From 3985afc65c20a020ae24820d9ccb23eb2fa40c85 Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Fri, 18 Jul 2025 16:45:45 +0200 Subject: [PATCH 1/2] nix-prefetch-git: Add --root-dir argument With this argument nix-prefetch-git will make a subdirectory of the Git repository a root of the resulting store path. This is helpful for dealing with monorepos where many projects are in separate directories and don't need a new source hash every time the monorepo is updated. The corresponding fetchgit change follows. --- pkgs/build-support/fetchgit/nix-prefetch-git | 62 ++++++++++++++------ 1 file changed, 45 insertions(+), 17 deletions(-) diff --git a/pkgs/build-support/fetchgit/nix-prefetch-git b/pkgs/build-support/fetchgit/nix-prefetch-git index f8dd93912644..e8bcca7f6b42 100755 --- a/pkgs/build-support/fetchgit/nix-prefetch-git +++ b/pkgs/build-support/fetchgit/nix-prefetch-git @@ -59,6 +59,7 @@ Options: --fetch-submodules Fetch submodules. --fetch-tags Fetch all tags (useful for git describe). --builder Clone as fetchgit does, but url, rev, and out option are mandatory. + --root-dir dir Directory in the repository that will be copied to the output instead of the full repository. --quiet Only print the final json summary. " exit 1 @@ -90,6 +91,7 @@ for arg; do --fetch-submodules) fetchSubmodules=true;; --fetch-tags) fetchTags=true;; --builder) builder=true;; + --root-dir) argfun=set_rootDir;; -h|--help) usage; exit;; *) : $((++argi)) @@ -150,10 +152,16 @@ url_to_name(){ local base base=$(basename "$url" .git | cut -d: -f2) - if [[ $ref =~ ^[a-z0-9]+$ ]]; then - echo "$base-${ref:0:7}" + if test -n "$rootDir"; then + # Sanitize by removing leading dots and replacing all invalid character sequences with dashes. + # See sanitizeDerivationName in ../../../lib/strings.nix for reference. + echo "$base-$(sed -E 's/^\.+//;s/[^[:alnum:]+._?=-]+/-/g' <<< $rootDir)" else - echo "$base" + if [[ $ref =~ ^[a-z0-9]+$ ]]; then + echo "$base-${ref:0:7}" + else + echo "$base" + fi fi } @@ -348,6 +356,23 @@ clone_user_rev() { fi } +clone_user_rev_to_tmpfile(){ + local url="$1" + local rev="${2:-HEAD}" + + # nix>=2.20 rejects adding symlinked paths to the store, so use realpath + # to resolve to a physical path. https://github.com/NixOS/nix/issues/11941 + tmpPath="$(realpath "$(mktemp -d --tmpdir git-checkout-tmp-XXXXXXXX)")" + exit_handlers+=(remove_tmpPath) + + tmpOut="$tmpPath/out/$storePathName" + tmpClone="$tmpPath/clone" + mkdir -p "$tmpPath/out" "$tmpClone" + + # Perform the checkout. + clone_user_rev "$tmpClone" "$url" "$rev" +} + exit_handlers=() run_exit_handlers() { @@ -413,7 +438,8 @@ print_results() { "fetchSubmodules": $([[ -n "$fetchSubmodules" ]] && echo true || echo false), "deepClone": $([[ -n "$deepClone" ]] && echo true || echo false), "fetchTags": $([[ -n "$fetchTags" ]] && echo true || echo false), - "leaveDotGit": $([[ -n "$leaveDotGit" ]] && echo true || echo false) + "leaveDotGit": $([[ -n "$leaveDotGit" ]] && echo true || echo false), + "rootDir": "$(json_escape "$rootDir")" } EOF fi @@ -451,8 +477,13 @@ export GIT_CONFIG_NOSYSTEM=1 if test -n "$builder"; then test -n "$out" -a -n "$url" -a -n "$rev" || usage - mkdir -p "$out" - clone_user_rev "$out" "$url" "$rev" + if test -n "$rootDir"; then + clone_user_rev_to_tmpfile "$url" "$rev" + mv "$tmpClone/$rootDir" "$out" + else + mkdir -p "$out" + clone_user_rev "$out" "$url" "$rev" + fi else if test -z "$hashType"; then hashType=sha256 @@ -471,22 +502,19 @@ else # If we don't know the hash or a path with that hash doesn't exist, # download the file and add it to the store. if test -z "$finalPath"; then - # nix>=2.20 rejects adding symlinked paths to the store, so use realpath - # to resolve to a physical path. https://github.com/NixOS/nix/issues/11941 - tmpPath="$(realpath "$(mktemp -d --tmpdir git-checkout-tmp-XXXXXXXX)")" - exit_handlers+=(remove_tmpPath) + clone_user_rev_to_tmpfile "$url" "$rev" - tmpFile="$tmpPath/$storePathName" - mkdir -p "$tmpFile" - - # Perform the checkout. - clone_user_rev "$tmpFile" "$url" "$rev" + if test -z "$rootDir"; then + mv "$tmpClone" "$tmpOut" + else + mv "$tmpClone/$rootDir" "$tmpOut" + fi # Compute the hash. - hash=$(nix-hash --type $hashType --base32 "$tmpFile") + hash=$(nix-hash --type $hashType --base32 "$tmpOut") # Add the downloaded file to the Nix store. - finalPath=$(nix-store --add-fixed --recursive "$hashType" "$tmpFile") + finalPath=$(nix-store --add-fixed --recursive "$hashType" "$tmpOut") if test -n "$expHash" -a "$expHash" != "$hash"; then echo "hash mismatch for URL \`$url'. Got \`$hash'; expected \`$expHash'." >&2 From 34612851db874ad188732c24429b3d1057f461fd Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Fri, 18 Jul 2025 16:45:45 +0200 Subject: [PATCH 2/2] fetchgit: Add rootDir argument With this argument fetchgit will make a subdirectory of the Git repository a root of the resulting store path. This is helpful for dealing with monorepos where many projects are in separate directories and don't need a new source hash every time the monorepo is updated. Commit hash is removed from the name of the derivation to prevent it from changing the store path when nothing in the subdirectory changes. --- doc/build-helpers/fetchers.chapter.md | 4 ++++ doc/release-notes/rl-2511.section.md | 1 + pkgs/build-support/fetchgit/builder.sh | 3 ++- pkgs/build-support/fetchgit/default.nix | 23 ++++++++++++++++++----- pkgs/build-support/fetchgit/tests.nix | 8 ++++++++ 5 files changed, 33 insertions(+), 6 deletions(-) diff --git a/doc/build-helpers/fetchers.chapter.md b/doc/build-helpers/fetchers.chapter.md index 8ff1fdf11a19..bd606ea924f1 100644 --- a/doc/build-helpers/fetchers.chapter.md +++ b/doc/build-helpers/fetchers.chapter.md @@ -827,6 +827,10 @@ Additionally, the following optional arguments can be given: See [git sparse-checkout](https://git-scm.com/docs/git-sparse-checkout) for more information. +*`rootDir`* (String) + +: When not empty, copy only contents of the subdirectory of the repository to the result. Automatically sets `sparseCheckout` and `nonConeMode` to avoid checking out any extra pieces. Incompatible with `leaveDotGit`. + Some additional parameters for niche use-cases can be found listed in the function parameters in the declaration of `fetchgit`: `pkgs/build-support/fetchgit/default.nix`. Future parameters additions might also happen without immediately being documented here. diff --git a/doc/release-notes/rl-2511.section.md b/doc/release-notes/rl-2511.section.md index 2e54b34da408..023650e2664c 100644 --- a/doc/release-notes/rl-2511.section.md +++ b/doc/release-notes/rl-2511.section.md @@ -75,6 +75,7 @@ * `$debug/lib/debug/.build-id/48/3bd7f7229bdb06462222e1e353e4f37e15c293.sourceoverlay` is a symlink to a directory with the same structure as the expanded `$sourceRoot` but containing only a copy of files which were patched during the build * `$debug/lib/debug/.build-id/48/3bd7f7229bdb06462222e1e353e4f37e15c293.debug` is the file containing debug symbols (like before). +- `fetchgit`: Add `rootDir` argument to limit the resulting source to one subdirectory of the whole Git repository. Corresponding `--root-dir` option added to `nix-prefetch-git`. ## Nixpkgs Library {#sec-nixpkgs-release-25.11-lib} diff --git a/pkgs/build-support/fetchgit/builder.sh b/pkgs/build-support/fetchgit/builder.sh index 819b0127a852..393d0a57d163 100644 --- a/pkgs/build-support/fetchgit/builder.sh +++ b/pkgs/build-support/fetchgit/builder.sh @@ -16,6 +16,7 @@ $SHELL $fetcher --builder --url "$url" --out "$out" --rev "$rev" --name "$name" ${fetchTags:+--fetch-tags} \ ${sparseCheckout:+--sparse-checkout "$sparseCheckout"} \ ${nonConeMode:+--non-cone-mode} \ - ${branchName:+--branch-name "$branchName"} + ${branchName:+--branch-name "$branchName"} \ + ${rootDir:+--root-dir "$rootDir"} runHook postFetch diff --git a/pkgs/build-support/fetchgit/default.nix b/pkgs/build-support/fetchgit/default.nix index 374d55b0456c..f6099fddd984 100644 --- a/pkgs/build-support/fetchgit/default.nix +++ b/pkgs/build-support/fetchgit/default.nix @@ -8,12 +8,16 @@ let urlToName = - url: rev: + { + url, + rev, + append, + }: let shortRev = lib.sources.shortRev rev; appendShort = lib.optionalString ((builtins.match "[a-f0-9]*" rev) != null) "-${shortRev}"; in - "${lib.sources.urlToName url}${appendShort}"; + "${lib.sources.urlToName url}${if append == "" then appendShort else append}"; in lib.makeOverridable ( @@ -24,15 +28,20 @@ lib.makeOverridable ( url, tag ? null, rev ? null, - name ? urlToName url (lib.revOrTag rev tag), + name ? urlToName { + inherit url; + rev = lib.revOrTag rev tag; + # when rootDir is specified, avoid invalidating the result when rev changes + append = if rootDir != "" then "-${lib.strings.sanitizeDerivationName rootDir}" else ""; + }, leaveDotGit ? deepClone || fetchTags, outputHash ? lib.fakeHash, outputHashAlgo ? null, fetchSubmodules ? true, deepClone ? false, branchName ? null, - sparseCheckout ? [ ], - nonConeMode ? false, + sparseCheckout ? lib.optional (rootDir != "") rootDir, + nonConeMode ? rootDir != "", nativeBuildInputs ? [ ], # Shell code executed before the file has been fetched. This, in # particular, can do things like set NIX_PREFETCH_GIT_CHECKOUT_HOOK to @@ -53,6 +62,8 @@ lib.makeOverridable ( allowedRequisites ? null, # fetch all tags after tree (useful for git describe) fetchTags ? false, + # make this subdirectory the root of the result + rootDir ? "", }: /* @@ -80,6 +91,7 @@ lib.makeOverridable ( assert nonConeMode -> (sparseCheckout != [ ]); assert fetchTags -> leaveDotGit; + assert rootDir != "" -> !leaveDotGit; let revWithTag = @@ -136,6 +148,7 @@ lib.makeOverridable ( preFetch postFetch fetchTags + rootDir ; rev = revWithTag; diff --git a/pkgs/build-support/fetchgit/tests.nix b/pkgs/build-support/fetchgit/tests.nix index fd6aebc335f7..e3fdd5fb8672 100644 --- a/pkgs/build-support/fetchgit/tests.nix +++ b/pkgs/build-support/fetchgit/tests.nix @@ -97,4 +97,12 @@ rm -rf .git ''; }; + + rootDir = testers.invalidateFetcherByDrvHash fetchgit { + name = "fetchgit-with-rootdir"; + url = "https://github.com/NixOS/nix"; + rev = "9d9dbe6ed05854e03811c361a3380e09183f4f4a"; + rootDir = "misc/systemd"; + sha256 = "sha256-UhxHk4SrXYq7ZDMtXLig5SigpbITrVgkpFTmryuvpcM="; + }; }