lib, treewide: introduce repoRevToName and use it to cleanup most fetch* functions (#316668)

lib/default.nix

@@ -433,6 +433,8 @@ let
         pathHasContext
         canCleanSource
         pathIsGitRepo
+        revOrTag
+        repoRevToName
         ;
       inherit (self.modules)
         evalModules

lib/sources.nix

@@ -3,7 +3,7 @@
 
 # Tested in lib/tests/sources.sh
 let
-  inherit (builtins)
+  inherit (lib.strings)
     match
     split
     storeDir
@@ -403,6 +403,101 @@ let
       };
     };
 
+  # urlToName : (URL | Path | String) -> String
+  #
+  # Transform a URL (or path, or string) into a clean package name.
+  urlToName =
+    url:
+    let
+      inherit (lib.strings) stringLength;
+      base = baseNameOf (lib.removeSuffix "/" (lib.last (lib.splitString ":" (toString url))));
+      # chop away one git or archive-related extension
+      removeExt =
+        name:
+        let
+          matchExt = match "(.*)\\.(git|tar|zip|gz|tgz|bz|tbz|bz2|tbz2|lzma|txz|xz|zstd)$" name;
+        in
+        if matchExt != null then lib.head matchExt else name;
+      # apply function f to string x while the result shrinks
+      shrink =
+        f: x:
+        let
+          v = f x;
+        in
+        if stringLength v < stringLength x then shrink f v else x;
+    in
+    shrink removeExt base;
+
+  # shortRev : (String | Integer) -> String
+  #
+  # Given a package revision (like "refs/tags/v12.0"), produce a short revision ("12.0").
+  shortRev =
+    rev:
+    let
+      baseRev = baseNameOf (toString rev);
+      matchHash = match "[a-f0-9]+" baseRev;
+      matchVer = match "([A-Za-z]+[-_. ]?)*(v)?([0-9.]+.*)" baseRev;
+    in
+    if matchHash != null then
+      builtins.substring 0 7 baseRev
+    else if matchVer != null then
+      lib.last matchVer
+    else
+      baseRev;
+
+  # revOrTag : String -> String -> String
+  #
+  # Turn git `rev` and `tag` pair into a revision usable in `repoRevToName*`.
+  revOrTag =
+    rev: tag:
+    if tag != null then
+      tag
+    else if rev != null then
+      rev
+    else
+      "HEAD";
+
+  # repoRevToNameFull : (URL | Path | String) -> (String | Integer | null) -> (String | null) -> String
+  #
+  # See `repoRevToName` below.
+  repoRevToNameFull =
+    repo_: rev_: suffix_:
+    let
+      repo = urlToName repo_;
+      rev = if rev_ != null then "-${shortRev rev_}" else "";
+      suffix = if suffix_ != null then "-${suffix_}" else "";
+    in
+    "${repo}${rev}${suffix}-source";
+
+  # repoRevToName : String -> (URL | Path | String) -> (String | Integer | null) -> String -> String
+  #
+  # Produce derivation.name attribute for a given repository URL/path/name and (optionally) its revision/version tag.
+  #
+  # This is used by fetch(zip|git|FromGitHub|hg|svn|etc) to generate discoverable
+  # /nix/store paths.
+  #
+  # This uses a different implementation depending on the `pretty` argument:
+  #  "source" -> name everything as "source"
+  #  "versioned" -> name everything as "${repo}-${rev}-source"
+  #  "full" -> name everything as "${repo}-${rev}-${fetcher}-source"
+  repoRevToName =
+    kind:
+    # match on `kind` first to minimize the thunk
+    if kind == "source" then
+      (
+        repo: rev: suffix:
+        "source"
+      )
+    else if kind == "versioned" then
+      (
+        repo: rev: suffix:
+        repoRevToNameFull repo rev null
+      )
+    else if kind == "full" then
+      repoRevToNameFull
+    else
+      throw "repoRevToName: invalid kind";
+
 in
 {
 
@@ -431,6 +526,11 @@ in
     pathHasContext
     canCleanSource
 
+    urlToName
+    shortRev
+    revOrTag
+    repoRevToName
+
     sourceByRegex
     sourceFilesBySuffices
 

pkgs/build-support/fetchbitbucket/default.nix

@@ -1,11 +1,15 @@
-{ fetchzip, lib }:
+{
+  lib,
+  repoRevToNameMaybe,
+  fetchzip,
+}:
 
 lib.makeOverridable (
   {
     owner,
     repo,
     rev,
-    name ? "source",
+    name ? repoRevToNameMaybe repo rev "bitbucket",
     ... # For hash agility
   }@args:
   fetchzip (

pkgs/build-support/fetchgit/default.nix

@@ -5,29 +5,26 @@
   git-lfs,
   cacert,
 }:
+
 let
   urlToName =
     url: rev:
     let
-      inherit (lib) removeSuffix splitString last;
+      shortRev = lib.sources.shortRev rev;
-      base = last (splitString ":" (baseNameOf (removeSuffix "/" url)));
+      appendShort = lib.optionalString ((builtins.match "[a-f0-9]*" rev) != null) "-${shortRev}";
-
-      matched = builtins.match "(.*)\\.git" base;
-
-      short = builtins.substring 0 7 rev;
-
-      appendShort = lib.optionalString ((builtins.match "[a-f0-9]*" rev) != null) "-${short}";
     in
-    "${if matched == null then base else builtins.head matched}${appendShort}";
+    "${lib.sources.urlToName url}${appendShort}";
 in
+
 lib.makeOverridable (
   lib.fetchers.withNormalizedHash { } (
     # NOTE Please document parameter additions or changes in
-    #   doc/build-helpers/fetchers.chapter.md
+    #   ../../../doc/build-helpers/fetchers.chapter.md
     {
       url,
       tag ? null,
       rev ? null,
+      name ? urlToName url (lib.revOrTag rev tag),
       leaveDotGit ? deepClone || fetchTags,
       outputHash ? lib.fakeHash,
       outputHashAlgo ? null,
@@ -36,7 +33,6 @@ lib.makeOverridable (
       branchName ? null,
       sparseCheckout ? [ ],
       nonConeMode ? false,
-      name ? null,
       nativeBuildInputs ? [ ],
       # Shell code executed before the file has been fetched.  This, in
       # particular, can do things like set NIX_PREFETCH_GIT_CHECKOUT_HOOK to
@@ -108,7 +104,7 @@ lib.makeOverridable (
         "Please provide directories/patterns for sparse checkout as a list of strings. Passing a (multi-line) string is not supported any more."
     else
       stdenvNoCC.mkDerivation {
-        name = if name != null then name else urlToName url revWithTag;
+        inherit name;
 
         builder = ./builder.sh;
         fetcher = ./nix-prefetch-git;

pkgs/build-support/fetchgithub/default.nix

@@ -1,5 +1,6 @@
 {
   lib,
+  repoRevToNameMaybe,
   fetchgit,
   fetchzip,
 }:
@@ -10,7 +11,7 @@ lib.makeOverridable (
     repo,
     tag ? null,
     rev ? null,
-    name ? "source",
+    name ? repoRevToNameMaybe repo (lib.revOrTag rev tag) "github",
     fetchSubmodules ? false,
     leaveDotGit ? null,
     deepClone ? false,

pkgs/build-support/fetchgitiles/default.nix

@@ -1,11 +1,15 @@
-{ fetchzip, lib }:
+{
+  fetchzip,
+  repoRevToNameMaybe,
+  lib,
+}:
 
 lib.makeOverridable (
   {
     url,
     rev ? null,
     tag ? null,
-    name ? "source",
+    name ? repoRevToNameMaybe url (lib.revOrTag rev tag) "gitiles",
     ...
   }@args:
 

pkgs/build-support/fetchgitlab/default.nix

@@ -1,5 +1,6 @@
 {
   lib,
+  repoRevToNameMaybe,
   fetchgit,
   fetchzip,
 }:
@@ -11,9 +12,9 @@ lib.makeOverridable (
     repo,
     rev ? null,
     tag ? null,
+    name ? repoRevToNameMaybe repo (lib.revOrTag rev tag) "gitlab",
     protocol ? "https",
     domain ? "gitlab.com",
-    name ? "source",
     group ? null,
     fetchSubmodules ? false,
     leaveDotGit ? false,

pkgs/build-support/fetchrepoorcz/default.nix

@@ -1,10 +1,14 @@
-{ fetchzip }:
+{
+  lib,
+  repoRevToNameMaybe,
+  fetchzip,
+}:
 
 # gitweb example, snapshot support is optional in gitweb
 {
   repo,
   rev,
-  name ? "source",
+  name ? repoRevToNameMaybe repo rev "repoorcz",
   ... # For hash agility
 }@args:
 fetchzip (

pkgs/build-support/fetchsavannah/default.nix

@@ -1,11 +1,15 @@
-{ fetchzip, lib }:
+{
+  lib,
+  repoRevToNameMaybe,
+  fetchzip,
+}:
 
 lib.makeOverridable (
   # cgit example, snapshot support is optional in cgit
   {
     repo,
     rev,
-    name ? "source",
+    name ? repoRevToNameMaybe repo rev "savannah",
     ... # For hash agility
   }@args:
   fetchzip (

pkgs/build-support/fetchsourcehut/default.nix

@@ -1,8 +1,9 @@
 {
+  lib,
+  repoRevToNameMaybe,
   fetchgit,
   fetchhg,
   fetchzip,
-  lib,
 }:
 
 let
@@ -18,9 +19,9 @@ makeOverridable (
     owner,
     repo,
     rev,
+    name ? repoRevToNameMaybe repo rev "sourcehut",
     domain ? "sr.ht",
     vc ? "git",
-    name ? "source",
     fetchSubmodules ? false,
     ... # For hash agility
   }@args:

pkgs/build-support/fetchsvn/default.nix

@@ -9,51 +9,55 @@
   openssh ? null,
 }:
 
+let
+  repoToName =
+    url: rev:
+    let
+      inherit (lib)
+        removeSuffix
+        splitString
+        reverseList
+        head
+        last
+        elemAt
+        ;
+      base = removeSuffix "/" (last (splitString ":" url));
+      path = reverseList (splitString "/" base);
+      repoName =
+        # ../repo/trunk -> repo
+        if head path == "trunk" then
+          elemAt path 1
+        # ../repo/branches/branch -> repo-branch
+        else if elemAt path 1 == "branches" then
+          "${elemAt path 2}-${head path}"
+        # ../repo/tags/tag -> repo-tag
+        else if elemAt path 1 == "tags" then
+          "${elemAt path 2}-${head path}"
+        # ../repo (no trunk) -> repo
+        else
+          head path;
+    in
+    "${repoName}-r${toString rev}";
+in
+
 {
   url,
   rev ? "HEAD",
+  name ? repoToName url rev,
   sha256 ? "",
   hash ? "",
   ignoreExternals ? false,
   ignoreKeywords ? false,
-  name ? null,
   preferLocalBuild ? true,
 }:
 
 assert sshSupport -> openssh != null;
 
-let
-  repoName =
-    let
-      fst = lib.head;
-      snd = l: lib.head (lib.tail l);
-      trd = l: lib.head (lib.tail (lib.tail l));
-      path_ =
-        (p: if lib.head p == "" then lib.tail p else p) # ~ drop final slash if any
-          (lib.reverseList (lib.splitString "/" url));
-      path = [ (lib.removeSuffix "/" (lib.head path_)) ] ++ (lib.tail path_);
-    in
-    # ../repo/trunk -> repo
-    if fst path == "trunk" then
-      snd path
-    # ../repo/branches/branch -> repo-branch
-    else if snd path == "branches" then
-      "${trd path}-${fst path}"
-    # ../repo/tags/tag -> repo-tag
-    else if snd path == "tags" then
-      "${trd path}-${fst path}"
-    # ../repo (no trunk) -> repo
-    else
-      fst path;
-
-  name_ = if name == null then "${repoName}-r${toString rev}" else name;
-in
-
 if hash != "" && sha256 != "" then
   throw "Only one of sha256 or hash can be set"
 else
   stdenvNoCC.mkDerivation {
-    name = name_;
+    inherit name;
     builder = ./builder.sh;
     nativeBuildInputs = [
       cacert

pkgs/build-support/fetchzip/default.nix

@@ -7,6 +7,7 @@
 
 {
   lib,
+  repoRevToNameMaybe,
   fetchurl,
   withUnzip ? true,
   unzip,
@@ -14,9 +15,9 @@
 }:
 
 {
-  name ? "source",
   url ? "",
   urls ? [ ],
+  name ? repoRevToNameMaybe (if url != "" then url else builtins.head urls) null "unpacked",
   nativeBuildInputs ? [ ],
   postFetch ? "",
   extraPostFetch ? "",

pkgs/top-level/all-packages.nix

@@ -576,6 +576,9 @@ with pkgs;
     stdenv = if stdenv.hostPlatform.isDarwin then llvmPackages_18.stdenv else stdenv;
   };
 
+  # this is used by most `fetch*` functions
+  repoRevToNameMaybe = lib.repoRevToName config.fetchedSourceNameDefault;
+
   fetchpatch =
     callPackage ../build-support/fetchpatch {
       # 0.3.4 would change hashes: https://github.com/NixOS/nixpkgs/issues/25154

pkgs/top-level/config.nix

@@ -59,6 +59,52 @@ let
       default = false;
     };
 
+    fetchedSourceNameDefault = mkOption {
+      type = types.uniq (
+        types.enum [
+          "source"
+          "versioned"
+          "full"
+        ]
+      );
+      default = "source";
+      description = ''
+        This controls the default derivation `name` attribute set by the
+        `fetch*` (`fetchzip`, `fetchFromGitHub`, etc) functions.
+
+        Possible values and the resulting `.name`:
+
+        - `"source"` -> `"source"`
+        - `"versioned"` -> `"''${repo}-''${rev}-source"`
+        - `"full"` -> `"''${repo}-''${rev}-''${fetcherName}-source"`
+
+        The default `"source"` is the best choice for minimal rebuilds, it
+        will ignore any non-hash changes (like branches being renamed, source
+        URLs changing, etc) at the cost of `/nix/store` being easily
+        cache-poisoned (see [NixOS/nix#969](https://github.com/NixOS/nix/issues/969)).
+
+        Setting this to `"versioned"` greatly helps with discoverability of
+        sources in `/nix/store` and makes cache-poisoning of `/nix/store` much
+        harder, at the cost of a single mass-rebuild for all `src`
+        derivations, and an occasional rebuild when a source changes some of
+        its non-hash attributes.
+
+        Setting this to `"full"` is similar to setting it to `"versioned"`,
+        but the use of `fetcherName` in the derivation name will force a
+        rebuild when `src` switches between `fetch*` functions, thus forcing
+        `nix` to check new derivation's `outputHash`, which is useful for
+        debugging.
+
+        Also, `"full"` is useful for easy collection and tracking of
+        statistics of where the packages you use are hosted.
+
+        If you are a developer, you should probably set this to at
+        least`"versioned"`.
+
+        Changing the default will cause a mass rebuild.
+      '';
+    };
+
     doCheckByDefault = mkMassRebuild {
       feature = "run `checkPhase` by default";
     };
@@ -150,8 +196,6 @@ let
     };
 
     cudaSupport = mkMassRebuild {
-      type = types.bool;
-      default = false;
       feature = "build packages with CUDA support by default";
     };
 
@@ -184,8 +228,6 @@ let
     };
 
     rocmSupport = mkMassRebuild {
-      type = types.bool;
-      default = false;
       feature = "build packages with ROCm support by default";
     };