fetchgit: take postCheckout to allow gathering revision and commit metadata without leaving .git (#465497)

pkgs/build-support/fetchgit/default.nix

@@ -83,6 +83,8 @@ lib.makeOverridable (
           # run operations between the checkout completing and deleting the .git
           # directory.
           preFetch ? "",
+          # Shell code executed after `git checkout` and before .git directory removal/sanitization.
+          postCheckout ? "",
           # Shell code executed after the file has been fetched
           # successfully. This can do things like check or transform the file.
           postFetch ? "",
@@ -191,6 +193,7 @@ lib.makeOverridable (
             deepClone
             branchName
             preFetch
+            postCheckout
             postFetch
             fetchTags
             rootDir
@@ -251,6 +254,10 @@ lib.makeOverridable (
 
           inherit preferLocalBuild meta;
 
+          env = {
+            NIX_PREFETCH_GIT_CHECKOUT_HOOK = finalAttrs.postCheckout;
+          };
+
           passthru = {
             gitRepoUrl = url;
           }

pkgs/build-support/fetchgit/nix-prefetch-git

@@ -12,6 +12,7 @@ fetchSubmodules=
 fetchLFS=
 builder=
 fetchTags=
+fetchTagsCompat=
 branchName=$NIX_PREFETCH_GIT_BRANCH_NAME
 
 # ENV params
@@ -116,11 +117,17 @@ for arg; do
     fi
 done
 
+# `deepClone` used to effectively imply `fetchTags`.
+# We avoid such behaviour to enhance the `postCheckout` reproducibility,
+# while keeping the old behaviour for `.git` for backward compatibility purposes.
+if [[ -n "$deepClone" ]] && [[ -z "$leaveDotGit" ]]; then
+    fetchTagsCompat=true
+fi
+
 if test -z "$url"; then
     usage
 fi
 
-
 init_remote(){
     local url=$1
     clean_git init --initial-branch=master
@@ -181,9 +188,30 @@ checkout_hash(){
         hash=$(hash_from_ref "$ref")
     fi
 
-    [[ -z "$deepClone" ]] && \
-    clean_git fetch ${builder:+--progress} --depth=1 origin "$hash" || \
-    clean_git fetch -t ${builder:+--progress} origin || return 1
+    local -a fetchTagsArgs
+    if [[ -n "$fetchTags" ]]; then
+        fetchTagsArgs=(--tags)
+    else
+        fetchTagsArgs=(--no-tags)
+    fi
+    local -a fetchTargetArgs
+    if [[ -n "$deepClone" ]]; then
+        fetchTargetArgs=(origin)
+    else
+        fetchTargetArgs=(--depth=1 origin "$hash")
+    fi
+    if ! clean_git fetch "${fetchTagsArgs[@]}" ${builder:+--progress} "${fetchTargetArgs[@]}"; then
+        echo "ERROR: \`git fetch' failed." >&2
+        # Git remotes using the "dumb" protocol does not support shallow fetch;
+        # fall back to deep fetch if shallow fetch failed.
+        # TODO(@ShamrockLee): Determine whether the transfer protocol is smart reliably.
+        if [[ -z "$deepClone" ]] || [[ -z "$fetchTags" ]]; then
+            echo "This might be due to the dumb transfer protocol not supporting shallow fetch or no-tag cloning. Trying with \`--tags' and without \`--depth=1'..." >&2
+            clean_git fetch --tags ${builder:+--progress} origin || return 1
+        else
+            return 1
+        fi
+    fi
 
     local object_type=$(git cat-file -t "$hash")
     if [[ "$object_type" == "commit" || "$object_type" == "tag" ]]; then
@@ -253,11 +281,19 @@ clone(){
     )
 
     # Fetch all tags if requested
-    if test -n "$fetchTags"; then
+    # The fetched tags are potentially non-reproducible, as tags are mutable parts of the Git tree.
+    if [[ -n "$fetchTags" ]] || [[ -n "$fetchTagsCompat" ]]; then
         echo "fetching all tags..." >&2
         clean_git fetch origin 'refs/tags/*:refs/tags/*' || echo "warning: failed to fetch some tags" >&2
     fi
 
+    # Name "$ref" to make `git describe` work reproducibly in `NIX_PREFETCH_GIT_CHECKOUT_HOOK`.
+    # Name only when not leaving `.git` for compatibility purposes.
+    if [[ -n "$ref" ]] && [[ -z "$leaveDotGit" ]]; then
+        echo "refer to FETCH_HEAD as its original name $ref"
+        clean_git update-ref "$ref" FETCH_HEAD
+    fi
+
     # Checkout linked sources.
     if test -n "$fetchSubmodules"; then
         init_submodules

pkgs/build-support/fetchgit/tests.nix

@@ -17,6 +17,33 @@
     sha256 = "sha256-7DszvbCNTjpzGRmpIVAWXk20P0/XTrWZ79KSOGLrUWY=";
   };
 
+  collect-rev = testers.invalidateFetcherByDrvHash fetchgit {
+    name = "collect-rev-nix-source";
+    url = "https://github.com/NixOS/nix";
+    rev = "9d9dbe6ed05854e03811c361a3380e09183f4f4a";
+    hash = "sha256-AUTX1K7J5+fojvKYJacXYVV5kio3hrWYz5MCekO6h68=";
+    postCheckout = ''
+      git -C "$out" rev-parse HEAD | tee "$out/revision.txt"
+    '';
+  };
+
+  simple-tag = testers.invalidateFetcherByDrvHash fetchgit {
+    name = "simple-tag-nix-source";
+    url = "https://github.com/NixOS/nix";
+    tag = "2.3.15";
+    hash = "sha256-7DszvbCNTjpzGRmpIVAWXk20P0/XTrWZ79KSOGLrUWY=";
+  };
+
+  describe-tag = testers.invalidateFetcherByDrvHash fetchgit {
+    name = "describe-tag-nix-source";
+    url = "https://github.com/NixOS/nix";
+    tag = "2.3.15";
+    hash = "sha256-y7l+46lVP2pzJwGON5qEV0EoxWofRoWAym5q9VXvpc8=";
+    postCheckout = ''
+      { git -C "$out" describe || echo "git describe failed"; } | tee "$out"/describe-output.txt
+    '';
+  };
+
   sparseCheckout = testers.invalidateFetcherByDrvHash fetchgit {
     name = "sparse-checkout-nix-source";
     url = "https://github.com/NixOS/nix";
@@ -77,6 +104,20 @@
     postFetch = "rm -r $out/.git";
   };
 
+  submodule-revision-count = testers.invalidateFetcherByDrvHash fetchgit {
+    name = "submodule-revision-count-source";
+    url = "https://github.com/pineapplehunter/nix-test-repo-with-submodule";
+    rev = "26473335b84ead88ee0a3b649b1c7fa4a91cfd4a";
+    hash = "sha256-ok1e6Pb0fII5TF8HXF8DXaRGSoq7kgRCoXqSEauh1wk=";
+    fetchSubmodules = true;
+    deepClone = true;
+    leaveDotGit = false;
+    postCheckout = ''
+      { git -C "$out" rev-list --count HEAD || echo "git rev-list failed"; } | tee "$out/revision_count.txt"
+      { git -C "$out/nix-test-repo-submodule" rev-list --count HEAD || echo "git rev-list failed"; } | tee "$out/nix-test-repo-submodule/revision_count.txt"
+    '';
+  };
+
   submodule-leave-git-deep = testers.invalidateFetcherByDrvHash fetchgit {
     name = "submodule-leave-git-deep-source";
     url = "https://github.com/pineapplehunter/nix-test-repo-with-submodule";