diff --git a/pkgs/build-support/fetchgit/default.nix b/pkgs/build-support/fetchgit/default.nix index 72a8c449262b..a2d22847df2f 100644 --- a/pkgs/build-support/fetchgit/default.nix +++ b/pkgs/build-support/fetchgit/default.nix @@ -83,6 +83,8 @@ lib.makeOverridable ( # run operations between the checkout completing and deleting the .git # directory. preFetch ? "", + # Shell code executed after `git checkout` and before .git directory removal/sanitization. + postCheckout ? "", # Shell code executed after the file has been fetched # successfully. This can do things like check or transform the file. postFetch ? "", @@ -191,6 +193,7 @@ lib.makeOverridable ( deepClone branchName preFetch + postCheckout postFetch fetchTags rootDir @@ -251,6 +254,10 @@ lib.makeOverridable ( inherit preferLocalBuild meta; + env = { + NIX_PREFETCH_GIT_CHECKOUT_HOOK = finalAttrs.postCheckout; + }; + passthru = { gitRepoUrl = url; } diff --git a/pkgs/build-support/fetchgit/nix-prefetch-git b/pkgs/build-support/fetchgit/nix-prefetch-git index f9c8af8428fe..0cd830f83a82 100755 --- a/pkgs/build-support/fetchgit/nix-prefetch-git +++ b/pkgs/build-support/fetchgit/nix-prefetch-git @@ -12,6 +12,7 @@ fetchSubmodules= fetchLFS= builder= fetchTags= +fetchTagsCompat= branchName=$NIX_PREFETCH_GIT_BRANCH_NAME # ENV params @@ -116,11 +117,17 @@ for arg; do fi done +# `deepClone` used to effectively imply `fetchTags`. +# We avoid such behaviour to enhance the `postCheckout` reproducibility, +# while keeping the old behaviour for `.git` for backward compatibility purposes. +if [[ -n "$deepClone" ]] && [[ -z "$leaveDotGit" ]]; then + fetchTagsCompat=true +fi + if test -z "$url"; then usage fi - init_remote(){ local url=$1 clean_git init --initial-branch=master @@ -181,9 +188,30 @@ checkout_hash(){ hash=$(hash_from_ref "$ref") fi - [[ -z "$deepClone" ]] && \ - clean_git fetch ${builder:+--progress} --depth=1 origin "$hash" || \ - clean_git fetch -t ${builder:+--progress} origin || return 1 + local -a fetchTagsArgs + if [[ -n "$fetchTags" ]]; then + fetchTagsArgs=(--tags) + else + fetchTagsArgs=(--no-tags) + fi + local -a fetchTargetArgs + if [[ -n "$deepClone" ]]; then + fetchTargetArgs=(origin) + else + fetchTargetArgs=(--depth=1 origin "$hash") + fi + if ! clean_git fetch "${fetchTagsArgs[@]}" ${builder:+--progress} "${fetchTargetArgs[@]}"; then + echo "ERROR: \`git fetch' failed." >&2 + # Git remotes using the "dumb" protocol does not support shallow fetch; + # fall back to deep fetch if shallow fetch failed. + # TODO(@ShamrockLee): Determine whether the transfer protocol is smart reliably. + if [[ -z "$deepClone" ]] || [[ -z "$fetchTags" ]]; then + echo "This might be due to the dumb transfer protocol not supporting shallow fetch or no-tag cloning. Trying with \`--tags' and without \`--depth=1'..." >&2 + clean_git fetch --tags ${builder:+--progress} origin || return 1 + else + return 1 + fi + fi local object_type=$(git cat-file -t "$hash") if [[ "$object_type" == "commit" || "$object_type" == "tag" ]]; then @@ -253,11 +281,19 @@ clone(){ ) # Fetch all tags if requested - if test -n "$fetchTags"; then + # The fetched tags are potentially non-reproducible, as tags are mutable parts of the Git tree. + if [[ -n "$fetchTags" ]] || [[ -n "$fetchTagsCompat" ]]; then echo "fetching all tags..." >&2 clean_git fetch origin 'refs/tags/*:refs/tags/*' || echo "warning: failed to fetch some tags" >&2 fi + # Name "$ref" to make `git describe` work reproducibly in `NIX_PREFETCH_GIT_CHECKOUT_HOOK`. + # Name only when not leaving `.git` for compatibility purposes. + if [[ -n "$ref" ]] && [[ -z "$leaveDotGit" ]]; then + echo "refer to FETCH_HEAD as its original name $ref" + clean_git update-ref "$ref" FETCH_HEAD + fi + # Checkout linked sources. if test -n "$fetchSubmodules"; then init_submodules diff --git a/pkgs/build-support/fetchgit/tests.nix b/pkgs/build-support/fetchgit/tests.nix index 9ccb3ff3058b..3fab7442ef25 100644 --- a/pkgs/build-support/fetchgit/tests.nix +++ b/pkgs/build-support/fetchgit/tests.nix @@ -17,6 +17,33 @@ sha256 = "sha256-7DszvbCNTjpzGRmpIVAWXk20P0/XTrWZ79KSOGLrUWY="; }; + collect-rev = testers.invalidateFetcherByDrvHash fetchgit { + name = "collect-rev-nix-source"; + url = "https://github.com/NixOS/nix"; + rev = "9d9dbe6ed05854e03811c361a3380e09183f4f4a"; + hash = "sha256-AUTX1K7J5+fojvKYJacXYVV5kio3hrWYz5MCekO6h68="; + postCheckout = '' + git -C "$out" rev-parse HEAD | tee "$out/revision.txt" + ''; + }; + + simple-tag = testers.invalidateFetcherByDrvHash fetchgit { + name = "simple-tag-nix-source"; + url = "https://github.com/NixOS/nix"; + tag = "2.3.15"; + hash = "sha256-7DszvbCNTjpzGRmpIVAWXk20P0/XTrWZ79KSOGLrUWY="; + }; + + describe-tag = testers.invalidateFetcherByDrvHash fetchgit { + name = "describe-tag-nix-source"; + url = "https://github.com/NixOS/nix"; + tag = "2.3.15"; + hash = "sha256-y7l+46lVP2pzJwGON5qEV0EoxWofRoWAym5q9VXvpc8="; + postCheckout = '' + { git -C "$out" describe || echo "git describe failed"; } | tee "$out"/describe-output.txt + ''; + }; + sparseCheckout = testers.invalidateFetcherByDrvHash fetchgit { name = "sparse-checkout-nix-source"; url = "https://github.com/NixOS/nix"; @@ -77,6 +104,20 @@ postFetch = "rm -r $out/.git"; }; + submodule-revision-count = testers.invalidateFetcherByDrvHash fetchgit { + name = "submodule-revision-count-source"; + url = "https://github.com/pineapplehunter/nix-test-repo-with-submodule"; + rev = "26473335b84ead88ee0a3b649b1c7fa4a91cfd4a"; + hash = "sha256-ok1e6Pb0fII5TF8HXF8DXaRGSoq7kgRCoXqSEauh1wk="; + fetchSubmodules = true; + deepClone = true; + leaveDotGit = false; + postCheckout = '' + { git -C "$out" rev-list --count HEAD || echo "git rev-list failed"; } | tee "$out/revision_count.txt" + { git -C "$out/nix-test-repo-submodule" rev-list --count HEAD || echo "git rev-list failed"; } | tee "$out/nix-test-repo-submodule/revision_count.txt" + ''; + }; + submodule-leave-git-deep = testers.invalidateFetcherByDrvHash fetchgit { name = "submodule-leave-git-deep-source"; url = "https://github.com/pineapplehunter/nix-test-repo-with-submodule";