From 025424913f19dfbee0c791c68619142b94a0e6f3 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Wed, 20 Aug 2025 12:10:23 +0200 Subject: [PATCH 1/7] workflows: support cachix in forks This allows setting up a custom cachix cache in a contributor's fork. That's most valuable when working on CI itself, testing and interating in a fork. It's required when working on cachix related features directly. --- .github/workflows/build.yml | 7 ++++--- .github/workflows/check.yml | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ef3c71ca4fbe..60aed67cea03 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -61,9 +61,10 @@ jobs: - uses: cachix/cachix-action@0fc020193b5a1fa3ac4575aa3a7d3aa6a35435ad # v16 with: - # This cache is for the nixpkgs repo checks and should not be trusted or used elsewhere. - name: nixpkgs-ci - authToken: "${{ secrets.CACHIX_AUTH_TOKEN }}" + # The nixpkgs-ci cache should not be trusted or used outside of Nixpkgs and its forks' CI. + name: ${{ vars.CACHIX_NAME || 'nixpkgs-ci' }} + extraPullNames: nixpkgs-ci + authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} - run: nix-env --install -f pinned -A nix-build-uncached diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index c06729fa7f5d..d58358e23aa7 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -112,9 +112,10 @@ jobs: - uses: cachix/cachix-action@0fc020193b5a1fa3ac4575aa3a7d3aa6a35435ad # v16 with: - # This cache is for the nixpkgs repo checks and should not be trusted or used elsewhere. - name: nixpkgs-ci - authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + # The nixpkgs-ci cache should not be trusted or used outside of Nixpkgs and its forks' CI. + name: ${{ vars.CACHIX_NAME || 'nixpkgs-ci' }} + extraPullNames: nixpkgs-ci + authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} - name: Build codeowners validator run: nix-build trusted/ci --arg nixpkgs ./pinned -A codeownersValidator From 2400bdf0d4d4c408964525b4d5f12c33647aa708 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Wed, 20 Aug 2025 12:11:59 +0200 Subject: [PATCH 2/7] workflows: never push source to cachix Pushing the source path to cachix is pointless, because the only source we're using is Nixpkgs - and that will always be available already via checkout. No need to ever substitute it, so no need to push it either. --- .github/workflows/build.yml | 1 + .github/workflows/check.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 60aed67cea03..3b289413e6fb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -65,6 +65,7 @@ jobs: name: ${{ vars.CACHIX_NAME || 'nixpkgs-ci' }} extraPullNames: nixpkgs-ci authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} + pushFilter: -source$ - run: nix-env --install -f pinned -A nix-build-uncached diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index d58358e23aa7..3dbbb9d2dff7 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -116,6 +116,7 @@ jobs: name: ${{ vars.CACHIX_NAME || 'nixpkgs-ci' }} extraPullNames: nixpkgs-ci authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} + pushFilter: -source$ - name: Build codeowners validator run: nix-build trusted/ci --arg nixpkgs ./pinned -A codeownersValidator From 6ee40e79314d07ffc58537fbe33ad27373748353 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Wed, 20 Aug 2025 12:20:01 +0200 Subject: [PATCH 3/7] workflows/lint: add cachix For PRs targeting a fast moving branch like master, it might not actually help because the merge commit will still be different on each run. But on slower moving branches or when testing in a fork, it can speed things up. Since each of these build results is only a few hundred bytes without any dependencies, it has essentially no cost. Not adding this to `treefmt`, because this looks at *all* the files in the repo, so needs to re-run even when iterating on CI only. --- .github/workflows/lint.yml | 23 +++++++++++++++++++++++ .github/workflows/merge-group.yml | 2 ++ .github/workflows/pr.yml | 2 ++ 3 files changed, 27 insertions(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 44cb7fe7bada..d9168c4210d0 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -9,6 +9,9 @@ on: targetSha: required: true type: string + secrets: + CACHIX_AUTH_TOKEN: + required: true permissions: {} @@ -33,6 +36,10 @@ jobs: - uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31 + # TODO: Figure out how to best enable caching for the treefmt job. Cachix won't work well, + # because the cache would be invalidated on every commit - treefmt checks every file. + # Maybe we can cache treefmt's eval-cache somehow. + - name: Check that files are formatted run: | # Note that it's fine to run this on untrusted code because: @@ -65,6 +72,14 @@ jobs: - uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31 + - uses: cachix/cachix-action@0fc020193b5a1fa3ac4575aa3a7d3aa6a35435ad # v16 + with: + # The nixpkgs-ci cache should not be trusted or used outside of Nixpkgs and its forks' CI. + name: ${{ vars.CACHIX_NAME || 'nixpkgs-ci' }} + extraPullNames: nixpkgs-ci + authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} + pushFilter: -source$ + - name: Parse all nix files run: | # Tests multiple versions at once, let's make sure all of them run, so keep-going. @@ -88,6 +103,14 @@ jobs: - uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31 + - uses: cachix/cachix-action@0fc020193b5a1fa3ac4575aa3a7d3aa6a35435ad # v16 + with: + # The nixpkgs-ci cache should not be trusted or used outside of Nixpkgs and its forks' CI. + name: ${{ vars.CACHIX_NAME || 'nixpkgs-ci' }} + extraPullNames: nixpkgs-ci + authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} + pushFilter: -source$ + - name: Running nixpkgs-vet env: # Force terminal colors to be enabled. The library that `nixpkgs-vet` uses respects https://bixense.com/clicolors/ diff --git a/.github/workflows/merge-group.yml b/.github/workflows/merge-group.yml index 72b8deeb2dbc..9ed7125c76d2 100644 --- a/.github/workflows/merge-group.yml +++ b/.github/workflows/merge-group.yml @@ -9,6 +9,8 @@ jobs: lint: name: Lint uses: ./.github/workflows/lint.yml + secrets: + CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} with: mergedSha: ${{ github.event.merge_group.head_sha }} targetSha: ${{ github.event.merge_group.base_sha }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 1de7d6614c9c..e7535378b734 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -100,6 +100,8 @@ jobs: name: Lint needs: [prepare] uses: ./.github/workflows/lint.yml + secrets: + CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} with: mergedSha: ${{ needs.prepare.outputs.mergedSha }} targetSha: ${{ needs.prepare.outputs.targetSha }} From c4fa63da496154e74c8fa790a7a9d42f1a4d94ca Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Wed, 20 Aug 2025 12:49:57 +0200 Subject: [PATCH 4/7] ci/nixpkgs-vet: memoize filesets Speeds up evaluation a bit, because `filtered head` doesn't need to be evaluated 3x. --- ci/nixpkgs-vet.nix | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ci/nixpkgs-vet.nix b/ci/nixpkgs-vet.nix index 58b5024589bd..d3103579730d 100644 --- a/ci/nixpkgs-vet.nix +++ b/ci/nixpkgs-vet.nix @@ -16,6 +16,9 @@ let fileset = (gitTracked path); root = path; }; + + filteredBase = filtered base; + filteredHead = filtered head; in runCommand "nixpkgs-vet" { @@ -27,11 +30,11 @@ runCommand "nixpkgs-vet" '' export NIX_STATE_DIR=$(mktemp -d) - nixpkgs-vet --base ${filtered base} ${filtered head} + nixpkgs-vet --base ${filteredBase} ${filteredHead} # TODO: Upstream into nixpkgs-vet, see: # https://github.com/NixOS/nixpkgs-vet/issues/164 - badFiles=$(find ${filtered head}/pkgs -type f -name '*.nix' -print | xargs grep -l '^[^#]* to refer to itself." echo "The offending files:" @@ -41,7 +44,7 @@ runCommand "nixpkgs-vet" # TODO: Upstream into nixpkgs-vet, see: # https://github.com/NixOS/nixpkgs-vet/issues/166 - conflictingPaths=$(find ${filtered head} | awk '{ print $1 " " tolower($1) }' | sort -k2 | uniq -D -f 1 | cut -d ' ' -f 1) + conflictingPaths=$(find ${filteredHead} | awk '{ print $1 " " tolower($1) }' | sort -k2 | uniq -D -f 1 | cut -d ' ' -f 1) if [[ -n $conflictingPaths ]]; then echo "Files in nixpkgs must not vary only by case." echo "The offending paths:" From 4d996cfb2f823395c5fb469d9dee918f6a710cc3 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Wed, 20 Aug 2025 12:51:09 +0200 Subject: [PATCH 5/7] ci/nixpkgs-vet: ignore .github/ and ci/ folders nixpkgs-vet doesn't care about our CI infrastructure, so ignoring these files will lead to more cache hits when iterating on CI related PRs. --- ci/nixpkgs-vet.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/nixpkgs-vet.nix b/ci/nixpkgs-vet.nix index d3103579730d..7c11dffc4f9b 100644 --- a/ci/nixpkgs-vet.nix +++ b/ci/nixpkgs-vet.nix @@ -13,7 +13,10 @@ let with lib.fileset; path: toSource { - fileset = (gitTracked path); + fileset = difference (gitTracked path) (unions [ + (path + /.github) + (path + /ci) + ]); root = path; }; From 795dd01e31791c3d2f1d78d3c2273243b240803f Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Wed, 20 Aug 2025 13:37:48 +0200 Subject: [PATCH 6/7] workflows/eval: add cachix At this stage, this only allows faster iteration when working on CI in forks. --- .github/workflows/eval.yml | 10 ++++++++++ .github/workflows/pr.yml | 1 + .github/workflows/push.yml | 2 ++ 3 files changed, 13 insertions(+) diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index a89f2e4d5f9b..a1aaaafe6fbb 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -16,6 +16,8 @@ on: default: false type: boolean secrets: + CACHIX_AUTH_TOKEN: + required: true OWNER_APP_PRIVATE_KEY: required: false @@ -97,6 +99,14 @@ jobs: - name: Install Nix uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31 + - uses: cachix/cachix-action@0fc020193b5a1fa3ac4575aa3a7d3aa6a35435ad # v16 + with: + # The nixpkgs-ci cache should not be trusted or used outside of Nixpkgs and its forks' CI. + name: ${{ vars.CACHIX_NAME || 'nixpkgs-ci' }} + extraPullNames: nixpkgs-ci + authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} + pushFilter: '(-source|-single-chunk)$' + - name: Evaluate the ${{ matrix.system }} output paths for all derivation attributes env: MATRIX_SYSTEM: ${{ matrix.system }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index e7535378b734..b6c4fabafa12 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -114,6 +114,7 @@ jobs: # compare statuses: write secrets: + CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} OWNER_APP_PRIVATE_KEY: ${{ secrets.OWNER_APP_PRIVATE_KEY }} with: mergedSha: ${{ needs.prepare.outputs.mergedSha }} diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index ae829ed0ff2b..265f0fba8927 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -43,6 +43,8 @@ jobs: issues: write pull-requests: write statuses: write + secrets: + CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} with: mergedSha: ${{ github.sha }} systems: ${{ needs.prepare.outputs.systems }} From ac938656970ac980f19370788fca225db9faff2b Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Wed, 20 Aug 2025 17:09:54 +0200 Subject: [PATCH 7/7] workflows/build: prevent pushing tarball to cachix By definition the tarball changes on every commit, so it makes no sense to cache it - it will just waste disk space. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3b289413e6fb..b00335dc5f27 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -65,7 +65,7 @@ jobs: name: ${{ vars.CACHIX_NAME || 'nixpkgs-ci' }} extraPullNames: nixpkgs-ci authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} - pushFilter: -source$ + pushFilter: '(-source$|-nixpkgs-tarball-)' - run: nix-env --install -f pinned -A nix-build-uncached