gador/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

workflows/eval: compare eval consistency and performance between Lix / Nix versions (#427724)

Browse Source
This commit is contained in:
Wolfgang Walther
2025-08-12 08:22:08 +00:00
committed by GitHub
parent 85e76ee5f8 14a6d9de46
commit 161a4f0a69
7 changed files with 195 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/eval.yml
+132 -5
View File
@@ -11,6 +11,10 @@ on:
systems:
required: true
type: string
testVersions:
required: false
default: false
type: boolean
secrets:
OWNER_APP_PRIVATE_KEY:
required: false
@@ -22,13 +26,49 @@ defaults:
shell: bash
jobs:
versions:
if: inputs.testVersions
runs-on: ubuntu-24.04-arm
outputs:
versions: ${{ steps.versions.outputs.versions }}
steps:
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
path: trusted
sparse-checkout: |
ci/supportedVersions.nix
- name: Check out the PR at the test merge commit
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
ref: ${{ inputs.mergedSha }}
path: untrusted
sparse-checkout: |
ci/pinned.json
- name: Install Nix
uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31
- name: Load supported versions
id: versions
run: |
echo "versions=$(trusted/ci/supportedVersions.nix --arg pinnedJson untrusted/ci/pinned.json)" >> "$GITHUB_OUTPUT"
eval:
runs-on: ubuntu-24.04-arm
needs: versions
if: ${{ !cancelled() }}
strategy:
fail-fast: false
matrix:
system: ${{ fromJSON(inputs.systems) }}
name: ${{ matrix.system }}
version:
- "" # Default Eval triggering rebuild labels and such.
- ${{ fromJSON(needs.versions.outputs.versions || '[]') }} # Only for ci/pinned.json updates.
# Failures for versioned Evals will be collected in a separate job below
# to not interrupt main Eval's compare step.
continue-on-error: ${{ matrix.version != '' }}
name: ${{ matrix.system }}${{ matrix.version && format(' @ {0}', matrix.version) || '' }}
outputs:
targetRunId: ${{ steps.targetRunId.outputs.targetRunId }}
timeout-minutes: 15
@@ -60,17 +100,19 @@ jobs:
- name: Evaluate the ${{ matrix.system }} output paths for all derivation attributes
env:
MATRIX_SYSTEM: ${{ matrix.system }}
MATRIX_VERSION: ${{ matrix.version || 'nixVersions.latest' }}
run: |
nix-build untrusted/ci --arg nixpkgs ./pinned -A eval.singleSystem \
--argstr evalSystem "$MATRIX_SYSTEM" \
--arg chunkSize 8000 \
--argstr nixPath "$MATRIX_VERSION" \
--out-link merged
# If it uses too much memory, slightly decrease chunkSize
- name: Upload the output paths and eval stats
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: merged-${{ matrix.system }}
name: ${{ matrix.version && format('{0}-', matrix.version) || '' }}merged-${{ matrix.system }}
path: merged/*
- name: Log current API rate limits
@@ -125,7 +167,7 @@ jobs:
GH_TOKEN: ${{ github.token }}
run: gh api /rate_limit | jq
- uses: actions/download-artifact@v5
- uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
if: steps.targetRunId.outputs.targetRunId
with:
run-id: ${{ steps.targetRunId.outputs.targetRunId }}
@@ -149,7 +191,7 @@ jobs:
if: steps.targetRunId.outputs.targetRunId
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: diff-${{ matrix.system }}
name: ${{ matrix.version && format('{0}-', matrix.version) || '' }}diff-${{ matrix.system }}
path: diff/*
compare:
@@ -171,7 +213,7 @@ jobs:
pinnedFrom: trusted
- name: Download output paths and eval stats for all systems
uses: actions/download-artifact@de96f4613b77ec03b5cf633e7c350c32bd3c5660 # v4.1.8
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
with:
pattern: diff-*
path: diff
@@ -240,6 +282,91 @@ jobs:
target_url
})
# Creates a matrix of Eval performance for various versions and systems.
report:
runs-on: ubuntu-24.04-arm
needs: [versions, eval]
steps:
- name: Download output paths and eval stats for all versions
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
with:
pattern: "*-diff-*"
path: versions
- name: Add version comparison table to job summary
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
env:
SYSTEMS: ${{ inputs.systems }}
VERSIONS: ${{ needs.versions.outputs.versions }}
with:
script: |
const { readFileSync } = require('node:fs')
const path = require('node:path')
const systems = JSON.parse(process.env.SYSTEMS)
const versions = JSON.parse(process.env.VERSIONS)
core.summary.addHeading('Lix/Nix version comparison')
core.summary.addTable(
[].concat(
[
[{ data: 'Version', header: true }].concat(
systems.map((system) => ({ data: system, header: true })),
),
],
versions.map((version) =>
[{ data: version }].concat(
systems.map((system) => {
try {
const artifact = path.join('versions', `${version}-diff-${system}`)
const time = Math.round(
parseFloat(
readFileSync(
path.join(artifact, 'after', system, 'total-time'),
'utf-8',
),
),
)
const diff = JSON.parse(
readFileSync(path.join(artifact, system, 'diff.json'), 'utf-8'),
)
const attrs = [].concat(
diff.added,
diff.removed,
diff.changed,
diff.rebuilds
).filter(attr =>
// Exceptions related to dev shells, which changed at some time between 2.18 and 2.24.
!attr.startsWith('tests.devShellTools.nixos.') &&
!attr.startsWith('tests.devShellTools.unstructuredDerivationInputEnv.')
)
if (attrs.length > 0) {
core.setFailed(
`${version} on ${system} has changed outpaths!\nNote: Please make sure to update ci/pinned.json separately from changes to other packages.`,
)
return { data: ':x:' }
}
return { data: time }
} catch {
core.warning(`${version} on ${system} did not produce artifact.`)
return { data: ':warning:' }
}
}),
),
),
),
)
core.summary.addRaw(
'\n*Evaluation time in seconds without downloading dependencies.*',
true,
)
core.summary.addRaw('\n*:warning: Job did not report a result.*', true)
core.summary.addRaw(
'\n*:x: Job produced different outpaths than the target branch.*',
true,
)
core.summary.write()
misc:
if: ${{ github.event_name != 'push' }}
runs-on: ubuntu-24.04-arm
.github/workflows/pr.yml
+16
View File
@@ -28,6 +28,7 @@ jobs:
mergedSha: ${{ steps.get-merge-commit.outputs.mergedSha }}
targetSha: ${{ steps.get-merge-commit.outputs.targetSha }}
systems: ${{ steps.systems.outputs.systems }}
touched: ${{ steps.files.outputs.touched }}
steps:
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
@@ -64,6 +65,20 @@ jobs:
core.setOutput('head', headClassification)
core.info('head classification:', headClassification)
- name: Determine changed files
id: files
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
with:
script: |
const files = (await github.paginate(github.rest.pulls.listFiles, {
...context.repo,
pull_number: context.payload.pull_request.number,
per_page: 100,
})).map(file => file.filename)
if (files.includes('ci/pinned.json')) core.setOutput('touched', ['pinned'])
else core.setOutput('touched', [])
check:
name: Check
needs: [prepare]
@@ -96,6 +111,7 @@ jobs:
mergedSha: ${{ needs.prepare.outputs.mergedSha }}
targetSha: ${{ needs.prepare.outputs.targetSha }}
systems: ${{ needs.prepare.outputs.systems }}
testVersions: ${{ contains(fromJSON(needs.prepare.outputs.touched), 'pinned') && !contains(fromJSON(needs.prepare.outputs.headBranch).type, 'development') }}
labels:
name: Labels
.github/workflows/reviewers.yml
+1 -1
View File
@@ -96,7 +96,7 @@ jobs:
run: gh api /rate_limit | jq
- name: Download the comparison results
uses: actions/download-artifact@de96f4613b77ec03b5cf633e7c350c32bd3c5660 # v4.1.8
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
with:
run-id: ${{ steps.eval.outputs.run-id }}
github-token: ${{ github.token }}
ci/default.nix
+2 -1
View File
@@ -5,6 +5,7 @@ in
system ? builtins.currentSystem,
nixpkgs ? null,
nixPath ? "nixVersions.latest",
}:
let
nixpkgs' =
@@ -115,7 +116,7 @@ rec {
# (nixVersions.stable and Lix) here somehow at some point to ensure we don't
# have eval divergence.
eval = pkgs.callPackage ./eval {
nix = pkgs.nixVersions.latest;
nix = pkgs.lib.getAttrFromPath (pkgs.lib.splitString "." nixPath) pkgs;
};
# CI jobs
ci/eval/compare/default.nix
+6 -7
View File
@@ -146,6 +146,12 @@ runCommand "compare"
cp ${changed-paths} $out/changed-paths.json
{
echo
echo "# Packages"
echo
jq -r -f ${./generate-step-summary.jq} < ${changed-paths}
} >> $out/step-summary.md
if jq -e '(.attrdiff.added | length == 0) and (.attrdiff.removed | length == 0)' "${changed-paths}" > /dev/null; then
# Chunks have changed between revisions
@@ -175,12 +181,5 @@ runCommand "compare"
} >> $out/step-summary.md
fi
{
echo
echo "# Packages"
echo
jq -r -f ${./generate-step-summary.jq} < ${changed-paths}
} >> $out/step-summary.md
cp "$maintainersPath" "$out/maintainers.json"
''
ci/pinned.json
+6 -6
View File
@@ -9,9 +9,9 @@
},
"branch": "nixpkgs-unstable",
"submodules": false,
"revision": "6a489c9482ca676ce23c0bcd7f2e1795383325fa",
"url": "https://github.com/NixOS/nixpkgs/archive/6a489c9482ca676ce23c0bcd7f2e1795383325fa.tar.gz",
"hash": "0vsvkhy3gb8yzq62vazhmpqixssmd4xinnll7w73l4vrqd611wlf"
"revision": "641d909c4a7538f1539da9240dedb1755c907e40",
"url": "https://github.com/NixOS/nixpkgs/archive/641d909c4a7538f1539da9240dedb1755c907e40.tar.gz",
"hash": "10hpb1aw884k3zzcy1mhf47dqvfagiyx7kr6hg0p5xcwg04mkx8x"
},
"treefmt-nix": {
"type": "Git",
@@ -22,9 +22,9 @@
},
"branch": "main",
"submodules": false,
"revision": "58bd4da459f0a39e506847109a2a5cfceb837796",
"url": "https://github.com/numtide/treefmt-nix/archive/58bd4da459f0a39e506847109a2a5cfceb837796.tar.gz",
"hash": "01bg9b4xzlv6s5q1q78vib6l2csw02b3rk5bm5yj4gx2sk2hvmrq"
"revision": "7d81f6fb2e19bf84f1c65135d1060d829fae2408",
"url": "https://github.com/numtide/treefmt-nix/archive/7d81f6fb2e19bf84f1c65135d1060d829fae2408.tar.gz",
"hash": "1cg20q8ja8k2nb7mzy95hgmd8whxapc3fbyndh1ip5dr6d1grxfs"
}
},
"version": 5
ci/supportedVersions.nix
Executable
+32
View File
@@ -0,0 +1,32 @@
#!/usr/bin/env -S nix-instantiate --eval --strict --json --arg unused true
# Unused argument to trigger nix-instantiate calling this function with the default arguments.
{
pinnedJson ? ./pinned.json,
}:
let
pinned = (builtins.fromJSON (builtins.readFile pinnedJson)).pins;
nixpkgs = fetchTarball {
inherit (pinned.nixpkgs) url;
sha256 = pinned.nixpkgs.hash;
};
pkgs = import nixpkgs {
config.allowAliases = false;
};
inherit (pkgs) lib;
lix = lib.pipe pkgs.lixPackageSets [
(lib.filterAttrs (_: set: lib.isDerivation set.lix or null && set.lix.meta.available))
lib.attrNames
(lib.filter (name: lib.match "lix_[0-9_]+|git" name != null))
(map (name: "lixPackageSets.${name}.lix"))
];
nix = lib.pipe pkgs.nixVersions [
(lib.filterAttrs (_: drv: lib.isDerivation drv && drv.meta.available))
lib.attrNames
(lib.filter (name: lib.match "nix_[0-9_]+|git" name != null))
(map (name: "nixVersions.${name}"))
];
in
lix ++ nix