498574f238
This creates a simple mapping of all packages to github ids of all their maintainers. This is uploaded as an artifact and is then available for download on the latest commit of each branch with a merge queue. This will allow scheduled jobs to use this information for setting maintainer-related labels, to request reviewers and to implement the merge-bot. The advantage over querying this information directly via Nix in each case: The scheduled job does not need to install Nix and does not need to checkout the target branch. Compared to obtaining the maintainer information just for a single PR during Eval, this will allow retroactively changing maintainers for a package: For example, it allows to request a new maintainer as reviewer for a PR that was created before they became maintainer, but is still open - and similarly for maintainer labels and merge-bot rights. None of these extensions are implemented by this PR, yet.
363 lines
14 KiB
YAML
363 lines
14 KiB
YAML
name: Eval
|
|
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
artifact-prefix:
|
|
required: true
|
|
type: string
|
|
mergedSha:
|
|
required: true
|
|
type: string
|
|
targetSha:
|
|
type: string
|
|
systems:
|
|
required: true
|
|
type: string
|
|
testVersions:
|
|
required: false
|
|
default: false
|
|
type: boolean
|
|
secrets:
|
|
CACHIX_AUTH_TOKEN:
|
|
required: true
|
|
|
|
permissions: {}
|
|
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
|
|
jobs:
|
|
versions:
|
|
if: inputs.testVersions
|
|
runs-on: ubuntu-24.04-arm
|
|
outputs:
|
|
versions: ${{ steps.versions.outputs.versions }}
|
|
steps:
|
|
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
|
with:
|
|
persist-credentials: false
|
|
path: trusted
|
|
sparse-checkout: |
|
|
ci/supportedVersions.nix
|
|
|
|
- name: Check out the PR at the test merge commit
|
|
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
|
with:
|
|
persist-credentials: false
|
|
ref: ${{ inputs.mergedSha }}
|
|
path: untrusted
|
|
sparse-checkout: |
|
|
ci/pinned.json
|
|
|
|
- name: Install Nix
|
|
uses: cachix/install-nix-action@fd24c48048070c1be9acd18c9d369a83f0fe94d7 # v31
|
|
|
|
- name: Load supported versions
|
|
id: versions
|
|
run: |
|
|
echo "versions=$(trusted/ci/supportedVersions.nix --arg pinnedJson untrusted/ci/pinned.json)" >> "$GITHUB_OUTPUT"
|
|
|
|
eval:
|
|
runs-on: ubuntu-24.04-arm
|
|
needs: versions
|
|
if: ${{ !cancelled() && !failure() }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
system: ${{ fromJSON(inputs.systems) }}
|
|
version:
|
|
- "" # Default Eval triggering rebuild labels and such.
|
|
- ${{ fromJSON(needs.versions.outputs.versions || '[]') }} # Only for ci/pinned.json updates.
|
|
# Failures for versioned Evals will be collected in a separate job below
|
|
# to not interrupt main Eval's compare step.
|
|
continue-on-error: ${{ matrix.version != '' }}
|
|
name: ${{ matrix.system }}${{ matrix.version && format(' @ {0}', matrix.version) || '' }}
|
|
timeout-minutes: 15
|
|
steps:
|
|
# This is not supposed to be used and just acts as a fallback.
|
|
# Without swap, when Eval runs OOM, it will fail badly with a
|
|
# job that is sometimes not interruptible anymore.
|
|
# If Eval starts swapping, decrease chunkSize to keep it fast.
|
|
- name: Enable swap
|
|
run: |
|
|
sudo fallocate -l 10G /swap
|
|
sudo chmod 600 /swap
|
|
sudo mkswap /swap
|
|
sudo swapon /swap
|
|
|
|
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
|
with:
|
|
persist-credentials: false
|
|
sparse-checkout: .github/actions
|
|
- name: Check out the PR at merged and target commits
|
|
uses: ./.github/actions/checkout
|
|
with:
|
|
merged-as-untrusted-at: ${{ inputs.mergedSha }}
|
|
target-as-trusted-at: ${{ inputs.targetSha }}
|
|
|
|
- name: Install Nix
|
|
uses: cachix/install-nix-action@fd24c48048070c1be9acd18c9d369a83f0fe94d7 # v31
|
|
|
|
- uses: cachix/cachix-action@0fc020193b5a1fa3ac4575aa3a7d3aa6a35435ad # v16
|
|
with:
|
|
# The nixpkgs-ci cache should not be trusted or used outside of Nixpkgs and its forks' CI.
|
|
name: ${{ vars.CACHIX_NAME || 'nixpkgs-ci' }}
|
|
extraPullNames: nixpkgs-ci
|
|
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
|
|
pushFilter: '(-source|-single-chunk)$'
|
|
|
|
- name: Evaluate the ${{ matrix.system }} output paths at the merge commit
|
|
env:
|
|
MATRIX_SYSTEM: ${{ matrix.system }}
|
|
MATRIX_VERSION: ${{ matrix.version || 'nixVersions.latest' }}
|
|
run: |
|
|
nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A eval.singleSystem \
|
|
--argstr evalSystem "$MATRIX_SYSTEM" \
|
|
--arg chunkSize 8000 \
|
|
--argstr nixPath "$MATRIX_VERSION" \
|
|
--out-link merged
|
|
# If it uses too much memory, slightly decrease chunkSize.
|
|
# Note: Keep the same further down in sync!
|
|
|
|
- name: Evaluate the ${{ matrix.system }} output paths at the target commit
|
|
env:
|
|
MATRIX_SYSTEM: ${{ matrix.system }}
|
|
run: |
|
|
TARGET_DRV=$(nix-instantiate nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.singleSystem \
|
|
--argstr evalSystem "$MATRIX_SYSTEM" \
|
|
--arg chunkSize 8000 \
|
|
--argstr nixPath "nixVersions.latest")
|
|
|
|
# Try to fetch this from Cachix a few times, for up to 30 seconds. This avoids running Eval
|
|
# twice in the Merge Queue, when a later item finishes Eval at the merge commit earlier.
|
|
for _i in {1..6}; do
|
|
# Using --max-jobs 0 will cause nix-build to fail if this can't be substituted from cachix.
|
|
if nix-build "$TARGET_DRV" --max-jobs 0; then
|
|
break
|
|
fi
|
|
sleep 5
|
|
done
|
|
|
|
# Either fetches from Cachix or runs Eval itself. The fallback is required
|
|
# for pull requests into wip-branches without merge queue.
|
|
nix-build "$TARGET_DRV" --out-link target
|
|
|
|
- name: Compare outpaths against the target branch
|
|
env:
|
|
MATRIX_SYSTEM: ${{ matrix.system }}
|
|
run: |
|
|
nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A eval.diff \
|
|
--arg beforeDir ./target \
|
|
--arg afterDir ./merged \
|
|
--argstr evalSystem "$MATRIX_SYSTEM" \
|
|
--out-link diff
|
|
|
|
- name: Upload outpaths diff and stats
|
|
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
|
|
with:
|
|
name: ${{ inputs.artifact-prefix }}${{ matrix.version && format('{0}-', matrix.version) || '' }}diff-${{ matrix.system }}
|
|
path: diff/*
|
|
|
|
compare:
|
|
runs-on: ubuntu-24.04-arm
|
|
needs: [eval]
|
|
if: ${{ !cancelled() && !failure() }}
|
|
permissions:
|
|
statuses: write
|
|
timeout-minutes: 5
|
|
steps:
|
|
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
|
with:
|
|
persist-credentials: false
|
|
sparse-checkout: .github/actions
|
|
- name: Check out the PR at the target commit
|
|
uses: ./.github/actions/checkout
|
|
with:
|
|
merged-as-untrusted-at: ${{ inputs.mergedSha }}
|
|
target-as-trusted-at: ${{ inputs.targetSha }}
|
|
|
|
- name: Download output paths and eval stats for all systems
|
|
uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
|
|
with:
|
|
pattern: ${{ inputs.artifact-prefix }}diff-*
|
|
path: diff
|
|
merge-multiple: true
|
|
|
|
- name: Install Nix
|
|
uses: cachix/install-nix-action@fd24c48048070c1be9acd18c9d369a83f0fe94d7 # v31
|
|
|
|
- name: Combine all output paths and eval stats
|
|
run: |
|
|
nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.combine \
|
|
--arg diffDir ./diff \
|
|
--out-link combined
|
|
|
|
- name: Upload the maintainer list
|
|
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
|
|
with:
|
|
name: ${{ inputs.artifact-prefix }}maintainers
|
|
path: combined/maintainers.json
|
|
|
|
- name: Compare against the target branch
|
|
env:
|
|
AUTHOR_ID: ${{ github.event.pull_request.user.id }}
|
|
TARGET_SHA: ${{ inputs.mergedSha }}
|
|
run: |
|
|
git -C nixpkgs/trusted diff --name-only "$TARGET_SHA" \
|
|
| jq --raw-input --slurp 'split("\n")[:-1]' > touched-files.json
|
|
|
|
# Use the target branch to get accurate maintainer info
|
|
nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.compare \
|
|
--arg combinedDir ./combined \
|
|
--arg touchedFilesJson ./touched-files.json \
|
|
--argstr githubAuthorId "$AUTHOR_ID" \
|
|
--out-link comparison
|
|
|
|
cat comparison/step-summary.md >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
- name: Upload the comparison results
|
|
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
|
|
with:
|
|
name: ${{ inputs.artifact-prefix }}comparison
|
|
path: comparison/*
|
|
|
|
- name: Add eval summary to commit statuses
|
|
if: ${{ github.event_name == 'pull_request_target' }}
|
|
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
|
|
with:
|
|
script: |
|
|
const { readFile } = require('node:fs/promises')
|
|
const changed = JSON.parse(await readFile('comparison/changed-paths.json', 'utf-8'))
|
|
const description =
|
|
'Package: ' + [
|
|
`added ${changed.attrdiff.added.length}`,
|
|
`removed ${changed.attrdiff.removed.length}`,
|
|
`changed ${changed.attrdiff.changed.length}`
|
|
].join(', ') +
|
|
' — Rebuild: ' + [
|
|
`linux ${changed.rebuildCountByKernel.linux}`,
|
|
`darwin ${changed.rebuildCountByKernel.darwin}`
|
|
].join(', ')
|
|
|
|
const { serverUrl, repo, runId, payload } = context
|
|
const target_url =
|
|
`${serverUrl}/${repo.owner}/${repo.repo}/actions/runs/${runId}?pr=${payload.pull_request.number}`
|
|
|
|
await github.rest.repos.createCommitStatus({
|
|
...repo,
|
|
sha: payload.pull_request.head.sha,
|
|
context: 'Eval Summary',
|
|
state: 'success',
|
|
description,
|
|
target_url
|
|
})
|
|
|
|
# Creates a matrix of Eval performance for various versions and systems.
|
|
report:
|
|
runs-on: ubuntu-24.04-arm
|
|
needs: [versions, eval]
|
|
steps:
|
|
- name: Download output paths and eval stats for all versions
|
|
uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
|
|
with:
|
|
pattern: "*-diff-*"
|
|
path: versions
|
|
|
|
- name: Add version comparison table to job summary
|
|
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
|
|
env:
|
|
ARTIFACT_PREFIX: ${{ inputs.artifact-prefix }}
|
|
SYSTEMS: ${{ inputs.systems }}
|
|
VERSIONS: ${{ needs.versions.outputs.versions }}
|
|
with:
|
|
script: |
|
|
const { readFileSync } = require('node:fs')
|
|
const path = require('node:path')
|
|
|
|
const prefix = process.env.ARTIFACT_PREFIX
|
|
const systems = JSON.parse(process.env.SYSTEMS)
|
|
const versions = JSON.parse(process.env.VERSIONS)
|
|
|
|
core.summary.addHeading('Lix/Nix version comparison')
|
|
core.summary.addTable(
|
|
[].concat(
|
|
[
|
|
[{ data: 'Version', header: true }].concat(
|
|
systems.map((system) => ({ data: system, header: true })),
|
|
),
|
|
],
|
|
versions.map((version) =>
|
|
[{ data: version }].concat(
|
|
systems.map((system) => {
|
|
try {
|
|
const artifact = path.join('versions', `${prefix}${version}-diff-${system}`)
|
|
const time = Math.round(
|
|
parseFloat(
|
|
readFileSync(
|
|
path.join(artifact, 'after', system, 'total-time'),
|
|
'utf-8',
|
|
),
|
|
),
|
|
)
|
|
const diff = JSON.parse(
|
|
readFileSync(path.join(artifact, system, 'diff.json'), 'utf-8'),
|
|
)
|
|
const attrs = []
|
|
.concat(diff.added, diff.removed, diff.changed, diff.rebuilds)
|
|
// There are some special attributes, which are ignored for rebuilds.
|
|
// These only have a single path component, because they lack the `.<system>` suffix.
|
|
.filter((attr) => attr.split('.').length > 1)
|
|
if (attrs.length > 0) {
|
|
core.setFailed(
|
|
`${version} on ${system} has changed outpaths!\nNote: Please make sure to update ci/pinned.json separately from changes to other packages.`,
|
|
)
|
|
return { data: ':x:' }
|
|
}
|
|
return { data: time }
|
|
} catch {
|
|
core.warning(`${version} on ${system} did not produce artifact.`)
|
|
return { data: ':warning:' }
|
|
}
|
|
}),
|
|
),
|
|
),
|
|
),
|
|
)
|
|
core.summary.addRaw(
|
|
'\n*Evaluation time in seconds without downloading dependencies.*',
|
|
true,
|
|
)
|
|
core.summary.addRaw('\n*:warning: Job did not report a result.*', true)
|
|
core.summary.addRaw(
|
|
'\n*:x: Job produced different outpaths than the target branch.*',
|
|
true,
|
|
)
|
|
core.summary.write()
|
|
|
|
misc:
|
|
if: ${{ github.event_name != 'push' }}
|
|
runs-on: ubuntu-24.04-arm
|
|
timeout-minutes: 10
|
|
steps:
|
|
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
|
with:
|
|
persist-credentials: false
|
|
sparse-checkout: .github/actions
|
|
- name: Checkout the merge commit
|
|
uses: ./.github/actions/checkout
|
|
with:
|
|
merged-as-untrusted-at: ${{ inputs.mergedSha }}
|
|
|
|
- name: Install Nix
|
|
uses: cachix/install-nix-action@fd24c48048070c1be9acd18c9d369a83f0fe94d7 # v31
|
|
|
|
- name: Run misc eval tasks in parallel
|
|
run: |
|
|
# Ensure flake outputs on all systems still evaluate
|
|
nix flake check --all-systems --no-build './nixpkgs/untrusted?shallow=1' &
|
|
# Query nixpkgs with aliases enabled to check for basic syntax errors
|
|
nix-env -I ./nixpkgs/untrusted -f ./nixpkgs/untrusted -qa '*' --option restrict-eval true --option allow-import-from-derivation false >/dev/null &
|
|
wait
|