Philip Taron
466 lines
19 KiB
YAML
466 lines
19 KiB
YAML
name: Eval
|
|
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
artifact-prefix:
|
|
required: true
|
|
type: string
|
|
mergedSha:
|
|
required: true
|
|
type: string
|
|
headSha:
|
|
required: false # only required when testVersions is true
|
|
type: string
|
|
targetSha:
|
|
required: true
|
|
type: string
|
|
systems:
|
|
required: true
|
|
type: string
|
|
testVersions:
|
|
required: false
|
|
default: false
|
|
type: boolean
|
|
secrets:
|
|
# Should only be provided in the merge queue, not in pull requests,
|
|
# where we're evaluating untrusted code.
|
|
CACHIX_AUTH_TOKEN_GHA:
|
|
required: false
|
|
|
|
permissions: {}
|
|
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
|
|
jobs:
|
|
versions:
|
|
if: inputs.testVersions
|
|
runs-on: ubuntu-slim
|
|
outputs:
|
|
versions: ${{ steps.versions.outputs.versions }}
|
|
ciPinBumpCommit: ${{ steps.find-pinned-commit.outputs.ciPinBumpCommit }}
|
|
ciPinBumpCommitShort: ${{ steps.find-pinned-commit.outputs.ciPinBumpCommitShort }}
|
|
steps:
|
|
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
persist-credentials: false
|
|
path: trusted
|
|
sparse-checkout: |
|
|
ci/supportedVersions.nix
|
|
|
|
- name: Check out the PR at the test merge commit
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
persist-credentials: false
|
|
ref: ${{ inputs.mergedSha }}
|
|
path: untrusted
|
|
sparse-checkout: |
|
|
ci/pinned.json
|
|
|
|
- name: Find commit that touched ci/pinned.json
|
|
id: find-pinned-commit
|
|
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
|
|
env:
|
|
TARGET_SHA: ${{ inputs.targetSha }}
|
|
HEAD_SHA: ${{ inputs.headSha }}
|
|
with:
|
|
script: |
|
|
const targetSha = process.env.TARGET_SHA
|
|
const headSha = process.env.HEAD_SHA
|
|
|
|
if (!targetSha || !headSha) {
|
|
core.setFailed('Error: Both targetSha and headSha inputs are required when testVersions is true.')
|
|
return
|
|
}
|
|
|
|
// Compare the two commits to get the list of commits in between
|
|
const comparison = await github.rest.repos.compareCommitsWithBasehead({
|
|
...context.repo,
|
|
basehead: `${targetSha}...${headSha}`,
|
|
})
|
|
|
|
if(comparison.data.commits.length > 50) {
|
|
core.setFailed('Error: Too many commits in comparison, cannot reliably find pinned.json change.')
|
|
return
|
|
}
|
|
|
|
const logRateLimit = async (label) => {
|
|
const { data } = await github.rest.rateLimit.get()
|
|
const { remaining, limit, used } = data.rate
|
|
core.info(`[Rate Limit ${label}] ${remaining}/${limit} remaining (${used} used)`)
|
|
}
|
|
|
|
await logRateLimit('before commit filtering')
|
|
|
|
// Filter commits that modified ci/pinned.json
|
|
const commitsModifyingPinned = (
|
|
await Promise.all(
|
|
comparison.data.commits.map(async (commit) => {
|
|
const commitDetails = await github.rest.repos.getCommit({
|
|
...context.repo,
|
|
ref: commit.sha,
|
|
})
|
|
const modifiesPinned = commitDetails.data.files?.some(
|
|
(file) => file.filename === "ci/pinned.json"
|
|
)
|
|
return modifiesPinned ? commit.sha : null
|
|
})
|
|
)
|
|
).filter((sha) => sha !== null)
|
|
|
|
await logRateLimit('after commit filtering')
|
|
|
|
if (commitsModifyingPinned.length === 0) {
|
|
// This should not happen as testVersions should only be true
|
|
// when ci/pinned.json was modified in the PR.
|
|
core.setFailed("Error: ci/pinned.json was not modified in this PR")
|
|
return
|
|
} else if (commitsModifyingPinned.length > 1) {
|
|
core.setFailed([
|
|
"Error: Multiple commits touch ci/pinned.json in this PR:",
|
|
...commitsModifyingPinned,
|
|
"Please ensure only a single commit modifies ci/pinned.json for accurate version matrix evaluation."
|
|
].join("\n"))
|
|
return
|
|
}
|
|
|
|
const ciPinBumpCommit = commitsModifyingPinned[0]
|
|
core.setOutput("ciPinBumpCommit", ciPinBumpCommit)
|
|
core.setOutput("ciPinBumpCommitShort", ciPinBumpCommit.substring(0, 7))
|
|
core.info(`Found pinned.json commit: ${ciPinBumpCommit}`)
|
|
|
|
- name: Install Nix
|
|
uses: cachix/install-nix-action@4e002c8ec80594ecd40e759629461e26c8abed15 # v31
|
|
|
|
- name: Load supported versions
|
|
id: versions
|
|
run: |
|
|
echo "versions=$(trusted/ci/supportedVersions.nix --arg pinnedJson untrusted/ci/pinned.json)" >> "$GITHUB_OUTPUT"
|
|
|
|
eval:
|
|
runs-on: ubuntu-24.04-arm
|
|
needs: versions
|
|
if: ${{ !cancelled() && !failure() }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
system: ${{ fromJSON(inputs.systems) }}
|
|
version:
|
|
- "" # Default Eval triggering rebuild labels and such.
|
|
- ${{ fromJSON(needs.versions.outputs.versions || '[]') }} # Only for ci/pinned.json updates.
|
|
# Failures for versioned Evals will be collected in a separate job below
|
|
# to not interrupt main Eval's compare step.
|
|
continue-on-error: ${{ matrix.version != '' }}
|
|
name: ${{ matrix.system }}${{ matrix.version && format(' @ {0} ({1})', matrix.version, needs.versions.outputs.ciPinBumpCommitShort) || '' }}
|
|
timeout-minutes: 15
|
|
steps:
|
|
# This is not supposed to be used and just acts as a fallback.
|
|
# Without swap, when Eval runs OOM, it will fail badly with a
|
|
# job that is sometimes not interruptible anymore.
|
|
# If Eval starts swapping, decrease chunkSize to keep it fast.
|
|
- name: Enable swap
|
|
run: |
|
|
sudo fallocate -l 10G /swap
|
|
sudo chmod 600 /swap
|
|
sudo mkswap /swap
|
|
sudo swapon /swap
|
|
|
|
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
persist-credentials: false
|
|
sparse-checkout: .github/actions
|
|
- name: Check out the PR at merged and target commits
|
|
uses: ./.github/actions/checkout
|
|
with:
|
|
# For versioned evals, use the target as the untrusted base and apply the pin-bump commit
|
|
merged-as-untrusted-at: ${{ matrix.version && inputs.targetSha || inputs.mergedSha }}
|
|
untrusted-pin-bump: ${{ matrix.version && needs.versions.outputs.ciPinBumpCommit }}
|
|
target-as-trusted-at: ${{ inputs.targetSha }}
|
|
|
|
- name: Install Nix
|
|
uses: cachix/install-nix-action@4e002c8ec80594ecd40e759629461e26c8abed15 # v31
|
|
|
|
- uses: cachix/cachix-action@3ba601ff5bbb07c7220846facfa2cd81eeee15a1 # v16
|
|
continue-on-error: true
|
|
with:
|
|
# The nixpkgs-gha cache should not be trusted or used outside of Nixpkgs and its forks' CI.
|
|
name: ${{ vars.CACHIX_NAME || 'nixpkgs-gha' }}
|
|
extraPullNames: nixpkgs-gha
|
|
authToken: ${{ secrets.CACHIX_AUTH_TOKEN_GHA }}
|
|
pushFilter: '(-source|-single-chunk)$'
|
|
|
|
- name: Evaluate the ${{ matrix.system }} output paths at the merge commit
|
|
env:
|
|
MATRIX_SYSTEM: ${{ matrix.system }}
|
|
MATRIX_VERSION: ${{ matrix.version || 'nixVersions.latest' }}
|
|
run: |
|
|
nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A eval.singleSystem \
|
|
--argstr evalSystem "$MATRIX_SYSTEM" \
|
|
--arg chunkSize 8000 \
|
|
--argstr nixPath "$MATRIX_VERSION" \
|
|
--out-link merged
|
|
# If it uses too much memory, slightly decrease chunkSize.
|
|
# Note: Keep the same further down in sync!
|
|
|
|
- name: Evaluate the ${{ matrix.system }} output paths at the target commit
|
|
env:
|
|
MATRIX_SYSTEM: ${{ matrix.system }}
|
|
run: |
|
|
TARGET_DRV=$(nix-instantiate nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.singleSystem \
|
|
--argstr evalSystem "$MATRIX_SYSTEM" \
|
|
--arg chunkSize 8000 \
|
|
--argstr nixPath "nixVersions.latest")
|
|
|
|
# Try to fetch this from Cachix a few times, for up to 30 seconds. This avoids running Eval
|
|
# twice in the Merge Queue, when a later item finishes Eval at the merge commit earlier.
|
|
for _i in {1..6}; do
|
|
# Using --max-jobs 0 will cause nix-build to fail if this can't be substituted from cachix.
|
|
if nix-build "$TARGET_DRV" --max-jobs 0; then
|
|
break
|
|
fi
|
|
sleep 5
|
|
done
|
|
|
|
# Either fetches from Cachix or runs Eval itself. The fallback is required
|
|
# for pull requests into wip-branches without merge queue.
|
|
nix-build "$TARGET_DRV" --out-link target
|
|
|
|
- name: Compare outpaths against the target branch
|
|
env:
|
|
MATRIX_SYSTEM: ${{ matrix.system }}
|
|
run: |
|
|
nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A eval.diff \
|
|
--arg beforeDir ./target \
|
|
--arg afterDir ./merged \
|
|
--argstr evalSystem "$MATRIX_SYSTEM" \
|
|
--out-link diff
|
|
|
|
- name: Upload outpaths diff and stats
|
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
|
|
with:
|
|
name: ${{ inputs.artifact-prefix }}${{ matrix.version && format('{0}-', matrix.version) || '' }}diff-${{ matrix.system }}
|
|
path: diff/*
|
|
|
|
compare:
|
|
runs-on: ubuntu-24.04-arm
|
|
needs: [eval]
|
|
if: ${{ !cancelled() && !failure() }}
|
|
permissions:
|
|
pull-requests: write
|
|
statuses: write
|
|
timeout-minutes: 5
|
|
steps:
|
|
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
persist-credentials: false
|
|
sparse-checkout: .github/actions
|
|
- name: Check out the PR at the target commit
|
|
uses: ./.github/actions/checkout
|
|
with:
|
|
merged-as-untrusted-at: ${{ inputs.mergedSha }}
|
|
target-as-trusted-at: ${{ inputs.targetSha }}
|
|
|
|
- name: Download output paths and eval stats for all systems
|
|
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
|
|
with:
|
|
pattern: ${{ inputs.artifact-prefix }}diff-*
|
|
path: diff
|
|
merge-multiple: true
|
|
|
|
- name: Install Nix
|
|
uses: cachix/install-nix-action@4e002c8ec80594ecd40e759629461e26c8abed15 # v31
|
|
|
|
- name: Combine all output paths and eval stats
|
|
run: |
|
|
nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.combine \
|
|
--arg diffDir ./diff \
|
|
--out-link combined
|
|
|
|
- name: Upload the maintainer list
|
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
|
|
with:
|
|
name: ${{ inputs.artifact-prefix }}maintainers
|
|
path: combined/maintainers.json
|
|
|
|
- name: Compare against the target branch
|
|
env:
|
|
TARGET_SHA: ${{ inputs.mergedSha }}
|
|
run: |
|
|
git -C nixpkgs/trusted diff --name-only "$TARGET_SHA" \
|
|
| jq --raw-input --slurp 'split("\n")[:-1]' > touched-files.json
|
|
|
|
# Use the target branch to get accurate maintainer info
|
|
nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.compare \
|
|
--arg combinedDir ./combined \
|
|
--arg touchedFilesJson ./touched-files.json \
|
|
--out-link comparison
|
|
|
|
cat comparison/step-summary.md >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
- name: Upload the comparison results
|
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
|
|
with:
|
|
name: ${{ inputs.artifact-prefix }}comparison
|
|
path: comparison/*
|
|
|
|
- name: Add eval summary to commit statuses
|
|
if: ${{ github.event_name == 'pull_request_target' }}
|
|
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
|
|
with:
|
|
script: |
|
|
const { readFile } = require('node:fs/promises')
|
|
const changed = JSON.parse(await readFile('comparison/changed-paths.json', 'utf-8'))
|
|
const description =
|
|
'Package: ' + [
|
|
`added ${changed.attrdiff.added.length}`,
|
|
`removed ${changed.attrdiff.removed.length}`,
|
|
`changed ${changed.attrdiff.changed.length}`
|
|
].join(', ') +
|
|
' — Rebuild: ' + [
|
|
`linux ${changed.rebuildCountByKernel.linux}`,
|
|
`darwin ${changed.rebuildCountByKernel.darwin}`
|
|
].join(', ')
|
|
|
|
const { serverUrl, repo, runId, payload } = context
|
|
const target_url =
|
|
`${serverUrl}/${repo.owner}/${repo.repo}/actions/runs/${runId}?pr=${payload.pull_request.number}`
|
|
|
|
await github.rest.repos.createCommitStatus({
|
|
...repo,
|
|
sha: payload.pull_request.head.sha,
|
|
context: 'Eval Summary',
|
|
state: 'success',
|
|
description,
|
|
target_url
|
|
})
|
|
- name: Request changes if PR is against an inappropriate branch
|
|
if: ${{ github.event_name == 'pull_request_target' }}
|
|
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
|
|
with:
|
|
script: |
|
|
require('./nixpkgs/trusted/ci/github-script/check-target-branch.js')({
|
|
github,
|
|
context,
|
|
core,
|
|
dry: context.eventName == 'pull_request',
|
|
})
|
|
|
|
# Creates a matrix of Eval performance for various versions and systems.
|
|
report:
|
|
runs-on: ubuntu-slim
|
|
needs: [versions, eval]
|
|
steps:
|
|
- name: Download output paths and eval stats for all versions
|
|
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
|
|
with:
|
|
pattern: "*-diff-*"
|
|
path: versions
|
|
|
|
- name: Add version comparison table to job summary
|
|
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
|
|
env:
|
|
ARTIFACT_PREFIX: ${{ inputs.artifact-prefix }}
|
|
SYSTEMS: ${{ inputs.systems }}
|
|
VERSIONS: ${{ needs.versions.outputs.versions }}
|
|
CI_PIN_BUMP_COMMIT: ${{ needs.versions.outputs.ciPinBumpCommit }}
|
|
with:
|
|
script: |
|
|
const { readFileSync } = require('node:fs')
|
|
const path = require('node:path')
|
|
|
|
const prefix = process.env.ARTIFACT_PREFIX
|
|
const systems = JSON.parse(process.env.SYSTEMS)
|
|
const versions = JSON.parse(process.env.VERSIONS)
|
|
const ciPinBumpCommit = process.env.CI_PIN_BUMP_COMMIT
|
|
|
|
core.summary.addHeading('Lix/Nix version comparison')
|
|
core.summary.addRaw(`\n*Evaluated at commit: \`${ciPinBumpCommit}\` (commit that modified ci/pinned.json)*\n`, true)
|
|
core.summary.addTable(
|
|
[].concat(
|
|
[
|
|
[{ data: 'Version', header: true }].concat(
|
|
systems.map((system) => ({ data: system, header: true })),
|
|
),
|
|
],
|
|
versions.map((version) =>
|
|
[{ data: version }].concat(
|
|
systems.map((system) => {
|
|
try {
|
|
const artifact = path.join('versions', `${prefix}${version}-diff-${system}`)
|
|
const time = Math.round(
|
|
parseFloat(
|
|
readFileSync(
|
|
path.join(artifact, 'after', system, 'total-time'),
|
|
'utf-8',
|
|
),
|
|
),
|
|
)
|
|
const diff = JSON.parse(
|
|
readFileSync(path.join(artifact, system, 'diff.json'), 'utf-8'),
|
|
)
|
|
const attrs = []
|
|
.concat(diff.added, diff.removed, diff.changed, diff.rebuilds)
|
|
// There are some special attributes, which are ignored for rebuilds.
|
|
// These only have a single path component, because they lack the `.<system>` suffix.
|
|
.filter((attr) => attr.split('.').length > 1)
|
|
if (attrs.length > 0) {
|
|
core.setFailed(
|
|
`${version} on ${system} has changed outpaths!\n` +
|
|
`Note: This indicates that commit ${ciPinBumpCommit} ` +
|
|
`(which modified ci/pinned.json) also contains other ` +
|
|
`changes affecting package outputs. ` +
|
|
`Please ensure ci/pinned.json is updated in a standalone commit.`
|
|
)
|
|
return { data: ':x:' }
|
|
}
|
|
return { data: time }
|
|
} catch {
|
|
core.warning(`${version} on ${system} did not produce artifact.`)
|
|
return { data: ':warning:' }
|
|
}
|
|
}),
|
|
),
|
|
),
|
|
),
|
|
)
|
|
core.summary.addRaw(
|
|
'\n*Evaluation time in seconds without downloading dependencies.*',
|
|
true,
|
|
)
|
|
core.summary.addRaw('\n*:warning: Job did not report a result.*', true)
|
|
core.summary.addRaw(
|
|
'\n*:x: Job produced different outpaths than the target branch.*',
|
|
true,
|
|
)
|
|
core.summary.write()
|
|
|
|
misc:
|
|
if: ${{ github.event_name != 'push' }}
|
|
runs-on: ubuntu-24.04-arm
|
|
timeout-minutes: 10
|
|
steps:
|
|
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
persist-credentials: false
|
|
sparse-checkout: .github/actions
|
|
- name: Checkout the merge commit
|
|
uses: ./.github/actions/checkout
|
|
with:
|
|
merged-as-untrusted-at: ${{ inputs.mergedSha }}
|
|
|
|
- name: Install Nix
|
|
uses: cachix/install-nix-action@4e002c8ec80594ecd40e759629461e26c8abed15 # v31
|
|
|
|
- name: Ensure flake outputs on all systems still evaluate
|
|
run: nix flake check --all-systems --no-build './nixpkgs/untrusted?shallow=1'
|
|
|
|
- name: Query nixpkgs with aliases enabled to check for basic syntax errors
|
|
run: |
|
|
time nix-env -I ./nixpkgs/untrusted -f ./nixpkgs/untrusted -qa '*' --option restrict-eval true --option allow-import-from-derivation false >/dev/null
|
|
|
|
- name: Ensure NixOS modules meta is valid
|
|
run: |
|
|
time nix-instantiate -I ./nixpkgs/untrusted --strict --eval --json ./nixpkgs/untrusted/nixos --arg configuration '{}' --attr config.meta --option restrict-eval true --option allow-import-from-derivation false
|