5d6dd23117
Evaluating untrusted code in the presence of secrets is unsafe in general, thus we only provide the cachix auth token when these jobs run in the merge queue. This is enough for all practical purposes, PRs will be able to pull stuff from cachix that was built in the Merge Queue previously.
363 lines
14 KiB
YAML
363 lines
14 KiB
YAML
name: Eval
|
|
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
artifact-prefix:
|
|
required: true
|
|
type: string
|
|
mergedSha:
|
|
required: true
|
|
type: string
|
|
targetSha:
|
|
type: string
|
|
systems:
|
|
required: true
|
|
type: string
|
|
testVersions:
|
|
required: false
|
|
default: false
|
|
type: boolean
|
|
secrets:
|
|
# Should only be provided in the merge queue, not in pull requests,
|
|
# where we're evaluating untrusted code.
|
|
CACHIX_AUTH_TOKEN:
|
|
required: false
|
|
|
|
permissions: {}
|
|
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
|
|
jobs:
|
|
versions:
|
|
if: inputs.testVersions
|
|
runs-on: ubuntu-24.04-arm
|
|
outputs:
|
|
versions: ${{ steps.versions.outputs.versions }}
|
|
steps:
|
|
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
|
with:
|
|
persist-credentials: false
|
|
path: trusted
|
|
sparse-checkout: |
|
|
ci/supportedVersions.nix
|
|
|
|
- name: Check out the PR at the test merge commit
|
|
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
|
with:
|
|
persist-credentials: false
|
|
ref: ${{ inputs.mergedSha }}
|
|
path: untrusted
|
|
sparse-checkout: |
|
|
ci/pinned.json
|
|
|
|
- name: Install Nix
|
|
uses: cachix/install-nix-action@456688f15bc354bef6d396e4a35f4f89d40bf2b7 # v31
|
|
|
|
- name: Load supported versions
|
|
id: versions
|
|
run: |
|
|
echo "versions=$(trusted/ci/supportedVersions.nix --arg pinnedJson untrusted/ci/pinned.json)" >> "$GITHUB_OUTPUT"
|
|
|
|
eval:
|
|
runs-on: ubuntu-24.04-arm
|
|
needs: versions
|
|
if: ${{ !cancelled() && !failure() }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
system: ${{ fromJSON(inputs.systems) }}
|
|
version:
|
|
- "" # Default Eval triggering rebuild labels and such.
|
|
- ${{ fromJSON(needs.versions.outputs.versions || '[]') }} # Only for ci/pinned.json updates.
|
|
# Failures for versioned Evals will be collected in a separate job below
|
|
# to not interrupt main Eval's compare step.
|
|
continue-on-error: ${{ matrix.version != '' }}
|
|
name: ${{ matrix.system }}${{ matrix.version && format(' @ {0}', matrix.version) || '' }}
|
|
timeout-minutes: 15
|
|
steps:
|
|
# This is not supposed to be used and just acts as a fallback.
|
|
# Without swap, when Eval runs OOM, it will fail badly with a
|
|
# job that is sometimes not interruptible anymore.
|
|
# If Eval starts swapping, decrease chunkSize to keep it fast.
|
|
- name: Enable swap
|
|
run: |
|
|
sudo fallocate -l 10G /swap
|
|
sudo chmod 600 /swap
|
|
sudo mkswap /swap
|
|
sudo swapon /swap
|
|
|
|
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
|
with:
|
|
persist-credentials: false
|
|
sparse-checkout: .github/actions
|
|
- name: Check out the PR at merged and target commits
|
|
uses: ./.github/actions/checkout
|
|
with:
|
|
merged-as-untrusted-at: ${{ inputs.mergedSha }}
|
|
target-as-trusted-at: ${{ inputs.targetSha }}
|
|
|
|
- name: Install Nix
|
|
uses: cachix/install-nix-action@456688f15bc354bef6d396e4a35f4f89d40bf2b7 # v31
|
|
|
|
- uses: cachix/cachix-action@0fc020193b5a1fa3ac4575aa3a7d3aa6a35435ad # v16
|
|
with:
|
|
# The nixpkgs-ci cache should not be trusted or used outside of Nixpkgs and its forks' CI.
|
|
name: ${{ vars.CACHIX_NAME || 'nixpkgs-ci' }}
|
|
extraPullNames: nixpkgs-ci
|
|
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
|
|
pushFilter: '(-source|-single-chunk)$'
|
|
|
|
- name: Evaluate the ${{ matrix.system }} output paths at the merge commit
|
|
env:
|
|
MATRIX_SYSTEM: ${{ matrix.system }}
|
|
MATRIX_VERSION: ${{ matrix.version || 'nixVersions.latest' }}
|
|
run: |
|
|
nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A eval.singleSystem \
|
|
--argstr evalSystem "$MATRIX_SYSTEM" \
|
|
--arg chunkSize 8000 \
|
|
--argstr nixPath "$MATRIX_VERSION" \
|
|
--out-link merged
|
|
# If it uses too much memory, slightly decrease chunkSize.
|
|
# Note: Keep the same further down in sync!
|
|
|
|
- name: Evaluate the ${{ matrix.system }} output paths at the target commit
|
|
env:
|
|
MATRIX_SYSTEM: ${{ matrix.system }}
|
|
run: |
|
|
TARGET_DRV=$(nix-instantiate nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.singleSystem \
|
|
--argstr evalSystem "$MATRIX_SYSTEM" \
|
|
--arg chunkSize 8000 \
|
|
--argstr nixPath "nixVersions.latest")
|
|
|
|
# Try to fetch this from Cachix a few times, for up to 30 seconds. This avoids running Eval
|
|
# twice in the Merge Queue, when a later item finishes Eval at the merge commit earlier.
|
|
for _i in {1..6}; do
|
|
# Using --max-jobs 0 will cause nix-build to fail if this can't be substituted from cachix.
|
|
if nix-build "$TARGET_DRV" --max-jobs 0; then
|
|
break
|
|
fi
|
|
sleep 5
|
|
done
|
|
|
|
# Either fetches from Cachix or runs Eval itself. The fallback is required
|
|
# for pull requests into wip-branches without merge queue.
|
|
nix-build "$TARGET_DRV" --out-link target
|
|
|
|
- name: Compare outpaths against the target branch
|
|
env:
|
|
MATRIX_SYSTEM: ${{ matrix.system }}
|
|
run: |
|
|
nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A eval.diff \
|
|
--arg beforeDir ./target \
|
|
--arg afterDir ./merged \
|
|
--argstr evalSystem "$MATRIX_SYSTEM" \
|
|
--out-link diff
|
|
|
|
- name: Upload outpaths diff and stats
|
|
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
|
|
with:
|
|
name: ${{ inputs.artifact-prefix }}${{ matrix.version && format('{0}-', matrix.version) || '' }}diff-${{ matrix.system }}
|
|
path: diff/*
|
|
|
|
compare:
|
|
runs-on: ubuntu-24.04-arm
|
|
needs: [eval]
|
|
if: ${{ !cancelled() && !failure() }}
|
|
permissions:
|
|
statuses: write
|
|
timeout-minutes: 5
|
|
steps:
|
|
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
|
with:
|
|
persist-credentials: false
|
|
sparse-checkout: .github/actions
|
|
- name: Check out the PR at the target commit
|
|
uses: ./.github/actions/checkout
|
|
with:
|
|
merged-as-untrusted-at: ${{ inputs.mergedSha }}
|
|
target-as-trusted-at: ${{ inputs.targetSha }}
|
|
|
|
- name: Download output paths and eval stats for all systems
|
|
uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
|
|
with:
|
|
pattern: ${{ inputs.artifact-prefix }}diff-*
|
|
path: diff
|
|
merge-multiple: true
|
|
|
|
- name: Install Nix
|
|
uses: cachix/install-nix-action@456688f15bc354bef6d396e4a35f4f89d40bf2b7 # v31
|
|
|
|
- name: Combine all output paths and eval stats
|
|
run: |
|
|
nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.combine \
|
|
--arg diffDir ./diff \
|
|
--out-link combined
|
|
|
|
- name: Upload the maintainer list
|
|
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
|
|
with:
|
|
name: ${{ inputs.artifact-prefix }}maintainers
|
|
path: combined/maintainers.json
|
|
|
|
- name: Compare against the target branch
|
|
env:
|
|
TARGET_SHA: ${{ inputs.mergedSha }}
|
|
run: |
|
|
git -C nixpkgs/trusted diff --name-only "$TARGET_SHA" \
|
|
| jq --raw-input --slurp 'split("\n")[:-1]' > touched-files.json
|
|
|
|
# Use the target branch to get accurate maintainer info
|
|
nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.compare \
|
|
--arg combinedDir ./combined \
|
|
--arg touchedFilesJson ./touched-files.json \
|
|
--out-link comparison
|
|
|
|
cat comparison/step-summary.md >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
- name: Upload the comparison results
|
|
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
|
|
with:
|
|
name: ${{ inputs.artifact-prefix }}comparison
|
|
path: comparison/*
|
|
|
|
- name: Add eval summary to commit statuses
|
|
if: ${{ github.event_name == 'pull_request_target' }}
|
|
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
|
|
with:
|
|
script: |
|
|
const { readFile } = require('node:fs/promises')
|
|
const changed = JSON.parse(await readFile('comparison/changed-paths.json', 'utf-8'))
|
|
const description =
|
|
'Package: ' + [
|
|
`added ${changed.attrdiff.added.length}`,
|
|
`removed ${changed.attrdiff.removed.length}`,
|
|
`changed ${changed.attrdiff.changed.length}`
|
|
].join(', ') +
|
|
' — Rebuild: ' + [
|
|
`linux ${changed.rebuildCountByKernel.linux}`,
|
|
`darwin ${changed.rebuildCountByKernel.darwin}`
|
|
].join(', ')
|
|
|
|
const { serverUrl, repo, runId, payload } = context
|
|
const target_url =
|
|
`${serverUrl}/${repo.owner}/${repo.repo}/actions/runs/${runId}?pr=${payload.pull_request.number}`
|
|
|
|
await github.rest.repos.createCommitStatus({
|
|
...repo,
|
|
sha: payload.pull_request.head.sha,
|
|
context: 'Eval Summary',
|
|
state: 'success',
|
|
description,
|
|
target_url
|
|
})
|
|
|
|
# Creates a matrix of Eval performance for various versions and systems.
|
|
report:
|
|
runs-on: ubuntu-24.04-arm
|
|
needs: [versions, eval]
|
|
steps:
|
|
- name: Download output paths and eval stats for all versions
|
|
uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
|
|
with:
|
|
pattern: "*-diff-*"
|
|
path: versions
|
|
|
|
- name: Add version comparison table to job summary
|
|
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
|
|
env:
|
|
ARTIFACT_PREFIX: ${{ inputs.artifact-prefix }}
|
|
SYSTEMS: ${{ inputs.systems }}
|
|
VERSIONS: ${{ needs.versions.outputs.versions }}
|
|
with:
|
|
script: |
|
|
const { readFileSync } = require('node:fs')
|
|
const path = require('node:path')
|
|
|
|
const prefix = process.env.ARTIFACT_PREFIX
|
|
const systems = JSON.parse(process.env.SYSTEMS)
|
|
const versions = JSON.parse(process.env.VERSIONS)
|
|
|
|
core.summary.addHeading('Lix/Nix version comparison')
|
|
core.summary.addTable(
|
|
[].concat(
|
|
[
|
|
[{ data: 'Version', header: true }].concat(
|
|
systems.map((system) => ({ data: system, header: true })),
|
|
),
|
|
],
|
|
versions.map((version) =>
|
|
[{ data: version }].concat(
|
|
systems.map((system) => {
|
|
try {
|
|
const artifact = path.join('versions', `${prefix}${version}-diff-${system}`)
|
|
const time = Math.round(
|
|
parseFloat(
|
|
readFileSync(
|
|
path.join(artifact, 'after', system, 'total-time'),
|
|
'utf-8',
|
|
),
|
|
),
|
|
)
|
|
const diff = JSON.parse(
|
|
readFileSync(path.join(artifact, system, 'diff.json'), 'utf-8'),
|
|
)
|
|
const attrs = []
|
|
.concat(diff.added, diff.removed, diff.changed, diff.rebuilds)
|
|
// There are some special attributes, which are ignored for rebuilds.
|
|
// These only have a single path component, because they lack the `.<system>` suffix.
|
|
.filter((attr) => attr.split('.').length > 1)
|
|
if (attrs.length > 0) {
|
|
core.setFailed(
|
|
`${version} on ${system} has changed outpaths!\nNote: Please make sure to update ci/pinned.json separately from changes to other packages.`,
|
|
)
|
|
return { data: ':x:' }
|
|
}
|
|
return { data: time }
|
|
} catch {
|
|
core.warning(`${version} on ${system} did not produce artifact.`)
|
|
return { data: ':warning:' }
|
|
}
|
|
}),
|
|
),
|
|
),
|
|
),
|
|
)
|
|
core.summary.addRaw(
|
|
'\n*Evaluation time in seconds without downloading dependencies.*',
|
|
true,
|
|
)
|
|
core.summary.addRaw('\n*:warning: Job did not report a result.*', true)
|
|
core.summary.addRaw(
|
|
'\n*:x: Job produced different outpaths than the target branch.*',
|
|
true,
|
|
)
|
|
core.summary.write()
|
|
|
|
misc:
|
|
if: ${{ github.event_name != 'push' }}
|
|
runs-on: ubuntu-24.04-arm
|
|
timeout-minutes: 10
|
|
steps:
|
|
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
|
with:
|
|
persist-credentials: false
|
|
sparse-checkout: .github/actions
|
|
- name: Checkout the merge commit
|
|
uses: ./.github/actions/checkout
|
|
with:
|
|
merged-as-untrusted-at: ${{ inputs.mergedSha }}
|
|
|
|
- name: Install Nix
|
|
uses: cachix/install-nix-action@456688f15bc354bef6d396e4a35f4f89d40bf2b7 # v31
|
|
|
|
- name: Run misc eval tasks in parallel
|
|
run: |
|
|
# Ensure flake outputs on all systems still evaluate
|
|
nix flake check --all-systems --no-build './nixpkgs/untrusted?shallow=1' &
|
|
# Query nixpkgs with aliases enabled to check for basic syntax errors
|
|
nix-env -I ./nixpkgs/untrusted -f ./nixpkgs/untrusted -qa '*' --option restrict-eval true --option allow-import-from-derivation false >/dev/null &
|
|
wait
|