267d5cdf64
The conclusion of the `versions` job propagates from through `eval` to `compare`, which meant the `compare` job was skipped. No rebuild labels, no reviewer requests. Also, we don't want to run eval when `versions` runs, but fails.
392 lines
15 KiB
YAML
392 lines
15 KiB
YAML
name: Eval
|
|
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
mergedSha:
|
|
required: true
|
|
type: string
|
|
targetSha:
|
|
type: string
|
|
systems:
|
|
required: true
|
|
type: string
|
|
testVersions:
|
|
required: false
|
|
default: false
|
|
type: boolean
|
|
secrets:
|
|
OWNER_APP_PRIVATE_KEY:
|
|
required: false
|
|
|
|
permissions: {}
|
|
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
|
|
jobs:
|
|
versions:
|
|
if: inputs.testVersions
|
|
runs-on: ubuntu-24.04-arm
|
|
outputs:
|
|
versions: ${{ steps.versions.outputs.versions }}
|
|
steps:
|
|
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
|
with:
|
|
path: trusted
|
|
sparse-checkout: |
|
|
ci/supportedVersions.nix
|
|
|
|
- name: Check out the PR at the test merge commit
|
|
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
|
with:
|
|
ref: ${{ inputs.mergedSha }}
|
|
path: untrusted
|
|
sparse-checkout: |
|
|
ci/pinned.json
|
|
|
|
- name: Install Nix
|
|
uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31
|
|
|
|
- name: Load supported versions
|
|
id: versions
|
|
run: |
|
|
echo "versions=$(trusted/ci/supportedVersions.nix --arg pinnedJson untrusted/ci/pinned.json)" >> "$GITHUB_OUTPUT"
|
|
|
|
eval:
|
|
runs-on: ubuntu-24.04-arm
|
|
needs: versions
|
|
if: ${{ !cancelled() && !failure() }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
system: ${{ fromJSON(inputs.systems) }}
|
|
version:
|
|
- "" # Default Eval triggering rebuild labels and such.
|
|
- ${{ fromJSON(needs.versions.outputs.versions || '[]') }} # Only for ci/pinned.json updates.
|
|
# Failures for versioned Evals will be collected in a separate job below
|
|
# to not interrupt main Eval's compare step.
|
|
continue-on-error: ${{ matrix.version != '' }}
|
|
name: ${{ matrix.system }}${{ matrix.version && format(' @ {0}', matrix.version) || '' }}
|
|
outputs:
|
|
targetRunId: ${{ steps.targetRunId.outputs.targetRunId }}
|
|
timeout-minutes: 15
|
|
steps:
|
|
# This is not supposed to be used and just acts as a fallback.
|
|
# Without swap, when Eval runs OOM, it will fail badly with a
|
|
# job that is sometimes not interruptible anymore.
|
|
# If Eval starts swapping, decrease chunkSize to keep it fast.
|
|
- name: Enable swap
|
|
run: |
|
|
sudo fallocate -l 10G /swap
|
|
sudo chmod 600 /swap
|
|
sudo mkswap /swap
|
|
sudo swapon /swap
|
|
|
|
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
|
with:
|
|
sparse-checkout: .github/actions
|
|
- name: Check out the PR at the test merge commit
|
|
uses: ./.github/actions/get-merge-commit
|
|
with:
|
|
mergedSha: ${{ inputs.mergedSha }}
|
|
merged-as-untrusted: true
|
|
pinnedFrom: untrusted
|
|
|
|
- name: Install Nix
|
|
uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31
|
|
|
|
- name: Evaluate the ${{ matrix.system }} output paths for all derivation attributes
|
|
env:
|
|
MATRIX_SYSTEM: ${{ matrix.system }}
|
|
MATRIX_VERSION: ${{ matrix.version || 'nixVersions.latest' }}
|
|
run: |
|
|
nix-build untrusted/ci --arg nixpkgs ./pinned -A eval.singleSystem \
|
|
--argstr evalSystem "$MATRIX_SYSTEM" \
|
|
--arg chunkSize 8000 \
|
|
--argstr nixPath "$MATRIX_VERSION" \
|
|
--out-link merged
|
|
# If it uses too much memory, slightly decrease chunkSize
|
|
|
|
- name: Upload the output paths and eval stats
|
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
|
with:
|
|
name: ${{ matrix.version && format('{0}-', matrix.version) || '' }}merged-${{ matrix.system }}
|
|
path: merged/*
|
|
|
|
- name: Log current API rate limits
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
run: gh api /rate_limit | jq
|
|
|
|
- name: Get target run id
|
|
if: inputs.targetSha
|
|
id: targetRunId
|
|
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
|
env:
|
|
MATRIX_SYSTEM: ${{ matrix.system }}
|
|
TARGET_SHA: ${{ inputs.targetSha }}
|
|
with:
|
|
script: |
|
|
const system = process.env.MATRIX_SYSTEM
|
|
const targetSha = process.env.TARGET_SHA
|
|
|
|
let run_id
|
|
try {
|
|
run_id = (await github.rest.actions.listWorkflowRuns({
|
|
...context.repo,
|
|
workflow_id: 'push.yml',
|
|
event: 'push',
|
|
head_sha: targetSha
|
|
})).data.workflow_runs[0].id
|
|
} catch {
|
|
throw new Error(`Could not find a push.yml workflow run for ${targetSha}.`)
|
|
}
|
|
|
|
// Waiting 120 * 5 sec = 10 min. max.
|
|
// Eval takes max 5-6 minutes, normally.
|
|
for (let i = 0; i < 120; i++) {
|
|
const result = await github.rest.actions.listWorkflowRunArtifacts({
|
|
...context.repo,
|
|
run_id,
|
|
name: `merged-${system}`
|
|
})
|
|
if (result.data.total_count > 0) {
|
|
core.setOutput('targetRunId', run_id)
|
|
return
|
|
}
|
|
await new Promise(resolve => setTimeout(resolve, 5000))
|
|
}
|
|
// No artifact found at this stage. This usually means that Eval failed on the target branch.
|
|
// This should only happen when Eval is broken on the target branch and this PR fixes it.
|
|
// Continue without targetRunId to skip the remaining steps, but pass the job.
|
|
|
|
- name: Log current API rate limits
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
run: gh api /rate_limit | jq
|
|
|
|
- uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
|
|
if: steps.targetRunId.outputs.targetRunId
|
|
with:
|
|
run-id: ${{ steps.targetRunId.outputs.targetRunId }}
|
|
name: merged-${{ matrix.system }}
|
|
path: target
|
|
github-token: ${{ github.token }}
|
|
merge-multiple: true
|
|
|
|
- name: Compare outpaths against the target branch
|
|
if: steps.targetRunId.outputs.targetRunId
|
|
env:
|
|
MATRIX_SYSTEM: ${{ matrix.system }}
|
|
run: |
|
|
nix-build untrusted/ci --arg nixpkgs ./pinned -A eval.diff \
|
|
--arg beforeDir ./target \
|
|
--arg afterDir "$(readlink ./merged)" \
|
|
--argstr evalSystem "$MATRIX_SYSTEM" \
|
|
--out-link diff
|
|
|
|
- name: Upload outpaths diff and stats
|
|
if: steps.targetRunId.outputs.targetRunId
|
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
|
with:
|
|
name: ${{ matrix.version && format('{0}-', matrix.version) || '' }}diff-${{ matrix.system }}
|
|
path: diff/*
|
|
|
|
compare:
|
|
runs-on: ubuntu-24.04-arm
|
|
needs: [eval]
|
|
if: needs.eval.outputs.targetRunId && !cancelled() && !failure()
|
|
permissions:
|
|
statuses: write
|
|
timeout-minutes: 5
|
|
steps:
|
|
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
|
with:
|
|
sparse-checkout: .github/actions
|
|
- name: Check out the PR at the target commit
|
|
uses: ./.github/actions/get-merge-commit
|
|
with:
|
|
targetSha: ${{ inputs.targetSha }}
|
|
target-as-trusted: true
|
|
pinnedFrom: trusted
|
|
|
|
- name: Download output paths and eval stats for all systems
|
|
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
|
|
with:
|
|
pattern: diff-*
|
|
path: diff
|
|
merge-multiple: true
|
|
|
|
- name: Install Nix
|
|
uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31
|
|
|
|
- name: Combine all output paths and eval stats
|
|
run: |
|
|
nix-build trusted/ci --arg nixpkgs ./pinned -A eval.combine \
|
|
--arg diffDir ./diff \
|
|
--out-link combined
|
|
|
|
- name: Compare against the target branch
|
|
env:
|
|
AUTHOR_ID: ${{ github.event.pull_request.user.id }}
|
|
run: |
|
|
git -C trusted fetch --depth 1 origin ${{ inputs.mergedSha }}
|
|
git -C trusted diff --name-only ${{ inputs.mergedSha }} \
|
|
| jq --raw-input --slurp 'split("\n")[:-1]' > touched-files.json
|
|
|
|
# Use the target branch to get accurate maintainer info
|
|
nix-build trusted/ci --arg nixpkgs ./pinned -A eval.compare \
|
|
--arg combinedDir "$(realpath ./combined)" \
|
|
--arg touchedFilesJson ./touched-files.json \
|
|
--argstr githubAuthorId "$AUTHOR_ID" \
|
|
--out-link comparison
|
|
|
|
cat comparison/step-summary.md >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
- name: Upload the comparison results
|
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
|
with:
|
|
name: comparison
|
|
path: comparison/*
|
|
|
|
- name: Add eval summary to commit statuses
|
|
if: ${{ github.event_name == 'pull_request_target' }}
|
|
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
|
with:
|
|
script: |
|
|
const { readFile } = require('node:fs/promises')
|
|
const changed = JSON.parse(await readFile('comparison/changed-paths.json', 'utf-8'))
|
|
const description =
|
|
'Package: ' + [
|
|
`added ${changed.attrdiff.added.length}`,
|
|
`removed ${changed.attrdiff.removed.length}`,
|
|
`changed ${changed.attrdiff.changed.length}`
|
|
].join(', ') +
|
|
' — Rebuild: ' + [
|
|
`linux ${changed.rebuildCountByKernel.linux}`,
|
|
`darwin ${changed.rebuildCountByKernel.darwin}`
|
|
].join(', ')
|
|
|
|
const { serverUrl, repo, runId, payload } = context
|
|
const target_url =
|
|
`${serverUrl}/${repo.owner}/${repo.repo}/actions/runs/${runId}?pr=${payload.pull_request.number}`
|
|
|
|
await github.rest.repos.createCommitStatus({
|
|
...repo,
|
|
sha: payload.pull_request.head.sha,
|
|
context: 'Eval Summary',
|
|
state: 'success',
|
|
description,
|
|
target_url
|
|
})
|
|
|
|
# Creates a matrix of Eval performance for various versions and systems.
|
|
report:
|
|
runs-on: ubuntu-24.04-arm
|
|
needs: [versions, eval]
|
|
steps:
|
|
- name: Download output paths and eval stats for all versions
|
|
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
|
|
with:
|
|
pattern: "*-diff-*"
|
|
path: versions
|
|
|
|
- name: Add version comparison table to job summary
|
|
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
|
env:
|
|
SYSTEMS: ${{ inputs.systems }}
|
|
VERSIONS: ${{ needs.versions.outputs.versions }}
|
|
with:
|
|
script: |
|
|
const { readFileSync } = require('node:fs')
|
|
const path = require('node:path')
|
|
|
|
const systems = JSON.parse(process.env.SYSTEMS)
|
|
const versions = JSON.parse(process.env.VERSIONS)
|
|
|
|
core.summary.addHeading('Lix/Nix version comparison')
|
|
core.summary.addTable(
|
|
[].concat(
|
|
[
|
|
[{ data: 'Version', header: true }].concat(
|
|
systems.map((system) => ({ data: system, header: true })),
|
|
),
|
|
],
|
|
versions.map((version) =>
|
|
[{ data: version }].concat(
|
|
systems.map((system) => {
|
|
try {
|
|
const artifact = path.join('versions', `${version}-diff-${system}`)
|
|
const time = Math.round(
|
|
parseFloat(
|
|
readFileSync(
|
|
path.join(artifact, 'after', system, 'total-time'),
|
|
'utf-8',
|
|
),
|
|
),
|
|
)
|
|
const diff = JSON.parse(
|
|
readFileSync(path.join(artifact, system, 'diff.json'), 'utf-8'),
|
|
)
|
|
const attrs = [].concat(
|
|
diff.added,
|
|
diff.removed,
|
|
diff.changed,
|
|
diff.rebuilds
|
|
).filter(attr =>
|
|
// Exceptions related to dev shells, which changed at some time between 2.18 and 2.24.
|
|
!attr.startsWith('tests.devShellTools.nixos.') &&
|
|
!attr.startsWith('tests.devShellTools.unstructuredDerivationInputEnv.')
|
|
)
|
|
if (attrs.length > 0) {
|
|
core.setFailed(
|
|
`${version} on ${system} has changed outpaths!\nNote: Please make sure to update ci/pinned.json separately from changes to other packages.`,
|
|
)
|
|
return { data: ':x:' }
|
|
}
|
|
return { data: time }
|
|
} catch {
|
|
core.warning(`${version} on ${system} did not produce artifact.`)
|
|
return { data: ':warning:' }
|
|
}
|
|
}),
|
|
),
|
|
),
|
|
),
|
|
)
|
|
core.summary.addRaw(
|
|
'\n*Evaluation time in seconds without downloading dependencies.*',
|
|
true,
|
|
)
|
|
core.summary.addRaw('\n*:warning: Job did not report a result.*', true)
|
|
core.summary.addRaw(
|
|
'\n*:x: Job produced different outpaths than the target branch.*',
|
|
true,
|
|
)
|
|
core.summary.write()
|
|
|
|
misc:
|
|
if: ${{ github.event_name != 'push' }}
|
|
runs-on: ubuntu-24.04-arm
|
|
timeout-minutes: 10
|
|
steps:
|
|
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
|
with:
|
|
sparse-checkout: .github/actions
|
|
- name: Check if the PR can be merged and checkout the merge commit
|
|
uses: ./.github/actions/get-merge-commit
|
|
with:
|
|
merged-as-untrusted: true
|
|
|
|
- name: Install Nix
|
|
uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31
|
|
|
|
- name: Ensure flake outputs on all systems still evaluate
|
|
run: nix flake check --all-systems --no-build ./untrusted
|
|
|
|
- name: Query nixpkgs with aliases enabled to check for basic syntax errors
|
|
run: |
|
|
time nix-env -I ./untrusted -f ./untrusted -qa '*' --option restrict-eval true --option allow-import-from-derivation false >/dev/null
|