From b523f257ac01c54d709907fae166c4b82684b382 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Wed, 23 Jul 2025 13:28:44 +0200 Subject: [PATCH] workflows/eval: test all available versions With this change, we start running Eval on all available Lix and Nix versions. Because this requires a lot of resources, this complete test is only run when `ci/pinned.json` is updated. The resulting outpaths are checked for consistency with the target branch. A difference will cause the `report` job to fail, thus blocking the merge, ensuring Eval consistency for Nixpkgs across different versions. This implements a kind of "ratchet style" check: Since we originally confirmed that the versions currently in Nixpkgs at the time of this commit match Eval behavior of Nix 2.3, we can ensure consistency with Nix 2.3 down the road, even without testing for it explicitly. There had been one regression in Eval consistency for Nix between 2.18 and 2.24 - two tests in `tests.devShellTools` produce different results between Lix 2.91+ (which was forked from Nix 2.18) and Nix 2.24+. I assume it's unlikely that such a change would be "fixed" by now, thus I added an exception for these. As a bonus, we also present the total time in seconds it takes for Eval to complete for every tested version in a summary table. This allows us to easily see performance improvements for Eval due to version updates. At this stage, this time only includes the "outpaths" step of Eval, but not the generation of attrpaths beforehand. --- .github/workflows/eval.yml | 133 ++++++++++++++++++++++++++++++++++++- .github/workflows/pr.yml | 16 +++++ ci/default.nix | 3 +- ci/supportedVersions.nix | 32 +++++++++ 4 files changed, 180 insertions(+), 4 deletions(-) create mode 100755 ci/supportedVersions.nix diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index 835fa4c43d99..1ac9c846e6f0 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -11,6 +11,10 @@ on: systems: required: true type: string + testVersions: + required: false + default: false + type: boolean secrets: OWNER_APP_PRIVATE_KEY: required: false @@ -22,13 +26,49 @@ defaults: shell: bash jobs: + versions: + if: inputs.testVersions + runs-on: ubuntu-24.04-arm + outputs: + versions: ${{ steps.versions.outputs.versions }} + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + path: trusted + sparse-checkout: | + ci/supportedVersions.nix + + - name: Check out the PR at the test merge commit + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + ref: ${{ inputs.mergedSha }} + path: untrusted + sparse-checkout: | + ci/pinned.json + + - name: Install Nix + uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31 + + - name: Load supported versions + id: versions + run: | + echo "versions=$(trusted/ci/supportedVersions.nix --arg pinnedJson untrusted/ci/pinned.json)" >> "$GITHUB_OUTPUT" + eval: runs-on: ubuntu-24.04-arm + needs: versions + if: ${{ !cancelled() }} strategy: fail-fast: false matrix: system: ${{ fromJSON(inputs.systems) }} - name: ${{ matrix.system }} + version: + - "" # Default Eval triggering rebuild labels and such. + - ${{ fromJSON(needs.versions.outputs.versions || '[]') }} # Only for ci/pinned.json updates. + # Failures for versioned Evals will be collected in a separate job below + # to not interrupt main Eval's compare step. + continue-on-error: ${{ matrix.version != '' }} + name: ${{ matrix.system }}${{ matrix.version && format(' @ {0}', matrix.version) || '' }} outputs: targetRunId: ${{ steps.targetRunId.outputs.targetRunId }} timeout-minutes: 15 @@ -60,17 +100,19 @@ jobs: - name: Evaluate the ${{ matrix.system }} output paths for all derivation attributes env: MATRIX_SYSTEM: ${{ matrix.system }} + MATRIX_VERSION: ${{ matrix.version || 'nixVersions.latest' }} run: | nix-build untrusted/ci --arg nixpkgs ./pinned -A eval.singleSystem \ --argstr evalSystem "$MATRIX_SYSTEM" \ --arg chunkSize 8000 \ + --argstr nixPath "$MATRIX_VERSION" \ --out-link merged # If it uses too much memory, slightly decrease chunkSize - name: Upload the output paths and eval stats uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: merged-${{ matrix.system }} + name: ${{ matrix.version && format('{0}-', matrix.version) || '' }}merged-${{ matrix.system }} path: merged/* - name: Log current API rate limits @@ -149,7 +191,7 @@ jobs: if: steps.targetRunId.outputs.targetRunId uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: diff-${{ matrix.system }} + name: ${{ matrix.version && format('{0}-', matrix.version) || '' }}diff-${{ matrix.system }} path: diff/* compare: @@ -240,6 +282,91 @@ jobs: target_url }) + # Creates a matrix of Eval performance for various versions and systems. + report: + runs-on: ubuntu-24.04-arm + needs: [versions, eval] + steps: + - name: Download output paths and eval stats for all versions + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + with: + pattern: "*-diff-*" + path: versions + + - name: Add version comparison table to job summary + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + env: + SYSTEMS: ${{ inputs.systems }} + VERSIONS: ${{ needs.versions.outputs.versions }} + with: + script: | + const { readFileSync } = require('node:fs') + const path = require('node:path') + + const systems = JSON.parse(process.env.SYSTEMS) + const versions = JSON.parse(process.env.VERSIONS) + + core.summary.addHeading('Lix/Nix version comparison') + core.summary.addTable( + [].concat( + [ + [{ data: 'Version', header: true }].concat( + systems.map((system) => ({ data: system, header: true })), + ), + ], + versions.map((version) => + [{ data: version }].concat( + systems.map((system) => { + try { + const artifact = path.join('versions', `${version}-diff-${system}`) + const time = Math.round( + parseFloat( + readFileSync( + path.join(artifact, 'after', system, 'total-time'), + 'utf-8', + ), + ), + ) + const diff = JSON.parse( + readFileSync(path.join(artifact, system, 'diff.json'), 'utf-8'), + ) + const attrs = [].concat( + diff.added, + diff.removed, + diff.changed, + diff.rebuilds + ).filter(attr => + // Exceptions related to dev shells, which changed at some time between 2.18 and 2.24. + !attr.startsWith('tests.devShellTools.nixos.') && + !attr.startsWith('tests.devShellTools.unstructuredDerivationInputEnv.') + ) + if (attrs.length > 0) { + core.setFailed( + `${version} on ${system} has changed outpaths!\nNote: Please make sure to update ci/pinned.json separately from changes to other packages.`, + ) + return { data: ':x:' } + } + return { data: time } + } catch { + core.warning(`${version} on ${system} did not produce artifact.`) + return { data: ':warning:' } + } + }), + ), + ), + ), + ) + core.summary.addRaw( + '\n*Evaluation time in seconds without downloading dependencies.*', + true, + ) + core.summary.addRaw('\n*:warning: Job did not report a result.*', true) + core.summary.addRaw( + '\n*:x: Job produced different outpaths than the target branch.*', + true, + ) + core.summary.write() + misc: if: ${{ github.event_name != 'push' }} runs-on: ubuntu-24.04-arm diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 086b14268f5b..db261a904ef5 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -28,6 +28,7 @@ jobs: mergedSha: ${{ steps.get-merge-commit.outputs.mergedSha }} targetSha: ${{ steps.get-merge-commit.outputs.targetSha }} systems: ${{ steps.systems.outputs.systems }} + touched: ${{ steps.files.outputs.touched }} steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: @@ -64,6 +65,20 @@ jobs: core.setOutput('head', headClassification) core.info('head classification:', headClassification) + - name: Determine changed files + id: files + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const files = (await github.paginate(github.rest.pulls.listFiles, { + ...context.repo, + pull_number: context.payload.pull_request.number, + per_page: 100, + })).map(file => file.filename) + + if (files.includes('ci/pinned.json')) core.setOutput('touched', ['pinned']) + else core.setOutput('touched', []) + check: name: Check needs: [prepare] @@ -96,6 +111,7 @@ jobs: mergedSha: ${{ needs.prepare.outputs.mergedSha }} targetSha: ${{ needs.prepare.outputs.targetSha }} systems: ${{ needs.prepare.outputs.systems }} + testVersions: ${{ contains(fromJSON(needs.prepare.outputs.touched), 'pinned') && !contains(fromJSON(needs.prepare.outputs.headBranch).type, 'development') }} labels: name: Labels diff --git a/ci/default.nix b/ci/default.nix index 603e11513aa7..2bae87eca058 100644 --- a/ci/default.nix +++ b/ci/default.nix @@ -5,6 +5,7 @@ in system ? builtins.currentSystem, nixpkgs ? null, + nixPath ? "nixVersions.latest", }: let nixpkgs' = @@ -115,7 +116,7 @@ rec { # (nixVersions.stable and Lix) here somehow at some point to ensure we don't # have eval divergence. eval = pkgs.callPackage ./eval { - nix = pkgs.nixVersions.latest; + nix = pkgs.lib.getAttrFromPath (pkgs.lib.splitString "." nixPath) pkgs; }; # CI jobs diff --git a/ci/supportedVersions.nix b/ci/supportedVersions.nix new file mode 100755 index 000000000000..77f6f7dd1475 --- /dev/null +++ b/ci/supportedVersions.nix @@ -0,0 +1,32 @@ +#!/usr/bin/env -S nix-instantiate --eval --strict --json --arg unused true +# Unused argument to trigger nix-instantiate calling this function with the default arguments. +{ + pinnedJson ? ./pinned.json, +}: +let + pinned = (builtins.fromJSON (builtins.readFile pinnedJson)).pins; + nixpkgs = fetchTarball { + inherit (pinned.nixpkgs) url; + sha256 = pinned.nixpkgs.hash; + }; + pkgs = import nixpkgs { + config.allowAliases = false; + }; + + inherit (pkgs) lib; + + lix = lib.pipe pkgs.lixPackageSets [ + (lib.filterAttrs (_: set: lib.isDerivation set.lix or null && set.lix.meta.available)) + lib.attrNames + (lib.filter (name: lib.match "lix_[0-9_]+|git" name != null)) + (map (name: "lixPackageSets.${name}.lix")) + ]; + + nix = lib.pipe pkgs.nixVersions [ + (lib.filterAttrs (_: drv: lib.isDerivation drv && drv.meta.available)) + lib.attrNames + (lib.filter (name: lib.match "nix_[0-9_]+|git" name != null)) + (map (name: "nixVersions.${name}")) + ]; +in +lix ++ nix