workflows/eval: test all available versions

With this change, we start running Eval on all available Lix and Nix
versions. Because this requires a lot of resources, this complete test
is only run when `ci/pinned.json` is updated.

The resulting outpaths are checked for consistency with the target
branch. A difference will cause the `report` job to fail, thus blocking
the merge, ensuring Eval consistency for Nixpkgs across different
versions.

This implements a kind of "ratchet style" check: Since we originally
confirmed that the versions currently in Nixpkgs at the time of this
commit match Eval behavior of Nix 2.3, we can ensure consistency with
Nix 2.3 down the road, even without testing for it explicitly.

There had been one regression in Eval consistency for Nix between 2.18
and 2.24 - two tests in `tests.devShellTools` produce different results
between Lix 2.91+ (which was forked from Nix 2.18) and Nix 2.24+. I
assume it's unlikely that such a change would be "fixed" by now, thus I
added an exception for these.

As a bonus, we also present the total time in seconds it takes for Eval
to complete for every tested version in a summary table. This allows us
to easily see performance improvements for Eval due to version updates.
At this stage, this time only includes the "outpaths" step of Eval, but
not the generation of attrpaths beforehand.

.github/workflows/eval.yml

@@ -11,6 +11,10 @@ on:
       systems:
         required: true
         type: string
+      testVersions:
+        required: false
+        default: false
+        type: boolean
     secrets:
       OWNER_APP_PRIVATE_KEY:
         required: false
@@ -22,13 +26,49 @@ defaults:
     shell: bash
 
 jobs:
+  versions:
+    if: inputs.testVersions
+    runs-on: ubuntu-24.04-arm
+    outputs:
+      versions: ${{ steps.versions.outputs.versions }}
+    steps:
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          path: trusted
+          sparse-checkout: |
+            ci/supportedVersions.nix
+
+      - name: Check out the PR at the test merge commit
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          ref: ${{ inputs.mergedSha }}
+          path: untrusted
+          sparse-checkout: |
+            ci/pinned.json
+
+      - name: Install Nix
+        uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31
+
+      - name: Load supported versions
+        id: versions
+        run: |
+          echo "versions=$(trusted/ci/supportedVersions.nix --arg pinnedJson untrusted/ci/pinned.json)" >> "$GITHUB_OUTPUT"
+
   eval:
     runs-on: ubuntu-24.04-arm
+    needs: versions
+    if: ${{ !cancelled() }}
     strategy:
       fail-fast: false
       matrix:
         system: ${{ fromJSON(inputs.systems) }}
-    name: ${{ matrix.system }}
+        version:
+          - "" # Default Eval triggering rebuild labels and such.
+          - ${{ fromJSON(needs.versions.outputs.versions || '[]') }} # Only for ci/pinned.json updates.
+    # Failures for versioned Evals will be collected in a separate job below
+    # to not interrupt main Eval's compare step.
+    continue-on-error: ${{ matrix.version != '' }}
+    name: ${{ matrix.system }}${{ matrix.version && format(' @ {0}', matrix.version) || '' }}
     outputs:
       targetRunId: ${{ steps.targetRunId.outputs.targetRunId }}
     timeout-minutes: 15
@@ -60,17 +100,19 @@ jobs:
       - name: Evaluate the ${{ matrix.system }} output paths for all derivation attributes
         env:
           MATRIX_SYSTEM: ${{ matrix.system }}
+          MATRIX_VERSION: ${{ matrix.version || 'nixVersions.latest' }}
         run: |
           nix-build untrusted/ci --arg nixpkgs ./pinned  -A eval.singleSystem \
             --argstr evalSystem "$MATRIX_SYSTEM" \
             --arg chunkSize 8000 \
+            --argstr nixPath "$MATRIX_VERSION" \
             --out-link merged
           # If it uses too much memory, slightly decrease chunkSize
 
       - name: Upload the output paths and eval stats
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
-          name: merged-${{ matrix.system }}
+          name: ${{ matrix.version && format('{0}-', matrix.version) || '' }}merged-${{ matrix.system }}
           path: merged/*
 
       - name: Log current API rate limits
@@ -149,7 +191,7 @@ jobs:
         if: steps.targetRunId.outputs.targetRunId
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
-          name: diff-${{ matrix.system }}
+          name: ${{ matrix.version && format('{0}-', matrix.version) || '' }}diff-${{ matrix.system }}
           path: diff/*
 
   compare:
@@ -240,6 +282,91 @@ jobs:
               target_url
             })
 
+  # Creates a matrix of Eval performance for various versions and systems.
+  report:
+    runs-on: ubuntu-24.04-arm
+    needs: [versions, eval]
+    steps:
+      - name: Download output paths and eval stats for all versions
+        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
+        with:
+          pattern: "*-diff-*"
+          path: versions
+
+      - name: Add version comparison table to job summary
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        env:
+          SYSTEMS: ${{ inputs.systems }}
+          VERSIONS: ${{ needs.versions.outputs.versions }}
+        with:
+          script: |
+            const { readFileSync } = require('node:fs')
+            const path = require('node:path')
+
+            const systems = JSON.parse(process.env.SYSTEMS)
+            const versions = JSON.parse(process.env.VERSIONS)
+
+            core.summary.addHeading('Lix/Nix version comparison')
+            core.summary.addTable(
+              [].concat(
+                [
+                  [{ data: 'Version', header: true }].concat(
+                    systems.map((system) => ({ data: system, header: true })),
+                  ),
+                ],
+                versions.map((version) =>
+                  [{ data: version }].concat(
+                    systems.map((system) => {
+                      try {
+                        const artifact = path.join('versions', `${version}-diff-${system}`)
+                        const time = Math.round(
+                          parseFloat(
+                            readFileSync(
+                              path.join(artifact, 'after', system, 'total-time'),
+                              'utf-8',
+                            ),
+                          ),
+                        )
+                        const diff = JSON.parse(
+                          readFileSync(path.join(artifact, system, 'diff.json'), 'utf-8'),
+                        )
+                        const attrs = [].concat(
+                          diff.added,
+                          diff.removed,
+                          diff.changed,
+                          diff.rebuilds
+                        ).filter(attr =>
+                          // Exceptions related to dev shells, which changed at some time between 2.18 and 2.24.
+                          !attr.startsWith('tests.devShellTools.nixos.') &&
+                          !attr.startsWith('tests.devShellTools.unstructuredDerivationInputEnv.')
+                        )
+                        if (attrs.length > 0) {
+                          core.setFailed(
+                            `${version} on ${system} has changed outpaths!\nNote: Please make sure to update ci/pinned.json separately from changes to other packages.`,
+                          )
+                          return { data: ':x:' }
+                        }
+                        return { data: time }
+                      } catch {
+                        core.warning(`${version} on ${system} did not produce artifact.`)
+                        return { data: ':warning:' }
+                      }
+                    }),
+                  ),
+                ),
+              ),
+            )
+            core.summary.addRaw(
+              '\n*Evaluation time in seconds without downloading dependencies.*',
+              true,
+            )
+            core.summary.addRaw('\n*:warning: Job did not report a result.*', true)
+            core.summary.addRaw(
+              '\n*:x: Job produced different outpaths than the target branch.*',
+              true,
+            )
+            core.summary.write()
+
   misc:
     if: ${{ github.event_name != 'push' }}
     runs-on: ubuntu-24.04-arm

.github/workflows/pr.yml

@@ -28,6 +28,7 @@ jobs:
       mergedSha: ${{ steps.get-merge-commit.outputs.mergedSha }}
       targetSha: ${{ steps.get-merge-commit.outputs.targetSha }}
       systems: ${{ steps.systems.outputs.systems }}
+      touched: ${{ steps.files.outputs.touched }}
     steps:
       - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
@@ -64,6 +65,20 @@ jobs:
             core.setOutput('head', headClassification)
             core.info('head classification:', headClassification)
 
+      - name: Determine changed files
+        id: files
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        with:
+          script: |
+            const files = (await github.paginate(github.rest.pulls.listFiles, {
+              ...context.repo,
+              pull_number: context.payload.pull_request.number,
+              per_page: 100,
+            })).map(file => file.filename)
+
+            if (files.includes('ci/pinned.json')) core.setOutput('touched', ['pinned'])
+            else core.setOutput('touched', [])
+
   check:
     name: Check
     needs: [prepare]
@@ -96,6 +111,7 @@ jobs:
       mergedSha: ${{ needs.prepare.outputs.mergedSha }}
       targetSha: ${{ needs.prepare.outputs.targetSha }}
       systems: ${{ needs.prepare.outputs.systems }}
+      testVersions: ${{ contains(fromJSON(needs.prepare.outputs.touched), 'pinned') && !contains(fromJSON(needs.prepare.outputs.headBranch).type, 'development') }}
 
   labels:
     name: Labels

ci/default.nix

@@ -5,6 +5,7 @@ in
   system ? builtins.currentSystem,
 
   nixpkgs ? null,
+  nixPath ? "nixVersions.latest",
 }:
 let
   nixpkgs' =
@@ -115,7 +116,7 @@ rec {
   # (nixVersions.stable and Lix) here somehow at some point to ensure we don't
   # have eval divergence.
   eval = pkgs.callPackage ./eval {
-    nix = pkgs.nixVersions.latest;
+    nix = pkgs.lib.getAttrFromPath (pkgs.lib.splitString "." nixPath) pkgs;
   };
 
   # CI jobs

ci/supportedVersions.nix

@@ -0,0 +1,32 @@
+#!/usr/bin/env -S nix-instantiate --eval --strict --json --arg unused true
+# Unused argument to trigger nix-instantiate calling this function with the default arguments.
+{
+  pinnedJson ? ./pinned.json,
+}:
+let
+  pinned = (builtins.fromJSON (builtins.readFile pinnedJson)).pins;
+  nixpkgs = fetchTarball {
+    inherit (pinned.nixpkgs) url;
+    sha256 = pinned.nixpkgs.hash;
+  };
+  pkgs = import nixpkgs {
+    config.allowAliases = false;
+  };
+
+  inherit (pkgs) lib;
+
+  lix = lib.pipe pkgs.lixPackageSets [
+    (lib.filterAttrs (_: set: lib.isDerivation set.lix or null && set.lix.meta.available))
+    lib.attrNames
+    (lib.filter (name: lib.match "lix_[0-9_]+|git" name != null))
+    (map (name: "lixPackageSets.${name}.lix"))
+  ];
+
+  nix = lib.pipe pkgs.nixVersions [
+    (lib.filterAttrs (_: drv: lib.isDerivation drv && drv.meta.available))
+    lib.attrNames
+    (lib.filter (name: lib.match "nix_[0-9_]+|git" name != null))
+    (map (name: "nixVersions.${name}"))
+  ];
+in
+lix ++ nix