NixOS test framework: add overriding methods (#421480)

nixos/doc/manual/development/writing-nixos-tests.section.md

@@ -275,6 +275,62 @@ added using the parameter `extraPythonPackages`. For example, you could add
 
 In that case, `numpy` is chosen from the generic `python3Packages`.
 
+## Overriding a test {#sec-override-nixos-test}
+
+The NixOS test framework returns tests with multiple overriding methods.
+
+`overrideTestDerivation` *function*
+:   Like applying `overrideAttrs` on the [test](#test-opt-test) derivation.
+
+    This is a convenience for `extend` with an override on the [`rawTestDerivationArg`](#test-opt-rawTestDerivationArg) option.
+
+    *function*
+    :   An extension function, e.g. `finalAttrs: prevAttrs: { /* … */ }`, the result of which is passed to [`mkDerivation`](https://nixos.org/manual/nixpkgs/stable/#sec-using-stdenv).
+        Just as with `overrideAttrs`, an abbreviated form can be used, e.g. `prevAttrs: { /* … */ }` or even `{ /* … */ }`.
+        See [`lib.extends`](https://nixos.org/manual/nixpkgs/stable/#function-library-lib.fixedPoints.extends).
+
+`extendNixOS { module = ` *module* `; specialArgs = ` *specialArgs* `; }`
+:   Evaluates the test with additional NixOS modules and/or arguments.
+
+    `module`
+    :   A NixOS module to add to all the nodes in the test. Sets test option [`extraBaseModules`](#test-opt-extraBaseModules).
+
+    `specialArgs`
+    :   An attribute set of arguments to pass to all NixOS modules. These override the existing arguments, as well as any `_module.args.<name>` that the modules may define. Sets test option [`node.specialArgs`](#test-opt-node.specialArgs).
+
+    This is a convenience function for `extend` that overrides the aforementioned test options.
+
+    :::{.example #ex-nixos-test-extendNixOS}
+
+    # Using extendNixOS in `passthru.tests` to make `(openssh.tests.overrideAttrs f).tests.nixos` coherent
+
+    ```nix
+    mkDerivation (finalAttrs: {
+      # …
+      passthru = {
+        tests = {
+          nixos = nixosTests.openssh.extendNixOS {
+            module = {
+              services.openssh.package = finalAttrs.finalPackage;
+            };
+          };
+        };
+      };
+    })
+    ```
+    :::
+
+`extend { modules = ` *modules* `; specialArgs = ` *specialArgs* `; }`
+:   Adds new `nixosTest` modules and/or module arguments to the test, which are evaluated together with the existing modules and [built-in options](#sec-test-options-reference).
+
+    If you're only looking to extend the _NixOS_ configurations of the test, and not something else about the test, you may use the `extendNixOS` convenience function instead.
+
+    `modules`
+    :   A list of modules to add to the test. These are added to the existing modules and then [evaluated](https://nixos.org/manual/nixpkgs/stable/index.html#module-system-lib-evalModules) together.
+
+    `specialArgs`
+    :   An attribute of arguments to pass to the test. These override the existing arguments, as well as any `_module.args.<name>` that the modules may define. See [`evalModules`/`specialArgs`](https://nixos.org/manual/nixpkgs/stable/#module-system-lib-evalModules-param-specialArgs).
+
 ## Test Options Reference {#sec-test-options-reference}
 
 The following options can be used when writing tests.

nixos/doc/manual/redirects.json

@@ -1,4 +1,13 @@
 {
+  "sec-override-nixos-test": [
+    "index.html#sec-override-nixos-test"
+  ],
+  "test-opt-rawTestDerivationArg": [
+    "index.html#test-opt-rawTestDerivationArg"
+  ],
+  "ex-nixos-test-extendNixOS": [
+    "index.html#ex-nixos-test-extendNixOS"
+  ],
   "book-nixos-manual": [
     "index.html#book-nixos-manual"
   ],

nixos/lib/testing/call-test.nix

@@ -1,6 +1,19 @@
-{ config, lib, ... }:
+{
+  config,
+  extendModules,
+  lib,
+  ...
+}:
 let
   inherit (lib) mkOption types;
+
+  unsafeGetAttrPosStringOr =
+    default: name: value:
+    let
+      p = builtins.unsafeGetAttrPos name value;
+    in
+    if p == null then default else p.file + ":" + toString p.line + ":" + toString p.column;
+
 in
 {
   options = {
@@ -9,4 +22,21 @@ in
       default = config;
     };
   };
+  config = {
+    # Docs: nixos/doc/manual/development/writing-nixos-tests.section.md
+    /**
+      See https://nixos.org/manual/nixos/unstable#sec-override-nixos-test
+    */
+    passthru.extend =
+      args@{
+        modules,
+        specialArgs ? { },
+      }:
+      (extendModules {
+        inherit specialArgs;
+        modules = map (lib.setDefaultModuleLocation (
+          unsafeGetAttrPosStringOr "<test.extend module>" "modules" args
+        )) modules;
+      }).config.test;
+  };
 }

nixos/lib/testing/nodes.nix

@@ -3,6 +3,7 @@ testModuleArgs@{
   lib,
   hostPkgs,
   nodes,
+  options,
   ...
 }:
 
@@ -73,6 +74,9 @@ let
     ];
   };
 
+  # TODO (lib): Dedup with run.nix, add to lib/options.nix
+  mkOneUp = opt: f: lib.mkOverride (opt.highestPrio - 1) (f opt.value);
+
 in
 
 {
@@ -233,5 +237,23 @@ in
       ))
     ];
 
+    # Docs: nixos/doc/manual/development/writing-nixos-tests.section.md
+    /**
+      See https://nixos.org/manual/nixos/unstable#sec-override-nixos-test
+    */
+    passthru.extendNixOS =
+      {
+        module,
+        specialArgs ? { },
+      }:
+      config.passthru.extend {
+        modules = [
+          {
+            extraBaseModules = module;
+            node.specialArgs = mkOneUp options.node.specialArgs (_: specialArgs);
+          }
+        ];
+      };
+
   };
 }

nixos/lib/testing/run.nix

@@ -2,10 +2,31 @@
   config,
   hostPkgs,
   lib,
+  options,
   ...
 }:
 let
   inherit (lib) types mkOption;
+
+  # TODO (lib): Also use lib equivalent in nodes.nix
+  /**
+    Create a module system definition that overrides an existing option from a different module evaluation.
+
+    Type: Option a -> (a -> a) -> Definition a
+  */
+  mkOneUp =
+    /**
+      Option from an existing module evaluation, e.g.
+      - `(lib.evalModules ...).options.x` when invoking `evalModules` again,
+      - or `{ options, ... }:` when invoking `extendModules`.
+    */
+    opt:
+    /**
+      Function from the old value to the new definition, which will be wrapped with `mkOverride`.
+    */
+    f:
+    lib.mkOverride (opt.highestPrio - 1) (f opt.value);
+
 in
 {
   options = {
@@ -30,6 +51,13 @@ in
       internal = true;
     };
 
+    rawTestDerivationArg = mkOption {
+      type = types.functionTo types.raw;
+      description = ''
+        Argument passed to `mkDerivation` to create the `rawTestDerivation`.
+      '';
+    };
+
     test = mkOption {
       type = types.package;
       # TODO: can the interactive driver be configured to access the network?
@@ -43,10 +71,12 @@ in
   };
 
   config = {
-    rawTestDerivation =
+    rawTestDerivation = hostPkgs.stdenv.mkDerivation config.rawTestDerivationArg;
+    rawTestDerivationArg =
+      finalAttrs:
       assert lib.assertMsg (!config.sshBackdoor.enable)
         "The SSH backdoor is currently not supported for non-interactive testing! Please make sure to only set `interactive.sshBackdoor.enable = true;`!";
-      hostPkgs.stdenv.mkDerivation {
+      {
         name = "vm-test-run-${config.name}";
 
         requiredSystemFeatures =
@@ -75,5 +105,19 @@ in
 
     # useful for inspection (debugging / exploration)
     passthru.config = config;
+
+    # Docs: nixos/doc/manual/development/writing-nixos-tests.section.md
+    /**
+      See https://nixos.org/manual/nixos/unstable#sec-override-nixos-test
+    */
+    passthru.overrideTestDerivation =
+      f:
+      config.passthru.extend {
+        modules = [
+          {
+            rawTestDerivationArg = mkOneUp options.rawTestDerivationArg (lib.extends (lib.toExtension f));
+          }
+        ];
+      };
   };
 }

pkgs/build-support/testers/test/default.nix

@@ -30,6 +30,24 @@ let
         __structuredAttrs = enable;
       });
     });
+  runNixOSTest-example = pkgs-with-overlay.testers.runNixOSTest (
+    { lib, ... }:
+    {
+      name = "runNixOSTest-test";
+      nodes.machine =
+        { pkgs, ... }:
+        {
+          system.nixos = dummyVersioning;
+          environment.systemPackages = [
+            pkgs.proof-of-overlay-hello
+            pkgs.figlet
+          ];
+        };
+      testScript = ''
+        machine.succeed("hello | figlet >/dev/console")
+      '';
+    }
+  );
 
 in
 lib.recurseIntoAttrs {
@@ -66,24 +84,27 @@ lib.recurseIntoAttrs {
     };
   };
 
-  runNixOSTest-example = pkgs-with-overlay.testers.runNixOSTest (
+  inherit runNixOSTest-example;
-    { lib, ... }:
+
-    {
+  runNixOSTest-extendNixOS =
-      name = "runNixOSTest-test";
+    let
-      nodes.machine =
+      t = runNixOSTest-example.extendNixOS {
-        { pkgs, ... }:
+        module =
-        {
+          { hi, lib, ... }:
-          system.nixos = dummyVersioning;
+          {
-          environment.systemPackages = [
+            config = {
-            pkgs.proof-of-overlay-hello
+              assertions = [ { assertion = hi; } ];
-            pkgs.figlet
+            };
-          ];
+            options = {
-        };
+              itsProofYay = lib.mkOption { };
-      testScript = ''
+            };
-        machine.succeed("hello | figlet >/dev/console")
+          };
-      '';
+        specialArgs.hi = true;
-    }
+      };
-  );
+    in
+    assert lib.isDerivation t;
+    assert t.nodes.machine ? itsProofYay;
+    t;
 
   # Check that the wiring of nixosTest is correct.
   # Correct operation of the NixOS test driver should be asserted elsewhere.