Set up hydra as a remote build machine.

nix/configuration/hosts/hydra/default.nix

@@ -18,10 +18,43 @@
   ];
 
   config = {
+    networking =
+      let
+        interface = "enp0s2";
+      in
+      {
         # Generate with `head -c4 /dev/urandom | od -A none -t x4`
-    networking.hostId = "6fbf418b";
+        hostId = "6fbf418b";
 
-    networking.hostName = "hydra"; # Define your hostname.
+        hostName = "hydra"; # Define your hostname.
+
+        interfaces = {
+          "${interface}" = {
+            ipv4.addresses = [
+              {
+                address = "10.215.1.219";
+                prefixLength = 24;
+              }
+            ];
+
+            ipv6.addresses = [
+              {
+                address = "2620:11f:7001:7:ffff:ffff:0ad7:01db";
+                prefixLength = 64;
+              }
+            ];
+          };
+        };
+        defaultGateway = "10.215.1.1";
+        defaultGateway6 = {
+          # address = "2620:11f:7001:7::1";
+          address = "2620:11f:7001:7:ffff:ffff:0ad7:0101";
+          inherit interface;
+        };
+
+        dhcpcd.enable = lib.mkForce false;
+        useDHCP = lib.mkForce false;
+      };
 
     time.timeZone = "America/New_York";
     i18n.defaultLocale = "en_US.UTF-8";
@@ -63,13 +96,42 @@
 
     environment.systemPackages = with pkgs; [
       htop
+      git # for building on hydra
+      tmux # for building on hydra
+      nix-output-monitor # for building on hydra
     ];
 
     # nix.sshServe.enable = true;
     # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ];
 
+    # Override garbage collection to keep things longer
+    # Automatic garbage collection
+    nix.gc = lib.mkForce {
+      automatic = true;
+      persistent = true;
+      dates = "weekly";
+      # randomizedDelaySec = "14m";
+      options = "--delete-older-than 60d";
+    };
+
+    # The default limit of files is 1024 which is too low for some nix builds.
+    #
+    # Check with `ulimit -n`
+    security.pam.loginLimits = [
+      {
+        domain = "*";
+        item = "nofile";
+        type = "-";
+        value = "8192";
+      }
+    ];
+
+    # systemd.user.extraConfig = "DefaultLimitNOFILE=8192";
+    # systemd.services."user@11400".serviceConfig.LimitNOFILE = "8192";
+
     me.build_in_ram.enable = true;
     me.dont_use_substituters.enable = true;
+    me.hydra.enable = true;
     me.minimal_base.enable = true;
     me.nix_worker.enable = true;
   };

nix/configuration/hosts/hydra/vm_disk.nix

@@ -26,7 +26,7 @@
         neededForBoot = true;
       };
 
-      # "/.disk" = lib.mkForce {
+      # "/.persist" = lib.mkForce {
       #   device = "bind9p";
       #   fsType = "9p";
       #   options = [
@@ -35,6 +35,10 @@
       #     "version=9p2000.L"
       #     "cache=mmap"
       #     "msize=512000"
+      #     "uname=root"
+      #     "dfltuid=0"
+      #     "dfltgid=0"
+      #     "nodevmap"
       #     # "noauto"
       #     # "x-systemd.automount"
       #   ];
@@ -67,25 +71,25 @@
         neededForBoot = true;
       };
 
-      "/nix/store" = lib.mkForce {
+      # "/nix/store" = lib.mkForce {
-        overlay = {
+      #   overlay = {
-          lowerdir = [ "/nix/.ro-store" ];
+      #     lowerdir = [ "/nix/.ro-store" ];
-          upperdir = "/.disk/persist/store";
+      #     upperdir = "/.disk/persist/store";
-          workdir = "/.disk/state/work";
+      #     workdir = "/.disk/state/work";
-        };
+      #   };
-        # fsType = "overlay";
+      #   # fsType = "overlay";
-        # device = "overlay";
+      #   # device = "overlay";
-        # options = [
+      #   # options = [
-        #   "lowerdir=/nix/.ro-store"
+      #   #   "lowerdir=/nix/.ro-store"
-        #   "upperdir=/.disk/persist/store"
+      #   #   "upperdir=/.disk/persist/store"
-        #   "workdir=/.disk/state/work"
+      #   #   "workdir=/.disk/state/work"
+      #   # ];
+      #   depends = [
+      #     "/nix/.ro-store"
+      #     "/.disk/persist/store"
+      #     "/.disk/state/work"
       #   ];
-        depends = [
+      # };
-          "/nix/.ro-store"
-          "/.disk/persist/store"
-          "/.disk/state/work"
-        ];
-      };
     };
   };
 }

nix/configuration/hosts/odo/SELF_BUILD

@@ -5,6 +5,12 @@ IFS=$'\n\t'
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
 : "${JOBS:="1"}"
+: "${NOM:="true"}"
 
 for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+
+if [ "$NOM" = "true" ]; then
     nixos-rebuild build --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#odo" --log-format internal-json -v "${@}" |& nom --json
+else
+    nixos-rebuild build --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#odo" -v "${@}"
+fi

nix/configuration/hosts/odo/distributed_build.nix

@@ -4,6 +4,12 @@
   config = {
     me.distributed_build.enable = true;
     me.distributed_build.machines.quark = {
+      enable = false;
+      additional_config = {
+        speedFactor = 2;
+      };
+    };
+    me.distributed_build.machines.hydra = {
       enable = true;
       additional_config = {
         speedFactor = 2;

nix/configuration/hosts/odowork/distributed_build.nix

@@ -4,6 +4,12 @@
   config = {
     me.distributed_build.enable = true;
     me.distributed_build.machines.quark = {
+      enable = false;
+      additional_config = {
+        speedFactor = 2;
+      };
+    };
+    me.distributed_build.machines.hydra = {
       enable = true;
       additional_config = {
         speedFactor = 2;

nix/configuration/hosts/quark/distributed_build.nix

@@ -3,5 +3,11 @@
 
   config = {
     me.distributed_build.enable = true;
+    me.distributed_build.machines.hydra = {
+      enable = true;
+      additional_config = {
+        speedFactor = 2;
+      };
+    };
   };
 }

nix/configuration/roles/distributed_build/default.nix

@@ -25,6 +25,13 @@ let
       };
       description = "Additional config values for the buildMachines entry. For example, speedFactor.";
     };
+
+    substituter_url = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      example = "ssh-ng://remote-host";
+      description = "URL to use as a substituter.";
+    };
   };
 
   static_host_configs = {
@@ -37,7 +44,40 @@ let
         # "aarch64-linux"
       ];
     };
+    hydra = {
+      # Does not work, so we have to use root's authorized keys. Not sure why. My best guess is it is related to overriding the ssh target via the ssh config.
+      #
+      # From: base64 -w0 /persist/ssh/ssh_host_ed25519_key.pub
+      # publicHostKey = "c3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSUNJRk9tU0NWV25xVVFFL2RKd2R0STdRQ29LTHhBNHRmWnRSYStFSG9XV0wgcm9vdEBoeWRyYQo=";
+      # publicHostKey = "c3NoLXJzYSBBQUFBQjNOemFDMXljMkVBQUFBREFRQUJBQUFDQVFDNk1nNytKMys1d2c5QkJmLzkweWJhbnhJK3ZiSXRzY1ZoTzRRVFNMT1FJM1QxbWMrNUxTK2oxdDY4a20zVmx1c2k5WWh0UmNOVk1NbmZQT3Q3ejFyZ0Q4Y05iY3UzU0xCOTNGR2hvb1ZtUktyVEMyaVE4bkxZYVoyV1lrdnZ3UjJ6b24zTjRlTlFJdWUwR0EzTEtNKy82RDN5V3didjRYV2pFeWtpa1NoVFJUVUtMTjU4aUVVOVRmcnEzeE1Ib0EvYmJZMGIxK1QzUDRkQ05wb3BFcFczaVJaYkV0NUpvNS92VFBsVjc4L3I3bnV2eHlaVjU3dWhzYjhFQlFzYUdCYTIraEt6SUhFblBHWXlPUDVRRWVCZ1ZBUzh6bFg4aktaZnZuaDA0NmdFdFoyMWJWdHNBZHNXcnVYdXNEUVQyanJ2VjJOVTYvc2cyMzZPbFZvWVU4Z1JYRHZiTXhoclVSVWNsajM1RGlzTi9wMEd2clZOckVjSktZK245MS9UMG9qU0gvVTVlRzVPclhZRXQwVzIra0NsVDk0T2kxM2JjQkVDYmVUUXJKMjJRc1hKSnVEVkVzOWhsU2RLemZkMDVaaVc3MllHaHRCRWptL015UG1nSHhxYzNTKzEwc0VvT2FIazdtRjQ0M0o1MzZoWEVHZDhGWjROMnQ3MlF3V3RuUGoyNENYOG8wbmNJN2ZIYTRHTWsybi9EWU84MlNUTEJHeVBMTkxnK040NUcyYUhaSk1NWGprWlplMUVZS0dQQm5tcTFlVUdFMVd0ZkVRSEhCTHd3Y0dDdm15aWI1NTliQ0p5NWExS2pnSGNBYlk0WVRKOTlwMWtPT2lIcVlvTnArczlhSklPZU93aE5xbkkrOUxrWnYrbEhCdXRoM3A1anRRNzA3bEJMMmVBd1E9PSByb290QGh5ZHJhCg==";
+      systems = [
+        "i686-linux"
+        "x86_64-linux"
+        # "aarch64-linux"
+      ];
+
+      hostName = lib.mkForce "hydra?remote-store=local?root=/home/nixworker/persist/root";
     };
+  };
+  joined_configs =
+    lib.genAttrs
+      (builtins.filter (hostname: config.me.distributed_build.machines."${hostname}".enable) (
+        builtins.attrNames all_nixos_configs
+      ))
+      (
+        hostname:
+        (lib.mkMerge [
+          {
+            hostName = hostname;
+            sshUser = "nixworker";
+            sshKey = "/persist/manual/ssh/root/keys/id_ed25519";
+            maxJobs = 1;
+            supportedFeatures = all_nixos_configs."${hostname}".config.me.optimizations.system_features;
+          }
+          static_host_configs."${hostname}"
+          config.me.distributed_build.machines."${hostname}".additional_config
+        ])
+      );
 in
 {
   imports = [ ];
@@ -58,9 +98,13 @@ in
       {
         nix.distributedBuilds = true;
 
-        # https://nix.dev/manual/nix/2.32/store/types/ssh-store.html
+        # Using an ssh-based substituter slows down the build because querying the remote store for paths takes ages.
-        # nix.settings.substituters = lib.mkForce [ "ssh://hydra?compress=true&log-fd=2&max-connections=4" ];
+        #
+        # nix.settings.substituters = lib.mkForce [
+        #   "ssh-ng://nixworker@ns1.fizz.buzz:65122?compress=true&ssh-key=/persist/manual/ssh/root/keys/id_ed25519&remote-store=/home/nixworker/persist/root"
+        # ];
         # nix.settings.substitute = lib.mkForce true;
+
         # nix.settings.post-build-hook = pkgs.writeShellScript "post-build-hook" ''
         #   set -euo pipefail
         #   IFS=$'\n\t'
@@ -87,6 +131,7 @@ in
                   sshKey = "/persist/manual/ssh/root/keys/id_ed25519";
                   maxJobs = 1;
                   supportedFeatures = all_nixos_configs."${hostname}".config.me.optimizations.system_features;
+                  protocol = "ssh-ng";
                 }
                 static_host_configs."${hostname}"
                 config.me.distributed_build.machines."${hostname}".additional_config
@@ -95,6 +140,12 @@ in
           ) (builtins.attrNames all_nixos_configs)
         );
       }
+      # {
+      #   nix.settings.substitute = lib.mkForce true;
+      #   nix.settings.substituters = lib.mkForce (
+      #     lib.mapAttrsToList (hostname: joined_config: "ssh-ng://${joined_config.hostName}") joined_configs
+      #   );
+      # }
     ]
   );
 }

nix/configuration/roles/dont_use_substituters/default.nix

@@ -20,6 +20,6 @@
   config = lib.mkIf config.me.dont_use_substituters.enable {
     # Disable substituters to avoid risk of cache poisoning.
     nix.settings.substitute = false;
-    nix.settings.substituters = lib.mkForce [ ];
+    nix.settings.substituters = lib.mkOverride 99 [ ];
   };
 }

nix/configuration/roles/hydra/default.nix

@@ -1,9 +1,35 @@
 {
   config,
   lib,
+  pkgs,
   ...
 }:
 
+let
+  patchScriptBin =
+    {
+      filename,
+      contents,
+      path ? [ ],
+    }:
+    ((pkgs.writeScriptBin filename contents).overrideAttrs (old: {
+      buildInputs = [ pkgs.makeWrapper ];
+      buildCommand = "${old.buildCommand}\n patchShebangs $out\nwrapProgram $out/bin/${filename} --prefix PATH : ${lib.makeBinPath path}";
+    }));
+  build_odo = (
+    patchScriptBin {
+      filename = "build_odo";
+      contents = (builtins.readFile ./files/build_odo.bash);
+      path = with pkgs; [
+        bash
+        git
+        nix
+        nix-output-monitor
+        nixos-rebuild
+      ];
+    }
+  );
+in
 {
   imports = [ ];
 
@@ -17,28 +43,59 @@
   };
 
   config = lib.mkIf config.me.hydra.enable {
-    services.hydra = {
+    environment.systemPackages = with pkgs; [
-      enable = true;
+      build_odo
-      hydraURL = "http://localhost:3000"; # Externally visible URL
+    ];
-      notificationSender = "hydra@localhost"; # "From" address for hydra emails.
+
-      # a standalone Hydra will require you to unset the buildMachinesFiles list to avoid using a nonexistant /etc/nix/machines
+    environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) {
-      buildMachinesFiles = [ ];
+      hideMounts = true;
-      useSubstitutes = true;
+      users.nixworker = {
+        directories = [
+          {
+            directory = "persist";
+            user = "nixworker";
+            group = "nixworker";
+            mode = "0700";
+          }
+        ];
+      };
     };
 
-    # nix.buildMachines = [
+    # Nix 2.30.0 (2025-07-07) changed the build directory from /tmp to /nix/var/nix/builds which broke a number of builds because my ZFS datasets were utf8only.
-    #   {
+    fileSystems."/home/nixworker/persist/root/nix/var/nix/builds" = {
-    #     hostName = "localhost";
+      device = "tmpfs";
-    #     protocol = null;
+      fsType = "tmpfs";
-    #     system = "x86_64-linux";
+      options = [
-    #     supportedFeatures = [
+        "size=40G" # adjust for your situation and needs
-    #       "kvm"
+        "mode=700"
-    #       "nixos-test"
+        "uid=11400"
-    #       "big-parallel"
+        "gid=11400"
-    #       "benchmark"
+      ];
-    #     ];
+    };
-    #     maxJobs = 8;
+
-    #   }
+    systemd.timers."build-cache" = {
-    # ];
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnCalendar = "Mon *-*-* 02:00:00 America/New_York";
+        Unit = "build-cache.service";
+      };
+    };
+
+    systemd.services."build-cache" = {
+      script = ''
+        set -euo pipefail
+        IFS=$'\n\t'
+        DIR="$( cd "$( dirname "''${BASH_SOURCE[0]}" )" && pwd )"
+
+        ${build_odo}/bin/build_odo
+      '';
+      restartIfChanged = false;
+      serviceConfig = {
+        Type = "simple";
+        User = "nixworker";
+        RemainAfterExit = true; # Prevents the service from automatically starting on rebuild. See https://discourse.nixos.org/t/how-to-prevent-custom-systemd-service-from-restarting-on-nixos-rebuild-switch/43431
+        LimitNOFILE = 8192;
+      };
+    };
   };
 }

nix/configuration/roles/hydra/files/build_odo.bash

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# : ${FOO:="1"}
+
+# MANUAL: doas install -d -o nixworker -g nixworker /persist/manual/manual_add_to_store
+# MANUAL: doas -u nixworker touch /persist/manual/manual_add_to_store/foo
+
+mkdir -p /home/nixworker/persist/machines/odo /home/nixworker/persist/root
+
+if [ ! -d /home/nixworker/persist/machine_setup ]; then
+    git clone --branch kubernetes https://code.fizz.buzz/talexander/machine_setup.git /home/nixworker/persist/machine_setup
+fi
+
+(cd /home/nixworker/persist/machines/odo && JOBS=1 NIX_REMOTE='local?root=/home/nixworker/persist/root' NOM='false' /home/nixworker/persist/machine_setup/nix/configuration/hosts/odo/SELF_BUILD)

nix/configuration/roles/nix_worker/default.nix

@@ -36,6 +36,7 @@
       createHome = true; # https://github.com/NixOS/nixpkgs/issues/6481
       group = "nixworker";
       # extraGroups = [ "wheel" ];
+      uid = 11400;
       # Generate with `mkpasswd -m scrypt`
       hashedPassword = "$7$CU..../....VXvNQ8za3wSGpdzGXNT50/$HcFtn/yvwPMCw4888BelpiAPLAxe/zU87fD.d/N6U48";
       openssh.authorizedKeys.keys = [
@@ -47,6 +48,8 @@
         "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIB/IlYTQ0M5pFN5tdoswh37CDl/gbULI3h+SsKXCansh talexander@odo"
       ];
     };
-    users.groups.nixworker = { };
+    users.groups.nixworker = {
+      gid = 11400;
+    };
   };
 }