Use rust nix-builder instead of bash script.

nix/configuration/roles/hydra/default.nix

@@ -6,29 +6,51 @@
 }:
 
 let
-  patchScriptBin =
+  # patchScriptBin =
-    {
+  #   {
-      filename,
+  #     filename,
-      contents,
+  #     contents,
-      path ? [ ],
+  #     path ? [ ],
-    }:
+  #   }:
-    ((pkgs.writeScriptBin filename contents).overrideAttrs (old: {
+  #   ((pkgs.writeScriptBin filename contents).overrideAttrs (old: {
-      buildInputs = [ pkgs.makeWrapper ];
+  #     buildInputs = [ pkgs.makeWrapper ];
-      buildCommand = "${old.buildCommand}\n patchShebangs $out\nwrapProgram $out/bin/${filename} --prefix PATH : ${lib.makeBinPath path}";
+  #     buildCommand = "${old.buildCommand}\n patchShebangs $out\nwrapProgram $out/bin/${filename} --prefix PATH : ${lib.makeBinPath path}";
-    }));
+  #   }));
-  build_odo = (
+  nix_builder = pkgs.rustPlatform.buildRustPackage rec {
-    patchScriptBin {
+    pname = "nix_builder";
-      filename = "build_odo";
+    version = "0.0.0";
-      contents = (builtins.readFile ./files/build_odo.bash);
+
-      path = with pkgs; [
+    src = pkgs.fetchgit {
-        bash
+      url = "https://code.fizz.buzz/talexander/nix_builder.git";
-        git
+      # tag = version;
-        nix
+      rev = "d0fc2331e7aadc8bdd98836b466172ac37628e7d";
-        nix-output-monitor
+      hash = "sha256-V1DU9U4+k96KfGV9BTxKYjxLzV6tWvQPM+a+5NU94G8=";
-        nixos-rebuild
+      leaveDotGit = false;
-      ];
+    };
-    }
+
-  );
+    cargoLock = {
+      lockFile = "${src}/Cargo.lock";
+    };
+
+    meta = with lib; {
+      description = "A builder of nix configs for a build server.";
+      homepage = "https://code.fizz.buzz/talexander/nix_builder";
+      license = licenses.bsd0;
+      maintainers = [ ];
+    };
+
+    nativeBuildInputs = [ pkgs.makeWrapper ];
+
+    postInstall = ''
+      wrapProgram $out/bin/nix-builder --prefix PATH : ${
+        lib.makeBinPath [
+          pkgs.git
+          pkgs.nix
+          pkgs.nixos-rebuild
+        ]
+      }
+    '';
+  };
 in
 {
   imports = [ ];
@@ -44,7 +66,8 @@ in
 
   config = lib.mkIf config.me.hydra.enable {
     environment.systemPackages = with pkgs; [
-      build_odo
+      nix_builder
+      sqlite # For manually inspecting the database.
     ];
 
     environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) {
@@ -76,7 +99,7 @@ in
     systemd.timers."build-cache" = {
       wantedBy = [ "timers.target" ];
       timerConfig = {
-        OnCalendar = "Mon *-*-* 02:00:00 America/New_York";
+        OnCalendar = "*-*-* 03:00:00 America/New_York";
         Unit = "build-cache.service";
       };
     };
@@ -87,13 +110,45 @@ in
         IFS=$'\n\t'
         DIR="$( cd "$( dirname "''${BASH_SOURCE[0]}" )" && pwd )"
 
-        ${build_odo}/bin/build_odo
+        NIX_REMOTE='local?root=/home/nixworker/persist/root' RUST_BACKTRACE=1 RUST_LOG=nix_builder=DEBUG ${nix_builder}/bin/nix-builder build --config ${./files/nix_builder.toml} --target odo --target odo_update --target odowork --target odowork_update --target quark --target quark_update --target hydra --target hydra_update --target controller0 --target controller0_update --target controller1 --target controller1_update --target controller2 --target controller2_update --target worker0 --target worker0_update --target worker1 --target worker1_update --target worker2 --target worker2_update
       '';
       restartIfChanged = false;
       serviceConfig = {
         Type = "simple";
         User = "nixworker";
-        RemainAfterExit = true; # Prevents the service from automatically starting on rebuild. See https://discourse.nixos.org/t/how-to-prevent-custom-systemd-service-from-restarting-on-nixos-rebuild-switch/43431
+        # restartIfChanged = false;
+        # RemainAfterExit = true; # Prevents the service from automatically starting on rebuild. See https://discourse.nixos.org/t/how-to-prevent-custom-systemd-service-from-restarting-on-nixos-rebuild-switch/43431
+        LimitNOFILE = 8192;
+      };
+    };
+
+    # TODO: This should move into nix-builder so we can only run clean when builds are passing. Otherwise partial builds will lose progress.
+    # TODO: In nix-builder maybe include setting to auto delete to make room during builds if we run out of space, just in case builds are failing for a long time and prevent cleanup from running.
+    systemd.timers."clean-cache" = {
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnCalendar = "*-*-01 02:00:00 America/New_York";
+        Unit = "clean-cache.service";
+      };
+    };
+
+    systemd.services."clean-cache" = {
+      script = ''
+        set -euo pipefail
+        IFS=$'\n\t'
+        DIR="$( cd "$( dirname "''${BASH_SOURCE[0]}" )" && pwd )"
+
+        NIX_REMOTE='local?root=/home/nixworker/persist/root' nix-collect-garbage -d
+      '';
+      path = with pkgs; [
+        pkgs.nix
+      ];
+      restartIfChanged = false;
+      serviceConfig = {
+        Type = "simple";
+        User = "nixworker";
+        # restartIfChanged = false;
+        # RemainAfterExit = true; # Prevents the service from automatically starting on rebuild. See https://discourse.nixos.org/t/how-to-prevent-custom-systemd-service-from-restarting-on-nixos-rebuild-switch/43431
         LimitNOFILE = 8192;
       };
     };

nix/configuration/roles/hydra/files/build_odo.bash

@@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-#
-set -euo pipefail
-IFS=$'\n\t'
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-# : ${FOO:="1"}
-
-# MANUAL: doas install -d -o nixworker -g nixworker /persist/manual/manual_add_to_store
-# MANUAL: doas -u nixworker touch /persist/manual/manual_add_to_store/foo
-
-mkdir -p /home/nixworker/persist/machines/odo /home/nixworker/persist/root
-
-if [ ! -d /home/nixworker/persist/machine_setup ]; then
-    git clone --branch kubernetes https://code.fizz.buzz/talexander/machine_setup.git /home/nixworker/persist/machine_setup
-fi
-
-(cd /home/nixworker/persist/machines/odo && JOBS=1 NIX_REMOTE='local?root=/home/nixworker/persist/root' NOM='false' /home/nixworker/persist/machine_setup/nix/configuration/hosts/odo/SELF_BUILD)

nix/configuration/roles/hydra/files/nix_builder.toml

@@ -0,0 +1,163 @@
+output_directory = "/home/nixworker/persist/nix_builder"
+
+[[targets]]
+  name = "odo"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "nix"
+  path = "nix/configuration"
+  attr = "nixosConfigurations.odo.config.system.build.toplevel"
+
+[[targets]]
+  name = "odo_update"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "nix"
+  path = "nix/configuration"
+  attr = "nixosConfigurations.odo.config.system.build.toplevel"
+  update = true
+  update_branch = "nix_update"
+
+[[targets]]
+  name = "odowork"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "nix"
+  path = "nix/configuration"
+  attr = "nixosConfigurations.odowork.config.system.build.toplevel"
+
+[[targets]]
+  name = "odowork_update"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "nix"
+  path = "nix/configuration"
+  attr = "nixosConfigurations.odowork.config.system.build.toplevel"
+  update = true
+  update_branch = "nix_update"
+
+[[targets]]
+  name = "quark"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "nix"
+  path = "nix/configuration"
+  attr = "nixosConfigurations.quark.config.system.build.toplevel"
+
+[[targets]]
+  name = "quark_update"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "nix"
+  path = "nix/configuration"
+  attr = "nixosConfigurations.quark.config.system.build.toplevel"
+  update = true
+  update_branch = "nix_update"
+
+[[targets]]
+  name = "hydra"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "nix"
+  path = "nix/configuration"
+  attr = "hydra.vm_iso"
+
+[[targets]]
+  name = "hydra_update"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "nix"
+  path = "nix/configuration"
+  attr = "hydra.vm_iso"
+  update = true
+  update_branch = "nix_update"
+
+[[targets]]
+  name = "controller0"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "kubernetes"
+  path = "nix/kubernetes"
+  attr = "controller0.vm_iso"
+
+[[targets]]
+  name = "controller0_update"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "kubernetes"
+  path = "nix/kubernetes"
+  attr = "controller0.vm_iso"
+  update = true
+  update_branch = "kubernetes_update"
+
+[[targets]]
+  name = "controller1"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "kubernetes"
+  path = "nix/kubernetes"
+  attr = "controller1.vm_iso"
+
+[[targets]]
+  name = "controller1_update"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "kubernetes"
+  path = "nix/kubernetes"
+  attr = "controller1.vm_iso"
+  update = true
+  update_branch = "kubernetes_update"
+
+[[targets]]
+  name = "controller2"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "kubernetes"
+  path = "nix/kubernetes"
+  attr = "controller2.vm_iso"
+
+[[targets]]
+  name = "controller2_update"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "kubernetes"
+  path = "nix/kubernetes"
+  attr = "controller2.vm_iso"
+  update = true
+  update_branch = "kubernetes_update"
+
+[[targets]]
+  name = "worker0"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "kubernetes"
+  path = "nix/kubernetes"
+  attr = "worker0.vm_iso"
+
+[[targets]]
+  name = "worker0_update"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "kubernetes"
+  path = "nix/kubernetes"
+  attr = "worker0.vm_iso"
+  update = true
+  update_branch = "kubernetes_update"
+
+[[targets]]
+  name = "worker1"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "kubernetes"
+  path = "nix/kubernetes"
+  attr = "worker1.vm_iso"
+
+[[targets]]
+  name = "worker1_update"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "kubernetes"
+  path = "nix/kubernetes"
+  attr = "worker1.vm_iso"
+  update = true
+  update_branch = "kubernetes_update"
+
+[[targets]]
+  name = "worker2"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "kubernetes"
+  path = "nix/kubernetes"
+  attr = "worker2.vm_iso"
+
+[[targets]]
+  name = "worker2_update"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "kubernetes"
+  path = "nix/kubernetes"
+  attr = "worker2.vm_iso"
+  update = true
+  update_branch = "kubernetes_update"
+
+# TODO: Add steam deck