Use rust nix-builder instead of bash script.

nix/configuration/roles/hydra/default.nix

@@ -6,29 +6,51 @@
 }:
 
 let
-  patchScriptBin =
-    {
-      filename,
-      contents,
-      path ? [ ],
-    }:
-    ((pkgs.writeScriptBin filename contents).overrideAttrs (old: {
-      buildInputs = [ pkgs.makeWrapper ];
-      buildCommand = "${old.buildCommand}\n patchShebangs $out\nwrapProgram $out/bin/${filename} --prefix PATH : ${lib.makeBinPath path}";
-    }));
-  build_odo = (
-    patchScriptBin {
-      filename = "build_odo";
-      contents = (builtins.readFile ./files/build_odo.bash);
-      path = with pkgs; [
-        bash
-        git
-        nix
-        nix-output-monitor
-        nixos-rebuild
-      ];
-    }
-  );
+  # patchScriptBin =
+  #   {
+  #     filename,
+  #     contents,
+  #     path ? [ ],
+  #   }:
+  #   ((pkgs.writeScriptBin filename contents).overrideAttrs (old: {
+  #     buildInputs = [ pkgs.makeWrapper ];
+  #     buildCommand = "${old.buildCommand}\n patchShebangs $out\nwrapProgram $out/bin/${filename} --prefix PATH : ${lib.makeBinPath path}";
+  #   }));
+  nix_builder = pkgs.rustPlatform.buildRustPackage rec {
+    pname = "nix_builder";
+    version = "0.0.0";
+
+    src = pkgs.fetchgit {
+      url = "https://code.fizz.buzz/talexander/nix_builder.git";
+      # tag = version;
+      rev = "d0fc2331e7aadc8bdd98836b466172ac37628e7d";
+      hash = "sha256-V1DU9U4+k96KfGV9BTxKYjxLzV6tWvQPM+a+5NU94G8=";
+      leaveDotGit = false;
+    };
+
+    cargoLock = {
+      lockFile = "${src}/Cargo.lock";
+    };
+
+    meta = with lib; {
+      description = "A builder of nix configs for a build server.";
+      homepage = "https://code.fizz.buzz/talexander/nix_builder";
+      license = licenses.bsd0;
+      maintainers = [ ];
+    };
+
+    nativeBuildInputs = [ pkgs.makeWrapper ];
+
+    postInstall = ''
+      wrapProgram $out/bin/nix-builder --prefix PATH : ${
+        lib.makeBinPath [
+          pkgs.git
+          pkgs.nix
+          pkgs.nixos-rebuild
+        ]
+      }
+    '';
+  };
 in
 {
   imports = [ ];
@@ -44,7 +66,8 @@ in
 
   config = lib.mkIf config.me.hydra.enable {
     environment.systemPackages = with pkgs; [
-      build_odo
+      nix_builder
+      sqlite # For manually inspecting the database.
     ];
 
     environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) {
@@ -76,7 +99,7 @@ in
     systemd.timers."build-cache" = {
       wantedBy = [ "timers.target" ];
       timerConfig = {
-        OnCalendar = "Mon *-*-* 02:00:00 America/New_York";
+        OnCalendar = "*-*-* 03:00:00 America/New_York";
         Unit = "build-cache.service";
       };
     };
@@ -87,13 +110,45 @@ in
         IFS=$'\n\t'
         DIR="$( cd "$( dirname "''${BASH_SOURCE[0]}" )" && pwd )"
 
-        ${build_odo}/bin/build_odo
+        NIX_REMOTE='local?root=/home/nixworker/persist/root' RUST_BACKTRACE=1 RUST_LOG=nix_builder=DEBUG ${nix_builder}/bin/nix-builder build --config ${./files/nix_builder.toml} --target odo --target odo_update --target odowork --target odowork_update --target quark --target quark_update --target hydra --target hydra_update --target controller0 --target controller0_update --target controller1 --target controller1_update --target controller2 --target controller2_update --target worker0 --target worker0_update --target worker1 --target worker1_update --target worker2 --target worker2_update
       '';
       restartIfChanged = false;
       serviceConfig = {
         Type = "simple";
         User = "nixworker";
-        RemainAfterExit = true; # Prevents the service from automatically starting on rebuild. See https://discourse.nixos.org/t/how-to-prevent-custom-systemd-service-from-restarting-on-nixos-rebuild-switch/43431
+        # restartIfChanged = false;
+        # RemainAfterExit = true; # Prevents the service from automatically starting on rebuild. See https://discourse.nixos.org/t/how-to-prevent-custom-systemd-service-from-restarting-on-nixos-rebuild-switch/43431
+        LimitNOFILE = 8192;
+      };
+    };
+
+    # TODO: This should move into nix-builder so we can only run clean when builds are passing. Otherwise partial builds will lose progress.
+    # TODO: In nix-builder maybe include setting to auto delete to make room during builds if we run out of space, just in case builds are failing for a long time and prevent cleanup from running.
+    systemd.timers."clean-cache" = {
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnCalendar = "*-*-01 02:00:00 America/New_York";
+        Unit = "clean-cache.service";
+      };
+    };
+
+    systemd.services."clean-cache" = {
+      script = ''
+        set -euo pipefail
+        IFS=$'\n\t'
+        DIR="$( cd "$( dirname "''${BASH_SOURCE[0]}" )" && pwd )"
+
+        NIX_REMOTE='local?root=/home/nixworker/persist/root' nix-collect-garbage -d
+      '';
+      path = with pkgs; [
+        pkgs.nix
+      ];
+      restartIfChanged = false;
+      serviceConfig = {
+        Type = "simple";
+        User = "nixworker";
+        # restartIfChanged = false;
+        # RemainAfterExit = true; # Prevents the service from automatically starting on rebuild. See https://discourse.nixos.org/t/how-to-prevent-custom-systemd-service-from-restarting-on-nixos-rebuild-switch/43431
         LimitNOFILE = 8192;
       };
     };