From c3bcc549a55b5b1507b420544523c801337187bc Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 11 Jan 2026 16:38:56 -0500 Subject: [PATCH] Set up hydra as a remote build machine. --- nix/configuration/hosts/hydra/default.nix | 68 ++++++++++++- nix/configuration/hosts/hydra/vm_disk.nix | 44 +++++---- nix/configuration/hosts/odo/SELF_BUILD | 8 +- .../hosts/odo/distributed_build.nix | 6 ++ .../hosts/odowork/distributed_build.nix | 6 ++ .../hosts/quark/distributed_build.nix | 6 ++ .../roles/distributed_build/default.nix | 54 +++++++++- .../roles/dont_use_substituters/default.nix | 2 +- nix/configuration/roles/hydra/default.nix | 99 +++++++++++++++---- .../roles/hydra/files/build_odo.bash | 18 ++++ .../roles/nix_worker/default.nix | 5 +- 11 files changed, 267 insertions(+), 49 deletions(-) create mode 100644 nix/configuration/roles/hydra/files/build_odo.bash diff --git a/nix/configuration/hosts/hydra/default.nix b/nix/configuration/hosts/hydra/default.nix index e551617c..57ff64ab 100644 --- a/nix/configuration/hosts/hydra/default.nix +++ b/nix/configuration/hosts/hydra/default.nix @@ -18,10 +18,43 @@ ]; config = { - # Generate with `head -c4 /dev/urandom | od -A none -t x4` - networking.hostId = "6fbf418b"; + networking = + let + interface = "enp0s2"; + in + { + # Generate with `head -c4 /dev/urandom | od -A none -t x4` + hostId = "6fbf418b"; - networking.hostName = "hydra"; # Define your hostname. + hostName = "hydra"; # Define your hostname. + + interfaces = { + "${interface}" = { + ipv4.addresses = [ + { + address = "10.215.1.219"; + prefixLength = 24; + } + ]; + + ipv6.addresses = [ + { + address = "2620:11f:7001:7:ffff:ffff:0ad7:01db"; + prefixLength = 64; + } + ]; + }; + }; + defaultGateway = "10.215.1.1"; + defaultGateway6 = { + # address = "2620:11f:7001:7::1"; + address = "2620:11f:7001:7:ffff:ffff:0ad7:0101"; + inherit interface; + }; + + dhcpcd.enable = lib.mkForce false; + useDHCP = lib.mkForce false; + }; time.timeZone = "America/New_York"; i18n.defaultLocale = "en_US.UTF-8"; @@ -63,13 +96,42 @@ environment.systemPackages = with pkgs; [ htop + git # for building on hydra + tmux # for building on hydra + nix-output-monitor # for building on hydra ]; # nix.sshServe.enable = true; # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ]; + # Override garbage collection to keep things longer + # Automatic garbage collection + nix.gc = lib.mkForce { + automatic = true; + persistent = true; + dates = "weekly"; + # randomizedDelaySec = "14m"; + options = "--delete-older-than 60d"; + }; + + # The default limit of files is 1024 which is too low for some nix builds. + # + # Check with `ulimit -n` + security.pam.loginLimits = [ + { + domain = "*"; + item = "nofile"; + type = "-"; + value = "8192"; + } + ]; + + # systemd.user.extraConfig = "DefaultLimitNOFILE=8192"; + # systemd.services."user@11400".serviceConfig.LimitNOFILE = "8192"; + me.build_in_ram.enable = true; me.dont_use_substituters.enable = true; + me.hydra.enable = true; me.minimal_base.enable = true; me.nix_worker.enable = true; }; diff --git a/nix/configuration/hosts/hydra/vm_disk.nix b/nix/configuration/hosts/hydra/vm_disk.nix index f67d28fa..bbbfb41b 100644 --- a/nix/configuration/hosts/hydra/vm_disk.nix +++ b/nix/configuration/hosts/hydra/vm_disk.nix @@ -26,7 +26,7 @@ neededForBoot = true; }; - # "/.disk" = lib.mkForce { + # "/.persist" = lib.mkForce { # device = "bind9p"; # fsType = "9p"; # options = [ @@ -35,6 +35,10 @@ # "version=9p2000.L" # "cache=mmap" # "msize=512000" + # "uname=root" + # "dfltuid=0" + # "dfltgid=0" + # "nodevmap" # # "noauto" # # "x-systemd.automount" # ]; @@ -67,25 +71,25 @@ neededForBoot = true; }; - "/nix/store" = lib.mkForce { - overlay = { - lowerdir = [ "/nix/.ro-store" ]; - upperdir = "/.disk/persist/store"; - workdir = "/.disk/state/work"; - }; - # fsType = "overlay"; - # device = "overlay"; - # options = [ - # "lowerdir=/nix/.ro-store" - # "upperdir=/.disk/persist/store" - # "workdir=/.disk/state/work" - # ]; - depends = [ - "/nix/.ro-store" - "/.disk/persist/store" - "/.disk/state/work" - ]; - }; + # "/nix/store" = lib.mkForce { + # overlay = { + # lowerdir = [ "/nix/.ro-store" ]; + # upperdir = "/.disk/persist/store"; + # workdir = "/.disk/state/work"; + # }; + # # fsType = "overlay"; + # # device = "overlay"; + # # options = [ + # # "lowerdir=/nix/.ro-store" + # # "upperdir=/.disk/persist/store" + # # "workdir=/.disk/state/work" + # # ]; + # depends = [ + # "/nix/.ro-store" + # "/.disk/persist/store" + # "/.disk/state/work" + # ]; + # }; }; }; } diff --git a/nix/configuration/hosts/odo/SELF_BUILD b/nix/configuration/hosts/odo/SELF_BUILD index 6de3e8b6..4dae4a98 100755 --- a/nix/configuration/hosts/odo/SELF_BUILD +++ b/nix/configuration/hosts/odo/SELF_BUILD @@ -5,6 +5,12 @@ IFS=$'\n\t' DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" : "${JOBS:="1"}" +: "${NOM:="true"}" for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done -nixos-rebuild build --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#odo" --log-format internal-json -v "${@}" |& nom --json + +if [ "$NOM" = "true" ]; then + nixos-rebuild build --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#odo" --log-format internal-json -v "${@}" |& nom --json +else + nixos-rebuild build --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#odo" -v "${@}" +fi diff --git a/nix/configuration/hosts/odo/distributed_build.nix b/nix/configuration/hosts/odo/distributed_build.nix index a6ce7fb0..723f54a0 100644 --- a/nix/configuration/hosts/odo/distributed_build.nix +++ b/nix/configuration/hosts/odo/distributed_build.nix @@ -4,6 +4,12 @@ config = { me.distributed_build.enable = true; me.distributed_build.machines.quark = { + enable = false; + additional_config = { + speedFactor = 2; + }; + }; + me.distributed_build.machines.hydra = { enable = true; additional_config = { speedFactor = 2; diff --git a/nix/configuration/hosts/odowork/distributed_build.nix b/nix/configuration/hosts/odowork/distributed_build.nix index a6ce7fb0..723f54a0 100644 --- a/nix/configuration/hosts/odowork/distributed_build.nix +++ b/nix/configuration/hosts/odowork/distributed_build.nix @@ -4,6 +4,12 @@ config = { me.distributed_build.enable = true; me.distributed_build.machines.quark = { + enable = false; + additional_config = { + speedFactor = 2; + }; + }; + me.distributed_build.machines.hydra = { enable = true; additional_config = { speedFactor = 2; diff --git a/nix/configuration/hosts/quark/distributed_build.nix b/nix/configuration/hosts/quark/distributed_build.nix index 1cddab6e..f4f49b05 100644 --- a/nix/configuration/hosts/quark/distributed_build.nix +++ b/nix/configuration/hosts/quark/distributed_build.nix @@ -3,5 +3,11 @@ config = { me.distributed_build.enable = true; + me.distributed_build.machines.hydra = { + enable = true; + additional_config = { + speedFactor = 2; + }; + }; }; } diff --git a/nix/configuration/roles/distributed_build/default.nix b/nix/configuration/roles/distributed_build/default.nix index b034e4c8..fc0b0296 100644 --- a/nix/configuration/roles/distributed_build/default.nix +++ b/nix/configuration/roles/distributed_build/default.nix @@ -25,6 +25,13 @@ let }; description = "Additional config values for the buildMachines entry. For example, speedFactor."; }; + + substituter_url = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "ssh-ng://remote-host"; + description = "URL to use as a substituter."; + }; }; static_host_configs = { @@ -37,7 +44,40 @@ let # "aarch64-linux" ]; }; + hydra = { + # Does not work, so we have to use root's authorized keys. Not sure why. My best guess is it is related to overriding the ssh target via the ssh config. + # + # From: base64 -w0 /persist/ssh/ssh_host_ed25519_key.pub + # publicHostKey = "c3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSUNJRk9tU0NWV25xVVFFL2RKd2R0STdRQ29LTHhBNHRmWnRSYStFSG9XV0wgcm9vdEBoeWRyYQo="; + # publicHostKey = "c3NoLXJzYSBBQUFBQjNOemFDMXljMkVBQUFBREFRQUJBQUFDQVFDNk1nNytKMys1d2c5QkJmLzkweWJhbnhJK3ZiSXRzY1ZoTzRRVFNMT1FJM1QxbWMrNUxTK2oxdDY4a20zVmx1c2k5WWh0UmNOVk1NbmZQT3Q3ejFyZ0Q4Y05iY3UzU0xCOTNGR2hvb1ZtUktyVEMyaVE4bkxZYVoyV1lrdnZ3UjJ6b24zTjRlTlFJdWUwR0EzTEtNKy82RDN5V3didjRYV2pFeWtpa1NoVFJUVUtMTjU4aUVVOVRmcnEzeE1Ib0EvYmJZMGIxK1QzUDRkQ05wb3BFcFczaVJaYkV0NUpvNS92VFBsVjc4L3I3bnV2eHlaVjU3dWhzYjhFQlFzYUdCYTIraEt6SUhFblBHWXlPUDVRRWVCZ1ZBUzh6bFg4aktaZnZuaDA0NmdFdFoyMWJWdHNBZHNXcnVYdXNEUVQyanJ2VjJOVTYvc2cyMzZPbFZvWVU4Z1JYRHZiTXhoclVSVWNsajM1RGlzTi9wMEd2clZOckVjSktZK245MS9UMG9qU0gvVTVlRzVPclhZRXQwVzIra0NsVDk0T2kxM2JjQkVDYmVUUXJKMjJRc1hKSnVEVkVzOWhsU2RLemZkMDVaaVc3MllHaHRCRWptL015UG1nSHhxYzNTKzEwc0VvT2FIazdtRjQ0M0o1MzZoWEVHZDhGWjROMnQ3MlF3V3RuUGoyNENYOG8wbmNJN2ZIYTRHTWsybi9EWU84MlNUTEJHeVBMTkxnK040NUcyYUhaSk1NWGprWlplMUVZS0dQQm5tcTFlVUdFMVd0ZkVRSEhCTHd3Y0dDdm15aWI1NTliQ0p5NWExS2pnSGNBYlk0WVRKOTlwMWtPT2lIcVlvTnArczlhSklPZU93aE5xbkkrOUxrWnYrbEhCdXRoM3A1anRRNzA3bEJMMmVBd1E9PSByb290QGh5ZHJhCg=="; + systems = [ + "i686-linux" + "x86_64-linux" + # "aarch64-linux" + ]; + + hostName = lib.mkForce "hydra?remote-store=local?root=/home/nixworker/persist/root"; + }; }; + joined_configs = + lib.genAttrs + (builtins.filter (hostname: config.me.distributed_build.machines."${hostname}".enable) ( + builtins.attrNames all_nixos_configs + )) + ( + hostname: + (lib.mkMerge [ + { + hostName = hostname; + sshUser = "nixworker"; + sshKey = "/persist/manual/ssh/root/keys/id_ed25519"; + maxJobs = 1; + supportedFeatures = all_nixos_configs."${hostname}".config.me.optimizations.system_features; + } + static_host_configs."${hostname}" + config.me.distributed_build.machines."${hostname}".additional_config + ]) + ); in { imports = [ ]; @@ -58,9 +98,13 @@ in { nix.distributedBuilds = true; - # https://nix.dev/manual/nix/2.32/store/types/ssh-store.html - # nix.settings.substituters = lib.mkForce [ "ssh://hydra?compress=true&log-fd=2&max-connections=4" ]; + # Using an ssh-based substituter slows down the build because querying the remote store for paths takes ages. + # + # nix.settings.substituters = lib.mkForce [ + # "ssh-ng://nixworker@ns1.fizz.buzz:65122?compress=true&ssh-key=/persist/manual/ssh/root/keys/id_ed25519&remote-store=/home/nixworker/persist/root" + # ]; # nix.settings.substitute = lib.mkForce true; + # nix.settings.post-build-hook = pkgs.writeShellScript "post-build-hook" '' # set -euo pipefail # IFS=$'\n\t' @@ -95,6 +139,12 @@ in ) (builtins.attrNames all_nixos_configs) ); } + # { + # nix.settings.substitute = lib.mkForce true; + # nix.settings.substituters = lib.mkForce ( + # lib.mapAttrsToList (hostname: joined_config: "ssh-ng://${joined_config.hostName}") joined_configs + # ); + # } ] ); } diff --git a/nix/configuration/roles/dont_use_substituters/default.nix b/nix/configuration/roles/dont_use_substituters/default.nix index 0982ff2e..4de81853 100644 --- a/nix/configuration/roles/dont_use_substituters/default.nix +++ b/nix/configuration/roles/dont_use_substituters/default.nix @@ -20,6 +20,6 @@ config = lib.mkIf config.me.dont_use_substituters.enable { # Disable substituters to avoid risk of cache poisoning. nix.settings.substitute = false; - nix.settings.substituters = lib.mkForce [ ]; + nix.settings.substituters = lib.mkOverride 99 [ ]; }; } diff --git a/nix/configuration/roles/hydra/default.nix b/nix/configuration/roles/hydra/default.nix index 88697bc7..883c504e 100644 --- a/nix/configuration/roles/hydra/default.nix +++ b/nix/configuration/roles/hydra/default.nix @@ -1,9 +1,35 @@ { config, lib, + pkgs, ... }: +let + patchScriptBin = + { + filename, + contents, + path ? [ ], + }: + ((pkgs.writeScriptBin filename contents).overrideAttrs (old: { + buildInputs = [ pkgs.makeWrapper ]; + buildCommand = "${old.buildCommand}\n patchShebangs $out\nwrapProgram $out/bin/${filename} --prefix PATH : ${lib.makeBinPath path}"; + })); + build_odo = ( + patchScriptBin { + filename = "build_odo"; + contents = (builtins.readFile ./files/build_odo.bash); + path = with pkgs; [ + bash + git + nix + nix-output-monitor + nixos-rebuild + ]; + } + ); +in { imports = [ ]; @@ -17,28 +43,59 @@ }; config = lib.mkIf config.me.hydra.enable { - services.hydra = { - enable = true; - hydraURL = "http://localhost:3000"; # Externally visible URL - notificationSender = "hydra@localhost"; # "From" address for hydra emails. - # a standalone Hydra will require you to unset the buildMachinesFiles list to avoid using a nonexistant /etc/nix/machines - buildMachinesFiles = [ ]; - useSubstitutes = true; + environment.systemPackages = with pkgs; [ + build_odo + ]; + + environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) { + hideMounts = true; + users.nixworker = { + directories = [ + { + directory = "persist"; + user = "nixworker"; + group = "nixworker"; + mode = "0700"; + } + ]; + }; }; - # nix.buildMachines = [ - # { - # hostName = "localhost"; - # protocol = null; - # system = "x86_64-linux"; - # supportedFeatures = [ - # "kvm" - # "nixos-test" - # "big-parallel" - # "benchmark" - # ]; - # maxJobs = 8; - # } - # ]; + # Nix 2.30.0 (2025-07-07) changed the build directory from /tmp to /nix/var/nix/builds which broke a number of builds because my ZFS datasets were utf8only. + fileSystems."/home/nixworker/persist/root/nix/var/nix/builds" = { + device = "tmpfs"; + fsType = "tmpfs"; + options = [ + "size=40G" # adjust for your situation and needs + "mode=700" + "uid=11400" + "gid=11400" + ]; + }; + + systemd.timers."build-cache" = { + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = "Mon *-*-* 02:00:00 America/New_York"; + Unit = "build-cache.service"; + }; + }; + + systemd.services."build-cache" = { + script = '' + set -euo pipefail + IFS=$'\n\t' + DIR="$( cd "$( dirname "''${BASH_SOURCE[0]}" )" && pwd )" + + ${build_odo}/bin/build_odo + ''; + restartIfChanged = false; + serviceConfig = { + Type = "simple"; + User = "nixworker"; + RemainAfterExit = true; # Prevents the service from automatically starting on rebuild. See https://discourse.nixos.org/t/how-to-prevent-custom-systemd-service-from-restarting-on-nixos-rebuild-switch/43431 + LimitNOFILE = 8192; + }; + }; }; } diff --git a/nix/configuration/roles/hydra/files/build_odo.bash b/nix/configuration/roles/hydra/files/build_odo.bash new file mode 100644 index 00000000..f68514ac --- /dev/null +++ b/nix/configuration/roles/hydra/files/build_odo.bash @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# : ${FOO:="1"} + +# MANUAL: doas install -d -o nixworker -g nixworker /persist/manual/manual_add_to_store +# MANUAL: doas -u nixworker touch /persist/manual/manual_add_to_store/foo + +mkdir -p /home/nixworker/persist/machines/odo /home/nixworker/persist/root + +if [ ! -d /home/nixworker/persist/machine_setup ]; then + git clone --branch kubernetes https://code.fizz.buzz/talexander/machine_setup.git /home/nixworker/persist/machine_setup +fi + +(cd /home/nixworker/persist/machines/odo && JOBS=1 NIX_REMOTE='local?root=/home/nixworker/persist/root' NOM='false' /home/nixworker/persist/machine_setup/nix/configuration/hosts/odo/SELF_BUILD) diff --git a/nix/configuration/roles/nix_worker/default.nix b/nix/configuration/roles/nix_worker/default.nix index 7244e98c..7e468062 100644 --- a/nix/configuration/roles/nix_worker/default.nix +++ b/nix/configuration/roles/nix_worker/default.nix @@ -36,6 +36,7 @@ createHome = true; # https://github.com/NixOS/nixpkgs/issues/6481 group = "nixworker"; # extraGroups = [ "wheel" ]; + uid = 11400; # Generate with `mkpasswd -m scrypt` hashedPassword = "$7$CU..../....VXvNQ8za3wSGpdzGXNT50/$HcFtn/yvwPMCw4888BelpiAPLAxe/zU87fD.d/N6U48"; openssh.authorizedKeys.keys = [ @@ -47,6 +48,8 @@ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIB/IlYTQ0M5pFN5tdoswh37CDl/gbULI3h+SsKXCansh talexander@odo" ]; }; - users.groups.nixworker = { }; + users.groups.nixworker = { + gid = 11400; + }; }; }