talexander/machine_setup
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Set up hydra as a remote build machine.

Browse Source
This commit is contained in:
Tom Alexander
2026-01-11 16:38:56 -05:00
committed by Tom Alexander
parent 24e03ed8f7
commit 776ed67675
11 changed files with 268 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
nix/configuration/roles/distributed_build/default.nix
View File

@@ -25,6 +25,13 @@ let
};
description = "Additional config values for the buildMachines entry. For example, speedFactor.";
};
substituter_url = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "ssh-ng://remote-host";
description = "URL to use as a substituter.";
};
};
static_host_configs = {
@@ -37,7 +44,40 @@ let
# "aarch64-linux"
];
};
hydra = {
# Does not work, so we have to use root's authorized keys. Not sure why. My best guess is it is related to overriding the ssh target via the ssh config.
#
# From: base64 -w0 /persist/ssh/ssh_host_ed25519_key.pub
# publicHostKey = "c3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSUNJRk9tU0NWV25xVVFFL2RKd2R0STdRQ29LTHhBNHRmWnRSYStFSG9XV0wgcm9vdEBoeWRyYQo=";
# publicHostKey = "c3NoLXJzYSBBQUFBQjNOemFDMXljMkVBQUFBREFRQUJBQUFDQVFDNk1nNytKMys1d2c5QkJmLzkweWJhbnhJK3ZiSXRzY1ZoTzRRVFNMT1FJM1QxbWMrNUxTK2oxdDY4a20zVmx1c2k5WWh0UmNOVk1NbmZQT3Q3ejFyZ0Q4Y05iY3UzU0xCOTNGR2hvb1ZtUktyVEMyaVE4bkxZYVoyV1lrdnZ3UjJ6b24zTjRlTlFJdWUwR0EzTEtNKy82RDN5V3didjRYV2pFeWtpa1NoVFJUVUtMTjU4aUVVOVRmcnEzeE1Ib0EvYmJZMGIxK1QzUDRkQ05wb3BFcFczaVJaYkV0NUpvNS92VFBsVjc4L3I3bnV2eHlaVjU3dWhzYjhFQlFzYUdCYTIraEt6SUhFblBHWXlPUDVRRWVCZ1ZBUzh6bFg4aktaZnZuaDA0NmdFdFoyMWJWdHNBZHNXcnVYdXNEUVQyanJ2VjJOVTYvc2cyMzZPbFZvWVU4Z1JYRHZiTXhoclVSVWNsajM1RGlzTi9wMEd2clZOckVjSktZK245MS9UMG9qU0gvVTVlRzVPclhZRXQwVzIra0NsVDk0T2kxM2JjQkVDYmVUUXJKMjJRc1hKSnVEVkVzOWhsU2RLemZkMDVaaVc3MllHaHRCRWptL015UG1nSHhxYzNTKzEwc0VvT2FIazdtRjQ0M0o1MzZoWEVHZDhGWjROMnQ3MlF3V3RuUGoyNENYOG8wbmNJN2ZIYTRHTWsybi9EWU84MlNUTEJHeVBMTkxnK040NUcyYUhaSk1NWGprWlplMUVZS0dQQm5tcTFlVUdFMVd0ZkVRSEhCTHd3Y0dDdm15aWI1NTliQ0p5NWExS2pnSGNBYlk0WVRKOTlwMWtPT2lIcVlvTnArczlhSklPZU93aE5xbkkrOUxrWnYrbEhCdXRoM3A1anRRNzA3bEJMMmVBd1E9PSByb290QGh5ZHJhCg==";
systems = [
"i686-linux"
"x86_64-linux"
# "aarch64-linux"
];
hostName = lib.mkForce "hydra?remote-store=local?root=/home/nixworker/persist/root";
};
};
joined_configs =
lib.genAttrs
(builtins.filter (hostname: config.me.distributed_build.machines."${hostname}".enable) (
builtins.attrNames all_nixos_configs
))
(
hostname:
(lib.mkMerge [
{
hostName = hostname;
sshUser = "nixworker";
sshKey = "/persist/manual/ssh/root/keys/id_ed25519";
maxJobs = 1;
supportedFeatures = all_nixos_configs."${hostname}".config.me.optimizations.system_features;
}
static_host_configs."${hostname}"
config.me.distributed_build.machines."${hostname}".additional_config
])
);
in
{
imports = [ ];
@@ -58,9 +98,13 @@ in
{
nix.distributedBuilds = true;
# https://nix.dev/manual/nix/2.32/store/types/ssh-store.html
# nix.settings.substituters = lib.mkForce [ "ssh://hydra?compress=true&log-fd=2&max-connections=4" ];
# Using an ssh-based substituter slows down the build because querying the remote store for paths takes ages.
#
# nix.settings.substituters = lib.mkForce [
# "ssh-ng://nixworker@ns1.fizz.buzz:65122?compress=true&ssh-key=/persist/manual/ssh/root/keys/id_ed25519&remote-store=/home/nixworker/persist/root"
# ];
# nix.settings.substitute = lib.mkForce true;
# nix.settings.post-build-hook = pkgs.writeShellScript "post-build-hook" ''
# set -euo pipefail
# IFS=$'\n\t'
@@ -87,6 +131,7 @@ in
sshKey = "/persist/manual/ssh/root/keys/id_ed25519";
maxJobs = 1;
supportedFeatures = all_nixos_configs."${hostname}".config.me.optimizations.system_features;
protocol = "ssh-ng";
}
static_host_configs."${hostname}"
config.me.distributed_build.machines."${hostname}".additional_config
@@ -95,6 +140,12 @@ in
) (builtins.attrNames all_nixos_configs)
);
}
# {
# nix.settings.substitute = lib.mkForce true;
# nix.settings.substituters = lib.mkForce (
# lib.mapAttrsToList (hostname: joined_config: "ssh-ng://${joined_config.hostName}") joined_configs
# );
# }
]
);
}

2
nix/configuration/roles/dont_use_substituters/default.nix
View File

@@ -20,6 +20,6 @@
config = lib.mkIf config.me.dont_use_substituters.enable {
# Disable substituters to avoid risk of cache poisoning.
nix.settings.substitute = false;
nix.settings.substituters = lib.mkForce [ ];
nix.settings.substituters = lib.mkOverride 99 [ ];
};
}

99
nix/configuration/roles/hydra/default.nix
View File

@@ -1,9 +1,35 @@
{
config,
lib,
pkgs,
...
}:
let
patchScriptBin =
{
filename,
contents,
path ? [ ],
}:
((pkgs.writeScriptBin filename contents).overrideAttrs (old: {
buildInputs = [ pkgs.makeWrapper ];
buildCommand = "${old.buildCommand}\n patchShebangs $out\nwrapProgram $out/bin/${filename} --prefix PATH : ${lib.makeBinPath path}";
}));
build_odo = (
patchScriptBin {
filename = "build_odo";
contents = (builtins.readFile ./files/build_odo.bash);
path = with pkgs; [
bash
git
nix
nix-output-monitor
nixos-rebuild
];
}
);
in
{
imports = [ ];
@@ -17,28 +43,59 @@
};
config = lib.mkIf config.me.hydra.enable {
services.hydra = {
enable = true;
hydraURL = "http://localhost:3000"; # Externally visible URL
notificationSender = "hydra@localhost"; # "From" address for hydra emails.
# a standalone Hydra will require you to unset the buildMachinesFiles list to avoid using a nonexistant /etc/nix/machines
buildMachinesFiles = [ ];
useSubstitutes = true;
environment.systemPackages = with pkgs; [
build_odo
];
environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) {
hideMounts = true;
users.nixworker = {
directories = [
{
directory = "persist";
user = "nixworker";
group = "nixworker";
mode = "0700";
}
];
};
};
# nix.buildMachines = [
# {
# hostName = "localhost";
# protocol = null;
# system = "x86_64-linux";
# supportedFeatures = [
# "kvm"
# "nixos-test"
# "big-parallel"
# "benchmark"
# ];
# maxJobs = 8;
# }
# ];
# Nix 2.30.0 (2025-07-07) changed the build directory from /tmp to /nix/var/nix/builds which broke a number of builds because my ZFS datasets were utf8only.
fileSystems."/home/nixworker/persist/root/nix/var/nix/builds" = {
device = "tmpfs";
fsType = "tmpfs";
options = [
"size=40G" # adjust for your situation and needs
"mode=700"
"uid=11400"
"gid=11400"
];
};
systemd.timers."build-cache" = {
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = "Mon *-*-* 02:00:00 America/New_York";
Unit = "build-cache.service";
};
};
systemd.services."build-cache" = {
script = ''
set -euo pipefail
IFS=$'\n\t'
DIR="$( cd "$( dirname "''${BASH_SOURCE[0]}" )" && pwd )"
${build_odo}/bin/build_odo
'';
restartIfChanged = false;
serviceConfig = {
Type = "simple";
User = "nixworker";
RemainAfterExit = true; # Prevents the service from automatically starting on rebuild. See https://discourse.nixos.org/t/how-to-prevent-custom-systemd-service-from-restarting-on-nixos-rebuild-switch/43431
LimitNOFILE = 8192;
};
};
};
}

18
nix/configuration/roles/hydra/files/build_odo.bash Normal file
View File

@@ -0,0 +1,18 @@
#!/usr/bin/env bash
#
set -euo pipefail
IFS=$'\n\t'
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# : ${FOO:="1"}
# MANUAL: doas install -d -o nixworker -g nixworker /persist/manual/manual_add_to_store
# MANUAL: doas -u nixworker touch /persist/manual/manual_add_to_store/foo
mkdir -p /home/nixworker/persist/machines/odo /home/nixworker/persist/root
if [ ! -d /home/nixworker/persist/machine_setup ]; then
git clone --branch kubernetes https://code.fizz.buzz/talexander/machine_setup.git /home/nixworker/persist/machine_setup
fi
(cd /home/nixworker/persist/machines/odo && JOBS=1 NIX_REMOTE='local?root=/home/nixworker/persist/root' NOM='false' /home/nixworker/persist/machine_setup/nix/configuration/hosts/odo/SELF_BUILD)

5
nix/configuration/roles/nix_worker/default.nix
View File

@@ -36,6 +36,7 @@
createHome = true; # https://github.com/NixOS/nixpkgs/issues/6481
group = "nixworker";
# extraGroups = [ "wheel" ];
uid = 11400;
# Generate with `mkpasswd -m scrypt`
hashedPassword = "$7$CU..../....VXvNQ8za3wSGpdzGXNT50/$HcFtn/yvwPMCw4888BelpiAPLAxe/zU87fD.d/N6U48";
openssh.authorizedKeys.keys = [
@@ -47,6 +48,8 @@
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIB/IlYTQ0M5pFN5tdoswh37CDl/gbULI3h+SsKXCansh talexander@odo"
];
};
users.groups.nixworker = { };
users.groups.nixworker = {
gid = 11400;
};
};
}