From f4a2f570624ccfee06e549e38d815294434d5213 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 7 Dec 2025 15:48:08 -0500 Subject: [PATCH] Add additional controllers. --- nix/kubernetes/flake.nix | 6 + nix/kubernetes/hosts/controller0/default.nix | 2 +- nix/kubernetes/hosts/controller0/vm_disk.nix | 31 ++++- nix/kubernetes/hosts/controller1/DEPLOY_BOOT | 13 ++ .../hosts/controller1/DEPLOY_SWITCH | 13 ++ nix/kubernetes/hosts/controller1/ISO | 10 ++ nix/kubernetes/hosts/controller1/SELF_BOOT | 10 ++ nix/kubernetes/hosts/controller1/SELF_BUILD | 10 ++ nix/kubernetes/hosts/controller1/SELF_SWITCH | 10 ++ nix/kubernetes/hosts/controller1/VM_ISO | 10 ++ nix/kubernetes/hosts/controller1/default.nix | 123 ++++++++++++++++++ .../controller1/hardware-configuration.nix | 31 +++++ nix/kubernetes/hosts/controller1/vm_disk.nix | 94 +++++++++++++ nix/kubernetes/hosts/controller2/DEPLOY_BOOT | 13 ++ .../hosts/controller2/DEPLOY_SWITCH | 13 ++ nix/kubernetes/hosts/controller2/ISO | 10 ++ nix/kubernetes/hosts/controller2/SELF_BOOT | 10 ++ nix/kubernetes/hosts/controller2/SELF_BUILD | 10 ++ nix/kubernetes/hosts/controller2/SELF_SWITCH | 10 ++ nix/kubernetes/hosts/controller2/VM_ISO | 10 ++ nix/kubernetes/hosts/controller2/default.nix | 123 ++++++++++++++++++ .../controller2/hardware-configuration.nix | 31 +++++ nix/kubernetes/hosts/controller2/vm_disk.nix | 94 +++++++++++++ nix/kubernetes/keys/scope.nix | 8 +- nix/kubernetes/roles/etcd/default.nix | 33 +++-- 25 files changed, 711 insertions(+), 17 deletions(-) create mode 100755 nix/kubernetes/hosts/controller1/DEPLOY_BOOT create mode 100755 nix/kubernetes/hosts/controller1/DEPLOY_SWITCH create mode 100755 nix/kubernetes/hosts/controller1/ISO create mode 100755 nix/kubernetes/hosts/controller1/SELF_BOOT create mode 100755 nix/kubernetes/hosts/controller1/SELF_BUILD create mode 100755 nix/kubernetes/hosts/controller1/SELF_SWITCH create mode 100755 nix/kubernetes/hosts/controller1/VM_ISO create mode 100644 nix/kubernetes/hosts/controller1/default.nix create mode 100644 nix/kubernetes/hosts/controller1/hardware-configuration.nix create mode 100644 nix/kubernetes/hosts/controller1/vm_disk.nix create mode 100755 nix/kubernetes/hosts/controller2/DEPLOY_BOOT create mode 100755 nix/kubernetes/hosts/controller2/DEPLOY_SWITCH create mode 100755 nix/kubernetes/hosts/controller2/ISO create mode 100755 nix/kubernetes/hosts/controller2/SELF_BOOT create mode 100755 nix/kubernetes/hosts/controller2/SELF_BUILD create mode 100755 nix/kubernetes/hosts/controller2/SELF_SWITCH create mode 100755 nix/kubernetes/hosts/controller2/VM_ISO create mode 100644 nix/kubernetes/hosts/controller2/default.nix create mode 100644 nix/kubernetes/hosts/controller2/hardware-configuration.nix create mode 100644 nix/kubernetes/hosts/controller2/vm_disk.nix diff --git a/nix/kubernetes/flake.nix b/nix/kubernetes/flake.nix index bfbbd02e..74eda61f 100644 --- a/nix/kubernetes/flake.nix +++ b/nix/kubernetes/flake.nix @@ -48,6 +48,12 @@ controller0 = { system = "x86_64-linux"; }; + controller1 = { + system = "x86_64-linux"; + }; + controller2 = { + system = "x86_64-linux"; + }; }; nixosConfigs = builtins.mapAttrs ( hostname: nodeConfig: format: diff --git a/nix/kubernetes/hosts/controller0/default.nix b/nix/kubernetes/hosts/controller0/default.nix index 847874fe..5339141f 100644 --- a/nix/kubernetes/hosts/controller0/default.nix +++ b/nix/kubernetes/hosts/controller0/default.nix @@ -102,7 +102,7 @@ # nix.sshServe.enable = true; # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ]; - me.etcd.cluster_name = "put a nix on it"; + me.etcd.cluster_name = "put-a-nix-on-it"; me.etcd.internal_ip = [ # "10.215.1.221" "[2620:11f:7001:7:ffff:ffff:0ad7:01dd]" diff --git a/nix/kubernetes/hosts/controller0/vm_disk.nix b/nix/kubernetes/hosts/controller0/vm_disk.nix index 6c23789f..83683f83 100644 --- a/nix/kubernetes/hosts/controller0/vm_disk.nix +++ b/nix/kubernetes/hosts/controller0/vm_disk.nix @@ -12,6 +12,16 @@ # Mount the local disk fileSystems = lib.mkIf config.me.mountPersistence { "/.disk" = lib.mkForce { + device = "/dev/nvme0n1p1"; + fsType = "ext4"; + options = [ + "noatime" + "discard" + ]; + neededForBoot = true; + }; + + "/.persist" = lib.mkForce { device = "bind9p"; fsType = "9p"; options = [ @@ -28,26 +38,26 @@ "/persist" = { fsType = "none"; - device = "/.disk/persist"; + device = "/.persist/persist"; options = [ "bind" "rw" ]; depends = [ - "/.disk/persist" + "/.persist/persist" ]; neededForBoot = true; }; "/state" = { fsType = "none"; - device = "/.disk/state"; + device = "/.persist/state"; options = [ "bind" "rw" ]; depends = [ - "/.disk/state" + "/.persist/state" ]; neededForBoot = true; }; @@ -66,6 +76,19 @@ ]; neededForBoot = true; }; + + "/disk" = { + fsType = "none"; + device = "/.disk/persist"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.disk/persist" + ]; + neededForBoot = true; + }; }; }; } diff --git a/nix/kubernetes/hosts/controller1/DEPLOY_BOOT b/nix/kubernetes/hosts/controller1/DEPLOY_BOOT new file mode 100755 index 00000000..a98d5d44 --- /dev/null +++ b/nix/kubernetes/hosts/controller1/DEPLOY_BOOT @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=controller1 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild boot --flake "$DIR/../../#controller1" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller1/DEPLOY_SWITCH b/nix/kubernetes/hosts/controller1/DEPLOY_SWITCH new file mode 100755 index 00000000..e7c22a0f --- /dev/null +++ b/nix/kubernetes/hosts/controller1/DEPLOY_SWITCH @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=controller1 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild switch --flake "$DIR/../../#controller1" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller1/ISO b/nix/kubernetes/hosts/controller1/ISO new file mode 100755 index 00000000..57320a27 --- /dev/null +++ b/nix/kubernetes/hosts/controller1/ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#controller1.iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller1/SELF_BOOT b/nix/kubernetes/hosts/controller1/SELF_BOOT new file mode 100755 index 00000000..6f387324 --- /dev/null +++ b/nix/kubernetes/hosts/controller1/SELF_BOOT @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild boot --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller1" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller1/SELF_BUILD b/nix/kubernetes/hosts/controller1/SELF_BUILD new file mode 100755 index 00000000..bc575651 --- /dev/null +++ b/nix/kubernetes/hosts/controller1/SELF_BUILD @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild build --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller1" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller1/SELF_SWITCH b/nix/kubernetes/hosts/controller1/SELF_SWITCH new file mode 100755 index 00000000..2879d19c --- /dev/null +++ b/nix/kubernetes/hosts/controller1/SELF_SWITCH @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild switch --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller1" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller1/VM_ISO b/nix/kubernetes/hosts/controller1/VM_ISO new file mode 100755 index 00000000..e33e16b1 --- /dev/null +++ b/nix/kubernetes/hosts/controller1/VM_ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#controller1.vm_iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller1/default.nix b/nix/kubernetes/hosts/controller1/default.nix new file mode 100644 index 00000000..5f6d50b2 --- /dev/null +++ b/nix/kubernetes/hosts/controller1/default.nix @@ -0,0 +1,123 @@ +# MANUAL: On client machines generate signing keys: +# nix-store --generate-binary-cache-key some-name /persist/manual/nix/nix-cache-key.sec /persist/manual/nix/nix-cache-key.pub +# +# Trust other machines and add the substituters: +# nix.binaryCachePublicKeys = [ "some-name:AzNW1MOlkNEsUAXS1jIFZ1QCFKXjV+Y/LrF37quAZ1A=" ]; +# nix.binaryCaches = [ "https://test.example/nix-cache" ]; + +{ + config, + lib, + pkgs, + ... +}: +{ + imports = [ + ./hardware-configuration.nix + ./vm_disk.nix + ]; + + config = { + networking = + let + interface = "enp0s2"; + in + { + # Generate with `head -c4 /dev/urandom | od -A none -t x4` + hostId = "59a83979"; + + hostName = "controller1"; # Define your hostname. + + interfaces = { + "${interface}" = { + ipv4.addresses = [ + { + address = "10.215.1.222"; + prefixLength = 24; + } + ]; + + ipv6.addresses = [ + { + address = "2620:11f:7001:7:ffff:ffff:0ad7:01de"; + prefixLength = 64; + } + ]; + }; + }; + defaultGateway = "10.215.1.1"; + defaultGateway6 = { + # address = "2620:11f:7001:7::1"; + address = "2620:11f:7001:7:ffff:ffff:0ad7:0101"; + inherit interface; + }; + nameservers = [ + "10.215.1.1" + ]; + + dhcpcd.enable = lib.mkForce false; + useDHCP = lib.mkForce false; + }; + + time.timeZone = "America/New_York"; + i18n.defaultLocale = "en_US.UTF-8"; + + me.boot.enable = true; + me.boot.secure = false; + me.mountPersistence = true; + boot.loader.timeout = lib.mkForce 0; # We can always generate a new ISO if we need to access other boot options. + + me.optimizations = { + enable = true; + arch = "znver4"; + # build_arch = "x86-64-v3"; + system_features = [ + "gccarch-znver4" + "gccarch-skylake" + "gccarch-kabylake" + # "gccarch-alderlake" missing WAITPKG + "gccarch-x86-64-v3" + "gccarch-x86-64-v4" + "benchmark" + "big-parallel" + "kvm" + "nixos-test" + ]; + }; + + # Mount tmpfs at /tmp + boot.tmp.useTmpfs = true; + + # Enable TRIM + # services.fstrim.enable = lib.mkDefault true; + + # nix.optimise.automatic = true; + # nix.optimise.dates = [ "03:45" ]; + # nix.optimise.persistent = true; + + environment.systemPackages = with pkgs; [ + htop + ]; + + # nix.sshServe.enable = true; + # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ]; + + me.etcd.cluster_name = "put-a-nix-on-it"; + me.etcd.internal_ip = [ + # "10.215.1.221" + "[2620:11f:7001:7:ffff:ffff:0ad7:01de]" + ]; + me.etcd.initial_cluster = [ + # "controller0=https://10.215.1.221:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01dd + # "controller1=https://10.215.1.222:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01de + # "controller2=https://10.215.1.223:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01df + "controller0=https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2380" # 10.215.1.221 + "controller1=https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2380" # 10.215.1.222 + "controller2=https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2380" # 10.215.1.223 + ]; + + me.dont_use_substituters.enable = true; + me.etcd.enable = true; + me.minimal_base.enable = true; + }; +} diff --git a/nix/kubernetes/hosts/controller1/hardware-configuration.nix b/nix/kubernetes/hosts/controller1/hardware-configuration.nix new file mode 100644 index 00000000..6029e088 --- /dev/null +++ b/nix/kubernetes/hosts/controller1/hardware-configuration.nix @@ -0,0 +1,31 @@ +{ + config, + lib, + modulesPath, + ... +}: + +{ + imports = [ + (modulesPath + "/installer/scan/not-detected.nix") + ]; + + config = { + boot.initrd.availableKernelModules = [ + "nvme" + "xhci_pci" + "thunderbolt" + ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ ]; + boot.extraModulePackages = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + # networking.useDHCP = lib.mkDefault true; + # networking.interfaces.eno1.useDHCP = lib.mkDefault true; + # networking.interfaces.wlp58s0.useDHCP = lib.mkDefault true; + }; +} diff --git a/nix/kubernetes/hosts/controller1/vm_disk.nix b/nix/kubernetes/hosts/controller1/vm_disk.nix new file mode 100644 index 00000000..83683f83 --- /dev/null +++ b/nix/kubernetes/hosts/controller1/vm_disk.nix @@ -0,0 +1,94 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + config = { + # Mount the local disk + fileSystems = lib.mkIf config.me.mountPersistence { + "/.disk" = lib.mkForce { + device = "/dev/nvme0n1p1"; + fsType = "ext4"; + options = [ + "noatime" + "discard" + ]; + neededForBoot = true; + }; + + "/.persist" = lib.mkForce { + device = "bind9p"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + + "/persist" = { + fsType = "none"; + device = "/.persist/persist"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.persist/persist" + ]; + neededForBoot = true; + }; + + "/state" = { + fsType = "none"; + device = "/.persist/state"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.persist/state" + ]; + neededForBoot = true; + }; + + "/k8spv" = lib.mkForce { + device = "k8spv"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + + "/disk" = { + fsType = "none"; + device = "/.disk/persist"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.disk/persist" + ]; + neededForBoot = true; + }; + }; + }; +} diff --git a/nix/kubernetes/hosts/controller2/DEPLOY_BOOT b/nix/kubernetes/hosts/controller2/DEPLOY_BOOT new file mode 100755 index 00000000..5c4bee4f --- /dev/null +++ b/nix/kubernetes/hosts/controller2/DEPLOY_BOOT @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=controller2 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild boot --flake "$DIR/../../#controller2" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller2/DEPLOY_SWITCH b/nix/kubernetes/hosts/controller2/DEPLOY_SWITCH new file mode 100755 index 00000000..53c95e73 --- /dev/null +++ b/nix/kubernetes/hosts/controller2/DEPLOY_SWITCH @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=controller2 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild switch --flake "$DIR/../../#controller2" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller2/ISO b/nix/kubernetes/hosts/controller2/ISO new file mode 100755 index 00000000..0850b604 --- /dev/null +++ b/nix/kubernetes/hosts/controller2/ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#controller2.iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller2/SELF_BOOT b/nix/kubernetes/hosts/controller2/SELF_BOOT new file mode 100755 index 00000000..2976bc91 --- /dev/null +++ b/nix/kubernetes/hosts/controller2/SELF_BOOT @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild boot --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller2" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller2/SELF_BUILD b/nix/kubernetes/hosts/controller2/SELF_BUILD new file mode 100755 index 00000000..a9a3b562 --- /dev/null +++ b/nix/kubernetes/hosts/controller2/SELF_BUILD @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild build --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller2" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller2/SELF_SWITCH b/nix/kubernetes/hosts/controller2/SELF_SWITCH new file mode 100755 index 00000000..dfca46e9 --- /dev/null +++ b/nix/kubernetes/hosts/controller2/SELF_SWITCH @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild switch --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller2" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller2/VM_ISO b/nix/kubernetes/hosts/controller2/VM_ISO new file mode 100755 index 00000000..f57c8834 --- /dev/null +++ b/nix/kubernetes/hosts/controller2/VM_ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#controller2.vm_iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller2/default.nix b/nix/kubernetes/hosts/controller2/default.nix new file mode 100644 index 00000000..d86dbbe7 --- /dev/null +++ b/nix/kubernetes/hosts/controller2/default.nix @@ -0,0 +1,123 @@ +# MANUAL: On client machines generate signing keys: +# nix-store --generate-binary-cache-key some-name /persist/manual/nix/nix-cache-key.sec /persist/manual/nix/nix-cache-key.pub +# +# Trust other machines and add the substituters: +# nix.binaryCachePublicKeys = [ "some-name:AzNW1MOlkNEsUAXS1jIFZ1QCFKXjV+Y/LrF37quAZ1A=" ]; +# nix.binaryCaches = [ "https://test.example/nix-cache" ]; + +{ + config, + lib, + pkgs, + ... +}: +{ + imports = [ + ./hardware-configuration.nix + ./vm_disk.nix + ]; + + config = { + networking = + let + interface = "enp0s2"; + in + { + # Generate with `head -c4 /dev/urandom | od -A none -t x4` + hostId = "26a43660"; + + hostName = "controller2"; # Define your hostname. + + interfaces = { + "${interface}" = { + ipv4.addresses = [ + { + address = "10.215.1.223"; + prefixLength = 24; + } + ]; + + ipv6.addresses = [ + { + address = "2620:11f:7001:7:ffff:ffff:0ad7:01df"; + prefixLength = 64; + } + ]; + }; + }; + defaultGateway = "10.215.1.1"; + defaultGateway6 = { + # address = "2620:11f:7001:7::1"; + address = "2620:11f:7001:7:ffff:ffff:0ad7:0101"; + inherit interface; + }; + nameservers = [ + "10.215.1.1" + ]; + + dhcpcd.enable = lib.mkForce false; + useDHCP = lib.mkForce false; + }; + + time.timeZone = "America/New_York"; + i18n.defaultLocale = "en_US.UTF-8"; + + me.boot.enable = true; + me.boot.secure = false; + me.mountPersistence = true; + boot.loader.timeout = lib.mkForce 0; # We can always generate a new ISO if we need to access other boot options. + + me.optimizations = { + enable = true; + arch = "znver4"; + # build_arch = "x86-64-v3"; + system_features = [ + "gccarch-znver4" + "gccarch-skylake" + "gccarch-kabylake" + # "gccarch-alderlake" missing WAITPKG + "gccarch-x86-64-v3" + "gccarch-x86-64-v4" + "benchmark" + "big-parallel" + "kvm" + "nixos-test" + ]; + }; + + # Mount tmpfs at /tmp + boot.tmp.useTmpfs = true; + + # Enable TRIM + # services.fstrim.enable = lib.mkDefault true; + + # nix.optimise.automatic = true; + # nix.optimise.dates = [ "03:45" ]; + # nix.optimise.persistent = true; + + environment.systemPackages = with pkgs; [ + htop + ]; + + # nix.sshServe.enable = true; + # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ]; + + me.etcd.cluster_name = "put-a-nix-on-it"; + me.etcd.internal_ip = [ + # "10.215.1.221" + "[2620:11f:7001:7:ffff:ffff:0ad7:01df]" + ]; + me.etcd.initial_cluster = [ + # "controller0=https://10.215.1.221:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01dd + # "controller1=https://10.215.1.222:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01de + # "controller2=https://10.215.1.223:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01df + "controller0=https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2380" # 10.215.1.221 + "controller1=https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2380" # 10.215.1.222 + "controller2=https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2380" # 10.215.1.223 + ]; + + me.dont_use_substituters.enable = true; + me.etcd.enable = true; + me.minimal_base.enable = true; + }; +} diff --git a/nix/kubernetes/hosts/controller2/hardware-configuration.nix b/nix/kubernetes/hosts/controller2/hardware-configuration.nix new file mode 100644 index 00000000..6029e088 --- /dev/null +++ b/nix/kubernetes/hosts/controller2/hardware-configuration.nix @@ -0,0 +1,31 @@ +{ + config, + lib, + modulesPath, + ... +}: + +{ + imports = [ + (modulesPath + "/installer/scan/not-detected.nix") + ]; + + config = { + boot.initrd.availableKernelModules = [ + "nvme" + "xhci_pci" + "thunderbolt" + ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ ]; + boot.extraModulePackages = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + # networking.useDHCP = lib.mkDefault true; + # networking.interfaces.eno1.useDHCP = lib.mkDefault true; + # networking.interfaces.wlp58s0.useDHCP = lib.mkDefault true; + }; +} diff --git a/nix/kubernetes/hosts/controller2/vm_disk.nix b/nix/kubernetes/hosts/controller2/vm_disk.nix new file mode 100644 index 00000000..83683f83 --- /dev/null +++ b/nix/kubernetes/hosts/controller2/vm_disk.nix @@ -0,0 +1,94 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + config = { + # Mount the local disk + fileSystems = lib.mkIf config.me.mountPersistence { + "/.disk" = lib.mkForce { + device = "/dev/nvme0n1p1"; + fsType = "ext4"; + options = [ + "noatime" + "discard" + ]; + neededForBoot = true; + }; + + "/.persist" = lib.mkForce { + device = "bind9p"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + + "/persist" = { + fsType = "none"; + device = "/.persist/persist"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.persist/persist" + ]; + neededForBoot = true; + }; + + "/state" = { + fsType = "none"; + device = "/.persist/state"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.persist/state" + ]; + neededForBoot = true; + }; + + "/k8spv" = lib.mkForce { + device = "k8spv"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + + "/disk" = { + fsType = "none"; + device = "/.disk/persist"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.disk/persist" + ]; + neededForBoot = true; + }; + }; + }; +} diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index 6f1f52c1..f86abcf9 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -52,18 +52,18 @@ makeScope newScope ( }; deploy_key = ( vm_name: file: '' - ${openssh}/bin/ssh mrmanager rm -f /vm/${vm_name}/persist/keys/${builtins.baseNameOf file} ~/${builtins.baseNameOf file} + ${openssh}/bin/ssh mrmanager rm -f /vm/${vm_name}/persist/keys/etcd/${builtins.baseNameOf file} ~/${builtins.baseNameOf file} ${openssh}/bin/scp ${file} mrmanager:~/${builtins.baseNameOf file} - ${openssh}/bin/ssh mrmanager doas install -o 11235 -g 998 -m 0640 ~/${builtins.baseNameOf file} /vm/${vm_name}/persist/keys/${builtins.baseNameOf file} + ${openssh}/bin/ssh mrmanager doas install -o 10016 -g 10016 -m 0640 ~/${builtins.baseNameOf file} /vm/${vm_name}/persist/keys/etcd/${builtins.baseNameOf file} ${openssh}/bin/ssh mrmanager rm -f ~/${builtins.baseNameOf file} - # chown to 11235:998 for talexander:etcd '' ); deploy_machine = ( vm_name: ( '' - ${openssh}/bin/ssh mrmanager doas install -d -o talexander -g talexander -m 0755 /vm/${vm_name}/persist/keys/ + ${openssh}/bin/ssh mrmanager doas install -d -o 11235 -g 11235 -m 0755 /vm/${vm_name}/persist/keys + ${openssh}/bin/ssh mrmanager doas install -d -o 10016 -g 10016 -m 0755 /vm/${vm_name}/persist/keys/etcd '' + (lib.concatMapStringsSep "\n" (deploy_key vm_name) [ "${self.kubernetes}/kubernetes.pem" diff --git a/nix/kubernetes/roles/etcd/default.nix b/nix/kubernetes/roles/etcd/default.nix index e554c37e..f0044d0a 100644 --- a/nix/kubernetes/roles/etcd/default.nix +++ b/nix/kubernetes/roles/etcd/default.nix @@ -55,13 +55,14 @@ enable = true; openFirewall = true; name = config.networking.hostName; - certFile = "/.disk/keys/kubernetes.pem"; - keyFile = "/.disk/keys/kubernetes-key.pem"; - peerCertFile = "/.disk/keys/kubernetes.pem"; - peerKeyFile = "/.disk/keys/kubernetes-key.pem"; - trustedCaFile = "/.disk/keys/ca.pem"; - peerTrustedCaFile = "/.disk/keys/ca.pem"; + certFile = "/.persist/keys/etcd/kubernetes.pem"; + keyFile = "/.persist/keys/etcd/kubernetes-key.pem"; + peerCertFile = "/.persist/keys/etcd/kubernetes.pem"; + peerKeyFile = "/.persist/keys/etcd/kubernetes-key.pem"; + trustedCaFile = "/.persist/keys/etcd/ca.pem"; + peerTrustedCaFile = "/.persist/keys/etcd/ca.pem"; peerClientCertAuth = true; + clientCertAuth = true; initialAdvertisePeerUrls = ( builtins.map (iip: "https://${iip}:2380") (builtins.attrNames config.me.etcd.internal_ip) ); @@ -82,11 +83,27 @@ initialClusterState = "new"; }; - environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) { + environment.persistence."/disk" = lib.mkIf (config.me.mountPersistence) { hideMounts = true; directories = [ - config.services.etcd.dataDir # "/var/lib/etcd" + { + directory = config.services.etcd.dataDir; # "/var/lib/etcd" + user = "etcd"; + group = "etcd"; + mode = "0700"; + } ]; }; + + users.users.etcd.uid = 10016; + users.groups.etcd.gid = 10016; + + environment.systemPackages = with pkgs; [ + net-tools # for debugging + tcpdump + e2fsprogs # mkfs.ext4 + gptfdisk # cgdisk + ]; + networking.firewall.enable = false; }; }