Add worker nodes.

nix/kubernetes/configuration.nix

@@ -7,6 +7,7 @@
 {
   imports = [
     ./roles/boot
+    ./roles/containerd
     ./roles/control_plane
     ./roles/doas
     ./roles/dont_use_substituters
@@ -24,8 +25,7 @@
     ./roles/ssh
     ./roles/sshd
     ./roles/user
-    ./roles/zfs
+    ./roles/worker_node
-    ./roles/zrepl
     ./roles/zsh
     ./util/install_files
     ./util/unfree_polyfill

nix/kubernetes/flake.nix

@@ -54,6 +54,15 @@
         controller2 = {
           system = "x86_64-linux";
         };
+        worker0 = {
+          system = "x86_64-linux";
+        };
+        worker1 = {
+          system = "x86_64-linux";
+        };
+        worker2 = {
+          system = "x86_64-linux";
+        };
       };
       nixosConfigs = builtins.mapAttrs (
         hostname: nodeConfig: format:

nix/kubernetes/hosts/worker0/DEPLOY_BOOT

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+TARGET=worker0
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+
+nixos-rebuild boot --flake "$DIR/../../#worker0" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/worker0/DEPLOY_SWITCH

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+TARGET=worker0
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+
+nixos-rebuild switch --flake "$DIR/../../#worker0" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/worker0/ISO

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#worker0.iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/worker0/SELF_BOOT

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nixos-rebuild boot --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker0" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/worker0/SELF_BUILD

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nixos-rebuild build --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker0" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/worker0/SELF_SWITCH

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nixos-rebuild switch --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker0" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/worker0/VM_ISO

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#worker0.vm_iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/worker0/default.nix

@@ -0,0 +1,109 @@
+# MANUAL: On client machines generate signing keys:
+#   nix-store --generate-binary-cache-key some-name /persist/manual/nix/nix-cache-key.sec /persist/manual/nix/nix-cache-key.pub
+#
+# Trust other machines and add the substituters:
+#   nix.binaryCachePublicKeys = [ "some-name:AzNW1MOlkNEsUAXS1jIFZ1QCFKXjV+Y/LrF37quAZ1A=" ];
+#   nix.binaryCaches = [ "https://test.example/nix-cache" ];
+
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  imports = [
+    ./hardware-configuration.nix
+    ./vm_disk.nix
+  ];
+
+  config = {
+    networking =
+      let
+        interface = "enp0s2";
+      in
+      {
+        # Generate with `head -c4 /dev/urandom | od -A none -t x4`
+        hostId = "0aadbb10";
+
+        hostName = "worker0"; # Define your hostname.
+
+        interfaces = {
+          "${interface}" = {
+            ipv4.addresses = [
+              {
+                address = "10.215.1.224";
+                prefixLength = 24;
+              }
+            ];
+
+            ipv6.addresses = [
+              {
+                address = "2620:11f:7001:7:ffff:ffff:0ad7:01e0";
+                prefixLength = 64;
+              }
+            ];
+          };
+        };
+        defaultGateway = "10.215.1.1";
+        defaultGateway6 = {
+          # address = "2620:11f:7001:7::1";
+          address = "2620:11f:7001:7:ffff:ffff:0ad7:0101";
+          inherit interface;
+        };
+        nameservers = [
+          "10.215.1.1"
+        ];
+
+        dhcpcd.enable = lib.mkForce false;
+        useDHCP = lib.mkForce false;
+      };
+
+    time.timeZone = "America/New_York";
+    i18n.defaultLocale = "en_US.UTF-8";
+
+    me.boot.enable = true;
+    me.boot.secure = false;
+    me.mountPersistence = true;
+    boot.loader.timeout = lib.mkForce 0; # We can always generate a new ISO if we need to access other boot options.
+
+    me.optimizations = {
+      enable = true;
+      arch = "znver4";
+      # build_arch = "x86-64-v3";
+      system_features = [
+        "gccarch-znver4"
+        "gccarch-skylake"
+        "gccarch-kabylake"
+        # "gccarch-alderlake" missing WAITPKG
+        "gccarch-x86-64-v3"
+        "gccarch-x86-64-v4"
+        "benchmark"
+        "big-parallel"
+        "kvm"
+        "nixos-test"
+      ];
+    };
+
+    # Mount tmpfs at /tmp
+    boot.tmp.useTmpfs = true;
+
+    # Enable TRIM
+    # services.fstrim.enable = lib.mkDefault true;
+
+    # nix.optimise.automatic = true;
+    # nix.optimise.dates = [ "03:45" ];
+    # nix.optimise.persistent = true;
+
+    environment.systemPackages = with pkgs; [
+      htop
+    ];
+
+    # nix.sshServe.enable = true;
+    # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ];
+
+    me.worker_node.enable = true;
+    me.dont_use_substituters.enable = true;
+    me.minimal_base.enable = true;
+  };
+}

nix/kubernetes/hosts/worker0/hardware-configuration.nix

@@ -0,0 +1,31 @@
+{
+  config,
+  lib,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [
+    (modulesPath + "/installer/scan/not-detected.nix")
+  ];
+
+  config = {
+    boot.initrd.availableKernelModules = [
+      "nvme"
+      "xhci_pci"
+      "thunderbolt"
+    ];
+    boot.initrd.kernelModules = [ ];
+    boot.kernelModules = [ ];
+    boot.extraModulePackages = [ ];
+
+    # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+    # (the default) this is the recommended approach. When using systemd-networkd it's
+    # still possible to use this option, but it's recommended to use it in conjunction
+    # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+    # networking.useDHCP = lib.mkDefault true;
+    # networking.interfaces.eno1.useDHCP = lib.mkDefault true;
+    # networking.interfaces.wlp58s0.useDHCP = lib.mkDefault true;
+  };
+}

nix/kubernetes/hosts/worker0/vm_disk.nix

@@ -0,0 +1,94 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  imports = [ ];
+
+  config = {
+    # Mount the local disk
+    fileSystems = lib.mkIf config.me.mountPersistence {
+      # "/.disk" = lib.mkForce {
+      #   device = "/dev/nvme0n1p1";
+      #   fsType = "ext4";
+      #   options = [
+      #     "noatime"
+      #     "discard"
+      #   ];
+      #   neededForBoot = true;
+      # };
+
+      "/.persist" = lib.mkForce {
+        device = "bind9p";
+        fsType = "9p";
+        options = [
+          "noatime"
+          "trans=virtio"
+          "version=9p2000.L"
+          "cache=mmap"
+          "msize=512000"
+          # "noauto"
+          # "x-systemd.automount"
+        ];
+        neededForBoot = true;
+      };
+
+      "/persist" = {
+        fsType = "none";
+        device = "/.persist/persist";
+        options = [
+          "bind"
+          "rw"
+        ];
+        depends = [
+          "/.persist/persist"
+        ];
+        neededForBoot = true;
+      };
+
+      "/state" = {
+        fsType = "none";
+        device = "/.persist/state";
+        options = [
+          "bind"
+          "rw"
+        ];
+        depends = [
+          "/.persist/state"
+        ];
+        neededForBoot = true;
+      };
+
+      "/k8spv" = lib.mkForce {
+        device = "k8spv";
+        fsType = "9p";
+        options = [
+          "noatime"
+          "trans=virtio"
+          "version=9p2000.L"
+          "cache=mmap"
+          "msize=512000"
+          # "noauto"
+          # "x-systemd.automount"
+        ];
+        neededForBoot = true;
+      };
+
+      # "/disk" = {
+      #   fsType = "none";
+      #   device = "/.disk/persist";
+      #   options = [
+      #     "bind"
+      #     "rw"
+      #   ];
+      #   depends = [
+      #     "/.disk/persist"
+      #   ];
+      #   neededForBoot = true;
+      # };
+    };
+  };
+}

nix/kubernetes/hosts/worker1/DEPLOY_BOOT

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+TARGET=worker1
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+
+nixos-rebuild boot --flake "$DIR/../../#worker1" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/worker1/DEPLOY_SWITCH

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+TARGET=worker1
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+
+nixos-rebuild switch --flake "$DIR/../../#worker1" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/worker1/ISO

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#worker1.iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/worker1/SELF_BOOT

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nixos-rebuild boot --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker1" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/worker1/SELF_BUILD

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nixos-rebuild build --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker1" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/worker1/SELF_SWITCH

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nixos-rebuild switch --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker1" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/worker1/VM_ISO

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#worker1.vm_iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/worker1/default.nix

@@ -0,0 +1,109 @@
+# MANUAL: On client machines generate signing keys:
+#   nix-store --generate-binary-cache-key some-name /persist/manual/nix/nix-cache-key.sec /persist/manual/nix/nix-cache-key.pub
+#
+# Trust other machines and add the substituters:
+#   nix.binaryCachePublicKeys = [ "some-name:AzNW1MOlkNEsUAXS1jIFZ1QCFKXjV+Y/LrF37quAZ1A=" ];
+#   nix.binaryCaches = [ "https://test.example/nix-cache" ];
+
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  imports = [
+    ./hardware-configuration.nix
+    ./vm_disk.nix
+  ];
+
+  config = {
+    networking =
+      let
+        interface = "enp0s2";
+      in
+      {
+        # Generate with `head -c4 /dev/urandom | od -A none -t x4`
+        hostId = "4324346d";
+
+        hostName = "worker1"; # Define your hostname.
+
+        interfaces = {
+          "${interface}" = {
+            ipv4.addresses = [
+              {
+                address = "10.215.1.225";
+                prefixLength = 24;
+              }
+            ];
+
+            ipv6.addresses = [
+              {
+                address = "2620:11f:7001:7:ffff:ffff:0ad7:01e1";
+                prefixLength = 64;
+              }
+            ];
+          };
+        };
+        defaultGateway = "10.215.1.1";
+        defaultGateway6 = {
+          # address = "2620:11f:7001:7::1";
+          address = "2620:11f:7001:7:ffff:ffff:0ad7:0101";
+          inherit interface;
+        };
+        nameservers = [
+          "10.215.1.1"
+        ];
+
+        dhcpcd.enable = lib.mkForce false;
+        useDHCP = lib.mkForce false;
+      };
+
+    time.timeZone = "America/New_York";
+    i18n.defaultLocale = "en_US.UTF-8";
+
+    me.boot.enable = true;
+    me.boot.secure = false;
+    me.mountPersistence = true;
+    boot.loader.timeout = lib.mkForce 0; # We can always generate a new ISO if we need to access other boot options.
+
+    me.optimizations = {
+      enable = true;
+      arch = "znver4";
+      # build_arch = "x86-64-v3";
+      system_features = [
+        "gccarch-znver4"
+        "gccarch-skylake"
+        "gccarch-kabylake"
+        # "gccarch-alderlake" missing WAITPKG
+        "gccarch-x86-64-v3"
+        "gccarch-x86-64-v4"
+        "benchmark"
+        "big-parallel"
+        "kvm"
+        "nixos-test"
+      ];
+    };
+
+    # Mount tmpfs at /tmp
+    boot.tmp.useTmpfs = true;
+
+    # Enable TRIM
+    # services.fstrim.enable = lib.mkDefault true;
+
+    # nix.optimise.automatic = true;
+    # nix.optimise.dates = [ "03:45" ];
+    # nix.optimise.persistent = true;
+
+    environment.systemPackages = with pkgs; [
+      htop
+    ];
+
+    # nix.sshServe.enable = true;
+    # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ];
+
+    me.worker_node.enable = true;
+    me.dont_use_substituters.enable = true;
+    me.minimal_base.enable = true;
+  };
+}

nix/kubernetes/hosts/worker1/hardware-configuration.nix

@@ -0,0 +1,31 @@
+{
+  config,
+  lib,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [
+    (modulesPath + "/installer/scan/not-detected.nix")
+  ];
+
+  config = {
+    boot.initrd.availableKernelModules = [
+      "nvme"
+      "xhci_pci"
+      "thunderbolt"
+    ];
+    boot.initrd.kernelModules = [ ];
+    boot.kernelModules = [ ];
+    boot.extraModulePackages = [ ];
+
+    # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+    # (the default) this is the recommended approach. When using systemd-networkd it's
+    # still possible to use this option, but it's recommended to use it in conjunction
+    # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+    # networking.useDHCP = lib.mkDefault true;
+    # networking.interfaces.eno1.useDHCP = lib.mkDefault true;
+    # networking.interfaces.wlp58s0.useDHCP = lib.mkDefault true;
+  };
+}

nix/kubernetes/hosts/worker1/vm_disk.nix

@@ -0,0 +1,94 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  imports = [ ];
+
+  config = {
+    # Mount the local disk
+    fileSystems = lib.mkIf config.me.mountPersistence {
+      # "/.disk" = lib.mkForce {
+      #   device = "/dev/nvme0n1p1";
+      #   fsType = "ext4";
+      #   options = [
+      #     "noatime"
+      #     "discard"
+      #   ];
+      #   neededForBoot = true;
+      # };
+
+      "/.persist" = lib.mkForce {
+        device = "bind9p";
+        fsType = "9p";
+        options = [
+          "noatime"
+          "trans=virtio"
+          "version=9p2000.L"
+          "cache=mmap"
+          "msize=512000"
+          # "noauto"
+          # "x-systemd.automount"
+        ];
+        neededForBoot = true;
+      };
+
+      "/persist" = {
+        fsType = "none";
+        device = "/.persist/persist";
+        options = [
+          "bind"
+          "rw"
+        ];
+        depends = [
+          "/.persist/persist"
+        ];
+        neededForBoot = true;
+      };
+
+      "/state" = {
+        fsType = "none";
+        device = "/.persist/state";
+        options = [
+          "bind"
+          "rw"
+        ];
+        depends = [
+          "/.persist/state"
+        ];
+        neededForBoot = true;
+      };
+
+      "/k8spv" = lib.mkForce {
+        device = "k8spv";
+        fsType = "9p";
+        options = [
+          "noatime"
+          "trans=virtio"
+          "version=9p2000.L"
+          "cache=mmap"
+          "msize=512000"
+          # "noauto"
+          # "x-systemd.automount"
+        ];
+        neededForBoot = true;
+      };
+
+      # "/disk" = {
+      #   fsType = "none";
+      #   device = "/.disk/persist";
+      #   options = [
+      #     "bind"
+      #     "rw"
+      #   ];
+      #   depends = [
+      #     "/.disk/persist"
+      #   ];
+      #   neededForBoot = true;
+      # };
+    };
+  };
+}

nix/kubernetes/hosts/worker2/DEPLOY_BOOT

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+TARGET=worker2
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+
+nixos-rebuild boot --flake "$DIR/../../#worker2" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/worker2/DEPLOY_SWITCH

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+TARGET=worker2
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+
+nixos-rebuild switch --flake "$DIR/../../#worker2" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/worker2/ISO

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#worker2.iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/worker2/SELF_BOOT

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nixos-rebuild boot --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker2" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/worker2/SELF_BUILD

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nixos-rebuild build --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker2" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/worker2/SELF_SWITCH

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nixos-rebuild switch --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker2" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/worker2/VM_ISO

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#worker2.vm_iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/worker2/default.nix

@@ -0,0 +1,109 @@
+# MANUAL: On client machines generate signing keys:
+#   nix-store --generate-binary-cache-key some-name /persist/manual/nix/nix-cache-key.sec /persist/manual/nix/nix-cache-key.pub
+#
+# Trust other machines and add the substituters:
+#   nix.binaryCachePublicKeys = [ "some-name:AzNW1MOlkNEsUAXS1jIFZ1QCFKXjV+Y/LrF37quAZ1A=" ];
+#   nix.binaryCaches = [ "https://test.example/nix-cache" ];
+
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  imports = [
+    ./hardware-configuration.nix
+    ./vm_disk.nix
+  ];
+
+  config = {
+    networking =
+      let
+        interface = "enp0s2";
+      in
+      {
+        # Generate with `head -c4 /dev/urandom | od -A none -t x4`
+        hostId = "ce017961";
+
+        hostName = "worker2"; # Define your hostname.
+
+        interfaces = {
+          "${interface}" = {
+            ipv4.addresses = [
+              {
+                address = "10.215.1.226";
+                prefixLength = 24;
+              }
+            ];
+
+            ipv6.addresses = [
+              {
+                address = "2620:11f:7001:7:ffff:ffff:0ad7:01e2";
+                prefixLength = 64;
+              }
+            ];
+          };
+        };
+        defaultGateway = "10.215.1.1";
+        defaultGateway6 = {
+          # address = "2620:11f:7001:7::1";
+          address = "2620:11f:7001:7:ffff:ffff:0ad7:0101";
+          inherit interface;
+        };
+        nameservers = [
+          "10.215.1.1"
+        ];
+
+        dhcpcd.enable = lib.mkForce false;
+        useDHCP = lib.mkForce false;
+      };
+
+    time.timeZone = "America/New_York";
+    i18n.defaultLocale = "en_US.UTF-8";
+
+    me.boot.enable = true;
+    me.boot.secure = false;
+    me.mountPersistence = true;
+    boot.loader.timeout = lib.mkForce 0; # We can always generate a new ISO if we need to access other boot options.
+
+    me.optimizations = {
+      enable = true;
+      arch = "znver4";
+      # build_arch = "x86-64-v3";
+      system_features = [
+        "gccarch-znver4"
+        "gccarch-skylake"
+        "gccarch-kabylake"
+        # "gccarch-alderlake" missing WAITPKG
+        "gccarch-x86-64-v3"
+        "gccarch-x86-64-v4"
+        "benchmark"
+        "big-parallel"
+        "kvm"
+        "nixos-test"
+      ];
+    };
+
+    # Mount tmpfs at /tmp
+    boot.tmp.useTmpfs = true;
+
+    # Enable TRIM
+    # services.fstrim.enable = lib.mkDefault true;
+
+    # nix.optimise.automatic = true;
+    # nix.optimise.dates = [ "03:45" ];
+    # nix.optimise.persistent = true;
+
+    environment.systemPackages = with pkgs; [
+      htop
+    ];
+
+    # nix.sshServe.enable = true;
+    # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ];
+
+    me.worker_node.enable = true;
+    me.dont_use_substituters.enable = true;
+    me.minimal_base.enable = true;
+  };
+}

nix/kubernetes/hosts/worker2/hardware-configuration.nix

@@ -0,0 +1,31 @@
+{
+  config,
+  lib,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [
+    (modulesPath + "/installer/scan/not-detected.nix")
+  ];
+
+  config = {
+    boot.initrd.availableKernelModules = [
+      "nvme"
+      "xhci_pci"
+      "thunderbolt"
+    ];
+    boot.initrd.kernelModules = [ ];
+    boot.kernelModules = [ ];
+    boot.extraModulePackages = [ ];
+
+    # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+    # (the default) this is the recommended approach. When using systemd-networkd it's
+    # still possible to use this option, but it's recommended to use it in conjunction
+    # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+    # networking.useDHCP = lib.mkDefault true;
+    # networking.interfaces.eno1.useDHCP = lib.mkDefault true;
+    # networking.interfaces.wlp58s0.useDHCP = lib.mkDefault true;
+  };
+}

nix/kubernetes/hosts/worker2/vm_disk.nix

@@ -0,0 +1,94 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  imports = [ ];
+
+  config = {
+    # Mount the local disk
+    fileSystems = lib.mkIf config.me.mountPersistence {
+      # "/.disk" = lib.mkForce {
+      #   device = "/dev/nvme0n1p1";
+      #   fsType = "ext4";
+      #   options = [
+      #     "noatime"
+      #     "discard"
+      #   ];
+      #   neededForBoot = true;
+      # };
+
+      "/.persist" = lib.mkForce {
+        device = "bind9p";
+        fsType = "9p";
+        options = [
+          "noatime"
+          "trans=virtio"
+          "version=9p2000.L"
+          "cache=mmap"
+          "msize=512000"
+          # "noauto"
+          # "x-systemd.automount"
+        ];
+        neededForBoot = true;
+      };
+
+      "/persist" = {
+        fsType = "none";
+        device = "/.persist/persist";
+        options = [
+          "bind"
+          "rw"
+        ];
+        depends = [
+          "/.persist/persist"
+        ];
+        neededForBoot = true;
+      };
+
+      "/state" = {
+        fsType = "none";
+        device = "/.persist/state";
+        options = [
+          "bind"
+          "rw"
+        ];
+        depends = [
+          "/.persist/state"
+        ];
+        neededForBoot = true;
+      };
+
+      "/k8spv" = lib.mkForce {
+        device = "k8spv";
+        fsType = "9p";
+        options = [
+          "noatime"
+          "trans=virtio"
+          "version=9p2000.L"
+          "cache=mmap"
+          "msize=512000"
+          # "noauto"
+          # "x-systemd.automount"
+        ];
+        neededForBoot = true;
+      };
+
+      # "/disk" = {
+      #   fsType = "none";
+      #   device = "/.disk/persist";
+      #   options = [
+      #     "bind"
+      #     "rw"
+      #   ];
+      #   depends = [
+      #     "/.disk/persist"
+      #   ];
+      #   neededForBoot = true;
+      # };
+    };
+  };
+}

nix/kubernetes/roles/containerd/default.nix

@@ -0,0 +1,46 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  imports = [ ];
+
+  options.me = {
+    containerd.enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+      description = "Whether we want to install containerd.";
+    };
+  };
+
+  config = lib.mkIf config.me.containerd.enable {
+    virtualisation.containerd.enable = true;
+    virtualisation.containerd.settings = {
+      "plugins" = {
+        "io.containerd.grpc.v1.cri" = {
+          "cni" = {
+            "bin_dir" = "/opt/cni/bin";
+            "conf_dir" = "/etc/cni/net.d";
+          };
+          "containerd" = {
+            "default_runtime_name" = "runc";
+            "runtimes" = {
+              "runc" = {
+                "options" = {
+                  "SystemdCgroup" = true;
+                };
+                "runtime_type" = "io.containerd.runc.v2";
+              };
+            };
+            "snapshotter" = "overlayfs";
+          };
+        };
+      };
+      "version" = 2;
+    };
+  };
+}

nix/kubernetes/roles/etcd/default.nix

@@ -105,5 +105,6 @@
       gptfdisk # cgdisk
     ];
     networking.firewall.enable = false; # TODO: This is just here for debugging / initial development.
+    # TODO: Maybe use networking.nftables.enable to switch to nftables?
   };
 }

nix/kubernetes/roles/minimal_base/default.nix

@@ -24,8 +24,6 @@
     me.ssh.enable = true;
     me.sshd.enable = true;
     me.user.enable = true;
-    me.zfs.enable = true;
-    me.zrepl.enable = true;
     me.zsh.enable = true;
 
     # TODO: Maybe add me.boot.enable ?

nix/kubernetes/roles/worker_node/default.nix

@@ -0,0 +1,24 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  imports = [ ];
+
+  options.me = {
+    worker_node.enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+      description = "Whether we want to install worker_node.";
+    };
+  };
+
+  config = lib.mkIf config.me.worker_node.enable {
+    me.containerd.enable = true;
+    me.kubernetes.enable = true;
+  };
+}

nix/kubernetes/roles/zfs/default.nix

@@ -1,68 +0,0 @@
-{
-  config,
-  lib,
-  pkgs,
-  ...
-}:
-
-let
-  zfs_clone_send =
-    (pkgs.writeScriptBin "zfs_clone_send" (builtins.readFile ./files/zfs_clone_send.bash)).overrideAttrs
-      (old: {
-        buildCommand = "${old.buildCommand}\n patchShebangs $out";
-
-      });
-  zfs_clone_recv =
-    (pkgs.writeScriptBin "zfs_clone_recv" (builtins.readFile ./files/zfs_clone_recv.bash)).overrideAttrs
-      (old: {
-        buildCommand = "${old.buildCommand}\n patchShebangs $out";
-
-      });
-  zfs_clone_resume =
-    (pkgs.writeScriptBin "zfs_clone_resume" (builtins.readFile ./files/zfs_clone_resume.bash))
-    .overrideAttrs
-      (old: {
-        buildCommand = "${old.buildCommand}\n patchShebangs $out";
-
-      });
-in
-{
-  imports = [ ];
-
-  options.me = {
-    zfs.enable = lib.mkOption {
-      type = lib.types.bool;
-      default = false;
-      example = true;
-      description = "Whether we want to install zfs.";
-    };
-  };
-
-  config = lib.mkIf config.me.zfs.enable {
-    # Technically only needed when building the ISO because nix detects ZFS in the filesystem list normally. I basically always want this so I'm just setting it to always be on.
-    boot.supportedFilesystems.zfs = true;
-
-    boot.zfs.devNodes = "/dev/disk/by-partuuid";
-
-    services.zfs = {
-      autoScrub = {
-        enable = true;
-        interval = "monthly";
-      };
-      trim.enable = true;
-    };
-
-    environment.systemPackages = [
-      zfs_clone_send
-      zfs_clone_recv
-      zfs_clone_resume
-    ];
-
-    environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) {
-      hideMounts = true;
-      directories = [
-        "/etc/zfs/zpool.cache" # Which zpools to import, the root zpool is already imported and does not need this cache file but this captures additional pools.
-      ];
-    };
-  };
-}

nix/kubernetes/roles/zfs/files/zfs_clone_recv.bash

@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-#
-# A zfs-send alias that creates a perfect clone with good defaults.
-set -euo pipefail
-IFS=$'\n\t'
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-# -s if the stream is interrupted, save the partial stream. The stream can then be resumed by doing a zfs send -t token where token is the receive_resume_token prop on the dataset we received into.
-# -u Do not mount the filesystem we are receiving. We can always mount afterwards but this avoids issues with streams with mountpoints to places like /
-# Can optionally add -F to destroy the dataset in the recv location.
-exec zfs recv -s -u "${@}"
-
-# To delete an interrupted recv, run `zfs receive -A dataset`

nix/kubernetes/roles/zfs/files/zfs_clone_resume.bash

@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-#
-# Resume a zfs send.
-set -euo pipefail
-IFS=$'\n\t'
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-function main {
-    local hst="$1"
-    local dst="$2"
-    local token
-    token=$(zfs get -H -o value receive_resume_token "$dst")
-    ssh "$hst" doas zfs send --verbose -t "$token" | doas zfs recv -s "$dst"
-
-}
-
-main "${@}"

nix/kubernetes/roles/zfs/files/zfs_clone_send.bash

@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-#
-# A zfs-send alias that creates a perfect clone with good defaults.
-set -euo pipefail
-IFS=$'\n\t'
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-exec zfs send --compressed --replicate --large-block --embed --verbose --raw "${@}"

nix/kubernetes/roles/zrepl/default.nix

@@ -1,55 +0,0 @@
-{
-  config,
-  lib,
-  ...
-}:
-
-{
-  imports = [ ];
-
-  options.me = {
-    zrepl.enable = lib.mkOption {
-      type = lib.types.bool;
-      default = false;
-      example = true;
-      description = "Whether we want to install zrepl.";
-    };
-  };
-
-  config = lib.mkIf config.me.zrepl.enable {
-    services.zrepl = {
-      enable = true;
-      settings = {
-        jobs = [
-          {
-            name = "snapjob";
-            type = "snap";
-            filesystems = {
-              "zroot/linux/nix/persist<" = true;
-              "zroot/bridge<" = true;
-            };
-            snapshotting = {
-              type = "periodic";
-              interval = "15m";
-              prefix = "zrepl_";
-            };
-            pruning = {
-              keep = [
-                {
-                  type = "grid";
-                  grid = "1x1h(keep=all) | 24x1h | 14x1d";
-                  regex = "^zrepl_.*";
-                }
-                {
-                  type = "regex";
-                  negate = true;
-                  regex = "^zrepl_.*";
-                }
-              ];
-            };
-          }
-        ];
-      };
-    };
-  };
-}