Add additional controllers.

nix/kubernetes/flake.nix

@@ -48,6 +48,12 @@
         controller0 = {
           system = "x86_64-linux";
         };
+        controller1 = {
+          system = "x86_64-linux";
+        };
+        controller2 = {
+          system = "x86_64-linux";
+        };
       };
       nixosConfigs = builtins.mapAttrs (
         hostname: nodeConfig: format:

nix/kubernetes/hosts/controller0/default.nix

@@ -102,7 +102,7 @@
     # nix.sshServe.enable = true;
     # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ];
 
-    me.etcd.cluster_name = "put a nix on it";
+    me.etcd.cluster_name = "put-a-nix-on-it";
     me.etcd.internal_ip = [
       # "10.215.1.221"
       "[2620:11f:7001:7:ffff:ffff:0ad7:01dd]"

nix/kubernetes/hosts/controller0/vm_disk.nix

@@ -12,6 +12,16 @@
     # Mount the local disk
     fileSystems = lib.mkIf config.me.mountPersistence {
       "/.disk" = lib.mkForce {
+        device = "/dev/nvme0n1p1";
+        fsType = "ext4";
+        options = [
+          "noatime"
+          "discard"
+        ];
+        neededForBoot = true;
+      };
+
+      "/.persist" = lib.mkForce {
         device = "bind9p";
         fsType = "9p";
         options = [
@@ -28,26 +38,26 @@
 
       "/persist" = {
         fsType = "none";
-        device = "/.disk/persist";
+        device = "/.persist/persist";
         options = [
           "bind"
           "rw"
         ];
         depends = [
-          "/.disk/persist"
+          "/.persist/persist"
         ];
         neededForBoot = true;
       };
 
       "/state" = {
         fsType = "none";
-        device = "/.disk/state";
+        device = "/.persist/state";
         options = [
           "bind"
           "rw"
         ];
         depends = [
-          "/.disk/state"
+          "/.persist/state"
         ];
         neededForBoot = true;
       };
@@ -66,6 +76,19 @@
         ];
         neededForBoot = true;
       };
+
+      "/disk" = {
+        fsType = "none";
+        device = "/.disk/persist";
+        options = [
+          "bind"
+          "rw"
+        ];
+        depends = [
+          "/.disk/persist"
+        ];
+        neededForBoot = true;
+      };
     };
   };
 }

nix/kubernetes/hosts/controller1/DEPLOY_BOOT

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+TARGET=controller1
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+
+nixos-rebuild boot --flake "$DIR/../../#controller1" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/controller1/DEPLOY_SWITCH

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+TARGET=controller1
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+
+nixos-rebuild switch --flake "$DIR/../../#controller1" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/controller1/ISO

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#controller1.iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/controller1/SELF_BOOT

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nixos-rebuild boot --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller1" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/controller1/SELF_BUILD

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nixos-rebuild build --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller1" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/controller1/SELF_SWITCH

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nixos-rebuild switch --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller1" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/controller1/VM_ISO

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#controller1.vm_iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/controller1/default.nix

@@ -0,0 +1,123 @@
+# MANUAL: On client machines generate signing keys:
+#   nix-store --generate-binary-cache-key some-name /persist/manual/nix/nix-cache-key.sec /persist/manual/nix/nix-cache-key.pub
+#
+# Trust other machines and add the substituters:
+#   nix.binaryCachePublicKeys = [ "some-name:AzNW1MOlkNEsUAXS1jIFZ1QCFKXjV+Y/LrF37quAZ1A=" ];
+#   nix.binaryCaches = [ "https://test.example/nix-cache" ];
+
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  imports = [
+    ./hardware-configuration.nix
+    ./vm_disk.nix
+  ];
+
+  config = {
+    networking =
+      let
+        interface = "enp0s2";
+      in
+      {
+        # Generate with `head -c4 /dev/urandom | od -A none -t x4`
+        hostId = "59a83979";
+
+        hostName = "controller1"; # Define your hostname.
+
+        interfaces = {
+          "${interface}" = {
+            ipv4.addresses = [
+              {
+                address = "10.215.1.222";
+                prefixLength = 24;
+              }
+            ];
+
+            ipv6.addresses = [
+              {
+                address = "2620:11f:7001:7:ffff:ffff:0ad7:01de";
+                prefixLength = 64;
+              }
+            ];
+          };
+        };
+        defaultGateway = "10.215.1.1";
+        defaultGateway6 = {
+          # address = "2620:11f:7001:7::1";
+          address = "2620:11f:7001:7:ffff:ffff:0ad7:0101";
+          inherit interface;
+        };
+        nameservers = [
+          "10.215.1.1"
+        ];
+
+        dhcpcd.enable = lib.mkForce false;
+        useDHCP = lib.mkForce false;
+      };
+
+    time.timeZone = "America/New_York";
+    i18n.defaultLocale = "en_US.UTF-8";
+
+    me.boot.enable = true;
+    me.boot.secure = false;
+    me.mountPersistence = true;
+    boot.loader.timeout = lib.mkForce 0; # We can always generate a new ISO if we need to access other boot options.
+
+    me.optimizations = {
+      enable = true;
+      arch = "znver4";
+      # build_arch = "x86-64-v3";
+      system_features = [
+        "gccarch-znver4"
+        "gccarch-skylake"
+        "gccarch-kabylake"
+        # "gccarch-alderlake" missing WAITPKG
+        "gccarch-x86-64-v3"
+        "gccarch-x86-64-v4"
+        "benchmark"
+        "big-parallel"
+        "kvm"
+        "nixos-test"
+      ];
+    };
+
+    # Mount tmpfs at /tmp
+    boot.tmp.useTmpfs = true;
+
+    # Enable TRIM
+    # services.fstrim.enable = lib.mkDefault true;
+
+    # nix.optimise.automatic = true;
+    # nix.optimise.dates = [ "03:45" ];
+    # nix.optimise.persistent = true;
+
+    environment.systemPackages = with pkgs; [
+      htop
+    ];
+
+    # nix.sshServe.enable = true;
+    # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ];
+
+    me.etcd.cluster_name = "put-a-nix-on-it";
+    me.etcd.internal_ip = [
+      # "10.215.1.221"
+      "[2620:11f:7001:7:ffff:ffff:0ad7:01de]"
+    ];
+    me.etcd.initial_cluster = [
+      # "controller0=https://10.215.1.221:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01dd
+      # "controller1=https://10.215.1.222:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01de
+      # "controller2=https://10.215.1.223:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01df
+      "controller0=https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2380" # 10.215.1.221
+      "controller1=https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2380" # 10.215.1.222
+      "controller2=https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2380" # 10.215.1.223
+    ];
+
+    me.dont_use_substituters.enable = true;
+    me.etcd.enable = true;
+    me.minimal_base.enable = true;
+  };
+}

nix/kubernetes/hosts/controller1/hardware-configuration.nix

@@ -0,0 +1,31 @@
+{
+  config,
+  lib,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [
+    (modulesPath + "/installer/scan/not-detected.nix")
+  ];
+
+  config = {
+    boot.initrd.availableKernelModules = [
+      "nvme"
+      "xhci_pci"
+      "thunderbolt"
+    ];
+    boot.initrd.kernelModules = [ ];
+    boot.kernelModules = [ ];
+    boot.extraModulePackages = [ ];
+
+    # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+    # (the default) this is the recommended approach. When using systemd-networkd it's
+    # still possible to use this option, but it's recommended to use it in conjunction
+    # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+    # networking.useDHCP = lib.mkDefault true;
+    # networking.interfaces.eno1.useDHCP = lib.mkDefault true;
+    # networking.interfaces.wlp58s0.useDHCP = lib.mkDefault true;
+  };
+}

nix/kubernetes/hosts/controller1/vm_disk.nix

@@ -0,0 +1,94 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  imports = [ ];
+
+  config = {
+    # Mount the local disk
+    fileSystems = lib.mkIf config.me.mountPersistence {
+      "/.disk" = lib.mkForce {
+        device = "/dev/nvme0n1p1";
+        fsType = "ext4";
+        options = [
+          "noatime"
+          "discard"
+        ];
+        neededForBoot = true;
+      };
+
+      "/.persist" = lib.mkForce {
+        device = "bind9p";
+        fsType = "9p";
+        options = [
+          "noatime"
+          "trans=virtio"
+          "version=9p2000.L"
+          "cache=mmap"
+          "msize=512000"
+          # "noauto"
+          # "x-systemd.automount"
+        ];
+        neededForBoot = true;
+      };
+
+      "/persist" = {
+        fsType = "none";
+        device = "/.persist/persist";
+        options = [
+          "bind"
+          "rw"
+        ];
+        depends = [
+          "/.persist/persist"
+        ];
+        neededForBoot = true;
+      };
+
+      "/state" = {
+        fsType = "none";
+        device = "/.persist/state";
+        options = [
+          "bind"
+          "rw"
+        ];
+        depends = [
+          "/.persist/state"
+        ];
+        neededForBoot = true;
+      };
+
+      "/k8spv" = lib.mkForce {
+        device = "k8spv";
+        fsType = "9p";
+        options = [
+          "noatime"
+          "trans=virtio"
+          "version=9p2000.L"
+          "cache=mmap"
+          "msize=512000"
+          # "noauto"
+          # "x-systemd.automount"
+        ];
+        neededForBoot = true;
+      };
+
+      "/disk" = {
+        fsType = "none";
+        device = "/.disk/persist";
+        options = [
+          "bind"
+          "rw"
+        ];
+        depends = [
+          "/.disk/persist"
+        ];
+        neededForBoot = true;
+      };
+    };
+  };
+}

nix/kubernetes/hosts/controller2/DEPLOY_BOOT

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+TARGET=controller2
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+
+nixos-rebuild boot --flake "$DIR/../../#controller2" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/controller2/DEPLOY_SWITCH

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+TARGET=controller2
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+
+nixos-rebuild switch --flake "$DIR/../../#controller2" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/controller2/ISO

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#controller2.iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/controller2/SELF_BOOT

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nixos-rebuild boot --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller2" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/controller2/SELF_BUILD

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nixos-rebuild build --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller2" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/controller2/SELF_SWITCH

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nixos-rebuild switch --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller2" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/controller2/VM_ISO

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#controller2.vm_iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json

nix/kubernetes/hosts/controller2/default.nix

@@ -0,0 +1,123 @@
+# MANUAL: On client machines generate signing keys:
+#   nix-store --generate-binary-cache-key some-name /persist/manual/nix/nix-cache-key.sec /persist/manual/nix/nix-cache-key.pub
+#
+# Trust other machines and add the substituters:
+#   nix.binaryCachePublicKeys = [ "some-name:AzNW1MOlkNEsUAXS1jIFZ1QCFKXjV+Y/LrF37quAZ1A=" ];
+#   nix.binaryCaches = [ "https://test.example/nix-cache" ];
+
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  imports = [
+    ./hardware-configuration.nix
+    ./vm_disk.nix
+  ];
+
+  config = {
+    networking =
+      let
+        interface = "enp0s2";
+      in
+      {
+        # Generate with `head -c4 /dev/urandom | od -A none -t x4`
+        hostId = "26a43660";
+
+        hostName = "controller2"; # Define your hostname.
+
+        interfaces = {
+          "${interface}" = {
+            ipv4.addresses = [
+              {
+                address = "10.215.1.223";
+                prefixLength = 24;
+              }
+            ];
+
+            ipv6.addresses = [
+              {
+                address = "2620:11f:7001:7:ffff:ffff:0ad7:01df";
+                prefixLength = 64;
+              }
+            ];
+          };
+        };
+        defaultGateway = "10.215.1.1";
+        defaultGateway6 = {
+          # address = "2620:11f:7001:7::1";
+          address = "2620:11f:7001:7:ffff:ffff:0ad7:0101";
+          inherit interface;
+        };
+        nameservers = [
+          "10.215.1.1"
+        ];
+
+        dhcpcd.enable = lib.mkForce false;
+        useDHCP = lib.mkForce false;
+      };
+
+    time.timeZone = "America/New_York";
+    i18n.defaultLocale = "en_US.UTF-8";
+
+    me.boot.enable = true;
+    me.boot.secure = false;
+    me.mountPersistence = true;
+    boot.loader.timeout = lib.mkForce 0; # We can always generate a new ISO if we need to access other boot options.
+
+    me.optimizations = {
+      enable = true;
+      arch = "znver4";
+      # build_arch = "x86-64-v3";
+      system_features = [
+        "gccarch-znver4"
+        "gccarch-skylake"
+        "gccarch-kabylake"
+        # "gccarch-alderlake" missing WAITPKG
+        "gccarch-x86-64-v3"
+        "gccarch-x86-64-v4"
+        "benchmark"
+        "big-parallel"
+        "kvm"
+        "nixos-test"
+      ];
+    };
+
+    # Mount tmpfs at /tmp
+    boot.tmp.useTmpfs = true;
+
+    # Enable TRIM
+    # services.fstrim.enable = lib.mkDefault true;
+
+    # nix.optimise.automatic = true;
+    # nix.optimise.dates = [ "03:45" ];
+    # nix.optimise.persistent = true;
+
+    environment.systemPackages = with pkgs; [
+      htop
+    ];
+
+    # nix.sshServe.enable = true;
+    # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ];
+
+    me.etcd.cluster_name = "put-a-nix-on-it";
+    me.etcd.internal_ip = [
+      # "10.215.1.221"
+      "[2620:11f:7001:7:ffff:ffff:0ad7:01df]"
+    ];
+    me.etcd.initial_cluster = [
+      # "controller0=https://10.215.1.221:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01dd
+      # "controller1=https://10.215.1.222:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01de
+      # "controller2=https://10.215.1.223:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01df
+      "controller0=https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2380" # 10.215.1.221
+      "controller1=https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2380" # 10.215.1.222
+      "controller2=https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2380" # 10.215.1.223
+    ];
+
+    me.dont_use_substituters.enable = true;
+    me.etcd.enable = true;
+    me.minimal_base.enable = true;
+  };
+}

nix/kubernetes/hosts/controller2/hardware-configuration.nix

@@ -0,0 +1,31 @@
+{
+  config,
+  lib,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [
+    (modulesPath + "/installer/scan/not-detected.nix")
+  ];
+
+  config = {
+    boot.initrd.availableKernelModules = [
+      "nvme"
+      "xhci_pci"
+      "thunderbolt"
+    ];
+    boot.initrd.kernelModules = [ ];
+    boot.kernelModules = [ ];
+    boot.extraModulePackages = [ ];
+
+    # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+    # (the default) this is the recommended approach. When using systemd-networkd it's
+    # still possible to use this option, but it's recommended to use it in conjunction
+    # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+    # networking.useDHCP = lib.mkDefault true;
+    # networking.interfaces.eno1.useDHCP = lib.mkDefault true;
+    # networking.interfaces.wlp58s0.useDHCP = lib.mkDefault true;
+  };
+}

nix/kubernetes/hosts/controller2/vm_disk.nix

@@ -0,0 +1,94 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  imports = [ ];
+
+  config = {
+    # Mount the local disk
+    fileSystems = lib.mkIf config.me.mountPersistence {
+      "/.disk" = lib.mkForce {
+        device = "/dev/nvme0n1p1";
+        fsType = "ext4";
+        options = [
+          "noatime"
+          "discard"
+        ];
+        neededForBoot = true;
+      };
+
+      "/.persist" = lib.mkForce {
+        device = "bind9p";
+        fsType = "9p";
+        options = [
+          "noatime"
+          "trans=virtio"
+          "version=9p2000.L"
+          "cache=mmap"
+          "msize=512000"
+          # "noauto"
+          # "x-systemd.automount"
+        ];
+        neededForBoot = true;
+      };
+
+      "/persist" = {
+        fsType = "none";
+        device = "/.persist/persist";
+        options = [
+          "bind"
+          "rw"
+        ];
+        depends = [
+          "/.persist/persist"
+        ];
+        neededForBoot = true;
+      };
+
+      "/state" = {
+        fsType = "none";
+        device = "/.persist/state";
+        options = [
+          "bind"
+          "rw"
+        ];
+        depends = [
+          "/.persist/state"
+        ];
+        neededForBoot = true;
+      };
+
+      "/k8spv" = lib.mkForce {
+        device = "k8spv";
+        fsType = "9p";
+        options = [
+          "noatime"
+          "trans=virtio"
+          "version=9p2000.L"
+          "cache=mmap"
+          "msize=512000"
+          # "noauto"
+          # "x-systemd.automount"
+        ];
+        neededForBoot = true;
+      };
+
+      "/disk" = {
+        fsType = "none";
+        device = "/.disk/persist";
+        options = [
+          "bind"
+          "rw"
+        ];
+        depends = [
+          "/.disk/persist"
+        ];
+        neededForBoot = true;
+      };
+    };
+  };
+}

nix/kubernetes/keys/scope.nix

@@ -52,18 +52,18 @@ makeScope newScope (
     };
     deploy_key = (
       vm_name: file: ''
-        ${openssh}/bin/ssh mrmanager rm -f /vm/${vm_name}/persist/keys/${builtins.baseNameOf file} ~/${builtins.baseNameOf file}
+        ${openssh}/bin/ssh mrmanager rm -f /vm/${vm_name}/persist/keys/etcd/${builtins.baseNameOf file} ~/${builtins.baseNameOf file}
         ${openssh}/bin/scp ${file} mrmanager:~/${builtins.baseNameOf file}
-        ${openssh}/bin/ssh mrmanager doas install -o 11235 -g 998 -m 0640 ~/${builtins.baseNameOf file} /vm/${vm_name}/persist/keys/${builtins.baseNameOf file}
+        ${openssh}/bin/ssh mrmanager doas install -o 10016 -g 10016 -m 0640 ~/${builtins.baseNameOf file} /vm/${vm_name}/persist/keys/etcd/${builtins.baseNameOf file}
         ${openssh}/bin/ssh mrmanager rm -f ~/${builtins.baseNameOf file}
-        # chown to 11235:998 for talexander:etcd
       ''
     );
     deploy_machine = (
       vm_name:
       (
         ''
-          ${openssh}/bin/ssh mrmanager doas install -d -o talexander -g talexander -m 0755 /vm/${vm_name}/persist/keys/
+          ${openssh}/bin/ssh mrmanager doas install -d -o 11235 -g 11235 -m 0755 /vm/${vm_name}/persist/keys
+          ${openssh}/bin/ssh mrmanager doas install -d -o 10016 -g 10016 -m 0755 /vm/${vm_name}/persist/keys/etcd
         ''
         + (lib.concatMapStringsSep "\n" (deploy_key vm_name) [
           "${self.kubernetes}/kubernetes.pem"

nix/kubernetes/roles/etcd/default.nix

@@ -55,13 +55,14 @@
       enable = true;
       openFirewall = true;
       name = config.networking.hostName;
-      certFile = "/.disk/keys/kubernetes.pem";
+      certFile = "/.persist/keys/etcd/kubernetes.pem";
-      keyFile = "/.disk/keys/kubernetes-key.pem";
+      keyFile = "/.persist/keys/etcd/kubernetes-key.pem";
-      peerCertFile = "/.disk/keys/kubernetes.pem";
+      peerCertFile = "/.persist/keys/etcd/kubernetes.pem";
-      peerKeyFile = "/.disk/keys/kubernetes-key.pem";
+      peerKeyFile = "/.persist/keys/etcd/kubernetes-key.pem";
-      trustedCaFile = "/.disk/keys/ca.pem";
+      trustedCaFile = "/.persist/keys/etcd/ca.pem";
-      peerTrustedCaFile = "/.disk/keys/ca.pem";
+      peerTrustedCaFile = "/.persist/keys/etcd/ca.pem";
       peerClientCertAuth = true;
+      clientCertAuth = true;
       initialAdvertisePeerUrls = (
         builtins.map (iip: "https://${iip}:2380") (builtins.attrNames config.me.etcd.internal_ip)
       );
@@ -82,11 +83,27 @@
       initialClusterState = "new";
     };
 
-    environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) {
+    environment.persistence."/disk" = lib.mkIf (config.me.mountPersistence) {
       hideMounts = true;
       directories = [
-        config.services.etcd.dataDir # "/var/lib/etcd"
+        {
+          directory = config.services.etcd.dataDir; # "/var/lib/etcd"
+          user = "etcd";
+          group = "etcd";
+          mode = "0700";
+        }
       ];
     };
+
+    users.users.etcd.uid = 10016;
+    users.groups.etcd.gid = 10016;
+
+    environment.systemPackages = with pkgs; [
+      net-tools # for debugging
+      tcpdump
+      e2fsprogs # mkfs.ext4
+      gptfdisk # cgdisk
+    ];
+    networking.firewall.enable = false;
   };
 }