Fix network for updated nix.

Fix proxy auth tls
Generate certificates for the aggregation layer.
2026-02-21 15:11:15 -05:00 · 2026-02-21 15:11:15 -05:00 · 2026-02-21 15:11:14 -05:00 · 2026-02-21 15:11:14 -05:00 · 2026-02-21 15:11:14 -05:00 · 2026-02-21 15:11:14 -05:00
210 changed files with 9877 additions and 56 deletions
--- a/nix/configuration/README.org
+++ b/nix/configuration/README.org
@ -0,0 +1,12 @@
+* To-do
+** Perhaps use overlay for /etc for speedup
+#+begin_src nix
+  system.etc.overlay.enable = true;
+#+end_src
+** read https://nixos.org/manual/nixos/stable/
+** Performance for mini pc
+#+begin_src nix
+  security.pam.loginLimits = [
+    { domain = "@users"; item = "rtprio"; type = "-"; value = 1; }
+  ];
+#+end_src
--- a/nix/configuration/roles/firefox/default.nix
+++ b/nix/configuration/roles/firefox/default.nix
@ -79,6 +79,20 @@
      policies = {
        DisableTelemetry = true;
        DisplayBookmarksToolbar = "newtab";
+        DisableFirefoxStudies = true;
+        FirefoxHome = {
+          SponsoredStories = false;
+          SponsoredTopSites = false;
+          Stories = false;
+        };
+        GenerativeAI = {
+          Enabled = false;
+        };
+        SearchEngines = {
+          Remove = [
+            "Perplexity"
+          ];
+        };

        # Check about:support for extension/add-on ID strings.
        # Valid strings for installation_mode are "allowed", "blocked",
--- a/nix/configuration/roles/hydra/default.nix
+++ b/nix/configuration/roles/hydra/default.nix
@ -6,29 +6,51 @@
 }:

 let
-  patchScriptBin =
-    {
-      filename,
-      contents,
-      path ? [ ],
-    }:
-    ((pkgs.writeScriptBin filename contents).overrideAttrs (old: {
-      buildInputs = [ pkgs.makeWrapper ];
-      buildCommand = "${old.buildCommand}\n patchShebangs $out\nwrapProgram $out/bin/${filename} --prefix PATH : ${lib.makeBinPath path}";
-    }));
-  build_odo = (
-    patchScriptBin {
-      filename = "build_odo";
-      contents = (builtins.readFile ./files/build_odo.bash);
-      path = with pkgs; [
-        bash
-        git
-        nix
-        nix-output-monitor
-        nixos-rebuild
-      ];
-    }
-  );
+  # patchScriptBin =
+  #   {
+  #     filename,
+  #     contents,
+  #     path ? [ ],
+  #   }:
+  #   ((pkgs.writeScriptBin filename contents).overrideAttrs (old: {
+  #     buildInputs = [ pkgs.makeWrapper ];
+  #     buildCommand = "${old.buildCommand}\n patchShebangs $out\nwrapProgram $out/bin/${filename} --prefix PATH : ${lib.makeBinPath path}";
+  #   }));
+  nix_builder = pkgs.rustPlatform.buildRustPackage rec {
+    pname = "nix_builder";
+    version = "0.0.0";
+
+    src = pkgs.fetchgit {
+      url = "https://code.fizz.buzz/talexander/nix_builder.git";
+      # tag = version;
+      rev = "d0fc2331e7aadc8bdd98836b466172ac37628e7d";
+      hash = "sha256-V1DU9U4+k96KfGV9BTxKYjxLzV6tWvQPM+a+5NU94G8=";
+      leaveDotGit = false;
+    };
+
+    cargoLock = {
+      lockFile = "${src}/Cargo.lock";
+    };
+
+    meta = with lib; {
+      description = "A builder of nix configs for a build server.";
+      homepage = "https://code.fizz.buzz/talexander/nix_builder";
+      license = licenses.bsd0;
+      maintainers = [ ];
+    };
+
+    nativeBuildInputs = [ pkgs.makeWrapper ];
+
+    postInstall = ''
+      wrapProgram $out/bin/nix-builder --prefix PATH : ${
+        lib.makeBinPath [
+          pkgs.git
+          pkgs.nix
+          pkgs.nixos-rebuild
+        ]
+      }
+    '';
+  };
 in
 {
  imports = [ ];
@ -44,7 +66,8 @@ in

  config = lib.mkIf config.me.hydra.enable {
    environment.systemPackages = with pkgs; [
-      build_odo
+      nix_builder
+      sqlite # For manually inspecting the database.
    ];

    environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) {
@ -76,7 +99,7 @@ in
    systemd.timers."build-cache" = {
      wantedBy = [ "timers.target" ];
      timerConfig = {
-        OnCalendar = "Mon *-*-* 02:00:00 America/New_York";
+        OnCalendar = "*-*-* 03:00:00 America/New_York";
        Unit = "build-cache.service";
      };
    };
@ -87,13 +110,45 @@ in
        IFS=$'\n\t'
        DIR="$( cd "$( dirname "''${BASH_SOURCE[0]}" )" && pwd )"

-        ${build_odo}/bin/build_odo
+        NIX_REMOTE='local?root=/home/nixworker/persist/root' RUST_BACKTRACE=1 RUST_LOG=nix_builder=DEBUG ${nix_builder}/bin/nix-builder build --config ${./files/nix_builder.toml} --target odo --target odo_update --target odowork --target odowork_update --target quark --target quark_update --target hydra --target hydra_update --target controller0 --target controller0_update --target controller1 --target controller1_update --target controller2 --target controller2_update --target worker0 --target worker0_update --target worker1 --target worker1_update --target worker2 --target worker2_update
      '';
      restartIfChanged = false;
      serviceConfig = {
        Type = "simple";
        User = "nixworker";
-        RemainAfterExit = true; # Prevents the service from automatically starting on rebuild. See https://discourse.nixos.org/t/how-to-prevent-custom-systemd-service-from-restarting-on-nixos-rebuild-switch/43431
+        # restartIfChanged = false;
+        # RemainAfterExit = true; # Prevents the service from automatically starting on rebuild. See https://discourse.nixos.org/t/how-to-prevent-custom-systemd-service-from-restarting-on-nixos-rebuild-switch/43431
+        LimitNOFILE = 8192;
+      };
+    };
+
+    # TODO: This should move into nix-builder so we can only run clean when builds are passing. Otherwise partial builds will lose progress.
+    # TODO: In nix-builder maybe include setting to auto delete to make room during builds if we run out of space, just in case builds are failing for a long time and prevent cleanup from running.
+    systemd.timers."clean-cache" = {
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnCalendar = "*-*-01 02:00:00 America/New_York";
+        Unit = "clean-cache.service";
+      };
+    };
+
+    systemd.services."clean-cache" = {
+      script = ''
+        set -euo pipefail
+        IFS=$'\n\t'
+        DIR="$( cd "$( dirname "''${BASH_SOURCE[0]}" )" && pwd )"
+
+        NIX_REMOTE='local?root=/home/nixworker/persist/root' nix-collect-garbage -d
+      '';
+      path = with pkgs; [
+        pkgs.nix
+      ];
+      restartIfChanged = false;
+      serviceConfig = {
+        Type = "simple";
+        User = "nixworker";
+        # restartIfChanged = false;
+        # RemainAfterExit = true; # Prevents the service from automatically starting on rebuild. See https://discourse.nixos.org/t/how-to-prevent-custom-systemd-service-from-restarting-on-nixos-rebuild-switch/43431
        LimitNOFILE = 8192;
      };
    };
--- a/nix/configuration/roles/hydra/files/build_odo.bash
+++ b/nix/configuration/roles/hydra/files/build_odo.bash
@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-#
-set -euo pipefail
-IFS=$'\n\t'
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-# : ${FOO:="1"}
-
-# MANUAL: doas install -d -o nixworker -g nixworker /persist/manual/manual_add_to_store
-# MANUAL: doas -u nixworker touch /persist/manual/manual_add_to_store/foo
-
-mkdir -p /home/nixworker/persist/machines/odo /home/nixworker/persist/root
-
-if [ ! -d /home/nixworker/persist/machine_setup ]; then
-    git clone --branch kubernetes https://code.fizz.buzz/talexander/machine_setup.git /home/nixworker/persist/machine_setup
-fi
-
-(cd /home/nixworker/persist/machines/odo && JOBS=1 NIX_REMOTE='local?root=/home/nixworker/persist/root' NOM='false' /home/nixworker/persist/machine_setup/nix/configuration/hosts/odo/SELF_BUILD)
--- a/nix/configuration/roles/hydra/files/nix_builder.toml
+++ b/nix/configuration/roles/hydra/files/nix_builder.toml
@ -0,0 +1,163 @@
+output_directory = "/home/nixworker/persist/nix_builder"
+
+[[targets]]
+  name = "odo"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "nix"
+  path = "nix/configuration"
+  attr = "nixosConfigurations.odo.config.system.build.toplevel"
+
+[[targets]]
+  name = "odo_update"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "nix"
+  path = "nix/configuration"
+  attr = "nixosConfigurations.odo.config.system.build.toplevel"
+  update = true
+  update_branch = "nix_update"
+
+[[targets]]
+  name = "odowork"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "nix"
+  path = "nix/configuration"
+  attr = "nixosConfigurations.odowork.config.system.build.toplevel"
+
+[[targets]]
+  name = "odowork_update"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "nix"
+  path = "nix/configuration"
+  attr = "nixosConfigurations.odowork.config.system.build.toplevel"
+  update = true
+  update_branch = "nix_update"
+
+[[targets]]
+  name = "quark"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "nix"
+  path = "nix/configuration"
+  attr = "nixosConfigurations.quark.config.system.build.toplevel"
+
+[[targets]]
+  name = "quark_update"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "nix"
+  path = "nix/configuration"
+  attr = "nixosConfigurations.quark.config.system.build.toplevel"
+  update = true
+  update_branch = "nix_update"
+
+[[targets]]
+  name = "hydra"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "nix"
+  path = "nix/configuration"
+  attr = "hydra.vm_iso"
+
+[[targets]]
+  name = "hydra_update"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "nix"
+  path = "nix/configuration"
+  attr = "hydra.vm_iso"
+  update = true
+  update_branch = "nix_update"
+
+[[targets]]
+  name = "controller0"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "kubernetes"
+  path = "nix/kubernetes"
+  attr = "controller0.vm_iso"
+
+[[targets]]
+  name = "controller0_update"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "kubernetes"
+  path = "nix/kubernetes"
+  attr = "controller0.vm_iso"
+  update = true
+  update_branch = "kubernetes_update"
+
+[[targets]]
+  name = "controller1"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "kubernetes"
+  path = "nix/kubernetes"
+  attr = "controller1.vm_iso"
+
+[[targets]]
+  name = "controller1_update"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "kubernetes"
+  path = "nix/kubernetes"
+  attr = "controller1.vm_iso"
+  update = true
+  update_branch = "kubernetes_update"
+
+[[targets]]
+  name = "controller2"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "kubernetes"
+  path = "nix/kubernetes"
+  attr = "controller2.vm_iso"
+
+[[targets]]
+  name = "controller2_update"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "kubernetes"
+  path = "nix/kubernetes"
+  attr = "controller2.vm_iso"
+  update = true
+  update_branch = "kubernetes_update"
+
+[[targets]]
+  name = "worker0"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "kubernetes"
+  path = "nix/kubernetes"
+  attr = "worker0.vm_iso"
+
+[[targets]]
+  name = "worker0_update"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "kubernetes"
+  path = "nix/kubernetes"
+  attr = "worker0.vm_iso"
+  update = true
+  update_branch = "kubernetes_update"
+
+[[targets]]
+  name = "worker1"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "kubernetes"
+  path = "nix/kubernetes"
+  attr = "worker1.vm_iso"
+
+[[targets]]
+  name = "worker1_update"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "kubernetes"
+  path = "nix/kubernetes"
+  attr = "worker1.vm_iso"
+  update = true
+  update_branch = "kubernetes_update"
+
+[[targets]]
+  name = "worker2"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "kubernetes"
+  path = "nix/kubernetes"
+  attr = "worker2.vm_iso"
+
+[[targets]]
+  name = "worker2_update"
+  repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+  branch = "kubernetes"
+  path = "nix/kubernetes"
+  attr = "worker2.vm_iso"
+  update = true
+  update_branch = "kubernetes_update"
+
+# TODO: Add steam deck
--- a/nix/kubernetes/.gitignore
+++ b/nix/kubernetes/.gitignore
@ -0,0 +1 @@
+result
--- a/nix/kubernetes/README.org
+++ b/nix/kubernetes/README.org
@ -0,0 +1,126 @@
+* To-do
+** Perhaps use overlay for /etc for speedup
+#+begin_src nix
+  system.etc.overlay.enable = true;
+#+end_src
+** read https://nixos.org/manual/nixos/stable/
+** Performance for mini pc
+#+begin_src nix
+  security.pam.loginLimits = [
+    { domain = "@users"; item = "rtprio"; type = "-"; value = 1; }
+  ];
+#+end_src
+* IP Ranges
+|               | IPv4                        | IPv6                                    |
+|---------------+-----------------------------+-----------------------------------------|
+| Pod           | 10.200.0.0/16               | 2620:11f:7001:7:ffff:eeee::/96          |
+| Service       | 10.197.0.0/16               | fd00:3e42:e349::/112                    |
+| Node          | 10.215.1.0/24               | 2620:11f:7001:7:ffff:ffff:0ad7:0100/120 |
+| Load Balancer | 74.80.180.139-74.80.180.142 | 2620:11f:7001:7:ffff:dddd::/96          |
+* Healthcheck
+** Check cilium status
+#+begin_src bash
+  kubectl -n kube-system exec ds/cilium -- cilium-dbg status --verbose
+  kubectl -n kube-system exec ds/cilium -- cilium-dbg status | grep KubeProxyReplacement
+#+end_src
+** Check connectivity
+#+begin_src bash
+  cilium connectivity test
+#+end_src
+** Show dropped packets
+#+begin_src bash
+  kubectl -n kube-system exec ds/cilium -- cilium-dbg monitor --type drop
+#+end_src
+** Install flux
+#+begin_src bash
+  nix shell 'nixpkgs#fluxcd'
+
+  flux bootstrap git \
+    --url=ssh://git@<host>/<org>/<repository> \
+    --branch=main \
+    --private-key-file=<path/to/private.key> \
+    --password=<key-passphrase> \
+    --path=clusters/my-cluster
+#+end_src
+
+#+begin_src bash
+  nix shell 'nixpkgs#kubernetes-helm'
+
+  helm template --dry-run=server flux-operator oci://ghcr.io/controlplaneio-fluxcd/charts/flux-operator \
+    --namespace flux-system \
+    --create-namespace
+#+end_src
+
+#+begin_src text
+  apiVersion: fluxcd.controlplane.io/v1
+  kind: FluxInstance
+  metadata:
+    name: flux
+    namespace: flux-system
+    annotations:
+      fluxcd.controlplane.io/reconcileEvery: "1h"
+      fluxcd.controlplane.io/reconcileTimeout: "5m"
+  spec:
+    distribution:
+      version: "2.x"
+      registry: "ghcr.io/fluxcd"
+      artifact: "oci://ghcr.io/controlplaneio-fluxcd/flux-operator-manifests"
+    components:
+      - source-controller
+      - kustomize-controller
+      - helm-controller
+      - notification-controller
+      - image-reflector-controller
+      - image-automation-controller
+    cluster:
+      type: kubernetes
+      size: medium
+      multitenant: false
+      networkPolicy: true
+      domain: "cluster.local"
+    kustomize:
+      patches:
+        - target:
+            kind: Deployment
+          patch: |
+            - op: replace
+              path: /spec/template/spec/nodeSelector
+              value:
+                kubernetes.io/os: linux
+            - op: add
+              path: /spec/template/spec/tolerations
+              value:
+                - key: "CriticalAddonsOnly"
+                  operator: "Exists"
+    sync:
+      kind: OCIRepository
+      url: "oci://ghcr.io/my-org/my-fleet-manifests"
+      ref: "latest"
+      path: "clusters/my-cluster"
+      pullSecret: "ghcr-auth"
+#+end_src
+
+#+begin_src text
+  apiVersion: fluxcd.controlplane.io/v1
+  kind: FluxInstance
+  metadata:
+    name: flux
+    namespace: flux-system
+  spec:
+    distribution:
+      version: "2.7.x"
+      registry: "ghcr.io/fluxcd"
+    sync:
+      kind: GitRepository
+      url: "ssh://git@10.215.1.210:22/repos/mrmanager"
+      ref: "refs/heads/nix"
+      path: "clusters/my-cluster"
+      pullSecret: "flux-system"
+#+end_src
+
+#+begin_src bash
+  flux create secret git flux-system \
+    --url=https://gitlab.com/my-org/my-fleet.git \
+    --username=git \
+    --password=$GITLAB_TOKEN
+#+end_src
--- a/nix/kubernetes/configuration.nix
+++ b/nix/kubernetes/configuration.nix
@ -0,0 +1,148 @@
+{
+  config,
+  lib,
+  ...
+}:
+
+{
+  imports = [
+    ./roles/boot
+    ./roles/cilium
+    ./roles/containerd
+    ./roles/control_plane
+    ./roles/debugging
+    ./roles/doas
+    ./roles/dont_use_substituters
+    ./roles/etcd
+    ./roles/firewall
+    ./roles/image_based_appliance
+    ./roles/iso
+    ./roles/kube_apiserver
+    ./roles/kube_controller_manager
+    ./roles/kube_proxy
+    ./roles/kube_scheduler
+    ./roles/kubelet
+    ./roles/kubernetes
+    ./roles/minimal_base
+    ./roles/network
+    ./roles/nvme
+    ./roles/optimized_build
+    ./roles/ssh
+    ./roles/sshd
+    ./roles/user
+    ./roles/worker_node
+    ./roles/zsh
+    ./util/install_files
+    ./util/unfree_polyfill
+  ];
+
+  config = {
+    nix.settings.experimental-features = [
+      "nix-command"
+      "flakes"
+      "ca-derivations"
+      # "blake3-hashes"
+      # "git-hashing"
+    ];
+    nix.settings.trusted-users = [ "@wheel" ];
+
+    hardware.enableRedistributableFirmware = true;
+
+    # Keep outputs so we can build offline.
+    nix.settings.keep-outputs = true;
+    nix.settings.keep-derivations = true;
+
+    # Automatic garbage collection
+    nix.gc = lib.mkIf (!config.me.buildingPortable) {
+      # Runs nix-collect-garbage --delete-older-than 5d
+      automatic = true;
+      persistent = true;
+      dates = "monthly";
+      # randomizedDelaySec = "14m";
+      options = "--delete-older-than 30d";
+    };
+    nix.settings.auto-optimise-store = !config.me.buildingPortable;
+
+    environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) {
+      hideMounts = true;
+      directories = [
+        "/var/lib/nixos" # Contains user information (uids/gids)
+        "/var/lib/systemd" # Systemd state directory for random seed, persistent timers, core dumps, persist hardware state like backlight and rfkill
+        "/var/log/journal" # Logs, alternatively set `services.journald.storage = "volatile";` to write to /run/log/journal
+      ];
+      files = [
+        "/etc/machine-id" # Systemd unique machine id "otherwise, the system journal may fail to list earlier boots, etc"
+      ];
+    };
+
+    # Write a list of the currently installed packages to /etc/current-system-packages
+    # environment.etc."current-system-packages".text =
+    #   let
+    #     packages = builtins.map (p: "${p.name}") config.environment.systemPackages;
+    #     sortedUnique = builtins.sort builtins.lessThan (lib.unique packages);
+    #     formatted = builtins.concatStringsSep "\n" sortedUnique;
+    #   in
+    #   formatted;
+
+    # nixpkgs.overlays = [
+    #   (final: prev: {
+    #     foot = throw "foo";
+    #   })
+    # ];
+
+    nixpkgs.overlays =
+      let
+        disableTests = (
+          package_name:
+          (final: prev: {
+            "${package_name}" = prev."${package_name}".overrideAttrs (old: {
+              doCheck = false;
+              doInstallCheck = false;
+            });
+          })
+        );
+      in
+      [
+        # (final: prev: {
+        #   imagemagick = prev.imagemagick.overrideAttrs (old: rec {
+        #     # 7.1.2-6 seems to no longer exist, so use 7.1.2-7
+        #     version = "7.1.2-7";
+
+        #     src = final.fetchFromGitHub {
+        #       owner = "ImageMagick";
+        #       repo = "ImageMagick";
+        #       tag = version;
+        #       hash = "sha256-9ARCYftoXiilpJoj+Y+aLCEqLmhHFYSrHfgA5DQHbGo=";
+        #     };
+        #   });
+        # })
+        # (final: prev: {
+        #   grub2 = (final.callPackage ./package/grub { });
+        # })
+        (final: prev: {
+          inherit (final.unoptimized)
+            libtpms
+            ;
+        })
+      ];
+
+    # This option defines the first version of NixOS you have installed on this particular machine,
+    # and is used to maintain compatibility with application data (e.g. databases) created on older NixOS versions.
+    #
+    # Most users should NEVER change this value after the initial install, for any reason,
+    # even if you've upgraded your system to a new NixOS release.
+    #
+    # This value does NOT affect the Nixpkgs version your packages and OS are pulled from,
+    # so changing it will NOT upgrade your system - see https://nixos.org/manual/nixos/stable/#sec-upgrading for how
+    # to actually do that.
+    #
+    # This value being lower than the current NixOS release does NOT mean your system is
+    # out of date, out of support, or vulnerable.
+    #
+    # Do NOT change this value unless you have manually inspected all the changes it would make to your configuration,
+    # and migrated your data accordingly.
+    #
+    # For more information, see `man configuration.nix` or https://nixos.org/manual/nixos/stable/options#opt-system.stateVersion .
+    system.stateVersion = "24.11"; # Did you read the comment?
+  };
+}
--- a/nix/kubernetes/flake.lock
+++ b/nix/kubernetes/flake.lock
@ -0,0 +1,256 @@
+{
+  "nodes": {
+    "crane": {
+      "locked": {
+        "lastModified": 1731098351,
+        "narHash": "sha256-HQkYvKvaLQqNa10KEFGgWHfMAbWBfFp+4cAgkut+NNE=",
+        "owner": "ipetkov",
+        "repo": "crane",
+        "rev": "ef80ead953c1b28316cc3f8613904edc2eb90c28",
+        "type": "github"
+      },
+      "original": {
+        "owner": "ipetkov",
+        "repo": "crane",
+        "type": "github"
+      }
+    },
+    "disko": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1769524058,
+        "narHash": "sha256-zygdD6X1PcVNR2PsyK4ptzrVEiAdbMqLos7utrMDEWE=",
+        "owner": "nix-community",
+        "repo": "disko",
+        "rev": "71a3fc97d80881e91710fe721f1158d3b96ae14d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "disko",
+        "type": "github"
+      }
+    },
+    "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1696426674,
+        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-parts": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "lanzaboote",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1730504689,
+        "narHash": "sha256-hgmguH29K2fvs9szpq2r3pz2/8cJd2LPS+b4tfNFCwE=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "506278e768c2a08bec68eb62932193e341f55c90",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "gitignore": {
+      "inputs": {
+        "nixpkgs": [
+          "lanzaboote",
+          "pre-commit-hooks-nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1709087332,
+        "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=",
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "rev": "637db329424fd7e46cf4185293b9cc8c88c95394",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "type": "github"
+      }
+    },
+    "home-manager": {
+      "inputs": {
+        "nixpkgs": [
+          "impermanence",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1768598210,
+        "narHash": "sha256-kkgA32s/f4jaa4UG+2f8C225Qvclxnqs76mf8zvTVPg=",
+        "owner": "nix-community",
+        "repo": "home-manager",
+        "rev": "c47b2cc64a629f8e075de52e4742de688f930dc6",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "home-manager",
+        "type": "github"
+      }
+    },
+    "impermanence": {
+      "inputs": {
+        "home-manager": "home-manager",
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1769548169,
+        "narHash": "sha256-03+JxvzmfwRu+5JafM0DLbxgHttOQZkUtDWBmeUkN8Y=",
+        "owner": "nix-community",
+        "repo": "impermanence",
+        "rev": "7b1d382faf603b6d264f58627330f9faa5cba149",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "impermanence",
+        "type": "github"
+      }
+    },
+    "lanzaboote": {
+      "inputs": {
+        "crane": "crane",
+        "flake-compat": "flake-compat",
+        "flake-parts": "flake-parts",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "pre-commit-hooks-nix": "pre-commit-hooks-nix",
+        "rust-overlay": "rust-overlay"
+      },
+      "locked": {
+        "lastModified": 1737639419,
+        "narHash": "sha256-AEEDktApTEZ5PZXNDkry2YV2k6t0dTgLPEmAZbnigXU=",
+        "owner": "nix-community",
+        "repo": "lanzaboote",
+        "rev": "a65905a09e2c43ff63be8c0e86a93712361f871e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "ref": "v0.4.2",
+        "repo": "lanzaboote",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1770197578,
+        "narHash": "sha256-AYqlWrX09+HvGs8zM6ebZ1pwUqjkfpnv8mewYwAo+iM=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "00c21e4c93d963c50d4c0c89bfa84ed6e0694df2",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs-stable": {
+      "locked": {
+        "lastModified": 1730741070,
+        "narHash": "sha256-edm8WG19kWozJ/GqyYx2VjW99EdhjKwbY3ZwdlPAAlo=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "d063c1dd113c91ab27959ba540c0d9753409edf3",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-24.05",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "pre-commit-hooks-nix": {
+      "inputs": {
+        "flake-compat": [
+          "lanzaboote",
+          "flake-compat"
+        ],
+        "gitignore": "gitignore",
+        "nixpkgs": [
+          "lanzaboote",
+          "nixpkgs"
+        ],
+        "nixpkgs-stable": "nixpkgs-stable"
+      },
+      "locked": {
+        "lastModified": 1731363552,
+        "narHash": "sha256-vFta1uHnD29VUY4HJOO/D6p6rxyObnf+InnSMT4jlMU=",
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "rev": "cd1af27aa85026ac759d5d3fccf650abe7e1bbf0",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "disko": "disko",
+        "impermanence": "impermanence",
+        "lanzaboote": "lanzaboote",
+        "nixpkgs": "nixpkgs"
+      }
+    },
+    "rust-overlay": {
+      "inputs": {
+        "nixpkgs": [
+          "lanzaboote",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1731897198,
+        "narHash": "sha256-Ou7vLETSKwmE/HRQz4cImXXJBr/k9gp4J4z/PF8LzTE=",
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "rev": "0be641045af6d8666c11c2c40e45ffc9667839b5",
+        "type": "github"
+      },
+      "original": {
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}
--- a/nix/kubernetes/flake.nix
+++ b/nix/kubernetes/flake.nix
@ -0,0 +1,183 @@
+# Get a repl for this flake
+#   nix repl --expr "builtins.getFlake \"$PWD\""
+
+# TODO maybe use `nix eval --raw .#odo.iso.outPath`
+
+#
+# Install on a new machine:
+#
+# Set
+#   me.disko.enable = true;
+#   me.disko.offline.enable = true;
+#
+# Run
+#   doas disko --mode destroy,format,mount hosts/recovery/disk-config.nix
+#   doas nixos-install --substituters "http://10.0.2.2:8080?trusted=1 https://cache.nixos.org/" --flake ".#recovery"
+
+{
+  description = "My system configuration";
+
+  inputs = {
+    impermanence = {
+      url = "github:nix-community/impermanence";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+    lanzaboote = {
+      url = "github:nix-community/lanzaboote/v0.4.2";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+    disko = {
+      url = "github:nix-community/disko";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+  };
+
+  outputs =
+    {
+      self,
+      nixpkgs,
+      disko,
+      impermanence,
+      lanzaboote,
+      ...
+    }:
+    let
+      forAllSystems =
+        func:
+        builtins.listToAttrs (
+          map (system: {
+            name = system;
+            value = func system;
+          }) nixpkgs.lib.systems.flakeExposed
+        );
+      nodes = {
+        controller0 = {
+          system = "x86_64-linux";
+        };
+        controller1 = {
+          system = "x86_64-linux";
+        };
+        controller2 = {
+          system = "x86_64-linux";
+        };
+        worker0 = {
+          system = "x86_64-linux";
+        };
+        worker1 = {
+          system = "x86_64-linux";
+        };
+        worker2 = {
+          system = "x86_64-linux";
+        };
+      };
+      nixosConfigs = builtins.mapAttrs (
+        hostname: nodeConfig: format:
+        nixpkgs.lib.nixosSystem {
+          specialArgs = {
+            inherit self;
+
+            this_nixos_config = self.nixosConfigurations."${hostname}";
+
+            all_nixos_configs = self.nixosConfigurations;
+          };
+          modules = [
+            impermanence.nixosModules.impermanence
+            lanzaboote.nixosModules.lanzaboote
+            disko.nixosModules.disko
+            ./configuration.nix
+            (./. + "/hosts/${hostname}")
+            (./. + "/formats/${format}.nix")
+            {
+              config = {
+                nixpkgs.hostPlatform.system = nodeConfig.system;
+                nixpkgs.overlays = [
+                  (final: prev: {
+                    # stable = nixpkgs-stable.legacyPackages."${prev.stdenv.hostPlatform.system}";
+                    unoptimized = import nixpkgs {
+                      system = prev.stdenv.hostPlatform.system;
+                      hostPlatform.gcc.arch = "default";
+                      hostPlatform.gcc.tune = "default";
+                    };
+                  })
+                ];
+              };
+            }
+            (
+              {
+                config,
+                lib,
+                pkgs,
+                ...
+              }:
+              let
+                nix-self-repl = pkgs.writeShellScriptBin "nix-self-repl" ''
+                  source /etc/set-environment
+                  nix repl --expr 'builtins.getFlake "${self}"'
+                '';
+                # If we wanted the current version of a flake then we'd just launch
+                #   nix repl
+                # and then run:
+                #   :lf /path/to/flake
+              in
+              {
+                config = {
+                  environment.systemPackages = lib.mkIf config.nix.enable [ nix-self-repl ];
+                };
+              }
+            )
+          ];
+        }
+      ) nodes;
+      installerConfig =
+        hostname: nodeConfig:
+        nixpkgs.lib.nixosSystem {
+          specialArgs = {
+            targetSystem = self.nixosConfigurations."${hostname}";
+          };
+          modules = [
+            ./formats/installer.nix
+            (
+              {
+                config,
+                lib,
+                pkgs,
+                ...
+              }:
+              let
+                nix-self-repl = pkgs.writeShellScriptBin "nix-self-repl" ''
+                  source /etc/set-environment
+                  nix repl --expr 'builtins.getFlake "${self}"'
+                '';
+                # If we wanted the current version of a flake then we'd just launch
+                #   nix repl
+                # and then run:
+                #   :lf /path/to/flake
+              in
+              {
+                config = {
+                  environment.systemPackages = lib.mkIf config.nix.enable [ nix-self-repl ];
+                };
+              }
+            )
+            ({ nixpkgs.hostPlatform.system = nodeConfig.system; })
+          ];
+        };
+    in
+    {
+      nixosConfigurations = (builtins.mapAttrs (name: value: value "toplevel") nixosConfigs);
+    }
+    // {
+      packages = (
+        forAllSystems (
+          system:
+          (builtins.mapAttrs (hostname: nodeConfig: {
+            iso = (nixosConfigs."${hostname}" "iso").config.system.build.isoImage;
+            vm_iso = (nixosConfigs."${hostname}" "vm_iso").config.system.build.isoImage;
+            sd = (nixosConfigs."${hostname}" "sd").config.system.build.sdImage;
+            installer = (installerConfig hostname nodes."${hostname}").config.system.build.isoImage;
+          }) (nixpkgs.lib.attrsets.filterAttrs (hostname: nodeConfig: nodeConfig.system == system) nodes))
+        )
+      );
+    };
+}
--- a/nix/kubernetes/formats/installer.nix
+++ b/nix/kubernetes/formats/installer.nix
@ -0,0 +1,74 @@
+{
+  config,
+  pkgs,
+  lib,
+  modulesPath,
+  targetSystem,
+  ...
+}:
+let
+  installer = pkgs.writeShellApplication {
+    name = "installer";
+    runtimeInputs = with pkgs; [
+      # clevis
+      dosfstools
+      e2fsprogs
+      gawk
+      nixos-install-tools
+      util-linux
+      config.nix.package
+    ];
+    text = ''
+      set -euo pipefail
+
+      ${targetSystem.config.system.build.diskoScript}
+
+      nixos-install --no-channel-copy --no-root-password --option substituters "" --system ${targetSystem.config.system.build.toplevel}
+    '';
+  };
+  installerFailsafe = pkgs.writeShellScript "failsafe" ''
+    ${lib.getExe installer} || echo "ERROR: Installation failure!"
+    sleep 3600
+  '';
+in
+{
+  imports = [
+    (modulesPath + "/installer/cd-dvd/iso-image.nix")
+    (modulesPath + "/profiles/all-hardware.nix")
+  ];
+
+  # boot.kernelPackages = pkgs.linuxPackagesFor pkgs.linux_6_17;
+  # boot.zfs.package = pkgs.zfs_unstable;
+  boot.kernelPackages = pkgs.linuxPackagesFor pkgs.linux;
+  boot.kernelParams = [
+    "quiet"
+    "systemd.unit=getty.target"
+  ];
+  boot.supportedFilesystems.zfs = true;
+  boot.initrd.systemd.enable = true;
+
+  networking.hostId = "04581ecf";
+
+  isoImage.makeEfiBootable = true;
+  isoImage.makeUsbBootable = true;
+  isoImage.squashfsCompression = "zstd -Xcompression-level 15";
+
+  environment.systemPackages = [
+    installer
+  ];
+
+  systemd.services."getty@tty1" = {
+    overrideStrategy = "asDropin";
+    serviceConfig = {
+      ExecStart = [
+        ""
+        installerFailsafe
+      ];
+      Restart = "no";
+      StandardInput = "null";
+    };
+  };
+
+  # system.stateVersion = lib.mkDefault lib.trivial.release;
+  system.stateVersion = "24.11";
+}
--- a/nix/kubernetes/formats/iso.nix
+++ b/nix/kubernetes/formats/iso.nix
@ -0,0 +1,36 @@
+{
+  config,
+  lib,
+  modulesPath,
+  pkgs,
+  ...
+}:
+
+{
+  imports = [
+    (modulesPath + "/installer/cd-dvd/iso-image.nix")
+  ];
+
+  config = {
+    isoImage.makeEfiBootable = true;
+    isoImage.makeUsbBootable = true;
+
+    networking.dhcpcd.enable = true;
+    networking.useDHCP = true;
+
+    me.buildingPortable = true;
+    me.disko.enable = true;
+    me.disko.offline.enable = true;
+    me.mountPersistence = lib.mkForce false;
+    # me.optimizations.enable = lib.mkForce false;
+
+    # Not doing image_based_appliance because this might be an install ISO, in which case we'd need nix to do the install.
+    # me.image_based_appliance.enable = true;
+
+    # TODO: Should I use this instead of doing a mkIf for the disk config?
+    # disko.enableConfig = false;
+
+    # Faster image generation for testing/development.
+    isoImage.squashfsCompression = "zstd -Xcompression-level 15";
+  };
+}
--- a/nix/kubernetes/formats/sd.nix
+++ b/nix/kubernetes/formats/sd.nix
@ -0,0 +1,32 @@
+{
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [
+    (modulesPath + "/installer/sd-card/sd-image.nix")
+  ];
+
+  config = {
+    isoImage.makeEfiBootable = true;
+    isoImage.makeUsbBootable = true;
+
+    boot.loader.grub.enable = false;
+    boot.loader.generic-extlinux-compatible.enable = true;
+
+    # TODO: image based appliance?
+
+    # TODO: Maybe this?
+    # fileSystems = {
+    #   "/" = {
+    #     device = "/dev/disk/by-label/NIXOS_SD";
+    #     fsType = "ext4";
+    #     options = [
+    #       "noatime"
+    #       "norelatime"
+    #     ];
+    #   };
+    # };
+  };
+}
--- a/nix/kubernetes/formats/toplevel.nix
+++ b/nix/kubernetes/formats/toplevel.nix
@ -0,0 +1 @@
+{ }
--- a/nix/kubernetes/formats/vm_iso.nix
+++ b/nix/kubernetes/formats/vm_iso.nix
@ -0,0 +1,22 @@
+{
+  lib,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [
+    (modulesPath + "/installer/cd-dvd/iso-image.nix")
+    (modulesPath + "/profiles/qemu-guest.nix") # VirtIO kernel modules
+  ];
+
+  config = {
+    isoImage.makeEfiBootable = true;
+    isoImage.makeUsbBootable = true;
+
+    networking.dhcpcd.enable = true;
+    networking.useDHCP = true;
+
+    me.image_based_appliance.enable = true;
+  };
+}
--- a/nix/kubernetes/functions/to_yaml.nix
+++ b/nix/kubernetes/functions/to_yaml.nix
@ -0,0 +1,24 @@
+{
+  pkgs,
+  ...
+}:
+let
+  to_yaml_file =
+    file_name: contents:
+    let
+      settingsFormat = pkgs.formats.yaml { };
+      yaml_file = settingsFormat.generate file_name contents;
+    in
+    yaml_file;
+  to_yaml =
+    file_name: contents:
+    let
+      settingsFormat = pkgs.formats.yaml { };
+      yaml_file = settingsFormat.generate file_name contents;
+      yaml_content = builtins.readFile yaml_file;
+    in
+    yaml_content;
+in
+{
+  inherit to_yaml to_yaml_file;
+}
--- a/nix/kubernetes/hosts/controller0/DEPLOY_BOOT
+++ b/nix/kubernetes/hosts/controller0/DEPLOY_BOOT
@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+TARGET=controller0
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+
+nixos-rebuild boot --flake "$DIR/../../#controller0" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/controller0/DEPLOY_SWITCH
+++ b/nix/kubernetes/hosts/controller0/DEPLOY_SWITCH
@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+TARGET=controller0
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+
+nixos-rebuild switch --flake "$DIR/../../#controller0" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/controller0/ISO
+++ b/nix/kubernetes/hosts/controller0/ISO
@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#controller0.iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/controller0/SELF_BOOT
+++ b/nix/kubernetes/hosts/controller0/SELF_BOOT
@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nixos-rebuild boot --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller0" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/controller0/SELF_BUILD
+++ b/nix/kubernetes/hosts/controller0/SELF_BUILD
@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nixos-rebuild build --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller0" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/controller0/SELF_SWITCH
+++ b/nix/kubernetes/hosts/controller0/SELF_SWITCH
@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nixos-rebuild switch --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller0" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/controller0/VM_ISO
+++ b/nix/kubernetes/hosts/controller0/VM_ISO
@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#controller0.vm_iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/controller0/default.nix
+++ b/nix/kubernetes/hosts/controller0/default.nix
@ -0,0 +1,130 @@
+# MANUAL: On client machines generate signing keys:
+#   nix-store --generate-binary-cache-key some-name /persist/manual/nix/nix-cache-key.sec /persist/manual/nix/nix-cache-key.pub
+#
+# Trust other machines and add the substituters:
+#   nix.binaryCachePublicKeys = [ "some-name:AzNW1MOlkNEsUAXS1jIFZ1QCFKXjV+Y/LrF37quAZ1A=" ];
+#   nix.binaryCaches = [ "https://test.example/nix-cache" ];
+
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  imports = [
+    ./hardware-configuration.nix
+    ./vm_disk.nix
+  ];
+
+  config = {
+    networking =
+      let
+        interface = "enp0s2";
+      in
+      {
+        # Generate with `head -c4 /dev/urandom | od -A none -t x4`
+        hostId = "769e1349";
+
+        hostName = "controller0"; # Define your hostname.
+
+        interfaces = {
+          "${interface}" = {
+            ipv4.addresses = [
+              {
+                address = "10.215.1.221";
+                prefixLength = 24;
+              }
+            ];
+
+            ipv6.addresses = [
+              {
+                address = "2620:11f:7001:7:ffff:ffff:0ad7:01dd";
+                prefixLength = 64;
+              }
+            ];
+          };
+        };
+        defaultGateway = "10.215.1.1";
+        defaultGateway6 = {
+          # address = "2620:11f:7001:7::1";
+          address = "2620:11f:7001:7:ffff:ffff:0ad7:0101";
+          inherit interface;
+        };
+
+        dhcpcd.enable = lib.mkForce false;
+        useDHCP = lib.mkForce false;
+      };
+
+    time.timeZone = "America/New_York";
+    i18n.defaultLocale = "en_US.UTF-8";
+
+    me.boot.enable = true;
+    me.boot.secure = false;
+    me.mountPersistence = true;
+    boot.loader.timeout = lib.mkForce 0; # We can always generate a new ISO if we need to access other boot options.
+
+    me.optimizations = {
+      enable = true;
+      arch = "znver4";
+      # build_arch = "x86-64-v3";
+      system_features = [
+        "gccarch-znver4"
+        "gccarch-skylake"
+        "gccarch-kabylake"
+        # "gccarch-alderlake" missing WAITPKG
+        "gccarch-x86-64-v3"
+        "gccarch-x86-64-v4"
+        "benchmark"
+        "big-parallel"
+        "kvm"
+        "nixos-test"
+      ];
+    };
+
+    # Mount tmpfs at /tmp
+    boot.tmp.useTmpfs = true;
+
+    # Enable TRIM
+    # services.fstrim.enable = lib.mkDefault true;
+
+    # nix.optimise.automatic = true;
+    # nix.optimise.dates = [ "03:45" ];
+    # nix.optimise.persistent = true;
+
+    environment.systemPackages = with pkgs; [
+      htop
+    ];
+
+    # nix.sshServe.enable = true;
+    # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ];
+
+    me.etcd.cluster_name = "put-a-nix-on-it";
+    me.etcd.internal_ip = [
+      # "10.215.1.221"
+      "[2620:11f:7001:7:ffff:ffff:0ad7:01dd]"
+    ];
+    me.etcd.initial_cluster = [
+      # "controller0=https://10.215.1.221:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01dd
+      # "controller1=https://10.215.1.222:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01de
+      # "controller2=https://10.215.1.223:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01df
+      "controller0=https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2380" # 10.215.1.221
+      "controller1=https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2380" # 10.215.1.222
+      "controller2=https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2380" # 10.215.1.223
+    ];
+
+    me.kube_apiserver.internal_ip = "2620:11f:7001:7:ffff:ffff:0ad7:01dd";
+    # me.kube_apiserver.external_ip = "74.80.180.138";
+    me.kube_apiserver.external_ip = "2620:11f:7001:7:ffff:ffff:0ad7:01dd";
+    me.kube_apiserver.etcd_services = [
+      "https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2379" # 10.215.1.221
+      "https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2379" # 10.215.1.222
+      "https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2379" # 10.215.1.223
+    ];
+
+    me.control_plane.enable = true;
+    me.dont_use_substituters.enable = true;
+    me.etcd.enable = true;
+    me.minimal_base.enable = true;
+  };
+}
--- a/nix/kubernetes/hosts/controller0/hardware-configuration.nix
+++ b/nix/kubernetes/hosts/controller0/hardware-configuration.nix
@ -0,0 +1,31 @@
+{
+  config,
+  lib,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [
+    (modulesPath + "/installer/scan/not-detected.nix")
+  ];
+
+  config = {
+    boot.initrd.availableKernelModules = [
+      "nvme"
+      "xhci_pci"
+      "thunderbolt"
+    ];
+    boot.initrd.kernelModules = [ ];
+    boot.kernelModules = [ ];
+    boot.extraModulePackages = [ ];
+
+    # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+    # (the default) this is the recommended approach. When using systemd-networkd it's
+    # still possible to use this option, but it's recommended to use it in conjunction
+    # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+    # networking.useDHCP = lib.mkDefault true;
+    # networking.interfaces.eno1.useDHCP = lib.mkDefault true;
+    # networking.interfaces.wlp58s0.useDHCP = lib.mkDefault true;
+  };
+}
--- a/nix/kubernetes/hosts/controller0/vm_disk.nix
+++ b/nix/kubernetes/hosts/controller0/vm_disk.nix
@ -0,0 +1,94 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  imports = [ ];
+
+  config = {
+    # Mount the local disk
+    fileSystems = lib.mkIf config.me.mountPersistence {
+      "/.disk" = lib.mkForce {
+        device = "/dev/nvme0n1p1";
+        fsType = "ext4";
+        options = [
+          "noatime"
+          "discard"
+        ];
+        neededForBoot = true;
+      };
+
+      "/.persist" = lib.mkForce {
+        device = "bind9p";
+        fsType = "9p";
+        options = [
+          "noatime"
+          "trans=virtio"
+          "version=9p2000.L"
+          "cache=mmap"
+          "msize=512000"
+          # "noauto"
+          # "x-systemd.automount"
+        ];
+        neededForBoot = true;
+      };
+
+      "/persist" = {
+        fsType = "none";
+        device = "/.persist/persist";
+        options = [
+          "bind"
+          "rw"
+        ];
+        depends = [
+          "/.persist/persist"
+        ];
+        neededForBoot = true;
+      };
+
+      "/state" = {
+        fsType = "none";
+        device = "/.persist/state";
+        options = [
+          "bind"
+          "rw"
+        ];
+        depends = [
+          "/.persist/state"
+        ];
+        neededForBoot = true;
+      };
+
+      "/k8spv" = lib.mkForce {
+        device = "k8spv";
+        fsType = "9p";
+        options = [
+          "noatime"
+          "trans=virtio"
+          "version=9p2000.L"
+          "cache=mmap"
+          "msize=512000"
+          # "noauto"
+          # "x-systemd.automount"
+        ];
+        neededForBoot = true;
+      };
+
+      "/disk" = {
+        fsType = "none";
+        device = "/.disk/persist";
+        options = [
+          "bind"
+          "rw"
+        ];
+        depends = [
+          "/.disk/persist"
+        ];
+        neededForBoot = true;
+      };
+    };
+  };
+}
--- a/nix/kubernetes/hosts/controller1/DEPLOY_BOOT
+++ b/nix/kubernetes/hosts/controller1/DEPLOY_BOOT
@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+TARGET=controller1
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+
+nixos-rebuild boot --flake "$DIR/../../#controller1" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/controller1/DEPLOY_SWITCH
+++ b/nix/kubernetes/hosts/controller1/DEPLOY_SWITCH
@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+TARGET=controller1
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+
+nixos-rebuild switch --flake "$DIR/../../#controller1" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/controller1/ISO
+++ b/nix/kubernetes/hosts/controller1/ISO
@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#controller1.iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/controller1/SELF_BOOT
+++ b/nix/kubernetes/hosts/controller1/SELF_BOOT
@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nixos-rebuild boot --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller1" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/controller1/SELF_BUILD
+++ b/nix/kubernetes/hosts/controller1/SELF_BUILD
@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nixos-rebuild build --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller1" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/controller1/SELF_SWITCH
+++ b/nix/kubernetes/hosts/controller1/SELF_SWITCH
@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nixos-rebuild switch --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller1" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/controller1/VM_ISO
+++ b/nix/kubernetes/hosts/controller1/VM_ISO
@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#controller1.vm_iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/controller1/default.nix
+++ b/nix/kubernetes/hosts/controller1/default.nix
@ -0,0 +1,130 @@
+# MANUAL: On client machines generate signing keys:
+#   nix-store --generate-binary-cache-key some-name /persist/manual/nix/nix-cache-key.sec /persist/manual/nix/nix-cache-key.pub
+#
+# Trust other machines and add the substituters:
+#   nix.binaryCachePublicKeys = [ "some-name:AzNW1MOlkNEsUAXS1jIFZ1QCFKXjV+Y/LrF37quAZ1A=" ];
+#   nix.binaryCaches = [ "https://test.example/nix-cache" ];
+
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  imports = [
+    ./hardware-configuration.nix
+    ./vm_disk.nix
+  ];
+
+  config = {
+    networking =
+      let
+        interface = "enp0s2";
+      in
+      {
+        # Generate with `head -c4 /dev/urandom | od -A none -t x4`
+        hostId = "59a83979";
+
+        hostName = "controller1"; # Define your hostname.
+
+        interfaces = {
+          "${interface}" = {
+            ipv4.addresses = [
+              {
+                address = "10.215.1.222";
+                prefixLength = 24;
+              }
+            ];
+
+            ipv6.addresses = [
+              {
+                address = "2620:11f:7001:7:ffff:ffff:0ad7:01de";
+                prefixLength = 64;
+              }
+            ];
+          };
+        };
+        defaultGateway = "10.215.1.1";
+        defaultGateway6 = {
+          # address = "2620:11f:7001:7::1";
+          address = "2620:11f:7001:7:ffff:ffff:0ad7:0101";
+          inherit interface;
+        };
+
+        dhcpcd.enable = lib.mkForce false;
+        useDHCP = lib.mkForce false;
+      };
+
+    time.timeZone = "America/New_York";
+    i18n.defaultLocale = "en_US.UTF-8";
+
+    me.boot.enable = true;
+    me.boot.secure = false;
+    me.mountPersistence = true;
+    boot.loader.timeout = lib.mkForce 0; # We can always generate a new ISO if we need to access other boot options.
+
+    me.optimizations = {
+      enable = true;
+      arch = "znver4";
+      # build_arch = "x86-64-v3";
+      system_features = [
+        "gccarch-znver4"
+        "gccarch-skylake"
+        "gccarch-kabylake"
+        # "gccarch-alderlake" missing WAITPKG
+        "gccarch-x86-64-v3"
+        "gccarch-x86-64-v4"
+        "benchmark"
+        "big-parallel"
+        "kvm"
+        "nixos-test"
+      ];
+    };
+
+    # Mount tmpfs at /tmp
+    boot.tmp.useTmpfs = true;
+
+    # Enable TRIM
+    # services.fstrim.enable = lib.mkDefault true;
+
+    # nix.optimise.automatic = true;
+    # nix.optimise.dates = [ "03:45" ];
+    # nix.optimise.persistent = true;
+
+    environment.systemPackages = with pkgs; [
+      htop
+    ];
+
+    # nix.sshServe.enable = true;
+    # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ];
+
+    me.etcd.cluster_name = "put-a-nix-on-it";
+    me.etcd.internal_ip = [
+      # "10.215.1.221"
+      "[2620:11f:7001:7:ffff:ffff:0ad7:01de]"
+    ];
+    me.etcd.initial_cluster = [
+      # "controller0=https://10.215.1.221:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01dd
+      # "controller1=https://10.215.1.222:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01de
+      # "controller2=https://10.215.1.223:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01df
+      "controller0=https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2380" # 10.215.1.221
+      "controller1=https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2380" # 10.215.1.222
+      "controller2=https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2380" # 10.215.1.223
+    ];
+
+    me.kube_apiserver.internal_ip = "2620:11f:7001:7:ffff:ffff:0ad7:01de";
+    # me.kube_apiserver.external_ip = "74.80.180.138";
+    me.kube_apiserver.external_ip = "2620:11f:7001:7:ffff:ffff:0ad7:01de";
+    me.kube_apiserver.etcd_services = [
+      "https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2379" # 10.215.1.221
+      "https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2379" # 10.215.1.222
+      "https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2379" # 10.215.1.223
+    ];
+
+    me.control_plane.enable = true;
+    me.dont_use_substituters.enable = true;
+    me.etcd.enable = true;
+    me.minimal_base.enable = true;
+  };
+}
--- a/nix/kubernetes/hosts/controller1/hardware-configuration.nix
+++ b/nix/kubernetes/hosts/controller1/hardware-configuration.nix
@ -0,0 +1,31 @@
+{
+  config,
+  lib,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [
+    (modulesPath + "/installer/scan/not-detected.nix")
+  ];
+
+  config = {
+    boot.initrd.availableKernelModules = [
+      "nvme"
+      "xhci_pci"
+      "thunderbolt"
+    ];
+    boot.initrd.kernelModules = [ ];
+    boot.kernelModules = [ ];
+    boot.extraModulePackages = [ ];
+
+    # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+    # (the default) this is the recommended approach. When using systemd-networkd it's
+    # still possible to use this option, but it's recommended to use it in conjunction
+    # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+    # networking.useDHCP = lib.mkDefault true;
+    # networking.interfaces.eno1.useDHCP = lib.mkDefault true;
+    # networking.interfaces.wlp58s0.useDHCP = lib.mkDefault true;
+  };
+}
--- a/nix/kubernetes/hosts/controller1/vm_disk.nix
+++ b/nix/kubernetes/hosts/controller1/vm_disk.nix
@ -0,0 +1,94 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  imports = [ ];
+
+  config = {
+    # Mount the local disk
+    fileSystems = lib.mkIf config.me.mountPersistence {
+      "/.disk" = lib.mkForce {
+        device = "/dev/nvme0n1p1";
+        fsType = "ext4";
+        options = [
+          "noatime"
+          "discard"
+        ];
+        neededForBoot = true;
+      };
+
+      "/.persist" = lib.mkForce {
+        device = "bind9p";
+        fsType = "9p";
+        options = [
+          "noatime"
+          "trans=virtio"
+          "version=9p2000.L"
+          "cache=mmap"
+          "msize=512000"
+          # "noauto"
+          # "x-systemd.automount"
+        ];
+        neededForBoot = true;
+      };
+
+      "/persist" = {
+        fsType = "none";
+        device = "/.persist/persist";
+        options = [
+          "bind"
+          "rw"
+        ];
+        depends = [
+          "/.persist/persist"
+        ];
+        neededForBoot = true;
+      };
+
+      "/state" = {
+        fsType = "none";
+        device = "/.persist/state";
+        options = [
+          "bind"
+          "rw"
+        ];
+        depends = [
+          "/.persist/state"
+        ];
+        neededForBoot = true;
+      };
+
+      "/k8spv" = lib.mkForce {
+        device = "k8spv";
+        fsType = "9p";
+        options = [
+          "noatime"
+          "trans=virtio"
+          "version=9p2000.L"
+          "cache=mmap"
+          "msize=512000"
+          # "noauto"
+          # "x-systemd.automount"
+        ];
+        neededForBoot = true;
+      };
+
+      "/disk" = {
+        fsType = "none";
+        device = "/.disk/persist";
+        options = [
+          "bind"
+          "rw"
+        ];
+        depends = [
+          "/.disk/persist"
+        ];
+        neededForBoot = true;
+      };
+    };
+  };
+}
--- a/nix/kubernetes/hosts/controller2/DEPLOY_BOOT
+++ b/nix/kubernetes/hosts/controller2/DEPLOY_BOOT
@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+TARGET=controller2
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+
+nixos-rebuild boot --flake "$DIR/../../#controller2" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/controller2/DEPLOY_SWITCH
+++ b/nix/kubernetes/hosts/controller2/DEPLOY_SWITCH
@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+TARGET=controller2
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+
+nixos-rebuild switch --flake "$DIR/../../#controller2" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/controller2/ISO
+++ b/nix/kubernetes/hosts/controller2/ISO
@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#controller2.iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/controller2/SELF_BOOT
+++ b/nix/kubernetes/hosts/controller2/SELF_BOOT
@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nixos-rebuild boot --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller2" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/controller2/SELF_BUILD
+++ b/nix/kubernetes/hosts/controller2/SELF_BUILD
@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nixos-rebuild build --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller2" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/controller2/SELF_SWITCH
+++ b/nix/kubernetes/hosts/controller2/SELF_SWITCH
@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nixos-rebuild switch --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller2" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/controller2/VM_ISO
+++ b/nix/kubernetes/hosts/controller2/VM_ISO
@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#controller2.vm_iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/controller2/default.nix
+++ b/nix/kubernetes/hosts/controller2/default.nix
@ -0,0 +1,130 @@
+# MANUAL: On client machines generate signing keys:
+#   nix-store --generate-binary-cache-key some-name /persist/manual/nix/nix-cache-key.sec /persist/manual/nix/nix-cache-key.pub
+#
+# Trust other machines and add the substituters:
+#   nix.binaryCachePublicKeys = [ "some-name:AzNW1MOlkNEsUAXS1jIFZ1QCFKXjV+Y/LrF37quAZ1A=" ];
+#   nix.binaryCaches = [ "https://test.example/nix-cache" ];
+
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  imports = [
+    ./hardware-configuration.nix
+    ./vm_disk.nix
+  ];
+
+  config = {
+    networking =
+      let
+        interface = "enp0s2";
+      in
+      {
+        # Generate with `head -c4 /dev/urandom | od -A none -t x4`
+        hostId = "26a43660";
+
+        hostName = "controller2"; # Define your hostname.
+
+        interfaces = {
+          "${interface}" = {
+            ipv4.addresses = [
+              {
+                address = "10.215.1.223";
+                prefixLength = 24;
+              }
+            ];
+
+            ipv6.addresses = [
+              {
+                address = "2620:11f:7001:7:ffff:ffff:0ad7:01df";
+                prefixLength = 64;
+              }
+            ];
+          };
+        };
+        defaultGateway = "10.215.1.1";
+        defaultGateway6 = {
+          # address = "2620:11f:7001:7::1";
+          address = "2620:11f:7001:7:ffff:ffff:0ad7:0101";
+          inherit interface;
+        };
+
+        dhcpcd.enable = lib.mkForce false;
+        useDHCP = lib.mkForce false;
+      };
+
+    time.timeZone = "America/New_York";
+    i18n.defaultLocale = "en_US.UTF-8";
+
+    me.boot.enable = true;
+    me.boot.secure = false;
+    me.mountPersistence = true;
+    boot.loader.timeout = lib.mkForce 0; # We can always generate a new ISO if we need to access other boot options.
+
+    me.optimizations = {
+      enable = true;
+      arch = "znver4";
+      # build_arch = "x86-64-v3";
+      system_features = [
+        "gccarch-znver4"
+        "gccarch-skylake"
+        "gccarch-kabylake"
+        # "gccarch-alderlake" missing WAITPKG
+        "gccarch-x86-64-v3"
+        "gccarch-x86-64-v4"
+        "benchmark"
+        "big-parallel"
+        "kvm"
+        "nixos-test"
+      ];
+    };
+
+    # Mount tmpfs at /tmp
+    boot.tmp.useTmpfs = true;
+
+    # Enable TRIM
+    # services.fstrim.enable = lib.mkDefault true;
+
+    # nix.optimise.automatic = true;
+    # nix.optimise.dates = [ "03:45" ];
+    # nix.optimise.persistent = true;
+
+    environment.systemPackages = with pkgs; [
+      htop
+    ];
+
+    # nix.sshServe.enable = true;
+    # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ];
+
+    me.etcd.cluster_name = "put-a-nix-on-it";
+    me.etcd.internal_ip = [
+      # "10.215.1.221"
+      "[2620:11f:7001:7:ffff:ffff:0ad7:01df]"
+    ];
+    me.etcd.initial_cluster = [
+      # "controller0=https://10.215.1.221:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01dd
+      # "controller1=https://10.215.1.222:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01de
+      # "controller2=https://10.215.1.223:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01df
+      "controller0=https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2380" # 10.215.1.221
+      "controller1=https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2380" # 10.215.1.222
+      "controller2=https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2380" # 10.215.1.223
+    ];
+
+    me.kube_apiserver.internal_ip = "2620:11f:7001:7:ffff:ffff:0ad7:01df";
+    # me.kube_apiserver.external_ip = "74.80.180.138";
+    me.kube_apiserver.external_ip = "2620:11f:7001:7:ffff:ffff:0ad7:01df";
+    me.kube_apiserver.etcd_services = [
+      "https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2379" # 10.215.1.221
+      "https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2379" # 10.215.1.222
+      "https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2379" # 10.215.1.223
+    ];
+
+    me.control_plane.enable = true;
+    me.dont_use_substituters.enable = true;
+    me.etcd.enable = true;
+    me.minimal_base.enable = true;
+  };
+}
--- a/nix/kubernetes/hosts/controller2/hardware-configuration.nix
+++ b/nix/kubernetes/hosts/controller2/hardware-configuration.nix
@ -0,0 +1,31 @@
+{
+  config,
+  lib,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [
+    (modulesPath + "/installer/scan/not-detected.nix")
+  ];
+
+  config = {
+    boot.initrd.availableKernelModules = [
+      "nvme"
+      "xhci_pci"
+      "thunderbolt"
+    ];
+    boot.initrd.kernelModules = [ ];
+    boot.kernelModules = [ ];
+    boot.extraModulePackages = [ ];
+
+    # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+    # (the default) this is the recommended approach. When using systemd-networkd it's
+    # still possible to use this option, but it's recommended to use it in conjunction
+    # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+    # networking.useDHCP = lib.mkDefault true;
+    # networking.interfaces.eno1.useDHCP = lib.mkDefault true;
+    # networking.interfaces.wlp58s0.useDHCP = lib.mkDefault true;
+  };
+}
--- a/nix/kubernetes/hosts/controller2/vm_disk.nix
+++ b/nix/kubernetes/hosts/controller2/vm_disk.nix
@ -0,0 +1,94 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  imports = [ ];
+
+  config = {
+    # Mount the local disk
+    fileSystems = lib.mkIf config.me.mountPersistence {
+      "/.disk" = lib.mkForce {
+        device = "/dev/nvme0n1p1";
+        fsType = "ext4";
+        options = [
+          "noatime"
+          "discard"
+        ];
+        neededForBoot = true;
+      };
+
+      "/.persist" = lib.mkForce {
+        device = "bind9p";
+        fsType = "9p";
+        options = [
+          "noatime"
+          "trans=virtio"
+          "version=9p2000.L"
+          "cache=mmap"
+          "msize=512000"
+          # "noauto"
+          # "x-systemd.automount"
+        ];
+        neededForBoot = true;
+      };
+
+      "/persist" = {
+        fsType = "none";
+        device = "/.persist/persist";
+        options = [
+          "bind"
+          "rw"
+        ];
+        depends = [
+          "/.persist/persist"
+        ];
+        neededForBoot = true;
+      };
+
+      "/state" = {
+        fsType = "none";
+        device = "/.persist/state";
+        options = [
+          "bind"
+          "rw"
+        ];
+        depends = [
+          "/.persist/state"
+        ];
+        neededForBoot = true;
+      };
+
+      "/k8spv" = lib.mkForce {
+        device = "k8spv";
+        fsType = "9p";
+        options = [
+          "noatime"
+          "trans=virtio"
+          "version=9p2000.L"
+          "cache=mmap"
+          "msize=512000"
+          # "noauto"
+          # "x-systemd.automount"
+        ];
+        neededForBoot = true;
+      };
+
+      "/disk" = {
+        fsType = "none";
+        device = "/.disk/persist";
+        options = [
+          "bind"
+          "rw"
+        ];
+        depends = [
+          "/.disk/persist"
+        ];
+        neededForBoot = true;
+      };
+    };
+  };
+}
--- a/nix/kubernetes/hosts/worker0/DEPLOY_BOOT
+++ b/nix/kubernetes/hosts/worker0/DEPLOY_BOOT
@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+TARGET=worker0
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+
+nixos-rebuild boot --flake "$DIR/../../#worker0" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/worker0/DEPLOY_SWITCH
+++ b/nix/kubernetes/hosts/worker0/DEPLOY_SWITCH
@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+TARGET=worker0
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+
+nixos-rebuild switch --flake "$DIR/../../#worker0" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/worker0/ISO
+++ b/nix/kubernetes/hosts/worker0/ISO
@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#worker0.iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/worker0/SELF_BOOT
+++ b/nix/kubernetes/hosts/worker0/SELF_BOOT
@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nixos-rebuild boot --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker0" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/worker0/SELF_BUILD
+++ b/nix/kubernetes/hosts/worker0/SELF_BUILD
@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nixos-rebuild build --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker0" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/worker0/SELF_SWITCH
+++ b/nix/kubernetes/hosts/worker0/SELF_SWITCH
@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nixos-rebuild switch --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker0" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/worker0/VM_ISO
+++ b/nix/kubernetes/hosts/worker0/VM_ISO
@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#worker0.vm_iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/worker0/default.nix
+++ b/nix/kubernetes/hosts/worker0/default.nix
@ -0,0 +1,106 @@
+# MANUAL: On client machines generate signing keys:
+#   nix-store --generate-binary-cache-key some-name /persist/manual/nix/nix-cache-key.sec /persist/manual/nix/nix-cache-key.pub
+#
+# Trust other machines and add the substituters:
+#   nix.binaryCachePublicKeys = [ "some-name:AzNW1MOlkNEsUAXS1jIFZ1QCFKXjV+Y/LrF37quAZ1A=" ];
+#   nix.binaryCaches = [ "https://test.example/nix-cache" ];
+
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  imports = [
+    ./hardware-configuration.nix
+    ./vm_disk.nix
+  ];
+
+  config = {
+    networking =
+      let
+        interface = "enp0s2";
+      in
+      {
+        # Generate with `head -c4 /dev/urandom | od -A none -t x4`
+        hostId = "0aadbb10";
+
+        hostName = "worker0"; # Define your hostname.
+
+        interfaces = {
+          "${interface}" = {
+            ipv4.addresses = [
+              {
+                address = "10.215.1.224";
+                prefixLength = 24;
+              }
+            ];
+
+            ipv6.addresses = [
+              {
+                address = "2620:11f:7001:7:ffff:ffff:0ad7:01e0";
+                prefixLength = 64;
+              }
+            ];
+          };
+        };
+        defaultGateway = "10.215.1.1";
+        defaultGateway6 = {
+          # address = "2620:11f:7001:7::1";
+          address = "2620:11f:7001:7:ffff:ffff:0ad7:0101";
+          inherit interface;
+        };
+
+        dhcpcd.enable = lib.mkForce false;
+        useDHCP = lib.mkForce false;
+      };
+
+    time.timeZone = "America/New_York";
+    i18n.defaultLocale = "en_US.UTF-8";
+
+    me.boot.enable = true;
+    me.boot.secure = false;
+    me.mountPersistence = true;
+    boot.loader.timeout = lib.mkForce 0; # We can always generate a new ISO if we need to access other boot options.
+
+    me.optimizations = {
+      enable = true;
+      arch = "znver4";
+      # build_arch = "x86-64-v3";
+      system_features = [
+        "gccarch-znver4"
+        "gccarch-skylake"
+        "gccarch-kabylake"
+        # "gccarch-alderlake" missing WAITPKG
+        "gccarch-x86-64-v3"
+        "gccarch-x86-64-v4"
+        "benchmark"
+        "big-parallel"
+        "kvm"
+        "nixos-test"
+      ];
+    };
+
+    # Mount tmpfs at /tmp
+    boot.tmp.useTmpfs = true;
+
+    # Enable TRIM
+    # services.fstrim.enable = lib.mkDefault true;
+
+    # nix.optimise.automatic = true;
+    # nix.optimise.dates = [ "03:45" ];
+    # nix.optimise.persistent = true;
+
+    environment.systemPackages = with pkgs; [
+      htop
+    ];
+
+    # nix.sshServe.enable = true;
+    # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ];
+
+    me.dont_use_substituters.enable = true;
+    me.minimal_base.enable = true;
+    me.worker_node.enable = true;
+  };
+}
--- a/nix/kubernetes/hosts/worker0/hardware-configuration.nix
+++ b/nix/kubernetes/hosts/worker0/hardware-configuration.nix
@ -0,0 +1,31 @@
+{
+  config,
+  lib,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [
+    (modulesPath + "/installer/scan/not-detected.nix")
+  ];
+
+  config = {
+    boot.initrd.availableKernelModules = [
+      "nvme"
+      "xhci_pci"
+      "thunderbolt"
+    ];
+    boot.initrd.kernelModules = [ ];
+    boot.kernelModules = [ ];
+    boot.extraModulePackages = [ ];
+
+    # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+    # (the default) this is the recommended approach. When using systemd-networkd it's
+    # still possible to use this option, but it's recommended to use it in conjunction
+    # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+    # networking.useDHCP = lib.mkDefault true;
+    # networking.interfaces.eno1.useDHCP = lib.mkDefault true;
+    # networking.interfaces.wlp58s0.useDHCP = lib.mkDefault true;
+  };
+}
--- a/nix/kubernetes/hosts/worker0/vm_disk.nix
+++ b/nix/kubernetes/hosts/worker0/vm_disk.nix
@ -0,0 +1,94 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  imports = [ ];
+
+  config = {
+    # Mount the local disk
+    fileSystems = lib.mkIf config.me.mountPersistence {
+      "/.disk" = lib.mkForce {
+        device = "/dev/nvme0n1p1";
+        fsType = "ext4";
+        options = [
+          "noatime"
+          "discard"
+        ];
+        neededForBoot = true;
+      };
+
+      "/.persist" = lib.mkForce {
+        device = "bind9p";
+        fsType = "9p";
+        options = [
+          "noatime"
+          "trans=virtio"
+          "version=9p2000.L"
+          "cache=mmap"
+          "msize=512000"
+          # "noauto"
+          # "x-systemd.automount"
+        ];
+        neededForBoot = true;
+      };
+
+      "/persist" = {
+        fsType = "none";
+        device = "/.persist/persist";
+        options = [
+          "bind"
+          "rw"
+        ];
+        depends = [
+          "/.persist/persist"
+        ];
+        neededForBoot = true;
+      };
+
+      "/state" = {
+        fsType = "none";
+        device = "/.persist/state";
+        options = [
+          "bind"
+          "rw"
+        ];
+        depends = [
+          "/.persist/state"
+        ];
+        neededForBoot = true;
+      };
+
+      "/k8spv" = lib.mkForce {
+        device = "k8spv";
+        fsType = "9p";
+        options = [
+          "noatime"
+          "trans=virtio"
+          "version=9p2000.L"
+          "cache=mmap"
+          "msize=512000"
+          # "noauto"
+          # "x-systemd.automount"
+        ];
+        neededForBoot = true;
+      };
+
+      "/disk" = {
+        fsType = "none";
+        device = "/.disk/persist";
+        options = [
+          "bind"
+          "rw"
+        ];
+        depends = [
+          "/.disk/persist"
+        ];
+        neededForBoot = true;
+      };
+    };
+  };
+}
--- a/nix/kubernetes/hosts/worker1/DEPLOY_BOOT
+++ b/nix/kubernetes/hosts/worker1/DEPLOY_BOOT
@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+TARGET=worker1
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+
+nixos-rebuild boot --flake "$DIR/../../#worker1" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/worker1/DEPLOY_SWITCH
+++ b/nix/kubernetes/hosts/worker1/DEPLOY_SWITCH
@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+TARGET=worker1
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+
+nixos-rebuild switch --flake "$DIR/../../#worker1" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/worker1/ISO
+++ b/nix/kubernetes/hosts/worker1/ISO
@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#worker1.iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/worker1/SELF_BOOT
+++ b/nix/kubernetes/hosts/worker1/SELF_BOOT
@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nixos-rebuild boot --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker1" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/worker1/SELF_BUILD
+++ b/nix/kubernetes/hosts/worker1/SELF_BUILD
@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nixos-rebuild build --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker1" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/worker1/SELF_SWITCH
+++ b/nix/kubernetes/hosts/worker1/SELF_SWITCH
@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nixos-rebuild switch --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker1" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/worker1/VM_ISO
+++ b/nix/kubernetes/hosts/worker1/VM_ISO
@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#worker1.vm_iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/worker1/default.nix
+++ b/nix/kubernetes/hosts/worker1/default.nix
@ -0,0 +1,106 @@
+# MANUAL: On client machines generate signing keys:
+#   nix-store --generate-binary-cache-key some-name /persist/manual/nix/nix-cache-key.sec /persist/manual/nix/nix-cache-key.pub
+#
+# Trust other machines and add the substituters:
+#   nix.binaryCachePublicKeys = [ "some-name:AzNW1MOlkNEsUAXS1jIFZ1QCFKXjV+Y/LrF37quAZ1A=" ];
+#   nix.binaryCaches = [ "https://test.example/nix-cache" ];
+
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  imports = [
+    ./hardware-configuration.nix
+    ./vm_disk.nix
+  ];
+
+  config = {
+    networking =
+      let
+        interface = "enp0s2";
+      in
+      {
+        # Generate with `head -c4 /dev/urandom | od -A none -t x4`
+        hostId = "4324346d";
+
+        hostName = "worker1"; # Define your hostname.
+
+        interfaces = {
+          "${interface}" = {
+            ipv4.addresses = [
+              {
+                address = "10.215.1.225";
+                prefixLength = 24;
+              }
+            ];
+
+            ipv6.addresses = [
+              {
+                address = "2620:11f:7001:7:ffff:ffff:0ad7:01e1";
+                prefixLength = 64;
+              }
+            ];
+          };
+        };
+        defaultGateway = "10.215.1.1";
+        defaultGateway6 = {
+          # address = "2620:11f:7001:7::1";
+          address = "2620:11f:7001:7:ffff:ffff:0ad7:0101";
+          inherit interface;
+        };
+
+        dhcpcd.enable = lib.mkForce false;
+        useDHCP = lib.mkForce false;
+      };
+
+    time.timeZone = "America/New_York";
+    i18n.defaultLocale = "en_US.UTF-8";
+
+    me.boot.enable = true;
+    me.boot.secure = false;
+    me.mountPersistence = true;
+    boot.loader.timeout = lib.mkForce 0; # We can always generate a new ISO if we need to access other boot options.
+
+    me.optimizations = {
+      enable = true;
+      arch = "znver4";
+      # build_arch = "x86-64-v3";
+      system_features = [
+        "gccarch-znver4"
+        "gccarch-skylake"
+        "gccarch-kabylake"
+        # "gccarch-alderlake" missing WAITPKG
+        "gccarch-x86-64-v3"
+        "gccarch-x86-64-v4"
+        "benchmark"
+        "big-parallel"
+        "kvm"
+        "nixos-test"
+      ];
+    };
+
+    # Mount tmpfs at /tmp
+    boot.tmp.useTmpfs = true;
+
+    # Enable TRIM
+    # services.fstrim.enable = lib.mkDefault true;
+
+    # nix.optimise.automatic = true;
+    # nix.optimise.dates = [ "03:45" ];
+    # nix.optimise.persistent = true;
+
+    environment.systemPackages = with pkgs; [
+      htop
+    ];
+
+    # nix.sshServe.enable = true;
+    # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ];
+
+    me.worker_node.enable = true;
+    me.dont_use_substituters.enable = true;
+    me.minimal_base.enable = true;
+  };
+}
--- a/nix/kubernetes/hosts/worker1/hardware-configuration.nix
+++ b/nix/kubernetes/hosts/worker1/hardware-configuration.nix
@ -0,0 +1,31 @@
+{
+  config,
+  lib,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [
+    (modulesPath + "/installer/scan/not-detected.nix")
+  ];
+
+  config = {
+    boot.initrd.availableKernelModules = [
+      "nvme"
+      "xhci_pci"
+      "thunderbolt"
+    ];
+    boot.initrd.kernelModules = [ ];
+    boot.kernelModules = [ ];
+    boot.extraModulePackages = [ ];
+
+    # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+    # (the default) this is the recommended approach. When using systemd-networkd it's
+    # still possible to use this option, but it's recommended to use it in conjunction
+    # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+    # networking.useDHCP = lib.mkDefault true;
+    # networking.interfaces.eno1.useDHCP = lib.mkDefault true;
+    # networking.interfaces.wlp58s0.useDHCP = lib.mkDefault true;
+  };
+}
--- a/nix/kubernetes/hosts/worker1/vm_disk.nix
+++ b/nix/kubernetes/hosts/worker1/vm_disk.nix
@ -0,0 +1,94 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  imports = [ ];
+
+  config = {
+    # Mount the local disk
+    fileSystems = lib.mkIf config.me.mountPersistence {
+      "/.disk" = lib.mkForce {
+        device = "/dev/nvme0n1p1";
+        fsType = "ext4";
+        options = [
+          "noatime"
+          "discard"
+        ];
+        neededForBoot = true;
+      };
+
+      "/.persist" = lib.mkForce {
+        device = "bind9p";
+        fsType = "9p";
+        options = [
+          "noatime"
+          "trans=virtio"
+          "version=9p2000.L"
+          "cache=mmap"
+          "msize=512000"
+          # "noauto"
+          # "x-systemd.automount"
+        ];
+        neededForBoot = true;
+      };
+
+      "/persist" = {
+        fsType = "none";
+        device = "/.persist/persist";
+        options = [
+          "bind"
+          "rw"
+        ];
+        depends = [
+          "/.persist/persist"
+        ];
+        neededForBoot = true;
+      };
+
+      "/state" = {
+        fsType = "none";
+        device = "/.persist/state";
+        options = [
+          "bind"
+          "rw"
+        ];
+        depends = [
+          "/.persist/state"
+        ];
+        neededForBoot = true;
+      };
+
+      "/k8spv" = lib.mkForce {
+        device = "k8spv";
+        fsType = "9p";
+        options = [
+          "noatime"
+          "trans=virtio"
+          "version=9p2000.L"
+          "cache=mmap"
+          "msize=512000"
+          # "noauto"
+          # "x-systemd.automount"
+        ];
+        neededForBoot = true;
+      };
+
+      "/disk" = {
+        fsType = "none";
+        device = "/.disk/persist";
+        options = [
+          "bind"
+          "rw"
+        ];
+        depends = [
+          "/.disk/persist"
+        ];
+        neededForBoot = true;
+      };
+    };
+  };
+}
--- a/nix/kubernetes/hosts/worker2/DEPLOY_BOOT
+++ b/nix/kubernetes/hosts/worker2/DEPLOY_BOOT
@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+TARGET=worker2
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+
+nixos-rebuild boot --flake "$DIR/../../#worker2" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/worker2/DEPLOY_SWITCH
+++ b/nix/kubernetes/hosts/worker2/DEPLOY_SWITCH
@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+TARGET=worker2
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+
+nixos-rebuild switch --flake "$DIR/../../#worker2" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/worker2/ISO
+++ b/nix/kubernetes/hosts/worker2/ISO
@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#worker2.iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/worker2/SELF_BOOT
+++ b/nix/kubernetes/hosts/worker2/SELF_BOOT
@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nixos-rebuild boot --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker2" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/worker2/SELF_BUILD
+++ b/nix/kubernetes/hosts/worker2/SELF_BUILD
@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nixos-rebuild build --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker2" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/worker2/SELF_SWITCH
+++ b/nix/kubernetes/hosts/worker2/SELF_SWITCH
@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nixos-rebuild switch --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker2" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/worker2/VM_ISO
+++ b/nix/kubernetes/hosts/worker2/VM_ISO
@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#worker2.vm_iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json
--- a/nix/kubernetes/hosts/worker2/default.nix
+++ b/nix/kubernetes/hosts/worker2/default.nix
@ -0,0 +1,106 @@
+# MANUAL: On client machines generate signing keys:
+#   nix-store --generate-binary-cache-key some-name /persist/manual/nix/nix-cache-key.sec /persist/manual/nix/nix-cache-key.pub
+#
+# Trust other machines and add the substituters:
+#   nix.binaryCachePublicKeys = [ "some-name:AzNW1MOlkNEsUAXS1jIFZ1QCFKXjV+Y/LrF37quAZ1A=" ];
+#   nix.binaryCaches = [ "https://test.example/nix-cache" ];
+
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  imports = [
+    ./hardware-configuration.nix
+    ./vm_disk.nix
+  ];
+
+  config = {
+    networking =
+      let
+        interface = "enp0s2";
+      in
+      {
+        # Generate with `head -c4 /dev/urandom | od -A none -t x4`
+        hostId = "ce017961";
+
+        hostName = "worker2"; # Define your hostname.
+
+        interfaces = {
+          "${interface}" = {
+            ipv4.addresses = [
+              {
+                address = "10.215.1.226";
+                prefixLength = 24;
+              }
+            ];
+
+            ipv6.addresses = [
+              {
+                address = "2620:11f:7001:7:ffff:ffff:0ad7:01e2";
+                prefixLength = 64;
+              }
+            ];
+          };
+        };
+        defaultGateway = "10.215.1.1";
+        defaultGateway6 = {
+          # address = "2620:11f:7001:7::1";
+          address = "2620:11f:7001:7:ffff:ffff:0ad7:0101";
+          inherit interface;
+        };
+
+        dhcpcd.enable = lib.mkForce false;
+        useDHCP = lib.mkForce false;
+      };
+
+    time.timeZone = "America/New_York";
+    i18n.defaultLocale = "en_US.UTF-8";
+
+    me.boot.enable = true;
+    me.boot.secure = false;
+    me.mountPersistence = true;
+    boot.loader.timeout = lib.mkForce 0; # We can always generate a new ISO if we need to access other boot options.
+
+    me.optimizations = {
+      enable = true;
+      arch = "znver4";
+      # build_arch = "x86-64-v3";
+      system_features = [
+        "gccarch-znver4"
+        "gccarch-skylake"
+        "gccarch-kabylake"
+        # "gccarch-alderlake" missing WAITPKG
+        "gccarch-x86-64-v3"
+        "gccarch-x86-64-v4"
+        "benchmark"
+        "big-parallel"
+        "kvm"
+        "nixos-test"
+      ];
+    };
+
+    # Mount tmpfs at /tmp
+    boot.tmp.useTmpfs = true;
+
+    # Enable TRIM
+    # services.fstrim.enable = lib.mkDefault true;
+
+    # nix.optimise.automatic = true;
+    # nix.optimise.dates = [ "03:45" ];
+    # nix.optimise.persistent = true;
+
+    environment.systemPackages = with pkgs; [
+      htop
+    ];
+
+    # nix.sshServe.enable = true;
+    # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ];
+
+    me.worker_node.enable = true;
+    me.dont_use_substituters.enable = true;
+    me.minimal_base.enable = true;
+  };
+}
--- a/nix/kubernetes/hosts/worker2/hardware-configuration.nix
+++ b/nix/kubernetes/hosts/worker2/hardware-configuration.nix
@ -0,0 +1,31 @@
+{
+  config,
+  lib,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [
+    (modulesPath + "/installer/scan/not-detected.nix")
+  ];
+
+  config = {
+    boot.initrd.availableKernelModules = [
+      "nvme"
+      "xhci_pci"
+      "thunderbolt"
+    ];
+    boot.initrd.kernelModules = [ ];
+    boot.kernelModules = [ ];
+    boot.extraModulePackages = [ ];
+
+    # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+    # (the default) this is the recommended approach. When using systemd-networkd it's
+    # still possible to use this option, but it's recommended to use it in conjunction
+    # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+    # networking.useDHCP = lib.mkDefault true;
+    # networking.interfaces.eno1.useDHCP = lib.mkDefault true;
+    # networking.interfaces.wlp58s0.useDHCP = lib.mkDefault true;
+  };
+}
--- a/nix/kubernetes/hosts/worker2/vm_disk.nix
+++ b/nix/kubernetes/hosts/worker2/vm_disk.nix
@ -0,0 +1,94 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  imports = [ ];
+
+  config = {
+    # Mount the local disk
+    fileSystems = lib.mkIf config.me.mountPersistence {
+      "/.disk" = lib.mkForce {
+        device = "/dev/nvme0n1p1";
+        fsType = "ext4";
+        options = [
+          "noatime"
+          "discard"
+        ];
+        neededForBoot = true;
+      };
+
+      "/.persist" = lib.mkForce {
+        device = "bind9p";
+        fsType = "9p";
+        options = [
+          "noatime"
+          "trans=virtio"
+          "version=9p2000.L"
+          "cache=mmap"
+          "msize=512000"
+          # "noauto"
+          # "x-systemd.automount"
+        ];
+        neededForBoot = true;
+      };
+
+      "/persist" = {
+        fsType = "none";
+        device = "/.persist/persist";
+        options = [
+          "bind"
+          "rw"
+        ];
+        depends = [
+          "/.persist/persist"
+        ];
+        neededForBoot = true;
+      };
+
+      "/state" = {
+        fsType = "none";
+        device = "/.persist/state";
+        options = [
+          "bind"
+          "rw"
+        ];
+        depends = [
+          "/.persist/state"
+        ];
+        neededForBoot = true;
+      };
+
+      "/k8spv" = lib.mkForce {
+        device = "k8spv";
+        fsType = "9p";
+        options = [
+          "noatime"
+          "trans=virtio"
+          "version=9p2000.L"
+          "cache=mmap"
+          "msize=512000"
+          # "noauto"
+          # "x-systemd.automount"
+        ];
+        neededForBoot = true;
+      };
+
+      "/disk" = {
+        fsType = "none";
+        device = "/.disk/persist";
+        options = [
+          "bind"
+          "rw"
+        ];
+        depends = [
+          "/.disk/persist"
+        ];
+        neededForBoot = true;
+      };
+    };
+  };
+}
--- a/nix/kubernetes/keys/Makefile
+++ b/nix/kubernetes/keys/Makefile
@ -0,0 +1,29 @@
+SHELL := bash
+.ONESHELL:
+.SHELLFLAGS := -eu -o pipefail -c
+.DELETE_ON_ERROR:
+MAKEFLAGS += --warn-undefined-variables
+MAKEFLAGS += --no-builtin-rules
+OUT=generated
+
+ifeq ($(origin .RECIPEPREFIX), undefined)
+  $(error This Make does not support .RECIPEPREFIX. Please use GNU Make 4.0 or later)
+endif
+.RECIPEPREFIX = >
+
+KUBERNETES_PUBLIC_ADDRESS := 74.80.180.138
+WORKERS := worker0 worker1 worker2 controller0 controller1 controller2
+
+.PHONY: all
+all: \
+    $(OUT)/known_hosts
+
+.PHONY: clean
+clean:
+> rm -rf $(OUT)
+
+$(OUT)/:
+> @mkdir -p $(@D)
+
+$(OUT)/known_hosts: | $(OUT)/
+> ssh-keyscan -p 65099 74.80.180.138 | sed 's/\[74.80.180.138\]:65099/\[10.215.1.210\]:22/g' > $@
--- a/nix/kubernetes/keys/contrib/base64/package.nix
+++ b/nix/kubernetes/keys/contrib/base64/package.nix
@ -0,0 +1,62 @@
+# From: https://gist.github.com/manveru/74eb41d850bc146b7e78c4cb059507e2
+# From: https://discourse.nixos.org/t/string-to-base-64/32624/3
+{ lib, ... }:
+{
+  toBase64 =
+    text:
+    let
+      inherit (lib)
+        sublist
+        mod
+        stringToCharacters
+        concatMapStrings
+        ;
+      inherit (lib.strings) charToInt;
+      inherit (builtins)
+        substring
+        foldl'
+        genList
+        elemAt
+        length
+        concatStringsSep
+        stringLength
+        ;
+      lookup = stringToCharacters "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+      sliceN =
+        size: list: n:
+        sublist (n * size) size list;
+      pows = [
+        (64 * 64 * 64)
+        (64 * 64)
+        64
+        1
+      ];
+      intSextets = i: map (j: mod (i / j) 64) pows;
+      compose =
+        f: g: x:
+        f (g x);
+      intToChar = elemAt lookup;
+      convertTripletInt = sliceInt: concatMapStrings intToChar (intSextets sliceInt);
+      sliceToInt = foldl' (acc: val: acc * 256 + val) 0;
+      convertTriplet = compose convertTripletInt sliceToInt;
+      join = concatStringsSep "";
+      convertLastSlice =
+        slice:
+        let
+          len = length slice;
+        in
+        if len == 1 then
+          (substring 0 2 (convertTripletInt ((sliceToInt slice) * 256 * 256))) + "=="
+        else if len == 2 then
+          (substring 0 3 (convertTripletInt ((sliceToInt slice) * 256))) + "="
+        else
+          "";
+      len = stringLength text;
+      nFullSlices = len / 3;
+      bytes = map charToInt (stringToCharacters text);
+      tripletAt = sliceN 3 bytes;
+      head = genList (compose convertTriplet tripletAt) nFullSlices;
+      tail = convertLastSlice (tripletAt nFullSlices);
+    in
+    join (head ++ [ tail ]);
+}
--- a/nix/kubernetes/keys/flake.lock
+++ b/nix/kubernetes/keys/flake.lock
@ -0,0 +1,27 @@
+{
+  "nodes": {
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1767892417,
+        "narHash": "sha256-dhhvQY67aboBk8b0/u0XB6vwHdgbROZT3fJAjyNh5Ww=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "3497aa5c9457a9d88d71fa93a4a8368816fbeeba",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "nixpkgs": "nixpkgs"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}
--- a/nix/kubernetes/keys/flake.nix
+++ b/nix/kubernetes/keys/flake.nix
@ -0,0 +1,54 @@
+{
+  description = "Build keys to manually deploy to kubernetes cluster.";
+
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+  };
+
+  outputs =
+    { self, nixpkgs }:
+    let
+      forAllSystems = nixpkgs.lib.genAttrs nixpkgs.lib.systems.flakeExposed;
+    in
+    {
+      packages = forAllSystems (
+        system:
+        let
+          pkgs = nixpkgs.legacyPackages.${system};
+          appliedOverlay = self.overlays.default pkgs pkgs;
+        in
+        {
+          deploy_script = appliedOverlay.k8s.deploy_script;
+          default = appliedOverlay.k8s.all_keys;
+          bootstrap_script = appliedOverlay.k8s.bootstrap_script;
+        }
+      );
+      overlays.default = (
+        final: prev: {
+          k8s = (final.callPackage ./scope.nix { inherit (final.lib) makeScope; });
+        }
+      );
+
+    }
+    // {
+      devShells = forAllSystems (
+        system:
+        let
+          pkgs = nixpkgs.legacyPackages.${system};
+          appliedOverlay = self.overlays.default pkgs pkgs;
+        in
+        {
+          default = pkgs.mkShell {
+            nativeBuildInputs = with pkgs; [
+              pkg-config
+            ];
+            buildInputs = with pkgs; [
+              kubernetes-helm # To generate cilium manifests
+              fluxcd # To generate flux manifests
+              cilium-cli # To check cilium status
+            ];
+          };
+        }
+      );
+    };
+}
--- a/nix/kubernetes/keys/generated/known_hosts
+++ b/nix/kubernetes/keys/generated/known_hosts
@ -0,0 +1,8 @@
+# 74.80.180.138:65099 SSH-2.0-OpenSSH_9.3 FreeBSD-20230316
+[10.215.1.210]:22 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC0hWY7Ighnlp3UfPfApyW9nEGG11f+on/kOkp6YdxTTVX0jvi00xvrZ8c23l48YDptmEKOMj7avUR+jdpRNaSwbw3Lm7swg+EpFZ73tnHK+r6HnOnNu8ECDvYOW10eI6vdRctFisRfyIKigmtmquxXYLhQDSA2INVW+Vuebdwa74VqKLLirUu7e3ymp8dH8ktcCAjWSd/+Ax7E+4AMa5WHFeTPBheA2GhfLhINDLpgdZ8WNZ4i3ow8MrQADiOVYUDPrXvI55MVWSQTQQcOco184Z67rtcCtqY/fcCp+38yzUT0Bm2syXM+HNOlFqM+fJBf0T9kiiy5XvWuN9bY+368JGOUUM6RsCUgERHSaU65nX3i8oIcNRt3w6sVsmRR8sX8x5qFjyEYuElIwKywcdtKpoklV6gu+lo+mIE8i95jJmXMj6lk3G83wMZICL9+dm+b8ckpRZEi6970EqahiPO3cV/Fa88gysf9HwiC8AxSc3m2BcOvaV3jadaT39Tymp8=
+# 74.80.180.138:65099 SSH-2.0-OpenSSH_9.3 FreeBSD-20230316
+# 74.80.180.138:65099 SSH-2.0-OpenSSH_9.3 FreeBSD-20230316
+[10.215.1.210]:22 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBH2euFJKLEDfTV9NTecrOoqL9FpiYvTbNp/Ty3FebJA5DKmVd1xBRz3sNs1R1ayn213vmRVLWSu2ikulbl65LLQ=
+# 74.80.180.138:65099 SSH-2.0-OpenSSH_9.3 FreeBSD-20230316
+[10.215.1.210]:22 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM1qjGgD2UdD5Lc+zGFxHX/+h6FBNmGW+O30LG0tiHvC
+# 74.80.180.138:65099 SSH-2.0-OpenSSH_9.3 FreeBSD-20230316
--- a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux.yaml
+++ b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux.yaml
--- a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_apply_git.yaml
+++ b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_apply_git.yaml
@ -0,0 +1,51 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: GitRepository
+metadata:
+  name: kubernetes
+  namespace: flux-system
+spec:
+  interval: 5m0s
+  ref:
+    branch: nix
+  secretRef:
+    name: kubernetes-deploy-key
+  # url: ssh://git@74.80.180.138:65099/repos/mrmanager
+  url: ssh://git@10.215.1.210:22/repos/mrmanager
+  ignore: |
+    bootstrap
+    .sops.yaml
+    secrets/
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: first
+  namespace: flux-system
+spec:
+  interval: 1m0s
+  path: "./k8s/first"
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: kubernetes
+  decryption:
+    provider: sops
+    secretRef:
+      name: sops-gpg
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: deferred
+  namespace: flux-system
+spec:
+  interval: 1m0s
+  path: "./k8s/deferred"
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: kubernetes
+  decryption:
+    provider: sops
+    secretRef:
+      name: sops-gpg
--- a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_instance.yaml
+++ b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_instance.yaml
@ -0,0 +1,9 @@
+apiVersion: fluxcd.controlplane.io/v1
+kind: FluxInstance
+metadata:
+  name: flux
+  namespace: flux-system
+spec:
+  distribution:
+    version: "2.7.x"
+    registry: "ghcr.io/fluxcd"
--- a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_namespace.yaml
+++ b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_namespace.yaml
@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: flux-system
--- a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/initial_clusterrole.yaml
+++ b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/initial_clusterrole.yaml
@ -0,0 +1,33 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations:
+    rbac.authorization.kubernetes.io/autoupdate: "true"
+  labels:
+    kubernetes.io/bootstrapping: rbac-defaults
+  name: system:kube-apiserver-to-kubelet
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - nodes/proxy
+      - nodes/stats
+      - nodes/log
+      - nodes/spec
+      - nodes/metrics
+    verbs:
+      - "*"
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system:kube-apiserver
+  namespace: ""
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:kube-apiserver-to-kubelet
+subjects:
+  - apiGroup: rbac.authorization.k8s.io
+    kind: User
+    name: kubernetes
--- a/nix/kubernetes/keys/package/bootstrap-script/package.nix
+++ b/nix/kubernetes/keys/package/bootstrap-script/package.nix
@ -0,0 +1,86 @@
+# unpackPhase
+# patchPhase
+# configurePhase
+# buildPhase
+# checkPhase
+# installPhase
+# fixupPhase
+# installCheckPhase
+# distPhase
+{
+  lib,
+  stdenv,
+  writeShellScript,
+  k8s,
+  ...
+}:
+let
+  bootstrap_script = (writeShellScript "bootstrap-script" bootstrap_script_body);
+  bootstrap_script_body = (''
+    set -euo pipefail
+    IFS=$'\n\t'
+    DIR="$( cd "$( dirname "''${BASH_SOURCE[0]}" )" && pwd )"
+
+    ${apply_manifests}
+    echo "Bootstrap finished"
+  '');
+  manifests = (
+    lib.concatMapStringsSep "," lib.escapeShellArg (
+      [
+        ./files/manifests/initial_clusterrole.yaml
+      ]
+      ++ gateway_crds
+      ++ [
+        "${k8s.cilium-manifest}/cilium.yaml"
+        "${k8s.coredns-manifest}/coredns.yaml"
+        ./files/manifests/flux_namespace.yaml
+        ./files/manifests/flux.yaml
+        ./files/manifests/flux_instance.yaml
+      ]
+      ++ (lib.attrsets.mapAttrsToList (
+        secret_name: secret_value: "${secret_value}/${secret_name}.yaml"
+      ) k8s.k8s-secrets-generic)
+      ++ [
+        ./files/manifests/flux_apply_git.yaml
+      ]
+    )
+  );
+  apply_manifests = "kubectl --kubeconfig=${k8s.client-configs.admin}/admin.kubeconfig apply --server-side --force-conflicts -f ${manifests}";
+  gateway_crds = [
+    (builtins.fetchurl {
+      url = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.2.0/config/crd/standard/gateway.networking.k8s.io_gatewayclasses.yaml";
+      sha256 = "0vf8c3kzlf7p6bf92gmdrzjc22fr2dwkrzvvbnxlsb43knv1nbzl";
+    })
+    (builtins.fetchurl {
+      url = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.2.0/config/crd/standard/gateway.networking.k8s.io_gateways.yaml";
+      sha256 = "1dqwlsypcb5f37y7x48rrv27yfgkizcx2alqd2nngijl1qzir3wa";
+    })
+    (builtins.fetchurl {
+      url = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.2.0/config/crd/standard/gateway.networking.k8s.io_httproutes.yaml";
+      sha256 = "05llfw6y66438r8kqy7krhyymyalkzxsaxjpa2zxzjk6z5mggbzq";
+    })
+    (builtins.fetchurl {
+      url = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.2.0/config/crd/standard/gateway.networking.k8s.io_referencegrants.yaml";
+      sha256 = "0a9q0vhqcazfrni3ajcq8vm2b254vcjbgmkchsdq9l6cbpvx79jd";
+    })
+    (builtins.fetchurl {
+      url = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.2.0/config/crd/standard/gateway.networking.k8s.io_grpcroutes.yaml";
+      sha256 = "19hwvdwdj0sc5fihdskw492g52ail3kjjzm6vpflvp2vlqam629p";
+    })
+    (builtins.fetchurl {
+      url = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.2.0/config/crd/experimental/gateway.networking.k8s.io_tlsroutes.yaml";
+      sha256 = "0b5pjihyzyyi4inz3avlkzvvccsynj9wsmx6znld04jmmvwpgxc9";
+    })
+  ];
+in
+stdenv.mkDerivation (finalAttrs: {
+  name = "bootstrap-script";
+  nativeBuildInputs = [ ];
+  buildInputs = [ ];
+
+  unpackPhase = "true";
+
+  installPhase = ''
+    cp ${bootstrap_script} "$out"
+  '';
+})
--- a/nix/kubernetes/keys/package/deploy-script/package.nix
+++ b/nix/kubernetes/keys/package/deploy-script/package.nix
@ -0,0 +1,301 @@
+# unpackPhase
+# patchPhase
+# configurePhase
+# buildPhase
+# checkPhase
+# installPhase
+# fixupPhase
+# installCheckPhase
+# distPhase
+{
+  config,
+  lib,
+  stdenv,
+  writeShellScript,
+  k8s,
+  openssh,
+  ...
+}:
+let
+  vm_name_to_hostname =
+    let
+      mapping = {
+        "nc0" = "controller0";
+        "nc1" = "controller1";
+        "nc2" = "controller2";
+        "nw0" = "worker0";
+        "nw1" = "worker1";
+        "nw2" = "worker2";
+      };
+    in
+    (vm_name: mapping."${vm_name}");
+
+  deploy_script_body = (
+    ''
+      set -euo pipefail
+      IFS=$'\n\t'
+      DIR="$( cd "$( dirname "''${BASH_SOURCE[0]}" )" && pwd )"
+    ''
+    + (lib.concatMapStringsSep "\n" deploy_control_plane [
+      "nc0"
+      "nc1"
+      "nc2"
+    ])
+    + (lib.concatMapStringsSep "\n" deploy_worker [
+      "nw0"
+      "nw1"
+      "nw2"
+    ])
+    + (trust_ssh_key {
+      public_key = "${k8s.ssh-keys.flux_ssh_key}/flux_ssh_key.pub";
+      destination = "/jail/admin_git/usr/home/git/.ssh/authorized_keys";
+      owner = "11236";
+      group = "11236";
+      mode = "0600";
+    })
+  );
+  deploy_script = (writeShellScript "deploy-script" deploy_script_body);
+  deploy_file = (
+    {
+      dest_dir,
+      file,
+      name ? (builtins.baseNameOf file),
+      owner,
+      group,
+      mode,
+    }:
+    ''
+      ##
+      ## deploy ${name} to ${dest_dir}
+      ##
+      ${openssh}/bin/ssh mrmanager doas rm -f ${dest_dir}/${name} ~/${name}
+      ${openssh}/bin/scp ${file} mrmanager:~/${name}
+      ${openssh}/bin/ssh mrmanager doas install -o ${toString owner} -g ${toString group} -m ${mode} ~/${name} ${dest_dir}/${name}
+      ${openssh}/bin/ssh mrmanager doas rm -f ~/${name}
+
+
+    ''
+  );
+  deploy_control_plane = (
+    vm_name:
+    (
+      ''
+        ##
+        ## Create directories on ${vm_name}
+        ##
+        ${openssh}/bin/ssh mrmanager doas install -d -o 0 -g 0 -m 0755 /vm/${vm_name}/persist/keys
+        ${openssh}/bin/ssh mrmanager doas install -d -o 10016 -g 10016 -m 0755 /vm/${vm_name}/persist/keys/etcd
+        ${openssh}/bin/ssh mrmanager doas install -d -o 10024 -g 10024 -m 0755 /vm/${vm_name}/persist/keys/kube
+
+
+      ''
+      + (lib.concatMapStringsSep "\n" deploy_file [
+        {
+          dest_dir = "/vm/${vm_name}/persist/keys/etcd";
+          file = "${k8s.keys.kube-api-server}/kube-api-server.crt";
+          owner = 10016;
+          group = 10016;
+          mode = "0640";
+        }
+        {
+          dest_dir = "/vm/${vm_name}/persist/keys/etcd";
+          file = "${k8s.keys.kube-api-server}/kube-api-server.key";
+          owner = 10016;
+          group = 10016;
+          mode = "0600";
+        }
+        {
+          dest_dir = "/vm/${vm_name}/persist/keys/etcd";
+          file = "${k8s.ca.client}/client-ca.crt";
+          owner = 10016;
+          group = 10016;
+          mode = "0640";
+        }
+        {
+          dest_dir = "/vm/${vm_name}/persist/keys/kube";
+          file = "${k8s.ca.client}/client-ca.crt";
+          owner = 10024;
+          group = 10024;
+          mode = "0640";
+        }
+        {
+          dest_dir = "/vm/${vm_name}/persist/keys/kube";
+          file = "${k8s.ca.client}/client-ca.key";
+          owner = 10024;
+          group = 10024;
+          mode = "0600";
+        }
+        {
+          dest_dir = "/vm/${vm_name}/persist/keys/kube";
+          file = "${k8s.keys.kube-api-server}/kube-api-server.crt";
+          owner = 10024;
+          group = 10024;
+          mode = "0640";
+        }
+        {
+          dest_dir = "/vm/${vm_name}/persist/keys/kube";
+          file = "${k8s.keys.kube-api-server}/kube-api-server.key";
+          owner = 10024;
+          group = 10024;
+          mode = "0600";
+        }
+        {
+          dest_dir = "/vm/${vm_name}/persist/keys/kube";
+          file = "${k8s.encryption_config}/encryption-config.yaml";
+          name = "encryption-config.yaml";
+          owner = 10024;
+          group = 10024;
+          mode = "0600";
+        }
+        {
+          dest_dir = "/vm/${vm_name}/persist/keys/kube";
+          file = "${k8s.keys.service-accounts}/service-accounts.crt";
+          owner = 10024;
+          group = 10024;
+          mode = "0640";
+        }
+        {
+          dest_dir = "/vm/${vm_name}/persist/keys/kube";
+          file = "${k8s.keys.service-accounts}/service-accounts.key";
+          owner = 10024;
+          group = 10024;
+          mode = "0600";
+        }
+        {
+          dest_dir = "/vm/${vm_name}/persist/keys/kube";
+          file = "${k8s.client-configs.kube-controller-manager}/kube-controller-manager.kubeconfig";
+          owner = 10024;
+          group = 10024;
+          mode = "0600";
+        }
+        {
+          dest_dir = "/vm/${vm_name}/persist/keys/kube";
+          file = "${k8s.client-configs.kube-scheduler}/kube-scheduler.kubeconfig";
+          owner = 10024;
+          group = 10024;
+          mode = "0600";
+        }
+        {
+          dest_dir = "/vm/${vm_name}/persist/keys/kube";
+          file = "${k8s.ca.requestheader-client}/requestheader-client-ca.crt";
+          owner = 10024;
+          group = 10024;
+          mode = "0640";
+        }
+        {
+          dest_dir = "/vm/${vm_name}/persist/keys/kube";
+          file = "${
+            k8s.keys."${vm_name_to_hostname vm_name}-proxy"
+          }/${vm_name_to_hostname vm_name}-proxy.crt";
+          name = "proxy.crt";
+          owner = 10024;
+          group = 10024;
+          mode = "0640";
+        }
+        {
+          dest_dir = "/vm/${vm_name}/persist/keys/kube";
+          file = "${
+            k8s.keys."${vm_name_to_hostname vm_name}-proxy"
+          }/${vm_name_to_hostname vm_name}-proxy.key";
+          name = "proxy.key";
+          owner = 10024;
+          group = 10024;
+          mode = "0600";
+        }
+      ])
+    )
+  );
+  deploy_worker = (
+    vm_name:
+    (
+      ''
+        ##
+        ## Create directories on ${vm_name}
+        ##
+        ${openssh}/bin/ssh mrmanager doas install -d -o 0 -g 0 -m 0755 /vm/${vm_name}/persist/keys
+        ${openssh}/bin/ssh mrmanager doas install -d -o 10024 -g 10024 -m 0755 /vm/${vm_name}/persist/keys/kube
+
+
+      ''
+      + (lib.concatMapStringsSep "\n" deploy_file [
+        {
+          dest_dir = "/vm/${vm_name}/persist/keys/kube";
+          file = "${k8s.ca.client}/client-ca.crt";
+          owner = 10024;
+          group = 10024;
+          mode = "0640";
+        }
+        {
+          dest_dir = "/vm/${vm_name}/persist/keys/kube";
+          file = "${k8s.keys."${vm_name_to_hostname vm_name}"}/${vm_name_to_hostname vm_name}.crt";
+          name = "kubelet.crt";
+          owner = 10024;
+          group = 10024;
+          mode = "0640";
+        }
+        {
+          dest_dir = "/vm/${vm_name}/persist/keys/kube";
+          file = "${k8s.keys."${vm_name_to_hostname vm_name}"}/${vm_name_to_hostname vm_name}.key";
+          name = "kubelet.key";
+          owner = 10024;
+          group = 10024;
+          mode = "0600";
+        }
+        {
+          dest_dir = "/vm/${vm_name}/persist/keys/kube";
+          file = "${
+            k8s.client-configs."${vm_name_to_hostname vm_name}"
+          }/${vm_name_to_hostname vm_name}.kubeconfig";
+          name = "kubelet.kubeconfig";
+          owner = 10024;
+          group = 10024;
+          mode = "0600";
+        }
+        {
+          dest_dir = "/vm/${vm_name}/persist/keys/kube";
+          file = "${k8s.client-configs.kube-proxy}/kube-proxy.kubeconfig";
+          owner = 10024;
+          group = 10024;
+          mode = "0600";
+        }
+      ])
+    )
+  );
+  trust_ssh_key =
+    {
+      public_key,
+      destination,
+      owner,
+      group,
+      mode,
+    }:
+    let
+      public_key_name = builtins.baseNameOf public_key;
+      public_key_contents = builtins.readFile public_key;
+      trimmed = lib.strings.trim public_key_contents;
+      escaped = lib.strings.escapeShellArg trimmed;
+    in
+    ''
+      ##
+      ## trust ${public_key_name} in ${destination}
+      ##
+      if ! ${openssh}/bin/ssh mrmanager doas grep -q "${escaped}" "${destination}"; then
+         ${openssh}/bin/ssh mrmanager doas tee -a "${destination}" <<<"$(cat ${public_key})"
+         ${openssh}/bin/ssh mrmanager doas chown "${owner}:${group}" "${destination}"
+         ${openssh}/bin/ssh mrmanager doas chmod "${mode}" "${destination}"
+      else
+         echo "${public_key_name} is already trusted in ${destination}"
+      fi
+    '';
+in
+stdenv.mkDerivation (finalAttrs: {
+  name = "deploy-script";
+  nativeBuildInputs = [ ];
+  buildInputs = [ ];
+
+  unpackPhase = "true";
+
+  installPhase = ''
+    cp ${deploy_script} "$out"
+  '';
+})
--- a/nix/kubernetes/keys/package/helm-manifest/package.nix
+++ b/nix/kubernetes/keys/package/helm-manifest/package.nix
@ -0,0 +1,48 @@
+# unpackPhase
+# patchPhase
+# configurePhase
+# buildPhase
+# checkPhase
+# installPhase
+# fixupPhase
+# installCheckPhase
+# distPhase
+{
+  lib,
+  pkgs,
+  stdenv,
+  kubernetes-helm,
+  helm_src,
+  helm_name,
+  helm_namespace,
+  helm_path ? ".",
+  helm_manifest_name,
+  helm_values ? { },
+  ...
+}:
+stdenv.mkDerivation (
+  finalAttrs:
+  let
+    to_yaml_file = ((import ../../../functions/to_yaml.nix) { inherit pkgs; }).to_yaml_file;
+  in
+  {
+    name = "${helm_name}-manifest";
+    nativeBuildInputs = [
+      kubernetes-helm
+    ];
+    buildInputs = [ ];
+
+    src = helm_src;
+
+    buildPhase = ''
+      helm template --dry-run=client ${lib.strings.escapeShellArg helm_name} $src/${helm_path} --namespace ${helm_namespace} \
+           --values ${to_yaml_file "values.yaml" helm_values} \
+           | tee $NIX_BUILD_TOP/${helm_manifest_name}
+    '';
+
+    installPhase = ''
+      mkdir -p "$out"
+      cp $NIX_BUILD_TOP/${helm_manifest_name} $out/
+    '';
+  }
+)
--- a/nix/kubernetes/keys/package/k8s-ca/files/client-ca.conf
+++ b/nix/kubernetes/keys/package/k8s-ca/files/client-ca.conf
@ -0,0 +1,305 @@
+[req]
+distinguished_name = req_distinguished_name
+prompt             = no
+x509_extensions    = ca_x509_extensions
+
+[ca_x509_extensions]
+basicConstraints = CA:TRUE
+keyUsage         = cRLSign, keyCertSign
+
+[req_distinguished_name]
+C   = US
+ST  = Washington
+L   = Seattle
+CN  = CA
+
+[admin]
+distinguished_name = admin_distinguished_name
+prompt             = no
+req_extensions     = default_req_extensions
+
+[admin_distinguished_name]
+CN = admin
+O  = system:masters
+
+# Service Accounts
+#
+# The Kubernetes Controller Manager leverages a key pair to generate
+# and sign service account tokens as described in the
+# [managing service accounts](https://kubernetes.io/docs/admin/service-accounts-admin/)
+# documentation.
+
+[service-accounts]
+distinguished_name = service-accounts_distinguished_name
+prompt             = no
+req_extensions     = default_req_extensions
+
+[service-accounts_distinguished_name]
+CN = service-accounts
+
+# Worker Nodes
+#
+# Kubernetes uses a [special-purpose authorization mode](https://kubernetes.io/docs/admin/authorization/node/)
+# called Node Authorizer, that specifically authorizes API requests made
+# by [Kubelets](https://kubernetes.io/docs/concepts/overview/components/#kubelet).
+# In order to be authorized by the Node Authorizer, Kubelets must use a credential
+# that identifies them as being in the `system:nodes` group, with a username
+# of `system:node:<nodeName>`.
+
+[controller0]
+distinguished_name = controller0_distinguished_name
+prompt             = no
+req_extensions     = controller0_req_extensions
+
+[controller0_req_extensions]
+basicConstraints     = CA:FALSE
+extendedKeyUsage     = clientAuth, serverAuth
+keyUsage             = critical, digitalSignature, keyEncipherment
+nsCertType           = client
+nsComment            = "controller0 Certificate"
+subjectAltName       = DNS:controller0, IP:127.0.0.1
+subjectKeyIdentifier = hash
+
+[controller0_distinguished_name]
+CN = system:node:controller0
+O  = system:nodes
+C  = US
+ST = Washington
+L  = Seattle
+
+[controller1]
+distinguished_name = controller1_distinguished_name
+prompt             = no
+req_extensions     = controller1_req_extensions
+
+[controller1_req_extensions]
+basicConstraints     = CA:FALSE
+extendedKeyUsage     = clientAuth, serverAuth
+keyUsage             = critical, digitalSignature, keyEncipherment
+nsCertType           = client
+nsComment            = "controller1 Certificate"
+subjectAltName       = DNS:controller1, IP:127.0.0.1
+subjectKeyIdentifier = hash
+
+[controller1_distinguished_name]
+CN = system:node:controller1
+O  = system:nodes
+C  = US
+ST = Washington
+L  = Seattle
+
+[controller2]
+distinguished_name = controller2_distinguished_name
+prompt             = no
+req_extensions     = controller2_req_extensions
+
+[controller2_req_extensions]
+basicConstraints     = CA:FALSE
+extendedKeyUsage     = clientAuth, serverAuth
+keyUsage             = critical, digitalSignature, keyEncipherment
+nsCertType           = client
+nsComment            = "controller2 Certificate"
+subjectAltName       = DNS:controller2, IP:127.0.0.1
+subjectKeyIdentifier = hash
+
+[controller2_distinguished_name]
+CN = system:node:controller2
+O  = system:nodes
+C  = US
+ST = Washington
+L  = Seattle
+
+[worker0]
+distinguished_name = worker0_distinguished_name
+prompt             = no
+req_extensions     = worker0_req_extensions
+
+[worker0_req_extensions]
+basicConstraints     = CA:FALSE
+extendedKeyUsage     = clientAuth, serverAuth
+keyUsage             = critical, digitalSignature, keyEncipherment
+nsCertType           = client
+nsComment            = "worker0 Certificate"
+subjectAltName       = DNS:worker0, IP:127.0.0.1
+subjectKeyIdentifier = hash
+
+[worker0_distinguished_name]
+CN = system:node:worker0
+O  = system:nodes
+C  = US
+ST = Washington
+L  = Seattle
+
+[worker1]
+distinguished_name = worker1_distinguished_name
+prompt             = no
+req_extensions     = worker1_req_extensions
+
+[worker1_req_extensions]
+basicConstraints     = CA:FALSE
+extendedKeyUsage     = clientAuth, serverAuth
+keyUsage             = critical, digitalSignature, keyEncipherment
+nsCertType           = client
+nsComment            = "worker1 Certificate"
+subjectAltName       = DNS:worker1, IP:127.0.0.1
+subjectKeyIdentifier = hash
+
+[worker1_distinguished_name]
+CN = system:node:worker1
+O  = system:nodes
+C  = US
+ST = Washington
+L  = Seattle
+
+[worker2]
+distinguished_name = worker2_distinguished_name
+prompt             = no
+req_extensions     = worker2_req_extensions
+
+[worker2_req_extensions]
+basicConstraints     = CA:FALSE
+extendedKeyUsage     = clientAuth, serverAuth
+keyUsage             = critical, digitalSignature, keyEncipherment
+nsCertType           = client
+nsComment            = "worker2 Certificate"
+subjectAltName       = DNS:worker2, IP:127.0.0.1
+subjectKeyIdentifier = hash
+
+[worker2_distinguished_name]
+CN = system:node:worker2
+O  = system:nodes
+C  = US
+ST = Washington
+L  = Seattle
+
+
+
+# Kube Proxy Section
+[kube-proxy]
+distinguished_name = kube-proxy_distinguished_name
+prompt             = no
+req_extensions     = kube-proxy_req_extensions
+
+[kube-proxy_req_extensions]
+basicConstraints     = CA:FALSE
+extendedKeyUsage     = clientAuth, serverAuth
+keyUsage             = critical, digitalSignature, keyEncipherment
+nsCertType           = client
+nsComment            = "Kube Proxy Certificate"
+subjectAltName       = DNS:kube-proxy, IP:127.0.0.1
+subjectKeyIdentifier = hash
+
+[kube-proxy_distinguished_name]
+CN = system:kube-proxy
+O  = system:node-proxier
+C  = US
+ST = Washington
+L  = Seattle
+
+
+# Controller Manager
+[kube-controller-manager]
+distinguished_name = kube-controller-manager_distinguished_name
+prompt             = no
+req_extensions     = kube-controller-manager_req_extensions
+
+[kube-controller-manager_req_extensions]
+basicConstraints     = CA:FALSE
+extendedKeyUsage     = clientAuth, serverAuth
+keyUsage             = critical, digitalSignature, keyEncipherment
+nsCertType           = client
+nsComment            = "Kube Controller Manager Certificate"
+subjectAltName       = DNS:kube-controller-manager, IP:127.0.0.1
+subjectKeyIdentifier = hash
+
+[kube-controller-manager_distinguished_name]
+CN = system:kube-controller-manager
+O  = system:kube-controller-manager
+C  = US
+ST = Washington
+L  = Seattle
+
+
+# Scheduler
+[kube-scheduler]
+distinguished_name = kube-scheduler_distinguished_name
+prompt             = no
+req_extensions     = kube-scheduler_req_extensions
+
+[kube-scheduler_req_extensions]
+basicConstraints     = CA:FALSE
+extendedKeyUsage     = clientAuth, serverAuth
+keyUsage             = critical, digitalSignature, keyEncipherment
+nsCertType           = client
+nsComment            = "Kube Scheduler Certificate"
+subjectAltName       = DNS:kube-scheduler, IP:127.0.0.1
+subjectKeyIdentifier = hash
+
+[kube-scheduler_distinguished_name]
+CN = system:kube-scheduler
+O  = system:system:kube-scheduler
+C  = US
+ST = Washington
+L  = Seattle
+
+
+# API Server
+#
+# The Kubernetes API server is automatically assigned the `kubernetes`
+# internal dns name, which will be linked to the first IP address (`10.32.0.1`)
+# from the address range (`10.32.0.0/24`) reserved for internal cluster
+# services.
+
+[kube-api-server]
+distinguished_name = kube-api-server_distinguished_name
+prompt             = no
+req_extensions     = kube-api-server_req_extensions
+
+[kube-api-server_req_extensions]
+basicConstraints     = CA:FALSE
+extendedKeyUsage     = clientAuth, serverAuth
+keyUsage             = critical, digitalSignature, keyEncipherment
+nsCertType           = client, server
+nsComment            = "Kube API Server Certificate"
+subjectAltName       = @kube-api-server_alt_names
+subjectKeyIdentifier = hash
+
+[kube-api-server_alt_names]
+IP.0  = 127.0.0.1
+IP.1  = 10.0.0.1
+IP.2  = 10.215.1.221
+IP.3  = 2620:11f:7001:7:ffff:ffff:0ad7:01dd
+IP.4  = 10.215.1.222
+IP.5  = 2620:11f:7001:7:ffff:ffff:0ad7:01de
+IP.6  = 10.215.1.223
+IP.7  = 2620:11f:7001:7:ffff:ffff:0ad7:01df
+IP.8  = 10.215.1.224
+IP.9  = 2620:11f:7001:7:ffff:ffff:0ad7:01e0
+IP.10  = 10.215.1.225
+IP.11  = 2620:11f:7001:7:ffff:ffff:0ad7:01e1
+IP.12  = 10.215.1.226
+IP.13  = 2620:11f:7001:7:ffff:ffff:0ad7:01e2
+IP.14  = fd00:3e42:e349::1
+IP.15  = 2620:11f:7001:7:ffff:eeee::1
+DNS.0 = kubernetes
+DNS.1 = kubernetes.default
+DNS.2 = kubernetes.default.svc
+DNS.3 = kubernetes.default.svc.cluster
+DNS.4 = kubernetes.svc.cluster.local
+DNS.5 = server.kubernetes.local
+DNS.6 = api-server.kubernetes.local
+
+[kube-api-server_distinguished_name]
+CN = kubernetes
+C  = US
+ST = Washington
+L  = Seattle
+
+
+[default_req_extensions]
+basicConstraints     = CA:FALSE
+extendedKeyUsage     = clientAuth
+keyUsage             = critical, digitalSignature, keyEncipherment
+nsCertType           = client
+nsComment            = "Admin Client Certificate"
+subjectKeyIdentifier = hash
--- a/nix/kubernetes/keys/package/k8s-ca/files/requestheader-client-ca.conf
+++ b/nix/kubernetes/keys/package/k8s-ca/files/requestheader-client-ca.conf
@ -0,0 +1,98 @@
+[req]
+distinguished_name = req_distinguished_name
+prompt             = no
+x509_extensions    = ca_x509_extensions
+
+[ca_x509_extensions]
+basicConstraints = CA:TRUE
+keyUsage         = cRLSign, keyCertSign
+
+[req_distinguished_name]
+C   = US
+ST  = Washington
+L   = Seattle
+CN  = Kubernetes
+O   = Kubernetes
+OU  = CA
+
+
+[controller0-proxy]
+distinguished_name = controller0_distinguished_name
+prompt             = no
+req_extensions     = controller0_req_extensions
+
+[controller0_req_extensions]
+basicConstraints     = CA:FALSE
+extendedKeyUsage     = clientAuth, serverAuth
+keyUsage             = critical, digitalSignature, keyEncipherment
+nsCertType           = client
+nsComment            = "controller0-proxy Certificate"
+subjectAltName       = @controller0_alt_names
+subjectKeyIdentifier = hash
+
+[controller0_distinguished_name]
+CN = system:node:controller0
+O  = system:nodes
+C  = US
+ST = Washington
+L  = Seattle
+
+[controller0_alt_names]
+IP.0  = 127.0.0.1
+IP.1  = 10.215.1.221
+IP.2  = 2620:11f:7001:7:ffff:ffff:0ad7:01dd
+DNS.0 = controller0
+
+[controller1-proxy]
+distinguished_name = controller1_distinguished_name
+prompt             = no
+req_extensions     = controller1_req_extensions
+
+[controller1_req_extensions]
+basicConstraints     = CA:FALSE
+extendedKeyUsage     = clientAuth, serverAuth
+keyUsage             = critical, digitalSignature, keyEncipherment
+nsCertType           = client
+nsComment            = "controller1-proxy Certificate"
+subjectAltName       = @controller1_alt_names
+subjectKeyIdentifier = hash
+
+[controller1_distinguished_name]
+CN = system:node:controller1
+O  = system:nodes
+C  = US
+ST = Washington
+L  = Seattle
+
+[controller1_alt_names]
+IP.0  = 127.0.0.1
+IP.4  = 10.215.1.222
+IP.5  = 2620:11f:7001:7:ffff:ffff:0ad7:01de
+DNS.0 = controller1
+
+[controller2-proxy]
+distinguished_name = controller2_distinguished_name
+prompt             = no
+req_extensions     = controller2_req_extensions
+
+[controller2_req_extensions]
+basicConstraints     = CA:FALSE
+extendedKeyUsage     = clientAuth, serverAuth
+keyUsage             = critical, digitalSignature, keyEncipherment
+nsCertType           = client
+nsComment            = "controller2-proxy Certificate"
+subjectAltName       = @controller2_alt_names
+subjectKeyIdentifier = hash
+
+[controller2_distinguished_name]
+CN = system:node:controller2
+O  = system:nodes
+C  = US
+ST = Washington
+L  = Seattle
+
+[controller2_alt_names]
+IP.0  = 127.0.0.1
+IP.1  = 10.215.1.223
+IP.2  = 2620:11f:7001:7:ffff:ffff:0ad7:01df
+DNS.0 = controller2
--- a/nix/kubernetes/keys/package/k8s-ca/package.nix
+++ b/nix/kubernetes/keys/package/k8s-ca/package.nix
@ -0,0 +1,37 @@
+# unpackPhase
+# patchPhase
+# configurePhase
+# buildPhase
+# checkPhase
+# installPhase
+# fixupPhase
+# installCheckPhase
+# distPhase
+{
+  stdenv,
+  openssl,
+  ca_name,
+  ca_config,
+  ...
+}:
+stdenv.mkDerivation (finalAttrs: {
+  name = "k8s-ca-${ca_name}";
+  nativeBuildInputs = [ openssl ];
+  buildInputs = [ ];
+
+  unpackPhase = "true";
+
+  buildPhase = ''
+    openssl genrsa -out "${ca_name}-ca.key" 4096
+
+    openssl req -x509 -new -sha512 -noenc \
+            -key "${ca_name}-ca.key" -days 3653 \
+            -config "${ca_config}" \
+            -out "${ca_name}-ca.crt"
+  '';
+
+  installPhase = ''
+    mkdir "$out"
+    cp "${ca_name}-ca.crt" "${ca_name}-ca.key" $out/
+  '';
+})
--- a/nix/kubernetes/keys/package/k8s-client-config/package.nix
+++ b/nix/kubernetes/keys/package/k8s-client-config/package.nix
@ -0,0 +1,53 @@
+# unpackPhase
+# patchPhase
+# configurePhase
+# buildPhase
+# checkPhase
+# installPhase
+# fixupPhase
+# installCheckPhase
+# distPhase
+{
+  lib,
+  stdenv,
+  k8s,
+  kubectl,
+  config_name,
+  config_user,
+  config_server,
+  ...
+}:
+stdenv.mkDerivation (finalAttrs: {
+  name = "k8s-client-config-${config_name}";
+  nativeBuildInputs = [ kubectl ];
+  buildInputs = [ ];
+
+  unpackPhase = "true";
+
+  buildPhase = ''
+    kubectl config set-cluster kubernetes-the-hard-way \
+      --certificate-authority=${k8s.ca.client}/client-ca.crt \
+      --embed-certs=true \
+      --server=${lib.strings.escapeShellArg config_server} \
+      --kubeconfig=${config_name}.kubeconfig
+
+    kubectl config set-credentials ${config_user} \
+      --client-certificate=${k8s.keys."${config_name}"}/${config_name}.crt \
+      --client-key=${k8s.keys."${config_name}"}/${config_name}.key \
+      --embed-certs=true \
+      --kubeconfig=${config_name}.kubeconfig
+
+    kubectl config set-context default \
+      --cluster=kubernetes-the-hard-way \
+      --user=${config_user} \
+      --kubeconfig=${config_name}.kubeconfig
+
+    kubectl config use-context default \
+      --kubeconfig=${config_name}.kubeconfig
+  '';
+
+  installPhase = ''
+    mkdir "$out"
+    cp "${config_name}.kubeconfig" $out/
+  '';
+})
--- a/nix/kubernetes/keys/package/k8s-encryption-key/package.nix
+++ b/nix/kubernetes/keys/package/k8s-encryption-key/package.nix
@ -0,0 +1,56 @@
+# unpackPhase
+# patchPhase
+# configurePhase
+# buildPhase
+# checkPhase
+# installPhase
+# fixupPhase
+# installCheckPhase
+# distPhase
+{
+  pkgs,
+  stdenv,
+  runCommand,
+  ...
+}:
+let
+  to_yaml_file = ((import ../../../functions/to_yaml.nix) {inherit pkgs;}).to_yaml_file;
+  kube_encryption_key = runCommand "kube_encryption_key" { } ''
+    head -c 32 /dev/urandom | base64 -w 0 | tee $out
+  '';
+  kube_encryption_config = {
+    kind = "EncryptionConfig";
+    apiVersion = "v1";
+    resources = [
+      {
+        resources = [ "secrets" ];
+        providers = [
+          {
+            aescbc = {
+              keys = [
+                {
+                  name = "key1";
+                  secret = (builtins.readFile "${kube_encryption_key}");
+                }
+              ];
+            };
+          }
+          { identity = { }; }
+        ];
+      }
+    ];
+  };
+  kube_encryption_config_yaml = (to_yaml_file "encryption-config.yaml" kube_encryption_config);
+in
+stdenv.mkDerivation (finalAttrs: {
+  name = "k8s-encryption-key";
+  nativeBuildInputs = [ ];
+  buildInputs = [ ];
+
+  unpackPhase = "true";
+
+  installPhase = ''
+    mkdir "$out"
+    cp "${kube_encryption_config_yaml}" $out/encryption-config.yaml
+  '';
+})
--- a/nix/kubernetes/keys/package/k8s-keys/package.nix
+++ b/nix/kubernetes/keys/package/k8s-keys/package.nix
@ -0,0 +1,26 @@
+{
+  k8s,
+  runCommand,
+  symlinkJoin,
+  ...
+}:
+let
+  scripts = runCommand "scripts" { } ''
+    mkdir $out
+    cp ${k8s.deploy_script} $out/deploy_script
+    cp ${k8s.bootstrap_script} $out/bootstrap_script
+  '';
+in
+symlinkJoin {
+  name = "k8s-keys";
+  paths = [
+    scripts
+    k8s.encryption_config
+  ]
+  ++ (builtins.attrValues k8s.ca)
+  ++ (builtins.attrValues k8s.keys)
+  ++ (builtins.attrValues k8s.client-configs)
+  ++ (builtins.attrValues k8s.ssh-keys)
+  ++ (builtins.attrValues k8s.pgp-keys)
+  ++ (builtins.attrValues k8s.k8s-secrets-generic);
+}
--- a/nix/kubernetes/keys/package/k8s-secret-generic/package.nix
+++ b/nix/kubernetes/keys/package/k8s-secret-generic/package.nix
@ -0,0 +1,50 @@
+# unpackPhase
+# patchPhase
+# configurePhase
+# buildPhase
+# checkPhase
+# installPhase
+# fixupPhase
+# installCheckPhase
+# distPhase
+{
+  lib,
+  pkgs,
+  stdenv,
+  k8s,
+  kubectl,
+  secret_name,
+  secret_namespace,
+  secret_values ? { },
+  ...
+}:
+let
+  toBase64 = (pkgs.callPackage ../../contrib/base64/package.nix { inherit lib; }).toBase64;
+  secret_yaml = {
+    apiVersion = "v1";
+    kind = "Secret";
+    metadata = {
+      name = "${secret_name}";
+      namespace = "${secret_namespace}";
+    };
+    data = (builtins.mapAttrs (key: val: (toBase64 val)) secret_values);
+  };
+  settingsFormat = pkgs.formats.yaml { };
+  yaml_body = settingsFormat.generate "${secret_name}.yaml" secret_yaml;
+  yaml_file = pkgs.writeTextFile {
+    name = "${secret_name}.yaml";
+    text = (builtins.readFile yaml_body);
+  };
+in
+stdenv.mkDerivation (finalAttrs: {
+  name = "k8s-secret-generic-${secret_name}";
+  nativeBuildInputs = [ kubectl ];
+  buildInputs = [ ];
+
+  unpackPhase = "true";
+
+  installPhase = ''
+    mkdir "$out"
+    cp "${yaml_file}" "$out/${secret_name}.yaml"
+  '';
+})
--- a/nix/kubernetes/keys/package/pgp-key/package.nix
+++ b/nix/kubernetes/keys/package/pgp-key/package.nix
@ -0,0 +1,50 @@
+# unpackPhase
+# patchPhase
+# configurePhase
+# buildPhase
+# checkPhase
+# installPhase
+# fixupPhase
+# installCheckPhase
+# distPhase
+{
+  stdenv,
+  gnupg,
+  key_name,
+  expire_date ? "0",
+  pgp_comment ? "${key_name}",
+  pgp_name ? "${key_name}",
+  ...
+}:
+stdenv.mkDerivation (finalAttrs: {
+  name = "pgp-key-${key_name}";
+  nativeBuildInputs = [ gnupg ];
+  buildInputs = [ ];
+
+  unpackPhase = "true";
+
+  buildPhase = ''
+    mkdir keyring
+    export GNUPGHOME=$(readlink -f keyring)
+
+    gpg --batch --full-generate-key <<EOF
+    %no-protection
+    Key-Type: 1
+    Key-Length: 4096
+    Subkey-Type: 1
+    Subkey-Length: 4096
+    Expire-Date: ${expire_date}
+    Name-Comment: ${pgp_comment}
+    Name-Real: ${pgp_name}
+    EOF
+
+
+  '';
+
+  installPhase = ''
+    export GNUPGHOME=$(readlink -f keyring)
+    mkdir "$out"
+    gpg --export-secret-keys --armor "${pgp_name}" > "$out/${key_name}_private_key.asc"
+    gpg --export --armor "${pgp_name}" > "$out/${key_name}_public_key.asc"
+  '';
+})
--- a/nix/kubernetes/keys/package/ssh-key/package.nix
+++ b/nix/kubernetes/keys/package/ssh-key/package.nix
@ -0,0 +1,33 @@
+# unpackPhase
+# patchPhase
+# configurePhase
+# buildPhase
+# checkPhase
+# installPhase
+# fixupPhase
+# installCheckPhase
+# distPhase
+{
+  lib,
+  stdenv,
+  k8s,
+  openssh,
+  key_name,
+  ...
+}:
+stdenv.mkDerivation (finalAttrs: {
+  name = "ssh-key-${key_name}";
+  nativeBuildInputs = [ openssh ];
+  buildInputs = [ ];
+
+  unpackPhase = "true";
+
+  buildPhase = ''
+    ssh-keygen -t ed25519 -f ${key_name} -N ""
+  '';
+
+  installPhase = ''
+    mkdir "$out"
+    cp "${key_name}" "${key_name}.pub" $out/
+  '';
+})
--- a/nix/kubernetes/keys/package/tls-key/package.nix
+++ b/nix/kubernetes/keys/package/tls-key/package.nix
@ -0,0 +1,47 @@
+# unpackPhase
+# patchPhase
+# configurePhase
+# buildPhase
+# checkPhase
+# installPhase
+# fixupPhase
+# installCheckPhase
+# distPhase
+{
+  stdenv,
+  openssl,
+  k8s,
+  key_name,
+  ca_name,
+  ca_config,
+  ...
+}:
+stdenv.mkDerivation (finalAttrs: {
+  name = "tls-key-${key_name}";
+  nativeBuildInputs = [ openssl ];
+  buildInputs = [ ];
+
+  unpackPhase = "true";
+
+  buildPhase = ''
+    cp ${k8s.ca."${ca_name}"}/${ca_name}-ca.crt ${k8s.ca."${ca_name}"}/${ca_name}-ca.key ./
+
+    openssl genrsa -out "${key_name}.key" 4096
+
+    openssl req -new -key "${key_name}.key" -sha256 \
+            -config "${ca_config}" -section ${key_name} \
+            -out "${key_name}.csr"
+
+    openssl x509 -req -days 3653 -in "${key_name}.csr" \
+            -copy_extensions copyall \
+            -sha256 -CA "./${ca_name}-ca.crt" \
+            -CAkey "./${ca_name}-ca.key" \
+            -CAcreateserial \
+            -out "${key_name}.crt"
+  '';
+
+  installPhase = ''
+    mkdir "$out"
+    cp "${key_name}.crt" "${key_name}.key" $out/
+  '';
+})
--- a/nix/kubernetes/keys/scope.nix
+++ b/nix/kubernetes/keys/scope.nix
@ -0,0 +1,374 @@
+{
+  makeScope,
+  newScope,
+  callPackage,
+  fetchFromGitHub,
+  lib,
+}:
+let
+  public_addresses = [
+    "74.80.180.138"
+  ];
+  internal_addresses = [
+    # nc0
+    "10.215.1.221"
+    "2620:11f:7001:7:ffff:ffff:0ad7:01dd"
+    # nc1
+    "10.215.1.222"
+    "2620:11f:7001:7:ffff:ffff:0ad7:01de"
+    # nc2
+    "10.215.1.223"
+    "2620:11f:7001:7:ffff:ffff:0ad7:01df"
+    # nw0
+    "10.215.1.224"
+    "2620:11f:7001:7:ffff:ffff:0ad7:01e0"
+    # nw1
+    "10.215.1.225"
+    "2620:11f:7001:7:ffff:ffff:0ad7:01e1"
+    # nw2
+    "10.215.1.226"
+    "2620:11f:7001:7:ffff:ffff:0ad7:01e2"
+  ];
+  all_hostnames = [
+    "10.197.0.1"
+    "10.0.0.1"
+    "127.0.0.1"
+    "kubernetes"
+    "kubernetes.default"
+    "kubernetes.default.svc"
+    "kubernetes.default.svc.cluster"
+    "kubernetes.svc.cluster.local"
+  ]
+  ++ public_addresses
+  ++ internal_addresses;
+  controllers = {
+    "controller0" = {
+      "internal_ips" = [
+        "10.215.1.221"
+        "2620:11f:7001:7:ffff:ffff:0ad7:01dd"
+      ];
+      "external_ips" = [
+        "2620:11f:7001:7:ffff:ffff:0ad7:01dd"
+      ];
+    };
+    "controller1" = {
+      "internal_ips" = [
+        "10.215.1.222"
+        "2620:11f:7001:7:ffff:ffff:0ad7:01de"
+      ];
+      "external_ips" = [
+        "2620:11f:7001:7:ffff:ffff:0ad7:01de"
+      ];
+    };
+    "controller2" = {
+      "internal_ips" = [
+        "10.215.1.223"
+        "2620:11f:7001:7:ffff:ffff:0ad7:01df"
+      ];
+      "external_ips" = [
+        "2620:11f:7001:7:ffff:ffff:0ad7:01df"
+      ];
+    };
+  };
+in
+makeScope newScope (
+  self:
+  let
+    additional_vars = {
+      inherit all_hostnames controllers;
+      k8s = self;
+    };
+    certificate_authorities = {
+      "client" = {
+        ca_config = ./package/k8s-ca/files/client-ca.conf;
+      };
+      "requestheader-client" = {
+        ca_config = ./package/k8s-ca/files/requestheader-client-ca.conf;
+      };
+    };
+    certificate_authorities_merged = (
+      builtins.mapAttrs (ca_name: ca_config: { inherit ca_name; } // ca_config) certificate_authorities
+    );
+  in
+  {
+    ca = (
+      builtins.mapAttrs (
+        ca_name: ca_config:
+        (callPackage ./package/k8s-ca/package.nix (additional_vars // { inherit ca_name; } // ca_config))
+      ) certificate_authorities
+    );
+    keys = (
+      builtins.mapAttrs
+        (
+          key_name: key_config:
+          (callPackage ./package/tls-key/package.nix (additional_vars // { inherit key_name; } // key_config))
+        )
+        {
+          "admin" = { } // certificate_authorities_merged.client;
+          "controller0" = { } // certificate_authorities_merged.client;
+          "controller1" = { } // certificate_authorities_merged.client;
+          "controller2" = { } // certificate_authorities_merged.client;
+          "worker0" = { } // certificate_authorities_merged.client;
+          "worker1" = { } // certificate_authorities_merged.client;
+          "worker2" = { } // certificate_authorities_merged.client;
+          "kube-proxy" = { } // certificate_authorities_merged.client;
+          "kube-scheduler" = { } // certificate_authorities_merged.client;
+          "kube-controller-manager" = { } // certificate_authorities_merged.client;
+          "kube-api-server" = { } // certificate_authorities_merged.client;
+          "service-accounts" = { } // certificate_authorities_merged.client;
+          "controller0-proxy" = { } // certificate_authorities_merged.requestheader-client;
+          "controller1-proxy" = { } // certificate_authorities_merged.requestheader-client;
+          "controller2-proxy" = { } // certificate_authorities_merged.requestheader-client;
+        }
+    );
+    ssh-keys = (
+      lib.genAttrs [
+        "flux_ssh_key"
+      ] (key_name: (callPackage ./package/ssh-key/package.nix (additional_vars // { inherit key_name; })))
+    );
+    pgp-keys = (
+      builtins.mapAttrs
+        (
+          key_name: key_config:
+          (callPackage ./package/pgp-key/package.nix (additional_vars // { inherit key_name; } // key_config))
+        )
+        {
+          "flux_gpg" = {
+            pgp_comment = "flux secrets";
+            pgp_name = "flux sops";
+          };
+        }
+    );
+    k8s-secrets-generic = (
+      builtins.mapAttrs
+        (
+          secret_name: secret_config:
+          (callPackage ./package/k8s-secret-generic/package.nix (
+            additional_vars // { inherit secret_name; } // secret_config
+          ))
+        )
+        {
+          "sops-gpg" = {
+            secret_namespace = "flux-system";
+            secret_values = {
+              "sops.asc" = (builtins.readFile "${self.pgp-keys.flux_gpg}/flux_gpg_private_key.asc");
+            };
+          };
+          "kubernetes-deploy-key" = {
+            secret_namespace = "flux-system";
+            secret_values = {
+              "identity" = builtins.readFile "${self.ssh-keys.flux_ssh_key}/flux_ssh_key";
+              "identity.pub" = builtins.readFile "${self.ssh-keys.flux_ssh_key}/flux_ssh_key.pub";
+              "known_hosts" = builtins.readFile ./generated/known_hosts;
+            };
+          };
+        }
+    );
+    client-configs = (
+      builtins.mapAttrs
+        (
+          config_name: config:
+          (callPackage ./package/k8s-client-config/package.nix (
+            additional_vars // { inherit config_name; } // config
+          ))
+        )
+        {
+          controller0 = {
+            config_user = "system:node:controller0";
+            config_server = "https://127.0.0.1:6443";
+            # config_server = "https://server.kubernetes.local:6443";
+          };
+          controller1 = {
+            config_user = "system:node:controller1";
+            config_server = "https://127.0.0.1:6443";
+            # config_server = "https://server.kubernetes.local:6443";
+          };
+          controller2 = {
+            config_user = "system:node:controller2";
+            config_server = "https://127.0.0.1:6443";
+            # config_server = "https://server.kubernetes.local:6443";
+          };
+          worker0 = {
+            config_user = "system:node:worker0";
+            config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443";
+            # config_server = "https://127.0.0.1:6443";
+            # config_server = "https://server.kubernetes.local:6443";
+          };
+          worker1 = {
+            config_user = "system:node:worker1";
+            config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443";
+            # config_server = "https://127.0.0.1:6443";
+            # config_server = "https://server.kubernetes.local:6443";
+          };
+          worker2 = {
+            config_user = "system:node:worker2";
+            config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443";
+            # config_server = "https://127.0.0.1:6443";
+            # config_server = "https://server.kubernetes.local:6443";
+          };
+          kube-proxy = {
+            config_user = "system:kube-proxy";
+            config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443";
+            # config_server = "https://127.0.0.1:6443";
+            # config_server = "https://server.kubernetes.local:6443";
+          };
+          kube-controller-manager = {
+            config_user = "system:kube-controller-manager";
+            # config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443";
+            config_server = "https://127.0.0.1:6443";
+            # config_server = "https://server.kubernetes.local:6443";
+          };
+          kube-scheduler = {
+            config_user = "system:kube-scheduler";
+            # config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443";
+            config_server = "https://127.0.0.1:6443";
+            # config_server = "https://server.kubernetes.local:6443";
+          };
+          admin = {
+            config_user = "admin";
+            config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443";
+            # config_server = "https://127.0.0.1:6443";
+          };
+        }
+    );
+    encryption_config = (callPackage ./package/k8s-encryption-key/package.nix additional_vars);
+    cilium-manifest =
+      let
+        version = "1.18.5";
+      in
+      (callPackage ./package/helm-manifest/package.nix (
+        additional_vars
+        // {
+          helm_src = fetchFromGitHub {
+            owner = "cilium";
+            repo = "cilium";
+            tag = "v${version}";
+            hash = "sha256-348inOOQ/fgwTYnaSHrQ363xGYnx2UPts3D4ycDRsWE=";
+          };
+          helm_name = "cilium";
+          helm_namespace = "kube-system";
+          helm_path = "install/kubernetes/cilium";
+          helm_manifest_name = "cilium.yaml";
+          helm_values = {
+            "kubeProxyReplacement" = true;
+            "ipam" = {
+              "mode" = "kubernetes";
+            };
+            "k8sServiceHost" = "2620:11f:7001:7:ffff:ffff:ad7:1dd";
+            "k8sServicePort" = 6443;
+            "ipv6" = {
+              "enabled" = true;
+            };
+            "ipv4" = {
+              "enabled" = true;
+            };
+            "enableIPv6Masquerade" = false;
+            "enableIPv4BIGTCP" = true;
+            "enableIPv6BIGTCP" = true;
+            "routingMode" = "native";
+            "autoDirectNodeRoutes" = true;
+            "ipv4NativeRoutingCIDR" = "10.200.0.0/16";
+            "ipv6NativeRoutingCIDR" = "2620:11f:7001:7:ffff::/80";
+
+            "hubble" = {
+              "relay" = {
+                "enabled" = true;
+              };
+              "ui" = {
+                "enabled" = true;
+              };
+
+              "gatewayAPI" = {
+                "enabled" = true;
+              };
+            };
+
+            "policyEnforcementMode" = "never";
+
+            # TODO: Read and maybe apply https://docs.cilium.io/en/stable/operations/performance/tuning/
+
+            # --set hostFirewall.enabled=true
+
+            # --set 'ipam.operator.clusterPoolIPv4PodCIDRList=["10.0.0.0/8"]' \
+            # --set 'ipam.operator.clusterPoolIPv6PodCIDRList=["fd00::/100"]' \
+
+            # --set encryption.enabled=true \
+            # --set encryption.type=wireguard
+            # --set encryption.nodeEncryption=true
+          };
+        }
+      ));
+    coredns-manifest =
+      let
+        version = "1.45.0";
+      in
+      (callPackage ./package/helm-manifest/package.nix (
+        additional_vars
+        // {
+          helm_src = fetchFromGitHub {
+            owner = "coredns";
+            repo = "helm";
+            tag = "coredns-${version}";
+            hash = "sha256-9YHd/jB33JXvySzx/p9DaP+/2p5ucyLjues4DNtOkmU=";
+          };
+          helm_name = "coredns";
+          helm_namespace = "kube-system";
+          helm_path = "charts/coredns";
+          helm_manifest_name = "coredns.yaml";
+          helm_values = {
+            "service" = {
+              "ipFamilyPolicy" = "PreferDualStack";
+              "clusterIP" = "fd00:3e42:e349::10";
+              "clusterIPs" = [
+                "fd00:3e42:e349::10"
+                "10.197.0.10"
+              ];
+            };
+
+            servers = [
+              {
+                zones = [
+                  {
+                    zone = ".";
+                    use_tcp = true;
+                  }
+                ];
+                port = 53;
+                plugins = [
+                  { name = "errors"; }
+                  {
+                    name = "health";
+                    configBlock = "lameduck 10s";
+                  }
+                  { name = "ready"; }
+                  {
+                    name = "kubernetes";
+                    parameters = "cluster.local in-addr.arpa ip6.arpa";
+                    configBlock = "pods insecure\nfallthrough in-addr.arpa ip6.arpa\nttl 30";
+                  }
+                  {
+                    name = "prometheus";
+                    parameters = "0.0.0.0:9153";
+                  }
+                  {
+                    name = "forward";
+                    parameters = ". /etc/resolv.conf";
+                  }
+                  {
+                    name = "cache";
+                    parameters = 300; # default 30
+                  }
+                  { name = "loop"; }
+                  { name = "reload"; }
+                  { name = "loadbalance"; }
+                ];
+              }
+            ];
+          };
+        }
+      ));
+    all_keys = (callPackage ./package/k8s-keys/package.nix additional_vars);
+    deploy_script = (callPackage ./package/deploy-script/package.nix additional_vars);
+    bootstrap_script = (callPackage ./package/bootstrap-script/package.nix additional_vars);
+  }
+)
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Tom Alexander	4b9322ab0d	Fix network for updated nix.	2026-02-21 15:11:15 -05:00
Tom Alexander	fc473c00da	Fix proxy auth tls	2026-02-21 15:11:15 -05:00
Tom Alexander	85d40613e6	Generate certificates for the aggregation layer.	2026-02-21 15:11:14 -05:00
Tom Alexander	481438f0fe	Add a note for the cilium connectivity test.	2026-02-21 15:11:14 -05:00
Tom Alexander	2e19d68ba5	Temporarily disable the firewall for debugging.	2026-02-21 15:11:14 -05:00
Tom Alexander	9830e34a45	Enable gateway support.	2026-02-21 15:11:14 -05:00
Tom Alexander	20007079df	Enable hubble.	2026-02-21 15:11:14 -05:00
Tom Alexander	657d5a5e55	Temporarily drop flux interval to 1 minute during early development. This is to reduce waiting time.	2026-02-21 15:11:14 -05:00
Tom Alexander	31dd69cfcb	Install deferred manifests.	2026-02-21 15:11:14 -05:00
Tom Alexander	a4353e438e	Add a custom nftables firewall config.	2026-02-21 15:11:13 -05:00
Tom Alexander	1902e132a7	Enable the firewall. Now that we have networking working, I can enable the firewall and confirm nothing breaks.	2026-02-21 15:11:13 -05:00
Tom Alexander	da9b91a47c	Fix CoreDNS IPv4 connectivity.	2026-02-21 15:11:13 -05:00
Tom Alexander	b9d916fdc7	Increase timeout for coredns cache.	2026-02-21 15:11:13 -05:00
Tom Alexander	7f6f8352c0	More changes to try to fix coredns.	2026-02-21 15:11:13 -05:00
Tom Alexander	8e043ba48a	Move the kubelet yaml config into nix.	2026-02-21 15:11:13 -05:00
Tom Alexander	218aaf97c6	Implement a generic helm templater package.	2026-02-21 15:11:12 -05:00
Tom Alexander	35a1b8c227	Switch to generating the coredns manifests via nix.	2026-02-21 15:11:12 -05:00
Tom Alexander	e3c61fe4db	Use CoreDNS for in-cluster DNS requests and caching.	2026-02-21 15:11:12 -05:00
Tom Alexander	9314691e7e	Enable native routing.	2026-02-21 15:11:12 -05:00
Tom Alexander	740e3a17e5	Build the cilium manifest automatically in nix.	2026-02-21 15:11:12 -05:00
Tom Alexander	5c4ac7ea59	Allow pods to directly speak to the public internet on their own public IPv6 addresses.	2026-02-21 15:11:12 -05:00
Tom Alexander	323e2fff83	Enable ipv4 and tunnel routing.	2026-02-21 15:11:12 -05:00
Tom Alexander	f5286bd10e	Switch to kubernetes ipam mode.	2026-02-21 15:11:11 -05:00
Tom Alexander	87429a2953	Fix service cluster ip range. Kubernetes only allows a /112 for service ip range.	2026-02-21 15:11:11 -05:00
Tom Alexander	a32eb663b1	Fix trailing line break in kubernetes encryption config.	2026-02-21 15:11:11 -05:00
Tom Alexander	bd7c86a10e	Move the yaml functions to their own file.	2026-02-21 15:11:11 -05:00
Tom Alexander	a99477797c	Introduce functions to generate yaml. The toYAML function is just an alias to toJSON which is technically fine since YAML is a superset of JSON, but these new functions will generate actual YAML.	2026-02-21 15:11:11 -05:00
Tom Alexander	b5f9a7c812	Add missing cidr declarations.	2026-02-21 15:11:11 -05:00
Tom Alexander	f0e6b30c98	Fix DNS resolution.	2026-02-21 15:11:10 -05:00
Tom Alexander	70c10aba40	Apply the git repo to the cluster.	2026-02-21 15:11:10 -05:00
Tom Alexander	32b27fba36	Trust flux's ssh key in the yaml git repo.	2026-02-21 15:11:10 -05:00
Tom Alexander	90a97f4e74	Generic secrets for ssh keys.	2026-02-21 15:11:10 -05:00
Tom Alexander	21ee9a631c	Generic secrets for pgp keys.	2026-02-21 15:11:10 -05:00
Tom Alexander	9529f4b805	Generate kubernetes secrets for ssh keys.	2026-02-21 15:11:10 -05:00
Tom Alexander	f3e7d56b66	Install CoreDNS.	2026-02-21 15:11:09 -05:00
Tom Alexander	5593a1fccd	Generate pgp keys for sops.	2026-02-21 15:11:09 -05:00
Tom Alexander	ff04f8d951	Generate ssh keys for flux bootstrap.	2026-02-21 15:11:09 -05:00
Tom Alexander	50ba6f7c87	Move the cluster bootstrap into the keys flake. Bootstrapping the cluster needs access to secrets, so I am moving it into the keys flake.	2026-02-21 15:11:09 -05:00
Tom Alexander	2f1075c20c	Set up flux.	2026-02-21 15:11:09 -05:00
Tom Alexander	41a9bc263c	Add a bootstrap role.	2026-02-21 15:11:09 -05:00
Tom Alexander	8ca78b023e	Add a bootstrap role to load manifests into the cluster.	2026-02-21 15:11:09 -05:00
Tom Alexander	118c63a90d	Fix launching of containers.	2026-02-21 15:11:08 -05:00
Tom Alexander	05f7f18e93	Create a debugging role.	2026-02-21 15:11:08 -05:00
Tom Alexander	2d49aff563	Some networking fixes.	2026-02-21 15:11:08 -05:00
Tom Alexander	ac9fa195ae	Add cilium bootstrap.	2026-02-21 15:11:08 -05:00
Tom Alexander	af7ec0414e	Installing the cni plugins.	2026-02-21 15:11:08 -05:00
Tom Alexander	fbfa3dc5dc	Add kube-proxy.	2026-02-21 15:11:08 -05:00
Tom Alexander	2ac0bfe5f8	Add kubelet.	2026-02-21 15:11:07 -05:00
Tom Alexander	fb0d592a84	Add worker nodes.	2026-02-21 15:11:07 -05:00
Tom Alexander	13f3237359	Add kube-scheduler.	2026-02-21 15:11:07 -05:00
Tom Alexander	7365efd97f	Add kube-controller-manager.	2026-02-21 15:11:07 -05:00
Tom Alexander	8c59a1216f	Fix launching kube-apiserver.	2026-02-21 15:11:07 -05:00
Tom Alexander	cd86934dde	Move the encryption config into a package.	2026-02-21 15:11:07 -05:00
Tom Alexander	f1346a52ce	Switch to generating certs with openssl.	2026-02-21 15:11:07 -05:00
Tom Alexander	f57df2d855	Add controller proxy certs.	2026-02-21 15:11:06 -05:00
Tom Alexander	5f459db540	Add requestheader-client-ca.	2026-02-21 15:11:06 -05:00
Tom Alexander	9956009638	Add service account.	2026-02-21 15:11:06 -05:00
Tom Alexander	f713ac372b	Install kubernetes.	2026-02-21 15:11:06 -05:00
Tom Alexander	a140d691f4	Add additional controllers.	2026-02-21 15:11:06 -05:00
Tom Alexander	4c029aa0b0	Add configs for a new kubernetes cluster on NixOS.	2026-02-21 15:11:06 -05:00
Tom Alexander	185b2dcba2	Remove home manager from the steam deck.	2026-02-21 15:11:05 -05:00
Tom Alexander	3577edab9d	Clean up steam deck config.	2026-02-21 15:11:05 -05:00
Tom Alexander	5c2df30757	Update packages on steam deck.	2026-02-21 15:11:05 -05:00
Tom Alexander	c6fa57b47e	Disable some stuff in firefox.	2026-02-21 15:11:05 -05:00
Tom Alexander	c0ab442fe7	Use rust nix-builder instead of bash script.	2026-02-21 15:11:05 -05:00