Add configs for a new kubernetes cluster on NixOS.

nix/kubernetes/roles/boot/default.nix

@@ -0,0 +1,130 @@
+# ISO does not work with systemd initrd yet https://github.com/NixOS/nixpkgs/pull/291750
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  imports = [ ];
+
+  options.me = {
+    boot.enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+      description = "Whether we want to install boot.";
+    };
+
+    boot.secure = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+      description = "Enable to use secure boot.";
+    };
+
+    rollback.enable = lib.mkOption {
+      type = lib.types.bool;
+      default = true;
+      example = true;
+      description = "Whether we want to enable rolling back during boot.";
+    };
+
+    rollback.dataset = lib.mkOption {
+      default = { };
+      example = lib.literalExpression ''
+        {
+          "zroot/linux/nix/root@blank" = true;
+          "zroot/linux/nix/home@blank" = lib.mkForce false;
+        }
+      '';
+      type = lib.types.coercedTo (lib.types.listOf lib.types.str) (
+        enabled: lib.listToAttrs (map (fs: lib.nameValuePair fs true) enabled)
+      ) (lib.types.attrsOf lib.types.bool);
+      description = "List of ZFS datasets to rollback to during boot.";
+    };
+  };
+
+  config = lib.mkIf config.me.boot.enable (
+    lib.mkMerge [
+      {
+        environment.systemPackages = with pkgs; [
+          tpm2-tools # For tpm2_eventlog to check for OptionRoms
+          # cp /sys/kernel/security/tpm0/binary_bios_measurements eventlog
+          # tpm2_eventlog eventlog | grep "BOOT_SERVICES_DRIVER"
+          sbctl # For debugging and troubleshooting Secure Boot.
+          efibootmgr # To set EFI boot order.
+        ];
+      }
+      (lib.mkIf (!config.me.buildingPortable) {
+
+        boot.loader.grub.enable = false;
+        # Use the systemd-boot EFI boot loader.
+        boot.loader.systemd-boot.enable = true;
+        # TODO: make not write bootx64.efi
+        boot.loader.efi.canTouchEfiVariables = false;
+
+        # Automatically delete old generations
+        boot.loader.systemd-boot.configurationLimit = 3;
+
+        boot.loader.systemd-boot.memtest86.enable = true;
+
+        # Check what will be lost with `zfs diff zroot/linux/root@blank`
+        boot.initrd.systemd.enable = lib.mkDefault true;
+        boot.initrd.systemd.services.zfs-rollback = lib.mkIf config.me.rollback.enable {
+          description = "Rollback ZFS root dataset to blank snapshot";
+          wantedBy = [
+            "initrd.target"
+          ];
+          after = [
+            "zfs-import-zroot.service"
+          ];
+          before = [
+            "sysroot.mount"
+          ];
+          unitConfig.DefaultDependencies = "no";
+          serviceConfig.Type = "oneshot";
+          script = lib.concatStringsSep "\n" (
+            (builtins.map (ds: "${config.boot.zfs.package}/sbin/zfs rollback -r '${ds}'") (
+              builtins.attrNames config.me.rollback.dataset
+            ))
+            ++ [ ''echo "rollback complete"'' ]
+          );
+        };
+
+        # boot.loader.systemd-boot.extraEntries = {
+        #   "windows.conf" = ''
+        #     title Windows
+        #     efi /EFI/Microsoft/Boot/bootmgfw.efi
+        #     options root=PARTUUID=17e325bf-a378-4d1d-be6a-f6df5476f0fa
+        #   '';
+        # };
+        environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) {
+          hideMounts = true;
+          directories = [
+            "/var/lib/sbctl" # Secure Boot Keys
+          ];
+        };
+      })
+      (lib.mkIf (config.me.boot.secure) {
+        environment.systemPackages = with pkgs; [
+          sbctl
+        ];
+        boot.loader.systemd-boot.enable = lib.mkForce false;
+        boot.lanzaboote = {
+          enable = true;
+          pkiBundle = "/var/lib/sbctl";
+        };
+      })
+    ]
+  );
+}
+# efibootmgr -c -d /dev/sda -p 1 -L NixOS-boot -l '\EFI\NixOS-boot\grubx64.efi'
+
+# Text-only:
+# sudo cp "$(nix-build '<nixpkgs>' --no-out-link -A 'refind')/share/refind/refind_x64.efi" /boot/EFI/boot/bootx64.efi
+
+# Full graphics:
+# $ sudo nix-shell -p refind efibootmgr
+# $ refind-install

nix/kubernetes/roles/doas/default.nix

@@ -0,0 +1,36 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  imports = [ ];
+
+  options.me = {
+    doas.enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+      description = "Whether we want to install doas.";
+    };
+  };
+
+  config = lib.mkIf config.me.doas.enable {
+    # Use doas instead of sudo
+    security.doas.enable = true;
+    security.doas.wheelNeedsPassword = false;
+    security.sudo.enable = false;
+    security.doas.extraRules = [
+      {
+        # Retain environment (for example NIX_PATH)
+        keepEnv = true;
+        persist = true; # Only ask for a password the first time.
+      }
+    ];
+    environment.systemPackages = with pkgs; [
+      doas-sudo-shim # To support --sudo for remote builds
+    ];
+  };
+}

nix/kubernetes/roles/dont_use_substituters/default.nix

@@ -0,0 +1,25 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  imports = [ ];
+
+  options.me = {
+    dont_use_substituters.enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+      description = "Whether we want to install dont_use_substituters.";
+    };
+  };
+
+  config = lib.mkIf config.me.dont_use_substituters.enable {
+    # Disable substituters to avoid risk of cache poisoning.
+    nix.settings.substitute = false;
+    nix.settings.substituters = lib.mkForce [ ];
+  };
+}

nix/kubernetes/roles/etcd/default.nix

@@ -0,0 +1,92 @@
+{
+  config,
+  lib,
+  pkgs,
+  self,
+  ...
+}:
+
+{
+  imports = [ ];
+
+  options.me = {
+    etcd.enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+      description = "Whether we want to install etcd.";
+    };
+
+    etcd.cluster_name = lib.mkOption {
+      type = lib.types.str;
+      default = false;
+      example = "lorem";
+      description = "The unique name for the cluster.";
+    };
+
+    etcd.internal_ip = lib.mkOption {
+      default = { };
+      example = lib.literalExpression ''
+        {
+          "172.16.0.10" = true;
+          "192.168.1.10" = lib.mkForce false;
+        }
+      '';
+      type = lib.types.coercedTo (lib.types.listOf lib.types.str) (
+        enabled: lib.listToAttrs (map (fs: lib.nameValuePair fs true) enabled)
+      ) (lib.types.attrsOf lib.types.bool);
+      description = "List internal IP addresses for accessing this node.";
+    };
+
+    etcd.initial_cluster = lib.mkOption {
+      default = [ ];
+      example = [
+        "controller0=https://10.215.1.221:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01dd
+        "controller1=https://10.215.1.222:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01de
+        "controller2=https://10.215.1.223:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01df
+      ];
+      type = lib.types.listOf lib.types.str;
+      description = "List of controller nodes to form the initial etcd cluster.";
+    };
+  };
+
+  config = lib.mkIf config.me.etcd.enable {
+    services.etcd = {
+      enable = true;
+      openFirewall = true;
+      name = config.networking.hostName;
+      certFile = "/.disk/keys/kubernetes.pem";
+      keyFile = "/.disk/keys/kubernetes-key.pem";
+      peerCertFile = "/.disk/keys/kubernetes.pem";
+      peerKeyFile = "/.disk/keys/kubernetes-key.pem";
+      trustedCaFile = "/.disk/keys/ca.pem";
+      peerTrustedCaFile = "/.disk/keys/ca.pem";
+      peerClientCertAuth = true;
+      initialAdvertisePeerUrls = (
+        builtins.map (iip: "https://${iip}:2380") (builtins.attrNames config.me.etcd.internal_ip)
+      );
+      listenPeerUrls = (
+        builtins.map (iip: "https://${iip}:2380") (builtins.attrNames config.me.etcd.internal_ip)
+      );
+      listenClientUrls = (
+        [
+          "https://127.0.0.1:2379"
+        ]
+        ++ (builtins.map (iip: "https://${iip}:2379") (builtins.attrNames config.me.etcd.internal_ip))
+      );
+      advertiseClientUrls = (
+        builtins.map (iip: "https://${iip}:2379") (builtins.attrNames config.me.etcd.internal_ip)
+      );
+      initialClusterToken = config.me.etcd.cluster_name;
+      initialCluster = config.me.etcd.initial_cluster;
+      initialClusterState = "new";
+    };
+
+    environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) {
+      hideMounts = true;
+      directories = [
+        config.services.etcd.dataDir # "/var/lib/etcd"
+      ];
+    };
+  };
+}

nix/kubernetes/roles/image_based_appliance/default.nix

@@ -0,0 +1,30 @@
+{
+  config,
+  lib,
+  ...
+}:
+
+{
+  imports = [ ];
+
+  options.me = {
+    image_based_appliance.enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+      description = "Whether we want to install image_based_appliance.";
+    };
+  };
+
+  config = lib.mkIf config.me.image_based_appliance.enable (
+    lib.mkMerge [
+      {
+        # Do not install nix. A full new image must be built to update
+        # the machine.
+        nix.enable = false;
+        system.switch.enable = false;
+        nix.gc.automatic = lib.mkForce false;
+      }
+    ]
+  );
+}

nix/kubernetes/roles/iso/default.nix

@@ -0,0 +1,22 @@
+{
+  lib,
+  ...
+}:
+
+{
+  imports = [ ];
+
+  options.me.buildingPortable = lib.mkOption {
+    type = lib.types.bool;
+    default = false;
+    example = true;
+    description = "Whether we are building a portable image (iso/sd). This would disable CPU-specific optimizations and persistent file mounts.";
+  };
+
+  options.me.mountPersistence = lib.mkOption {
+    type = lib.types.bool;
+    default = false;
+    example = true;
+    description = "Whether we should mount persistent directories.";
+  };
+}

nix/kubernetes/roles/minimal_base/default.nix

@@ -0,0 +1,33 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  imports = [ ];
+
+  options.me = {
+    minimal_base.enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+      description = "Whether we want to install minimal_base.";
+    };
+  };
+
+  config = lib.mkIf config.me.minimal_base.enable {
+    me.doas.enable = true;
+    me.network.enable = true;
+    me.nvme.enable = true;
+    me.ssh.enable = true;
+    me.sshd.enable = true;
+    me.user.enable = true;
+    me.zfs.enable = true;
+    me.zrepl.enable = true;
+    me.zsh.enable = true;
+
+    # TODO: Maybe add me.boot.enable ?
+  };
+}

nix/kubernetes/roles/network/default.nix

@@ -0,0 +1,87 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+# Alternative DNS servers:
+# "1.0.0.1#cloudflare-dns.com"
+# "1.1.1.1#cloudflare-dns.com"
+# "2606:4700:4700::1001#cloudflare-dns.com"
+# "2606:4700:4700::1111#cloudflare-dns.com"
+# "8.8.4.4#dns.google"
+# "8.8.8.8#dns.google"
+# "2001:4860:4860::8844#dns.google"
+# "2001:4860:4860::8888#dns.google"
+
+{
+  imports = [ ];
+
+  options.me = {
+    network.enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+      description = "Whether we want to install network.";
+    };
+  };
+
+  config = lib.mkIf config.me.network.enable {
+    networking.dhcpcd.enable = lib.mkDefault false;
+    networking.useDHCP = lib.mkDefault false;
+    networking.nameservers = [
+      "194.242.2.2#doh.mullvad.net"
+      "2a07:e340::2#doh.mullvad.net"
+    ];
+    services.resolved = {
+      enable = true;
+      # dnssec = "true";
+      domains = [ "~." ];
+      fallbackDns = [ ];
+      dnsovertls = "true";
+    };
+
+    # Without this, systemd-resolved will send DNS requests for <X>.home.arpa to the per-link DNS server (172.16.0.1) which does not support DNS-over-TLS. This leads to the connection hanging and timing out. This causes firefox startup to take an extra 10+ seconds.
+    #
+    # Test with: drill @127.0.0.53 odo.home.arpa
+    # TODO: The 127.0.0.1 address should probably be moved to a host-specific file.
+    networking.extraHosts = ''
+      127.0.0.1 ${config.networking.hostName}.home.arpa
+    '';
+
+    environment.systemPackages = with pkgs; [
+      iw
+      ldns # for drill
+      arp-scan # To find devices on the network
+      wavemon
+      dhcpcd # For Android USB tethering.
+    ];
+
+    boot.extraModprobeConfig = ''
+      # Set wifi to US
+      options cfg80211 ieee80211_regdom=US
+    '';
+
+    boot.kernel.sysctl = {
+      # Enable TCP packetization-layer PMTUD when an ICMP black hole is detected.
+      "net.ipv4.tcp_mtu_probing" = 1;
+      # Switch to bbr tcp congestion control which should be better on lossy connections like bad wifi.
+      # We set this in the kernel config, but include this here for unoptimized builds.
+      "net.ipv4.tcp_congestion_control" = "bbr";
+      # Don't do a slow start after a connection has been idle for a single RTO.
+      "net.ipv4.tcp_slow_start_after_idle" = 0;
+      # 3x time to accumulate filesystem changes before flushing to disk.
+      "vm.dirty_writeback_centisecs" = 1500;
+      # Adjust ttl
+      "net.ipv4.ip_default_ttl" = 65;
+      "net.ipv6.conf.all.hop_limit" = 65;
+      "net.ipv6.conf.default.hop_limit" = 65;
+      # Enable IPv6 Privacy Extensions
+      "net.ipv6.conf.all.use_tempaddr" = 2;
+      # Enable IPv6 Privacy Extensions
+      # This is enabled by default in nixos.
+      # "net.ipv6.conf.default.use_tempaddr" = 2;
+    };
+  };
+}

nix/kubernetes/roles/nvme/default.nix

@@ -0,0 +1,25 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  imports = [ ];
+
+  options.me = {
+    nvme.enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+      description = "Whether we want to install nvme.";
+    };
+  };
+
+  config = lib.mkIf config.me.nvme.enable {
+    environment.systemPackages = with pkgs; [
+      nvme-cli
+    ];
+  };
+}

nix/kubernetes/roles/optimized_build/default.nix

@@ -0,0 +1,134 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  imports = [ ];
+
+  options.me = {
+    optimizations.enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+      description = "Whether we want to enable CPU optimizations (will trigger a rebuild from source).";
+    };
+
+    optimizations.arch = lib.mkOption {
+      type = lib.types.str;
+      default = null;
+      example = "znver4";
+      description = "The CPU arch for which programs should be optimized.";
+    };
+
+    optimizations.build_arch = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      example = "znver4";
+      description = "The CPU arch for which programs should be optimized.";
+    };
+
+    optimizations.system_features = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      example = [
+        "gccarch-znver4"
+        "gccarch-znver5"
+        "gccarch-skylake"
+        "gccarch-x86-64-v3"
+        "gccarch-x86-64-v4"
+        "benchmark"
+        "big-parallel"
+        "kvm"
+        "nixos-test"
+      ];
+      description = "The list of CPU features that should be enabled on this machine.";
+    };
+  };
+
+  config = lib.mkMerge [
+    (lib.mkIf (!config.me.optimizations.enable) (
+      lib.mkMerge [
+        {
+          # boot.kernelPackages = pkgs.linuxPackagesFor pkgs.linux_6_17;
+          boot.kernelPackages = pkgs.linuxPackagesFor pkgs.linux;
+        }
+      ]
+    ))
+    (lib.mkIf config.me.optimizations.enable (
+      lib.mkMerge [
+        {
+          boot.kernelPackages = pkgs.linuxPackagesFor pkgs.linux_me;
+
+          nixpkgs.hostPlatform = {
+            gcc.arch = config.me.optimizations.arch;
+            gcc.tune = config.me.optimizations.arch;
+          };
+
+          nixpkgs.overlays = [
+            (
+              final: prev:
+              let
+                addConfig =
+                  additionalConfig: pkg:
+                  pkg.override (oldconfig: {
+                    structuredExtraConfig = pkg.structuredExtraConfig // additionalConfig;
+                  });
+              in
+              {
+                linux_me = addConfig {
+                  # Full preemption
+                  PREEMPT = lib.mkOverride 60 lib.kernel.yes;
+                  PREEMPT_VOLUNTARY = lib.mkOverride 60 lib.kernel.no;
+
+                  # Google's BBRv3 TCP congestion Control
+                  TCP_CONG_BBR = lib.kernel.yes;
+                  DEFAULT_BBR = lib.kernel.yes;
+
+                  # Preemptive Full Tickless Kernel at 300Hz
+                  HZ = lib.kernel.freeform "300";
+                  HZ_300 = lib.kernel.yes;
+                  HZ_1000 = lib.kernel.no;
+                } prev.linux; # or prev.linux_6_17
+              }
+            )
+            (final: prev: {
+              inherit (final.unoptimized)
+                assimp
+                binaryen
+                gsl
+                rapidjson
+                ffmpeg-headless
+                ffmpeg
+                pipewire
+                chromaprint
+                gtkmm
+                ;
+            })
+          ];
+        }
+      ]
+    ))
+    (lib.mkIf (config.me.optimizations.build_arch != null) (
+      lib.mkMerge [
+        {
+          # Enable cross-compiling
+          nixpkgs.buildPlatform = {
+            gcc.arch = config.me.optimizations.build_arch;
+            gcc.tune = "generic";
+            system = "x86_64-linux";
+          };
+        }
+      ]
+    ))
+    (lib.mkIf (config.me.optimizations.system_features != [ ]) (
+      lib.mkMerge [
+        {
+          nix.settings.system-features = lib.mkForce config.me.optimizations.system_features;
+        }
+      ]
+    ))
+  ];
+}

nix/kubernetes/roles/ssh/default.nix

@@ -0,0 +1,50 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  imports = [ ];
+
+  options.me = {
+    ssh.enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+      description = "Whether we want to install ssh.";
+    };
+  };
+
+  config = lib.mkIf config.me.ssh.enable {
+    environment.systemPackages = with pkgs; [
+      sshfs
+    ];
+
+    environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) {
+      hideMounts = true;
+      users.talexander = {
+        files = [
+          ".ssh/known_hosts"
+        ];
+      };
+      users.root = {
+        files = [
+          ".ssh/known_hosts"
+        ];
+      };
+    };
+
+    me.install.user.root.file = {
+      ".ssh/config" = {
+        source = ./files/ssh_config_root;
+      };
+    };
+    me.install.user.talexander.file = {
+      ".ssh/config" = {
+        source = ./files/ssh_config;
+      };
+    };
+  };
+}

nix/kubernetes/roles/ssh/files/ssh_config

@@ -0,0 +1,42 @@
+Host poudriere
+  ProxyJump talexander@mrmanager
+  HostName 10.215.1.203
+
+Host controller0
+  ProxyJump talexander@mrmanager
+  HostName 10.215.1.204
+
+Host controller1
+  ProxyJump talexander@mrmanager
+  HostName 10.215.1.205
+
+Host controller2
+  ProxyJump talexander@mrmanager
+  HostName 10.215.1.206
+
+Host worker0
+  ProxyJump talexander@mrmanager
+  HostName 10.215.1.207
+
+Host worker1
+  ProxyJump talexander@mrmanager
+  HostName 10.215.1.208
+
+Host worker2
+  ProxyJump talexander@mrmanager
+  HostName 10.215.1.209
+
+Host brianai
+  ProxyJump talexander@mrmanager
+  HostName 10.215.1.215
+
+Host hydra
+  ProxyJump talexander@mrmanager
+  HostName 10.215.1.219
+
+Host i_only_boot_zfs
+  HostName 127.0.0.1
+  Port 60022
+
+Host *
+  Compression yes

nix/kubernetes/roles/ssh/files/ssh_config_root

@@ -0,0 +1,9 @@
+Host hydra
+  HostName ns1.fizz.buzz
+  Port 65122
+  User nixworker
+  IdentitiesOnly yes
+  IdentityFile /persist/manual/ssh/root/keys/id_ed25519
+
+Host *
+  Compression yes

nix/kubernetes/roles/sshd/default.nix

@@ -0,0 +1,49 @@
+{
+  config,
+  lib,
+  ...
+}:
+
+{
+  imports = [ ];
+
+  options.me = {
+    sshd.enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+      description = "Whether we want to install sshd.";
+    };
+  };
+
+  config = lib.mkIf config.me.sshd.enable {
+    services.openssh = {
+      enable = true;
+      settings = {
+        PasswordAuthentication = false;
+        KbdInteractiveAuthentication = false;
+      };
+      hostKeys = [
+        {
+          path = "/persist/ssh/ssh_host_ed25519_key";
+          type = "ed25519";
+        }
+        {
+          path = "/persist/ssh/ssh_host_rsa_key";
+          type = "rsa";
+          bits = 4096;
+        }
+      ];
+    };
+
+    environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) {
+      hideMounts = true;
+      files = [
+        "/etc/ssh/ssh_host_rsa_key"
+        "/etc/ssh/ssh_host_rsa_key.pub"
+        "/etc/ssh/ssh_host_ed25519_key"
+        "/etc/ssh/ssh_host_ed25519_key.pub"
+      ];
+    };
+  };
+}

nix/kubernetes/roles/user/default.nix

@@ -0,0 +1,59 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  imports = [ ];
+
+  options.me = {
+    user.enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+      description = "Whether we want to create my user.";
+    };
+  };
+
+  config = lib.mkIf config.me.user.enable {
+    services.getty = {
+      autologinUser = "talexander"; # I use full disk encryption so the user password is irrelevant.
+      autologinOnce = true;
+    };
+    users.mutableUsers = false;
+    users.users.talexander = {
+      isNormalUser = true;
+      createHome = true; # https://github.com/NixOS/nixpkgs/issues/6481
+      group = "talexander";
+      extraGroups = [ "wheel" ];
+      uid = 11235;
+      packages = with pkgs; [
+        tree
+      ];
+      # Generate with `mkpasswd -m scrypt`
+      hashedPassword = "$7$CU..../....VXvNQ8za3wSGpdzGXNT50/$HcFtn/yvwPMCw4888BelpiAPLAxe/zU87fD.d/N6U48";
+      openssh.authorizedKeys.keys = [
+        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID0+4zi26M3eYWnIrciR54kOlGxzfgCXG+o4ea1zpzrk openpgp:0x7FF123C8"
+        "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIEI6mu6I5Jp+Ib0vJxapGHbEShZjyvzV8jz5DnzDrI39AAAABHNzaDo="
+        "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIAFNcSXwvy+brYTOGo56G93Ptuq2MmZsjvRWAfMqbmMLAAAABHNzaDo="
+      ];
+    };
+    users.groups.talexander.gid = 11235;
+
+    environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) {
+      hideMounts = true;
+      users.talexander = {
+        directories = [
+          {
+            directory = "persist";
+            user = "talexander";
+            group = "talexander";
+            mode = "0700";
+          }
+        ];
+      };
+    };
+  };
+}

nix/kubernetes/roles/zfs/default.nix

@@ -0,0 +1,68 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  zfs_clone_send =
+    (pkgs.writeScriptBin "zfs_clone_send" (builtins.readFile ./files/zfs_clone_send.bash)).overrideAttrs
+      (old: {
+        buildCommand = "${old.buildCommand}\n patchShebangs $out";
+
+      });
+  zfs_clone_recv =
+    (pkgs.writeScriptBin "zfs_clone_recv" (builtins.readFile ./files/zfs_clone_recv.bash)).overrideAttrs
+      (old: {
+        buildCommand = "${old.buildCommand}\n patchShebangs $out";
+
+      });
+  zfs_clone_resume =
+    (pkgs.writeScriptBin "zfs_clone_resume" (builtins.readFile ./files/zfs_clone_resume.bash))
+    .overrideAttrs
+      (old: {
+        buildCommand = "${old.buildCommand}\n patchShebangs $out";
+
+      });
+in
+{
+  imports = [ ];
+
+  options.me = {
+    zfs.enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+      description = "Whether we want to install zfs.";
+    };
+  };
+
+  config = lib.mkIf config.me.zfs.enable {
+    # Technically only needed when building the ISO because nix detects ZFS in the filesystem list normally. I basically always want this so I'm just setting it to always be on.
+    boot.supportedFilesystems.zfs = true;
+
+    boot.zfs.devNodes = "/dev/disk/by-partuuid";
+
+    services.zfs = {
+      autoScrub = {
+        enable = true;
+        interval = "monthly";
+      };
+      trim.enable = true;
+    };
+
+    environment.systemPackages = [
+      zfs_clone_send
+      zfs_clone_recv
+      zfs_clone_resume
+    ];
+
+    environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) {
+      hideMounts = true;
+      directories = [
+        "/etc/zfs/zpool.cache" # Which zpools to import, the root zpool is already imported and does not need this cache file but this captures additional pools.
+      ];
+    };
+  };
+}

nix/kubernetes/roles/zfs/files/zfs_clone_recv.bash

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+# A zfs-send alias that creates a perfect clone with good defaults.
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# -s if the stream is interrupted, save the partial stream. The stream can then be resumed by doing a zfs send -t token where token is the receive_resume_token prop on the dataset we received into.
+# -u Do not mount the filesystem we are receiving. We can always mount afterwards but this avoids issues with streams with mountpoints to places like /
+# Can optionally add -F to destroy the dataset in the recv location.
+exec zfs recv -s -u "${@}"
+
+# To delete an interrupted recv, run `zfs receive -A dataset`

nix/kubernetes/roles/zfs/files/zfs_clone_resume.bash

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+#
+# Resume a zfs send.
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+function main {
+    local hst="$1"
+    local dst="$2"
+    local token
+    token=$(zfs get -H -o value receive_resume_token "$dst")
+    ssh "$hst" doas zfs send --verbose -t "$token" | doas zfs recv -s "$dst"
+
+}
+
+main "${@}"

nix/kubernetes/roles/zfs/files/zfs_clone_send.bash

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+#
+# A zfs-send alias that creates a perfect clone with good defaults.
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+exec zfs send --compressed --replicate --large-block --embed --verbose --raw "${@}"

nix/kubernetes/roles/zrepl/default.nix

@@ -0,0 +1,55 @@
+{
+  config,
+  lib,
+  ...
+}:
+
+{
+  imports = [ ];
+
+  options.me = {
+    zrepl.enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+      description = "Whether we want to install zrepl.";
+    };
+  };
+
+  config = lib.mkIf config.me.zrepl.enable {
+    services.zrepl = {
+      enable = true;
+      settings = {
+        jobs = [
+          {
+            name = "snapjob";
+            type = "snap";
+            filesystems = {
+              "zroot/linux/nix/persist<" = true;
+              "zroot/bridge<" = true;
+            };
+            snapshotting = {
+              type = "periodic";
+              interval = "15m";
+              prefix = "zrepl_";
+            };
+            pruning = {
+              keep = [
+                {
+                  type = "grid";
+                  grid = "1x1h(keep=all) | 24x1h | 14x1d";
+                  regex = "^zrepl_.*";
+                }
+                {
+                  type = "regex";
+                  negate = true;
+                  regex = "^zrepl_.*";
+                }
+              ];
+            };
+          }
+        ];
+      };
+    };
+  };
+}

nix/kubernetes/roles/zsh/default.nix

@@ -0,0 +1,117 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  zshrc = pkgs.writeTextFile {
+    name = ".zshrc";
+    text = ''
+      # Lines configured by zsh-newuser-install
+      HISTFILE=~/.zhistory
+      HISTSIZE=100000
+      SAVEHIST=100000
+      setopt appendhistory notify
+      unsetopt beep
+      bindkey -e
+      # End of lines configured by zsh-newuser-install
+      # The following lines were added by compinstall
+      #
+
+      # Use menu complete immediately instead of after the first tab
+      setopt MENU_COMPLETE
+
+      zstyle :compinstall filename "$HOME/.zshrc"
+
+      autoload -Uz compinit
+      compinit
+      # End of lines added by compinstall
+
+      # Enable the 2d menu for tab completion
+      zstyle ':completion:*' menu select
+
+      autoload colors zsh/terminfo
+      if [[ "$terminfo[colors]" -ge 8 ]]; then
+          colors
+      fi
+      for color in RED GREEN YELLOW BLUE MAGENTA CYAN WHITE; do
+          eval PR_$color='%{$terminfo[bold]$fg[''${(L)color}]%}'
+          eval PR_LIGHT_$color='%{$fg[''${(L)color}]%}'
+          (( count = $count + 1 ))
+      done
+      PR_NO_COLOR="%{$terminfo[sgr0]%}"
+      PS1="[$PR_BLUE%n$PR_WHITE@$PR_GREEN%U%m%u$PR_NO_COLOR:$PR_RED%2c$PR_NO_COLOR]%(!.#.$) "
+
+      source ${pkgs.zsh-histdb}/share/zsh/plugins/zsh-histdb/sqlite-history.zsh
+      autoload -Uz add-zsh-hook
+
+      source ${pkgs.zsh-histdb}/share/zsh/plugins/zsh-histdb/histdb-interactive.zsh
+      bindkey '^r' _histdb-isearch
+
+      ${lib.concatMapStringsSep "\n" (item: "source ${item}") config.me.zsh.includes}
+    '';
+  };
+in
+{
+  imports = [ ];
+
+  options.me = {
+    zsh.enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+      description = "Whether we want to install zsh.";
+    };
+    zsh.includes = lib.mkOption {
+      type = lib.types.listOf lib.types.package;
+      default = [ ];
+      example = lib.literalExpression ''
+        [ (pkgs.writeTextFile {
+            name = "launch-kanshi.conf";
+            text = "exec kanshi";
+          }) ]'';
+      description = "List of zshrc files to import.";
+    };
+  };
+
+  config = lib.mkIf config.me.zsh.enable {
+    environment.systemPackages = with pkgs; [
+      zsh
+    ];
+
+    users.users.talexander.shell = pkgs.zsh;
+    environment.shells = with pkgs; [ zsh ];
+
+    programs.zsh = {
+      enable = true;
+    };
+
+    me.install.user.talexander.file = {
+      ".zshrc" = {
+        source = "${zshrc}";
+      };
+    };
+
+    environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) {
+      hideMounts = true;
+      users.talexander = {
+        directories = [
+          {
+            directory = ".histdb";
+            user = "talexander";
+            group = "talexander";
+            mode = "0700";
+          }
+        ];
+      };
+    };
+
+    nixpkgs.overlays = [
+      (final: prev: {
+        zsh-histdb = (final.callPackage ./package/zsh-histdb/package.nix { });
+      })
+    ];
+  };
+}

nix/kubernetes/roles/zsh/package/zsh-histdb/package.nix

@@ -0,0 +1,36 @@
+# unpackPhase
+# patchPhase
+# configurePhase
+# buildPhase
+# checkPhase
+# installPhase
+# fixupPhase
+# installCheckPhase
+# distPhase
+{
+  stdenv,
+  pkgs,
+  sqlite,
+  ...
+}:
+stdenv.mkDerivation {
+  name = "zsh-histdb";
+  src = pkgs.fetchgit {
+    url = "https://github.com/larkery/zsh-histdb.git";
+    rev = "90a6c104d0fcc0410d665e148fa7da28c49684eb";
+    sha256 = "sha256-vtG1poaRVbfb/wKPChk1WpPgDq+7udLqLfYfLqap4Vg=";
+  };
+  buildInputs = [ sqlite ];
+  phases = [
+    "installPhase"
+  ];
+  installPhase = ''
+    runHook preInstall
+    mkdir -p $out/share/zsh/plugins/zsh-histdb
+    cp -r $src/histdb-* $src/*.zsh $src/db_migrations $out/share/zsh/plugins/zsh-histdb/
+    runHook postInstall
+  '';
+  postInstall = ''
+    substituteInPlace $out/share/zsh/plugins/zsh-histdb/sqlite-history.zsh $out/share/zsh/plugins/zsh-histdb/histdb-merge $out/share/zsh/plugins/zsh-histdb/histdb-migrate --replace-fail "sqlite3" "${sqlite}/bin/sqlite3"
+  '';
+}