Add configs for a new kubernetes cluster on NixOS.

2025-11-30 14:32:36 -05:00
parent d35cfaacbd
commit b16db4325f
58 changed files with 3221 additions and 0 deletions
--- a/nix/kubernetes/roles/boot/default.nix
+++ b/nix/kubernetes/roles/boot/default.nix
@@ -0,0 +1,130 @@
+# ISO does not work with systemd initrd yet https://github.com/NixOS/nixpkgs/pull/291750
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  imports = [ ];
+
+  options.me = {
+    boot.enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+      description = "Whether we want to install boot.";
+    };
+
+    boot.secure = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+      description = "Enable to use secure boot.";
+    };
+
+    rollback.enable = lib.mkOption {
+      type = lib.types.bool;
+      default = true;
+      example = true;
+      description = "Whether we want to enable rolling back during boot.";
+    };
+
+    rollback.dataset = lib.mkOption {
+      default = { };
+      example = lib.literalExpression ''
+        {
+          "zroot/linux/nix/root@blank" = true;
+          "zroot/linux/nix/home@blank" = lib.mkForce false;
+        }
+      '';
+      type = lib.types.coercedTo (lib.types.listOf lib.types.str) (
+        enabled: lib.listToAttrs (map (fs: lib.nameValuePair fs true) enabled)
+      ) (lib.types.attrsOf lib.types.bool);
+      description = "List of ZFS datasets to rollback to during boot.";
+    };
+  };
+
+  config = lib.mkIf config.me.boot.enable (
+    lib.mkMerge [
+      {
+        environment.systemPackages = with pkgs; [
+          tpm2-tools # For tpm2_eventlog to check for OptionRoms
+          # cp /sys/kernel/security/tpm0/binary_bios_measurements eventlog
+          # tpm2_eventlog eventlog | grep "BOOT_SERVICES_DRIVER"
+          sbctl # For debugging and troubleshooting Secure Boot.
+          efibootmgr # To set EFI boot order.
+        ];
+      }
+      (lib.mkIf (!config.me.buildingPortable) {
+
+        boot.loader.grub.enable = false;
+        # Use the systemd-boot EFI boot loader.
+        boot.loader.systemd-boot.enable = true;
+        # TODO: make not write bootx64.efi
+        boot.loader.efi.canTouchEfiVariables = false;
+
+        # Automatically delete old generations
+        boot.loader.systemd-boot.configurationLimit = 3;
+
+        boot.loader.systemd-boot.memtest86.enable = true;
+
+        # Check what will be lost with `zfs diff zroot/linux/root@blank`
+        boot.initrd.systemd.enable = lib.mkDefault true;
+        boot.initrd.systemd.services.zfs-rollback = lib.mkIf config.me.rollback.enable {
+          description = "Rollback ZFS root dataset to blank snapshot";
+          wantedBy = [
+            "initrd.target"
+          ];
+          after = [
+            "zfs-import-zroot.service"
+          ];
+          before = [
+            "sysroot.mount"
+          ];
+          unitConfig.DefaultDependencies = "no";
+          serviceConfig.Type = "oneshot";
+          script = lib.concatStringsSep "\n" (
+            (builtins.map (ds: "${config.boot.zfs.package}/sbin/zfs rollback -r '${ds}'") (
+              builtins.attrNames config.me.rollback.dataset
+            ))
+            ++ [ ''echo "rollback complete"'' ]
+          );
+        };
+
+        # boot.loader.systemd-boot.extraEntries = {
+        #   "windows.conf" = ''
+        #     title Windows
+        #     efi /EFI/Microsoft/Boot/bootmgfw.efi
+        #     options root=PARTUUID=17e325bf-a378-4d1d-be6a-f6df5476f0fa
+        #   '';
+        # };
+        environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) {
+          hideMounts = true;
+          directories = [
+            "/var/lib/sbctl" # Secure Boot Keys
+          ];
+        };
+      })
+      (lib.mkIf (config.me.boot.secure) {
+        environment.systemPackages = with pkgs; [
+          sbctl
+        ];
+        boot.loader.systemd-boot.enable = lib.mkForce false;
+        boot.lanzaboote = {
+          enable = true;
+          pkiBundle = "/var/lib/sbctl";
+        };
+      })
+    ]
+  );
+}
+# efibootmgr -c -d /dev/sda -p 1 -L NixOS-boot -l '\EFI\NixOS-boot\grubx64.efi'
+
+# Text-only:
+# sudo cp "$(nix-build '<nixpkgs>' --no-out-link -A 'refind')/share/refind/refind_x64.efi" /boot/EFI/boot/bootx64.efi
+
+# Full graphics:
+# $ sudo nix-shell -p refind efibootmgr
+# $ refind-install