Install kubernetes.

nix/kubernetes/configuration.nix

@@ -7,15 +7,18 @@
 {
   imports = [
     ./roles/boot
+    ./roles/control_plane
     ./roles/doas
+    ./roles/dont_use_substituters
     ./roles/etcd
     ./roles/image_based_appliance
     ./roles/iso
-    ./roles/optimized_build
+    ./roles/kube_apiserver
-    ./roles/dont_use_substituters
+    ./roles/kubernetes
     ./roles/minimal_base
     ./roles/network
     ./roles/nvme
+    ./roles/optimized_build
     ./roles/ssh
     ./roles/sshd
     ./roles/user

nix/kubernetes/flake.nix

@@ -95,10 +95,9 @@
                 ...
               }:
               let
-                repl_path = toString ./.;
                 nix-self-repl = pkgs.writeShellScriptBin "nix-self-repl" ''
                   source /etc/set-environment
-                  nix repl "${repl_path}/repl.nix" "$@"
+                  nix repl --expr 'builtins.getFlake "${self}"'
                 '';
                 # If we wanted the current version of a flake then we'd just launch
                 #   nix repl
@@ -130,10 +129,9 @@
                 ...
               }:
               let
-                repl_path = toString ./.;
                 nix-self-repl = pkgs.writeShellScriptBin "nix-self-repl" ''
                   source /etc/set-environment
-                  nix repl "${repl_path}/repl.nix" "$@"
+                  nix repl --expr 'builtins.getFlake "${self}"'
                 '';
                 # If we wanted the current version of a flake then we'd just launch
                 #   nix repl

nix/kubernetes/hosts/controller0/default.nix

@@ -116,6 +116,14 @@
       "controller2=https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2380" # 10.215.1.223
     ];
 
+    me.kube_apiserver.internal_ip = "2620:11f:7001:7:ffff:ffff:0ad7:01dd";
+    me.kube_apiserver.etcd_services = [
+      "https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2379" # 10.215.1.221
+      "https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2379" # 10.215.1.222
+      "https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2379" # 10.215.1.223
+    ];
+
+    me.control_plane.enable = true;
     me.dont_use_substituters.enable = true;
     me.etcd.enable = true;
     me.minimal_base.enable = true;

nix/kubernetes/hosts/controller1/default.nix

@@ -116,6 +116,14 @@
       "controller2=https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2380" # 10.215.1.223
     ];
 
+    me.kube_apiserver.internal_ip = "2620:11f:7001:7:ffff:ffff:0ad7:01de";
+    me.kube_apiserver.etcd_services = [
+      "https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2379" # 10.215.1.221
+      "https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2379" # 10.215.1.222
+      "https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2379" # 10.215.1.223
+    ];
+
+    me.control_plane.enable = true;
     me.dont_use_substituters.enable = true;
     me.etcd.enable = true;
     me.minimal_base.enable = true;

nix/kubernetes/hosts/controller2/default.nix

@@ -116,6 +116,14 @@
       "controller2=https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2380" # 10.215.1.223
     ];
 
+    me.kube_apiserver.internal_ip = "2620:11f:7001:7:ffff:ffff:0ad7:01df";
+    me.kube_apiserver.etcd_services = [
+      "https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2379" # 10.215.1.221
+      "https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2379" # 10.215.1.222
+      "https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2379" # 10.215.1.223
+    ];
+
+    me.control_plane.enable = true;
     me.dont_use_substituters.enable = true;
     me.etcd.enable = true;
     me.minimal_base.enable = true;

nix/kubernetes/keys/scope.nix

@@ -4,6 +4,8 @@
   callPackage,
   writeShellScript,
   openssh,
+  runCommand,
+  writeText,
   lib,
 }:
 let
@@ -50,25 +52,91 @@ makeScope newScope (
       inherit all_hostnames;
       k8s = self;
     };
-    deploy_key = (
+    deploy_file = (
-      vm_name: file: ''
+      {
-        ${openssh}/bin/ssh mrmanager rm -f /vm/${vm_name}/persist/keys/etcd/${builtins.baseNameOf file} ~/${builtins.baseNameOf file}
+        dest_dir,
-        ${openssh}/bin/scp ${file} mrmanager:~/${builtins.baseNameOf file}
+        file,
-        ${openssh}/bin/ssh mrmanager doas install -o 10016 -g 10016 -m 0640 ~/${builtins.baseNameOf file} /vm/${vm_name}/persist/keys/etcd/${builtins.baseNameOf file}
+        name ? (builtins.baseNameOf file),
-        ${openssh}/bin/ssh mrmanager rm -f ~/${builtins.baseNameOf file}
+        owner,
+        group,
+        mode,
+      }:
+      ''
+        ##
+        ## deploy ${name} to ${dest_dir}
+        ##
+        ${openssh}/bin/ssh mrmanager doas rm -f ${dest_dir}/${name} ~/${name}
+        ${openssh}/bin/scp ${file} mrmanager:~/${name}
+        ${openssh}/bin/ssh mrmanager doas install -o ${toString owner} -g ${toString group} -m ${mode} ~/${name} ${dest_dir}/${name}
+        ${openssh}/bin/ssh mrmanager doas rm -f ~/${name}
+
+
       ''
     );
     deploy_machine = (
       vm_name:
       (
         ''
+          ##
+          ## Create directories on ${vm_name}
+          ##
           ${openssh}/bin/ssh mrmanager doas install -d -o 11235 -g 11235 -m 0755 /vm/${vm_name}/persist/keys
           ${openssh}/bin/ssh mrmanager doas install -d -o 10016 -g 10016 -m 0755 /vm/${vm_name}/persist/keys/etcd
+          ${openssh}/bin/ssh mrmanager doas install -d -o 10024 -g 10024 -m 0755 /vm/${vm_name}/persist/keys/kube
+
+
         ''
-        + (lib.concatMapStringsSep "\n" (deploy_key vm_name) [
+        + (lib.concatMapStringsSep "\n" deploy_file [
-          "${self.kubernetes}/kubernetes.pem"
+          {
-          "${self.kubernetes}/kubernetes-key.pem"
+            dest_dir = "/vm/${vm_name}/persist/keys/etcd";
-          "${self.ca}/ca.pem"
+            file = "${self.kubernetes}/kubernetes.pem";
+            owner = 10016;
+            group = 10016;
+            mode = "0640";
+          }
+          {
+            dest_dir = "/vm/${vm_name}/persist/keys/etcd";
+            file = "${self.kubernetes}/kubernetes-key.pem";
+            owner = 10016;
+            group = 10016;
+            mode = "0600";
+          }
+          {
+            dest_dir = "/vm/${vm_name}/persist/keys/etcd";
+            file = "${self.ca}/ca.pem";
+            owner = 10016;
+            group = 10016;
+            mode = "0640";
+          }
+          {
+            dest_dir = "/vm/${vm_name}/persist/keys/kube";
+            file = "${self.kubernetes}/kubernetes.pem";
+            owner = 10024;
+            group = 10024;
+            mode = "0640";
+          }
+          {
+            dest_dir = "/vm/${vm_name}/persist/keys/kube";
+            file = "${self.kubernetes}/kubernetes-key.pem";
+            owner = 10024;
+            group = 10024;
+            mode = "0640";
+          }
+          {
+            dest_dir = "/vm/${vm_name}/persist/keys/kube";
+            file = "${self.ca}/ca.pem";
+            owner = 10024;
+            group = 10024;
+            mode = "0600";
+          }
+          {
+            dest_dir = "/vm/${vm_name}/persist/keys/kube";
+            file = (writeText "encryption-config.yaml" (lib.generators.toYAML { } kube_encryption_config));
+            name = "encryption-config.yaml";
+            owner = 10024;
+            group = 10024;
+            mode = "0600";
+          }
         ])
       )
     );
@@ -84,6 +152,31 @@ makeScope newScope (
         "nc2"
       ])
     );
+    kube_encryption_key = runCommand "kube_encryption_key" { } ''
+      head -c 32 /dev/urandom | base64 | tee $out
+    '';
+    kube_encryption_config = {
+      kind = "EncryptionConfig";
+      apiVersion = "v1";
+      resources = [
+        {
+          resources = [ "secrets" ];
+          providers = [
+            {
+              aescbc = {
+                keys = [
+                  {
+                    name = "key1";
+                    secret = (builtins.readFile "${kube_encryption_key}");
+                  }
+                ];
+              };
+            }
+            { identity = { }; }
+          ];
+        }
+      ];
+    };
   in
   {
     ca = (callPackage ./package/k8s-ca/package.nix additional_vars);

nix/kubernetes/repl.nix

@@ -1,5 +0,0 @@
-let
-  flake = builtins.getFlake (toString ./.);
-  nixpkgs = import <nixpkgs> { };
-in
-{ inherit flake; } // flake // builtins // nixpkgs // nixpkgs.lib // flake.nixosConfigurations

nix/kubernetes/roles/control_plane/default.nix

@@ -0,0 +1,24 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  imports = [ ];
+
+  options.me = {
+    control_plane.enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+      description = "Whether we want to install control_plane.";
+    };
+  };
+
+  config = lib.mkIf config.me.control_plane.enable {
+    me.kubernetes.enable = true;
+    me.kube_apiserver.enable = true;
+  };
+}

nix/kubernetes/roles/kube_apiserver/default.nix

@@ -0,0 +1,98 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  shellCommand = cmd: (lib.concatMapStringsSep " " lib.strings.escapeShellArg cmd);
+in
+{
+  imports = [ ];
+
+  options.me = {
+    kube_apiserver.enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+      description = "Whether we want to install kube_apiserver.";
+    };
+
+    kube_apiserver.internal_ip = lib.mkOption {
+      # default = { };
+      example = "192.168.1.10";
+      type = lib.types.str;
+      description = "IP address this server should advertise.";
+    };
+
+    kube_apiserver.etcd_services = lib.mkOption {
+      default = [ ];
+      example = [ "https://192.168.1.10:2379" ];
+      type = lib.types.listOf lib.types.str;
+      description = "Endpoints for etcd servers.";
+    };
+  };
+
+  config = lib.mkIf config.me.kube_apiserver.enable {
+    systemd.services.kube-apiserver = {
+      enable = true;
+      description = "Kubernetes API Server";
+      documentation = [ "https://github.com/kubernetes/kubernetes" ];
+      wantedBy = [ "kubernetes.target" ];
+      # path = with pkgs; [
+      #   zfs
+      # ];
+      unitConfig.DefaultDependencies = "no";
+      serviceConfig = {
+        Type = "notify";
+        ExecStart = (
+          shellCommand [
+            "${pkgs.kubernetes}/bin/kube-apiserver"
+            "--advertise-address=${config.me.kube_apiserver.internal_ip}"
+            "--allow-privileged=true"
+            "--apiserver-count=3"
+            "--audit-log-maxage=30"
+            "--audit-log-maxbackup=3"
+            "--audit-log-maxsize=100"
+            "--audit-log-path=/var/log/audit.log"
+            "--authorization-mode=Node,RBAC"
+            "--bind-address=0.0.0.0"
+            "--client-ca-file=/.persist/keys/kube/ca.pem"
+            "--requestheader-client-ca-file=/var/lib/kubernetes/requestheader-client-ca.pem"
+            ''--requestheader-allowed-names=""''
+            "--requestheader-extra-headers-prefix=X-Remote-Extra-"
+            "--requestheader-group-headers=X-Remote-Group"
+            "--requestheader-username-headers=X-Remote-User"
+            "--proxy-client-cert-file=/var/lib/kubernetes/{{ node_name }}-proxy.pem"
+            "--proxy-client-key-file=/var/lib/kubernetes/{{ node_name }}-proxy-key.pem"
+            "--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota"
+            "--etcd-cafile=/.persist/keys/kube/ca.pem"
+            "--etcd-certfile=/.persist/keys/kube/kubernetes.pem"
+            "--etcd-keyfile=/.persist/keys/kube/kubernetes-key.pem"
+            "--etcd-servers=${builtins.concatStringsSep "," config.me.kube_apiserver.etcd_services}"
+            "--event-ttl=1h"
+            "--encryption-provider-config=/.persist/keys/kube/encryption-config.yaml"
+            "--kubelet-certificate-authority=/.persist/keys/kube/ca.pem"
+            "--kubelet-client-certificate=/.persist/keys/kube/kubernetes.pem"
+            "--kubelet-client-key=/.persist/keys/kube/kubernetes-key.pem"
+            "--runtime-config='api/all=true'"
+            "--service-account-key-file=/var/lib/kubernetes/service-account.pem"
+            "--service-account-signing-key-file=/var/lib/kubernetes/service-account-key.pem"
+            "--service-account-issuer=https://{{ kubernetes_public_address }}:6443"
+            "--service-node-port-range=30000-32767"
+            "--tls-cert-file=/.persist/keys/kube/kubernetes.pem"
+            "--tls-private-key-file=/.persist/keys/kube/kubernetes-key.pem"
+            "--tls-min-version=VersionTLS13"
+            "--kubelet-preferred-address-types=InternalIP,ExternalDNS,ExternalIP,Hostname,InternalDNS"
+            "--service-cluster-ip-range=10.197.0.0/16"
+            "--enable-aggregator-routing=true"
+            "--v=2"
+          ]
+        );
+        Restart = "on-failure";
+        RestartSec = 5;
+      };
+    };
+  };
+}

nix/kubernetes/roles/kubernetes/default.nix

@@ -0,0 +1,53 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  imports = [ ];
+
+  options.me = {
+    kubernetes.enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+      description = "Whether we want to install kubernetes.";
+    };
+  };
+
+  config = lib.mkIf config.me.kubernetes.enable {
+    environment.systemPackages = with pkgs; [
+      kubernetes
+    ];
+
+    systemd.targets.kubernetes = {
+      description = "Kubernetes";
+      wantedBy = [ "multi-user.target" ];
+    };
+
+    users.users.kubernetes = {
+      uid = 10024;
+      description = "Kubernetes";
+      group = "kubernetes";
+      home = "/var/lib/kubernetes";
+      createHome = true;
+      homeMode = "755";
+      isSystemUser = true;
+    };
+    users.groups.kubernetes.gid = 10024;
+
+    environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) {
+      hideMounts = true;
+      directories = [
+        {
+          directory = "/var/lib/kubernetes";
+          user = "kubernetes";
+          group = "kubernetes";
+          mode = "0755";
+        }
+      ];
+    };
+  };
+}