Add kubelet.

nix/kubernetes/configuration.nix

@@ -17,6 +17,7 @@
     ./roles/kube_apiserver
     ./roles/kube_controller_manager
     ./roles/kube_scheduler
+    ./roles/kubelet
     ./roles/kubernetes
     ./roles/minimal_base
     ./roles/network

nix/kubernetes/keys/package/deploy-script/package.nix

@@ -8,6 +8,7 @@
 # installCheckPhase
 # distPhase
 {
+  config,
   lib,
   stdenv,
   writeShellScript,
@@ -16,17 +17,35 @@
   ...
 }:
 let
+  vm_name_to_hostname =
+    let
+      mapping = {
+        "nc0" = "controller0";
+        "nc1" = "controller1";
+        "nc2" = "controller2";
+        "nw0" = "worker0";
+        "nw1" = "worker1";
+        "nw2" = "worker2";
+      };
+    in
+    (vm_name: mapping."${vm_name}");
+
   deploy_script_body = (
     ''
       set -euo pipefail
       IFS=$'\n\t'
       DIR="$( cd "$( dirname "''${BASH_SOURCE[0]}" )" && pwd )"
     ''
-    + (lib.concatMapStringsSep "\n" deploy_machine [
+    + (lib.concatMapStringsSep "\n" deploy_control_plane [
       "nc0"
       "nc1"
       "nc2"
     ])
+    + (lib.concatMapStringsSep "\n" deploy_worker [
+      "nw0"
+      "nw1"
+      "nw2"
+    ])
   );
   deploy_script = (writeShellScript "deploy-script" deploy_script_body);
   deploy_file = (
@@ -50,14 +69,14 @@ let
 
     ''
   );
-  deploy_machine = (
+  deploy_control_plane = (
     vm_name:
     (
       ''
         ##
         ## Create directories on ${vm_name}
         ##
-        ${openssh}/bin/ssh mrmanager doas install -d -o 11235 -g 11235 -m 0755 /vm/${vm_name}/persist/keys
+        ${openssh}/bin/ssh mrmanager doas install -d -o 0 -g 0 -m 0755 /vm/${vm_name}/persist/keys
         ${openssh}/bin/ssh mrmanager doas install -d -o 10016 -g 10016 -m 0755 /vm/${vm_name}/persist/keys/etcd
         ${openssh}/bin/ssh mrmanager doas install -d -o 10024 -g 10024 -m 0755 /vm/${vm_name}/persist/keys/kube
 
@@ -152,6 +171,55 @@ let
       ])
     )
   );
+  deploy_worker = (
+    vm_name:
+    (
+      ''
+        ##
+        ## Create directories on ${vm_name}
+        ##
+        ${openssh}/bin/ssh mrmanager doas install -d -o 0 -g 0 -m 0755 /vm/${vm_name}/persist/keys
+        ${openssh}/bin/ssh mrmanager doas install -d -o 10024 -g 10024 -m 0755 /vm/${vm_name}/persist/keys/kube
+
+
+      ''
+      + (lib.concatMapStringsSep "\n" deploy_file [
+        {
+          dest_dir = "/vm/${vm_name}/persist/keys/kube";
+          file = "${k8s.ca}/ca.crt";
+          owner = 10024;
+          group = 10024;
+          mode = "0640";
+        }
+        {
+          dest_dir = "/vm/${vm_name}/persist/keys/kube";
+          file = "${k8s.keys."${vm_name_to_hostname vm_name}"}/${vm_name_to_hostname vm_name}.crt";
+          name = "kubelet.crt";
+          owner = 10024;
+          group = 10024;
+          mode = "0640";
+        }
+        {
+          dest_dir = "/vm/${vm_name}/persist/keys/kube";
+          file = "${k8s.keys."${vm_name_to_hostname vm_name}"}/${vm_name_to_hostname vm_name}.key";
+          name = "kubelet.key";
+          owner = 10024;
+          group = 10024;
+          mode = "0600";
+        }
+        {
+          dest_dir = "/vm/${vm_name}/persist/keys/kube";
+          file = "${
+            k8s.client-configs."${vm_name_to_hostname vm_name}"
+          }/${vm_name_to_hostname vm_name}.kubeconfig";
+          name = "kubeconfig";
+          owner = 10024;
+          group = 10024;
+          mode = "0600";
+        }
+      ])
+    )
+  );
 in
 stdenv.mkDerivation (finalAttrs: {
   name = "deploy-script";

nix/kubernetes/keys/scope.nix

@@ -69,12 +69,6 @@ let
       ];
     };
   };
-  # _vm_name_to_hostname = {
-  #   "nc0" = "controller0";
-  #   "nc1" = "controller1";
-  #   "nc2" = "controller2";
-  # };
-  # vm_name_to_hostname = (vm_name: _vm_name_to_hostname."${vm_name}");
 in
 makeScope newScope (
   self:
@@ -113,27 +107,36 @@ makeScope newScope (
         {
           controller0 = {
             config_user = "system:node:controller0";
-            config_server = "https://server.kubernetes.local:6443";
+            config_server = "https://127.0.0.1:6443";
+            # config_server = "https://server.kubernetes.local:6443";
           };
           controller1 = {
             config_user = "system:node:controller1";
-            config_server = "https://server.kubernetes.local:6443";
+            config_server = "https://127.0.0.1:6443";
+            # config_server = "https://server.kubernetes.local:6443";
           };
           controller2 = {
             config_user = "system:node:controller2";
-            config_server = "https://server.kubernetes.local:6443";
+            config_server = "https://127.0.0.1:6443";
+            # config_server = "https://server.kubernetes.local:6443";
           };
           worker0 = {
             config_user = "system:node:worker0";
-            config_server = "https://server.kubernetes.local:6443";
+            config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443";
+            # config_server = "https://127.0.0.1:6443";
+            # config_server = "https://server.kubernetes.local:6443";
           };
           worker1 = {
             config_user = "system:node:worker1";
-            config_server = "https://server.kubernetes.local:6443";
+            config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443";
+            # config_server = "https://127.0.0.1:6443";
+            # config_server = "https://server.kubernetes.local:6443";
           };
           worker2 = {
             config_user = "system:node:worker2";
-            config_server = "https://server.kubernetes.local:6443";
+            config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443";
+            # config_server = "https://127.0.0.1:6443";
+            # config_server = "https://server.kubernetes.local:6443";
           };
           kube-proxy = {
             config_user = "system:kube-proxy";

nix/kubernetes/roles/kube_controller_manager/default.nix

@@ -35,7 +35,6 @@ in
       serviceConfig = {
         ExecStart = (
           shellCommand [
-            # NEW:
             "${pkgs.kubernetes}/bin/kube-controller-manager"
             "--bind-address=0.0.0.0"
             # "--cluster-cidr=10.200.0.0/16"

nix/kubernetes/roles/kubelet/default.nix

@@ -0,0 +1,101 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  # shellCommand = cmd: (lib.concatMapStringsSep " " lib.strings.escapeShellArg cmd);
+  shellCommand = cmd: (builtins.concatStringsSep " " cmd);
+  settingsFormat = pkgs.formats.yaml { };
+  config_file = settingsFormat.generate "kubelet-config.yaml" config.me.kubelet.settings;
+in
+{
+  imports = [ ];
+
+  options.me = {
+    kubelet.enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+      description = "Whether we want to install kubelet.";
+    };
+
+    kubelet.settings = lib.mkOption {
+      type = settingsFormat.type;
+      default = {
+        kind = "KubeletConfiguration";
+        apiVersion = "kubelet.config.k8s.io/v1beta1";
+        address = "0.0.0.0";
+        authentication = {
+          anonymous = {
+            enabled = false;
+          };
+          webhook = {
+            enabled = true;
+          };
+          x509 = {
+            clientCAFile = "/var/lib/kubelet/ca.crt";
+          };
+        };
+        authorization = {
+          mode = "Webhook";
+        };
+        cgroupDriver = "systemd";
+        containerRuntimeEndpoint = "unix:///var/run/containerd/containerd.sock";
+        enableServer = true;
+        failSwapOn = false;
+        maxPods = 16;
+        memorySwap = {
+          swapBehavior = "NoSwap";
+        };
+        port = 10250;
+        resolvConf = "/etc/resolv.conf";
+        registerNode = true;
+        runtimeRequestTimeout = "15m";
+        tlsCertFile = "/var/lib/kubelet/kubelet.crt";
+        tlsPrivateKeyFile = "/var/lib/kubelet/kubelet.key";
+      };
+      description = ''
+        kubelet-config.yaml
+      '';
+    };
+  };
+
+  config = lib.mkIf config.me.kubelet.enable {
+    systemd.services.kubelet = {
+      enable = true;
+      description = "Kubernetes Kubelet";
+      documentation = [ "https://github.com/kubernetes/kubernetes" ];
+      wantedBy = [ "kubernetes.target" ];
+      after = [ "containerd.service" ];
+      requires = [ "containerd.service" ];
+      # path = with pkgs; [
+      #   zfs
+      # ];
+      unitConfig.DefaultDependencies = "no";
+      serviceConfig = {
+        ExecStart = (
+          shellCommand [
+            "${pkgs.kubernetes}/bin/kubelet"
+            # "--config=${config_file}"
+            "--config=${./files/kubelet-config.yaml}"
+            "--kubeconfig=/.persist/keys/kube/kubeconfig"
+            "--v=2"
+          ]
+        );
+        Restart = "on-failure";
+        RestartSec = 5;
+        # ConfigurationDirectory = "kubernetes";
+        # CPUAccounting = "true";
+        # IPAccounting = "true";
+        # KillMode = "process";
+        # MemoryAccounting = "true";
+        # StartLimitInterval = 0;
+        # RuntimeDirectory = "kubelet";
+        # StateDirectory = "kubelet";
+      };
+    };
+  };
+}

nix/kubernetes/roles/kubelet/files/kubelet-config.yaml

@@ -0,0 +1,25 @@
+kind: KubeletConfiguration
+apiVersion: kubelet.config.k8s.io/v1beta1
+address: "0.0.0.0"
+authentication:
+  anonymous:
+    enabled: false
+  webhook:
+    enabled: true
+  x509:
+    clientCAFile: "/.persist/keys/kube/ca.crt"
+authorization:
+  mode: Webhook
+cgroupDriver: systemd
+containerRuntimeEndpoint: "unix:///var/run/containerd/containerd.sock"
+enableServer: true
+failSwapOn: false
+maxPods: 16
+memorySwap:
+  swapBehavior: NoSwap
+port: 10250
+resolvConf: "/etc/resolv.conf"
+registerNode: true
+runtimeRequestTimeout: "15m"
+tlsCertFile: "/.persist/keys/kube/kubelet.crt"
+tlsPrivateKeyFile: "/.persist/keys/kube/kubelet.key"

nix/kubernetes/roles/worker_node/default.nix

@@ -19,6 +19,7 @@
 
   config = lib.mkIf config.me.worker_node.enable {
     me.containerd.enable = true;
+    me.kubelet.enable = true;
     me.kubernetes.enable = true;
   };
 }