Add kube-scheduler.

nix/kubernetes/configuration.nix

@@ -15,6 +15,7 @@
     ./roles/iso
     ./roles/kube_apiserver
     ./roles/kube_controller_manager
+    ./roles/kube_scheduler
     ./roles/kubernetes
     ./roles/minimal_base
     ./roles/network

nix/kubernetes/keys/package/deploy-script/package.nix

@@ -142,70 +142,13 @@ let
           group = 10024;
           mode = "0600";
         }
-        # {
+        {
-        #   dest_dir = "/vm/${vm_name}/persist/keys/kube";
+          dest_dir = "/vm/${vm_name}/persist/keys/kube";
-        #   file = "${self.kubernetes}/kubernetes.pem";
+          file = "${k8s.client-configs.kube-scheduler}/kube-scheduler.kubeconfig";
-        #   owner = 10024;
+          owner = 10024;
-        #   group = 10024;
+          group = 10024;
-        #   mode = "0640";
+          mode = "0600";
-        # }
+        }
-        # {
-        #   dest_dir = "/vm/${vm_name}/persist/keys/kube";
-        #   file = "${self.kubernetes}/kubernetes-key.pem";
-        #   owner = 10024;
-        #   group = 10024;
-        #   mode = "0640";
-        # }
-        # {
-        #   dest_dir = "/vm/${vm_name}/persist/keys/kube";
-        #   file = "${self.ca}/ca.pem";
-        #   owner = 10024;
-        #   group = 10024;
-        #   mode = "0600";
-        # }
-        # {
-        #   dest_dir = "/vm/${vm_name}/persist/keys/kube";
-        #   file = (writeText "encryption-config.yaml" (lib.generators.toYAML { } kube_encryption_config));
-        #   name = "encryption-config.yaml";
-        #   owner = 10024;
-        #   group = 10024;
-        #   mode = "0600";
-        # }
-        # {
-        #   dest_dir = "/vm/${vm_name}/persist/keys/kube";
-        #   file = "${self.service_account}/service-account.pem";
-        #   owner = 10024;
-        #   group = 10024;
-        #   mode = "0600";
-        # }
-        # {
-        #   dest_dir = "/vm/${vm_name}/persist/keys/kube";
-        #   file = "${self.service_account}/service-account-key.pem";
-        #   owner = 10024;
-        #   group = 10024;
-        #   mode = "0600";
-        # }
-        # {
-        #   dest_dir = "/vm/${vm_name}/persist/keys/kube";
-        #   file = "${self.requestheader-client-ca}/requestheader-client-ca.pem";
-        #   owner = 10024;
-        #   group = 10024;
-        #   mode = "0600";
-        # }
-        # {
-        #   dest_dir = "/vm/${vm_name}/persist/keys/kube";
-        #   file = "${self.controller-proxy}/${vm_name_to_hostname vm_name}-proxy.pem";
-        #   owner = 10024;
-        #   group = 10024;
-        #   mode = "0600";
-        # }
-        # {
-        #   dest_dir = "/vm/${vm_name}/persist/keys/kube";
-        #   file = "${self.controller-proxy}/${vm_name_to_hostname vm_name}-proxy-key.pem";
-        #   owner = 10024;
-        #   group = 10024;
-        #   mode = "0600";
-        # }
       ])
     )
   );

nix/kubernetes/keys/package/k8s-client-config/package.nix

@@ -8,6 +8,7 @@
 # installCheckPhase
 # distPhase
 {
+  lib,
   stdenv,
   k8s,
   kubectl,
@@ -27,7 +28,7 @@ stdenv.mkDerivation (finalAttrs: {
     kubectl config set-cluster kubernetes-the-hard-way \
       --certificate-authority=${k8s.ca}/ca.crt \
       --embed-certs=true \
-      --server=${config_server} \
+      --server=${lib.strings.escapeShellArg config_server} \
       --kubeconfig=${config_name}.kubeconfig
 
     kubectl config set-credentials ${config_user} \

nix/kubernetes/keys/scope.nix

@@ -141,15 +141,20 @@ makeScope newScope (
           };
           kube-controller-manager = {
             config_user = "system:kube-controller-manager";
-            config_server = "https://server.kubernetes.local:6443";
+            # config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443";
+            config_server = "https://127.0.0.1:6443";
+            # config_server = "https://server.kubernetes.local:6443";
           };
           kube-scheduler = {
             config_user = "system:kube-scheduler";
-            config_server = "https://server.kubernetes.local:6443";
+            # config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443";
+            config_server = "https://127.0.0.1:6443";
+            # config_server = "https://server.kubernetes.local:6443";
           };
           admin = {
             config_user = "admin";
-            config_server = "https://127.0.0.1:6443";
+            config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443";
+            # config_server = "https://127.0.0.1:6443";
           };
         }
     );

nix/kubernetes/roles/control_plane/default.nix

@@ -18,8 +18,9 @@
   };
 
   config = lib.mkIf config.me.control_plane.enable {
-    me.kubernetes.enable = true;
     me.kube_apiserver.enable = true;
     me.kube_controller_manager.enable = true;
+    me.kube_scheduler.enable = true;
+    me.kubernetes.enable = true;
   };
 }

nix/kubernetes/roles/etcd/default.nix

@@ -104,6 +104,6 @@
       e2fsprogs # mkfs.ext4
       gptfdisk # cgdisk
     ];
-    networking.firewall.enable = false;
+    networking.firewall.enable = false; # TODO: This is just here for debugging / initial development.
   };
 }

nix/kubernetes/roles/kube_apiserver/default.nix

@@ -131,7 +131,17 @@ in
         );
         Restart = "on-failure";
         RestartSec = 5;
+        LimitNOFILE = 65536;
+        User = "kubernetes";
       };
     };
+
+    networking.firewall.allowedTCPPorts = [
+      6443
+    ];
+
+    systemd.tmpfiles.rules = [
+      "f /var/log/audit.log 0600 kubernetes kubernetes - -"
+    ];
   };
 }

nix/kubernetes/roles/kube_controller_manager/default.nix

@@ -27,33 +27,37 @@ in
       description = "Kubernetes Controller Manager";
       documentation = [ "https://github.com/kubernetes/kubernetes" ];
       wantedBy = [ "kubernetes.target" ];
+      after = [ "kube-apiserver.service" ];
       # path = with pkgs; [
       #   zfs
       # ];
       unitConfig.DefaultDependencies = "no";
       serviceConfig = {
-        Type = "notify";
         ExecStart = (
           shellCommand [
             # NEW:
             "${pkgs.kubernetes}/bin/kube-controller-manager"
             "--bind-address=0.0.0.0"
             # "--cluster-cidr=10.200.0.0/16"
-            "--cluster-cidr=2620:11f:7001:7:ffff:ffff:0ac8:0000/16"
+            # "--cluster-cidr=2620:11f:7001:7:ffff:ffff:0ac8:0000/16"
+            "--cluster-cidr=fd49:0595:2bba::/48"
             "--cluster-name=kubernetes"
             "--cluster-signing-cert-file=/.persist/keys/kube/ca.crt"
             "--cluster-signing-key-file=/.persist/keys/kube/ca.key"
             "--kubeconfig=/.persist/keys/kube/kube-controller-manager.kubeconfig"
             "--root-ca-file=/.persist/keys/kube/ca.crt"
             "--service-account-private-key-file=/.persist/keys/kube/service-accounts.key"
-            "--service-cluster-ip-range=2620:11f:7001:7:ffff:ffff:0ac5:0000/16"
             # "--service-cluster-ip-range=10.197.0.0/16"
+            # "--service-cluster-ip-range=2620:11f:7001:7:ffff:ffff:0ac5:0000/16"
+            "--service-cluster-ip-range=fd00:3e42:e349::/48"
             "--use-service-account-credentials=true"
             "--v=2"
           ]
         );
         Restart = "on-failure";
         RestartSec = 5;
+        LimitNOFILE = 65536;
+        User = "kubernetes";
       };
     };
   };

nix/kubernetes/roles/kube_scheduler/default.nix

@@ -0,0 +1,51 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  # shellCommand = cmd: (lib.concatMapStringsSep " " lib.strings.escapeShellArg cmd);
+  shellCommand = cmd: (builtins.concatStringsSep " " cmd);
+in
+{
+  imports = [ ];
+
+  options.me = {
+    kube_scheduler.enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+      description = "Whether we want to install kube_scheduler.";
+    };
+  };
+
+  config = lib.mkIf config.me.kube_scheduler.enable {
+    systemd.services.kube-scheduler = {
+      enable = true;
+      description = "Kubernetes Scheduler";
+      documentation = [ "https://github.com/kubernetes/kubernetes" ];
+      wantedBy = [ "kubernetes.target" ];
+      after = [ "kube-apiserver.service" ];
+      # path = with pkgs; [
+      #   zfs
+      # ];
+      unitConfig.DefaultDependencies = "no";
+      serviceConfig = {
+        ExecStart = (
+          shellCommand [
+            # NEW:
+            "${pkgs.kubernetes}/bin/kube-scheduler"
+            "--config=${./files/kube-scheduler.yaml}"
+            "--v=2"
+          ]
+        );
+        Restart = "on-failure";
+        RestartSec = 5;
+        LimitNOFILE = 65536;
+        User = "kubernetes";
+      };
+    };
+  };
+}

nix/kubernetes/roles/kube_scheduler/files/kube-scheduler.yaml

@@ -0,0 +1,6 @@
+apiVersion: kubescheduler.config.k8s.io/v1
+kind: KubeSchedulerConfiguration
+clientConnection:
+  kubeconfig: "/.persist/keys/kube/kube-scheduler.kubeconfig"
+leaderElection:
+  leaderElect: true