From 0463d2cbd1c8c982ba3c910ba0e1c9acdb85144c Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Tue, 16 Dec 2025 19:31:33 -0500 Subject: [PATCH] Add kubelet. --- nix/kubernetes/configuration.nix | 1 + .../keys/package/deploy-script/package.nix | 74 ++++++++++++- nix/kubernetes/keys/scope.nix | 27 ++--- .../roles/kube_controller_manager/default.nix | 1 - nix/kubernetes/roles/kubelet/default.nix | 101 ++++++++++++++++++ .../roles/kubelet/files/kubelet-config.yaml | 25 +++++ nix/kubernetes/roles/worker_node/default.nix | 1 + 7 files changed, 214 insertions(+), 16 deletions(-) create mode 100644 nix/kubernetes/roles/kubelet/default.nix create mode 100644 nix/kubernetes/roles/kubelet/files/kubelet-config.yaml diff --git a/nix/kubernetes/configuration.nix b/nix/kubernetes/configuration.nix index 4cde4057..617128d7 100644 --- a/nix/kubernetes/configuration.nix +++ b/nix/kubernetes/configuration.nix @@ -17,6 +17,7 @@ ./roles/kube_apiserver ./roles/kube_controller_manager ./roles/kube_scheduler + ./roles/kubelet ./roles/kubernetes ./roles/minimal_base ./roles/network diff --git a/nix/kubernetes/keys/package/deploy-script/package.nix b/nix/kubernetes/keys/package/deploy-script/package.nix index d3e8da30..78152719 100644 --- a/nix/kubernetes/keys/package/deploy-script/package.nix +++ b/nix/kubernetes/keys/package/deploy-script/package.nix @@ -8,6 +8,7 @@ # installCheckPhase # distPhase { + config, lib, stdenv, writeShellScript, @@ -16,17 +17,35 @@ ... }: let + vm_name_to_hostname = + let + mapping = { + "nc0" = "controller0"; + "nc1" = "controller1"; + "nc2" = "controller2"; + "nw0" = "worker0"; + "nw1" = "worker1"; + "nw2" = "worker2"; + }; + in + (vm_name: mapping."${vm_name}"); + deploy_script_body = ( '' set -euo pipefail IFS=$'\n\t' DIR="$( cd "$( dirname "''${BASH_SOURCE[0]}" )" && pwd )" '' - + (lib.concatMapStringsSep "\n" deploy_machine [ + + (lib.concatMapStringsSep "\n" deploy_control_plane [ "nc0" "nc1" "nc2" ]) + + (lib.concatMapStringsSep "\n" deploy_worker [ + "nw0" + "nw1" + "nw2" + ]) ); deploy_script = (writeShellScript "deploy-script" deploy_script_body); deploy_file = ( @@ -50,14 +69,14 @@ let '' ); - deploy_machine = ( + deploy_control_plane = ( vm_name: ( '' ## ## Create directories on ${vm_name} ## - ${openssh}/bin/ssh mrmanager doas install -d -o 11235 -g 11235 -m 0755 /vm/${vm_name}/persist/keys + ${openssh}/bin/ssh mrmanager doas install -d -o 0 -g 0 -m 0755 /vm/${vm_name}/persist/keys ${openssh}/bin/ssh mrmanager doas install -d -o 10016 -g 10016 -m 0755 /vm/${vm_name}/persist/keys/etcd ${openssh}/bin/ssh mrmanager doas install -d -o 10024 -g 10024 -m 0755 /vm/${vm_name}/persist/keys/kube @@ -152,6 +171,55 @@ let ]) ) ); + deploy_worker = ( + vm_name: + ( + '' + ## + ## Create directories on ${vm_name} + ## + ${openssh}/bin/ssh mrmanager doas install -d -o 0 -g 0 -m 0755 /vm/${vm_name}/persist/keys + ${openssh}/bin/ssh mrmanager doas install -d -o 10024 -g 10024 -m 0755 /vm/${vm_name}/persist/keys/kube + + + '' + + (lib.concatMapStringsSep "\n" deploy_file [ + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.ca}/ca.crt"; + owner = 10024; + group = 10024; + mode = "0640"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.keys."${vm_name_to_hostname vm_name}"}/${vm_name_to_hostname vm_name}.crt"; + name = "kubelet.crt"; + owner = 10024; + group = 10024; + mode = "0640"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.keys."${vm_name_to_hostname vm_name}"}/${vm_name_to_hostname vm_name}.key"; + name = "kubelet.key"; + owner = 10024; + group = 10024; + mode = "0600"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${ + k8s.client-configs."${vm_name_to_hostname vm_name}" + }/${vm_name_to_hostname vm_name}.kubeconfig"; + name = "kubeconfig"; + owner = 10024; + group = 10024; + mode = "0600"; + } + ]) + ) + ); in stdenv.mkDerivation (finalAttrs: { name = "deploy-script"; diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index 1aef3f1c..aeca4011 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -69,12 +69,6 @@ let ]; }; }; - # _vm_name_to_hostname = { - # "nc0" = "controller0"; - # "nc1" = "controller1"; - # "nc2" = "controller2"; - # }; - # vm_name_to_hostname = (vm_name: _vm_name_to_hostname."${vm_name}"); in makeScope newScope ( self: @@ -113,27 +107,36 @@ makeScope newScope ( { controller0 = { config_user = "system:node:controller0"; - config_server = "https://server.kubernetes.local:6443"; + config_server = "https://127.0.0.1:6443"; + # config_server = "https://server.kubernetes.local:6443"; }; controller1 = { config_user = "system:node:controller1"; - config_server = "https://server.kubernetes.local:6443"; + config_server = "https://127.0.0.1:6443"; + # config_server = "https://server.kubernetes.local:6443"; }; controller2 = { config_user = "system:node:controller2"; - config_server = "https://server.kubernetes.local:6443"; + config_server = "https://127.0.0.1:6443"; + # config_server = "https://server.kubernetes.local:6443"; }; worker0 = { config_user = "system:node:worker0"; - config_server = "https://server.kubernetes.local:6443"; + config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443"; + # config_server = "https://127.0.0.1:6443"; + # config_server = "https://server.kubernetes.local:6443"; }; worker1 = { config_user = "system:node:worker1"; - config_server = "https://server.kubernetes.local:6443"; + config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443"; + # config_server = "https://127.0.0.1:6443"; + # config_server = "https://server.kubernetes.local:6443"; }; worker2 = { config_user = "system:node:worker2"; - config_server = "https://server.kubernetes.local:6443"; + config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443"; + # config_server = "https://127.0.0.1:6443"; + # config_server = "https://server.kubernetes.local:6443"; }; kube-proxy = { config_user = "system:kube-proxy"; diff --git a/nix/kubernetes/roles/kube_controller_manager/default.nix b/nix/kubernetes/roles/kube_controller_manager/default.nix index f31de3f8..955858a9 100644 --- a/nix/kubernetes/roles/kube_controller_manager/default.nix +++ b/nix/kubernetes/roles/kube_controller_manager/default.nix @@ -35,7 +35,6 @@ in serviceConfig = { ExecStart = ( shellCommand [ - # NEW: "${pkgs.kubernetes}/bin/kube-controller-manager" "--bind-address=0.0.0.0" # "--cluster-cidr=10.200.0.0/16" diff --git a/nix/kubernetes/roles/kubelet/default.nix b/nix/kubernetes/roles/kubelet/default.nix new file mode 100644 index 00000000..e3dc4752 --- /dev/null +++ b/nix/kubernetes/roles/kubelet/default.nix @@ -0,0 +1,101 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + # shellCommand = cmd: (lib.concatMapStringsSep " " lib.strings.escapeShellArg cmd); + shellCommand = cmd: (builtins.concatStringsSep " " cmd); + settingsFormat = pkgs.formats.yaml { }; + config_file = settingsFormat.generate "kubelet-config.yaml" config.me.kubelet.settings; +in +{ + imports = [ ]; + + options.me = { + kubelet.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install kubelet."; + }; + + kubelet.settings = lib.mkOption { + type = settingsFormat.type; + default = { + kind = "KubeletConfiguration"; + apiVersion = "kubelet.config.k8s.io/v1beta1"; + address = "0.0.0.0"; + authentication = { + anonymous = { + enabled = false; + }; + webhook = { + enabled = true; + }; + x509 = { + clientCAFile = "/var/lib/kubelet/ca.crt"; + }; + }; + authorization = { + mode = "Webhook"; + }; + cgroupDriver = "systemd"; + containerRuntimeEndpoint = "unix:///var/run/containerd/containerd.sock"; + enableServer = true; + failSwapOn = false; + maxPods = 16; + memorySwap = { + swapBehavior = "NoSwap"; + }; + port = 10250; + resolvConf = "/etc/resolv.conf"; + registerNode = true; + runtimeRequestTimeout = "15m"; + tlsCertFile = "/var/lib/kubelet/kubelet.crt"; + tlsPrivateKeyFile = "/var/lib/kubelet/kubelet.key"; + }; + description = '' + kubelet-config.yaml + ''; + }; + }; + + config = lib.mkIf config.me.kubelet.enable { + systemd.services.kubelet = { + enable = true; + description = "Kubernetes Kubelet"; + documentation = [ "https://github.com/kubernetes/kubernetes" ]; + wantedBy = [ "kubernetes.target" ]; + after = [ "containerd.service" ]; + requires = [ "containerd.service" ]; + # path = with pkgs; [ + # zfs + # ]; + unitConfig.DefaultDependencies = "no"; + serviceConfig = { + ExecStart = ( + shellCommand [ + "${pkgs.kubernetes}/bin/kubelet" + # "--config=${config_file}" + "--config=${./files/kubelet-config.yaml}" + "--kubeconfig=/.persist/keys/kube/kubeconfig" + "--v=2" + ] + ); + Restart = "on-failure"; + RestartSec = 5; + # ConfigurationDirectory = "kubernetes"; + # CPUAccounting = "true"; + # IPAccounting = "true"; + # KillMode = "process"; + # MemoryAccounting = "true"; + # StartLimitInterval = 0; + # RuntimeDirectory = "kubelet"; + # StateDirectory = "kubelet"; + }; + }; + }; +} diff --git a/nix/kubernetes/roles/kubelet/files/kubelet-config.yaml b/nix/kubernetes/roles/kubelet/files/kubelet-config.yaml new file mode 100644 index 00000000..2313c535 --- /dev/null +++ b/nix/kubernetes/roles/kubelet/files/kubelet-config.yaml @@ -0,0 +1,25 @@ +kind: KubeletConfiguration +apiVersion: kubelet.config.k8s.io/v1beta1 +address: "0.0.0.0" +authentication: + anonymous: + enabled: false + webhook: + enabled: true + x509: + clientCAFile: "/.persist/keys/kube/ca.crt" +authorization: + mode: Webhook +cgroupDriver: systemd +containerRuntimeEndpoint: "unix:///var/run/containerd/containerd.sock" +enableServer: true +failSwapOn: false +maxPods: 16 +memorySwap: + swapBehavior: NoSwap +port: 10250 +resolvConf: "/etc/resolv.conf" +registerNode: true +runtimeRequestTimeout: "15m" +tlsCertFile: "/.persist/keys/kube/kubelet.crt" +tlsPrivateKeyFile: "/.persist/keys/kube/kubelet.key" diff --git a/nix/kubernetes/roles/worker_node/default.nix b/nix/kubernetes/roles/worker_node/default.nix index 9e7d4bd7..af082757 100644 --- a/nix/kubernetes/roles/worker_node/default.nix +++ b/nix/kubernetes/roles/worker_node/default.nix @@ -19,6 +19,7 @@ config = lib.mkIf config.me.worker_node.enable { me.containerd.enable = true; + me.kubelet.enable = true; me.kubernetes.enable = true; }; }