From 3b96f8d26c3ca69cb2f6fd9026fa2f2475165b47 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 15 Dec 2025 20:09:46 -0500 Subject: [PATCH] Add kube-scheduler. --- nix/kubernetes/configuration.nix | 1 + .../keys/package/deploy-script/package.nix | 71 ++----------------- .../package/k8s-client-config/package.nix | 3 +- nix/kubernetes/keys/scope.nix | 11 ++- .../roles/control_plane/default.nix | 3 +- nix/kubernetes/roles/etcd/default.nix | 2 +- .../roles/kube_apiserver/default.nix | 10 +++ .../roles/kube_controller_manager/default.nix | 10 ++- .../roles/kube_scheduler/default.nix | 51 +++++++++++++ .../kube_scheduler/files/kube-scheduler.yaml | 6 ++ 10 files changed, 95 insertions(+), 73 deletions(-) create mode 100644 nix/kubernetes/roles/kube_scheduler/default.nix create mode 100644 nix/kubernetes/roles/kube_scheduler/files/kube-scheduler.yaml diff --git a/nix/kubernetes/configuration.nix b/nix/kubernetes/configuration.nix index 598ded20..11b95381 100644 --- a/nix/kubernetes/configuration.nix +++ b/nix/kubernetes/configuration.nix @@ -15,6 +15,7 @@ ./roles/iso ./roles/kube_apiserver ./roles/kube_controller_manager + ./roles/kube_scheduler ./roles/kubernetes ./roles/minimal_base ./roles/network diff --git a/nix/kubernetes/keys/package/deploy-script/package.nix b/nix/kubernetes/keys/package/deploy-script/package.nix index d639c01e..d3e8da30 100644 --- a/nix/kubernetes/keys/package/deploy-script/package.nix +++ b/nix/kubernetes/keys/package/deploy-script/package.nix @@ -142,70 +142,13 @@ let group = 10024; mode = "0600"; } - # { - # dest_dir = "/vm/${vm_name}/persist/keys/kube"; - # file = "${self.kubernetes}/kubernetes.pem"; - # owner = 10024; - # group = 10024; - # mode = "0640"; - # } - # { - # dest_dir = "/vm/${vm_name}/persist/keys/kube"; - # file = "${self.kubernetes}/kubernetes-key.pem"; - # owner = 10024; - # group = 10024; - # mode = "0640"; - # } - # { - # dest_dir = "/vm/${vm_name}/persist/keys/kube"; - # file = "${self.ca}/ca.pem"; - # owner = 10024; - # group = 10024; - # mode = "0600"; - # } - # { - # dest_dir = "/vm/${vm_name}/persist/keys/kube"; - # file = (writeText "encryption-config.yaml" (lib.generators.toYAML { } kube_encryption_config)); - # name = "encryption-config.yaml"; - # owner = 10024; - # group = 10024; - # mode = "0600"; - # } - # { - # dest_dir = "/vm/${vm_name}/persist/keys/kube"; - # file = "${self.service_account}/service-account.pem"; - # owner = 10024; - # group = 10024; - # mode = "0600"; - # } - # { - # dest_dir = "/vm/${vm_name}/persist/keys/kube"; - # file = "${self.service_account}/service-account-key.pem"; - # owner = 10024; - # group = 10024; - # mode = "0600"; - # } - # { - # dest_dir = "/vm/${vm_name}/persist/keys/kube"; - # file = "${self.requestheader-client-ca}/requestheader-client-ca.pem"; - # owner = 10024; - # group = 10024; - # mode = "0600"; - # } - # { - # dest_dir = "/vm/${vm_name}/persist/keys/kube"; - # file = "${self.controller-proxy}/${vm_name_to_hostname vm_name}-proxy.pem"; - # owner = 10024; - # group = 10024; - # mode = "0600"; - # } - # { - # dest_dir = "/vm/${vm_name}/persist/keys/kube"; - # file = "${self.controller-proxy}/${vm_name_to_hostname vm_name}-proxy-key.pem"; - # owner = 10024; - # group = 10024; - # mode = "0600"; - # } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.client-configs.kube-scheduler}/kube-scheduler.kubeconfig"; + owner = 10024; + group = 10024; + mode = "0600"; + } ]) ) ); diff --git a/nix/kubernetes/keys/package/k8s-client-config/package.nix b/nix/kubernetes/keys/package/k8s-client-config/package.nix index d81f44f4..2d4ac886 100644 --- a/nix/kubernetes/keys/package/k8s-client-config/package.nix +++ b/nix/kubernetes/keys/package/k8s-client-config/package.nix @@ -8,6 +8,7 @@ # installCheckPhase # distPhase { + lib, stdenv, k8s, kubectl, @@ -27,7 +28,7 @@ stdenv.mkDerivation (finalAttrs: { kubectl config set-cluster kubernetes-the-hard-way \ --certificate-authority=${k8s.ca}/ca.crt \ --embed-certs=true \ - --server=${config_server} \ + --server=${lib.strings.escapeShellArg config_server} \ --kubeconfig=${config_name}.kubeconfig kubectl config set-credentials ${config_user} \ diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index 785653f4..1aef3f1c 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -141,15 +141,20 @@ makeScope newScope ( }; kube-controller-manager = { config_user = "system:kube-controller-manager"; - config_server = "https://server.kubernetes.local:6443"; + # config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443"; + config_server = "https://127.0.0.1:6443"; + # config_server = "https://server.kubernetes.local:6443"; }; kube-scheduler = { config_user = "system:kube-scheduler"; - config_server = "https://server.kubernetes.local:6443"; + # config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443"; + config_server = "https://127.0.0.1:6443"; + # config_server = "https://server.kubernetes.local:6443"; }; admin = { config_user = "admin"; - config_server = "https://127.0.0.1:6443"; + config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443"; + # config_server = "https://127.0.0.1:6443"; }; } ); diff --git a/nix/kubernetes/roles/control_plane/default.nix b/nix/kubernetes/roles/control_plane/default.nix index b460144d..70399829 100644 --- a/nix/kubernetes/roles/control_plane/default.nix +++ b/nix/kubernetes/roles/control_plane/default.nix @@ -18,8 +18,9 @@ }; config = lib.mkIf config.me.control_plane.enable { - me.kubernetes.enable = true; me.kube_apiserver.enable = true; me.kube_controller_manager.enable = true; + me.kube_scheduler.enable = true; + me.kubernetes.enable = true; }; } diff --git a/nix/kubernetes/roles/etcd/default.nix b/nix/kubernetes/roles/etcd/default.nix index c3de9202..468b8657 100644 --- a/nix/kubernetes/roles/etcd/default.nix +++ b/nix/kubernetes/roles/etcd/default.nix @@ -104,6 +104,6 @@ e2fsprogs # mkfs.ext4 gptfdisk # cgdisk ]; - networking.firewall.enable = false; + networking.firewall.enable = false; # TODO: This is just here for debugging / initial development. }; } diff --git a/nix/kubernetes/roles/kube_apiserver/default.nix b/nix/kubernetes/roles/kube_apiserver/default.nix index a78f4dcb..66b31aad 100644 --- a/nix/kubernetes/roles/kube_apiserver/default.nix +++ b/nix/kubernetes/roles/kube_apiserver/default.nix @@ -131,7 +131,17 @@ in ); Restart = "on-failure"; RestartSec = 5; + LimitNOFILE = 65536; + User = "kubernetes"; }; }; + + networking.firewall.allowedTCPPorts = [ + 6443 + ]; + + systemd.tmpfiles.rules = [ + "f /var/log/audit.log 0600 kubernetes kubernetes - -" + ]; }; } diff --git a/nix/kubernetes/roles/kube_controller_manager/default.nix b/nix/kubernetes/roles/kube_controller_manager/default.nix index 9b33b497..f31de3f8 100644 --- a/nix/kubernetes/roles/kube_controller_manager/default.nix +++ b/nix/kubernetes/roles/kube_controller_manager/default.nix @@ -27,33 +27,37 @@ in description = "Kubernetes Controller Manager"; documentation = [ "https://github.com/kubernetes/kubernetes" ]; wantedBy = [ "kubernetes.target" ]; + after = [ "kube-apiserver.service" ]; # path = with pkgs; [ # zfs # ]; unitConfig.DefaultDependencies = "no"; serviceConfig = { - Type = "notify"; ExecStart = ( shellCommand [ # NEW: "${pkgs.kubernetes}/bin/kube-controller-manager" "--bind-address=0.0.0.0" # "--cluster-cidr=10.200.0.0/16" - "--cluster-cidr=2620:11f:7001:7:ffff:ffff:0ac8:0000/16" + # "--cluster-cidr=2620:11f:7001:7:ffff:ffff:0ac8:0000/16" + "--cluster-cidr=fd49:0595:2bba::/48" "--cluster-name=kubernetes" "--cluster-signing-cert-file=/.persist/keys/kube/ca.crt" "--cluster-signing-key-file=/.persist/keys/kube/ca.key" "--kubeconfig=/.persist/keys/kube/kube-controller-manager.kubeconfig" "--root-ca-file=/.persist/keys/kube/ca.crt" "--service-account-private-key-file=/.persist/keys/kube/service-accounts.key" - "--service-cluster-ip-range=2620:11f:7001:7:ffff:ffff:0ac5:0000/16" # "--service-cluster-ip-range=10.197.0.0/16" + # "--service-cluster-ip-range=2620:11f:7001:7:ffff:ffff:0ac5:0000/16" + "--service-cluster-ip-range=fd00:3e42:e349::/48" "--use-service-account-credentials=true" "--v=2" ] ); Restart = "on-failure"; RestartSec = 5; + LimitNOFILE = 65536; + User = "kubernetes"; }; }; }; diff --git a/nix/kubernetes/roles/kube_scheduler/default.nix b/nix/kubernetes/roles/kube_scheduler/default.nix new file mode 100644 index 00000000..89a76971 --- /dev/null +++ b/nix/kubernetes/roles/kube_scheduler/default.nix @@ -0,0 +1,51 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + # shellCommand = cmd: (lib.concatMapStringsSep " " lib.strings.escapeShellArg cmd); + shellCommand = cmd: (builtins.concatStringsSep " " cmd); +in +{ + imports = [ ]; + + options.me = { + kube_scheduler.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install kube_scheduler."; + }; + }; + + config = lib.mkIf config.me.kube_scheduler.enable { + systemd.services.kube-scheduler = { + enable = true; + description = "Kubernetes Scheduler"; + documentation = [ "https://github.com/kubernetes/kubernetes" ]; + wantedBy = [ "kubernetes.target" ]; + after = [ "kube-apiserver.service" ]; + # path = with pkgs; [ + # zfs + # ]; + unitConfig.DefaultDependencies = "no"; + serviceConfig = { + ExecStart = ( + shellCommand [ + # NEW: + "${pkgs.kubernetes}/bin/kube-scheduler" + "--config=${./files/kube-scheduler.yaml}" + "--v=2" + ] + ); + Restart = "on-failure"; + RestartSec = 5; + LimitNOFILE = 65536; + User = "kubernetes"; + }; + }; + }; +} diff --git a/nix/kubernetes/roles/kube_scheduler/files/kube-scheduler.yaml b/nix/kubernetes/roles/kube_scheduler/files/kube-scheduler.yaml new file mode 100644 index 00000000..38814532 --- /dev/null +++ b/nix/kubernetes/roles/kube_scheduler/files/kube-scheduler.yaml @@ -0,0 +1,6 @@ +apiVersion: kubescheduler.config.k8s.io/v1 +kind: KubeSchedulerConfiguration +clientConnection: + kubeconfig: "/.persist/keys/kube/kube-scheduler.kubeconfig" +leaderElection: + leaderElect: true