talexander/machine_setup
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Add kube-scheduler.

Browse Source
This commit is contained in:
Tom Alexander 2025-12-15 20:09:46 -05:00 committed by Tom Alexander
parent 6483b20b89
commit 3b96f8d26c
Signed by: talexander
GPG Key ID: 36C99E8B3C39D85F
10 changed files with 95 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
nix/kubernetes/configuration.nix
View File

@ -15,6 +15,7 @@
./roles/iso
./roles/kube_apiserver
./roles/kube_controller_manager
./roles/kube_scheduler
./roles/kubernetes
./roles/minimal_base
./roles/network

71
nix/kubernetes/keys/package/deploy-script/package.nix
View File

@ -142,70 +142,13 @@ let
group = 10024;
mode = "0600";
}
# {
# dest_dir = "/vm/${vm_name}/persist/keys/kube";
# file = "${self.kubernetes}/kubernetes.pem";
# owner = 10024;
# group = 10024;
# mode = "0640";
# }
# {
# dest_dir = "/vm/${vm_name}/persist/keys/kube";
# file = "${self.kubernetes}/kubernetes-key.pem";
# owner = 10024;
# group = 10024;
# mode = "0640";
# }
# {
# dest_dir = "/vm/${vm_name}/persist/keys/kube";
# file = "${self.ca}/ca.pem";
# owner = 10024;
# group = 10024;
# mode = "0600";
# }
# {
# dest_dir = "/vm/${vm_name}/persist/keys/kube";
# file = (writeText "encryption-config.yaml" (lib.generators.toYAML { } kube_encryption_config));
# name = "encryption-config.yaml";
# owner = 10024;
# group = 10024;
# mode = "0600";
# }
# {
# dest_dir = "/vm/${vm_name}/persist/keys/kube";
# file = "${self.service_account}/service-account.pem";
# owner = 10024;
# group = 10024;
# mode = "0600";
# }
# {
# dest_dir = "/vm/${vm_name}/persist/keys/kube";
# file = "${self.service_account}/service-account-key.pem";
# owner = 10024;
# group = 10024;
# mode = "0600";
# }
# {
# dest_dir = "/vm/${vm_name}/persist/keys/kube";
# file = "${self.requestheader-client-ca}/requestheader-client-ca.pem";
# owner = 10024;
# group = 10024;
# mode = "0600";
# }
# {
# dest_dir = "/vm/${vm_name}/persist/keys/kube";
# file = "${self.controller-proxy}/${vm_name_to_hostname vm_name}-proxy.pem";
# owner = 10024;
# group = 10024;
# mode = "0600";
# }
# {
# dest_dir = "/vm/${vm_name}/persist/keys/kube";
# file = "${self.controller-proxy}/${vm_name_to_hostname vm_name}-proxy-key.pem";
# owner = 10024;
# group = 10024;
# mode = "0600";
# }
{
dest_dir = "/vm/${vm_name}/persist/keys/kube";
file = "${k8s.client-configs.kube-scheduler}/kube-scheduler.kubeconfig";
owner = 10024;
group = 10024;
mode = "0600";
}
])
)
);

3
nix/kubernetes/keys/package/k8s-client-config/package.nix
View File

@ -8,6 +8,7 @@
# installCheckPhase
# distPhase
{
lib,
stdenv,
k8s,
kubectl,
@ -27,7 +28,7 @@ stdenv.mkDerivation (finalAttrs: {
kubectl config set-cluster kubernetes-the-hard-way \
--certificate-authority=${k8s.ca}/ca.crt \
--embed-certs=true \
--server=${config_server} \
--server=${lib.strings.escapeShellArg config_server} \
--kubeconfig=${config_name}.kubeconfig
kubectl config set-credentials ${config_user} \

11
nix/kubernetes/keys/scope.nix
View File

@ -141,15 +141,20 @@ makeScope newScope (
};
kube-controller-manager = {
config_user = "system:kube-controller-manager";
config_server = "https://server.kubernetes.local:6443";
# config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443";
config_server = "https://127.0.0.1:6443";
# config_server = "https://server.kubernetes.local:6443";
};
kube-scheduler = {
config_user = "system:kube-scheduler";
config_server = "https://server.kubernetes.local:6443";
# config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443";
config_server = "https://127.0.0.1:6443";
# config_server = "https://server.kubernetes.local:6443";
};
admin = {
config_user = "admin";
config_server = "https://127.0.0.1:6443";
config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443";
# config_server = "https://127.0.0.1:6443";
};
}
);

3
nix/kubernetes/roles/control_plane/default.nix
View File

@ -18,8 +18,9 @@
};
config = lib.mkIf config.me.control_plane.enable {
me.kubernetes.enable = true;
me.kube_apiserver.enable = true;
me.kube_controller_manager.enable = true;
me.kube_scheduler.enable = true;
me.kubernetes.enable = true;
};
}

2
nix/kubernetes/roles/etcd/default.nix
View File

@ -104,6 +104,6 @@
e2fsprogs # mkfs.ext4
gptfdisk # cgdisk
];
networking.firewall.enable = false;
networking.firewall.enable = false; # TODO: This is just here for debugging / initial development.
};
}

10
nix/kubernetes/roles/kube_apiserver/default.nix
View File

@ -131,7 +131,17 @@ in
);
Restart = "on-failure";
RestartSec = 5;
LimitNOFILE = 65536;
User = "kubernetes";
};
};
networking.firewall.allowedTCPPorts = [
6443
];
systemd.tmpfiles.rules = [
"f /var/log/audit.log 0600 kubernetes kubernetes - -"
];
};
}

10
nix/kubernetes/roles/kube_controller_manager/default.nix
View File

@ -27,33 +27,37 @@ in
description = "Kubernetes Controller Manager";
documentation = [ "https://github.com/kubernetes/kubernetes" ];
wantedBy = [ "kubernetes.target" ];
after = [ "kube-apiserver.service" ];
# path = with pkgs; [
# zfs
# ];
unitConfig.DefaultDependencies = "no";
serviceConfig = {
Type = "notify";
ExecStart = (
shellCommand [
# NEW:
"${pkgs.kubernetes}/bin/kube-controller-manager"
"--bind-address=0.0.0.0"
# "--cluster-cidr=10.200.0.0/16"
"--cluster-cidr=2620:11f:7001:7:ffff:ffff:0ac8:0000/16"
# "--cluster-cidr=2620:11f:7001:7:ffff:ffff:0ac8:0000/16"
"--cluster-cidr=fd49:0595:2bba::/48"
"--cluster-name=kubernetes"
"--cluster-signing-cert-file=/.persist/keys/kube/ca.crt"
"--cluster-signing-key-file=/.persist/keys/kube/ca.key"
"--kubeconfig=/.persist/keys/kube/kube-controller-manager.kubeconfig"
"--root-ca-file=/.persist/keys/kube/ca.crt"
"--service-account-private-key-file=/.persist/keys/kube/service-accounts.key"
"--service-cluster-ip-range=2620:11f:7001:7:ffff:ffff:0ac5:0000/16"
# "--service-cluster-ip-range=10.197.0.0/16"
# "--service-cluster-ip-range=2620:11f:7001:7:ffff:ffff:0ac5:0000/16"
"--service-cluster-ip-range=fd00:3e42:e349::/48"
"--use-service-account-credentials=true"
"--v=2"
]
);
Restart = "on-failure";
RestartSec = 5;
LimitNOFILE = 65536;
User = "kubernetes";
};
};
};

51
nix/kubernetes/roles/kube_scheduler/default.nix Normal file
View File

@ -0,0 +1,51 @@
{
config,
lib,
pkgs,
...
}:
let
# shellCommand = cmd: (lib.concatMapStringsSep " " lib.strings.escapeShellArg cmd);
shellCommand = cmd: (builtins.concatStringsSep " " cmd);
in
{
imports = [ ];
options.me = {
kube_scheduler.enable = lib.mkOption {
type = lib.types.bool;
default = false;
example = true;
description = "Whether we want to install kube_scheduler.";
};
};
config = lib.mkIf config.me.kube_scheduler.enable {
systemd.services.kube-scheduler = {
enable = true;
description = "Kubernetes Scheduler";
documentation = [ "https://github.com/kubernetes/kubernetes" ];
wantedBy = [ "kubernetes.target" ];
after = [ "kube-apiserver.service" ];
# path = with pkgs; [
# zfs
# ];
unitConfig.DefaultDependencies = "no";
serviceConfig = {
ExecStart = (
shellCommand [
# NEW:
"${pkgs.kubernetes}/bin/kube-scheduler"
"--config=${./files/kube-scheduler.yaml}"
"--v=2"
]
);
Restart = "on-failure";
RestartSec = 5;
LimitNOFILE = 65536;
User = "kubernetes";
};
};
};
}

6
nix/kubernetes/roles/kube_scheduler/files/kube-scheduler.yaml Normal file
View File

@ -0,0 +1,6 @@
apiVersion: kubescheduler.config.k8s.io/v1
kind: KubeSchedulerConfiguration
clientConnection:
kubeconfig: "/.persist/keys/kube/kube-scheduler.kubeconfig"
leaderElection:
leaderElect: true