talexander/machine_setup
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Add kubelet.

Browse Source
This commit is contained in:
Tom Alexander 2025-12-16 19:31:33 -05:00 committed by Tom Alexander
parent fa24540bb3
commit 0463d2cbd1
Signed by: talexander
GPG Key ID: 36C99E8B3C39D85F
7 changed files with 214 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
nix/kubernetes/configuration.nix
View File

@ -17,6 +17,7 @@
./roles/kube_apiserver
./roles/kube_controller_manager
./roles/kube_scheduler
./roles/kubelet
./roles/kubernetes
./roles/minimal_base
./roles/network

74
nix/kubernetes/keys/package/deploy-script/package.nix
View File

@ -8,6 +8,7 @@
# installCheckPhase
# distPhase
{
config,
lib,
stdenv,
writeShellScript,
@ -16,17 +17,35 @@
...
}:
let
vm_name_to_hostname =
let
mapping = {
"nc0" = "controller0";
"nc1" = "controller1";
"nc2" = "controller2";
"nw0" = "worker0";
"nw1" = "worker1";
"nw2" = "worker2";
};
in
(vm_name: mapping."${vm_name}");
deploy_script_body = (
''
set -euo pipefail
IFS=$'\n\t'
DIR="$( cd "$( dirname "''${BASH_SOURCE[0]}" )" && pwd )"
''
+ (lib.concatMapStringsSep "\n" deploy_machine [
+ (lib.concatMapStringsSep "\n" deploy_control_plane [
"nc0"
"nc1"
"nc2"
])
+ (lib.concatMapStringsSep "\n" deploy_worker [
"nw0"
"nw1"
"nw2"
])
);
deploy_script = (writeShellScript "deploy-script" deploy_script_body);
deploy_file = (
@ -50,14 +69,14 @@ let
''
);
deploy_machine = (
deploy_control_plane = (
vm_name:
(
''
##
## Create directories on ${vm_name}
##
${openssh}/bin/ssh mrmanager doas install -d -o 11235 -g 11235 -m 0755 /vm/${vm_name}/persist/keys
${openssh}/bin/ssh mrmanager doas install -d -o 0 -g 0 -m 0755 /vm/${vm_name}/persist/keys
${openssh}/bin/ssh mrmanager doas install -d -o 10016 -g 10016 -m 0755 /vm/${vm_name}/persist/keys/etcd
${openssh}/bin/ssh mrmanager doas install -d -o 10024 -g 10024 -m 0755 /vm/${vm_name}/persist/keys/kube
@ -152,6 +171,55 @@ let
])
)
);
deploy_worker = (
vm_name:
(
''
##
## Create directories on ${vm_name}
##
${openssh}/bin/ssh mrmanager doas install -d -o 0 -g 0 -m 0755 /vm/${vm_name}/persist/keys
${openssh}/bin/ssh mrmanager doas install -d -o 10024 -g 10024 -m 0755 /vm/${vm_name}/persist/keys/kube
''
+ (lib.concatMapStringsSep "\n" deploy_file [
{
dest_dir = "/vm/${vm_name}/persist/keys/kube";
file = "${k8s.ca}/ca.crt";
owner = 10024;
group = 10024;
mode = "0640";
}
{
dest_dir = "/vm/${vm_name}/persist/keys/kube";
file = "${k8s.keys."${vm_name_to_hostname vm_name}"}/${vm_name_to_hostname vm_name}.crt";
name = "kubelet.crt";
owner = 10024;
group = 10024;
mode = "0640";
}
{
dest_dir = "/vm/${vm_name}/persist/keys/kube";
file = "${k8s.keys."${vm_name_to_hostname vm_name}"}/${vm_name_to_hostname vm_name}.key";
name = "kubelet.key";
owner = 10024;
group = 10024;
mode = "0600";
}
{
dest_dir = "/vm/${vm_name}/persist/keys/kube";
file = "${
k8s.client-configs."${vm_name_to_hostname vm_name}"
}/${vm_name_to_hostname vm_name}.kubeconfig";
name = "kubeconfig";
owner = 10024;
group = 10024;
mode = "0600";
}
])
)
);
in
stdenv.mkDerivation (finalAttrs: {
name = "deploy-script";

27
nix/kubernetes/keys/scope.nix
View File

@ -69,12 +69,6 @@ let
];
};
};
# _vm_name_to_hostname = {
# "nc0" = "controller0";
# "nc1" = "controller1";
# "nc2" = "controller2";
# };
# vm_name_to_hostname = (vm_name: _vm_name_to_hostname."${vm_name}");
in
makeScope newScope (
self:
@ -113,27 +107,36 @@ makeScope newScope (
{
controller0 = {
config_user = "system:node:controller0";
config_server = "https://server.kubernetes.local:6443";
config_server = "https://127.0.0.1:6443";
# config_server = "https://server.kubernetes.local:6443";
};
controller1 = {
config_user = "system:node:controller1";
config_server = "https://server.kubernetes.local:6443";
config_server = "https://127.0.0.1:6443";
# config_server = "https://server.kubernetes.local:6443";
};
controller2 = {
config_user = "system:node:controller2";
config_server = "https://server.kubernetes.local:6443";
config_server = "https://127.0.0.1:6443";
# config_server = "https://server.kubernetes.local:6443";
};
worker0 = {
config_user = "system:node:worker0";
config_server = "https://server.kubernetes.local:6443";
config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443";
# config_server = "https://127.0.0.1:6443";
# config_server = "https://server.kubernetes.local:6443";
};
worker1 = {
config_user = "system:node:worker1";
config_server = "https://server.kubernetes.local:6443";
config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443";
# config_server = "https://127.0.0.1:6443";
# config_server = "https://server.kubernetes.local:6443";
};
worker2 = {
config_user = "system:node:worker2";
config_server = "https://server.kubernetes.local:6443";
config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443";
# config_server = "https://127.0.0.1:6443";
# config_server = "https://server.kubernetes.local:6443";
};
kube-proxy = {
config_user = "system:kube-proxy";

1
nix/kubernetes/roles/kube_controller_manager/default.nix
View File

@ -35,7 +35,6 @@ in
serviceConfig = {
ExecStart = (
shellCommand [
# NEW:
"${pkgs.kubernetes}/bin/kube-controller-manager"
"--bind-address=0.0.0.0"
# "--cluster-cidr=10.200.0.0/16"

101
nix/kubernetes/roles/kubelet/default.nix Normal file
View File

@ -0,0 +1,101 @@
{
config,
lib,
pkgs,
...
}:
let
# shellCommand = cmd: (lib.concatMapStringsSep " " lib.strings.escapeShellArg cmd);
shellCommand = cmd: (builtins.concatStringsSep " " cmd);
settingsFormat = pkgs.formats.yaml { };
config_file = settingsFormat.generate "kubelet-config.yaml" config.me.kubelet.settings;
in
{
imports = [ ];
options.me = {
kubelet.enable = lib.mkOption {
type = lib.types.bool;
default = false;
example = true;
description = "Whether we want to install kubelet.";
};
kubelet.settings = lib.mkOption {
type = settingsFormat.type;
default = {
kind = "KubeletConfiguration";
apiVersion = "kubelet.config.k8s.io/v1beta1";
address = "0.0.0.0";
authentication = {
anonymous = {
enabled = false;
};
webhook = {
enabled = true;
};
x509 = {
clientCAFile = "/var/lib/kubelet/ca.crt";
};
};
authorization = {
mode = "Webhook";
};
cgroupDriver = "systemd";
containerRuntimeEndpoint = "unix:///var/run/containerd/containerd.sock";
enableServer = true;
failSwapOn = false;
maxPods = 16;
memorySwap = {
swapBehavior = "NoSwap";
};
port = 10250;
resolvConf = "/etc/resolv.conf";
registerNode = true;
runtimeRequestTimeout = "15m";
tlsCertFile = "/var/lib/kubelet/kubelet.crt";
tlsPrivateKeyFile = "/var/lib/kubelet/kubelet.key";
};
description = ''
kubelet-config.yaml
'';
};
};
config = lib.mkIf config.me.kubelet.enable {
systemd.services.kubelet = {
enable = true;
description = "Kubernetes Kubelet";
documentation = [ "https://github.com/kubernetes/kubernetes" ];
wantedBy = [ "kubernetes.target" ];
after = [ "containerd.service" ];
requires = [ "containerd.service" ];
# path = with pkgs; [
# zfs
# ];
unitConfig.DefaultDependencies = "no";
serviceConfig = {
ExecStart = (
shellCommand [
"${pkgs.kubernetes}/bin/kubelet"
# "--config=${config_file}"
"--config=${./files/kubelet-config.yaml}"
"--kubeconfig=/.persist/keys/kube/kubeconfig"
"--v=2"
]
);
Restart = "on-failure";
RestartSec = 5;
# ConfigurationDirectory = "kubernetes";
# CPUAccounting = "true";
# IPAccounting = "true";
# KillMode = "process";
# MemoryAccounting = "true";
# StartLimitInterval = 0;
# RuntimeDirectory = "kubelet";
# StateDirectory = "kubelet";
};
};
};
}

25
nix/kubernetes/roles/kubelet/files/kubelet-config.yaml Normal file
View File

@ -0,0 +1,25 @@
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
address: "0.0.0.0"
authentication:
anonymous:
enabled: false
webhook:
enabled: true
x509:
clientCAFile: "/.persist/keys/kube/ca.crt"
authorization:
mode: Webhook
cgroupDriver: systemd
containerRuntimeEndpoint: "unix:///var/run/containerd/containerd.sock"
enableServer: true
failSwapOn: false
maxPods: 16
memorySwap:
swapBehavior: NoSwap
port: 10250
resolvConf: "/etc/resolv.conf"
registerNode: true
runtimeRequestTimeout: "15m"
tlsCertFile: "/.persist/keys/kube/kubelet.crt"
tlsPrivateKeyFile: "/.persist/keys/kube/kubelet.key"

1
nix/kubernetes/roles/worker_node/default.nix
View File

@ -19,6 +19,7 @@
config = lib.mkIf config.me.worker_node.enable {
me.containerd.enable = true;
me.kubelet.enable = true;
me.kubernetes.enable = true;
};
}