talexander/machine_setup
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Install kubernetes.

Browse Source
This commit is contained in:
Tom Alexander 2025-12-08 20:33:41 -05:00 committed by Tom Alexander
parent f4a2f57062
commit 342a5e338c
Signed by: talexander
GPG Key ID: 36C99E8B3C39D85F
10 changed files with 309 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
nix/kubernetes/configuration.nix
View File

@ -7,15 +7,18 @@
{
imports = [
./roles/boot
./roles/control_plane
./roles/doas
./roles/dont_use_substituters
./roles/etcd
./roles/image_based_appliance
./roles/iso
./roles/optimized_build
./roles/dont_use_substituters
./roles/kube_apiserver
./roles/kubernetes
./roles/minimal_base
./roles/network
./roles/nvme
./roles/optimized_build
./roles/ssh
./roles/sshd
./roles/user

6
nix/kubernetes/flake.nix
View File

@ -95,10 +95,9 @@
...
}:
let
repl_path = toString ./.;
nix-self-repl = pkgs.writeShellScriptBin "nix-self-repl" ''
source /etc/set-environment
nix repl "${repl_path}/repl.nix" "$@"
nix repl --expr 'builtins.getFlake "${self}"'
'';
# If we wanted the current version of a flake then we'd just launch
# nix repl
@ -130,10 +129,9 @@
...
}:
let
repl_path = toString ./.;
nix-self-repl = pkgs.writeShellScriptBin "nix-self-repl" ''
source /etc/set-environment
nix repl "${repl_path}/repl.nix" "$@"
nix repl --expr 'builtins.getFlake "${self}"'
'';
# If we wanted the current version of a flake then we'd just launch
# nix repl

8
nix/kubernetes/hosts/controller0/default.nix
View File

@ -116,6 +116,14 @@
"controller2=https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2380" # 10.215.1.223
];
me.kube_apiserver.internal_ip = "2620:11f:7001:7:ffff:ffff:0ad7:01dd";
me.kube_apiserver.etcd_services = [
"https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2379" # 10.215.1.221
"https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2379" # 10.215.1.222
"https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2379" # 10.215.1.223
];
me.control_plane.enable = true;
me.dont_use_substituters.enable = true;
me.etcd.enable = true;
me.minimal_base.enable = true;

8
nix/kubernetes/hosts/controller1/default.nix
View File

@ -116,6 +116,14 @@
"controller2=https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2380" # 10.215.1.223
];
me.kube_apiserver.internal_ip = "2620:11f:7001:7:ffff:ffff:0ad7:01de";
me.kube_apiserver.etcd_services = [
"https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2379" # 10.215.1.221
"https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2379" # 10.215.1.222
"https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2379" # 10.215.1.223
];
me.control_plane.enable = true;
me.dont_use_substituters.enable = true;
me.etcd.enable = true;
me.minimal_base.enable = true;

8
nix/kubernetes/hosts/controller2/default.nix
View File

@ -116,6 +116,14 @@
"controller2=https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2380" # 10.215.1.223
];
me.kube_apiserver.internal_ip = "2620:11f:7001:7:ffff:ffff:0ad7:01df";
me.kube_apiserver.etcd_services = [
"https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2379" # 10.215.1.221
"https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2379" # 10.215.1.222
"https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2379" # 10.215.1.223
];
me.control_plane.enable = true;
me.dont_use_substituters.enable = true;
me.etcd.enable = true;
me.minimal_base.enable = true;

113
nix/kubernetes/keys/scope.nix
View File

@ -4,6 +4,8 @@
callPackage,
writeShellScript,
openssh,
runCommand,
writeText,
lib,
}:
let
@ -50,25 +52,91 @@ makeScope newScope (
inherit all_hostnames;
k8s = self;
};
deploy_key = (
vm_name: file: ''
${openssh}/bin/ssh mrmanager rm -f /vm/${vm_name}/persist/keys/etcd/${builtins.baseNameOf file} ~/${builtins.baseNameOf file}
${openssh}/bin/scp ${file} mrmanager:~/${builtins.baseNameOf file}
${openssh}/bin/ssh mrmanager doas install -o 10016 -g 10016 -m 0640 ~/${builtins.baseNameOf file} /vm/${vm_name}/persist/keys/etcd/${builtins.baseNameOf file}
${openssh}/bin/ssh mrmanager rm -f ~/${builtins.baseNameOf file}
deploy_file = (
{
dest_dir,
file,
name ? (builtins.baseNameOf file),
owner,
group,
mode,
}:
''
##
## deploy ${name} to ${dest_dir}
##
${openssh}/bin/ssh mrmanager doas rm -f ${dest_dir}/${name} ~/${name}
${openssh}/bin/scp ${file} mrmanager:~/${name}
${openssh}/bin/ssh mrmanager doas install -o ${toString owner} -g ${toString group} -m ${mode} ~/${name} ${dest_dir}/${name}
${openssh}/bin/ssh mrmanager doas rm -f ~/${name}
''
);
deploy_machine = (
vm_name:
(
''
##
## Create directories on ${vm_name}
##
${openssh}/bin/ssh mrmanager doas install -d -o 11235 -g 11235 -m 0755 /vm/${vm_name}/persist/keys
${openssh}/bin/ssh mrmanager doas install -d -o 10016 -g 10016 -m 0755 /vm/${vm_name}/persist/keys/etcd
${openssh}/bin/ssh mrmanager doas install -d -o 10024 -g 10024 -m 0755 /vm/${vm_name}/persist/keys/kube
''
+ (lib.concatMapStringsSep "\n" (deploy_key vm_name) [
"${self.kubernetes}/kubernetes.pem"
"${self.kubernetes}/kubernetes-key.pem"
"${self.ca}/ca.pem"
+ (lib.concatMapStringsSep "\n" deploy_file [
{
dest_dir = "/vm/${vm_name}/persist/keys/etcd";
file = "${self.kubernetes}/kubernetes.pem";
owner = 10016;
group = 10016;
mode = "0640";
}
{
dest_dir = "/vm/${vm_name}/persist/keys/etcd";
file = "${self.kubernetes}/kubernetes-key.pem";
owner = 10016;
group = 10016;
mode = "0600";
}
{
dest_dir = "/vm/${vm_name}/persist/keys/etcd";
file = "${self.ca}/ca.pem";
owner = 10016;
group = 10016;
mode = "0640";
}
{
dest_dir = "/vm/${vm_name}/persist/keys/kube";
file = "${self.kubernetes}/kubernetes.pem";
owner = 10024;
group = 10024;
mode = "0640";
}
{
dest_dir = "/vm/${vm_name}/persist/keys/kube";
file = "${self.kubernetes}/kubernetes-key.pem";
owner = 10024;
group = 10024;
mode = "0640";
}
{
dest_dir = "/vm/${vm_name}/persist/keys/kube";
file = "${self.ca}/ca.pem";
owner = 10024;
group = 10024;
mode = "0600";
}
{
dest_dir = "/vm/${vm_name}/persist/keys/kube";
file = (writeText "encryption-config.yaml" (lib.generators.toYAML { } kube_encryption_config));
name = "encryption-config.yaml";
owner = 10024;
group = 10024;
mode = "0600";
}
])
)
);
@ -84,6 +152,31 @@ makeScope newScope (
"nc2"
])
);
kube_encryption_key = runCommand "kube_encryption_key" { } ''
head -c 32 /dev/urandom | base64 | tee $out
'';
kube_encryption_config = {
kind = "EncryptionConfig";
apiVersion = "v1";
resources = [
{
resources = [ "secrets" ];
providers = [
{
aescbc = {
keys = [
{
name = "key1";
secret = (builtins.readFile "${kube_encryption_key}");
}
];
};
}
{ identity = { }; }
];
}
];
};
in
{
ca = (callPackage ./package/k8s-ca/package.nix additional_vars);

5
nix/kubernetes/repl.nix
View File

@ -1,5 +0,0 @@
let
flake = builtins.getFlake (toString ./.);
nixpkgs = import <nixpkgs> { };
in
{ inherit flake; } // flake // builtins // nixpkgs // nixpkgs.lib // flake.nixosConfigurations

24
nix/kubernetes/roles/control_plane/default.nix Normal file
View File

@ -0,0 +1,24 @@
{
config,
lib,
pkgs,
...
}:
{
imports = [ ];
options.me = {
control_plane.enable = lib.mkOption {
type = lib.types.bool;
default = false;
example = true;
description = "Whether we want to install control_plane.";
};
};
config = lib.mkIf config.me.control_plane.enable {
me.kubernetes.enable = true;
me.kube_apiserver.enable = true;
};
}

98
nix/kubernetes/roles/kube_apiserver/default.nix Normal file
View File

@ -0,0 +1,98 @@
{
config,
lib,
pkgs,
...
}:
let
shellCommand = cmd: (lib.concatMapStringsSep " " lib.strings.escapeShellArg cmd);
in
{
imports = [ ];
options.me = {
kube_apiserver.enable = lib.mkOption {
type = lib.types.bool;
default = false;
example = true;
description = "Whether we want to install kube_apiserver.";
};
kube_apiserver.internal_ip = lib.mkOption {
# default = { };
example = "192.168.1.10";
type = lib.types.str;
description = "IP address this server should advertise.";
};
kube_apiserver.etcd_services = lib.mkOption {
default = [ ];
example = [ "https://192.168.1.10:2379" ];
type = lib.types.listOf lib.types.str;
description = "Endpoints for etcd servers.";
};
};
config = lib.mkIf config.me.kube_apiserver.enable {
systemd.services.kube-apiserver = {
enable = true;
description = "Kubernetes API Server";
documentation = [ "https://github.com/kubernetes/kubernetes" ];
wantedBy = [ "kubernetes.target" ];
# path = with pkgs; [
# zfs
# ];
unitConfig.DefaultDependencies = "no";
serviceConfig = {
Type = "notify";
ExecStart = (
shellCommand [
"${pkgs.kubernetes}/bin/kube-apiserver"
"--advertise-address=${config.me.kube_apiserver.internal_ip}"
"--allow-privileged=true"
"--apiserver-count=3"
"--audit-log-maxage=30"
"--audit-log-maxbackup=3"
"--audit-log-maxsize=100"
"--audit-log-path=/var/log/audit.log"
"--authorization-mode=Node,RBAC"
"--bind-address=0.0.0.0"
"--client-ca-file=/.persist/keys/kube/ca.pem"
"--requestheader-client-ca-file=/var/lib/kubernetes/requestheader-client-ca.pem"
''--requestheader-allowed-names=""''
"--requestheader-extra-headers-prefix=X-Remote-Extra-"
"--requestheader-group-headers=X-Remote-Group"
"--requestheader-username-headers=X-Remote-User"
"--proxy-client-cert-file=/var/lib/kubernetes/{{ node_name }}-proxy.pem"
"--proxy-client-key-file=/var/lib/kubernetes/{{ node_name }}-proxy-key.pem"
"--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota"
"--etcd-cafile=/.persist/keys/kube/ca.pem"
"--etcd-certfile=/.persist/keys/kube/kubernetes.pem"
"--etcd-keyfile=/.persist/keys/kube/kubernetes-key.pem"
"--etcd-servers=${builtins.concatStringsSep "," config.me.kube_apiserver.etcd_services}"
"--event-ttl=1h"
"--encryption-provider-config=/.persist/keys/kube/encryption-config.yaml"
"--kubelet-certificate-authority=/.persist/keys/kube/ca.pem"
"--kubelet-client-certificate=/.persist/keys/kube/kubernetes.pem"
"--kubelet-client-key=/.persist/keys/kube/kubernetes-key.pem"
"--runtime-config='api/all=true'"
"--service-account-key-file=/var/lib/kubernetes/service-account.pem"
"--service-account-signing-key-file=/var/lib/kubernetes/service-account-key.pem"
"--service-account-issuer=https://{{ kubernetes_public_address }}:6443"
"--service-node-port-range=30000-32767"
"--tls-cert-file=/.persist/keys/kube/kubernetes.pem"
"--tls-private-key-file=/.persist/keys/kube/kubernetes-key.pem"
"--tls-min-version=VersionTLS13"
"--kubelet-preferred-address-types=InternalIP,ExternalDNS,ExternalIP,Hostname,InternalDNS"
"--service-cluster-ip-range=10.197.0.0/16"
"--enable-aggregator-routing=true"
"--v=2"
]
);
Restart = "on-failure";
RestartSec = 5;
};
};
};
}

53
nix/kubernetes/roles/kubernetes/default.nix Normal file
View File

@ -0,0 +1,53 @@
{
config,
lib,
pkgs,
...
}:
{
imports = [ ];
options.me = {
kubernetes.enable = lib.mkOption {
type = lib.types.bool;
default = false;
example = true;
description = "Whether we want to install kubernetes.";
};
};
config = lib.mkIf config.me.kubernetes.enable {
environment.systemPackages = with pkgs; [
kubernetes
];
systemd.targets.kubernetes = {
description = "Kubernetes";
wantedBy = [ "multi-user.target" ];
};
users.users.kubernetes = {
uid = 10024;
description = "Kubernetes";
group = "kubernetes";
home = "/var/lib/kubernetes";
createHome = true;
homeMode = "755";
isSystemUser = true;
};
users.groups.kubernetes.gid = 10024;
environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) {
hideMounts = true;
directories = [
{
directory = "/var/lib/kubernetes";
user = "kubernetes";
group = "kubernetes";
mode = "0755";
}
];
};
};
}