324 lines
9.0 KiB
Nix
324 lines
9.0 KiB
Nix
{
|
|
makeScope,
|
|
newScope,
|
|
callPackage,
|
|
writeShellScript,
|
|
openssh,
|
|
runCommand,
|
|
writeText,
|
|
lib,
|
|
}:
|
|
let
|
|
public_addresses = [
|
|
"74.80.180.138"
|
|
];
|
|
internal_addresses = [
|
|
# nc0
|
|
"10.215.1.221"
|
|
"2620:11f:7001:7:ffff:ffff:0ad7:01dd"
|
|
# nc1
|
|
"10.215.1.222"
|
|
"2620:11f:7001:7:ffff:ffff:0ad7:01de"
|
|
# nc2
|
|
"10.215.1.223"
|
|
"2620:11f:7001:7:ffff:ffff:0ad7:01df"
|
|
# nw0
|
|
"10.215.1.224"
|
|
"2620:11f:7001:7:ffff:ffff:0ad7:01e0"
|
|
# nw1
|
|
"10.215.1.225"
|
|
"2620:11f:7001:7:ffff:ffff:0ad7:01e1"
|
|
# nw2
|
|
"10.215.1.226"
|
|
"2620:11f:7001:7:ffff:ffff:0ad7:01e2"
|
|
];
|
|
all_hostnames = [
|
|
"10.197.0.1"
|
|
"10.0.0.1"
|
|
"127.0.0.1"
|
|
"kubernetes"
|
|
"kubernetes.default"
|
|
"kubernetes.default.svc"
|
|
"kubernetes.default.svc.cluster"
|
|
"kubernetes.svc.cluster.local"
|
|
]
|
|
++ public_addresses
|
|
++ internal_addresses;
|
|
controllers = {
|
|
"controller0" = {
|
|
"internal_ips" = [
|
|
"10.215.1.221"
|
|
"2620:11f:7001:7:ffff:ffff:0ad7:01dd"
|
|
];
|
|
"external_ips" = [
|
|
"2620:11f:7001:7:ffff:ffff:0ad7:01dd"
|
|
];
|
|
};
|
|
"controller1" = {
|
|
"internal_ips" = [
|
|
"10.215.1.222"
|
|
"2620:11f:7001:7:ffff:ffff:0ad7:01de"
|
|
];
|
|
"external_ips" = [
|
|
"2620:11f:7001:7:ffff:ffff:0ad7:01de"
|
|
];
|
|
};
|
|
"controller2" = {
|
|
"internal_ips" = [
|
|
"10.215.1.223"
|
|
"2620:11f:7001:7:ffff:ffff:0ad7:01df"
|
|
];
|
|
"external_ips" = [
|
|
"2620:11f:7001:7:ffff:ffff:0ad7:01df"
|
|
];
|
|
};
|
|
};
|
|
_vm_name_to_hostname = {
|
|
"nc0" = "controller0";
|
|
"nc1" = "controller1";
|
|
"nc2" = "controller2";
|
|
};
|
|
vm_name_to_hostname = (vm_name: _vm_name_to_hostname."${vm_name}");
|
|
in
|
|
makeScope newScope (
|
|
self:
|
|
let
|
|
additional_vars = {
|
|
inherit all_hostnames controllers;
|
|
k8s = self;
|
|
};
|
|
deploy_file = (
|
|
{
|
|
dest_dir,
|
|
file,
|
|
name ? (builtins.baseNameOf file),
|
|
owner,
|
|
group,
|
|
mode,
|
|
}:
|
|
''
|
|
##
|
|
## deploy ${name} to ${dest_dir}
|
|
##
|
|
${openssh}/bin/ssh mrmanager doas rm -f ${dest_dir}/${name} ~/${name}
|
|
${openssh}/bin/scp ${file} mrmanager:~/${name}
|
|
${openssh}/bin/ssh mrmanager doas install -o ${toString owner} -g ${toString group} -m ${mode} ~/${name} ${dest_dir}/${name}
|
|
${openssh}/bin/ssh mrmanager doas rm -f ~/${name}
|
|
|
|
|
|
''
|
|
);
|
|
deploy_machine = (
|
|
vm_name:
|
|
(
|
|
''
|
|
##
|
|
## Create directories on ${vm_name}
|
|
##
|
|
${openssh}/bin/ssh mrmanager doas install -d -o 11235 -g 11235 -m 0755 /vm/${vm_name}/persist/keys
|
|
${openssh}/bin/ssh mrmanager doas install -d -o 10016 -g 10016 -m 0755 /vm/${vm_name}/persist/keys/etcd
|
|
${openssh}/bin/ssh mrmanager doas install -d -o 10024 -g 10024 -m 0755 /vm/${vm_name}/persist/keys/kube
|
|
|
|
|
|
''
|
|
+ (lib.concatMapStringsSep "\n" deploy_file [
|
|
{
|
|
dest_dir = "/vm/${vm_name}/persist/keys/etcd";
|
|
file = "${self.kubernetes}/kubernetes.pem";
|
|
owner = 10016;
|
|
group = 10016;
|
|
mode = "0640";
|
|
}
|
|
{
|
|
dest_dir = "/vm/${vm_name}/persist/keys/etcd";
|
|
file = "${self.kubernetes}/kubernetes-key.pem";
|
|
owner = 10016;
|
|
group = 10016;
|
|
mode = "0600";
|
|
}
|
|
{
|
|
dest_dir = "/vm/${vm_name}/persist/keys/etcd";
|
|
file = "${self.ca}/ca.pem";
|
|
owner = 10016;
|
|
group = 10016;
|
|
mode = "0640";
|
|
}
|
|
{
|
|
dest_dir = "/vm/${vm_name}/persist/keys/kube";
|
|
file = "${self.kubernetes}/kubernetes.pem";
|
|
owner = 10024;
|
|
group = 10024;
|
|
mode = "0640";
|
|
}
|
|
{
|
|
dest_dir = "/vm/${vm_name}/persist/keys/kube";
|
|
file = "${self.kubernetes}/kubernetes-key.pem";
|
|
owner = 10024;
|
|
group = 10024;
|
|
mode = "0640";
|
|
}
|
|
{
|
|
dest_dir = "/vm/${vm_name}/persist/keys/kube";
|
|
file = "${self.ca}/ca.pem";
|
|
owner = 10024;
|
|
group = 10024;
|
|
mode = "0600";
|
|
}
|
|
{
|
|
dest_dir = "/vm/${vm_name}/persist/keys/kube";
|
|
file = (writeText "encryption-config.yaml" (lib.generators.toYAML { } kube_encryption_config));
|
|
name = "encryption-config.yaml";
|
|
owner = 10024;
|
|
group = 10024;
|
|
mode = "0600";
|
|
}
|
|
{
|
|
dest_dir = "/vm/${vm_name}/persist/keys/kube";
|
|
file = "${self.service_account}/service-account.pem";
|
|
owner = 10024;
|
|
group = 10024;
|
|
mode = "0600";
|
|
}
|
|
{
|
|
dest_dir = "/vm/${vm_name}/persist/keys/kube";
|
|
file = "${self.service_account}/service-account-key.pem";
|
|
owner = 10024;
|
|
group = 10024;
|
|
mode = "0600";
|
|
}
|
|
{
|
|
dest_dir = "/vm/${vm_name}/persist/keys/kube";
|
|
file = "${self.requestheader-client-ca}/requestheader-client-ca.pem";
|
|
owner = 10024;
|
|
group = 10024;
|
|
mode = "0600";
|
|
}
|
|
{
|
|
dest_dir = "/vm/${vm_name}/persist/keys/kube";
|
|
file = "${self.controller-proxy}/${vm_name_to_hostname vm_name}-proxy.pem";
|
|
owner = 10024;
|
|
group = 10024;
|
|
mode = "0600";
|
|
}
|
|
{
|
|
dest_dir = "/vm/${vm_name}/persist/keys/kube";
|
|
file = "${self.controller-proxy}/${vm_name_to_hostname vm_name}-proxy-key.pem";
|
|
owner = 10024;
|
|
group = 10024;
|
|
mode = "0600";
|
|
}
|
|
])
|
|
)
|
|
);
|
|
deploy_script = (
|
|
''
|
|
set -euo pipefail
|
|
IFS=$'\n\t'
|
|
DIR="$( cd "$( dirname "''${BASH_SOURCE[0]}" )" && pwd )"
|
|
''
|
|
+ (lib.concatMapStringsSep "\n" deploy_machine [
|
|
"nc0"
|
|
"nc1"
|
|
"nc2"
|
|
])
|
|
);
|
|
kube_encryption_key = runCommand "kube_encryption_key" { } ''
|
|
head -c 32 /dev/urandom | base64 | tee $out
|
|
'';
|
|
kube_encryption_config = {
|
|
kind = "EncryptionConfig";
|
|
apiVersion = "v1";
|
|
resources = [
|
|
{
|
|
resources = [ "secrets" ];
|
|
providers = [
|
|
{
|
|
aescbc = {
|
|
keys = [
|
|
{
|
|
name = "key1";
|
|
secret = (builtins.readFile "${kube_encryption_key}");
|
|
}
|
|
];
|
|
};
|
|
}
|
|
{ identity = { }; }
|
|
];
|
|
}
|
|
];
|
|
};
|
|
in
|
|
{
|
|
ca = (callPackage ./package/k8s-ca/package.nix additional_vars);
|
|
keys = (
|
|
lib.genAttrs [
|
|
"admin"
|
|
"controller0"
|
|
"controller1"
|
|
"controller2"
|
|
"worker0"
|
|
"worker1"
|
|
"worker2"
|
|
"kube-proxy"
|
|
"kube-scheduler"
|
|
"kube-controller-manager"
|
|
"kube-api-server"
|
|
"service-accounts"
|
|
] (key_name: (callPackage ./package/tls-key/package.nix (additional_vars // { inherit key_name; })))
|
|
);
|
|
client-configs = (
|
|
builtins.mapAttrs
|
|
(
|
|
config_name: config:
|
|
(callPackage ./package/k8s-client-config/package.nix (
|
|
additional_vars // { inherit config_name; } // config
|
|
))
|
|
)
|
|
{
|
|
controller0 = {
|
|
config_user = "system:node:controller0";
|
|
config_server = "https://server.kubernetes.local:6443";
|
|
};
|
|
controller1 = {
|
|
config_user = "system:node:controller1";
|
|
config_server = "https://server.kubernetes.local:6443";
|
|
};
|
|
controller2 = {
|
|
config_user = "system:node:controller2";
|
|
config_server = "https://server.kubernetes.local:6443";
|
|
};
|
|
worker0 = {
|
|
config_user = "system:node:worker0";
|
|
config_server = "https://server.kubernetes.local:6443";
|
|
};
|
|
worker1 = {
|
|
config_user = "system:node:worker1";
|
|
config_server = "https://server.kubernetes.local:6443";
|
|
};
|
|
worker2 = {
|
|
config_user = "system:node:worker2";
|
|
config_server = "https://server.kubernetes.local:6443";
|
|
};
|
|
kube-proxy = {
|
|
config_user = "system:kube-proxy";
|
|
config_server = "https://server.kubernetes.local:6443";
|
|
};
|
|
kube-controller-manager = {
|
|
config_user = "system:kube-controller-manager";
|
|
config_server = "https://server.kubernetes.local:6443";
|
|
};
|
|
kube-scheduler = {
|
|
config_user = "system:kube-scheduler";
|
|
config_server = "https://server.kubernetes.local:6443";
|
|
};
|
|
admin = {
|
|
config_user = "admin";
|
|
config_server = "https://127.0.0.1:6443";
|
|
};
|
|
}
|
|
);
|
|
all_keys = (callPackage ./package/k8s-keys/package.nix additional_vars);
|
|
deploy_script = (callPackage ./package/deploy-script/package.nix additional_vars);
|
|
}
|
|
)
|