diff --git a/nix/kubernetes/keys/package/deploy-script/package.nix b/nix/kubernetes/keys/package/deploy-script/package.nix index dfe85edd..7e21d75f 100644 --- a/nix/kubernetes/keys/package/deploy-script/package.nix +++ b/nix/kubernetes/keys/package/deploy-script/package.nix @@ -8,14 +8,150 @@ # installCheckPhase # distPhase { + lib, stdenv, writeShellScript, k8s, + openssh, ... }: let - deploy_script_body = ""; + deploy_script_body = ( + '' + set -euo pipefail + IFS=$'\n\t' + DIR="$( cd "$( dirname "''${BASH_SOURCE[0]}" )" && pwd )" + '' + + (lib.concatMapStringsSep "\n" deploy_machine [ + "nc0" + "nc1" + "nc2" + ]) + ); deploy_script = (writeShellScript "deploy-script" deploy_script_body); + deploy_file = ( + { + dest_dir, + file, + name ? (builtins.baseNameOf file), + owner, + group, + mode, + }: + '' + ## + ## deploy ${name} to ${dest_dir} + ## + ${openssh}/bin/ssh mrmanager doas rm -f ${dest_dir}/${name} ~/${name} + ${openssh}/bin/scp ${file} mrmanager:~/${name} + ${openssh}/bin/ssh mrmanager doas install -o ${toString owner} -g ${toString group} -m ${mode} ~/${name} ${dest_dir}/${name} + ${openssh}/bin/ssh mrmanager doas rm -f ~/${name} + + + '' + ); + deploy_machine = ( + vm_name: + ( + '' + ## + ## Create directories on ${vm_name} + ## + ${openssh}/bin/ssh mrmanager doas install -d -o 11235 -g 11235 -m 0755 /vm/${vm_name}/persist/keys + ${openssh}/bin/ssh mrmanager doas install -d -o 10016 -g 10016 -m 0755 /vm/${vm_name}/persist/keys/etcd + ${openssh}/bin/ssh mrmanager doas install -d -o 10024 -g 10024 -m 0755 /vm/${vm_name}/persist/keys/kube + + + '' + + (lib.concatMapStringsSep "\n" deploy_file [ + { + dest_dir = "/vm/${vm_name}/persist/keys/etcd"; + file = "${k8s.keys.kube-api-server}/kube-api-server.crt"; + owner = 10016; + group = 10016; + mode = "0640"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/etcd"; + file = "${k8s.keys.kube-api-server}/kube-api-server.key"; + owner = 10016; + group = 10016; + mode = "0600"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/etcd"; + file = "${k8s.ca}/ca.crt"; + owner = 10016; + group = 10016; + mode = "0640"; + } + # { + # dest_dir = "/vm/${vm_name}/persist/keys/kube"; + # file = "${self.kubernetes}/kubernetes.pem"; + # owner = 10024; + # group = 10024; + # mode = "0640"; + # } + # { + # dest_dir = "/vm/${vm_name}/persist/keys/kube"; + # file = "${self.kubernetes}/kubernetes-key.pem"; + # owner = 10024; + # group = 10024; + # mode = "0640"; + # } + # { + # dest_dir = "/vm/${vm_name}/persist/keys/kube"; + # file = "${self.ca}/ca.pem"; + # owner = 10024; + # group = 10024; + # mode = "0600"; + # } + # { + # dest_dir = "/vm/${vm_name}/persist/keys/kube"; + # file = (writeText "encryption-config.yaml" (lib.generators.toYAML { } kube_encryption_config)); + # name = "encryption-config.yaml"; + # owner = 10024; + # group = 10024; + # mode = "0600"; + # } + # { + # dest_dir = "/vm/${vm_name}/persist/keys/kube"; + # file = "${self.service_account}/service-account.pem"; + # owner = 10024; + # group = 10024; + # mode = "0600"; + # } + # { + # dest_dir = "/vm/${vm_name}/persist/keys/kube"; + # file = "${self.service_account}/service-account-key.pem"; + # owner = 10024; + # group = 10024; + # mode = "0600"; + # } + # { + # dest_dir = "/vm/${vm_name}/persist/keys/kube"; + # file = "${self.requestheader-client-ca}/requestheader-client-ca.pem"; + # owner = 10024; + # group = 10024; + # mode = "0600"; + # } + # { + # dest_dir = "/vm/${vm_name}/persist/keys/kube"; + # file = "${self.controller-proxy}/${vm_name_to_hostname vm_name}-proxy.pem"; + # owner = 10024; + # group = 10024; + # mode = "0600"; + # } + # { + # dest_dir = "/vm/${vm_name}/persist/keys/kube"; + # file = "${self.controller-proxy}/${vm_name_to_hostname vm_name}-proxy-key.pem"; + # owner = 10024; + # group = 10024; + # mode = "0600"; + # } + ]) + ) + ); in stdenv.mkDerivation (finalAttrs: { name = "deploy-script"; diff --git a/nix/kubernetes/keys/package/k8s-ca/files/ca.conf b/nix/kubernetes/keys/package/k8s-ca/files/ca.conf index 3c7021c7..b26ddbf4 100644 --- a/nix/kubernetes/keys/package/k8s-ca/files/ca.conf +++ b/nix/kubernetes/keys/package/k8s-ca/files/ca.conf @@ -266,7 +266,19 @@ subjectKeyIdentifier = hash [kube-api-server_alt_names] IP.0 = 127.0.0.1 -IP.1 = 10.32.0.1 +IP.1 = 10.0.0.1 +IP.2 = 10.215.1.221 +IP.3 = 2620:11f:7001:7:ffff:ffff:0ad7:01dd +IP.4 = 10.215.1.222 +IP.5 = 2620:11f:7001:7:ffff:ffff:0ad7:01de +IP.6 = 10.215.1.223 +IP.7 = 2620:11f:7001:7:ffff:ffff:0ad7:01df +IP.8 = 10.215.1.224 +IP.9 = 2620:11f:7001:7:ffff:ffff:0ad7:01e0 +IP.10 = 10.215.1.225 +IP.11 = 2620:11f:7001:7:ffff:ffff:0ad7:01e1 +IP.12 = 10.215.1.226 +IP.13 = 2620:11f:7001:7:ffff:ffff:0ad7:01e2 DNS.0 = kubernetes DNS.1 = kubernetes.default DNS.2 = kubernetes.default.svc diff --git a/nix/kubernetes/keys/package/k8s-encryption-key/package.nix b/nix/kubernetes/keys/package/k8s-encryption-key/package.nix new file mode 100644 index 00000000..1f88575e --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-encryption-key/package.nix @@ -0,0 +1,58 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + lib, + stdenv, + runCommand, + writeText, + ... +}: +let + kube_encryption_key = runCommand "kube_encryption_key" { } '' + head -c 32 /dev/urandom | base64 | tee $out + ''; + kube_encryption_config = { + kind = "EncryptionConfig"; + apiVersion = "v1"; + resources = [ + { + resources = [ "secrets" ]; + providers = [ + { + aescbc = { + keys = [ + { + name = "key1"; + secret = (builtins.readFile "${kube_encryption_key}"); + } + ]; + }; + } + { identity = { }; } + ]; + } + ]; + }; + kube_encryption_config_yaml = ( + writeText "encryption-config.yaml" (lib.generators.toYAML { } kube_encryption_config) + ); +in +stdenv.mkDerivation (finalAttrs: { + name = "k8s-encryption-key"; + nativeBuildInputs = [ ]; + buildInputs = [ ]; + + unpackPhase = "true"; + + installPhase = '' + mkdir "$out" + cp "${kube_encryption_config_yaml}" $out/encryption-config.yaml + ''; +}) diff --git a/nix/kubernetes/keys/package/k8s-keys/package.nix b/nix/kubernetes/keys/package/k8s-keys/package.nix index fd858828..05bc44e7 100644 --- a/nix/kubernetes/keys/package/k8s-keys/package.nix +++ b/nix/kubernetes/keys/package/k8s-keys/package.nix @@ -7,6 +7,7 @@ symlinkJoin { name = "k8s-keys"; paths = [ k8s.ca + k8s.encryption_config ] ++ (builtins.attrValues k8s.keys) ++ (builtins.attrValues k8s.client-configs); diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index 4e32a2dd..785653f4 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -2,10 +2,6 @@ makeScope, newScope, callPackage, - writeShellScript, - openssh, - runCommand, - writeText, lib, }: let @@ -73,12 +69,12 @@ let ]; }; }; - _vm_name_to_hostname = { - "nc0" = "controller0"; - "nc1" = "controller1"; - "nc2" = "controller2"; - }; - vm_name_to_hostname = (vm_name: _vm_name_to_hostname."${vm_name}"); + # _vm_name_to_hostname = { + # "nc0" = "controller0"; + # "nc1" = "controller1"; + # "nc2" = "controller2"; + # }; + # vm_name_to_hostname = (vm_name: _vm_name_to_hostname."${vm_name}"); in makeScope newScope ( self: @@ -87,166 +83,6 @@ makeScope newScope ( inherit all_hostnames controllers; k8s = self; }; - deploy_file = ( - { - dest_dir, - file, - name ? (builtins.baseNameOf file), - owner, - group, - mode, - }: - '' - ## - ## deploy ${name} to ${dest_dir} - ## - ${openssh}/bin/ssh mrmanager doas rm -f ${dest_dir}/${name} ~/${name} - ${openssh}/bin/scp ${file} mrmanager:~/${name} - ${openssh}/bin/ssh mrmanager doas install -o ${toString owner} -g ${toString group} -m ${mode} ~/${name} ${dest_dir}/${name} - ${openssh}/bin/ssh mrmanager doas rm -f ~/${name} - - - '' - ); - deploy_machine = ( - vm_name: - ( - '' - ## - ## Create directories on ${vm_name} - ## - ${openssh}/bin/ssh mrmanager doas install -d -o 11235 -g 11235 -m 0755 /vm/${vm_name}/persist/keys - ${openssh}/bin/ssh mrmanager doas install -d -o 10016 -g 10016 -m 0755 /vm/${vm_name}/persist/keys/etcd - ${openssh}/bin/ssh mrmanager doas install -d -o 10024 -g 10024 -m 0755 /vm/${vm_name}/persist/keys/kube - - - '' - + (lib.concatMapStringsSep "\n" deploy_file [ - { - dest_dir = "/vm/${vm_name}/persist/keys/etcd"; - file = "${self.kubernetes}/kubernetes.pem"; - owner = 10016; - group = 10016; - mode = "0640"; - } - { - dest_dir = "/vm/${vm_name}/persist/keys/etcd"; - file = "${self.kubernetes}/kubernetes-key.pem"; - owner = 10016; - group = 10016; - mode = "0600"; - } - { - dest_dir = "/vm/${vm_name}/persist/keys/etcd"; - file = "${self.ca}/ca.pem"; - owner = 10016; - group = 10016; - mode = "0640"; - } - { - dest_dir = "/vm/${vm_name}/persist/keys/kube"; - file = "${self.kubernetes}/kubernetes.pem"; - owner = 10024; - group = 10024; - mode = "0640"; - } - { - dest_dir = "/vm/${vm_name}/persist/keys/kube"; - file = "${self.kubernetes}/kubernetes-key.pem"; - owner = 10024; - group = 10024; - mode = "0640"; - } - { - dest_dir = "/vm/${vm_name}/persist/keys/kube"; - file = "${self.ca}/ca.pem"; - owner = 10024; - group = 10024; - mode = "0600"; - } - { - dest_dir = "/vm/${vm_name}/persist/keys/kube"; - file = (writeText "encryption-config.yaml" (lib.generators.toYAML { } kube_encryption_config)); - name = "encryption-config.yaml"; - owner = 10024; - group = 10024; - mode = "0600"; - } - { - dest_dir = "/vm/${vm_name}/persist/keys/kube"; - file = "${self.service_account}/service-account.pem"; - owner = 10024; - group = 10024; - mode = "0600"; - } - { - dest_dir = "/vm/${vm_name}/persist/keys/kube"; - file = "${self.service_account}/service-account-key.pem"; - owner = 10024; - group = 10024; - mode = "0600"; - } - { - dest_dir = "/vm/${vm_name}/persist/keys/kube"; - file = "${self.requestheader-client-ca}/requestheader-client-ca.pem"; - owner = 10024; - group = 10024; - mode = "0600"; - } - { - dest_dir = "/vm/${vm_name}/persist/keys/kube"; - file = "${self.controller-proxy}/${vm_name_to_hostname vm_name}-proxy.pem"; - owner = 10024; - group = 10024; - mode = "0600"; - } - { - dest_dir = "/vm/${vm_name}/persist/keys/kube"; - file = "${self.controller-proxy}/${vm_name_to_hostname vm_name}-proxy-key.pem"; - owner = 10024; - group = 10024; - mode = "0600"; - } - ]) - ) - ); - deploy_script = ( - '' - set -euo pipefail - IFS=$'\n\t' - DIR="$( cd "$( dirname "''${BASH_SOURCE[0]}" )" && pwd )" - '' - + (lib.concatMapStringsSep "\n" deploy_machine [ - "nc0" - "nc1" - "nc2" - ]) - ); - kube_encryption_key = runCommand "kube_encryption_key" { } '' - head -c 32 /dev/urandom | base64 | tee $out - ''; - kube_encryption_config = { - kind = "EncryptionConfig"; - apiVersion = "v1"; - resources = [ - { - resources = [ "secrets" ]; - providers = [ - { - aescbc = { - keys = [ - { - name = "key1"; - secret = (builtins.readFile "${kube_encryption_key}"); - } - ]; - }; - } - { identity = { }; } - ]; - } - ]; - }; in { ca = (callPackage ./package/k8s-ca/package.nix additional_vars); @@ -317,6 +153,7 @@ makeScope newScope ( }; } ); + encryption_config = (callPackage ./package/k8s-encryption-key/package.nix additional_vars); all_keys = (callPackage ./package/k8s-keys/package.nix additional_vars); deploy_script = (callPackage ./package/deploy-script/package.nix additional_vars); } diff --git a/nix/kubernetes/roles/etcd/default.nix b/nix/kubernetes/roles/etcd/default.nix index f0044d0a..c3de9202 100644 --- a/nix/kubernetes/roles/etcd/default.nix +++ b/nix/kubernetes/roles/etcd/default.nix @@ -55,12 +55,12 @@ enable = true; openFirewall = true; name = config.networking.hostName; - certFile = "/.persist/keys/etcd/kubernetes.pem"; - keyFile = "/.persist/keys/etcd/kubernetes-key.pem"; - peerCertFile = "/.persist/keys/etcd/kubernetes.pem"; - peerKeyFile = "/.persist/keys/etcd/kubernetes-key.pem"; - trustedCaFile = "/.persist/keys/etcd/ca.pem"; - peerTrustedCaFile = "/.persist/keys/etcd/ca.pem"; + certFile = "/.persist/keys/etcd/kube-api-server.crt"; + keyFile = "/.persist/keys/etcd/kube-api-server.key"; + peerCertFile = "/.persist/keys/etcd/kube-api-server.crt"; + peerKeyFile = "/.persist/keys/etcd/kube-api-server.key"; + trustedCaFile = "/.persist/keys/etcd/ca.crt"; + peerTrustedCaFile = "/.persist/keys/etcd/ca.crt"; peerClientCertAuth = true; clientCertAuth = true; initialAdvertisePeerUrls = (