diff --git a/nix/kubernetes/README.org b/nix/kubernetes/README.org index b681dea7..9437c61a 100644 --- a/nix/kubernetes/README.org +++ b/nix/kubernetes/README.org @@ -16,7 +16,7 @@ | Pod | 10.200.0.0/16 | 2620:11f:7001:7:ffff:eeee::/96 | | Service | 10.197.0.0/16 | fd00:3e42:e349::/112 | | Node | 10.215.1.0/24 | 2620:11f:7001:7:ffff:ffff:0ad7:0100/120 | -| Load Balancer | 74.80.180.139-74.80.180.142 | 2620:11f:7001:7:ffff:dddd::/96 | +| Load Balancer | 74.80.180.139-74.80.180.142 | 2620:11f:7001:7:ffff:dddd::/96 | * Healthcheck ** Check cilium status #+begin_src bash diff --git a/nix/kubernetes/keys/package/deploy-script/package.nix b/nix/kubernetes/keys/package/deploy-script/package.nix index e809d9a7..78fff16d 100644 --- a/nix/kubernetes/keys/package/deploy-script/package.nix +++ b/nix/kubernetes/keys/package/deploy-script/package.nix @@ -106,21 +106,21 @@ let } { dest_dir = "/vm/${vm_name}/persist/keys/etcd"; - file = "${k8s.ca}/ca.crt"; + file = "${k8s.ca.client}/client-ca.crt"; owner = 10016; group = 10016; mode = "0640"; } { dest_dir = "/vm/${vm_name}/persist/keys/kube"; - file = "${k8s.ca}/ca.crt"; + file = "${k8s.ca.client}/client-ca.crt"; owner = 10024; group = 10024; mode = "0640"; } { dest_dir = "/vm/${vm_name}/persist/keys/kube"; - file = "${k8s.ca}/ca.key"; + file = "${k8s.ca.client}/client-ca.key"; owner = 10024; group = 10024; mode = "0600"; @@ -175,6 +175,33 @@ let group = 10024; mode = "0600"; } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.ca.requestheader-client}/requestheader-client-ca.crt"; + owner = 10024; + group = 10024; + mode = "0640"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${ + k8s.keys."${vm_name_to_hostname vm_name}-proxy" + }/${vm_name_to_hostname vm_name}-proxy.crt"; + name = "proxy.crt"; + owner = 10024; + group = 10024; + mode = "0640"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${ + k8s.keys."${vm_name_to_hostname vm_name}-proxy" + }/${vm_name_to_hostname vm_name}-proxy.key"; + name = "proxy.key"; + owner = 10024; + group = 10024; + mode = "0600"; + } ]) ) ); @@ -193,7 +220,7 @@ let + (lib.concatMapStringsSep "\n" deploy_file [ { dest_dir = "/vm/${vm_name}/persist/keys/kube"; - file = "${k8s.ca}/ca.crt"; + file = "${k8s.ca.client}/client-ca.crt"; owner = 10024; group = 10024; mode = "0640"; diff --git a/nix/kubernetes/keys/package/k8s-ca/files/ca.conf b/nix/kubernetes/keys/package/k8s-ca/files/client-ca.conf similarity index 100% rename from nix/kubernetes/keys/package/k8s-ca/files/ca.conf rename to nix/kubernetes/keys/package/k8s-ca/files/client-ca.conf diff --git a/nix/kubernetes/keys/package/k8s-ca/files/requestheader-client-ca.conf b/nix/kubernetes/keys/package/k8s-ca/files/requestheader-client-ca.conf new file mode 100644 index 00000000..f31e75c7 --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-ca/files/requestheader-client-ca.conf @@ -0,0 +1,95 @@ +[req] +distinguished_name = req_distinguished_name +prompt = no +x509_extensions = ca_x509_extensions + +[ca_x509_extensions] +basicConstraints = CA:TRUE +keyUsage = cRLSign, keyCertSign + +[req_distinguished_name] +C = US +ST = Washington +L = Seattle +CN = CA + +[controller0-proxy] +distinguished_name = controller0_distinguished_name +prompt = no +req_extensions = controller0_req_extensions + +[controller0_req_extensions] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth, serverAuth +keyUsage = critical, digitalSignature, keyEncipherment +nsCertType = client +nsComment = "controller0 Certificate" +subjectAltName = @controller0_alt_names +subjectKeyIdentifier = hash + +[controller0_distinguished_name] +CN = system:node:controller0 +O = system:nodes +C = US +ST = Washington +L = Seattle + +[controller0_alt_names] +IP.0 = 127.0.0.1 +IP.4 = 10.215.1.221 +IP.5 = 2620:11f:7001:7:ffff:ffff:0ad7:01dd +DNS.0 = controller0 + +[controller1-proxy] +distinguished_name = controller1_distinguished_name +prompt = no +req_extensions = controller1_req_extensions + +[controller1_req_extensions] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth, serverAuth +keyUsage = critical, digitalSignature, keyEncipherment +nsCertType = client +nsComment = "controller1 Certificate" +subjectAltName = @controller1_alt_names +subjectKeyIdentifier = hash + +[controller1_distinguished_name] +CN = system:node:controller1 +O = system:nodes +C = US +ST = Washington +L = Seattle + +[controller1_alt_names] +IP.0 = 127.0.0.1 +IP.4 = 10.215.1.222 +IP.5 = 2620:11f:7001:7:ffff:ffff:0ad7:01de +DNS.0 = controller1 + +[controller2-proxy] +distinguished_name = controller2_distinguished_name +prompt = no +req_extensions = controller2_req_extensions + +[controller2_req_extensions] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth, serverAuth +keyUsage = critical, digitalSignature, keyEncipherment +nsCertType = client +nsComment = "controller2 Certificate" +subjectAltName = @controller2_alt_names +subjectKeyIdentifier = hash + +[controller2_distinguished_name] +CN = system:node:controller2 +O = system:nodes +C = US +ST = Washington +L = Seattle + +[controller2_alt_names] +IP.0 = 127.0.0.1 +IP.6 = 10.215.1.223 +IP.7 = 2620:11f:7001:7:ffff:ffff:0ad7:01df +DNS.0 = controller2 diff --git a/nix/kubernetes/keys/package/k8s-ca/package.nix b/nix/kubernetes/keys/package/k8s-ca/package.nix index eed2a777..6c9329b3 100644 --- a/nix/kubernetes/keys/package/k8s-ca/package.nix +++ b/nix/kubernetes/keys/package/k8s-ca/package.nix @@ -10,23 +10,28 @@ { stdenv, openssl, + ca_name, + ca_config, ... }: stdenv.mkDerivation (finalAttrs: { - name = "k8s-ca"; + name = "k8s-ca-${ca_name}"; nativeBuildInputs = [ openssl ]; buildInputs = [ ]; unpackPhase = "true"; - installPhase = '' - mkdir -p "$out" - cd "$out" + buildPhase = '' + openssl genrsa -out "${ca_name}-ca.key" 4096 - openssl genrsa -out ca.key 4096 openssl req -x509 -new -sha512 -noenc \ - -key ca.key -days 3653 \ - -config ${./files/ca.conf} \ - -out ca.crt + -key "${ca_name}-ca.key" -days 3653 \ + -config "${ca_config}" \ + -out "${ca_name}-ca.crt" + ''; + + installPhase = '' + mkdir "$out" + cp "${ca_name}-ca.crt" "${ca_name}-ca.key" $out/ ''; }) diff --git a/nix/kubernetes/keys/package/k8s-client-config/package.nix b/nix/kubernetes/keys/package/k8s-client-config/package.nix index 2d4ac886..853c3df8 100644 --- a/nix/kubernetes/keys/package/k8s-client-config/package.nix +++ b/nix/kubernetes/keys/package/k8s-client-config/package.nix @@ -26,7 +26,7 @@ stdenv.mkDerivation (finalAttrs: { buildPhase = '' kubectl config set-cluster kubernetes-the-hard-way \ - --certificate-authority=${k8s.ca}/ca.crt \ + --certificate-authority=${k8s.ca.client}/client-ca.crt \ --embed-certs=true \ --server=${lib.strings.escapeShellArg config_server} \ --kubeconfig=${config_name}.kubeconfig diff --git a/nix/kubernetes/keys/package/k8s-keys/package.nix b/nix/kubernetes/keys/package/k8s-keys/package.nix index 3c62aece..71596b87 100644 --- a/nix/kubernetes/keys/package/k8s-keys/package.nix +++ b/nix/kubernetes/keys/package/k8s-keys/package.nix @@ -15,9 +15,9 @@ symlinkJoin { name = "k8s-keys"; paths = [ scripts - k8s.ca k8s.encryption_config ] + ++ (builtins.attrValues k8s.ca) ++ (builtins.attrValues k8s.keys) ++ (builtins.attrValues k8s.client-configs) ++ (builtins.attrValues k8s.ssh-keys) diff --git a/nix/kubernetes/keys/package/tls-key/package.nix b/nix/kubernetes/keys/package/tls-key/package.nix index 437099e1..5daee362 100644 --- a/nix/kubernetes/keys/package/tls-key/package.nix +++ b/nix/kubernetes/keys/package/tls-key/package.nix @@ -12,6 +12,8 @@ openssl, k8s, key_name, + ca_name, + ca_config, ... }: stdenv.mkDerivation (finalAttrs: { @@ -22,18 +24,18 @@ stdenv.mkDerivation (finalAttrs: { unpackPhase = "true"; buildPhase = '' - cp ${k8s.ca}/ca.crt ${k8s.ca}/ca.key ./ + cp ${k8s.ca."${ca_name}"}/${ca_name}-ca.crt ${k8s.ca."${ca_name}"}/${ca_name}-ca.key ./ openssl genrsa -out "${key_name}.key" 4096 openssl req -new -key "${key_name}.key" -sha256 \ - -config "${../k8s-ca/files/ca.conf}" -section ${key_name} \ + -config "${ca_config}" -section ${key_name} \ -out "${key_name}.csr" openssl x509 -req -days 3653 -in "${key_name}.csr" \ -copy_extensions copyall \ - -sha256 -CA "./ca.crt" \ - -CAkey "./ca.key" \ + -sha256 -CA "./${ca_name}-ca.crt" \ + -CAkey "./${ca_name}-ca.key" \ -CAcreateserial \ -out "${key_name}.crt" ''; diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index 1414f6e2..d5b970ce 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -78,24 +78,48 @@ makeScope newScope ( inherit all_hostnames controllers; k8s = self; }; + certificate_authorities = { + "client" = { + ca_config = ./package/k8s-ca/files/client-ca.conf; + }; + "requestheader-client" = { + ca_config = ./package/k8s-ca/files/requestheader-client-ca.conf; + }; + }; + certificate_authorities_merged = ( + builtins.mapAttrs (ca_name: ca_config: { inherit ca_name; } // ca_config) certificate_authorities + ); in { - ca = (callPackage ./package/k8s-ca/package.nix additional_vars); + ca = ( + builtins.mapAttrs ( + ca_name: ca_config: + (callPackage ./package/k8s-ca/package.nix (additional_vars // { inherit ca_name; } // ca_config)) + ) certificate_authorities + ); keys = ( - lib.genAttrs [ - "admin" - "controller0" - "controller1" - "controller2" - "worker0" - "worker1" - "worker2" - "kube-proxy" - "kube-scheduler" - "kube-controller-manager" - "kube-api-server" - "service-accounts" - ] (key_name: (callPackage ./package/tls-key/package.nix (additional_vars // { inherit key_name; }))) + builtins.mapAttrs + ( + key_name: key_config: + (callPackage ./package/tls-key/package.nix (additional_vars // { inherit key_name; } // key_config)) + ) + { + "admin" = { } // certificate_authorities_merged.client; + "controller0" = { } // certificate_authorities_merged.client; + "controller1" = { } // certificate_authorities_merged.client; + "controller2" = { } // certificate_authorities_merged.client; + "worker0" = { } // certificate_authorities_merged.client; + "worker1" = { } // certificate_authorities_merged.client; + "worker2" = { } // certificate_authorities_merged.client; + "kube-proxy" = { } // certificate_authorities_merged.client; + "kube-scheduler" = { } // certificate_authorities_merged.client; + "kube-controller-manager" = { } // certificate_authorities_merged.client; + "kube-api-server" = { } // certificate_authorities_merged.client; + "service-accounts" = { } // certificate_authorities_merged.client; + "controller0-proxy" = { } // certificate_authorities_merged.requestheader-client; + "controller1-proxy" = { } // certificate_authorities_merged.requestheader-client; + "controller2-proxy" = { } // certificate_authorities_merged.requestheader-client; + } ); ssh-keys = ( lib.genAttrs [ diff --git a/nix/kubernetes/roles/etcd/default.nix b/nix/kubernetes/roles/etcd/default.nix index 4ebc159c..f0863157 100644 --- a/nix/kubernetes/roles/etcd/default.nix +++ b/nix/kubernetes/roles/etcd/default.nix @@ -59,8 +59,8 @@ keyFile = "/.persist/keys/etcd/kube-api-server.key"; peerCertFile = "/.persist/keys/etcd/kube-api-server.crt"; peerKeyFile = "/.persist/keys/etcd/kube-api-server.key"; - trustedCaFile = "/.persist/keys/etcd/ca.crt"; - peerTrustedCaFile = "/.persist/keys/etcd/ca.crt"; + trustedCaFile = "/.persist/keys/etcd/client-ca.crt"; + peerTrustedCaFile = "/.persist/keys/etcd/client-ca.crt"; peerClientCertAuth = true; clientCertAuth = true; initialAdvertisePeerUrls = ( diff --git a/nix/kubernetes/roles/kube_apiserver/default.nix b/nix/kubernetes/roles/kube_apiserver/default.nix index 1c2edf78..267f642f 100644 --- a/nix/kubernetes/roles/kube_apiserver/default.nix +++ b/nix/kubernetes/roles/kube_apiserver/default.nix @@ -65,15 +65,15 @@ in "--audit-log-path=/var/log/audit.log" "--authorization-mode=Node,RBAC" "--bind-address=0.0.0.0" - "--client-ca-file=/.persist/keys/kube/ca.crt" + "--client-ca-file=/.persist/keys/kube/client-ca.crt" "--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota" - "--etcd-cafile=/.persist/keys/kube/ca.crt" + "--etcd-cafile=/.persist/keys/kube/client-ca.crt" "--etcd-certfile=/.persist/keys/kube/kube-api-server.crt" "--etcd-keyfile=/.persist/keys/kube/kube-api-server.key" "--etcd-servers=${builtins.concatStringsSep "," config.me.kube_apiserver.etcd_services}" "--event-ttl=1h" "--encryption-provider-config=/.persist/keys/kube/encryption-config.yaml" - "--kubelet-certificate-authority=/.persist/keys/kube/ca.crt" + "--kubelet-certificate-authority=/.persist/keys/kube/client-ca.crt" "--kubelet-client-certificate=/.persist/keys/kube/kube-api-server.crt" "--kubelet-client-key=/.persist/keys/kube/kube-api-server.key" "--runtime-config='api/all=true'" @@ -85,6 +85,14 @@ in "--tls-private-key-file=/.persist/keys/kube/kube-api-server.key" "--tls-min-version=VersionTLS13" "--service-cluster-ip-range=fd00:3e42:e349::/112,10.197.0.0/16" + "--requestheader-client-ca-file=/.persist/keys/kube/requestheader-client-ca.crt" + "--requestheader-allowed-names=\"\"" # CN must be in this list to be valid. Blank = accept all CN. + "--requestheader-extra-headers-prefix=X-Remote-Extra" + "--requestheader-group-headers=X-Remote-Group" + "--requestheader-username-headers=X-Remote-User" + "--proxy-client-cert-file=/.persist/keys/kube/proxy.crt" + "--proxy-client-key-file=/.persist/keys/kube/proxy.key" + "--enable-aggregator-routing=true" "--v=2" # OLD: diff --git a/nix/kubernetes/roles/kube_controller_manager/default.nix b/nix/kubernetes/roles/kube_controller_manager/default.nix index fb93f9af..10eaff67 100644 --- a/nix/kubernetes/roles/kube_controller_manager/default.nix +++ b/nix/kubernetes/roles/kube_controller_manager/default.nix @@ -44,10 +44,10 @@ in "--node-cidr-mask-size-ipv4=20" # default is 24 "--node-cidr-mask-size-ipv6=112" # default is 64, must be smaller than cluster-cidr mask "--cluster-name=kubernetes" - "--cluster-signing-cert-file=/.persist/keys/kube/ca.crt" - "--cluster-signing-key-file=/.persist/keys/kube/ca.key" + "--cluster-signing-cert-file=/.persist/keys/kube/client-ca.crt" + "--cluster-signing-key-file=/.persist/keys/kube/client-ca.key" "--kubeconfig=/.persist/keys/kube/kube-controller-manager.kubeconfig" - "--root-ca-file=/.persist/keys/kube/ca.crt" + "--root-ca-file=/.persist/keys/kube/client-ca.crt" "--service-account-private-key-file=/.persist/keys/kube/service-accounts.key" # "--service-cluster-ip-range=10.197.0.0/16" # "--service-cluster-ip-range=2620:11f:7001:7:ffff:ffff:0ac5:0000/16" diff --git a/nix/kubernetes/roles/kubelet/default.nix b/nix/kubernetes/roles/kubelet/default.nix index 3da95a12..5f4e78e6 100644 --- a/nix/kubernetes/roles/kubelet/default.nix +++ b/nix/kubernetes/roles/kubelet/default.nix @@ -22,7 +22,7 @@ let enabled = true; }; x509 = { - clientCAFile = "/.persist/keys/kube/ca.crt"; + clientCAFile = "/.persist/keys/kube/client-ca.crt"; }; }; authorization = {