Generate certificates for the aggregation layer.

nix/kubernetes/keys/package/deploy-script/package.nix

@@ -106,21 +106,21 @@ let
         }
         {
           dest_dir = "/vm/${vm_name}/persist/keys/etcd";
-          file = "${k8s.ca}/ca.crt";
+          file = "${k8s.ca.client}/client-ca.crt";
           owner = 10016;
           group = 10016;
           mode = "0640";
         }
         {
           dest_dir = "/vm/${vm_name}/persist/keys/kube";
-          file = "${k8s.ca}/ca.crt";
+          file = "${k8s.ca.client}/client-ca.crt";
           owner = 10024;
           group = 10024;
           mode = "0640";
         }
         {
           dest_dir = "/vm/${vm_name}/persist/keys/kube";
-          file = "${k8s.ca}/ca.key";
+          file = "${k8s.ca.client}/client-ca.key";
           owner = 10024;
           group = 10024;
           mode = "0600";
@@ -175,6 +175,33 @@ let
           group = 10024;
           mode = "0600";
         }
+        {
+          dest_dir = "/vm/${vm_name}/persist/keys/kube";
+          file = "${k8s.ca.requestheader-client}/requestheader-client-ca.crt";
+          owner = 10024;
+          group = 10024;
+          mode = "0640";
+        }
+        {
+          dest_dir = "/vm/${vm_name}/persist/keys/kube";
+          file = "${
+            k8s.keys."${vm_name_to_hostname vm_name}-proxy"
+          }/${vm_name_to_hostname vm_name}-proxy.crt";
+          name = "proxy.crt";
+          owner = 10024;
+          group = 10024;
+          mode = "0640";
+        }
+        {
+          dest_dir = "/vm/${vm_name}/persist/keys/kube";
+          file = "${
+            k8s.keys."${vm_name_to_hostname vm_name}-proxy"
+          }/${vm_name_to_hostname vm_name}-proxy.key";
+          name = "proxy.key";
+          owner = 10024;
+          group = 10024;
+          mode = "0600";
+        }
       ])
     )
   );
@@ -193,7 +220,7 @@ let
       + (lib.concatMapStringsSep "\n" deploy_file [
         {
           dest_dir = "/vm/${vm_name}/persist/keys/kube";
-          file = "${k8s.ca}/ca.crt";
+          file = "${k8s.ca.client}/client-ca.crt";
           owner = 10024;
           group = 10024;
           mode = "0640";

nix/kubernetes/keys/package/k8s-ca/files/requestheader-client-ca.conf

@@ -0,0 +1,95 @@
+[req]
+distinguished_name = req_distinguished_name
+prompt             = no
+x509_extensions    = ca_x509_extensions
+
+[ca_x509_extensions]
+basicConstraints = CA:TRUE
+keyUsage         = cRLSign, keyCertSign
+
+[req_distinguished_name]
+C   = US
+ST  = Washington
+L   = Seattle
+CN  = CA
+
+[controller0-proxy]
+distinguished_name = controller0_distinguished_name
+prompt             = no
+req_extensions     = controller0_req_extensions
+
+[controller0_req_extensions]
+basicConstraints     = CA:FALSE
+extendedKeyUsage     = clientAuth, serverAuth
+keyUsage             = critical, digitalSignature, keyEncipherment
+nsCertType           = client
+nsComment            = "controller0 Certificate"
+subjectAltName       = @controller0_alt_names
+subjectKeyIdentifier = hash
+
+[controller0_distinguished_name]
+CN = system:node:controller0
+O  = system:nodes
+C  = US
+ST = Washington
+L  = Seattle
+
+[controller0_alt_names]
+IP.0  = 127.0.0.1
+IP.4  = 10.215.1.221
+IP.5  = 2620:11f:7001:7:ffff:ffff:0ad7:01dd
+DNS.0 = controller0
+
+[controller1-proxy]
+distinguished_name = controller1_distinguished_name
+prompt             = no
+req_extensions     = controller1_req_extensions
+
+[controller1_req_extensions]
+basicConstraints     = CA:FALSE
+extendedKeyUsage     = clientAuth, serverAuth
+keyUsage             = critical, digitalSignature, keyEncipherment
+nsCertType           = client
+nsComment            = "controller1 Certificate"
+subjectAltName       = @controller1_alt_names
+subjectKeyIdentifier = hash
+
+[controller1_distinguished_name]
+CN = system:node:controller1
+O  = system:nodes
+C  = US
+ST = Washington
+L  = Seattle
+
+[controller1_alt_names]
+IP.0  = 127.0.0.1
+IP.4  = 10.215.1.222
+IP.5  = 2620:11f:7001:7:ffff:ffff:0ad7:01de
+DNS.0 = controller1
+
+[controller2-proxy]
+distinguished_name = controller2_distinguished_name
+prompt             = no
+req_extensions     = controller2_req_extensions
+
+[controller2_req_extensions]
+basicConstraints     = CA:FALSE
+extendedKeyUsage     = clientAuth, serverAuth
+keyUsage             = critical, digitalSignature, keyEncipherment
+nsCertType           = client
+nsComment            = "controller2 Certificate"
+subjectAltName       = @controller2_alt_names
+subjectKeyIdentifier = hash
+
+[controller2_distinguished_name]
+CN = system:node:controller2
+O  = system:nodes
+C  = US
+ST = Washington
+L  = Seattle
+
+[controller2_alt_names]
+IP.0  = 127.0.0.1
+IP.6  = 10.215.1.223
+IP.7  = 2620:11f:7001:7:ffff:ffff:0ad7:01df
+DNS.0 = controller2

nix/kubernetes/keys/package/k8s-ca/package.nix

@@ -10,23 +10,28 @@
 {
   stdenv,
   openssl,
+  ca_name,
+  ca_config,
   ...
 }:
 stdenv.mkDerivation (finalAttrs: {
-  name = "k8s-ca";
+  name = "k8s-ca-${ca_name}";
   nativeBuildInputs = [ openssl ];
   buildInputs = [ ];
 
   unpackPhase = "true";
 
-  installPhase = ''
+  buildPhase = ''
-    mkdir -p "$out"
+    openssl genrsa -out "${ca_name}-ca.key" 4096
-    cd "$out"
 
-    openssl genrsa -out ca.key 4096
     openssl req -x509 -new -sha512 -noenc \
-            -key ca.key -days 3653 \
+            -key "${ca_name}-ca.key" -days 3653 \
-            -config ${./files/ca.conf} \
+            -config "${ca_config}" \
-            -out ca.crt
+            -out "${ca_name}-ca.crt"
+  '';
+
+  installPhase = ''
+    mkdir "$out"
+    cp "${ca_name}-ca.crt" "${ca_name}-ca.key" $out/
   '';
 })

nix/kubernetes/keys/package/k8s-client-config/package.nix

@@ -26,7 +26,7 @@ stdenv.mkDerivation (finalAttrs: {
 
   buildPhase = ''
     kubectl config set-cluster kubernetes-the-hard-way \
-      --certificate-authority=${k8s.ca}/ca.crt \
+      --certificate-authority=${k8s.ca.client}/client-ca.crt \
       --embed-certs=true \
       --server=${lib.strings.escapeShellArg config_server} \
       --kubeconfig=${config_name}.kubeconfig

nix/kubernetes/keys/package/k8s-keys/package.nix

@@ -15,9 +15,9 @@ symlinkJoin {
   name = "k8s-keys";
   paths = [
     scripts
-    k8s.ca
     k8s.encryption_config
   ]
+  ++ (builtins.attrValues k8s.ca)
   ++ (builtins.attrValues k8s.keys)
   ++ (builtins.attrValues k8s.client-configs)
   ++ (builtins.attrValues k8s.ssh-keys)

nix/kubernetes/keys/package/tls-key/package.nix

@@ -12,6 +12,8 @@
   openssl,
   k8s,
   key_name,
+  ca_name,
+  ca_config,
   ...
 }:
 stdenv.mkDerivation (finalAttrs: {
@@ -22,18 +24,18 @@ stdenv.mkDerivation (finalAttrs: {
   unpackPhase = "true";
 
   buildPhase = ''
-    cp ${k8s.ca}/ca.crt ${k8s.ca}/ca.key ./
+    cp ${k8s.ca."${ca_name}"}/${ca_name}-ca.crt ${k8s.ca."${ca_name}"}/${ca_name}-ca.key ./
 
     openssl genrsa -out "${key_name}.key" 4096
 
     openssl req -new -key "${key_name}.key" -sha256 \
-            -config "${../k8s-ca/files/ca.conf}" -section ${key_name} \
+            -config "${ca_config}" -section ${key_name} \
             -out "${key_name}.csr"
 
     openssl x509 -req -days 3653 -in "${key_name}.csr" \
             -copy_extensions copyall \
-            -sha256 -CA "./ca.crt" \
+            -sha256 -CA "./${ca_name}-ca.crt" \
-            -CAkey "./ca.key" \
+            -CAkey "./${ca_name}-ca.key" \
             -CAcreateserial \
             -out "${key_name}.crt"
   '';

nix/kubernetes/keys/scope.nix

@@ -78,24 +78,48 @@ makeScope newScope (
       inherit all_hostnames controllers;
       k8s = self;
     };
+    certificate_authorities = {
+      "client" = {
+        ca_config = ./package/k8s-ca/files/client-ca.conf;
+      };
+      "requestheader-client" = {
+        ca_config = ./package/k8s-ca/files/requestheader-client-ca.conf;
+      };
+    };
+    certificate_authorities_merged = (
+      builtins.mapAttrs (ca_name: ca_config: { inherit ca_name; } // ca_config) certificate_authorities
+    );
   in
   {
-    ca = (callPackage ./package/k8s-ca/package.nix additional_vars);
+    ca = (
+      builtins.mapAttrs (
+        ca_name: ca_config:
+        (callPackage ./package/k8s-ca/package.nix (additional_vars // { inherit ca_name; } // ca_config))
+      ) certificate_authorities
+    );
     keys = (
-      lib.genAttrs [
+      builtins.mapAttrs
-        "admin"
+        (
-        "controller0"
+          key_name: key_config:
-        "controller1"
+          (callPackage ./package/tls-key/package.nix (additional_vars // { inherit key_name; } // key_config))
-        "controller2"
+        )
-        "worker0"
+        {
-        "worker1"
+          "admin" = { } // certificate_authorities_merged.client;
-        "worker2"
+          "controller0" = { } // certificate_authorities_merged.client;
-        "kube-proxy"
+          "controller1" = { } // certificate_authorities_merged.client;
-        "kube-scheduler"
+          "controller2" = { } // certificate_authorities_merged.client;
-        "kube-controller-manager"
+          "worker0" = { } // certificate_authorities_merged.client;
-        "kube-api-server"
+          "worker1" = { } // certificate_authorities_merged.client;
-        "service-accounts"
+          "worker2" = { } // certificate_authorities_merged.client;
-      ] (key_name: (callPackage ./package/tls-key/package.nix (additional_vars // { inherit key_name; })))
+          "kube-proxy" = { } // certificate_authorities_merged.client;
+          "kube-scheduler" = { } // certificate_authorities_merged.client;
+          "kube-controller-manager" = { } // certificate_authorities_merged.client;
+          "kube-api-server" = { } // certificate_authorities_merged.client;
+          "service-accounts" = { } // certificate_authorities_merged.client;
+          "controller0-proxy" = { } // certificate_authorities_merged.requestheader-client;
+          "controller1-proxy" = { } // certificate_authorities_merged.requestheader-client;
+          "controller2-proxy" = { } // certificate_authorities_merged.requestheader-client;
+        }
     );
     ssh-keys = (
       lib.genAttrs [

nix/kubernetes/roles/etcd/default.nix

@@ -59,8 +59,8 @@
       keyFile = "/.persist/keys/etcd/kube-api-server.key";
       peerCertFile = "/.persist/keys/etcd/kube-api-server.crt";
       peerKeyFile = "/.persist/keys/etcd/kube-api-server.key";
-      trustedCaFile = "/.persist/keys/etcd/ca.crt";
+      trustedCaFile = "/.persist/keys/etcd/client-ca.crt";
-      peerTrustedCaFile = "/.persist/keys/etcd/ca.crt";
+      peerTrustedCaFile = "/.persist/keys/etcd/client-ca.crt";
       peerClientCertAuth = true;
       clientCertAuth = true;
       initialAdvertisePeerUrls = (

nix/kubernetes/roles/kube_apiserver/default.nix

@@ -65,15 +65,15 @@ in
             "--audit-log-path=/var/log/audit.log"
             "--authorization-mode=Node,RBAC"
             "--bind-address=0.0.0.0"
-            "--client-ca-file=/.persist/keys/kube/ca.crt"
+            "--client-ca-file=/.persist/keys/kube/client-ca.crt"
             "--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota"
-            "--etcd-cafile=/.persist/keys/kube/ca.crt"
+            "--etcd-cafile=/.persist/keys/kube/client-ca.crt"
             "--etcd-certfile=/.persist/keys/kube/kube-api-server.crt"
             "--etcd-keyfile=/.persist/keys/kube/kube-api-server.key"
             "--etcd-servers=${builtins.concatStringsSep "," config.me.kube_apiserver.etcd_services}"
             "--event-ttl=1h"
             "--encryption-provider-config=/.persist/keys/kube/encryption-config.yaml"
-            "--kubelet-certificate-authority=/.persist/keys/kube/ca.crt"
+            "--kubelet-certificate-authority=/.persist/keys/kube/client-ca.crt"
             "--kubelet-client-certificate=/.persist/keys/kube/kube-api-server.crt"
             "--kubelet-client-key=/.persist/keys/kube/kube-api-server.key"
             "--runtime-config='api/all=true'"
@@ -85,6 +85,14 @@ in
             "--tls-private-key-file=/.persist/keys/kube/kube-api-server.key"
             "--tls-min-version=VersionTLS13"
             "--service-cluster-ip-range=fd00:3e42:e349::/112,10.197.0.0/16"
+            "--requestheader-client-ca-file=/.persist/keys/kube/requestheader-client-ca.crt"
+            "--requestheader-allowed-names=\"\"" # CN must be in this list to be valid. Blank = accept all CN.
+            "--requestheader-extra-headers-prefix=X-Remote-Extra"
+            "--requestheader-group-headers=X-Remote-Group"
+            "--requestheader-username-headers=X-Remote-User"
+            "--proxy-client-cert-file=/.persist/keys/kube/proxy.crt"
+            "--proxy-client-key-file=/.persist/keys/kube/proxy.key"
+            "--enable-aggregator-routing=true"
             "--v=2"
 
             # OLD:

nix/kubernetes/roles/kube_controller_manager/default.nix

@@ -44,10 +44,10 @@ in
             "--node-cidr-mask-size-ipv4=20" # default is 24
             "--node-cidr-mask-size-ipv6=112" # default is 64, must be smaller than cluster-cidr mask
             "--cluster-name=kubernetes"
-            "--cluster-signing-cert-file=/.persist/keys/kube/ca.crt"
+            "--cluster-signing-cert-file=/.persist/keys/kube/client-ca.crt"
-            "--cluster-signing-key-file=/.persist/keys/kube/ca.key"
+            "--cluster-signing-key-file=/.persist/keys/kube/client-ca.key"
             "--kubeconfig=/.persist/keys/kube/kube-controller-manager.kubeconfig"
-            "--root-ca-file=/.persist/keys/kube/ca.crt"
+            "--root-ca-file=/.persist/keys/kube/client-ca.crt"
             "--service-account-private-key-file=/.persist/keys/kube/service-accounts.key"
             # "--service-cluster-ip-range=10.197.0.0/16"
             # "--service-cluster-ip-range=2620:11f:7001:7:ffff:ffff:0ac5:0000/16"

nix/kubernetes/roles/kubelet/default.nix

@@ -22,7 +22,7 @@ let
         enabled = true;
       };
       x509 = {
-        clientCAFile = "/.persist/keys/kube/ca.crt";
+        clientCAFile = "/.persist/keys/kube/client-ca.crt";
       };
     };
     authorization = {