Add controller proxy certs.

nix/kubernetes/keys/package/k8s-controller-proxy/files/ca-config.json

@@ -0,0 +1,13 @@
+{
+  "signing": {
+    "default": {
+      "expiry": "8760h"
+    },
+    "profiles": {
+      "kubernetes": {
+        "usages": ["signing", "key encipherment", "server auth", "client auth"],
+        "expiry": "8760h"
+      }
+    }
+  }
+}

nix/kubernetes/keys/package/k8s-controller-proxy/files/controller0-proxy-csr.json

@@ -0,0 +1,16 @@
+{
+  "CN": "system:node:controller0",
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "names": [
+    {
+      "C": "US",
+      "L": "Portland",
+      "O": "system:nodes",
+      "OU": "Kubernetes The Hard Way",
+      "ST": "Oregon"
+    }
+  ]
+}

nix/kubernetes/keys/package/k8s-controller-proxy/files/controller1-proxy-csr.json

@@ -0,0 +1,16 @@
+{
+  "CN": "system:node:controller1",
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "names": [
+    {
+      "C": "US",
+      "L": "Portland",
+      "O": "system:nodes",
+      "OU": "Kubernetes The Hard Way",
+      "ST": "Oregon"
+    }
+  ]
+}

nix/kubernetes/keys/package/k8s-controller-proxy/files/controller2-proxy-csr.json

@@ -0,0 +1,16 @@
+{
+  "CN": "system:node:controller2",
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "names": [
+    {
+      "C": "US",
+      "L": "Portland",
+      "O": "system:nodes",
+      "OU": "Kubernetes The Hard Way",
+      "ST": "Oregon"
+    }
+  ]
+}

nix/kubernetes/keys/package/k8s-controller-proxy/package.nix

@@ -0,0 +1,48 @@
+# unpackPhase
+# patchPhase
+# configurePhase
+# buildPhase
+# checkPhase
+# installPhase
+# fixupPhase
+# installCheckPhase
+# distPhase
+{
+  lib,
+  stdenv,
+  sqlite,
+  cfssl,
+  k8s,
+  all_hostnames,
+  controllers,
+  ...
+}:
+let
+  get_hostnames = (
+    hostname: (builtins.concatStringsSep "," ([ hostname ] ++ controllers."${hostname}".internal_ips))
+  );
+  install_body = (
+    lib.concatMapStringsSep "\n" (hostname: ''
+      cfssl gencert \
+            -ca=${k8s.requestheader-client-ca}/requestheader-client-ca.pem \
+            -ca-key=${k8s.requestheader-client-ca}/requestheader-client-ca-key.pem \
+            -config=${./files/ca-config.json} \
+            -hostname=${get_hostnames hostname} \
+            -profile=kubernetes \
+            ${./files}/${hostname}-proxy-csr.json | cfssljson -bare ${hostname}-proxy
+    '') (builtins.attrNames controllers)
+  );
+in
+stdenv.mkDerivation (finalAttrs: {
+  name = "k8s-controller-proxy";
+  nativeBuildInputs = [ cfssl ];
+  buildInputs = [ ];
+
+  unpackPhase = "true";
+
+  installPhase = ''
+    mkdir -p "$out"
+    cd "$out"
+  ''
+  + install_body;
+})

nix/kubernetes/keys/package/k8s-keys/package.nix

@@ -10,5 +10,6 @@ symlinkJoin {
     k8s.ca
     k8s.service_account
     k8s.requestheader-client-ca
+    k8s.controller-proxy
   ];
 }

nix/kubernetes/keys/scope.nix

@@ -44,12 +44,47 @@ let
   ]
   ++ public_addresses
   ++ internal_addresses;
+  controllers = {
+    "controller0" = {
+      "internal_ips" = [
+        "10.215.1.221"
+        "2620:11f:7001:7:ffff:ffff:0ad7:01dd"
+      ];
+      "external_ips" = [
+        "2620:11f:7001:7:ffff:ffff:0ad7:01dd"
+      ];
+    };
+    "controller1" = {
+      "internal_ips" = [
+        "10.215.1.222"
+        "2620:11f:7001:7:ffff:ffff:0ad7:01de"
+      ];
+      "external_ips" = [
+        "2620:11f:7001:7:ffff:ffff:0ad7:01de"
+      ];
+    };
+    "controller2" = {
+      "internal_ips" = [
+        "10.215.1.223"
+        "2620:11f:7001:7:ffff:ffff:0ad7:01df"
+      ];
+      "external_ips" = [
+        "2620:11f:7001:7:ffff:ffff:0ad7:01df"
+      ];
+    };
+  };
+  _vm_name_to_hostname = {
+    "nc0" = "controller0";
+    "nc1" = "controller1";
+    "nc2" = "controller2";
+  };
+  vm_name_to_hostname = (vm_name: _vm_name_to_hostname."${vm_name}");
 in
 makeScope newScope (
   self:
   let
     additional_vars = {
-      inherit all_hostnames;
+      inherit all_hostnames controllers;
       k8s = self;
     };
     deploy_file = (
@@ -158,6 +193,20 @@ makeScope newScope (
             group = 10024;
             mode = "0600";
           }
+          {
+            dest_dir = "/vm/${vm_name}/persist/keys/kube";
+            file = "${self.controller-proxy}/${vm_name_to_hostname vm_name}-proxy.pem";
+            owner = 10024;
+            group = 10024;
+            mode = "0600";
+          }
+          {
+            dest_dir = "/vm/${vm_name}/persist/keys/kube";
+            file = "${self.controller-proxy}/${vm_name_to_hostname vm_name}-proxy-key.pem";
+            owner = 10024;
+            group = 10024;
+            mode = "0600";
+          }
         ])
       )
     );
@@ -206,6 +255,7 @@ makeScope newScope (
     requestheader-client-ca = (
       callPackage ./package/k8s-requestheader-client-ca/package.nix additional_vars
     );
+    controller-proxy = (callPackage ./package/k8s-controller-proxy/package.nix additional_vars);
     keys = (callPackage ./package/k8s-keys/package.nix additional_vars);
     deploy_script = (writeShellScript "deploy-keys" deploy_script);
   }

nix/kubernetes/roles/kube_apiserver/default.nix

@@ -70,8 +70,8 @@ in
             "--requestheader-extra-headers-prefix=X-Remote-Extra-"
             "--requestheader-group-headers=X-Remote-Group"
             "--requestheader-username-headers=X-Remote-User"
-            "--proxy-client-cert-file=/var/lib/kubernetes/{{ node_name }}-proxy.pem"
+            "--proxy-client-cert-file=/.persist/keys/kube/${config.networking.hostName}-proxy.pem"
-            "--proxy-client-key-file=/var/lib/kubernetes/{{ node_name }}-proxy-key.pem"
+            "--proxy-client-key-file=/.persist/keys/kube/${config.networking.hostName}-proxy-key.pem"
             "--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota"
             "--etcd-cafile=/.persist/keys/kube/ca.pem"
             "--etcd-certfile=/.persist/keys/kube/kubernetes.pem"
@@ -91,7 +91,8 @@ in
             "--tls-private-key-file=/.persist/keys/kube/kubernetes-key.pem"
             "--tls-min-version=VersionTLS13"
             "--kubelet-preferred-address-types=InternalIP,ExternalDNS,ExternalIP,Hostname,InternalDNS"
-            "--service-cluster-ip-range=10.197.0.0/16"
+            # "--service-cluster-ip-range=10.197.0.0/16"
+            "--service-cluster-ip-range=2620:11f:7001:7:ffff:ffff:0ac5:0000/16"
             "--enable-aggregator-routing=true"
             "--v=2"
           ]