diff --git a/nix/kubernetes/keys/package/k8s-controller-proxy/files/ca-config.json b/nix/kubernetes/keys/package/k8s-controller-proxy/files/ca-config.json new file mode 100644 index 00000000..a63e0dd2 --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-controller-proxy/files/ca-config.json @@ -0,0 +1,13 @@ +{ + "signing": { + "default": { + "expiry": "8760h" + }, + "profiles": { + "kubernetes": { + "usages": ["signing", "key encipherment", "server auth", "client auth"], + "expiry": "8760h" + } + } + } +} diff --git a/nix/kubernetes/keys/package/k8s-controller-proxy/files/controller0-proxy-csr.json b/nix/kubernetes/keys/package/k8s-controller-proxy/files/controller0-proxy-csr.json new file mode 100644 index 00000000..31a18eac --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-controller-proxy/files/controller0-proxy-csr.json @@ -0,0 +1,16 @@ +{ + "CN": "system:node:controller0", + "key": { + "algo": "rsa", + "size": 2048 + }, + "names": [ + { + "C": "US", + "L": "Portland", + "O": "system:nodes", + "OU": "Kubernetes The Hard Way", + "ST": "Oregon" + } + ] +} diff --git a/nix/kubernetes/keys/package/k8s-controller-proxy/files/controller1-proxy-csr.json b/nix/kubernetes/keys/package/k8s-controller-proxy/files/controller1-proxy-csr.json new file mode 100644 index 00000000..b5defb2b --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-controller-proxy/files/controller1-proxy-csr.json @@ -0,0 +1,16 @@ +{ + "CN": "system:node:controller1", + "key": { + "algo": "rsa", + "size": 2048 + }, + "names": [ + { + "C": "US", + "L": "Portland", + "O": "system:nodes", + "OU": "Kubernetes The Hard Way", + "ST": "Oregon" + } + ] +} diff --git a/nix/kubernetes/keys/package/k8s-controller-proxy/files/controller2-proxy-csr.json b/nix/kubernetes/keys/package/k8s-controller-proxy/files/controller2-proxy-csr.json new file mode 100644 index 00000000..09c26666 --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-controller-proxy/files/controller2-proxy-csr.json @@ -0,0 +1,16 @@ +{ + "CN": "system:node:controller2", + "key": { + "algo": "rsa", + "size": 2048 + }, + "names": [ + { + "C": "US", + "L": "Portland", + "O": "system:nodes", + "OU": "Kubernetes The Hard Way", + "ST": "Oregon" + } + ] +} diff --git a/nix/kubernetes/keys/package/k8s-controller-proxy/package.nix b/nix/kubernetes/keys/package/k8s-controller-proxy/package.nix new file mode 100644 index 00000000..ba965bbb --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-controller-proxy/package.nix @@ -0,0 +1,48 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + lib, + stdenv, + sqlite, + cfssl, + k8s, + all_hostnames, + controllers, + ... +}: +let + get_hostnames = ( + hostname: (builtins.concatStringsSep "," ([ hostname ] ++ controllers."${hostname}".internal_ips)) + ); + install_body = ( + lib.concatMapStringsSep "\n" (hostname: '' + cfssl gencert \ + -ca=${k8s.requestheader-client-ca}/requestheader-client-ca.pem \ + -ca-key=${k8s.requestheader-client-ca}/requestheader-client-ca-key.pem \ + -config=${./files/ca-config.json} \ + -hostname=${get_hostnames hostname} \ + -profile=kubernetes \ + ${./files}/${hostname}-proxy-csr.json | cfssljson -bare ${hostname}-proxy + '') (builtins.attrNames controllers) + ); +in +stdenv.mkDerivation (finalAttrs: { + name = "k8s-controller-proxy"; + nativeBuildInputs = [ cfssl ]; + buildInputs = [ ]; + + unpackPhase = "true"; + + installPhase = '' + mkdir -p "$out" + cd "$out" + '' + + install_body; +}) diff --git a/nix/kubernetes/keys/package/k8s-keys/package.nix b/nix/kubernetes/keys/package/k8s-keys/package.nix index 42d98349..151e9ecb 100644 --- a/nix/kubernetes/keys/package/k8s-keys/package.nix +++ b/nix/kubernetes/keys/package/k8s-keys/package.nix @@ -10,5 +10,6 @@ symlinkJoin { k8s.ca k8s.service_account k8s.requestheader-client-ca + k8s.controller-proxy ]; } diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index 8f490e9b..21acdcec 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -44,12 +44,47 @@ let ] ++ public_addresses ++ internal_addresses; + controllers = { + "controller0" = { + "internal_ips" = [ + "10.215.1.221" + "2620:11f:7001:7:ffff:ffff:0ad7:01dd" + ]; + "external_ips" = [ + "2620:11f:7001:7:ffff:ffff:0ad7:01dd" + ]; + }; + "controller1" = { + "internal_ips" = [ + "10.215.1.222" + "2620:11f:7001:7:ffff:ffff:0ad7:01de" + ]; + "external_ips" = [ + "2620:11f:7001:7:ffff:ffff:0ad7:01de" + ]; + }; + "controller2" = { + "internal_ips" = [ + "10.215.1.223" + "2620:11f:7001:7:ffff:ffff:0ad7:01df" + ]; + "external_ips" = [ + "2620:11f:7001:7:ffff:ffff:0ad7:01df" + ]; + }; + }; + _vm_name_to_hostname = { + "nc0" = "controller0"; + "nc1" = "controller1"; + "nc2" = "controller2"; + }; + vm_name_to_hostname = (vm_name: _vm_name_to_hostname."${vm_name}"); in makeScope newScope ( self: let additional_vars = { - inherit all_hostnames; + inherit all_hostnames controllers; k8s = self; }; deploy_file = ( @@ -158,6 +193,20 @@ makeScope newScope ( group = 10024; mode = "0600"; } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${self.controller-proxy}/${vm_name_to_hostname vm_name}-proxy.pem"; + owner = 10024; + group = 10024; + mode = "0600"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${self.controller-proxy}/${vm_name_to_hostname vm_name}-proxy-key.pem"; + owner = 10024; + group = 10024; + mode = "0600"; + } ]) ) ); @@ -206,6 +255,7 @@ makeScope newScope ( requestheader-client-ca = ( callPackage ./package/k8s-requestheader-client-ca/package.nix additional_vars ); + controller-proxy = (callPackage ./package/k8s-controller-proxy/package.nix additional_vars); keys = (callPackage ./package/k8s-keys/package.nix additional_vars); deploy_script = (writeShellScript "deploy-keys" deploy_script); } diff --git a/nix/kubernetes/roles/kube_apiserver/default.nix b/nix/kubernetes/roles/kube_apiserver/default.nix index 9701d991..4a0881f0 100644 --- a/nix/kubernetes/roles/kube_apiserver/default.nix +++ b/nix/kubernetes/roles/kube_apiserver/default.nix @@ -70,8 +70,8 @@ in "--requestheader-extra-headers-prefix=X-Remote-Extra-" "--requestheader-group-headers=X-Remote-Group" "--requestheader-username-headers=X-Remote-User" - "--proxy-client-cert-file=/var/lib/kubernetes/{{ node_name }}-proxy.pem" - "--proxy-client-key-file=/var/lib/kubernetes/{{ node_name }}-proxy-key.pem" + "--proxy-client-cert-file=/.persist/keys/kube/${config.networking.hostName}-proxy.pem" + "--proxy-client-key-file=/.persist/keys/kube/${config.networking.hostName}-proxy-key.pem" "--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota" "--etcd-cafile=/.persist/keys/kube/ca.pem" "--etcd-certfile=/.persist/keys/kube/kubernetes.pem" @@ -91,7 +91,8 @@ in "--tls-private-key-file=/.persist/keys/kube/kubernetes-key.pem" "--tls-min-version=VersionTLS13" "--kubelet-preferred-address-types=InternalIP,ExternalDNS,ExternalIP,Hostname,InternalDNS" - "--service-cluster-ip-range=10.197.0.0/16" + # "--service-cluster-ip-range=10.197.0.0/16" + "--service-cluster-ip-range=2620:11f:7001:7:ffff:ffff:0ac5:0000/16" "--enable-aggregator-routing=true" "--v=2" ]