Add configs for a new kubernetes cluster on NixOS.

2025-11-30 14:32:36 -05:00
parent d35cfaacbd
commit b16db4325f
58 changed files with 3221 additions and 0 deletions
--- a/nix/kubernetes/keys/Makefile
+++ b/nix/kubernetes/keys/Makefile
@@ -0,0 +1,357 @@
+SHELL := bash
+.ONESHELL:
+.SHELLFLAGS := -eu -o pipefail -c
+.DELETE_ON_ERROR:
+MAKEFLAGS += --warn-undefined-variables
+MAKEFLAGS += --no-builtin-rules
+OUT=generated
+
+ifeq ($(origin .RECIPEPREFIX), undefined)
+  $(error This Make does not support .RECIPEPREFIX. Please use GNU Make 4.0 or later)
+endif
+.RECIPEPREFIX = >
+
+KUBERNETES_PUBLIC_ADDRESS := 74.80.180.138
+WORKERS := worker0 worker1 worker2 controller0 controller1 controller2
+
+.PHONY: all
+all: \
+    $(OUT)/ca-key.pem \
+    $(OUT)/admin-key.pem \
+    $(OUT)/worker0-key.pem \
+    $(OUT)/worker1-key.pem \
+    $(OUT)/worker2-key.pem \
+    $(OUT)/controller0-proxy-key.pem \
+    $(OUT)/controller1-proxy-key.pem \
+    $(OUT)/controller2-proxy-key.pem \
+    $(OUT)/kube-controller-manager-key.pem \
+    $(OUT)/kube-proxy-key.pem \
+    $(OUT)/kube-scheduler-key.pem \
+    $(OUT)/kubernetes-key.pem \
+    $(OUT)/service-account-key.pem \
+    $(OUT)/worker0.kubeconfig \
+    $(OUT)/worker1.kubeconfig \
+    $(OUT)/worker2.kubeconfig \
+    $(OUT)/controller0.kubeconfig \
+    $(OUT)/controller1.kubeconfig \
+    $(OUT)/controller2.kubeconfig \
+    $(OUT)/kube-proxy.kubeconfig \
+    $(OUT)/kube-controller-manager.kubeconfig \
+    $(OUT)/kube-scheduler.kubeconfig \
+    $(OUT)/admin.kubeconfig \
+    $(OUT)/encryption-config.yaml \
+    $(OUT)/remote_admin.kubeconfig \
+    $(OUT)/requestheader-client-ca-key.pem
+
+.PHONY: clean
+clean:
+> rm -rf $(OUT)
+
+# Requestheader client ca
+$(OUT)/requestheader-client-ca-key.pem: requestheader-client-ca-csr.json ca-config.json
+> @mkdir -p $(@D)
+> cd $(@D) && cfssl gencert -initca ../requestheader-client-ca-csr.json | cfssljson -bare requestheader-client-ca
+
+# Certificate authority
+$(OUT)/ca-key.pem: ca-csr.json ca-config.json
+> @mkdir -p $(@D)
+> cd $(@D) && cfssl gencert -initca ../ca-csr.json | cfssljson -bare ca
+
+# Admin client certificate
+$(OUT)/admin-key.pem: admin-csr.json ca-config.json
+> @mkdir -p $(@D)
+> cd $(@D) && cfssl gencert \
+>   -ca=ca.pem \
+>   -ca-key=ca-key.pem \
+>   -config=../ca-config.json \
+>   -profile=kubernetes \
+>   ../admin-csr.json | cfssljson -bare admin
+
+# Worker kubelet client certificate
+$(OUT)/worker0-key.pem: worker0-csr.json ca-config.json
+> @mkdir -p $(@D)
+> cd $(@D) && cfssl gencert \
+>   -ca=ca.pem \
+>   -ca-key=ca-key.pem \
+>   -config=../ca-config.json \
+>   -hostname=worker0,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.207 \
+>   -profile=kubernetes \
+>   ../worker0-csr.json | cfssljson -bare worker0
+
+# Worker kubelet client certificate
+$(OUT)/worker1-key.pem: worker1-csr.json ca-config.json
+> @mkdir -p $(@D)
+> cd $(@D) && cfssl gencert \
+>   -ca=ca.pem \
+>   -ca-key=ca-key.pem \
+>   -config=../ca-config.json \
+>   -hostname=worker1,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.208 \
+>   -profile=kubernetes \
+>   ../worker1-csr.json | cfssljson -bare worker1
+
+# Worker kubelet client certificate
+$(OUT)/worker2-key.pem: worker2-csr.json ca-config.json
+> @mkdir -p $(@D)
+> cd $(@D) && cfssl gencert \
+>   -ca=ca.pem \
+>   -ca-key=ca-key.pem \
+>   -config=../ca-config.json \
+>   -hostname=worker2,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.209 \
+>   -profile=kubernetes \
+>   ../worker2-csr.json | cfssljson -bare worker2
+
+# Controller kubelet client certificate
+$(OUT)/controller0-key.pem: controller0-csr.json ca-config.json
+> @mkdir -p $(@D)
+> cd $(@D) && cfssl gencert \
+>   -ca=ca.pem \
+>   -ca-key=ca-key.pem \
+>   -config=../ca-config.json \
+>   -hostname=controller0,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.204 \
+>   -profile=kubernetes \
+>   ../controller0-csr.json | cfssljson -bare controller0
+
+# Controller kubelet client certificate
+$(OUT)/controller1-key.pem: controller1-csr.json ca-config.json
+> @mkdir -p $(@D)
+> cd $(@D) && cfssl gencert \
+>   -ca=ca.pem \
+>   -ca-key=ca-key.pem \
+>   -config=../ca-config.json \
+>   -hostname=controller1,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.205 \
+>   -profile=kubernetes \
+>   ../controller1-csr.json | cfssljson -bare controller1
+
+# Controller kubelet client certificate
+$(OUT)/controller2-key.pem: controller2-csr.json ca-config.json
+> @mkdir -p $(@D)
+> cd $(@D) && cfssl gencert \
+>   -ca=ca.pem \
+>   -ca-key=ca-key.pem \
+>   -config=../ca-config.json \
+>   -hostname=controller2,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.206 \
+>   -profile=kubernetes \
+>   ../controller2-csr.json | cfssljson -bare controller2
+
+# Controller kubelet client certificate
+$(OUT)/controller0-proxy-key.pem: controller0-proxy-csr.json ca-config.json $(OUT)/requestheader-client-ca-key.pem
+> @mkdir -p $(@D)
+> cd $(@D) && cfssl gencert \
+>   -ca=requestheader-client-ca.pem \
+>   -ca-key=requestheader-client-ca-key.pem \
+>   -config=../ca-config.json \
+>   -hostname=controller0,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.204 \
+>   -profile=kubernetes \
+>   ../controller0-proxy-csr.json | cfssljson -bare controller0-proxy
+
+# Controller kubelet client certificate
+$(OUT)/controller1-proxy-key.pem: controller1-proxy-csr.json ca-config.json $(OUT)/requestheader-client-ca-key.pem
+> @mkdir -p $(@D)
+> cd $(@D) && cfssl gencert \
+>   -ca=requestheader-client-ca.pem \
+>   -ca-key=requestheader-client-ca-key.pem \
+>   -config=../ca-config.json \
+>   -hostname=controller1,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.205 \
+>   -profile=kubernetes \
+>   ../controller1-proxy-csr.json | cfssljson -bare controller1-proxy
+
+# Controller kubelet client certificate
+$(OUT)/controller2-proxy-key.pem: controller2-proxy-csr.json ca-config.json $(OUT)/requestheader-client-ca-key.pem
+> @mkdir -p $(@D)
+> cd $(@D) && cfssl gencert \
+>   -ca=requestheader-client-ca.pem \
+>   -ca-key=requestheader-client-ca-key.pem \
+>   -config=../ca-config.json \
+>   -hostname=controller2,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.206 \
+>   -profile=kubernetes \
+>   ../controller2-proxy-csr.json | cfssljson -bare controller2-proxy
+
+# Controller manager client certificate
+$(OUT)/kube-controller-manager-key.pem: kube-controller-manager-csr.json ca-config.json
+> @mkdir -p $(@D)
+> cd $(@D) && cfssl gencert \
+>   -ca=ca.pem \
+>   -ca-key=ca-key.pem \
+>   -config=../ca-config.json \
+>   -profile=kubernetes \
+>   ../kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager
+
+# Kube proxy client certificate
+$(OUT)/kube-proxy-key.pem: kube-proxy-csr.json ca-config.json
+> @mkdir -p $(@D)
+> cd $(@D) && cfssl gencert \
+>   -ca=ca.pem \
+>   -ca-key=ca-key.pem \
+>   -config=../ca-config.json \
+>   -profile=kubernetes \
+>   ../kube-proxy-csr.json | cfssljson -bare kube-proxy
+
+# Kube scheduler client certificate
+$(OUT)/kube-scheduler-key.pem: kube-scheduler-csr.json ca-config.json
+> @mkdir -p $(@D)
+> cd $(@D) && cfssl gencert \
+>   -ca=ca.pem \
+>   -ca-key=ca-key.pem \
+>   -config=../ca-config.json \
+>   -profile=kubernetes \
+>   ../kube-scheduler-csr.json | cfssljson -bare kube-scheduler
+
+# Kuberntes API server certificate
+# TODO: Replace 10.32.0.1 with kubernetes api server local ip address from lab 8
+$(OUT)/kubernetes-key.pem: kubernetes-csr.json ca-config.json
+> @mkdir -p $(@D)
+> cd $(@D) && cfssl gencert \
+>   -ca=ca.pem \
+>   -ca-key=ca-key.pem \
+>   -config=../ca-config.json \
+>   -hostname=10.197.0.1,10.0.0.1,10.215.1.204,10.215.1.205,10.215.1.206,10.215.1.207,10.215.1.208,10.215.1.209,$(KUBERNETES_PUBLIC_ADDRESS),127.0.0.1,kubernetes,kubernetes.default,kubernetes.default.svc,kubernetes.default.svc.cluster,kubernetes.svc.cluster.local \
+>   -profile=kubernetes \
+>   ../kubernetes-csr.json | cfssljson -bare kubernetes
+
+# Service account keypair
+$(OUT)/service-account-key.pem: service-account-csr.json ca-config.json
+> @mkdir -p $(@D)
+> cd $(@D) && cfssl gencert \
+>   -ca=ca.pem \
+>   -ca-key=ca-key.pem \
+>   -config=../ca-config.json \
+>   -profile=kubernetes \
+>   ../service-account-csr.json | cfssljson -bare service-account
+
+# Generate worker kubeconfigs
+$(patsubst %,$(OUT)/%.kubeconfig,$(WORKERS)): $(OUT)/%.kubeconfig: $(OUT)/%-key.pem $(OUT)/%.pem
+> @mkdir -p $(@D)
+> kubectl config set-cluster kubernetes-the-hard-way \
+>   --certificate-authority=$(OUT)/ca.pem \
+>   --embed-certs=true \
+>   --server=https://$(KUBERNETES_PUBLIC_ADDRESS):6443 \
+>   --kubeconfig=$@
+>
+> kubectl config set-credentials system:node:$* \
+>   --client-certificate=$(OUT)/$*.pem \
+>   --client-key=$(OUT)/$*-key.pem \
+>   --embed-certs=true \
+>   --kubeconfig=$@
+>
+> kubectl config set-context default \
+>   --cluster=kubernetes-the-hard-way \
+>   --user=system:node:$* \
+>   --kubeconfig=$@
+>
+> kubectl config use-context default --kubeconfig=$@
+
+# Generate kube-proxy kubeconfig
+$(OUT)/kube-proxy.kubeconfig: $(OUT)/%.kubeconfig: $(OUT)/%-key.pem $(OUT)/%.pem
+> @mkdir -p $(@D)
+> kubectl config set-cluster kubernetes-the-hard-way \
+>   --certificate-authority=$(OUT)/ca.pem \
+>   --embed-certs=true \
+>   --server=https://$(KUBERNETES_PUBLIC_ADDRESS):6443 \
+>   --kubeconfig=$@
+>
+> kubectl config set-credentials system:$* \
+>   --client-certificate=$(OUT)/$*.pem \
+>   --client-key=$(OUT)/$*-key.pem \
+>   --embed-certs=true \
+>   --kubeconfig=$@
+>
+> kubectl config set-context default \
+>   --cluster=kubernetes-the-hard-way \
+>   --user=system:$* \
+>   --kubeconfig=$@
+>
+> kubectl config use-context default --kubeconfig=$@
+
+# Generate kube-controller-manager kubeconfig
+$(OUT)/kube-controller-manager.kubeconfig: $(OUT)/%.kubeconfig: $(OUT)/%-key.pem $(OUT)/%.pem
+> @mkdir -p $(@D)
+> kubectl config set-cluster kubernetes-the-hard-way \
+>   --certificate-authority=$(OUT)/ca.pem \
+>   --embed-certs=true \
+>   --server=https://127.0.0.1:6443 \
+>   --kubeconfig=$@
+>
+> kubectl config set-credentials system:$* \
+>   --client-certificate=$(OUT)/$*.pem \
+>   --client-key=$(OUT)/$*-key.pem \
+>   --embed-certs=true \
+>   --kubeconfig=$@
+>
+> kubectl config set-context default \
+>   --cluster=kubernetes-the-hard-way \
+>   --user=system:$* \
+>   --kubeconfig=$@
+>
+> kubectl config use-context default --kubeconfig=$@
+
+# Generate kube-scheduler kubeconfig
+$(OUT)/kube-scheduler.kubeconfig: $(OUT)/%.kubeconfig: $(OUT)/%-key.pem $(OUT)/%.pem
+> @mkdir -p $(@D)
+> kubectl config set-cluster kubernetes-the-hard-way \
+>   --certificate-authority=$(OUT)/ca.pem \
+>   --embed-certs=true \
+>   --server=https://127.0.0.1:6443 \
+>   --kubeconfig=$@
+>
+> kubectl config set-credentials system:$* \
+>   --client-certificate=$(OUT)/$*.pem \
+>   --client-key=$(OUT)/$*-key.pem \
+>   --embed-certs=true \
+>   --kubeconfig=$@
+>
+> kubectl config set-context default \
+>   --cluster=kubernetes-the-hard-way \
+>   --user=system:$* \
+>   --kubeconfig=$@
+>
+> kubectl config use-context default --kubeconfig=$@
+
+# Generate admin kubeconfig
+$(OUT)/admin.kubeconfig: $(OUT)/%.kubeconfig: $(OUT)/%-key.pem $(OUT)/%.pem
+> @mkdir -p $(@D)
+> kubectl config set-cluster kubernetes-the-hard-way \
+>   --certificate-authority=$(OUT)/ca.pem \
+>   --embed-certs=true \
+>   --server=https://127.0.0.1:6443 \
+>   --kubeconfig=$@
+>
+> kubectl config set-credentials $* \
+>   --client-certificate=$(OUT)/$*.pem \
+>   --client-key=$(OUT)/$*-key.pem \
+>   --embed-certs=true \
+>   --kubeconfig=$@
+>
+> kubectl config set-context default \
+>   --cluster=kubernetes-the-hard-way \
+>   --user=$* \
+>   --kubeconfig=$@
+>
+> kubectl config use-context default --kubeconfig=$@
+
+# Generate data encryption key for encrypting data at rest
+$(OUT)/encryption-config.yaml:
+> @mkdir -p $(@D)
+> ENCRYPTION_KEY=$(shell head -c 32 /dev/urandom | base64)
+> cat encryption-config-template.yaml | sed "s@ENCRYPTION_KEY@$$ENCRYPTION_KEY@g" > $@
+
+# Generate remote admin kubeconfig
+$(OUT)/remote_admin.kubeconfig: $(OUT)/remote_%.kubeconfig: $(OUT)/%-key.pem $(OUT)/%.pem
+> @mkdir -p $(@D)
+> kubectl config set-cluster kubernetes-the-hard-way \
+>   --certificate-authority=$(OUT)/ca.pem \
+>   --embed-certs=true \
+>   --server=https://$(KUBERNETES_PUBLIC_ADDRESS):6443 \
+>   --kubeconfig=$@
+>
+> kubectl config set-credentials $* \
+>   --client-certificate=$(OUT)/$*.pem \
+>   --client-key=$(OUT)/$*-key.pem \
+>   --embed-certs=true \
+>   --kubeconfig=$@
+>
+> kubectl config set-context default \
+>   --cluster=kubernetes-the-hard-way \
+>   --user=$* \
+>   --kubeconfig=$@
+>
+> kubectl config use-context default --kubeconfig=$@
--- a/nix/kubernetes/keys/flake.lock
+++ b/nix/kubernetes/keys/flake.lock
@@ -0,0 +1,27 @@
+{
+  "nodes": {
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1767892417,
+        "narHash": "sha256-dhhvQY67aboBk8b0/u0XB6vwHdgbROZT3fJAjyNh5Ww=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "3497aa5c9457a9d88d71fa93a4a8368816fbeeba",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "nixpkgs": "nixpkgs"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}
--- a/nix/kubernetes/keys/flake.nix
+++ b/nix/kubernetes/keys/flake.nix
@@ -0,0 +1,31 @@
+{
+  description = "Build keys to manually deploy to kubernetes cluster.";
+
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+  };
+
+  outputs =
+    { self, nixpkgs }:
+    let
+      forAllSystems = nixpkgs.lib.genAttrs nixpkgs.lib.systems.flakeExposed;
+    in
+    {
+      packages = forAllSystems (
+        system:
+        let
+          pkgs = nixpkgs.legacyPackages.${system};
+          appliedOverlay = self.overlays.default pkgs pkgs;
+        in
+        {
+          deploy_script = appliedOverlay.k8s.deploy_script;
+          default = appliedOverlay.k8s.keys;
+        }
+      );
+      overlays.default = (
+        final: prev: {
+          k8s = (final.callPackage ./scope.nix { inherit (final.lib) makeScope; });
+        }
+      );
+    };
+}
--- a/nix/kubernetes/keys/package/k8s-ca/files/ca-csr.json
+++ b/nix/kubernetes/keys/package/k8s-ca/files/ca-csr.json
@@ -0,0 +1,16 @@
+{
+  "CN": "Kubernetes",
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "names": [
+    {
+      "C": "US",
+      "L": "Portland",
+      "O": "Kubernetes",
+      "OU": "CA",
+      "ST": "Oregon"
+    }
+  ]
+}
--- a/nix/kubernetes/keys/package/k8s-ca/package.nix
+++ b/nix/kubernetes/keys/package/k8s-ca/package.nix
@@ -0,0 +1,28 @@
+# unpackPhase
+# patchPhase
+# configurePhase
+# buildPhase
+# checkPhase
+# installPhase
+# fixupPhase
+# installCheckPhase
+# distPhase
+{
+  stdenv,
+  sqlite,
+  cfssl,
+  ...
+}:
+stdenv.mkDerivation (finalAttrs: {
+  name = "k8s-ca";
+  nativeBuildInputs = [ cfssl ];
+  buildInputs = [ ];
+
+  unpackPhase = "true";
+
+  installPhase = ''
+    mkdir -p "$out"
+    cd "$out"
+    cfssl gencert -initca ${./files/ca-csr.json} | cfssljson -bare ca
+  '';
+})
--- a/nix/kubernetes/keys/package/k8s-keys/files/ca-config.json
+++ b/nix/kubernetes/keys/package/k8s-keys/files/ca-config.json
@@ -0,0 +1,13 @@
+{
+  "signing": {
+    "default": {
+      "expiry": "8760h"
+    },
+    "profiles": {
+      "kubernetes": {
+        "usages": ["signing", "key encipherment", "server auth", "client auth"],
+        "expiry": "8760h"
+      }
+    }
+  }
+}
--- a/nix/kubernetes/keys/package/k8s-keys/files/kubernetes-csr.json
+++ b/nix/kubernetes/keys/package/k8s-keys/files/kubernetes-csr.json
@@ -0,0 +1,16 @@
+{
+  "CN": "kubernetes",
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "names": [
+    {
+      "C": "US",
+      "L": "Portland",
+      "O": "Kubernetes",
+      "OU": "Kubernetes The Hard Way",
+      "ST": "Oregon"
+    }
+  ]
+}
--- a/nix/kubernetes/keys/package/k8s-keys/package.nix
+++ b/nix/kubernetes/keys/package/k8s-keys/package.nix
@@ -0,0 +1,12 @@
+{
+  k8s,
+  symlinkJoin,
+  ...
+}:
+symlinkJoin {
+  name = "k8s-keys";
+  paths = [
+    k8s.kubernetes
+    k8s.ca
+  ];
+}
--- a/nix/kubernetes/keys/package/k8s-kubernetes/files/ca-config.json
+++ b/nix/kubernetes/keys/package/k8s-kubernetes/files/ca-config.json
@@ -0,0 +1,13 @@
+{
+  "signing": {
+    "default": {
+      "expiry": "8760h"
+    },
+    "profiles": {
+      "kubernetes": {
+        "usages": ["signing", "key encipherment", "server auth", "client auth"],
+        "expiry": "8760h"
+      }
+    }
+  }
+}
--- a/nix/kubernetes/keys/package/k8s-kubernetes/files/kubernetes-csr.json
+++ b/nix/kubernetes/keys/package/k8s-kubernetes/files/kubernetes-csr.json
@@ -0,0 +1,16 @@
+{
+  "CN": "kubernetes",
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "names": [
+    {
+      "C": "US",
+      "L": "Portland",
+      "O": "Kubernetes",
+      "OU": "Kubernetes The Hard Way",
+      "ST": "Oregon"
+    }
+  ]
+}
--- a/nix/kubernetes/keys/package/k8s-kubernetes/package.nix
+++ b/nix/kubernetes/keys/package/k8s-kubernetes/package.nix
@@ -0,0 +1,36 @@
+# unpackPhase
+# patchPhase
+# configurePhase
+# buildPhase
+# checkPhase
+# installPhase
+# fixupPhase
+# installCheckPhase
+# distPhase
+{
+  stdenv,
+  sqlite,
+  cfssl,
+  k8s,
+  all_hostnames,
+  ...
+}:
+stdenv.mkDerivation (finalAttrs: {
+  name = "k8s-keys";
+  nativeBuildInputs = [ cfssl ];
+  buildInputs = [ ];
+
+  unpackPhase = "true";
+
+  installPhase = ''
+    mkdir -p "$out"
+    cd "$out"
+    cfssl gencert \
+       -ca=${k8s.ca}/ca.pem \
+       -ca-key=${k8s.ca}/ca-key.pem \
+       -config=${./files/ca-config.json} \
+       -hostname=${builtins.concatStringsSep "," all_hostnames} \
+       -profile=kubernetes \
+       ${./files/kubernetes-csr.json} | cfssljson -bare kubernetes
+  '';
+})
--- a/nix/kubernetes/keys/scope.nix
+++ b/nix/kubernetes/keys/scope.nix
@@ -0,0 +1,94 @@
+{
+  makeScope,
+  newScope,
+  callPackage,
+  writeShellScript,
+  openssh,
+  lib,
+}:
+let
+  public_addresses = [
+    "74.80.180.138"
+  ];
+  internal_addresses = [
+    # nc0
+    "10.215.1.221"
+    "2620:11f:7001:7:ffff:ffff:0ad7:01dd"
+    # nc1
+    "10.215.1.222"
+    "2620:11f:7001:7:ffff:ffff:0ad7:01de"
+    # nc2
+    "10.215.1.223"
+    "2620:11f:7001:7:ffff:ffff:0ad7:01df"
+    # nw0
+    "10.215.1.224"
+    "2620:11f:7001:7:ffff:ffff:0ad7:01e0"
+    # nw1
+    "10.215.1.225"
+    "2620:11f:7001:7:ffff:ffff:0ad7:01e1"
+    # nw2
+    "10.215.1.226"
+    "2620:11f:7001:7:ffff:ffff:0ad7:01e2"
+  ];
+  all_hostnames = [
+    "10.197.0.1"
+    "10.0.0.1"
+    "127.0.0.1"
+    "kubernetes"
+    "kubernetes.default"
+    "kubernetes.default.svc"
+    "kubernetes.default.svc.cluster"
+    "kubernetes.svc.cluster.local"
+  ]
+  ++ public_addresses
+  ++ internal_addresses;
+in
+makeScope newScope (
+  self:
+  let
+    additional_vars = {
+      inherit all_hostnames;
+      k8s = self;
+    };
+    deploy_key = (
+      vm_name: file: ''
+        ${openssh}/bin/ssh mrmanager rm -f /vm/${vm_name}/persist/keys/${builtins.baseNameOf file} ~/${builtins.baseNameOf file}
+        ${openssh}/bin/scp ${file} mrmanager:~/${builtins.baseNameOf file}
+        ${openssh}/bin/ssh mrmanager doas install -o 11235 -g 998 -m 0640 ~/${builtins.baseNameOf file} /vm/${vm_name}/persist/keys/${builtins.baseNameOf file}
+        ${openssh}/bin/ssh mrmanager rm -f ~/${builtins.baseNameOf file}
+        # chown to 11235:998 for talexander:etcd
+      ''
+    );
+    deploy_machine = (
+      vm_name:
+      (
+        ''
+          ${openssh}/bin/ssh mrmanager doas install -d -o talexander -g talexander -m 0755 /vm/${vm_name}/persist/keys/
+        ''
+        + (lib.concatMapStringsSep "\n" (deploy_key vm_name) [
+          "${self.kubernetes}/kubernetes.pem"
+          "${self.kubernetes}/kubernetes-key.pem"
+          "${self.ca}/ca.pem"
+        ])
+      )
+    );
+    deploy_script = (
+      ''
+        set -euo pipefail
+        IFS=$'\n\t'
+        DIR="$( cd "$( dirname "''${BASH_SOURCE[0]}" )" && pwd )"
+      ''
+      + (lib.concatMapStringsSep "\n" deploy_machine [
+        "nc0"
+        "nc1"
+        "nc2"
+      ])
+    );
+  in
+  {
+    ca = (callPackage ./package/k8s-ca/package.nix additional_vars);
+    kubernetes = (callPackage ./package/k8s-kubernetes/package.nix additional_vars);
+    keys = (callPackage ./package/k8s-keys/package.nix additional_vars);
+    deploy_script = (writeShellScript "deploy-keys" deploy_script);
+  }
+)