Apply the git repo to the cluster.

nix/kubernetes/keys/Makefile

@@ -16,342 +16,14 @@ WORKERS := worker0 worker1 worker2 controller0 controller1 controller2
 
 .PHONY: all
 all: \
-    $(OUT)/ca-key.pem \
+    $(OUT)/known_hosts
-    $(OUT)/admin-key.pem \
-    $(OUT)/worker0-key.pem \
-    $(OUT)/worker1-key.pem \
-    $(OUT)/worker2-key.pem \
-    $(OUT)/controller0-proxy-key.pem \
-    $(OUT)/controller1-proxy-key.pem \
-    $(OUT)/controller2-proxy-key.pem \
-    $(OUT)/kube-controller-manager-key.pem \
-    $(OUT)/kube-proxy-key.pem \
-    $(OUT)/kube-scheduler-key.pem \
-    $(OUT)/kubernetes-key.pem \
-    $(OUT)/service-account-key.pem \
-    $(OUT)/worker0.kubeconfig \
-    $(OUT)/worker1.kubeconfig \
-    $(OUT)/worker2.kubeconfig \
-    $(OUT)/controller0.kubeconfig \
-    $(OUT)/controller1.kubeconfig \
-    $(OUT)/controller2.kubeconfig \
-    $(OUT)/kube-proxy.kubeconfig \
-    $(OUT)/kube-controller-manager.kubeconfig \
-    $(OUT)/kube-scheduler.kubeconfig \
-    $(OUT)/admin.kubeconfig \
-    $(OUT)/encryption-config.yaml \
-    $(OUT)/remote_admin.kubeconfig \
-    $(OUT)/requestheader-client-ca-key.pem
 
 .PHONY: clean
 clean:
 > rm -rf $(OUT)
 
-# Requestheader client ca
+$(OUT)/:
-$(OUT)/requestheader-client-ca-key.pem: requestheader-client-ca-csr.json ca-config.json
 > @mkdir -p $(@D)
-> cd $(@D) && cfssl gencert -initca ../requestheader-client-ca-csr.json | cfssljson -bare requestheader-client-ca
 
-# Certificate authority
+$(OUT)/known_hosts: | $(OUT)/
-$(OUT)/ca-key.pem: ca-csr.json ca-config.json
+> ssh-keyscan -p 65099 74.80.180.138 | sed 's/\[74.80.180.138\]:65099/\[10.215.1.210\]:22/g' > $@
-> @mkdir -p $(@D)
-> cd $(@D) && cfssl gencert -initca ../ca-csr.json | cfssljson -bare ca
-
-# Admin client certificate
-$(OUT)/admin-key.pem: admin-csr.json ca-config.json
-> @mkdir -p $(@D)
-> cd $(@D) && cfssl gencert \
->   -ca=ca.pem \
->   -ca-key=ca-key.pem \
->   -config=../ca-config.json \
->   -profile=kubernetes \
->   ../admin-csr.json | cfssljson -bare admin
-
-# Worker kubelet client certificate
-$(OUT)/worker0-key.pem: worker0-csr.json ca-config.json
-> @mkdir -p $(@D)
-> cd $(@D) && cfssl gencert \
->   -ca=ca.pem \
->   -ca-key=ca-key.pem \
->   -config=../ca-config.json \
->   -hostname=worker0,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.207 \
->   -profile=kubernetes \
->   ../worker0-csr.json | cfssljson -bare worker0
-
-# Worker kubelet client certificate
-$(OUT)/worker1-key.pem: worker1-csr.json ca-config.json
-> @mkdir -p $(@D)
-> cd $(@D) && cfssl gencert \
->   -ca=ca.pem \
->   -ca-key=ca-key.pem \
->   -config=../ca-config.json \
->   -hostname=worker1,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.208 \
->   -profile=kubernetes \
->   ../worker1-csr.json | cfssljson -bare worker1
-
-# Worker kubelet client certificate
-$(OUT)/worker2-key.pem: worker2-csr.json ca-config.json
-> @mkdir -p $(@D)
-> cd $(@D) && cfssl gencert \
->   -ca=ca.pem \
->   -ca-key=ca-key.pem \
->   -config=../ca-config.json \
->   -hostname=worker2,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.209 \
->   -profile=kubernetes \
->   ../worker2-csr.json | cfssljson -bare worker2
-
-# Controller kubelet client certificate
-$(OUT)/controller0-key.pem: controller0-csr.json ca-config.json
-> @mkdir -p $(@D)
-> cd $(@D) && cfssl gencert \
->   -ca=ca.pem \
->   -ca-key=ca-key.pem \
->   -config=../ca-config.json \
->   -hostname=controller0,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.204 \
->   -profile=kubernetes \
->   ../controller0-csr.json | cfssljson -bare controller0
-
-# Controller kubelet client certificate
-$(OUT)/controller1-key.pem: controller1-csr.json ca-config.json
-> @mkdir -p $(@D)
-> cd $(@D) && cfssl gencert \
->   -ca=ca.pem \
->   -ca-key=ca-key.pem \
->   -config=../ca-config.json \
->   -hostname=controller1,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.205 \
->   -profile=kubernetes \
->   ../controller1-csr.json | cfssljson -bare controller1
-
-# Controller kubelet client certificate
-$(OUT)/controller2-key.pem: controller2-csr.json ca-config.json
-> @mkdir -p $(@D)
-> cd $(@D) && cfssl gencert \
->   -ca=ca.pem \
->   -ca-key=ca-key.pem \
->   -config=../ca-config.json \
->   -hostname=controller2,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.206 \
->   -profile=kubernetes \
->   ../controller2-csr.json | cfssljson -bare controller2
-
-# Controller kubelet client certificate
-$(OUT)/controller0-proxy-key.pem: controller0-proxy-csr.json ca-config.json $(OUT)/requestheader-client-ca-key.pem
-> @mkdir -p $(@D)
-> cd $(@D) && cfssl gencert \
->   -ca=requestheader-client-ca.pem \
->   -ca-key=requestheader-client-ca-key.pem \
->   -config=../ca-config.json \
->   -hostname=controller0,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.204 \
->   -profile=kubernetes \
->   ../controller0-proxy-csr.json | cfssljson -bare controller0-proxy
-
-# Controller kubelet client certificate
-$(OUT)/controller1-proxy-key.pem: controller1-proxy-csr.json ca-config.json $(OUT)/requestheader-client-ca-key.pem
-> @mkdir -p $(@D)
-> cd $(@D) && cfssl gencert \
->   -ca=requestheader-client-ca.pem \
->   -ca-key=requestheader-client-ca-key.pem \
->   -config=../ca-config.json \
->   -hostname=controller1,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.205 \
->   -profile=kubernetes \
->   ../controller1-proxy-csr.json | cfssljson -bare controller1-proxy
-
-# Controller kubelet client certificate
-$(OUT)/controller2-proxy-key.pem: controller2-proxy-csr.json ca-config.json $(OUT)/requestheader-client-ca-key.pem
-> @mkdir -p $(@D)
-> cd $(@D) && cfssl gencert \
->   -ca=requestheader-client-ca.pem \
->   -ca-key=requestheader-client-ca-key.pem \
->   -config=../ca-config.json \
->   -hostname=controller2,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.206 \
->   -profile=kubernetes \
->   ../controller2-proxy-csr.json | cfssljson -bare controller2-proxy
-
-# Controller manager client certificate
-$(OUT)/kube-controller-manager-key.pem: kube-controller-manager-csr.json ca-config.json
-> @mkdir -p $(@D)
-> cd $(@D) && cfssl gencert \
->   -ca=ca.pem \
->   -ca-key=ca-key.pem \
->   -config=../ca-config.json \
->   -profile=kubernetes \
->   ../kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager
-
-# Kube proxy client certificate
-$(OUT)/kube-proxy-key.pem: kube-proxy-csr.json ca-config.json
-> @mkdir -p $(@D)
-> cd $(@D) && cfssl gencert \
->   -ca=ca.pem \
->   -ca-key=ca-key.pem \
->   -config=../ca-config.json \
->   -profile=kubernetes \
->   ../kube-proxy-csr.json | cfssljson -bare kube-proxy
-
-# Kube scheduler client certificate
-$(OUT)/kube-scheduler-key.pem: kube-scheduler-csr.json ca-config.json
-> @mkdir -p $(@D)
-> cd $(@D) && cfssl gencert \
->   -ca=ca.pem \
->   -ca-key=ca-key.pem \
->   -config=../ca-config.json \
->   -profile=kubernetes \
->   ../kube-scheduler-csr.json | cfssljson -bare kube-scheduler
-
-# Kuberntes API server certificate
-# TODO: Replace 10.32.0.1 with kubernetes api server local ip address from lab 8
-$(OUT)/kubernetes-key.pem: kubernetes-csr.json ca-config.json
-> @mkdir -p $(@D)
-> cd $(@D) && cfssl gencert \
->   -ca=ca.pem \
->   -ca-key=ca-key.pem \
->   -config=../ca-config.json \
->   -hostname=10.197.0.1,10.0.0.1,10.215.1.204,10.215.1.205,10.215.1.206,10.215.1.207,10.215.1.208,10.215.1.209,$(KUBERNETES_PUBLIC_ADDRESS),127.0.0.1,kubernetes,kubernetes.default,kubernetes.default.svc,kubernetes.default.svc.cluster,kubernetes.svc.cluster.local \
->   -profile=kubernetes \
->   ../kubernetes-csr.json | cfssljson -bare kubernetes
-
-# Service account keypair
-$(OUT)/service-account-key.pem: service-account-csr.json ca-config.json
-> @mkdir -p $(@D)
-> cd $(@D) && cfssl gencert \
->   -ca=ca.pem \
->   -ca-key=ca-key.pem \
->   -config=../ca-config.json \
->   -profile=kubernetes \
->   ../service-account-csr.json | cfssljson -bare service-account
-
-# Generate worker kubeconfigs
-$(patsubst %,$(OUT)/%.kubeconfig,$(WORKERS)): $(OUT)/%.kubeconfig: $(OUT)/%-key.pem $(OUT)/%.pem
-> @mkdir -p $(@D)
-> kubectl config set-cluster kubernetes-the-hard-way \
->   --certificate-authority=$(OUT)/ca.pem \
->   --embed-certs=true \
->   --server=https://$(KUBERNETES_PUBLIC_ADDRESS):6443 \
->   --kubeconfig=$@
->
-> kubectl config set-credentials system:node:$* \
->   --client-certificate=$(OUT)/$*.pem \
->   --client-key=$(OUT)/$*-key.pem \
->   --embed-certs=true \
->   --kubeconfig=$@
->
-> kubectl config set-context default \
->   --cluster=kubernetes-the-hard-way \
->   --user=system:node:$* \
->   --kubeconfig=$@
->
-> kubectl config use-context default --kubeconfig=$@
-
-# Generate kube-proxy kubeconfig
-$(OUT)/kube-proxy.kubeconfig: $(OUT)/%.kubeconfig: $(OUT)/%-key.pem $(OUT)/%.pem
-> @mkdir -p $(@D)
-> kubectl config set-cluster kubernetes-the-hard-way \
->   --certificate-authority=$(OUT)/ca.pem \
->   --embed-certs=true \
->   --server=https://$(KUBERNETES_PUBLIC_ADDRESS):6443 \
->   --kubeconfig=$@
->
-> kubectl config set-credentials system:$* \
->   --client-certificate=$(OUT)/$*.pem \
->   --client-key=$(OUT)/$*-key.pem \
->   --embed-certs=true \
->   --kubeconfig=$@
->
-> kubectl config set-context default \
->   --cluster=kubernetes-the-hard-way \
->   --user=system:$* \
->   --kubeconfig=$@
->
-> kubectl config use-context default --kubeconfig=$@
-
-# Generate kube-controller-manager kubeconfig
-$(OUT)/kube-controller-manager.kubeconfig: $(OUT)/%.kubeconfig: $(OUT)/%-key.pem $(OUT)/%.pem
-> @mkdir -p $(@D)
-> kubectl config set-cluster kubernetes-the-hard-way \
->   --certificate-authority=$(OUT)/ca.pem \
->   --embed-certs=true \
->   --server=https://127.0.0.1:6443 \
->   --kubeconfig=$@
->
-> kubectl config set-credentials system:$* \
->   --client-certificate=$(OUT)/$*.pem \
->   --client-key=$(OUT)/$*-key.pem \
->   --embed-certs=true \
->   --kubeconfig=$@
->
-> kubectl config set-context default \
->   --cluster=kubernetes-the-hard-way \
->   --user=system:$* \
->   --kubeconfig=$@
->
-> kubectl config use-context default --kubeconfig=$@
-
-# Generate kube-scheduler kubeconfig
-$(OUT)/kube-scheduler.kubeconfig: $(OUT)/%.kubeconfig: $(OUT)/%-key.pem $(OUT)/%.pem
-> @mkdir -p $(@D)
-> kubectl config set-cluster kubernetes-the-hard-way \
->   --certificate-authority=$(OUT)/ca.pem \
->   --embed-certs=true \
->   --server=https://127.0.0.1:6443 \
->   --kubeconfig=$@
->
-> kubectl config set-credentials system:$* \
->   --client-certificate=$(OUT)/$*.pem \
->   --client-key=$(OUT)/$*-key.pem \
->   --embed-certs=true \
->   --kubeconfig=$@
->
-> kubectl config set-context default \
->   --cluster=kubernetes-the-hard-way \
->   --user=system:$* \
->   --kubeconfig=$@
->
-> kubectl config use-context default --kubeconfig=$@
-
-# Generate admin kubeconfig
-$(OUT)/admin.kubeconfig: $(OUT)/%.kubeconfig: $(OUT)/%-key.pem $(OUT)/%.pem
-> @mkdir -p $(@D)
-> kubectl config set-cluster kubernetes-the-hard-way \
->   --certificate-authority=$(OUT)/ca.pem \
->   --embed-certs=true \
->   --server=https://127.0.0.1:6443 \
->   --kubeconfig=$@
->
-> kubectl config set-credentials $* \
->   --client-certificate=$(OUT)/$*.pem \
->   --client-key=$(OUT)/$*-key.pem \
->   --embed-certs=true \
->   --kubeconfig=$@
->
-> kubectl config set-context default \
->   --cluster=kubernetes-the-hard-way \
->   --user=$* \
->   --kubeconfig=$@
->
-> kubectl config use-context default --kubeconfig=$@
-
-# Generate data encryption key for encrypting data at rest
-$(OUT)/encryption-config.yaml:
-> @mkdir -p $(@D)
-> ENCRYPTION_KEY=$(shell head -c 32 /dev/urandom | base64)
-> cat encryption-config-template.yaml | sed "s@ENCRYPTION_KEY@$$ENCRYPTION_KEY@g" > $@
-
-# Generate remote admin kubeconfig
-$(OUT)/remote_admin.kubeconfig: $(OUT)/remote_%.kubeconfig: $(OUT)/%-key.pem $(OUT)/%.pem
-> @mkdir -p $(@D)
-> kubectl config set-cluster kubernetes-the-hard-way \
->   --certificate-authority=$(OUT)/ca.pem \
->   --embed-certs=true \
->   --server=https://$(KUBERNETES_PUBLIC_ADDRESS):6443 \
->   --kubeconfig=$@
->
-> kubectl config set-credentials $* \
->   --client-certificate=$(OUT)/$*.pem \
->   --client-key=$(OUT)/$*-key.pem \
->   --embed-certs=true \
->   --kubeconfig=$@
->
-> kubectl config set-context default \
->   --cluster=kubernetes-the-hard-way \
->   --user=$* \
->   --kubeconfig=$@
->
-> kubectl config use-context default --kubeconfig=$@

nix/kubernetes/keys/generated/known_hosts

@@ -0,0 +1,8 @@
+# 74.80.180.138:65099 SSH-2.0-OpenSSH_9.3 FreeBSD-20230316
+[10.215.1.210]:22 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC0hWY7Ighnlp3UfPfApyW9nEGG11f+on/kOkp6YdxTTVX0jvi00xvrZ8c23l48YDptmEKOMj7avUR+jdpRNaSwbw3Lm7swg+EpFZ73tnHK+r6HnOnNu8ECDvYOW10eI6vdRctFisRfyIKigmtmquxXYLhQDSA2INVW+Vuebdwa74VqKLLirUu7e3ymp8dH8ktcCAjWSd/+Ax7E+4AMa5WHFeTPBheA2GhfLhINDLpgdZ8WNZ4i3ow8MrQADiOVYUDPrXvI55MVWSQTQQcOco184Z67rtcCtqY/fcCp+38yzUT0Bm2syXM+HNOlFqM+fJBf0T9kiiy5XvWuN9bY+368JGOUUM6RsCUgERHSaU65nX3i8oIcNRt3w6sVsmRR8sX8x5qFjyEYuElIwKywcdtKpoklV6gu+lo+mIE8i95jJmXMj6lk3G83wMZICL9+dm+b8ckpRZEi6970EqahiPO3cV/Fa88gysf9HwiC8AxSc3m2BcOvaV3jadaT39Tymp8=
+# 74.80.180.138:65099 SSH-2.0-OpenSSH_9.3 FreeBSD-20230316
+# 74.80.180.138:65099 SSH-2.0-OpenSSH_9.3 FreeBSD-20230316
+[10.215.1.210]:22 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBH2euFJKLEDfTV9NTecrOoqL9FpiYvTbNp/Ty3FebJA5DKmVd1xBRz3sNs1R1ayn213vmRVLWSu2ikulbl65LLQ=
+# 74.80.180.138:65099 SSH-2.0-OpenSSH_9.3 FreeBSD-20230316
+[10.215.1.210]:22 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM1qjGgD2UdD5Lc+zGFxHX/+h6FBNmGW+O30LG0tiHvC
+# 74.80.180.138:65099 SSH-2.0-OpenSSH_9.3 FreeBSD-20230316

nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_apply_git.yaml

@@ -0,0 +1,34 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: GitRepository
+metadata:
+  name: kubernetes
+  namespace: flux-system
+spec:
+  interval: 5m0s
+  ref:
+    branch: nix
+  secretRef:
+    name: kubernetes-deploy-key
+  # url: ssh://git@74.80.180.138:65099/repos/mrmanager
+  url: ssh://git@10.215.1.210:22/repos/mrmanager
+  ignore: |
+    bootstrap
+    .sops.yaml
+    secrets/
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: backend
+  namespace: flux-system
+spec:
+  interval: 5m0s
+  path: "./k8s"
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: kubernetes
+  decryption:
+    provider: sops
+    secretRef:
+      name: sops-gpg

nix/kubernetes/keys/package/k8s-keys/package.nix

@@ -1,11 +1,20 @@
 {
   k8s,
+  runCommand,
   symlinkJoin,
   ...
 }:
+let
+  scripts = runCommand "scripts" { } ''
+    mkdir $out
+    cp ${k8s.deploy_script} $out/deploy_script
+    cp ${k8s.bootstrap_script} $out/bootstrap_script
+  '';
+in
 symlinkJoin {
   name = "k8s-keys";
   paths = [
+    scripts
     k8s.ca
     k8s.encryption_config
   ]

nix/kubernetes/keys/scope.nix

@@ -134,6 +134,7 @@ makeScope newScope (
             secret_values = {
               "identity" = builtins.readFile "${self.ssh-keys.flux_ssh_key}/flux_ssh_key";
               "identity.pub" = builtins.readFile "${self.ssh-keys.flux_ssh_key}/flux_ssh_key.pub";
+              "known_hosts" = builtins.readFile ./generated/known_hosts;
             };
           };
         }