Delete images after 24 hours of being unused.

nix/kubernetes/README.org

@@ -12,11 +12,12 @@
 #+end_src
 * IP Ranges
 |                                | IPv4                        | IPv6                                    |
-|------------------------------+-----------------------------+-----------------------------------------|
+|--------------------------------+-----------------------------+-----------------------------------------|
 | Pod                            | 10.200.0.0/16               | 2620:11f:7001:7:ffff:eeee::/96          |
 | Service                        | 10.197.0.0/16               | fd00:3e42:e349::/112                    |
 | Node                           | 10.215.1.0/24               | 2620:11f:7001:7:ffff:ffff:0ad7:0100/120 |
 | Load Balancer                  | 74.80.180.139-74.80.180.142 | 2620:11f:7001:7:ffff:dddd::/96          |
+| Load Balancer Private (unused) | 10.198.0.0/16               | fd9c:0bd5:22a4::/112                    |
 | PowerDNS from inside cluster   | 10.215.1.211                |                                         |
 * Healthcheck
 ** Check cilium status

nix/kubernetes/keys/package/deploy-script/package.nix

@@ -119,8 +119,6 @@ let
         ${openssh}/bin/ssh mrmanager doas install -d -o 0 -g 0 -m 0755 /vm/${vm_name}/persist/keys
         ${openssh}/bin/ssh mrmanager doas install -d -o 10016 -g 10016 -m 0755 /vm/${vm_name}/persist/keys/etcd
         ${openssh}/bin/ssh mrmanager doas install -d -o 10024 -g 10024 -m 0755 /vm/${vm_name}/persist/keys/kube
-
-
       ''
       + (lib.concatMapStringsSep "\n" deploy_file [
         {
@@ -248,7 +246,8 @@ let
         ${openssh}/bin/ssh mrmanager doas install -d -o 0 -g 0 -m 0755 /vm/${vm_name}/persist/keys
         ${openssh}/bin/ssh mrmanager doas install -d -o 10024 -g 10024 -m 0755 /vm/${vm_name}/persist/keys/kube
 
-
+        ${openssh}/bin/ssh mrmanager doas install -d -o 0 -g 0 -m 0700 /vm/${vm_name}/persist/containerd/certs.d/docker.io
+        ${openssh}/bin/ssh mrmanager doas install -d -o 0 -g 0 -m 0700 /vm/${vm_name}/persist/containerd/certs.d/harbor.fizz.buzz
       ''
       + (lib.concatMapStringsSep "\n" deploy_file [
         {
@@ -291,6 +290,22 @@ let
           group = 10024;
           mode = "0600";
         }
+        {
+          dest_dir = "/vm/${vm_name}/persist/containerd/certs.d/docker.io";
+          file = "${./files/containerd/docker.io/hosts.toml}";
+          name = "hosts.toml";
+          owner = 0;
+          group = 0;
+          mode = "0600";
+        }
+        {
+          dest_dir = "/vm/${vm_name}/persist/containerd/certs.d/harbor.fizz.buzz";
+          file = "${./files/containerd/harbor.fizz.buzz/hosts.toml}";
+          name = "hosts.toml";
+          owner = 0;
+          group = 0;
+          mode = "0600";
+        }
       ])
     )
   );

nix/kubernetes/keys/package/k8s-secret-generic/package.nix

@@ -16,19 +16,29 @@
   secret_name,
   secret_namespace,
   secret_values ? { },
+  secret_type ? null,
+  secret_annotations ? null,
   ...
 }:
 let
   toBase64 = (pkgs.callPackage ../../contrib/base64/package.nix { inherit lib; }).toBase64;
-  secret_yaml = {
-    apiVersion = "v1";
-    kind = "Secret";
   metadata = {
     name = "${secret_name}";
     namespace = "${secret_namespace}";
+  }
+  // lib.optionalAttrs (secret_annotations != null) {
+    "annotations" = secret_annotations;
   };
+  secret_yaml = {
+    apiVersion = "v1";
+    kind = "Secret";
+    metadata = metadata;
     data = (builtins.mapAttrs (key: val: (toBase64 val)) secret_values);
+  }
+  // lib.optionalAttrs (secret_type != null) {
+    "type" = secret_type;
   };
+
   settingsFormat = pkgs.formats.yaml { };
   yaml_body = settingsFormat.generate "${secret_name}.yaml" secret_yaml;
   yaml_file = pkgs.writeTextFile {

nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix

@@ -13,13 +13,33 @@ let
       (
         secret_namespace: secrets:
         (builtins.mapAttrs (
-          secret_name: secret_values:
+          secret_name: original_secret_values:
+          let
+            secret_type = original_secret_values."__type" or null;
+            secret_annotations = original_secret_values."__annotations" or null;
+            secret_values = removeAttrs original_secret_values [
+              "__type"
+              "__annotations"
+            ];
+          in
           (callPackage ../../package/k8s-secret-generic/package.nix {
-            inherit secret_name secret_namespace secret_values;
+            inherit
+              secret_name
+              secret_namespace
+              secret_values
+              secret_type
+              secret_annotations
+              ;
           })
         ) secrets)
       )
       {
+        "archive-box" = {
+          "archive-box-auth" = {
+            "username" = (builtins.readFile "${./secrets/archive-box/archive-box-auth/username}");
+            "password" = (builtins.readFile "${./secrets/archive-box/archive-box-auth/password}");
+          };
+        };
         "cert-manager" = {
           "rfc2136" = {
             "TSIG_SECRET" = (builtins.readFile "${./secrets/cert-manager/rfc2136/TSIG_SECRET}");
@@ -37,6 +57,11 @@ let
             );
           };
         };
+        "flux-system" = {
+          "webhook-token" = {
+            "token" = generate_key 64 "flux-system.webhook-token.token";
+          };
+        };
         "gitea" = {
           "gitea-env" = {
             "GITEA_ADMIN_USERNAME" = (builtins.readFile "${./secrets/gitea/gitea-env/GITEA_ADMIN_USERNAME}");
@@ -59,6 +84,23 @@ let
             );
           };
         };
+        "tekton-gateway" = {
+          "oauth2-env" = oauth2_env { dex_id = "tekton"; };
+        };
+        "webhook-bridge" = {
+          "webhook-bridge" = {
+            "HMAC_TOKEN" = (builtins.readFile "${./secrets/webhook-bridge/webhook-bridge/HMAC_TOKEN}");
+            "OAUTH_TOKEN" = (builtins.readFile "${./secrets/webhook-bridge/webhook-bridge/OAUTH_TOKEN}");
+          };
+          "deployer-key" = {
+            "__annotations" = {
+              "tekton.dev/git-0" = "code.fizz.buzz";
+            };
+            "__type" = "kubernetes.io/ssh-auth";
+            "ssh-privatekey" = (builtins.readFile "${./secrets/webhook-bridge/deployer-key/ssh-privatekey}");
+            "ssh-publickey" = (builtins.readFile "${./secrets/webhook-bridge/deployer-key/ssh-publickey}");
+          };
+        };
       };
   encrypted_secrets = (
     builtins.mapAttrs (
@@ -101,8 +143,31 @@ let
         dd if=/dev/urandom | tr --complement --delete '[:alnum:]' | dd bs=${toString len} count=1 of="$out"
       ''
     );
-  helm_json_escape = json: builtins.toJSON json;
+  # helm_json_escape = json: builtins.toJSON json;
-
+  helm_json_escape =
+    json:
+    builtins.replaceStrings
+      [
+        "="
+        "["
+        "]"
+        ","
+        "."
+        "\""
+        "{"
+        "}"
+      ]
+      [
+        "\\="
+        "\\["
+        "\\]"
+        "\\,"
+        "\\."
+        "\\\""
+        "\\{"
+        "\\}"
+      ]
+      json;
   ## dex
   get_dex_config =
     client_id:

nix/kubernetes/roles/containerd/default.nix

@@ -1,3 +1,4 @@
+# TODO: Set up a proxy to harbor for OCI compliance: https://github.com/moby/moby/pull/34319#issuecomment-720606627
 {
   config,
   lib,
@@ -29,30 +30,43 @@ in
 
   config = lib.mkIf config.me.containerd.enable {
     virtualisation.containerd.enable = true;
-    virtualisation.containerd.settings = {
+    virtualisation.containerd.settings = lib.mkForce {
       "plugins" = {
-        "io.containerd.grpc.v1.cri" = {
+        "io.containerd.cri.v1.images" = {
+          "registry" = {
+            "config_path" = "/.persist/containerd/certs.d";
+          };
+          "snapshotter" = "overlayfs";
+        };
+        "io.containerd.cri.v1.runtime" = {
           "cni" = {
-            "bin_dir" = "/opt/cni/bin";
+            "bin_dirs" = [
+              "/opt/cni/bin"
+            ];
             "conf_dir" = "/etc/cni/net.d";
-            # "bin_dir" = "${my-cni-plugins}/bin";
-            # "conf_dir" = "${my-cni-configs}";
           };
           "containerd" = {
             "default_runtime_name" = "runc";
             "runtimes" = {
               "runc" = {
-                "options" = {
-                  "SystemdCgroup" = true;
-                };
                 "runtime_type" = "io.containerd.runc.v2";
               };
             };
-            "snapshotter" = "overlayfs";
+          };
+        };
+        "io.containerd.cri.v1.services" = {
+          "containerd" = {
+            "runtimes" = {
+              "runc" = {
+                "options" = {
+                  "SystemdCgroup" = true;
                 };
               };
             };
-      "version" = 2;
+          };
+        };
+      };
+      "version" = 3;
     };
 
     systemd.services.containerd.preStart = ''

nix/kubernetes/roles/kubelet/default.nix

@@ -32,7 +32,7 @@ let
     containerRuntimeEndpoint = "unix:///var/run/containerd/containerd.sock";
     enableServer = true;
     failSwapOn = false;
-    maxPods = 16;
+    maxPods = 110;
     memorySwap = {
       swapBehavior = "NoSwap";
     };
@@ -47,6 +47,7 @@ let
       "10.197.0.10"
       "fd00:3e42:e349::10"
     ];
+    imageMaximumGCAge = "24h"; # Delete unused images after 1 day.
   };
   kubelet_config_file = (to_yaml_file "kubelet-config.yaml" kubelet_config);
 in