Add kube-proxy.

nix/configuration/README.org

@@ -0,0 +1,12 @@
+* To-do
+** Perhaps use overlay for /etc for speedup
+#+begin_src nix
+  system.etc.overlay.enable = true;
+#+end_src
+** read https://nixos.org/manual/nixos/stable/
+** Performance for mini pc
+#+begin_src nix
+  security.pam.loginLimits = [
+    { domain = "@users"; item = "rtprio"; type = "-"; value = 1; }
+  ];
+#+end_src

nix/kubernetes/README.org

@@ -10,3 +10,17 @@
     { domain = "@users"; item = "rtprio"; type = "-"; value = 1; }
   ];
 #+end_src
+* Bootstrap
+** Install cilium
+#+begin_src bash
+  helm repo add cilium https://helm.cilium.io/
+  helm install --dry-run cilium cilium/cilium --version 1.18.4 --namespace kube-system \
+       --set kubeProxyReplacement=true \
+       --set k8sServiceHost=${API_SERVER_IP} \
+       --set k8sServicePort=${API_SERVER_PORT}
+
+
+  kubectl -n kube-system exec ds/cilium -- cilium-dbg status --verbose
+  kubectl -n kube-system exec ds/cilium -- cilium-dbg status | grep KubeProxyReplacement
+
+#+end_src

nix/kubernetes/configuration.nix

@@ -12,10 +12,12 @@
     ./roles/doas
     ./roles/dont_use_substituters
     ./roles/etcd
+    ./roles/firewall
     ./roles/image_based_appliance
     ./roles/iso
     ./roles/kube_apiserver
     ./roles/kube_controller_manager
+    ./roles/kube_proxy
     ./roles/kube_scheduler
     ./roles/kubelet
     ./roles/kubernetes

nix/kubernetes/keys/package/deploy-script/package.nix

@@ -212,7 +212,14 @@ let
           file = "${
             k8s.client-configs."${vm_name_to_hostname vm_name}"
           }/${vm_name_to_hostname vm_name}.kubeconfig";
-          name = "kubeconfig";
+          name = "kubelet.kubeconfig";
+          owner = 10024;
+          group = 10024;
+          mode = "0600";
+        }
+        {
+          dest_dir = "/vm/${vm_name}/persist/keys/kube";
+          file = "${k8s.client-configs.kube-proxy}/kube-proxy.kubeconfig";
           owner = 10024;
           group = 10024;
           mode = "0600";

nix/kubernetes/keys/scope.nix

@@ -140,7 +140,9 @@ makeScope newScope (
           };
           kube-proxy = {
             config_user = "system:kube-proxy";
-            config_server = "https://server.kubernetes.local:6443";
+            config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443";
+            # config_server = "https://127.0.0.1:6443";
+            # config_server = "https://server.kubernetes.local:6443";
           };
           kube-controller-manager = {
             config_user = "system:kube-controller-manager";

nix/kubernetes/roles/containerd/default.nix

@@ -19,28 +19,39 @@
 
   config = lib.mkIf config.me.containerd.enable {
     virtualisation.containerd.enable = true;
-    virtualisation.containerd.settings = {
+    virtualisation.containerd.settings =
-      "plugins" = {
+      # let
-        "io.containerd.grpc.v1.cri" = {
+      #   my-cni-plugins = pkgs.buildEnv {
-          "cni" = {
+      #     name = "my-cni-plugins";
-            "bin_dir" = "/opt/cni/bin";
+      #     paths = with pkgs; [
-            "conf_dir" = "/etc/cni/net.d";
+      #       cni-plugins
-          };
+      #       cni-plugin-flannel
-          "containerd" = {
+      #     ];
-            "default_runtime_name" = "runc";
+      #   };
-            "runtimes" = {
+      # in
-              "runc" = {
+      {
-                "options" = {
+        "plugins" = {
-                  "SystemdCgroup" = true;
+          "io.containerd.grpc.v1.cri" = {
-                };
+            "cni" = {
-                "runtime_type" = "io.containerd.runc.v2";
+              "bin_dir" = "/opt/cni/bin";
-              };
+              # "bin_dir" = "${my-cni-plugins}/bin";
+              "conf_dir" = "/etc/cni/net.d";
+            };
+            "containerd" = {
+              "default_runtime_name" = "runc";
+              "runtimes" = {
+                "runc" = {
+                  "options" = {
+                    "SystemdCgroup" = true;
+                  };
+                  "runtime_type" = "io.containerd.runc.v2";
+                };
+              };
+              "snapshotter" = "overlayfs";
             };
-            "snapshotter" = "overlayfs";
           };
         };
+        "version" = 2;
       };
-      "version" = 2;
-    };
   };
 }

nix/kubernetes/roles/control_plane/default.nix

@@ -18,6 +18,7 @@
   };
 
   config = lib.mkIf config.me.control_plane.enable {
+    me.firewall.enable = true;
     me.kube_apiserver.enable = true;
     me.kube_controller_manager.enable = true;
     me.kube_scheduler.enable = true;

nix/kubernetes/roles/firewall/default.nix

@@ -0,0 +1,36 @@
+{
+  config,
+  lib,
+  ...
+}:
+
+{
+  imports = [ ];
+
+  options.me = {
+    firewall.enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+      description = "Whether we want to install firewall.";
+    };
+  };
+
+  config = lib.mkIf config.me.firewall.enable {
+    # kernel modules and settings required by Kubernetes
+    boot.kernelModules = [
+      "overlay"
+      "br_netfilter"
+    ];
+    boot.kernel.sysctl = {
+      "net.bridge.bridge-nf-call-iptables" = 1;
+      "net.bridge.bridge-nf-call-ip6tables" = 1;
+      "net.ipv4.ip_forward" = 1;
+    };
+
+    networking.nftables.enable = true;
+    # We want to filter forwarded traffic.
+    # Also needed for `networking.firewall.extraForwardRules` to do anything.
+    networking.firewall.filterForward = true;
+  };
+}

nix/kubernetes/roles/kube_proxy/default.nix

@@ -0,0 +1,68 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  # shellCommand = cmd: (lib.concatMapStringsSep " " lib.strings.escapeShellArg cmd);
+  shellCommand = cmd: (builtins.concatStringsSep " " cmd);
+  settingsFormat = pkgs.formats.yaml { };
+  config_file = settingsFormat.generate "kube-proxy-config.yaml" config.me.kube-proxy.settings;
+in
+{
+  imports = [ ];
+
+  options.me = {
+    kube-proxy.enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+      description = "Whether we want to install kube-proxy.";
+    };
+
+    kube-proxy.settings = lib.mkOption {
+      type = settingsFormat.type;
+      default = {
+        kind = "KubeProxyConfiguration";
+        apiVersion = "kubeproxy.config.k8s.io/v1alpha1";
+        clientConnection = {
+          kubeconfig = "/.persist/keys/kube/kube-proxy.kubeconfig";
+        };
+        mode = "iptables";
+        # clusterCIDR = "10.200.0.0/16";
+        # clusterCIDR = "2620:11f:7001:7:ffff:ffff:0ac8:0000/16";
+        clusterCIDR = "fd49:0595:2bba::/48";
+      };
+      description = ''
+        kubelet-config.yaml
+      '';
+    };
+  };
+
+  config = lib.mkIf config.me.kube-proxy.enable {
+    systemd.services.kube-proxy = {
+      enable = true;
+      description = "Kubernetes Kube Proxy";
+      documentation = [ "https://github.com/kubernetes/kubernetes" ];
+      wantedBy = [ "kubernetes.target" ];
+      path = with pkgs; [
+        iptables
+      ];
+      unitConfig.DefaultDependencies = "no";
+      serviceConfig = {
+        ExecStart = (
+          shellCommand [
+            "${pkgs.kubernetes}/bin/kube-proxy"
+            "--config=${config_file}"
+            "--nodeport-addresses=primary"
+            "--proxy-mode=nftables"
+          ]
+        );
+        Restart = "on-failure";
+        RestartSec = 5;
+      };
+    };
+  };
+}

nix/kubernetes/roles/kubelet/default.nix

@@ -8,8 +8,6 @@
 let
   # shellCommand = cmd: (lib.concatMapStringsSep " " lib.strings.escapeShellArg cmd);
   shellCommand = cmd: (builtins.concatStringsSep " " cmd);
-  settingsFormat = pkgs.formats.yaml { };
-  config_file = settingsFormat.generate "kubelet-config.yaml" config.me.kubelet.settings;
 in
 {
   imports = [ ];
@@ -21,46 +19,6 @@ in
       example = true;
       description = "Whether we want to install kubelet.";
     };
-
-    kubelet.settings = lib.mkOption {
-      type = settingsFormat.type;
-      default = {
-        kind = "KubeletConfiguration";
-        apiVersion = "kubelet.config.k8s.io/v1beta1";
-        address = "0.0.0.0";
-        authentication = {
-          anonymous = {
-            enabled = false;
-          };
-          webhook = {
-            enabled = true;
-          };
-          x509 = {
-            clientCAFile = "/var/lib/kubelet/ca.crt";
-          };
-        };
-        authorization = {
-          mode = "Webhook";
-        };
-        cgroupDriver = "systemd";
-        containerRuntimeEndpoint = "unix:///var/run/containerd/containerd.sock";
-        enableServer = true;
-        failSwapOn = false;
-        maxPods = 16;
-        memorySwap = {
-          swapBehavior = "NoSwap";
-        };
-        port = 10250;
-        resolvConf = "/etc/resolv.conf";
-        registerNode = true;
-        runtimeRequestTimeout = "15m";
-        tlsCertFile = "/var/lib/kubelet/kubelet.crt";
-        tlsPrivateKeyFile = "/var/lib/kubelet/kubelet.key";
-      };
-      description = ''
-        kubelet-config.yaml
-      '';
-    };
   };
 
   config = lib.mkIf config.me.kubelet.enable {
@@ -79,9 +37,8 @@ in
         ExecStart = (
           shellCommand [
             "${pkgs.kubernetes}/bin/kubelet"
-            # "--config=${config_file}"
             "--config=${./files/kubelet-config.yaml}"
-            "--kubeconfig=/.persist/keys/kube/kubeconfig"
+            "--kubeconfig=/.persist/keys/kube/kubelet.kubeconfig"
             "--v=2"
           ]
         );

nix/kubernetes/roles/kubernetes/default.nix

@@ -18,6 +18,14 @@
   };
 
   config = lib.mkIf config.me.kubernetes.enable {
+    assertions = [
+      {
+        # Kubernetes should only upgrade 1 minor version at a time, so this assert is here to prevent unwittingly jumping versions.
+        assertion = lib.hasPrefix "1.35." pkgs.kubernetes.version;
+        message = "Unexpected Kubernetes package version: ${pkgs.kubernetes.version}";
+      }
+    ];
+
     environment.systemPackages = with pkgs; [
       kubernetes
     ];

nix/kubernetes/roles/worker_node/default.nix

@@ -19,6 +19,8 @@
 
   config = lib.mkIf config.me.worker_node.enable {
     me.containerd.enable = true;
+    me.firewall.enable = true;
+    me.kube-proxy.enable = true;
     me.kubelet.enable = true;
     me.kubernetes.enable = true;
   };