Some networking fixes.

nix/kubernetes/roles/cilium/default.nix

@@ -0,0 +1,29 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  imports = [ ];
+
+  options.me = {
+    cilium.enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+      description = "Whether we want to install cilium.";
+    };
+  };
+
+  config = lib.mkIf config.me.cilium.enable {
+    networking.firewall.allowedTCPPorts = [
+      4240 # Health checks
+    ];
+    networking.firewall.allowedUDPPorts = [
+      8472 # vxlan
+      51871 # wireguard
+    ];
+  };
+}

nix/kubernetes/roles/containerd/default.nix

@@ -5,6 +5,16 @@
   ...
 }:
 
+let
+  my-cni-plugins = pkgs.buildEnv {
+    name = "my-cni-plugins";
+    paths = with pkgs; [
+      cni-plugins
+      cni-plugin-flannel
+    ];
+  };
+  my-cni-configs = pkgs.callPackage ./package/cni_conf/package.nix { };
+in
 {
   imports = [ ];
 
@@ -19,40 +29,37 @@
 
   config = lib.mkIf config.me.containerd.enable {
     virtualisation.containerd.enable = true;
-    virtualisation.containerd.settings =
-      let
-        my-cni-plugins = pkgs.buildEnv {
-          name = "my-cni-plugins";
-          paths = with pkgs; [
-            cni-plugins
-            cni-plugin-flannel
-          ];
-        };
-      in
-      {
-        "plugins" = {
-          "io.containerd.grpc.v1.cri" = {
-            "cni" = {
-              # "bin_dir" = "/opt/cni/bin";
-              "bin_dir" = "${my-cni-plugins}/bin";
-              # "conf_dir" = "/etc/cni/net.d";
-              "conf_dir" = "${pkgs.callPackage ./package/cni_conf/package.nix { }}";
-            };
-            "containerd" = {
-              "default_runtime_name" = "runc";
-              "runtimes" = {
-                "runc" = {
-                  "options" = {
-                    "SystemdCgroup" = true;
-                  };
-                  "runtime_type" = "io.containerd.runc.v2";
+    virtualisation.containerd.settings = {
+      "plugins" = {
+        "io.containerd.grpc.v1.cri" = {
+          "cni" = {
+            "bin_dir" = "/opt/cni/bin";
+            "conf_dir" = "/etc/cni/net.d";
+            # "bin_dir" = "${my-cni-plugins}/bin";
+            # "conf_dir" = "${my-cni-configs}";
+          };
+          "containerd" = {
+            "default_runtime_name" = "runc";
+            "runtimes" = {
+              "runc" = {
+                "options" = {
+                  "SystemdCgroup" = true;
                 };
+                "runtime_type" = "io.containerd.runc.v2";
               };
-              "snapshotter" = "overlayfs";
             };
+            "snapshotter" = "overlayfs";
           };
         };
-        "version" = 2;
       };
+      "version" = 2;
+    };
+
+    systemd.services.containerd.preStart = ''
+      ${pkgs.toybox}/bin/install -d -m 0755 /opt/cni/bin /etc/cni/net.d
+      ${pkgs.toybox}/bin/install ${my-cni-plugins}/bin/* /opt/cni/bin/
+      ${pkgs.toybox}/bin/install ${my-cni-configs}/* /etc/cni/net.d/
+      echo "Copied CNI plugins/config."
+    '';
   };
 }

nix/kubernetes/roles/firewall/default.nix

@@ -32,5 +32,11 @@
     # We want to filter forwarded traffic.
     # Also needed for `networking.firewall.extraForwardRules` to do anything.
     networking.firewall.filterForward = true;
+
+    # This can make debugging easier by rejecting packets instead of dropping them:
+    # networking.firewall.rejectPackets = true;
+
+    # Check logs for blocked connections:
+    # journalctl -k or dmesg
   };
 }

nix/kubernetes/roles/kube_controller_manager/default.nix

@@ -59,5 +59,9 @@ in
         User = "kubernetes";
       };
     };
+
+    networking.firewall.allowedTCPPorts = [
+      10257
+    ];
   };
 }

nix/kubernetes/roles/kube_proxy/default.nix

@@ -57,12 +57,15 @@ in
             "${pkgs.kubernetes}/bin/kube-proxy"
             "--config=${config_file}"
             "--nodeport-addresses=primary"
-            "--proxy-mode=nftables"
           ]
         );
         Restart = "on-failure";
         RestartSec = 5;
       };
     };
+
+    networking.firewall.allowedTCPPorts = [
+      10256
+    ];
   };
 }

nix/kubernetes/roles/kube_scheduler/default.nix

@@ -47,5 +47,9 @@ in
         User = "kubernetes";
       };
     };
+
+    networking.firewall.allowedTCPPorts = [
+      10259
+    ];
   };
 }

nix/kubernetes/roles/kubelet/default.nix

@@ -54,5 +54,9 @@ in
         # StateDirectory = "kubelet";
       };
     };
+
+    networking.firewall.allowedTCPPorts = [
+      10250
+    ];
   };
 }

nix/kubernetes/roles/network/default.nix

@@ -48,6 +48,12 @@
     # TODO: The 127.0.0.1 address should probably be moved to a host-specific file.
     networking.extraHosts = ''
       127.0.0.1 ${config.networking.hostName}.home.arpa
+      2620:11f:7001:7:ffff:ffff:0ad7:01dd controller0.kubernetes.local controller0
+      2620:11f:7001:7:ffff:ffff:0ad7:01de controller1.kubernetes.local controller1
+      2620:11f:7001:7:ffff:ffff:0ad7:01df controller2.kubernetes.local controller2
+      2620:11f:7001:7:ffff:ffff:0ad7:01e0 worker0.kubernetes.local worker0
+      2620:11f:7001:7:ffff:ffff:0ad7:01e1 worker1.kubernetes.local worker1
+      2620:11f:7001:7:ffff:ffff:0ad7:01e2 worker2.kubernetes.local worker2
     '';
 
     environment.systemPackages = with pkgs; [
@@ -56,6 +62,7 @@
       arp-scan # To find devices on the network
       wavemon
       dhcpcd # For Android USB tethering.
+      net-tools # for netstat
     ];
 
     boot.extraModprobeConfig = ''

nix/kubernetes/roles/worker_node/default.nix

@@ -18,10 +18,27 @@
   };
 
   config = lib.mkIf config.me.worker_node.enable {
+    me.cilium.enable = true;
     me.containerd.enable = true;
     me.firewall.enable = true;
     # me.kube-proxy.enable = true;
     me.kubelet.enable = true;
     me.kubernetes.enable = true;
+
+    networking.firewall.allowedTCPPortRanges = [
+      {
+        # NodePort services
+        from = 30000;
+        to = 32767;
+      }
+    ];
+
+    networking.firewall.allowedUDPPortRanges = [
+      {
+        # NodePort services
+        from = 30000;
+        to = 32767;
+      }
+    ];
   };
 }