Build the cilium manifest automatically in nix.

nix/kubernetes/README.org

@@ -10,43 +10,11 @@
     { domain = "@users"; item = "rtprio"; type = "-"; value = 1; }
   ];
 #+end_src
-* Bootstrap
+* Healthcheck
-** Install cilium
+** Check cilium status
 #+begin_src bash
-  # nix shell nixpkgs#cilium-cli
+  kubectl -n kube-system exec ds/cilium -- cilium-dbg status --verbose
-  nix shell 'nixpkgs#kubernetes-helm'
-
-  helm repo add cilium https://helm.cilium.io/
-  helm template --dry-run=client cilium cilium/cilium --version 1.18.5 --namespace kube-system \
-       --set kubeProxyReplacement=true \
-       --set ipam.mode=kubernetes \
-       --set k8sServiceHost="2620:11f:7001:7:ffff:ffff:ad7:1dd" \
-       --set k8sServicePort=6443 \
-       --set ipv6.enabled=true \
-       --set ipv4.enabled=true \
-       --set enableIPv6Masquerade=false
-       # --set enableIPv4BIGTCP=true \
-       # --set enableIPv6BIGTCP=true
-       # --set routingMode=native \
-       # --set ipv4NativeRoutingCIDR=10.0.0.0/8 \
-       # --set ipv6NativeRoutingCIDR=fd00::/100
-
-  kubec
-  tl -n kube-system exec ds/cilium -- cilium-dbg status --verbose
   kubectl -n kube-system exec ds/cilium -- cilium-dbg status | grep KubeProxyReplacement
-
-  #      --set hostFirewall.enabled=true
-  # routingMode=native
-
-       # --set ipv4-native-routing-cidr=10.0.0.0/8 \
-       # --set ipv6-native-routing-cidr=fd00::/100
-  # --set 'ipam.operator.clusterPoolIPv4PodCIDRList=["10.0.0.0/8"]' \
-  # --set 'ipam.operator.clusterPoolIPv6PodCIDRList=["fd00::/100"]' \
-
-
-  # --set encryption.enabled=true \
-  #   --set encryption.type=wireguard
-  #   --set encryption.nodeEncryption=true
 #+end_src
 ** Install flux
 #+begin_src bash

nix/kubernetes/hosts/controller0/default.nix

@@ -51,9 +51,6 @@
           address = "2620:11f:7001:7:ffff:ffff:0ad7:0101";
           inherit interface;
         };
-        nameservers = [
-          "10.215.1.1"
-        ];
 
         dhcpcd.enable = lib.mkForce false;
         useDHCP = lib.mkForce false;

nix/kubernetes/hosts/controller1/default.nix

@@ -51,9 +51,6 @@
           address = "2620:11f:7001:7:ffff:ffff:0ad7:0101";
           inherit interface;
         };
-        nameservers = [
-          "10.215.1.1"
-        ];
 
         dhcpcd.enable = lib.mkForce false;
         useDHCP = lib.mkForce false;

nix/kubernetes/hosts/controller2/default.nix

@@ -51,9 +51,6 @@
           address = "2620:11f:7001:7:ffff:ffff:0ad7:0101";
           inherit interface;
         };
-        nameservers = [
-          "10.215.1.1"
-        ];
 
         dhcpcd.enable = lib.mkForce false;
         useDHCP = lib.mkForce false;

nix/kubernetes/hosts/worker0/default.nix

@@ -51,9 +51,6 @@
           address = "2620:11f:7001:7:ffff:ffff:0ad7:0101";
           inherit interface;
         };
-        nameservers = [
-          "10.215.1.1"
-        ];
 
         dhcpcd.enable = lib.mkForce false;
         useDHCP = lib.mkForce false;

nix/kubernetes/hosts/worker1/default.nix

@@ -51,9 +51,6 @@
           address = "2620:11f:7001:7:ffff:ffff:0ad7:0101";
           inherit interface;
         };
-        nameservers = [
-          "10.215.1.1"
-        ];
 
         dhcpcd.enable = lib.mkForce false;
         useDHCP = lib.mkForce false;

nix/kubernetes/hosts/worker2/default.nix

@@ -51,9 +51,6 @@
           address = "2620:11f:7001:7:ffff:ffff:0ad7:0101";
           inherit interface;
         };
-        nameservers = [
-          "10.215.1.1"
-        ];
 
         dhcpcd.enable = lib.mkForce false;
         useDHCP = lib.mkForce false;

nix/kubernetes/keys/package/bootstrap-script/package.nix

@@ -8,12 +8,10 @@
 # installCheckPhase
 # distPhase
 {
-  config,
   lib,
   stdenv,
   writeShellScript,
   k8s,
-  openssh,
   ...
 }:
 let
@@ -30,7 +28,7 @@ let
     lib.concatMapStringsSep "," lib.escapeShellArg (
       [
         ./files/manifests/initial_clusterrole.yaml
-        ./files/manifests/cilium.yaml
+        "${k8s.cilium-manifest}/cilium.yaml"
         ./files/manifests/coredns.yaml
         ./files/manifests/flux_namespace.yaml
         ./files/manifests/flux.yaml

nix/kubernetes/keys/package/cilium-manifest/package.nix

@@ -0,0 +1,70 @@
+# unpackPhase
+# patchPhase
+# configurePhase
+# buildPhase
+# checkPhase
+# installPhase
+# fixupPhase
+# installCheckPhase
+# distPhase
+{
+  stdenv,
+  openssl,
+  fetchFromGitHub,
+  kubernetes-helm,
+  ...
+}:
+stdenv.mkDerivation (
+  finalAttrs:
+  let
+    version = "1.18.5";
+  in
+  {
+    name = "cilium-manifest";
+    nativeBuildInputs = [
+      openssl
+      kubernetes-helm
+    ];
+    buildInputs = [ ];
+
+    src = fetchFromGitHub {
+      owner = "cilium";
+      repo = "cilium";
+      tag = "v${version}";
+      hash = "sha256-348inOOQ/fgwTYnaSHrQ363xGYnx2UPts3D4ycDRsWE=";
+    };
+
+    buildPhase = ''
+      helm template --dry-run=client cilium $src/install/kubernetes/cilium --version 1.18.5 --namespace kube-system \
+           --set kubeProxyReplacement=true \
+           --set ipam.mode=kubernetes \
+           --set k8sServiceHost="2620:11f:7001:7:ffff:ffff:ad7:1dd" \
+           --set k8sServicePort=6443 \
+           --set ipv6.enabled=true \
+           --set ipv4.enabled=true \
+           --set enableIPv6Masquerade=false \
+           | tee $NIX_BUILD_TOP/cilium.yaml
+    '';
+
+    # --set enableIPv4BIGTCP=false \
+    # --set enableIPv6BIGTCP=false \
+    # --set routingMode=native \
+    # --set ipv4NativeRoutingCIDR=10.0.0.0/8 \
+    # --set ipv6NativeRoutingCIDR=2620:11f:7001:7:ffff::/96 \
+
+    # --set hostFirewall.enabled=true
+    # --set routingMode=native
+
+    # --set 'ipam.operator.clusterPoolIPv4PodCIDRList=["10.0.0.0/8"]' \
+    # --set 'ipam.operator.clusterPoolIPv6PodCIDRList=["fd00::/100"]' \
+
+    # --set encryption.enabled=true \
+    # --set encryption.type=wireguard
+    # --set encryption.nodeEncryption=true
+
+    installPhase = ''
+      mkdir -p "$out"
+      cp $NIX_BUILD_TOP/cilium.yaml $out/
+    '';
+  }
+)

nix/kubernetes/keys/scope.nix

@@ -207,6 +207,7 @@ makeScope newScope (
         }
     );
     encryption_config = (callPackage ./package/k8s-encryption-key/package.nix additional_vars);
+    cilium-manifest = (callPackage ./package/cilium-manifest/package.nix additional_vars);
     all_keys = (callPackage ./package/k8s-keys/package.nix additional_vars);
     deploy_script = (callPackage ./package/deploy-script/package.nix additional_vars);
     bootstrap_script = (callPackage ./package/bootstrap-script/package.nix additional_vars);

nix/kubernetes/roles/network/default.nix

@@ -30,16 +30,21 @@
   config = lib.mkIf config.me.network.enable {
     networking.dhcpcd.enable = lib.mkDefault false;
     networking.useDHCP = lib.mkDefault false;
+    # Nameservers configured in host-specific files.
+    # networking.nameservers = [
+    #   "194.242.2.2#doh.mullvad.net"
+    #   "2a07:e340::2#doh.mullvad.net"
+    # ];
     networking.nameservers = [
-      "194.242.2.2#doh.mullvad.net"
+      "10.215.1.1"
-      "2a07:e340::2#doh.mullvad.net"
+      "2620:11f:7001:7:ffff:ffff:0ad7:0101"
     ];
     services.resolved = {
       enable = true;
       # dnssec = "true";
       domains = [ "~." ];
       fallbackDns = [ ];
-      dnsovertls = "true";
+      # dnsovertls = "true";
     };
 
     # Without this, systemd-resolved will send DNS requests for <X>.home.arpa to the per-link DNS server (172.16.0.1) which does not support DNS-over-TLS. This leads to the connection hanging and timing out. This causes firefox startup to take an extra 10+ seconds.