diff --git a/nix/configuration/README.org b/nix/configuration/README.org new file mode 100644 index 00000000..ba881163 --- /dev/null +++ b/nix/configuration/README.org @@ -0,0 +1,12 @@ +* To-do +** Perhaps use overlay for /etc for speedup +#+begin_src nix + system.etc.overlay.enable = true; +#+end_src +** read https://nixos.org/manual/nixos/stable/ +** Performance for mini pc +#+begin_src nix + security.pam.loginLimits = [ + { domain = "@users"; item = "rtprio"; type = "-"; value = 1; } + ]; +#+end_src diff --git a/nix/kubernetes/README.org b/nix/kubernetes/README.org index ba881163..5f156a26 100644 --- a/nix/kubernetes/README.org +++ b/nix/kubernetes/README.org @@ -10,3 +10,17 @@ { domain = "@users"; item = "rtprio"; type = "-"; value = 1; } ]; #+end_src +* Bootstrap +** Install cilium +#+begin_src bash + helm repo add cilium https://helm.cilium.io/ + helm install --dry-run cilium cilium/cilium --version 1.18.4 --namespace kube-system \ + --set kubeProxyReplacement=true \ + --set k8sServiceHost=${API_SERVER_IP} \ + --set k8sServicePort=${API_SERVER_PORT} + + + kubectl -n kube-system exec ds/cilium -- cilium-dbg status --verbose + kubectl -n kube-system exec ds/cilium -- cilium-dbg status | grep KubeProxyReplacement + +#+end_src diff --git a/nix/kubernetes/configuration.nix b/nix/kubernetes/configuration.nix index 617128d7..b6ca2c97 100644 --- a/nix/kubernetes/configuration.nix +++ b/nix/kubernetes/configuration.nix @@ -12,10 +12,12 @@ ./roles/doas ./roles/dont_use_substituters ./roles/etcd + ./roles/firewall ./roles/image_based_appliance ./roles/iso ./roles/kube_apiserver ./roles/kube_controller_manager + ./roles/kube_proxy ./roles/kube_scheduler ./roles/kubelet ./roles/kubernetes diff --git a/nix/kubernetes/keys/package/deploy-script/package.nix b/nix/kubernetes/keys/package/deploy-script/package.nix index 78152719..8b6844dc 100644 --- a/nix/kubernetes/keys/package/deploy-script/package.nix +++ b/nix/kubernetes/keys/package/deploy-script/package.nix @@ -212,7 +212,14 @@ let file = "${ k8s.client-configs."${vm_name_to_hostname vm_name}" }/${vm_name_to_hostname vm_name}.kubeconfig"; - name = "kubeconfig"; + name = "kubelet.kubeconfig"; + owner = 10024; + group = 10024; + mode = "0600"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.client-configs.kube-proxy}/kube-proxy.kubeconfig"; owner = 10024; group = 10024; mode = "0600"; diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index aeca4011..bdb5d539 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -140,7 +140,9 @@ makeScope newScope ( }; kube-proxy = { config_user = "system:kube-proxy"; - config_server = "https://server.kubernetes.local:6443"; + config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443"; + # config_server = "https://127.0.0.1:6443"; + # config_server = "https://server.kubernetes.local:6443"; }; kube-controller-manager = { config_user = "system:kube-controller-manager"; diff --git a/nix/kubernetes/roles/containerd/default.nix b/nix/kubernetes/roles/containerd/default.nix index 18dac5c2..88b1a073 100644 --- a/nix/kubernetes/roles/containerd/default.nix +++ b/nix/kubernetes/roles/containerd/default.nix @@ -19,28 +19,39 @@ config = lib.mkIf config.me.containerd.enable { virtualisation.containerd.enable = true; - virtualisation.containerd.settings = { - "plugins" = { - "io.containerd.grpc.v1.cri" = { - "cni" = { - "bin_dir" = "/opt/cni/bin"; - "conf_dir" = "/etc/cni/net.d"; - }; - "containerd" = { - "default_runtime_name" = "runc"; - "runtimes" = { - "runc" = { - "options" = { - "SystemdCgroup" = true; - }; - "runtime_type" = "io.containerd.runc.v2"; - }; + virtualisation.containerd.settings = + # let + # my-cni-plugins = pkgs.buildEnv { + # name = "my-cni-plugins"; + # paths = with pkgs; [ + # cni-plugins + # cni-plugin-flannel + # ]; + # }; + # in + { + "plugins" = { + "io.containerd.grpc.v1.cri" = { + "cni" = { + "bin_dir" = "/opt/cni/bin"; + # "bin_dir" = "${my-cni-plugins}/bin"; + "conf_dir" = "/etc/cni/net.d"; + }; + "containerd" = { + "default_runtime_name" = "runc"; + "runtimes" = { + "runc" = { + "options" = { + "SystemdCgroup" = true; + }; + "runtime_type" = "io.containerd.runc.v2"; + }; + }; + "snapshotter" = "overlayfs"; }; - "snapshotter" = "overlayfs"; }; }; + "version" = 2; }; - "version" = 2; - }; }; } diff --git a/nix/kubernetes/roles/control_plane/default.nix b/nix/kubernetes/roles/control_plane/default.nix index 70399829..ec8a7603 100644 --- a/nix/kubernetes/roles/control_plane/default.nix +++ b/nix/kubernetes/roles/control_plane/default.nix @@ -18,6 +18,7 @@ }; config = lib.mkIf config.me.control_plane.enable { + me.firewall.enable = true; me.kube_apiserver.enable = true; me.kube_controller_manager.enable = true; me.kube_scheduler.enable = true; diff --git a/nix/kubernetes/roles/firewall/default.nix b/nix/kubernetes/roles/firewall/default.nix new file mode 100644 index 00000000..a5dafab4 --- /dev/null +++ b/nix/kubernetes/roles/firewall/default.nix @@ -0,0 +1,36 @@ +{ + config, + lib, + ... +}: + +{ + imports = [ ]; + + options.me = { + firewall.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install firewall."; + }; + }; + + config = lib.mkIf config.me.firewall.enable { + # kernel modules and settings required by Kubernetes + boot.kernelModules = [ + "overlay" + "br_netfilter" + ]; + boot.kernel.sysctl = { + "net.bridge.bridge-nf-call-iptables" = 1; + "net.bridge.bridge-nf-call-ip6tables" = 1; + "net.ipv4.ip_forward" = 1; + }; + + networking.nftables.enable = true; + # We want to filter forwarded traffic. + # Also needed for `networking.firewall.extraForwardRules` to do anything. + networking.firewall.filterForward = true; + }; +} diff --git a/nix/kubernetes/roles/kube_proxy/default.nix b/nix/kubernetes/roles/kube_proxy/default.nix new file mode 100644 index 00000000..3a13fad6 --- /dev/null +++ b/nix/kubernetes/roles/kube_proxy/default.nix @@ -0,0 +1,68 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + # shellCommand = cmd: (lib.concatMapStringsSep " " lib.strings.escapeShellArg cmd); + shellCommand = cmd: (builtins.concatStringsSep " " cmd); + settingsFormat = pkgs.formats.yaml { }; + config_file = settingsFormat.generate "kube-proxy-config.yaml" config.me.kube-proxy.settings; +in +{ + imports = [ ]; + + options.me = { + kube-proxy.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install kube-proxy."; + }; + + kube-proxy.settings = lib.mkOption { + type = settingsFormat.type; + default = { + kind = "KubeProxyConfiguration"; + apiVersion = "kubeproxy.config.k8s.io/v1alpha1"; + clientConnection = { + kubeconfig = "/.persist/keys/kube/kube-proxy.kubeconfig"; + }; + mode = "iptables"; + # clusterCIDR = "10.200.0.0/16"; + # clusterCIDR = "2620:11f:7001:7:ffff:ffff:0ac8:0000/16"; + clusterCIDR = "fd49:0595:2bba::/48"; + }; + description = '' + kubelet-config.yaml + ''; + }; + }; + + config = lib.mkIf config.me.kube-proxy.enable { + systemd.services.kube-proxy = { + enable = true; + description = "Kubernetes Kube Proxy"; + documentation = [ "https://github.com/kubernetes/kubernetes" ]; + wantedBy = [ "kubernetes.target" ]; + path = with pkgs; [ + iptables + ]; + unitConfig.DefaultDependencies = "no"; + serviceConfig = { + ExecStart = ( + shellCommand [ + "${pkgs.kubernetes}/bin/kube-proxy" + "--config=${config_file}" + "--nodeport-addresses=primary" + "--proxy-mode=nftables" + ] + ); + Restart = "on-failure"; + RestartSec = 5; + }; + }; + }; +} diff --git a/nix/kubernetes/roles/kubelet/default.nix b/nix/kubernetes/roles/kubelet/default.nix index e3dc4752..76ab5a17 100644 --- a/nix/kubernetes/roles/kubelet/default.nix +++ b/nix/kubernetes/roles/kubelet/default.nix @@ -8,8 +8,6 @@ let # shellCommand = cmd: (lib.concatMapStringsSep " " lib.strings.escapeShellArg cmd); shellCommand = cmd: (builtins.concatStringsSep " " cmd); - settingsFormat = pkgs.formats.yaml { }; - config_file = settingsFormat.generate "kubelet-config.yaml" config.me.kubelet.settings; in { imports = [ ]; @@ -21,46 +19,6 @@ in example = true; description = "Whether we want to install kubelet."; }; - - kubelet.settings = lib.mkOption { - type = settingsFormat.type; - default = { - kind = "KubeletConfiguration"; - apiVersion = "kubelet.config.k8s.io/v1beta1"; - address = "0.0.0.0"; - authentication = { - anonymous = { - enabled = false; - }; - webhook = { - enabled = true; - }; - x509 = { - clientCAFile = "/var/lib/kubelet/ca.crt"; - }; - }; - authorization = { - mode = "Webhook"; - }; - cgroupDriver = "systemd"; - containerRuntimeEndpoint = "unix:///var/run/containerd/containerd.sock"; - enableServer = true; - failSwapOn = false; - maxPods = 16; - memorySwap = { - swapBehavior = "NoSwap"; - }; - port = 10250; - resolvConf = "/etc/resolv.conf"; - registerNode = true; - runtimeRequestTimeout = "15m"; - tlsCertFile = "/var/lib/kubelet/kubelet.crt"; - tlsPrivateKeyFile = "/var/lib/kubelet/kubelet.key"; - }; - description = '' - kubelet-config.yaml - ''; - }; }; config = lib.mkIf config.me.kubelet.enable { @@ -79,9 +37,8 @@ in ExecStart = ( shellCommand [ "${pkgs.kubernetes}/bin/kubelet" - # "--config=${config_file}" "--config=${./files/kubelet-config.yaml}" - "--kubeconfig=/.persist/keys/kube/kubeconfig" + "--kubeconfig=/.persist/keys/kube/kubelet.kubeconfig" "--v=2" ] ); diff --git a/nix/kubernetes/roles/kubernetes/default.nix b/nix/kubernetes/roles/kubernetes/default.nix index 9d3646d9..f4d70f1e 100644 --- a/nix/kubernetes/roles/kubernetes/default.nix +++ b/nix/kubernetes/roles/kubernetes/default.nix @@ -18,6 +18,14 @@ }; config = lib.mkIf config.me.kubernetes.enable { + assertions = [ + { + # Kubernetes should only upgrade 1 minor version at a time, so this assert is here to prevent unwittingly jumping versions. + assertion = lib.hasPrefix "1.35." pkgs.kubernetes.version; + message = "Unexpected Kubernetes package version: ${pkgs.kubernetes.version}"; + } + ]; + environment.systemPackages = with pkgs; [ kubernetes ]; diff --git a/nix/kubernetes/roles/worker_node/default.nix b/nix/kubernetes/roles/worker_node/default.nix index af082757..905b0368 100644 --- a/nix/kubernetes/roles/worker_node/default.nix +++ b/nix/kubernetes/roles/worker_node/default.nix @@ -19,6 +19,8 @@ config = lib.mkIf config.me.worker_node.enable { me.containerd.enable = true; + me.firewall.enable = true; + me.kube-proxy.enable = true; me.kubelet.enable = true; me.kubernetes.enable = true; };