From fa59f17df583ecdc1f2c089615baf109788e7f63 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 30 Nov 2025 14:32:36 -0500 Subject: [PATCH 01/81] Add configs for a new kubernetes cluster on NixOS. --- nix/kubernetes/.gitignore | 1 + nix/kubernetes/README.org | 12 + nix/kubernetes/configuration.nix | 138 +++++++ nix/kubernetes/flake.lock | 256 +++++++++++++ nix/kubernetes/flake.nix | 163 ++++++++ nix/kubernetes/formats/installer.nix | 74 ++++ nix/kubernetes/formats/iso.nix | 36 ++ nix/kubernetes/formats/sd.nix | 32 ++ nix/kubernetes/formats/toplevel.nix | 1 + nix/kubernetes/formats/vm_iso.nix | 22 ++ nix/kubernetes/hosts/controller0/DEPLOY_BOOT | 13 + .../hosts/controller0/DEPLOY_SWITCH | 13 + nix/kubernetes/hosts/controller0/ISO | 10 + nix/kubernetes/hosts/controller0/SELF_BOOT | 10 + nix/kubernetes/hosts/controller0/SELF_BUILD | 10 + nix/kubernetes/hosts/controller0/SELF_SWITCH | 10 + nix/kubernetes/hosts/controller0/VM_ISO | 10 + nix/kubernetes/hosts/controller0/default.nix | 123 ++++++ .../controller0/hardware-configuration.nix | 31 ++ nix/kubernetes/hosts/controller0/vm_disk.nix | 71 ++++ nix/kubernetes/keys/Makefile | 357 ++++++++++++++++++ nix/kubernetes/keys/flake.lock | 27 ++ nix/kubernetes/keys/flake.nix | 31 ++ .../keys/package/k8s-ca/files/ca-csr.json | 16 + .../keys/package/k8s-ca/package.nix | 28 ++ .../package/k8s-keys/files/ca-config.json | 13 + .../k8s-keys/files/kubernetes-csr.json | 16 + .../keys/package/k8s-keys/package.nix | 12 + .../k8s-kubernetes/files/ca-config.json | 13 + .../k8s-kubernetes/files/kubernetes-csr.json | 16 + .../keys/package/k8s-kubernetes/package.nix | 36 ++ nix/kubernetes/keys/scope.nix | 94 +++++ nix/kubernetes/repl.nix | 5 + nix/kubernetes/roles/boot/default.nix | 130 +++++++ nix/kubernetes/roles/doas/default.nix | 36 ++ .../roles/dont_use_substituters/default.nix | 25 ++ nix/kubernetes/roles/etcd/default.nix | 92 +++++ .../roles/image_based_appliance/default.nix | 30 ++ nix/kubernetes/roles/iso/default.nix | 22 ++ nix/kubernetes/roles/minimal_base/default.nix | 33 ++ nix/kubernetes/roles/network/default.nix | 87 +++++ nix/kubernetes/roles/nvme/default.nix | 25 ++ .../roles/optimized_build/default.nix | 134 +++++++ nix/kubernetes/roles/ssh/default.nix | 50 +++ nix/kubernetes/roles/ssh/files/ssh_config | 42 +++ .../roles/ssh/files/ssh_config_root | 9 + nix/kubernetes/roles/sshd/default.nix | 49 +++ nix/kubernetes/roles/user/default.nix | 59 +++ nix/kubernetes/roles/zfs/default.nix | 68 ++++ .../roles/zfs/files/zfs_clone_recv.bash | 13 + .../roles/zfs/files/zfs_clone_resume.bash | 17 + .../roles/zfs/files/zfs_clone_send.bash | 8 + nix/kubernetes/roles/zrepl/default.nix | 55 +++ nix/kubernetes/roles/zsh/default.nix | 117 ++++++ .../roles/zsh/package/zsh-histdb/package.nix | 36 ++ nix/kubernetes/util/install_files/default.nix | 331 ++++++++++++++++ .../util/install_files/files/lib.bash | 38 ++ .../util/unfree_polyfill/default.nix | 15 + 58 files changed, 3221 insertions(+) create mode 100644 nix/kubernetes/.gitignore create mode 100644 nix/kubernetes/README.org create mode 100644 nix/kubernetes/configuration.nix create mode 100644 nix/kubernetes/flake.lock create mode 100644 nix/kubernetes/flake.nix create mode 100644 nix/kubernetes/formats/installer.nix create mode 100644 nix/kubernetes/formats/iso.nix create mode 100644 nix/kubernetes/formats/sd.nix create mode 100644 nix/kubernetes/formats/toplevel.nix create mode 100644 nix/kubernetes/formats/vm_iso.nix create mode 100755 nix/kubernetes/hosts/controller0/DEPLOY_BOOT create mode 100755 nix/kubernetes/hosts/controller0/DEPLOY_SWITCH create mode 100755 nix/kubernetes/hosts/controller0/ISO create mode 100755 nix/kubernetes/hosts/controller0/SELF_BOOT create mode 100755 nix/kubernetes/hosts/controller0/SELF_BUILD create mode 100755 nix/kubernetes/hosts/controller0/SELF_SWITCH create mode 100755 nix/kubernetes/hosts/controller0/VM_ISO create mode 100644 nix/kubernetes/hosts/controller0/default.nix create mode 100644 nix/kubernetes/hosts/controller0/hardware-configuration.nix create mode 100644 nix/kubernetes/hosts/controller0/vm_disk.nix create mode 100644 nix/kubernetes/keys/Makefile create mode 100644 nix/kubernetes/keys/flake.lock create mode 100644 nix/kubernetes/keys/flake.nix create mode 100644 nix/kubernetes/keys/package/k8s-ca/files/ca-csr.json create mode 100644 nix/kubernetes/keys/package/k8s-ca/package.nix create mode 100644 nix/kubernetes/keys/package/k8s-keys/files/ca-config.json create mode 100644 nix/kubernetes/keys/package/k8s-keys/files/kubernetes-csr.json create mode 100644 nix/kubernetes/keys/package/k8s-keys/package.nix create mode 100644 nix/kubernetes/keys/package/k8s-kubernetes/files/ca-config.json create mode 100644 nix/kubernetes/keys/package/k8s-kubernetes/files/kubernetes-csr.json create mode 100644 nix/kubernetes/keys/package/k8s-kubernetes/package.nix create mode 100644 nix/kubernetes/keys/scope.nix create mode 100644 nix/kubernetes/repl.nix create mode 100644 nix/kubernetes/roles/boot/default.nix create mode 100644 nix/kubernetes/roles/doas/default.nix create mode 100644 nix/kubernetes/roles/dont_use_substituters/default.nix create mode 100644 nix/kubernetes/roles/etcd/default.nix create mode 100644 nix/kubernetes/roles/image_based_appliance/default.nix create mode 100644 nix/kubernetes/roles/iso/default.nix create mode 100644 nix/kubernetes/roles/minimal_base/default.nix create mode 100644 nix/kubernetes/roles/network/default.nix create mode 100644 nix/kubernetes/roles/nvme/default.nix create mode 100644 nix/kubernetes/roles/optimized_build/default.nix create mode 100644 nix/kubernetes/roles/ssh/default.nix create mode 100644 nix/kubernetes/roles/ssh/files/ssh_config create mode 100644 nix/kubernetes/roles/ssh/files/ssh_config_root create mode 100644 nix/kubernetes/roles/sshd/default.nix create mode 100644 nix/kubernetes/roles/user/default.nix create mode 100644 nix/kubernetes/roles/zfs/default.nix create mode 100644 nix/kubernetes/roles/zfs/files/zfs_clone_recv.bash create mode 100644 nix/kubernetes/roles/zfs/files/zfs_clone_resume.bash create mode 100644 nix/kubernetes/roles/zfs/files/zfs_clone_send.bash create mode 100644 nix/kubernetes/roles/zrepl/default.nix create mode 100644 nix/kubernetes/roles/zsh/default.nix create mode 100644 nix/kubernetes/roles/zsh/package/zsh-histdb/package.nix create mode 100644 nix/kubernetes/util/install_files/default.nix create mode 100644 nix/kubernetes/util/install_files/files/lib.bash create mode 100644 nix/kubernetes/util/unfree_polyfill/default.nix diff --git a/nix/kubernetes/.gitignore b/nix/kubernetes/.gitignore new file mode 100644 index 0000000..b2be92b --- /dev/null +++ b/nix/kubernetes/.gitignore @@ -0,0 +1 @@ +result diff --git a/nix/kubernetes/README.org b/nix/kubernetes/README.org new file mode 100644 index 0000000..ba88116 --- /dev/null +++ b/nix/kubernetes/README.org @@ -0,0 +1,12 @@ +* To-do +** Perhaps use overlay for /etc for speedup +#+begin_src nix + system.etc.overlay.enable = true; +#+end_src +** read https://nixos.org/manual/nixos/stable/ +** Performance for mini pc +#+begin_src nix + security.pam.loginLimits = [ + { domain = "@users"; item = "rtprio"; type = "-"; value = 1; } + ]; +#+end_src diff --git a/nix/kubernetes/configuration.nix b/nix/kubernetes/configuration.nix new file mode 100644 index 0000000..2be3505 --- /dev/null +++ b/nix/kubernetes/configuration.nix @@ -0,0 +1,138 @@ +{ + config, + lib, + ... +}: + +{ + imports = [ + ./roles/boot + ./roles/doas + ./roles/etcd + ./roles/image_based_appliance + ./roles/iso + ./roles/optimized_build + ./roles/dont_use_substituters + ./roles/minimal_base + ./roles/network + ./roles/nvme + ./roles/ssh + ./roles/sshd + ./roles/user + ./roles/zfs + ./roles/zrepl + ./roles/zsh + ./util/install_files + ./util/unfree_polyfill + ]; + + config = { + nix.settings.experimental-features = [ + "nix-command" + "flakes" + "ca-derivations" + # "blake3-hashes" + # "git-hashing" + ]; + nix.settings.trusted-users = [ "@wheel" ]; + + hardware.enableRedistributableFirmware = true; + + # Keep outputs so we can build offline. + nix.settings.keep-outputs = true; + nix.settings.keep-derivations = true; + + # Automatic garbage collection + nix.gc = lib.mkIf (!config.me.buildingPortable) { + # Runs nix-collect-garbage --delete-older-than 5d + automatic = true; + persistent = true; + dates = "monthly"; + # randomizedDelaySec = "14m"; + options = "--delete-older-than 30d"; + }; + nix.settings.auto-optimise-store = !config.me.buildingPortable; + + environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) { + hideMounts = true; + directories = [ + "/var/lib/nixos" # Contains user information (uids/gids) + "/var/lib/systemd" # Systemd state directory for random seed, persistent timers, core dumps, persist hardware state like backlight and rfkill + "/var/log/journal" # Logs, alternatively set `services.journald.storage = "volatile";` to write to /run/log/journal + ]; + files = [ + "/etc/machine-id" # Systemd unique machine id "otherwise, the system journal may fail to list earlier boots, etc" + ]; + }; + + # Write a list of the currently installed packages to /etc/current-system-packages + # environment.etc."current-system-packages".text = + # let + # packages = builtins.map (p: "${p.name}") config.environment.systemPackages; + # sortedUnique = builtins.sort builtins.lessThan (lib.unique packages); + # formatted = builtins.concatStringsSep "\n" sortedUnique; + # in + # formatted; + + # nixpkgs.overlays = [ + # (final: prev: { + # foot = throw "foo"; + # }) + # ]; + + nixpkgs.overlays = + let + disableTests = ( + package_name: + (final: prev: { + "${package_name}" = prev."${package_name}".overrideAttrs (old: { + doCheck = false; + doInstallCheck = false; + }); + }) + ); + in + [ + # (final: prev: { + # imagemagick = prev.imagemagick.overrideAttrs (old: rec { + # # 7.1.2-6 seems to no longer exist, so use 7.1.2-7 + # version = "7.1.2-7"; + + # src = final.fetchFromGitHub { + # owner = "ImageMagick"; + # repo = "ImageMagick"; + # tag = version; + # hash = "sha256-9ARCYftoXiilpJoj+Y+aLCEqLmhHFYSrHfgA5DQHbGo="; + # }; + # }); + # }) + # (final: prev: { + # grub2 = (final.callPackage ./package/grub { }); + # }) + (final: prev: { + inherit (final.unoptimized) + libtpms + ; + }) + ]; + + # This option defines the first version of NixOS you have installed on this particular machine, + # and is used to maintain compatibility with application data (e.g. databases) created on older NixOS versions. + # + # Most users should NEVER change this value after the initial install, for any reason, + # even if you've upgraded your system to a new NixOS release. + # + # This value does NOT affect the Nixpkgs version your packages and OS are pulled from, + # so changing it will NOT upgrade your system - see https://nixos.org/manual/nixos/stable/#sec-upgrading for how + # to actually do that. + # + # This value being lower than the current NixOS release does NOT mean your system is + # out of date, out of support, or vulnerable. + # + # Do NOT change this value unless you have manually inspected all the changes it would make to your configuration, + # and migrated your data accordingly. + # + # For more information, see `man configuration.nix` or https://nixos.org/manual/nixos/stable/options#opt-system.stateVersion . + system.stateVersion = "24.11"; # Did you read the comment? + }; +} diff --git a/nix/kubernetes/flake.lock b/nix/kubernetes/flake.lock new file mode 100644 index 0000000..28ca666 --- /dev/null +++ b/nix/kubernetes/flake.lock @@ -0,0 +1,256 @@ +{ + "nodes": { + "crane": { + "locked": { + "lastModified": 1731098351, + "narHash": "sha256-HQkYvKvaLQqNa10KEFGgWHfMAbWBfFp+4cAgkut+NNE=", + "owner": "ipetkov", + "repo": "crane", + "rev": "ef80ead953c1b28316cc3f8613904edc2eb90c28", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "repo": "crane", + "type": "github" + } + }, + "disko": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1769524058, + "narHash": "sha256-zygdD6X1PcVNR2PsyK4ptzrVEiAdbMqLos7utrMDEWE=", + "owner": "nix-community", + "repo": "disko", + "rev": "71a3fc97d80881e91710fe721f1158d3b96ae14d", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "disko", + "type": "github" + } + }, + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "lanzaboote", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1730504689, + "narHash": "sha256-hgmguH29K2fvs9szpq2r3pz2/8cJd2LPS+b4tfNFCwE=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "506278e768c2a08bec68eb62932193e341f55c90", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "gitignore": { + "inputs": { + "nixpkgs": [ + "lanzaboote", + "pre-commit-hooks-nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1709087332, + "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=", + "owner": "hercules-ci", + "repo": "gitignore.nix", + "rev": "637db329424fd7e46cf4185293b9cc8c88c95394", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "gitignore.nix", + "type": "github" + } + }, + "home-manager": { + "inputs": { + "nixpkgs": [ + "impermanence", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1768598210, + "narHash": "sha256-kkgA32s/f4jaa4UG+2f8C225Qvclxnqs76mf8zvTVPg=", + "owner": "nix-community", + "repo": "home-manager", + "rev": "c47b2cc64a629f8e075de52e4742de688f930dc6", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "home-manager", + "type": "github" + } + }, + "impermanence": { + "inputs": { + "home-manager": "home-manager", + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1769548169, + "narHash": "sha256-03+JxvzmfwRu+5JafM0DLbxgHttOQZkUtDWBmeUkN8Y=", + "owner": "nix-community", + "repo": "impermanence", + "rev": "7b1d382faf603b6d264f58627330f9faa5cba149", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "impermanence", + "type": "github" + } + }, + "lanzaboote": { + "inputs": { + "crane": "crane", + "flake-compat": "flake-compat", + "flake-parts": "flake-parts", + "nixpkgs": [ + "nixpkgs" + ], + "pre-commit-hooks-nix": "pre-commit-hooks-nix", + "rust-overlay": "rust-overlay" + }, + "locked": { + "lastModified": 1737639419, + "narHash": "sha256-AEEDktApTEZ5PZXNDkry2YV2k6t0dTgLPEmAZbnigXU=", + "owner": "nix-community", + "repo": "lanzaboote", + "rev": "a65905a09e2c43ff63be8c0e86a93712361f871e", + "type": "github" + }, + "original": { + "owner": "nix-community", + "ref": "v0.4.2", + "repo": "lanzaboote", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1770197578, + "narHash": "sha256-AYqlWrX09+HvGs8zM6ebZ1pwUqjkfpnv8mewYwAo+iM=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-stable": { + "locked": { + "lastModified": 1730741070, + "narHash": "sha256-edm8WG19kWozJ/GqyYx2VjW99EdhjKwbY3ZwdlPAAlo=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "d063c1dd113c91ab27959ba540c0d9753409edf3", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-24.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "pre-commit-hooks-nix": { + "inputs": { + "flake-compat": [ + "lanzaboote", + "flake-compat" + ], + "gitignore": "gitignore", + "nixpkgs": [ + "lanzaboote", + "nixpkgs" + ], + "nixpkgs-stable": "nixpkgs-stable" + }, + "locked": { + "lastModified": 1731363552, + "narHash": "sha256-vFta1uHnD29VUY4HJOO/D6p6rxyObnf+InnSMT4jlMU=", + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "rev": "cd1af27aa85026ac759d5d3fccf650abe7e1bbf0", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "type": "github" + } + }, + "root": { + "inputs": { + "disko": "disko", + "impermanence": "impermanence", + "lanzaboote": "lanzaboote", + "nixpkgs": "nixpkgs" + } + }, + "rust-overlay": { + "inputs": { + "nixpkgs": [ + "lanzaboote", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1731897198, + "narHash": "sha256-Ou7vLETSKwmE/HRQz4cImXXJBr/k9gp4J4z/PF8LzTE=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "0be641045af6d8666c11c2c40e45ffc9667839b5", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/nix/kubernetes/flake.nix b/nix/kubernetes/flake.nix new file mode 100644 index 0000000..bfbbd02 --- /dev/null +++ b/nix/kubernetes/flake.nix @@ -0,0 +1,163 @@ +# Get a repl for this flake +# nix repl --expr "builtins.getFlake \"$PWD\"" + +# TODO maybe use `nix eval --raw .#odo.iso.outPath` + +# +# Install on a new machine: +# +# Set +# me.disko.enable = true; +# me.disko.offline.enable = true; +# +# Run +# doas disko --mode destroy,format,mount hosts/recovery/disk-config.nix +# doas nixos-install --substituters "http://10.0.2.2:8080?trusted=1 https://cache.nixos.org/" --flake ".#recovery" + +{ + description = "My system configuration"; + + inputs = { + impermanence = { + url = "github:nix-community/impermanence"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + lanzaboote = { + url = "github:nix-community/lanzaboote/v0.4.2"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + disko = { + url = "github:nix-community/disko"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + }; + + outputs = + { + self, + nixpkgs, + disko, + impermanence, + lanzaboote, + ... + }: + let + forAllSystems = nixpkgs.lib.genAttrs nixpkgs.lib.systems.flakeExposed; + nodes = { + controller0 = { + system = "x86_64-linux"; + }; + }; + nixosConfigs = builtins.mapAttrs ( + hostname: nodeConfig: format: + nixpkgs.lib.nixosSystem { + specialArgs = { + inherit self; + + this_nixos_config = self.nixosConfigurations."${hostname}"; + + all_nixos_configs = self.nixosConfigurations; + }; + modules = [ + impermanence.nixosModules.impermanence + lanzaboote.nixosModules.lanzaboote + disko.nixosModules.disko + ./configuration.nix + (./. + "/hosts/${hostname}") + (./. + "/formats/${format}.nix") + { + config = { + nixpkgs.hostPlatform.system = nodeConfig.system; + nixpkgs.overlays = [ + (final: prev: { + # stable = nixpkgs-stable.legacyPackages."${prev.stdenv.hostPlatform.system}"; + unoptimized = import nixpkgs { + system = prev.stdenv.hostPlatform.system; + hostPlatform.gcc.arch = "default"; + hostPlatform.gcc.tune = "default"; + }; + }) + ]; + }; + } + ( + { + config, + lib, + pkgs, + ... + }: + let + repl_path = toString ./.; + nix-self-repl = pkgs.writeShellScriptBin "nix-self-repl" '' + source /etc/set-environment + nix repl "${repl_path}/repl.nix" "$@" + ''; + # If we wanted the current version of a flake then we'd just launch + # nix repl + # and then run: + # :lf /path/to/flake + in + { + config = { + environment.systemPackages = lib.mkIf config.nix.enable [ nix-self-repl ]; + }; + } + ) + ]; + } + ) nodes; + installerConfig = + hostname: nodeConfig: + nixpkgs.lib.nixosSystem { + specialArgs = { + targetSystem = self.nixosConfigurations."${hostname}"; + }; + modules = [ + ./formats/installer.nix + ( + { + config, + lib, + pkgs, + ... + }: + let + repl_path = toString ./.; + nix-self-repl = pkgs.writeShellScriptBin "nix-self-repl" '' + source /etc/set-environment + nix repl "${repl_path}/repl.nix" "$@" + ''; + # If we wanted the current version of a flake then we'd just launch + # nix repl + # and then run: + # :lf /path/to/flake + in + { + config = { + environment.systemPackages = lib.mkIf config.nix.enable [ nix-self-repl ]; + }; + } + ) + ({ nixpkgs.hostPlatform.system = nodeConfig.system; }) + ]; + }; + in + { + nixosConfigurations = (builtins.mapAttrs (name: value: value "toplevel") nixosConfigs); + } + // { + packages = ( + forAllSystems ( + system: + (builtins.mapAttrs (hostname: nodeConfig: { + iso = (nixosConfigs."${hostname}" "iso").config.system.build.isoImage; + vm_iso = (nixosConfigs."${hostname}" "vm_iso").config.system.build.isoImage; + sd = (nixosConfigs."${hostname}" "sd").config.system.build.sdImage; + installer = (installerConfig hostname nodes."${hostname}").config.system.build.isoImage; + }) (nixpkgs.lib.attrsets.filterAttrs (hostname: nodeConfig: nodeConfig.system == system) nodes)) + ) + ); + }; +} diff --git a/nix/kubernetes/formats/installer.nix b/nix/kubernetes/formats/installer.nix new file mode 100644 index 0000000..93b1187 --- /dev/null +++ b/nix/kubernetes/formats/installer.nix @@ -0,0 +1,74 @@ +{ + config, + pkgs, + lib, + modulesPath, + targetSystem, + ... +}: +let + installer = pkgs.writeShellApplication { + name = "installer"; + runtimeInputs = with pkgs; [ + # clevis + dosfstools + e2fsprogs + gawk + nixos-install-tools + util-linux + config.nix.package + ]; + text = '' + set -euo pipefail + + ${targetSystem.config.system.build.diskoScript} + + nixos-install --no-channel-copy --no-root-password --option substituters "" --system ${targetSystem.config.system.build.toplevel} + ''; + }; + installerFailsafe = pkgs.writeShellScript "failsafe" '' + ${lib.getExe installer} || echo "ERROR: Installation failure!" + sleep 3600 + ''; +in +{ + imports = [ + (modulesPath + "/installer/cd-dvd/iso-image.nix") + (modulesPath + "/profiles/all-hardware.nix") + ]; + + # boot.kernelPackages = pkgs.linuxPackagesFor pkgs.linux_6_17; + # boot.zfs.package = pkgs.zfs_unstable; + boot.kernelPackages = pkgs.linuxPackagesFor pkgs.linux; + boot.kernelParams = [ + "quiet" + "systemd.unit=getty.target" + ]; + boot.supportedFilesystems.zfs = true; + boot.initrd.systemd.enable = true; + + networking.hostId = "04581ecf"; + + isoImage.makeEfiBootable = true; + isoImage.makeUsbBootable = true; + isoImage.squashfsCompression = "zstd -Xcompression-level 15"; + + environment.systemPackages = [ + installer + ]; + + systemd.services."getty@tty1" = { + overrideStrategy = "asDropin"; + serviceConfig = { + ExecStart = [ + "" + installerFailsafe + ]; + Restart = "no"; + StandardInput = "null"; + }; + }; + + # system.stateVersion = lib.mkDefault lib.trivial.release; + system.stateVersion = "24.11"; +} diff --git a/nix/kubernetes/formats/iso.nix b/nix/kubernetes/formats/iso.nix new file mode 100644 index 0000000..974d710 --- /dev/null +++ b/nix/kubernetes/formats/iso.nix @@ -0,0 +1,36 @@ +{ + config, + lib, + modulesPath, + pkgs, + ... +}: + +{ + imports = [ + (modulesPath + "/installer/cd-dvd/iso-image.nix") + ]; + + config = { + isoImage.makeEfiBootable = true; + isoImage.makeUsbBootable = true; + + networking.dhcpcd.enable = true; + networking.useDHCP = true; + + me.buildingPortable = true; + me.disko.enable = true; + me.disko.offline.enable = true; + me.mountPersistence = lib.mkForce false; + # me.optimizations.enable = lib.mkForce false; + + # Not doing image_based_appliance because this might be an install ISO, in which case we'd need nix to do the install. + # me.image_based_appliance.enable = true; + + # TODO: Should I use this instead of doing a mkIf for the disk config? + # disko.enableConfig = false; + + # Faster image generation for testing/development. + isoImage.squashfsCompression = "zstd -Xcompression-level 15"; + }; +} diff --git a/nix/kubernetes/formats/sd.nix b/nix/kubernetes/formats/sd.nix new file mode 100644 index 0000000..208195f --- /dev/null +++ b/nix/kubernetes/formats/sd.nix @@ -0,0 +1,32 @@ +{ + modulesPath, + ... +}: + +{ + imports = [ + (modulesPath + "/installer/sd-card/sd-image.nix") + ]; + + config = { + isoImage.makeEfiBootable = true; + isoImage.makeUsbBootable = true; + + boot.loader.grub.enable = false; + boot.loader.generic-extlinux-compatible.enable = true; + + # TODO: image based appliance? + + # TODO: Maybe this? + # fileSystems = { + # "/" = { + # device = "/dev/disk/by-label/NIXOS_SD"; + # fsType = "ext4"; + # options = [ + # "noatime" + # "norelatime" + # ]; + # }; + # }; + }; +} diff --git a/nix/kubernetes/formats/toplevel.nix b/nix/kubernetes/formats/toplevel.nix new file mode 100644 index 0000000..ffcd441 --- /dev/null +++ b/nix/kubernetes/formats/toplevel.nix @@ -0,0 +1 @@ +{ } diff --git a/nix/kubernetes/formats/vm_iso.nix b/nix/kubernetes/formats/vm_iso.nix new file mode 100644 index 0000000..ef487de --- /dev/null +++ b/nix/kubernetes/formats/vm_iso.nix @@ -0,0 +1,22 @@ +{ + lib, + modulesPath, + ... +}: + +{ + imports = [ + (modulesPath + "/installer/cd-dvd/iso-image.nix") + (modulesPath + "/profiles/qemu-guest.nix") # VirtIO kernel modules + ]; + + config = { + isoImage.makeEfiBootable = true; + isoImage.makeUsbBootable = true; + + networking.dhcpcd.enable = true; + networking.useDHCP = true; + + me.image_based_appliance.enable = true; + }; +} diff --git a/nix/kubernetes/hosts/controller0/DEPLOY_BOOT b/nix/kubernetes/hosts/controller0/DEPLOY_BOOT new file mode 100755 index 0000000..2648e46 --- /dev/null +++ b/nix/kubernetes/hosts/controller0/DEPLOY_BOOT @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=controller0 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild boot --flake "$DIR/../../#controller0" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller0/DEPLOY_SWITCH b/nix/kubernetes/hosts/controller0/DEPLOY_SWITCH new file mode 100755 index 0000000..e8c3f61 --- /dev/null +++ b/nix/kubernetes/hosts/controller0/DEPLOY_SWITCH @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=controller0 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild switch --flake "$DIR/../../#controller0" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller0/ISO b/nix/kubernetes/hosts/controller0/ISO new file mode 100755 index 0000000..f86f8df --- /dev/null +++ b/nix/kubernetes/hosts/controller0/ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#controller0.iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller0/SELF_BOOT b/nix/kubernetes/hosts/controller0/SELF_BOOT new file mode 100755 index 0000000..c447544 --- /dev/null +++ b/nix/kubernetes/hosts/controller0/SELF_BOOT @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild boot --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller0" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller0/SELF_BUILD b/nix/kubernetes/hosts/controller0/SELF_BUILD new file mode 100755 index 0000000..e92bea4 --- /dev/null +++ b/nix/kubernetes/hosts/controller0/SELF_BUILD @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild build --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller0" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller0/SELF_SWITCH b/nix/kubernetes/hosts/controller0/SELF_SWITCH new file mode 100755 index 0000000..c86ce07 --- /dev/null +++ b/nix/kubernetes/hosts/controller0/SELF_SWITCH @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild switch --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller0" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller0/VM_ISO b/nix/kubernetes/hosts/controller0/VM_ISO new file mode 100755 index 0000000..98bcd23 --- /dev/null +++ b/nix/kubernetes/hosts/controller0/VM_ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#controller0.vm_iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller0/default.nix b/nix/kubernetes/hosts/controller0/default.nix new file mode 100644 index 0000000..847874f --- /dev/null +++ b/nix/kubernetes/hosts/controller0/default.nix @@ -0,0 +1,123 @@ +# MANUAL: On client machines generate signing keys: +# nix-store --generate-binary-cache-key some-name /persist/manual/nix/nix-cache-key.sec /persist/manual/nix/nix-cache-key.pub +# +# Trust other machines and add the substituters: +# nix.binaryCachePublicKeys = [ "some-name:AzNW1MOlkNEsUAXS1jIFZ1QCFKXjV+Y/LrF37quAZ1A=" ]; +# nix.binaryCaches = [ "https://test.example/nix-cache" ]; + +{ + config, + lib, + pkgs, + ... +}: +{ + imports = [ + ./hardware-configuration.nix + ./vm_disk.nix + ]; + + config = { + networking = + let + interface = "enp0s2"; + in + { + # Generate with `head -c4 /dev/urandom | od -A none -t x4` + hostId = "769e1349"; + + hostName = "controller0"; # Define your hostname. + + interfaces = { + "${interface}" = { + ipv4.addresses = [ + { + address = "10.215.1.221"; + prefixLength = 24; + } + ]; + + ipv6.addresses = [ + { + address = "2620:11f:7001:7:ffff:ffff:0ad7:01dd"; + prefixLength = 64; + } + ]; + }; + }; + defaultGateway = "10.215.1.1"; + defaultGateway6 = { + # address = "2620:11f:7001:7::1"; + address = "2620:11f:7001:7:ffff:ffff:0ad7:0101"; + inherit interface; + }; + nameservers = [ + "10.215.1.1" + ]; + + dhcpcd.enable = lib.mkForce false; + useDHCP = lib.mkForce false; + }; + + time.timeZone = "America/New_York"; + i18n.defaultLocale = "en_US.UTF-8"; + + me.boot.enable = true; + me.boot.secure = false; + me.mountPersistence = true; + boot.loader.timeout = lib.mkForce 0; # We can always generate a new ISO if we need to access other boot options. + + me.optimizations = { + enable = true; + arch = "znver4"; + # build_arch = "x86-64-v3"; + system_features = [ + "gccarch-znver4" + "gccarch-skylake" + "gccarch-kabylake" + # "gccarch-alderlake" missing WAITPKG + "gccarch-x86-64-v3" + "gccarch-x86-64-v4" + "benchmark" + "big-parallel" + "kvm" + "nixos-test" + ]; + }; + + # Mount tmpfs at /tmp + boot.tmp.useTmpfs = true; + + # Enable TRIM + # services.fstrim.enable = lib.mkDefault true; + + # nix.optimise.automatic = true; + # nix.optimise.dates = [ "03:45" ]; + # nix.optimise.persistent = true; + + environment.systemPackages = with pkgs; [ + htop + ]; + + # nix.sshServe.enable = true; + # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ]; + + me.etcd.cluster_name = "put a nix on it"; + me.etcd.internal_ip = [ + # "10.215.1.221" + "[2620:11f:7001:7:ffff:ffff:0ad7:01dd]" + ]; + me.etcd.initial_cluster = [ + # "controller0=https://10.215.1.221:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01dd + # "controller1=https://10.215.1.222:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01de + # "controller2=https://10.215.1.223:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01df + "controller0=https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2380" # 10.215.1.221 + "controller1=https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2380" # 10.215.1.222 + "controller2=https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2380" # 10.215.1.223 + ]; + + me.dont_use_substituters.enable = true; + me.etcd.enable = true; + me.minimal_base.enable = true; + }; +} diff --git a/nix/kubernetes/hosts/controller0/hardware-configuration.nix b/nix/kubernetes/hosts/controller0/hardware-configuration.nix new file mode 100644 index 0000000..6029e08 --- /dev/null +++ b/nix/kubernetes/hosts/controller0/hardware-configuration.nix @@ -0,0 +1,31 @@ +{ + config, + lib, + modulesPath, + ... +}: + +{ + imports = [ + (modulesPath + "/installer/scan/not-detected.nix") + ]; + + config = { + boot.initrd.availableKernelModules = [ + "nvme" + "xhci_pci" + "thunderbolt" + ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ ]; + boot.extraModulePackages = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + # networking.useDHCP = lib.mkDefault true; + # networking.interfaces.eno1.useDHCP = lib.mkDefault true; + # networking.interfaces.wlp58s0.useDHCP = lib.mkDefault true; + }; +} diff --git a/nix/kubernetes/hosts/controller0/vm_disk.nix b/nix/kubernetes/hosts/controller0/vm_disk.nix new file mode 100644 index 0000000..6c23789 --- /dev/null +++ b/nix/kubernetes/hosts/controller0/vm_disk.nix @@ -0,0 +1,71 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + config = { + # Mount the local disk + fileSystems = lib.mkIf config.me.mountPersistence { + "/.disk" = lib.mkForce { + device = "bind9p"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + + "/persist" = { + fsType = "none"; + device = "/.disk/persist"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.disk/persist" + ]; + neededForBoot = true; + }; + + "/state" = { + fsType = "none"; + device = "/.disk/state"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.disk/state" + ]; + neededForBoot = true; + }; + + "/k8spv" = lib.mkForce { + device = "k8spv"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + }; + }; +} diff --git a/nix/kubernetes/keys/Makefile b/nix/kubernetes/keys/Makefile new file mode 100644 index 0000000..1f2bb24 --- /dev/null +++ b/nix/kubernetes/keys/Makefile @@ -0,0 +1,357 @@ +SHELL := bash +.ONESHELL: +.SHELLFLAGS := -eu -o pipefail -c +.DELETE_ON_ERROR: +MAKEFLAGS += --warn-undefined-variables +MAKEFLAGS += --no-builtin-rules +OUT=generated + +ifeq ($(origin .RECIPEPREFIX), undefined) + $(error This Make does not support .RECIPEPREFIX. Please use GNU Make 4.0 or later) +endif +.RECIPEPREFIX = > + +KUBERNETES_PUBLIC_ADDRESS := 74.80.180.138 +WORKERS := worker0 worker1 worker2 controller0 controller1 controller2 + +.PHONY: all +all: \ + $(OUT)/ca-key.pem \ + $(OUT)/admin-key.pem \ + $(OUT)/worker0-key.pem \ + $(OUT)/worker1-key.pem \ + $(OUT)/worker2-key.pem \ + $(OUT)/controller0-proxy-key.pem \ + $(OUT)/controller1-proxy-key.pem \ + $(OUT)/controller2-proxy-key.pem \ + $(OUT)/kube-controller-manager-key.pem \ + $(OUT)/kube-proxy-key.pem \ + $(OUT)/kube-scheduler-key.pem \ + $(OUT)/kubernetes-key.pem \ + $(OUT)/service-account-key.pem \ + $(OUT)/worker0.kubeconfig \ + $(OUT)/worker1.kubeconfig \ + $(OUT)/worker2.kubeconfig \ + $(OUT)/controller0.kubeconfig \ + $(OUT)/controller1.kubeconfig \ + $(OUT)/controller2.kubeconfig \ + $(OUT)/kube-proxy.kubeconfig \ + $(OUT)/kube-controller-manager.kubeconfig \ + $(OUT)/kube-scheduler.kubeconfig \ + $(OUT)/admin.kubeconfig \ + $(OUT)/encryption-config.yaml \ + $(OUT)/remote_admin.kubeconfig \ + $(OUT)/requestheader-client-ca-key.pem + +.PHONY: clean +clean: +> rm -rf $(OUT) + +# Requestheader client ca +$(OUT)/requestheader-client-ca-key.pem: requestheader-client-ca-csr.json ca-config.json +> @mkdir -p $(@D) +> cd $(@D) && cfssl gencert -initca ../requestheader-client-ca-csr.json | cfssljson -bare requestheader-client-ca + +# Certificate authority +$(OUT)/ca-key.pem: ca-csr.json ca-config.json +> @mkdir -p $(@D) +> cd $(@D) && cfssl gencert -initca ../ca-csr.json | cfssljson -bare ca + +# Admin client certificate +$(OUT)/admin-key.pem: admin-csr.json ca-config.json +> @mkdir -p $(@D) +> cd $(@D) && cfssl gencert \ +> -ca=ca.pem \ +> -ca-key=ca-key.pem \ +> -config=../ca-config.json \ +> -profile=kubernetes \ +> ../admin-csr.json | cfssljson -bare admin + +# Worker kubelet client certificate +$(OUT)/worker0-key.pem: worker0-csr.json ca-config.json +> @mkdir -p $(@D) +> cd $(@D) && cfssl gencert \ +> -ca=ca.pem \ +> -ca-key=ca-key.pem \ +> -config=../ca-config.json \ +> -hostname=worker0,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.207 \ +> -profile=kubernetes \ +> ../worker0-csr.json | cfssljson -bare worker0 + +# Worker kubelet client certificate +$(OUT)/worker1-key.pem: worker1-csr.json ca-config.json +> @mkdir -p $(@D) +> cd $(@D) && cfssl gencert \ +> -ca=ca.pem \ +> -ca-key=ca-key.pem \ +> -config=../ca-config.json \ +> -hostname=worker1,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.208 \ +> -profile=kubernetes \ +> ../worker1-csr.json | cfssljson -bare worker1 + +# Worker kubelet client certificate +$(OUT)/worker2-key.pem: worker2-csr.json ca-config.json +> @mkdir -p $(@D) +> cd $(@D) && cfssl gencert \ +> -ca=ca.pem \ +> -ca-key=ca-key.pem \ +> -config=../ca-config.json \ +> -hostname=worker2,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.209 \ +> -profile=kubernetes \ +> ../worker2-csr.json | cfssljson -bare worker2 + +# Controller kubelet client certificate +$(OUT)/controller0-key.pem: controller0-csr.json ca-config.json +> @mkdir -p $(@D) +> cd $(@D) && cfssl gencert \ +> -ca=ca.pem \ +> -ca-key=ca-key.pem \ +> -config=../ca-config.json \ +> -hostname=controller0,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.204 \ +> -profile=kubernetes \ +> ../controller0-csr.json | cfssljson -bare controller0 + +# Controller kubelet client certificate +$(OUT)/controller1-key.pem: controller1-csr.json ca-config.json +> @mkdir -p $(@D) +> cd $(@D) && cfssl gencert \ +> -ca=ca.pem \ +> -ca-key=ca-key.pem \ +> -config=../ca-config.json \ +> -hostname=controller1,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.205 \ +> -profile=kubernetes \ +> ../controller1-csr.json | cfssljson -bare controller1 + +# Controller kubelet client certificate +$(OUT)/controller2-key.pem: controller2-csr.json ca-config.json +> @mkdir -p $(@D) +> cd $(@D) && cfssl gencert \ +> -ca=ca.pem \ +> -ca-key=ca-key.pem \ +> -config=../ca-config.json \ +> -hostname=controller2,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.206 \ +> -profile=kubernetes \ +> ../controller2-csr.json | cfssljson -bare controller2 + +# Controller kubelet client certificate +$(OUT)/controller0-proxy-key.pem: controller0-proxy-csr.json ca-config.json $(OUT)/requestheader-client-ca-key.pem +> @mkdir -p $(@D) +> cd $(@D) && cfssl gencert \ +> -ca=requestheader-client-ca.pem \ +> -ca-key=requestheader-client-ca-key.pem \ +> -config=../ca-config.json \ +> -hostname=controller0,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.204 \ +> -profile=kubernetes \ +> ../controller0-proxy-csr.json | cfssljson -bare controller0-proxy + +# Controller kubelet client certificate +$(OUT)/controller1-proxy-key.pem: controller1-proxy-csr.json ca-config.json $(OUT)/requestheader-client-ca-key.pem +> @mkdir -p $(@D) +> cd $(@D) && cfssl gencert \ +> -ca=requestheader-client-ca.pem \ +> -ca-key=requestheader-client-ca-key.pem \ +> -config=../ca-config.json \ +> -hostname=controller1,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.205 \ +> -profile=kubernetes \ +> ../controller1-proxy-csr.json | cfssljson -bare controller1-proxy + +# Controller kubelet client certificate +$(OUT)/controller2-proxy-key.pem: controller2-proxy-csr.json ca-config.json $(OUT)/requestheader-client-ca-key.pem +> @mkdir -p $(@D) +> cd $(@D) && cfssl gencert \ +> -ca=requestheader-client-ca.pem \ +> -ca-key=requestheader-client-ca-key.pem \ +> -config=../ca-config.json \ +> -hostname=controller2,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.206 \ +> -profile=kubernetes \ +> ../controller2-proxy-csr.json | cfssljson -bare controller2-proxy + +# Controller manager client certificate +$(OUT)/kube-controller-manager-key.pem: kube-controller-manager-csr.json ca-config.json +> @mkdir -p $(@D) +> cd $(@D) && cfssl gencert \ +> -ca=ca.pem \ +> -ca-key=ca-key.pem \ +> -config=../ca-config.json \ +> -profile=kubernetes \ +> ../kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager + +# Kube proxy client certificate +$(OUT)/kube-proxy-key.pem: kube-proxy-csr.json ca-config.json +> @mkdir -p $(@D) +> cd $(@D) && cfssl gencert \ +> -ca=ca.pem \ +> -ca-key=ca-key.pem \ +> -config=../ca-config.json \ +> -profile=kubernetes \ +> ../kube-proxy-csr.json | cfssljson -bare kube-proxy + +# Kube scheduler client certificate +$(OUT)/kube-scheduler-key.pem: kube-scheduler-csr.json ca-config.json +> @mkdir -p $(@D) +> cd $(@D) && cfssl gencert \ +> -ca=ca.pem \ +> -ca-key=ca-key.pem \ +> -config=../ca-config.json \ +> -profile=kubernetes \ +> ../kube-scheduler-csr.json | cfssljson -bare kube-scheduler + +# Kuberntes API server certificate +# TODO: Replace 10.32.0.1 with kubernetes api server local ip address from lab 8 +$(OUT)/kubernetes-key.pem: kubernetes-csr.json ca-config.json +> @mkdir -p $(@D) +> cd $(@D) && cfssl gencert \ +> -ca=ca.pem \ +> -ca-key=ca-key.pem \ +> -config=../ca-config.json \ +> -hostname=10.197.0.1,10.0.0.1,10.215.1.204,10.215.1.205,10.215.1.206,10.215.1.207,10.215.1.208,10.215.1.209,$(KUBERNETES_PUBLIC_ADDRESS),127.0.0.1,kubernetes,kubernetes.default,kubernetes.default.svc,kubernetes.default.svc.cluster,kubernetes.svc.cluster.local \ +> -profile=kubernetes \ +> ../kubernetes-csr.json | cfssljson -bare kubernetes + +# Service account keypair +$(OUT)/service-account-key.pem: service-account-csr.json ca-config.json +> @mkdir -p $(@D) +> cd $(@D) && cfssl gencert \ +> -ca=ca.pem \ +> -ca-key=ca-key.pem \ +> -config=../ca-config.json \ +> -profile=kubernetes \ +> ../service-account-csr.json | cfssljson -bare service-account + +# Generate worker kubeconfigs +$(patsubst %,$(OUT)/%.kubeconfig,$(WORKERS)): $(OUT)/%.kubeconfig: $(OUT)/%-key.pem $(OUT)/%.pem +> @mkdir -p $(@D) +> kubectl config set-cluster kubernetes-the-hard-way \ +> --certificate-authority=$(OUT)/ca.pem \ +> --embed-certs=true \ +> --server=https://$(KUBERNETES_PUBLIC_ADDRESS):6443 \ +> --kubeconfig=$@ +> +> kubectl config set-credentials system:node:$* \ +> --client-certificate=$(OUT)/$*.pem \ +> --client-key=$(OUT)/$*-key.pem \ +> --embed-certs=true \ +> --kubeconfig=$@ +> +> kubectl config set-context default \ +> --cluster=kubernetes-the-hard-way \ +> --user=system:node:$* \ +> --kubeconfig=$@ +> +> kubectl config use-context default --kubeconfig=$@ + +# Generate kube-proxy kubeconfig +$(OUT)/kube-proxy.kubeconfig: $(OUT)/%.kubeconfig: $(OUT)/%-key.pem $(OUT)/%.pem +> @mkdir -p $(@D) +> kubectl config set-cluster kubernetes-the-hard-way \ +> --certificate-authority=$(OUT)/ca.pem \ +> --embed-certs=true \ +> --server=https://$(KUBERNETES_PUBLIC_ADDRESS):6443 \ +> --kubeconfig=$@ +> +> kubectl config set-credentials system:$* \ +> --client-certificate=$(OUT)/$*.pem \ +> --client-key=$(OUT)/$*-key.pem \ +> --embed-certs=true \ +> --kubeconfig=$@ +> +> kubectl config set-context default \ +> --cluster=kubernetes-the-hard-way \ +> --user=system:$* \ +> --kubeconfig=$@ +> +> kubectl config use-context default --kubeconfig=$@ + +# Generate kube-controller-manager kubeconfig +$(OUT)/kube-controller-manager.kubeconfig: $(OUT)/%.kubeconfig: $(OUT)/%-key.pem $(OUT)/%.pem +> @mkdir -p $(@D) +> kubectl config set-cluster kubernetes-the-hard-way \ +> --certificate-authority=$(OUT)/ca.pem \ +> --embed-certs=true \ +> --server=https://127.0.0.1:6443 \ +> --kubeconfig=$@ +> +> kubectl config set-credentials system:$* \ +> --client-certificate=$(OUT)/$*.pem \ +> --client-key=$(OUT)/$*-key.pem \ +> --embed-certs=true \ +> --kubeconfig=$@ +> +> kubectl config set-context default \ +> --cluster=kubernetes-the-hard-way \ +> --user=system:$* \ +> --kubeconfig=$@ +> +> kubectl config use-context default --kubeconfig=$@ + +# Generate kube-scheduler kubeconfig +$(OUT)/kube-scheduler.kubeconfig: $(OUT)/%.kubeconfig: $(OUT)/%-key.pem $(OUT)/%.pem +> @mkdir -p $(@D) +> kubectl config set-cluster kubernetes-the-hard-way \ +> --certificate-authority=$(OUT)/ca.pem \ +> --embed-certs=true \ +> --server=https://127.0.0.1:6443 \ +> --kubeconfig=$@ +> +> kubectl config set-credentials system:$* \ +> --client-certificate=$(OUT)/$*.pem \ +> --client-key=$(OUT)/$*-key.pem \ +> --embed-certs=true \ +> --kubeconfig=$@ +> +> kubectl config set-context default \ +> --cluster=kubernetes-the-hard-way \ +> --user=system:$* \ +> --kubeconfig=$@ +> +> kubectl config use-context default --kubeconfig=$@ + +# Generate admin kubeconfig +$(OUT)/admin.kubeconfig: $(OUT)/%.kubeconfig: $(OUT)/%-key.pem $(OUT)/%.pem +> @mkdir -p $(@D) +> kubectl config set-cluster kubernetes-the-hard-way \ +> --certificate-authority=$(OUT)/ca.pem \ +> --embed-certs=true \ +> --server=https://127.0.0.1:6443 \ +> --kubeconfig=$@ +> +> kubectl config set-credentials $* \ +> --client-certificate=$(OUT)/$*.pem \ +> --client-key=$(OUT)/$*-key.pem \ +> --embed-certs=true \ +> --kubeconfig=$@ +> +> kubectl config set-context default \ +> --cluster=kubernetes-the-hard-way \ +> --user=$* \ +> --kubeconfig=$@ +> +> kubectl config use-context default --kubeconfig=$@ + +# Generate data encryption key for encrypting data at rest +$(OUT)/encryption-config.yaml: +> @mkdir -p $(@D) +> ENCRYPTION_KEY=$(shell head -c 32 /dev/urandom | base64) +> cat encryption-config-template.yaml | sed "s@ENCRYPTION_KEY@$$ENCRYPTION_KEY@g" > $@ + +# Generate remote admin kubeconfig +$(OUT)/remote_admin.kubeconfig: $(OUT)/remote_%.kubeconfig: $(OUT)/%-key.pem $(OUT)/%.pem +> @mkdir -p $(@D) +> kubectl config set-cluster kubernetes-the-hard-way \ +> --certificate-authority=$(OUT)/ca.pem \ +> --embed-certs=true \ +> --server=https://$(KUBERNETES_PUBLIC_ADDRESS):6443 \ +> --kubeconfig=$@ +> +> kubectl config set-credentials $* \ +> --client-certificate=$(OUT)/$*.pem \ +> --client-key=$(OUT)/$*-key.pem \ +> --embed-certs=true \ +> --kubeconfig=$@ +> +> kubectl config set-context default \ +> --cluster=kubernetes-the-hard-way \ +> --user=$* \ +> --kubeconfig=$@ +> +> kubectl config use-context default --kubeconfig=$@ diff --git a/nix/kubernetes/keys/flake.lock b/nix/kubernetes/keys/flake.lock new file mode 100644 index 0000000..42c9926 --- /dev/null +++ b/nix/kubernetes/keys/flake.lock @@ -0,0 +1,27 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1767892417, + "narHash": "sha256-dhhvQY67aboBk8b0/u0XB6vwHdgbROZT3fJAjyNh5Ww=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "3497aa5c9457a9d88d71fa93a4a8368816fbeeba", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/nix/kubernetes/keys/flake.nix b/nix/kubernetes/keys/flake.nix new file mode 100644 index 0000000..3fae683 --- /dev/null +++ b/nix/kubernetes/keys/flake.nix @@ -0,0 +1,31 @@ +{ + description = "Build keys to manually deploy to kubernetes cluster."; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + }; + + outputs = + { self, nixpkgs }: + let + forAllSystems = nixpkgs.lib.genAttrs nixpkgs.lib.systems.flakeExposed; + in + { + packages = forAllSystems ( + system: + let + pkgs = nixpkgs.legacyPackages.${system}; + appliedOverlay = self.overlays.default pkgs pkgs; + in + { + deploy_script = appliedOverlay.k8s.deploy_script; + default = appliedOverlay.k8s.keys; + } + ); + overlays.default = ( + final: prev: { + k8s = (final.callPackage ./scope.nix { inherit (final.lib) makeScope; }); + } + ); + }; +} diff --git a/nix/kubernetes/keys/package/k8s-ca/files/ca-csr.json b/nix/kubernetes/keys/package/k8s-ca/files/ca-csr.json new file mode 100644 index 0000000..8145e50 --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-ca/files/ca-csr.json @@ -0,0 +1,16 @@ +{ + "CN": "Kubernetes", + "key": { + "algo": "rsa", + "size": 2048 + }, + "names": [ + { + "C": "US", + "L": "Portland", + "O": "Kubernetes", + "OU": "CA", + "ST": "Oregon" + } + ] +} diff --git a/nix/kubernetes/keys/package/k8s-ca/package.nix b/nix/kubernetes/keys/package/k8s-ca/package.nix new file mode 100644 index 0000000..a2425cb --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-ca/package.nix @@ -0,0 +1,28 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + stdenv, + sqlite, + cfssl, + ... +}: +stdenv.mkDerivation (finalAttrs: { + name = "k8s-ca"; + nativeBuildInputs = [ cfssl ]; + buildInputs = [ ]; + + unpackPhase = "true"; + + installPhase = '' + mkdir -p "$out" + cd "$out" + cfssl gencert -initca ${./files/ca-csr.json} | cfssljson -bare ca + ''; +}) diff --git a/nix/kubernetes/keys/package/k8s-keys/files/ca-config.json b/nix/kubernetes/keys/package/k8s-keys/files/ca-config.json new file mode 100644 index 0000000..a63e0dd --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-keys/files/ca-config.json @@ -0,0 +1,13 @@ +{ + "signing": { + "default": { + "expiry": "8760h" + }, + "profiles": { + "kubernetes": { + "usages": ["signing", "key encipherment", "server auth", "client auth"], + "expiry": "8760h" + } + } + } +} diff --git a/nix/kubernetes/keys/package/k8s-keys/files/kubernetes-csr.json b/nix/kubernetes/keys/package/k8s-keys/files/kubernetes-csr.json new file mode 100644 index 0000000..778db63 --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-keys/files/kubernetes-csr.json @@ -0,0 +1,16 @@ +{ + "CN": "kubernetes", + "key": { + "algo": "rsa", + "size": 2048 + }, + "names": [ + { + "C": "US", + "L": "Portland", + "O": "Kubernetes", + "OU": "Kubernetes The Hard Way", + "ST": "Oregon" + } + ] +} diff --git a/nix/kubernetes/keys/package/k8s-keys/package.nix b/nix/kubernetes/keys/package/k8s-keys/package.nix new file mode 100644 index 0000000..a890aa3 --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-keys/package.nix @@ -0,0 +1,12 @@ +{ + k8s, + symlinkJoin, + ... +}: +symlinkJoin { + name = "k8s-keys"; + paths = [ + k8s.kubernetes + k8s.ca + ]; +} diff --git a/nix/kubernetes/keys/package/k8s-kubernetes/files/ca-config.json b/nix/kubernetes/keys/package/k8s-kubernetes/files/ca-config.json new file mode 100644 index 0000000..a63e0dd --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-kubernetes/files/ca-config.json @@ -0,0 +1,13 @@ +{ + "signing": { + "default": { + "expiry": "8760h" + }, + "profiles": { + "kubernetes": { + "usages": ["signing", "key encipherment", "server auth", "client auth"], + "expiry": "8760h" + } + } + } +} diff --git a/nix/kubernetes/keys/package/k8s-kubernetes/files/kubernetes-csr.json b/nix/kubernetes/keys/package/k8s-kubernetes/files/kubernetes-csr.json new file mode 100644 index 0000000..778db63 --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-kubernetes/files/kubernetes-csr.json @@ -0,0 +1,16 @@ +{ + "CN": "kubernetes", + "key": { + "algo": "rsa", + "size": 2048 + }, + "names": [ + { + "C": "US", + "L": "Portland", + "O": "Kubernetes", + "OU": "Kubernetes The Hard Way", + "ST": "Oregon" + } + ] +} diff --git a/nix/kubernetes/keys/package/k8s-kubernetes/package.nix b/nix/kubernetes/keys/package/k8s-kubernetes/package.nix new file mode 100644 index 0000000..4bef3ed --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-kubernetes/package.nix @@ -0,0 +1,36 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + stdenv, + sqlite, + cfssl, + k8s, + all_hostnames, + ... +}: +stdenv.mkDerivation (finalAttrs: { + name = "k8s-keys"; + nativeBuildInputs = [ cfssl ]; + buildInputs = [ ]; + + unpackPhase = "true"; + + installPhase = '' + mkdir -p "$out" + cd "$out" + cfssl gencert \ + -ca=${k8s.ca}/ca.pem \ + -ca-key=${k8s.ca}/ca-key.pem \ + -config=${./files/ca-config.json} \ + -hostname=${builtins.concatStringsSep "," all_hostnames} \ + -profile=kubernetes \ + ${./files/kubernetes-csr.json} | cfssljson -bare kubernetes + ''; +}) diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix new file mode 100644 index 0000000..6f1f52c --- /dev/null +++ b/nix/kubernetes/keys/scope.nix @@ -0,0 +1,94 @@ +{ + makeScope, + newScope, + callPackage, + writeShellScript, + openssh, + lib, +}: +let + public_addresses = [ + "74.80.180.138" + ]; + internal_addresses = [ + # nc0 + "10.215.1.221" + "2620:11f:7001:7:ffff:ffff:0ad7:01dd" + # nc1 + "10.215.1.222" + "2620:11f:7001:7:ffff:ffff:0ad7:01de" + # nc2 + "10.215.1.223" + "2620:11f:7001:7:ffff:ffff:0ad7:01df" + # nw0 + "10.215.1.224" + "2620:11f:7001:7:ffff:ffff:0ad7:01e0" + # nw1 + "10.215.1.225" + "2620:11f:7001:7:ffff:ffff:0ad7:01e1" + # nw2 + "10.215.1.226" + "2620:11f:7001:7:ffff:ffff:0ad7:01e2" + ]; + all_hostnames = [ + "10.197.0.1" + "10.0.0.1" + "127.0.0.1" + "kubernetes" + "kubernetes.default" + "kubernetes.default.svc" + "kubernetes.default.svc.cluster" + "kubernetes.svc.cluster.local" + ] + ++ public_addresses + ++ internal_addresses; +in +makeScope newScope ( + self: + let + additional_vars = { + inherit all_hostnames; + k8s = self; + }; + deploy_key = ( + vm_name: file: '' + ${openssh}/bin/ssh mrmanager rm -f /vm/${vm_name}/persist/keys/${builtins.baseNameOf file} ~/${builtins.baseNameOf file} + ${openssh}/bin/scp ${file} mrmanager:~/${builtins.baseNameOf file} + ${openssh}/bin/ssh mrmanager doas install -o 11235 -g 998 -m 0640 ~/${builtins.baseNameOf file} /vm/${vm_name}/persist/keys/${builtins.baseNameOf file} + ${openssh}/bin/ssh mrmanager rm -f ~/${builtins.baseNameOf file} + # chown to 11235:998 for talexander:etcd + '' + ); + deploy_machine = ( + vm_name: + ( + '' + ${openssh}/bin/ssh mrmanager doas install -d -o talexander -g talexander -m 0755 /vm/${vm_name}/persist/keys/ + '' + + (lib.concatMapStringsSep "\n" (deploy_key vm_name) [ + "${self.kubernetes}/kubernetes.pem" + "${self.kubernetes}/kubernetes-key.pem" + "${self.ca}/ca.pem" + ]) + ) + ); + deploy_script = ( + '' + set -euo pipefail + IFS=$'\n\t' + DIR="$( cd "$( dirname "''${BASH_SOURCE[0]}" )" && pwd )" + '' + + (lib.concatMapStringsSep "\n" deploy_machine [ + "nc0" + "nc1" + "nc2" + ]) + ); + in + { + ca = (callPackage ./package/k8s-ca/package.nix additional_vars); + kubernetes = (callPackage ./package/k8s-kubernetes/package.nix additional_vars); + keys = (callPackage ./package/k8s-keys/package.nix additional_vars); + deploy_script = (writeShellScript "deploy-keys" deploy_script); + } +) diff --git a/nix/kubernetes/repl.nix b/nix/kubernetes/repl.nix new file mode 100644 index 0000000..e21db83 --- /dev/null +++ b/nix/kubernetes/repl.nix @@ -0,0 +1,5 @@ +let + flake = builtins.getFlake (toString ./.); + nixpkgs = import { }; +in +{ inherit flake; } // flake // builtins // nixpkgs // nixpkgs.lib // flake.nixosConfigurations diff --git a/nix/kubernetes/roles/boot/default.nix b/nix/kubernetes/roles/boot/default.nix new file mode 100644 index 0000000..035f747 --- /dev/null +++ b/nix/kubernetes/roles/boot/default.nix @@ -0,0 +1,130 @@ +# ISO does not work with systemd initrd yet https://github.com/NixOS/nixpkgs/pull/291750 +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + options.me = { + boot.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install boot."; + }; + + boot.secure = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Enable to use secure boot."; + }; + + rollback.enable = lib.mkOption { + type = lib.types.bool; + default = true; + example = true; + description = "Whether we want to enable rolling back during boot."; + }; + + rollback.dataset = lib.mkOption { + default = { }; + example = lib.literalExpression '' + { + "zroot/linux/nix/root@blank" = true; + "zroot/linux/nix/home@blank" = lib.mkForce false; + } + ''; + type = lib.types.coercedTo (lib.types.listOf lib.types.str) ( + enabled: lib.listToAttrs (map (fs: lib.nameValuePair fs true) enabled) + ) (lib.types.attrsOf lib.types.bool); + description = "List of ZFS datasets to rollback to during boot."; + }; + }; + + config = lib.mkIf config.me.boot.enable ( + lib.mkMerge [ + { + environment.systemPackages = with pkgs; [ + tpm2-tools # For tpm2_eventlog to check for OptionRoms + # cp /sys/kernel/security/tpm0/binary_bios_measurements eventlog + # tpm2_eventlog eventlog | grep "BOOT_SERVICES_DRIVER" + sbctl # For debugging and troubleshooting Secure Boot. + efibootmgr # To set EFI boot order. + ]; + } + (lib.mkIf (!config.me.buildingPortable) { + + boot.loader.grub.enable = false; + # Use the systemd-boot EFI boot loader. + boot.loader.systemd-boot.enable = true; + # TODO: make not write bootx64.efi + boot.loader.efi.canTouchEfiVariables = false; + + # Automatically delete old generations + boot.loader.systemd-boot.configurationLimit = 3; + + boot.loader.systemd-boot.memtest86.enable = true; + + # Check what will be lost with `zfs diff zroot/linux/root@blank` + boot.initrd.systemd.enable = lib.mkDefault true; + boot.initrd.systemd.services.zfs-rollback = lib.mkIf config.me.rollback.enable { + description = "Rollback ZFS root dataset to blank snapshot"; + wantedBy = [ + "initrd.target" + ]; + after = [ + "zfs-import-zroot.service" + ]; + before = [ + "sysroot.mount" + ]; + unitConfig.DefaultDependencies = "no"; + serviceConfig.Type = "oneshot"; + script = lib.concatStringsSep "\n" ( + (builtins.map (ds: "${config.boot.zfs.package}/sbin/zfs rollback -r '${ds}'") ( + builtins.attrNames config.me.rollback.dataset + )) + ++ [ ''echo "rollback complete"'' ] + ); + }; + + # boot.loader.systemd-boot.extraEntries = { + # "windows.conf" = '' + # title Windows + # efi /EFI/Microsoft/Boot/bootmgfw.efi + # options root=PARTUUID=17e325bf-a378-4d1d-be6a-f6df5476f0fa + # ''; + # }; + environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) { + hideMounts = true; + directories = [ + "/var/lib/sbctl" # Secure Boot Keys + ]; + }; + }) + (lib.mkIf (config.me.boot.secure) { + environment.systemPackages = with pkgs; [ + sbctl + ]; + boot.loader.systemd-boot.enable = lib.mkForce false; + boot.lanzaboote = { + enable = true; + pkiBundle = "/var/lib/sbctl"; + }; + }) + ] + ); +} +# efibootmgr -c -d /dev/sda -p 1 -L NixOS-boot -l '\EFI\NixOS-boot\grubx64.efi' + +# Text-only: +# sudo cp "$(nix-build '' --no-out-link -A 'refind')/share/refind/refind_x64.efi" /boot/EFI/boot/bootx64.efi + +# Full graphics: +# $ sudo nix-shell -p refind efibootmgr +# $ refind-install diff --git a/nix/kubernetes/roles/doas/default.nix b/nix/kubernetes/roles/doas/default.nix new file mode 100644 index 0000000..47e1773 --- /dev/null +++ b/nix/kubernetes/roles/doas/default.nix @@ -0,0 +1,36 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + options.me = { + doas.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install doas."; + }; + }; + + config = lib.mkIf config.me.doas.enable { + # Use doas instead of sudo + security.doas.enable = true; + security.doas.wheelNeedsPassword = false; + security.sudo.enable = false; + security.doas.extraRules = [ + { + # Retain environment (for example NIX_PATH) + keepEnv = true; + persist = true; # Only ask for a password the first time. + } + ]; + environment.systemPackages = with pkgs; [ + doas-sudo-shim # To support --sudo for remote builds + ]; + }; +} diff --git a/nix/kubernetes/roles/dont_use_substituters/default.nix b/nix/kubernetes/roles/dont_use_substituters/default.nix new file mode 100644 index 0000000..0982ff2 --- /dev/null +++ b/nix/kubernetes/roles/dont_use_substituters/default.nix @@ -0,0 +1,25 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + options.me = { + dont_use_substituters.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install dont_use_substituters."; + }; + }; + + config = lib.mkIf config.me.dont_use_substituters.enable { + # Disable substituters to avoid risk of cache poisoning. + nix.settings.substitute = false; + nix.settings.substituters = lib.mkForce [ ]; + }; +} diff --git a/nix/kubernetes/roles/etcd/default.nix b/nix/kubernetes/roles/etcd/default.nix new file mode 100644 index 0000000..e554c37 --- /dev/null +++ b/nix/kubernetes/roles/etcd/default.nix @@ -0,0 +1,92 @@ +{ + config, + lib, + pkgs, + self, + ... +}: + +{ + imports = [ ]; + + options.me = { + etcd.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install etcd."; + }; + + etcd.cluster_name = lib.mkOption { + type = lib.types.str; + default = false; + example = "lorem"; + description = "The unique name for the cluster."; + }; + + etcd.internal_ip = lib.mkOption { + default = { }; + example = lib.literalExpression '' + { + "172.16.0.10" = true; + "192.168.1.10" = lib.mkForce false; + } + ''; + type = lib.types.coercedTo (lib.types.listOf lib.types.str) ( + enabled: lib.listToAttrs (map (fs: lib.nameValuePair fs true) enabled) + ) (lib.types.attrsOf lib.types.bool); + description = "List internal IP addresses for accessing this node."; + }; + + etcd.initial_cluster = lib.mkOption { + default = [ ]; + example = [ + "controller0=https://10.215.1.221:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01dd + "controller1=https://10.215.1.222:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01de + "controller2=https://10.215.1.223:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01df + ]; + type = lib.types.listOf lib.types.str; + description = "List of controller nodes to form the initial etcd cluster."; + }; + }; + + config = lib.mkIf config.me.etcd.enable { + services.etcd = { + enable = true; + openFirewall = true; + name = config.networking.hostName; + certFile = "/.disk/keys/kubernetes.pem"; + keyFile = "/.disk/keys/kubernetes-key.pem"; + peerCertFile = "/.disk/keys/kubernetes.pem"; + peerKeyFile = "/.disk/keys/kubernetes-key.pem"; + trustedCaFile = "/.disk/keys/ca.pem"; + peerTrustedCaFile = "/.disk/keys/ca.pem"; + peerClientCertAuth = true; + initialAdvertisePeerUrls = ( + builtins.map (iip: "https://${iip}:2380") (builtins.attrNames config.me.etcd.internal_ip) + ); + listenPeerUrls = ( + builtins.map (iip: "https://${iip}:2380") (builtins.attrNames config.me.etcd.internal_ip) + ); + listenClientUrls = ( + [ + "https://127.0.0.1:2379" + ] + ++ (builtins.map (iip: "https://${iip}:2379") (builtins.attrNames config.me.etcd.internal_ip)) + ); + advertiseClientUrls = ( + builtins.map (iip: "https://${iip}:2379") (builtins.attrNames config.me.etcd.internal_ip) + ); + initialClusterToken = config.me.etcd.cluster_name; + initialCluster = config.me.etcd.initial_cluster; + initialClusterState = "new"; + }; + + environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) { + hideMounts = true; + directories = [ + config.services.etcd.dataDir # "/var/lib/etcd" + ]; + }; + }; +} diff --git a/nix/kubernetes/roles/image_based_appliance/default.nix b/nix/kubernetes/roles/image_based_appliance/default.nix new file mode 100644 index 0000000..9257580 --- /dev/null +++ b/nix/kubernetes/roles/image_based_appliance/default.nix @@ -0,0 +1,30 @@ +{ + config, + lib, + ... +}: + +{ + imports = [ ]; + + options.me = { + image_based_appliance.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install image_based_appliance."; + }; + }; + + config = lib.mkIf config.me.image_based_appliance.enable ( + lib.mkMerge [ + { + # Do not install nix. A full new image must be built to update + # the machine. + nix.enable = false; + system.switch.enable = false; + nix.gc.automatic = lib.mkForce false; + } + ] + ); +} diff --git a/nix/kubernetes/roles/iso/default.nix b/nix/kubernetes/roles/iso/default.nix new file mode 100644 index 0000000..f09f8c7 --- /dev/null +++ b/nix/kubernetes/roles/iso/default.nix @@ -0,0 +1,22 @@ +{ + lib, + ... +}: + +{ + imports = [ ]; + + options.me.buildingPortable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we are building a portable image (iso/sd). This would disable CPU-specific optimizations and persistent file mounts."; + }; + + options.me.mountPersistence = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we should mount persistent directories."; + }; +} diff --git a/nix/kubernetes/roles/minimal_base/default.nix b/nix/kubernetes/roles/minimal_base/default.nix new file mode 100644 index 0000000..1654729 --- /dev/null +++ b/nix/kubernetes/roles/minimal_base/default.nix @@ -0,0 +1,33 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + options.me = { + minimal_base.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install minimal_base."; + }; + }; + + config = lib.mkIf config.me.minimal_base.enable { + me.doas.enable = true; + me.network.enable = true; + me.nvme.enable = true; + me.ssh.enable = true; + me.sshd.enable = true; + me.user.enable = true; + me.zfs.enable = true; + me.zrepl.enable = true; + me.zsh.enable = true; + + # TODO: Maybe add me.boot.enable ? + }; +} diff --git a/nix/kubernetes/roles/network/default.nix b/nix/kubernetes/roles/network/default.nix new file mode 100644 index 0000000..3aca1d8 --- /dev/null +++ b/nix/kubernetes/roles/network/default.nix @@ -0,0 +1,87 @@ +{ + config, + lib, + pkgs, + ... +}: + +# Alternative DNS servers: +# "1.0.0.1#cloudflare-dns.com" +# "1.1.1.1#cloudflare-dns.com" +# "2606:4700:4700::1001#cloudflare-dns.com" +# "2606:4700:4700::1111#cloudflare-dns.com" +# "8.8.4.4#dns.google" +# "8.8.8.8#dns.google" +# "2001:4860:4860::8844#dns.google" +# "2001:4860:4860::8888#dns.google" + +{ + imports = [ ]; + + options.me = { + network.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install network."; + }; + }; + + config = lib.mkIf config.me.network.enable { + networking.dhcpcd.enable = lib.mkDefault false; + networking.useDHCP = lib.mkDefault false; + networking.nameservers = [ + "194.242.2.2#doh.mullvad.net" + "2a07:e340::2#doh.mullvad.net" + ]; + services.resolved = { + enable = true; + # dnssec = "true"; + domains = [ "~." ]; + fallbackDns = [ ]; + dnsovertls = "true"; + }; + + # Without this, systemd-resolved will send DNS requests for .home.arpa to the per-link DNS server (172.16.0.1) which does not support DNS-over-TLS. This leads to the connection hanging and timing out. This causes firefox startup to take an extra 10+ seconds. + # + # Test with: drill @127.0.0.53 odo.home.arpa + # TODO: The 127.0.0.1 address should probably be moved to a host-specific file. + networking.extraHosts = '' + 127.0.0.1 ${config.networking.hostName}.home.arpa + ''; + + environment.systemPackages = with pkgs; [ + iw + ldns # for drill + arp-scan # To find devices on the network + wavemon + dhcpcd # For Android USB tethering. + ]; + + boot.extraModprobeConfig = '' + # Set wifi to US + options cfg80211 ieee80211_regdom=US + ''; + + boot.kernel.sysctl = { + # Enable TCP packetization-layer PMTUD when an ICMP black hole is detected. + "net.ipv4.tcp_mtu_probing" = 1; + # Switch to bbr tcp congestion control which should be better on lossy connections like bad wifi. + # We set this in the kernel config, but include this here for unoptimized builds. + "net.ipv4.tcp_congestion_control" = "bbr"; + # Don't do a slow start after a connection has been idle for a single RTO. + "net.ipv4.tcp_slow_start_after_idle" = 0; + # 3x time to accumulate filesystem changes before flushing to disk. + "vm.dirty_writeback_centisecs" = 1500; + # Adjust ttl + "net.ipv4.ip_default_ttl" = 65; + "net.ipv6.conf.all.hop_limit" = 65; + "net.ipv6.conf.default.hop_limit" = 65; + # Enable IPv6 Privacy Extensions + "net.ipv6.conf.all.use_tempaddr" = 2; + # Enable IPv6 Privacy Extensions + # This is enabled by default in nixos. + # "net.ipv6.conf.default.use_tempaddr" = 2; + }; + }; +} diff --git a/nix/kubernetes/roles/nvme/default.nix b/nix/kubernetes/roles/nvme/default.nix new file mode 100644 index 0000000..646ae6d --- /dev/null +++ b/nix/kubernetes/roles/nvme/default.nix @@ -0,0 +1,25 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + options.me = { + nvme.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install nvme."; + }; + }; + + config = lib.mkIf config.me.nvme.enable { + environment.systemPackages = with pkgs; [ + nvme-cli + ]; + }; +} diff --git a/nix/kubernetes/roles/optimized_build/default.nix b/nix/kubernetes/roles/optimized_build/default.nix new file mode 100644 index 0000000..d569df4 --- /dev/null +++ b/nix/kubernetes/roles/optimized_build/default.nix @@ -0,0 +1,134 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + options.me = { + optimizations.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to enable CPU optimizations (will trigger a rebuild from source)."; + }; + + optimizations.arch = lib.mkOption { + type = lib.types.str; + default = null; + example = "znver4"; + description = "The CPU arch for which programs should be optimized."; + }; + + optimizations.build_arch = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "znver4"; + description = "The CPU arch for which programs should be optimized."; + }; + + optimizations.system_features = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + example = [ + "gccarch-znver4" + "gccarch-znver5" + "gccarch-skylake" + "gccarch-x86-64-v3" + "gccarch-x86-64-v4" + "benchmark" + "big-parallel" + "kvm" + "nixos-test" + ]; + description = "The list of CPU features that should be enabled on this machine."; + }; + }; + + config = lib.mkMerge [ + (lib.mkIf (!config.me.optimizations.enable) ( + lib.mkMerge [ + { + # boot.kernelPackages = pkgs.linuxPackagesFor pkgs.linux_6_17; + boot.kernelPackages = pkgs.linuxPackagesFor pkgs.linux; + } + ] + )) + (lib.mkIf config.me.optimizations.enable ( + lib.mkMerge [ + { + boot.kernelPackages = pkgs.linuxPackagesFor pkgs.linux_me; + + nixpkgs.hostPlatform = { + gcc.arch = config.me.optimizations.arch; + gcc.tune = config.me.optimizations.arch; + }; + + nixpkgs.overlays = [ + ( + final: prev: + let + addConfig = + additionalConfig: pkg: + pkg.override (oldconfig: { + structuredExtraConfig = pkg.structuredExtraConfig // additionalConfig; + }); + in + { + linux_me = addConfig { + # Full preemption + PREEMPT = lib.mkOverride 60 lib.kernel.yes; + PREEMPT_VOLUNTARY = lib.mkOverride 60 lib.kernel.no; + + # Google's BBRv3 TCP congestion Control + TCP_CONG_BBR = lib.kernel.yes; + DEFAULT_BBR = lib.kernel.yes; + + # Preemptive Full Tickless Kernel at 300Hz + HZ = lib.kernel.freeform "300"; + HZ_300 = lib.kernel.yes; + HZ_1000 = lib.kernel.no; + } prev.linux; # or prev.linux_6_17 + } + ) + (final: prev: { + inherit (final.unoptimized) + assimp + binaryen + gsl + rapidjson + ffmpeg-headless + ffmpeg + pipewire + chromaprint + gtkmm + ; + }) + ]; + } + ] + )) + (lib.mkIf (config.me.optimizations.build_arch != null) ( + lib.mkMerge [ + { + # Enable cross-compiling + nixpkgs.buildPlatform = { + gcc.arch = config.me.optimizations.build_arch; + gcc.tune = "generic"; + system = "x86_64-linux"; + }; + } + ] + )) + (lib.mkIf (config.me.optimizations.system_features != [ ]) ( + lib.mkMerge [ + { + nix.settings.system-features = lib.mkForce config.me.optimizations.system_features; + } + ] + )) + ]; +} diff --git a/nix/kubernetes/roles/ssh/default.nix b/nix/kubernetes/roles/ssh/default.nix new file mode 100644 index 0000000..8532dd1 --- /dev/null +++ b/nix/kubernetes/roles/ssh/default.nix @@ -0,0 +1,50 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + options.me = { + ssh.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install ssh."; + }; + }; + + config = lib.mkIf config.me.ssh.enable { + environment.systemPackages = with pkgs; [ + sshfs + ]; + + environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) { + hideMounts = true; + users.talexander = { + files = [ + ".ssh/known_hosts" + ]; + }; + users.root = { + files = [ + ".ssh/known_hosts" + ]; + }; + }; + + me.install.user.root.file = { + ".ssh/config" = { + source = ./files/ssh_config_root; + }; + }; + me.install.user.talexander.file = { + ".ssh/config" = { + source = ./files/ssh_config; + }; + }; + }; +} diff --git a/nix/kubernetes/roles/ssh/files/ssh_config b/nix/kubernetes/roles/ssh/files/ssh_config new file mode 100644 index 0000000..6b6fa9f --- /dev/null +++ b/nix/kubernetes/roles/ssh/files/ssh_config @@ -0,0 +1,42 @@ +Host poudriere + ProxyJump talexander@mrmanager + HostName 10.215.1.203 + +Host controller0 + ProxyJump talexander@mrmanager + HostName 10.215.1.204 + +Host controller1 + ProxyJump talexander@mrmanager + HostName 10.215.1.205 + +Host controller2 + ProxyJump talexander@mrmanager + HostName 10.215.1.206 + +Host worker0 + ProxyJump talexander@mrmanager + HostName 10.215.1.207 + +Host worker1 + ProxyJump talexander@mrmanager + HostName 10.215.1.208 + +Host worker2 + ProxyJump talexander@mrmanager + HostName 10.215.1.209 + +Host brianai + ProxyJump talexander@mrmanager + HostName 10.215.1.215 + +Host hydra + ProxyJump talexander@mrmanager + HostName 10.215.1.219 + +Host i_only_boot_zfs + HostName 127.0.0.1 + Port 60022 + +Host * + Compression yes diff --git a/nix/kubernetes/roles/ssh/files/ssh_config_root b/nix/kubernetes/roles/ssh/files/ssh_config_root new file mode 100644 index 0000000..0d340b3 --- /dev/null +++ b/nix/kubernetes/roles/ssh/files/ssh_config_root @@ -0,0 +1,9 @@ +Host hydra + HostName ns1.fizz.buzz + Port 65122 + User nixworker + IdentitiesOnly yes + IdentityFile /persist/manual/ssh/root/keys/id_ed25519 + +Host * + Compression yes diff --git a/nix/kubernetes/roles/sshd/default.nix b/nix/kubernetes/roles/sshd/default.nix new file mode 100644 index 0000000..928a059 --- /dev/null +++ b/nix/kubernetes/roles/sshd/default.nix @@ -0,0 +1,49 @@ +{ + config, + lib, + ... +}: + +{ + imports = [ ]; + + options.me = { + sshd.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install sshd."; + }; + }; + + config = lib.mkIf config.me.sshd.enable { + services.openssh = { + enable = true; + settings = { + PasswordAuthentication = false; + KbdInteractiveAuthentication = false; + }; + hostKeys = [ + { + path = "/persist/ssh/ssh_host_ed25519_key"; + type = "ed25519"; + } + { + path = "/persist/ssh/ssh_host_rsa_key"; + type = "rsa"; + bits = 4096; + } + ]; + }; + + environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) { + hideMounts = true; + files = [ + "/etc/ssh/ssh_host_rsa_key" + "/etc/ssh/ssh_host_rsa_key.pub" + "/etc/ssh/ssh_host_ed25519_key" + "/etc/ssh/ssh_host_ed25519_key.pub" + ]; + }; + }; +} diff --git a/nix/kubernetes/roles/user/default.nix b/nix/kubernetes/roles/user/default.nix new file mode 100644 index 0000000..4d7d45d --- /dev/null +++ b/nix/kubernetes/roles/user/default.nix @@ -0,0 +1,59 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + options.me = { + user.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to create my user."; + }; + }; + + config = lib.mkIf config.me.user.enable { + services.getty = { + autologinUser = "talexander"; # I use full disk encryption so the user password is irrelevant. + autologinOnce = true; + }; + users.mutableUsers = false; + users.users.talexander = { + isNormalUser = true; + createHome = true; # https://github.com/NixOS/nixpkgs/issues/6481 + group = "talexander"; + extraGroups = [ "wheel" ]; + uid = 11235; + packages = with pkgs; [ + tree + ]; + # Generate with `mkpasswd -m scrypt` + hashedPassword = "$7$CU..../....VXvNQ8za3wSGpdzGXNT50/$HcFtn/yvwPMCw4888BelpiAPLAxe/zU87fD.d/N6U48"; + openssh.authorizedKeys.keys = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID0+4zi26M3eYWnIrciR54kOlGxzfgCXG+o4ea1zpzrk openpgp:0x7FF123C8" + "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIEI6mu6I5Jp+Ib0vJxapGHbEShZjyvzV8jz5DnzDrI39AAAABHNzaDo=" + "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIAFNcSXwvy+brYTOGo56G93Ptuq2MmZsjvRWAfMqbmMLAAAABHNzaDo=" + ]; + }; + users.groups.talexander.gid = 11235; + + environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) { + hideMounts = true; + users.talexander = { + directories = [ + { + directory = "persist"; + user = "talexander"; + group = "talexander"; + mode = "0700"; + } + ]; + }; + }; + }; +} diff --git a/nix/kubernetes/roles/zfs/default.nix b/nix/kubernetes/roles/zfs/default.nix new file mode 100644 index 0000000..ed53227 --- /dev/null +++ b/nix/kubernetes/roles/zfs/default.nix @@ -0,0 +1,68 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + zfs_clone_send = + (pkgs.writeScriptBin "zfs_clone_send" (builtins.readFile ./files/zfs_clone_send.bash)).overrideAttrs + (old: { + buildCommand = "${old.buildCommand}\n patchShebangs $out"; + + }); + zfs_clone_recv = + (pkgs.writeScriptBin "zfs_clone_recv" (builtins.readFile ./files/zfs_clone_recv.bash)).overrideAttrs + (old: { + buildCommand = "${old.buildCommand}\n patchShebangs $out"; + + }); + zfs_clone_resume = + (pkgs.writeScriptBin "zfs_clone_resume" (builtins.readFile ./files/zfs_clone_resume.bash)) + .overrideAttrs + (old: { + buildCommand = "${old.buildCommand}\n patchShebangs $out"; + + }); +in +{ + imports = [ ]; + + options.me = { + zfs.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install zfs."; + }; + }; + + config = lib.mkIf config.me.zfs.enable { + # Technically only needed when building the ISO because nix detects ZFS in the filesystem list normally. I basically always want this so I'm just setting it to always be on. + boot.supportedFilesystems.zfs = true; + + boot.zfs.devNodes = "/dev/disk/by-partuuid"; + + services.zfs = { + autoScrub = { + enable = true; + interval = "monthly"; + }; + trim.enable = true; + }; + + environment.systemPackages = [ + zfs_clone_send + zfs_clone_recv + zfs_clone_resume + ]; + + environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) { + hideMounts = true; + directories = [ + "/etc/zfs/zpool.cache" # Which zpools to import, the root zpool is already imported and does not need this cache file but this captures additional pools. + ]; + }; + }; +} diff --git a/nix/kubernetes/roles/zfs/files/zfs_clone_recv.bash b/nix/kubernetes/roles/zfs/files/zfs_clone_recv.bash new file mode 100644 index 0000000..e51e2c7 --- /dev/null +++ b/nix/kubernetes/roles/zfs/files/zfs_clone_recv.bash @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +# A zfs-send alias that creates a perfect clone with good defaults. +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# -s if the stream is interrupted, save the partial stream. The stream can then be resumed by doing a zfs send -t token where token is the receive_resume_token prop on the dataset we received into. +# -u Do not mount the filesystem we are receiving. We can always mount afterwards but this avoids issues with streams with mountpoints to places like / +# Can optionally add -F to destroy the dataset in the recv location. +exec zfs recv -s -u "${@}" + +# To delete an interrupted recv, run `zfs receive -A dataset` diff --git a/nix/kubernetes/roles/zfs/files/zfs_clone_resume.bash b/nix/kubernetes/roles/zfs/files/zfs_clone_resume.bash new file mode 100644 index 0000000..8b9ddac --- /dev/null +++ b/nix/kubernetes/roles/zfs/files/zfs_clone_resume.bash @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# +# Resume a zfs send. +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +function main { + local hst="$1" + local dst="$2" + local token + token=$(zfs get -H -o value receive_resume_token "$dst") + ssh "$hst" doas zfs send --verbose -t "$token" | doas zfs recv -s "$dst" + +} + +main "${@}" diff --git a/nix/kubernetes/roles/zfs/files/zfs_clone_send.bash b/nix/kubernetes/roles/zfs/files/zfs_clone_send.bash new file mode 100644 index 0000000..92a325d --- /dev/null +++ b/nix/kubernetes/roles/zfs/files/zfs_clone_send.bash @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# +# A zfs-send alias that creates a perfect clone with good defaults. +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +exec zfs send --compressed --replicate --large-block --embed --verbose --raw "${@}" diff --git a/nix/kubernetes/roles/zrepl/default.nix b/nix/kubernetes/roles/zrepl/default.nix new file mode 100644 index 0000000..2c60485 --- /dev/null +++ b/nix/kubernetes/roles/zrepl/default.nix @@ -0,0 +1,55 @@ +{ + config, + lib, + ... +}: + +{ + imports = [ ]; + + options.me = { + zrepl.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install zrepl."; + }; + }; + + config = lib.mkIf config.me.zrepl.enable { + services.zrepl = { + enable = true; + settings = { + jobs = [ + { + name = "snapjob"; + type = "snap"; + filesystems = { + "zroot/linux/nix/persist<" = true; + "zroot/bridge<" = true; + }; + snapshotting = { + type = "periodic"; + interval = "15m"; + prefix = "zrepl_"; + }; + pruning = { + keep = [ + { + type = "grid"; + grid = "1x1h(keep=all) | 24x1h | 14x1d"; + regex = "^zrepl_.*"; + } + { + type = "regex"; + negate = true; + regex = "^zrepl_.*"; + } + ]; + }; + } + ]; + }; + }; + }; +} diff --git a/nix/kubernetes/roles/zsh/default.nix b/nix/kubernetes/roles/zsh/default.nix new file mode 100644 index 0000000..645b305 --- /dev/null +++ b/nix/kubernetes/roles/zsh/default.nix @@ -0,0 +1,117 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + zshrc = pkgs.writeTextFile { + name = ".zshrc"; + text = '' + # Lines configured by zsh-newuser-install + HISTFILE=~/.zhistory + HISTSIZE=100000 + SAVEHIST=100000 + setopt appendhistory notify + unsetopt beep + bindkey -e + # End of lines configured by zsh-newuser-install + # The following lines were added by compinstall + # + + # Use menu complete immediately instead of after the first tab + setopt MENU_COMPLETE + + zstyle :compinstall filename "$HOME/.zshrc" + + autoload -Uz compinit + compinit + # End of lines added by compinstall + + # Enable the 2d menu for tab completion + zstyle ':completion:*' menu select + + autoload colors zsh/terminfo + if [[ "$terminfo[colors]" -ge 8 ]]; then + colors + fi + for color in RED GREEN YELLOW BLUE MAGENTA CYAN WHITE; do + eval PR_$color='%{$terminfo[bold]$fg[''${(L)color}]%}' + eval PR_LIGHT_$color='%{$fg[''${(L)color}]%}' + (( count = $count + 1 )) + done + PR_NO_COLOR="%{$terminfo[sgr0]%}" + PS1="[$PR_BLUE%n$PR_WHITE@$PR_GREEN%U%m%u$PR_NO_COLOR:$PR_RED%2c$PR_NO_COLOR]%(!.#.$) " + + source ${pkgs.zsh-histdb}/share/zsh/plugins/zsh-histdb/sqlite-history.zsh + autoload -Uz add-zsh-hook + + source ${pkgs.zsh-histdb}/share/zsh/plugins/zsh-histdb/histdb-interactive.zsh + bindkey '^r' _histdb-isearch + + ${lib.concatMapStringsSep "\n" (item: "source ${item}") config.me.zsh.includes} + ''; + }; +in +{ + imports = [ ]; + + options.me = { + zsh.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install zsh."; + }; + zsh.includes = lib.mkOption { + type = lib.types.listOf lib.types.package; + default = [ ]; + example = lib.literalExpression '' + [ (pkgs.writeTextFile { + name = "launch-kanshi.conf"; + text = "exec kanshi"; + }) ]''; + description = "List of zshrc files to import."; + }; + }; + + config = lib.mkIf config.me.zsh.enable { + environment.systemPackages = with pkgs; [ + zsh + ]; + + users.users.talexander.shell = pkgs.zsh; + environment.shells = with pkgs; [ zsh ]; + + programs.zsh = { + enable = true; + }; + + me.install.user.talexander.file = { + ".zshrc" = { + source = "${zshrc}"; + }; + }; + + environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) { + hideMounts = true; + users.talexander = { + directories = [ + { + directory = ".histdb"; + user = "talexander"; + group = "talexander"; + mode = "0700"; + } + ]; + }; + }; + + nixpkgs.overlays = [ + (final: prev: { + zsh-histdb = (final.callPackage ./package/zsh-histdb/package.nix { }); + }) + ]; + }; +} diff --git a/nix/kubernetes/roles/zsh/package/zsh-histdb/package.nix b/nix/kubernetes/roles/zsh/package/zsh-histdb/package.nix new file mode 100644 index 0000000..f3b92a3 --- /dev/null +++ b/nix/kubernetes/roles/zsh/package/zsh-histdb/package.nix @@ -0,0 +1,36 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + stdenv, + pkgs, + sqlite, + ... +}: +stdenv.mkDerivation { + name = "zsh-histdb"; + src = pkgs.fetchgit { + url = "https://github.com/larkery/zsh-histdb.git"; + rev = "90a6c104d0fcc0410d665e148fa7da28c49684eb"; + sha256 = "sha256-vtG1poaRVbfb/wKPChk1WpPgDq+7udLqLfYfLqap4Vg="; + }; + buildInputs = [ sqlite ]; + phases = [ + "installPhase" + ]; + installPhase = '' + runHook preInstall + mkdir -p $out/share/zsh/plugins/zsh-histdb + cp -r $src/histdb-* $src/*.zsh $src/db_migrations $out/share/zsh/plugins/zsh-histdb/ + runHook postInstall + ''; + postInstall = '' + substituteInPlace $out/share/zsh/plugins/zsh-histdb/sqlite-history.zsh $out/share/zsh/plugins/zsh-histdb/histdb-merge $out/share/zsh/plugins/zsh-histdb/histdb-migrate --replace-fail "sqlite3" "${sqlite}/bin/sqlite3" + ''; +} diff --git a/nix/kubernetes/util/install_files/default.nix b/nix/kubernetes/util/install_files/default.nix new file mode 100644 index 0000000..171e038 --- /dev/null +++ b/nix/kubernetes/util/install_files/default.nix @@ -0,0 +1,331 @@ +{ + config, + lib, + ... +}: + +let + cfg = config.me.install; + inherit (lib) + filter + attrNames + ; + + get_shell_values = + target: + let + homedir = config.users.users."${target.username}".home; + group = config.users.users."${target.username}".group; + in + { + source = lib.strings.escapeShellArg "${target.source}"; + destination = lib.strings.escapeShellArg "${homedir}/${target.target}"; + mode = lib.strings.escapeShellArg "${target.mode}"; + dir_mode = lib.strings.escapeShellArg "${target.dir_mode}"; + username = lib.strings.escapeShellArg "${target.username}"; + group = lib.strings.escapeShellArg "${group}"; + }; + install_user_file = + let + constructors = { + "overwrite" = install_user_file_overwrite; + "symlink" = install_user_file_symlink; + }; + in + stage: target: (constructors."${target.method}"."${stage}" target); + install_user_file_overwrite = { + "check" = (target: ""); + "install" = ( + target: + let + inherit (get_shell_values target) + source + destination + mode + dir_mode + username + group + ; + flags = lib.strings.concatStringsSep " " [ + (if mode != "" then "-m ${mode}" else "") + (if username != "" then "-o ${username}" else "") + (if group != "" then "-g ${group}" else "") + ]; + dir_flags = lib.strings.concatStringsSep " " [ + (if dir_mode != "" then "-m ${dir_mode}" else "") + (if username != "" then "-o ${username}" else "") + (if group != "" then "-g ${group}" else "") + ]; + in + if target.recursive then + [ + '' + find ${source} -type f -print0 | while read -r -d "" file; do + relative_path=$(realpath -s --relative-to ${source} "$file") + full_dest=${destination}/"$relative_path" + create_containing_directories "$full_dest" ${dir_flags} + $DRY_RUN_CMD install $VERBOSE_ARG --compare ${flags} "$file" "$full_dest" + done + '' + ] + else + [ + '' + create_containing_directories ${destination} ${dir_flags} + $DRY_RUN_CMD install $VERBOSE_ARG --compare ${flags} ${source} ${destination} + '' + ] + ); + "uninstall" = ( + target: + let + inherit (get_shell_values target) + source + destination + ; + in + if target.recursive then + [ + '' + find ${source} -type f -print0 | while read -r -d "" file; do + relative_path=$(realpath -s --relative-to ${source} "$file") + full_dest=${destination}/"$relative_path" + $DRY_RUN_CMD echo rm -f "$full_dest" + done + '' + ] + else + [ + '' + $DRY_RUN_CMD echo rm -f ${destination} + '' + ] + ); + }; + install_user_file_symlink = { + "check" = (target: ""); + "install" = ( + target: + let + inherit (get_shell_values target) + source + destination + mode + dir_mode + username + group + ; + owner = lib.strings.concatStringsSep ":" ( + filter (val: val != "") [ + username + group + ] + ); + dir_flags = lib.strings.concatStringsSep " " [ + (if dir_mode != "" then "-m ${dir_mode}" else "") + (if username != "" then "-o ${username}" else "") + (if group != "" then "-g ${group}" else "") + ]; + in + if target.recursive then + [ + '' + find ${source} -type f -print0 | while read -r -d "" file; do + relative_path=$(realpath -s --relative-to ${source} "$file") + full_dest=${destination}/"$relative_path" + create_containing_directories "$full_dest" ${dir_flags} + $DRY_RUN_CMD ln $VERBOSE_ARG -s "$file" "$full_dest" + $DRY_RUN_CMD chown $VERBOSE_ARG -h ${owner} "$full_dest" + done + '' + ] + else + [ + '' + create_containing_directories ${destination} ${dir_flags} + $DRY_RUN_CMD ln $VERBOSE_ARG -s ${source} ${destination} + $DRY_RUN_CMD chown $VERBOSE_ARG -h ${owner} ${destination} + '' + ] + ); + "uninstall" = ( + target: + let + inherit (get_shell_values target) + source + destination + ; + in + if target.recursive then + [ + '' + find ${source} -type f -print0 | while read -r -d "" file; do + relative_path=$(realpath -s --relative-to ${source} "$file") + full_dest=${destination}/"$relative_path" + $DRY_RUN_CMD echo rm -f "$full_dest" + done + '' + ] + else + [ + '' + $DRY_RUN_CMD echo rm -f ${destination} + '' + ] + ); + }; +in +{ + imports = [ ]; + + options.me.install = { + user = lib.mkOption { + default = { }; + type = lib.types.attrsOf ( + lib.types.submodule ( + { name, config, ... }: + let + username = name; + in + { + options = { + enable = lib.mkOption { + type = lib.types.bool; + default = true; + defaultText = "enable"; + example = lib.literalExpression false; + description = "Whether we want to install files in this user's home directory."; + }; + + file = lib.mkOption { + type = lib.types.attrsOf ( + lib.types.submodule ( + { name, config, ... }: + let + path = name; + in + { + options = { + enable = lib.mkOption { + type = lib.types.bool; + default = true; + defaultText = "enable"; + example = lib.literalExpression false; + description = "Whether we want to install this file in this user's home directory."; + }; + username = lib.mkOption { + type = lib.types.str; + defaultText = "username"; + example = "root"; + description = "The username for the user whose home directory will contain the file."; + }; + target = lib.mkOption { + type = lib.types.str; + defaultText = "target"; + example = ".local/share/foo/bar.txt"; + description = "The path where the file should be written."; + }; + method = lib.mkOption { + type = lib.types.enum [ + "symlink" + "overwrite" + # "bind_mount" TODO: for directories? + ]; + default = "symlink"; + defaultText = "me.install.file.‹path›.method"; + example = "overwrite"; + description = "The way in which the file should be installed."; + }; + mode = lib.mkOption { + type = lib.types.str; + default = "0444"; + defaultText = "me.install.file.‹path›.mode"; + example = "0750"; + description = "The read, write, execute permission flags."; + }; + dir_mode = lib.mkOption { + type = lib.types.str; + default = "0755"; + defaultText = "dir_mode"; + example = "0755"; + description = "The read, write, execute permission flags for any parent directories that need to be created."; + }; + source = lib.mkOption { + type = lib.types.path; + defaultText = "me.install.file.‹path›.source"; + example = ./files/foo.txt; + description = "The source file to install into the destination."; + }; + recursive = lib.mkOption { + type = lib.types.bool; + default = false; + defaultText = "recursive"; + example = lib.literalExpression false; + description = "Whether we want to recurse through the directory doing individual installs for each file."; + }; + }; + + config = { + username = lib.mkDefault username; + target = lib.mkDefault path; + }; + } + ) + ); + }; + }; + } + ) + ); + }; + }; + + config = + let + all_users = builtins.map (username: cfg.user."${username}") (attrNames cfg.user); + enabled_users = filter (user: user.enable) all_users; + all_file_targets = lib.flatten ( + builtins.map (user: (builtins.map (path: user.file."${path}") (attrNames user.file))) enabled_users + ); + enabled_file_targets = filter (target: target.enable) all_file_targets; + check_commands = lib.flatten (builtins.map (install_user_file "check") enabled_file_targets); + install_commands = lib.flatten (builtins.map (install_user_file "install") enabled_file_targets); + uninstall_commands = lib.flatten ( + builtins.map (install_user_file "uninstall") enabled_file_targets + ); + in + { + systemd.services.me-install-file = { + enable = true; + description = "me-install-file"; + wantedBy = [ "multi-user.target" ]; + wants = [ "multi-user.target" ]; + before = [ "multi-user.target" ]; + # path = with pkgs; [ + # zfs + # ]; + unitConfig.DefaultDependencies = "no"; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = "yes"; + }; + script = '' + set -o pipefail + IFS=$'\n\t' + source ${./files/lib.bash} + '' + + (lib.strings.concatStringsSep "\n" ( + [ + ] + ++ check_commands + ++ install_commands + )); + preStop = '' + set -o pipefail + IFS=$'\n\t' + source ${./files/lib.bash} + '' + + (lib.strings.concatStringsSep "\n" uninstall_commands); + }; + }; +} diff --git a/nix/kubernetes/util/install_files/files/lib.bash b/nix/kubernetes/util/install_files/files/lib.bash new file mode 100644 index 0000000..1a5c178 --- /dev/null +++ b/nix/kubernetes/util/install_files/files/lib.bash @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# + +############## Setup ######################### + +function die { + local status_code="$1" + shift + (>&2 echo "${@}") + exit "$status_code" +} + +function log { + (>&2 echo "${@}") +} + +############## Program ######################### + +function create_containing_directories { + local full_dest="$1" + shift 1 + local dirs_to_create=() + local containing_directory="$full_dest" + while true; do + containing_directory=$(dirname "$containing_directory") + if [ -e "$containing_directory" ] || [ "$containing_directory" = "/" ]; then + break + fi + dirs_to_create+=($containing_directory) + done + + for (( idx=${#dirs_to_create[@]}-1 ; idx>=0 ; idx-- )) ; do + local containing_directory="${dirs_to_create[idx]}" + log "Creating $containing_directory" + $DRY_RUN_CMD install $VERBOSE_ARG -d "${@}" "$containing_directory" + done + +} diff --git a/nix/kubernetes/util/unfree_polyfill/default.nix b/nix/kubernetes/util/unfree_polyfill/default.nix new file mode 100644 index 0000000..d744cf4 --- /dev/null +++ b/nix/kubernetes/util/unfree_polyfill/default.nix @@ -0,0 +1,15 @@ +{ config, lib, ... }: + +let + inherit (builtins) elem; + inherit (lib) getName mkOption; + inherit (lib.types) listOf str; +in +{ + # Pending https://github.com/NixOS/nixpkgs/issues/55674 + options.allowedUnfree = mkOption { + type = listOf str; + default = [ ]; + }; + config.nixpkgs.config.allowUnfreePredicate = p: elem (getName p) config.allowedUnfree; +} From ac07941b8ec103d8c85c9e9422881c7a415e4700 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 7 Dec 2025 15:48:08 -0500 Subject: [PATCH 02/81] Add additional controllers. --- nix/kubernetes/flake.nix | 6 + nix/kubernetes/hosts/controller0/default.nix | 2 +- nix/kubernetes/hosts/controller0/vm_disk.nix | 31 ++++- nix/kubernetes/hosts/controller1/DEPLOY_BOOT | 13 ++ .../hosts/controller1/DEPLOY_SWITCH | 13 ++ nix/kubernetes/hosts/controller1/ISO | 10 ++ nix/kubernetes/hosts/controller1/SELF_BOOT | 10 ++ nix/kubernetes/hosts/controller1/SELF_BUILD | 10 ++ nix/kubernetes/hosts/controller1/SELF_SWITCH | 10 ++ nix/kubernetes/hosts/controller1/VM_ISO | 10 ++ nix/kubernetes/hosts/controller1/default.nix | 123 ++++++++++++++++++ .../controller1/hardware-configuration.nix | 31 +++++ nix/kubernetes/hosts/controller1/vm_disk.nix | 94 +++++++++++++ nix/kubernetes/hosts/controller2/DEPLOY_BOOT | 13 ++ .../hosts/controller2/DEPLOY_SWITCH | 13 ++ nix/kubernetes/hosts/controller2/ISO | 10 ++ nix/kubernetes/hosts/controller2/SELF_BOOT | 10 ++ nix/kubernetes/hosts/controller2/SELF_BUILD | 10 ++ nix/kubernetes/hosts/controller2/SELF_SWITCH | 10 ++ nix/kubernetes/hosts/controller2/VM_ISO | 10 ++ nix/kubernetes/hosts/controller2/default.nix | 123 ++++++++++++++++++ .../controller2/hardware-configuration.nix | 31 +++++ nix/kubernetes/hosts/controller2/vm_disk.nix | 94 +++++++++++++ nix/kubernetes/keys/scope.nix | 8 +- nix/kubernetes/roles/etcd/default.nix | 33 +++-- 25 files changed, 711 insertions(+), 17 deletions(-) create mode 100755 nix/kubernetes/hosts/controller1/DEPLOY_BOOT create mode 100755 nix/kubernetes/hosts/controller1/DEPLOY_SWITCH create mode 100755 nix/kubernetes/hosts/controller1/ISO create mode 100755 nix/kubernetes/hosts/controller1/SELF_BOOT create mode 100755 nix/kubernetes/hosts/controller1/SELF_BUILD create mode 100755 nix/kubernetes/hosts/controller1/SELF_SWITCH create mode 100755 nix/kubernetes/hosts/controller1/VM_ISO create mode 100644 nix/kubernetes/hosts/controller1/default.nix create mode 100644 nix/kubernetes/hosts/controller1/hardware-configuration.nix create mode 100644 nix/kubernetes/hosts/controller1/vm_disk.nix create mode 100755 nix/kubernetes/hosts/controller2/DEPLOY_BOOT create mode 100755 nix/kubernetes/hosts/controller2/DEPLOY_SWITCH create mode 100755 nix/kubernetes/hosts/controller2/ISO create mode 100755 nix/kubernetes/hosts/controller2/SELF_BOOT create mode 100755 nix/kubernetes/hosts/controller2/SELF_BUILD create mode 100755 nix/kubernetes/hosts/controller2/SELF_SWITCH create mode 100755 nix/kubernetes/hosts/controller2/VM_ISO create mode 100644 nix/kubernetes/hosts/controller2/default.nix create mode 100644 nix/kubernetes/hosts/controller2/hardware-configuration.nix create mode 100644 nix/kubernetes/hosts/controller2/vm_disk.nix diff --git a/nix/kubernetes/flake.nix b/nix/kubernetes/flake.nix index bfbbd02..74eda61 100644 --- a/nix/kubernetes/flake.nix +++ b/nix/kubernetes/flake.nix @@ -48,6 +48,12 @@ controller0 = { system = "x86_64-linux"; }; + controller1 = { + system = "x86_64-linux"; + }; + controller2 = { + system = "x86_64-linux"; + }; }; nixosConfigs = builtins.mapAttrs ( hostname: nodeConfig: format: diff --git a/nix/kubernetes/hosts/controller0/default.nix b/nix/kubernetes/hosts/controller0/default.nix index 847874f..5339141 100644 --- a/nix/kubernetes/hosts/controller0/default.nix +++ b/nix/kubernetes/hosts/controller0/default.nix @@ -102,7 +102,7 @@ # nix.sshServe.enable = true; # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ]; - me.etcd.cluster_name = "put a nix on it"; + me.etcd.cluster_name = "put-a-nix-on-it"; me.etcd.internal_ip = [ # "10.215.1.221" "[2620:11f:7001:7:ffff:ffff:0ad7:01dd]" diff --git a/nix/kubernetes/hosts/controller0/vm_disk.nix b/nix/kubernetes/hosts/controller0/vm_disk.nix index 6c23789..83683f8 100644 --- a/nix/kubernetes/hosts/controller0/vm_disk.nix +++ b/nix/kubernetes/hosts/controller0/vm_disk.nix @@ -12,6 +12,16 @@ # Mount the local disk fileSystems = lib.mkIf config.me.mountPersistence { "/.disk" = lib.mkForce { + device = "/dev/nvme0n1p1"; + fsType = "ext4"; + options = [ + "noatime" + "discard" + ]; + neededForBoot = true; + }; + + "/.persist" = lib.mkForce { device = "bind9p"; fsType = "9p"; options = [ @@ -28,26 +38,26 @@ "/persist" = { fsType = "none"; - device = "/.disk/persist"; + device = "/.persist/persist"; options = [ "bind" "rw" ]; depends = [ - "/.disk/persist" + "/.persist/persist" ]; neededForBoot = true; }; "/state" = { fsType = "none"; - device = "/.disk/state"; + device = "/.persist/state"; options = [ "bind" "rw" ]; depends = [ - "/.disk/state" + "/.persist/state" ]; neededForBoot = true; }; @@ -66,6 +76,19 @@ ]; neededForBoot = true; }; + + "/disk" = { + fsType = "none"; + device = "/.disk/persist"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.disk/persist" + ]; + neededForBoot = true; + }; }; }; } diff --git a/nix/kubernetes/hosts/controller1/DEPLOY_BOOT b/nix/kubernetes/hosts/controller1/DEPLOY_BOOT new file mode 100755 index 0000000..a98d5d4 --- /dev/null +++ b/nix/kubernetes/hosts/controller1/DEPLOY_BOOT @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=controller1 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild boot --flake "$DIR/../../#controller1" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller1/DEPLOY_SWITCH b/nix/kubernetes/hosts/controller1/DEPLOY_SWITCH new file mode 100755 index 0000000..e7c22a0 --- /dev/null +++ b/nix/kubernetes/hosts/controller1/DEPLOY_SWITCH @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=controller1 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild switch --flake "$DIR/../../#controller1" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller1/ISO b/nix/kubernetes/hosts/controller1/ISO new file mode 100755 index 0000000..57320a2 --- /dev/null +++ b/nix/kubernetes/hosts/controller1/ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#controller1.iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller1/SELF_BOOT b/nix/kubernetes/hosts/controller1/SELF_BOOT new file mode 100755 index 0000000..6f38732 --- /dev/null +++ b/nix/kubernetes/hosts/controller1/SELF_BOOT @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild boot --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller1" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller1/SELF_BUILD b/nix/kubernetes/hosts/controller1/SELF_BUILD new file mode 100755 index 0000000..bc57565 --- /dev/null +++ b/nix/kubernetes/hosts/controller1/SELF_BUILD @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild build --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller1" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller1/SELF_SWITCH b/nix/kubernetes/hosts/controller1/SELF_SWITCH new file mode 100755 index 0000000..2879d19 --- /dev/null +++ b/nix/kubernetes/hosts/controller1/SELF_SWITCH @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild switch --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller1" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller1/VM_ISO b/nix/kubernetes/hosts/controller1/VM_ISO new file mode 100755 index 0000000..e33e16b --- /dev/null +++ b/nix/kubernetes/hosts/controller1/VM_ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#controller1.vm_iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller1/default.nix b/nix/kubernetes/hosts/controller1/default.nix new file mode 100644 index 0000000..5f6d50b --- /dev/null +++ b/nix/kubernetes/hosts/controller1/default.nix @@ -0,0 +1,123 @@ +# MANUAL: On client machines generate signing keys: +# nix-store --generate-binary-cache-key some-name /persist/manual/nix/nix-cache-key.sec /persist/manual/nix/nix-cache-key.pub +# +# Trust other machines and add the substituters: +# nix.binaryCachePublicKeys = [ "some-name:AzNW1MOlkNEsUAXS1jIFZ1QCFKXjV+Y/LrF37quAZ1A=" ]; +# nix.binaryCaches = [ "https://test.example/nix-cache" ]; + +{ + config, + lib, + pkgs, + ... +}: +{ + imports = [ + ./hardware-configuration.nix + ./vm_disk.nix + ]; + + config = { + networking = + let + interface = "enp0s2"; + in + { + # Generate with `head -c4 /dev/urandom | od -A none -t x4` + hostId = "59a83979"; + + hostName = "controller1"; # Define your hostname. + + interfaces = { + "${interface}" = { + ipv4.addresses = [ + { + address = "10.215.1.222"; + prefixLength = 24; + } + ]; + + ipv6.addresses = [ + { + address = "2620:11f:7001:7:ffff:ffff:0ad7:01de"; + prefixLength = 64; + } + ]; + }; + }; + defaultGateway = "10.215.1.1"; + defaultGateway6 = { + # address = "2620:11f:7001:7::1"; + address = "2620:11f:7001:7:ffff:ffff:0ad7:0101"; + inherit interface; + }; + nameservers = [ + "10.215.1.1" + ]; + + dhcpcd.enable = lib.mkForce false; + useDHCP = lib.mkForce false; + }; + + time.timeZone = "America/New_York"; + i18n.defaultLocale = "en_US.UTF-8"; + + me.boot.enable = true; + me.boot.secure = false; + me.mountPersistence = true; + boot.loader.timeout = lib.mkForce 0; # We can always generate a new ISO if we need to access other boot options. + + me.optimizations = { + enable = true; + arch = "znver4"; + # build_arch = "x86-64-v3"; + system_features = [ + "gccarch-znver4" + "gccarch-skylake" + "gccarch-kabylake" + # "gccarch-alderlake" missing WAITPKG + "gccarch-x86-64-v3" + "gccarch-x86-64-v4" + "benchmark" + "big-parallel" + "kvm" + "nixos-test" + ]; + }; + + # Mount tmpfs at /tmp + boot.tmp.useTmpfs = true; + + # Enable TRIM + # services.fstrim.enable = lib.mkDefault true; + + # nix.optimise.automatic = true; + # nix.optimise.dates = [ "03:45" ]; + # nix.optimise.persistent = true; + + environment.systemPackages = with pkgs; [ + htop + ]; + + # nix.sshServe.enable = true; + # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ]; + + me.etcd.cluster_name = "put-a-nix-on-it"; + me.etcd.internal_ip = [ + # "10.215.1.221" + "[2620:11f:7001:7:ffff:ffff:0ad7:01de]" + ]; + me.etcd.initial_cluster = [ + # "controller0=https://10.215.1.221:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01dd + # "controller1=https://10.215.1.222:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01de + # "controller2=https://10.215.1.223:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01df + "controller0=https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2380" # 10.215.1.221 + "controller1=https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2380" # 10.215.1.222 + "controller2=https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2380" # 10.215.1.223 + ]; + + me.dont_use_substituters.enable = true; + me.etcd.enable = true; + me.minimal_base.enable = true; + }; +} diff --git a/nix/kubernetes/hosts/controller1/hardware-configuration.nix b/nix/kubernetes/hosts/controller1/hardware-configuration.nix new file mode 100644 index 0000000..6029e08 --- /dev/null +++ b/nix/kubernetes/hosts/controller1/hardware-configuration.nix @@ -0,0 +1,31 @@ +{ + config, + lib, + modulesPath, + ... +}: + +{ + imports = [ + (modulesPath + "/installer/scan/not-detected.nix") + ]; + + config = { + boot.initrd.availableKernelModules = [ + "nvme" + "xhci_pci" + "thunderbolt" + ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ ]; + boot.extraModulePackages = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + # networking.useDHCP = lib.mkDefault true; + # networking.interfaces.eno1.useDHCP = lib.mkDefault true; + # networking.interfaces.wlp58s0.useDHCP = lib.mkDefault true; + }; +} diff --git a/nix/kubernetes/hosts/controller1/vm_disk.nix b/nix/kubernetes/hosts/controller1/vm_disk.nix new file mode 100644 index 0000000..83683f8 --- /dev/null +++ b/nix/kubernetes/hosts/controller1/vm_disk.nix @@ -0,0 +1,94 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + config = { + # Mount the local disk + fileSystems = lib.mkIf config.me.mountPersistence { + "/.disk" = lib.mkForce { + device = "/dev/nvme0n1p1"; + fsType = "ext4"; + options = [ + "noatime" + "discard" + ]; + neededForBoot = true; + }; + + "/.persist" = lib.mkForce { + device = "bind9p"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + + "/persist" = { + fsType = "none"; + device = "/.persist/persist"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.persist/persist" + ]; + neededForBoot = true; + }; + + "/state" = { + fsType = "none"; + device = "/.persist/state"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.persist/state" + ]; + neededForBoot = true; + }; + + "/k8spv" = lib.mkForce { + device = "k8spv"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + + "/disk" = { + fsType = "none"; + device = "/.disk/persist"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.disk/persist" + ]; + neededForBoot = true; + }; + }; + }; +} diff --git a/nix/kubernetes/hosts/controller2/DEPLOY_BOOT b/nix/kubernetes/hosts/controller2/DEPLOY_BOOT new file mode 100755 index 0000000..5c4bee4 --- /dev/null +++ b/nix/kubernetes/hosts/controller2/DEPLOY_BOOT @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=controller2 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild boot --flake "$DIR/../../#controller2" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller2/DEPLOY_SWITCH b/nix/kubernetes/hosts/controller2/DEPLOY_SWITCH new file mode 100755 index 0000000..53c95e7 --- /dev/null +++ b/nix/kubernetes/hosts/controller2/DEPLOY_SWITCH @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=controller2 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild switch --flake "$DIR/../../#controller2" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller2/ISO b/nix/kubernetes/hosts/controller2/ISO new file mode 100755 index 0000000..0850b60 --- /dev/null +++ b/nix/kubernetes/hosts/controller2/ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#controller2.iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller2/SELF_BOOT b/nix/kubernetes/hosts/controller2/SELF_BOOT new file mode 100755 index 0000000..2976bc9 --- /dev/null +++ b/nix/kubernetes/hosts/controller2/SELF_BOOT @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild boot --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller2" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller2/SELF_BUILD b/nix/kubernetes/hosts/controller2/SELF_BUILD new file mode 100755 index 0000000..a9a3b56 --- /dev/null +++ b/nix/kubernetes/hosts/controller2/SELF_BUILD @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild build --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller2" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller2/SELF_SWITCH b/nix/kubernetes/hosts/controller2/SELF_SWITCH new file mode 100755 index 0000000..dfca46e --- /dev/null +++ b/nix/kubernetes/hosts/controller2/SELF_SWITCH @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild switch --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller2" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller2/VM_ISO b/nix/kubernetes/hosts/controller2/VM_ISO new file mode 100755 index 0000000..f57c883 --- /dev/null +++ b/nix/kubernetes/hosts/controller2/VM_ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#controller2.vm_iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller2/default.nix b/nix/kubernetes/hosts/controller2/default.nix new file mode 100644 index 0000000..d86dbbe --- /dev/null +++ b/nix/kubernetes/hosts/controller2/default.nix @@ -0,0 +1,123 @@ +# MANUAL: On client machines generate signing keys: +# nix-store --generate-binary-cache-key some-name /persist/manual/nix/nix-cache-key.sec /persist/manual/nix/nix-cache-key.pub +# +# Trust other machines and add the substituters: +# nix.binaryCachePublicKeys = [ "some-name:AzNW1MOlkNEsUAXS1jIFZ1QCFKXjV+Y/LrF37quAZ1A=" ]; +# nix.binaryCaches = [ "https://test.example/nix-cache" ]; + +{ + config, + lib, + pkgs, + ... +}: +{ + imports = [ + ./hardware-configuration.nix + ./vm_disk.nix + ]; + + config = { + networking = + let + interface = "enp0s2"; + in + { + # Generate with `head -c4 /dev/urandom | od -A none -t x4` + hostId = "26a43660"; + + hostName = "controller2"; # Define your hostname. + + interfaces = { + "${interface}" = { + ipv4.addresses = [ + { + address = "10.215.1.223"; + prefixLength = 24; + } + ]; + + ipv6.addresses = [ + { + address = "2620:11f:7001:7:ffff:ffff:0ad7:01df"; + prefixLength = 64; + } + ]; + }; + }; + defaultGateway = "10.215.1.1"; + defaultGateway6 = { + # address = "2620:11f:7001:7::1"; + address = "2620:11f:7001:7:ffff:ffff:0ad7:0101"; + inherit interface; + }; + nameservers = [ + "10.215.1.1" + ]; + + dhcpcd.enable = lib.mkForce false; + useDHCP = lib.mkForce false; + }; + + time.timeZone = "America/New_York"; + i18n.defaultLocale = "en_US.UTF-8"; + + me.boot.enable = true; + me.boot.secure = false; + me.mountPersistence = true; + boot.loader.timeout = lib.mkForce 0; # We can always generate a new ISO if we need to access other boot options. + + me.optimizations = { + enable = true; + arch = "znver4"; + # build_arch = "x86-64-v3"; + system_features = [ + "gccarch-znver4" + "gccarch-skylake" + "gccarch-kabylake" + # "gccarch-alderlake" missing WAITPKG + "gccarch-x86-64-v3" + "gccarch-x86-64-v4" + "benchmark" + "big-parallel" + "kvm" + "nixos-test" + ]; + }; + + # Mount tmpfs at /tmp + boot.tmp.useTmpfs = true; + + # Enable TRIM + # services.fstrim.enable = lib.mkDefault true; + + # nix.optimise.automatic = true; + # nix.optimise.dates = [ "03:45" ]; + # nix.optimise.persistent = true; + + environment.systemPackages = with pkgs; [ + htop + ]; + + # nix.sshServe.enable = true; + # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ]; + + me.etcd.cluster_name = "put-a-nix-on-it"; + me.etcd.internal_ip = [ + # "10.215.1.221" + "[2620:11f:7001:7:ffff:ffff:0ad7:01df]" + ]; + me.etcd.initial_cluster = [ + # "controller0=https://10.215.1.221:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01dd + # "controller1=https://10.215.1.222:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01de + # "controller2=https://10.215.1.223:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01df + "controller0=https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2380" # 10.215.1.221 + "controller1=https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2380" # 10.215.1.222 + "controller2=https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2380" # 10.215.1.223 + ]; + + me.dont_use_substituters.enable = true; + me.etcd.enable = true; + me.minimal_base.enable = true; + }; +} diff --git a/nix/kubernetes/hosts/controller2/hardware-configuration.nix b/nix/kubernetes/hosts/controller2/hardware-configuration.nix new file mode 100644 index 0000000..6029e08 --- /dev/null +++ b/nix/kubernetes/hosts/controller2/hardware-configuration.nix @@ -0,0 +1,31 @@ +{ + config, + lib, + modulesPath, + ... +}: + +{ + imports = [ + (modulesPath + "/installer/scan/not-detected.nix") + ]; + + config = { + boot.initrd.availableKernelModules = [ + "nvme" + "xhci_pci" + "thunderbolt" + ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ ]; + boot.extraModulePackages = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + # networking.useDHCP = lib.mkDefault true; + # networking.interfaces.eno1.useDHCP = lib.mkDefault true; + # networking.interfaces.wlp58s0.useDHCP = lib.mkDefault true; + }; +} diff --git a/nix/kubernetes/hosts/controller2/vm_disk.nix b/nix/kubernetes/hosts/controller2/vm_disk.nix new file mode 100644 index 0000000..83683f8 --- /dev/null +++ b/nix/kubernetes/hosts/controller2/vm_disk.nix @@ -0,0 +1,94 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + config = { + # Mount the local disk + fileSystems = lib.mkIf config.me.mountPersistence { + "/.disk" = lib.mkForce { + device = "/dev/nvme0n1p1"; + fsType = "ext4"; + options = [ + "noatime" + "discard" + ]; + neededForBoot = true; + }; + + "/.persist" = lib.mkForce { + device = "bind9p"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + + "/persist" = { + fsType = "none"; + device = "/.persist/persist"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.persist/persist" + ]; + neededForBoot = true; + }; + + "/state" = { + fsType = "none"; + device = "/.persist/state"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.persist/state" + ]; + neededForBoot = true; + }; + + "/k8spv" = lib.mkForce { + device = "k8spv"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + + "/disk" = { + fsType = "none"; + device = "/.disk/persist"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.disk/persist" + ]; + neededForBoot = true; + }; + }; + }; +} diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index 6f1f52c..f86abcf 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -52,18 +52,18 @@ makeScope newScope ( }; deploy_key = ( vm_name: file: '' - ${openssh}/bin/ssh mrmanager rm -f /vm/${vm_name}/persist/keys/${builtins.baseNameOf file} ~/${builtins.baseNameOf file} + ${openssh}/bin/ssh mrmanager rm -f /vm/${vm_name}/persist/keys/etcd/${builtins.baseNameOf file} ~/${builtins.baseNameOf file} ${openssh}/bin/scp ${file} mrmanager:~/${builtins.baseNameOf file} - ${openssh}/bin/ssh mrmanager doas install -o 11235 -g 998 -m 0640 ~/${builtins.baseNameOf file} /vm/${vm_name}/persist/keys/${builtins.baseNameOf file} + ${openssh}/bin/ssh mrmanager doas install -o 10016 -g 10016 -m 0640 ~/${builtins.baseNameOf file} /vm/${vm_name}/persist/keys/etcd/${builtins.baseNameOf file} ${openssh}/bin/ssh mrmanager rm -f ~/${builtins.baseNameOf file} - # chown to 11235:998 for talexander:etcd '' ); deploy_machine = ( vm_name: ( '' - ${openssh}/bin/ssh mrmanager doas install -d -o talexander -g talexander -m 0755 /vm/${vm_name}/persist/keys/ + ${openssh}/bin/ssh mrmanager doas install -d -o 11235 -g 11235 -m 0755 /vm/${vm_name}/persist/keys + ${openssh}/bin/ssh mrmanager doas install -d -o 10016 -g 10016 -m 0755 /vm/${vm_name}/persist/keys/etcd '' + (lib.concatMapStringsSep "\n" (deploy_key vm_name) [ "${self.kubernetes}/kubernetes.pem" diff --git a/nix/kubernetes/roles/etcd/default.nix b/nix/kubernetes/roles/etcd/default.nix index e554c37..f0044d0 100644 --- a/nix/kubernetes/roles/etcd/default.nix +++ b/nix/kubernetes/roles/etcd/default.nix @@ -55,13 +55,14 @@ enable = true; openFirewall = true; name = config.networking.hostName; - certFile = "/.disk/keys/kubernetes.pem"; - keyFile = "/.disk/keys/kubernetes-key.pem"; - peerCertFile = "/.disk/keys/kubernetes.pem"; - peerKeyFile = "/.disk/keys/kubernetes-key.pem"; - trustedCaFile = "/.disk/keys/ca.pem"; - peerTrustedCaFile = "/.disk/keys/ca.pem"; + certFile = "/.persist/keys/etcd/kubernetes.pem"; + keyFile = "/.persist/keys/etcd/kubernetes-key.pem"; + peerCertFile = "/.persist/keys/etcd/kubernetes.pem"; + peerKeyFile = "/.persist/keys/etcd/kubernetes-key.pem"; + trustedCaFile = "/.persist/keys/etcd/ca.pem"; + peerTrustedCaFile = "/.persist/keys/etcd/ca.pem"; peerClientCertAuth = true; + clientCertAuth = true; initialAdvertisePeerUrls = ( builtins.map (iip: "https://${iip}:2380") (builtins.attrNames config.me.etcd.internal_ip) ); @@ -82,11 +83,27 @@ initialClusterState = "new"; }; - environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) { + environment.persistence."/disk" = lib.mkIf (config.me.mountPersistence) { hideMounts = true; directories = [ - config.services.etcd.dataDir # "/var/lib/etcd" + { + directory = config.services.etcd.dataDir; # "/var/lib/etcd" + user = "etcd"; + group = "etcd"; + mode = "0700"; + } ]; }; + + users.users.etcd.uid = 10016; + users.groups.etcd.gid = 10016; + + environment.systemPackages = with pkgs; [ + net-tools # for debugging + tcpdump + e2fsprogs # mkfs.ext4 + gptfdisk # cgdisk + ]; + networking.firewall.enable = false; }; } From 732e2cd296753b314c353eac25da34cc840bb49f Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 8 Dec 2025 20:33:41 -0500 Subject: [PATCH 03/81] Install kubernetes. --- nix/kubernetes/configuration.nix | 7 +- nix/kubernetes/flake.nix | 6 +- nix/kubernetes/hosts/controller0/default.nix | 8 ++ nix/kubernetes/hosts/controller1/default.nix | 8 ++ nix/kubernetes/hosts/controller2/default.nix | 8 ++ nix/kubernetes/keys/scope.nix | 113 ++++++++++++++++-- nix/kubernetes/repl.nix | 5 - .../roles/control_plane/default.nix | 24 ++++ .../roles/kube_apiserver/default.nix | 98 +++++++++++++++ nix/kubernetes/roles/kubernetes/default.nix | 53 ++++++++ 10 files changed, 309 insertions(+), 21 deletions(-) delete mode 100644 nix/kubernetes/repl.nix create mode 100644 nix/kubernetes/roles/control_plane/default.nix create mode 100644 nix/kubernetes/roles/kube_apiserver/default.nix create mode 100644 nix/kubernetes/roles/kubernetes/default.nix diff --git a/nix/kubernetes/configuration.nix b/nix/kubernetes/configuration.nix index 2be3505..16fcf75 100644 --- a/nix/kubernetes/configuration.nix +++ b/nix/kubernetes/configuration.nix @@ -7,15 +7,18 @@ { imports = [ ./roles/boot + ./roles/control_plane ./roles/doas + ./roles/dont_use_substituters ./roles/etcd ./roles/image_based_appliance ./roles/iso - ./roles/optimized_build - ./roles/dont_use_substituters + ./roles/kube_apiserver + ./roles/kubernetes ./roles/minimal_base ./roles/network ./roles/nvme + ./roles/optimized_build ./roles/ssh ./roles/sshd ./roles/user diff --git a/nix/kubernetes/flake.nix b/nix/kubernetes/flake.nix index 74eda61..30e8c39 100644 --- a/nix/kubernetes/flake.nix +++ b/nix/kubernetes/flake.nix @@ -95,10 +95,9 @@ ... }: let - repl_path = toString ./.; nix-self-repl = pkgs.writeShellScriptBin "nix-self-repl" '' source /etc/set-environment - nix repl "${repl_path}/repl.nix" "$@" + nix repl --expr 'builtins.getFlake "${self}"' ''; # If we wanted the current version of a flake then we'd just launch # nix repl @@ -130,10 +129,9 @@ ... }: let - repl_path = toString ./.; nix-self-repl = pkgs.writeShellScriptBin "nix-self-repl" '' source /etc/set-environment - nix repl "${repl_path}/repl.nix" "$@" + nix repl --expr 'builtins.getFlake "${self}"' ''; # If we wanted the current version of a flake then we'd just launch # nix repl diff --git a/nix/kubernetes/hosts/controller0/default.nix b/nix/kubernetes/hosts/controller0/default.nix index 5339141..ef6a64f 100644 --- a/nix/kubernetes/hosts/controller0/default.nix +++ b/nix/kubernetes/hosts/controller0/default.nix @@ -116,6 +116,14 @@ "controller2=https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2380" # 10.215.1.223 ]; + me.kube_apiserver.internal_ip = "2620:11f:7001:7:ffff:ffff:0ad7:01dd"; + me.kube_apiserver.etcd_services = [ + "https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2379" # 10.215.1.221 + "https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2379" # 10.215.1.222 + "https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2379" # 10.215.1.223 + ]; + + me.control_plane.enable = true; me.dont_use_substituters.enable = true; me.etcd.enable = true; me.minimal_base.enable = true; diff --git a/nix/kubernetes/hosts/controller1/default.nix b/nix/kubernetes/hosts/controller1/default.nix index 5f6d50b..61f26a5 100644 --- a/nix/kubernetes/hosts/controller1/default.nix +++ b/nix/kubernetes/hosts/controller1/default.nix @@ -116,6 +116,14 @@ "controller2=https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2380" # 10.215.1.223 ]; + me.kube_apiserver.internal_ip = "2620:11f:7001:7:ffff:ffff:0ad7:01de"; + me.kube_apiserver.etcd_services = [ + "https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2379" # 10.215.1.221 + "https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2379" # 10.215.1.222 + "https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2379" # 10.215.1.223 + ]; + + me.control_plane.enable = true; me.dont_use_substituters.enable = true; me.etcd.enable = true; me.minimal_base.enable = true; diff --git a/nix/kubernetes/hosts/controller2/default.nix b/nix/kubernetes/hosts/controller2/default.nix index d86dbbe..8336a01 100644 --- a/nix/kubernetes/hosts/controller2/default.nix +++ b/nix/kubernetes/hosts/controller2/default.nix @@ -116,6 +116,14 @@ "controller2=https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2380" # 10.215.1.223 ]; + me.kube_apiserver.internal_ip = "2620:11f:7001:7:ffff:ffff:0ad7:01df"; + me.kube_apiserver.etcd_services = [ + "https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2379" # 10.215.1.221 + "https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2379" # 10.215.1.222 + "https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2379" # 10.215.1.223 + ]; + + me.control_plane.enable = true; me.dont_use_substituters.enable = true; me.etcd.enable = true; me.minimal_base.enable = true; diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index f86abcf..402c733 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -4,6 +4,8 @@ callPackage, writeShellScript, openssh, + runCommand, + writeText, lib, }: let @@ -50,25 +52,91 @@ makeScope newScope ( inherit all_hostnames; k8s = self; }; - deploy_key = ( - vm_name: file: '' - ${openssh}/bin/ssh mrmanager rm -f /vm/${vm_name}/persist/keys/etcd/${builtins.baseNameOf file} ~/${builtins.baseNameOf file} - ${openssh}/bin/scp ${file} mrmanager:~/${builtins.baseNameOf file} - ${openssh}/bin/ssh mrmanager doas install -o 10016 -g 10016 -m 0640 ~/${builtins.baseNameOf file} /vm/${vm_name}/persist/keys/etcd/${builtins.baseNameOf file} - ${openssh}/bin/ssh mrmanager rm -f ~/${builtins.baseNameOf file} + deploy_file = ( + { + dest_dir, + file, + name ? (builtins.baseNameOf file), + owner, + group, + mode, + }: + '' + ## + ## deploy ${name} to ${dest_dir} + ## + ${openssh}/bin/ssh mrmanager doas rm -f ${dest_dir}/${name} ~/${name} + ${openssh}/bin/scp ${file} mrmanager:~/${name} + ${openssh}/bin/ssh mrmanager doas install -o ${toString owner} -g ${toString group} -m ${mode} ~/${name} ${dest_dir}/${name} + ${openssh}/bin/ssh mrmanager doas rm -f ~/${name} + + '' ); deploy_machine = ( vm_name: ( '' + ## + ## Create directories on ${vm_name} + ## ${openssh}/bin/ssh mrmanager doas install -d -o 11235 -g 11235 -m 0755 /vm/${vm_name}/persist/keys ${openssh}/bin/ssh mrmanager doas install -d -o 10016 -g 10016 -m 0755 /vm/${vm_name}/persist/keys/etcd + ${openssh}/bin/ssh mrmanager doas install -d -o 10024 -g 10024 -m 0755 /vm/${vm_name}/persist/keys/kube + + '' - + (lib.concatMapStringsSep "\n" (deploy_key vm_name) [ - "${self.kubernetes}/kubernetes.pem" - "${self.kubernetes}/kubernetes-key.pem" - "${self.ca}/ca.pem" + + (lib.concatMapStringsSep "\n" deploy_file [ + { + dest_dir = "/vm/${vm_name}/persist/keys/etcd"; + file = "${self.kubernetes}/kubernetes.pem"; + owner = 10016; + group = 10016; + mode = "0640"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/etcd"; + file = "${self.kubernetes}/kubernetes-key.pem"; + owner = 10016; + group = 10016; + mode = "0600"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/etcd"; + file = "${self.ca}/ca.pem"; + owner = 10016; + group = 10016; + mode = "0640"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${self.kubernetes}/kubernetes.pem"; + owner = 10024; + group = 10024; + mode = "0640"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${self.kubernetes}/kubernetes-key.pem"; + owner = 10024; + group = 10024; + mode = "0640"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${self.ca}/ca.pem"; + owner = 10024; + group = 10024; + mode = "0600"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = (writeText "encryption-config.yaml" (lib.generators.toYAML { } kube_encryption_config)); + name = "encryption-config.yaml"; + owner = 10024; + group = 10024; + mode = "0600"; + } ]) ) ); @@ -84,6 +152,31 @@ makeScope newScope ( "nc2" ]) ); + kube_encryption_key = runCommand "kube_encryption_key" { } '' + head -c 32 /dev/urandom | base64 | tee $out + ''; + kube_encryption_config = { + kind = "EncryptionConfig"; + apiVersion = "v1"; + resources = [ + { + resources = [ "secrets" ]; + providers = [ + { + aescbc = { + keys = [ + { + name = "key1"; + secret = (builtins.readFile "${kube_encryption_key}"); + } + ]; + }; + } + { identity = { }; } + ]; + } + ]; + }; in { ca = (callPackage ./package/k8s-ca/package.nix additional_vars); diff --git a/nix/kubernetes/repl.nix b/nix/kubernetes/repl.nix deleted file mode 100644 index e21db83..0000000 --- a/nix/kubernetes/repl.nix +++ /dev/null @@ -1,5 +0,0 @@ -let - flake = builtins.getFlake (toString ./.); - nixpkgs = import { }; -in -{ inherit flake; } // flake // builtins // nixpkgs // nixpkgs.lib // flake.nixosConfigurations diff --git a/nix/kubernetes/roles/control_plane/default.nix b/nix/kubernetes/roles/control_plane/default.nix new file mode 100644 index 0000000..1c0b58b --- /dev/null +++ b/nix/kubernetes/roles/control_plane/default.nix @@ -0,0 +1,24 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + options.me = { + control_plane.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install control_plane."; + }; + }; + + config = lib.mkIf config.me.control_plane.enable { + me.kubernetes.enable = true; + me.kube_apiserver.enable = true; + }; +} diff --git a/nix/kubernetes/roles/kube_apiserver/default.nix b/nix/kubernetes/roles/kube_apiserver/default.nix new file mode 100644 index 0000000..4c67ff1 --- /dev/null +++ b/nix/kubernetes/roles/kube_apiserver/default.nix @@ -0,0 +1,98 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + shellCommand = cmd: (lib.concatMapStringsSep " " lib.strings.escapeShellArg cmd); +in +{ + imports = [ ]; + + options.me = { + kube_apiserver.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install kube_apiserver."; + }; + + kube_apiserver.internal_ip = lib.mkOption { + # default = { }; + example = "192.168.1.10"; + type = lib.types.str; + description = "IP address this server should advertise."; + }; + + kube_apiserver.etcd_services = lib.mkOption { + default = [ ]; + example = [ "https://192.168.1.10:2379" ]; + type = lib.types.listOf lib.types.str; + description = "Endpoints for etcd servers."; + }; + }; + + config = lib.mkIf config.me.kube_apiserver.enable { + systemd.services.kube-apiserver = { + enable = true; + description = "Kubernetes API Server"; + documentation = [ "https://github.com/kubernetes/kubernetes" ]; + wantedBy = [ "kubernetes.target" ]; + # path = with pkgs; [ + # zfs + # ]; + unitConfig.DefaultDependencies = "no"; + serviceConfig = { + Type = "notify"; + ExecStart = ( + shellCommand [ + "${pkgs.kubernetes}/bin/kube-apiserver" + "--advertise-address=${config.me.kube_apiserver.internal_ip}" + "--allow-privileged=true" + "--apiserver-count=3" + "--audit-log-maxage=30" + "--audit-log-maxbackup=3" + "--audit-log-maxsize=100" + "--audit-log-path=/var/log/audit.log" + "--authorization-mode=Node,RBAC" + "--bind-address=0.0.0.0" + "--client-ca-file=/.persist/keys/kube/ca.pem" + "--requestheader-client-ca-file=/var/lib/kubernetes/requestheader-client-ca.pem" + ''--requestheader-allowed-names=""'' + "--requestheader-extra-headers-prefix=X-Remote-Extra-" + "--requestheader-group-headers=X-Remote-Group" + "--requestheader-username-headers=X-Remote-User" + "--proxy-client-cert-file=/var/lib/kubernetes/{{ node_name }}-proxy.pem" + "--proxy-client-key-file=/var/lib/kubernetes/{{ node_name }}-proxy-key.pem" + "--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota" + "--etcd-cafile=/.persist/keys/kube/ca.pem" + "--etcd-certfile=/.persist/keys/kube/kubernetes.pem" + "--etcd-keyfile=/.persist/keys/kube/kubernetes-key.pem" + "--etcd-servers=${builtins.concatStringsSep "," config.me.kube_apiserver.etcd_services}" + "--event-ttl=1h" + "--encryption-provider-config=/.persist/keys/kube/encryption-config.yaml" + "--kubelet-certificate-authority=/.persist/keys/kube/ca.pem" + "--kubelet-client-certificate=/.persist/keys/kube/kubernetes.pem" + "--kubelet-client-key=/.persist/keys/kube/kubernetes-key.pem" + "--runtime-config='api/all=true'" + "--service-account-key-file=/var/lib/kubernetes/service-account.pem" + "--service-account-signing-key-file=/var/lib/kubernetes/service-account-key.pem" + "--service-account-issuer=https://{{ kubernetes_public_address }}:6443" + "--service-node-port-range=30000-32767" + "--tls-cert-file=/.persist/keys/kube/kubernetes.pem" + "--tls-private-key-file=/.persist/keys/kube/kubernetes-key.pem" + "--tls-min-version=VersionTLS13" + "--kubelet-preferred-address-types=InternalIP,ExternalDNS,ExternalIP,Hostname,InternalDNS" + "--service-cluster-ip-range=10.197.0.0/16" + "--enable-aggregator-routing=true" + "--v=2" + ] + ); + Restart = "on-failure"; + RestartSec = 5; + }; + }; + }; +} diff --git a/nix/kubernetes/roles/kubernetes/default.nix b/nix/kubernetes/roles/kubernetes/default.nix new file mode 100644 index 0000000..9d3646d --- /dev/null +++ b/nix/kubernetes/roles/kubernetes/default.nix @@ -0,0 +1,53 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + options.me = { + kubernetes.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install kubernetes."; + }; + }; + + config = lib.mkIf config.me.kubernetes.enable { + environment.systemPackages = with pkgs; [ + kubernetes + ]; + + systemd.targets.kubernetes = { + description = "Kubernetes"; + wantedBy = [ "multi-user.target" ]; + }; + + users.users.kubernetes = { + uid = 10024; + description = "Kubernetes"; + group = "kubernetes"; + home = "/var/lib/kubernetes"; + createHome = true; + homeMode = "755"; + isSystemUser = true; + }; + users.groups.kubernetes.gid = 10024; + + environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) { + hideMounts = true; + directories = [ + { + directory = "/var/lib/kubernetes"; + user = "kubernetes"; + group = "kubernetes"; + mode = "0755"; + } + ]; + }; + }; +} From 6aa87faecb4f128fc469ce3958d40bfb48cbbbee Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 14 Dec 2025 13:37:46 -0500 Subject: [PATCH 04/81] Add service account. --- .../keys/package/k8s-keys/package.nix | 1 + .../keys/package/k8s-kubernetes/package.nix | 2 +- .../k8s-service-account/files/ca-config.json | 13 +++++++ .../files/service-account-csr.json | 16 +++++++++ .../package/k8s-service-account/package.nix | 35 +++++++++++++++++++ nix/kubernetes/keys/scope.nix | 15 ++++++++ .../roles/kube_apiserver/default.nix | 4 +-- 7 files changed, 83 insertions(+), 3 deletions(-) create mode 100644 nix/kubernetes/keys/package/k8s-service-account/files/ca-config.json create mode 100644 nix/kubernetes/keys/package/k8s-service-account/files/service-account-csr.json create mode 100644 nix/kubernetes/keys/package/k8s-service-account/package.nix diff --git a/nix/kubernetes/keys/package/k8s-keys/package.nix b/nix/kubernetes/keys/package/k8s-keys/package.nix index a890aa3..497f1e0 100644 --- a/nix/kubernetes/keys/package/k8s-keys/package.nix +++ b/nix/kubernetes/keys/package/k8s-keys/package.nix @@ -8,5 +8,6 @@ symlinkJoin { paths = [ k8s.kubernetes k8s.ca + k8s.service_account ]; } diff --git a/nix/kubernetes/keys/package/k8s-kubernetes/package.nix b/nix/kubernetes/keys/package/k8s-kubernetes/package.nix index 4bef3ed..daadf88 100644 --- a/nix/kubernetes/keys/package/k8s-kubernetes/package.nix +++ b/nix/kubernetes/keys/package/k8s-kubernetes/package.nix @@ -16,7 +16,7 @@ ... }: stdenv.mkDerivation (finalAttrs: { - name = "k8s-keys"; + name = "k8s-kubernetes"; nativeBuildInputs = [ cfssl ]; buildInputs = [ ]; diff --git a/nix/kubernetes/keys/package/k8s-service-account/files/ca-config.json b/nix/kubernetes/keys/package/k8s-service-account/files/ca-config.json new file mode 100644 index 0000000..a63e0dd --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-service-account/files/ca-config.json @@ -0,0 +1,13 @@ +{ + "signing": { + "default": { + "expiry": "8760h" + }, + "profiles": { + "kubernetes": { + "usages": ["signing", "key encipherment", "server auth", "client auth"], + "expiry": "8760h" + } + } + } +} diff --git a/nix/kubernetes/keys/package/k8s-service-account/files/service-account-csr.json b/nix/kubernetes/keys/package/k8s-service-account/files/service-account-csr.json new file mode 100644 index 0000000..be3c0ca --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-service-account/files/service-account-csr.json @@ -0,0 +1,16 @@ +{ + "CN": "service-accounts", + "key": { + "algo": "rsa", + "size": 2048 + }, + "names": [ + { + "C": "US", + "L": "Portland", + "O": "Kubernetes", + "OU": "Kubernetes The Hard Way", + "ST": "Oregon" + } + ] +} diff --git a/nix/kubernetes/keys/package/k8s-service-account/package.nix b/nix/kubernetes/keys/package/k8s-service-account/package.nix new file mode 100644 index 0000000..717f059 --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-service-account/package.nix @@ -0,0 +1,35 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + stdenv, + sqlite, + cfssl, + k8s, + all_hostnames, + ... +}: +stdenv.mkDerivation (finalAttrs: { + name = "k8s-service-account"; + nativeBuildInputs = [ cfssl ]; + buildInputs = [ ]; + + unpackPhase = "true"; + + installPhase = '' + mkdir -p "$out" + cd "$out" + cfssl gencert \ + -ca=${k8s.ca}/ca.pem \ + -ca-key=${k8s.ca}/ca-key.pem \ + -config=${./files/ca-config.json} \ + -profile=kubernetes \ + ${./files/service-account-csr.json} | cfssljson -bare service-account + ''; +}) diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index 402c733..9135e96 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -137,6 +137,20 @@ makeScope newScope ( group = 10024; mode = "0600"; } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${self.service_account}/service-account.pem"; + owner = 10024; + group = 10024; + mode = "0600"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${self.service_account}/service-account-key.pem"; + owner = 10024; + group = 10024; + mode = "0600"; + } ]) ) ); @@ -181,6 +195,7 @@ makeScope newScope ( { ca = (callPackage ./package/k8s-ca/package.nix additional_vars); kubernetes = (callPackage ./package/k8s-kubernetes/package.nix additional_vars); + service_account = (callPackage ./package/k8s-service-account/package.nix additional_vars); keys = (callPackage ./package/k8s-keys/package.nix additional_vars); deploy_script = (writeShellScript "deploy-keys" deploy_script); } diff --git a/nix/kubernetes/roles/kube_apiserver/default.nix b/nix/kubernetes/roles/kube_apiserver/default.nix index 4c67ff1..bbaee42 100644 --- a/nix/kubernetes/roles/kube_apiserver/default.nix +++ b/nix/kubernetes/roles/kube_apiserver/default.nix @@ -77,8 +77,8 @@ in "--kubelet-client-certificate=/.persist/keys/kube/kubernetes.pem" "--kubelet-client-key=/.persist/keys/kube/kubernetes-key.pem" "--runtime-config='api/all=true'" - "--service-account-key-file=/var/lib/kubernetes/service-account.pem" - "--service-account-signing-key-file=/var/lib/kubernetes/service-account-key.pem" + "--service-account-key-file=/.persist/keys/kube/service-account.pem" + "--service-account-signing-key-file=/.persist/keys/kube/service-account-key.pem" "--service-account-issuer=https://{{ kubernetes_public_address }}:6443" "--service-node-port-range=30000-32767" "--tls-cert-file=/.persist/keys/kube/kubernetes.pem" From 6c07157f215e790583c2ddf0b2c1d6d1536908a8 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 14 Dec 2025 13:44:56 -0500 Subject: [PATCH 05/81] Add requestheader-client-ca. --- nix/kubernetes/hosts/controller0/default.nix | 1 + nix/kubernetes/hosts/controller1/default.nix | 1 + nix/kubernetes/hosts/controller2/default.nix | 1 + .../keys/package/k8s-keys/package.nix | 1 + .../files/requestheader-client-ca-csr.json | 16 ++++++++++ .../k8s-requestheader-client-ca/package.nix | 30 +++++++++++++++++++ nix/kubernetes/keys/scope.nix | 10 +++++++ .../roles/kube_apiserver/default.nix | 10 +++++-- 8 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 nix/kubernetes/keys/package/k8s-requestheader-client-ca/files/requestheader-client-ca-csr.json create mode 100644 nix/kubernetes/keys/package/k8s-requestheader-client-ca/package.nix diff --git a/nix/kubernetes/hosts/controller0/default.nix b/nix/kubernetes/hosts/controller0/default.nix index ef6a64f..5361612 100644 --- a/nix/kubernetes/hosts/controller0/default.nix +++ b/nix/kubernetes/hosts/controller0/default.nix @@ -117,6 +117,7 @@ ]; me.kube_apiserver.internal_ip = "2620:11f:7001:7:ffff:ffff:0ad7:01dd"; + me.kube_apiserver.external_ip = "74.80.180.138"; me.kube_apiserver.etcd_services = [ "https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2379" # 10.215.1.221 "https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2379" # 10.215.1.222 diff --git a/nix/kubernetes/hosts/controller1/default.nix b/nix/kubernetes/hosts/controller1/default.nix index 61f26a5..d6571c0 100644 --- a/nix/kubernetes/hosts/controller1/default.nix +++ b/nix/kubernetes/hosts/controller1/default.nix @@ -117,6 +117,7 @@ ]; me.kube_apiserver.internal_ip = "2620:11f:7001:7:ffff:ffff:0ad7:01de"; + me.kube_apiserver.external_ip = "74.80.180.138"; me.kube_apiserver.etcd_services = [ "https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2379" # 10.215.1.221 "https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2379" # 10.215.1.222 diff --git a/nix/kubernetes/hosts/controller2/default.nix b/nix/kubernetes/hosts/controller2/default.nix index 8336a01..c6c95c0 100644 --- a/nix/kubernetes/hosts/controller2/default.nix +++ b/nix/kubernetes/hosts/controller2/default.nix @@ -117,6 +117,7 @@ ]; me.kube_apiserver.internal_ip = "2620:11f:7001:7:ffff:ffff:0ad7:01df"; + me.kube_apiserver.external_ip = "74.80.180.138"; me.kube_apiserver.etcd_services = [ "https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2379" # 10.215.1.221 "https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2379" # 10.215.1.222 diff --git a/nix/kubernetes/keys/package/k8s-keys/package.nix b/nix/kubernetes/keys/package/k8s-keys/package.nix index 497f1e0..42d9834 100644 --- a/nix/kubernetes/keys/package/k8s-keys/package.nix +++ b/nix/kubernetes/keys/package/k8s-keys/package.nix @@ -9,5 +9,6 @@ symlinkJoin { k8s.kubernetes k8s.ca k8s.service_account + k8s.requestheader-client-ca ]; } diff --git a/nix/kubernetes/keys/package/k8s-requestheader-client-ca/files/requestheader-client-ca-csr.json b/nix/kubernetes/keys/package/k8s-requestheader-client-ca/files/requestheader-client-ca-csr.json new file mode 100644 index 0000000..8145e50 --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-requestheader-client-ca/files/requestheader-client-ca-csr.json @@ -0,0 +1,16 @@ +{ + "CN": "Kubernetes", + "key": { + "algo": "rsa", + "size": 2048 + }, + "names": [ + { + "C": "US", + "L": "Portland", + "O": "Kubernetes", + "OU": "CA", + "ST": "Oregon" + } + ] +} diff --git a/nix/kubernetes/keys/package/k8s-requestheader-client-ca/package.nix b/nix/kubernetes/keys/package/k8s-requestheader-client-ca/package.nix new file mode 100644 index 0000000..4d907f5 --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-requestheader-client-ca/package.nix @@ -0,0 +1,30 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + stdenv, + sqlite, + cfssl, + k8s, + all_hostnames, + ... +}: +stdenv.mkDerivation (finalAttrs: { + name = "k8s-service-account"; + nativeBuildInputs = [ cfssl ]; + buildInputs = [ ]; + + unpackPhase = "true"; + + installPhase = '' + mkdir -p "$out" + cd "$out" + cfssl gencert -initca ${./files/requestheader-client-ca-csr.json} | cfssljson -bare requestheader-client-ca + ''; +}) diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index 9135e96..8f490e9 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -151,6 +151,13 @@ makeScope newScope ( group = 10024; mode = "0600"; } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${self.requestheader-client-ca}/requestheader-client-ca.pem"; + owner = 10024; + group = 10024; + mode = "0600"; + } ]) ) ); @@ -196,6 +203,9 @@ makeScope newScope ( ca = (callPackage ./package/k8s-ca/package.nix additional_vars); kubernetes = (callPackage ./package/k8s-kubernetes/package.nix additional_vars); service_account = (callPackage ./package/k8s-service-account/package.nix additional_vars); + requestheader-client-ca = ( + callPackage ./package/k8s-requestheader-client-ca/package.nix additional_vars + ); keys = (callPackage ./package/k8s-keys/package.nix additional_vars); deploy_script = (writeShellScript "deploy-keys" deploy_script); } diff --git a/nix/kubernetes/roles/kube_apiserver/default.nix b/nix/kubernetes/roles/kube_apiserver/default.nix index bbaee42..9701d99 100644 --- a/nix/kubernetes/roles/kube_apiserver/default.nix +++ b/nix/kubernetes/roles/kube_apiserver/default.nix @@ -26,6 +26,12 @@ in description = "IP address this server should advertise."; }; + kube_apiserver.external_ip = lib.mkOption { + example = "192.168.1.10"; + type = lib.types.str; + description = "IP address to reach this cluster externally."; + }; + kube_apiserver.etcd_services = lib.mkOption { default = [ ]; example = [ "https://192.168.1.10:2379" ]; @@ -59,7 +65,7 @@ in "--authorization-mode=Node,RBAC" "--bind-address=0.0.0.0" "--client-ca-file=/.persist/keys/kube/ca.pem" - "--requestheader-client-ca-file=/var/lib/kubernetes/requestheader-client-ca.pem" + "--requestheader-client-ca-file=/.persist/keys/kube/requestheader-client-ca.pem" ''--requestheader-allowed-names=""'' "--requestheader-extra-headers-prefix=X-Remote-Extra-" "--requestheader-group-headers=X-Remote-Group" @@ -79,7 +85,7 @@ in "--runtime-config='api/all=true'" "--service-account-key-file=/.persist/keys/kube/service-account.pem" "--service-account-signing-key-file=/.persist/keys/kube/service-account-key.pem" - "--service-account-issuer=https://{{ kubernetes_public_address }}:6443" + "--service-account-issuer=https://${config.me.kube_apiserver.external_ip}:6443" "--service-node-port-range=30000-32767" "--tls-cert-file=/.persist/keys/kube/kubernetes.pem" "--tls-private-key-file=/.persist/keys/kube/kubernetes-key.pem" From f8ad309c0b8dfed19e5dcad004a2893e2bdb37e3 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 14 Dec 2025 14:48:53 -0500 Subject: [PATCH 06/81] Add controller proxy certs. --- .../k8s-controller-proxy/files/ca-config.json | 13 +++++ .../files/controller0-proxy-csr.json | 16 ++++++ .../files/controller1-proxy-csr.json | 16 ++++++ .../files/controller2-proxy-csr.json | 16 ++++++ .../package/k8s-controller-proxy/package.nix | 48 +++++++++++++++++ .../keys/package/k8s-keys/package.nix | 1 + nix/kubernetes/keys/scope.nix | 52 ++++++++++++++++++- .../roles/kube_apiserver/default.nix | 7 +-- 8 files changed, 165 insertions(+), 4 deletions(-) create mode 100644 nix/kubernetes/keys/package/k8s-controller-proxy/files/ca-config.json create mode 100644 nix/kubernetes/keys/package/k8s-controller-proxy/files/controller0-proxy-csr.json create mode 100644 nix/kubernetes/keys/package/k8s-controller-proxy/files/controller1-proxy-csr.json create mode 100644 nix/kubernetes/keys/package/k8s-controller-proxy/files/controller2-proxy-csr.json create mode 100644 nix/kubernetes/keys/package/k8s-controller-proxy/package.nix diff --git a/nix/kubernetes/keys/package/k8s-controller-proxy/files/ca-config.json b/nix/kubernetes/keys/package/k8s-controller-proxy/files/ca-config.json new file mode 100644 index 0000000..a63e0dd --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-controller-proxy/files/ca-config.json @@ -0,0 +1,13 @@ +{ + "signing": { + "default": { + "expiry": "8760h" + }, + "profiles": { + "kubernetes": { + "usages": ["signing", "key encipherment", "server auth", "client auth"], + "expiry": "8760h" + } + } + } +} diff --git a/nix/kubernetes/keys/package/k8s-controller-proxy/files/controller0-proxy-csr.json b/nix/kubernetes/keys/package/k8s-controller-proxy/files/controller0-proxy-csr.json new file mode 100644 index 0000000..31a18ea --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-controller-proxy/files/controller0-proxy-csr.json @@ -0,0 +1,16 @@ +{ + "CN": "system:node:controller0", + "key": { + "algo": "rsa", + "size": 2048 + }, + "names": [ + { + "C": "US", + "L": "Portland", + "O": "system:nodes", + "OU": "Kubernetes The Hard Way", + "ST": "Oregon" + } + ] +} diff --git a/nix/kubernetes/keys/package/k8s-controller-proxy/files/controller1-proxy-csr.json b/nix/kubernetes/keys/package/k8s-controller-proxy/files/controller1-proxy-csr.json new file mode 100644 index 0000000..b5defb2 --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-controller-proxy/files/controller1-proxy-csr.json @@ -0,0 +1,16 @@ +{ + "CN": "system:node:controller1", + "key": { + "algo": "rsa", + "size": 2048 + }, + "names": [ + { + "C": "US", + "L": "Portland", + "O": "system:nodes", + "OU": "Kubernetes The Hard Way", + "ST": "Oregon" + } + ] +} diff --git a/nix/kubernetes/keys/package/k8s-controller-proxy/files/controller2-proxy-csr.json b/nix/kubernetes/keys/package/k8s-controller-proxy/files/controller2-proxy-csr.json new file mode 100644 index 0000000..09c2666 --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-controller-proxy/files/controller2-proxy-csr.json @@ -0,0 +1,16 @@ +{ + "CN": "system:node:controller2", + "key": { + "algo": "rsa", + "size": 2048 + }, + "names": [ + { + "C": "US", + "L": "Portland", + "O": "system:nodes", + "OU": "Kubernetes The Hard Way", + "ST": "Oregon" + } + ] +} diff --git a/nix/kubernetes/keys/package/k8s-controller-proxy/package.nix b/nix/kubernetes/keys/package/k8s-controller-proxy/package.nix new file mode 100644 index 0000000..ba965bb --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-controller-proxy/package.nix @@ -0,0 +1,48 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + lib, + stdenv, + sqlite, + cfssl, + k8s, + all_hostnames, + controllers, + ... +}: +let + get_hostnames = ( + hostname: (builtins.concatStringsSep "," ([ hostname ] ++ controllers."${hostname}".internal_ips)) + ); + install_body = ( + lib.concatMapStringsSep "\n" (hostname: '' + cfssl gencert \ + -ca=${k8s.requestheader-client-ca}/requestheader-client-ca.pem \ + -ca-key=${k8s.requestheader-client-ca}/requestheader-client-ca-key.pem \ + -config=${./files/ca-config.json} \ + -hostname=${get_hostnames hostname} \ + -profile=kubernetes \ + ${./files}/${hostname}-proxy-csr.json | cfssljson -bare ${hostname}-proxy + '') (builtins.attrNames controllers) + ); +in +stdenv.mkDerivation (finalAttrs: { + name = "k8s-controller-proxy"; + nativeBuildInputs = [ cfssl ]; + buildInputs = [ ]; + + unpackPhase = "true"; + + installPhase = '' + mkdir -p "$out" + cd "$out" + '' + + install_body; +}) diff --git a/nix/kubernetes/keys/package/k8s-keys/package.nix b/nix/kubernetes/keys/package/k8s-keys/package.nix index 42d9834..151e9ec 100644 --- a/nix/kubernetes/keys/package/k8s-keys/package.nix +++ b/nix/kubernetes/keys/package/k8s-keys/package.nix @@ -10,5 +10,6 @@ symlinkJoin { k8s.ca k8s.service_account k8s.requestheader-client-ca + k8s.controller-proxy ]; } diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index 8f490e9..21acdce 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -44,12 +44,47 @@ let ] ++ public_addresses ++ internal_addresses; + controllers = { + "controller0" = { + "internal_ips" = [ + "10.215.1.221" + "2620:11f:7001:7:ffff:ffff:0ad7:01dd" + ]; + "external_ips" = [ + "2620:11f:7001:7:ffff:ffff:0ad7:01dd" + ]; + }; + "controller1" = { + "internal_ips" = [ + "10.215.1.222" + "2620:11f:7001:7:ffff:ffff:0ad7:01de" + ]; + "external_ips" = [ + "2620:11f:7001:7:ffff:ffff:0ad7:01de" + ]; + }; + "controller2" = { + "internal_ips" = [ + "10.215.1.223" + "2620:11f:7001:7:ffff:ffff:0ad7:01df" + ]; + "external_ips" = [ + "2620:11f:7001:7:ffff:ffff:0ad7:01df" + ]; + }; + }; + _vm_name_to_hostname = { + "nc0" = "controller0"; + "nc1" = "controller1"; + "nc2" = "controller2"; + }; + vm_name_to_hostname = (vm_name: _vm_name_to_hostname."${vm_name}"); in makeScope newScope ( self: let additional_vars = { - inherit all_hostnames; + inherit all_hostnames controllers; k8s = self; }; deploy_file = ( @@ -158,6 +193,20 @@ makeScope newScope ( group = 10024; mode = "0600"; } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${self.controller-proxy}/${vm_name_to_hostname vm_name}-proxy.pem"; + owner = 10024; + group = 10024; + mode = "0600"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${self.controller-proxy}/${vm_name_to_hostname vm_name}-proxy-key.pem"; + owner = 10024; + group = 10024; + mode = "0600"; + } ]) ) ); @@ -206,6 +255,7 @@ makeScope newScope ( requestheader-client-ca = ( callPackage ./package/k8s-requestheader-client-ca/package.nix additional_vars ); + controller-proxy = (callPackage ./package/k8s-controller-proxy/package.nix additional_vars); keys = (callPackage ./package/k8s-keys/package.nix additional_vars); deploy_script = (writeShellScript "deploy-keys" deploy_script); } diff --git a/nix/kubernetes/roles/kube_apiserver/default.nix b/nix/kubernetes/roles/kube_apiserver/default.nix index 9701d99..4a0881f 100644 --- a/nix/kubernetes/roles/kube_apiserver/default.nix +++ b/nix/kubernetes/roles/kube_apiserver/default.nix @@ -70,8 +70,8 @@ in "--requestheader-extra-headers-prefix=X-Remote-Extra-" "--requestheader-group-headers=X-Remote-Group" "--requestheader-username-headers=X-Remote-User" - "--proxy-client-cert-file=/var/lib/kubernetes/{{ node_name }}-proxy.pem" - "--proxy-client-key-file=/var/lib/kubernetes/{{ node_name }}-proxy-key.pem" + "--proxy-client-cert-file=/.persist/keys/kube/${config.networking.hostName}-proxy.pem" + "--proxy-client-key-file=/.persist/keys/kube/${config.networking.hostName}-proxy-key.pem" "--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota" "--etcd-cafile=/.persist/keys/kube/ca.pem" "--etcd-certfile=/.persist/keys/kube/kubernetes.pem" @@ -91,7 +91,8 @@ in "--tls-private-key-file=/.persist/keys/kube/kubernetes-key.pem" "--tls-min-version=VersionTLS13" "--kubelet-preferred-address-types=InternalIP,ExternalDNS,ExternalIP,Hostname,InternalDNS" - "--service-cluster-ip-range=10.197.0.0/16" + # "--service-cluster-ip-range=10.197.0.0/16" + "--service-cluster-ip-range=2620:11f:7001:7:ffff:ffff:0ac5:0000/16" "--enable-aggregator-routing=true" "--v=2" ] From f4891f345ebc1e4876bbec1e192bd7ab0b94db0a Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 14 Dec 2025 18:24:24 -0500 Subject: [PATCH 07/81] Switch to generating certs with openssl. --- nix/kubernetes/keys/flake.nix | 2 +- .../package.nix | 16 +- .../keys/package/k8s-ca/files/ca-csr.json | 16 - .../keys/package/k8s-ca/files/ca.conf | 291 ++++++++++++++++++ .../keys/package/k8s-ca/package.nix | 12 +- .../package/k8s-client-config/package.nix | 52 ++++ .../k8s-controller-proxy/files/ca-config.json | 13 - .../files/controller0-proxy-csr.json | 16 - .../files/controller1-proxy-csr.json | 16 - .../files/controller2-proxy-csr.json | 16 - .../package/k8s-controller-proxy/package.nix | 48 --- .../package/k8s-keys/files/ca-config.json | 13 - .../k8s-keys/files/kubernetes-csr.json | 16 - .../keys/package/k8s-keys/package.nix | 8 +- .../k8s-kubernetes/files/ca-config.json | 13 - .../k8s-kubernetes/files/kubernetes-csr.json | 16 - .../keys/package/k8s-kubernetes/package.nix | 36 --- .../files/requestheader-client-ca-csr.json | 16 - .../k8s-service-account/files/ca-config.json | 13 - .../files/service-account-csr.json | 16 - .../package/k8s-service-account/package.nix | 35 --- .../keys/package/tls-key/package.nix | 45 +++ nix/kubernetes/keys/scope.nix | 75 ++++- 23 files changed, 476 insertions(+), 324 deletions(-) rename nix/kubernetes/keys/package/{k8s-requestheader-client-ca => deploy-script}/package.nix (52%) delete mode 100644 nix/kubernetes/keys/package/k8s-ca/files/ca-csr.json create mode 100644 nix/kubernetes/keys/package/k8s-ca/files/ca.conf create mode 100644 nix/kubernetes/keys/package/k8s-client-config/package.nix delete mode 100644 nix/kubernetes/keys/package/k8s-controller-proxy/files/ca-config.json delete mode 100644 nix/kubernetes/keys/package/k8s-controller-proxy/files/controller0-proxy-csr.json delete mode 100644 nix/kubernetes/keys/package/k8s-controller-proxy/files/controller1-proxy-csr.json delete mode 100644 nix/kubernetes/keys/package/k8s-controller-proxy/files/controller2-proxy-csr.json delete mode 100644 nix/kubernetes/keys/package/k8s-controller-proxy/package.nix delete mode 100644 nix/kubernetes/keys/package/k8s-keys/files/ca-config.json delete mode 100644 nix/kubernetes/keys/package/k8s-keys/files/kubernetes-csr.json delete mode 100644 nix/kubernetes/keys/package/k8s-kubernetes/files/ca-config.json delete mode 100644 nix/kubernetes/keys/package/k8s-kubernetes/files/kubernetes-csr.json delete mode 100644 nix/kubernetes/keys/package/k8s-kubernetes/package.nix delete mode 100644 nix/kubernetes/keys/package/k8s-requestheader-client-ca/files/requestheader-client-ca-csr.json delete mode 100644 nix/kubernetes/keys/package/k8s-service-account/files/ca-config.json delete mode 100644 nix/kubernetes/keys/package/k8s-service-account/files/service-account-csr.json delete mode 100644 nix/kubernetes/keys/package/k8s-service-account/package.nix create mode 100644 nix/kubernetes/keys/package/tls-key/package.nix diff --git a/nix/kubernetes/keys/flake.nix b/nix/kubernetes/keys/flake.nix index 3fae683..af515b5 100644 --- a/nix/kubernetes/keys/flake.nix +++ b/nix/kubernetes/keys/flake.nix @@ -19,7 +19,7 @@ in { deploy_script = appliedOverlay.k8s.deploy_script; - default = appliedOverlay.k8s.keys; + default = appliedOverlay.k8s.all_keys; } ); overlays.default = ( diff --git a/nix/kubernetes/keys/package/k8s-requestheader-client-ca/package.nix b/nix/kubernetes/keys/package/deploy-script/package.nix similarity index 52% rename from nix/kubernetes/keys/package/k8s-requestheader-client-ca/package.nix rename to nix/kubernetes/keys/package/deploy-script/package.nix index 4d907f5..dfe85ed 100644 --- a/nix/kubernetes/keys/package/k8s-requestheader-client-ca/package.nix +++ b/nix/kubernetes/keys/package/deploy-script/package.nix @@ -9,22 +9,22 @@ # distPhase { stdenv, - sqlite, - cfssl, + writeShellScript, k8s, - all_hostnames, ... }: +let + deploy_script_body = ""; + deploy_script = (writeShellScript "deploy-script" deploy_script_body); +in stdenv.mkDerivation (finalAttrs: { - name = "k8s-service-account"; - nativeBuildInputs = [ cfssl ]; + name = "deploy-script"; + nativeBuildInputs = [ ]; buildInputs = [ ]; unpackPhase = "true"; installPhase = '' - mkdir -p "$out" - cd "$out" - cfssl gencert -initca ${./files/requestheader-client-ca-csr.json} | cfssljson -bare requestheader-client-ca + cp ${deploy_script} "$out" ''; }) diff --git a/nix/kubernetes/keys/package/k8s-ca/files/ca-csr.json b/nix/kubernetes/keys/package/k8s-ca/files/ca-csr.json deleted file mode 100644 index 8145e50..0000000 --- a/nix/kubernetes/keys/package/k8s-ca/files/ca-csr.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "CN": "Kubernetes", - "key": { - "algo": "rsa", - "size": 2048 - }, - "names": [ - { - "C": "US", - "L": "Portland", - "O": "Kubernetes", - "OU": "CA", - "ST": "Oregon" - } - ] -} diff --git a/nix/kubernetes/keys/package/k8s-ca/files/ca.conf b/nix/kubernetes/keys/package/k8s-ca/files/ca.conf new file mode 100644 index 0000000..3c7021c --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-ca/files/ca.conf @@ -0,0 +1,291 @@ +[req] +distinguished_name = req_distinguished_name +prompt = no +x509_extensions = ca_x509_extensions + +[ca_x509_extensions] +basicConstraints = CA:TRUE +keyUsage = cRLSign, keyCertSign + +[req_distinguished_name] +C = US +ST = Washington +L = Seattle +CN = CA + +[admin] +distinguished_name = admin_distinguished_name +prompt = no +req_extensions = default_req_extensions + +[admin_distinguished_name] +CN = admin +O = system:masters + +# Service Accounts +# +# The Kubernetes Controller Manager leverages a key pair to generate +# and sign service account tokens as described in the +# [managing service accounts](https://kubernetes.io/docs/admin/service-accounts-admin/) +# documentation. + +[service-accounts] +distinguished_name = service-accounts_distinguished_name +prompt = no +req_extensions = default_req_extensions + +[service-accounts_distinguished_name] +CN = service-accounts + +# Worker Nodes +# +# Kubernetes uses a [special-purpose authorization mode](https://kubernetes.io/docs/admin/authorization/node/) +# called Node Authorizer, that specifically authorizes API requests made +# by [Kubelets](https://kubernetes.io/docs/concepts/overview/components/#kubelet). +# In order to be authorized by the Node Authorizer, Kubelets must use a credential +# that identifies them as being in the `system:nodes` group, with a username +# of `system:node:`. + +[controller0] +distinguished_name = controller0_distinguished_name +prompt = no +req_extensions = controller0_req_extensions + +[controller0_req_extensions] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth, serverAuth +keyUsage = critical, digitalSignature, keyEncipherment +nsCertType = client +nsComment = "controller0 Certificate" +subjectAltName = DNS:controller0, IP:127.0.0.1 +subjectKeyIdentifier = hash + +[controller0_distinguished_name] +CN = system:node:controller0 +O = system:nodes +C = US +ST = Washington +L = Seattle + +[controller1] +distinguished_name = controller1_distinguished_name +prompt = no +req_extensions = controller1_req_extensions + +[controller1_req_extensions] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth, serverAuth +keyUsage = critical, digitalSignature, keyEncipherment +nsCertType = client +nsComment = "controller1 Certificate" +subjectAltName = DNS:controller1, IP:127.0.0.1 +subjectKeyIdentifier = hash + +[controller1_distinguished_name] +CN = system:node:controller1 +O = system:nodes +C = US +ST = Washington +L = Seattle + +[controller2] +distinguished_name = controller2_distinguished_name +prompt = no +req_extensions = controller2_req_extensions + +[controller2_req_extensions] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth, serverAuth +keyUsage = critical, digitalSignature, keyEncipherment +nsCertType = client +nsComment = "controller2 Certificate" +subjectAltName = DNS:controller2, IP:127.0.0.1 +subjectKeyIdentifier = hash + +[controller2_distinguished_name] +CN = system:node:controller2 +O = system:nodes +C = US +ST = Washington +L = Seattle + +[worker0] +distinguished_name = worker0_distinguished_name +prompt = no +req_extensions = worker0_req_extensions + +[worker0_req_extensions] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth, serverAuth +keyUsage = critical, digitalSignature, keyEncipherment +nsCertType = client +nsComment = "worker0 Certificate" +subjectAltName = DNS:worker0, IP:127.0.0.1 +subjectKeyIdentifier = hash + +[worker0_distinguished_name] +CN = system:node:worker0 +O = system:nodes +C = US +ST = Washington +L = Seattle + +[worker1] +distinguished_name = worker1_distinguished_name +prompt = no +req_extensions = worker1_req_extensions + +[worker1_req_extensions] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth, serverAuth +keyUsage = critical, digitalSignature, keyEncipherment +nsCertType = client +nsComment = "worker1 Certificate" +subjectAltName = DNS:worker1, IP:127.0.0.1 +subjectKeyIdentifier = hash + +[worker1_distinguished_name] +CN = system:node:worker1 +O = system:nodes +C = US +ST = Washington +L = Seattle + +[worker2] +distinguished_name = worker2_distinguished_name +prompt = no +req_extensions = worker2_req_extensions + +[worker2_req_extensions] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth, serverAuth +keyUsage = critical, digitalSignature, keyEncipherment +nsCertType = client +nsComment = "worker2 Certificate" +subjectAltName = DNS:worker2, IP:127.0.0.1 +subjectKeyIdentifier = hash + +[worker2_distinguished_name] +CN = system:node:worker2 +O = system:nodes +C = US +ST = Washington +L = Seattle + + + +# Kube Proxy Section +[kube-proxy] +distinguished_name = kube-proxy_distinguished_name +prompt = no +req_extensions = kube-proxy_req_extensions + +[kube-proxy_req_extensions] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth, serverAuth +keyUsage = critical, digitalSignature, keyEncipherment +nsCertType = client +nsComment = "Kube Proxy Certificate" +subjectAltName = DNS:kube-proxy, IP:127.0.0.1 +subjectKeyIdentifier = hash + +[kube-proxy_distinguished_name] +CN = system:kube-proxy +O = system:node-proxier +C = US +ST = Washington +L = Seattle + + +# Controller Manager +[kube-controller-manager] +distinguished_name = kube-controller-manager_distinguished_name +prompt = no +req_extensions = kube-controller-manager_req_extensions + +[kube-controller-manager_req_extensions] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth, serverAuth +keyUsage = critical, digitalSignature, keyEncipherment +nsCertType = client +nsComment = "Kube Controller Manager Certificate" +subjectAltName = DNS:kube-controller-manager, IP:127.0.0.1 +subjectKeyIdentifier = hash + +[kube-controller-manager_distinguished_name] +CN = system:kube-controller-manager +O = system:kube-controller-manager +C = US +ST = Washington +L = Seattle + + +# Scheduler +[kube-scheduler] +distinguished_name = kube-scheduler_distinguished_name +prompt = no +req_extensions = kube-scheduler_req_extensions + +[kube-scheduler_req_extensions] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth, serverAuth +keyUsage = critical, digitalSignature, keyEncipherment +nsCertType = client +nsComment = "Kube Scheduler Certificate" +subjectAltName = DNS:kube-scheduler, IP:127.0.0.1 +subjectKeyIdentifier = hash + +[kube-scheduler_distinguished_name] +CN = system:kube-scheduler +O = system:system:kube-scheduler +C = US +ST = Washington +L = Seattle + + +# API Server +# +# The Kubernetes API server is automatically assigned the `kubernetes` +# internal dns name, which will be linked to the first IP address (`10.32.0.1`) +# from the address range (`10.32.0.0/24`) reserved for internal cluster +# services. + +[kube-api-server] +distinguished_name = kube-api-server_distinguished_name +prompt = no +req_extensions = kube-api-server_req_extensions + +[kube-api-server_req_extensions] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth, serverAuth +keyUsage = critical, digitalSignature, keyEncipherment +nsCertType = client, server +nsComment = "Kube API Server Certificate" +subjectAltName = @kube-api-server_alt_names +subjectKeyIdentifier = hash + +[kube-api-server_alt_names] +IP.0 = 127.0.0.1 +IP.1 = 10.32.0.1 +DNS.0 = kubernetes +DNS.1 = kubernetes.default +DNS.2 = kubernetes.default.svc +DNS.3 = kubernetes.default.svc.cluster +DNS.4 = kubernetes.svc.cluster.local +DNS.5 = server.kubernetes.local +DNS.6 = api-server.kubernetes.local + +[kube-api-server_distinguished_name] +CN = kubernetes +C = US +ST = Washington +L = Seattle + + +[default_req_extensions] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth +keyUsage = critical, digitalSignature, keyEncipherment +nsCertType = client +nsComment = "Admin Client Certificate" +subjectKeyIdentifier = hash diff --git a/nix/kubernetes/keys/package/k8s-ca/package.nix b/nix/kubernetes/keys/package/k8s-ca/package.nix index a2425cb..eed2a77 100644 --- a/nix/kubernetes/keys/package/k8s-ca/package.nix +++ b/nix/kubernetes/keys/package/k8s-ca/package.nix @@ -9,13 +9,12 @@ # distPhase { stdenv, - sqlite, - cfssl, + openssl, ... }: stdenv.mkDerivation (finalAttrs: { name = "k8s-ca"; - nativeBuildInputs = [ cfssl ]; + nativeBuildInputs = [ openssl ]; buildInputs = [ ]; unpackPhase = "true"; @@ -23,6 +22,11 @@ stdenv.mkDerivation (finalAttrs: { installPhase = '' mkdir -p "$out" cd "$out" - cfssl gencert -initca ${./files/ca-csr.json} | cfssljson -bare ca + + openssl genrsa -out ca.key 4096 + openssl req -x509 -new -sha512 -noenc \ + -key ca.key -days 3653 \ + -config ${./files/ca.conf} \ + -out ca.crt ''; }) diff --git a/nix/kubernetes/keys/package/k8s-client-config/package.nix b/nix/kubernetes/keys/package/k8s-client-config/package.nix new file mode 100644 index 0000000..d81f44f --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-client-config/package.nix @@ -0,0 +1,52 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + stdenv, + k8s, + kubectl, + config_name, + config_user, + config_server, + ... +}: +stdenv.mkDerivation (finalAttrs: { + name = "k8s-client-config-${config_name}"; + nativeBuildInputs = [ kubectl ]; + buildInputs = [ ]; + + unpackPhase = "true"; + + buildPhase = '' + kubectl config set-cluster kubernetes-the-hard-way \ + --certificate-authority=${k8s.ca}/ca.crt \ + --embed-certs=true \ + --server=${config_server} \ + --kubeconfig=${config_name}.kubeconfig + + kubectl config set-credentials ${config_user} \ + --client-certificate=${k8s.keys."${config_name}"}/${config_name}.crt \ + --client-key=${k8s.keys."${config_name}"}/${config_name}.key \ + --embed-certs=true \ + --kubeconfig=${config_name}.kubeconfig + + kubectl config set-context default \ + --cluster=kubernetes-the-hard-way \ + --user=${config_user} \ + --kubeconfig=${config_name}.kubeconfig + + kubectl config use-context default \ + --kubeconfig=${config_name}.kubeconfig + ''; + + installPhase = '' + mkdir "$out" + cp "${config_name}.kubeconfig" $out/ + ''; +}) diff --git a/nix/kubernetes/keys/package/k8s-controller-proxy/files/ca-config.json b/nix/kubernetes/keys/package/k8s-controller-proxy/files/ca-config.json deleted file mode 100644 index a63e0dd..0000000 --- a/nix/kubernetes/keys/package/k8s-controller-proxy/files/ca-config.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "signing": { - "default": { - "expiry": "8760h" - }, - "profiles": { - "kubernetes": { - "usages": ["signing", "key encipherment", "server auth", "client auth"], - "expiry": "8760h" - } - } - } -} diff --git a/nix/kubernetes/keys/package/k8s-controller-proxy/files/controller0-proxy-csr.json b/nix/kubernetes/keys/package/k8s-controller-proxy/files/controller0-proxy-csr.json deleted file mode 100644 index 31a18ea..0000000 --- a/nix/kubernetes/keys/package/k8s-controller-proxy/files/controller0-proxy-csr.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "CN": "system:node:controller0", - "key": { - "algo": "rsa", - "size": 2048 - }, - "names": [ - { - "C": "US", - "L": "Portland", - "O": "system:nodes", - "OU": "Kubernetes The Hard Way", - "ST": "Oregon" - } - ] -} diff --git a/nix/kubernetes/keys/package/k8s-controller-proxy/files/controller1-proxy-csr.json b/nix/kubernetes/keys/package/k8s-controller-proxy/files/controller1-proxy-csr.json deleted file mode 100644 index b5defb2..0000000 --- a/nix/kubernetes/keys/package/k8s-controller-proxy/files/controller1-proxy-csr.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "CN": "system:node:controller1", - "key": { - "algo": "rsa", - "size": 2048 - }, - "names": [ - { - "C": "US", - "L": "Portland", - "O": "system:nodes", - "OU": "Kubernetes The Hard Way", - "ST": "Oregon" - } - ] -} diff --git a/nix/kubernetes/keys/package/k8s-controller-proxy/files/controller2-proxy-csr.json b/nix/kubernetes/keys/package/k8s-controller-proxy/files/controller2-proxy-csr.json deleted file mode 100644 index 09c2666..0000000 --- a/nix/kubernetes/keys/package/k8s-controller-proxy/files/controller2-proxy-csr.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "CN": "system:node:controller2", - "key": { - "algo": "rsa", - "size": 2048 - }, - "names": [ - { - "C": "US", - "L": "Portland", - "O": "system:nodes", - "OU": "Kubernetes The Hard Way", - "ST": "Oregon" - } - ] -} diff --git a/nix/kubernetes/keys/package/k8s-controller-proxy/package.nix b/nix/kubernetes/keys/package/k8s-controller-proxy/package.nix deleted file mode 100644 index ba965bb..0000000 --- a/nix/kubernetes/keys/package/k8s-controller-proxy/package.nix +++ /dev/null @@ -1,48 +0,0 @@ -# unpackPhase -# patchPhase -# configurePhase -# buildPhase -# checkPhase -# installPhase -# fixupPhase -# installCheckPhase -# distPhase -{ - lib, - stdenv, - sqlite, - cfssl, - k8s, - all_hostnames, - controllers, - ... -}: -let - get_hostnames = ( - hostname: (builtins.concatStringsSep "," ([ hostname ] ++ controllers."${hostname}".internal_ips)) - ); - install_body = ( - lib.concatMapStringsSep "\n" (hostname: '' - cfssl gencert \ - -ca=${k8s.requestheader-client-ca}/requestheader-client-ca.pem \ - -ca-key=${k8s.requestheader-client-ca}/requestheader-client-ca-key.pem \ - -config=${./files/ca-config.json} \ - -hostname=${get_hostnames hostname} \ - -profile=kubernetes \ - ${./files}/${hostname}-proxy-csr.json | cfssljson -bare ${hostname}-proxy - '') (builtins.attrNames controllers) - ); -in -stdenv.mkDerivation (finalAttrs: { - name = "k8s-controller-proxy"; - nativeBuildInputs = [ cfssl ]; - buildInputs = [ ]; - - unpackPhase = "true"; - - installPhase = '' - mkdir -p "$out" - cd "$out" - '' - + install_body; -}) diff --git a/nix/kubernetes/keys/package/k8s-keys/files/ca-config.json b/nix/kubernetes/keys/package/k8s-keys/files/ca-config.json deleted file mode 100644 index a63e0dd..0000000 --- a/nix/kubernetes/keys/package/k8s-keys/files/ca-config.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "signing": { - "default": { - "expiry": "8760h" - }, - "profiles": { - "kubernetes": { - "usages": ["signing", "key encipherment", "server auth", "client auth"], - "expiry": "8760h" - } - } - } -} diff --git a/nix/kubernetes/keys/package/k8s-keys/files/kubernetes-csr.json b/nix/kubernetes/keys/package/k8s-keys/files/kubernetes-csr.json deleted file mode 100644 index 778db63..0000000 --- a/nix/kubernetes/keys/package/k8s-keys/files/kubernetes-csr.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "CN": "kubernetes", - "key": { - "algo": "rsa", - "size": 2048 - }, - "names": [ - { - "C": "US", - "L": "Portland", - "O": "Kubernetes", - "OU": "Kubernetes The Hard Way", - "ST": "Oregon" - } - ] -} diff --git a/nix/kubernetes/keys/package/k8s-keys/package.nix b/nix/kubernetes/keys/package/k8s-keys/package.nix index 151e9ec..fd85882 100644 --- a/nix/kubernetes/keys/package/k8s-keys/package.nix +++ b/nix/kubernetes/keys/package/k8s-keys/package.nix @@ -6,10 +6,8 @@ symlinkJoin { name = "k8s-keys"; paths = [ - k8s.kubernetes k8s.ca - k8s.service_account - k8s.requestheader-client-ca - k8s.controller-proxy - ]; + ] + ++ (builtins.attrValues k8s.keys) + ++ (builtins.attrValues k8s.client-configs); } diff --git a/nix/kubernetes/keys/package/k8s-kubernetes/files/ca-config.json b/nix/kubernetes/keys/package/k8s-kubernetes/files/ca-config.json deleted file mode 100644 index a63e0dd..0000000 --- a/nix/kubernetes/keys/package/k8s-kubernetes/files/ca-config.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "signing": { - "default": { - "expiry": "8760h" - }, - "profiles": { - "kubernetes": { - "usages": ["signing", "key encipherment", "server auth", "client auth"], - "expiry": "8760h" - } - } - } -} diff --git a/nix/kubernetes/keys/package/k8s-kubernetes/files/kubernetes-csr.json b/nix/kubernetes/keys/package/k8s-kubernetes/files/kubernetes-csr.json deleted file mode 100644 index 778db63..0000000 --- a/nix/kubernetes/keys/package/k8s-kubernetes/files/kubernetes-csr.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "CN": "kubernetes", - "key": { - "algo": "rsa", - "size": 2048 - }, - "names": [ - { - "C": "US", - "L": "Portland", - "O": "Kubernetes", - "OU": "Kubernetes The Hard Way", - "ST": "Oregon" - } - ] -} diff --git a/nix/kubernetes/keys/package/k8s-kubernetes/package.nix b/nix/kubernetes/keys/package/k8s-kubernetes/package.nix deleted file mode 100644 index daadf88..0000000 --- a/nix/kubernetes/keys/package/k8s-kubernetes/package.nix +++ /dev/null @@ -1,36 +0,0 @@ -# unpackPhase -# patchPhase -# configurePhase -# buildPhase -# checkPhase -# installPhase -# fixupPhase -# installCheckPhase -# distPhase -{ - stdenv, - sqlite, - cfssl, - k8s, - all_hostnames, - ... -}: -stdenv.mkDerivation (finalAttrs: { - name = "k8s-kubernetes"; - nativeBuildInputs = [ cfssl ]; - buildInputs = [ ]; - - unpackPhase = "true"; - - installPhase = '' - mkdir -p "$out" - cd "$out" - cfssl gencert \ - -ca=${k8s.ca}/ca.pem \ - -ca-key=${k8s.ca}/ca-key.pem \ - -config=${./files/ca-config.json} \ - -hostname=${builtins.concatStringsSep "," all_hostnames} \ - -profile=kubernetes \ - ${./files/kubernetes-csr.json} | cfssljson -bare kubernetes - ''; -}) diff --git a/nix/kubernetes/keys/package/k8s-requestheader-client-ca/files/requestheader-client-ca-csr.json b/nix/kubernetes/keys/package/k8s-requestheader-client-ca/files/requestheader-client-ca-csr.json deleted file mode 100644 index 8145e50..0000000 --- a/nix/kubernetes/keys/package/k8s-requestheader-client-ca/files/requestheader-client-ca-csr.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "CN": "Kubernetes", - "key": { - "algo": "rsa", - "size": 2048 - }, - "names": [ - { - "C": "US", - "L": "Portland", - "O": "Kubernetes", - "OU": "CA", - "ST": "Oregon" - } - ] -} diff --git a/nix/kubernetes/keys/package/k8s-service-account/files/ca-config.json b/nix/kubernetes/keys/package/k8s-service-account/files/ca-config.json deleted file mode 100644 index a63e0dd..0000000 --- a/nix/kubernetes/keys/package/k8s-service-account/files/ca-config.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "signing": { - "default": { - "expiry": "8760h" - }, - "profiles": { - "kubernetes": { - "usages": ["signing", "key encipherment", "server auth", "client auth"], - "expiry": "8760h" - } - } - } -} diff --git a/nix/kubernetes/keys/package/k8s-service-account/files/service-account-csr.json b/nix/kubernetes/keys/package/k8s-service-account/files/service-account-csr.json deleted file mode 100644 index be3c0ca..0000000 --- a/nix/kubernetes/keys/package/k8s-service-account/files/service-account-csr.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "CN": "service-accounts", - "key": { - "algo": "rsa", - "size": 2048 - }, - "names": [ - { - "C": "US", - "L": "Portland", - "O": "Kubernetes", - "OU": "Kubernetes The Hard Way", - "ST": "Oregon" - } - ] -} diff --git a/nix/kubernetes/keys/package/k8s-service-account/package.nix b/nix/kubernetes/keys/package/k8s-service-account/package.nix deleted file mode 100644 index 717f059..0000000 --- a/nix/kubernetes/keys/package/k8s-service-account/package.nix +++ /dev/null @@ -1,35 +0,0 @@ -# unpackPhase -# patchPhase -# configurePhase -# buildPhase -# checkPhase -# installPhase -# fixupPhase -# installCheckPhase -# distPhase -{ - stdenv, - sqlite, - cfssl, - k8s, - all_hostnames, - ... -}: -stdenv.mkDerivation (finalAttrs: { - name = "k8s-service-account"; - nativeBuildInputs = [ cfssl ]; - buildInputs = [ ]; - - unpackPhase = "true"; - - installPhase = '' - mkdir -p "$out" - cd "$out" - cfssl gencert \ - -ca=${k8s.ca}/ca.pem \ - -ca-key=${k8s.ca}/ca-key.pem \ - -config=${./files/ca-config.json} \ - -profile=kubernetes \ - ${./files/service-account-csr.json} | cfssljson -bare service-account - ''; -}) diff --git a/nix/kubernetes/keys/package/tls-key/package.nix b/nix/kubernetes/keys/package/tls-key/package.nix new file mode 100644 index 0000000..437099e --- /dev/null +++ b/nix/kubernetes/keys/package/tls-key/package.nix @@ -0,0 +1,45 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + stdenv, + openssl, + k8s, + key_name, + ... +}: +stdenv.mkDerivation (finalAttrs: { + name = "tls-key-${key_name}"; + nativeBuildInputs = [ openssl ]; + buildInputs = [ ]; + + unpackPhase = "true"; + + buildPhase = '' + cp ${k8s.ca}/ca.crt ${k8s.ca}/ca.key ./ + + openssl genrsa -out "${key_name}.key" 4096 + + openssl req -new -key "${key_name}.key" -sha256 \ + -config "${../k8s-ca/files/ca.conf}" -section ${key_name} \ + -out "${key_name}.csr" + + openssl x509 -req -days 3653 -in "${key_name}.csr" \ + -copy_extensions copyall \ + -sha256 -CA "./ca.crt" \ + -CAkey "./ca.key" \ + -CAcreateserial \ + -out "${key_name}.crt" + ''; + + installPhase = '' + mkdir "$out" + cp "${key_name}.crt" "${key_name}.key" $out/ + ''; +}) diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index 21acdce..4e32a2d 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -250,13 +250,74 @@ makeScope newScope ( in { ca = (callPackage ./package/k8s-ca/package.nix additional_vars); - kubernetes = (callPackage ./package/k8s-kubernetes/package.nix additional_vars); - service_account = (callPackage ./package/k8s-service-account/package.nix additional_vars); - requestheader-client-ca = ( - callPackage ./package/k8s-requestheader-client-ca/package.nix additional_vars + keys = ( + lib.genAttrs [ + "admin" + "controller0" + "controller1" + "controller2" + "worker0" + "worker1" + "worker2" + "kube-proxy" + "kube-scheduler" + "kube-controller-manager" + "kube-api-server" + "service-accounts" + ] (key_name: (callPackage ./package/tls-key/package.nix (additional_vars // { inherit key_name; }))) ); - controller-proxy = (callPackage ./package/k8s-controller-proxy/package.nix additional_vars); - keys = (callPackage ./package/k8s-keys/package.nix additional_vars); - deploy_script = (writeShellScript "deploy-keys" deploy_script); + client-configs = ( + builtins.mapAttrs + ( + config_name: config: + (callPackage ./package/k8s-client-config/package.nix ( + additional_vars // { inherit config_name; } // config + )) + ) + { + controller0 = { + config_user = "system:node:controller0"; + config_server = "https://server.kubernetes.local:6443"; + }; + controller1 = { + config_user = "system:node:controller1"; + config_server = "https://server.kubernetes.local:6443"; + }; + controller2 = { + config_user = "system:node:controller2"; + config_server = "https://server.kubernetes.local:6443"; + }; + worker0 = { + config_user = "system:node:worker0"; + config_server = "https://server.kubernetes.local:6443"; + }; + worker1 = { + config_user = "system:node:worker1"; + config_server = "https://server.kubernetes.local:6443"; + }; + worker2 = { + config_user = "system:node:worker2"; + config_server = "https://server.kubernetes.local:6443"; + }; + kube-proxy = { + config_user = "system:kube-proxy"; + config_server = "https://server.kubernetes.local:6443"; + }; + kube-controller-manager = { + config_user = "system:kube-controller-manager"; + config_server = "https://server.kubernetes.local:6443"; + }; + kube-scheduler = { + config_user = "system:kube-scheduler"; + config_server = "https://server.kubernetes.local:6443"; + }; + admin = { + config_user = "admin"; + config_server = "https://127.0.0.1:6443"; + }; + } + ); + all_keys = (callPackage ./package/k8s-keys/package.nix additional_vars); + deploy_script = (callPackage ./package/deploy-script/package.nix additional_vars); } ) From 380ac011f14dafb9d9d76aac05c0e7971264c89b Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 14 Dec 2025 20:28:48 -0500 Subject: [PATCH 08/81] Move the encryption config into a package. --- .../keys/package/deploy-script/package.nix | 138 +++++++++++++- .../keys/package/k8s-ca/files/ca.conf | 14 +- .../package/k8s-encryption-key/package.nix | 58 ++++++ .../keys/package/k8s-keys/package.nix | 1 + nix/kubernetes/keys/scope.nix | 177 +----------------- nix/kubernetes/roles/etcd/default.nix | 12 +- 6 files changed, 222 insertions(+), 178 deletions(-) create mode 100644 nix/kubernetes/keys/package/k8s-encryption-key/package.nix diff --git a/nix/kubernetes/keys/package/deploy-script/package.nix b/nix/kubernetes/keys/package/deploy-script/package.nix index dfe85ed..7e21d75 100644 --- a/nix/kubernetes/keys/package/deploy-script/package.nix +++ b/nix/kubernetes/keys/package/deploy-script/package.nix @@ -8,14 +8,150 @@ # installCheckPhase # distPhase { + lib, stdenv, writeShellScript, k8s, + openssh, ... }: let - deploy_script_body = ""; + deploy_script_body = ( + '' + set -euo pipefail + IFS=$'\n\t' + DIR="$( cd "$( dirname "''${BASH_SOURCE[0]}" )" && pwd )" + '' + + (lib.concatMapStringsSep "\n" deploy_machine [ + "nc0" + "nc1" + "nc2" + ]) + ); deploy_script = (writeShellScript "deploy-script" deploy_script_body); + deploy_file = ( + { + dest_dir, + file, + name ? (builtins.baseNameOf file), + owner, + group, + mode, + }: + '' + ## + ## deploy ${name} to ${dest_dir} + ## + ${openssh}/bin/ssh mrmanager doas rm -f ${dest_dir}/${name} ~/${name} + ${openssh}/bin/scp ${file} mrmanager:~/${name} + ${openssh}/bin/ssh mrmanager doas install -o ${toString owner} -g ${toString group} -m ${mode} ~/${name} ${dest_dir}/${name} + ${openssh}/bin/ssh mrmanager doas rm -f ~/${name} + + + '' + ); + deploy_machine = ( + vm_name: + ( + '' + ## + ## Create directories on ${vm_name} + ## + ${openssh}/bin/ssh mrmanager doas install -d -o 11235 -g 11235 -m 0755 /vm/${vm_name}/persist/keys + ${openssh}/bin/ssh mrmanager doas install -d -o 10016 -g 10016 -m 0755 /vm/${vm_name}/persist/keys/etcd + ${openssh}/bin/ssh mrmanager doas install -d -o 10024 -g 10024 -m 0755 /vm/${vm_name}/persist/keys/kube + + + '' + + (lib.concatMapStringsSep "\n" deploy_file [ + { + dest_dir = "/vm/${vm_name}/persist/keys/etcd"; + file = "${k8s.keys.kube-api-server}/kube-api-server.crt"; + owner = 10016; + group = 10016; + mode = "0640"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/etcd"; + file = "${k8s.keys.kube-api-server}/kube-api-server.key"; + owner = 10016; + group = 10016; + mode = "0600"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/etcd"; + file = "${k8s.ca}/ca.crt"; + owner = 10016; + group = 10016; + mode = "0640"; + } + # { + # dest_dir = "/vm/${vm_name}/persist/keys/kube"; + # file = "${self.kubernetes}/kubernetes.pem"; + # owner = 10024; + # group = 10024; + # mode = "0640"; + # } + # { + # dest_dir = "/vm/${vm_name}/persist/keys/kube"; + # file = "${self.kubernetes}/kubernetes-key.pem"; + # owner = 10024; + # group = 10024; + # mode = "0640"; + # } + # { + # dest_dir = "/vm/${vm_name}/persist/keys/kube"; + # file = "${self.ca}/ca.pem"; + # owner = 10024; + # group = 10024; + # mode = "0600"; + # } + # { + # dest_dir = "/vm/${vm_name}/persist/keys/kube"; + # file = (writeText "encryption-config.yaml" (lib.generators.toYAML { } kube_encryption_config)); + # name = "encryption-config.yaml"; + # owner = 10024; + # group = 10024; + # mode = "0600"; + # } + # { + # dest_dir = "/vm/${vm_name}/persist/keys/kube"; + # file = "${self.service_account}/service-account.pem"; + # owner = 10024; + # group = 10024; + # mode = "0600"; + # } + # { + # dest_dir = "/vm/${vm_name}/persist/keys/kube"; + # file = "${self.service_account}/service-account-key.pem"; + # owner = 10024; + # group = 10024; + # mode = "0600"; + # } + # { + # dest_dir = "/vm/${vm_name}/persist/keys/kube"; + # file = "${self.requestheader-client-ca}/requestheader-client-ca.pem"; + # owner = 10024; + # group = 10024; + # mode = "0600"; + # } + # { + # dest_dir = "/vm/${vm_name}/persist/keys/kube"; + # file = "${self.controller-proxy}/${vm_name_to_hostname vm_name}-proxy.pem"; + # owner = 10024; + # group = 10024; + # mode = "0600"; + # } + # { + # dest_dir = "/vm/${vm_name}/persist/keys/kube"; + # file = "${self.controller-proxy}/${vm_name_to_hostname vm_name}-proxy-key.pem"; + # owner = 10024; + # group = 10024; + # mode = "0600"; + # } + ]) + ) + ); in stdenv.mkDerivation (finalAttrs: { name = "deploy-script"; diff --git a/nix/kubernetes/keys/package/k8s-ca/files/ca.conf b/nix/kubernetes/keys/package/k8s-ca/files/ca.conf index 3c7021c..b26ddbf 100644 --- a/nix/kubernetes/keys/package/k8s-ca/files/ca.conf +++ b/nix/kubernetes/keys/package/k8s-ca/files/ca.conf @@ -266,7 +266,19 @@ subjectKeyIdentifier = hash [kube-api-server_alt_names] IP.0 = 127.0.0.1 -IP.1 = 10.32.0.1 +IP.1 = 10.0.0.1 +IP.2 = 10.215.1.221 +IP.3 = 2620:11f:7001:7:ffff:ffff:0ad7:01dd +IP.4 = 10.215.1.222 +IP.5 = 2620:11f:7001:7:ffff:ffff:0ad7:01de +IP.6 = 10.215.1.223 +IP.7 = 2620:11f:7001:7:ffff:ffff:0ad7:01df +IP.8 = 10.215.1.224 +IP.9 = 2620:11f:7001:7:ffff:ffff:0ad7:01e0 +IP.10 = 10.215.1.225 +IP.11 = 2620:11f:7001:7:ffff:ffff:0ad7:01e1 +IP.12 = 10.215.1.226 +IP.13 = 2620:11f:7001:7:ffff:ffff:0ad7:01e2 DNS.0 = kubernetes DNS.1 = kubernetes.default DNS.2 = kubernetes.default.svc diff --git a/nix/kubernetes/keys/package/k8s-encryption-key/package.nix b/nix/kubernetes/keys/package/k8s-encryption-key/package.nix new file mode 100644 index 0000000..1f88575 --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-encryption-key/package.nix @@ -0,0 +1,58 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + lib, + stdenv, + runCommand, + writeText, + ... +}: +let + kube_encryption_key = runCommand "kube_encryption_key" { } '' + head -c 32 /dev/urandom | base64 | tee $out + ''; + kube_encryption_config = { + kind = "EncryptionConfig"; + apiVersion = "v1"; + resources = [ + { + resources = [ "secrets" ]; + providers = [ + { + aescbc = { + keys = [ + { + name = "key1"; + secret = (builtins.readFile "${kube_encryption_key}"); + } + ]; + }; + } + { identity = { }; } + ]; + } + ]; + }; + kube_encryption_config_yaml = ( + writeText "encryption-config.yaml" (lib.generators.toYAML { } kube_encryption_config) + ); +in +stdenv.mkDerivation (finalAttrs: { + name = "k8s-encryption-key"; + nativeBuildInputs = [ ]; + buildInputs = [ ]; + + unpackPhase = "true"; + + installPhase = '' + mkdir "$out" + cp "${kube_encryption_config_yaml}" $out/encryption-config.yaml + ''; +}) diff --git a/nix/kubernetes/keys/package/k8s-keys/package.nix b/nix/kubernetes/keys/package/k8s-keys/package.nix index fd85882..05bc44e 100644 --- a/nix/kubernetes/keys/package/k8s-keys/package.nix +++ b/nix/kubernetes/keys/package/k8s-keys/package.nix @@ -7,6 +7,7 @@ symlinkJoin { name = "k8s-keys"; paths = [ k8s.ca + k8s.encryption_config ] ++ (builtins.attrValues k8s.keys) ++ (builtins.attrValues k8s.client-configs); diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index 4e32a2d..785653f 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -2,10 +2,6 @@ makeScope, newScope, callPackage, - writeShellScript, - openssh, - runCommand, - writeText, lib, }: let @@ -73,12 +69,12 @@ let ]; }; }; - _vm_name_to_hostname = { - "nc0" = "controller0"; - "nc1" = "controller1"; - "nc2" = "controller2"; - }; - vm_name_to_hostname = (vm_name: _vm_name_to_hostname."${vm_name}"); + # _vm_name_to_hostname = { + # "nc0" = "controller0"; + # "nc1" = "controller1"; + # "nc2" = "controller2"; + # }; + # vm_name_to_hostname = (vm_name: _vm_name_to_hostname."${vm_name}"); in makeScope newScope ( self: @@ -87,166 +83,6 @@ makeScope newScope ( inherit all_hostnames controllers; k8s = self; }; - deploy_file = ( - { - dest_dir, - file, - name ? (builtins.baseNameOf file), - owner, - group, - mode, - }: - '' - ## - ## deploy ${name} to ${dest_dir} - ## - ${openssh}/bin/ssh mrmanager doas rm -f ${dest_dir}/${name} ~/${name} - ${openssh}/bin/scp ${file} mrmanager:~/${name} - ${openssh}/bin/ssh mrmanager doas install -o ${toString owner} -g ${toString group} -m ${mode} ~/${name} ${dest_dir}/${name} - ${openssh}/bin/ssh mrmanager doas rm -f ~/${name} - - - '' - ); - deploy_machine = ( - vm_name: - ( - '' - ## - ## Create directories on ${vm_name} - ## - ${openssh}/bin/ssh mrmanager doas install -d -o 11235 -g 11235 -m 0755 /vm/${vm_name}/persist/keys - ${openssh}/bin/ssh mrmanager doas install -d -o 10016 -g 10016 -m 0755 /vm/${vm_name}/persist/keys/etcd - ${openssh}/bin/ssh mrmanager doas install -d -o 10024 -g 10024 -m 0755 /vm/${vm_name}/persist/keys/kube - - - '' - + (lib.concatMapStringsSep "\n" deploy_file [ - { - dest_dir = "/vm/${vm_name}/persist/keys/etcd"; - file = "${self.kubernetes}/kubernetes.pem"; - owner = 10016; - group = 10016; - mode = "0640"; - } - { - dest_dir = "/vm/${vm_name}/persist/keys/etcd"; - file = "${self.kubernetes}/kubernetes-key.pem"; - owner = 10016; - group = 10016; - mode = "0600"; - } - { - dest_dir = "/vm/${vm_name}/persist/keys/etcd"; - file = "${self.ca}/ca.pem"; - owner = 10016; - group = 10016; - mode = "0640"; - } - { - dest_dir = "/vm/${vm_name}/persist/keys/kube"; - file = "${self.kubernetes}/kubernetes.pem"; - owner = 10024; - group = 10024; - mode = "0640"; - } - { - dest_dir = "/vm/${vm_name}/persist/keys/kube"; - file = "${self.kubernetes}/kubernetes-key.pem"; - owner = 10024; - group = 10024; - mode = "0640"; - } - { - dest_dir = "/vm/${vm_name}/persist/keys/kube"; - file = "${self.ca}/ca.pem"; - owner = 10024; - group = 10024; - mode = "0600"; - } - { - dest_dir = "/vm/${vm_name}/persist/keys/kube"; - file = (writeText "encryption-config.yaml" (lib.generators.toYAML { } kube_encryption_config)); - name = "encryption-config.yaml"; - owner = 10024; - group = 10024; - mode = "0600"; - } - { - dest_dir = "/vm/${vm_name}/persist/keys/kube"; - file = "${self.service_account}/service-account.pem"; - owner = 10024; - group = 10024; - mode = "0600"; - } - { - dest_dir = "/vm/${vm_name}/persist/keys/kube"; - file = "${self.service_account}/service-account-key.pem"; - owner = 10024; - group = 10024; - mode = "0600"; - } - { - dest_dir = "/vm/${vm_name}/persist/keys/kube"; - file = "${self.requestheader-client-ca}/requestheader-client-ca.pem"; - owner = 10024; - group = 10024; - mode = "0600"; - } - { - dest_dir = "/vm/${vm_name}/persist/keys/kube"; - file = "${self.controller-proxy}/${vm_name_to_hostname vm_name}-proxy.pem"; - owner = 10024; - group = 10024; - mode = "0600"; - } - { - dest_dir = "/vm/${vm_name}/persist/keys/kube"; - file = "${self.controller-proxy}/${vm_name_to_hostname vm_name}-proxy-key.pem"; - owner = 10024; - group = 10024; - mode = "0600"; - } - ]) - ) - ); - deploy_script = ( - '' - set -euo pipefail - IFS=$'\n\t' - DIR="$( cd "$( dirname "''${BASH_SOURCE[0]}" )" && pwd )" - '' - + (lib.concatMapStringsSep "\n" deploy_machine [ - "nc0" - "nc1" - "nc2" - ]) - ); - kube_encryption_key = runCommand "kube_encryption_key" { } '' - head -c 32 /dev/urandom | base64 | tee $out - ''; - kube_encryption_config = { - kind = "EncryptionConfig"; - apiVersion = "v1"; - resources = [ - { - resources = [ "secrets" ]; - providers = [ - { - aescbc = { - keys = [ - { - name = "key1"; - secret = (builtins.readFile "${kube_encryption_key}"); - } - ]; - }; - } - { identity = { }; } - ]; - } - ]; - }; in { ca = (callPackage ./package/k8s-ca/package.nix additional_vars); @@ -317,6 +153,7 @@ makeScope newScope ( }; } ); + encryption_config = (callPackage ./package/k8s-encryption-key/package.nix additional_vars); all_keys = (callPackage ./package/k8s-keys/package.nix additional_vars); deploy_script = (callPackage ./package/deploy-script/package.nix additional_vars); } diff --git a/nix/kubernetes/roles/etcd/default.nix b/nix/kubernetes/roles/etcd/default.nix index f0044d0..c3de920 100644 --- a/nix/kubernetes/roles/etcd/default.nix +++ b/nix/kubernetes/roles/etcd/default.nix @@ -55,12 +55,12 @@ enable = true; openFirewall = true; name = config.networking.hostName; - certFile = "/.persist/keys/etcd/kubernetes.pem"; - keyFile = "/.persist/keys/etcd/kubernetes-key.pem"; - peerCertFile = "/.persist/keys/etcd/kubernetes.pem"; - peerKeyFile = "/.persist/keys/etcd/kubernetes-key.pem"; - trustedCaFile = "/.persist/keys/etcd/ca.pem"; - peerTrustedCaFile = "/.persist/keys/etcd/ca.pem"; + certFile = "/.persist/keys/etcd/kube-api-server.crt"; + keyFile = "/.persist/keys/etcd/kube-api-server.key"; + peerCertFile = "/.persist/keys/etcd/kube-api-server.crt"; + peerKeyFile = "/.persist/keys/etcd/kube-api-server.key"; + trustedCaFile = "/.persist/keys/etcd/ca.crt"; + peerTrustedCaFile = "/.persist/keys/etcd/ca.crt"; peerClientCertAuth = true; clientCertAuth = true; initialAdvertisePeerUrls = ( From fd15dc5befcb4ea98a147710f81aea961e0dfb1d Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 14 Dec 2025 23:24:23 -0500 Subject: [PATCH 09/81] Fix launching kube-apiserver. --- .../keys/package/deploy-script/package.nix | 43 ++++++++++ .../roles/kube_apiserver/default.nix | 82 +++++++++++++------ 2 files changed, 100 insertions(+), 25 deletions(-) diff --git a/nix/kubernetes/keys/package/deploy-script/package.nix b/nix/kubernetes/keys/package/deploy-script/package.nix index 7e21d75..6d32543 100644 --- a/nix/kubernetes/keys/package/deploy-script/package.nix +++ b/nix/kubernetes/keys/package/deploy-script/package.nix @@ -85,6 +85,49 @@ let group = 10016; mode = "0640"; } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.ca}/ca.crt"; + owner = 10024; + group = 10024; + mode = "0640"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.keys.kube-api-server}/kube-api-server.crt"; + owner = 10024; + group = 10024; + mode = "0640"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.keys.kube-api-server}/kube-api-server.key"; + owner = 10024; + group = 10024; + mode = "0600"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.encryption_config}/encryption-config.yaml"; + name = "encryption-config.yaml"; + owner = 10024; + group = 10024; + mode = "0600"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.keys.service-accounts}/service-accounts.crt"; + owner = 10024; + group = 10024; + mode = "0640"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.keys.service-accounts}/service-accounts.key"; + owner = 10024; + group = 10024; + mode = "0600"; + } # { # dest_dir = "/vm/${vm_name}/persist/keys/kube"; # file = "${self.kubernetes}/kubernetes.pem"; diff --git a/nix/kubernetes/roles/kube_apiserver/default.nix b/nix/kubernetes/roles/kube_apiserver/default.nix index 4a0881f..2a76188 100644 --- a/nix/kubernetes/roles/kube_apiserver/default.nix +++ b/nix/kubernetes/roles/kube_apiserver/default.nix @@ -6,7 +6,8 @@ }: let - shellCommand = cmd: (lib.concatMapStringsSep " " lib.strings.escapeShellArg cmd); + # shellCommand = cmd: (lib.concatMapStringsSep " " lib.strings.escapeShellArg cmd); + shellCommand = cmd: (builtins.concatStringsSep " " cmd); in { imports = [ ]; @@ -54,47 +55,78 @@ in Type = "notify"; ExecStart = ( shellCommand [ + # NEW: "${pkgs.kubernetes}/bin/kube-apiserver" - "--advertise-address=${config.me.kube_apiserver.internal_ip}" "--allow-privileged=true" - "--apiserver-count=3" "--audit-log-maxage=30" "--audit-log-maxbackup=3" "--audit-log-maxsize=100" "--audit-log-path=/var/log/audit.log" "--authorization-mode=Node,RBAC" "--bind-address=0.0.0.0" - "--client-ca-file=/.persist/keys/kube/ca.pem" - "--requestheader-client-ca-file=/.persist/keys/kube/requestheader-client-ca.pem" - ''--requestheader-allowed-names=""'' - "--requestheader-extra-headers-prefix=X-Remote-Extra-" - "--requestheader-group-headers=X-Remote-Group" - "--requestheader-username-headers=X-Remote-User" - "--proxy-client-cert-file=/.persist/keys/kube/${config.networking.hostName}-proxy.pem" - "--proxy-client-key-file=/.persist/keys/kube/${config.networking.hostName}-proxy-key.pem" + "--client-ca-file=/.persist/keys/kube/ca.crt" "--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota" "--etcd-cafile=/.persist/keys/kube/ca.pem" - "--etcd-certfile=/.persist/keys/kube/kubernetes.pem" - "--etcd-keyfile=/.persist/keys/kube/kubernetes-key.pem" + "--etcd-certfile=/.persist/keys/kube/kube-api-server.crt" + "--etcd-keyfile=/.persist/keys/kube/kube-api-server.key" "--etcd-servers=${builtins.concatStringsSep "," config.me.kube_apiserver.etcd_services}" "--event-ttl=1h" "--encryption-provider-config=/.persist/keys/kube/encryption-config.yaml" - "--kubelet-certificate-authority=/.persist/keys/kube/ca.pem" - "--kubelet-client-certificate=/.persist/keys/kube/kubernetes.pem" - "--kubelet-client-key=/.persist/keys/kube/kubernetes-key.pem" + "--kubelet-certificate-authority=/.persist/keys/kube/ca.crt" + "--kubelet-client-certificate=/.persist/keys/kube/kube-api-server.crt" + "--kubelet-client-key=/.persist/keys/kube/kube-api-server.key" "--runtime-config='api/all=true'" - "--service-account-key-file=/.persist/keys/kube/service-account.pem" - "--service-account-signing-key-file=/.persist/keys/kube/service-account-key.pem" - "--service-account-issuer=https://${config.me.kube_apiserver.external_ip}:6443" + "--service-account-key-file=/.persist/keys/kube/service-accounts.crt" + "--service-account-signing-key-file=/.persist/keys/kube/service-accounts.key" + "--service-account-issuer=https://server.kubernetes.local:6443" "--service-node-port-range=30000-32767" - "--tls-cert-file=/.persist/keys/kube/kubernetes.pem" - "--tls-private-key-file=/.persist/keys/kube/kubernetes-key.pem" + "--tls-cert-file=/.persist/keys/kube/kube-api-server.crt" + "--tls-private-key-file=/.persist/keys/kube/kube-api-server.key" "--tls-min-version=VersionTLS13" - "--kubelet-preferred-address-types=InternalIP,ExternalDNS,ExternalIP,Hostname,InternalDNS" - # "--service-cluster-ip-range=10.197.0.0/16" - "--service-cluster-ip-range=2620:11f:7001:7:ffff:ffff:0ac5:0000/16" - "--enable-aggregator-routing=true" "--v=2" + + # OLD: + # "${pkgs.kubernetes}/bin/kube-apiserver" + # "--advertise-address=${config.me.kube_apiserver.internal_ip}" + # "--allow-privileged=true" + # "--apiserver-count=3" + # "--audit-log-maxage=30" + # "--audit-log-maxbackup=3" + # "--audit-log-maxsize=100" + # "--audit-log-path=/var/log/audit.log" + # "--authorization-mode=Node,RBAC" + # "--bind-address=0.0.0.0" + # "--client-ca-file=/.persist/keys/kube/ca.pem" + # "--requestheader-client-ca-file=/.persist/keys/kube/requestheader-client-ca.pem" + # ''--requestheader-allowed-names=""'' + # "--requestheader-extra-headers-prefix=X-Remote-Extra-" + # "--requestheader-group-headers=X-Remote-Group" + # "--requestheader-username-headers=X-Remote-User" + # "--proxy-client-cert-file=/.persist/keys/kube/${config.networking.hostName}-proxy.pem" + # "--proxy-client-key-file=/.persist/keys/kube/${config.networking.hostName}-proxy-key.pem" + # "--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota" + # "--etcd-cafile=/.persist/keys/kube/ca.pem" + # "--etcd-certfile=/.persist/keys/kube/kubernetes.pem" + # "--etcd-keyfile=/.persist/keys/kube/kubernetes-key.pem" + # "--etcd-servers=${builtins.concatStringsSep "," config.me.kube_apiserver.etcd_services}" + # "--event-ttl=1h" + # "--encryption-provider-config=/.persist/keys/kube/encryption-config.yaml" + # "--kubelet-certificate-authority=/.persist/keys/kube/ca.pem" + # "--kubelet-client-certificate=/.persist/keys/kube/kubernetes.pem" + # "--kubelet-client-key=/.persist/keys/kube/kubernetes-key.pem" + # "--runtime-config='api/all=true'" + # "--service-account-key-file=/.persist/keys/kube/service-account.pem" + # "--service-account-signing-key-file=/.persist/keys/kube/service-account-key.pem" + # "--service-account-issuer=https://${config.me.kube_apiserver.external_ip}:6443" + # "--service-node-port-range=30000-32767" + # "--tls-cert-file=/.persist/keys/kube/kubernetes.pem" + # "--tls-private-key-file=/.persist/keys/kube/kubernetes-key.pem" + # "--tls-min-version=VersionTLS13" + # "--kubelet-preferred-address-types=InternalIP,ExternalDNS,ExternalIP,Hostname,InternalDNS" + # # "--service-cluster-ip-range=10.197.0.0/16" + # "--service-cluster-ip-range=2620:11f:7001:7:ffff:ffff:0ac5:0000/16" + # "--enable-aggregator-routing=true" + # "--v=2" ] ); Restart = "on-failure"; From d0439a6d2f3d194e77b5223b6655e7ac97c6b80b Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 15 Dec 2025 19:47:35 -0500 Subject: [PATCH 10/81] Add kube-controller-manager. --- nix/kubernetes/configuration.nix | 1 + .../keys/package/deploy-script/package.nix | 14 +++++ .../roles/control_plane/default.nix | 1 + .../roles/kube_apiserver/default.nix | 2 +- .../roles/kube_controller_manager/default.nix | 60 +++++++++++++++++++ 5 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 nix/kubernetes/roles/kube_controller_manager/default.nix diff --git a/nix/kubernetes/configuration.nix b/nix/kubernetes/configuration.nix index 16fcf75..598ded2 100644 --- a/nix/kubernetes/configuration.nix +++ b/nix/kubernetes/configuration.nix @@ -14,6 +14,7 @@ ./roles/image_based_appliance ./roles/iso ./roles/kube_apiserver + ./roles/kube_controller_manager ./roles/kubernetes ./roles/minimal_base ./roles/network diff --git a/nix/kubernetes/keys/package/deploy-script/package.nix b/nix/kubernetes/keys/package/deploy-script/package.nix index 6d32543..d639c01 100644 --- a/nix/kubernetes/keys/package/deploy-script/package.nix +++ b/nix/kubernetes/keys/package/deploy-script/package.nix @@ -92,6 +92,13 @@ let group = 10024; mode = "0640"; } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.ca}/ca.key"; + owner = 10024; + group = 10024; + mode = "0600"; + } { dest_dir = "/vm/${vm_name}/persist/keys/kube"; file = "${k8s.keys.kube-api-server}/kube-api-server.crt"; @@ -128,6 +135,13 @@ let group = 10024; mode = "0600"; } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.client-configs.kube-controller-manager}/kube-controller-manager.kubeconfig"; + owner = 10024; + group = 10024; + mode = "0600"; + } # { # dest_dir = "/vm/${vm_name}/persist/keys/kube"; # file = "${self.kubernetes}/kubernetes.pem"; diff --git a/nix/kubernetes/roles/control_plane/default.nix b/nix/kubernetes/roles/control_plane/default.nix index 1c0b58b..b460144 100644 --- a/nix/kubernetes/roles/control_plane/default.nix +++ b/nix/kubernetes/roles/control_plane/default.nix @@ -20,5 +20,6 @@ config = lib.mkIf config.me.control_plane.enable { me.kubernetes.enable = true; me.kube_apiserver.enable = true; + me.kube_controller_manager.enable = true; }; } diff --git a/nix/kubernetes/roles/kube_apiserver/default.nix b/nix/kubernetes/roles/kube_apiserver/default.nix index 2a76188..a78f4dc 100644 --- a/nix/kubernetes/roles/kube_apiserver/default.nix +++ b/nix/kubernetes/roles/kube_apiserver/default.nix @@ -66,7 +66,7 @@ in "--bind-address=0.0.0.0" "--client-ca-file=/.persist/keys/kube/ca.crt" "--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota" - "--etcd-cafile=/.persist/keys/kube/ca.pem" + "--etcd-cafile=/.persist/keys/kube/ca.crt" "--etcd-certfile=/.persist/keys/kube/kube-api-server.crt" "--etcd-keyfile=/.persist/keys/kube/kube-api-server.key" "--etcd-servers=${builtins.concatStringsSep "," config.me.kube_apiserver.etcd_services}" diff --git a/nix/kubernetes/roles/kube_controller_manager/default.nix b/nix/kubernetes/roles/kube_controller_manager/default.nix new file mode 100644 index 0000000..9b33b49 --- /dev/null +++ b/nix/kubernetes/roles/kube_controller_manager/default.nix @@ -0,0 +1,60 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + # shellCommand = cmd: (lib.concatMapStringsSep " " lib.strings.escapeShellArg cmd); + shellCommand = cmd: (builtins.concatStringsSep " " cmd); +in +{ + imports = [ ]; + + options.me = { + kube_controller_manager.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install kube_controller_manager."; + }; + }; + + config = lib.mkIf config.me.kube_controller_manager.enable { + systemd.services.kube-controller-manager = { + enable = true; + description = "Kubernetes Controller Manager"; + documentation = [ "https://github.com/kubernetes/kubernetes" ]; + wantedBy = [ "kubernetes.target" ]; + # path = with pkgs; [ + # zfs + # ]; + unitConfig.DefaultDependencies = "no"; + serviceConfig = { + Type = "notify"; + ExecStart = ( + shellCommand [ + # NEW: + "${pkgs.kubernetes}/bin/kube-controller-manager" + "--bind-address=0.0.0.0" + # "--cluster-cidr=10.200.0.0/16" + "--cluster-cidr=2620:11f:7001:7:ffff:ffff:0ac8:0000/16" + "--cluster-name=kubernetes" + "--cluster-signing-cert-file=/.persist/keys/kube/ca.crt" + "--cluster-signing-key-file=/.persist/keys/kube/ca.key" + "--kubeconfig=/.persist/keys/kube/kube-controller-manager.kubeconfig" + "--root-ca-file=/.persist/keys/kube/ca.crt" + "--service-account-private-key-file=/.persist/keys/kube/service-accounts.key" + "--service-cluster-ip-range=2620:11f:7001:7:ffff:ffff:0ac5:0000/16" + # "--service-cluster-ip-range=10.197.0.0/16" + "--use-service-account-credentials=true" + "--v=2" + ] + ); + Restart = "on-failure"; + RestartSec = 5; + }; + }; + }; +} From 868b9b98897b98aa169f7fb6131a61d8d939e80c Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 15 Dec 2025 20:09:46 -0500 Subject: [PATCH 11/81] Add kube-scheduler. --- nix/kubernetes/configuration.nix | 1 + .../keys/package/deploy-script/package.nix | 71 ++----------------- .../package/k8s-client-config/package.nix | 3 +- nix/kubernetes/keys/scope.nix | 11 ++- .../roles/control_plane/default.nix | 3 +- nix/kubernetes/roles/etcd/default.nix | 2 +- .../roles/kube_apiserver/default.nix | 10 +++ .../roles/kube_controller_manager/default.nix | 10 ++- .../roles/kube_scheduler/default.nix | 51 +++++++++++++ .../kube_scheduler/files/kube-scheduler.yaml | 6 ++ 10 files changed, 95 insertions(+), 73 deletions(-) create mode 100644 nix/kubernetes/roles/kube_scheduler/default.nix create mode 100644 nix/kubernetes/roles/kube_scheduler/files/kube-scheduler.yaml diff --git a/nix/kubernetes/configuration.nix b/nix/kubernetes/configuration.nix index 598ded2..11b9538 100644 --- a/nix/kubernetes/configuration.nix +++ b/nix/kubernetes/configuration.nix @@ -15,6 +15,7 @@ ./roles/iso ./roles/kube_apiserver ./roles/kube_controller_manager + ./roles/kube_scheduler ./roles/kubernetes ./roles/minimal_base ./roles/network diff --git a/nix/kubernetes/keys/package/deploy-script/package.nix b/nix/kubernetes/keys/package/deploy-script/package.nix index d639c01..d3e8da3 100644 --- a/nix/kubernetes/keys/package/deploy-script/package.nix +++ b/nix/kubernetes/keys/package/deploy-script/package.nix @@ -142,70 +142,13 @@ let group = 10024; mode = "0600"; } - # { - # dest_dir = "/vm/${vm_name}/persist/keys/kube"; - # file = "${self.kubernetes}/kubernetes.pem"; - # owner = 10024; - # group = 10024; - # mode = "0640"; - # } - # { - # dest_dir = "/vm/${vm_name}/persist/keys/kube"; - # file = "${self.kubernetes}/kubernetes-key.pem"; - # owner = 10024; - # group = 10024; - # mode = "0640"; - # } - # { - # dest_dir = "/vm/${vm_name}/persist/keys/kube"; - # file = "${self.ca}/ca.pem"; - # owner = 10024; - # group = 10024; - # mode = "0600"; - # } - # { - # dest_dir = "/vm/${vm_name}/persist/keys/kube"; - # file = (writeText "encryption-config.yaml" (lib.generators.toYAML { } kube_encryption_config)); - # name = "encryption-config.yaml"; - # owner = 10024; - # group = 10024; - # mode = "0600"; - # } - # { - # dest_dir = "/vm/${vm_name}/persist/keys/kube"; - # file = "${self.service_account}/service-account.pem"; - # owner = 10024; - # group = 10024; - # mode = "0600"; - # } - # { - # dest_dir = "/vm/${vm_name}/persist/keys/kube"; - # file = "${self.service_account}/service-account-key.pem"; - # owner = 10024; - # group = 10024; - # mode = "0600"; - # } - # { - # dest_dir = "/vm/${vm_name}/persist/keys/kube"; - # file = "${self.requestheader-client-ca}/requestheader-client-ca.pem"; - # owner = 10024; - # group = 10024; - # mode = "0600"; - # } - # { - # dest_dir = "/vm/${vm_name}/persist/keys/kube"; - # file = "${self.controller-proxy}/${vm_name_to_hostname vm_name}-proxy.pem"; - # owner = 10024; - # group = 10024; - # mode = "0600"; - # } - # { - # dest_dir = "/vm/${vm_name}/persist/keys/kube"; - # file = "${self.controller-proxy}/${vm_name_to_hostname vm_name}-proxy-key.pem"; - # owner = 10024; - # group = 10024; - # mode = "0600"; - # } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.client-configs.kube-scheduler}/kube-scheduler.kubeconfig"; + owner = 10024; + group = 10024; + mode = "0600"; + } ]) ) ); diff --git a/nix/kubernetes/keys/package/k8s-client-config/package.nix b/nix/kubernetes/keys/package/k8s-client-config/package.nix index d81f44f..2d4ac88 100644 --- a/nix/kubernetes/keys/package/k8s-client-config/package.nix +++ b/nix/kubernetes/keys/package/k8s-client-config/package.nix @@ -8,6 +8,7 @@ # installCheckPhase # distPhase { + lib, stdenv, k8s, kubectl, @@ -27,7 +28,7 @@ stdenv.mkDerivation (finalAttrs: { kubectl config set-cluster kubernetes-the-hard-way \ --certificate-authority=${k8s.ca}/ca.crt \ --embed-certs=true \ - --server=${config_server} \ + --server=${lib.strings.escapeShellArg config_server} \ --kubeconfig=${config_name}.kubeconfig kubectl config set-credentials ${config_user} \ diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index 785653f..1aef3f1 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -141,15 +141,20 @@ makeScope newScope ( }; kube-controller-manager = { config_user = "system:kube-controller-manager"; - config_server = "https://server.kubernetes.local:6443"; + # config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443"; + config_server = "https://127.0.0.1:6443"; + # config_server = "https://server.kubernetes.local:6443"; }; kube-scheduler = { config_user = "system:kube-scheduler"; - config_server = "https://server.kubernetes.local:6443"; + # config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443"; + config_server = "https://127.0.0.1:6443"; + # config_server = "https://server.kubernetes.local:6443"; }; admin = { config_user = "admin"; - config_server = "https://127.0.0.1:6443"; + config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443"; + # config_server = "https://127.0.0.1:6443"; }; } ); diff --git a/nix/kubernetes/roles/control_plane/default.nix b/nix/kubernetes/roles/control_plane/default.nix index b460144..7039982 100644 --- a/nix/kubernetes/roles/control_plane/default.nix +++ b/nix/kubernetes/roles/control_plane/default.nix @@ -18,8 +18,9 @@ }; config = lib.mkIf config.me.control_plane.enable { - me.kubernetes.enable = true; me.kube_apiserver.enable = true; me.kube_controller_manager.enable = true; + me.kube_scheduler.enable = true; + me.kubernetes.enable = true; }; } diff --git a/nix/kubernetes/roles/etcd/default.nix b/nix/kubernetes/roles/etcd/default.nix index c3de920..468b865 100644 --- a/nix/kubernetes/roles/etcd/default.nix +++ b/nix/kubernetes/roles/etcd/default.nix @@ -104,6 +104,6 @@ e2fsprogs # mkfs.ext4 gptfdisk # cgdisk ]; - networking.firewall.enable = false; + networking.firewall.enable = false; # TODO: This is just here for debugging / initial development. }; } diff --git a/nix/kubernetes/roles/kube_apiserver/default.nix b/nix/kubernetes/roles/kube_apiserver/default.nix index a78f4dc..66b31aa 100644 --- a/nix/kubernetes/roles/kube_apiserver/default.nix +++ b/nix/kubernetes/roles/kube_apiserver/default.nix @@ -131,7 +131,17 @@ in ); Restart = "on-failure"; RestartSec = 5; + LimitNOFILE = 65536; + User = "kubernetes"; }; }; + + networking.firewall.allowedTCPPorts = [ + 6443 + ]; + + systemd.tmpfiles.rules = [ + "f /var/log/audit.log 0600 kubernetes kubernetes - -" + ]; }; } diff --git a/nix/kubernetes/roles/kube_controller_manager/default.nix b/nix/kubernetes/roles/kube_controller_manager/default.nix index 9b33b49..f31de3f 100644 --- a/nix/kubernetes/roles/kube_controller_manager/default.nix +++ b/nix/kubernetes/roles/kube_controller_manager/default.nix @@ -27,33 +27,37 @@ in description = "Kubernetes Controller Manager"; documentation = [ "https://github.com/kubernetes/kubernetes" ]; wantedBy = [ "kubernetes.target" ]; + after = [ "kube-apiserver.service" ]; # path = with pkgs; [ # zfs # ]; unitConfig.DefaultDependencies = "no"; serviceConfig = { - Type = "notify"; ExecStart = ( shellCommand [ # NEW: "${pkgs.kubernetes}/bin/kube-controller-manager" "--bind-address=0.0.0.0" # "--cluster-cidr=10.200.0.0/16" - "--cluster-cidr=2620:11f:7001:7:ffff:ffff:0ac8:0000/16" + # "--cluster-cidr=2620:11f:7001:7:ffff:ffff:0ac8:0000/16" + "--cluster-cidr=fd49:0595:2bba::/48" "--cluster-name=kubernetes" "--cluster-signing-cert-file=/.persist/keys/kube/ca.crt" "--cluster-signing-key-file=/.persist/keys/kube/ca.key" "--kubeconfig=/.persist/keys/kube/kube-controller-manager.kubeconfig" "--root-ca-file=/.persist/keys/kube/ca.crt" "--service-account-private-key-file=/.persist/keys/kube/service-accounts.key" - "--service-cluster-ip-range=2620:11f:7001:7:ffff:ffff:0ac5:0000/16" # "--service-cluster-ip-range=10.197.0.0/16" + # "--service-cluster-ip-range=2620:11f:7001:7:ffff:ffff:0ac5:0000/16" + "--service-cluster-ip-range=fd00:3e42:e349::/48" "--use-service-account-credentials=true" "--v=2" ] ); Restart = "on-failure"; RestartSec = 5; + LimitNOFILE = 65536; + User = "kubernetes"; }; }; }; diff --git a/nix/kubernetes/roles/kube_scheduler/default.nix b/nix/kubernetes/roles/kube_scheduler/default.nix new file mode 100644 index 0000000..89a7697 --- /dev/null +++ b/nix/kubernetes/roles/kube_scheduler/default.nix @@ -0,0 +1,51 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + # shellCommand = cmd: (lib.concatMapStringsSep " " lib.strings.escapeShellArg cmd); + shellCommand = cmd: (builtins.concatStringsSep " " cmd); +in +{ + imports = [ ]; + + options.me = { + kube_scheduler.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install kube_scheduler."; + }; + }; + + config = lib.mkIf config.me.kube_scheduler.enable { + systemd.services.kube-scheduler = { + enable = true; + description = "Kubernetes Scheduler"; + documentation = [ "https://github.com/kubernetes/kubernetes" ]; + wantedBy = [ "kubernetes.target" ]; + after = [ "kube-apiserver.service" ]; + # path = with pkgs; [ + # zfs + # ]; + unitConfig.DefaultDependencies = "no"; + serviceConfig = { + ExecStart = ( + shellCommand [ + # NEW: + "${pkgs.kubernetes}/bin/kube-scheduler" + "--config=${./files/kube-scheduler.yaml}" + "--v=2" + ] + ); + Restart = "on-failure"; + RestartSec = 5; + LimitNOFILE = 65536; + User = "kubernetes"; + }; + }; + }; +} diff --git a/nix/kubernetes/roles/kube_scheduler/files/kube-scheduler.yaml b/nix/kubernetes/roles/kube_scheduler/files/kube-scheduler.yaml new file mode 100644 index 0000000..3881453 --- /dev/null +++ b/nix/kubernetes/roles/kube_scheduler/files/kube-scheduler.yaml @@ -0,0 +1,6 @@ +apiVersion: kubescheduler.config.k8s.io/v1 +kind: KubeSchedulerConfiguration +clientConnection: + kubeconfig: "/.persist/keys/kube/kube-scheduler.kubeconfig" +leaderElection: + leaderElect: true From 68979154552dd81961447f604bec1f72698fc79a Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 15 Dec 2025 22:32:32 -0500 Subject: [PATCH 12/81] Add worker nodes. --- nix/kubernetes/configuration.nix | 4 +- nix/kubernetes/flake.nix | 9 ++ nix/kubernetes/hosts/worker0/DEPLOY_BOOT | 13 +++ nix/kubernetes/hosts/worker0/DEPLOY_SWITCH | 13 +++ nix/kubernetes/hosts/worker0/ISO | 10 ++ nix/kubernetes/hosts/worker0/SELF_BOOT | 10 ++ nix/kubernetes/hosts/worker0/SELF_BUILD | 10 ++ nix/kubernetes/hosts/worker0/SELF_SWITCH | 10 ++ nix/kubernetes/hosts/worker0/VM_ISO | 10 ++ nix/kubernetes/hosts/worker0/default.nix | 109 ++++++++++++++++++ .../hosts/worker0/hardware-configuration.nix | 31 +++++ nix/kubernetes/hosts/worker0/vm_disk.nix | 94 +++++++++++++++ nix/kubernetes/hosts/worker1/DEPLOY_BOOT | 13 +++ nix/kubernetes/hosts/worker1/DEPLOY_SWITCH | 13 +++ nix/kubernetes/hosts/worker1/ISO | 10 ++ nix/kubernetes/hosts/worker1/SELF_BOOT | 10 ++ nix/kubernetes/hosts/worker1/SELF_BUILD | 10 ++ nix/kubernetes/hosts/worker1/SELF_SWITCH | 10 ++ nix/kubernetes/hosts/worker1/VM_ISO | 10 ++ nix/kubernetes/hosts/worker1/default.nix | 109 ++++++++++++++++++ .../hosts/worker1/hardware-configuration.nix | 31 +++++ nix/kubernetes/hosts/worker1/vm_disk.nix | 94 +++++++++++++++ nix/kubernetes/hosts/worker2/DEPLOY_BOOT | 13 +++ nix/kubernetes/hosts/worker2/DEPLOY_SWITCH | 13 +++ nix/kubernetes/hosts/worker2/ISO | 10 ++ nix/kubernetes/hosts/worker2/SELF_BOOT | 10 ++ nix/kubernetes/hosts/worker2/SELF_BUILD | 10 ++ nix/kubernetes/hosts/worker2/SELF_SWITCH | 10 ++ nix/kubernetes/hosts/worker2/VM_ISO | 10 ++ nix/kubernetes/hosts/worker2/default.nix | 109 ++++++++++++++++++ .../hosts/worker2/hardware-configuration.nix | 31 +++++ nix/kubernetes/hosts/worker2/vm_disk.nix | 94 +++++++++++++++ nix/kubernetes/roles/containerd/default.nix | 46 ++++++++ nix/kubernetes/roles/etcd/default.nix | 1 + nix/kubernetes/roles/minimal_base/default.nix | 2 - nix/kubernetes/roles/worker_node/default.nix | 24 ++++ nix/kubernetes/roles/zfs/default.nix | 68 ----------- .../roles/zfs/files/zfs_clone_recv.bash | 13 --- .../roles/zfs/files/zfs_clone_resume.bash | 17 --- .../roles/zfs/files/zfs_clone_send.bash | 8 -- nix/kubernetes/roles/zrepl/default.nix | 55 --------- 41 files changed, 1012 insertions(+), 165 deletions(-) create mode 100755 nix/kubernetes/hosts/worker0/DEPLOY_BOOT create mode 100755 nix/kubernetes/hosts/worker0/DEPLOY_SWITCH create mode 100755 nix/kubernetes/hosts/worker0/ISO create mode 100755 nix/kubernetes/hosts/worker0/SELF_BOOT create mode 100755 nix/kubernetes/hosts/worker0/SELF_BUILD create mode 100755 nix/kubernetes/hosts/worker0/SELF_SWITCH create mode 100755 nix/kubernetes/hosts/worker0/VM_ISO create mode 100644 nix/kubernetes/hosts/worker0/default.nix create mode 100644 nix/kubernetes/hosts/worker0/hardware-configuration.nix create mode 100644 nix/kubernetes/hosts/worker0/vm_disk.nix create mode 100755 nix/kubernetes/hosts/worker1/DEPLOY_BOOT create mode 100755 nix/kubernetes/hosts/worker1/DEPLOY_SWITCH create mode 100755 nix/kubernetes/hosts/worker1/ISO create mode 100755 nix/kubernetes/hosts/worker1/SELF_BOOT create mode 100755 nix/kubernetes/hosts/worker1/SELF_BUILD create mode 100755 nix/kubernetes/hosts/worker1/SELF_SWITCH create mode 100755 nix/kubernetes/hosts/worker1/VM_ISO create mode 100644 nix/kubernetes/hosts/worker1/default.nix create mode 100644 nix/kubernetes/hosts/worker1/hardware-configuration.nix create mode 100644 nix/kubernetes/hosts/worker1/vm_disk.nix create mode 100755 nix/kubernetes/hosts/worker2/DEPLOY_BOOT create mode 100755 nix/kubernetes/hosts/worker2/DEPLOY_SWITCH create mode 100755 nix/kubernetes/hosts/worker2/ISO create mode 100755 nix/kubernetes/hosts/worker2/SELF_BOOT create mode 100755 nix/kubernetes/hosts/worker2/SELF_BUILD create mode 100755 nix/kubernetes/hosts/worker2/SELF_SWITCH create mode 100755 nix/kubernetes/hosts/worker2/VM_ISO create mode 100644 nix/kubernetes/hosts/worker2/default.nix create mode 100644 nix/kubernetes/hosts/worker2/hardware-configuration.nix create mode 100644 nix/kubernetes/hosts/worker2/vm_disk.nix create mode 100644 nix/kubernetes/roles/containerd/default.nix create mode 100644 nix/kubernetes/roles/worker_node/default.nix delete mode 100644 nix/kubernetes/roles/zfs/default.nix delete mode 100644 nix/kubernetes/roles/zfs/files/zfs_clone_recv.bash delete mode 100644 nix/kubernetes/roles/zfs/files/zfs_clone_resume.bash delete mode 100644 nix/kubernetes/roles/zfs/files/zfs_clone_send.bash delete mode 100644 nix/kubernetes/roles/zrepl/default.nix diff --git a/nix/kubernetes/configuration.nix b/nix/kubernetes/configuration.nix index 11b9538..4cde405 100644 --- a/nix/kubernetes/configuration.nix +++ b/nix/kubernetes/configuration.nix @@ -7,6 +7,7 @@ { imports = [ ./roles/boot + ./roles/containerd ./roles/control_plane ./roles/doas ./roles/dont_use_substituters @@ -24,8 +25,7 @@ ./roles/ssh ./roles/sshd ./roles/user - ./roles/zfs - ./roles/zrepl + ./roles/worker_node ./roles/zsh ./util/install_files ./util/unfree_polyfill diff --git a/nix/kubernetes/flake.nix b/nix/kubernetes/flake.nix index 30e8c39..36a66e7 100644 --- a/nix/kubernetes/flake.nix +++ b/nix/kubernetes/flake.nix @@ -54,6 +54,15 @@ controller2 = { system = "x86_64-linux"; }; + worker0 = { + system = "x86_64-linux"; + }; + worker1 = { + system = "x86_64-linux"; + }; + worker2 = { + system = "x86_64-linux"; + }; }; nixosConfigs = builtins.mapAttrs ( hostname: nodeConfig: format: diff --git a/nix/kubernetes/hosts/worker0/DEPLOY_BOOT b/nix/kubernetes/hosts/worker0/DEPLOY_BOOT new file mode 100755 index 0000000..d716872 --- /dev/null +++ b/nix/kubernetes/hosts/worker0/DEPLOY_BOOT @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=worker0 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild boot --flake "$DIR/../../#worker0" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker0/DEPLOY_SWITCH b/nix/kubernetes/hosts/worker0/DEPLOY_SWITCH new file mode 100755 index 0000000..a47bf6a --- /dev/null +++ b/nix/kubernetes/hosts/worker0/DEPLOY_SWITCH @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=worker0 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild switch --flake "$DIR/../../#worker0" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker0/ISO b/nix/kubernetes/hosts/worker0/ISO new file mode 100755 index 0000000..46caa98 --- /dev/null +++ b/nix/kubernetes/hosts/worker0/ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#worker0.iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker0/SELF_BOOT b/nix/kubernetes/hosts/worker0/SELF_BOOT new file mode 100755 index 0000000..35e6059 --- /dev/null +++ b/nix/kubernetes/hosts/worker0/SELF_BOOT @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild boot --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker0" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker0/SELF_BUILD b/nix/kubernetes/hosts/worker0/SELF_BUILD new file mode 100755 index 0000000..69fbd95 --- /dev/null +++ b/nix/kubernetes/hosts/worker0/SELF_BUILD @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild build --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker0" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker0/SELF_SWITCH b/nix/kubernetes/hosts/worker0/SELF_SWITCH new file mode 100755 index 0000000..b127a67 --- /dev/null +++ b/nix/kubernetes/hosts/worker0/SELF_SWITCH @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild switch --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker0" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker0/VM_ISO b/nix/kubernetes/hosts/worker0/VM_ISO new file mode 100755 index 0000000..3e518fc --- /dev/null +++ b/nix/kubernetes/hosts/worker0/VM_ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#worker0.vm_iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker0/default.nix b/nix/kubernetes/hosts/worker0/default.nix new file mode 100644 index 0000000..1fac5fa --- /dev/null +++ b/nix/kubernetes/hosts/worker0/default.nix @@ -0,0 +1,109 @@ +# MANUAL: On client machines generate signing keys: +# nix-store --generate-binary-cache-key some-name /persist/manual/nix/nix-cache-key.sec /persist/manual/nix/nix-cache-key.pub +# +# Trust other machines and add the substituters: +# nix.binaryCachePublicKeys = [ "some-name:AzNW1MOlkNEsUAXS1jIFZ1QCFKXjV+Y/LrF37quAZ1A=" ]; +# nix.binaryCaches = [ "https://test.example/nix-cache" ]; + +{ + config, + lib, + pkgs, + ... +}: +{ + imports = [ + ./hardware-configuration.nix + ./vm_disk.nix + ]; + + config = { + networking = + let + interface = "enp0s2"; + in + { + # Generate with `head -c4 /dev/urandom | od -A none -t x4` + hostId = "0aadbb10"; + + hostName = "worker0"; # Define your hostname. + + interfaces = { + "${interface}" = { + ipv4.addresses = [ + { + address = "10.215.1.224"; + prefixLength = 24; + } + ]; + + ipv6.addresses = [ + { + address = "2620:11f:7001:7:ffff:ffff:0ad7:01e0"; + prefixLength = 64; + } + ]; + }; + }; + defaultGateway = "10.215.1.1"; + defaultGateway6 = { + # address = "2620:11f:7001:7::1"; + address = "2620:11f:7001:7:ffff:ffff:0ad7:0101"; + inherit interface; + }; + nameservers = [ + "10.215.1.1" + ]; + + dhcpcd.enable = lib.mkForce false; + useDHCP = lib.mkForce false; + }; + + time.timeZone = "America/New_York"; + i18n.defaultLocale = "en_US.UTF-8"; + + me.boot.enable = true; + me.boot.secure = false; + me.mountPersistence = true; + boot.loader.timeout = lib.mkForce 0; # We can always generate a new ISO if we need to access other boot options. + + me.optimizations = { + enable = true; + arch = "znver4"; + # build_arch = "x86-64-v3"; + system_features = [ + "gccarch-znver4" + "gccarch-skylake" + "gccarch-kabylake" + # "gccarch-alderlake" missing WAITPKG + "gccarch-x86-64-v3" + "gccarch-x86-64-v4" + "benchmark" + "big-parallel" + "kvm" + "nixos-test" + ]; + }; + + # Mount tmpfs at /tmp + boot.tmp.useTmpfs = true; + + # Enable TRIM + # services.fstrim.enable = lib.mkDefault true; + + # nix.optimise.automatic = true; + # nix.optimise.dates = [ "03:45" ]; + # nix.optimise.persistent = true; + + environment.systemPackages = with pkgs; [ + htop + ]; + + # nix.sshServe.enable = true; + # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ]; + + me.worker_node.enable = true; + me.dont_use_substituters.enable = true; + me.minimal_base.enable = true; + }; +} diff --git a/nix/kubernetes/hosts/worker0/hardware-configuration.nix b/nix/kubernetes/hosts/worker0/hardware-configuration.nix new file mode 100644 index 0000000..6029e08 --- /dev/null +++ b/nix/kubernetes/hosts/worker0/hardware-configuration.nix @@ -0,0 +1,31 @@ +{ + config, + lib, + modulesPath, + ... +}: + +{ + imports = [ + (modulesPath + "/installer/scan/not-detected.nix") + ]; + + config = { + boot.initrd.availableKernelModules = [ + "nvme" + "xhci_pci" + "thunderbolt" + ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ ]; + boot.extraModulePackages = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + # networking.useDHCP = lib.mkDefault true; + # networking.interfaces.eno1.useDHCP = lib.mkDefault true; + # networking.interfaces.wlp58s0.useDHCP = lib.mkDefault true; + }; +} diff --git a/nix/kubernetes/hosts/worker0/vm_disk.nix b/nix/kubernetes/hosts/worker0/vm_disk.nix new file mode 100644 index 0000000..751f4ba --- /dev/null +++ b/nix/kubernetes/hosts/worker0/vm_disk.nix @@ -0,0 +1,94 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + config = { + # Mount the local disk + fileSystems = lib.mkIf config.me.mountPersistence { + # "/.disk" = lib.mkForce { + # device = "/dev/nvme0n1p1"; + # fsType = "ext4"; + # options = [ + # "noatime" + # "discard" + # ]; + # neededForBoot = true; + # }; + + "/.persist" = lib.mkForce { + device = "bind9p"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + + "/persist" = { + fsType = "none"; + device = "/.persist/persist"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.persist/persist" + ]; + neededForBoot = true; + }; + + "/state" = { + fsType = "none"; + device = "/.persist/state"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.persist/state" + ]; + neededForBoot = true; + }; + + "/k8spv" = lib.mkForce { + device = "k8spv"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + + # "/disk" = { + # fsType = "none"; + # device = "/.disk/persist"; + # options = [ + # "bind" + # "rw" + # ]; + # depends = [ + # "/.disk/persist" + # ]; + # neededForBoot = true; + # }; + }; + }; +} diff --git a/nix/kubernetes/hosts/worker1/DEPLOY_BOOT b/nix/kubernetes/hosts/worker1/DEPLOY_BOOT new file mode 100755 index 0000000..631d46d --- /dev/null +++ b/nix/kubernetes/hosts/worker1/DEPLOY_BOOT @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=worker1 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild boot --flake "$DIR/../../#worker1" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker1/DEPLOY_SWITCH b/nix/kubernetes/hosts/worker1/DEPLOY_SWITCH new file mode 100755 index 0000000..271829c --- /dev/null +++ b/nix/kubernetes/hosts/worker1/DEPLOY_SWITCH @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=worker1 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild switch --flake "$DIR/../../#worker1" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker1/ISO b/nix/kubernetes/hosts/worker1/ISO new file mode 100755 index 0000000..4c2b969 --- /dev/null +++ b/nix/kubernetes/hosts/worker1/ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#worker1.iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker1/SELF_BOOT b/nix/kubernetes/hosts/worker1/SELF_BOOT new file mode 100755 index 0000000..cf928cd --- /dev/null +++ b/nix/kubernetes/hosts/worker1/SELF_BOOT @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild boot --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker1" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker1/SELF_BUILD b/nix/kubernetes/hosts/worker1/SELF_BUILD new file mode 100755 index 0000000..73c846e --- /dev/null +++ b/nix/kubernetes/hosts/worker1/SELF_BUILD @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild build --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker1" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker1/SELF_SWITCH b/nix/kubernetes/hosts/worker1/SELF_SWITCH new file mode 100755 index 0000000..d6a989d --- /dev/null +++ b/nix/kubernetes/hosts/worker1/SELF_SWITCH @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild switch --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker1" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker1/VM_ISO b/nix/kubernetes/hosts/worker1/VM_ISO new file mode 100755 index 0000000..ffab210 --- /dev/null +++ b/nix/kubernetes/hosts/worker1/VM_ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#worker1.vm_iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker1/default.nix b/nix/kubernetes/hosts/worker1/default.nix new file mode 100644 index 0000000..6f951f0 --- /dev/null +++ b/nix/kubernetes/hosts/worker1/default.nix @@ -0,0 +1,109 @@ +# MANUAL: On client machines generate signing keys: +# nix-store --generate-binary-cache-key some-name /persist/manual/nix/nix-cache-key.sec /persist/manual/nix/nix-cache-key.pub +# +# Trust other machines and add the substituters: +# nix.binaryCachePublicKeys = [ "some-name:AzNW1MOlkNEsUAXS1jIFZ1QCFKXjV+Y/LrF37quAZ1A=" ]; +# nix.binaryCaches = [ "https://test.example/nix-cache" ]; + +{ + config, + lib, + pkgs, + ... +}: +{ + imports = [ + ./hardware-configuration.nix + ./vm_disk.nix + ]; + + config = { + networking = + let + interface = "enp0s2"; + in + { + # Generate with `head -c4 /dev/urandom | od -A none -t x4` + hostId = "4324346d"; + + hostName = "worker1"; # Define your hostname. + + interfaces = { + "${interface}" = { + ipv4.addresses = [ + { + address = "10.215.1.225"; + prefixLength = 24; + } + ]; + + ipv6.addresses = [ + { + address = "2620:11f:7001:7:ffff:ffff:0ad7:01e1"; + prefixLength = 64; + } + ]; + }; + }; + defaultGateway = "10.215.1.1"; + defaultGateway6 = { + # address = "2620:11f:7001:7::1"; + address = "2620:11f:7001:7:ffff:ffff:0ad7:0101"; + inherit interface; + }; + nameservers = [ + "10.215.1.1" + ]; + + dhcpcd.enable = lib.mkForce false; + useDHCP = lib.mkForce false; + }; + + time.timeZone = "America/New_York"; + i18n.defaultLocale = "en_US.UTF-8"; + + me.boot.enable = true; + me.boot.secure = false; + me.mountPersistence = true; + boot.loader.timeout = lib.mkForce 0; # We can always generate a new ISO if we need to access other boot options. + + me.optimizations = { + enable = true; + arch = "znver4"; + # build_arch = "x86-64-v3"; + system_features = [ + "gccarch-znver4" + "gccarch-skylake" + "gccarch-kabylake" + # "gccarch-alderlake" missing WAITPKG + "gccarch-x86-64-v3" + "gccarch-x86-64-v4" + "benchmark" + "big-parallel" + "kvm" + "nixos-test" + ]; + }; + + # Mount tmpfs at /tmp + boot.tmp.useTmpfs = true; + + # Enable TRIM + # services.fstrim.enable = lib.mkDefault true; + + # nix.optimise.automatic = true; + # nix.optimise.dates = [ "03:45" ]; + # nix.optimise.persistent = true; + + environment.systemPackages = with pkgs; [ + htop + ]; + + # nix.sshServe.enable = true; + # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ]; + + me.worker_node.enable = true; + me.dont_use_substituters.enable = true; + me.minimal_base.enable = true; + }; +} diff --git a/nix/kubernetes/hosts/worker1/hardware-configuration.nix b/nix/kubernetes/hosts/worker1/hardware-configuration.nix new file mode 100644 index 0000000..6029e08 --- /dev/null +++ b/nix/kubernetes/hosts/worker1/hardware-configuration.nix @@ -0,0 +1,31 @@ +{ + config, + lib, + modulesPath, + ... +}: + +{ + imports = [ + (modulesPath + "/installer/scan/not-detected.nix") + ]; + + config = { + boot.initrd.availableKernelModules = [ + "nvme" + "xhci_pci" + "thunderbolt" + ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ ]; + boot.extraModulePackages = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + # networking.useDHCP = lib.mkDefault true; + # networking.interfaces.eno1.useDHCP = lib.mkDefault true; + # networking.interfaces.wlp58s0.useDHCP = lib.mkDefault true; + }; +} diff --git a/nix/kubernetes/hosts/worker1/vm_disk.nix b/nix/kubernetes/hosts/worker1/vm_disk.nix new file mode 100644 index 0000000..751f4ba --- /dev/null +++ b/nix/kubernetes/hosts/worker1/vm_disk.nix @@ -0,0 +1,94 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + config = { + # Mount the local disk + fileSystems = lib.mkIf config.me.mountPersistence { + # "/.disk" = lib.mkForce { + # device = "/dev/nvme0n1p1"; + # fsType = "ext4"; + # options = [ + # "noatime" + # "discard" + # ]; + # neededForBoot = true; + # }; + + "/.persist" = lib.mkForce { + device = "bind9p"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + + "/persist" = { + fsType = "none"; + device = "/.persist/persist"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.persist/persist" + ]; + neededForBoot = true; + }; + + "/state" = { + fsType = "none"; + device = "/.persist/state"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.persist/state" + ]; + neededForBoot = true; + }; + + "/k8spv" = lib.mkForce { + device = "k8spv"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + + # "/disk" = { + # fsType = "none"; + # device = "/.disk/persist"; + # options = [ + # "bind" + # "rw" + # ]; + # depends = [ + # "/.disk/persist" + # ]; + # neededForBoot = true; + # }; + }; + }; +} diff --git a/nix/kubernetes/hosts/worker2/DEPLOY_BOOT b/nix/kubernetes/hosts/worker2/DEPLOY_BOOT new file mode 100755 index 0000000..eb5db11 --- /dev/null +++ b/nix/kubernetes/hosts/worker2/DEPLOY_BOOT @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=worker2 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild boot --flake "$DIR/../../#worker2" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker2/DEPLOY_SWITCH b/nix/kubernetes/hosts/worker2/DEPLOY_SWITCH new file mode 100755 index 0000000..c321328 --- /dev/null +++ b/nix/kubernetes/hosts/worker2/DEPLOY_SWITCH @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=worker2 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild switch --flake "$DIR/../../#worker2" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker2/ISO b/nix/kubernetes/hosts/worker2/ISO new file mode 100755 index 0000000..ad8a821 --- /dev/null +++ b/nix/kubernetes/hosts/worker2/ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#worker2.iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker2/SELF_BOOT b/nix/kubernetes/hosts/worker2/SELF_BOOT new file mode 100755 index 0000000..bdfde92 --- /dev/null +++ b/nix/kubernetes/hosts/worker2/SELF_BOOT @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild boot --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker2" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker2/SELF_BUILD b/nix/kubernetes/hosts/worker2/SELF_BUILD new file mode 100755 index 0000000..36cc5b5 --- /dev/null +++ b/nix/kubernetes/hosts/worker2/SELF_BUILD @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild build --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker2" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker2/SELF_SWITCH b/nix/kubernetes/hosts/worker2/SELF_SWITCH new file mode 100755 index 0000000..7af1d15 --- /dev/null +++ b/nix/kubernetes/hosts/worker2/SELF_SWITCH @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild switch --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker2" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker2/VM_ISO b/nix/kubernetes/hosts/worker2/VM_ISO new file mode 100755 index 0000000..7f1f203 --- /dev/null +++ b/nix/kubernetes/hosts/worker2/VM_ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#worker2.vm_iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker2/default.nix b/nix/kubernetes/hosts/worker2/default.nix new file mode 100644 index 0000000..d595c0d --- /dev/null +++ b/nix/kubernetes/hosts/worker2/default.nix @@ -0,0 +1,109 @@ +# MANUAL: On client machines generate signing keys: +# nix-store --generate-binary-cache-key some-name /persist/manual/nix/nix-cache-key.sec /persist/manual/nix/nix-cache-key.pub +# +# Trust other machines and add the substituters: +# nix.binaryCachePublicKeys = [ "some-name:AzNW1MOlkNEsUAXS1jIFZ1QCFKXjV+Y/LrF37quAZ1A=" ]; +# nix.binaryCaches = [ "https://test.example/nix-cache" ]; + +{ + config, + lib, + pkgs, + ... +}: +{ + imports = [ + ./hardware-configuration.nix + ./vm_disk.nix + ]; + + config = { + networking = + let + interface = "enp0s2"; + in + { + # Generate with `head -c4 /dev/urandom | od -A none -t x4` + hostId = "ce017961"; + + hostName = "worker2"; # Define your hostname. + + interfaces = { + "${interface}" = { + ipv4.addresses = [ + { + address = "10.215.1.226"; + prefixLength = 24; + } + ]; + + ipv6.addresses = [ + { + address = "2620:11f:7001:7:ffff:ffff:0ad7:01e2"; + prefixLength = 64; + } + ]; + }; + }; + defaultGateway = "10.215.1.1"; + defaultGateway6 = { + # address = "2620:11f:7001:7::1"; + address = "2620:11f:7001:7:ffff:ffff:0ad7:0101"; + inherit interface; + }; + nameservers = [ + "10.215.1.1" + ]; + + dhcpcd.enable = lib.mkForce false; + useDHCP = lib.mkForce false; + }; + + time.timeZone = "America/New_York"; + i18n.defaultLocale = "en_US.UTF-8"; + + me.boot.enable = true; + me.boot.secure = false; + me.mountPersistence = true; + boot.loader.timeout = lib.mkForce 0; # We can always generate a new ISO if we need to access other boot options. + + me.optimizations = { + enable = true; + arch = "znver4"; + # build_arch = "x86-64-v3"; + system_features = [ + "gccarch-znver4" + "gccarch-skylake" + "gccarch-kabylake" + # "gccarch-alderlake" missing WAITPKG + "gccarch-x86-64-v3" + "gccarch-x86-64-v4" + "benchmark" + "big-parallel" + "kvm" + "nixos-test" + ]; + }; + + # Mount tmpfs at /tmp + boot.tmp.useTmpfs = true; + + # Enable TRIM + # services.fstrim.enable = lib.mkDefault true; + + # nix.optimise.automatic = true; + # nix.optimise.dates = [ "03:45" ]; + # nix.optimise.persistent = true; + + environment.systemPackages = with pkgs; [ + htop + ]; + + # nix.sshServe.enable = true; + # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ]; + + me.worker_node.enable = true; + me.dont_use_substituters.enable = true; + me.minimal_base.enable = true; + }; +} diff --git a/nix/kubernetes/hosts/worker2/hardware-configuration.nix b/nix/kubernetes/hosts/worker2/hardware-configuration.nix new file mode 100644 index 0000000..6029e08 --- /dev/null +++ b/nix/kubernetes/hosts/worker2/hardware-configuration.nix @@ -0,0 +1,31 @@ +{ + config, + lib, + modulesPath, + ... +}: + +{ + imports = [ + (modulesPath + "/installer/scan/not-detected.nix") + ]; + + config = { + boot.initrd.availableKernelModules = [ + "nvme" + "xhci_pci" + "thunderbolt" + ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ ]; + boot.extraModulePackages = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + # networking.useDHCP = lib.mkDefault true; + # networking.interfaces.eno1.useDHCP = lib.mkDefault true; + # networking.interfaces.wlp58s0.useDHCP = lib.mkDefault true; + }; +} diff --git a/nix/kubernetes/hosts/worker2/vm_disk.nix b/nix/kubernetes/hosts/worker2/vm_disk.nix new file mode 100644 index 0000000..751f4ba --- /dev/null +++ b/nix/kubernetes/hosts/worker2/vm_disk.nix @@ -0,0 +1,94 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + config = { + # Mount the local disk + fileSystems = lib.mkIf config.me.mountPersistence { + # "/.disk" = lib.mkForce { + # device = "/dev/nvme0n1p1"; + # fsType = "ext4"; + # options = [ + # "noatime" + # "discard" + # ]; + # neededForBoot = true; + # }; + + "/.persist" = lib.mkForce { + device = "bind9p"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + + "/persist" = { + fsType = "none"; + device = "/.persist/persist"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.persist/persist" + ]; + neededForBoot = true; + }; + + "/state" = { + fsType = "none"; + device = "/.persist/state"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.persist/state" + ]; + neededForBoot = true; + }; + + "/k8spv" = lib.mkForce { + device = "k8spv"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + + # "/disk" = { + # fsType = "none"; + # device = "/.disk/persist"; + # options = [ + # "bind" + # "rw" + # ]; + # depends = [ + # "/.disk/persist" + # ]; + # neededForBoot = true; + # }; + }; + }; +} diff --git a/nix/kubernetes/roles/containerd/default.nix b/nix/kubernetes/roles/containerd/default.nix new file mode 100644 index 0000000..18dac5c --- /dev/null +++ b/nix/kubernetes/roles/containerd/default.nix @@ -0,0 +1,46 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + options.me = { + containerd.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install containerd."; + }; + }; + + config = lib.mkIf config.me.containerd.enable { + virtualisation.containerd.enable = true; + virtualisation.containerd.settings = { + "plugins" = { + "io.containerd.grpc.v1.cri" = { + "cni" = { + "bin_dir" = "/opt/cni/bin"; + "conf_dir" = "/etc/cni/net.d"; + }; + "containerd" = { + "default_runtime_name" = "runc"; + "runtimes" = { + "runc" = { + "options" = { + "SystemdCgroup" = true; + }; + "runtime_type" = "io.containerd.runc.v2"; + }; + }; + "snapshotter" = "overlayfs"; + }; + }; + }; + "version" = 2; + }; + }; +} diff --git a/nix/kubernetes/roles/etcd/default.nix b/nix/kubernetes/roles/etcd/default.nix index 468b865..ae34542 100644 --- a/nix/kubernetes/roles/etcd/default.nix +++ b/nix/kubernetes/roles/etcd/default.nix @@ -105,5 +105,6 @@ gptfdisk # cgdisk ]; networking.firewall.enable = false; # TODO: This is just here for debugging / initial development. + # TODO: Maybe use networking.nftables.enable to switch to nftables? }; } diff --git a/nix/kubernetes/roles/minimal_base/default.nix b/nix/kubernetes/roles/minimal_base/default.nix index 1654729..e57f1f2 100644 --- a/nix/kubernetes/roles/minimal_base/default.nix +++ b/nix/kubernetes/roles/minimal_base/default.nix @@ -24,8 +24,6 @@ me.ssh.enable = true; me.sshd.enable = true; me.user.enable = true; - me.zfs.enable = true; - me.zrepl.enable = true; me.zsh.enable = true; # TODO: Maybe add me.boot.enable ? diff --git a/nix/kubernetes/roles/worker_node/default.nix b/nix/kubernetes/roles/worker_node/default.nix new file mode 100644 index 0000000..9e7d4bd --- /dev/null +++ b/nix/kubernetes/roles/worker_node/default.nix @@ -0,0 +1,24 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + options.me = { + worker_node.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install worker_node."; + }; + }; + + config = lib.mkIf config.me.worker_node.enable { + me.containerd.enable = true; + me.kubernetes.enable = true; + }; +} diff --git a/nix/kubernetes/roles/zfs/default.nix b/nix/kubernetes/roles/zfs/default.nix deleted file mode 100644 index ed53227..0000000 --- a/nix/kubernetes/roles/zfs/default.nix +++ /dev/null @@ -1,68 +0,0 @@ -{ - config, - lib, - pkgs, - ... -}: - -let - zfs_clone_send = - (pkgs.writeScriptBin "zfs_clone_send" (builtins.readFile ./files/zfs_clone_send.bash)).overrideAttrs - (old: { - buildCommand = "${old.buildCommand}\n patchShebangs $out"; - - }); - zfs_clone_recv = - (pkgs.writeScriptBin "zfs_clone_recv" (builtins.readFile ./files/zfs_clone_recv.bash)).overrideAttrs - (old: { - buildCommand = "${old.buildCommand}\n patchShebangs $out"; - - }); - zfs_clone_resume = - (pkgs.writeScriptBin "zfs_clone_resume" (builtins.readFile ./files/zfs_clone_resume.bash)) - .overrideAttrs - (old: { - buildCommand = "${old.buildCommand}\n patchShebangs $out"; - - }); -in -{ - imports = [ ]; - - options.me = { - zfs.enable = lib.mkOption { - type = lib.types.bool; - default = false; - example = true; - description = "Whether we want to install zfs."; - }; - }; - - config = lib.mkIf config.me.zfs.enable { - # Technically only needed when building the ISO because nix detects ZFS in the filesystem list normally. I basically always want this so I'm just setting it to always be on. - boot.supportedFilesystems.zfs = true; - - boot.zfs.devNodes = "/dev/disk/by-partuuid"; - - services.zfs = { - autoScrub = { - enable = true; - interval = "monthly"; - }; - trim.enable = true; - }; - - environment.systemPackages = [ - zfs_clone_send - zfs_clone_recv - zfs_clone_resume - ]; - - environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) { - hideMounts = true; - directories = [ - "/etc/zfs/zpool.cache" # Which zpools to import, the root zpool is already imported and does not need this cache file but this captures additional pools. - ]; - }; - }; -} diff --git a/nix/kubernetes/roles/zfs/files/zfs_clone_recv.bash b/nix/kubernetes/roles/zfs/files/zfs_clone_recv.bash deleted file mode 100644 index e51e2c7..0000000 --- a/nix/kubernetes/roles/zfs/files/zfs_clone_recv.bash +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -# -# A zfs-send alias that creates a perfect clone with good defaults. -set -euo pipefail -IFS=$'\n\t' -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -# -s if the stream is interrupted, save the partial stream. The stream can then be resumed by doing a zfs send -t token where token is the receive_resume_token prop on the dataset we received into. -# -u Do not mount the filesystem we are receiving. We can always mount afterwards but this avoids issues with streams with mountpoints to places like / -# Can optionally add -F to destroy the dataset in the recv location. -exec zfs recv -s -u "${@}" - -# To delete an interrupted recv, run `zfs receive -A dataset` diff --git a/nix/kubernetes/roles/zfs/files/zfs_clone_resume.bash b/nix/kubernetes/roles/zfs/files/zfs_clone_resume.bash deleted file mode 100644 index 8b9ddac..0000000 --- a/nix/kubernetes/roles/zfs/files/zfs_clone_resume.bash +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -# -# Resume a zfs send. -set -euo pipefail -IFS=$'\n\t' -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -function main { - local hst="$1" - local dst="$2" - local token - token=$(zfs get -H -o value receive_resume_token "$dst") - ssh "$hst" doas zfs send --verbose -t "$token" | doas zfs recv -s "$dst" - -} - -main "${@}" diff --git a/nix/kubernetes/roles/zfs/files/zfs_clone_send.bash b/nix/kubernetes/roles/zfs/files/zfs_clone_send.bash deleted file mode 100644 index 92a325d..0000000 --- a/nix/kubernetes/roles/zfs/files/zfs_clone_send.bash +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -# -# A zfs-send alias that creates a perfect clone with good defaults. -set -euo pipefail -IFS=$'\n\t' -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -exec zfs send --compressed --replicate --large-block --embed --verbose --raw "${@}" diff --git a/nix/kubernetes/roles/zrepl/default.nix b/nix/kubernetes/roles/zrepl/default.nix deleted file mode 100644 index 2c60485..0000000 --- a/nix/kubernetes/roles/zrepl/default.nix +++ /dev/null @@ -1,55 +0,0 @@ -{ - config, - lib, - ... -}: - -{ - imports = [ ]; - - options.me = { - zrepl.enable = lib.mkOption { - type = lib.types.bool; - default = false; - example = true; - description = "Whether we want to install zrepl."; - }; - }; - - config = lib.mkIf config.me.zrepl.enable { - services.zrepl = { - enable = true; - settings = { - jobs = [ - { - name = "snapjob"; - type = "snap"; - filesystems = { - "zroot/linux/nix/persist<" = true; - "zroot/bridge<" = true; - }; - snapshotting = { - type = "periodic"; - interval = "15m"; - prefix = "zrepl_"; - }; - pruning = { - keep = [ - { - type = "grid"; - grid = "1x1h(keep=all) | 24x1h | 14x1d"; - regex = "^zrepl_.*"; - } - { - type = "regex"; - negate = true; - regex = "^zrepl_.*"; - } - ]; - }; - } - ]; - }; - }; - }; -} From 95ec23126a2d9a35ea088782c4927cb698556265 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Tue, 16 Dec 2025 19:31:33 -0500 Subject: [PATCH 13/81] Add kubelet. --- nix/kubernetes/configuration.nix | 1 + .../keys/package/deploy-script/package.nix | 74 ++++++++++++- nix/kubernetes/keys/scope.nix | 27 ++--- .../roles/kube_controller_manager/default.nix | 1 - nix/kubernetes/roles/kubelet/default.nix | 101 ++++++++++++++++++ .../roles/kubelet/files/kubelet-config.yaml | 25 +++++ nix/kubernetes/roles/worker_node/default.nix | 1 + 7 files changed, 214 insertions(+), 16 deletions(-) create mode 100644 nix/kubernetes/roles/kubelet/default.nix create mode 100644 nix/kubernetes/roles/kubelet/files/kubelet-config.yaml diff --git a/nix/kubernetes/configuration.nix b/nix/kubernetes/configuration.nix index 4cde405..617128d 100644 --- a/nix/kubernetes/configuration.nix +++ b/nix/kubernetes/configuration.nix @@ -17,6 +17,7 @@ ./roles/kube_apiserver ./roles/kube_controller_manager ./roles/kube_scheduler + ./roles/kubelet ./roles/kubernetes ./roles/minimal_base ./roles/network diff --git a/nix/kubernetes/keys/package/deploy-script/package.nix b/nix/kubernetes/keys/package/deploy-script/package.nix index d3e8da3..7815271 100644 --- a/nix/kubernetes/keys/package/deploy-script/package.nix +++ b/nix/kubernetes/keys/package/deploy-script/package.nix @@ -8,6 +8,7 @@ # installCheckPhase # distPhase { + config, lib, stdenv, writeShellScript, @@ -16,17 +17,35 @@ ... }: let + vm_name_to_hostname = + let + mapping = { + "nc0" = "controller0"; + "nc1" = "controller1"; + "nc2" = "controller2"; + "nw0" = "worker0"; + "nw1" = "worker1"; + "nw2" = "worker2"; + }; + in + (vm_name: mapping."${vm_name}"); + deploy_script_body = ( '' set -euo pipefail IFS=$'\n\t' DIR="$( cd "$( dirname "''${BASH_SOURCE[0]}" )" && pwd )" '' - + (lib.concatMapStringsSep "\n" deploy_machine [ + + (lib.concatMapStringsSep "\n" deploy_control_plane [ "nc0" "nc1" "nc2" ]) + + (lib.concatMapStringsSep "\n" deploy_worker [ + "nw0" + "nw1" + "nw2" + ]) ); deploy_script = (writeShellScript "deploy-script" deploy_script_body); deploy_file = ( @@ -50,14 +69,14 @@ let '' ); - deploy_machine = ( + deploy_control_plane = ( vm_name: ( '' ## ## Create directories on ${vm_name} ## - ${openssh}/bin/ssh mrmanager doas install -d -o 11235 -g 11235 -m 0755 /vm/${vm_name}/persist/keys + ${openssh}/bin/ssh mrmanager doas install -d -o 0 -g 0 -m 0755 /vm/${vm_name}/persist/keys ${openssh}/bin/ssh mrmanager doas install -d -o 10016 -g 10016 -m 0755 /vm/${vm_name}/persist/keys/etcd ${openssh}/bin/ssh mrmanager doas install -d -o 10024 -g 10024 -m 0755 /vm/${vm_name}/persist/keys/kube @@ -152,6 +171,55 @@ let ]) ) ); + deploy_worker = ( + vm_name: + ( + '' + ## + ## Create directories on ${vm_name} + ## + ${openssh}/bin/ssh mrmanager doas install -d -o 0 -g 0 -m 0755 /vm/${vm_name}/persist/keys + ${openssh}/bin/ssh mrmanager doas install -d -o 10024 -g 10024 -m 0755 /vm/${vm_name}/persist/keys/kube + + + '' + + (lib.concatMapStringsSep "\n" deploy_file [ + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.ca}/ca.crt"; + owner = 10024; + group = 10024; + mode = "0640"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.keys."${vm_name_to_hostname vm_name}"}/${vm_name_to_hostname vm_name}.crt"; + name = "kubelet.crt"; + owner = 10024; + group = 10024; + mode = "0640"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.keys."${vm_name_to_hostname vm_name}"}/${vm_name_to_hostname vm_name}.key"; + name = "kubelet.key"; + owner = 10024; + group = 10024; + mode = "0600"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${ + k8s.client-configs."${vm_name_to_hostname vm_name}" + }/${vm_name_to_hostname vm_name}.kubeconfig"; + name = "kubeconfig"; + owner = 10024; + group = 10024; + mode = "0600"; + } + ]) + ) + ); in stdenv.mkDerivation (finalAttrs: { name = "deploy-script"; diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index 1aef3f1..aeca401 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -69,12 +69,6 @@ let ]; }; }; - # _vm_name_to_hostname = { - # "nc0" = "controller0"; - # "nc1" = "controller1"; - # "nc2" = "controller2"; - # }; - # vm_name_to_hostname = (vm_name: _vm_name_to_hostname."${vm_name}"); in makeScope newScope ( self: @@ -113,27 +107,36 @@ makeScope newScope ( { controller0 = { config_user = "system:node:controller0"; - config_server = "https://server.kubernetes.local:6443"; + config_server = "https://127.0.0.1:6443"; + # config_server = "https://server.kubernetes.local:6443"; }; controller1 = { config_user = "system:node:controller1"; - config_server = "https://server.kubernetes.local:6443"; + config_server = "https://127.0.0.1:6443"; + # config_server = "https://server.kubernetes.local:6443"; }; controller2 = { config_user = "system:node:controller2"; - config_server = "https://server.kubernetes.local:6443"; + config_server = "https://127.0.0.1:6443"; + # config_server = "https://server.kubernetes.local:6443"; }; worker0 = { config_user = "system:node:worker0"; - config_server = "https://server.kubernetes.local:6443"; + config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443"; + # config_server = "https://127.0.0.1:6443"; + # config_server = "https://server.kubernetes.local:6443"; }; worker1 = { config_user = "system:node:worker1"; - config_server = "https://server.kubernetes.local:6443"; + config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443"; + # config_server = "https://127.0.0.1:6443"; + # config_server = "https://server.kubernetes.local:6443"; }; worker2 = { config_user = "system:node:worker2"; - config_server = "https://server.kubernetes.local:6443"; + config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443"; + # config_server = "https://127.0.0.1:6443"; + # config_server = "https://server.kubernetes.local:6443"; }; kube-proxy = { config_user = "system:kube-proxy"; diff --git a/nix/kubernetes/roles/kube_controller_manager/default.nix b/nix/kubernetes/roles/kube_controller_manager/default.nix index f31de3f..955858a 100644 --- a/nix/kubernetes/roles/kube_controller_manager/default.nix +++ b/nix/kubernetes/roles/kube_controller_manager/default.nix @@ -35,7 +35,6 @@ in serviceConfig = { ExecStart = ( shellCommand [ - # NEW: "${pkgs.kubernetes}/bin/kube-controller-manager" "--bind-address=0.0.0.0" # "--cluster-cidr=10.200.0.0/16" diff --git a/nix/kubernetes/roles/kubelet/default.nix b/nix/kubernetes/roles/kubelet/default.nix new file mode 100644 index 0000000..e3dc475 --- /dev/null +++ b/nix/kubernetes/roles/kubelet/default.nix @@ -0,0 +1,101 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + # shellCommand = cmd: (lib.concatMapStringsSep " " lib.strings.escapeShellArg cmd); + shellCommand = cmd: (builtins.concatStringsSep " " cmd); + settingsFormat = pkgs.formats.yaml { }; + config_file = settingsFormat.generate "kubelet-config.yaml" config.me.kubelet.settings; +in +{ + imports = [ ]; + + options.me = { + kubelet.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install kubelet."; + }; + + kubelet.settings = lib.mkOption { + type = settingsFormat.type; + default = { + kind = "KubeletConfiguration"; + apiVersion = "kubelet.config.k8s.io/v1beta1"; + address = "0.0.0.0"; + authentication = { + anonymous = { + enabled = false; + }; + webhook = { + enabled = true; + }; + x509 = { + clientCAFile = "/var/lib/kubelet/ca.crt"; + }; + }; + authorization = { + mode = "Webhook"; + }; + cgroupDriver = "systemd"; + containerRuntimeEndpoint = "unix:///var/run/containerd/containerd.sock"; + enableServer = true; + failSwapOn = false; + maxPods = 16; + memorySwap = { + swapBehavior = "NoSwap"; + }; + port = 10250; + resolvConf = "/etc/resolv.conf"; + registerNode = true; + runtimeRequestTimeout = "15m"; + tlsCertFile = "/var/lib/kubelet/kubelet.crt"; + tlsPrivateKeyFile = "/var/lib/kubelet/kubelet.key"; + }; + description = '' + kubelet-config.yaml + ''; + }; + }; + + config = lib.mkIf config.me.kubelet.enable { + systemd.services.kubelet = { + enable = true; + description = "Kubernetes Kubelet"; + documentation = [ "https://github.com/kubernetes/kubernetes" ]; + wantedBy = [ "kubernetes.target" ]; + after = [ "containerd.service" ]; + requires = [ "containerd.service" ]; + # path = with pkgs; [ + # zfs + # ]; + unitConfig.DefaultDependencies = "no"; + serviceConfig = { + ExecStart = ( + shellCommand [ + "${pkgs.kubernetes}/bin/kubelet" + # "--config=${config_file}" + "--config=${./files/kubelet-config.yaml}" + "--kubeconfig=/.persist/keys/kube/kubeconfig" + "--v=2" + ] + ); + Restart = "on-failure"; + RestartSec = 5; + # ConfigurationDirectory = "kubernetes"; + # CPUAccounting = "true"; + # IPAccounting = "true"; + # KillMode = "process"; + # MemoryAccounting = "true"; + # StartLimitInterval = 0; + # RuntimeDirectory = "kubelet"; + # StateDirectory = "kubelet"; + }; + }; + }; +} diff --git a/nix/kubernetes/roles/kubelet/files/kubelet-config.yaml b/nix/kubernetes/roles/kubelet/files/kubelet-config.yaml new file mode 100644 index 0000000..2313c53 --- /dev/null +++ b/nix/kubernetes/roles/kubelet/files/kubelet-config.yaml @@ -0,0 +1,25 @@ +kind: KubeletConfiguration +apiVersion: kubelet.config.k8s.io/v1beta1 +address: "0.0.0.0" +authentication: + anonymous: + enabled: false + webhook: + enabled: true + x509: + clientCAFile: "/.persist/keys/kube/ca.crt" +authorization: + mode: Webhook +cgroupDriver: systemd +containerRuntimeEndpoint: "unix:///var/run/containerd/containerd.sock" +enableServer: true +failSwapOn: false +maxPods: 16 +memorySwap: + swapBehavior: NoSwap +port: 10250 +resolvConf: "/etc/resolv.conf" +registerNode: true +runtimeRequestTimeout: "15m" +tlsCertFile: "/.persist/keys/kube/kubelet.crt" +tlsPrivateKeyFile: "/.persist/keys/kube/kubelet.key" diff --git a/nix/kubernetes/roles/worker_node/default.nix b/nix/kubernetes/roles/worker_node/default.nix index 9e7d4bd..af08275 100644 --- a/nix/kubernetes/roles/worker_node/default.nix +++ b/nix/kubernetes/roles/worker_node/default.nix @@ -19,6 +19,7 @@ config = lib.mkIf config.me.worker_node.enable { me.containerd.enable = true; + me.kubelet.enable = true; me.kubernetes.enable = true; }; } From 19b2da0d4016a6668809a71144506452bca81a93 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Tue, 16 Dec 2025 21:07:39 -0500 Subject: [PATCH 14/81] Add kube-proxy. --- nix/configuration/README.org | 12 ++++ nix/kubernetes/README.org | 14 ++++ nix/kubernetes/configuration.nix | 2 + .../keys/package/deploy-script/package.nix | 9 ++- nix/kubernetes/keys/scope.nix | 4 +- nix/kubernetes/roles/containerd/default.nix | 49 +++++++------ .../roles/control_plane/default.nix | 1 + nix/kubernetes/roles/firewall/default.nix | 36 ++++++++++ nix/kubernetes/roles/kube_proxy/default.nix | 68 +++++++++++++++++++ nix/kubernetes/roles/kubelet/default.nix | 45 +----------- nix/kubernetes/roles/kubernetes/default.nix | 8 +++ nix/kubernetes/roles/worker_node/default.nix | 2 + 12 files changed, 185 insertions(+), 65 deletions(-) create mode 100644 nix/configuration/README.org create mode 100644 nix/kubernetes/roles/firewall/default.nix create mode 100644 nix/kubernetes/roles/kube_proxy/default.nix diff --git a/nix/configuration/README.org b/nix/configuration/README.org new file mode 100644 index 0000000..ba88116 --- /dev/null +++ b/nix/configuration/README.org @@ -0,0 +1,12 @@ +* To-do +** Perhaps use overlay for /etc for speedup +#+begin_src nix + system.etc.overlay.enable = true; +#+end_src +** read https://nixos.org/manual/nixos/stable/ +** Performance for mini pc +#+begin_src nix + security.pam.loginLimits = [ + { domain = "@users"; item = "rtprio"; type = "-"; value = 1; } + ]; +#+end_src diff --git a/nix/kubernetes/README.org b/nix/kubernetes/README.org index ba88116..5f156a2 100644 --- a/nix/kubernetes/README.org +++ b/nix/kubernetes/README.org @@ -10,3 +10,17 @@ { domain = "@users"; item = "rtprio"; type = "-"; value = 1; } ]; #+end_src +* Bootstrap +** Install cilium +#+begin_src bash + helm repo add cilium https://helm.cilium.io/ + helm install --dry-run cilium cilium/cilium --version 1.18.4 --namespace kube-system \ + --set kubeProxyReplacement=true \ + --set k8sServiceHost=${API_SERVER_IP} \ + --set k8sServicePort=${API_SERVER_PORT} + + + kubectl -n kube-system exec ds/cilium -- cilium-dbg status --verbose + kubectl -n kube-system exec ds/cilium -- cilium-dbg status | grep KubeProxyReplacement + +#+end_src diff --git a/nix/kubernetes/configuration.nix b/nix/kubernetes/configuration.nix index 617128d..b6ca2c9 100644 --- a/nix/kubernetes/configuration.nix +++ b/nix/kubernetes/configuration.nix @@ -12,10 +12,12 @@ ./roles/doas ./roles/dont_use_substituters ./roles/etcd + ./roles/firewall ./roles/image_based_appliance ./roles/iso ./roles/kube_apiserver ./roles/kube_controller_manager + ./roles/kube_proxy ./roles/kube_scheduler ./roles/kubelet ./roles/kubernetes diff --git a/nix/kubernetes/keys/package/deploy-script/package.nix b/nix/kubernetes/keys/package/deploy-script/package.nix index 7815271..8b6844d 100644 --- a/nix/kubernetes/keys/package/deploy-script/package.nix +++ b/nix/kubernetes/keys/package/deploy-script/package.nix @@ -212,7 +212,14 @@ let file = "${ k8s.client-configs."${vm_name_to_hostname vm_name}" }/${vm_name_to_hostname vm_name}.kubeconfig"; - name = "kubeconfig"; + name = "kubelet.kubeconfig"; + owner = 10024; + group = 10024; + mode = "0600"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.client-configs.kube-proxy}/kube-proxy.kubeconfig"; owner = 10024; group = 10024; mode = "0600"; diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index aeca401..bdb5d53 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -140,7 +140,9 @@ makeScope newScope ( }; kube-proxy = { config_user = "system:kube-proxy"; - config_server = "https://server.kubernetes.local:6443"; + config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443"; + # config_server = "https://127.0.0.1:6443"; + # config_server = "https://server.kubernetes.local:6443"; }; kube-controller-manager = { config_user = "system:kube-controller-manager"; diff --git a/nix/kubernetes/roles/containerd/default.nix b/nix/kubernetes/roles/containerd/default.nix index 18dac5c..88b1a07 100644 --- a/nix/kubernetes/roles/containerd/default.nix +++ b/nix/kubernetes/roles/containerd/default.nix @@ -19,28 +19,39 @@ config = lib.mkIf config.me.containerd.enable { virtualisation.containerd.enable = true; - virtualisation.containerd.settings = { - "plugins" = { - "io.containerd.grpc.v1.cri" = { - "cni" = { - "bin_dir" = "/opt/cni/bin"; - "conf_dir" = "/etc/cni/net.d"; - }; - "containerd" = { - "default_runtime_name" = "runc"; - "runtimes" = { - "runc" = { - "options" = { - "SystemdCgroup" = true; - }; - "runtime_type" = "io.containerd.runc.v2"; - }; + virtualisation.containerd.settings = + # let + # my-cni-plugins = pkgs.buildEnv { + # name = "my-cni-plugins"; + # paths = with pkgs; [ + # cni-plugins + # cni-plugin-flannel + # ]; + # }; + # in + { + "plugins" = { + "io.containerd.grpc.v1.cri" = { + "cni" = { + "bin_dir" = "/opt/cni/bin"; + # "bin_dir" = "${my-cni-plugins}/bin"; + "conf_dir" = "/etc/cni/net.d"; + }; + "containerd" = { + "default_runtime_name" = "runc"; + "runtimes" = { + "runc" = { + "options" = { + "SystemdCgroup" = true; + }; + "runtime_type" = "io.containerd.runc.v2"; + }; + }; + "snapshotter" = "overlayfs"; }; - "snapshotter" = "overlayfs"; }; }; + "version" = 2; }; - "version" = 2; - }; }; } diff --git a/nix/kubernetes/roles/control_plane/default.nix b/nix/kubernetes/roles/control_plane/default.nix index 7039982..ec8a760 100644 --- a/nix/kubernetes/roles/control_plane/default.nix +++ b/nix/kubernetes/roles/control_plane/default.nix @@ -18,6 +18,7 @@ }; config = lib.mkIf config.me.control_plane.enable { + me.firewall.enable = true; me.kube_apiserver.enable = true; me.kube_controller_manager.enable = true; me.kube_scheduler.enable = true; diff --git a/nix/kubernetes/roles/firewall/default.nix b/nix/kubernetes/roles/firewall/default.nix new file mode 100644 index 0000000..a5dafab --- /dev/null +++ b/nix/kubernetes/roles/firewall/default.nix @@ -0,0 +1,36 @@ +{ + config, + lib, + ... +}: + +{ + imports = [ ]; + + options.me = { + firewall.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install firewall."; + }; + }; + + config = lib.mkIf config.me.firewall.enable { + # kernel modules and settings required by Kubernetes + boot.kernelModules = [ + "overlay" + "br_netfilter" + ]; + boot.kernel.sysctl = { + "net.bridge.bridge-nf-call-iptables" = 1; + "net.bridge.bridge-nf-call-ip6tables" = 1; + "net.ipv4.ip_forward" = 1; + }; + + networking.nftables.enable = true; + # We want to filter forwarded traffic. + # Also needed for `networking.firewall.extraForwardRules` to do anything. + networking.firewall.filterForward = true; + }; +} diff --git a/nix/kubernetes/roles/kube_proxy/default.nix b/nix/kubernetes/roles/kube_proxy/default.nix new file mode 100644 index 0000000..3a13fad --- /dev/null +++ b/nix/kubernetes/roles/kube_proxy/default.nix @@ -0,0 +1,68 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + # shellCommand = cmd: (lib.concatMapStringsSep " " lib.strings.escapeShellArg cmd); + shellCommand = cmd: (builtins.concatStringsSep " " cmd); + settingsFormat = pkgs.formats.yaml { }; + config_file = settingsFormat.generate "kube-proxy-config.yaml" config.me.kube-proxy.settings; +in +{ + imports = [ ]; + + options.me = { + kube-proxy.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install kube-proxy."; + }; + + kube-proxy.settings = lib.mkOption { + type = settingsFormat.type; + default = { + kind = "KubeProxyConfiguration"; + apiVersion = "kubeproxy.config.k8s.io/v1alpha1"; + clientConnection = { + kubeconfig = "/.persist/keys/kube/kube-proxy.kubeconfig"; + }; + mode = "iptables"; + # clusterCIDR = "10.200.0.0/16"; + # clusterCIDR = "2620:11f:7001:7:ffff:ffff:0ac8:0000/16"; + clusterCIDR = "fd49:0595:2bba::/48"; + }; + description = '' + kubelet-config.yaml + ''; + }; + }; + + config = lib.mkIf config.me.kube-proxy.enable { + systemd.services.kube-proxy = { + enable = true; + description = "Kubernetes Kube Proxy"; + documentation = [ "https://github.com/kubernetes/kubernetes" ]; + wantedBy = [ "kubernetes.target" ]; + path = with pkgs; [ + iptables + ]; + unitConfig.DefaultDependencies = "no"; + serviceConfig = { + ExecStart = ( + shellCommand [ + "${pkgs.kubernetes}/bin/kube-proxy" + "--config=${config_file}" + "--nodeport-addresses=primary" + "--proxy-mode=nftables" + ] + ); + Restart = "on-failure"; + RestartSec = 5; + }; + }; + }; +} diff --git a/nix/kubernetes/roles/kubelet/default.nix b/nix/kubernetes/roles/kubelet/default.nix index e3dc475..76ab5a1 100644 --- a/nix/kubernetes/roles/kubelet/default.nix +++ b/nix/kubernetes/roles/kubelet/default.nix @@ -8,8 +8,6 @@ let # shellCommand = cmd: (lib.concatMapStringsSep " " lib.strings.escapeShellArg cmd); shellCommand = cmd: (builtins.concatStringsSep " " cmd); - settingsFormat = pkgs.formats.yaml { }; - config_file = settingsFormat.generate "kubelet-config.yaml" config.me.kubelet.settings; in { imports = [ ]; @@ -21,46 +19,6 @@ in example = true; description = "Whether we want to install kubelet."; }; - - kubelet.settings = lib.mkOption { - type = settingsFormat.type; - default = { - kind = "KubeletConfiguration"; - apiVersion = "kubelet.config.k8s.io/v1beta1"; - address = "0.0.0.0"; - authentication = { - anonymous = { - enabled = false; - }; - webhook = { - enabled = true; - }; - x509 = { - clientCAFile = "/var/lib/kubelet/ca.crt"; - }; - }; - authorization = { - mode = "Webhook"; - }; - cgroupDriver = "systemd"; - containerRuntimeEndpoint = "unix:///var/run/containerd/containerd.sock"; - enableServer = true; - failSwapOn = false; - maxPods = 16; - memorySwap = { - swapBehavior = "NoSwap"; - }; - port = 10250; - resolvConf = "/etc/resolv.conf"; - registerNode = true; - runtimeRequestTimeout = "15m"; - tlsCertFile = "/var/lib/kubelet/kubelet.crt"; - tlsPrivateKeyFile = "/var/lib/kubelet/kubelet.key"; - }; - description = '' - kubelet-config.yaml - ''; - }; }; config = lib.mkIf config.me.kubelet.enable { @@ -79,9 +37,8 @@ in ExecStart = ( shellCommand [ "${pkgs.kubernetes}/bin/kubelet" - # "--config=${config_file}" "--config=${./files/kubelet-config.yaml}" - "--kubeconfig=/.persist/keys/kube/kubeconfig" + "--kubeconfig=/.persist/keys/kube/kubelet.kubeconfig" "--v=2" ] ); diff --git a/nix/kubernetes/roles/kubernetes/default.nix b/nix/kubernetes/roles/kubernetes/default.nix index 9d3646d..f4d70f1 100644 --- a/nix/kubernetes/roles/kubernetes/default.nix +++ b/nix/kubernetes/roles/kubernetes/default.nix @@ -18,6 +18,14 @@ }; config = lib.mkIf config.me.kubernetes.enable { + assertions = [ + { + # Kubernetes should only upgrade 1 minor version at a time, so this assert is here to prevent unwittingly jumping versions. + assertion = lib.hasPrefix "1.35." pkgs.kubernetes.version; + message = "Unexpected Kubernetes package version: ${pkgs.kubernetes.version}"; + } + ]; + environment.systemPackages = with pkgs; [ kubernetes ]; diff --git a/nix/kubernetes/roles/worker_node/default.nix b/nix/kubernetes/roles/worker_node/default.nix index af08275..905b036 100644 --- a/nix/kubernetes/roles/worker_node/default.nix +++ b/nix/kubernetes/roles/worker_node/default.nix @@ -19,6 +19,8 @@ config = lib.mkIf config.me.worker_node.enable { me.containerd.enable = true; + me.firewall.enable = true; + me.kube-proxy.enable = true; me.kubelet.enable = true; me.kubernetes.enable = true; }; From 14787d6730b7137452a4a453d2cf61a0efc9c25e Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Thu, 18 Dec 2025 00:27:18 -0500 Subject: [PATCH 15/81] Installing the cni plugins. --- nix/kubernetes/roles/containerd/default.nix | 25 ++++++++--------- .../package/cni_conf/files/10-bridge.conf | 15 +++++++++++ .../package/cni_conf/files/99-loopback.conf | 5 ++++ .../containerd/package/cni_conf/package.nix | 27 +++++++++++++++++++ 4 files changed, 60 insertions(+), 12 deletions(-) create mode 100644 nix/kubernetes/roles/containerd/package/cni_conf/files/10-bridge.conf create mode 100644 nix/kubernetes/roles/containerd/package/cni_conf/files/99-loopback.conf create mode 100644 nix/kubernetes/roles/containerd/package/cni_conf/package.nix diff --git a/nix/kubernetes/roles/containerd/default.nix b/nix/kubernetes/roles/containerd/default.nix index 88b1a07..e2c8d68 100644 --- a/nix/kubernetes/roles/containerd/default.nix +++ b/nix/kubernetes/roles/containerd/default.nix @@ -20,22 +20,23 @@ config = lib.mkIf config.me.containerd.enable { virtualisation.containerd.enable = true; virtualisation.containerd.settings = - # let - # my-cni-plugins = pkgs.buildEnv { - # name = "my-cni-plugins"; - # paths = with pkgs; [ - # cni-plugins - # cni-plugin-flannel - # ]; - # }; - # in + let + my-cni-plugins = pkgs.buildEnv { + name = "my-cni-plugins"; + paths = with pkgs; [ + cni-plugins + cni-plugin-flannel + ]; + }; + in { "plugins" = { "io.containerd.grpc.v1.cri" = { "cni" = { - "bin_dir" = "/opt/cni/bin"; - # "bin_dir" = "${my-cni-plugins}/bin"; - "conf_dir" = "/etc/cni/net.d"; + # "bin_dir" = "/opt/cni/bin"; + "bin_dir" = "${my-cni-plugins}/bin"; + # "conf_dir" = "/etc/cni/net.d"; + "conf_dir" = "${pkgs.callPackage ./package/cni_conf/package.nix { }}"; }; "containerd" = { "default_runtime_name" = "runc"; diff --git a/nix/kubernetes/roles/containerd/package/cni_conf/files/10-bridge.conf b/nix/kubernetes/roles/containerd/package/cni_conf/files/10-bridge.conf new file mode 100644 index 0000000..e9a3bff --- /dev/null +++ b/nix/kubernetes/roles/containerd/package/cni_conf/files/10-bridge.conf @@ -0,0 +1,15 @@ +{ + "cniVersion": "1.0.0", + "name": "bridge", + "type": "bridge", + "bridge": "cni0", + "isGateway": true, + "ipMasq": true, + "ipam": { + "type": "host-local", + "ranges": [ + [{"subnet": "SUBNET"}] + ], + "routes": [{"dst": "0.0.0.0/0"}] + } +} \ No newline at end of file diff --git a/nix/kubernetes/roles/containerd/package/cni_conf/files/99-loopback.conf b/nix/kubernetes/roles/containerd/package/cni_conf/files/99-loopback.conf new file mode 100644 index 0000000..98d6dc1 --- /dev/null +++ b/nix/kubernetes/roles/containerd/package/cni_conf/files/99-loopback.conf @@ -0,0 +1,5 @@ +{ + "cniVersion": "1.1.0", + "name": "lo", + "type": "loopback" +} \ No newline at end of file diff --git a/nix/kubernetes/roles/containerd/package/cni_conf/package.nix b/nix/kubernetes/roles/containerd/package/cni_conf/package.nix new file mode 100644 index 0000000..6476c94 --- /dev/null +++ b/nix/kubernetes/roles/containerd/package/cni_conf/package.nix @@ -0,0 +1,27 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + stdenv, + openssl, + ... +}: +stdenv.mkDerivation (finalAttrs: { + name = "cni-conf"; + nativeBuildInputs = [ openssl ]; + buildInputs = [ ]; + + unpackPhase = "true"; + + installPhase = '' + mkdir -p "$out" + cd "$out" + install ${./files/10-bridge.conf} ${./files/99-loopback.conf} $out/ + ''; +}) From 10cb394ddec8ec7490f3a6daf91e8d1bd438eceb Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Tue, 16 Dec 2025 21:48:44 -0500 Subject: [PATCH 16/81] Add cilium bootstrap. --- nix/kubernetes/README.org | 8 +- .../cilium/cilium-agent/clusterrole.yaml | 112 ++++ .../cilium-agent/clusterrolebinding.yaml | 16 + .../cilium/cilium-agent/daemonset.yaml | 522 ++++++++++++++++++ .../bootstrap/cilium/cilium-agent/role.yaml | 36 ++ .../cilium/cilium-agent/rolebinding.yaml | 34 ++ .../cilium/cilium-agent/serviceaccount.yaml | 7 + .../bootstrap/cilium/cilium-ca-secret.yaml | 10 + .../bootstrap/cilium/cilium-configmap.yaml | 259 +++++++++ .../cilium/cilium-envoy/configmap.yaml | 11 + .../cilium/cilium-envoy/daemonset.yaml | 171 ++++++ .../cilium/cilium-envoy/service.yaml | 25 + .../cilium/cilium-envoy/serviceaccount.yaml | 7 + .../cilium/cilium-operator/clusterrole.yaml | 240 ++++++++ .../cilium-operator/clusterrolebinding.yaml | 16 + .../cilium/cilium-operator/deployment.yaml | 140 +++++ .../cilium/cilium-operator/role.yaml | 19 + .../cilium/cilium-operator/rolebinding.yaml | 17 + .../cilium-operator/serviceaccount.yaml | 7 + .../cilium/cilium-secrets-namespace.yaml | 9 + .../bootstrap/cilium/hubble/peer-service.yaml | 21 + .../cilium/hubble/tls-helm/server-secret.yaml | 12 + nix/kubernetes/roles/kubelet/default.nix | 6 +- nix/kubernetes/roles/worker_node/default.nix | 2 +- 24 files changed, 1699 insertions(+), 8 deletions(-) create mode 100644 nix/kubernetes/bootstrap/cilium/cilium-agent/clusterrole.yaml create mode 100644 nix/kubernetes/bootstrap/cilium/cilium-agent/clusterrolebinding.yaml create mode 100644 nix/kubernetes/bootstrap/cilium/cilium-agent/daemonset.yaml create mode 100644 nix/kubernetes/bootstrap/cilium/cilium-agent/role.yaml create mode 100644 nix/kubernetes/bootstrap/cilium/cilium-agent/rolebinding.yaml create mode 100644 nix/kubernetes/bootstrap/cilium/cilium-agent/serviceaccount.yaml create mode 100644 nix/kubernetes/bootstrap/cilium/cilium-ca-secret.yaml create mode 100644 nix/kubernetes/bootstrap/cilium/cilium-configmap.yaml create mode 100644 nix/kubernetes/bootstrap/cilium/cilium-envoy/configmap.yaml create mode 100644 nix/kubernetes/bootstrap/cilium/cilium-envoy/daemonset.yaml create mode 100644 nix/kubernetes/bootstrap/cilium/cilium-envoy/service.yaml create mode 100644 nix/kubernetes/bootstrap/cilium/cilium-envoy/serviceaccount.yaml create mode 100644 nix/kubernetes/bootstrap/cilium/cilium-operator/clusterrole.yaml create mode 100644 nix/kubernetes/bootstrap/cilium/cilium-operator/clusterrolebinding.yaml create mode 100644 nix/kubernetes/bootstrap/cilium/cilium-operator/deployment.yaml create mode 100644 nix/kubernetes/bootstrap/cilium/cilium-operator/role.yaml create mode 100644 nix/kubernetes/bootstrap/cilium/cilium-operator/rolebinding.yaml create mode 100644 nix/kubernetes/bootstrap/cilium/cilium-operator/serviceaccount.yaml create mode 100644 nix/kubernetes/bootstrap/cilium/cilium-secrets-namespace.yaml create mode 100644 nix/kubernetes/bootstrap/cilium/hubble/peer-service.yaml create mode 100644 nix/kubernetes/bootstrap/cilium/hubble/tls-helm/server-secret.yaml diff --git a/nix/kubernetes/README.org b/nix/kubernetes/README.org index 5f156a2..60b01f1 100644 --- a/nix/kubernetes/README.org +++ b/nix/kubernetes/README.org @@ -14,11 +14,11 @@ ** Install cilium #+begin_src bash helm repo add cilium https://helm.cilium.io/ - helm install --dry-run cilium cilium/cilium --version 1.18.4 --namespace kube-system \ + helm template --dry-run=server cilium cilium/cilium --version 1.18.4 --namespace kube-system --output-dir cilium \ --set kubeProxyReplacement=true \ - --set k8sServiceHost=${API_SERVER_IP} \ - --set k8sServicePort=${API_SERVER_PORT} - + --set k8sServiceHost="2620:11f:7001:7:ffff:ffff:ad7:1dd" \ + --set k8sServicePort=6443 \ + --set ipv6.enabled=true kubectl -n kube-system exec ds/cilium -- cilium-dbg status --verbose kubectl -n kube-system exec ds/cilium -- cilium-dbg status | grep KubeProxyReplacement diff --git a/nix/kubernetes/bootstrap/cilium/cilium-agent/clusterrole.yaml b/nix/kubernetes/bootstrap/cilium/cilium-agent/clusterrole.yaml new file mode 100644 index 0000000..34f88bb --- /dev/null +++ b/nix/kubernetes/bootstrap/cilium/cilium-agent/clusterrole.yaml @@ -0,0 +1,112 @@ +--- +# Source: cilium/templates/cilium-agent/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + - services + - pods + - endpoints + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch + # This is used when validating policies in preflight. This will need to stay + # until we figure out how to avoid "get" inside the preflight, and then + # should be removed ideally. + - get +- apiGroups: + - cilium.io + resources: + - ciliumloadbalancerippools + - ciliumbgppeeringpolicies + - ciliumbgpnodeconfigs + - ciliumbgpadvertisements + - ciliumbgppeerconfigs + - ciliumclusterwideenvoyconfigs + - ciliumclusterwidenetworkpolicies + - ciliumegressgatewaypolicies + - ciliumendpoints + - ciliumendpointslices + - ciliumenvoyconfigs + - ciliumidentities + - ciliumlocalredirectpolicies + - ciliumnetworkpolicies + - ciliumnodes + - ciliumnodeconfigs + - ciliumcidrgroups + - ciliuml2announcementpolicies + - ciliumpodippools + verbs: + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumidentities + - ciliumendpoints + - ciliumnodes + verbs: + - create +- apiGroups: + - cilium.io + # To synchronize garbage collection of such resources + resources: + - ciliumidentities + verbs: + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints + verbs: + - delete + - get +- apiGroups: + - cilium.io + resources: + - ciliumnodes + - ciliumnodes/status + verbs: + - get + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints/status + - ciliumendpoints + - ciliuml2announcementpolicies/status + - ciliumbgpnodeconfigs/status + verbs: + - patch diff --git a/nix/kubernetes/bootstrap/cilium/cilium-agent/clusterrolebinding.yaml b/nix/kubernetes/bootstrap/cilium/cilium-agent/clusterrolebinding.yaml new file mode 100644 index 0000000..9f906ad --- /dev/null +++ b/nix/kubernetes/bootstrap/cilium/cilium-agent/clusterrolebinding.yaml @@ -0,0 +1,16 @@ +--- +# Source: cilium/templates/cilium-agent/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium +subjects: +- kind: ServiceAccount + name: "cilium" + namespace: kube-system diff --git a/nix/kubernetes/bootstrap/cilium/cilium-agent/daemonset.yaml b/nix/kubernetes/bootstrap/cilium/cilium-agent/daemonset.yaml new file mode 100644 index 0000000..edd9478 --- /dev/null +++ b/nix/kubernetes/bootstrap/cilium/cilium-agent/daemonset.yaml @@ -0,0 +1,522 @@ +--- +# Source: cilium/templates/cilium-agent/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cilium + namespace: kube-system + labels: + k8s-app: cilium + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-agent +spec: + selector: + matchLabels: + k8s-app: cilium + updateStrategy: + rollingUpdate: + maxUnavailable: 2 + type: RollingUpdate + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: cilium-agent + labels: + k8s-app: cilium + app.kubernetes.io/name: cilium-agent + app.kubernetes.io/part-of: cilium + spec: + securityContext: + appArmorProfile: + type: Unconfined + seccompProfile: + type: Unconfined + containers: + - name: cilium-agent + image: "quay.io/cilium/cilium:v1.18.4@sha256:49d87af187eeeb9e9e3ec2bc6bd372261a0b5cb2d845659463ba7cc10fe9e45f" + imagePullPolicy: IfNotPresent + command: + - cilium-agent + args: + - --config-dir=/tmp/cilium/config-map + startupProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9879 + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + failureThreshold: 300 + periodSeconds: 2 + successThreshold: 1 + initialDelaySeconds: 5 + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9879 + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + - name: "require-k8s-connectivity" + value: "false" + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 10 + timeoutSeconds: 5 + readinessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9879 + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 5 + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_CLUSTERMESH_CONFIG + value: /var/lib/cilium/clustermesh/ + - name: GOMEMLIMIT + valueFrom: + resourceFieldRef: + resource: limits.memory + divisor: '1' + - name: KUBERNETES_SERVICE_HOST + value: "2620:11f:7001:7:ffff:ffff:ad7:1dd" + - name: KUBERNETES_SERVICE_PORT + value: "6443" + - name: KUBE_CLIENT_BACKOFF_BASE + value: "1" + - name: KUBE_CLIENT_BACKOFF_DURATION + value: "120" + lifecycle: + postStart: + exec: + command: + - "bash" + - "-c" + - | + set -o errexit + set -o pipefail + set -o nounset + + # When running in AWS ENI mode, it's likely that 'aws-node' has + # had a chance to install SNAT iptables rules. These can result + # in dropped traffic, so we should attempt to remove them. + # We do it using a 'postStart' hook since this may need to run + # for nodes which might have already been init'ed but may still + # have dangling rules. This is safe because there are no + # dependencies on anything that is part of the startup script + # itself, and can be safely run multiple times per node (e.g. in + # case of a restart). + if [[ "$(iptables-save | grep -E -c 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN')" != "0" ]]; + then + echo 'Deleting iptables rules created by the AWS CNI VPC plugin' + iptables-save | grep -E -v 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN' | iptables-restore + fi + echo 'Done!' + + preStop: + exec: + command: + - /cni-uninstall.sh + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - CHOWN + - KILL + - NET_ADMIN + - NET_RAW + - IPC_LOCK + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: envoy-sockets + mountPath: /var/run/cilium/envoy/sockets + readOnly: false + # Unprivileged containers need to mount /proc/sys/net from the host + # to have write access + - mountPath: /host/proc/sys/net + name: host-proc-sys-net + # Unprivileged containers need to mount /proc/sys/kernel from the host + # to have write access + - mountPath: /host/proc/sys/kernel + name: host-proc-sys-kernel + - name: bpf-maps + mountPath: /sys/fs/bpf + # Unprivileged containers can't set mount propagation to bidirectional + # in this case we will mount the bpf fs from an init container that + # is privileged and set the mount propagation from host to container + # in Cilium. + mountPropagation: HostToContainer + - name: cilium-run + mountPath: /var/run/cilium + - name: cilium-netns + mountPath: /var/run/cilium/netns + mountPropagation: HostToContainer + - name: etc-cni-netd + mountPath: /host/etc/cni/net.d + - name: clustermesh-secrets + mountPath: /var/lib/cilium/clustermesh + readOnly: true + # Needed to be able to load kernel modules + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: xtables-lock + mountPath: /run/xtables.lock + - name: hubble-tls + mountPath: /var/lib/cilium/tls/hubble + readOnly: true + - name: tmp + mountPath: /tmp + + initContainers: + - name: config + image: "quay.io/cilium/cilium:v1.18.4@sha256:49d87af187eeeb9e9e3ec2bc6bd372261a0b5cb2d845659463ba7cc10fe9e45f" + imagePullPolicy: IfNotPresent + command: + - cilium-dbg + - build-config + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: KUBERNETES_SERVICE_HOST + value: "2620:11f:7001:7:ffff:ffff:ad7:1dd" + - name: KUBERNETES_SERVICE_PORT + value: "6443" + volumeMounts: + - name: tmp + mountPath: /tmp + terminationMessagePolicy: FallbackToLogsOnError + # Required to mount cgroup2 filesystem on the underlying Kubernetes node. + # We use nsenter command with host's cgroup and mount namespaces enabled. + - name: mount-cgroup + image: "quay.io/cilium/cilium:v1.18.4@sha256:49d87af187eeeb9e9e3ec2bc6bd372261a0b5cb2d845659463ba7cc10fe9e45f" + imagePullPolicy: IfNotPresent + env: + - name: CGROUP_ROOT + value: /run/cilium/cgroupv2 + - name: BIN_PATH + value: /opt/cni/bin + command: + - sh + - -ec + # The statically linked Go program binary is invoked to avoid any + # dependency on utilities like sh and mount that can be missing on certain + # distros installed on the underlying host. Copy the binary to the + # same directory where we install cilium cni plugin so that exec permissions + # are available. + - | + cp /usr/bin/cilium-mount /hostbin/cilium-mount; + nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-mount" $CGROUP_ROOT; + rm /hostbin/cilium-mount + volumeMounts: + - name: hostproc + mountPath: /hostproc + - name: cni-path + mountPath: /hostbin + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL + - name: apply-sysctl-overwrites + image: "quay.io/cilium/cilium:v1.18.4@sha256:49d87af187eeeb9e9e3ec2bc6bd372261a0b5cb2d845659463ba7cc10fe9e45f" + imagePullPolicy: IfNotPresent + env: + - name: BIN_PATH + value: /opt/cni/bin + command: + - sh + - -ec + # The statically linked Go program binary is invoked to avoid any + # dependency on utilities like sh that can be missing on certain + # distros installed on the underlying host. Copy the binary to the + # same directory where we install cilium cni plugin so that exec permissions + # are available. + - | + cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix; + nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix"; + rm /hostbin/cilium-sysctlfix + volumeMounts: + - name: hostproc + mountPath: /hostproc + - name: cni-path + mountPath: /hostbin + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL + # Mount the bpf fs if it is not mounted. We will perform this task + # from a privileged container because the mount propagation bidirectional + # only works from privileged containers. + - name: mount-bpf-fs + image: "quay.io/cilium/cilium:v1.18.4@sha256:49d87af187eeeb9e9e3ec2bc6bd372261a0b5cb2d845659463ba7cc10fe9e45f" + imagePullPolicy: IfNotPresent + args: + - 'mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf' + command: + - /bin/bash + - -c + - -- + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + privileged: true + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf + mountPropagation: Bidirectional + - name: clean-cilium-state + image: "quay.io/cilium/cilium:v1.18.4@sha256:49d87af187eeeb9e9e3ec2bc6bd372261a0b5cb2d845659463ba7cc10fe9e45f" + imagePullPolicy: IfNotPresent + command: + - /init-container.sh + env: + - name: CILIUM_ALL_STATE + valueFrom: + configMapKeyRef: + name: cilium-config + key: clean-cilium-state + optional: true + - name: CILIUM_BPF_STATE + valueFrom: + configMapKeyRef: + name: cilium-config + key: clean-cilium-bpf-state + optional: true + - name: WRITE_CNI_CONF_WHEN_READY + valueFrom: + configMapKeyRef: + name: cilium-config + key: write-cni-conf-when-ready + optional: true + - name: KUBERNETES_SERVICE_HOST + value: "2620:11f:7001:7:ffff:ffff:ad7:1dd" + - name: KUBERNETES_SERVICE_PORT + value: "6443" + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - NET_ADMIN + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + drop: + - ALL + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf + # Required to mount cgroup filesystem from the host to cilium agent pod + - name: cilium-cgroup + mountPath: /run/cilium/cgroupv2 + mountPropagation: HostToContainer + - name: cilium-run + mountPath: /var/run/cilium # wait-for-kube-proxy + # Install the CNI binaries in an InitContainer so we don't have a writable host mount in the agent + - name: install-cni-binaries + image: "quay.io/cilium/cilium:v1.18.4@sha256:49d87af187eeeb9e9e3ec2bc6bd372261a0b5cb2d845659463ba7cc10fe9e45f" + imagePullPolicy: IfNotPresent + command: + - "/install-plugin.sh" + resources: + requests: + cpu: 100m + memory: 10Mi + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: cni-path + mountPath: /host/opt/cni/bin # .Values.cni.install + restartPolicy: Always + priorityClassName: system-node-critical + serviceAccountName: "cilium" + automountServiceAccountToken: true + terminationGracePeriodSeconds: 1 + hostNetwork: true + + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: cilium + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - operator: Exists + volumes: + # For sharing configuration between the "config" initContainer and the agent + - name: tmp + emptyDir: {} + # To keep state between restarts / upgrades + - name: cilium-run + hostPath: + path: /var/run/cilium + type: DirectoryOrCreate + # To exec into pod network namespaces + - name: cilium-netns + hostPath: + path: /var/run/netns + type: DirectoryOrCreate + # To keep state between restarts / upgrades for bpf maps + - name: bpf-maps + hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate + # To mount cgroup2 filesystem on the host or apply sysctlfix + - name: hostproc + hostPath: + path: /proc + type: Directory + # To keep state between restarts / upgrades for cgroup2 filesystem + - name: cilium-cgroup + hostPath: + path: /run/cilium/cgroupv2 + type: DirectoryOrCreate + # To install cilium cni plugin in the host + - name: cni-path + hostPath: + path: /opt/cni/bin + type: DirectoryOrCreate + # To install cilium cni configuration in the host + - name: etc-cni-netd + hostPath: + path: /etc/cni/net.d + type: DirectoryOrCreate + # To be able to load kernel modules + - name: lib-modules + hostPath: + path: /lib/modules + # To access iptables concurrently with other processes (e.g. kube-proxy) + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + # Sharing socket with Cilium Envoy on the same node by using a host path + - name: envoy-sockets + hostPath: + path: "/var/run/cilium/envoy/sockets" + type: DirectoryOrCreate + # To read the clustermesh configuration + - name: clustermesh-secrets + projected: + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + sources: + - secret: + name: cilium-clustermesh + optional: true + # note: items are not explicitly listed here, since the entries of this secret + # depend on the peers configured, and that would cause a restart of all agents + # at every addition/removal. Leaving the field empty makes each secret entry + # to be automatically projected into the volume as a file whose name is the key. + - secret: + name: clustermesh-apiserver-remote-cert + optional: true + items: + - key: tls.key + path: common-etcd-client.key + - key: tls.crt + path: common-etcd-client.crt + - key: ca.crt + path: common-etcd-client-ca.crt + # note: we configure the volume for the kvstoremesh-specific certificate + # regardless of whether KVStoreMesh is enabled or not, so that it can be + # automatically mounted in case KVStoreMesh gets subsequently enabled, + # without requiring an agent restart. + - secret: + name: clustermesh-apiserver-local-cert + optional: true + items: + - key: tls.key + path: local-etcd-client.key + - key: tls.crt + path: local-etcd-client.crt + - key: ca.crt + path: local-etcd-client-ca.crt + - name: host-proc-sys-net + hostPath: + path: /proc/sys/net + type: Directory + - name: host-proc-sys-kernel + hostPath: + path: /proc/sys/kernel + type: Directory + - name: hubble-tls + projected: + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + sources: + - secret: + name: hubble-server-certs + optional: true + items: + - key: tls.crt + path: server.crt + - key: tls.key + path: server.key + - key: ca.crt + path: client-ca.crt diff --git a/nix/kubernetes/bootstrap/cilium/cilium-agent/role.yaml b/nix/kubernetes/bootstrap/cilium/cilium-agent/role.yaml new file mode 100644 index 0000000..9153be3 --- /dev/null +++ b/nix/kubernetes/bootstrap/cilium/cilium-agent/role.yaml @@ -0,0 +1,36 @@ +--- +# Source: cilium/templates/cilium-agent/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cilium-config-agent + namespace: kube-system + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch +--- +# Source: cilium/templates/cilium-agent/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cilium-tlsinterception-secrets + namespace: "cilium-secrets" + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch diff --git a/nix/kubernetes/bootstrap/cilium/cilium-agent/rolebinding.yaml b/nix/kubernetes/bootstrap/cilium/cilium-agent/rolebinding.yaml new file mode 100644 index 0000000..9e206ab --- /dev/null +++ b/nix/kubernetes/bootstrap/cilium/cilium-agent/rolebinding.yaml @@ -0,0 +1,34 @@ +--- +# Source: cilium/templates/cilium-agent/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cilium-config-agent + namespace: kube-system + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-config-agent +subjects: + - kind: ServiceAccount + name: "cilium" + namespace: kube-system +--- +# Source: cilium/templates/cilium-agent/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cilium-tlsinterception-secrets + namespace: "cilium-secrets" + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-tlsinterception-secrets +subjects: +- kind: ServiceAccount + name: "cilium" + namespace: kube-system diff --git a/nix/kubernetes/bootstrap/cilium/cilium-agent/serviceaccount.yaml b/nix/kubernetes/bootstrap/cilium/cilium-agent/serviceaccount.yaml new file mode 100644 index 0000000..4880adb --- /dev/null +++ b/nix/kubernetes/bootstrap/cilium/cilium-agent/serviceaccount.yaml @@ -0,0 +1,7 @@ +--- +# Source: cilium/templates/cilium-agent/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "cilium" + namespace: kube-system diff --git a/nix/kubernetes/bootstrap/cilium/cilium-ca-secret.yaml b/nix/kubernetes/bootstrap/cilium/cilium-ca-secret.yaml new file mode 100644 index 0000000..05b06fa --- /dev/null +++ b/nix/kubernetes/bootstrap/cilium/cilium-ca-secret.yaml @@ -0,0 +1,10 @@ +--- +# Source: cilium/templates/cilium-ca-secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: cilium-ca + namespace: kube-system +data: + ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURFekNDQWZ1Z0F3SUJBZ0lRVFl2T2xxVU5TdVlFcGpzamtlaSsvREFOQmdrcWhraUc5dzBCQVFzRkFEQVUKTVJJd0VBWURWUVFERXdsRGFXeHBkVzBnUTBFd0hoY05NalV4TWpFNU1ERXhPREEwV2hjTk1qZ3hNakU0TURFeApPREEwV2pBVU1SSXdFQVlEVlFRREV3bERhV3hwZFcwZ1EwRXdnZ0VpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElCCkR3QXdnZ0VLQW9JQkFRRFNDeEd3cDhaU3Q1Y29RMjNnZ285ZjYvUUE3cUthcnR3bXhlTHkyRjlTcnU2YWNodTUKdURCWm1memtGeDJhRXp6ek94U0REclAvaFFBQUtiSG9kWkVJQ2JkZnY1bzVqanduY0xaSUFMVlZjMlRBeGsxcAphMHVkQVNkVFByYS8rcFRvbjNpeW9LV0JFc2VqY2FXNU1XckFvc1JhaTlLaHl3MTRxSnlsTC9sdDBxVWorQVNaCkxTTndEeU5CK3RlUFhxc0l1VWRmOHcyNlJHUTVlbTZnblNPYmFYZnU5SUhkRDRZYnhQTW9kbkp6dUl3cUFBS1QKOTFqZVVjYkN0Y1Q0UDRTb3RzM1RteXl0Q2VRd1FRclRSY2tIYSt0RmErbnRhVnhIalExSE1GM0pWOFIrK1MvcQpSellMWElaZjR3d2t5SDVFZVJoSjU3WXpBVG1ibFI3SFZobkRBZ01CQUFHallUQmZNQTRHQTFVZER3RUIvd1FFCkF3SUNwREFkQmdOVkhTVUVGakFVQmdnckJnRUZCUWNEQVFZSUt3WUJCUVVIQXdJd0R3WURWUjBUQVFIL0JBVXcKQXdFQi96QWRCZ05WSFE0RUZnUVV3bGd4ZC9qa0FGMTZ4ak52VUhPdExLSGIvMW93RFFZSktvWklodmNOQVFFTApCUUFEZ2dFQkFFbkFvdTArRU9DK1A5YXNGdVJmNy90cHk3UDZoR09vSzZjbElpaFo0UXdpcU82RGdrQ2Frd2ZQCnVCVURYZXZERCt0M3FTYkxOZ3JyNDdWS3R2Qnc5QzhCR3NFZU1vS0w2RThYUlRZSTk5VzBWUmo3MDZHRWsxd0wKSzZXM0R4V251TXRqMHJEWi90VWpDVzhvdkg4SDQyRDhmRHNsT3dSY0NqZ3pMQk5YTXduOFBLdEN6VVRFQzRJZgpFcEZVTXhKOFg0MW5PQnNFdEdkTExEQWpmcjJ3Z0tNWWpsVldtSkE1ZW40WVhtU2hJTmZJZURheDd2WExwZk9OCnVsMmZVaHYxaWVucDlTVjF4d3V2RGxkcFdOQWF2L0dPTjc3MUN5VTRUMzZRS1JzQkptQ0RrQW1iSXloMEdvNEEKQ3NJZXVwYlJCZEd1ck4rbjZYRWtrMFBrWW9DU1JkST0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + ca.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBMGdzUnNLZkdVcmVYS0VOdDRJS1BYK3YwQU82aW1xN2NKc1hpOHRoZlVxN3VtbkliCnViZ3dXWm44NUJjZG1oTTg4enNVZ3c2ei80VUFBQ214NkhXUkNBbTNYNythT1k0OEozQzJTQUMxVlhOa3dNWk4KYVd0TG5RRW5VejYydi9xVTZKOTRzcUNsZ1JMSG8zR2x1VEZxd0tMRVdvdlNvY3NOZUtpY3BTLzViZEtsSS9nRQptUzBqY0E4alFmclhqMTZyQ0xsSFgvTU51a1JrT1hwdW9KMGptMmwzN3ZTQjNRK0dHOFR6S0haeWM3aU1LZ0FDCmsvZFkzbEhHd3JYRStEK0VxTGJOMDVzc3JRbmtNRUVLMDBYSkIydnJSV3ZwN1dsY1I0ME5SekJkeVZmRWZ2a3YKNmtjMkMxeUdYK01NSk1oK1JIa1lTZWUyTXdFNW01VWV4MVlad3dJREFRQUJBb0lCQUJVK3dHQjZ2OXZGclJqZgpUOHhVZUdCajZJVEs4N3NyUjZKTHJzRVFjMUovZ05hSFhJTGtVZmo3ZmR4V1NEZGtPR3lEdzh5SkhxOWxpbEtrCmo2c0FiK3NudHhxM1hIS2VPVHNRc3FkRmNieW9haW1qaFMzNHNuME96UHI3RGhkaC9GRFdoeW40M01vRVFSbCsKUThjZndVNmx5QVdUNTA5WHowK0QxOVdrMTI1WG44ODZDbkdVZExaZUtZZzNsMUxjTDJLWlJEK2ZSa0VHUUVIeQoyN2JDbmFHd1lzZDByQ1V4N0Z3KzdSZ3RBYzdLSUgyY3lockxTVjFLcjByQUE5RVJRRmNuZG8xdHB5bUpNbTR6CjNsclF2MWpOdGcxVDdTSHBKYXF0K0UxL21ObFpDYjBsSVR6dG5DYWFMQkRCSjNuQkpzS2RDLzBaUFFNVnVyUngKMXRIUEpXVUNnWUVBMCtqMFZ1NzErK0duODdqV0dtWXRIR3FhNW5aR2MyYmRCNnR4YVVUQzJVaGtqOWRISnRUcgp6Ykk2aUFNbTg0NjI3ZkxtVnhGT1lYYXJJMEZsb1I4aHNqYjAzYks1Wmh5Vlc5dTY2aDZhNnAzWFRqcittSEFiCkZJYTNXUXZ6VXdjOVRaSi9RWkRkcTdpcEVEWVhmRXY3Y3EzWk9JKzgwRnRjT0J3bzNaTzRpZmNDZ1lFQS9iNnYKallYTkxCZ3BsNHhZR2M5bldCaUNndFpyOCtRekhrUnNNZkpNMjdhemRCNXJCNXROT0pBOUtSOVNGWFBwb1pLQgo2RmE2ZmsrOHdmOWFxVUxpcEVrRm5kWFByREZtU2pCN0Q2ZWFrbG1hVW1BaFZpNTZ6aUQvWm9tTWtqcmRDWlJSCisrcE1SVGYvQzdGc2NtYytLcXFrbzdOenNuVy9uMkVJeVlwMVc1VUNnWUJJcDhpT3ZlT3M5dE9rNXF6UjVGSjMKT2IzZVlwTUpJaTJLWWFmQzFnYVFoUmVsa2NRZGRrZGJBVTY4TDRoOVhXTXU5bWN2VndtdXRRYzhVVUhOR21WdApPeFo5cExlWVlSaDhwRHZUNWFacjVxNVpialM0ZzBkbHBFTGN6eElnVjQwWE9iRlVBNTFkTVRVV0Q5WEJrak9tClFDRUlHWkE5Ui9XNGJ6ODdxVmhPUlFLQmdRQzFrREl4ZytJeGdRQ1J6ZnhrUzRIWkNZQ3BlaUE0bmJydUYydUwKdWFMQlBGUGY3THdNNzFVcitobXBTUjRFOTdIZXlPUm5pVmRjZGxYTVFwbHVyOHRZWGwvRWRtOW8rTmdHa2ZZYwpWNE5FNFJTSmlQdVJuU1NHUmhvNkZvWWRSRDFSVTIzdVlkSTlYVG9EOWVweFdlcWp3UUtabUJoYlErSGZleUU4CmhQUkFtUUtCZ1FDR2RyY251THVVb2tlNTk5WnhxWE5jUnlXRjJqMDdFWHpDSnlpUG93ejdINzBFV2EyRkQ4VXQKTm53aGw2MFhhclJEeWFwSlVTRFhsOGh3c3lYVFVWYXhoeDBrSHlVcE9iSG55Uk5TRjhnODYveUoyOEF4UE44WApoSnlxaERMVlBWZ09OU0huSTF4YkE5ZU9JQWk3QWhxRnp6QWNpUVJudHFoNXVmYzNwVnFCRUE9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo= diff --git a/nix/kubernetes/bootstrap/cilium/cilium-configmap.yaml b/nix/kubernetes/bootstrap/cilium/cilium-configmap.yaml new file mode 100644 index 0000000..f84524f --- /dev/null +++ b/nix/kubernetes/bootstrap/cilium/cilium-configmap.yaml @@ -0,0 +1,259 @@ +--- +# Source: cilium/templates/cilium-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: cilium-config + namespace: kube-system +data: + + # Identity allocation mode selects how identities are shared between cilium + # nodes by setting how they are stored. The options are "crd", "kvstore" or + # "doublewrite-readkvstore" / "doublewrite-readcrd". + # - "crd" stores identities in kubernetes as CRDs (custom resource definition). + # These can be queried with: + # kubectl get ciliumid + # - "kvstore" stores identities in an etcd kvstore, that is + # configured below. Cilium versions before 1.6 supported only the kvstore + # backend. Upgrades from these older cilium versions should continue using + # the kvstore by commenting out the identity-allocation-mode below, or + # setting it to "kvstore". + # - "doublewrite" modes store identities in both the kvstore and CRDs. This is useful + # for seamless migrations from the kvstore mode to the crd mode. Consult the + # documentation for more information on how to perform the migration. + identity-allocation-mode: crd + + identity-heartbeat-timeout: "30m0s" + identity-gc-interval: "15m0s" + cilium-endpoint-gc-interval: "5m0s" + nodes-gc-interval: "5m0s" + + # If you want to run cilium in debug mode change this value to true + debug: "false" + debug-verbose: "" + metrics-sampling-interval: "5m" + # The agent can be put into the following three policy enforcement modes + # default, always and never. + # https://docs.cilium.io/en/latest/security/policy/intro/#policy-enforcement-modes + enable-policy: "default" + policy-cidr-match-mode: "" + # If you want metrics enabled in cilium-operator, set the port for + # which the Cilium Operator will have their metrics exposed. + # NOTE that this will open the port on the nodes where Cilium operator pod + # is scheduled. + operator-prometheus-serve-addr: ":9963" + enable-metrics: "true" + enable-policy-secrets-sync: "true" + policy-secrets-only-from-secrets-namespace: "true" + policy-secrets-namespace: "cilium-secrets" + + # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 + # address. + enable-ipv4: "true" + + # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6 + # address. + enable-ipv6: "true" + # Users who wish to specify their own custom CNI configuration file must set + # custom-cni-conf to "true", otherwise Cilium may overwrite the configuration. + custom-cni-conf: "false" + enable-bpf-clock-probe: "false" + # If you want cilium monitor to aggregate tracing for packets, set this level + # to "low", "medium", or "maximum". The higher the level, the less packets + # that will be seen in monitor output. + monitor-aggregation: medium + + # The monitor aggregation interval governs the typical time between monitor + # notification events for each allowed connection. + # + # Only effective when monitor aggregation is set to "medium" or higher. + monitor-aggregation-interval: "5s" + + # The monitor aggregation flags determine which TCP flags which, upon the + # first observation, cause monitor notifications to be generated. + # + # Only effective when monitor aggregation is set to "medium" or higher. + monitor-aggregation-flags: all + # Specifies the ratio (0.0-1.0] of total system memory to use for dynamic + # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps. + bpf-map-dynamic-size-ratio: "0.0025" + # bpf-policy-map-max specifies the maximum number of entries in endpoint + # policy map (per endpoint) + bpf-policy-map-max: "16384" + # bpf-policy-stats-map-max specifies the maximum number of entries in global + # policy stats map + bpf-policy-stats-map-max: "65536" + # bpf-lb-map-max specifies the maximum number of entries in bpf lb service, + # backend and affinity maps. + bpf-lb-map-max: "65536" + bpf-lb-external-clusterip: "false" + bpf-lb-source-range-all-types: "false" + bpf-lb-algorithm-annotation: "false" + bpf-lb-mode-annotation: "false" + + bpf-distributed-lru: "false" + bpf-events-drop-enabled: "true" + bpf-events-policy-verdict-enabled: "true" + bpf-events-trace-enabled: "true" + + # Pre-allocation of map entries allows per-packet latency to be reduced, at + # the expense of up-front memory allocation for the entries in the maps. The + # default value below will minimize memory usage in the default installation; + # users who are sensitive to latency may consider setting this to "true". + # + # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore + # this option and behave as though it is set to "true". + # + # If this value is modified, then during the next Cilium startup the restore + # of existing endpoints and tracking of ongoing connections may be disrupted. + # As a result, reply packets may be dropped and the load-balancing decisions + # for established connections may change. + # + # If this option is set to "false" during an upgrade from 1.3 or earlier to + # 1.4 or later, then it may cause one-time disruptions during the upgrade. + preallocate-bpf-maps: "false" + + # Name of the cluster. Only relevant when building a mesh of clusters. + cluster-name: "default" + # Unique ID of the cluster. Must be unique across all conneted clusters and + # in the range of 1 and 255. Only relevant when building a mesh of clusters. + cluster-id: "0" + + # Encapsulation mode for communication between nodes + # Possible values: + # - disabled + # - vxlan (default) + # - geneve + + routing-mode: "tunnel" + tunnel-protocol: "vxlan" + tunnel-source-port-range: "0-0" + service-no-backend-response: "reject" + + + # Enables L7 proxy for L7 policy enforcement and visibility + enable-l7-proxy: "true" + enable-ipv4-masquerade: "true" + enable-ipv4-big-tcp: "false" + enable-ipv6-big-tcp: "false" + enable-ipv6-masquerade: "true" + enable-tcx: "true" + datapath-mode: "veth" + enable-masquerade-to-route-source: "false" + + enable-xt-socket-fallback: "true" + install-no-conntrack-iptables-rules: "false" + iptables-random-fully: "false" + + auto-direct-node-routes: "false" + direct-routing-skip-unreachable: "false" + + + + kube-proxy-replacement: "true" + kube-proxy-replacement-healthz-bind-address: "" + bpf-lb-sock: "false" + nodeport-addresses: "" + enable-health-check-nodeport: "true" + enable-health-check-loadbalancer-ip: "false" + node-port-bind-protection: "true" + enable-auto-protect-node-port-range: "true" + bpf-lb-acceleration: "disabled" + enable-svc-source-range-check: "true" + enable-l2-neigh-discovery: "false" + k8s-require-ipv4-pod-cidr: "false" + k8s-require-ipv6-pod-cidr: "false" + enable-k8s-networkpolicy: "true" + enable-endpoint-lockdown-on-policy-overflow: "false" + # Tell the agent to generate and write a CNI configuration file + write-cni-conf-when-ready: /host/etc/cni/net.d/05-cilium.conflist + cni-exclusive: "true" + cni-log-file: "/var/run/cilium/cilium-cni.log" + enable-endpoint-health-checking: "true" + enable-health-checking: "true" + health-check-icmp-failure-threshold: "3" + enable-well-known-identities: "false" + enable-node-selector-labels: "false" + synchronize-k8s-nodes: "true" + operator-api-serve-addr: "127.0.0.1:9234" + + enable-hubble: "true" + # UNIX domain socket for Hubble server to listen to. + hubble-socket-path: "/var/run/cilium/hubble.sock" + hubble-network-policy-correlation-enabled: "true" + # An additional address for Hubble server to listen to (e.g. ":4244"). + hubble-listen-address: ":4244" + hubble-disable-tls: "false" + hubble-tls-cert-file: /var/lib/cilium/tls/hubble/server.crt + hubble-tls-key-file: /var/lib/cilium/tls/hubble/server.key + hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/client-ca.crt + ipam: "cluster-pool" + ipam-cilium-node-update-rate: "15s" + cluster-pool-ipv4-cidr: "10.0.0.0/8" + cluster-pool-ipv4-mask-size: "24" + cluster-pool-ipv6-cidr: "fd00::/104" + cluster-pool-ipv6-mask-size: "120" + + default-lb-service-ipam: "lbipam" + egress-gateway-reconciliation-trigger-interval: "1s" + enable-vtep: "false" + vtep-endpoint: "" + vtep-cidr: "" + vtep-mask: "" + vtep-mac: "" + procfs: "/host/proc" + bpf-root: "/sys/fs/bpf" + cgroup-root: "/run/cilium/cgroupv2" + + identity-management-mode: "agent" + enable-sctp: "false" + remove-cilium-node-taints: "true" + set-cilium-node-taints: "true" + set-cilium-is-up-condition: "true" + unmanaged-pod-watcher-interval: "15" + # default DNS proxy to transparent mode in non-chaining modes + dnsproxy-enable-transparent-mode: "true" + dnsproxy-socket-linger-timeout: "10" + tofqdns-dns-reject-response-code: "refused" + tofqdns-enable-dns-compression: "true" + tofqdns-endpoint-max-ip-per-hostname: "1000" + tofqdns-idle-connection-grace-period: "0s" + tofqdns-max-deferred-connection-deletes: "10000" + tofqdns-proxy-response-max-delay: "100ms" + tofqdns-preallocate-identities: "true" + agent-not-ready-taint-key: "node.cilium.io/agent-not-ready" + + mesh-auth-enabled: "true" + mesh-auth-queue-size: "1024" + mesh-auth-rotated-identities-queue-size: "1024" + mesh-auth-gc-interval: "5m0s" + + proxy-xff-num-trusted-hops-ingress: "0" + proxy-xff-num-trusted-hops-egress: "0" + proxy-connect-timeout: "2" + proxy-initial-fetch-timeout: "30" + proxy-max-requests-per-connection: "0" + proxy-max-connection-duration-seconds: "0" + proxy-idle-timeout-seconds: "60" + proxy-max-concurrent-retries: "128" + http-retry-count: "3" + http-stream-idle-timeout: "300" + + external-envoy-proxy: "true" + envoy-base-id: "0" + envoy-access-log-buffer-size: "4096" + envoy-keep-cap-netbindservice: "false" + max-connected-clusters: "255" + clustermesh-enable-endpoint-sync: "false" + clustermesh-enable-mcs-api: "false" + policy-default-local-cluster: "false" + + nat-map-stats-entries: "32" + nat-map-stats-interval: "30s" + enable-internal-traffic-policy: "true" + enable-lb-ipam: "true" + enable-non-default-deny-policies: "true" + enable-source-ip-verification: "true" + +# Extra config allows adding arbitrary properties to the cilium config. +# By putting it at the end of the ConfigMap, it's also possible to override existing properties. diff --git a/nix/kubernetes/bootstrap/cilium/cilium-envoy/configmap.yaml b/nix/kubernetes/bootstrap/cilium/cilium-envoy/configmap.yaml new file mode 100644 index 0000000..8cd818b --- /dev/null +++ b/nix/kubernetes/bootstrap/cilium/cilium-envoy/configmap.yaml @@ -0,0 +1,11 @@ +--- +# Source: cilium/templates/cilium-envoy/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: cilium-envoy-config + namespace: kube-system +data: + # Keep the key name as bootstrap-config.json to avoid breaking changes + bootstrap-config.json: | + {"admin":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/admin.sock"}}},"applicationLogConfig":{"logFormat":{"textFormat":"[%Y-%m-%d %T.%e][%t][%l][%n] [%g:%#] %v"}},"bootstrapExtensions":[{"name":"envoy.bootstrap.internal_listener","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener"}}],"dynamicResources":{"cdsConfig":{"apiConfigSource":{"apiType":"GRPC","grpcServices":[{"envoyGrpc":{"clusterName":"xds-grpc-cilium"}}],"setNodeOnFirstMessageOnly":true,"transportApiVersion":"V3"},"initialFetchTimeout":"30s","resourceApiVersion":"V3"},"ldsConfig":{"apiConfigSource":{"apiType":"GRPC","grpcServices":[{"envoyGrpc":{"clusterName":"xds-grpc-cilium"}}],"setNodeOnFirstMessageOnly":true,"transportApiVersion":"V3"},"initialFetchTimeout":"30s","resourceApiVersion":"V3"}},"node":{"cluster":"ingress-cluster","id":"host~127.0.0.1~no-id~localdomain"},"overloadManager":{"resourceMonitors":[{"name":"envoy.resource_monitors.global_downstream_max_connections","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig","max_active_downstream_connections":"50000"}}]},"staticResources":{"clusters":[{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"ingress-cluster","type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"egress-cluster-tls","transportSocket":{"name":"cilium.tls_wrapper","typedConfig":{"@type":"type.googleapis.com/cilium.UpstreamTlsWrapperContext"}},"type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"upstreamHttpProtocolOptions":{},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"egress-cluster","type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"ingress-cluster-tls","transportSocket":{"name":"cilium.tls_wrapper","typedConfig":{"@type":"type.googleapis.com/cilium.UpstreamTlsWrapperContext"}},"type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"upstreamHttpProtocolOptions":{},"useDownstreamProtocolConfig":{}}}},{"connectTimeout":"2s","loadAssignment":{"clusterName":"xds-grpc-cilium","endpoints":[{"lbEndpoints":[{"endpoint":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/xds.sock"}}}}]}]},"name":"xds-grpc-cilium","type":"STATIC","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","explicitHttpConfig":{"http2ProtocolOptions":{}}}}},{"connectTimeout":"2s","loadAssignment":{"clusterName":"/envoy-admin","endpoints":[{"lbEndpoints":[{"endpoint":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/admin.sock"}}}}]}]},"name":"/envoy-admin","type":"STATIC"}],"listeners":[{"additionalAddresses":[{"address":{"socketAddress":{"address":"::","portValue":9964}}}],"address":{"socketAddress":{"address":"0.0.0.0","portValue":9964}},"filterChains":[{"filters":[{"name":"envoy.filters.network.http_connection_manager","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager","httpFilters":[{"name":"envoy.filters.http.router","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"}}],"internalAddressConfig":{"cidrRanges":[{"addressPrefix":"10.0.0.0","prefixLen":8},{"addressPrefix":"172.16.0.0","prefixLen":12},{"addressPrefix":"192.168.0.0","prefixLen":16},{"addressPrefix":"127.0.0.1","prefixLen":32},{"addressPrefix":"::1","prefixLen":128}]},"routeConfig":{"virtualHosts":[{"domains":["*"],"name":"prometheus_metrics_route","routes":[{"match":{"prefix":"/metrics"},"name":"prometheus_metrics_route","route":{"cluster":"/envoy-admin","prefixRewrite":"/stats/prometheus"}}]}]},"statPrefix":"envoy-prometheus-metrics-listener","streamIdleTimeout":"300s"}}]}],"name":"envoy-prometheus-metrics-listener"},{"additionalAddresses":[{"address":{"socketAddress":{"address":"::1","portValue":9878}}}],"address":{"socketAddress":{"address":"127.0.0.1","portValue":9878}},"filterChains":[{"filters":[{"name":"envoy.filters.network.http_connection_manager","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager","httpFilters":[{"name":"envoy.filters.http.router","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"}}],"internalAddressConfig":{"cidrRanges":[{"addressPrefix":"10.0.0.0","prefixLen":8},{"addressPrefix":"172.16.0.0","prefixLen":12},{"addressPrefix":"192.168.0.0","prefixLen":16},{"addressPrefix":"127.0.0.1","prefixLen":32},{"addressPrefix":"::1","prefixLen":128}]},"routeConfig":{"virtual_hosts":[{"domains":["*"],"name":"health","routes":[{"match":{"prefix":"/healthz"},"name":"health","route":{"cluster":"/envoy-admin","prefixRewrite":"/ready"}}]}]},"statPrefix":"envoy-health-listener","streamIdleTimeout":"300s"}}]}],"name":"envoy-health-listener"}]}} diff --git a/nix/kubernetes/bootstrap/cilium/cilium-envoy/daemonset.yaml b/nix/kubernetes/bootstrap/cilium/cilium-envoy/daemonset.yaml new file mode 100644 index 0000000..1a855fe --- /dev/null +++ b/nix/kubernetes/bootstrap/cilium/cilium-envoy/daemonset.yaml @@ -0,0 +1,171 @@ +--- +# Source: cilium/templates/cilium-envoy/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cilium-envoy + namespace: kube-system + labels: + k8s-app: cilium-envoy + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-envoy + name: cilium-envoy +spec: + selector: + matchLabels: + k8s-app: cilium-envoy + updateStrategy: + rollingUpdate: + maxUnavailable: 2 + type: RollingUpdate + template: + metadata: + annotations: + labels: + k8s-app: cilium-envoy + name: cilium-envoy + app.kubernetes.io/name: cilium-envoy + app.kubernetes.io/part-of: cilium + spec: + securityContext: + appArmorProfile: + type: Unconfined + containers: + - name: cilium-envoy + image: "quay.io/cilium/cilium-envoy:v1.34.10-1762597008-ff7ae7d623be00078865cff1b0672cc5d9bfc6d5@sha256:1deb6709afcb5523579bf1abbc3255adf9e354565a88c4a9162c8d9cb1d77ab5" + imagePullPolicy: IfNotPresent + command: + - /usr/bin/cilium-envoy-starter + args: + - '--' + - '-c /var/run/cilium/envoy/bootstrap-config.json' + - '--base-id 0' + - '--log-level info' + startupProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9878 + scheme: HTTP + failureThreshold: 105 + periodSeconds: 2 + successThreshold: 1 + initialDelaySeconds: 5 + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9878 + scheme: HTTP + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 10 + timeoutSeconds: 5 + readinessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9878 + scheme: HTTP + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 5 + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: KUBERNETES_SERVICE_HOST + value: "2620:11f:7001:7:ffff:ffff:ad7:1dd" + - name: KUBERNETES_SERVICE_PORT + value: "6443" + ports: + - name: envoy-metrics + containerPort: 9964 + hostPort: 9964 + protocol: TCP + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - NET_ADMIN + - SYS_ADMIN + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: envoy-sockets + mountPath: /var/run/cilium/envoy/sockets + readOnly: false + - name: envoy-artifacts + mountPath: /var/run/cilium/envoy/artifacts + readOnly: true + - name: envoy-config + mountPath: /var/run/cilium/envoy/ + readOnly: true + - name: bpf-maps + mountPath: /sys/fs/bpf + mountPropagation: HostToContainer + restartPolicy: Always + priorityClassName: system-node-critical + serviceAccountName: "cilium-envoy" + automountServiceAccountToken: true + terminationGracePeriodSeconds: 1 + hostNetwork: true + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cilium.io/no-schedule + operator: NotIn + values: + - "true" + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: cilium + topologyKey: kubernetes.io/hostname + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: cilium-envoy + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - operator: Exists + volumes: + - name: envoy-sockets + hostPath: + path: "/var/run/cilium/envoy/sockets" + type: DirectoryOrCreate + - name: envoy-artifacts + hostPath: + path: "/var/run/cilium/envoy/artifacts" + type: DirectoryOrCreate + - name: envoy-config + configMap: + name: "cilium-envoy-config" + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + items: + - key: bootstrap-config.json + path: bootstrap-config.json + # To keep state between restarts / upgrades + # To keep state between restarts / upgrades for bpf maps + - name: bpf-maps + hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate diff --git a/nix/kubernetes/bootstrap/cilium/cilium-envoy/service.yaml b/nix/kubernetes/bootstrap/cilium/cilium-envoy/service.yaml new file mode 100644 index 0000000..04e9a0f --- /dev/null +++ b/nix/kubernetes/bootstrap/cilium/cilium-envoy/service.yaml @@ -0,0 +1,25 @@ +--- +# Source: cilium/templates/cilium-envoy/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: cilium-envoy + namespace: kube-system + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9964" + labels: + k8s-app: cilium-envoy + app.kubernetes.io/name: cilium-envoy + app.kubernetes.io/part-of: cilium + io.cilium/app: proxy +spec: + clusterIP: None + type: ClusterIP + selector: + k8s-app: cilium-envoy + ports: + - name: envoy-metrics + port: 9964 + protocol: TCP + targetPort: envoy-metrics diff --git a/nix/kubernetes/bootstrap/cilium/cilium-envoy/serviceaccount.yaml b/nix/kubernetes/bootstrap/cilium/cilium-envoy/serviceaccount.yaml new file mode 100644 index 0000000..eda0123 --- /dev/null +++ b/nix/kubernetes/bootstrap/cilium/cilium-envoy/serviceaccount.yaml @@ -0,0 +1,7 @@ +--- +# Source: cilium/templates/cilium-envoy/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "cilium-envoy" + namespace: kube-system diff --git a/nix/kubernetes/bootstrap/cilium/cilium-operator/clusterrole.yaml b/nix/kubernetes/bootstrap/cilium/cilium-operator/clusterrole.yaml new file mode 100644 index 0000000..e3f5294 --- /dev/null +++ b/nix/kubernetes/bootstrap/cilium/cilium-operator/clusterrole.yaml @@ -0,0 +1,240 @@ +--- +# Source: cilium/templates/cilium-operator/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium-operator + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + # to automatically delete [core|kube]dns pods so that are starting to being + # managed by Cilium + - delete +- apiGroups: + - "" + resources: + - configmaps + resourceNames: + - cilium-config + verbs: + # allow patching of the configmap to set annotations + - patch +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + # To remove node taints + - nodes + # To set NetworkUnavailable false on startup + - nodes/status + verbs: + - patch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + # to perform LB IP allocation for BGP + - services/status + verbs: + - update + - patch +- apiGroups: + - "" + resources: + # to check apiserver connectivity + - namespaces + - secrets + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + # to perform the translation of a CNP that contains `ToGroup` to its endpoints + - services + - endpoints + verbs: + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies + - ciliumclusterwidenetworkpolicies + verbs: + # Create auto-generated CNPs and CCNPs from Policies that have 'toGroups' + - create + - update + - deletecollection + # To update the status of the CNPs and CCNPs + - patch + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies/status + - ciliumclusterwidenetworkpolicies/status + verbs: + # Update the auto-generated CNPs and CCNPs status. + - patch + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints + - ciliumidentities + verbs: + # To perform garbage collection of such resources + - delete + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumidentities + verbs: + # To synchronize garbage collection of such resources + - update +- apiGroups: + - cilium.io + resources: + - ciliumnodes + verbs: + - create + - update + - get + - list + - watch + # To perform CiliumNode garbage collector + - delete +- apiGroups: + - cilium.io + resources: + - ciliumnodes/status + verbs: + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpointslices + - ciliumenvoyconfigs + - ciliumbgppeerconfigs + - ciliumbgpadvertisements + - ciliumbgpnodeconfigs + verbs: + - create + - update + - get + - list + - watch + - delete + - patch +- apiGroups: + - cilium.io + resources: + - ciliumbgpclusterconfigs/status + - ciliumbgppeerconfigs/status + verbs: + - update +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - update + resourceNames: + - ciliumloadbalancerippools.cilium.io + - ciliumbgppeeringpolicies.cilium.io + - ciliumbgpclusterconfigs.cilium.io + - ciliumbgppeerconfigs.cilium.io + - ciliumbgpadvertisements.cilium.io + - ciliumbgpnodeconfigs.cilium.io + - ciliumbgpnodeconfigoverrides.cilium.io + - ciliumclusterwideenvoyconfigs.cilium.io + - ciliumclusterwidenetworkpolicies.cilium.io + - ciliumegressgatewaypolicies.cilium.io + - ciliumendpoints.cilium.io + - ciliumendpointslices.cilium.io + - ciliumenvoyconfigs.cilium.io + - ciliumidentities.cilium.io + - ciliumlocalredirectpolicies.cilium.io + - ciliumnetworkpolicies.cilium.io + - ciliumnodes.cilium.io + - ciliumnodeconfigs.cilium.io + - ciliumcidrgroups.cilium.io + - ciliuml2announcementpolicies.cilium.io + - ciliumpodippools.cilium.io + - ciliumgatewayclassconfigs.cilium.io +- apiGroups: + - cilium.io + resources: + - ciliumloadbalancerippools + - ciliumpodippools + - ciliumbgppeeringpolicies + - ciliumbgpclusterconfigs + - ciliumbgpnodeconfigoverrides + - ciliumbgppeerconfigs + verbs: + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumpodippools + verbs: + - create +- apiGroups: + - cilium.io + resources: + - ciliumloadbalancerippools/status + verbs: + - patch +# For cilium-operator running in HA mode. +# +# Cilium operator running in HA mode requires the use of ResourceLock for Leader Election +# between multiple running instances. +# The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less +# common and fewer objects in the cluster watch "all Leases". +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - update diff --git a/nix/kubernetes/bootstrap/cilium/cilium-operator/clusterrolebinding.yaml b/nix/kubernetes/bootstrap/cilium/cilium-operator/clusterrolebinding.yaml new file mode 100644 index 0000000..feb5ff3 --- /dev/null +++ b/nix/kubernetes/bootstrap/cilium/cilium-operator/clusterrolebinding.yaml @@ -0,0 +1,16 @@ +--- +# Source: cilium/templates/cilium-operator/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium-operator + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium-operator +subjects: +- kind: ServiceAccount + name: "cilium-operator" + namespace: kube-system diff --git a/nix/kubernetes/bootstrap/cilium/cilium-operator/deployment.yaml b/nix/kubernetes/bootstrap/cilium/cilium-operator/deployment.yaml new file mode 100644 index 0000000..f2d3d43 --- /dev/null +++ b/nix/kubernetes/bootstrap/cilium/cilium-operator/deployment.yaml @@ -0,0 +1,140 @@ +--- +# Source: cilium/templates/cilium-operator/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cilium-operator + namespace: kube-system + labels: + io.cilium/app: operator + name: cilium-operator + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-operator +spec: + # See docs on ServerCapabilities.LeasesResourceLock in file pkg/k8s/version/version.go + # for more details. + replicas: 2 + selector: + matchLabels: + io.cilium/app: operator + name: cilium-operator + # ensure operator update on single node k8s clusters, by using rolling update with maxUnavailable=100% in case + # of one replica and no user configured Recreate strategy. + # otherwise an update might get stuck due to the default maxUnavailable=50% in combination with the + # podAntiAffinity which prevents deployments of multiple operator replicas on the same node. + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 50% + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/port: "9963" + prometheus.io/scrape: "true" + labels: + io.cilium/app: operator + name: cilium-operator + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-operator + spec: + securityContext: + seccompProfile: + type: RuntimeDefault + containers: + - name: cilium-operator + image: "quay.io/cilium/operator-generic:v1.18.4@sha256:1b22b9ff28affdf574378a70dade4ef835b00b080c2ee2418530809dd62c3012" + imagePullPolicy: IfNotPresent + command: + - cilium-operator-generic + args: + - --config-dir=/tmp/cilium/config-map + - --debug=$(CILIUM_DEBUG) + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_DEBUG + valueFrom: + configMapKeyRef: + key: debug + name: cilium-config + optional: true + - name: KUBERNETES_SERVICE_HOST + value: "2620:11f:7001:7:ffff:ffff:ad7:1dd" + - name: KUBERNETES_SERVICE_PORT + value: "6443" + ports: + - name: prometheus + containerPort: 9963 + hostPort: 9963 + protocol: TCP + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9234 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 3 + readinessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9234 + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 5 + volumeMounts: + - name: cilium-config-path + mountPath: /tmp/cilium/config-map + readOnly: true + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + hostNetwork: true + restartPolicy: Always + priorityClassName: system-cluster-critical + serviceAccountName: "cilium-operator" + automountServiceAccountToken: true + # In HA mode, cilium-operator pods must not be scheduled on the same + # node as they will clash with each other. + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + io.cilium/app: operator + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - key: node-role.kubernetes.io/master + operator: Exists + - key: node.kubernetes.io/not-ready + operator: Exists + - key: node.cloudprovider.kubernetes.io/uninitialized + operator: Exists + - key: node.cilium.io/agent-not-ready + operator: Exists + + volumes: + # To read the configuration from the config map + - name: cilium-config-path + configMap: + name: cilium-config diff --git a/nix/kubernetes/bootstrap/cilium/cilium-operator/role.yaml b/nix/kubernetes/bootstrap/cilium/cilium-operator/role.yaml new file mode 100644 index 0000000..1400c0b --- /dev/null +++ b/nix/kubernetes/bootstrap/cilium/cilium-operator/role.yaml @@ -0,0 +1,19 @@ +--- +# Source: cilium/templates/cilium-operator/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cilium-operator-tlsinterception-secrets + namespace: "cilium-secrets" + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - update + - patch diff --git a/nix/kubernetes/bootstrap/cilium/cilium-operator/rolebinding.yaml b/nix/kubernetes/bootstrap/cilium/cilium-operator/rolebinding.yaml new file mode 100644 index 0000000..ddba34d --- /dev/null +++ b/nix/kubernetes/bootstrap/cilium/cilium-operator/rolebinding.yaml @@ -0,0 +1,17 @@ +--- +# Source: cilium/templates/cilium-operator/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cilium-operator-tlsinterception-secrets + namespace: "cilium-secrets" + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-operator-tlsinterception-secrets +subjects: +- kind: ServiceAccount + name: "cilium-operator" + namespace: kube-system diff --git a/nix/kubernetes/bootstrap/cilium/cilium-operator/serviceaccount.yaml b/nix/kubernetes/bootstrap/cilium/cilium-operator/serviceaccount.yaml new file mode 100644 index 0000000..d313595 --- /dev/null +++ b/nix/kubernetes/bootstrap/cilium/cilium-operator/serviceaccount.yaml @@ -0,0 +1,7 @@ +--- +# Source: cilium/templates/cilium-operator/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "cilium-operator" + namespace: kube-system diff --git a/nix/kubernetes/bootstrap/cilium/cilium-secrets-namespace.yaml b/nix/kubernetes/bootstrap/cilium/cilium-secrets-namespace.yaml new file mode 100644 index 0000000..ceb5245 --- /dev/null +++ b/nix/kubernetes/bootstrap/cilium/cilium-secrets-namespace.yaml @@ -0,0 +1,9 @@ +--- +# Source: cilium/templates/cilium-secrets-namespace.yaml +apiVersion: v1 +kind: Namespace +metadata: + name: "cilium-secrets" + labels: + app.kubernetes.io/part-of: cilium + annotations: diff --git a/nix/kubernetes/bootstrap/cilium/hubble/peer-service.yaml b/nix/kubernetes/bootstrap/cilium/hubble/peer-service.yaml new file mode 100644 index 0000000..4abab3a --- /dev/null +++ b/nix/kubernetes/bootstrap/cilium/hubble/peer-service.yaml @@ -0,0 +1,21 @@ +--- +# Source: cilium/templates/hubble/peer-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: hubble-peer + namespace: kube-system + labels: + k8s-app: cilium + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: hubble-peer + +spec: + selector: + k8s-app: cilium + ports: + - name: peer-service + port: 443 + protocol: TCP + targetPort: 4244 + internalTrafficPolicy: Local diff --git a/nix/kubernetes/bootstrap/cilium/hubble/tls-helm/server-secret.yaml b/nix/kubernetes/bootstrap/cilium/hubble/tls-helm/server-secret.yaml new file mode 100644 index 0000000..02cbf1c --- /dev/null +++ b/nix/kubernetes/bootstrap/cilium/hubble/tls-helm/server-secret.yaml @@ -0,0 +1,12 @@ +--- +# Source: cilium/templates/hubble/tls-helm/server-secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: hubble-server-certs + namespace: kube-system +type: kubernetes.io/tls +data: + ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURFekNDQWZ1Z0F3SUJBZ0lRVFl2T2xxVU5TdVlFcGpzamtlaSsvREFOQmdrcWhraUc5dzBCQVFzRkFEQVUKTVJJd0VBWURWUVFERXdsRGFXeHBkVzBnUTBFd0hoY05NalV4TWpFNU1ERXhPREEwV2hjTk1qZ3hNakU0TURFeApPREEwV2pBVU1SSXdFQVlEVlFRREV3bERhV3hwZFcwZ1EwRXdnZ0VpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElCCkR3QXdnZ0VLQW9JQkFRRFNDeEd3cDhaU3Q1Y29RMjNnZ285ZjYvUUE3cUthcnR3bXhlTHkyRjlTcnU2YWNodTUKdURCWm1memtGeDJhRXp6ek94U0REclAvaFFBQUtiSG9kWkVJQ2JkZnY1bzVqanduY0xaSUFMVlZjMlRBeGsxcAphMHVkQVNkVFByYS8rcFRvbjNpeW9LV0JFc2VqY2FXNU1XckFvc1JhaTlLaHl3MTRxSnlsTC9sdDBxVWorQVNaCkxTTndEeU5CK3RlUFhxc0l1VWRmOHcyNlJHUTVlbTZnblNPYmFYZnU5SUhkRDRZYnhQTW9kbkp6dUl3cUFBS1QKOTFqZVVjYkN0Y1Q0UDRTb3RzM1RteXl0Q2VRd1FRclRSY2tIYSt0RmErbnRhVnhIalExSE1GM0pWOFIrK1MvcQpSellMWElaZjR3d2t5SDVFZVJoSjU3WXpBVG1ibFI3SFZobkRBZ01CQUFHallUQmZNQTRHQTFVZER3RUIvd1FFCkF3SUNwREFkQmdOVkhTVUVGakFVQmdnckJnRUZCUWNEQVFZSUt3WUJCUVVIQXdJd0R3WURWUjBUQVFIL0JBVXcKQXdFQi96QWRCZ05WSFE0RUZnUVV3bGd4ZC9qa0FGMTZ4ak52VUhPdExLSGIvMW93RFFZSktvWklodmNOQVFFTApCUUFEZ2dFQkFFbkFvdTArRU9DK1A5YXNGdVJmNy90cHk3UDZoR09vSzZjbElpaFo0UXdpcU82RGdrQ2Frd2ZQCnVCVURYZXZERCt0M3FTYkxOZ3JyNDdWS3R2Qnc5QzhCR3NFZU1vS0w2RThYUlRZSTk5VzBWUmo3MDZHRWsxd0wKSzZXM0R4V251TXRqMHJEWi90VWpDVzhvdkg4SDQyRDhmRHNsT3dSY0NqZ3pMQk5YTXduOFBLdEN6VVRFQzRJZgpFcEZVTXhKOFg0MW5PQnNFdEdkTExEQWpmcjJ3Z0tNWWpsVldtSkE1ZW40WVhtU2hJTmZJZURheDd2WExwZk9OCnVsMmZVaHYxaWVucDlTVjF4d3V2RGxkcFdOQWF2L0dPTjc3MUN5VTRUMzZRS1JzQkptQ0RrQW1iSXloMEdvNEEKQ3NJZXVwYlJCZEd1ck4rbjZYRWtrMFBrWW9DU1JkST0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURWakNDQWo2Z0F3SUJBZ0lRVVN2aDNTdzFBSXR5MjYvd2JYZ0ZnakFOQmdrcWhraUc5dzBCQVFzRkFEQVUKTVJJd0VBWURWUVFERXdsRGFXeHBkVzBnUTBFd0hoY05NalV4TWpFNU1ERXhPREEwV2hjTk1qWXhNakU1TURFeApPREEwV2pBcU1TZ3dKZ1lEVlFRRERCOHFMbVJsWm1GMWJIUXVhSFZpWW14bExXZHljR011WTJsc2FYVnRMbWx2Ck1JSUJJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdTBPVUd3cU9BN1lWcDE0ZE9GcDYKdjgxaGpPUVNmejl3YXduQUVxSStGQWZ3YXBlbkJ6SlZrbktiNlBYWTUydHNkNU5KTS9YaVFzTWRYYXY2OHJUMwpmRXU4TnlrRVQvWVRIOXpyYkplVndGOVdtTUtKdm9CUjBqVWpLZ3RodmNwK2xPclZoNjJyRHRzbWlIMGtUZTNGCkhYeHlOOTdWMmQxRHNHbGFUc0dVM3B3UzNVNFl3eEJTQWlqOVdhRFprcWJYREZxSUJNQXpjQ1owcWRMTTlXR0cKS1dIbGxwUGdRblNtRnVxWG1TSXFRYndGem14cEt5OXFmMmlndkIwVkIwdDhvSFdVLzY4ckZKQ3d1Q0didncwUApORGpWeVdWdmJoWHZZM0tzWjJCQk45VHdzWkxkWWU4Nkk3cWhyWHVXZWVrN0tCeUpYUkJTQ1BOUnhLZHArMmpxCjl3SURBUUFCbzRHTk1JR0tNQTRHQTFVZER3RUIvd1FFQXdJRm9EQWRCZ05WSFNVRUZqQVVCZ2dyQmdFRkJRY0QKQVFZSUt3WUJCUVVIQXdJd0RBWURWUjBUQVFIL0JBSXdBREFmQmdOVkhTTUVHREFXZ0JUQ1dERjMrT1FBWFhyRwpNMjlRYzYwc29kdi9XakFxQmdOVkhSRUVJekFoZ2g4cUxtUmxabUYxYkhRdWFIVmlZbXhsTFdkeWNHTXVZMmxzCmFYVnRMbWx2TUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFCa3A5a3pwcU1qQ1NxSlRDT3dyZFFMemdRdDlyYmkKSTZUTlFqSVh4T0hjc2JtNFhHNU5LMlBpVS8yQkphamtHMW1NTzVoYkw2aDJzcDM4UklBUTE3Yzg2VWpEM0cvWQppVmFzamhISTBuY2IrMXdvZ3ArYVc3TGYwbGpDMHJQSUZBNjNieGMyYmRzWnkwZmJnUEkxNlBLSjVYWU9SZ0FrCnJnUnZOaTdMQWFlelBNSGUrT0p4WnBLSmNnVWhFQW1aMjFJdVQ3Mm9paWk2NGQ5WVRqTnVvRllBOHg2cjlNSmwKUXNKenBidTAyR0VoZnZjNEwzTzUxOUp2b1VQV0JITHg1VXNqd2tyc05nU3krcDJ2dDQ4Sm0vVUZLa3NyRkNSSgpudTlsa2xTWjZUTnczMjlTQ0lqd3BUVXREZ0QzaHpWRk9oOWkzTFI2UkdrbUdIeFQvME9iMXl3VwotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBdTBPVUd3cU9BN1lWcDE0ZE9GcDZ2ODFoak9RU2Z6OXdhd25BRXFJK0ZBZndhcGVuCkJ6SlZrbktiNlBYWTUydHNkNU5KTS9YaVFzTWRYYXY2OHJUM2ZFdThOeWtFVC9ZVEg5enJiSmVWd0Y5V21NS0oKdm9CUjBqVWpLZ3RodmNwK2xPclZoNjJyRHRzbWlIMGtUZTNGSFh4eU45N1YyZDFEc0dsYVRzR1UzcHdTM1U0WQp3eEJTQWlqOVdhRFprcWJYREZxSUJNQXpjQ1owcWRMTTlXR0dLV0hsbHBQZ1FuU21GdXFYbVNJcVFid0Z6bXhwCkt5OXFmMmlndkIwVkIwdDhvSFdVLzY4ckZKQ3d1Q0didncwUE5EalZ5V1Z2YmhYdlkzS3NaMkJCTjlUd3NaTGQKWWU4Nkk3cWhyWHVXZWVrN0tCeUpYUkJTQ1BOUnhLZHArMmpxOXdJREFRQUJBb0lCQUFQWnJseGwrYWlLNEdRYQpFOGk2VjRiRkRzbnFVSlVNTldBR0NjeDRTSVY1NmxBT3dURHRNSVE3MWdmU3NqeUQ0VzJOK2pYV3l1bXJnRm1TCk5TMHpXbGxVWjFETkI1a3JTcFpmb1BtUkhsT2Z3alNPMzIwcXowUGVzdFBrK3dQL2UxM05mRWwzWVNxRGlBWEwKYU9rdlI5dkUyTVpjVHhKVk1kU1pQZkdibWxkL1hmN1ppeEpldzhxSERUN2xGQ083ZjU5S3hSaUpLRmlYYjJmcApXRTBYaGsvc3I2L3FwM0E5TEk1VlJxbWVkUTZXZkpScVREWjlzTlNweXMyaWNnMlBXUkZxWXo3djRyTWoyZUpKCjN3Unh6UmIzQkw5NlR6SlpaNFpCck93ZHhna1gxcHhqbmdBZjlvVzJFdnhuZFhiemlZdkh6b0R1NVlZOWF0Y1oKTW9qWlR4RUNnWUVBNjNxVGJwWEUwRWFrM0FDeVBnRXlYcFZQT2dGV2lKZloreTJNVXhrbU13NXZOZVFYN1BBeAp3SzJNY3RVQnpmSzRCY21hVjhneEtWaDF0ckl0K3l3N1laYjQzckdhSFcwMm9abzhPSjN1ZjVhNGVrTTgwblFaCnFGOWpJR3JveWR1R29oUlpGaXZMSVRLc21yUXRUZi93R1orcm5DWDdnUEk4WkZ6ODdrUlZlTkVDZ1lFQXk1VmEKRXZmWW9lRDQ2cnpsU3NSWUFEOGRlQ0dTV0lrZU51MFRuWmE3UzV4VFhac05PQVViZzVIZG56cW03ZE5zcGVRcQp3OUhGU3orMzFockxpZkdZTS9nMUR2YnpabzluaXcyYXFwanY2Zjd3K2QvUnFkMTRFd1orRlFDYjZmSFlZb1RkClg0ZlhDREN1Sk9acHdxYk85S01SQnVsK0syWWpTK3J2bkhwb0dVY0NnWUJPeVFFZzQ3MG96aW9FYmJzMUlhTmYKbitjVkNqT1FtZlhWb3lTcS9PUVordjljY1lkTHdBVjFhTnR2NURLcFhFY1RzS01TTWFtNUlwNm4xSytydE4zZApSZXpZWFdXcDZNdDVnQ2diN1VOSnJ2emo4ekViWExyazZaMDUwR3M0ZGxseVN3cVBiM0dVRnVkSEQzZUUvRmV4Cnl3bFl3aUNHd1FWUnN3YnhGUzNmc1FLQmdRQ0hpQ0IxTEVjdVRjV3BFS1JkaUNWRjFMN2JzUTNxbnk1Y1JxeWUKWnpDOWY0OEliQ2I2dW51OHRha2pUanEvYmRlUkFUdHZMZ0g0dkt5NGZ4SXQvWWJrWWtPQWVPdEZ4Z2FhemkyLwprUjEzWEdBMUdjcE5HY0hNZUxBTGIySmM5c0tsbytwMHBNeUZDVk9HQVZjN1hTMitNNmtzUHRuZUxwc2srWDBYCllXNVlGUUtCZ1FEcEk5NDcrVEhZTEphdnNxY3g5eXhaWnVVM2t0eGdGT0lwWDdNb0l4N1AwZUw0TEhBNTJmbm4KSkIxeUhoUzk0eTU4VHIrV20vZ2FIM0UvQVl4NjN6V1N3NXNNZkZCVVlDU0NPdlJ5RjUyTVY0aEpxT1orUndacQphcTJVLzN1U1NaTWJsRWs1U1VNdCtKWFFWL3NlZ20zOWM5R2VOM1hCcEF4VDk0TCtLaTZKekE9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo= diff --git a/nix/kubernetes/roles/kubelet/default.nix b/nix/kubernetes/roles/kubelet/default.nix index 76ab5a1..99f4fff 100644 --- a/nix/kubernetes/roles/kubelet/default.nix +++ b/nix/kubernetes/roles/kubelet/default.nix @@ -29,9 +29,9 @@ in wantedBy = [ "kubernetes.target" ]; after = [ "containerd.service" ]; requires = [ "containerd.service" ]; - # path = with pkgs; [ - # zfs - # ]; + path = with pkgs; [ + util-linux + ]; unitConfig.DefaultDependencies = "no"; serviceConfig = { ExecStart = ( diff --git a/nix/kubernetes/roles/worker_node/default.nix b/nix/kubernetes/roles/worker_node/default.nix index 905b036..918e3b8 100644 --- a/nix/kubernetes/roles/worker_node/default.nix +++ b/nix/kubernetes/roles/worker_node/default.nix @@ -20,7 +20,7 @@ config = lib.mkIf config.me.worker_node.enable { me.containerd.enable = true; me.firewall.enable = true; - me.kube-proxy.enable = true; + # me.kube-proxy.enable = true; me.kubelet.enable = true; me.kubernetes.enable = true; }; From e9db45fb6192393f92b4bc576531c947a9721959 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Thu, 18 Dec 2025 22:28:03 -0500 Subject: [PATCH 17/81] Some networking fixes. --- nix/kubernetes/README.org | 4 +- nix/kubernetes/bootstrap/cilium.yaml | 1691 +++++++++++++++++ .../cilium/cilium-agent/clusterrole.yaml | 112 -- .../cilium-agent/clusterrolebinding.yaml | 16 - .../cilium/cilium-agent/daemonset.yaml | 522 ----- .../bootstrap/cilium/cilium-agent/role.yaml | 36 - .../cilium/cilium-agent/rolebinding.yaml | 34 - .../cilium/cilium-agent/serviceaccount.yaml | 7 - .../bootstrap/cilium/cilium-ca-secret.yaml | 10 - .../bootstrap/cilium/cilium-configmap.yaml | 259 --- .../cilium/cilium-envoy/configmap.yaml | 11 - .../cilium/cilium-envoy/daemonset.yaml | 171 -- .../cilium/cilium-envoy/service.yaml | 25 - .../cilium/cilium-envoy/serviceaccount.yaml | 7 - .../cilium/cilium-operator/clusterrole.yaml | 240 --- .../cilium-operator/clusterrolebinding.yaml | 16 - .../cilium/cilium-operator/deployment.yaml | 140 -- .../cilium/cilium-operator/role.yaml | 19 - .../cilium/cilium-operator/rolebinding.yaml | 17 - .../cilium-operator/serviceaccount.yaml | 7 - .../cilium/cilium-secrets-namespace.yaml | 9 - .../bootstrap/cilium/hubble/peer-service.yaml | 21 - .../cilium/hubble/tls-helm/server-secret.yaml | 12 - nix/kubernetes/configuration.nix | 1 + nix/kubernetes/roles/cilium/default.nix | 29 + nix/kubernetes/roles/containerd/default.nix | 65 +- nix/kubernetes/roles/firewall/default.nix | 6 + .../roles/kube_controller_manager/default.nix | 4 + nix/kubernetes/roles/kube_proxy/default.nix | 5 +- .../roles/kube_scheduler/default.nix | 4 + nix/kubernetes/roles/kubelet/default.nix | 4 + nix/kubernetes/roles/network/default.nix | 7 + nix/kubernetes/roles/worker_node/default.nix | 17 + 33 files changed, 1806 insertions(+), 1722 deletions(-) create mode 100644 nix/kubernetes/bootstrap/cilium.yaml delete mode 100644 nix/kubernetes/bootstrap/cilium/cilium-agent/clusterrole.yaml delete mode 100644 nix/kubernetes/bootstrap/cilium/cilium-agent/clusterrolebinding.yaml delete mode 100644 nix/kubernetes/bootstrap/cilium/cilium-agent/daemonset.yaml delete mode 100644 nix/kubernetes/bootstrap/cilium/cilium-agent/role.yaml delete mode 100644 nix/kubernetes/bootstrap/cilium/cilium-agent/rolebinding.yaml delete mode 100644 nix/kubernetes/bootstrap/cilium/cilium-agent/serviceaccount.yaml delete mode 100644 nix/kubernetes/bootstrap/cilium/cilium-ca-secret.yaml delete mode 100644 nix/kubernetes/bootstrap/cilium/cilium-configmap.yaml delete mode 100644 nix/kubernetes/bootstrap/cilium/cilium-envoy/configmap.yaml delete mode 100644 nix/kubernetes/bootstrap/cilium/cilium-envoy/daemonset.yaml delete mode 100644 nix/kubernetes/bootstrap/cilium/cilium-envoy/service.yaml delete mode 100644 nix/kubernetes/bootstrap/cilium/cilium-envoy/serviceaccount.yaml delete mode 100644 nix/kubernetes/bootstrap/cilium/cilium-operator/clusterrole.yaml delete mode 100644 nix/kubernetes/bootstrap/cilium/cilium-operator/clusterrolebinding.yaml delete mode 100644 nix/kubernetes/bootstrap/cilium/cilium-operator/deployment.yaml delete mode 100644 nix/kubernetes/bootstrap/cilium/cilium-operator/role.yaml delete mode 100644 nix/kubernetes/bootstrap/cilium/cilium-operator/rolebinding.yaml delete mode 100644 nix/kubernetes/bootstrap/cilium/cilium-operator/serviceaccount.yaml delete mode 100644 nix/kubernetes/bootstrap/cilium/cilium-secrets-namespace.yaml delete mode 100644 nix/kubernetes/bootstrap/cilium/hubble/peer-service.yaml delete mode 100644 nix/kubernetes/bootstrap/cilium/hubble/tls-helm/server-secret.yaml create mode 100644 nix/kubernetes/roles/cilium/default.nix diff --git a/nix/kubernetes/README.org b/nix/kubernetes/README.org index 60b01f1..8ea141a 100644 --- a/nix/kubernetes/README.org +++ b/nix/kubernetes/README.org @@ -14,7 +14,7 @@ ** Install cilium #+begin_src bash helm repo add cilium https://helm.cilium.io/ - helm template --dry-run=server cilium cilium/cilium --version 1.18.4 --namespace kube-system --output-dir cilium \ + helm template --dry-run=server cilium cilium/cilium --version 1.18.4 --namespace kube-system \ --set kubeProxyReplacement=true \ --set k8sServiceHost="2620:11f:7001:7:ffff:ffff:ad7:1dd" \ --set k8sServicePort=6443 \ @@ -23,4 +23,6 @@ kubectl -n kube-system exec ds/cilium -- cilium-dbg status --verbose kubectl -n kube-system exec ds/cilium -- cilium-dbg status | grep KubeProxyReplacement + # --set hostFirewall.enabled=true + # routingMode=native #+end_src diff --git a/nix/kubernetes/bootstrap/cilium.yaml b/nix/kubernetes/bootstrap/cilium.yaml new file mode 100644 index 0000000..799db4d --- /dev/null +++ b/nix/kubernetes/bootstrap/cilium.yaml @@ -0,0 +1,1691 @@ +--- +# Source: cilium/templates/cilium-secrets-namespace.yaml +apiVersion: v1 +kind: Namespace +metadata: + name: "cilium-secrets" + labels: + app.kubernetes.io/part-of: cilium + annotations: +--- +# Source: cilium/templates/cilium-agent/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "cilium" + namespace: kube-system +--- +# Source: cilium/templates/cilium-envoy/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "cilium-envoy" + namespace: kube-system +--- +# Source: cilium/templates/cilium-operator/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "cilium-operator" + namespace: kube-system +--- +# Source: cilium/templates/cilium-ca-secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: cilium-ca + namespace: kube-system +data: + ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lSQVByWlQ4U29LSnRBS3dLMTdYSXhhK0l3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkxTVRJeE9UQTBNamt3T1ZvWERUSTRNVEl4T0RBMApNamt3T1Zvd0ZERVNNQkFHQTFVRUF4TUpRMmxzYVhWdElFTkJNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFNSUlCQ2dLQ0FRRUFyc2xsWWZQMVN1U0Rtd2crQTg1WGJPWDdJSE5MZGpEKzErUXZrZ2I1dVNickRZbjUKUnNtWXZ1NERVdnRWWXBYc1UvUVU5L1Bmb00xVkNQZFJkZGU5eHlBT01mOGF1bW4vZ3g5S1BYbU1aZk10MzBWTApweWJrbE5lVkw1aGhsWkZrNit6VW1yVU1lWXZ0cjFIRHNlaXlNL2lUT3VQUjlYS3A1UUp1cDRxdHJRaW8zT21lCkw2U3lSZGtoU3BEL3FwVjJYWnI5WEJBMEhSZ082dlY5SHY1V0xBclBUWU9xWFFGaFV2MTZQcE1TY3FkL1hNSDYKcUtrQ0ZJVDJGd1prb3dnS0V5bktCTmFVdjFkMWdsMVduM0tTNG5kdjRDdFY3VVRGbWhUOU51RCtsbWU1VmoyVApzYVlMdmp5WGI0MUdxWVhqRkppWTZvY3FFT2hrdnRhbGNTRXUzUUlEQVFBQm8yRXdYekFPQmdOVkhROEJBZjhFCkJBTUNBcVF3SFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGTGducmxPZGwzZmRWT2dsNkN0NFBha0pjODlWTUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQi9taVdES3RYQWtOU0lXblIwcWRNbkVlR2JaOXd4NHprY3UzajBqSlk4VW1MSnJqVXFTRXQxClh0dEJ2QVZHV3RKKzU0R3I1bWJXTy9oSXE3UkZmY04vYUZoQWZDdGdPZ0Nacm1yejJobkVKK294cWpRM1c2NlkKQXd1TkNNZkJjUmNyeE9Yc3hkNzc2R24rZVVwYUpWTlh6S1JJSXdHWWJYOVE5Tkd3dlhKQjRoVWJXVXZIQTJkTwp4TjFNMmFzOWVxYmVUejlPTjNZdWd6ZnJBckVxbHhuNDIyR1NVOWQ4bktobi9DUnhvK1U0dGVJRXRTUXBaN0hIClhxYjhLZjdOTXJweTVjb3gxOFU5YVovcFBMT0tzckxlQUxYc3c1ejdtT3l2eWhVVE9vbUVlYnR5TU5VRGhCVHkKeUhaMUp2RVNoaVVLRmkvZW5SdGxyVUwxZG1qd1FHZzAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + ca.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb1FJQkFBS0NBUUVBcnNsbFlmUDFTdVNEbXdnK0E4NVhiT1g3SUhOTGRqRCsxK1F2a2diNXVTYnJEWW41ClJzbVl2dTREVXZ0VllwWHNVL1FVOS9QZm9NMVZDUGRSZGRlOXh5QU9NZjhhdW1uL2d4OUtQWG1NWmZNdDMwVkwKcHlia2xOZVZMNWhobFpGazYrelVtclVNZVl2dHIxSERzZWl5TS9pVE91UFI5WEtwNVFKdXA0cXRyUWlvM09tZQpMNlN5UmRraFNwRC9xcFYyWFpyOVhCQTBIUmdPNnZWOUh2NVdMQXJQVFlPcVhRRmhVdjE2UHBNU2NxZC9YTUg2CnFLa0NGSVQyRndaa293Z0tFeW5LQk5hVXYxZDFnbDFXbjNLUzRuZHY0Q3RWN1VURm1oVDlOdUQrbG1lNVZqMlQKc2FZTHZqeVhiNDFHcVlYakZKaVk2b2NxRU9oa3Z0YWxjU0V1M1FJREFRQUJBb0lCQUROWG4zdEZnVEZOck1iMgpoZGp5WE1YMVRHNDh6OWJaRDVMMVNXLzRzcFJMdFZJVnpHNlp1MCtpZE1HRWFKRjcvcDZ1blJZMHdlVG5TK3BGClU5MXMwU2RUNkRlUmV4VjFHNFExeGNZSEZWSEJQQ0ZRcHFka0NsTWFmSTB0a3NTdm45aVppZDRqL3BVWm9Ha1cKZUlpakZPMmxRQUpWZDVCRTdWa0MvT1FrSFY0dEo2cExDck1WNkE0OXVvaWNDMFZOUnRxZHBXc0RxQ3RydDY3dApQRmRNemVZY290TjR6RFg5SHlaeFdnako4QXhPNVMwM3ZRWXVhZCs3YW1QWXAvWGkzaDBKM0xvc25ETmh6QWtxCktRYlVtekZMM3I4TnVUclBQV05rQXB4UEU2aEl5dkd2eWc0UkxhdGUvZDBNbld4NEZKaDJGdG5HYXYzSXkyMHcKS1I1M05wVUNnWUVBMjFvZ1hCZmtCOFBKdC94RkMxYU1oamphWmNZc0gzM2FBQWVvU1k1TlJUeXNXWElpQ1puUgpvTDhJY2NuK1puZlMvMnM4cjYwcTJlcWxMQURRS0JEbkxBU29SYWJ6QlBtTjJoeVZhM2llQTZzT2lNMVBzU0l4CmlLLy81R05vU2ROSVVLd1RMMEhvTGpmZW1uSFZRU0dFanVJbndZblFXVXJDZTBnZW9mRGNyWXNDZ1lFQXkvMHIKOFZ2ZXlMN3FMZzhIeHZBRFoyMmEvNHBqSFM5WFZoY1Z5UGw2WWRDTTVkbnpQdE5RQmxHS1MvNStxbThZYUFGcAp0YWo4Y29Zci9FRzFnN0hWU0JIVXJER2RoNEJZYUtjalZ2K294TThJYVdVcW1vRytZWFpVeTEvTDJJQ2diRkMyCm92YzBsSVpZVlY1KzBoRWRRUHpOQVhSYUNkVkZVYTQwSHJOUWNqY0NnWUVBdGJMSFlXSE0ydjI4aDNJcGcwWFQKSzQvR29VMXgyVGczcjVTb094M3NOZkVYVnhScXk0Y21GVUNJN3pFQ1VjRHAvL2ZGa2tRZ1hkaTNyMmFSQnM5UgpucTNROGJsa1R2ZWxJRXBUSDI0UGZqeGcwc00zUXlSdTRhbmdqMEFrOEJwYktiQVFTRElROE5BSXplaURYT3MxCjBlSFdTc1FzMlBwTGRURjlabW1OY3hjQ2YzNnh2NWtvZGpZNFhUbytHNVp6ZTZuQmFZTk5jRUd3Slk1U3lEVUsKdmZnQ2o0QXkzZUVuWFl5UEdGWWF1cEVnYndncDJIMzdkcytxaGFjbktKV2kzZFRsUGpHMG55S0psRjlyYjRlWgpRamo2WFQ4WWxsMWdOOTVRb0Ztd0pldFIrRkZOMENGbmFFRFhNQVhWcXU3TnNsMnFwYWxCbXFxQXk0enZwVzlyCjc0c0NnWUFJcHgzdWw4aDF1bmhtQStKQnl6d2xHYnE1VzVuZFErUFFnZGZoUmpZeW05Qy9xaXFuVGJyeUN5QzUKZkNRcWpHRUwxRXloUFAwaDljMEdZc3U5VHdtUmh2N2x2eUlRZlkrMnl2U2FtZDc4UWpHTjZUSklnRnlwTWtuRwpmMU1ZT01Td1M1ckhuZzdxcGtuTW1CTjI5SHAyWnFIRWhoNWYyRTVlbUZub2puT2drQT09Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== +--- +# Source: cilium/templates/hubble/tls-helm/server-secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: hubble-server-certs + namespace: kube-system +type: kubernetes.io/tls +data: + ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lSQVByWlQ4U29LSnRBS3dLMTdYSXhhK0l3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkxTVRJeE9UQTBNamt3T1ZvWERUSTRNVEl4T0RBMApNamt3T1Zvd0ZERVNNQkFHQTFVRUF4TUpRMmxzYVhWdElFTkJNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFNSUlCQ2dLQ0FRRUFyc2xsWWZQMVN1U0Rtd2crQTg1WGJPWDdJSE5MZGpEKzErUXZrZ2I1dVNickRZbjUKUnNtWXZ1NERVdnRWWXBYc1UvUVU5L1Bmb00xVkNQZFJkZGU5eHlBT01mOGF1bW4vZ3g5S1BYbU1aZk10MzBWTApweWJrbE5lVkw1aGhsWkZrNit6VW1yVU1lWXZ0cjFIRHNlaXlNL2lUT3VQUjlYS3A1UUp1cDRxdHJRaW8zT21lCkw2U3lSZGtoU3BEL3FwVjJYWnI5WEJBMEhSZ082dlY5SHY1V0xBclBUWU9xWFFGaFV2MTZQcE1TY3FkL1hNSDYKcUtrQ0ZJVDJGd1prb3dnS0V5bktCTmFVdjFkMWdsMVduM0tTNG5kdjRDdFY3VVRGbWhUOU51RCtsbWU1VmoyVApzYVlMdmp5WGI0MUdxWVhqRkppWTZvY3FFT2hrdnRhbGNTRXUzUUlEQVFBQm8yRXdYekFPQmdOVkhROEJBZjhFCkJBTUNBcVF3SFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGTGducmxPZGwzZmRWT2dsNkN0NFBha0pjODlWTUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQi9taVdES3RYQWtOU0lXblIwcWRNbkVlR2JaOXd4NHprY3UzajBqSlk4VW1MSnJqVXFTRXQxClh0dEJ2QVZHV3RKKzU0R3I1bWJXTy9oSXE3UkZmY04vYUZoQWZDdGdPZ0Nacm1yejJobkVKK294cWpRM1c2NlkKQXd1TkNNZkJjUmNyeE9Yc3hkNzc2R24rZVVwYUpWTlh6S1JJSXdHWWJYOVE5Tkd3dlhKQjRoVWJXVXZIQTJkTwp4TjFNMmFzOWVxYmVUejlPTjNZdWd6ZnJBckVxbHhuNDIyR1NVOWQ4bktobi9DUnhvK1U0dGVJRXRTUXBaN0hIClhxYjhLZjdOTXJweTVjb3gxOFU5YVovcFBMT0tzckxlQUxYc3c1ejdtT3l2eWhVVE9vbUVlYnR5TU5VRGhCVHkKeUhaMUp2RVNoaVVLRmkvZW5SdGxyVUwxZG1qd1FHZzAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURWakNDQWo2Z0F3SUJBZ0lRUE41WFQxYWZ6VGtwZUtnMXFkZXZEREFOQmdrcWhraUc5dzBCQVFzRkFEQVUKTVJJd0VBWURWUVFERXdsRGFXeHBkVzBnUTBFd0hoY05NalV4TWpFNU1EUXlPVEE1V2hjTk1qWXhNakU1TURReQpPVEE1V2pBcU1TZ3dKZ1lEVlFRRERCOHFMbVJsWm1GMWJIUXVhSFZpWW14bExXZHljR011WTJsc2FYVnRMbWx2Ck1JSUJJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBNk55UDV4VnVsT3l4UitURkU2cXEKTGtvbHgydXBZa1JMMC9zN1RwTnVLa2pqSXF5cWd3dUVEOW1ndWdkeUp0VDdOaXkxM3lOU04wSWJTYkRRU0ZpeQpmbStmVnR3eGllUGRCVmorNXV4QWFWc1RURTI5SThYbVpHRG5ORHpRN0xNWGNiTkFJM1NzSVpoMm5kdTBWaWtPCnVTQ2gzd0RZMWgrd2szeGlhc2oxY2tiVUFHRnBGajRhRDNqUDNVWEw0Z2ZLOWhQN00rb2tTbWlzZnZqa01QakwKa2hteFkvdDFzc1o1WXpmMjFsZ2dBbDdldExKN0tNRlN5WkYwRkdHTmlzNnJGYzF1dm5vdDNDUVo2Sk5URWpHMQprakJxZVo2MHAyakh1dGpTdWpVZUJSV1N5Sk9zM2NLVDI3QmVEZmVTQkNmUktpYlQvaWwzdXZENmZvTStOSjJtCk1RSURBUUFCbzRHTk1JR0tNQTRHQTFVZER3RUIvd1FFQXdJRm9EQWRCZ05WSFNVRUZqQVVCZ2dyQmdFRkJRY0QKQVFZSUt3WUJCUVVIQXdJd0RBWURWUjBUQVFIL0JBSXdBREFmQmdOVkhTTUVHREFXZ0JTNEo2NVRuWmQzM1ZUbwpKZWdyZUQycENYUFBWVEFxQmdOVkhSRUVJekFoZ2g4cUxtUmxabUYxYkhRdWFIVmlZbXhsTFdkeWNHTXVZMmxzCmFYVnRMbWx2TUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFDVEVSY09weWpYa3lLVmdHRWs2c25jamRLbVIxRUcKQUk2cWtFZ3VWanN2SUpLcnh0TXN3c0F5Y0NuYWpodDNDQXNaVi9mK1lCQ3c0aGkzeHppNUZHRmN1TDAxL0FHaQp6Njc4dkw2eHZETW1ERG0rL2swbzFNYnZVVDJBRmVWZE1qdndVdFlsSytkd3pxMHl1Sjh6ajBzeEtqa0l0d3JYCjRwRUcrcGhGZjJvYWRybjBXamdhcVRpR2NJNWRlNzMwYStIb0M3WDdneG4xWmd3Um50RXI4V3AvcG1LeUtBRkoKT2ZodkFaLzVhSzd4WFBxZGRDQXF0S2ZHOVpaWkJwZzE3a3o5NnFhMlU2NlN6ZlFsUFNaWmwxWGJnRHVIR0pEawpPUkxuSDFhbmo3ZzNFTGM0K3NhYzNVSUlaTkVkb2cyWTZqM3lWZHJnTytHOW1vQ3pSQTJkK0tiUgotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBNk55UDV4VnVsT3l4UitURkU2cXFMa29seDJ1cFlrUkwwL3M3VHBOdUtrampJcXlxCmd3dUVEOW1ndWdkeUp0VDdOaXkxM3lOU04wSWJTYkRRU0ZpeWZtK2ZWdHd4aWVQZEJWais1dXhBYVZzVFRFMjkKSThYbVpHRG5ORHpRN0xNWGNiTkFJM1NzSVpoMm5kdTBWaWtPdVNDaDN3RFkxaCt3azN4aWFzajFja2JVQUdGcApGajRhRDNqUDNVWEw0Z2ZLOWhQN00rb2tTbWlzZnZqa01QakxraG14WS90MXNzWjVZemYyMWxnZ0FsN2V0TEo3CktNRlN5WkYwRkdHTmlzNnJGYzF1dm5vdDNDUVo2Sk5URWpHMWtqQnFlWjYwcDJqSHV0alN1alVlQlJXU3lKT3MKM2NLVDI3QmVEZmVTQkNmUktpYlQvaWwzdXZENmZvTStOSjJtTVFJREFRQUJBb0lCQUdxaEpSd1NDbE1UbDVKdQpSWGhkL0JPNlRGWm5ER2M5RXVQODlRbGtKVndqaUM3R2xrWHZyZ2Q4QmtkcFJYSldzTG43KzBRMGJsOXpXLytBCmdiKy9MZTBMdkJFUVAzYis2cVNhUGc4TU8rcUNrL0dxMzJOVUtjM3BPYllINnp1WVBySUgvT1MvZGxyajc5MFYKVUJ6c0xCdkZ2NW5ZVDNLdGpEUGREMHdLdmFvWHVSd0NMSTMrbUl4UDJTczAwY1k5azdDUXN3eDVpaXl3NEJSMQpIdllPa2IxN2hYYkJlQS9Ua28zOXVRcnF2bUhpZFZUVWJuaExxN240ZzBGcktoOGRvVkJuMGkzbWxGYkRBTzYvCnlySGdCZ01TcFQ1Z0tQN3dETzJtcis5Zzc3eEhxb0kyNWhUODFJSkc2WnRWSmRSMXYxcXRMNG53TW9RbXBhMnEKOE9qMHhxa0NnWUVBNi8wczYrRWJPY1lFVUwxQVJGMnRYRktQSjBpK1lWZnB6Y1lxKytBbisxRU1KTWN3VmdYSwpDcW9FZ3FCcnpidU84K3J5S1FVc0VITWU2TnBpc21UU2hzS0hJSEQ5eUFuS1JEUEdXSk1JTmxuQjNDQ1F3QmphCkYwL0RIL3Rwdk5SdjhhazJBenFHRU04MmtwYTc2b1R2YTNuL2h6aEl1MjU3dFFzbzVCVjM3QjhDZ1lFQS9KdC8KV1EzTkJValNCNEJtTUVJVHJrOGh4bW5Ubm5Pc0lFOWJjMGpORDlLb1dqOEFWSGZwbUF5QjVEcENLVUcwS21YQwpSWk52QWhzczhSQnFFamdGUDdEWUxCa0l0SWVVQ0V0T2RZSmRDZ1RyVGFCNkUzcWpadjJLRzl1QUtsUWZNdjZ5CisybHhiQnY5TjJ6d1ZsTDhHdlVYb3RZRkVsRXc3U3lsbnNKRkk2OENnWUJsMlo0U3E5UHhTbHBBY24yVFJBNzMKUXBhUU1PQWg2U3g2YUNYVEVYcVFCL25Dd2NsRnh6ZmRCR1h2N3NhNFExTytTRTk5anNMbTBwaXJqTjE5K3UrMQp3NjFJUGh2NXd1ZnVLWjVZU2hVSVNCTFlLNkh0bG94QjlXWFVudENTOCtqZ2x0UTk3bnRBTS81c28rdTBtdTB1CktMZzFKd0NWREdrTnlTUTcxRnd1cHdLQmdDVURsL1p4bWtBUytvdVUyZVJxV2ozcFpIYVJYU3ZUb1Q0ZXdyU3kKenpGcFdiZlhrWEhQSEZCbVRNWW5iemhpWTBidklEZFhzaVZwMi96NnZjQXJxMEU0NVlRWXF2amhBT1R5TyszbQpDMElEbElNTVo3YmcrbjZGMmkzWXpQL20wYWVFUUsrc05LMTFzSFhscFpYUWxSWlNJWU4vdnNCL1d4NzFzdEwvCmZtZ0xBb0dCQUxpY2I0UzQ2bmFKbWYvdXFLV1RMUi91NE5hOWhSR1JzU2Z3a3dTaUxKaGk3T2VmOVZuZk9QUVUKakRaajRHMDRuRUtkdzA1a3o0S0VpVU1aYnE3cnVZS05pRUloS1NSWWF6SU9sMnpjSGNuNGIrNUtOVjBqdmtlTwpIc3Q5YzREZVJxQ2xXNk5rY2JtT0NIQ2RjSkRsRE1YeEU1VzdscitENnMyZ1JLQW9FZ0RPCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== +--- +# Source: cilium/templates/cilium-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: cilium-config + namespace: kube-system +data: + + # Identity allocation mode selects how identities are shared between cilium + # nodes by setting how they are stored. The options are "crd", "kvstore" or + # "doublewrite-readkvstore" / "doublewrite-readcrd". + # - "crd" stores identities in kubernetes as CRDs (custom resource definition). + # These can be queried with: + # kubectl get ciliumid + # - "kvstore" stores identities in an etcd kvstore, that is + # configured below. Cilium versions before 1.6 supported only the kvstore + # backend. Upgrades from these older cilium versions should continue using + # the kvstore by commenting out the identity-allocation-mode below, or + # setting it to "kvstore". + # - "doublewrite" modes store identities in both the kvstore and CRDs. This is useful + # for seamless migrations from the kvstore mode to the crd mode. Consult the + # documentation for more information on how to perform the migration. + identity-allocation-mode: crd + + identity-heartbeat-timeout: "30m0s" + identity-gc-interval: "15m0s" + cilium-endpoint-gc-interval: "5m0s" + nodes-gc-interval: "5m0s" + + # If you want to run cilium in debug mode change this value to true + debug: "false" + debug-verbose: "" + metrics-sampling-interval: "5m" + # The agent can be put into the following three policy enforcement modes + # default, always and never. + # https://docs.cilium.io/en/latest/security/policy/intro/#policy-enforcement-modes + enable-policy: "default" + policy-cidr-match-mode: "" + # If you want metrics enabled in cilium-operator, set the port for + # which the Cilium Operator will have their metrics exposed. + # NOTE that this will open the port on the nodes where Cilium operator pod + # is scheduled. + operator-prometheus-serve-addr: ":9963" + enable-metrics: "true" + enable-policy-secrets-sync: "true" + policy-secrets-only-from-secrets-namespace: "true" + policy-secrets-namespace: "cilium-secrets" + + # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 + # address. + enable-ipv4: "true" + + # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6 + # address. + enable-ipv6: "true" + # Users who wish to specify their own custom CNI configuration file must set + # custom-cni-conf to "true", otherwise Cilium may overwrite the configuration. + custom-cni-conf: "false" + enable-bpf-clock-probe: "false" + # If you want cilium monitor to aggregate tracing for packets, set this level + # to "low", "medium", or "maximum". The higher the level, the less packets + # that will be seen in monitor output. + monitor-aggregation: medium + + # The monitor aggregation interval governs the typical time between monitor + # notification events for each allowed connection. + # + # Only effective when monitor aggregation is set to "medium" or higher. + monitor-aggregation-interval: "5s" + + # The monitor aggregation flags determine which TCP flags which, upon the + # first observation, cause monitor notifications to be generated. + # + # Only effective when monitor aggregation is set to "medium" or higher. + monitor-aggregation-flags: all + # Specifies the ratio (0.0-1.0] of total system memory to use for dynamic + # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps. + bpf-map-dynamic-size-ratio: "0.0025" + # bpf-policy-map-max specifies the maximum number of entries in endpoint + # policy map (per endpoint) + bpf-policy-map-max: "16384" + # bpf-policy-stats-map-max specifies the maximum number of entries in global + # policy stats map + bpf-policy-stats-map-max: "65536" + # bpf-lb-map-max specifies the maximum number of entries in bpf lb service, + # backend and affinity maps. + bpf-lb-map-max: "65536" + bpf-lb-external-clusterip: "false" + bpf-lb-source-range-all-types: "false" + bpf-lb-algorithm-annotation: "false" + bpf-lb-mode-annotation: "false" + + bpf-distributed-lru: "false" + bpf-events-drop-enabled: "true" + bpf-events-policy-verdict-enabled: "true" + bpf-events-trace-enabled: "true" + + # Pre-allocation of map entries allows per-packet latency to be reduced, at + # the expense of up-front memory allocation for the entries in the maps. The + # default value below will minimize memory usage in the default installation; + # users who are sensitive to latency may consider setting this to "true". + # + # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore + # this option and behave as though it is set to "true". + # + # If this value is modified, then during the next Cilium startup the restore + # of existing endpoints and tracking of ongoing connections may be disrupted. + # As a result, reply packets may be dropped and the load-balancing decisions + # for established connections may change. + # + # If this option is set to "false" during an upgrade from 1.3 or earlier to + # 1.4 or later, then it may cause one-time disruptions during the upgrade. + preallocate-bpf-maps: "false" + + # Name of the cluster. Only relevant when building a mesh of clusters. + cluster-name: "default" + # Unique ID of the cluster. Must be unique across all conneted clusters and + # in the range of 1 and 255. Only relevant when building a mesh of clusters. + cluster-id: "0" + + # Encapsulation mode for communication between nodes + # Possible values: + # - disabled + # - vxlan (default) + # - geneve + + routing-mode: "tunnel" + tunnel-protocol: "vxlan" + tunnel-source-port-range: "0-0" + service-no-backend-response: "reject" + + + # Enables L7 proxy for L7 policy enforcement and visibility + enable-l7-proxy: "true" + enable-ipv4-masquerade: "true" + enable-ipv4-big-tcp: "false" + enable-ipv6-big-tcp: "false" + enable-ipv6-masquerade: "true" + enable-tcx: "true" + datapath-mode: "veth" + enable-masquerade-to-route-source: "false" + + enable-xt-socket-fallback: "true" + install-no-conntrack-iptables-rules: "false" + iptables-random-fully: "false" + + auto-direct-node-routes: "false" + direct-routing-skip-unreachable: "false" + + + + kube-proxy-replacement: "true" + kube-proxy-replacement-healthz-bind-address: "" + bpf-lb-sock: "false" + nodeport-addresses: "" + enable-health-check-nodeport: "true" + enable-health-check-loadbalancer-ip: "false" + node-port-bind-protection: "true" + enable-auto-protect-node-port-range: "true" + bpf-lb-acceleration: "disabled" + enable-svc-source-range-check: "true" + enable-l2-neigh-discovery: "false" + k8s-require-ipv4-pod-cidr: "false" + k8s-require-ipv6-pod-cidr: "false" + enable-k8s-networkpolicy: "true" + enable-endpoint-lockdown-on-policy-overflow: "false" + # Tell the agent to generate and write a CNI configuration file + write-cni-conf-when-ready: /host/etc/cni/net.d/05-cilium.conflist + cni-exclusive: "true" + cni-log-file: "/var/run/cilium/cilium-cni.log" + enable-endpoint-health-checking: "true" + enable-health-checking: "true" + health-check-icmp-failure-threshold: "3" + enable-well-known-identities: "false" + enable-node-selector-labels: "false" + synchronize-k8s-nodes: "true" + operator-api-serve-addr: "127.0.0.1:9234" + + enable-hubble: "true" + # UNIX domain socket for Hubble server to listen to. + hubble-socket-path: "/var/run/cilium/hubble.sock" + hubble-network-policy-correlation-enabled: "true" + # An additional address for Hubble server to listen to (e.g. ":4244"). + hubble-listen-address: ":4244" + hubble-disable-tls: "false" + hubble-tls-cert-file: /var/lib/cilium/tls/hubble/server.crt + hubble-tls-key-file: /var/lib/cilium/tls/hubble/server.key + hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/client-ca.crt + ipam: "cluster-pool" + ipam-cilium-node-update-rate: "15s" + cluster-pool-ipv4-cidr: "10.0.0.0/8" + cluster-pool-ipv4-mask-size: "24" + cluster-pool-ipv6-cidr: "fd00::/104" + cluster-pool-ipv6-mask-size: "120" + + default-lb-service-ipam: "lbipam" + egress-gateway-reconciliation-trigger-interval: "1s" + enable-vtep: "false" + vtep-endpoint: "" + vtep-cidr: "" + vtep-mask: "" + vtep-mac: "" + procfs: "/host/proc" + bpf-root: "/sys/fs/bpf" + cgroup-root: "/run/cilium/cgroupv2" + + identity-management-mode: "agent" + enable-sctp: "false" + remove-cilium-node-taints: "true" + set-cilium-node-taints: "true" + set-cilium-is-up-condition: "true" + unmanaged-pod-watcher-interval: "15" + # default DNS proxy to transparent mode in non-chaining modes + dnsproxy-enable-transparent-mode: "true" + dnsproxy-socket-linger-timeout: "10" + tofqdns-dns-reject-response-code: "refused" + tofqdns-enable-dns-compression: "true" + tofqdns-endpoint-max-ip-per-hostname: "1000" + tofqdns-idle-connection-grace-period: "0s" + tofqdns-max-deferred-connection-deletes: "10000" + tofqdns-proxy-response-max-delay: "100ms" + tofqdns-preallocate-identities: "true" + agent-not-ready-taint-key: "node.cilium.io/agent-not-ready" + + mesh-auth-enabled: "true" + mesh-auth-queue-size: "1024" + mesh-auth-rotated-identities-queue-size: "1024" + mesh-auth-gc-interval: "5m0s" + + proxy-xff-num-trusted-hops-ingress: "0" + proxy-xff-num-trusted-hops-egress: "0" + proxy-connect-timeout: "2" + proxy-initial-fetch-timeout: "30" + proxy-max-requests-per-connection: "0" + proxy-max-connection-duration-seconds: "0" + proxy-idle-timeout-seconds: "60" + proxy-max-concurrent-retries: "128" + http-retry-count: "3" + http-stream-idle-timeout: "300" + + external-envoy-proxy: "true" + envoy-base-id: "0" + envoy-access-log-buffer-size: "4096" + envoy-keep-cap-netbindservice: "false" + max-connected-clusters: "255" + clustermesh-enable-endpoint-sync: "false" + clustermesh-enable-mcs-api: "false" + policy-default-local-cluster: "false" + + nat-map-stats-entries: "32" + nat-map-stats-interval: "30s" + enable-internal-traffic-policy: "true" + enable-lb-ipam: "true" + enable-non-default-deny-policies: "true" + enable-source-ip-verification: "true" + +# Extra config allows adding arbitrary properties to the cilium config. +# By putting it at the end of the ConfigMap, it's also possible to override existing properties. +--- +# Source: cilium/templates/cilium-envoy/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: cilium-envoy-config + namespace: kube-system +data: + # Keep the key name as bootstrap-config.json to avoid breaking changes + bootstrap-config.json: | + {"admin":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/admin.sock"}}},"applicationLogConfig":{"logFormat":{"textFormat":"[%Y-%m-%d %T.%e][%t][%l][%n] [%g:%#] %v"}},"bootstrapExtensions":[{"name":"envoy.bootstrap.internal_listener","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener"}}],"dynamicResources":{"cdsConfig":{"apiConfigSource":{"apiType":"GRPC","grpcServices":[{"envoyGrpc":{"clusterName":"xds-grpc-cilium"}}],"setNodeOnFirstMessageOnly":true,"transportApiVersion":"V3"},"initialFetchTimeout":"30s","resourceApiVersion":"V3"},"ldsConfig":{"apiConfigSource":{"apiType":"GRPC","grpcServices":[{"envoyGrpc":{"clusterName":"xds-grpc-cilium"}}],"setNodeOnFirstMessageOnly":true,"transportApiVersion":"V3"},"initialFetchTimeout":"30s","resourceApiVersion":"V3"}},"node":{"cluster":"ingress-cluster","id":"host~127.0.0.1~no-id~localdomain"},"overloadManager":{"resourceMonitors":[{"name":"envoy.resource_monitors.global_downstream_max_connections","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig","max_active_downstream_connections":"50000"}}]},"staticResources":{"clusters":[{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"ingress-cluster","type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"egress-cluster-tls","transportSocket":{"name":"cilium.tls_wrapper","typedConfig":{"@type":"type.googleapis.com/cilium.UpstreamTlsWrapperContext"}},"type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"upstreamHttpProtocolOptions":{},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"egress-cluster","type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"ingress-cluster-tls","transportSocket":{"name":"cilium.tls_wrapper","typedConfig":{"@type":"type.googleapis.com/cilium.UpstreamTlsWrapperContext"}},"type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"upstreamHttpProtocolOptions":{},"useDownstreamProtocolConfig":{}}}},{"connectTimeout":"2s","loadAssignment":{"clusterName":"xds-grpc-cilium","endpoints":[{"lbEndpoints":[{"endpoint":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/xds.sock"}}}}]}]},"name":"xds-grpc-cilium","type":"STATIC","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","explicitHttpConfig":{"http2ProtocolOptions":{}}}}},{"connectTimeout":"2s","loadAssignment":{"clusterName":"/envoy-admin","endpoints":[{"lbEndpoints":[{"endpoint":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/admin.sock"}}}}]}]},"name":"/envoy-admin","type":"STATIC"}],"listeners":[{"additionalAddresses":[{"address":{"socketAddress":{"address":"::","portValue":9964}}}],"address":{"socketAddress":{"address":"0.0.0.0","portValue":9964}},"filterChains":[{"filters":[{"name":"envoy.filters.network.http_connection_manager","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager","httpFilters":[{"name":"envoy.filters.http.router","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"}}],"internalAddressConfig":{"cidrRanges":[{"addressPrefix":"10.0.0.0","prefixLen":8},{"addressPrefix":"172.16.0.0","prefixLen":12},{"addressPrefix":"192.168.0.0","prefixLen":16},{"addressPrefix":"127.0.0.1","prefixLen":32},{"addressPrefix":"::1","prefixLen":128}]},"routeConfig":{"virtualHosts":[{"domains":["*"],"name":"prometheus_metrics_route","routes":[{"match":{"prefix":"/metrics"},"name":"prometheus_metrics_route","route":{"cluster":"/envoy-admin","prefixRewrite":"/stats/prometheus"}}]}]},"statPrefix":"envoy-prometheus-metrics-listener","streamIdleTimeout":"300s"}}]}],"name":"envoy-prometheus-metrics-listener"},{"additionalAddresses":[{"address":{"socketAddress":{"address":"::1","portValue":9878}}}],"address":{"socketAddress":{"address":"127.0.0.1","portValue":9878}},"filterChains":[{"filters":[{"name":"envoy.filters.network.http_connection_manager","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager","httpFilters":[{"name":"envoy.filters.http.router","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"}}],"internalAddressConfig":{"cidrRanges":[{"addressPrefix":"10.0.0.0","prefixLen":8},{"addressPrefix":"172.16.0.0","prefixLen":12},{"addressPrefix":"192.168.0.0","prefixLen":16},{"addressPrefix":"127.0.0.1","prefixLen":32},{"addressPrefix":"::1","prefixLen":128}]},"routeConfig":{"virtual_hosts":[{"domains":["*"],"name":"health","routes":[{"match":{"prefix":"/healthz"},"name":"health","route":{"cluster":"/envoy-admin","prefixRewrite":"/ready"}}]}]},"statPrefix":"envoy-health-listener","streamIdleTimeout":"300s"}}]}],"name":"envoy-health-listener"}]}} +--- +# Source: cilium/templates/cilium-agent/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + - services + - pods + - endpoints + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch + # This is used when validating policies in preflight. This will need to stay + # until we figure out how to avoid "get" inside the preflight, and then + # should be removed ideally. + - get +- apiGroups: + - cilium.io + resources: + - ciliumloadbalancerippools + - ciliumbgppeeringpolicies + - ciliumbgpnodeconfigs + - ciliumbgpadvertisements + - ciliumbgppeerconfigs + - ciliumclusterwideenvoyconfigs + - ciliumclusterwidenetworkpolicies + - ciliumegressgatewaypolicies + - ciliumendpoints + - ciliumendpointslices + - ciliumenvoyconfigs + - ciliumidentities + - ciliumlocalredirectpolicies + - ciliumnetworkpolicies + - ciliumnodes + - ciliumnodeconfigs + - ciliumcidrgroups + - ciliuml2announcementpolicies + - ciliumpodippools + verbs: + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumidentities + - ciliumendpoints + - ciliumnodes + verbs: + - create +- apiGroups: + - cilium.io + # To synchronize garbage collection of such resources + resources: + - ciliumidentities + verbs: + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints + verbs: + - delete + - get +- apiGroups: + - cilium.io + resources: + - ciliumnodes + - ciliumnodes/status + verbs: + - get + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints/status + - ciliumendpoints + - ciliuml2announcementpolicies/status + - ciliumbgpnodeconfigs/status + verbs: + - patch +--- +# Source: cilium/templates/cilium-operator/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium-operator + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + # to automatically delete [core|kube]dns pods so that are starting to being + # managed by Cilium + - delete +- apiGroups: + - "" + resources: + - configmaps + resourceNames: + - cilium-config + verbs: + # allow patching of the configmap to set annotations + - patch +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + # To remove node taints + - nodes + # To set NetworkUnavailable false on startup + - nodes/status + verbs: + - patch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + # to perform LB IP allocation for BGP + - services/status + verbs: + - update + - patch +- apiGroups: + - "" + resources: + # to check apiserver connectivity + - namespaces + - secrets + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + # to perform the translation of a CNP that contains `ToGroup` to its endpoints + - services + - endpoints + verbs: + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies + - ciliumclusterwidenetworkpolicies + verbs: + # Create auto-generated CNPs and CCNPs from Policies that have 'toGroups' + - create + - update + - deletecollection + # To update the status of the CNPs and CCNPs + - patch + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies/status + - ciliumclusterwidenetworkpolicies/status + verbs: + # Update the auto-generated CNPs and CCNPs status. + - patch + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints + - ciliumidentities + verbs: + # To perform garbage collection of such resources + - delete + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumidentities + verbs: + # To synchronize garbage collection of such resources + - update +- apiGroups: + - cilium.io + resources: + - ciliumnodes + verbs: + - create + - update + - get + - list + - watch + # To perform CiliumNode garbage collector + - delete +- apiGroups: + - cilium.io + resources: + - ciliumnodes/status + verbs: + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpointslices + - ciliumenvoyconfigs + - ciliumbgppeerconfigs + - ciliumbgpadvertisements + - ciliumbgpnodeconfigs + verbs: + - create + - update + - get + - list + - watch + - delete + - patch +- apiGroups: + - cilium.io + resources: + - ciliumbgpclusterconfigs/status + - ciliumbgppeerconfigs/status + verbs: + - update +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - update + resourceNames: + - ciliumloadbalancerippools.cilium.io + - ciliumbgppeeringpolicies.cilium.io + - ciliumbgpclusterconfigs.cilium.io + - ciliumbgppeerconfigs.cilium.io + - ciliumbgpadvertisements.cilium.io + - ciliumbgpnodeconfigs.cilium.io + - ciliumbgpnodeconfigoverrides.cilium.io + - ciliumclusterwideenvoyconfigs.cilium.io + - ciliumclusterwidenetworkpolicies.cilium.io + - ciliumegressgatewaypolicies.cilium.io + - ciliumendpoints.cilium.io + - ciliumendpointslices.cilium.io + - ciliumenvoyconfigs.cilium.io + - ciliumidentities.cilium.io + - ciliumlocalredirectpolicies.cilium.io + - ciliumnetworkpolicies.cilium.io + - ciliumnodes.cilium.io + - ciliumnodeconfigs.cilium.io + - ciliumcidrgroups.cilium.io + - ciliuml2announcementpolicies.cilium.io + - ciliumpodippools.cilium.io + - ciliumgatewayclassconfigs.cilium.io +- apiGroups: + - cilium.io + resources: + - ciliumloadbalancerippools + - ciliumpodippools + - ciliumbgppeeringpolicies + - ciliumbgpclusterconfigs + - ciliumbgpnodeconfigoverrides + - ciliumbgppeerconfigs + verbs: + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumpodippools + verbs: + - create +- apiGroups: + - cilium.io + resources: + - ciliumloadbalancerippools/status + verbs: + - patch +# For cilium-operator running in HA mode. +# +# Cilium operator running in HA mode requires the use of ResourceLock for Leader Election +# between multiple running instances. +# The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less +# common and fewer objects in the cluster watch "all Leases". +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - update +--- +# Source: cilium/templates/cilium-agent/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium +subjects: +- kind: ServiceAccount + name: "cilium" + namespace: kube-system +--- +# Source: cilium/templates/cilium-operator/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium-operator + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium-operator +subjects: +- kind: ServiceAccount + name: "cilium-operator" + namespace: kube-system +--- +# Source: cilium/templates/cilium-agent/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cilium-config-agent + namespace: kube-system + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch +--- +# Source: cilium/templates/cilium-agent/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cilium-tlsinterception-secrets + namespace: "cilium-secrets" + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch +--- +# Source: cilium/templates/cilium-operator/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cilium-operator-tlsinterception-secrets + namespace: "cilium-secrets" + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - update + - patch +--- +# Source: cilium/templates/cilium-agent/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cilium-config-agent + namespace: kube-system + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-config-agent +subjects: + - kind: ServiceAccount + name: "cilium" + namespace: kube-system +--- +# Source: cilium/templates/cilium-agent/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cilium-tlsinterception-secrets + namespace: "cilium-secrets" + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-tlsinterception-secrets +subjects: +- kind: ServiceAccount + name: "cilium" + namespace: kube-system +--- +# Source: cilium/templates/cilium-operator/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cilium-operator-tlsinterception-secrets + namespace: "cilium-secrets" + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-operator-tlsinterception-secrets +subjects: +- kind: ServiceAccount + name: "cilium-operator" + namespace: kube-system +--- +# Source: cilium/templates/cilium-envoy/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: cilium-envoy + namespace: kube-system + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9964" + labels: + k8s-app: cilium-envoy + app.kubernetes.io/name: cilium-envoy + app.kubernetes.io/part-of: cilium + io.cilium/app: proxy +spec: + clusterIP: None + type: ClusterIP + selector: + k8s-app: cilium-envoy + ports: + - name: envoy-metrics + port: 9964 + protocol: TCP + targetPort: envoy-metrics +--- +# Source: cilium/templates/hubble/peer-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: hubble-peer + namespace: kube-system + labels: + k8s-app: cilium + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: hubble-peer + +spec: + selector: + k8s-app: cilium + ports: + - name: peer-service + port: 443 + protocol: TCP + targetPort: 4244 + internalTrafficPolicy: Local +--- +# Source: cilium/templates/cilium-agent/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cilium + namespace: kube-system + labels: + k8s-app: cilium + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-agent +spec: + selector: + matchLabels: + k8s-app: cilium + updateStrategy: + rollingUpdate: + maxUnavailable: 2 + type: RollingUpdate + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: cilium-agent + labels: + k8s-app: cilium + app.kubernetes.io/name: cilium-agent + app.kubernetes.io/part-of: cilium + spec: + securityContext: + appArmorProfile: + type: Unconfined + seccompProfile: + type: Unconfined + containers: + - name: cilium-agent + image: "quay.io/cilium/cilium:v1.18.4@sha256:49d87af187eeeb9e9e3ec2bc6bd372261a0b5cb2d845659463ba7cc10fe9e45f" + imagePullPolicy: IfNotPresent + command: + - cilium-agent + args: + - --config-dir=/tmp/cilium/config-map + startupProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9879 + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + failureThreshold: 300 + periodSeconds: 2 + successThreshold: 1 + initialDelaySeconds: 5 + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9879 + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + - name: "require-k8s-connectivity" + value: "false" + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 10 + timeoutSeconds: 5 + readinessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9879 + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 5 + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_CLUSTERMESH_CONFIG + value: /var/lib/cilium/clustermesh/ + - name: GOMEMLIMIT + valueFrom: + resourceFieldRef: + resource: limits.memory + divisor: '1' + - name: KUBERNETES_SERVICE_HOST + value: "2620:11f:7001:7:ffff:ffff:ad7:1dd" + - name: KUBERNETES_SERVICE_PORT + value: "6443" + - name: KUBE_CLIENT_BACKOFF_BASE + value: "1" + - name: KUBE_CLIENT_BACKOFF_DURATION + value: "120" + lifecycle: + postStart: + exec: + command: + - "bash" + - "-c" + - | + set -o errexit + set -o pipefail + set -o nounset + + # When running in AWS ENI mode, it's likely that 'aws-node' has + # had a chance to install SNAT iptables rules. These can result + # in dropped traffic, so we should attempt to remove them. + # We do it using a 'postStart' hook since this may need to run + # for nodes which might have already been init'ed but may still + # have dangling rules. This is safe because there are no + # dependencies on anything that is part of the startup script + # itself, and can be safely run multiple times per node (e.g. in + # case of a restart). + if [[ "$(iptables-save | grep -E -c 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN')" != "0" ]]; + then + echo 'Deleting iptables rules created by the AWS CNI VPC plugin' + iptables-save | grep -E -v 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN' | iptables-restore + fi + echo 'Done!' + + preStop: + exec: + command: + - /cni-uninstall.sh + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - CHOWN + - KILL + - NET_ADMIN + - NET_RAW + - IPC_LOCK + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: envoy-sockets + mountPath: /var/run/cilium/envoy/sockets + readOnly: false + # Unprivileged containers need to mount /proc/sys/net from the host + # to have write access + - mountPath: /host/proc/sys/net + name: host-proc-sys-net + # Unprivileged containers need to mount /proc/sys/kernel from the host + # to have write access + - mountPath: /host/proc/sys/kernel + name: host-proc-sys-kernel + - name: bpf-maps + mountPath: /sys/fs/bpf + # Unprivileged containers can't set mount propagation to bidirectional + # in this case we will mount the bpf fs from an init container that + # is privileged and set the mount propagation from host to container + # in Cilium. + mountPropagation: HostToContainer + - name: cilium-run + mountPath: /var/run/cilium + - name: cilium-netns + mountPath: /var/run/cilium/netns + mountPropagation: HostToContainer + - name: etc-cni-netd + mountPath: /host/etc/cni/net.d + - name: clustermesh-secrets + mountPath: /var/lib/cilium/clustermesh + readOnly: true + # Needed to be able to load kernel modules + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: xtables-lock + mountPath: /run/xtables.lock + - name: hubble-tls + mountPath: /var/lib/cilium/tls/hubble + readOnly: true + - name: tmp + mountPath: /tmp + + initContainers: + - name: config + image: "quay.io/cilium/cilium:v1.18.4@sha256:49d87af187eeeb9e9e3ec2bc6bd372261a0b5cb2d845659463ba7cc10fe9e45f" + imagePullPolicy: IfNotPresent + command: + - cilium-dbg + - build-config + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: KUBERNETES_SERVICE_HOST + value: "2620:11f:7001:7:ffff:ffff:ad7:1dd" + - name: KUBERNETES_SERVICE_PORT + value: "6443" + volumeMounts: + - name: tmp + mountPath: /tmp + terminationMessagePolicy: FallbackToLogsOnError + # Required to mount cgroup2 filesystem on the underlying Kubernetes node. + # We use nsenter command with host's cgroup and mount namespaces enabled. + - name: mount-cgroup + image: "quay.io/cilium/cilium:v1.18.4@sha256:49d87af187eeeb9e9e3ec2bc6bd372261a0b5cb2d845659463ba7cc10fe9e45f" + imagePullPolicy: IfNotPresent + env: + - name: CGROUP_ROOT + value: /run/cilium/cgroupv2 + - name: BIN_PATH + value: /opt/cni/bin + command: + - sh + - -ec + # The statically linked Go program binary is invoked to avoid any + # dependency on utilities like sh and mount that can be missing on certain + # distros installed on the underlying host. Copy the binary to the + # same directory where we install cilium cni plugin so that exec permissions + # are available. + - | + cp /usr/bin/cilium-mount /hostbin/cilium-mount; + nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-mount" $CGROUP_ROOT; + rm /hostbin/cilium-mount + volumeMounts: + - name: hostproc + mountPath: /hostproc + - name: cni-path + mountPath: /hostbin + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL + - name: apply-sysctl-overwrites + image: "quay.io/cilium/cilium:v1.18.4@sha256:49d87af187eeeb9e9e3ec2bc6bd372261a0b5cb2d845659463ba7cc10fe9e45f" + imagePullPolicy: IfNotPresent + env: + - name: BIN_PATH + value: /opt/cni/bin + command: + - sh + - -ec + # The statically linked Go program binary is invoked to avoid any + # dependency on utilities like sh that can be missing on certain + # distros installed on the underlying host. Copy the binary to the + # same directory where we install cilium cni plugin so that exec permissions + # are available. + - | + cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix; + nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix"; + rm /hostbin/cilium-sysctlfix + volumeMounts: + - name: hostproc + mountPath: /hostproc + - name: cni-path + mountPath: /hostbin + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL + # Mount the bpf fs if it is not mounted. We will perform this task + # from a privileged container because the mount propagation bidirectional + # only works from privileged containers. + - name: mount-bpf-fs + image: "quay.io/cilium/cilium:v1.18.4@sha256:49d87af187eeeb9e9e3ec2bc6bd372261a0b5cb2d845659463ba7cc10fe9e45f" + imagePullPolicy: IfNotPresent + args: + - 'mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf' + command: + - /bin/bash + - -c + - -- + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + privileged: true + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf + mountPropagation: Bidirectional + - name: clean-cilium-state + image: "quay.io/cilium/cilium:v1.18.4@sha256:49d87af187eeeb9e9e3ec2bc6bd372261a0b5cb2d845659463ba7cc10fe9e45f" + imagePullPolicy: IfNotPresent + command: + - /init-container.sh + env: + - name: CILIUM_ALL_STATE + valueFrom: + configMapKeyRef: + name: cilium-config + key: clean-cilium-state + optional: true + - name: CILIUM_BPF_STATE + valueFrom: + configMapKeyRef: + name: cilium-config + key: clean-cilium-bpf-state + optional: true + - name: WRITE_CNI_CONF_WHEN_READY + valueFrom: + configMapKeyRef: + name: cilium-config + key: write-cni-conf-when-ready + optional: true + - name: KUBERNETES_SERVICE_HOST + value: "2620:11f:7001:7:ffff:ffff:ad7:1dd" + - name: KUBERNETES_SERVICE_PORT + value: "6443" + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - NET_ADMIN + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + drop: + - ALL + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf + # Required to mount cgroup filesystem from the host to cilium agent pod + - name: cilium-cgroup + mountPath: /run/cilium/cgroupv2 + mountPropagation: HostToContainer + - name: cilium-run + mountPath: /var/run/cilium # wait-for-kube-proxy + # Install the CNI binaries in an InitContainer so we don't have a writable host mount in the agent + - name: install-cni-binaries + image: "quay.io/cilium/cilium:v1.18.4@sha256:49d87af187eeeb9e9e3ec2bc6bd372261a0b5cb2d845659463ba7cc10fe9e45f" + imagePullPolicy: IfNotPresent + command: + - "/install-plugin.sh" + resources: + requests: + cpu: 100m + memory: 10Mi + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: cni-path + mountPath: /host/opt/cni/bin # .Values.cni.install + restartPolicy: Always + priorityClassName: system-node-critical + serviceAccountName: "cilium" + automountServiceAccountToken: true + terminationGracePeriodSeconds: 1 + hostNetwork: true + + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: cilium + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - operator: Exists + volumes: + # For sharing configuration between the "config" initContainer and the agent + - name: tmp + emptyDir: {} + # To keep state between restarts / upgrades + - name: cilium-run + hostPath: + path: /var/run/cilium + type: DirectoryOrCreate + # To exec into pod network namespaces + - name: cilium-netns + hostPath: + path: /var/run/netns + type: DirectoryOrCreate + # To keep state between restarts / upgrades for bpf maps + - name: bpf-maps + hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate + # To mount cgroup2 filesystem on the host or apply sysctlfix + - name: hostproc + hostPath: + path: /proc + type: Directory + # To keep state between restarts / upgrades for cgroup2 filesystem + - name: cilium-cgroup + hostPath: + path: /run/cilium/cgroupv2 + type: DirectoryOrCreate + # To install cilium cni plugin in the host + - name: cni-path + hostPath: + path: /opt/cni/bin + type: DirectoryOrCreate + # To install cilium cni configuration in the host + - name: etc-cni-netd + hostPath: + path: /etc/cni/net.d + type: DirectoryOrCreate + # To be able to load kernel modules + - name: lib-modules + hostPath: + path: /lib/modules + # To access iptables concurrently with other processes (e.g. kube-proxy) + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + # Sharing socket with Cilium Envoy on the same node by using a host path + - name: envoy-sockets + hostPath: + path: "/var/run/cilium/envoy/sockets" + type: DirectoryOrCreate + # To read the clustermesh configuration + - name: clustermesh-secrets + projected: + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + sources: + - secret: + name: cilium-clustermesh + optional: true + # note: items are not explicitly listed here, since the entries of this secret + # depend on the peers configured, and that would cause a restart of all agents + # at every addition/removal. Leaving the field empty makes each secret entry + # to be automatically projected into the volume as a file whose name is the key. + - secret: + name: clustermesh-apiserver-remote-cert + optional: true + items: + - key: tls.key + path: common-etcd-client.key + - key: tls.crt + path: common-etcd-client.crt + - key: ca.crt + path: common-etcd-client-ca.crt + # note: we configure the volume for the kvstoremesh-specific certificate + # regardless of whether KVStoreMesh is enabled or not, so that it can be + # automatically mounted in case KVStoreMesh gets subsequently enabled, + # without requiring an agent restart. + - secret: + name: clustermesh-apiserver-local-cert + optional: true + items: + - key: tls.key + path: local-etcd-client.key + - key: tls.crt + path: local-etcd-client.crt + - key: ca.crt + path: local-etcd-client-ca.crt + - name: host-proc-sys-net + hostPath: + path: /proc/sys/net + type: Directory + - name: host-proc-sys-kernel + hostPath: + path: /proc/sys/kernel + type: Directory + - name: hubble-tls + projected: + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + sources: + - secret: + name: hubble-server-certs + optional: true + items: + - key: tls.crt + path: server.crt + - key: tls.key + path: server.key + - key: ca.crt + path: client-ca.crt +--- +# Source: cilium/templates/cilium-envoy/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cilium-envoy + namespace: kube-system + labels: + k8s-app: cilium-envoy + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-envoy + name: cilium-envoy +spec: + selector: + matchLabels: + k8s-app: cilium-envoy + updateStrategy: + rollingUpdate: + maxUnavailable: 2 + type: RollingUpdate + template: + metadata: + annotations: + labels: + k8s-app: cilium-envoy + name: cilium-envoy + app.kubernetes.io/name: cilium-envoy + app.kubernetes.io/part-of: cilium + spec: + securityContext: + appArmorProfile: + type: Unconfined + containers: + - name: cilium-envoy + image: "quay.io/cilium/cilium-envoy:v1.34.10-1762597008-ff7ae7d623be00078865cff1b0672cc5d9bfc6d5@sha256:1deb6709afcb5523579bf1abbc3255adf9e354565a88c4a9162c8d9cb1d77ab5" + imagePullPolicy: IfNotPresent + command: + - /usr/bin/cilium-envoy-starter + args: + - '--' + - '-c /var/run/cilium/envoy/bootstrap-config.json' + - '--base-id 0' + - '--log-level info' + startupProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9878 + scheme: HTTP + failureThreshold: 105 + periodSeconds: 2 + successThreshold: 1 + initialDelaySeconds: 5 + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9878 + scheme: HTTP + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 10 + timeoutSeconds: 5 + readinessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9878 + scheme: HTTP + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 5 + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: KUBERNETES_SERVICE_HOST + value: "2620:11f:7001:7:ffff:ffff:ad7:1dd" + - name: KUBERNETES_SERVICE_PORT + value: "6443" + ports: + - name: envoy-metrics + containerPort: 9964 + hostPort: 9964 + protocol: TCP + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - NET_ADMIN + - SYS_ADMIN + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: envoy-sockets + mountPath: /var/run/cilium/envoy/sockets + readOnly: false + - name: envoy-artifacts + mountPath: /var/run/cilium/envoy/artifacts + readOnly: true + - name: envoy-config + mountPath: /var/run/cilium/envoy/ + readOnly: true + - name: bpf-maps + mountPath: /sys/fs/bpf + mountPropagation: HostToContainer + restartPolicy: Always + priorityClassName: system-node-critical + serviceAccountName: "cilium-envoy" + automountServiceAccountToken: true + terminationGracePeriodSeconds: 1 + hostNetwork: true + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cilium.io/no-schedule + operator: NotIn + values: + - "true" + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: cilium + topologyKey: kubernetes.io/hostname + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: cilium-envoy + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - operator: Exists + volumes: + - name: envoy-sockets + hostPath: + path: "/var/run/cilium/envoy/sockets" + type: DirectoryOrCreate + - name: envoy-artifacts + hostPath: + path: "/var/run/cilium/envoy/artifacts" + type: DirectoryOrCreate + - name: envoy-config + configMap: + name: "cilium-envoy-config" + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + items: + - key: bootstrap-config.json + path: bootstrap-config.json + # To keep state between restarts / upgrades + # To keep state between restarts / upgrades for bpf maps + - name: bpf-maps + hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate +--- +# Source: cilium/templates/cilium-operator/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cilium-operator + namespace: kube-system + labels: + io.cilium/app: operator + name: cilium-operator + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-operator +spec: + # See docs on ServerCapabilities.LeasesResourceLock in file pkg/k8s/version/version.go + # for more details. + replicas: 2 + selector: + matchLabels: + io.cilium/app: operator + name: cilium-operator + # ensure operator update on single node k8s clusters, by using rolling update with maxUnavailable=100% in case + # of one replica and no user configured Recreate strategy. + # otherwise an update might get stuck due to the default maxUnavailable=50% in combination with the + # podAntiAffinity which prevents deployments of multiple operator replicas on the same node. + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 50% + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/port: "9963" + prometheus.io/scrape: "true" + labels: + io.cilium/app: operator + name: cilium-operator + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-operator + spec: + securityContext: + seccompProfile: + type: RuntimeDefault + containers: + - name: cilium-operator + image: "quay.io/cilium/operator-generic:v1.18.4@sha256:1b22b9ff28affdf574378a70dade4ef835b00b080c2ee2418530809dd62c3012" + imagePullPolicy: IfNotPresent + command: + - cilium-operator-generic + args: + - --config-dir=/tmp/cilium/config-map + - --debug=$(CILIUM_DEBUG) + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_DEBUG + valueFrom: + configMapKeyRef: + key: debug + name: cilium-config + optional: true + - name: KUBERNETES_SERVICE_HOST + value: "2620:11f:7001:7:ffff:ffff:ad7:1dd" + - name: KUBERNETES_SERVICE_PORT + value: "6443" + ports: + - name: prometheus + containerPort: 9963 + hostPort: 9963 + protocol: TCP + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9234 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 3 + readinessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9234 + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 5 + volumeMounts: + - name: cilium-config-path + mountPath: /tmp/cilium/config-map + readOnly: true + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + hostNetwork: true + restartPolicy: Always + priorityClassName: system-cluster-critical + serviceAccountName: "cilium-operator" + automountServiceAccountToken: true + # In HA mode, cilium-operator pods must not be scheduled on the same + # node as they will clash with each other. + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + io.cilium/app: operator + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - key: node-role.kubernetes.io/master + operator: Exists + - key: node.kubernetes.io/not-ready + operator: Exists + - key: node.cloudprovider.kubernetes.io/uninitialized + operator: Exists + - key: node.cilium.io/agent-not-ready + operator: Exists + + volumes: + # To read the configuration from the config map + - name: cilium-config-path + configMap: + name: cilium-config diff --git a/nix/kubernetes/bootstrap/cilium/cilium-agent/clusterrole.yaml b/nix/kubernetes/bootstrap/cilium/cilium-agent/clusterrole.yaml deleted file mode 100644 index 34f88bb..0000000 --- a/nix/kubernetes/bootstrap/cilium/cilium-agent/clusterrole.yaml +++ /dev/null @@ -1,112 +0,0 @@ ---- -# Source: cilium/templates/cilium-agent/clusterrole.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: cilium - labels: - app.kubernetes.io/part-of: cilium -rules: -- apiGroups: - - networking.k8s.io - resources: - - networkpolicies - verbs: - - get - - list - - watch -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - namespaces - - services - - pods - - endpoints - - nodes - verbs: - - get - - list - - watch -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - list - - watch - # This is used when validating policies in preflight. This will need to stay - # until we figure out how to avoid "get" inside the preflight, and then - # should be removed ideally. - - get -- apiGroups: - - cilium.io - resources: - - ciliumloadbalancerippools - - ciliumbgppeeringpolicies - - ciliumbgpnodeconfigs - - ciliumbgpadvertisements - - ciliumbgppeerconfigs - - ciliumclusterwideenvoyconfigs - - ciliumclusterwidenetworkpolicies - - ciliumegressgatewaypolicies - - ciliumendpoints - - ciliumendpointslices - - ciliumenvoyconfigs - - ciliumidentities - - ciliumlocalredirectpolicies - - ciliumnetworkpolicies - - ciliumnodes - - ciliumnodeconfigs - - ciliumcidrgroups - - ciliuml2announcementpolicies - - ciliumpodippools - verbs: - - list - - watch -- apiGroups: - - cilium.io - resources: - - ciliumidentities - - ciliumendpoints - - ciliumnodes - verbs: - - create -- apiGroups: - - cilium.io - # To synchronize garbage collection of such resources - resources: - - ciliumidentities - verbs: - - update -- apiGroups: - - cilium.io - resources: - - ciliumendpoints - verbs: - - delete - - get -- apiGroups: - - cilium.io - resources: - - ciliumnodes - - ciliumnodes/status - verbs: - - get - - update -- apiGroups: - - cilium.io - resources: - - ciliumendpoints/status - - ciliumendpoints - - ciliuml2announcementpolicies/status - - ciliumbgpnodeconfigs/status - verbs: - - patch diff --git a/nix/kubernetes/bootstrap/cilium/cilium-agent/clusterrolebinding.yaml b/nix/kubernetes/bootstrap/cilium/cilium-agent/clusterrolebinding.yaml deleted file mode 100644 index 9f906ad..0000000 --- a/nix/kubernetes/bootstrap/cilium/cilium-agent/clusterrolebinding.yaml +++ /dev/null @@ -1,16 +0,0 @@ ---- -# Source: cilium/templates/cilium-agent/clusterrolebinding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: cilium - labels: - app.kubernetes.io/part-of: cilium -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cilium -subjects: -- kind: ServiceAccount - name: "cilium" - namespace: kube-system diff --git a/nix/kubernetes/bootstrap/cilium/cilium-agent/daemonset.yaml b/nix/kubernetes/bootstrap/cilium/cilium-agent/daemonset.yaml deleted file mode 100644 index edd9478..0000000 --- a/nix/kubernetes/bootstrap/cilium/cilium-agent/daemonset.yaml +++ /dev/null @@ -1,522 +0,0 @@ ---- -# Source: cilium/templates/cilium-agent/daemonset.yaml -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: cilium - namespace: kube-system - labels: - k8s-app: cilium - app.kubernetes.io/part-of: cilium - app.kubernetes.io/name: cilium-agent -spec: - selector: - matchLabels: - k8s-app: cilium - updateStrategy: - rollingUpdate: - maxUnavailable: 2 - type: RollingUpdate - template: - metadata: - annotations: - kubectl.kubernetes.io/default-container: cilium-agent - labels: - k8s-app: cilium - app.kubernetes.io/name: cilium-agent - app.kubernetes.io/part-of: cilium - spec: - securityContext: - appArmorProfile: - type: Unconfined - seccompProfile: - type: Unconfined - containers: - - name: cilium-agent - image: "quay.io/cilium/cilium:v1.18.4@sha256:49d87af187eeeb9e9e3ec2bc6bd372261a0b5cb2d845659463ba7cc10fe9e45f" - imagePullPolicy: IfNotPresent - command: - - cilium-agent - args: - - --config-dir=/tmp/cilium/config-map - startupProbe: - httpGet: - host: "127.0.0.1" - path: /healthz - port: 9879 - scheme: HTTP - httpHeaders: - - name: "brief" - value: "true" - failureThreshold: 300 - periodSeconds: 2 - successThreshold: 1 - initialDelaySeconds: 5 - livenessProbe: - httpGet: - host: "127.0.0.1" - path: /healthz - port: 9879 - scheme: HTTP - httpHeaders: - - name: "brief" - value: "true" - - name: "require-k8s-connectivity" - value: "false" - periodSeconds: 30 - successThreshold: 1 - failureThreshold: 10 - timeoutSeconds: 5 - readinessProbe: - httpGet: - host: "127.0.0.1" - path: /healthz - port: 9879 - scheme: HTTP - httpHeaders: - - name: "brief" - value: "true" - periodSeconds: 30 - successThreshold: 1 - failureThreshold: 3 - timeoutSeconds: 5 - env: - - name: K8S_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: CILIUM_K8S_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: CILIUM_CLUSTERMESH_CONFIG - value: /var/lib/cilium/clustermesh/ - - name: GOMEMLIMIT - valueFrom: - resourceFieldRef: - resource: limits.memory - divisor: '1' - - name: KUBERNETES_SERVICE_HOST - value: "2620:11f:7001:7:ffff:ffff:ad7:1dd" - - name: KUBERNETES_SERVICE_PORT - value: "6443" - - name: KUBE_CLIENT_BACKOFF_BASE - value: "1" - - name: KUBE_CLIENT_BACKOFF_DURATION - value: "120" - lifecycle: - postStart: - exec: - command: - - "bash" - - "-c" - - | - set -o errexit - set -o pipefail - set -o nounset - - # When running in AWS ENI mode, it's likely that 'aws-node' has - # had a chance to install SNAT iptables rules. These can result - # in dropped traffic, so we should attempt to remove them. - # We do it using a 'postStart' hook since this may need to run - # for nodes which might have already been init'ed but may still - # have dangling rules. This is safe because there are no - # dependencies on anything that is part of the startup script - # itself, and can be safely run multiple times per node (e.g. in - # case of a restart). - if [[ "$(iptables-save | grep -E -c 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN')" != "0" ]]; - then - echo 'Deleting iptables rules created by the AWS CNI VPC plugin' - iptables-save | grep -E -v 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN' | iptables-restore - fi - echo 'Done!' - - preStop: - exec: - command: - - /cni-uninstall.sh - securityContext: - seLinuxOptions: - level: s0 - type: spc_t - capabilities: - add: - - CHOWN - - KILL - - NET_ADMIN - - NET_RAW - - IPC_LOCK - - SYS_MODULE - - SYS_ADMIN - - SYS_RESOURCE - - DAC_OVERRIDE - - FOWNER - - SETGID - - SETUID - drop: - - ALL - terminationMessagePolicy: FallbackToLogsOnError - volumeMounts: - - name: envoy-sockets - mountPath: /var/run/cilium/envoy/sockets - readOnly: false - # Unprivileged containers need to mount /proc/sys/net from the host - # to have write access - - mountPath: /host/proc/sys/net - name: host-proc-sys-net - # Unprivileged containers need to mount /proc/sys/kernel from the host - # to have write access - - mountPath: /host/proc/sys/kernel - name: host-proc-sys-kernel - - name: bpf-maps - mountPath: /sys/fs/bpf - # Unprivileged containers can't set mount propagation to bidirectional - # in this case we will mount the bpf fs from an init container that - # is privileged and set the mount propagation from host to container - # in Cilium. - mountPropagation: HostToContainer - - name: cilium-run - mountPath: /var/run/cilium - - name: cilium-netns - mountPath: /var/run/cilium/netns - mountPropagation: HostToContainer - - name: etc-cni-netd - mountPath: /host/etc/cni/net.d - - name: clustermesh-secrets - mountPath: /var/lib/cilium/clustermesh - readOnly: true - # Needed to be able to load kernel modules - - name: lib-modules - mountPath: /lib/modules - readOnly: true - - name: xtables-lock - mountPath: /run/xtables.lock - - name: hubble-tls - mountPath: /var/lib/cilium/tls/hubble - readOnly: true - - name: tmp - mountPath: /tmp - - initContainers: - - name: config - image: "quay.io/cilium/cilium:v1.18.4@sha256:49d87af187eeeb9e9e3ec2bc6bd372261a0b5cb2d845659463ba7cc10fe9e45f" - imagePullPolicy: IfNotPresent - command: - - cilium-dbg - - build-config - env: - - name: K8S_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: CILIUM_K8S_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: KUBERNETES_SERVICE_HOST - value: "2620:11f:7001:7:ffff:ffff:ad7:1dd" - - name: KUBERNETES_SERVICE_PORT - value: "6443" - volumeMounts: - - name: tmp - mountPath: /tmp - terminationMessagePolicy: FallbackToLogsOnError - # Required to mount cgroup2 filesystem on the underlying Kubernetes node. - # We use nsenter command with host's cgroup and mount namespaces enabled. - - name: mount-cgroup - image: "quay.io/cilium/cilium:v1.18.4@sha256:49d87af187eeeb9e9e3ec2bc6bd372261a0b5cb2d845659463ba7cc10fe9e45f" - imagePullPolicy: IfNotPresent - env: - - name: CGROUP_ROOT - value: /run/cilium/cgroupv2 - - name: BIN_PATH - value: /opt/cni/bin - command: - - sh - - -ec - # The statically linked Go program binary is invoked to avoid any - # dependency on utilities like sh and mount that can be missing on certain - # distros installed on the underlying host. Copy the binary to the - # same directory where we install cilium cni plugin so that exec permissions - # are available. - - | - cp /usr/bin/cilium-mount /hostbin/cilium-mount; - nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-mount" $CGROUP_ROOT; - rm /hostbin/cilium-mount - volumeMounts: - - name: hostproc - mountPath: /hostproc - - name: cni-path - mountPath: /hostbin - terminationMessagePolicy: FallbackToLogsOnError - securityContext: - seLinuxOptions: - level: s0 - type: spc_t - capabilities: - add: - - SYS_ADMIN - - SYS_CHROOT - - SYS_PTRACE - drop: - - ALL - - name: apply-sysctl-overwrites - image: "quay.io/cilium/cilium:v1.18.4@sha256:49d87af187eeeb9e9e3ec2bc6bd372261a0b5cb2d845659463ba7cc10fe9e45f" - imagePullPolicy: IfNotPresent - env: - - name: BIN_PATH - value: /opt/cni/bin - command: - - sh - - -ec - # The statically linked Go program binary is invoked to avoid any - # dependency on utilities like sh that can be missing on certain - # distros installed on the underlying host. Copy the binary to the - # same directory where we install cilium cni plugin so that exec permissions - # are available. - - | - cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix; - nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix"; - rm /hostbin/cilium-sysctlfix - volumeMounts: - - name: hostproc - mountPath: /hostproc - - name: cni-path - mountPath: /hostbin - terminationMessagePolicy: FallbackToLogsOnError - securityContext: - seLinuxOptions: - level: s0 - type: spc_t - capabilities: - add: - - SYS_ADMIN - - SYS_CHROOT - - SYS_PTRACE - drop: - - ALL - # Mount the bpf fs if it is not mounted. We will perform this task - # from a privileged container because the mount propagation bidirectional - # only works from privileged containers. - - name: mount-bpf-fs - image: "quay.io/cilium/cilium:v1.18.4@sha256:49d87af187eeeb9e9e3ec2bc6bd372261a0b5cb2d845659463ba7cc10fe9e45f" - imagePullPolicy: IfNotPresent - args: - - 'mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf' - command: - - /bin/bash - - -c - - -- - terminationMessagePolicy: FallbackToLogsOnError - securityContext: - privileged: true - volumeMounts: - - name: bpf-maps - mountPath: /sys/fs/bpf - mountPropagation: Bidirectional - - name: clean-cilium-state - image: "quay.io/cilium/cilium:v1.18.4@sha256:49d87af187eeeb9e9e3ec2bc6bd372261a0b5cb2d845659463ba7cc10fe9e45f" - imagePullPolicy: IfNotPresent - command: - - /init-container.sh - env: - - name: CILIUM_ALL_STATE - valueFrom: - configMapKeyRef: - name: cilium-config - key: clean-cilium-state - optional: true - - name: CILIUM_BPF_STATE - valueFrom: - configMapKeyRef: - name: cilium-config - key: clean-cilium-bpf-state - optional: true - - name: WRITE_CNI_CONF_WHEN_READY - valueFrom: - configMapKeyRef: - name: cilium-config - key: write-cni-conf-when-ready - optional: true - - name: KUBERNETES_SERVICE_HOST - value: "2620:11f:7001:7:ffff:ffff:ad7:1dd" - - name: KUBERNETES_SERVICE_PORT - value: "6443" - terminationMessagePolicy: FallbackToLogsOnError - securityContext: - seLinuxOptions: - level: s0 - type: spc_t - capabilities: - add: - - NET_ADMIN - - SYS_MODULE - - SYS_ADMIN - - SYS_RESOURCE - drop: - - ALL - volumeMounts: - - name: bpf-maps - mountPath: /sys/fs/bpf - # Required to mount cgroup filesystem from the host to cilium agent pod - - name: cilium-cgroup - mountPath: /run/cilium/cgroupv2 - mountPropagation: HostToContainer - - name: cilium-run - mountPath: /var/run/cilium # wait-for-kube-proxy - # Install the CNI binaries in an InitContainer so we don't have a writable host mount in the agent - - name: install-cni-binaries - image: "quay.io/cilium/cilium:v1.18.4@sha256:49d87af187eeeb9e9e3ec2bc6bd372261a0b5cb2d845659463ba7cc10fe9e45f" - imagePullPolicy: IfNotPresent - command: - - "/install-plugin.sh" - resources: - requests: - cpu: 100m - memory: 10Mi - securityContext: - seLinuxOptions: - level: s0 - type: spc_t - capabilities: - drop: - - ALL - terminationMessagePolicy: FallbackToLogsOnError - volumeMounts: - - name: cni-path - mountPath: /host/opt/cni/bin # .Values.cni.install - restartPolicy: Always - priorityClassName: system-node-critical - serviceAccountName: "cilium" - automountServiceAccountToken: true - terminationGracePeriodSeconds: 1 - hostNetwork: true - - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - k8s-app: cilium - topologyKey: kubernetes.io/hostname - nodeSelector: - kubernetes.io/os: linux - tolerations: - - operator: Exists - volumes: - # For sharing configuration between the "config" initContainer and the agent - - name: tmp - emptyDir: {} - # To keep state between restarts / upgrades - - name: cilium-run - hostPath: - path: /var/run/cilium - type: DirectoryOrCreate - # To exec into pod network namespaces - - name: cilium-netns - hostPath: - path: /var/run/netns - type: DirectoryOrCreate - # To keep state between restarts / upgrades for bpf maps - - name: bpf-maps - hostPath: - path: /sys/fs/bpf - type: DirectoryOrCreate - # To mount cgroup2 filesystem on the host or apply sysctlfix - - name: hostproc - hostPath: - path: /proc - type: Directory - # To keep state between restarts / upgrades for cgroup2 filesystem - - name: cilium-cgroup - hostPath: - path: /run/cilium/cgroupv2 - type: DirectoryOrCreate - # To install cilium cni plugin in the host - - name: cni-path - hostPath: - path: /opt/cni/bin - type: DirectoryOrCreate - # To install cilium cni configuration in the host - - name: etc-cni-netd - hostPath: - path: /etc/cni/net.d - type: DirectoryOrCreate - # To be able to load kernel modules - - name: lib-modules - hostPath: - path: /lib/modules - # To access iptables concurrently with other processes (e.g. kube-proxy) - - name: xtables-lock - hostPath: - path: /run/xtables.lock - type: FileOrCreate - # Sharing socket with Cilium Envoy on the same node by using a host path - - name: envoy-sockets - hostPath: - path: "/var/run/cilium/envoy/sockets" - type: DirectoryOrCreate - # To read the clustermesh configuration - - name: clustermesh-secrets - projected: - # note: the leading zero means this number is in octal representation: do not remove it - defaultMode: 0400 - sources: - - secret: - name: cilium-clustermesh - optional: true - # note: items are not explicitly listed here, since the entries of this secret - # depend on the peers configured, and that would cause a restart of all agents - # at every addition/removal. Leaving the field empty makes each secret entry - # to be automatically projected into the volume as a file whose name is the key. - - secret: - name: clustermesh-apiserver-remote-cert - optional: true - items: - - key: tls.key - path: common-etcd-client.key - - key: tls.crt - path: common-etcd-client.crt - - key: ca.crt - path: common-etcd-client-ca.crt - # note: we configure the volume for the kvstoremesh-specific certificate - # regardless of whether KVStoreMesh is enabled or not, so that it can be - # automatically mounted in case KVStoreMesh gets subsequently enabled, - # without requiring an agent restart. - - secret: - name: clustermesh-apiserver-local-cert - optional: true - items: - - key: tls.key - path: local-etcd-client.key - - key: tls.crt - path: local-etcd-client.crt - - key: ca.crt - path: local-etcd-client-ca.crt - - name: host-proc-sys-net - hostPath: - path: /proc/sys/net - type: Directory - - name: host-proc-sys-kernel - hostPath: - path: /proc/sys/kernel - type: Directory - - name: hubble-tls - projected: - # note: the leading zero means this number is in octal representation: do not remove it - defaultMode: 0400 - sources: - - secret: - name: hubble-server-certs - optional: true - items: - - key: tls.crt - path: server.crt - - key: tls.key - path: server.key - - key: ca.crt - path: client-ca.crt diff --git a/nix/kubernetes/bootstrap/cilium/cilium-agent/role.yaml b/nix/kubernetes/bootstrap/cilium/cilium-agent/role.yaml deleted file mode 100644 index 9153be3..0000000 --- a/nix/kubernetes/bootstrap/cilium/cilium-agent/role.yaml +++ /dev/null @@ -1,36 +0,0 @@ ---- -# Source: cilium/templates/cilium-agent/role.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: cilium-config-agent - namespace: kube-system - labels: - app.kubernetes.io/part-of: cilium -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch ---- -# Source: cilium/templates/cilium-agent/role.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: cilium-tlsinterception-secrets - namespace: "cilium-secrets" - labels: - app.kubernetes.io/part-of: cilium -rules: -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - watch diff --git a/nix/kubernetes/bootstrap/cilium/cilium-agent/rolebinding.yaml b/nix/kubernetes/bootstrap/cilium/cilium-agent/rolebinding.yaml deleted file mode 100644 index 9e206ab..0000000 --- a/nix/kubernetes/bootstrap/cilium/cilium-agent/rolebinding.yaml +++ /dev/null @@ -1,34 +0,0 @@ ---- -# Source: cilium/templates/cilium-agent/rolebinding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: cilium-config-agent - namespace: kube-system - labels: - app.kubernetes.io/part-of: cilium -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: cilium-config-agent -subjects: - - kind: ServiceAccount - name: "cilium" - namespace: kube-system ---- -# Source: cilium/templates/cilium-agent/rolebinding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: cilium-tlsinterception-secrets - namespace: "cilium-secrets" - labels: - app.kubernetes.io/part-of: cilium -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: cilium-tlsinterception-secrets -subjects: -- kind: ServiceAccount - name: "cilium" - namespace: kube-system diff --git a/nix/kubernetes/bootstrap/cilium/cilium-agent/serviceaccount.yaml b/nix/kubernetes/bootstrap/cilium/cilium-agent/serviceaccount.yaml deleted file mode 100644 index 4880adb..0000000 --- a/nix/kubernetes/bootstrap/cilium/cilium-agent/serviceaccount.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -# Source: cilium/templates/cilium-agent/serviceaccount.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: "cilium" - namespace: kube-system diff --git a/nix/kubernetes/bootstrap/cilium/cilium-ca-secret.yaml b/nix/kubernetes/bootstrap/cilium/cilium-ca-secret.yaml deleted file mode 100644 index 05b06fa..0000000 --- a/nix/kubernetes/bootstrap/cilium/cilium-ca-secret.yaml +++ /dev/null @@ -1,10 +0,0 @@ ---- -# Source: cilium/templates/cilium-ca-secret.yaml -apiVersion: v1 -kind: Secret -metadata: - name: cilium-ca - namespace: kube-system -data: - ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURFekNDQWZ1Z0F3SUJBZ0lRVFl2T2xxVU5TdVlFcGpzamtlaSsvREFOQmdrcWhraUc5dzBCQVFzRkFEQVUKTVJJd0VBWURWUVFERXdsRGFXeHBkVzBnUTBFd0hoY05NalV4TWpFNU1ERXhPREEwV2hjTk1qZ3hNakU0TURFeApPREEwV2pBVU1SSXdFQVlEVlFRREV3bERhV3hwZFcwZ1EwRXdnZ0VpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElCCkR3QXdnZ0VLQW9JQkFRRFNDeEd3cDhaU3Q1Y29RMjNnZ285ZjYvUUE3cUthcnR3bXhlTHkyRjlTcnU2YWNodTUKdURCWm1memtGeDJhRXp6ek94U0REclAvaFFBQUtiSG9kWkVJQ2JkZnY1bzVqanduY0xaSUFMVlZjMlRBeGsxcAphMHVkQVNkVFByYS8rcFRvbjNpeW9LV0JFc2VqY2FXNU1XckFvc1JhaTlLaHl3MTRxSnlsTC9sdDBxVWorQVNaCkxTTndEeU5CK3RlUFhxc0l1VWRmOHcyNlJHUTVlbTZnblNPYmFYZnU5SUhkRDRZYnhQTW9kbkp6dUl3cUFBS1QKOTFqZVVjYkN0Y1Q0UDRTb3RzM1RteXl0Q2VRd1FRclRSY2tIYSt0RmErbnRhVnhIalExSE1GM0pWOFIrK1MvcQpSellMWElaZjR3d2t5SDVFZVJoSjU3WXpBVG1ibFI3SFZobkRBZ01CQUFHallUQmZNQTRHQTFVZER3RUIvd1FFCkF3SUNwREFkQmdOVkhTVUVGakFVQmdnckJnRUZCUWNEQVFZSUt3WUJCUVVIQXdJd0R3WURWUjBUQVFIL0JBVXcKQXdFQi96QWRCZ05WSFE0RUZnUVV3bGd4ZC9qa0FGMTZ4ak52VUhPdExLSGIvMW93RFFZSktvWklodmNOQVFFTApCUUFEZ2dFQkFFbkFvdTArRU9DK1A5YXNGdVJmNy90cHk3UDZoR09vSzZjbElpaFo0UXdpcU82RGdrQ2Frd2ZQCnVCVURYZXZERCt0M3FTYkxOZ3JyNDdWS3R2Qnc5QzhCR3NFZU1vS0w2RThYUlRZSTk5VzBWUmo3MDZHRWsxd0wKSzZXM0R4V251TXRqMHJEWi90VWpDVzhvdkg4SDQyRDhmRHNsT3dSY0NqZ3pMQk5YTXduOFBLdEN6VVRFQzRJZgpFcEZVTXhKOFg0MW5PQnNFdEdkTExEQWpmcjJ3Z0tNWWpsVldtSkE1ZW40WVhtU2hJTmZJZURheDd2WExwZk9OCnVsMmZVaHYxaWVucDlTVjF4d3V2RGxkcFdOQWF2L0dPTjc3MUN5VTRUMzZRS1JzQkptQ0RrQW1iSXloMEdvNEEKQ3NJZXVwYlJCZEd1ck4rbjZYRWtrMFBrWW9DU1JkST0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= - ca.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBMGdzUnNLZkdVcmVYS0VOdDRJS1BYK3YwQU82aW1xN2NKc1hpOHRoZlVxN3VtbkliCnViZ3dXWm44NUJjZG1oTTg4enNVZ3c2ei80VUFBQ214NkhXUkNBbTNYNythT1k0OEozQzJTQUMxVlhOa3dNWk4KYVd0TG5RRW5VejYydi9xVTZKOTRzcUNsZ1JMSG8zR2x1VEZxd0tMRVdvdlNvY3NOZUtpY3BTLzViZEtsSS9nRQptUzBqY0E4alFmclhqMTZyQ0xsSFgvTU51a1JrT1hwdW9KMGptMmwzN3ZTQjNRK0dHOFR6S0haeWM3aU1LZ0FDCmsvZFkzbEhHd3JYRStEK0VxTGJOMDVzc3JRbmtNRUVLMDBYSkIydnJSV3ZwN1dsY1I0ME5SekJkeVZmRWZ2a3YKNmtjMkMxeUdYK01NSk1oK1JIa1lTZWUyTXdFNW01VWV4MVlad3dJREFRQUJBb0lCQUJVK3dHQjZ2OXZGclJqZgpUOHhVZUdCajZJVEs4N3NyUjZKTHJzRVFjMUovZ05hSFhJTGtVZmo3ZmR4V1NEZGtPR3lEdzh5SkhxOWxpbEtrCmo2c0FiK3NudHhxM1hIS2VPVHNRc3FkRmNieW9haW1qaFMzNHNuME96UHI3RGhkaC9GRFdoeW40M01vRVFSbCsKUThjZndVNmx5QVdUNTA5WHowK0QxOVdrMTI1WG44ODZDbkdVZExaZUtZZzNsMUxjTDJLWlJEK2ZSa0VHUUVIeQoyN2JDbmFHd1lzZDByQ1V4N0Z3KzdSZ3RBYzdLSUgyY3lockxTVjFLcjByQUE5RVJRRmNuZG8xdHB5bUpNbTR6CjNsclF2MWpOdGcxVDdTSHBKYXF0K0UxL21ObFpDYjBsSVR6dG5DYWFMQkRCSjNuQkpzS2RDLzBaUFFNVnVyUngKMXRIUEpXVUNnWUVBMCtqMFZ1NzErK0duODdqV0dtWXRIR3FhNW5aR2MyYmRCNnR4YVVUQzJVaGtqOWRISnRUcgp6Ykk2aUFNbTg0NjI3ZkxtVnhGT1lYYXJJMEZsb1I4aHNqYjAzYks1Wmh5Vlc5dTY2aDZhNnAzWFRqcittSEFiCkZJYTNXUXZ6VXdjOVRaSi9RWkRkcTdpcEVEWVhmRXY3Y3EzWk9JKzgwRnRjT0J3bzNaTzRpZmNDZ1lFQS9iNnYKallYTkxCZ3BsNHhZR2M5bldCaUNndFpyOCtRekhrUnNNZkpNMjdhemRCNXJCNXROT0pBOUtSOVNGWFBwb1pLQgo2RmE2ZmsrOHdmOWFxVUxpcEVrRm5kWFByREZtU2pCN0Q2ZWFrbG1hVW1BaFZpNTZ6aUQvWm9tTWtqcmRDWlJSCisrcE1SVGYvQzdGc2NtYytLcXFrbzdOenNuVy9uMkVJeVlwMVc1VUNnWUJJcDhpT3ZlT3M5dE9rNXF6UjVGSjMKT2IzZVlwTUpJaTJLWWFmQzFnYVFoUmVsa2NRZGRrZGJBVTY4TDRoOVhXTXU5bWN2VndtdXRRYzhVVUhOR21WdApPeFo5cExlWVlSaDhwRHZUNWFacjVxNVpialM0ZzBkbHBFTGN6eElnVjQwWE9iRlVBNTFkTVRVV0Q5WEJrak9tClFDRUlHWkE5Ui9XNGJ6ODdxVmhPUlFLQmdRQzFrREl4ZytJeGdRQ1J6ZnhrUzRIWkNZQ3BlaUE0bmJydUYydUwKdWFMQlBGUGY3THdNNzFVcitobXBTUjRFOTdIZXlPUm5pVmRjZGxYTVFwbHVyOHRZWGwvRWRtOW8rTmdHa2ZZYwpWNE5FNFJTSmlQdVJuU1NHUmhvNkZvWWRSRDFSVTIzdVlkSTlYVG9EOWVweFdlcWp3UUtabUJoYlErSGZleUU4CmhQUkFtUUtCZ1FDR2RyY251THVVb2tlNTk5WnhxWE5jUnlXRjJqMDdFWHpDSnlpUG93ejdINzBFV2EyRkQ4VXQKTm53aGw2MFhhclJEeWFwSlVTRFhsOGh3c3lYVFVWYXhoeDBrSHlVcE9iSG55Uk5TRjhnODYveUoyOEF4UE44WApoSnlxaERMVlBWZ09OU0huSTF4YkE5ZU9JQWk3QWhxRnp6QWNpUVJudHFoNXVmYzNwVnFCRUE9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo= diff --git a/nix/kubernetes/bootstrap/cilium/cilium-configmap.yaml b/nix/kubernetes/bootstrap/cilium/cilium-configmap.yaml deleted file mode 100644 index f84524f..0000000 --- a/nix/kubernetes/bootstrap/cilium/cilium-configmap.yaml +++ /dev/null @@ -1,259 +0,0 @@ ---- -# Source: cilium/templates/cilium-configmap.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: cilium-config - namespace: kube-system -data: - - # Identity allocation mode selects how identities are shared between cilium - # nodes by setting how they are stored. The options are "crd", "kvstore" or - # "doublewrite-readkvstore" / "doublewrite-readcrd". - # - "crd" stores identities in kubernetes as CRDs (custom resource definition). - # These can be queried with: - # kubectl get ciliumid - # - "kvstore" stores identities in an etcd kvstore, that is - # configured below. Cilium versions before 1.6 supported only the kvstore - # backend. Upgrades from these older cilium versions should continue using - # the kvstore by commenting out the identity-allocation-mode below, or - # setting it to "kvstore". - # - "doublewrite" modes store identities in both the kvstore and CRDs. This is useful - # for seamless migrations from the kvstore mode to the crd mode. Consult the - # documentation for more information on how to perform the migration. - identity-allocation-mode: crd - - identity-heartbeat-timeout: "30m0s" - identity-gc-interval: "15m0s" - cilium-endpoint-gc-interval: "5m0s" - nodes-gc-interval: "5m0s" - - # If you want to run cilium in debug mode change this value to true - debug: "false" - debug-verbose: "" - metrics-sampling-interval: "5m" - # The agent can be put into the following three policy enforcement modes - # default, always and never. - # https://docs.cilium.io/en/latest/security/policy/intro/#policy-enforcement-modes - enable-policy: "default" - policy-cidr-match-mode: "" - # If you want metrics enabled in cilium-operator, set the port for - # which the Cilium Operator will have their metrics exposed. - # NOTE that this will open the port on the nodes where Cilium operator pod - # is scheduled. - operator-prometheus-serve-addr: ":9963" - enable-metrics: "true" - enable-policy-secrets-sync: "true" - policy-secrets-only-from-secrets-namespace: "true" - policy-secrets-namespace: "cilium-secrets" - - # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 - # address. - enable-ipv4: "true" - - # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6 - # address. - enable-ipv6: "true" - # Users who wish to specify their own custom CNI configuration file must set - # custom-cni-conf to "true", otherwise Cilium may overwrite the configuration. - custom-cni-conf: "false" - enable-bpf-clock-probe: "false" - # If you want cilium monitor to aggregate tracing for packets, set this level - # to "low", "medium", or "maximum". The higher the level, the less packets - # that will be seen in monitor output. - monitor-aggregation: medium - - # The monitor aggregation interval governs the typical time between monitor - # notification events for each allowed connection. - # - # Only effective when monitor aggregation is set to "medium" or higher. - monitor-aggregation-interval: "5s" - - # The monitor aggregation flags determine which TCP flags which, upon the - # first observation, cause monitor notifications to be generated. - # - # Only effective when monitor aggregation is set to "medium" or higher. - monitor-aggregation-flags: all - # Specifies the ratio (0.0-1.0] of total system memory to use for dynamic - # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps. - bpf-map-dynamic-size-ratio: "0.0025" - # bpf-policy-map-max specifies the maximum number of entries in endpoint - # policy map (per endpoint) - bpf-policy-map-max: "16384" - # bpf-policy-stats-map-max specifies the maximum number of entries in global - # policy stats map - bpf-policy-stats-map-max: "65536" - # bpf-lb-map-max specifies the maximum number of entries in bpf lb service, - # backend and affinity maps. - bpf-lb-map-max: "65536" - bpf-lb-external-clusterip: "false" - bpf-lb-source-range-all-types: "false" - bpf-lb-algorithm-annotation: "false" - bpf-lb-mode-annotation: "false" - - bpf-distributed-lru: "false" - bpf-events-drop-enabled: "true" - bpf-events-policy-verdict-enabled: "true" - bpf-events-trace-enabled: "true" - - # Pre-allocation of map entries allows per-packet latency to be reduced, at - # the expense of up-front memory allocation for the entries in the maps. The - # default value below will minimize memory usage in the default installation; - # users who are sensitive to latency may consider setting this to "true". - # - # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore - # this option and behave as though it is set to "true". - # - # If this value is modified, then during the next Cilium startup the restore - # of existing endpoints and tracking of ongoing connections may be disrupted. - # As a result, reply packets may be dropped and the load-balancing decisions - # for established connections may change. - # - # If this option is set to "false" during an upgrade from 1.3 or earlier to - # 1.4 or later, then it may cause one-time disruptions during the upgrade. - preallocate-bpf-maps: "false" - - # Name of the cluster. Only relevant when building a mesh of clusters. - cluster-name: "default" - # Unique ID of the cluster. Must be unique across all conneted clusters and - # in the range of 1 and 255. Only relevant when building a mesh of clusters. - cluster-id: "0" - - # Encapsulation mode for communication between nodes - # Possible values: - # - disabled - # - vxlan (default) - # - geneve - - routing-mode: "tunnel" - tunnel-protocol: "vxlan" - tunnel-source-port-range: "0-0" - service-no-backend-response: "reject" - - - # Enables L7 proxy for L7 policy enforcement and visibility - enable-l7-proxy: "true" - enable-ipv4-masquerade: "true" - enable-ipv4-big-tcp: "false" - enable-ipv6-big-tcp: "false" - enable-ipv6-masquerade: "true" - enable-tcx: "true" - datapath-mode: "veth" - enable-masquerade-to-route-source: "false" - - enable-xt-socket-fallback: "true" - install-no-conntrack-iptables-rules: "false" - iptables-random-fully: "false" - - auto-direct-node-routes: "false" - direct-routing-skip-unreachable: "false" - - - - kube-proxy-replacement: "true" - kube-proxy-replacement-healthz-bind-address: "" - bpf-lb-sock: "false" - nodeport-addresses: "" - enable-health-check-nodeport: "true" - enable-health-check-loadbalancer-ip: "false" - node-port-bind-protection: "true" - enable-auto-protect-node-port-range: "true" - bpf-lb-acceleration: "disabled" - enable-svc-source-range-check: "true" - enable-l2-neigh-discovery: "false" - k8s-require-ipv4-pod-cidr: "false" - k8s-require-ipv6-pod-cidr: "false" - enable-k8s-networkpolicy: "true" - enable-endpoint-lockdown-on-policy-overflow: "false" - # Tell the agent to generate and write a CNI configuration file - write-cni-conf-when-ready: /host/etc/cni/net.d/05-cilium.conflist - cni-exclusive: "true" - cni-log-file: "/var/run/cilium/cilium-cni.log" - enable-endpoint-health-checking: "true" - enable-health-checking: "true" - health-check-icmp-failure-threshold: "3" - enable-well-known-identities: "false" - enable-node-selector-labels: "false" - synchronize-k8s-nodes: "true" - operator-api-serve-addr: "127.0.0.1:9234" - - enable-hubble: "true" - # UNIX domain socket for Hubble server to listen to. - hubble-socket-path: "/var/run/cilium/hubble.sock" - hubble-network-policy-correlation-enabled: "true" - # An additional address for Hubble server to listen to (e.g. ":4244"). - hubble-listen-address: ":4244" - hubble-disable-tls: "false" - hubble-tls-cert-file: /var/lib/cilium/tls/hubble/server.crt - hubble-tls-key-file: /var/lib/cilium/tls/hubble/server.key - hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/client-ca.crt - ipam: "cluster-pool" - ipam-cilium-node-update-rate: "15s" - cluster-pool-ipv4-cidr: "10.0.0.0/8" - cluster-pool-ipv4-mask-size: "24" - cluster-pool-ipv6-cidr: "fd00::/104" - cluster-pool-ipv6-mask-size: "120" - - default-lb-service-ipam: "lbipam" - egress-gateway-reconciliation-trigger-interval: "1s" - enable-vtep: "false" - vtep-endpoint: "" - vtep-cidr: "" - vtep-mask: "" - vtep-mac: "" - procfs: "/host/proc" - bpf-root: "/sys/fs/bpf" - cgroup-root: "/run/cilium/cgroupv2" - - identity-management-mode: "agent" - enable-sctp: "false" - remove-cilium-node-taints: "true" - set-cilium-node-taints: "true" - set-cilium-is-up-condition: "true" - unmanaged-pod-watcher-interval: "15" - # default DNS proxy to transparent mode in non-chaining modes - dnsproxy-enable-transparent-mode: "true" - dnsproxy-socket-linger-timeout: "10" - tofqdns-dns-reject-response-code: "refused" - tofqdns-enable-dns-compression: "true" - tofqdns-endpoint-max-ip-per-hostname: "1000" - tofqdns-idle-connection-grace-period: "0s" - tofqdns-max-deferred-connection-deletes: "10000" - tofqdns-proxy-response-max-delay: "100ms" - tofqdns-preallocate-identities: "true" - agent-not-ready-taint-key: "node.cilium.io/agent-not-ready" - - mesh-auth-enabled: "true" - mesh-auth-queue-size: "1024" - mesh-auth-rotated-identities-queue-size: "1024" - mesh-auth-gc-interval: "5m0s" - - proxy-xff-num-trusted-hops-ingress: "0" - proxy-xff-num-trusted-hops-egress: "0" - proxy-connect-timeout: "2" - proxy-initial-fetch-timeout: "30" - proxy-max-requests-per-connection: "0" - proxy-max-connection-duration-seconds: "0" - proxy-idle-timeout-seconds: "60" - proxy-max-concurrent-retries: "128" - http-retry-count: "3" - http-stream-idle-timeout: "300" - - external-envoy-proxy: "true" - envoy-base-id: "0" - envoy-access-log-buffer-size: "4096" - envoy-keep-cap-netbindservice: "false" - max-connected-clusters: "255" - clustermesh-enable-endpoint-sync: "false" - clustermesh-enable-mcs-api: "false" - policy-default-local-cluster: "false" - - nat-map-stats-entries: "32" - nat-map-stats-interval: "30s" - enable-internal-traffic-policy: "true" - enable-lb-ipam: "true" - enable-non-default-deny-policies: "true" - enable-source-ip-verification: "true" - -# Extra config allows adding arbitrary properties to the cilium config. -# By putting it at the end of the ConfigMap, it's also possible to override existing properties. diff --git a/nix/kubernetes/bootstrap/cilium/cilium-envoy/configmap.yaml b/nix/kubernetes/bootstrap/cilium/cilium-envoy/configmap.yaml deleted file mode 100644 index 8cd818b..0000000 --- a/nix/kubernetes/bootstrap/cilium/cilium-envoy/configmap.yaml +++ /dev/null @@ -1,11 +0,0 @@ ---- -# Source: cilium/templates/cilium-envoy/configmap.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: cilium-envoy-config - namespace: kube-system -data: - # Keep the key name as bootstrap-config.json to avoid breaking changes - bootstrap-config.json: | - {"admin":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/admin.sock"}}},"applicationLogConfig":{"logFormat":{"textFormat":"[%Y-%m-%d %T.%e][%t][%l][%n] [%g:%#] %v"}},"bootstrapExtensions":[{"name":"envoy.bootstrap.internal_listener","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener"}}],"dynamicResources":{"cdsConfig":{"apiConfigSource":{"apiType":"GRPC","grpcServices":[{"envoyGrpc":{"clusterName":"xds-grpc-cilium"}}],"setNodeOnFirstMessageOnly":true,"transportApiVersion":"V3"},"initialFetchTimeout":"30s","resourceApiVersion":"V3"},"ldsConfig":{"apiConfigSource":{"apiType":"GRPC","grpcServices":[{"envoyGrpc":{"clusterName":"xds-grpc-cilium"}}],"setNodeOnFirstMessageOnly":true,"transportApiVersion":"V3"},"initialFetchTimeout":"30s","resourceApiVersion":"V3"}},"node":{"cluster":"ingress-cluster","id":"host~127.0.0.1~no-id~localdomain"},"overloadManager":{"resourceMonitors":[{"name":"envoy.resource_monitors.global_downstream_max_connections","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig","max_active_downstream_connections":"50000"}}]},"staticResources":{"clusters":[{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"ingress-cluster","type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"egress-cluster-tls","transportSocket":{"name":"cilium.tls_wrapper","typedConfig":{"@type":"type.googleapis.com/cilium.UpstreamTlsWrapperContext"}},"type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"upstreamHttpProtocolOptions":{},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"egress-cluster","type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"ingress-cluster-tls","transportSocket":{"name":"cilium.tls_wrapper","typedConfig":{"@type":"type.googleapis.com/cilium.UpstreamTlsWrapperContext"}},"type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"upstreamHttpProtocolOptions":{},"useDownstreamProtocolConfig":{}}}},{"connectTimeout":"2s","loadAssignment":{"clusterName":"xds-grpc-cilium","endpoints":[{"lbEndpoints":[{"endpoint":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/xds.sock"}}}}]}]},"name":"xds-grpc-cilium","type":"STATIC","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","explicitHttpConfig":{"http2ProtocolOptions":{}}}}},{"connectTimeout":"2s","loadAssignment":{"clusterName":"/envoy-admin","endpoints":[{"lbEndpoints":[{"endpoint":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/admin.sock"}}}}]}]},"name":"/envoy-admin","type":"STATIC"}],"listeners":[{"additionalAddresses":[{"address":{"socketAddress":{"address":"::","portValue":9964}}}],"address":{"socketAddress":{"address":"0.0.0.0","portValue":9964}},"filterChains":[{"filters":[{"name":"envoy.filters.network.http_connection_manager","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager","httpFilters":[{"name":"envoy.filters.http.router","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"}}],"internalAddressConfig":{"cidrRanges":[{"addressPrefix":"10.0.0.0","prefixLen":8},{"addressPrefix":"172.16.0.0","prefixLen":12},{"addressPrefix":"192.168.0.0","prefixLen":16},{"addressPrefix":"127.0.0.1","prefixLen":32},{"addressPrefix":"::1","prefixLen":128}]},"routeConfig":{"virtualHosts":[{"domains":["*"],"name":"prometheus_metrics_route","routes":[{"match":{"prefix":"/metrics"},"name":"prometheus_metrics_route","route":{"cluster":"/envoy-admin","prefixRewrite":"/stats/prometheus"}}]}]},"statPrefix":"envoy-prometheus-metrics-listener","streamIdleTimeout":"300s"}}]}],"name":"envoy-prometheus-metrics-listener"},{"additionalAddresses":[{"address":{"socketAddress":{"address":"::1","portValue":9878}}}],"address":{"socketAddress":{"address":"127.0.0.1","portValue":9878}},"filterChains":[{"filters":[{"name":"envoy.filters.network.http_connection_manager","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager","httpFilters":[{"name":"envoy.filters.http.router","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"}}],"internalAddressConfig":{"cidrRanges":[{"addressPrefix":"10.0.0.0","prefixLen":8},{"addressPrefix":"172.16.0.0","prefixLen":12},{"addressPrefix":"192.168.0.0","prefixLen":16},{"addressPrefix":"127.0.0.1","prefixLen":32},{"addressPrefix":"::1","prefixLen":128}]},"routeConfig":{"virtual_hosts":[{"domains":["*"],"name":"health","routes":[{"match":{"prefix":"/healthz"},"name":"health","route":{"cluster":"/envoy-admin","prefixRewrite":"/ready"}}]}]},"statPrefix":"envoy-health-listener","streamIdleTimeout":"300s"}}]}],"name":"envoy-health-listener"}]}} diff --git a/nix/kubernetes/bootstrap/cilium/cilium-envoy/daemonset.yaml b/nix/kubernetes/bootstrap/cilium/cilium-envoy/daemonset.yaml deleted file mode 100644 index 1a855fe..0000000 --- a/nix/kubernetes/bootstrap/cilium/cilium-envoy/daemonset.yaml +++ /dev/null @@ -1,171 +0,0 @@ ---- -# Source: cilium/templates/cilium-envoy/daemonset.yaml -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: cilium-envoy - namespace: kube-system - labels: - k8s-app: cilium-envoy - app.kubernetes.io/part-of: cilium - app.kubernetes.io/name: cilium-envoy - name: cilium-envoy -spec: - selector: - matchLabels: - k8s-app: cilium-envoy - updateStrategy: - rollingUpdate: - maxUnavailable: 2 - type: RollingUpdate - template: - metadata: - annotations: - labels: - k8s-app: cilium-envoy - name: cilium-envoy - app.kubernetes.io/name: cilium-envoy - app.kubernetes.io/part-of: cilium - spec: - securityContext: - appArmorProfile: - type: Unconfined - containers: - - name: cilium-envoy - image: "quay.io/cilium/cilium-envoy:v1.34.10-1762597008-ff7ae7d623be00078865cff1b0672cc5d9bfc6d5@sha256:1deb6709afcb5523579bf1abbc3255adf9e354565a88c4a9162c8d9cb1d77ab5" - imagePullPolicy: IfNotPresent - command: - - /usr/bin/cilium-envoy-starter - args: - - '--' - - '-c /var/run/cilium/envoy/bootstrap-config.json' - - '--base-id 0' - - '--log-level info' - startupProbe: - httpGet: - host: "127.0.0.1" - path: /healthz - port: 9878 - scheme: HTTP - failureThreshold: 105 - periodSeconds: 2 - successThreshold: 1 - initialDelaySeconds: 5 - livenessProbe: - httpGet: - host: "127.0.0.1" - path: /healthz - port: 9878 - scheme: HTTP - periodSeconds: 30 - successThreshold: 1 - failureThreshold: 10 - timeoutSeconds: 5 - readinessProbe: - httpGet: - host: "127.0.0.1" - path: /healthz - port: 9878 - scheme: HTTP - periodSeconds: 30 - successThreshold: 1 - failureThreshold: 3 - timeoutSeconds: 5 - env: - - name: K8S_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: CILIUM_K8S_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: KUBERNETES_SERVICE_HOST - value: "2620:11f:7001:7:ffff:ffff:ad7:1dd" - - name: KUBERNETES_SERVICE_PORT - value: "6443" - ports: - - name: envoy-metrics - containerPort: 9964 - hostPort: 9964 - protocol: TCP - securityContext: - seLinuxOptions: - level: s0 - type: spc_t - capabilities: - add: - - NET_ADMIN - - SYS_ADMIN - drop: - - ALL - terminationMessagePolicy: FallbackToLogsOnError - volumeMounts: - - name: envoy-sockets - mountPath: /var/run/cilium/envoy/sockets - readOnly: false - - name: envoy-artifacts - mountPath: /var/run/cilium/envoy/artifacts - readOnly: true - - name: envoy-config - mountPath: /var/run/cilium/envoy/ - readOnly: true - - name: bpf-maps - mountPath: /sys/fs/bpf - mountPropagation: HostToContainer - restartPolicy: Always - priorityClassName: system-node-critical - serviceAccountName: "cilium-envoy" - automountServiceAccountToken: true - terminationGracePeriodSeconds: 1 - hostNetwork: true - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: cilium.io/no-schedule - operator: NotIn - values: - - "true" - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - k8s-app: cilium - topologyKey: kubernetes.io/hostname - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - k8s-app: cilium-envoy - topologyKey: kubernetes.io/hostname - nodeSelector: - kubernetes.io/os: linux - tolerations: - - operator: Exists - volumes: - - name: envoy-sockets - hostPath: - path: "/var/run/cilium/envoy/sockets" - type: DirectoryOrCreate - - name: envoy-artifacts - hostPath: - path: "/var/run/cilium/envoy/artifacts" - type: DirectoryOrCreate - - name: envoy-config - configMap: - name: "cilium-envoy-config" - # note: the leading zero means this number is in octal representation: do not remove it - defaultMode: 0400 - items: - - key: bootstrap-config.json - path: bootstrap-config.json - # To keep state between restarts / upgrades - # To keep state between restarts / upgrades for bpf maps - - name: bpf-maps - hostPath: - path: /sys/fs/bpf - type: DirectoryOrCreate diff --git a/nix/kubernetes/bootstrap/cilium/cilium-envoy/service.yaml b/nix/kubernetes/bootstrap/cilium/cilium-envoy/service.yaml deleted file mode 100644 index 04e9a0f..0000000 --- a/nix/kubernetes/bootstrap/cilium/cilium-envoy/service.yaml +++ /dev/null @@ -1,25 +0,0 @@ ---- -# Source: cilium/templates/cilium-envoy/service.yaml -apiVersion: v1 -kind: Service -metadata: - name: cilium-envoy - namespace: kube-system - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9964" - labels: - k8s-app: cilium-envoy - app.kubernetes.io/name: cilium-envoy - app.kubernetes.io/part-of: cilium - io.cilium/app: proxy -spec: - clusterIP: None - type: ClusterIP - selector: - k8s-app: cilium-envoy - ports: - - name: envoy-metrics - port: 9964 - protocol: TCP - targetPort: envoy-metrics diff --git a/nix/kubernetes/bootstrap/cilium/cilium-envoy/serviceaccount.yaml b/nix/kubernetes/bootstrap/cilium/cilium-envoy/serviceaccount.yaml deleted file mode 100644 index eda0123..0000000 --- a/nix/kubernetes/bootstrap/cilium/cilium-envoy/serviceaccount.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -# Source: cilium/templates/cilium-envoy/serviceaccount.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: "cilium-envoy" - namespace: kube-system diff --git a/nix/kubernetes/bootstrap/cilium/cilium-operator/clusterrole.yaml b/nix/kubernetes/bootstrap/cilium/cilium-operator/clusterrole.yaml deleted file mode 100644 index e3f5294..0000000 --- a/nix/kubernetes/bootstrap/cilium/cilium-operator/clusterrole.yaml +++ /dev/null @@ -1,240 +0,0 @@ ---- -# Source: cilium/templates/cilium-operator/clusterrole.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: cilium-operator - labels: - app.kubernetes.io/part-of: cilium -rules: -- apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - watch - # to automatically delete [core|kube]dns pods so that are starting to being - # managed by Cilium - - delete -- apiGroups: - - "" - resources: - - configmaps - resourceNames: - - cilium-config - verbs: - # allow patching of the configmap to set annotations - - patch -- apiGroups: - - "" - resources: - - nodes - verbs: - - list - - watch -- apiGroups: - - "" - resources: - # To remove node taints - - nodes - # To set NetworkUnavailable false on startup - - nodes/status - verbs: - - patch -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - # to perform LB IP allocation for BGP - - services/status - verbs: - - update - - patch -- apiGroups: - - "" - resources: - # to check apiserver connectivity - - namespaces - - secrets - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - # to perform the translation of a CNP that contains `ToGroup` to its endpoints - - services - - endpoints - verbs: - - get - - list - - watch -- apiGroups: - - cilium.io - resources: - - ciliumnetworkpolicies - - ciliumclusterwidenetworkpolicies - verbs: - # Create auto-generated CNPs and CCNPs from Policies that have 'toGroups' - - create - - update - - deletecollection - # To update the status of the CNPs and CCNPs - - patch - - get - - list - - watch -- apiGroups: - - cilium.io - resources: - - ciliumnetworkpolicies/status - - ciliumclusterwidenetworkpolicies/status - verbs: - # Update the auto-generated CNPs and CCNPs status. - - patch - - update -- apiGroups: - - cilium.io - resources: - - ciliumendpoints - - ciliumidentities - verbs: - # To perform garbage collection of such resources - - delete - - list - - watch -- apiGroups: - - cilium.io - resources: - - ciliumidentities - verbs: - # To synchronize garbage collection of such resources - - update -- apiGroups: - - cilium.io - resources: - - ciliumnodes - verbs: - - create - - update - - get - - list - - watch - # To perform CiliumNode garbage collector - - delete -- apiGroups: - - cilium.io - resources: - - ciliumnodes/status - verbs: - - update -- apiGroups: - - cilium.io - resources: - - ciliumendpointslices - - ciliumenvoyconfigs - - ciliumbgppeerconfigs - - ciliumbgpadvertisements - - ciliumbgpnodeconfigs - verbs: - - create - - update - - get - - list - - watch - - delete - - patch -- apiGroups: - - cilium.io - resources: - - ciliumbgpclusterconfigs/status - - ciliumbgppeerconfigs/status - verbs: - - update -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - create - - get - - list - - watch -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - update - resourceNames: - - ciliumloadbalancerippools.cilium.io - - ciliumbgppeeringpolicies.cilium.io - - ciliumbgpclusterconfigs.cilium.io - - ciliumbgppeerconfigs.cilium.io - - ciliumbgpadvertisements.cilium.io - - ciliumbgpnodeconfigs.cilium.io - - ciliumbgpnodeconfigoverrides.cilium.io - - ciliumclusterwideenvoyconfigs.cilium.io - - ciliumclusterwidenetworkpolicies.cilium.io - - ciliumegressgatewaypolicies.cilium.io - - ciliumendpoints.cilium.io - - ciliumendpointslices.cilium.io - - ciliumenvoyconfigs.cilium.io - - ciliumidentities.cilium.io - - ciliumlocalredirectpolicies.cilium.io - - ciliumnetworkpolicies.cilium.io - - ciliumnodes.cilium.io - - ciliumnodeconfigs.cilium.io - - ciliumcidrgroups.cilium.io - - ciliuml2announcementpolicies.cilium.io - - ciliumpodippools.cilium.io - - ciliumgatewayclassconfigs.cilium.io -- apiGroups: - - cilium.io - resources: - - ciliumloadbalancerippools - - ciliumpodippools - - ciliumbgppeeringpolicies - - ciliumbgpclusterconfigs - - ciliumbgpnodeconfigoverrides - - ciliumbgppeerconfigs - verbs: - - get - - list - - watch -- apiGroups: - - cilium.io - resources: - - ciliumpodippools - verbs: - - create -- apiGroups: - - cilium.io - resources: - - ciliumloadbalancerippools/status - verbs: - - patch -# For cilium-operator running in HA mode. -# -# Cilium operator running in HA mode requires the use of ResourceLock for Leader Election -# between multiple running instances. -# The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less -# common and fewer objects in the cluster watch "all Leases". -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create - - get - - update diff --git a/nix/kubernetes/bootstrap/cilium/cilium-operator/clusterrolebinding.yaml b/nix/kubernetes/bootstrap/cilium/cilium-operator/clusterrolebinding.yaml deleted file mode 100644 index feb5ff3..0000000 --- a/nix/kubernetes/bootstrap/cilium/cilium-operator/clusterrolebinding.yaml +++ /dev/null @@ -1,16 +0,0 @@ ---- -# Source: cilium/templates/cilium-operator/clusterrolebinding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: cilium-operator - labels: - app.kubernetes.io/part-of: cilium -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cilium-operator -subjects: -- kind: ServiceAccount - name: "cilium-operator" - namespace: kube-system diff --git a/nix/kubernetes/bootstrap/cilium/cilium-operator/deployment.yaml b/nix/kubernetes/bootstrap/cilium/cilium-operator/deployment.yaml deleted file mode 100644 index f2d3d43..0000000 --- a/nix/kubernetes/bootstrap/cilium/cilium-operator/deployment.yaml +++ /dev/null @@ -1,140 +0,0 @@ ---- -# Source: cilium/templates/cilium-operator/deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: cilium-operator - namespace: kube-system - labels: - io.cilium/app: operator - name: cilium-operator - app.kubernetes.io/part-of: cilium - app.kubernetes.io/name: cilium-operator -spec: - # See docs on ServerCapabilities.LeasesResourceLock in file pkg/k8s/version/version.go - # for more details. - replicas: 2 - selector: - matchLabels: - io.cilium/app: operator - name: cilium-operator - # ensure operator update on single node k8s clusters, by using rolling update with maxUnavailable=100% in case - # of one replica and no user configured Recreate strategy. - # otherwise an update might get stuck due to the default maxUnavailable=50% in combination with the - # podAntiAffinity which prevents deployments of multiple operator replicas on the same node. - strategy: - rollingUpdate: - maxSurge: 25% - maxUnavailable: 50% - type: RollingUpdate - template: - metadata: - annotations: - prometheus.io/port: "9963" - prometheus.io/scrape: "true" - labels: - io.cilium/app: operator - name: cilium-operator - app.kubernetes.io/part-of: cilium - app.kubernetes.io/name: cilium-operator - spec: - securityContext: - seccompProfile: - type: RuntimeDefault - containers: - - name: cilium-operator - image: "quay.io/cilium/operator-generic:v1.18.4@sha256:1b22b9ff28affdf574378a70dade4ef835b00b080c2ee2418530809dd62c3012" - imagePullPolicy: IfNotPresent - command: - - cilium-operator-generic - args: - - --config-dir=/tmp/cilium/config-map - - --debug=$(CILIUM_DEBUG) - env: - - name: K8S_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: CILIUM_K8S_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: CILIUM_DEBUG - valueFrom: - configMapKeyRef: - key: debug - name: cilium-config - optional: true - - name: KUBERNETES_SERVICE_HOST - value: "2620:11f:7001:7:ffff:ffff:ad7:1dd" - - name: KUBERNETES_SERVICE_PORT - value: "6443" - ports: - - name: prometheus - containerPort: 9963 - hostPort: 9963 - protocol: TCP - livenessProbe: - httpGet: - host: "127.0.0.1" - path: /healthz - port: 9234 - scheme: HTTP - initialDelaySeconds: 60 - periodSeconds: 10 - timeoutSeconds: 3 - readinessProbe: - httpGet: - host: "127.0.0.1" - path: /healthz - port: 9234 - scheme: HTTP - initialDelaySeconds: 0 - periodSeconds: 5 - timeoutSeconds: 3 - failureThreshold: 5 - volumeMounts: - - name: cilium-config-path - mountPath: /tmp/cilium/config-map - readOnly: true - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - terminationMessagePolicy: FallbackToLogsOnError - hostNetwork: true - restartPolicy: Always - priorityClassName: system-cluster-critical - serviceAccountName: "cilium-operator" - automountServiceAccountToken: true - # In HA mode, cilium-operator pods must not be scheduled on the same - # node as they will clash with each other. - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - io.cilium/app: operator - topologyKey: kubernetes.io/hostname - nodeSelector: - kubernetes.io/os: linux - tolerations: - - key: node-role.kubernetes.io/control-plane - operator: Exists - - key: node-role.kubernetes.io/master - operator: Exists - - key: node.kubernetes.io/not-ready - operator: Exists - - key: node.cloudprovider.kubernetes.io/uninitialized - operator: Exists - - key: node.cilium.io/agent-not-ready - operator: Exists - - volumes: - # To read the configuration from the config map - - name: cilium-config-path - configMap: - name: cilium-config diff --git a/nix/kubernetes/bootstrap/cilium/cilium-operator/role.yaml b/nix/kubernetes/bootstrap/cilium/cilium-operator/role.yaml deleted file mode 100644 index 1400c0b..0000000 --- a/nix/kubernetes/bootstrap/cilium/cilium-operator/role.yaml +++ /dev/null @@ -1,19 +0,0 @@ ---- -# Source: cilium/templates/cilium-operator/role.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: cilium-operator-tlsinterception-secrets - namespace: "cilium-secrets" - labels: - app.kubernetes.io/part-of: cilium -rules: -- apiGroups: - - "" - resources: - - secrets - verbs: - - create - - delete - - update - - patch diff --git a/nix/kubernetes/bootstrap/cilium/cilium-operator/rolebinding.yaml b/nix/kubernetes/bootstrap/cilium/cilium-operator/rolebinding.yaml deleted file mode 100644 index ddba34d..0000000 --- a/nix/kubernetes/bootstrap/cilium/cilium-operator/rolebinding.yaml +++ /dev/null @@ -1,17 +0,0 @@ ---- -# Source: cilium/templates/cilium-operator/rolebinding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: cilium-operator-tlsinterception-secrets - namespace: "cilium-secrets" - labels: - app.kubernetes.io/part-of: cilium -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: cilium-operator-tlsinterception-secrets -subjects: -- kind: ServiceAccount - name: "cilium-operator" - namespace: kube-system diff --git a/nix/kubernetes/bootstrap/cilium/cilium-operator/serviceaccount.yaml b/nix/kubernetes/bootstrap/cilium/cilium-operator/serviceaccount.yaml deleted file mode 100644 index d313595..0000000 --- a/nix/kubernetes/bootstrap/cilium/cilium-operator/serviceaccount.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -# Source: cilium/templates/cilium-operator/serviceaccount.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: "cilium-operator" - namespace: kube-system diff --git a/nix/kubernetes/bootstrap/cilium/cilium-secrets-namespace.yaml b/nix/kubernetes/bootstrap/cilium/cilium-secrets-namespace.yaml deleted file mode 100644 index ceb5245..0000000 --- a/nix/kubernetes/bootstrap/cilium/cilium-secrets-namespace.yaml +++ /dev/null @@ -1,9 +0,0 @@ ---- -# Source: cilium/templates/cilium-secrets-namespace.yaml -apiVersion: v1 -kind: Namespace -metadata: - name: "cilium-secrets" - labels: - app.kubernetes.io/part-of: cilium - annotations: diff --git a/nix/kubernetes/bootstrap/cilium/hubble/peer-service.yaml b/nix/kubernetes/bootstrap/cilium/hubble/peer-service.yaml deleted file mode 100644 index 4abab3a..0000000 --- a/nix/kubernetes/bootstrap/cilium/hubble/peer-service.yaml +++ /dev/null @@ -1,21 +0,0 @@ ---- -# Source: cilium/templates/hubble/peer-service.yaml -apiVersion: v1 -kind: Service -metadata: - name: hubble-peer - namespace: kube-system - labels: - k8s-app: cilium - app.kubernetes.io/part-of: cilium - app.kubernetes.io/name: hubble-peer - -spec: - selector: - k8s-app: cilium - ports: - - name: peer-service - port: 443 - protocol: TCP - targetPort: 4244 - internalTrafficPolicy: Local diff --git a/nix/kubernetes/bootstrap/cilium/hubble/tls-helm/server-secret.yaml b/nix/kubernetes/bootstrap/cilium/hubble/tls-helm/server-secret.yaml deleted file mode 100644 index 02cbf1c..0000000 --- a/nix/kubernetes/bootstrap/cilium/hubble/tls-helm/server-secret.yaml +++ /dev/null @@ -1,12 +0,0 @@ ---- -# Source: cilium/templates/hubble/tls-helm/server-secret.yaml -apiVersion: v1 -kind: Secret -metadata: - name: hubble-server-certs - namespace: kube-system -type: kubernetes.io/tls -data: - ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURFekNDQWZ1Z0F3SUJBZ0lRVFl2T2xxVU5TdVlFcGpzamtlaSsvREFOQmdrcWhraUc5dzBCQVFzRkFEQVUKTVJJd0VBWURWUVFERXdsRGFXeHBkVzBnUTBFd0hoY05NalV4TWpFNU1ERXhPREEwV2hjTk1qZ3hNakU0TURFeApPREEwV2pBVU1SSXdFQVlEVlFRREV3bERhV3hwZFcwZ1EwRXdnZ0VpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElCCkR3QXdnZ0VLQW9JQkFRRFNDeEd3cDhaU3Q1Y29RMjNnZ285ZjYvUUE3cUthcnR3bXhlTHkyRjlTcnU2YWNodTUKdURCWm1memtGeDJhRXp6ek94U0REclAvaFFBQUtiSG9kWkVJQ2JkZnY1bzVqanduY0xaSUFMVlZjMlRBeGsxcAphMHVkQVNkVFByYS8rcFRvbjNpeW9LV0JFc2VqY2FXNU1XckFvc1JhaTlLaHl3MTRxSnlsTC9sdDBxVWorQVNaCkxTTndEeU5CK3RlUFhxc0l1VWRmOHcyNlJHUTVlbTZnblNPYmFYZnU5SUhkRDRZYnhQTW9kbkp6dUl3cUFBS1QKOTFqZVVjYkN0Y1Q0UDRTb3RzM1RteXl0Q2VRd1FRclRSY2tIYSt0RmErbnRhVnhIalExSE1GM0pWOFIrK1MvcQpSellMWElaZjR3d2t5SDVFZVJoSjU3WXpBVG1ibFI3SFZobkRBZ01CQUFHallUQmZNQTRHQTFVZER3RUIvd1FFCkF3SUNwREFkQmdOVkhTVUVGakFVQmdnckJnRUZCUWNEQVFZSUt3WUJCUVVIQXdJd0R3WURWUjBUQVFIL0JBVXcKQXdFQi96QWRCZ05WSFE0RUZnUVV3bGd4ZC9qa0FGMTZ4ak52VUhPdExLSGIvMW93RFFZSktvWklodmNOQVFFTApCUUFEZ2dFQkFFbkFvdTArRU9DK1A5YXNGdVJmNy90cHk3UDZoR09vSzZjbElpaFo0UXdpcU82RGdrQ2Frd2ZQCnVCVURYZXZERCt0M3FTYkxOZ3JyNDdWS3R2Qnc5QzhCR3NFZU1vS0w2RThYUlRZSTk5VzBWUmo3MDZHRWsxd0wKSzZXM0R4V251TXRqMHJEWi90VWpDVzhvdkg4SDQyRDhmRHNsT3dSY0NqZ3pMQk5YTXduOFBLdEN6VVRFQzRJZgpFcEZVTXhKOFg0MW5PQnNFdEdkTExEQWpmcjJ3Z0tNWWpsVldtSkE1ZW40WVhtU2hJTmZJZURheDd2WExwZk9OCnVsMmZVaHYxaWVucDlTVjF4d3V2RGxkcFdOQWF2L0dPTjc3MUN5VTRUMzZRS1JzQkptQ0RrQW1iSXloMEdvNEEKQ3NJZXVwYlJCZEd1ck4rbjZYRWtrMFBrWW9DU1JkST0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= - tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURWakNDQWo2Z0F3SUJBZ0lRVVN2aDNTdzFBSXR5MjYvd2JYZ0ZnakFOQmdrcWhraUc5dzBCQVFzRkFEQVUKTVJJd0VBWURWUVFERXdsRGFXeHBkVzBnUTBFd0hoY05NalV4TWpFNU1ERXhPREEwV2hjTk1qWXhNakU1TURFeApPREEwV2pBcU1TZ3dKZ1lEVlFRRERCOHFMbVJsWm1GMWJIUXVhSFZpWW14bExXZHljR011WTJsc2FYVnRMbWx2Ck1JSUJJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdTBPVUd3cU9BN1lWcDE0ZE9GcDYKdjgxaGpPUVNmejl3YXduQUVxSStGQWZ3YXBlbkJ6SlZrbktiNlBYWTUydHNkNU5KTS9YaVFzTWRYYXY2OHJUMwpmRXU4TnlrRVQvWVRIOXpyYkplVndGOVdtTUtKdm9CUjBqVWpLZ3RodmNwK2xPclZoNjJyRHRzbWlIMGtUZTNGCkhYeHlOOTdWMmQxRHNHbGFUc0dVM3B3UzNVNFl3eEJTQWlqOVdhRFprcWJYREZxSUJNQXpjQ1owcWRMTTlXR0cKS1dIbGxwUGdRblNtRnVxWG1TSXFRYndGem14cEt5OXFmMmlndkIwVkIwdDhvSFdVLzY4ckZKQ3d1Q0didncwUApORGpWeVdWdmJoWHZZM0tzWjJCQk45VHdzWkxkWWU4Nkk3cWhyWHVXZWVrN0tCeUpYUkJTQ1BOUnhLZHArMmpxCjl3SURBUUFCbzRHTk1JR0tNQTRHQTFVZER3RUIvd1FFQXdJRm9EQWRCZ05WSFNVRUZqQVVCZ2dyQmdFRkJRY0QKQVFZSUt3WUJCUVVIQXdJd0RBWURWUjBUQVFIL0JBSXdBREFmQmdOVkhTTUVHREFXZ0JUQ1dERjMrT1FBWFhyRwpNMjlRYzYwc29kdi9XakFxQmdOVkhSRUVJekFoZ2g4cUxtUmxabUYxYkhRdWFIVmlZbXhsTFdkeWNHTXVZMmxzCmFYVnRMbWx2TUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFCa3A5a3pwcU1qQ1NxSlRDT3dyZFFMemdRdDlyYmkKSTZUTlFqSVh4T0hjc2JtNFhHNU5LMlBpVS8yQkphamtHMW1NTzVoYkw2aDJzcDM4UklBUTE3Yzg2VWpEM0cvWQppVmFzamhISTBuY2IrMXdvZ3ArYVc3TGYwbGpDMHJQSUZBNjNieGMyYmRzWnkwZmJnUEkxNlBLSjVYWU9SZ0FrCnJnUnZOaTdMQWFlelBNSGUrT0p4WnBLSmNnVWhFQW1aMjFJdVQ3Mm9paWk2NGQ5WVRqTnVvRllBOHg2cjlNSmwKUXNKenBidTAyR0VoZnZjNEwzTzUxOUp2b1VQV0JITHg1VXNqd2tyc05nU3krcDJ2dDQ4Sm0vVUZLa3NyRkNSSgpudTlsa2xTWjZUTnczMjlTQ0lqd3BUVXREZ0QzaHpWRk9oOWkzTFI2UkdrbUdIeFQvME9iMXl3VwotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== - tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBdTBPVUd3cU9BN1lWcDE0ZE9GcDZ2ODFoak9RU2Z6OXdhd25BRXFJK0ZBZndhcGVuCkJ6SlZrbktiNlBYWTUydHNkNU5KTS9YaVFzTWRYYXY2OHJUM2ZFdThOeWtFVC9ZVEg5enJiSmVWd0Y5V21NS0oKdm9CUjBqVWpLZ3RodmNwK2xPclZoNjJyRHRzbWlIMGtUZTNGSFh4eU45N1YyZDFEc0dsYVRzR1UzcHdTM1U0WQp3eEJTQWlqOVdhRFprcWJYREZxSUJNQXpjQ1owcWRMTTlXR0dLV0hsbHBQZ1FuU21GdXFYbVNJcVFid0Z6bXhwCkt5OXFmMmlndkIwVkIwdDhvSFdVLzY4ckZKQ3d1Q0didncwUE5EalZ5V1Z2YmhYdlkzS3NaMkJCTjlUd3NaTGQKWWU4Nkk3cWhyWHVXZWVrN0tCeUpYUkJTQ1BOUnhLZHArMmpxOXdJREFRQUJBb0lCQUFQWnJseGwrYWlLNEdRYQpFOGk2VjRiRkRzbnFVSlVNTldBR0NjeDRTSVY1NmxBT3dURHRNSVE3MWdmU3NqeUQ0VzJOK2pYV3l1bXJnRm1TCk5TMHpXbGxVWjFETkI1a3JTcFpmb1BtUkhsT2Z3alNPMzIwcXowUGVzdFBrK3dQL2UxM05mRWwzWVNxRGlBWEwKYU9rdlI5dkUyTVpjVHhKVk1kU1pQZkdibWxkL1hmN1ppeEpldzhxSERUN2xGQ083ZjU5S3hSaUpLRmlYYjJmcApXRTBYaGsvc3I2L3FwM0E5TEk1VlJxbWVkUTZXZkpScVREWjlzTlNweXMyaWNnMlBXUkZxWXo3djRyTWoyZUpKCjN3Unh6UmIzQkw5NlR6SlpaNFpCck93ZHhna1gxcHhqbmdBZjlvVzJFdnhuZFhiemlZdkh6b0R1NVlZOWF0Y1oKTW9qWlR4RUNnWUVBNjNxVGJwWEUwRWFrM0FDeVBnRXlYcFZQT2dGV2lKZloreTJNVXhrbU13NXZOZVFYN1BBeAp3SzJNY3RVQnpmSzRCY21hVjhneEtWaDF0ckl0K3l3N1laYjQzckdhSFcwMm9abzhPSjN1ZjVhNGVrTTgwblFaCnFGOWpJR3JveWR1R29oUlpGaXZMSVRLc21yUXRUZi93R1orcm5DWDdnUEk4WkZ6ODdrUlZlTkVDZ1lFQXk1VmEKRXZmWW9lRDQ2cnpsU3NSWUFEOGRlQ0dTV0lrZU51MFRuWmE3UzV4VFhac05PQVViZzVIZG56cW03ZE5zcGVRcQp3OUhGU3orMzFockxpZkdZTS9nMUR2YnpabzluaXcyYXFwanY2Zjd3K2QvUnFkMTRFd1orRlFDYjZmSFlZb1RkClg0ZlhDREN1Sk9acHdxYk85S01SQnVsK0syWWpTK3J2bkhwb0dVY0NnWUJPeVFFZzQ3MG96aW9FYmJzMUlhTmYKbitjVkNqT1FtZlhWb3lTcS9PUVordjljY1lkTHdBVjFhTnR2NURLcFhFY1RzS01TTWFtNUlwNm4xSytydE4zZApSZXpZWFdXcDZNdDVnQ2diN1VOSnJ2emo4ekViWExyazZaMDUwR3M0ZGxseVN3cVBiM0dVRnVkSEQzZUUvRmV4Cnl3bFl3aUNHd1FWUnN3YnhGUzNmc1FLQmdRQ0hpQ0IxTEVjdVRjV3BFS1JkaUNWRjFMN2JzUTNxbnk1Y1JxeWUKWnpDOWY0OEliQ2I2dW51OHRha2pUanEvYmRlUkFUdHZMZ0g0dkt5NGZ4SXQvWWJrWWtPQWVPdEZ4Z2FhemkyLwprUjEzWEdBMUdjcE5HY0hNZUxBTGIySmM5c0tsbytwMHBNeUZDVk9HQVZjN1hTMitNNmtzUHRuZUxwc2srWDBYCllXNVlGUUtCZ1FEcEk5NDcrVEhZTEphdnNxY3g5eXhaWnVVM2t0eGdGT0lwWDdNb0l4N1AwZUw0TEhBNTJmbm4KSkIxeUhoUzk0eTU4VHIrV20vZ2FIM0UvQVl4NjN6V1N3NXNNZkZCVVlDU0NPdlJ5RjUyTVY0aEpxT1orUndacQphcTJVLzN1U1NaTWJsRWs1U1VNdCtKWFFWL3NlZ20zOWM5R2VOM1hCcEF4VDk0TCtLaTZKekE9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo= diff --git a/nix/kubernetes/configuration.nix b/nix/kubernetes/configuration.nix index b6ca2c9..2909efa 100644 --- a/nix/kubernetes/configuration.nix +++ b/nix/kubernetes/configuration.nix @@ -7,6 +7,7 @@ { imports = [ ./roles/boot + ./roles/cilium ./roles/containerd ./roles/control_plane ./roles/doas diff --git a/nix/kubernetes/roles/cilium/default.nix b/nix/kubernetes/roles/cilium/default.nix new file mode 100644 index 0000000..86a0439 --- /dev/null +++ b/nix/kubernetes/roles/cilium/default.nix @@ -0,0 +1,29 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + options.me = { + cilium.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install cilium."; + }; + }; + + config = lib.mkIf config.me.cilium.enable { + networking.firewall.allowedTCPPorts = [ + 4240 # Health checks + ]; + networking.firewall.allowedUDPPorts = [ + 8472 # vxlan + 51871 # wireguard + ]; + }; +} diff --git a/nix/kubernetes/roles/containerd/default.nix b/nix/kubernetes/roles/containerd/default.nix index e2c8d68..0a462b3 100644 --- a/nix/kubernetes/roles/containerd/default.nix +++ b/nix/kubernetes/roles/containerd/default.nix @@ -5,6 +5,16 @@ ... }: +let + my-cni-plugins = pkgs.buildEnv { + name = "my-cni-plugins"; + paths = with pkgs; [ + cni-plugins + cni-plugin-flannel + ]; + }; + my-cni-configs = pkgs.callPackage ./package/cni_conf/package.nix { }; +in { imports = [ ]; @@ -19,40 +29,37 @@ config = lib.mkIf config.me.containerd.enable { virtualisation.containerd.enable = true; - virtualisation.containerd.settings = - let - my-cni-plugins = pkgs.buildEnv { - name = "my-cni-plugins"; - paths = with pkgs; [ - cni-plugins - cni-plugin-flannel - ]; - }; - in - { - "plugins" = { - "io.containerd.grpc.v1.cri" = { - "cni" = { - # "bin_dir" = "/opt/cni/bin"; - "bin_dir" = "${my-cni-plugins}/bin"; - # "conf_dir" = "/etc/cni/net.d"; - "conf_dir" = "${pkgs.callPackage ./package/cni_conf/package.nix { }}"; - }; - "containerd" = { - "default_runtime_name" = "runc"; - "runtimes" = { - "runc" = { - "options" = { - "SystemdCgroup" = true; - }; - "runtime_type" = "io.containerd.runc.v2"; + virtualisation.containerd.settings = { + "plugins" = { + "io.containerd.grpc.v1.cri" = { + "cni" = { + "bin_dir" = "/opt/cni/bin"; + "conf_dir" = "/etc/cni/net.d"; + # "bin_dir" = "${my-cni-plugins}/bin"; + # "conf_dir" = "${my-cni-configs}"; + }; + "containerd" = { + "default_runtime_name" = "runc"; + "runtimes" = { + "runc" = { + "options" = { + "SystemdCgroup" = true; }; + "runtime_type" = "io.containerd.runc.v2"; }; - "snapshotter" = "overlayfs"; }; + "snapshotter" = "overlayfs"; }; }; - "version" = 2; }; + "version" = 2; + }; + + systemd.services.containerd.preStart = '' + ${pkgs.toybox}/bin/install -d -m 0755 /opt/cni/bin /etc/cni/net.d + ${pkgs.toybox}/bin/install ${my-cni-plugins}/bin/* /opt/cni/bin/ + ${pkgs.toybox}/bin/install ${my-cni-configs}/* /etc/cni/net.d/ + echo "Copied CNI plugins/config." + ''; }; } diff --git a/nix/kubernetes/roles/firewall/default.nix b/nix/kubernetes/roles/firewall/default.nix index a5dafab..9dedd29 100644 --- a/nix/kubernetes/roles/firewall/default.nix +++ b/nix/kubernetes/roles/firewall/default.nix @@ -32,5 +32,11 @@ # We want to filter forwarded traffic. # Also needed for `networking.firewall.extraForwardRules` to do anything. networking.firewall.filterForward = true; + + # This can make debugging easier by rejecting packets instead of dropping them: + # networking.firewall.rejectPackets = true; + + # Check logs for blocked connections: + # journalctl -k or dmesg }; } diff --git a/nix/kubernetes/roles/kube_controller_manager/default.nix b/nix/kubernetes/roles/kube_controller_manager/default.nix index 955858a..40ac486 100644 --- a/nix/kubernetes/roles/kube_controller_manager/default.nix +++ b/nix/kubernetes/roles/kube_controller_manager/default.nix @@ -59,5 +59,9 @@ in User = "kubernetes"; }; }; + + networking.firewall.allowedTCPPorts = [ + 10257 + ]; }; } diff --git a/nix/kubernetes/roles/kube_proxy/default.nix b/nix/kubernetes/roles/kube_proxy/default.nix index 3a13fad..213aa3a 100644 --- a/nix/kubernetes/roles/kube_proxy/default.nix +++ b/nix/kubernetes/roles/kube_proxy/default.nix @@ -57,12 +57,15 @@ in "${pkgs.kubernetes}/bin/kube-proxy" "--config=${config_file}" "--nodeport-addresses=primary" - "--proxy-mode=nftables" ] ); Restart = "on-failure"; RestartSec = 5; }; }; + + networking.firewall.allowedTCPPorts = [ + 10256 + ]; }; } diff --git a/nix/kubernetes/roles/kube_scheduler/default.nix b/nix/kubernetes/roles/kube_scheduler/default.nix index 89a7697..ed701e5 100644 --- a/nix/kubernetes/roles/kube_scheduler/default.nix +++ b/nix/kubernetes/roles/kube_scheduler/default.nix @@ -47,5 +47,9 @@ in User = "kubernetes"; }; }; + + networking.firewall.allowedTCPPorts = [ + 10259 + ]; }; } diff --git a/nix/kubernetes/roles/kubelet/default.nix b/nix/kubernetes/roles/kubelet/default.nix index 99f4fff..cf79068 100644 --- a/nix/kubernetes/roles/kubelet/default.nix +++ b/nix/kubernetes/roles/kubelet/default.nix @@ -54,5 +54,9 @@ in # StateDirectory = "kubelet"; }; }; + + networking.firewall.allowedTCPPorts = [ + 10250 + ]; }; } diff --git a/nix/kubernetes/roles/network/default.nix b/nix/kubernetes/roles/network/default.nix index 3aca1d8..31f7e4a 100644 --- a/nix/kubernetes/roles/network/default.nix +++ b/nix/kubernetes/roles/network/default.nix @@ -48,6 +48,12 @@ # TODO: The 127.0.0.1 address should probably be moved to a host-specific file. networking.extraHosts = '' 127.0.0.1 ${config.networking.hostName}.home.arpa + 2620:11f:7001:7:ffff:ffff:0ad7:01dd controller0.kubernetes.local controller0 + 2620:11f:7001:7:ffff:ffff:0ad7:01de controller1.kubernetes.local controller1 + 2620:11f:7001:7:ffff:ffff:0ad7:01df controller2.kubernetes.local controller2 + 2620:11f:7001:7:ffff:ffff:0ad7:01e0 worker0.kubernetes.local worker0 + 2620:11f:7001:7:ffff:ffff:0ad7:01e1 worker1.kubernetes.local worker1 + 2620:11f:7001:7:ffff:ffff:0ad7:01e2 worker2.kubernetes.local worker2 ''; environment.systemPackages = with pkgs; [ @@ -56,6 +62,7 @@ arp-scan # To find devices on the network wavemon dhcpcd # For Android USB tethering. + net-tools # for netstat ]; boot.extraModprobeConfig = '' diff --git a/nix/kubernetes/roles/worker_node/default.nix b/nix/kubernetes/roles/worker_node/default.nix index 918e3b8..804d6f8 100644 --- a/nix/kubernetes/roles/worker_node/default.nix +++ b/nix/kubernetes/roles/worker_node/default.nix @@ -18,10 +18,27 @@ }; config = lib.mkIf config.me.worker_node.enable { + me.cilium.enable = true; me.containerd.enable = true; me.firewall.enable = true; # me.kube-proxy.enable = true; me.kubelet.enable = true; me.kubernetes.enable = true; + + networking.firewall.allowedTCPPortRanges = [ + { + # NodePort services + from = 30000; + to = 32767; + } + ]; + + networking.firewall.allowedUDPPortRanges = [ + { + # NodePort services + from = 30000; + to = 32767; + } + ]; }; } From 601086779639020a96daa34008908be8058d4a03 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Fri, 19 Dec 2025 21:21:57 -0500 Subject: [PATCH 18/81] Create a debugging role. --- nix/kubernetes/configuration.nix | 1 + nix/kubernetes/roles/containerd/default.nix | 7 +++++ nix/kubernetes/roles/debugging/default.nix | 33 +++++++++++++++++++++ nix/kubernetes/roles/etcd/default.nix | 9 ------ nix/kubernetes/roles/network/default.nix | 9 ------ 5 files changed, 41 insertions(+), 18 deletions(-) create mode 100644 nix/kubernetes/roles/debugging/default.nix diff --git a/nix/kubernetes/configuration.nix b/nix/kubernetes/configuration.nix index 2909efa..e6d5ef5 100644 --- a/nix/kubernetes/configuration.nix +++ b/nix/kubernetes/configuration.nix @@ -10,6 +10,7 @@ ./roles/cilium ./roles/containerd ./roles/control_plane + ./roles/debugging ./roles/doas ./roles/dont_use_substituters ./roles/etcd diff --git a/nix/kubernetes/roles/containerd/default.nix b/nix/kubernetes/roles/containerd/default.nix index 0a462b3..17085d1 100644 --- a/nix/kubernetes/roles/containerd/default.nix +++ b/nix/kubernetes/roles/containerd/default.nix @@ -61,5 +61,12 @@ in ${pkgs.toybox}/bin/install ${my-cni-configs}/* /etc/cni/net.d/ echo "Copied CNI plugins/config." ''; + + environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) { + hideMounts = lib.mkForce false; + directories = [ + "/var/lib/containerd" + ]; + }; }; } diff --git a/nix/kubernetes/roles/debugging/default.nix b/nix/kubernetes/roles/debugging/default.nix new file mode 100644 index 0000000..14bcb75 --- /dev/null +++ b/nix/kubernetes/roles/debugging/default.nix @@ -0,0 +1,33 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + options.me = { + debugging.enable = lib.mkOption { + type = lib.types.bool; + default = true; + example = true; + description = "Whether we want to install debugging."; + }; + }; + + config = lib.mkIf config.me.debugging.enable { + environment.systemPackages = with pkgs; [ + net-tools # for netstat + tcpdump + e2fsprogs # mkfs.ext4 + gptfdisk # cgdisk + arp-scan # To find devices on the network + ldns # for drill + ]; + + networking.firewall.enable = false; # TODO: This is just here for debugging / initial development. + # TODO: Maybe use networking.nftables.enable to switch to nftables? + }; +} diff --git a/nix/kubernetes/roles/etcd/default.nix b/nix/kubernetes/roles/etcd/default.nix index ae34542..4ebc159 100644 --- a/nix/kubernetes/roles/etcd/default.nix +++ b/nix/kubernetes/roles/etcd/default.nix @@ -97,14 +97,5 @@ users.users.etcd.uid = 10016; users.groups.etcd.gid = 10016; - - environment.systemPackages = with pkgs; [ - net-tools # for debugging - tcpdump - e2fsprogs # mkfs.ext4 - gptfdisk # cgdisk - ]; - networking.firewall.enable = false; # TODO: This is just here for debugging / initial development. - # TODO: Maybe use networking.nftables.enable to switch to nftables? }; } diff --git a/nix/kubernetes/roles/network/default.nix b/nix/kubernetes/roles/network/default.nix index 31f7e4a..c75d0d1 100644 --- a/nix/kubernetes/roles/network/default.nix +++ b/nix/kubernetes/roles/network/default.nix @@ -56,15 +56,6 @@ 2620:11f:7001:7:ffff:ffff:0ad7:01e2 worker2.kubernetes.local worker2 ''; - environment.systemPackages = with pkgs; [ - iw - ldns # for drill - arp-scan # To find devices on the network - wavemon - dhcpcd # For Android USB tethering. - net-tools # for netstat - ]; - boot.extraModprobeConfig = '' # Set wifi to US options cfg80211 ieee80211_regdom=US From 773db6e567fe321f8932f397bd8c7ecd668bfaf3 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Fri, 19 Dec 2025 22:49:32 -0500 Subject: [PATCH 19/81] Fix launching of containers. --- .../bootstrap/initial_clusterrole.yaml | 33 +++++++++++++++ nix/kubernetes/hosts/worker0/vm_disk.nix | 42 +++++++++---------- nix/kubernetes/hosts/worker1/vm_disk.nix | 42 +++++++++---------- nix/kubernetes/hosts/worker2/vm_disk.nix | 42 +++++++++---------- nix/kubernetes/roles/containerd/default.nix | 2 +- 5 files changed, 97 insertions(+), 64 deletions(-) create mode 100644 nix/kubernetes/bootstrap/initial_clusterrole.yaml diff --git a/nix/kubernetes/bootstrap/initial_clusterrole.yaml b/nix/kubernetes/bootstrap/initial_clusterrole.yaml new file mode 100644 index 0000000..e56770a --- /dev/null +++ b/nix/kubernetes/bootstrap/initial_clusterrole.yaml @@ -0,0 +1,33 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: system:kube-apiserver-to-kubelet +rules: + - apiGroups: + - "" + resources: + - nodes/proxy + - nodes/stats + - nodes/log + - nodes/spec + - nodes/metrics + verbs: + - "*" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:kube-apiserver + namespace: "" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-apiserver-to-kubelet +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: User + name: kubernetes \ No newline at end of file diff --git a/nix/kubernetes/hosts/worker0/vm_disk.nix b/nix/kubernetes/hosts/worker0/vm_disk.nix index 751f4ba..83683f8 100644 --- a/nix/kubernetes/hosts/worker0/vm_disk.nix +++ b/nix/kubernetes/hosts/worker0/vm_disk.nix @@ -11,15 +11,15 @@ config = { # Mount the local disk fileSystems = lib.mkIf config.me.mountPersistence { - # "/.disk" = lib.mkForce { - # device = "/dev/nvme0n1p1"; - # fsType = "ext4"; - # options = [ - # "noatime" - # "discard" - # ]; - # neededForBoot = true; - # }; + "/.disk" = lib.mkForce { + device = "/dev/nvme0n1p1"; + fsType = "ext4"; + options = [ + "noatime" + "discard" + ]; + neededForBoot = true; + }; "/.persist" = lib.mkForce { device = "bind9p"; @@ -77,18 +77,18 @@ neededForBoot = true; }; - # "/disk" = { - # fsType = "none"; - # device = "/.disk/persist"; - # options = [ - # "bind" - # "rw" - # ]; - # depends = [ - # "/.disk/persist" - # ]; - # neededForBoot = true; - # }; + "/disk" = { + fsType = "none"; + device = "/.disk/persist"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.disk/persist" + ]; + neededForBoot = true; + }; }; }; } diff --git a/nix/kubernetes/hosts/worker1/vm_disk.nix b/nix/kubernetes/hosts/worker1/vm_disk.nix index 751f4ba..83683f8 100644 --- a/nix/kubernetes/hosts/worker1/vm_disk.nix +++ b/nix/kubernetes/hosts/worker1/vm_disk.nix @@ -11,15 +11,15 @@ config = { # Mount the local disk fileSystems = lib.mkIf config.me.mountPersistence { - # "/.disk" = lib.mkForce { - # device = "/dev/nvme0n1p1"; - # fsType = "ext4"; - # options = [ - # "noatime" - # "discard" - # ]; - # neededForBoot = true; - # }; + "/.disk" = lib.mkForce { + device = "/dev/nvme0n1p1"; + fsType = "ext4"; + options = [ + "noatime" + "discard" + ]; + neededForBoot = true; + }; "/.persist" = lib.mkForce { device = "bind9p"; @@ -77,18 +77,18 @@ neededForBoot = true; }; - # "/disk" = { - # fsType = "none"; - # device = "/.disk/persist"; - # options = [ - # "bind" - # "rw" - # ]; - # depends = [ - # "/.disk/persist" - # ]; - # neededForBoot = true; - # }; + "/disk" = { + fsType = "none"; + device = "/.disk/persist"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.disk/persist" + ]; + neededForBoot = true; + }; }; }; } diff --git a/nix/kubernetes/hosts/worker2/vm_disk.nix b/nix/kubernetes/hosts/worker2/vm_disk.nix index 751f4ba..83683f8 100644 --- a/nix/kubernetes/hosts/worker2/vm_disk.nix +++ b/nix/kubernetes/hosts/worker2/vm_disk.nix @@ -11,15 +11,15 @@ config = { # Mount the local disk fileSystems = lib.mkIf config.me.mountPersistence { - # "/.disk" = lib.mkForce { - # device = "/dev/nvme0n1p1"; - # fsType = "ext4"; - # options = [ - # "noatime" - # "discard" - # ]; - # neededForBoot = true; - # }; + "/.disk" = lib.mkForce { + device = "/dev/nvme0n1p1"; + fsType = "ext4"; + options = [ + "noatime" + "discard" + ]; + neededForBoot = true; + }; "/.persist" = lib.mkForce { device = "bind9p"; @@ -77,18 +77,18 @@ neededForBoot = true; }; - # "/disk" = { - # fsType = "none"; - # device = "/.disk/persist"; - # options = [ - # "bind" - # "rw" - # ]; - # depends = [ - # "/.disk/persist" - # ]; - # neededForBoot = true; - # }; + "/disk" = { + fsType = "none"; + device = "/.disk/persist"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.disk/persist" + ]; + neededForBoot = true; + }; }; }; } diff --git a/nix/kubernetes/roles/containerd/default.nix b/nix/kubernetes/roles/containerd/default.nix index 17085d1..e3b61d3 100644 --- a/nix/kubernetes/roles/containerd/default.nix +++ b/nix/kubernetes/roles/containerd/default.nix @@ -62,7 +62,7 @@ in echo "Copied CNI plugins/config." ''; - environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) { + environment.persistence."/disk" = lib.mkIf (config.me.mountPersistence) { hideMounts = lib.mkForce false; directories = [ "/var/lib/containerd" From 3396a4f482267ea03850d516881a3dcdd641dc13 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sat, 20 Dec 2025 20:49:40 -0500 Subject: [PATCH 20/81] Add a bootstrap role to load manifests into the cluster. --- nix/kubernetes/configuration.nix | 1 + nix/kubernetes/roles/bootstrap/default.nix | 53 ++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 nix/kubernetes/roles/bootstrap/default.nix diff --git a/nix/kubernetes/configuration.nix b/nix/kubernetes/configuration.nix index e6d5ef5..34e2f86 100644 --- a/nix/kubernetes/configuration.nix +++ b/nix/kubernetes/configuration.nix @@ -7,6 +7,7 @@ { imports = [ ./roles/boot + ./roles/bootstrap ./roles/cilium ./roles/containerd ./roles/control_plane diff --git a/nix/kubernetes/roles/bootstrap/default.nix b/nix/kubernetes/roles/bootstrap/default.nix new file mode 100644 index 0000000..f068c63 --- /dev/null +++ b/nix/kubernetes/roles/bootstrap/default.nix @@ -0,0 +1,53 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + options.me = { + bootstrap.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install bootstrap."; + }; + + bootstrap.manifests = lib.mkOption { + type = lib.types.listOf lib.types.path; + default = [ ]; + example = lib.literalExpression ''[ ${./files/clusterrole.yaml} ]''; + description = "List of kubernetes manifests to load into the cluster."; + }; + }; + + config = + lib.mkIf (config.me.bootstrap.enable && ((builtins.length config.me.bootstrap.manifests) > 0)) + { + systemd.services.kube-bootstrap = { + enable = true; + description = "Load initial kubernetes manifests into the cluster."; + after = [ "kubernetes.target" ]; + path = with pkgs; [ + kubectl + ]; + unitConfig.DefaultDependencies = "no"; + serviceConfig = { + Type = "oneshot"; + }; + script = + let + manifests = (lib.concatMapStringsSep " " lib.escapeShellArgs config.me.bootstrap.manifests); + in + '' + set -o pipefail + IFS=$'\n\t' + + kubectl apply --server-side --force-conflicts -f ${manifests} + ''; + }; + }; +} From 3760bca64b73eff2bfe504c7034b94b4b074b7d9 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sat, 20 Dec 2025 22:14:28 -0500 Subject: [PATCH 21/81] Add a bootstrap role. --- nix/kubernetes/hosts/worker0/default.nix | 3 ++- nix/kubernetes/roles/bootstrap/default.nix | 9 +++++---- .../{bootstrap => roles/bootstrap/files}/cilium.yaml | 0 .../bootstrap/files}/initial_clusterrole.yaml | 0 nix/kubernetes/roles/containerd/default.nix | 2 +- nix/kubernetes/roles/worker_node/default.nix | 4 ++++ 6 files changed, 12 insertions(+), 6 deletions(-) rename nix/kubernetes/{bootstrap => roles/bootstrap/files}/cilium.yaml (100%) rename nix/kubernetes/{bootstrap => roles/bootstrap/files}/initial_clusterrole.yaml (100%) diff --git a/nix/kubernetes/hosts/worker0/default.nix b/nix/kubernetes/hosts/worker0/default.nix index 1fac5fa..e7e954f 100644 --- a/nix/kubernetes/hosts/worker0/default.nix +++ b/nix/kubernetes/hosts/worker0/default.nix @@ -102,8 +102,9 @@ # nix.sshServe.enable = true; # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ]; - me.worker_node.enable = true; + me.bootstrap.enable = true; me.dont_use_substituters.enable = true; me.minimal_base.enable = true; + me.worker_node.enable = true; }; } diff --git a/nix/kubernetes/roles/bootstrap/default.nix b/nix/kubernetes/roles/bootstrap/default.nix index f068c63..2eda302 100644 --- a/nix/kubernetes/roles/bootstrap/default.nix +++ b/nix/kubernetes/roles/bootstrap/default.nix @@ -17,7 +17,7 @@ }; bootstrap.manifests = lib.mkOption { - type = lib.types.listOf lib.types.path; + type = lib.types.listOf lib.types.str; default = [ ]; example = lib.literalExpression ''[ ${./files/clusterrole.yaml} ]''; description = "List of kubernetes manifests to load into the cluster."; @@ -29,8 +29,9 @@ { systemd.services.kube-bootstrap = { enable = true; - description = "Load initial kubernetes manifests into the cluster."; + description = "Load initial kubernetes manifests into the cluster"; after = [ "kubernetes.target" ]; + wantedBy = [ "multi-user.target" ]; path = with pkgs; [ kubectl ]; @@ -40,13 +41,13 @@ }; script = let - manifests = (lib.concatMapStringsSep " " lib.escapeShellArgs config.me.bootstrap.manifests); + manifests = (lib.concatMapStringsSep "," lib.escapeShellArg config.me.bootstrap.manifests); in '' set -o pipefail IFS=$'\n\t' - kubectl apply --server-side --force-conflicts -f ${manifests} + kubectl --kubeconfig=/.persist/keys/kube/kubelet.kubeconfig apply --server-side --force-conflicts -f ${manifests} ''; }; }; diff --git a/nix/kubernetes/bootstrap/cilium.yaml b/nix/kubernetes/roles/bootstrap/files/cilium.yaml similarity index 100% rename from nix/kubernetes/bootstrap/cilium.yaml rename to nix/kubernetes/roles/bootstrap/files/cilium.yaml diff --git a/nix/kubernetes/bootstrap/initial_clusterrole.yaml b/nix/kubernetes/roles/bootstrap/files/initial_clusterrole.yaml similarity index 100% rename from nix/kubernetes/bootstrap/initial_clusterrole.yaml rename to nix/kubernetes/roles/bootstrap/files/initial_clusterrole.yaml diff --git a/nix/kubernetes/roles/containerd/default.nix b/nix/kubernetes/roles/containerd/default.nix index e3b61d3..f7b48ec 100644 --- a/nix/kubernetes/roles/containerd/default.nix +++ b/nix/kubernetes/roles/containerd/default.nix @@ -10,7 +10,7 @@ let name = "my-cni-plugins"; paths = with pkgs; [ cni-plugins - cni-plugin-flannel + # cni-plugin-flannel ]; }; my-cni-configs = pkgs.callPackage ./package/cni_conf/package.nix { }; diff --git a/nix/kubernetes/roles/worker_node/default.nix b/nix/kubernetes/roles/worker_node/default.nix index 804d6f8..7a2edc3 100644 --- a/nix/kubernetes/roles/worker_node/default.nix +++ b/nix/kubernetes/roles/worker_node/default.nix @@ -18,6 +18,10 @@ }; config = lib.mkIf config.me.worker_node.enable { + me.bootstrap.manifests = [ + "${../bootstrap/files/initial_clusterrole.yaml}" + "${../bootstrap/files/cilium.yaml}" + ]; me.cilium.enable = true; me.containerd.enable = true; me.firewall.enable = true; From 46c8f5e0ce396b1bc3b2d0e99bdee8287351eff7 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sat, 20 Dec 2025 23:01:19 -0500 Subject: [PATCH 22/81] Set up flux. --- nix/kubernetes/README.org | 96 + .../roles/bootstrap/files/flux.yaml | 1954 +++++++++++++++++ .../roles/bootstrap/files/flux_namespace.yaml | 4 + 3 files changed, 2054 insertions(+) create mode 100644 nix/kubernetes/roles/bootstrap/files/flux.yaml create mode 100644 nix/kubernetes/roles/bootstrap/files/flux_namespace.yaml diff --git a/nix/kubernetes/README.org b/nix/kubernetes/README.org index 8ea141a..1662ba4 100644 --- a/nix/kubernetes/README.org +++ b/nix/kubernetes/README.org @@ -13,6 +13,9 @@ * Bootstrap ** Install cilium #+begin_src bash + # nix shell nixpkgs#cilium-cli + nix shell 'nixpkgs#kubernetes-helm' + helm repo add cilium https://helm.cilium.io/ helm template --dry-run=server cilium cilium/cilium --version 1.18.4 --namespace kube-system \ --set kubeProxyReplacement=true \ @@ -26,3 +29,96 @@ # --set hostFirewall.enabled=true # routingMode=native #+end_src +** Install flux +#+begin_src bash + nix shell 'nixpkgs#fluxcd' + + flux bootstrap git \ + --url=ssh://git@// \ + --branch=main \ + --private-key-file= \ + --password= \ + --path=clusters/my-cluster +#+end_src + +#+begin_src bash + nix shell 'nixpkgs#kubernetes-helm' + + helm template --dry-run=server flux-operator oci://ghcr.io/controlplaneio-fluxcd/charts/flux-operator \ + --namespace flux-system \ + --create-namespace +#+end_src + +#+begin_src text + apiVersion: fluxcd.controlplane.io/v1 + kind: FluxInstance + metadata: + name: flux + namespace: flux-system + annotations: + fluxcd.controlplane.io/reconcileEvery: "1h" + fluxcd.controlplane.io/reconcileTimeout: "5m" + spec: + distribution: + version: "2.x" + registry: "ghcr.io/fluxcd" + artifact: "oci://ghcr.io/controlplaneio-fluxcd/flux-operator-manifests" + components: + - source-controller + - kustomize-controller + - helm-controller + - notification-controller + - image-reflector-controller + - image-automation-controller + cluster: + type: kubernetes + size: medium + multitenant: false + networkPolicy: true + domain: "cluster.local" + kustomize: + patches: + - target: + kind: Deployment + patch: | + - op: replace + path: /spec/template/spec/nodeSelector + value: + kubernetes.io/os: linux + - op: add + path: /spec/template/spec/tolerations + value: + - key: "CriticalAddonsOnly" + operator: "Exists" + sync: + kind: OCIRepository + url: "oci://ghcr.io/my-org/my-fleet-manifests" + ref: "latest" + path: "clusters/my-cluster" + pullSecret: "ghcr-auth" +#+end_src + +#+begin_src text + apiVersion: fluxcd.controlplane.io/v1 + kind: FluxInstance + metadata: + name: flux + namespace: flux-system + spec: + distribution: + version: "2.7.x" + registry: "ghcr.io/fluxcd" + sync: + kind: GitRepository + url: "https://gitlab.com/my-org/my-fleet.git" + ref: "refs/heads/main" + path: "clusters/my-cluster" + pullSecret: "flux-system" +#+end_src + +#+begin_src bash + flux create secret git flux-system \ + --url=https://gitlab.com/my-org/my-fleet.git \ + --username=git \ + --password=$GITLAB_TOKEN +#+end_src diff --git a/nix/kubernetes/roles/bootstrap/files/flux.yaml b/nix/kubernetes/roles/bootstrap/files/flux.yaml new file mode 100644 index 0000000..d521a2f --- /dev/null +++ b/nix/kubernetes/roles/bootstrap/files/flux.yaml @@ -0,0 +1,1954 @@ +--- +# Source: flux-operator/templates/networkpolicy.yaml +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: flux-operator-web + namespace: flux-system + labels: + helm.sh/chart: flux-operator-0.37.1 + app.kubernetes.io/name: flux-operator + app.kubernetes.io/instance: flux-operator + app.kubernetes.io/version: "v0.37.1" + app.kubernetes.io/managed-by: Helm +spec: + policyTypes: + - Ingress + podSelector: + matchLabels: + app.kubernetes.io/name: flux-operator + app.kubernetes.io/instance: flux-operator + ingress: + - from: + - namespaceSelector: {} + ports: + - protocol: TCP + port: 9080 +--- +# Source: flux-operator/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flux-operator + namespace: flux-system + labels: + helm.sh/chart: flux-operator-0.37.1 + app.kubernetes.io/name: flux-operator + app.kubernetes.io/instance: flux-operator + app.kubernetes.io/version: "v0.37.1" + app.kubernetes.io/managed-by: Helm +automountServiceAccountToken: true +--- +# Source: flux-operator/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + helm.sh/resource-policy: keep + labels: + app.kubernetes.io/instance: 'flux-operator' + app.kubernetes.io/managed-by: 'Helm' + app.kubernetes.io/name: 'flux-operator' + app.kubernetes.io/version: 'v0.37.1' + helm.sh/chart: 'flux-operator-0.37.1' + name: fluxinstances.fluxcd.controlplane.io +spec: + group: fluxcd.controlplane.io + names: + kind: FluxInstance + listKind: FluxInstanceList + plural: fluxinstances + singular: fluxinstance + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + - jsonPath: .status.lastAttemptedRevision + name: Revision + type: string + name: v1 + schema: + openAPIV3Schema: + description: FluxInstance is the Schema for the fluxinstances API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: FluxInstanceSpec defines the desired state of FluxInstance + properties: + cluster: + description: Cluster holds the specification of the Kubernetes cluster. + properties: + domain: + default: cluster.local + description: |- + Domain is the cluster domain used for generating the FQDN of services. + Defaults to 'cluster.local'. + type: string + multitenant: + default: false + description: Multitenant enables the multitenancy lockdown. Defaults + to false. + type: boolean + multitenantWorkloadIdentity: + default: false + description: |- + MultitenantWorkloadIdentity enables the multitenancy lockdown for + workload identity. Defaults to false. + type: boolean + networkPolicy: + default: true + description: |- + NetworkPolicy restricts network access to the current namespace. + Defaults to true. + type: boolean + objectLevelWorkloadIdentity: + description: |- + ObjectLevelWorkloadIdentity enables the feature gate + required for object-level workload identity. + This feature is only available in Flux v2.6.0 and later. + type: boolean + size: + description: |- + Size defines the vertical scaling profile of the Flux controllers. + The size is used to determine the concurrency and CPU/Memory limits for the Flux controllers. + Accepted values are: 'small', 'medium' and 'large'. + enum: + - small + - medium + - large + type: string + tenantDefaultDecryptionServiceAccount: + description: |- + TenantDefaultDecryptionServiceAccount is the name of the service account + to use as default for kustomize-controller SOPS decryption when the + multitenant lockdown for workload identity is enabled. Defaults to the + 'default' service account from the tenant namespace. + type: string + tenantDefaultKubeConfigServiceAccount: + description: |- + TenantDefaultKubeConfigServiceAccount is the name of the service account + to use as default for kustomize-controller and helm-controller remote + cluster access via spec.kubeConfig.configMapRef when the multitenant + lockdown for workload identity is enabled. Defaults to the 'default' + service account from the tenant namespace. + type: string + tenantDefaultServiceAccount: + description: |- + TenantDefaultServiceAccount is the name of the service account + to use as default when the multitenant lockdown is enabled, for + kustomize-controller and helm-controller. + This field will also be used for multitenant workload identity + lockdown for source-controller, notification-controller, + image-reflector-controller and image-automation-controller. + Defaults to the 'default' service account from the tenant namespace. + type: string + type: + default: kubernetes + description: |- + Type specifies the distro of the Kubernetes cluster. + Defaults to 'kubernetes'. + enum: + - kubernetes + - openshift + - aws + - azure + - gcp + type: string + type: object + x-kubernetes-validations: + - message: .objectLevelWorkloadIdentity must be set to true when .multitenantWorkloadIdentity + is set to true + rule: (has(self.objectLevelWorkloadIdentity) && self.objectLevelWorkloadIdentity) + || !has(self.multitenantWorkloadIdentity) || !self.multitenantWorkloadIdentity + commonMetadata: + description: |- + CommonMetadata specifies the common labels and annotations that are + applied to all resources. Any existing label or annotation will be + overridden if its key matches a common one. + properties: + annotations: + additionalProperties: + type: string + description: Annotations to be added to the object's metadata. + type: object + labels: + additionalProperties: + type: string + description: Labels to be added to the object's metadata. + type: object + type: object + components: + description: |- + Components is the list of controllers to install. + Defaults to a commonly used subset. + items: + description: Component is the name of a controller to install. + enum: + - source-controller + - kustomize-controller + - helm-controller + - notification-controller + - image-reflector-controller + - image-automation-controller + - source-watcher + type: string + type: array + distribution: + description: Distribution specifies the version and container registry + to pull images from. + properties: + artifact: + description: |- + Artifact is the URL to the OCI artifact containing + the latest Kubernetes manifests for the distribution, + e.g. 'oci://ghcr.io/controlplaneio-fluxcd/flux-operator-manifests:latest'. + pattern: ^oci://.*$ + type: string + artifactPullSecret: + description: |- + ArtifactPullSecret is the name of the Kubernetes secret + to use for pulling the Kubernetes manifests for the distribution specified in the Artifact field. + type: string + imagePullSecret: + description: |- + ImagePullSecret is the name of the Kubernetes secret + to use for pulling images. + type: string + registry: + description: |- + Registry address to pull the distribution images from + e.g. 'ghcr.io/fluxcd'. + type: string + variant: + description: |- + Variant specifies the Flux distribution flavor stored + in the registry. + enum: + - upstream-alpine + - enterprise-alpine + - enterprise-distroless + - enterprise-distroless-fips + type: string + version: + description: Version semver expression e.g. '2.x', '2.3.x'. + type: string + required: + - registry + - version + type: object + kustomize: + description: |- + Kustomize holds a set of patches that can be applied to the + Flux installation, to customize the way Flux operates. + properties: + patches: + description: |- + Strategic merge and JSON patches, defined as inline YAML objects, + capable of targeting objects based on kind, label and annotation selectors. + items: + description: |- + Patch contains an inline StrategicMerge or JSON6902 patch, and the target the patch should + be applied to. + properties: + patch: + description: |- + Patch contains an inline StrategicMerge patch or an inline JSON6902 patch with + an array of operation objects. + type: string + target: + description: Target points to the resources that the patch + document should be applied to. + properties: + annotationSelector: + description: |- + AnnotationSelector is a string that follows the label selection expression + https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#api + It matches with the resource annotations. + type: string + group: + description: |- + Group is the API group to select resources from. + Together with Version and Kind it is capable of unambiguously identifying and/or selecting resources. + https://github.com/kubernetes/community/blob/master/contributors/design-proposals/api-machinery/api-group.md + type: string + kind: + description: |- + Kind of the API Group to select resources from. + Together with Group and Version it is capable of unambiguously + identifying and/or selecting resources. + https://github.com/kubernetes/community/blob/master/contributors/design-proposals/api-machinery/api-group.md + type: string + labelSelector: + description: |- + LabelSelector is a string that follows the label selection expression + https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#api + It matches with the resource labels. + type: string + name: + description: Name to match resources with. + type: string + namespace: + description: Namespace to select resources from. + type: string + version: + description: |- + Version of the API Group to select resources from. + Together with Group and Kind it is capable of unambiguously identifying and/or selecting resources. + https://github.com/kubernetes/community/blob/master/contributors/design-proposals/api-machinery/api-group.md + type: string + type: object + required: + - patch + type: object + type: array + type: object + migrateResources: + default: true + description: |- + MigrateResources instructs the controller to migrate the Flux custom resources + from the previous version to the latest API version specified in the CRD. + Defaults to true. + type: boolean + sharding: + description: Sharding holds the specification of the sharding configuration. + properties: + key: + default: sharding.fluxcd.io/key + description: Key is the label key used to shard the resources. + type: string + shards: + description: Shards is the list of shard names. + items: + type: string + minItems: 1 + type: array + storage: + description: |- + Storage defines if the source-controller shards + should use an emptyDir or a persistent volume claim for storage. + Accepted values are 'ephemeral' or 'persistent', defaults to 'ephemeral'. + For 'persistent' to take effect, the '.spec.storage' field must be set. + enum: + - ephemeral + - persistent + type: string + required: + - shards + type: object + storage: + description: |- + Storage holds the specification of the source-controller + persistent volume claim. + properties: + class: + description: Class is the storage class to use for the PVC. + type: string + size: + description: Size is the size of the PVC. + type: string + required: + - class + - size + type: object + sync: + description: |- + Sync specifies the source for the cluster sync operation. + When set, a Flux source (GitRepository, OCIRepository or Bucket) + and Flux Kustomization are created to sync the cluster state + with the source repository. + properties: + interval: + default: 1m + description: Interval is the time between syncs. + pattern: ^([0-9]+(\.[0-9]+)?(ms|s|m|h))+$ + type: string + kind: + description: Kind is the kind of the source. + enum: + - OCIRepository + - GitRepository + - Bucket + type: string + name: + description: |- + Name is the name of the Flux source and kustomization resources. + When not specified, the name is set to the namespace name of the FluxInstance. + maxLength: 63 + type: string + x-kubernetes-validations: + - message: Sync name is immutable + rule: self == oldSelf + path: + description: |- + Path is the path to the source directory containing + the kustomize overlay or plain Kubernetes manifests. + type: string + provider: + description: |- + Provider specifies OIDC provider for source authentication. + For OCIRepository and Bucket the provider can be set to 'aws', 'azure' or 'gcp'. + for GitRepository the accepted value can be set to 'azure' or 'github'. + To disable OIDC authentication the provider can be set to 'generic' or left empty. + enum: + - generic + - aws + - azure + - gcp + - github + type: string + pullSecret: + description: |- + PullSecret specifies the Kubernetes Secret containing the + authentication credentials for the source. + For Git over HTTP/S sources, the secret must contain username and password fields. + For Git over SSH sources, the secret must contain known_hosts and identity fields. + For OCI sources, the secret must be of type kubernetes.io/dockerconfigjson. + For Bucket sources, the secret must contain accesskey and secretkey fields. + type: string + ref: + description: |- + Ref is the source reference, can be a Git ref name e.g. 'refs/heads/main', + an OCI tag e.g. 'latest' or a bucket name e.g. 'flux'. + type: string + url: + description: |- + URL is the source URL, can be a Git repository HTTP/S or SSH address, + an OCI repository address or a Bucket endpoint. + type: string + required: + - kind + - path + - ref + - url + type: object + wait: + default: true + description: |- + Wait instructs the controller to check the health of all the reconciled + resources. Defaults to true. + type: boolean + required: + - distribution + type: object + status: + description: FluxInstanceStatus defines the observed state of FluxInstance + properties: + components: + description: Components contains the container images used by the + components. + items: + description: ComponentImage represents a container image used by + a component. + properties: + digest: + description: Digest of the container image. + type: string + name: + description: Name of the component. + type: string + repository: + description: Repository address of the container image. + type: string + tag: + description: Tag of the container image. + type: string + required: + - name + - repository + - tag + type: object + type: array + conditions: + description: Conditions contains the readiness conditions of the object. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + history: + description: |- + History contains the reconciliation history of the FluxInstance + as a list of snapshots ordered by the last reconciled time. + items: + description: |- + Snapshot represents a point-in-time record of a group of resources reconciliation, + including timing information, status, and a unique digest identifier. + properties: + digest: + description: Digest is the checksum in the format `:` + of the resources in this snapshot. + type: string + firstReconciled: + description: FirstReconciled is the time when this revision + was first reconciled to the cluster. + format: date-time + type: string + lastReconciled: + description: LastReconciled is the time when this revision was + last reconciled to the cluster. + format: date-time + type: string + lastReconciledDuration: + description: LastReconciledDuration is time it took to reconcile + the resources in this revision. + type: string + lastReconciledStatus: + description: LastReconciledStatus is the status of the last + reconciliation. + type: string + metadata: + additionalProperties: + type: string + description: Metadata contains additional information about + the snapshot. + type: object + totalReconciliations: + description: TotalReconciliations is the total number of reconciliations + that have occurred for this snapshot. + format: int64 + type: integer + required: + - digest + - firstReconciled + - lastReconciled + - lastReconciledDuration + - lastReconciledStatus + - totalReconciliations + type: object + type: array + inventory: + description: |- + Inventory contains a list of Kubernetes resource object references + last applied on the cluster. + properties: + entries: + description: Entries of Kubernetes resource object references. + items: + description: ResourceRef contains the information necessary + to locate a resource within a cluster. + properties: + id: + description: |- + ID is the string representation of the Kubernetes resource object's metadata, + in the format '___'. + type: string + v: + description: Version is the API version of the Kubernetes + resource object's kind. + type: string + required: + - id + - v + type: object + type: array + required: + - entries + type: object + lastAppliedRevision: + description: |- + LastAppliedRevision is the version and digest of the + distribution config that was last reconcile. + type: string + lastArtifactRevision: + description: |- + LastArtifactRevision is the digest of the last pulled + distribution artifact. + type: string + lastAttemptedRevision: + description: |- + LastAttemptedRevision is the version and digest of the + distribution config that was last attempted to reconcile. + type: string + lastHandledForceAt: + description: |- + LastHandledForceAt holds the value of the most recent + force request value, so a change of the annotation value + can be detected. + type: string + lastHandledReconcileAt: + description: |- + LastHandledReconcileAt holds the value of the most recent + reconcile request value, so a change of the annotation value + can be detected. + type: string + type: object + type: object + x-kubernetes-validations: + - message: the only accepted name for a FluxInstance is 'flux' + rule: self.metadata.name == 'flux' + served: true + storage: true + subresources: + status: {} +--- +# Source: flux-operator/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + helm.sh/resource-policy: keep + labels: + app.kubernetes.io/instance: 'flux-operator' + app.kubernetes.io/managed-by: 'Helm' + app.kubernetes.io/name: 'flux-operator' + app.kubernetes.io/version: 'v0.37.1' + helm.sh/chart: 'flux-operator-0.37.1' + name: fluxreports.fluxcd.controlplane.io +spec: + group: fluxcd.controlplane.io + names: + kind: FluxReport + listKind: FluxReportList + plural: fluxreports + singular: fluxreport + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.distribution.entitlement + name: Entitlement + priority: 10 + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].lastTransitionTime + name: LastUpdated + type: string + name: v1 + schema: + openAPIV3Schema: + description: FluxReport is the Schema for the fluxreports API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: FluxReportSpec defines the observed state of a Flux installation. + properties: + cluster: + description: Cluster is the version information of the Kubernetes + cluster. + properties: + nodes: + description: Nodes is the number of nodes in the Kubernetes cluster. + type: integer + platform: + description: Platform is the os/arch of the Kubernetes control + plane. + type: string + serverVersion: + description: ServerVersion is the version of the Kubernetes API + server. + type: string + required: + - platform + - serverVersion + type: object + components: + description: ComponentsStatus is the status of the Flux controller + deployments. + items: + description: FluxComponentStatus defines the observed state of a + Flux component. + properties: + image: + description: Image is the container image of the Flux component. + type: string + name: + description: Name is the name of the Flux component. + type: string + ready: + description: Ready is the readiness status of the Flux component. + type: boolean + status: + description: |- + Status is a human-readable message indicating details + about the Flux component observed state. + type: string + required: + - image + - name + - ready + - status + type: object + type: array + distribution: + description: Distribution is the version information of the Flux installation. + properties: + entitlement: + description: Entitlement is the entitlement verification status. + type: string + managedBy: + description: ManagedBy is the name of the operator managing the + Flux instance. + type: string + status: + description: |- + Status is a human-readable message indicating details + about the distribution observed state. + type: string + version: + description: Version is the version of the Flux instance. + type: string + required: + - entitlement + - status + type: object + operator: + description: Operator is the version information of the Flux Operator. + properties: + apiVersion: + description: APIVersion is the API version of the Flux Operator. + type: string + platform: + description: Platform is the os/arch of Flux Operator. + type: string + version: + description: Version is the version number of Flux Operator. + type: string + required: + - apiVersion + - platform + - version + type: object + reconcilers: + description: |- + ReconcilersStatus is the list of Flux reconcilers and + their statistics grouped by API kind. + items: + description: FluxReconcilerStatus defines the observed state of + a Flux reconciler. + properties: + apiVersion: + description: APIVersion is the API version of the Flux resource. + type: string + kind: + description: Kind is the kind of the Flux resource. + type: string + stats: + description: Stats is the reconcile statics of the Flux resource + kind. + properties: + failing: + description: |- + Failing is the number of reconciled + resources in the Failing state. + type: integer + running: + description: |- + Running is the number of reconciled + resources in the Running state. + type: integer + suspended: + description: |- + Suspended is the number of reconciled + resources in the Suspended state. + type: integer + totalSize: + description: TotalSize is the total size of the artifacts + in storage. + type: string + required: + - failing + - running + - suspended + type: object + required: + - apiVersion + - kind + type: object + type: array + sync: + description: |- + SyncStatus is the status of the cluster sync + Source and Kustomization resources. + properties: + id: + description: ID is the identifier of the sync. + type: string + path: + description: Path is the kustomize path of the sync. + type: string + ready: + description: Ready is the readiness status of the sync. + type: boolean + source: + description: Source is the URL of the source repository. + type: string + status: + description: |- + Status is a human-readable message indicating details + about the sync observed state. + type: string + required: + - id + - ready + - status + type: object + required: + - distribution + type: object + status: + description: FluxReportStatus defines the readiness of a FluxReport. + properties: + conditions: + description: Conditions contains the readiness conditions of the object. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + lastHandledReconcileAt: + description: |- + LastHandledReconcileAt holds the value of the most recent + reconcile request value, so a change of the annotation value + can be detected. + type: string + type: object + type: object + x-kubernetes-validations: + - message: the only accepted name for a FluxReport is 'flux' + rule: self.metadata.name == 'flux' + served: true + storage: true + subresources: + status: {} +--- +# Source: flux-operator/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + helm.sh/resource-policy: keep + labels: + app.kubernetes.io/instance: 'flux-operator' + app.kubernetes.io/managed-by: 'Helm' + app.kubernetes.io/name: 'flux-operator' + app.kubernetes.io/version: 'v0.37.1' + helm.sh/chart: 'flux-operator-0.37.1' + name: resourcesetinputproviders.fluxcd.controlplane.io +spec: + group: fluxcd.controlplane.io + names: + kind: ResourceSetInputProvider + listKind: ResourceSetInputProviderList + plural: resourcesetinputproviders + shortNames: + - rsip + singular: resourcesetinputprovider + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + name: v1 + schema: + openAPIV3Schema: + description: ResourceSetInputProvider is the Schema for the ResourceSetInputProviders + API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ResourceSetInputProviderSpec defines the desired state of + ResourceSetInputProvider + properties: + certSecretRef: + description: |- + CertSecretRef specifies the Kubernetes Secret containing either or both of + + - a PEM-encoded CA certificate (`ca.crt`) + - a PEM-encoded client certificate (`tls.crt`) and private key (`tls.key`) + + When connecting to a Git or OCI provider that uses self-signed certificates, the CA certificate + must be set in the Secret under the 'ca.crt' key to establish the trust relationship. + When connecting to an OCI provider that supports client certificates (mTLS), the client certificate + and private key must be set in the Secret under the 'tls.crt' and 'tls.key' keys, respectively. + properties: + name: + description: Name of the referent. + type: string + required: + - name + type: object + defaultValues: + additionalProperties: + x-kubernetes-preserve-unknown-fields: true + description: |- + DefaultValues contains the default values for the inputs. + These values are used to populate the inputs when the provider + response does not contain them. + type: object + filter: + description: Filter defines the filter to apply to the input provider + response. + properties: + excludeBranch: + description: |- + ExcludeBranch specifies the regular expression to filter the branches + that the input provider should exclude. + type: string + excludeEnvironment: + description: |- + ExcludeEnvironment specifies the regular expression to filter the environments + that the input provider should exclude. + type: string + excludeTag: + description: |- + ExcludeTag specifies the regular expression to filter the tags + that the input provider should exclude. + type: string + includeBranch: + description: |- + IncludeBranch specifies the regular expression to filter the branches + that the input provider should include. + type: string + includeEnvironment: + description: |- + IncludeEnvironment specifies the regular expression to filter the environments + that the input provider should include. + type: string + includeTag: + description: |- + IncludeTag specifies the regular expression to filter the tags + that the input provider should include. + type: string + labels: + description: Labels specifies the list of labels to filter the + input provider response. + items: + type: string + type: array + limit: + default: 100 + description: |- + Limit specifies the maximum number of input sets to return. + When not set, the default limit is 100. + type: integer + semver: + description: |- + Semver specifies a semantic version range to filter and sort the tags. + If this field is not specified, the tags will be sorted in reverse + alphabetical order. + Supported only for tags at the moment. + type: string + type: object + schedule: + description: Schedule defines the schedules for the input provider + to run. + items: + description: Schedule defines a schedule for something to run. + properties: + cron: + description: Cron specifies the cron expression for the schedule. + type: string + timeZone: + default: UTC + description: TimeZone specifies the time zone for the cron schedule. + Defaults to UTC. + type: string + window: + default: 0s + description: |- + Window defines the time window during which the execution is allowed. + Defaults to 0s, meaning no window is applied. + pattern: ^([0-9]+(\.[0-9]+)?(ms|s|m|h))+$ + type: string + required: + - cron + type: object + type: array + secretRef: + description: |- + SecretRef specifies the Kubernetes Secret containing the basic-auth credentials + to access the input provider. + When connecting to a Git provider, the secret must contain the keys + 'username' and 'password', and the password should be a personal access token + that grants read-only access to the repository. + When connecting to an OCI provider, the secret must contain a Kubernetes + Image Pull Secret, as if created by `kubectl create secret docker-registry`. + properties: + name: + description: Name of the referent. + type: string + required: + - name + type: object + serviceAccountName: + description: |- + ServiceAccountName specifies the name of the Kubernetes ServiceAccount + used for authentication with AWS, Azure or GCP services through + workload identity federation features. If not specified, the + authentication for these cloud providers will use the ServiceAccount + of the operator (or any other environment authentication configuration). + type: string + skip: + description: Skip defines whether we need to skip input provider response + updates. + properties: + labels: + description: |- + Labels specifies list of labels to skip input provider response when any of the label conditions matched. + When prefixed with !, input provider response will be skipped if it does not have this label. + items: + type: string + type: array + type: object + type: + description: Type specifies the type of the input provider. + enum: + - Static + - GitHubBranch + - GitHubTag + - GitHubPullRequest + - GitLabBranch + - GitLabTag + - GitLabMergeRequest + - GitLabEnvironment + - AzureDevOpsBranch + - AzureDevOpsTag + - AzureDevOpsPullRequest + - OCIArtifactTag + - ACRArtifactTag + - ECRArtifactTag + - GARArtifactTag + type: string + url: + description: |- + URL specifies the HTTP/S or OCI address of the input provider API. + When connecting to a Git provider, the URL should point to the repository address. + When connecting to an OCI provider, the URL should point to the OCI repository address. + pattern: ^((http|https|oci)://.*){0,1}$ + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: spec.url must be empty when spec.type is 'Static' + rule: self.type != 'Static' || !has(self.url) + - message: spec.url must not be empty when spec.type is not 'Static' + rule: self.type == 'Static' || has(self.url) + - message: spec.url must start with 'http://' or 'https://' when spec.type + is a Git provider + rule: '!self.type.startsWith(''Git'') || self.url.startsWith(''http'')' + - message: spec.url must start with 'http://' or 'https://' when spec.type + is a Git provider + rule: '!self.type.startsWith(''AzureDevOps'') || self.url.startsWith(''http'')' + - message: spec.url must start with 'oci://' when spec.type is an OCI + provider + rule: '!self.type.endsWith(''ArtifactTag'') || self.url.startsWith(''oci'')' + - message: cannot specify spec.serviceAccountName when spec.type is not + one of AzureDevOps* or *ArtifactTag + rule: '!has(self.serviceAccountName) || self.type.startsWith(''AzureDevOps'') + || self.type.endsWith(''ArtifactTag'')' + - message: cannot specify spec.certSecretRef when spec.type is one of + Static, AzureDevOps*, ACRArtifactTag, ECRArtifactTag or GARArtifactTag + rule: '!has(self.certSecretRef) || !(self.url == ''Static'' || self.type.startsWith(''AzureDevOps'') + || (self.type.endsWith(''ArtifactTag'') && self.type != ''OCIArtifactTag''))' + - message: cannot specify spec.secretRef when spec.type is one of Static, + ACRArtifactTag, ECRArtifactTag or GARArtifactTag + rule: '!has(self.secretRef) || !(self.url == ''Static'' || (self.type.endsWith(''ArtifactTag'') + && self.type != ''OCIArtifactTag''))' + status: + description: ResourceSetInputProviderStatus defines the observed state + of ResourceSetInputProvider. + properties: + conditions: + description: Conditions contains the readiness conditions of the object. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + exportedInputs: + description: ExportedInputs contains the list of inputs exported by + the provider. + items: + additionalProperties: + x-kubernetes-preserve-unknown-fields: true + description: ResourceSetInput defines the key-value pairs of the + ResourceSet input. + type: object + type: array + lastExportedRevision: + description: |- + LastExportedRevision is the digest of the + inputs that were last reconcile. + type: string + lastHandledForceAt: + description: |- + LastHandledForceAt holds the value of the most recent + force request value, so a change of the annotation value + can be detected. + type: string + lastHandledReconcileAt: + description: |- + LastHandledReconcileAt holds the value of the most recent + reconcile request value, so a change of the annotation value + can be detected. + type: string + nextSchedule: + description: NextSchedule is the next schedule when the input provider + will run. + properties: + cron: + description: Cron specifies the cron expression for the schedule. + type: string + timeZone: + default: UTC + description: TimeZone specifies the time zone for the cron schedule. + Defaults to UTC. + type: string + when: + description: When is the next time the schedule will run. + format: date-time + type: string + window: + default: 0s + description: |- + Window defines the time window during which the execution is allowed. + Defaults to 0s, meaning no window is applied. + pattern: ^([0-9]+(\.[0-9]+)?(ms|s|m|h))+$ + type: string + required: + - cron + - when + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: flux-operator/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + helm.sh/resource-policy: keep + labels: + app.kubernetes.io/instance: 'flux-operator' + app.kubernetes.io/managed-by: 'Helm' + app.kubernetes.io/name: 'flux-operator' + app.kubernetes.io/version: 'v0.37.1' + helm.sh/chart: 'flux-operator-0.37.1' + name: resourcesets.fluxcd.controlplane.io +spec: + group: fluxcd.controlplane.io + names: + kind: ResourceSet + listKind: ResourceSetList + plural: resourcesets + shortNames: + - rset + singular: resourceset + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + name: v1 + schema: + openAPIV3Schema: + description: ResourceSet is the Schema for the ResourceSets API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ResourceSetSpec defines the desired state of ResourceSet + properties: + commonMetadata: + description: |- + CommonMetadata specifies the common labels and annotations that are + applied to all resources. Any existing label or annotation will be + overridden if its key matches a common one. + properties: + annotations: + additionalProperties: + type: string + description: Annotations to be added to the object's metadata. + type: object + labels: + additionalProperties: + type: string + description: Labels to be added to the object's metadata. + type: object + type: object + dependsOn: + description: |- + DependsOn specifies the list of Kubernetes resources that must + exist on the cluster before the reconciliation process starts. + items: + description: Dependency defines a ResourceSet dependency on a Kubernetes + resource. + properties: + apiVersion: + description: APIVersion of the resource to depend on. + type: string + kind: + description: Kind of the resource to depend on. + type: string + name: + description: Name of the resource to depend on. + type: string + namespace: + description: Namespace of the resource to depend on. + type: string + ready: + description: Ready checks if the resource Ready status condition + is true. + type: boolean + readyExpr: + description: |- + ReadyExpr checks if the resource satisfies the given CEL expression. + The expression replaces the default readiness check and + is only evaluated if Ready is set to 'true'. + type: string + required: + - apiVersion + - kind + - name + type: object + type: array + inputStrategy: + description: |- + InputStrategy defines how the inputs are combined when multiple + input provider objects are used. Defaults to flattening all inputs + from all providers into a single list of input sets. + properties: + name: + description: |- + Name defines how the inputs are combined when multiple + input provider objects are used. Supported values are: + - Flatten: all inputs sets from all input provider objects are + flattened into a single list of input sets. + - Permute: all inputs sets from all input provider objects are + combined using a Cartesian product, resulting in a list of input sets + that contains every possible combination of input values. + For example, if provider A has inputs [{x: 1}, {x: 2}] and provider B has + inputs [{y: "a"}, {y: "b"}], the resulting input sets will be: + [{x: 1, y: "a"}, {x: 1, y: "b"}, {x: 2, y: "a"}, {x: 2, y: "b"}]. + This strategy can lead to a large number of input sets and should be + used with caution. Users should use filtering features from + ResourceSetInputProvider to limit the amount of exported inputs. + enum: + - Flatten + - Permute + type: string + required: + - name + type: object + inputs: + description: Inputs contains the list of ResourceSet inputs. + items: + additionalProperties: + x-kubernetes-preserve-unknown-fields: true + description: ResourceSetInput defines the key-value pairs of the + ResourceSet input. + type: object + type: array + inputsFrom: + description: |- + InputsFrom contains the list of references to input providers. + When set, the inputs are fetched from the providers and concatenated + with the in-line inputs defined in the ResourceSet. + items: + description: |- + InputProviderReference defines a reference to an input provider resource + in the same namespace as the ResourceSet. + properties: + apiVersion: + description: |- + APIVersion of the input provider resource. + When not set, the APIVersion of the ResourceSet is used. + enum: + - fluxcd.controlplane.io/v1 + type: string + kind: + description: Kind of the input provider resource. + enum: + - ResourceSetInputProvider + type: string + name: + description: |- + Name of the input provider resource. Cannot be set + when the Selector field is set. + type: string + selector: + description: |- + Selector is a label selector to filter the input provider resources + as an alternative to the Name field. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: object + x-kubernetes-validations: + - message: at least one of name or selector must be set for input + provider references + rule: has(self.name) || has(self.selector) + - message: cannot set both name and selector for input provider + references + rule: '!has(self.name) || !has(self.selector)' + type: array + resources: + description: Resources contains the list of Kubernetes resources to + reconcile. + items: + x-kubernetes-preserve-unknown-fields: true + type: array + resourcesTemplate: + description: |- + ResourcesTemplate is a Go template that generates the list of + Kubernetes resources to reconcile. The template is rendered + as multi-document YAML, the resources should be separated by '---'. + When both Resources and ResourcesTemplate are set, the resulting + objects are merged and deduplicated, with the ones from Resources taking precedence. + type: string + serviceAccountName: + description: |- + The name of the Kubernetes service account to impersonate + when reconciling the generated resources. + type: string + wait: + description: |- + Wait instructs the controller to check the health + of all the reconciled resources. + type: boolean + type: object + status: + description: ResourceSetStatus defines the observed state of ResourceSet. + properties: + conditions: + description: Conditions contains the readiness conditions of the object. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + history: + description: |- + History contains the reconciliation history of the ResourceSet + as a list of snapshots ordered by the last reconciled time. + items: + description: |- + Snapshot represents a point-in-time record of a group of resources reconciliation, + including timing information, status, and a unique digest identifier. + properties: + digest: + description: Digest is the checksum in the format `:` + of the resources in this snapshot. + type: string + firstReconciled: + description: FirstReconciled is the time when this revision + was first reconciled to the cluster. + format: date-time + type: string + lastReconciled: + description: LastReconciled is the time when this revision was + last reconciled to the cluster. + format: date-time + type: string + lastReconciledDuration: + description: LastReconciledDuration is time it took to reconcile + the resources in this revision. + type: string + lastReconciledStatus: + description: LastReconciledStatus is the status of the last + reconciliation. + type: string + metadata: + additionalProperties: + type: string + description: Metadata contains additional information about + the snapshot. + type: object + totalReconciliations: + description: TotalReconciliations is the total number of reconciliations + that have occurred for this snapshot. + format: int64 + type: integer + required: + - digest + - firstReconciled + - lastReconciled + - lastReconciledDuration + - lastReconciledStatus + - totalReconciliations + type: object + type: array + inventory: + description: |- + Inventory contains a list of Kubernetes resource object references + last applied on the cluster. + properties: + entries: + description: Entries of Kubernetes resource object references. + items: + description: ResourceRef contains the information necessary + to locate a resource within a cluster. + properties: + id: + description: |- + ID is the string representation of the Kubernetes resource object's metadata, + in the format '___'. + type: string + v: + description: Version is the API version of the Kubernetes + resource object's kind. + type: string + required: + - id + - v + type: object + type: array + required: + - entries + type: object + lastAppliedRevision: + description: |- + LastAppliedRevision is the digest of the + generated resources that were last reconcile. + type: string + lastHandledReconcileAt: + description: |- + LastHandledReconcileAt holds the value of the most recent + reconcile request value, so a change of the annotation value + can be detected. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: flux-operator/templates/aggregate-clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: flux-operator-edit + labels: + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-admin: "true" + helm.sh/chart: flux-operator-0.37.1 + app.kubernetes.io/name: flux-operator + app.kubernetes.io/instance: flux-operator + app.kubernetes.io/version: "v0.37.1" + app.kubernetes.io/managed-by: Helm +rules: + - apiGroups: + - fluxcd.controlplane.io + resources: + - resourcesets + - resourcesetinputproviders + verbs: + - create + - delete + - deletecollection + - patch + - update +--- +# Source: flux-operator/templates/aggregate-clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: flux-operator-view + labels: + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-view: "true" + helm.sh/chart: flux-operator-0.37.1 + app.kubernetes.io/name: flux-operator + app.kubernetes.io/instance: flux-operator + app.kubernetes.io/version: "v0.37.1" + app.kubernetes.io/managed-by: Helm +rules: + - apiGroups: + - fluxcd.controlplane.io + resources: + - resourcesets + - resourcesetinputproviders + verbs: + - get + - list + - watch +--- +# Source: flux-operator/templates/admin-clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: flux-operator + labels: + helm.sh/chart: flux-operator-0.37.1 + app.kubernetes.io/name: flux-operator + app.kubernetes.io/instance: flux-operator + app.kubernetes.io/version: "v0.37.1" + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: flux-operator + namespace: flux-system +--- +# Source: flux-operator/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: flux-operator + namespace: flux-system + labels: + helm.sh/chart: flux-operator-0.37.1 + app.kubernetes.io/name: flux-operator + app.kubernetes.io/instance: flux-operator + app.kubernetes.io/version: "v0.37.1" + app.kubernetes.io/managed-by: Helm +spec: + ports: + - port: 8080 + targetPort: http-metrics + protocol: TCP + name: http + - port: 9080 + targetPort: http-web + protocol: TCP + name: http-web + selector: + app.kubernetes.io/name: flux-operator + app.kubernetes.io/instance: flux-operator +--- +# Source: flux-operator/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flux-operator + namespace: flux-system + labels: + helm.sh/chart: flux-operator-0.37.1 + app.kubernetes.io/name: flux-operator + app.kubernetes.io/instance: flux-operator + app.kubernetes.io/version: "v0.37.1" + app.kubernetes.io/managed-by: Helm +spec: + selector: + matchLabels: + app.kubernetes.io/name: flux-operator + app.kubernetes.io/instance: flux-operator + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + prometheus.io/path: "/metrics" + labels: + helm.sh/chart: flux-operator-0.37.1 + app.kubernetes.io/name: flux-operator + app.kubernetes.io/instance: flux-operator + app.kubernetes.io/version: "v0.37.1" + app.kubernetes.io/managed-by: Helm + spec: + serviceAccountName: flux-operator + containers: + - name: manager + args: + - --log-level=info + env: + - name: RUNTIME_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: REPORTING_INTERVAL + value: 5m + - name: WEB_SERVER_PORT + value: "9080" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + image: "ghcr.io/controlplaneio-fluxcd/flux-operator:v0.37.1" + imagePullPolicy: "IfNotPresent" + ports: + - name: http-metrics + containerPort: 8080 + protocol: TCP + - name: http + containerPort: 8081 + protocol: TCP + - name: http-web + containerPort: 9080 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 2000m + memory: 1Gi + requests: + cpu: 100m + memory: 64Mi + volumeMounts: + - name: temp + mountPath: /tmp + volumes: + - name: temp + emptyDir: {} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux diff --git a/nix/kubernetes/roles/bootstrap/files/flux_namespace.yaml b/nix/kubernetes/roles/bootstrap/files/flux_namespace.yaml new file mode 100644 index 0000000..c00a432 --- /dev/null +++ b/nix/kubernetes/roles/bootstrap/files/flux_namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: flux-system From a3f1dff3513a81fbe15dec86ab1cc6b8ff93d658 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sat, 20 Dec 2025 23:13:51 -0500 Subject: [PATCH 23/81] Move the cluster bootstrap into the keys flake. Bootstrapping the cluster needs access to secrets, so I am moving it into the keys flake. --- nix/kubernetes/configuration.nix | 1 - nix/kubernetes/hosts/worker0/default.nix | 1 - nix/kubernetes/keys/flake.nix | 1 + .../files/manifests}/cilium.yaml | 0 .../files/manifests}/flux.yaml | 0 .../files/manifests}/flux_namespace.yaml | 0 .../files/manifests}/initial_clusterrole.yaml | 0 .../keys/package/bootstrap-script/package.nix | 49 +++++++++++++++++ nix/kubernetes/keys/scope.nix | 1 + nix/kubernetes/roles/bootstrap/default.nix | 54 ------------------- nix/kubernetes/roles/worker_node/default.nix | 4 -- 11 files changed, 51 insertions(+), 60 deletions(-) rename nix/kubernetes/{roles/bootstrap/files => keys/package/bootstrap-script/files/manifests}/cilium.yaml (100%) rename nix/kubernetes/{roles/bootstrap/files => keys/package/bootstrap-script/files/manifests}/flux.yaml (100%) rename nix/kubernetes/{roles/bootstrap/files => keys/package/bootstrap-script/files/manifests}/flux_namespace.yaml (100%) rename nix/kubernetes/{roles/bootstrap/files => keys/package/bootstrap-script/files/manifests}/initial_clusterrole.yaml (100%) create mode 100644 nix/kubernetes/keys/package/bootstrap-script/package.nix delete mode 100644 nix/kubernetes/roles/bootstrap/default.nix diff --git a/nix/kubernetes/configuration.nix b/nix/kubernetes/configuration.nix index 34e2f86..e6d5ef5 100644 --- a/nix/kubernetes/configuration.nix +++ b/nix/kubernetes/configuration.nix @@ -7,7 +7,6 @@ { imports = [ ./roles/boot - ./roles/bootstrap ./roles/cilium ./roles/containerd ./roles/control_plane diff --git a/nix/kubernetes/hosts/worker0/default.nix b/nix/kubernetes/hosts/worker0/default.nix index e7e954f..59f80f3 100644 --- a/nix/kubernetes/hosts/worker0/default.nix +++ b/nix/kubernetes/hosts/worker0/default.nix @@ -102,7 +102,6 @@ # nix.sshServe.enable = true; # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ]; - me.bootstrap.enable = true; me.dont_use_substituters.enable = true; me.minimal_base.enable = true; me.worker_node.enable = true; diff --git a/nix/kubernetes/keys/flake.nix b/nix/kubernetes/keys/flake.nix index af515b5..e763209 100644 --- a/nix/kubernetes/keys/flake.nix +++ b/nix/kubernetes/keys/flake.nix @@ -20,6 +20,7 @@ { deploy_script = appliedOverlay.k8s.deploy_script; default = appliedOverlay.k8s.all_keys; + bootstrap_script = appliedOverlay.k8s.bootstrap_script; } ); overlays.default = ( diff --git a/nix/kubernetes/roles/bootstrap/files/cilium.yaml b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/cilium.yaml similarity index 100% rename from nix/kubernetes/roles/bootstrap/files/cilium.yaml rename to nix/kubernetes/keys/package/bootstrap-script/files/manifests/cilium.yaml diff --git a/nix/kubernetes/roles/bootstrap/files/flux.yaml b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux.yaml similarity index 100% rename from nix/kubernetes/roles/bootstrap/files/flux.yaml rename to nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux.yaml diff --git a/nix/kubernetes/roles/bootstrap/files/flux_namespace.yaml b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_namespace.yaml similarity index 100% rename from nix/kubernetes/roles/bootstrap/files/flux_namespace.yaml rename to nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_namespace.yaml diff --git a/nix/kubernetes/roles/bootstrap/files/initial_clusterrole.yaml b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/initial_clusterrole.yaml similarity index 100% rename from nix/kubernetes/roles/bootstrap/files/initial_clusterrole.yaml rename to nix/kubernetes/keys/package/bootstrap-script/files/manifests/initial_clusterrole.yaml diff --git a/nix/kubernetes/keys/package/bootstrap-script/package.nix b/nix/kubernetes/keys/package/bootstrap-script/package.nix new file mode 100644 index 0000000..bfaa847 --- /dev/null +++ b/nix/kubernetes/keys/package/bootstrap-script/package.nix @@ -0,0 +1,49 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + config, + lib, + stdenv, + writeShellScript, + k8s, + openssh, + ... +}: +let + bootstrap_script = (writeShellScript "bootstrap-script" bootstrap_script_body); + bootstrap_script_body = ('' + set -euo pipefail + IFS=$'\n\t' + DIR="$( cd "$( dirname "''${BASH_SOURCE[0]}" )" && pwd )" + + ${apply_manifests} + echo "Bootstrap finished" + ''); + manifests = ( + lib.concatMapStringsSep "," lib.escapeShellArg [ + ./files/manifests/initial_clusterrole.yaml + ./files/manifests/cilium.yaml + ./files/manifests/flux_namespace.yaml + ./files/manifests/flux.yaml + ] + ); + apply_manifests = "kubectl --kubeconfig=${k8s.client-configs.admin}/admin.kubeconfig apply --server-side --force-conflicts -f ${manifests}"; +in +stdenv.mkDerivation (finalAttrs: { + name = "bootstrap-script"; + nativeBuildInputs = [ ]; + buildInputs = [ ]; + + unpackPhase = "true"; + + installPhase = '' + cp ${bootstrap_script} "$out" + ''; +}) diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index bdb5d53..e36cde1 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -166,5 +166,6 @@ makeScope newScope ( encryption_config = (callPackage ./package/k8s-encryption-key/package.nix additional_vars); all_keys = (callPackage ./package/k8s-keys/package.nix additional_vars); deploy_script = (callPackage ./package/deploy-script/package.nix additional_vars); + bootstrap_script = (callPackage ./package/bootstrap-script/package.nix additional_vars); } ) diff --git a/nix/kubernetes/roles/bootstrap/default.nix b/nix/kubernetes/roles/bootstrap/default.nix deleted file mode 100644 index 2eda302..0000000 --- a/nix/kubernetes/roles/bootstrap/default.nix +++ /dev/null @@ -1,54 +0,0 @@ -{ - config, - lib, - pkgs, - ... -}: - -{ - imports = [ ]; - - options.me = { - bootstrap.enable = lib.mkOption { - type = lib.types.bool; - default = false; - example = true; - description = "Whether we want to install bootstrap."; - }; - - bootstrap.manifests = lib.mkOption { - type = lib.types.listOf lib.types.str; - default = [ ]; - example = lib.literalExpression ''[ ${./files/clusterrole.yaml} ]''; - description = "List of kubernetes manifests to load into the cluster."; - }; - }; - - config = - lib.mkIf (config.me.bootstrap.enable && ((builtins.length config.me.bootstrap.manifests) > 0)) - { - systemd.services.kube-bootstrap = { - enable = true; - description = "Load initial kubernetes manifests into the cluster"; - after = [ "kubernetes.target" ]; - wantedBy = [ "multi-user.target" ]; - path = with pkgs; [ - kubectl - ]; - unitConfig.DefaultDependencies = "no"; - serviceConfig = { - Type = "oneshot"; - }; - script = - let - manifests = (lib.concatMapStringsSep "," lib.escapeShellArg config.me.bootstrap.manifests); - in - '' - set -o pipefail - IFS=$'\n\t' - - kubectl --kubeconfig=/.persist/keys/kube/kubelet.kubeconfig apply --server-side --force-conflicts -f ${manifests} - ''; - }; - }; -} diff --git a/nix/kubernetes/roles/worker_node/default.nix b/nix/kubernetes/roles/worker_node/default.nix index 7a2edc3..804d6f8 100644 --- a/nix/kubernetes/roles/worker_node/default.nix +++ b/nix/kubernetes/roles/worker_node/default.nix @@ -18,10 +18,6 @@ }; config = lib.mkIf config.me.worker_node.enable { - me.bootstrap.manifests = [ - "${../bootstrap/files/initial_clusterrole.yaml}" - "${../bootstrap/files/cilium.yaml}" - ]; me.cilium.enable = true; me.containerd.enable = true; me.firewall.enable = true; From ceb627bb35f8d513ec5c1242726d88b5e92fdc8e Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 21 Dec 2025 00:01:57 -0500 Subject: [PATCH 24/81] Generate ssh keys for flux bootstrap. --- .../keys/package/k8s-keys/package.nix | 3 +- .../keys/package/ssh-key/package.nix | 33 +++++++++++++++++++ nix/kubernetes/keys/scope.nix | 5 +++ 3 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 nix/kubernetes/keys/package/ssh-key/package.nix diff --git a/nix/kubernetes/keys/package/k8s-keys/package.nix b/nix/kubernetes/keys/package/k8s-keys/package.nix index 05bc44e..f6e5efb 100644 --- a/nix/kubernetes/keys/package/k8s-keys/package.nix +++ b/nix/kubernetes/keys/package/k8s-keys/package.nix @@ -10,5 +10,6 @@ symlinkJoin { k8s.encryption_config ] ++ (builtins.attrValues k8s.keys) - ++ (builtins.attrValues k8s.client-configs); + ++ (builtins.attrValues k8s.client-configs) + ++ (builtins.attrValues k8s.ssh-keys); } diff --git a/nix/kubernetes/keys/package/ssh-key/package.nix b/nix/kubernetes/keys/package/ssh-key/package.nix new file mode 100644 index 0000000..01a2366 --- /dev/null +++ b/nix/kubernetes/keys/package/ssh-key/package.nix @@ -0,0 +1,33 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + lib, + stdenv, + k8s, + openssh, + key_name, + ... +}: +stdenv.mkDerivation (finalAttrs: { + name = "ssh-key-${key_name}"; + nativeBuildInputs = [ openssh ]; + buildInputs = [ ]; + + unpackPhase = "true"; + + buildPhase = '' + ssh-keygen -t ed25519 -f ${key_name} -N "" + ''; + + installPhase = '' + mkdir "$out" + cp "${key_name}" "${key_name}.pub" $out/ + ''; +}) diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index e36cde1..ae62bd2 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -96,6 +96,11 @@ makeScope newScope ( "service-accounts" ] (key_name: (callPackage ./package/tls-key/package.nix (additional_vars // { inherit key_name; }))) ); + ssh-keys = ( + lib.genAttrs [ + "flux_ssh_key" + ] (key_name: (callPackage ./package/ssh-key/package.nix (additional_vars // { inherit key_name; }))) + ); client-configs = ( builtins.mapAttrs ( From 158c622840913ceac06195ed2e7ca179e8fce1c5 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 21 Dec 2025 14:17:31 -0500 Subject: [PATCH 25/81] Generate pgp keys for sops. --- .../keys/package/k8s-keys/package.nix | 3 +- .../keys/package/pgp-key/package.nix | 50 +++++++++++++++++++ nix/kubernetes/keys/scope.nix | 13 +++++ 3 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 nix/kubernetes/keys/package/pgp-key/package.nix diff --git a/nix/kubernetes/keys/package/k8s-keys/package.nix b/nix/kubernetes/keys/package/k8s-keys/package.nix index f6e5efb..13def7e 100644 --- a/nix/kubernetes/keys/package/k8s-keys/package.nix +++ b/nix/kubernetes/keys/package/k8s-keys/package.nix @@ -11,5 +11,6 @@ symlinkJoin { ] ++ (builtins.attrValues k8s.keys) ++ (builtins.attrValues k8s.client-configs) - ++ (builtins.attrValues k8s.ssh-keys); + ++ (builtins.attrValues k8s.ssh-keys) + ++ (builtins.attrValues k8s.pgp-keys); } diff --git a/nix/kubernetes/keys/package/pgp-key/package.nix b/nix/kubernetes/keys/package/pgp-key/package.nix new file mode 100644 index 0000000..fa69f12 --- /dev/null +++ b/nix/kubernetes/keys/package/pgp-key/package.nix @@ -0,0 +1,50 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + stdenv, + gnupg, + key_name, + expire_date ? "0", + pgp_comment ? "${key_name}", + pgp_name ? "${key_name}", + ... +}: +stdenv.mkDerivation (finalAttrs: { + name = "pgp-key-${key_name}"; + nativeBuildInputs = [ gnupg ]; + buildInputs = [ ]; + + unpackPhase = "true"; + + buildPhase = '' + mkdir keyring + export GNUPGHOME=$(readlink -f keyring) + + gpg --batch --full-generate-key < "$out/${key_name}_private_key.asc" + gpg --export --armor "${pgp_name}" > "$out/${key_name}_public_key.asc" + ''; +}) diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index ae62bd2..5f33152 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -101,6 +101,19 @@ makeScope newScope ( "flux_ssh_key" ] (key_name: (callPackage ./package/ssh-key/package.nix (additional_vars // { inherit key_name; }))) ); + pgp-keys = ( + builtins.mapAttrs + ( + key_name: key_config: + (callPackage ./package/pgp-key/package.nix (additional_vars // { inherit key_name; } // key_config)) + ) + { + "flux_gpg" = { + pgp_comment = "flux secrets"; + pgp_name = "flux sops"; + }; + } + ); client-configs = ( builtins.mapAttrs ( From 903c1c1f4199a6c63a9cbb14ba408e6c0244dc0e Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 21 Dec 2025 18:23:27 -0500 Subject: [PATCH 26/81] Install CoreDNS. --- .../files/manifests/coredns.yaml | 214 ++++++++++++++++++ .../files/manifests/flux_instance.yaml | 9 + .../keys/package/bootstrap-script/package.nix | 2 + .../roles/kubelet/files/kubelet-config.yaml | 2 +- 4 files changed, 226 insertions(+), 1 deletion(-) create mode 100644 nix/kubernetes/keys/package/bootstrap-script/files/manifests/coredns.yaml create mode 100644 nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_instance.yaml diff --git a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/coredns.yaml b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/coredns.yaml new file mode 100644 index 0000000..e1046cd --- /dev/null +++ b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/coredns.yaml @@ -0,0 +1,214 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: coredns + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + kubernetes.io/bootstrapping: rbac-defaults + addonmanager.kubernetes.io/mode: Reconcile + name: system:coredns +rules: + - apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + kubernetes.io/bootstrapping: rbac-defaults + addonmanager.kubernetes.io/mode: EnsureExists + name: system:coredns +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:coredns +subjects: + - kind: ServiceAccount + name: coredns + namespace: kube-system +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: EnsureExists +data: + Corefile: | + .:53 { + errors + health { + lameduck 5s + } + ready + kubernetes cluster.local in-addr.arpa ip6.arpa { + pods insecure + fallthrough in-addr.arpa ip6.arpa + ttl 30 + } + prometheus :9153 + forward . /etc/resolv.conf { + max_concurrent 1000 + } + cache 30 + loop + reload + loadbalance + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: coredns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/name: "CoreDNS" +spec: + # replicas: not specified here: + # 1. In order to make Addon Manager do not reconcile this replicas parameter. + # 2. Default is 1. + # 3. Will be tuned in real time if DNS horizontal auto-scaling is turned on. + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: kube-dns + template: + metadata: + labels: + k8s-app: kube-dns + spec: + securityContext: + seccompProfile: + type: RuntimeDefault + priorityClassName: system-cluster-critical + serviceAccountName: coredns + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: k8s-app + operator: In + values: ["kube-dns"] + topologyKey: kubernetes.io/hostname + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + nodeSelector: + kubernetes.io/os: linux + containers: + - name: coredns + image: registry.k8s.io/coredns/coredns:v1.13.1 + imagePullPolicy: IfNotPresent + resources: + limits: + memory: 70Mi + requests: + cpu: 100m + memory: 70Mi + args: ["-conf", "/etc/coredns/Corefile"] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + readOnly: true + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9153 + name: metrics + protocol: TCP + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /ready + port: 8181 + scheme: HTTP + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + readOnlyRootFilesystem: true + dnsPolicy: Default + volumes: + - name: config-volume + configMap: + name: coredns + items: + - key: Corefile + path: Corefile +--- +apiVersion: v1 +kind: Service +metadata: + name: kube-dns + namespace: kube-system + annotations: + prometheus.io/port: "9153" + prometheus.io/scrape: "true" + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/name: "CoreDNS" +spec: + selector: + k8s-app: kube-dns + # clusterIP: $DNS_SERVER_IP + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP + - name: metrics + port: 9153 + protocol: TCP diff --git a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_instance.yaml b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_instance.yaml new file mode 100644 index 0000000..e0c756a --- /dev/null +++ b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_instance.yaml @@ -0,0 +1,9 @@ +apiVersion: fluxcd.controlplane.io/v1 +kind: FluxInstance +metadata: + name: flux + namespace: flux-system +spec: + distribution: + version: "2.7.x" + registry: "ghcr.io/fluxcd" diff --git a/nix/kubernetes/keys/package/bootstrap-script/package.nix b/nix/kubernetes/keys/package/bootstrap-script/package.nix index bfaa847..ce3d0e3 100644 --- a/nix/kubernetes/keys/package/bootstrap-script/package.nix +++ b/nix/kubernetes/keys/package/bootstrap-script/package.nix @@ -30,8 +30,10 @@ let lib.concatMapStringsSep "," lib.escapeShellArg [ ./files/manifests/initial_clusterrole.yaml ./files/manifests/cilium.yaml + ./files/manifests/coredns.yaml ./files/manifests/flux_namespace.yaml ./files/manifests/flux.yaml + ./files/manifests/flux_instance.yaml ] ); apply_manifests = "kubectl --kubeconfig=${k8s.client-configs.admin}/admin.kubeconfig apply --server-side --force-conflicts -f ${manifests}"; diff --git a/nix/kubernetes/roles/kubelet/files/kubelet-config.yaml b/nix/kubernetes/roles/kubelet/files/kubelet-config.yaml index 2313c53..7089087 100644 --- a/nix/kubernetes/roles/kubelet/files/kubelet-config.yaml +++ b/nix/kubernetes/roles/kubelet/files/kubelet-config.yaml @@ -18,7 +18,7 @@ maxPods: 16 memorySwap: swapBehavior: NoSwap port: 10250 -resolvConf: "/etc/resolv.conf" +resolvConf: "/run/systemd/resolve/resolv.conf" registerNode: true runtimeRequestTimeout: "15m" tlsCertFile: "/.persist/keys/kube/kubelet.crt" From 3c20fd49f022502574efefd0259789759db38d7e Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 21 Dec 2025 18:45:49 -0500 Subject: [PATCH 27/81] Generate kubernetes secrets for ssh keys. --- nix/kubernetes/README.org | 4 +- .../keys/package/bootstrap-script/package.nix | 1 + .../keys/package/k8s-keys/package.nix | 3 +- .../keys/package/k8s-secret-ssh/package.nix | 39 +++++++++++++++++++ nix/kubernetes/keys/scope.nix | 15 +++++++ 5 files changed, 59 insertions(+), 3 deletions(-) create mode 100644 nix/kubernetes/keys/package/k8s-secret-ssh/package.nix diff --git a/nix/kubernetes/README.org b/nix/kubernetes/README.org index 1662ba4..40e71b7 100644 --- a/nix/kubernetes/README.org +++ b/nix/kubernetes/README.org @@ -110,8 +110,8 @@ registry: "ghcr.io/fluxcd" sync: kind: GitRepository - url: "https://gitlab.com/my-org/my-fleet.git" - ref: "refs/heads/main" + url: "ssh://git@10.215.1.210:22/repos/mrmanager" + ref: "refs/heads/nix" path: "clusters/my-cluster" pullSecret: "flux-system" #+end_src diff --git a/nix/kubernetes/keys/package/bootstrap-script/package.nix b/nix/kubernetes/keys/package/bootstrap-script/package.nix index ce3d0e3..5ebacb2 100644 --- a/nix/kubernetes/keys/package/bootstrap-script/package.nix +++ b/nix/kubernetes/keys/package/bootstrap-script/package.nix @@ -34,6 +34,7 @@ let ./files/manifests/flux_namespace.yaml ./files/manifests/flux.yaml ./files/manifests/flux_instance.yaml + "${k8s.k8s-ssh-secrets.kubernetes-deploy-key}/kubernetes-deploy-key.yaml" ] ); apply_manifests = "kubectl --kubeconfig=${k8s.client-configs.admin}/admin.kubeconfig apply --server-side --force-conflicts -f ${manifests}"; diff --git a/nix/kubernetes/keys/package/k8s-keys/package.nix b/nix/kubernetes/keys/package/k8s-keys/package.nix index 13def7e..5e144bb 100644 --- a/nix/kubernetes/keys/package/k8s-keys/package.nix +++ b/nix/kubernetes/keys/package/k8s-keys/package.nix @@ -12,5 +12,6 @@ symlinkJoin { ++ (builtins.attrValues k8s.keys) ++ (builtins.attrValues k8s.client-configs) ++ (builtins.attrValues k8s.ssh-keys) - ++ (builtins.attrValues k8s.pgp-keys); + ++ (builtins.attrValues k8s.pgp-keys) + ++ (builtins.attrValues k8s.k8s-ssh-secrets); } diff --git a/nix/kubernetes/keys/package/k8s-secret-ssh/package.nix b/nix/kubernetes/keys/package/k8s-secret-ssh/package.nix new file mode 100644 index 0000000..88411dd --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-secret-ssh/package.nix @@ -0,0 +1,39 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + stdenv, + k8s, + kubectl, + secret_name, + secret_namespace, + ssh_key_name, + ... +}: +stdenv.mkDerivation (finalAttrs: { + name = "k8s-secret-ssh-${secret_name}"; + nativeBuildInputs = [ kubectl ]; + buildInputs = [ ]; + + unpackPhase = "true"; + + buildPhase = '' + kubectl create secret generic ${secret_name} \ + --namespace ${secret_namespace} \ + --from-file=identity=${k8s.ssh-keys."${ssh_key_name}"}/${ssh_key_name} \ + --from-file=identity.pub=${k8s.ssh-keys."${ssh_key_name}"}/${ssh_key_name}.pub \ + --dry-run=client -o yaml > ${secret_name}.yaml + ''; + # --from-file=known_hosts=$(OUT)/known_hosts \ + + installPhase = '' + mkdir "$out" + cp "${secret_name}.yaml" $out/ + ''; +}) diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index 5f33152..fb8a640 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -114,6 +114,21 @@ makeScope newScope ( }; } ); + k8s-ssh-secrets = ( + builtins.mapAttrs + ( + secret_name: secret_config: + (callPackage ./package/k8s-secret-ssh/package.nix ( + additional_vars // { inherit secret_name; } // secret_config + )) + ) + { + "kubernetes-deploy-key" = { + secret_namespace = "flux-system"; + ssh_key_name = "flux_ssh_key"; + }; + } + ); client-configs = ( builtins.mapAttrs ( From 1700c429265d26044d82f1dde3b0aeb0b75d98de Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 21 Dec 2025 19:43:43 -0500 Subject: [PATCH 28/81] Generic secrets for pgp keys. --- .../keys/contrib/base64/package.nix | 62 +++++++++++++++++++ .../keys/package/k8s-keys/package.nix | 3 +- .../package/k8s-secret-generic/package.nix | 52 ++++++++++++++++ nix/kubernetes/keys/scope.nix | 17 +++++ 4 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 nix/kubernetes/keys/contrib/base64/package.nix create mode 100644 nix/kubernetes/keys/package/k8s-secret-generic/package.nix diff --git a/nix/kubernetes/keys/contrib/base64/package.nix b/nix/kubernetes/keys/contrib/base64/package.nix new file mode 100644 index 0000000..50e7a4b --- /dev/null +++ b/nix/kubernetes/keys/contrib/base64/package.nix @@ -0,0 +1,62 @@ +# From: https://gist.github.com/manveru/74eb41d850bc146b7e78c4cb059507e2 +# From: https://discourse.nixos.org/t/string-to-base-64/32624/3 +{ lib, ... }: +{ + toBase64 = + text: + let + inherit (lib) + sublist + mod + stringToCharacters + concatMapStrings + ; + inherit (lib.strings) charToInt; + inherit (builtins) + substring + foldl' + genList + elemAt + length + concatStringsSep + stringLength + ; + lookup = stringToCharacters "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + sliceN = + size: list: n: + sublist (n * size) size list; + pows = [ + (64 * 64 * 64) + (64 * 64) + 64 + 1 + ]; + intSextets = i: map (j: mod (i / j) 64) pows; + compose = + f: g: x: + f (g x); + intToChar = elemAt lookup; + convertTripletInt = sliceInt: concatMapStrings intToChar (intSextets sliceInt); + sliceToInt = foldl' (acc: val: acc * 256 + val) 0; + convertTriplet = compose convertTripletInt sliceToInt; + join = concatStringsSep ""; + convertLastSlice = + slice: + let + len = length slice; + in + if len == 1 then + (substring 0 2 (convertTripletInt ((sliceToInt slice) * 256 * 256))) + "==" + else if len == 2 then + (substring 0 3 (convertTripletInt ((sliceToInt slice) * 256))) + "=" + else + ""; + len = stringLength text; + nFullSlices = len / 3; + bytes = map charToInt (stringToCharacters text); + tripletAt = sliceN 3 bytes; + head = genList (compose convertTriplet tripletAt) nFullSlices; + tail = convertLastSlice (tripletAt nFullSlices); + in + join (head ++ [ tail ]); +} diff --git a/nix/kubernetes/keys/package/k8s-keys/package.nix b/nix/kubernetes/keys/package/k8s-keys/package.nix index 5e144bb..5f3ac7a 100644 --- a/nix/kubernetes/keys/package/k8s-keys/package.nix +++ b/nix/kubernetes/keys/package/k8s-keys/package.nix @@ -13,5 +13,6 @@ symlinkJoin { ++ (builtins.attrValues k8s.client-configs) ++ (builtins.attrValues k8s.ssh-keys) ++ (builtins.attrValues k8s.pgp-keys) - ++ (builtins.attrValues k8s.k8s-ssh-secrets); + ++ (builtins.attrValues k8s.k8s-ssh-secrets) + ++ (builtins.attrValues k8s.k8s-secrets-generic); } diff --git a/nix/kubernetes/keys/package/k8s-secret-generic/package.nix b/nix/kubernetes/keys/package/k8s-secret-generic/package.nix new file mode 100644 index 0000000..7d4249b --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-secret-generic/package.nix @@ -0,0 +1,52 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + lib, + pkgs, + stdenv, + k8s, + kubectl, + secret_name, + secret_namespace, + secret_values ? { }, + ... +}: +let + toBase64 = (pkgs.callPackage ../../contrib/base64/package.nix { inherit lib; }).toBase64; + secret_yaml = { + apiVersion = "v1"; + kind = "Secret"; + metadata = { + name = "${secret_name}"; + namespace = "${secret_namespace}"; + }; + data = (builtins.mapAttrs (key: val: (toBase64 val)) secret_values); + }; + settingsFormat = pkgs.formats.yaml { }; + yaml_body = settingsFormat.generate "${secret_name}.yaml" secret_yaml; + yaml_file = pkgs.writeTextFile { + name = "${secret_name}.yaml"; + text = (builtins.readFile yaml_body); + }; +in +stdenv.mkDerivation (finalAttrs: { + name = "k8s-secret-generic-${secret_name}"; + nativeBuildInputs = [ kubectl ]; + buildInputs = [ ]; + + unpackPhase = "true"; + + # lib.attrsets.mapAttrsToList + + installPhase = '' + mkdir "$out" + cp "${yaml_file}" "$out/${secret_name}.yaml" + ''; +}) diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index fb8a640..abc7b77 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -129,6 +129,23 @@ makeScope newScope ( }; } ); + k8s-secrets-generic = ( + builtins.mapAttrs + ( + secret_name: secret_config: + (callPackage ./package/k8s-secret-generic/package.nix ( + additional_vars // { inherit secret_name; } // secret_config + )) + ) + { + "sops-gpg" = { + secret_namespace = "flux-system"; + secret_values = { + "sops.asc" = (builtins.readFile "${self.pgp-keys.flux_gpg}/flux_gpg_private_key.asc"); + }; + }; + } + ); client-configs = ( builtins.mapAttrs ( From dbd82b998d94c9591106f0c0a0a500e3c481bcde Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 21 Dec 2025 22:41:21 -0500 Subject: [PATCH 29/81] Generic secrets for ssh keys. --- .../keys/package/bootstrap-script/package.nix | 22 ++++++----- .../keys/package/k8s-keys/package.nix | 1 - .../package/k8s-secret-generic/package.nix | 2 - .../keys/package/k8s-secret-ssh/package.nix | 39 ------------------- nix/kubernetes/keys/scope.nix | 22 ++++------- 5 files changed, 20 insertions(+), 66 deletions(-) delete mode 100644 nix/kubernetes/keys/package/k8s-secret-ssh/package.nix diff --git a/nix/kubernetes/keys/package/bootstrap-script/package.nix b/nix/kubernetes/keys/package/bootstrap-script/package.nix index 5ebacb2..31a7df0 100644 --- a/nix/kubernetes/keys/package/bootstrap-script/package.nix +++ b/nix/kubernetes/keys/package/bootstrap-script/package.nix @@ -27,15 +27,19 @@ let echo "Bootstrap finished" ''); manifests = ( - lib.concatMapStringsSep "," lib.escapeShellArg [ - ./files/manifests/initial_clusterrole.yaml - ./files/manifests/cilium.yaml - ./files/manifests/coredns.yaml - ./files/manifests/flux_namespace.yaml - ./files/manifests/flux.yaml - ./files/manifests/flux_instance.yaml - "${k8s.k8s-ssh-secrets.kubernetes-deploy-key}/kubernetes-deploy-key.yaml" - ] + lib.concatMapStringsSep "," lib.escapeShellArg ( + [ + ./files/manifests/initial_clusterrole.yaml + ./files/manifests/cilium.yaml + ./files/manifests/coredns.yaml + ./files/manifests/flux_namespace.yaml + ./files/manifests/flux.yaml + ./files/manifests/flux_instance.yaml + ] + ++ (lib.attrsets.mapAttrsToList ( + secret_name: secret_value: "${secret_value}/${secret_name}.yaml" + ) k8s.k8s-secrets-generic) + ) ); apply_manifests = "kubectl --kubeconfig=${k8s.client-configs.admin}/admin.kubeconfig apply --server-side --force-conflicts -f ${manifests}"; in diff --git a/nix/kubernetes/keys/package/k8s-keys/package.nix b/nix/kubernetes/keys/package/k8s-keys/package.nix index 5f3ac7a..587749a 100644 --- a/nix/kubernetes/keys/package/k8s-keys/package.nix +++ b/nix/kubernetes/keys/package/k8s-keys/package.nix @@ -13,6 +13,5 @@ symlinkJoin { ++ (builtins.attrValues k8s.client-configs) ++ (builtins.attrValues k8s.ssh-keys) ++ (builtins.attrValues k8s.pgp-keys) - ++ (builtins.attrValues k8s.k8s-ssh-secrets) ++ (builtins.attrValues k8s.k8s-secrets-generic); } diff --git a/nix/kubernetes/keys/package/k8s-secret-generic/package.nix b/nix/kubernetes/keys/package/k8s-secret-generic/package.nix index 7d4249b..6aea5e0 100644 --- a/nix/kubernetes/keys/package/k8s-secret-generic/package.nix +++ b/nix/kubernetes/keys/package/k8s-secret-generic/package.nix @@ -43,8 +43,6 @@ stdenv.mkDerivation (finalAttrs: { unpackPhase = "true"; - # lib.attrsets.mapAttrsToList - installPhase = '' mkdir "$out" cp "${yaml_file}" "$out/${secret_name}.yaml" diff --git a/nix/kubernetes/keys/package/k8s-secret-ssh/package.nix b/nix/kubernetes/keys/package/k8s-secret-ssh/package.nix deleted file mode 100644 index 88411dd..0000000 --- a/nix/kubernetes/keys/package/k8s-secret-ssh/package.nix +++ /dev/null @@ -1,39 +0,0 @@ -# unpackPhase -# patchPhase -# configurePhase -# buildPhase -# checkPhase -# installPhase -# fixupPhase -# installCheckPhase -# distPhase -{ - stdenv, - k8s, - kubectl, - secret_name, - secret_namespace, - ssh_key_name, - ... -}: -stdenv.mkDerivation (finalAttrs: { - name = "k8s-secret-ssh-${secret_name}"; - nativeBuildInputs = [ kubectl ]; - buildInputs = [ ]; - - unpackPhase = "true"; - - buildPhase = '' - kubectl create secret generic ${secret_name} \ - --namespace ${secret_namespace} \ - --from-file=identity=${k8s.ssh-keys."${ssh_key_name}"}/${ssh_key_name} \ - --from-file=identity.pub=${k8s.ssh-keys."${ssh_key_name}"}/${ssh_key_name}.pub \ - --dry-run=client -o yaml > ${secret_name}.yaml - ''; - # --from-file=known_hosts=$(OUT)/known_hosts \ - - installPhase = '' - mkdir "$out" - cp "${secret_name}.yaml" $out/ - ''; -}) diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index abc7b77..93dff6f 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -114,21 +114,6 @@ makeScope newScope ( }; } ); - k8s-ssh-secrets = ( - builtins.mapAttrs - ( - secret_name: secret_config: - (callPackage ./package/k8s-secret-ssh/package.nix ( - additional_vars // { inherit secret_name; } // secret_config - )) - ) - { - "kubernetes-deploy-key" = { - secret_namespace = "flux-system"; - ssh_key_name = "flux_ssh_key"; - }; - } - ); k8s-secrets-generic = ( builtins.mapAttrs ( @@ -144,6 +129,13 @@ makeScope newScope ( "sops.asc" = (builtins.readFile "${self.pgp-keys.flux_gpg}/flux_gpg_private_key.asc"); }; }; + "kubernetes-deploy-key" = { + secret_namespace = "flux-system"; + secret_values = { + "identity" = builtins.readFile "${self.ssh-keys.flux_ssh_key}/flux_ssh_key"; + "identity.pub" = builtins.readFile "${self.ssh-keys.flux_ssh_key}/flux_ssh_key.pub"; + }; + }; } ); client-configs = ( From 88b1c36c0d39ef10dbd1be8dc73739319f98fbfb Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 21 Dec 2025 23:26:15 -0500 Subject: [PATCH 30/81] Trust flux's ssh key in the yaml git repo. --- .../keys/package/bootstrap-script/package.nix | 3 ++ .../keys/package/deploy-script/package.nix | 33 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/nix/kubernetes/keys/package/bootstrap-script/package.nix b/nix/kubernetes/keys/package/bootstrap-script/package.nix index 31a7df0..298ac62 100644 --- a/nix/kubernetes/keys/package/bootstrap-script/package.nix +++ b/nix/kubernetes/keys/package/bootstrap-script/package.nix @@ -39,6 +39,9 @@ let ++ (lib.attrsets.mapAttrsToList ( secret_name: secret_value: "${secret_value}/${secret_name}.yaml" ) k8s.k8s-secrets-generic) + ++ [ + ./files/manifests/flux_apply_git.yaml + ] ) ); apply_manifests = "kubectl --kubeconfig=${k8s.client-configs.admin}/admin.kubeconfig apply --server-side --force-conflicts -f ${manifests}"; diff --git a/nix/kubernetes/keys/package/deploy-script/package.nix b/nix/kubernetes/keys/package/deploy-script/package.nix index 8b6844d..e809d9a 100644 --- a/nix/kubernetes/keys/package/deploy-script/package.nix +++ b/nix/kubernetes/keys/package/deploy-script/package.nix @@ -46,6 +46,13 @@ let "nw1" "nw2" ]) + + (trust_ssh_key { + public_key = "${k8s.ssh-keys.flux_ssh_key}/flux_ssh_key.pub"; + destination = "/jail/admin_git/usr/home/git/.ssh/authorized_keys"; + owner = "11236"; + group = "11236"; + mode = "0600"; + }) ); deploy_script = (writeShellScript "deploy-script" deploy_script_body); deploy_file = ( @@ -227,6 +234,32 @@ let ]) ) ); + trust_ssh_key = + { + public_key, + destination, + owner, + group, + mode, + }: + let + public_key_name = builtins.baseNameOf public_key; + public_key_contents = builtins.readFile public_key; + trimmed = lib.strings.trim public_key_contents; + escaped = lib.strings.escapeShellArg trimmed; + in + '' + ## + ## trust ${public_key_name} in ${destination} + ## + if ! ${openssh}/bin/ssh mrmanager doas grep -q "${escaped}" "${destination}"; then + ${openssh}/bin/ssh mrmanager doas tee -a "${destination}" <<<"$(cat ${public_key})" + ${openssh}/bin/ssh mrmanager doas chown "${owner}:${group}" "${destination}" + ${openssh}/bin/ssh mrmanager doas chmod "${mode}" "${destination}" + else + echo "${public_key_name} is already trusted in ${destination}" + fi + ''; in stdenv.mkDerivation (finalAttrs: { name = "deploy-script"; From c289f46cc5f46e03fde3fd4381443d666fe41009 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 21 Dec 2025 23:48:17 -0500 Subject: [PATCH 31/81] Apply the git repo to the cluster. --- nix/kubernetes/keys/Makefile | 336 +----------------- nix/kubernetes/keys/generated/known_hosts | 8 + .../files/manifests/flux_apply_git.yaml | 34 ++ .../keys/package/k8s-keys/package.nix | 9 + nix/kubernetes/keys/scope.nix | 1 + 5 files changed, 56 insertions(+), 332 deletions(-) create mode 100644 nix/kubernetes/keys/generated/known_hosts create mode 100644 nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_apply_git.yaml diff --git a/nix/kubernetes/keys/Makefile b/nix/kubernetes/keys/Makefile index 1f2bb24..24020a7 100644 --- a/nix/kubernetes/keys/Makefile +++ b/nix/kubernetes/keys/Makefile @@ -16,342 +16,14 @@ WORKERS := worker0 worker1 worker2 controller0 controller1 controller2 .PHONY: all all: \ - $(OUT)/ca-key.pem \ - $(OUT)/admin-key.pem \ - $(OUT)/worker0-key.pem \ - $(OUT)/worker1-key.pem \ - $(OUT)/worker2-key.pem \ - $(OUT)/controller0-proxy-key.pem \ - $(OUT)/controller1-proxy-key.pem \ - $(OUT)/controller2-proxy-key.pem \ - $(OUT)/kube-controller-manager-key.pem \ - $(OUT)/kube-proxy-key.pem \ - $(OUT)/kube-scheduler-key.pem \ - $(OUT)/kubernetes-key.pem \ - $(OUT)/service-account-key.pem \ - $(OUT)/worker0.kubeconfig \ - $(OUT)/worker1.kubeconfig \ - $(OUT)/worker2.kubeconfig \ - $(OUT)/controller0.kubeconfig \ - $(OUT)/controller1.kubeconfig \ - $(OUT)/controller2.kubeconfig \ - $(OUT)/kube-proxy.kubeconfig \ - $(OUT)/kube-controller-manager.kubeconfig \ - $(OUT)/kube-scheduler.kubeconfig \ - $(OUT)/admin.kubeconfig \ - $(OUT)/encryption-config.yaml \ - $(OUT)/remote_admin.kubeconfig \ - $(OUT)/requestheader-client-ca-key.pem + $(OUT)/known_hosts .PHONY: clean clean: > rm -rf $(OUT) -# Requestheader client ca -$(OUT)/requestheader-client-ca-key.pem: requestheader-client-ca-csr.json ca-config.json +$(OUT)/: > @mkdir -p $(@D) -> cd $(@D) && cfssl gencert -initca ../requestheader-client-ca-csr.json | cfssljson -bare requestheader-client-ca -# Certificate authority -$(OUT)/ca-key.pem: ca-csr.json ca-config.json -> @mkdir -p $(@D) -> cd $(@D) && cfssl gencert -initca ../ca-csr.json | cfssljson -bare ca - -# Admin client certificate -$(OUT)/admin-key.pem: admin-csr.json ca-config.json -> @mkdir -p $(@D) -> cd $(@D) && cfssl gencert \ -> -ca=ca.pem \ -> -ca-key=ca-key.pem \ -> -config=../ca-config.json \ -> -profile=kubernetes \ -> ../admin-csr.json | cfssljson -bare admin - -# Worker kubelet client certificate -$(OUT)/worker0-key.pem: worker0-csr.json ca-config.json -> @mkdir -p $(@D) -> cd $(@D) && cfssl gencert \ -> -ca=ca.pem \ -> -ca-key=ca-key.pem \ -> -config=../ca-config.json \ -> -hostname=worker0,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.207 \ -> -profile=kubernetes \ -> ../worker0-csr.json | cfssljson -bare worker0 - -# Worker kubelet client certificate -$(OUT)/worker1-key.pem: worker1-csr.json ca-config.json -> @mkdir -p $(@D) -> cd $(@D) && cfssl gencert \ -> -ca=ca.pem \ -> -ca-key=ca-key.pem \ -> -config=../ca-config.json \ -> -hostname=worker1,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.208 \ -> -profile=kubernetes \ -> ../worker1-csr.json | cfssljson -bare worker1 - -# Worker kubelet client certificate -$(OUT)/worker2-key.pem: worker2-csr.json ca-config.json -> @mkdir -p $(@D) -> cd $(@D) && cfssl gencert \ -> -ca=ca.pem \ -> -ca-key=ca-key.pem \ -> -config=../ca-config.json \ -> -hostname=worker2,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.209 \ -> -profile=kubernetes \ -> ../worker2-csr.json | cfssljson -bare worker2 - -# Controller kubelet client certificate -$(OUT)/controller0-key.pem: controller0-csr.json ca-config.json -> @mkdir -p $(@D) -> cd $(@D) && cfssl gencert \ -> -ca=ca.pem \ -> -ca-key=ca-key.pem \ -> -config=../ca-config.json \ -> -hostname=controller0,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.204 \ -> -profile=kubernetes \ -> ../controller0-csr.json | cfssljson -bare controller0 - -# Controller kubelet client certificate -$(OUT)/controller1-key.pem: controller1-csr.json ca-config.json -> @mkdir -p $(@D) -> cd $(@D) && cfssl gencert \ -> -ca=ca.pem \ -> -ca-key=ca-key.pem \ -> -config=../ca-config.json \ -> -hostname=controller1,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.205 \ -> -profile=kubernetes \ -> ../controller1-csr.json | cfssljson -bare controller1 - -# Controller kubelet client certificate -$(OUT)/controller2-key.pem: controller2-csr.json ca-config.json -> @mkdir -p $(@D) -> cd $(@D) && cfssl gencert \ -> -ca=ca.pem \ -> -ca-key=ca-key.pem \ -> -config=../ca-config.json \ -> -hostname=controller2,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.206 \ -> -profile=kubernetes \ -> ../controller2-csr.json | cfssljson -bare controller2 - -# Controller kubelet client certificate -$(OUT)/controller0-proxy-key.pem: controller0-proxy-csr.json ca-config.json $(OUT)/requestheader-client-ca-key.pem -> @mkdir -p $(@D) -> cd $(@D) && cfssl gencert \ -> -ca=requestheader-client-ca.pem \ -> -ca-key=requestheader-client-ca-key.pem \ -> -config=../ca-config.json \ -> -hostname=controller0,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.204 \ -> -profile=kubernetes \ -> ../controller0-proxy-csr.json | cfssljson -bare controller0-proxy - -# Controller kubelet client certificate -$(OUT)/controller1-proxy-key.pem: controller1-proxy-csr.json ca-config.json $(OUT)/requestheader-client-ca-key.pem -> @mkdir -p $(@D) -> cd $(@D) && cfssl gencert \ -> -ca=requestheader-client-ca.pem \ -> -ca-key=requestheader-client-ca-key.pem \ -> -config=../ca-config.json \ -> -hostname=controller1,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.205 \ -> -profile=kubernetes \ -> ../controller1-proxy-csr.json | cfssljson -bare controller1-proxy - -# Controller kubelet client certificate -$(OUT)/controller2-proxy-key.pem: controller2-proxy-csr.json ca-config.json $(OUT)/requestheader-client-ca-key.pem -> @mkdir -p $(@D) -> cd $(@D) && cfssl gencert \ -> -ca=requestheader-client-ca.pem \ -> -ca-key=requestheader-client-ca-key.pem \ -> -config=../ca-config.json \ -> -hostname=controller2,$(KUBERNETES_PUBLIC_ADDRESS),10.215.1.206 \ -> -profile=kubernetes \ -> ../controller2-proxy-csr.json | cfssljson -bare controller2-proxy - -# Controller manager client certificate -$(OUT)/kube-controller-manager-key.pem: kube-controller-manager-csr.json ca-config.json -> @mkdir -p $(@D) -> cd $(@D) && cfssl gencert \ -> -ca=ca.pem \ -> -ca-key=ca-key.pem \ -> -config=../ca-config.json \ -> -profile=kubernetes \ -> ../kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager - -# Kube proxy client certificate -$(OUT)/kube-proxy-key.pem: kube-proxy-csr.json ca-config.json -> @mkdir -p $(@D) -> cd $(@D) && cfssl gencert \ -> -ca=ca.pem \ -> -ca-key=ca-key.pem \ -> -config=../ca-config.json \ -> -profile=kubernetes \ -> ../kube-proxy-csr.json | cfssljson -bare kube-proxy - -# Kube scheduler client certificate -$(OUT)/kube-scheduler-key.pem: kube-scheduler-csr.json ca-config.json -> @mkdir -p $(@D) -> cd $(@D) && cfssl gencert \ -> -ca=ca.pem \ -> -ca-key=ca-key.pem \ -> -config=../ca-config.json \ -> -profile=kubernetes \ -> ../kube-scheduler-csr.json | cfssljson -bare kube-scheduler - -# Kuberntes API server certificate -# TODO: Replace 10.32.0.1 with kubernetes api server local ip address from lab 8 -$(OUT)/kubernetes-key.pem: kubernetes-csr.json ca-config.json -> @mkdir -p $(@D) -> cd $(@D) && cfssl gencert \ -> -ca=ca.pem \ -> -ca-key=ca-key.pem \ -> -config=../ca-config.json \ -> -hostname=10.197.0.1,10.0.0.1,10.215.1.204,10.215.1.205,10.215.1.206,10.215.1.207,10.215.1.208,10.215.1.209,$(KUBERNETES_PUBLIC_ADDRESS),127.0.0.1,kubernetes,kubernetes.default,kubernetes.default.svc,kubernetes.default.svc.cluster,kubernetes.svc.cluster.local \ -> -profile=kubernetes \ -> ../kubernetes-csr.json | cfssljson -bare kubernetes - -# Service account keypair -$(OUT)/service-account-key.pem: service-account-csr.json ca-config.json -> @mkdir -p $(@D) -> cd $(@D) && cfssl gencert \ -> -ca=ca.pem \ -> -ca-key=ca-key.pem \ -> -config=../ca-config.json \ -> -profile=kubernetes \ -> ../service-account-csr.json | cfssljson -bare service-account - -# Generate worker kubeconfigs -$(patsubst %,$(OUT)/%.kubeconfig,$(WORKERS)): $(OUT)/%.kubeconfig: $(OUT)/%-key.pem $(OUT)/%.pem -> @mkdir -p $(@D) -> kubectl config set-cluster kubernetes-the-hard-way \ -> --certificate-authority=$(OUT)/ca.pem \ -> --embed-certs=true \ -> --server=https://$(KUBERNETES_PUBLIC_ADDRESS):6443 \ -> --kubeconfig=$@ -> -> kubectl config set-credentials system:node:$* \ -> --client-certificate=$(OUT)/$*.pem \ -> --client-key=$(OUT)/$*-key.pem \ -> --embed-certs=true \ -> --kubeconfig=$@ -> -> kubectl config set-context default \ -> --cluster=kubernetes-the-hard-way \ -> --user=system:node:$* \ -> --kubeconfig=$@ -> -> kubectl config use-context default --kubeconfig=$@ - -# Generate kube-proxy kubeconfig -$(OUT)/kube-proxy.kubeconfig: $(OUT)/%.kubeconfig: $(OUT)/%-key.pem $(OUT)/%.pem -> @mkdir -p $(@D) -> kubectl config set-cluster kubernetes-the-hard-way \ -> --certificate-authority=$(OUT)/ca.pem \ -> --embed-certs=true \ -> --server=https://$(KUBERNETES_PUBLIC_ADDRESS):6443 \ -> --kubeconfig=$@ -> -> kubectl config set-credentials system:$* \ -> --client-certificate=$(OUT)/$*.pem \ -> --client-key=$(OUT)/$*-key.pem \ -> --embed-certs=true \ -> --kubeconfig=$@ -> -> kubectl config set-context default \ -> --cluster=kubernetes-the-hard-way \ -> --user=system:$* \ -> --kubeconfig=$@ -> -> kubectl config use-context default --kubeconfig=$@ - -# Generate kube-controller-manager kubeconfig -$(OUT)/kube-controller-manager.kubeconfig: $(OUT)/%.kubeconfig: $(OUT)/%-key.pem $(OUT)/%.pem -> @mkdir -p $(@D) -> kubectl config set-cluster kubernetes-the-hard-way \ -> --certificate-authority=$(OUT)/ca.pem \ -> --embed-certs=true \ -> --server=https://127.0.0.1:6443 \ -> --kubeconfig=$@ -> -> kubectl config set-credentials system:$* \ -> --client-certificate=$(OUT)/$*.pem \ -> --client-key=$(OUT)/$*-key.pem \ -> --embed-certs=true \ -> --kubeconfig=$@ -> -> kubectl config set-context default \ -> --cluster=kubernetes-the-hard-way \ -> --user=system:$* \ -> --kubeconfig=$@ -> -> kubectl config use-context default --kubeconfig=$@ - -# Generate kube-scheduler kubeconfig -$(OUT)/kube-scheduler.kubeconfig: $(OUT)/%.kubeconfig: $(OUT)/%-key.pem $(OUT)/%.pem -> @mkdir -p $(@D) -> kubectl config set-cluster kubernetes-the-hard-way \ -> --certificate-authority=$(OUT)/ca.pem \ -> --embed-certs=true \ -> --server=https://127.0.0.1:6443 \ -> --kubeconfig=$@ -> -> kubectl config set-credentials system:$* \ -> --client-certificate=$(OUT)/$*.pem \ -> --client-key=$(OUT)/$*-key.pem \ -> --embed-certs=true \ -> --kubeconfig=$@ -> -> kubectl config set-context default \ -> --cluster=kubernetes-the-hard-way \ -> --user=system:$* \ -> --kubeconfig=$@ -> -> kubectl config use-context default --kubeconfig=$@ - -# Generate admin kubeconfig -$(OUT)/admin.kubeconfig: $(OUT)/%.kubeconfig: $(OUT)/%-key.pem $(OUT)/%.pem -> @mkdir -p $(@D) -> kubectl config set-cluster kubernetes-the-hard-way \ -> --certificate-authority=$(OUT)/ca.pem \ -> --embed-certs=true \ -> --server=https://127.0.0.1:6443 \ -> --kubeconfig=$@ -> -> kubectl config set-credentials $* \ -> --client-certificate=$(OUT)/$*.pem \ -> --client-key=$(OUT)/$*-key.pem \ -> --embed-certs=true \ -> --kubeconfig=$@ -> -> kubectl config set-context default \ -> --cluster=kubernetes-the-hard-way \ -> --user=$* \ -> --kubeconfig=$@ -> -> kubectl config use-context default --kubeconfig=$@ - -# Generate data encryption key for encrypting data at rest -$(OUT)/encryption-config.yaml: -> @mkdir -p $(@D) -> ENCRYPTION_KEY=$(shell head -c 32 /dev/urandom | base64) -> cat encryption-config-template.yaml | sed "s@ENCRYPTION_KEY@$$ENCRYPTION_KEY@g" > $@ - -# Generate remote admin kubeconfig -$(OUT)/remote_admin.kubeconfig: $(OUT)/remote_%.kubeconfig: $(OUT)/%-key.pem $(OUT)/%.pem -> @mkdir -p $(@D) -> kubectl config set-cluster kubernetes-the-hard-way \ -> --certificate-authority=$(OUT)/ca.pem \ -> --embed-certs=true \ -> --server=https://$(KUBERNETES_PUBLIC_ADDRESS):6443 \ -> --kubeconfig=$@ -> -> kubectl config set-credentials $* \ -> --client-certificate=$(OUT)/$*.pem \ -> --client-key=$(OUT)/$*-key.pem \ -> --embed-certs=true \ -> --kubeconfig=$@ -> -> kubectl config set-context default \ -> --cluster=kubernetes-the-hard-way \ -> --user=$* \ -> --kubeconfig=$@ -> -> kubectl config use-context default --kubeconfig=$@ +$(OUT)/known_hosts: | $(OUT)/ +> ssh-keyscan -p 65099 74.80.180.138 | sed 's/\[74.80.180.138\]:65099/\[10.215.1.210\]:22/g' > $@ diff --git a/nix/kubernetes/keys/generated/known_hosts b/nix/kubernetes/keys/generated/known_hosts new file mode 100644 index 0000000..3406283 --- /dev/null +++ b/nix/kubernetes/keys/generated/known_hosts @@ -0,0 +1,8 @@ +# 74.80.180.138:65099 SSH-2.0-OpenSSH_9.3 FreeBSD-20230316 +[10.215.1.210]:22 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC0hWY7Ighnlp3UfPfApyW9nEGG11f+on/kOkp6YdxTTVX0jvi00xvrZ8c23l48YDptmEKOMj7avUR+jdpRNaSwbw3Lm7swg+EpFZ73tnHK+r6HnOnNu8ECDvYOW10eI6vdRctFisRfyIKigmtmquxXYLhQDSA2INVW+Vuebdwa74VqKLLirUu7e3ymp8dH8ktcCAjWSd/+Ax7E+4AMa5WHFeTPBheA2GhfLhINDLpgdZ8WNZ4i3ow8MrQADiOVYUDPrXvI55MVWSQTQQcOco184Z67rtcCtqY/fcCp+38yzUT0Bm2syXM+HNOlFqM+fJBf0T9kiiy5XvWuN9bY+368JGOUUM6RsCUgERHSaU65nX3i8oIcNRt3w6sVsmRR8sX8x5qFjyEYuElIwKywcdtKpoklV6gu+lo+mIE8i95jJmXMj6lk3G83wMZICL9+dm+b8ckpRZEi6970EqahiPO3cV/Fa88gysf9HwiC8AxSc3m2BcOvaV3jadaT39Tymp8= +# 74.80.180.138:65099 SSH-2.0-OpenSSH_9.3 FreeBSD-20230316 +# 74.80.180.138:65099 SSH-2.0-OpenSSH_9.3 FreeBSD-20230316 +[10.215.1.210]:22 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBH2euFJKLEDfTV9NTecrOoqL9FpiYvTbNp/Ty3FebJA5DKmVd1xBRz3sNs1R1ayn213vmRVLWSu2ikulbl65LLQ= +# 74.80.180.138:65099 SSH-2.0-OpenSSH_9.3 FreeBSD-20230316 +[10.215.1.210]:22 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM1qjGgD2UdD5Lc+zGFxHX/+h6FBNmGW+O30LG0tiHvC +# 74.80.180.138:65099 SSH-2.0-OpenSSH_9.3 FreeBSD-20230316 diff --git a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_apply_git.yaml b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_apply_git.yaml new file mode 100644 index 0000000..a0fba73 --- /dev/null +++ b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_apply_git.yaml @@ -0,0 +1,34 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: GitRepository +metadata: + name: kubernetes + namespace: flux-system +spec: + interval: 5m0s + ref: + branch: nix + secretRef: + name: kubernetes-deploy-key + # url: ssh://git@74.80.180.138:65099/repos/mrmanager + url: ssh://git@10.215.1.210:22/repos/mrmanager + ignore: | + bootstrap + .sops.yaml + secrets/ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: backend + namespace: flux-system +spec: + interval: 5m0s + path: "./k8s" + prune: true + sourceRef: + kind: GitRepository + name: kubernetes + decryption: + provider: sops + secretRef: + name: sops-gpg diff --git a/nix/kubernetes/keys/package/k8s-keys/package.nix b/nix/kubernetes/keys/package/k8s-keys/package.nix index 587749a..3c62aec 100644 --- a/nix/kubernetes/keys/package/k8s-keys/package.nix +++ b/nix/kubernetes/keys/package/k8s-keys/package.nix @@ -1,11 +1,20 @@ { k8s, + runCommand, symlinkJoin, ... }: +let + scripts = runCommand "scripts" { } '' + mkdir $out + cp ${k8s.deploy_script} $out/deploy_script + cp ${k8s.bootstrap_script} $out/bootstrap_script + ''; +in symlinkJoin { name = "k8s-keys"; paths = [ + scripts k8s.ca k8s.encryption_config ] diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index 93dff6f..0aa2695 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -134,6 +134,7 @@ makeScope newScope ( secret_values = { "identity" = builtins.readFile "${self.ssh-keys.flux_ssh_key}/flux_ssh_key"; "identity.pub" = builtins.readFile "${self.ssh-keys.flux_ssh_key}/flux_ssh_key.pub"; + "known_hosts" = builtins.readFile ./generated/known_hosts; }; }; } From 2bbf44e528c41d00bc0bb3d0fe91ae7ba7f7774c Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 22 Dec 2025 00:14:17 -0500 Subject: [PATCH 32/81] Fix DNS resolution. --- .../keys/package/bootstrap-script/files/manifests/coredns.yaml | 2 +- nix/kubernetes/roles/kubelet/files/kubelet-config.yaml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/coredns.yaml b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/coredns.yaml index e1046cd..74698a6 100644 --- a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/coredns.yaml +++ b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/coredns.yaml @@ -201,7 +201,7 @@ metadata: spec: selector: k8s-app: kube-dns - # clusterIP: $DNS_SERVER_IP + clusterIP: 10.197.0.10 ports: - name: dns port: 53 diff --git a/nix/kubernetes/roles/kubelet/files/kubelet-config.yaml b/nix/kubernetes/roles/kubelet/files/kubelet-config.yaml index 7089087..05f5320 100644 --- a/nix/kubernetes/roles/kubelet/files/kubelet-config.yaml +++ b/nix/kubernetes/roles/kubelet/files/kubelet-config.yaml @@ -23,3 +23,6 @@ registerNode: true runtimeRequestTimeout: "15m" tlsCertFile: "/.persist/keys/kube/kubelet.crt" tlsPrivateKeyFile: "/.persist/keys/kube/kubelet.key" +clusterDomain: "cluster.local" +clusterDNS: + - "10.197.0.10" From d924aa3a81f3c609e3dd7211c0c384f7abb9e593 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Tue, 23 Dec 2025 06:48:06 -0500 Subject: [PATCH 33/81] Add missing cidr declarations. --- nix/kubernetes/hosts/controller0/default.nix | 3 ++- nix/kubernetes/hosts/controller1/default.nix | 3 ++- nix/kubernetes/hosts/controller2/default.nix | 3 ++- .../package/bootstrap-script/files/manifests/coredns.yaml | 3 ++- nix/kubernetes/roles/kube_apiserver/default.nix | 2 ++ nix/kubernetes/roles/kube_proxy/default.nix | 1 + nix/kubernetes/roles/kubelet/files/kubelet-config.yaml | 6 +++--- 7 files changed, 14 insertions(+), 7 deletions(-) diff --git a/nix/kubernetes/hosts/controller0/default.nix b/nix/kubernetes/hosts/controller0/default.nix index 5361612..4521b7e 100644 --- a/nix/kubernetes/hosts/controller0/default.nix +++ b/nix/kubernetes/hosts/controller0/default.nix @@ -117,7 +117,8 @@ ]; me.kube_apiserver.internal_ip = "2620:11f:7001:7:ffff:ffff:0ad7:01dd"; - me.kube_apiserver.external_ip = "74.80.180.138"; + # me.kube_apiserver.external_ip = "74.80.180.138"; + me.kube_apiserver.external_ip = "2620:11f:7001:7:ffff:ffff:0ad7:01dd"; me.kube_apiserver.etcd_services = [ "https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2379" # 10.215.1.221 "https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2379" # 10.215.1.222 diff --git a/nix/kubernetes/hosts/controller1/default.nix b/nix/kubernetes/hosts/controller1/default.nix index d6571c0..3af358d 100644 --- a/nix/kubernetes/hosts/controller1/default.nix +++ b/nix/kubernetes/hosts/controller1/default.nix @@ -117,7 +117,8 @@ ]; me.kube_apiserver.internal_ip = "2620:11f:7001:7:ffff:ffff:0ad7:01de"; - me.kube_apiserver.external_ip = "74.80.180.138"; + # me.kube_apiserver.external_ip = "74.80.180.138"; + me.kube_apiserver.external_ip = "2620:11f:7001:7:ffff:ffff:0ad7:01de"; me.kube_apiserver.etcd_services = [ "https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2379" # 10.215.1.221 "https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2379" # 10.215.1.222 diff --git a/nix/kubernetes/hosts/controller2/default.nix b/nix/kubernetes/hosts/controller2/default.nix index c6c95c0..a25ddae 100644 --- a/nix/kubernetes/hosts/controller2/default.nix +++ b/nix/kubernetes/hosts/controller2/default.nix @@ -117,7 +117,8 @@ ]; me.kube_apiserver.internal_ip = "2620:11f:7001:7:ffff:ffff:0ad7:01df"; - me.kube_apiserver.external_ip = "74.80.180.138"; + # me.kube_apiserver.external_ip = "74.80.180.138"; + me.kube_apiserver.external_ip = "2620:11f:7001:7:ffff:ffff:0ad7:01df"; me.kube_apiserver.etcd_services = [ "https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2379" # 10.215.1.221 "https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2379" # 10.215.1.222 diff --git a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/coredns.yaml b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/coredns.yaml index 74698a6..1f12a63 100644 --- a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/coredns.yaml +++ b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/coredns.yaml @@ -201,7 +201,8 @@ metadata: spec: selector: k8s-app: kube-dns - clusterIP: 10.197.0.10 + # clusterIP: 10.197.0.10 + clusterIP: "fd00:3e42:e349::10" ports: - name: dns port: 53 diff --git a/nix/kubernetes/roles/kube_apiserver/default.nix b/nix/kubernetes/roles/kube_apiserver/default.nix index 66b31aa..4bd798e 100644 --- a/nix/kubernetes/roles/kube_apiserver/default.nix +++ b/nix/kubernetes/roles/kube_apiserver/default.nix @@ -57,6 +57,7 @@ in shellCommand [ # NEW: "${pkgs.kubernetes}/bin/kube-apiserver" + "--advertise-address=${config.me.kube_apiserver.external_ip}" "--allow-privileged=true" "--audit-log-maxage=30" "--audit-log-maxbackup=3" @@ -83,6 +84,7 @@ in "--tls-cert-file=/.persist/keys/kube/kube-api-server.crt" "--tls-private-key-file=/.persist/keys/kube/kube-api-server.key" "--tls-min-version=VersionTLS13" + "--service-cluster-ip-range=fd00:3e42:e349::/48" "--v=2" # OLD: diff --git a/nix/kubernetes/roles/kube_proxy/default.nix b/nix/kubernetes/roles/kube_proxy/default.nix index 213aa3a..bd23e46 100644 --- a/nix/kubernetes/roles/kube_proxy/default.nix +++ b/nix/kubernetes/roles/kube_proxy/default.nix @@ -57,6 +57,7 @@ in "${pkgs.kubernetes}/bin/kube-proxy" "--config=${config_file}" "--nodeport-addresses=primary" + "--cluster-cidr=fd49:0595:2bba::/48" ] ); Restart = "on-failure"; diff --git a/nix/kubernetes/roles/kubelet/files/kubelet-config.yaml b/nix/kubernetes/roles/kubelet/files/kubelet-config.yaml index 05f5320..c790198 100644 --- a/nix/kubernetes/roles/kubelet/files/kubelet-config.yaml +++ b/nix/kubernetes/roles/kubelet/files/kubelet-config.yaml @@ -23,6 +23,6 @@ registerNode: true runtimeRequestTimeout: "15m" tlsCertFile: "/.persist/keys/kube/kubelet.crt" tlsPrivateKeyFile: "/.persist/keys/kube/kubelet.key" -clusterDomain: "cluster.local" -clusterDNS: - - "10.197.0.10" +# clusterDomain: "cluster.local" +# clusterDNS: +# - "10.197.0.10" From 8e9bd9acc19cabbe341bdadfdde19d1155a87e60 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sat, 27 Dec 2025 21:03:25 -0500 Subject: [PATCH 34/81] Introduce functions to generate yaml. The toYAML function is just an alias to toJSON which is technically fine since YAML is a superset of JSON, but these new functions will generate actual YAML. --- .../package/k8s-encryption-key/package.nix | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/nix/kubernetes/keys/package/k8s-encryption-key/package.nix b/nix/kubernetes/keys/package/k8s-encryption-key/package.nix index 1f88575..fcbbca1 100644 --- a/nix/kubernetes/keys/package/k8s-encryption-key/package.nix +++ b/nix/kubernetes/keys/package/k8s-encryption-key/package.nix @@ -9,12 +9,28 @@ # distPhase { lib, + pkgs, stdenv, runCommand, writeText, ... }: let + to_yaml_file = + file_name: contents: + let + settingsFormat = pkgs.formats.yaml { }; + yaml_file = settingsFormat.generate file_name contents; + in + yaml_file; + to_yaml = + file_name: contents: + let + settingsFormat = pkgs.formats.yaml { }; + yaml_file = settingsFormat.generate file_name contents; + yaml_content = builtins.readFile yaml_file; + in + yaml_content; kube_encryption_key = runCommand "kube_encryption_key" { } '' head -c 32 /dev/urandom | base64 | tee $out ''; @@ -40,9 +56,7 @@ let } ]; }; - kube_encryption_config_yaml = ( - writeText "encryption-config.yaml" (lib.generators.toYAML { } kube_encryption_config) - ); + kube_encryption_config_yaml = (to_yaml_file "encryption-config.yaml" kube_encryption_config); in stdenv.mkDerivation (finalAttrs: { name = "k8s-encryption-key"; From 2314bef2ad3bb5adcfeb909d7d68a0014104828a Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sat, 27 Dec 2025 21:15:00 -0500 Subject: [PATCH 35/81] Move the yaml functions to their own file. --- nix/kubernetes/functions/to_yaml.nix | 24 +++++++++++++++++++ .../package/k8s-encryption-key/package.nix | 18 +------------- 2 files changed, 25 insertions(+), 17 deletions(-) create mode 100644 nix/kubernetes/functions/to_yaml.nix diff --git a/nix/kubernetes/functions/to_yaml.nix b/nix/kubernetes/functions/to_yaml.nix new file mode 100644 index 0000000..c32d4ca --- /dev/null +++ b/nix/kubernetes/functions/to_yaml.nix @@ -0,0 +1,24 @@ +{ + pkgs, + ... +}: +let + to_yaml_file = + file_name: contents: + let + settingsFormat = pkgs.formats.yaml { }; + yaml_file = settingsFormat.generate file_name contents; + in + yaml_file; + to_yaml = + file_name: contents: + let + settingsFormat = pkgs.formats.yaml { }; + yaml_file = settingsFormat.generate file_name contents; + yaml_content = builtins.readFile yaml_file; + in + yaml_content; +in +{ + inherit to_yaml to_yaml_file; +} diff --git a/nix/kubernetes/keys/package/k8s-encryption-key/package.nix b/nix/kubernetes/keys/package/k8s-encryption-key/package.nix index fcbbca1..6ba483b 100644 --- a/nix/kubernetes/keys/package/k8s-encryption-key/package.nix +++ b/nix/kubernetes/keys/package/k8s-encryption-key/package.nix @@ -8,29 +8,13 @@ # installCheckPhase # distPhase { - lib, pkgs, stdenv, runCommand, - writeText, ... }: let - to_yaml_file = - file_name: contents: - let - settingsFormat = pkgs.formats.yaml { }; - yaml_file = settingsFormat.generate file_name contents; - in - yaml_file; - to_yaml = - file_name: contents: - let - settingsFormat = pkgs.formats.yaml { }; - yaml_file = settingsFormat.generate file_name contents; - yaml_content = builtins.readFile yaml_file; - in - yaml_content; + to_yaml_file = ((import ../../../functions/to_yaml.nix) {inherit pkgs;}).to_yaml_file; kube_encryption_key = runCommand "kube_encryption_key" { } '' head -c 32 /dev/urandom | base64 | tee $out ''; From efbfe865e0135220c74420d42ef2ab3e5e5ad86b Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sat, 27 Dec 2025 21:22:06 -0500 Subject: [PATCH 36/81] Fix trailing line break in kubernetes encryption config. --- nix/kubernetes/keys/package/k8s-encryption-key/package.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/kubernetes/keys/package/k8s-encryption-key/package.nix b/nix/kubernetes/keys/package/k8s-encryption-key/package.nix index 6ba483b..9855e32 100644 --- a/nix/kubernetes/keys/package/k8s-encryption-key/package.nix +++ b/nix/kubernetes/keys/package/k8s-encryption-key/package.nix @@ -16,7 +16,7 @@ let to_yaml_file = ((import ../../../functions/to_yaml.nix) {inherit pkgs;}).to_yaml_file; kube_encryption_key = runCommand "kube_encryption_key" { } '' - head -c 32 /dev/urandom | base64 | tee $out + head -c 32 /dev/urandom | base64 -w 0 | tee $out ''; kube_encryption_config = { kind = "EncryptionConfig"; From 7df824da6a16ae68079c21283bd164b90c08e1a1 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 29 Dec 2025 04:58:49 -0500 Subject: [PATCH 37/81] Fix service cluster ip range. Kubernetes only allows a /112 for service ip range. --- nix/kubernetes/keys/package/k8s-ca/files/ca.conf | 1 + nix/kubernetes/roles/kube_apiserver/default.nix | 2 +- nix/kubernetes/roles/kube_controller_manager/default.nix | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/nix/kubernetes/keys/package/k8s-ca/files/ca.conf b/nix/kubernetes/keys/package/k8s-ca/files/ca.conf index b26ddbf..8893f9d 100644 --- a/nix/kubernetes/keys/package/k8s-ca/files/ca.conf +++ b/nix/kubernetes/keys/package/k8s-ca/files/ca.conf @@ -279,6 +279,7 @@ IP.10 = 10.215.1.225 IP.11 = 2620:11f:7001:7:ffff:ffff:0ad7:01e1 IP.12 = 10.215.1.226 IP.13 = 2620:11f:7001:7:ffff:ffff:0ad7:01e2 +IP.14 = fd00:3e42:e349::1 DNS.0 = kubernetes DNS.1 = kubernetes.default DNS.2 = kubernetes.default.svc diff --git a/nix/kubernetes/roles/kube_apiserver/default.nix b/nix/kubernetes/roles/kube_apiserver/default.nix index 4bd798e..8cf4fb9 100644 --- a/nix/kubernetes/roles/kube_apiserver/default.nix +++ b/nix/kubernetes/roles/kube_apiserver/default.nix @@ -84,7 +84,7 @@ in "--tls-cert-file=/.persist/keys/kube/kube-api-server.crt" "--tls-private-key-file=/.persist/keys/kube/kube-api-server.key" "--tls-min-version=VersionTLS13" - "--service-cluster-ip-range=fd00:3e42:e349::/48" + "--service-cluster-ip-range=fd00:3e42:e349::/112" "--v=2" # OLD: diff --git a/nix/kubernetes/roles/kube_controller_manager/default.nix b/nix/kubernetes/roles/kube_controller_manager/default.nix index 40ac486..282d0a5 100644 --- a/nix/kubernetes/roles/kube_controller_manager/default.nix +++ b/nix/kubernetes/roles/kube_controller_manager/default.nix @@ -39,6 +39,7 @@ in "--bind-address=0.0.0.0" # "--cluster-cidr=10.200.0.0/16" # "--cluster-cidr=2620:11f:7001:7:ffff:ffff:0ac8:0000/16" + "--allocate-node-cidrs=true" "--cluster-cidr=fd49:0595:2bba::/48" "--cluster-name=kubernetes" "--cluster-signing-cert-file=/.persist/keys/kube/ca.crt" @@ -48,7 +49,7 @@ in "--service-account-private-key-file=/.persist/keys/kube/service-accounts.key" # "--service-cluster-ip-range=10.197.0.0/16" # "--service-cluster-ip-range=2620:11f:7001:7:ffff:ffff:0ac5:0000/16" - "--service-cluster-ip-range=fd00:3e42:e349::/48" + "--service-cluster-ip-range=fd00:3e42:e349::/112" "--use-service-account-credentials=true" "--v=2" ] From 383acd4fd2723c01cf985b4dad954312940507a4 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 29 Dec 2025 15:59:00 -0500 Subject: [PATCH 38/81] Switch to kubernetes ipam mode. --- nix/kubernetes/README.org | 16 ++++- nix/kubernetes/keys/flake.nix | 22 +++++++ .../files/manifests/cilium.yaml | 63 +++++++++---------- 3 files changed, 67 insertions(+), 34 deletions(-) diff --git a/nix/kubernetes/README.org b/nix/kubernetes/README.org index 40e71b7..f5d38e2 100644 --- a/nix/kubernetes/README.org +++ b/nix/kubernetes/README.org @@ -17,17 +17,29 @@ nix shell 'nixpkgs#kubernetes-helm' helm repo add cilium https://helm.cilium.io/ - helm template --dry-run=server cilium cilium/cilium --version 1.18.4 --namespace kube-system \ + helm template --dry-run=client cilium cilium/cilium --version 1.18.5 --namespace kube-system \ --set kubeProxyReplacement=true \ + --set ipam.mode=kubernetes \ --set k8sServiceHost="2620:11f:7001:7:ffff:ffff:ad7:1dd" \ --set k8sServicePort=6443 \ - --set ipv6.enabled=true + --set ipv6.enabled=true \ + --set ipv4.enabled=false \ + --set enableIPv4BIGTCP=false \ + --set enableIPv6BIGTCP=true \ + --set routingMode=native \ + --set ipv4NativeRoutingCIDR=10.0.0.0/8 \ + --set ipv6NativeRoutingCIDR=fd00::/100 kubectl -n kube-system exec ds/cilium -- cilium-dbg status --verbose kubectl -n kube-system exec ds/cilium -- cilium-dbg status | grep KubeProxyReplacement # --set hostFirewall.enabled=true # routingMode=native + + # --set ipv4-native-routing-cidr=10.0.0.0/8 \ + # --set ipv6-native-routing-cidr=fd00::/100 + # --set 'ipam.operator.clusterPoolIPv4PodCIDRList=["10.0.0.0/8"]' \ + # --set 'ipam.operator.clusterPoolIPv6PodCIDRList=["fd00::/100"]' \ #+end_src ** Install flux #+begin_src bash diff --git a/nix/kubernetes/keys/flake.nix b/nix/kubernetes/keys/flake.nix index e763209..3bdd04b 100644 --- a/nix/kubernetes/keys/flake.nix +++ b/nix/kubernetes/keys/flake.nix @@ -28,5 +28,27 @@ k8s = (final.callPackage ./scope.nix { inherit (final.lib) makeScope; }); } ); + + } + // { + devShells = forAllSystems ( + system: + let + pkgs = nixpkgs.legacyPackages.${system}; + appliedOverlay = self.overlays.default pkgs pkgs; + in + { + default = pkgs.mkShell { + nativeBuildInputs = with pkgs; [ + pkg-config + ]; + buildInputs = with pkgs; [ + kubernetes-helm # To generate cilium manifests + fluxcd # To generate flux manifests + cilium-cli # To check cilium status + ]; + }; + } + ); }; } diff --git a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/cilium.yaml b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/cilium.yaml index 799db4d..2113b65 100644 --- a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/cilium.yaml +++ b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/cilium.yaml @@ -36,8 +36,8 @@ metadata: name: cilium-ca namespace: kube-system data: - ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lSQVByWlQ4U29LSnRBS3dLMTdYSXhhK0l3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkxTVRJeE9UQTBNamt3T1ZvWERUSTRNVEl4T0RBMApNamt3T1Zvd0ZERVNNQkFHQTFVRUF4TUpRMmxzYVhWdElFTkJNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFNSUlCQ2dLQ0FRRUFyc2xsWWZQMVN1U0Rtd2crQTg1WGJPWDdJSE5MZGpEKzErUXZrZ2I1dVNickRZbjUKUnNtWXZ1NERVdnRWWXBYc1UvUVU5L1Bmb00xVkNQZFJkZGU5eHlBT01mOGF1bW4vZ3g5S1BYbU1aZk10MzBWTApweWJrbE5lVkw1aGhsWkZrNit6VW1yVU1lWXZ0cjFIRHNlaXlNL2lUT3VQUjlYS3A1UUp1cDRxdHJRaW8zT21lCkw2U3lSZGtoU3BEL3FwVjJYWnI5WEJBMEhSZ082dlY5SHY1V0xBclBUWU9xWFFGaFV2MTZQcE1TY3FkL1hNSDYKcUtrQ0ZJVDJGd1prb3dnS0V5bktCTmFVdjFkMWdsMVduM0tTNG5kdjRDdFY3VVRGbWhUOU51RCtsbWU1VmoyVApzYVlMdmp5WGI0MUdxWVhqRkppWTZvY3FFT2hrdnRhbGNTRXUzUUlEQVFBQm8yRXdYekFPQmdOVkhROEJBZjhFCkJBTUNBcVF3SFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGTGducmxPZGwzZmRWT2dsNkN0NFBha0pjODlWTUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQi9taVdES3RYQWtOU0lXblIwcWRNbkVlR2JaOXd4NHprY3UzajBqSlk4VW1MSnJqVXFTRXQxClh0dEJ2QVZHV3RKKzU0R3I1bWJXTy9oSXE3UkZmY04vYUZoQWZDdGdPZ0Nacm1yejJobkVKK294cWpRM1c2NlkKQXd1TkNNZkJjUmNyeE9Yc3hkNzc2R24rZVVwYUpWTlh6S1JJSXdHWWJYOVE5Tkd3dlhKQjRoVWJXVXZIQTJkTwp4TjFNMmFzOWVxYmVUejlPTjNZdWd6ZnJBckVxbHhuNDIyR1NVOWQ4bktobi9DUnhvK1U0dGVJRXRTUXBaN0hIClhxYjhLZjdOTXJweTVjb3gxOFU5YVovcFBMT0tzckxlQUxYc3c1ejdtT3l2eWhVVE9vbUVlYnR5TU5VRGhCVHkKeUhaMUp2RVNoaVVLRmkvZW5SdGxyVUwxZG1qd1FHZzAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= - ca.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb1FJQkFBS0NBUUVBcnNsbFlmUDFTdVNEbXdnK0E4NVhiT1g3SUhOTGRqRCsxK1F2a2diNXVTYnJEWW41ClJzbVl2dTREVXZ0VllwWHNVL1FVOS9QZm9NMVZDUGRSZGRlOXh5QU9NZjhhdW1uL2d4OUtQWG1NWmZNdDMwVkwKcHlia2xOZVZMNWhobFpGazYrelVtclVNZVl2dHIxSERzZWl5TS9pVE91UFI5WEtwNVFKdXA0cXRyUWlvM09tZQpMNlN5UmRraFNwRC9xcFYyWFpyOVhCQTBIUmdPNnZWOUh2NVdMQXJQVFlPcVhRRmhVdjE2UHBNU2NxZC9YTUg2CnFLa0NGSVQyRndaa293Z0tFeW5LQk5hVXYxZDFnbDFXbjNLUzRuZHY0Q3RWN1VURm1oVDlOdUQrbG1lNVZqMlQKc2FZTHZqeVhiNDFHcVlYakZKaVk2b2NxRU9oa3Z0YWxjU0V1M1FJREFRQUJBb0lCQUROWG4zdEZnVEZOck1iMgpoZGp5WE1YMVRHNDh6OWJaRDVMMVNXLzRzcFJMdFZJVnpHNlp1MCtpZE1HRWFKRjcvcDZ1blJZMHdlVG5TK3BGClU5MXMwU2RUNkRlUmV4VjFHNFExeGNZSEZWSEJQQ0ZRcHFka0NsTWFmSTB0a3NTdm45aVppZDRqL3BVWm9Ha1cKZUlpakZPMmxRQUpWZDVCRTdWa0MvT1FrSFY0dEo2cExDck1WNkE0OXVvaWNDMFZOUnRxZHBXc0RxQ3RydDY3dApQRmRNemVZY290TjR6RFg5SHlaeFdnako4QXhPNVMwM3ZRWXVhZCs3YW1QWXAvWGkzaDBKM0xvc25ETmh6QWtxCktRYlVtekZMM3I4TnVUclBQV05rQXB4UEU2aEl5dkd2eWc0UkxhdGUvZDBNbld4NEZKaDJGdG5HYXYzSXkyMHcKS1I1M05wVUNnWUVBMjFvZ1hCZmtCOFBKdC94RkMxYU1oamphWmNZc0gzM2FBQWVvU1k1TlJUeXNXWElpQ1puUgpvTDhJY2NuK1puZlMvMnM4cjYwcTJlcWxMQURRS0JEbkxBU29SYWJ6QlBtTjJoeVZhM2llQTZzT2lNMVBzU0l4CmlLLy81R05vU2ROSVVLd1RMMEhvTGpmZW1uSFZRU0dFanVJbndZblFXVXJDZTBnZW9mRGNyWXNDZ1lFQXkvMHIKOFZ2ZXlMN3FMZzhIeHZBRFoyMmEvNHBqSFM5WFZoY1Z5UGw2WWRDTTVkbnpQdE5RQmxHS1MvNStxbThZYUFGcAp0YWo4Y29Zci9FRzFnN0hWU0JIVXJER2RoNEJZYUtjalZ2K294TThJYVdVcW1vRytZWFpVeTEvTDJJQ2diRkMyCm92YzBsSVpZVlY1KzBoRWRRUHpOQVhSYUNkVkZVYTQwSHJOUWNqY0NnWUVBdGJMSFlXSE0ydjI4aDNJcGcwWFQKSzQvR29VMXgyVGczcjVTb094M3NOZkVYVnhScXk0Y21GVUNJN3pFQ1VjRHAvL2ZGa2tRZ1hkaTNyMmFSQnM5UgpucTNROGJsa1R2ZWxJRXBUSDI0UGZqeGcwc00zUXlSdTRhbmdqMEFrOEJwYktiQVFTRElROE5BSXplaURYT3MxCjBlSFdTc1FzMlBwTGRURjlabW1OY3hjQ2YzNnh2NWtvZGpZNFhUbytHNVp6ZTZuQmFZTk5jRUd3Slk1U3lEVUsKdmZnQ2o0QXkzZUVuWFl5UEdGWWF1cEVnYndncDJIMzdkcytxaGFjbktKV2kzZFRsUGpHMG55S0psRjlyYjRlWgpRamo2WFQ4WWxsMWdOOTVRb0Ztd0pldFIrRkZOMENGbmFFRFhNQVhWcXU3TnNsMnFwYWxCbXFxQXk0enZwVzlyCjc0c0NnWUFJcHgzdWw4aDF1bmhtQStKQnl6d2xHYnE1VzVuZFErUFFnZGZoUmpZeW05Qy9xaXFuVGJyeUN5QzUKZkNRcWpHRUwxRXloUFAwaDljMEdZc3U5VHdtUmh2N2x2eUlRZlkrMnl2U2FtZDc4UWpHTjZUSklnRnlwTWtuRwpmMU1ZT01Td1M1ckhuZzdxcGtuTW1CTjI5SHAyWnFIRWhoNWYyRTVlbUZub2puT2drQT09Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== + ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lSQVBxa2tMQ3FLbm5tTThxWXczTW1Xakl3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkxTVRJeU9USXdOVEF4TlZvWERUSTRNVEl5T0RJdwpOVEF4TlZvd0ZERVNNQkFHQTFVRUF4TUpRMmxzYVhWdElFTkJNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFNSUlCQ2dLQ0FRRUFxc0cvUjI4bHF4Z0JCRDYrdllrL3JPMk1xRTYvZGMxaDROZlEvK2RPM01NeGVpM0gKbXk1eVVXRXg1QWVCaCtzUGhTVGd1L1lrUkJWVDFDS3VtbVNiZE5VdkRRN3Rwc2pPNWQ4YjdObUhqb0Yya25OcQorVVc1MGF0a0M1dVpjR1dEekdaU0kyNElTdVl6Qnd3VzVoSklvNDUzQ21Zd3pJSlVaZWEzam1EQWQ2QUswMnNxCjViRzNuVkpNSHcyNlJQNUJuZWJaQS82dWZlNFlCcDIxR1dvT2gzcERhR0NSS2FRRkpyT0VGSU41aFJYRHJJN1MKUmh3TVNYSVFLallDL0NHNTkrV3FEemhTSTdtMHAyWGRPODU4SEpxL3hSemVrUmZtRkhlcEErNFVTWWZoRUZCUgpRcTV5Z0w4TDZNZEZ5QnU3TXNlMzVxZ3kvMFd6a21uamthUkk2d0lEQVFBQm8yRXdYekFPQmdOVkhROEJBZjhFCkJBTUNBcVF3SFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGRkx0ZUlCVW9PTFZ6K1dRZWxzWkNFVTIyYVFQTUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQ1ZVeW8rTDAyQUJybUIvRUU0V3lmVkFBOWxjeEFYQ0w2QVFzUGpBcGl5QkwrWWFvNkQrN09uCkJCUTRsRklrMFAxbXViOU1OS2xnMTRPcmVCMDhVYkJ3Mm1uYzh5Uk1keERuUjdpVzAwSXJramtkM0Mzc2FLWG4Kcmp2V3dIdzZDcThVUzhZNFMxZjIyUlNNT1h5dE8yd29SenF1UDBYWmtzVUFoeXNTMnVGS1RtQlN1SXNzWWtwRQoxeGhQY1dScUwxRUt3MHRYUHQ4QnhpOGlyckpTeHFHU0pwdTFRVWNra05ISHV2SlNGRXZPZXZPTGJydVRiZmNBCnRDd3p3am1aQVo3clFPMGR5NFNDZ0ppdlhsMktDODRGQ0JvcVkyaVl5d2pubFRzSHMwRnNGSmFsWGtjczhkdzEKUkF6VmZQek54OCtkUFBobGNqdkh3Y1lqT25tdFJqaFAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + ca.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBcXNHL1IyOGxxeGdCQkQ2K3ZZay9yTzJNcUU2L2RjMWg0TmZRLytkTzNNTXhlaTNICm15NXlVV0V4NUFlQmgrc1BoU1RndS9Za1JCVlQxQ0t1bW1TYmROVXZEUTd0cHNqTzVkOGI3Tm1Iam9GMmtuTnEKK1VXNTBhdGtDNXVaY0dXRHpHWlNJMjRJU3VZekJ3d1c1aEpJbzQ1M0NtWXd6SUpVWmVhM2ptREFkNkFLMDJzcQo1YkczblZKTUh3MjZSUDVCbmViWkEvNnVmZTRZQnAyMUdXb09oM3BEYUdDUkthUUZKck9FRklONWhSWERySTdTClJod01TWElRS2pZQy9DRzU5K1dxRHpoU0k3bTBwMlhkTzg1OEhKcS94Unpla1JmbUZIZXBBKzRVU1lmaEVGQlIKUXE1eWdMOEw2TWRGeUJ1N01zZTM1cWd5LzBXemttbmprYVJJNndJREFRQUJBb0lCQUFRa05kVHdNNnd1SlZ3cQpmMnJPYU41elZuMkFoM0UrZlhpOEUwUVpvcGFGSDI2VXJLb3RXclZ4azRteHl1ajJnUVpjS2gwMDc1VTlKUzdZCmRuNmxJUVlZNDQ3QUJYQ05uME1CQWNiaFVjSFpqdnduVTNsZlFRMytSNnRtWnpiTUo0R3VHNkRqV3MwaVB5dVYKRmZzWGgyNThGcFFJZlhlQVU3YXkvVkcxTkFYU3NEb25qZE01c1d1dkhYTDJTeFVQQjJBa1ZHek1ENEg2ZzZQSQpBOC94ekdTOHRQYnF0eGhaYW1Nbk9pN1BId0hLZDNWajJBTHh3ZEcyZ0tWVFhBNzNkQ2pWekN4dXJiZVJVWEIvCjhpam5jeEVBMFovY3Vzb3g1RENNUDUrd1prM2xlSjFaMWg3WW9MVFVmUldWL25KU244RDFhUVlRUmZXSGQ4NE4KWnVndDFia0NnWUVBeVBDb3psdkZmVmNkS0orYmtqa3o3bGN3bmhZRWpkTEswdkRUaFp5WUkzMTdiS3hHK28wSQplSDVWd21zRjFXUm9KYmZxYzFIN25nczFVUVp1ZnJvSko4TzJVZkp6YW9YdjIxMUswRVVNOCsvSFR6Q3ZwZDlhCmUvVElEVGlvUE1YRzEwOENDQ0l2aUM5NzBpWHMyajRuY3dKb2ZVZVBXdGZ3ek1vdStwN0dtV2tDZ1lFQTJZdlMKMzRONlFaOUVMUkY4Uzc1M0VjZHhtL0lxdDd2TnVzVW41OUxqR01ZRFpYM25MUEt3MmZRMG9nSXZKODVOUThsdQpIRTluRjVsc21uSkxNd2h2aFlPc1owdVVlNnNtaVRGbTRmeGdHT1J6QnRFV2pHQWlxRTB4eTZOVHNaNWRWNU1DCmcxSFIyZVY3N25SOGhNWVkvWnMwOUZNSHhNY3hRUTBoRVI2TjBUTUNnWUJ3b0JzdGx0aUptVk01bXNRNjRCOFEKWHdSY0J5RkVVTHJvSXAyY1RSb2xBa2Mwc1JjVGhnc1NabFV4SklWZmh1aXRBM25BYTFvcHlrUjNCckRXRFBYbQp1TmF1OWNMbEdPOUl0L0NmN2FRRjhnWjNoMlZjQm1XVUJxNCtZdHhCTytWb2R0cy9yZEdRZG5UalkraHVwdG03CnA4VGpPMEhKZkRGMngzZ3ZkdUQveVFLQmdIVThhdm5WS3U5dVd4bEdrUXRvcTFaRGV6QWxRMndjbzNRQmg4M2cKMGsycGVHOGJlSmU2ZWlBYkFMcEdIdzkrbUJqdDFvOGZJMW1PYnY2VWxYMzlEOHlOOXp6N2VCdGxVckVnbEdjNAowTGpaZER1Nk5rS1VGRmxLc0lrampaanMxVFM2bnVJU0Y5c3JWcXhjempYblk5Ulg3aU8xRXp5eWRCS3dkcDk1CkxjZ05Bb0dCQUxaS1IxRjl4SElnelpoUEVLM04vRis1VzZMLytVR29jRkt5K2VTU0tLbnA4eWlUVHJiRUdvYUMKU1pHdlo2eFBod1g5RGJ3ZnpyZHpXZWpFSUpYU3BwdmloekhUcXE1WVd4VnVkbnlFVzVxRmhENkZOencrV1hRegp4R28wM2R3N21JeXF6ZDM2K1RDcWFvNmJUeWc1bUUvZUJLREhCVGZ3UFNsanZrQmVPUU1YCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== --- # Source: cilium/templates/hubble/tls-helm/server-secret.yaml apiVersion: v1 @@ -47,9 +47,9 @@ metadata: namespace: kube-system type: kubernetes.io/tls data: - ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lSQVByWlQ4U29LSnRBS3dLMTdYSXhhK0l3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkxTVRJeE9UQTBNamt3T1ZvWERUSTRNVEl4T0RBMApNamt3T1Zvd0ZERVNNQkFHQTFVRUF4TUpRMmxzYVhWdElFTkJNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFNSUlCQ2dLQ0FRRUFyc2xsWWZQMVN1U0Rtd2crQTg1WGJPWDdJSE5MZGpEKzErUXZrZ2I1dVNickRZbjUKUnNtWXZ1NERVdnRWWXBYc1UvUVU5L1Bmb00xVkNQZFJkZGU5eHlBT01mOGF1bW4vZ3g5S1BYbU1aZk10MzBWTApweWJrbE5lVkw1aGhsWkZrNit6VW1yVU1lWXZ0cjFIRHNlaXlNL2lUT3VQUjlYS3A1UUp1cDRxdHJRaW8zT21lCkw2U3lSZGtoU3BEL3FwVjJYWnI5WEJBMEhSZ082dlY5SHY1V0xBclBUWU9xWFFGaFV2MTZQcE1TY3FkL1hNSDYKcUtrQ0ZJVDJGd1prb3dnS0V5bktCTmFVdjFkMWdsMVduM0tTNG5kdjRDdFY3VVRGbWhUOU51RCtsbWU1VmoyVApzYVlMdmp5WGI0MUdxWVhqRkppWTZvY3FFT2hrdnRhbGNTRXUzUUlEQVFBQm8yRXdYekFPQmdOVkhROEJBZjhFCkJBTUNBcVF3SFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGTGducmxPZGwzZmRWT2dsNkN0NFBha0pjODlWTUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQi9taVdES3RYQWtOU0lXblIwcWRNbkVlR2JaOXd4NHprY3UzajBqSlk4VW1MSnJqVXFTRXQxClh0dEJ2QVZHV3RKKzU0R3I1bWJXTy9oSXE3UkZmY04vYUZoQWZDdGdPZ0Nacm1yejJobkVKK294cWpRM1c2NlkKQXd1TkNNZkJjUmNyeE9Yc3hkNzc2R24rZVVwYUpWTlh6S1JJSXdHWWJYOVE5Tkd3dlhKQjRoVWJXVXZIQTJkTwp4TjFNMmFzOWVxYmVUejlPTjNZdWd6ZnJBckVxbHhuNDIyR1NVOWQ4bktobi9DUnhvK1U0dGVJRXRTUXBaN0hIClhxYjhLZjdOTXJweTVjb3gxOFU5YVovcFBMT0tzckxlQUxYc3c1ejdtT3l2eWhVVE9vbUVlYnR5TU5VRGhCVHkKeUhaMUp2RVNoaVVLRmkvZW5SdGxyVUwxZG1qd1FHZzAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= - tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURWakNDQWo2Z0F3SUJBZ0lRUE41WFQxYWZ6VGtwZUtnMXFkZXZEREFOQmdrcWhraUc5dzBCQVFzRkFEQVUKTVJJd0VBWURWUVFERXdsRGFXeHBkVzBnUTBFd0hoY05NalV4TWpFNU1EUXlPVEE1V2hjTk1qWXhNakU1TURReQpPVEE1V2pBcU1TZ3dKZ1lEVlFRRERCOHFMbVJsWm1GMWJIUXVhSFZpWW14bExXZHljR011WTJsc2FYVnRMbWx2Ck1JSUJJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBNk55UDV4VnVsT3l4UitURkU2cXEKTGtvbHgydXBZa1JMMC9zN1RwTnVLa2pqSXF5cWd3dUVEOW1ndWdkeUp0VDdOaXkxM3lOU04wSWJTYkRRU0ZpeQpmbStmVnR3eGllUGRCVmorNXV4QWFWc1RURTI5SThYbVpHRG5ORHpRN0xNWGNiTkFJM1NzSVpoMm5kdTBWaWtPCnVTQ2gzd0RZMWgrd2szeGlhc2oxY2tiVUFHRnBGajRhRDNqUDNVWEw0Z2ZLOWhQN00rb2tTbWlzZnZqa01QakwKa2hteFkvdDFzc1o1WXpmMjFsZ2dBbDdldExKN0tNRlN5WkYwRkdHTmlzNnJGYzF1dm5vdDNDUVo2Sk5URWpHMQprakJxZVo2MHAyakh1dGpTdWpVZUJSV1N5Sk9zM2NLVDI3QmVEZmVTQkNmUktpYlQvaWwzdXZENmZvTStOSjJtCk1RSURBUUFCbzRHTk1JR0tNQTRHQTFVZER3RUIvd1FFQXdJRm9EQWRCZ05WSFNVRUZqQVVCZ2dyQmdFRkJRY0QKQVFZSUt3WUJCUVVIQXdJd0RBWURWUjBUQVFIL0JBSXdBREFmQmdOVkhTTUVHREFXZ0JTNEo2NVRuWmQzM1ZUbwpKZWdyZUQycENYUFBWVEFxQmdOVkhSRUVJekFoZ2g4cUxtUmxabUYxYkhRdWFIVmlZbXhsTFdkeWNHTXVZMmxzCmFYVnRMbWx2TUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFDVEVSY09weWpYa3lLVmdHRWs2c25jamRLbVIxRUcKQUk2cWtFZ3VWanN2SUpLcnh0TXN3c0F5Y0NuYWpodDNDQXNaVi9mK1lCQ3c0aGkzeHppNUZHRmN1TDAxL0FHaQp6Njc4dkw2eHZETW1ERG0rL2swbzFNYnZVVDJBRmVWZE1qdndVdFlsSytkd3pxMHl1Sjh6ajBzeEtqa0l0d3JYCjRwRUcrcGhGZjJvYWRybjBXamdhcVRpR2NJNWRlNzMwYStIb0M3WDdneG4xWmd3Um50RXI4V3AvcG1LeUtBRkoKT2ZodkFaLzVhSzd4WFBxZGRDQXF0S2ZHOVpaWkJwZzE3a3o5NnFhMlU2NlN6ZlFsUFNaWmwxWGJnRHVIR0pEawpPUkxuSDFhbmo3ZzNFTGM0K3NhYzNVSUlaTkVkb2cyWTZqM3lWZHJnTytHOW1vQ3pSQTJkK0tiUgotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== - tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBNk55UDV4VnVsT3l4UitURkU2cXFMa29seDJ1cFlrUkwwL3M3VHBOdUtrampJcXlxCmd3dUVEOW1ndWdkeUp0VDdOaXkxM3lOU04wSWJTYkRRU0ZpeWZtK2ZWdHd4aWVQZEJWais1dXhBYVZzVFRFMjkKSThYbVpHRG5ORHpRN0xNWGNiTkFJM1NzSVpoMm5kdTBWaWtPdVNDaDN3RFkxaCt3azN4aWFzajFja2JVQUdGcApGajRhRDNqUDNVWEw0Z2ZLOWhQN00rb2tTbWlzZnZqa01QakxraG14WS90MXNzWjVZemYyMWxnZ0FsN2V0TEo3CktNRlN5WkYwRkdHTmlzNnJGYzF1dm5vdDNDUVo2Sk5URWpHMWtqQnFlWjYwcDJqSHV0alN1alVlQlJXU3lKT3MKM2NLVDI3QmVEZmVTQkNmUktpYlQvaWwzdXZENmZvTStOSjJtTVFJREFRQUJBb0lCQUdxaEpSd1NDbE1UbDVKdQpSWGhkL0JPNlRGWm5ER2M5RXVQODlRbGtKVndqaUM3R2xrWHZyZ2Q4QmtkcFJYSldzTG43KzBRMGJsOXpXLytBCmdiKy9MZTBMdkJFUVAzYis2cVNhUGc4TU8rcUNrL0dxMzJOVUtjM3BPYllINnp1WVBySUgvT1MvZGxyajc5MFYKVUJ6c0xCdkZ2NW5ZVDNLdGpEUGREMHdLdmFvWHVSd0NMSTMrbUl4UDJTczAwY1k5azdDUXN3eDVpaXl3NEJSMQpIdllPa2IxN2hYYkJlQS9Ua28zOXVRcnF2bUhpZFZUVWJuaExxN240ZzBGcktoOGRvVkJuMGkzbWxGYkRBTzYvCnlySGdCZ01TcFQ1Z0tQN3dETzJtcis5Zzc3eEhxb0kyNWhUODFJSkc2WnRWSmRSMXYxcXRMNG53TW9RbXBhMnEKOE9qMHhxa0NnWUVBNi8wczYrRWJPY1lFVUwxQVJGMnRYRktQSjBpK1lWZnB6Y1lxKytBbisxRU1KTWN3VmdYSwpDcW9FZ3FCcnpidU84K3J5S1FVc0VITWU2TnBpc21UU2hzS0hJSEQ5eUFuS1JEUEdXSk1JTmxuQjNDQ1F3QmphCkYwL0RIL3Rwdk5SdjhhazJBenFHRU04MmtwYTc2b1R2YTNuL2h6aEl1MjU3dFFzbzVCVjM3QjhDZ1lFQS9KdC8KV1EzTkJValNCNEJtTUVJVHJrOGh4bW5Ubm5Pc0lFOWJjMGpORDlLb1dqOEFWSGZwbUF5QjVEcENLVUcwS21YQwpSWk52QWhzczhSQnFFamdGUDdEWUxCa0l0SWVVQ0V0T2RZSmRDZ1RyVGFCNkUzcWpadjJLRzl1QUtsUWZNdjZ5CisybHhiQnY5TjJ6d1ZsTDhHdlVYb3RZRkVsRXc3U3lsbnNKRkk2OENnWUJsMlo0U3E5UHhTbHBBY24yVFJBNzMKUXBhUU1PQWg2U3g2YUNYVEVYcVFCL25Dd2NsRnh6ZmRCR1h2N3NhNFExTytTRTk5anNMbTBwaXJqTjE5K3UrMQp3NjFJUGh2NXd1ZnVLWjVZU2hVSVNCTFlLNkh0bG94QjlXWFVudENTOCtqZ2x0UTk3bnRBTS81c28rdTBtdTB1CktMZzFKd0NWREdrTnlTUTcxRnd1cHdLQmdDVURsL1p4bWtBUytvdVUyZVJxV2ozcFpIYVJYU3ZUb1Q0ZXdyU3kKenpGcFdiZlhrWEhQSEZCbVRNWW5iemhpWTBidklEZFhzaVZwMi96NnZjQXJxMEU0NVlRWXF2amhBT1R5TyszbQpDMElEbElNTVo3YmcrbjZGMmkzWXpQL20wYWVFUUsrc05LMTFzSFhscFpYUWxSWlNJWU4vdnNCL1d4NzFzdEwvCmZtZ0xBb0dCQUxpY2I0UzQ2bmFKbWYvdXFLV1RMUi91NE5hOWhSR1JzU2Z3a3dTaUxKaGk3T2VmOVZuZk9QUVUKakRaajRHMDRuRUtkdzA1a3o0S0VpVU1aYnE3cnVZS05pRUloS1NSWWF6SU9sMnpjSGNuNGIrNUtOVjBqdmtlTwpIc3Q5YzREZVJxQ2xXNk5rY2JtT0NIQ2RjSkRsRE1YeEU1VzdscitENnMyZ1JLQW9FZ0RPCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== + ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lSQVBxa2tMQ3FLbm5tTThxWXczTW1Xakl3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkxTVRJeU9USXdOVEF4TlZvWERUSTRNVEl5T0RJdwpOVEF4TlZvd0ZERVNNQkFHQTFVRUF4TUpRMmxzYVhWdElFTkJNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFNSUlCQ2dLQ0FRRUFxc0cvUjI4bHF4Z0JCRDYrdllrL3JPMk1xRTYvZGMxaDROZlEvK2RPM01NeGVpM0gKbXk1eVVXRXg1QWVCaCtzUGhTVGd1L1lrUkJWVDFDS3VtbVNiZE5VdkRRN3Rwc2pPNWQ4YjdObUhqb0Yya25OcQorVVc1MGF0a0M1dVpjR1dEekdaU0kyNElTdVl6Qnd3VzVoSklvNDUzQ21Zd3pJSlVaZWEzam1EQWQ2QUswMnNxCjViRzNuVkpNSHcyNlJQNUJuZWJaQS82dWZlNFlCcDIxR1dvT2gzcERhR0NSS2FRRkpyT0VGSU41aFJYRHJJN1MKUmh3TVNYSVFLallDL0NHNTkrV3FEemhTSTdtMHAyWGRPODU4SEpxL3hSemVrUmZtRkhlcEErNFVTWWZoRUZCUgpRcTV5Z0w4TDZNZEZ5QnU3TXNlMzVxZ3kvMFd6a21uamthUkk2d0lEQVFBQm8yRXdYekFPQmdOVkhROEJBZjhFCkJBTUNBcVF3SFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGRkx0ZUlCVW9PTFZ6K1dRZWxzWkNFVTIyYVFQTUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQ1ZVeW8rTDAyQUJybUIvRUU0V3lmVkFBOWxjeEFYQ0w2QVFzUGpBcGl5QkwrWWFvNkQrN09uCkJCUTRsRklrMFAxbXViOU1OS2xnMTRPcmVCMDhVYkJ3Mm1uYzh5Uk1keERuUjdpVzAwSXJramtkM0Mzc2FLWG4Kcmp2V3dIdzZDcThVUzhZNFMxZjIyUlNNT1h5dE8yd29SenF1UDBYWmtzVUFoeXNTMnVGS1RtQlN1SXNzWWtwRQoxeGhQY1dScUwxRUt3MHRYUHQ4QnhpOGlyckpTeHFHU0pwdTFRVWNra05ISHV2SlNGRXZPZXZPTGJydVRiZmNBCnRDd3p3am1aQVo3clFPMGR5NFNDZ0ppdlhsMktDODRGQ0JvcVkyaVl5d2pubFRzSHMwRnNGSmFsWGtjczhkdzEKUkF6VmZQek54OCtkUFBobGNqdkh3Y1lqT25tdFJqaFAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURWakNDQWo2Z0F3SUJBZ0lRTmkxWC8xeWtubUpZczJ3VlZCTG9UekFOQmdrcWhraUc5dzBCQVFzRkFEQVUKTVJJd0VBWURWUVFERXdsRGFXeHBkVzBnUTBFd0hoY05NalV4TWpJNU1qQTFNREUxV2hjTk1qWXhNakk1TWpBMQpNREUxV2pBcU1TZ3dKZ1lEVlFRRERCOHFMbVJsWm1GMWJIUXVhSFZpWW14bExXZHljR011WTJsc2FYVnRMbWx2Ck1JSUJJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBcmF5QWhJUmY3OWY0bVBrc1ZIc2kKalBlTHpKaUdzbzlHemVNT0FKRlhSZDRzYmozdjFybVdyRjI5blczTnBzd09lOFgrY2FHR2JqVWNmdTkvaXdLdApCRjRBWU5aeW9sQzVwdlFIQUsvL3lLbFR5RVRRTTlTYmRvRlZoeVZ2TWUxL09WeU5yVlNVK090dGR2YUlZeUZBClJPdHUzY0hWUk5PMXlGcTJEVml1K0p0YTFwVmszOHNYZ1oxeVNCbW5qNXJhSVJOd2JwMVczMSs2UDJ3N3JCUFkKaUJPV0JsVFhZMm43eXMvSGJxRDhzVDY1VUsyN0ZVWTlXajg5QWNQWVppVVBGZjRwNVVLZ2ExUmtjQnFBQWlhUgpuWjNlSGhzNlpER0RPUStEZ2FXQW0wY1pJTTRubW42ZHh2ZlRVTmJJV21iMzBzYzZxVnUrMzBybThzM1R4Z1lDCkF3SURBUUFCbzRHTk1JR0tNQTRHQTFVZER3RUIvd1FFQXdJRm9EQWRCZ05WSFNVRUZqQVVCZ2dyQmdFRkJRY0QKQVFZSUt3WUJCUVVIQXdJd0RBWURWUjBUQVFIL0JBSXdBREFmQmdOVkhTTUVHREFXZ0JSUzdYaUFWS0RpMWMvbAprSHBiR1FoRk50bWtEekFxQmdOVkhSRUVJekFoZ2g4cUxtUmxabUYxYkhRdWFIVmlZbXhsTFdkeWNHTXVZMmxzCmFYVnRMbWx2TUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFDTEh4ZUd4QlRJT3cwa2N2VmYvSTBQNjNNRHR5Q0MKQ01IMzdENy9kWEd4RGVEYTMzUVNDQUwrZ2tHUVBHb0c3MU12Z09uNHJIZm54MkcwcVZrTERjQVhxUXg5dUlkcgpkOU1XNEN0SWpueVA2UzFSTEtJSmdQQmRJeTRlU2xZS2NKRU1HNVhVWERiWnh0TGEzWlo0N0E0VzZpMTY3d2pGCm5TZ05LNFBNdUxSd3ZqTDFScGt5blZDWmplVk1nUGE4K3ZIa1RDYW9rNVp2R09rQVhtbGhXS0pVcnlocjVKVmkKUXZiVk95dkhUaW1TM2RoZkt3dmV2OG5FZFJ3ak1UK0llUWxkb2hDMkZ0YWlVMU93V2d5Yi9uZFo1RCsrS3VjSwpodDlNbUorT1FCYk9PV2RWS0hZK0dUTmZ0NWdXS3IwT2Z3ZllnTUNGLzJ5Y3Z4ZFh3NlUxNjNIUQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBcmF5QWhJUmY3OWY0bVBrc1ZIc2lqUGVMekppR3NvOUd6ZU1PQUpGWFJkNHNiajN2CjFybVdyRjI5blczTnBzd09lOFgrY2FHR2JqVWNmdTkvaXdLdEJGNEFZTlp5b2xDNXB2UUhBSy8veUtsVHlFVFEKTTlTYmRvRlZoeVZ2TWUxL09WeU5yVlNVK090dGR2YUlZeUZBUk90dTNjSFZSTk8xeUZxMkRWaXUrSnRhMXBWawozOHNYZ1oxeVNCbW5qNXJhSVJOd2JwMVczMSs2UDJ3N3JCUFlpQk9XQmxUWFkybjd5cy9IYnFEOHNUNjVVSzI3CkZVWTlXajg5QWNQWVppVVBGZjRwNVVLZ2ExUmtjQnFBQWlhUm5aM2VIaHM2WkRHRE9RK0RnYVdBbTBjWklNNG4KbW42ZHh2ZlRVTmJJV21iMzBzYzZxVnUrMzBybThzM1R4Z1lDQXdJREFRQUJBb0lCQUI2anNrQ3V3ZVlrYWYxQwo2R2szR0E5YWxOVlJpMkh4R1FLMXJnQzNvM2YyREV5ZXJMTnJKb1BlUGRlS1NaL0ZuRUNGMjYvY1dVVVVKYVRYClI5OFVaOFcvZDcyZDE3ZC9HbUd6NTVCZGZIakFOc0c4TWljU3VTS2RDcU8vSFBEbi8ydjFRaUYwTEp3Rm5OaEMKQWluSzhmVzJ1Mlp3RGRvSXA2ZGpLWDhOU0VFZmpPNmtjMWlCWjR0TTR1Q2VXNHBKaTUxZ3ErQk9PTWVidkc2TwpjVjlORllzVURpeXFUZFV2WFlndzBlVmh3R2FHZVo3N0M5cFhKTUhqVVc1SWx5ZDBzWk9JRzVtWWlSOFNwYjZ3Cjhsa1ZDU09CKzdhSWlmbmd6RUNwRjV2WUhmNzQzRGtUeEQrZStUbFFWK1VlZ2hyOVN0dUphMklkVVNaZ29mSisKbUlVRDlURUNnWUVBMGdYWHkzK1crS2Z1MzZrYWNZVXQvOENrVFRVWTB0VUhMUXAxZGw0TnR1SGxpRFV5SUdSUgpTdHZpMHh0bFVPRTUzUmdJQ0h2SXhXS0JQcVcybHpVdThYYk1mOVk1Z0xUOWR0N1pjUEJuUnZKWWNNSi8vZjBzCk9ORC9oUnFPNFRvNlc5SjVhbzlnS3laMVVZR2h2R0l4TEVYM3NGSjcyUTZKaHRqWWVtOEY5QlVDZ1lFQTA3R1UKUFRPalp3OXVsSEtaRjB0R21EM2pGa1c4MGRseU1NZ2tLWkV3N3pVTHRQZ2phWTZML1BvZkZMRUlRMjV4dEdEdQpGUGMzRkNHTEsvclpvaFRZVmEzYjBad3BuSmxXM1ppTHB4ZWJNalo0MjJPL3RyOUNjVXlJY0FNYWVvNUp2Q2Z3CkdSOFFRaXpPMHYzMkRKbkVrUjlXUmU2MG12Q09MVlFBV2wzeEs3Y0NnWUFPaldNWGNaN0hXZk4rUWR4WU0wOHQKb201Ly93YmFUUEp6VjM4dUZYUkQwTUV6ditQYVJMa3hRNjBpellvbW9ZUjk0bFZrS1JlUFBiVVEzekNZcDVHNgpIMjRxQ1hEd2p4bms2Nm9MWGt6NXFIOGlTUWZQVklTRTUwNWRMYXJxeGpzVzg2MkFDYkV6TERZQzBNV2hKUGlkClJiZlUydEpFQnZMV2loM3QxZFdHeFFLQmdIMUJVQjNhUk8vQTRNUHkvb1FhNVIxRldPMGZ2RFpLTzllRFJHcFgKM3lWYU5LQVNSaG01M3ppQzRnc0RLSW1GZXIrb3JwWTNKQjdFTGkyS2ttVGxnbzlIZkk0TW9VNW9LRTZiYU40bwpDSXJ3VWhnMWNBVDRLNThRRXBaOEw1bWVRU0NVWE9yd1IxVndYN3ZWeWxUd1VxOHZaSU9pVnJocVp5V0kwMkNrCnZ3TUhBb0dBUjlPZStGWnRiNlFyS0VIcGJ3c2xoNWZScnY1WmVLeGJqYTZTU3NsejVWdktWZXppcG9BNWJUTjcKQVV6djZRS1hXaHRLdGk5cUZDWHA0V25KRTd1SVk0VG1BMnZMRUV3NHZlOG9XaUFQZXlpWWVFZXFWOWsyQy9TRgp3MFdSOHkxOEprN05PMXN3UTZ3TDNqWXNyMXNiZEEyelROU1c2N2FBMGRxZjRHZHVlOTg9Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== --- # Source: cilium/templates/cilium-configmap.yaml apiVersion: v1 @@ -101,7 +101,7 @@ data: # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 # address. - enable-ipv4: "true" + enable-ipv4: "false" # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6 # address. @@ -177,7 +177,7 @@ data: # - vxlan (default) # - geneve - routing-mode: "tunnel" + routing-mode: "native" tunnel-protocol: "vxlan" tunnel-source-port-range: "0-0" service-no-backend-response: "reject" @@ -187,7 +187,7 @@ data: enable-l7-proxy: "true" enable-ipv4-masquerade: "true" enable-ipv4-big-tcp: "false" - enable-ipv6-big-tcp: "false" + enable-ipv6-big-tcp: "true" enable-ipv6-masquerade: "true" enable-tcx: "true" datapath-mode: "veth" @@ -201,6 +201,8 @@ data: direct-routing-skip-unreachable: "false" + ipv4-native-routing-cidr: 10.0.0.0/8 + ipv6-native-routing-cidr: fd00::/100 kube-proxy-replacement: "true" kube-proxy-replacement-healthz-bind-address: "" @@ -227,7 +229,7 @@ data: enable-well-known-identities: "false" enable-node-selector-labels: "false" synchronize-k8s-nodes: "true" - operator-api-serve-addr: "127.0.0.1:9234" + operator-api-serve-addr: "[::1]:9234" enable-hubble: "true" # UNIX domain socket for Hubble server to listen to. @@ -239,12 +241,9 @@ data: hubble-tls-cert-file: /var/lib/cilium/tls/hubble/server.crt hubble-tls-key-file: /var/lib/cilium/tls/hubble/server.key hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/client-ca.crt - ipam: "cluster-pool" + hubble-prefer-ipv6: "true" + ipam: "kubernetes" ipam-cilium-node-update-rate: "15s" - cluster-pool-ipv4-cidr: "10.0.0.0/8" - cluster-pool-ipv4-mask-size: "24" - cluster-pool-ipv6-cidr: "fd00::/104" - cluster-pool-ipv6-mask-size: "120" default-lb-service-ipam: "lbipam" egress-gateway-reconciliation-trigger-interval: "1s" @@ -319,7 +318,7 @@ metadata: data: # Keep the key name as bootstrap-config.json to avoid breaking changes bootstrap-config.json: | - {"admin":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/admin.sock"}}},"applicationLogConfig":{"logFormat":{"textFormat":"[%Y-%m-%d %T.%e][%t][%l][%n] [%g:%#] %v"}},"bootstrapExtensions":[{"name":"envoy.bootstrap.internal_listener","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener"}}],"dynamicResources":{"cdsConfig":{"apiConfigSource":{"apiType":"GRPC","grpcServices":[{"envoyGrpc":{"clusterName":"xds-grpc-cilium"}}],"setNodeOnFirstMessageOnly":true,"transportApiVersion":"V3"},"initialFetchTimeout":"30s","resourceApiVersion":"V3"},"ldsConfig":{"apiConfigSource":{"apiType":"GRPC","grpcServices":[{"envoyGrpc":{"clusterName":"xds-grpc-cilium"}}],"setNodeOnFirstMessageOnly":true,"transportApiVersion":"V3"},"initialFetchTimeout":"30s","resourceApiVersion":"V3"}},"node":{"cluster":"ingress-cluster","id":"host~127.0.0.1~no-id~localdomain"},"overloadManager":{"resourceMonitors":[{"name":"envoy.resource_monitors.global_downstream_max_connections","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig","max_active_downstream_connections":"50000"}}]},"staticResources":{"clusters":[{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"ingress-cluster","type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"egress-cluster-tls","transportSocket":{"name":"cilium.tls_wrapper","typedConfig":{"@type":"type.googleapis.com/cilium.UpstreamTlsWrapperContext"}},"type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"upstreamHttpProtocolOptions":{},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"egress-cluster","type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"ingress-cluster-tls","transportSocket":{"name":"cilium.tls_wrapper","typedConfig":{"@type":"type.googleapis.com/cilium.UpstreamTlsWrapperContext"}},"type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"upstreamHttpProtocolOptions":{},"useDownstreamProtocolConfig":{}}}},{"connectTimeout":"2s","loadAssignment":{"clusterName":"xds-grpc-cilium","endpoints":[{"lbEndpoints":[{"endpoint":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/xds.sock"}}}}]}]},"name":"xds-grpc-cilium","type":"STATIC","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","explicitHttpConfig":{"http2ProtocolOptions":{}}}}},{"connectTimeout":"2s","loadAssignment":{"clusterName":"/envoy-admin","endpoints":[{"lbEndpoints":[{"endpoint":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/admin.sock"}}}}]}]},"name":"/envoy-admin","type":"STATIC"}],"listeners":[{"additionalAddresses":[{"address":{"socketAddress":{"address":"::","portValue":9964}}}],"address":{"socketAddress":{"address":"0.0.0.0","portValue":9964}},"filterChains":[{"filters":[{"name":"envoy.filters.network.http_connection_manager","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager","httpFilters":[{"name":"envoy.filters.http.router","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"}}],"internalAddressConfig":{"cidrRanges":[{"addressPrefix":"10.0.0.0","prefixLen":8},{"addressPrefix":"172.16.0.0","prefixLen":12},{"addressPrefix":"192.168.0.0","prefixLen":16},{"addressPrefix":"127.0.0.1","prefixLen":32},{"addressPrefix":"::1","prefixLen":128}]},"routeConfig":{"virtualHosts":[{"domains":["*"],"name":"prometheus_metrics_route","routes":[{"match":{"prefix":"/metrics"},"name":"prometheus_metrics_route","route":{"cluster":"/envoy-admin","prefixRewrite":"/stats/prometheus"}}]}]},"statPrefix":"envoy-prometheus-metrics-listener","streamIdleTimeout":"300s"}}]}],"name":"envoy-prometheus-metrics-listener"},{"additionalAddresses":[{"address":{"socketAddress":{"address":"::1","portValue":9878}}}],"address":{"socketAddress":{"address":"127.0.0.1","portValue":9878}},"filterChains":[{"filters":[{"name":"envoy.filters.network.http_connection_manager","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager","httpFilters":[{"name":"envoy.filters.http.router","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"}}],"internalAddressConfig":{"cidrRanges":[{"addressPrefix":"10.0.0.0","prefixLen":8},{"addressPrefix":"172.16.0.0","prefixLen":12},{"addressPrefix":"192.168.0.0","prefixLen":16},{"addressPrefix":"127.0.0.1","prefixLen":32},{"addressPrefix":"::1","prefixLen":128}]},"routeConfig":{"virtual_hosts":[{"domains":["*"],"name":"health","routes":[{"match":{"prefix":"/healthz"},"name":"health","route":{"cluster":"/envoy-admin","prefixRewrite":"/ready"}}]}]},"statPrefix":"envoy-health-listener","streamIdleTimeout":"300s"}}]}],"name":"envoy-health-listener"}]}} + {"admin":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/admin.sock"}}},"applicationLogConfig":{"logFormat":{"textFormat":"[%Y-%m-%d %T.%e][%t][%l][%n] [%g:%#] %v"}},"bootstrapExtensions":[{"name":"envoy.bootstrap.internal_listener","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener"}}],"dynamicResources":{"cdsConfig":{"apiConfigSource":{"apiType":"GRPC","grpcServices":[{"envoyGrpc":{"clusterName":"xds-grpc-cilium"}}],"setNodeOnFirstMessageOnly":true,"transportApiVersion":"V3"},"initialFetchTimeout":"30s","resourceApiVersion":"V3"},"ldsConfig":{"apiConfigSource":{"apiType":"GRPC","grpcServices":[{"envoyGrpc":{"clusterName":"xds-grpc-cilium"}}],"setNodeOnFirstMessageOnly":true,"transportApiVersion":"V3"},"initialFetchTimeout":"30s","resourceApiVersion":"V3"}},"node":{"cluster":"ingress-cluster","id":"host~127.0.0.1~no-id~localdomain"},"overloadManager":{"resourceMonitors":[{"name":"envoy.resource_monitors.global_downstream_max_connections","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig","max_active_downstream_connections":"50000"}}]},"staticResources":{"clusters":[{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"ingress-cluster","type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"egress-cluster-tls","transportSocket":{"name":"cilium.tls_wrapper","typedConfig":{"@type":"type.googleapis.com/cilium.UpstreamTlsWrapperContext"}},"type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"upstreamHttpProtocolOptions":{},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"egress-cluster","type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"ingress-cluster-tls","transportSocket":{"name":"cilium.tls_wrapper","typedConfig":{"@type":"type.googleapis.com/cilium.UpstreamTlsWrapperContext"}},"type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"upstreamHttpProtocolOptions":{},"useDownstreamProtocolConfig":{}}}},{"connectTimeout":"2s","loadAssignment":{"clusterName":"xds-grpc-cilium","endpoints":[{"lbEndpoints":[{"endpoint":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/xds.sock"}}}}]}]},"name":"xds-grpc-cilium","type":"STATIC","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","explicitHttpConfig":{"http2ProtocolOptions":{}}}}},{"connectTimeout":"2s","loadAssignment":{"clusterName":"/envoy-admin","endpoints":[{"lbEndpoints":[{"endpoint":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/admin.sock"}}}}]}]},"name":"/envoy-admin","type":"STATIC"}],"listeners":[{"address":{"socketAddress":{"address":"::","portValue":9964}},"filterChains":[{"filters":[{"name":"envoy.filters.network.http_connection_manager","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager","httpFilters":[{"name":"envoy.filters.http.router","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"}}],"internalAddressConfig":{"cidrRanges":[{"addressPrefix":"::1","prefixLen":128}]},"routeConfig":{"virtualHosts":[{"domains":["*"],"name":"prometheus_metrics_route","routes":[{"match":{"prefix":"/metrics"},"name":"prometheus_metrics_route","route":{"cluster":"/envoy-admin","prefixRewrite":"/stats/prometheus"}}]}]},"statPrefix":"envoy-prometheus-metrics-listener","streamIdleTimeout":"300s"}}]}],"name":"envoy-prometheus-metrics-listener"},{"address":{"socketAddress":{"address":"::1","portValue":9878}},"filterChains":[{"filters":[{"name":"envoy.filters.network.http_connection_manager","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager","httpFilters":[{"name":"envoy.filters.http.router","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"}}],"internalAddressConfig":{"cidrRanges":[{"addressPrefix":"::1","prefixLen":128}]},"routeConfig":{"virtual_hosts":[{"domains":["*"],"name":"health","routes":[{"match":{"prefix":"/healthz"},"name":"health","route":{"cluster":"/envoy-admin","prefixRewrite":"/ready"}}]}]},"statPrefix":"envoy-health-listener","streamIdleTimeout":"300s"}}]}],"name":"envoy-health-listener"}]}} --- # Source: cilium/templates/cilium-agent/clusterrole.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -891,7 +890,7 @@ spec: type: Unconfined containers: - name: cilium-agent - image: "quay.io/cilium/cilium:v1.18.4@sha256:49d87af187eeeb9e9e3ec2bc6bd372261a0b5cb2d845659463ba7cc10fe9e45f" + image: "quay.io/cilium/cilium:v1.18.5@sha256:2c92fb05962a346eaf0ce11b912ba434dc10bd54b9989e970416681f4a069628" imagePullPolicy: IfNotPresent command: - cilium-agent @@ -899,7 +898,7 @@ spec: - --config-dir=/tmp/cilium/config-map startupProbe: httpGet: - host: "127.0.0.1" + host: "::1" path: /healthz port: 9879 scheme: HTTP @@ -912,7 +911,7 @@ spec: initialDelaySeconds: 5 livenessProbe: httpGet: - host: "127.0.0.1" + host: "::1" path: /healthz port: 9879 scheme: HTTP @@ -927,7 +926,7 @@ spec: timeoutSeconds: 5 readinessProbe: httpGet: - host: "127.0.0.1" + host: "::1" path: /healthz port: 9879 scheme: HTTP @@ -1059,7 +1058,7 @@ spec: initContainers: - name: config - image: "quay.io/cilium/cilium:v1.18.4@sha256:49d87af187eeeb9e9e3ec2bc6bd372261a0b5cb2d845659463ba7cc10fe9e45f" + image: "quay.io/cilium/cilium:v1.18.5@sha256:2c92fb05962a346eaf0ce11b912ba434dc10bd54b9989e970416681f4a069628" imagePullPolicy: IfNotPresent command: - cilium-dbg @@ -1086,7 +1085,7 @@ spec: # Required to mount cgroup2 filesystem on the underlying Kubernetes node. # We use nsenter command with host's cgroup and mount namespaces enabled. - name: mount-cgroup - image: "quay.io/cilium/cilium:v1.18.4@sha256:49d87af187eeeb9e9e3ec2bc6bd372261a0b5cb2d845659463ba7cc10fe9e45f" + image: "quay.io/cilium/cilium:v1.18.5@sha256:2c92fb05962a346eaf0ce11b912ba434dc10bd54b9989e970416681f4a069628" imagePullPolicy: IfNotPresent env: - name: CGROUP_ROOT @@ -1123,7 +1122,7 @@ spec: drop: - ALL - name: apply-sysctl-overwrites - image: "quay.io/cilium/cilium:v1.18.4@sha256:49d87af187eeeb9e9e3ec2bc6bd372261a0b5cb2d845659463ba7cc10fe9e45f" + image: "quay.io/cilium/cilium:v1.18.5@sha256:2c92fb05962a346eaf0ce11b912ba434dc10bd54b9989e970416681f4a069628" imagePullPolicy: IfNotPresent env: - name: BIN_PATH @@ -1161,7 +1160,7 @@ spec: # from a privileged container because the mount propagation bidirectional # only works from privileged containers. - name: mount-bpf-fs - image: "quay.io/cilium/cilium:v1.18.4@sha256:49d87af187eeeb9e9e3ec2bc6bd372261a0b5cb2d845659463ba7cc10fe9e45f" + image: "quay.io/cilium/cilium:v1.18.5@sha256:2c92fb05962a346eaf0ce11b912ba434dc10bd54b9989e970416681f4a069628" imagePullPolicy: IfNotPresent args: - 'mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf' @@ -1177,7 +1176,7 @@ spec: mountPath: /sys/fs/bpf mountPropagation: Bidirectional - name: clean-cilium-state - image: "quay.io/cilium/cilium:v1.18.4@sha256:49d87af187eeeb9e9e3ec2bc6bd372261a0b5cb2d845659463ba7cc10fe9e45f" + image: "quay.io/cilium/cilium:v1.18.5@sha256:2c92fb05962a346eaf0ce11b912ba434dc10bd54b9989e970416681f4a069628" imagePullPolicy: IfNotPresent command: - /init-container.sh @@ -1228,7 +1227,7 @@ spec: mountPath: /var/run/cilium # wait-for-kube-proxy # Install the CNI binaries in an InitContainer so we don't have a writable host mount in the agent - name: install-cni-binaries - image: "quay.io/cilium/cilium:v1.18.4@sha256:49d87af187eeeb9e9e3ec2bc6bd372261a0b5cb2d845659463ba7cc10fe9e45f" + image: "quay.io/cilium/cilium:v1.18.5@sha256:2c92fb05962a346eaf0ce11b912ba434dc10bd54b9989e970416681f4a069628" imagePullPolicy: IfNotPresent command: - "/install-plugin.sh" @@ -1412,7 +1411,7 @@ spec: type: Unconfined containers: - name: cilium-envoy - image: "quay.io/cilium/cilium-envoy:v1.34.10-1762597008-ff7ae7d623be00078865cff1b0672cc5d9bfc6d5@sha256:1deb6709afcb5523579bf1abbc3255adf9e354565a88c4a9162c8d9cb1d77ab5" + image: "quay.io/cilium/cilium-envoy:v1.34.12-1765374555-6a93b0bbba8d6dc75b651cbafeedb062b2997716@sha256:3108521821c6922695ff1f6ef24b09026c94b195283f8bfbfc0fa49356a156e1" imagePullPolicy: IfNotPresent command: - /usr/bin/cilium-envoy-starter @@ -1423,7 +1422,7 @@ spec: - '--log-level info' startupProbe: httpGet: - host: "127.0.0.1" + host: "::1" path: /healthz port: 9878 scheme: HTTP @@ -1433,7 +1432,7 @@ spec: initialDelaySeconds: 5 livenessProbe: httpGet: - host: "127.0.0.1" + host: "::1" path: /healthz port: 9878 scheme: HTTP @@ -1443,7 +1442,7 @@ spec: timeoutSeconds: 5 readinessProbe: httpGet: - host: "127.0.0.1" + host: "::1" path: /healthz port: 9878 scheme: HTTP @@ -1594,7 +1593,7 @@ spec: type: RuntimeDefault containers: - name: cilium-operator - image: "quay.io/cilium/operator-generic:v1.18.4@sha256:1b22b9ff28affdf574378a70dade4ef835b00b080c2ee2418530809dd62c3012" + image: "quay.io/cilium/operator-generic:v1.18.5@sha256:36c3f6f14c8ced7f45b40b0a927639894b44269dd653f9528e7a0dc363a4eb99" imagePullPolicy: IfNotPresent command: - cilium-operator-generic @@ -1629,7 +1628,7 @@ spec: protocol: TCP livenessProbe: httpGet: - host: "127.0.0.1" + host: "::1" path: /healthz port: 9234 scheme: HTTP @@ -1638,7 +1637,7 @@ spec: timeoutSeconds: 3 readinessProbe: httpGet: - host: "127.0.0.1" + host: "::1" path: /healthz port: 9234 scheme: HTTP From 01d230bde61e70508e236d03e94c058f9f14b831 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 29 Dec 2025 17:00:23 -0500 Subject: [PATCH 39/81] Enable ipv4 and tunnel routing. --- nix/kubernetes/README.org | 21 ++++++---- .../files/manifests/cilium.yaml | 39 +++++++++---------- .../roles/kube_apiserver/default.nix | 2 +- .../roles/kube_controller_manager/default.nix | 4 +- nix/kubernetes/roles/kube_proxy/default.nix | 4 +- 5 files changed, 37 insertions(+), 33 deletions(-) diff --git a/nix/kubernetes/README.org b/nix/kubernetes/README.org index f5d38e2..8e103f3 100644 --- a/nix/kubernetes/README.org +++ b/nix/kubernetes/README.org @@ -23,14 +23,16 @@ --set k8sServiceHost="2620:11f:7001:7:ffff:ffff:ad7:1dd" \ --set k8sServicePort=6443 \ --set ipv6.enabled=true \ - --set ipv4.enabled=false \ - --set enableIPv4BIGTCP=false \ - --set enableIPv6BIGTCP=true \ - --set routingMode=native \ - --set ipv4NativeRoutingCIDR=10.0.0.0/8 \ - --set ipv6NativeRoutingCIDR=fd00::/100 + --set ipv4.enabled=true + # --set enableIPv6Masquerade=false + # --set enableIPv4BIGTCP=true \ + # --set enableIPv6BIGTCP=true + # --set routingMode=native \ + # --set ipv4NativeRoutingCIDR=10.0.0.0/8 \ + # --set ipv6NativeRoutingCIDR=fd00::/100 - kubectl -n kube-system exec ds/cilium -- cilium-dbg status --verbose + kubec + tl -n kube-system exec ds/cilium -- cilium-dbg status --verbose kubectl -n kube-system exec ds/cilium -- cilium-dbg status | grep KubeProxyReplacement # --set hostFirewall.enabled=true @@ -40,6 +42,11 @@ # --set ipv6-native-routing-cidr=fd00::/100 # --set 'ipam.operator.clusterPoolIPv4PodCIDRList=["10.0.0.0/8"]' \ # --set 'ipam.operator.clusterPoolIPv6PodCIDRList=["fd00::/100"]' \ + + + # --set encryption.enabled=true \ + # --set encryption.type=wireguard + # --set encryption.nodeEncryption=true #+end_src ** Install flux #+begin_src bash diff --git a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/cilium.yaml b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/cilium.yaml index 2113b65..8425169 100644 --- a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/cilium.yaml +++ b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/cilium.yaml @@ -36,8 +36,8 @@ metadata: name: cilium-ca namespace: kube-system data: - ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lSQVBxa2tMQ3FLbm5tTThxWXczTW1Xakl3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkxTVRJeU9USXdOVEF4TlZvWERUSTRNVEl5T0RJdwpOVEF4TlZvd0ZERVNNQkFHQTFVRUF4TUpRMmxzYVhWdElFTkJNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFNSUlCQ2dLQ0FRRUFxc0cvUjI4bHF4Z0JCRDYrdllrL3JPMk1xRTYvZGMxaDROZlEvK2RPM01NeGVpM0gKbXk1eVVXRXg1QWVCaCtzUGhTVGd1L1lrUkJWVDFDS3VtbVNiZE5VdkRRN3Rwc2pPNWQ4YjdObUhqb0Yya25OcQorVVc1MGF0a0M1dVpjR1dEekdaU0kyNElTdVl6Qnd3VzVoSklvNDUzQ21Zd3pJSlVaZWEzam1EQWQ2QUswMnNxCjViRzNuVkpNSHcyNlJQNUJuZWJaQS82dWZlNFlCcDIxR1dvT2gzcERhR0NSS2FRRkpyT0VGSU41aFJYRHJJN1MKUmh3TVNYSVFLallDL0NHNTkrV3FEemhTSTdtMHAyWGRPODU4SEpxL3hSemVrUmZtRkhlcEErNFVTWWZoRUZCUgpRcTV5Z0w4TDZNZEZ5QnU3TXNlMzVxZ3kvMFd6a21uamthUkk2d0lEQVFBQm8yRXdYekFPQmdOVkhROEJBZjhFCkJBTUNBcVF3SFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGRkx0ZUlCVW9PTFZ6K1dRZWxzWkNFVTIyYVFQTUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQ1ZVeW8rTDAyQUJybUIvRUU0V3lmVkFBOWxjeEFYQ0w2QVFzUGpBcGl5QkwrWWFvNkQrN09uCkJCUTRsRklrMFAxbXViOU1OS2xnMTRPcmVCMDhVYkJ3Mm1uYzh5Uk1keERuUjdpVzAwSXJramtkM0Mzc2FLWG4Kcmp2V3dIdzZDcThVUzhZNFMxZjIyUlNNT1h5dE8yd29SenF1UDBYWmtzVUFoeXNTMnVGS1RtQlN1SXNzWWtwRQoxeGhQY1dScUwxRUt3MHRYUHQ4QnhpOGlyckpTeHFHU0pwdTFRVWNra05ISHV2SlNGRXZPZXZPTGJydVRiZmNBCnRDd3p3am1aQVo3clFPMGR5NFNDZ0ppdlhsMktDODRGQ0JvcVkyaVl5d2pubFRzSHMwRnNGSmFsWGtjczhkdzEKUkF6VmZQek54OCtkUFBobGNqdkh3Y1lqT25tdFJqaFAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= - ca.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBcXNHL1IyOGxxeGdCQkQ2K3ZZay9yTzJNcUU2L2RjMWg0TmZRLytkTzNNTXhlaTNICm15NXlVV0V4NUFlQmgrc1BoU1RndS9Za1JCVlQxQ0t1bW1TYmROVXZEUTd0cHNqTzVkOGI3Tm1Iam9GMmtuTnEKK1VXNTBhdGtDNXVaY0dXRHpHWlNJMjRJU3VZekJ3d1c1aEpJbzQ1M0NtWXd6SUpVWmVhM2ptREFkNkFLMDJzcQo1YkczblZKTUh3MjZSUDVCbmViWkEvNnVmZTRZQnAyMUdXb09oM3BEYUdDUkthUUZKck9FRklONWhSWERySTdTClJod01TWElRS2pZQy9DRzU5K1dxRHpoU0k3bTBwMlhkTzg1OEhKcS94Unpla1JmbUZIZXBBKzRVU1lmaEVGQlIKUXE1eWdMOEw2TWRGeUJ1N01zZTM1cWd5LzBXemttbmprYVJJNndJREFRQUJBb0lCQUFRa05kVHdNNnd1SlZ3cQpmMnJPYU41elZuMkFoM0UrZlhpOEUwUVpvcGFGSDI2VXJLb3RXclZ4azRteHl1ajJnUVpjS2gwMDc1VTlKUzdZCmRuNmxJUVlZNDQ3QUJYQ05uME1CQWNiaFVjSFpqdnduVTNsZlFRMytSNnRtWnpiTUo0R3VHNkRqV3MwaVB5dVYKRmZzWGgyNThGcFFJZlhlQVU3YXkvVkcxTkFYU3NEb25qZE01c1d1dkhYTDJTeFVQQjJBa1ZHek1ENEg2ZzZQSQpBOC94ekdTOHRQYnF0eGhaYW1Nbk9pN1BId0hLZDNWajJBTHh3ZEcyZ0tWVFhBNzNkQ2pWekN4dXJiZVJVWEIvCjhpam5jeEVBMFovY3Vzb3g1RENNUDUrd1prM2xlSjFaMWg3WW9MVFVmUldWL25KU244RDFhUVlRUmZXSGQ4NE4KWnVndDFia0NnWUVBeVBDb3psdkZmVmNkS0orYmtqa3o3bGN3bmhZRWpkTEswdkRUaFp5WUkzMTdiS3hHK28wSQplSDVWd21zRjFXUm9KYmZxYzFIN25nczFVUVp1ZnJvSko4TzJVZkp6YW9YdjIxMUswRVVNOCsvSFR6Q3ZwZDlhCmUvVElEVGlvUE1YRzEwOENDQ0l2aUM5NzBpWHMyajRuY3dKb2ZVZVBXdGZ3ek1vdStwN0dtV2tDZ1lFQTJZdlMKMzRONlFaOUVMUkY4Uzc1M0VjZHhtL0lxdDd2TnVzVW41OUxqR01ZRFpYM25MUEt3MmZRMG9nSXZKODVOUThsdQpIRTluRjVsc21uSkxNd2h2aFlPc1owdVVlNnNtaVRGbTRmeGdHT1J6QnRFV2pHQWlxRTB4eTZOVHNaNWRWNU1DCmcxSFIyZVY3N25SOGhNWVkvWnMwOUZNSHhNY3hRUTBoRVI2TjBUTUNnWUJ3b0JzdGx0aUptVk01bXNRNjRCOFEKWHdSY0J5RkVVTHJvSXAyY1RSb2xBa2Mwc1JjVGhnc1NabFV4SklWZmh1aXRBM25BYTFvcHlrUjNCckRXRFBYbQp1TmF1OWNMbEdPOUl0L0NmN2FRRjhnWjNoMlZjQm1XVUJxNCtZdHhCTytWb2R0cy9yZEdRZG5UalkraHVwdG03CnA4VGpPMEhKZkRGMngzZ3ZkdUQveVFLQmdIVThhdm5WS3U5dVd4bEdrUXRvcTFaRGV6QWxRMndjbzNRQmg4M2cKMGsycGVHOGJlSmU2ZWlBYkFMcEdIdzkrbUJqdDFvOGZJMW1PYnY2VWxYMzlEOHlOOXp6N2VCdGxVckVnbEdjNAowTGpaZER1Nk5rS1VGRmxLc0lrampaanMxVFM2bnVJU0Y5c3JWcXhjempYblk5Ulg3aU8xRXp5eWRCS3dkcDk1CkxjZ05Bb0dCQUxaS1IxRjl4SElnelpoUEVLM04vRis1VzZMLytVR29jRkt5K2VTU0tLbnA4eWlUVHJiRUdvYUMKU1pHdlo2eFBod1g5RGJ3ZnpyZHpXZWpFSUpYU3BwdmloekhUcXE1WVd4VnVkbnlFVzVxRmhENkZOencrV1hRegp4R28wM2R3N21JeXF6ZDM2K1RDcWFvNmJUeWc1bUUvZUJLREhCVGZ3UFNsanZrQmVPUU1YCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== + ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lSQVBMcVRXajlHdHFqRXNMRTgxemtUUkl3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkxTVRJeU9USXlNVGt6TkZvWERUSTRNVEl5T0RJeQpNVGt6TkZvd0ZERVNNQkFHQTFVRUF4TUpRMmxzYVhWdElFTkJNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFNSUlCQ2dLQ0FRRUE0UVIvSy9WOUEvNnNPdzU3N3ZodTVTcEw4anJHMFFWR1VPeXVqM2V5LytVRno5MWYKZlBiVDhzK3FCY1BreFZ4YVV4d05McVJGcFkrLzhKUm5NWi9Hb21jdEpiY2VQLzBuRnhMekRUdFhqWHFWTmMrSgpZSUx2cWFEYVhyOStBZFhmdUNuVFRtenlWVTBCekRGSFU3VTYyMldOL05FbEhZK3YvaTY2L3RWWFh5VWNpcXNzCit5Rjh0Vkd5cVNFTG5Wd0lDKzVvL1BhV29ybzEwZ0xCLzNBSGdNRlBDTG14QnF0Z3RKTzVaSWF5Mzc5VHJhamgKMytpQ2s0UGNIT2RhbFVDMGFiMHlUYjlvS0xjTHZoVFBMUjFpdGphVmRhaU9VYUtONDUyWUQ4UnJ1NFlyZ05rUwphMmtlcnNlbjZKNTRDZHFQL3VHcXJwaDV5TTFtSFNoa1dLaEdsd0lEQVFBQm8yRXdYekFPQmdOVkhROEJBZjhFCkJBTUNBcVF3SFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGREIvYzlIRnFSY0prREpHeVV3aWlQZ0k2TDhmTUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQlZSc3NtVlFIWkphVi9uNk8ydFJNQ2ZOckcwR01xRkpRNFlBRjg4ZW0wT0hNWWszRjNocGZmCkhUZHVrbTJ4OS9DZlNHNkdra3pCbFVhK1cxOVJNUFhJcXdybmwrOHEzVk4ySDA3THZFajVzZHhOK1hnSVR4dG4Kb3B0MndidXpJZWNjR2FnNEd1aHJOcHBXVndHODZURTVMelU0MXE0dlNUS01CUWtIVG5aVnhaTVV3RkxIUzNEVQpPalA0b3prYjc0OEZZNkJvbjR2dGJVYjk2U0I2V2FBRnhGZW8vanRSN0RUSjVxNjZzOFp1c05tMHBKZURzVFQ2Ck9xdzUvWDhJeVRPN2tZUmV4RTIvUEZwRWZOUXVyRTBlNGlvTG9rSC9acmtNZmtncTdISU84RDRUUFJVL0s0YXIKMTdwZmhBWDVieVlScy9FZTRrV0dQclByb3EzcGpFZ1kKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + ca.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBNFFSL0svVjlBLzZzT3c1Nzd2aHU1U3BMOGpyRzBRVkdVT3l1ajNleS8rVUZ6OTFmCmZQYlQ4cytxQmNQa3hWeGFVeHdOTHFSRnBZKy84SlJuTVovR29tY3RKYmNlUC8wbkZ4THpEVHRYalhxVk5jK0oKWUlMdnFhRGFYcjkrQWRYZnVDblRUbXp5VlUwQnpERkhVN1U2MjJXTi9ORWxIWSt2L2k2Ni90VlhYeVVjaXFzcworeUY4dFZHeXFTRUxuVndJQys1by9QYVdvcm8xMGdMQi8zQUhnTUZQQ0xteEJxdGd0Sk81WklheTM3OVRyYWpoCjMraUNrNFBjSE9kYWxVQzBhYjB5VGI5b0tMY0x2aFRQTFIxaXRqYVZkYWlPVWFLTjQ1MllEOFJydTRZcmdOa1MKYTJrZXJzZW42SjU0Q2RxUC91R3FycGg1eU0xbUhTaGtXS2hHbHdJREFRQUJBb0lCQUE5Zi9VaUZwNXNrUmFScgpZdkI3TFNpZmNUMEY0eHZaSG1yZElUaFFWM3pBcTFyK3AwMmtsK3JaWWFhdk1leUNXUEdnMHczQ3o3ZDJVUWtoCk9zUGJSUGxIejMvcU9UanFGVWV3VmNjcnJOblA3RzRXMWk0d0JDdzAvM2JGNHRoQlF0NFVqWW1vVEE2a0NtRm4KWkpaRnBkWGo1SVhIeko5dWQvb0lPMks2TlJRZEZpL01YaDdxdDhpaWtSUWllcVZQTEZlZExYSlBVSHk3ZEw5eApMaHpzSTNWak9Ed1VYZGJtSXlMYXhpUkpwY3NWb3RqcitGWEdEaVFvdnFJYnk0SWlId2ZZeFMzR3RkUUI1M2J3Cm5FOW9Ga1o1WHFCRys0dmxtQ0N2cVQvaFZkN3ZRU3U4S3p4dVd1MFh3WWJUelppZTBMd0hESndxTDRNdDVUbEEKVDlXQ2I5RUNnWUVBL3pCR3Y2cEFCUUZuL1l5VDRIRzNxcXNXKzhDUjFHZEhDUHI1YVZxYXpXSldOc3VwNjExKwpsUnZqT2RyVThtUkRFS2pGRzJMS2J4QmNlRkFIb2hkZGtoNnkyK1NsMFM0VnBKc3dPMGtXMkhtOWtzeFFYNGVRCmM4Ui8xVWFpVmNENC9KYnl5d012NjlQU3lSM3p6MEl2QTJZa2p5aGh0U0tVell5NlEramlZbE1DZ1lFQTRidXAKUXlxazhZSW9DWkQ0dHdRMHlzRHBibWlaTVpmem11VWFvdlQzOXEvdUpXMWtSWWdFeG9KSTJpVnUzRGx1YmxJaQplWEMvTG52SklZenZhQjM2MlhaaXFVWUJRamJtdUp3eks0QVNrRU9VOWNMNmI4dnpBRVB4Mi9kS3NJdnNhM1NnCnRPaE4wT1ozTGNQWVFXVXo0Q0R0RjY2SGdpUkJDS201cENtclNpMENnWUJES1JGMW4vajVXRVZtRlhsVnVKZjkKOHNrNXVKU29zZ1dmOTIxc3JNdlJBZkJ1dTJzYzVwNUozKzBOYk9wZVlNVEZ2YVVpYi8yVy9WZFkwcmN2a2JhdQpuaVcwZVppcVZOYWRXN3AxMythRFVvYngrNU9ya0tJVDFjTk8vaWY3S2E1ZHk5eGFVWnhyRkhTRk9ielE3em93ClN0R1VXNnhiWDU2SitsK0xQOTlVd1FLQmdCYVdqc04zMnZXSHpyWFdXZTBHY0xueFArcWFjT256aUo3eFdnOEEKY0dleEJ5V2Jvck1LZjEvVjNiQm1tb2RTLytmdU5DRHUxVkt6ZEZiMUlXZkx5RFJHa216WjRucGJ3QTZ0SXJteApvU25xZVZkMllWVWFsWUJyVENKMHhuYVFLZUkrMDI0RzZQS0VmVnlKQW5UWXlhNGQzVnZIVFN3S3NLOUxQSlplCnpSTXBBb0dBYk84NnRRMjNIWnB1Q1Z1eWprTkdoSklrTSsyZ0xoZURFVnBxVmsxU1kvL1RLYzNQSDJQY1lHNjcKM3pmTTJkTE01d2ZNenBVOVNsamNnQXZ3TjZBalJvdm1kOGJDZzl2RzEwZmRjSXFiRko1WU4xellRM0JPTDZudgpuNjBXVWQ5V2FXeDdDSGxSWjN6N1NBOFZuMlJ6NHFkdGRXdnlDblZtTnN0VmM1WnIwWk09Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== --- # Source: cilium/templates/hubble/tls-helm/server-secret.yaml apiVersion: v1 @@ -47,9 +47,9 @@ metadata: namespace: kube-system type: kubernetes.io/tls data: - ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lSQVBxa2tMQ3FLbm5tTThxWXczTW1Xakl3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkxTVRJeU9USXdOVEF4TlZvWERUSTRNVEl5T0RJdwpOVEF4TlZvd0ZERVNNQkFHQTFVRUF4TUpRMmxzYVhWdElFTkJNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFNSUlCQ2dLQ0FRRUFxc0cvUjI4bHF4Z0JCRDYrdllrL3JPMk1xRTYvZGMxaDROZlEvK2RPM01NeGVpM0gKbXk1eVVXRXg1QWVCaCtzUGhTVGd1L1lrUkJWVDFDS3VtbVNiZE5VdkRRN3Rwc2pPNWQ4YjdObUhqb0Yya25OcQorVVc1MGF0a0M1dVpjR1dEekdaU0kyNElTdVl6Qnd3VzVoSklvNDUzQ21Zd3pJSlVaZWEzam1EQWQ2QUswMnNxCjViRzNuVkpNSHcyNlJQNUJuZWJaQS82dWZlNFlCcDIxR1dvT2gzcERhR0NSS2FRRkpyT0VGSU41aFJYRHJJN1MKUmh3TVNYSVFLallDL0NHNTkrV3FEemhTSTdtMHAyWGRPODU4SEpxL3hSemVrUmZtRkhlcEErNFVTWWZoRUZCUgpRcTV5Z0w4TDZNZEZ5QnU3TXNlMzVxZ3kvMFd6a21uamthUkk2d0lEQVFBQm8yRXdYekFPQmdOVkhROEJBZjhFCkJBTUNBcVF3SFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGRkx0ZUlCVW9PTFZ6K1dRZWxzWkNFVTIyYVFQTUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQ1ZVeW8rTDAyQUJybUIvRUU0V3lmVkFBOWxjeEFYQ0w2QVFzUGpBcGl5QkwrWWFvNkQrN09uCkJCUTRsRklrMFAxbXViOU1OS2xnMTRPcmVCMDhVYkJ3Mm1uYzh5Uk1keERuUjdpVzAwSXJramtkM0Mzc2FLWG4Kcmp2V3dIdzZDcThVUzhZNFMxZjIyUlNNT1h5dE8yd29SenF1UDBYWmtzVUFoeXNTMnVGS1RtQlN1SXNzWWtwRQoxeGhQY1dScUwxRUt3MHRYUHQ4QnhpOGlyckpTeHFHU0pwdTFRVWNra05ISHV2SlNGRXZPZXZPTGJydVRiZmNBCnRDd3p3am1aQVo3clFPMGR5NFNDZ0ppdlhsMktDODRGQ0JvcVkyaVl5d2pubFRzSHMwRnNGSmFsWGtjczhkdzEKUkF6VmZQek54OCtkUFBobGNqdkh3Y1lqT25tdFJqaFAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= - tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURWakNDQWo2Z0F3SUJBZ0lRTmkxWC8xeWtubUpZczJ3VlZCTG9UekFOQmdrcWhraUc5dzBCQVFzRkFEQVUKTVJJd0VBWURWUVFERXdsRGFXeHBkVzBnUTBFd0hoY05NalV4TWpJNU1qQTFNREUxV2hjTk1qWXhNakk1TWpBMQpNREUxV2pBcU1TZ3dKZ1lEVlFRRERCOHFMbVJsWm1GMWJIUXVhSFZpWW14bExXZHljR011WTJsc2FYVnRMbWx2Ck1JSUJJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBcmF5QWhJUmY3OWY0bVBrc1ZIc2kKalBlTHpKaUdzbzlHemVNT0FKRlhSZDRzYmozdjFybVdyRjI5blczTnBzd09lOFgrY2FHR2JqVWNmdTkvaXdLdApCRjRBWU5aeW9sQzVwdlFIQUsvL3lLbFR5RVRRTTlTYmRvRlZoeVZ2TWUxL09WeU5yVlNVK090dGR2YUlZeUZBClJPdHUzY0hWUk5PMXlGcTJEVml1K0p0YTFwVmszOHNYZ1oxeVNCbW5qNXJhSVJOd2JwMVczMSs2UDJ3N3JCUFkKaUJPV0JsVFhZMm43eXMvSGJxRDhzVDY1VUsyN0ZVWTlXajg5QWNQWVppVVBGZjRwNVVLZ2ExUmtjQnFBQWlhUgpuWjNlSGhzNlpER0RPUStEZ2FXQW0wY1pJTTRubW42ZHh2ZlRVTmJJV21iMzBzYzZxVnUrMzBybThzM1R4Z1lDCkF3SURBUUFCbzRHTk1JR0tNQTRHQTFVZER3RUIvd1FFQXdJRm9EQWRCZ05WSFNVRUZqQVVCZ2dyQmdFRkJRY0QKQVFZSUt3WUJCUVVIQXdJd0RBWURWUjBUQVFIL0JBSXdBREFmQmdOVkhTTUVHREFXZ0JSUzdYaUFWS0RpMWMvbAprSHBiR1FoRk50bWtEekFxQmdOVkhSRUVJekFoZ2g4cUxtUmxabUYxYkhRdWFIVmlZbXhsTFdkeWNHTXVZMmxzCmFYVnRMbWx2TUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFDTEh4ZUd4QlRJT3cwa2N2VmYvSTBQNjNNRHR5Q0MKQ01IMzdENy9kWEd4RGVEYTMzUVNDQUwrZ2tHUVBHb0c3MU12Z09uNHJIZm54MkcwcVZrTERjQVhxUXg5dUlkcgpkOU1XNEN0SWpueVA2UzFSTEtJSmdQQmRJeTRlU2xZS2NKRU1HNVhVWERiWnh0TGEzWlo0N0E0VzZpMTY3d2pGCm5TZ05LNFBNdUxSd3ZqTDFScGt5blZDWmplVk1nUGE4K3ZIa1RDYW9rNVp2R09rQVhtbGhXS0pVcnlocjVKVmkKUXZiVk95dkhUaW1TM2RoZkt3dmV2OG5FZFJ3ak1UK0llUWxkb2hDMkZ0YWlVMU93V2d5Yi9uZFo1RCsrS3VjSwpodDlNbUorT1FCYk9PV2RWS0hZK0dUTmZ0NWdXS3IwT2Z3ZllnTUNGLzJ5Y3Z4ZFh3NlUxNjNIUQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== - tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBcmF5QWhJUmY3OWY0bVBrc1ZIc2lqUGVMekppR3NvOUd6ZU1PQUpGWFJkNHNiajN2CjFybVdyRjI5blczTnBzd09lOFgrY2FHR2JqVWNmdTkvaXdLdEJGNEFZTlp5b2xDNXB2UUhBSy8veUtsVHlFVFEKTTlTYmRvRlZoeVZ2TWUxL09WeU5yVlNVK090dGR2YUlZeUZBUk90dTNjSFZSTk8xeUZxMkRWaXUrSnRhMXBWawozOHNYZ1oxeVNCbW5qNXJhSVJOd2JwMVczMSs2UDJ3N3JCUFlpQk9XQmxUWFkybjd5cy9IYnFEOHNUNjVVSzI3CkZVWTlXajg5QWNQWVppVVBGZjRwNVVLZ2ExUmtjQnFBQWlhUm5aM2VIaHM2WkRHRE9RK0RnYVdBbTBjWklNNG4KbW42ZHh2ZlRVTmJJV21iMzBzYzZxVnUrMzBybThzM1R4Z1lDQXdJREFRQUJBb0lCQUI2anNrQ3V3ZVlrYWYxQwo2R2szR0E5YWxOVlJpMkh4R1FLMXJnQzNvM2YyREV5ZXJMTnJKb1BlUGRlS1NaL0ZuRUNGMjYvY1dVVVVKYVRYClI5OFVaOFcvZDcyZDE3ZC9HbUd6NTVCZGZIakFOc0c4TWljU3VTS2RDcU8vSFBEbi8ydjFRaUYwTEp3Rm5OaEMKQWluSzhmVzJ1Mlp3RGRvSXA2ZGpLWDhOU0VFZmpPNmtjMWlCWjR0TTR1Q2VXNHBKaTUxZ3ErQk9PTWVidkc2TwpjVjlORllzVURpeXFUZFV2WFlndzBlVmh3R2FHZVo3N0M5cFhKTUhqVVc1SWx5ZDBzWk9JRzVtWWlSOFNwYjZ3Cjhsa1ZDU09CKzdhSWlmbmd6RUNwRjV2WUhmNzQzRGtUeEQrZStUbFFWK1VlZ2hyOVN0dUphMklkVVNaZ29mSisKbUlVRDlURUNnWUVBMGdYWHkzK1crS2Z1MzZrYWNZVXQvOENrVFRVWTB0VUhMUXAxZGw0TnR1SGxpRFV5SUdSUgpTdHZpMHh0bFVPRTUzUmdJQ0h2SXhXS0JQcVcybHpVdThYYk1mOVk1Z0xUOWR0N1pjUEJuUnZKWWNNSi8vZjBzCk9ORC9oUnFPNFRvNlc5SjVhbzlnS3laMVVZR2h2R0l4TEVYM3NGSjcyUTZKaHRqWWVtOEY5QlVDZ1lFQTA3R1UKUFRPalp3OXVsSEtaRjB0R21EM2pGa1c4MGRseU1NZ2tLWkV3N3pVTHRQZ2phWTZML1BvZkZMRUlRMjV4dEdEdQpGUGMzRkNHTEsvclpvaFRZVmEzYjBad3BuSmxXM1ppTHB4ZWJNalo0MjJPL3RyOUNjVXlJY0FNYWVvNUp2Q2Z3CkdSOFFRaXpPMHYzMkRKbkVrUjlXUmU2MG12Q09MVlFBV2wzeEs3Y0NnWUFPaldNWGNaN0hXZk4rUWR4WU0wOHQKb201Ly93YmFUUEp6VjM4dUZYUkQwTUV6ditQYVJMa3hRNjBpellvbW9ZUjk0bFZrS1JlUFBiVVEzekNZcDVHNgpIMjRxQ1hEd2p4bms2Nm9MWGt6NXFIOGlTUWZQVklTRTUwNWRMYXJxeGpzVzg2MkFDYkV6TERZQzBNV2hKUGlkClJiZlUydEpFQnZMV2loM3QxZFdHeFFLQmdIMUJVQjNhUk8vQTRNUHkvb1FhNVIxRldPMGZ2RFpLTzllRFJHcFgKM3lWYU5LQVNSaG01M3ppQzRnc0RLSW1GZXIrb3JwWTNKQjdFTGkyS2ttVGxnbzlIZkk0TW9VNW9LRTZiYU40bwpDSXJ3VWhnMWNBVDRLNThRRXBaOEw1bWVRU0NVWE9yd1IxVndYN3ZWeWxUd1VxOHZaSU9pVnJocVp5V0kwMkNrCnZ3TUhBb0dBUjlPZStGWnRiNlFyS0VIcGJ3c2xoNWZScnY1WmVLeGJqYTZTU3NsejVWdktWZXppcG9BNWJUTjcKQVV6djZRS1hXaHRLdGk5cUZDWHA0V25KRTd1SVk0VG1BMnZMRUV3NHZlOG9XaUFQZXlpWWVFZXFWOWsyQy9TRgp3MFdSOHkxOEprN05PMXN3UTZ3TDNqWXNyMXNiZEEyelROU1c2N2FBMGRxZjRHZHVlOTg9Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== + ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lSQVBMcVRXajlHdHFqRXNMRTgxemtUUkl3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkxTVRJeU9USXlNVGt6TkZvWERUSTRNVEl5T0RJeQpNVGt6TkZvd0ZERVNNQkFHQTFVRUF4TUpRMmxzYVhWdElFTkJNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFNSUlCQ2dLQ0FRRUE0UVIvSy9WOUEvNnNPdzU3N3ZodTVTcEw4anJHMFFWR1VPeXVqM2V5LytVRno5MWYKZlBiVDhzK3FCY1BreFZ4YVV4d05McVJGcFkrLzhKUm5NWi9Hb21jdEpiY2VQLzBuRnhMekRUdFhqWHFWTmMrSgpZSUx2cWFEYVhyOStBZFhmdUNuVFRtenlWVTBCekRGSFU3VTYyMldOL05FbEhZK3YvaTY2L3RWWFh5VWNpcXNzCit5Rjh0Vkd5cVNFTG5Wd0lDKzVvL1BhV29ybzEwZ0xCLzNBSGdNRlBDTG14QnF0Z3RKTzVaSWF5Mzc5VHJhamgKMytpQ2s0UGNIT2RhbFVDMGFiMHlUYjlvS0xjTHZoVFBMUjFpdGphVmRhaU9VYUtONDUyWUQ4UnJ1NFlyZ05rUwphMmtlcnNlbjZKNTRDZHFQL3VHcXJwaDV5TTFtSFNoa1dLaEdsd0lEQVFBQm8yRXdYekFPQmdOVkhROEJBZjhFCkJBTUNBcVF3SFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGREIvYzlIRnFSY0prREpHeVV3aWlQZ0k2TDhmTUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQlZSc3NtVlFIWkphVi9uNk8ydFJNQ2ZOckcwR01xRkpRNFlBRjg4ZW0wT0hNWWszRjNocGZmCkhUZHVrbTJ4OS9DZlNHNkdra3pCbFVhK1cxOVJNUFhJcXdybmwrOHEzVk4ySDA3THZFajVzZHhOK1hnSVR4dG4Kb3B0MndidXpJZWNjR2FnNEd1aHJOcHBXVndHODZURTVMelU0MXE0dlNUS01CUWtIVG5aVnhaTVV3RkxIUzNEVQpPalA0b3prYjc0OEZZNkJvbjR2dGJVYjk2U0I2V2FBRnhGZW8vanRSN0RUSjVxNjZzOFp1c05tMHBKZURzVFQ2Ck9xdzUvWDhJeVRPN2tZUmV4RTIvUEZwRWZOUXVyRTBlNGlvTG9rSC9acmtNZmtncTdISU84RDRUUFJVL0s0YXIKMTdwZmhBWDVieVlScy9FZTRrV0dQclByb3EzcGpFZ1kKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURWekNDQWorZ0F3SUJBZ0lSQVB0VkxoVHZtMkFJbEFVYTRwaFB6Ykl3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkxTVRJeU9USXlNVGt6TkZvWERUSTJNVEl5T1RJeQpNVGt6TkZvd0tqRW9NQ1lHQTFVRUF3d2ZLaTVrWldaaGRXeDBMbWgxWW1Kc1pTMW5jbkJqTG1OcGJHbDFiUzVwCmJ6Q0NBU0l3RFFZSktvWklodmNOQVFFQkJRQURnZ0VQQURDQ0FRb0NnZ0VCQUwyaWI2V0E4bUNZcitjWlZKTEIKbXBZd0VwQlo0bEpZYU1tbHZTMnlPQ09HbEE3bEh6c1QyZkVLT2dFVFZoNnZkSkJwVkVRdk5pb0VGejZ3YUd1OApuL1dmVVV4STZldUhkZmNjdXNwK1pLRld6dGJ5R0JnK01DUG42OEE5eFN1WWorMXpEWTA3WGVEem0xYU1jTGZGClFleER1OFQ5UnJ4UUJoTDBkWEVraWpFdFJYN1BSUTB1ODYrbnROVmVrRk4wa3lSTzZQMVhLcmpKWjFLeFJYZFMKOGRQaU1peFg5OXZZREwrbDR1a2hqQXJFUFYwK0d5OXo1cklobzMzYktRMFJBV1NtQi9SNU5aUllHNTR2U2VZZwplQzJsM2JQZmFMVGhuSU5NajIrb1JnRzErK21FMWxDMkZJTU4xaTFWUmN6cVdYZGlOZUxGMVdNNnBlaWkrSkZVCnVhOENBd0VBQWFPQmpUQ0JpakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdIUVlEVlIwbEJCWXdGQVlJS3dZQkJRVUgKQXdFR0NDc0dBUVVGQndNQ01Bd0dBMVVkRXdFQi93UUNNQUF3SHdZRFZSMGpCQmd3Rm9BVU1IOXowY1dwRndtUQpNa2JKVENLSStBam92eDh3S2dZRFZSMFJCQ013SVlJZktpNWtaV1poZFd4MExtaDFZbUpzWlMxbmNuQmpMbU5wCmJHbDFiUzVwYnpBTkJna3Foa2lHOXcwQkFRc0ZBQU9DQVFFQVpHNk9zMng2ZmdKQ01qSUNEUS9UaDFQSWhVakwKQ3FJQzlzWVZRRHM0dWpRMXJkK0pMMHBLQWVVa04ySmMxWktwanVSb2YwSGE2cmpzZVArTVZLMithemx6Qm9xTApDWVh1ZHJRb0h4U1RjVU9na01oWmtqM2dnUkthODBraEh1UjJOY21KbnZOTnVGdU4vVFBTd0JBUXpVczFLQkFpClJ0eUlOWjdERU9OVWd5Y0dwRGg5NkFKK2JUWlBGT1U1d28xS1hKajRCaVV0c1dxaFd3T3ozTWhkd1NsVC9yVTIKR1hmbXdibDY1THpneEZGaWt0TUpoUTdRdkRzdDVyM3prSXVVZWRPZ0U0RnRUMTRoZVgvMnlveitSWkxqaWFJQgowMWJFVEtGc1B6bFVQMnVLUzFaUnFqWlN0SjFZMXEycDYzRHJkM0hMNllWMytwa3lONG8waWhYM1NBPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBdmFKdnBZRHlZSml2NXhsVWtzR2FsakFTa0ZuaVVsaG95YVc5TGJJNEk0YVVEdVVmCk94UFo4UW82QVJOV0hxOTBrR2xVUkM4MktnUVhQckJvYTd5ZjlaOVJURWpwNjRkMTl4eTZ5bjVrb1ZiTzF2SVkKR0Q0d0krZnJ3RDNGSzVpUDdYTU5qVHRkNFBPYlZveHd0OFZCN0VPN3hQMUd2RkFHRXZSMWNTU0tNUzFGZnM5RgpEUzd6cjZlMDFWNlFVM1NUSkU3by9WY3F1TWxuVXJGRmQxTHgwK0l5TEZmMzI5Z012NlhpNlNHTUNzUTlYVDRiCkwzUG1zaUdqZmRzcERSRUJaS1lIOUhrMWxGZ2JuaTlKNWlCNExhWGRzOTlvdE9HY2cweVBiNmhHQWJYNzZZVFcKVUxZVWd3M1dMVlZGek9wWmQySTE0c1hWWXpxbDZLTDRrVlM1cndJREFRQUJBb0lCQUFVaVZiVjhaVDR2bVd1VQpxRkV2aEFPVzc1bTY1VUZuZXJGNFFabEZkRXhQUTNPWHlSMnNydW96WmEzMmNRN0puMHFkbjJ0dEtuYVVGUnV4CnVQeXZFYXhHWmtsb3hwOHdLVnNmQ3dEdno4SlNld0oweHRRNU1zMGd1cVdYY1RRa29FT1oxQ25EdHJTZ0dTVGgKdVN6Yk5BQUtqQlhobEQyd1NQUjdGb2JtcWdYUGNQQWtjK1JWR014bXpGUTBMOXg5bGZsZE1pSzNBR096UUJtMQpsNC9WSnBYUTEzaEkyU1dybGx0NEtYTUpTWnRnWllDdXgrS0FHUE1PK3BqbnFCZExUMllnbFRIc1VMN2pIU3FBCk91d3VFVCt3b3N6RVZiZFRwQkw3VVB0Z25FNFdibW1hOTR5M05xcGlVcXg3WXlkVWRQbHgrbGNkd0U5RmNRZG4KMVM5QzNWRUNnWUVBd2ZjamhQQkhHcGJldC85Q2liUVY2NHl2NmF3eENIVlNWWHVzekNwVlhRNy9zaEJmckZVZgpGUzQvenZhUkYyMVE3Sy9OT3ZTUWpJVUFSYnYySi9NY0VBamMwVThaWUplVFJyN1BjbkwrZ0FqT1ZGby9meHFqCkpWS0NQUHdiVkpSYUl2UTJERm8rdlQrdkwvRUg0VjdITDltRUZpMHFCS3VhcWUwaktDK1RSM2tDZ1lFQStraTEKYVljeGRiWmxoWkNpOTBkOHNhYW1WT004ZTJ4Wkl1czJmK3luUGQ2WjR2aDZ0YVFsVVNoSno1UVpZVlIrR3Y1OApNUzJxUDdQWmhEK3JiL2FaT2F6NFRGdjFac2JVSGhQdVlIVEU4WDUweDFVSlQ2Z3dCeVBNQXJWL08vZVJpeU5mCjF2SVRaWFhxMVNlUXVpeE9UbXdNemlFNlhYdVhxMEdLKzQ1VXVHY0NnWUJ2UUtlWTJwQ09DYmNaWmtudDNlTXMKeGZjb1dtR05ibTJaSm1USWZnZVVac1AyaURtNENPTFpMVHZnSThDNDVUU2piWHFUdEM4c3lpU0wydkdubkdPZQpOdGNoSHZONVdiZFp2cHdTRXk4eWxOcHp1NGZzQ1lWR0pQc2FxNmVwYmFYOW9vRlZ1SFYvNndVNnhFODJ6endFCmtBaVpCN0t3RnhXUkhia3FsWTh1VVFLQmdHNzlwVzJaNVlZbEQ5cHViTWFxTGhMK0swOER3b09kWmQ0Rlh4TFYKMk1pb0dhZzh0dllzUjl3NHVKclVPM2tkSmh0RWRjQzlWbjJQZlV1WGpLaEhQR1lHWGNwSEVZbTFiTVcxNHdWbApZeDBSRGlxRGZIQ1Z6azZzUWtHRlNWcEhqSVNlZUZieTNVVW1TTENrTGh0Um9KeEljRmxOQlB3RjNobTFKRFF3ClIwUExBb0dCQUpaclVxUDR4QkNSUEE2Tm41aFN6VklZcTJac1kxMnB0Y0F2WG8zTjY3MnVEZjh4QmRMTndac2YKMW1jOGVERmxqelZ0a2o2aWd2UzFFTkJMdTJkcnBRSmJOdDdHNS9tUGJHMnVBL21yOENuSFBFaGwreHVObVFrMQpoeEhoblZxekV2VkZPOHh3Tmh1TUx0YU1OSTYrQWNyS1c5WFE2UFVWbm1oUHZjamRzN3lYCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== --- # Source: cilium/templates/cilium-configmap.yaml apiVersion: v1 @@ -101,7 +101,7 @@ data: # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 # address. - enable-ipv4: "false" + enable-ipv4: "true" # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6 # address. @@ -177,7 +177,7 @@ data: # - vxlan (default) # - geneve - routing-mode: "native" + routing-mode: "tunnel" tunnel-protocol: "vxlan" tunnel-source-port-range: "0-0" service-no-backend-response: "reject" @@ -187,7 +187,7 @@ data: enable-l7-proxy: "true" enable-ipv4-masquerade: "true" enable-ipv4-big-tcp: "false" - enable-ipv6-big-tcp: "true" + enable-ipv6-big-tcp: "false" enable-ipv6-masquerade: "true" enable-tcx: "true" datapath-mode: "veth" @@ -201,8 +201,6 @@ data: direct-routing-skip-unreachable: "false" - ipv4-native-routing-cidr: 10.0.0.0/8 - ipv6-native-routing-cidr: fd00::/100 kube-proxy-replacement: "true" kube-proxy-replacement-healthz-bind-address: "" @@ -229,7 +227,7 @@ data: enable-well-known-identities: "false" enable-node-selector-labels: "false" synchronize-k8s-nodes: "true" - operator-api-serve-addr: "[::1]:9234" + operator-api-serve-addr: "127.0.0.1:9234" enable-hubble: "true" # UNIX domain socket for Hubble server to listen to. @@ -241,7 +239,6 @@ data: hubble-tls-cert-file: /var/lib/cilium/tls/hubble/server.crt hubble-tls-key-file: /var/lib/cilium/tls/hubble/server.key hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/client-ca.crt - hubble-prefer-ipv6: "true" ipam: "kubernetes" ipam-cilium-node-update-rate: "15s" @@ -318,7 +315,7 @@ metadata: data: # Keep the key name as bootstrap-config.json to avoid breaking changes bootstrap-config.json: | - {"admin":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/admin.sock"}}},"applicationLogConfig":{"logFormat":{"textFormat":"[%Y-%m-%d %T.%e][%t][%l][%n] [%g:%#] %v"}},"bootstrapExtensions":[{"name":"envoy.bootstrap.internal_listener","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener"}}],"dynamicResources":{"cdsConfig":{"apiConfigSource":{"apiType":"GRPC","grpcServices":[{"envoyGrpc":{"clusterName":"xds-grpc-cilium"}}],"setNodeOnFirstMessageOnly":true,"transportApiVersion":"V3"},"initialFetchTimeout":"30s","resourceApiVersion":"V3"},"ldsConfig":{"apiConfigSource":{"apiType":"GRPC","grpcServices":[{"envoyGrpc":{"clusterName":"xds-grpc-cilium"}}],"setNodeOnFirstMessageOnly":true,"transportApiVersion":"V3"},"initialFetchTimeout":"30s","resourceApiVersion":"V3"}},"node":{"cluster":"ingress-cluster","id":"host~127.0.0.1~no-id~localdomain"},"overloadManager":{"resourceMonitors":[{"name":"envoy.resource_monitors.global_downstream_max_connections","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig","max_active_downstream_connections":"50000"}}]},"staticResources":{"clusters":[{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"ingress-cluster","type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"egress-cluster-tls","transportSocket":{"name":"cilium.tls_wrapper","typedConfig":{"@type":"type.googleapis.com/cilium.UpstreamTlsWrapperContext"}},"type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"upstreamHttpProtocolOptions":{},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"egress-cluster","type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"ingress-cluster-tls","transportSocket":{"name":"cilium.tls_wrapper","typedConfig":{"@type":"type.googleapis.com/cilium.UpstreamTlsWrapperContext"}},"type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"upstreamHttpProtocolOptions":{},"useDownstreamProtocolConfig":{}}}},{"connectTimeout":"2s","loadAssignment":{"clusterName":"xds-grpc-cilium","endpoints":[{"lbEndpoints":[{"endpoint":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/xds.sock"}}}}]}]},"name":"xds-grpc-cilium","type":"STATIC","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","explicitHttpConfig":{"http2ProtocolOptions":{}}}}},{"connectTimeout":"2s","loadAssignment":{"clusterName":"/envoy-admin","endpoints":[{"lbEndpoints":[{"endpoint":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/admin.sock"}}}}]}]},"name":"/envoy-admin","type":"STATIC"}],"listeners":[{"address":{"socketAddress":{"address":"::","portValue":9964}},"filterChains":[{"filters":[{"name":"envoy.filters.network.http_connection_manager","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager","httpFilters":[{"name":"envoy.filters.http.router","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"}}],"internalAddressConfig":{"cidrRanges":[{"addressPrefix":"::1","prefixLen":128}]},"routeConfig":{"virtualHosts":[{"domains":["*"],"name":"prometheus_metrics_route","routes":[{"match":{"prefix":"/metrics"},"name":"prometheus_metrics_route","route":{"cluster":"/envoy-admin","prefixRewrite":"/stats/prometheus"}}]}]},"statPrefix":"envoy-prometheus-metrics-listener","streamIdleTimeout":"300s"}}]}],"name":"envoy-prometheus-metrics-listener"},{"address":{"socketAddress":{"address":"::1","portValue":9878}},"filterChains":[{"filters":[{"name":"envoy.filters.network.http_connection_manager","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager","httpFilters":[{"name":"envoy.filters.http.router","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"}}],"internalAddressConfig":{"cidrRanges":[{"addressPrefix":"::1","prefixLen":128}]},"routeConfig":{"virtual_hosts":[{"domains":["*"],"name":"health","routes":[{"match":{"prefix":"/healthz"},"name":"health","route":{"cluster":"/envoy-admin","prefixRewrite":"/ready"}}]}]},"statPrefix":"envoy-health-listener","streamIdleTimeout":"300s"}}]}],"name":"envoy-health-listener"}]}} + {"admin":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/admin.sock"}}},"applicationLogConfig":{"logFormat":{"textFormat":"[%Y-%m-%d %T.%e][%t][%l][%n] [%g:%#] %v"}},"bootstrapExtensions":[{"name":"envoy.bootstrap.internal_listener","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener"}}],"dynamicResources":{"cdsConfig":{"apiConfigSource":{"apiType":"GRPC","grpcServices":[{"envoyGrpc":{"clusterName":"xds-grpc-cilium"}}],"setNodeOnFirstMessageOnly":true,"transportApiVersion":"V3"},"initialFetchTimeout":"30s","resourceApiVersion":"V3"},"ldsConfig":{"apiConfigSource":{"apiType":"GRPC","grpcServices":[{"envoyGrpc":{"clusterName":"xds-grpc-cilium"}}],"setNodeOnFirstMessageOnly":true,"transportApiVersion":"V3"},"initialFetchTimeout":"30s","resourceApiVersion":"V3"}},"node":{"cluster":"ingress-cluster","id":"host~127.0.0.1~no-id~localdomain"},"overloadManager":{"resourceMonitors":[{"name":"envoy.resource_monitors.global_downstream_max_connections","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig","max_active_downstream_connections":"50000"}}]},"staticResources":{"clusters":[{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"ingress-cluster","type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"egress-cluster-tls","transportSocket":{"name":"cilium.tls_wrapper","typedConfig":{"@type":"type.googleapis.com/cilium.UpstreamTlsWrapperContext"}},"type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"upstreamHttpProtocolOptions":{},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"egress-cluster","type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"ingress-cluster-tls","transportSocket":{"name":"cilium.tls_wrapper","typedConfig":{"@type":"type.googleapis.com/cilium.UpstreamTlsWrapperContext"}},"type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"upstreamHttpProtocolOptions":{},"useDownstreamProtocolConfig":{}}}},{"connectTimeout":"2s","loadAssignment":{"clusterName":"xds-grpc-cilium","endpoints":[{"lbEndpoints":[{"endpoint":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/xds.sock"}}}}]}]},"name":"xds-grpc-cilium","type":"STATIC","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","explicitHttpConfig":{"http2ProtocolOptions":{}}}}},{"connectTimeout":"2s","loadAssignment":{"clusterName":"/envoy-admin","endpoints":[{"lbEndpoints":[{"endpoint":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/admin.sock"}}}}]}]},"name":"/envoy-admin","type":"STATIC"}],"listeners":[{"additionalAddresses":[{"address":{"socketAddress":{"address":"::","portValue":9964}}}],"address":{"socketAddress":{"address":"0.0.0.0","portValue":9964}},"filterChains":[{"filters":[{"name":"envoy.filters.network.http_connection_manager","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager","httpFilters":[{"name":"envoy.filters.http.router","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"}}],"internalAddressConfig":{"cidrRanges":[{"addressPrefix":"10.0.0.0","prefixLen":8},{"addressPrefix":"172.16.0.0","prefixLen":12},{"addressPrefix":"192.168.0.0","prefixLen":16},{"addressPrefix":"127.0.0.1","prefixLen":32},{"addressPrefix":"::1","prefixLen":128}]},"routeConfig":{"virtualHosts":[{"domains":["*"],"name":"prometheus_metrics_route","routes":[{"match":{"prefix":"/metrics"},"name":"prometheus_metrics_route","route":{"cluster":"/envoy-admin","prefixRewrite":"/stats/prometheus"}}]}]},"statPrefix":"envoy-prometheus-metrics-listener","streamIdleTimeout":"300s"}}]}],"name":"envoy-prometheus-metrics-listener"},{"additionalAddresses":[{"address":{"socketAddress":{"address":"::1","portValue":9878}}}],"address":{"socketAddress":{"address":"127.0.0.1","portValue":9878}},"filterChains":[{"filters":[{"name":"envoy.filters.network.http_connection_manager","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager","httpFilters":[{"name":"envoy.filters.http.router","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"}}],"internalAddressConfig":{"cidrRanges":[{"addressPrefix":"10.0.0.0","prefixLen":8},{"addressPrefix":"172.16.0.0","prefixLen":12},{"addressPrefix":"192.168.0.0","prefixLen":16},{"addressPrefix":"127.0.0.1","prefixLen":32},{"addressPrefix":"::1","prefixLen":128}]},"routeConfig":{"virtual_hosts":[{"domains":["*"],"name":"health","routes":[{"match":{"prefix":"/healthz"},"name":"health","route":{"cluster":"/envoy-admin","prefixRewrite":"/ready"}}]}]},"statPrefix":"envoy-health-listener","streamIdleTimeout":"300s"}}]}],"name":"envoy-health-listener"}]}} --- # Source: cilium/templates/cilium-agent/clusterrole.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -898,7 +895,7 @@ spec: - --config-dir=/tmp/cilium/config-map startupProbe: httpGet: - host: "::1" + host: "127.0.0.1" path: /healthz port: 9879 scheme: HTTP @@ -911,7 +908,7 @@ spec: initialDelaySeconds: 5 livenessProbe: httpGet: - host: "::1" + host: "127.0.0.1" path: /healthz port: 9879 scheme: HTTP @@ -926,7 +923,7 @@ spec: timeoutSeconds: 5 readinessProbe: httpGet: - host: "::1" + host: "127.0.0.1" path: /healthz port: 9879 scheme: HTTP @@ -1422,7 +1419,7 @@ spec: - '--log-level info' startupProbe: httpGet: - host: "::1" + host: "127.0.0.1" path: /healthz port: 9878 scheme: HTTP @@ -1432,7 +1429,7 @@ spec: initialDelaySeconds: 5 livenessProbe: httpGet: - host: "::1" + host: "127.0.0.1" path: /healthz port: 9878 scheme: HTTP @@ -1442,7 +1439,7 @@ spec: timeoutSeconds: 5 readinessProbe: httpGet: - host: "::1" + host: "127.0.0.1" path: /healthz port: 9878 scheme: HTTP @@ -1628,7 +1625,7 @@ spec: protocol: TCP livenessProbe: httpGet: - host: "::1" + host: "127.0.0.1" path: /healthz port: 9234 scheme: HTTP @@ -1637,7 +1634,7 @@ spec: timeoutSeconds: 3 readinessProbe: httpGet: - host: "::1" + host: "127.0.0.1" path: /healthz port: 9234 scheme: HTTP diff --git a/nix/kubernetes/roles/kube_apiserver/default.nix b/nix/kubernetes/roles/kube_apiserver/default.nix index 8cf4fb9..1c2edf7 100644 --- a/nix/kubernetes/roles/kube_apiserver/default.nix +++ b/nix/kubernetes/roles/kube_apiserver/default.nix @@ -84,7 +84,7 @@ in "--tls-cert-file=/.persist/keys/kube/kube-api-server.crt" "--tls-private-key-file=/.persist/keys/kube/kube-api-server.key" "--tls-min-version=VersionTLS13" - "--service-cluster-ip-range=fd00:3e42:e349::/112" + "--service-cluster-ip-range=fd00:3e42:e349::/112,10.197.0.0/16" "--v=2" # OLD: diff --git a/nix/kubernetes/roles/kube_controller_manager/default.nix b/nix/kubernetes/roles/kube_controller_manager/default.nix index 282d0a5..75dc659 100644 --- a/nix/kubernetes/roles/kube_controller_manager/default.nix +++ b/nix/kubernetes/roles/kube_controller_manager/default.nix @@ -40,7 +40,7 @@ in # "--cluster-cidr=10.200.0.0/16" # "--cluster-cidr=2620:11f:7001:7:ffff:ffff:0ac8:0000/16" "--allocate-node-cidrs=true" - "--cluster-cidr=fd49:0595:2bba::/48" + "--cluster-cidr=10.200.0.0/16,fd49:0595:2bba::/48" "--cluster-name=kubernetes" "--cluster-signing-cert-file=/.persist/keys/kube/ca.crt" "--cluster-signing-key-file=/.persist/keys/kube/ca.key" @@ -49,7 +49,7 @@ in "--service-account-private-key-file=/.persist/keys/kube/service-accounts.key" # "--service-cluster-ip-range=10.197.0.0/16" # "--service-cluster-ip-range=2620:11f:7001:7:ffff:ffff:0ac5:0000/16" - "--service-cluster-ip-range=fd00:3e42:e349::/112" + "--service-cluster-ip-range=10.197.0.0/16,fd00:3e42:e349::/112" "--use-service-account-credentials=true" "--v=2" ] diff --git a/nix/kubernetes/roles/kube_proxy/default.nix b/nix/kubernetes/roles/kube_proxy/default.nix index bd23e46..a7977e4 100644 --- a/nix/kubernetes/roles/kube_proxy/default.nix +++ b/nix/kubernetes/roles/kube_proxy/default.nix @@ -33,7 +33,7 @@ in mode = "iptables"; # clusterCIDR = "10.200.0.0/16"; # clusterCIDR = "2620:11f:7001:7:ffff:ffff:0ac8:0000/16"; - clusterCIDR = "fd49:0595:2bba::/48"; + clusterCIDR = "10.200.0.0/16,fd49:0595:2bba::/48"; }; description = '' kubelet-config.yaml @@ -57,7 +57,7 @@ in "${pkgs.kubernetes}/bin/kube-proxy" "--config=${config_file}" "--nodeport-addresses=primary" - "--cluster-cidr=fd49:0595:2bba::/48" + "--cluster-cidr=10.200.0.0/16,fd49:0595:2bba::/48" ] ); Restart = "on-failure"; From 37efcdb367a1b927fe727c1ba7b76b7c988682fb Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 29 Dec 2025 18:35:20 -0500 Subject: [PATCH 40/81] Allow pods to directly speak to the public internet on their own public IPv6 addresses. --- nix/kubernetes/README.org | 4 ++-- .../bootstrap-script/files/manifests/cilium.yaml | 12 ++++++------ nix/kubernetes/keys/package/k8s-ca/files/ca.conf | 1 + .../roles/kube_controller_manager/default.nix | 6 ++++-- nix/kubernetes/roles/kube_proxy/default.nix | 4 ++-- 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/nix/kubernetes/README.org b/nix/kubernetes/README.org index 8e103f3..bace10e 100644 --- a/nix/kubernetes/README.org +++ b/nix/kubernetes/README.org @@ -23,8 +23,8 @@ --set k8sServiceHost="2620:11f:7001:7:ffff:ffff:ad7:1dd" \ --set k8sServicePort=6443 \ --set ipv6.enabled=true \ - --set ipv4.enabled=true - # --set enableIPv6Masquerade=false + --set ipv4.enabled=true \ + --set enableIPv6Masquerade=false # --set enableIPv4BIGTCP=true \ # --set enableIPv6BIGTCP=true # --set routingMode=native \ diff --git a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/cilium.yaml b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/cilium.yaml index 8425169..585612e 100644 --- a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/cilium.yaml +++ b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/cilium.yaml @@ -36,8 +36,8 @@ metadata: name: cilium-ca namespace: kube-system data: - ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lSQVBMcVRXajlHdHFqRXNMRTgxemtUUkl3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkxTVRJeU9USXlNVGt6TkZvWERUSTRNVEl5T0RJeQpNVGt6TkZvd0ZERVNNQkFHQTFVRUF4TUpRMmxzYVhWdElFTkJNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFNSUlCQ2dLQ0FRRUE0UVIvSy9WOUEvNnNPdzU3N3ZodTVTcEw4anJHMFFWR1VPeXVqM2V5LytVRno5MWYKZlBiVDhzK3FCY1BreFZ4YVV4d05McVJGcFkrLzhKUm5NWi9Hb21jdEpiY2VQLzBuRnhMekRUdFhqWHFWTmMrSgpZSUx2cWFEYVhyOStBZFhmdUNuVFRtenlWVTBCekRGSFU3VTYyMldOL05FbEhZK3YvaTY2L3RWWFh5VWNpcXNzCit5Rjh0Vkd5cVNFTG5Wd0lDKzVvL1BhV29ybzEwZ0xCLzNBSGdNRlBDTG14QnF0Z3RKTzVaSWF5Mzc5VHJhamgKMytpQ2s0UGNIT2RhbFVDMGFiMHlUYjlvS0xjTHZoVFBMUjFpdGphVmRhaU9VYUtONDUyWUQ4UnJ1NFlyZ05rUwphMmtlcnNlbjZKNTRDZHFQL3VHcXJwaDV5TTFtSFNoa1dLaEdsd0lEQVFBQm8yRXdYekFPQmdOVkhROEJBZjhFCkJBTUNBcVF3SFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGREIvYzlIRnFSY0prREpHeVV3aWlQZ0k2TDhmTUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQlZSc3NtVlFIWkphVi9uNk8ydFJNQ2ZOckcwR01xRkpRNFlBRjg4ZW0wT0hNWWszRjNocGZmCkhUZHVrbTJ4OS9DZlNHNkdra3pCbFVhK1cxOVJNUFhJcXdybmwrOHEzVk4ySDA3THZFajVzZHhOK1hnSVR4dG4Kb3B0MndidXpJZWNjR2FnNEd1aHJOcHBXVndHODZURTVMelU0MXE0dlNUS01CUWtIVG5aVnhaTVV3RkxIUzNEVQpPalA0b3prYjc0OEZZNkJvbjR2dGJVYjk2U0I2V2FBRnhGZW8vanRSN0RUSjVxNjZzOFp1c05tMHBKZURzVFQ2Ck9xdzUvWDhJeVRPN2tZUmV4RTIvUEZwRWZOUXVyRTBlNGlvTG9rSC9acmtNZmtncTdISU84RDRUUFJVL0s0YXIKMTdwZmhBWDVieVlScy9FZTRrV0dQclByb3EzcGpFZ1kKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= - ca.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBNFFSL0svVjlBLzZzT3c1Nzd2aHU1U3BMOGpyRzBRVkdVT3l1ajNleS8rVUZ6OTFmCmZQYlQ4cytxQmNQa3hWeGFVeHdOTHFSRnBZKy84SlJuTVovR29tY3RKYmNlUC8wbkZ4THpEVHRYalhxVk5jK0oKWUlMdnFhRGFYcjkrQWRYZnVDblRUbXp5VlUwQnpERkhVN1U2MjJXTi9ORWxIWSt2L2k2Ni90VlhYeVVjaXFzcworeUY4dFZHeXFTRUxuVndJQys1by9QYVdvcm8xMGdMQi8zQUhnTUZQQ0xteEJxdGd0Sk81WklheTM3OVRyYWpoCjMraUNrNFBjSE9kYWxVQzBhYjB5VGI5b0tMY0x2aFRQTFIxaXRqYVZkYWlPVWFLTjQ1MllEOFJydTRZcmdOa1MKYTJrZXJzZW42SjU0Q2RxUC91R3FycGg1eU0xbUhTaGtXS2hHbHdJREFRQUJBb0lCQUE5Zi9VaUZwNXNrUmFScgpZdkI3TFNpZmNUMEY0eHZaSG1yZElUaFFWM3pBcTFyK3AwMmtsK3JaWWFhdk1leUNXUEdnMHczQ3o3ZDJVUWtoCk9zUGJSUGxIejMvcU9UanFGVWV3VmNjcnJOblA3RzRXMWk0d0JDdzAvM2JGNHRoQlF0NFVqWW1vVEE2a0NtRm4KWkpaRnBkWGo1SVhIeko5dWQvb0lPMks2TlJRZEZpL01YaDdxdDhpaWtSUWllcVZQTEZlZExYSlBVSHk3ZEw5eApMaHpzSTNWak9Ed1VYZGJtSXlMYXhpUkpwY3NWb3RqcitGWEdEaVFvdnFJYnk0SWlId2ZZeFMzR3RkUUI1M2J3Cm5FOW9Ga1o1WHFCRys0dmxtQ0N2cVQvaFZkN3ZRU3U4S3p4dVd1MFh3WWJUelppZTBMd0hESndxTDRNdDVUbEEKVDlXQ2I5RUNnWUVBL3pCR3Y2cEFCUUZuL1l5VDRIRzNxcXNXKzhDUjFHZEhDUHI1YVZxYXpXSldOc3VwNjExKwpsUnZqT2RyVThtUkRFS2pGRzJMS2J4QmNlRkFIb2hkZGtoNnkyK1NsMFM0VnBKc3dPMGtXMkhtOWtzeFFYNGVRCmM4Ui8xVWFpVmNENC9KYnl5d012NjlQU3lSM3p6MEl2QTJZa2p5aGh0U0tVell5NlEramlZbE1DZ1lFQTRidXAKUXlxazhZSW9DWkQ0dHdRMHlzRHBibWlaTVpmem11VWFvdlQzOXEvdUpXMWtSWWdFeG9KSTJpVnUzRGx1YmxJaQplWEMvTG52SklZenZhQjM2MlhaaXFVWUJRamJtdUp3eks0QVNrRU9VOWNMNmI4dnpBRVB4Mi9kS3NJdnNhM1NnCnRPaE4wT1ozTGNQWVFXVXo0Q0R0RjY2SGdpUkJDS201cENtclNpMENnWUJES1JGMW4vajVXRVZtRlhsVnVKZjkKOHNrNXVKU29zZ1dmOTIxc3JNdlJBZkJ1dTJzYzVwNUozKzBOYk9wZVlNVEZ2YVVpYi8yVy9WZFkwcmN2a2JhdQpuaVcwZVppcVZOYWRXN3AxMythRFVvYngrNU9ya0tJVDFjTk8vaWY3S2E1ZHk5eGFVWnhyRkhTRk9ielE3em93ClN0R1VXNnhiWDU2SitsK0xQOTlVd1FLQmdCYVdqc04zMnZXSHpyWFdXZTBHY0xueFArcWFjT256aUo3eFdnOEEKY0dleEJ5V2Jvck1LZjEvVjNiQm1tb2RTLytmdU5DRHUxVkt6ZEZiMUlXZkx5RFJHa216WjRucGJ3QTZ0SXJteApvU25xZVZkMllWVWFsWUJyVENKMHhuYVFLZUkrMDI0RzZQS0VmVnlKQW5UWXlhNGQzVnZIVFN3S3NLOUxQSlplCnpSTXBBb0dBYk84NnRRMjNIWnB1Q1Z1eWprTkdoSklrTSsyZ0xoZURFVnBxVmsxU1kvL1RLYzNQSDJQY1lHNjcKM3pmTTJkTE01d2ZNenBVOVNsamNnQXZ3TjZBalJvdm1kOGJDZzl2RzEwZmRjSXFiRko1WU4xellRM0JPTDZudgpuNjBXVWQ5V2FXeDdDSGxSWjN6N1NBOFZuMlJ6NHFkdGRXdnlDblZtTnN0VmM1WnIwWk09Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== + ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lSQU53OW1DamlKdysveDFOQTJ6Sm5zc0F3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkxTVRJeU9USXlOVEF3TVZvWERUSTRNVEl5T0RJeQpOVEF3TVZvd0ZERVNNQkFHQTFVRUF4TUpRMmxzYVhWdElFTkJNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFNSUlCQ2dLQ0FRRUF6SGM0aThFL2JUMmlrOVB6Z0ZQeWVoTHIvUVZEbXl4SU41ZVBIUVFDSjVudXRZOHAKRTFjSGdlVzI4SGJhclZ3U3ZSQXFObUdsV2FmZDVtckgyN1E2YThpVm1DcDZYeGVsNFMrYVlRWHkrZFgyVXdOeApLZTlRNVA4WXhUMWNGWkV0QW5tclBzS0dyTm9YdDdQNC8wdHBSVTVvWktnMEJQK3RnamxiUHd3TUtueFROWjMvClBrZzI0M0Z2bEVDWHV2VXNENy9wcFhkUzRsNEVzWGpkbW1IaXpGelQ2S3A1VWJicUtsaWh5aHhkSzhlZXB4emMKM3dyUEE1ZVZXNjdCUlc4dTU1ZWFSMnVpOGRDZkgwZ2dYbXVHL0hiSTJMREtOQ09RanRFRk5XVWhUM08zb0ZuVAovWS9ON2lJc1VobE9IcnZJNUNKYWhKV2NuMVRHd1FuWWthcStxd0lEQVFBQm8yRXdYekFPQmdOVkhROEJBZjhFCkJBTUNBcVF3SFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGRFc0WXJXYkN4c0ZlQU9nT1Vsb1A1bUllVVgrTUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQ0VOdnBiblI4TUtaMWpBVWRlMU55elJueUhjemR2T0Jla1F1YVFkS01hOWhFMk12Y0hTQ05qClBwNUNOenNXWk5FMW5RekF6RFNDcEV5Z0lpRm5mNitYaTBBZlZncXdpcDQ2SytkbUd1d1B5dnBRNWorU1YxNHoKSmlFOElJYTdYMzQrWnlrM2ZZUjd0elpobllqQ21UcXJYbnZSbzBGZHNuU0ZvRUU4T3RRMjU5dllJeXNuRXVPVwpOTEwvSTVQeGpNQlQ1WWNkaXdtbGR2RGxwZzk5YkRUQ0w0UFJ1MldKV2VFUmhzdGIzbWdRSk9Mb0RSM1VWWmowCjBFOGxQTjJiTjZMbVZjYjB1QlVyVjJQOHR6ZmxhVnF4SzQ5SlZFQmpFVHFaSzY3TTVYY3R4WG55SHdPVEpRckIKRGhVbWpkSGNZWCtYL0t5MDNOa1dhc2ZJeWdnV3IvR3gKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + ca.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBekhjNGk4RS9iVDJpazlQemdGUHllaExyL1FWRG15eElONWVQSFFRQ0o1bnV0WThwCkUxY0hnZVcyOEhiYXJWd1N2UkFxTm1HbFdhZmQ1bXJIMjdRNmE4aVZtQ3A2WHhlbDRTK2FZUVh5K2RYMlV3TngKS2U5UTVQOFl4VDFjRlpFdEFubXJQc0tHck5vWHQ3UDQvMHRwUlU1b1pLZzBCUCt0Z2psYlB3d01LbnhUTlozLwpQa2cyNDNGdmxFQ1h1dlVzRDcvcHBYZFM0bDRFc1hqZG1tSGl6RnpUNktwNVViYnFLbGloeWh4ZEs4ZWVweHpjCjN3clBBNWVWVzY3QlJXOHU1NWVhUjJ1aThkQ2ZIMGdnWG11Ry9IYkkyTERLTkNPUWp0RUZOV1VoVDNPM29GblQKL1kvTjdpSXNVaGxPSHJ2STVDSmFoSldjbjFUR3dRbllrYXErcXdJREFRQUJBb0lCQUEySWtYNEVWWVRZUU5VQwpwYnhzQ09ZL1U4NlVrSG00TGFFODcxSTZmQjhyTXNBWmNMZ2xhNHRjMGM4dlhzVTBNZlF1aUNndENZMSs3V1Q0CjB2amVBRmRiYyt5SHhxMStFVndCN3NIV3ROTXMzZTdrL3JkQWt1WUZZL1dmN3RDT0VDREI5UzJiUmJ1ZmZIbUwKblF3REpJYkpiRDRReTRaZ215bVpUeVlmVDVkb3hTSEJVMU9GMkFPTlg1VkYya1oySDI5dVNsMlgydm9jMlZidQowUlMvZFp6VExkcTRocUFPQ1VsbER6Nzk1U29aVU4zdmsranBSOGdxN1J2WTV4MzFDdmZQTENKNFNGQ25WSWtEClZvcnNCRW51V3ZaN0tBc051Tms3ZGMvYys0STk3d3cxNEt2NWVXV3p2ak05c1RFZWF1M3liRlluTmFTMUwzbjUKR3ZzeERERUNnWUVBMGJWNEJwS0xhM2g1c3BpbHZ0UlpSMVV3WndpZG13REMxZVB6dEZHSmx0SUg1b1I0T1BaVApzWjR2ak9WU21MUndFQ1VGdkNIaWZpV0ljTjd0NG5oQ1VCNmpJQXh5aS8wU1BSU1pkN2dtMXdYNGZyWVRVTkJtCk1DZ1krZlNLbjN2R3o2ODNlbUlFbDYyOFl5amt0aWYzQXdjeHVwdk9lSjVqQVBaekNoTG8ycE1DZ1lFQStabDIKMVp1Ym10RUI5T3pJZmlhRUV3R1k5TXdkKzNKL3daUU9LUWd6SGZBRU1DQnpGcnZZMWNvWkFpeWt3NWtoSTEwWQpZRlpFQlVhdFF3L3JpRFYvR1pDRkJyRHBYTFVjeFdMcTZoTGRtYmVZZHdXVXUvakdsSTUrYnhpNkVhUnlDcys4ClpUZVFNSDNLWVFJZ2RmQ3J3N0VTSEp1OU5ualFpMmhOZkt5STRva0NnWUVBaExrN2U3MW42OTdWOXdqUmJkTnIKcGMvdTBHY05pTHFVbFZhdU9TT1oxeENhMjNSZnVuMThQdUFFN0VGL1l4SmdFbmU4QjNQU3ExQUo2SlhQTFJRNwp0QVdQN0lxMFBKRXc1K09QdGN1aEdWbTRDa29tTTNHU0cweGxjbDBwRndMNXN4d01HckxLZ1V6OS9DdzNoR29LCjFhbkorWGIrMkN3Mk1MZkoyMGhZZzIwQ2dZRUE3VWhlZDMyQjlURGpPbE5yMnJtRTc0aWlQMzVZdG5WSVRPQVMKZ2lHQWJ1S0JLTHVBamNrd3Z6VnNodXVvQ3libElQaGN0eURyYzFTWVhGdWpIdzgwY0RvNnJIeThyTnlrcUdWOApTK0I1ZUt2WUxyWklpbFpiZWxqb2kzY25WS1JQb2tXUXBXeW9EK0ZWNXRrZHdPRjJlUWc1M3FhVHYxZ2xjWkpMCnI4MWFLSGtDZ1lBSXVxaDZzOU9RRGpjckVMeklJZENiZUxxLzhEdWUzcEZjbmFIWjgxemcwY0oxNnM1MU53aHQKaUQxWC9oRS9sdkp1VTFjTEFpNlJLZEk3UXIwL0x2SkFDSTlCakp6VVVYS1hQZ3JJZU44UUxCOWNhK2lka2FheApIWmNzdnBSN1g1b2cvTVpldWVTUk1OaU9wdVJSZ0xlYlh5R29lMkhmbE05STUvazJERUU5bmc9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo= --- # Source: cilium/templates/hubble/tls-helm/server-secret.yaml apiVersion: v1 @@ -47,9 +47,9 @@ metadata: namespace: kube-system type: kubernetes.io/tls data: - ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lSQVBMcVRXajlHdHFqRXNMRTgxemtUUkl3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkxTVRJeU9USXlNVGt6TkZvWERUSTRNVEl5T0RJeQpNVGt6TkZvd0ZERVNNQkFHQTFVRUF4TUpRMmxzYVhWdElFTkJNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFNSUlCQ2dLQ0FRRUE0UVIvSy9WOUEvNnNPdzU3N3ZodTVTcEw4anJHMFFWR1VPeXVqM2V5LytVRno5MWYKZlBiVDhzK3FCY1BreFZ4YVV4d05McVJGcFkrLzhKUm5NWi9Hb21jdEpiY2VQLzBuRnhMekRUdFhqWHFWTmMrSgpZSUx2cWFEYVhyOStBZFhmdUNuVFRtenlWVTBCekRGSFU3VTYyMldOL05FbEhZK3YvaTY2L3RWWFh5VWNpcXNzCit5Rjh0Vkd5cVNFTG5Wd0lDKzVvL1BhV29ybzEwZ0xCLzNBSGdNRlBDTG14QnF0Z3RKTzVaSWF5Mzc5VHJhamgKMytpQ2s0UGNIT2RhbFVDMGFiMHlUYjlvS0xjTHZoVFBMUjFpdGphVmRhaU9VYUtONDUyWUQ4UnJ1NFlyZ05rUwphMmtlcnNlbjZKNTRDZHFQL3VHcXJwaDV5TTFtSFNoa1dLaEdsd0lEQVFBQm8yRXdYekFPQmdOVkhROEJBZjhFCkJBTUNBcVF3SFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGREIvYzlIRnFSY0prREpHeVV3aWlQZ0k2TDhmTUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQlZSc3NtVlFIWkphVi9uNk8ydFJNQ2ZOckcwR01xRkpRNFlBRjg4ZW0wT0hNWWszRjNocGZmCkhUZHVrbTJ4OS9DZlNHNkdra3pCbFVhK1cxOVJNUFhJcXdybmwrOHEzVk4ySDA3THZFajVzZHhOK1hnSVR4dG4Kb3B0MndidXpJZWNjR2FnNEd1aHJOcHBXVndHODZURTVMelU0MXE0dlNUS01CUWtIVG5aVnhaTVV3RkxIUzNEVQpPalA0b3prYjc0OEZZNkJvbjR2dGJVYjk2U0I2V2FBRnhGZW8vanRSN0RUSjVxNjZzOFp1c05tMHBKZURzVFQ2Ck9xdzUvWDhJeVRPN2tZUmV4RTIvUEZwRWZOUXVyRTBlNGlvTG9rSC9acmtNZmtncTdISU84RDRUUFJVL0s0YXIKMTdwZmhBWDVieVlScy9FZTRrV0dQclByb3EzcGpFZ1kKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= - tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURWekNDQWorZ0F3SUJBZ0lSQVB0VkxoVHZtMkFJbEFVYTRwaFB6Ykl3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkxTVRJeU9USXlNVGt6TkZvWERUSTJNVEl5T1RJeQpNVGt6TkZvd0tqRW9NQ1lHQTFVRUF3d2ZLaTVrWldaaGRXeDBMbWgxWW1Kc1pTMW5jbkJqTG1OcGJHbDFiUzVwCmJ6Q0NBU0l3RFFZSktvWklodmNOQVFFQkJRQURnZ0VQQURDQ0FRb0NnZ0VCQUwyaWI2V0E4bUNZcitjWlZKTEIKbXBZd0VwQlo0bEpZYU1tbHZTMnlPQ09HbEE3bEh6c1QyZkVLT2dFVFZoNnZkSkJwVkVRdk5pb0VGejZ3YUd1OApuL1dmVVV4STZldUhkZmNjdXNwK1pLRld6dGJ5R0JnK01DUG42OEE5eFN1WWorMXpEWTA3WGVEem0xYU1jTGZGClFleER1OFQ5UnJ4UUJoTDBkWEVraWpFdFJYN1BSUTB1ODYrbnROVmVrRk4wa3lSTzZQMVhLcmpKWjFLeFJYZFMKOGRQaU1peFg5OXZZREwrbDR1a2hqQXJFUFYwK0d5OXo1cklobzMzYktRMFJBV1NtQi9SNU5aUllHNTR2U2VZZwplQzJsM2JQZmFMVGhuSU5NajIrb1JnRzErK21FMWxDMkZJTU4xaTFWUmN6cVdYZGlOZUxGMVdNNnBlaWkrSkZVCnVhOENBd0VBQWFPQmpUQ0JpakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdIUVlEVlIwbEJCWXdGQVlJS3dZQkJRVUgKQXdFR0NDc0dBUVVGQndNQ01Bd0dBMVVkRXdFQi93UUNNQUF3SHdZRFZSMGpCQmd3Rm9BVU1IOXowY1dwRndtUQpNa2JKVENLSStBam92eDh3S2dZRFZSMFJCQ013SVlJZktpNWtaV1poZFd4MExtaDFZbUpzWlMxbmNuQmpMbU5wCmJHbDFiUzVwYnpBTkJna3Foa2lHOXcwQkFRc0ZBQU9DQVFFQVpHNk9zMng2ZmdKQ01qSUNEUS9UaDFQSWhVakwKQ3FJQzlzWVZRRHM0dWpRMXJkK0pMMHBLQWVVa04ySmMxWktwanVSb2YwSGE2cmpzZVArTVZLMithemx6Qm9xTApDWVh1ZHJRb0h4U1RjVU9na01oWmtqM2dnUkthODBraEh1UjJOY21KbnZOTnVGdU4vVFBTd0JBUXpVczFLQkFpClJ0eUlOWjdERU9OVWd5Y0dwRGg5NkFKK2JUWlBGT1U1d28xS1hKajRCaVV0c1dxaFd3T3ozTWhkd1NsVC9yVTIKR1hmbXdibDY1THpneEZGaWt0TUpoUTdRdkRzdDVyM3prSXVVZWRPZ0U0RnRUMTRoZVgvMnlveitSWkxqaWFJQgowMWJFVEtGc1B6bFVQMnVLUzFaUnFqWlN0SjFZMXEycDYzRHJkM0hMNllWMytwa3lONG8waWhYM1NBPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= - tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBdmFKdnBZRHlZSml2NXhsVWtzR2FsakFTa0ZuaVVsaG95YVc5TGJJNEk0YVVEdVVmCk94UFo4UW82QVJOV0hxOTBrR2xVUkM4MktnUVhQckJvYTd5ZjlaOVJURWpwNjRkMTl4eTZ5bjVrb1ZiTzF2SVkKR0Q0d0krZnJ3RDNGSzVpUDdYTU5qVHRkNFBPYlZveHd0OFZCN0VPN3hQMUd2RkFHRXZSMWNTU0tNUzFGZnM5RgpEUzd6cjZlMDFWNlFVM1NUSkU3by9WY3F1TWxuVXJGRmQxTHgwK0l5TEZmMzI5Z012NlhpNlNHTUNzUTlYVDRiCkwzUG1zaUdqZmRzcERSRUJaS1lIOUhrMWxGZ2JuaTlKNWlCNExhWGRzOTlvdE9HY2cweVBiNmhHQWJYNzZZVFcKVUxZVWd3M1dMVlZGek9wWmQySTE0c1hWWXpxbDZLTDRrVlM1cndJREFRQUJBb0lCQUFVaVZiVjhaVDR2bVd1VQpxRkV2aEFPVzc1bTY1VUZuZXJGNFFabEZkRXhQUTNPWHlSMnNydW96WmEzMmNRN0puMHFkbjJ0dEtuYVVGUnV4CnVQeXZFYXhHWmtsb3hwOHdLVnNmQ3dEdno4SlNld0oweHRRNU1zMGd1cVdYY1RRa29FT1oxQ25EdHJTZ0dTVGgKdVN6Yk5BQUtqQlhobEQyd1NQUjdGb2JtcWdYUGNQQWtjK1JWR014bXpGUTBMOXg5bGZsZE1pSzNBR096UUJtMQpsNC9WSnBYUTEzaEkyU1dybGx0NEtYTUpTWnRnWllDdXgrS0FHUE1PK3BqbnFCZExUMllnbFRIc1VMN2pIU3FBCk91d3VFVCt3b3N6RVZiZFRwQkw3VVB0Z25FNFdibW1hOTR5M05xcGlVcXg3WXlkVWRQbHgrbGNkd0U5RmNRZG4KMVM5QzNWRUNnWUVBd2ZjamhQQkhHcGJldC85Q2liUVY2NHl2NmF3eENIVlNWWHVzekNwVlhRNy9zaEJmckZVZgpGUzQvenZhUkYyMVE3Sy9OT3ZTUWpJVUFSYnYySi9NY0VBamMwVThaWUplVFJyN1BjbkwrZ0FqT1ZGby9meHFqCkpWS0NQUHdiVkpSYUl2UTJERm8rdlQrdkwvRUg0VjdITDltRUZpMHFCS3VhcWUwaktDK1RSM2tDZ1lFQStraTEKYVljeGRiWmxoWkNpOTBkOHNhYW1WT004ZTJ4Wkl1czJmK3luUGQ2WjR2aDZ0YVFsVVNoSno1UVpZVlIrR3Y1OApNUzJxUDdQWmhEK3JiL2FaT2F6NFRGdjFac2JVSGhQdVlIVEU4WDUweDFVSlQ2Z3dCeVBNQXJWL08vZVJpeU5mCjF2SVRaWFhxMVNlUXVpeE9UbXdNemlFNlhYdVhxMEdLKzQ1VXVHY0NnWUJ2UUtlWTJwQ09DYmNaWmtudDNlTXMKeGZjb1dtR05ibTJaSm1USWZnZVVac1AyaURtNENPTFpMVHZnSThDNDVUU2piWHFUdEM4c3lpU0wydkdubkdPZQpOdGNoSHZONVdiZFp2cHdTRXk4eWxOcHp1NGZzQ1lWR0pQc2FxNmVwYmFYOW9vRlZ1SFYvNndVNnhFODJ6endFCmtBaVpCN0t3RnhXUkhia3FsWTh1VVFLQmdHNzlwVzJaNVlZbEQ5cHViTWFxTGhMK0swOER3b09kWmQ0Rlh4TFYKMk1pb0dhZzh0dllzUjl3NHVKclVPM2tkSmh0RWRjQzlWbjJQZlV1WGpLaEhQR1lHWGNwSEVZbTFiTVcxNHdWbApZeDBSRGlxRGZIQ1Z6azZzUWtHRlNWcEhqSVNlZUZieTNVVW1TTENrTGh0Um9KeEljRmxOQlB3RjNobTFKRFF3ClIwUExBb0dCQUpaclVxUDR4QkNSUEE2Tm41aFN6VklZcTJac1kxMnB0Y0F2WG8zTjY3MnVEZjh4QmRMTndac2YKMW1jOGVERmxqelZ0a2o2aWd2UzFFTkJMdTJkcnBRSmJOdDdHNS9tUGJHMnVBL21yOENuSFBFaGwreHVObVFrMQpoeEhoblZxekV2VkZPOHh3Tmh1TUx0YU1OSTYrQWNyS1c5WFE2UFVWbm1oUHZjamRzN3lYCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== + ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lSQU53OW1DamlKdysveDFOQTJ6Sm5zc0F3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkxTVRJeU9USXlOVEF3TVZvWERUSTRNVEl5T0RJeQpOVEF3TVZvd0ZERVNNQkFHQTFVRUF4TUpRMmxzYVhWdElFTkJNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFNSUlCQ2dLQ0FRRUF6SGM0aThFL2JUMmlrOVB6Z0ZQeWVoTHIvUVZEbXl4SU41ZVBIUVFDSjVudXRZOHAKRTFjSGdlVzI4SGJhclZ3U3ZSQXFObUdsV2FmZDVtckgyN1E2YThpVm1DcDZYeGVsNFMrYVlRWHkrZFgyVXdOeApLZTlRNVA4WXhUMWNGWkV0QW5tclBzS0dyTm9YdDdQNC8wdHBSVTVvWktnMEJQK3RnamxiUHd3TUtueFROWjMvClBrZzI0M0Z2bEVDWHV2VXNENy9wcFhkUzRsNEVzWGpkbW1IaXpGelQ2S3A1VWJicUtsaWh5aHhkSzhlZXB4emMKM3dyUEE1ZVZXNjdCUlc4dTU1ZWFSMnVpOGRDZkgwZ2dYbXVHL0hiSTJMREtOQ09RanRFRk5XVWhUM08zb0ZuVAovWS9ON2lJc1VobE9IcnZJNUNKYWhKV2NuMVRHd1FuWWthcStxd0lEQVFBQm8yRXdYekFPQmdOVkhROEJBZjhFCkJBTUNBcVF3SFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGRFc0WXJXYkN4c0ZlQU9nT1Vsb1A1bUllVVgrTUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQ0VOdnBiblI4TUtaMWpBVWRlMU55elJueUhjemR2T0Jla1F1YVFkS01hOWhFMk12Y0hTQ05qClBwNUNOenNXWk5FMW5RekF6RFNDcEV5Z0lpRm5mNitYaTBBZlZncXdpcDQ2SytkbUd1d1B5dnBRNWorU1YxNHoKSmlFOElJYTdYMzQrWnlrM2ZZUjd0elpobllqQ21UcXJYbnZSbzBGZHNuU0ZvRUU4T3RRMjU5dllJeXNuRXVPVwpOTEwvSTVQeGpNQlQ1WWNkaXdtbGR2RGxwZzk5YkRUQ0w0UFJ1MldKV2VFUmhzdGIzbWdRSk9Mb0RSM1VWWmowCjBFOGxQTjJiTjZMbVZjYjB1QlVyVjJQOHR6ZmxhVnF4SzQ5SlZFQmpFVHFaSzY3TTVYY3R4WG55SHdPVEpRckIKRGhVbWpkSGNZWCtYL0t5MDNOa1dhc2ZJeWdnV3IvR3gKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURWekNDQWorZ0F3SUJBZ0lSQVBsUTMyUWVldjRpclVZaXJMRmVSU0l3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkxTVRJeU9USXlOVEF3TVZvWERUSTJNVEl5T1RJeQpOVEF3TVZvd0tqRW9NQ1lHQTFVRUF3d2ZLaTVrWldaaGRXeDBMbWgxWW1Kc1pTMW5jbkJqTG1OcGJHbDFiUzVwCmJ6Q0NBU0l3RFFZSktvWklodmNOQVFFQkJRQURnZ0VQQURDQ0FRb0NnZ0VCQU5IMWI3VXQ0eDdaSitUUUxBWjAKdXg0VlY4V2Z5RXRGY0hUbHdxeGZGcmMzZHBrNTQzWnpwN0NSbUg3Mkw2NlQ3VW1iRElydHZsdHcxdmh5MSs2eQpaY0FqNlZIcUVDdHhEVzd0MlNjTlRzdzFkME1aV3hiVm5QeVdiWm13aW0xV1Z4ODJ4THJMUWkrY3Y4RFhaUGpBCkZldm9rV21lNURMeTBJVmIwc0thbng0cGtQdTJsZlN0U0lveVpqOXBCYWFST2NwYXlVemNCZFU2RHBGdDlMRDIKdTIyTnVzbm00ZUh2bm9wRm5FMVI2UnprbVRGNHh4R2IyeTVweHdMTTV6RzhkZERJL3hhd3hNcHdENVVFYlRXWQorUjhaVGh5a242RHpmaitwcGI5dWZocmNOalZXRzNhY0o1elg4cElKU25ya1VTQmpLdm1ieVh3WEt4T0c4eGYzClltMENBd0VBQWFPQmpUQ0JpakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdIUVlEVlIwbEJCWXdGQVlJS3dZQkJRVUgKQXdFR0NDc0dBUVVGQndNQ01Bd0dBMVVkRXdFQi93UUNNQUF3SHdZRFZSMGpCQmd3Rm9BVU5iaGl0WnNMR3dWNApBNkE1U1dnL21ZaDVSZjR3S2dZRFZSMFJCQ013SVlJZktpNWtaV1poZFd4MExtaDFZbUpzWlMxbmNuQmpMbU5wCmJHbDFiUzVwYnpBTkJna3Foa2lHOXcwQkFRc0ZBQU9DQVFFQVRrb24wcTVDdzJPT2JCcy9Nb0g2a041WXJidTAKeTR1c1hVc0x4V3oyNDFadisvY3JZQ0x4d092bGZNTGNnU0crL0xtTkIrMGpGc0RnMnRhQk9zejNHbzRtUzRrQwprV1QxUENSQ2o5MFdZU2hzRHNMdUV5MlI1bndTYmZCL3NHc3MxZnNwcDNhTldXSUNjRjJlZmh1TXRsREdMcEtRClFKQytlbHNYSFNoNkpWMWROTXhFYkhQVk9kQzBobUswNHRJMjJ0Tytsc0Iwd090S3VCbUJUb2VtR1ZScUtRbmEKem81NUtLc2cyQ1FwODhSTUJ4OUk2aXJjMThqSmFJeFhaRTE5RGMzWkRiNk42ZzVGc2hNeHhUK3F5cEt3VUVWego1SGRQNks5NjhyS3RjZnZXSXpieDVsYnIvY3hrZWZ0ZTE5SThXbmlxTzl4SlkxN0duVXV1aU5DREx3PT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBMGZWdnRTM2pIdGtuNU5Bc0JuUzdIaFZYeFovSVMwVndkT1hDckY4V3R6ZDJtVG5qCmRuT25zSkdZZnZZdnJwUHRTWnNNaXUyK1czRFcrSExYN3JKbHdDUHBVZW9RSzNFTmJ1M1pKdzFPekRWM1F4bGIKRnRXYy9KWnRtYkNLYlZaWEh6YkV1c3RDTDV5L3dOZGsrTUFWNitpUmFaN2tNdkxRaFZ2U3dwcWZIaW1RKzdhVgo5SzFJaWpKbVAya0ZwcEU1eWxySlROd0YxVG9Pa1czMHNQYTdiWTI2eWViaDRlK2Vpa1djVFZIcEhPU1pNWGpICkVadmJMbW5IQXN6bk1ieDEwTWovRnJERXluQVBsUVJ0TlpqNUh4bE9IS1Nmb1BOK1A2bWx2MjUrR3R3Mk5WWWIKZHB3bm5OZnlrZ2xLZXVSUklHTXErWnZKZkJjckU0YnpGL2RpYlFJREFRQUJBb0lCQUROMzhKbmY5VkN2R0IzNQp3QWtYSFhXYThzakJ5Z1pWcitNZ0hiQkdvUmxwMGJ0dkd1a0RJZ1RoTkJwUGZGSFY5bkc0UGdOaWsycjBCdWFoCnRIaFJxQ3JKUXQxdlBPYUJVTE0wUUNyWUVzeHhnTnBrZ1ZWZ0tSc3NQeExSV2FYQzZCbUJRZ211WVB1dDk3MnMKcjVaZzlCN0FCTUY1RDJURDdFVDh4UFlxbEQzK3JjTUpIelZmWkhyWXhkSzZsaldHelBkYkhrQ251VUdpS0lHWgp6VTI4cVNJLzBzeld6SEU0K2NrZFplc2dTUXovMVV2WUJ3Qk1DM0k3V3dvSVlZbXV1RVhreHdiTy9iMjltSVJHClhyaWY1M1FBNDA4a0YvODFic3piUmFuT01YRUhrN2NnSlFKUzVwaXV2UVV1KzA4bzJWZEhuZUJoQ1VqK1JWWDcKZElxWEhuRUNnWUVBOGpJaGdjMGtxTlA2Z3dldjJxSkszbGd3UzhqR3pWQVlJZDIzQkNuQWtQRGp5czROdVY2QQpZaEVGejNGWkpLV0J4a3pXeDVlNXZpeHJOR0RMUnNGVDgxR1ZTZUtnZGljR3NxYTZNL0Q2d2FRZXh6T2RTRFJNCk15cjBKeWs3dmRVTjdEVHovbTl4UjNzdkluM0dpMGhXaU1ELzFJUWpJOEhyLzkraVc1UkMwQ2tDZ1lFQTNlenYKZXhHM3BsUXhDbmNJbUlyaU5SS1NiRTljODluNGQ5VVJzcEs3cFVZVWQvSFExZmlTZ2ZpL2twZnBQb0dXREtOTAppOTA4WFN3MWNxTTZCTXVWTnlmT2xEKzRQaTZXb3phU1pHOTI4Ym8zNFBzbjhSeEZqSjJ3NTZyVC9iU2p5WUllClR3NUNpQzJmQUhGQmlUTkV0TEwzY09EMzlxcmF1NWR3MVZjMGVLVUNnWUFQd1JFSzUzUTJBeXZ0Y0FlUldqTXkKaVZ3QzRmbUVpMncyYjd5aTZiQmIvVDlrQnNrL3dKVHJUQjRyb3p6Z05GL2ZyVW5mUzlCS1BZdHZxY2d2UHc0ZAo2cldpUzdxU1ZQR0xsMnJQVENLVGpBQndocnY1WVdWL2dwREZKMXA2ZTZ4ZkxBYWZDMUs4Q3BoRFR4a21JRHQ0Cnc4MGdYc1FHWkd2Y2hnaUNtbjlLR1FLQmdGRUVMMFQ1YWRieHcxbHpyUktyR1B1UkJSMC9OOHJaMXdoQUk0N1MKWEdod2xnWlhwQXFKRFVzZmVTaFdCdE5IMFhSWnBMbXhrVmplUzhERzcrWlNQKzM3dlVHSHBZWWwwZDVSak0zWApsMCtWME5KMFBkZWFuNVUrK0JjSzJRczBoOXFIZ3ZNUFhLQ0VMeGlsUCt5TFo5aWp3UXRYUlk2cVB1SGUzbFV6CjJiYjFBb0dBWUhNazlYV29rOU5MRS81c1N1c2QyWFJqVjJVN1dGZnV5K21lKzJGVU1PdGliUUMrNHgwNnBLMlUKanRMc2VZU0thcURTQ1F5bG15SXlLVG5VNlZ0YnY1Z0FvRWhQZ2tYV2p2SWhPbDduamhHWXlyOFpLcWFvaTdOZwpNa29BQUg3KzBsL2VzeVkzSmZ3UlVyNUhoNGhmaDg1bHFFaDZjejlkTGhOa0U0VTV0TkU9Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== --- # Source: cilium/templates/cilium-configmap.yaml apiVersion: v1 @@ -188,7 +188,7 @@ data: enable-ipv4-masquerade: "true" enable-ipv4-big-tcp: "false" enable-ipv6-big-tcp: "false" - enable-ipv6-masquerade: "true" + enable-ipv6-masquerade: "false" enable-tcx: "true" datapath-mode: "veth" enable-masquerade-to-route-source: "false" diff --git a/nix/kubernetes/keys/package/k8s-ca/files/ca.conf b/nix/kubernetes/keys/package/k8s-ca/files/ca.conf index 8893f9d..c752da0 100644 --- a/nix/kubernetes/keys/package/k8s-ca/files/ca.conf +++ b/nix/kubernetes/keys/package/k8s-ca/files/ca.conf @@ -280,6 +280,7 @@ IP.11 = 2620:11f:7001:7:ffff:ffff:0ad7:01e1 IP.12 = 10.215.1.226 IP.13 = 2620:11f:7001:7:ffff:ffff:0ad7:01e2 IP.14 = fd00:3e42:e349::1 +IP.15 = 2620:11f:7001:7:ffff:eeee::1 DNS.0 = kubernetes DNS.1 = kubernetes.default DNS.2 = kubernetes.default.svc diff --git a/nix/kubernetes/roles/kube_controller_manager/default.nix b/nix/kubernetes/roles/kube_controller_manager/default.nix index 75dc659..4197828 100644 --- a/nix/kubernetes/roles/kube_controller_manager/default.nix +++ b/nix/kubernetes/roles/kube_controller_manager/default.nix @@ -38,9 +38,11 @@ in "${pkgs.kubernetes}/bin/kube-controller-manager" "--bind-address=0.0.0.0" # "--cluster-cidr=10.200.0.0/16" - # "--cluster-cidr=2620:11f:7001:7:ffff:ffff:0ac8:0000/16" + # "--cluster-cidr=2620:11f:7001:7:ffff:ffff:0ac8:0000/96" "--allocate-node-cidrs=true" - "--cluster-cidr=10.200.0.0/16,fd49:0595:2bba::/48" + "--cluster-cidr=10.200.0.0/16,2620:11f:7001:7:ffff:eeee::/96" + "--node-cidr-mask-size-ipv4=20" # default is 24 + "--node-cidr-mask-size-ipv6=112" # default is 64, must be smaller than cluster-cidr mask "--cluster-name=kubernetes" "--cluster-signing-cert-file=/.persist/keys/kube/ca.crt" "--cluster-signing-key-file=/.persist/keys/kube/ca.key" diff --git a/nix/kubernetes/roles/kube_proxy/default.nix b/nix/kubernetes/roles/kube_proxy/default.nix index a7977e4..340044f 100644 --- a/nix/kubernetes/roles/kube_proxy/default.nix +++ b/nix/kubernetes/roles/kube_proxy/default.nix @@ -33,7 +33,7 @@ in mode = "iptables"; # clusterCIDR = "10.200.0.0/16"; # clusterCIDR = "2620:11f:7001:7:ffff:ffff:0ac8:0000/16"; - clusterCIDR = "10.200.0.0/16,fd49:0595:2bba::/48"; + clusterCIDR = "10.200.0.0/16,2620:11f:7001:7:ffff:eeee::/96"; }; description = '' kubelet-config.yaml @@ -57,7 +57,7 @@ in "${pkgs.kubernetes}/bin/kube-proxy" "--config=${config_file}" "--nodeport-addresses=primary" - "--cluster-cidr=10.200.0.0/16,fd49:0595:2bba::/48" + "--cluster-cidr=10.200.0.0/16,2620:11f:7001:7:ffff:eeee::/96" ] ); Restart = "on-failure"; From 51e027ba676dbae47730895edc9e059e42404768 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 29 Dec 2025 19:11:55 -0500 Subject: [PATCH 41/81] Build the cilium manifest automatically in nix. --- nix/kubernetes/README.org | 38 +- nix/kubernetes/hosts/controller0/default.nix | 3 - nix/kubernetes/hosts/controller1/default.nix | 3 - nix/kubernetes/hosts/controller2/default.nix | 3 - nix/kubernetes/hosts/worker0/default.nix | 3 - nix/kubernetes/hosts/worker1/default.nix | 3 - nix/kubernetes/hosts/worker2/default.nix | 3 - .../files/manifests/cilium.yaml | 1687 ----------------- .../keys/package/bootstrap-script/package.nix | 4 +- .../keys/package/cilium-manifest/package.nix | 70 + nix/kubernetes/keys/scope.nix | 1 + nix/kubernetes/roles/network/default.nix | 11 +- 12 files changed, 83 insertions(+), 1746 deletions(-) delete mode 100644 nix/kubernetes/keys/package/bootstrap-script/files/manifests/cilium.yaml create mode 100644 nix/kubernetes/keys/package/cilium-manifest/package.nix diff --git a/nix/kubernetes/README.org b/nix/kubernetes/README.org index bace10e..70f8a68 100644 --- a/nix/kubernetes/README.org +++ b/nix/kubernetes/README.org @@ -10,43 +10,11 @@ { domain = "@users"; item = "rtprio"; type = "-"; value = 1; } ]; #+end_src -* Bootstrap -** Install cilium +* Healthcheck +** Check cilium status #+begin_src bash - # nix shell nixpkgs#cilium-cli - nix shell 'nixpkgs#kubernetes-helm' - - helm repo add cilium https://helm.cilium.io/ - helm template --dry-run=client cilium cilium/cilium --version 1.18.5 --namespace kube-system \ - --set kubeProxyReplacement=true \ - --set ipam.mode=kubernetes \ - --set k8sServiceHost="2620:11f:7001:7:ffff:ffff:ad7:1dd" \ - --set k8sServicePort=6443 \ - --set ipv6.enabled=true \ - --set ipv4.enabled=true \ - --set enableIPv6Masquerade=false - # --set enableIPv4BIGTCP=true \ - # --set enableIPv6BIGTCP=true - # --set routingMode=native \ - # --set ipv4NativeRoutingCIDR=10.0.0.0/8 \ - # --set ipv6NativeRoutingCIDR=fd00::/100 - - kubec - tl -n kube-system exec ds/cilium -- cilium-dbg status --verbose + kubectl -n kube-system exec ds/cilium -- cilium-dbg status --verbose kubectl -n kube-system exec ds/cilium -- cilium-dbg status | grep KubeProxyReplacement - - # --set hostFirewall.enabled=true - # routingMode=native - - # --set ipv4-native-routing-cidr=10.0.0.0/8 \ - # --set ipv6-native-routing-cidr=fd00::/100 - # --set 'ipam.operator.clusterPoolIPv4PodCIDRList=["10.0.0.0/8"]' \ - # --set 'ipam.operator.clusterPoolIPv6PodCIDRList=["fd00::/100"]' \ - - - # --set encryption.enabled=true \ - # --set encryption.type=wireguard - # --set encryption.nodeEncryption=true #+end_src ** Install flux #+begin_src bash diff --git a/nix/kubernetes/hosts/controller0/default.nix b/nix/kubernetes/hosts/controller0/default.nix index 4521b7e..bb32f50 100644 --- a/nix/kubernetes/hosts/controller0/default.nix +++ b/nix/kubernetes/hosts/controller0/default.nix @@ -51,9 +51,6 @@ address = "2620:11f:7001:7:ffff:ffff:0ad7:0101"; inherit interface; }; - nameservers = [ - "10.215.1.1" - ]; dhcpcd.enable = lib.mkForce false; useDHCP = lib.mkForce false; diff --git a/nix/kubernetes/hosts/controller1/default.nix b/nix/kubernetes/hosts/controller1/default.nix index 3af358d..f4b5c87 100644 --- a/nix/kubernetes/hosts/controller1/default.nix +++ b/nix/kubernetes/hosts/controller1/default.nix @@ -51,9 +51,6 @@ address = "2620:11f:7001:7:ffff:ffff:0ad7:0101"; inherit interface; }; - nameservers = [ - "10.215.1.1" - ]; dhcpcd.enable = lib.mkForce false; useDHCP = lib.mkForce false; diff --git a/nix/kubernetes/hosts/controller2/default.nix b/nix/kubernetes/hosts/controller2/default.nix index a25ddae..e9c318a 100644 --- a/nix/kubernetes/hosts/controller2/default.nix +++ b/nix/kubernetes/hosts/controller2/default.nix @@ -51,9 +51,6 @@ address = "2620:11f:7001:7:ffff:ffff:0ad7:0101"; inherit interface; }; - nameservers = [ - "10.215.1.1" - ]; dhcpcd.enable = lib.mkForce false; useDHCP = lib.mkForce false; diff --git a/nix/kubernetes/hosts/worker0/default.nix b/nix/kubernetes/hosts/worker0/default.nix index 59f80f3..6e4570f 100644 --- a/nix/kubernetes/hosts/worker0/default.nix +++ b/nix/kubernetes/hosts/worker0/default.nix @@ -51,9 +51,6 @@ address = "2620:11f:7001:7:ffff:ffff:0ad7:0101"; inherit interface; }; - nameservers = [ - "10.215.1.1" - ]; dhcpcd.enable = lib.mkForce false; useDHCP = lib.mkForce false; diff --git a/nix/kubernetes/hosts/worker1/default.nix b/nix/kubernetes/hosts/worker1/default.nix index 6f951f0..bb56099 100644 --- a/nix/kubernetes/hosts/worker1/default.nix +++ b/nix/kubernetes/hosts/worker1/default.nix @@ -51,9 +51,6 @@ address = "2620:11f:7001:7:ffff:ffff:0ad7:0101"; inherit interface; }; - nameservers = [ - "10.215.1.1" - ]; dhcpcd.enable = lib.mkForce false; useDHCP = lib.mkForce false; diff --git a/nix/kubernetes/hosts/worker2/default.nix b/nix/kubernetes/hosts/worker2/default.nix index d595c0d..ca83144 100644 --- a/nix/kubernetes/hosts/worker2/default.nix +++ b/nix/kubernetes/hosts/worker2/default.nix @@ -51,9 +51,6 @@ address = "2620:11f:7001:7:ffff:ffff:0ad7:0101"; inherit interface; }; - nameservers = [ - "10.215.1.1" - ]; dhcpcd.enable = lib.mkForce false; useDHCP = lib.mkForce false; diff --git a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/cilium.yaml b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/cilium.yaml deleted file mode 100644 index 585612e..0000000 --- a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/cilium.yaml +++ /dev/null @@ -1,1687 +0,0 @@ ---- -# Source: cilium/templates/cilium-secrets-namespace.yaml -apiVersion: v1 -kind: Namespace -metadata: - name: "cilium-secrets" - labels: - app.kubernetes.io/part-of: cilium - annotations: ---- -# Source: cilium/templates/cilium-agent/serviceaccount.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: "cilium" - namespace: kube-system ---- -# Source: cilium/templates/cilium-envoy/serviceaccount.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: "cilium-envoy" - namespace: kube-system ---- -# Source: cilium/templates/cilium-operator/serviceaccount.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: "cilium-operator" - namespace: kube-system ---- -# Source: cilium/templates/cilium-ca-secret.yaml -apiVersion: v1 -kind: Secret -metadata: - name: cilium-ca - namespace: kube-system -data: - ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lSQU53OW1DamlKdysveDFOQTJ6Sm5zc0F3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkxTVRJeU9USXlOVEF3TVZvWERUSTRNVEl5T0RJeQpOVEF3TVZvd0ZERVNNQkFHQTFVRUF4TUpRMmxzYVhWdElFTkJNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFNSUlCQ2dLQ0FRRUF6SGM0aThFL2JUMmlrOVB6Z0ZQeWVoTHIvUVZEbXl4SU41ZVBIUVFDSjVudXRZOHAKRTFjSGdlVzI4SGJhclZ3U3ZSQXFObUdsV2FmZDVtckgyN1E2YThpVm1DcDZYeGVsNFMrYVlRWHkrZFgyVXdOeApLZTlRNVA4WXhUMWNGWkV0QW5tclBzS0dyTm9YdDdQNC8wdHBSVTVvWktnMEJQK3RnamxiUHd3TUtueFROWjMvClBrZzI0M0Z2bEVDWHV2VXNENy9wcFhkUzRsNEVzWGpkbW1IaXpGelQ2S3A1VWJicUtsaWh5aHhkSzhlZXB4emMKM3dyUEE1ZVZXNjdCUlc4dTU1ZWFSMnVpOGRDZkgwZ2dYbXVHL0hiSTJMREtOQ09RanRFRk5XVWhUM08zb0ZuVAovWS9ON2lJc1VobE9IcnZJNUNKYWhKV2NuMVRHd1FuWWthcStxd0lEQVFBQm8yRXdYekFPQmdOVkhROEJBZjhFCkJBTUNBcVF3SFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGRFc0WXJXYkN4c0ZlQU9nT1Vsb1A1bUllVVgrTUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQ0VOdnBiblI4TUtaMWpBVWRlMU55elJueUhjemR2T0Jla1F1YVFkS01hOWhFMk12Y0hTQ05qClBwNUNOenNXWk5FMW5RekF6RFNDcEV5Z0lpRm5mNitYaTBBZlZncXdpcDQ2SytkbUd1d1B5dnBRNWorU1YxNHoKSmlFOElJYTdYMzQrWnlrM2ZZUjd0elpobllqQ21UcXJYbnZSbzBGZHNuU0ZvRUU4T3RRMjU5dllJeXNuRXVPVwpOTEwvSTVQeGpNQlQ1WWNkaXdtbGR2RGxwZzk5YkRUQ0w0UFJ1MldKV2VFUmhzdGIzbWdRSk9Mb0RSM1VWWmowCjBFOGxQTjJiTjZMbVZjYjB1QlVyVjJQOHR6ZmxhVnF4SzQ5SlZFQmpFVHFaSzY3TTVYY3R4WG55SHdPVEpRckIKRGhVbWpkSGNZWCtYL0t5MDNOa1dhc2ZJeWdnV3IvR3gKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= - ca.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBekhjNGk4RS9iVDJpazlQemdGUHllaExyL1FWRG15eElONWVQSFFRQ0o1bnV0WThwCkUxY0hnZVcyOEhiYXJWd1N2UkFxTm1HbFdhZmQ1bXJIMjdRNmE4aVZtQ3A2WHhlbDRTK2FZUVh5K2RYMlV3TngKS2U5UTVQOFl4VDFjRlpFdEFubXJQc0tHck5vWHQ3UDQvMHRwUlU1b1pLZzBCUCt0Z2psYlB3d01LbnhUTlozLwpQa2cyNDNGdmxFQ1h1dlVzRDcvcHBYZFM0bDRFc1hqZG1tSGl6RnpUNktwNVViYnFLbGloeWh4ZEs4ZWVweHpjCjN3clBBNWVWVzY3QlJXOHU1NWVhUjJ1aThkQ2ZIMGdnWG11Ry9IYkkyTERLTkNPUWp0RUZOV1VoVDNPM29GblQKL1kvTjdpSXNVaGxPSHJ2STVDSmFoSldjbjFUR3dRbllrYXErcXdJREFRQUJBb0lCQUEySWtYNEVWWVRZUU5VQwpwYnhzQ09ZL1U4NlVrSG00TGFFODcxSTZmQjhyTXNBWmNMZ2xhNHRjMGM4dlhzVTBNZlF1aUNndENZMSs3V1Q0CjB2amVBRmRiYyt5SHhxMStFVndCN3NIV3ROTXMzZTdrL3JkQWt1WUZZL1dmN3RDT0VDREI5UzJiUmJ1ZmZIbUwKblF3REpJYkpiRDRReTRaZ215bVpUeVlmVDVkb3hTSEJVMU9GMkFPTlg1VkYya1oySDI5dVNsMlgydm9jMlZidQowUlMvZFp6VExkcTRocUFPQ1VsbER6Nzk1U29aVU4zdmsranBSOGdxN1J2WTV4MzFDdmZQTENKNFNGQ25WSWtEClZvcnNCRW51V3ZaN0tBc051Tms3ZGMvYys0STk3d3cxNEt2NWVXV3p2ak05c1RFZWF1M3liRlluTmFTMUwzbjUKR3ZzeERERUNnWUVBMGJWNEJwS0xhM2g1c3BpbHZ0UlpSMVV3WndpZG13REMxZVB6dEZHSmx0SUg1b1I0T1BaVApzWjR2ak9WU21MUndFQ1VGdkNIaWZpV0ljTjd0NG5oQ1VCNmpJQXh5aS8wU1BSU1pkN2dtMXdYNGZyWVRVTkJtCk1DZ1krZlNLbjN2R3o2ODNlbUlFbDYyOFl5amt0aWYzQXdjeHVwdk9lSjVqQVBaekNoTG8ycE1DZ1lFQStabDIKMVp1Ym10RUI5T3pJZmlhRUV3R1k5TXdkKzNKL3daUU9LUWd6SGZBRU1DQnpGcnZZMWNvWkFpeWt3NWtoSTEwWQpZRlpFQlVhdFF3L3JpRFYvR1pDRkJyRHBYTFVjeFdMcTZoTGRtYmVZZHdXVXUvakdsSTUrYnhpNkVhUnlDcys4ClpUZVFNSDNLWVFJZ2RmQ3J3N0VTSEp1OU5ualFpMmhOZkt5STRva0NnWUVBaExrN2U3MW42OTdWOXdqUmJkTnIKcGMvdTBHY05pTHFVbFZhdU9TT1oxeENhMjNSZnVuMThQdUFFN0VGL1l4SmdFbmU4QjNQU3ExQUo2SlhQTFJRNwp0QVdQN0lxMFBKRXc1K09QdGN1aEdWbTRDa29tTTNHU0cweGxjbDBwRndMNXN4d01HckxLZ1V6OS9DdzNoR29LCjFhbkorWGIrMkN3Mk1MZkoyMGhZZzIwQ2dZRUE3VWhlZDMyQjlURGpPbE5yMnJtRTc0aWlQMzVZdG5WSVRPQVMKZ2lHQWJ1S0JLTHVBamNrd3Z6VnNodXVvQ3libElQaGN0eURyYzFTWVhGdWpIdzgwY0RvNnJIeThyTnlrcUdWOApTK0I1ZUt2WUxyWklpbFpiZWxqb2kzY25WS1JQb2tXUXBXeW9EK0ZWNXRrZHdPRjJlUWc1M3FhVHYxZ2xjWkpMCnI4MWFLSGtDZ1lBSXVxaDZzOU9RRGpjckVMeklJZENiZUxxLzhEdWUzcEZjbmFIWjgxemcwY0oxNnM1MU53aHQKaUQxWC9oRS9sdkp1VTFjTEFpNlJLZEk3UXIwL0x2SkFDSTlCakp6VVVYS1hQZ3JJZU44UUxCOWNhK2lka2FheApIWmNzdnBSN1g1b2cvTVpldWVTUk1OaU9wdVJSZ0xlYlh5R29lMkhmbE05STUvazJERUU5bmc9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo= ---- -# Source: cilium/templates/hubble/tls-helm/server-secret.yaml -apiVersion: v1 -kind: Secret -metadata: - name: hubble-server-certs - namespace: kube-system -type: kubernetes.io/tls -data: - ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lSQU53OW1DamlKdysveDFOQTJ6Sm5zc0F3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkxTVRJeU9USXlOVEF3TVZvWERUSTRNVEl5T0RJeQpOVEF3TVZvd0ZERVNNQkFHQTFVRUF4TUpRMmxzYVhWdElFTkJNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFNSUlCQ2dLQ0FRRUF6SGM0aThFL2JUMmlrOVB6Z0ZQeWVoTHIvUVZEbXl4SU41ZVBIUVFDSjVudXRZOHAKRTFjSGdlVzI4SGJhclZ3U3ZSQXFObUdsV2FmZDVtckgyN1E2YThpVm1DcDZYeGVsNFMrYVlRWHkrZFgyVXdOeApLZTlRNVA4WXhUMWNGWkV0QW5tclBzS0dyTm9YdDdQNC8wdHBSVTVvWktnMEJQK3RnamxiUHd3TUtueFROWjMvClBrZzI0M0Z2bEVDWHV2VXNENy9wcFhkUzRsNEVzWGpkbW1IaXpGelQ2S3A1VWJicUtsaWh5aHhkSzhlZXB4emMKM3dyUEE1ZVZXNjdCUlc4dTU1ZWFSMnVpOGRDZkgwZ2dYbXVHL0hiSTJMREtOQ09RanRFRk5XVWhUM08zb0ZuVAovWS9ON2lJc1VobE9IcnZJNUNKYWhKV2NuMVRHd1FuWWthcStxd0lEQVFBQm8yRXdYekFPQmdOVkhROEJBZjhFCkJBTUNBcVF3SFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGRFc0WXJXYkN4c0ZlQU9nT1Vsb1A1bUllVVgrTUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQ0VOdnBiblI4TUtaMWpBVWRlMU55elJueUhjemR2T0Jla1F1YVFkS01hOWhFMk12Y0hTQ05qClBwNUNOenNXWk5FMW5RekF6RFNDcEV5Z0lpRm5mNitYaTBBZlZncXdpcDQ2SytkbUd1d1B5dnBRNWorU1YxNHoKSmlFOElJYTdYMzQrWnlrM2ZZUjd0elpobllqQ21UcXJYbnZSbzBGZHNuU0ZvRUU4T3RRMjU5dllJeXNuRXVPVwpOTEwvSTVQeGpNQlQ1WWNkaXdtbGR2RGxwZzk5YkRUQ0w0UFJ1MldKV2VFUmhzdGIzbWdRSk9Mb0RSM1VWWmowCjBFOGxQTjJiTjZMbVZjYjB1QlVyVjJQOHR6ZmxhVnF4SzQ5SlZFQmpFVHFaSzY3TTVYY3R4WG55SHdPVEpRckIKRGhVbWpkSGNZWCtYL0t5MDNOa1dhc2ZJeWdnV3IvR3gKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= - tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURWekNDQWorZ0F3SUJBZ0lSQVBsUTMyUWVldjRpclVZaXJMRmVSU0l3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkxTVRJeU9USXlOVEF3TVZvWERUSTJNVEl5T1RJeQpOVEF3TVZvd0tqRW9NQ1lHQTFVRUF3d2ZLaTVrWldaaGRXeDBMbWgxWW1Kc1pTMW5jbkJqTG1OcGJHbDFiUzVwCmJ6Q0NBU0l3RFFZSktvWklodmNOQVFFQkJRQURnZ0VQQURDQ0FRb0NnZ0VCQU5IMWI3VXQ0eDdaSitUUUxBWjAKdXg0VlY4V2Z5RXRGY0hUbHdxeGZGcmMzZHBrNTQzWnpwN0NSbUg3Mkw2NlQ3VW1iRElydHZsdHcxdmh5MSs2eQpaY0FqNlZIcUVDdHhEVzd0MlNjTlRzdzFkME1aV3hiVm5QeVdiWm13aW0xV1Z4ODJ4THJMUWkrY3Y4RFhaUGpBCkZldm9rV21lNURMeTBJVmIwc0thbng0cGtQdTJsZlN0U0lveVpqOXBCYWFST2NwYXlVemNCZFU2RHBGdDlMRDIKdTIyTnVzbm00ZUh2bm9wRm5FMVI2UnprbVRGNHh4R2IyeTVweHdMTTV6RzhkZERJL3hhd3hNcHdENVVFYlRXWQorUjhaVGh5a242RHpmaitwcGI5dWZocmNOalZXRzNhY0o1elg4cElKU25ya1VTQmpLdm1ieVh3WEt4T0c4eGYzClltMENBd0VBQWFPQmpUQ0JpakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdIUVlEVlIwbEJCWXdGQVlJS3dZQkJRVUgKQXdFR0NDc0dBUVVGQndNQ01Bd0dBMVVkRXdFQi93UUNNQUF3SHdZRFZSMGpCQmd3Rm9BVU5iaGl0WnNMR3dWNApBNkE1U1dnL21ZaDVSZjR3S2dZRFZSMFJCQ013SVlJZktpNWtaV1poZFd4MExtaDFZbUpzWlMxbmNuQmpMbU5wCmJHbDFiUzVwYnpBTkJna3Foa2lHOXcwQkFRc0ZBQU9DQVFFQVRrb24wcTVDdzJPT2JCcy9Nb0g2a041WXJidTAKeTR1c1hVc0x4V3oyNDFadisvY3JZQ0x4d092bGZNTGNnU0crL0xtTkIrMGpGc0RnMnRhQk9zejNHbzRtUzRrQwprV1QxUENSQ2o5MFdZU2hzRHNMdUV5MlI1bndTYmZCL3NHc3MxZnNwcDNhTldXSUNjRjJlZmh1TXRsREdMcEtRClFKQytlbHNYSFNoNkpWMWROTXhFYkhQVk9kQzBobUswNHRJMjJ0Tytsc0Iwd090S3VCbUJUb2VtR1ZScUtRbmEKem81NUtLc2cyQ1FwODhSTUJ4OUk2aXJjMThqSmFJeFhaRTE5RGMzWkRiNk42ZzVGc2hNeHhUK3F5cEt3VUVWego1SGRQNks5NjhyS3RjZnZXSXpieDVsYnIvY3hrZWZ0ZTE5SThXbmlxTzl4SlkxN0duVXV1aU5DREx3PT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= - tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBMGZWdnRTM2pIdGtuNU5Bc0JuUzdIaFZYeFovSVMwVndkT1hDckY4V3R6ZDJtVG5qCmRuT25zSkdZZnZZdnJwUHRTWnNNaXUyK1czRFcrSExYN3JKbHdDUHBVZW9RSzNFTmJ1M1pKdzFPekRWM1F4bGIKRnRXYy9KWnRtYkNLYlZaWEh6YkV1c3RDTDV5L3dOZGsrTUFWNitpUmFaN2tNdkxRaFZ2U3dwcWZIaW1RKzdhVgo5SzFJaWpKbVAya0ZwcEU1eWxySlROd0YxVG9Pa1czMHNQYTdiWTI2eWViaDRlK2Vpa1djVFZIcEhPU1pNWGpICkVadmJMbW5IQXN6bk1ieDEwTWovRnJERXluQVBsUVJ0TlpqNUh4bE9IS1Nmb1BOK1A2bWx2MjUrR3R3Mk5WWWIKZHB3bm5OZnlrZ2xLZXVSUklHTXErWnZKZkJjckU0YnpGL2RpYlFJREFRQUJBb0lCQUROMzhKbmY5VkN2R0IzNQp3QWtYSFhXYThzakJ5Z1pWcitNZ0hiQkdvUmxwMGJ0dkd1a0RJZ1RoTkJwUGZGSFY5bkc0UGdOaWsycjBCdWFoCnRIaFJxQ3JKUXQxdlBPYUJVTE0wUUNyWUVzeHhnTnBrZ1ZWZ0tSc3NQeExSV2FYQzZCbUJRZ211WVB1dDk3MnMKcjVaZzlCN0FCTUY1RDJURDdFVDh4UFlxbEQzK3JjTUpIelZmWkhyWXhkSzZsaldHelBkYkhrQ251VUdpS0lHWgp6VTI4cVNJLzBzeld6SEU0K2NrZFplc2dTUXovMVV2WUJ3Qk1DM0k3V3dvSVlZbXV1RVhreHdiTy9iMjltSVJHClhyaWY1M1FBNDA4a0YvODFic3piUmFuT01YRUhrN2NnSlFKUzVwaXV2UVV1KzA4bzJWZEhuZUJoQ1VqK1JWWDcKZElxWEhuRUNnWUVBOGpJaGdjMGtxTlA2Z3dldjJxSkszbGd3UzhqR3pWQVlJZDIzQkNuQWtQRGp5czROdVY2QQpZaEVGejNGWkpLV0J4a3pXeDVlNXZpeHJOR0RMUnNGVDgxR1ZTZUtnZGljR3NxYTZNL0Q2d2FRZXh6T2RTRFJNCk15cjBKeWs3dmRVTjdEVHovbTl4UjNzdkluM0dpMGhXaU1ELzFJUWpJOEhyLzkraVc1UkMwQ2tDZ1lFQTNlenYKZXhHM3BsUXhDbmNJbUlyaU5SS1NiRTljODluNGQ5VVJzcEs3cFVZVWQvSFExZmlTZ2ZpL2twZnBQb0dXREtOTAppOTA4WFN3MWNxTTZCTXVWTnlmT2xEKzRQaTZXb3phU1pHOTI4Ym8zNFBzbjhSeEZqSjJ3NTZyVC9iU2p5WUllClR3NUNpQzJmQUhGQmlUTkV0TEwzY09EMzlxcmF1NWR3MVZjMGVLVUNnWUFQd1JFSzUzUTJBeXZ0Y0FlUldqTXkKaVZ3QzRmbUVpMncyYjd5aTZiQmIvVDlrQnNrL3dKVHJUQjRyb3p6Z05GL2ZyVW5mUzlCS1BZdHZxY2d2UHc0ZAo2cldpUzdxU1ZQR0xsMnJQVENLVGpBQndocnY1WVdWL2dwREZKMXA2ZTZ4ZkxBYWZDMUs4Q3BoRFR4a21JRHQ0Cnc4MGdYc1FHWkd2Y2hnaUNtbjlLR1FLQmdGRUVMMFQ1YWRieHcxbHpyUktyR1B1UkJSMC9OOHJaMXdoQUk0N1MKWEdod2xnWlhwQXFKRFVzZmVTaFdCdE5IMFhSWnBMbXhrVmplUzhERzcrWlNQKzM3dlVHSHBZWWwwZDVSak0zWApsMCtWME5KMFBkZWFuNVUrK0JjSzJRczBoOXFIZ3ZNUFhLQ0VMeGlsUCt5TFo5aWp3UXRYUlk2cVB1SGUzbFV6CjJiYjFBb0dBWUhNazlYV29rOU5MRS81c1N1c2QyWFJqVjJVN1dGZnV5K21lKzJGVU1PdGliUUMrNHgwNnBLMlUKanRMc2VZU0thcURTQ1F5bG15SXlLVG5VNlZ0YnY1Z0FvRWhQZ2tYV2p2SWhPbDduamhHWXlyOFpLcWFvaTdOZwpNa29BQUg3KzBsL2VzeVkzSmZ3UlVyNUhoNGhmaDg1bHFFaDZjejlkTGhOa0U0VTV0TkU9Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== ---- -# Source: cilium/templates/cilium-configmap.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: cilium-config - namespace: kube-system -data: - - # Identity allocation mode selects how identities are shared between cilium - # nodes by setting how they are stored. The options are "crd", "kvstore" or - # "doublewrite-readkvstore" / "doublewrite-readcrd". - # - "crd" stores identities in kubernetes as CRDs (custom resource definition). - # These can be queried with: - # kubectl get ciliumid - # - "kvstore" stores identities in an etcd kvstore, that is - # configured below. Cilium versions before 1.6 supported only the kvstore - # backend. Upgrades from these older cilium versions should continue using - # the kvstore by commenting out the identity-allocation-mode below, or - # setting it to "kvstore". - # - "doublewrite" modes store identities in both the kvstore and CRDs. This is useful - # for seamless migrations from the kvstore mode to the crd mode. Consult the - # documentation for more information on how to perform the migration. - identity-allocation-mode: crd - - identity-heartbeat-timeout: "30m0s" - identity-gc-interval: "15m0s" - cilium-endpoint-gc-interval: "5m0s" - nodes-gc-interval: "5m0s" - - # If you want to run cilium in debug mode change this value to true - debug: "false" - debug-verbose: "" - metrics-sampling-interval: "5m" - # The agent can be put into the following three policy enforcement modes - # default, always and never. - # https://docs.cilium.io/en/latest/security/policy/intro/#policy-enforcement-modes - enable-policy: "default" - policy-cidr-match-mode: "" - # If you want metrics enabled in cilium-operator, set the port for - # which the Cilium Operator will have their metrics exposed. - # NOTE that this will open the port on the nodes where Cilium operator pod - # is scheduled. - operator-prometheus-serve-addr: ":9963" - enable-metrics: "true" - enable-policy-secrets-sync: "true" - policy-secrets-only-from-secrets-namespace: "true" - policy-secrets-namespace: "cilium-secrets" - - # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 - # address. - enable-ipv4: "true" - - # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6 - # address. - enable-ipv6: "true" - # Users who wish to specify their own custom CNI configuration file must set - # custom-cni-conf to "true", otherwise Cilium may overwrite the configuration. - custom-cni-conf: "false" - enable-bpf-clock-probe: "false" - # If you want cilium monitor to aggregate tracing for packets, set this level - # to "low", "medium", or "maximum". The higher the level, the less packets - # that will be seen in monitor output. - monitor-aggregation: medium - - # The monitor aggregation interval governs the typical time between monitor - # notification events for each allowed connection. - # - # Only effective when monitor aggregation is set to "medium" or higher. - monitor-aggregation-interval: "5s" - - # The monitor aggregation flags determine which TCP flags which, upon the - # first observation, cause monitor notifications to be generated. - # - # Only effective when monitor aggregation is set to "medium" or higher. - monitor-aggregation-flags: all - # Specifies the ratio (0.0-1.0] of total system memory to use for dynamic - # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps. - bpf-map-dynamic-size-ratio: "0.0025" - # bpf-policy-map-max specifies the maximum number of entries in endpoint - # policy map (per endpoint) - bpf-policy-map-max: "16384" - # bpf-policy-stats-map-max specifies the maximum number of entries in global - # policy stats map - bpf-policy-stats-map-max: "65536" - # bpf-lb-map-max specifies the maximum number of entries in bpf lb service, - # backend and affinity maps. - bpf-lb-map-max: "65536" - bpf-lb-external-clusterip: "false" - bpf-lb-source-range-all-types: "false" - bpf-lb-algorithm-annotation: "false" - bpf-lb-mode-annotation: "false" - - bpf-distributed-lru: "false" - bpf-events-drop-enabled: "true" - bpf-events-policy-verdict-enabled: "true" - bpf-events-trace-enabled: "true" - - # Pre-allocation of map entries allows per-packet latency to be reduced, at - # the expense of up-front memory allocation for the entries in the maps. The - # default value below will minimize memory usage in the default installation; - # users who are sensitive to latency may consider setting this to "true". - # - # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore - # this option and behave as though it is set to "true". - # - # If this value is modified, then during the next Cilium startup the restore - # of existing endpoints and tracking of ongoing connections may be disrupted. - # As a result, reply packets may be dropped and the load-balancing decisions - # for established connections may change. - # - # If this option is set to "false" during an upgrade from 1.3 or earlier to - # 1.4 or later, then it may cause one-time disruptions during the upgrade. - preallocate-bpf-maps: "false" - - # Name of the cluster. Only relevant when building a mesh of clusters. - cluster-name: "default" - # Unique ID of the cluster. Must be unique across all conneted clusters and - # in the range of 1 and 255. Only relevant when building a mesh of clusters. - cluster-id: "0" - - # Encapsulation mode for communication between nodes - # Possible values: - # - disabled - # - vxlan (default) - # - geneve - - routing-mode: "tunnel" - tunnel-protocol: "vxlan" - tunnel-source-port-range: "0-0" - service-no-backend-response: "reject" - - - # Enables L7 proxy for L7 policy enforcement and visibility - enable-l7-proxy: "true" - enable-ipv4-masquerade: "true" - enable-ipv4-big-tcp: "false" - enable-ipv6-big-tcp: "false" - enable-ipv6-masquerade: "false" - enable-tcx: "true" - datapath-mode: "veth" - enable-masquerade-to-route-source: "false" - - enable-xt-socket-fallback: "true" - install-no-conntrack-iptables-rules: "false" - iptables-random-fully: "false" - - auto-direct-node-routes: "false" - direct-routing-skip-unreachable: "false" - - - - kube-proxy-replacement: "true" - kube-proxy-replacement-healthz-bind-address: "" - bpf-lb-sock: "false" - nodeport-addresses: "" - enable-health-check-nodeport: "true" - enable-health-check-loadbalancer-ip: "false" - node-port-bind-protection: "true" - enable-auto-protect-node-port-range: "true" - bpf-lb-acceleration: "disabled" - enable-svc-source-range-check: "true" - enable-l2-neigh-discovery: "false" - k8s-require-ipv4-pod-cidr: "false" - k8s-require-ipv6-pod-cidr: "false" - enable-k8s-networkpolicy: "true" - enable-endpoint-lockdown-on-policy-overflow: "false" - # Tell the agent to generate and write a CNI configuration file - write-cni-conf-when-ready: /host/etc/cni/net.d/05-cilium.conflist - cni-exclusive: "true" - cni-log-file: "/var/run/cilium/cilium-cni.log" - enable-endpoint-health-checking: "true" - enable-health-checking: "true" - health-check-icmp-failure-threshold: "3" - enable-well-known-identities: "false" - enable-node-selector-labels: "false" - synchronize-k8s-nodes: "true" - operator-api-serve-addr: "127.0.0.1:9234" - - enable-hubble: "true" - # UNIX domain socket for Hubble server to listen to. - hubble-socket-path: "/var/run/cilium/hubble.sock" - hubble-network-policy-correlation-enabled: "true" - # An additional address for Hubble server to listen to (e.g. ":4244"). - hubble-listen-address: ":4244" - hubble-disable-tls: "false" - hubble-tls-cert-file: /var/lib/cilium/tls/hubble/server.crt - hubble-tls-key-file: /var/lib/cilium/tls/hubble/server.key - hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/client-ca.crt - ipam: "kubernetes" - ipam-cilium-node-update-rate: "15s" - - default-lb-service-ipam: "lbipam" - egress-gateway-reconciliation-trigger-interval: "1s" - enable-vtep: "false" - vtep-endpoint: "" - vtep-cidr: "" - vtep-mask: "" - vtep-mac: "" - procfs: "/host/proc" - bpf-root: "/sys/fs/bpf" - cgroup-root: "/run/cilium/cgroupv2" - - identity-management-mode: "agent" - enable-sctp: "false" - remove-cilium-node-taints: "true" - set-cilium-node-taints: "true" - set-cilium-is-up-condition: "true" - unmanaged-pod-watcher-interval: "15" - # default DNS proxy to transparent mode in non-chaining modes - dnsproxy-enable-transparent-mode: "true" - dnsproxy-socket-linger-timeout: "10" - tofqdns-dns-reject-response-code: "refused" - tofqdns-enable-dns-compression: "true" - tofqdns-endpoint-max-ip-per-hostname: "1000" - tofqdns-idle-connection-grace-period: "0s" - tofqdns-max-deferred-connection-deletes: "10000" - tofqdns-proxy-response-max-delay: "100ms" - tofqdns-preallocate-identities: "true" - agent-not-ready-taint-key: "node.cilium.io/agent-not-ready" - - mesh-auth-enabled: "true" - mesh-auth-queue-size: "1024" - mesh-auth-rotated-identities-queue-size: "1024" - mesh-auth-gc-interval: "5m0s" - - proxy-xff-num-trusted-hops-ingress: "0" - proxy-xff-num-trusted-hops-egress: "0" - proxy-connect-timeout: "2" - proxy-initial-fetch-timeout: "30" - proxy-max-requests-per-connection: "0" - proxy-max-connection-duration-seconds: "0" - proxy-idle-timeout-seconds: "60" - proxy-max-concurrent-retries: "128" - http-retry-count: "3" - http-stream-idle-timeout: "300" - - external-envoy-proxy: "true" - envoy-base-id: "0" - envoy-access-log-buffer-size: "4096" - envoy-keep-cap-netbindservice: "false" - max-connected-clusters: "255" - clustermesh-enable-endpoint-sync: "false" - clustermesh-enable-mcs-api: "false" - policy-default-local-cluster: "false" - - nat-map-stats-entries: "32" - nat-map-stats-interval: "30s" - enable-internal-traffic-policy: "true" - enable-lb-ipam: "true" - enable-non-default-deny-policies: "true" - enable-source-ip-verification: "true" - -# Extra config allows adding arbitrary properties to the cilium config. -# By putting it at the end of the ConfigMap, it's also possible to override existing properties. ---- -# Source: cilium/templates/cilium-envoy/configmap.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: cilium-envoy-config - namespace: kube-system -data: - # Keep the key name as bootstrap-config.json to avoid breaking changes - bootstrap-config.json: | - {"admin":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/admin.sock"}}},"applicationLogConfig":{"logFormat":{"textFormat":"[%Y-%m-%d %T.%e][%t][%l][%n] [%g:%#] %v"}},"bootstrapExtensions":[{"name":"envoy.bootstrap.internal_listener","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener"}}],"dynamicResources":{"cdsConfig":{"apiConfigSource":{"apiType":"GRPC","grpcServices":[{"envoyGrpc":{"clusterName":"xds-grpc-cilium"}}],"setNodeOnFirstMessageOnly":true,"transportApiVersion":"V3"},"initialFetchTimeout":"30s","resourceApiVersion":"V3"},"ldsConfig":{"apiConfigSource":{"apiType":"GRPC","grpcServices":[{"envoyGrpc":{"clusterName":"xds-grpc-cilium"}}],"setNodeOnFirstMessageOnly":true,"transportApiVersion":"V3"},"initialFetchTimeout":"30s","resourceApiVersion":"V3"}},"node":{"cluster":"ingress-cluster","id":"host~127.0.0.1~no-id~localdomain"},"overloadManager":{"resourceMonitors":[{"name":"envoy.resource_monitors.global_downstream_max_connections","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig","max_active_downstream_connections":"50000"}}]},"staticResources":{"clusters":[{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"ingress-cluster","type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"egress-cluster-tls","transportSocket":{"name":"cilium.tls_wrapper","typedConfig":{"@type":"type.googleapis.com/cilium.UpstreamTlsWrapperContext"}},"type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"upstreamHttpProtocolOptions":{},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"egress-cluster","type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"ingress-cluster-tls","transportSocket":{"name":"cilium.tls_wrapper","typedConfig":{"@type":"type.googleapis.com/cilium.UpstreamTlsWrapperContext"}},"type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"upstreamHttpProtocolOptions":{},"useDownstreamProtocolConfig":{}}}},{"connectTimeout":"2s","loadAssignment":{"clusterName":"xds-grpc-cilium","endpoints":[{"lbEndpoints":[{"endpoint":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/xds.sock"}}}}]}]},"name":"xds-grpc-cilium","type":"STATIC","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","explicitHttpConfig":{"http2ProtocolOptions":{}}}}},{"connectTimeout":"2s","loadAssignment":{"clusterName":"/envoy-admin","endpoints":[{"lbEndpoints":[{"endpoint":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/admin.sock"}}}}]}]},"name":"/envoy-admin","type":"STATIC"}],"listeners":[{"additionalAddresses":[{"address":{"socketAddress":{"address":"::","portValue":9964}}}],"address":{"socketAddress":{"address":"0.0.0.0","portValue":9964}},"filterChains":[{"filters":[{"name":"envoy.filters.network.http_connection_manager","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager","httpFilters":[{"name":"envoy.filters.http.router","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"}}],"internalAddressConfig":{"cidrRanges":[{"addressPrefix":"10.0.0.0","prefixLen":8},{"addressPrefix":"172.16.0.0","prefixLen":12},{"addressPrefix":"192.168.0.0","prefixLen":16},{"addressPrefix":"127.0.0.1","prefixLen":32},{"addressPrefix":"::1","prefixLen":128}]},"routeConfig":{"virtualHosts":[{"domains":["*"],"name":"prometheus_metrics_route","routes":[{"match":{"prefix":"/metrics"},"name":"prometheus_metrics_route","route":{"cluster":"/envoy-admin","prefixRewrite":"/stats/prometheus"}}]}]},"statPrefix":"envoy-prometheus-metrics-listener","streamIdleTimeout":"300s"}}]}],"name":"envoy-prometheus-metrics-listener"},{"additionalAddresses":[{"address":{"socketAddress":{"address":"::1","portValue":9878}}}],"address":{"socketAddress":{"address":"127.0.0.1","portValue":9878}},"filterChains":[{"filters":[{"name":"envoy.filters.network.http_connection_manager","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager","httpFilters":[{"name":"envoy.filters.http.router","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"}}],"internalAddressConfig":{"cidrRanges":[{"addressPrefix":"10.0.0.0","prefixLen":8},{"addressPrefix":"172.16.0.0","prefixLen":12},{"addressPrefix":"192.168.0.0","prefixLen":16},{"addressPrefix":"127.0.0.1","prefixLen":32},{"addressPrefix":"::1","prefixLen":128}]},"routeConfig":{"virtual_hosts":[{"domains":["*"],"name":"health","routes":[{"match":{"prefix":"/healthz"},"name":"health","route":{"cluster":"/envoy-admin","prefixRewrite":"/ready"}}]}]},"statPrefix":"envoy-health-listener","streamIdleTimeout":"300s"}}]}],"name":"envoy-health-listener"}]}} ---- -# Source: cilium/templates/cilium-agent/clusterrole.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: cilium - labels: - app.kubernetes.io/part-of: cilium -rules: -- apiGroups: - - networking.k8s.io - resources: - - networkpolicies - verbs: - - get - - list - - watch -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - namespaces - - services - - pods - - endpoints - - nodes - verbs: - - get - - list - - watch -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - list - - watch - # This is used when validating policies in preflight. This will need to stay - # until we figure out how to avoid "get" inside the preflight, and then - # should be removed ideally. - - get -- apiGroups: - - cilium.io - resources: - - ciliumloadbalancerippools - - ciliumbgppeeringpolicies - - ciliumbgpnodeconfigs - - ciliumbgpadvertisements - - ciliumbgppeerconfigs - - ciliumclusterwideenvoyconfigs - - ciliumclusterwidenetworkpolicies - - ciliumegressgatewaypolicies - - ciliumendpoints - - ciliumendpointslices - - ciliumenvoyconfigs - - ciliumidentities - - ciliumlocalredirectpolicies - - ciliumnetworkpolicies - - ciliumnodes - - ciliumnodeconfigs - - ciliumcidrgroups - - ciliuml2announcementpolicies - - ciliumpodippools - verbs: - - list - - watch -- apiGroups: - - cilium.io - resources: - - ciliumidentities - - ciliumendpoints - - ciliumnodes - verbs: - - create -- apiGroups: - - cilium.io - # To synchronize garbage collection of such resources - resources: - - ciliumidentities - verbs: - - update -- apiGroups: - - cilium.io - resources: - - ciliumendpoints - verbs: - - delete - - get -- apiGroups: - - cilium.io - resources: - - ciliumnodes - - ciliumnodes/status - verbs: - - get - - update -- apiGroups: - - cilium.io - resources: - - ciliumendpoints/status - - ciliumendpoints - - ciliuml2announcementpolicies/status - - ciliumbgpnodeconfigs/status - verbs: - - patch ---- -# Source: cilium/templates/cilium-operator/clusterrole.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: cilium-operator - labels: - app.kubernetes.io/part-of: cilium -rules: -- apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - watch - # to automatically delete [core|kube]dns pods so that are starting to being - # managed by Cilium - - delete -- apiGroups: - - "" - resources: - - configmaps - resourceNames: - - cilium-config - verbs: - # allow patching of the configmap to set annotations - - patch -- apiGroups: - - "" - resources: - - nodes - verbs: - - list - - watch -- apiGroups: - - "" - resources: - # To remove node taints - - nodes - # To set NetworkUnavailable false on startup - - nodes/status - verbs: - - patch -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - # to perform LB IP allocation for BGP - - services/status - verbs: - - update - - patch -- apiGroups: - - "" - resources: - # to check apiserver connectivity - - namespaces - - secrets - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - # to perform the translation of a CNP that contains `ToGroup` to its endpoints - - services - - endpoints - verbs: - - get - - list - - watch -- apiGroups: - - cilium.io - resources: - - ciliumnetworkpolicies - - ciliumclusterwidenetworkpolicies - verbs: - # Create auto-generated CNPs and CCNPs from Policies that have 'toGroups' - - create - - update - - deletecollection - # To update the status of the CNPs and CCNPs - - patch - - get - - list - - watch -- apiGroups: - - cilium.io - resources: - - ciliumnetworkpolicies/status - - ciliumclusterwidenetworkpolicies/status - verbs: - # Update the auto-generated CNPs and CCNPs status. - - patch - - update -- apiGroups: - - cilium.io - resources: - - ciliumendpoints - - ciliumidentities - verbs: - # To perform garbage collection of such resources - - delete - - list - - watch -- apiGroups: - - cilium.io - resources: - - ciliumidentities - verbs: - # To synchronize garbage collection of such resources - - update -- apiGroups: - - cilium.io - resources: - - ciliumnodes - verbs: - - create - - update - - get - - list - - watch - # To perform CiliumNode garbage collector - - delete -- apiGroups: - - cilium.io - resources: - - ciliumnodes/status - verbs: - - update -- apiGroups: - - cilium.io - resources: - - ciliumendpointslices - - ciliumenvoyconfigs - - ciliumbgppeerconfigs - - ciliumbgpadvertisements - - ciliumbgpnodeconfigs - verbs: - - create - - update - - get - - list - - watch - - delete - - patch -- apiGroups: - - cilium.io - resources: - - ciliumbgpclusterconfigs/status - - ciliumbgppeerconfigs/status - verbs: - - update -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - create - - get - - list - - watch -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - update - resourceNames: - - ciliumloadbalancerippools.cilium.io - - ciliumbgppeeringpolicies.cilium.io - - ciliumbgpclusterconfigs.cilium.io - - ciliumbgppeerconfigs.cilium.io - - ciliumbgpadvertisements.cilium.io - - ciliumbgpnodeconfigs.cilium.io - - ciliumbgpnodeconfigoverrides.cilium.io - - ciliumclusterwideenvoyconfigs.cilium.io - - ciliumclusterwidenetworkpolicies.cilium.io - - ciliumegressgatewaypolicies.cilium.io - - ciliumendpoints.cilium.io - - ciliumendpointslices.cilium.io - - ciliumenvoyconfigs.cilium.io - - ciliumidentities.cilium.io - - ciliumlocalredirectpolicies.cilium.io - - ciliumnetworkpolicies.cilium.io - - ciliumnodes.cilium.io - - ciliumnodeconfigs.cilium.io - - ciliumcidrgroups.cilium.io - - ciliuml2announcementpolicies.cilium.io - - ciliumpodippools.cilium.io - - ciliumgatewayclassconfigs.cilium.io -- apiGroups: - - cilium.io - resources: - - ciliumloadbalancerippools - - ciliumpodippools - - ciliumbgppeeringpolicies - - ciliumbgpclusterconfigs - - ciliumbgpnodeconfigoverrides - - ciliumbgppeerconfigs - verbs: - - get - - list - - watch -- apiGroups: - - cilium.io - resources: - - ciliumpodippools - verbs: - - create -- apiGroups: - - cilium.io - resources: - - ciliumloadbalancerippools/status - verbs: - - patch -# For cilium-operator running in HA mode. -# -# Cilium operator running in HA mode requires the use of ResourceLock for Leader Election -# between multiple running instances. -# The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less -# common and fewer objects in the cluster watch "all Leases". -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create - - get - - update ---- -# Source: cilium/templates/cilium-agent/clusterrolebinding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: cilium - labels: - app.kubernetes.io/part-of: cilium -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cilium -subjects: -- kind: ServiceAccount - name: "cilium" - namespace: kube-system ---- -# Source: cilium/templates/cilium-operator/clusterrolebinding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: cilium-operator - labels: - app.kubernetes.io/part-of: cilium -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cilium-operator -subjects: -- kind: ServiceAccount - name: "cilium-operator" - namespace: kube-system ---- -# Source: cilium/templates/cilium-agent/role.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: cilium-config-agent - namespace: kube-system - labels: - app.kubernetes.io/part-of: cilium -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch ---- -# Source: cilium/templates/cilium-agent/role.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: cilium-tlsinterception-secrets - namespace: "cilium-secrets" - labels: - app.kubernetes.io/part-of: cilium -rules: -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - watch ---- -# Source: cilium/templates/cilium-operator/role.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: cilium-operator-tlsinterception-secrets - namespace: "cilium-secrets" - labels: - app.kubernetes.io/part-of: cilium -rules: -- apiGroups: - - "" - resources: - - secrets - verbs: - - create - - delete - - update - - patch ---- -# Source: cilium/templates/cilium-agent/rolebinding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: cilium-config-agent - namespace: kube-system - labels: - app.kubernetes.io/part-of: cilium -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: cilium-config-agent -subjects: - - kind: ServiceAccount - name: "cilium" - namespace: kube-system ---- -# Source: cilium/templates/cilium-agent/rolebinding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: cilium-tlsinterception-secrets - namespace: "cilium-secrets" - labels: - app.kubernetes.io/part-of: cilium -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: cilium-tlsinterception-secrets -subjects: -- kind: ServiceAccount - name: "cilium" - namespace: kube-system ---- -# Source: cilium/templates/cilium-operator/rolebinding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: cilium-operator-tlsinterception-secrets - namespace: "cilium-secrets" - labels: - app.kubernetes.io/part-of: cilium -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: cilium-operator-tlsinterception-secrets -subjects: -- kind: ServiceAccount - name: "cilium-operator" - namespace: kube-system ---- -# Source: cilium/templates/cilium-envoy/service.yaml -apiVersion: v1 -kind: Service -metadata: - name: cilium-envoy - namespace: kube-system - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9964" - labels: - k8s-app: cilium-envoy - app.kubernetes.io/name: cilium-envoy - app.kubernetes.io/part-of: cilium - io.cilium/app: proxy -spec: - clusterIP: None - type: ClusterIP - selector: - k8s-app: cilium-envoy - ports: - - name: envoy-metrics - port: 9964 - protocol: TCP - targetPort: envoy-metrics ---- -# Source: cilium/templates/hubble/peer-service.yaml -apiVersion: v1 -kind: Service -metadata: - name: hubble-peer - namespace: kube-system - labels: - k8s-app: cilium - app.kubernetes.io/part-of: cilium - app.kubernetes.io/name: hubble-peer - -spec: - selector: - k8s-app: cilium - ports: - - name: peer-service - port: 443 - protocol: TCP - targetPort: 4244 - internalTrafficPolicy: Local ---- -# Source: cilium/templates/cilium-agent/daemonset.yaml -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: cilium - namespace: kube-system - labels: - k8s-app: cilium - app.kubernetes.io/part-of: cilium - app.kubernetes.io/name: cilium-agent -spec: - selector: - matchLabels: - k8s-app: cilium - updateStrategy: - rollingUpdate: - maxUnavailable: 2 - type: RollingUpdate - template: - metadata: - annotations: - kubectl.kubernetes.io/default-container: cilium-agent - labels: - k8s-app: cilium - app.kubernetes.io/name: cilium-agent - app.kubernetes.io/part-of: cilium - spec: - securityContext: - appArmorProfile: - type: Unconfined - seccompProfile: - type: Unconfined - containers: - - name: cilium-agent - image: "quay.io/cilium/cilium:v1.18.5@sha256:2c92fb05962a346eaf0ce11b912ba434dc10bd54b9989e970416681f4a069628" - imagePullPolicy: IfNotPresent - command: - - cilium-agent - args: - - --config-dir=/tmp/cilium/config-map - startupProbe: - httpGet: - host: "127.0.0.1" - path: /healthz - port: 9879 - scheme: HTTP - httpHeaders: - - name: "brief" - value: "true" - failureThreshold: 300 - periodSeconds: 2 - successThreshold: 1 - initialDelaySeconds: 5 - livenessProbe: - httpGet: - host: "127.0.0.1" - path: /healthz - port: 9879 - scheme: HTTP - httpHeaders: - - name: "brief" - value: "true" - - name: "require-k8s-connectivity" - value: "false" - periodSeconds: 30 - successThreshold: 1 - failureThreshold: 10 - timeoutSeconds: 5 - readinessProbe: - httpGet: - host: "127.0.0.1" - path: /healthz - port: 9879 - scheme: HTTP - httpHeaders: - - name: "brief" - value: "true" - periodSeconds: 30 - successThreshold: 1 - failureThreshold: 3 - timeoutSeconds: 5 - env: - - name: K8S_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: CILIUM_K8S_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: CILIUM_CLUSTERMESH_CONFIG - value: /var/lib/cilium/clustermesh/ - - name: GOMEMLIMIT - valueFrom: - resourceFieldRef: - resource: limits.memory - divisor: '1' - - name: KUBERNETES_SERVICE_HOST - value: "2620:11f:7001:7:ffff:ffff:ad7:1dd" - - name: KUBERNETES_SERVICE_PORT - value: "6443" - - name: KUBE_CLIENT_BACKOFF_BASE - value: "1" - - name: KUBE_CLIENT_BACKOFF_DURATION - value: "120" - lifecycle: - postStart: - exec: - command: - - "bash" - - "-c" - - | - set -o errexit - set -o pipefail - set -o nounset - - # When running in AWS ENI mode, it's likely that 'aws-node' has - # had a chance to install SNAT iptables rules. These can result - # in dropped traffic, so we should attempt to remove them. - # We do it using a 'postStart' hook since this may need to run - # for nodes which might have already been init'ed but may still - # have dangling rules. This is safe because there are no - # dependencies on anything that is part of the startup script - # itself, and can be safely run multiple times per node (e.g. in - # case of a restart). - if [[ "$(iptables-save | grep -E -c 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN')" != "0" ]]; - then - echo 'Deleting iptables rules created by the AWS CNI VPC plugin' - iptables-save | grep -E -v 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN' | iptables-restore - fi - echo 'Done!' - - preStop: - exec: - command: - - /cni-uninstall.sh - securityContext: - seLinuxOptions: - level: s0 - type: spc_t - capabilities: - add: - - CHOWN - - KILL - - NET_ADMIN - - NET_RAW - - IPC_LOCK - - SYS_MODULE - - SYS_ADMIN - - SYS_RESOURCE - - DAC_OVERRIDE - - FOWNER - - SETGID - - SETUID - drop: - - ALL - terminationMessagePolicy: FallbackToLogsOnError - volumeMounts: - - name: envoy-sockets - mountPath: /var/run/cilium/envoy/sockets - readOnly: false - # Unprivileged containers need to mount /proc/sys/net from the host - # to have write access - - mountPath: /host/proc/sys/net - name: host-proc-sys-net - # Unprivileged containers need to mount /proc/sys/kernel from the host - # to have write access - - mountPath: /host/proc/sys/kernel - name: host-proc-sys-kernel - - name: bpf-maps - mountPath: /sys/fs/bpf - # Unprivileged containers can't set mount propagation to bidirectional - # in this case we will mount the bpf fs from an init container that - # is privileged and set the mount propagation from host to container - # in Cilium. - mountPropagation: HostToContainer - - name: cilium-run - mountPath: /var/run/cilium - - name: cilium-netns - mountPath: /var/run/cilium/netns - mountPropagation: HostToContainer - - name: etc-cni-netd - mountPath: /host/etc/cni/net.d - - name: clustermesh-secrets - mountPath: /var/lib/cilium/clustermesh - readOnly: true - # Needed to be able to load kernel modules - - name: lib-modules - mountPath: /lib/modules - readOnly: true - - name: xtables-lock - mountPath: /run/xtables.lock - - name: hubble-tls - mountPath: /var/lib/cilium/tls/hubble - readOnly: true - - name: tmp - mountPath: /tmp - - initContainers: - - name: config - image: "quay.io/cilium/cilium:v1.18.5@sha256:2c92fb05962a346eaf0ce11b912ba434dc10bd54b9989e970416681f4a069628" - imagePullPolicy: IfNotPresent - command: - - cilium-dbg - - build-config - env: - - name: K8S_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: CILIUM_K8S_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: KUBERNETES_SERVICE_HOST - value: "2620:11f:7001:7:ffff:ffff:ad7:1dd" - - name: KUBERNETES_SERVICE_PORT - value: "6443" - volumeMounts: - - name: tmp - mountPath: /tmp - terminationMessagePolicy: FallbackToLogsOnError - # Required to mount cgroup2 filesystem on the underlying Kubernetes node. - # We use nsenter command with host's cgroup and mount namespaces enabled. - - name: mount-cgroup - image: "quay.io/cilium/cilium:v1.18.5@sha256:2c92fb05962a346eaf0ce11b912ba434dc10bd54b9989e970416681f4a069628" - imagePullPolicy: IfNotPresent - env: - - name: CGROUP_ROOT - value: /run/cilium/cgroupv2 - - name: BIN_PATH - value: /opt/cni/bin - command: - - sh - - -ec - # The statically linked Go program binary is invoked to avoid any - # dependency on utilities like sh and mount that can be missing on certain - # distros installed on the underlying host. Copy the binary to the - # same directory where we install cilium cni plugin so that exec permissions - # are available. - - | - cp /usr/bin/cilium-mount /hostbin/cilium-mount; - nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-mount" $CGROUP_ROOT; - rm /hostbin/cilium-mount - volumeMounts: - - name: hostproc - mountPath: /hostproc - - name: cni-path - mountPath: /hostbin - terminationMessagePolicy: FallbackToLogsOnError - securityContext: - seLinuxOptions: - level: s0 - type: spc_t - capabilities: - add: - - SYS_ADMIN - - SYS_CHROOT - - SYS_PTRACE - drop: - - ALL - - name: apply-sysctl-overwrites - image: "quay.io/cilium/cilium:v1.18.5@sha256:2c92fb05962a346eaf0ce11b912ba434dc10bd54b9989e970416681f4a069628" - imagePullPolicy: IfNotPresent - env: - - name: BIN_PATH - value: /opt/cni/bin - command: - - sh - - -ec - # The statically linked Go program binary is invoked to avoid any - # dependency on utilities like sh that can be missing on certain - # distros installed on the underlying host. Copy the binary to the - # same directory where we install cilium cni plugin so that exec permissions - # are available. - - | - cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix; - nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix"; - rm /hostbin/cilium-sysctlfix - volumeMounts: - - name: hostproc - mountPath: /hostproc - - name: cni-path - mountPath: /hostbin - terminationMessagePolicy: FallbackToLogsOnError - securityContext: - seLinuxOptions: - level: s0 - type: spc_t - capabilities: - add: - - SYS_ADMIN - - SYS_CHROOT - - SYS_PTRACE - drop: - - ALL - # Mount the bpf fs if it is not mounted. We will perform this task - # from a privileged container because the mount propagation bidirectional - # only works from privileged containers. - - name: mount-bpf-fs - image: "quay.io/cilium/cilium:v1.18.5@sha256:2c92fb05962a346eaf0ce11b912ba434dc10bd54b9989e970416681f4a069628" - imagePullPolicy: IfNotPresent - args: - - 'mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf' - command: - - /bin/bash - - -c - - -- - terminationMessagePolicy: FallbackToLogsOnError - securityContext: - privileged: true - volumeMounts: - - name: bpf-maps - mountPath: /sys/fs/bpf - mountPropagation: Bidirectional - - name: clean-cilium-state - image: "quay.io/cilium/cilium:v1.18.5@sha256:2c92fb05962a346eaf0ce11b912ba434dc10bd54b9989e970416681f4a069628" - imagePullPolicy: IfNotPresent - command: - - /init-container.sh - env: - - name: CILIUM_ALL_STATE - valueFrom: - configMapKeyRef: - name: cilium-config - key: clean-cilium-state - optional: true - - name: CILIUM_BPF_STATE - valueFrom: - configMapKeyRef: - name: cilium-config - key: clean-cilium-bpf-state - optional: true - - name: WRITE_CNI_CONF_WHEN_READY - valueFrom: - configMapKeyRef: - name: cilium-config - key: write-cni-conf-when-ready - optional: true - - name: KUBERNETES_SERVICE_HOST - value: "2620:11f:7001:7:ffff:ffff:ad7:1dd" - - name: KUBERNETES_SERVICE_PORT - value: "6443" - terminationMessagePolicy: FallbackToLogsOnError - securityContext: - seLinuxOptions: - level: s0 - type: spc_t - capabilities: - add: - - NET_ADMIN - - SYS_MODULE - - SYS_ADMIN - - SYS_RESOURCE - drop: - - ALL - volumeMounts: - - name: bpf-maps - mountPath: /sys/fs/bpf - # Required to mount cgroup filesystem from the host to cilium agent pod - - name: cilium-cgroup - mountPath: /run/cilium/cgroupv2 - mountPropagation: HostToContainer - - name: cilium-run - mountPath: /var/run/cilium # wait-for-kube-proxy - # Install the CNI binaries in an InitContainer so we don't have a writable host mount in the agent - - name: install-cni-binaries - image: "quay.io/cilium/cilium:v1.18.5@sha256:2c92fb05962a346eaf0ce11b912ba434dc10bd54b9989e970416681f4a069628" - imagePullPolicy: IfNotPresent - command: - - "/install-plugin.sh" - resources: - requests: - cpu: 100m - memory: 10Mi - securityContext: - seLinuxOptions: - level: s0 - type: spc_t - capabilities: - drop: - - ALL - terminationMessagePolicy: FallbackToLogsOnError - volumeMounts: - - name: cni-path - mountPath: /host/opt/cni/bin # .Values.cni.install - restartPolicy: Always - priorityClassName: system-node-critical - serviceAccountName: "cilium" - automountServiceAccountToken: true - terminationGracePeriodSeconds: 1 - hostNetwork: true - - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - k8s-app: cilium - topologyKey: kubernetes.io/hostname - nodeSelector: - kubernetes.io/os: linux - tolerations: - - operator: Exists - volumes: - # For sharing configuration between the "config" initContainer and the agent - - name: tmp - emptyDir: {} - # To keep state between restarts / upgrades - - name: cilium-run - hostPath: - path: /var/run/cilium - type: DirectoryOrCreate - # To exec into pod network namespaces - - name: cilium-netns - hostPath: - path: /var/run/netns - type: DirectoryOrCreate - # To keep state between restarts / upgrades for bpf maps - - name: bpf-maps - hostPath: - path: /sys/fs/bpf - type: DirectoryOrCreate - # To mount cgroup2 filesystem on the host or apply sysctlfix - - name: hostproc - hostPath: - path: /proc - type: Directory - # To keep state between restarts / upgrades for cgroup2 filesystem - - name: cilium-cgroup - hostPath: - path: /run/cilium/cgroupv2 - type: DirectoryOrCreate - # To install cilium cni plugin in the host - - name: cni-path - hostPath: - path: /opt/cni/bin - type: DirectoryOrCreate - # To install cilium cni configuration in the host - - name: etc-cni-netd - hostPath: - path: /etc/cni/net.d - type: DirectoryOrCreate - # To be able to load kernel modules - - name: lib-modules - hostPath: - path: /lib/modules - # To access iptables concurrently with other processes (e.g. kube-proxy) - - name: xtables-lock - hostPath: - path: /run/xtables.lock - type: FileOrCreate - # Sharing socket with Cilium Envoy on the same node by using a host path - - name: envoy-sockets - hostPath: - path: "/var/run/cilium/envoy/sockets" - type: DirectoryOrCreate - # To read the clustermesh configuration - - name: clustermesh-secrets - projected: - # note: the leading zero means this number is in octal representation: do not remove it - defaultMode: 0400 - sources: - - secret: - name: cilium-clustermesh - optional: true - # note: items are not explicitly listed here, since the entries of this secret - # depend on the peers configured, and that would cause a restart of all agents - # at every addition/removal. Leaving the field empty makes each secret entry - # to be automatically projected into the volume as a file whose name is the key. - - secret: - name: clustermesh-apiserver-remote-cert - optional: true - items: - - key: tls.key - path: common-etcd-client.key - - key: tls.crt - path: common-etcd-client.crt - - key: ca.crt - path: common-etcd-client-ca.crt - # note: we configure the volume for the kvstoremesh-specific certificate - # regardless of whether KVStoreMesh is enabled or not, so that it can be - # automatically mounted in case KVStoreMesh gets subsequently enabled, - # without requiring an agent restart. - - secret: - name: clustermesh-apiserver-local-cert - optional: true - items: - - key: tls.key - path: local-etcd-client.key - - key: tls.crt - path: local-etcd-client.crt - - key: ca.crt - path: local-etcd-client-ca.crt - - name: host-proc-sys-net - hostPath: - path: /proc/sys/net - type: Directory - - name: host-proc-sys-kernel - hostPath: - path: /proc/sys/kernel - type: Directory - - name: hubble-tls - projected: - # note: the leading zero means this number is in octal representation: do not remove it - defaultMode: 0400 - sources: - - secret: - name: hubble-server-certs - optional: true - items: - - key: tls.crt - path: server.crt - - key: tls.key - path: server.key - - key: ca.crt - path: client-ca.crt ---- -# Source: cilium/templates/cilium-envoy/daemonset.yaml -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: cilium-envoy - namespace: kube-system - labels: - k8s-app: cilium-envoy - app.kubernetes.io/part-of: cilium - app.kubernetes.io/name: cilium-envoy - name: cilium-envoy -spec: - selector: - matchLabels: - k8s-app: cilium-envoy - updateStrategy: - rollingUpdate: - maxUnavailable: 2 - type: RollingUpdate - template: - metadata: - annotations: - labels: - k8s-app: cilium-envoy - name: cilium-envoy - app.kubernetes.io/name: cilium-envoy - app.kubernetes.io/part-of: cilium - spec: - securityContext: - appArmorProfile: - type: Unconfined - containers: - - name: cilium-envoy - image: "quay.io/cilium/cilium-envoy:v1.34.12-1765374555-6a93b0bbba8d6dc75b651cbafeedb062b2997716@sha256:3108521821c6922695ff1f6ef24b09026c94b195283f8bfbfc0fa49356a156e1" - imagePullPolicy: IfNotPresent - command: - - /usr/bin/cilium-envoy-starter - args: - - '--' - - '-c /var/run/cilium/envoy/bootstrap-config.json' - - '--base-id 0' - - '--log-level info' - startupProbe: - httpGet: - host: "127.0.0.1" - path: /healthz - port: 9878 - scheme: HTTP - failureThreshold: 105 - periodSeconds: 2 - successThreshold: 1 - initialDelaySeconds: 5 - livenessProbe: - httpGet: - host: "127.0.0.1" - path: /healthz - port: 9878 - scheme: HTTP - periodSeconds: 30 - successThreshold: 1 - failureThreshold: 10 - timeoutSeconds: 5 - readinessProbe: - httpGet: - host: "127.0.0.1" - path: /healthz - port: 9878 - scheme: HTTP - periodSeconds: 30 - successThreshold: 1 - failureThreshold: 3 - timeoutSeconds: 5 - env: - - name: K8S_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: CILIUM_K8S_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: KUBERNETES_SERVICE_HOST - value: "2620:11f:7001:7:ffff:ffff:ad7:1dd" - - name: KUBERNETES_SERVICE_PORT - value: "6443" - ports: - - name: envoy-metrics - containerPort: 9964 - hostPort: 9964 - protocol: TCP - securityContext: - seLinuxOptions: - level: s0 - type: spc_t - capabilities: - add: - - NET_ADMIN - - SYS_ADMIN - drop: - - ALL - terminationMessagePolicy: FallbackToLogsOnError - volumeMounts: - - name: envoy-sockets - mountPath: /var/run/cilium/envoy/sockets - readOnly: false - - name: envoy-artifacts - mountPath: /var/run/cilium/envoy/artifacts - readOnly: true - - name: envoy-config - mountPath: /var/run/cilium/envoy/ - readOnly: true - - name: bpf-maps - mountPath: /sys/fs/bpf - mountPropagation: HostToContainer - restartPolicy: Always - priorityClassName: system-node-critical - serviceAccountName: "cilium-envoy" - automountServiceAccountToken: true - terminationGracePeriodSeconds: 1 - hostNetwork: true - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: cilium.io/no-schedule - operator: NotIn - values: - - "true" - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - k8s-app: cilium - topologyKey: kubernetes.io/hostname - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - k8s-app: cilium-envoy - topologyKey: kubernetes.io/hostname - nodeSelector: - kubernetes.io/os: linux - tolerations: - - operator: Exists - volumes: - - name: envoy-sockets - hostPath: - path: "/var/run/cilium/envoy/sockets" - type: DirectoryOrCreate - - name: envoy-artifacts - hostPath: - path: "/var/run/cilium/envoy/artifacts" - type: DirectoryOrCreate - - name: envoy-config - configMap: - name: "cilium-envoy-config" - # note: the leading zero means this number is in octal representation: do not remove it - defaultMode: 0400 - items: - - key: bootstrap-config.json - path: bootstrap-config.json - # To keep state between restarts / upgrades - # To keep state between restarts / upgrades for bpf maps - - name: bpf-maps - hostPath: - path: /sys/fs/bpf - type: DirectoryOrCreate ---- -# Source: cilium/templates/cilium-operator/deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: cilium-operator - namespace: kube-system - labels: - io.cilium/app: operator - name: cilium-operator - app.kubernetes.io/part-of: cilium - app.kubernetes.io/name: cilium-operator -spec: - # See docs on ServerCapabilities.LeasesResourceLock in file pkg/k8s/version/version.go - # for more details. - replicas: 2 - selector: - matchLabels: - io.cilium/app: operator - name: cilium-operator - # ensure operator update on single node k8s clusters, by using rolling update with maxUnavailable=100% in case - # of one replica and no user configured Recreate strategy. - # otherwise an update might get stuck due to the default maxUnavailable=50% in combination with the - # podAntiAffinity which prevents deployments of multiple operator replicas on the same node. - strategy: - rollingUpdate: - maxSurge: 25% - maxUnavailable: 50% - type: RollingUpdate - template: - metadata: - annotations: - prometheus.io/port: "9963" - prometheus.io/scrape: "true" - labels: - io.cilium/app: operator - name: cilium-operator - app.kubernetes.io/part-of: cilium - app.kubernetes.io/name: cilium-operator - spec: - securityContext: - seccompProfile: - type: RuntimeDefault - containers: - - name: cilium-operator - image: "quay.io/cilium/operator-generic:v1.18.5@sha256:36c3f6f14c8ced7f45b40b0a927639894b44269dd653f9528e7a0dc363a4eb99" - imagePullPolicy: IfNotPresent - command: - - cilium-operator-generic - args: - - --config-dir=/tmp/cilium/config-map - - --debug=$(CILIUM_DEBUG) - env: - - name: K8S_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: CILIUM_K8S_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: CILIUM_DEBUG - valueFrom: - configMapKeyRef: - key: debug - name: cilium-config - optional: true - - name: KUBERNETES_SERVICE_HOST - value: "2620:11f:7001:7:ffff:ffff:ad7:1dd" - - name: KUBERNETES_SERVICE_PORT - value: "6443" - ports: - - name: prometheus - containerPort: 9963 - hostPort: 9963 - protocol: TCP - livenessProbe: - httpGet: - host: "127.0.0.1" - path: /healthz - port: 9234 - scheme: HTTP - initialDelaySeconds: 60 - periodSeconds: 10 - timeoutSeconds: 3 - readinessProbe: - httpGet: - host: "127.0.0.1" - path: /healthz - port: 9234 - scheme: HTTP - initialDelaySeconds: 0 - periodSeconds: 5 - timeoutSeconds: 3 - failureThreshold: 5 - volumeMounts: - - name: cilium-config-path - mountPath: /tmp/cilium/config-map - readOnly: true - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - terminationMessagePolicy: FallbackToLogsOnError - hostNetwork: true - restartPolicy: Always - priorityClassName: system-cluster-critical - serviceAccountName: "cilium-operator" - automountServiceAccountToken: true - # In HA mode, cilium-operator pods must not be scheduled on the same - # node as they will clash with each other. - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - io.cilium/app: operator - topologyKey: kubernetes.io/hostname - nodeSelector: - kubernetes.io/os: linux - tolerations: - - key: node-role.kubernetes.io/control-plane - operator: Exists - - key: node-role.kubernetes.io/master - operator: Exists - - key: node.kubernetes.io/not-ready - operator: Exists - - key: node.cloudprovider.kubernetes.io/uninitialized - operator: Exists - - key: node.cilium.io/agent-not-ready - operator: Exists - - volumes: - # To read the configuration from the config map - - name: cilium-config-path - configMap: - name: cilium-config diff --git a/nix/kubernetes/keys/package/bootstrap-script/package.nix b/nix/kubernetes/keys/package/bootstrap-script/package.nix index 298ac62..9190b7a 100644 --- a/nix/kubernetes/keys/package/bootstrap-script/package.nix +++ b/nix/kubernetes/keys/package/bootstrap-script/package.nix @@ -8,12 +8,10 @@ # installCheckPhase # distPhase { - config, lib, stdenv, writeShellScript, k8s, - openssh, ... }: let @@ -30,7 +28,7 @@ let lib.concatMapStringsSep "," lib.escapeShellArg ( [ ./files/manifests/initial_clusterrole.yaml - ./files/manifests/cilium.yaml + "${k8s.cilium-manifest}/cilium.yaml" ./files/manifests/coredns.yaml ./files/manifests/flux_namespace.yaml ./files/manifests/flux.yaml diff --git a/nix/kubernetes/keys/package/cilium-manifest/package.nix b/nix/kubernetes/keys/package/cilium-manifest/package.nix new file mode 100644 index 0000000..f8c7a09 --- /dev/null +++ b/nix/kubernetes/keys/package/cilium-manifest/package.nix @@ -0,0 +1,70 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + stdenv, + openssl, + fetchFromGitHub, + kubernetes-helm, + ... +}: +stdenv.mkDerivation ( + finalAttrs: + let + version = "1.18.5"; + in + { + name = "cilium-manifest"; + nativeBuildInputs = [ + openssl + kubernetes-helm + ]; + buildInputs = [ ]; + + src = fetchFromGitHub { + owner = "cilium"; + repo = "cilium"; + tag = "v${version}"; + hash = "sha256-348inOOQ/fgwTYnaSHrQ363xGYnx2UPts3D4ycDRsWE="; + }; + + buildPhase = '' + helm template --dry-run=client cilium $src/install/kubernetes/cilium --version 1.18.5 --namespace kube-system \ + --set kubeProxyReplacement=true \ + --set ipam.mode=kubernetes \ + --set k8sServiceHost="2620:11f:7001:7:ffff:ffff:ad7:1dd" \ + --set k8sServicePort=6443 \ + --set ipv6.enabled=true \ + --set ipv4.enabled=true \ + --set enableIPv6Masquerade=false \ + | tee $NIX_BUILD_TOP/cilium.yaml + ''; + + # --set enableIPv4BIGTCP=false \ + # --set enableIPv6BIGTCP=false \ + # --set routingMode=native \ + # --set ipv4NativeRoutingCIDR=10.0.0.0/8 \ + # --set ipv6NativeRoutingCIDR=2620:11f:7001:7:ffff::/96 \ + + # --set hostFirewall.enabled=true + # --set routingMode=native + + # --set 'ipam.operator.clusterPoolIPv4PodCIDRList=["10.0.0.0/8"]' \ + # --set 'ipam.operator.clusterPoolIPv6PodCIDRList=["fd00::/100"]' \ + + # --set encryption.enabled=true \ + # --set encryption.type=wireguard + # --set encryption.nodeEncryption=true + + installPhase = '' + mkdir -p "$out" + cp $NIX_BUILD_TOP/cilium.yaml $out/ + ''; + } +) diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index 0aa2695..784b20c 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -207,6 +207,7 @@ makeScope newScope ( } ); encryption_config = (callPackage ./package/k8s-encryption-key/package.nix additional_vars); + cilium-manifest = (callPackage ./package/cilium-manifest/package.nix additional_vars); all_keys = (callPackage ./package/k8s-keys/package.nix additional_vars); deploy_script = (callPackage ./package/deploy-script/package.nix additional_vars); bootstrap_script = (callPackage ./package/bootstrap-script/package.nix additional_vars); diff --git a/nix/kubernetes/roles/network/default.nix b/nix/kubernetes/roles/network/default.nix index c75d0d1..a25e70d 100644 --- a/nix/kubernetes/roles/network/default.nix +++ b/nix/kubernetes/roles/network/default.nix @@ -30,16 +30,21 @@ config = lib.mkIf config.me.network.enable { networking.dhcpcd.enable = lib.mkDefault false; networking.useDHCP = lib.mkDefault false; + # Nameservers configured in host-specific files. + # networking.nameservers = [ + # "194.242.2.2#doh.mullvad.net" + # "2a07:e340::2#doh.mullvad.net" + # ]; networking.nameservers = [ - "194.242.2.2#doh.mullvad.net" - "2a07:e340::2#doh.mullvad.net" + "10.215.1.1" + "2620:11f:7001:7:ffff:ffff:0ad7:0101" ]; services.resolved = { enable = true; # dnssec = "true"; domains = [ "~." ]; fallbackDns = [ ]; - dnsovertls = "true"; + # dnsovertls = "true"; }; # Without this, systemd-resolved will send DNS requests for .home.arpa to the per-link DNS server (172.16.0.1) which does not support DNS-over-TLS. This leads to the connection hanging and timing out. This causes firefox startup to take an extra 10+ seconds. From 5d58d550d364850040fe11a20a37869b2cad8eb4 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 29 Dec 2025 20:45:01 -0500 Subject: [PATCH 42/81] Enable native routing. --- .../keys/package/cilium-manifest/package.nix | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/nix/kubernetes/keys/package/cilium-manifest/package.nix b/nix/kubernetes/keys/package/cilium-manifest/package.nix index f8c7a09..e52470f 100644 --- a/nix/kubernetes/keys/package/cilium-manifest/package.nix +++ b/nix/kubernetes/keys/package/cilium-manifest/package.nix @@ -43,15 +43,14 @@ stdenv.mkDerivation ( --set ipv6.enabled=true \ --set ipv4.enabled=true \ --set enableIPv6Masquerade=false \ + --set enableIPv4BIGTCP=false \ + --set enableIPv6BIGTCP=false \ + --set routingMode=native \ + --set ipv4NativeRoutingCIDR=10.0.0.0/8 \ + --set ipv6NativeRoutingCIDR=2620:11f:7001:7:ffff::/96 \ | tee $NIX_BUILD_TOP/cilium.yaml ''; - # --set enableIPv4BIGTCP=false \ - # --set enableIPv6BIGTCP=false \ - # --set routingMode=native \ - # --set ipv4NativeRoutingCIDR=10.0.0.0/8 \ - # --set ipv6NativeRoutingCIDR=2620:11f:7001:7:ffff::/96 \ - # --set hostFirewall.enabled=true # --set routingMode=native From dc80c06ee9009f4e43501482a7e3c6f860e9ce76 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 29 Dec 2025 21:15:29 -0500 Subject: [PATCH 43/81] Use CoreDNS for in-cluster DNS requests and caching. --- .../package/bootstrap-script/files/manifests/coredns.yaml | 5 ++++- nix/kubernetes/roles/kubelet/files/kubelet-config.yaml | 7 ++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/coredns.yaml b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/coredns.yaml index 1f12a63..4b4bdf5 100644 --- a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/coredns.yaml +++ b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/coredns.yaml @@ -201,8 +201,11 @@ metadata: spec: selector: k8s-app: kube-dns - # clusterIP: 10.197.0.10 + ipFamilyPolicy: PreferDualStack clusterIP: "fd00:3e42:e349::10" + clusterIPs: + - "fd00:3e42:e349::10" + - "10.197.0.10" ports: - name: dns port: 53 diff --git a/nix/kubernetes/roles/kubelet/files/kubelet-config.yaml b/nix/kubernetes/roles/kubelet/files/kubelet-config.yaml index c790198..1f5302f 100644 --- a/nix/kubernetes/roles/kubelet/files/kubelet-config.yaml +++ b/nix/kubernetes/roles/kubelet/files/kubelet-config.yaml @@ -23,6 +23,7 @@ registerNode: true runtimeRequestTimeout: "15m" tlsCertFile: "/.persist/keys/kube/kubelet.crt" tlsPrivateKeyFile: "/.persist/keys/kube/kubelet.key" -# clusterDomain: "cluster.local" -# clusterDNS: -# - "10.197.0.10" +clusterDomain: "cluster.local" +clusterDNS: + - "10.197.0.10" + - "fd00:3e42:e349::10" From f545376aec0c4685b44d00f3f94dff51e7458602 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 29 Dec 2025 21:19:50 -0500 Subject: [PATCH 44/81] Switch to generating the coredns manifests via nix. --- .../keys/package/bootstrap-script/package.nix | 2 +- .../keys/package/coredns-manifest/package.nix | 45 +++++++++++++++++++ nix/kubernetes/keys/scope.nix | 1 + 3 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 nix/kubernetes/keys/package/coredns-manifest/package.nix diff --git a/nix/kubernetes/keys/package/bootstrap-script/package.nix b/nix/kubernetes/keys/package/bootstrap-script/package.nix index 9190b7a..eacc1fc 100644 --- a/nix/kubernetes/keys/package/bootstrap-script/package.nix +++ b/nix/kubernetes/keys/package/bootstrap-script/package.nix @@ -29,7 +29,7 @@ let [ ./files/manifests/initial_clusterrole.yaml "${k8s.cilium-manifest}/cilium.yaml" - ./files/manifests/coredns.yaml + "${k8s.coredns-manifest}/coredns.yaml" ./files/manifests/flux_namespace.yaml ./files/manifests/flux.yaml ./files/manifests/flux_instance.yaml diff --git a/nix/kubernetes/keys/package/coredns-manifest/package.nix b/nix/kubernetes/keys/package/coredns-manifest/package.nix new file mode 100644 index 0000000..ef90407 --- /dev/null +++ b/nix/kubernetes/keys/package/coredns-manifest/package.nix @@ -0,0 +1,45 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + stdenv, + fetchFromGitHub, + kubernetes-helm, + ... +}: +stdenv.mkDerivation ( + finalAttrs: + let + version = "1.45.0"; + in + { + name = "coredns-manifest"; + nativeBuildInputs = [ + kubernetes-helm + ]; + buildInputs = [ ]; + + src = fetchFromGitHub { + owner = "coredns"; + repo = "helm"; + tag = "coredns-${version}"; + hash = "sha256-9YHd/jB33JXvySzx/p9DaP+/2p5ucyLjues4DNtOkmU="; + }; + + buildPhase = '' + helm template --dry-run=client coredns $src/charts/coredns --namespace kube-system \ + | tee $NIX_BUILD_TOP/coredns.yaml + ''; + + installPhase = '' + mkdir -p "$out" + cp $NIX_BUILD_TOP/coredns.yaml $out/ + ''; + } +) diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index 784b20c..60650ac 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -208,6 +208,7 @@ makeScope newScope ( ); encryption_config = (callPackage ./package/k8s-encryption-key/package.nix additional_vars); cilium-manifest = (callPackage ./package/cilium-manifest/package.nix additional_vars); + coredns-manifest = (callPackage ./package/coredns-manifest/package.nix additional_vars); all_keys = (callPackage ./package/k8s-keys/package.nix additional_vars); deploy_script = (callPackage ./package/deploy-script/package.nix additional_vars); bootstrap_script = (callPackage ./package/bootstrap-script/package.nix additional_vars); From 361ec69a8bdcb4e4d68f8285fd77798fe4ce5833 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 29 Dec 2025 21:41:23 -0500 Subject: [PATCH 45/81] Implement a generic helm templater package. --- .../keys/package/cilium-manifest/package.nix | 69 ------------------- .../keys/package/coredns-manifest/package.nix | 45 ------------ .../keys/package/helm-manifest/package.nix | 48 +++++++++++++ nix/kubernetes/keys/scope.nix | 65 ++++++++++++++++- 4 files changed, 111 insertions(+), 116 deletions(-) delete mode 100644 nix/kubernetes/keys/package/cilium-manifest/package.nix delete mode 100644 nix/kubernetes/keys/package/coredns-manifest/package.nix create mode 100644 nix/kubernetes/keys/package/helm-manifest/package.nix diff --git a/nix/kubernetes/keys/package/cilium-manifest/package.nix b/nix/kubernetes/keys/package/cilium-manifest/package.nix deleted file mode 100644 index e52470f..0000000 --- a/nix/kubernetes/keys/package/cilium-manifest/package.nix +++ /dev/null @@ -1,69 +0,0 @@ -# unpackPhase -# patchPhase -# configurePhase -# buildPhase -# checkPhase -# installPhase -# fixupPhase -# installCheckPhase -# distPhase -{ - stdenv, - openssl, - fetchFromGitHub, - kubernetes-helm, - ... -}: -stdenv.mkDerivation ( - finalAttrs: - let - version = "1.18.5"; - in - { - name = "cilium-manifest"; - nativeBuildInputs = [ - openssl - kubernetes-helm - ]; - buildInputs = [ ]; - - src = fetchFromGitHub { - owner = "cilium"; - repo = "cilium"; - tag = "v${version}"; - hash = "sha256-348inOOQ/fgwTYnaSHrQ363xGYnx2UPts3D4ycDRsWE="; - }; - - buildPhase = '' - helm template --dry-run=client cilium $src/install/kubernetes/cilium --version 1.18.5 --namespace kube-system \ - --set kubeProxyReplacement=true \ - --set ipam.mode=kubernetes \ - --set k8sServiceHost="2620:11f:7001:7:ffff:ffff:ad7:1dd" \ - --set k8sServicePort=6443 \ - --set ipv6.enabled=true \ - --set ipv4.enabled=true \ - --set enableIPv6Masquerade=false \ - --set enableIPv4BIGTCP=false \ - --set enableIPv6BIGTCP=false \ - --set routingMode=native \ - --set ipv4NativeRoutingCIDR=10.0.0.0/8 \ - --set ipv6NativeRoutingCIDR=2620:11f:7001:7:ffff::/96 \ - | tee $NIX_BUILD_TOP/cilium.yaml - ''; - - # --set hostFirewall.enabled=true - # --set routingMode=native - - # --set 'ipam.operator.clusterPoolIPv4PodCIDRList=["10.0.0.0/8"]' \ - # --set 'ipam.operator.clusterPoolIPv6PodCIDRList=["fd00::/100"]' \ - - # --set encryption.enabled=true \ - # --set encryption.type=wireguard - # --set encryption.nodeEncryption=true - - installPhase = '' - mkdir -p "$out" - cp $NIX_BUILD_TOP/cilium.yaml $out/ - ''; - } -) diff --git a/nix/kubernetes/keys/package/coredns-manifest/package.nix b/nix/kubernetes/keys/package/coredns-manifest/package.nix deleted file mode 100644 index ef90407..0000000 --- a/nix/kubernetes/keys/package/coredns-manifest/package.nix +++ /dev/null @@ -1,45 +0,0 @@ -# unpackPhase -# patchPhase -# configurePhase -# buildPhase -# checkPhase -# installPhase -# fixupPhase -# installCheckPhase -# distPhase -{ - stdenv, - fetchFromGitHub, - kubernetes-helm, - ... -}: -stdenv.mkDerivation ( - finalAttrs: - let - version = "1.45.0"; - in - { - name = "coredns-manifest"; - nativeBuildInputs = [ - kubernetes-helm - ]; - buildInputs = [ ]; - - src = fetchFromGitHub { - owner = "coredns"; - repo = "helm"; - tag = "coredns-${version}"; - hash = "sha256-9YHd/jB33JXvySzx/p9DaP+/2p5ucyLjues4DNtOkmU="; - }; - - buildPhase = '' - helm template --dry-run=client coredns $src/charts/coredns --namespace kube-system \ - | tee $NIX_BUILD_TOP/coredns.yaml - ''; - - installPhase = '' - mkdir -p "$out" - cp $NIX_BUILD_TOP/coredns.yaml $out/ - ''; - } -) diff --git a/nix/kubernetes/keys/package/helm-manifest/package.nix b/nix/kubernetes/keys/package/helm-manifest/package.nix new file mode 100644 index 0000000..9bb6aa8 --- /dev/null +++ b/nix/kubernetes/keys/package/helm-manifest/package.nix @@ -0,0 +1,48 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + lib, + pkgs, + stdenv, + kubernetes-helm, + helm_src, + helm_name, + helm_namespace, + helm_path ? ".", + helm_manifest_name, + helm_values ? { }, + ... +}: +stdenv.mkDerivation ( + finalAttrs: + let + to_yaml_file = ((import ../../../functions/to_yaml.nix) { inherit pkgs; }).to_yaml_file; + in + { + name = "${helm_name}-manifest"; + nativeBuildInputs = [ + kubernetes-helm + ]; + buildInputs = [ ]; + + src = helm_src; + + buildPhase = '' + helm template --dry-run=client ${lib.strings.escapeShellArg helm_name} $src/${helm_path} --namespace ${helm_namespace} \ + --values ${to_yaml_file "values.yaml" helm_values} \ + | tee $NIX_BUILD_TOP/${helm_manifest_name} + ''; + + installPhase = '' + mkdir -p "$out" + cp $NIX_BUILD_TOP/${helm_manifest_name} $out/ + ''; + } +) diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index 60650ac..20fd33a 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -2,6 +2,7 @@ makeScope, newScope, callPackage, + fetchFromGitHub, lib, }: let @@ -207,8 +208,68 @@ makeScope newScope ( } ); encryption_config = (callPackage ./package/k8s-encryption-key/package.nix additional_vars); - cilium-manifest = (callPackage ./package/cilium-manifest/package.nix additional_vars); - coredns-manifest = (callPackage ./package/coredns-manifest/package.nix additional_vars); + cilium-manifest = + let + version = "1.18.5"; + in + (callPackage ./package/helm-manifest/package.nix ( + additional_vars + // { + helm_src = fetchFromGitHub { + owner = "cilium"; + repo = "cilium"; + tag = "v${version}"; + hash = "sha256-348inOOQ/fgwTYnaSHrQ363xGYnx2UPts3D4ycDRsWE="; + }; + helm_name = "cilium"; + helm_namespace = "kube-system"; + helm_path = "install/kubernetes/cilium"; + helm_manifest_name = "cilium.yaml"; + helm_values = { + "kubeProxyReplacement" = true; + "ipam.mode" = "kubernetes"; + "k8sServiceHost" = "2620:11f:7001:7:ffff:ffff:ad7:1dd"; + "k8sServicePort" = 6443; + "ipv6.enabled" = true; + "ipv4.enabled" = true; + "enableIPv6Masquerade" = false; + "enableIPv4BIGTCP" = false; + "enableIPv6BIGTCP" = false; + "routingMode" = "native"; + "ipv4NativeRoutingCIDR" = "10.0.0.0/8"; + "ipv6NativeRoutingCIDR" = "2620:11f:7001:7:ffff::/96"; + # --set hostFirewall.enabled=true + # --set routingMode=native + + # --set 'ipam.operator.clusterPoolIPv4PodCIDRList=["10.0.0.0/8"]' \ + # --set 'ipam.operator.clusterPoolIPv6PodCIDRList=["fd00::/100"]' \ + + # --set encryption.enabled=true \ + # --set encryption.type=wireguard + # --set encryption.nodeEncryption=true + }; + } + )); + coredns-manifest = + let + version = "1.45.0"; + in + (callPackage ./package/helm-manifest/package.nix ( + additional_vars + // { + helm_src = fetchFromGitHub { + owner = "coredns"; + repo = "helm"; + tag = "coredns-${version}"; + hash = "sha256-9YHd/jB33JXvySzx/p9DaP+/2p5ucyLjues4DNtOkmU="; + }; + helm_name = "coredns"; + helm_namespace = "kube-system"; + helm_path = "charts/coredns"; + helm_manifest_name = "coredns.yaml"; + helm_values = { }; + } + )); all_keys = (callPackage ./package/k8s-keys/package.nix additional_vars); deploy_script = (callPackage ./package/deploy-script/package.nix additional_vars); bootstrap_script = (callPackage ./package/bootstrap-script/package.nix additional_vars); From 4061d146510696c359e2fd6cd211296f111090d9 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 29 Dec 2025 21:54:27 -0500 Subject: [PATCH 46/81] Move the kubelet yaml config into nix. --- nix/kubernetes/keys/scope.nix | 10 ++++- nix/kubernetes/roles/kubelet/default.nix | 44 ++++++++++++++++++- .../roles/kubelet/files/kubelet-config.yaml | 29 ------------ .../roles/kubelet/files/resolv.conf | 1 + 4 files changed, 53 insertions(+), 31 deletions(-) delete mode 100644 nix/kubernetes/roles/kubelet/files/kubelet-config.yaml create mode 100644 nix/kubernetes/roles/kubelet/files/resolv.conf diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index 20fd33a..5732d9f 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -267,7 +267,15 @@ makeScope newScope ( helm_namespace = "kube-system"; helm_path = "charts/coredns"; helm_manifest_name = "coredns.yaml"; - helm_values = { }; + helm_values = { + "service" = { + "clusterIP" = "fd00:3e42:e349::10"; + "clusterIPs" = [ + "fd00:3e42:e349::10" + "10.197.0.10" + ]; + }; + }; } )); all_keys = (callPackage ./package/k8s-keys/package.nix additional_vars); diff --git a/nix/kubernetes/roles/kubelet/default.nix b/nix/kubernetes/roles/kubelet/default.nix index cf79068..3014d18 100644 --- a/nix/kubernetes/roles/kubelet/default.nix +++ b/nix/kubernetes/roles/kubelet/default.nix @@ -8,6 +8,48 @@ let # shellCommand = cmd: (lib.concatMapStringsSep " " lib.strings.escapeShellArg cmd); shellCommand = cmd: (builtins.concatStringsSep " " cmd); + to_yaml_file = ((import ../../../functions/to_yaml.nix) { inherit pkgs; }).to_yaml_file; + + kubelet_config = { + kind = "KubeletConfiguration"; + apiVersion = "kubelet.config.k8s.io/v1beta1"; + address = "0.0.0.0"; + authentication = { + anonymous = { + enabled = false; + }; + webhook = { + enabled = true; + }; + x509 = { + clientCAFile = "/.persist/keys/kube/ca.crt"; + }; + }; + authorization = { + mode = "Webhook"; + }; + cgroupDriver = "systemd"; + containerRuntimeEndpoint = "unix:///var/run/containerd/containerd.sock"; + enableServer = true; + failSwapOn = false; + maxPods = 16; + memorySwap = { + swapBehavior = "NoSwap"; + }; + port = 10250; + # resolvConf = "/run/systemd/resolve/resolv.conf"; + resolvConf = "${./files/resolv.conf}"; + registerNode = true; + runtimeRequestTimeout = "15m"; + tlsCertFile = "/.persist/keys/kube/kubelet.crt"; + tlsPrivateKeyFile = "/.persist/keys/kube/kubelet.key"; + # clusterDomain = "cluster.local"; + # clusterDNS = [ + # "10.197.0.10" + # "fd00:3e42:e349::10" + # ]; + }; + kubelet_config_file = (to_yaml_file "kubelet-config.yaml" kubelet_config); in { imports = [ ]; @@ -37,7 +79,7 @@ in ExecStart = ( shellCommand [ "${pkgs.kubernetes}/bin/kubelet" - "--config=${./files/kubelet-config.yaml}" + "--config=${kubelet_config_file}" "--kubeconfig=/.persist/keys/kube/kubelet.kubeconfig" "--v=2" ] diff --git a/nix/kubernetes/roles/kubelet/files/kubelet-config.yaml b/nix/kubernetes/roles/kubelet/files/kubelet-config.yaml deleted file mode 100644 index 1f5302f..0000000 --- a/nix/kubernetes/roles/kubelet/files/kubelet-config.yaml +++ /dev/null @@ -1,29 +0,0 @@ -kind: KubeletConfiguration -apiVersion: kubelet.config.k8s.io/v1beta1 -address: "0.0.0.0" -authentication: - anonymous: - enabled: false - webhook: - enabled: true - x509: - clientCAFile: "/.persist/keys/kube/ca.crt" -authorization: - mode: Webhook -cgroupDriver: systemd -containerRuntimeEndpoint: "unix:///var/run/containerd/containerd.sock" -enableServer: true -failSwapOn: false -maxPods: 16 -memorySwap: - swapBehavior: NoSwap -port: 10250 -resolvConf: "/run/systemd/resolve/resolv.conf" -registerNode: true -runtimeRequestTimeout: "15m" -tlsCertFile: "/.persist/keys/kube/kubelet.crt" -tlsPrivateKeyFile: "/.persist/keys/kube/kubelet.key" -clusterDomain: "cluster.local" -clusterDNS: - - "10.197.0.10" - - "fd00:3e42:e349::10" diff --git a/nix/kubernetes/roles/kubelet/files/resolv.conf b/nix/kubernetes/roles/kubelet/files/resolv.conf new file mode 100644 index 0000000..5c089d6 --- /dev/null +++ b/nix/kubernetes/roles/kubelet/files/resolv.conf @@ -0,0 +1 @@ +nameserver 127.0.0.53 From ab1b2a70f3adbac757c005cb68b48e2cdc65fa71 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 29 Dec 2025 22:44:04 -0500 Subject: [PATCH 47/81] More changes to try to fix coredns. --- .../files/manifests/coredns.yaml | 218 ------------------ nix/kubernetes/keys/scope.nix | 20 +- .../roles/kube_controller_manager/default.nix | 2 +- nix/kubernetes/roles/kubelet/default.nix | 15 +- .../roles/kubelet/files/resolv.conf | 1 - 5 files changed, 22 insertions(+), 234 deletions(-) delete mode 100644 nix/kubernetes/keys/package/bootstrap-script/files/manifests/coredns.yaml delete mode 100644 nix/kubernetes/roles/kubelet/files/resolv.conf diff --git a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/coredns.yaml b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/coredns.yaml deleted file mode 100644 index 4b4bdf5..0000000 --- a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/coredns.yaml +++ /dev/null @@ -1,218 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: coredns - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: Reconcile ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - kubernetes.io/bootstrapping: rbac-defaults - addonmanager.kubernetes.io/mode: Reconcile - name: system:coredns -rules: - - apiGroups: - - "" - resources: - - endpoints - - services - - pods - - namespaces - verbs: - - list - - watch - - apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - labels: - kubernetes.io/bootstrapping: rbac-defaults - addonmanager.kubernetes.io/mode: EnsureExists - name: system:coredns -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:coredns -subjects: - - kind: ServiceAccount - name: coredns - namespace: kube-system ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: coredns - namespace: kube-system - labels: - addonmanager.kubernetes.io/mode: EnsureExists -data: - Corefile: | - .:53 { - errors - health { - lameduck 5s - } - ready - kubernetes cluster.local in-addr.arpa ip6.arpa { - pods insecure - fallthrough in-addr.arpa ip6.arpa - ttl 30 - } - prometheus :9153 - forward . /etc/resolv.conf { - max_concurrent 1000 - } - cache 30 - loop - reload - loadbalance - } ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: coredns - namespace: kube-system - labels: - k8s-app: kube-dns - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: Reconcile - kubernetes.io/name: "CoreDNS" -spec: - # replicas: not specified here: - # 1. In order to make Addon Manager do not reconcile this replicas parameter. - # 2. Default is 1. - # 3. Will be tuned in real time if DNS horizontal auto-scaling is turned on. - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - selector: - matchLabels: - k8s-app: kube-dns - template: - metadata: - labels: - k8s-app: kube-dns - spec: - securityContext: - seccompProfile: - type: RuntimeDefault - priorityClassName: system-cluster-critical - serviceAccountName: coredns - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: k8s-app - operator: In - values: ["kube-dns"] - topologyKey: kubernetes.io/hostname - tolerations: - - key: "CriticalAddonsOnly" - operator: "Exists" - nodeSelector: - kubernetes.io/os: linux - containers: - - name: coredns - image: registry.k8s.io/coredns/coredns:v1.13.1 - imagePullPolicy: IfNotPresent - resources: - limits: - memory: 70Mi - requests: - cpu: 100m - memory: 70Mi - args: ["-conf", "/etc/coredns/Corefile"] - volumeMounts: - - name: config-volume - mountPath: /etc/coredns - readOnly: true - ports: - - containerPort: 53 - name: dns - protocol: UDP - - containerPort: 53 - name: dns-tcp - protocol: TCP - - containerPort: 9153 - name: metrics - protocol: TCP - livenessProbe: - httpGet: - path: /health - port: 8080 - scheme: HTTP - initialDelaySeconds: 60 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 5 - readinessProbe: - httpGet: - path: /ready - port: 8181 - scheme: HTTP - securityContext: - allowPrivilegeEscalation: false - capabilities: - add: - - NET_BIND_SERVICE - drop: - - ALL - readOnlyRootFilesystem: true - dnsPolicy: Default - volumes: - - name: config-volume - configMap: - name: coredns - items: - - key: Corefile - path: Corefile ---- -apiVersion: v1 -kind: Service -metadata: - name: kube-dns - namespace: kube-system - annotations: - prometheus.io/port: "9153" - prometheus.io/scrape: "true" - labels: - k8s-app: kube-dns - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: Reconcile - kubernetes.io/name: "CoreDNS" -spec: - selector: - k8s-app: kube-dns - ipFamilyPolicy: PreferDualStack - clusterIP: "fd00:3e42:e349::10" - clusterIPs: - - "fd00:3e42:e349::10" - - "10.197.0.10" - ports: - - name: dns - port: 53 - protocol: UDP - - name: dns-tcp - port: 53 - protocol: TCP - - name: metrics - port: 9153 - protocol: TCP diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index 5732d9f..15d773f 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -227,17 +227,24 @@ makeScope newScope ( helm_manifest_name = "cilium.yaml"; helm_values = { "kubeProxyReplacement" = true; - "ipam.mode" = "kubernetes"; + "ipam" = { + "mode" = "kubernetes"; + }; "k8sServiceHost" = "2620:11f:7001:7:ffff:ffff:ad7:1dd"; "k8sServicePort" = 6443; - "ipv6.enabled" = true; - "ipv4.enabled" = true; + "ipv6" = { + "enabled" = true; + }; + "ipv4" = { + "enabled" = true; + }; "enableIPv6Masquerade" = false; - "enableIPv4BIGTCP" = false; - "enableIPv6BIGTCP" = false; + "enableIPv4BIGTCP" = false; # "true" causes high lag for me (I assume PMTUD?) + "enableIPv6BIGTCP" = false; # "true" causes high lag for me (I assume PMTUD?) "routingMode" = "native"; "ipv4NativeRoutingCIDR" = "10.0.0.0/8"; - "ipv6NativeRoutingCIDR" = "2620:11f:7001:7:ffff::/96"; + "ipv6NativeRoutingCIDR" = "2620:11f:7001:7:ffff::/80"; + # --set hostFirewall.enabled=true # --set routingMode=native @@ -269,6 +276,7 @@ makeScope newScope ( helm_manifest_name = "coredns.yaml"; helm_values = { "service" = { + "ipFamilyPolicy" = "PreferDualStack"; "clusterIP" = "fd00:3e42:e349::10"; "clusterIPs" = [ "fd00:3e42:e349::10" diff --git a/nix/kubernetes/roles/kube_controller_manager/default.nix b/nix/kubernetes/roles/kube_controller_manager/default.nix index 4197828..fb93f9a 100644 --- a/nix/kubernetes/roles/kube_controller_manager/default.nix +++ b/nix/kubernetes/roles/kube_controller_manager/default.nix @@ -51,7 +51,7 @@ in "--service-account-private-key-file=/.persist/keys/kube/service-accounts.key" # "--service-cluster-ip-range=10.197.0.0/16" # "--service-cluster-ip-range=2620:11f:7001:7:ffff:ffff:0ac5:0000/16" - "--service-cluster-ip-range=10.197.0.0/16,fd00:3e42:e349::/112" + "--service-cluster-ip-range=fd00:3e42:e349::/112,10.197.0.0/16" "--use-service-account-credentials=true" "--v=2" ] diff --git a/nix/kubernetes/roles/kubelet/default.nix b/nix/kubernetes/roles/kubelet/default.nix index 3014d18..3da95a1 100644 --- a/nix/kubernetes/roles/kubelet/default.nix +++ b/nix/kubernetes/roles/kubelet/default.nix @@ -8,7 +8,7 @@ let # shellCommand = cmd: (lib.concatMapStringsSep " " lib.strings.escapeShellArg cmd); shellCommand = cmd: (builtins.concatStringsSep " " cmd); - to_yaml_file = ((import ../../../functions/to_yaml.nix) { inherit pkgs; }).to_yaml_file; + to_yaml_file = ((import ../../functions/to_yaml.nix) { inherit pkgs; }).to_yaml_file; kubelet_config = { kind = "KubeletConfiguration"; @@ -37,17 +37,16 @@ let swapBehavior = "NoSwap"; }; port = 10250; - # resolvConf = "/run/systemd/resolve/resolv.conf"; - resolvConf = "${./files/resolv.conf}"; + resolvConf = "/run/systemd/resolve/resolv.conf"; registerNode = true; runtimeRequestTimeout = "15m"; tlsCertFile = "/.persist/keys/kube/kubelet.crt"; tlsPrivateKeyFile = "/.persist/keys/kube/kubelet.key"; - # clusterDomain = "cluster.local"; - # clusterDNS = [ - # "10.197.0.10" - # "fd00:3e42:e349::10" - # ]; + clusterDomain = "cluster.local"; + clusterDNS = [ + "10.197.0.10" + "fd00:3e42:e349::10" + ]; }; kubelet_config_file = (to_yaml_file "kubelet-config.yaml" kubelet_config); in diff --git a/nix/kubernetes/roles/kubelet/files/resolv.conf b/nix/kubernetes/roles/kubelet/files/resolv.conf deleted file mode 100644 index 5c089d6..0000000 --- a/nix/kubernetes/roles/kubelet/files/resolv.conf +++ /dev/null @@ -1 +0,0 @@ -nameserver 127.0.0.53 From 9b5fbadc441e7785dd32e23d373082cbfa15bc61 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Tue, 30 Dec 2025 21:57:09 -0500 Subject: [PATCH 48/81] Increase timeout for coredns cache. --- nix/kubernetes/keys/scope.nix | 40 +++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index 15d773f..e21322e 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -283,6 +283,46 @@ makeScope newScope ( "10.197.0.10" ]; }; + + servers = [ + { + zones = [ + { + zone = "."; + use_tcp = true; + } + ]; + port = 53; + plugins = [ + { name = "errors"; } + { + name = "health"; + configBlock = "lameduck 10s"; + } + { name = "ready"; } + { + name = "kubernetes"; + parameters = "cluster.local in-addr.arpa ip6.arpa"; + configBlock = "pods insecure\nfallthrough in-addr.arpa ip6.arpa\nttl 30"; + } + { + name = "prometheus"; + parameters = "0.0.0.0:9153"; + } + { + name = "forward"; + parameters = ". /etc/resolv.conf"; + } + { + name = "cache"; + parameters = 300; # default 30 + } + { name = "loop"; } + { name = "reload"; } + { name = "loadbalance"; } + ]; + } + ]; }; } )); From 316982272651989ea1ff4ff0c70d06cdf0a88193 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Wed, 31 Dec 2025 20:12:56 -0500 Subject: [PATCH 49/81] Fix CoreDNS IPv4 connectivity. --- nix/kubernetes/keys/scope.nix | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index e21322e..b75bb80 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -239,14 +239,14 @@ makeScope newScope ( "enabled" = true; }; "enableIPv6Masquerade" = false; - "enableIPv4BIGTCP" = false; # "true" causes high lag for me (I assume PMTUD?) - "enableIPv6BIGTCP" = false; # "true" causes high lag for me (I assume PMTUD?) + "enableIPv4BIGTCP" = true; + "enableIPv6BIGTCP" = true; "routingMode" = "native"; - "ipv4NativeRoutingCIDR" = "10.0.0.0/8"; + "autoDirectNodeRoutes" = true; + "ipv4NativeRoutingCIDR" = "10.200.0.0/16"; "ipv6NativeRoutingCIDR" = "2620:11f:7001:7:ffff::/80"; # --set hostFirewall.enabled=true - # --set routingMode=native # --set 'ipam.operator.clusterPoolIPv4PodCIDRList=["10.0.0.0/8"]' \ # --set 'ipam.operator.clusterPoolIPv6PodCIDRList=["fd00::/100"]' \ From 9e729a8cc645fc20fcf604b2b94cabce92952cde Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Thu, 1 Jan 2026 10:21:36 -0500 Subject: [PATCH 50/81] Enable the firewall. Now that we have networking working, I can enable the firewall and confirm nothing breaks. --- nix/kubernetes/README.org | 6 ++++++ nix/kubernetes/keys/scope.nix | 2 ++ nix/kubernetes/roles/debugging/default.nix | 6 ++++-- nix/kubernetes/roles/firewall/default.nix | 18 ++++++++++++++++-- 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/nix/kubernetes/README.org b/nix/kubernetes/README.org index 70f8a68..fbeeb07 100644 --- a/nix/kubernetes/README.org +++ b/nix/kubernetes/README.org @@ -10,6 +10,12 @@ { domain = "@users"; item = "rtprio"; type = "-"; value = 1; } ]; #+end_src +* IP Ranges +| | IPv4 | IPv6 | +|---------+---------------+-----------------------------------------| +| Pod | 10.200.0.0/16 | 2620:11f:7001:7:ffff:eeee::/96 | +| Service | 10.197.0.0/16 | fd00:3e42:e349::/112 | +| Node | 10.215.1.0/24 | 2620:11f:7001:7:ffff:ffff:0ad7:0100/120 | * Healthcheck ** Check cilium status #+begin_src bash diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index b75bb80..5c7382e 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -246,6 +246,8 @@ makeScope newScope ( "ipv4NativeRoutingCIDR" = "10.200.0.0/16"; "ipv6NativeRoutingCIDR" = "2620:11f:7001:7:ffff::/80"; + # TODO: Read and maybe apply https://docs.cilium.io/en/stable/operations/performance/tuning/ + # --set hostFirewall.enabled=true # --set 'ipam.operator.clusterPoolIPv4PodCIDRList=["10.0.0.0/8"]' \ diff --git a/nix/kubernetes/roles/debugging/default.nix b/nix/kubernetes/roles/debugging/default.nix index 14bcb75..35b9cd1 100644 --- a/nix/kubernetes/roles/debugging/default.nix +++ b/nix/kubernetes/roles/debugging/default.nix @@ -27,7 +27,9 @@ ldns # for drill ]; - networking.firewall.enable = false; # TODO: This is just here for debugging / initial development. - # TODO: Maybe use networking.nftables.enable to switch to nftables? + # This can make debugging easier by rejecting packets instead of dropping them: + networking.firewall.rejectPackets = true; + # Log each rejected packet instead of just each connection. + networking.firewall.logRefusedPackets = true; }; } diff --git a/nix/kubernetes/roles/firewall/default.nix b/nix/kubernetes/roles/firewall/default.nix index 9dedd29..771aa1d 100644 --- a/nix/kubernetes/roles/firewall/default.nix +++ b/nix/kubernetes/roles/firewall/default.nix @@ -26,15 +26,29 @@ "net.bridge.bridge-nf-call-iptables" = 1; "net.bridge.bridge-nf-call-ip6tables" = 1; "net.ipv4.ip_forward" = 1; + + # Enable forwarding on all interfaces. + # "net.ipv4.conf.all.forwarding" = 1; + # "net.ipv6.conf.all.forwarding" = 1; }; + networking.firewall.enable = false; networking.nftables.enable = true; # We want to filter forwarded traffic. # Also needed for `networking.firewall.extraForwardRules` to do anything. networking.firewall.filterForward = true; - # This can make debugging easier by rejecting packets instead of dropping them: - # networking.firewall.rejectPackets = true; + networking.firewall.extraInputRules = '' + ip6 saddr 2620:11f:7001:7:ffff:eeee::/96 accept + ip6 saddr fd00:3e42:e349::/112 accept + ip6 saddr 2620:11f:7001:7:ffff:ffff:0ad7:0100/120 accept + ''; + + networking.firewall.extraForwardRules = '' + ip6 daddr 2620:11f:7001:7:ffff:eeee::/96 accept + ip6 daddr fd00:3e42:e349::/112 accept + ip6 daddr 2620:11f:7001:7:ffff:ffff:0ad7:0100/120 accept + ''; # Check logs for blocked connections: # journalctl -k or dmesg From 23d9bbb8a1b9565f49e6310fe2b1d09259f55141 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Fri, 2 Jan 2026 23:28:29 -0500 Subject: [PATCH 51/81] Add a custom nftables firewall config. --- nix/kubernetes/roles/firewall/default.nix | 5 + nix/kubernetes/roles/firewall/files/my-fw.nft | 140 ++++++++++++++++++ 2 files changed, 145 insertions(+) create mode 100644 nix/kubernetes/roles/firewall/files/my-fw.nft diff --git a/nix/kubernetes/roles/firewall/default.nix b/nix/kubernetes/roles/firewall/default.nix index 771aa1d..b983de6 100644 --- a/nix/kubernetes/roles/firewall/default.nix +++ b/nix/kubernetes/roles/firewall/default.nix @@ -52,5 +52,10 @@ # Check logs for blocked connections: # journalctl -k or dmesg + + networking.nftables.tables."my-fw" = { + family = "inet"; + content = (builtins.readFile ./files/my-fw.nft); + }; }; } diff --git a/nix/kubernetes/roles/firewall/files/my-fw.nft b/nix/kubernetes/roles/firewall/files/my-fw.nft new file mode 100644 index 0000000..c434f6f --- /dev/null +++ b/nix/kubernetes/roles/firewall/files/my-fw.nft @@ -0,0 +1,140 @@ +set internal-iface { + type ifname + # Tell the kernel this set may include ranges, so it can pick the best datastructure. + flags interval + elements = { + "lxc*", + "cilium_net" + } +} + +set node-cidr-ipv4 { + type ipv4_addr + flags constant, interval + elements = { 10.215.1.0/24 } +} + +set node-cidr-ipv6 { + type ipv6_addr + flags constant, interval + elements = { 2620:11f:7001:7:ffff:ffff:0ad7:0100/120 } +} + +set service-cidr-ipv4 { + type ipv4_addr + flags constant, interval + elements = { 10.197.0.0/16 } +} + +set service-cidr-ipv6 { + type ipv6_addr + flags constant, interval + elements = { fd00:3e42:e349::/112 } +} + +set pod-cidr-ipv4 { + type ipv4_addr + flags constant, interval + elements = { 10.200.0.0/16 } +} + +set pod-cidr-ipv6 { + type ipv6_addr + flags constant, interval + elements = { 2620:11f:7001:7:ffff:eeee::/96 } +} + +set public-ports { + # These are open to all IP addresses + type inet_proto . inet_service + flags constant + elements = { + tcp . 22, # ssh + tcp . 6443 # kubernetes API server + } +} + +set node-to-node-ports { + # Ports open for nodes sending packets to nodes + type inet_proto . inet_service + flags constant, interval + elements = { + tcp . 2379-2380, # etcd + tcp . 4240, # cilium health monitoring + tcp . 10250, # kubelet API + # udp . 51871, # cilium wireguard + # tcp . 30000-32767, # nodeport range + # udp . 30000-32767, # nodeport range + # udp . 8472, # cilium vxlan + tcp . 7946, # MetalLB memberlist + udp . 7946 # MetalLB memberlist + } +} + +chain rpfilter { + type filter hook prerouting priority mangle + 10; policy drop; + meta nfproto ipv4 udp sport . udp dport { 68 . 67, 67 . 68 } accept comment "DHCPv4 client/server" + # Reverse path forwarding filter. Check that a route exists back to the source address on the interface which received the packet. If the packet came from the wrong interface, then the packet is likely spoofed. + fib saddr . mark . iif check exists accept + jump rpfilter-allow + meta pkttype host log prefix "Failed rpfilter: " level info +} + +chain rpfilter-allow { + # Allow packets on internal interfaces from pods + meta iifname @internal-iface ip saddr @pod-cidr-ipv4 accept + meta iifname @internal-iface ip6 saddr @pod-cidr-ipv6 accept +} + + +chain input { + type filter hook input priority filter; policy drop; + iifname "lo" accept comment "trusted interfaces" + # Drop invalid connections, accept packets for established or related connections and send packets for new or untracked connections to the input-allow chain + ct state vmap { invalid : drop, established : accept, related : accept, new : jump input-allow, untracked : jump input-allow } + # If the packet is a new connection and reaches this point, then we are going to reject it. So log that rejection. + tcp flags & (fin | syn | rst | ack) == syn log prefix "refused connection: " level info + # Log rejected packets destined for this machine (as opposed to packets being routed or broadcast packets) + meta pkttype host log prefix "refused packet: " level info + # When rejecting packets, send a TCP Reset (RST) instead of simply dropping the packet. + meta l4proto tcp reject with tcp reset + # Reject any packets that make it here. + reject +} + +chain input-allow { + # Allow pings. + icmp type echo-request accept comment "allow ping" + icmpv6 type != { nd-redirect, 139 } accept comment "Accept all ICMPv6 messages except redirects and node information queries (type 139). See RFC 4890, section 4.4." + ip6 daddr fe80::/64 udp dport 546 accept comment "DHCPv6 client" + + # Allow public ports + meta l4proto . th dport @public-ports accept + + # Allow node to node + ip saddr @node-cidr-ipv4 ip daddr @node-cidr-ipv4 meta l4proto . th dport @node-to-node-ports accept + ip6 saddr @node-cidr-ipv6 ip6 daddr @node-cidr-ipv6 meta l4proto . th dport @node-to-node-ports accept +} + +chain forward { + type filter hook forward priority filter; policy drop; + # Drop invalid connections, accept packets for established or related connections and send packets for new or untracked connections to the forward-allow chain + ct state vmap { invalid : drop, established : accept, related : accept, new : jump forward-allow, untracked : jump forward-allow } + + log prefix "blocked forwarding packet: " level info +} + +chain forward-allow { + icmpv6 type != { router-renumbering, 139 } accept comment "Accept all ICMPv6 messages except renumbering and node information queries (type 139). See RFC 4890, section 4.3." + + # When connection tracking (ct) shows the status as destination nat (dnat) then accept the packet. + ct status dnat accept comment "allow port forward" + + # Allow packets from pods + ip saddr @pod-cidr-ipv4 accept + ip6 saddr @pod-cidr-ipv6 accept + + # Allow node-to-pod + ip saddr @node-cidr-ipv4 ip daddr @pod-cidr-ipv4 accept + ip6 saddr @node-cidr-ipv6 ip6 daddr @pod-cidr-ipv6 accept +} From f3b969642c54d363721c7d9f29d22e45c9c34c9a Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sat, 3 Jan 2026 14:57:26 -0500 Subject: [PATCH 52/81] Install deferred manifests. --- nix/kubernetes/README.org | 11 +++++----- .../files/manifests/flux_apply_git.yaml | 21 +++++++++++++++++-- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/nix/kubernetes/README.org b/nix/kubernetes/README.org index fbeeb07..eeae564 100644 --- a/nix/kubernetes/README.org +++ b/nix/kubernetes/README.org @@ -11,11 +11,12 @@ ]; #+end_src * IP Ranges -| | IPv4 | IPv6 | -|---------+---------------+-----------------------------------------| -| Pod | 10.200.0.0/16 | 2620:11f:7001:7:ffff:eeee::/96 | -| Service | 10.197.0.0/16 | fd00:3e42:e349::/112 | -| Node | 10.215.1.0/24 | 2620:11f:7001:7:ffff:ffff:0ad7:0100/120 | +| | IPv4 | IPv6 | +|---------------+-----------------------------+-----------------------------------------| +| Pod | 10.200.0.0/16 | 2620:11f:7001:7:ffff:eeee::/96 | +| Service | 10.197.0.0/16 | fd00:3e42:e349::/112 | +| Node | 10.215.1.0/24 | 2620:11f:7001:7:ffff:ffff:0ad7:0100/120 | +| Load Balancer | 74.80.180.139-74.80.180.142 | 2620:11f:7001:7:ffff:dddd::/96 | * Healthcheck ** Check cilium status #+begin_src bash diff --git a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_apply_git.yaml b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_apply_git.yaml index a0fba73..eeb6add 100644 --- a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_apply_git.yaml +++ b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_apply_git.yaml @@ -19,11 +19,28 @@ spec: apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: - name: backend + name: first namespace: flux-system spec: interval: 5m0s - path: "./k8s" + path: "./k8s/first" + prune: true + sourceRef: + kind: GitRepository + name: kubernetes + decryption: + provider: sops + secretRef: + name: sops-gpg +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: deferred + namespace: flux-system +spec: + interval: 5m0s + path: "./k8s/deferred" prune: true sourceRef: kind: GitRepository From 1ed3832730e64e6a7a6d09caff057acb412bbdca Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sat, 3 Jan 2026 15:00:44 -0500 Subject: [PATCH 53/81] Temporarily drop flux interval to 1 minute during early development. This is to reduce waiting time. --- .../bootstrap-script/files/manifests/flux_apply_git.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_apply_git.yaml b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_apply_git.yaml index eeb6add..d60e9bb 100644 --- a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_apply_git.yaml +++ b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_apply_git.yaml @@ -22,7 +22,7 @@ metadata: name: first namespace: flux-system spec: - interval: 5m0s + interval: 1m0s path: "./k8s/first" prune: true sourceRef: @@ -39,7 +39,7 @@ metadata: name: deferred namespace: flux-system spec: - interval: 5m0s + interval: 1m0s path: "./k8s/deferred" prune: true sourceRef: From 612158c3e18f9eea3b35ab9c151468d45d98db96 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 4 Jan 2026 12:13:58 -0500 Subject: [PATCH 54/81] Enable hubble. --- nix/kubernetes/keys/scope.nix | 9 +++++++++ nix/kubernetes/roles/firewall/files/my-fw.nft | 13 +++++++++++++ nix/kubernetes/roles/kubelet/files/resolv.conf | 4 ++++ 3 files changed, 26 insertions(+) create mode 100644 nix/kubernetes/roles/kubelet/files/resolv.conf diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index 5c7382e..350947f 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -246,6 +246,15 @@ makeScope newScope ( "ipv4NativeRoutingCIDR" = "10.200.0.0/16"; "ipv6NativeRoutingCIDR" = "2620:11f:7001:7:ffff::/80"; + "hubble" = { + "relay" = { + "enabled" = true; + }; + "ui" = { + "enabled" = true; + }; + }; + # TODO: Read and maybe apply https://docs.cilium.io/en/stable/operations/performance/tuning/ # --set hostFirewall.enabled=true diff --git a/nix/kubernetes/roles/firewall/files/my-fw.nft b/nix/kubernetes/roles/firewall/files/my-fw.nft index c434f6f..61609cd 100644 --- a/nix/kubernetes/roles/firewall/files/my-fw.nft +++ b/nix/kubernetes/roles/firewall/files/my-fw.nft @@ -71,6 +71,15 @@ set node-to-node-ports { } } +set pod-to-node-ports { + # Ports open for nodes sending packets to nodes + type inet_proto . inet_service + flags constant, interval + elements = { + tcp . 4244 # hubble ui + } +} + chain rpfilter { type filter hook prerouting priority mangle + 10; policy drop; meta nfproto ipv4 udp sport . udp dport { 68 . 67, 67 . 68 } accept comment "DHCPv4 client/server" @@ -114,6 +123,10 @@ chain input-allow { # Allow node to node ip saddr @node-cidr-ipv4 ip daddr @node-cidr-ipv4 meta l4proto . th dport @node-to-node-ports accept ip6 saddr @node-cidr-ipv6 ip6 daddr @node-cidr-ipv6 meta l4proto . th dport @node-to-node-ports accept + + # Allow pod to node + ip saddr @pod-cidr-ipv4 ip daddr @node-cidr-ipv4 meta l4proto . th dport @pod-to-node-ports accept + ip6 saddr @pod-cidr-ipv6 ip6 daddr @node-cidr-ipv6 meta l4proto . th dport @pod-to-node-ports accept } chain forward { diff --git a/nix/kubernetes/roles/kubelet/files/resolv.conf b/nix/kubernetes/roles/kubelet/files/resolv.conf new file mode 100644 index 0000000..a91e07e --- /dev/null +++ b/nix/kubernetes/roles/kubelet/files/resolv.conf @@ -0,0 +1,4 @@ +search svc.cluster.local cluster.local +nameserver 10.197.0.10 +nameserver fd00:3e42:e349::10 +options ndots:5 From cce4f981258a939a98fa7babfd2daf3d5a5d1a0e Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 4 Jan 2026 22:27:00 -0500 Subject: [PATCH 55/81] Enable gateway support. --- .../keys/package/bootstrap-script/package.nix | 29 +++++++++++++++++++ nix/kubernetes/keys/scope.nix | 4 +++ 2 files changed, 33 insertions(+) diff --git a/nix/kubernetes/keys/package/bootstrap-script/package.nix b/nix/kubernetes/keys/package/bootstrap-script/package.nix index eacc1fc..2ae16d0 100644 --- a/nix/kubernetes/keys/package/bootstrap-script/package.nix +++ b/nix/kubernetes/keys/package/bootstrap-script/package.nix @@ -28,6 +28,9 @@ let lib.concatMapStringsSep "," lib.escapeShellArg ( [ ./files/manifests/initial_clusterrole.yaml + ] + ++ gateway_crds + ++ [ "${k8s.cilium-manifest}/cilium.yaml" "${k8s.coredns-manifest}/coredns.yaml" ./files/manifests/flux_namespace.yaml @@ -43,6 +46,32 @@ let ) ); apply_manifests = "kubectl --kubeconfig=${k8s.client-configs.admin}/admin.kubeconfig apply --server-side --force-conflicts -f ${manifests}"; + gateway_crds = [ + (builtins.fetchurl { + url = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.2.0/config/crd/standard/gateway.networking.k8s.io_gatewayclasses.yaml"; + sha256 = "0vf8c3kzlf7p6bf92gmdrzjc22fr2dwkrzvvbnxlsb43knv1nbzl"; + }) + (builtins.fetchurl { + url = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.2.0/config/crd/standard/gateway.networking.k8s.io_gateways.yaml"; + sha256 = "1dqwlsypcb5f37y7x48rrv27yfgkizcx2alqd2nngijl1qzir3wa"; + }) + (builtins.fetchurl { + url = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.2.0/config/crd/standard/gateway.networking.k8s.io_httproutes.yaml"; + sha256 = "05llfw6y66438r8kqy7krhyymyalkzxsaxjpa2zxzjk6z5mggbzq"; + }) + (builtins.fetchurl { + url = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.2.0/config/crd/standard/gateway.networking.k8s.io_referencegrants.yaml"; + sha256 = "0a9q0vhqcazfrni3ajcq8vm2b254vcjbgmkchsdq9l6cbpvx79jd"; + }) + (builtins.fetchurl { + url = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.2.0/config/crd/standard/gateway.networking.k8s.io_grpcroutes.yaml"; + sha256 = "19hwvdwdj0sc5fihdskw492g52ail3kjjzm6vpflvp2vlqam629p"; + }) + (builtins.fetchurl { + url = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.2.0/config/crd/experimental/gateway.networking.k8s.io_tlsroutes.yaml"; + sha256 = "0b5pjihyzyyi4inz3avlkzvvccsynj9wsmx6znld04jmmvwpgxc9"; + }) + ]; in stdenv.mkDerivation (finalAttrs: { name = "bootstrap-script"; diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index 350947f..47ee352 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -253,6 +253,10 @@ makeScope newScope ( "ui" = { "enabled" = true; }; + + "gatewayAPI" = { + "enabled" = true; + }; }; # TODO: Read and maybe apply https://docs.cilium.io/en/stable/operations/performance/tuning/ From d6f089bdb455fcffe5d4638e3f8cfc88d469df1c Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 4 Jan 2026 22:27:46 -0500 Subject: [PATCH 56/81] Temporarily disable the firewall for debugging. --- nix/kubernetes/keys/scope.nix | 2 ++ nix/kubernetes/roles/firewall/default.nix | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index 47ee352..1414f6e 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -259,6 +259,8 @@ makeScope newScope ( }; }; + "policyEnforcementMode" = "never"; + # TODO: Read and maybe apply https://docs.cilium.io/en/stable/operations/performance/tuning/ # --set hostFirewall.enabled=true diff --git a/nix/kubernetes/roles/firewall/default.nix b/nix/kubernetes/roles/firewall/default.nix index b983de6..3630b5c 100644 --- a/nix/kubernetes/roles/firewall/default.nix +++ b/nix/kubernetes/roles/firewall/default.nix @@ -53,9 +53,9 @@ # Check logs for blocked connections: # journalctl -k or dmesg - networking.nftables.tables."my-fw" = { - family = "inet"; - content = (builtins.readFile ./files/my-fw.nft); - }; + # networking.nftables.tables."my-fw" = { + # family = "inet"; + # content = (builtins.readFile ./files/my-fw.nft); + # }; }; } From 2a319deea00eb0a09240a14327bd5b20e250f868 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Fri, 9 Jan 2026 17:32:28 -0500 Subject: [PATCH 57/81] Add a note for the cilium connectivity test. --- nix/kubernetes/README.org | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/nix/kubernetes/README.org b/nix/kubernetes/README.org index eeae564..b681dea 100644 --- a/nix/kubernetes/README.org +++ b/nix/kubernetes/README.org @@ -23,6 +23,14 @@ kubectl -n kube-system exec ds/cilium -- cilium-dbg status --verbose kubectl -n kube-system exec ds/cilium -- cilium-dbg status | grep KubeProxyReplacement #+end_src +** Check connectivity +#+begin_src bash + cilium connectivity test +#+end_src +** Show dropped packets +#+begin_src bash + kubectl -n kube-system exec ds/cilium -- cilium-dbg monitor --type drop +#+end_src ** Install flux #+begin_src bash nix shell 'nixpkgs#fluxcd' From 92727aa1f7ef9f834c1e5eef45fce4b8f2bfc193 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Fri, 9 Jan 2026 18:19:34 -0500 Subject: [PATCH 58/81] Generate certificates for the aggregation layer. --- nix/kubernetes/README.org | 2 +- .../keys/package/deploy-script/package.nix | 35 ++++++- .../k8s-ca/files/{ca.conf => client-ca.conf} | 0 .../k8s-ca/files/requestheader-client-ca.conf | 95 +++++++++++++++++++ .../keys/package/k8s-ca/package.nix | 21 ++-- .../package/k8s-client-config/package.nix | 2 +- .../keys/package/k8s-keys/package.nix | 2 +- .../keys/package/tls-key/package.nix | 10 +- nix/kubernetes/keys/scope.nix | 54 ++++++++--- nix/kubernetes/roles/etcd/default.nix | 4 +- .../roles/kube_apiserver/default.nix | 14 ++- .../roles/kube_controller_manager/default.nix | 6 +- nix/kubernetes/roles/kubelet/default.nix | 2 +- 13 files changed, 204 insertions(+), 43 deletions(-) rename nix/kubernetes/keys/package/k8s-ca/files/{ca.conf => client-ca.conf} (100%) create mode 100644 nix/kubernetes/keys/package/k8s-ca/files/requestheader-client-ca.conf diff --git a/nix/kubernetes/README.org b/nix/kubernetes/README.org index b681dea..9437c61 100644 --- a/nix/kubernetes/README.org +++ b/nix/kubernetes/README.org @@ -16,7 +16,7 @@ | Pod | 10.200.0.0/16 | 2620:11f:7001:7:ffff:eeee::/96 | | Service | 10.197.0.0/16 | fd00:3e42:e349::/112 | | Node | 10.215.1.0/24 | 2620:11f:7001:7:ffff:ffff:0ad7:0100/120 | -| Load Balancer | 74.80.180.139-74.80.180.142 | 2620:11f:7001:7:ffff:dddd::/96 | +| Load Balancer | 74.80.180.139-74.80.180.142 | 2620:11f:7001:7:ffff:dddd::/96 | * Healthcheck ** Check cilium status #+begin_src bash diff --git a/nix/kubernetes/keys/package/deploy-script/package.nix b/nix/kubernetes/keys/package/deploy-script/package.nix index e809d9a..78fff16 100644 --- a/nix/kubernetes/keys/package/deploy-script/package.nix +++ b/nix/kubernetes/keys/package/deploy-script/package.nix @@ -106,21 +106,21 @@ let } { dest_dir = "/vm/${vm_name}/persist/keys/etcd"; - file = "${k8s.ca}/ca.crt"; + file = "${k8s.ca.client}/client-ca.crt"; owner = 10016; group = 10016; mode = "0640"; } { dest_dir = "/vm/${vm_name}/persist/keys/kube"; - file = "${k8s.ca}/ca.crt"; + file = "${k8s.ca.client}/client-ca.crt"; owner = 10024; group = 10024; mode = "0640"; } { dest_dir = "/vm/${vm_name}/persist/keys/kube"; - file = "${k8s.ca}/ca.key"; + file = "${k8s.ca.client}/client-ca.key"; owner = 10024; group = 10024; mode = "0600"; @@ -175,6 +175,33 @@ let group = 10024; mode = "0600"; } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.ca.requestheader-client}/requestheader-client-ca.crt"; + owner = 10024; + group = 10024; + mode = "0640"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${ + k8s.keys."${vm_name_to_hostname vm_name}-proxy" + }/${vm_name_to_hostname vm_name}-proxy.crt"; + name = "proxy.crt"; + owner = 10024; + group = 10024; + mode = "0640"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${ + k8s.keys."${vm_name_to_hostname vm_name}-proxy" + }/${vm_name_to_hostname vm_name}-proxy.key"; + name = "proxy.key"; + owner = 10024; + group = 10024; + mode = "0600"; + } ]) ) ); @@ -193,7 +220,7 @@ let + (lib.concatMapStringsSep "\n" deploy_file [ { dest_dir = "/vm/${vm_name}/persist/keys/kube"; - file = "${k8s.ca}/ca.crt"; + file = "${k8s.ca.client}/client-ca.crt"; owner = 10024; group = 10024; mode = "0640"; diff --git a/nix/kubernetes/keys/package/k8s-ca/files/ca.conf b/nix/kubernetes/keys/package/k8s-ca/files/client-ca.conf similarity index 100% rename from nix/kubernetes/keys/package/k8s-ca/files/ca.conf rename to nix/kubernetes/keys/package/k8s-ca/files/client-ca.conf diff --git a/nix/kubernetes/keys/package/k8s-ca/files/requestheader-client-ca.conf b/nix/kubernetes/keys/package/k8s-ca/files/requestheader-client-ca.conf new file mode 100644 index 0000000..f31e75c --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-ca/files/requestheader-client-ca.conf @@ -0,0 +1,95 @@ +[req] +distinguished_name = req_distinguished_name +prompt = no +x509_extensions = ca_x509_extensions + +[ca_x509_extensions] +basicConstraints = CA:TRUE +keyUsage = cRLSign, keyCertSign + +[req_distinguished_name] +C = US +ST = Washington +L = Seattle +CN = CA + +[controller0-proxy] +distinguished_name = controller0_distinguished_name +prompt = no +req_extensions = controller0_req_extensions + +[controller0_req_extensions] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth, serverAuth +keyUsage = critical, digitalSignature, keyEncipherment +nsCertType = client +nsComment = "controller0 Certificate" +subjectAltName = @controller0_alt_names +subjectKeyIdentifier = hash + +[controller0_distinguished_name] +CN = system:node:controller0 +O = system:nodes +C = US +ST = Washington +L = Seattle + +[controller0_alt_names] +IP.0 = 127.0.0.1 +IP.4 = 10.215.1.221 +IP.5 = 2620:11f:7001:7:ffff:ffff:0ad7:01dd +DNS.0 = controller0 + +[controller1-proxy] +distinguished_name = controller1_distinguished_name +prompt = no +req_extensions = controller1_req_extensions + +[controller1_req_extensions] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth, serverAuth +keyUsage = critical, digitalSignature, keyEncipherment +nsCertType = client +nsComment = "controller1 Certificate" +subjectAltName = @controller1_alt_names +subjectKeyIdentifier = hash + +[controller1_distinguished_name] +CN = system:node:controller1 +O = system:nodes +C = US +ST = Washington +L = Seattle + +[controller1_alt_names] +IP.0 = 127.0.0.1 +IP.4 = 10.215.1.222 +IP.5 = 2620:11f:7001:7:ffff:ffff:0ad7:01de +DNS.0 = controller1 + +[controller2-proxy] +distinguished_name = controller2_distinguished_name +prompt = no +req_extensions = controller2_req_extensions + +[controller2_req_extensions] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth, serverAuth +keyUsage = critical, digitalSignature, keyEncipherment +nsCertType = client +nsComment = "controller2 Certificate" +subjectAltName = @controller2_alt_names +subjectKeyIdentifier = hash + +[controller2_distinguished_name] +CN = system:node:controller2 +O = system:nodes +C = US +ST = Washington +L = Seattle + +[controller2_alt_names] +IP.0 = 127.0.0.1 +IP.6 = 10.215.1.223 +IP.7 = 2620:11f:7001:7:ffff:ffff:0ad7:01df +DNS.0 = controller2 diff --git a/nix/kubernetes/keys/package/k8s-ca/package.nix b/nix/kubernetes/keys/package/k8s-ca/package.nix index eed2a77..6c9329b 100644 --- a/nix/kubernetes/keys/package/k8s-ca/package.nix +++ b/nix/kubernetes/keys/package/k8s-ca/package.nix @@ -10,23 +10,28 @@ { stdenv, openssl, + ca_name, + ca_config, ... }: stdenv.mkDerivation (finalAttrs: { - name = "k8s-ca"; + name = "k8s-ca-${ca_name}"; nativeBuildInputs = [ openssl ]; buildInputs = [ ]; unpackPhase = "true"; - installPhase = '' - mkdir -p "$out" - cd "$out" + buildPhase = '' + openssl genrsa -out "${ca_name}-ca.key" 4096 - openssl genrsa -out ca.key 4096 openssl req -x509 -new -sha512 -noenc \ - -key ca.key -days 3653 \ - -config ${./files/ca.conf} \ - -out ca.crt + -key "${ca_name}-ca.key" -days 3653 \ + -config "${ca_config}" \ + -out "${ca_name}-ca.crt" + ''; + + installPhase = '' + mkdir "$out" + cp "${ca_name}-ca.crt" "${ca_name}-ca.key" $out/ ''; }) diff --git a/nix/kubernetes/keys/package/k8s-client-config/package.nix b/nix/kubernetes/keys/package/k8s-client-config/package.nix index 2d4ac88..853c3df 100644 --- a/nix/kubernetes/keys/package/k8s-client-config/package.nix +++ b/nix/kubernetes/keys/package/k8s-client-config/package.nix @@ -26,7 +26,7 @@ stdenv.mkDerivation (finalAttrs: { buildPhase = '' kubectl config set-cluster kubernetes-the-hard-way \ - --certificate-authority=${k8s.ca}/ca.crt \ + --certificate-authority=${k8s.ca.client}/client-ca.crt \ --embed-certs=true \ --server=${lib.strings.escapeShellArg config_server} \ --kubeconfig=${config_name}.kubeconfig diff --git a/nix/kubernetes/keys/package/k8s-keys/package.nix b/nix/kubernetes/keys/package/k8s-keys/package.nix index 3c62aec..71596b8 100644 --- a/nix/kubernetes/keys/package/k8s-keys/package.nix +++ b/nix/kubernetes/keys/package/k8s-keys/package.nix @@ -15,9 +15,9 @@ symlinkJoin { name = "k8s-keys"; paths = [ scripts - k8s.ca k8s.encryption_config ] + ++ (builtins.attrValues k8s.ca) ++ (builtins.attrValues k8s.keys) ++ (builtins.attrValues k8s.client-configs) ++ (builtins.attrValues k8s.ssh-keys) diff --git a/nix/kubernetes/keys/package/tls-key/package.nix b/nix/kubernetes/keys/package/tls-key/package.nix index 437099e..5daee36 100644 --- a/nix/kubernetes/keys/package/tls-key/package.nix +++ b/nix/kubernetes/keys/package/tls-key/package.nix @@ -12,6 +12,8 @@ openssl, k8s, key_name, + ca_name, + ca_config, ... }: stdenv.mkDerivation (finalAttrs: { @@ -22,18 +24,18 @@ stdenv.mkDerivation (finalAttrs: { unpackPhase = "true"; buildPhase = '' - cp ${k8s.ca}/ca.crt ${k8s.ca}/ca.key ./ + cp ${k8s.ca."${ca_name}"}/${ca_name}-ca.crt ${k8s.ca."${ca_name}"}/${ca_name}-ca.key ./ openssl genrsa -out "${key_name}.key" 4096 openssl req -new -key "${key_name}.key" -sha256 \ - -config "${../k8s-ca/files/ca.conf}" -section ${key_name} \ + -config "${ca_config}" -section ${key_name} \ -out "${key_name}.csr" openssl x509 -req -days 3653 -in "${key_name}.csr" \ -copy_extensions copyall \ - -sha256 -CA "./ca.crt" \ - -CAkey "./ca.key" \ + -sha256 -CA "./${ca_name}-ca.crt" \ + -CAkey "./${ca_name}-ca.key" \ -CAcreateserial \ -out "${key_name}.crt" ''; diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index 1414f6e..d5b970c 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -78,24 +78,48 @@ makeScope newScope ( inherit all_hostnames controllers; k8s = self; }; + certificate_authorities = { + "client" = { + ca_config = ./package/k8s-ca/files/client-ca.conf; + }; + "requestheader-client" = { + ca_config = ./package/k8s-ca/files/requestheader-client-ca.conf; + }; + }; + certificate_authorities_merged = ( + builtins.mapAttrs (ca_name: ca_config: { inherit ca_name; } // ca_config) certificate_authorities + ); in { - ca = (callPackage ./package/k8s-ca/package.nix additional_vars); + ca = ( + builtins.mapAttrs ( + ca_name: ca_config: + (callPackage ./package/k8s-ca/package.nix (additional_vars // { inherit ca_name; } // ca_config)) + ) certificate_authorities + ); keys = ( - lib.genAttrs [ - "admin" - "controller0" - "controller1" - "controller2" - "worker0" - "worker1" - "worker2" - "kube-proxy" - "kube-scheduler" - "kube-controller-manager" - "kube-api-server" - "service-accounts" - ] (key_name: (callPackage ./package/tls-key/package.nix (additional_vars // { inherit key_name; }))) + builtins.mapAttrs + ( + key_name: key_config: + (callPackage ./package/tls-key/package.nix (additional_vars // { inherit key_name; } // key_config)) + ) + { + "admin" = { } // certificate_authorities_merged.client; + "controller0" = { } // certificate_authorities_merged.client; + "controller1" = { } // certificate_authorities_merged.client; + "controller2" = { } // certificate_authorities_merged.client; + "worker0" = { } // certificate_authorities_merged.client; + "worker1" = { } // certificate_authorities_merged.client; + "worker2" = { } // certificate_authorities_merged.client; + "kube-proxy" = { } // certificate_authorities_merged.client; + "kube-scheduler" = { } // certificate_authorities_merged.client; + "kube-controller-manager" = { } // certificate_authorities_merged.client; + "kube-api-server" = { } // certificate_authorities_merged.client; + "service-accounts" = { } // certificate_authorities_merged.client; + "controller0-proxy" = { } // certificate_authorities_merged.requestheader-client; + "controller1-proxy" = { } // certificate_authorities_merged.requestheader-client; + "controller2-proxy" = { } // certificate_authorities_merged.requestheader-client; + } ); ssh-keys = ( lib.genAttrs [ diff --git a/nix/kubernetes/roles/etcd/default.nix b/nix/kubernetes/roles/etcd/default.nix index 4ebc159..f086315 100644 --- a/nix/kubernetes/roles/etcd/default.nix +++ b/nix/kubernetes/roles/etcd/default.nix @@ -59,8 +59,8 @@ keyFile = "/.persist/keys/etcd/kube-api-server.key"; peerCertFile = "/.persist/keys/etcd/kube-api-server.crt"; peerKeyFile = "/.persist/keys/etcd/kube-api-server.key"; - trustedCaFile = "/.persist/keys/etcd/ca.crt"; - peerTrustedCaFile = "/.persist/keys/etcd/ca.crt"; + trustedCaFile = "/.persist/keys/etcd/client-ca.crt"; + peerTrustedCaFile = "/.persist/keys/etcd/client-ca.crt"; peerClientCertAuth = true; clientCertAuth = true; initialAdvertisePeerUrls = ( diff --git a/nix/kubernetes/roles/kube_apiserver/default.nix b/nix/kubernetes/roles/kube_apiserver/default.nix index 1c2edf7..267f642 100644 --- a/nix/kubernetes/roles/kube_apiserver/default.nix +++ b/nix/kubernetes/roles/kube_apiserver/default.nix @@ -65,15 +65,15 @@ in "--audit-log-path=/var/log/audit.log" "--authorization-mode=Node,RBAC" "--bind-address=0.0.0.0" - "--client-ca-file=/.persist/keys/kube/ca.crt" + "--client-ca-file=/.persist/keys/kube/client-ca.crt" "--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota" - "--etcd-cafile=/.persist/keys/kube/ca.crt" + "--etcd-cafile=/.persist/keys/kube/client-ca.crt" "--etcd-certfile=/.persist/keys/kube/kube-api-server.crt" "--etcd-keyfile=/.persist/keys/kube/kube-api-server.key" "--etcd-servers=${builtins.concatStringsSep "," config.me.kube_apiserver.etcd_services}" "--event-ttl=1h" "--encryption-provider-config=/.persist/keys/kube/encryption-config.yaml" - "--kubelet-certificate-authority=/.persist/keys/kube/ca.crt" + "--kubelet-certificate-authority=/.persist/keys/kube/client-ca.crt" "--kubelet-client-certificate=/.persist/keys/kube/kube-api-server.crt" "--kubelet-client-key=/.persist/keys/kube/kube-api-server.key" "--runtime-config='api/all=true'" @@ -85,6 +85,14 @@ in "--tls-private-key-file=/.persist/keys/kube/kube-api-server.key" "--tls-min-version=VersionTLS13" "--service-cluster-ip-range=fd00:3e42:e349::/112,10.197.0.0/16" + "--requestheader-client-ca-file=/.persist/keys/kube/requestheader-client-ca.crt" + "--requestheader-allowed-names=\"\"" # CN must be in this list to be valid. Blank = accept all CN. + "--requestheader-extra-headers-prefix=X-Remote-Extra" + "--requestheader-group-headers=X-Remote-Group" + "--requestheader-username-headers=X-Remote-User" + "--proxy-client-cert-file=/.persist/keys/kube/proxy.crt" + "--proxy-client-key-file=/.persist/keys/kube/proxy.key" + "--enable-aggregator-routing=true" "--v=2" # OLD: diff --git a/nix/kubernetes/roles/kube_controller_manager/default.nix b/nix/kubernetes/roles/kube_controller_manager/default.nix index fb93f9a..10eaff6 100644 --- a/nix/kubernetes/roles/kube_controller_manager/default.nix +++ b/nix/kubernetes/roles/kube_controller_manager/default.nix @@ -44,10 +44,10 @@ in "--node-cidr-mask-size-ipv4=20" # default is 24 "--node-cidr-mask-size-ipv6=112" # default is 64, must be smaller than cluster-cidr mask "--cluster-name=kubernetes" - "--cluster-signing-cert-file=/.persist/keys/kube/ca.crt" - "--cluster-signing-key-file=/.persist/keys/kube/ca.key" + "--cluster-signing-cert-file=/.persist/keys/kube/client-ca.crt" + "--cluster-signing-key-file=/.persist/keys/kube/client-ca.key" "--kubeconfig=/.persist/keys/kube/kube-controller-manager.kubeconfig" - "--root-ca-file=/.persist/keys/kube/ca.crt" + "--root-ca-file=/.persist/keys/kube/client-ca.crt" "--service-account-private-key-file=/.persist/keys/kube/service-accounts.key" # "--service-cluster-ip-range=10.197.0.0/16" # "--service-cluster-ip-range=2620:11f:7001:7:ffff:ffff:0ac5:0000/16" diff --git a/nix/kubernetes/roles/kubelet/default.nix b/nix/kubernetes/roles/kubelet/default.nix index 3da95a1..5f4e78e 100644 --- a/nix/kubernetes/roles/kubelet/default.nix +++ b/nix/kubernetes/roles/kubelet/default.nix @@ -22,7 +22,7 @@ let enabled = true; }; x509 = { - clientCAFile = "/.persist/keys/kube/ca.crt"; + clientCAFile = "/.persist/keys/kube/client-ca.crt"; }; }; authorization = { From 3cc0fb1d3f02ec3850d8795282f12b85c35e7330 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 11 Jan 2026 13:03:20 -0500 Subject: [PATCH 59/81] Fix proxy auth tls --- .../k8s-ca/files/requestheader-client-ca.conf | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/nix/kubernetes/keys/package/k8s-ca/files/requestheader-client-ca.conf b/nix/kubernetes/keys/package/k8s-ca/files/requestheader-client-ca.conf index f31e75c..a25c479 100644 --- a/nix/kubernetes/keys/package/k8s-ca/files/requestheader-client-ca.conf +++ b/nix/kubernetes/keys/package/k8s-ca/files/requestheader-client-ca.conf @@ -11,7 +11,10 @@ keyUsage = cRLSign, keyCertSign C = US ST = Washington L = Seattle -CN = CA +CN = Kubernetes +O = Kubernetes +OU = CA + [controller0-proxy] distinguished_name = controller0_distinguished_name @@ -23,7 +26,7 @@ basicConstraints = CA:FALSE extendedKeyUsage = clientAuth, serverAuth keyUsage = critical, digitalSignature, keyEncipherment nsCertType = client -nsComment = "controller0 Certificate" +nsComment = "controller0-proxy Certificate" subjectAltName = @controller0_alt_names subjectKeyIdentifier = hash @@ -36,8 +39,8 @@ L = Seattle [controller0_alt_names] IP.0 = 127.0.0.1 -IP.4 = 10.215.1.221 -IP.5 = 2620:11f:7001:7:ffff:ffff:0ad7:01dd +IP.1 = 10.215.1.221 +IP.2 = 2620:11f:7001:7:ffff:ffff:0ad7:01dd DNS.0 = controller0 [controller1-proxy] @@ -50,7 +53,7 @@ basicConstraints = CA:FALSE extendedKeyUsage = clientAuth, serverAuth keyUsage = critical, digitalSignature, keyEncipherment nsCertType = client -nsComment = "controller1 Certificate" +nsComment = "controller1-proxy Certificate" subjectAltName = @controller1_alt_names subjectKeyIdentifier = hash @@ -77,7 +80,7 @@ basicConstraints = CA:FALSE extendedKeyUsage = clientAuth, serverAuth keyUsage = critical, digitalSignature, keyEncipherment nsCertType = client -nsComment = "controller2 Certificate" +nsComment = "controller2-proxy Certificate" subjectAltName = @controller2_alt_names subjectKeyIdentifier = hash @@ -90,6 +93,6 @@ L = Seattle [controller2_alt_names] IP.0 = 127.0.0.1 -IP.6 = 10.215.1.223 -IP.7 = 2620:11f:7001:7:ffff:ffff:0ad7:01df +IP.1 = 10.215.1.223 +IP.2 = 2620:11f:7001:7:ffff:ffff:0ad7:01df DNS.0 = controller2 From e09e659d91ba7b0f88a0468835de38b947f0f1e0 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sat, 14 Feb 2026 12:34:24 -0500 Subject: [PATCH 60/81] Fix network for updated nix. --- nix/kubernetes/flake.nix | 9 ++++++++- nix/kubernetes/roles/network/default.nix | 6 +++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/nix/kubernetes/flake.nix b/nix/kubernetes/flake.nix index 36a66e7..a38d663 100644 --- a/nix/kubernetes/flake.nix +++ b/nix/kubernetes/flake.nix @@ -43,7 +43,14 @@ ... }: let - forAllSystems = nixpkgs.lib.genAttrs nixpkgs.lib.systems.flakeExposed; + forAllSystems = + func: + builtins.listToAttrs ( + map (system: { + name = system; + value = func system; + }) nixpkgs.lib.systems.flakeExposed + ); nodes = { controller0 = { system = "x86_64-linux"; diff --git a/nix/kubernetes/roles/network/default.nix b/nix/kubernetes/roles/network/default.nix index a25e70d..db04bed 100644 --- a/nix/kubernetes/roles/network/default.nix +++ b/nix/kubernetes/roles/network/default.nix @@ -42,9 +42,9 @@ services.resolved = { enable = true; # dnssec = "true"; - domains = [ "~." ]; - fallbackDns = [ ]; - # dnsovertls = "true"; + settings.Resolve.Domains = [ "~." ]; + settings.Resolve.FallbackDNS = [ ]; + # settings.Resolve.DNSOverTLS = "true"; }; # Without this, systemd-resolved will send DNS requests for .home.arpa to the per-link DNS server (172.16.0.1) which does not support DNS-over-TLS. This leads to the connection hanging and timing out. This causes firefox startup to take an extra 10+ seconds. From e54f272a6983ae972a5161e8c0b4dec10c90444a Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sat, 7 Mar 2026 20:14:07 -0500 Subject: [PATCH 61/81] Update packages in kubernetes/keys. --- nix/kubernetes/keys/flake.lock | 6 ++-- nix/kubernetes/keys/flake.nix | 13 +++---- .../keys/package/bootstrap-script/package.nix | 36 ++++++++++++------- nix/kubernetes/keys/scope.nix | 13 ++++--- 4 files changed, 43 insertions(+), 25 deletions(-) diff --git a/nix/kubernetes/keys/flake.lock b/nix/kubernetes/keys/flake.lock index 42c9926..a6e534e 100644 --- a/nix/kubernetes/keys/flake.lock +++ b/nix/kubernetes/keys/flake.lock @@ -2,11 +2,11 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1767892417, - "narHash": "sha256-dhhvQY67aboBk8b0/u0XB6vwHdgbROZT3fJAjyNh5Ww=", + "lastModified": 1772773019, + "narHash": "sha256-E1bxHxNKfDoQUuvriG71+f+s/NT0qWkImXsYZNFFfCs=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "3497aa5c9457a9d88d71fa93a4a8368816fbeeba", + "rev": "aca4d95fce4914b3892661bcb80b8087293536c6", "type": "github" }, "original": { diff --git a/nix/kubernetes/keys/flake.nix b/nix/kubernetes/keys/flake.nix index 3bdd04b..5bc06a6 100644 --- a/nix/kubernetes/keys/flake.nix +++ b/nix/kubernetes/keys/flake.nix @@ -14,13 +14,15 @@ packages = forAllSystems ( system: let - pkgs = nixpkgs.legacyPackages.${system}; - appliedOverlay = self.overlays.default pkgs pkgs; + pkgs = import nixpkgs { + inherit system; + overlays = [ self.overlays.default ]; + }; in { - deploy_script = appliedOverlay.k8s.deploy_script; - default = appliedOverlay.k8s.all_keys; - bootstrap_script = appliedOverlay.k8s.bootstrap_script; + deploy_script = pkgs.k8s.deploy_script; + default = pkgs.k8s.all_keys; + bootstrap_script = pkgs.k8s.bootstrap_script; } ); overlays.default = ( @@ -35,7 +37,6 @@ system: let pkgs = nixpkgs.legacyPackages.${system}; - appliedOverlay = self.overlays.default pkgs pkgs; in { default = pkgs.mkShell { diff --git a/nix/kubernetes/keys/package/bootstrap-script/package.nix b/nix/kubernetes/keys/package/bootstrap-script/package.nix index 2ae16d0..37bf3a6 100644 --- a/nix/kubernetes/keys/package/bootstrap-script/package.nix +++ b/nix/kubernetes/keys/package/bootstrap-script/package.nix @@ -48,28 +48,40 @@ let apply_manifests = "kubectl --kubeconfig=${k8s.client-configs.admin}/admin.kubeconfig apply --server-side --force-conflicts -f ${manifests}"; gateway_crds = [ (builtins.fetchurl { - url = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.2.0/config/crd/standard/gateway.networking.k8s.io_gatewayclasses.yaml"; - sha256 = "0vf8c3kzlf7p6bf92gmdrzjc22fr2dwkrzvvbnxlsb43knv1nbzl"; + url = "https://github.com/kubernetes-sigs/gateway-api/raw/refs/tags/v1.5.1/config/crd/standard/gateway.networking.k8s.io_backendtlspolicies.yaml"; + sha256 = "0wbrylglinba48ibqnrzs5vp4raa1azb0b83hjf2zmsk44bii24v"; }) (builtins.fetchurl { - url = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.2.0/config/crd/standard/gateway.networking.k8s.io_gateways.yaml"; - sha256 = "1dqwlsypcb5f37y7x48rrv27yfgkizcx2alqd2nngijl1qzir3wa"; + url = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/refs/tags/v1.5.1/config/crd/standard/gateway.networking.k8s.io_gatewayclasses.yaml"; + sha256 = "1x5yws3q7grd5xlnz071v6ymn707vycbp1s1d9cv7qbyfnrd8ji3"; }) (builtins.fetchurl { - url = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.2.0/config/crd/standard/gateway.networking.k8s.io_httproutes.yaml"; - sha256 = "05llfw6y66438r8kqy7krhyymyalkzxsaxjpa2zxzjk6z5mggbzq"; + url = "https://github.com/kubernetes-sigs/gateway-api/raw/refs/tags/v1.5.1/config/crd/standard/gateway.networking.k8s.io_gateways.yaml"; + sha256 = "0cbwwzmy3kqrn224a440pklcpfjv0w4mci133akw1n5l1qqfh5kl"; }) (builtins.fetchurl { - url = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.2.0/config/crd/standard/gateway.networking.k8s.io_referencegrants.yaml"; - sha256 = "0a9q0vhqcazfrni3ajcq8vm2b254vcjbgmkchsdq9l6cbpvx79jd"; + url = "https://github.com/kubernetes-sigs/gateway-api/raw/refs/tags/v1.5.1/config/crd/standard/gateway.networking.k8s.io_grpcroutes.yaml"; + sha256 = "1pr7g06q3m9dx2mfi4ri892nrrzq9z8d205sb53g4gadshjl37wp"; }) (builtins.fetchurl { - url = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.2.0/config/crd/standard/gateway.networking.k8s.io_grpcroutes.yaml"; - sha256 = "19hwvdwdj0sc5fihdskw492g52ail3kjjzm6vpflvp2vlqam629p"; + url = "https://github.com/kubernetes-sigs/gateway-api/raw/refs/tags/v1.5.1/config/crd/standard/gateway.networking.k8s.io_httproutes.yaml"; + sha256 = "0w632khanl080fzjf34vzqi7vhf2gf7mffh7726v3v5s16qh68k8"; }) (builtins.fetchurl { - url = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.2.0/config/crd/experimental/gateway.networking.k8s.io_tlsroutes.yaml"; - sha256 = "0b5pjihyzyyi4inz3avlkzvvccsynj9wsmx6znld04jmmvwpgxc9"; + url = "https://github.com/kubernetes-sigs/gateway-api/raw/refs/tags/v1.5.1/config/crd/standard/gateway.networking.k8s.io_listenersets.yaml"; + sha256 = "1fz0y0w8n6rn20jgynlp0xvg4r5cmdjfzc8kc41b1yzx366lc8cj"; + }) + (builtins.fetchurl { + url = "https://github.com/kubernetes-sigs/gateway-api/raw/refs/tags/v1.5.1/config/crd/standard/gateway.networking.k8s.io_referencegrants.yaml"; + sha256 = "0ldv1ydvdjq1vhml0j400gmih2dsr9n4g2mvylwp62zddr42r458"; + }) + (builtins.fetchurl { + url = "https://github.com/kubernetes-sigs/gateway-api/raw/refs/tags/v1.5.1/config/crd/standard/gateway.networking.k8s.io_tlsroutes.yaml"; + sha256 = "0ickl2fj23ch5j0l9pd8zr82qy2nws8ib1d24wjhx939qhkli3l1"; + }) + (builtins.fetchurl { + url = "https://github.com/kubernetes-sigs/gateway-api/raw/refs/tags/v1.5.1/config/crd/standard/gateway.networking.k8s.io_vap_safeupgrades.yaml"; + sha256 = "18aqz4abwyi9kiqx035rakq4g6a257r6y00y0my5djq64ylls6lq"; }) ]; in diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index d5b970c..09af1be 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -234,7 +234,7 @@ makeScope newScope ( encryption_config = (callPackage ./package/k8s-encryption-key/package.nix additional_vars); cilium-manifest = let - version = "1.18.5"; + version = "1.19.1"; in (callPackage ./package/helm-manifest/package.nix ( additional_vars @@ -243,7 +243,7 @@ makeScope newScope ( owner = "cilium"; repo = "cilium"; tag = "v${version}"; - hash = "sha256-348inOOQ/fgwTYnaSHrQ363xGYnx2UPts3D4ycDRsWE="; + hash = "sha256-wswY4u2Z7Z8hvGVnLONxSD1Mu1RV1AglC4ijUHsCCW4="; }; helm_name = "cilium"; helm_namespace = "kube-system"; @@ -262,13 +262,18 @@ makeScope newScope ( "ipv4" = { "enabled" = true; }; + "externalIPs" = { + "enabled" = true; + }; "enableIPv6Masquerade" = false; "enableIPv4BIGTCP" = true; "enableIPv6BIGTCP" = true; "routingMode" = "native"; "autoDirectNodeRoutes" = true; "ipv4NativeRoutingCIDR" = "10.200.0.0/16"; - "ipv6NativeRoutingCIDR" = "2620:11f:7001:7:ffff::/80"; + "ipv6NativeRoutingCIDR" = "2620:11f:7001:7:ffff:eeee::/96"; + # "ipv6NativeRoutingCIDR" = "2620:11f:7001:7:ffff::/80"; + # "l7Proxy" = true; # Needed for cilium gateway controller "hubble" = { "relay" = { @@ -283,7 +288,7 @@ makeScope newScope ( }; }; - "policyEnforcementMode" = "never"; + "policyEnforcementMode" = "never"; # This is temporary for debugging # TODO: Read and maybe apply https://docs.cilium.io/en/stable/operations/performance/tuning/ From 1d3d178fc8470ada85f6e59b0b79d2c9c1a3ee39 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 15 Mar 2026 12:22:26 -0400 Subject: [PATCH 62/81] Downgrade to gateway 1.4.1. 1.5 came out recently, so no gateway providers support it. --- .../keys/package/bootstrap-script/package.nix | 36 +++++++------------ 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/nix/kubernetes/keys/package/bootstrap-script/package.nix b/nix/kubernetes/keys/package/bootstrap-script/package.nix index 37bf3a6..cd2995c 100644 --- a/nix/kubernetes/keys/package/bootstrap-script/package.nix +++ b/nix/kubernetes/keys/package/bootstrap-script/package.nix @@ -48,40 +48,28 @@ let apply_manifests = "kubectl --kubeconfig=${k8s.client-configs.admin}/admin.kubeconfig apply --server-side --force-conflicts -f ${manifests}"; gateway_crds = [ (builtins.fetchurl { - url = "https://github.com/kubernetes-sigs/gateway-api/raw/refs/tags/v1.5.1/config/crd/standard/gateway.networking.k8s.io_backendtlspolicies.yaml"; - sha256 = "0wbrylglinba48ibqnrzs5vp4raa1azb0b83hjf2zmsk44bii24v"; + url = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/refs/tags/v1.4.1/config/crd/standard/gateway.networking.k8s.io_backendtlspolicies.yaml"; + sha256 = "0kspa45s5kmbw8gm74r1wnj1chb8k8nnfvisnljnrawwq6q38kzr"; }) (builtins.fetchurl { - url = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/refs/tags/v1.5.1/config/crd/standard/gateway.networking.k8s.io_gatewayclasses.yaml"; - sha256 = "1x5yws3q7grd5xlnz071v6ymn707vycbp1s1d9cv7qbyfnrd8ji3"; + url = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/refs/tags/v1.4.1/config/crd/standard/gateway.networking.k8s.io_gatewayclasses.yaml"; + sha256 = "1j78da9p6ndqp199f6rmz7dxxzpirag6v10ncscs992yjv9qm5bx"; }) (builtins.fetchurl { - url = "https://github.com/kubernetes-sigs/gateway-api/raw/refs/tags/v1.5.1/config/crd/standard/gateway.networking.k8s.io_gateways.yaml"; - sha256 = "0cbwwzmy3kqrn224a440pklcpfjv0w4mci133akw1n5l1qqfh5kl"; + url = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/refs/tags/v1.4.1/config/crd/standard/gateway.networking.k8s.io_gateways.yaml"; + sha256 = "0vmsydfbizdlp7kcc529k6b6h7sy6xbdv7cccrxij7whzhjs8bm0"; }) (builtins.fetchurl { - url = "https://github.com/kubernetes-sigs/gateway-api/raw/refs/tags/v1.5.1/config/crd/standard/gateway.networking.k8s.io_grpcroutes.yaml"; - sha256 = "1pr7g06q3m9dx2mfi4ri892nrrzq9z8d205sb53g4gadshjl37wp"; + url = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/refs/tags/v1.4.1/config/crd/standard/gateway.networking.k8s.io_grpcroutes.yaml"; + sha256 = "1asd3kjglxn7ygcp2bmy98b4v40nzgz8snlyc355285k5js6h85p"; }) (builtins.fetchurl { - url = "https://github.com/kubernetes-sigs/gateway-api/raw/refs/tags/v1.5.1/config/crd/standard/gateway.networking.k8s.io_httproutes.yaml"; - sha256 = "0w632khanl080fzjf34vzqi7vhf2gf7mffh7726v3v5s16qh68k8"; + url = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/refs/tags/v1.4.1/config/crd/standard/gateway.networking.k8s.io_httproutes.yaml"; + sha256 = "055grkxs7zwxdwykhwj2vpzwj1i0cgp8ihp9ja93379h49y7gilq"; }) (builtins.fetchurl { - url = "https://github.com/kubernetes-sigs/gateway-api/raw/refs/tags/v1.5.1/config/crd/standard/gateway.networking.k8s.io_listenersets.yaml"; - sha256 = "1fz0y0w8n6rn20jgynlp0xvg4r5cmdjfzc8kc41b1yzx366lc8cj"; - }) - (builtins.fetchurl { - url = "https://github.com/kubernetes-sigs/gateway-api/raw/refs/tags/v1.5.1/config/crd/standard/gateway.networking.k8s.io_referencegrants.yaml"; - sha256 = "0ldv1ydvdjq1vhml0j400gmih2dsr9n4g2mvylwp62zddr42r458"; - }) - (builtins.fetchurl { - url = "https://github.com/kubernetes-sigs/gateway-api/raw/refs/tags/v1.5.1/config/crd/standard/gateway.networking.k8s.io_tlsroutes.yaml"; - sha256 = "0ickl2fj23ch5j0l9pd8zr82qy2nws8ib1d24wjhx939qhkli3l1"; - }) - (builtins.fetchurl { - url = "https://github.com/kubernetes-sigs/gateway-api/raw/refs/tags/v1.5.1/config/crd/standard/gateway.networking.k8s.io_vap_safeupgrades.yaml"; - sha256 = "18aqz4abwyi9kiqx035rakq4g6a257r6y00y0my5djq64ylls6lq"; + url = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/refs/tags/v1.4.1/config/crd/standard/gateway.networking.k8s.io_referencegrants.yaml"; + sha256 = "11351y1aq7q5lbib6h2n1vil06h143bs44ywsv2g9500x7wc4kyp"; }) ]; in From fc32a42750d6560397b4c3b766476aa7a78bafa9 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 16 Mar 2026 19:04:56 -0400 Subject: [PATCH 63/81] Enforce cilium network policies. --- nix/kubernetes/keys/scope.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index 09af1be..f1f87b3 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -288,8 +288,6 @@ makeScope newScope ( }; }; - "policyEnforcementMode" = "never"; # This is temporary for debugging - # TODO: Read and maybe apply https://docs.cilium.io/en/stable/operations/performance/tuning/ # --set hostFirewall.enabled=true From 8f5f59ee87593b83a7d8b68f9754f1713b06ab08 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Thu, 19 Mar 2026 18:16:20 -0400 Subject: [PATCH 64/81] Add generation for in-repo secrets. --- nix/kubernetes/README.org | 13 ++-- nix/kubernetes/keys/flake.nix | 1 + .../files/manifests/flux_apply_git.yaml | 1 - .../keys/package/k8s-keys/package.nix | 5 ++ .../package/k8s-secret-encrypted/package.nix | 65 +++++++++++++++++ .../mrmanager-repo-secrets/package.nix | 70 +++++++++++++++++++ nix/kubernetes/keys/scope.nix | 1 + 7 files changed, 149 insertions(+), 7 deletions(-) create mode 100644 nix/kubernetes/keys/package/k8s-secret-encrypted/package.nix create mode 100644 nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix diff --git a/nix/kubernetes/README.org b/nix/kubernetes/README.org index 9437c61..bce9cee 100644 --- a/nix/kubernetes/README.org +++ b/nix/kubernetes/README.org @@ -11,12 +11,13 @@ ]; #+end_src * IP Ranges -| | IPv4 | IPv6 | -|---------------+-----------------------------+-----------------------------------------| -| Pod | 10.200.0.0/16 | 2620:11f:7001:7:ffff:eeee::/96 | -| Service | 10.197.0.0/16 | fd00:3e42:e349::/112 | -| Node | 10.215.1.0/24 | 2620:11f:7001:7:ffff:ffff:0ad7:0100/120 | -| Load Balancer | 74.80.180.139-74.80.180.142 | 2620:11f:7001:7:ffff:dddd::/96 | +| | IPv4 | IPv6 | +|------------------------------+-----------------------------+-----------------------------------------| +| Pod | 10.200.0.0/16 | 2620:11f:7001:7:ffff:eeee::/96 | +| Service | 10.197.0.0/16 | fd00:3e42:e349::/112 | +| Node | 10.215.1.0/24 | 2620:11f:7001:7:ffff:ffff:0ad7:0100/120 | +| Load Balancer | 74.80.180.139-74.80.180.142 | 2620:11f:7001:7:ffff:dddd::/96 | +| PowerDNS from inside cluster | 10.215.1.211 | | * Healthcheck ** Check cilium status #+begin_src bash diff --git a/nix/kubernetes/keys/flake.nix b/nix/kubernetes/keys/flake.nix index 5bc06a6..851cd71 100644 --- a/nix/kubernetes/keys/flake.nix +++ b/nix/kubernetes/keys/flake.nix @@ -23,6 +23,7 @@ deploy_script = pkgs.k8s.deploy_script; default = pkgs.k8s.all_keys; bootstrap_script = pkgs.k8s.bootstrap_script; + mrmanager_repo_secrets = pkgs.k8s.mrmanager_repo_secrets; } ); overlays.default = ( diff --git a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_apply_git.yaml b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_apply_git.yaml index d60e9bb..91e6f46 100644 --- a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_apply_git.yaml +++ b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_apply_git.yaml @@ -14,7 +14,6 @@ spec: ignore: | bootstrap .sops.yaml - secrets/ --- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization diff --git a/nix/kubernetes/keys/package/k8s-keys/package.nix b/nix/kubernetes/keys/package/k8s-keys/package.nix index 71596b8..b6b2b87 100644 --- a/nix/kubernetes/keys/package/k8s-keys/package.nix +++ b/nix/kubernetes/keys/package/k8s-keys/package.nix @@ -10,12 +10,17 @@ let cp ${k8s.deploy_script} $out/deploy_script cp ${k8s.bootstrap_script} $out/bootstrap_script ''; + mrmanager_repo_secrets = runCommand "mrmanager_repo_secrets" { } '' + mkdir $out + cp -r ${k8s.mrmanager_repo_secrets} $out/mrmanager_repo_secrets + ''; in symlinkJoin { name = "k8s-keys"; paths = [ scripts k8s.encryption_config + mrmanager_repo_secrets ] ++ (builtins.attrValues k8s.ca) ++ (builtins.attrValues k8s.keys) diff --git a/nix/kubernetes/keys/package/k8s-secret-encrypted/package.nix b/nix/kubernetes/keys/package/k8s-secret-encrypted/package.nix new file mode 100644 index 0000000..6d2fbbc --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-secret-encrypted/package.nix @@ -0,0 +1,65 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + pkgs, + stdenv, + kubectl, + gnupg, + source_file, + output_filename, + pgp_public_key, + ... +}: +let + pgp_key_id_command = pkgs.runCommand "pgp_key_id_command" { } '' + mkdir keyring + export GNUPGHOME=$(readlink -f keyring) + ${gnupg}/bin/gpg --with-fingerprint --with-colons --keyid-format LONG "${pgp_public_key}" | grep '^pub' | cut -d ':' -f 5 > $out + ''; + pgp_key_id = builtins.readFile pgp_key_id_command; + sops_config = { + creation_rules = [ + { + "path_regex" = ".*.yaml"; + "encrypted_regex" = "^(data|stringData)$"; + "pgp" = pgp_key_id; + } + ]; + }; + settingsFormat = pkgs.formats.yaml { }; + yaml_body = settingsFormat.generate ".sops.yaml" sops_config; + yaml_file = pkgs.writeTextFile { + name = ".sops.yaml"; + text = (builtins.readFile yaml_body); + }; +in +stdenv.mkDerivation (finalAttrs: { + name = "k8s-secret-encrypted-${output_filename}"; + nativeBuildInputs = [ + kubectl + gnupg + ]; + buildInputs = [ ]; + + unpackPhase = "true"; + + buildPhase = '' + mkdir keyring + export GNUPGHOME=$(readlink -f keyring) + cat "${pgp_public_key}" | gpg --import + ''; + + installPhase = '' + set -x + export GNUPGHOME=$(readlink -f keyring) + mkdir "$out" + cat "${source_file}" | ${pkgs.sops}/bin/sops --config "${yaml_file}" encrypt --filename-override "${output_filename}" | tee "$out/${output_filename}" + ''; +}) diff --git a/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix b/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix new file mode 100644 index 0000000..c08fa86 --- /dev/null +++ b/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix @@ -0,0 +1,70 @@ +{ + lib, + k8s, + callPackage, + runCommand, + symlinkJoin, + ... +}: +let + pre_encryption_secrets = + builtins.mapAttrs + ( + secret_namespace: secrets: + (builtins.mapAttrs ( + secret_name: secret_values: + (callPackage ../../package/k8s-secret-generic/package.nix { + inherit secret_name secret_namespace secret_values; + }) + ) secrets) + ) + { + "external-dns" = { + "rfc2136" = { + "EXTERNAL_DNS_RFC2136_TSIG_SECRET" = ( + builtins.readFile "${./secrets/external-dns/rfc2136/EXTERNAL_DNS_RFC2136_TSIG_SECRET}" + ); + }; + }; + "cert-manager" = { + "rfc2136" = { + "TSIG_SECRET" = (builtins.readFile "${./secrets/cert-manager/rfc2136/TSIG_SECRET}"); + }; + }; + }; + encrypted_secrets = ( + builtins.mapAttrs ( + secret_namespace: secrets: + (builtins.mapAttrs ( + secret_name: secret_package: + (callPackage ../../package/k8s-secret-encrypted/package.nix { + source_file = "${ + pre_encryption_secrets."${secret_namespace}"."${secret_name}" + }/${secret_name}.yaml"; + output_filename = "${secret_name}.yaml"; + pgp_public_key = "${k8s.pgp-keys.flux_gpg}/flux_gpg_public_key.asc"; + }) + ) secrets) + ) pre_encryption_secrets + ); + combined_script = ( + lib.concatMapStringsSep "\n" ( + secret_namespace: + '' + mkdir -p $out/${secret_namespace} + '' + + (lib.concatMapStringsSep "\n" (secret_name: '' + cat ${ + encrypted_secrets."${secret_namespace}"."${secret_name}" + }/${secret_name}.yaml > $out/${secret_namespace}/${secret_name}.yaml + '') (builtins.attrNames encrypted_secrets."${secret_namespace}")) + ) (builtins.attrNames encrypted_secrets) + ); + gen_in_repo_secrets = runCommand "gen_in_repo_secrets" { } combined_script; +in +symlinkJoin { + name = "in-repo-secrets"; + paths = [ + gen_in_repo_secrets + ]; +} diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index f1f87b3..cd0ca35 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -373,5 +373,6 @@ makeScope newScope ( all_keys = (callPackage ./package/k8s-keys/package.nix additional_vars); deploy_script = (callPackage ./package/deploy-script/package.nix additional_vars); bootstrap_script = (callPackage ./package/bootstrap-script/package.nix additional_vars); + mrmanager_repo_secrets = (callPackage ./package/mrmanager-repo-secrets/package.nix additional_vars); } ) From b593fdc8a7bba6e6a5ca4781b9b0ef4c9719a23e Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Thu, 16 Apr 2026 20:33:54 -0400 Subject: [PATCH 65/81] Add gitea secrets. --- .../keys/package/mrmanager-repo-secrets/package.nix | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix b/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix index c08fa86..f42fe55 100644 --- a/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix +++ b/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix @@ -31,6 +31,12 @@ let "TSIG_SECRET" = (builtins.readFile "${./secrets/cert-manager/rfc2136/TSIG_SECRET}"); }; }; + "gitea" = { + "gitea-env" = { + "GITEA_ADMIN_USERNAME" = (builtins.readFile "${./secrets/gitea/gitea-env/GITEA_ADMIN_USERNAME}"); + "GITEA_ADMIN_PASSWORD" = (builtins.readFile "${./secrets/gitea/gitea-env/GITEA_ADMIN_PASSWORD}"); + }; + }; }; encrypted_secrets = ( builtins.mapAttrs ( From a4b20c1ead65f792b2a2b65bfb8f259c31550308 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sat, 18 Apr 2026 15:49:08 -0400 Subject: [PATCH 66/81] Enable the firewall. --- nix/kubernetes/README.org | 4 ++ .../keys/package/deploy-script/package.nix | 41 +++++++++++++++ nix/kubernetes/roles/firewall/default.nix | 51 +++++++++++++++---- 3 files changed, 86 insertions(+), 10 deletions(-) diff --git a/nix/kubernetes/README.org b/nix/kubernetes/README.org index bce9cee..4e1aa19 100644 --- a/nix/kubernetes/README.org +++ b/nix/kubernetes/README.org @@ -32,6 +32,10 @@ #+begin_src bash kubectl -n kube-system exec ds/cilium -- cilium-dbg monitor --type drop #+end_src +** Show dropped packets for a specific pod +#+begin_src bash + kubectl -n kube-system exec ds/cilium -- hubble observe --since 30s --pod cnpg-system/cnpg-controller-manager-84d498b97-q5m4n --type drop +#+end_src ** Install flux #+begin_src bash nix shell 'nixpkgs#fluxcd' diff --git a/nix/kubernetes/keys/package/deploy-script/package.nix b/nix/kubernetes/keys/package/deploy-script/package.nix index 78fff16..11db150 100644 --- a/nix/kubernetes/keys/package/deploy-script/package.nix +++ b/nix/kubernetes/keys/package/deploy-script/package.nix @@ -53,6 +53,33 @@ let group = "11236"; mode = "0600"; }) + + (lib.concatMapStringsSep "\n" create_pv_dir [ + { + path = "manual-pv/gitea-psql"; + owner = "26"; + group = "26"; + mode = "0777"; + } + # { + # path = "manual-pv/gitea"; + # owner = "1000"; + # group = "1000"; + # mode = "0777"; + # } + # { + # path = "manual-pv/gitea/gitea"; + # owner = "1000"; + # group = "1000"; + # mode = "0700"; + # } + # { + # path = "manual-pv/gitea/gitea/public"; + # owner = "1000"; + # group = "1000"; + # mode = "0755"; + # } + ]) + ); deploy_script = (writeShellScript "deploy-script" deploy_script_body); deploy_file = ( @@ -287,6 +314,20 @@ let echo "${public_key_name} is already trusted in ${destination}" fi ''; + create_pv_dir = + { + path, + owner, + group, + mode, + }: + '' + ## + ## create pv directory ${path} + ## + ${openssh}/bin/ssh mrmanager doas install -d -o "${owner}" -g "${group}" -m "${mode}" "/nk8spv/${path}" + ''; + in stdenv.mkDerivation (finalAttrs: { name = "deploy-script"; diff --git a/nix/kubernetes/roles/firewall/default.nix b/nix/kubernetes/roles/firewall/default.nix index 3630b5c..984fed0 100644 --- a/nix/kubernetes/roles/firewall/default.nix +++ b/nix/kubernetes/roles/firewall/default.nix @@ -32,23 +32,54 @@ # "net.ipv6.conf.all.forwarding" = 1; }; - networking.firewall.enable = false; + networking.firewall.enable = true; networking.nftables.enable = true; # We want to filter forwarded traffic. # Also needed for `networking.firewall.extraForwardRules` to do anything. networking.firewall.filterForward = true; - networking.firewall.extraInputRules = '' - ip6 saddr 2620:11f:7001:7:ffff:eeee::/96 accept - ip6 saddr fd00:3e42:e349::/112 accept - ip6 saddr 2620:11f:7001:7:ffff:ffff:0ad7:0100/120 accept + # Allow traffic from the pods on the lxc interfaces even though the interfaces do not have the correct ip addressses set for the return path. + networking.firewall.extraReversePathFilterRules = '' + iifname "lxc*" ip6 saddr 2620:11f:7001:7:ffff:eeee::/96 accept + iifname "lxc*" ip saddr 10.200.0.0/16 accept ''; - networking.firewall.extraForwardRules = '' - ip6 daddr 2620:11f:7001:7:ffff:eeee::/96 accept - ip6 daddr fd00:3e42:e349::/112 accept - ip6 daddr 2620:11f:7001:7:ffff:ffff:0ad7:0100/120 accept - ''; + networking.firewall.extraInputRules = builtins.concatStringsSep "\n" [ + # Allow pod-to-node communication + '' + ip6 saddr 2620:11f:7001:7:ffff:eeee::/96 accept + '' + ]; + + # networking.firewall.extraInputRules = '' + # ip6 saddr 2620:11f:7001:7:ffff:eeee::/96 accept + # ip6 saddr fd00:3e42:e349::/112 accept + # ip6 saddr 2620:11f:7001:7:ffff:ffff:0ad7:0100/120 accept + # ''; + + networking.firewall.extraForwardRules = builtins.concatStringsSep "\n" [ + # Allow pod to external communication + '' + iifname "lxc*" ip6 saddr 2620:11f:7001:7:ffff:eeee::/96 accept + iifname "lxc*" ip saddr 10.200.0.0/16 accept + '' + # Allow pod-to-pod communication + '' + ip saddr 10.200.0.0/16 ip daddr 10.200.0.0/16 accept + ip6 saddr 2620:11f:7001:7:ffff:eeee::/96 ip6 daddr 2620:11f:7001:7:ffff:eeee::/96 accept + '' + # Allow external-to-pod communication + '' + ip daddr 10.200.0.0/16 accept + ip6 daddr 2620:11f:7001:7:ffff:eeee::/96 accept + '' + ]; + + # networking.firewall.extraForwardRules = '' + # ip6 daddr 2620:11f:7001:7:ffff:eeee::/96 accept + # ip6 daddr fd00:3e42:e349::/112 accept + # ip6 daddr 2620:11f:7001:7:ffff:ffff:0ad7:0100/120 accept + # ''; # Check logs for blocked connections: # journalctl -k or dmesg From d14d8fe4a424c42edd4971eb229bc1f23aa4ea47 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sat, 18 Apr 2026 16:22:47 -0400 Subject: [PATCH 67/81] Add dex secrets. --- .../mrmanager-repo-secrets/package.nix | 90 +++++++++++++++++-- 1 file changed, 85 insertions(+), 5 deletions(-) diff --git a/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix b/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix index f42fe55..1691b46 100644 --- a/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix +++ b/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix @@ -1,5 +1,6 @@ { lib, + pkgs, k8s, callPackage, runCommand, @@ -19,6 +20,16 @@ let ) secrets) ) { + "cert-manager" = { + "rfc2136" = { + "TSIG_SECRET" = (builtins.readFile "${./secrets/cert-manager/rfc2136/TSIG_SECRET}"); + }; + }; + "dex" = { + "files" = { + "config.yaml" = dex_config_yaml; + }; + }; "external-dns" = { "rfc2136" = { "EXTERNAL_DNS_RFC2136_TSIG_SECRET" = ( @@ -26,11 +37,6 @@ let ); }; }; - "cert-manager" = { - "rfc2136" = { - "TSIG_SECRET" = (builtins.readFile "${./secrets/cert-manager/rfc2136/TSIG_SECRET}"); - }; - }; "gitea" = { "gitea-env" = { "GITEA_ADMIN_USERNAME" = (builtins.readFile "${./secrets/gitea/gitea-env/GITEA_ADMIN_USERNAME}"); @@ -67,6 +73,80 @@ let ) (builtins.attrNames encrypted_secrets) ); gen_in_repo_secrets = runCommand "gen_in_repo_secrets" { } combined_script; + + ## Utilities + inherit ((import ../../../functions/to_yaml.nix) { inherit pkgs; }) to_yaml; + + ## dex + dex_static_client = + { + id, + name, + redirectURIs, + }: + let + generate_key = runCommand "generate_key" { } '' + set +o pipefail + dd if=/dev/urandom | tr --complement --delete '[:alnum:]' | dd bs=32 count=1 of="$out" + ''; + in + { + inherit id name redirectURIs; + secret = builtins.readFile generate_key; + }; + dex_config = { + issuer = "https://dex.fizz.buzz"; + storage = { + config = { + inCluster = true; + }; + type = "kubernetes"; + }; + logger = { + level = "debug"; + }; + web = { + http = "0.0.0.0:5556"; + }; + oauth2 = { + alwaysShowLoginScreen = false; + skipApprovalScreen = true; + }; + staticClients = map dex_static_client [ + { + id = "prometheus"; + name = "Prometheus"; + redirectURIs = [ "https://prometheus.fizz.buzz/oauth2/callback" ]; + } + { + id = "harbor"; + name = "Harbor"; + redirectURIs = [ "https://harbor.fizz.buzz/c/oidc/callback" ]; + } + { + id = "tekton"; + name = "Tekton"; + redirectURIs = [ "https://tekton.fizz.buzz/oauth2/callback" ]; + } + { + id = "homepage-staging"; + name = "Homepage staging"; + redirectURIs = [ "https://staging.fizz.buzz/oauth2/callback" ]; + } + { + id = "gitea"; + name = "gitea"; + redirectURIs = [ "https://code.fizz.buzz/oauth2/callback" ]; + } + ]; + enablePasswordDB = true; + staticPasswords = (import ./secrets/dex/static_passwords.nix); + expiry = { + idTokens = "1h"; + signingKeys = "4h"; + }; + }; + dex_config_yaml = to_yaml "config.yml" dex_config; in symlinkJoin { name = "in-repo-secrets"; From 7416e412df513d5b91f2bcba50a9d7ae49ffccc3 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sat, 18 Apr 2026 17:45:47 -0400 Subject: [PATCH 68/81] Add oauth2 proxy secrets. --- .../mrmanager-repo-secrets/package.nix | 29 ++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix b/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix index 1691b46..5824a1c 100644 --- a/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix +++ b/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix @@ -42,6 +42,7 @@ let "GITEA_ADMIN_USERNAME" = (builtins.readFile "${./secrets/gitea/gitea-env/GITEA_ADMIN_USERNAME}"); "GITEA_ADMIN_PASSWORD" = (builtins.readFile "${./secrets/gitea/gitea-env/GITEA_ADMIN_PASSWORD}"); }; + "oauth2-env" = oauth2_env { dex_id = "gitea"; }; }; }; encrypted_secrets = ( @@ -76,6 +77,15 @@ let ## Utilities inherit ((import ../../../functions/to_yaml.nix) { inherit pkgs; }) to_yaml; + generate_key = + len: name: + builtins.readFile ( + runCommand "generate_key" { } '' + set +o pipefail + # ${name} + dd if=/dev/urandom | tr --complement --delete '[:alnum:]' | dd bs=${toString len} count=1 of="$out" + '' + ); ## dex dex_static_client = @@ -84,15 +94,9 @@ let name, redirectURIs, }: - let - generate_key = runCommand "generate_key" { } '' - set +o pipefail - dd if=/dev/urandom | tr --complement --delete '[:alnum:]' | dd bs=32 count=1 of="$out" - ''; - in { inherit id name redirectURIs; - secret = builtins.readFile generate_key; + secret = generate_key 32 "dex_static_client ${id}"; }; dex_config = { issuer = "https://dex.fizz.buzz"; @@ -147,6 +151,17 @@ let }; }; dex_config_yaml = to_yaml "config.yml" dex_config; + + ## oauth2-proxy + oauth2_env = + { dex_id }: + { + "OAUTH2_PROXY_CLIENT_SECRET" = + (builtins.head ( + builtins.filter (static_client: static_client.id == dex_id) dex_config.staticClients + )).secret; + "OAUTH2_PROXY_COOKIE_SECRET" = generate_key 32 "OAUTH2_PROXY_COOKIE_SECRET ${dex_id}"; + }; in symlinkJoin { name = "in-repo-secrets"; From cdb29944479b986f8c695688040202fe2feb76fe Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sat, 18 Apr 2026 19:20:16 -0400 Subject: [PATCH 69/81] Switch to the experimental gateway CRDs for TCPRoute support. --- .../keys/package/bootstrap-script/package.nix | 43 ++++++++----------- 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/nix/kubernetes/keys/package/bootstrap-script/package.nix b/nix/kubernetes/keys/package/bootstrap-script/package.nix index cd2995c..28647b7 100644 --- a/nix/kubernetes/keys/package/bootstrap-script/package.nix +++ b/nix/kubernetes/keys/package/bootstrap-script/package.nix @@ -10,6 +10,7 @@ { lib, stdenv, + fetchFromGitHub, writeShellScript, k8s, ... @@ -46,31 +47,25 @@ let ) ); apply_manifests = "kubectl --kubeconfig=${k8s.client-configs.admin}/admin.kubeconfig apply --server-side --force-conflicts -f ${manifests}"; + gateway_crds_repo = fetchFromGitHub { + owner = "kubernetes-sigs"; + repo = "gateway-api"; + rev = "v1.4.1"; + sha256 = "sha256-/GHyikcC2QGDN0ndpY6/xvSEEnpSsLrNU+lFElCKBs8="; + }; gateway_crds = [ - (builtins.fetchurl { - url = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/refs/tags/v1.4.1/config/crd/standard/gateway.networking.k8s.io_backendtlspolicies.yaml"; - sha256 = "0kspa45s5kmbw8gm74r1wnj1chb8k8nnfvisnljnrawwq6q38kzr"; - }) - (builtins.fetchurl { - url = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/refs/tags/v1.4.1/config/crd/standard/gateway.networking.k8s.io_gatewayclasses.yaml"; - sha256 = "1j78da9p6ndqp199f6rmz7dxxzpirag6v10ncscs992yjv9qm5bx"; - }) - (builtins.fetchurl { - url = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/refs/tags/v1.4.1/config/crd/standard/gateway.networking.k8s.io_gateways.yaml"; - sha256 = "0vmsydfbizdlp7kcc529k6b6h7sy6xbdv7cccrxij7whzhjs8bm0"; - }) - (builtins.fetchurl { - url = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/refs/tags/v1.4.1/config/crd/standard/gateway.networking.k8s.io_grpcroutes.yaml"; - sha256 = "1asd3kjglxn7ygcp2bmy98b4v40nzgz8snlyc355285k5js6h85p"; - }) - (builtins.fetchurl { - url = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/refs/tags/v1.4.1/config/crd/standard/gateway.networking.k8s.io_httproutes.yaml"; - sha256 = "055grkxs7zwxdwykhwj2vpzwj1i0cgp8ihp9ja93379h49y7gilq"; - }) - (builtins.fetchurl { - url = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/refs/tags/v1.4.1/config/crd/standard/gateway.networking.k8s.io_referencegrants.yaml"; - sha256 = "11351y1aq7q5lbib6h2n1vil06h143bs44ywsv2g9500x7wc4kyp"; - }) + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_backendtlspolicies.yaml" + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_referencegrants.yaml" + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.x-k8s.io_xmeshes.yaml" + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_gatewayclasses.yaml" + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_udproutes.yaml" + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_tlsroutes.yaml" + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.x-k8s.io_xbackendtrafficpolicies.yaml" + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_gateways.yaml" + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.x-k8s.io_xlistenersets.yaml" + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_tcproutes.yaml" + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_httproutes.yaml" + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_grpcroutes.yaml" ]; in stdenv.mkDerivation (finalAttrs: { From c1ec96044e60be265f45ea41e8c1455a845014c4 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 26 Apr 2026 09:11:35 -0400 Subject: [PATCH 70/81] Add harbor secrets. --- .../keys/package/deploy-script/package.nix | 6 +++ .../mrmanager-repo-secrets/package.nix | 41 +++++++++++++++++-- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/nix/kubernetes/keys/package/deploy-script/package.nix b/nix/kubernetes/keys/package/deploy-script/package.nix index 11db150..f3d84ce 100644 --- a/nix/kubernetes/keys/package/deploy-script/package.nix +++ b/nix/kubernetes/keys/package/deploy-script/package.nix @@ -60,6 +60,12 @@ let group = "26"; mode = "0777"; } + { + path = "manual-pv/harbor-psql"; + owner = "26"; + group = "26"; + mode = "0755"; + } # { # path = "manual-pv/gitea"; # owner = "1000"; diff --git a/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix b/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix index 5824a1c..a97fcb5 100644 --- a/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix +++ b/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix @@ -44,6 +44,21 @@ let }; "oauth2-env" = oauth2_env { dex_id = "gitea"; }; }; + "harbor" = { + "harbor-config" = { + "config.json" = helm_json_escape harbor_config_json; + }; + "dockerhub-auth-config" = { + "basic_auth.include" = ( + builtins.readFile "${./secrets/harbor/dockerhub-auth-config/basic_auth.include}" + ); + }; + "harbor-admin-password" = { + "HARBOR_ADMIN_PASSWORD" = ( + builtins.readFile "${./secrets/harbor/harbor-admin-password/HARBOR_ADMIN_PASSWORD}" + ); + }; + }; }; encrypted_secrets = ( builtins.mapAttrs ( @@ -86,8 +101,14 @@ let dd if=/dev/urandom | tr --complement --delete '[:alnum:]' | dd bs=${toString len} count=1 of="$out" '' ); + helm_json_escape = json: builtins.toJSON json; ## dex + get_dex_config = + client_id: + (builtins.head ( + builtins.filter (static_client: static_client.id == client_id) dex_config.staticClients + )); dex_static_client = { id, @@ -156,12 +177,24 @@ let oauth2_env = { dex_id }: { - "OAUTH2_PROXY_CLIENT_SECRET" = - (builtins.head ( - builtins.filter (static_client: static_client.id == dex_id) dex_config.staticClients - )).secret; + "OAUTH2_PROXY_CLIENT_SECRET" = (get_dex_config dex_id).secret; "OAUTH2_PROXY_COOKIE_SECRET" = generate_key 32 "OAUTH2_PROXY_COOKIE_SECRET ${dex_id}"; }; + + ## harbor + harbor_dex_config = get_dex_config "harbor"; + harbor_config = { + "auth_mode" = "oidc_auth"; + "self_registration" = "false"; + "oidc_name" = "harbor"; + "oidc_endpoint" = "https://dex.fizz.buzz"; + "oidc_client_id" = harbor_dex_config.id; + "oidc_client_secret" = harbor_dex_config.secret; + "oidc_admin_group" = "TODO"; + "oidc_scope" = "openid,profile,email,offline_access,groups"; + }; + # harbor_config_json = pkgs.writeText "config.json" (builtins.toJSON harbor_config); + harbor_config_json = builtins.toJSON harbor_config; in symlinkJoin { name = "in-repo-secrets"; From deaea6954e9e8ff0e63e0c02d44d2da265d2a185 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 26 Apr 2026 09:28:31 -0400 Subject: [PATCH 71/81] Use numbers for folder order. --- .../bootstrap-script/files/manifests/flux_apply_git.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_apply_git.yaml b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_apply_git.yaml index 91e6f46..5869736 100644 --- a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_apply_git.yaml +++ b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_apply_git.yaml @@ -18,11 +18,11 @@ spec: apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: - name: first + name: apply1 namespace: flux-system spec: interval: 1m0s - path: "./k8s/first" + path: "./k8s/1" prune: true sourceRef: kind: GitRepository @@ -35,11 +35,11 @@ spec: apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: - name: deferred + name: apply2 namespace: flux-system spec: interval: 1m0s - path: "./k8s/deferred" + path: "./k8s/2" prune: true sourceRef: kind: GitRepository From a36016e63193a28055855c384956a1e023850c66 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 26 Apr 2026 12:44:24 -0400 Subject: [PATCH 72/81] Allow node-to-node communication. --- nix/kubernetes/roles/firewall/default.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nix/kubernetes/roles/firewall/default.nix b/nix/kubernetes/roles/firewall/default.nix index 984fed0..d69f1c8 100644 --- a/nix/kubernetes/roles/firewall/default.nix +++ b/nix/kubernetes/roles/firewall/default.nix @@ -49,6 +49,13 @@ '' ip6 saddr 2620:11f:7001:7:ffff:eeee::/96 accept '' + # Allow node-to-node communication + # Needed for: + # - metallb port 7946 + '' + iifname "enp*" ip saddr 10.215.1.0/24 ip daddr 10.215.1.0/24 accept + iifname "enp*" ip6 saddr 2620:11f:7001:7:ffff:ffff:0ad7:0100/120 ip6 daddr 2620:11f:7001:7:ffff:ffff:0ad7:0100/120 accept + '' ]; # networking.firewall.extraInputRules = '' From c8c18bde88694e0846b1fa12a57f884aa9dcb1e0 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Tue, 28 Apr 2026 17:45:02 -0400 Subject: [PATCH 73/81] Add secrets for archive-box, webhook-bridge, and tekton. --- nix/kubernetes/README.org | 15 ++-- .../package/k8s-secret-generic/package.nix | 18 ++++- .../mrmanager-repo-secrets/package.nix | 79 ++++++++++++++++++- nix/kubernetes/roles/kubelet/default.nix | 2 +- 4 files changed, 98 insertions(+), 16 deletions(-) diff --git a/nix/kubernetes/README.org b/nix/kubernetes/README.org index 4e1aa19..6cf08ee 100644 --- a/nix/kubernetes/README.org +++ b/nix/kubernetes/README.org @@ -11,13 +11,14 @@ ]; #+end_src * IP Ranges -| | IPv4 | IPv6 | -|------------------------------+-----------------------------+-----------------------------------------| -| Pod | 10.200.0.0/16 | 2620:11f:7001:7:ffff:eeee::/96 | -| Service | 10.197.0.0/16 | fd00:3e42:e349::/112 | -| Node | 10.215.1.0/24 | 2620:11f:7001:7:ffff:ffff:0ad7:0100/120 | -| Load Balancer | 74.80.180.139-74.80.180.142 | 2620:11f:7001:7:ffff:dddd::/96 | -| PowerDNS from inside cluster | 10.215.1.211 | | +| | IPv4 | IPv6 | +|--------------------------------+-----------------------------+-----------------------------------------| +| Pod | 10.200.0.0/16 | 2620:11f:7001:7:ffff:eeee::/96 | +| Service | 10.197.0.0/16 | fd00:3e42:e349::/112 | +| Node | 10.215.1.0/24 | 2620:11f:7001:7:ffff:ffff:0ad7:0100/120 | +| Load Balancer | 74.80.180.139-74.80.180.142 | 2620:11f:7001:7:ffff:dddd::/96 | +| Load Balancer Private (unused) | 10.198.0.0/16 | fd9c:0bd5:22a4::/112 | +| PowerDNS from inside cluster | 10.215.1.211 | | * Healthcheck ** Check cilium status #+begin_src bash diff --git a/nix/kubernetes/keys/package/k8s-secret-generic/package.nix b/nix/kubernetes/keys/package/k8s-secret-generic/package.nix index 6aea5e0..c708d24 100644 --- a/nix/kubernetes/keys/package/k8s-secret-generic/package.nix +++ b/nix/kubernetes/keys/package/k8s-secret-generic/package.nix @@ -16,19 +16,29 @@ secret_name, secret_namespace, secret_values ? { }, + secret_type ? null, + secret_annotations ? null, ... }: let toBase64 = (pkgs.callPackage ../../contrib/base64/package.nix { inherit lib; }).toBase64; + metadata = { + name = "${secret_name}"; + namespace = "${secret_namespace}"; + } + // lib.optionalAttrs (secret_annotations != null) { + "annotations" = secret_annotations; + }; secret_yaml = { apiVersion = "v1"; kind = "Secret"; - metadata = { - name = "${secret_name}"; - namespace = "${secret_namespace}"; - }; + metadata = metadata; data = (builtins.mapAttrs (key: val: (toBase64 val)) secret_values); + } + // lib.optionalAttrs (secret_type != null) { + "type" = secret_type; }; + settingsFormat = pkgs.formats.yaml { }; yaml_body = settingsFormat.generate "${secret_name}.yaml" secret_yaml; yaml_file = pkgs.writeTextFile { diff --git a/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix b/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix index a97fcb5..3663bc4 100644 --- a/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix +++ b/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix @@ -13,13 +13,33 @@ let ( secret_namespace: secrets: (builtins.mapAttrs ( - secret_name: secret_values: + secret_name: original_secret_values: + let + secret_type = original_secret_values."__type" or null; + secret_annotations = original_secret_values."__annotations" or null; + secret_values = removeAttrs original_secret_values [ + "__type" + "__annotations" + ]; + in (callPackage ../../package/k8s-secret-generic/package.nix { - inherit secret_name secret_namespace secret_values; + inherit + secret_name + secret_namespace + secret_values + secret_type + secret_annotations + ; }) ) secrets) ) { + "archive-box" = { + "archive-box-auth" = { + "username" = (builtins.readFile "${./secrets/archive-box/archive-box-auth/username}"); + "password" = (builtins.readFile "${./secrets/archive-box/archive-box-auth/password}"); + }; + }; "cert-manager" = { "rfc2136" = { "TSIG_SECRET" = (builtins.readFile "${./secrets/cert-manager/rfc2136/TSIG_SECRET}"); @@ -37,6 +57,11 @@ let ); }; }; + "flux-system" = { + "webhook-token" = { + "token" = generate_key 64 "flux-system.webhook-token.token"; + }; + }; "gitea" = { "gitea-env" = { "GITEA_ADMIN_USERNAME" = (builtins.readFile "${./secrets/gitea/gitea-env/GITEA_ADMIN_USERNAME}"); @@ -59,6 +84,29 @@ let ); }; }; + "tekton-gateway" = { + "oauth2-env" = oauth2_env { dex_id = "tekton"; }; + }; + "webhook-bridge" = { + "webhook-bridge" = { + "HMAC_TOKEN" = (builtins.readFile "${./secrets/webhook-bridge/webhook-bridge/HMAC_TOKEN}"); + "OAUTH_TOKEN" = (builtins.readFile "${./secrets/webhook-bridge/webhook-bridge/OAUTH_TOKEN}"); + }; + "deployer-key" = { + "__annotations" = { + "tekton.dev/git-0" = "code.fizz.buzz"; + }; + "__type" = "kubernetes.io/ssh-auth"; + "ssh-privatekey" = (builtins.readFile "${./secrets/webhook-bridge/deployer-key/ssh-privatekey}"); + "ssh-publickey" = (builtins.readFile "${./secrets/webhook-bridge/deployer-key/ssh-publickey}"); + }; + "gitea" = { + "token" = (builtins.readFile "${./secrets/webhook-bridge/gitea/token}"); + }; + "harbor-plain" = { + "config.json" = (builtins.readFile "${./secrets/webhook-bridge/harbor-plain/config.json}"); + }; + }; }; encrypted_secrets = ( builtins.mapAttrs ( @@ -101,8 +149,31 @@ let dd if=/dev/urandom | tr --complement --delete '[:alnum:]' | dd bs=${toString len} count=1 of="$out" '' ); - helm_json_escape = json: builtins.toJSON json; - + # helm_json_escape = json: builtins.toJSON json; + helm_json_escape = + json: + builtins.replaceStrings + [ + "=" + "[" + "]" + "," + "." + "\"" + "{" + "}" + ] + [ + "\\=" + "\\[" + "\\]" + "\\," + "\\." + "\\\"" + "\\{" + "\\}" + ] + json; ## dex get_dex_config = client_id: diff --git a/nix/kubernetes/roles/kubelet/default.nix b/nix/kubernetes/roles/kubelet/default.nix index 5f4e78e..a226bc4 100644 --- a/nix/kubernetes/roles/kubelet/default.nix +++ b/nix/kubernetes/roles/kubelet/default.nix @@ -32,7 +32,7 @@ let containerRuntimeEndpoint = "unix:///var/run/containerd/containerd.sock"; enableServer = true; failSwapOn = false; - maxPods = 16; + maxPods = 110; memorySwap = { swapBehavior = "NoSwap"; }; From 058f0cd59b6716f1ef45577eda40d2d741b3c21d Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sat, 2 May 2026 12:50:36 -0400 Subject: [PATCH 74/81] Set up containerd use harbor.fizz.buzz. --- .../keys/package/deploy-script/package.nix | 21 ++++++++++-- nix/kubernetes/roles/containerd/default.nix | 34 +++++++++++++------ 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/nix/kubernetes/keys/package/deploy-script/package.nix b/nix/kubernetes/keys/package/deploy-script/package.nix index f3d84ce..c18242e 100644 --- a/nix/kubernetes/keys/package/deploy-script/package.nix +++ b/nix/kubernetes/keys/package/deploy-script/package.nix @@ -119,8 +119,6 @@ let ${openssh}/bin/ssh mrmanager doas install -d -o 0 -g 0 -m 0755 /vm/${vm_name}/persist/keys ${openssh}/bin/ssh mrmanager doas install -d -o 10016 -g 10016 -m 0755 /vm/${vm_name}/persist/keys/etcd ${openssh}/bin/ssh mrmanager doas install -d -o 10024 -g 10024 -m 0755 /vm/${vm_name}/persist/keys/kube - - '' + (lib.concatMapStringsSep "\n" deploy_file [ { @@ -248,7 +246,8 @@ let ${openssh}/bin/ssh mrmanager doas install -d -o 0 -g 0 -m 0755 /vm/${vm_name}/persist/keys ${openssh}/bin/ssh mrmanager doas install -d -o 10024 -g 10024 -m 0755 /vm/${vm_name}/persist/keys/kube - + ${openssh}/bin/ssh mrmanager doas install -d -o 0 -g 0 -m 0700 /vm/${vm_name}/persist/containerd/certs.d/docker.io + ${openssh}/bin/ssh mrmanager doas install -d -o 0 -g 0 -m 0700 /vm/${vm_name}/persist/containerd/certs.d/harbor.fizz.buzz '' + (lib.concatMapStringsSep "\n" deploy_file [ { @@ -291,6 +290,22 @@ let group = 10024; mode = "0600"; } + { + dest_dir = "/vm/${vm_name}/persist/containerd/certs.d/docker.io"; + file = "${./files/containerd/docker.io/hosts.toml}"; + name = "hosts.toml"; + owner = 0; + group = 0; + mode = "0600"; + } + { + dest_dir = "/vm/${vm_name}/persist/containerd/certs.d/harbor.fizz.buzz"; + file = "${./files/containerd/harbor.fizz.buzz/hosts.toml}"; + name = "hosts.toml"; + owner = 0; + group = 0; + mode = "0600"; + } ]) ) ); diff --git a/nix/kubernetes/roles/containerd/default.nix b/nix/kubernetes/roles/containerd/default.nix index f7b48ec..a45a914 100644 --- a/nix/kubernetes/roles/containerd/default.nix +++ b/nix/kubernetes/roles/containerd/default.nix @@ -1,3 +1,4 @@ +# TODO: Set up a proxy to harbor for OCI compliance: https://github.com/moby/moby/pull/34319#issuecomment-720606627 { config, lib, @@ -29,30 +30,43 @@ in config = lib.mkIf config.me.containerd.enable { virtualisation.containerd.enable = true; - virtualisation.containerd.settings = { + virtualisation.containerd.settings = lib.mkForce { "plugins" = { - "io.containerd.grpc.v1.cri" = { + "io.containerd.cri.v1.images" = { + "registry" = { + "config_path" = "/.persist/containerd/certs.d"; + }; + "snapshotter" = "overlayfs"; + }; + "io.containerd.cri.v1.runtime" = { "cni" = { - "bin_dir" = "/opt/cni/bin"; + "bin_dirs" = [ + "/opt/cni/bin" + ]; "conf_dir" = "/etc/cni/net.d"; - # "bin_dir" = "${my-cni-plugins}/bin"; - # "conf_dir" = "${my-cni-configs}"; }; "containerd" = { "default_runtime_name" = "runc"; "runtimes" = { "runc" = { - "options" = { - "SystemdCgroup" = true; - }; "runtime_type" = "io.containerd.runc.v2"; }; }; - "snapshotter" = "overlayfs"; + }; + }; + "io.containerd.cri.v1.services" = { + "containerd" = { + "runtimes" = { + "runc" = { + "options" = { + "SystemdCgroup" = true; + }; + }; + }; }; }; }; - "version" = 2; + "version" = 3; }; systemd.services.containerd.preStart = '' From a00115fb4e17f486a5a044a2faad30258fbffc55 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sat, 2 May 2026 15:53:35 -0400 Subject: [PATCH 75/81] Delete images after 24 hours of being unused. --- nix/kubernetes/roles/kubelet/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/kubernetes/roles/kubelet/default.nix b/nix/kubernetes/roles/kubelet/default.nix index a226bc4..1227f3d 100644 --- a/nix/kubernetes/roles/kubelet/default.nix +++ b/nix/kubernetes/roles/kubelet/default.nix @@ -47,6 +47,7 @@ let "10.197.0.10" "fd00:3e42:e349::10" ]; + imageMaximumGCAge = "24h"; # Delete unused images after 1 day. }; kubelet_config_file = (to_yaml_file "kubelet-config.yaml" kubelet_config); in From b0f025d340830a255399ae9e45eb53ce8c1f8eaa Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sat, 2 May 2026 20:35:13 -0400 Subject: [PATCH 76/81] Add IP addresses to worker certs for the metrics server. --- nix/kubernetes/keys/package/k8s-ca/files/client-ca.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nix/kubernetes/keys/package/k8s-ca/files/client-ca.conf b/nix/kubernetes/keys/package/k8s-ca/files/client-ca.conf index c752da0..ac8d404 100644 --- a/nix/kubernetes/keys/package/k8s-ca/files/client-ca.conf +++ b/nix/kubernetes/keys/package/k8s-ca/files/client-ca.conf @@ -120,7 +120,7 @@ extendedKeyUsage = clientAuth, serverAuth keyUsage = critical, digitalSignature, keyEncipherment nsCertType = client nsComment = "worker0 Certificate" -subjectAltName = DNS:worker0, IP:127.0.0.1 +subjectAltName = DNS:worker0, IP:127.0.0.1, IP:10.215.1.224, IP:2620:11f:7001:7:ffff:ffff:ad7:1e0 subjectKeyIdentifier = hash [worker0_distinguished_name] @@ -141,7 +141,7 @@ extendedKeyUsage = clientAuth, serverAuth keyUsage = critical, digitalSignature, keyEncipherment nsCertType = client nsComment = "worker1 Certificate" -subjectAltName = DNS:worker1, IP:127.0.0.1 +subjectAltName = DNS:worker1, IP:127.0.0.1, IP:10.215.1.225, IP:2620:11f:7001:7:ffff:ffff:ad7:1e1 subjectKeyIdentifier = hash [worker1_distinguished_name] @@ -162,7 +162,7 @@ extendedKeyUsage = clientAuth, serverAuth keyUsage = critical, digitalSignature, keyEncipherment nsCertType = client nsComment = "worker2 Certificate" -subjectAltName = DNS:worker2, IP:127.0.0.1 +subjectAltName = DNS:worker2, IP:127.0.0.1, IP:10.215.1.226, IP:2620:11f:7001:7:ffff:ffff:ad7:1e2 subjectKeyIdentifier = hash [worker2_distinguished_name] From 65c2021473a93abae0fd1fcd09281a6a8e8ad546 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 3 May 2026 14:52:53 -0400 Subject: [PATCH 77/81] Update flux and install the image automation controller. --- .../files/manifests/flux.yaml | 188 ++++++++++++++---- .../files/manifests/flux_instance.yaml | 10 +- .../keys/package/bootstrap-script/package.nix | 4 + .../mrmanager-repo-secrets/package.nix | 27 +++ 4 files changed, 193 insertions(+), 36 deletions(-) diff --git a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux.yaml b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux.yaml index d521a2f..e724deb 100644 --- a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux.yaml +++ b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux.yaml @@ -6,10 +6,10 @@ metadata: name: flux-operator-web namespace: flux-system labels: - helm.sh/chart: flux-operator-0.37.1 + helm.sh/chart: flux-operator-0.48.0 app.kubernetes.io/name: flux-operator app.kubernetes.io/instance: flux-operator - app.kubernetes.io/version: "v0.37.1" + app.kubernetes.io/version: "v0.48.0" app.kubernetes.io/managed-by: Helm spec: policyTypes: @@ -32,10 +32,10 @@ metadata: name: flux-operator namespace: flux-system labels: - helm.sh/chart: flux-operator-0.37.1 + helm.sh/chart: flux-operator-0.48.0 app.kubernetes.io/name: flux-operator app.kubernetes.io/instance: flux-operator - app.kubernetes.io/version: "v0.37.1" + app.kubernetes.io/version: "v0.48.0" app.kubernetes.io/managed-by: Helm automountServiceAccountToken: true --- @@ -44,14 +44,14 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.19.0 + controller-gen.kubebuilder.io/version: v0.20.0 helm.sh/resource-policy: keep labels: app.kubernetes.io/instance: 'flux-operator' app.kubernetes.io/managed-by: 'Helm' app.kubernetes.io/name: 'flux-operator' - app.kubernetes.io/version: 'v0.37.1' - helm.sh/chart: 'flux-operator-0.37.1' + app.kubernetes.io/version: 'v0.48.0' + helm.sh/chart: 'flux-operator-0.48.0' name: fluxinstances.fluxcd.controlplane.io spec: group: fluxcd.controlplane.io @@ -205,7 +205,11 @@ spec: components: description: |- Components is the list of controllers to install. - Defaults to a commonly used subset. + Defaults to the core Flux controllers: + - source-controller + - kustomize-controller + - helm-controller + - notification-controller items: description: Component is the name of a controller to install. enum: @@ -661,14 +665,14 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.19.0 + controller-gen.kubebuilder.io/version: v0.20.0 helm.sh/resource-policy: keep labels: app.kubernetes.io/instance: 'flux-operator' app.kubernetes.io/managed-by: 'Helm' app.kubernetes.io/name: 'flux-operator' - app.kubernetes.io/version: 'v0.37.1' - helm.sh/chart: 'flux-operator-0.37.1' + app.kubernetes.io/version: 'v0.48.0' + helm.sh/chart: 'flux-operator-0.48.0' name: fluxreports.fluxcd.controlplane.io spec: group: fluxcd.controlplane.io @@ -828,7 +832,7 @@ spec: failing: description: |- Failing is the number of reconciled - resources in the Failing state. + resources in the Failing state and not Suspended. type: integer running: description: |- @@ -965,14 +969,14 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.19.0 + controller-gen.kubebuilder.io/version: v0.20.0 helm.sh/resource-policy: keep labels: app.kubernetes.io/instance: 'flux-operator' app.kubernetes.io/managed-by: 'Helm' app.kubernetes.io/name: 'flux-operator' - app.kubernetes.io/version: 'v0.37.1' - helm.sh/chart: 'flux-operator-0.37.1' + app.kubernetes.io/version: 'v0.48.0' + helm.sh/chart: 'flux-operator-0.48.0' name: resourcesetinputproviders.fluxcd.controlplane.io spec: group: fluxcd.controlplane.io @@ -1029,9 +1033,9 @@ spec: - a PEM-encoded CA certificate (`ca.crt`) - a PEM-encoded client certificate (`tls.crt`) and private key (`tls.key`) - When connecting to a Git or OCI provider that uses self-signed certificates, the CA certificate - must be set in the Secret under the 'ca.crt' key to establish the trust relationship. - When connecting to an OCI provider that supports client certificates (mTLS), the client certificate + When connecting to a Git, OCI, or ExternalService provider that uses self-signed certificates, + the CA certificate must be set in the Secret under the 'ca.crt' key to establish the trust relationship. + When connecting to a provider that supports client certificates (mTLS), the client certificate and private key must be set in the Secret under the 'tls.crt' and 'tls.key' keys, respectively. properties: name: @@ -1102,6 +1106,11 @@ spec: Supported only for tags at the moment. type: string type: object + insecure: + description: |- + Insecure allows connecting to an ExternalService or OCIArtifactTag provider + over plain HTTP without TLS. When not set, the URL must use HTTPS. + type: boolean schedule: description: Schedule defines the schedules for the input provider to run. @@ -1129,13 +1138,16 @@ spec: type: array secretRef: description: |- - SecretRef specifies the Kubernetes Secret containing the basic-auth credentials + SecretRef specifies the Kubernetes Secret containing the credentials to access the input provider. When connecting to a Git provider, the secret must contain the keys 'username' and 'password', and the password should be a personal access token that grants read-only access to the repository. When connecting to an OCI provider, the secret must contain a Kubernetes Image Pull Secret, as if created by `kubectl create secret docker-registry`. + When connecting to an ExternalService provider, the secret must contain either + a 'token' key for bearer token authentication, or 'username' and 'password' + keys for basic authentication. properties: name: description: Name of the referent. @@ -1177,10 +1189,14 @@ spec: - AzureDevOpsBranch - AzureDevOpsTag - AzureDevOpsPullRequest + - GiteaBranch + - GiteaTag + - GiteaPullRequest - OCIArtifactTag - ACRArtifactTag - ECRArtifactTag - GARArtifactTag + - ExternalService type: string url: description: |- @@ -1206,6 +1222,16 @@ spec: - message: spec.url must start with 'oci://' when spec.type is an OCI provider rule: '!self.type.endsWith(''ArtifactTag'') || self.url.startsWith(''oci'')' + - message: spec.url must start with 'http://' or 'https://' when spec.type + is 'ExternalService' + rule: self.type != 'ExternalService' || self.url.startsWith('http') + - message: spec.insecure can only be set when spec.type is 'ExternalService' + or 'OCIArtifactTag' + rule: '!has(self.insecure) || !self.insecure || self.type == ''ExternalService'' + || self.type == ''OCIArtifactTag''' + - message: spec.url must use 'https://' unless spec.insecure is true + rule: self.type != 'ExternalService' || !self.url.startsWith('http://') + || (has(self.insecure) && self.insecure) - message: cannot specify spec.serviceAccountName when spec.type is not one of AzureDevOps* or *ArtifactTag rule: '!has(self.serviceAccountName) || self.type.startsWith(''AzureDevOps'') @@ -1345,14 +1371,14 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.19.0 + controller-gen.kubebuilder.io/version: v0.20.0 helm.sh/resource-policy: keep labels: app.kubernetes.io/instance: 'flux-operator' app.kubernetes.io/managed-by: 'Helm' app.kubernetes.io/name: 'flux-operator' - app.kubernetes.io/version: 'v0.37.1' - helm.sh/chart: 'flux-operator-0.37.1' + app.kubernetes.io/version: 'v0.48.0' + helm.sh/chart: 'flux-operator-0.48.0' name: resourcesets.fluxcd.controlplane.io spec: group: fluxcd.controlplane.io @@ -1459,6 +1485,15 @@ spec: input provider objects are used. Defaults to flattening all inputs from all providers into a single list of input sets. properties: + includeEmptyProviders: + description: |- + IncludeEmptyProviders controls how input providers that export no + inputs are treated. Only applies when Name is Permute. When true, if + any provider has zero inputs the resulting permutation set is empty + (mathematically correct Cartesian product behavior). When false or + unset (default), providers with zero inputs are silently skipped and + the remaining providers still permute among themselves. + type: boolean name: description: |- Name defines how the inputs are combined when multiple @@ -1481,6 +1516,9 @@ spec: required: - name type: object + x-kubernetes-validations: + - message: includeEmptyProviders only applies when name is Permute + rule: '!has(self.includeEmptyProviders) || self.name == ''Permute''' inputs: description: Inputs contains the list of ResourceSet inputs. items: @@ -1659,6 +1697,16 @@ spec: - type type: object type: array + externalChecksumRefs: + description: |- + ExternalChecksumRefs lists the ConfigMap and Secret references + discovered in checksumFrom annotations on the last reconciliation + that point to objects not rendered by this ResourceSet. Each entry + has the form "Kind/namespace/name". It is used to trigger a + reconciliation when one of the referenced objects changes. + items: + type: string + type: array history: description: |- History contains the reconciliation history of the ResourceSet @@ -1764,10 +1812,10 @@ metadata: labels: rbac.authorization.k8s.io/aggregate-to-edit: "true" rbac.authorization.k8s.io/aggregate-to-admin: "true" - helm.sh/chart: flux-operator-0.37.1 + helm.sh/chart: flux-operator-0.48.0 app.kubernetes.io/name: flux-operator app.kubernetes.io/instance: flux-operator - app.kubernetes.io/version: "v0.37.1" + app.kubernetes.io/version: "v0.48.0" app.kubernetes.io/managed-by: Helm rules: - apiGroups: @@ -1791,10 +1839,10 @@ metadata: rbac.authorization.k8s.io/aggregate-to-admin: "true" rbac.authorization.k8s.io/aggregate-to-edit: "true" rbac.authorization.k8s.io/aggregate-to-view: "true" - helm.sh/chart: flux-operator-0.37.1 + helm.sh/chart: flux-operator-0.48.0 app.kubernetes.io/name: flux-operator app.kubernetes.io/instance: flux-operator - app.kubernetes.io/version: "v0.37.1" + app.kubernetes.io/version: "v0.48.0" app.kubernetes.io/managed-by: Helm rules: - apiGroups: @@ -1807,16 +1855,86 @@ rules: - list - watch --- +# Source: flux-operator/templates/web-standard-roles.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: flux-web-user + labels: + helm.sh/chart: flux-operator-0.48.0 + app.kubernetes.io/name: flux-operator + app.kubernetes.io/instance: flux-operator + app.kubernetes.io/version: "v0.48.0" + app.kubernetes.io/managed-by: Helm +rules: + - apiGroups: ["*"] + resources: ["*"] + verbs: ["get", "list", "watch"] +--- +# Source: flux-operator/templates/web-standard-roles.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: flux-web-admin + labels: + helm.sh/chart: flux-operator-0.48.0 + app.kubernetes.io/name: flux-operator + app.kubernetes.io/instance: flux-operator + app.kubernetes.io/version: "v0.48.0" + app.kubernetes.io/managed-by: Helm +rules: + - apiGroups: ["*"] + resources: ["*"] + verbs: ["get", "list", "watch"] + - apiGroups: + - fluxcd.controlplane.io + - source.toolkit.fluxcd.io + - source.extensions.fluxcd.io + - kustomize.toolkit.fluxcd.io + - helm.toolkit.fluxcd.io + - image.toolkit.fluxcd.io + - notification.toolkit.fluxcd.io + resources: ["*"] + verbs: + - patch + - reconcile + - suspend + - resume + - download + - apiGroups: + - apps + resources: + - deployments + - statefulsets + - daemonsets + verbs: + - patch + - restart + - apiGroups: + - batch + resources: + - cronjobs + - jobs + verbs: + - create + - restart + - apiGroups: + - "" + resources: + - pods + verbs: + - delete +--- # Source: flux-operator/templates/admin-clusterrole.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: flux-operator labels: - helm.sh/chart: flux-operator-0.37.1 + helm.sh/chart: flux-operator-0.48.0 app.kubernetes.io/name: flux-operator app.kubernetes.io/instance: flux-operator - app.kubernetes.io/version: "v0.37.1" + app.kubernetes.io/version: "v0.48.0" app.kubernetes.io/managed-by: Helm roleRef: apiGroup: rbac.authorization.k8s.io @@ -1834,10 +1952,10 @@ metadata: name: flux-operator namespace: flux-system labels: - helm.sh/chart: flux-operator-0.37.1 + helm.sh/chart: flux-operator-0.48.0 app.kubernetes.io/name: flux-operator app.kubernetes.io/instance: flux-operator - app.kubernetes.io/version: "v0.37.1" + app.kubernetes.io/version: "v0.48.0" app.kubernetes.io/managed-by: Helm spec: ports: @@ -1860,10 +1978,10 @@ metadata: name: flux-operator namespace: flux-system labels: - helm.sh/chart: flux-operator-0.37.1 + helm.sh/chart: flux-operator-0.48.0 app.kubernetes.io/name: flux-operator app.kubernetes.io/instance: flux-operator - app.kubernetes.io/version: "v0.37.1" + app.kubernetes.io/version: "v0.48.0" app.kubernetes.io/managed-by: Helm spec: selector: @@ -1877,10 +1995,10 @@ spec: prometheus.io/port: "8080" prometheus.io/path: "/metrics" labels: - helm.sh/chart: flux-operator-0.37.1 + helm.sh/chart: flux-operator-0.48.0 app.kubernetes.io/name: flux-operator app.kubernetes.io/instance: flux-operator - app.kubernetes.io/version: "v0.37.1" + app.kubernetes.io/version: "v0.48.0" app.kubernetes.io/managed-by: Helm spec: serviceAccountName: flux-operator @@ -1906,7 +2024,7 @@ spec: runAsNonRoot: true seccompProfile: type: RuntimeDefault - image: "ghcr.io/controlplaneio-fluxcd/flux-operator:v0.37.1" + image: "ghcr.io/controlplaneio-fluxcd/flux-operator:v0.48.0" imagePullPolicy: "IfNotPresent" ports: - name: http-metrics diff --git a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_instance.yaml b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_instance.yaml index e0c756a..211fd7c 100644 --- a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_instance.yaml +++ b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_instance.yaml @@ -5,5 +5,13 @@ metadata: namespace: flux-system spec: distribution: - version: "2.7.x" + version: "2.8.x" registry: "ghcr.io/fluxcd" + components: + - source-controller + - kustomize-controller + - helm-controller + - notification-controller + - image-automation-controller + - image-reflector-controller + # - source-watcher diff --git a/nix/kubernetes/keys/package/bootstrap-script/package.nix b/nix/kubernetes/keys/package/bootstrap-script/package.nix index 28647b7..c8d4cd9 100644 --- a/nix/kubernetes/keys/package/bootstrap-script/package.nix +++ b/nix/kubernetes/keys/package/bootstrap-script/package.nix @@ -35,6 +35,10 @@ let "${k8s.cilium-manifest}/cilium.yaml" "${k8s.coredns-manifest}/coredns.yaml" ./files/manifests/flux_namespace.yaml + + # + # Generate with: helm template --dry-run=server flux-operator oci://ghcr.io/controlplaneio-fluxcd/charts/flux-operator --namespace flux-system --create-namespace + # ./files/manifests/flux.yaml ./files/manifests/flux_instance.yaml ] diff --git a/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix b/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix index 3663bc4..d0743d8 100644 --- a/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix +++ b/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix @@ -58,6 +58,17 @@ let }; }; "flux-system" = { + "registry-credentials" = + (generate_docker_secret { + username = builtins.readFile "${./secrets/flux-system/registry-credentials/username}"; + password = builtins.readFile "${./secrets/flux-system/registry-credentials/password}"; + email = builtins.readFile "${./secrets/flux-system/registry-credentials/email}"; + }) + // { + # "__annotations" = { + # "tekton.dev/docker-0" = "https://harbor.fizz.buzz"; + # }; + }; "webhook-token" = { "token" = generate_key 64 "flux-system.webhook-token.token"; }; @@ -140,6 +151,7 @@ let ## Utilities inherit ((import ../../../functions/to_yaml.nix) { inherit pkgs; }) to_yaml; + inherit (pkgs.callPackage ../../contrib/base64/package.nix { inherit lib; }) toBase64; generate_key = len: name: builtins.readFile ( @@ -174,6 +186,21 @@ let "\\}" ] json; + generate_docker_secret = + { + username, + password, + email, + }: + let + in + { + "__type" = "kubernetes.io/dockerconfigjson"; + ".dockerconfigjson" = builtins.toJSON { + inherit username password email; + "auth" = toBase64 "${username}:${password}"; + }; + }; ## dex get_dex_config = client_id: From 889de9dbf64c538b3eacb0e899d0c311638ca122 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 3 May 2026 16:25:01 -0400 Subject: [PATCH 78/81] Add a secret for the harbor webhooks to flux. --- .../keys/package/mrmanager-repo-secrets/package.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix b/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix index d0743d8..70eba2c 100644 --- a/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix +++ b/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix @@ -70,8 +70,13 @@ let # }; }; "webhook-token" = { + # This token is used for gitea webhooks "token" = generate_key 64 "flux-system.webhook-token.token"; }; + "harbor-webhook-token" = { + # This token is used for harbor webhooks + "token" = generate_key 64 "flux-system.harbor-webhook-token.token"; + }; }; "gitea" = { "gitea-env" = { From 345ecfc7cfb120e0897d6c03db810c8e36f8511c Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 3 May 2026 17:03:04 -0400 Subject: [PATCH 79/81] Add secret for homepage-staging. --- nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix b/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix index 70eba2c..60ccdec 100644 --- a/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix +++ b/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix @@ -100,6 +100,9 @@ let ); }; }; + "homepage-staging" = { + "oauth2-env" = oauth2_env { dex_id = "homepage-staging"; }; + }; "tekton-gateway" = { "oauth2-env" = oauth2_env { dex_id = "tekton"; }; }; From 297577471ec3ce319a8b637567618f31114b50e2 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sat, 30 May 2026 19:35:06 -0400 Subject: [PATCH 80/81] Update kubernetes packages and update to gateway API 1.5.1. --- nix/kubernetes/flake.lock | 12 +++++------ .../keys/package/bootstrap-script/package.nix | 21 ++++++++++--------- nix/kubernetes/roles/kubernetes/default.nix | 2 +- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/nix/kubernetes/flake.lock b/nix/kubernetes/flake.lock index 28ca666..2824a29 100644 --- a/nix/kubernetes/flake.lock +++ b/nix/kubernetes/flake.lock @@ -22,11 +22,11 @@ ] }, "locked": { - "lastModified": 1769524058, - "narHash": "sha256-zygdD6X1PcVNR2PsyK4ptzrVEiAdbMqLos7utrMDEWE=", + "lastModified": 1780290312, + "narHash": "sha256-eTAlX0CwgB84Ts3GaBd944A3DRXVMzgA0EqroZBISUo=", "owner": "nix-community", "repo": "disko", - "rev": "71a3fc97d80881e91710fe721f1158d3b96ae14d", + "rev": "115e5211780054d8a890b41f0b7734cafad54dfe", "type": "github" }, "original": { @@ -164,11 +164,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1770197578, - "narHash": "sha256-AYqlWrX09+HvGs8zM6ebZ1pwUqjkfpnv8mewYwAo+iM=", + "lastModified": 1780243769, + "narHash": "sha256-x5UQuRsH3MqI0U9afaXSNqzTPSeZlRLvFAav2Ux1pNw=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "00c21e4c93d963c50d4c0c89bfa84ed6e0694df2", + "rev": "331800de5053fcebacf6813adb5db9c9dca22a0c", "type": "github" }, "original": { diff --git a/nix/kubernetes/keys/package/bootstrap-script/package.nix b/nix/kubernetes/keys/package/bootstrap-script/package.nix index c8d4cd9..2e050b9 100644 --- a/nix/kubernetes/keys/package/bootstrap-script/package.nix +++ b/nix/kubernetes/keys/package/bootstrap-script/package.nix @@ -54,22 +54,23 @@ let gateway_crds_repo = fetchFromGitHub { owner = "kubernetes-sigs"; repo = "gateway-api"; - rev = "v1.4.1"; - sha256 = "sha256-/GHyikcC2QGDN0ndpY6/xvSEEnpSsLrNU+lFElCKBs8="; + rev = "v1.5.1"; + sha256 = "sha256-mWMvJG6esOqDBSbhExvt7L3ZTiQUOfeRBohew/m67A0="; }; gateway_crds = [ "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_backendtlspolicies.yaml" - "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_referencegrants.yaml" - "${gateway_crds_repo}/config/crd/experimental/gateway.networking.x-k8s.io_xmeshes.yaml" "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_gatewayclasses.yaml" - "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_udproutes.yaml" - "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_tlsroutes.yaml" - "${gateway_crds_repo}/config/crd/experimental/gateway.networking.x-k8s.io_xbackendtrafficpolicies.yaml" "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_gateways.yaml" - "${gateway_crds_repo}/config/crd/experimental/gateway.networking.x-k8s.io_xlistenersets.yaml" - "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_tcproutes.yaml" - "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_httproutes.yaml" "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_grpcroutes.yaml" + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_httproutes.yaml" + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_listenersets.yaml" + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_referencegrants.yaml" + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_tcproutes.yaml" + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_tlsroutes.yaml" + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_udproutes.yaml" + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_vap_safeupgrades.yaml" + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.x-k8s.io_xbackendtrafficpolicies.yaml" + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.x-k8s.io_xmeshes.yaml" ]; in stdenv.mkDerivation (finalAttrs: { diff --git a/nix/kubernetes/roles/kubernetes/default.nix b/nix/kubernetes/roles/kubernetes/default.nix index f4d70f1..0f268d8 100644 --- a/nix/kubernetes/roles/kubernetes/default.nix +++ b/nix/kubernetes/roles/kubernetes/default.nix @@ -21,7 +21,7 @@ assertions = [ { # Kubernetes should only upgrade 1 minor version at a time, so this assert is here to prevent unwittingly jumping versions. - assertion = lib.hasPrefix "1.35." pkgs.kubernetes.version; + assertion = lib.hasPrefix "1.36." pkgs.kubernetes.version; message = "Unexpected Kubernetes package version: ${pkgs.kubernetes.version}"; } ]; From 7c8d278923d3cd2f4f228136c5eabb00bd83d3df Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sat, 30 May 2026 21:40:12 -0400 Subject: [PATCH 81/81] Configure kernel preemption. --- nix/kubernetes/configuration.nix | 1 + nix/kubernetes/flake.lock | 6 +- nix/kubernetes/roles/kernel/default.nix | 192 ++++++++++++++++++ nix/kubernetes/roles/minimal_base/default.nix | 1 + .../roles/optimized_build/default.nix | 66 ++---- 5 files changed, 212 insertions(+), 54 deletions(-) create mode 100644 nix/kubernetes/roles/kernel/default.nix diff --git a/nix/kubernetes/configuration.nix b/nix/kubernetes/configuration.nix index e6d5ef5..ef64f2f 100644 --- a/nix/kubernetes/configuration.nix +++ b/nix/kubernetes/configuration.nix @@ -17,6 +17,7 @@ ./roles/firewall ./roles/image_based_appliance ./roles/iso + ./roles/kernel ./roles/kube_apiserver ./roles/kube_controller_manager ./roles/kube_proxy diff --git a/nix/kubernetes/flake.lock b/nix/kubernetes/flake.lock index 2824a29..8f09179 100644 --- a/nix/kubernetes/flake.lock +++ b/nix/kubernetes/flake.lock @@ -164,11 +164,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1780243769, - "narHash": "sha256-x5UQuRsH3MqI0U9afaXSNqzTPSeZlRLvFAav2Ux1pNw=", + "lastModified": 1780749050, + "narHash": "sha256-3av0pIjlOWQ6rDbNOmpUSvbNnJkGORQKKjb4LtCZsIY=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "331800de5053fcebacf6813adb5db9c9dca22a0c", + "rev": "a799d3e3886da994fa307f817a6bc705ae538eeb", "type": "github" }, "original": { diff --git a/nix/kubernetes/roles/kernel/default.nix b/nix/kubernetes/roles/kernel/default.nix new file mode 100644 index 0000000..7979a68 --- /dev/null +++ b/nix/kubernetes/roles/kernel/default.nix @@ -0,0 +1,192 @@ +# Check current config: +# nix build '/persist/machine_setup/nix/configuration#nixosConfigurations.hydra.pkgs.linux_me.configfile' +# cat $(nix eval --raw '/persist/machine_setup/nix/configuration#nixosConfigurations.hydra.pkgs.linux_me.configfile') | less + +{ + config, + lib, + pkgs, + ... +}: + +let + preemption_type = with lib.kernel; { + full = { + PREEMPT_DYNAMIC = yes; + PREEMPT = yes; + PREEMPT_VOLUNTARY = lib.mkForce no; + PREEMPT_LAZY = lib.mkForce no; + PREEMPT_NONE = no; + }; + lazy = { + PREEMPT_DYNAMIC = yes; + PREEMPT = no; + PREEMPT_VOLUNTARY = lib.mkForce no; + PREEMPT_LAZY = yes; + PREEMPT_NONE = no; + }; + voluntary = { + PREEMPT_DYNAMIC = no; + PREEMPT = no; + PREEMPT_VOLUNTARY = yes; + PREEMPT_LAZY = lib.mkForce no; + PREEMPT_NONE = no; + }; + none = { + PREEMPT_DYNAMIC = no; + PREEMPT = no; + PREEMPT_VOLUNTARY = lib.mkForce no; + PREEMPT_LAZY = lib.mkForce no; + PREEMPT_NONE = yes; + }; + }; + tick_hz = + with lib.kernel; + { + "1000" = { + HZ_1000 = yes; + HZ = freeform "1000"; + }; + } + // lib.genAttrs [ "100" "250" "300" "500" "600" "750" ] (hz: { + HZ_1000 = no; + "HZ_${hz}" = yes; + HZ = freeform hz; + }); + performance_governor = with lib.kernel; { + default = { + CPU_FREQ_DEFAULT_GOV_SCHEDUTIL = yes; + }; + performance = { + CPU_FREQ_DEFAULT_GOV_SCHEDUTIL = no; + CPU_FREQ_DEFAULT_GOV_PERFORMANCE = yes; + }; + }; + tick_rate = with lib.kernel; { + # Always tick at the hz frequency. + periodic = { + NO_HZ_IDLE = no; + NO_HZ_FULL = no; + NO_HZ = no; + NO_HZ_COMMON = no; + HZ_PERIODIC = yes; + }; + # Idle - Do not disturb the CPU when idle. This can save power but increase latency. + idle = { + HZ_PERIODIC = no; + NO_HZ_FULL = no; + NO_HZ_IDLE = yes; + NO_HZ = yes; + NO_HZ_COMMON = yes; + }; + # Full dyntick system (tickless) - The kernel tries to shut down the tick whenever possible. + tickless = { + HZ_PERIODIC = no; + NO_HZ_IDLE = no; + NO_HZ_FULL = yes; + NO_HZ = yes; + NO_HZ_COMMON = yes; + CONTEXT_TRACKING = yes; + }; + }; + huge_page = with lib.kernel; { + always = { + TRANSPARENT_HUGEPAGE_MADVISE = no; + TRANSPARENT_HUGEPAGE_ALWAYS = yes; + }; + madvise = { + TRANSPARENT_HUGEPAGE_ALWAYS = no; + TRANSPARENT_HUGEPAGE_MADVISE = yes; + }; + }; + common_config = with lib.kernel; { + # Google's BBRv3 TCP congestion Control + TCP_CONG_BBR = yes; + DEFAULT_BBR = yes; + }; + flavors = { + server = lib.mkMerge [ + preemption_type.none + tick_hz."300" + performance_governor.default + tick_rate.tickless + huge_page.madvise + ]; + interactive = + with lib.kernel; + lib.mkMerge [ + { + # Enable RCU Lazy - Reduces power consumption when idle or lightly loaded. Useful for battery-powered devices like laptops. + RCU_LAZY = yes; + } + preemption_type.lazy + tick_hz."300" + performance_governor.default + tick_rate.tickless + huge_page.madvise + ]; + }; +in +{ + imports = [ ]; + + options.me = { + kernel.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install kernel."; + }; + + kernel.version = lib.mkOption { + type = lib.types.str; + default = "linux"; # LTS + example = "linux_6_18"; + description = "What version of the kernl should we use."; + }; + + kernel.flavor = lib.mkOption { + type = lib.types.str; + default = "server"; + example = "interactive"; + description = "What type of kernel should be built."; + }; + }; + + config = lib.mkIf config.me.kernel.enable ( + lib.mkMerge [ + { + boot.kernelPackages = pkgs.linuxPackagesFor pkgs.linux_me; + } + (lib.mkIf (!config.me.optimizations.enable) { + nixpkgs.overlays = [ + (final: prev: { + linux_me = final."${config.me.kernel.version}"; + }) + ]; + }) + (lib.mkIf (config.me.optimizations.enable) { + nixpkgs.overlays = [ + ( + final: prev: + let + addConfig = + additionalConfig: pkg: + pkg.override (oldconfig: { + structuredExtraConfig = lib.mkMerge ([ pkg.structuredExtraConfig ] ++ additionalConfig); + # stdenv = pkgs.llvmPackages_latest.stdenv; + # stdenv = pkgs.clangStdenv; + }); + in + { + linux_me = addConfig ([ + common_config + flavors."${config.me.kernel.flavor}" + ]) final."${config.me.kernel.version}"; + } + ) + ]; + }) + ] + ); +} diff --git a/nix/kubernetes/roles/minimal_base/default.nix b/nix/kubernetes/roles/minimal_base/default.nix index e57f1f2..54261f2 100644 --- a/nix/kubernetes/roles/minimal_base/default.nix +++ b/nix/kubernetes/roles/minimal_base/default.nix @@ -19,6 +19,7 @@ config = lib.mkIf config.me.minimal_base.enable { me.doas.enable = true; + me.kernel.enable = true; me.network.enable = true; me.nvme.enable = true; me.ssh.enable = true; diff --git a/nix/kubernetes/roles/optimized_build/default.nix b/nix/kubernetes/roles/optimized_build/default.nix index d569df4..65fd074 100644 --- a/nix/kubernetes/roles/optimized_build/default.nix +++ b/nix/kubernetes/roles/optimized_build/default.nix @@ -49,65 +49,29 @@ }; config = lib.mkMerge [ - (lib.mkIf (!config.me.optimizations.enable) ( - lib.mkMerge [ - { - # boot.kernelPackages = pkgs.linuxPackagesFor pkgs.linux_6_17; - boot.kernelPackages = pkgs.linuxPackagesFor pkgs.linux; - } - ] - )) (lib.mkIf config.me.optimizations.enable ( lib.mkMerge [ { - boot.kernelPackages = pkgs.linuxPackagesFor pkgs.linux_me; - nixpkgs.hostPlatform = { gcc.arch = config.me.optimizations.arch; gcc.tune = config.me.optimizations.arch; }; - nixpkgs.overlays = [ - ( - final: prev: - let - addConfig = - additionalConfig: pkg: - pkg.override (oldconfig: { - structuredExtraConfig = pkg.structuredExtraConfig // additionalConfig; - }); - in - { - linux_me = addConfig { - # Full preemption - PREEMPT = lib.mkOverride 60 lib.kernel.yes; - PREEMPT_VOLUNTARY = lib.mkOverride 60 lib.kernel.no; - - # Google's BBRv3 TCP congestion Control - TCP_CONG_BBR = lib.kernel.yes; - DEFAULT_BBR = lib.kernel.yes; - - # Preemptive Full Tickless Kernel at 300Hz - HZ = lib.kernel.freeform "300"; - HZ_300 = lib.kernel.yes; - HZ_1000 = lib.kernel.no; - } prev.linux; # or prev.linux_6_17 - } - ) - (final: prev: { - inherit (final.unoptimized) - assimp - binaryen - gsl - rapidjson - ffmpeg-headless - ffmpeg - pipewire - chromaprint - gtkmm - ; - }) - ]; + # nixpkgs.overlays = [ + # (final: prev: { + # inherit (final.unoptimized) + # assimp + # binaryen + # gsl + # rapidjson + # ffmpeg-headless + # ffmpeg + # pipewire + # chromaprint + # gtkmm + # ; + # }) + # ]; } ] ))