diff --git a/nix/configuration/README.org b/nix/configuration/README.org new file mode 100644 index 0000000..ba88116 --- /dev/null +++ b/nix/configuration/README.org @@ -0,0 +1,12 @@ +* To-do +** Perhaps use overlay for /etc for speedup +#+begin_src nix + system.etc.overlay.enable = true; +#+end_src +** read https://nixos.org/manual/nixos/stable/ +** Performance for mini pc +#+begin_src nix + security.pam.loginLimits = [ + { domain = "@users"; item = "rtprio"; type = "-"; value = 1; } + ]; +#+end_src diff --git a/nix/kubernetes/.gitignore b/nix/kubernetes/.gitignore new file mode 100644 index 0000000..b2be92b --- /dev/null +++ b/nix/kubernetes/.gitignore @@ -0,0 +1 @@ +result diff --git a/nix/kubernetes/README.org b/nix/kubernetes/README.org new file mode 100644 index 0000000..6cf08ee --- /dev/null +++ b/nix/kubernetes/README.org @@ -0,0 +1,132 @@ +* To-do +** Perhaps use overlay for /etc for speedup +#+begin_src nix + system.etc.overlay.enable = true; +#+end_src +** read https://nixos.org/manual/nixos/stable/ +** Performance for mini pc +#+begin_src nix + security.pam.loginLimits = [ + { domain = "@users"; item = "rtprio"; type = "-"; value = 1; } + ]; +#+end_src +* IP Ranges +| | IPv4 | IPv6 | +|--------------------------------+-----------------------------+-----------------------------------------| +| Pod | 10.200.0.0/16 | 2620:11f:7001:7:ffff:eeee::/96 | +| Service | 10.197.0.0/16 | fd00:3e42:e349::/112 | +| Node | 10.215.1.0/24 | 2620:11f:7001:7:ffff:ffff:0ad7:0100/120 | +| Load Balancer | 74.80.180.139-74.80.180.142 | 2620:11f:7001:7:ffff:dddd::/96 | +| Load Balancer Private (unused) | 10.198.0.0/16 | fd9c:0bd5:22a4::/112 | +| PowerDNS from inside cluster | 10.215.1.211 | | +* Healthcheck +** Check cilium status +#+begin_src bash + kubectl -n kube-system exec ds/cilium -- cilium-dbg status --verbose + kubectl -n kube-system exec ds/cilium -- cilium-dbg status | grep KubeProxyReplacement +#+end_src +** Check connectivity +#+begin_src bash + cilium connectivity test +#+end_src +** Show dropped packets +#+begin_src bash + kubectl -n kube-system exec ds/cilium -- cilium-dbg monitor --type drop +#+end_src +** Show dropped packets for a specific pod +#+begin_src bash + kubectl -n kube-system exec ds/cilium -- hubble observe --since 30s --pod cnpg-system/cnpg-controller-manager-84d498b97-q5m4n --type drop +#+end_src +** Install flux +#+begin_src bash + nix shell 'nixpkgs#fluxcd' + + flux bootstrap git \ + --url=ssh://git@// \ + --branch=main \ + --private-key-file= \ + --password= \ + --path=clusters/my-cluster +#+end_src + +#+begin_src bash + nix shell 'nixpkgs#kubernetes-helm' + + helm template --dry-run=server flux-operator oci://ghcr.io/controlplaneio-fluxcd/charts/flux-operator \ + --namespace flux-system \ + --create-namespace +#+end_src + +#+begin_src text + apiVersion: fluxcd.controlplane.io/v1 + kind: FluxInstance + metadata: + name: flux + namespace: flux-system + annotations: + fluxcd.controlplane.io/reconcileEvery: "1h" + fluxcd.controlplane.io/reconcileTimeout: "5m" + spec: + distribution: + version: "2.x" + registry: "ghcr.io/fluxcd" + artifact: "oci://ghcr.io/controlplaneio-fluxcd/flux-operator-manifests" + components: + - source-controller + - kustomize-controller + - helm-controller + - notification-controller + - image-reflector-controller + - image-automation-controller + cluster: + type: kubernetes + size: medium + multitenant: false + networkPolicy: true + domain: "cluster.local" + kustomize: + patches: + - target: + kind: Deployment + patch: | + - op: replace + path: /spec/template/spec/nodeSelector + value: + kubernetes.io/os: linux + - op: add + path: /spec/template/spec/tolerations + value: + - key: "CriticalAddonsOnly" + operator: "Exists" + sync: + kind: OCIRepository + url: "oci://ghcr.io/my-org/my-fleet-manifests" + ref: "latest" + path: "clusters/my-cluster" + pullSecret: "ghcr-auth" +#+end_src + +#+begin_src text + apiVersion: fluxcd.controlplane.io/v1 + kind: FluxInstance + metadata: + name: flux + namespace: flux-system + spec: + distribution: + version: "2.7.x" + registry: "ghcr.io/fluxcd" + sync: + kind: GitRepository + url: "ssh://git@10.215.1.210:22/repos/mrmanager" + ref: "refs/heads/nix" + path: "clusters/my-cluster" + pullSecret: "flux-system" +#+end_src + +#+begin_src bash + flux create secret git flux-system \ + --url=https://gitlab.com/my-org/my-fleet.git \ + --username=git \ + --password=$GITLAB_TOKEN +#+end_src diff --git a/nix/kubernetes/configuration.nix b/nix/kubernetes/configuration.nix new file mode 100644 index 0000000..ef64f2f --- /dev/null +++ b/nix/kubernetes/configuration.nix @@ -0,0 +1,149 @@ +{ + config, + lib, + ... +}: + +{ + imports = [ + ./roles/boot + ./roles/cilium + ./roles/containerd + ./roles/control_plane + ./roles/debugging + ./roles/doas + ./roles/dont_use_substituters + ./roles/etcd + ./roles/firewall + ./roles/image_based_appliance + ./roles/iso + ./roles/kernel + ./roles/kube_apiserver + ./roles/kube_controller_manager + ./roles/kube_proxy + ./roles/kube_scheduler + ./roles/kubelet + ./roles/kubernetes + ./roles/minimal_base + ./roles/network + ./roles/nvme + ./roles/optimized_build + ./roles/ssh + ./roles/sshd + ./roles/user + ./roles/worker_node + ./roles/zsh + ./util/install_files + ./util/unfree_polyfill + ]; + + config = { + nix.settings.experimental-features = [ + "nix-command" + "flakes" + "ca-derivations" + # "blake3-hashes" + # "git-hashing" + ]; + nix.settings.trusted-users = [ "@wheel" ]; + + hardware.enableRedistributableFirmware = true; + + # Keep outputs so we can build offline. + nix.settings.keep-outputs = true; + nix.settings.keep-derivations = true; + + # Automatic garbage collection + nix.gc = lib.mkIf (!config.me.buildingPortable) { + # Runs nix-collect-garbage --delete-older-than 5d + automatic = true; + persistent = true; + dates = "monthly"; + # randomizedDelaySec = "14m"; + options = "--delete-older-than 30d"; + }; + nix.settings.auto-optimise-store = !config.me.buildingPortable; + + environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) { + hideMounts = true; + directories = [ + "/var/lib/nixos" # Contains user information (uids/gids) + "/var/lib/systemd" # Systemd state directory for random seed, persistent timers, core dumps, persist hardware state like backlight and rfkill + "/var/log/journal" # Logs, alternatively set `services.journald.storage = "volatile";` to write to /run/log/journal + ]; + files = [ + "/etc/machine-id" # Systemd unique machine id "otherwise, the system journal may fail to list earlier boots, etc" + ]; + }; + + # Write a list of the currently installed packages to /etc/current-system-packages + # environment.etc."current-system-packages".text = + # let + # packages = builtins.map (p: "${p.name}") config.environment.systemPackages; + # sortedUnique = builtins.sort builtins.lessThan (lib.unique packages); + # formatted = builtins.concatStringsSep "\n" sortedUnique; + # in + # formatted; + + # nixpkgs.overlays = [ + # (final: prev: { + # foot = throw "foo"; + # }) + # ]; + + nixpkgs.overlays = + let + disableTests = ( + package_name: + (final: prev: { + "${package_name}" = prev."${package_name}".overrideAttrs (old: { + doCheck = false; + doInstallCheck = false; + }); + }) + ); + in + [ + # (final: prev: { + # imagemagick = prev.imagemagick.overrideAttrs (old: rec { + # # 7.1.2-6 seems to no longer exist, so use 7.1.2-7 + # version = "7.1.2-7"; + + # src = final.fetchFromGitHub { + # owner = "ImageMagick"; + # repo = "ImageMagick"; + # tag = version; + # hash = "sha256-9ARCYftoXiilpJoj+Y+aLCEqLmhHFYSrHfgA5DQHbGo="; + # }; + # }); + # }) + # (final: prev: { + # grub2 = (final.callPackage ./package/grub { }); + # }) + (final: prev: { + inherit (final.unoptimized) + libtpms + ; + }) + ]; + + # This option defines the first version of NixOS you have installed on this particular machine, + # and is used to maintain compatibility with application data (e.g. databases) created on older NixOS versions. + # + # Most users should NEVER change this value after the initial install, for any reason, + # even if you've upgraded your system to a new NixOS release. + # + # This value does NOT affect the Nixpkgs version your packages and OS are pulled from, + # so changing it will NOT upgrade your system - see https://nixos.org/manual/nixos/stable/#sec-upgrading for how + # to actually do that. + # + # This value being lower than the current NixOS release does NOT mean your system is + # out of date, out of support, or vulnerable. + # + # Do NOT change this value unless you have manually inspected all the changes it would make to your configuration, + # and migrated your data accordingly. + # + # For more information, see `man configuration.nix` or https://nixos.org/manual/nixos/stable/options#opt-system.stateVersion . + system.stateVersion = "24.11"; # Did you read the comment? + }; +} diff --git a/nix/kubernetes/flake.lock b/nix/kubernetes/flake.lock new file mode 100644 index 0000000..8f09179 --- /dev/null +++ b/nix/kubernetes/flake.lock @@ -0,0 +1,256 @@ +{ + "nodes": { + "crane": { + "locked": { + "lastModified": 1731098351, + "narHash": "sha256-HQkYvKvaLQqNa10KEFGgWHfMAbWBfFp+4cAgkut+NNE=", + "owner": "ipetkov", + "repo": "crane", + "rev": "ef80ead953c1b28316cc3f8613904edc2eb90c28", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "repo": "crane", + "type": "github" + } + }, + "disko": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1780290312, + "narHash": "sha256-eTAlX0CwgB84Ts3GaBd944A3DRXVMzgA0EqroZBISUo=", + "owner": "nix-community", + "repo": "disko", + "rev": "115e5211780054d8a890b41f0b7734cafad54dfe", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "disko", + "type": "github" + } + }, + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "lanzaboote", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1730504689, + "narHash": "sha256-hgmguH29K2fvs9szpq2r3pz2/8cJd2LPS+b4tfNFCwE=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "506278e768c2a08bec68eb62932193e341f55c90", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "gitignore": { + "inputs": { + "nixpkgs": [ + "lanzaboote", + "pre-commit-hooks-nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1709087332, + "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=", + "owner": "hercules-ci", + "repo": "gitignore.nix", + "rev": "637db329424fd7e46cf4185293b9cc8c88c95394", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "gitignore.nix", + "type": "github" + } + }, + "home-manager": { + "inputs": { + "nixpkgs": [ + "impermanence", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1768598210, + "narHash": "sha256-kkgA32s/f4jaa4UG+2f8C225Qvclxnqs76mf8zvTVPg=", + "owner": "nix-community", + "repo": "home-manager", + "rev": "c47b2cc64a629f8e075de52e4742de688f930dc6", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "home-manager", + "type": "github" + } + }, + "impermanence": { + "inputs": { + "home-manager": "home-manager", + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1769548169, + "narHash": "sha256-03+JxvzmfwRu+5JafM0DLbxgHttOQZkUtDWBmeUkN8Y=", + "owner": "nix-community", + "repo": "impermanence", + "rev": "7b1d382faf603b6d264f58627330f9faa5cba149", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "impermanence", + "type": "github" + } + }, + "lanzaboote": { + "inputs": { + "crane": "crane", + "flake-compat": "flake-compat", + "flake-parts": "flake-parts", + "nixpkgs": [ + "nixpkgs" + ], + "pre-commit-hooks-nix": "pre-commit-hooks-nix", + "rust-overlay": "rust-overlay" + }, + "locked": { + "lastModified": 1737639419, + "narHash": "sha256-AEEDktApTEZ5PZXNDkry2YV2k6t0dTgLPEmAZbnigXU=", + "owner": "nix-community", + "repo": "lanzaboote", + "rev": "a65905a09e2c43ff63be8c0e86a93712361f871e", + "type": "github" + }, + "original": { + "owner": "nix-community", + "ref": "v0.4.2", + "repo": "lanzaboote", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1780749050, + "narHash": "sha256-3av0pIjlOWQ6rDbNOmpUSvbNnJkGORQKKjb4LtCZsIY=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a799d3e3886da994fa307f817a6bc705ae538eeb", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-stable": { + "locked": { + "lastModified": 1730741070, + "narHash": "sha256-edm8WG19kWozJ/GqyYx2VjW99EdhjKwbY3ZwdlPAAlo=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "d063c1dd113c91ab27959ba540c0d9753409edf3", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-24.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "pre-commit-hooks-nix": { + "inputs": { + "flake-compat": [ + "lanzaboote", + "flake-compat" + ], + "gitignore": "gitignore", + "nixpkgs": [ + "lanzaboote", + "nixpkgs" + ], + "nixpkgs-stable": "nixpkgs-stable" + }, + "locked": { + "lastModified": 1731363552, + "narHash": "sha256-vFta1uHnD29VUY4HJOO/D6p6rxyObnf+InnSMT4jlMU=", + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "rev": "cd1af27aa85026ac759d5d3fccf650abe7e1bbf0", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "type": "github" + } + }, + "root": { + "inputs": { + "disko": "disko", + "impermanence": "impermanence", + "lanzaboote": "lanzaboote", + "nixpkgs": "nixpkgs" + } + }, + "rust-overlay": { + "inputs": { + "nixpkgs": [ + "lanzaboote", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1731897198, + "narHash": "sha256-Ou7vLETSKwmE/HRQz4cImXXJBr/k9gp4J4z/PF8LzTE=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "0be641045af6d8666c11c2c40e45ffc9667839b5", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/nix/kubernetes/flake.nix b/nix/kubernetes/flake.nix new file mode 100644 index 0000000..a38d663 --- /dev/null +++ b/nix/kubernetes/flake.nix @@ -0,0 +1,183 @@ +# Get a repl for this flake +# nix repl --expr "builtins.getFlake \"$PWD\"" + +# TODO maybe use `nix eval --raw .#odo.iso.outPath` + +# +# Install on a new machine: +# +# Set +# me.disko.enable = true; +# me.disko.offline.enable = true; +# +# Run +# doas disko --mode destroy,format,mount hosts/recovery/disk-config.nix +# doas nixos-install --substituters "http://10.0.2.2:8080?trusted=1 https://cache.nixos.org/" --flake ".#recovery" + +{ + description = "My system configuration"; + + inputs = { + impermanence = { + url = "github:nix-community/impermanence"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + lanzaboote = { + url = "github:nix-community/lanzaboote/v0.4.2"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + disko = { + url = "github:nix-community/disko"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + }; + + outputs = + { + self, + nixpkgs, + disko, + impermanence, + lanzaboote, + ... + }: + let + forAllSystems = + func: + builtins.listToAttrs ( + map (system: { + name = system; + value = func system; + }) nixpkgs.lib.systems.flakeExposed + ); + nodes = { + controller0 = { + system = "x86_64-linux"; + }; + controller1 = { + system = "x86_64-linux"; + }; + controller2 = { + system = "x86_64-linux"; + }; + worker0 = { + system = "x86_64-linux"; + }; + worker1 = { + system = "x86_64-linux"; + }; + worker2 = { + system = "x86_64-linux"; + }; + }; + nixosConfigs = builtins.mapAttrs ( + hostname: nodeConfig: format: + nixpkgs.lib.nixosSystem { + specialArgs = { + inherit self; + + this_nixos_config = self.nixosConfigurations."${hostname}"; + + all_nixos_configs = self.nixosConfigurations; + }; + modules = [ + impermanence.nixosModules.impermanence + lanzaboote.nixosModules.lanzaboote + disko.nixosModules.disko + ./configuration.nix + (./. + "/hosts/${hostname}") + (./. + "/formats/${format}.nix") + { + config = { + nixpkgs.hostPlatform.system = nodeConfig.system; + nixpkgs.overlays = [ + (final: prev: { + # stable = nixpkgs-stable.legacyPackages."${prev.stdenv.hostPlatform.system}"; + unoptimized = import nixpkgs { + system = prev.stdenv.hostPlatform.system; + hostPlatform.gcc.arch = "default"; + hostPlatform.gcc.tune = "default"; + }; + }) + ]; + }; + } + ( + { + config, + lib, + pkgs, + ... + }: + let + nix-self-repl = pkgs.writeShellScriptBin "nix-self-repl" '' + source /etc/set-environment + nix repl --expr 'builtins.getFlake "${self}"' + ''; + # If we wanted the current version of a flake then we'd just launch + # nix repl + # and then run: + # :lf /path/to/flake + in + { + config = { + environment.systemPackages = lib.mkIf config.nix.enable [ nix-self-repl ]; + }; + } + ) + ]; + } + ) nodes; + installerConfig = + hostname: nodeConfig: + nixpkgs.lib.nixosSystem { + specialArgs = { + targetSystem = self.nixosConfigurations."${hostname}"; + }; + modules = [ + ./formats/installer.nix + ( + { + config, + lib, + pkgs, + ... + }: + let + nix-self-repl = pkgs.writeShellScriptBin "nix-self-repl" '' + source /etc/set-environment + nix repl --expr 'builtins.getFlake "${self}"' + ''; + # If we wanted the current version of a flake then we'd just launch + # nix repl + # and then run: + # :lf /path/to/flake + in + { + config = { + environment.systemPackages = lib.mkIf config.nix.enable [ nix-self-repl ]; + }; + } + ) + ({ nixpkgs.hostPlatform.system = nodeConfig.system; }) + ]; + }; + in + { + nixosConfigurations = (builtins.mapAttrs (name: value: value "toplevel") nixosConfigs); + } + // { + packages = ( + forAllSystems ( + system: + (builtins.mapAttrs (hostname: nodeConfig: { + iso = (nixosConfigs."${hostname}" "iso").config.system.build.isoImage; + vm_iso = (nixosConfigs."${hostname}" "vm_iso").config.system.build.isoImage; + sd = (nixosConfigs."${hostname}" "sd").config.system.build.sdImage; + installer = (installerConfig hostname nodes."${hostname}").config.system.build.isoImage; + }) (nixpkgs.lib.attrsets.filterAttrs (hostname: nodeConfig: nodeConfig.system == system) nodes)) + ) + ); + }; +} diff --git a/nix/kubernetes/formats/installer.nix b/nix/kubernetes/formats/installer.nix new file mode 100644 index 0000000..93b1187 --- /dev/null +++ b/nix/kubernetes/formats/installer.nix @@ -0,0 +1,74 @@ +{ + config, + pkgs, + lib, + modulesPath, + targetSystem, + ... +}: +let + installer = pkgs.writeShellApplication { + name = "installer"; + runtimeInputs = with pkgs; [ + # clevis + dosfstools + e2fsprogs + gawk + nixos-install-tools + util-linux + config.nix.package + ]; + text = '' + set -euo pipefail + + ${targetSystem.config.system.build.diskoScript} + + nixos-install --no-channel-copy --no-root-password --option substituters "" --system ${targetSystem.config.system.build.toplevel} + ''; + }; + installerFailsafe = pkgs.writeShellScript "failsafe" '' + ${lib.getExe installer} || echo "ERROR: Installation failure!" + sleep 3600 + ''; +in +{ + imports = [ + (modulesPath + "/installer/cd-dvd/iso-image.nix") + (modulesPath + "/profiles/all-hardware.nix") + ]; + + # boot.kernelPackages = pkgs.linuxPackagesFor pkgs.linux_6_17; + # boot.zfs.package = pkgs.zfs_unstable; + boot.kernelPackages = pkgs.linuxPackagesFor pkgs.linux; + boot.kernelParams = [ + "quiet" + "systemd.unit=getty.target" + ]; + boot.supportedFilesystems.zfs = true; + boot.initrd.systemd.enable = true; + + networking.hostId = "04581ecf"; + + isoImage.makeEfiBootable = true; + isoImage.makeUsbBootable = true; + isoImage.squashfsCompression = "zstd -Xcompression-level 15"; + + environment.systemPackages = [ + installer + ]; + + systemd.services."getty@tty1" = { + overrideStrategy = "asDropin"; + serviceConfig = { + ExecStart = [ + "" + installerFailsafe + ]; + Restart = "no"; + StandardInput = "null"; + }; + }; + + # system.stateVersion = lib.mkDefault lib.trivial.release; + system.stateVersion = "24.11"; +} diff --git a/nix/kubernetes/formats/iso.nix b/nix/kubernetes/formats/iso.nix new file mode 100644 index 0000000..974d710 --- /dev/null +++ b/nix/kubernetes/formats/iso.nix @@ -0,0 +1,36 @@ +{ + config, + lib, + modulesPath, + pkgs, + ... +}: + +{ + imports = [ + (modulesPath + "/installer/cd-dvd/iso-image.nix") + ]; + + config = { + isoImage.makeEfiBootable = true; + isoImage.makeUsbBootable = true; + + networking.dhcpcd.enable = true; + networking.useDHCP = true; + + me.buildingPortable = true; + me.disko.enable = true; + me.disko.offline.enable = true; + me.mountPersistence = lib.mkForce false; + # me.optimizations.enable = lib.mkForce false; + + # Not doing image_based_appliance because this might be an install ISO, in which case we'd need nix to do the install. + # me.image_based_appliance.enable = true; + + # TODO: Should I use this instead of doing a mkIf for the disk config? + # disko.enableConfig = false; + + # Faster image generation for testing/development. + isoImage.squashfsCompression = "zstd -Xcompression-level 15"; + }; +} diff --git a/nix/kubernetes/formats/sd.nix b/nix/kubernetes/formats/sd.nix new file mode 100644 index 0000000..208195f --- /dev/null +++ b/nix/kubernetes/formats/sd.nix @@ -0,0 +1,32 @@ +{ + modulesPath, + ... +}: + +{ + imports = [ + (modulesPath + "/installer/sd-card/sd-image.nix") + ]; + + config = { + isoImage.makeEfiBootable = true; + isoImage.makeUsbBootable = true; + + boot.loader.grub.enable = false; + boot.loader.generic-extlinux-compatible.enable = true; + + # TODO: image based appliance? + + # TODO: Maybe this? + # fileSystems = { + # "/" = { + # device = "/dev/disk/by-label/NIXOS_SD"; + # fsType = "ext4"; + # options = [ + # "noatime" + # "norelatime" + # ]; + # }; + # }; + }; +} diff --git a/nix/kubernetes/formats/toplevel.nix b/nix/kubernetes/formats/toplevel.nix new file mode 100644 index 0000000..ffcd441 --- /dev/null +++ b/nix/kubernetes/formats/toplevel.nix @@ -0,0 +1 @@ +{ } diff --git a/nix/kubernetes/formats/vm_iso.nix b/nix/kubernetes/formats/vm_iso.nix new file mode 100644 index 0000000..ef487de --- /dev/null +++ b/nix/kubernetes/formats/vm_iso.nix @@ -0,0 +1,22 @@ +{ + lib, + modulesPath, + ... +}: + +{ + imports = [ + (modulesPath + "/installer/cd-dvd/iso-image.nix") + (modulesPath + "/profiles/qemu-guest.nix") # VirtIO kernel modules + ]; + + config = { + isoImage.makeEfiBootable = true; + isoImage.makeUsbBootable = true; + + networking.dhcpcd.enable = true; + networking.useDHCP = true; + + me.image_based_appliance.enable = true; + }; +} diff --git a/nix/kubernetes/functions/to_yaml.nix b/nix/kubernetes/functions/to_yaml.nix new file mode 100644 index 0000000..c32d4ca --- /dev/null +++ b/nix/kubernetes/functions/to_yaml.nix @@ -0,0 +1,24 @@ +{ + pkgs, + ... +}: +let + to_yaml_file = + file_name: contents: + let + settingsFormat = pkgs.formats.yaml { }; + yaml_file = settingsFormat.generate file_name contents; + in + yaml_file; + to_yaml = + file_name: contents: + let + settingsFormat = pkgs.formats.yaml { }; + yaml_file = settingsFormat.generate file_name contents; + yaml_content = builtins.readFile yaml_file; + in + yaml_content; +in +{ + inherit to_yaml to_yaml_file; +} diff --git a/nix/kubernetes/hosts/controller0/DEPLOY_BOOT b/nix/kubernetes/hosts/controller0/DEPLOY_BOOT new file mode 100755 index 0000000..2648e46 --- /dev/null +++ b/nix/kubernetes/hosts/controller0/DEPLOY_BOOT @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=controller0 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild boot --flake "$DIR/../../#controller0" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller0/DEPLOY_SWITCH b/nix/kubernetes/hosts/controller0/DEPLOY_SWITCH new file mode 100755 index 0000000..e8c3f61 --- /dev/null +++ b/nix/kubernetes/hosts/controller0/DEPLOY_SWITCH @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=controller0 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild switch --flake "$DIR/../../#controller0" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller0/ISO b/nix/kubernetes/hosts/controller0/ISO new file mode 100755 index 0000000..f86f8df --- /dev/null +++ b/nix/kubernetes/hosts/controller0/ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#controller0.iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller0/SELF_BOOT b/nix/kubernetes/hosts/controller0/SELF_BOOT new file mode 100755 index 0000000..c447544 --- /dev/null +++ b/nix/kubernetes/hosts/controller0/SELF_BOOT @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild boot --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller0" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller0/SELF_BUILD b/nix/kubernetes/hosts/controller0/SELF_BUILD new file mode 100755 index 0000000..e92bea4 --- /dev/null +++ b/nix/kubernetes/hosts/controller0/SELF_BUILD @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild build --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller0" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller0/SELF_SWITCH b/nix/kubernetes/hosts/controller0/SELF_SWITCH new file mode 100755 index 0000000..c86ce07 --- /dev/null +++ b/nix/kubernetes/hosts/controller0/SELF_SWITCH @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild switch --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller0" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller0/VM_ISO b/nix/kubernetes/hosts/controller0/VM_ISO new file mode 100755 index 0000000..98bcd23 --- /dev/null +++ b/nix/kubernetes/hosts/controller0/VM_ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#controller0.vm_iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller0/default.nix b/nix/kubernetes/hosts/controller0/default.nix new file mode 100644 index 0000000..bb32f50 --- /dev/null +++ b/nix/kubernetes/hosts/controller0/default.nix @@ -0,0 +1,130 @@ +# MANUAL: On client machines generate signing keys: +# nix-store --generate-binary-cache-key some-name /persist/manual/nix/nix-cache-key.sec /persist/manual/nix/nix-cache-key.pub +# +# Trust other machines and add the substituters: +# nix.binaryCachePublicKeys = [ "some-name:AzNW1MOlkNEsUAXS1jIFZ1QCFKXjV+Y/LrF37quAZ1A=" ]; +# nix.binaryCaches = [ "https://test.example/nix-cache" ]; + +{ + config, + lib, + pkgs, + ... +}: +{ + imports = [ + ./hardware-configuration.nix + ./vm_disk.nix + ]; + + config = { + networking = + let + interface = "enp0s2"; + in + { + # Generate with `head -c4 /dev/urandom | od -A none -t x4` + hostId = "769e1349"; + + hostName = "controller0"; # Define your hostname. + + interfaces = { + "${interface}" = { + ipv4.addresses = [ + { + address = "10.215.1.221"; + prefixLength = 24; + } + ]; + + ipv6.addresses = [ + { + address = "2620:11f:7001:7:ffff:ffff:0ad7:01dd"; + prefixLength = 64; + } + ]; + }; + }; + defaultGateway = "10.215.1.1"; + defaultGateway6 = { + # address = "2620:11f:7001:7::1"; + address = "2620:11f:7001:7:ffff:ffff:0ad7:0101"; + inherit interface; + }; + + dhcpcd.enable = lib.mkForce false; + useDHCP = lib.mkForce false; + }; + + time.timeZone = "America/New_York"; + i18n.defaultLocale = "en_US.UTF-8"; + + me.boot.enable = true; + me.boot.secure = false; + me.mountPersistence = true; + boot.loader.timeout = lib.mkForce 0; # We can always generate a new ISO if we need to access other boot options. + + me.optimizations = { + enable = true; + arch = "znver4"; + # build_arch = "x86-64-v3"; + system_features = [ + "gccarch-znver4" + "gccarch-skylake" + "gccarch-kabylake" + # "gccarch-alderlake" missing WAITPKG + "gccarch-x86-64-v3" + "gccarch-x86-64-v4" + "benchmark" + "big-parallel" + "kvm" + "nixos-test" + ]; + }; + + # Mount tmpfs at /tmp + boot.tmp.useTmpfs = true; + + # Enable TRIM + # services.fstrim.enable = lib.mkDefault true; + + # nix.optimise.automatic = true; + # nix.optimise.dates = [ "03:45" ]; + # nix.optimise.persistent = true; + + environment.systemPackages = with pkgs; [ + htop + ]; + + # nix.sshServe.enable = true; + # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ]; + + me.etcd.cluster_name = "put-a-nix-on-it"; + me.etcd.internal_ip = [ + # "10.215.1.221" + "[2620:11f:7001:7:ffff:ffff:0ad7:01dd]" + ]; + me.etcd.initial_cluster = [ + # "controller0=https://10.215.1.221:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01dd + # "controller1=https://10.215.1.222:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01de + # "controller2=https://10.215.1.223:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01df + "controller0=https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2380" # 10.215.1.221 + "controller1=https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2380" # 10.215.1.222 + "controller2=https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2380" # 10.215.1.223 + ]; + + me.kube_apiserver.internal_ip = "2620:11f:7001:7:ffff:ffff:0ad7:01dd"; + # me.kube_apiserver.external_ip = "74.80.180.138"; + me.kube_apiserver.external_ip = "2620:11f:7001:7:ffff:ffff:0ad7:01dd"; + me.kube_apiserver.etcd_services = [ + "https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2379" # 10.215.1.221 + "https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2379" # 10.215.1.222 + "https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2379" # 10.215.1.223 + ]; + + me.control_plane.enable = true; + me.dont_use_substituters.enable = true; + me.etcd.enable = true; + me.minimal_base.enable = true; + }; +} diff --git a/nix/kubernetes/hosts/controller0/hardware-configuration.nix b/nix/kubernetes/hosts/controller0/hardware-configuration.nix new file mode 100644 index 0000000..6029e08 --- /dev/null +++ b/nix/kubernetes/hosts/controller0/hardware-configuration.nix @@ -0,0 +1,31 @@ +{ + config, + lib, + modulesPath, + ... +}: + +{ + imports = [ + (modulesPath + "/installer/scan/not-detected.nix") + ]; + + config = { + boot.initrd.availableKernelModules = [ + "nvme" + "xhci_pci" + "thunderbolt" + ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ ]; + boot.extraModulePackages = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + # networking.useDHCP = lib.mkDefault true; + # networking.interfaces.eno1.useDHCP = lib.mkDefault true; + # networking.interfaces.wlp58s0.useDHCP = lib.mkDefault true; + }; +} diff --git a/nix/kubernetes/hosts/controller0/vm_disk.nix b/nix/kubernetes/hosts/controller0/vm_disk.nix new file mode 100644 index 0000000..83683f8 --- /dev/null +++ b/nix/kubernetes/hosts/controller0/vm_disk.nix @@ -0,0 +1,94 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + config = { + # Mount the local disk + fileSystems = lib.mkIf config.me.mountPersistence { + "/.disk" = lib.mkForce { + device = "/dev/nvme0n1p1"; + fsType = "ext4"; + options = [ + "noatime" + "discard" + ]; + neededForBoot = true; + }; + + "/.persist" = lib.mkForce { + device = "bind9p"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + + "/persist" = { + fsType = "none"; + device = "/.persist/persist"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.persist/persist" + ]; + neededForBoot = true; + }; + + "/state" = { + fsType = "none"; + device = "/.persist/state"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.persist/state" + ]; + neededForBoot = true; + }; + + "/k8spv" = lib.mkForce { + device = "k8spv"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + + "/disk" = { + fsType = "none"; + device = "/.disk/persist"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.disk/persist" + ]; + neededForBoot = true; + }; + }; + }; +} diff --git a/nix/kubernetes/hosts/controller1/DEPLOY_BOOT b/nix/kubernetes/hosts/controller1/DEPLOY_BOOT new file mode 100755 index 0000000..a98d5d4 --- /dev/null +++ b/nix/kubernetes/hosts/controller1/DEPLOY_BOOT @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=controller1 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild boot --flake "$DIR/../../#controller1" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller1/DEPLOY_SWITCH b/nix/kubernetes/hosts/controller1/DEPLOY_SWITCH new file mode 100755 index 0000000..e7c22a0 --- /dev/null +++ b/nix/kubernetes/hosts/controller1/DEPLOY_SWITCH @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=controller1 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild switch --flake "$DIR/../../#controller1" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller1/ISO b/nix/kubernetes/hosts/controller1/ISO new file mode 100755 index 0000000..57320a2 --- /dev/null +++ b/nix/kubernetes/hosts/controller1/ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#controller1.iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller1/SELF_BOOT b/nix/kubernetes/hosts/controller1/SELF_BOOT new file mode 100755 index 0000000..6f38732 --- /dev/null +++ b/nix/kubernetes/hosts/controller1/SELF_BOOT @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild boot --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller1" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller1/SELF_BUILD b/nix/kubernetes/hosts/controller1/SELF_BUILD new file mode 100755 index 0000000..bc57565 --- /dev/null +++ b/nix/kubernetes/hosts/controller1/SELF_BUILD @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild build --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller1" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller1/SELF_SWITCH b/nix/kubernetes/hosts/controller1/SELF_SWITCH new file mode 100755 index 0000000..2879d19 --- /dev/null +++ b/nix/kubernetes/hosts/controller1/SELF_SWITCH @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild switch --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller1" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller1/VM_ISO b/nix/kubernetes/hosts/controller1/VM_ISO new file mode 100755 index 0000000..e33e16b --- /dev/null +++ b/nix/kubernetes/hosts/controller1/VM_ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#controller1.vm_iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller1/default.nix b/nix/kubernetes/hosts/controller1/default.nix new file mode 100644 index 0000000..f4b5c87 --- /dev/null +++ b/nix/kubernetes/hosts/controller1/default.nix @@ -0,0 +1,130 @@ +# MANUAL: On client machines generate signing keys: +# nix-store --generate-binary-cache-key some-name /persist/manual/nix/nix-cache-key.sec /persist/manual/nix/nix-cache-key.pub +# +# Trust other machines and add the substituters: +# nix.binaryCachePublicKeys = [ "some-name:AzNW1MOlkNEsUAXS1jIFZ1QCFKXjV+Y/LrF37quAZ1A=" ]; +# nix.binaryCaches = [ "https://test.example/nix-cache" ]; + +{ + config, + lib, + pkgs, + ... +}: +{ + imports = [ + ./hardware-configuration.nix + ./vm_disk.nix + ]; + + config = { + networking = + let + interface = "enp0s2"; + in + { + # Generate with `head -c4 /dev/urandom | od -A none -t x4` + hostId = "59a83979"; + + hostName = "controller1"; # Define your hostname. + + interfaces = { + "${interface}" = { + ipv4.addresses = [ + { + address = "10.215.1.222"; + prefixLength = 24; + } + ]; + + ipv6.addresses = [ + { + address = "2620:11f:7001:7:ffff:ffff:0ad7:01de"; + prefixLength = 64; + } + ]; + }; + }; + defaultGateway = "10.215.1.1"; + defaultGateway6 = { + # address = "2620:11f:7001:7::1"; + address = "2620:11f:7001:7:ffff:ffff:0ad7:0101"; + inherit interface; + }; + + dhcpcd.enable = lib.mkForce false; + useDHCP = lib.mkForce false; + }; + + time.timeZone = "America/New_York"; + i18n.defaultLocale = "en_US.UTF-8"; + + me.boot.enable = true; + me.boot.secure = false; + me.mountPersistence = true; + boot.loader.timeout = lib.mkForce 0; # We can always generate a new ISO if we need to access other boot options. + + me.optimizations = { + enable = true; + arch = "znver4"; + # build_arch = "x86-64-v3"; + system_features = [ + "gccarch-znver4" + "gccarch-skylake" + "gccarch-kabylake" + # "gccarch-alderlake" missing WAITPKG + "gccarch-x86-64-v3" + "gccarch-x86-64-v4" + "benchmark" + "big-parallel" + "kvm" + "nixos-test" + ]; + }; + + # Mount tmpfs at /tmp + boot.tmp.useTmpfs = true; + + # Enable TRIM + # services.fstrim.enable = lib.mkDefault true; + + # nix.optimise.automatic = true; + # nix.optimise.dates = [ "03:45" ]; + # nix.optimise.persistent = true; + + environment.systemPackages = with pkgs; [ + htop + ]; + + # nix.sshServe.enable = true; + # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ]; + + me.etcd.cluster_name = "put-a-nix-on-it"; + me.etcd.internal_ip = [ + # "10.215.1.221" + "[2620:11f:7001:7:ffff:ffff:0ad7:01de]" + ]; + me.etcd.initial_cluster = [ + # "controller0=https://10.215.1.221:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01dd + # "controller1=https://10.215.1.222:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01de + # "controller2=https://10.215.1.223:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01df + "controller0=https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2380" # 10.215.1.221 + "controller1=https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2380" # 10.215.1.222 + "controller2=https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2380" # 10.215.1.223 + ]; + + me.kube_apiserver.internal_ip = "2620:11f:7001:7:ffff:ffff:0ad7:01de"; + # me.kube_apiserver.external_ip = "74.80.180.138"; + me.kube_apiserver.external_ip = "2620:11f:7001:7:ffff:ffff:0ad7:01de"; + me.kube_apiserver.etcd_services = [ + "https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2379" # 10.215.1.221 + "https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2379" # 10.215.1.222 + "https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2379" # 10.215.1.223 + ]; + + me.control_plane.enable = true; + me.dont_use_substituters.enable = true; + me.etcd.enable = true; + me.minimal_base.enable = true; + }; +} diff --git a/nix/kubernetes/hosts/controller1/hardware-configuration.nix b/nix/kubernetes/hosts/controller1/hardware-configuration.nix new file mode 100644 index 0000000..6029e08 --- /dev/null +++ b/nix/kubernetes/hosts/controller1/hardware-configuration.nix @@ -0,0 +1,31 @@ +{ + config, + lib, + modulesPath, + ... +}: + +{ + imports = [ + (modulesPath + "/installer/scan/not-detected.nix") + ]; + + config = { + boot.initrd.availableKernelModules = [ + "nvme" + "xhci_pci" + "thunderbolt" + ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ ]; + boot.extraModulePackages = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + # networking.useDHCP = lib.mkDefault true; + # networking.interfaces.eno1.useDHCP = lib.mkDefault true; + # networking.interfaces.wlp58s0.useDHCP = lib.mkDefault true; + }; +} diff --git a/nix/kubernetes/hosts/controller1/vm_disk.nix b/nix/kubernetes/hosts/controller1/vm_disk.nix new file mode 100644 index 0000000..83683f8 --- /dev/null +++ b/nix/kubernetes/hosts/controller1/vm_disk.nix @@ -0,0 +1,94 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + config = { + # Mount the local disk + fileSystems = lib.mkIf config.me.mountPersistence { + "/.disk" = lib.mkForce { + device = "/dev/nvme0n1p1"; + fsType = "ext4"; + options = [ + "noatime" + "discard" + ]; + neededForBoot = true; + }; + + "/.persist" = lib.mkForce { + device = "bind9p"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + + "/persist" = { + fsType = "none"; + device = "/.persist/persist"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.persist/persist" + ]; + neededForBoot = true; + }; + + "/state" = { + fsType = "none"; + device = "/.persist/state"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.persist/state" + ]; + neededForBoot = true; + }; + + "/k8spv" = lib.mkForce { + device = "k8spv"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + + "/disk" = { + fsType = "none"; + device = "/.disk/persist"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.disk/persist" + ]; + neededForBoot = true; + }; + }; + }; +} diff --git a/nix/kubernetes/hosts/controller2/DEPLOY_BOOT b/nix/kubernetes/hosts/controller2/DEPLOY_BOOT new file mode 100755 index 0000000..5c4bee4 --- /dev/null +++ b/nix/kubernetes/hosts/controller2/DEPLOY_BOOT @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=controller2 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild boot --flake "$DIR/../../#controller2" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller2/DEPLOY_SWITCH b/nix/kubernetes/hosts/controller2/DEPLOY_SWITCH new file mode 100755 index 0000000..53c95e7 --- /dev/null +++ b/nix/kubernetes/hosts/controller2/DEPLOY_SWITCH @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=controller2 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild switch --flake "$DIR/../../#controller2" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller2/ISO b/nix/kubernetes/hosts/controller2/ISO new file mode 100755 index 0000000..0850b60 --- /dev/null +++ b/nix/kubernetes/hosts/controller2/ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#controller2.iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller2/SELF_BOOT b/nix/kubernetes/hosts/controller2/SELF_BOOT new file mode 100755 index 0000000..2976bc9 --- /dev/null +++ b/nix/kubernetes/hosts/controller2/SELF_BOOT @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild boot --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller2" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller2/SELF_BUILD b/nix/kubernetes/hosts/controller2/SELF_BUILD new file mode 100755 index 0000000..a9a3b56 --- /dev/null +++ b/nix/kubernetes/hosts/controller2/SELF_BUILD @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild build --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller2" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller2/SELF_SWITCH b/nix/kubernetes/hosts/controller2/SELF_SWITCH new file mode 100755 index 0000000..dfca46e --- /dev/null +++ b/nix/kubernetes/hosts/controller2/SELF_SWITCH @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild switch --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#controller2" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller2/VM_ISO b/nix/kubernetes/hosts/controller2/VM_ISO new file mode 100755 index 0000000..f57c883 --- /dev/null +++ b/nix/kubernetes/hosts/controller2/VM_ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#controller2.vm_iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/controller2/default.nix b/nix/kubernetes/hosts/controller2/default.nix new file mode 100644 index 0000000..e9c318a --- /dev/null +++ b/nix/kubernetes/hosts/controller2/default.nix @@ -0,0 +1,130 @@ +# MANUAL: On client machines generate signing keys: +# nix-store --generate-binary-cache-key some-name /persist/manual/nix/nix-cache-key.sec /persist/manual/nix/nix-cache-key.pub +# +# Trust other machines and add the substituters: +# nix.binaryCachePublicKeys = [ "some-name:AzNW1MOlkNEsUAXS1jIFZ1QCFKXjV+Y/LrF37quAZ1A=" ]; +# nix.binaryCaches = [ "https://test.example/nix-cache" ]; + +{ + config, + lib, + pkgs, + ... +}: +{ + imports = [ + ./hardware-configuration.nix + ./vm_disk.nix + ]; + + config = { + networking = + let + interface = "enp0s2"; + in + { + # Generate with `head -c4 /dev/urandom | od -A none -t x4` + hostId = "26a43660"; + + hostName = "controller2"; # Define your hostname. + + interfaces = { + "${interface}" = { + ipv4.addresses = [ + { + address = "10.215.1.223"; + prefixLength = 24; + } + ]; + + ipv6.addresses = [ + { + address = "2620:11f:7001:7:ffff:ffff:0ad7:01df"; + prefixLength = 64; + } + ]; + }; + }; + defaultGateway = "10.215.1.1"; + defaultGateway6 = { + # address = "2620:11f:7001:7::1"; + address = "2620:11f:7001:7:ffff:ffff:0ad7:0101"; + inherit interface; + }; + + dhcpcd.enable = lib.mkForce false; + useDHCP = lib.mkForce false; + }; + + time.timeZone = "America/New_York"; + i18n.defaultLocale = "en_US.UTF-8"; + + me.boot.enable = true; + me.boot.secure = false; + me.mountPersistence = true; + boot.loader.timeout = lib.mkForce 0; # We can always generate a new ISO if we need to access other boot options. + + me.optimizations = { + enable = true; + arch = "znver4"; + # build_arch = "x86-64-v3"; + system_features = [ + "gccarch-znver4" + "gccarch-skylake" + "gccarch-kabylake" + # "gccarch-alderlake" missing WAITPKG + "gccarch-x86-64-v3" + "gccarch-x86-64-v4" + "benchmark" + "big-parallel" + "kvm" + "nixos-test" + ]; + }; + + # Mount tmpfs at /tmp + boot.tmp.useTmpfs = true; + + # Enable TRIM + # services.fstrim.enable = lib.mkDefault true; + + # nix.optimise.automatic = true; + # nix.optimise.dates = [ "03:45" ]; + # nix.optimise.persistent = true; + + environment.systemPackages = with pkgs; [ + htop + ]; + + # nix.sshServe.enable = true; + # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ]; + + me.etcd.cluster_name = "put-a-nix-on-it"; + me.etcd.internal_ip = [ + # "10.215.1.221" + "[2620:11f:7001:7:ffff:ffff:0ad7:01df]" + ]; + me.etcd.initial_cluster = [ + # "controller0=https://10.215.1.221:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01dd + # "controller1=https://10.215.1.222:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01de + # "controller2=https://10.215.1.223:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01df + "controller0=https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2380" # 10.215.1.221 + "controller1=https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2380" # 10.215.1.222 + "controller2=https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2380" # 10.215.1.223 + ]; + + me.kube_apiserver.internal_ip = "2620:11f:7001:7:ffff:ffff:0ad7:01df"; + # me.kube_apiserver.external_ip = "74.80.180.138"; + me.kube_apiserver.external_ip = "2620:11f:7001:7:ffff:ffff:0ad7:01df"; + me.kube_apiserver.etcd_services = [ + "https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2379" # 10.215.1.221 + "https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2379" # 10.215.1.222 + "https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2379" # 10.215.1.223 + ]; + + me.control_plane.enable = true; + me.dont_use_substituters.enable = true; + me.etcd.enable = true; + me.minimal_base.enable = true; + }; +} diff --git a/nix/kubernetes/hosts/controller2/hardware-configuration.nix b/nix/kubernetes/hosts/controller2/hardware-configuration.nix new file mode 100644 index 0000000..6029e08 --- /dev/null +++ b/nix/kubernetes/hosts/controller2/hardware-configuration.nix @@ -0,0 +1,31 @@ +{ + config, + lib, + modulesPath, + ... +}: + +{ + imports = [ + (modulesPath + "/installer/scan/not-detected.nix") + ]; + + config = { + boot.initrd.availableKernelModules = [ + "nvme" + "xhci_pci" + "thunderbolt" + ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ ]; + boot.extraModulePackages = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + # networking.useDHCP = lib.mkDefault true; + # networking.interfaces.eno1.useDHCP = lib.mkDefault true; + # networking.interfaces.wlp58s0.useDHCP = lib.mkDefault true; + }; +} diff --git a/nix/kubernetes/hosts/controller2/vm_disk.nix b/nix/kubernetes/hosts/controller2/vm_disk.nix new file mode 100644 index 0000000..83683f8 --- /dev/null +++ b/nix/kubernetes/hosts/controller2/vm_disk.nix @@ -0,0 +1,94 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + config = { + # Mount the local disk + fileSystems = lib.mkIf config.me.mountPersistence { + "/.disk" = lib.mkForce { + device = "/dev/nvme0n1p1"; + fsType = "ext4"; + options = [ + "noatime" + "discard" + ]; + neededForBoot = true; + }; + + "/.persist" = lib.mkForce { + device = "bind9p"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + + "/persist" = { + fsType = "none"; + device = "/.persist/persist"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.persist/persist" + ]; + neededForBoot = true; + }; + + "/state" = { + fsType = "none"; + device = "/.persist/state"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.persist/state" + ]; + neededForBoot = true; + }; + + "/k8spv" = lib.mkForce { + device = "k8spv"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + + "/disk" = { + fsType = "none"; + device = "/.disk/persist"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.disk/persist" + ]; + neededForBoot = true; + }; + }; + }; +} diff --git a/nix/kubernetes/hosts/worker0/DEPLOY_BOOT b/nix/kubernetes/hosts/worker0/DEPLOY_BOOT new file mode 100755 index 0000000..d716872 --- /dev/null +++ b/nix/kubernetes/hosts/worker0/DEPLOY_BOOT @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=worker0 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild boot --flake "$DIR/../../#worker0" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker0/DEPLOY_SWITCH b/nix/kubernetes/hosts/worker0/DEPLOY_SWITCH new file mode 100755 index 0000000..a47bf6a --- /dev/null +++ b/nix/kubernetes/hosts/worker0/DEPLOY_SWITCH @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=worker0 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild switch --flake "$DIR/../../#worker0" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker0/ISO b/nix/kubernetes/hosts/worker0/ISO new file mode 100755 index 0000000..46caa98 --- /dev/null +++ b/nix/kubernetes/hosts/worker0/ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#worker0.iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker0/SELF_BOOT b/nix/kubernetes/hosts/worker0/SELF_BOOT new file mode 100755 index 0000000..35e6059 --- /dev/null +++ b/nix/kubernetes/hosts/worker0/SELF_BOOT @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild boot --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker0" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker0/SELF_BUILD b/nix/kubernetes/hosts/worker0/SELF_BUILD new file mode 100755 index 0000000..69fbd95 --- /dev/null +++ b/nix/kubernetes/hosts/worker0/SELF_BUILD @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild build --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker0" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker0/SELF_SWITCH b/nix/kubernetes/hosts/worker0/SELF_SWITCH new file mode 100755 index 0000000..b127a67 --- /dev/null +++ b/nix/kubernetes/hosts/worker0/SELF_SWITCH @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild switch --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker0" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker0/VM_ISO b/nix/kubernetes/hosts/worker0/VM_ISO new file mode 100755 index 0000000..3e518fc --- /dev/null +++ b/nix/kubernetes/hosts/worker0/VM_ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#worker0.vm_iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker0/default.nix b/nix/kubernetes/hosts/worker0/default.nix new file mode 100644 index 0000000..6e4570f --- /dev/null +++ b/nix/kubernetes/hosts/worker0/default.nix @@ -0,0 +1,106 @@ +# MANUAL: On client machines generate signing keys: +# nix-store --generate-binary-cache-key some-name /persist/manual/nix/nix-cache-key.sec /persist/manual/nix/nix-cache-key.pub +# +# Trust other machines and add the substituters: +# nix.binaryCachePublicKeys = [ "some-name:AzNW1MOlkNEsUAXS1jIFZ1QCFKXjV+Y/LrF37quAZ1A=" ]; +# nix.binaryCaches = [ "https://test.example/nix-cache" ]; + +{ + config, + lib, + pkgs, + ... +}: +{ + imports = [ + ./hardware-configuration.nix + ./vm_disk.nix + ]; + + config = { + networking = + let + interface = "enp0s2"; + in + { + # Generate with `head -c4 /dev/urandom | od -A none -t x4` + hostId = "0aadbb10"; + + hostName = "worker0"; # Define your hostname. + + interfaces = { + "${interface}" = { + ipv4.addresses = [ + { + address = "10.215.1.224"; + prefixLength = 24; + } + ]; + + ipv6.addresses = [ + { + address = "2620:11f:7001:7:ffff:ffff:0ad7:01e0"; + prefixLength = 64; + } + ]; + }; + }; + defaultGateway = "10.215.1.1"; + defaultGateway6 = { + # address = "2620:11f:7001:7::1"; + address = "2620:11f:7001:7:ffff:ffff:0ad7:0101"; + inherit interface; + }; + + dhcpcd.enable = lib.mkForce false; + useDHCP = lib.mkForce false; + }; + + time.timeZone = "America/New_York"; + i18n.defaultLocale = "en_US.UTF-8"; + + me.boot.enable = true; + me.boot.secure = false; + me.mountPersistence = true; + boot.loader.timeout = lib.mkForce 0; # We can always generate a new ISO if we need to access other boot options. + + me.optimizations = { + enable = true; + arch = "znver4"; + # build_arch = "x86-64-v3"; + system_features = [ + "gccarch-znver4" + "gccarch-skylake" + "gccarch-kabylake" + # "gccarch-alderlake" missing WAITPKG + "gccarch-x86-64-v3" + "gccarch-x86-64-v4" + "benchmark" + "big-parallel" + "kvm" + "nixos-test" + ]; + }; + + # Mount tmpfs at /tmp + boot.tmp.useTmpfs = true; + + # Enable TRIM + # services.fstrim.enable = lib.mkDefault true; + + # nix.optimise.automatic = true; + # nix.optimise.dates = [ "03:45" ]; + # nix.optimise.persistent = true; + + environment.systemPackages = with pkgs; [ + htop + ]; + + # nix.sshServe.enable = true; + # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ]; + + me.dont_use_substituters.enable = true; + me.minimal_base.enable = true; + me.worker_node.enable = true; + }; +} diff --git a/nix/kubernetes/hosts/worker0/hardware-configuration.nix b/nix/kubernetes/hosts/worker0/hardware-configuration.nix new file mode 100644 index 0000000..6029e08 --- /dev/null +++ b/nix/kubernetes/hosts/worker0/hardware-configuration.nix @@ -0,0 +1,31 @@ +{ + config, + lib, + modulesPath, + ... +}: + +{ + imports = [ + (modulesPath + "/installer/scan/not-detected.nix") + ]; + + config = { + boot.initrd.availableKernelModules = [ + "nvme" + "xhci_pci" + "thunderbolt" + ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ ]; + boot.extraModulePackages = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + # networking.useDHCP = lib.mkDefault true; + # networking.interfaces.eno1.useDHCP = lib.mkDefault true; + # networking.interfaces.wlp58s0.useDHCP = lib.mkDefault true; + }; +} diff --git a/nix/kubernetes/hosts/worker0/vm_disk.nix b/nix/kubernetes/hosts/worker0/vm_disk.nix new file mode 100644 index 0000000..83683f8 --- /dev/null +++ b/nix/kubernetes/hosts/worker0/vm_disk.nix @@ -0,0 +1,94 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + config = { + # Mount the local disk + fileSystems = lib.mkIf config.me.mountPersistence { + "/.disk" = lib.mkForce { + device = "/dev/nvme0n1p1"; + fsType = "ext4"; + options = [ + "noatime" + "discard" + ]; + neededForBoot = true; + }; + + "/.persist" = lib.mkForce { + device = "bind9p"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + + "/persist" = { + fsType = "none"; + device = "/.persist/persist"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.persist/persist" + ]; + neededForBoot = true; + }; + + "/state" = { + fsType = "none"; + device = "/.persist/state"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.persist/state" + ]; + neededForBoot = true; + }; + + "/k8spv" = lib.mkForce { + device = "k8spv"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + + "/disk" = { + fsType = "none"; + device = "/.disk/persist"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.disk/persist" + ]; + neededForBoot = true; + }; + }; + }; +} diff --git a/nix/kubernetes/hosts/worker1/DEPLOY_BOOT b/nix/kubernetes/hosts/worker1/DEPLOY_BOOT new file mode 100755 index 0000000..631d46d --- /dev/null +++ b/nix/kubernetes/hosts/worker1/DEPLOY_BOOT @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=worker1 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild boot --flake "$DIR/../../#worker1" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker1/DEPLOY_SWITCH b/nix/kubernetes/hosts/worker1/DEPLOY_SWITCH new file mode 100755 index 0000000..271829c --- /dev/null +++ b/nix/kubernetes/hosts/worker1/DEPLOY_SWITCH @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=worker1 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild switch --flake "$DIR/../../#worker1" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker1/ISO b/nix/kubernetes/hosts/worker1/ISO new file mode 100755 index 0000000..4c2b969 --- /dev/null +++ b/nix/kubernetes/hosts/worker1/ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#worker1.iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker1/SELF_BOOT b/nix/kubernetes/hosts/worker1/SELF_BOOT new file mode 100755 index 0000000..cf928cd --- /dev/null +++ b/nix/kubernetes/hosts/worker1/SELF_BOOT @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild boot --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker1" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker1/SELF_BUILD b/nix/kubernetes/hosts/worker1/SELF_BUILD new file mode 100755 index 0000000..73c846e --- /dev/null +++ b/nix/kubernetes/hosts/worker1/SELF_BUILD @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild build --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker1" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker1/SELF_SWITCH b/nix/kubernetes/hosts/worker1/SELF_SWITCH new file mode 100755 index 0000000..d6a989d --- /dev/null +++ b/nix/kubernetes/hosts/worker1/SELF_SWITCH @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild switch --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker1" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker1/VM_ISO b/nix/kubernetes/hosts/worker1/VM_ISO new file mode 100755 index 0000000..ffab210 --- /dev/null +++ b/nix/kubernetes/hosts/worker1/VM_ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#worker1.vm_iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker1/default.nix b/nix/kubernetes/hosts/worker1/default.nix new file mode 100644 index 0000000..bb56099 --- /dev/null +++ b/nix/kubernetes/hosts/worker1/default.nix @@ -0,0 +1,106 @@ +# MANUAL: On client machines generate signing keys: +# nix-store --generate-binary-cache-key some-name /persist/manual/nix/nix-cache-key.sec /persist/manual/nix/nix-cache-key.pub +# +# Trust other machines and add the substituters: +# nix.binaryCachePublicKeys = [ "some-name:AzNW1MOlkNEsUAXS1jIFZ1QCFKXjV+Y/LrF37quAZ1A=" ]; +# nix.binaryCaches = [ "https://test.example/nix-cache" ]; + +{ + config, + lib, + pkgs, + ... +}: +{ + imports = [ + ./hardware-configuration.nix + ./vm_disk.nix + ]; + + config = { + networking = + let + interface = "enp0s2"; + in + { + # Generate with `head -c4 /dev/urandom | od -A none -t x4` + hostId = "4324346d"; + + hostName = "worker1"; # Define your hostname. + + interfaces = { + "${interface}" = { + ipv4.addresses = [ + { + address = "10.215.1.225"; + prefixLength = 24; + } + ]; + + ipv6.addresses = [ + { + address = "2620:11f:7001:7:ffff:ffff:0ad7:01e1"; + prefixLength = 64; + } + ]; + }; + }; + defaultGateway = "10.215.1.1"; + defaultGateway6 = { + # address = "2620:11f:7001:7::1"; + address = "2620:11f:7001:7:ffff:ffff:0ad7:0101"; + inherit interface; + }; + + dhcpcd.enable = lib.mkForce false; + useDHCP = lib.mkForce false; + }; + + time.timeZone = "America/New_York"; + i18n.defaultLocale = "en_US.UTF-8"; + + me.boot.enable = true; + me.boot.secure = false; + me.mountPersistence = true; + boot.loader.timeout = lib.mkForce 0; # We can always generate a new ISO if we need to access other boot options. + + me.optimizations = { + enable = true; + arch = "znver4"; + # build_arch = "x86-64-v3"; + system_features = [ + "gccarch-znver4" + "gccarch-skylake" + "gccarch-kabylake" + # "gccarch-alderlake" missing WAITPKG + "gccarch-x86-64-v3" + "gccarch-x86-64-v4" + "benchmark" + "big-parallel" + "kvm" + "nixos-test" + ]; + }; + + # Mount tmpfs at /tmp + boot.tmp.useTmpfs = true; + + # Enable TRIM + # services.fstrim.enable = lib.mkDefault true; + + # nix.optimise.automatic = true; + # nix.optimise.dates = [ "03:45" ]; + # nix.optimise.persistent = true; + + environment.systemPackages = with pkgs; [ + htop + ]; + + # nix.sshServe.enable = true; + # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ]; + + me.worker_node.enable = true; + me.dont_use_substituters.enable = true; + me.minimal_base.enable = true; + }; +} diff --git a/nix/kubernetes/hosts/worker1/hardware-configuration.nix b/nix/kubernetes/hosts/worker1/hardware-configuration.nix new file mode 100644 index 0000000..6029e08 --- /dev/null +++ b/nix/kubernetes/hosts/worker1/hardware-configuration.nix @@ -0,0 +1,31 @@ +{ + config, + lib, + modulesPath, + ... +}: + +{ + imports = [ + (modulesPath + "/installer/scan/not-detected.nix") + ]; + + config = { + boot.initrd.availableKernelModules = [ + "nvme" + "xhci_pci" + "thunderbolt" + ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ ]; + boot.extraModulePackages = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + # networking.useDHCP = lib.mkDefault true; + # networking.interfaces.eno1.useDHCP = lib.mkDefault true; + # networking.interfaces.wlp58s0.useDHCP = lib.mkDefault true; + }; +} diff --git a/nix/kubernetes/hosts/worker1/vm_disk.nix b/nix/kubernetes/hosts/worker1/vm_disk.nix new file mode 100644 index 0000000..83683f8 --- /dev/null +++ b/nix/kubernetes/hosts/worker1/vm_disk.nix @@ -0,0 +1,94 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + config = { + # Mount the local disk + fileSystems = lib.mkIf config.me.mountPersistence { + "/.disk" = lib.mkForce { + device = "/dev/nvme0n1p1"; + fsType = "ext4"; + options = [ + "noatime" + "discard" + ]; + neededForBoot = true; + }; + + "/.persist" = lib.mkForce { + device = "bind9p"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + + "/persist" = { + fsType = "none"; + device = "/.persist/persist"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.persist/persist" + ]; + neededForBoot = true; + }; + + "/state" = { + fsType = "none"; + device = "/.persist/state"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.persist/state" + ]; + neededForBoot = true; + }; + + "/k8spv" = lib.mkForce { + device = "k8spv"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + + "/disk" = { + fsType = "none"; + device = "/.disk/persist"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.disk/persist" + ]; + neededForBoot = true; + }; + }; + }; +} diff --git a/nix/kubernetes/hosts/worker2/DEPLOY_BOOT b/nix/kubernetes/hosts/worker2/DEPLOY_BOOT new file mode 100755 index 0000000..eb5db11 --- /dev/null +++ b/nix/kubernetes/hosts/worker2/DEPLOY_BOOT @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=worker2 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild boot --flake "$DIR/../../#worker2" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker2/DEPLOY_SWITCH b/nix/kubernetes/hosts/worker2/DEPLOY_SWITCH new file mode 100755 index 0000000..c321328 --- /dev/null +++ b/nix/kubernetes/hosts/worker2/DEPLOY_SWITCH @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=worker2 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild switch --flake "$DIR/../../#worker2" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker2/ISO b/nix/kubernetes/hosts/worker2/ISO new file mode 100755 index 0000000..ad8a821 --- /dev/null +++ b/nix/kubernetes/hosts/worker2/ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#worker2.iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker2/SELF_BOOT b/nix/kubernetes/hosts/worker2/SELF_BOOT new file mode 100755 index 0000000..bdfde92 --- /dev/null +++ b/nix/kubernetes/hosts/worker2/SELF_BOOT @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild boot --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker2" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker2/SELF_BUILD b/nix/kubernetes/hosts/worker2/SELF_BUILD new file mode 100755 index 0000000..36cc5b5 --- /dev/null +++ b/nix/kubernetes/hosts/worker2/SELF_BUILD @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild build --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker2" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker2/SELF_SWITCH b/nix/kubernetes/hosts/worker2/SELF_SWITCH new file mode 100755 index 0000000..7af1d15 --- /dev/null +++ b/nix/kubernetes/hosts/worker2/SELF_SWITCH @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild switch --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker2" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker2/VM_ISO b/nix/kubernetes/hosts/worker2/VM_ISO new file mode 100755 index 0000000..7f1f203 --- /dev/null +++ b/nix/kubernetes/hosts/worker2/VM_ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#worker2.vm_iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker2/default.nix b/nix/kubernetes/hosts/worker2/default.nix new file mode 100644 index 0000000..ca83144 --- /dev/null +++ b/nix/kubernetes/hosts/worker2/default.nix @@ -0,0 +1,106 @@ +# MANUAL: On client machines generate signing keys: +# nix-store --generate-binary-cache-key some-name /persist/manual/nix/nix-cache-key.sec /persist/manual/nix/nix-cache-key.pub +# +# Trust other machines and add the substituters: +# nix.binaryCachePublicKeys = [ "some-name:AzNW1MOlkNEsUAXS1jIFZ1QCFKXjV+Y/LrF37quAZ1A=" ]; +# nix.binaryCaches = [ "https://test.example/nix-cache" ]; + +{ + config, + lib, + pkgs, + ... +}: +{ + imports = [ + ./hardware-configuration.nix + ./vm_disk.nix + ]; + + config = { + networking = + let + interface = "enp0s2"; + in + { + # Generate with `head -c4 /dev/urandom | od -A none -t x4` + hostId = "ce017961"; + + hostName = "worker2"; # Define your hostname. + + interfaces = { + "${interface}" = { + ipv4.addresses = [ + { + address = "10.215.1.226"; + prefixLength = 24; + } + ]; + + ipv6.addresses = [ + { + address = "2620:11f:7001:7:ffff:ffff:0ad7:01e2"; + prefixLength = 64; + } + ]; + }; + }; + defaultGateway = "10.215.1.1"; + defaultGateway6 = { + # address = "2620:11f:7001:7::1"; + address = "2620:11f:7001:7:ffff:ffff:0ad7:0101"; + inherit interface; + }; + + dhcpcd.enable = lib.mkForce false; + useDHCP = lib.mkForce false; + }; + + time.timeZone = "America/New_York"; + i18n.defaultLocale = "en_US.UTF-8"; + + me.boot.enable = true; + me.boot.secure = false; + me.mountPersistence = true; + boot.loader.timeout = lib.mkForce 0; # We can always generate a new ISO if we need to access other boot options. + + me.optimizations = { + enable = true; + arch = "znver4"; + # build_arch = "x86-64-v3"; + system_features = [ + "gccarch-znver4" + "gccarch-skylake" + "gccarch-kabylake" + # "gccarch-alderlake" missing WAITPKG + "gccarch-x86-64-v3" + "gccarch-x86-64-v4" + "benchmark" + "big-parallel" + "kvm" + "nixos-test" + ]; + }; + + # Mount tmpfs at /tmp + boot.tmp.useTmpfs = true; + + # Enable TRIM + # services.fstrim.enable = lib.mkDefault true; + + # nix.optimise.automatic = true; + # nix.optimise.dates = [ "03:45" ]; + # nix.optimise.persistent = true; + + environment.systemPackages = with pkgs; [ + htop + ]; + + # nix.sshServe.enable = true; + # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ]; + + me.worker_node.enable = true; + me.dont_use_substituters.enable = true; + me.minimal_base.enable = true; + }; +} diff --git a/nix/kubernetes/hosts/worker2/hardware-configuration.nix b/nix/kubernetes/hosts/worker2/hardware-configuration.nix new file mode 100644 index 0000000..6029e08 --- /dev/null +++ b/nix/kubernetes/hosts/worker2/hardware-configuration.nix @@ -0,0 +1,31 @@ +{ + config, + lib, + modulesPath, + ... +}: + +{ + imports = [ + (modulesPath + "/installer/scan/not-detected.nix") + ]; + + config = { + boot.initrd.availableKernelModules = [ + "nvme" + "xhci_pci" + "thunderbolt" + ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ ]; + boot.extraModulePackages = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + # networking.useDHCP = lib.mkDefault true; + # networking.interfaces.eno1.useDHCP = lib.mkDefault true; + # networking.interfaces.wlp58s0.useDHCP = lib.mkDefault true; + }; +} diff --git a/nix/kubernetes/hosts/worker2/vm_disk.nix b/nix/kubernetes/hosts/worker2/vm_disk.nix new file mode 100644 index 0000000..83683f8 --- /dev/null +++ b/nix/kubernetes/hosts/worker2/vm_disk.nix @@ -0,0 +1,94 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + config = { + # Mount the local disk + fileSystems = lib.mkIf config.me.mountPersistence { + "/.disk" = lib.mkForce { + device = "/dev/nvme0n1p1"; + fsType = "ext4"; + options = [ + "noatime" + "discard" + ]; + neededForBoot = true; + }; + + "/.persist" = lib.mkForce { + device = "bind9p"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + + "/persist" = { + fsType = "none"; + device = "/.persist/persist"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.persist/persist" + ]; + neededForBoot = true; + }; + + "/state" = { + fsType = "none"; + device = "/.persist/state"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.persist/state" + ]; + neededForBoot = true; + }; + + "/k8spv" = lib.mkForce { + device = "k8spv"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + + "/disk" = { + fsType = "none"; + device = "/.disk/persist"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.disk/persist" + ]; + neededForBoot = true; + }; + }; + }; +} diff --git a/nix/kubernetes/keys/Makefile b/nix/kubernetes/keys/Makefile new file mode 100644 index 0000000..24020a7 --- /dev/null +++ b/nix/kubernetes/keys/Makefile @@ -0,0 +1,29 @@ +SHELL := bash +.ONESHELL: +.SHELLFLAGS := -eu -o pipefail -c +.DELETE_ON_ERROR: +MAKEFLAGS += --warn-undefined-variables +MAKEFLAGS += --no-builtin-rules +OUT=generated + +ifeq ($(origin .RECIPEPREFIX), undefined) + $(error This Make does not support .RECIPEPREFIX. Please use GNU Make 4.0 or later) +endif +.RECIPEPREFIX = > + +KUBERNETES_PUBLIC_ADDRESS := 74.80.180.138 +WORKERS := worker0 worker1 worker2 controller0 controller1 controller2 + +.PHONY: all +all: \ + $(OUT)/known_hosts + +.PHONY: clean +clean: +> rm -rf $(OUT) + +$(OUT)/: +> @mkdir -p $(@D) + +$(OUT)/known_hosts: | $(OUT)/ +> ssh-keyscan -p 65099 74.80.180.138 | sed 's/\[74.80.180.138\]:65099/\[10.215.1.210\]:22/g' > $@ diff --git a/nix/kubernetes/keys/contrib/base64/package.nix b/nix/kubernetes/keys/contrib/base64/package.nix new file mode 100644 index 0000000..50e7a4b --- /dev/null +++ b/nix/kubernetes/keys/contrib/base64/package.nix @@ -0,0 +1,62 @@ +# From: https://gist.github.com/manveru/74eb41d850bc146b7e78c4cb059507e2 +# From: https://discourse.nixos.org/t/string-to-base-64/32624/3 +{ lib, ... }: +{ + toBase64 = + text: + let + inherit (lib) + sublist + mod + stringToCharacters + concatMapStrings + ; + inherit (lib.strings) charToInt; + inherit (builtins) + substring + foldl' + genList + elemAt + length + concatStringsSep + stringLength + ; + lookup = stringToCharacters "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + sliceN = + size: list: n: + sublist (n * size) size list; + pows = [ + (64 * 64 * 64) + (64 * 64) + 64 + 1 + ]; + intSextets = i: map (j: mod (i / j) 64) pows; + compose = + f: g: x: + f (g x); + intToChar = elemAt lookup; + convertTripletInt = sliceInt: concatMapStrings intToChar (intSextets sliceInt); + sliceToInt = foldl' (acc: val: acc * 256 + val) 0; + convertTriplet = compose convertTripletInt sliceToInt; + join = concatStringsSep ""; + convertLastSlice = + slice: + let + len = length slice; + in + if len == 1 then + (substring 0 2 (convertTripletInt ((sliceToInt slice) * 256 * 256))) + "==" + else if len == 2 then + (substring 0 3 (convertTripletInt ((sliceToInt slice) * 256))) + "=" + else + ""; + len = stringLength text; + nFullSlices = len / 3; + bytes = map charToInt (stringToCharacters text); + tripletAt = sliceN 3 bytes; + head = genList (compose convertTriplet tripletAt) nFullSlices; + tail = convertLastSlice (tripletAt nFullSlices); + in + join (head ++ [ tail ]); +} diff --git a/nix/kubernetes/keys/flake.lock b/nix/kubernetes/keys/flake.lock new file mode 100644 index 0000000..a6e534e --- /dev/null +++ b/nix/kubernetes/keys/flake.lock @@ -0,0 +1,27 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1772773019, + "narHash": "sha256-E1bxHxNKfDoQUuvriG71+f+s/NT0qWkImXsYZNFFfCs=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "aca4d95fce4914b3892661bcb80b8087293536c6", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/nix/kubernetes/keys/flake.nix b/nix/kubernetes/keys/flake.nix new file mode 100644 index 0000000..851cd71 --- /dev/null +++ b/nix/kubernetes/keys/flake.nix @@ -0,0 +1,56 @@ +{ + description = "Build keys to manually deploy to kubernetes cluster."; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + }; + + outputs = + { self, nixpkgs }: + let + forAllSystems = nixpkgs.lib.genAttrs nixpkgs.lib.systems.flakeExposed; + in + { + packages = forAllSystems ( + system: + let + pkgs = import nixpkgs { + inherit system; + overlays = [ self.overlays.default ]; + }; + in + { + deploy_script = pkgs.k8s.deploy_script; + default = pkgs.k8s.all_keys; + bootstrap_script = pkgs.k8s.bootstrap_script; + mrmanager_repo_secrets = pkgs.k8s.mrmanager_repo_secrets; + } + ); + overlays.default = ( + final: prev: { + k8s = (final.callPackage ./scope.nix { inherit (final.lib) makeScope; }); + } + ); + + } + // { + devShells = forAllSystems ( + system: + let + pkgs = nixpkgs.legacyPackages.${system}; + in + { + default = pkgs.mkShell { + nativeBuildInputs = with pkgs; [ + pkg-config + ]; + buildInputs = with pkgs; [ + kubernetes-helm # To generate cilium manifests + fluxcd # To generate flux manifests + cilium-cli # To check cilium status + ]; + }; + } + ); + }; +} diff --git a/nix/kubernetes/keys/generated/known_hosts b/nix/kubernetes/keys/generated/known_hosts new file mode 100644 index 0000000..3406283 --- /dev/null +++ b/nix/kubernetes/keys/generated/known_hosts @@ -0,0 +1,8 @@ +# 74.80.180.138:65099 SSH-2.0-OpenSSH_9.3 FreeBSD-20230316 +[10.215.1.210]:22 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC0hWY7Ighnlp3UfPfApyW9nEGG11f+on/kOkp6YdxTTVX0jvi00xvrZ8c23l48YDptmEKOMj7avUR+jdpRNaSwbw3Lm7swg+EpFZ73tnHK+r6HnOnNu8ECDvYOW10eI6vdRctFisRfyIKigmtmquxXYLhQDSA2INVW+Vuebdwa74VqKLLirUu7e3ymp8dH8ktcCAjWSd/+Ax7E+4AMa5WHFeTPBheA2GhfLhINDLpgdZ8WNZ4i3ow8MrQADiOVYUDPrXvI55MVWSQTQQcOco184Z67rtcCtqY/fcCp+38yzUT0Bm2syXM+HNOlFqM+fJBf0T9kiiy5XvWuN9bY+368JGOUUM6RsCUgERHSaU65nX3i8oIcNRt3w6sVsmRR8sX8x5qFjyEYuElIwKywcdtKpoklV6gu+lo+mIE8i95jJmXMj6lk3G83wMZICL9+dm+b8ckpRZEi6970EqahiPO3cV/Fa88gysf9HwiC8AxSc3m2BcOvaV3jadaT39Tymp8= +# 74.80.180.138:65099 SSH-2.0-OpenSSH_9.3 FreeBSD-20230316 +# 74.80.180.138:65099 SSH-2.0-OpenSSH_9.3 FreeBSD-20230316 +[10.215.1.210]:22 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBH2euFJKLEDfTV9NTecrOoqL9FpiYvTbNp/Ty3FebJA5DKmVd1xBRz3sNs1R1ayn213vmRVLWSu2ikulbl65LLQ= +# 74.80.180.138:65099 SSH-2.0-OpenSSH_9.3 FreeBSD-20230316 +[10.215.1.210]:22 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM1qjGgD2UdD5Lc+zGFxHX/+h6FBNmGW+O30LG0tiHvC +# 74.80.180.138:65099 SSH-2.0-OpenSSH_9.3 FreeBSD-20230316 diff --git a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux.yaml b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux.yaml new file mode 100644 index 0000000..e724deb --- /dev/null +++ b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux.yaml @@ -0,0 +1,2072 @@ +--- +# Source: flux-operator/templates/networkpolicy.yaml +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: flux-operator-web + namespace: flux-system + labels: + helm.sh/chart: flux-operator-0.48.0 + app.kubernetes.io/name: flux-operator + app.kubernetes.io/instance: flux-operator + app.kubernetes.io/version: "v0.48.0" + app.kubernetes.io/managed-by: Helm +spec: + policyTypes: + - Ingress + podSelector: + matchLabels: + app.kubernetes.io/name: flux-operator + app.kubernetes.io/instance: flux-operator + ingress: + - from: + - namespaceSelector: {} + ports: + - protocol: TCP + port: 9080 +--- +# Source: flux-operator/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flux-operator + namespace: flux-system + labels: + helm.sh/chart: flux-operator-0.48.0 + app.kubernetes.io/name: flux-operator + app.kubernetes.io/instance: flux-operator + app.kubernetes.io/version: "v0.48.0" + app.kubernetes.io/managed-by: Helm +automountServiceAccountToken: true +--- +# Source: flux-operator/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.0 + helm.sh/resource-policy: keep + labels: + app.kubernetes.io/instance: 'flux-operator' + app.kubernetes.io/managed-by: 'Helm' + app.kubernetes.io/name: 'flux-operator' + app.kubernetes.io/version: 'v0.48.0' + helm.sh/chart: 'flux-operator-0.48.0' + name: fluxinstances.fluxcd.controlplane.io +spec: + group: fluxcd.controlplane.io + names: + kind: FluxInstance + listKind: FluxInstanceList + plural: fluxinstances + singular: fluxinstance + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + - jsonPath: .status.lastAttemptedRevision + name: Revision + type: string + name: v1 + schema: + openAPIV3Schema: + description: FluxInstance is the Schema for the fluxinstances API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: FluxInstanceSpec defines the desired state of FluxInstance + properties: + cluster: + description: Cluster holds the specification of the Kubernetes cluster. + properties: + domain: + default: cluster.local + description: |- + Domain is the cluster domain used for generating the FQDN of services. + Defaults to 'cluster.local'. + type: string + multitenant: + default: false + description: Multitenant enables the multitenancy lockdown. Defaults + to false. + type: boolean + multitenantWorkloadIdentity: + default: false + description: |- + MultitenantWorkloadIdentity enables the multitenancy lockdown for + workload identity. Defaults to false. + type: boolean + networkPolicy: + default: true + description: |- + NetworkPolicy restricts network access to the current namespace. + Defaults to true. + type: boolean + objectLevelWorkloadIdentity: + description: |- + ObjectLevelWorkloadIdentity enables the feature gate + required for object-level workload identity. + This feature is only available in Flux v2.6.0 and later. + type: boolean + size: + description: |- + Size defines the vertical scaling profile of the Flux controllers. + The size is used to determine the concurrency and CPU/Memory limits for the Flux controllers. + Accepted values are: 'small', 'medium' and 'large'. + enum: + - small + - medium + - large + type: string + tenantDefaultDecryptionServiceAccount: + description: |- + TenantDefaultDecryptionServiceAccount is the name of the service account + to use as default for kustomize-controller SOPS decryption when the + multitenant lockdown for workload identity is enabled. Defaults to the + 'default' service account from the tenant namespace. + type: string + tenantDefaultKubeConfigServiceAccount: + description: |- + TenantDefaultKubeConfigServiceAccount is the name of the service account + to use as default for kustomize-controller and helm-controller remote + cluster access via spec.kubeConfig.configMapRef when the multitenant + lockdown for workload identity is enabled. Defaults to the 'default' + service account from the tenant namespace. + type: string + tenantDefaultServiceAccount: + description: |- + TenantDefaultServiceAccount is the name of the service account + to use as default when the multitenant lockdown is enabled, for + kustomize-controller and helm-controller. + This field will also be used for multitenant workload identity + lockdown for source-controller, notification-controller, + image-reflector-controller and image-automation-controller. + Defaults to the 'default' service account from the tenant namespace. + type: string + type: + default: kubernetes + description: |- + Type specifies the distro of the Kubernetes cluster. + Defaults to 'kubernetes'. + enum: + - kubernetes + - openshift + - aws + - azure + - gcp + type: string + type: object + x-kubernetes-validations: + - message: .objectLevelWorkloadIdentity must be set to true when .multitenantWorkloadIdentity + is set to true + rule: (has(self.objectLevelWorkloadIdentity) && self.objectLevelWorkloadIdentity) + || !has(self.multitenantWorkloadIdentity) || !self.multitenantWorkloadIdentity + commonMetadata: + description: |- + CommonMetadata specifies the common labels and annotations that are + applied to all resources. Any existing label or annotation will be + overridden if its key matches a common one. + properties: + annotations: + additionalProperties: + type: string + description: Annotations to be added to the object's metadata. + type: object + labels: + additionalProperties: + type: string + description: Labels to be added to the object's metadata. + type: object + type: object + components: + description: |- + Components is the list of controllers to install. + Defaults to the core Flux controllers: + - source-controller + - kustomize-controller + - helm-controller + - notification-controller + items: + description: Component is the name of a controller to install. + enum: + - source-controller + - kustomize-controller + - helm-controller + - notification-controller + - image-reflector-controller + - image-automation-controller + - source-watcher + type: string + type: array + distribution: + description: Distribution specifies the version and container registry + to pull images from. + properties: + artifact: + description: |- + Artifact is the URL to the OCI artifact containing + the latest Kubernetes manifests for the distribution, + e.g. 'oci://ghcr.io/controlplaneio-fluxcd/flux-operator-manifests:latest'. + pattern: ^oci://.*$ + type: string + artifactPullSecret: + description: |- + ArtifactPullSecret is the name of the Kubernetes secret + to use for pulling the Kubernetes manifests for the distribution specified in the Artifact field. + type: string + imagePullSecret: + description: |- + ImagePullSecret is the name of the Kubernetes secret + to use for pulling images. + type: string + registry: + description: |- + Registry address to pull the distribution images from + e.g. 'ghcr.io/fluxcd'. + type: string + variant: + description: |- + Variant specifies the Flux distribution flavor stored + in the registry. + enum: + - upstream-alpine + - enterprise-alpine + - enterprise-distroless + - enterprise-distroless-fips + type: string + version: + description: Version semver expression e.g. '2.x', '2.3.x'. + type: string + required: + - registry + - version + type: object + kustomize: + description: |- + Kustomize holds a set of patches that can be applied to the + Flux installation, to customize the way Flux operates. + properties: + patches: + description: |- + Strategic merge and JSON patches, defined as inline YAML objects, + capable of targeting objects based on kind, label and annotation selectors. + items: + description: |- + Patch contains an inline StrategicMerge or JSON6902 patch, and the target the patch should + be applied to. + properties: + patch: + description: |- + Patch contains an inline StrategicMerge patch or an inline JSON6902 patch with + an array of operation objects. + type: string + target: + description: Target points to the resources that the patch + document should be applied to. + properties: + annotationSelector: + description: |- + AnnotationSelector is a string that follows the label selection expression + https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#api + It matches with the resource annotations. + type: string + group: + description: |- + Group is the API group to select resources from. + Together with Version and Kind it is capable of unambiguously identifying and/or selecting resources. + https://github.com/kubernetes/community/blob/master/contributors/design-proposals/api-machinery/api-group.md + type: string + kind: + description: |- + Kind of the API Group to select resources from. + Together with Group and Version it is capable of unambiguously + identifying and/or selecting resources. + https://github.com/kubernetes/community/blob/master/contributors/design-proposals/api-machinery/api-group.md + type: string + labelSelector: + description: |- + LabelSelector is a string that follows the label selection expression + https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#api + It matches with the resource labels. + type: string + name: + description: Name to match resources with. + type: string + namespace: + description: Namespace to select resources from. + type: string + version: + description: |- + Version of the API Group to select resources from. + Together with Group and Kind it is capable of unambiguously identifying and/or selecting resources. + https://github.com/kubernetes/community/blob/master/contributors/design-proposals/api-machinery/api-group.md + type: string + type: object + required: + - patch + type: object + type: array + type: object + migrateResources: + default: true + description: |- + MigrateResources instructs the controller to migrate the Flux custom resources + from the previous version to the latest API version specified in the CRD. + Defaults to true. + type: boolean + sharding: + description: Sharding holds the specification of the sharding configuration. + properties: + key: + default: sharding.fluxcd.io/key + description: Key is the label key used to shard the resources. + type: string + shards: + description: Shards is the list of shard names. + items: + type: string + minItems: 1 + type: array + storage: + description: |- + Storage defines if the source-controller shards + should use an emptyDir or a persistent volume claim for storage. + Accepted values are 'ephemeral' or 'persistent', defaults to 'ephemeral'. + For 'persistent' to take effect, the '.spec.storage' field must be set. + enum: + - ephemeral + - persistent + type: string + required: + - shards + type: object + storage: + description: |- + Storage holds the specification of the source-controller + persistent volume claim. + properties: + class: + description: Class is the storage class to use for the PVC. + type: string + size: + description: Size is the size of the PVC. + type: string + required: + - class + - size + type: object + sync: + description: |- + Sync specifies the source for the cluster sync operation. + When set, a Flux source (GitRepository, OCIRepository or Bucket) + and Flux Kustomization are created to sync the cluster state + with the source repository. + properties: + interval: + default: 1m + description: Interval is the time between syncs. + pattern: ^([0-9]+(\.[0-9]+)?(ms|s|m|h))+$ + type: string + kind: + description: Kind is the kind of the source. + enum: + - OCIRepository + - GitRepository + - Bucket + type: string + name: + description: |- + Name is the name of the Flux source and kustomization resources. + When not specified, the name is set to the namespace name of the FluxInstance. + maxLength: 63 + type: string + x-kubernetes-validations: + - message: Sync name is immutable + rule: self == oldSelf + path: + description: |- + Path is the path to the source directory containing + the kustomize overlay or plain Kubernetes manifests. + type: string + provider: + description: |- + Provider specifies OIDC provider for source authentication. + For OCIRepository and Bucket the provider can be set to 'aws', 'azure' or 'gcp'. + for GitRepository the accepted value can be set to 'azure' or 'github'. + To disable OIDC authentication the provider can be set to 'generic' or left empty. + enum: + - generic + - aws + - azure + - gcp + - github + type: string + pullSecret: + description: |- + PullSecret specifies the Kubernetes Secret containing the + authentication credentials for the source. + For Git over HTTP/S sources, the secret must contain username and password fields. + For Git over SSH sources, the secret must contain known_hosts and identity fields. + For OCI sources, the secret must be of type kubernetes.io/dockerconfigjson. + For Bucket sources, the secret must contain accesskey and secretkey fields. + type: string + ref: + description: |- + Ref is the source reference, can be a Git ref name e.g. 'refs/heads/main', + an OCI tag e.g. 'latest' or a bucket name e.g. 'flux'. + type: string + url: + description: |- + URL is the source URL, can be a Git repository HTTP/S or SSH address, + an OCI repository address or a Bucket endpoint. + type: string + required: + - kind + - path + - ref + - url + type: object + wait: + default: true + description: |- + Wait instructs the controller to check the health of all the reconciled + resources. Defaults to true. + type: boolean + required: + - distribution + type: object + status: + description: FluxInstanceStatus defines the observed state of FluxInstance + properties: + components: + description: Components contains the container images used by the + components. + items: + description: ComponentImage represents a container image used by + a component. + properties: + digest: + description: Digest of the container image. + type: string + name: + description: Name of the component. + type: string + repository: + description: Repository address of the container image. + type: string + tag: + description: Tag of the container image. + type: string + required: + - name + - repository + - tag + type: object + type: array + conditions: + description: Conditions contains the readiness conditions of the object. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + history: + description: |- + History contains the reconciliation history of the FluxInstance + as a list of snapshots ordered by the last reconciled time. + items: + description: |- + Snapshot represents a point-in-time record of a group of resources reconciliation, + including timing information, status, and a unique digest identifier. + properties: + digest: + description: Digest is the checksum in the format `:` + of the resources in this snapshot. + type: string + firstReconciled: + description: FirstReconciled is the time when this revision + was first reconciled to the cluster. + format: date-time + type: string + lastReconciled: + description: LastReconciled is the time when this revision was + last reconciled to the cluster. + format: date-time + type: string + lastReconciledDuration: + description: LastReconciledDuration is time it took to reconcile + the resources in this revision. + type: string + lastReconciledStatus: + description: LastReconciledStatus is the status of the last + reconciliation. + type: string + metadata: + additionalProperties: + type: string + description: Metadata contains additional information about + the snapshot. + type: object + totalReconciliations: + description: TotalReconciliations is the total number of reconciliations + that have occurred for this snapshot. + format: int64 + type: integer + required: + - digest + - firstReconciled + - lastReconciled + - lastReconciledDuration + - lastReconciledStatus + - totalReconciliations + type: object + type: array + inventory: + description: |- + Inventory contains a list of Kubernetes resource object references + last applied on the cluster. + properties: + entries: + description: Entries of Kubernetes resource object references. + items: + description: ResourceRef contains the information necessary + to locate a resource within a cluster. + properties: + id: + description: |- + ID is the string representation of the Kubernetes resource object's metadata, + in the format '___'. + type: string + v: + description: Version is the API version of the Kubernetes + resource object's kind. + type: string + required: + - id + - v + type: object + type: array + required: + - entries + type: object + lastAppliedRevision: + description: |- + LastAppliedRevision is the version and digest of the + distribution config that was last reconcile. + type: string + lastArtifactRevision: + description: |- + LastArtifactRevision is the digest of the last pulled + distribution artifact. + type: string + lastAttemptedRevision: + description: |- + LastAttemptedRevision is the version and digest of the + distribution config that was last attempted to reconcile. + type: string + lastHandledForceAt: + description: |- + LastHandledForceAt holds the value of the most recent + force request value, so a change of the annotation value + can be detected. + type: string + lastHandledReconcileAt: + description: |- + LastHandledReconcileAt holds the value of the most recent + reconcile request value, so a change of the annotation value + can be detected. + type: string + type: object + type: object + x-kubernetes-validations: + - message: the only accepted name for a FluxInstance is 'flux' + rule: self.metadata.name == 'flux' + served: true + storage: true + subresources: + status: {} +--- +# Source: flux-operator/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.0 + helm.sh/resource-policy: keep + labels: + app.kubernetes.io/instance: 'flux-operator' + app.kubernetes.io/managed-by: 'Helm' + app.kubernetes.io/name: 'flux-operator' + app.kubernetes.io/version: 'v0.48.0' + helm.sh/chart: 'flux-operator-0.48.0' + name: fluxreports.fluxcd.controlplane.io +spec: + group: fluxcd.controlplane.io + names: + kind: FluxReport + listKind: FluxReportList + plural: fluxreports + singular: fluxreport + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.distribution.entitlement + name: Entitlement + priority: 10 + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].lastTransitionTime + name: LastUpdated + type: string + name: v1 + schema: + openAPIV3Schema: + description: FluxReport is the Schema for the fluxreports API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: FluxReportSpec defines the observed state of a Flux installation. + properties: + cluster: + description: Cluster is the version information of the Kubernetes + cluster. + properties: + nodes: + description: Nodes is the number of nodes in the Kubernetes cluster. + type: integer + platform: + description: Platform is the os/arch of the Kubernetes control + plane. + type: string + serverVersion: + description: ServerVersion is the version of the Kubernetes API + server. + type: string + required: + - platform + - serverVersion + type: object + components: + description: ComponentsStatus is the status of the Flux controller + deployments. + items: + description: FluxComponentStatus defines the observed state of a + Flux component. + properties: + image: + description: Image is the container image of the Flux component. + type: string + name: + description: Name is the name of the Flux component. + type: string + ready: + description: Ready is the readiness status of the Flux component. + type: boolean + status: + description: |- + Status is a human-readable message indicating details + about the Flux component observed state. + type: string + required: + - image + - name + - ready + - status + type: object + type: array + distribution: + description: Distribution is the version information of the Flux installation. + properties: + entitlement: + description: Entitlement is the entitlement verification status. + type: string + managedBy: + description: ManagedBy is the name of the operator managing the + Flux instance. + type: string + status: + description: |- + Status is a human-readable message indicating details + about the distribution observed state. + type: string + version: + description: Version is the version of the Flux instance. + type: string + required: + - entitlement + - status + type: object + operator: + description: Operator is the version information of the Flux Operator. + properties: + apiVersion: + description: APIVersion is the API version of the Flux Operator. + type: string + platform: + description: Platform is the os/arch of Flux Operator. + type: string + version: + description: Version is the version number of Flux Operator. + type: string + required: + - apiVersion + - platform + - version + type: object + reconcilers: + description: |- + ReconcilersStatus is the list of Flux reconcilers and + their statistics grouped by API kind. + items: + description: FluxReconcilerStatus defines the observed state of + a Flux reconciler. + properties: + apiVersion: + description: APIVersion is the API version of the Flux resource. + type: string + kind: + description: Kind is the kind of the Flux resource. + type: string + stats: + description: Stats is the reconcile statics of the Flux resource + kind. + properties: + failing: + description: |- + Failing is the number of reconciled + resources in the Failing state and not Suspended. + type: integer + running: + description: |- + Running is the number of reconciled + resources in the Running state. + type: integer + suspended: + description: |- + Suspended is the number of reconciled + resources in the Suspended state. + type: integer + totalSize: + description: TotalSize is the total size of the artifacts + in storage. + type: string + required: + - failing + - running + - suspended + type: object + required: + - apiVersion + - kind + type: object + type: array + sync: + description: |- + SyncStatus is the status of the cluster sync + Source and Kustomization resources. + properties: + id: + description: ID is the identifier of the sync. + type: string + path: + description: Path is the kustomize path of the sync. + type: string + ready: + description: Ready is the readiness status of the sync. + type: boolean + source: + description: Source is the URL of the source repository. + type: string + status: + description: |- + Status is a human-readable message indicating details + about the sync observed state. + type: string + required: + - id + - ready + - status + type: object + required: + - distribution + type: object + status: + description: FluxReportStatus defines the readiness of a FluxReport. + properties: + conditions: + description: Conditions contains the readiness conditions of the object. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + lastHandledReconcileAt: + description: |- + LastHandledReconcileAt holds the value of the most recent + reconcile request value, so a change of the annotation value + can be detected. + type: string + type: object + type: object + x-kubernetes-validations: + - message: the only accepted name for a FluxReport is 'flux' + rule: self.metadata.name == 'flux' + served: true + storage: true + subresources: + status: {} +--- +# Source: flux-operator/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.0 + helm.sh/resource-policy: keep + labels: + app.kubernetes.io/instance: 'flux-operator' + app.kubernetes.io/managed-by: 'Helm' + app.kubernetes.io/name: 'flux-operator' + app.kubernetes.io/version: 'v0.48.0' + helm.sh/chart: 'flux-operator-0.48.0' + name: resourcesetinputproviders.fluxcd.controlplane.io +spec: + group: fluxcd.controlplane.io + names: + kind: ResourceSetInputProvider + listKind: ResourceSetInputProviderList + plural: resourcesetinputproviders + shortNames: + - rsip + singular: resourcesetinputprovider + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + name: v1 + schema: + openAPIV3Schema: + description: ResourceSetInputProvider is the Schema for the ResourceSetInputProviders + API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ResourceSetInputProviderSpec defines the desired state of + ResourceSetInputProvider + properties: + certSecretRef: + description: |- + CertSecretRef specifies the Kubernetes Secret containing either or both of + + - a PEM-encoded CA certificate (`ca.crt`) + - a PEM-encoded client certificate (`tls.crt`) and private key (`tls.key`) + + When connecting to a Git, OCI, or ExternalService provider that uses self-signed certificates, + the CA certificate must be set in the Secret under the 'ca.crt' key to establish the trust relationship. + When connecting to a provider that supports client certificates (mTLS), the client certificate + and private key must be set in the Secret under the 'tls.crt' and 'tls.key' keys, respectively. + properties: + name: + description: Name of the referent. + type: string + required: + - name + type: object + defaultValues: + additionalProperties: + x-kubernetes-preserve-unknown-fields: true + description: |- + DefaultValues contains the default values for the inputs. + These values are used to populate the inputs when the provider + response does not contain them. + type: object + filter: + description: Filter defines the filter to apply to the input provider + response. + properties: + excludeBranch: + description: |- + ExcludeBranch specifies the regular expression to filter the branches + that the input provider should exclude. + type: string + excludeEnvironment: + description: |- + ExcludeEnvironment specifies the regular expression to filter the environments + that the input provider should exclude. + type: string + excludeTag: + description: |- + ExcludeTag specifies the regular expression to filter the tags + that the input provider should exclude. + type: string + includeBranch: + description: |- + IncludeBranch specifies the regular expression to filter the branches + that the input provider should include. + type: string + includeEnvironment: + description: |- + IncludeEnvironment specifies the regular expression to filter the environments + that the input provider should include. + type: string + includeTag: + description: |- + IncludeTag specifies the regular expression to filter the tags + that the input provider should include. + type: string + labels: + description: Labels specifies the list of labels to filter the + input provider response. + items: + type: string + type: array + limit: + default: 100 + description: |- + Limit specifies the maximum number of input sets to return. + When not set, the default limit is 100. + type: integer + semver: + description: |- + Semver specifies a semantic version range to filter and sort the tags. + If this field is not specified, the tags will be sorted in reverse + alphabetical order. + Supported only for tags at the moment. + type: string + type: object + insecure: + description: |- + Insecure allows connecting to an ExternalService or OCIArtifactTag provider + over plain HTTP without TLS. When not set, the URL must use HTTPS. + type: boolean + schedule: + description: Schedule defines the schedules for the input provider + to run. + items: + description: Schedule defines a schedule for something to run. + properties: + cron: + description: Cron specifies the cron expression for the schedule. + type: string + timeZone: + default: UTC + description: TimeZone specifies the time zone for the cron schedule. + Defaults to UTC. + type: string + window: + default: 0s + description: |- + Window defines the time window during which the execution is allowed. + Defaults to 0s, meaning no window is applied. + pattern: ^([0-9]+(\.[0-9]+)?(ms|s|m|h))+$ + type: string + required: + - cron + type: object + type: array + secretRef: + description: |- + SecretRef specifies the Kubernetes Secret containing the credentials + to access the input provider. + When connecting to a Git provider, the secret must contain the keys + 'username' and 'password', and the password should be a personal access token + that grants read-only access to the repository. + When connecting to an OCI provider, the secret must contain a Kubernetes + Image Pull Secret, as if created by `kubectl create secret docker-registry`. + When connecting to an ExternalService provider, the secret must contain either + a 'token' key for bearer token authentication, or 'username' and 'password' + keys for basic authentication. + properties: + name: + description: Name of the referent. + type: string + required: + - name + type: object + serviceAccountName: + description: |- + ServiceAccountName specifies the name of the Kubernetes ServiceAccount + used for authentication with AWS, Azure or GCP services through + workload identity federation features. If not specified, the + authentication for these cloud providers will use the ServiceAccount + of the operator (or any other environment authentication configuration). + type: string + skip: + description: Skip defines whether we need to skip input provider response + updates. + properties: + labels: + description: |- + Labels specifies list of labels to skip input provider response when any of the label conditions matched. + When prefixed with !, input provider response will be skipped if it does not have this label. + items: + type: string + type: array + type: object + type: + description: Type specifies the type of the input provider. + enum: + - Static + - GitHubBranch + - GitHubTag + - GitHubPullRequest + - GitLabBranch + - GitLabTag + - GitLabMergeRequest + - GitLabEnvironment + - AzureDevOpsBranch + - AzureDevOpsTag + - AzureDevOpsPullRequest + - GiteaBranch + - GiteaTag + - GiteaPullRequest + - OCIArtifactTag + - ACRArtifactTag + - ECRArtifactTag + - GARArtifactTag + - ExternalService + type: string + url: + description: |- + URL specifies the HTTP/S or OCI address of the input provider API. + When connecting to a Git provider, the URL should point to the repository address. + When connecting to an OCI provider, the URL should point to the OCI repository address. + pattern: ^((http|https|oci)://.*){0,1}$ + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: spec.url must be empty when spec.type is 'Static' + rule: self.type != 'Static' || !has(self.url) + - message: spec.url must not be empty when spec.type is not 'Static' + rule: self.type == 'Static' || has(self.url) + - message: spec.url must start with 'http://' or 'https://' when spec.type + is a Git provider + rule: '!self.type.startsWith(''Git'') || self.url.startsWith(''http'')' + - message: spec.url must start with 'http://' or 'https://' when spec.type + is a Git provider + rule: '!self.type.startsWith(''AzureDevOps'') || self.url.startsWith(''http'')' + - message: spec.url must start with 'oci://' when spec.type is an OCI + provider + rule: '!self.type.endsWith(''ArtifactTag'') || self.url.startsWith(''oci'')' + - message: spec.url must start with 'http://' or 'https://' when spec.type + is 'ExternalService' + rule: self.type != 'ExternalService' || self.url.startsWith('http') + - message: spec.insecure can only be set when spec.type is 'ExternalService' + or 'OCIArtifactTag' + rule: '!has(self.insecure) || !self.insecure || self.type == ''ExternalService'' + || self.type == ''OCIArtifactTag''' + - message: spec.url must use 'https://' unless spec.insecure is true + rule: self.type != 'ExternalService' || !self.url.startsWith('http://') + || (has(self.insecure) && self.insecure) + - message: cannot specify spec.serviceAccountName when spec.type is not + one of AzureDevOps* or *ArtifactTag + rule: '!has(self.serviceAccountName) || self.type.startsWith(''AzureDevOps'') + || self.type.endsWith(''ArtifactTag'')' + - message: cannot specify spec.certSecretRef when spec.type is one of + Static, AzureDevOps*, ACRArtifactTag, ECRArtifactTag or GARArtifactTag + rule: '!has(self.certSecretRef) || !(self.url == ''Static'' || self.type.startsWith(''AzureDevOps'') + || (self.type.endsWith(''ArtifactTag'') && self.type != ''OCIArtifactTag''))' + - message: cannot specify spec.secretRef when spec.type is one of Static, + ACRArtifactTag, ECRArtifactTag or GARArtifactTag + rule: '!has(self.secretRef) || !(self.url == ''Static'' || (self.type.endsWith(''ArtifactTag'') + && self.type != ''OCIArtifactTag''))' + status: + description: ResourceSetInputProviderStatus defines the observed state + of ResourceSetInputProvider. + properties: + conditions: + description: Conditions contains the readiness conditions of the object. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + exportedInputs: + description: ExportedInputs contains the list of inputs exported by + the provider. + items: + additionalProperties: + x-kubernetes-preserve-unknown-fields: true + description: ResourceSetInput defines the key-value pairs of the + ResourceSet input. + type: object + type: array + lastExportedRevision: + description: |- + LastExportedRevision is the digest of the + inputs that were last reconcile. + type: string + lastHandledForceAt: + description: |- + LastHandledForceAt holds the value of the most recent + force request value, so a change of the annotation value + can be detected. + type: string + lastHandledReconcileAt: + description: |- + LastHandledReconcileAt holds the value of the most recent + reconcile request value, so a change of the annotation value + can be detected. + type: string + nextSchedule: + description: NextSchedule is the next schedule when the input provider + will run. + properties: + cron: + description: Cron specifies the cron expression for the schedule. + type: string + timeZone: + default: UTC + description: TimeZone specifies the time zone for the cron schedule. + Defaults to UTC. + type: string + when: + description: When is the next time the schedule will run. + format: date-time + type: string + window: + default: 0s + description: |- + Window defines the time window during which the execution is allowed. + Defaults to 0s, meaning no window is applied. + pattern: ^([0-9]+(\.[0-9]+)?(ms|s|m|h))+$ + type: string + required: + - cron + - when + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: flux-operator/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.0 + helm.sh/resource-policy: keep + labels: + app.kubernetes.io/instance: 'flux-operator' + app.kubernetes.io/managed-by: 'Helm' + app.kubernetes.io/name: 'flux-operator' + app.kubernetes.io/version: 'v0.48.0' + helm.sh/chart: 'flux-operator-0.48.0' + name: resourcesets.fluxcd.controlplane.io +spec: + group: fluxcd.controlplane.io + names: + kind: ResourceSet + listKind: ResourceSetList + plural: resourcesets + shortNames: + - rset + singular: resourceset + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + name: v1 + schema: + openAPIV3Schema: + description: ResourceSet is the Schema for the ResourceSets API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ResourceSetSpec defines the desired state of ResourceSet + properties: + commonMetadata: + description: |- + CommonMetadata specifies the common labels and annotations that are + applied to all resources. Any existing label or annotation will be + overridden if its key matches a common one. + properties: + annotations: + additionalProperties: + type: string + description: Annotations to be added to the object's metadata. + type: object + labels: + additionalProperties: + type: string + description: Labels to be added to the object's metadata. + type: object + type: object + dependsOn: + description: |- + DependsOn specifies the list of Kubernetes resources that must + exist on the cluster before the reconciliation process starts. + items: + description: Dependency defines a ResourceSet dependency on a Kubernetes + resource. + properties: + apiVersion: + description: APIVersion of the resource to depend on. + type: string + kind: + description: Kind of the resource to depend on. + type: string + name: + description: Name of the resource to depend on. + type: string + namespace: + description: Namespace of the resource to depend on. + type: string + ready: + description: Ready checks if the resource Ready status condition + is true. + type: boolean + readyExpr: + description: |- + ReadyExpr checks if the resource satisfies the given CEL expression. + The expression replaces the default readiness check and + is only evaluated if Ready is set to 'true'. + type: string + required: + - apiVersion + - kind + - name + type: object + type: array + inputStrategy: + description: |- + InputStrategy defines how the inputs are combined when multiple + input provider objects are used. Defaults to flattening all inputs + from all providers into a single list of input sets. + properties: + includeEmptyProviders: + description: |- + IncludeEmptyProviders controls how input providers that export no + inputs are treated. Only applies when Name is Permute. When true, if + any provider has zero inputs the resulting permutation set is empty + (mathematically correct Cartesian product behavior). When false or + unset (default), providers with zero inputs are silently skipped and + the remaining providers still permute among themselves. + type: boolean + name: + description: |- + Name defines how the inputs are combined when multiple + input provider objects are used. Supported values are: + - Flatten: all inputs sets from all input provider objects are + flattened into a single list of input sets. + - Permute: all inputs sets from all input provider objects are + combined using a Cartesian product, resulting in a list of input sets + that contains every possible combination of input values. + For example, if provider A has inputs [{x: 1}, {x: 2}] and provider B has + inputs [{y: "a"}, {y: "b"}], the resulting input sets will be: + [{x: 1, y: "a"}, {x: 1, y: "b"}, {x: 2, y: "a"}, {x: 2, y: "b"}]. + This strategy can lead to a large number of input sets and should be + used with caution. Users should use filtering features from + ResourceSetInputProvider to limit the amount of exported inputs. + enum: + - Flatten + - Permute + type: string + required: + - name + type: object + x-kubernetes-validations: + - message: includeEmptyProviders only applies when name is Permute + rule: '!has(self.includeEmptyProviders) || self.name == ''Permute''' + inputs: + description: Inputs contains the list of ResourceSet inputs. + items: + additionalProperties: + x-kubernetes-preserve-unknown-fields: true + description: ResourceSetInput defines the key-value pairs of the + ResourceSet input. + type: object + type: array + inputsFrom: + description: |- + InputsFrom contains the list of references to input providers. + When set, the inputs are fetched from the providers and concatenated + with the in-line inputs defined in the ResourceSet. + items: + description: |- + InputProviderReference defines a reference to an input provider resource + in the same namespace as the ResourceSet. + properties: + apiVersion: + description: |- + APIVersion of the input provider resource. + When not set, the APIVersion of the ResourceSet is used. + enum: + - fluxcd.controlplane.io/v1 + type: string + kind: + description: Kind of the input provider resource. + enum: + - ResourceSetInputProvider + type: string + name: + description: |- + Name of the input provider resource. Cannot be set + when the Selector field is set. + type: string + selector: + description: |- + Selector is a label selector to filter the input provider resources + as an alternative to the Name field. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: object + x-kubernetes-validations: + - message: at least one of name or selector must be set for input + provider references + rule: has(self.name) || has(self.selector) + - message: cannot set both name and selector for input provider + references + rule: '!has(self.name) || !has(self.selector)' + type: array + resources: + description: Resources contains the list of Kubernetes resources to + reconcile. + items: + x-kubernetes-preserve-unknown-fields: true + type: array + resourcesTemplate: + description: |- + ResourcesTemplate is a Go template that generates the list of + Kubernetes resources to reconcile. The template is rendered + as multi-document YAML, the resources should be separated by '---'. + When both Resources and ResourcesTemplate are set, the resulting + objects are merged and deduplicated, with the ones from Resources taking precedence. + type: string + serviceAccountName: + description: |- + The name of the Kubernetes service account to impersonate + when reconciling the generated resources. + type: string + wait: + description: |- + Wait instructs the controller to check the health + of all the reconciled resources. + type: boolean + type: object + status: + description: ResourceSetStatus defines the observed state of ResourceSet. + properties: + conditions: + description: Conditions contains the readiness conditions of the object. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + externalChecksumRefs: + description: |- + ExternalChecksumRefs lists the ConfigMap and Secret references + discovered in checksumFrom annotations on the last reconciliation + that point to objects not rendered by this ResourceSet. Each entry + has the form "Kind/namespace/name". It is used to trigger a + reconciliation when one of the referenced objects changes. + items: + type: string + type: array + history: + description: |- + History contains the reconciliation history of the ResourceSet + as a list of snapshots ordered by the last reconciled time. + items: + description: |- + Snapshot represents a point-in-time record of a group of resources reconciliation, + including timing information, status, and a unique digest identifier. + properties: + digest: + description: Digest is the checksum in the format `:` + of the resources in this snapshot. + type: string + firstReconciled: + description: FirstReconciled is the time when this revision + was first reconciled to the cluster. + format: date-time + type: string + lastReconciled: + description: LastReconciled is the time when this revision was + last reconciled to the cluster. + format: date-time + type: string + lastReconciledDuration: + description: LastReconciledDuration is time it took to reconcile + the resources in this revision. + type: string + lastReconciledStatus: + description: LastReconciledStatus is the status of the last + reconciliation. + type: string + metadata: + additionalProperties: + type: string + description: Metadata contains additional information about + the snapshot. + type: object + totalReconciliations: + description: TotalReconciliations is the total number of reconciliations + that have occurred for this snapshot. + format: int64 + type: integer + required: + - digest + - firstReconciled + - lastReconciled + - lastReconciledDuration + - lastReconciledStatus + - totalReconciliations + type: object + type: array + inventory: + description: |- + Inventory contains a list of Kubernetes resource object references + last applied on the cluster. + properties: + entries: + description: Entries of Kubernetes resource object references. + items: + description: ResourceRef contains the information necessary + to locate a resource within a cluster. + properties: + id: + description: |- + ID is the string representation of the Kubernetes resource object's metadata, + in the format '___'. + type: string + v: + description: Version is the API version of the Kubernetes + resource object's kind. + type: string + required: + - id + - v + type: object + type: array + required: + - entries + type: object + lastAppliedRevision: + description: |- + LastAppliedRevision is the digest of the + generated resources that were last reconcile. + type: string + lastHandledReconcileAt: + description: |- + LastHandledReconcileAt holds the value of the most recent + reconcile request value, so a change of the annotation value + can be detected. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: flux-operator/templates/aggregate-clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: flux-operator-edit + labels: + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-admin: "true" + helm.sh/chart: flux-operator-0.48.0 + app.kubernetes.io/name: flux-operator + app.kubernetes.io/instance: flux-operator + app.kubernetes.io/version: "v0.48.0" + app.kubernetes.io/managed-by: Helm +rules: + - apiGroups: + - fluxcd.controlplane.io + resources: + - resourcesets + - resourcesetinputproviders + verbs: + - create + - delete + - deletecollection + - patch + - update +--- +# Source: flux-operator/templates/aggregate-clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: flux-operator-view + labels: + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-view: "true" + helm.sh/chart: flux-operator-0.48.0 + app.kubernetes.io/name: flux-operator + app.kubernetes.io/instance: flux-operator + app.kubernetes.io/version: "v0.48.0" + app.kubernetes.io/managed-by: Helm +rules: + - apiGroups: + - fluxcd.controlplane.io + resources: + - resourcesets + - resourcesetinputproviders + verbs: + - get + - list + - watch +--- +# Source: flux-operator/templates/web-standard-roles.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: flux-web-user + labels: + helm.sh/chart: flux-operator-0.48.0 + app.kubernetes.io/name: flux-operator + app.kubernetes.io/instance: flux-operator + app.kubernetes.io/version: "v0.48.0" + app.kubernetes.io/managed-by: Helm +rules: + - apiGroups: ["*"] + resources: ["*"] + verbs: ["get", "list", "watch"] +--- +# Source: flux-operator/templates/web-standard-roles.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: flux-web-admin + labels: + helm.sh/chart: flux-operator-0.48.0 + app.kubernetes.io/name: flux-operator + app.kubernetes.io/instance: flux-operator + app.kubernetes.io/version: "v0.48.0" + app.kubernetes.io/managed-by: Helm +rules: + - apiGroups: ["*"] + resources: ["*"] + verbs: ["get", "list", "watch"] + - apiGroups: + - fluxcd.controlplane.io + - source.toolkit.fluxcd.io + - source.extensions.fluxcd.io + - kustomize.toolkit.fluxcd.io + - helm.toolkit.fluxcd.io + - image.toolkit.fluxcd.io + - notification.toolkit.fluxcd.io + resources: ["*"] + verbs: + - patch + - reconcile + - suspend + - resume + - download + - apiGroups: + - apps + resources: + - deployments + - statefulsets + - daemonsets + verbs: + - patch + - restart + - apiGroups: + - batch + resources: + - cronjobs + - jobs + verbs: + - create + - restart + - apiGroups: + - "" + resources: + - pods + verbs: + - delete +--- +# Source: flux-operator/templates/admin-clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: flux-operator + labels: + helm.sh/chart: flux-operator-0.48.0 + app.kubernetes.io/name: flux-operator + app.kubernetes.io/instance: flux-operator + app.kubernetes.io/version: "v0.48.0" + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: flux-operator + namespace: flux-system +--- +# Source: flux-operator/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: flux-operator + namespace: flux-system + labels: + helm.sh/chart: flux-operator-0.48.0 + app.kubernetes.io/name: flux-operator + app.kubernetes.io/instance: flux-operator + app.kubernetes.io/version: "v0.48.0" + app.kubernetes.io/managed-by: Helm +spec: + ports: + - port: 8080 + targetPort: http-metrics + protocol: TCP + name: http + - port: 9080 + targetPort: http-web + protocol: TCP + name: http-web + selector: + app.kubernetes.io/name: flux-operator + app.kubernetes.io/instance: flux-operator +--- +# Source: flux-operator/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flux-operator + namespace: flux-system + labels: + helm.sh/chart: flux-operator-0.48.0 + app.kubernetes.io/name: flux-operator + app.kubernetes.io/instance: flux-operator + app.kubernetes.io/version: "v0.48.0" + app.kubernetes.io/managed-by: Helm +spec: + selector: + matchLabels: + app.kubernetes.io/name: flux-operator + app.kubernetes.io/instance: flux-operator + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + prometheus.io/path: "/metrics" + labels: + helm.sh/chart: flux-operator-0.48.0 + app.kubernetes.io/name: flux-operator + app.kubernetes.io/instance: flux-operator + app.kubernetes.io/version: "v0.48.0" + app.kubernetes.io/managed-by: Helm + spec: + serviceAccountName: flux-operator + containers: + - name: manager + args: + - --log-level=info + env: + - name: RUNTIME_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: REPORTING_INTERVAL + value: 5m + - name: WEB_SERVER_PORT + value: "9080" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + image: "ghcr.io/controlplaneio-fluxcd/flux-operator:v0.48.0" + imagePullPolicy: "IfNotPresent" + ports: + - name: http-metrics + containerPort: 8080 + protocol: TCP + - name: http + containerPort: 8081 + protocol: TCP + - name: http-web + containerPort: 9080 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 2000m + memory: 1Gi + requests: + cpu: 100m + memory: 64Mi + volumeMounts: + - name: temp + mountPath: /tmp + volumes: + - name: temp + emptyDir: {} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux diff --git a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_apply_git.yaml b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_apply_git.yaml new file mode 100644 index 0000000..5869736 --- /dev/null +++ b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_apply_git.yaml @@ -0,0 +1,50 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: GitRepository +metadata: + name: kubernetes + namespace: flux-system +spec: + interval: 5m0s + ref: + branch: nix + secretRef: + name: kubernetes-deploy-key + # url: ssh://git@74.80.180.138:65099/repos/mrmanager + url: ssh://git@10.215.1.210:22/repos/mrmanager + ignore: | + bootstrap + .sops.yaml +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: apply1 + namespace: flux-system +spec: + interval: 1m0s + path: "./k8s/1" + prune: true + sourceRef: + kind: GitRepository + name: kubernetes + decryption: + provider: sops + secretRef: + name: sops-gpg +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: apply2 + namespace: flux-system +spec: + interval: 1m0s + path: "./k8s/2" + prune: true + sourceRef: + kind: GitRepository + name: kubernetes + decryption: + provider: sops + secretRef: + name: sops-gpg diff --git a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_instance.yaml b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_instance.yaml new file mode 100644 index 0000000..211fd7c --- /dev/null +++ b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_instance.yaml @@ -0,0 +1,17 @@ +apiVersion: fluxcd.controlplane.io/v1 +kind: FluxInstance +metadata: + name: flux + namespace: flux-system +spec: + distribution: + version: "2.8.x" + registry: "ghcr.io/fluxcd" + components: + - source-controller + - kustomize-controller + - helm-controller + - notification-controller + - image-automation-controller + - image-reflector-controller + # - source-watcher diff --git a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_namespace.yaml b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_namespace.yaml new file mode 100644 index 0000000..c00a432 --- /dev/null +++ b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/flux_namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: flux-system diff --git a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/initial_clusterrole.yaml b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/initial_clusterrole.yaml new file mode 100644 index 0000000..e56770a --- /dev/null +++ b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/initial_clusterrole.yaml @@ -0,0 +1,33 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: system:kube-apiserver-to-kubelet +rules: + - apiGroups: + - "" + resources: + - nodes/proxy + - nodes/stats + - nodes/log + - nodes/spec + - nodes/metrics + verbs: + - "*" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:kube-apiserver + namespace: "" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-apiserver-to-kubelet +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: User + name: kubernetes \ No newline at end of file diff --git a/nix/kubernetes/keys/package/bootstrap-script/package.nix b/nix/kubernetes/keys/package/bootstrap-script/package.nix new file mode 100644 index 0000000..2e050b9 --- /dev/null +++ b/nix/kubernetes/keys/package/bootstrap-script/package.nix @@ -0,0 +1,86 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + lib, + stdenv, + fetchFromGitHub, + writeShellScript, + k8s, + ... +}: +let + bootstrap_script = (writeShellScript "bootstrap-script" bootstrap_script_body); + bootstrap_script_body = ('' + set -euo pipefail + IFS=$'\n\t' + DIR="$( cd "$( dirname "''${BASH_SOURCE[0]}" )" && pwd )" + + ${apply_manifests} + echo "Bootstrap finished" + ''); + manifests = ( + lib.concatMapStringsSep "," lib.escapeShellArg ( + [ + ./files/manifests/initial_clusterrole.yaml + ] + ++ gateway_crds + ++ [ + "${k8s.cilium-manifest}/cilium.yaml" + "${k8s.coredns-manifest}/coredns.yaml" + ./files/manifests/flux_namespace.yaml + + # + # Generate with: helm template --dry-run=server flux-operator oci://ghcr.io/controlplaneio-fluxcd/charts/flux-operator --namespace flux-system --create-namespace + # + ./files/manifests/flux.yaml + ./files/manifests/flux_instance.yaml + ] + ++ (lib.attrsets.mapAttrsToList ( + secret_name: secret_value: "${secret_value}/${secret_name}.yaml" + ) k8s.k8s-secrets-generic) + ++ [ + ./files/manifests/flux_apply_git.yaml + ] + ) + ); + apply_manifests = "kubectl --kubeconfig=${k8s.client-configs.admin}/admin.kubeconfig apply --server-side --force-conflicts -f ${manifests}"; + gateway_crds_repo = fetchFromGitHub { + owner = "kubernetes-sigs"; + repo = "gateway-api"; + rev = "v1.5.1"; + sha256 = "sha256-mWMvJG6esOqDBSbhExvt7L3ZTiQUOfeRBohew/m67A0="; + }; + gateway_crds = [ + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_backendtlspolicies.yaml" + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_gatewayclasses.yaml" + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_gateways.yaml" + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_grpcroutes.yaml" + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_httproutes.yaml" + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_listenersets.yaml" + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_referencegrants.yaml" + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_tcproutes.yaml" + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_tlsroutes.yaml" + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_udproutes.yaml" + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.k8s.io_vap_safeupgrades.yaml" + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.x-k8s.io_xbackendtrafficpolicies.yaml" + "${gateway_crds_repo}/config/crd/experimental/gateway.networking.x-k8s.io_xmeshes.yaml" + ]; +in +stdenv.mkDerivation (finalAttrs: { + name = "bootstrap-script"; + nativeBuildInputs = [ ]; + buildInputs = [ ]; + + unpackPhase = "true"; + + installPhase = '' + cp ${bootstrap_script} "$out" + ''; +}) diff --git a/nix/kubernetes/keys/package/deploy-script/package.nix b/nix/kubernetes/keys/package/deploy-script/package.nix new file mode 100644 index 0000000..c18242e --- /dev/null +++ b/nix/kubernetes/keys/package/deploy-script/package.nix @@ -0,0 +1,363 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + config, + lib, + stdenv, + writeShellScript, + k8s, + openssh, + ... +}: +let + vm_name_to_hostname = + let + mapping = { + "nc0" = "controller0"; + "nc1" = "controller1"; + "nc2" = "controller2"; + "nw0" = "worker0"; + "nw1" = "worker1"; + "nw2" = "worker2"; + }; + in + (vm_name: mapping."${vm_name}"); + + deploy_script_body = ( + '' + set -euo pipefail + IFS=$'\n\t' + DIR="$( cd "$( dirname "''${BASH_SOURCE[0]}" )" && pwd )" + '' + + (lib.concatMapStringsSep "\n" deploy_control_plane [ + "nc0" + "nc1" + "nc2" + ]) + + (lib.concatMapStringsSep "\n" deploy_worker [ + "nw0" + "nw1" + "nw2" + ]) + + (trust_ssh_key { + public_key = "${k8s.ssh-keys.flux_ssh_key}/flux_ssh_key.pub"; + destination = "/jail/admin_git/usr/home/git/.ssh/authorized_keys"; + owner = "11236"; + group = "11236"; + mode = "0600"; + }) + + (lib.concatMapStringsSep "\n" create_pv_dir [ + { + path = "manual-pv/gitea-psql"; + owner = "26"; + group = "26"; + mode = "0777"; + } + { + path = "manual-pv/harbor-psql"; + owner = "26"; + group = "26"; + mode = "0755"; + } + # { + # path = "manual-pv/gitea"; + # owner = "1000"; + # group = "1000"; + # mode = "0777"; + # } + # { + # path = "manual-pv/gitea/gitea"; + # owner = "1000"; + # group = "1000"; + # mode = "0700"; + # } + # { + # path = "manual-pv/gitea/gitea/public"; + # owner = "1000"; + # group = "1000"; + # mode = "0755"; + # } + ]) + + ); + deploy_script = (writeShellScript "deploy-script" deploy_script_body); + deploy_file = ( + { + dest_dir, + file, + name ? (builtins.baseNameOf file), + owner, + group, + mode, + }: + '' + ## + ## deploy ${name} to ${dest_dir} + ## + ${openssh}/bin/ssh mrmanager doas rm -f ${dest_dir}/${name} ~/${name} + ${openssh}/bin/scp ${file} mrmanager:~/${name} + ${openssh}/bin/ssh mrmanager doas install -o ${toString owner} -g ${toString group} -m ${mode} ~/${name} ${dest_dir}/${name} + ${openssh}/bin/ssh mrmanager doas rm -f ~/${name} + + + '' + ); + deploy_control_plane = ( + vm_name: + ( + '' + ## + ## Create directories on ${vm_name} + ## + ${openssh}/bin/ssh mrmanager doas install -d -o 0 -g 0 -m 0755 /vm/${vm_name}/persist/keys + ${openssh}/bin/ssh mrmanager doas install -d -o 10016 -g 10016 -m 0755 /vm/${vm_name}/persist/keys/etcd + ${openssh}/bin/ssh mrmanager doas install -d -o 10024 -g 10024 -m 0755 /vm/${vm_name}/persist/keys/kube + '' + + (lib.concatMapStringsSep "\n" deploy_file [ + { + dest_dir = "/vm/${vm_name}/persist/keys/etcd"; + file = "${k8s.keys.kube-api-server}/kube-api-server.crt"; + owner = 10016; + group = 10016; + mode = "0640"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/etcd"; + file = "${k8s.keys.kube-api-server}/kube-api-server.key"; + owner = 10016; + group = 10016; + mode = "0600"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/etcd"; + file = "${k8s.ca.client}/client-ca.crt"; + owner = 10016; + group = 10016; + mode = "0640"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.ca.client}/client-ca.crt"; + owner = 10024; + group = 10024; + mode = "0640"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.ca.client}/client-ca.key"; + owner = 10024; + group = 10024; + mode = "0600"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.keys.kube-api-server}/kube-api-server.crt"; + owner = 10024; + group = 10024; + mode = "0640"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.keys.kube-api-server}/kube-api-server.key"; + owner = 10024; + group = 10024; + mode = "0600"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.encryption_config}/encryption-config.yaml"; + name = "encryption-config.yaml"; + owner = 10024; + group = 10024; + mode = "0600"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.keys.service-accounts}/service-accounts.crt"; + owner = 10024; + group = 10024; + mode = "0640"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.keys.service-accounts}/service-accounts.key"; + owner = 10024; + group = 10024; + mode = "0600"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.client-configs.kube-controller-manager}/kube-controller-manager.kubeconfig"; + owner = 10024; + group = 10024; + mode = "0600"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.client-configs.kube-scheduler}/kube-scheduler.kubeconfig"; + owner = 10024; + group = 10024; + mode = "0600"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.ca.requestheader-client}/requestheader-client-ca.crt"; + owner = 10024; + group = 10024; + mode = "0640"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${ + k8s.keys."${vm_name_to_hostname vm_name}-proxy" + }/${vm_name_to_hostname vm_name}-proxy.crt"; + name = "proxy.crt"; + owner = 10024; + group = 10024; + mode = "0640"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${ + k8s.keys."${vm_name_to_hostname vm_name}-proxy" + }/${vm_name_to_hostname vm_name}-proxy.key"; + name = "proxy.key"; + owner = 10024; + group = 10024; + mode = "0600"; + } + ]) + ) + ); + deploy_worker = ( + vm_name: + ( + '' + ## + ## Create directories on ${vm_name} + ## + ${openssh}/bin/ssh mrmanager doas install -d -o 0 -g 0 -m 0755 /vm/${vm_name}/persist/keys + ${openssh}/bin/ssh mrmanager doas install -d -o 10024 -g 10024 -m 0755 /vm/${vm_name}/persist/keys/kube + + ${openssh}/bin/ssh mrmanager doas install -d -o 0 -g 0 -m 0700 /vm/${vm_name}/persist/containerd/certs.d/docker.io + ${openssh}/bin/ssh mrmanager doas install -d -o 0 -g 0 -m 0700 /vm/${vm_name}/persist/containerd/certs.d/harbor.fizz.buzz + '' + + (lib.concatMapStringsSep "\n" deploy_file [ + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.ca.client}/client-ca.crt"; + owner = 10024; + group = 10024; + mode = "0640"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.keys."${vm_name_to_hostname vm_name}"}/${vm_name_to_hostname vm_name}.crt"; + name = "kubelet.crt"; + owner = 10024; + group = 10024; + mode = "0640"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.keys."${vm_name_to_hostname vm_name}"}/${vm_name_to_hostname vm_name}.key"; + name = "kubelet.key"; + owner = 10024; + group = 10024; + mode = "0600"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${ + k8s.client-configs."${vm_name_to_hostname vm_name}" + }/${vm_name_to_hostname vm_name}.kubeconfig"; + name = "kubelet.kubeconfig"; + owner = 10024; + group = 10024; + mode = "0600"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${k8s.client-configs.kube-proxy}/kube-proxy.kubeconfig"; + owner = 10024; + group = 10024; + mode = "0600"; + } + { + dest_dir = "/vm/${vm_name}/persist/containerd/certs.d/docker.io"; + file = "${./files/containerd/docker.io/hosts.toml}"; + name = "hosts.toml"; + owner = 0; + group = 0; + mode = "0600"; + } + { + dest_dir = "/vm/${vm_name}/persist/containerd/certs.d/harbor.fizz.buzz"; + file = "${./files/containerd/harbor.fizz.buzz/hosts.toml}"; + name = "hosts.toml"; + owner = 0; + group = 0; + mode = "0600"; + } + ]) + ) + ); + trust_ssh_key = + { + public_key, + destination, + owner, + group, + mode, + }: + let + public_key_name = builtins.baseNameOf public_key; + public_key_contents = builtins.readFile public_key; + trimmed = lib.strings.trim public_key_contents; + escaped = lib.strings.escapeShellArg trimmed; + in + '' + ## + ## trust ${public_key_name} in ${destination} + ## + if ! ${openssh}/bin/ssh mrmanager doas grep -q "${escaped}" "${destination}"; then + ${openssh}/bin/ssh mrmanager doas tee -a "${destination}" <<<"$(cat ${public_key})" + ${openssh}/bin/ssh mrmanager doas chown "${owner}:${group}" "${destination}" + ${openssh}/bin/ssh mrmanager doas chmod "${mode}" "${destination}" + else + echo "${public_key_name} is already trusted in ${destination}" + fi + ''; + create_pv_dir = + { + path, + owner, + group, + mode, + }: + '' + ## + ## create pv directory ${path} + ## + ${openssh}/bin/ssh mrmanager doas install -d -o "${owner}" -g "${group}" -m "${mode}" "/nk8spv/${path}" + ''; + +in +stdenv.mkDerivation (finalAttrs: { + name = "deploy-script"; + nativeBuildInputs = [ ]; + buildInputs = [ ]; + + unpackPhase = "true"; + + installPhase = '' + cp ${deploy_script} "$out" + ''; +}) diff --git a/nix/kubernetes/keys/package/helm-manifest/package.nix b/nix/kubernetes/keys/package/helm-manifest/package.nix new file mode 100644 index 0000000..9bb6aa8 --- /dev/null +++ b/nix/kubernetes/keys/package/helm-manifest/package.nix @@ -0,0 +1,48 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + lib, + pkgs, + stdenv, + kubernetes-helm, + helm_src, + helm_name, + helm_namespace, + helm_path ? ".", + helm_manifest_name, + helm_values ? { }, + ... +}: +stdenv.mkDerivation ( + finalAttrs: + let + to_yaml_file = ((import ../../../functions/to_yaml.nix) { inherit pkgs; }).to_yaml_file; + in + { + name = "${helm_name}-manifest"; + nativeBuildInputs = [ + kubernetes-helm + ]; + buildInputs = [ ]; + + src = helm_src; + + buildPhase = '' + helm template --dry-run=client ${lib.strings.escapeShellArg helm_name} $src/${helm_path} --namespace ${helm_namespace} \ + --values ${to_yaml_file "values.yaml" helm_values} \ + | tee $NIX_BUILD_TOP/${helm_manifest_name} + ''; + + installPhase = '' + mkdir -p "$out" + cp $NIX_BUILD_TOP/${helm_manifest_name} $out/ + ''; + } +) diff --git a/nix/kubernetes/keys/package/k8s-ca/files/client-ca.conf b/nix/kubernetes/keys/package/k8s-ca/files/client-ca.conf new file mode 100644 index 0000000..ac8d404 --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-ca/files/client-ca.conf @@ -0,0 +1,305 @@ +[req] +distinguished_name = req_distinguished_name +prompt = no +x509_extensions = ca_x509_extensions + +[ca_x509_extensions] +basicConstraints = CA:TRUE +keyUsage = cRLSign, keyCertSign + +[req_distinguished_name] +C = US +ST = Washington +L = Seattle +CN = CA + +[admin] +distinguished_name = admin_distinguished_name +prompt = no +req_extensions = default_req_extensions + +[admin_distinguished_name] +CN = admin +O = system:masters + +# Service Accounts +# +# The Kubernetes Controller Manager leverages a key pair to generate +# and sign service account tokens as described in the +# [managing service accounts](https://kubernetes.io/docs/admin/service-accounts-admin/) +# documentation. + +[service-accounts] +distinguished_name = service-accounts_distinguished_name +prompt = no +req_extensions = default_req_extensions + +[service-accounts_distinguished_name] +CN = service-accounts + +# Worker Nodes +# +# Kubernetes uses a [special-purpose authorization mode](https://kubernetes.io/docs/admin/authorization/node/) +# called Node Authorizer, that specifically authorizes API requests made +# by [Kubelets](https://kubernetes.io/docs/concepts/overview/components/#kubelet). +# In order to be authorized by the Node Authorizer, Kubelets must use a credential +# that identifies them as being in the `system:nodes` group, with a username +# of `system:node:`. + +[controller0] +distinguished_name = controller0_distinguished_name +prompt = no +req_extensions = controller0_req_extensions + +[controller0_req_extensions] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth, serverAuth +keyUsage = critical, digitalSignature, keyEncipherment +nsCertType = client +nsComment = "controller0 Certificate" +subjectAltName = DNS:controller0, IP:127.0.0.1 +subjectKeyIdentifier = hash + +[controller0_distinguished_name] +CN = system:node:controller0 +O = system:nodes +C = US +ST = Washington +L = Seattle + +[controller1] +distinguished_name = controller1_distinguished_name +prompt = no +req_extensions = controller1_req_extensions + +[controller1_req_extensions] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth, serverAuth +keyUsage = critical, digitalSignature, keyEncipherment +nsCertType = client +nsComment = "controller1 Certificate" +subjectAltName = DNS:controller1, IP:127.0.0.1 +subjectKeyIdentifier = hash + +[controller1_distinguished_name] +CN = system:node:controller1 +O = system:nodes +C = US +ST = Washington +L = Seattle + +[controller2] +distinguished_name = controller2_distinguished_name +prompt = no +req_extensions = controller2_req_extensions + +[controller2_req_extensions] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth, serverAuth +keyUsage = critical, digitalSignature, keyEncipherment +nsCertType = client +nsComment = "controller2 Certificate" +subjectAltName = DNS:controller2, IP:127.0.0.1 +subjectKeyIdentifier = hash + +[controller2_distinguished_name] +CN = system:node:controller2 +O = system:nodes +C = US +ST = Washington +L = Seattle + +[worker0] +distinguished_name = worker0_distinguished_name +prompt = no +req_extensions = worker0_req_extensions + +[worker0_req_extensions] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth, serverAuth +keyUsage = critical, digitalSignature, keyEncipherment +nsCertType = client +nsComment = "worker0 Certificate" +subjectAltName = DNS:worker0, IP:127.0.0.1, IP:10.215.1.224, IP:2620:11f:7001:7:ffff:ffff:ad7:1e0 +subjectKeyIdentifier = hash + +[worker0_distinguished_name] +CN = system:node:worker0 +O = system:nodes +C = US +ST = Washington +L = Seattle + +[worker1] +distinguished_name = worker1_distinguished_name +prompt = no +req_extensions = worker1_req_extensions + +[worker1_req_extensions] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth, serverAuth +keyUsage = critical, digitalSignature, keyEncipherment +nsCertType = client +nsComment = "worker1 Certificate" +subjectAltName = DNS:worker1, IP:127.0.0.1, IP:10.215.1.225, IP:2620:11f:7001:7:ffff:ffff:ad7:1e1 +subjectKeyIdentifier = hash + +[worker1_distinguished_name] +CN = system:node:worker1 +O = system:nodes +C = US +ST = Washington +L = Seattle + +[worker2] +distinguished_name = worker2_distinguished_name +prompt = no +req_extensions = worker2_req_extensions + +[worker2_req_extensions] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth, serverAuth +keyUsage = critical, digitalSignature, keyEncipherment +nsCertType = client +nsComment = "worker2 Certificate" +subjectAltName = DNS:worker2, IP:127.0.0.1, IP:10.215.1.226, IP:2620:11f:7001:7:ffff:ffff:ad7:1e2 +subjectKeyIdentifier = hash + +[worker2_distinguished_name] +CN = system:node:worker2 +O = system:nodes +C = US +ST = Washington +L = Seattle + + + +# Kube Proxy Section +[kube-proxy] +distinguished_name = kube-proxy_distinguished_name +prompt = no +req_extensions = kube-proxy_req_extensions + +[kube-proxy_req_extensions] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth, serverAuth +keyUsage = critical, digitalSignature, keyEncipherment +nsCertType = client +nsComment = "Kube Proxy Certificate" +subjectAltName = DNS:kube-proxy, IP:127.0.0.1 +subjectKeyIdentifier = hash + +[kube-proxy_distinguished_name] +CN = system:kube-proxy +O = system:node-proxier +C = US +ST = Washington +L = Seattle + + +# Controller Manager +[kube-controller-manager] +distinguished_name = kube-controller-manager_distinguished_name +prompt = no +req_extensions = kube-controller-manager_req_extensions + +[kube-controller-manager_req_extensions] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth, serverAuth +keyUsage = critical, digitalSignature, keyEncipherment +nsCertType = client +nsComment = "Kube Controller Manager Certificate" +subjectAltName = DNS:kube-controller-manager, IP:127.0.0.1 +subjectKeyIdentifier = hash + +[kube-controller-manager_distinguished_name] +CN = system:kube-controller-manager +O = system:kube-controller-manager +C = US +ST = Washington +L = Seattle + + +# Scheduler +[kube-scheduler] +distinguished_name = kube-scheduler_distinguished_name +prompt = no +req_extensions = kube-scheduler_req_extensions + +[kube-scheduler_req_extensions] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth, serverAuth +keyUsage = critical, digitalSignature, keyEncipherment +nsCertType = client +nsComment = "Kube Scheduler Certificate" +subjectAltName = DNS:kube-scheduler, IP:127.0.0.1 +subjectKeyIdentifier = hash + +[kube-scheduler_distinguished_name] +CN = system:kube-scheduler +O = system:system:kube-scheduler +C = US +ST = Washington +L = Seattle + + +# API Server +# +# The Kubernetes API server is automatically assigned the `kubernetes` +# internal dns name, which will be linked to the first IP address (`10.32.0.1`) +# from the address range (`10.32.0.0/24`) reserved for internal cluster +# services. + +[kube-api-server] +distinguished_name = kube-api-server_distinguished_name +prompt = no +req_extensions = kube-api-server_req_extensions + +[kube-api-server_req_extensions] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth, serverAuth +keyUsage = critical, digitalSignature, keyEncipherment +nsCertType = client, server +nsComment = "Kube API Server Certificate" +subjectAltName = @kube-api-server_alt_names +subjectKeyIdentifier = hash + +[kube-api-server_alt_names] +IP.0 = 127.0.0.1 +IP.1 = 10.0.0.1 +IP.2 = 10.215.1.221 +IP.3 = 2620:11f:7001:7:ffff:ffff:0ad7:01dd +IP.4 = 10.215.1.222 +IP.5 = 2620:11f:7001:7:ffff:ffff:0ad7:01de +IP.6 = 10.215.1.223 +IP.7 = 2620:11f:7001:7:ffff:ffff:0ad7:01df +IP.8 = 10.215.1.224 +IP.9 = 2620:11f:7001:7:ffff:ffff:0ad7:01e0 +IP.10 = 10.215.1.225 +IP.11 = 2620:11f:7001:7:ffff:ffff:0ad7:01e1 +IP.12 = 10.215.1.226 +IP.13 = 2620:11f:7001:7:ffff:ffff:0ad7:01e2 +IP.14 = fd00:3e42:e349::1 +IP.15 = 2620:11f:7001:7:ffff:eeee::1 +DNS.0 = kubernetes +DNS.1 = kubernetes.default +DNS.2 = kubernetes.default.svc +DNS.3 = kubernetes.default.svc.cluster +DNS.4 = kubernetes.svc.cluster.local +DNS.5 = server.kubernetes.local +DNS.6 = api-server.kubernetes.local + +[kube-api-server_distinguished_name] +CN = kubernetes +C = US +ST = Washington +L = Seattle + + +[default_req_extensions] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth +keyUsage = critical, digitalSignature, keyEncipherment +nsCertType = client +nsComment = "Admin Client Certificate" +subjectKeyIdentifier = hash diff --git a/nix/kubernetes/keys/package/k8s-ca/files/requestheader-client-ca.conf b/nix/kubernetes/keys/package/k8s-ca/files/requestheader-client-ca.conf new file mode 100644 index 0000000..a25c479 --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-ca/files/requestheader-client-ca.conf @@ -0,0 +1,98 @@ +[req] +distinguished_name = req_distinguished_name +prompt = no +x509_extensions = ca_x509_extensions + +[ca_x509_extensions] +basicConstraints = CA:TRUE +keyUsage = cRLSign, keyCertSign + +[req_distinguished_name] +C = US +ST = Washington +L = Seattle +CN = Kubernetes +O = Kubernetes +OU = CA + + +[controller0-proxy] +distinguished_name = controller0_distinguished_name +prompt = no +req_extensions = controller0_req_extensions + +[controller0_req_extensions] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth, serverAuth +keyUsage = critical, digitalSignature, keyEncipherment +nsCertType = client +nsComment = "controller0-proxy Certificate" +subjectAltName = @controller0_alt_names +subjectKeyIdentifier = hash + +[controller0_distinguished_name] +CN = system:node:controller0 +O = system:nodes +C = US +ST = Washington +L = Seattle + +[controller0_alt_names] +IP.0 = 127.0.0.1 +IP.1 = 10.215.1.221 +IP.2 = 2620:11f:7001:7:ffff:ffff:0ad7:01dd +DNS.0 = controller0 + +[controller1-proxy] +distinguished_name = controller1_distinguished_name +prompt = no +req_extensions = controller1_req_extensions + +[controller1_req_extensions] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth, serverAuth +keyUsage = critical, digitalSignature, keyEncipherment +nsCertType = client +nsComment = "controller1-proxy Certificate" +subjectAltName = @controller1_alt_names +subjectKeyIdentifier = hash + +[controller1_distinguished_name] +CN = system:node:controller1 +O = system:nodes +C = US +ST = Washington +L = Seattle + +[controller1_alt_names] +IP.0 = 127.0.0.1 +IP.4 = 10.215.1.222 +IP.5 = 2620:11f:7001:7:ffff:ffff:0ad7:01de +DNS.0 = controller1 + +[controller2-proxy] +distinguished_name = controller2_distinguished_name +prompt = no +req_extensions = controller2_req_extensions + +[controller2_req_extensions] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth, serverAuth +keyUsage = critical, digitalSignature, keyEncipherment +nsCertType = client +nsComment = "controller2-proxy Certificate" +subjectAltName = @controller2_alt_names +subjectKeyIdentifier = hash + +[controller2_distinguished_name] +CN = system:node:controller2 +O = system:nodes +C = US +ST = Washington +L = Seattle + +[controller2_alt_names] +IP.0 = 127.0.0.1 +IP.1 = 10.215.1.223 +IP.2 = 2620:11f:7001:7:ffff:ffff:0ad7:01df +DNS.0 = controller2 diff --git a/nix/kubernetes/keys/package/k8s-ca/package.nix b/nix/kubernetes/keys/package/k8s-ca/package.nix new file mode 100644 index 0000000..6c9329b --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-ca/package.nix @@ -0,0 +1,37 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + stdenv, + openssl, + ca_name, + ca_config, + ... +}: +stdenv.mkDerivation (finalAttrs: { + name = "k8s-ca-${ca_name}"; + nativeBuildInputs = [ openssl ]; + buildInputs = [ ]; + + unpackPhase = "true"; + + buildPhase = '' + openssl genrsa -out "${ca_name}-ca.key" 4096 + + openssl req -x509 -new -sha512 -noenc \ + -key "${ca_name}-ca.key" -days 3653 \ + -config "${ca_config}" \ + -out "${ca_name}-ca.crt" + ''; + + installPhase = '' + mkdir "$out" + cp "${ca_name}-ca.crt" "${ca_name}-ca.key" $out/ + ''; +}) diff --git a/nix/kubernetes/keys/package/k8s-client-config/package.nix b/nix/kubernetes/keys/package/k8s-client-config/package.nix new file mode 100644 index 0000000..853c3df --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-client-config/package.nix @@ -0,0 +1,53 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + lib, + stdenv, + k8s, + kubectl, + config_name, + config_user, + config_server, + ... +}: +stdenv.mkDerivation (finalAttrs: { + name = "k8s-client-config-${config_name}"; + nativeBuildInputs = [ kubectl ]; + buildInputs = [ ]; + + unpackPhase = "true"; + + buildPhase = '' + kubectl config set-cluster kubernetes-the-hard-way \ + --certificate-authority=${k8s.ca.client}/client-ca.crt \ + --embed-certs=true \ + --server=${lib.strings.escapeShellArg config_server} \ + --kubeconfig=${config_name}.kubeconfig + + kubectl config set-credentials ${config_user} \ + --client-certificate=${k8s.keys."${config_name}"}/${config_name}.crt \ + --client-key=${k8s.keys."${config_name}"}/${config_name}.key \ + --embed-certs=true \ + --kubeconfig=${config_name}.kubeconfig + + kubectl config set-context default \ + --cluster=kubernetes-the-hard-way \ + --user=${config_user} \ + --kubeconfig=${config_name}.kubeconfig + + kubectl config use-context default \ + --kubeconfig=${config_name}.kubeconfig + ''; + + installPhase = '' + mkdir "$out" + cp "${config_name}.kubeconfig" $out/ + ''; +}) diff --git a/nix/kubernetes/keys/package/k8s-encryption-key/package.nix b/nix/kubernetes/keys/package/k8s-encryption-key/package.nix new file mode 100644 index 0000000..9855e32 --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-encryption-key/package.nix @@ -0,0 +1,56 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + pkgs, + stdenv, + runCommand, + ... +}: +let + to_yaml_file = ((import ../../../functions/to_yaml.nix) {inherit pkgs;}).to_yaml_file; + kube_encryption_key = runCommand "kube_encryption_key" { } '' + head -c 32 /dev/urandom | base64 -w 0 | tee $out + ''; + kube_encryption_config = { + kind = "EncryptionConfig"; + apiVersion = "v1"; + resources = [ + { + resources = [ "secrets" ]; + providers = [ + { + aescbc = { + keys = [ + { + name = "key1"; + secret = (builtins.readFile "${kube_encryption_key}"); + } + ]; + }; + } + { identity = { }; } + ]; + } + ]; + }; + kube_encryption_config_yaml = (to_yaml_file "encryption-config.yaml" kube_encryption_config); +in +stdenv.mkDerivation (finalAttrs: { + name = "k8s-encryption-key"; + nativeBuildInputs = [ ]; + buildInputs = [ ]; + + unpackPhase = "true"; + + installPhase = '' + mkdir "$out" + cp "${kube_encryption_config_yaml}" $out/encryption-config.yaml + ''; +}) diff --git a/nix/kubernetes/keys/package/k8s-keys/package.nix b/nix/kubernetes/keys/package/k8s-keys/package.nix new file mode 100644 index 0000000..b6b2b87 --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-keys/package.nix @@ -0,0 +1,31 @@ +{ + k8s, + runCommand, + symlinkJoin, + ... +}: +let + scripts = runCommand "scripts" { } '' + mkdir $out + cp ${k8s.deploy_script} $out/deploy_script + cp ${k8s.bootstrap_script} $out/bootstrap_script + ''; + mrmanager_repo_secrets = runCommand "mrmanager_repo_secrets" { } '' + mkdir $out + cp -r ${k8s.mrmanager_repo_secrets} $out/mrmanager_repo_secrets + ''; +in +symlinkJoin { + name = "k8s-keys"; + paths = [ + scripts + k8s.encryption_config + mrmanager_repo_secrets + ] + ++ (builtins.attrValues k8s.ca) + ++ (builtins.attrValues k8s.keys) + ++ (builtins.attrValues k8s.client-configs) + ++ (builtins.attrValues k8s.ssh-keys) + ++ (builtins.attrValues k8s.pgp-keys) + ++ (builtins.attrValues k8s.k8s-secrets-generic); +} diff --git a/nix/kubernetes/keys/package/k8s-secret-encrypted/package.nix b/nix/kubernetes/keys/package/k8s-secret-encrypted/package.nix new file mode 100644 index 0000000..6d2fbbc --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-secret-encrypted/package.nix @@ -0,0 +1,65 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + pkgs, + stdenv, + kubectl, + gnupg, + source_file, + output_filename, + pgp_public_key, + ... +}: +let + pgp_key_id_command = pkgs.runCommand "pgp_key_id_command" { } '' + mkdir keyring + export GNUPGHOME=$(readlink -f keyring) + ${gnupg}/bin/gpg --with-fingerprint --with-colons --keyid-format LONG "${pgp_public_key}" | grep '^pub' | cut -d ':' -f 5 > $out + ''; + pgp_key_id = builtins.readFile pgp_key_id_command; + sops_config = { + creation_rules = [ + { + "path_regex" = ".*.yaml"; + "encrypted_regex" = "^(data|stringData)$"; + "pgp" = pgp_key_id; + } + ]; + }; + settingsFormat = pkgs.formats.yaml { }; + yaml_body = settingsFormat.generate ".sops.yaml" sops_config; + yaml_file = pkgs.writeTextFile { + name = ".sops.yaml"; + text = (builtins.readFile yaml_body); + }; +in +stdenv.mkDerivation (finalAttrs: { + name = "k8s-secret-encrypted-${output_filename}"; + nativeBuildInputs = [ + kubectl + gnupg + ]; + buildInputs = [ ]; + + unpackPhase = "true"; + + buildPhase = '' + mkdir keyring + export GNUPGHOME=$(readlink -f keyring) + cat "${pgp_public_key}" | gpg --import + ''; + + installPhase = '' + set -x + export GNUPGHOME=$(readlink -f keyring) + mkdir "$out" + cat "${source_file}" | ${pkgs.sops}/bin/sops --config "${yaml_file}" encrypt --filename-override "${output_filename}" | tee "$out/${output_filename}" + ''; +}) diff --git a/nix/kubernetes/keys/package/k8s-secret-generic/package.nix b/nix/kubernetes/keys/package/k8s-secret-generic/package.nix new file mode 100644 index 0000000..c708d24 --- /dev/null +++ b/nix/kubernetes/keys/package/k8s-secret-generic/package.nix @@ -0,0 +1,60 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + lib, + pkgs, + stdenv, + k8s, + kubectl, + secret_name, + secret_namespace, + secret_values ? { }, + secret_type ? null, + secret_annotations ? null, + ... +}: +let + toBase64 = (pkgs.callPackage ../../contrib/base64/package.nix { inherit lib; }).toBase64; + metadata = { + name = "${secret_name}"; + namespace = "${secret_namespace}"; + } + // lib.optionalAttrs (secret_annotations != null) { + "annotations" = secret_annotations; + }; + secret_yaml = { + apiVersion = "v1"; + kind = "Secret"; + metadata = metadata; + data = (builtins.mapAttrs (key: val: (toBase64 val)) secret_values); + } + // lib.optionalAttrs (secret_type != null) { + "type" = secret_type; + }; + + settingsFormat = pkgs.formats.yaml { }; + yaml_body = settingsFormat.generate "${secret_name}.yaml" secret_yaml; + yaml_file = pkgs.writeTextFile { + name = "${secret_name}.yaml"; + text = (builtins.readFile yaml_body); + }; +in +stdenv.mkDerivation (finalAttrs: { + name = "k8s-secret-generic-${secret_name}"; + nativeBuildInputs = [ kubectl ]; + buildInputs = [ ]; + + unpackPhase = "true"; + + installPhase = '' + mkdir "$out" + cp "${yaml_file}" "$out/${secret_name}.yaml" + ''; +}) diff --git a/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix b/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix new file mode 100644 index 0000000..60ccdec --- /dev/null +++ b/nix/kubernetes/keys/package/mrmanager-repo-secrets/package.nix @@ -0,0 +1,310 @@ +{ + lib, + pkgs, + k8s, + callPackage, + runCommand, + symlinkJoin, + ... +}: +let + pre_encryption_secrets = + builtins.mapAttrs + ( + secret_namespace: secrets: + (builtins.mapAttrs ( + secret_name: original_secret_values: + let + secret_type = original_secret_values."__type" or null; + secret_annotations = original_secret_values."__annotations" or null; + secret_values = removeAttrs original_secret_values [ + "__type" + "__annotations" + ]; + in + (callPackage ../../package/k8s-secret-generic/package.nix { + inherit + secret_name + secret_namespace + secret_values + secret_type + secret_annotations + ; + }) + ) secrets) + ) + { + "archive-box" = { + "archive-box-auth" = { + "username" = (builtins.readFile "${./secrets/archive-box/archive-box-auth/username}"); + "password" = (builtins.readFile "${./secrets/archive-box/archive-box-auth/password}"); + }; + }; + "cert-manager" = { + "rfc2136" = { + "TSIG_SECRET" = (builtins.readFile "${./secrets/cert-manager/rfc2136/TSIG_SECRET}"); + }; + }; + "dex" = { + "files" = { + "config.yaml" = dex_config_yaml; + }; + }; + "external-dns" = { + "rfc2136" = { + "EXTERNAL_DNS_RFC2136_TSIG_SECRET" = ( + builtins.readFile "${./secrets/external-dns/rfc2136/EXTERNAL_DNS_RFC2136_TSIG_SECRET}" + ); + }; + }; + "flux-system" = { + "registry-credentials" = + (generate_docker_secret { + username = builtins.readFile "${./secrets/flux-system/registry-credentials/username}"; + password = builtins.readFile "${./secrets/flux-system/registry-credentials/password}"; + email = builtins.readFile "${./secrets/flux-system/registry-credentials/email}"; + }) + // { + # "__annotations" = { + # "tekton.dev/docker-0" = "https://harbor.fizz.buzz"; + # }; + }; + "webhook-token" = { + # This token is used for gitea webhooks + "token" = generate_key 64 "flux-system.webhook-token.token"; + }; + "harbor-webhook-token" = { + # This token is used for harbor webhooks + "token" = generate_key 64 "flux-system.harbor-webhook-token.token"; + }; + }; + "gitea" = { + "gitea-env" = { + "GITEA_ADMIN_USERNAME" = (builtins.readFile "${./secrets/gitea/gitea-env/GITEA_ADMIN_USERNAME}"); + "GITEA_ADMIN_PASSWORD" = (builtins.readFile "${./secrets/gitea/gitea-env/GITEA_ADMIN_PASSWORD}"); + }; + "oauth2-env" = oauth2_env { dex_id = "gitea"; }; + }; + "harbor" = { + "harbor-config" = { + "config.json" = helm_json_escape harbor_config_json; + }; + "dockerhub-auth-config" = { + "basic_auth.include" = ( + builtins.readFile "${./secrets/harbor/dockerhub-auth-config/basic_auth.include}" + ); + }; + "harbor-admin-password" = { + "HARBOR_ADMIN_PASSWORD" = ( + builtins.readFile "${./secrets/harbor/harbor-admin-password/HARBOR_ADMIN_PASSWORD}" + ); + }; + }; + "homepage-staging" = { + "oauth2-env" = oauth2_env { dex_id = "homepage-staging"; }; + }; + "tekton-gateway" = { + "oauth2-env" = oauth2_env { dex_id = "tekton"; }; + }; + "webhook-bridge" = { + "webhook-bridge" = { + "HMAC_TOKEN" = (builtins.readFile "${./secrets/webhook-bridge/webhook-bridge/HMAC_TOKEN}"); + "OAUTH_TOKEN" = (builtins.readFile "${./secrets/webhook-bridge/webhook-bridge/OAUTH_TOKEN}"); + }; + "deployer-key" = { + "__annotations" = { + "tekton.dev/git-0" = "code.fizz.buzz"; + }; + "__type" = "kubernetes.io/ssh-auth"; + "ssh-privatekey" = (builtins.readFile "${./secrets/webhook-bridge/deployer-key/ssh-privatekey}"); + "ssh-publickey" = (builtins.readFile "${./secrets/webhook-bridge/deployer-key/ssh-publickey}"); + }; + "gitea" = { + "token" = (builtins.readFile "${./secrets/webhook-bridge/gitea/token}"); + }; + "harbor-plain" = { + "config.json" = (builtins.readFile "${./secrets/webhook-bridge/harbor-plain/config.json}"); + }; + }; + }; + encrypted_secrets = ( + builtins.mapAttrs ( + secret_namespace: secrets: + (builtins.mapAttrs ( + secret_name: secret_package: + (callPackage ../../package/k8s-secret-encrypted/package.nix { + source_file = "${ + pre_encryption_secrets."${secret_namespace}"."${secret_name}" + }/${secret_name}.yaml"; + output_filename = "${secret_name}.yaml"; + pgp_public_key = "${k8s.pgp-keys.flux_gpg}/flux_gpg_public_key.asc"; + }) + ) secrets) + ) pre_encryption_secrets + ); + combined_script = ( + lib.concatMapStringsSep "\n" ( + secret_namespace: + '' + mkdir -p $out/${secret_namespace} + '' + + (lib.concatMapStringsSep "\n" (secret_name: '' + cat ${ + encrypted_secrets."${secret_namespace}"."${secret_name}" + }/${secret_name}.yaml > $out/${secret_namespace}/${secret_name}.yaml + '') (builtins.attrNames encrypted_secrets."${secret_namespace}")) + ) (builtins.attrNames encrypted_secrets) + ); + gen_in_repo_secrets = runCommand "gen_in_repo_secrets" { } combined_script; + + ## Utilities + inherit ((import ../../../functions/to_yaml.nix) { inherit pkgs; }) to_yaml; + inherit (pkgs.callPackage ../../contrib/base64/package.nix { inherit lib; }) toBase64; + generate_key = + len: name: + builtins.readFile ( + runCommand "generate_key" { } '' + set +o pipefail + # ${name} + dd if=/dev/urandom | tr --complement --delete '[:alnum:]' | dd bs=${toString len} count=1 of="$out" + '' + ); + # helm_json_escape = json: builtins.toJSON json; + helm_json_escape = + json: + builtins.replaceStrings + [ + "=" + "[" + "]" + "," + "." + "\"" + "{" + "}" + ] + [ + "\\=" + "\\[" + "\\]" + "\\," + "\\." + "\\\"" + "\\{" + "\\}" + ] + json; + generate_docker_secret = + { + username, + password, + email, + }: + let + in + { + "__type" = "kubernetes.io/dockerconfigjson"; + ".dockerconfigjson" = builtins.toJSON { + inherit username password email; + "auth" = toBase64 "${username}:${password}"; + }; + }; + ## dex + get_dex_config = + client_id: + (builtins.head ( + builtins.filter (static_client: static_client.id == client_id) dex_config.staticClients + )); + dex_static_client = + { + id, + name, + redirectURIs, + }: + { + inherit id name redirectURIs; + secret = generate_key 32 "dex_static_client ${id}"; + }; + dex_config = { + issuer = "https://dex.fizz.buzz"; + storage = { + config = { + inCluster = true; + }; + type = "kubernetes"; + }; + logger = { + level = "debug"; + }; + web = { + http = "0.0.0.0:5556"; + }; + oauth2 = { + alwaysShowLoginScreen = false; + skipApprovalScreen = true; + }; + staticClients = map dex_static_client [ + { + id = "prometheus"; + name = "Prometheus"; + redirectURIs = [ "https://prometheus.fizz.buzz/oauth2/callback" ]; + } + { + id = "harbor"; + name = "Harbor"; + redirectURIs = [ "https://harbor.fizz.buzz/c/oidc/callback" ]; + } + { + id = "tekton"; + name = "Tekton"; + redirectURIs = [ "https://tekton.fizz.buzz/oauth2/callback" ]; + } + { + id = "homepage-staging"; + name = "Homepage staging"; + redirectURIs = [ "https://staging.fizz.buzz/oauth2/callback" ]; + } + { + id = "gitea"; + name = "gitea"; + redirectURIs = [ "https://code.fizz.buzz/oauth2/callback" ]; + } + ]; + enablePasswordDB = true; + staticPasswords = (import ./secrets/dex/static_passwords.nix); + expiry = { + idTokens = "1h"; + signingKeys = "4h"; + }; + }; + dex_config_yaml = to_yaml "config.yml" dex_config; + + ## oauth2-proxy + oauth2_env = + { dex_id }: + { + "OAUTH2_PROXY_CLIENT_SECRET" = (get_dex_config dex_id).secret; + "OAUTH2_PROXY_COOKIE_SECRET" = generate_key 32 "OAUTH2_PROXY_COOKIE_SECRET ${dex_id}"; + }; + + ## harbor + harbor_dex_config = get_dex_config "harbor"; + harbor_config = { + "auth_mode" = "oidc_auth"; + "self_registration" = "false"; + "oidc_name" = "harbor"; + "oidc_endpoint" = "https://dex.fizz.buzz"; + "oidc_client_id" = harbor_dex_config.id; + "oidc_client_secret" = harbor_dex_config.secret; + "oidc_admin_group" = "TODO"; + "oidc_scope" = "openid,profile,email,offline_access,groups"; + }; + # harbor_config_json = pkgs.writeText "config.json" (builtins.toJSON harbor_config); + harbor_config_json = builtins.toJSON harbor_config; +in +symlinkJoin { + name = "in-repo-secrets"; + paths = [ + gen_in_repo_secrets + ]; +} diff --git a/nix/kubernetes/keys/package/pgp-key/package.nix b/nix/kubernetes/keys/package/pgp-key/package.nix new file mode 100644 index 0000000..fa69f12 --- /dev/null +++ b/nix/kubernetes/keys/package/pgp-key/package.nix @@ -0,0 +1,50 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + stdenv, + gnupg, + key_name, + expire_date ? "0", + pgp_comment ? "${key_name}", + pgp_name ? "${key_name}", + ... +}: +stdenv.mkDerivation (finalAttrs: { + name = "pgp-key-${key_name}"; + nativeBuildInputs = [ gnupg ]; + buildInputs = [ ]; + + unpackPhase = "true"; + + buildPhase = '' + mkdir keyring + export GNUPGHOME=$(readlink -f keyring) + + gpg --batch --full-generate-key < "$out/${key_name}_private_key.asc" + gpg --export --armor "${pgp_name}" > "$out/${key_name}_public_key.asc" + ''; +}) diff --git a/nix/kubernetes/keys/package/ssh-key/package.nix b/nix/kubernetes/keys/package/ssh-key/package.nix new file mode 100644 index 0000000..01a2366 --- /dev/null +++ b/nix/kubernetes/keys/package/ssh-key/package.nix @@ -0,0 +1,33 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + lib, + stdenv, + k8s, + openssh, + key_name, + ... +}: +stdenv.mkDerivation (finalAttrs: { + name = "ssh-key-${key_name}"; + nativeBuildInputs = [ openssh ]; + buildInputs = [ ]; + + unpackPhase = "true"; + + buildPhase = '' + ssh-keygen -t ed25519 -f ${key_name} -N "" + ''; + + installPhase = '' + mkdir "$out" + cp "${key_name}" "${key_name}.pub" $out/ + ''; +}) diff --git a/nix/kubernetes/keys/package/tls-key/package.nix b/nix/kubernetes/keys/package/tls-key/package.nix new file mode 100644 index 0000000..5daee36 --- /dev/null +++ b/nix/kubernetes/keys/package/tls-key/package.nix @@ -0,0 +1,47 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + stdenv, + openssl, + k8s, + key_name, + ca_name, + ca_config, + ... +}: +stdenv.mkDerivation (finalAttrs: { + name = "tls-key-${key_name}"; + nativeBuildInputs = [ openssl ]; + buildInputs = [ ]; + + unpackPhase = "true"; + + buildPhase = '' + cp ${k8s.ca."${ca_name}"}/${ca_name}-ca.crt ${k8s.ca."${ca_name}"}/${ca_name}-ca.key ./ + + openssl genrsa -out "${key_name}.key" 4096 + + openssl req -new -key "${key_name}.key" -sha256 \ + -config "${ca_config}" -section ${key_name} \ + -out "${key_name}.csr" + + openssl x509 -req -days 3653 -in "${key_name}.csr" \ + -copy_extensions copyall \ + -sha256 -CA "./${ca_name}-ca.crt" \ + -CAkey "./${ca_name}-ca.key" \ + -CAcreateserial \ + -out "${key_name}.crt" + ''; + + installPhase = '' + mkdir "$out" + cp "${key_name}.crt" "${key_name}.key" $out/ + ''; +}) diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix new file mode 100644 index 0000000..cd0ca35 --- /dev/null +++ b/nix/kubernetes/keys/scope.nix @@ -0,0 +1,378 @@ +{ + makeScope, + newScope, + callPackage, + fetchFromGitHub, + lib, +}: +let + public_addresses = [ + "74.80.180.138" + ]; + internal_addresses = [ + # nc0 + "10.215.1.221" + "2620:11f:7001:7:ffff:ffff:0ad7:01dd" + # nc1 + "10.215.1.222" + "2620:11f:7001:7:ffff:ffff:0ad7:01de" + # nc2 + "10.215.1.223" + "2620:11f:7001:7:ffff:ffff:0ad7:01df" + # nw0 + "10.215.1.224" + "2620:11f:7001:7:ffff:ffff:0ad7:01e0" + # nw1 + "10.215.1.225" + "2620:11f:7001:7:ffff:ffff:0ad7:01e1" + # nw2 + "10.215.1.226" + "2620:11f:7001:7:ffff:ffff:0ad7:01e2" + ]; + all_hostnames = [ + "10.197.0.1" + "10.0.0.1" + "127.0.0.1" + "kubernetes" + "kubernetes.default" + "kubernetes.default.svc" + "kubernetes.default.svc.cluster" + "kubernetes.svc.cluster.local" + ] + ++ public_addresses + ++ internal_addresses; + controllers = { + "controller0" = { + "internal_ips" = [ + "10.215.1.221" + "2620:11f:7001:7:ffff:ffff:0ad7:01dd" + ]; + "external_ips" = [ + "2620:11f:7001:7:ffff:ffff:0ad7:01dd" + ]; + }; + "controller1" = { + "internal_ips" = [ + "10.215.1.222" + "2620:11f:7001:7:ffff:ffff:0ad7:01de" + ]; + "external_ips" = [ + "2620:11f:7001:7:ffff:ffff:0ad7:01de" + ]; + }; + "controller2" = { + "internal_ips" = [ + "10.215.1.223" + "2620:11f:7001:7:ffff:ffff:0ad7:01df" + ]; + "external_ips" = [ + "2620:11f:7001:7:ffff:ffff:0ad7:01df" + ]; + }; + }; +in +makeScope newScope ( + self: + let + additional_vars = { + inherit all_hostnames controllers; + k8s = self; + }; + certificate_authorities = { + "client" = { + ca_config = ./package/k8s-ca/files/client-ca.conf; + }; + "requestheader-client" = { + ca_config = ./package/k8s-ca/files/requestheader-client-ca.conf; + }; + }; + certificate_authorities_merged = ( + builtins.mapAttrs (ca_name: ca_config: { inherit ca_name; } // ca_config) certificate_authorities + ); + in + { + ca = ( + builtins.mapAttrs ( + ca_name: ca_config: + (callPackage ./package/k8s-ca/package.nix (additional_vars // { inherit ca_name; } // ca_config)) + ) certificate_authorities + ); + keys = ( + builtins.mapAttrs + ( + key_name: key_config: + (callPackage ./package/tls-key/package.nix (additional_vars // { inherit key_name; } // key_config)) + ) + { + "admin" = { } // certificate_authorities_merged.client; + "controller0" = { } // certificate_authorities_merged.client; + "controller1" = { } // certificate_authorities_merged.client; + "controller2" = { } // certificate_authorities_merged.client; + "worker0" = { } // certificate_authorities_merged.client; + "worker1" = { } // certificate_authorities_merged.client; + "worker2" = { } // certificate_authorities_merged.client; + "kube-proxy" = { } // certificate_authorities_merged.client; + "kube-scheduler" = { } // certificate_authorities_merged.client; + "kube-controller-manager" = { } // certificate_authorities_merged.client; + "kube-api-server" = { } // certificate_authorities_merged.client; + "service-accounts" = { } // certificate_authorities_merged.client; + "controller0-proxy" = { } // certificate_authorities_merged.requestheader-client; + "controller1-proxy" = { } // certificate_authorities_merged.requestheader-client; + "controller2-proxy" = { } // certificate_authorities_merged.requestheader-client; + } + ); + ssh-keys = ( + lib.genAttrs [ + "flux_ssh_key" + ] (key_name: (callPackage ./package/ssh-key/package.nix (additional_vars // { inherit key_name; }))) + ); + pgp-keys = ( + builtins.mapAttrs + ( + key_name: key_config: + (callPackage ./package/pgp-key/package.nix (additional_vars // { inherit key_name; } // key_config)) + ) + { + "flux_gpg" = { + pgp_comment = "flux secrets"; + pgp_name = "flux sops"; + }; + } + ); + k8s-secrets-generic = ( + builtins.mapAttrs + ( + secret_name: secret_config: + (callPackage ./package/k8s-secret-generic/package.nix ( + additional_vars // { inherit secret_name; } // secret_config + )) + ) + { + "sops-gpg" = { + secret_namespace = "flux-system"; + secret_values = { + "sops.asc" = (builtins.readFile "${self.pgp-keys.flux_gpg}/flux_gpg_private_key.asc"); + }; + }; + "kubernetes-deploy-key" = { + secret_namespace = "flux-system"; + secret_values = { + "identity" = builtins.readFile "${self.ssh-keys.flux_ssh_key}/flux_ssh_key"; + "identity.pub" = builtins.readFile "${self.ssh-keys.flux_ssh_key}/flux_ssh_key.pub"; + "known_hosts" = builtins.readFile ./generated/known_hosts; + }; + }; + } + ); + client-configs = ( + builtins.mapAttrs + ( + config_name: config: + (callPackage ./package/k8s-client-config/package.nix ( + additional_vars // { inherit config_name; } // config + )) + ) + { + controller0 = { + config_user = "system:node:controller0"; + config_server = "https://127.0.0.1:6443"; + # config_server = "https://server.kubernetes.local:6443"; + }; + controller1 = { + config_user = "system:node:controller1"; + config_server = "https://127.0.0.1:6443"; + # config_server = "https://server.kubernetes.local:6443"; + }; + controller2 = { + config_user = "system:node:controller2"; + config_server = "https://127.0.0.1:6443"; + # config_server = "https://server.kubernetes.local:6443"; + }; + worker0 = { + config_user = "system:node:worker0"; + config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443"; + # config_server = "https://127.0.0.1:6443"; + # config_server = "https://server.kubernetes.local:6443"; + }; + worker1 = { + config_user = "system:node:worker1"; + config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443"; + # config_server = "https://127.0.0.1:6443"; + # config_server = "https://server.kubernetes.local:6443"; + }; + worker2 = { + config_user = "system:node:worker2"; + config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443"; + # config_server = "https://127.0.0.1:6443"; + # config_server = "https://server.kubernetes.local:6443"; + }; + kube-proxy = { + config_user = "system:kube-proxy"; + config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443"; + # config_server = "https://127.0.0.1:6443"; + # config_server = "https://server.kubernetes.local:6443"; + }; + kube-controller-manager = { + config_user = "system:kube-controller-manager"; + # config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443"; + config_server = "https://127.0.0.1:6443"; + # config_server = "https://server.kubernetes.local:6443"; + }; + kube-scheduler = { + config_user = "system:kube-scheduler"; + # config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443"; + config_server = "https://127.0.0.1:6443"; + # config_server = "https://server.kubernetes.local:6443"; + }; + admin = { + config_user = "admin"; + config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443"; + # config_server = "https://127.0.0.1:6443"; + }; + } + ); + encryption_config = (callPackage ./package/k8s-encryption-key/package.nix additional_vars); + cilium-manifest = + let + version = "1.19.1"; + in + (callPackage ./package/helm-manifest/package.nix ( + additional_vars + // { + helm_src = fetchFromGitHub { + owner = "cilium"; + repo = "cilium"; + tag = "v${version}"; + hash = "sha256-wswY4u2Z7Z8hvGVnLONxSD1Mu1RV1AglC4ijUHsCCW4="; + }; + helm_name = "cilium"; + helm_namespace = "kube-system"; + helm_path = "install/kubernetes/cilium"; + helm_manifest_name = "cilium.yaml"; + helm_values = { + "kubeProxyReplacement" = true; + "ipam" = { + "mode" = "kubernetes"; + }; + "k8sServiceHost" = "2620:11f:7001:7:ffff:ffff:ad7:1dd"; + "k8sServicePort" = 6443; + "ipv6" = { + "enabled" = true; + }; + "ipv4" = { + "enabled" = true; + }; + "externalIPs" = { + "enabled" = true; + }; + "enableIPv6Masquerade" = false; + "enableIPv4BIGTCP" = true; + "enableIPv6BIGTCP" = true; + "routingMode" = "native"; + "autoDirectNodeRoutes" = true; + "ipv4NativeRoutingCIDR" = "10.200.0.0/16"; + "ipv6NativeRoutingCIDR" = "2620:11f:7001:7:ffff:eeee::/96"; + # "ipv6NativeRoutingCIDR" = "2620:11f:7001:7:ffff::/80"; + # "l7Proxy" = true; # Needed for cilium gateway controller + + "hubble" = { + "relay" = { + "enabled" = true; + }; + "ui" = { + "enabled" = true; + }; + + "gatewayAPI" = { + "enabled" = true; + }; + }; + + # TODO: Read and maybe apply https://docs.cilium.io/en/stable/operations/performance/tuning/ + + # --set hostFirewall.enabled=true + + # --set 'ipam.operator.clusterPoolIPv4PodCIDRList=["10.0.0.0/8"]' \ + # --set 'ipam.operator.clusterPoolIPv6PodCIDRList=["fd00::/100"]' \ + + # --set encryption.enabled=true \ + # --set encryption.type=wireguard + # --set encryption.nodeEncryption=true + }; + } + )); + coredns-manifest = + let + version = "1.45.0"; + in + (callPackage ./package/helm-manifest/package.nix ( + additional_vars + // { + helm_src = fetchFromGitHub { + owner = "coredns"; + repo = "helm"; + tag = "coredns-${version}"; + hash = "sha256-9YHd/jB33JXvySzx/p9DaP+/2p5ucyLjues4DNtOkmU="; + }; + helm_name = "coredns"; + helm_namespace = "kube-system"; + helm_path = "charts/coredns"; + helm_manifest_name = "coredns.yaml"; + helm_values = { + "service" = { + "ipFamilyPolicy" = "PreferDualStack"; + "clusterIP" = "fd00:3e42:e349::10"; + "clusterIPs" = [ + "fd00:3e42:e349::10" + "10.197.0.10" + ]; + }; + + servers = [ + { + zones = [ + { + zone = "."; + use_tcp = true; + } + ]; + port = 53; + plugins = [ + { name = "errors"; } + { + name = "health"; + configBlock = "lameduck 10s"; + } + { name = "ready"; } + { + name = "kubernetes"; + parameters = "cluster.local in-addr.arpa ip6.arpa"; + configBlock = "pods insecure\nfallthrough in-addr.arpa ip6.arpa\nttl 30"; + } + { + name = "prometheus"; + parameters = "0.0.0.0:9153"; + } + { + name = "forward"; + parameters = ". /etc/resolv.conf"; + } + { + name = "cache"; + parameters = 300; # default 30 + } + { name = "loop"; } + { name = "reload"; } + { name = "loadbalance"; } + ]; + } + ]; + }; + } + )); + all_keys = (callPackage ./package/k8s-keys/package.nix additional_vars); + deploy_script = (callPackage ./package/deploy-script/package.nix additional_vars); + bootstrap_script = (callPackage ./package/bootstrap-script/package.nix additional_vars); + mrmanager_repo_secrets = (callPackage ./package/mrmanager-repo-secrets/package.nix additional_vars); + } +) diff --git a/nix/kubernetes/roles/boot/default.nix b/nix/kubernetes/roles/boot/default.nix new file mode 100644 index 0000000..035f747 --- /dev/null +++ b/nix/kubernetes/roles/boot/default.nix @@ -0,0 +1,130 @@ +# ISO does not work with systemd initrd yet https://github.com/NixOS/nixpkgs/pull/291750 +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + options.me = { + boot.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install boot."; + }; + + boot.secure = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Enable to use secure boot."; + }; + + rollback.enable = lib.mkOption { + type = lib.types.bool; + default = true; + example = true; + description = "Whether we want to enable rolling back during boot."; + }; + + rollback.dataset = lib.mkOption { + default = { }; + example = lib.literalExpression '' + { + "zroot/linux/nix/root@blank" = true; + "zroot/linux/nix/home@blank" = lib.mkForce false; + } + ''; + type = lib.types.coercedTo (lib.types.listOf lib.types.str) ( + enabled: lib.listToAttrs (map (fs: lib.nameValuePair fs true) enabled) + ) (lib.types.attrsOf lib.types.bool); + description = "List of ZFS datasets to rollback to during boot."; + }; + }; + + config = lib.mkIf config.me.boot.enable ( + lib.mkMerge [ + { + environment.systemPackages = with pkgs; [ + tpm2-tools # For tpm2_eventlog to check for OptionRoms + # cp /sys/kernel/security/tpm0/binary_bios_measurements eventlog + # tpm2_eventlog eventlog | grep "BOOT_SERVICES_DRIVER" + sbctl # For debugging and troubleshooting Secure Boot. + efibootmgr # To set EFI boot order. + ]; + } + (lib.mkIf (!config.me.buildingPortable) { + + boot.loader.grub.enable = false; + # Use the systemd-boot EFI boot loader. + boot.loader.systemd-boot.enable = true; + # TODO: make not write bootx64.efi + boot.loader.efi.canTouchEfiVariables = false; + + # Automatically delete old generations + boot.loader.systemd-boot.configurationLimit = 3; + + boot.loader.systemd-boot.memtest86.enable = true; + + # Check what will be lost with `zfs diff zroot/linux/root@blank` + boot.initrd.systemd.enable = lib.mkDefault true; + boot.initrd.systemd.services.zfs-rollback = lib.mkIf config.me.rollback.enable { + description = "Rollback ZFS root dataset to blank snapshot"; + wantedBy = [ + "initrd.target" + ]; + after = [ + "zfs-import-zroot.service" + ]; + before = [ + "sysroot.mount" + ]; + unitConfig.DefaultDependencies = "no"; + serviceConfig.Type = "oneshot"; + script = lib.concatStringsSep "\n" ( + (builtins.map (ds: "${config.boot.zfs.package}/sbin/zfs rollback -r '${ds}'") ( + builtins.attrNames config.me.rollback.dataset + )) + ++ [ ''echo "rollback complete"'' ] + ); + }; + + # boot.loader.systemd-boot.extraEntries = { + # "windows.conf" = '' + # title Windows + # efi /EFI/Microsoft/Boot/bootmgfw.efi + # options root=PARTUUID=17e325bf-a378-4d1d-be6a-f6df5476f0fa + # ''; + # }; + environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) { + hideMounts = true; + directories = [ + "/var/lib/sbctl" # Secure Boot Keys + ]; + }; + }) + (lib.mkIf (config.me.boot.secure) { + environment.systemPackages = with pkgs; [ + sbctl + ]; + boot.loader.systemd-boot.enable = lib.mkForce false; + boot.lanzaboote = { + enable = true; + pkiBundle = "/var/lib/sbctl"; + }; + }) + ] + ); +} +# efibootmgr -c -d /dev/sda -p 1 -L NixOS-boot -l '\EFI\NixOS-boot\grubx64.efi' + +# Text-only: +# sudo cp "$(nix-build '' --no-out-link -A 'refind')/share/refind/refind_x64.efi" /boot/EFI/boot/bootx64.efi + +# Full graphics: +# $ sudo nix-shell -p refind efibootmgr +# $ refind-install diff --git a/nix/kubernetes/roles/cilium/default.nix b/nix/kubernetes/roles/cilium/default.nix new file mode 100644 index 0000000..86a0439 --- /dev/null +++ b/nix/kubernetes/roles/cilium/default.nix @@ -0,0 +1,29 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + options.me = { + cilium.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install cilium."; + }; + }; + + config = lib.mkIf config.me.cilium.enable { + networking.firewall.allowedTCPPorts = [ + 4240 # Health checks + ]; + networking.firewall.allowedUDPPorts = [ + 8472 # vxlan + 51871 # wireguard + ]; + }; +} diff --git a/nix/kubernetes/roles/containerd/default.nix b/nix/kubernetes/roles/containerd/default.nix new file mode 100644 index 0000000..a45a914 --- /dev/null +++ b/nix/kubernetes/roles/containerd/default.nix @@ -0,0 +1,86 @@ +# TODO: Set up a proxy to harbor for OCI compliance: https://github.com/moby/moby/pull/34319#issuecomment-720606627 +{ + config, + lib, + pkgs, + ... +}: + +let + my-cni-plugins = pkgs.buildEnv { + name = "my-cni-plugins"; + paths = with pkgs; [ + cni-plugins + # cni-plugin-flannel + ]; + }; + my-cni-configs = pkgs.callPackage ./package/cni_conf/package.nix { }; +in +{ + imports = [ ]; + + options.me = { + containerd.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install containerd."; + }; + }; + + config = lib.mkIf config.me.containerd.enable { + virtualisation.containerd.enable = true; + virtualisation.containerd.settings = lib.mkForce { + "plugins" = { + "io.containerd.cri.v1.images" = { + "registry" = { + "config_path" = "/.persist/containerd/certs.d"; + }; + "snapshotter" = "overlayfs"; + }; + "io.containerd.cri.v1.runtime" = { + "cni" = { + "bin_dirs" = [ + "/opt/cni/bin" + ]; + "conf_dir" = "/etc/cni/net.d"; + }; + "containerd" = { + "default_runtime_name" = "runc"; + "runtimes" = { + "runc" = { + "runtime_type" = "io.containerd.runc.v2"; + }; + }; + }; + }; + "io.containerd.cri.v1.services" = { + "containerd" = { + "runtimes" = { + "runc" = { + "options" = { + "SystemdCgroup" = true; + }; + }; + }; + }; + }; + }; + "version" = 3; + }; + + systemd.services.containerd.preStart = '' + ${pkgs.toybox}/bin/install -d -m 0755 /opt/cni/bin /etc/cni/net.d + ${pkgs.toybox}/bin/install ${my-cni-plugins}/bin/* /opt/cni/bin/ + ${pkgs.toybox}/bin/install ${my-cni-configs}/* /etc/cni/net.d/ + echo "Copied CNI plugins/config." + ''; + + environment.persistence."/disk" = lib.mkIf (config.me.mountPersistence) { + hideMounts = lib.mkForce false; + directories = [ + "/var/lib/containerd" + ]; + }; + }; +} diff --git a/nix/kubernetes/roles/containerd/package/cni_conf/files/10-bridge.conf b/nix/kubernetes/roles/containerd/package/cni_conf/files/10-bridge.conf new file mode 100644 index 0000000..e9a3bff --- /dev/null +++ b/nix/kubernetes/roles/containerd/package/cni_conf/files/10-bridge.conf @@ -0,0 +1,15 @@ +{ + "cniVersion": "1.0.0", + "name": "bridge", + "type": "bridge", + "bridge": "cni0", + "isGateway": true, + "ipMasq": true, + "ipam": { + "type": "host-local", + "ranges": [ + [{"subnet": "SUBNET"}] + ], + "routes": [{"dst": "0.0.0.0/0"}] + } +} \ No newline at end of file diff --git a/nix/kubernetes/roles/containerd/package/cni_conf/files/99-loopback.conf b/nix/kubernetes/roles/containerd/package/cni_conf/files/99-loopback.conf new file mode 100644 index 0000000..98d6dc1 --- /dev/null +++ b/nix/kubernetes/roles/containerd/package/cni_conf/files/99-loopback.conf @@ -0,0 +1,5 @@ +{ + "cniVersion": "1.1.0", + "name": "lo", + "type": "loopback" +} \ No newline at end of file diff --git a/nix/kubernetes/roles/containerd/package/cni_conf/package.nix b/nix/kubernetes/roles/containerd/package/cni_conf/package.nix new file mode 100644 index 0000000..6476c94 --- /dev/null +++ b/nix/kubernetes/roles/containerd/package/cni_conf/package.nix @@ -0,0 +1,27 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + stdenv, + openssl, + ... +}: +stdenv.mkDerivation (finalAttrs: { + name = "cni-conf"; + nativeBuildInputs = [ openssl ]; + buildInputs = [ ]; + + unpackPhase = "true"; + + installPhase = '' + mkdir -p "$out" + cd "$out" + install ${./files/10-bridge.conf} ${./files/99-loopback.conf} $out/ + ''; +}) diff --git a/nix/kubernetes/roles/control_plane/default.nix b/nix/kubernetes/roles/control_plane/default.nix new file mode 100644 index 0000000..ec8a760 --- /dev/null +++ b/nix/kubernetes/roles/control_plane/default.nix @@ -0,0 +1,27 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + options.me = { + control_plane.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install control_plane."; + }; + }; + + config = lib.mkIf config.me.control_plane.enable { + me.firewall.enable = true; + me.kube_apiserver.enable = true; + me.kube_controller_manager.enable = true; + me.kube_scheduler.enable = true; + me.kubernetes.enable = true; + }; +} diff --git a/nix/kubernetes/roles/debugging/default.nix b/nix/kubernetes/roles/debugging/default.nix new file mode 100644 index 0000000..35b9cd1 --- /dev/null +++ b/nix/kubernetes/roles/debugging/default.nix @@ -0,0 +1,35 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + options.me = { + debugging.enable = lib.mkOption { + type = lib.types.bool; + default = true; + example = true; + description = "Whether we want to install debugging."; + }; + }; + + config = lib.mkIf config.me.debugging.enable { + environment.systemPackages = with pkgs; [ + net-tools # for netstat + tcpdump + e2fsprogs # mkfs.ext4 + gptfdisk # cgdisk + arp-scan # To find devices on the network + ldns # for drill + ]; + + # This can make debugging easier by rejecting packets instead of dropping them: + networking.firewall.rejectPackets = true; + # Log each rejected packet instead of just each connection. + networking.firewall.logRefusedPackets = true; + }; +} diff --git a/nix/kubernetes/roles/doas/default.nix b/nix/kubernetes/roles/doas/default.nix new file mode 100644 index 0000000..47e1773 --- /dev/null +++ b/nix/kubernetes/roles/doas/default.nix @@ -0,0 +1,36 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + options.me = { + doas.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install doas."; + }; + }; + + config = lib.mkIf config.me.doas.enable { + # Use doas instead of sudo + security.doas.enable = true; + security.doas.wheelNeedsPassword = false; + security.sudo.enable = false; + security.doas.extraRules = [ + { + # Retain environment (for example NIX_PATH) + keepEnv = true; + persist = true; # Only ask for a password the first time. + } + ]; + environment.systemPackages = with pkgs; [ + doas-sudo-shim # To support --sudo for remote builds + ]; + }; +} diff --git a/nix/kubernetes/roles/dont_use_substituters/default.nix b/nix/kubernetes/roles/dont_use_substituters/default.nix new file mode 100644 index 0000000..0982ff2 --- /dev/null +++ b/nix/kubernetes/roles/dont_use_substituters/default.nix @@ -0,0 +1,25 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + options.me = { + dont_use_substituters.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install dont_use_substituters."; + }; + }; + + config = lib.mkIf config.me.dont_use_substituters.enable { + # Disable substituters to avoid risk of cache poisoning. + nix.settings.substitute = false; + nix.settings.substituters = lib.mkForce [ ]; + }; +} diff --git a/nix/kubernetes/roles/etcd/default.nix b/nix/kubernetes/roles/etcd/default.nix new file mode 100644 index 0000000..f086315 --- /dev/null +++ b/nix/kubernetes/roles/etcd/default.nix @@ -0,0 +1,101 @@ +{ + config, + lib, + pkgs, + self, + ... +}: + +{ + imports = [ ]; + + options.me = { + etcd.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install etcd."; + }; + + etcd.cluster_name = lib.mkOption { + type = lib.types.str; + default = false; + example = "lorem"; + description = "The unique name for the cluster."; + }; + + etcd.internal_ip = lib.mkOption { + default = { }; + example = lib.literalExpression '' + { + "172.16.0.10" = true; + "192.168.1.10" = lib.mkForce false; + } + ''; + type = lib.types.coercedTo (lib.types.listOf lib.types.str) ( + enabled: lib.listToAttrs (map (fs: lib.nameValuePair fs true) enabled) + ) (lib.types.attrsOf lib.types.bool); + description = "List internal IP addresses for accessing this node."; + }; + + etcd.initial_cluster = lib.mkOption { + default = [ ]; + example = [ + "controller0=https://10.215.1.221:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01dd + "controller1=https://10.215.1.222:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01de + "controller2=https://10.215.1.223:2380" # 2620:11f:7001:7:ffff:ffff:0ad7:01df + ]; + type = lib.types.listOf lib.types.str; + description = "List of controller nodes to form the initial etcd cluster."; + }; + }; + + config = lib.mkIf config.me.etcd.enable { + services.etcd = { + enable = true; + openFirewall = true; + name = config.networking.hostName; + certFile = "/.persist/keys/etcd/kube-api-server.crt"; + keyFile = "/.persist/keys/etcd/kube-api-server.key"; + peerCertFile = "/.persist/keys/etcd/kube-api-server.crt"; + peerKeyFile = "/.persist/keys/etcd/kube-api-server.key"; + trustedCaFile = "/.persist/keys/etcd/client-ca.crt"; + peerTrustedCaFile = "/.persist/keys/etcd/client-ca.crt"; + peerClientCertAuth = true; + clientCertAuth = true; + initialAdvertisePeerUrls = ( + builtins.map (iip: "https://${iip}:2380") (builtins.attrNames config.me.etcd.internal_ip) + ); + listenPeerUrls = ( + builtins.map (iip: "https://${iip}:2380") (builtins.attrNames config.me.etcd.internal_ip) + ); + listenClientUrls = ( + [ + "https://127.0.0.1:2379" + ] + ++ (builtins.map (iip: "https://${iip}:2379") (builtins.attrNames config.me.etcd.internal_ip)) + ); + advertiseClientUrls = ( + builtins.map (iip: "https://${iip}:2379") (builtins.attrNames config.me.etcd.internal_ip) + ); + initialClusterToken = config.me.etcd.cluster_name; + initialCluster = config.me.etcd.initial_cluster; + initialClusterState = "new"; + }; + + environment.persistence."/disk" = lib.mkIf (config.me.mountPersistence) { + hideMounts = true; + directories = [ + { + directory = config.services.etcd.dataDir; # "/var/lib/etcd" + user = "etcd"; + group = "etcd"; + mode = "0700"; + } + ]; + }; + + users.users.etcd.uid = 10016; + users.groups.etcd.gid = 10016; + }; +} diff --git a/nix/kubernetes/roles/firewall/default.nix b/nix/kubernetes/roles/firewall/default.nix new file mode 100644 index 0000000..d69f1c8 --- /dev/null +++ b/nix/kubernetes/roles/firewall/default.nix @@ -0,0 +1,99 @@ +{ + config, + lib, + ... +}: + +{ + imports = [ ]; + + options.me = { + firewall.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install firewall."; + }; + }; + + config = lib.mkIf config.me.firewall.enable { + # kernel modules and settings required by Kubernetes + boot.kernelModules = [ + "overlay" + "br_netfilter" + ]; + boot.kernel.sysctl = { + "net.bridge.bridge-nf-call-iptables" = 1; + "net.bridge.bridge-nf-call-ip6tables" = 1; + "net.ipv4.ip_forward" = 1; + + # Enable forwarding on all interfaces. + # "net.ipv4.conf.all.forwarding" = 1; + # "net.ipv6.conf.all.forwarding" = 1; + }; + + networking.firewall.enable = true; + networking.nftables.enable = true; + # We want to filter forwarded traffic. + # Also needed for `networking.firewall.extraForwardRules` to do anything. + networking.firewall.filterForward = true; + + # Allow traffic from the pods on the lxc interfaces even though the interfaces do not have the correct ip addressses set for the return path. + networking.firewall.extraReversePathFilterRules = '' + iifname "lxc*" ip6 saddr 2620:11f:7001:7:ffff:eeee::/96 accept + iifname "lxc*" ip saddr 10.200.0.0/16 accept + ''; + + networking.firewall.extraInputRules = builtins.concatStringsSep "\n" [ + # Allow pod-to-node communication + '' + ip6 saddr 2620:11f:7001:7:ffff:eeee::/96 accept + '' + # Allow node-to-node communication + # Needed for: + # - metallb port 7946 + '' + iifname "enp*" ip saddr 10.215.1.0/24 ip daddr 10.215.1.0/24 accept + iifname "enp*" ip6 saddr 2620:11f:7001:7:ffff:ffff:0ad7:0100/120 ip6 daddr 2620:11f:7001:7:ffff:ffff:0ad7:0100/120 accept + '' + ]; + + # networking.firewall.extraInputRules = '' + # ip6 saddr 2620:11f:7001:7:ffff:eeee::/96 accept + # ip6 saddr fd00:3e42:e349::/112 accept + # ip6 saddr 2620:11f:7001:7:ffff:ffff:0ad7:0100/120 accept + # ''; + + networking.firewall.extraForwardRules = builtins.concatStringsSep "\n" [ + # Allow pod to external communication + '' + iifname "lxc*" ip6 saddr 2620:11f:7001:7:ffff:eeee::/96 accept + iifname "lxc*" ip saddr 10.200.0.0/16 accept + '' + # Allow pod-to-pod communication + '' + ip saddr 10.200.0.0/16 ip daddr 10.200.0.0/16 accept + ip6 saddr 2620:11f:7001:7:ffff:eeee::/96 ip6 daddr 2620:11f:7001:7:ffff:eeee::/96 accept + '' + # Allow external-to-pod communication + '' + ip daddr 10.200.0.0/16 accept + ip6 daddr 2620:11f:7001:7:ffff:eeee::/96 accept + '' + ]; + + # networking.firewall.extraForwardRules = '' + # ip6 daddr 2620:11f:7001:7:ffff:eeee::/96 accept + # ip6 daddr fd00:3e42:e349::/112 accept + # ip6 daddr 2620:11f:7001:7:ffff:ffff:0ad7:0100/120 accept + # ''; + + # Check logs for blocked connections: + # journalctl -k or dmesg + + # networking.nftables.tables."my-fw" = { + # family = "inet"; + # content = (builtins.readFile ./files/my-fw.nft); + # }; + }; +} diff --git a/nix/kubernetes/roles/firewall/files/my-fw.nft b/nix/kubernetes/roles/firewall/files/my-fw.nft new file mode 100644 index 0000000..61609cd --- /dev/null +++ b/nix/kubernetes/roles/firewall/files/my-fw.nft @@ -0,0 +1,153 @@ +set internal-iface { + type ifname + # Tell the kernel this set may include ranges, so it can pick the best datastructure. + flags interval + elements = { + "lxc*", + "cilium_net" + } +} + +set node-cidr-ipv4 { + type ipv4_addr + flags constant, interval + elements = { 10.215.1.0/24 } +} + +set node-cidr-ipv6 { + type ipv6_addr + flags constant, interval + elements = { 2620:11f:7001:7:ffff:ffff:0ad7:0100/120 } +} + +set service-cidr-ipv4 { + type ipv4_addr + flags constant, interval + elements = { 10.197.0.0/16 } +} + +set service-cidr-ipv6 { + type ipv6_addr + flags constant, interval + elements = { fd00:3e42:e349::/112 } +} + +set pod-cidr-ipv4 { + type ipv4_addr + flags constant, interval + elements = { 10.200.0.0/16 } +} + +set pod-cidr-ipv6 { + type ipv6_addr + flags constant, interval + elements = { 2620:11f:7001:7:ffff:eeee::/96 } +} + +set public-ports { + # These are open to all IP addresses + type inet_proto . inet_service + flags constant + elements = { + tcp . 22, # ssh + tcp . 6443 # kubernetes API server + } +} + +set node-to-node-ports { + # Ports open for nodes sending packets to nodes + type inet_proto . inet_service + flags constant, interval + elements = { + tcp . 2379-2380, # etcd + tcp . 4240, # cilium health monitoring + tcp . 10250, # kubelet API + # udp . 51871, # cilium wireguard + # tcp . 30000-32767, # nodeport range + # udp . 30000-32767, # nodeport range + # udp . 8472, # cilium vxlan + tcp . 7946, # MetalLB memberlist + udp . 7946 # MetalLB memberlist + } +} + +set pod-to-node-ports { + # Ports open for nodes sending packets to nodes + type inet_proto . inet_service + flags constant, interval + elements = { + tcp . 4244 # hubble ui + } +} + +chain rpfilter { + type filter hook prerouting priority mangle + 10; policy drop; + meta nfproto ipv4 udp sport . udp dport { 68 . 67, 67 . 68 } accept comment "DHCPv4 client/server" + # Reverse path forwarding filter. Check that a route exists back to the source address on the interface which received the packet. If the packet came from the wrong interface, then the packet is likely spoofed. + fib saddr . mark . iif check exists accept + jump rpfilter-allow + meta pkttype host log prefix "Failed rpfilter: " level info +} + +chain rpfilter-allow { + # Allow packets on internal interfaces from pods + meta iifname @internal-iface ip saddr @pod-cidr-ipv4 accept + meta iifname @internal-iface ip6 saddr @pod-cidr-ipv6 accept +} + + +chain input { + type filter hook input priority filter; policy drop; + iifname "lo" accept comment "trusted interfaces" + # Drop invalid connections, accept packets for established or related connections and send packets for new or untracked connections to the input-allow chain + ct state vmap { invalid : drop, established : accept, related : accept, new : jump input-allow, untracked : jump input-allow } + # If the packet is a new connection and reaches this point, then we are going to reject it. So log that rejection. + tcp flags & (fin | syn | rst | ack) == syn log prefix "refused connection: " level info + # Log rejected packets destined for this machine (as opposed to packets being routed or broadcast packets) + meta pkttype host log prefix "refused packet: " level info + # When rejecting packets, send a TCP Reset (RST) instead of simply dropping the packet. + meta l4proto tcp reject with tcp reset + # Reject any packets that make it here. + reject +} + +chain input-allow { + # Allow pings. + icmp type echo-request accept comment "allow ping" + icmpv6 type != { nd-redirect, 139 } accept comment "Accept all ICMPv6 messages except redirects and node information queries (type 139). See RFC 4890, section 4.4." + ip6 daddr fe80::/64 udp dport 546 accept comment "DHCPv6 client" + + # Allow public ports + meta l4proto . th dport @public-ports accept + + # Allow node to node + ip saddr @node-cidr-ipv4 ip daddr @node-cidr-ipv4 meta l4proto . th dport @node-to-node-ports accept + ip6 saddr @node-cidr-ipv6 ip6 daddr @node-cidr-ipv6 meta l4proto . th dport @node-to-node-ports accept + + # Allow pod to node + ip saddr @pod-cidr-ipv4 ip daddr @node-cidr-ipv4 meta l4proto . th dport @pod-to-node-ports accept + ip6 saddr @pod-cidr-ipv6 ip6 daddr @node-cidr-ipv6 meta l4proto . th dport @pod-to-node-ports accept +} + +chain forward { + type filter hook forward priority filter; policy drop; + # Drop invalid connections, accept packets for established or related connections and send packets for new or untracked connections to the forward-allow chain + ct state vmap { invalid : drop, established : accept, related : accept, new : jump forward-allow, untracked : jump forward-allow } + + log prefix "blocked forwarding packet: " level info +} + +chain forward-allow { + icmpv6 type != { router-renumbering, 139 } accept comment "Accept all ICMPv6 messages except renumbering and node information queries (type 139). See RFC 4890, section 4.3." + + # When connection tracking (ct) shows the status as destination nat (dnat) then accept the packet. + ct status dnat accept comment "allow port forward" + + # Allow packets from pods + ip saddr @pod-cidr-ipv4 accept + ip6 saddr @pod-cidr-ipv6 accept + + # Allow node-to-pod + ip saddr @node-cidr-ipv4 ip daddr @pod-cidr-ipv4 accept + ip6 saddr @node-cidr-ipv6 ip6 daddr @pod-cidr-ipv6 accept +} diff --git a/nix/kubernetes/roles/image_based_appliance/default.nix b/nix/kubernetes/roles/image_based_appliance/default.nix new file mode 100644 index 0000000..9257580 --- /dev/null +++ b/nix/kubernetes/roles/image_based_appliance/default.nix @@ -0,0 +1,30 @@ +{ + config, + lib, + ... +}: + +{ + imports = [ ]; + + options.me = { + image_based_appliance.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install image_based_appliance."; + }; + }; + + config = lib.mkIf config.me.image_based_appliance.enable ( + lib.mkMerge [ + { + # Do not install nix. A full new image must be built to update + # the machine. + nix.enable = false; + system.switch.enable = false; + nix.gc.automatic = lib.mkForce false; + } + ] + ); +} diff --git a/nix/kubernetes/roles/iso/default.nix b/nix/kubernetes/roles/iso/default.nix new file mode 100644 index 0000000..f09f8c7 --- /dev/null +++ b/nix/kubernetes/roles/iso/default.nix @@ -0,0 +1,22 @@ +{ + lib, + ... +}: + +{ + imports = [ ]; + + options.me.buildingPortable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we are building a portable image (iso/sd). This would disable CPU-specific optimizations and persistent file mounts."; + }; + + options.me.mountPersistence = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we should mount persistent directories."; + }; +} diff --git a/nix/kubernetes/roles/kernel/default.nix b/nix/kubernetes/roles/kernel/default.nix new file mode 100644 index 0000000..7979a68 --- /dev/null +++ b/nix/kubernetes/roles/kernel/default.nix @@ -0,0 +1,192 @@ +# Check current config: +# nix build '/persist/machine_setup/nix/configuration#nixosConfigurations.hydra.pkgs.linux_me.configfile' +# cat $(nix eval --raw '/persist/machine_setup/nix/configuration#nixosConfigurations.hydra.pkgs.linux_me.configfile') | less + +{ + config, + lib, + pkgs, + ... +}: + +let + preemption_type = with lib.kernel; { + full = { + PREEMPT_DYNAMIC = yes; + PREEMPT = yes; + PREEMPT_VOLUNTARY = lib.mkForce no; + PREEMPT_LAZY = lib.mkForce no; + PREEMPT_NONE = no; + }; + lazy = { + PREEMPT_DYNAMIC = yes; + PREEMPT = no; + PREEMPT_VOLUNTARY = lib.mkForce no; + PREEMPT_LAZY = yes; + PREEMPT_NONE = no; + }; + voluntary = { + PREEMPT_DYNAMIC = no; + PREEMPT = no; + PREEMPT_VOLUNTARY = yes; + PREEMPT_LAZY = lib.mkForce no; + PREEMPT_NONE = no; + }; + none = { + PREEMPT_DYNAMIC = no; + PREEMPT = no; + PREEMPT_VOLUNTARY = lib.mkForce no; + PREEMPT_LAZY = lib.mkForce no; + PREEMPT_NONE = yes; + }; + }; + tick_hz = + with lib.kernel; + { + "1000" = { + HZ_1000 = yes; + HZ = freeform "1000"; + }; + } + // lib.genAttrs [ "100" "250" "300" "500" "600" "750" ] (hz: { + HZ_1000 = no; + "HZ_${hz}" = yes; + HZ = freeform hz; + }); + performance_governor = with lib.kernel; { + default = { + CPU_FREQ_DEFAULT_GOV_SCHEDUTIL = yes; + }; + performance = { + CPU_FREQ_DEFAULT_GOV_SCHEDUTIL = no; + CPU_FREQ_DEFAULT_GOV_PERFORMANCE = yes; + }; + }; + tick_rate = with lib.kernel; { + # Always tick at the hz frequency. + periodic = { + NO_HZ_IDLE = no; + NO_HZ_FULL = no; + NO_HZ = no; + NO_HZ_COMMON = no; + HZ_PERIODIC = yes; + }; + # Idle - Do not disturb the CPU when idle. This can save power but increase latency. + idle = { + HZ_PERIODIC = no; + NO_HZ_FULL = no; + NO_HZ_IDLE = yes; + NO_HZ = yes; + NO_HZ_COMMON = yes; + }; + # Full dyntick system (tickless) - The kernel tries to shut down the tick whenever possible. + tickless = { + HZ_PERIODIC = no; + NO_HZ_IDLE = no; + NO_HZ_FULL = yes; + NO_HZ = yes; + NO_HZ_COMMON = yes; + CONTEXT_TRACKING = yes; + }; + }; + huge_page = with lib.kernel; { + always = { + TRANSPARENT_HUGEPAGE_MADVISE = no; + TRANSPARENT_HUGEPAGE_ALWAYS = yes; + }; + madvise = { + TRANSPARENT_HUGEPAGE_ALWAYS = no; + TRANSPARENT_HUGEPAGE_MADVISE = yes; + }; + }; + common_config = with lib.kernel; { + # Google's BBRv3 TCP congestion Control + TCP_CONG_BBR = yes; + DEFAULT_BBR = yes; + }; + flavors = { + server = lib.mkMerge [ + preemption_type.none + tick_hz."300" + performance_governor.default + tick_rate.tickless + huge_page.madvise + ]; + interactive = + with lib.kernel; + lib.mkMerge [ + { + # Enable RCU Lazy - Reduces power consumption when idle or lightly loaded. Useful for battery-powered devices like laptops. + RCU_LAZY = yes; + } + preemption_type.lazy + tick_hz."300" + performance_governor.default + tick_rate.tickless + huge_page.madvise + ]; + }; +in +{ + imports = [ ]; + + options.me = { + kernel.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install kernel."; + }; + + kernel.version = lib.mkOption { + type = lib.types.str; + default = "linux"; # LTS + example = "linux_6_18"; + description = "What version of the kernl should we use."; + }; + + kernel.flavor = lib.mkOption { + type = lib.types.str; + default = "server"; + example = "interactive"; + description = "What type of kernel should be built."; + }; + }; + + config = lib.mkIf config.me.kernel.enable ( + lib.mkMerge [ + { + boot.kernelPackages = pkgs.linuxPackagesFor pkgs.linux_me; + } + (lib.mkIf (!config.me.optimizations.enable) { + nixpkgs.overlays = [ + (final: prev: { + linux_me = final."${config.me.kernel.version}"; + }) + ]; + }) + (lib.mkIf (config.me.optimizations.enable) { + nixpkgs.overlays = [ + ( + final: prev: + let + addConfig = + additionalConfig: pkg: + pkg.override (oldconfig: { + structuredExtraConfig = lib.mkMerge ([ pkg.structuredExtraConfig ] ++ additionalConfig); + # stdenv = pkgs.llvmPackages_latest.stdenv; + # stdenv = pkgs.clangStdenv; + }); + in + { + linux_me = addConfig ([ + common_config + flavors."${config.me.kernel.flavor}" + ]) final."${config.me.kernel.version}"; + } + ) + ]; + }) + ] + ); +} diff --git a/nix/kubernetes/roles/kube_apiserver/default.nix b/nix/kubernetes/roles/kube_apiserver/default.nix new file mode 100644 index 0000000..267f642 --- /dev/null +++ b/nix/kubernetes/roles/kube_apiserver/default.nix @@ -0,0 +1,157 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + # shellCommand = cmd: (lib.concatMapStringsSep " " lib.strings.escapeShellArg cmd); + shellCommand = cmd: (builtins.concatStringsSep " " cmd); +in +{ + imports = [ ]; + + options.me = { + kube_apiserver.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install kube_apiserver."; + }; + + kube_apiserver.internal_ip = lib.mkOption { + # default = { }; + example = "192.168.1.10"; + type = lib.types.str; + description = "IP address this server should advertise."; + }; + + kube_apiserver.external_ip = lib.mkOption { + example = "192.168.1.10"; + type = lib.types.str; + description = "IP address to reach this cluster externally."; + }; + + kube_apiserver.etcd_services = lib.mkOption { + default = [ ]; + example = [ "https://192.168.1.10:2379" ]; + type = lib.types.listOf lib.types.str; + description = "Endpoints for etcd servers."; + }; + }; + + config = lib.mkIf config.me.kube_apiserver.enable { + systemd.services.kube-apiserver = { + enable = true; + description = "Kubernetes API Server"; + documentation = [ "https://github.com/kubernetes/kubernetes" ]; + wantedBy = [ "kubernetes.target" ]; + # path = with pkgs; [ + # zfs + # ]; + unitConfig.DefaultDependencies = "no"; + serviceConfig = { + Type = "notify"; + ExecStart = ( + shellCommand [ + # NEW: + "${pkgs.kubernetes}/bin/kube-apiserver" + "--advertise-address=${config.me.kube_apiserver.external_ip}" + "--allow-privileged=true" + "--audit-log-maxage=30" + "--audit-log-maxbackup=3" + "--audit-log-maxsize=100" + "--audit-log-path=/var/log/audit.log" + "--authorization-mode=Node,RBAC" + "--bind-address=0.0.0.0" + "--client-ca-file=/.persist/keys/kube/client-ca.crt" + "--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota" + "--etcd-cafile=/.persist/keys/kube/client-ca.crt" + "--etcd-certfile=/.persist/keys/kube/kube-api-server.crt" + "--etcd-keyfile=/.persist/keys/kube/kube-api-server.key" + "--etcd-servers=${builtins.concatStringsSep "," config.me.kube_apiserver.etcd_services}" + "--event-ttl=1h" + "--encryption-provider-config=/.persist/keys/kube/encryption-config.yaml" + "--kubelet-certificate-authority=/.persist/keys/kube/client-ca.crt" + "--kubelet-client-certificate=/.persist/keys/kube/kube-api-server.crt" + "--kubelet-client-key=/.persist/keys/kube/kube-api-server.key" + "--runtime-config='api/all=true'" + "--service-account-key-file=/.persist/keys/kube/service-accounts.crt" + "--service-account-signing-key-file=/.persist/keys/kube/service-accounts.key" + "--service-account-issuer=https://server.kubernetes.local:6443" + "--service-node-port-range=30000-32767" + "--tls-cert-file=/.persist/keys/kube/kube-api-server.crt" + "--tls-private-key-file=/.persist/keys/kube/kube-api-server.key" + "--tls-min-version=VersionTLS13" + "--service-cluster-ip-range=fd00:3e42:e349::/112,10.197.0.0/16" + "--requestheader-client-ca-file=/.persist/keys/kube/requestheader-client-ca.crt" + "--requestheader-allowed-names=\"\"" # CN must be in this list to be valid. Blank = accept all CN. + "--requestheader-extra-headers-prefix=X-Remote-Extra" + "--requestheader-group-headers=X-Remote-Group" + "--requestheader-username-headers=X-Remote-User" + "--proxy-client-cert-file=/.persist/keys/kube/proxy.crt" + "--proxy-client-key-file=/.persist/keys/kube/proxy.key" + "--enable-aggregator-routing=true" + "--v=2" + + # OLD: + # "${pkgs.kubernetes}/bin/kube-apiserver" + # "--advertise-address=${config.me.kube_apiserver.internal_ip}" + # "--allow-privileged=true" + # "--apiserver-count=3" + # "--audit-log-maxage=30" + # "--audit-log-maxbackup=3" + # "--audit-log-maxsize=100" + # "--audit-log-path=/var/log/audit.log" + # "--authorization-mode=Node,RBAC" + # "--bind-address=0.0.0.0" + # "--client-ca-file=/.persist/keys/kube/ca.pem" + # "--requestheader-client-ca-file=/.persist/keys/kube/requestheader-client-ca.pem" + # ''--requestheader-allowed-names=""'' + # "--requestheader-extra-headers-prefix=X-Remote-Extra-" + # "--requestheader-group-headers=X-Remote-Group" + # "--requestheader-username-headers=X-Remote-User" + # "--proxy-client-cert-file=/.persist/keys/kube/${config.networking.hostName}-proxy.pem" + # "--proxy-client-key-file=/.persist/keys/kube/${config.networking.hostName}-proxy-key.pem" + # "--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota" + # "--etcd-cafile=/.persist/keys/kube/ca.pem" + # "--etcd-certfile=/.persist/keys/kube/kubernetes.pem" + # "--etcd-keyfile=/.persist/keys/kube/kubernetes-key.pem" + # "--etcd-servers=${builtins.concatStringsSep "," config.me.kube_apiserver.etcd_services}" + # "--event-ttl=1h" + # "--encryption-provider-config=/.persist/keys/kube/encryption-config.yaml" + # "--kubelet-certificate-authority=/.persist/keys/kube/ca.pem" + # "--kubelet-client-certificate=/.persist/keys/kube/kubernetes.pem" + # "--kubelet-client-key=/.persist/keys/kube/kubernetes-key.pem" + # "--runtime-config='api/all=true'" + # "--service-account-key-file=/.persist/keys/kube/service-account.pem" + # "--service-account-signing-key-file=/.persist/keys/kube/service-account-key.pem" + # "--service-account-issuer=https://${config.me.kube_apiserver.external_ip}:6443" + # "--service-node-port-range=30000-32767" + # "--tls-cert-file=/.persist/keys/kube/kubernetes.pem" + # "--tls-private-key-file=/.persist/keys/kube/kubernetes-key.pem" + # "--tls-min-version=VersionTLS13" + # "--kubelet-preferred-address-types=InternalIP,ExternalDNS,ExternalIP,Hostname,InternalDNS" + # # "--service-cluster-ip-range=10.197.0.0/16" + # "--service-cluster-ip-range=2620:11f:7001:7:ffff:ffff:0ac5:0000/16" + # "--enable-aggregator-routing=true" + # "--v=2" + ] + ); + Restart = "on-failure"; + RestartSec = 5; + LimitNOFILE = 65536; + User = "kubernetes"; + }; + }; + + networking.firewall.allowedTCPPorts = [ + 6443 + ]; + + systemd.tmpfiles.rules = [ + "f /var/log/audit.log 0600 kubernetes kubernetes - -" + ]; + }; +} diff --git a/nix/kubernetes/roles/kube_controller_manager/default.nix b/nix/kubernetes/roles/kube_controller_manager/default.nix new file mode 100644 index 0000000..10eaff6 --- /dev/null +++ b/nix/kubernetes/roles/kube_controller_manager/default.nix @@ -0,0 +1,70 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + # shellCommand = cmd: (lib.concatMapStringsSep " " lib.strings.escapeShellArg cmd); + shellCommand = cmd: (builtins.concatStringsSep " " cmd); +in +{ + imports = [ ]; + + options.me = { + kube_controller_manager.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install kube_controller_manager."; + }; + }; + + config = lib.mkIf config.me.kube_controller_manager.enable { + systemd.services.kube-controller-manager = { + enable = true; + description = "Kubernetes Controller Manager"; + documentation = [ "https://github.com/kubernetes/kubernetes" ]; + wantedBy = [ "kubernetes.target" ]; + after = [ "kube-apiserver.service" ]; + # path = with pkgs; [ + # zfs + # ]; + unitConfig.DefaultDependencies = "no"; + serviceConfig = { + ExecStart = ( + shellCommand [ + "${pkgs.kubernetes}/bin/kube-controller-manager" + "--bind-address=0.0.0.0" + # "--cluster-cidr=10.200.0.0/16" + # "--cluster-cidr=2620:11f:7001:7:ffff:ffff:0ac8:0000/96" + "--allocate-node-cidrs=true" + "--cluster-cidr=10.200.0.0/16,2620:11f:7001:7:ffff:eeee::/96" + "--node-cidr-mask-size-ipv4=20" # default is 24 + "--node-cidr-mask-size-ipv6=112" # default is 64, must be smaller than cluster-cidr mask + "--cluster-name=kubernetes" + "--cluster-signing-cert-file=/.persist/keys/kube/client-ca.crt" + "--cluster-signing-key-file=/.persist/keys/kube/client-ca.key" + "--kubeconfig=/.persist/keys/kube/kube-controller-manager.kubeconfig" + "--root-ca-file=/.persist/keys/kube/client-ca.crt" + "--service-account-private-key-file=/.persist/keys/kube/service-accounts.key" + # "--service-cluster-ip-range=10.197.0.0/16" + # "--service-cluster-ip-range=2620:11f:7001:7:ffff:ffff:0ac5:0000/16" + "--service-cluster-ip-range=fd00:3e42:e349::/112,10.197.0.0/16" + "--use-service-account-credentials=true" + "--v=2" + ] + ); + Restart = "on-failure"; + RestartSec = 5; + LimitNOFILE = 65536; + User = "kubernetes"; + }; + }; + + networking.firewall.allowedTCPPorts = [ + 10257 + ]; + }; +} diff --git a/nix/kubernetes/roles/kube_proxy/default.nix b/nix/kubernetes/roles/kube_proxy/default.nix new file mode 100644 index 0000000..340044f --- /dev/null +++ b/nix/kubernetes/roles/kube_proxy/default.nix @@ -0,0 +1,72 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + # shellCommand = cmd: (lib.concatMapStringsSep " " lib.strings.escapeShellArg cmd); + shellCommand = cmd: (builtins.concatStringsSep " " cmd); + settingsFormat = pkgs.formats.yaml { }; + config_file = settingsFormat.generate "kube-proxy-config.yaml" config.me.kube-proxy.settings; +in +{ + imports = [ ]; + + options.me = { + kube-proxy.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install kube-proxy."; + }; + + kube-proxy.settings = lib.mkOption { + type = settingsFormat.type; + default = { + kind = "KubeProxyConfiguration"; + apiVersion = "kubeproxy.config.k8s.io/v1alpha1"; + clientConnection = { + kubeconfig = "/.persist/keys/kube/kube-proxy.kubeconfig"; + }; + mode = "iptables"; + # clusterCIDR = "10.200.0.0/16"; + # clusterCIDR = "2620:11f:7001:7:ffff:ffff:0ac8:0000/16"; + clusterCIDR = "10.200.0.0/16,2620:11f:7001:7:ffff:eeee::/96"; + }; + description = '' + kubelet-config.yaml + ''; + }; + }; + + config = lib.mkIf config.me.kube-proxy.enable { + systemd.services.kube-proxy = { + enable = true; + description = "Kubernetes Kube Proxy"; + documentation = [ "https://github.com/kubernetes/kubernetes" ]; + wantedBy = [ "kubernetes.target" ]; + path = with pkgs; [ + iptables + ]; + unitConfig.DefaultDependencies = "no"; + serviceConfig = { + ExecStart = ( + shellCommand [ + "${pkgs.kubernetes}/bin/kube-proxy" + "--config=${config_file}" + "--nodeport-addresses=primary" + "--cluster-cidr=10.200.0.0/16,2620:11f:7001:7:ffff:eeee::/96" + ] + ); + Restart = "on-failure"; + RestartSec = 5; + }; + }; + + networking.firewall.allowedTCPPorts = [ + 10256 + ]; + }; +} diff --git a/nix/kubernetes/roles/kube_scheduler/default.nix b/nix/kubernetes/roles/kube_scheduler/default.nix new file mode 100644 index 0000000..ed701e5 --- /dev/null +++ b/nix/kubernetes/roles/kube_scheduler/default.nix @@ -0,0 +1,55 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + # shellCommand = cmd: (lib.concatMapStringsSep " " lib.strings.escapeShellArg cmd); + shellCommand = cmd: (builtins.concatStringsSep " " cmd); +in +{ + imports = [ ]; + + options.me = { + kube_scheduler.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install kube_scheduler."; + }; + }; + + config = lib.mkIf config.me.kube_scheduler.enable { + systemd.services.kube-scheduler = { + enable = true; + description = "Kubernetes Scheduler"; + documentation = [ "https://github.com/kubernetes/kubernetes" ]; + wantedBy = [ "kubernetes.target" ]; + after = [ "kube-apiserver.service" ]; + # path = with pkgs; [ + # zfs + # ]; + unitConfig.DefaultDependencies = "no"; + serviceConfig = { + ExecStart = ( + shellCommand [ + # NEW: + "${pkgs.kubernetes}/bin/kube-scheduler" + "--config=${./files/kube-scheduler.yaml}" + "--v=2" + ] + ); + Restart = "on-failure"; + RestartSec = 5; + LimitNOFILE = 65536; + User = "kubernetes"; + }; + }; + + networking.firewall.allowedTCPPorts = [ + 10259 + ]; + }; +} diff --git a/nix/kubernetes/roles/kube_scheduler/files/kube-scheduler.yaml b/nix/kubernetes/roles/kube_scheduler/files/kube-scheduler.yaml new file mode 100644 index 0000000..3881453 --- /dev/null +++ b/nix/kubernetes/roles/kube_scheduler/files/kube-scheduler.yaml @@ -0,0 +1,6 @@ +apiVersion: kubescheduler.config.k8s.io/v1 +kind: KubeSchedulerConfiguration +clientConnection: + kubeconfig: "/.persist/keys/kube/kube-scheduler.kubeconfig" +leaderElection: + leaderElect: true diff --git a/nix/kubernetes/roles/kubelet/default.nix b/nix/kubernetes/roles/kubelet/default.nix new file mode 100644 index 0000000..1227f3d --- /dev/null +++ b/nix/kubernetes/roles/kubelet/default.nix @@ -0,0 +1,104 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + # shellCommand = cmd: (lib.concatMapStringsSep " " lib.strings.escapeShellArg cmd); + shellCommand = cmd: (builtins.concatStringsSep " " cmd); + to_yaml_file = ((import ../../functions/to_yaml.nix) { inherit pkgs; }).to_yaml_file; + + kubelet_config = { + kind = "KubeletConfiguration"; + apiVersion = "kubelet.config.k8s.io/v1beta1"; + address = "0.0.0.0"; + authentication = { + anonymous = { + enabled = false; + }; + webhook = { + enabled = true; + }; + x509 = { + clientCAFile = "/.persist/keys/kube/client-ca.crt"; + }; + }; + authorization = { + mode = "Webhook"; + }; + cgroupDriver = "systemd"; + containerRuntimeEndpoint = "unix:///var/run/containerd/containerd.sock"; + enableServer = true; + failSwapOn = false; + maxPods = 110; + memorySwap = { + swapBehavior = "NoSwap"; + }; + port = 10250; + resolvConf = "/run/systemd/resolve/resolv.conf"; + registerNode = true; + runtimeRequestTimeout = "15m"; + tlsCertFile = "/.persist/keys/kube/kubelet.crt"; + tlsPrivateKeyFile = "/.persist/keys/kube/kubelet.key"; + clusterDomain = "cluster.local"; + clusterDNS = [ + "10.197.0.10" + "fd00:3e42:e349::10" + ]; + imageMaximumGCAge = "24h"; # Delete unused images after 1 day. + }; + kubelet_config_file = (to_yaml_file "kubelet-config.yaml" kubelet_config); +in +{ + imports = [ ]; + + options.me = { + kubelet.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install kubelet."; + }; + }; + + config = lib.mkIf config.me.kubelet.enable { + systemd.services.kubelet = { + enable = true; + description = "Kubernetes Kubelet"; + documentation = [ "https://github.com/kubernetes/kubernetes" ]; + wantedBy = [ "kubernetes.target" ]; + after = [ "containerd.service" ]; + requires = [ "containerd.service" ]; + path = with pkgs; [ + util-linux + ]; + unitConfig.DefaultDependencies = "no"; + serviceConfig = { + ExecStart = ( + shellCommand [ + "${pkgs.kubernetes}/bin/kubelet" + "--config=${kubelet_config_file}" + "--kubeconfig=/.persist/keys/kube/kubelet.kubeconfig" + "--v=2" + ] + ); + Restart = "on-failure"; + RestartSec = 5; + # ConfigurationDirectory = "kubernetes"; + # CPUAccounting = "true"; + # IPAccounting = "true"; + # KillMode = "process"; + # MemoryAccounting = "true"; + # StartLimitInterval = 0; + # RuntimeDirectory = "kubelet"; + # StateDirectory = "kubelet"; + }; + }; + + networking.firewall.allowedTCPPorts = [ + 10250 + ]; + }; +} diff --git a/nix/kubernetes/roles/kubelet/files/resolv.conf b/nix/kubernetes/roles/kubelet/files/resolv.conf new file mode 100644 index 0000000..a91e07e --- /dev/null +++ b/nix/kubernetes/roles/kubelet/files/resolv.conf @@ -0,0 +1,4 @@ +search svc.cluster.local cluster.local +nameserver 10.197.0.10 +nameserver fd00:3e42:e349::10 +options ndots:5 diff --git a/nix/kubernetes/roles/kubernetes/default.nix b/nix/kubernetes/roles/kubernetes/default.nix new file mode 100644 index 0000000..0f268d8 --- /dev/null +++ b/nix/kubernetes/roles/kubernetes/default.nix @@ -0,0 +1,61 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + options.me = { + kubernetes.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install kubernetes."; + }; + }; + + config = lib.mkIf config.me.kubernetes.enable { + assertions = [ + { + # Kubernetes should only upgrade 1 minor version at a time, so this assert is here to prevent unwittingly jumping versions. + assertion = lib.hasPrefix "1.36." pkgs.kubernetes.version; + message = "Unexpected Kubernetes package version: ${pkgs.kubernetes.version}"; + } + ]; + + environment.systemPackages = with pkgs; [ + kubernetes + ]; + + systemd.targets.kubernetes = { + description = "Kubernetes"; + wantedBy = [ "multi-user.target" ]; + }; + + users.users.kubernetes = { + uid = 10024; + description = "Kubernetes"; + group = "kubernetes"; + home = "/var/lib/kubernetes"; + createHome = true; + homeMode = "755"; + isSystemUser = true; + }; + users.groups.kubernetes.gid = 10024; + + environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) { + hideMounts = true; + directories = [ + { + directory = "/var/lib/kubernetes"; + user = "kubernetes"; + group = "kubernetes"; + mode = "0755"; + } + ]; + }; + }; +} diff --git a/nix/kubernetes/roles/minimal_base/default.nix b/nix/kubernetes/roles/minimal_base/default.nix new file mode 100644 index 0000000..54261f2 --- /dev/null +++ b/nix/kubernetes/roles/minimal_base/default.nix @@ -0,0 +1,32 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + options.me = { + minimal_base.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install minimal_base."; + }; + }; + + config = lib.mkIf config.me.minimal_base.enable { + me.doas.enable = true; + me.kernel.enable = true; + me.network.enable = true; + me.nvme.enable = true; + me.ssh.enable = true; + me.sshd.enable = true; + me.user.enable = true; + me.zsh.enable = true; + + # TODO: Maybe add me.boot.enable ? + }; +} diff --git a/nix/kubernetes/roles/network/default.nix b/nix/kubernetes/roles/network/default.nix new file mode 100644 index 0000000..db04bed --- /dev/null +++ b/nix/kubernetes/roles/network/default.nix @@ -0,0 +1,90 @@ +{ + config, + lib, + pkgs, + ... +}: + +# Alternative DNS servers: +# "1.0.0.1#cloudflare-dns.com" +# "1.1.1.1#cloudflare-dns.com" +# "2606:4700:4700::1001#cloudflare-dns.com" +# "2606:4700:4700::1111#cloudflare-dns.com" +# "8.8.4.4#dns.google" +# "8.8.8.8#dns.google" +# "2001:4860:4860::8844#dns.google" +# "2001:4860:4860::8888#dns.google" + +{ + imports = [ ]; + + options.me = { + network.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install network."; + }; + }; + + config = lib.mkIf config.me.network.enable { + networking.dhcpcd.enable = lib.mkDefault false; + networking.useDHCP = lib.mkDefault false; + # Nameservers configured in host-specific files. + # networking.nameservers = [ + # "194.242.2.2#doh.mullvad.net" + # "2a07:e340::2#doh.mullvad.net" + # ]; + networking.nameservers = [ + "10.215.1.1" + "2620:11f:7001:7:ffff:ffff:0ad7:0101" + ]; + services.resolved = { + enable = true; + # dnssec = "true"; + settings.Resolve.Domains = [ "~." ]; + settings.Resolve.FallbackDNS = [ ]; + # settings.Resolve.DNSOverTLS = "true"; + }; + + # Without this, systemd-resolved will send DNS requests for .home.arpa to the per-link DNS server (172.16.0.1) which does not support DNS-over-TLS. This leads to the connection hanging and timing out. This causes firefox startup to take an extra 10+ seconds. + # + # Test with: drill @127.0.0.53 odo.home.arpa + # TODO: The 127.0.0.1 address should probably be moved to a host-specific file. + networking.extraHosts = '' + 127.0.0.1 ${config.networking.hostName}.home.arpa + 2620:11f:7001:7:ffff:ffff:0ad7:01dd controller0.kubernetes.local controller0 + 2620:11f:7001:7:ffff:ffff:0ad7:01de controller1.kubernetes.local controller1 + 2620:11f:7001:7:ffff:ffff:0ad7:01df controller2.kubernetes.local controller2 + 2620:11f:7001:7:ffff:ffff:0ad7:01e0 worker0.kubernetes.local worker0 + 2620:11f:7001:7:ffff:ffff:0ad7:01e1 worker1.kubernetes.local worker1 + 2620:11f:7001:7:ffff:ffff:0ad7:01e2 worker2.kubernetes.local worker2 + ''; + + boot.extraModprobeConfig = '' + # Set wifi to US + options cfg80211 ieee80211_regdom=US + ''; + + boot.kernel.sysctl = { + # Enable TCP packetization-layer PMTUD when an ICMP black hole is detected. + "net.ipv4.tcp_mtu_probing" = 1; + # Switch to bbr tcp congestion control which should be better on lossy connections like bad wifi. + # We set this in the kernel config, but include this here for unoptimized builds. + "net.ipv4.tcp_congestion_control" = "bbr"; + # Don't do a slow start after a connection has been idle for a single RTO. + "net.ipv4.tcp_slow_start_after_idle" = 0; + # 3x time to accumulate filesystem changes before flushing to disk. + "vm.dirty_writeback_centisecs" = 1500; + # Adjust ttl + "net.ipv4.ip_default_ttl" = 65; + "net.ipv6.conf.all.hop_limit" = 65; + "net.ipv6.conf.default.hop_limit" = 65; + # Enable IPv6 Privacy Extensions + "net.ipv6.conf.all.use_tempaddr" = 2; + # Enable IPv6 Privacy Extensions + # This is enabled by default in nixos. + # "net.ipv6.conf.default.use_tempaddr" = 2; + }; + }; +} diff --git a/nix/kubernetes/roles/nvme/default.nix b/nix/kubernetes/roles/nvme/default.nix new file mode 100644 index 0000000..646ae6d --- /dev/null +++ b/nix/kubernetes/roles/nvme/default.nix @@ -0,0 +1,25 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + options.me = { + nvme.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install nvme."; + }; + }; + + config = lib.mkIf config.me.nvme.enable { + environment.systemPackages = with pkgs; [ + nvme-cli + ]; + }; +} diff --git a/nix/kubernetes/roles/optimized_build/default.nix b/nix/kubernetes/roles/optimized_build/default.nix new file mode 100644 index 0000000..65fd074 --- /dev/null +++ b/nix/kubernetes/roles/optimized_build/default.nix @@ -0,0 +1,98 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + options.me = { + optimizations.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to enable CPU optimizations (will trigger a rebuild from source)."; + }; + + optimizations.arch = lib.mkOption { + type = lib.types.str; + default = null; + example = "znver4"; + description = "The CPU arch for which programs should be optimized."; + }; + + optimizations.build_arch = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "znver4"; + description = "The CPU arch for which programs should be optimized."; + }; + + optimizations.system_features = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + example = [ + "gccarch-znver4" + "gccarch-znver5" + "gccarch-skylake" + "gccarch-x86-64-v3" + "gccarch-x86-64-v4" + "benchmark" + "big-parallel" + "kvm" + "nixos-test" + ]; + description = "The list of CPU features that should be enabled on this machine."; + }; + }; + + config = lib.mkMerge [ + (lib.mkIf config.me.optimizations.enable ( + lib.mkMerge [ + { + nixpkgs.hostPlatform = { + gcc.arch = config.me.optimizations.arch; + gcc.tune = config.me.optimizations.arch; + }; + + # nixpkgs.overlays = [ + # (final: prev: { + # inherit (final.unoptimized) + # assimp + # binaryen + # gsl + # rapidjson + # ffmpeg-headless + # ffmpeg + # pipewire + # chromaprint + # gtkmm + # ; + # }) + # ]; + } + ] + )) + (lib.mkIf (config.me.optimizations.build_arch != null) ( + lib.mkMerge [ + { + # Enable cross-compiling + nixpkgs.buildPlatform = { + gcc.arch = config.me.optimizations.build_arch; + gcc.tune = "generic"; + system = "x86_64-linux"; + }; + } + ] + )) + (lib.mkIf (config.me.optimizations.system_features != [ ]) ( + lib.mkMerge [ + { + nix.settings.system-features = lib.mkForce config.me.optimizations.system_features; + } + ] + )) + ]; +} diff --git a/nix/kubernetes/roles/ssh/default.nix b/nix/kubernetes/roles/ssh/default.nix new file mode 100644 index 0000000..8532dd1 --- /dev/null +++ b/nix/kubernetes/roles/ssh/default.nix @@ -0,0 +1,50 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + options.me = { + ssh.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install ssh."; + }; + }; + + config = lib.mkIf config.me.ssh.enable { + environment.systemPackages = with pkgs; [ + sshfs + ]; + + environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) { + hideMounts = true; + users.talexander = { + files = [ + ".ssh/known_hosts" + ]; + }; + users.root = { + files = [ + ".ssh/known_hosts" + ]; + }; + }; + + me.install.user.root.file = { + ".ssh/config" = { + source = ./files/ssh_config_root; + }; + }; + me.install.user.talexander.file = { + ".ssh/config" = { + source = ./files/ssh_config; + }; + }; + }; +} diff --git a/nix/kubernetes/roles/ssh/files/ssh_config b/nix/kubernetes/roles/ssh/files/ssh_config new file mode 100644 index 0000000..6b6fa9f --- /dev/null +++ b/nix/kubernetes/roles/ssh/files/ssh_config @@ -0,0 +1,42 @@ +Host poudriere + ProxyJump talexander@mrmanager + HostName 10.215.1.203 + +Host controller0 + ProxyJump talexander@mrmanager + HostName 10.215.1.204 + +Host controller1 + ProxyJump talexander@mrmanager + HostName 10.215.1.205 + +Host controller2 + ProxyJump talexander@mrmanager + HostName 10.215.1.206 + +Host worker0 + ProxyJump talexander@mrmanager + HostName 10.215.1.207 + +Host worker1 + ProxyJump talexander@mrmanager + HostName 10.215.1.208 + +Host worker2 + ProxyJump talexander@mrmanager + HostName 10.215.1.209 + +Host brianai + ProxyJump talexander@mrmanager + HostName 10.215.1.215 + +Host hydra + ProxyJump talexander@mrmanager + HostName 10.215.1.219 + +Host i_only_boot_zfs + HostName 127.0.0.1 + Port 60022 + +Host * + Compression yes diff --git a/nix/kubernetes/roles/ssh/files/ssh_config_root b/nix/kubernetes/roles/ssh/files/ssh_config_root new file mode 100644 index 0000000..0d340b3 --- /dev/null +++ b/nix/kubernetes/roles/ssh/files/ssh_config_root @@ -0,0 +1,9 @@ +Host hydra + HostName ns1.fizz.buzz + Port 65122 + User nixworker + IdentitiesOnly yes + IdentityFile /persist/manual/ssh/root/keys/id_ed25519 + +Host * + Compression yes diff --git a/nix/kubernetes/roles/sshd/default.nix b/nix/kubernetes/roles/sshd/default.nix new file mode 100644 index 0000000..928a059 --- /dev/null +++ b/nix/kubernetes/roles/sshd/default.nix @@ -0,0 +1,49 @@ +{ + config, + lib, + ... +}: + +{ + imports = [ ]; + + options.me = { + sshd.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install sshd."; + }; + }; + + config = lib.mkIf config.me.sshd.enable { + services.openssh = { + enable = true; + settings = { + PasswordAuthentication = false; + KbdInteractiveAuthentication = false; + }; + hostKeys = [ + { + path = "/persist/ssh/ssh_host_ed25519_key"; + type = "ed25519"; + } + { + path = "/persist/ssh/ssh_host_rsa_key"; + type = "rsa"; + bits = 4096; + } + ]; + }; + + environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) { + hideMounts = true; + files = [ + "/etc/ssh/ssh_host_rsa_key" + "/etc/ssh/ssh_host_rsa_key.pub" + "/etc/ssh/ssh_host_ed25519_key" + "/etc/ssh/ssh_host_ed25519_key.pub" + ]; + }; + }; +} diff --git a/nix/kubernetes/roles/user/default.nix b/nix/kubernetes/roles/user/default.nix new file mode 100644 index 0000000..4d7d45d --- /dev/null +++ b/nix/kubernetes/roles/user/default.nix @@ -0,0 +1,59 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + options.me = { + user.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to create my user."; + }; + }; + + config = lib.mkIf config.me.user.enable { + services.getty = { + autologinUser = "talexander"; # I use full disk encryption so the user password is irrelevant. + autologinOnce = true; + }; + users.mutableUsers = false; + users.users.talexander = { + isNormalUser = true; + createHome = true; # https://github.com/NixOS/nixpkgs/issues/6481 + group = "talexander"; + extraGroups = [ "wheel" ]; + uid = 11235; + packages = with pkgs; [ + tree + ]; + # Generate with `mkpasswd -m scrypt` + hashedPassword = "$7$CU..../....VXvNQ8za3wSGpdzGXNT50/$HcFtn/yvwPMCw4888BelpiAPLAxe/zU87fD.d/N6U48"; + openssh.authorizedKeys.keys = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID0+4zi26M3eYWnIrciR54kOlGxzfgCXG+o4ea1zpzrk openpgp:0x7FF123C8" + "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIEI6mu6I5Jp+Ib0vJxapGHbEShZjyvzV8jz5DnzDrI39AAAABHNzaDo=" + "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIAFNcSXwvy+brYTOGo56G93Ptuq2MmZsjvRWAfMqbmMLAAAABHNzaDo=" + ]; + }; + users.groups.talexander.gid = 11235; + + environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) { + hideMounts = true; + users.talexander = { + directories = [ + { + directory = "persist"; + user = "talexander"; + group = "talexander"; + mode = "0700"; + } + ]; + }; + }; + }; +} diff --git a/nix/kubernetes/roles/worker_node/default.nix b/nix/kubernetes/roles/worker_node/default.nix new file mode 100644 index 0000000..804d6f8 --- /dev/null +++ b/nix/kubernetes/roles/worker_node/default.nix @@ -0,0 +1,44 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + options.me = { + worker_node.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install worker_node."; + }; + }; + + config = lib.mkIf config.me.worker_node.enable { + me.cilium.enable = true; + me.containerd.enable = true; + me.firewall.enable = true; + # me.kube-proxy.enable = true; + me.kubelet.enable = true; + me.kubernetes.enable = true; + + networking.firewall.allowedTCPPortRanges = [ + { + # NodePort services + from = 30000; + to = 32767; + } + ]; + + networking.firewall.allowedUDPPortRanges = [ + { + # NodePort services + from = 30000; + to = 32767; + } + ]; + }; +} diff --git a/nix/kubernetes/roles/zsh/default.nix b/nix/kubernetes/roles/zsh/default.nix new file mode 100644 index 0000000..645b305 --- /dev/null +++ b/nix/kubernetes/roles/zsh/default.nix @@ -0,0 +1,117 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + zshrc = pkgs.writeTextFile { + name = ".zshrc"; + text = '' + # Lines configured by zsh-newuser-install + HISTFILE=~/.zhistory + HISTSIZE=100000 + SAVEHIST=100000 + setopt appendhistory notify + unsetopt beep + bindkey -e + # End of lines configured by zsh-newuser-install + # The following lines were added by compinstall + # + + # Use menu complete immediately instead of after the first tab + setopt MENU_COMPLETE + + zstyle :compinstall filename "$HOME/.zshrc" + + autoload -Uz compinit + compinit + # End of lines added by compinstall + + # Enable the 2d menu for tab completion + zstyle ':completion:*' menu select + + autoload colors zsh/terminfo + if [[ "$terminfo[colors]" -ge 8 ]]; then + colors + fi + for color in RED GREEN YELLOW BLUE MAGENTA CYAN WHITE; do + eval PR_$color='%{$terminfo[bold]$fg[''${(L)color}]%}' + eval PR_LIGHT_$color='%{$fg[''${(L)color}]%}' + (( count = $count + 1 )) + done + PR_NO_COLOR="%{$terminfo[sgr0]%}" + PS1="[$PR_BLUE%n$PR_WHITE@$PR_GREEN%U%m%u$PR_NO_COLOR:$PR_RED%2c$PR_NO_COLOR]%(!.#.$) " + + source ${pkgs.zsh-histdb}/share/zsh/plugins/zsh-histdb/sqlite-history.zsh + autoload -Uz add-zsh-hook + + source ${pkgs.zsh-histdb}/share/zsh/plugins/zsh-histdb/histdb-interactive.zsh + bindkey '^r' _histdb-isearch + + ${lib.concatMapStringsSep "\n" (item: "source ${item}") config.me.zsh.includes} + ''; + }; +in +{ + imports = [ ]; + + options.me = { + zsh.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install zsh."; + }; + zsh.includes = lib.mkOption { + type = lib.types.listOf lib.types.package; + default = [ ]; + example = lib.literalExpression '' + [ (pkgs.writeTextFile { + name = "launch-kanshi.conf"; + text = "exec kanshi"; + }) ]''; + description = "List of zshrc files to import."; + }; + }; + + config = lib.mkIf config.me.zsh.enable { + environment.systemPackages = with pkgs; [ + zsh + ]; + + users.users.talexander.shell = pkgs.zsh; + environment.shells = with pkgs; [ zsh ]; + + programs.zsh = { + enable = true; + }; + + me.install.user.talexander.file = { + ".zshrc" = { + source = "${zshrc}"; + }; + }; + + environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) { + hideMounts = true; + users.talexander = { + directories = [ + { + directory = ".histdb"; + user = "talexander"; + group = "talexander"; + mode = "0700"; + } + ]; + }; + }; + + nixpkgs.overlays = [ + (final: prev: { + zsh-histdb = (final.callPackage ./package/zsh-histdb/package.nix { }); + }) + ]; + }; +} diff --git a/nix/kubernetes/roles/zsh/package/zsh-histdb/package.nix b/nix/kubernetes/roles/zsh/package/zsh-histdb/package.nix new file mode 100644 index 0000000..f3b92a3 --- /dev/null +++ b/nix/kubernetes/roles/zsh/package/zsh-histdb/package.nix @@ -0,0 +1,36 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + stdenv, + pkgs, + sqlite, + ... +}: +stdenv.mkDerivation { + name = "zsh-histdb"; + src = pkgs.fetchgit { + url = "https://github.com/larkery/zsh-histdb.git"; + rev = "90a6c104d0fcc0410d665e148fa7da28c49684eb"; + sha256 = "sha256-vtG1poaRVbfb/wKPChk1WpPgDq+7udLqLfYfLqap4Vg="; + }; + buildInputs = [ sqlite ]; + phases = [ + "installPhase" + ]; + installPhase = '' + runHook preInstall + mkdir -p $out/share/zsh/plugins/zsh-histdb + cp -r $src/histdb-* $src/*.zsh $src/db_migrations $out/share/zsh/plugins/zsh-histdb/ + runHook postInstall + ''; + postInstall = '' + substituteInPlace $out/share/zsh/plugins/zsh-histdb/sqlite-history.zsh $out/share/zsh/plugins/zsh-histdb/histdb-merge $out/share/zsh/plugins/zsh-histdb/histdb-migrate --replace-fail "sqlite3" "${sqlite}/bin/sqlite3" + ''; +} diff --git a/nix/kubernetes/util/install_files/default.nix b/nix/kubernetes/util/install_files/default.nix new file mode 100644 index 0000000..171e038 --- /dev/null +++ b/nix/kubernetes/util/install_files/default.nix @@ -0,0 +1,331 @@ +{ + config, + lib, + ... +}: + +let + cfg = config.me.install; + inherit (lib) + filter + attrNames + ; + + get_shell_values = + target: + let + homedir = config.users.users."${target.username}".home; + group = config.users.users."${target.username}".group; + in + { + source = lib.strings.escapeShellArg "${target.source}"; + destination = lib.strings.escapeShellArg "${homedir}/${target.target}"; + mode = lib.strings.escapeShellArg "${target.mode}"; + dir_mode = lib.strings.escapeShellArg "${target.dir_mode}"; + username = lib.strings.escapeShellArg "${target.username}"; + group = lib.strings.escapeShellArg "${group}"; + }; + install_user_file = + let + constructors = { + "overwrite" = install_user_file_overwrite; + "symlink" = install_user_file_symlink; + }; + in + stage: target: (constructors."${target.method}"."${stage}" target); + install_user_file_overwrite = { + "check" = (target: ""); + "install" = ( + target: + let + inherit (get_shell_values target) + source + destination + mode + dir_mode + username + group + ; + flags = lib.strings.concatStringsSep " " [ + (if mode != "" then "-m ${mode}" else "") + (if username != "" then "-o ${username}" else "") + (if group != "" then "-g ${group}" else "") + ]; + dir_flags = lib.strings.concatStringsSep " " [ + (if dir_mode != "" then "-m ${dir_mode}" else "") + (if username != "" then "-o ${username}" else "") + (if group != "" then "-g ${group}" else "") + ]; + in + if target.recursive then + [ + '' + find ${source} -type f -print0 | while read -r -d "" file; do + relative_path=$(realpath -s --relative-to ${source} "$file") + full_dest=${destination}/"$relative_path" + create_containing_directories "$full_dest" ${dir_flags} + $DRY_RUN_CMD install $VERBOSE_ARG --compare ${flags} "$file" "$full_dest" + done + '' + ] + else + [ + '' + create_containing_directories ${destination} ${dir_flags} + $DRY_RUN_CMD install $VERBOSE_ARG --compare ${flags} ${source} ${destination} + '' + ] + ); + "uninstall" = ( + target: + let + inherit (get_shell_values target) + source + destination + ; + in + if target.recursive then + [ + '' + find ${source} -type f -print0 | while read -r -d "" file; do + relative_path=$(realpath -s --relative-to ${source} "$file") + full_dest=${destination}/"$relative_path" + $DRY_RUN_CMD echo rm -f "$full_dest" + done + '' + ] + else + [ + '' + $DRY_RUN_CMD echo rm -f ${destination} + '' + ] + ); + }; + install_user_file_symlink = { + "check" = (target: ""); + "install" = ( + target: + let + inherit (get_shell_values target) + source + destination + mode + dir_mode + username + group + ; + owner = lib.strings.concatStringsSep ":" ( + filter (val: val != "") [ + username + group + ] + ); + dir_flags = lib.strings.concatStringsSep " " [ + (if dir_mode != "" then "-m ${dir_mode}" else "") + (if username != "" then "-o ${username}" else "") + (if group != "" then "-g ${group}" else "") + ]; + in + if target.recursive then + [ + '' + find ${source} -type f -print0 | while read -r -d "" file; do + relative_path=$(realpath -s --relative-to ${source} "$file") + full_dest=${destination}/"$relative_path" + create_containing_directories "$full_dest" ${dir_flags} + $DRY_RUN_CMD ln $VERBOSE_ARG -s "$file" "$full_dest" + $DRY_RUN_CMD chown $VERBOSE_ARG -h ${owner} "$full_dest" + done + '' + ] + else + [ + '' + create_containing_directories ${destination} ${dir_flags} + $DRY_RUN_CMD ln $VERBOSE_ARG -s ${source} ${destination} + $DRY_RUN_CMD chown $VERBOSE_ARG -h ${owner} ${destination} + '' + ] + ); + "uninstall" = ( + target: + let + inherit (get_shell_values target) + source + destination + ; + in + if target.recursive then + [ + '' + find ${source} -type f -print0 | while read -r -d "" file; do + relative_path=$(realpath -s --relative-to ${source} "$file") + full_dest=${destination}/"$relative_path" + $DRY_RUN_CMD echo rm -f "$full_dest" + done + '' + ] + else + [ + '' + $DRY_RUN_CMD echo rm -f ${destination} + '' + ] + ); + }; +in +{ + imports = [ ]; + + options.me.install = { + user = lib.mkOption { + default = { }; + type = lib.types.attrsOf ( + lib.types.submodule ( + { name, config, ... }: + let + username = name; + in + { + options = { + enable = lib.mkOption { + type = lib.types.bool; + default = true; + defaultText = "enable"; + example = lib.literalExpression false; + description = "Whether we want to install files in this user's home directory."; + }; + + file = lib.mkOption { + type = lib.types.attrsOf ( + lib.types.submodule ( + { name, config, ... }: + let + path = name; + in + { + options = { + enable = lib.mkOption { + type = lib.types.bool; + default = true; + defaultText = "enable"; + example = lib.literalExpression false; + description = "Whether we want to install this file in this user's home directory."; + }; + username = lib.mkOption { + type = lib.types.str; + defaultText = "username"; + example = "root"; + description = "The username for the user whose home directory will contain the file."; + }; + target = lib.mkOption { + type = lib.types.str; + defaultText = "target"; + example = ".local/share/foo/bar.txt"; + description = "The path where the file should be written."; + }; + method = lib.mkOption { + type = lib.types.enum [ + "symlink" + "overwrite" + # "bind_mount" TODO: for directories? + ]; + default = "symlink"; + defaultText = "me.install.file.‹path›.method"; + example = "overwrite"; + description = "The way in which the file should be installed."; + }; + mode = lib.mkOption { + type = lib.types.str; + default = "0444"; + defaultText = "me.install.file.‹path›.mode"; + example = "0750"; + description = "The read, write, execute permission flags."; + }; + dir_mode = lib.mkOption { + type = lib.types.str; + default = "0755"; + defaultText = "dir_mode"; + example = "0755"; + description = "The read, write, execute permission flags for any parent directories that need to be created."; + }; + source = lib.mkOption { + type = lib.types.path; + defaultText = "me.install.file.‹path›.source"; + example = ./files/foo.txt; + description = "The source file to install into the destination."; + }; + recursive = lib.mkOption { + type = lib.types.bool; + default = false; + defaultText = "recursive"; + example = lib.literalExpression false; + description = "Whether we want to recurse through the directory doing individual installs for each file."; + }; + }; + + config = { + username = lib.mkDefault username; + target = lib.mkDefault path; + }; + } + ) + ); + }; + }; + } + ) + ); + }; + }; + + config = + let + all_users = builtins.map (username: cfg.user."${username}") (attrNames cfg.user); + enabled_users = filter (user: user.enable) all_users; + all_file_targets = lib.flatten ( + builtins.map (user: (builtins.map (path: user.file."${path}") (attrNames user.file))) enabled_users + ); + enabled_file_targets = filter (target: target.enable) all_file_targets; + check_commands = lib.flatten (builtins.map (install_user_file "check") enabled_file_targets); + install_commands = lib.flatten (builtins.map (install_user_file "install") enabled_file_targets); + uninstall_commands = lib.flatten ( + builtins.map (install_user_file "uninstall") enabled_file_targets + ); + in + { + systemd.services.me-install-file = { + enable = true; + description = "me-install-file"; + wantedBy = [ "multi-user.target" ]; + wants = [ "multi-user.target" ]; + before = [ "multi-user.target" ]; + # path = with pkgs; [ + # zfs + # ]; + unitConfig.DefaultDependencies = "no"; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = "yes"; + }; + script = '' + set -o pipefail + IFS=$'\n\t' + source ${./files/lib.bash} + '' + + (lib.strings.concatStringsSep "\n" ( + [ + ] + ++ check_commands + ++ install_commands + )); + preStop = '' + set -o pipefail + IFS=$'\n\t' + source ${./files/lib.bash} + '' + + (lib.strings.concatStringsSep "\n" uninstall_commands); + }; + }; +} diff --git a/nix/kubernetes/util/install_files/files/lib.bash b/nix/kubernetes/util/install_files/files/lib.bash new file mode 100644 index 0000000..1a5c178 --- /dev/null +++ b/nix/kubernetes/util/install_files/files/lib.bash @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# + +############## Setup ######################### + +function die { + local status_code="$1" + shift + (>&2 echo "${@}") + exit "$status_code" +} + +function log { + (>&2 echo "${@}") +} + +############## Program ######################### + +function create_containing_directories { + local full_dest="$1" + shift 1 + local dirs_to_create=() + local containing_directory="$full_dest" + while true; do + containing_directory=$(dirname "$containing_directory") + if [ -e "$containing_directory" ] || [ "$containing_directory" = "/" ]; then + break + fi + dirs_to_create+=($containing_directory) + done + + for (( idx=${#dirs_to_create[@]}-1 ; idx>=0 ; idx-- )) ; do + local containing_directory="${dirs_to_create[idx]}" + log "Creating $containing_directory" + $DRY_RUN_CMD install $VERBOSE_ARG -d "${@}" "$containing_directory" + done + +} diff --git a/nix/kubernetes/util/unfree_polyfill/default.nix b/nix/kubernetes/util/unfree_polyfill/default.nix new file mode 100644 index 0000000..d744cf4 --- /dev/null +++ b/nix/kubernetes/util/unfree_polyfill/default.nix @@ -0,0 +1,15 @@ +{ config, lib, ... }: + +let + inherit (builtins) elem; + inherit (lib) getName mkOption; + inherit (lib.types) listOf str; +in +{ + # Pending https://github.com/NixOS/nixpkgs/issues/55674 + options.allowedUnfree = mkOption { + type = listOf str; + default = [ ]; + }; + config.nixpkgs.config.allowUnfreePredicate = p: elem (getName p) config.allowedUnfree; +}