diff --git a/nix/kubernetes/configuration.nix b/nix/kubernetes/configuration.nix index 11b95381..4cde4057 100644 --- a/nix/kubernetes/configuration.nix +++ b/nix/kubernetes/configuration.nix @@ -7,6 +7,7 @@ { imports = [ ./roles/boot + ./roles/containerd ./roles/control_plane ./roles/doas ./roles/dont_use_substituters @@ -24,8 +25,7 @@ ./roles/ssh ./roles/sshd ./roles/user - ./roles/zfs - ./roles/zrepl + ./roles/worker_node ./roles/zsh ./util/install_files ./util/unfree_polyfill diff --git a/nix/kubernetes/flake.nix b/nix/kubernetes/flake.nix index 30e8c399..36a66e7b 100644 --- a/nix/kubernetes/flake.nix +++ b/nix/kubernetes/flake.nix @@ -54,6 +54,15 @@ controller2 = { system = "x86_64-linux"; }; + worker0 = { + system = "x86_64-linux"; + }; + worker1 = { + system = "x86_64-linux"; + }; + worker2 = { + system = "x86_64-linux"; + }; }; nixosConfigs = builtins.mapAttrs ( hostname: nodeConfig: format: diff --git a/nix/kubernetes/hosts/worker0/DEPLOY_BOOT b/nix/kubernetes/hosts/worker0/DEPLOY_BOOT new file mode 100755 index 00000000..d7168729 --- /dev/null +++ b/nix/kubernetes/hosts/worker0/DEPLOY_BOOT @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=worker0 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild boot --flake "$DIR/../../#worker0" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker0/DEPLOY_SWITCH b/nix/kubernetes/hosts/worker0/DEPLOY_SWITCH new file mode 100755 index 00000000..a47bf6a8 --- /dev/null +++ b/nix/kubernetes/hosts/worker0/DEPLOY_SWITCH @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=worker0 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild switch --flake "$DIR/../../#worker0" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker0/ISO b/nix/kubernetes/hosts/worker0/ISO new file mode 100755 index 00000000..46caa981 --- /dev/null +++ b/nix/kubernetes/hosts/worker0/ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#worker0.iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker0/SELF_BOOT b/nix/kubernetes/hosts/worker0/SELF_BOOT new file mode 100755 index 00000000..35e60597 --- /dev/null +++ b/nix/kubernetes/hosts/worker0/SELF_BOOT @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild boot --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker0" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker0/SELF_BUILD b/nix/kubernetes/hosts/worker0/SELF_BUILD new file mode 100755 index 00000000..69fbd95f --- /dev/null +++ b/nix/kubernetes/hosts/worker0/SELF_BUILD @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild build --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker0" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker0/SELF_SWITCH b/nix/kubernetes/hosts/worker0/SELF_SWITCH new file mode 100755 index 00000000..b127a674 --- /dev/null +++ b/nix/kubernetes/hosts/worker0/SELF_SWITCH @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild switch --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker0" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker0/VM_ISO b/nix/kubernetes/hosts/worker0/VM_ISO new file mode 100755 index 00000000..3e518fc0 --- /dev/null +++ b/nix/kubernetes/hosts/worker0/VM_ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#worker0.vm_iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker0/default.nix b/nix/kubernetes/hosts/worker0/default.nix new file mode 100644 index 00000000..1fac5fae --- /dev/null +++ b/nix/kubernetes/hosts/worker0/default.nix @@ -0,0 +1,109 @@ +# MANUAL: On client machines generate signing keys: +# nix-store --generate-binary-cache-key some-name /persist/manual/nix/nix-cache-key.sec /persist/manual/nix/nix-cache-key.pub +# +# Trust other machines and add the substituters: +# nix.binaryCachePublicKeys = [ "some-name:AzNW1MOlkNEsUAXS1jIFZ1QCFKXjV+Y/LrF37quAZ1A=" ]; +# nix.binaryCaches = [ "https://test.example/nix-cache" ]; + +{ + config, + lib, + pkgs, + ... +}: +{ + imports = [ + ./hardware-configuration.nix + ./vm_disk.nix + ]; + + config = { + networking = + let + interface = "enp0s2"; + in + { + # Generate with `head -c4 /dev/urandom | od -A none -t x4` + hostId = "0aadbb10"; + + hostName = "worker0"; # Define your hostname. + + interfaces = { + "${interface}" = { + ipv4.addresses = [ + { + address = "10.215.1.224"; + prefixLength = 24; + } + ]; + + ipv6.addresses = [ + { + address = "2620:11f:7001:7:ffff:ffff:0ad7:01e0"; + prefixLength = 64; + } + ]; + }; + }; + defaultGateway = "10.215.1.1"; + defaultGateway6 = { + # address = "2620:11f:7001:7::1"; + address = "2620:11f:7001:7:ffff:ffff:0ad7:0101"; + inherit interface; + }; + nameservers = [ + "10.215.1.1" + ]; + + dhcpcd.enable = lib.mkForce false; + useDHCP = lib.mkForce false; + }; + + time.timeZone = "America/New_York"; + i18n.defaultLocale = "en_US.UTF-8"; + + me.boot.enable = true; + me.boot.secure = false; + me.mountPersistence = true; + boot.loader.timeout = lib.mkForce 0; # We can always generate a new ISO if we need to access other boot options. + + me.optimizations = { + enable = true; + arch = "znver4"; + # build_arch = "x86-64-v3"; + system_features = [ + "gccarch-znver4" + "gccarch-skylake" + "gccarch-kabylake" + # "gccarch-alderlake" missing WAITPKG + "gccarch-x86-64-v3" + "gccarch-x86-64-v4" + "benchmark" + "big-parallel" + "kvm" + "nixos-test" + ]; + }; + + # Mount tmpfs at /tmp + boot.tmp.useTmpfs = true; + + # Enable TRIM + # services.fstrim.enable = lib.mkDefault true; + + # nix.optimise.automatic = true; + # nix.optimise.dates = [ "03:45" ]; + # nix.optimise.persistent = true; + + environment.systemPackages = with pkgs; [ + htop + ]; + + # nix.sshServe.enable = true; + # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ]; + + me.worker_node.enable = true; + me.dont_use_substituters.enable = true; + me.minimal_base.enable = true; + }; +} diff --git a/nix/kubernetes/hosts/worker0/hardware-configuration.nix b/nix/kubernetes/hosts/worker0/hardware-configuration.nix new file mode 100644 index 00000000..6029e088 --- /dev/null +++ b/nix/kubernetes/hosts/worker0/hardware-configuration.nix @@ -0,0 +1,31 @@ +{ + config, + lib, + modulesPath, + ... +}: + +{ + imports = [ + (modulesPath + "/installer/scan/not-detected.nix") + ]; + + config = { + boot.initrd.availableKernelModules = [ + "nvme" + "xhci_pci" + "thunderbolt" + ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ ]; + boot.extraModulePackages = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + # networking.useDHCP = lib.mkDefault true; + # networking.interfaces.eno1.useDHCP = lib.mkDefault true; + # networking.interfaces.wlp58s0.useDHCP = lib.mkDefault true; + }; +} diff --git a/nix/kubernetes/hosts/worker0/vm_disk.nix b/nix/kubernetes/hosts/worker0/vm_disk.nix new file mode 100644 index 00000000..751f4bad --- /dev/null +++ b/nix/kubernetes/hosts/worker0/vm_disk.nix @@ -0,0 +1,94 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + config = { + # Mount the local disk + fileSystems = lib.mkIf config.me.mountPersistence { + # "/.disk" = lib.mkForce { + # device = "/dev/nvme0n1p1"; + # fsType = "ext4"; + # options = [ + # "noatime" + # "discard" + # ]; + # neededForBoot = true; + # }; + + "/.persist" = lib.mkForce { + device = "bind9p"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + + "/persist" = { + fsType = "none"; + device = "/.persist/persist"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.persist/persist" + ]; + neededForBoot = true; + }; + + "/state" = { + fsType = "none"; + device = "/.persist/state"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.persist/state" + ]; + neededForBoot = true; + }; + + "/k8spv" = lib.mkForce { + device = "k8spv"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + + # "/disk" = { + # fsType = "none"; + # device = "/.disk/persist"; + # options = [ + # "bind" + # "rw" + # ]; + # depends = [ + # "/.disk/persist" + # ]; + # neededForBoot = true; + # }; + }; + }; +} diff --git a/nix/kubernetes/hosts/worker1/DEPLOY_BOOT b/nix/kubernetes/hosts/worker1/DEPLOY_BOOT new file mode 100755 index 00000000..631d46d4 --- /dev/null +++ b/nix/kubernetes/hosts/worker1/DEPLOY_BOOT @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=worker1 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild boot --flake "$DIR/../../#worker1" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker1/DEPLOY_SWITCH b/nix/kubernetes/hosts/worker1/DEPLOY_SWITCH new file mode 100755 index 00000000..271829cc --- /dev/null +++ b/nix/kubernetes/hosts/worker1/DEPLOY_SWITCH @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=worker1 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild switch --flake "$DIR/../../#worker1" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker1/ISO b/nix/kubernetes/hosts/worker1/ISO new file mode 100755 index 00000000..4c2b9693 --- /dev/null +++ b/nix/kubernetes/hosts/worker1/ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#worker1.iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker1/SELF_BOOT b/nix/kubernetes/hosts/worker1/SELF_BOOT new file mode 100755 index 00000000..cf928cd7 --- /dev/null +++ b/nix/kubernetes/hosts/worker1/SELF_BOOT @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild boot --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker1" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker1/SELF_BUILD b/nix/kubernetes/hosts/worker1/SELF_BUILD new file mode 100755 index 00000000..73c846ee --- /dev/null +++ b/nix/kubernetes/hosts/worker1/SELF_BUILD @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild build --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker1" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker1/SELF_SWITCH b/nix/kubernetes/hosts/worker1/SELF_SWITCH new file mode 100755 index 00000000..d6a989d0 --- /dev/null +++ b/nix/kubernetes/hosts/worker1/SELF_SWITCH @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild switch --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker1" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker1/VM_ISO b/nix/kubernetes/hosts/worker1/VM_ISO new file mode 100755 index 00000000..ffab210a --- /dev/null +++ b/nix/kubernetes/hosts/worker1/VM_ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#worker1.vm_iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker1/default.nix b/nix/kubernetes/hosts/worker1/default.nix new file mode 100644 index 00000000..6f951f0a --- /dev/null +++ b/nix/kubernetes/hosts/worker1/default.nix @@ -0,0 +1,109 @@ +# MANUAL: On client machines generate signing keys: +# nix-store --generate-binary-cache-key some-name /persist/manual/nix/nix-cache-key.sec /persist/manual/nix/nix-cache-key.pub +# +# Trust other machines and add the substituters: +# nix.binaryCachePublicKeys = [ "some-name:AzNW1MOlkNEsUAXS1jIFZ1QCFKXjV+Y/LrF37quAZ1A=" ]; +# nix.binaryCaches = [ "https://test.example/nix-cache" ]; + +{ + config, + lib, + pkgs, + ... +}: +{ + imports = [ + ./hardware-configuration.nix + ./vm_disk.nix + ]; + + config = { + networking = + let + interface = "enp0s2"; + in + { + # Generate with `head -c4 /dev/urandom | od -A none -t x4` + hostId = "4324346d"; + + hostName = "worker1"; # Define your hostname. + + interfaces = { + "${interface}" = { + ipv4.addresses = [ + { + address = "10.215.1.225"; + prefixLength = 24; + } + ]; + + ipv6.addresses = [ + { + address = "2620:11f:7001:7:ffff:ffff:0ad7:01e1"; + prefixLength = 64; + } + ]; + }; + }; + defaultGateway = "10.215.1.1"; + defaultGateway6 = { + # address = "2620:11f:7001:7::1"; + address = "2620:11f:7001:7:ffff:ffff:0ad7:0101"; + inherit interface; + }; + nameservers = [ + "10.215.1.1" + ]; + + dhcpcd.enable = lib.mkForce false; + useDHCP = lib.mkForce false; + }; + + time.timeZone = "America/New_York"; + i18n.defaultLocale = "en_US.UTF-8"; + + me.boot.enable = true; + me.boot.secure = false; + me.mountPersistence = true; + boot.loader.timeout = lib.mkForce 0; # We can always generate a new ISO if we need to access other boot options. + + me.optimizations = { + enable = true; + arch = "znver4"; + # build_arch = "x86-64-v3"; + system_features = [ + "gccarch-znver4" + "gccarch-skylake" + "gccarch-kabylake" + # "gccarch-alderlake" missing WAITPKG + "gccarch-x86-64-v3" + "gccarch-x86-64-v4" + "benchmark" + "big-parallel" + "kvm" + "nixos-test" + ]; + }; + + # Mount tmpfs at /tmp + boot.tmp.useTmpfs = true; + + # Enable TRIM + # services.fstrim.enable = lib.mkDefault true; + + # nix.optimise.automatic = true; + # nix.optimise.dates = [ "03:45" ]; + # nix.optimise.persistent = true; + + environment.systemPackages = with pkgs; [ + htop + ]; + + # nix.sshServe.enable = true; + # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ]; + + me.worker_node.enable = true; + me.dont_use_substituters.enable = true; + me.minimal_base.enable = true; + }; +} diff --git a/nix/kubernetes/hosts/worker1/hardware-configuration.nix b/nix/kubernetes/hosts/worker1/hardware-configuration.nix new file mode 100644 index 00000000..6029e088 --- /dev/null +++ b/nix/kubernetes/hosts/worker1/hardware-configuration.nix @@ -0,0 +1,31 @@ +{ + config, + lib, + modulesPath, + ... +}: + +{ + imports = [ + (modulesPath + "/installer/scan/not-detected.nix") + ]; + + config = { + boot.initrd.availableKernelModules = [ + "nvme" + "xhci_pci" + "thunderbolt" + ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ ]; + boot.extraModulePackages = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + # networking.useDHCP = lib.mkDefault true; + # networking.interfaces.eno1.useDHCP = lib.mkDefault true; + # networking.interfaces.wlp58s0.useDHCP = lib.mkDefault true; + }; +} diff --git a/nix/kubernetes/hosts/worker1/vm_disk.nix b/nix/kubernetes/hosts/worker1/vm_disk.nix new file mode 100644 index 00000000..751f4bad --- /dev/null +++ b/nix/kubernetes/hosts/worker1/vm_disk.nix @@ -0,0 +1,94 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + config = { + # Mount the local disk + fileSystems = lib.mkIf config.me.mountPersistence { + # "/.disk" = lib.mkForce { + # device = "/dev/nvme0n1p1"; + # fsType = "ext4"; + # options = [ + # "noatime" + # "discard" + # ]; + # neededForBoot = true; + # }; + + "/.persist" = lib.mkForce { + device = "bind9p"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + + "/persist" = { + fsType = "none"; + device = "/.persist/persist"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.persist/persist" + ]; + neededForBoot = true; + }; + + "/state" = { + fsType = "none"; + device = "/.persist/state"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.persist/state" + ]; + neededForBoot = true; + }; + + "/k8spv" = lib.mkForce { + device = "k8spv"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + + # "/disk" = { + # fsType = "none"; + # device = "/.disk/persist"; + # options = [ + # "bind" + # "rw" + # ]; + # depends = [ + # "/.disk/persist" + # ]; + # neededForBoot = true; + # }; + }; + }; +} diff --git a/nix/kubernetes/hosts/worker2/DEPLOY_BOOT b/nix/kubernetes/hosts/worker2/DEPLOY_BOOT new file mode 100755 index 00000000..eb5db119 --- /dev/null +++ b/nix/kubernetes/hosts/worker2/DEPLOY_BOOT @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=worker2 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild boot --flake "$DIR/../../#worker2" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker2/DEPLOY_SWITCH b/nix/kubernetes/hosts/worker2/DEPLOY_SWITCH new file mode 100755 index 00000000..c3213281 --- /dev/null +++ b/nix/kubernetes/hosts/worker2/DEPLOY_SWITCH @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +TARGET=worker2 + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done + +nixos-rebuild switch --flake "$DIR/../../#worker2" --target-host "$TARGET" --build-host "$TARGET" --sudo --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker2/ISO b/nix/kubernetes/hosts/worker2/ISO new file mode 100755 index 00000000..ad8a821d --- /dev/null +++ b/nix/kubernetes/hosts/worker2/ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#worker2.iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker2/SELF_BOOT b/nix/kubernetes/hosts/worker2/SELF_BOOT new file mode 100755 index 00000000..bdfde927 --- /dev/null +++ b/nix/kubernetes/hosts/worker2/SELF_BOOT @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild boot --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker2" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker2/SELF_BUILD b/nix/kubernetes/hosts/worker2/SELF_BUILD new file mode 100755 index 00000000..36cc5b57 --- /dev/null +++ b/nix/kubernetes/hosts/worker2/SELF_BUILD @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild build --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker2" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker2/SELF_SWITCH b/nix/kubernetes/hosts/worker2/SELF_SWITCH new file mode 100755 index 00000000..7af1d157 --- /dev/null +++ b/nix/kubernetes/hosts/worker2/SELF_SWITCH @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nixos-rebuild switch --show-trace --sudo --max-jobs "$JOBS" --flake "$DIR/../../#worker2" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker2/VM_ISO b/nix/kubernetes/hosts/worker2/VM_ISO new file mode 100755 index 00000000..7f1f2030 --- /dev/null +++ b/nix/kubernetes/hosts/worker2/VM_ISO @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +: "${JOBS:="1"}" + +for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done +nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#worker2.vm_iso" --max-jobs "$JOBS" --log-format internal-json -v "${@}" |& nom --json diff --git a/nix/kubernetes/hosts/worker2/default.nix b/nix/kubernetes/hosts/worker2/default.nix new file mode 100644 index 00000000..d595c0df --- /dev/null +++ b/nix/kubernetes/hosts/worker2/default.nix @@ -0,0 +1,109 @@ +# MANUAL: On client machines generate signing keys: +# nix-store --generate-binary-cache-key some-name /persist/manual/nix/nix-cache-key.sec /persist/manual/nix/nix-cache-key.pub +# +# Trust other machines and add the substituters: +# nix.binaryCachePublicKeys = [ "some-name:AzNW1MOlkNEsUAXS1jIFZ1QCFKXjV+Y/LrF37quAZ1A=" ]; +# nix.binaryCaches = [ "https://test.example/nix-cache" ]; + +{ + config, + lib, + pkgs, + ... +}: +{ + imports = [ + ./hardware-configuration.nix + ./vm_disk.nix + ]; + + config = { + networking = + let + interface = "enp0s2"; + in + { + # Generate with `head -c4 /dev/urandom | od -A none -t x4` + hostId = "ce017961"; + + hostName = "worker2"; # Define your hostname. + + interfaces = { + "${interface}" = { + ipv4.addresses = [ + { + address = "10.215.1.226"; + prefixLength = 24; + } + ]; + + ipv6.addresses = [ + { + address = "2620:11f:7001:7:ffff:ffff:0ad7:01e2"; + prefixLength = 64; + } + ]; + }; + }; + defaultGateway = "10.215.1.1"; + defaultGateway6 = { + # address = "2620:11f:7001:7::1"; + address = "2620:11f:7001:7:ffff:ffff:0ad7:0101"; + inherit interface; + }; + nameservers = [ + "10.215.1.1" + ]; + + dhcpcd.enable = lib.mkForce false; + useDHCP = lib.mkForce false; + }; + + time.timeZone = "America/New_York"; + i18n.defaultLocale = "en_US.UTF-8"; + + me.boot.enable = true; + me.boot.secure = false; + me.mountPersistence = true; + boot.loader.timeout = lib.mkForce 0; # We can always generate a new ISO if we need to access other boot options. + + me.optimizations = { + enable = true; + arch = "znver4"; + # build_arch = "x86-64-v3"; + system_features = [ + "gccarch-znver4" + "gccarch-skylake" + "gccarch-kabylake" + # "gccarch-alderlake" missing WAITPKG + "gccarch-x86-64-v3" + "gccarch-x86-64-v4" + "benchmark" + "big-parallel" + "kvm" + "nixos-test" + ]; + }; + + # Mount tmpfs at /tmp + boot.tmp.useTmpfs = true; + + # Enable TRIM + # services.fstrim.enable = lib.mkDefault true; + + # nix.optimise.automatic = true; + # nix.optimise.dates = [ "03:45" ]; + # nix.optimise.persistent = true; + + environment.systemPackages = with pkgs; [ + htop + ]; + + # nix.sshServe.enable = true; + # nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ]; + + me.worker_node.enable = true; + me.dont_use_substituters.enable = true; + me.minimal_base.enable = true; + }; +} diff --git a/nix/kubernetes/hosts/worker2/hardware-configuration.nix b/nix/kubernetes/hosts/worker2/hardware-configuration.nix new file mode 100644 index 00000000..6029e088 --- /dev/null +++ b/nix/kubernetes/hosts/worker2/hardware-configuration.nix @@ -0,0 +1,31 @@ +{ + config, + lib, + modulesPath, + ... +}: + +{ + imports = [ + (modulesPath + "/installer/scan/not-detected.nix") + ]; + + config = { + boot.initrd.availableKernelModules = [ + "nvme" + "xhci_pci" + "thunderbolt" + ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ ]; + boot.extraModulePackages = [ ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + # networking.useDHCP = lib.mkDefault true; + # networking.interfaces.eno1.useDHCP = lib.mkDefault true; + # networking.interfaces.wlp58s0.useDHCP = lib.mkDefault true; + }; +} diff --git a/nix/kubernetes/hosts/worker2/vm_disk.nix b/nix/kubernetes/hosts/worker2/vm_disk.nix new file mode 100644 index 00000000..751f4bad --- /dev/null +++ b/nix/kubernetes/hosts/worker2/vm_disk.nix @@ -0,0 +1,94 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + config = { + # Mount the local disk + fileSystems = lib.mkIf config.me.mountPersistence { + # "/.disk" = lib.mkForce { + # device = "/dev/nvme0n1p1"; + # fsType = "ext4"; + # options = [ + # "noatime" + # "discard" + # ]; + # neededForBoot = true; + # }; + + "/.persist" = lib.mkForce { + device = "bind9p"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + + "/persist" = { + fsType = "none"; + device = "/.persist/persist"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.persist/persist" + ]; + neededForBoot = true; + }; + + "/state" = { + fsType = "none"; + device = "/.persist/state"; + options = [ + "bind" + "rw" + ]; + depends = [ + "/.persist/state" + ]; + neededForBoot = true; + }; + + "/k8spv" = lib.mkForce { + device = "k8spv"; + fsType = "9p"; + options = [ + "noatime" + "trans=virtio" + "version=9p2000.L" + "cache=mmap" + "msize=512000" + # "noauto" + # "x-systemd.automount" + ]; + neededForBoot = true; + }; + + # "/disk" = { + # fsType = "none"; + # device = "/.disk/persist"; + # options = [ + # "bind" + # "rw" + # ]; + # depends = [ + # "/.disk/persist" + # ]; + # neededForBoot = true; + # }; + }; + }; +} diff --git a/nix/kubernetes/roles/containerd/default.nix b/nix/kubernetes/roles/containerd/default.nix new file mode 100644 index 00000000..18dac5c2 --- /dev/null +++ b/nix/kubernetes/roles/containerd/default.nix @@ -0,0 +1,46 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + options.me = { + containerd.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install containerd."; + }; + }; + + config = lib.mkIf config.me.containerd.enable { + virtualisation.containerd.enable = true; + virtualisation.containerd.settings = { + "plugins" = { + "io.containerd.grpc.v1.cri" = { + "cni" = { + "bin_dir" = "/opt/cni/bin"; + "conf_dir" = "/etc/cni/net.d"; + }; + "containerd" = { + "default_runtime_name" = "runc"; + "runtimes" = { + "runc" = { + "options" = { + "SystemdCgroup" = true; + }; + "runtime_type" = "io.containerd.runc.v2"; + }; + }; + "snapshotter" = "overlayfs"; + }; + }; + }; + "version" = 2; + }; + }; +} diff --git a/nix/kubernetes/roles/etcd/default.nix b/nix/kubernetes/roles/etcd/default.nix index 468b8657..ae345427 100644 --- a/nix/kubernetes/roles/etcd/default.nix +++ b/nix/kubernetes/roles/etcd/default.nix @@ -105,5 +105,6 @@ gptfdisk # cgdisk ]; networking.firewall.enable = false; # TODO: This is just here for debugging / initial development. + # TODO: Maybe use networking.nftables.enable to switch to nftables? }; } diff --git a/nix/kubernetes/roles/minimal_base/default.nix b/nix/kubernetes/roles/minimal_base/default.nix index 16547296..e57f1f24 100644 --- a/nix/kubernetes/roles/minimal_base/default.nix +++ b/nix/kubernetes/roles/minimal_base/default.nix @@ -24,8 +24,6 @@ me.ssh.enable = true; me.sshd.enable = true; me.user.enable = true; - me.zfs.enable = true; - me.zrepl.enable = true; me.zsh.enable = true; # TODO: Maybe add me.boot.enable ? diff --git a/nix/kubernetes/roles/worker_node/default.nix b/nix/kubernetes/roles/worker_node/default.nix new file mode 100644 index 00000000..9e7d4bd7 --- /dev/null +++ b/nix/kubernetes/roles/worker_node/default.nix @@ -0,0 +1,24 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + options.me = { + worker_node.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install worker_node."; + }; + }; + + config = lib.mkIf config.me.worker_node.enable { + me.containerd.enable = true; + me.kubernetes.enable = true; + }; +} diff --git a/nix/kubernetes/roles/zfs/default.nix b/nix/kubernetes/roles/zfs/default.nix deleted file mode 100644 index ed532277..00000000 --- a/nix/kubernetes/roles/zfs/default.nix +++ /dev/null @@ -1,68 +0,0 @@ -{ - config, - lib, - pkgs, - ... -}: - -let - zfs_clone_send = - (pkgs.writeScriptBin "zfs_clone_send" (builtins.readFile ./files/zfs_clone_send.bash)).overrideAttrs - (old: { - buildCommand = "${old.buildCommand}\n patchShebangs $out"; - - }); - zfs_clone_recv = - (pkgs.writeScriptBin "zfs_clone_recv" (builtins.readFile ./files/zfs_clone_recv.bash)).overrideAttrs - (old: { - buildCommand = "${old.buildCommand}\n patchShebangs $out"; - - }); - zfs_clone_resume = - (pkgs.writeScriptBin "zfs_clone_resume" (builtins.readFile ./files/zfs_clone_resume.bash)) - .overrideAttrs - (old: { - buildCommand = "${old.buildCommand}\n patchShebangs $out"; - - }); -in -{ - imports = [ ]; - - options.me = { - zfs.enable = lib.mkOption { - type = lib.types.bool; - default = false; - example = true; - description = "Whether we want to install zfs."; - }; - }; - - config = lib.mkIf config.me.zfs.enable { - # Technically only needed when building the ISO because nix detects ZFS in the filesystem list normally. I basically always want this so I'm just setting it to always be on. - boot.supportedFilesystems.zfs = true; - - boot.zfs.devNodes = "/dev/disk/by-partuuid"; - - services.zfs = { - autoScrub = { - enable = true; - interval = "monthly"; - }; - trim.enable = true; - }; - - environment.systemPackages = [ - zfs_clone_send - zfs_clone_recv - zfs_clone_resume - ]; - - environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) { - hideMounts = true; - directories = [ - "/etc/zfs/zpool.cache" # Which zpools to import, the root zpool is already imported and does not need this cache file but this captures additional pools. - ]; - }; - }; -} diff --git a/nix/kubernetes/roles/zfs/files/zfs_clone_recv.bash b/nix/kubernetes/roles/zfs/files/zfs_clone_recv.bash deleted file mode 100644 index e51e2c7f..00000000 --- a/nix/kubernetes/roles/zfs/files/zfs_clone_recv.bash +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -# -# A zfs-send alias that creates a perfect clone with good defaults. -set -euo pipefail -IFS=$'\n\t' -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -# -s if the stream is interrupted, save the partial stream. The stream can then be resumed by doing a zfs send -t token where token is the receive_resume_token prop on the dataset we received into. -# -u Do not mount the filesystem we are receiving. We can always mount afterwards but this avoids issues with streams with mountpoints to places like / -# Can optionally add -F to destroy the dataset in the recv location. -exec zfs recv -s -u "${@}" - -# To delete an interrupted recv, run `zfs receive -A dataset` diff --git a/nix/kubernetes/roles/zfs/files/zfs_clone_resume.bash b/nix/kubernetes/roles/zfs/files/zfs_clone_resume.bash deleted file mode 100644 index 8b9ddac5..00000000 --- a/nix/kubernetes/roles/zfs/files/zfs_clone_resume.bash +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -# -# Resume a zfs send. -set -euo pipefail -IFS=$'\n\t' -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -function main { - local hst="$1" - local dst="$2" - local token - token=$(zfs get -H -o value receive_resume_token "$dst") - ssh "$hst" doas zfs send --verbose -t "$token" | doas zfs recv -s "$dst" - -} - -main "${@}" diff --git a/nix/kubernetes/roles/zfs/files/zfs_clone_send.bash b/nix/kubernetes/roles/zfs/files/zfs_clone_send.bash deleted file mode 100644 index 92a325dc..00000000 --- a/nix/kubernetes/roles/zfs/files/zfs_clone_send.bash +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -# -# A zfs-send alias that creates a perfect clone with good defaults. -set -euo pipefail -IFS=$'\n\t' -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -exec zfs send --compressed --replicate --large-block --embed --verbose --raw "${@}" diff --git a/nix/kubernetes/roles/zrepl/default.nix b/nix/kubernetes/roles/zrepl/default.nix deleted file mode 100644 index 2c60485f..00000000 --- a/nix/kubernetes/roles/zrepl/default.nix +++ /dev/null @@ -1,55 +0,0 @@ -{ - config, - lib, - ... -}: - -{ - imports = [ ]; - - options.me = { - zrepl.enable = lib.mkOption { - type = lib.types.bool; - default = false; - example = true; - description = "Whether we want to install zrepl."; - }; - }; - - config = lib.mkIf config.me.zrepl.enable { - services.zrepl = { - enable = true; - settings = { - jobs = [ - { - name = "snapjob"; - type = "snap"; - filesystems = { - "zroot/linux/nix/persist<" = true; - "zroot/bridge<" = true; - }; - snapshotting = { - type = "periodic"; - interval = "15m"; - prefix = "zrepl_"; - }; - pruning = { - keep = [ - { - type = "grid"; - grid = "1x1h(keep=all) | 24x1h | 14x1d"; - regex = "^zrepl_.*"; - } - { - type = "regex"; - negate = true; - regex = "^zrepl_.*"; - } - ]; - }; - } - ]; - }; - }; - }; -}