Add a new ionlybootzfs host for the test VM.

This way it will install far less software.

nix/configuration/configuration.nix

@@ -170,6 +170,7 @@
     ipcalc
     gptfdisk # for cgdisk
     nix-output-monitor # For better view into nixos-rebuild
+    nix-serve-ng # Serve nix store over http
   ];
 
   services.openssh = {

nix/configuration/flake.nix

@@ -29,12 +29,12 @@
 # Install on a new machine:
 #
 #
-# doas nix --experimental-features "nix-command flakes" run github:nix-community/disko/latest -- --mode destroy,format,mount hosts/odo/disk-config.nix
+# doas nix --substituters "http://10.0.2.2:8080?trusted=1 https://cache.nixos.org/" --experimental-features "nix-command flakes" run github:nix-community/disko/latest -- --mode destroy,format,mount hosts/odo/disk-config.nix
 
 # nix flake update zsh-histdb --flake .
 # nix flake update ansible-sshjail --flake .
 # for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
-# nixos-install --flake ".#quark" --substituters "http://10.0.2.2:8080"
+# nixos-install --substituters "http://10.0.2.2:8080?trusted=1 https://cache.nixos.org/" --flake ".#vm_ionlybootzfs"
 #
 
 {
@@ -223,6 +223,23 @@
                 modules = main.modules ++ additional_vm_modules ++ additional_iso_modules;
               };
             };
+          ionlybootzfs = rec {
+            main = base_x86_64_linux // {
+              modules = base_x86_64_linux.modules ++ [
+                ./hosts/ionlybootzfs
+              ];
+            };
+            iso = main // {
+              modules = main.modules ++ additional_iso_modules;
+            };
+            vm = main // {
+              modules = main.modules ++ additional_vm_modules;
+            };
+            vm_iso = main // {
+              modules = main.modules ++ additional_vm_modules ++ additional_iso_modules;
+            };
+          };
+
         };
     in
     {
@@ -245,5 +262,11 @@
       iso.hydra = (nixpkgs.lib.nixosSystem systems.hydra.iso).config.system.build.isoImage;
       nixosConfigurations.vm_hydra = nixpkgs.lib.nixosSystem systems.hydra.vm;
       vm_iso.hydra = (nixpkgs.lib.nixosSystem systems.hydra.vm_iso).config.system.build.isoImage;
+
+      nixosConfigurations.ionlybootzfs = nixpkgs.lib.nixosSystem systems.ionlybootzfs.main;
+      iso.ionlybootzfs = (nixpkgs.lib.nixosSystem systems.ionlybootzfs.iso).config.system.build.isoImage;
+      nixosConfigurations.vm_ionlybootzfs = nixpkgs.lib.nixosSystem systems.ionlybootzfs.vm;
+      vm_iso.ionlybootzfs =
+        (nixpkgs.lib.nixosSystem systems.ionlybootzfs.vm_iso).config.system.build.isoImage;
     };
 }

nix/configuration/hosts/ionlybootzfs/DEPLOY_BOOT

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+#TARGET=10.216.1.14
+# TARGET=192.168.211.250
+TARGET="ionlybootzfs"
+
+nix flake update zsh-histdb --flake "$DIR/../../"
+nix flake update ansible-sshjail --flake "$DIR/../../"
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+
+nixos-rebuild boot --flake "$DIR/../../#ionlybootzfs" --target-host "$TARGET" --build-host "$TARGET" --fast --use-remote-sudo --max-jobs "$JOBS" "${@}" |& nom
+
+# rsync -av --progress --delete --exclude=.git "$DIR/../../../configuration" "talexander@${TARGET}:/persist/manual/" && ssh talexander@${TARGET} 'cd /persist/manual/configuration && nix flake update zsh-histdb && nix flake update ansible-sshjail && doas nice -n 19 nixos-rebuild boot --flake /persist/manual/configuration#ionlybootzfs'

nix/configuration/hosts/ionlybootzfs/DEPLOY_SWITCH

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+#TARGET=10.216.1.14
+# TARGET=192.168.211.250
+TARGET=ionlybootzfs
+
+nix flake update zsh-histdb --flake "$DIR/../../"
+nix flake update ansible-sshjail --flake "$DIR/../../"
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+
+nixos-rebuild switch --flake "$DIR/../../#ionlybootzfs" --target-host "$TARGET" --build-host "$TARGET" --fast --use-remote-sudo --max-jobs "$JOBS" "${@}" |& nom
+
+# rsync -av --progress --delete --exclude=.git "$DIR/../../../configuration" "talexander@${TARGET}:/persist/manual/" && ssh talexander@${TARGET} 'cd /persist/manual/configuration && nix flake update zsh-histdb && nix flake update ansible-sshjail && doas nice -n 19 nixos-rebuild switch --flake /persist/manual/configuration#ionlybootzfs'

nix/configuration/hosts/ionlybootzfs/ISO

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: "${JOBS:="1"}"
+
+nix flake update zsh-histdb --flake "$DIR/../../"
+nix flake update ansible-sshjail --flake "$DIR/../../"
+for f in /persist/manual/manual_add_to_store/*; do nix-store --add-fixed sha256 "$f"; done
+nix build --extra-experimental-features nix-command --extra-experimental-features flakes "$DIR/../..#iso.ionlybootzfs" --max-jobs "$JOBS" "${@}" |& nom

nix/configuration/hosts/ionlybootzfs/default.nix

@@ -0,0 +1,49 @@
+#
+# Testing:
+# doas "$(nix-build '<nixpkgs>' --no-out-link -A 'qemu')/bin/qemu-system-x86_64" \
+#      -accel kvm \
+#      -cpu host \
+#      -smp cores=8 \
+#      -m 32768 \
+#      -drive "file=$(nix-build '<nixpkgs>' --no-out-link -A 'OVMF.fd')/FV/OVMF.fd,if=pflash,format=raw,readonly=on" \
+#      -drive file=/tmp/localdisk.img,if=none,id=nvm,format=raw \
+#      -device nvme,serial=deadbeef,drive=nvm \
+#      -nic user,hostfwd=tcp::60022-:22 \
+#      -boot order=d \
+#      -cdrom "$(readlink -f /persist/machine_setup/nix/configuration/result/iso/nixos*.iso)" \
+#      -display vnc=127.0.0.1:0
+#
+
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  imports = [
+    ./wrapped-disk-config.nix
+    ./hardware-configuration.nix
+    ./optimized_build.nix
+  ];
+
+  # Generate with `head -c4 /dev/urandom | od -A none -t x4`
+  networking.hostId = "fbd233d8";
+
+  networking.hostName = "ionlybootzfs"; # Define your hostname.
+
+  time.timeZone = "America/New_York";
+  i18n.defaultLocale = "en_US.UTF-8";
+
+  me.optimizations.enable = false;
+  me.secureBoot.enable = true;
+
+  # Mount tmpfs at /tmp
+  boot.tmp.useTmpfs = true;
+
+  me.emacs_flavor = "plainmacs";
+  me.graphical = false;
+  me.wireguard.activated = [ ];
+  me.wireguard.deactivated = [ ];
+  me.zsh.enable = true;
+}

nix/configuration/hosts/ionlybootzfs/disk-config.nix

@@ -0,0 +1,142 @@
+# Manual Step:
+#   Check if drive supports 4kn: nvme id-ns -H /dev/nvme0n1
+#   Format the drive to 4kn: nvme format --lbaf=1 /dev/nvme0n1
+
+{
+  disko.devices = {
+    disk = {
+      main = {
+        type = "disk";
+        device = "/dev/nvme0n1";
+        content = {
+          type = "gpt";
+          partitions = {
+            ESP = {
+              size = "1G";
+              type = "EF00";
+              content = {
+                type = "filesystem";
+                format = "vfat";
+                mountpoint = "/boot";
+                mountOptions = [
+                  "umask=0077"
+                  "noatime"
+                  "discard"
+                ];
+              };
+            };
+            zfs = {
+              size = "100%";
+              content = {
+                type = "zfs";
+                pool = "zroot";
+              };
+            };
+          };
+        };
+      };
+    };
+    zpool = {
+      zroot = {
+        type = "zpool";
+        # mode = "mirror";
+        # Workaround: cannot import 'zroot': I/O error in disko tests
+        options.cachefile = "none";
+        options = {
+          ashift = "12";
+          compatibility = "openzfs-2.2-freebsd";
+          autotrim = "on";
+        };
+        rootFsOptions = {
+          acltype = "posixacl";
+          atime = "off";
+          relatime = "off";
+          xattr = "sa";
+          mountpoint = "none";
+          compression = "lz4";
+          canmount = "off";
+          utf8only = "on";
+          dnodesize = "auto";
+          normalization = "formD";
+        };
+
+        datasets = {
+          "linux/nix" = {
+            type = "zfs_fs";
+            options.mountpoint = "none";
+            options = {
+              encryption = "aes-256-gcm";
+              keyformat = "passphrase";
+              # keylocation = "file:///tmp/secret.key";
+            };
+          };
+          "linux/nix/root" = {
+            type = "zfs_fs";
+            options.mountpoint = "legacy";
+            mountpoint = "/";
+            postCreateHook = "zfs list -t snapshot -H -o name | grep -E '^zroot/linux/nix/root@blank$' || zfs snapshot zroot/linux/nix/root@blank";
+          };
+          "linux/nix/nix" = {
+            type = "zfs_fs";
+            options.mountpoint = "legacy";
+            mountpoint = "/nix";
+            postCreateHook = "zfs list -t snapshot -H -o name | grep -E '^zroot/linux/nix/nix@blank$' || zfs snapshot zroot/linux/nix/nix@blank";
+            options = {
+              recordsize = "16MiB";
+              compression = "zstd-19";
+            };
+          };
+          "linux/nix/home" = {
+            type = "zfs_fs";
+            options.mountpoint = "legacy";
+            mountpoint = "/home";
+            postCreateHook = "zfs list -t snapshot -H -o name | grep -E '^zroot/linux/nix/home@blank$' || zfs snapshot zroot/linux/nix/home@blank";
+          };
+          "linux/nix/persist" = {
+            type = "zfs_fs";
+            options.mountpoint = "legacy";
+            mountpoint = "/persist";
+            postCreateHook = "zfs list -t snapshot -H -o name | grep -E '^zroot/linux/nix/persist@blank$' || zfs snapshot zroot/linux/nix/persist@blank";
+          };
+          "linux/nix/state" = {
+            type = "zfs_fs";
+            options.mountpoint = "legacy";
+            mountpoint = "/state";
+            postCreateHook = "zfs list -t snapshot -H -o name | grep -E '^zroot/linux/nix/state@blank$' || zfs snapshot zroot/linux/nix/state@blank";
+          };
+        };
+      };
+    };
+  };
+
+  # Make sure all persistent volumes are marked as neededForBoot
+  #
+  # Also mounts /home so it is mounted before the user home directories are created.
+  fileSystems."/persist".neededForBoot = true;
+  fileSystems."/state".neededForBoot = true;
+  fileSystems."/home".neededForBoot = true;
+
+  fileSystems."/".options = [
+    "noatime"
+    "norelatime"
+  ];
+  fileSystems."/nix".options = [
+    "noatime"
+    "norelatime"
+  ];
+  fileSystems."/persist".options = [
+    "noatime"
+    "norelatime"
+  ];
+  fileSystems."/state".options = [
+    "noatime"
+    "norelatime"
+  ];
+  fileSystems."/home".options = [
+    "noatime"
+    "norelatime"
+  ];
+
+  # Only attempt to decrypt the main pool. Otherwise it attempts to decrypt pools that aren't even used.
+  boot.zfs.requestEncryptionCredentials = [ "zroot/linux/nix" ];
+}

nix/configuration/hosts/ionlybootzfs/hardware-configuration.nix

@@ -0,0 +1,38 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{
+  config,
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [
+    (modulesPath + "/installer/scan/not-detected.nix")
+  ];
+
+  boot.initrd.availableKernelModules = [
+    "xhci_pci"
+    "nvme"
+    "usbhid"
+    "usb_storage"
+    "sd_mod"
+    "sdhci_pci"
+  ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.dhcpcd.enable = lib.mkForce true;
+  networking.useDHCP = lib.mkForce true;
+  # systemd.network.enable = true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+}

nix/configuration/hosts/ionlybootzfs/optimized_build.nix

@@ -0,0 +1,131 @@
+{
+  config,
+  lib,
+  pkgs,
+  pkgs-unoptimized,
+  ...
+}:
+{
+  imports = [ ];
+
+  config = lib.mkMerge [
+    { }
+    (lib.mkIf (!config.me.optimizations.enable) {
+      boot.kernelPackages = pkgs.linuxPackagesFor pkgs.linux_6_14;
+    })
+    (lib.mkIf (config.me.optimizations.enable) {
+      nixpkgs.hostPlatform = {
+        gcc.arch = "znver4";
+        gcc.tune = "znver4";
+        system = "x86_64-linux";
+      };
+
+      nixpkgs.overlays = [
+        (
+          final: prev:
+          let
+            addConfig =
+              additionalConfig: pkg:
+              pkg.override (oldconfig: {
+                structuredExtraConfig = pkg.structuredExtraConfig // additionalConfig;
+              });
+          in
+          {
+            linux_me = addConfig {
+              # Full preemption
+              PREEMPT = lib.mkOverride 60 lib.kernel.yes;
+              PREEMPT_VOLUNTARY = lib.mkOverride 60 lib.kernel.no;
+
+              # Google's BBRv3 TCP congestion Control
+              TCP_CONG_BBR = lib.kernel.yes;
+              DEFAULT_BBR = lib.kernel.yes;
+
+              # Preemptive Full Tickless Kernel at 300Hz
+              HZ = lib.kernel.freeform "300";
+              HZ_300 = lib.kernel.yes;
+              HZ_1000 = lib.kernel.no;
+            } prev.linux_6_14;
+            # gsl = prev.gsl.overrideAttrs (old: {
+            #   # gsl tests fails when optimizations are enabled.
+            #   # > FAIL: cholesky_invert unscaled hilbert (  4,  4)[0,2]: 2.55795384873636067e-13                        0
+            #   # >  (2.55795384873636067e-13 observed vs 0 expected) [28259614]
+            #   doCheck = false;
+            # });
+          }
+        )
+        (final: prev: {
+          haskellPackages = prev.haskellPackages.extend (
+            final': prev': {
+              inherit (pkgs-unoptimized.haskellPackages)
+                crypton
+                crypton-connection
+                crypton-x509
+                crypton-x509-store
+                crypton-x509-system
+                crypton-x509-validation
+                hspec-wai
+                http-client-tls
+                http2
+                pandoc
+                pandoc-cli
+                pandoc-lua-engine
+                pandoc-server
+                servant-server
+                tls
+                wai-app-static
+                wai-extra
+                warp
+                ;
+            }
+          );
+        })
+        (final: prev: {
+          inherit (pkgs-unoptimized)
+            gsl
+            redis
+            valkey
+            ;
+        })
+      ];
+
+      boot.kernelPackages = pkgs.linuxPackagesFor pkgs.linux_me;
+    })
+    (lib.mkIf (!config.me.buildingIso) {
+      nix.settings.system-features = lib.mkForce [
+        "gccarch-znver4"
+        "gccarch-skylake"
+        # "gccarch-alderlake" missing WAITPKG
+        "gccarch-x86-64-v3"
+        "gccarch-x86-64-v4"
+        "benchmark"
+        "big-parallel"
+        "kvm"
+        "nixos-test"
+      ];
+
+      # Keep ALL dependencies so we can rebuild offline. This DRASTICALLY increase disk usage, but disk space is cheap.
+      # system.includeBuildDependencies = true;
+
+      # This also should enable building offline? TODO: test.
+      nix.extraOptions = ''
+        keep-outputs = true
+        keep-derivations = true
+      '';
+
+      # # building ON
+      # nixpkgs.localSystem = { system = "aarch64-linux"; };
+      # # building FOR
+      # nixpkgs.crossSystem = { system = "aarch64-linux"; };
+
+      # nixpkgs.config = {
+      #   replaceStdenv = ({ pkgs }: pkgs.clangStdenv);
+      # };
+      # or maybe an overlay
+      # stdenv = prev.clangStdenv;
+
+    })
+    (lib.mkIf (config.me.buildingIso) {
+      boot.supportedFilesystems.zfs = true;
+    })
+  ];
+}

nix/configuration/hosts/ionlybootzfs/wrapped-disk-config.nix

@@ -0,0 +1,8 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+lib.mkIf (!config.me.buildingIso) (import ./disk-config.nix)

nix/configuration/roles/ssh/files/ssh_config

@@ -34,5 +34,9 @@ Host hydra
   ProxyJump talexander@mrmanager
   HostName 10.215.1.219
 
+Host ionlybootzfs
+  HostName 127.0.0.1
+  Port 60022
+
 Host *
   Compression yes