Configure kernel preemption.

nix/kubernetes/configuration.nix

@@ -17,6 +17,7 @@
     ./roles/firewall
     ./roles/image_based_appliance
     ./roles/iso
+    ./roles/kernel
     ./roles/kube_apiserver
     ./roles/kube_controller_manager
     ./roles/kube_proxy

nix/kubernetes/flake.lock

@@ -164,11 +164,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1780243769,
+        "lastModified": 1780749050,
-        "narHash": "sha256-x5UQuRsH3MqI0U9afaXSNqzTPSeZlRLvFAav2Ux1pNw=",
+        "narHash": "sha256-3av0pIjlOWQ6rDbNOmpUSvbNnJkGORQKKjb4LtCZsIY=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "331800de5053fcebacf6813adb5db9c9dca22a0c",
+        "rev": "a799d3e3886da994fa307f817a6bc705ae538eeb",
         "type": "github"
       },
       "original": {

nix/kubernetes/roles/kernel/default.nix

@@ -0,0 +1,192 @@
+# Check current config:
+#   nix build '/persist/machine_setup/nix/configuration#nixosConfigurations.hydra.pkgs.linux_me.configfile'
+#   cat $(nix eval --raw '/persist/machine_setup/nix/configuration#nixosConfigurations.hydra.pkgs.linux_me.configfile') | less
+
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  preemption_type = with lib.kernel; {
+    full = {
+      PREEMPT_DYNAMIC = yes;
+      PREEMPT = yes;
+      PREEMPT_VOLUNTARY = lib.mkForce no;
+      PREEMPT_LAZY = lib.mkForce no;
+      PREEMPT_NONE = no;
+    };
+    lazy = {
+      PREEMPT_DYNAMIC = yes;
+      PREEMPT = no;
+      PREEMPT_VOLUNTARY = lib.mkForce no;
+      PREEMPT_LAZY = yes;
+      PREEMPT_NONE = no;
+    };
+    voluntary = {
+      PREEMPT_DYNAMIC = no;
+      PREEMPT = no;
+      PREEMPT_VOLUNTARY = yes;
+      PREEMPT_LAZY = lib.mkForce no;
+      PREEMPT_NONE = no;
+    };
+    none = {
+      PREEMPT_DYNAMIC = no;
+      PREEMPT = no;
+      PREEMPT_VOLUNTARY = lib.mkForce no;
+      PREEMPT_LAZY = lib.mkForce no;
+      PREEMPT_NONE = yes;
+    };
+  };
+  tick_hz =
+    with lib.kernel;
+    {
+      "1000" = {
+        HZ_1000 = yes;
+        HZ = freeform "1000";
+      };
+    }
+    // lib.genAttrs [ "100" "250" "300" "500" "600" "750" ] (hz: {
+      HZ_1000 = no;
+      "HZ_${hz}" = yes;
+      HZ = freeform hz;
+    });
+  performance_governor = with lib.kernel; {
+    default = {
+      CPU_FREQ_DEFAULT_GOV_SCHEDUTIL = yes;
+    };
+    performance = {
+      CPU_FREQ_DEFAULT_GOV_SCHEDUTIL = no;
+      CPU_FREQ_DEFAULT_GOV_PERFORMANCE = yes;
+    };
+  };
+  tick_rate = with lib.kernel; {
+    # Always tick at the hz frequency.
+    periodic = {
+      NO_HZ_IDLE = no;
+      NO_HZ_FULL = no;
+      NO_HZ = no;
+      NO_HZ_COMMON = no;
+      HZ_PERIODIC = yes;
+    };
+    # Idle - Do not disturb the CPU when idle. This can save power but increase latency.
+    idle = {
+      HZ_PERIODIC = no;
+      NO_HZ_FULL = no;
+      NO_HZ_IDLE = yes;
+      NO_HZ = yes;
+      NO_HZ_COMMON = yes;
+    };
+    # Full dyntick system (tickless) - The kernel tries to shut down the tick whenever possible.
+    tickless = {
+      HZ_PERIODIC = no;
+      NO_HZ_IDLE = no;
+      NO_HZ_FULL = yes;
+      NO_HZ = yes;
+      NO_HZ_COMMON = yes;
+      CONTEXT_TRACKING = yes;
+    };
+  };
+  huge_page = with lib.kernel; {
+    always = {
+      TRANSPARENT_HUGEPAGE_MADVISE = no;
+      TRANSPARENT_HUGEPAGE_ALWAYS = yes;
+    };
+    madvise = {
+      TRANSPARENT_HUGEPAGE_ALWAYS = no;
+      TRANSPARENT_HUGEPAGE_MADVISE = yes;
+    };
+  };
+  common_config = with lib.kernel; {
+    # Google's BBRv3 TCP congestion Control
+    TCP_CONG_BBR = yes;
+    DEFAULT_BBR = yes;
+  };
+  flavors = {
+    server = lib.mkMerge [
+      preemption_type.none
+      tick_hz."300"
+      performance_governor.default
+      tick_rate.tickless
+      huge_page.madvise
+    ];
+    interactive =
+      with lib.kernel;
+      lib.mkMerge [
+        {
+          # Enable RCU Lazy - Reduces power consumption when idle or lightly loaded. Useful for battery-powered devices like laptops.
+          RCU_LAZY = yes;
+        }
+        preemption_type.lazy
+        tick_hz."300"
+        performance_governor.default
+        tick_rate.tickless
+        huge_page.madvise
+      ];
+  };
+in
+{
+  imports = [ ];
+
+  options.me = {
+    kernel.enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+      description = "Whether we want to install kernel.";
+    };
+
+    kernel.version = lib.mkOption {
+      type = lib.types.str;
+      default = "linux"; # LTS
+      example = "linux_6_18";
+      description = "What version of the kernl should we use.";
+    };
+
+    kernel.flavor = lib.mkOption {
+      type = lib.types.str;
+      default = "server";
+      example = "interactive";
+      description = "What type of kernel should be built.";
+    };
+  };
+
+  config = lib.mkIf config.me.kernel.enable (
+    lib.mkMerge [
+      {
+        boot.kernelPackages = pkgs.linuxPackagesFor pkgs.linux_me;
+      }
+      (lib.mkIf (!config.me.optimizations.enable) {
+        nixpkgs.overlays = [
+          (final: prev: {
+            linux_me = final."${config.me.kernel.version}";
+          })
+        ];
+      })
+      (lib.mkIf (config.me.optimizations.enable) {
+        nixpkgs.overlays = [
+          (
+            final: prev:
+            let
+              addConfig =
+                additionalConfig: pkg:
+                pkg.override (oldconfig: {
+                  structuredExtraConfig = lib.mkMerge ([ pkg.structuredExtraConfig ] ++ additionalConfig);
+                  # stdenv = pkgs.llvmPackages_latest.stdenv;
+                  # stdenv = pkgs.clangStdenv;
+                });
+            in
+            {
+              linux_me = addConfig ([
+                common_config
+                flavors."${config.me.kernel.flavor}"
+              ]) final."${config.me.kernel.version}";
+            }
+          )
+        ];
+      })
+    ]
+  );
+}

nix/kubernetes/roles/minimal_base/default.nix

@@ -19,6 +19,7 @@
 
   config = lib.mkIf config.me.minimal_base.enable {
     me.doas.enable = true;
+    me.kernel.enable = true;
     me.network.enable = true;
     me.nvme.enable = true;
     me.ssh.enable = true;

nix/kubernetes/roles/optimized_build/default.nix

@@ -49,65 +49,29 @@
   };
 
   config = lib.mkMerge [
-    (lib.mkIf (!config.me.optimizations.enable) (
-      lib.mkMerge [
-        {
-          # boot.kernelPackages = pkgs.linuxPackagesFor pkgs.linux_6_17;
-          boot.kernelPackages = pkgs.linuxPackagesFor pkgs.linux;
-        }
-      ]
-    ))
     (lib.mkIf config.me.optimizations.enable (
       lib.mkMerge [
         {
-          boot.kernelPackages = pkgs.linuxPackagesFor pkgs.linux_me;
-
           nixpkgs.hostPlatform = {
             gcc.arch = config.me.optimizations.arch;
             gcc.tune = config.me.optimizations.arch;
           };
 
-          nixpkgs.overlays = [
+          # nixpkgs.overlays = [
-            (
+          #   (final: prev: {
-              final: prev:
+          #     inherit (final.unoptimized)
-              let
+          #       assimp
-                addConfig =
+          #       binaryen
-                  additionalConfig: pkg:
+          #       gsl
-                  pkg.override (oldconfig: {
+          #       rapidjson
-                    structuredExtraConfig = pkg.structuredExtraConfig // additionalConfig;
+          #       ffmpeg-headless
-                  });
+          #       ffmpeg
-              in
+          #       pipewire
-              {
+          #       chromaprint
-                linux_me = addConfig {
+          #       gtkmm
-                  # Full preemption
+          #       ;
-                  PREEMPT = lib.mkOverride 60 lib.kernel.yes;
+          #   })
-                  PREEMPT_VOLUNTARY = lib.mkOverride 60 lib.kernel.no;
+          # ];
-
-                  # Google's BBRv3 TCP congestion Control
-                  TCP_CONG_BBR = lib.kernel.yes;
-                  DEFAULT_BBR = lib.kernel.yes;
-
-                  # Preemptive Full Tickless Kernel at 300Hz
-                  HZ = lib.kernel.freeform "300";
-                  HZ_300 = lib.kernel.yes;
-                  HZ_1000 = lib.kernel.no;
-                } prev.linux; # or prev.linux_6_17
-              }
-            )
-            (final: prev: {
-              inherit (final.unoptimized)
-                assimp
-                binaryen
-                gsl
-                rapidjson
-                ffmpeg-headless
-                ffmpeg
-                pipewire
-                chromaprint
-                gtkmm
-                ;
-            })
-          ];
         }
       ]
     ))