diff --git a/nix/kubernetes/configuration.nix b/nix/kubernetes/configuration.nix index e6d5ef5..ef64f2f 100644 --- a/nix/kubernetes/configuration.nix +++ b/nix/kubernetes/configuration.nix @@ -17,6 +17,7 @@ ./roles/firewall ./roles/image_based_appliance ./roles/iso + ./roles/kernel ./roles/kube_apiserver ./roles/kube_controller_manager ./roles/kube_proxy diff --git a/nix/kubernetes/flake.lock b/nix/kubernetes/flake.lock index 2824a29..8f09179 100644 --- a/nix/kubernetes/flake.lock +++ b/nix/kubernetes/flake.lock @@ -164,11 +164,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1780243769, - "narHash": "sha256-x5UQuRsH3MqI0U9afaXSNqzTPSeZlRLvFAav2Ux1pNw=", + "lastModified": 1780749050, + "narHash": "sha256-3av0pIjlOWQ6rDbNOmpUSvbNnJkGORQKKjb4LtCZsIY=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "331800de5053fcebacf6813adb5db9c9dca22a0c", + "rev": "a799d3e3886da994fa307f817a6bc705ae538eeb", "type": "github" }, "original": { diff --git a/nix/kubernetes/roles/kernel/default.nix b/nix/kubernetes/roles/kernel/default.nix new file mode 100644 index 0000000..7979a68 --- /dev/null +++ b/nix/kubernetes/roles/kernel/default.nix @@ -0,0 +1,192 @@ +# Check current config: +# nix build '/persist/machine_setup/nix/configuration#nixosConfigurations.hydra.pkgs.linux_me.configfile' +# cat $(nix eval --raw '/persist/machine_setup/nix/configuration#nixosConfigurations.hydra.pkgs.linux_me.configfile') | less + +{ + config, + lib, + pkgs, + ... +}: + +let + preemption_type = with lib.kernel; { + full = { + PREEMPT_DYNAMIC = yes; + PREEMPT = yes; + PREEMPT_VOLUNTARY = lib.mkForce no; + PREEMPT_LAZY = lib.mkForce no; + PREEMPT_NONE = no; + }; + lazy = { + PREEMPT_DYNAMIC = yes; + PREEMPT = no; + PREEMPT_VOLUNTARY = lib.mkForce no; + PREEMPT_LAZY = yes; + PREEMPT_NONE = no; + }; + voluntary = { + PREEMPT_DYNAMIC = no; + PREEMPT = no; + PREEMPT_VOLUNTARY = yes; + PREEMPT_LAZY = lib.mkForce no; + PREEMPT_NONE = no; + }; + none = { + PREEMPT_DYNAMIC = no; + PREEMPT = no; + PREEMPT_VOLUNTARY = lib.mkForce no; + PREEMPT_LAZY = lib.mkForce no; + PREEMPT_NONE = yes; + }; + }; + tick_hz = + with lib.kernel; + { + "1000" = { + HZ_1000 = yes; + HZ = freeform "1000"; + }; + } + // lib.genAttrs [ "100" "250" "300" "500" "600" "750" ] (hz: { + HZ_1000 = no; + "HZ_${hz}" = yes; + HZ = freeform hz; + }); + performance_governor = with lib.kernel; { + default = { + CPU_FREQ_DEFAULT_GOV_SCHEDUTIL = yes; + }; + performance = { + CPU_FREQ_DEFAULT_GOV_SCHEDUTIL = no; + CPU_FREQ_DEFAULT_GOV_PERFORMANCE = yes; + }; + }; + tick_rate = with lib.kernel; { + # Always tick at the hz frequency. + periodic = { + NO_HZ_IDLE = no; + NO_HZ_FULL = no; + NO_HZ = no; + NO_HZ_COMMON = no; + HZ_PERIODIC = yes; + }; + # Idle - Do not disturb the CPU when idle. This can save power but increase latency. + idle = { + HZ_PERIODIC = no; + NO_HZ_FULL = no; + NO_HZ_IDLE = yes; + NO_HZ = yes; + NO_HZ_COMMON = yes; + }; + # Full dyntick system (tickless) - The kernel tries to shut down the tick whenever possible. + tickless = { + HZ_PERIODIC = no; + NO_HZ_IDLE = no; + NO_HZ_FULL = yes; + NO_HZ = yes; + NO_HZ_COMMON = yes; + CONTEXT_TRACKING = yes; + }; + }; + huge_page = with lib.kernel; { + always = { + TRANSPARENT_HUGEPAGE_MADVISE = no; + TRANSPARENT_HUGEPAGE_ALWAYS = yes; + }; + madvise = { + TRANSPARENT_HUGEPAGE_ALWAYS = no; + TRANSPARENT_HUGEPAGE_MADVISE = yes; + }; + }; + common_config = with lib.kernel; { + # Google's BBRv3 TCP congestion Control + TCP_CONG_BBR = yes; + DEFAULT_BBR = yes; + }; + flavors = { + server = lib.mkMerge [ + preemption_type.none + tick_hz."300" + performance_governor.default + tick_rate.tickless + huge_page.madvise + ]; + interactive = + with lib.kernel; + lib.mkMerge [ + { + # Enable RCU Lazy - Reduces power consumption when idle or lightly loaded. Useful for battery-powered devices like laptops. + RCU_LAZY = yes; + } + preemption_type.lazy + tick_hz."300" + performance_governor.default + tick_rate.tickless + huge_page.madvise + ]; + }; +in +{ + imports = [ ]; + + options.me = { + kernel.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install kernel."; + }; + + kernel.version = lib.mkOption { + type = lib.types.str; + default = "linux"; # LTS + example = "linux_6_18"; + description = "What version of the kernl should we use."; + }; + + kernel.flavor = lib.mkOption { + type = lib.types.str; + default = "server"; + example = "interactive"; + description = "What type of kernel should be built."; + }; + }; + + config = lib.mkIf config.me.kernel.enable ( + lib.mkMerge [ + { + boot.kernelPackages = pkgs.linuxPackagesFor pkgs.linux_me; + } + (lib.mkIf (!config.me.optimizations.enable) { + nixpkgs.overlays = [ + (final: prev: { + linux_me = final."${config.me.kernel.version}"; + }) + ]; + }) + (lib.mkIf (config.me.optimizations.enable) { + nixpkgs.overlays = [ + ( + final: prev: + let + addConfig = + additionalConfig: pkg: + pkg.override (oldconfig: { + structuredExtraConfig = lib.mkMerge ([ pkg.structuredExtraConfig ] ++ additionalConfig); + # stdenv = pkgs.llvmPackages_latest.stdenv; + # stdenv = pkgs.clangStdenv; + }); + in + { + linux_me = addConfig ([ + common_config + flavors."${config.me.kernel.flavor}" + ]) final."${config.me.kernel.version}"; + } + ) + ]; + }) + ] + ); +} diff --git a/nix/kubernetes/roles/minimal_base/default.nix b/nix/kubernetes/roles/minimal_base/default.nix index e57f1f2..54261f2 100644 --- a/nix/kubernetes/roles/minimal_base/default.nix +++ b/nix/kubernetes/roles/minimal_base/default.nix @@ -19,6 +19,7 @@ config = lib.mkIf config.me.minimal_base.enable { me.doas.enable = true; + me.kernel.enable = true; me.network.enable = true; me.nvme.enable = true; me.ssh.enable = true; diff --git a/nix/kubernetes/roles/optimized_build/default.nix b/nix/kubernetes/roles/optimized_build/default.nix index d569df4..65fd074 100644 --- a/nix/kubernetes/roles/optimized_build/default.nix +++ b/nix/kubernetes/roles/optimized_build/default.nix @@ -49,65 +49,29 @@ }; config = lib.mkMerge [ - (lib.mkIf (!config.me.optimizations.enable) ( - lib.mkMerge [ - { - # boot.kernelPackages = pkgs.linuxPackagesFor pkgs.linux_6_17; - boot.kernelPackages = pkgs.linuxPackagesFor pkgs.linux; - } - ] - )) (lib.mkIf config.me.optimizations.enable ( lib.mkMerge [ { - boot.kernelPackages = pkgs.linuxPackagesFor pkgs.linux_me; - nixpkgs.hostPlatform = { gcc.arch = config.me.optimizations.arch; gcc.tune = config.me.optimizations.arch; }; - nixpkgs.overlays = [ - ( - final: prev: - let - addConfig = - additionalConfig: pkg: - pkg.override (oldconfig: { - structuredExtraConfig = pkg.structuredExtraConfig // additionalConfig; - }); - in - { - linux_me = addConfig { - # Full preemption - PREEMPT = lib.mkOverride 60 lib.kernel.yes; - PREEMPT_VOLUNTARY = lib.mkOverride 60 lib.kernel.no; - - # Google's BBRv3 TCP congestion Control - TCP_CONG_BBR = lib.kernel.yes; - DEFAULT_BBR = lib.kernel.yes; - - # Preemptive Full Tickless Kernel at 300Hz - HZ = lib.kernel.freeform "300"; - HZ_300 = lib.kernel.yes; - HZ_1000 = lib.kernel.no; - } prev.linux; # or prev.linux_6_17 - } - ) - (final: prev: { - inherit (final.unoptimized) - assimp - binaryen - gsl - rapidjson - ffmpeg-headless - ffmpeg - pipewire - chromaprint - gtkmm - ; - }) - ]; + # nixpkgs.overlays = [ + # (final: prev: { + # inherit (final.unoptimized) + # assimp + # binaryen + # gsl + # rapidjson + # ffmpeg-headless + # ffmpeg + # pipewire + # chromaprint + # gtkmm + # ; + # }) + # ]; } ] ))