talexander/machine_setup
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Add kube-proxy.

Browse Source
This commit is contained in:
Tom Alexander 2025-12-16 21:07:39 -05:00 committed by Tom Alexander
parent 0463d2cbd1
commit c61da527f2
Signed by: talexander
GPG Key ID: 36C99E8B3C39D85F
12 changed files with 185 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
nix/configuration/README.org Normal file
View File

@ -0,0 +1,12 @@
* To-do
** Perhaps use overlay for /etc for speedup
#+begin_src nix
system.etc.overlay.enable = true;
#+end_src
** read https://nixos.org/manual/nixos/stable/
** Performance for mini pc
#+begin_src nix
security.pam.loginLimits = [
{ domain = "@users"; item = "rtprio"; type = "-"; value = 1; }
];
#+end_src

14
nix/kubernetes/README.org
View File

@ -10,3 +10,17 @@
{ domain = "@users"; item = "rtprio"; type = "-"; value = 1; }
];
#+end_src
* Bootstrap
** Install cilium
#+begin_src bash
helm repo add cilium https://helm.cilium.io/
helm install --dry-run cilium cilium/cilium --version 1.18.4 --namespace kube-system \
--set kubeProxyReplacement=true \
--set k8sServiceHost=${API_SERVER_IP} \
--set k8sServicePort=${API_SERVER_PORT}
kubectl -n kube-system exec ds/cilium -- cilium-dbg status --verbose
kubectl -n kube-system exec ds/cilium -- cilium-dbg status | grep KubeProxyReplacement
#+end_src

2
nix/kubernetes/configuration.nix
View File

@ -12,10 +12,12 @@
./roles/doas
./roles/dont_use_substituters
./roles/etcd
./roles/firewall
./roles/image_based_appliance
./roles/iso
./roles/kube_apiserver
./roles/kube_controller_manager
./roles/kube_proxy
./roles/kube_scheduler
./roles/kubelet
./roles/kubernetes

9
nix/kubernetes/keys/package/deploy-script/package.nix
View File

@ -212,7 +212,14 @@ let
file = "${
k8s.client-configs."${vm_name_to_hostname vm_name}"
}/${vm_name_to_hostname vm_name}.kubeconfig";
name = "kubeconfig";
name = "kubelet.kubeconfig";
owner = 10024;
group = 10024;
mode = "0600";
}
{
dest_dir = "/vm/${vm_name}/persist/keys/kube";
file = "${k8s.client-configs.kube-proxy}/kube-proxy.kubeconfig";
owner = 10024;
group = 10024;
mode = "0600";

4
nix/kubernetes/keys/scope.nix
View File

@ -140,7 +140,9 @@ makeScope newScope (
};
kube-proxy = {
config_user = "system:kube-proxy";
config_server = "https://server.kubernetes.local:6443";
config_server = "https://[2620:11f:7001:7:ffff:ffff:ad7:1dd]:6443";
# config_server = "https://127.0.0.1:6443";
# config_server = "https://server.kubernetes.local:6443";
};
kube-controller-manager = {
config_user = "system:kube-controller-manager";

49
nix/kubernetes/roles/containerd/default.nix
View File

@ -19,28 +19,39 @@
config = lib.mkIf config.me.containerd.enable {
virtualisation.containerd.enable = true;
virtualisation.containerd.settings = {
"plugins" = {
"io.containerd.grpc.v1.cri" = {
"cni" = {
"bin_dir" = "/opt/cni/bin";
"conf_dir" = "/etc/cni/net.d";
};
"containerd" = {
"default_runtime_name" = "runc";
"runtimes" = {
"runc" = {
"options" = {
"SystemdCgroup" = true;
};
"runtime_type" = "io.containerd.runc.v2";
};
virtualisation.containerd.settings =
# let
# my-cni-plugins = pkgs.buildEnv {
# name = "my-cni-plugins";
# paths = with pkgs; [
# cni-plugins
# cni-plugin-flannel
# ];
# };
# in
{
"plugins" = {
"io.containerd.grpc.v1.cri" = {
"cni" = {
"bin_dir" = "/opt/cni/bin";
# "bin_dir" = "${my-cni-plugins}/bin";
"conf_dir" = "/etc/cni/net.d";
};
"containerd" = {
"default_runtime_name" = "runc";
"runtimes" = {
"runc" = {
"options" = {
"SystemdCgroup" = true;
};
"runtime_type" = "io.containerd.runc.v2";
};
};
"snapshotter" = "overlayfs";
};
"snapshotter" = "overlayfs";
};
};
"version" = 2;
};
"version" = 2;
};
};
}

1
nix/kubernetes/roles/control_plane/default.nix
View File

@ -18,6 +18,7 @@
};
config = lib.mkIf config.me.control_plane.enable {
me.firewall.enable = true;
me.kube_apiserver.enable = true;
me.kube_controller_manager.enable = true;
me.kube_scheduler.enable = true;

36
nix/kubernetes/roles/firewall/default.nix Normal file
View File

@ -0,0 +1,36 @@
{
config,
lib,
...
}:
{
imports = [ ];
options.me = {
firewall.enable = lib.mkOption {
type = lib.types.bool;
default = false;
example = true;
description = "Whether we want to install firewall.";
};
};
config = lib.mkIf config.me.firewall.enable {
# kernel modules and settings required by Kubernetes
boot.kernelModules = [
"overlay"
"br_netfilter"
];
boot.kernel.sysctl = {
"net.bridge.bridge-nf-call-iptables" = 1;
"net.bridge.bridge-nf-call-ip6tables" = 1;
"net.ipv4.ip_forward" = 1;
};
networking.nftables.enable = true;
# We want to filter forwarded traffic.
# Also needed for `networking.firewall.extraForwardRules` to do anything.
networking.firewall.filterForward = true;
};
}

68
nix/kubernetes/roles/kube_proxy/default.nix Normal file
View File

@ -0,0 +1,68 @@
{
config,
lib,
pkgs,
...
}:
let
# shellCommand = cmd: (lib.concatMapStringsSep " " lib.strings.escapeShellArg cmd);
shellCommand = cmd: (builtins.concatStringsSep " " cmd);
settingsFormat = pkgs.formats.yaml { };
config_file = settingsFormat.generate "kube-proxy-config.yaml" config.me.kube-proxy.settings;
in
{
imports = [ ];
options.me = {
kube-proxy.enable = lib.mkOption {
type = lib.types.bool;
default = false;
example = true;
description = "Whether we want to install kube-proxy.";
};
kube-proxy.settings = lib.mkOption {
type = settingsFormat.type;
default = {
kind = "KubeProxyConfiguration";
apiVersion = "kubeproxy.config.k8s.io/v1alpha1";
clientConnection = {
kubeconfig = "/.persist/keys/kube/kube-proxy.kubeconfig";
};
mode = "iptables";
# clusterCIDR = "10.200.0.0/16";
# clusterCIDR = "2620:11f:7001:7:ffff:ffff:0ac8:0000/16";
clusterCIDR = "fd49:0595:2bba::/48";
};
description = ''
kubelet-config.yaml
'';
};
};
config = lib.mkIf config.me.kube-proxy.enable {
systemd.services.kube-proxy = {
enable = true;
description = "Kubernetes Kube Proxy";
documentation = [ "https://github.com/kubernetes/kubernetes" ];
wantedBy = [ "kubernetes.target" ];
path = with pkgs; [
iptables
];
unitConfig.DefaultDependencies = "no";
serviceConfig = {
ExecStart = (
shellCommand [
"${pkgs.kubernetes}/bin/kube-proxy"
"--config=${config_file}"
"--nodeport-addresses=primary"
"--proxy-mode=nftables"
]
);
Restart = "on-failure";
RestartSec = 5;
};
};
};
}

45
nix/kubernetes/roles/kubelet/default.nix
View File

@ -8,8 +8,6 @@
let
# shellCommand = cmd: (lib.concatMapStringsSep " " lib.strings.escapeShellArg cmd);
shellCommand = cmd: (builtins.concatStringsSep " " cmd);
settingsFormat = pkgs.formats.yaml { };
config_file = settingsFormat.generate "kubelet-config.yaml" config.me.kubelet.settings;
in
{
imports = [ ];
@ -21,46 +19,6 @@ in
example = true;
description = "Whether we want to install kubelet.";
};
kubelet.settings = lib.mkOption {
type = settingsFormat.type;
default = {
kind = "KubeletConfiguration";
apiVersion = "kubelet.config.k8s.io/v1beta1";
address = "0.0.0.0";
authentication = {
anonymous = {
enabled = false;
};
webhook = {
enabled = true;
};
x509 = {
clientCAFile = "/var/lib/kubelet/ca.crt";
};
};
authorization = {
mode = "Webhook";
};
cgroupDriver = "systemd";
containerRuntimeEndpoint = "unix:///var/run/containerd/containerd.sock";
enableServer = true;
failSwapOn = false;
maxPods = 16;
memorySwap = {
swapBehavior = "NoSwap";
};
port = 10250;
resolvConf = "/etc/resolv.conf";
registerNode = true;
runtimeRequestTimeout = "15m";
tlsCertFile = "/var/lib/kubelet/kubelet.crt";
tlsPrivateKeyFile = "/var/lib/kubelet/kubelet.key";
};
description = ''
kubelet-config.yaml
'';
};
};
config = lib.mkIf config.me.kubelet.enable {
@ -79,9 +37,8 @@ in
ExecStart = (
shellCommand [
"${pkgs.kubernetes}/bin/kubelet"
# "--config=${config_file}"
"--config=${./files/kubelet-config.yaml}"
"--kubeconfig=/.persist/keys/kube/kubeconfig"
"--kubeconfig=/.persist/keys/kube/kubelet.kubeconfig"
"--v=2"
]
);

8
nix/kubernetes/roles/kubernetes/default.nix
View File

@ -18,6 +18,14 @@
};
config = lib.mkIf config.me.kubernetes.enable {
assertions = [
{
# Kubernetes should only upgrade 1 minor version at a time, so this assert is here to prevent unwittingly jumping versions.
assertion = lib.hasPrefix "1.35." pkgs.kubernetes.version;
message = "Unexpected Kubernetes package version: ${pkgs.kubernetes.version}";
}
];
environment.systemPackages = with pkgs; [
kubernetes
];

2
nix/kubernetes/roles/worker_node/default.nix
View File

@ -19,6 +19,8 @@
config = lib.mkIf config.me.worker_node.enable {
me.containerd.enable = true;
me.firewall.enable = true;
me.kube-proxy.enable = true;
me.kubelet.enable = true;
me.kubernetes.enable = true;
};