diff --git a/nix/kubernetes/configuration.nix b/nix/kubernetes/configuration.nix index 2be35057..16fcf75c 100644 --- a/nix/kubernetes/configuration.nix +++ b/nix/kubernetes/configuration.nix @@ -7,15 +7,18 @@ { imports = [ ./roles/boot + ./roles/control_plane ./roles/doas + ./roles/dont_use_substituters ./roles/etcd ./roles/image_based_appliance ./roles/iso - ./roles/optimized_build - ./roles/dont_use_substituters + ./roles/kube_apiserver + ./roles/kubernetes ./roles/minimal_base ./roles/network ./roles/nvme + ./roles/optimized_build ./roles/ssh ./roles/sshd ./roles/user diff --git a/nix/kubernetes/flake.nix b/nix/kubernetes/flake.nix index 74eda61f..30e8c399 100644 --- a/nix/kubernetes/flake.nix +++ b/nix/kubernetes/flake.nix @@ -95,10 +95,9 @@ ... }: let - repl_path = toString ./.; nix-self-repl = pkgs.writeShellScriptBin "nix-self-repl" '' source /etc/set-environment - nix repl "${repl_path}/repl.nix" "$@" + nix repl --expr 'builtins.getFlake "${self}"' ''; # If we wanted the current version of a flake then we'd just launch # nix repl @@ -130,10 +129,9 @@ ... }: let - repl_path = toString ./.; nix-self-repl = pkgs.writeShellScriptBin "nix-self-repl" '' source /etc/set-environment - nix repl "${repl_path}/repl.nix" "$@" + nix repl --expr 'builtins.getFlake "${self}"' ''; # If we wanted the current version of a flake then we'd just launch # nix repl diff --git a/nix/kubernetes/hosts/controller0/default.nix b/nix/kubernetes/hosts/controller0/default.nix index 5339141f..ef6a64fe 100644 --- a/nix/kubernetes/hosts/controller0/default.nix +++ b/nix/kubernetes/hosts/controller0/default.nix @@ -116,6 +116,14 @@ "controller2=https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2380" # 10.215.1.223 ]; + me.kube_apiserver.internal_ip = "2620:11f:7001:7:ffff:ffff:0ad7:01dd"; + me.kube_apiserver.etcd_services = [ + "https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2379" # 10.215.1.221 + "https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2379" # 10.215.1.222 + "https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2379" # 10.215.1.223 + ]; + + me.control_plane.enable = true; me.dont_use_substituters.enable = true; me.etcd.enable = true; me.minimal_base.enable = true; diff --git a/nix/kubernetes/hosts/controller1/default.nix b/nix/kubernetes/hosts/controller1/default.nix index 5f6d50b2..61f26a5e 100644 --- a/nix/kubernetes/hosts/controller1/default.nix +++ b/nix/kubernetes/hosts/controller1/default.nix @@ -116,6 +116,14 @@ "controller2=https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2380" # 10.215.1.223 ]; + me.kube_apiserver.internal_ip = "2620:11f:7001:7:ffff:ffff:0ad7:01de"; + me.kube_apiserver.etcd_services = [ + "https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2379" # 10.215.1.221 + "https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2379" # 10.215.1.222 + "https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2379" # 10.215.1.223 + ]; + + me.control_plane.enable = true; me.dont_use_substituters.enable = true; me.etcd.enable = true; me.minimal_base.enable = true; diff --git a/nix/kubernetes/hosts/controller2/default.nix b/nix/kubernetes/hosts/controller2/default.nix index d86dbbe7..8336a011 100644 --- a/nix/kubernetes/hosts/controller2/default.nix +++ b/nix/kubernetes/hosts/controller2/default.nix @@ -116,6 +116,14 @@ "controller2=https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2380" # 10.215.1.223 ]; + me.kube_apiserver.internal_ip = "2620:11f:7001:7:ffff:ffff:0ad7:01df"; + me.kube_apiserver.etcd_services = [ + "https://[2620:11f:7001:7:ffff:ffff:0ad7:01dd]:2379" # 10.215.1.221 + "https://[2620:11f:7001:7:ffff:ffff:0ad7:01de]:2379" # 10.215.1.222 + "https://[2620:11f:7001:7:ffff:ffff:0ad7:01df]:2379" # 10.215.1.223 + ]; + + me.control_plane.enable = true; me.dont_use_substituters.enable = true; me.etcd.enable = true; me.minimal_base.enable = true; diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index f86abcf9..402c7338 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -4,6 +4,8 @@ callPackage, writeShellScript, openssh, + runCommand, + writeText, lib, }: let @@ -50,25 +52,91 @@ makeScope newScope ( inherit all_hostnames; k8s = self; }; - deploy_key = ( - vm_name: file: '' - ${openssh}/bin/ssh mrmanager rm -f /vm/${vm_name}/persist/keys/etcd/${builtins.baseNameOf file} ~/${builtins.baseNameOf file} - ${openssh}/bin/scp ${file} mrmanager:~/${builtins.baseNameOf file} - ${openssh}/bin/ssh mrmanager doas install -o 10016 -g 10016 -m 0640 ~/${builtins.baseNameOf file} /vm/${vm_name}/persist/keys/etcd/${builtins.baseNameOf file} - ${openssh}/bin/ssh mrmanager rm -f ~/${builtins.baseNameOf file} + deploy_file = ( + { + dest_dir, + file, + name ? (builtins.baseNameOf file), + owner, + group, + mode, + }: + '' + ## + ## deploy ${name} to ${dest_dir} + ## + ${openssh}/bin/ssh mrmanager doas rm -f ${dest_dir}/${name} ~/${name} + ${openssh}/bin/scp ${file} mrmanager:~/${name} + ${openssh}/bin/ssh mrmanager doas install -o ${toString owner} -g ${toString group} -m ${mode} ~/${name} ${dest_dir}/${name} + ${openssh}/bin/ssh mrmanager doas rm -f ~/${name} + + '' ); deploy_machine = ( vm_name: ( '' + ## + ## Create directories on ${vm_name} + ## ${openssh}/bin/ssh mrmanager doas install -d -o 11235 -g 11235 -m 0755 /vm/${vm_name}/persist/keys ${openssh}/bin/ssh mrmanager doas install -d -o 10016 -g 10016 -m 0755 /vm/${vm_name}/persist/keys/etcd + ${openssh}/bin/ssh mrmanager doas install -d -o 10024 -g 10024 -m 0755 /vm/${vm_name}/persist/keys/kube + + '' - + (lib.concatMapStringsSep "\n" (deploy_key vm_name) [ - "${self.kubernetes}/kubernetes.pem" - "${self.kubernetes}/kubernetes-key.pem" - "${self.ca}/ca.pem" + + (lib.concatMapStringsSep "\n" deploy_file [ + { + dest_dir = "/vm/${vm_name}/persist/keys/etcd"; + file = "${self.kubernetes}/kubernetes.pem"; + owner = 10016; + group = 10016; + mode = "0640"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/etcd"; + file = "${self.kubernetes}/kubernetes-key.pem"; + owner = 10016; + group = 10016; + mode = "0600"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/etcd"; + file = "${self.ca}/ca.pem"; + owner = 10016; + group = 10016; + mode = "0640"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${self.kubernetes}/kubernetes.pem"; + owner = 10024; + group = 10024; + mode = "0640"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${self.kubernetes}/kubernetes-key.pem"; + owner = 10024; + group = 10024; + mode = "0640"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = "${self.ca}/ca.pem"; + owner = 10024; + group = 10024; + mode = "0600"; + } + { + dest_dir = "/vm/${vm_name}/persist/keys/kube"; + file = (writeText "encryption-config.yaml" (lib.generators.toYAML { } kube_encryption_config)); + name = "encryption-config.yaml"; + owner = 10024; + group = 10024; + mode = "0600"; + } ]) ) ); @@ -84,6 +152,31 @@ makeScope newScope ( "nc2" ]) ); + kube_encryption_key = runCommand "kube_encryption_key" { } '' + head -c 32 /dev/urandom | base64 | tee $out + ''; + kube_encryption_config = { + kind = "EncryptionConfig"; + apiVersion = "v1"; + resources = [ + { + resources = [ "secrets" ]; + providers = [ + { + aescbc = { + keys = [ + { + name = "key1"; + secret = (builtins.readFile "${kube_encryption_key}"); + } + ]; + }; + } + { identity = { }; } + ]; + } + ]; + }; in { ca = (callPackage ./package/k8s-ca/package.nix additional_vars); diff --git a/nix/kubernetes/repl.nix b/nix/kubernetes/repl.nix deleted file mode 100644 index e21db83b..00000000 --- a/nix/kubernetes/repl.nix +++ /dev/null @@ -1,5 +0,0 @@ -let - flake = builtins.getFlake (toString ./.); - nixpkgs = import { }; -in -{ inherit flake; } // flake // builtins // nixpkgs // nixpkgs.lib // flake.nixosConfigurations diff --git a/nix/kubernetes/roles/control_plane/default.nix b/nix/kubernetes/roles/control_plane/default.nix new file mode 100644 index 00000000..1c0b58b9 --- /dev/null +++ b/nix/kubernetes/roles/control_plane/default.nix @@ -0,0 +1,24 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + options.me = { + control_plane.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install control_plane."; + }; + }; + + config = lib.mkIf config.me.control_plane.enable { + me.kubernetes.enable = true; + me.kube_apiserver.enable = true; + }; +} diff --git a/nix/kubernetes/roles/kube_apiserver/default.nix b/nix/kubernetes/roles/kube_apiserver/default.nix new file mode 100644 index 00000000..4c67ff16 --- /dev/null +++ b/nix/kubernetes/roles/kube_apiserver/default.nix @@ -0,0 +1,98 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + shellCommand = cmd: (lib.concatMapStringsSep " " lib.strings.escapeShellArg cmd); +in +{ + imports = [ ]; + + options.me = { + kube_apiserver.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install kube_apiserver."; + }; + + kube_apiserver.internal_ip = lib.mkOption { + # default = { }; + example = "192.168.1.10"; + type = lib.types.str; + description = "IP address this server should advertise."; + }; + + kube_apiserver.etcd_services = lib.mkOption { + default = [ ]; + example = [ "https://192.168.1.10:2379" ]; + type = lib.types.listOf lib.types.str; + description = "Endpoints for etcd servers."; + }; + }; + + config = lib.mkIf config.me.kube_apiserver.enable { + systemd.services.kube-apiserver = { + enable = true; + description = "Kubernetes API Server"; + documentation = [ "https://github.com/kubernetes/kubernetes" ]; + wantedBy = [ "kubernetes.target" ]; + # path = with pkgs; [ + # zfs + # ]; + unitConfig.DefaultDependencies = "no"; + serviceConfig = { + Type = "notify"; + ExecStart = ( + shellCommand [ + "${pkgs.kubernetes}/bin/kube-apiserver" + "--advertise-address=${config.me.kube_apiserver.internal_ip}" + "--allow-privileged=true" + "--apiserver-count=3" + "--audit-log-maxage=30" + "--audit-log-maxbackup=3" + "--audit-log-maxsize=100" + "--audit-log-path=/var/log/audit.log" + "--authorization-mode=Node,RBAC" + "--bind-address=0.0.0.0" + "--client-ca-file=/.persist/keys/kube/ca.pem" + "--requestheader-client-ca-file=/var/lib/kubernetes/requestheader-client-ca.pem" + ''--requestheader-allowed-names=""'' + "--requestheader-extra-headers-prefix=X-Remote-Extra-" + "--requestheader-group-headers=X-Remote-Group" + "--requestheader-username-headers=X-Remote-User" + "--proxy-client-cert-file=/var/lib/kubernetes/{{ node_name }}-proxy.pem" + "--proxy-client-key-file=/var/lib/kubernetes/{{ node_name }}-proxy-key.pem" + "--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota" + "--etcd-cafile=/.persist/keys/kube/ca.pem" + "--etcd-certfile=/.persist/keys/kube/kubernetes.pem" + "--etcd-keyfile=/.persist/keys/kube/kubernetes-key.pem" + "--etcd-servers=${builtins.concatStringsSep "," config.me.kube_apiserver.etcd_services}" + "--event-ttl=1h" + "--encryption-provider-config=/.persist/keys/kube/encryption-config.yaml" + "--kubelet-certificate-authority=/.persist/keys/kube/ca.pem" + "--kubelet-client-certificate=/.persist/keys/kube/kubernetes.pem" + "--kubelet-client-key=/.persist/keys/kube/kubernetes-key.pem" + "--runtime-config='api/all=true'" + "--service-account-key-file=/var/lib/kubernetes/service-account.pem" + "--service-account-signing-key-file=/var/lib/kubernetes/service-account-key.pem" + "--service-account-issuer=https://{{ kubernetes_public_address }}:6443" + "--service-node-port-range=30000-32767" + "--tls-cert-file=/.persist/keys/kube/kubernetes.pem" + "--tls-private-key-file=/.persist/keys/kube/kubernetes-key.pem" + "--tls-min-version=VersionTLS13" + "--kubelet-preferred-address-types=InternalIP,ExternalDNS,ExternalIP,Hostname,InternalDNS" + "--service-cluster-ip-range=10.197.0.0/16" + "--enable-aggregator-routing=true" + "--v=2" + ] + ); + Restart = "on-failure"; + RestartSec = 5; + }; + }; + }; +} diff --git a/nix/kubernetes/roles/kubernetes/default.nix b/nix/kubernetes/roles/kubernetes/default.nix new file mode 100644 index 00000000..9d3646d9 --- /dev/null +++ b/nix/kubernetes/roles/kubernetes/default.nix @@ -0,0 +1,53 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ ]; + + options.me = { + kubernetes.enable = lib.mkOption { + type = lib.types.bool; + default = false; + example = true; + description = "Whether we want to install kubernetes."; + }; + }; + + config = lib.mkIf config.me.kubernetes.enable { + environment.systemPackages = with pkgs; [ + kubernetes + ]; + + systemd.targets.kubernetes = { + description = "Kubernetes"; + wantedBy = [ "multi-user.target" ]; + }; + + users.users.kubernetes = { + uid = 10024; + description = "Kubernetes"; + group = "kubernetes"; + home = "/var/lib/kubernetes"; + createHome = true; + homeMode = "755"; + isSystemUser = true; + }; + users.groups.kubernetes.gid = 10024; + + environment.persistence."/persist" = lib.mkIf (config.me.mountPersistence) { + hideMounts = true; + directories = [ + { + directory = "/var/lib/kubernetes"; + user = "kubernetes"; + group = "kubernetes"; + mode = "0755"; + } + ]; + }; + }; +}