machine_setup/nix/kubernetes/roles/kube_apiserver/default.nix

{
  config,
  lib,
  pkgs,
  ...
}:

let
  # shellCommand = cmd: (lib.concatMapStringsSep " " lib.strings.escapeShellArg cmd);
  shellCommand = cmd: (builtins.concatStringsSep " " cmd);
in
{
  imports = [ ];

  options.me = {
    kube_apiserver.enable = lib.mkOption {
      type = lib.types.bool;
      default = false;
      example = true;
      description = "Whether we want to install kube_apiserver.";
    };

    kube_apiserver.internal_ip = lib.mkOption {
      # default = { };
      example = "192.168.1.10";
      type = lib.types.str;
      description = "IP address this server should advertise.";
    };

    kube_apiserver.external_ip = lib.mkOption {
      example = "192.168.1.10";
      type = lib.types.str;
      description = "IP address to reach this cluster externally.";
    };

    kube_apiserver.etcd_services = lib.mkOption {
      default = [ ];
      example = [ "https://192.168.1.10:2379" ];
      type = lib.types.listOf lib.types.str;
      description = "Endpoints for etcd servers.";
    };
  };

  config = lib.mkIf config.me.kube_apiserver.enable {
    systemd.services.kube-apiserver = {
      enable = true;
      description = "Kubernetes API Server";
      documentation = [ "https://github.com/kubernetes/kubernetes" ];
      wantedBy = [ "kubernetes.target" ];
      # path = with pkgs; [
      #   zfs
      # ];
      unitConfig.DefaultDependencies = "no";
      serviceConfig = {
        Type = "notify";
        ExecStart = (
          shellCommand [
            # NEW:
            "${pkgs.kubernetes}/bin/kube-apiserver"
            "--advertise-address=${config.me.kube_apiserver.external_ip}"
            "--allow-privileged=true"
            "--audit-log-maxage=30"
            "--audit-log-maxbackup=3"
            "--audit-log-maxsize=100"
            "--audit-log-path=/var/log/audit.log"
            "--authorization-mode=Node,RBAC"
            "--bind-address=0.0.0.0"
            "--client-ca-file=/.persist/keys/kube/ca.crt"
            "--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota"
            "--etcd-cafile=/.persist/keys/kube/ca.crt"
            "--etcd-certfile=/.persist/keys/kube/kube-api-server.crt"
            "--etcd-keyfile=/.persist/keys/kube/kube-api-server.key"
            "--etcd-servers=${builtins.concatStringsSep "," config.me.kube_apiserver.etcd_services}"
            "--event-ttl=1h"
            "--encryption-provider-config=/.persist/keys/kube/encryption-config.yaml"
            "--kubelet-certificate-authority=/.persist/keys/kube/ca.crt"
            "--kubelet-client-certificate=/.persist/keys/kube/kube-api-server.crt"
            "--kubelet-client-key=/.persist/keys/kube/kube-api-server.key"
            "--runtime-config='api/all=true'"
            "--service-account-key-file=/.persist/keys/kube/service-accounts.crt"
            "--service-account-signing-key-file=/.persist/keys/kube/service-accounts.key"
            "--service-account-issuer=https://server.kubernetes.local:6443"
            "--service-node-port-range=30000-32767"
            "--tls-cert-file=/.persist/keys/kube/kube-api-server.crt"
            "--tls-private-key-file=/.persist/keys/kube/kube-api-server.key"
            "--tls-min-version=VersionTLS13"
            "--service-cluster-ip-range=fd00:3e42:e349::/112"
            "--v=2"

            # OLD:
            # "${pkgs.kubernetes}/bin/kube-apiserver"
            # "--advertise-address=${config.me.kube_apiserver.internal_ip}"
            # "--allow-privileged=true"
            # "--apiserver-count=3"
            # "--audit-log-maxage=30"
            # "--audit-log-maxbackup=3"
            # "--audit-log-maxsize=100"
            # "--audit-log-path=/var/log/audit.log"
            # "--authorization-mode=Node,RBAC"
            # "--bind-address=0.0.0.0"
            # "--client-ca-file=/.persist/keys/kube/ca.pem"
            # "--requestheader-client-ca-file=/.persist/keys/kube/requestheader-client-ca.pem"
            # ''--requestheader-allowed-names=""''
            # "--requestheader-extra-headers-prefix=X-Remote-Extra-"
            # "--requestheader-group-headers=X-Remote-Group"
            # "--requestheader-username-headers=X-Remote-User"
            # "--proxy-client-cert-file=/.persist/keys/kube/${config.networking.hostName}-proxy.pem"
            # "--proxy-client-key-file=/.persist/keys/kube/${config.networking.hostName}-proxy-key.pem"
            # "--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota"
            # "--etcd-cafile=/.persist/keys/kube/ca.pem"
            # "--etcd-certfile=/.persist/keys/kube/kubernetes.pem"
            # "--etcd-keyfile=/.persist/keys/kube/kubernetes-key.pem"
            # "--etcd-servers=${builtins.concatStringsSep "," config.me.kube_apiserver.etcd_services}"
            # "--event-ttl=1h"
            # "--encryption-provider-config=/.persist/keys/kube/encryption-config.yaml"
            # "--kubelet-certificate-authority=/.persist/keys/kube/ca.pem"
            # "--kubelet-client-certificate=/.persist/keys/kube/kubernetes.pem"
            # "--kubelet-client-key=/.persist/keys/kube/kubernetes-key.pem"
            # "--runtime-config='api/all=true'"
            # "--service-account-key-file=/.persist/keys/kube/service-account.pem"
            # "--service-account-signing-key-file=/.persist/keys/kube/service-account-key.pem"
            # "--service-account-issuer=https://${config.me.kube_apiserver.external_ip}:6443"
            # "--service-node-port-range=30000-32767"
            # "--tls-cert-file=/.persist/keys/kube/kubernetes.pem"
            # "--tls-private-key-file=/.persist/keys/kube/kubernetes-key.pem"
            # "--tls-min-version=VersionTLS13"
            # "--kubelet-preferred-address-types=InternalIP,ExternalDNS,ExternalIP,Hostname,InternalDNS"
            # # "--service-cluster-ip-range=10.197.0.0/16"
            # "--service-cluster-ip-range=2620:11f:7001:7:ffff:ffff:0ac5:0000/16"
            # "--enable-aggregator-routing=true"
            # "--v=2"
          ]
        );
        Restart = "on-failure";
        RestartSec = 5;
        LimitNOFILE = 65536;
        User = "kubernetes";
      };
    };

    networking.firewall.allowedTCPPorts = [
      6443
    ];

    systemd.tmpfiles.rules = [
      "f /var/log/audit.log 0600 kubernetes kubernetes - -"
    ];
  };
}