machine_setup/nix/kubernetes/keys/scope.nix

{
  makeScope,
  newScope,
  callPackage,
  writeShellScript,
  openssh,
  runCommand,
  writeText,
  lib,
}:
let
  public_addresses = [
    "74.80.180.138"
  ];
  internal_addresses = [
    # nc0
    "10.215.1.221"
    "2620:11f:7001:7:ffff:ffff:0ad7:01dd"
    # nc1
    "10.215.1.222"
    "2620:11f:7001:7:ffff:ffff:0ad7:01de"
    # nc2
    "10.215.1.223"
    "2620:11f:7001:7:ffff:ffff:0ad7:01df"
    # nw0
    "10.215.1.224"
    "2620:11f:7001:7:ffff:ffff:0ad7:01e0"
    # nw1
    "10.215.1.225"
    "2620:11f:7001:7:ffff:ffff:0ad7:01e1"
    # nw2
    "10.215.1.226"
    "2620:11f:7001:7:ffff:ffff:0ad7:01e2"
  ];
  all_hostnames = [
    "10.197.0.1"
    "10.0.0.1"
    "127.0.0.1"
    "kubernetes"
    "kubernetes.default"
    "kubernetes.default.svc"
    "kubernetes.default.svc.cluster"
    "kubernetes.svc.cluster.local"
  ]
  ++ public_addresses
  ++ internal_addresses;
  controllers = {
    "controller0" = {
      "internal_ips" = [
        "10.215.1.221"
        "2620:11f:7001:7:ffff:ffff:0ad7:01dd"
      ];
      "external_ips" = [
        "2620:11f:7001:7:ffff:ffff:0ad7:01dd"
      ];
    };
    "controller1" = {
      "internal_ips" = [
        "10.215.1.222"
        "2620:11f:7001:7:ffff:ffff:0ad7:01de"
      ];
      "external_ips" = [
        "2620:11f:7001:7:ffff:ffff:0ad7:01de"
      ];
    };
    "controller2" = {
      "internal_ips" = [
        "10.215.1.223"
        "2620:11f:7001:7:ffff:ffff:0ad7:01df"
      ];
      "external_ips" = [
        "2620:11f:7001:7:ffff:ffff:0ad7:01df"
      ];
    };
  };
  _vm_name_to_hostname = {
    "nc0" = "controller0";
    "nc1" = "controller1";
    "nc2" = "controller2";
  };
  vm_name_to_hostname = (vm_name: _vm_name_to_hostname."${vm_name}");
in
makeScope newScope (
  self:
  let
    additional_vars = {
      inherit all_hostnames controllers;
      k8s = self;
    };
    deploy_file = (
      {
        dest_dir,
        file,
        name ? (builtins.baseNameOf file),
        owner,
        group,
        mode,
      }:
      ''
        ##
        ## deploy ${name} to ${dest_dir}
        ##
        ${openssh}/bin/ssh mrmanager doas rm -f ${dest_dir}/${name} ~/${name}
        ${openssh}/bin/scp ${file} mrmanager:~/${name}
        ${openssh}/bin/ssh mrmanager doas install -o ${toString owner} -g ${toString group} -m ${mode} ~/${name} ${dest_dir}/${name}
        ${openssh}/bin/ssh mrmanager doas rm -f ~/${name}


      ''
    );
    deploy_machine = (
      vm_name:
      (
        ''
          ##
          ## Create directories on ${vm_name}
          ##
          ${openssh}/bin/ssh mrmanager doas install -d -o 11235 -g 11235 -m 0755 /vm/${vm_name}/persist/keys
          ${openssh}/bin/ssh mrmanager doas install -d -o 10016 -g 10016 -m 0755 /vm/${vm_name}/persist/keys/etcd
          ${openssh}/bin/ssh mrmanager doas install -d -o 10024 -g 10024 -m 0755 /vm/${vm_name}/persist/keys/kube


        ''
        + (lib.concatMapStringsSep "\n" deploy_file [
          {
            dest_dir = "/vm/${vm_name}/persist/keys/etcd";
            file = "${self.kubernetes}/kubernetes.pem";
            owner = 10016;
            group = 10016;
            mode = "0640";
          }
          {
            dest_dir = "/vm/${vm_name}/persist/keys/etcd";
            file = "${self.kubernetes}/kubernetes-key.pem";
            owner = 10016;
            group = 10016;
            mode = "0600";
          }
          {
            dest_dir = "/vm/${vm_name}/persist/keys/etcd";
            file = "${self.ca}/ca.pem";
            owner = 10016;
            group = 10016;
            mode = "0640";
          }
          {
            dest_dir = "/vm/${vm_name}/persist/keys/kube";
            file = "${self.kubernetes}/kubernetes.pem";
            owner = 10024;
            group = 10024;
            mode = "0640";
          }
          {
            dest_dir = "/vm/${vm_name}/persist/keys/kube";
            file = "${self.kubernetes}/kubernetes-key.pem";
            owner = 10024;
            group = 10024;
            mode = "0640";
          }
          {
            dest_dir = "/vm/${vm_name}/persist/keys/kube";
            file = "${self.ca}/ca.pem";
            owner = 10024;
            group = 10024;
            mode = "0600";
          }
          {
            dest_dir = "/vm/${vm_name}/persist/keys/kube";
            file = (writeText "encryption-config.yaml" (lib.generators.toYAML { } kube_encryption_config));
            name = "encryption-config.yaml";
            owner = 10024;
            group = 10024;
            mode = "0600";
          }
          {
            dest_dir = "/vm/${vm_name}/persist/keys/kube";
            file = "${self.service_account}/service-account.pem";
            owner = 10024;
            group = 10024;
            mode = "0600";
          }
          {
            dest_dir = "/vm/${vm_name}/persist/keys/kube";
            file = "${self.service_account}/service-account-key.pem";
            owner = 10024;
            group = 10024;
            mode = "0600";
          }
          {
            dest_dir = "/vm/${vm_name}/persist/keys/kube";
            file = "${self.requestheader-client-ca}/requestheader-client-ca.pem";
            owner = 10024;
            group = 10024;
            mode = "0600";
          }
          {
            dest_dir = "/vm/${vm_name}/persist/keys/kube";
            file = "${self.controller-proxy}/${vm_name_to_hostname vm_name}-proxy.pem";
            owner = 10024;
            group = 10024;
            mode = "0600";
          }
          {
            dest_dir = "/vm/${vm_name}/persist/keys/kube";
            file = "${self.controller-proxy}/${vm_name_to_hostname vm_name}-proxy-key.pem";
            owner = 10024;
            group = 10024;
            mode = "0600";
          }
        ])
      )
    );
    deploy_script = (
      ''
        set -euo pipefail
        IFS=$'\n\t'
        DIR="$( cd "$( dirname "''${BASH_SOURCE[0]}" )" && pwd )"
      ''
      + (lib.concatMapStringsSep "\n" deploy_machine [
        "nc0"
        "nc1"
        "nc2"
      ])
    );
    kube_encryption_key = runCommand "kube_encryption_key" { } ''
      head -c 32 /dev/urandom | base64 | tee $out
    '';
    kube_encryption_config = {
      kind = "EncryptionConfig";
      apiVersion = "v1";
      resources = [
        {
          resources = [ "secrets" ];
          providers = [
            {
              aescbc = {
                keys = [
                  {
                    name = "key1";
                    secret = (builtins.readFile "${kube_encryption_key}");
                  }
                ];
              };
            }
            { identity = { }; }
          ];
        }
      ];
    };
  in
  {
    ca = (callPackage ./package/k8s-ca/package.nix additional_vars);
    kubernetes = (callPackage ./package/k8s-kubernetes/package.nix additional_vars);
    service_account = (callPackage ./package/k8s-service-account/package.nix additional_vars);
    requestheader-client-ca = (
      callPackage ./package/k8s-requestheader-client-ca/package.nix additional_vars
    );
    controller-proxy = (callPackage ./package/k8s-controller-proxy/package.nix additional_vars);
    keys = (callPackage ./package/k8s-keys/package.nix additional_vars);
    deploy_script = (writeShellScript "deploy-keys" deploy_script);
  }
)