machine_setup/nix/configuration/roles/distributed_build/default.nix

{
  config,
  lib,
  all_nixos_configs,
  pkgs,
  ...
}:

# TODO: FreeBSD is x86_64-freebsd

let
  make_machine_config = name: {
    enable = lib.mkOption {
      type = lib.types.bool;
      default = false;
      example = true;
      description = "Whether we want to use the ${name} machine during distributed builds.";
    };

    additional_config = lib.mkOption {
      type = lib.types.attrs;
      default = { };
      example = lib.literalExpression {
        speedFactor = 2;
      };
      description = "Additional config values for the buildMachines entry. For example, speedFactor.";
    };

    substituter_url = lib.mkOption {
      type = lib.types.nullOr lib.types.str;
      default = null;
      example = "ssh-ng://remote-host";
      description = "URL to use as a substituter.";
    };
  };

  static_host_configs = {
    quark = {
      # From: base64 -w0 /persist/ssh/ssh_host_ed25519_key.pub
      publicHostKey = "c3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSUx0alplYlVYTkRkU3Y1enVGbjM3eFNMZUN3S2hPKzFMdWovM2FYNFJRTEEgcm9vdEBxdWFyawo=";
      systems = [
        "i686-linux"
        "x86_64-linux"
        # "aarch64-linux"
      ];
    };
    hydra = {
      # Does not work, so we have to use root's authorized keys. Not sure why. My best guess is it is related to overriding the ssh target via the ssh config.
      #
      # From: base64 -w0 /persist/ssh/ssh_host_ed25519_key.pub
      # publicHostKey = "c3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSUNJRk9tU0NWV25xVVFFL2RKd2R0STdRQ29LTHhBNHRmWnRSYStFSG9XV0wgcm9vdEBoeWRyYQo=";
      # publicHostKey = "c3NoLXJzYSBBQUFBQjNOemFDMXljMkVBQUFBREFRQUJBQUFDQVFDNk1nNytKMys1d2c5QkJmLzkweWJhbnhJK3ZiSXRzY1ZoTzRRVFNMT1FJM1QxbWMrNUxTK2oxdDY4a20zVmx1c2k5WWh0UmNOVk1NbmZQT3Q3ejFyZ0Q4Y05iY3UzU0xCOTNGR2hvb1ZtUktyVEMyaVE4bkxZYVoyV1lrdnZ3UjJ6b24zTjRlTlFJdWUwR0EzTEtNKy82RDN5V3didjRYV2pFeWtpa1NoVFJUVUtMTjU4aUVVOVRmcnEzeE1Ib0EvYmJZMGIxK1QzUDRkQ05wb3BFcFczaVJaYkV0NUpvNS92VFBsVjc4L3I3bnV2eHlaVjU3dWhzYjhFQlFzYUdCYTIraEt6SUhFblBHWXlPUDVRRWVCZ1ZBUzh6bFg4aktaZnZuaDA0NmdFdFoyMWJWdHNBZHNXcnVYdXNEUVQyanJ2VjJOVTYvc2cyMzZPbFZvWVU4Z1JYRHZiTXhoclVSVWNsajM1RGlzTi9wMEd2clZOckVjSktZK245MS9UMG9qU0gvVTVlRzVPclhZRXQwVzIra0NsVDk0T2kxM2JjQkVDYmVUUXJKMjJRc1hKSnVEVkVzOWhsU2RLemZkMDVaaVc3MllHaHRCRWptL015UG1nSHhxYzNTKzEwc0VvT2FIazdtRjQ0M0o1MzZoWEVHZDhGWjROMnQ3MlF3V3RuUGoyNENYOG8wbmNJN2ZIYTRHTWsybi9EWU84MlNUTEJHeVBMTkxnK040NUcyYUhaSk1NWGprWlplMUVZS0dQQm5tcTFlVUdFMVd0ZkVRSEhCTHd3Y0dDdm15aWI1NTliQ0p5NWExS2pnSGNBYlk0WVRKOTlwMWtPT2lIcVlvTnArczlhSklPZU93aE5xbkkrOUxrWnYrbEhCdXRoM3A1anRRNzA3bEJMMmVBd1E9PSByb290QGh5ZHJhCg==";
      systems = [
        "i686-linux"
        "x86_64-linux"
        # "aarch64-linux"
      ];

      hostName = lib.mkForce "hydra?remote-store=local?root=/home/nixworker/persist/root";
    };
  };
  joined_configs =
    lib.genAttrs
      (builtins.filter (hostname: config.me.distributed_build.machines."${hostname}".enable) (
        builtins.attrNames all_nixos_configs
      ))
      (
        hostname:
        (lib.mkMerge [
          {
            hostName = hostname;
            sshUser = "nixworker";
            sshKey = "/persist/manual/ssh/root/keys/id_ed25519";
            maxJobs = 1;
            supportedFeatures = all_nixos_configs."${hostname}".config.me.optimizations.system_features;
          }
          static_host_configs."${hostname}"
          config.me.distributed_build.machines."${hostname}".additional_config
        ])
      );
in
{
  imports = [ ];

  options.me = {
    distributed_build.enable = lib.mkOption {
      type = lib.types.bool;
      default = false;
      example = true;
      description = "Whether we want to use multiple machines to perform a nixos-rebuild.";
    };

    distributed_build.machines = lib.mapAttrs (name: value: make_machine_config name) all_nixos_configs;
  };

  config = lib.mkIf config.me.distributed_build.enable (
    lib.mkMerge [
      {
        nix.distributedBuilds = true;

        # Using an ssh-based substituter slows down the build because querying the remote store for paths takes ages.
        #
        # nix.settings.substituters = lib.mkForce [
        #   "ssh-ng://nixworker@ns1.fizz.buzz:65122?compress=true&ssh-key=/persist/manual/ssh/root/keys/id_ed25519&remote-store=/home/nixworker/persist/root"
        # ];
        # nix.settings.substitute = lib.mkForce true;

        # nix.settings.post-build-hook = pkgs.writeShellScript "post-build-hook" ''
        #   set -euo pipefail
        #   IFS=$'\n\t'
        #   set -f # disable globbing
        #   echo "Signing and uploading paths" $OUT_PATHS
        #   exec nix copy --to 'ssh://hydra' $OUT_PATHS
        # '';
        nix.settings.secret-key-files = [ "/persist/manual/nix/nix-cache-key.sec" ];
        nix.settings.trusted-public-keys = lib.mkForce [
          "odo:0S/XKSFjjIrihQ7lbHEIebXk/c/xuoodhm0Gz26YhjA="
          "odowork:zg3UKBAyLy3xtZkL0hMtbxHjxgn5A2QY8NNAgyRT6Yo="
          "quark:Eb6ygkIiVlcUqb5hOjEVIQcfYLpCz40YVYA3/rxrgBc="
        ];
      }
      {
        nix.buildMachines = (
          map (
            hostname:
            (lib.mkIf config.me.distributed_build.machines."${hostname}".enable (
              lib.mkMerge [
                {
                  hostName = hostname;
                  sshUser = "nixworker";
                  sshKey = "/persist/manual/ssh/root/keys/id_ed25519";
                  maxJobs = 1;
                  supportedFeatures = all_nixos_configs."${hostname}".config.me.optimizations.system_features;
                }
                static_host_configs."${hostname}"
                config.me.distributed_build.machines."${hostname}".additional_config
              ]
            ))
          ) (builtins.attrNames all_nixos_configs)
        );
      }
      # {
      #   nix.settings.substitute = lib.mkForce true;
      #   nix.settings.substituters = lib.mkForce (
      #     lib.mapAttrsToList (hostname: joined_config: "ssh-ng://${joined_config.hostName}") joined_configs
      #   );
      # }
    ]
  );
}