Make public IP address quota based on actual use.

README.md

@@ -304,6 +304,8 @@ Question and Answer
 
 [GKE assigns a separate IP address to each `Ingress`](https://cloud.google.com/kubernetes-engine/docs/concepts/ingress#limitations), but we can have a single `Gateway` with an IP address and then any quantity of `HTTPRoute`. This is a design choice for GKE, and not a limitation of kubernetes.
 
+If you need to use `Ingress`, we can achieve the same efficiency for IP addresses by using the nginx ingress controller. This can be enabled by passing `-var ingress_type=nginx`. If you need to use the built-in ingress controller instead of nginx you can set `-var ingress_type=gce` but then each `Ingress` will cost 1 IP address.
+
 Clean Up
 ========
 Just like we did a 2-stage apply by toggling the `cluster_exists` variable, we will need to do a 2-stage destroy. First we tear down any kubernetes resources by running *apply* with the `cluster_exists` variable set to `false`. Then we can destroy the entire project.

terraform/external_dns.tf

@@ -43,7 +43,7 @@ resource "google_project_iam_member" "external_dns" {
   role    = "roles/dns.reader"
 }
 
-resource "google_dns_managed_zone_iam_member" "member" {
+resource "google_dns_managed_zone_iam_member" "external_dns" {
   project      = google_project.project.project_id
   managed_zone = google_dns_managed_zone.zone.name
   role         = "roles/dns.admin"

terraform/main.tf

@@ -1,3 +1,4 @@
+# TODO: put IP address ranges into variables
 terraform {
   backend "gcs" {
     bucket = "tf-state-4b00"
@@ -48,6 +49,11 @@ variable "ingress_type" {
   description = "What controller should we use to handle incoming http(s) connections."
   type        = string
   default     = "gateway"
+
+  validation {
+    condition     = contains(["gateway", "nginx", "gce"], var.ingress_type)
+    error_message = "Must be either \"gateway\", \"nginx\", or \"gce\"."
+  }
 }
 
 variable "cluster_exists" {

terraform/modules/cluster/main.tf

@@ -46,12 +46,6 @@ variable "public_ingress" {
 variable "ingress_type" {
   description = "What controller should we use to handle incoming http(s) connections."
   type        = string
-  default     = "gateway"
-
-  validation {
-    condition     = contains(["gateway"], var.ingress_type)
-    error_message = "Currently only \"gateway\" is supported."
-  }
 }
 
 variable "main_k8s_namespace" {

terraform/modules/k8s_workload/external_dns.tf

@@ -12,7 +12,8 @@ variable "external_dns_gcp_service_account_email" {
 
 locals {
   external_dns_namespace     = length(kubernetes_namespace.external_dns) == 0 ? var.external_dns_k8s_namespace : kubernetes_namespace.external_dns[0].metadata[0].name
-  external_dns_domain_filter = trimsuffix("${var.cluster.name}.${var.dns_managed_zone.dns_name}", ".")
+  external_dns_domain_filter = trimsuffix("${var.dns_managed_zone.dns_name}", ".")
+  # external_dns_domain_filter needs to match a google_dns_managed_zone so to keep things simple I am only filtering to the dns_root. If we wanted to filter to the cluster subdomain, we could create a separate google_dns_managed_zone for each cluster (and set IAM permissions accordingly).
 }
 
 resource "kubernetes_namespace" "external_dns" {

terraform/modules/k8s_workload/ingress_nginx.tf

@@ -8,8 +8,9 @@
 #   controller: k8s.io/ingress-nginx
 
 module "nginx_ingress_controller" {
-  count  = var.ingress_type == "nginx" ? 1 : 0
+  count          = var.ingress_type == "nginx" ? 1 : 0
-  source = "../nginx_ingress_controller"
+  source         = "../nginx_ingress_controller"
+  public_ingress = var.public_ingress
 }
 
 resource "kubernetes_ingress_v1" "ingress_nginx" {
@@ -18,7 +19,7 @@ resource "kubernetes_ingress_v1" "ingress_nginx" {
   metadata {
     name = "${var.cluster.name}-${each.value.metadata[0].name}"
     annotations = {
-      "kubernetes.io/ingress.class" = var.public_ingress ? "gce" : "gce-internal"
+      "kubernetes.io/ingress.class" = "nginx"
     }
   }
 
@@ -41,5 +42,5 @@ resource "kubernetes_ingress_v1" "ingress_nginx" {
     }
   }
 
-  depends_on = [time_sleep.wait_service_cleanup]
+  depends_on = [time_sleep.wait_service_cleanup, module.nginx_ingress_controller]
 }

terraform/modules/nginx_ingress_controller/ingress-nginx-controller-v1.12.0.tf

@@ -514,7 +514,6 @@ resource "kubernetes_manifest" "clusterrolebinding_ingress_nginx_admission" {
 resource "kubernetes_manifest" "configmap_ingress_nginx_ingress_nginx_controller" {
   manifest = {
     "apiVersion" = "v1"
-    "data"       = null
     "kind"       = "ConfigMap"
     "metadata" = {
       "labels" = {
@@ -535,6 +534,9 @@ resource "kubernetes_manifest" "service_ingress_nginx_ingress_nginx_controller"
     "apiVersion" = "v1"
     "kind"       = "Service"
     "metadata" = {
+      "annotations" = {
+        "networking.gke.io/load-balancer-type" = var.public_ingress ? "External" : "Internal"
+      }
       "labels" = {
         "app.kubernetes.io/component" = "controller"
         "app.kubernetes.io/instance"  = "ingress-nginx"
@@ -612,6 +614,7 @@ resource "kubernetes_manifest" "service_ingress_nginx_ingress_nginx_controller_a
 }
 
 resource "kubernetes_manifest" "deployment_ingress_nginx_ingress_nginx_controller" {
+  computed_fields = ["metadata.annotations", "metadata.labels", "spec.template.metadata.labels"]
   manifest = {
     "apiVersion" = "apps/v1"
     "kind"       = "Deployment"
@@ -627,7 +630,6 @@ resource "kubernetes_manifest" "deployment_ingress_nginx_ingress_nginx_controlle
       "namespace" = kubernetes_manifest.namespace_ingress_nginx.manifest.metadata.name
     }
     "spec" = {
-      "minReadySeconds"      = 0
       "revisionHistoryLimit" = 10
       "selector" = {
         "matchLabels" = {
@@ -795,6 +797,7 @@ resource "kubernetes_manifest" "deployment_ingress_nginx_ingress_nginx_controlle
 }
 
 resource "kubernetes_manifest" "job_ingress_nginx_ingress_nginx_admission_create" {
+  computed_fields = ["metadata.annotations", "metadata.labels", "spec.template.metadata.labels"]
   manifest = {
     "apiVersion" = "batch/v1"
     "kind"       = "Job"
@@ -872,6 +875,7 @@ resource "kubernetes_manifest" "job_ingress_nginx_ingress_nginx_admission_create
 }
 
 resource "kubernetes_manifest" "job_ingress_nginx_ingress_nginx_admission_patch" {
+  computed_fields = ["metadata.annotations", "metadata.labels", "spec.template.metadata.labels"]
   manifest = {
     "apiVersion" = "batch/v1"
     "kind"       = "Job"

terraform/modules/nginx_ingress_controller/main.tf

@@ -7,6 +7,11 @@ terraform {
   }
 }
 
+variable "public_ingress" {
+  description = "Set to true to make the kubernetes ingresses exposed to the public internet."
+  type        = bool
+}
+
 data "google_client_config" "default" {}
 
 resource "kubernetes_cluster_role_binding" "cluster_admin_binding" {

terraform/quota.tf

@@ -1,5 +1,10 @@
 # TODO: Make public IP quota dependent on var.public_ingress and update the amount to match what is expected to be spun up.
 
+locals {
+  num_clusters            = 14
+  num_public_ip_addresses = var.ingress_type == "gce" ? 10 * local.num_clusters : local.num_clusters
+}
+
 # resource "google_cloud_quotas_quota_preference" "clusters_per_region" {
 #   count         = var.quota_email == null ? 0 : 1
 #   parent        = "projects/${google_project.project.project_id}"
@@ -24,7 +29,7 @@
 #   quota_id      = "IN-USE-ADDRESSES-per-project-region"
 #   contact_email = var.quota_email
 #   quota_config {
-#     preferred_value = 70
+#     preferred_value = local.num_public_ip_addresses
 #   }
 #   justification = var.quota_justification
 #   depends_on    = [google_project_service.service["cloudquotas"], ]