Switch to not including trailing period in root domain.

README.md

@@ -88,8 +88,8 @@ gcloud auth application-default login
 Then go into the `terraform` folder and apply the configuration. We need to apply the config in two phases via the `cluster_exists` variable because the kubernetes terraform provider does not have native support for the Gateway API and the `kubernetes_manifest` terraform resource [has a shortcoming that requires the cluster exists at plan time](https://github.com/hashicorp/terraform-provider-kubernetes/issues/1775).
 
 ```
-terraform apply -var dns_root="k8sdemo.mydomain.example." -var quota_email="MrManager@mydomain.example" -var quota_justification="Explain why you need quotas increased here." -var cluster_exists=false
+terraform apply -var dns_root="k8sdemo.mydomain.example" -var quota_email="MrManager@mydomain.example" -var quota_justification="Explain why you need quotas increased here." -var cluster_exists=false
-terraform apply -var dns_root="k8sdemo.mydomain.example." -var quota_email="MrManager@mydomain.example" -var quota_justification="Explain why you need quotas increased here." -var cluster_exists=true
+terraform apply -var dns_root="k8sdemo.mydomain.example" -var quota_email="MrManager@mydomain.example" -var quota_justification="Explain why you need quotas increased here." -var cluster_exists=true
 ```
 
 Please note that this will exceed the default quotas on new Google Cloud projects. The terraform configuration will automatically put in requests for quota increases but they can take multiple days to be approved or denied. You should be able to fit 3 clusters in the default quota until then.
@@ -279,7 +279,7 @@ But that doesn't mean that we need to use the valuable RFC-1918 IP address space
 To demonstrate, we can apply the terraform config again but with the `enable_snat=true` variable set:
 
 ```
-terraform apply -var dns_root="k8sdemo.mydomain.example." -var quota_email="MrManager@mydomain.example" -var quota_justification="Explain why you need quotas increased here." -var cluster_exists=true -var enable_snat=true
+terraform apply -var dns_root="k8sdemo.mydomain.example" -var quota_email="MrManager@mydomain.example" -var quota_justification="Explain why you need quotas increased here." -var cluster_exists=true -var enable_snat=true
 ```
 
 Then in our kubernetes pod, we can run the `curl` again:
@@ -304,11 +304,13 @@ Question and Answer
 
 [GKE assigns a separate IP address to each `Ingress`](https://cloud.google.com/kubernetes-engine/docs/concepts/ingress#limitations), but we can have a single `Gateway` with an IP address and then any quantity of `HTTPRoute`. This is a design choice for GKE, and not a limitation of kubernetes.
 
+If you need to use `Ingress`, we can achieve the same efficiency for IP addresses by using the nginx ingress controller. This can be enabled by passing `-var ingress_type=nginx`. If you need to use the built-in ingress controller instead of nginx you can set `-var ingress_type=gce` but then each `Ingress` will cost 1 IP address.
+
 Clean Up
 ========
 Just like we did a 2-stage apply by toggling the `cluster_exists` variable, we will need to do a 2-stage destroy. First we tear down any kubernetes resources by running *apply* with the `cluster_exists` variable set to `false`. Then we can destroy the entire project.
 
 ```
-terraform apply -var dns_root="k8sdemo.mydomain.example." -var quota_email="MrManager@mydomain.example" -var quota_justification="Explain why you need quotas increased here." -var cluster_exists=false
+terraform apply -var dns_root="k8sdemo.mydomain.example" -var quota_email="MrManager@mydomain.example" -var quota_justification="Explain why you need quotas increased here." -var cluster_exists=false
-terraform destroy -var dns_root="k8sdemo.mydomain.example." -var quota_email="MrManager@mydomain.example" -var quota_justification="Explain why you need quotas increased here." -var cluster_exists=false
+terraform destroy -var dns_root="k8sdemo.mydomain.example" -var quota_email="MrManager@mydomain.example" -var quota_justification="Explain why you need quotas increased here." -var cluster_exists=false
 ```

terraform/dns.tf

@@ -1,7 +1,5 @@
-# TODO: Switch to not requiring trailing period?
-
 variable "dns_root" {
-  description = "DNS domain root with trailing period. Example: \"foo.bar.com.\""
+  description = "DNS domain root. Example: \"k8sdemo.mydomain.example\""
   type        = string
 }
 
@@ -9,7 +7,7 @@ variable "dns_root" {
 resource "google_dns_managed_zone" "zone" {
   project  = google_project.project.project_id
   name     = "dns-zone"
-  dns_name = var.dns_root
+  dns_name = "${var.dns_root}."
 
   depends_on = [google_project_service.service["dns"], ]
 }

terraform/external_dns.tf

@@ -43,7 +43,7 @@ resource "google_project_iam_member" "external_dns" {
   role    = "roles/dns.reader"
 }
 
-resource "google_dns_managed_zone_iam_member" "member" {
+resource "google_dns_managed_zone_iam_member" "external_dns" {
   project      = google_project.project.project_id
   managed_zone = google_dns_managed_zone.zone.name
   role         = "roles/dns.admin"

terraform/main.tf

@@ -1,3 +1,4 @@
+# TODO: put IP address ranges into variables
 terraform {
   backend "gcs" {
     bucket = "tf-state-4b00"
@@ -48,6 +49,11 @@ variable "ingress_type" {
   description = "What controller should we use to handle incoming http(s) connections."
   type        = string
   default     = "gateway"
+
+  validation {
+    condition     = contains(["gateway", "nginx", "gce"], var.ingress_type)
+    error_message = "Must be either \"gateway\", \"nginx\", or \"gce\"."
+  }
 }
 
 variable "cluster_exists" {

terraform/modules/cluster/main.tf

@@ -46,12 +46,6 @@ variable "public_ingress" {
 variable "ingress_type" {
   description = "What controller should we use to handle incoming http(s) connections."
   type        = string
-  default     = "gateway"
-
-  validation {
-    condition     = contains(["gateway"], var.ingress_type)
-    error_message = "Currently only \"gateway\" is supported."
-  }
 }
 
 variable "main_k8s_namespace" {

terraform/modules/k8s_workload/external_dns.tf

@@ -12,7 +12,8 @@ variable "external_dns_gcp_service_account_email" {
 
 locals {
   external_dns_namespace     = length(kubernetes_namespace.external_dns) == 0 ? var.external_dns_k8s_namespace : kubernetes_namespace.external_dns[0].metadata[0].name
-  external_dns_domain_filter = trimsuffix("${var.cluster.name}.${var.dns_managed_zone.dns_name}", ".")
+  external_dns_domain_filter = trimsuffix("${var.dns_managed_zone.dns_name}", ".")
+  # external_dns_domain_filter needs to match a google_dns_managed_zone so to keep things simple I am only filtering to the dns_root. If we wanted to filter to the cluster subdomain, we could create a separate google_dns_managed_zone for each cluster (and set IAM permissions accordingly).
 }
 
 resource "kubernetes_namespace" "external_dns" {

terraform/modules/k8s_workload/ingress_nginx.tf

@@ -10,6 +10,7 @@
 module "nginx_ingress_controller" {
   count          = var.ingress_type == "nginx" ? 1 : 0
   source         = "../nginx_ingress_controller"
+  public_ingress = var.public_ingress
 }
 
 resource "kubernetes_ingress_v1" "ingress_nginx" {
@@ -18,7 +19,7 @@ resource "kubernetes_ingress_v1" "ingress_nginx" {
   metadata {
     name = "${var.cluster.name}-${each.value.metadata[0].name}"
     annotations = {
-      "kubernetes.io/ingress.class" = var.public_ingress ? "gce" : "gce-internal"
+      "kubernetes.io/ingress.class" = "nginx"
     }
   }
 
@@ -41,5 +42,5 @@ resource "kubernetes_ingress_v1" "ingress_nginx" {
     }
   }
 
-  depends_on = [time_sleep.wait_service_cleanup]
+  depends_on = [time_sleep.wait_service_cleanup, module.nginx_ingress_controller]
 }

terraform/modules/nginx_ingress_controller/ingress-nginx-controller-v1.12.0.tf

@@ -514,7 +514,6 @@ resource "kubernetes_manifest" "clusterrolebinding_ingress_nginx_admission" {
 resource "kubernetes_manifest" "configmap_ingress_nginx_ingress_nginx_controller" {
   manifest = {
     "apiVersion" = "v1"
-    "data"       = null
     "kind"       = "ConfigMap"
     "metadata" = {
       "labels" = {
@@ -535,6 +534,9 @@ resource "kubernetes_manifest" "service_ingress_nginx_ingress_nginx_controller"
     "apiVersion" = "v1"
     "kind"       = "Service"
     "metadata" = {
+      "annotations" = {
+        "networking.gke.io/load-balancer-type" = var.public_ingress ? "External" : "Internal"
+      }
       "labels" = {
         "app.kubernetes.io/component" = "controller"
         "app.kubernetes.io/instance"  = "ingress-nginx"
@@ -612,6 +614,7 @@ resource "kubernetes_manifest" "service_ingress_nginx_ingress_nginx_controller_a
 }
 
 resource "kubernetes_manifest" "deployment_ingress_nginx_ingress_nginx_controller" {
+  computed_fields = ["metadata.annotations", "metadata.labels", "spec.template.metadata.labels"]
   manifest = {
     "apiVersion" = "apps/v1"
     "kind"       = "Deployment"
@@ -627,7 +630,6 @@ resource "kubernetes_manifest" "deployment_ingress_nginx_ingress_nginx_controlle
       "namespace" = kubernetes_manifest.namespace_ingress_nginx.manifest.metadata.name
     }
     "spec" = {
-      "minReadySeconds"      = 0
       "revisionHistoryLimit" = 10
       "selector" = {
         "matchLabels" = {
@@ -795,6 +797,7 @@ resource "kubernetes_manifest" "deployment_ingress_nginx_ingress_nginx_controlle
 }
 
 resource "kubernetes_manifest" "job_ingress_nginx_ingress_nginx_admission_create" {
+  computed_fields = ["metadata.annotations", "metadata.labels", "spec.template.metadata.labels"]
   manifest = {
     "apiVersion" = "batch/v1"
     "kind"       = "Job"
@@ -872,6 +875,7 @@ resource "kubernetes_manifest" "job_ingress_nginx_ingress_nginx_admission_create
 }
 
 resource "kubernetes_manifest" "job_ingress_nginx_ingress_nginx_admission_patch" {
+  computed_fields = ["metadata.annotations", "metadata.labels", "spec.template.metadata.labels"]
   manifest = {
     "apiVersion" = "batch/v1"
     "kind"       = "Job"

terraform/modules/nginx_ingress_controller/main.tf

@@ -7,6 +7,11 @@ terraform {
   }
 }
 
+variable "public_ingress" {
+  description = "Set to true to make the kubernetes ingresses exposed to the public internet."
+  type        = bool
+}
+
 data "google_client_config" "default" {}
 
 resource "kubernetes_cluster_role_binding" "cluster_admin_binding" {

terraform/quota.tf

@@ -1,5 +1,10 @@
 # TODO: Make public IP quota dependent on var.public_ingress and update the amount to match what is expected to be spun up.
 
+locals {
+  num_clusters            = 14
+  num_public_ip_addresses = var.ingress_type == "gce" ? 10 * local.num_clusters : local.num_clusters
+}
+
 # resource "google_cloud_quotas_quota_preference" "clusters_per_region" {
 #   count         = var.quota_email == null ? 0 : 1
 #   parent        = "projects/${google_project.project.project_id}"
@@ -16,7 +21,7 @@
 # }
 
 # resource "google_cloud_quotas_quota_preference" "public_ip_per_project_region" {
-#   count         = var.quota_email == null ? 0 : 1
+#   count         = var.quota_email == null && var.public_ingress == true ? 0 : 1
 #   parent        = "projects/${google_project.project.project_id}"
 #   name          = "compute-IN-USE-ADDRESSES-per-project-region"
 #   dimensions    = { region = var.region }
@@ -24,7 +29,7 @@
 #   quota_id      = "IN-USE-ADDRESSES-per-project-region"
 #   contact_email = var.quota_email
 #   quota_config {
-#     preferred_value = 70
+#     preferred_value = local.num_public_ip_addresses
 #   }
 #   justification = var.quota_justification
 #   depends_on    = [google_project_service.service["cloudquotas"], ]