Demonstrate conservative RFC1918 IP address use on GKE.

This is a terraform config demonstrating spinning up 14 clusters in only a /26 (64 addresses) to demonstrate the GKE clusters do not need to consume large amounts of RFC1918 IP addresses.

terraform/modules/k8s_workload/external_dns.tf

@@ -0,0 +1,154 @@
+variable "external_dns_k8s_namespace" {
+  type = string
+}
+
+variable "external_dns_k8s_service_account" {
+  type = string
+}
+
+variable "external_dns_gcp_service_account_email" {
+  type = string
+}
+
+locals {
+  external_dns_namespace     = length(kubernetes_namespace.external_dns) == 0 ? var.external_dns_k8s_namespace : kubernetes_namespace.external_dns[0].metadata[0].name
+  external_dns_domain_filter = trimsuffix("${var.cluster.name}.${var.dns_managed_zone.dns_name}", ".")
+}
+
+resource "kubernetes_namespace" "external_dns" {
+  count = var.external_dns_k8s_namespace != "default" ? 1 : 0
+  metadata {
+    name = var.external_dns_k8s_namespace
+  }
+
+  timeouts {
+    delete = "60m"
+  }
+
+  depends_on = [var.node_pool]
+}
+
+
+resource "kubernetes_service_account" "external_dns" {
+  metadata {
+    name      = var.external_dns_k8s_service_account
+    namespace = local.external_dns_namespace
+    annotations = {
+      "iam.gke.io/gcp-service-account" = var.external_dns_gcp_service_account_email
+    }
+  }
+
+  depends_on = [var.node_pool]
+}
+
+resource "kubernetes_cluster_role" "external_dns" {
+  metadata {
+    name = "external-dns"
+
+    labels = {
+      "app.kubernetes.io/name" = "external-dns"
+    }
+  }
+
+  rule {
+    verbs      = ["get", "watch", "list"]
+    api_groups = [""]
+    resources  = ["services", "endpoints", "pods", "nodes", "namespaces"]
+  }
+
+  rule {
+    verbs      = ["get", "watch", "list"]
+    api_groups = ["extensions", "networking.k8s.io"]
+    resources  = ["ingresses"]
+  }
+
+  rule {
+    verbs      = ["get", "watch", "list"]
+    api_groups = ["gateway.networking.k8s.io"]
+    resources  = ["gateways", "httproutes", "tlsroutes", "tcproutes", "udproutes"]
+  }
+
+  depends_on = [var.node_pool]
+}
+
+resource "kubernetes_cluster_role_binding" "external_dns_viewer" {
+  metadata {
+    name = "external-dns-viewer"
+
+    labels = {
+      "app.kubernetes.io/name" = "external-dns"
+    }
+  }
+
+  subject {
+    kind      = "ServiceAccount"
+    name      = kubernetes_service_account.external_dns.metadata[0].name
+    namespace = kubernetes_service_account.external_dns.metadata[0].namespace
+  }
+
+  role_ref {
+    api_group = "rbac.authorization.k8s.io"
+    kind      = "ClusterRole"
+    name      = "external-dns"
+  }
+
+  depends_on = [var.node_pool]
+}
+
+resource "kubernetes_deployment" "external_dns" {
+  metadata {
+    name      = "external-dns"
+    namespace = local.external_dns_namespace
+
+    labels = {
+      "app.kubernetes.io/name" = "external-dns"
+    }
+  }
+
+  spec {
+    selector {
+      match_labels = {
+        "app.kubernetes.io/name" = "external-dns"
+      }
+    }
+
+    revision_history_limit = 0
+
+    template {
+      metadata {
+        labels = {
+          "app.kubernetes.io/name" = "external-dns"
+        }
+      }
+
+      spec {
+        container {
+          name  = "external-dns"
+          image = "registry.k8s.io/external-dns/external-dns:v0.15.1"
+          args = [
+            "--source=service",
+            "--source=ingress",
+            "--source=gateway-httproute",
+            # "--source=gateway-tlsroute",
+            # "--source=gateway-tcproute",
+            # "--source=gateway-udproute",
+            "--domain-filter=${local.external_dns_domain_filter}",
+            "--provider=google",
+            "--log-format=json",
+            "--registry=txt",
+            "--txt-owner-id=k8s-${var.cluster.name}",
+            # "--log-level=debug",
+          ]
+        }
+
+        service_account_name = kubernetes_service_account.external_dns.metadata[0].name
+      }
+    }
+
+    strategy {
+      type = "Recreate"
+    }
+  }
+
+  depends_on = [var.node_pool, time_sleep.wait_service_cleanup]
+}

terraform/modules/k8s_workload/ingress_gateway.tf

@@ -0,0 +1,79 @@
+# Terraform does not support gateway API so we need to use the generic kubernetes_manifest type instead.
+# https://github.com/hashicorp/terraform-provider-kubernetes/issues/2474
+
+resource "kubernetes_manifest" "gateway" {
+  count = var.ingress_type == "gateway" ? 1 : 0
+
+  manifest = {
+    "apiVersion" = "gateway.networking.k8s.io/v1"
+    "kind"       = "Gateway"
+    "metadata" = {
+      "name"      = "${var.cluster.name}-gateway"
+      "namespace" = var.main_k8s_namespace
+    }
+    "spec" = {
+      "gatewayClassName" = var.public_ingress ? "gke-l7-gxlb" : "gke-l7-rilb"
+      "listeners" = [
+        {
+          "name"     = "plain-http"
+          "protocol" = "HTTP"
+          "port"     = 80
+          "allowedRoutes" = {
+            "kinds" = [
+              {
+                "kind" = "HTTPRoute"
+              }
+            ]
+            "namespaces" = {
+              "from" : "All"
+            }
+          }
+        }
+      ]
+    }
+  }
+
+  depends_on = [time_sleep.wait_service_cleanup, var.cluster]
+}
+
+resource "kubernetes_manifest" "httproute" {
+  for_each = var.ingress_type == "gateway" ? { for k, v in kubernetes_service_v1.default : k => v } : {}
+
+  manifest = {
+    "apiVersion" = "gateway.networking.k8s.io/v1"
+    "kind"       = "HTTPRoute"
+    "metadata" = {
+      "name"      = "${var.cluster.name}-${each.value.metadata[0].name}"
+      "namespace" = var.main_k8s_namespace
+    }
+    "spec" = {
+      "parentRefs" = [
+        {
+          "name" : kubernetes_manifest.gateway[0].manifest.metadata.name
+          "namespace" : kubernetes_manifest.gateway[0].manifest.metadata.namespace
+        }
+      ]
+      "hostnames" = [trimsuffix("${each.value.metadata[0].name}.${var.cluster.name}.${var.dns_managed_zone.dns_name}", ".")]
+      "rules" = [
+        {
+          "backendRefs" = [
+            {
+              "name" = each.value.metadata[0].name
+              "port" = 80
+            },
+          ]
+          "matches" = [
+            {
+              "path" = {
+                "type"  = "PathPrefix"
+                "value" = "/"
+              }
+            },
+          ]
+        }
+      ]
+    }
+  }
+
+  depends_on = [time_sleep.wait_service_cleanup, var.cluster]
+}

terraform/modules/k8s_workload/ingress_gce.tf

@@ -0,0 +1,31 @@
+resource "kubernetes_ingress_v1" "ingress_gce" {
+  for_each = var.ingress_type == "gce" ? { for k, v in kubernetes_service_v1.default : k => v } : {}
+
+  metadata {
+    name = "${var.cluster.name}-${each.value.metadata[0].name}"
+    annotations = {
+      "kubernetes.io/ingress.class" = var.public_ingress ? "gce" : "gce-internal"
+    }
+  }
+
+  spec {
+    rule {
+      host = trimsuffix("${each.value.metadata[0].name}.${var.cluster.name}.${var.dns_managed_zone.dns_name}", ".")
+      http {
+        path {
+          path = "/"
+          backend {
+            service {
+              name = each.value.metadata[0].name
+              port {
+                number = 80
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  depends_on = [time_sleep.wait_service_cleanup]
+}

terraform/modules/k8s_workload/ingress_nginx.tf

@@ -0,0 +1,45 @@
+# apiVersion: networking.k8s.io/v1
+# kind: IngressClass
+# metadata:
+#   name: nginx-public
+#   annotations:
+#     ingressclass.kubernetes.io/is-default-class: "true"
+# spec:
+#   controller: k8s.io/ingress-nginx
+
+module "nginx_ingress_controller" {
+  count  = var.ingress_type == "nginx" ? 1 : 0
+  source = "../nginx_ingress_controller"
+}
+
+resource "kubernetes_ingress_v1" "ingress_nginx" {
+  for_each = var.ingress_type == "nginx" ? { for k, v in kubernetes_service_v1.default : k => v } : {}
+
+  metadata {
+    name = "${var.cluster.name}-${each.value.metadata[0].name}"
+    annotations = {
+      "kubernetes.io/ingress.class" = var.public_ingress ? "gce" : "gce-internal"
+    }
+  }
+
+  spec {
+    rule {
+      host = trimsuffix("${each.value.metadata[0].name}.${var.cluster.name}.${var.dns_managed_zone.dns_name}", ".")
+      http {
+        path {
+          path = "/"
+          backend {
+            service {
+              name = each.value.metadata[0].name
+              port {
+                number = 80
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  depends_on = [time_sleep.wait_service_cleanup]
+}

terraform/modules/k8s_workload/main.tf

@@ -0,0 +1,143 @@
+variable "project" {
+  type = string
+}
+
+variable "region" {
+  type = string
+}
+
+variable "cluster" {
+}
+
+variable "node_pool" {
+}
+
+variable "dns_managed_zone" {
+}
+
+variable "public_ingress" {
+  description = "Set to true to make the kubernetes ingresses exposed to the public internet."
+  type        = bool
+}
+
+variable "ingress_type" {
+  description = "What controller should we use to handle incoming http(s) connections."
+  type        = string
+}
+
+variable "main_k8s_namespace" {
+  type = string
+}
+
+# Provide time for Service cleanup
+resource "time_sleep" "wait_service_cleanup" {
+  depends_on = [var.cluster]
+
+  destroy_duration = "180s"
+}
+
+
+resource "kubernetes_deployment_v1" "default" {
+  count = 12
+  metadata {
+    name = "deployment${count.index + 1}"
+  }
+
+  spec {
+    replicas = 2
+
+    selector {
+      match_labels = {
+        app = "hello-app-${count.index + 1}"
+      }
+    }
+
+    template {
+      metadata {
+        labels = {
+          app = "hello-app-${count.index + 1}"
+        }
+      }
+
+      spec {
+        container {
+          image = "us-docker.pkg.dev/google-samples/containers/gke/hello-app:2.0"
+          name  = "hello-app-container"
+
+          port {
+            container_port = 8080
+            name           = "hello-app-svc"
+          }
+
+          security_context {
+            allow_privilege_escalation = false
+            privileged                 = false
+            read_only_root_filesystem  = false
+
+            capabilities {
+              add  = []
+              drop = ["NET_RAW"]
+            }
+          }
+
+          liveness_probe {
+            http_get {
+              path = "/"
+              port = "hello-app-svc"
+            }
+
+            initial_delay_seconds = 3
+            period_seconds        = 3
+          }
+        }
+
+        security_context {
+          run_as_non_root = true
+
+          seccomp_profile {
+            type = "RuntimeDefault"
+          }
+        }
+
+        # Toleration is currently required to prevent perpetual diff:
+        # https://github.com/hashicorp/terraform-provider-kubernetes/pull/2380
+        toleration {
+          effect   = "NoSchedule"
+          key      = "kubernetes.io/arch"
+          operator = "Equal"
+          value    = "amd64"
+        }
+      }
+    }
+  }
+
+  depends_on = [var.node_pool]
+}
+
+resource "kubernetes_service_v1" "default" {
+  count = 12
+  metadata {
+    name = "service${count.index + 1}"
+    annotations = {
+      # TODO: Revisit this, is this needed with the gateway API?
+      "networking.gke.io/load-balancer-type" = "Internal" # Remove to create an external loadbalancer
+    }
+  }
+
+  spec {
+    selector = {
+      app = kubernetes_deployment_v1.default[count.index].spec[0].selector[0].match_labels.app
+    }
+
+    ip_family_policy = "SingleStack"
+
+    port {
+      port        = 80
+      target_port = kubernetes_deployment_v1.default[count.index].spec[0].template[0].spec[0].container[0].port[0].name
+    }
+
+    type = "ClusterIP"
+  }
+
+  depends_on = [var.node_pool, time_sleep.wait_service_cleanup]
+}