From d8182298bc4759ce8ffe75b9b9dd65ede3fbbe9c Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 12 Jul 2021 23:24:11 -0400 Subject: [PATCH 1/6] Remove the old redis code. --- terraform/basic_gke/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/terraform/basic_gke/main.tf b/terraform/basic_gke/main.tf index 46b7963..751e6b5 100644 --- a/terraform/basic_gke/main.tf +++ b/terraform/basic_gke/main.tf @@ -285,3 +285,4 @@ output "redis_port" { value = module.redis.redis_port } + From a6c408c3c6ce54dcfd6ef2f477e8f81ad3f5b10b Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 12 Jul 2021 23:55:17 -0400 Subject: [PATCH 2/6] Reduce range for private services to a /24 --- terraform/modules/networking/networking.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/modules/networking/networking.tf b/terraform/modules/networking/networking.tf index 4a7b757..b85b96e 100644 --- a/terraform/modules/networking/networking.tf +++ b/terraform/modules/networking/networking.tf @@ -28,7 +28,7 @@ resource "google_compute_global_address" "private_ip_address" { name = "private-ip-address" purpose = "VPC_PEERING" address_type = "INTERNAL" - prefix_length = 16 + prefix_length = 24 network = google_compute_network.private_network.id } From 9d8a1f21426177e3c2c406d9812c84f03dabef3a Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Tue, 13 Jul 2021 01:10:23 -0400 Subject: [PATCH 3/6] Move GKE to its own module. --- terraform/basic_gke/main.tf | 181 ++------------------------------ terraform/modules/gke/gke.tf | 198 +++++++++++++++++++++++++++++++++++ 2 files changed, 208 insertions(+), 171 deletions(-) create mode 100644 terraform/modules/gke/gke.tf diff --git a/terraform/basic_gke/main.tf b/terraform/basic_gke/main.tf index 751e6b5..b3b267e 100644 --- a/terraform/basic_gke/main.tf +++ b/terraform/basic_gke/main.tf @@ -70,183 +70,22 @@ resource "google_project_service" "cloudkms" { disable_dependent_services = true } -resource "random_id" "gke_db" { - byte_length = 4 -} - -resource "google_kms_key_ring" "gke_db" { - project = var.project - name = "gke-db-${random_id.gke_db.hex}" - location = var.region - - lifecycle { - prevent_destroy = true - } - - depends_on = [ - google_project_service.cloudkms - ] -} - -resource "google_kms_key_ring_iam_policy" "gke_db" { - key_ring_id = google_kms_key_ring.gke_db.id - policy_data = data.google_iam_policy.gke_db.policy_data - - depends_on = [ - google_project_service.cloudkms - ] -} - -resource "google_kms_crypto_key" "gke_db" { - name = "gke-db-key" - key_ring = google_kms_key_ring.gke_db.id - - lifecycle { - prevent_destroy = true - } - - depends_on = [ - google_project_service.container - ] -} - -data "google_iam_policy" "gke_db" { - binding { - role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" - - members = [ - "serviceAccount:service-${data.google_project.project.number}@container-engine-robot.iam.gserviceaccount.com" - ] - } -} - #################### GKE ################################## -resource "google_project_service" "container" { - project = var.project - service = "container.googleapis.com" - disable_dependent_services = true -} - -resource "google_project_service" "containerregistry" { - project = var.project - service = "containerregistry.googleapis.com" - disable_dependent_services = true -} - -resource "google_service_account" "gke" { - project = var.project - account_id = "gke-service-account" - display_name = "GKE Service Account" -} - -# Allow GKE to access custom docker images in GCR -resource "google_storage_bucket_iam_member" "gke_gcr" { - bucket = "artifacts.${google_service_account.gke.project}.appspot.com" - role = "roles/storage.objectViewer" - member = "serviceAccount:${google_service_account.gke.email}" - - depends_on = [ - google_project_service.containerregistry - ] -} - -resource "google_container_cluster" "primary" { - project = var.project - name = "gke-cluster" - location = var.region - - remove_default_node_pool = true - initial_node_count = 1 - enable_shielded_nodes = true - min_master_version = "1.19.10-gke.1000" - - database_encryption { - state = "ENCRYPTED" - key_name = google_kms_crypto_key.gke_db.self_link - } - - maintenance_policy { - daily_maintenance_window { - start_time = "03:00" - } - } - - workload_identity_config { - identity_namespace = "${data.google_project.project.project_id}.svc.id.goog" - } - - release_channel { - channel = "STABLE" - } - - master_auth { - username = "" - password = "" - } - - ip_allocation_policy { - cluster_ipv4_cidr_block = "10.1.0.0/16" - services_ipv4_cidr_block = "10.2.0.0/20" - } - - lifecycle { - prevent_destroy = true - } - - depends_on = [ - google_project_service.container, - google_kms_key_ring_iam_policy.gke_db - ] -} - -resource "google_container_node_pool" "primary" { - project = google_container_cluster.primary.project - name_prefix = "node-pool" - location = var.region - cluster = google_container_cluster.primary.name - initial_node_count = 1 +module "gke" { + source = "../modules/gke" + project = var.project + region = var.region + service_cloudkms = google_project_service.cloudkms - autoscaling { - min_node_count = 0 - max_node_count = 20 - } - - node_config { - preemptible = true - machine_type = "e2-medium" - - service_account = google_service_account.gke.email - oauth_scopes = [ - "https://www.googleapis.com/auth/cloud-platform" - ] - - metadata = { - disable-legacy-endpoints = "true" - } - - tags = [] - - shielded_instance_config { - enable_secure_boot = true - enable_integrity_monitoring = true - } - } - - lifecycle { - ignore_changes = [ - node_count - ] - } - - depends_on = [ - google_project_service.container - ] + # depends_on = [ + # module.networking + # ] } output "gke_connect_command" { - description = "Command to run to connect to the kubernetes cluster." - value = "gcloud container clusters get-credentials ${google_container_cluster.primary.name} --region ${var.region} --project ${var.project}" + # description = "Command to run to connect to the kubernetes cluster." + value = module.gke.gke_connect_command } #################### SQL ################################## diff --git a/terraform/modules/gke/gke.tf b/terraform/modules/gke/gke.tf new file mode 100644 index 0000000..f39e01c --- /dev/null +++ b/terraform/modules/gke/gke.tf @@ -0,0 +1,198 @@ +variable "project" { + description = "Project ID." + type = string +} + +variable "region" { + description = "Region." + type = string +} + +variable "service_cloudkms" { + description = "cloudkms service." +} + +output "gke_connect_command" { + description = "Command to run to connect to the kubernetes cluster." + value = "gcloud container clusters get-credentials ${google_container_cluster.primary.name} --region ${var.region} --project ${var.project}" +} + +data "google_project" "project" { + project_id = var.project +} + +#################### KMS ################################## + +resource "random_id" "gke_db" { + byte_length = 4 +} + +resource "google_kms_key_ring" "gke_db" { + project = var.project + name = "gke-db-${random_id.gke_db.hex}" + location = var.region + + lifecycle { + prevent_destroy = true + } + + depends_on = [ + var.service_cloudkms + ] +} + +resource "google_kms_key_ring_iam_policy" "gke_db" { + key_ring_id = google_kms_key_ring.gke_db.id + policy_data = data.google_iam_policy.gke_db.policy_data + + depends_on = [ + var.service_cloudkms + ] +} + +resource "google_kms_crypto_key" "gke_db" { + name = "gke-db-key" + key_ring = google_kms_key_ring.gke_db.id + + lifecycle { + prevent_destroy = true + } + + depends_on = [ + google_project_service.container + ] +} + +data "google_iam_policy" "gke_db" { + binding { + role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" + + members = [ + "serviceAccount:service-${data.google_project.project.number}@container-engine-robot.iam.gserviceaccount.com" + ] + } +} + +#################### GKE ################################## + +resource "google_project_service" "container" { + project = var.project + service = "container.googleapis.com" + disable_dependent_services = true +} + +resource "google_project_service" "containerregistry" { + project = var.project + service = "containerregistry.googleapis.com" + disable_dependent_services = true +} + +resource "google_service_account" "gke" { + project = var.project + account_id = "gke-service-account" + display_name = "GKE Service Account" +} + +# Allow GKE to access custom docker images in GCR +resource "google_storage_bucket_iam_member" "gke_gcr" { + bucket = "artifacts.${google_service_account.gke.project}.appspot.com" + role = "roles/storage.objectViewer" + member = "serviceAccount:${google_service_account.gke.email}" + + depends_on = [ + google_project_service.containerregistry + ] +} + +resource "google_container_cluster" "primary" { + project = var.project + name = "gke-cluster" + location = var.region + + remove_default_node_pool = true + initial_node_count = 1 + enable_shielded_nodes = true + min_master_version = "1.19.10-gke.1000" + + database_encryption { + state = "ENCRYPTED" + key_name = google_kms_crypto_key.gke_db.self_link + } + + maintenance_policy { + daily_maintenance_window { + start_time = "03:00" + } + } + + workload_identity_config { + identity_namespace = "${data.google_project.project.project_id}.svc.id.goog" + } + + release_channel { + channel = "STABLE" + } + + master_auth { + username = "" + password = "" + } + + ip_allocation_policy { + cluster_ipv4_cidr_block = "10.1.0.0/16" + services_ipv4_cidr_block = "10.2.0.0/20" + } + + lifecycle { + prevent_destroy = true + } + + depends_on = [ + google_project_service.container, + google_kms_key_ring_iam_policy.gke_db + ] +} + +resource "google_container_node_pool" "primary" { + project = google_container_cluster.primary.project + name_prefix = "node-pool" + location = var.region + cluster = google_container_cluster.primary.name + initial_node_count = 1 + + autoscaling { + min_node_count = 0 + max_node_count = 20 + } + + node_config { + preemptible = true + machine_type = "e2-medium" + + service_account = google_service_account.gke.email + oauth_scopes = [ + "https://www.googleapis.com/auth/cloud-platform" + ] + + metadata = { + disable-legacy-endpoints = "true" + } + + tags = [] + + shielded_instance_config { + enable_secure_boot = true + enable_integrity_monitoring = true + } + } + + lifecycle { + ignore_changes = [ + node_count + ] + } + + depends_on = [ + google_project_service.container + ] +} From b92396d3217d060b52b817d594d1b654babe26f5 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Tue, 13 Jul 2021 01:30:22 -0400 Subject: [PATCH 4/6] Switch to using an explicit net/subnet. --- terraform/basic_gke/main.tf | 11 +++--- terraform/modules/gke/gke.tf | 40 +++++++++++++++++++--- terraform/modules/networking/networking.tf | 17 +++++++++ 3 files changed, 59 insertions(+), 9 deletions(-) diff --git a/terraform/basic_gke/main.tf b/terraform/basic_gke/main.tf index b3b267e..eb36c27 100644 --- a/terraform/basic_gke/main.tf +++ b/terraform/basic_gke/main.tf @@ -48,6 +48,7 @@ data "google_project" "project" { module "networking" { source = "../modules/networking" project = var.project + region = var.region } #################### Workload Identity #################### @@ -73,10 +74,12 @@ resource "google_project_service" "cloudkms" { #################### GKE ################################## module "gke" { - source = "../modules/gke" - project = var.project - region = var.region - service_cloudkms = google_project_service.cloudkms + source = "../modules/gke" + project = var.project + region = var.region + private_network_id = module.networking.private_network_id + private_subnetwork_id = module.networking.private_subnetwork_id + service_cloudkms = google_project_service.cloudkms # depends_on = [ # module.networking diff --git a/terraform/modules/gke/gke.tf b/terraform/modules/gke/gke.tf index f39e01c..79a8980 100644 --- a/terraform/modules/gke/gke.tf +++ b/terraform/modules/gke/gke.tf @@ -12,6 +12,16 @@ variable "service_cloudkms" { description = "cloudkms service." } +variable "private_network_id" { + description = "Private network id." + type = string +} + +variable "private_subnetwork_id" { + description = "Private subnetwork id." + type = string +} + output "gke_connect_command" { description = "Command to run to connect to the kubernetes cluster." value = "gcloud container clusters get-credentials ${google_container_cluster.primary.name} --region ${var.region} --project ${var.project}" @@ -104,10 +114,30 @@ resource "google_storage_bucket_iam_member" "gke_gcr" { ] } +resource "google_compute_global_address" "gke_cluster_range" { + project = var.project + name = "gke-cluster-range" + purpose = "VPC_PEERING" + address_type = "INTERNAL" + prefix_length = 16 + network = var.private_network_id +} + +resource "google_compute_global_address" "gke_services_range" { + project = var.project + name = "gke-services-range" + purpose = "VPC_PEERING" + address_type = "INTERNAL" + prefix_length = 20 + network = var.private_network_id +} + resource "google_container_cluster" "primary" { - project = var.project - name = "gke-cluster" - location = var.region + project = var.project + name = "gke-cluster" + location = var.region + network = var.private_network_id + subnetwork = var.private_subnetwork_id remove_default_node_pool = true initial_node_count = 1 @@ -139,8 +169,8 @@ resource "google_container_cluster" "primary" { } ip_allocation_policy { - cluster_ipv4_cidr_block = "10.1.0.0/16" - services_ipv4_cidr_block = "10.2.0.0/20" + cluster_secondary_range_name = google_compute_global_address.gke_cluster_range.name + services_secondary_range_name = google_compute_global_address.gke_services_range.name } lifecycle { diff --git a/terraform/modules/networking/networking.tf b/terraform/modules/networking/networking.tf index b85b96e..a4b33a7 100644 --- a/terraform/modules/networking/networking.tf +++ b/terraform/modules/networking/networking.tf @@ -3,11 +3,21 @@ variable "project" { type = string } +variable "region" { + description = "Region." + type = string +} + output "private_network_id" { description = "Private network id." value = google_compute_network.private_network.id } +output "private_subnetwork_id" { + description = "Private subnetwork id." + value = google_compute_subnetwork.subnet.id +} + resource "google_project_service" "servicenetworking" { project = var.project service = "servicenetworking.googleapis.com" @@ -23,6 +33,13 @@ resource "google_compute_network" "private_network" { ] } +resource "google_compute_subnetwork" "subnet" { + name = "private-subnetwork" + ip_cidr_range = "10.100.0.0/16" + region = var.region + network = google_compute_network.private_network.id +} + resource "google_compute_global_address" "private_ip_address" { project = google_compute_network.private_network.project name = "private-ip-address" From 2cf9f93de64dff8023903360148d297dc97b8fb5 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Tue, 13 Jul 2021 01:41:06 -0400 Subject: [PATCH 5/6] Add in secondary range, but not working due to wrong network. --- terraform/modules/networking/networking.tf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/terraform/modules/networking/networking.tf b/terraform/modules/networking/networking.tf index a4b33a7..0cbd570 100644 --- a/terraform/modules/networking/networking.tf +++ b/terraform/modules/networking/networking.tf @@ -34,10 +34,16 @@ resource "google_compute_network" "private_network" { } resource "google_compute_subnetwork" "subnet" { + project = google_compute_network.private_network.project name = "private-subnetwork" ip_cidr_range = "10.100.0.0/16" region = var.region network = google_compute_network.private_network.id + + secondary_ip_range { + range_name = "private-subnetwork-secondary" + ip_cidr_range = "192.168.10.0/24" + } } resource "google_compute_global_address" "private_ip_address" { From 40f4a8c3983ee427ddf3b87699e403334467c66d Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Tue, 13 Jul 2021 20:50:43 -0400 Subject: [PATCH 6/6] Fix ip exhaustion by increasing services ip address range. --- terraform/basic_gke/main.tf | 10 +++++--- terraform/modules/cloudsql/cloudsql.tf | 14 +++++++---- terraform/modules/gke/gke.tf | 28 ++++------------------ terraform/modules/networking/networking.tf | 7 +----- 4 files changed, 22 insertions(+), 37 deletions(-) diff --git a/terraform/basic_gke/main.tf b/terraform/basic_gke/main.tf index eb36c27..77aee44 100644 --- a/terraform/basic_gke/main.tf +++ b/terraform/basic_gke/main.tf @@ -81,9 +81,9 @@ module "gke" { private_subnetwork_id = module.networking.private_subnetwork_id service_cloudkms = google_project_service.cloudkms - # depends_on = [ - # module.networking - # ] + depends_on = [ + module.networking + ] } output "gke_connect_command" { @@ -128,3 +128,7 @@ output "redis_port" { } + + + + diff --git a/terraform/modules/cloudsql/cloudsql.tf b/terraform/modules/cloudsql/cloudsql.tf index 44b1d3f..b9c9520 100644 --- a/terraform/modules/cloudsql/cloudsql.tf +++ b/terraform/modules/cloudsql/cloudsql.tf @@ -25,11 +25,14 @@ variable "private_network_id" { type = string } -resource "google_sql_database_instance" "instance" { - project = var.project - region = var.region - name = "my-database-instance" +resource "random_id" "cloudsql" { + byte_length = 4 +} +resource "google_sql_database_instance" "instance" { + project = var.project + region = var.region + name = "my-database-instance-${random_id.cloudsql.hex}" database_version = var.db_version settings { @@ -41,5 +44,6 @@ resource "google_sql_database_instance" "instance" { } } - deletion_protection = "true" + deletion_protection = "false" + # deletion_protection = "true" } diff --git a/terraform/modules/gke/gke.tf b/terraform/modules/gke/gke.tf index 79a8980..7e5e9aa 100644 --- a/terraform/modules/gke/gke.tf +++ b/terraform/modules/gke/gke.tf @@ -43,7 +43,7 @@ resource "google_kms_key_ring" "gke_db" { location = var.region lifecycle { - prevent_destroy = true + #prevent_destroy = true } depends_on = [ @@ -65,7 +65,7 @@ resource "google_kms_crypto_key" "gke_db" { key_ring = google_kms_key_ring.gke_db.id lifecycle { - prevent_destroy = true + #prevent_destroy = true } depends_on = [ @@ -114,24 +114,6 @@ resource "google_storage_bucket_iam_member" "gke_gcr" { ] } -resource "google_compute_global_address" "gke_cluster_range" { - project = var.project - name = "gke-cluster-range" - purpose = "VPC_PEERING" - address_type = "INTERNAL" - prefix_length = 16 - network = var.private_network_id -} - -resource "google_compute_global_address" "gke_services_range" { - project = var.project - name = "gke-services-range" - purpose = "VPC_PEERING" - address_type = "INTERNAL" - prefix_length = 20 - network = var.private_network_id -} - resource "google_container_cluster" "primary" { project = var.project name = "gke-cluster" @@ -169,12 +151,12 @@ resource "google_container_cluster" "primary" { } ip_allocation_policy { - cluster_secondary_range_name = google_compute_global_address.gke_cluster_range.name - services_secondary_range_name = google_compute_global_address.gke_services_range.name + cluster_ipv4_cidr_block = "/16" + services_ipv4_cidr_block = "/20" } lifecycle { - prevent_destroy = true + #prevent_destroy = true } depends_on = [ diff --git a/terraform/modules/networking/networking.tf b/terraform/modules/networking/networking.tf index 0cbd570..97aee17 100644 --- a/terraform/modules/networking/networking.tf +++ b/terraform/modules/networking/networking.tf @@ -39,11 +39,6 @@ resource "google_compute_subnetwork" "subnet" { ip_cidr_range = "10.100.0.0/16" region = var.region network = google_compute_network.private_network.id - - secondary_ip_range { - range_name = "private-subnetwork-secondary" - ip_cidr_range = "192.168.10.0/24" - } } resource "google_compute_global_address" "private_ip_address" { @@ -51,7 +46,7 @@ resource "google_compute_global_address" "private_ip_address" { name = "private-ip-address" purpose = "VPC_PEERING" address_type = "INTERNAL" - prefix_length = 24 + prefix_length = 16 network = google_compute_network.private_network.id }