From 879ea6cc6086c81f6ab081369460f206f9db6822 Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Tue, 20 Jul 2021 00:04:07 -0400
Subject: [PATCH] Add pubsub topic and subscription.

---
 terraform/basic_gke/main.tf            | 10 +++
 terraform/modules/bigquery/bigquery.tf | 84 ++++++++++++++++++++++++++
 2 files changed, 94 insertions(+)
 create mode 100644 terraform/modules/bigquery/bigquery.tf

diff --git a/terraform/basic_gke/main.tf b/terraform/basic_gke/main.tf
index 3606993..477a9b9 100644
--- a/terraform/basic_gke/main.tf
+++ b/terraform/basic_gke/main.tf
@@ -184,3 +184,13 @@ output "redis_port" {
   description = "Port for redis database."
   value       = module.redis.redis_port
 }
+
+#################### PubSub to BigQuery ###################
+
+module "bigquery" {
+  source           = "../modules/bigquery"
+  project          = var.project
+  region           = var.region
+  service_cloudkms = google_project_service.cloudkms
+}
+
diff --git a/terraform/modules/bigquery/bigquery.tf b/terraform/modules/bigquery/bigquery.tf
new file mode 100644
index 0000000..9b9b2e4
--- /dev/null
+++ b/terraform/modules/bigquery/bigquery.tf
@@ -0,0 +1,84 @@
+variable "project" {
+  description = "Project ID."
+  type        = string
+}
+
+variable "region" {
+  description = "Region."
+  type        = string
+}
+
+variable "service_cloudkms" {
+  description = "cloudkms service."
+}
+
+data "google_project" "project" {
+  project_id = var.project
+}
+
+#################### IAM ##################################
+
+resource "google_project_iam_binding" "pubsub_kms" {
+  project = var.project
+  role    = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
+
+  members = [
+    "serviceAccount:service-${data.google_project.project.number}@gcp-sa-pubsub.iam.gserviceaccount.com"
+  ]
+}
+
+#################### KMS ##################################
+
+resource "random_id" "bigquery_etl_keyring" {
+  byte_length = 4
+}
+
+resource "google_kms_crypto_key" "bigquery_etl_key" {
+  name     = "bigquery-etl-key"
+  key_ring = google_kms_key_ring.bigquery_etl_keyring.id
+}
+
+resource "google_kms_key_ring" "bigquery_etl_keyring" {
+  project  = var.project
+  name     = "bigquery-etl-keyring-${random_id.bigquery_etl_keyring.hex}"
+  location = var.region
+
+  lifecycle {
+    #prevent_destroy = true
+  }
+
+  depends_on = [
+    var.service_cloudkms
+  ]
+}
+
+#################### PubSub ###############################
+
+resource "google_pubsub_topic" "bigquery_etl" {
+  project      = var.project
+  name         = "bigquery-etl"
+  kms_key_name = google_kms_crypto_key.bigquery_etl_key.id
+  depends_on = [
+    google_project_iam_binding.pubsub_kms
+  ]
+}
+
+resource "google_pubsub_subscription" "bigquery_etl" {
+  project = var.project
+  name    = "bigquery-etl-sub"
+  topic   = google_pubsub_topic.bigquery_etl.name
+
+  # 20 minutes
+  message_retention_duration = "1200s"
+  retain_acked_messages      = false
+  ack_deadline_seconds       = 20
+
+  expiration_policy {
+    ttl = ""
+  }
+
+  retry_policy {
+    maximum_backoff = "600s"
+    minimum_backoff = "10s"
+  }
+}