From 5241df2d9477a52d5dd6f0d2f0659cf837745b7e Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Tue, 20 Jul 2021 01:12:01 -0400
Subject: [PATCH] Add bigquery table.

---
 terraform/basic_gke/README.md          |  2 +
 terraform/modules/bigquery/bigquery.tf | 53 +++++++++++++++++++++++++-
 2 files changed, 53 insertions(+), 2 deletions(-)

diff --git a/terraform/basic_gke/README.md b/terraform/basic_gke/README.md
index ac3a929..7469a3b 100644
--- a/terraform/basic_gke/README.md
+++ b/terraform/basic_gke/README.md
@@ -6,3 +6,5 @@ A small terraform script to launch GKE with:
 - GCR for custom docker images
 - Auto-scaling node pool
 - K8s database encryption
+
+Contains some settings that are inadvisable in production (for example, disabling deletion protection). I include them because this project's goal is to provide a template for quickly spinning up a test/dev environment which may involve frequent deletions.
diff --git a/terraform/modules/bigquery/bigquery.tf b/terraform/modules/bigquery/bigquery.tf
index 6374899..caf0ab9 100644
--- a/terraform/modules/bigquery/bigquery.tf
+++ b/terraform/modules/bigquery/bigquery.tf
@@ -1,3 +1,6 @@
+# Example message:
+# {"time": "2021-07-20T05:05:47", "service": "foo", "log": "bar"}
+
 variable "project" {
   description = "Project ID."
   type        = string
@@ -116,10 +119,56 @@ resource "google_dataflow_job" "etl_job" {
 
   parameters = {
     inputSubscription = google_pubsub_subscription.bigquery_etl.id
-    outputTableSpec   = "your-project:your-dataset.your-table-name"
+    outputTableSpec   = "${google_bigquery_table.pubsub_etl.project}:${google_bigquery_table.pubsub_etl.dataset_id}.${google_bigquery_table.pubsub_etl.table_id}"
   }
 
   additional_experiments = [
-    "enable_streaming_engine"
+    "enable_streaming_engine",
+    "enable_windmill_service"
   ]
 }
+
+#################### BigQuery #############################
+
+resource "google_bigquery_dataset" "pubsub_etl" {
+  project       = var.project
+  dataset_id    = "pubsub_etl"
+  friendly_name = "PubSub ETL"
+  description   = "Dataset where PubSub ETL data goes."
+  location      = "US"
+}
+
+resource "google_bigquery_table" "pubsub_etl" {
+  project    = var.project
+  dataset_id = google_bigquery_dataset.pubsub_etl.dataset_id
+  table_id   = "pubsub_etl"
+
+  deletion_protection = false
+
+  time_partitioning {
+    type = "DAY"
+  }
+
+  schema = <<EOF
+[
+  {
+    "description": "Time of the event",
+    "mode": "NULLABLE",
+    "name": "time",
+    "type": "DATETIME"
+  },
+  {
+    "description": "Name of the service",
+    "mode": "NULLABLE",
+    "name": "service",
+    "type": "STRING"
+  },
+  {
+    "description": "Text of the log",
+    "mode": "NULLABLE",
+    "name": "log",
+    "type": "STRING"
+  }
+]
+EOF
+}