From e3b6049f60ed41c4e751aca41b2c988e9ced0605 Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Tue, 20 Jul 2021 22:24:32 -0400
Subject: [PATCH 1/6] Initial setup of a cloud function.

---
 terraform/basic_gke/main.tf                   | 31 ++++++++
 .../modules/cf_to_pubsub/built/.gitignore     |  0
 terraform/modules/cf_to_pubsub/built/README   |  1 +
 .../modules/cf_to_pubsub/cf_to_pubsub.tf      | 71 +++++++++++++++++++
 .../functions/cf_to_pubsub/main.py            | 18 +++++
 .../functions/cf_to_pubsub/requirements.txt   |  1 +
 6 files changed, 122 insertions(+)
 create mode 100644 terraform/modules/cf_to_pubsub/built/.gitignore
 create mode 100644 terraform/modules/cf_to_pubsub/built/README
 create mode 100644 terraform/modules/cf_to_pubsub/cf_to_pubsub.tf
 create mode 100644 terraform/modules/cf_to_pubsub/functions/cf_to_pubsub/main.py
 create mode 100644 terraform/modules/cf_to_pubsub/functions/cf_to_pubsub/requirements.txt

diff --git a/terraform/basic_gke/main.tf b/terraform/basic_gke/main.tf
index 477a9b9..37988c7 100644
--- a/terraform/basic_gke/main.tf
+++ b/terraform/basic_gke/main.tf
@@ -185,6 +185,37 @@ output "redis_port" {
   value       = module.redis.redis_port
 }
 
+#################### Cloudfunction to PubSub ##############
+
+resource "google_project_service" "cloudbuild" {
+  project                    = var.project
+  service                    = "cloudbuild.googleapis.com"
+  disable_dependent_services = true
+}
+
+resource "random_id" "cf_bucket_id" {
+  byte_length = 4
+}
+
+resource "google_storage_bucket" "bucket" {
+  project       = var.project
+  name          = "cloudfunc-${random_id.cf_bucket_id.hex}"
+  force_destroy = true
+}
+
+module "cf_to_pubsub" {
+  source             = "../modules/cf_to_pubsub"
+  project            = var.project
+  region             = var.region
+  source_bucket      = google_storage_bucket.bucket
+  service_cloudbuild = google_project_service.cloudbuild
+}
+
+output "log_to_bq_endpoint" {
+  description = "https endpoint to log to BigQuery."
+  value       = module.cf_to_pubsub.https_trigger_url
+}
+
 #################### PubSub to BigQuery ###################
 
 module "bigquery" {
diff --git a/terraform/modules/cf_to_pubsub/built/.gitignore b/terraform/modules/cf_to_pubsub/built/.gitignore
new file mode 100644
index 0000000..e69de29
diff --git a/terraform/modules/cf_to_pubsub/built/README b/terraform/modules/cf_to_pubsub/built/README
new file mode 100644
index 0000000..ebcf151
--- /dev/null
+++ b/terraform/modules/cf_to_pubsub/built/README
@@ -0,0 +1 @@
+This folder will get populated with zip files of the source code for the cloud functions.
diff --git a/terraform/modules/cf_to_pubsub/cf_to_pubsub.tf b/terraform/modules/cf_to_pubsub/cf_to_pubsub.tf
new file mode 100644
index 0000000..8cfe250
--- /dev/null
+++ b/terraform/modules/cf_to_pubsub/cf_to_pubsub.tf
@@ -0,0 +1,71 @@
+variable "project" {
+  description = "Project ID."
+  type        = string
+}
+
+variable "region" {
+  description = "Region."
+  type        = string
+}
+
+variable "source_bucket" {
+  description = "Google storage bucket where the source code will be stored."
+}
+
+variable "function_source_name" {
+  description = "Name of the folder containing the source code for the function."
+  type        = string
+  default     = "cf_to_pubsub"
+}
+
+variable "service_cloudbuild" {
+  description = "THe cloudbuild google_project_service."
+}
+
+output "https_trigger_url" {
+  description = "https endpoint for the cloud function."
+  value       = google_cloudfunctions_function.function.https_trigger_url
+}
+
+resource "random_id" "cf_bucket_id" {
+  byte_length = 4
+}
+
+data "archive_file" "source_archive" {
+  type        = "zip"
+  source_dir  = "${path.module}/functions/${var.function_source_name}"
+  output_path = "${path.module}/built/${var.function_source_name}.zip"
+  excludes    = [".python-version"]
+}
+
+resource "google_storage_bucket_object" "remote_archive" {
+  name   = "${var.function_source_name}-${data.archive_file.source_archive.output_base64sha256}.zip"
+  bucket = var.source_bucket.name
+  source = data.archive_file.source_archive.output_path
+}
+
+resource "google_cloudfunctions_function" "function" {
+  name        = "cf-to-pubsub"
+  description = "CloudFunction to PubSub"
+  runtime     = "python39"
+
+  available_memory_mb   = 128
+  source_archive_bucket = var.source_bucket.name
+  source_archive_object = google_storage_bucket_object.remote_archive.name
+  trigger_http          = true
+  entry_point           = "main"
+
+  depends_on = [
+    var.service_cloudbuild
+  ]
+}
+
+# Allow unauthenticated access over http
+resource "google_cloudfunctions_function_iam_member" "invoker" {
+  project        = google_cloudfunctions_function.function.project
+  region         = google_cloudfunctions_function.function.region
+  cloud_function = google_cloudfunctions_function.function.name
+
+  role   = "roles/cloudfunctions.invoker"
+  member = "allUsers"
+}
diff --git a/terraform/modules/cf_to_pubsub/functions/cf_to_pubsub/main.py b/terraform/modules/cf_to_pubsub/functions/cf_to_pubsub/main.py
new file mode 100644
index 0000000..fe14ebd
--- /dev/null
+++ b/terraform/modules/cf_to_pubsub/functions/cf_to_pubsub/main.py
@@ -0,0 +1,18 @@
+import json
+
+
+def push_to_pubsub(request_params):
+    print(json.dumps(request_params))
+
+
+def main(request):
+    request_json = request.get_json(silent=True)
+    request_args = request.args
+
+    if request_json:
+        push_to_pubsub(request_json)
+    elif request_args:
+        push_to_pubsub(request_args)
+    else:
+        raise Exception("No data provided.")
+    return {"status": "ok"}
diff --git a/terraform/modules/cf_to_pubsub/functions/cf_to_pubsub/requirements.txt b/terraform/modules/cf_to_pubsub/functions/cf_to_pubsub/requirements.txt
new file mode 100644
index 0000000..46a48dd
--- /dev/null
+++ b/terraform/modules/cf_to_pubsub/functions/cf_to_pubsub/requirements.txt
@@ -0,0 +1 @@
+Flask==1.1.2

From 7fa8ffee884a68034cbd2a682affe005a626be0e Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Tue, 20 Jul 2021 22:31:20 -0400
Subject: [PATCH 2/6] Allow external connections.

---
 terraform/modules/cf_to_pubsub/cf_to_pubsub.tf | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/terraform/modules/cf_to_pubsub/cf_to_pubsub.tf b/terraform/modules/cf_to_pubsub/cf_to_pubsub.tf
index 8cfe250..a46baef 100644
--- a/terraform/modules/cf_to_pubsub/cf_to_pubsub.tf
+++ b/terraform/modules/cf_to_pubsub/cf_to_pubsub.tf
@@ -54,6 +54,8 @@ resource "google_cloudfunctions_function" "function" {
   source_archive_object = google_storage_bucket_object.remote_archive.name
   trigger_http          = true
   entry_point           = "main"
+  ingress_settings      = "ALLOW_ALL"
+  # ingress_settings      = "ALLOW_INTERNAL_ONLY"
 
   depends_on = [
     var.service_cloudbuild

From 75f760695b142bce2e1c7eb002bf9e15f9421fb7 Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Tue, 20 Jul 2021 22:34:07 -0400
Subject: [PATCH 3/6] Ignore built zips.

---
 terraform/modules/cf_to_pubsub/built/.gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/terraform/modules/cf_to_pubsub/built/.gitignore b/terraform/modules/cf_to_pubsub/built/.gitignore
index e69de29..c4c4ffc 100644
--- a/terraform/modules/cf_to_pubsub/built/.gitignore
+++ b/terraform/modules/cf_to_pubsub/built/.gitignore
@@ -0,0 +1 @@
+*.zip

From d90b267848ae4caa6d92506e59e170d18b15f343 Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Tue, 20 Jul 2021 22:38:16 -0400
Subject: [PATCH 4/6] Add example curl.

---
 terraform/modules/cf_to_pubsub/cf_to_pubsub.tf | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/terraform/modules/cf_to_pubsub/cf_to_pubsub.tf b/terraform/modules/cf_to_pubsub/cf_to_pubsub.tf
index a46baef..d957e26 100644
--- a/terraform/modules/cf_to_pubsub/cf_to_pubsub.tf
+++ b/terraform/modules/cf_to_pubsub/cf_to_pubsub.tf
@@ -1,3 +1,6 @@
+# Example message:
+# curl -H "Content-Type: application/json" -d '{"time": "2021-07-20T05:05:47", "service": "foo", "log": "bar"}' -X POST 'https://us-central1-hip-wharf-319304.cloudfunctions.net/cf-to-pubsub'
+
 variable "project" {
   description = "Project ID."
   type        = string

From f457cf2a6c496535c0de8b7c66fda34889fce94f Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Tue, 20 Jul 2021 23:03:01 -0400
Subject: [PATCH 5/6] Put on a limit of max instances.

---
 terraform/modules/cf_to_pubsub/cf_to_pubsub.tf | 1 +
 1 file changed, 1 insertion(+)

diff --git a/terraform/modules/cf_to_pubsub/cf_to_pubsub.tf b/terraform/modules/cf_to_pubsub/cf_to_pubsub.tf
index d957e26..411e297 100644
--- a/terraform/modules/cf_to_pubsub/cf_to_pubsub.tf
+++ b/terraform/modules/cf_to_pubsub/cf_to_pubsub.tf
@@ -57,6 +57,7 @@ resource "google_cloudfunctions_function" "function" {
   source_archive_object = google_storage_bucket_object.remote_archive.name
   trigger_http          = true
   entry_point           = "main"
+  max_instances         = 4
   ingress_settings      = "ALLOW_ALL"
   # ingress_settings      = "ALLOW_INTERNAL_ONLY"
 

From 7bf8370650bb05d92c28c1ef0e533e1e52aa8ac6 Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Tue, 20 Jul 2021 23:26:14 -0400
Subject: [PATCH 6/6] Push events to pubsub.

---
 terraform/basic_gke/main.tf                            |  1 +
 terraform/modules/cf_to_pubsub/cf_to_pubsub.tf         | 10 ++++++++++
 .../cf_to_pubsub/functions/cf_to_pubsub/main.py        |  8 +++++++-
 .../functions/cf_to_pubsub/requirements.txt            |  1 +
 4 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/terraform/basic_gke/main.tf b/terraform/basic_gke/main.tf
index 37988c7..80f8624 100644
--- a/terraform/basic_gke/main.tf
+++ b/terraform/basic_gke/main.tf
@@ -207,6 +207,7 @@ module "cf_to_pubsub" {
   source             = "../modules/cf_to_pubsub"
   project            = var.project
   region             = var.region
+  topic_name         = "bigquery-etl"
   source_bucket      = google_storage_bucket.bucket
   service_cloudbuild = google_project_service.cloudbuild
 }
diff --git a/terraform/modules/cf_to_pubsub/cf_to_pubsub.tf b/terraform/modules/cf_to_pubsub/cf_to_pubsub.tf
index 411e297..09d3305 100644
--- a/terraform/modules/cf_to_pubsub/cf_to_pubsub.tf
+++ b/terraform/modules/cf_to_pubsub/cf_to_pubsub.tf
@@ -11,6 +11,11 @@ variable "region" {
   type        = string
 }
 
+variable "topic_name" {
+  description = "The name of topic where the events should be published."
+  type        = string
+}
+
 variable "source_bucket" {
   description = "Google storage bucket where the source code will be stored."
 }
@@ -61,6 +66,11 @@ resource "google_cloudfunctions_function" "function" {
   ingress_settings      = "ALLOW_ALL"
   # ingress_settings      = "ALLOW_INTERNAL_ONLY"
 
+  environment_variables = {
+    GCP_PROJECT = var.project
+    GCP_TOPIC   = var.topic_name
+  }
+
   depends_on = [
     var.service_cloudbuild
   ]
diff --git a/terraform/modules/cf_to_pubsub/functions/cf_to_pubsub/main.py b/terraform/modules/cf_to_pubsub/functions/cf_to_pubsub/main.py
index fe14ebd..14e321f 100644
--- a/terraform/modules/cf_to_pubsub/functions/cf_to_pubsub/main.py
+++ b/terraform/modules/cf_to_pubsub/functions/cf_to_pubsub/main.py
@@ -1,8 +1,14 @@
 import json
+import os
+
+from google.cloud import pubsub_v1
+
+publisher = pubsub_v1.PublisherClient()
+topic = publisher.topic_path(os.environ.get("GCP_PROJECT"), os.environ.get("GCP_TOPIC"))
 
 
 def push_to_pubsub(request_params):
-    print(json.dumps(request_params))
+    publisher.publish(topic, json.dumps(request_params).encode("utf-8")).result()
 
 
 def main(request):
diff --git a/terraform/modules/cf_to_pubsub/functions/cf_to_pubsub/requirements.txt b/terraform/modules/cf_to_pubsub/functions/cf_to_pubsub/requirements.txt
index 46a48dd..fd39a8c 100644
--- a/terraform/modules/cf_to_pubsub/functions/cf_to_pubsub/requirements.txt
+++ b/terraform/modules/cf_to_pubsub/functions/cf_to_pubsub/requirements.txt
@@ -1 +1,2 @@
 Flask==1.1.2
+google-cloud-pubsub==2.6.1