Merge branch 'cloudfunc'

terraform/basic_gke/main.tf

@@ -185,6 +185,38 @@ output "redis_port" {
   value       = module.redis.redis_port
 }
 
+#################### Cloudfunction to PubSub ##############
+
+resource "google_project_service" "cloudbuild" {
+  project                    = var.project
+  service                    = "cloudbuild.googleapis.com"
+  disable_dependent_services = true
+}
+
+resource "random_id" "cf_bucket_id" {
+  byte_length = 4
+}
+
+resource "google_storage_bucket" "bucket" {
+  project       = var.project
+  name          = "cloudfunc-${random_id.cf_bucket_id.hex}"
+  force_destroy = true
+}
+
+module "cf_to_pubsub" {
+  source             = "../modules/cf_to_pubsub"
+  project            = var.project
+  region             = var.region
+  topic_name         = "bigquery-etl"
+  source_bucket      = google_storage_bucket.bucket
+  service_cloudbuild = google_project_service.cloudbuild
+}
+
+output "log_to_bq_endpoint" {
+  description = "https endpoint to log to BigQuery."
+  value       = module.cf_to_pubsub.https_trigger_url
+}
+
 #################### PubSub to BigQuery ###################
 
 module "bigquery" {

terraform/modules/cf_to_pubsub/built/.gitignore

@@ -0,0 +1 @@
+*.zip

terraform/modules/cf_to_pubsub/built/README

@@ -0,0 +1 @@
+This folder will get populated with zip files of the source code for the cloud functions.

terraform/modules/cf_to_pubsub/cf_to_pubsub.tf

@@ -0,0 +1,87 @@
+# Example message:
+# curl -H "Content-Type: application/json" -d '{"time": "2021-07-20T05:05:47", "service": "foo", "log": "bar"}' -X POST 'https://us-central1-hip-wharf-319304.cloudfunctions.net/cf-to-pubsub'
+
+variable "project" {
+  description = "Project ID."
+  type        = string
+}
+
+variable "region" {
+  description = "Region."
+  type        = string
+}
+
+variable "topic_name" {
+  description = "The name of topic where the events should be published."
+  type        = string
+}
+
+variable "source_bucket" {
+  description = "Google storage bucket where the source code will be stored."
+}
+
+variable "function_source_name" {
+  description = "Name of the folder containing the source code for the function."
+  type        = string
+  default     = "cf_to_pubsub"
+}
+
+variable "service_cloudbuild" {
+  description = "THe cloudbuild google_project_service."
+}
+
+output "https_trigger_url" {
+  description = "https endpoint for the cloud function."
+  value       = google_cloudfunctions_function.function.https_trigger_url
+}
+
+resource "random_id" "cf_bucket_id" {
+  byte_length = 4
+}
+
+data "archive_file" "source_archive" {
+  type        = "zip"
+  source_dir  = "${path.module}/functions/${var.function_source_name}"
+  output_path = "${path.module}/built/${var.function_source_name}.zip"
+  excludes    = [".python-version"]
+}
+
+resource "google_storage_bucket_object" "remote_archive" {
+  name   = "${var.function_source_name}-${data.archive_file.source_archive.output_base64sha256}.zip"
+  bucket = var.source_bucket.name
+  source = data.archive_file.source_archive.output_path
+}
+
+resource "google_cloudfunctions_function" "function" {
+  name        = "cf-to-pubsub"
+  description = "CloudFunction to PubSub"
+  runtime     = "python39"
+
+  available_memory_mb   = 128
+  source_archive_bucket = var.source_bucket.name
+  source_archive_object = google_storage_bucket_object.remote_archive.name
+  trigger_http          = true
+  entry_point           = "main"
+  max_instances         = 4
+  ingress_settings      = "ALLOW_ALL"
+  # ingress_settings      = "ALLOW_INTERNAL_ONLY"
+
+  environment_variables = {
+    GCP_PROJECT = var.project
+    GCP_TOPIC   = var.topic_name
+  }
+
+  depends_on = [
+    var.service_cloudbuild
+  ]
+}
+
+# Allow unauthenticated access over http
+resource "google_cloudfunctions_function_iam_member" "invoker" {
+  project        = google_cloudfunctions_function.function.project
+  region         = google_cloudfunctions_function.function.region
+  cloud_function = google_cloudfunctions_function.function.name
+
+  role   = "roles/cloudfunctions.invoker"
+  member = "allUsers"
+}

terraform/modules/cf_to_pubsub/functions/cf_to_pubsub/main.py

@@ -0,0 +1,24 @@
+import json
+import os
+
+from google.cloud import pubsub_v1
+
+publisher = pubsub_v1.PublisherClient()
+topic = publisher.topic_path(os.environ.get("GCP_PROJECT"), os.environ.get("GCP_TOPIC"))
+
+
+def push_to_pubsub(request_params):
+    publisher.publish(topic, json.dumps(request_params).encode("utf-8")).result()
+
+
+def main(request):
+    request_json = request.get_json(silent=True)
+    request_args = request.args
+
+    if request_json:
+        push_to_pubsub(request_json)
+    elif request_args:
+        push_to_pubsub(request_args)
+    else:
+        raise Exception("No data provided.")
+    return {"status": "ok"}

terraform/modules/cf_to_pubsub/functions/cf_to_pubsub/requirements.txt

@@ -0,0 +1,2 @@
+Flask==1.1.2
+google-cloud-pubsub==2.6.1