diff --git a/terraform/basic_gke/main.tf b/terraform/basic_gke/main.tf index 477a9b9..80f8624 100644 --- a/terraform/basic_gke/main.tf +++ b/terraform/basic_gke/main.tf @@ -185,6 +185,38 @@ output "redis_port" { value = module.redis.redis_port } +#################### Cloudfunction to PubSub ############## + +resource "google_project_service" "cloudbuild" { + project = var.project + service = "cloudbuild.googleapis.com" + disable_dependent_services = true +} + +resource "random_id" "cf_bucket_id" { + byte_length = 4 +} + +resource "google_storage_bucket" "bucket" { + project = var.project + name = "cloudfunc-${random_id.cf_bucket_id.hex}" + force_destroy = true +} + +module "cf_to_pubsub" { + source = "../modules/cf_to_pubsub" + project = var.project + region = var.region + topic_name = "bigquery-etl" + source_bucket = google_storage_bucket.bucket + service_cloudbuild = google_project_service.cloudbuild +} + +output "log_to_bq_endpoint" { + description = "https endpoint to log to BigQuery." + value = module.cf_to_pubsub.https_trigger_url +} + #################### PubSub to BigQuery ################### module "bigquery" { diff --git a/terraform/modules/cf_to_pubsub/built/.gitignore b/terraform/modules/cf_to_pubsub/built/.gitignore new file mode 100644 index 0000000..c4c4ffc --- /dev/null +++ b/terraform/modules/cf_to_pubsub/built/.gitignore @@ -0,0 +1 @@ +*.zip diff --git a/terraform/modules/cf_to_pubsub/built/README b/terraform/modules/cf_to_pubsub/built/README new file mode 100644 index 0000000..ebcf151 --- /dev/null +++ b/terraform/modules/cf_to_pubsub/built/README @@ -0,0 +1 @@ +This folder will get populated with zip files of the source code for the cloud functions. diff --git a/terraform/modules/cf_to_pubsub/cf_to_pubsub.tf b/terraform/modules/cf_to_pubsub/cf_to_pubsub.tf new file mode 100644 index 0000000..09d3305 --- /dev/null +++ b/terraform/modules/cf_to_pubsub/cf_to_pubsub.tf @@ -0,0 +1,87 @@ +# Example message: +# curl -H "Content-Type: application/json" -d '{"time": "2021-07-20T05:05:47", "service": "foo", "log": "bar"}' -X POST 'https://us-central1-hip-wharf-319304.cloudfunctions.net/cf-to-pubsub' + +variable "project" { + description = "Project ID." + type = string +} + +variable "region" { + description = "Region." + type = string +} + +variable "topic_name" { + description = "The name of topic where the events should be published." + type = string +} + +variable "source_bucket" { + description = "Google storage bucket where the source code will be stored." +} + +variable "function_source_name" { + description = "Name of the folder containing the source code for the function." + type = string + default = "cf_to_pubsub" +} + +variable "service_cloudbuild" { + description = "THe cloudbuild google_project_service." +} + +output "https_trigger_url" { + description = "https endpoint for the cloud function." + value = google_cloudfunctions_function.function.https_trigger_url +} + +resource "random_id" "cf_bucket_id" { + byte_length = 4 +} + +data "archive_file" "source_archive" { + type = "zip" + source_dir = "${path.module}/functions/${var.function_source_name}" + output_path = "${path.module}/built/${var.function_source_name}.zip" + excludes = [".python-version"] +} + +resource "google_storage_bucket_object" "remote_archive" { + name = "${var.function_source_name}-${data.archive_file.source_archive.output_base64sha256}.zip" + bucket = var.source_bucket.name + source = data.archive_file.source_archive.output_path +} + +resource "google_cloudfunctions_function" "function" { + name = "cf-to-pubsub" + description = "CloudFunction to PubSub" + runtime = "python39" + + available_memory_mb = 128 + source_archive_bucket = var.source_bucket.name + source_archive_object = google_storage_bucket_object.remote_archive.name + trigger_http = true + entry_point = "main" + max_instances = 4 + ingress_settings = "ALLOW_ALL" + # ingress_settings = "ALLOW_INTERNAL_ONLY" + + environment_variables = { + GCP_PROJECT = var.project + GCP_TOPIC = var.topic_name + } + + depends_on = [ + var.service_cloudbuild + ] +} + +# Allow unauthenticated access over http +resource "google_cloudfunctions_function_iam_member" "invoker" { + project = google_cloudfunctions_function.function.project + region = google_cloudfunctions_function.function.region + cloud_function = google_cloudfunctions_function.function.name + + role = "roles/cloudfunctions.invoker" + member = "allUsers" +} diff --git a/terraform/modules/cf_to_pubsub/functions/cf_to_pubsub/main.py b/terraform/modules/cf_to_pubsub/functions/cf_to_pubsub/main.py new file mode 100644 index 0000000..14e321f --- /dev/null +++ b/terraform/modules/cf_to_pubsub/functions/cf_to_pubsub/main.py @@ -0,0 +1,24 @@ +import json +import os + +from google.cloud import pubsub_v1 + +publisher = pubsub_v1.PublisherClient() +topic = publisher.topic_path(os.environ.get("GCP_PROJECT"), os.environ.get("GCP_TOPIC")) + + +def push_to_pubsub(request_params): + publisher.publish(topic, json.dumps(request_params).encode("utf-8")).result() + + +def main(request): + request_json = request.get_json(silent=True) + request_args = request.args + + if request_json: + push_to_pubsub(request_json) + elif request_args: + push_to_pubsub(request_args) + else: + raise Exception("No data provided.") + return {"status": "ok"} diff --git a/terraform/modules/cf_to_pubsub/functions/cf_to_pubsub/requirements.txt b/terraform/modules/cf_to_pubsub/functions/cf_to_pubsub/requirements.txt new file mode 100644 index 0000000..fd39a8c --- /dev/null +++ b/terraform/modules/cf_to_pubsub/functions/cf_to_pubsub/requirements.txt @@ -0,0 +1,2 @@ +Flask==1.1.2 +google-cloud-pubsub==2.6.1