kubernetes/autoscale/k8s/autoscale.yaml

@@ -1,70 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: autoscale-test
-  namespace: default
-spec:
-  selector:
-    matchLabels:
-      app: autoscale-test
-  template:
-    metadata:
-      labels:
-        app: autoscale-test
-    spec:
-      containers:
-        - name: echoservice
-          image: gcr.io/hip-wharf-319304/echoservice:latest
-          imagePullPolicy: Always
-          env:
-            - name: "WORKERS"
-              value: "8"
-          ports:
-            - name: http
-              containerPort: 8080
-              protocol: TCP
-          livenessProbe:
-            httpGet:
-              path: /
-              port: http
-          readinessProbe:
-            httpGet:
-              path: /
-              port: http
-          resources:
-            requests:
-              cpu: "250m"
-              memory: "256Mi"
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: autoscale-test
-  namespace: default
-spec:
-  selector:
-    app: autoscale-test
-  type: LoadBalancer
-  ports:
-    - port: 80
-      targetPort: "http"
----
-apiVersion: autoscaling/v2beta2
-kind: HorizontalPodAutoscaler
-metadata:
-  name: autoscale-test
-  namespace: default
-spec:
-  scaleTargetRef:
-    apiVersion: apps/v1
-    kind: Deployment
-    name: autoscale-test
-  minReplicas: 1
-  maxReplicas: 10
-  metrics:
-    - type: Resource
-      resource:
-        name: cpu
-        target:
-          type: Utilization
-          averageUtilization: 50

kubernetes/autoscale/service/Dockerfile

@@ -1,17 +0,0 @@
-FROM python:3.9
-
-RUN useradd -m -g nogroup server
-USER server
-
-WORKDIR /home/server
-ENV FLASK_APP=main.py
-
-# Perform the install of requirements first to avoid re-installing on every code change
-COPY requirements.txt /home/server/
-RUN pip install -r /home/server/requirements.txt
-
-COPY *.py /home/server/
-
-# CMD ["python", "/home/server/main.py"]
-
-CMD ["/home/server/.local/bin/gunicorn", "--config", "/home/server/gunicorn.conf.py", "main:app"]

kubernetes/autoscale/service/Makefile

@@ -1,22 +0,0 @@
-IMAGE_NAME:=echoservice
-GCP_PROJECT:=hip-wharf-319304
-
-.PHONY: all
-all: build push
-
-.PHONY: build
-build:
-	docker build -t $(IMAGE_NAME) .
-
-.PHONY: push
-push:
-	docker tag $(IMAGE_NAME) gcr.io/$(GCP_PROJECT)/$(IMAGE_NAME)
-	docker push gcr.io/$(GCP_PROJECT)/$(IMAGE_NAME)
-
-.PHONY: clean
-clean:
-	docker rmi $(IMAGE_NAME) gcr.io/$(GCP_PROJECT)/$(IMAGE_NAME) $(IMAGE_NAME)
-
-.PHONY: run
-run:
-	docker run --rm -i -t -p "8080:8080" $(IMAGE_NAME)

kubernetes/autoscale/service/gunicorn.conf.py

@@ -1,9 +0,0 @@
-import os
-
-env_port = int(os.environ.get("PORT", 8080))
-
-bind = f"0.0.0.0:{env_port}"
-workers = int(os.environ.get("WORKERS", 8))
-preload_app = True
-max_requests = 100
-max_requests_jitter = 50

kubernetes/autoscale/service/main.py

@@ -1,28 +0,0 @@
-import json
-import os
-
-from flask import Flask, request
-
-app = Flask(__name__)
-
-
-def get_request_params():
-    request_json = request.get_json(silent=True)
-    request_args = request.args
-    if request_json:
-        return request_json
-    elif request_args:
-        return request_args
-    else:
-        return None
-
-
-@app.route("/", methods=("GET", "POST"))
-def index():
-    return json.dumps(
-        {"headers": {k: v for k, v in request.headers}, "params": get_request_params()}
-    )
-
-
-if __name__ == "__main__":
-    app.run(host="0.0.0.0", port=int(os.environ.get("PORT", 8080)), debug=True)

kubernetes/autoscale/service/requirements.txt

@@ -1,2 +0,0 @@
-flask==1.1.2
-gunicorn==20.1.0

terraform/basic_gke/main.tf

@@ -204,46 +204,19 @@ resource "google_storage_bucket" "bucket" {
 }
 
 module "cf_to_pubsub" {
-  source               = "../modules/cf_to_pubsub"
+  source             = "../modules/cf_to_pubsub"
-  project              = var.project
+  project            = var.project
-  region               = var.region
+  region             = var.region
-  function_name        = "cf-to-pubsub"
+  topic_name         = "bigquery-etl"
-  function_description = "CloudFunction to PubSub"
+  source_bucket      = google_storage_bucket.bucket
-  function_source_name = "cf_to_pubsub"
+  service_cloudbuild = google_project_service.cloudbuild
-  source_bucket        = google_storage_bucket.bucket
-  service_cloudbuild   = google_project_service.cloudbuild
-
-  environment_variables = {
-    GCP_PROJECT = var.project
-    GCP_TOPIC   = "bigquery-etl"
-  }
 }
 
-output "cf_to_pubsub_endpoint" {
+output "log_to_bq_endpoint" {
-  description = "https endpoint to log to BigQuery through pubsub."
+  description = "https endpoint to log to BigQuery."
   value       = module.cf_to_pubsub.https_trigger_url
 }
 
-module "cf_to_bq" {
-  source               = "../modules/cf_to_pubsub"
-  project              = var.project
-  region               = var.region
-  function_name        = "cf-to-bq"
-  function_description = "CloudFunction to BigQuery"
-  function_source_name = "cf_to_bq"
-  source_bucket        = google_storage_bucket.bucket
-  service_cloudbuild   = google_project_service.cloudbuild
-
-  environment_variables = {
-    BQ_TABLE = "${var.project}.pubsub_etl.pubsub_etl"
-  }
-}
-
-output "cf_to_bq_endpoint" {
-  description = "https endpoint to log to BigQuery directly."
-  value       = module.cf_to_bq.https_trigger_url
-}
-
 #################### PubSub to BigQuery ###################
 
 module "bigquery" {

terraform/modules/bigquery/bigquery.tf

@@ -99,13 +99,9 @@ resource "google_project_service" "dataflow" {
   disable_dependent_services = true
 }
 
-resource "random_id" "temp_storage" {
-  byte_length = 4
-}
-
 resource "google_storage_bucket" "temp_storage" {
   project       = var.project
-  name          = "${var.project}-etl-temp-${random_id.temp_storage.hex}"
+  name          = "${var.project}-bigquery-etl"
   force_destroy = true
 }
 

terraform/modules/cf_to_pubsub/cf_to_pubsub.tf

@@ -11,22 +11,9 @@ variable "region" {
   type        = string
 }
 
-variable "function_name" {
+variable "topic_name" {
-  description = "Name for the cloud function. If unspecified, one will be generated."
+  description = "The name of topic where the events should be published."
   type        = string
-  default     = ""
-}
-
-variable "function_description" {
-  description = "Description for the cloud function."
-  type        = string
-  default     = ""
-}
-
-variable "environment_variables" {
-  description = "Environment variables for the execution of the cloud function."
-  type        = map(any)
-  default     = {}
 }
 
 variable "source_bucket" {
@@ -36,16 +23,11 @@ variable "source_bucket" {
 variable "function_source_name" {
   description = "Name of the folder containing the source code for the function."
   type        = string
+  default     = "cf_to_pubsub"
 }
 
 variable "service_cloudbuild" {
-  description = "The cloudbuild google_project_service."
+  description = "THe cloudbuild google_project_service."
-}
-
-variable "allow_external" {
-  description = "Whether or not to allow outside traffic ingress."
-  type        = bool
-  default     = true
 }
 
 output "https_trigger_url" {
@@ -53,14 +35,6 @@ output "https_trigger_url" {
   value       = google_cloudfunctions_function.function.https_trigger_url
 }
 
-locals {
-  function_name = var.function_name == "" ? "cf-${random_id.function_id.hex}" : var.function_name
-}
-
-resource "random_id" "function_id" {
-  byte_length = 4
-}
-
 resource "random_id" "cf_bucket_id" {
   byte_length = 4
 }
@@ -79,8 +53,8 @@ resource "google_storage_bucket_object" "remote_archive" {
 }
 
 resource "google_cloudfunctions_function" "function" {
-  name        = local.function_name
+  name        = "cf-to-pubsub"
-  description = var.function_description
+  description = "CloudFunction to PubSub"
   runtime     = "python39"
 
   available_memory_mb   = 128
@@ -89,9 +63,13 @@ resource "google_cloudfunctions_function" "function" {
   trigger_http          = true
   entry_point           = "main"
   max_instances         = 4
-  ingress_settings      = var.allow_external ? "ALLOW_ALL" : "ALLOW_INTERNAL_ONLY"
+  ingress_settings      = "ALLOW_ALL"
+  # ingress_settings      = "ALLOW_INTERNAL_ONLY"
 
-  environment_variables = var.environment_variables
+  environment_variables = {
+    GCP_PROJECT = var.project
+    GCP_TOPIC   = var.topic_name
+  }
 
   depends_on = [
     var.service_cloudbuild

terraform/modules/cf_to_pubsub/functions/cf_to_bq/main.py

@@ -1,25 +0,0 @@
-import json
-import os
-
-from google.cloud import bigquery
-
-client = bigquery.Client()
-
-
-def push_to_pubsub(request_params):
-    errors = client.insert_rows_json(os.environ["BQ_TABLE"], [request_params])
-    if errors != []:
-        raise Exception("Encountered errors while inserting rows: {}".format(errors))
-
-
-def main(request):
-    request_json = request.get_json(silent=True)
-    request_args = request.args
-
-    if request_json:
-        push_to_pubsub(request_json)
-    elif request_args:
-        push_to_pubsub(request_args)
-    else:
-        return ("No data provided.", 400)
-    return {"status": "ok", "source": "cf_to_bq"}

terraform/modules/cf_to_pubsub/functions/cf_to_bq/requirements.txt

@@ -1,2 +0,0 @@
-Flask==1.1.2
-google-cloud-bigquery==2.22.0

terraform/modules/cf_to_pubsub/functions/cf_to_pubsub/main.py

@@ -4,7 +4,7 @@ import os
 from google.cloud import pubsub_v1
 
 publisher = pubsub_v1.PublisherClient()
-topic = publisher.topic_path(os.environ["GCP_PROJECT"], os.environ["GCP_TOPIC"])
+topic = publisher.topic_path(os.environ.get("GCP_PROJECT"), os.environ.get("GCP_TOPIC"))
 
 
 def push_to_pubsub(request_params):
@@ -20,5 +20,5 @@ def main(request):
     elif request_args:
         push_to_pubsub(request_args)
     else:
-        return ("No data provided.", 400)
+        raise Exception("No data provided.")
-    return {"status": "ok", "source": "cf_to_pubsub"}
+    return {"status": "ok"}