Add bigquery table.
This commit is contained in:
@@ -1,3 +1,6 @@
|
||||
# Example message:
|
||||
# {"time": "2021-07-20T05:05:47", "service": "foo", "log": "bar"}
|
||||
|
||||
variable "project" {
|
||||
description = "Project ID."
|
||||
type = string
|
||||
@@ -116,10 +119,56 @@ resource "google_dataflow_job" "etl_job" {
|
||||
|
||||
parameters = {
|
||||
inputSubscription = google_pubsub_subscription.bigquery_etl.id
|
||||
outputTableSpec = "your-project:your-dataset.your-table-name"
|
||||
outputTableSpec = "${google_bigquery_table.pubsub_etl.project}:${google_bigquery_table.pubsub_etl.dataset_id}.${google_bigquery_table.pubsub_etl.table_id}"
|
||||
}
|
||||
|
||||
additional_experiments = [
|
||||
"enable_streaming_engine"
|
||||
"enable_streaming_engine",
|
||||
"enable_windmill_service"
|
||||
]
|
||||
}
|
||||
|
||||
#################### BigQuery #############################
|
||||
|
||||
resource "google_bigquery_dataset" "pubsub_etl" {
|
||||
project = var.project
|
||||
dataset_id = "pubsub_etl"
|
||||
friendly_name = "PubSub ETL"
|
||||
description = "Dataset where PubSub ETL data goes."
|
||||
location = "US"
|
||||
}
|
||||
|
||||
resource "google_bigquery_table" "pubsub_etl" {
|
||||
project = var.project
|
||||
dataset_id = google_bigquery_dataset.pubsub_etl.dataset_id
|
||||
table_id = "pubsub_etl"
|
||||
|
||||
deletion_protection = false
|
||||
|
||||
time_partitioning {
|
||||
type = "DAY"
|
||||
}
|
||||
|
||||
schema = <<EOF
|
||||
[
|
||||
{
|
||||
"description": "Time of the event",
|
||||
"mode": "NULLABLE",
|
||||
"name": "time",
|
||||
"type": "DATETIME"
|
||||
},
|
||||
{
|
||||
"description": "Name of the service",
|
||||
"mode": "NULLABLE",
|
||||
"name": "service",
|
||||
"type": "STRING"
|
||||
},
|
||||
{
|
||||
"description": "Text of the log",
|
||||
"mode": "NULLABLE",
|
||||
"name": "log",
|
||||
"type": "STRING"
|
||||
}
|
||||
]
|
||||
EOF
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user