feat!: ✨ added bigquery subcription capability

README.md

@@ -49,6 +49,14 @@ module "pubsub" {
       service_account              = "service2@project2.iam.gserviceaccount.com"          // optional
       enable_exactly_once_delivery = true                                                 // optional
     }
+  bigquery_subscriptions = [
+    {
+      name                = "bigquery"              // required
+      table               = "project.dataset.table" // required
+      use_topic_schema    = true                    // optional
+      write_metadata      = false                   // optional
+      drop_unknown_fields = false                   // optional
+    }
   ]
 }
 ```
@@ -58,6 +66,7 @@ module "pubsub" {
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
+| bigquery\_subscriptions | The list of the bigquery push subscriptions. | `list(map(string))` | `[]` | no |
 | create\_subscriptions | Specify true if you want to create subscriptions. | `bool` | `true` | no |
 | create\_topic | Specify true if you want to create a topic. | `bool` | `true` | no |
 | grant\_token\_creator | Specify true if you want to add token creator role to the default Pub/Sub SA. | `bool` | `true` | no |

examples/bigquery/README.md

@@ -0,0 +1,35 @@
+# Bigquery Example
+
+This example illustrates how to use the `pubsub` module.
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| project\_id | The project ID to manage the Pub/Sub resources | `string` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| project\_id | The project ID |
+| topic\_labels | The labels of the Pub/Sub topic created |
+| topic\_name | The name of the Pub/Sub topic created |
+
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+
+## Requirements
+
+The following sections describe the requirements which must be met in
+order to invoke this example. The requirements of the
+[root module][root-module-requirements] must be met.
+
+## Usage
+
+To provision this example, populate `terraform.tfvars` with the [required variables](#inputs) and run the following commands within
+this directory:
+- `terraform init` to get the plugins
+- `terraform plan` to see the infrastructure plan
+- `terraform apply` to apply the infrastructure build
+- `terraform destroy` to destroy the built infrastructure

examples/bigquery/main.tf

@@ -0,0 +1,65 @@
+/**
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+provider "google" {
+  region = "europe-west1"
+}
+
+module "pubsub" {
+  source     = "../../"
+  project_id = var.project_id
+  topic      = "cft-tf-pubsub-topic"
+  topic_labels = {
+    foo_label = "foo_value"
+    bar_label = "bar_value"
+  }
+
+  bigquery_subscriptions = [
+    {
+      name  = "example_subscription"
+      table = "${var.project_id}:example_dataset.example_table"
+    },
+  ]
+
+  depends_on = [
+    google_bigquery_table.test
+  ]
+
+}
+
+resource "google_bigquery_dataset" "test" {
+  project    = var.project_id
+  dataset_id = "example_dataset"
+  location   = "europe-west1"
+}
+
+resource "google_bigquery_table" "test" {
+  project             = var.project_id
+  deletion_protection = false
+  table_id            = "example_table"
+  dataset_id          = google_bigquery_dataset.test.dataset_id
+
+  schema = <<EOF
+[
+  {
+    "name": "data",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "The data"
+  }
+]
+EOF
+}

examples/bigquery/outputs.tf

@@ -0,0 +1,30 @@
+/**
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "project_id" {
+  value       = var.project_id
+  description = "The project ID"
+}
+
+output "topic_name" {
+  value       = module.pubsub.topic
+  description = "The name of the Pub/Sub topic created"
+}
+
+output "topic_labels" {
+  value       = module.pubsub.topic_labels
+  description = "The labels of the Pub/Sub topic created"
+}

examples/bigquery/variables.tf

@@ -0,0 +1,20 @@
+/**
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "project_id" {
+  type        = string
+  description = "The project ID to manage the Pub/Sub resources"
+}

examples/bigquery/versions.tf

@@ -0,0 +1,25 @@
+/**
+ * Copyright 2021 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+terraform {
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "~> 4.23"
+    }
+  }
+  required_version = ">= 0.13"
+}

main.tf

@@ -31,6 +31,19 @@ resource "google_pubsub_schema" "schema" {
   definition = var.schema.definition
 }
 
+resource "google_project_iam_member" "bigquery_metadata_viewer_binding" {
+  count   = length(var.bigquery_subscriptions) != 0 ? 1 : 0
+  project = var.project_id
+  role    = "roles/bigquery.metadataViewer"
+  member  = "serviceAccount:${local.pubsub_svc_account_email}"
+}
+
+resource "google_project_iam_member" "bigquery_data_editor_binding" {
+  count   = length(var.bigquery_subscriptions) != 0 ? 1 : 0
+  project = var.project_id
+  role    = "roles/bigquery.dataEditor"
+  member  = "serviceAccount:${local.pubsub_svc_account_email}"
+}
 
 resource "google_project_iam_member" "token_creator_binding" {
   count   = var.grant_token_creator ? 1 : 0
@@ -254,6 +267,76 @@ resource "google_pubsub_subscription" "pull_subscriptions" {
   ]
 }
 
+resource "google_pubsub_subscription" "bigquery_subscriptions" {
+  for_each = var.create_subscriptions ? { for i in var.bigquery_subscriptions : i.name => i } : {}
+
+  name    = each.value.name
+  topic   = var.create_topic ? google_pubsub_topic.topic.0.name : var.topic
+  project = var.project_id
+  labels  = var.subscription_labels
+  ack_deadline_seconds = lookup(
+    each.value,
+    "ack_deadline_seconds",
+    local.default_ack_deadline_seconds,
+  )
+  message_retention_duration = lookup(
+    each.value,
+    "message_retention_duration",
+    null,
+  )
+  retain_acked_messages = lookup(
+    each.value,
+    "retain_acked_messages",
+    null,
+  )
+  filter = lookup(
+    each.value,
+    "filter",
+    null,
+  )
+  enable_message_ordering = lookup(
+    each.value,
+    "enable_message_ordering",
+    null,
+  )
+  dynamic "expiration_policy" {
+    // check if the 'expiration_policy' key exists, if yes, return a list containing it.
+    for_each = contains(keys(each.value), "expiration_policy") ? [each.value.expiration_policy] : []
+    content {
+      ttl = expiration_policy.value
+    }
+  }
+
+  dynamic "dead_letter_policy" {
+    for_each = (lookup(each.value, "dead_letter_topic", "") != "") ? [each.value.dead_letter_topic] : []
+    content {
+      dead_letter_topic     = lookup(each.value, "dead_letter_topic", "")
+      max_delivery_attempts = lookup(each.value, "max_delivery_attempts", "5")
+    }
+  }
+
+  dynamic "retry_policy" {
+    for_each = (lookup(each.value, "maximum_backoff", "") != "") ? [each.value.maximum_backoff] : []
+    content {
+      maximum_backoff = lookup(each.value, "maximum_backoff", "")
+      minimum_backoff = lookup(each.value, "minimum_backoff", "")
+    }
+  }
+
+  bigquery_config {
+    table               = each.value["table"]
+    use_topic_schema    = lookup(each.value, "use_topic_schema", false)
+    write_metadata      = lookup(each.value, "write_metadata", false)
+    drop_unknown_fields = lookup(each.value, "drop_unknown_fields", false)
+  }
+
+  depends_on = [
+    google_pubsub_topic.topic,
+    google_project_iam_member.bigquery_metadata_viewer_binding,
+    google_project_iam_member.bigquery_data_editor_binding
+  ]
+}
+
 resource "google_pubsub_subscription_iam_member" "pull_subscription_sa_binding_subscriber" {
   for_each = var.create_subscriptions ? { for i in var.pull_subscriptions : i.name => i if lookup(i, "service_account", null) != null } : {}
 

outputs.tf

@@ -38,6 +38,7 @@ output "subscription_names" {
   value = concat(
     values({ for k, v in google_pubsub_subscription.push_subscriptions : k => v.name }),
     values({ for k, v in google_pubsub_subscription.pull_subscriptions : k => v.name }),
+    values({ for k, v in google_pubsub_subscription.bigquery_subscriptions : k => v.name }),
   )
 
   description = "The name list of Pub/Sub subscriptions"
@@ -47,6 +48,7 @@ output "subscription_paths" {
   value = concat(
     values({ for k, v in google_pubsub_subscription.push_subscriptions : k => v.id }),
     values({ for k, v in google_pubsub_subscription.pull_subscriptions : k => v.id }),
+    values({ for k, v in google_pubsub_subscription.bigquery_subscriptions : k => v.name }),
   )
 
   description = "The path list of Pub/Sub subscriptions"

test/setup/iam.tf

@@ -18,7 +18,8 @@ locals {
   int_required_roles = [
     "roles/cloudiot.admin",
     "roles/pubsub.admin",
-    "roles/resourcemanager.projectIamAdmin"
+    "roles/resourcemanager.projectIamAdmin",
+    "roles/bigquery.admin"
   ]
 }
 

test/setup/main.tf

@@ -28,6 +28,7 @@ module "project-ci-int-pubsub" {
     "cloudiot.googleapis.com",
     "cloudresourcemanager.googleapis.com",
     "pubsub.googleapis.com",
-    "serviceusage.googleapis.com"
+    "serviceusage.googleapis.com",
+    "bigquery.googleapis.com",
   ]
 }

test/setup/versions.tf

@@ -19,11 +19,11 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = "~> 4.1"
+      version = "~> 4.23"
     }
     google-beta = {
       source  = "hashicorp/google-beta"
-      version = "~> 4.1"
+      version = "~> 4.23"
     }
     null = {
       source = "hashicorp/null"

variables.tf

@@ -53,6 +53,12 @@ variable "pull_subscriptions" {
   default     = []
 }
 
+variable "bigquery_subscriptions" {
+  type        = list(map(string))
+  description = "The list of the bigquery push subscriptions."
+  default     = []
+}
+
 variable "subscription_labels" {
   type        = map(string)
   description = "A map of labels to assign to every Pub/Sub subscription."

versions.tf

@@ -20,7 +20,7 @@ terraform {
 
     google = {
       source  = "hashicorp/google"
-      version = ">= 4.17, < 5.0"
+      version = ">= 4.32, < 5.0"
     }
   }