diff --git a/blueprints/data-solutions/data-platform-foundations/02-load.tf b/blueprints/data-solutions/data-platform-foundations/02-load.tf
index 52fb5276ff..2810a38f8c 100644
--- a/blueprints/data-solutions/data-platform-foundations/02-load.tf
+++ b/blueprints/data-solutions/data-platform-foundations/02-load.tf
@@ -17,7 +17,8 @@
locals {
load_iam = {
data_engineers = [
- "roles/dataflow.admin"
+ "roles/dataflow.admin",
+ "roles/dataflow.developer"
]
robots_dataflow_load = [
"roles/storage.objectAdmin"
@@ -54,6 +55,7 @@ module "load-project" {
"cloudkms.googleapis.com",
"compute.googleapis.com",
"dataflow.googleapis.com",
+ "datalineage.googleapis.com",
"dlp.googleapis.com",
"pubsub.googleapis.com",
"servicenetworking.googleapis.com",
diff --git a/blueprints/data-solutions/data-platform-foundations/03-composer.tf b/blueprints/data-solutions/data-platform-foundations/03-composer.tf
index 8c803e4b6a..af169e7195 100644
--- a/blueprints/data-solutions/data-platform-foundations/03-composer.tf
+++ b/blueprints/data-solutions/data-platform-foundations/03-composer.tf
@@ -68,16 +68,20 @@ module "orch-sa-cmp-0" {
}
resource "google_composer_environment" "orch-cmp-0" {
- count = var.composer_config.disable_deployment == true ? 0 : 1
- project = module.orch-project.project_id
- name = "${var.prefix}-orc-cmp-0"
- region = var.region
+ count = var.composer_config.disable_deployment == true ? 0 : 1
+ provider = google-beta
+ project = module.orch-project.project_id
+ name = "${var.prefix}-orc-cmp-0"
+ region = var.region
config {
software_config {
airflow_config_overrides = try(var.composer_config.software_config.airflow_config_overrides, null)
pypi_packages = try(var.composer_config.software_config.pypi_packages, null)
env_variables = local.env_variables
image_version = try(var.composer_config.software_config.image_version, null)
+ cloud_data_lineage_integration {
+ enabled = var.composer_config.software_config.cloud_data_lineage_integration
+ }
}
dynamic "workloads_config" {
for_each = (try(var.composer_config.workloads_config, null) != null ? { 1 = 1 } : {})
diff --git a/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf b/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf
index 804250a5ca..c97721a47a 100644
--- a/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf
+++ b/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf
@@ -21,10 +21,13 @@ locals {
"roles/bigquery.dataEditor",
"roles/bigquery.jobUser",
"roles/cloudbuild.builds.editor",
+ "roles/composer.admin",
+ "roles/composer.user",
"roles/composer.environmentAndStorageObjectAdmin",
"roles/iam.serviceAccountUser",
"roles/iap.httpsResourceAccessor",
- "roles/serviceusage.serviceUsageConsumer"
+ "roles/serviceusage.serviceUsageConsumer",
+ "roles/storage.objectAdmin"
]
robots_cloudbuild = [
"roles/storage.objectAdmin"
@@ -33,6 +36,10 @@ locals {
"roles/composer.ServiceAgentV2Ext",
"roles/storage.objectAdmin"
]
+ sa_df_build = [
+ "roles/cloudbuild.serviceAgent",
+ "roles/storage.objectAdmin"
+ ]
sa_load = [
"roles/artifactregistry.reader",
"roles/bigquery.dataEditor",
@@ -63,9 +70,7 @@ module "orch-project" {
)
iam = local.use_projects ? {} : local.orch_iam_auth
iam_bindings_additive = !local.use_projects ? {} : local.orch_iam_additive
- compute_metadata = {
- enable-oslogin = "false"
- }
+
services = concat(var.project_services, [
"artifactregistry.googleapis.com",
"bigquery.googleapis.com",
@@ -79,6 +84,7 @@ module "orch-project" {
"containerregistry.googleapis.com",
"artifactregistry.googleapis.com",
"dataflow.googleapis.com",
+ "datalineage.googleapis.com",
"orgpolicy.googleapis.com",
"pubsub.googleapis.com",
"servicenetworking.googleapis.com",
diff --git a/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf b/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf
index 8ba0f0f969..7cbd4bfae4 100644
--- a/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf
+++ b/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf
@@ -19,12 +19,14 @@ locals {
data_analysts = [
"roles/bigquery.dataViewer",
"roles/bigquery.jobUser",
+ "roles/datacatalog.tagTemplateViewer",
"roles/datacatalog.viewer",
"roles/storage.objectViewer"
]
data_engineers = [
"roles/bigquery.dataViewer",
"roles/bigquery.jobUser",
+ "roles/datacatalog.tagTemplateViewer",
"roles/datacatalog.viewer",
"roles/storage.objectViewer"
]
@@ -41,10 +43,13 @@ locals {
data_engineers = [
"roles/bigquery.dataViewer",
"roles/bigquery.jobUser",
+ "roles/datacatalog.tagTemplateViewer",
"roles/datacatalog.viewer",
"roles/storage.objectViewer"
]
sa_load = [
+ "roles/bigquery.dataOwner",
+ "roles/bigquery.jobUser",
"roles/storage.objectCreator"
]
sa_transf_bq = [
@@ -52,9 +57,7 @@ locals {
"roles/datacatalog.categoryAdmin"
]
sa_transf_df = [
- "roles/bigquery.dataOwner",
- "roles/bigquery.dataViewer",
- "roles/bigquery.jobUser"
+ "roles/bigquery.dataViewer"
]
}
}
diff --git a/blueprints/data-solutions/data-platform-foundations/README.md b/blueprints/data-solutions/data-platform-foundations/README.md
index 5c2cabee9d..f313614b92 100644
--- a/blueprints/data-solutions/data-platform-foundations/README.md
+++ b/blueprints/data-solutions/data-platform-foundations/README.md
@@ -228,7 +228,7 @@ module "data-platform" {
}
prefix = "myprefix"
}
-# tftest modules=43 resources=279
+# tftest modules=43 resources=290
```
## Customizations
@@ -255,24 +255,43 @@ Once you have identified the required project granularity for your use case, we
The application layer is out of scope of this script. As a demo purpuse only, several Cloud Composer DAGs are provided. Demos will import data from the `drop off` area to the `Data Warehouse Confidential` dataset suing different features.
You can find examples in the `[demo](./demo)` folder.
+
+## Cleanup
+
+If you want to destroy the Data Platform deployment, follow these steps.
+
+**ATTENTION**: The following procedure will permanently delete all of your data in an irreversible manner.
+
+```bash
+# remove GCS buckets and BQ dataset manually. Projects will be destroyed anyway
+for x in $(terraform state list | grep google_storage_bucket.bucket); do
+ terraform state rm "$x";
+done
+
+for x in $(terraform state list | grep google_bigquery_dataset); do
+ terraform state rm "$x";
+done
+
+terraform destroy
+```
## Variables
| name | description | type | required | default |
|---|---|:---:|:---:|:---:|
-| [organization_domain](variables.tf#L164) | Organization domain. | string | ✓ | |
-| [prefix](variables.tf#L169) | Prefix used for resource names. | string | ✓ | |
-| [project_config](variables.tf#L178) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | ✓ | |
-| [composer_config](variables.tf#L17) | Cloud Composer config. | object({…}) | | {…} |
-| [data_catalog_tags](variables.tf#L105) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(object({…})) | | {…} |
-| [data_force_destroy](variables.tf#L119) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | bool | | false |
-| [groups](variables.tf#L125) | User groups. | map(string) | | {…} |
-| [location](variables.tf#L135) | Location used for multi-regional resources. | string | | "eu" |
-| [network_config](variables.tf#L141) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null |
-| [project_services](variables.tf#L212) | List of core services enabled on all projects. | list(string) | | […] |
-| [project_suffix](variables.tf#L223) | Suffix used only for project ids. | string | | null |
-| [region](variables.tf#L229) | Region used for regional resources. | string | | "europe-west1" |
-| [service_encryption_keys](variables.tf#L235) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null |
+| [organization_domain](variables.tf#L165) | Organization domain. | string | ✓ | |
+| [prefix](variables.tf#L170) | Prefix used for resource names. | string | ✓ | |
+| [project_config](variables.tf#L179) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | ✓ | |
+| [composer_config](variables.tf#L17) | Cloud Composer config. | object({…}) | | {…} |
+| [data_catalog_tags](variables.tf#L106) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(object({…})) | | {…} |
+| [data_force_destroy](variables.tf#L120) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | bool | | false |
+| [groups](variables.tf#L126) | User groups. | map(string) | | {…} |
+| [location](variables.tf#L136) | Location used for multi-regional resources. | string | | "eu" |
+| [network_config](variables.tf#L142) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null |
+| [project_services](variables.tf#L213) | List of core services enabled on all projects. | list(string) | | […] |
+| [project_suffix](variables.tf#L224) | Suffix used only for project ids. | string | | null |
+| [region](variables.tf#L230) | Region used for regional resources. | string | | "europe-west1" |
+| [service_encryption_keys](variables.tf#L236) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null |
## Outputs
diff --git a/blueprints/data-solutions/data-platform-foundations/demo/datapipeline.py b/blueprints/data-solutions/data-platform-foundations/demo/datapipeline.py
index 45b71b30d8..e23fd1162f 100644
--- a/blueprints/data-solutions/data-platform-foundations/demo/datapipeline.py
+++ b/blueprints/data-solutions/data-platform-foundations/demo/datapipeline.py
@@ -38,9 +38,6 @@
DWH_CONFIDENTIAL_PRJ = Variable.get("DWH_CONFIDENTIAL_PRJ")
DWH_CONFIDENTIAL_BQ_DATASET = Variable.get("DWH_CONFIDENTIAL_BQ_DATASET")
DWH_CONFIDENTIAL_GCS = Variable.get("DWH_CONFIDENTIAL_GCS")
-DWH_PLG_PRJ = Variable.get("DWH_PLG_PRJ")
-DWH_PLG_BQ_DATASET = Variable.get("DWH_PLG_BQ_DATASET")
-DWH_PLG_GCS = Variable.get("DWH_PLG_GCS")
GCP_REGION = Variable.get("GCP_REGION")
DRP_PRJ = Variable.get("DRP_PRJ")
DRP_BQ = Variable.get("DRP_BQ")
diff --git a/blueprints/data-solutions/data-platform-foundations/demo/datapipeline_dc_tags.py b/blueprints/data-solutions/data-platform-foundations/demo/datapipeline_dc_tags.py
index 5e86472af1..65311dba21 100644
--- a/blueprints/data-solutions/data-platform-foundations/demo/datapipeline_dc_tags.py
+++ b/blueprints/data-solutions/data-platform-foundations/demo/datapipeline_dc_tags.py
@@ -39,9 +39,6 @@
DWH_CONFIDENTIAL_PRJ = Variable.get("DWH_CONFIDENTIAL_PRJ")
DWH_CONFIDENTIAL_BQ_DATASET = Variable.get("DWH_CONFIDENTIAL_BQ_DATASET")
DWH_CONFIDENTIAL_GCS = Variable.get("DWH_CONFIDENTIAL_GCS")
-DWH_PLG_PRJ = Variable.get("DWH_PLG_PRJ")
-DWH_PLG_BQ_DATASET = Variable.get("DWH_PLG_BQ_DATASET")
-DWH_PLG_GCS = Variable.get("DWH_PLG_GCS")
GCP_REGION = Variable.get("GCP_REGION")
DRP_PRJ = Variable.get("DRP_PRJ")
DRP_BQ = Variable.get("DRP_BQ")
diff --git a/blueprints/data-solutions/data-platform-foundations/demo/datapipeline_dc_tags_flex.py b/blueprints/data-solutions/data-platform-foundations/demo/datapipeline_dc_tags_flex.py
index 7bbf67a160..a81ecef9b4 100644
--- a/blueprints/data-solutions/data-platform-foundations/demo/datapipeline_dc_tags_flex.py
+++ b/blueprints/data-solutions/data-platform-foundations/demo/datapipeline_dc_tags_flex.py
@@ -40,9 +40,6 @@
DWH_CONFIDENTIAL_PRJ = Variable.get("DWH_CONFIDENTIAL_PRJ")
DWH_CONFIDENTIAL_BQ_DATASET = Variable.get("DWH_CONFIDENTIAL_BQ_DATASET")
DWH_CONFIDENTIAL_GCS = Variable.get("DWH_CONFIDENTIAL_GCS")
-DWH_PLG_PRJ = Variable.get("DWH_PLG_PRJ")
-DWH_PLG_BQ_DATASET = Variable.get("DWH_PLG_BQ_DATASET")
-DWH_PLG_GCS = Variable.get("DWH_PLG_GCS")
GCP_REGION = Variable.get("GCP_REGION")
DRP_PRJ = Variable.get("DRP_PRJ")
DRP_BQ = Variable.get("DRP_BQ")
diff --git a/blueprints/data-solutions/data-platform-foundations/demo/datapipeline_flex.py b/blueprints/data-solutions/data-platform-foundations/demo/datapipeline_flex.py
index 5e60c62f0b..e948fac69d 100644
--- a/blueprints/data-solutions/data-platform-foundations/demo/datapipeline_flex.py
+++ b/blueprints/data-solutions/data-platform-foundations/demo/datapipeline_flex.py
@@ -39,9 +39,6 @@
DWH_CONFIDENTIAL_PRJ = Variable.get("DWH_CONFIDENTIAL_PRJ")
DWH_CONFIDENTIAL_BQ_DATASET = Variable.get("DWH_CONFIDENTIAL_BQ_DATASET")
DWH_CONFIDENTIAL_GCS = Variable.get("DWH_CONFIDENTIAL_GCS")
-DWH_PLG_PRJ = Variable.get("DWH_PLG_PRJ")
-DWH_PLG_BQ_DATASET = Variable.get("DWH_PLG_BQ_DATASET")
-DWH_PLG_GCS = Variable.get("DWH_PLG_GCS")
GCP_REGION = Variable.get("GCP_REGION")
DRP_PRJ = Variable.get("DRP_PRJ")
DRP_BQ = Variable.get("DRP_BQ")
diff --git a/blueprints/data-solutions/data-platform-foundations/demo/delete_table.py b/blueprints/data-solutions/data-platform-foundations/demo/delete_table.py
index 252400ad66..9ae3f384b2 100644
--- a/blueprints/data-solutions/data-platform-foundations/demo/delete_table.py
+++ b/blueprints/data-solutions/data-platform-foundations/demo/delete_table.py
@@ -43,9 +43,6 @@
DWH_CONFIDENTIAL_PRJ = Variable.get("DWH_CONFIDENTIAL_PRJ")
DWH_CONFIDENTIAL_BQ_DATASET = Variable.get("DWH_CONFIDENTIAL_BQ_DATASET")
DWH_CONFIDENTIAL_GCS = Variable.get("DWH_CONFIDENTIAL_GCS")
-DWH_PLG_PRJ = Variable.get("DWH_PLG_PRJ")
-DWH_PLG_BQ_DATASET = Variable.get("DWH_PLG_BQ_DATASET")
-DWH_PLG_GCS = Variable.get("DWH_PLG_GCS")
GCP_REGION = Variable.get("GCP_REGION")
DRP_PRJ = Variable.get("DRP_PRJ")
DRP_BQ = Variable.get("DRP_BQ")
diff --git a/blueprints/data-solutions/data-platform-foundations/locals-05-datawarehouse.tf b/blueprints/data-solutions/data-platform-foundations/locals-05-datawarehouse.tf
index 5bd652c856..47c91b1ace 100644
--- a/blueprints/data-solutions/data-platform-foundations/locals-05-datawarehouse.tf
+++ b/blueprints/data-solutions/data-platform-foundations/locals-05-datawarehouse.tf
@@ -50,6 +50,7 @@ locals {
"cloudkms.googleapis.com",
"compute.googleapis.com",
"dataflow.googleapis.com",
+ "datalineage.googleapis.com",
"pubsub.googleapis.com",
"servicenetworking.googleapis.com",
"storage.googleapis.com",
diff --git a/blueprints/data-solutions/data-platform-foundations/main.tf b/blueprints/data-solutions/data-platform-foundations/main.tf
index 77944f4df2..8a22f38645 100644
--- a/blueprints/data-solutions/data-platform-foundations/main.tf
+++ b/blueprints/data-solutions/data-platform-foundations/main.tf
@@ -43,6 +43,7 @@ locals {
robots_composer = "serviceAccount:${module.orch-project.service_accounts.robots.composer}"
robots_dataflow_load = "serviceAccount:${module.load-project.service_accounts.robots.dataflow}"
robots_dataflow_trf = "serviceAccount:${module.transf-project.service_accounts.robots.dataflow}"
+ sa_df_build = module.orch-sa-df-build.iam_email
sa_drop_bq = module.drop-sa-bq-0.iam_email
sa_drop_cs = module.drop-sa-cs-0.iam_email
sa_drop_ps = module.drop-sa-ps-0.iam_email
diff --git a/blueprints/data-solutions/data-platform-foundations/variables.tf b/blueprints/data-solutions/data-platform-foundations/variables.tf
index 92a6316b44..335eede86f 100644
--- a/blueprints/data-solutions/data-platform-foundations/variables.tf
+++ b/blueprints/data-solutions/data-platform-foundations/variables.tf
@@ -21,10 +21,11 @@ variable "composer_config" {
environment_size = optional(string, "ENVIRONMENT_SIZE_SMALL")
software_config = optional(
object({
- airflow_config_overrides = optional(any)
- pypi_packages = optional(any)
- env_variables = optional(map(string))
- image_version = string
+ airflow_config_overrides = optional(any)
+ pypi_packages = optional(any)
+ env_variables = optional(map(string))
+ image_version = string
+ cloud_data_lineage_integration = optional(bool, true)
}),
{ image_version = "composer-2-airflow-2" }
)
diff --git a/fast/stages/3-data-platform/dev/IAM.md b/fast/stages/3-data-platform/dev/IAM.md
index 70622c2e03..02a5df7a91 100644
--- a/fast/stages/3-data-platform/dev/IAM.md
+++ b/fast/stages/3-data-platform/dev/IAM.md
@@ -2,108 +2,88 @@
Legend: + additive, • conditional.
-## Project dev-data-cmn-0
+## Project cmn
| members | roles |
|---|---|
|gcp-data-analysts
group|[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) |
|gcp-data-engineers
group|[roles/dlp.estimatesAdmin](https://cloud.google.com/iam/docs/understanding-roles#dlp.estimatesAdmin)
[roles/dlp.reader](https://cloud.google.com/iam/docs/understanding-roles#dlp.reader)
[roles/dlp.user](https://cloud.google.com/iam/docs/understanding-roles#dlp.user) |
|gcp-data-security
group|[roles/datacatalog.admin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.admin)
[roles/dlp.admin](https://cloud.google.com/iam/docs/understanding-roles#dlp.admin) |
-|dev-data-load-df-0
serviceAccount|[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/dlp.user](https://cloud.google.com/iam/docs/understanding-roles#dlp.user) |
-|dev-data-trf-bq-0
serviceAccount|[roles/datacatalog.categoryFineGrainedReader](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryFineGrainedReader)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) |
-|dev-data-trf-df-0
serviceAccount|[roles/datacatalog.categoryFineGrainedReader](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryFineGrainedReader)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/dlp.user](https://cloud.google.com/iam/docs/understanding-roles#dlp.user) |
+|load-df
serviceAccount|[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/dlp.user](https://cloud.google.com/iam/docs/understanding-roles#dlp.user) |
+|trf-bq
serviceAccount|[roles/datacatalog.categoryFineGrainedReader](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryFineGrainedReader)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) |
+|trf-df
serviceAccount|[roles/datacatalog.categoryFineGrainedReader](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryFineGrainedReader)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/dlp.user](https://cloud.google.com/iam/docs/understanding-roles#dlp.user) |
-## Project dev-data-dtl-0-0
+## Project drp
| members | roles |
|---|---|
-|gcp-data-analysts
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/bigquery.metadataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.metadataViewer)
[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
-|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
-|SERVICE_IDENTITY_service-networking
serviceAccount|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) +|
-|dev-data-load-df-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator) |
-|dev-data-trf-bq-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) |
-|dev-data-trf-df-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) |
+|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user) |
+|drp-bq
serviceAccount|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) |
+|drp-cs
serviceAccount|[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator) |
+|drp-ps
serviceAccount|[roles/pubsub.publisher](https://cloud.google.com/iam/docs/understanding-roles#pubsub.publisher) |
+|load-df
serviceAccount|[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/pubsub.subscriber](https://cloud.google.com/iam/docs/understanding-roles#pubsub.subscriber)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
+|orc-cmp
serviceAccount|[roles/pubsub.subscriber](https://cloud.google.com/iam/docs/understanding-roles#pubsub.subscriber)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
-## Project dev-data-dtl-1-0
+## Project dwh-conf
| members | roles |
|---|---|
-|gcp-data-analysts
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/bigquery.metadataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.metadataViewer)
[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
-|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
+|gcp-data-analysts
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
+|gcp-data-engineers
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
|SERVICE_IDENTITY_service-networking
serviceAccount|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) +|
-|dev-data-load-df-0
serviceAccount|[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) |
-|dev-data-trf-bq-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) |
-|dev-data-trf-df-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
+|trf-bq
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) |
+|trf-df
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
-## Project dev-data-dtl-2-0
+## Project dwh-cur
| members | roles |
|---|---|
-|gcp-data-analysts
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/bigquery.metadataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.metadataViewer)
[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
-|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
+|gcp-data-analysts
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
+|gcp-data-engineers
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
|SERVICE_IDENTITY_service-networking
serviceAccount|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) +|
-|dev-data-load-df-0
serviceAccount|[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) |
-|dev-data-trf-bq-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) |
-|dev-data-trf-df-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
+|trf-bq
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) |
+|trf-df
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
-## Project dev-data-dtl-plg-0
+## Project dwh-lnd
| members | roles |
|---|---|
-|gcp-data-analysts
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/bigquery.metadataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.metadataViewer)
[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
-|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
+|gcp-data-engineers
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
|SERVICE_IDENTITY_service-networking
serviceAccount|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) +|
+|load-df
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator) |
+|trf-bq
serviceAccount|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) |
+|trf-df
serviceAccount|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer) |
-## Project dev-data-lnd-0
-
-| members | roles |
-|---|---|
-|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/pubsub.editor](https://cloud.google.com/iam/docs/understanding-roles#pubsub.editor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
-|dev-data-lnd-bq-0
serviceAccount|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) |
-|dev-data-lnd-cs-0
serviceAccount|[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator) |
-|dev-data-lnd-ps-0
serviceAccount|[roles/pubsub.publisher](https://cloud.google.com/iam/docs/understanding-roles#pubsub.publisher) |
-|dev-data-load-df-0
serviceAccount|[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/pubsub.subscriber](https://cloud.google.com/iam/docs/understanding-roles#pubsub.subscriber)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
-|dev-data-orc-cmp-0
serviceAccount|[roles/pubsub.subscriber](https://cloud.google.com/iam/docs/understanding-roles#pubsub.subscriber)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
-
-## Project dev-data-lod-0
+## Project lod
| members | roles |
|---|---|
-|gcp-data-engineers
group|[roles/compute.viewer](https://cloud.google.com/iam/docs/understanding-roles#compute.viewer)
[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin)
[roles/dataflow.developer](https://cloud.google.com/iam/docs/understanding-roles#dataflow.developer)
[roles/viewer](https://cloud.google.com/iam/docs/understanding-roles#viewer) |
+|gcp-data-engineers
group|[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin)
[roles/dataflow.developer](https://cloud.google.com/iam/docs/understanding-roles#dataflow.developer) |
|SERVICE_IDENTITY_dataflow-service-producer-prod
serviceAccount|[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
|SERVICE_IDENTITY_service-networking
serviceAccount|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) +|
-|dev-data-load-df-0
serviceAccount|[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin)
[roles/dataflow.worker](https://cloud.google.com/iam/docs/understanding-roles#dataflow.worker)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
-|dev-data-orc-cmp-0
serviceAccount|[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin) |
+|load-df
serviceAccount|[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin)
[roles/dataflow.worker](https://cloud.google.com/iam/docs/understanding-roles#dataflow.worker)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
+|orc-cmp
serviceAccount|[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin) |
-## Project dev-data-orc-0
+## Project orc
| members | roles |
|---|---|
-|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/cloudbuild.builds.editor](https://cloud.google.com/iam/docs/understanding-roles#cloudbuild.builds.editor)
[roles/composer.admin](https://cloud.google.com/iam/docs/understanding-roles#composer.admin)
[roles/composer.environmentAndStorageObjectAdmin](https://cloud.google.com/iam/docs/understanding-roles#composer.environmentAndStorageObjectAdmin)
[roles/iam.serviceAccountUser](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountUser)
[roles/iap.httpsResourceAccessor](https://cloud.google.com/iam/docs/understanding-roles#iap.httpsResourceAccessor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
-|SERVICE_IDENTITY_cloudcomposer-accounts
serviceAccount|[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
+|gcp-data-engineers
group|[roles/artifactregistry.admin](https://cloud.google.com/iam/docs/understanding-roles#artifactregistry.admin)
[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/cloudbuild.builds.editor](https://cloud.google.com/iam/docs/understanding-roles#cloudbuild.builds.editor)
[roles/composer.admin](https://cloud.google.com/iam/docs/understanding-roles#composer.admin)
[roles/composer.environmentAndStorageObjectAdmin](https://cloud.google.com/iam/docs/understanding-roles#composer.environmentAndStorageObjectAdmin)
[roles/composer.user](https://cloud.google.com/iam/docs/understanding-roles#composer.user)
[roles/iam.serviceAccountUser](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountUser)
[roles/iap.httpsResourceAccessor](https://cloud.google.com/iam/docs/understanding-roles#iap.httpsResourceAccessor)
[roles/serviceusage.serviceUsageConsumer](https://cloud.google.com/iam/docs/understanding-roles#serviceusage.serviceUsageConsumer)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
+|SERVICE_IDENTITY_cloudcomposer-accounts
serviceAccount|[roles/composer.ServiceAgentV2Ext](https://cloud.google.com/iam/docs/understanding-roles#composer.ServiceAgentV2Ext)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
+|SERVICE_IDENTITY_gcp-sa-cloudbuild
serviceAccount|[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
|SERVICE_IDENTITY_service-networking
serviceAccount|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) +|
-|dev-data-load-df-0
serviceAccount|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
-|dev-data-orc-cmp-0
serviceAccount|[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/composer.worker](https://cloud.google.com/iam/docs/understanding-roles#composer.worker)
[roles/iam.serviceAccountUser](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountUser)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
-|dev-data-trf-df-0
serviceAccount|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) |
+|load-df
serviceAccount|[roles/artifactregistry.reader](https://cloud.google.com/iam/docs/understanding-roles#artifactregistry.reader)
[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
+|orc-cmp
serviceAccount|[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/composer.worker](https://cloud.google.com/iam/docs/understanding-roles#composer.worker)
[roles/iam.serviceAccountUser](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountUser)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
+|orc-sa-df-build
serviceAccount|[roles/cloudbuild.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#cloudbuild.serviceAgent)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
+|trf-df
serviceAccount|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) |
-## Project dev-data-trf-0
+## Project trf
| members | roles |
|---|---|
|gcp-data-engineers
group|[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin) |
|SERVICE_IDENTITY_dataflow-service-producer-prod
serviceAccount|[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
|SERVICE_IDENTITY_service-networking
serviceAccount|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) +|
-|dev-data-orc-cmp-0
serviceAccount|[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin) |
-|dev-data-trf-bq-0
serviceAccount|[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) |
-|dev-data-trf-df-0
serviceAccount|[roles/dataflow.worker](https://cloud.google.com/iam/docs/understanding-roles#dataflow.worker)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
-
-## Project dev-net-spoke-0
-
-| members | roles |
-|---|---|
-|PROJECT_CLOUD_SERVICES
serviceAccount|[roles/compute.networkUser](https://cloud.google.com/iam/docs/understanding-roles#compute.networkUser) +|
-|SERVICE_IDENTITY_cloudcomposer-accounts
serviceAccount|[roles/composer.sharedVpcAgent](https://cloud.google.com/iam/docs/understanding-roles#composer.sharedVpcAgent) +|
-|SERVICE_IDENTITY_container-engine-robot
serviceAccount|[roles/compute.networkUser](https://cloud.google.com/iam/docs/understanding-roles#compute.networkUser) +
[roles/container.hostServiceAgentUser](https://cloud.google.com/iam/docs/understanding-roles#container.hostServiceAgentUser) +|
-|SERVICE_IDENTITY_dataflow-service-producer-prod
serviceAccount|[roles/compute.networkUser](https://cloud.google.com/iam/docs/understanding-roles#compute.networkUser) +
[roles/compute.networkUser](https://cloud.google.com/iam/docs/understanding-roles#compute.networkUser) +
[roles/compute.networkUser](https://cloud.google.com/iam/docs/understanding-roles#compute.networkUser) +
[roles/container.hostServiceAgentUser](https://cloud.google.com/iam/docs/understanding-roles#container.hostServiceAgentUser) +|
-|dev-data-load-df-0
serviceAccount|[roles/compute.networkUser](https://cloud.google.com/iam/docs/understanding-roles#compute.networkUser) +|
-|dev-data-trf-df-0
serviceAccount|[roles/compute.networkUser](https://cloud.google.com/iam/docs/understanding-roles#compute.networkUser) +|
+|orc-cmp
serviceAccount|[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin) |
+|trf-bq
serviceAccount|[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) |
+|trf-df
serviceAccount|[roles/dataflow.worker](https://cloud.google.com/iam/docs/understanding-roles#dataflow.worker)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
diff --git a/fast/stages/3-data-platform/dev/README.md b/fast/stages/3-data-platform/dev/README.md
index 6641e27b36..397f2d20ff 100644
--- a/fast/stages/3-data-platform/dev/README.md
+++ b/fast/stages/3-data-platform/dev/README.md
@@ -185,22 +185,23 @@ You can find examples in the `[demo](../../../../blueprints/data-solutions/data-
|---|---|:---:|:---:|:---:|:---:|
| [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 0-bootstrap |
| [billing_account](variables.tf#L25) | Billing account id. If billing account is not part of the same org set `is_org_level` to false. | object({…}) | ✓ | | 0-bootstrap |
-| [folder_ids](variables.tf#L105) | Folder to be used for the networking resources in folders/nnnn format. | object({…}) | ✓ | | 1-resman |
-| [host_project_ids](variables.tf#L123) | Shared VPC project ids. | object({…}) | ✓ | | 2-networking |
-| [organization](variables.tf#L153) | Organization details. | object({…}) | ✓ | | 00-globals |
-| [prefix](variables.tf#L169) | Unique prefix used for resource names. Not used for projects if 'project_create' is null. | string | ✓ | | 00-globals |
-| [composer_config](variables.tf#L38) | Cloud Composer configuration options. | object({…}) | | {…} | |
-| [data_catalog_tags](variables.tf#L85) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(object({…})) | | {…} | |
-| [data_force_destroy](variables.tf#L99) | Flag to set 'force_destroy' on data services like BigQery or Cloud Storage. | bool | | false | |
-| [groups](variables.tf#L113) | Groups. | map(string) | | {…} | |
-| [location](variables.tf#L131) | Location used for multi-regional resources. | string | | "eu" | |
-| [network_config_composer](variables.tf#L137) | Network configurations to use for Composer. | object({…}) | | {…} | |
-| [outputs_location](variables.tf#L163) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | |
-| [project_services](variables.tf#L179) | List of core services enabled on all projects. | list(string) | | […] | |
-| [region](variables.tf#L190) | Region used for regional resources. | string | | "europe-west1" | |
-| [service_encryption_keys](variables.tf#L196) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null | |
-| [subnet_self_links](variables.tf#L208) | Shared VPC subnet self links. | object({…}) | | null | 2-networking |
-| [vpc_self_links](variables.tf#L217) | Shared VPC self links. | object({…}) | | null | 2-networking |
+| [folder_ids](variables.tf#L107) | Folder to be used for the networking resources in folders/nnnn format. | object({…}) | ✓ | | 1-resman |
+| [host_project_ids](variables.tf#L125) | Shared VPC project ids. | object({…}) | ✓ | | 2-networking |
+| [organization](variables.tf#L155) | Organization details. | object({…}) | ✓ | | 00-globals |
+| [prefix](variables.tf#L171) | Unique prefix used for resource names. Not used for projects if 'project_create' is null. | string | ✓ | | 00-globals |
+| [composer_config](variables.tf#L38) | Cloud Composer configuration options. | object({…}) | | {…} | |
+| [data_catalog_tags](variables.tf#L87) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(object({…})) | | {…} | |
+| [data_force_destroy](variables.tf#L101) | Flag to set 'force_destroy' on data services like BigQery or Cloud Storage. | bool | | false | |
+| [groups-dp](variables.tf#L115) | Data Platform groups. | map(string) | | {…} | |
+| [location](variables.tf#L133) | Location used for multi-regional resources. | string | | "eu" | |
+| [network_config_composer](variables.tf#L139) | Network configurations to use for Composer. | object({…}) | | {…} | |
+| [outputs_location](variables.tf#L165) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | |
+| [project_services](variables.tf#L181) | List of core services enabled on all projects. | list(string) | | […] | |
+| [project_suffix](variables.tf#L192) | Suffix used only for project ids. | string | | null | |
+| [region](variables.tf#L198) | Region used for regional resources. | string | | "europe-west1" | |
+| [service_encryption_keys](variables.tf#L204) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null | |
+| [subnet_self_links](variables.tf#L216) | Shared VPC subnet self links. | object({…}) | | null | 2-networking |
+| [vpc_self_links](variables.tf#L225) | Shared VPC self links. | object({…}) | | null | 2-networking |
## Outputs
diff --git a/fast/stages/3-data-platform/dev/main.tf b/fast/stages/3-data-platform/dev/main.tf
index 6e699837b5..25dfe24aa2 100644
--- a/fast/stages/3-data-platform/dev/main.tf
+++ b/fast/stages/3-data-platform/dev/main.tf
@@ -25,7 +25,7 @@ module "data-platform" {
billing_account_id = var.billing_account.id
parent = var.folder_ids.data-platform-dev
}
- groups = var.groups
+ groups = var.groups-dp
location = var.location
network_config = {
host_project = var.host_project_ids.dev-spoke-0
@@ -46,9 +46,9 @@ module "data-platform" {
}
}
organization_domain = var.organization.domain
- prefix = "${var.prefix}-dev-dt"
+ prefix = "${var.prefix}-dev-dp"
project_services = var.project_services
- project_suffix = "0"
+ project_suffix = var.project_suffix
region = var.region
service_encryption_keys = var.service_encryption_keys
}
diff --git a/fast/stages/3-data-platform/dev/variables.tf b/fast/stages/3-data-platform/dev/variables.tf
index 940c12b68c..ef28fb27fc 100644
--- a/fast/stages/3-data-platform/dev/variables.tf
+++ b/fast/stages/3-data-platform/dev/variables.tf
@@ -41,10 +41,11 @@ variable "composer_config" {
disable_deployment = optional(bool)
environment_size = string
software_config = object({
- airflow_config_overrides = optional(any)
- pypi_packages = optional(any)
- env_variables = optional(map(string))
- image_version = string
+ airflow_config_overrides = optional(any)
+ pypi_packages = optional(any)
+ env_variables = optional(map(string))
+ image_version = string
+ cloud_data_lineage_integration = optional(bool, true)
})
workloads_config = object({
scheduler = object(
@@ -76,7 +77,8 @@ variable "composer_config" {
default = {
environment_size = "ENVIRONMENT_SIZE_SMALL"
software_config = {
- image_version = "composer-2-airflow-2"
+ image_version = "composer-2-airflow-2"
+ cloud_data_lineage_integration = true
}
workloads_config = null
}
@@ -110,8 +112,8 @@ variable "folder_ids" {
})
}
-variable "groups" {
- description = "Groups."
+variable "groups-dp" {
+ description = "Data Platform groups."
type = map(string)
default = {
data-analysts = "gcp-data-analysts"
@@ -187,6 +189,12 @@ variable "project_services" {
]
}
+variable "project_suffix" {
+ description = "Suffix used only for project ids."
+ type = string
+ default = null
+}
+
variable "region" {
description = "Region used for regional resources."
type = string
diff --git a/fast/stages/CLEANUP.md b/fast/stages/CLEANUP.md
index 83ad251cd9..f00580753a 100644
--- a/fast/stages/CLEANUP.md
+++ b/fast/stages/CLEANUP.md
@@ -26,6 +26,25 @@ done
terraform destroy
```
+## Stage 3 (Data Platform)
+
+Terraform refuses to delete non-empty GCS buckets and BigQuery datasets, so they need to be removed manually from the state.
+
+```bash
+cd $FAST_PWD/3-data-platform/dev/
+
+# remove GCS buckets and BQ dataset manually. Projects will be destroyed anyway
+for x in $(terraform state list | grep google_storage_bucket.bucket); do
+ terraform state rm "$x";
+done
+
+for x in $(terraform state list | grep google_bigquery_dataset); do
+ terraform state rm "$x";
+done
+
+terraform destroy
+```
+
## Stage 2 (Security)
```bash