Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix data platform roles #1725

Merged
merged 5 commits into from
Oct 4, 2023
Merged

Fix data platform roles #1725

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
92f5438
Fix Data Platform roles
lcaggio Oct 3, 2023
ce7d6cc
Fix README
lcaggio Oct 3, 2023
b93bb95
Fix blueprint tests
lcaggio Oct 3, 2023
5e5025c
Update cleanup dp steps
lcaggio Oct 3, 2023
7662a9b
Merge branch 'master' into lcaggio/dp-roles
ludoo Oct 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion blueprints/data-solutions/data-platform-foundations/02-load.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@
locals {
load_iam = {
data_engineers = [
"roles/dataflow.admin"
"roles/dataflow.admin",
"roles/dataflow.developer"
]
robots_dataflow_load = [
"roles/storage.objectAdmin"
Expand Down Expand Up @@ -54,6 +55,7 @@ module "load-project" {
"cloudkms.googleapis.com",
"compute.googleapis.com",
"dataflow.googleapis.com",
"datalineage.googleapis.com",
"dlp.googleapis.com",
"pubsub.googleapis.com",
"servicenetworking.googleapis.com",
Expand Down
12 changes: 8 additions & 4 deletions blueprints/data-solutions/data-platform-foundations/03-composer.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,16 +68,20 @@ module "orch-sa-cmp-0" {
}

resource "google_composer_environment" "orch-cmp-0" {
count = var.composer_config.disable_deployment == true ? 0 : 1
project = module.orch-project.project_id
name = "${var.prefix}-orc-cmp-0"
region = var.region
count = var.composer_config.disable_deployment == true ? 0 : 1
provider = google-beta
project = module.orch-project.project_id
name = "${var.prefix}-orc-cmp-0"
region = var.region
config {
software_config {
airflow_config_overrides = try(var.composer_config.software_config.airflow_config_overrides, null)
pypi_packages = try(var.composer_config.software_config.pypi_packages, null)
env_variables = local.env_variables
image_version = try(var.composer_config.software_config.image_version, null)
cloud_data_lineage_integration {
enabled = var.composer_config.software_config.cloud_data_lineage_integration
}
}
dynamic "workloads_config" {
for_each = (try(var.composer_config.workloads_config, null) != null ? { 1 = 1 } : {})
Expand Down
14 changes: 10 additions & 4 deletions blueprints/data-solutions/data-platform-foundations/03-orchestration.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,13 @@ locals {
"roles/bigquery.dataEditor",
"roles/bigquery.jobUser",
"roles/cloudbuild.builds.editor",
"roles/composer.admin",
"roles/composer.user",
"roles/composer.environmentAndStorageObjectAdmin",
"roles/iam.serviceAccountUser",
"roles/iap.httpsResourceAccessor",
"roles/serviceusage.serviceUsageConsumer"
"roles/serviceusage.serviceUsageConsumer",
"roles/storage.objectAdmin"
]
robots_cloudbuild = [
"roles/storage.objectAdmin"
Expand All @@ -33,6 +36,10 @@ locals {
"roles/composer.ServiceAgentV2Ext",
"roles/storage.objectAdmin"
]
sa_df_build = [
"roles/cloudbuild.serviceAgent",
"roles/storage.objectAdmin"
]
sa_load = [
"roles/artifactregistry.reader",
"roles/bigquery.dataEditor",
Expand Down Expand Up @@ -63,9 +70,7 @@ module "orch-project" {
)
iam = local.use_projects ? {} : local.orch_iam_auth
iam_bindings_additive = !local.use_projects ? {} : local.orch_iam_additive
compute_metadata = {
enable-oslogin = "false"
}

services = concat(var.project_services, [
"artifactregistry.googleapis.com",
"bigquery.googleapis.com",
Expand All @@ -79,6 +84,7 @@ module "orch-project" {
"containerregistry.googleapis.com",
"artifactregistry.googleapis.com",
"dataflow.googleapis.com",
"datalineage.googleapis.com",
"orgpolicy.googleapis.com",
"pubsub.googleapis.com",
"servicenetworking.googleapis.com",
Expand Down
9 changes: 6 additions & 3 deletions blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,14 @@ locals {
data_analysts = [
"roles/bigquery.dataViewer",
"roles/bigquery.jobUser",
"roles/datacatalog.tagTemplateViewer",
"roles/datacatalog.viewer",
"roles/storage.objectViewer"
]
data_engineers = [
"roles/bigquery.dataViewer",
"roles/bigquery.jobUser",
"roles/datacatalog.tagTemplateViewer",
"roles/datacatalog.viewer",
"roles/storage.objectViewer"
]
Expand All @@ -41,20 +43,21 @@ locals {
data_engineers = [
"roles/bigquery.dataViewer",
"roles/bigquery.jobUser",
"roles/datacatalog.tagTemplateViewer",
"roles/datacatalog.viewer",
"roles/storage.objectViewer"
]
sa_load = [
"roles/bigquery.dataOwner",
"roles/bigquery.jobUser",
"roles/storage.objectCreator"
]
sa_transf_bq = [
"roles/bigquery.dataViewer",
"roles/datacatalog.categoryAdmin"
]
sa_transf_df = [
"roles/bigquery.dataOwner",
"roles/bigquery.dataViewer",
"roles/bigquery.jobUser"
"roles/bigquery.dataViewer"
]
}
}
Expand Down
47 changes: 33 additions & 14 deletions blueprints/data-solutions/data-platform-foundations/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,7 @@ module "data-platform" {
}
prefix = "myprefix"
}
# tftest modules=43 resources=279
# tftest modules=43 resources=290
```

## Customizations
Expand All @@ -255,24 +255,43 @@ Once you have identified the required project granularity for your use case, we
The application layer is out of scope of this script. As a demo purpuse only, several Cloud Composer DAGs are provided. Demos will import data from the `drop off` area to the `Data Warehouse Confidential` dataset suing different features.

You can find examples in the `[demo](./demo)` folder.

## Cleanup

If you want to destroy the Data Platform deployment, follow these steps.

**ATTENTION**: The following procedure will permanently delete all of your data in an irreversible manner.

```bash
# remove GCS buckets and BQ dataset manually. Projects will be destroyed anyway
for x in $(terraform state list | grep google_storage_bucket.bucket); do
terraform state rm "$x";
done

for x in $(terraform state list | grep google_bigquery_dataset); do
terraform state rm "$x";
done

terraform destroy
```
<!-- BEGIN TFDOC -->
## Variables

| name | description | type | required | default |
|---|---|:---:|:---:|:---:|
| [organization_domain](variables.tf#L164) | Organization domain. | <code>string</code> | ✓ | |
| [prefix](variables.tf#L169) | Prefix used for resource names. | <code>string</code> | ✓ | |
| [project_config](variables.tf#L178) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | <code title="object&#40;&#123;&#10; billing_account_id &#61; optional&#40;string, null&#41;&#10; parent &#61; string&#10; project_ids &#61; optional&#40;object&#40;&#123;&#10; drop &#61; string&#10; load &#61; string&#10; orc &#61; string&#10; trf &#61; string&#10; dwh-lnd &#61; string&#10; dwh-cur &#61; string&#10; dwh-conf &#61; string&#10; common &#61; string&#10; exp &#61; string&#10; &#125;&#41;, &#123;&#10; drop &#61; &#34;drp&#34;&#10; load &#61; &#34;lod&#34;&#10; orc &#61; &#34;orc&#34;&#10; trf &#61; &#34;trf&#34;&#10; dwh-lnd &#61; &#34;dwh-lnd&#34;&#10; dwh-cur &#61; &#34;dwh-cur&#34;&#10; dwh-conf &#61; &#34;dwh-conf&#34;&#10; common &#61; &#34;cmn&#34;&#10; exp &#61; &#34;exp&#34;&#10; &#125;&#10; &#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ | |
| [composer_config](variables.tf#L17) | Cloud Composer config. | <code title="object&#40;&#123;&#10; disable_deployment &#61; optional&#40;bool&#41;&#10; environment_size &#61; optional&#40;string, &#34;ENVIRONMENT_SIZE_SMALL&#34;&#41;&#10; software_config &#61; optional&#40;&#10; object&#40;&#123;&#10; airflow_config_overrides &#61; optional&#40;any&#41;&#10; pypi_packages &#61; optional&#40;any&#41;&#10; env_variables &#61; optional&#40;map&#40;string&#41;&#41;&#10; image_version &#61; string&#10; &#125;&#41;,&#10; &#123; image_version &#61; &#34;composer-2-airflow-2&#34; &#125;&#10; &#41;&#10; workloads_config &#61; optional&#40;&#10; object&#40;&#123;&#10; scheduler &#61; optional&#40;&#10; object&#40;&#123;&#10; cpu &#61; number&#10; memory_gb &#61; number&#10; storage_gb &#61; number&#10; count &#61; number&#10; &#125;&#41;,&#10; &#123;&#10; cpu &#61; 0.5&#10; memory_gb &#61; 1.875&#10; storage_gb &#61; 1&#10; count &#61; 1&#10; &#125;&#10; &#41;&#10; web_server &#61; optional&#40;&#10; object&#40;&#123;&#10; cpu &#61; number&#10; memory_gb &#61; number&#10; storage_gb &#61; number&#10; &#125;&#41;,&#10; &#123;&#10; cpu &#61; 0.5&#10; memory_gb &#61; 1.875&#10; storage_gb &#61; 1&#10; &#125;&#10; &#41;&#10; worker &#61; optional&#40;&#10; object&#40;&#123;&#10; cpu &#61; number&#10; memory_gb &#61; number&#10; storage_gb &#61; number&#10; min_count &#61; number&#10; max_count &#61; number&#10; &#125;&#41;,&#10; &#123;&#10; cpu &#61; 0.5&#10; memory_gb &#61; 1.875&#10; storage_gb &#61; 1&#10; min_count &#61; 1&#10; max_count &#61; 3&#10; &#125;&#10; &#41;&#10; &#125;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code title="&#123;&#10; environment_size &#61; &#34;ENVIRONMENT_SIZE_SMALL&#34;&#10; software_config &#61; &#123;&#10; image_version &#61; &#34;composer-2-airflow-2&#34;&#10; &#125;&#10; workloads_config &#61; &#123;&#10; scheduler &#61; &#123;&#10; cpu &#61; 0.5&#10; memory_gb &#61; 1.875&#10; storage_gb &#61; 1&#10; count &#61; 1&#10; &#125;&#10; web_server &#61; &#123;&#10; cpu &#61; 0.5&#10; memory_gb &#61; 1.875&#10; storage_gb &#61; 1&#10; &#125;&#10; worker &#61; &#123;&#10; cpu &#61; 0.5&#10; memory_gb &#61; 1.875&#10; storage_gb &#61; 1&#10; min_count &#61; 1&#10; max_count &#61; 3&#10; &#125;&#10; &#125;&#10;&#125;">&#123;&#8230;&#125;</code> |
| [data_catalog_tags](variables.tf#L105) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | <code title="map&#40;object&#40;&#123;&#10; description &#61; optional&#40;string&#41;&#10; iam &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | | <code title="&#123;&#10; &#34;3_Confidential&#34; &#61; &#123;&#125;&#10; &#34;2_Private&#34; &#61; &#123;&#125;&#10; &#34;1_Sensitive&#34; &#61; &#123;&#125;&#10;&#125;">&#123;&#8230;&#125;</code> |
| [data_force_destroy](variables.tf#L119) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | <code>bool</code> | | <code>false</code> |
| [groups](variables.tf#L125) | User groups. | <code>map&#40;string&#41;</code> | | <code title="&#123;&#10; data-analysts &#61; &#34;gcp-data-analysts&#34;&#10; data-engineers &#61; &#34;gcp-data-engineers&#34;&#10; data-security &#61; &#34;gcp-data-security&#34;&#10;&#125;">&#123;&#8230;&#125;</code> |
| [location](variables.tf#L135) | Location used for multi-regional resources. | <code>string</code> | | <code>&#34;eu&#34;</code> |
| [network_config](variables.tf#L141) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | <code title="object&#40;&#123;&#10; host_project &#61; string&#10; network_self_link &#61; string&#10; subnet_self_links &#61; object&#40;&#123;&#10; load &#61; string&#10; transformation &#61; string&#10; orchestration &#61; string&#10; &#125;&#41;&#10; composer_ip_ranges &#61; object&#40;&#123;&#10; cloudsql &#61; string&#10; gke_master &#61; string&#10; &#125;&#41;&#10; composer_secondary_ranges &#61; object&#40;&#123;&#10; pods &#61; string&#10; services &#61; string&#10; &#125;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> |
| [project_services](variables.tf#L212) | List of core services enabled on all projects. | <code>list&#40;string&#41;</code> | | <code title="&#91;&#10; &#34;cloudresourcemanager.googleapis.com&#34;,&#10; &#34;iam.googleapis.com&#34;,&#10; &#34;serviceusage.googleapis.com&#34;,&#10; &#34;stackdriver.googleapis.com&#34;&#10;&#93;">&#91;&#8230;&#93;</code> |
| [project_suffix](variables.tf#L223) | Suffix used only for project ids. | <code>string</code> | | <code>null</code> |
| [region](variables.tf#L229) | Region used for regional resources. | <code>string</code> | | <code>&#34;europe-west1&#34;</code> |
| [service_encryption_keys](variables.tf#L235) | Cloud KMS to use to encrypt different services. Key location should match service region. | <code title="object&#40;&#123;&#10; bq &#61; string&#10; composer &#61; string&#10; dataflow &#61; string&#10; storage &#61; string&#10; pubsub &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> |
| [organization_domain](variables.tf#L165) | Organization domain. | <code>string</code> | ✓ | |
| [prefix](variables.tf#L170) | Prefix used for resource names. | <code>string</code> | ✓ | |
| [project_config](variables.tf#L179) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | <code title="object&#40;&#123;&#10; billing_account_id &#61; optional&#40;string, null&#41;&#10; parent &#61; string&#10; project_ids &#61; optional&#40;object&#40;&#123;&#10; drop &#61; string&#10; load &#61; string&#10; orc &#61; string&#10; trf &#61; string&#10; dwh-lnd &#61; string&#10; dwh-cur &#61; string&#10; dwh-conf &#61; string&#10; common &#61; string&#10; exp &#61; string&#10; &#125;&#41;, &#123;&#10; drop &#61; &#34;drp&#34;&#10; load &#61; &#34;lod&#34;&#10; orc &#61; &#34;orc&#34;&#10; trf &#61; &#34;trf&#34;&#10; dwh-lnd &#61; &#34;dwh-lnd&#34;&#10; dwh-cur &#61; &#34;dwh-cur&#34;&#10; dwh-conf &#61; &#34;dwh-conf&#34;&#10; common &#61; &#34;cmn&#34;&#10; exp &#61; &#34;exp&#34;&#10; &#125;&#10; &#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ | |
| [composer_config](variables.tf#L17) | Cloud Composer config. | <code title="object&#40;&#123;&#10; disable_deployment &#61; optional&#40;bool&#41;&#10; environment_size &#61; optional&#40;string, &#34;ENVIRONMENT_SIZE_SMALL&#34;&#41;&#10; software_config &#61; optional&#40;&#10; object&#40;&#123;&#10; airflow_config_overrides &#61; optional&#40;any&#41;&#10; pypi_packages &#61; optional&#40;any&#41;&#10; env_variables &#61; optional&#40;map&#40;string&#41;&#41;&#10; image_version &#61; string&#10; cloud_data_lineage_integration &#61; optional&#40;bool, true&#41;&#10; &#125;&#41;,&#10; &#123; image_version &#61; &#34;composer-2-airflow-2&#34; &#125;&#10; &#41;&#10; workloads_config &#61; optional&#40;&#10; object&#40;&#123;&#10; scheduler &#61; optional&#40;&#10; object&#40;&#123;&#10; cpu &#61; number&#10; memory_gb &#61; number&#10; storage_gb &#61; number&#10; count &#61; number&#10; &#125;&#41;,&#10; &#123;&#10; cpu &#61; 0.5&#10; memory_gb &#61; 1.875&#10; storage_gb &#61; 1&#10; count &#61; 1&#10; &#125;&#10; &#41;&#10; web_server &#61; optional&#40;&#10; object&#40;&#123;&#10; cpu &#61; number&#10; memory_gb &#61; number&#10; storage_gb &#61; number&#10; &#125;&#41;,&#10; &#123;&#10; cpu &#61; 0.5&#10; memory_gb &#61; 1.875&#10; storage_gb &#61; 1&#10; &#125;&#10; &#41;&#10; worker &#61; optional&#40;&#10; object&#40;&#123;&#10; cpu &#61; number&#10; memory_gb &#61; number&#10; storage_gb &#61; number&#10; min_count &#61; number&#10; max_count &#61; number&#10; &#125;&#41;,&#10; &#123;&#10; cpu &#61; 0.5&#10; memory_gb &#61; 1.875&#10; storage_gb &#61; 1&#10; min_count &#61; 1&#10; max_count &#61; 3&#10; &#125;&#10; &#41;&#10; &#125;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code title="&#123;&#10; environment_size &#61; &#34;ENVIRONMENT_SIZE_SMALL&#34;&#10; software_config &#61; &#123;&#10; image_version &#61; &#34;composer-2-airflow-2&#34;&#10; &#125;&#10; workloads_config &#61; &#123;&#10; scheduler &#61; &#123;&#10; cpu &#61; 0.5&#10; memory_gb &#61; 1.875&#10; storage_gb &#61; 1&#10; count &#61; 1&#10; &#125;&#10; web_server &#61; &#123;&#10; cpu &#61; 0.5&#10; memory_gb &#61; 1.875&#10; storage_gb &#61; 1&#10; &#125;&#10; worker &#61; &#123;&#10; cpu &#61; 0.5&#10; memory_gb &#61; 1.875&#10; storage_gb &#61; 1&#10; min_count &#61; 1&#10; max_count &#61; 3&#10; &#125;&#10; &#125;&#10;&#125;">&#123;&#8230;&#125;</code> |
| [data_catalog_tags](variables.tf#L106) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | <code title="map&#40;object&#40;&#123;&#10; description &#61; optional&#40;string&#41;&#10; iam &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | | <code title="&#123;&#10; &#34;3_Confidential&#34; &#61; &#123;&#125;&#10; &#34;2_Private&#34; &#61; &#123;&#125;&#10; &#34;1_Sensitive&#34; &#61; &#123;&#125;&#10;&#125;">&#123;&#8230;&#125;</code> |
| [data_force_destroy](variables.tf#L120) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | <code>bool</code> | | <code>false</code> |
| [groups](variables.tf#L126) | User groups. | <code>map&#40;string&#41;</code> | | <code title="&#123;&#10; data-analysts &#61; &#34;gcp-data-analysts&#34;&#10; data-engineers &#61; &#34;gcp-data-engineers&#34;&#10; data-security &#61; &#34;gcp-data-security&#34;&#10;&#125;">&#123;&#8230;&#125;</code> |
| [location](variables.tf#L136) | Location used for multi-regional resources. | <code>string</code> | | <code>&#34;eu&#34;</code> |
| [network_config](variables.tf#L142) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | <code title="object&#40;&#123;&#10; host_project &#61; string&#10; network_self_link &#61; string&#10; subnet_self_links &#61; object&#40;&#123;&#10; load &#61; string&#10; transformation &#61; string&#10; orchestration &#61; string&#10; &#125;&#41;&#10; composer_ip_ranges &#61; object&#40;&#123;&#10; cloudsql &#61; string&#10; gke_master &#61; string&#10; &#125;&#41;&#10; composer_secondary_ranges &#61; object&#40;&#123;&#10; pods &#61; string&#10; services &#61; string&#10; &#125;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> |
| [project_services](variables.tf#L213) | List of core services enabled on all projects. | <code>list&#40;string&#41;</code> | | <code title="&#91;&#10; &#34;cloudresourcemanager.googleapis.com&#34;,&#10; &#34;iam.googleapis.com&#34;,&#10; &#34;serviceusage.googleapis.com&#34;,&#10; &#34;stackdriver.googleapis.com&#34;&#10;&#93;">&#91;&#8230;&#93;</code> |
| [project_suffix](variables.tf#L224) | Suffix used only for project ids. | <code>string</code> | | <code>null</code> |
| [region](variables.tf#L230) | Region used for regional resources. | <code>string</code> | | <code>&#34;europe-west1&#34;</code> |
| [service_encryption_keys](variables.tf#L236) | Cloud KMS to use to encrypt different services. Key location should match service region. | <code title="object&#40;&#123;&#10; bq &#61; string&#10; composer &#61; string&#10; dataflow &#61; string&#10; storage &#61; string&#10; pubsub &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> |

## Outputs

Expand Down
Loading