From b73a5b6554a51c76130b9a607baf7a59ec67276d Mon Sep 17 00:00:00 2001 From: maunope Date: Wed, 6 Dec 2023 11:09:32 +0100 Subject: [PATCH 01/11] fist test --- .../cloud-operations/quota-monitoring/main.tf | 5 +- .../quota-monitoring/src/main.py | 62 +++++++++++++++++-- 2 files changed, 59 insertions(+), 8 deletions(-) diff --git a/blueprints/cloud-operations/quota-monitoring/main.tf b/blueprints/cloud-operations/quota-monitoring/main.tf index a49891c00c..307f038fc1 100644 --- a/blueprints/cloud-operations/quota-monitoring/main.tf +++ b/blueprints/cloud-operations/quota-monitoring/main.tf @@ -29,8 +29,9 @@ module "project" { parent = try(var.project_create_config.parent, null) project_create = var.project_create_config != null services = [ - "compute.googleapis.com", - "cloudfunctions.googleapis.com" + "cloudasset.googleapis.com", + "cloudfunctions.googleapis.com", + "compute.googleapis.com" ] } diff --git a/blueprints/cloud-operations/quota-monitoring/src/main.py b/blueprints/cloud-operations/quota-monitoring/src/main.py index 5a84536417..c4b37583b9 100755 --- a/blueprints/cloud-operations/quota-monitoring/src/main.py +++ b/blueprints/cloud-operations/quota-monitoring/src/main.py @@ -32,6 +32,8 @@ import requests.exceptions from google.auth.transport.requests import AuthorizedSession +from google.cloud import asset_v1 +from google.protobuf.json_format import MessageToDict BASE = 'custom.googleapis.com/quota' HTTP = AuthorizedSession(google.auth.default()[0]) @@ -39,6 +41,9 @@ URL_PROJECT = 'https://compute.googleapis.com/compute/v1/projects/{}' URL_REGION = 'https://compute.googleapis.com/compute/v1/projects/{}/regions/{}' URL_TS = 'https://monitoring.googleapis.com/v3/projects/{}/timeSeries' +URL_DISCOVERY='https://cloudasset.googleapis.com/v1/{}/assets?assetTypes=cloudresourcemanager.googleapis.com%2FProject&contentType=RESOURCE&pageSize=100' + + _Quota = collections.namedtuple('_Quota', 'project region tstamp metric limit usage') @@ -161,12 +166,18 @@ def get_quotas(project, region='global'): yield Quota(project, region, ts, **quota) +#def get_discovered_projects(discovery_root) +# if discovery_root.partition('/')[0] not in ('folders', 'organizations'): +# raise SystemExit('Invalid discovery root.') + @click.command() @click.argument('project-id', required=True) @click.option( - '--project-ids', multiple=True, help= - 'Project ids to monitor (multiple). Defaults to monitoring project if not set.' -) + '--discovery-root', '-dr', required=False, + help='Root node for asset discovery, organizations/nnn or folders/nnn.') +@click.option( + '--project-ids', multiple=True, + help='Project ids to monitor (multiple). Defaults to monitoring project if not set, values are appended to those found under discovery-root') @click.option('--regions', multiple=True, help='Regions (multiple). Defaults to "global" if not set.') @click.option('--include', multiple=True, @@ -175,11 +186,11 @@ def get_quotas(project, region='global'): help='Exclude quotas starting with keyword (multiple).') @click.option('--dry-run', is_flag=True, help='Do not write metrics.') @click.option('--verbose', is_flag=True, help='Verbose output.') -def main_cli(project_id=None, project_ids=None, regions=None, include=None, +def main_cli(project_id=None, discovery_root=None, project_ids=None, regions=None, include=None, exclude=None, dry_run=False, verbose=False): 'Fetch GCE quotas and writes them as custom metrics to Stackdriver.' try: - _main(project_id, project_ids, regions, include, exclude, dry_run, verbose) + _main(project_id,discovery_root,project_ids, regions, include, exclude, dry_run, verbose) except RuntimeError as e: logging.exception(f'exception raised: {e.args[0]}') @@ -193,11 +204,50 @@ def main(event, context): raise -def _main(monitoring_project, projects=None, regions=None, include=None, +def _main(monitoring_project, discovery_root=None, projects=None, regions=None, include=None, exclude=None, dry_run=False, verbose=False): """Module entry point used by cli and cloud function wrappers.""" configure_logging(verbose=verbose) + + + # Create the Cloud Asset Inventory client + #client = asset_v1.AssetServiceClient() + + # Define the parent resource (organization) + parent = f"organizations/"+discovery_root + + # Define the asset types to list (projects) + #asset_types = ["organization.googleapis.com/Project"] + + # Build the query + #query = asset_v1.types.ListAssetsRequest() + #query.content_type="RESOURCE" + #query.parent = parent + #query.asset_types = ["cloudresourcemanager.googleapis.com/Project"] + #query.page_size=100 + request = HTTPRequest(URL_DISCOVERY.format(parent)) + print(request) + resp = fetch(request) + print(resp) + + + last_assets_page_reached=False + discovered_projects=[] + # List projects + while not last_assets_page_reached: + list_assets_results = MessageToDict(client.list_assets(query)._pb) + if not "nextPageToken" in list_assets_results: + last_assets_page_reached=True + + for asset in list_assets_results["assets"]: + if asset["assetType"] in query.asset_types: + discovered_projects.append(asset["resource"]["data"]["projectId"]) + + projects = projects or [monitoring_project] + projects= (projects + tuple(set(discovered_projects) - set(projects))) + + regions = regions or ['global'] include = set(include or []) exclude = set(exclude or []) From 2ba2ba0bed40f02e31a9fb355d84fbe8e510430e Mon Sep 17 00:00:00 2001 From: maunope Date: Wed, 6 Dec 2023 12:25:53 +0100 Subject: [PATCH 02/11] dev complete --- .../quota-monitoring/src/main.py | 61 ++++++------------- 1 file changed, 20 insertions(+), 41 deletions(-) diff --git a/blueprints/cloud-operations/quota-monitoring/src/main.py b/blueprints/cloud-operations/quota-monitoring/src/main.py index c4b37583b9..0d64b2e934 100755 --- a/blueprints/cloud-operations/quota-monitoring/src/main.py +++ b/blueprints/cloud-operations/quota-monitoring/src/main.py @@ -41,7 +41,7 @@ URL_PROJECT = 'https://compute.googleapis.com/compute/v1/projects/{}' URL_REGION = 'https://compute.googleapis.com/compute/v1/projects/{}/regions/{}' URL_TS = 'https://monitoring.googleapis.com/v3/projects/{}/timeSeries' -URL_DISCOVERY='https://cloudasset.googleapis.com/v1/{}/assets?assetTypes=cloudresourcemanager.googleapis.com%2FProject&contentType=RESOURCE&pageSize=100' +URL_DISCOVERY='https://cloudasset.googleapis.com/v1/{}/assets?assetTypes=cloudresourcemanager.googleapis.com%2FProject&contentType=RESOURCE&pageSize=1&pageToken={}' @@ -166,10 +166,6 @@ def get_quotas(project, region='global'): yield Quota(project, region, ts, **quota) -#def get_discovered_projects(discovery_root) -# if discovery_root.partition('/')[0] not in ('folders', 'organizations'): -# raise SystemExit('Invalid discovery root.') - @click.command() @click.argument('project-id', required=True) @click.option( @@ -209,43 +205,26 @@ def _main(monitoring_project, discovery_root=None, projects=None, regions=None, """Module entry point used by cli and cloud function wrappers.""" configure_logging(verbose=verbose) - - # Create the Cloud Asset Inventory client - #client = asset_v1.AssetServiceClient() - - # Define the parent resource (organization) - parent = f"organizations/"+discovery_root - - # Define the asset types to list (projects) - #asset_types = ["organization.googleapis.com/Project"] - - # Build the query - #query = asset_v1.types.ListAssetsRequest() - #query.content_type="RESOURCE" - #query.parent = parent - #query.asset_types = ["cloudresourcemanager.googleapis.com/Project"] - #query.page_size=100 - request = HTTPRequest(URL_DISCOVERY.format(parent)) - print(request) - resp = fetch(request) - print(resp) - - - last_assets_page_reached=False - discovered_projects=[] - # List projects - while not last_assets_page_reached: - list_assets_results = MessageToDict(client.list_assets(query)._pb) - if not "nextPageToken" in list_assets_results: - last_assets_page_reached=True + + projects = projects or {monitoring_project} + + if (discovery_root): + if discovery_root.partition('/')[0] not in ('folders', 'organizations'): + raise SystemExit(f'Invalid discovery root {discovery_root}.') - for asset in list_assets_results["assets"]: - if asset["assetType"] in query.asset_types: - discovered_projects.append(asset["resource"]["data"]["projectId"]) - - - projects = projects or [monitoring_project] - projects= (projects + tuple(set(discovered_projects) - set(projects))) + last_assets_page_reached=False + discovered_projects=[] + nextPageToken="" + while not last_assets_page_reached: + list_assets_results = fetch(HTTPRequest(URL_DISCOVERY.format(discovery_root,nextPageToken))) + if "assets" in list_assets_results: + for asset in list_assets_results["assets"]: + if (asset["resource"]["data"]["lifecycleState"] == "ACTIVE"): + discovered_projects.append(asset["resource"]["data"]["projectId"]) + last_assets_page_reached = False if "nextPageToken" in list_assets_results else True + nextPageToken="" if last_assets_page_reached==True else list_assets_results["nextPageToken"] + #merge discovered projects with those received as an input + projects= tuple(projects)+ tuple(set(discovered_projects) - set(projects)) regions = regions or ['global'] From 0defa77cd585ea5d9be671338ce0f3e4a4a9b139 Mon Sep 17 00:00:00 2001 From: maunope Date: Wed, 6 Dec 2023 14:59:51 +0100 Subject: [PATCH 03/11] update tf with permissions, enabled APIs and discovery root management --- .../cloud-operations/quota-monitoring/main.tf | 52 +++++++++++++++++++ .../quota-monitoring/src/main.py | 2 - .../quota-monitoring/variables.tf | 3 +- 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/blueprints/cloud-operations/quota-monitoring/main.tf b/blueprints/cloud-operations/quota-monitoring/main.tf index 307f038fc1..4e9b1c9508 100644 --- a/blueprints/cloud-operations/quota-monitoring/main.tf +++ b/blueprints/cloud-operations/quota-monitoring/main.tf @@ -20,6 +20,8 @@ locals { ? [var.project_id] : var.quota_config.projects ) + discovery_root_type = split("/", var.quota_config["discovery_root"])[0] + discovery_root_id = split("/", var.quota_config["discovery_root"])[1] } module "project" { @@ -30,7 +32,9 @@ module "project" { project_create = var.project_create_config != null services = [ "cloudasset.googleapis.com", + "cloudbuild.googleapis.com", "cloudfunctions.googleapis.com", + "cloudscheduler.googleapis.com", "compute.googleapis.com" ] } @@ -82,6 +86,54 @@ resource "google_cloud_scheduler_job" "default" { } } + + +resource "google_organization_iam_member" "org_asset_viewer" { + count= local.discovery_root_type=="organizations"?1:0 + org_id = local.discovery_root_id + role = "roles/cloudasset.viewer" + member = module.cf.service_account_iam_email +} + +resource "google_organization_iam_member" "org_network_viewer" { + count= local.discovery_root_type=="organizations"?1:0 + org_id = local.discovery_root_id + role = "roles/compute.networkViewer" + member = module.cf.service_account_iam_email +} + +resource "google_organization_iam_member" "org_quota_viewer" { + count= local.discovery_root_type=="organizations"?1:0 + org_id = local.discovery_root_id + role = "roles/servicemanagement.quotaViewer" + member = module.cf.service_account_iam_email +} + +resource "google_folder_iam_member" "folder_asset_viewer" { + count= local.discovery_root_type=="folders"?1:0 + folder = local.discovery_root_id + role = "roles/cloudasset.viewer" + member = module.cf.service_account_iam_email +} + +resource "google_folder_iam_member" "folder_network_viewer" { + count= local.discovery_root_type=="folders"?1:0 + folder = local.discovery_root_id + role = "roles/compute.networkViewer" + member = module.cf.service_account_iam_email +} + +resource "google_folder_iam_member" "folder_quota_viewer" { + count= local.discovery_root_type=="folders"?1:0 + folder = local.discovery_root_id + role = "roles/servicemanagement.quotaViewer" + member = module.cf.service_account_iam_email +} + + + + + resource "google_project_iam_member" "metric_writer" { project = module.project.project_id role = "roles/monitoring.metricWriter" diff --git a/blueprints/cloud-operations/quota-monitoring/src/main.py b/blueprints/cloud-operations/quota-monitoring/src/main.py index 0d64b2e934..1681e5830a 100755 --- a/blueprints/cloud-operations/quota-monitoring/src/main.py +++ b/blueprints/cloud-operations/quota-monitoring/src/main.py @@ -32,8 +32,6 @@ import requests.exceptions from google.auth.transport.requests import AuthorizedSession -from google.cloud import asset_v1 -from google.protobuf.json_format import MessageToDict BASE = 'custom.googleapis.com/quota' HTTP = AuthorizedSession(google.auth.default()[0]) diff --git a/blueprints/cloud-operations/quota-monitoring/variables.tf b/blueprints/cloud-operations/quota-monitoring/variables.tf index 21cf765395..95b500962a 100644 --- a/blueprints/cloud-operations/quota-monitoring/variables.tf +++ b/blueprints/cloud-operations/quota-monitoring/variables.tf @@ -63,10 +63,11 @@ variable "quota_config" { "a2", "c2", "c2d", "committed", "g2", "interconnect", "m1", "m2", "m3", "nvidia", "preemptible" ]) + discovery_root = optional(string) + dry_run = optional(bool, false) include = optional(list(string)) projects = optional(list(string)) regions = optional(list(string)) - dry_run = optional(bool, false) verbose = optional(bool, false) }) nullable = false From aee642e994ec78a85cdeb16a9ac765accd62125c Mon Sep 17 00:00:00 2001 From: maunope Date: Wed, 6 Dec 2023 15:02:57 +0100 Subject: [PATCH 04/11] updated readme --- blueprints/cloud-operations/quota-monitoring/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/blueprints/cloud-operations/quota-monitoring/README.md b/blueprints/cloud-operations/quota-monitoring/README.md index aeaa9b4d91..0e645e98d5 100644 --- a/blueprints/cloud-operations/quota-monitoring/README.md +++ b/blueprints/cloud-operations/quota-monitoring/README.md @@ -38,9 +38,10 @@ The region, location of the bundle used to deploy the function, and scheduling f The `quota_config` variable mirrors the arguments accepted by the Python program, and allows configuring several different aspects of its behaviour: +- `quota_config.discover_root` organization or folder to be used to discover all underlying projects to track quotas for - `quota_config.exclude` do not generate metrics for quotas matching prefixes listed here - `quota_config.include` only generate metrics for quotas matching prefixes listed here -- `quota_config.projects` projects to track quotas for, defaults to the project where metrics are stored +- `quota_config.projects` projects to track quotas for, defaults to the project where metrics are stored, if projects are automatically discovered, those in this list are appended - `quota_config.regions` regions to track quotas for, defaults to the `global` region for project-level quotas - `dry_run` do not write actual metrics - `verbose` increase logging verbosity From ba56eb6ddbc0ad1b08d65dba471d2e89397b5a5d Mon Sep 17 00:00:00 2001 From: maunope Date: Mon, 11 Dec 2023 15:54:51 +0100 Subject: [PATCH 05/11] moved projects discovery to a separate method --- .../quota-monitoring/src/main.py | 37 +++++++++++-------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/blueprints/cloud-operations/quota-monitoring/src/main.py b/blueprints/cloud-operations/quota-monitoring/src/main.py index 1681e5830a..dea6591bf7 100755 --- a/blueprints/cloud-operations/quota-monitoring/src/main.py +++ b/blueprints/cloud-operations/quota-monitoring/src/main.py @@ -115,6 +115,25 @@ def configure_logging(verbose=True): warnings.filterwarnings('ignore', r'.*end user credentials.*', UserWarning) +def discover_projects(discovery_root): + 'Discovers projects under a folder or organization' + if discovery_root.partition('/')[0] not in ('folders', 'organizations'): + raise SystemExit(f'Invalid discovery root {discovery_root}.') + last_assets_page_reached=False + discovered_projects=[] + nextPageToken="" + while not last_assets_page_reached: + list_assets_results = fetch(HTTPRequest(URL_DISCOVERY.format(discovery_root,nextPageToken))) + if "assets" in list_assets_results: + for asset in list_assets_results["assets"]: + if (asset["resource"]["data"]["lifecycleState"] == "ACTIVE"): + discovered_projects.append(asset["resource"]["data"]["projectId"]) + last_assets_page_reached = False if "nextPageToken" in list_assets_results else True + nextPageToken="" if last_assets_page_reached==True else list_assets_results["nextPageToken"] + return discover_projects + + + def fetch(request, delete=False): 'Minimal HTTP client interface for API calls.' logging.debug(f'fetch {"POST" if request.data else "GET"} {request.url}') @@ -204,24 +223,10 @@ def _main(monitoring_project, discovery_root=None, projects=None, regions=None, configure_logging(verbose=verbose) + # default to monitoring scope project if projects parameter is not passed, then merge the list with discovered projects, if any projects = projects or {monitoring_project} - if (discovery_root): - if discovery_root.partition('/')[0] not in ('folders', 'organizations'): - raise SystemExit(f'Invalid discovery root {discovery_root}.') - - last_assets_page_reached=False - discovered_projects=[] - nextPageToken="" - while not last_assets_page_reached: - list_assets_results = fetch(HTTPRequest(URL_DISCOVERY.format(discovery_root,nextPageToken))) - if "assets" in list_assets_results: - for asset in list_assets_results["assets"]: - if (asset["resource"]["data"]["lifecycleState"] == "ACTIVE"): - discovered_projects.append(asset["resource"]["data"]["projectId"]) - last_assets_page_reached = False if "nextPageToken" in list_assets_results else True - nextPageToken="" if last_assets_page_reached==True else list_assets_results["nextPageToken"] - #merge discovered projects with those received as an input + discovered_projects=discover_projects(discovery_root) projects= tuple(projects)+ tuple(set(discovered_projects) - set(projects)) From 6d82a87be20f6bbbce5b8537ed03422243652355 Mon Sep 17 00:00:00 2001 From: Ludo Date: Tue, 12 Dec 2023 13:11:45 +0100 Subject: [PATCH 06/11] reviewed Mauri's changes --- .../quota-monitoring/README.md | 10 ++- .../cloud-operations/quota-monitoring/main.tf | 42 +++++------ .../quota-monitoring/src/main.py | 69 +++++++++---------- .../quota-monitoring/variables.tf | 18 +++-- 4 files changed, 72 insertions(+), 67 deletions(-) diff --git a/blueprints/cloud-operations/quota-monitoring/README.md b/blueprints/cloud-operations/quota-monitoring/README.md index 0e645e98d5..2e8ddd9d42 100644 --- a/blueprints/cloud-operations/quota-monitoring/README.md +++ b/blueprints/cloud-operations/quota-monitoring/README.md @@ -38,7 +38,7 @@ The region, location of the bundle used to deploy the function, and scheduling f The `quota_config` variable mirrors the arguments accepted by the Python program, and allows configuring several different aspects of its behaviour: -- `quota_config.discover_root` organization or folder to be used to discover all underlying projects to track quotas for +- `quota_config.discover_root` organization or folder to be used to discover all underlying projects to track quotas for, in `organizations/nnnnn` or `folders/nnnnn` format - `quota_config.exclude` do not generate metrics for quotas matching prefixes listed here - `quota_config.include` only generate metrics for quotas matching prefixes listed here - `quota_config.projects` projects to track quotas for, defaults to the project where metrics are stored, if projects are automatically discovered, those in this list are appended @@ -55,7 +55,6 @@ Clone this repository or [open it in cloud shell](https://ssh.cloud.google.com/c - `terraform init` - `terraform apply -var project_id=my-project-id` - ## Variables | name | description | type | required | default | @@ -65,10 +64,9 @@ Clone this repository or [open it in cloud shell](https://ssh.cloud.google.com/c | [bundle_path](variables.tf#L33) | Path used to write the intermediate Cloud Function code bundle. | string | | "./bundle.zip" | | [name](variables.tf#L39) | Arbitrary string used to name created resources. | string | | "quota-monitor" | | [project_create_config](variables.tf#L45) | Create project instead of using an existing one. | object({…}) | | null | -| [quota_config](variables.tf#L59) | Cloud function configuration. | object({…}) | | {} | -| [region](variables.tf#L76) | Compute region used in the example. | string | | "europe-west1" | -| [schedule_config](variables.tf#L82) | Schedule timer configuration in crontab format. | string | | "0 * * * *" | - +| [quota_config](variables.tf#L59) | Cloud function configuration. | object({…}) | | {} | +| [region](variables.tf#L85) | Compute region used in the example. | string | | "europe-west1" | +| [schedule_config](variables.tf#L91) | Schedule timer configuration in crontab format. | string | | "0 * * * *" | ## Test diff --git a/blueprints/cloud-operations/quota-monitoring/main.tf b/blueprints/cloud-operations/quota-monitoring/main.tf index 4e9b1c9508..44983fd01b 100644 --- a/blueprints/cloud-operations/quota-monitoring/main.tf +++ b/blueprints/cloud-operations/quota-monitoring/main.tf @@ -21,7 +21,7 @@ locals { : var.quota_config.projects ) discovery_root_type = split("/", var.quota_config["discovery_root"])[0] - discovery_root_id = split("/", var.quota_config["discovery_root"])[1] + discovery_root_id = split("/", var.quota_config["discovery_root"])[1] } module "project" { @@ -86,48 +86,48 @@ resource "google_cloud_scheduler_job" "default" { } } - - resource "google_organization_iam_member" "org_asset_viewer" { - count= local.discovery_root_type=="organizations"?1:0 + count = local.discovery_root_type == "organizations" ? 1 : 0 org_id = local.discovery_root_id - role = "roles/cloudasset.viewer" - member = module.cf.service_account_iam_email + role = "roles/cloudasset.viewer" + member = module.cf.service_account_iam_email } +# TODO: document why this role is needed + resource "google_organization_iam_member" "org_network_viewer" { - count= local.discovery_root_type=="organizations"?1:0 + count = local.discovery_root_type == "organizations" ? 1 : 0 org_id = local.discovery_root_id - role = "roles/compute.networkViewer" - member = module.cf.service_account_iam_email + role = "roles/compute.networkViewer" + member = module.cf.service_account_iam_email } resource "google_organization_iam_member" "org_quota_viewer" { - count= local.discovery_root_type=="organizations"?1:0 + count = local.discovery_root_type == "organizations" ? 1 : 0 org_id = local.discovery_root_id - role = "roles/servicemanagement.quotaViewer" - member = module.cf.service_account_iam_email + role = "roles/servicemanagement.quotaViewer" + member = module.cf.service_account_iam_email } resource "google_folder_iam_member" "folder_asset_viewer" { - count= local.discovery_root_type=="folders"?1:0 + count = local.discovery_root_type == "folders" ? 1 : 0 folder = local.discovery_root_id - role = "roles/cloudasset.viewer" - member = module.cf.service_account_iam_email + role = "roles/cloudasset.viewer" + member = module.cf.service_account_iam_email } resource "google_folder_iam_member" "folder_network_viewer" { - count= local.discovery_root_type=="folders"?1:0 + count = local.discovery_root_type == "folders" ? 1 : 0 folder = local.discovery_root_id - role = "roles/compute.networkViewer" - member = module.cf.service_account_iam_email + role = "roles/compute.networkViewer" + member = module.cf.service_account_iam_email } resource "google_folder_iam_member" "folder_quota_viewer" { - count= local.discovery_root_type=="folders"?1:0 + count = local.discovery_root_type == "folders" ? 1 : 0 folder = local.discovery_root_id - role = "roles/servicemanagement.quotaViewer" - member = module.cf.service_account_iam_email + role = "roles/servicemanagement.quotaViewer" + member = module.cf.service_account_iam_email } diff --git a/blueprints/cloud-operations/quota-monitoring/src/main.py b/blueprints/cloud-operations/quota-monitoring/src/main.py index dea6591bf7..ba1e281e46 100755 --- a/blueprints/cloud-operations/quota-monitoring/src/main.py +++ b/blueprints/cloud-operations/quota-monitoring/src/main.py @@ -39,9 +39,9 @@ URL_PROJECT = 'https://compute.googleapis.com/compute/v1/projects/{}' URL_REGION = 'https://compute.googleapis.com/compute/v1/projects/{}/regions/{}' URL_TS = 'https://monitoring.googleapis.com/v3/projects/{}/timeSeries' -URL_DISCOVERY='https://cloudasset.googleapis.com/v1/{}/assets?assetTypes=cloudresourcemanager.googleapis.com%2FProject&contentType=RESOURCE&pageSize=1&pageToken={}' - - +URL_DISCOVERY = ('https://cloudasset.googleapis.com/v1/{}/assets?' + 'assetTypes=cloudresourcemanager.googleapis.com%2FProject&' + 'contentType=RESOURCE&pageSize=1&pageToken={}') _Quota = collections.namedtuple('_Quota', 'project region tstamp metric limit usage') @@ -83,8 +83,8 @@ def _api_format(self, name, value): else: d['valueType'] = 'INT64' d['points'][0]['value'] = {'int64Value': value} - # remove this label if cardinality gets too high - d['metric']['labels']['quota'] = f'{self.usage}/{self.limit}' + # re-enable the following line if cardinality is not a problem + # d['metric']['labels']['quota'] = f'{self.usage}/{self.limit}' return d @property @@ -95,7 +95,7 @@ def timeseries(self): ratio = 0 yield self._api_format('ratio', ratio) yield self._api_format('usage', self.usage) - # yield self._api_format('limit', self.limit) + yield self._api_format('limit', self.limit) def batched(iterable, n): @@ -116,22 +116,21 @@ def configure_logging(verbose=True): def discover_projects(discovery_root): - 'Discovers projects under a folder or organization' + 'Discovers projects under a folder or organization.' if discovery_root.partition('/')[0] not in ('folders', 'organizations'): raise SystemExit(f'Invalid discovery root {discovery_root}.') - last_assets_page_reached=False - discovered_projects=[] - nextPageToken="" + last_assets_page_reached = False + next_page_token = '' while not last_assets_page_reached: - list_assets_results = fetch(HTTPRequest(URL_DISCOVERY.format(discovery_root,nextPageToken))) - if "assets" in list_assets_results: - for asset in list_assets_results["assets"]: - if (asset["resource"]["data"]["lifecycleState"] == "ACTIVE"): - discovered_projects.append(asset["resource"]["data"]["projectId"]) - last_assets_page_reached = False if "nextPageToken" in list_assets_results else True - nextPageToken="" if last_assets_page_reached==True else list_assets_results["nextPageToken"] - return discover_projects - + list_assets_results = fetch( + HTTPRequest(URL_DISCOVERY.format(discovery_root, next_page_token))) + if 'assets' in list_assets_results: + for asset in list_assets_results['assets']: + if (asset['resource']['data']['lifecycleState'] == 'ACTIVE'): + yield asset['resource']['data']['projectId'] + next_page_token = list_assets_results.get('nextPageToken') + if not next_page_token: + break def fetch(request, delete=False): @@ -186,11 +185,13 @@ def get_quotas(project, region='global'): @click.command() @click.argument('project-id', required=True) @click.option( - '--discovery-root', '-dr', required=False, - help='Root node for asset discovery, organizations/nnn or folders/nnn.') + '--discovery-root', '-dr', required=False, help= + 'Root node used to dynamically fetch projects, in organizations/nnn or folders/nnn format.' +) @click.option( - '--project-ids', multiple=True, - help='Project ids to monitor (multiple). Defaults to monitoring project if not set, values are appended to those found under discovery-root') + '--project-ids', multiple=True, help= + 'Project ids to monitor (multiple). Defaults to monitoring project if not set, values are appended to those found under discovery-root' +) @click.option('--regions', multiple=True, help='Regions (multiple). Defaults to "global" if not set.') @click.option('--include', multiple=True, @@ -199,11 +200,13 @@ def get_quotas(project, region='global'): help='Exclude quotas starting with keyword (multiple).') @click.option('--dry-run', is_flag=True, help='Do not write metrics.') @click.option('--verbose', is_flag=True, help='Verbose output.') -def main_cli(project_id=None, discovery_root=None, project_ids=None, regions=None, include=None, - exclude=None, dry_run=False, verbose=False): +def main_cli(project_id=None, discovery_root=None, project_ids=None, + regions=None, include=None, exclude=None, dry_run=False, + verbose=False): 'Fetch GCE quotas and writes them as custom metrics to Stackdriver.' try: - _main(project_id,discovery_root,project_ids, regions, include, exclude, dry_run, verbose) + _main(project_id, discovery_root, project_ids, regions, include, exclude, + dry_run, verbose) except RuntimeError as e: logging.exception(f'exception raised: {e.args[0]}') @@ -217,22 +220,18 @@ def main(event, context): raise -def _main(monitoring_project, discovery_root=None, projects=None, regions=None, include=None, - exclude=None, dry_run=False, verbose=False): +def _main(monitoring_project, discovery_root=None, projects=None, regions=None, + include=None, exclude=None, dry_run=False, verbose=False): """Module entry point used by cli and cloud function wrappers.""" configure_logging(verbose=verbose) - # default to monitoring scope project if projects parameter is not passed, then merge the list with discovered projects, if any - projects = projects or {monitoring_project} - if (discovery_root): - discovered_projects=discover_projects(discovery_root) - projects= tuple(projects)+ tuple(set(discovered_projects) - set(projects)) - - regions = regions or ['global'] include = set(include or []) exclude = set(exclude or []) + projects = projects or [monitoring_project] + if (discovery_root): + projects = set(projects + list(discover_projects(discovery_root))) for k in ('monitoring_project', 'projects', 'regions', 'include', 'exclude'): logging.debug(f'{k} {locals().get(k)}') timeseries = [] diff --git a/blueprints/cloud-operations/quota-monitoring/variables.tf b/blueprints/cloud-operations/quota-monitoring/variables.tf index 95b500962a..4aee3b543e 100644 --- a/blueprints/cloud-operations/quota-monitoring/variables.tf +++ b/blueprints/cloud-operations/quota-monitoring/variables.tf @@ -64,14 +64,22 @@ variable "quota_config" { "nvidia", "preemptible" ]) discovery_root = optional(string) - dry_run = optional(bool, false) - include = optional(list(string)) - projects = optional(list(string)) - regions = optional(list(string)) - verbose = optional(bool, false) + dry_run = optional(bool, false) + include = optional(list(string)) + projects = optional(list(string)) + regions = optional(list(string)) + verbose = optional(bool, false) }) nullable = false default = {} + validation { + condition = ( + var.quota_config.discovery_root == null || + startswith(var.quota_config.discovery_root, "folders/") || + startswith(var.quota_config.discovery_root, "organizations/") + ) + error_message = "non-null discovery root needs to start with folders/ or organizations/" + } } variable "region" { From bf6331e2ed5ef52d55afe28b4acba3ac980f8bb8 Mon Sep 17 00:00:00 2001 From: Ludo Date: Tue, 12 Dec 2023 14:02:18 +0100 Subject: [PATCH 07/11] add missing lines from last change --- blueprints/cloud-operations/quota-monitoring/src/main.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/blueprints/cloud-operations/quota-monitoring/src/main.py b/blueprints/cloud-operations/quota-monitoring/src/main.py index ba1e281e46..36a2d478b0 100755 --- a/blueprints/cloud-operations/quota-monitoring/src/main.py +++ b/blueprints/cloud-operations/quota-monitoring/src/main.py @@ -119,9 +119,8 @@ def discover_projects(discovery_root): 'Discovers projects under a folder or organization.' if discovery_root.partition('/')[0] not in ('folders', 'organizations'): raise SystemExit(f'Invalid discovery root {discovery_root}.') - last_assets_page_reached = False next_page_token = '' - while not last_assets_page_reached: + while True: list_assets_results = fetch( HTTPRequest(URL_DISCOVERY.format(discovery_root, next_page_token))) if 'assets' in list_assets_results: From c58965093cfbdb2c36bec9ad002e8710a77bbcf7 Mon Sep 17 00:00:00 2001 From: maunope Date: Tue, 12 Dec 2023 17:31:08 +0100 Subject: [PATCH 08/11] - fixed discovery page size to 100 - removed last_asset_page_reached var from discover_projects - added cast to list for projects var in _main, to make the script work both using CLI and pub/sub --- blueprints/cloud-operations/quota-monitoring/README.md | 2 +- blueprints/cloud-operations/quota-monitoring/main.tf | 3 ++- blueprints/cloud-operations/quota-monitoring/src/main.py | 7 +++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/blueprints/cloud-operations/quota-monitoring/README.md b/blueprints/cloud-operations/quota-monitoring/README.md index 2e8ddd9d42..19747ece85 100644 --- a/blueprints/cloud-operations/quota-monitoring/README.md +++ b/blueprints/cloud-operations/quota-monitoring/README.md @@ -41,7 +41,7 @@ The `quota_config` variable mirrors the arguments accepted by the Python program - `quota_config.discover_root` organization or folder to be used to discover all underlying projects to track quotas for, in `organizations/nnnnn` or `folders/nnnnn` format - `quota_config.exclude` do not generate metrics for quotas matching prefixes listed here - `quota_config.include` only generate metrics for quotas matching prefixes listed here -- `quota_config.projects` projects to track quotas for, defaults to the project where metrics are stored, if projects are automatically discovered, those in this list are appended +- `quota_config.projects` projects to track quotas for, defaults to the project where metrics are stored, if projects are automatically discovered, those in this list are appended. - `quota_config.regions` regions to track quotas for, defaults to the `global` region for project-level quotas - `dry_run` do not write actual metrics - `verbose` increase logging verbosity diff --git a/blueprints/cloud-operations/quota-monitoring/main.tf b/blueprints/cloud-operations/quota-monitoring/main.tf index 44983fd01b..c0dbf5bfe8 100644 --- a/blueprints/cloud-operations/quota-monitoring/main.tf +++ b/blueprints/cloud-operations/quota-monitoring/main.tf @@ -93,8 +93,8 @@ resource "google_organization_iam_member" "org_asset_viewer" { member = module.cf.service_account_iam_email } -# TODO: document why this role is needed +# role with the least privilege including compute.projects.get permission resource "google_organization_iam_member" "org_network_viewer" { count = local.discovery_root_type == "organizations" ? 1 : 0 org_id = local.discovery_root_id @@ -116,6 +116,7 @@ resource "google_folder_iam_member" "folder_asset_viewer" { member = module.cf.service_account_iam_email } +# role with the least privilege including compute.projects.get permission resource "google_folder_iam_member" "folder_network_viewer" { count = local.discovery_root_type == "folders" ? 1 : 0 folder = local.discovery_root_id diff --git a/blueprints/cloud-operations/quota-monitoring/src/main.py b/blueprints/cloud-operations/quota-monitoring/src/main.py index ba1e281e46..e681afada9 100755 --- a/blueprints/cloud-operations/quota-monitoring/src/main.py +++ b/blueprints/cloud-operations/quota-monitoring/src/main.py @@ -41,7 +41,7 @@ URL_TS = 'https://monitoring.googleapis.com/v3/projects/{}/timeSeries' URL_DISCOVERY = ('https://cloudasset.googleapis.com/v1/{}/assets?' 'assetTypes=cloudresourcemanager.googleapis.com%2FProject&' - 'contentType=RESOURCE&pageSize=1&pageToken={}') + 'contentType=RESOURCE&pageSize=100&pageToken={}') _Quota = collections.namedtuple('_Quota', 'project region tstamp metric limit usage') @@ -119,9 +119,8 @@ def discover_projects(discovery_root): 'Discovers projects under a folder or organization.' if discovery_root.partition('/')[0] not in ('folders', 'organizations'): raise SystemExit(f'Invalid discovery root {discovery_root}.') - last_assets_page_reached = False next_page_token = '' - while not last_assets_page_reached: + while True: list_assets_results = fetch( HTTPRequest(URL_DISCOVERY.format(discovery_root, next_page_token))) if 'assets' in list_assets_results: @@ -231,7 +230,7 @@ def _main(monitoring_project, discovery_root=None, projects=None, regions=None, exclude = set(exclude or []) projects = projects or [monitoring_project] if (discovery_root): - projects = set(projects + list(discover_projects(discovery_root))) + projects = set(list(projects) + list(discover_projects(discovery_root))) for k in ('monitoring_project', 'projects', 'regions', 'include', 'exclude'): logging.debug(f'{k} {locals().get(k)}') timeseries = [] From 2d710e2be7655ef4427423fe428d1f98063fc497 Mon Sep 17 00:00:00 2001 From: maunope Date: Tue, 12 Dec 2023 18:18:02 +0100 Subject: [PATCH 09/11] fixed discovery_root default value to work when no value is passed --- blueprints/cloud-operations/quota-monitoring/main.tf | 4 ++-- blueprints/cloud-operations/quota-monitoring/variables.tf | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/blueprints/cloud-operations/quota-monitoring/main.tf b/blueprints/cloud-operations/quota-monitoring/main.tf index c0dbf5bfe8..d5f9a48666 100644 --- a/blueprints/cloud-operations/quota-monitoring/main.tf +++ b/blueprints/cloud-operations/quota-monitoring/main.tf @@ -20,8 +20,8 @@ locals { ? [var.project_id] : var.quota_config.projects ) - discovery_root_type = split("/", var.quota_config["discovery_root"])[0] - discovery_root_id = split("/", var.quota_config["discovery_root"])[1] + discovery_root_type = split("/", coalesce(var.quota_config["discovery_root"], "/"))[0] + discovery_root_id = split("/", coalesce(var.quota_config["discovery_root"], "/"))[1] } module "project" { diff --git a/blueprints/cloud-operations/quota-monitoring/variables.tf b/blueprints/cloud-operations/quota-monitoring/variables.tf index 4aee3b543e..737cf0f91d 100644 --- a/blueprints/cloud-operations/quota-monitoring/variables.tf +++ b/blueprints/cloud-operations/quota-monitoring/variables.tf @@ -63,7 +63,7 @@ variable "quota_config" { "a2", "c2", "c2d", "committed", "g2", "interconnect", "m1", "m2", "m3", "nvidia", "preemptible" ]) - discovery_root = optional(string) + discovery_root = optional(string, "") dry_run = optional(bool, false) include = optional(list(string)) projects = optional(list(string)) @@ -74,7 +74,7 @@ variable "quota_config" { default = {} validation { condition = ( - var.quota_config.discovery_root == null || + var.quota_config.discovery_root == "" || startswith(var.quota_config.discovery_root, "folders/") || startswith(var.quota_config.discovery_root, "organizations/") ) From 07d89b3c84ad75d40709dbf828fc11f1e5adc05e Mon Sep 17 00:00:00 2001 From: maunope Date: Tue, 12 Dec 2023 18:34:59 +0100 Subject: [PATCH 10/11] fixed tfdoc --- blueprints/cloud-operations/quota-monitoring/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blueprints/cloud-operations/quota-monitoring/README.md b/blueprints/cloud-operations/quota-monitoring/README.md index 19747ece85..86bec2cbdf 100644 --- a/blueprints/cloud-operations/quota-monitoring/README.md +++ b/blueprints/cloud-operations/quota-monitoring/README.md @@ -64,7 +64,7 @@ Clone this repository or [open it in cloud shell](https://ssh.cloud.google.com/c | [bundle_path](variables.tf#L33) | Path used to write the intermediate Cloud Function code bundle. | string | | "./bundle.zip" | | [name](variables.tf#L39) | Arbitrary string used to name created resources. | string | | "quota-monitor" | | [project_create_config](variables.tf#L45) | Create project instead of using an existing one. | object({…}) | | null | -| [quota_config](variables.tf#L59) | Cloud function configuration. | object({…}) | | {} | +| [quota_config](variables.tf#L59) | Cloud function configuration. | object({…}) | | {} | | [region](variables.tf#L85) | Compute region used in the example. | string | | "europe-west1" | | [schedule_config](variables.tf#L91) | Schedule timer configuration in crontab format. | string | | "0 * * * *" | From d9a3e8b65fc907b7554da387d9ba1d061cbe87cc Mon Sep 17 00:00:00 2001 From: maunope Date: Tue, 12 Dec 2023 19:00:52 +0100 Subject: [PATCH 11/11] fixed tftest resources # --- blueprints/cloud-operations/quota-monitoring/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blueprints/cloud-operations/quota-monitoring/README.md b/blueprints/cloud-operations/quota-monitoring/README.md index 86bec2cbdf..26a702e822 100644 --- a/blueprints/cloud-operations/quota-monitoring/README.md +++ b/blueprints/cloud-operations/quota-monitoring/README.md @@ -79,5 +79,5 @@ module "test" { billing_account = "12345-ABCDE-12345" } } -# tftest modules=4 resources=14 +# tftest modules=4 resources=19 ```