From bbdc4c90c6de72f4599f97d92b68ebc12065027d Mon Sep 17 00:00:00 2001 From: erohmensing Date: Mon, 20 Mar 2023 15:28:21 -0500 Subject: [PATCH 1/2] ci_credentials: fix overwriting 'data' before getting nextPageToken --- .../ci_credentials/secrets_manager.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tools/ci_credentials/ci_credentials/secrets_manager.py b/tools/ci_credentials/ci_credentials/secrets_manager.py index 62a5bbfb1723..b6d345199657 100644 --- a/tools/ci_credentials/ci_credentials/secrets_manager.py +++ b/tools/ci_credentials/ci_credentials/secrets_manager.py @@ -81,8 +81,8 @@ def __load_gsm_secrets(self) -> List[RemoteSecret]: if next_token: params["pageToken"] = next_token - data = self.api.get(url, params=params) - for secret_info in data.get("secrets") or []: + all_secrets_data = self.api.get(url, params=params) + for secret_info in all_secrets_data.get("secrets") or []: secret_name = secret_info["name"] connector_name = secret_info.get("labels", {}).get("connector") if not connector_name: @@ -103,14 +103,14 @@ def __load_gsm_secrets(self) -> List[RemoteSecret]: self.logger.info(f"found GSM secret: {log_name} = > {filename}") versions_url = f"https://secretmanager.googleapis.com/v1/{secret_name}/versions" - data = self.api.get(versions_url) - enabled_versions = [version["name"] for version in data["versions"] if version["state"] == "ENABLED"] + versions_data = self.api.get(versions_url) + enabled_versions = [version["name"] for version in versions_data["versions"] if version["state"] == "ENABLED"] if len(enabled_versions) > 1: self.logger.critical(f"{log_name} should have one enabled version at the same time!!!") enabled_version = enabled_versions[0] secret_url = f"https://secretmanager.googleapis.com/v1/{enabled_version}:access" - data = self.api.get(secret_url) - secret_value = data.get("payload", {}).get("data") + secret_data = self.api.get(secret_url) + secret_value = secret_data.get("payload", {}).get("data") if not secret_value: self.logger.warning(f"{log_name} has empty value") continue @@ -126,7 +126,7 @@ def __load_gsm_secrets(self) -> List[RemoteSecret]: remote_secret = RemoteSecret(connector_name, filename, secret_value, enabled_version) secrets.append(remote_secret) - next_token = data.get("nextPageToken") + next_token = all_secrets_data.get("nextPageToken") if not next_token: break From 449603277b2d99ae3f05cd78d92948393cf683b2 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Mon, 20 Mar 2023 21:19:53 -0500 Subject: [PATCH 2/2] Updates to readme --- tools/ci_credentials/README.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tools/ci_credentials/README.md b/tools/ci_credentials/README.md index b08ec2067cc5..acce97d6c8e0 100644 --- a/tools/ci_credentials/README.md +++ b/tools/ci_credentials/README.md @@ -29,7 +29,7 @@ Download a Service account json key that has access to Google Secrets Manager. * Click on "ADD KEY -> Create new key" and select JSON. This will download a file on your computer ### Setup ci_credentials -* In your .zshrc, add: export GCP_GSM_CREDENTIALS=`cat ` +* In your .zshrc, add: `export GCP_GSM_CREDENTIALS=cat $()` * Follow README.md under `tools/ci_credentials` After making a change, you have to reinstall it to run the bash command: `pip install --quiet -e ./tools/ci_*` @@ -44,12 +44,19 @@ The `VERSION=dev` will make it so it knows to use your local current working dir ci_credentials --help ``` -### Write to storage +### Write credentials for a specific connector to local storage To download GSM secrets to `airbyte-integrations/connectors/source-bings-ads/secrets`: ```bash ci_credentials source-bing-ads write-to-storage ``` +### Write credentials for all connectors to local storage +To download GSM secrets to for all available connectors into their respective `secrets` directories: +```bash +ci_credentials all write-to-storage +``` + + ### Update secrets To upload to GSM newly updated configurations from `airbyte-integrations/connectors/source-bings-ads/secrets/updated_configurations`: