Skip to content

Commit

Permalink
Metadata Upload: target production GCS bucket (#25529)
Browse files Browse the repository at this point in the history
* Update spec mask to be generated from sensor

* Update publish to use production bucket

* Update env variable
  • Loading branch information
bnchrch authored Apr 26, 2023
1 parent 3a308ba commit bd43a4f
Showing 2 changed files with 8 additions and 9 deletions.
6 changes: 3 additions & 3 deletions .github/workflows/publish_connectors.yml
Original file line number Diff line number Diff line change
@@ -26,9 +26,9 @@ jobs:
DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
GCS_CREDENTIALS: ${{ secrets.METADATA_SERVICE_DEV_GCS_CREDENTIALS }}
METADATA_SERVICE_ACCOUNT_KEY: ${{ secrets.METADATA_SERVICE_DEV_GCS_CREDENTIALS }}
METADATA_SERVICE_BUCKET_NAME: dev-airbyte-cloud-connector-metadata-service
GCS_CREDENTIALS: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
METADATA_SERVICE_ACCOUNT_KEY: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
METADATA_SERVICE_BUCKET_NAME: prod-airbyte-cloud-connector-metadata-service
SPEC_CACHE_BUCKET_NAME: io-airbyte-cloud-spec-cache
SPEC_CACHE_SERVICE_ACCOUNT_KEY: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }}
TEST_REPORTS_BUCKET_NAME: "airbyte-connector-build-status"
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
from dagster import define_asset_job, AssetSelection

registries_inclusive = AssetSelection.keys(
"metadata_directory_report", "cloud_registry_from_metadata", "oss_registry_from_metadata"
"metadata_directory_report", "cloud_registry_from_metadata", "oss_registry_from_metadata", "specs_secrets_mask_yaml"
).upstream()
registry_reports_inclusive = AssetSelection.keys("connector_registry_report").upstream()

generate_registry = define_asset_job(name="generate_registry", selection=registries_inclusive)

registry_reports_inclusive = AssetSelection.keys("connector_registry_report").upstream()
generate_registry_reports = define_asset_job(name="generate_registry_reports", selection=registry_reports_inclusive)

# TODO Change to a Asset selection
generate_local_metadata_files = define_asset_job(name="generate_local_metadata_files", selection=["persist_metadata_definitions"])
generate_specs_secrets_mask_file = define_asset_job(name="generate_specs_secrets_mask_file", selection=["persist_specs_secrets_mask"])
metadata_inclusive = AssetSelection.keys("persist_metadata_definitions").upstream()
generate_local_metadata_files = define_asset_job(name="generate_local_metadata_files", selection=metadata_inclusive)

0 comments on commit bd43a4f

Please sign in to comment.