From 7f7c0fa5f3d7f6dcc8dfd2dd55253e8c4f88bc12 Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 13 Jan 2026 14:45:04 +0900 Subject: [PATCH 1/6] feat: generate workflow for testing third-party PRs with secrets --- adbc_drivers_dev/templates/dev.yaml | 4 +- adbc_drivers_dev/templates/dev_pr.yaml | 2 +- adbc_drivers_dev/templates/go_test_pr.yaml | 108 ++++++++++++++++++ adbc_drivers_dev/templates/pixi.toml | 5 + adbc_drivers_dev/templates/test.yaml | 125 +++++++++++++++++++-- adbc_drivers_dev/workflow.py | 18 +++ 6 files changed, 248 insertions(+), 14 deletions(-) create mode 100644 adbc_drivers_dev/templates/go_test_pr.yaml diff --git a/adbc_drivers_dev/templates/dev.yaml b/adbc_drivers_dev/templates/dev.yaml index a7cd952..323e46f 100644 --- a/adbc_drivers_dev/templates/dev.yaml +++ b/adbc_drivers_dev/templates/dev.yaml @@ -44,13 +44,13 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 persist-credentials: false <% if lang.get("go") %> - - uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0 + - uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0 with: cache-dependency-path: go/go.sum check-latest: true diff --git a/adbc_drivers_dev/templates/dev_pr.yaml b/adbc_drivers_dev/templates/dev_pr.yaml index 263901a..28bde6f 100644 --- a/adbc_drivers_dev/templates/dev_pr.yaml +++ b/adbc_drivers_dev/templates/dev_pr.yaml @@ -47,7 +47,7 @@ jobs: runs-on: ubuntu-slim steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 1 persist-credentials: false diff --git a/adbc_drivers_dev/templates/go_test_pr.yaml b/adbc_drivers_dev/templates/go_test_pr.yaml new file mode 100644 index 0000000..fe373b9 --- /dev/null +++ b/adbc_drivers_dev/templates/go_test_pr.yaml @@ -0,0 +1,108 @@ +# Copyright (c) 2025 ADBC Drivers Contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# !!!! AUTO-GENERATED FILE. DO NOT EDIT. !!!! +# USE adbc-gen-workflow (see adbc-drivers/dev) TO UPDATE THIS FILE. + +# This is a common workflow for testing a PR when permissions are needed. + +name: Go Test PR + +on: + workflow_dispatch: + inputs: + pr: + description: "The PR to test" + required: true + type: string + commit: + description: "The commit to checkout" + required: true + type: string + +concurrency: + group: ${{ github.repository }}-${{ github.workflow }}-${{ inputs.pr }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +permissions: + contents: read + packages: read + +jobs: + setup: + name: configure job + runs-on: ubuntu-slim + outputs: + repository: ${{ steps.get_repo.outputs.repository }} + steps: + - name: get repository + id: get_repo + env: + GH_TOKEN: ${{ github.token }} + run: | + echo repository=$(gh pr --repo ${{ github.repository }} view ${{ inputs.pr }} --json headRepository,headRepositoryOwner --jq '"\(.headRepositoryOwner.login)/\(.headRepository.name)"') | tee -a $GITHUB_OUTPUT + + - name: set job summary + run: | + echo "**PR:** https://github.com/${{ github.repository }}/pull/${{ inputs.pr }}" >> $GITHUB_STEP_SUMMARY + echo "**Remote:** ${{ steps.get_repo.outputs.repository }}@${{ inputs.commit }}" >> $GITHUB_STEP_SUMMARY + + test: + uses: adbc-drivers/<{driver}>/.github/workflows/go_test.yaml@main + needs: + - setup + with: + repository: ${{ needs.setup.outputs.repository }} + commit: ${{ inputs.commit }} + secrets: + # https://github.com/orgs/community/discussions/25238#discussioncomment-3247035 +<% for name, val in secrets["all"].items() %> + <{val}>: ${{ secrets.<{val}> }} +<% endfor %> + + report: + name: comment on PR + runs-on: ubuntu-slim + if: ${{ always() }} + needs: + - setup + - test + permissions: + pull-requests: write + steps: + - id: get_run + env: + GH_TOKEN: ${{ github.token }} + run: | + echo workflow_run_url=$(gh run --repo ${{ github.repository }} view ${{ github.run_id }} --json url --jq '.url') | tee -a $GITHUB_OUTPUT + if [[ '${{ needs.test.result }}' == 'success' ]]; then + echo message=":heavy_check_mark: **Test passed:** " | tee -a $GITHUB_OUTPUT + else + echo message=":x: **Test failed:** " | tee -a $GITHUB_OUTPUT + fi + + - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + with: + github-token: ${{ github.token }} + script: | + github.rest.issues.createComment({ + issue_number: ${{ inputs.pr }}, + owner: context.repo.owner, + repo: context.repo.repo, + body: '${{ steps.get_run.outputs.message }} ${{ needs.setup.outputs.repository }}@${{ inputs.commit }}\nWorkflow run: ${{ steps.get_run.outputs.workflow_run_url }}' + }) diff --git a/adbc_drivers_dev/templates/pixi.toml b/adbc_drivers_dev/templates/pixi.toml index 8b3b885..5b3f2f0 100644 --- a/adbc_drivers_dev/templates/pixi.toml +++ b/adbc_drivers_dev/templates/pixi.toml @@ -39,6 +39,11 @@ gendocs = "python -m validation.tests.generate_documentation" [dependencies] python = ">=3.13.5,<3.14" +<% if "extra-dependencies" in validation %> +<% for key, value in validation["extra-dependencies"].items() %> +<{key}> = "<{value}>" +<% endfor %> +<% endif %> [pypi-dependencies] adbc-drivers-dev = { git = "https://github.com/adbc-drivers/dev" } diff --git a/adbc_drivers_dev/templates/test.yaml b/adbc_drivers_dev/templates/test.yaml index 85e9174..eb14bf8 100644 --- a/adbc_drivers_dev/templates/test.yaml +++ b/adbc_drivers_dev/templates/test.yaml @@ -54,8 +54,26 @@ on: <% endfor %> <% endif %> +<% if not release and secrets["all"] %> + workflow_call: + inputs: + repository: + description: "The repository to checkout" + required: true + type: string + commit: + description: "The commit to checkout" + required: true + type: string + secrets: +<% for name, val in secrets["all"].items() %> + <{val}>: + required: true +<% endfor %> +<% endif %> +<% if release %> workflow_dispatch: {} - +<% endif %> concurrency: # Must share concurrency group with release workflow since it also builds/tests @@ -93,12 +111,29 @@ jobs: # https://github.com/actions/runner-images/issues/2840 sudo rm -rf "$AGENT_TOOLSDIRECTORY" - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 +<% if release %> + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + persist-credentials: false +<% else %> + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + if: github.event_name != 'workflow_dispatch' + with: + fetch-depth: 0 + persist-credentials: false + + - name: "checkout remote" + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + if: github.event_name == 'workflow_dispatch' with: + repository: ${{ inputs.repository }} + ref: ${{ inputs.commit }} fetch-depth: 0 persist-credentials: false +<% endif %> - - uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0 + - uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0 with: cache-dependency-path: go/go.sum check-latest: true @@ -211,12 +246,29 @@ jobs: # https://github.com/actions/runner-images/issues/2840 sudo rm -rf "$AGENT_TOOLSDIRECTORY" - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 +<% if release %> + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + persist-credentials: false +<% else %> + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + if: github.event_name != 'workflow_dispatch' + with: + fetch-depth: 0 + persist-credentials: false + + - name: "checkout remote" + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + if: github.event_name == 'workflow_dispatch' with: + repository: ${{ inputs.repository }} + ref: ${{ inputs.commit }} fetch-depth: 0 persist-credentials: false +<% endif %> - - uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0 + - uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0 with: cache-dependency-path: go/go.sum check-latest: true @@ -339,12 +391,29 @@ jobs: packages: read steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 +<% if release %> + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 persist-credentials: false +<% else %> + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + if: github.event_name != 'workflow_dispatch' + with: + fetch-depth: 0 + persist-credentials: false + + - name: "checkout remote" + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + if: github.event_name == 'workflow_dispatch' + with: + repository: ${{ inputs.repository }} + ref: ${{ inputs.commit }} + fetch-depth: 0 + persist-credentials: false +<% endif %> - - uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0 + - uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0 with: cache-dependency-path: go/go.sum check-latest: true @@ -398,12 +467,29 @@ jobs: contents: read steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 +<% if release %> + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + persist-credentials: false +<% else %> + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + if: github.event_name != 'workflow_dispatch' with: fetch-depth: 0 persist-credentials: false - - uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0 + - name: "checkout remote" + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + if: github.event_name == 'workflow_dispatch' + with: + repository: ${{ inputs.repository }} + ref: ${{ inputs.commit }} + fetch-depth: 0 + persist-credentials: false +<% endif %> + + - uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0 with: check-latest: true go-version: "stable" @@ -473,12 +559,29 @@ jobs: <% endif %> steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 +<% if release %> + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 persist-credentials: false +<% else %> + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + if: github.event_name != 'workflow_dispatch' + with: + fetch-depth: 0 + persist-credentials: false + + - name: "checkout remote" + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + if: github.event_name == 'workflow_dispatch' + with: + repository: ${{ inputs.repository }} + ref: ${{ inputs.commit }} + fetch-depth: 0 + persist-credentials: false +<% endif %> - - uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0 + - uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0 with: check-latest: true go-version: "stable" diff --git a/adbc_drivers_dev/workflow.py b/adbc_drivers_dev/workflow.py index 5374c76..abfe6f9 100644 --- a/adbc_drivers_dev/workflow.py +++ b/adbc_drivers_dev/workflow.py @@ -40,6 +40,7 @@ # TOML does not support nulls MORE_DEFAULTS = { "environment": None, + "validation": {"extra-dependencies": {}}, } @@ -112,6 +113,12 @@ def generate_workflows(args) -> int: f"Secret {secret} must be a string or mapping, not {type(secret_value)}" ) + all_secrets = {} + for context_secrets in secrets.values(): + all_secrets.update(context_secrets) + secrets["all"] = all_secrets + # TODO: secrets["all"] should contain GCloud, AWS secrets + if params["lang"].get("go"): template = env.get_template("test.yaml") write_workflow( @@ -138,6 +145,17 @@ def generate_workflows(args) -> int: "workflow_name": "Release", }, ) + template = env.get_template("go_test_pr.yaml") + if secrets["all"]: + write_workflow( + workflows, + template, + "go_test_pr.yaml", + { + **params, + "secrets": secrets, + }, + ) for dev in ["dev.yaml", "dev_issues.yaml", "dev_pr.yaml"]: template = env.get_template(dev) From b59f307b08d82c3d39595f93640809966f063543 Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 13 Jan 2026 16:00:42 +0900 Subject: [PATCH 2/6] refactor: parse `generate.toml` into consistent structure --- adbc_drivers_dev/templates/go_test_pr.yaml | 9 + adbc_drivers_dev/templates/pixi.toml | 4 +- adbc_drivers_dev/templates/test.yaml | 10 +- adbc_drivers_dev/workflow.py | 179 +++++++++++++------ tests/test_workflow.py | 198 +++++++++++++++++++++ 5 files changed, 337 insertions(+), 63 deletions(-) create mode 100644 tests/test_workflow.py diff --git a/adbc_drivers_dev/templates/go_test_pr.yaml b/adbc_drivers_dev/templates/go_test_pr.yaml index fe373b9..eb7656c 100644 --- a/adbc_drivers_dev/templates/go_test_pr.yaml +++ b/adbc_drivers_dev/templates/go_test_pr.yaml @@ -1,3 +1,6 @@ +<% if private %> +# Copyright (c) 2025 Columnar Technologies Inc. All rights reserved. +<% else %> # Copyright (c) 2025 ADBC Drivers Contributors # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -11,6 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +<% endif %> # !!!! AUTO-GENERATED FILE. DO NOT EDIT. !!!! # USE adbc-gen-workflow (see adbc-drivers/dev) TO UPDATE THIS FILE. @@ -41,7 +45,11 @@ defaults: permissions: contents: read +<% if permissions.get("id_token") %> + id-token: write +<% endif %> packages: read + pull-requests: read jobs: setup: @@ -83,6 +91,7 @@ jobs: - setup - test permissions: + actions: read pull-requests: write steps: - id: get_run diff --git a/adbc_drivers_dev/templates/pixi.toml b/adbc_drivers_dev/templates/pixi.toml index 5b3f2f0..2db6be4 100644 --- a/adbc_drivers_dev/templates/pixi.toml +++ b/adbc_drivers_dev/templates/pixi.toml @@ -39,11 +39,9 @@ gendocs = "python -m validation.tests.generate_documentation" [dependencies] python = ">=3.13.5,<3.14" -<% if "extra-dependencies" in validation %> -<% for key, value in validation["extra-dependencies"].items() %> +<% for key, value in validation["extra_dependencies"].items() %> <{key}> = "<{value}>" <% endfor %> -<% endif %> [pypi-dependencies] adbc-drivers-dev = { git = "https://github.com/adbc-drivers/dev" } diff --git a/adbc_drivers_dev/templates/test.yaml b/adbc_drivers_dev/templates/test.yaml index eb14bf8..a928171 100644 --- a/adbc_drivers_dev/templates/test.yaml +++ b/adbc_drivers_dev/templates/test.yaml @@ -111,7 +111,7 @@ jobs: # https://github.com/actions/runner-images/issues/2840 sudo rm -rf "$AGENT_TOOLSDIRECTORY" -<% if release %> +<% if release or not secrets["all"] %> - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 @@ -246,7 +246,7 @@ jobs: # https://github.com/actions/runner-images/issues/2840 sudo rm -rf "$AGENT_TOOLSDIRECTORY" -<% if release %> +<% if release or not secrets["all"] %> - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 @@ -391,7 +391,7 @@ jobs: packages: read steps: -<% if release %> +<% if release or not secrets["all"] %> - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 @@ -467,7 +467,7 @@ jobs: contents: read steps: -<% if release %> +<% if release or not secrets["all"] %> - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 @@ -559,7 +559,7 @@ jobs: <% endif %> steps: -<% if release %> +<% if release or not secrets["all"] %> - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 diff --git a/adbc_drivers_dev/workflow.py b/adbc_drivers_dev/workflow.py index abfe6f9..a63c54e 100644 --- a/adbc_drivers_dev/workflow.py +++ b/adbc_drivers_dev/workflow.py @@ -17,7 +17,6 @@ import argparse import functools -import itertools import re import subprocess import sys @@ -28,19 +27,125 @@ import packaging.version import tomlkit + +def _require_bool(value: typing.Any, path: list[str]) -> bool: + if not isinstance(value, bool): + raise TypeError(f"Expected bool at `{'.'.join(path)}`, got {type(value)}") + return value + + +def _require_str_optional_nonempty(value: typing.Any, path: list[str]) -> str: + if value is None: + return value + if not isinstance(value, str): + raise TypeError(f"Expected bool at `{'.'.join(path)}`, got {type(value)}") + if not value.strip(): + raise ValueError(f"Expected non-empty string at `{'.'.join(path)}`") + return value + + +def _unknown_keys(path, keys) -> Exception: + if path: + return ValueError(f"Unknown keys in `{'.'.join(path)}`: {', '.join(keys)}") + return ValueError(f"Unknown keys at root: {', '.join(keys)}") + + +class Params: + def __init__(self, raw: dict[str, typing.Any]) -> None: + self.driver: str = raw.pop("driver", "(unknown)") + self.environment: str | None = _require_str_optional_nonempty( + raw.pop("environment", None), ["environment"] + ) + self.private: bool = _require_bool(raw.pop("private", False), ["private"]) + self.lang: dict[str, bool] = {} + for lang, enabled in raw.pop("lang", {}).items(): + self.lang[lang] = _require_bool(enabled, ["lang", lang]) + + self.secrets: dict[str, dict[str, str]] = { + "build:release": {}, + "test": {}, + "validate": {}, + } + for secret, secret_value in raw.pop("secrets", {}).items(): + if isinstance(secret_value, str): + for context in self.secrets: + self.secrets[context][secret] = secret_value + elif isinstance(secret_value, dict): + name = secret_value.pop("secret") + for scope in secret_value.pop("contexts", self.secrets.keys()): + self.secrets[scope][secret] = name + + if secret_value: + raise _unknown_keys(["secrets", secret], secret_value.keys()) + else: + raise TypeError( + f"Secret {secret} must be a string or mapping, not {type(secret_value)}" + ) + all_secrets = {} + for context_secrets in self.secrets.values(): + all_secrets.update(context_secrets) + self.secrets["all"] = all_secrets + + self.permissions: dict[str, bool] = {} + + self.aws = {} + if aws := raw.pop("aws", {}): + self.secrets["all"]["AWS_ROLE"] = "AWS_ROLE" + self.secrets["all"]["AWS_ROLE_SESSION_NAME"] = "AWS_ROLE_SESSION_NAME" + self.aws["region"] = aws.pop("region") + if aws: + raise _unknown_keys(["aws"], aws.keys()) + + self.gcloud = _require_bool(raw.pop("gcloud", False), ["gcloud"]) + if self.gcloud: + self.secrets["all"]["GCLOUD_SERVICE_ACCOUNT"] = "GCLOUD_SERVICE_ACCOUNT" + self.secrets["all"]["GCLOUD_WORKLOAD_IDENTITY_PROVIDER"] = ( + "GCLOUD_WORKLOAD_IDENTITY_PROVIDER" + ) + + if self.aws or self.gcloud: + # TODO: it might be better to have this be "write" but for now we + # don't need the flexibility + self.permissions["id_token"] = True + + self.validation = { + "extra_dependencies": {}, + } + if validation := raw.pop("validation", {}): + if extra_deps := validation.pop("extra-dependencies", {}): + self.validation["extra_dependencies"] = extra_deps + + if validation: + raise ValueError( + f"Unknown validation parameters: {', '.join(validation.keys())}" + ) + + if raw: + raise ValueError(f"Unknown parameters: {', '.join(raw.keys())}") + + def to_dict(self) -> dict[str, typing.Any]: + return { + "driver": self.driver, + "environment": self.environment, + "private": self.private, + "lang": self.lang, + "secrets": self.secrets, + "permissions": self.permissions, + "aws": self.aws, + "gcloud": self.gcloud, + "validation": self.validation, + } + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Params): + return NotImplemented + return self.to_dict() == other.to_dict() + + DEFAULT_PARAMS = { "driver": "(unknown)", "private": False, "lang": {}, - "permissions": {}, - "aws": {}, - "gcloud": {}, -} - -# TOML does not support nulls -MORE_DEFAULTS = { - "environment": None, - "validation": {"extra-dependencies": {}}, } @@ -82,52 +187,18 @@ def generate_workflows(args) -> int: return 1 - for key, value in itertools.chain(DEFAULT_PARAMS.items(), MORE_DEFAULTS.items()): - if key not in params: - params[key] = value - - if params["aws"] or params["gcloud"]: - params["permissions"]["id_token"] = True + params = Params(params) workflows = args.repository / ".github/workflows" - secrets = { - "build:release": {}, - "test": {}, - "validate": {}, - } - - if "secrets" in params: - defined_secrets = params.pop("secrets") - - for secret, secret_value in defined_secrets.items(): - if isinstance(secret_value, str): - for context in secrets: - secrets[context][secret] = secret_value - elif isinstance(secret_value, dict): - name = secret_value["secret"] - for scope in secret_value.get("contexts", secrets.keys()): - secrets[scope][secret] = name - else: - raise TypeError( - f"Secret {secret} must be a string or mapping, not {type(secret_value)}" - ) - - all_secrets = {} - for context_secrets in secrets.values(): - all_secrets.update(context_secrets) - secrets["all"] = all_secrets - # TODO: secrets["all"] should contain GCloud, AWS secrets - - if params["lang"].get("go"): + if params.lang.get("go"): template = env.get_template("test.yaml") write_workflow( workflows, template, "go_test.yaml", { - **params, - "secrets": secrets, + **params.to_dict(), "pull_request_trigger_paths": [".github/workflows/go_test.yaml"], "release": False, "workflow_name": "Test", @@ -138,22 +209,20 @@ def generate_workflows(args) -> int: template, "go_release.yaml", { - **params, - "secrets": secrets, + **params.to_dict(), "pull_request_trigger_paths": [".github/workflows/go_release.yaml"], "release": True, "workflow_name": "Release", }, ) template = env.get_template("go_test_pr.yaml") - if secrets["all"]: + if params.secrets["all"]: write_workflow( workflows, template, "go_test_pr.yaml", { - **params, - "secrets": secrets, + **params.to_dict(), }, ) @@ -164,14 +233,14 @@ def generate_workflows(args) -> int: template, dev, { - **params, + **params.to_dict(), }, ) template = env.get_template("pixi.toml") retcode = 0 - for lang, enabled in params["lang"].items(): + for lang, enabled in params.lang.items(): if not enabled: continue write_workflow( @@ -179,7 +248,7 @@ def generate_workflows(args) -> int: template, "pixi.toml", { - **params, + **params.to_dict(), }, ) diff --git a/tests/test_workflow.py b/tests/test_workflow.py new file mode 100644 index 0000000..62ca70a --- /dev/null +++ b/tests/test_workflow.py @@ -0,0 +1,198 @@ +# Copyright (c) 2025 ADBC Drivers Contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from adbc_drivers_dev.workflow import Params + + +def test_params_default() -> None: + params = Params({}) + assert params.driver == "(unknown)" + assert params.environment is None + assert params.private is False + assert params.lang == {} + assert params.secrets == { + "all": {}, + "build:release": {}, + "test": {}, + "validate": {}, + } + assert params.permissions == {} + assert params.aws == {} + assert params.gcloud is False + assert params.validation == {"extra_dependencies": {}} + + assert params.to_dict() == { + "driver": "(unknown)", + "environment": None, + "private": False, + "lang": {}, + "secrets": { + "all": {}, + "build:release": {}, + "test": {}, + "validate": {}, + }, + "permissions": {}, + "aws": {}, + "gcloud": False, + "validation": {"extra_dependencies": {}}, + } + + assert params == Params({}) + + +def test_params_custom() -> None: + params = Params({"driver": "postgresql"}) + assert params.driver == "postgresql" + assert params.to_dict()["driver"] == "postgresql" + assert params == params + assert params != Params({}) + + params = Params({"environment": "ci-env"}) + assert params.environment == "ci-env" + assert params.to_dict()["environment"] == "ci-env" + assert params == params + assert params != Params({}) + + params = Params({"private": True}) + assert params.private is True + assert params.to_dict()["private"] is True + assert params == params + assert params != Params({}) + + params = Params({"lang": {"python": True, "java": False}}) + assert params.lang == {"python": True, "java": False} + assert params.to_dict()["lang"] == {"python": True, "java": False} + assert params == params + assert params != Params({}) + + +def test_params_secrets() -> None: + params = Params( + { + "secrets": { + "foo": "bar", + "spam": {"secret": "eggs"}, + "fizz": {"secret": "buzz", "contexts": ["test", "validate"]}, + } + } + ) + assert params.secrets == { + "all": {"foo": "bar", "spam": "eggs", "fizz": "buzz"}, + "build:release": {"foo": "bar", "spam": "eggs"}, + "test": {"foo": "bar", "spam": "eggs", "fizz": "buzz"}, + "validate": {"foo": "bar", "spam": "eggs", "fizz": "buzz"}, + } + assert params.to_dict()["secrets"] == { + "all": {"foo": "bar", "spam": "eggs", "fizz": "buzz"}, + "build:release": {"foo": "bar", "spam": "eggs"}, + "test": {"foo": "bar", "spam": "eggs", "fizz": "buzz"}, + "validate": {"foo": "bar", "spam": "eggs", "fizz": "buzz"}, + } + + +def test_params_aws() -> None: + params = Params({"aws": {"region": "us-west-2"}}) + assert params.aws == {"region": "us-west-2"} + assert params.permissions == {"id_token": True} + assert params.to_dict()["permissions"] == {"id_token": True} + assert params.secrets == { + "all": { + "AWS_ROLE": "AWS_ROLE", + "AWS_ROLE_SESSION_NAME": "AWS_ROLE_SESSION_NAME", + }, + "build:release": {}, + "test": {}, + "validate": {}, + } + assert params.to_dict()["secrets"] == { + "all": { + "AWS_ROLE": "AWS_ROLE", + "AWS_ROLE_SESSION_NAME": "AWS_ROLE_SESSION_NAME", + }, + "build:release": {}, + "test": {}, + "validate": {}, + } + + +def test_params_gcloud() -> None: + params = Params({"gcloud": True}) + assert params.gcloud is True + assert params.permissions == {"id_token": True} + assert params.to_dict()["permissions"] == {"id_token": True} + assert params.secrets == { + "all": { + "GCLOUD_SERVICE_ACCOUNT": "GCLOUD_SERVICE_ACCOUNT", + "GCLOUD_WORKLOAD_IDENTITY_PROVIDER": "GCLOUD_WORKLOAD_IDENTITY_PROVIDER", + }, + "build:release": {}, + "test": {}, + "validate": {}, + } + assert params.to_dict()["secrets"] == { + "all": { + "GCLOUD_SERVICE_ACCOUNT": "GCLOUD_SERVICE_ACCOUNT", + "GCLOUD_WORKLOAD_IDENTITY_PROVIDER": "GCLOUD_WORKLOAD_IDENTITY_PROVIDER", + }, + "build:release": {}, + "test": {}, + "validate": {}, + } + + +def test_params_invalid() -> None: + with pytest.raises(TypeError): + Params({"private": ""}) + + with pytest.raises(TypeError): + Params({"environment": 2}) + + with pytest.raises(ValueError): + Params({"environment": ""}) + + with pytest.raises(ValueError): + Params( + { + "secrets": { + "fizz": { + "secret": "buzz", + "contexts": ["test", "validate"], + "foo": "bar", + }, + } + } + ) + + with pytest.raises(KeyError): + Params( + { + "secrets": { + "fizz": {"secret": "buzz", "contexts": ["asdf"], "foo": "bar"}, + } + } + ) + + +def test_params_unknown() -> None: + with pytest.raises(ValueError): + Params({"unknown_key": "value"}) + + with pytest.raises(KeyError): + Params({"aws": {"foo": "bar"}}) + + with pytest.raises(ValueError): + Params({"aws": {"region": "foo", "foo": "bar"}}) From e885ef2155989a11cbf0b4bcc604495831cc7f06 Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 14 Jan 2026 17:19:52 +0900 Subject: [PATCH 3/6] Apply suggestions from code review Co-authored-by: Bryce Mecum --- adbc_drivers_dev/templates/test.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/adbc_drivers_dev/templates/test.yaml b/adbc_drivers_dev/templates/test.yaml index a928171..99d6a49 100644 --- a/adbc_drivers_dev/templates/test.yaml +++ b/adbc_drivers_dev/templates/test.yaml @@ -58,11 +58,11 @@ on: workflow_call: inputs: repository: - description: "The repository to checkout" + description: "The repository to checkout (in owner/repo short format)" required: true type: string - commit: - description: "The commit to checkout" + ref: + description: "The ref to checkout" required: true type: string secrets: @@ -128,7 +128,7 @@ jobs: if: github.event_name == 'workflow_dispatch' with: repository: ${{ inputs.repository }} - ref: ${{ inputs.commit }} + ref: ${{ inputs.ref }} fetch-depth: 0 persist-credentials: false <% endif %> @@ -263,7 +263,7 @@ jobs: if: github.event_name == 'workflow_dispatch' with: repository: ${{ inputs.repository }} - ref: ${{ inputs.commit }} + ref: ${{ inputs.ref }} fetch-depth: 0 persist-credentials: false <% endif %> From 3f212c3ac6bf0dfd0cf8f4f1ef6bd65ca5a8daca Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 14 Jan 2026 17:20:55 +0900 Subject: [PATCH 4/6] rename --- adbc_drivers_dev/templates/go_test_pr.yaml | 8 ++++---- adbc_drivers_dev/templates/test.yaml | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/adbc_drivers_dev/templates/go_test_pr.yaml b/adbc_drivers_dev/templates/go_test_pr.yaml index eb7656c..4fbdc74 100644 --- a/adbc_drivers_dev/templates/go_test_pr.yaml +++ b/adbc_drivers_dev/templates/go_test_pr.yaml @@ -27,11 +27,11 @@ on: workflow_dispatch: inputs: pr: - description: "The PR to test" + description: "The ID of the PR to test (e.g. 42)" required: true type: string - commit: - description: "The commit to checkout" + ref: + description: "The commit/ref to checkout" required: true type: string @@ -76,7 +76,7 @@ jobs: - setup with: repository: ${{ needs.setup.outputs.repository }} - commit: ${{ inputs.commit }} + ref: ${{ inputs.ref }} secrets: # https://github.com/orgs/community/discussions/25238#discussioncomment-3247035 <% for name, val in secrets["all"].items() %> diff --git a/adbc_drivers_dev/templates/test.yaml b/adbc_drivers_dev/templates/test.yaml index 99d6a49..41fdcbc 100644 --- a/adbc_drivers_dev/templates/test.yaml +++ b/adbc_drivers_dev/templates/test.yaml @@ -408,7 +408,7 @@ jobs: if: github.event_name == 'workflow_dispatch' with: repository: ${{ inputs.repository }} - ref: ${{ inputs.commit }} + ref: ${{ inputs.ref }} fetch-depth: 0 persist-credentials: false <% endif %> @@ -484,7 +484,7 @@ jobs: if: github.event_name == 'workflow_dispatch' with: repository: ${{ inputs.repository }} - ref: ${{ inputs.commit }} + ref: ${{ inputs.ref }} fetch-depth: 0 persist-credentials: false <% endif %> @@ -576,7 +576,7 @@ jobs: if: github.event_name == 'workflow_dispatch' with: repository: ${{ inputs.repository }} - ref: ${{ inputs.commit }} + ref: ${{ inputs.ref }} fetch-depth: 0 persist-credentials: false <% endif %> From f3df817009d847a47c3a5acb4ba0ba9f23c2fc14 Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 14 Jan 2026 17:24:06 +0900 Subject: [PATCH 5/6] rename --- adbc_drivers_dev/templates/go_test_pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/adbc_drivers_dev/templates/go_test_pr.yaml b/adbc_drivers_dev/templates/go_test_pr.yaml index 4fbdc74..4201250 100644 --- a/adbc_drivers_dev/templates/go_test_pr.yaml +++ b/adbc_drivers_dev/templates/go_test_pr.yaml @@ -68,7 +68,7 @@ jobs: - name: set job summary run: | echo "**PR:** https://github.com/${{ github.repository }}/pull/${{ inputs.pr }}" >> $GITHUB_STEP_SUMMARY - echo "**Remote:** ${{ steps.get_repo.outputs.repository }}@${{ inputs.commit }}" >> $GITHUB_STEP_SUMMARY + echo "**Remote:** ${{ steps.get_repo.outputs.repository }}@${{ inputs.ref }}" >> $GITHUB_STEP_SUMMARY test: uses: adbc-drivers/<{driver}>/.github/workflows/go_test.yaml@main @@ -113,5 +113,5 @@ jobs: issue_number: ${{ inputs.pr }}, owner: context.repo.owner, repo: context.repo.repo, - body: '${{ steps.get_run.outputs.message }} ${{ needs.setup.outputs.repository }}@${{ inputs.commit }}\nWorkflow run: ${{ steps.get_run.outputs.workflow_run_url }}' + body: '${{ steps.get_run.outputs.message }} ${{ needs.setup.outputs.repository }}@${{ inputs.ref }}\nWorkflow run: ${{ steps.get_run.outputs.workflow_run_url }}' }) From 281834573a8858ce1a23dbf9a3165f35fc084817 Mon Sep 17 00:00:00 2001 From: David Li Date: Thu, 15 Jan 2026 10:00:36 +0900 Subject: [PATCH 6/6] add secrets to template --- adbc_drivers_dev/templates/test.yaml | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/adbc_drivers_dev/templates/test.yaml b/adbc_drivers_dev/templates/test.yaml index 41fdcbc..7617733 100644 --- a/adbc_drivers_dev/templates/test.yaml +++ b/adbc_drivers_dev/templates/test.yaml @@ -182,6 +182,12 @@ jobs: - name: Test if: runner.os == 'Linux' +<% if secrets["test"] %> + env: +<% for name, val in secrets["test"].items() %> + <{name}>: ${{ secrets.<{val}> }} +<% endfor %> +<% endif %> working-directory: go run: | set -a @@ -193,11 +199,6 @@ jobs: fi set +a - if [[ -n "${{ secrets.environment }}" ]]; then - echo "Loading secret environment variables" - eval "${{ secrets.environment }}" - fi - if [[ -f ci/scripts/pre-test.sh ]]; then echo "Loading pre-test" ./ci/scripts/pre-test.sh @@ -342,11 +343,6 @@ jobs: fi set +a - if [[ -n "${{ secrets.environment }}" ]]; then - echo "Loading secret environment variables" - eval "${{ secrets.environment }}" - fi - if [[ -f ci/scripts/pre-test.sh ]]; then echo "Loading pre-test" ./ci/scripts/pre-test.sh