From eec86e9b755f3d16df9bf7919c552bb71c58e710 Mon Sep 17 00:00:00 2001 From: Helder Correia <174525+helderco@users.noreply.github.com> Date: Mon, 7 Aug 2023 08:14:04 +0000 Subject: [PATCH] dagger/connectors-ci: use kwargs for optional arguments (#28729) --- airbyte-ci/connectors/pipelines/README.md | 3 ++- .../connectors/pipelines/pipelines/actions/environments.py | 2 +- .../pipelines/pipelines/actions/remote_storage.py | 4 ++-- airbyte-ci/connectors/pipelines/pipelines/gradle.py | 4 ++-- airbyte-ci/connectors/pipelines/pipelines/hacks.py | 4 ++-- airbyte-ci/connectors/pipelines/pipelines/publish.py | 6 +++--- airbyte-ci/connectors/pipelines/pyproject.toml | 2 +- airbyte-ci/connectors/pipelines/tests/tests/test_common.py | 2 +- 8 files changed, 14 insertions(+), 13 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index 5b0f7ccc6d7f..e23ea25beb46 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -380,8 +380,9 @@ This command runs the Python tests for a airbyte-ci poetry package. | Version | PR | Description | |---------|-----------------------------------------------------------|----------------------------------------------------------------------------------------------| +| 0.4.6 | [#28729](https://github.com/airbytehq/airbyte/pull/28729) | Use keyword args instead of positional argument for optional paramater in Dagger's API | | 0.4.5 | [#29034](https://github.com/airbytehq/airbyte/pull/29034) | Disable Dagger terminal UI when running publish. | -| 0.4.4 | [#29064](https://github.com/airbytehq/airbyte/pull/29064) | Make connector modified files a frozen set. | +| 0.4.4 | [#29064](https://github.com/airbytehq/airbyte/pull/29064) | Make connector modified files a frozen set. | | 0.4.3 | [#29033](https://github.com/airbytehq/airbyte/pull/29033) | Disable dependency scanning for Java connectors. | | 0.4.2 | [#29030](https://github.com/airbytehq/airbyte/pull/29030) | Make report path always have the same prefix: `airbyte-ci/`. | | 0.4.1 | [#28855](https://github.com/airbytehq/airbyte/pull/28855) | Improve the selected connectors detection for connectors commands. | diff --git a/airbyte-ci/connectors/pipelines/pipelines/actions/environments.py b/airbyte-ci/connectors/pipelines/pipelines/actions/environments.py index b5b48f582f1a..513f3ae79161 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/actions/environments.py +++ b/airbyte-ci/connectors/pipelines/pipelines/actions/environments.py @@ -955,7 +955,7 @@ async def with_airbyte_python_connector_full_dagger(context: ConnectorContext, b base.with_workdir("/airbyte/integration_code") .with_directory("/usr/local", builder.directory("/install")) .with_file("/usr/localtime", builder.file("/usr/share/zoneinfo/Etc/UTC")) - .with_new_file("/etc/timezone", "Etc/UTC") + .with_new_file("/etc/timezone", contents="Etc/UTC") .with_exec(["apt-get", "install", "-y", "bash"]) .with_file("main.py", (await context.get_connector_dir(include="main.py")).file("main.py")) .with_directory(snake_case_name, (await context.get_connector_dir(include=snake_case_name)).directory(snake_case_name)) diff --git a/airbyte-ci/connectors/pipelines/pipelines/actions/remote_storage.py b/airbyte-ci/connectors/pipelines/pipelines/actions/remote_storage.py index d45c88390ccb..b5f2fb961478 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/actions/remote_storage.py +++ b/airbyte-ci/connectors/pipelines/pipelines/actions/remote_storage.py @@ -7,8 +7,8 @@ from pathlib import Path from typing import List, Optional, Tuple -from pipelines.utils import get_exec_result, secret_host_variable, with_exit_code from dagger import Client, File, Secret +from pipelines.utils import get_exec_result, secret_host_variable, with_exit_code GOOGLE_CLOUD_SDK_TAG = "425.0.0-slim" @@ -68,7 +68,7 @@ async def upload_to_gcs( dagger_client.container() .from_(f"google/cloud-sdk:{GOOGLE_CLOUD_SDK_TAG}") .with_workdir("/upload") - .with_new_file("credentials.json", await gcs_credentials.plaintext()) + .with_new_file("credentials.json", contents=await gcs_credentials.plaintext()) .with_env_variable("GOOGLE_APPLICATION_CREDENTIALS", "/upload/credentials.json") .with_file("to_upload", file_to_upload) ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/gradle.py b/airbyte-ci/connectors/pipelines/pipelines/gradle.py index d09b81e32b90..93522fba7e8e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/gradle.py +++ b/airbyte-ci/connectors/pipelines/pipelines/gradle.py @@ -7,10 +7,10 @@ from abc import ABC from typing import ClassVar, List, Tuple +from dagger import CacheVolume, Container, Directory, QueryError from pipelines import consts from pipelines.actions import environments from pipelines.bases import Step, StepResult -from dagger import CacheVolume, Container, Directory, QueryError class GradleTask(Step, ABC): @@ -58,7 +58,7 @@ async def _get_patched_build_src_dir(self) -> Directory: cat_gradle_plugin_content = cat_gradle_plugin_content.replace( "project.integrationTest.dependsOn(project.connectorAcceptanceTest)", "" ) - return build_src_dir.with_new_file("src/main/groovy/airbyte-connector-acceptance-test.gradle", cat_gradle_plugin_content) + return build_src_dir.with_new_file("src/main/groovy/airbyte-connector-acceptance-test.gradle", contents=cat_gradle_plugin_content) def _get_gradle_command(self, extra_options: Tuple[str] = ("--no-daemon", "--scan", "--build-cache")) -> List: command = ( diff --git a/airbyte-ci/connectors/pipelines/pipelines/hacks.py b/airbyte-ci/connectors/pipelines/pipelines/hacks.py index 0557786d5266..9fa287209eee 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/hacks.py +++ b/airbyte-ci/connectors/pipelines/pipelines/hacks.py @@ -57,7 +57,7 @@ async def _patch_gradle_file(context: ConnectorContext, connector_dir: Directory for line in gradle_file_content.splitlines(): if not any(line_to_remove in line for line_to_remove in LINES_TO_REMOVE_FROM_GRADLE_FILE): patched_gradle_file.append(line) - return connector_dir.with_new_file("build.gradle", "\n".join(patched_gradle_file)) + return connector_dir.with_new_file("build.gradle", contents="\n".join(patched_gradle_file)) def _patch_cat_config(context: ConnectorContext, connector_dir: Directory) -> Directory: @@ -86,7 +86,7 @@ def _patch_cat_config(context: ConnectorContext, connector_dir: Directory) -> Di patched_cat_config["connector_image"] = context.connector.acceptance_test_config["connector_image"].replace( ":dev", f":{context.git_revision}" ) - return connector_dir.with_new_file("acceptance-test-config.yml", yaml.safe_dump(patched_cat_config)) + return connector_dir.with_new_file("acceptance-test-config.yml", contents=yaml.safe_dump(patched_cat_config)) async def patch_connector_dir(context: ConnectorContext, connector_dir: Directory) -> Directory: diff --git a/airbyte-ci/connectors/pipelines/pipelines/publish.py b/airbyte-ci/connectors/pipelines/pipelines/publish.py index a7ad9ae75afb..d5e48eb9875d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/publish.py +++ b/airbyte-ci/connectors/pipelines/pipelines/publish.py @@ -7,13 +7,13 @@ import anyio from airbyte_protocol.models.airbyte_protocol import ConnectorSpecification +from dagger import Container, ExecError, File, ImageLayerCompression, QueryError from pipelines import builds, consts from pipelines.actions import environments from pipelines.actions.remote_storage import upload_to_gcs from pipelines.bases import ConnectorReport, Step, StepResult, StepStatus from pipelines.contexts import PublishConnectorContext from pipelines.pipelines import metadata -from dagger import Container, ExecError, File, ImageLayerCompression, QueryError from pydantic import ValidationError @@ -174,7 +174,7 @@ async def _get_connector_spec(self, connector: Container, deployment_mode: str) return self._parse_spec_output(spec_output) async def _get_spec_as_file(self, spec: str, name="spec_to_cache.json") -> File: - return (await self.context.get_connector_dir()).with_new_file(name, spec).file(name) + return (await self.context.get_connector_dir()).with_new_file(name, contents=spec).file(name) async def _run(self, built_connector: Container) -> StepResult: try: @@ -186,7 +186,7 @@ async def _run(self, built_connector: Container) -> StepResult: specs_to_uploads: List[Tuple[str, File]] = [(self.oss_spec_key, await self._get_spec_as_file(oss_spec))] if oss_spec != cloud_spec: - specs_to_uploads.append(self.cloud_spec_key, await self._get_spec_as_file(cloud_spec, "cloud_spec_to_cache.json")) + specs_to_uploads.append((self.cloud_spec_key, await self._get_spec_as_file(cloud_spec, "cloud_spec_to_cache.json"))) for key, file in specs_to_uploads: exit_code, stdout, stderr = await upload_to_gcs( diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 0a425efe14d4..b4c4f7dee61c 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "0.4.5" +version = "0.4.6" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "] diff --git a/airbyte-ci/connectors/pipelines/tests/tests/test_common.py b/airbyte-ci/connectors/pipelines/tests/tests/test_common.py index b46733f4419c..93aae42d2919 100644 --- a/airbyte-ci/connectors/pipelines/tests/tests/test_common.py +++ b/airbyte-ci/connectors/pipelines/tests/tests/test_common.py @@ -32,7 +32,7 @@ def get_dummy_cat_container(dagger_client: dagger.Client, exit_code: int, secret secret_dir_name = str(pathlib.Path(secret_file_path).parent) container = container.with_exec(["mkdir", "-p", secret_dir_name]) container = container.with_exec(["sh", "-c", f"echo foo > {secret_file_path}"]) - return container.with_new_file("/stupid_bash_script.sh", f"echo {stdout}; echo {stderr} >&2; exit {exit_code}") + return container.with_new_file("/stupid_bash_script.sh", contents=f"echo {stdout}; echo {stderr} >&2; exit {exit_code}") @pytest.fixture def test_context(self, dagger_client):