From d1deac0b63d8c9054fb7178ab24a6048ad879d74 Mon Sep 17 00:00:00 2001 From: hantmac Date: Sat, 11 Feb 2023 08:05:50 +0800 Subject: [PATCH 1/7] fix: upgrade databend-sqlalchemy --- airbyte-integrations/connectors/destination-databend/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-databend/setup.py b/airbyte-integrations/connectors/destination-databend/setup.py index 9e03ca8e57e6..f045900d2cbf 100644 --- a/airbyte-integrations/connectors/destination-databend/setup.py +++ b/airbyte-integrations/connectors/destination-databend/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk", "requests", "databend-sqlalchemy"] +MAIN_REQUIREMENTS = ["airbyte-cdk", "requests", "databend-sqlalchemy==0.1.0"] TEST_REQUIREMENTS = ["pytest~=6.1"] setup( From eb8088278e3e04a67fac87f2f51eea7be295df4d Mon Sep 17 00:00:00 2001 From: hantmac Date: Sat, 11 Feb 2023 08:17:08 +0800 Subject: [PATCH 2/7] add docs --- .../connectors/destination-databend/Dockerfile | 2 +- docs/integrations/destinations/databend.md | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/destination-databend/Dockerfile b/airbyte-integrations/connectors/destination-databend/Dockerfile index dabbd4dc78e9..df2af6877d65 100644 --- a/airbyte-integrations/connectors/destination-databend/Dockerfile +++ b/airbyte-integrations/connectors/destination-databend/Dockerfile @@ -34,5 +34,5 @@ COPY destination_databend ./destination_databend ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/destination-databend diff --git a/docs/integrations/destinations/databend.md b/docs/integrations/destinations/databend.md index cf14368bdaf6..e25a80f7ec88 100644 --- a/docs/integrations/destinations/databend.md +++ b/docs/integrations/destinations/databend.md @@ -51,11 +51,14 @@ You should now have all the requirements needed to configure Databend as a desti * **Password** * **Database** +## Compatibility +If your databend version >= v0.9.0 or later, you need to use databend-sqlalchemy version >= v0.1.0. And the [Databend Cloud](https://app.databend.com/) will only support databend version > 0.9.0. ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------| -| 0.1.1 | 2022-01-09 | [21182](https://github.com/airbytehq/airbyte/pull/21182) | Remove protocol option and enforce HTTPS | -| 0.1.0 | 2022-01-09 | [20909](https://github.com/airbytehq/airbyte/pull/20909) | Destination Databend | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------| +| 0.1.2 | 2023-02-11 | [22855](https://github.com/airbytehq/airbyte/pull/22855) | Fix compatibility issue with databend-query 0.9 | +| 0.1.1 | 2022-01-09 | [21182](https://github.com/airbytehq/airbyte/pull/21182) | Remove protocol option and enforce HTTPS | +| 0.1.0 | 2022-01-09 | [20909](https://github.com/airbytehq/airbyte/pull/20909) | Destination Databend | From 6d6d3423dd2c6990fcaf54523298e3fbb5d8dcfb Mon Sep 17 00:00:00 2001 From: hantmac Date: Sat, 11 Feb 2023 11:28:24 +0800 Subject: [PATCH 3/7] fix --- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- .../init/src/main/resources/seed/destination_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index e254f48df385..291dfdabd0f8 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -426,7 +426,7 @@ - name: Databend destinationDefinitionId: 302e4d8e-08d3-4098-acd4-ac67ca365b88 dockerRepository: airbyte/destination-databend - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.2 icon: databend.svg documentationUrl: https://docs.airbyte.com/integrations/destinations/databend releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index cd036d4ced2e..8c58a3fb3947 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -7180,7 +7180,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-databend:0.1.0" +- dockerImage: "airbyte/destination-databend:0.1.2" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/databend" connectionSpecification: From ebc0b405328078a42040cf87e2728ee50a1bef37 Mon Sep 17 00:00:00 2001 From: hantmac Date: Mon, 13 Feb 2023 10:51:35 +0800 Subject: [PATCH 4/7] update --- .../destination_databend/client.py | 2 +- .../integration_tests/integration_test.py | 20 +++++++++---------- .../connectors/destination-databend/setup.py | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/airbyte-integrations/connectors/destination-databend/destination_databend/client.py b/airbyte-integrations/connectors/destination-databend/destination_databend/client.py index c71b3df14fa6..1764093aa094 100644 --- a/airbyte-integrations/connectors/destination-databend/destination_databend/client.py +++ b/airbyte-integrations/connectors/destination-databend/destination_databend/client.py @@ -15,6 +15,6 @@ def __init__(self, host: str, port: int, database: str, table: str, username: st self.password = password def open(self): - handle = connector.connect(f"https://{self.username}:{self.password}@{self.host}:{self.port}").cursor() + handle = connector.connect(f"https://{self.username}:{self.password}@{self.host}:{self.port}/{self.database}").cursor() return handle diff --git a/airbyte-integrations/connectors/destination-databend/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-databend/integration_tests/integration_test.py index 9ddcbc0d6038..906e0429c10c 100644 --- a/airbyte-integrations/connectors/destination-databend/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-databend/integration_tests/integration_test.py @@ -23,7 +23,7 @@ from destination_databend.client import DatabendClient -@pytest.fixture(name="config") +@pytest.fixture(name="databendConfig") def config_fixture() -> Mapping[str, Any]: with open("secrets/config.json", "r") as f: return json.loads(f.read()) @@ -49,20 +49,20 @@ def configured_catalog_fixture() -> ConfiguredAirbyteCatalog: @pytest.fixture(autouse=True) -def teardown(config: Mapping): +def teardown(databendConfig: Mapping): yield - client = DatabendClient(**config) + client = DatabendClient(**databendConfig) cursor = client.open() cursor.close() @pytest.fixture(name="client") -def client_fixture(config) -> DatabendClient: - return DatabendClient(**config) +def client_fixture(databendConfig) -> DatabendClient: + return DatabendClient(**databendConfig) -def test_check_valid_config(config: Mapping): - outcome = DestinationDatabend().check(logging.getLogger('airbyte'), config) +def test_check_valid_config(databendConfig: Mapping): + outcome = DestinationDatabend().check(logging.getLogger('airbyte'), databendConfig) assert outcome.status == Status.SUCCEEDED @@ -103,7 +103,7 @@ def retrieve_all_records(client: DatabendClient) -> List[AirbyteRecordMessage]: return overwrite_out + append_out -def test_write(config: Mapping, configured_catalog: ConfiguredAirbyteCatalog, client: DatabendClient): +def test_write(databendConfig: Mapping, configured_catalog: ConfiguredAirbyteCatalog, client: DatabendClient): """ This test verifies that: 1. writing a stream in "overwrite" mode overwrites any existing data for that stream @@ -124,7 +124,7 @@ def test_write(config: Mapping, configured_catalog: ConfiguredAirbyteCatalog, cl expected_states = [first_state_message, second_state_message] output_states = list( destination.write( - config, configured_catalog, [*first_record_chunk, first_state_message, *second_record_chunk, second_state_message] + databendConfig, configured_catalog, [*first_record_chunk, first_state_message, *second_record_chunk, second_state_message] ) ) assert expected_states == output_states, "Checkpoint state messages were expected from the destination" @@ -139,7 +139,7 @@ def test_write(config: Mapping, configured_catalog: ConfiguredAirbyteCatalog, cl _record(overwrite_stream, str(i), i) for i in range(10, 15) ] - output_states = list(destination.write(config, configured_catalog, [*third_record_chunk, third_state_message])) + output_states = list(destination.write(databendConfig, configured_catalog, [*third_record_chunk, third_state_message])) assert [third_state_message] == output_states records_in_destination = retrieve_all_records(client) diff --git a/airbyte-integrations/connectors/destination-databend/setup.py b/airbyte-integrations/connectors/destination-databend/setup.py index f045900d2cbf..64ea3347e2f2 100644 --- a/airbyte-integrations/connectors/destination-databend/setup.py +++ b/airbyte-integrations/connectors/destination-databend/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk", "requests", "databend-sqlalchemy==0.1.0"] +MAIN_REQUIREMENTS = ["airbyte-cdk", "requests", "databend-sqlalchemy==0.1.3"] TEST_REQUIREMENTS = ["pytest~=6.1"] setup( From 11fdfcc9225bbdcd208c828b2a91e04a66d26c64 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Tue, 28 Feb 2023 21:55:14 +0800 Subject: [PATCH 5/7] Update setup.py --- airbyte-integrations/connectors/destination-databend/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-databend/setup.py b/airbyte-integrations/connectors/destination-databend/setup.py index 64ea3347e2f2..49878e343bd6 100644 --- a/airbyte-integrations/connectors/destination-databend/setup.py +++ b/airbyte-integrations/connectors/destination-databend/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk", "requests", "databend-sqlalchemy==0.1.3"] +MAIN_REQUIREMENTS = ["airbyte-cdk", "requests", "databend-sqlalchemy==0.1.6"] TEST_REQUIREMENTS = ["pytest~=6.1"] setup( From c29bc7c7aee5bd07a41bdd93c386f00a35ab9d42 Mon Sep 17 00:00:00 2001 From: natalyjazzviolin Date: Thu, 23 Mar 2023 14:05:00 +0000 Subject: [PATCH 6/7] Automated Change --- connectors.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/connectors.md b/connectors.md index 2efe4fa39612..ff03756ffd66 100644 --- a/connectors.md +++ b/connectors.md @@ -295,7 +295,7 @@ | **Chargify (Keen)** | Chargify (Keen) icon | Destination | airbyte/destination-keen:0.2.4 | alpha | [link](https://docs.airbyte.com/integrations/destinations/keen) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-keen) | `81740ce8-d764-4ea7-94df-16bb41de36ae` | | **Clickhouse** | Clickhouse icon | Destination | airbyte/destination-clickhouse:0.2.2 | alpha | [link](https://docs.airbyte.com/integrations/destinations/clickhouse) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-clickhouse) | `ce0d828e-1dc4-496c-b122-2da42e637e48` | | **Cloudflare R2** | Cloudflare R2 icon | Destination | airbyte/destination-r2:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/destinations/r2) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-r2) | `0fb07be9-7c3b-4336-850d-5efc006152ee` | -| **Databend** | Databend icon | Destination | airbyte/destination-databend:0.1.1 | alpha | [link](https://docs.airbyte.com/integrations/destinations/databend) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-databend) | `302e4d8e-08d3-4098-acd4-ac67ca365b88` | +| **Databend** | Databend icon | Destination | airbyte/destination-databend:0.1.2 | alpha | [link](https://docs.airbyte.com/integrations/destinations/databend) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-databend) | `302e4d8e-08d3-4098-acd4-ac67ca365b88` | | **Databricks Lakehouse** | Databricks Lakehouse icon | Destination | airbyte/destination-databricks:1.0.0 | alpha | [link](https://docs.airbyte.com/integrations/destinations/databricks) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-databricks) | `072d5540-f236-4294-ba7c-ade8fd918496` | | **DuckDB** | DuckDB icon | Destination | airbyte/destination-duckdb:0.1.0 | alpha | [link](https://docs.airbyte.io/integrations/destinations/duckdb) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-duckdb) | `94bd199c-2ff0-4aa2-b98e-17f0acb72610` | | **DynamoDB** | DynamoDB icon | Destination | airbyte/destination-dynamodb:0.1.7 | alpha | [link](https://docs.airbyte.com/integrations/destinations/dynamodb) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-dynamodb) | `8ccd8909-4e99-4141-b48d-4984b70b2d89` | From bdb51af1bc65fb273f9b4c5f471a51625fb1ee17 Mon Sep 17 00:00:00 2001 From: natalyjazzviolin Date: Mon, 27 Mar 2023 14:29:25 +0000 Subject: [PATCH 7/7] Automated Change --- connectors.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/connectors.md b/connectors.md index 4502aa032db3..599b453f0751 100644 --- a/connectors.md +++ b/connectors.md @@ -298,7 +298,7 @@ | **Clickhouse** | Clickhouse icon | Destination | airbyte/destination-clickhouse:0.2.2 | alpha | [link](https://docs.airbyte.com/integrations/destinations/clickhouse) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-clickhouse) | `ce0d828e-1dc4-496c-b122-2da42e637e48` | | **Cloudflare R2** | Cloudflare R2 icon | Destination | airbyte/destination-r2:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/destinations/r2) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-r2) | `0fb07be9-7c3b-4336-850d-5efc006152ee` | | **Convex** | Convex icon | Destination | airbyte/destination-convex:0.1.0 | alpha | [link](https://docs.airbyte.io/integrations/destinations/convex) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-convex) | `3eb4d99c-11fa-4561-a259-fc88e0c2f8f4` | -| **Databend** | Databend icon | Destination | airbyte/destination-databend:0.1.1 | alpha | [link](https://docs.airbyte.com/integrations/destinations/databend) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-databend) | `302e4d8e-08d3-4098-acd4-ac67ca365b88` | +| **Databend** | Databend icon | Destination | airbyte/destination-databend:0.1.2 | alpha | [link](https://docs.airbyte.com/integrations/destinations/databend) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-databend) | `302e4d8e-08d3-4098-acd4-ac67ca365b88` | | **Databricks Lakehouse** | Databricks Lakehouse icon | Destination | airbyte/destination-databricks:1.0.0 | alpha | [link](https://docs.airbyte.com/integrations/destinations/databricks) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-databricks) | `072d5540-f236-4294-ba7c-ade8fd918496` | | **DuckDB** | DuckDB icon | Destination | airbyte/destination-duckdb:0.1.0 | alpha | [link](https://docs.airbyte.io/integrations/destinations/duckdb) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-duckdb) | `94bd199c-2ff0-4aa2-b98e-17f0acb72610` | | **DynamoDB** | DynamoDB icon | Destination | airbyte/destination-dynamodb:0.1.7 | alpha | [link](https://docs.airbyte.com/integrations/destinations/dynamodb) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-dynamodb) | `8ccd8909-4e99-4141-b48d-4984b70b2d89` |