From f47c8e31e5c3b3a0842804f73d0172191c7995ea Mon Sep 17 00:00:00 2001 From: grishick Date: Wed, 18 Jan 2023 16:22:54 -0800 Subject: [PATCH 1/8] Use ClickhouseDestinationStrictEncrypt in strict-encrypt clickhouse destination --- .../connectors/destination-clickhouse-strict-encrypt/Dockerfile | 2 +- .../destination-clickhouse-strict-encrypt/build.gradle | 2 +- .../connectors/destination-clickhouse/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile index 1682d0974303..8a6e418ae246 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-clickhouse-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.2.1 +LABEL io.airbyte.version=0.2.2 LABEL io.airbyte.name=airbyte/destination-clickhouse-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle index ece5ce8ecf7e..5e0280c4d223 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle @@ -5,7 +5,7 @@ plugins { } application { - mainClass = 'io.airbyte.integrations.destination.clickhouse.ClickhouseDestination' + mainClass = 'io.airbyte.integrations.destination.clickhouse.ClickhouseDestinationStrictEncrypt' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } diff --git a/airbyte-integrations/connectors/destination-clickhouse/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse/Dockerfile index 4d9d2eb31d8a..604ed1f756d1 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/Dockerfile +++ b/airbyte-integrations/connectors/destination-clickhouse/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-clickhouse COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.2.1 +LABEL io.airbyte.version=0.2.2 LABEL io.airbyte.name=airbyte/destination-clickhouse From a4623471aee8911cae1074daaf3e2703cbb572f0 Mon Sep 17 00:00:00 2001 From: grishick Date: Wed, 18 Jan 2023 16:41:14 -0800 Subject: [PATCH 2/8] Do not merge this commit. Trying to unblock tests --- .github/workflows/test-command.yml | 13 ------ .../tests/test_secrets_manager.py | 45 ------------------- 2 files changed, 58 deletions(-) diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index 069d1cd46624..864aa53e4f2f 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -129,19 +129,6 @@ jobs: command: ./tools/bin/ci_integration_test.sh ${{ github.event.inputs.connector }} attempt_limit: 3 attempt_delay: 10000 # in ms - - name: Update Integration Test Credentials after test run for ${{ github.event.inputs.connector }} - if: always() - run: | - source venv/bin/activate - ci_credentials ${{ github.event.inputs.connector }} update-secrets - # normalization also runs destination-specific tests, so fetch their creds also - if [ 'bases/base-normalization' = "${{ github.event.inputs.connector }}" ] || [ 'base-normalization' = "${{ github.event.inputs.connector }}" ]; then - ci_credentials destination-bigquery update-secrets - ci_credentials destination-postgres update-secrets - ci_credentials destination-snowflake update-secrets - fi - env: - GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - name: Archive test reports artifacts if: github.event.inputs.comment-id && failure() uses: actions/upload-artifact@v3 diff --git a/tools/ci_credentials/tests/test_secrets_manager.py b/tools/ci_credentials/tests/test_secrets_manager.py index e5acdbec9f97..2017223d47c8 100644 --- a/tools/ci_credentials/tests/test_secrets_manager.py +++ b/tools/ci_credentials/tests/test_secrets_manager.py @@ -159,48 +159,3 @@ def test_validate_mask_values(connector_name, dict_json_value, expected_secret, manager.mask_secrets_from_action_log(None, json_value) assert expected_secret in capsys.readouterr().out - -@patch("ci_common_utils.GoogleApi.get_access_token", lambda *args: ("fake_token", None)) -@patch("ci_common_utils.GoogleApi.project_id", "fake_id") -@pytest.mark.parametrize( - "old_secret_value, updated_configurations", - [ - (json.dumps({"key": "value"}), [json.dumps({"key": "new_value_1"}), json.dumps({"key": "new_value_2"})]), - (json.dumps({"key": "value"}), [json.dumps({"key": "value"})]), - ], -) -def test_update_secrets(tmp_path, matchers, old_secret_value, updated_configurations): - existing_secret = RemoteSecret("source-test", "config.json", old_secret_value, "previous_version") - existing_secrets = [existing_secret] - - manager = SecretsManager(connector_name="source-test", gsm_credentials={}) - manager.base_folder = tmp_path - updated_configuration_directory = tmp_path / "airbyte-integrations/connectors/source-test/secrets/updated_configurations" - updated_configuration_directory.mkdir(parents=True) - - for i, updated_configuration in enumerate(updated_configurations): - stem, ext = existing_secret.configuration_file_name.split(".") - updated_configuration_file_name = f"{stem}|{i}.{ext}" - updated_configuration_path = updated_configuration_directory / updated_configuration_file_name - with open(updated_configuration_path, "w") as f: - f.write(updated_configuration) - - with requests_mock.Mocker() as m: - add_version_adapter = m.post(matchers["addVersion"], json={"name": "new_version"}) - disable_version_adapter = m.post(matchers["disable"], json={}) - updated_secrets = manager.update_secrets(existing_secrets) - - if old_secret_value != updated_configurations[-1]: - # We confirm the new version was created from the latest updated_configuration value - for secret in updated_secrets: - assert secret.connector_name == "source-test" - assert secret.configuration_file_name == "config.json" - assert secret.value == updated_configurations[-1] - assert secret.enabled_version == "new_version" - expected_add_version_payload = {"payload": {"data": base64.b64encode(updated_configurations[-1].encode()).decode("utf-8")}} - assert add_version_adapter.last_request.json() == expected_add_version_payload - assert disable_version_adapter.called_once - else: - assert not updated_secrets - assert not add_version_adapter.called - assert not disable_version_adapter.called From 7ea5ff47f074baf66177a61088a79774518cebab Mon Sep 17 00:00:00 2001 From: grishick Date: Wed, 18 Jan 2023 16:51:30 -0800 Subject: [PATCH 3/8] format --- tools/ci_credentials/tests/test_secrets_manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/ci_credentials/tests/test_secrets_manager.py b/tools/ci_credentials/tests/test_secrets_manager.py index 2017223d47c8..ccb0ff4526de 100644 --- a/tools/ci_credentials/tests/test_secrets_manager.py +++ b/tools/ci_credentials/tests/test_secrets_manager.py @@ -158,4 +158,3 @@ def test_validate_mask_values(connector_name, dict_json_value, expected_secret, json_value = json.loads(dict_json_value) manager.mask_secrets_from_action_log(None, json_value) assert expected_secret in capsys.readouterr().out - From 66e52e9995030bc755737033711fbd19c4c5ba3f Mon Sep 17 00:00:00 2001 From: grishick Date: Wed, 18 Jan 2023 16:52:37 -0800 Subject: [PATCH 4/8] skip failing CI step --- .github/workflows/publish-command.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 89677481489d..1d93f25db19f 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -318,19 +318,6 @@ jobs: ./tools/integrations/manage.sh publish airbyte-integrations/${{ matrix.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache attempt_limit: 3 attempt_delay: 5000 in # ms - - name: Update Integration Test Credentials after test run for ${{ github.event.inputs.connector }} - if: always() - run: | - source venv/bin/activate - ci_credentials ${{ matrix.connector }} update-secrets - # normalization also runs destination-specific tests, so fetch their creds also - if [ 'bases/base-normalization' = "${{ matrix.connector }}" ] || [ 'base-normalization' = "${{ matrix.connector }}" ]; then - ci_credentials destination-bigquery update-secrets - ci_credentials destination-postgres update-secrets - ci_credentials destination-snowflake update-secrets - fi - env: - GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - name: Create Sentry Release if: startsWith(matrix.connector, 'connectors') && success() run: | From 57c3cb89a48456997dd47eb4036d25c0310f9053 Mon Sep 17 00:00:00 2001 From: grishick Date: Thu, 19 Jan 2023 08:18:32 -0800 Subject: [PATCH 5/8] remove workaround --- .github/workflows/test-command.yml | 13 ++++++ .../tests/test_secrets_manager.py | 46 +++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index 864aa53e4f2f..069d1cd46624 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -129,6 +129,19 @@ jobs: command: ./tools/bin/ci_integration_test.sh ${{ github.event.inputs.connector }} attempt_limit: 3 attempt_delay: 10000 # in ms + - name: Update Integration Test Credentials after test run for ${{ github.event.inputs.connector }} + if: always() + run: | + source venv/bin/activate + ci_credentials ${{ github.event.inputs.connector }} update-secrets + # normalization also runs destination-specific tests, so fetch their creds also + if [ 'bases/base-normalization' = "${{ github.event.inputs.connector }}" ] || [ 'base-normalization' = "${{ github.event.inputs.connector }}" ]; then + ci_credentials destination-bigquery update-secrets + ci_credentials destination-postgres update-secrets + ci_credentials destination-snowflake update-secrets + fi + env: + GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - name: Archive test reports artifacts if: github.event.inputs.comment-id && failure() uses: actions/upload-artifact@v3 diff --git a/tools/ci_credentials/tests/test_secrets_manager.py b/tools/ci_credentials/tests/test_secrets_manager.py index ccb0ff4526de..e5acdbec9f97 100644 --- a/tools/ci_credentials/tests/test_secrets_manager.py +++ b/tools/ci_credentials/tests/test_secrets_manager.py @@ -158,3 +158,49 @@ def test_validate_mask_values(connector_name, dict_json_value, expected_secret, json_value = json.loads(dict_json_value) manager.mask_secrets_from_action_log(None, json_value) assert expected_secret in capsys.readouterr().out + + +@patch("ci_common_utils.GoogleApi.get_access_token", lambda *args: ("fake_token", None)) +@patch("ci_common_utils.GoogleApi.project_id", "fake_id") +@pytest.mark.parametrize( + "old_secret_value, updated_configurations", + [ + (json.dumps({"key": "value"}), [json.dumps({"key": "new_value_1"}), json.dumps({"key": "new_value_2"})]), + (json.dumps({"key": "value"}), [json.dumps({"key": "value"})]), + ], +) +def test_update_secrets(tmp_path, matchers, old_secret_value, updated_configurations): + existing_secret = RemoteSecret("source-test", "config.json", old_secret_value, "previous_version") + existing_secrets = [existing_secret] + + manager = SecretsManager(connector_name="source-test", gsm_credentials={}) + manager.base_folder = tmp_path + updated_configuration_directory = tmp_path / "airbyte-integrations/connectors/source-test/secrets/updated_configurations" + updated_configuration_directory.mkdir(parents=True) + + for i, updated_configuration in enumerate(updated_configurations): + stem, ext = existing_secret.configuration_file_name.split(".") + updated_configuration_file_name = f"{stem}|{i}.{ext}" + updated_configuration_path = updated_configuration_directory / updated_configuration_file_name + with open(updated_configuration_path, "w") as f: + f.write(updated_configuration) + + with requests_mock.Mocker() as m: + add_version_adapter = m.post(matchers["addVersion"], json={"name": "new_version"}) + disable_version_adapter = m.post(matchers["disable"], json={}) + updated_secrets = manager.update_secrets(existing_secrets) + + if old_secret_value != updated_configurations[-1]: + # We confirm the new version was created from the latest updated_configuration value + for secret in updated_secrets: + assert secret.connector_name == "source-test" + assert secret.configuration_file_name == "config.json" + assert secret.value == updated_configurations[-1] + assert secret.enabled_version == "new_version" + expected_add_version_payload = {"payload": {"data": base64.b64encode(updated_configurations[-1].encode()).decode("utf-8")}} + assert add_version_adapter.last_request.json() == expected_add_version_payload + assert disable_version_adapter.called_once + else: + assert not updated_secrets + assert not add_version_adapter.called + assert not disable_version_adapter.called From 95267b244e0c768bc7e42fb31f9faa974996d8f4 Mon Sep 17 00:00:00 2001 From: grishick Date: Thu, 19 Jan 2023 08:20:12 -0800 Subject: [PATCH 6/8] remove workaround --- .github/workflows/publish-command.yml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 1d93f25db19f..93e0be8770f9 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -260,12 +260,13 @@ jobs: uses: actions/setup-python@v4 with: python-version: "3.9" - - name: Install Pyenv + - name: Install Pyenv and Tox run: | python3 -m pip install --quiet virtualenv==16.7.9 --user rm -r venv || echo "no pre-existing venv" python3 -m virtualenv venv source venv/bin/activate + pip install --quiet tox==3.24.4 - name: Install yq if: github.event.inputs.auto-bump-version == 'true' && success() run: | @@ -273,10 +274,11 @@ jobs: sudo add-apt-repository ppa:rmescandon/yq sudo apt update sudo apt install yq -y - - name: Install CI scripts + - name: Test and install CI scripts # all CI python packages have the prefix "ci_" run: | source venv/bin/activate + tox -r -c ./tools/tox_ci.ini pip install --quiet -e ./tools/ci_* - name: Write Integration Test Credentials for ${{ matrix.connector }} run: | @@ -318,6 +320,19 @@ jobs: ./tools/integrations/manage.sh publish airbyte-integrations/${{ matrix.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache attempt_limit: 3 attempt_delay: 5000 in # ms + - name: Update Integration Test Credentials after test run for ${{ github.event.inputs.connector }} + if: always() + run: | + source venv/bin/activate + ci_credentials ${{ matrix.connector }} update-secrets + # normalization also runs destination-specific tests, so fetch their creds also + if [ 'bases/base-normalization' = "${{ matrix.connector }}" ] || [ 'base-normalization' = "${{ matrix.connector }}" ]; then + ci_credentials destination-bigquery update-secrets + ci_credentials destination-postgres update-secrets + ci_credentials destination-snowflake update-secrets + fi + env: + GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - name: Create Sentry Release if: startsWith(matrix.connector, 'connectors') && success() run: | From 37aa980deac9acdb08fecc4115270c7433884af3 Mon Sep 17 00:00:00 2001 From: grishick Date: Thu, 19 Jan 2023 08:21:58 -0800 Subject: [PATCH 7/8] remove workaround --- .github/workflows/publish-command.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 93e0be8770f9..89677481489d 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -260,13 +260,12 @@ jobs: uses: actions/setup-python@v4 with: python-version: "3.9" - - name: Install Pyenv and Tox + - name: Install Pyenv run: | python3 -m pip install --quiet virtualenv==16.7.9 --user rm -r venv || echo "no pre-existing venv" python3 -m virtualenv venv source venv/bin/activate - pip install --quiet tox==3.24.4 - name: Install yq if: github.event.inputs.auto-bump-version == 'true' && success() run: | @@ -274,11 +273,10 @@ jobs: sudo add-apt-repository ppa:rmescandon/yq sudo apt update sudo apt install yq -y - - name: Test and install CI scripts + - name: Install CI scripts # all CI python packages have the prefix "ci_" run: | source venv/bin/activate - tox -r -c ./tools/tox_ci.ini pip install --quiet -e ./tools/ci_* - name: Write Integration Test Credentials for ${{ matrix.connector }} run: | From e35bee64f1cfae51f19583ac8abd7e36eb6df6be Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Thu, 19 Jan 2023 16:44:05 +0000 Subject: [PATCH 8/8] auto-bump connector version --- .../src/main/resources/seed/destination_definitions.yaml | 2 +- .../init/src/main/resources/seed/destination_specs.yaml | 2 +- docs/reference/api/generated-api-html/index.html | 7 ------- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 507ddbb3b2ab..39285b789ebe 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -85,7 +85,7 @@ - name: Clickhouse destinationDefinitionId: ce0d828e-1dc4-496c-b122-2da42e637e48 dockerRepository: airbyte/destination-clickhouse - dockerImageTag: 0.2.1 + dockerImageTag: 0.2.2 documentationUrl: https://docs.airbyte.com/integrations/destinations/clickhouse icon: clickhouse.svg releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 7473ae8005e4..859ec2b6d8ab 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -1133,7 +1133,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-clickhouse:0.2.1" +- dockerImage: "airbyte/destination-clickhouse:0.2.2" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/clickhouse" connectionSpecification: diff --git a/docs/reference/api/generated-api-html/index.html b/docs/reference/api/generated-api-html/index.html index 7b3127f63cdb..340fc7c45794 100644 --- a/docs/reference/api/generated-api-html/index.html +++ b/docs/reference/api/generated-api-html/index.html @@ -1971,7 +1971,6 @@

Return type

Example data

Content-Type: application/json
{
-  "didUpdateConfiguration" : true,
   "message" : "message",
   "jobInfo" : {
     "createdAt" : 0,
@@ -2042,7 +2041,6 @@ 
Return type

     
Example data

     
Content-Type: application/json

     
{
-  "didUpdateConfiguration" : true,
   "message" : "message",
   "jobInfo" : {
     "createdAt" : 0,
@@ -5873,7 +5871,6 @@ 
Return type

     
Example data

     
Content-Type: application/json

     
{
-  "didUpdateConfiguration" : true,
   "message" : "message",
   "jobInfo" : {
     "createdAt" : 0,
@@ -5941,7 +5938,6 @@ 
Return type

     
Example data

     
Content-Type: application/json

     
{
-  "didUpdateConfiguration" : true,
   "message" : "message",
   "jobInfo" : {
     "createdAt" : 0,
@@ -6167,7 +6163,6 @@ 
Return type

     
Example data

     
Content-Type: application/json

     
{
-  "didUpdateConfiguration" : true,
   "message" : "message",
   "jobInfo" : {
     "createdAt" : 0,
@@ -6238,7 +6233,6 @@ 
Return type

     
Example data

     
Content-Type: application/json

     
{
-  "didUpdateConfiguration" : true,
   "message" : "message",
   "jobInfo" : {
     "createdAt" : 0,
@@ -10566,7 +10560,6 @@ 
CheckConnectionRead -  Enum:
         
succeeded
failed

 
message (optional)
String  

-
didUpdateConfiguration (optional)
Boolean  

 
jobInfo 
SynchronousJobRead