From ea3fb898aa0d9ec71094d4a5a11cc6b1b795c477 Mon Sep 17 00:00:00 2001 From: Edward Gao Date: Mon, 5 Dec 2022 17:08:21 -0800 Subject: [PATCH] Source Postgres: Fix not being able to configure wal2json plugin (unpublished) (#19985) * switch const to default * update expected specs * try updating system-stubs? * Revert "try updating system-stubs?" This reverts commit 0311a5a1d15eedb83e4c9b82ecf33f4c264ae60d. * switch to system stubs * changelog * fix more tests --- .../sources/CdcBinlogsMySqlSourceDatatypeTest.java | 2 +- .../connectors/source-postgres/src/main/resources/spec.json | 2 +- .../src/test-integration/resources/expected_spec.json | 2 +- .../resources/expected_strict_encrypt_spec.json | 2 +- .../source-postgres/src/test/resources/expected_spec.json | 2 +- docs/integrations/sources/postgres.md | 1 + 6 files changed, 6 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcBinlogsMySqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcBinlogsMySqlSourceDatatypeTest.java index b0bbe03185f5..3b5a43206a2f 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcBinlogsMySqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcBinlogsMySqlSourceDatatypeTest.java @@ -45,7 +45,7 @@ protected List runRead(final ConfiguredAirbyteCatalog configured } @Override - protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { + protected void postSetup() throws Exception { final Database database = setupDatabase(); initTests(); for (final TestDataHolder test : testDataHolders) { diff --git a/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json b/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json index 31c5acf4a36b..9e5a16a33181 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json @@ -247,7 +247,7 @@ "title": "Plugin", "description": "A logical decoding plugin installed on the PostgreSQL server. The `pgoutput` plugin is used by default. If the replication table contains a lot of big jsonb values it is recommended to use `wal2json` plugin. Read more about selecting replication plugins.", "enum": ["pgoutput", "wal2json"], - "const": "pgoutput", + "default": "pgoutput", "order": 2 }, "replication_slot": { diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/resources/expected_spec.json b/airbyte-integrations/connectors/source-postgres/src/test-integration/resources/expected_spec.json index 8b0fa6875170..7ee9abfc9e98 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/resources/expected_spec.json @@ -247,7 +247,7 @@ "title": "Plugin", "description": "A logical decoding plugin installed on the PostgreSQL server. The `pgoutput` plugin is used by default. If the replication table contains a lot of big jsonb values it is recommended to use `wal2json` plugin. Read more about selecting replication plugins.", "enum": ["pgoutput", "wal2json"], - "const": "pgoutput", + "default": "pgoutput", "order": 2 }, "replication_slot": { diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/resources/expected_strict_encrypt_spec.json b/airbyte-integrations/connectors/source-postgres/src/test-integration/resources/expected_strict_encrypt_spec.json index 596d49c26e02..78f0de73bf00 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/resources/expected_strict_encrypt_spec.json +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/resources/expected_strict_encrypt_spec.json @@ -227,7 +227,7 @@ "title": "Plugin", "description": "A logical decoding plugin installed on the PostgreSQL server. The `pgoutput` plugin is used by default. If the replication table contains a lot of big jsonb values it is recommended to use `wal2json` plugin. Read more about selecting replication plugins.", "enum": ["pgoutput", "wal2json"], - "const": "pgoutput", + "default": "pgoutput", "order": 2 }, "replication_slot": { diff --git a/airbyte-integrations/connectors/source-postgres/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/source-postgres/src/test/resources/expected_spec.json index 596d49c26e02..78f0de73bf00 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-postgres/src/test/resources/expected_spec.json @@ -227,7 +227,7 @@ "title": "Plugin", "description": "A logical decoding plugin installed on the PostgreSQL server. The `pgoutput` plugin is used by default. If the replication table contains a lot of big jsonb values it is recommended to use `wal2json` plugin. Read more about selecting replication plugins.", "enum": ["pgoutput", "wal2json"], - "const": "pgoutput", + "default": "pgoutput", "order": 2 }, "replication_slot": { diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index 472c32b7cec4..a94731ff8c50 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -415,6 +415,7 @@ The root causes is that the WALs needed for the incremental sync has been remove | Version | Date | Pull Request | Subject | |:--------|:-----------|:-------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| (none) | 2022-12-02 | [19985](https://github.com/airbytehq/airbyte/pull/19985) | Reenable incorrectly-disabled `wal2json` CDC plugin | | 1.0.30 | 2022-11-29 | [19024](https://github.com/airbytehq/airbyte/pull/19024) | Skip tables from schema where user do not have Usage permission during discovery. | | 1.0.29 | 2022-11-29 | [19623](https://github.com/airbytehq/airbyte/pull/19623) | Mark PSQLException related to using replica that is configured as a hot standby server as config error. | | 1.0.28 | 2022-11-28 | [19514](https://github.com/airbytehq/airbyte/pull/19514) | Adjust batch selection memory limits databases. |