From 200a74d5e32a2ce45291bedbe9a5eec344249680 Mon Sep 17 00:00:00 2001 From: Neil Macneale V Date: Thu, 8 Dec 2022 15:37:56 -0800 Subject: [PATCH 1/5] Fix a bug where no terms or no values would crash discover() --- .../connectors/source-fauna/source_fauna/source.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/airbyte-integrations/connectors/source-fauna/source_fauna/source.py b/airbyte-integrations/connectors/source-fauna/source_fauna/source.py index 3a2d03ff3e2c..a0ba2a94b614 100644 --- a/airbyte-integrations/connectors/source-fauna/source_fauna/source.py +++ b/airbyte-integrations/connectors/source-fauna/source_fauna/source.py @@ -239,6 +239,9 @@ def discover(self, logger: AirbyteLogger, config: json) -> AirbyteCatalog: if ( type(source) is Ref and source.collection() == Ref("collections") + # If either of these are missing, the index has no terms or values + and "terms" in index + and "values" in index # Index must have 2 values and no terms and len(index["values"]) == 2 and len(index["terms"]) == 0 From 95d0af8d8bbd7011ba982518d97b004f402927f6 Mon Sep 17 00:00:00 2001 From: Neil Macneale V Date: Thu, 8 Dec 2022 15:38:14 -0800 Subject: [PATCH 2/5] Add a test to validate discover() against a real database --- .../source-fauna/unit_tests/database_test.py | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-fauna/unit_tests/database_test.py b/airbyte-integrations/connectors/source-fauna/unit_tests/database_test.py index 8ebaf61d675f..5acf67ae504e 100644 --- a/airbyte-integrations/connectors/source-fauna/unit_tests/database_test.py +++ b/airbyte-integrations/connectors/source-fauna/unit_tests/database_test.py @@ -89,6 +89,16 @@ def setup_database(source: SourceFauna): } ), ) + # This index just *existing* used to crash the connector, because it has no + # terms or values. + source.client.query( + q.create_index( + { + "name": "breaks_things", + "source": q.collection("foo"), + } + ), + ) print("Database is setup!") # Store all the refs and ts of the documents we created, so that we can validate them @@ -160,6 +170,28 @@ def setup_container(): raise +def run_discover_test(source: SourceFauna, logger): + # See `test_util.py` for these values + catalog = source.discover(logger, { + "secret": "secret", + "domain": "localhost", + "port": 9000, + "scheme": "http", + "collection": { + "page_size": 64, + "deletions": { + "deletion_mode": "ignore", + } + } + }) + assert len(catalog.streams) == 1 + stream = catalog.streams[0] + assert stream.name == "foo" + assert stream.supported_sync_modes == [SyncMode.full_refresh, SyncMode.incremental] + assert stream.source_defined_cursor is True + assert stream.default_cursor_field == ["ts"] + + def run_add_removes_test(source: SourceFauna, logger, stream: ConfiguredAirbyteStream): source._setup_client(FullConfig.localhost()) source.client.query(q.create(ref(105, "foo"), {"data": {"a": 10}})) @@ -477,6 +509,7 @@ def run_updates_test(db_data, source: SourceFauna, logger, catalog: ConfiguredAi def run_test(db_data, source: SourceFauna): logger = mock_logger() + run_discover_test(source, logger) stream = ConfiguredAirbyteStream( stream=AirbyteStream(name="foo", json_schema={}, supported_sync_modes=[SyncMode.incremental, SyncMode.full_refresh]), sync_mode=SyncMode.incremental, @@ -489,7 +522,7 @@ def run_test(db_data, source: SourceFauna): run_general_remove_test(source, logger) -def test_incremental_reads(): +def test_database(): container, db_data, source = setup_container() try: From 25f45ccc5230402921e37f6df5187ddbb3817819 Mon Sep 17 00:00:00 2001 From: Neil Macneale V Date: Thu, 8 Dec 2022 15:52:00 -0800 Subject: [PATCH 3/5] Cleanup conditional, and allow an index without terms as well as an index with no terms --- .../connectors/source-fauna/source_fauna/source.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/source-fauna/source_fauna/source.py b/airbyte-integrations/connectors/source-fauna/source_fauna/source.py index a0ba2a94b614..25a9915ace41 100644 --- a/airbyte-integrations/connectors/source-fauna/source_fauna/source.py +++ b/airbyte-integrations/connectors/source-fauna/source_fauna/source.py @@ -239,12 +239,9 @@ def discover(self, logger: AirbyteLogger, config: json) -> AirbyteCatalog: if ( type(source) is Ref and source.collection() == Ref("collections") - # If either of these are missing, the index has no terms or values - and "terms" in index - and "values" in index # Index must have 2 values and no terms - and len(index["values"]) == 2 - and len(index["terms"]) == 0 + and ("values" in index and len(index["values"]) == 2) + and (("terms" in index and len(index["terms"]) == 0) or "terms" not in index) # Index values must be ts and ref and index["values"][0] == {"field": "ts"} and index["values"][1] == {"field": "ref"} From 3799dff7e85c44d915cae5192c712cdd82eb4c2f Mon Sep 17 00:00:00 2001 From: Neil Macneale V Date: Mon, 12 Dec 2022 09:03:46 -0800 Subject: [PATCH 4/5] Bump connector version, and add PR to changelog --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-integrations/connectors/source-fauna/Dockerfile | 2 +- docs/integrations/sources/fauna.md | 7 ++++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 76037e114f4f..f18b506b3799 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -539,7 +539,7 @@ - name: Fauna sourceDefinitionId: 3825db3e-c94b-42ac-bd53-b5a9507ace2b dockerRepository: airbyte/source-fauna - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.1 documentationUrl: https://docs.airbyte.com/integrations/sources/fauna icon: fauna.svg sourceType: database diff --git a/airbyte-integrations/connectors/source-fauna/Dockerfile b/airbyte-integrations/connectors/source-fauna/Dockerfile index 3762d70736b4..8ec7ca3d45c7 100644 --- a/airbyte-integrations/connectors/source-fauna/Dockerfile +++ b/airbyte-integrations/connectors/source-fauna/Dockerfile @@ -34,5 +34,5 @@ COPY source_fauna ./source_fauna ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/source-fauna diff --git a/docs/integrations/sources/fauna.md b/docs/integrations/sources/fauna.md index a4c4b55079bd..5e93bbd48008 100644 --- a/docs/integrations/sources/fauna.md +++ b/docs/integrations/sources/fauna.md @@ -222,6 +222,7 @@ FQL [`Select`](https://docs.fauna.com/fauna/current/api/fql/functions/select) is ## Changelog -| Version | Date | Pull Request | Subject | -| ------- | ---------- | -------------------------------------------------------- | ---------------- | -| 0.1.0 | 2022-11-17 | [15274](https://github.com/airbytehq/airbyte/pull/15274) | Add Fauna Source | +| Version | Date | Pull Request | Subject | +| ------- | ---------- | -------------------------------------------------------- | ------------------------------- | +| 0.1.1 | 2022-12-12 | [20275](https://github.com/airbytehq/airbyte/pull/20275) | Fix index lookup with no values | +| 0.1.0 | 2022-11-17 | [15274](https://github.com/airbytehq/airbyte/pull/15274) | Add Fauna Source | From 0a350143a0f3a39f1027ef8989b955292e9fc4fc Mon Sep 17 00:00:00 2001 From: Neil Macneale V Date: Mon, 6 Feb 2023 15:36:32 -0800 Subject: [PATCH 5/5] Run ./gradlew :airbyte-config:init:processResources --- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 9a7ac6ef01fd..99e12515bfef 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -4112,7 +4112,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-fauna:0.1.0" +- dockerImage: "airbyte/source-fauna:0.1.1" spec: documentationUrl: "https://github.com/fauna/airbyte/blob/source-fauna/docs/integrations/sources/fauna.md" connectionSpecification: