Connector acceptance test: Fix discovered catalog caching for differe…

…nt configs (#22301) Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>
airbytehq · Feb 6, 2023 · 449c3d8 · 449c3d8
1 parent e39b90f
commit 449c3d8
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 3 deletions.
diff --git a/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md b/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md
@@ -1,5 +1,8 @@
 # Changelog
 
+## 0.5.1
+Fix discovered catalog caching for different configs. [#22301](https://github.com/airbytehq/airbyte/pull/22301)
+
 ## 0.5.0
 Re-release of 0.3.0 [#21451](https://github.com/airbytehq/airbyte/pull/21451)
 

diff --git a/airbyte-integrations/bases/connector-acceptance-test/Dockerfile b/airbyte-integrations/bases/connector-acceptance-test/Dockerfile
@@ -33,7 +33,7 @@ COPY pytest.ini setup.py ./
 COPY connector_acceptance_test ./connector_acceptance_test
 RUN pip install .
 
-LABEL io.airbyte.version=0.5.0
+LABEL io.airbyte.version=0.5.1
 LABEL io.airbyte.name=airbyte/connector-acceptance-test
 
 ENTRYPOINT ["python", "-m", "pytest", "-p", "connector_acceptance_test.plugin", "-r", "fEsx"]
diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py
@@ -26,6 +26,7 @@
     filter_output,
     load_config,
     load_yaml_or_json_path,
+    make_hashable,
 )
 from docker import errors
 
@@ -259,7 +260,10 @@ def discovered_catalog_fixture(
     connector_config, docker_runner: ConnectorRunner, cached_schemas, cache_discovered_catalog: bool
 ) -> MutableMapping[str, AirbyteStream]:
     """JSON schemas for each stream"""
-    if not cached_schemas or not cache_discovered_catalog:
+    cached_schemas = cached_schemas.setdefault(make_hashable(connector_config), {})
+    if not cache_discovered_catalog:
+        cached_schemas.clear()
+    if not cached_schemas:
         output = docker_runner.call_discover(config=connector_config)
         catalogs = [message.catalog for message in output if message.type == Type.CATALOG]
         for stream in catalogs[-1].streams:
@@ -269,14 +273,17 @@ def discovered_catalog_fixture(
 
 @pytest.fixture(name="previous_discovered_catalog")
 def previous_discovered_catalog_fixture(
-    connector_config, previous_connector_docker_runner: ConnectorRunner, previous_cached_schemas
+    connector_config, previous_connector_docker_runner: ConnectorRunner, previous_cached_schemas, cache_discovered_catalog: bool
 ) -> MutableMapping[str, AirbyteStream]:
     """JSON schemas for each stream"""
     if previous_connector_docker_runner is None:
         logging.warning(
             "\n We could not retrieve the previous discovered catalog as a connector runner for the previous connector version could not be instantiated."
         )
         return None
+    previous_cached_schemas = previous_cached_schemas.setdefault(make_hashable(connector_config), {})
+    if not cache_discovered_catalog:
+        previous_cached_schemas.clear()
     if not previous_cached_schemas:
         output = previous_connector_docker_runner.call_discover(config=connector_config)
         catalogs = [message.catalog for message in output if message.type == Type.CATALOG]