SAT: declare bypass_reason in acceptance-test-config.yml (#18364)

airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## 0.2.12
+Declare `bypass_reason` field in test configuration. [#18364](https://github.com/airbytehq/airbyte/pull/18364).
+
 ## 0.2.11
 Declare `test_strictness_level` field in test configuration. [#18218](https://github.com/airbytehq/airbyte/pull/18218).
 

airbyte-integrations/bases/source-acceptance-test/Dockerfile

@@ -33,7 +33,7 @@ COPY pytest.ini setup.py ./
 COPY source_acceptance_test ./source_acceptance_test
 RUN pip install .
 
-LABEL io.airbyte.version=0.2.11
+LABEL io.airbyte.version=0.2.12
 LABEL io.airbyte.name=airbyte/source-acceptance-test
 
 ENTRYPOINT ["python", "-m", "pytest", "-p", "source_acceptance_test.plugin", "-r", "fEsx"]

airbyte-integrations/bases/source-acceptance-test/README.md

@@ -56,4 +56,15 @@ These iterations are more conveniently achieved by remaining in the current dire
 12. Open a PR on our GitHub repository
 13. Run the unit test on the CI by running `/test connector=bases/source-acceptance-test` in a GitHub comment
 14. Publish the new SAT version if your PR is approved by running `/publish connector=bases/source-acceptance-test auto-bump-version=false` in a GitHub comment
-15. Merge your PR
+15. Merge your PR
+
+## Migrating `acceptance-test-config.yml` to latest configuration format
+We introduced changes in the structure of `acceptance-test-config.yml` files in version 0.2.12.
+The *legacy* configuration format is still supported but should be deprecated soon.
+To migrate a legacy configuration to the latest configuration format please run:
+
+```bash
+python -m venv .venv # If you don't have a virtualenv already
+source ./.venv/bin/activate # If you're not in your virtualenv already
+python source_acceptance_test/utils/config_migration.py ../../connectors/source-to-migrate/acceptance-test-config.yml
+```

airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/config.py

@@ -3,11 +3,14 @@
 #
 
 
+import logging
+from copy import deepcopy
 from enum import Enum
 from pathlib import Path
-from typing import List, Mapping, Optional, Set
+from typing import Generic, List, Mapping, Optional, Set, TypeVar
 
-from pydantic import BaseModel, Field, validator
+from pydantic import BaseModel, Field, root_validator, validator
+from pydantic.generics import GenericModel
 
 config_path: str = Field(default="secrets/config.json", description="Path to a JSON object representing a valid connector configuration")
 invalid_config_path: str = Field(description="Path to a JSON object representing an invalid connector configuration")
@@ -18,13 +21,17 @@
 timeout_seconds: int = Field(default=None, description="Test execution timeout_seconds", ge=0)
 
 SEMVER_REGEX = r"(0|(?:[1-9]\d*))(?:\.(0|(?:[1-9]\d*))(?:\.(0|(?:[1-9]\d*)))?(?:\-([\w][\w\.\-_]*))?)?"
+ALLOW_LEGACY_CONFIG = True
 
 
 class BaseConfig(BaseModel):
     class Config:
         extra = "forbid"
 
 
+TestConfigT = TypeVar("TestConfigT")
+
+
 class BackwardCompatibilityTestsConfig(BaseConfig):
     previous_connector_version: str = Field(
         regex=SEMVER_REGEX, default="latest", description="Previous connector version to use for backward compatibility tests."
@@ -133,22 +140,86 @@ class IncrementalConfig(BaseConfig):
     )
 
 
-class TestConfig(BaseConfig):
-    spec: Optional[List[SpecTestConfig]] = Field(description="TODO")
-    connection: Optional[List[ConnectionTestConfig]] = Field(description="TODO")
-    discovery: Optional[List[DiscoveryTestConfig]] = Field(description="TODO")
-    basic_read: Optional[List[BasicReadTestConfig]] = Field(description="TODO")
-    full_refresh: Optional[List[FullRefreshConfig]] = Field(description="TODO")
-    incremental: Optional[List[IncrementalConfig]] = Field(description="TODO")
+class GenericTestConfig(GenericModel, Generic[TestConfigT]):
+    bypass_reason: Optional[str]
+    tests: Optional[List[TestConfigT]]
+
+    @validator("tests", always=True)
+    def no_bypass_reason_when_tests_is_set(cls, tests, values):
+        if tests and values.get("bypass_reason"):
+            raise ValueError("You can't set a bypass_reason if tests are set.")
+        return tests
+
+
+class AcceptanceTestConfigurations(BaseConfig):
+    spec: Optional[GenericTestConfig[SpecTestConfig]]
+    connection: Optional[GenericTestConfig[ConnectionTestConfig]]
+    discovery: Optional[GenericTestConfig[DiscoveryTestConfig]]
+    basic_read: Optional[GenericTestConfig[BasicReadTestConfig]]
+    full_refresh: Optional[GenericTestConfig[FullRefreshConfig]]
+    incremental: Optional[GenericTestConfig[IncrementalConfig]]
 
 
 class Config(BaseConfig):
     class TestStrictnessLevel(str, Enum):
         high = "high"
+        low = "low"
 
     connector_image: str = Field(description="Docker image to test, for example 'airbyte/source-hubspot:dev'")
-    tests: TestConfig = Field(description="List of the tests with their configs")
+    acceptance_tests: AcceptanceTestConfigurations = Field(description="List of the acceptance test to run with their configs")
     base_path: Optional[str] = Field(description="Base path for all relative paths")
     test_strictness_level: Optional[TestStrictnessLevel] = Field(
-        description="Corresponds to a strictness level of the test suite and will change which tests are mandatory for a successful run."
+        default=TestStrictnessLevel.low,
+        description="Corresponds to a strictness level of the test suite and will change which tests are mandatory for a successful run.",
     )
+
+    @staticmethod
+    def is_legacy(config: dict) -> bool:
+        """Check if a configuration is 'legacy'.
+        We consider it is legacy if a 'tests' field exists at its root level (prior to v0.2.12).
+
+        Args:
+            config (dict): A configuration
+
+        Returns:
+            bool: Whether the configuration is legacy.
+        """
+        return "tests" in config
+
+    @staticmethod
+    def migrate_legacy_to_current_config(legacy_config: dict) -> dict:
+        """Convert configuration structure created prior to v0.2.12 into the current structure.
+        e.g.
+        This structure:
+            {"connector_image": "my-connector-image", "tests": {"spec": [{"spec_path": "my/spec/path.json"}]}
+        Gets converted to:
+            {"connector_image": "my-connector-image", "acceptance_tests": {"spec": {"tests": [{"spec_path": "my/spec/path.json"}]}}
+
+        Args:
+            legacy_config (dict): A legacy configuration
+
+        Returns:
+            dict: A migrated configuration
+        """
+        migrated_config = deepcopy(legacy_config)
+        migrated_config.pop("tests")
+        migrated_config["acceptance_tests"] = {}
+        for test_name, test_configs in legacy_config["tests"].items():
+            migrated_config["acceptance_tests"][test_name] = {"tests": test_configs}
+        return migrated_config
+
+    @root_validator(pre=True)
+    def legacy_format_adapter(cls, values: dict) -> dict:
+        """Root level validator executed 'pre' field validation to migrate a legacy config to the current structure.
+
+        Args:
+            values (dict): The raw configuration.
+
+        Returns:
+            dict: The migrated configuration if needed.
+        """
+        if ALLOW_LEGACY_CONFIG and cls.is_legacy(values):
+            logging.warn("The acceptance-test-config.yml file is in a legacy format. Please migrate to the latest format.")
+            return cls.migrate_legacy_to_current_config(values)
+        else:
+            return values

airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/plugin.py

@@ -56,10 +56,10 @@ def pytest_generate_tests(metafunc):
         config_key = metafunc.cls.config_key()
         test_name = f"{metafunc.cls.__name__}.{metafunc.function.__name__}"
         config = load_config(metafunc.config.getoption("--acceptance-test-config"))
-        if not hasattr(config.tests, config_key) or not getattr(config.tests, config_key):
+        if not hasattr(config.acceptance_tests, config_key) or not getattr(config.acceptance_tests, config_key):
             pytest.skip(f"Skipping {test_name} because not found in the config")
         else:
-            test_inputs = getattr(config.tests, config_key)
+            test_inputs = getattr(config.acceptance_tests, config_key).tests
             if not test_inputs:
                 pytest.skip(f"Skipping {test_name} because no inputs provided")
 
@@ -87,8 +87,9 @@ def pytest_collection_modifyitems(config, items):
         if not hasattr(items[0].cls, "config_key"):
             # Skip user defined test classes from integration_tests/ directory.
             continue
-        test_configs = getattr(config.tests, items[0].cls.config_key())
-        for test_config, item in zip(test_configs, items):
+        test_configs = getattr(config.acceptance_tests, items[0].cls.config_key())
+
+        for test_config, item in zip(test_configs.tests, items):
             default_timeout = item.get_closest_marker("default_timeout")
             if test_config.timeout_seconds:
                 item.add_marker(pytest.mark.timeout(test_config.timeout_seconds))

airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/config_migration.py

@@ -0,0 +1,37 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+import argparse
+from pathlib import Path
+
+import yaml
+from source_acceptance_test.config import Config
+from yaml import load
+
+try:
+    from yaml import CLoader as Loader
+except ImportError:
+    from yaml import Loader
+
+parser = argparse.ArgumentParser(description="Migrate legacy acceptance-test-config.yml to the latest configuration format.")
+parser.add_argument("config_path", type=str, help="Path to the acceptance-test-config.yml to migrate.")
+
+
+def migrate_legacy_configuration(config_path: Path):
+
+    with open(config_path, "r") as file:
+        to_migrate = load(file, Loader=Loader)
+
+    if Config.is_legacy(to_migrate):
+        migrated_config = Config.migrate_legacy_to_current_config(to_migrate)
+        with open(config_path, "w") as output_file:
+            yaml.dump(migrated_config, output_file)
+        print(f"Your configuration was successfully migrated to the latest configuration format: {config_path}")
+    else:
+        print("Your configuration is not in a legacy format.")
+
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+    migrate_legacy_configuration(Path(args.config_path))