metadata: fix tests not running, fix failing non-running tests, fix v…

…alidate base images exist (#33127) Co-authored-by: erohmensing <erohmensing@users.noreply.github.com>
airbytehq · Dec 6, 2023 · 62b2020 · 62b2020
1 parent ff96866
commit 62b2020
Show file tree

Hide file tree

Showing 21 changed files with 180 additions and 45 deletions.
diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/gcs_upload.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/gcs_upload.py
@@ -217,7 +217,11 @@ def upload_metadata_to_gcs(bucket_name: str, metadata_file_path: Path, validator
     if metadata is None:
         raise ValueError(f"Metadata file {metadata_file_path} is invalid for uploading: {error}")
 
-    service_account_info = json.loads(os.environ.get("GCS_CREDENTIALS"))
+    gcs_creds = os.environ.get("GCS_CREDENTIALS")
+    if not gcs_creds:
+        raise ValueError("Please set the GCS_CREDENTIALS env var.")
+
+    service_account_info = json.loads(gcs_creds)
     credentials = service_account.Credentials.from_service_account_info(service_account_info)
     storage_client = storage.Client(credentials=credentials)
     bucket = storage_client.bucket(bucket_name)

diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/validators/metadata_validator.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/validators/metadata_validator.py
@@ -71,7 +71,7 @@ def validate_metadata_images_in_dockerhub(
 
     print(f"Checking that the following images are on dockerhub: {images_to_check}")
     for image, version in images_to_check:
-        if not is_image_on_docker_hub(image, version):
+        if not is_image_on_docker_hub(image, version, retries=3):
             return False, f"Image {image}:{version} does not exist in DockerHub"
 
     return True, None
@@ -151,7 +151,7 @@ def validate_metadata_base_images_in_dockerhub(
 ) -> ValidationResult:
     metadata_definition_dict = metadata_definition.dict()
 
-    image_address = get(metadata_definition_dict, "data.connectorOptions.baseImage")
+    image_address = get(metadata_definition_dict, "data.connectorBuildOptions.baseImage")
     if image_address is None:
         return True, None
 

diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/__init__.py b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/__init__.py
@@ -7,28 +7,45 @@
 
 def list_all_paths_in_fixture_directory(folder_name: str) -> List[str]:
     file_path = os.path.join(os.path.dirname(__file__), folder_name)
+
+    # If folder_name has subdirectories, os.walk will return a list of tuples,
+    # one for folder_name and one for each of its subdirectories.
+    fixture_files = []
     for root, dirs, files in os.walk(file_path):
-        return [os.path.join(root, file_name) for file_name in files]
+        fixture_files.extend(os.path.join(root, file_name) for file_name in files)
+    return fixture_files
 
 
 @pytest.fixture(scope="session")
 def valid_metadata_yaml_files() -> List[str]:
-    return list_all_paths_in_fixture_directory("metadata_validate/valid")
+    files = list_all_paths_in_fixture_directory("metadata_validate/valid")
+    if not files:
+        pytest.fail("No files found in metadata_validate/valid")
+    return files
 
 
 @pytest.fixture(scope="session")
 def invalid_metadata_yaml_files() -> List[str]:
-    return list_all_paths_in_fixture_directory("metadata_validate/invalid")
+    files = list_all_paths_in_fixture_directory("metadata_validate/invalid")
+    if not files:
+        pytest.fail("No files found in metadata_validate/invalid")
+    return files
 
 
 @pytest.fixture(scope="session")
 def valid_metadata_upload_files() -> List[str]:
-    return list_all_paths_in_fixture_directory("metadata_upload/valid")
+    files = list_all_paths_in_fixture_directory("metadata_upload/valid")
+    if not files:
+        pytest.fail("No files found in metadata_upload/valid")
+    return files
 
 
 @pytest.fixture(scope="session")
 def invalid_metadata_upload_files() -> List[str]:
-    return list_all_paths_in_fixture_directory("metadata_upload/invalid")
+    files = list_all_paths_in_fixture_directory("metadata_upload/invalid")
+    if not files:
+        pytest.fail("No files found in metadata_upload/invalid")
+    return files
 
 
 @pytest.fixture(scope="session")

diff --git a/...ata_upload/invalid/tag_nonexistent/metadata_breaking_change_image_tag_does_not_exist.yaml b/...ata_upload/invalid/tag_nonexistent/metadata_breaking_change_image_tag_does_not_exist.yaml
@@ -16,7 +16,7 @@ data:
   releaseStage: alpha
   releases:
     breakingChanges:
-      6.0.0: # tag does not exist
+      0.0.0: # tag does not exist
         upgradeDeadline: 2023-08-22
         message: "This version made a change."
   documentationUrl: https://docs.airbyte.com/integrations/sources/onesignal

diff --git a/...ures/metadata_upload/invalid/tag_nonexistent/metadata_cloud_image_tag_does_not_exist.yaml b/...ures/metadata_upload/invalid/tag_nonexistent/metadata_cloud_image_tag_does_not_exist.yaml
@@ -18,7 +18,7 @@ data:
     cloud:
       enabled: true
       dockerRepository: airbyte/exists-3
-      dockerImageTag: 6.6.6 # tag does not exist
+      dockerImageTag: 99.99.99 # tag does not exist
     oss:
       enabled: true
       dockerRepository: airbyte/exists-4

diff --git a/...tures/metadata_upload/invalid/tag_nonexistent/metadata_main_image_tag_does_not_exist.yaml b/...tures/metadata_upload/invalid/tag_nonexistent/metadata_main_image_tag_does_not_exist.yaml
@@ -5,7 +5,7 @@ data:
   connectorType: source
   dockerRepository: airbyte/exists-1
   githubIssueLabel: source-alloydb-strict-encrypt
-  dockerImageTag: 6.6.6 # tag does not exist
+  dockerImageTag: 99.99.99 # tag does not exist
   documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource
   connectorSubtype: database
   releaseStage: generally_available

diff --git a/...adata_upload/invalid/tag_nonexistent/metadata_normalization_image_tag_does_not_exist.yaml b/...adata_upload/invalid/tag_nonexistent/metadata_normalization_image_tag_does_not_exist.yaml
@@ -13,7 +13,7 @@ data:
   normalizationConfig:
     normalizationIntegrationType: postgres
     normalizationRepository: airbyte/exists-2
-    dockerImageTag: 6.6.6 # tag does not exist
+    normalizationTag: 99.99.99 # tag does not exist
   registries:
     cloud:
       enabled: true

diff --git a/...ts/fixtures/metadata_upload/invalid/tag_nonexistent/metadata_oss_repo_does_not_exist.yaml b/...ts/fixtures/metadata_upload/invalid/tag_nonexistent/metadata_oss_repo_does_not_exist.yaml
@@ -22,6 +22,6 @@ data:
     oss:
       enabled: true
       dockerRepository: airbyte/exists-4
-      dockerImageTag: 6.6.6 # tag does not exist
+      dockerImageTag: 99.99.99 # tag does not exist
   tags:
     - language:java
diff --git a/...e_tag_does_not_exist_but_is_overrode.yaml → ...e_tag_does_not_exist_but_is_overrode.yaml b/...e_tag_does_not_exist_but_is_overrode.yaml → ...e_tag_does_not_exist_but_is_overrode.yaml
@@ -5,7 +5,7 @@ data:
   connectorType: source
   dockerRepository: airbyte/exists-1
   githubIssueLabel: source-alloydb-strict-encrypt
-  dockerImageTag: 6.6.6 # tag does not exist
+  dockerImageTag: 99.99.99 # tag does not exist
   documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource
   connectorSubtype: database
   releaseStage: generally_available

diff --git a/..._repo_does_not_exist_but_is_overrode.yaml → ..._repo_does_not_exist_but_is_overrode.yaml b/..._repo_does_not_exist_but_is_overrode.yaml → ..._repo_does_not_exist_but_is_overrode.yaml
diff --git a/...metadata_service/lib/tests/fixtures/metadata_upload/valid/metadata_base_image_exists.yaml b/...metadata_service/lib/tests/fixtures/metadata_upload/valid/metadata_base_image_exists.yaml
@@ -6,7 +6,7 @@ data:
     hosts:
       - zopim.com
   connectorBuildOptions:
-    baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c
+    baseImage: docker.io/airbyte/base-repo-exists:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c
   connectorSubtype: api
   connectorType: source
   definitionId: 40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4

diff --git a/...ta_base_image_digest_does_not_exists.yaml → ...ta_base_image_digest_does_not_exists.yaml b/...ta_base_image_digest_does_not_exists.yaml → ...ta_base_image_digest_does_not_exists.yaml
@@ -6,7 +6,7 @@ data:
     hosts:
       - zopim.com
   connectorBuildOptions:
-    baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:doesnotexists
+    baseImage: docker.io/airbyte/base-repo-exists:1.1.0@sha256:MISSINGSHA
   connectorSubtype: api
   connectorType: source
   definitionId: 40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4

diff --git a/...data_base_image_name_does_not_exists.yaml → ...data_base_image_name_does_not_exists.yaml b/...data_base_image_name_does_not_exists.yaml → ...data_base_image_name_does_not_exists.yaml
diff --git a/...adata_base_image_tag_does_not_exists.yaml → ...adata_base_image_tag_does_not_exists.yaml b/...adata_base_image_tag_does_not_exists.yaml → ...adata_base_image_tag_does_not_exists.yaml
@@ -6,7 +6,7 @@ data:
     hosts:
       - zopim.com
   connectorBuildOptions:
-    baseImage: docker.io/airbyte/python-connector-base:does-not-exists@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c
+    baseImage: docker.io/airbyte/base-repo-exists:99.99.99@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c
   connectorSubtype: api
   connectorType: source
   definitionId: 40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4

diff --git a/...s/fixtures/metadata_validate/invalid/metadata_breaking_change_versions_under_releases.yml b/...s/fixtures/metadata_validate/invalid/metadata_breaking_change_versions_under_releases.yml
@@ -0,0 +1,18 @@
+metadataSpecVersion: 1.0
+data:
+  name: AlloyDB for PostgreSQL
+  definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002
+  connectorType: source
+  dockerRepository: airbyte/image-exists-1
+  githubIssueLabel: source-alloydb-strict-encrypt
+  dockerImageTag: 0.0.1
+  documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb
+  connectorSubtype: database
+  releaseStage: generally_available
+  license: MIT
+  releasestests/fixtures/metadata_validate/invalid/metadata_breaking_change_versions_under_releases.yml:
+    2.1.3:
+      message: "This version changes the connector’s authentication method from `ApiKey` to `oAuth`, per the [API guide](https://amazon-sqs.com/api/someguide)."
+      upgradeDeadline: 2023-08-22
+  tags:
+    - language:java
diff --git a/...tests/fixtures/metadata_validate/invalid/metadata_breaking_changes_not_under_releases.yml b/...tests/fixtures/metadata_validate/invalid/metadata_breaking_changes_not_under_releases.yml
@@ -0,0 +1,18 @@
+metadataSpecVersion: 1.0
+data:
+  name: AlloyDB for PostgreSQL
+  definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002
+  connectorType: source
+  dockerRepository: airbyte/image-exists-1
+  githubIssueLabel: source-alloydb-strict-encrypt
+  dockerImageTag: 0.0.1
+  documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb
+  connectorSubtype: database
+  releaseStage: generally_available
+  license: MIT
+  breakingChanges:
+    2.1.3:
+      message: "This version changes the connector’s authentication method from `ApiKey` to `oAuth`, per the [API guide](https://amazon-sqs.com/api/someguide)."
+      upgradeDeadline: 2023-08-22
+  tags:
+    - language:java
diff --git a/...rvice/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_base_image_no_sha.yml b/...rvice/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_base_image_no_sha.yml
@@ -0,0 +1,29 @@
+data:
+  allowedHosts:
+    hosts:
+      - "*.googleapis.com"
+  connectorBuildOptions:
+    baseImage: docker.io/airbyte/base-repo-exists:1.1.0
+  connectorSubtype: file
+  connectorType: source
+  definitionId: 71607ba1-c0ac-4799-8049-7f4b90dd50f7
+  dockerImageTag: 0.3.7
+  dockerRepository: airbyte/source-google-sheets
+  githubIssueLabel: source-google-sheets
+  icon: google-sheets.svg
+  license: Elv2
+  name: Google Sheets
+  registries:
+    cloud:
+      enabled: true
+    oss:
+      enabled: true
+  releaseStage: generally_available
+  documentationUrl: https://docs.airbyte.com/integrations/sources/google-sheets
+  tags:
+    - language:python
+  ab_internal:
+    sl: 300
+    ql: 400
+  supportLevel: certified
+metadataSpecVersion: "1.0"
diff --git a/...etadata_validate/invalid/metadata_major_version_no_breaking_change_entry_for_version.yaml b/...etadata_validate/invalid/metadata_major_version_no_breaking_change_entry_for_version.yaml
@@ -16,9 +16,10 @@ data:
       enabled: true
   releaseStage: alpha
   releases:
-    1.0.0:
-      upgradeDeadline: 2023-08-22
-      message: "This version made a change."
+    breakingChanges:
+      1.0.0:
+        upgradeDeadline: 2023-08-22
+        message: "This version made a change."
   documentationUrl: https://docs.airbyte.com/integrations/sources/onesignal
   tags:
     - language:python
diff --git a/...etadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_build_base_image.yaml b/...etadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_build_base_image.yaml
@@ -3,7 +3,7 @@ data:
     hosts:
       - "*.googleapis.com"
   connectorBuildOptions:
-    baseImage: airbyte/python-connector-base:1.0.0
+    baseImage: docker.io/airbyte/base-repo-exists:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c
   connectorSubtype: file
   connectorType: source
   definitionId: 71607ba1-c0ac-4799-8049-7f4b90dd50f7

diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/test_commands.py b/airbyte-ci/connectors/metadata_service/lib/tests/test_commands.py
@@ -10,12 +10,16 @@
 from metadata_service.gcs_upload import MetadataUploadInfo, UploadedFile
 from metadata_service.validators.metadata_validator import ValidatorOptions
 from pydantic import BaseModel, ValidationError, error_wrappers
+from test_gcs_upload import stub_is_image_on_docker_hub
 
 
 # TEST VALIDATE COMMAND
-def test_valid_metadata_yaml_files(valid_metadata_yaml_files, tmp_path):
+def test_valid_metadata_yaml_files(mocker, valid_metadata_yaml_files, tmp_path):
     runner = CliRunner()
 
+    # Mock dockerhub for base image checks
+    mocker.patch("metadata_service.validators.metadata_validator.is_image_on_docker_hub", side_effect=stub_is_image_on_docker_hub)
+
     assert len(valid_metadata_yaml_files) > 0, "No files found"
 
     for file_path in valid_metadata_yaml_files:
@@ -30,13 +34,13 @@ def test_invalid_metadata_yaml_files(invalid_metadata_yaml_files, tmp_path):
 
     for file_path in invalid_metadata_yaml_files:
         result = runner.invoke(commands.validate, [file_path, str(tmp_path)])
-        assert result.exit_code != 0, f"Validation succeeded (when it shouldve failed) for {file_path}"
+        assert result.exit_code != 0, f"Validation succeeded (when it should have failed) for {file_path}"
 
 
 def test_metadata_file_not_found_fails(tmp_path):
     runner = CliRunner()
     result = runner.invoke(commands.validate, ["non_existent_file.yaml", str(tmp_path)])
-    assert result.exit_code != 0, "Validation succeeded (when it shouldve failed) for non_existent_file.yaml"
+    assert result.exit_code != 0, "Validation succeeded (when it should have failed) for non_existent_file.yaml"
 
 
 def test_docs_path_not_found_fails(valid_metadata_yaml_files):
@@ -45,7 +49,7 @@ def test_docs_path_not_found_fails(valid_metadata_yaml_files):
     assert len(valid_metadata_yaml_files) > 0, "No files found"
 
     result = runner.invoke(commands.validate, [valid_metadata_yaml_files[0], "non_existent_docs_path"])
-    assert result.exit_code != 0, "Validation succeeded (when it shouldve failed) for non_existent_docs_path"
+    assert result.exit_code != 0, "Validation succeeded (when it should have failed) for non_existent_docs_path"
 
 
 def mock_metadata_upload_info(