Ensure we do an upload validate on the overrode metadata file

airbyte-ci/connectors/metadata_service/lib/metadata_service/gcs_upload.py

@@ -91,25 +91,28 @@ def upload_file_if_changed(
 
 
 def _latest_upload(metadata: ConnectorMetadataDefinitionV0, bucket: storage.bucket.Bucket, metadata_file_path: Path) -> Tuple[bool, str]:
-    version_path = get_metadata_remote_file_path(metadata.data.dockerRepository, metadata.data.dockerImageTag)
     latest_path = get_metadata_remote_file_path(metadata.data.dockerRepository, "latest")
-    latest_icon_path = get_icon_remote_file_path(metadata.data.dockerRepository, "latest")
+    return upload_file_if_changed(metadata_file_path, bucket, latest_path, disable_cache=True)
+
+
+def _version_upload(metadata: ConnectorMetadataDefinitionV0, bucket: storage.bucket.Bucket, metadata_file_path: Path) -> Tuple[bool, str]:
+    version_path = get_metadata_remote_file_path(metadata.data.dockerRepository, metadata.data.dockerImageTag)
+    return upload_file_if_changed(metadata_file_path, bucket, version_path, disable_cache=True)
 
-    (
-        version_uploaded,
-        version_blob_id,
-    ) = upload_file_if_changed(metadata_file_path, bucket, version_path, disable_cache=True)
-    latest_uploaded, _latest_blob_id = upload_file_if_changed(metadata_file_path, bucket, latest_path, disable_cache=True)
 
-    # Replace metadata file name with icon file name
+def _icon_upload(metadata: ConnectorMetadataDefinitionV0, bucket: storage.bucket.Bucket, metadata_file_path: Path) -> Tuple[bool, str]:
     local_icon_path = metadata_file_path.parent / ICON_FILE_NAME
-    if local_icon_path.exists():
-        upload_file_if_changed(local_icon_path, bucket, latest_icon_path)
+    latest_icon_path = get_icon_remote_file_path(metadata.data.dockerRepository, "latest")
+    if not local_icon_path.exists():
+        return False, f"No Icon found at {local_icon_path}"
+    return upload_file_if_changed(local_icon_path, bucket, latest_icon_path)
 
-    return version_uploaded or latest_uploaded, version_blob_id
 
+def create_prerelease_metadata_file(metadata_file_path: Path, prerelease_tag: str) -> Path:
+    metadata, error = validate_and_load(metadata_file_path, [])
+    if metadata is None:
+        raise ValueError(f"Metadata file {metadata_file_path} is invalid for uploading: {error}")
 
-def _prerelease_upload(metadata: ConnectorMetadataDefinitionV0, bucket: storage.bucket.Bucket, prerelease_tag: str) -> Tuple[bool, str]:
     # replace any dockerImageTag references with the actual tag
     # this includes metadata.data.dockerImageTag, metadata.data.registries[].dockerImageTag
     # where registries is a dictionary of registry name to registry object
@@ -125,8 +128,7 @@ def _prerelease_upload(metadata: ConnectorMetadataDefinitionV0, bucket: storage.
     with open(tmp_metadata_file_path, "w") as f:
         yaml.dump(metadata_dict, f)
 
-    prerelease_remote_path = get_metadata_remote_file_path(metadata.data.dockerRepository, prerelease_tag)
-    return upload_file_if_changed(tmp_metadata_file_path, bucket, prerelease_remote_path, disable_cache=True)
+    return tmp_metadata_file_path
 
 
 def upload_metadata_to_gcs(bucket_name: str, metadata_file_path: Path, prerelease: Optional[str] = None) -> Tuple[bool, str]:
@@ -143,6 +145,12 @@ def upload_metadata_to_gcs(bucket_name: str, metadata_file_path: Path, prereleas
     Returns:
         Tuple[bool, str]: Whether the metadata file was uploaded and its blob id.
     """
+    version_uploaded = False
+    latest_uploaded = False
+
+    if prerelease:
+        metadata_file_path = create_prerelease_metadata_file(metadata_file_path, prerelease)
+
     metadata, error = validate_and_load(metadata_file_path, POST_UPLOAD_VALIDATORS)
 
     if metadata is None:
@@ -153,7 +161,10 @@ def upload_metadata_to_gcs(bucket_name: str, metadata_file_path: Path, prereleas
     storage_client = storage.Client(credentials=credentials)
     bucket = storage_client.bucket(bucket_name)
 
-    if prerelease:
-        return _prerelease_upload(metadata, bucket, prerelease)
-    else:
-        return _latest_upload(metadata, bucket, metadata_file_path)
+    _icon_upload(metadata, bucket, metadata_file_path)
+
+    version_uploaded, version_blob_id = _version_upload(metadata, bucket, metadata_file_path)
+    if not prerelease:
+        latest_uploaded, _latest_blob_id = _latest_upload(metadata, bucket, metadata_file_path)
+
+    return version_uploaded or latest_uploaded, version_blob_id

airbyte-ci/connectors/metadata_service/lib/tests/test_gcs_upload.py

@@ -84,7 +84,7 @@ def setup_upload_mocks(mocker, version_blob_md5_hash, latest_blob_md5_hash, loca
 def test_upload_metadata_to_gcs_valid_metadata(
     mocker, valid_metadata_upload_files, version_blob_md5_hash, latest_blob_md5_hash, local_file_md5_hash
 ):
-    mocker.spy(gcs_upload, "_prerelease_upload")
+    mocker.spy(gcs_upload, "_version_upload")
     mocker.spy(gcs_upload, "_latest_upload")
 
     for valid_metadata_upload_file in valid_metadata_upload_files:
@@ -107,7 +107,7 @@ def test_upload_metadata_to_gcs_valid_metadata(
 
         # Assertions
 
-        gcs_upload._prerelease_upload.assert_not_called()
+        gcs_upload._version_upload.assert_called()
         gcs_upload._latest_upload.assert_called()
 
         gcs_upload.service_account.Credentials.from_service_account_info.assert_called_with(json.loads(mocks["service_account_json"]))
@@ -133,7 +133,7 @@ def test_upload_metadata_to_gcs_valid_metadata(
 
         # clear the call count
         gcs_upload._latest_upload.reset_mock()
-        gcs_upload._prerelease_upload.assert_not_called()
+        gcs_upload._version_upload.reset_mock()
 
 
 def test_upload_metadata_to_gcs_non_existent_metadata_file():
@@ -173,14 +173,14 @@ def test_upload_metadata_to_gcs_invalid_docker_images(mocker, invalid_metadata_u
 def test_upload_metadata_to_gcs_with_prerelease(mocker, valid_metadata_upload_files):
     # Arrange
     setup_upload_mocks(mocker, "new_md5_hash1", "new_md5_hash2", "new_md5_hash3")
-    mocker.patch("metadata_service.gcs_upload._latest_upload", return_value=None)
-    mocker.patch("metadata_service.gcs_upload.upload_file_if_changed", return_value=None)
-    mocker.spy(gcs_upload, "_prerelease_upload")
+    mocker.patch("metadata_service.gcs_upload._latest_upload", return_value=(True, "someid"))
+    mocker.patch("metadata_service.gcs_upload.upload_file_if_changed", return_value=(True, "someid"))
+    mocker.spy(gcs_upload, "_version_upload")
 
     for valid_metadata_upload_file in valid_metadata_upload_files:
         # Assuming there is a valid metadata file in the list, if not, you might need to create one
         metadata_file_path = pathlib.Path(valid_metadata_upload_file)
-        prerelease_image_tag = "prerelease_image_tag"
+        prerelease_image_tag = "1.5.6-dev.f80318f754"
 
         gcs_upload.upload_metadata_to_gcs(
             "my_bucket",
@@ -190,18 +190,16 @@ def test_upload_metadata_to_gcs_with_prerelease(mocker, valid_metadata_upload_fi
 
         gcs_upload._latest_upload.assert_not_called()
 
-        # Assert that _prerelease_upload is called and the third argument is prerelease_image_tag
+        # Assert that _version_upload is called and the third argument is prerelease_image_tag
         # Ignore the first and second arguments (_ and __ are often used as placeholder variable names in Python when you don't care about the value)
-        _, __, prerelease_tag = gcs_upload._prerelease_upload.call_args[0]
-        assert prerelease_tag == prerelease_image_tag
+        _, __, tmp_metadata_file_path = gcs_upload._version_upload.call_args[0]
+        assert prerelease_image_tag in str(tmp_metadata_file_path)
 
         # Verify the tmp metadata is overridden
-        metadata, error = gcs_upload.validate_and_load(metadata_file_path, [])
-        expected_tmp_metadata_file_path = pathlib.Path("/tmp") / metadata.data.dockerRepository / prerelease_image_tag / METADATA_FILE_NAME
-        assert expected_tmp_metadata_file_path.exists(), f"{expected_tmp_metadata_file_path} does not exist"
+        assert tmp_metadata_file_path.exists(), f"{tmp_metadata_file_path} does not exist"
 
         # verify that the metadata is overrode
-        tmp_metadata, error = gcs_upload.validate_and_load(expected_tmp_metadata_file_path, [])
+        tmp_metadata, error = gcs_upload.validate_and_load(tmp_metadata_file_path, [])
         tmp_metadata_dict = to_json_sanitized_dict(tmp_metadata, exclude_none=True)
         assert tmp_metadata_dict["data"]["dockerImageTag"] == prerelease_image_tag
         for registry in get(tmp_metadata_dict, "data.registries", {}).values():