Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Revert "Upload Flow - Reduce API Calls" #10778

Merged
merged 1 commit into from
Jan 7, 2021
Merged

Revert "Upload Flow - Reduce API Calls" #10778

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
132 changes: 34 additions & 98 deletions Tests/Marketplace/Tests/marketplace_services_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

from Tests.Marketplace.marketplace_services import Pack, Metadata, input_to_list, get_valid_bool, convert_price, \
get_higher_server_version, GCPConfig, BucketUploadFlow, PackStatus, load_json, \
store_successful_and_failed_packs_in_ci_artifacts, PACKS_FOLDER
store_successful_and_failed_packs_in_ci_artifacts


@pytest.fixture(scope="module")
Expand Down Expand Up @@ -330,6 +330,27 @@ def test_remove_unwanted_files(self):
assert os.path.isdir('Tests/Marketplace/Tests/test_data/pack_to_test/Integrations')
shutil.rmtree('Tests/Marketplace/Tests/test_data/pack_to_test')

@pytest.mark.parametrize('file_name, result', [
('Author_image.png', False),
('Integration_image.png', True),
('Integration_image.jpeg', False)
])
def test_is_integration_image(self, file_name, result):
"""
Given:
- Image name of an author.
- Image name of integration.
- Image name of integration with the wrong extension.
When:
- Checking whether the image in integration image or not
Then:
- Validate that the answer is False
- Validate that the answer is True
- Validate that the answer is False
"""
from Tests.Marketplace.marketplace_services import is_integration_image
assert is_integration_image(file_name) == result


class TestVersionSorting:
""" Class for sorting of changelog.json versions
Expand Down Expand Up @@ -623,18 +644,14 @@ def test_upload_integration_images_with_special_character(self, mocker, dummy_pa
"""
temp_image_name = f'{integration_name.replace(" ", "")}_image.png'
search_for_images_return_value = [{'display_name': integration_name,
'image_path': f'/path/{temp_image_name}',
'integration_path_basename': 'fake_unified_integration_path'}]
'image_path': f'/path/{temp_image_name}'}]
mocker.patch("marketplace_services_test.Pack._search_for_images", return_value=search_for_images_return_value)
mocker.patch("marketplace_services_test.Pack.need_to_upload_integration_image", return_value=True)
mocker.patch('builtins.open', mock_open(read_data="image_data"))
mocker.patch("Tests.Marketplace.marketplace_services.logging")
dummy_storage_bucket = mocker.MagicMock()
dummy_file = mocker.MagicMock()
dummy_file.a_path = os.path.join(PACKS_FOLDER, "TestPack", temp_image_name)
dummy_storage_bucket.blob.return_value.name = os.path.join(GCPConfig.STORAGE_BASE_PATH, "TestPack",
temp_image_name)
task_status, integration_images = dummy_pack.upload_integration_images(dummy_storage_bucket, [dummy_file])
task_status, integration_images = dummy_pack.upload_integration_images(storage_bucket=dummy_storage_bucket)

assert task_status
assert len(expected_result) == len(integration_images)
Expand All @@ -658,24 +675,20 @@ def test_upload_integration_images_without_special_character(self, mocker, dummy
"""
temp_image_name = f'{integration_name.replace(" ", "")}_image.png'
search_for_images_return_value = [{'display_name': integration_name,
'image_path': f'/path/{temp_image_name}',
'integration_path_basename': 'fake_unified_integration_path'}]
'image_path': f'/path/{temp_image_name}'}]
mocker.patch("marketplace_services_test.Pack._search_for_images", return_value=search_for_images_return_value)
mocker.patch("marketplace_services_test.Pack.need_to_upload_integration_image", return_value=True)
mocker.patch("builtins.open", mock_open(read_data="image_data"))
mocker.patch("Tests.Marketplace.marketplace_services.logging")
dummy_storage_bucket = mocker.MagicMock()
dummy_file = mocker.MagicMock()
dummy_file.a_path = os.path.join(PACKS_FOLDER, "TestPack", temp_image_name)
dummy_storage_bucket.blob.return_value.name = os.path.join(GCPConfig.STORAGE_BASE_PATH, "TestPack",
temp_image_name)
task_status, integration_images = dummy_pack.upload_integration_images(dummy_storage_bucket, [dummy_file])
task_status, integration_images = dummy_pack.upload_integration_images(storage_bucket=dummy_storage_bucket)

assert task_status
assert len(expected_result) == len(integration_images)
assert integration_images == expected_result

def test_copy_integration_images(self, mocker, dummy_pack):
def test_copy_and_upload_integration_images(self, mocker, dummy_pack):
"""
Given:
- Integration image.
Expand All @@ -688,13 +701,13 @@ def test_copy_integration_images(self, mocker, dummy_pack):
dummy_prod_bucket = mocker.MagicMock()
blob_name = "content/packs/TestPack/IntegrationName_image.png"
dummy_build_bucket.list_blobs.return_value = [Blob(blob_name, dummy_build_bucket)]
mocker.patch("Tests.Marketplace.marketplace_services.is_integration_image", return_value=True)
mocker.patch("Tests.Marketplace.marketplace_services.logging")
dummy_build_bucket.copy_blob.return_value = Blob('copied_blob', dummy_prod_bucket)
images_data = {"TestPack": {BucketUploadFlow.INTEGRATIONS: [os.path.basename(blob_name)]}}
task_status = dummy_pack.copy_integration_images(dummy_prod_bucket, dummy_build_bucket, images_data)
task_status = dummy_pack.copy_integration_images(dummy_prod_bucket, dummy_build_bucket)
assert task_status

def test_copy_author_image(self, mocker, dummy_pack):
def test_copy_and_upload_author_image(self, mocker, dummy_pack):
"""
Given:
- Author image.
Expand All @@ -707,9 +720,8 @@ def test_copy_author_image(self, mocker, dummy_pack):
dummy_prod_bucket = mocker.MagicMock()
mocker.patch("Tests.Marketplace.marketplace_services.logging")
blob_name = "content/packs/TestPack/Author_image.png"
images_data = {"TestPack": {BucketUploadFlow.AUTHOR: True}}
dummy_build_bucket.copy_blob.return_value = Blob(blob_name, dummy_prod_bucket)
task_status = dummy_pack.copy_author_image(dummy_prod_bucket, dummy_build_bucket, images_data)
task_status = dummy_pack.copy_author_image(dummy_prod_bucket, dummy_build_bucket)
assert task_status


Expand Down Expand Up @@ -1363,18 +1375,16 @@ def test_get_successful_and_failed_packs(self, tmp_path):
file = os.path.join(tmp_path, BucketUploadFlow.PACKS_RESULTS_FILE)

# Case 1: assert file does not exist
successful, failed, images = get_successful_and_failed_packs(file, BucketUploadFlow.PREPARE_CONTENT_FOR_TESTING)
successful, failed = get_successful_and_failed_packs(file, BucketUploadFlow.PREPARE_CONTENT_FOR_TESTING)
assert successful == {}
assert failed == {}
assert images == {}

# Case 2: assert empty file
with open(file, "w") as f:
f.write('')
successful, failed, imges = get_successful_and_failed_packs(file, BucketUploadFlow.PREPARE_CONTENT_FOR_TESTING)
successful, failed = get_successful_and_failed_packs(file, BucketUploadFlow.PREPARE_CONTENT_FOR_TESTING)
assert successful == {}
assert failed == {}
assert images == {}

# Case 3: assert valid file
with open(file, "w") as f:
Expand All @@ -1391,16 +1401,10 @@ def test_get_successful_and_failed_packs(self, tmp_path):
f"{BucketUploadFlow.STATUS}": "status1",
f"{BucketUploadFlow.AGGREGATED}": True
}
},
f"{BucketUploadFlow.IMAGES}": {
"TestPack1": {
f"{BucketUploadFlow.AUTHOR}": True,
f"{BucketUploadFlow.INTEGRATIONS}": ["integration_image.png"]
}
}
}
}))
successful, failed, images = get_successful_and_failed_packs(file, BucketUploadFlow.PREPARE_CONTENT_FOR_TESTING)
successful, failed = get_successful_and_failed_packs(file, BucketUploadFlow.PREPARE_CONTENT_FOR_TESTING)
assert successful == {"TestPack1": {
f"{BucketUploadFlow.STATUS}": "status1",
f"{BucketUploadFlow.AGGREGATED}": True
Expand All @@ -1415,71 +1419,3 @@ def test_get_successful_and_failed_packs(self, tmp_path):
failed_list = [*failed]
ans = 'TestPack2' in failed_list
assert ans
assert "TestPack1" in images
test_pack_images = images.get("TestPack1", {})
assert BucketUploadFlow.AUTHOR in test_pack_images
assert test_pack_images.get(BucketUploadFlow.AUTHOR, False)
assert BucketUploadFlow.INTEGRATIONS in test_pack_images
integration_images = test_pack_images.get(BucketUploadFlow.INTEGRATIONS, [])
assert len(integration_images) == 1
assert integration_images[0] == "integration_image.png"


class TestImageClassification:
""" Test class for all image classifications.
"""

@pytest.fixture(scope="class")
def dummy_pack(self):
""" dummy pack fixture
"""
return Pack(pack_name="TestPack", pack_path="dummy_path")

@pytest.mark.parametrize('file_path, result', [
('Packs/TestPack/Author_image.png', False),
('Packs/TestPack/Integration_image.png', True),
('Packs/TestPack/Integration_image.jpeg', False),
('Integration_image.png', False),
('Integration_pic.png', False),
])
def test_is_integration_image(self, file_path, result, dummy_pack):
"""
Given:
- File path of an author image.
- File path of an integration image.
- File path of an integration image with the wrong extension.
- File path not starting with Packs/TestPack
- File path not containing the 'image' constant
When:
- Checking whether the image in integration image or not
Then:
- Validate that the answer is False
- Validate that the answer is True
- Validate that the answer is False
- Validate that the answer is False
- Validate that the answer is False
"""
assert dummy_pack.is_integration_image(file_path) is result

@pytest.mark.parametrize('file_path, result', [
('Packs/TestPack/Author_image.png', True),
('Packs/TestPack/Author_image.jpeg', False),
('Packs/TestPack/Integration_image.png', False),
('Author_image.png', False)
])
def test_is_author_image(self, file_path, result, dummy_pack):
"""
Given:
- File path of an author image.
- File path of an author image with bad suffix.
- File path of an integration image.
- File path not starting with Packs/TestPack
When:
- Checking whether the image in integration image or not
Then:
- Validate that the answer is True
- Validate that the answer is False
- Validate that the answer is False
- Validate that the answer is False
"""
assert dummy_pack.is_author_image(file_path) is result
6 changes: 3 additions & 3 deletions Tests/Marketplace/copy_and_upload_packs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -315,7 +315,7 @@ def main():

# Get the successful and failed packs file from Prepare Content step in Create Instances job if there are
packs_results_file_path = os.path.join(os.path.dirname(packs_artifacts_path), BucketUploadFlow.PACKS_RESULTS_FILE)
pc_successful_packs_dict, pc_failed_packs_dict, pc_uploaded_images = get_successful_and_failed_packs(
pc_successful_packs_dict, pc_failed_packs_dict = get_successful_and_failed_packs(
packs_results_file_path, BucketUploadFlow.PREPARE_CONTENT_FOR_TESTING
)
logging.debug(f"Successful packs from Prepare Content: {pc_successful_packs_dict}")
Expand Down Expand Up @@ -345,13 +345,13 @@ def main():
pack.cleanup()
continue

task_status = pack.copy_integration_images(production_bucket, build_bucket, pc_uploaded_images)
task_status = pack.copy_integration_images(production_bucket, build_bucket)
if not task_status:
pack.status = PackStatus.FAILED_IMAGES_UPLOAD.name
pack.cleanup()
continue

task_status = pack.copy_author_image(production_bucket, build_bucket, pc_uploaded_images)
task_status = pack.copy_author_image(production_bucket, build_bucket)
if not task_status:
pack.status = PackStatus.FAILED_AUTHOR_IMAGE_UPLOAD.name
pack.cleanup()
Expand Down
Loading