diff --git a/cato_common/images/image_comparators/ssim_image_comparator.py b/cato_common/images/image_comparators/ssim_image_comparator.py index 3bab2163..08ad9a03 100644 --- a/cato_common/images/image_comparators/ssim_image_comparator.py +++ b/cato_common/images/image_comparators/ssim_image_comparator.py @@ -73,7 +73,11 @@ def compare( reference_image = self._normalize_image(reference_image) (score, diff) = metrics.structural_similarity( - reference_image, output_image, full=True, channel_axis=-1 + reference_image, + output_image, + full=True, + channel_axis=-1, + data_range=output_image.max() - output_image.min(), ) score = float(score) if math.isnan(score): @@ -145,7 +149,9 @@ def _create_diff_image( composited_diff_image = Image.composite(output_image, heatmap, l_thresholded) target_path = os.path.join(workdir, f"diff_image_{uuid.uuid4()}.png") - composited_diff_image.save(target_path) + no_alpha = Image.new("RGB", composited_diff_image.size, (0, 0, 0)) + no_alpha.paste(composited_diff_image, (0, 0), composited_diff_image) + no_alpha.save(target_path) return target_path def _normalize_image(self, image): diff --git a/tests/integrationtests/cato_cmd/snapshots/snap_test_cato_cli.py b/tests/integrationtests/cato_cmd/snapshots/snap_test_cato_cli.py index 0bf406af..97ec3b0b 100644 --- a/tests/integrationtests/cato_cmd/snapshots/snap_test_cato_cli.py +++ b/tests/integrationtests/cato_cmd/snapshots/snap_test_cato_cli.py @@ -39,6 +39,12 @@ [INFO] Found image output at path [INFO] Found image output at path [INFO] Comparing images locally.. + + + + + + [INFO] PythonOutputVersion succeeded in 0.12 seconds [INFO] Reporting test result of test PythonTestSuite/PythonOutputVersion.. [INFO] Uploading output of test PythonTestSuite/PythonOutputVersion.. diff --git a/tests/integrationtests/cato_cmd/test_cato_cli.py b/tests/integrationtests/cato_cmd/test_cato_cli.py index faf5255a..b96cd814 100644 --- a/tests/integrationtests/cato_cmd/test_cato_cli.py +++ b/tests/integrationtests/cato_cmd/test_cato_cli.py @@ -107,6 +107,10 @@ def test_worker_run_command( "Copy .* to .*": "Copy to ", "Found image output at path .*": "Found image output at path ", "Found reference image at path .*": "Found image output at path ", + ".*RuntimeWarning.*": "", + ".*S = \(A1.*": "", + ".*heatmap.clip.*": "", + ".*cv2.cvtColor.*": "", }, env=env_with_api_token, ) diff --git a/tests/integrationtests/cato_cmd/test_run_command.py b/tests/integrationtests/cato_cmd/test_run_command.py index 45260f37..089b0e99 100644 --- a/tests/integrationtests/cato_cmd/test_run_command.py +++ b/tests/integrationtests/cato_cmd/test_run_command.py @@ -240,14 +240,14 @@ def assert_the_success_result_is_available_on_the_server( "id": 2, "name": "write_black_image", "testIdentifier": "WriteImages/write_black_image", - "thumbnailFileId": 16, + "thumbnailFileId": 14, }, { "unifiedTestStatus": "SUCCESS", "id": 1, "name": "write_white_image", "testIdentifier": "WriteImages/write_white_image", - "thumbnailFileId": 8, + "thumbnailFileId": 7, }, ] diff --git a/tests/resources/sphere_test_images/expected_diff_ssim/exr_singlechannel_16_bit.png b/tests/resources/sphere_test_images/expected_diff_ssim/exr_singlechannel_16_bit.png index 1d3a9134..37aec08a 100644 Binary files a/tests/resources/sphere_test_images/expected_diff_ssim/exr_singlechannel_16_bit.png and b/tests/resources/sphere_test_images/expected_diff_ssim/exr_singlechannel_16_bit.png differ diff --git a/tests/resources/sphere_test_images/expected_diff_ssim/exr_singlechannel_32_bit.png b/tests/resources/sphere_test_images/expected_diff_ssim/exr_singlechannel_32_bit.png index 7d7f01ca..07d62568 100644 Binary files a/tests/resources/sphere_test_images/expected_diff_ssim/exr_singlechannel_32_bit.png and b/tests/resources/sphere_test_images/expected_diff_ssim/exr_singlechannel_32_bit.png differ diff --git a/tests/resources/sphere_test_images/expected_diff_ssim/jpeg.png b/tests/resources/sphere_test_images/expected_diff_ssim/jpeg.png index 9598c0eb..4606925b 100644 Binary files a/tests/resources/sphere_test_images/expected_diff_ssim/jpeg.png and b/tests/resources/sphere_test_images/expected_diff_ssim/jpeg.png differ diff --git a/tests/resources/sphere_test_images/expected_diff_ssim/png_16_bit.png b/tests/resources/sphere_test_images/expected_diff_ssim/png_16_bit.png index c3d63163..dc947603 100644 Binary files a/tests/resources/sphere_test_images/expected_diff_ssim/png_16_bit.png and b/tests/resources/sphere_test_images/expected_diff_ssim/png_16_bit.png differ diff --git a/tests/resources/sphere_test_images/expected_diff_ssim/png_8_bit.png b/tests/resources/sphere_test_images/expected_diff_ssim/png_8_bit.png index 1196102d..0ad549e4 100644 Binary files a/tests/resources/sphere_test_images/expected_diff_ssim/png_8_bit.png and b/tests/resources/sphere_test_images/expected_diff_ssim/png_8_bit.png differ diff --git a/tests/resources/sphere_test_images/expected_diff_ssim/tiff_16_bit.png b/tests/resources/sphere_test_images/expected_diff_ssim/tiff_16_bit.png index f3c8cbcb..476abf1e 100644 Binary files a/tests/resources/sphere_test_images/expected_diff_ssim/tiff_16_bit.png and b/tests/resources/sphere_test_images/expected_diff_ssim/tiff_16_bit.png differ diff --git a/tests/resources/sphere_test_images/expected_diff_ssim/tiff_8_bit.png b/tests/resources/sphere_test_images/expected_diff_ssim/tiff_8_bit.png index e6231bea..ba35124f 100644 Binary files a/tests/resources/sphere_test_images/expected_diff_ssim/tiff_8_bit.png and b/tests/resources/sphere_test_images/expected_diff_ssim/tiff_8_bit.png differ diff --git a/tests/resources/sphere_test_images/expected_diff_ssim/transparent_red_over_opaque_blue.png b/tests/resources/sphere_test_images/expected_diff_ssim/transparent_red_over_opaque_blue.png index f1a33ab8..2c5d77da 100644 Binary files a/tests/resources/sphere_test_images/expected_diff_ssim/transparent_red_over_opaque_blue.png and b/tests/resources/sphere_test_images/expected_diff_ssim/transparent_red_over_opaque_blue.png differ diff --git a/tests/resources/with_watermark_diff_ssim.png b/tests/resources/with_watermark_diff_ssim.png index 21671be8..60017151 100644 Binary files a/tests/resources/with_watermark_diff_ssim.png and b/tests/resources/with_watermark_diff_ssim.png differ diff --git a/tests/unittests/cato_common/images/image_comparators/test_ssim_image_comparator.py b/tests/unittests/cato_common/images/image_comparators/test_ssim_image_comparator.py index a47ce1ed..065ddf56 100644 --- a/tests/unittests/cato_common/images/image_comparators/test_ssim_image_comparator.py +++ b/tests/unittests/cato_common/images/image_comparators/test_ssim_image_comparator.py @@ -58,11 +58,11 @@ def test_compare_image_should_fail_one_pixel_different( assert comparison_result == ComparisonResult( status=ResultStatus.FAILED, - message="Images are not equal! SSIM score was 0.995, max threshold is 1.000", + message="Images are not equal! SSIM score was 0.994, max threshold is 1.000", diff_image=str( tmpdir.join("diff_image_c04b964d-f443-4ae9-8b43-47fe6d2422d0.png") ), - error=0.994861900806427, + error=0.9944998621940613, ) @@ -84,9 +84,9 @@ def test_compare_image_should_fail_waith_and_without_watermark( assert comparison_result == ComparisonResult( status=ResultStatus.FAILED, - message="Images are not equal! SSIM score was 0.918, max threshold is 1.000", + message="Images are not equal! SSIM score was 0.911, max threshold is 1.000", diff_image=tmpdir.join("diff_image_c04b964d-f443-4ae9-8b43-47fe6d2422d0.png"), - error=0.9175751805305481, + error=0.9112997651100159, ) assert images_are_equal( comparison_result.diff_image, @@ -187,7 +187,7 @@ def test_compare_image_should_succeed_threshold_not_exceeded( status=ResultStatus.SUCCESS, message=None, diff_image=tmpdir.join("diff_image_c04b964d-f443-4ae9-8b43-47fe6d2422d0.png"), - error=0.9175751805305481, + error=0.9112997651100159, ) diff --git a/tests/unittests/cato_common/images/image_comparators/utils.py b/tests/unittests/cato_common/images/image_comparators/utils.py index 145d1df0..e01587aa 100644 --- a/tests/unittests/cato_common/images/image_comparators/utils.py +++ b/tests/unittests/cato_common/images/image_comparators/utils.py @@ -34,7 +34,11 @@ def images_are_visually_equal(image1, image2, threshold): ) (score, diff) = metrics.structural_similarity( - image_one, image_two, full=True, multichannel=True + image_one, + image_two, + full=True, + channel_axis=-1, + data_range=image_two.max() - image_two.min(), ) print(score) return score > threshold