Merge pull request #2999 from gerritholl/katrins-behave-tests

Add Accsos image comparison tests

satpy/tests/behave/features/image_comparison.feature

@@ -0,0 +1,13 @@
+Feature: Image Comparison
+
+  Scenario Outline: Compare generated image with reference image
+    Given I have a <composite> reference image file from <satellite>
+    When I generate a new <composite> image file from <satellite>
+    Then the generated image should be the same as the reference image
+
+    Examples:
+      |satellite |composite  |
+      |GOES17   |airmass  |
+      |GOES16   |airmass  |
+      |GOES16   |ash      |
+      |GOES17   |ash      |

satpy/tests/behave/features/steps/image_comparison.py

@@ -0,0 +1,122 @@
+# Copyright (c) 2024 Satpy developers
+#
+# This file is part of satpy.
+#
+# satpy is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# satpy is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# satpy.  If not, see <http://www.gnu.org/licenses/>.
+"""Image comparison tests."""
+
+import os
+import warnings
+from datetime import datetime
+from glob import glob
+
+import cv2
+import dask
+import numpy as np
+from behave import given, then, when
+
+from satpy import Scene
+
+ext_data_path = "/app/ext_data"
+threshold = 2000
+
+def before_all(context):
+    """Define a before_all hook to create the timestamp and test results directory."""
+    tm = datetime.now()
+    context.timestamp = tm.strftime("%Y-%m-%d-%H-%M-%S")
+    context.test_results_dir = f"{ext_data_path}/test_results/image_comparison/{context.timestamp}"
+    os.makedirs(os.path.join(context.test_results_dir, "generated"), exist_ok=True)
+    os.makedirs(os.path.join(context.test_results_dir, "difference"), exist_ok=True)
+
+    # Write the timestamp to test_results.txt
+    results_file = os.path.join(context.test_results_dir, "test_results.txt")
+    with open(results_file, "a") as f:
+        f.write(f"Test executed at {context.timestamp}.\n\n")
+
+def setup_hooks():
+    """Register the before_all hook."""
+    from behave import use_fixture
+    from behave.runner import Context
+
+    use_fixture(before_all, Context)
+
+setup_hooks()
+@given("I have a {composite} reference image file from {satellite}")
+def step_given_reference_image(context, composite, satellite):
+    """Prepare a reference image."""
+    reference_image = f"reference_image_{satellite}_{composite}.png"
+    context.reference_image = cv2.imread(f"{ext_data_path}/reference_images/{reference_image}")
+    context.satellite = satellite
+    context.composite = composite
+
+
+@when("I generate a new {composite} image file from {satellite}")
+def step_when_generate_image(context, composite, satellite):
+    """Generate test images."""
+    os.environ["OMP_NUM_THREADS"] = os.environ["MKL_NUM_THREADS"] = "2"
+    os.environ["PYTROLL_CHUNK_SIZE"] = "1024"
+    warnings.simplefilter("ignore")
+    dask.config.set(scheduler="threads", num_workers=4)
+
+    # Get the list of satellite files to open
+    filenames = glob(f"{ext_data_path}/satellite_data/{satellite}/*.nc")
+
+    scn = Scene(reader="abi_l1b", filenames=filenames)
+
+    scn.load([composite])
+
+    # Save the generated image in the generated folder
+    generated_image_path = os.path.join(context.test_results_dir, "generated",
+                                        f"generated_{context.satellite}_{context.composite}.png")
+    scn.save_datasets(writer="simple_image", filename=generated_image_path)
+
+    # Save the generated image in the context
+    context.generated_image = cv2.imread(generated_image_path)
+
+
+@then("the generated image should be the same as the reference image")
+def step_then_compare_images(context):
+    """Compare test image to reference image."""
+    # Load the images
+    imageA = cv2.cvtColor(context.reference_image, cv2.COLOR_BGR2GRAY)
+    imageB = cv2.cvtColor(context.generated_image, cv2.COLOR_BGR2GRAY)
+    # Ensure both images have the same dimensions
+    if imageA.shape != imageB.shape:
+        raise ValueError("Both images must have the same dimensions")
+    array1 = np.array(imageA)
+    array2 = np.array(imageB)
+    # Perform pixel-wise comparison
+    result_matrix = (array1 != array2).astype(np.uint8) * 255
+
+    # Save the resulting numpy array as an image in the difference folder
+    diff_image_path = os.path.join(context.test_results_dir, "difference",
+                                   f"diff_{context.satellite}_{context.composite}.png")
+    cv2.imwrite(diff_image_path, result_matrix)
+
+    # Count non-zero pixels in the result matrix
+    non_zero_count = np.count_nonzero(result_matrix)
+
+    # Write the results to a file in the test results directory
+    results_file = os.path.join(context.test_results_dir, "test_results.txt")
+    with open(results_file, "a") as f:
+        f.write(f"Test for {context.satellite} - {context.composite}\n")
+        f.write(f"Non-zero pixel differences: {non_zero_count}\n")
+        if non_zero_count < threshold:
+            f.write(f"Result: Passed - {non_zero_count} pixel differences.\n\n")
+        else:
+            f.write(f"Result: Failed - {non_zero_count} pixel differences exceed the threshold of {threshold}.\n\n")
+
+    # Assert that the number of differences is below the threshold
+    assert non_zero_count < threshold, (f"Images are not similar enough. "
+            f"{non_zero_count} pixel differences exceed the threshold of "
+            f"{threshold}.")

utils/create_reference.py

@@ -0,0 +1,51 @@
+# Copyright (c) 2024 Satpy developers
+#
+# This file is part of satpy.
+#
+# satpy is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# satpy is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# satpy.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Script to create image testing references.
+
+Script to create reference images for the automated image testing system.
+
+create_reference.py <input-data> <output-directory> <satellite-name>
+
+The input data directory must follow the data structure from the
+image-comparison-tests repository with satellite_data/<satellite-name>.
+
+This script is a work in progress and expected to change significantly.
+It is absolutely not intended for any operational production of satellite
+imagery.
+"""
+
+import sys
+from glob import glob
+
+from dask.diagnostics import ProgressBar
+
+from satpy import Scene
+
+ext_data_path = sys.argv[1]
+outdir = sys.argv[2]
+satellite = sys.argv[3]
+
+filenames = glob(f"{ext_data_path}/satellite_data/{satellite}/*.nc")
+
+scn = Scene(reader="abi_l1b", filenames=filenames)
+
+composites = ["ash", "airmass"]
+scn.load(composites)
+ls = scn.resample(resampler="native")
+with ProgressBar():
+    ls.save_datasets(writer="simple_image", filename=outdir +
+                      "/satpy-reference-image-{platform_name}-{sensor}-{start_time:%Y%m%d%H%M}-{area.area_id}-{name}.png")