[SCSB-174] move DMS requirement <-> test correlations from @metrics_logger decorators to romancal/tests/dms_requirement_tests.json

changes/1399.general.rst

@@ -0,0 +1 @@
+move DMS requirement <-> test correlations from ``@metrics_logger()`` decorators to ``romancal/tests/dms_requirement_tests.json``

pyproject.toml

@@ -60,7 +60,6 @@ test = [
  "ci-watson >=0.5.0",
  "pytest >=4.6.0, <8.0.0",
  "pytest-astropy",
- "metrics_logger >= 0.1.0",
  "deepdiff",
 ]
 dev = [

requirements-dev-st.txt

@@ -9,7 +9,6 @@ git+https://github.com/spacetelescope/stpipe.git
 # Other important upstream packages
 git+https://github.com/spacetelescope/crds.git
 git+https://github.com/spacetelescope/gwcs.git
-git+https://github.com/spacetelescope/metrics_logger.git
 git+https://github.com/spacetelescope/tweakwcs.git
 git+https://github.com/spacetelescope/webbpsf.git
 # although not directly used install the dev version of poppy for webbpsf

romancal/regtest/test_catalog.py

@@ -2,7 +2,6 @@
 
 import asdf
 import pytest
-from metrics_logger.decorators import metrics_logger
 
 from romancal.source_catalog.source_catalog_step import SourceCatalogStep
 from romancal.stpipe import RomanStep
@@ -48,7 +47,6 @@ def check_catalog_fields(model, log, modeltype):
 
 @pytest.mark.bigdata
 @pytest.mark.soctests
-@metrics_logger("DMS374", "DMS375", "DMS376", "DMS386", "DMS387")
 def test_catalog_l3(rtdata, ignore_asdf_paths):
  # DMS374: positions on ICRF
  # DMS375: fluxes
@@ -76,7 +74,6 @@ def test_catalog_l3(rtdata, ignore_asdf_paths):
 
 @pytest.mark.bigdata
 @pytest.mark.soctests
-@metrics_logger("DMS374", "DMS375", "DMS376", "DMS386", "DMS387")
 def test_catalog_l2(rtdata, ignore_asdf_paths):
  # DMS374: positions on ICRF
  # DMS375: fluxes

romancal/regtest/test_mos_pipeline.py

@@ -4,7 +4,6 @@
 
 import pytest
 import roman_datamodels as rdm
-from metrics_logger.decorators import metrics_logger
 
 from romancal.pipeline.mosaic_pipeline import MosaicPipeline
 
@@ -20,7 +19,6 @@ def passfail(bool_expr):
 
 @pytest.mark.bigdata
 @pytest.mark.soctests
-@metrics_logger("DMS356", "DMS374", "DMS400")
 def test_level3_mos_pipeline(rtdata, ignore_asdf_paths):
  """Tests for level 3 processing requirements DMS356"""
  rtdata.get_asn("WFI/image/L3_regtest_asn.json")
@@ -131,7 +129,6 @@ def test_level3_mos_pipeline(rtdata, ignore_asdf_paths):
 
 @pytest.mark.bigdata
 @pytest.mark.soctests
-@metrics_logger("DMS373")
 def test_hlp_mosaic_pipeline(rtdata, ignore_asdf_paths):
  """Tests for level 3 mosaic requirements DMS373"""
  rtdata.get_asn("WFI/image/L3_mosaic_asn.json")

romancal/regtest/test_ramp_fitting.py

@@ -10,7 +10,6 @@
 
 import pytest
 import roman_datamodels as rdm
-from metrics_logger.decorators import metrics_logger
 
 from romancal.lib.dms import log_result
 from romancal.lib.suffix import replace_suffix
@@ -188,7 +187,6 @@ def test_rampfit_step(rampfit_result, rtdata_module, ignore_asdf_paths):
  except AssertionError as e:
  error_msgs.append(str(e))
 
- @metrics_logger(requirement)
  def test_success():
  assert not len(error_msgs), "\n".join(error_msgs)
 

romancal/regtest/test_resample.py

@@ -1,6 +1,5 @@
 import numpy as np
 import pytest
-from metrics_logger.decorators import metrics_logger
 from roman_datamodels import datamodels as rdm
 
 from romancal.resample.resample_step import ResampleStep
@@ -9,7 +8,6 @@
 from .regtestdata import compare_asdf
 
 
-@metrics_logger("DMS342", "DMS343", "DMS344", "DMS345")
 @pytest.mark.bigdata
 def test_resample_single_file(rtdata, ignore_asdf_paths):
  output_data = "mosaic_resamplestep.asdf"

romancal/regtest/test_tweakreg.py

@@ -2,7 +2,6 @@
 import numpy as np
 import pytest
 from astropy import units as u
-from metrics_logger.decorators import metrics_logger
 from roman_datamodels import datamodels as rdm
 
 from romancal.stpipe import RomanStep
@@ -11,7 +10,6 @@
 from .regtestdata import compare_asdf
 
 
-@metrics_logger("DMS280", "DMS405", "DMS488")
 @pytest.mark.bigdata
 def test_tweakreg(rtdata, ignore_asdf_paths, tmp_path):
  # N.B.: uncal file is from simulator

romancal/regtest/test_wfi_16resultants.py

@@ -2,7 +2,6 @@
 
 import pytest
 import roman_datamodels as rdm
-from metrics_logger.decorators import metrics_logger
 
 from romancal.pipeline.exposure_pipeline import ExposurePipeline
 
@@ -16,7 +15,6 @@ def passfail(bool_expr):
 
 @pytest.mark.bigdata
 @pytest.mark.soctests
-@metrics_logger("DMS413")
 def test_16resultants_image_processing(rtdata, ignore_asdf_paths):
  """Tests for imaging processing requirements for 16 resultants (DMS413)"""
  # The input data is from INS for stress testing at some point this should be generated
@@ -72,7 +70,6 @@ def test_16resultants_image_processing(rtdata, ignore_asdf_paths):
 
 @pytest.mark.bigdata
 @pytest.mark.soctests
-@metrics_logger("DMS414")
 def test_16resultants_spectral_processing(rtdata, ignore_asdf_paths):
  """Tests for imaging processing requirements for 16 resultants (DMS413)"""
  # The input data is from INS for stress testing at some point this should be generated

romancal/regtest/test_wfi_dq_init.py

@@ -4,15 +4,13 @@
 
 import pytest
 import roman_datamodels as rdm
-from metrics_logger.decorators import metrics_logger
 
 from romancal.step import DQInitStep
 from romancal.stpipe import RomanStep
 
 from .regtestdata import compare_asdf
 
 
-@metrics_logger("DMS25")
 @pytest.mark.bigdata
 def test_dq_init_image_step(rtdata, ignore_asdf_paths):
  """DMS25 Test: Testing retrieval of best ref file for image data,
@@ -68,7 +66,6 @@ def test_dq_init_image_step(rtdata, ignore_asdf_paths):
  assert diff.identical, diff.report()
 
 
-@metrics_logger("DMS25")
 @pytest.mark.bigdata
 def test_dq_init_grism_step(rtdata, ignore_asdf_paths):
  """DMS25 Test: Testing retrieval of best ref file for grism data,

romancal/regtest/test_wfi_flat_field.py

@@ -3,7 +3,6 @@
 import pytest
 import roman_datamodels as rdm
 from crds.core.exceptions import CrdsLookupError
-from metrics_logger.decorators import metrics_logger
 
 from romancal.step import FlatFieldStep
 from romancal.stpipe import RomanStep
@@ -69,7 +68,6 @@ def test_flat_field_grism_step(rtdata, ignore_asdf_paths):
  assert diff.identical, diff.report()
 
 
-@metrics_logger("DMS79")
 @pytest.mark.bigdata
 @pytest.mark.soctests
 def test_flat_field_crds_match_image_step(rtdata, ignore_asdf_paths):

romancal/regtest/test_wfi_photom.py

@@ -5,15 +5,13 @@
 import pytest
 import roman_datamodels as rdm
 from astropy import units as u
-from metrics_logger.decorators import metrics_logger
 
 from romancal.step import PhotomStep
 from romancal.stpipe import RomanStep
 
 from .regtestdata import compare_asdf
 
 
-@metrics_logger("DMS140")
 @pytest.mark.bigdata
 def test_absolute_photometric_calibration(rtdata, ignore_asdf_paths):
  """DMS140 Test: Testing application of photometric correction using

romancal/regtest/test_wfi_pipeline.py

@@ -7,7 +7,6 @@
 import pytest
 import roman_datamodels as rdm
 from gwcs.wcstools import grid_from_bounding_box
-from metrics_logger.decorators import metrics_logger
 from numpy.testing import assert_allclose
 from roman_datamodels.dqflags import pixel
 
@@ -26,7 +25,6 @@ def passfail(bool_expr):
 
 @pytest.mark.bigdata
 @pytest.mark.soctests
-@metrics_logger("DMS86", "DMS87", "DMS88", "DMS361", "DMS280")
 def test_level2_image_processing_pipeline(rtdata, ignore_asdf_paths):
  """Tests for flat field imaging processing requirements DMS86 & DMS 87"""
  input_data = "r0000101001001001001_01101_0001_WFI01_uncal.asdf"
@@ -273,7 +271,6 @@ def test_level2_image_processing_pipeline(rtdata, ignore_asdf_paths):
 
 @pytest.mark.bigdata
 @pytest.mark.soctests
-@metrics_logger("DMS278", "DMS90", "DMS91", "DMS9", "DMS365")
 def test_level2_grism_processing_pipeline(rtdata, ignore_asdf_paths):
  """Tests for flat field grism processing requirements DMS90, DMS91 and DMS 278"""
  input_data = "r0000201001001001001_01101_0001_WFI01_uncal.asdf"

romancal/tests/dms_requirement_tests.json

@@ -0,0 +1,109 @@
+{
+ "DMS9": [
+ "romancal.regtest.test_wfi_pipeline.test_level2_grism_processing_pipeline"
+ ],
+ "DMS25": [
+ "romancal.regtest.test_wfi_dq_init.test_dq_init_grism_step",
+ "romancal.regtest.test_wfi_dq_init.test_dq_init_image_step"
+ ],
+ "DMS79": [
+ "romancal.regtest.test_wfi_flat_field.test_flat_field_crds_match_image_step"
+ ],
+ "DMS86": [
+ "romancal.regtest.test_wfi_pipeline.test_level2_image_processing_pipeline"
+ ],
+ "DMS87": [
+ "romancal.regtest.test_wfi_pipeline.test_level2_image_processing_pipeline"
+ ],
+ "DMS88": [
+ "romancal.regtest.test_wfi_pipeline.test_level2_image_processing_pipeline"
+ ],
+ "DMS90": [
+ "romancal.regtest.test_wfi_pipeline.test_level2_grism_processing_pipeline"
+ ],
+ "DMS91": [
+ "romancal.regtest.test_wfi_pipeline.test_level2_grism_processing_pipeline"
+ ],
+ "DMS140": [
+ "romancal.regtest.test_wfi_photom.test_absolute_photometric_calibration"
+ ],
+ "DMS278": [
+ "romancal.regtest.test_wfi_pipeline.test_level2_grism_processing_pipeline"
+ ],
+ "DMS280": [
+ "romancal.regtest.test_tweakreg.test_tweakreg",
+ "romancal.regtest.test_wfi_pipeline.test_level2_image_processing_pipeline"
+ ],
+ "DMS342": [
+ "romancal.regtest.test_resample.test_resample_single_file"
+ ],
+ "DMS343": [
+ "romancal.regtest.test_resample.test_resample_single_file"
+ ],
+ "DMS344": [
+ "romancal.regtest.test_resample.test_resample_single_file"
+ ],
+ "DMS345": [
+ "romancal.regtest.test_resample.test_resample_single_file"
+ ],
+ "DMS356": [
+ "romancal.regtest.test_mos_pipeline.test_level3_mos_pipeline"
+ ],
+ "DMS361": [
+ "romancal.regtest.test_wfi_pipeline.test_level2_image_processing_pipeline"
+ ],
+ "DMS362": [
+ "romancal.regtest.test_ramp_fitting.test_rampfit_step"
+ ],
+ "DMS363": [
+ "romancal.regtest.test_ramp_fitting.test_rampfit_step"
+ ],
+ "DMS365": [
+ "romancal.regtest.test_wfi_pipeline.test_level2_grism_processing_pipeline"
+ ],
+ "DMS366": [
+ "romancal.regtest.test_ramp_fitting.test_rampfit_step"
+ ],
+ "DMS367": [
+ "romancal.regtest.test_ramp_fitting.test_rampfit_step"
+ ],
+ "DMS373": [
+ "romancal.regtest.test_mos_pipeline.test_hlp_mosaic_pipeline"
+ ],
+ "DMS374": [
+ "romancal.regtest.test_catalog.test_catalog_l2",
+ "romancal.regtest.test_catalog.test_catalog_l3",
+ "romancal.regtest.test_mos_pipeline.test_level3_mos_pipeline"
+ ],
+ "DMS375": [
+ "romancal.regtest.test_catalog.test_catalog_l2",
+ "romancal.regtest.test_catalog.test_catalog_l3"
+ ],
+ "DMS376": [
+ "romancal.regtest.test_catalog.test_catalog_l2",
+ "romancal.regtest.test_catalog.test_catalog_l3"
+ ],
+ "DMS386": [
+ "romancal.regtest.test_catalog.test_catalog_l2",
+ "romancal.regtest.test_catalog.test_catalog_l3"
+ ],
+ "DMS387": [
+ "romancal.regtest.test_catalog.test_catalog_l2",
+ "romancal.regtest.test_catalog.test_catalog_l3"
+ ],
+ "DMS400": [
+ "romancal.regtest.test_mos_pipeline.test_level3_mos_pipeline"
+ ],
+ "DMS405": [
+ "romancal.regtest.test_tweakreg.test_tweakreg"
+ ],
+ "DMS413": [
+ "romancal.regtest.test_wfi_16resultants.test_16resultants_image_processing"
+ ],
+ "DMS414": [
+ "romancal.regtest.test_wfi_16resultants.test_16resultants_spectral_processing"
+ ],
+ "DMS488": [
+ "romancal.regtest.test_tweakreg.test_tweakreg"
+ ]
+}

romancal/tests/test_dms_requirements.py

@@ -0,0 +1,36 @@
+import json
+import re
+from pathlib import Path
+
+TEST_DIRECTORY = Path(__file__).parent.parent
+TEST_REQUIREMENTS_FILENAME = TEST_DIRECTORY / "tests" / "dms_requirement_tests.json"
+
+
+def test_requirements():
+ test_requirements_filename = TEST_REQUIREMENTS_FILENAME.expanduser().absolute()
+ test_directory = TEST_DIRECTORY.expanduser().absolute()
+
+ with open(test_requirements_filename) as test_requirements_file:
+ requirements = json.load(test_requirements_file)
+
+ required_tests = {
+ test
+ for requirement_tests in requirements.values()
+ for test in requirement_tests
+ }
+
+ existing_tests = set()
+ test_regex = re.compile(r"def (test_[^\(]+)\(.*\):")
+ for test_filename in test_directory.glob("**/test_*.py"):
+ with open(test_filename) as test_file:
+ test_file_contents = test_file.read()
+
+ for match in re.finditer(test_regex, test_file_contents):
+ test = f"{test_directory.stem}.{str(test_filename.relative_to(test_directory).parent).replace('/', '.')}.{test_filename.stem}.{match.group(1)}"
+ if test in required_tests:
+ existing_tests.add(test)
+
+ missing_tests = required_tests - existing_tests
+ assert (
+ not missing_tests
+ ), f"could not find the following tests correlated with DMS requirements: {missing_tests}"