[SCSB-174] move DMS requirement <-> test correlations from `@metrics_…

…logger` decorators to `romanisim/tests/dms_requirement_tests.json` (#146)

* add `test_requirements.json` to repository to track requirements

* remove usage of `metrics_logger`

* add test checking if tests exist in source code

* use pytest to obtain loaded tests

* `test_requirements.json` -> `romanisim/tests/dms_requirement_tests.json`

* reword assertion error message

* scan source directory instead of using pytest

* remove cython-generated file that slipped in

pyproject.toml

@@ -58,7 +58,6 @@ test = [
  "pytest-doctestplus >=0.10.0",
  "pytest-cov >=2.9.0",
  "flake8 >=3.6.0",
- "metrics_logger >= 0.1.0",
 ]
 
 [project.urls]

requirements-dev.txt

@@ -12,6 +12,3 @@ scipy>=0.0.dev0
 # Roman upstream packages
 git+https://github.com/spacetelescope/roman_datamodels
 git+https://github.com/spacetelescope/rad
-
-# Testing upstream packages
-git+https://github.com/spacetelescope/metrics_logger

romanisim/tests/dms_requirement_tests.json

@@ -0,0 +1,63 @@
+{
+ "DMS214": [
+ "romanisim.tests.test_image.test_image_rendering"
+ ],
+ "DMS215": [
+ "romanisim.tests.test_image.test_image_rendering"
+ ],
+ "DMS216": [
+ "romanisim.tests.test_image.test_simulate"
+ ],
+ "DMS217": [
+ "romanisim.tests.test_catalog.test_table_catalog"
+ ],
+ "DMS218": [
+ "romanisim.tests.test_image.test_add_objects",
+ "romanisim.tests.test_image.test_simulate"
+ ],
+ "DMS219": [
+ "romanisim.tests.test_l3.test_sim_mosaic"
+ ],
+ "DMS220": [
+ "romanisim.tests.test_l1.test_apportion_counts_to_resultants"
+ ],
+ "DMS221": [
+ "romanisim.tests.test_image.test_simulate"
+ ],
+ "DMS222": [
+ "romanisim.tests.test_l1.test_linearized_counts_to_resultants"
+ ],
+ "DMS223": [
+ "romanisim.tests.test_l1.test_apportion_counts_to_resultants"
+ ],
+ "DMS224": [
+ "romanisim.tests.test_image.test_simulate"
+ ],
+ "DMS225": [
+ "romanisim.tests.test_l1.test_inject_source_into_ramp"
+ ],
+ "DMS226": [
+ "romanisim.tests.test_l1.test_ipc"
+ ],
+ "DMS227": [
+ "romanisim.tests.test_l1.test_make_l1_and_asdf"
+ ],
+ "DMS228": [
+ "romanisim.tests.test_image.test_image_input"
+ ],
+ "DMS229": [
+ "romanisim.tests.test_l1.test_apportion_counts_to_resultants"
+ ],
+ "DMS230": [
+ "romanisim.tests.test_image.test_simulate_counts_generic"
+ ],
+ "DMS231": [
+ "romanisim.tests.test_image.test_inject_source_into_image"
+ ],
+ "DMS232": [
+ "romanisim.tests.test_l3.test_inject_sources_into_mosaic"
+ ],
+ "DMS233": [
+ "romanisim.tests.test_bandpass.test_convert_flux_to_counts"
+ ]
+}

romanisim/tests/test_bandpass.py

@@ -8,7 +8,6 @@
 import os
 import pytest
 import asdf
-from metrics_logger.decorators import metrics_logger
 import numpy as np
 from romanisim import bandpass
 from astropy import constants
@@ -81,7 +80,6 @@ def test_get_abflux(filter, value):
  assert np.isclose(bandpass.get_abflux(filter), value, rtol=1.0e-1)
 
 
-@metrics_logger("DMS233")
 @pytest.mark.soctests
 def test_convert_flux_to_counts():
  # Define dirac delta wavelength

romanisim/tests/test_catalog.py

@@ -4,7 +4,6 @@
 
 import os
 import numpy as np
-from metrics_logger.decorators import metrics_logger
 import galsim
 from romanisim import catalog
 from astropy.coordinates import SkyCoord
@@ -36,7 +35,6 @@ def test_make_dummy_catalog():
  assert not cat[0].profile.spectral
 
 
-@metrics_logger("DMS217")
 def test_table_catalog(tmp_path):
  """Test generation of sources with different magnitudes and sizes
  Demonstrates DMS217: generate parametric distributions

romanisim/tests/test_dms_requirements.py

@@ -0,0 +1,34 @@
+import json
+import re
+from pathlib import Path
+
+TEST_DIRECTORY = Path(__file__).parent.parent
+TEST_REQUIREMENTS_FILENAME = Path(__file__).parent / "dms_requirement_tests.json"
+
+
+def test_requirements():
+ test_requirements_filename = TEST_REQUIREMENTS_FILENAME.expanduser().absolute()
+ test_directory = TEST_DIRECTORY.expanduser().absolute()
+
+ with open(test_requirements_filename) as test_requirements_file:
+ requirements = json.load(test_requirements_file)
+
+ required_tests = {
+ test
+ for requirement_tests in requirements.values()
+ for test in requirement_tests
+ }
+
+ existing_tests = set()
+ test_regex = re.compile(r"def (test_[^\(]+)\(.*\):")
+ for test_filename in test_directory.glob("**/test_*.py"):
+ with open(test_filename) as test_file:
+ test_file_contents = test_file.read()
+
+ for match in re.finditer(test_regex, test_file_contents):
+ test = f"{test_directory.stem}.{str(test_filename.relative_to(test_directory).parent).replace('/', '.')}.{test_filename.stem}.{match.group(1)}"
+ if test in required_tests:
+ existing_tests.add(test)
+
+ missing_tests = required_tests - existing_tests
+ assert not missing_tests, f"could not find the following tests correlated with DMS requirements: {missing_tests}"

romanisim/tests/test_image.py

@@ -27,7 +27,6 @@
 import webbpsf
 from astropy.modeling.functional_models import Sersic2D
 import pytest
-from metrics_logger.decorators import metrics_logger
 from romanisim import log
 from roman_datamodels.stnode import WfiScienceRaw, WfiImage
 import romanisim.bandpass
@@ -160,7 +159,6 @@ def central_stamp(im, sz):
  center - szo2: center + szo2 + 1]
 
 
-@metrics_logger("DMS214", "DMS215")
 @pytest.mark.soctests
 def test_image_rendering():
  """Tests for image rendering routines. This is demonstrates:
@@ -276,7 +274,6 @@ def test_image_rendering():
  # Poisson errors and containing 100k counts.
 
 
-@metrics_logger("DMS218")
 def test_add_objects():
  """Test adding objects to images.
  Demonstrates profile sensitivity to distortion component of DMS218.
@@ -321,7 +318,6 @@ def test_add_objects():
  # the actual ratio was 42.
 
 
-@metrics_logger("DMS230")
 def test_simulate_counts_generic():
  """Test adding poisson noise to images.
  Demonstrates DMS230: poisson noise
@@ -446,7 +442,6 @@ def test_simulate_counts():
  assert np.all(m)
 
 
-@metrics_logger("DMS216", "DMS218", "DMS221", "DMS224")
 @pytest.mark.soctests
 def test_simulate():
  """Test convolved image generation and L2 simulation framework.
@@ -631,7 +626,6 @@ def test_reference_file_crds_match(level):
  assert (type(im) is WfiImage)
 
 
-@metrics_logger("DMS231")
 @pytest.mark.soctests
 def test_inject_source_into_image():
  """Inject a source into an image.
@@ -688,7 +682,6 @@ def test_inject_source_into_image():
  af.write_to(os.path.join(artifactdir, 'dms231.asdf'))
 
 
-@metrics_logger("DMS228")
 @pytest.mark.soctests
 def test_image_input(tmpdir):
  # make some simple example images

romanisim/tests/test_l1.py

@@ -11,7 +11,6 @@
 """
 
 import pytest
-from metrics_logger.decorators import metrics_logger
 import numpy as np
 from romanisim import l1, log, parameters, nonlinearity
 import galsim
@@ -58,7 +57,6 @@ def test_tij_to_pij():
  pij, np.diff(np.concatenate(tij), prepend=0) / tij[-1][-1])
 
 
-@metrics_logger("DMS220", "DMS229", "DMS223")
 @pytest.mark.soctests
 def test_apportion_counts_to_resultants():
  """Test that we can apportion counts to resultants and appropriately add
@@ -131,7 +129,6 @@ def test_apportion_counts_to_resultants():
  af.write_to(os.path.join(artifactdir, 'dms223.asdf'))
 
 
-@metrics_logger("DMS222")
 @pytest.mark.soctests
 def test_linearized_counts_to_resultants():
  """Test that we can apportion linearized counts to resultants.
@@ -183,7 +180,6 @@ def test_linearized_counts_to_resultants():
  af.write_to(os.path.join(artifactdir, 'dms222.asdf'))
 
 
-@metrics_logger("DMS225")
 @pytest.mark.soctests
 def test_inject_source_into_ramp():
  """Inject a source into a ramp.
@@ -223,7 +219,6 @@ def test_inject_source_into_ramp():
  af.write_to(os.path.join(artifactdir, 'dms225.asdf'))
 
 
-@metrics_logger("DMS226")
 @pytest.mark.soctests
 def test_ipc():
  """Convolve an image with an IPC kernel.
@@ -250,7 +245,6 @@ def test_read_pattern_to_tij():
  assert l1.validate_times(tij)
 
 
-@metrics_logger("DMS227")
 @pytest.mark.soctests
 def test_make_l1_and_asdf(tmp_path):
  """Make an L1 file and save it appropriately.

romanisim/tests/test_l3.py

@@ -13,14 +13,12 @@
 from astropy.stats import mad_std
 import asdf
 import pytest
-from metrics_logger.decorators import metrics_logger
 import roman_datamodels.maker_utils as maker_utils
 import romanisim.bandpass
 from galsim import roman
 from astropy.coordinates import SkyCoord
 
 
-@metrics_logger("DMS232")
 @pytest.mark.soctests
 def test_inject_sources_into_mosaic():
  """Inject sources into a mosaic.
@@ -132,7 +130,6 @@ def test_inject_sources_into_mosaic():
  af.write_to(os.path.join(artifactdir, 'dms232.asdf'))
 
 
-@metrics_logger("DMS219")
 @pytest.mark.soctests
 def test_sim_mosaic():
  """Generating mosaic from catalog file.