Merge pull request #2520 from cta-observatory/refactor/rename_datawri…

…ter_options2

Refactor/rename datawriter options (redux)

docs/changes/2520.api.rst

@@ -0,0 +1,18 @@
+Unified the options for DataWriter and the data level names:
+
++-------------------------+--------------------------+
+| Old                     | New                      |
++=========================+==========================+
+| ``write_raw_waveforms`` | ``write_r0_waveforms``   |
++-------------------------+--------------------------+
+| ``write_waveforms``     | ``write_r1_waveforms``   |
++-------------------------+--------------------------+
+| ``write_images``        | ``write_dl1_images``     |
++-------------------------+--------------------------+
+| ``write_parameters``    | ``write_dl1_parameters`` |
++-------------------------+--------------------------+
+| ``write_showers``       | ``write_dl2``            |
++-------------------------+--------------------------+
+
+This changes requires that existing configuration files are updated
+if they use these parameters

examples/tutorials/ctapipe_overview.py

@@ -369,9 +369,7 @@
 
 f = tempfile.NamedTemporaryFile(suffix=".hdf5")
 
-with DataWriter(
-    source, output_path=f.name, overwrite=True, write_showers=True
-) as writer:
+with DataWriter(source, output_path=f.name, overwrite=True, write_dl2=True) as writer:
     for event in source:
         energy = event.simulation.shower.energy
         n_telescopes_r1 = len(event.r1.tel)

examples/tutorials/tableloader_and_eventsources.py

@@ -138,9 +138,9 @@
 with DataWriter(
     event_source=source,
     output_path="events.dl1.h5",
-    write_parameters=False,
+    write_dl1_parameters=False,
     overwrite=True,
-    write_images=True,
+    write_dl1_images=True,
 ) as write_data:
     for event in source:
         calibrate(event)

src/ctapipe/conftest.py

@@ -415,7 +415,7 @@ def dl1_image_file(dl1_tmp_path, prod5_gamma_simtel_path):
             f"--input={prod5_gamma_simtel_path}",
             f"--output={output}",
             "--write-images",
-            "--DataWriter.write_parameters=False",
+            "--DataWriter.write_dl1_parameters=False",
             "--max-events=20",
             "--DataWriter.Contact.name=αℓℓ the äüöß",
         ]

src/ctapipe/io/datawriter.py

@@ -183,23 +183,23 @@ class DataWriter(Component):
         help="output filename", default_value=pathlib.Path("events.dl1.h5")
     ).tag(config=True)
 
-    write_raw_waveforms = Bool(
+    write_r0_waveforms = Bool(
         help="Store R0 waveforms if available", default_value=False
     ).tag(config=True)
 
-    write_waveforms = Bool(
+    write_r1_waveforms = Bool(
         help="Store R1 waveforms if available", default_value=False
     ).tag(config=True)
 
-    write_images = Bool(help="Store DL1 Images if available", default_value=False).tag(
-        config=True
-    )
+    write_dl1_images = Bool(
+        help="Store DL1 Images if available", default_value=False
+    ).tag(config=True)
 
-    write_parameters = Bool(
+    write_dl1_parameters = Bool(
         help="Store DL1 image parameters if available", default_value=True
     ).tag(config=True)
 
-    write_showers = Bool(
+    write_dl2 = Bool(
         help="Store DL2 stereo shower parameters if available", default_value=False
     ).tag(config=True)
 
@@ -322,17 +322,17 @@ def __call__(self, event: ArrayEventContainer):
                     [tel_index, sim.impact],
                 )
 
-        if self.write_waveforms:
+        if self.write_r1_waveforms:
             self._write_r1_telescope_events(event)
 
-        if self.write_raw_waveforms:
+        if self.write_r0_waveforms:
             self._write_r0_telescope_events(event)
 
         # write telescope event data
         self._write_dl1_telescope_events(event)
 
         # write DL2 info if requested
-        if self.write_showers:
+        if self.write_dl2:
             self._write_dl2_telescope_events(event)
             self._write_dl2_stereo_event(event)
 
@@ -392,17 +392,17 @@ def finish(self):
     def datalevels(self):
         """returns a list of data levels requested"""
         data_levels = []
-        if self.write_images:
+        if self.write_dl1_images:
             data_levels.append(DataLevel.DL1_IMAGES)
-        if self.write_parameters:
+        if self.write_dl1_parameters:
             data_levels.append(DataLevel.DL1_PARAMETERS)
         if self.write_muon_parameters:
             data_levels.append(DataLevel.DL1_MUON)
-        if self.write_showers:
+        if self.write_dl2:
             data_levels.append(DataLevel.DL2)
-        if self.write_raw_waveforms:
+        if self.write_r0_waveforms:
             data_levels.append(DataLevel.R0)
-        if self.write_waveforms:
+        if self.write_r1_waveforms:
             data_levels.append(DataLevel.R1)
         return data_levels
 
@@ -435,10 +435,10 @@ def _setup_output_path(self):
 
         # check that options make sense
         writable_things = [
-            self.write_parameters,
-            self.write_images,
-            self.write_showers,
-            self.write_waveforms,
+            self.write_dl1_parameters,
+            self.write_dl1_images,
+            self.write_dl2,
+            self.write_r1_waveforms,
             self.write_muon_parameters,
         ]
         if not any(writable_things):
@@ -472,10 +472,10 @@ def _setup_writer(self):
         writer.exclude("/dl1/event/telescope/images/.*", "parameters")
         writer.exclude("/simulation/event/telescope/images/.*", "true_parameters")
 
-        if not self.write_images:
+        if not self.write_dl1_images:
             writer.exclude("/simulation/event/telescope/images/.*", "true_image")
 
-        if not self.write_parameters:
+        if not self.write_dl1_parameters:
             writer.exclude("/dl1/event/telescope/images/.*", "image_mask")
 
         # Set up transforms
@@ -671,16 +671,16 @@ def _write_dl1_telescope_events(self, event: ArrayEventContainer):
 
             table_name = self.table_name(tel_id)
 
-            if self.write_parameters:
+            if self.write_dl1_parameters:
                 self._writer.write(
                     table_name=f"dl1/event/telescope/parameters/{table_name}",
                     containers=[tel_index, *dl1_camera.parameters.values()],
                 )
 
-            if self.write_images:
+            if self.write_dl1_images:
                 if dl1_camera.image is None:
                     raise ValueError(
-                        "DataWriter.write_images is True but event does not contain image"
+                        "DataWriter.write_dl1_images is True but event does not contain image"
                     )
 
                 self._writer.write(
@@ -699,7 +699,7 @@ def _write_dl1_telescope_events(self, event: ArrayEventContainer):
                     tel_id in event.simulation.tel
                     and event.simulation.tel[tel_id].true_image is not None
                 )
-                if self.write_parameters and has_sim_image:
+                if self.write_dl1_parameters and has_sim_image:
                     true_parameters = event.simulation.tel[tel_id].true_parameters
                     # only write the available containers, no peak time related
                     # features for true image available.
@@ -783,15 +783,15 @@ def _generate_table_indices(self, h5file, start_node):
     def _generate_indices(self):
         """generate PyTables index tables for common columns"""
         self.log.debug("Writing index tables")
-        if self.write_images:
+        if self.write_dl1_images:
             self._generate_table_indices(
                 self._writer.h5file, "/dl1/event/telescope/images"
             )
             if self._is_simulation:
                 self._generate_table_indices(
                     self._writer.h5file, "/simulation/event/telescope/images"
                 )
-        if self.write_parameters:
+        if self.write_dl1_parameters:
             self._generate_table_indices(
                 self._writer.h5file, "/dl1/event/telescope/parameters"
             )

src/ctapipe/io/tests/conftest.py

@@ -20,11 +20,11 @@ def r1_hdf5_file(prod5_proton_simtel_path, r1_path):
     writer = DataWriter(
         event_source=source,
         output_path=path,
-        write_parameters=False,
-        write_images=False,
-        write_showers=False,
-        write_raw_waveforms=False,
-        write_waveforms=True,
+        write_dl1_parameters=False,
+        write_dl1_images=False,
+        write_dl2=False,
+        write_r0_waveforms=False,
+        write_r1_waveforms=True,
     )
 
     for e in source:

src/ctapipe/io/tests/test_datawriter.py

@@ -80,11 +80,11 @@ def test_write(tmpdir: Path):
     with DataWriter(
         event_source=source,
         output_path=output_path,
-        write_parameters=False,
-        write_images=True,
-        write_showers=True,
-        write_raw_waveforms=True,
-        write_waveforms=True,
+        write_dl1_parameters=False,
+        write_dl1_images=True,
+        write_dl2=True,
+        write_r0_waveforms=True,
+        write_r1_waveforms=True,
     ) as writer:
         writer.log.level = logging.DEBUG
         for event in source:
@@ -158,15 +158,15 @@ def test_roundtrip(tmpdir: Path):
     with DataWriter(
         event_source=source,
         output_path=output_path,
-        write_parameters=False,
-        write_images=True,
+        write_dl1_parameters=False,
+        write_dl1_images=True,
         transform_image=True,
         image_dtype="int32",
         image_scale=10,
         transform_peak_time=True,
         peak_time_dtype="int16",
         peak_time_scale=100,
-        write_showers=True,
+        write_dl2=True,
     ) as write:
         write.log.level = logging.DEBUG
         for event in source:
@@ -229,8 +229,8 @@ def test_dl1writer_no_events(tmpdir: Path):
     with DataWriter(
         event_source=source,
         output_path=output_path,
-        write_parameters=True,
-        write_images=True,
+        write_dl1_parameters=True,
+        write_dl1_images=True,
     ) as writer:
         writer.log.level = logging.DEBUG
         writer.write_simulation_histograms(source)
@@ -269,8 +269,8 @@ def test_metadata(tmpdir: Path):
         with DataWriter(
             event_source=source,
             output_path=output_path,
-            write_parameters=True,
-            write_images=True,
+            write_dl1_parameters=True,
+            write_dl1_images=True,
             config=config,
         ):
             pass

src/ctapipe/resources/base_config.yaml

@@ -21,11 +21,11 @@ DataWriter:
   # write nothing (can be overridden on the command-line or in subsequent config
   # files)
   overwrite: false # do not overwrite existing files
-  write_images: false # store DL1 images
-  write_parameters: false # store DL1 parameters
-  write_showers: false # store DL2 stereo geometry
-  write_raw_waveforms: false # write R0 waveforms
-  write_waveforms: false # write R1 waveforms
+  write_dl1_images: false # store DL1 images
+  write_dl1_parameters: false # store DL1 parameters
+  write_dl2: false # store DL2 stereo geometry
+  write_r0_waveforms: false # write R0 waveforms
+  write_r1_waveforms: false # write R1 waveforms
   write_muon_parameters: false # analyze and write muon parameters
 
 # The CameraCalibrator takes data from R1 or DL0 to DL1a level, applying finer
@@ -95,7 +95,7 @@ ImageProcessor:
 
 
 # The ShowerProcessor performs the DL1 to DL2a (reconstructed shower geometry)
-# transition. It is run only if the parameter DataWriter.write_showers=True.
+# transition. It is run only if the parameter DataWriter.write_dl2=True.
 ShowerProcessor:
   # choose between HillasReconstructor and HillasIntersection or both (two
   # implementations of the standard stereo line-intersection method)

src/ctapipe/resources/image_modification_config.json

@@ -6,9 +6,9 @@
             "organization": "CTA Consortium"
         },
         "overwrite": false,
-        "write_images": true,
-        "write_parameters": true,
-        "write_showers": false,
+        "write_dl1_images": true,
+        "write_dl1_parameters": true,
+        "write_dl2": false,
         "transform_image": true,
         "transform_peak_time": true
     },

src/ctapipe/resources/ml_preprocessing_config.json

@@ -6,9 +6,9 @@
             "organization": "YOUR-ORGANIZATION"
         },
         "overwrite": false,
-        "write_images": false,
-        "write_parameters": true,
-        "write_showers": true,
+        "write_dl1_images": false,
+        "write_dl1_parameters": true,
+        "write_dl2": true,
         "transform_image": true,
         "transform_peak_time": true
     },

src/ctapipe/resources/ml_preprocessing_config.yaml

@@ -5,13 +5,13 @@
 # Perform Stage 1 and Stage 2a processing to generate features needed for
 # training machine learning algorithms.
 #
-# Note that here write_images is disabled by default (to avoid very large
+# Note that here write_dl1_images is disabled by default (to avoid very large
 # files), however for training deep learning algorithms, it should be turned on.
 # ======================================================================
 #
 #  Make sure you first include `--config base_config.yaml` before including this file
 
 DataWriter:
-  write_images: false
-  write_parameters: true
-  write_showers: true
+  write_dl1_images: false
+  write_dl1_parameters: true
+  write_dl2: true

src/ctapipe/resources/stage1_config.json

@@ -6,9 +6,9 @@
             "organization": "YOUR-ORGANIZATION"
         },
         "overwrite": false,
-        "write_images": true,
-        "write_parameters": true,
-        "write_showers": false,
+        "write_dl1_images": true,
+        "write_dl1_parameters": true,
+        "write_dl2": false,
         "transform_image": true,
         "transform_peak_time": true
     },

src/ctapipe/resources/stage1_config.toml

@@ -8,9 +8,9 @@ organization = "YOUR-ORGANIZATION"
 [DataWriter]
 # options that control what is stored in the output file
 overwrite = false
-write_images = true
-write_parameters = true
-write_showers = false
+write_dl1_images = true
+write_dl1_parameters = true
+write_dl2 = false
 transform_image = true
 transform_peak_time = true
 

src/ctapipe/resources/stage1_config.yaml

@@ -10,6 +10,6 @@
 #  Make sure you first include `--config base_config.yaml` before including this file
 
 DataWriter:
-  write_images: true
-  write_parameters: true
-  write_showers: false
+  write_dl1_images: true
+  write_dl1_parameters: true
+  write_dl2: false

src/ctapipe/resources/stage2_config.json

@@ -6,9 +6,9 @@
             "organization": "YOUR-ORGANIZATION"
         },
         "overwrite": false,
-        "write_images": false,
-        "write_parameters": false,
-        "write_showers": true,
+        "write_dl1_images": false,
+        "write_dl1_parameters": false,
+        "write_dl2": true,
         "transform_image": true,
         "transform_peak_time": true
     },