From 31c42cc5d6e5eb329067b5aa8f912fd045c4e033 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Mon, 4 Mar 2024 14:45:48 +0100 Subject: [PATCH 1/6] write_raw_waveforms -> write_r0_waveforms --- src/ctapipe/io/datawriter.py | 6 +++--- src/ctapipe/io/tests/conftest.py | 2 +- src/ctapipe/io/tests/test_datawriter.py | 2 +- src/ctapipe/resources/base_config.yaml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ctapipe/io/datawriter.py b/src/ctapipe/io/datawriter.py index 5408fbbba38..c47e4959517 100644 --- a/src/ctapipe/io/datawriter.py +++ b/src/ctapipe/io/datawriter.py @@ -183,7 +183,7 @@ class DataWriter(Component): help="output filename", default_value=pathlib.Path("events.dl1.h5") ).tag(config=True) - write_raw_waveforms = Bool( + write_r0_waveforms = Bool( help="Store R0 waveforms if available", default_value=False ).tag(config=True) @@ -325,7 +325,7 @@ def __call__(self, event: ArrayEventContainer): if self.write_waveforms: self._write_r1_telescope_events(event) - if self.write_raw_waveforms: + if self.write_r0_waveforms: self._write_r0_telescope_events(event) # write telescope event data @@ -400,7 +400,7 @@ def datalevels(self): data_levels.append(DataLevel.DL1_MUON) if self.write_showers: data_levels.append(DataLevel.DL2) - if self.write_raw_waveforms: + if self.write_r0_waveforms: data_levels.append(DataLevel.R0) if self.write_waveforms: data_levels.append(DataLevel.R1) diff --git a/src/ctapipe/io/tests/conftest.py b/src/ctapipe/io/tests/conftest.py index 758ee7d1cac..317d446f47f 100644 --- a/src/ctapipe/io/tests/conftest.py +++ b/src/ctapipe/io/tests/conftest.py @@ -23,7 +23,7 @@ def r1_hdf5_file(prod5_proton_simtel_path, r1_path): write_parameters=False, write_images=False, write_showers=False, - write_raw_waveforms=False, + write_r0_waveforms=False, write_waveforms=True, ) diff --git a/src/ctapipe/io/tests/test_datawriter.py b/src/ctapipe/io/tests/test_datawriter.py index a39ad4c4f67..3560bad6691 100644 --- a/src/ctapipe/io/tests/test_datawriter.py +++ b/src/ctapipe/io/tests/test_datawriter.py @@ -83,7 +83,7 @@ def test_write(tmpdir: Path): write_parameters=False, write_images=True, write_showers=True, - write_raw_waveforms=True, + write_r0_waveforms=True, write_waveforms=True, ) as writer: writer.log.level = logging.DEBUG diff --git a/src/ctapipe/resources/base_config.yaml b/src/ctapipe/resources/base_config.yaml index 13fa1a20765..3cac5a9a043 100644 --- a/src/ctapipe/resources/base_config.yaml +++ b/src/ctapipe/resources/base_config.yaml @@ -24,7 +24,7 @@ DataWriter: write_images: false # store DL1 images write_parameters: false # store DL1 parameters write_showers: false # store DL2 stereo geometry - write_raw_waveforms: false # write R0 waveforms + write_r0_waveforms: false # write R0 waveforms write_waveforms: false # write R1 waveforms write_muon_parameters: false # analyze and write muon parameters From a325af87efeee6ab205dca0e8d2c597e9bdc79af Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Mon, 4 Mar 2024 14:46:40 +0100 Subject: [PATCH 2/6] write_waveforms -> write_r1_waveforms --- src/ctapipe/io/datawriter.py | 8 ++++---- src/ctapipe/io/tests/conftest.py | 2 +- src/ctapipe/io/tests/test_datawriter.py | 2 +- src/ctapipe/resources/base_config.yaml | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/ctapipe/io/datawriter.py b/src/ctapipe/io/datawriter.py index c47e4959517..1436b52f52d 100644 --- a/src/ctapipe/io/datawriter.py +++ b/src/ctapipe/io/datawriter.py @@ -187,7 +187,7 @@ class DataWriter(Component): help="Store R0 waveforms if available", default_value=False ).tag(config=True) - write_waveforms = Bool( + write_r1_waveforms = Bool( help="Store R1 waveforms if available", default_value=False ).tag(config=True) @@ -322,7 +322,7 @@ def __call__(self, event: ArrayEventContainer): [tel_index, sim.impact], ) - if self.write_waveforms: + if self.write_r1_waveforms: self._write_r1_telescope_events(event) if self.write_r0_waveforms: @@ -402,7 +402,7 @@ def datalevels(self): data_levels.append(DataLevel.DL2) if self.write_r0_waveforms: data_levels.append(DataLevel.R0) - if self.write_waveforms: + if self.write_r1_waveforms: data_levels.append(DataLevel.R1) return data_levels @@ -438,7 +438,7 @@ def _setup_output_path(self): self.write_parameters, self.write_images, self.write_showers, - self.write_waveforms, + self.write_r1_waveforms, self.write_muon_parameters, ] if not any(writable_things): diff --git a/src/ctapipe/io/tests/conftest.py b/src/ctapipe/io/tests/conftest.py index 317d446f47f..4e1de6c97e3 100644 --- a/src/ctapipe/io/tests/conftest.py +++ b/src/ctapipe/io/tests/conftest.py @@ -24,7 +24,7 @@ def r1_hdf5_file(prod5_proton_simtel_path, r1_path): write_images=False, write_showers=False, write_r0_waveforms=False, - write_waveforms=True, + write_r1_waveforms=True, ) for e in source: diff --git a/src/ctapipe/io/tests/test_datawriter.py b/src/ctapipe/io/tests/test_datawriter.py index 3560bad6691..d79f67a01cb 100644 --- a/src/ctapipe/io/tests/test_datawriter.py +++ b/src/ctapipe/io/tests/test_datawriter.py @@ -84,7 +84,7 @@ def test_write(tmpdir: Path): write_images=True, write_showers=True, write_r0_waveforms=True, - write_waveforms=True, + write_r1_waveforms=True, ) as writer: writer.log.level = logging.DEBUG for event in source: diff --git a/src/ctapipe/resources/base_config.yaml b/src/ctapipe/resources/base_config.yaml index 3cac5a9a043..824399f8786 100644 --- a/src/ctapipe/resources/base_config.yaml +++ b/src/ctapipe/resources/base_config.yaml @@ -25,7 +25,7 @@ DataWriter: write_parameters: false # store DL1 parameters write_showers: false # store DL2 stereo geometry write_r0_waveforms: false # write R0 waveforms - write_waveforms: false # write R1 waveforms + write_r1_waveforms: false # write R1 waveforms write_muon_parameters: false # analyze and write muon parameters # The CameraCalibrator takes data from R1 or DL0 to DL1a level, applying finer From 7e8569148f9b9d274f2dc59025bbaa318c9df696 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Mon, 4 Mar 2024 14:48:39 +0100 Subject: [PATCH 3/6] write_showers -> write_dl2 --- examples/tutorials/ctapipe_overview.py | 4 +--- src/ctapipe/io/datawriter.py | 8 ++++---- src/ctapipe/io/tests/conftest.py | 2 +- src/ctapipe/io/tests/test_datawriter.py | 4 ++-- src/ctapipe/resources/base_config.yaml | 4 ++-- src/ctapipe/resources/image_modification_config.json | 2 +- src/ctapipe/resources/ml_preprocessing_config.json | 2 +- src/ctapipe/resources/ml_preprocessing_config.yaml | 2 +- src/ctapipe/resources/stage1_config.json | 2 +- src/ctapipe/resources/stage1_config.toml | 2 +- src/ctapipe/resources/stage1_config.yaml | 2 +- src/ctapipe/resources/stage2_config.json | 2 +- src/ctapipe/resources/stage2_config.yaml | 2 +- src/ctapipe/tools/process.py | 4 ++-- src/ctapipe/tools/tests/test_process.py | 5 +---- 15 files changed, 21 insertions(+), 26 deletions(-) diff --git a/examples/tutorials/ctapipe_overview.py b/examples/tutorials/ctapipe_overview.py index ced2606c844..38ed0ac3873 100644 --- a/examples/tutorials/ctapipe_overview.py +++ b/examples/tutorials/ctapipe_overview.py @@ -369,9 +369,7 @@ f = tempfile.NamedTemporaryFile(suffix=".hdf5") -with DataWriter( - source, output_path=f.name, overwrite=True, write_showers=True -) as writer: +with DataWriter(source, output_path=f.name, overwrite=True, write_dl2=True) as writer: for event in source: energy = event.simulation.shower.energy n_telescopes_r1 = len(event.r1.tel) diff --git a/src/ctapipe/io/datawriter.py b/src/ctapipe/io/datawriter.py index 1436b52f52d..1624b7e5dfe 100644 --- a/src/ctapipe/io/datawriter.py +++ b/src/ctapipe/io/datawriter.py @@ -199,7 +199,7 @@ class DataWriter(Component): help="Store DL1 image parameters if available", default_value=True ).tag(config=True) - write_showers = Bool( + write_dl2 = Bool( help="Store DL2 stereo shower parameters if available", default_value=False ).tag(config=True) @@ -332,7 +332,7 @@ def __call__(self, event: ArrayEventContainer): self._write_dl1_telescope_events(event) # write DL2 info if requested - if self.write_showers: + if self.write_dl2: self._write_dl2_telescope_events(event) self._write_dl2_stereo_event(event) @@ -398,7 +398,7 @@ def datalevels(self): data_levels.append(DataLevel.DL1_PARAMETERS) if self.write_muon_parameters: data_levels.append(DataLevel.DL1_MUON) - if self.write_showers: + if self.write_dl2: data_levels.append(DataLevel.DL2) if self.write_r0_waveforms: data_levels.append(DataLevel.R0) @@ -437,7 +437,7 @@ def _setup_output_path(self): writable_things = [ self.write_parameters, self.write_images, - self.write_showers, + self.write_dl2, self.write_r1_waveforms, self.write_muon_parameters, ] diff --git a/src/ctapipe/io/tests/conftest.py b/src/ctapipe/io/tests/conftest.py index 4e1de6c97e3..842c1116c05 100644 --- a/src/ctapipe/io/tests/conftest.py +++ b/src/ctapipe/io/tests/conftest.py @@ -22,7 +22,7 @@ def r1_hdf5_file(prod5_proton_simtel_path, r1_path): output_path=path, write_parameters=False, write_images=False, - write_showers=False, + write_dl2=False, write_r0_waveforms=False, write_r1_waveforms=True, ) diff --git a/src/ctapipe/io/tests/test_datawriter.py b/src/ctapipe/io/tests/test_datawriter.py index d79f67a01cb..6a919ce34ff 100644 --- a/src/ctapipe/io/tests/test_datawriter.py +++ b/src/ctapipe/io/tests/test_datawriter.py @@ -82,7 +82,7 @@ def test_write(tmpdir: Path): output_path=output_path, write_parameters=False, write_images=True, - write_showers=True, + write_dl2=True, write_r0_waveforms=True, write_r1_waveforms=True, ) as writer: @@ -166,7 +166,7 @@ def test_roundtrip(tmpdir: Path): transform_peak_time=True, peak_time_dtype="int16", peak_time_scale=100, - write_showers=True, + write_dl2=True, ) as write: write.log.level = logging.DEBUG for event in source: diff --git a/src/ctapipe/resources/base_config.yaml b/src/ctapipe/resources/base_config.yaml index 824399f8786..7daf1ede8e6 100644 --- a/src/ctapipe/resources/base_config.yaml +++ b/src/ctapipe/resources/base_config.yaml @@ -23,7 +23,7 @@ DataWriter: overwrite: false # do not overwrite existing files write_images: false # store DL1 images write_parameters: false # store DL1 parameters - write_showers: false # store DL2 stereo geometry + write_dl2: false # store DL2 stereo geometry write_r0_waveforms: false # write R0 waveforms write_r1_waveforms: false # write R1 waveforms write_muon_parameters: false # analyze and write muon parameters @@ -95,7 +95,7 @@ ImageProcessor: # The ShowerProcessor performs the DL1 to DL2a (reconstructed shower geometry) -# transition. It is run only if the parameter DataWriter.write_showers=True. +# transition. It is run only if the parameter DataWriter.write_dl2=True. ShowerProcessor: # choose between HillasReconstructor and HillasIntersection or both (two # implementations of the standard stereo line-intersection method) diff --git a/src/ctapipe/resources/image_modification_config.json b/src/ctapipe/resources/image_modification_config.json index 90e92c4c6f3..373f3562a13 100644 --- a/src/ctapipe/resources/image_modification_config.json +++ b/src/ctapipe/resources/image_modification_config.json @@ -8,7 +8,7 @@ "overwrite": false, "write_images": true, "write_parameters": true, - "write_showers": false, + "write_dl2": false, "transform_image": true, "transform_peak_time": true }, diff --git a/src/ctapipe/resources/ml_preprocessing_config.json b/src/ctapipe/resources/ml_preprocessing_config.json index 04109454d02..14322822b61 100644 --- a/src/ctapipe/resources/ml_preprocessing_config.json +++ b/src/ctapipe/resources/ml_preprocessing_config.json @@ -8,7 +8,7 @@ "overwrite": false, "write_images": false, "write_parameters": true, - "write_showers": true, + "write_dl2": true, "transform_image": true, "transform_peak_time": true }, diff --git a/src/ctapipe/resources/ml_preprocessing_config.yaml b/src/ctapipe/resources/ml_preprocessing_config.yaml index 28f8644d821..e25964a8c67 100644 --- a/src/ctapipe/resources/ml_preprocessing_config.yaml +++ b/src/ctapipe/resources/ml_preprocessing_config.yaml @@ -14,4 +14,4 @@ DataWriter: write_images: false write_parameters: true - write_showers: true + write_dl2: true diff --git a/src/ctapipe/resources/stage1_config.json b/src/ctapipe/resources/stage1_config.json index c414f5d3678..f280b57200f 100644 --- a/src/ctapipe/resources/stage1_config.json +++ b/src/ctapipe/resources/stage1_config.json @@ -8,7 +8,7 @@ "overwrite": false, "write_images": true, "write_parameters": true, - "write_showers": false, + "write_dl2": false, "transform_image": true, "transform_peak_time": true }, diff --git a/src/ctapipe/resources/stage1_config.toml b/src/ctapipe/resources/stage1_config.toml index 296798bcbd1..d90cae502e6 100644 --- a/src/ctapipe/resources/stage1_config.toml +++ b/src/ctapipe/resources/stage1_config.toml @@ -10,7 +10,7 @@ organization = "YOUR-ORGANIZATION" overwrite = false write_images = true write_parameters = true -write_showers = false +write_dl2 = false transform_image = true transform_peak_time = true diff --git a/src/ctapipe/resources/stage1_config.yaml b/src/ctapipe/resources/stage1_config.yaml index d0413061eb1..e82ce7d4b7c 100644 --- a/src/ctapipe/resources/stage1_config.yaml +++ b/src/ctapipe/resources/stage1_config.yaml @@ -12,4 +12,4 @@ DataWriter: write_images: true write_parameters: true - write_showers: false + write_dl2: false diff --git a/src/ctapipe/resources/stage2_config.json b/src/ctapipe/resources/stage2_config.json index 76c05ffff1f..0b1770811aa 100644 --- a/src/ctapipe/resources/stage2_config.json +++ b/src/ctapipe/resources/stage2_config.json @@ -8,7 +8,7 @@ "overwrite": false, "write_images": false, "write_parameters": false, - "write_showers": true, + "write_dl2": true, "transform_image": true, "transform_peak_time": true }, diff --git a/src/ctapipe/resources/stage2_config.yaml b/src/ctapipe/resources/stage2_config.yaml index 992fbe52276..1a930eaa563 100644 --- a/src/ctapipe/resources/stage2_config.yaml +++ b/src/ctapipe/resources/stage2_config.yaml @@ -13,4 +13,4 @@ DataWriter: write_images: false write_parameters: false - write_showers: true + write_dl2: true diff --git a/src/ctapipe/tools/process.py b/src/ctapipe/tools/process.py index 06e739cfd43..00815841e40 100644 --- a/src/ctapipe/tools/process.py +++ b/src/ctapipe/tools/process.py @@ -122,7 +122,7 @@ class ProcessorTool(Tool): ), **flag( "write-showers", - "DataWriter.write_showers", + "DataWriter.write_dl2", "store DL2/Event parameters in output", "don't DL2/Event parameters in output", ), @@ -209,7 +209,7 @@ def should_compute_dl2(self): if self.force_recompute_dl2: return True - return self.write.write_showers + return self.write.write_dl2 @property def should_compute_dl1(self): diff --git a/src/ctapipe/tools/tests/test_process.py b/src/ctapipe/tools/tests/test_process.py index 95237160535..769574c71f4 100644 --- a/src/ctapipe/tools/tests/test_process.py +++ b/src/ctapipe/tools/tests/test_process.py @@ -62,10 +62,7 @@ def test_multiple_configs(dl1_image_file): tool.setup() # ensure the overwriting works (base config has this option disabled) - assert ( - tool.get_current_config()["ProcessorTool"]["DataWriter"]["write_showers"] - is True - ) + assert tool.get_current_config()["ProcessorTool"]["DataWriter"]["write_dl2"] is True def test_stage_1_dl1(tmp_path, dl1_image_file, dl1_parameters_file): From 2c5a4f475c25cc210b43f7b0e4e9bb0a16c3a1fa Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Mon, 4 Mar 2024 14:50:23 +0100 Subject: [PATCH 4/6] write_images -> write_dl1_images --- .../tutorials/tableloader_and_eventsources.py | 2 +- src/ctapipe/io/datawriter.py | 18 +++++++++--------- src/ctapipe/io/tests/conftest.py | 2 +- src/ctapipe/io/tests/test_datawriter.py | 8 ++++---- src/ctapipe/resources/base_config.yaml | 2 +- .../resources/image_modification_config.json | 2 +- .../resources/ml_preprocessing_config.json | 2 +- .../resources/ml_preprocessing_config.yaml | 4 ++-- src/ctapipe/resources/stage1_config.json | 2 +- src/ctapipe/resources/stage1_config.toml | 2 +- src/ctapipe/resources/stage1_config.yaml | 2 +- src/ctapipe/resources/stage2_config.json | 2 +- src/ctapipe/resources/stage2_config.yaml | 2 +- src/ctapipe/tools/process.py | 4 ++-- 14 files changed, 27 insertions(+), 27 deletions(-) diff --git a/examples/tutorials/tableloader_and_eventsources.py b/examples/tutorials/tableloader_and_eventsources.py index 65db252cc5d..5590b07c39b 100644 --- a/examples/tutorials/tableloader_and_eventsources.py +++ b/examples/tutorials/tableloader_and_eventsources.py @@ -140,7 +140,7 @@ output_path="events.dl1.h5", write_parameters=False, overwrite=True, - write_images=True, + write_dl1_images=True, ) as write_data: for event in source: calibrate(event) diff --git a/src/ctapipe/io/datawriter.py b/src/ctapipe/io/datawriter.py index 1624b7e5dfe..72f69ad63a9 100644 --- a/src/ctapipe/io/datawriter.py +++ b/src/ctapipe/io/datawriter.py @@ -191,9 +191,9 @@ class DataWriter(Component): help="Store R1 waveforms if available", default_value=False ).tag(config=True) - write_images = Bool(help="Store DL1 Images if available", default_value=False).tag( - config=True - ) + write_dl1_images = Bool( + help="Store DL1 Images if available", default_value=False + ).tag(config=True) write_parameters = Bool( help="Store DL1 image parameters if available", default_value=True @@ -392,7 +392,7 @@ def finish(self): def datalevels(self): """returns a list of data levels requested""" data_levels = [] - if self.write_images: + if self.write_dl1_images: data_levels.append(DataLevel.DL1_IMAGES) if self.write_parameters: data_levels.append(DataLevel.DL1_PARAMETERS) @@ -436,7 +436,7 @@ def _setup_output_path(self): # check that options make sense writable_things = [ self.write_parameters, - self.write_images, + self.write_dl1_images, self.write_dl2, self.write_r1_waveforms, self.write_muon_parameters, @@ -472,7 +472,7 @@ def _setup_writer(self): writer.exclude("/dl1/event/telescope/images/.*", "parameters") writer.exclude("/simulation/event/telescope/images/.*", "true_parameters") - if not self.write_images: + if not self.write_dl1_images: writer.exclude("/simulation/event/telescope/images/.*", "true_image") if not self.write_parameters: @@ -677,10 +677,10 @@ def _write_dl1_telescope_events(self, event: ArrayEventContainer): containers=[tel_index, *dl1_camera.parameters.values()], ) - if self.write_images: + if self.write_dl1_images: if dl1_camera.image is None: raise ValueError( - "DataWriter.write_images is True but event does not contain image" + "DataWriter.write_dl1_images is True but event does not contain image" ) self._writer.write( @@ -783,7 +783,7 @@ def _generate_table_indices(self, h5file, start_node): def _generate_indices(self): """generate PyTables index tables for common columns""" self.log.debug("Writing index tables") - if self.write_images: + if self.write_dl1_images: self._generate_table_indices( self._writer.h5file, "/dl1/event/telescope/images" ) diff --git a/src/ctapipe/io/tests/conftest.py b/src/ctapipe/io/tests/conftest.py index 842c1116c05..ce7b4d1aff6 100644 --- a/src/ctapipe/io/tests/conftest.py +++ b/src/ctapipe/io/tests/conftest.py @@ -21,7 +21,7 @@ def r1_hdf5_file(prod5_proton_simtel_path, r1_path): event_source=source, output_path=path, write_parameters=False, - write_images=False, + write_dl1_images=False, write_dl2=False, write_r0_waveforms=False, write_r1_waveforms=True, diff --git a/src/ctapipe/io/tests/test_datawriter.py b/src/ctapipe/io/tests/test_datawriter.py index 6a919ce34ff..bdebf916493 100644 --- a/src/ctapipe/io/tests/test_datawriter.py +++ b/src/ctapipe/io/tests/test_datawriter.py @@ -81,7 +81,7 @@ def test_write(tmpdir: Path): event_source=source, output_path=output_path, write_parameters=False, - write_images=True, + write_dl1_images=True, write_dl2=True, write_r0_waveforms=True, write_r1_waveforms=True, @@ -159,7 +159,7 @@ def test_roundtrip(tmpdir: Path): event_source=source, output_path=output_path, write_parameters=False, - write_images=True, + write_dl1_images=True, transform_image=True, image_dtype="int32", image_scale=10, @@ -230,7 +230,7 @@ def test_dl1writer_no_events(tmpdir: Path): event_source=source, output_path=output_path, write_parameters=True, - write_images=True, + write_dl1_images=True, ) as writer: writer.log.level = logging.DEBUG writer.write_simulation_histograms(source) @@ -270,7 +270,7 @@ def test_metadata(tmpdir: Path): event_source=source, output_path=output_path, write_parameters=True, - write_images=True, + write_dl1_images=True, config=config, ): pass diff --git a/src/ctapipe/resources/base_config.yaml b/src/ctapipe/resources/base_config.yaml index 7daf1ede8e6..80d8fee61bf 100644 --- a/src/ctapipe/resources/base_config.yaml +++ b/src/ctapipe/resources/base_config.yaml @@ -21,7 +21,7 @@ DataWriter: # write nothing (can be overridden on the command-line or in subsequent config # files) overwrite: false # do not overwrite existing files - write_images: false # store DL1 images + write_dl1_images: false # store DL1 images write_parameters: false # store DL1 parameters write_dl2: false # store DL2 stereo geometry write_r0_waveforms: false # write R0 waveforms diff --git a/src/ctapipe/resources/image_modification_config.json b/src/ctapipe/resources/image_modification_config.json index 373f3562a13..80bab70f6ec 100644 --- a/src/ctapipe/resources/image_modification_config.json +++ b/src/ctapipe/resources/image_modification_config.json @@ -6,7 +6,7 @@ "organization": "CTA Consortium" }, "overwrite": false, - "write_images": true, + "write_dl1_images": true, "write_parameters": true, "write_dl2": false, "transform_image": true, diff --git a/src/ctapipe/resources/ml_preprocessing_config.json b/src/ctapipe/resources/ml_preprocessing_config.json index 14322822b61..e9aa2cbdcd1 100644 --- a/src/ctapipe/resources/ml_preprocessing_config.json +++ b/src/ctapipe/resources/ml_preprocessing_config.json @@ -6,7 +6,7 @@ "organization": "YOUR-ORGANIZATION" }, "overwrite": false, - "write_images": false, + "write_dl1_images": false, "write_parameters": true, "write_dl2": true, "transform_image": true, diff --git a/src/ctapipe/resources/ml_preprocessing_config.yaml b/src/ctapipe/resources/ml_preprocessing_config.yaml index e25964a8c67..663cecb124f 100644 --- a/src/ctapipe/resources/ml_preprocessing_config.yaml +++ b/src/ctapipe/resources/ml_preprocessing_config.yaml @@ -5,13 +5,13 @@ # Perform Stage 1 and Stage 2a processing to generate features needed for # training machine learning algorithms. # -# Note that here write_images is disabled by default (to avoid very large +# Note that here write_dl1_images is disabled by default (to avoid very large # files), however for training deep learning algorithms, it should be turned on. # ====================================================================== # # Make sure you first include `--config base_config.yaml` before including this file DataWriter: - write_images: false + write_dl1_images: false write_parameters: true write_dl2: true diff --git a/src/ctapipe/resources/stage1_config.json b/src/ctapipe/resources/stage1_config.json index f280b57200f..a2f331cc6f8 100644 --- a/src/ctapipe/resources/stage1_config.json +++ b/src/ctapipe/resources/stage1_config.json @@ -6,7 +6,7 @@ "organization": "YOUR-ORGANIZATION" }, "overwrite": false, - "write_images": true, + "write_dl1_images": true, "write_parameters": true, "write_dl2": false, "transform_image": true, diff --git a/src/ctapipe/resources/stage1_config.toml b/src/ctapipe/resources/stage1_config.toml index d90cae502e6..e0472d2cb1c 100644 --- a/src/ctapipe/resources/stage1_config.toml +++ b/src/ctapipe/resources/stage1_config.toml @@ -8,7 +8,7 @@ organization = "YOUR-ORGANIZATION" [DataWriter] # options that control what is stored in the output file overwrite = false -write_images = true +write_dl1_images = true write_parameters = true write_dl2 = false transform_image = true diff --git a/src/ctapipe/resources/stage1_config.yaml b/src/ctapipe/resources/stage1_config.yaml index e82ce7d4b7c..044811c45b6 100644 --- a/src/ctapipe/resources/stage1_config.yaml +++ b/src/ctapipe/resources/stage1_config.yaml @@ -10,6 +10,6 @@ # Make sure you first include `--config base_config.yaml` before including this file DataWriter: - write_images: true + write_dl1_images: true write_parameters: true write_dl2: false diff --git a/src/ctapipe/resources/stage2_config.json b/src/ctapipe/resources/stage2_config.json index 0b1770811aa..560715716d5 100644 --- a/src/ctapipe/resources/stage2_config.json +++ b/src/ctapipe/resources/stage2_config.json @@ -6,7 +6,7 @@ "organization": "YOUR-ORGANIZATION" }, "overwrite": false, - "write_images": false, + "write_dl1_images": false, "write_parameters": false, "write_dl2": true, "transform_image": true, diff --git a/src/ctapipe/resources/stage2_config.yaml b/src/ctapipe/resources/stage2_config.yaml index 1a930eaa563..f561ae9dc17 100644 --- a/src/ctapipe/resources/stage2_config.yaml +++ b/src/ctapipe/resources/stage2_config.yaml @@ -11,6 +11,6 @@ # Make sure you first include `--config base_config.yaml` before including this file DataWriter: - write_images: false + write_dl1_images: false write_parameters: false write_dl2: true diff --git a/src/ctapipe/tools/process.py b/src/ctapipe/tools/process.py index 00815841e40..63c0d3edd0a 100644 --- a/src/ctapipe/tools/process.py +++ b/src/ctapipe/tools/process.py @@ -110,7 +110,7 @@ class ProcessorTool(Tool): ), **flag( "write-images", - "DataWriter.write_images", + "DataWriter.write_dl1_images", "store DL1/Event/Telescope images in output", "don't store DL1/Event/Telescope images in output", ), @@ -233,7 +233,7 @@ def should_calibrate(self): return True if ( - self.write.write_images + self.write.write_dl1_images and DataLevel.DL1_IMAGES not in self.event_source.datalevels ): return True From 84be5e65f1e7ed8262073ba228e769c13a69d6bb Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Mon, 4 Mar 2024 15:06:42 +0100 Subject: [PATCH 5/6] write_parameters -> write_dl1_parameters --- examples/tutorials/tableloader_and_eventsources.py | 2 +- src/ctapipe/conftest.py | 2 +- src/ctapipe/io/datawriter.py | 14 +++++++------- src/ctapipe/io/tests/conftest.py | 2 +- src/ctapipe/io/tests/test_datawriter.py | 8 ++++---- src/ctapipe/resources/base_config.yaml | 2 +- .../resources/image_modification_config.json | 2 +- src/ctapipe/resources/ml_preprocessing_config.json | 2 +- src/ctapipe/resources/ml_preprocessing_config.yaml | 2 +- src/ctapipe/resources/stage1_config.json | 2 +- src/ctapipe/resources/stage1_config.toml | 2 +- src/ctapipe/resources/stage1_config.yaml | 2 +- src/ctapipe/resources/stage2_config.json | 2 +- src/ctapipe/resources/stage2_config.yaml | 2 +- src/ctapipe/tools/process.py | 6 +++--- 15 files changed, 26 insertions(+), 26 deletions(-) diff --git a/examples/tutorials/tableloader_and_eventsources.py b/examples/tutorials/tableloader_and_eventsources.py index 5590b07c39b..815654a40ca 100644 --- a/examples/tutorials/tableloader_and_eventsources.py +++ b/examples/tutorials/tableloader_and_eventsources.py @@ -138,7 +138,7 @@ with DataWriter( event_source=source, output_path="events.dl1.h5", - write_parameters=False, + write_dl1_parameters=False, overwrite=True, write_dl1_images=True, ) as write_data: diff --git a/src/ctapipe/conftest.py b/src/ctapipe/conftest.py index 7efaa9925c9..b521d82c7c5 100644 --- a/src/ctapipe/conftest.py +++ b/src/ctapipe/conftest.py @@ -415,7 +415,7 @@ def dl1_image_file(dl1_tmp_path, prod5_gamma_simtel_path): f"--input={prod5_gamma_simtel_path}", f"--output={output}", "--write-images", - "--DataWriter.write_parameters=False", + "--DataWriter.write_dl1_parameters=False", "--max-events=20", "--DataWriter.Contact.name=αℓℓ the äüöß", ] diff --git a/src/ctapipe/io/datawriter.py b/src/ctapipe/io/datawriter.py index 72f69ad63a9..637177a4eb5 100644 --- a/src/ctapipe/io/datawriter.py +++ b/src/ctapipe/io/datawriter.py @@ -195,7 +195,7 @@ class DataWriter(Component): help="Store DL1 Images if available", default_value=False ).tag(config=True) - write_parameters = Bool( + write_dl1_parameters = Bool( help="Store DL1 image parameters if available", default_value=True ).tag(config=True) @@ -394,7 +394,7 @@ def datalevels(self): data_levels = [] if self.write_dl1_images: data_levels.append(DataLevel.DL1_IMAGES) - if self.write_parameters: + if self.write_dl1_parameters: data_levels.append(DataLevel.DL1_PARAMETERS) if self.write_muon_parameters: data_levels.append(DataLevel.DL1_MUON) @@ -435,7 +435,7 @@ def _setup_output_path(self): # check that options make sense writable_things = [ - self.write_parameters, + self.write_dl1_parameters, self.write_dl1_images, self.write_dl2, self.write_r1_waveforms, @@ -475,7 +475,7 @@ def _setup_writer(self): if not self.write_dl1_images: writer.exclude("/simulation/event/telescope/images/.*", "true_image") - if not self.write_parameters: + if not self.write_dl1_parameters: writer.exclude("/dl1/event/telescope/images/.*", "image_mask") # Set up transforms @@ -671,7 +671,7 @@ def _write_dl1_telescope_events(self, event: ArrayEventContainer): table_name = self.table_name(tel_id) - if self.write_parameters: + if self.write_dl1_parameters: self._writer.write( table_name=f"dl1/event/telescope/parameters/{table_name}", containers=[tel_index, *dl1_camera.parameters.values()], @@ -699,7 +699,7 @@ def _write_dl1_telescope_events(self, event: ArrayEventContainer): tel_id in event.simulation.tel and event.simulation.tel[tel_id].true_image is not None ) - if self.write_parameters and has_sim_image: + if self.write_dl1_parameters and has_sim_image: true_parameters = event.simulation.tel[tel_id].true_parameters # only write the available containers, no peak time related # features for true image available. @@ -791,7 +791,7 @@ def _generate_indices(self): self._generate_table_indices( self._writer.h5file, "/simulation/event/telescope/images" ) - if self.write_parameters: + if self.write_dl1_parameters: self._generate_table_indices( self._writer.h5file, "/dl1/event/telescope/parameters" ) diff --git a/src/ctapipe/io/tests/conftest.py b/src/ctapipe/io/tests/conftest.py index ce7b4d1aff6..9f4e6e21ab0 100644 --- a/src/ctapipe/io/tests/conftest.py +++ b/src/ctapipe/io/tests/conftest.py @@ -20,7 +20,7 @@ def r1_hdf5_file(prod5_proton_simtel_path, r1_path): writer = DataWriter( event_source=source, output_path=path, - write_parameters=False, + write_dl1_parameters=False, write_dl1_images=False, write_dl2=False, write_r0_waveforms=False, diff --git a/src/ctapipe/io/tests/test_datawriter.py b/src/ctapipe/io/tests/test_datawriter.py index bdebf916493..72450ea1093 100644 --- a/src/ctapipe/io/tests/test_datawriter.py +++ b/src/ctapipe/io/tests/test_datawriter.py @@ -80,7 +80,7 @@ def test_write(tmpdir: Path): with DataWriter( event_source=source, output_path=output_path, - write_parameters=False, + write_dl1_parameters=False, write_dl1_images=True, write_dl2=True, write_r0_waveforms=True, @@ -158,7 +158,7 @@ def test_roundtrip(tmpdir: Path): with DataWriter( event_source=source, output_path=output_path, - write_parameters=False, + write_dl1_parameters=False, write_dl1_images=True, transform_image=True, image_dtype="int32", @@ -229,7 +229,7 @@ def test_dl1writer_no_events(tmpdir: Path): with DataWriter( event_source=source, output_path=output_path, - write_parameters=True, + write_dl1_parameters=True, write_dl1_images=True, ) as writer: writer.log.level = logging.DEBUG @@ -269,7 +269,7 @@ def test_metadata(tmpdir: Path): with DataWriter( event_source=source, output_path=output_path, - write_parameters=True, + write_dl1_parameters=True, write_dl1_images=True, config=config, ): diff --git a/src/ctapipe/resources/base_config.yaml b/src/ctapipe/resources/base_config.yaml index 80d8fee61bf..d6d570fa547 100644 --- a/src/ctapipe/resources/base_config.yaml +++ b/src/ctapipe/resources/base_config.yaml @@ -22,7 +22,7 @@ DataWriter: # files) overwrite: false # do not overwrite existing files write_dl1_images: false # store DL1 images - write_parameters: false # store DL1 parameters + write_dl1_parameters: false # store DL1 parameters write_dl2: false # store DL2 stereo geometry write_r0_waveforms: false # write R0 waveforms write_r1_waveforms: false # write R1 waveforms diff --git a/src/ctapipe/resources/image_modification_config.json b/src/ctapipe/resources/image_modification_config.json index 80bab70f6ec..253e10be02e 100644 --- a/src/ctapipe/resources/image_modification_config.json +++ b/src/ctapipe/resources/image_modification_config.json @@ -7,7 +7,7 @@ }, "overwrite": false, "write_dl1_images": true, - "write_parameters": true, + "write_dl1_parameters": true, "write_dl2": false, "transform_image": true, "transform_peak_time": true diff --git a/src/ctapipe/resources/ml_preprocessing_config.json b/src/ctapipe/resources/ml_preprocessing_config.json index e9aa2cbdcd1..1fecee8513f 100644 --- a/src/ctapipe/resources/ml_preprocessing_config.json +++ b/src/ctapipe/resources/ml_preprocessing_config.json @@ -7,7 +7,7 @@ }, "overwrite": false, "write_dl1_images": false, - "write_parameters": true, + "write_dl1_parameters": true, "write_dl2": true, "transform_image": true, "transform_peak_time": true diff --git a/src/ctapipe/resources/ml_preprocessing_config.yaml b/src/ctapipe/resources/ml_preprocessing_config.yaml index 663cecb124f..eeeef143d0b 100644 --- a/src/ctapipe/resources/ml_preprocessing_config.yaml +++ b/src/ctapipe/resources/ml_preprocessing_config.yaml @@ -13,5 +13,5 @@ DataWriter: write_dl1_images: false - write_parameters: true + write_dl1_parameters: true write_dl2: true diff --git a/src/ctapipe/resources/stage1_config.json b/src/ctapipe/resources/stage1_config.json index a2f331cc6f8..1dcb765a23e 100644 --- a/src/ctapipe/resources/stage1_config.json +++ b/src/ctapipe/resources/stage1_config.json @@ -7,7 +7,7 @@ }, "overwrite": false, "write_dl1_images": true, - "write_parameters": true, + "write_dl1_parameters": true, "write_dl2": false, "transform_image": true, "transform_peak_time": true diff --git a/src/ctapipe/resources/stage1_config.toml b/src/ctapipe/resources/stage1_config.toml index e0472d2cb1c..776293a59d5 100644 --- a/src/ctapipe/resources/stage1_config.toml +++ b/src/ctapipe/resources/stage1_config.toml @@ -9,7 +9,7 @@ organization = "YOUR-ORGANIZATION" # options that control what is stored in the output file overwrite = false write_dl1_images = true -write_parameters = true +write_dl1_parameters = true write_dl2 = false transform_image = true transform_peak_time = true diff --git a/src/ctapipe/resources/stage1_config.yaml b/src/ctapipe/resources/stage1_config.yaml index 044811c45b6..9f5cb10046d 100644 --- a/src/ctapipe/resources/stage1_config.yaml +++ b/src/ctapipe/resources/stage1_config.yaml @@ -11,5 +11,5 @@ DataWriter: write_dl1_images: true - write_parameters: true + write_dl1_parameters: true write_dl2: false diff --git a/src/ctapipe/resources/stage2_config.json b/src/ctapipe/resources/stage2_config.json index 560715716d5..3fe35426d58 100644 --- a/src/ctapipe/resources/stage2_config.json +++ b/src/ctapipe/resources/stage2_config.json @@ -7,7 +7,7 @@ }, "overwrite": false, "write_dl1_images": false, - "write_parameters": false, + "write_dl1_parameters": false, "write_dl2": true, "transform_image": true, "transform_peak_time": true diff --git a/src/ctapipe/resources/stage2_config.yaml b/src/ctapipe/resources/stage2_config.yaml index f561ae9dc17..e00002427fd 100644 --- a/src/ctapipe/resources/stage2_config.yaml +++ b/src/ctapipe/resources/stage2_config.yaml @@ -12,5 +12,5 @@ DataWriter: write_dl1_images: false - write_parameters: false + write_dl1_parameters: false write_dl2: true diff --git a/src/ctapipe/tools/process.py b/src/ctapipe/tools/process.py index 63c0d3edd0a..dc0123c5d08 100644 --- a/src/ctapipe/tools/process.py +++ b/src/ctapipe/tools/process.py @@ -43,7 +43,7 @@ class ProcessorTool(Tool): Note that the muon analysis and shower reconstruction both depend on parametrized images and therefore compute image parameters even if - DataWriter.write_parameters=False in case these are not already present + DataWriter.write_dl1_parameters=False in case these are not already present in the input file. """ @@ -116,7 +116,7 @@ class ProcessorTool(Tool): ), **flag( "write-parameters", - "DataWriter.write_parameters", + "DataWriter.write_dl1_parameters", "store DL1/Event/Telescope parameters in output", "don't store DL1/Event/Telescope parameters in output", ), @@ -221,7 +221,7 @@ def should_compute_dl1(self): return False return ( - self.write.write_parameters + self.write.write_dl1_parameters or self.should_compute_dl2 or self.should_compute_muon_parameters ) From d1a2ef5f6c057ebcf3d3362800dee4e9dc6218cd Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Mon, 4 Mar 2024 15:24:16 +0100 Subject: [PATCH 6/6] add changelog --- docs/changes/2520.api.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 docs/changes/2520.api.rst diff --git a/docs/changes/2520.api.rst b/docs/changes/2520.api.rst new file mode 100644 index 00000000000..bd20f32532a --- /dev/null +++ b/docs/changes/2520.api.rst @@ -0,0 +1,18 @@ +Unified the options for DataWriter and the data level names: + ++-------------------------+--------------------------+ +| Old | New | ++=========================+==========================+ +| ``write_raw_waveforms`` | ``write_r0_waveforms`` | ++-------------------------+--------------------------+ +| ``write_waveforms`` | ``write_r1_waveforms`` | ++-------------------------+--------------------------+ +| ``write_images`` | ``write_dl1_images`` | ++-------------------------+--------------------------+ +| ``write_parameters`` | ``write_dl1_parameters`` | ++-------------------------+--------------------------+ +| ``write_showers`` | ``write_dl2`` | ++-------------------------+--------------------------+ + +This changes requires that existing configuration files are updated +if they use these parameters