From 50ad0c8cf346b819645ef037091249322902f043 Mon Sep 17 00:00:00 2001 From: Andrew Laing Date: Thu, 28 May 2020 18:15:42 +0200 Subject: [PATCH 1/7] Add buffer writer capable of writing nexus event as multiple events --- invisible_cities/io/rwf_io.py | 77 ++++++++++++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 2 deletions(-) diff --git a/invisible_cities/io/rwf_io.py b/invisible_cities/io/rwf_io.py index d7920427d2..3c99cd228b 100644 --- a/invisible_cities/io/rwf_io.py +++ b/invisible_cities/io/rwf_io.py @@ -1,9 +1,13 @@ import numpy as np import tables as tb -from typing import Callable +from functools import partial +from typing import Callable +from typing import List +from typing import Tuple -from .. reco import tbl_functions as tbl +from .. reco import tbl_functions as tbl +from . run_and_event_io import run_and_event_writer def rwf_writer(h5out : tb.file.File , @@ -54,3 +58,72 @@ def write_rwf(waveform : np.ndarray) -> None: """ rwf_table.append(waveform.reshape(1, n_sensors, waveform_length)) return write_rwf + + +class EventMap(tb.IsDescription): + """ + Maps the output event number and + the original nexus event number + NEEDS TO BE REVIEWED FOR INTEGRATION + """ + evt_number = tb.Int32Col(shape=(), pos=0) + nexus_evt = tb.Int32Col(shape=(), pos=1) + + +def buffer_writer(h5out, *, + run_number : int , + n_sens_eng : int , + n_sens_trk : int , + length_eng : int , + length_trk : int , + group_name : str = None, + compression: str = 'ZLIB4' + ) -> Callable[[int, List, List, List], None]: + """ + Generalised buffer writer which defines a raw waveform writer + for each type of sensor as well as an event info writer + with written event, timestamp and a mapping to the + nexus event number in case of event splitting. + """ + + eng_writer = rwf_writer(h5out, + group_name = group_name, + compression = compression, + table_name = 'pmtrd', + n_sensors = n_sens_eng, + waveform_length = length_eng) + + trk_writer = rwf_writer(h5out, + group_name = group_name, + compression = compression, + table_name = 'sipmrd', + n_sensors = n_sens_trk, + waveform_length = length_trk) + + run_and_event = partial(run_and_event_writer(h5out , + compression=compression), + run_number = run_number ) + + evt_group = getattr(h5out.root, 'Run') + nexus_map = h5out.create_table(evt_group, "eventMap", EventMap, + "event & nexus evt \ + for each index", + tbl.filters(compression)) + + def write_buffers(nexus_evt : int , + timestamps : List[ int], + events : List[Tuple]) -> None: + + for i, (t_stamp, (eng, trk)) in enumerate(zip(timestamps, events)): + ## The exact way to log MC event splitting + ## still to be decided. + run_and_event(event_number=nexus_evt, timestamp=t_stamp) + mrow = nexus_map.row + mrow["evt_number"] = i + mrow["nexus_evt"] = nexus_evt + mrow.append() + ## + + eng_writer(eng) + trk_writer(trk) + return write_buffers From d1e573f8b6b890ff79bafcd626ef535b97834950 Mon Sep 17 00:00:00 2001 From: Andrew Laing Date: Thu, 28 May 2020 18:17:07 +0200 Subject: [PATCH 2/7] Test for buffer writer --- invisible_cities/io/rwf_io_test.py | 54 ++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/invisible_cities/io/rwf_io_test.py b/invisible_cities/io/rwf_io_test.py index 19feaa3c2c..29f4b7b551 100644 --- a/invisible_cities/io/rwf_io_test.py +++ b/invisible_cities/io/rwf_io_test.py @@ -3,9 +3,11 @@ import numpy as np import tables as tb -from pytest import mark +from pytest import fixture +from pytest import mark -from . rwf_io import rwf_writer +from . rwf_io import rwf_writer +from . rwf_io import buffer_writer @mark.parametrize("group_name", (None, 'RD', 'BLR')) @@ -42,3 +44,51 @@ def test_rwf_writer(config_tmpdir, group_name): table = getattr(group, table_name) assert table.shape == (nevt, nsensor, nsample) assert np.all(test_data == table.read()) + + +@mark.parametrize("event triggers".split(), + ((2, [10]), (3, [10, 1100]), (4, [20]))) +def test_buffer_writer(config_tmpdir, event, triggers): + + run_number = -6400 + n_pmt = 12 + nsamp_pmt = 100 + n_sipm = 1792 + nsamp_sipm = 10 + + buffers = [(np.random.poisson(5, (n_pmt , nsamp_pmt )), + np.random.poisson(5, (n_sipm, nsamp_sipm))) for _ in triggers] + + out_name = os.path.join(config_tmpdir, 'test_buffers.h5') + with tb.open_file(out_name, 'w') as data_out: + + buffer_writer_ = buffer_writer(data_out, + run_number = run_number, + n_sens_eng = n_pmt, + n_sens_trk = n_sipm, + length_eng = nsamp_pmt, + length_trk = nsamp_sipm) + + buffer_writer_(event, triggers, buffers) + + pmt_wf = np.array([b[0] for b in buffers]) + sipm_wf = np.array([b[1] for b in buffers]) + with tb.open_file(out_name) as h5saved: + assert 'Run' in h5saved.root + assert 'pmtrd' in h5saved.root + assert 'sipmrd' in h5saved.root + assert 'events' in h5saved.root.Run + assert 'runInfo' in h5saved.root.Run + assert 'eventMap' in h5saved.root.Run + + nsaves = len(triggers) + assert len(h5saved.root.Run.events ) == nsaves + assert len(h5saved.root.Run.eventMap) == nsaves + assert len(h5saved.root.Run.runInfo ) == nsaves + assert np.all([r[0] == run_number for r in h5saved.root.Run.runInfo]) + + assert h5saved.root.pmtrd .shape == (nsaves, n_pmt , nsamp_pmt) + assert np.all(h5saved.root.pmtrd [:] == pmt_wf) + + assert h5saved.root.sipmrd.shape == (nsaves, n_sipm, nsamp_sipm) + assert np.all(h5saved.root.sipmrd[:] == sipm_wf) From ad0a847df5ca06aacb4fd4836c975d4f8b3ad3e7 Mon Sep 17 00:00:00 2001 From: Andrew Laing Date: Thu, 28 May 2020 18:51:24 +0200 Subject: [PATCH 3/7] Improve docstrings for new functions --- invisible_cities/io/rwf_io.py | 59 ++++++++++++++++++++++++++--------- 1 file changed, 45 insertions(+), 14 deletions(-) diff --git a/invisible_cities/io/rwf_io.py b/invisible_cities/io/rwf_io.py index 3c99cd228b..6645e44ef0 100644 --- a/invisible_cities/io/rwf_io.py +++ b/invisible_cities/io/rwf_io.py @@ -4,6 +4,7 @@ from functools import partial from typing import Callable from typing import List +from typing import Optional from typing import Tuple from .. reco import tbl_functions as tbl @@ -62,28 +63,45 @@ def write_rwf(waveform : np.ndarray) -> None: class EventMap(tb.IsDescription): """ - Maps the output event number and - the original nexus event number - NEEDS TO BE REVIEWED FOR INTEGRATION + Map between event index and original + event. """ evt_number = tb.Int32Col(shape=(), pos=0) nexus_evt = tb.Int32Col(shape=(), pos=1) def buffer_writer(h5out, *, - run_number : int , - n_sens_eng : int , - n_sens_trk : int , - length_eng : int , - length_trk : int , - group_name : str = None, - compression: str = 'ZLIB4' - ) -> Callable[[int, List, List, List], None]: + run_number : int , + n_sens_eng : int , + n_sens_trk : int , + length_eng : int , + length_trk : int , + group_name : Optional[str] = None, + compression: Optional[str] = 'ZLIB4' + ) -> Callable[[int, List, List], None]: """ Generalised buffer writer which defines a raw waveform writer - for each type of sensor as well as an event info writer - with written event, timestamp and a mapping to the - nexus event number in case of event splitting. + for each type of sensor as well as an event info writer. + Each call gives a list of 'triggers' to be written as + separate events in the output. + + parameters + ---------- + run_number : int + Run number to be saved in runInfo. + n_sens_eng : int + Number of sensors in the energy plane. + n_sens_trk : int + Number of sensors in the tracking plane. + length_eng : int + Number of samples per waveform for energy plane. + length_trk : int + Number of samples per waveform for tracking plane. + group_name : Optional[str] default None + Group name within root where waveforms to be saved. + Default directly in root + compression : Optional[str] default 'ZLIB4' + Compression level for output file. """ eng_writer = rwf_writer(h5out, @@ -113,6 +131,19 @@ def buffer_writer(h5out, *, def write_buffers(nexus_evt : int , timestamps : List[ int], events : List[Tuple]) -> None: + """ + Write run info and event waveforms to file. + + parameters + ---------- + nexus_evt : int + Event number from MC output file. + timestamps : List[int] + List of event times + events : List[Tuple] + List of tuples containing the energy and + tracking plane info for each identified 'trigger'. + """ for i, (t_stamp, (eng, trk)) in enumerate(zip(timestamps, events)): ## The exact way to log MC event splitting From a3f11079d63fc0662814be37a52d65524f0f1f59 Mon Sep 17 00:00:00 2001 From: Andrew Laing Date: Wed, 3 Jun 2020 12:20:12 +0200 Subject: [PATCH 4/7] Add return info to buffer_writer docstring --- invisible_cities/io/rwf_io.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/invisible_cities/io/rwf_io.py b/invisible_cities/io/rwf_io.py index 6645e44ef0..b2d202c9b4 100644 --- a/invisible_cities/io/rwf_io.py +++ b/invisible_cities/io/rwf_io.py @@ -102,6 +102,13 @@ def buffer_writer(h5out, *, Default directly in root compression : Optional[str] default 'ZLIB4' Compression level for output file. + + returns + ------- + write_buffers : Callable + A function which takes event information + for the tracking and energy planes and + the event timestamps and saves to file. """ eng_writer = rwf_writer(h5out, From 5dfdf052d713997beef908673e8c31a25d45b075 Mon Sep 17 00:00:00 2001 From: Andrew Laing Date: Wed, 3 Jun 2020 12:24:20 +0200 Subject: [PATCH 5/7] Move EventMap class from rwf_io to evm/nh5 and rename MCEventMap --- invisible_cities/evm/nh5.py | 6 ++++++ invisible_cities/io/rwf_io.py | 12 ++---------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/invisible_cities/evm/nh5.py b/invisible_cities/evm/nh5.py index e683a323d8..b2b022b874 100644 --- a/invisible_cities/evm/nh5.py +++ b/invisible_cities/evm/nh5.py @@ -43,6 +43,12 @@ class MCExtentInfo(tb.IsDescription): last_particle = tb.UInt64Col(pos=2) +class MCEventMap(tb.IsDescription): + """Map between event index and original event.""" + evt_number = tb.Int32Col(shape=(), pos=0) + nexus_evt = tb.Int32Col(shape=(), pos=1) + + class MCHitInfo(tb.IsDescription): """Stores the simulated hits as metadata using Pytables. """ diff --git a/invisible_cities/io/rwf_io.py b/invisible_cities/io/rwf_io.py index b2d202c9b4..ffee11f94a 100644 --- a/invisible_cities/io/rwf_io.py +++ b/invisible_cities/io/rwf_io.py @@ -7,6 +7,7 @@ from typing import Optional from typing import Tuple +from .. evm .nh5 import MCEventMap from .. reco import tbl_functions as tbl from . run_and_event_io import run_and_event_writer @@ -61,15 +62,6 @@ def write_rwf(waveform : np.ndarray) -> None: return write_rwf -class EventMap(tb.IsDescription): - """ - Map between event index and original - event. - """ - evt_number = tb.Int32Col(shape=(), pos=0) - nexus_evt = tb.Int32Col(shape=(), pos=1) - - def buffer_writer(h5out, *, run_number : int , n_sens_eng : int , @@ -130,7 +122,7 @@ def buffer_writer(h5out, *, run_number = run_number ) evt_group = getattr(h5out.root, 'Run') - nexus_map = h5out.create_table(evt_group, "eventMap", EventMap, + nexus_map = h5out.create_table(evt_group, "eventMap", MCEventMap, "event & nexus evt \ for each index", tbl.filters(compression)) From 182d078a37c1b1d9df801b7e241414741a71486b Mon Sep 17 00:00:00 2001 From: Andrew Laing Date: Wed, 3 Jun 2020 12:37:27 +0200 Subject: [PATCH 6/7] Use tables_io.make_table to make nexus_map in buffer_writer --- invisible_cities/io/rwf_io.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/invisible_cities/io/rwf_io.py b/invisible_cities/io/rwf_io.py index ffee11f94a..1eb0a0fece 100644 --- a/invisible_cities/io/rwf_io.py +++ b/invisible_cities/io/rwf_io.py @@ -10,6 +10,7 @@ from .. evm .nh5 import MCEventMap from .. reco import tbl_functions as tbl from . run_and_event_io import run_and_event_writer +from . table_io import make_table def rwf_writer(h5out : tb.file.File , @@ -121,11 +122,8 @@ def buffer_writer(h5out, *, compression=compression), run_number = run_number ) - evt_group = getattr(h5out.root, 'Run') - nexus_map = h5out.create_table(evt_group, "eventMap", MCEventMap, - "event & nexus evt \ - for each index", - tbl.filters(compression)) + nexus_map = make_table(h5out, 'Run', 'eventMap', MCEventMap, + "event & nexus evt for each index", compression) def write_buffers(nexus_evt : int , timestamps : List[ int], From 0e630e2f1fe3de07a8c381b0dd3a488c751bdbf6 Mon Sep 17 00:00:00 2001 From: Andrew Laing Date: Thu, 4 Jun 2020 12:58:04 +0200 Subject: [PATCH 7/7] Improve names in MCEventMap --- invisible_cities/evm/nh5.py | 2 +- invisible_cities/io/rwf_io.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/invisible_cities/evm/nh5.py b/invisible_cities/evm/nh5.py index b2b022b874..e243f4ac60 100644 --- a/invisible_cities/evm/nh5.py +++ b/invisible_cities/evm/nh5.py @@ -46,7 +46,7 @@ class MCExtentInfo(tb.IsDescription): class MCEventMap(tb.IsDescription): """Map between event index and original event.""" evt_number = tb.Int32Col(shape=(), pos=0) - nexus_evt = tb.Int32Col(shape=(), pos=1) + sub_evt = tb.Int32Col(shape=(), pos=1) class MCHitInfo(tb.IsDescription): diff --git a/invisible_cities/io/rwf_io.py b/invisible_cities/io/rwf_io.py index 1eb0a0fece..fbeb98863f 100644 --- a/invisible_cities/io/rwf_io.py +++ b/invisible_cities/io/rwf_io.py @@ -147,8 +147,8 @@ def write_buffers(nexus_evt : int , ## still to be decided. run_and_event(event_number=nexus_evt, timestamp=t_stamp) mrow = nexus_map.row - mrow["evt_number"] = i - mrow["nexus_evt"] = nexus_evt + mrow["evt_number"] = nexus_evt + mrow[ "sub_evt"] = i mrow.append() ##