Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove NoSuchNodeError catch in dst loading #801

Merged
merged 3 commits into from
Sep 8, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 26 additions & 20 deletions invisible_cities/io/dst_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,28 +18,34 @@ def _decode_str_columns(df):
df = df.apply(to_string)
return df

def load_dst(filename, group, node, evt_list=None):
def load_dst(filename, group, node, evt_list=None, ignore_errors=False):
"""load a kdst if filename, group and node correctly found"""
try:

def read_dst_(filename, group, node, evt_list):
with tb.open_file(filename) as h5in:
try:
table = getattr(getattr(h5in.root, group), node)
if evt_list is None:
values = table.read()
else:
events = table.read(field='event')
values = table.read_coordinates(np.where(np.isin(events, evt_list)))
return _decode_str_columns(pd.DataFrame.from_records(values, columns=table.colnames))
except NoSuchNodeError:
warnings.warn(f' not of kdst type: file= {filename} ', UserWarning)
except HDF5ExtError:
warnings.warn(f' corrupted: file = {filename} ', UserWarning)
except IOError:
warnings.warn(f' does not exist: file = {filename} ', UserWarning)


def load_dsts(dst_list, group, node, evt_list=None):
dsts = [load_dst(filename, group, node, evt_list) for filename in dst_list]
table = getattr(getattr(h5in.root, group), node)
if evt_list is None:
values = table.read()
else:
events = table.read(field='event')
values = table.read_coordinates(np.where(np.isin(events, evt_list)))
return _decode_str_columns(pd.DataFrame.from_records(values, columns=table.colnames))

if ignore_errors:
try:
return read_dst_(filename, group, node, evt_list)
except NoSuchNodeError:
warnings.warn(f' not of kdst type: file= {filename} ', UserWarning)
except HDF5ExtError:
warnings.warn(f' corrupted: file = {filename} ', UserWarning)
except IOError:
warnings.warn(f' does not exist: file = {filename} ', UserWarning)
else:
return read_dst_(filename, group, node, evt_list)


def load_dsts(dst_list, group, node, evt_list=None, ignore_errors=False):
dsts = [load_dst(filename, group, node, evt_list, ignore_errors) for filename in dst_list]
return pd.concat(dsts, ignore_index=True)

def _make_tabledef(column_types : np.dtype, str_col_length : int) -> dict:
Expand Down
4 changes: 2 additions & 2 deletions invisible_cities/io/dst_io_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ def test_load_dsts_warns_not_of_kdst_type(ICDATADIR):
group = "DST"
node = "Events"
with pytest.warns(UserWarning, match='not of kdst type'):
load_dsts([good_file, wrong_file], group, node)
load_dsts([good_file, wrong_file], group, node, ignore_errors=True)


def test_load_dsts_warns_if_not_existing_file(ICDATADIR):
Expand All @@ -124,7 +124,7 @@ def test_load_dsts_warns_if_not_existing_file(ICDATADIR):
group = "DST"
node = "Events"
with pytest.warns(UserWarning, match='does not exist'):
load_dsts([good_file, wrong_file], group, node)
load_dsts([good_file, wrong_file], group, node, ignore_errors=True)


def test_load_dst_converts_from_bytes(ICDATADIR, fixed_dataframe):
Expand Down
8 changes: 2 additions & 6 deletions invisible_cities/io/mcinfo_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -132,16 +132,12 @@ def safe_copy_nexus_eventmap(h5out : tb.file.File,
'evt_number': the evt_number copied to the output;
'nexus_evt' : the nexus event_ids copied to the output.
"""
# Suppress warning raised when table not
# present since case dealt with in try except.
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UserWarning)
evt_map = load_eventnumbermap(file_in)
try:
evt_map = load_eventnumbermap(file_in)
evt_mask = evt_map.evt_number.isin(evt_arr)
df_writer(h5out, evt_map[evt_mask], 'Run', 'eventMap')
return evt_map[evt_mask].to_dict('list')
except AttributeError:
except (AttributeError, tb.exceptions.NoSuchNodeError):
## Invoked when the eventMap table not found
## which occurs for pre-2020 MC files and
## at the first processing stage from nexus
Expand Down