Skip to content

Commit

Permalink
Merge pull request #29 from spice-herald/feature/calib
Browse files Browse the repository at this point in the history 
Modify for BOR calib file type
  • Loading branch information
@serfass
serfass authored Aug 5, 2024
2 parents 785e8b7 + 412ab59 commit 9e89c18
Show file tree
Hide file tree
Showing 7 changed files with 226 additions and 117 deletions.
32 changes: 13 additions & 19 deletions detprocess/core/eventbuilder.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -254,15 +254,17 @@ def build_event(self, event_metadata=None,
[event_metadata['event_num']]*nb_triggers).astype(int)
if 'dump_num' in event_metadata.keys():
metadata_dict['dump_number'] = np.array(
[event_metadata['dump_num']]*nb_triggers) .astype(int)
[event_metadata['dump_num']]*nb_triggers).astype(int)
if 'run_type' in event_metadata.keys():
metadata_dict['data_type'] = np.array(
[event_metadata['run_type']]*nb_triggers)
[event_metadata['run_type']]*nb_triggers).astype(str)
elif 'data_type' in event_metadata.keys():
metadata_dict['data_type'] = np.array(
[event_metadata['data_type']]*nb_triggers).astype(str)
if 'fridge_run' in event_metadata.keys():
metadata_dict['fridge_run_number'] = np.array(
[event_metadata['fridge_run']]*nb_triggers).astype(int)


[event_metadata['fridge_run']]*nb_triggers).astype(int)

# event times
trigger_times = self._event_df['trigger_time'].values
event_times = trigger_times + event_time_start
Expand All @@ -282,19 +284,14 @@ def build_event(self, event_metadata=None,
np.array(range(nb_triggers))
+ int(self._current_trigger_id)
+ 1)



self._current_trigger_id = metadata_dict['trigger_prod_id'][-1]



# add to dataframe
for key,val in metadata_dict.items():
for key, val in metadata_dict.items():
self._event_df[key] = val





def _merge_coincident_triggers(self, fs=None,
coincident_window_msec=None,
coincident_window_samples=None):
Expand Down Expand Up @@ -323,21 +320,18 @@ def _merge_coincident_triggers(self, fs=None,
# let's convert vaex dataframe to pandas so we can modify it
# more easily
df_pandas = self._event_df.to_pandas_df()



# get trigger index and amplitude
trigger_indices = np.array(df_pandas['trigger_index'].values)
trigger_amplitudes = np.array(df_pandas['trigger_amplitude'].values)
trigger_names = np.array(df_pandas['trigger_channel'].values)



# find list of indices within merge_window
# then store in list of index ranges
lgc_coincident = np.diff(trigger_indices) < merge_window
lgc_coincident = np.concatenate(([0], lgc_coincident, [0]))
lgc_coincident_diff = np.abs(np.diff(lgc_coincident))
coincident_ranges = np.where(lgc_coincident_diff == 1)[0].reshape(-1, 2)


# let's first loop through ranges
# then
Expand Down
47 changes: 31 additions & 16 deletions detprocess/core/noise.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,7 @@ def generate_randoms(self, raw_path, series=None,
min_separation_msec=100,
edge_exclusion_msec=50,
restricted=False,
calib=False,
ncores=1):
"""
Generate randoms from continuous data
Expand All @@ -185,7 +186,8 @@ def generate_randoms(self, raw_path, series=None,
raw_data, output_base_path, group_name = (
self._get_file_list(raw_path,
series=series,
restricted=restricted)
restricted=restricted,
calib=calib)
)

if not raw_data:
Expand All @@ -199,9 +201,13 @@ def generate_randoms(self, raw_path, series=None,
self._series_list = list(raw_data.keys())
self._detector_config = dict()


# generate randoms
rand_inst = Randoms(raw_path, series=series, verbose=self._verbose)
rand_inst = Randoms(raw_path, series=series,
verbose=self._verbose,
restricted=restricted,
calib=calib)

self._dataframe = rand_inst.process(
random_rate=random_rate,
nrandoms=nevents,
Expand Down Expand Up @@ -743,7 +749,8 @@ def _load_dataframe(self, dataframe_path):
def _get_file_list(self, file_path,
series=None,
is_raw=True,
restricted=False):
restricted=False,
calib=False):
"""
Get file list from path. Return as a dictionary
with key=series and value=list of files
Expand Down Expand Up @@ -866,25 +873,33 @@ def _get_file_list(self, file_path,
continue

# skip didv
if 'didv_' in file_name:
continue

# skip iv
if 'iv_' in file_name:
if ('didv_' in file_name
or 'iv_' in file_name):
continue

if 'treshtrig_' in file_name:
continue

# restricted
if (restricted
and 'restricted' not in file_name):
# calibration
if (calib
and 'calib_' not in file_name):
continue

# not restricted
if (not restricted
and 'restricted' in file_name):
continue
# not calibration
if not calib:

if 'calib_' in file_name:
continue

# restricted
if (restricted
and 'restricted' not in file_name):
continue

# not restricted
if (not restricted
and 'restricted' in file_name):
continue

# append file if series already in dictionary
if (series_name is not None
Expand Down
70 changes: 50 additions & 20 deletions detprocess/process/features.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ def __init__(self, raw_path, config_file,
external_file=None,
processing_id=None,
restricted=False,
calib=False,
verbose=True):
"""
Intialize data processing
Expand Down Expand Up @@ -89,6 +90,9 @@ def __init__(self, raw_path, config_file,
if True, use restricted data
if False (default), exclude restricted data
calib : boolean
if True, use only "calib" files
if False, no calib files included
verbose : bool, optional
if True, display info
Expand All @@ -106,6 +110,11 @@ def __init__(self, raw_path, config_file,

# restricted data
self._restricted = restricted

# calib
self._calib = calib
if calib:
self._restricted = False

# display
self._verbose = verbose
Expand All @@ -124,7 +133,8 @@ def __init__(self, raw_path, config_file,
raw_files, raw_path, group_name = (
self._get_file_list(raw_path,
series=raw_series,
restricted=restricted)
restricted=restricted,
calib=calib)
)

if not raw_files:
Expand All @@ -144,7 +154,8 @@ def __init__(self, raw_path, config_file,
self._get_file_list(trigger_dataframe_path,
series=dataframe_series,
is_raw=False,
restricted=restricted)
restricted=restricted,
calib=calib)
)
if not trigger_files:
raise ValueError(f'No dataframe files were found! '
Expand Down Expand Up @@ -291,7 +302,8 @@ def process(self,
self._create_output_directory(
save_path,
self._processing_data.get_facility(),
restricted=self._restricted
restricted=self._restricted,
calib=self._calib
)
)

Expand Down Expand Up @@ -429,6 +441,8 @@ def _process(self, node_num,
file_prefix = self._processing_id + '_feature'
if self._restricted:
file_prefix += '_restricted'
elif self._calib:
file_prefix += '_calib'

series_name = h5io.extract_series_name(
int(output_series_num+node_num)
Expand Down Expand Up @@ -738,7 +752,8 @@ def _process(self, node_num,
def _get_file_list(self, file_path,
is_raw=True,
series=None,
restricted=False):
restricted=False,
calib=False):
"""
Get file list from path. Return as a dictionary
with key=series and value=list of files
Expand All @@ -758,7 +773,9 @@ def _get_file_list(self, file_path,
if True, use restricted data
if False, exclude restricted data
calib : boolean
if True, use only "calib" files
if False, no calib files included
Return
-------
Expand Down Expand Up @@ -875,23 +892,32 @@ def _get_file_list(self, file_path,
if 'filter' in file_name:
continue

# skip iv or didv
if 'didv_' in file_name:
continue
if 'iv_' in file_name:
continue

# restricted
if (restricted
and 'restricted' not in file_name):
# skip didv and iv
if ('didv_' in file_name
or 'iv_' in file_name):
continue

# not restricted
if (not restricted
and 'restricted' in file_name):
# calibration
if (calib
and 'calib_' not in file_name):
continue



# not calibration
if not calib:

if 'calib_' in file_name:
continue

# restricted
if (restricted
and 'restricted' not in file_name):
continue

# not restricted
if (not restricted
and 'restricted' in file_name):
continue

# append file if series already in dictionary
if (series_name is not None
and series_name in file_name
Expand Down Expand Up @@ -1083,7 +1109,8 @@ def _read_config(self, yaml_file, available_channels):


def _create_output_directory(self, base_path, facility,
restricted=False):
restricted=False,
calib=False):
"""
Create output directory
Expand Down Expand Up @@ -1117,6 +1144,9 @@ def _create_output_directory(self, base_path, facility,
prefix = self._processing_id + '_feature'
if restricted:
prefix += '_restricted'
elif calib:
prefix += '_calib'

output_dir = base_path + '/' + prefix + '_' + series_name


Expand Down
9 changes: 4 additions & 5 deletions detprocess/process/processing_data.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -242,8 +242,8 @@ def instantiate_OF_base(self, processing_config, channel=None):
if nb_samples != csd.shape[-1]:
raise ValueError(
f'Number of samples is not '
f'consistent between raw data '
f'and csd for channel {chan}, '
f'consistent between raw data (={nb_samples}) '
f'and csd (={csd.shape[-1]})for channel {chan}, '
f'algorithm {algo}!'
)

Expand Down Expand Up @@ -751,8 +751,8 @@ def get_event_admin(self, return_all=False):
admin_dict['series_number'] = np.int64(self._current_admin_info['series_num'])
admin_dict['event_id'] = np.int32(self._current_admin_info['event_id'])
admin_dict['event_time'] = np.int64(self._current_admin_info['event_time'])
admin_dict['run_type'] = self._current_admin_info['run_type']
admin_dict['data_type'] = self._current_admin_info['run_type']
admin_dict['run_type'] = np.str(self._current_admin_info['run_type'])
admin_dict['data_type'] = np.str(self._current_admin_info['run_type'])

# group name
if self._group_name is not None:
Expand Down Expand Up @@ -814,7 +814,6 @@ def get_event_admin(self, return_all=False):
else:
admin_dict['group_start_time'] = np.nan


return admin_dict


Expand Down
Loading

0 comments on commit 9e89c18

Please sign in to comment.