From 82ba22ecb1ce2314f3b87294644b20404fd8f74c Mon Sep 17 00:00:00 2001 From: Bruno Serfass Date: Fri, 2 Aug 2024 14:04:36 -0700 Subject: [PATCH 1/2] change error message --- detprocess/process/processing_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/detprocess/process/processing_data.py b/detprocess/process/processing_data.py index 692c211..58cd811 100644 --- a/detprocess/process/processing_data.py +++ b/detprocess/process/processing_data.py @@ -242,8 +242,8 @@ def instantiate_OF_base(self, processing_config, channel=None): if nb_samples != csd.shape[-1]: raise ValueError( f'Number of samples is not ' - f'consistent between raw data ' - f'and csd for channel {chan}, ' + f'consistent between raw data (={nb_samples}) ' + f'and csd (={csd.shape[-1]})for channel {chan}, ' f'algorithm {algo}!' ) From 412ab59123684978e234943bfc2104f10a4b400b Mon Sep 17 00:00:00 2001 From: Bruno Serfass Date: Mon, 5 Aug 2024 14:35:37 -0700 Subject: [PATCH 2/2] add calib parameter --- detprocess/core/eventbuilder.py | 32 ++++------ detprocess/core/noise.py | 47 +++++++++----- detprocess/process/features.py | 70 +++++++++++++++------ detprocess/process/processing_data.py | 5 +- detprocess/process/randoms.py | 88 +++++++++++++++++++-------- detprocess/process/triggers.py | 73 +++++++++++++++------- scripts/process.py | 24 +++++--- 7 files changed, 224 insertions(+), 115 deletions(-) diff --git a/detprocess/core/eventbuilder.py b/detprocess/core/eventbuilder.py index 63b6e21..58ad96a 100644 --- a/detprocess/core/eventbuilder.py +++ b/detprocess/core/eventbuilder.py @@ -254,15 +254,17 @@ def build_event(self, event_metadata=None, [event_metadata['event_num']]*nb_triggers).astype(int) if 'dump_num' in event_metadata.keys(): metadata_dict['dump_number'] = np.array( - [event_metadata['dump_num']]*nb_triggers) .astype(int) + [event_metadata['dump_num']]*nb_triggers).astype(int) if 'run_type' in event_metadata.keys(): metadata_dict['data_type'] = np.array( - [event_metadata['run_type']]*nb_triggers) + [event_metadata['run_type']]*nb_triggers).astype(str) + elif 'data_type' in event_metadata.keys(): + metadata_dict['data_type'] = np.array( + [event_metadata['data_type']]*nb_triggers).astype(str) if 'fridge_run' in event_metadata.keys(): metadata_dict['fridge_run_number'] = np.array( - [event_metadata['fridge_run']]*nb_triggers).astype(int) - - + [event_metadata['fridge_run']]*nb_triggers).astype(int) + # event times trigger_times = self._event_df['trigger_time'].values event_times = trigger_times + event_time_start @@ -282,19 +284,14 @@ def build_event(self, event_metadata=None, np.array(range(nb_triggers)) + int(self._current_trigger_id) + 1) - - + self._current_trigger_id = metadata_dict['trigger_prod_id'][-1] - - + # add to dataframe - for key,val in metadata_dict.items(): + for key, val in metadata_dict.items(): self._event_df[key] = val - - - - + def _merge_coincident_triggers(self, fs=None, coincident_window_msec=None, coincident_window_samples=None): @@ -323,21 +320,18 @@ def _merge_coincident_triggers(self, fs=None, # let's convert vaex dataframe to pandas so we can modify it # more easily df_pandas = self._event_df.to_pandas_df() - - + # get trigger index and amplitude trigger_indices = np.array(df_pandas['trigger_index'].values) trigger_amplitudes = np.array(df_pandas['trigger_amplitude'].values) trigger_names = np.array(df_pandas['trigger_channel'].values) - - + # find list of indices within merge_window # then store in list of index ranges lgc_coincident = np.diff(trigger_indices) < merge_window lgc_coincident = np.concatenate(([0], lgc_coincident, [0])) lgc_coincident_diff = np.abs(np.diff(lgc_coincident)) coincident_ranges = np.where(lgc_coincident_diff == 1)[0].reshape(-1, 2) - # let's first loop through ranges # then diff --git a/detprocess/core/noise.py b/detprocess/core/noise.py index 70093c0..3fd1e1a 100644 --- a/detprocess/core/noise.py +++ b/detprocess/core/noise.py @@ -173,6 +173,7 @@ def generate_randoms(self, raw_path, series=None, min_separation_msec=100, edge_exclusion_msec=50, restricted=False, + calib=False, ncores=1): """ Generate randoms from continuous data @@ -185,7 +186,8 @@ def generate_randoms(self, raw_path, series=None, raw_data, output_base_path, group_name = ( self._get_file_list(raw_path, series=series, - restricted=restricted) + restricted=restricted, + calib=calib) ) if not raw_data: @@ -199,9 +201,13 @@ def generate_randoms(self, raw_path, series=None, self._series_list = list(raw_data.keys()) self._detector_config = dict() - + # generate randoms - rand_inst = Randoms(raw_path, series=series, verbose=self._verbose) + rand_inst = Randoms(raw_path, series=series, + verbose=self._verbose, + restricted=restricted, + calib=calib) + self._dataframe = rand_inst.process( random_rate=random_rate, nrandoms=nevents, @@ -743,7 +749,8 @@ def _load_dataframe(self, dataframe_path): def _get_file_list(self, file_path, series=None, is_raw=True, - restricted=False): + restricted=False, + calib=False): """ Get file list from path. Return as a dictionary with key=series and value=list of files @@ -866,25 +873,33 @@ def _get_file_list(self, file_path, continue # skip didv - if 'didv_' in file_name: - continue - - # skip iv - if 'iv_' in file_name: + if ('didv_' in file_name + or 'iv_' in file_name): continue if 'treshtrig_' in file_name: continue - # restricted - if (restricted - and 'restricted' not in file_name): + # calibration + if (calib + and 'calib_' not in file_name): continue - # not restricted - if (not restricted - and 'restricted' in file_name): - continue + # not calibration + if not calib: + + if 'calib_' in file_name: + continue + + # restricted + if (restricted + and 'restricted' not in file_name): + continue + + # not restricted + if (not restricted + and 'restricted' in file_name): + continue # append file if series already in dictionary if (series_name is not None diff --git a/detprocess/process/features.py b/detprocess/process/features.py index 29bd3c7..8c3ef5f 100644 --- a/detprocess/process/features.py +++ b/detprocess/process/features.py @@ -48,6 +48,7 @@ def __init__(self, raw_path, config_file, external_file=None, processing_id=None, restricted=False, + calib=False, verbose=True): """ Intialize data processing @@ -89,6 +90,9 @@ def __init__(self, raw_path, config_file, if True, use restricted data if False (default), exclude restricted data + calib : boolean + if True, use only "calib" files + if False, no calib files included verbose : bool, optional if True, display info @@ -106,6 +110,11 @@ def __init__(self, raw_path, config_file, # restricted data self._restricted = restricted + + # calib + self._calib = calib + if calib: + self._restricted = False # display self._verbose = verbose @@ -124,7 +133,8 @@ def __init__(self, raw_path, config_file, raw_files, raw_path, group_name = ( self._get_file_list(raw_path, series=raw_series, - restricted=restricted) + restricted=restricted, + calib=calib) ) if not raw_files: @@ -144,7 +154,8 @@ def __init__(self, raw_path, config_file, self._get_file_list(trigger_dataframe_path, series=dataframe_series, is_raw=False, - restricted=restricted) + restricted=restricted, + calib=calib) ) if not trigger_files: raise ValueError(f'No dataframe files were found! ' @@ -291,7 +302,8 @@ def process(self, self._create_output_directory( save_path, self._processing_data.get_facility(), - restricted=self._restricted + restricted=self._restricted, + calib=self._calib ) ) @@ -429,6 +441,8 @@ def _process(self, node_num, file_prefix = self._processing_id + '_feature' if self._restricted: file_prefix += '_restricted' + elif self._calib: + file_prefix += '_calib' series_name = h5io.extract_series_name( int(output_series_num+node_num) @@ -738,7 +752,8 @@ def _process(self, node_num, def _get_file_list(self, file_path, is_raw=True, series=None, - restricted=False): + restricted=False, + calib=False): """ Get file list from path. Return as a dictionary with key=series and value=list of files @@ -758,7 +773,9 @@ def _get_file_list(self, file_path, if True, use restricted data if False, exclude restricted data - + calib : boolean + if True, use only "calib" files + if False, no calib files included Return ------- @@ -875,23 +892,32 @@ def _get_file_list(self, file_path, if 'filter' in file_name: continue - # skip iv or didv - if 'didv_' in file_name: - continue - if 'iv_' in file_name: - continue - - # restricted - if (restricted - and 'restricted' not in file_name): + # skip didv and iv + if ('didv_' in file_name + or 'iv_' in file_name): continue - # not restricted - if (not restricted - and 'restricted' in file_name): + # calibration + if (calib + and 'calib_' not in file_name): continue - - + + # not calibration + if not calib: + + if 'calib_' in file_name: + continue + + # restricted + if (restricted + and 'restricted' not in file_name): + continue + + # not restricted + if (not restricted + and 'restricted' in file_name): + continue + # append file if series already in dictionary if (series_name is not None and series_name in file_name @@ -1083,7 +1109,8 @@ def _read_config(self, yaml_file, available_channels): def _create_output_directory(self, base_path, facility, - restricted=False): + restricted=False, + calib=False): """ Create output directory @@ -1117,6 +1144,9 @@ def _create_output_directory(self, base_path, facility, prefix = self._processing_id + '_feature' if restricted: prefix += '_restricted' + elif calib: + prefix += '_calib' + output_dir = base_path + '/' + prefix + '_' + series_name diff --git a/detprocess/process/processing_data.py b/detprocess/process/processing_data.py index 58cd811..e2571d9 100644 --- a/detprocess/process/processing_data.py +++ b/detprocess/process/processing_data.py @@ -751,8 +751,8 @@ def get_event_admin(self, return_all=False): admin_dict['series_number'] = np.int64(self._current_admin_info['series_num']) admin_dict['event_id'] = np.int32(self._current_admin_info['event_id']) admin_dict['event_time'] = np.int64(self._current_admin_info['event_time']) - admin_dict['run_type'] = self._current_admin_info['run_type'] - admin_dict['data_type'] = self._current_admin_info['run_type'] + admin_dict['run_type'] = np.str(self._current_admin_info['run_type']) + admin_dict['data_type'] = np.str(self._current_admin_info['run_type']) # group name if self._group_name is not None: @@ -814,7 +814,6 @@ def get_event_admin(self, return_all=False): else: admin_dict['group_start_time'] = np.nan - return admin_dict diff --git a/detprocess/process/randoms.py b/detprocess/process/randoms.py index 90b434e..6f40f1e 100644 --- a/detprocess/process/randoms.py +++ b/detprocess/process/randoms.py @@ -32,6 +32,7 @@ class Randoms: def __init__(self, raw_path, series=None, processing_id=None, restricted=False, + calib=False, verbose=True): """ Initialize randoms acquisition @@ -47,10 +48,17 @@ def __init__(self, raw_path, series=None, series : str or list of str, optional series to be process, disregard other data from raw_path + processing_id : str, optional + + restricted : boolean if True, use restricted data if False (default), exclude restricted data - + + calib : boolean + if True, use only "calib" files + if False, no calib files included + verbose : bool, optional if True, display info @@ -69,12 +77,18 @@ def __init__(self, raw_path, series=None, # restricted self._restricted = restricted + # calibration + self._calib = calib + if calib: + self._restricted = False + # extract input file list input_file_dict, input_base_path, input_group_name, facility = ( self._get_file_list(raw_path, series=series, - restricted=restricted) + restricted=restricted, + calib=calib) ) - + if not input_file_dict: raise ValueError('No files were found! Check configuration...') @@ -83,8 +97,6 @@ def __init__(self, raw_path, series=None, self._series_dict = input_file_dict self._facility = facility - - # initialize output path self._output_group_path = None @@ -208,7 +220,8 @@ def process(self, random_rate=None, save_path, self._facility, output_group_name=output_group_name, - restricted=self._restricted + restricted=self._restricted, + calib=self._calib ) ) @@ -324,6 +337,8 @@ def _process(self, node_num, if self._restricted: file_prefix += '_restricted' + elif self._calib: + file_prefix += '_calib' series_name = h5io.extract_series_name( int(output_series_num+node_num) @@ -530,7 +545,7 @@ def _process(self, node_num, feature_dict['trigger_index'].append(int(trigger_index)) feature_dict['trigger_time'].append(trigger_time) feature_dict['trigger_type'].append(3) - feature_dict['data_type'].append((metadata['run_type'])) + feature_dict['data_type'].append(metadata['run_type']) feature_dict['fridge_run_number'].append(int(metadata['fridge_run'])) feature_dict['trigger_prod_id'].append(trigger_id) feature_dict['trigger_prod_group_name'].append(trigger_prod_group_name) @@ -626,7 +641,7 @@ def _split_series(self, ncores): def _create_output_directory(self, base_path, facility, output_group_name=None, - restricted=False): + restricted=False, calib=False): """ Create output directory @@ -642,6 +657,9 @@ def _create_output_directory(self, base_path, facility, restricted : boolean if True, create directory name that includes "restricted" + calib : boolean + if True, create directory name that includes "calib" + Return ------ output_dir : str @@ -665,6 +683,9 @@ def _create_output_directory(self, base_path, facility, prefix = self._processing_id + '_trigger' if restricted: prefix += '_restricted' + elif calib: + prefix += '_calib' + output_dir = base_path + '/' + prefix + '_' + series_name else: if output_group_name not in base_path: @@ -684,7 +705,8 @@ def _create_output_directory(self, base_path, facility, def _get_file_list(self, file_path, series=None, - restricted=False): + restricted=False, + calib=False): """ Get file list from path. Return as a dictionary with key=series and value=list of files @@ -705,6 +727,10 @@ def _get_file_list(self, file_path, series=None, if True, use restricted data if False, exclude restricted data + calib : boolean + if True, use only "calib" files + if False, no calib files included + Return ------- @@ -805,32 +831,42 @@ def _get_file_list(self, file_path, series=None, if 'filter' in file_name: continue - # restricted - if (restricted - and 'restricted' not in file_name): - continue - - # not restricted - if (not restricted - and 'restricted' in file_name): - continue - - # skip iv - if 'iv_' in file_name: - continue - - # skip didv / external trigger + # skip didv and iv if ('didv_' in file_name + or 'iv_' in file_name or 'exttrig_' in file_name): continue - + # skip if trigger data already if ('thresh_' in file_name or 'threshtrig_' in file_name): continue + # calibration + if (calib + and 'calib_' not in file_name): + continue + + # not calibration + if not calib: + + if 'calib_' in file_name: + continue + + # restricted + if (restricted + and 'restricted' not in file_name): + continue + + # not restricted + if (not restricted + and 'restricted' in file_name): + continue + + # case unrecognized - if ('cont_' not in file_name + if ('calib_' not in file_name + and 'cont_' not in file_name and 'rand_' not in file_name): # unknown file -> check trigger type diff --git a/detprocess/process/triggers.py b/detprocess/process/triggers.py index a11af81..87b8f42 100644 --- a/detprocess/process/triggers.py +++ b/detprocess/process/triggers.py @@ -45,6 +45,7 @@ def __init__(self, raw_path, config_file, series=None, processing_id=None, restricted=False, + calib=False, verbose=True): """ Intialize data processing @@ -63,9 +64,6 @@ def __init__(self, raw_path, config_file, series : str or list of str, optional series to be process, disregard other data from raw_path - - output_path : str, optional (default=No saved file) - base directory where output feature file will be saved processing_id : str, optional an optional processing name. This is used to be build @@ -78,6 +76,10 @@ def __init__(self, raw_path, config_file, if True, use restricted data if False (default), exclude restricted data + calib : boolean + if True, use only "calib" files + if False, no calib files included + verbose : bool, optional if True, display info @@ -96,13 +98,19 @@ def __init__(self, raw_path, config_file, # restricted self._restricted = restricted + + # calibration data + self._calib = calib + if calib: + self._restricted = False # extract input file list input_data_dict, input_base_path, group_name = ( self._get_file_list(raw_path, series=series, - restricted=restricted) + restricted=restricted, + calib=calib) ) - + if not input_data_dict: raise ValueError('No files were found! Check configuration...') @@ -223,7 +231,8 @@ def process(self, ntriggers=-1, save_path, self._processing_data_inst.get_facility(), output_group_name=output_group_name, - restricted=self._restricted + restricted=self._restricted, + calib=self._calib ) ) if self._verbose: @@ -432,6 +441,8 @@ def _process(self, node_num, if self._restricted: file_prefix += '_restricted' + elif self._calib: + file_prefix += '_calib' series_name = h5io.extract_series_name( int(output_series_num+node_num) @@ -684,7 +695,7 @@ def _process(self, node_num, def _get_file_list(self, file_path, series=None, - restricted=False): + restricted=False, calib=False): """ Get file list from path. Return as a dictionary with key=series and value=list of files @@ -704,7 +715,9 @@ def _get_file_list(self, file_path, series=None, if True, use restricted data if False, exclude restricted data - + calib : boolean + if True, use only "calib" files + if False, no calib files included Return ------- @@ -790,7 +803,7 @@ def _get_file_list(self, file_path, series=None, # sort file_list.sort() - # get list of series + # get list of files series_dict = dict() h5reader = h5io.H5Reader() series_name = None @@ -804,24 +817,34 @@ def _get_file_list(self, file_path, series=None, continue # skip didv and iv - if 'didv_' in file_name: - continue - - if 'iv_' in file_name: - continue - # restricted - if (restricted - and 'restricted' not in file_name): + if ('didv_' in file_name + or 'iv_' in file_name): continue - # not restricted - if (not restricted - and 'restricted' in file_name): + # calibration + if (calib + and 'calib_' not in file_name): continue - # now check if continuous data + # not calibration + if not calib: + + if 'calib_' in file_name: + continue + + # restricted + if (restricted + and 'restricted' not in file_name): + continue + + # not restricted + if (not restricted + and 'restricted' in file_name): + continue + # case unrecognized - if 'cont_' not in file_name: + if ('calib_' not in file_name + and 'cont_' not in file_name): # unknown file -> check trigger type # check trigger type of first event @@ -1005,7 +1028,8 @@ def _read_config(self, yaml_file, available_channels): def _create_output_directory(self, base_path, facility, output_group_name=None, - restricted=False): + restricted=False, + calib=False): """ Create output directory @@ -1046,6 +1070,9 @@ def _create_output_directory(self, base_path, facility, prefix = self._processing_id + '_trigger' if restricted: prefix += '_restricted' + elif calib: + prefix += '_calib' + output_dir = base_path + '/' + prefix + '_' + series_name else: if output_group_name not in base_path: diff --git a/scripts/process.py b/scripts/process.py index 665a81d..b55f2d9 100644 --- a/scripts/process.py +++ b/scripts/process.py @@ -32,7 +32,7 @@ action='store_true', help='Process IV sweep data') parser.add_argument('--enable-trig', '--enable-triggers', '--enable_trig', dest='enable_trig', - action='store_true', help='Acquire randoms') + action='store_true', help='Acquire triggers') parser.add_argument('--trigger_dataframe_path', dest='trigger_dataframe_path', type=str, help='Path to trigger dataframe (threshold and/or randoms)') @@ -64,7 +64,12 @@ parser.add_argument('--restricted', action='store_true', - help=('Processing restricted (blinded) data')) + help=('Process restricted (blinded) data')) + + parser.add_argument('--calib', + action='store_true', + help=('Processing calibration data')) + args = parser.parse_args() @@ -105,12 +110,14 @@ 'type of processing (trigger or feature processing)') exit() - - restricted = False if args.restricted: restricted = True + calib = False + if args.calib: + calib = True + restricted = False # processing id processing_id = None @@ -145,8 +152,6 @@ save_path = None if args.save_path: save_path = str(args.save_path) - - # ------------------ # check setup file @@ -206,6 +211,7 @@ processing_id=processing_id, series=series, restricted=restricted, + calib=calib, verbose=True) myproc.process(random_rate=random_rate, @@ -240,12 +246,13 @@ if args.ntriggers: ntriggers = int(args.ntriggers) - + print(calib) myproc = TriggerProcessing(args.input_group_path, processing_setup, series=series, processing_id=processing_id, restricted=restricted, + calib=calib, verbose=True) myproc.process(ntriggers=ntriggers, @@ -276,7 +283,8 @@ trigger_dataframe_path=dataframe_path, external_file=None, processing_id=processing_id, - restricted=restricted) + restricted=restricted, + calib=calib) myproc.process(nevents=-1, lgc_save=True,