Yaml configuration

detprocess/core/__init__.py

@@ -8,3 +8,4 @@
 from .didv  import *
 from .noisemodel  import *
 from .salting import *
+from .rawdata import *

detprocess/core/algorithms.py

@@ -750,7 +750,7 @@ def psd_amp(channel, of_base,
         trace_fft = of_base.signal_fft(channel)
 
         # sample rate
-        fs = 2*np.max(np.abs(freqs))
+        fs = utils.estimate_sampling_rate(freqs)
         if 'fs' in kwargs:
             fs = kwargs['fs']
 

detprocess/core/eventbuilder.py

@@ -4,8 +4,11 @@
 import pandas as pd
 import warnings
 warnings.filterwarnings('ignore')
+import pyarrow as pa
 
-
+vx.settings.main.thread_count = 1
+vx.settings.main.thread_count_io = 1
+pa.set_cpu_count(1)
 
 class EventBuilder:
     """
@@ -93,7 +96,7 @@ def add_trigger_data(self, trigger_name, trigger_data):
         Add trigger data dictionary for a specific
         trigger channel
         """
-
+        
         # intialize if needed
         if self._trigger_names is None:
             self._trigger_names = list()
@@ -127,7 +130,7 @@ def acquire_triggers(self, trigger_name, trace, thresh,
         """
         calc
         """
-
+        
         # find trigger object
         if trigger_name not in self._trigger_objects.keys():
             raise ValueError(
@@ -167,12 +170,13 @@ def build_event(self, event_metadata=None,
                     fs=None,
                     coincident_window_msec=None,
                     coincident_window_samples=None,
+                    nb_trigger_channels=None,
                     trace_length_continuous_sec=None):
         """
         Function to merge coincident 
         events based on user defined window (in msec or samples)
         """
-
+        
         # metadata
         if event_metadata is None:
             event_metadata = dict()
@@ -222,57 +226,82 @@ def build_event(self, event_metadata=None,
             return
 
         # merge coincident events
-        self._merge_coincident_triggers(
-            fs=fs,
-            coincident_window_msec=coincident_window_msec,
-            coincident_window_samples=coincident_window_samples)
-
+        if (nb_trigger_channels is None
+            or nb_trigger_channels > 1):
+            self._merge_coincident_triggers(
+                fs=fs,
+                coincident_window_msec=coincident_window_msec,
+                coincident_window_samples=coincident_window_samples
+            )
+
 
         # number of triggers (after merging coincident events)
         nb_triggers = len(self._event_df)
 
-        #  add metadata
-        default_val = np.array([np.nan]*nb_triggers)
+        # Add string column in dataframe
+        default_val_string = pa.array([None]*nb_triggers, type=pa.string())
+        metadata_string_dict = {'processing_id':default_val_string,
+                                'data_type': default_val_string,
+                                'group_name': default_val_string}
+
+        # replace value if available
+        for key in metadata_string_dict.keys():
+            if key in event_metadata.keys():
+                key = str(key)
+                val = str(event_metadata[key])
+                if '\0' in val:
+                    val = val.replace('\0', '')  # Remove all null characters
+                metadata_string_dict[key] = pa.array([val]*nb_triggers,
+                                                     type=pa.string())
+
+        # string that have name change
+        if 'run_type' in event_metadata.keys():
+            val = str(event_metadata['run_type'])
+            if '\0' in val:
+                val = val.replace('\0', '')
+            metadata_string_dict['data_type'] = pa.array([val]*nb_triggers,
+                                                  type=pa.string())
+        # add to dataframe
+        for key, val in metadata_string_dict.items():
+            self._event_df[key] = val
+
+        # integer parameters
+
+        default_val  = np.array([-1]*nb_triggers, dtype=np.int64)
         metadata_dict = {'series_number': default_val,
                          'event_number': default_val,
                          'dump_number': default_val,
                          'series_start_time': default_val,
                          'group_start_time': default_val,
                          'fridge_run_start_time': default_val,
-                         'fridge_run_number': default_val,
-                         'data_type': default_val,
-                         'group_name':default_val}
+                         'fridge_run_number': default_val}
+
 
         # replace value if available
         for key in metadata_dict.keys():
             if key in event_metadata.keys():
                 metadata_dict[key] = np.array(
-                    [event_metadata[key]]*nb_triggers)
+                    [np.int64(event_metadata[key])]*nb_triggers
+                )
 
         # some parameters have different names
         if 'series_num' in event_metadata.keys():
             metadata_dict['series_number'] = np.array(
-                [event_metadata['series_num']]*nb_triggers).astype(int)    
+                [np.int64(event_metadata['series_num'])]*nb_triggers)
         if 'event_num' in event_metadata.keys():
             metadata_dict['event_number'] = np.array(
-                [event_metadata['event_num']]*nb_triggers).astype(int)    
+                [np.int64(event_metadata['event_num'])]*nb_triggers)
         if 'dump_num' in event_metadata.keys():
             metadata_dict['dump_number'] = np.array(
-                [event_metadata['dump_num']]*nb_triggers).astype(int)   
-        if 'run_type' in event_metadata.keys():
-            metadata_dict['data_type'] = np.array(
-                [event_metadata['run_type']]*nb_triggers).astype(str)
-        elif 'data_type' in event_metadata.keys():
-            metadata_dict['data_type'] = np.array(
-                [event_metadata['data_type']]*nb_triggers).astype(str)
+                [np.int64(event_metadata['dump_num'])]*nb_triggers)
         if 'fridge_run' in event_metadata.keys():
             metadata_dict['fridge_run_number'] = np.array(
-                [event_metadata['fridge_run']]*nb_triggers).astype(int)
+                [np.int64(event_metadata['fridge_run'])]*nb_triggers)
 
         # event times
         trigger_times = self._event_df['trigger_time'].values
         event_times = trigger_times + event_time_start
-        event_times_int = np.around(event_times).astype(int)
+        event_times_int = np.int64(np.around(event_times))
 
         # add new parameters in dictionary
         metadata_dict['event_time'] = event_times_int
@@ -286,7 +315,7 @@ def build_event(self, event_metadata=None,
         # trigger id
         metadata_dict['trigger_prod_id'] = (
             np.array(range(nb_triggers))
-            + int(self._current_trigger_id)
+            + np.int64(self._current_trigger_id)
             + 1)
 
         self._current_trigger_id = metadata_dict['trigger_prod_id'][-1]
@@ -295,15 +324,15 @@ def build_event(self, event_metadata=None,
         for key, val in metadata_dict.items():
             self._event_df[key] = val
 
-
+                
     def _merge_coincident_triggers(self, fs=None,
                                   coincident_window_msec=None,
                                   coincident_window_samples=None):
         """
         Function to merge coincident 
         events based on user defined window (in msec or samples)
         """
-
+                      
         # check
         if (self._event_df is None or len(self._event_df)==0):
             raise ValueError('ERROR: No trigger data '
@@ -321,6 +350,9 @@ def _merge_coincident_triggers(self, fs=None,
             merge_window = coincident_window_samples
 
 
+        if  merge_window == 0:
+            return
+
         # let's convert vaex dataframe to pandas so we can modify it
         # more easily
         df_pandas = self._event_df.to_pandas_df()
@@ -456,15 +488,3 @@ def _merge_coincident_triggers(self, fs=None,
         # convert back to vaex
         self._event_df = vx.from_pandas(df_pandas, copy_index=False)
 
-
-
-
-
-
-
-
-
-
-
-
-

detprocess/core/filterdata.py

@@ -9,7 +9,7 @@
 import qetpy.plotting as plotting
 from qetpy import calc_corrcoeff_from_csd
 import copy
-
+from detprocess.utils import estimate_sampling_rate
 
 class FilterData:
     """
@@ -223,6 +223,18 @@ def load_hdf5(self, file_name, overwrite=True):
             print('INFO: Loading filter data from file '
                   + file_name)
 
+        # update
+        self.set_data(data, overwrite=overwrite)
+
+    def set_data(self, data, overwrite=False):
+        """
+        Set data directly
+        """
+
+        if not isinstance(data, dict):
+            raise ValueError('ERROR: filter data should be a '
+                             'dictionary!')
+
         # update
         if overwrite or not self._filter_data:
             self._filter_data.update(data)
@@ -236,7 +248,7 @@ def load_hdf5(self, file_name, overwrite=True):
                         self._filter_data[key][par_name] = (
                             data[key][par_name]
                         )
-
+                        
 
 
     def save_hdf5(self, file_name, overwrite=False):
@@ -316,7 +328,7 @@ def get_psd(self, channels, tag='default',
                 if 'sample_rate' in metadata:
                     sample_rate = float(metadata['sample_rate'])
                 else:
-                    sample_rate = 2*np.max(np.abs(psd_freqs))
+                    sample_rate = estimate_sampling_rate(psd_freqs)
 
                 psd_freqs, psd = fold_spectrum(psd, sample_rate)
 
@@ -382,11 +394,11 @@ def get_csd(self, channels, tag='default',
         channel_name = convert_channel_list_to_name(channels)
         nb_channels = len(channel_list)
 
-        if nb_channels < 2:
-            raise ValueError(
-                'ERROR: At least 2 channels required to calculate csd'
-            )
-
+        if nb_channels == 1:
+            return self.get_psd(channel_name, tag=tag,
+                                fold=fold,
+                                return_metadata=return_metadata)
+                
         # get values
         output_metadata = dict()
         csd, csd_freqs, metadata = (
@@ -405,16 +417,16 @@ def get_csd(self, channels, tag='default',
             if 'sample_rate' in metadata:
                 sample_rate = float(metadata['sample_rate'])
             else:
-                sample_rate = 2*np.max(np.abs(csd_freqs))
+                sample_rate = estimate_sampling_rate(csd_freqs)
 
             csd_freqs, csd = fold_spectrum(csd, sample_rate)
 
         if return_metadata:
             return csd, csd_freqs, output_metadata
         else:
             return csd, csd_freqs
-            
-
+
+      
     def get_template(self, channel, tag='default',
                      return_metadata=False):
         """
@@ -599,7 +611,7 @@ def set_psd(self, channels, psd, psd_freqs,
                              'for "psd_freqs" argument')
 
         # add dimension if needed
-        if psd_freqs.ndim == 1:
+        if psd_freqs.ndim == 2:
             psd_freqs = psd_freqs[np.newaxis, :]
 
         # check if folded -> NOT ALLOWED
@@ -608,10 +620,10 @@ def set_psd(self, channels, psd, psd_freqs,
             raise ValueError('ERROR: psd needs to be two-sided!')
 
 
-        sample_rate_array = 2*np.max(np.abs(psd_freqs))
+        sample_rate_array = estimate_sampling_rate(psd_freqs[0,:])
         if sample_rate is None:
             sample_rate = sample_rate_array
-        elif sample_rate_array != sample_rate:
+        elif round(sample_rate_array) != round(sample_rate):
             raise ValueError('ERROR: sample_rate is inconsistent with '
                              'frequency array!')
 
@@ -729,10 +741,10 @@ def set_csd(self, channels, csd, csd_freqs,
             raise ValueError('ERROR: psd needs to be two-sided!')
 
 
-        sample_rate_array = 2*np.max(np.abs(csd_freqs))
+        sample_rate_array = estimate_sampling_rate(csd_freqs)
         if sample_rate is None:
             sample_rate = sample_rate_array
-        elif sample_rate_array != sample_rate:
+        elif round(sample_rate_array) != round(sample_rate):
             raise ValueError('ERROR: sample_rate is inconsistent with '
                              'frequency array!')
 
@@ -798,10 +810,10 @@ def set_dpdi(self, channels,
         if is_folded:
             raise ValueError('ERROR: dpdi needs to be two-sided!')
 
-        sample_rate_array = 2*np.max(np.abs(dpdi_freqs))
+        sample_rate_array = estimate_sampling_rate(dpdi_freqs[0,:])
         if sample_rate is None:
             sample_rate = sample_rate_array
-        elif sample_rate_array != sample_rate:
+        elif round(sample_rate_array) != round(sample_rate):
             raise ValueError(f'ERROR: sample rate ({sample_rate}) '
                              f'is inconsistent with sample rate from '
                              f'frequency array ({sample_rate_array})!')