From a38985a1216fe581a4e672ee5ac47531fc89c32a Mon Sep 17 00:00:00 2001 From: alessandrofelder Date: Thu, 3 Dec 2020 18:50:14 +0000 Subject: [PATCH 01/10] wip:bringing subsample up-to-date --- src/silverlabnwb/subsample_nwb.py | 35 +++++++++++++------------------ 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/src/silverlabnwb/subsample_nwb.py b/src/silverlabnwb/subsample_nwb.py index 040c637..d9bf34d 100644 --- a/src/silverlabnwb/subsample_nwb.py +++ b/src/silverlabnwb/subsample_nwb.py @@ -37,11 +37,10 @@ def subsample_nwb(nwb, input_path, output_path, ntrials=2, nrois=10): input_path, output_path, ntrials, nrois, orig_nrois) # Figure out time duration for given ntrials - last_trial = nwb['/epochs/trial_{:04d}'.format(ntrials)] - end_time = last_trial['stop_time'].value + end_time = nwb['/intervals/epochs/']['stop_time'][ntrials] print('Trial {} ends at {}'.format(ntrials, end_time)) # Copy truncated speed data - copy_speed_data(input_path, output_path, last_trial) + copy_speed_data(input_path, output_path, nwb['/intervals/epochs/']['timeseries'][ntrials]) # Figure out which Zstack planes have ROIs zstack_planes = find_used_planes(nwb, nrois) # Copy pockels file @@ -77,7 +76,7 @@ def find_used_planes(nwb, nrois): used_planes = set() seg_iface = nwb['/processing/Acquired_ROIs/ImageSegmentation'] for plane_name in seg_iface.keys(): - plane_num = int(plane_name[-4:]) + plane_num = int(plane_name[6:10]) for roi_name in seg_iface[plane_name].keys(): if roi_name.startswith('ROI_'): roi_num = int(roi_name[4:]) @@ -103,13 +102,13 @@ def copy_zstack(input_path, output_path, zstack_planes): im.save(dest, format='TIFF', compression='tiff_lzw') -def copy_speed_data(input_path, output_path, last_trial): +def copy_speed_data(input_path, output_path, last_trial_speed_data): fname = 'Speed_Data/Speed data 001.txt' os.makedirs(os.path.join(output_path, 'Speed_Data'), exist_ok=True) src = os.path.join(input_path, fname) dest = os.path.join(output_path, fname) - speed_data_ts = last_trial['speed_data'] - end_index = speed_data_ts['idx_start'].value + speed_data_ts['count'].value + speed_data_ts = last_trial_speed_data + end_index = speed_data_ts['idx_start'] + speed_data_ts['count'] copy_and_truncate(src, dest, end_index + 1) @@ -185,32 +184,26 @@ def copy_and_truncate(src, dest, nlines): def cycles_per_trial(nwb): """Get the number of microscope cycles/trial. + #TODO: FIXME this function currently only works for pointing data! That is, the number of times each point is imaged in each trial. Currently looks at the first imaging timeseries in the first trial, and assumes they're all the same. """ - trial1 = nwb['/epochs/trial_0001'] - for ts_name in trial1: - ts = trial1[ts_name] - is_image_series = ts['timeseries/pixel_time_offsets'] is not None - if is_image_series: - return ts['count'].value - else: - raise ValueError('No imaging timeseries found') + n_all_trials = len(nwb['/intervals/trials/id']) + return np.int(nwb['acquisition/ROI_001_Red/timestamps'].shape[0] / n_all_trials) def copy_tdms(nwb, in_path, out_path, nrois): - num_all_rois = nwb['/processing/Acquired_ROIs/roi_spec'].shape[0] + num_all_rois = int(len(list(nwb['/processing/Acquired_ROIs/ImageSegmentation'].keys()))/2) # divide by n_channels print('Copying {} of {} ROIs from {} to {}'.format( nrois, num_all_rois, in_path, out_path)) in_tdms = nptdms.TdmsFile(in_path) - group_name = 'Functional Imaging Data' + group_name = "'Functional Imaging Data'" with nptdms.TdmsWriter(out_path) as out_tdms: - root, group = in_tdms.object(), in_tdms.object(group_name) - out_tdms.write_segment([root, group]) + root, group = in_tdms.objects['/'], in_tdms.objects['/'+group_name] for ch, channel in {'0': 'Red', '1': 'Green'}.items(): - ch_name = 'Channel {} Data'.format(ch) - ch_obj = in_tdms.object(group_name, ch_name) + ch_name = "'Channel {} Data'".format(ch) + ch_obj = in_tdms.objects['/'+group_name+'/'+ch_name] shape = (cycles_per_trial(nwb), num_all_rois, -1) ch_data = ch_obj.data.reshape(shape) subset = ch_data[:, :nrois, :].reshape(-1) From 022422f45d31965a3b99df5fcf3d1098fb463930 Mon Sep 17 00:00:00 2001 From: alessandrofelder Date: Fri, 4 Dec 2020 12:42:56 +0000 Subject: [PATCH 02/10] wip: fix speed data index, fix tdms group names --- src/silverlabnwb/subsample_nwb.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/silverlabnwb/subsample_nwb.py b/src/silverlabnwb/subsample_nwb.py index d9bf34d..d121294 100644 --- a/src/silverlabnwb/subsample_nwb.py +++ b/src/silverlabnwb/subsample_nwb.py @@ -40,7 +40,7 @@ def subsample_nwb(nwb, input_path, output_path, ntrials=2, nrois=10): end_time = nwb['/intervals/epochs/']['stop_time'][ntrials] print('Trial {} ends at {}'.format(ntrials, end_time)) # Copy truncated speed data - copy_speed_data(input_path, output_path, nwb['/intervals/epochs/']['timeseries'][ntrials]) + copy_speed_data(input_path, output_path, nwb['/intervals/epochs/']['timeseries'][ntrials-1]) # Figure out which Zstack planes have ROIs zstack_planes = find_used_planes(nwb, nrois) # Copy pockels file @@ -56,7 +56,7 @@ def subsample_nwb(nwb, input_path, output_path, ntrials=2, nrois=10): tdms_out = os.path.join(output_path, tdms_path, tdms_name) copy_tdms(nwb, tdms_in, tdms_out, nrois) # Find videos defined - video_names = [name for name in nwb['/acquisition/timeseries'].keys() + video_names = [name for name in nwb['/acquisition'].keys() if name.endswith('Cam')] print('Videos:', video_names) # Compress videos @@ -194,7 +194,7 @@ def cycles_per_trial(nwb): def copy_tdms(nwb, in_path, out_path, nrois): - num_all_rois = int(len(list(nwb['/processing/Acquired_ROIs/ImageSegmentation'].keys()))/2) # divide by n_channels + num_all_rois = len([key for key in nwb['/acquisition/'].keys() if key.startswith('ROI') and key.endswith('Red')]) print('Copying {} of {} ROIs from {} to {}'.format( nrois, num_all_rois, in_path, out_path)) in_tdms = nptdms.TdmsFile(in_path) @@ -207,7 +207,7 @@ def copy_tdms(nwb, in_path, out_path, nrois): shape = (cycles_per_trial(nwb), num_all_rois, -1) ch_data = ch_obj.data.reshape(shape) subset = ch_data[:, :nrois, :].reshape(-1) - new_obj = nptdms.ChannelObject(group_name, ch_name, subset, properties={}) + new_obj = nptdms.ChannelObject(group_name.split('\'')[1], ch_name.split('\'')[1], subset, properties={}) out_tdms.write_segment([new_obj]) From 3744ca87a72afb02307dd9eb0815a27ff261fe27 Mon Sep 17 00:00:00 2001 From: alessandrofelder Date: Fri, 4 Dec 2020 13:11:06 +0000 Subject: [PATCH 03/10] cosmetics of accessing tdms files --- src/silverlabnwb/subsample_nwb.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/silverlabnwb/subsample_nwb.py b/src/silverlabnwb/subsample_nwb.py index d121294..fd86225 100644 --- a/src/silverlabnwb/subsample_nwb.py +++ b/src/silverlabnwb/subsample_nwb.py @@ -198,16 +198,17 @@ def copy_tdms(nwb, in_path, out_path, nrois): print('Copying {} of {} ROIs from {} to {}'.format( nrois, num_all_rois, in_path, out_path)) in_tdms = nptdms.TdmsFile(in_path) - group_name = "'Functional Imaging Data'" + group_name = "Functional Imaging Data" with nptdms.TdmsWriter(out_path) as out_tdms: - root, group = in_tdms.objects['/'], in_tdms.objects['/'+group_name] + group = in_tdms[group_name] + out_tdms.write_segment([group]) for ch, channel in {'0': 'Red', '1': 'Green'}.items(): - ch_name = "'Channel {} Data'".format(ch) - ch_obj = in_tdms.objects['/'+group_name+'/'+ch_name] + ch_name = "Channel {} Data".format(ch) + ch_obj = in_tdms[group_name][ch_name] shape = (cycles_per_trial(nwb), num_all_rois, -1) ch_data = ch_obj.data.reshape(shape) subset = ch_data[:, :nrois, :].reshape(-1) - new_obj = nptdms.ChannelObject(group_name.split('\'')[1], ch_name.split('\'')[1], subset, properties={}) + new_obj = nptdms.ChannelObject(group_name, ch_name, subset, properties={}) out_tdms.write_segment([new_obj]) From addf8e3640e0aafeb1a0622a0fe06ec167660599 Mon Sep 17 00:00:00 2001 From: alessandrofelder Date: Fri, 4 Dec 2020 16:17:32 +0000 Subject: [PATCH 04/10] fix index, revert change from single to double quotes --- src/silverlabnwb/subsample_nwb.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/silverlabnwb/subsample_nwb.py b/src/silverlabnwb/subsample_nwb.py index fd86225..6a809cd 100644 --- a/src/silverlabnwb/subsample_nwb.py +++ b/src/silverlabnwb/subsample_nwb.py @@ -37,7 +37,7 @@ def subsample_nwb(nwb, input_path, output_path, ntrials=2, nrois=10): input_path, output_path, ntrials, nrois, orig_nrois) # Figure out time duration for given ntrials - end_time = nwb['/intervals/epochs/']['stop_time'][ntrials] + end_time = nwb['/intervals/epochs/']['stop_time'][ntrials-1] print('Trial {} ends at {}'.format(ntrials, end_time)) # Copy truncated speed data copy_speed_data(input_path, output_path, nwb['/intervals/epochs/']['timeseries'][ntrials-1]) @@ -198,12 +198,12 @@ def copy_tdms(nwb, in_path, out_path, nrois): print('Copying {} of {} ROIs from {} to {}'.format( nrois, num_all_rois, in_path, out_path)) in_tdms = nptdms.TdmsFile(in_path) - group_name = "Functional Imaging Data" + group_name = 'Functional Imaging Data' with nptdms.TdmsWriter(out_path) as out_tdms: group = in_tdms[group_name] out_tdms.write_segment([group]) for ch, channel in {'0': 'Red', '1': 'Green'}.items(): - ch_name = "Channel {} Data".format(ch) + ch_name = 'Channel {} Data'.format(ch) ch_obj = in_tdms[group_name][ch_name] shape = (cycles_per_trial(nwb), num_all_rois, -1) ch_data = ch_obj.data.reshape(shape) From fbaf4f01f94b81df3d99ee86ef48231b74ee9cd3 Mon Sep 17 00:00:00 2001 From: alessandrofelder Date: Fri, 4 Dec 2020 17:02:40 +0000 Subject: [PATCH 05/10] remove unnecessary FIXME cycles per trial independent of whether scan is patch or pointing I've tested that new version of code give the same result as earlier for sample_miniscan_fred.... --- src/silverlabnwb/subsample_nwb.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/silverlabnwb/subsample_nwb.py b/src/silverlabnwb/subsample_nwb.py index 6a809cd..f508842 100644 --- a/src/silverlabnwb/subsample_nwb.py +++ b/src/silverlabnwb/subsample_nwb.py @@ -184,7 +184,6 @@ def copy_and_truncate(src, dest, nlines): def cycles_per_trial(nwb): """Get the number of microscope cycles/trial. - #TODO: FIXME this function currently only works for pointing data! That is, the number of times each point is imaged in each trial. Currently looks at the first imaging timeseries in the first trial, and assumes they're all the same. From eff30a36f60d755ce7ecbfdb67d3af2c41036703 Mon Sep 17 00:00:00 2001 From: alessandrofelder Date: Fri, 11 Dec 2020 11:02:30 +0000 Subject: [PATCH 06/10] wip: getting copy_tdms to work with variable rois --- src/silverlabnwb/subsample_nwb.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/silverlabnwb/subsample_nwb.py b/src/silverlabnwb/subsample_nwb.py index f508842..d1ab4b7 100644 --- a/src/silverlabnwb/subsample_nwb.py +++ b/src/silverlabnwb/subsample_nwb.py @@ -193,7 +193,9 @@ def cycles_per_trial(nwb): def copy_tdms(nwb, in_path, out_path, nrois): - num_all_rois = len([key for key in nwb['/acquisition/'].keys() if key.startswith('ROI') and key.endswith('Red')]) + all_rois = [nwb['/acquisition/'+key] for key in nwb['/acquisition/'].keys() if key.startswith('ROI') and key.endswith('Red')] + num_all_rois = len(all_rois) + all_roi_dimensions_pixels = np.array([roi['dimension'] for roi in all_rois]) print('Copying {} of {} ROIs from {} to {}'.format( nrois, num_all_rois, in_path, out_path)) in_tdms = nptdms.TdmsFile(in_path) @@ -204,9 +206,19 @@ def copy_tdms(nwb, in_path, out_path, nrois): for ch, channel in {'0': 'Red', '1': 'Green'}.items(): ch_name = 'Channel {} Data'.format(ch) ch_obj = in_tdms[group_name][ch_name] - shape = (cycles_per_trial(nwb), num_all_rois, -1) - ch_data = ch_obj.data.reshape(shape) - subset = ch_data[:, :nrois, :].reshape(-1) + # The number of pixels for each ROI for one cycle, and for all ROIs + all_roi_pixels = all_roi_dimensions_pixels.prod(axis=1) + total_pixels = all_roi_pixels.sum() + # How many pixels do the previous ROIs take up within a cycle's data? + # We'll need this to see where to start reading for a particular ROI. + previous_pixels = np.concatenate(([0], all_roi_pixels[:-1].cumsum())) + subset = np.array(()) + for roi_index, roi_to_keep in enumerate(all_rois[0:nrois]): + starts = np.arange(cycles_per_trial(nwb)) * total_pixels + previous_pixels[roi_index] + stops = starts + all_roi_pixels[roi_index] + inds = np.array([np.arange(start, stop) for (start, stop) in zip(starts, stops)]) + ch_data = ch_obj.data[inds].flatten() + subset = np.concatenate((subset, ch_data)) new_obj = nptdms.ChannelObject(group_name, ch_name, subset, properties={}) out_tdms.write_segment([new_obj]) From 3232fbdc4ce91e173cc7c5f2ac82d4707f833adf Mon Sep 17 00:00:00 2001 From: Anastasis Georgoulas Date: Sun, 13 Dec 2020 18:20:35 +0000 Subject: [PATCH 07/10] Put the TDMS data in the right order May be wrong! Previous version had all data from the first ROI followed by the second etc, but the original TDMS stores data in complete cycles (first cycle for all ROIs, then second cycle for all ROIs etc). This is a bit less general than the previous version, which could have been easily adapted for non-consecutive ROIs. However, the whole file is based on the assumption that we are keeping only the first N ROIs, so the simplification is probably worth the loss of generality in one function... --- src/silverlabnwb/subsample_nwb.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/src/silverlabnwb/subsample_nwb.py b/src/silverlabnwb/subsample_nwb.py index d1ab4b7..c8b9107 100644 --- a/src/silverlabnwb/subsample_nwb.py +++ b/src/silverlabnwb/subsample_nwb.py @@ -209,17 +209,15 @@ def copy_tdms(nwb, in_path, out_path, nrois): # The number of pixels for each ROI for one cycle, and for all ROIs all_roi_pixels = all_roi_dimensions_pixels.prod(axis=1) total_pixels = all_roi_pixels.sum() - # How many pixels do the previous ROIs take up within a cycle's data? - # We'll need this to see where to start reading for a particular ROI. - previous_pixels = np.concatenate(([0], all_roi_pixels[:-1].cumsum())) - subset = np.array(()) - for roi_index, roi_to_keep in enumerate(all_rois[0:nrois]): - starts = np.arange(cycles_per_trial(nwb)) * total_pixels + previous_pixels[roi_index] - stops = starts + all_roi_pixels[roi_index] - inds = np.array([np.arange(start, stop) for (start, stop) in zip(starts, stops)]) - ch_data = ch_obj.data[inds].flatten() - subset = np.concatenate((subset, ch_data)) - new_obj = nptdms.ChannelObject(group_name, ch_name, subset, properties={}) + # How many pixels to keep in each cycle from the chosen ROIs + pixels_kept_per_cycle = all_roi_pixels[:nrois].sum() + # Indices of the first and last pixels we're keeping from each cycle + firsts = np.arange(cycles_per_trial(nwb)) * total_pixels + lasts = firsts + pixels_kept_per_cycle + # Get all pixels in these ranges and store them in a 1d array + inds = np.array([np.arange(first, last) for (first, last) in zip(firsts, lasts)]) + ch_data = ch_obj.data[inds].flatten() + new_obj = nptdms.ChannelObject(group_name, ch_name, ch_data, properties={}) out_tdms.write_segment([new_obj]) From a926212389741973ed7e615af63e4249c7dc686b Mon Sep 17 00:00:00 2001 From: Anastasis Georgoulas Date: Sun, 13 Dec 2020 18:33:41 +0000 Subject: [PATCH 08/10] Simplify how we get indices of pixels to keep --- src/silverlabnwb/subsample_nwb.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/silverlabnwb/subsample_nwb.py b/src/silverlabnwb/subsample_nwb.py index c8b9107..e8575a1 100644 --- a/src/silverlabnwb/subsample_nwb.py +++ b/src/silverlabnwb/subsample_nwb.py @@ -211,11 +211,11 @@ def copy_tdms(nwb, in_path, out_path, nrois): total_pixels = all_roi_pixels.sum() # How many pixels to keep in each cycle from the chosen ROIs pixels_kept_per_cycle = all_roi_pixels[:nrois].sum() - # Indices of the first and last pixels we're keeping from each cycle - firsts = np.arange(cycles_per_trial(nwb)) * total_pixels - lasts = firsts + pixels_kept_per_cycle + # Build the indices of the pixels we're keeping from each cycle + first_pixels = np.arange(cycles_per_trial(nwb)) * total_pixels + remaining_pixels_per_cycle = np.arange(pixels_kept_per_cycle) + inds = first_pixels[:, np.newaxis] + remaining_pixels_per_cycle # Get all pixels in these ranges and store them in a 1d array - inds = np.array([np.arange(first, last) for (first, last) in zip(firsts, lasts)]) ch_data = ch_obj.data[inds].flatten() new_obj = nptdms.ChannelObject(group_name, ch_name, ch_data, properties={}) out_tdms.write_segment([new_obj]) From c89b31d9300d6fc7cf93a2cd83c92c89ea7d1cc7 Mon Sep 17 00:00:00 2001 From: alessandrofelder Date: Mon, 14 Dec 2020 14:31:16 +0000 Subject: [PATCH 09/10] add assertion for data dimensions this ensures the dimensions listed ROI.dat and those of the TDMS data are consistent with each other. --- src/silverlabnwb/nwb_file.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/silverlabnwb/nwb_file.py b/src/silverlabnwb/nwb_file.py index ce49093..68b74ea 100644 --- a/src/silverlabnwb/nwb_file.py +++ b/src/silverlabnwb/nwb_file.py @@ -819,6 +819,7 @@ def _write_roi_data(self, all_rois, num_trials, cycles_per_trial, # Reshape the TDMS data into an nd array # TODO: Consider precision: the round() here is to match the exported data... ch_data = np.round(tdms_file['Functional Imaging Data'][f'Channel {ch} Data'].data) + assert ch_data.size == total_pixels * cycles_per_trial # Copy each ROI's data into the NWB for roi_num, data_paths in all_rois.items(): roi_shape = all_roi_dimensions_pixels[roi_num - 1, :] From 65d3b84a1bbc2f8983f340383160dfb6d14aec41 Mon Sep 17 00:00:00 2001 From: alessandrofelder Date: Thu, 7 Jan 2021 11:21:32 +0000 Subject: [PATCH 10/10] add functionality to subsample intermediate ROIs e.g. so we don't have to start from ROI 1. --- src/silverlabnwb/subsample_nwb.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/silverlabnwb/subsample_nwb.py b/src/silverlabnwb/subsample_nwb.py index e8575a1..7760132 100644 --- a/src/silverlabnwb/subsample_nwb.py +++ b/src/silverlabnwb/subsample_nwb.py @@ -192,12 +192,12 @@ def cycles_per_trial(nwb): return np.int(nwb['acquisition/ROI_001_Red/timestamps'].shape[0] / n_all_trials) -def copy_tdms(nwb, in_path, out_path, nrois): +def copy_tdms(nwb, in_path, out_path, nrois, start_roi=0): all_rois = [nwb['/acquisition/'+key] for key in nwb['/acquisition/'].keys() if key.startswith('ROI') and key.endswith('Red')] num_all_rois = len(all_rois) all_roi_dimensions_pixels = np.array([roi['dimension'] for roi in all_rois]) - print('Copying {} of {} ROIs from {} to {}'.format( - nrois, num_all_rois, in_path, out_path)) + print('Copying ROIs {} to {} out of a total of {} from {} to {}'.format( + start_roi+1, nrois, num_all_rois, in_path, out_path)) in_tdms = nptdms.TdmsFile(in_path) group_name = 'Functional Imaging Data' with nptdms.TdmsWriter(out_path) as out_tdms: @@ -210,10 +210,11 @@ def copy_tdms(nwb, in_path, out_path, nrois): all_roi_pixels = all_roi_dimensions_pixels.prod(axis=1) total_pixels = all_roi_pixels.sum() # How many pixels to keep in each cycle from the chosen ROIs - pixels_kept_per_cycle = all_roi_pixels[:nrois].sum() + pixels_kept_per_cycle = all_roi_pixels[start_roi:nrois].sum() + preceding_pixels_per_cycle = all_roi_pixels[:start_roi].sum() # Build the indices of the pixels we're keeping from each cycle first_pixels = np.arange(cycles_per_trial(nwb)) * total_pixels - remaining_pixels_per_cycle = np.arange(pixels_kept_per_cycle) + remaining_pixels_per_cycle = preceding_pixels_per_cycle + np.arange(pixels_kept_per_cycle) inds = first_pixels[:, np.newaxis] + remaining_pixels_per_cycle # Get all pixels in these ranges and store them in a 1d array ch_data = ch_obj.data[inds].flatten()