From 4a1b40e728f0e546524f62a5e399dbc72b590292 Mon Sep 17 00:00:00 2001 From: Samuel Bray Date: Mon, 10 Jun 2024 12:57:57 -0700 Subject: [PATCH 01/94] Fix bug in change in analysis_file object_id (#1004) * fix bug in change in analysis_file_object_id * update changelog --- CHANGELOG.md | 2 +- src/spyglass/common/common_nwbfile.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b898826f2..36c3523f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,7 @@ - Common - `PositionVideo` table now inserts into self after `make` #966 - - Files created by `AnalysisNwbfile.create()` receive new object_id #999 + - Files created by `AnalysisNwbfile.create()` receive new object_id #999, #1004 - Decoding: Default values for classes on `ImportError` #966 - DLC - Allow dlc without pre-existing tracking data #973, #975 diff --git a/src/spyglass/common/common_nwbfile.py b/src/spyglass/common/common_nwbfile.py index 82070d5fb..bcdc50c28 100644 --- a/src/spyglass/common/common_nwbfile.py +++ b/src/spyglass/common/common_nwbfile.py @@ -225,7 +225,7 @@ def create(self, nwb_file_name): self._alter_spyglass_version(analysis_file_abs_path) # create a new object id for the file - with h5py.File(nwb_file_abspath, "a") as f: + with h5py.File(analysis_file_abs_path, "a") as f: f.attrs["object_id"] = str(uuid4()) # change the permissions to only allow owner to write From 5d957f1cc3699fe21d63c3ce2f046a4b237ab71e Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Tue, 11 Jun 2024 12:50:44 -0500 Subject: [PATCH 02/94] Remove classes for usused tables (#1003) * #976 * Remove notebook reference --- CHANGELOG.md | 12 +- notebooks/21_DLC.ipynb | 49 ----- notebooks/50_MUA_Detection.ipynb | 12 +- notebooks/py_scripts/21_DLC.py | 23 --- src/spyglass/common/common_ephys.py | 10 - src/spyglass/position/position_merge.py | 177 ------------------ src/spyglass/spikesorting/v0/__init__.py | 1 - .../spikesorting/v0/spikesorting_curation.py | 95 ---------- 8 files changed, 15 insertions(+), 364 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 36c3523f6..15cf86478 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,18 +24,22 @@ - Common - `PositionVideo` table now inserts into self after `make` #966 - - Files created by `AnalysisNwbfile.create()` receive new object_id #999, #1004 + - Don't insert lab member when creating lab team #983 + - Files created by `AnalysisNwbfile.create()` receive new object_id #999 + - Remove unused `ElectrodeBrainRegion` table #1003 + - Files created by `AnalysisNwbfile.create()` receive new object_id #999, + #1004 - Decoding: Default values for classes on `ImportError` #966 -- DLC +- Position - Allow dlc without pre-existing tracking data #973, #975 - Raise `KeyError` for missing input parameters across helper funcs #966 - `DLCPosVideo` table now inserts into self after `make` #966 -- Common - - Don't insert lab member when creating lab team #983 + - Remove unused `PositionVideoSelection` and `PositionVideo` tables #1003 - Spikesorting - Allow user to set smoothing timescale in `SortedSpikesGroup.get_firing_rate` #994 - Update docstrings #996 + - Remove unused `UnitInclusionParameters` table from `spikesorting.v0` #1003 ## [0.5.2] (April 22, 2024) diff --git a/notebooks/21_DLC.ipynb b/notebooks/21_DLC.ipynb index ffc0d450c..b976ae5eb 100644 --- a/notebooks/21_DLC.ipynb +++ b/notebooks/21_DLC.ipynb @@ -2097,55 +2097,6 @@ "(PositionOutput.DLCPosV1() & dlc_key).fetch1_dataframe()" ] }, - { - "cell_type": "markdown", - "id": "e48c7a4e-0bbc-4101-baf2-e84f1f5739d5", - "metadata": {}, - "source": [ - "#### [PositionVideo](#TableOfContents)\n" - ] - }, - { - "cell_type": "markdown", - "id": "388e6602-8e80-47fa-be78-4ae120d52e41", - "metadata": {}, - "source": [ - "We can use the `PositionVideo` table to create a video that overlays just the\n", - "centroid and orientation on the video. This table uses the parameter `plot` to\n", - "determine whether to plot the entry deriving from the DLC arm or from the Trodes\n", - "arm of the position pipeline. This parameter also accepts 'all', which will plot\n", - "both (if they exist) in order to compare results.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b2a782ce-0a14-4725-887f-ae6f341635f8", - "metadata": {}, - "outputs": [], - "source": [ - "sgp.PositionVideoSelection().insert1(\n", - " {\n", - " \"nwb_file_name\": \"J1620210604_.nwb\",\n", - " \"interval_list_name\": \"pos 13 valid times\",\n", - " \"trodes_position_id\": 0,\n", - " \"dlc_position_id\": 1,\n", - " \"plot\": \"DLC\",\n", - " \"output_dir\": \"/home/dgramling/Src/\",\n", - " }\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c32993e7-5b32-46f9-a2f9-9634aef785f2", - "metadata": {}, - "outputs": [], - "source": [ - "sgp.PositionVideo.populate({\"plot\": \"DLC\"})" - ] - }, { "cell_type": "markdown", "id": "be097052-3789-4d55-aca1-e44d426c39b4", diff --git a/notebooks/50_MUA_Detection.ipynb b/notebooks/50_MUA_Detection.ipynb index 2c8fa8511..1da12af6a 100644 --- a/notebooks/50_MUA_Detection.ipynb +++ b/notebooks/50_MUA_Detection.ipynb @@ -96,10 +96,10 @@ "source": [ "from spyglass.position import PositionOutput\n", "\n", - "#First, select the file of interest\n", + "# First, select the file of interest\n", "nwb_copy_file_name = \"mediumnwb20230802_.nwb\"\n", "\n", - "#Then, get position data\n", + "# Then, get position data\n", "trodes_s_key = {\n", " \"nwb_file_name\": nwb_copy_file_name,\n", " \"interval_list_name\": \"pos 0 valid times\",\n", @@ -215,7 +215,7 @@ " SortedSpikesGroup,\n", ")\n", "\n", - "#Select sorted spikes data\n", + "# Select sorted spikes data\n", "sorted_spikes_group_key = {\n", " \"nwb_file_name\": nwb_copy_file_name,\n", " \"sorted_spikes_group_name\": \"test_group\",\n", @@ -880,10 +880,12 @@ "import numpy as np\n", "\n", "fig, axes = plt.subplots(2, 1, sharex=True, figsize=(15, 4))\n", - "speed = MuaEventsV1.get_speed(mua_key) #get speed from MuaEventsV1 table\n", + "speed = MuaEventsV1.get_speed(mua_key) # get speed from MuaEventsV1 table\n", "time = speed.index.to_numpy()\n", "speed = speed.to_numpy()\n", - "multiunit_firing_rate = MuaEventsV1.get_firing_rate(mua_key, time) #get firing rate from MuaEventsV1 table\n", + "multiunit_firing_rate = MuaEventsV1.get_firing_rate(\n", + " mua_key, time\n", + ") # get firing rate from MuaEventsV1 table\n", "\n", "time_slice = slice(\n", " np.searchsorted(time, mua_times.loc[10].start_time) - 1_000,\n", diff --git a/notebooks/py_scripts/21_DLC.py b/notebooks/py_scripts/21_DLC.py index 63ffe4d0c..8a55441e8 100644 --- a/notebooks/py_scripts/21_DLC.py +++ b/notebooks/py_scripts/21_DLC.py @@ -772,29 +772,6 @@ (PositionOutput.DLCPosV1() & dlc_key).fetch1_dataframe() -# #### [PositionVideo](#TableOfContents) -# - -# We can use the `PositionVideo` table to create a video that overlays just the -# centroid and orientation on the video. This table uses the parameter `plot` to -# determine whether to plot the entry deriving from the DLC arm or from the Trodes -# arm of the position pipeline. This parameter also accepts 'all', which will plot -# both (if they exist) in order to compare results. -# - -sgp.PositionVideoSelection().insert1( - { - "nwb_file_name": "J1620210604_.nwb", - "interval_list_name": "pos 13 valid times", - "trodes_position_id": 0, - "dlc_position_id": 1, - "plot": "DLC", - "output_dir": "/home/dgramling/Src/", - } -) - -sgp.PositionVideo.populate({"plot": "DLC"}) - # ### _CONGRATULATIONS!!_ # # Please treat yourself to a nice tea break :-) diff --git a/src/spyglass/common/common_ephys.py b/src/spyglass/common/common_ephys.py index f9abff647..7e394bd2d 100644 --- a/src/spyglass/common/common_ephys.py +++ b/src/spyglass/common/common_ephys.py @@ -907,13 +907,3 @@ def fetch1_dataframe(self, *attrs, **kwargs): filtered_nwb["filtered_data"].timestamps, name="time" ), ) - - -@schema -class ElectrodeBrainRegion(SpyglassMixin, dj.Manual): - definition = """ - # Table with brain region of electrodes determined post-experiment e.g. via histological analysis or CT - -> Electrode - --- - -> BrainRegion - """ diff --git a/src/spyglass/position/position_merge.py b/src/spyglass/position/position_merge.py index 72e330af6..ea2d574a2 100644 --- a/src/spyglass/position/position_merge.py +++ b/src/spyglass/position/position_merge.py @@ -1,7 +1,3 @@ -import functools as ft -import os -from pathlib import Path - import datajoint as dj import numpy as np import pandas as pd @@ -87,176 +83,3 @@ def fetch1_dataframe(self): & key ) return query.fetch1_dataframe() - - -@schema -class PositionVideoSelection(SpyglassMixin, dj.Manual): - definition = """ - nwb_file_name : varchar(255) # name of the NWB file - interval_list_name : varchar(170) # descriptive name of this interval list - plot_id : int - plot : varchar(40) # Which position info to overlay on video file - --- - output_dir : varchar(255) # directory where to save output video - """ - - # NOTE: See #630, #664. Excessive key length. - - def insert1(self, key, **kwargs): - key["plot_id"] = self.get_plotid(key) - super().insert1(key, **kwargs) - - def get_plotid(self, key): - fields = list(self.primary_key) - temp_key = {k: val for k, val in key.items() if k in fields} - plot_id = temp_key.get("plot_id", None) - if plot_id is None: - plot_id = ( - dj.U().aggr(self & temp_key, n="max(plot_id)").fetch1("n") or 0 - ) + 1 - else: - id = (self & temp_key).fetch("plot_id") - if len(id) > 0: - plot_id = max(id) + 1 - else: - plot_id = max(0, plot_id) - return plot_id - - -@schema -class PositionVideo(SpyglassMixin, dj.Computed): - """Creates a video of the computed head position and orientation as well as - the original LED positions overlaid on the video of the animal. - - Use for debugging the effect of position extraction parameters.""" - - definition = """ - -> PositionVideoSelection - --- - """ - - def make(self, key): - raise NotImplementedError("work in progress -DPG") - - plot = key.get("plot") - if plot not in ["DLC", "Trodes", "Common", "All"]: - raise ValueError(f"Plot {key['plot']} not supported") - # CBroz: I was told only tests should `assert`, code should `raise` - - M_TO_CM = 100 - output_dir = (PositionVideoSelection & key).fetch1("output_dir") - - logger.info("Loading position data...") - # raw_position_df = ( - # RawPosition() - # & { - # "nwb_file_name": key["nwb_file_name"], - # "interval_list_name": key["interval_list_name"], - # } - # ).fetch1_dataframe() - - query = { - "nwb_file_name": key["nwb_file_name"], - "interval_list_name": key["interval_list_name"], - } - merge_entries = { - "DLC": PositionOutput.DLCPosV1 & query, - "Trodes": PositionOutput.TrodesPosV1 & query, - "Common": PositionOutput.CommonPos & query, - } - - position_mean_dict = {} - if plot == "All": - # Check which entries exist in PositionOutput - merge_dict = {} - for source, entries in merge_entries.items(): - if entries: - merge_dict[source] = entries.fetch1_dataframe().drop( - columns=["velocity_x", "velocity_y", "speed"] - ) - - pos_df = ft.reduce( - lambda left, right,: pd.merge( - left[1], - right[1], - left_index=True, - right_index=True, - suffixes=[f"_{left[0]}", f"_{right[0]}"], - ), - merge_dict.items(), - ) - position_mean_dict = { - source: { - "position": np.asarray( - pos_df[[f"position_x_{source}", f"position_y_{source}"]] - ), - "orientation": np.asarray( - pos_df[[f"orientation_{source}"]] - ), - } - for source in merge_dict.keys() - } - else: - if plot == "DLC": - # CBroz - why is this extra step needed for DLC? - pos_df_key = merge_entries[plot].fetch1(as_dict=True) - pos_df = (PositionOutput & pos_df_key).fetch1_dataframe() - elif plot in ["Trodes", "Common"]: - pos_df = merge_entries[plot].fetch1_dataframe() - - position_mean_dict[plot]["position"] = np.asarray( - pos_df[["position_x", "position_y"]] - ) - position_mean_dict[plot]["orientation"] = np.asarray( - pos_df[["orientation"]] - ) - - logger.info("Loading video data...") - epoch = int("".join(filter(str.isdigit, key["interval_list_name"]))) + 1 - - ( - video_path, - video_filename, - meters_per_pixel, - video_time, - ) = get_video_path( - {"nwb_file_name": key["nwb_file_name"], "epoch": epoch} - ) - video_dir = os.path.dirname(video_path) + "/" - video_frame_col_name = [ - col for col in pos_df.columns if "video_frame_ind" in col - ][0] - video_frame_inds = pos_df[video_frame_col_name].astype(int).to_numpy() - if plot in ["DLC", "All"]: - video_path = ( - DLCPoseEstimationSelection - & (PositionOutput.DLCPosV1 & key).fetch1("KEY") - ).fetch1("video_path") - else: - video_path = check_videofile( - video_dir, key["output_dir"], video_filename - )[0] - - nwb_base_filename = key["nwb_file_name"].replace(".nwb", "") - output_video_filename = Path( - f"{Path(output_dir).as_posix()}/{nwb_base_filename}{epoch:02d}_" - f"{key['plot']}_pos_overlay.mp4" - ).as_posix() - - # centroids = {'red': np.asarray(raw_position_df[['xloc', 'yloc']]), - # 'green': np.asarray(raw_position_df[['xloc2', 'yloc2']])} - - logger.info("Making video...") - - make_video( - video_path, - video_frame_inds, - position_mean_dict, - video_time, - np.asarray(pos_df.index), - processor="opencv", - output_video_filename=output_video_filename, - cm_to_pixels=meters_per_pixel * M_TO_CM, - disable_progressbar=False, - ) - self.insert1(key) diff --git a/src/spyglass/spikesorting/v0/__init__.py b/src/spyglass/spikesorting/v0/__init__.py index f15d25230..8b6035023 100644 --- a/src/spyglass/spikesorting/v0/__init__.py +++ b/src/spyglass/spikesorting/v0/__init__.py @@ -22,7 +22,6 @@ MetricParameters, MetricSelection, QualityMetrics, - UnitInclusionParameters, WaveformParameters, Waveforms, WaveformSelection, diff --git a/src/spyglass/spikesorting/v0/spikesorting_curation.py b/src/spyglass/spikesorting/v0/spikesorting_curation.py index acdebe352..78ed93bba 100644 --- a/src/spyglass/spikesorting/v0/spikesorting_curation.py +++ b/src/spyglass/spikesorting/v0/spikesorting_curation.py @@ -1077,98 +1077,3 @@ def get_sort_group_info(cls, key): * SortGroup.SortGroupElectrode() ) * BrainRegion() return sort_group_info - - -@schema -class UnitInclusionParameters(SpyglassMixin, dj.Manual): - definition = """ - unit_inclusion_param_name: varchar(80) # the name of the list of thresholds for unit inclusion - --- - inclusion_param_dict: blob # the dictionary of inclusion / exclusion parameters - """ - - def insert1(self, key, **kwargs): - # check to see that the dictionary fits the specifications - # The inclusion parameter dict has the following form: - # param_dict['metric_name'] = (operator, value) - # where operator is '<', '>', <=', '>=', or '==' and value is the comparison (float) value to be used () - # param_dict['exclude_labels'] = [list of labels to exclude] - pdict = key["inclusion_param_dict"] - metrics_list = CuratedSpikeSorting().metrics_fields() - - for k in pdict: - if k not in metrics_list and k != "exclude_labels": - raise Exception( - f"key {k} is not a valid element of the inclusion_param_dict" - ) - if k in metrics_list: - if pdict[k][0] not in _comparison_to_function: - raise Exception( - f"operator {pdict[k][0]} for metric {k} is not in the valid operators list: {_comparison_to_function.keys()}" - ) - if k == "exclude_labels": - for label in pdict[k]: - if label not in valid_labels: - raise Exception( - f"exclude label {label} is not in the valid_labels list: {valid_labels}" - ) - super().insert1(key, **kwargs) - - def get_included_units( - self, curated_sorting_key, unit_inclusion_param_name - ): - """Given a reference to a set of curated sorting units and the name of - a unit inclusion parameter list, returns unit key - - Parameters - ---------- - curated_sorting_key : dict - key to select a set of curated sorting - unit_inclusion_param_name : str - name of a unit inclusion parameter entry - - Returns - ------- - dict - key to select all of the included units - """ - curated_sortings = (CuratedSpikeSorting() & curated_sorting_key).fetch() - inc_param_dict = ( - UnitInclusionParameters - & {"unit_inclusion_param_name": unit_inclusion_param_name} - ).fetch1("inclusion_param_dict") - units = (CuratedSpikeSorting().Unit() & curated_sortings).fetch() - units_key = (CuratedSpikeSorting().Unit() & curated_sortings).fetch( - "KEY" - ) - # get the list of labels to exclude if there is one - if "exclude_labels" in inc_param_dict: - exclude_labels = inc_param_dict["exclude_labels"] - del inc_param_dict["exclude_labels"] - else: - exclude_labels = [] - - # create a list of the units to kepp. - keep = np.asarray([True] * len(units)) - for metric in inc_param_dict: - # for all units, go through each metric, compare it to the value - # specified, and update the list to be kept - keep = np.logical_and( - keep, - _comparison_to_function[inc_param_dict[metric][0]]( - units[metric], inc_param_dict[metric][1] - ), - ) - - # now exclude by label if it is specified - if len(exclude_labels): - for unit_ind in np.ravel(np.argwhere(keep)): - labels = units[unit_ind]["label"].split(",") - for label in labels: - if label in exclude_labels: - keep[unit_ind] = False - break - - # return units that passed all of the tests - # TODO: Make this more efficient - return {i: units_key[i] for i in np.ravel(np.argwhere(keep))} From 8eadc303dab0291c8bb69772d9db992e5c2d3bb7 Mon Sep 17 00:00:00 2001 From: Samuel Bray Date: Fri, 14 Jun 2024 14:06:55 -0700 Subject: [PATCH 03/94] Non-daemon parallel populate (#1001) * initial non daemon parallel commit * resolve namespace and pickling errors * fix linting * update changelog * implement review comments * add parallel_make flag to spikesorting recording tables * fix multiprocessing spawn error on mac * move propert --------- Co-authored-by: Samuel Bray --- CHANGELOG.md | 1 + src/spyglass/decoding/v1/waveform_features.py | 2 + .../spikesorting/v0/spikesorting_recording.py | 2 + src/spyglass/spikesorting/v1/recording.py | 2 + src/spyglass/utils/dj_helper_fn.py | 43 +++++++++++++++++++ src/spyglass/utils/dj_mixin.py | 40 ++++++++++++++++- 6 files changed, 89 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15cf86478..4e6c8640c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ - Add pytests for position pipeline, various `test_mode` exceptions #966 - Migrate `pip` dependencies from `environment.yml`s to `pyproject.toml` #966 - Add documentation for common error messages #997 +- Allow mixin tables with parallelization in `make` to run populate with `processes > 1` #1001 ### Pipelines diff --git a/src/spyglass/decoding/v1/waveform_features.py b/src/spyglass/decoding/v1/waveform_features.py index 536ed4864..67f7ab692 100644 --- a/src/spyglass/decoding/v1/waveform_features.py +++ b/src/spyglass/decoding/v1/waveform_features.py @@ -102,6 +102,8 @@ class UnitWaveformFeatures(SpyglassMixin, dj.Computed): object_id: varchar(40) # the NWB object that stores the waveforms """ + _parallel_make = True + def make(self, key): AnalysisNwbfile()._creation_times["pre_create_time"] = time() # get the list of feature parameters diff --git a/src/spyglass/spikesorting/v0/spikesorting_recording.py b/src/spyglass/spikesorting/v0/spikesorting_recording.py index e2eda9b43..26016cf91 100644 --- a/src/spyglass/spikesorting/v0/spikesorting_recording.py +++ b/src/spyglass/spikesorting/v0/spikesorting_recording.py @@ -381,6 +381,8 @@ class SpikeSortingRecording(SpyglassMixin, dj.Computed): -> IntervalList.proj(sort_interval_list_name='interval_list_name') """ + _parallel_make = True + def make(self, key): sort_interval_valid_times = self._get_sort_interval_valid_times(key) recording = self._get_filtered_recording(key) diff --git a/src/spyglass/spikesorting/v1/recording.py b/src/spyglass/spikesorting/v1/recording.py index 43ccd5495..5fa069a28 100644 --- a/src/spyglass/spikesorting/v1/recording.py +++ b/src/spyglass/spikesorting/v1/recording.py @@ -216,6 +216,8 @@ class SpikeSortingRecordingSelection(SpyglassMixin, dj.Manual): -> LabTeam """ + _parallel_make = True + @classmethod def insert_selection(cls, key: dict): """Insert a row into SpikeSortingRecordingSelection with an diff --git a/src/spyglass/utils/dj_helper_fn.py b/src/spyglass/utils/dj_helper_fn.py index 3fa18191c..85ff1922a 100644 --- a/src/spyglass/utils/dj_helper_fn.py +++ b/src/spyglass/utils/dj_helper_fn.py @@ -1,6 +1,7 @@ """Helper functions for manipulating information from DataJoint fetch calls.""" import inspect +import multiprocessing.pool import os from pathlib import Path from typing import List, Type, Union @@ -465,3 +466,45 @@ def make_file_obj_id_unique(nwb_path: str): f.attrs["object_id"] = new_id _resolve_external_table(nwb_path, nwb_path.split("/")[-1]) return new_id + + +def populate_pass_function(value): + """Pass function for parallel populate. + + Note: To avoid pickling errors, the table must be passed by class, NOT by instance. + Note: This function must be defined in the global namespace. + + Parameters + ---------- + value : (table, key, kwargs) + Class of table to populate, key to populate, and kwargs for populate + """ + table, key, kwargs = value + return table.populate(key, **kwargs) + + +class NonDaemonPool(multiprocessing.pool.Pool): + """NonDaemonPool. Used to create a pool of non-daemonized processes, + which are required for parallel populate operations in DataJoint. + """ + + # Explicitly set the start method to 'fork' + # Allows the pool to be used in MacOS, where the default start method is 'spawn' + multiprocessing.set_start_method("fork", force=True) + + def Process(self, *args, **kwds): + proc = super(NonDaemonPool, self).Process(*args, **kwds) + + class NonDaemonProcess(proc.__class__): + """Monkey-patch process to ensure it is never daemonized""" + + @property + def daemon(self): + return False + + @daemon.setter + def daemon(self, val): + pass + + proc.__class__ = NonDaemonProcess + return proc diff --git a/src/spyglass/utils/dj_mixin.py b/src/spyglass/utils/dj_mixin.py index 35e54ea7a..51f398436 100644 --- a/src/spyglass/utils/dj_mixin.py +++ b/src/spyglass/utils/dj_mixin.py @@ -1,3 +1,4 @@ +import multiprocessing.pool from atexit import register as exit_register from atexit import unregister as exit_unregister from collections import OrderedDict @@ -19,7 +20,12 @@ from pymysql.err import DataError from spyglass.utils.database_settings import SHARED_MODULES -from spyglass.utils.dj_helper_fn import fetch_nwb, get_nwb_table +from spyglass.utils.dj_helper_fn import ( # NonDaemonPool, + NonDaemonPool, + fetch_nwb, + get_nwb_table, + populate_pass_function, +) from spyglass.utils.dj_merge_tables import RESERVED_PRIMARY_KEY as MERGE_PK from spyglass.utils.dj_merge_tables import Merge, is_merge_table from spyglass.utils.logging import logger @@ -72,6 +78,7 @@ class SpyglassMixin: _member_pk = None # LabMember primary key. Mixin ambivalent table structure _banned_search_tables = set() # Tables to avoid in restrict_by + _parallel_make = False # Tables that use parallel processing in make def __init__(self, *args, **kwargs): """Initialize SpyglassMixin. @@ -655,6 +662,37 @@ def super_delete(self, warn=True, *args, **kwargs): self._log_delete(start=time(), super_delete=True) super().delete(*args, **kwargs) + # -------------------------- non-daemon populate -------------------------- + def populate(self, *restrictions, **kwargs): + """Populate table in parallel. + + Supersedes datajoint.table.Table.populate for classes with that + spawn processes in their make function + """ + + # Pass through to super if not parallel in the make function or only a single process + processes = kwargs.pop("processes", 1) + if processes == 1 or not self._parallel_make: + return super().populate(*restrictions, **kwargs) + + # If parallel in both make and populate, use non-daemon processes + # Get keys to populate + keys = (self._jobs_to_do(restrictions) - self.target).fetch( + "KEY", limit=kwargs.get("limit", None) + ) + # package the call list + call_list = [(type(self), key, kwargs) for key in keys] + + # Create a pool of non-daemon processes to populate a single entry each + pool = NonDaemonPool(processes=processes) + try: + pool.map(populate_pass_function, call_list) + except Exception as e: + raise e + finally: + pool.close() + pool.terminate() + # ------------------------------- Export Log ------------------------------- @cached_property From 97933e7a6d3dad383a72a8664a52aca2de626339 Mon Sep 17 00:00:00 2001 From: Kyu Hyun Lee Date: Tue, 18 Jun 2024 07:55:09 -0700 Subject: [PATCH 04/94] Give UUID to artifact interval (#993) * Give UUID to artifact interval * Add ability to set smoothing sigma in get_firing_rate (#994) * add option to set spike smoothing sigma * update changelog * Add docstrings to SortedSpikesGroup and Decoding methods (#996) * Add docstrings * update changelog * fix spelling --------- Co-authored-by: Samuel Bray * Add Common Errors doc (#997) * Add Common Errors * Update changelog * Mua notebook (#998) * documented some of mua notebook * mua notebook documented * documented some of mua notebook * synced py script * Dandi export and read (#956) * compile exported files, download dandiset, and organize * add function to translate files into dandi-compatible names * add table to store dandi name translation and steps to populate * add dandiset validation * add function to fetch nwb from dandi * add function to change obj_id of nwb_file * add dandi upload call and fix circular import * debug dandi file streaming * fix circular import * resolve dandi-streamed files with fetch_nwb * implement review comments * add admin tools to fix common dandi discrepencies * implement tool to cleanup common dandi errors * add dandi export to tutorial * fix linting * update changelog * fix spelling * style changes from review * reorganize function locations * fix circular import * make dandi dependency optional in imports * store dandi instance of data in DandiPath * resolve case of pre-existing dandi entries for export * cleanup bugs from refactor * update notebook * Apply suggestions from code review Co-authored-by: Chris Broz * add requested changes from review * make method check_admin_privilege in LabMember --------- Co-authored-by: Chris Broz * Minor fixes (#999) * give analysis nwb new uuid when created * fix function argument * update changelog * Fix bug in change in analysis_file object_id (#1004) * fix bug in change in analysis_file_object_id * update changelog * Remove classes for usused tables (#1003) * #976 * Remove notebook reference * Non-daemon parallel populate (#1001) * initial non daemon parallel commit * resolve namespace and pickling errors * fix linting * update changelog * implement review comments * add parallel_make flag to spikesorting recording tables * fix multiprocessing spawn error on mac * move propert --------- Co-authored-by: Samuel Bray * Update pipeline column for IntervalList --------- Co-authored-by: Samuel Bray Co-authored-by: Samuel Bray Co-authored-by: Chris Broz Co-authored-by: Denisse Morales-Rodriguez <68555303+denissemorales@users.noreply.github.com> Co-authored-by: Samuel Bray --- src/spyglass/lfp/v1/lfp_artifact.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/spyglass/lfp/v1/lfp_artifact.py b/src/spyglass/lfp/v1/lfp_artifact.py index 1f47a0884..70b8f2f2b 100644 --- a/src/spyglass/lfp/v1/lfp_artifact.py +++ b/src/spyglass/lfp/v1/lfp_artifact.py @@ -1,3 +1,5 @@ +import uuid + import datajoint as dj import numpy as np @@ -183,15 +185,7 @@ def make(self, key): dict( artifact_times=artifact_times, artifact_removed_valid_times=artifact_removed_valid_times, - # name for no-artifact time name using recording id - artifact_removed_interval_list_name="_".join( - [ - key["nwb_file_name"], - key["target_interval_list_name"], - "LFP", - key["artifact_params_name"], - ] - ), + artifact_removed_interval_list_name=uuid.uuid4(), ) ) @@ -199,11 +193,11 @@ def make(self, key): "nwb_file_name": key["nwb_file_name"], "interval_list_name": key["artifact_removed_interval_list_name"], "valid_times": key["artifact_removed_valid_times"], - "pipeline": "lfp_artifact", + "pipeline": self.full_table_name, } - LFPArtifactRemovedIntervalList.insert1(key, replace=True) - IntervalList.insert1(interval_key, replace=True) + LFPArtifactRemovedIntervalList.insert1(key) + IntervalList.insert1(interval_key) self.insert1(key) From d4f61af61ac4a360d06cc16f91f4a6bdf7e3d497 Mon Sep 17 00:00:00 2001 From: Kyu Hyun Lee Date: Mon, 24 Jun 2024 20:14:07 -0700 Subject: [PATCH 05/94] Fix artifact `list_triggers` (#1009) * Fix artifact list_triggers * Black * Update changelog --- CHANGELOG.md | 2 ++ src/spyglass/spikesorting/v1/sorting.py | 16 +++++++--------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e6c8640c..e61c612b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,8 @@ #994 - Update docstrings #996 - Remove unused `UnitInclusionParameters` table from `spikesorting.v0` #1003 + - Fix bug in identification of artifact samples to be zeroed out in `spikesorting.v1.SpikeSorting` #1009 + ## [0.5.2] (April 22, 2024) diff --git a/src/spyglass/spikesorting/v1/sorting.py b/src/spyglass/spikesorting/v1/sorting.py index 84a936eea..f738c5ff0 100644 --- a/src/spyglass/spikesorting/v1/sorting.py +++ b/src/spyglass/spikesorting/v1/sorting.py @@ -171,15 +171,15 @@ def make(self, key: dict): sorter, sorter_params = ( SpikeSorterParameters * SpikeSortingSelection & key ).fetch1("sorter", "sorter_params") + recording_analysis_nwb_file_abs_path = AnalysisNwbfile.get_abs_path( + recording_key["analysis_file_name"] + ) # DO: # - load recording # - concatenate artifact removed intervals # - run spike sorting # - save output to NWB file - recording_analysis_nwb_file_abs_path = AnalysisNwbfile.get_abs_path( - recording_key["analysis_file_name"] - ) recording = se.read_nwb_recording( recording_analysis_nwb_file_abs_path, load_time_vector=True ) @@ -200,7 +200,7 @@ def make(self, key: dict): list_triggers = [] if artifact_removed_intervals_ind[0][0] > 0: list_triggers.append( - np.array([0, artifact_removed_intervals_ind[0][0]]) + np.arange(0, artifact_removed_intervals_ind[0][0]) ) for interval_ind in range(len(artifact_removed_intervals_ind) - 1): list_triggers.append( @@ -211,11 +211,9 @@ def make(self, key: dict): ) if artifact_removed_intervals_ind[-1][1] < len(timestamps): list_triggers.append( - np.array( - [ - artifact_removed_intervals_ind[-1][1], - len(timestamps) - 1, - ] + np.arange( + artifact_removed_intervals_ind[-1][1], + len(timestamps) - 1, ) ) From b5ba05a2f25e16e72551d2b9fa59df13180e392f Mon Sep 17 00:00:00 2001 From: Samuel Bray Date: Tue, 25 Jun 2024 11:20:42 -0700 Subject: [PATCH 06/94] remove problem key in DLCPosV1 fetch_nwb attrs (#1011) * remove problem key in DLCPosV1 fetch_nwb attrs * update changelog --- CHANGELOG.md | 2 +- src/spyglass/position/v1/position_dlc_selection.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e61c612b4..17adc0a97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ - Raise `KeyError` for missing input parameters across helper funcs #966 - `DLCPosVideo` table now inserts into self after `make` #966 - Remove unused `PositionVideoSelection` and `PositionVideo` tables #1003 + - Fix SQL query error in `DLCPosV1.fetch_nwb` #1011 - Spikesorting - Allow user to set smoothing timescale in `SortedSpikesGroup.get_firing_rate` #994 @@ -43,7 +44,6 @@ - Remove unused `UnitInclusionParameters` table from `spikesorting.v0` #1003 - Fix bug in identification of artifact samples to be zeroed out in `spikesorting.v1.SpikeSorting` #1009 - ## [0.5.2] (April 22, 2024) ### Infrastructure diff --git a/src/spyglass/position/v1/position_dlc_selection.py b/src/spyglass/position/v1/position_dlc_selection.py index 02692ce14..8a283bb1d 100644 --- a/src/spyglass/position/v1/position_dlc_selection.py +++ b/src/spyglass/position/v1/position_dlc_selection.py @@ -180,6 +180,10 @@ def fetch1_dataframe(self): index=index, ) + def fetch_nwb(self, **kwargs): + attrs = [a for a in self.heading.names if not a == "pose_eval_result"] + return super().fetch_nwb(*attrs, **kwargs) + @classmethod def evaluate_pose_estimation(cls, key): likelihood_thresh = [] From fc4116783caddf1aa2abeff547d91927bb64af5c Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Wed, 26 Jun 2024 10:51:43 -0500 Subject: [PATCH 07/94] Tidy position (#870) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * WIP: pytests for common & lfp * WIP: pytests for utils 1 * ✅ : pytests for utils, position, linearization * Remove unnecessary lfp_band checks. Add deprecation warn on permission set * WIP: Tidy position 1 * WIP: Tidy position 2 * Spellcheck tests * Reduce pop_all_common redundancy * PosIntervalMap #849 * Logger decorator, unify make_video logic * Update changelog/requirements * Misc edits * Change deprecation warning * Video func name specificity * Revise centroid calc * Fix errors * Vectorize orient calc. Remove multitable stack warn * Revert blit --- .gitignore | 1 + CHANGELOG.md | 24 +- pyproject.toml | 1 + src/spyglass/common/common_behav.py | 118 +- src/spyglass/common/common_dandi.py | 2 + src/spyglass/common/common_device.py | 2 + src/spyglass/common/common_ephys.py | 10 +- src/spyglass/common/common_lab.py | 4 +- src/spyglass/common/common_position.py | 2 +- src/spyglass/common/common_session.py | 26 +- src/spyglass/common/common_usage.py | 19 + src/spyglass/common/populate_all_common.py | 4 + src/spyglass/data_import/insert_sessions.py | 3 +- src/spyglass/position/position_merge.py | 12 +- src/spyglass/position/v1/__init__.py | 4 +- src/spyglass/position/v1/dlc_decorators.py | 27 - src/spyglass/position/v1/dlc_reader.py | 2 + src/spyglass/position/v1/dlc_utils.py | 1258 ++++++----------- src/spyglass/position/v1/dlc_utils_makevid.py | 562 ++++++++ .../position/v1/position_dlc_centroid.py | 799 +++-------- .../position/v1/position_dlc_cohort.py | 82 +- .../position/v1/position_dlc_model.py | 60 +- .../position/v1/position_dlc_orient.py | 134 +- .../v1/position_dlc_pose_estimation.py | 355 +++-- .../position/v1/position_dlc_position.py | 291 ++-- .../position/v1/position_dlc_project.py | 412 +++--- .../position/v1/position_dlc_selection.py | 215 ++- .../position/v1/position_dlc_training.py | 286 ++-- .../position/v1/position_trodes_position.py | 252 +--- src/spyglass/utils/dj_helper_fn.py | 29 +- src/spyglass/utils/dj_merge_tables.py | 9 +- src/spyglass/utils/dj_mixin.py | 13 +- src/spyglass/utils/nwb_helper_fn.py | 12 +- tests/common/test_behav.py | 10 +- tests/common/test_device.py | 2 +- tests/common/test_ephys.py | 4 +- tests/common/test_insert.py | 30 +- tests/common/test_interval.py | 4 +- tests/common/test_interval_helpers.py | 2 +- tests/common/test_lab.py | 2 +- tests/common/test_region.py | 2 +- tests/common/test_session.py | 3 +- tests/conftest.py | 136 +- tests/container.py | 4 +- tests/data_downloader.py | 63 +- tests/lfp/test_lfp.py | 8 +- tests/position/conftest.py | 1 + tests/position/test_dlc_cent.py | 27 +- tests/position/test_dlc_model.py | 2 +- tests/position/test_dlc_pos_est.py | 4 +- tests/position/test_dlc_sel.py | 2 +- tests/position/test_dlc_train.py | 6 +- tests/position/test_trodes.py | 3 +- tests/utils/test_db_settings.py | 6 +- tests/utils/test_graph.py | 4 +- tests/utils/test_mixin.py | 4 +- tests/utils/test_nwb_helper_fn.py | 4 +- 57 files changed, 2424 insertions(+), 2939 deletions(-) delete mode 100644 src/spyglass/position/v1/dlc_decorators.py create mode 100644 src/spyglass/position/v1/dlc_utils_makevid.py diff --git a/.gitignore b/.gitignore index 6319e5f1c..1f18f4178 100644 --- a/.gitignore +++ b/.gitignore @@ -137,6 +137,7 @@ dmypy.json *.videoTimeStamps *.cameraHWSync *.stateScriptLog +tests/_data/* *.nwb *.DS_Store diff --git a/CHANGELOG.md b/CHANGELOG.md index 17adc0a97..69d2d0a87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ +```python +from spyglass.common.common_behav import PositionIntervalMap + +PositionIntervalMap.alter() +``` + ### Infrastructure - Create class `SpyglassGroupPart` to aid delete propagations #899 @@ -19,7 +25,8 @@ - Add pytests for position pipeline, various `test_mode` exceptions #966 - Migrate `pip` dependencies from `environment.yml`s to `pyproject.toml` #966 - Add documentation for common error messages #997 -- Allow mixin tables with parallelization in `make` to run populate with `processes > 1` #1001 +- Allow mixin tables with parallelization in `make` to run populate with + `processes > 1` #1001 ### Pipelines @@ -30,6 +37,9 @@ - Remove unused `ElectrodeBrainRegion` table #1003 - Files created by `AnalysisNwbfile.create()` receive new object_id #999, #1004 + - Remove redundant calls to tables in `populate_all_common` #870 + - Improve logging clarity in `populate_all_common` #870 + - `PositionIntervalMap` now inserts null entries for missing intervals #870 - Decoding: Default values for classes on `ImportError` #966 - Position - Allow dlc without pre-existing tracking data #973, #975 @@ -37,12 +47,18 @@ - `DLCPosVideo` table now inserts into self after `make` #966 - Remove unused `PositionVideoSelection` and `PositionVideo` tables #1003 - Fix SQL query error in `DLCPosV1.fetch_nwb` #1011 + - Add keyword args to all calls of `convert_to_pixels` #870 + - Unify `make_video` logic across `DLCPosVideo` and `TrodesVideo` #870 + - Replace `OutputLogger` context manager with decorator #870 + - Rename `check_videofile` -> `find_mp4` and `get_video_path` -> + `get_video_info` to reflect actual use #870 - Spikesorting - Allow user to set smoothing timescale in `SortedSpikesGroup.get_firing_rate` #994 - Update docstrings #996 - Remove unused `UnitInclusionParameters` table from `spikesorting.v0` #1003 - - Fix bug in identification of artifact samples to be zeroed out in `spikesorting.v1.SpikeSorting` #1009 + - Fix bug in identification of artifact samples to be zeroed out in + `spikesorting.v1.SpikeSorting` #1009 ## [0.5.2] (April 22, 2024) @@ -86,11 +102,15 @@ ### Pipelines +- Common: + - Add ActivityLog to `common_usage` to track unreferenced utilities. #870 - Position: - Fixes to `environment-dlc.yml` restricting tensortflow #834 - Video restriction for multicamera epochs #834 - Fixes to `_convert_mp4` #834 - Replace deprecated calls to `yaml.safe_load()` #834 + - Refactoring to reduce redundancy #870 + - Migrate `OutputLogger` behavior to decorator #870 - Spikesorting: - Increase`spikeinterface` version to >=0.99.1, \<0.100 #852 - Bug fix in single artifact interval edge case #859 diff --git a/pyproject.toml b/pyproject.toml index 78d189b73..2538b00dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,6 +72,7 @@ test = [ "kachery", # database access "kachery-client", "kachery-cloud>=0.4.0", + "opencv-python-headless", # for headless testing of Qt "pre-commit", # linting "pytest", # unit testing "pytest-cov", # code coverage diff --git a/src/spyglass/common/common_behav.py b/src/spyglass/common/common_behav.py index 67e6e35d9..a1397769b 100644 --- a/src/spyglass/common/common_behav.py +++ b/src/spyglass/common/common_behav.py @@ -14,7 +14,7 @@ from spyglass.common.common_nwbfile import Nwbfile from spyglass.common.common_session import Session # noqa: F401 from spyglass.common.common_task import TaskEpoch -from spyglass.settings import video_dir +from spyglass.settings import test_mode, video_dir from spyglass.utils import SpyglassMixin, logger from spyglass.utils.nwb_helper_fn import ( get_all_spatial_series, @@ -56,8 +56,8 @@ def make(self, keys: Union[List[Dict], dj.Table]): keys = [keys] if isinstance(keys[0], (dj.Table, dj.expression.QueryExpression)): keys = [k for tbl in keys for k in tbl.fetch("KEY", as_dict=True)] - for key in keys: - nwb_file_name = key.get("nwb_file_name") + nwb_files = set(key.get("nwb_file_name") for key in keys) + for nwb_file_name in nwb_files: # Only unique nwb files if not nwb_file_name: raise ValueError("PositionSource.make requires nwb_file_name") self.insert_from_nwbfile(nwb_file_name, skip_duplicates=True) @@ -311,7 +311,7 @@ def make(self, key): if associated_files is None: logger.info( "Unable to import StateScriptFile: no processing module named " - + '"associated_files" found in {nwb_file_name}.' + + f'"associated_files" found in {nwb_file_name}.' ) return # See #849 @@ -377,10 +377,12 @@ class VideoFile(SpyglassMixin, dj.Imported): def make(self, key): self._no_transaction_make(key) - def _no_transaction_make(self, key, verbose=True): + def _no_transaction_make(self, key, verbose=True, skip_duplicates=False): if not self.connection.in_transaction: self.populate(key) return + if test_mode: + skip_duplicates = True nwb_file_name = key["nwb_file_name"] nwb_file_abspath = Nwbfile.get_abs_path(nwb_file_name) @@ -404,6 +406,7 @@ def _no_transaction_make(self, key, verbose=True): "interval_list_name": interval_list_name, } ).fetch1("valid_times") + cam_device_str = r"camera_device (\d+)" is_found = False for ind, video in enumerate(videos.values()): @@ -413,28 +416,35 @@ def _no_transaction_make(self, key, verbose=True): # check to see if the times for this video_object are largely # overlapping with the task epoch times - if len( + if not len( interval_list_contains(valid_times, video_obj.timestamps) > 0.9 * len(video_obj.timestamps) ): - nwb_cam_device = video_obj.device.name - # returns whatever was captured in the first group (within the parentheses) of the regular expression -- in this case, 0 - key["video_file_num"] = int( - re.match(cam_device_str, nwb_cam_device)[1] - ) - camera_name = video_obj.device.camera_name - if CameraDevice & {"camera_name": camera_name}: - key["camera_name"] = video_obj.device.camera_name - else: - raise KeyError( - f"No camera with camera_name: {camera_name} found " - + "in CameraDevice table." - ) - key["video_file_object_id"] = video_obj.object_id - self.insert1( - key, skip_duplicates=True, allow_direct_insert=True + continue + + nwb_cam_device = video_obj.device.name + + # returns whatever was captured in the first group (within the + # parentheses) of the regular expression - in this case, 0 + + key["video_file_num"] = int( + re.match(cam_device_str, nwb_cam_device)[1] + ) + camera_name = video_obj.device.camera_name + if CameraDevice & {"camera_name": camera_name}: + key["camera_name"] = video_obj.device.camera_name + else: + raise KeyError( + f"No camera with camera_name: {camera_name} found " + + "in CameraDevice table." ) - is_found = True + key["video_file_object_id"] = video_obj.object_id + self.insert1( + key, + skip_duplicates=skip_duplicates, + allow_direct_insert=True, + ) + is_found = True if not is_found and verbose: logger.info( @@ -443,7 +453,7 @@ def _no_transaction_make(self, key, verbose=True): ) @classmethod - def update_entries(cls, restrict={}): + def update_entries(cls, restrict=True): existing_entries = (cls & restrict).fetch("KEY") for row in existing_entries: if (cls & row).fetch1("camera_name"): @@ -495,9 +505,11 @@ class PositionIntervalMap(SpyglassMixin, dj.Computed): definition = """ -> IntervalList --- - position_interval_name: varchar(200) # name of the corresponding position interval + position_interval_name="": varchar(200) # name of the corresponding interval """ + # #849 - Insert null to avoid rerun + def make(self, key): self._no_transaction_make(key) @@ -510,6 +522,8 @@ def _no_transaction_make(self, key): # if not called in the context of a make function, call its own make function self.populate(key) return + if self & key: + return # *** HARD CODED VALUES *** EPSILON = 0.51 # tolerated time diff in bounds across epoch/pos @@ -517,11 +531,13 @@ def _no_transaction_make(self, key): nwb_file_name = key["nwb_file_name"] pos_intervals = get_pos_interval_list_names(nwb_file_name) + null_key = dict(key, position_interval_name="") + insert_opts = dict(allow_direct_insert=True, skip_duplicates=True) # Skip populating if no pos interval list names if len(pos_intervals) == 0: - # TODO: Now that populate_all accept errors, raise here? logger.error(f"NO POS INTERVALS FOR {key}; {no_pop_msg}") + self.insert1(null_key, **insert_opts) return valid_times = (IntervalList & key).fetch1("valid_times") @@ -535,7 +551,6 @@ def _no_transaction_make(self, key): f"nwb_file_name='{nwb_file_name}' AND interval_list_name=" + "'{}'" ) for pos_interval in pos_intervals: - # cbroz: fetch1->fetch. fetch1 would fail w/o result pos_times = (IntervalList & restr.format(pos_interval)).fetch( "valid_times" ) @@ -558,16 +573,18 @@ def _no_transaction_make(self, key): # Check that each pos interval was matched to only one epoch if len(matching_pos_intervals) != 1: - # TODO: Now that populate_all accept errors, raise here? logger.warning( - f"Found {len(matching_pos_intervals)} pos intervals for {key}; " - + f"{no_pop_msg}\n{matching_pos_intervals}" + f"{no_pop_msg}. Found {len(matching_pos_intervals)} pos intervals for " + + f"\n\t{key}\n\tMatching intervals: {matching_pos_intervals}" ) + self.insert1(null_key, **insert_opts) return # Insert into table - key["position_interval_name"] = matching_pos_intervals[0] - self.insert1(key, skip_duplicates=True, allow_direct_insert=True) + self.insert1( + dict(key, position_interval_name=matching_pos_intervals[0]), + **insert_opts, + ) logger.info( "Populated PosIntervalMap for " + f'{nwb_file_name}, {key["interval_list_name"]}' @@ -609,19 +626,27 @@ def convert_epoch_interval_name_to_position_interval_name( ) pos_query = PositionIntervalMap & key + pos_str = "position_interval_name" - if len(pos_query) == 0: - if populate_missing: - PositionIntervalMap()._no_transaction_make(key) - pos_query = PositionIntervalMap & key + no_entries = len(pos_query) == 0 + null_entry = pos_query.fetch(pos_str)[0] == "" if len(pos_query) else False - if len(pos_query) == 0: + if populate_missing and (no_entries or null_entry): + if null_entry: + pos_query.delete(safemode=False) # no prompt + PositionIntervalMap()._no_transaction_make(key) + pos_query = PositionIntervalMap & key + + if pos_query.fetch(pos_str)[0] == "": logger.info(f"No position intervals found for {key}") return [] if len(pos_query) == 1: return pos_query.fetch1("position_interval_name") + else: + raise ValueError(f"Multiple intervals found for {key}: {pos_query}") + def get_interval_list_name_from_epoch(nwb_file_name: str, epoch: int) -> str: """Returns the interval list name for the given epoch. @@ -653,13 +678,12 @@ def get_interval_list_name_from_epoch(nwb_file_name: str, epoch: int) -> str: def populate_position_interval_map_session(nwb_file_name: str): - for interval_name in (TaskEpoch & {"nwb_file_name": nwb_file_name}).fetch( - "interval_list_name" - ): - with PositionIntervalMap._safe_context(): - PositionIntervalMap().make( - { - "nwb_file_name": nwb_file_name, - "interval_list_name": interval_name, - } - ) + # 1. remove redundancy in interval names + # 2. let PositionIntervalMap handle transaction context + nwb_dict = dict(nwb_file_name=nwb_file_name) + intervals = (TaskEpoch & nwb_dict).fetch("interval_list_name") + for interval_name in set(intervals): + interval_dict = dict(interval_list_name=interval_name) + if PositionIntervalMap & interval_dict: + continue + PositionIntervalMap().make(dict(nwb_dict, **interval_dict)) diff --git a/src/spyglass/common/common_dandi.py b/src/spyglass/common/common_dandi.py index e3c2836e1..8264de4cb 100644 --- a/src/spyglass/common/common_dandi.py +++ b/src/spyglass/common/common_dandi.py @@ -8,6 +8,8 @@ import pynwb from fsspec.implementations.cached import CachingFileSystem +from spyglass.utils import logger + try: import dandi.download import dandi.organize diff --git a/src/spyglass/common/common_device.py b/src/spyglass/common/common_device.py index 96fa11d44..b2e764b24 100644 --- a/src/spyglass/common/common_device.py +++ b/src/spyglass/common/common_device.py @@ -703,6 +703,8 @@ def prompt_insert( if table_type: table_type += " " + else: + table_type = "" logger.info( f"{table}{table_type} '{name}' was not found in the" diff --git a/src/spyglass/common/common_ephys.py b/src/spyglass/common/common_ephys.py index 7e394bd2d..146efeea1 100644 --- a/src/spyglass/common/common_ephys.py +++ b/src/spyglass/common/common_ephys.py @@ -98,14 +98,10 @@ class Electrode(SpyglassMixin, dj.Imported): """ def make(self, key): - """Make without transaction - - Allows populate_all_common to work within a single transaction.""" - nwb_file_name = key["nwb_file_name"] nwb_file_abspath = Nwbfile.get_abs_path(nwb_file_name) nwbf = get_nwb_file(nwb_file_abspath) - config = get_config(nwb_file_abspath) + config = get_config(nwb_file_abspath, calling_table=self.camel_name) if "Electrode" in config: electrode_config_dicts = { @@ -202,7 +198,7 @@ def create_from_config(cls, nwb_file_name: str): """ nwb_file_abspath = Nwbfile.get_abs_path(nwb_file_name) nwbf = get_nwb_file(nwb_file_abspath) - config = get_config(nwb_file_abspath) + config = get_config(nwb_file_abspath, calling_table=cls.__name__) if "Electrode" not in config: return # See #849 @@ -323,7 +319,7 @@ def make(self, key): # same nwb_object_id logger.info( - f'Importing raw data: Sampling rate:\t{key["sampling_rate"]} Hz\n' + f'Importing raw data: Sampling rate:\t{key["sampling_rate"]} Hz\n\t' + f'Number of valid intervals:\t{len(interval_dict["valid_times"])}' ) diff --git a/src/spyglass/common/common_lab.py b/src/spyglass/common/common_lab.py index bd8a90262..5958a4de9 100644 --- a/src/spyglass/common/common_lab.py +++ b/src/spyglass/common/common_lab.py @@ -213,8 +213,8 @@ def create_new_team( ) if not query: logger.info( - f"Please add the Google user ID for {team_member} in " - + "LabMember.LabMemberInfo to help manage permissions." + "To help manage permissions in LabMemberInfo, please add Google " + + f"user ID for {team_member}" ) labteammember_dict = { "team_name": team_name, diff --git a/src/spyglass/common/common_position.py b/src/spyglass/common/common_position.py index ed91aa463..f94cfff67 100644 --- a/src/spyglass/common/common_position.py +++ b/src/spyglass/common/common_position.py @@ -705,7 +705,7 @@ def make_video( head_position = head_position_mean[time_ind] head_position = self.convert_to_pixels( - head_position, frame_size, cm_to_pixels + data=head_position, cm_to_pixels=cm_to_pixels ) head_orientation = head_orientation_mean[time_ind] diff --git a/src/spyglass/common/common_session.py b/src/spyglass/common/common_session.py index b8139939a..186562444 100644 --- a/src/spyglass/common/common_session.py +++ b/src/spyglass/common/common_session.py @@ -64,7 +64,7 @@ def make(self, key): nwb_file_name = key["nwb_file_name"] nwb_file_abspath = Nwbfile.get_abs_path(nwb_file_name) nwbf = get_nwb_file(nwb_file_abspath) - config = get_config(nwb_file_abspath) + config = get_config(nwb_file_abspath, calling_table=self.camel_name) # certain data are not associated with a single NWB file / session # because they may apply to multiple sessions. these data go into @@ -77,26 +77,26 @@ def make(self, key): # via fields of Session (e.g., Subject, Institution, Lab) or part # tables (e.g., Experimenter, DataAcquisitionDevice). - logger.info("Institution...") + logger.info("Session populates Institution...") Institution().insert_from_nwbfile(nwbf) - logger.info("Lab...") + logger.info("Session populates Lab...") Lab().insert_from_nwbfile(nwbf) - logger.info("LabMember...") + logger.info("Session populates LabMember...") LabMember().insert_from_nwbfile(nwbf) - logger.info("Subject...") + logger.info("Session populates Subject...") Subject().insert_from_nwbfile(nwbf) if not debug_mode: # TODO: remove when demo files agree on device - logger.info("Populate DataAcquisitionDevice...") + logger.info("Session populates Populate DataAcquisitionDevice...") DataAcquisitionDevice.insert_from_nwbfile(nwbf, config) - logger.info("Populate CameraDevice...") + logger.info("Session populates Populate CameraDevice...") CameraDevice.insert_from_nwbfile(nwbf) - logger.info("Populate Probe...") + logger.info("Session populates Populate Probe...") Probe.insert_from_nwbfile(nwbf, config) if nwbf.subject is not None: @@ -126,7 +126,7 @@ def make(self, key): # interval lists depend on Session (as a primary key) but users may want to add these manually so this is # a manual table that is also populated from NWB files - logger.info("IntervalList...") + logger.info("Session populates IntervalList...") IntervalList().insert_from_nwbfile(nwbf, nwb_file_name=nwb_file_name) # logger.info('Unit...') @@ -148,8 +148,8 @@ def _add_data_acquisition_device_part(self, nwb_file_name, nwbf, config): } if len(query) == 0: logger.warn( - f"DataAcquisitionDevice with name {device_name} does not exist. " - "Cannot link Session with DataAcquisitionDevice in Session.DataAcquisitionDevice." + "Cannot link Session with DataAcquisitionDevice.\n" + + f"DataAcquisitionDevice does not exist: {device_name}" ) continue key = dict() @@ -166,8 +166,8 @@ def _add_experimenter_part(self, nwb_file_name, nwbf): query = LabMember & {"lab_member_name": name} if len(query) == 0: logger.warn( - f"LabMember with name {name} does not exist. " - "Cannot link Session with LabMember in Session.Experimenter." + "Cannot link Session with LabMember. " + + f"LabMember does not exist: {name}" ) continue diff --git a/src/spyglass/common/common_usage.py b/src/spyglass/common/common_usage.py index 9d408b5bc..6616fedf6 100644 --- a/src/spyglass/common/common_usage.py +++ b/src/spyglass/common/common_usage.py @@ -55,6 +55,25 @@ class InsertError(dj.Manual): """ +@schema +class ActivityLog(dj.Manual): + """A log of suspected low-use features worth deprecating.""" + + definition = """ + id: int auto_increment + --- + function: varchar(64) + dj_user: varchar(64) + timestamp=CURRENT_TIMESTAMP: timestamp + """ + + @classmethod + def deprecate_log(cls, name, warning=True) -> None: + if warning: + logger.warning(f"DEPRECATION scheduled for version 0.6: {name}") + cls.insert1(dict(dj_user=dj.config["database.user"], function=name)) + + @schema class ExportSelection(SpyglassMixin, dj.Manual): definition = """ diff --git a/src/spyglass/common/populate_all_common.py b/src/spyglass/common/populate_all_common.py index e78b68de1..d3b0d7f62 100644 --- a/src/spyglass/common/populate_all_common.py +++ b/src/spyglass/common/populate_all_common.py @@ -79,6 +79,10 @@ def single_transaction_make( for parent in parents[1:]: key_source *= parent.proj() + if table.__name__ == "PositionSource": + # PositionSource only uses nwb_file_name - full calls redundant + key_source = dj.U("nwb_file_name") & key_source + for pop_key in (key_source & file_restr).fetch("KEY"): try: table().make(pop_key) diff --git a/src/spyglass/data_import/insert_sessions.py b/src/spyglass/data_import/insert_sessions.py index a5d539e8e..b4dc1d406 100644 --- a/src/spyglass/data_import/insert_sessions.py +++ b/src/spyglass/data_import/insert_sessions.py @@ -114,8 +114,7 @@ def copy_nwb_link_raw_ephys(nwb_file_name, out_nwb_file_name): if debug_mode: return out_nwb_file_abs_path logger.warning( - f"Output file {out_nwb_file_abs_path} exists and will be " - + "overwritten." + f"Output file exists, will be overwritten: {out_nwb_file_abs_path}" ) with pynwb.NWBHDF5IO( diff --git a/src/spyglass/position/position_merge.py b/src/spyglass/position/position_merge.py index ea2d574a2..b6346b938 100644 --- a/src/spyglass/position/position_merge.py +++ b/src/spyglass/position/position_merge.py @@ -1,20 +1,10 @@ import datajoint as dj -import numpy as np -import pandas as pd from datajoint.utils import to_camel_case from spyglass.common.common_position import IntervalPositionInfo as CommonPos -from spyglass.position.v1.dlc_utils import ( - check_videofile, - get_video_path, - make_video, -) -from spyglass.position.v1.position_dlc_pose_estimation import ( - DLCPoseEstimationSelection, -) from spyglass.position.v1.position_dlc_selection import DLCPosV1 from spyglass.position.v1.position_trodes_position import TrodesPosV1 -from spyglass.utils import SpyglassMixin, _Merge, logger +from spyglass.utils import SpyglassMixin, _Merge schema = dj.schema("position_merge") diff --git a/src/spyglass/position/v1/__init__.py b/src/spyglass/position/v1/__init__.py index 20f4cf071..9fc821cb6 100644 --- a/src/spyglass/position/v1/__init__.py +++ b/src/spyglass/position/v1/__init__.py @@ -1,12 +1,12 @@ from .dlc_reader import do_pose_estimation, read_yaml, save_yaml from .dlc_utils import ( _convert_mp4, - check_videofile, find_full_path, + find_mp4, find_root_directory, get_dlc_processed_data_dir, get_dlc_root_data_dir, - get_video_path, + get_video_info, ) from .position_dlc_centroid import ( DLCCentroid, diff --git a/src/spyglass/position/v1/dlc_decorators.py b/src/spyglass/position/v1/dlc_decorators.py deleted file mode 100644 index 111d7508a..000000000 --- a/src/spyglass/position/v1/dlc_decorators.py +++ /dev/null @@ -1,27 +0,0 @@ -## dlc_decorators - - -def accepts(*vals, **kwargs): - is_method = kwargs.pop("is_method", True) - - def check_accepts(f): - if is_method: - assert len(vals) == f.__code__.co_argcount - 1 - else: - assert len(vals) == f.__code__.co_argcount - - def new_f(*args, **kwargs): - pargs = args[1:] if is_method else args - for a, t in zip(pargs, vals): # assume first arg is self or cls - if t is None: - continue - assert a in t, "arg %r is not in %s" % (a, t) - if is_method: - return f(args[0], *pargs, **kwargs) - else: - return f(*args, **kwargs) - - new_f.__name__ = f.__name__ - return new_f - - return check_accepts diff --git a/src/spyglass/position/v1/dlc_reader.py b/src/spyglass/position/v1/dlc_reader.py index caa3c2e5c..c3d8e8af8 100644 --- a/src/spyglass/position/v1/dlc_reader.py +++ b/src/spyglass/position/v1/dlc_reader.py @@ -8,6 +8,7 @@ import pandas as pd import ruamel.yaml as yaml +from spyglass.common.common_usage import ActivityLog from spyglass.settings import test_mode @@ -20,6 +21,7 @@ def __init__( yml_path=None, filename_prefix="", ): + ActivityLog.deprecate_log("dlc_reader: PoseEstimation") if dlc_dir is None: assert pkl_path and h5_path and yml_path, ( 'If "dlc_dir" is not provided, then pkl_path, h5_path, and yml_path ' diff --git a/src/spyglass/position/v1/dlc_utils.py b/src/spyglass/position/v1/dlc_utils.py index 6d27615e4..1523e01b4 100644 --- a/src/spyglass/position/v1/dlc_utils.py +++ b/src/spyglass/position/v1/dlc_utils.py @@ -3,25 +3,25 @@ import grp import logging import os -import pathlib import pwd import subprocess import sys from collections import abc -from contextlib import redirect_stdout -from itertools import groupby +from functools import reduce +from itertools import combinations, groupby from operator import itemgetter +from pathlib import Path, PosixPath from typing import Iterable, Union import datajoint as dj -import matplotlib.pyplot as plt import numpy as np import pandas as pd -from tqdm import tqdm as tqdm +from position_tools import get_distance from spyglass.common.common_behav import VideoFile -from spyglass.settings import dlc_output_dir, dlc_video_dir, raw_dir, test_mode -from spyglass.utils import logger +from spyglass.common.common_usage import ActivityLog +from spyglass.settings import dlc_output_dir, dlc_video_dir, raw_dir +from spyglass.utils.logging import logger, stream_handler def validate_option( @@ -111,6 +111,7 @@ def validate_smooth_params(params): if not params.get("smooth"): return smoothing_params = params.get("smoothing_params") + validate_option(option=smoothing_params, name="smoothing_params") validate_option( option=smoothing_params.get("smooth_method"), name="smooth_method", @@ -145,8 +146,9 @@ def _set_permissions(directory, mode, username: str, groupname: str = None): ------- None """ + ActivityLog().deprecate_log("dlc_utils: _set_permissions") - directory = pathlib.Path(directory) + directory = Path(directory) assert directory.exists(), f"Target directory: {directory} does not exist" uid = pwd.getpwnam(username).pw_uid if groupname: @@ -161,135 +163,52 @@ def _set_permissions(directory, mode, username: str, groupname: str = None): os.chmod(os.path.join(dirpath, filename), mode) -class OutputLogger: # TODO: migrate to spyglass.utils.logger - """ - A class to wrap a logging.Logger object in order to provide context manager capabilities. - - This class uses contextlib.redirect_stdout to temporarily redirect sys.stdout and thus - print statements to the log file instead of, or as well as the console. +def file_log(logger, console=False): + """Decorator to add a file handler to a logger. - Attributes + Parameters ---------- logger : logging.Logger - logger object - name : str - name of logger - level : int - level of logging that the logger is set to handle + Logger to add file handler to. + console : bool, optional + If True, logged info will also be printed to console. Default False. - Methods + Example ------- - setup_logger(name_logfile, path_logfile, print_console=False) - initialize or get logger object with name_logfile - that writes to path_logfile - - Examples - -------- - >>> with OutputLogger(name, path, print_console=True) as logger: - ... print("this will print to logfile") - ... logger.logger.info("this will log to the logfile") - ... print("this will print to the console") - ... logger.logger.info("this will log to the logfile") - + @file_log(logger, console=True) + def func(self, *args, **kwargs): + pass """ - def __init__(self, name, path, level="INFO", **kwargs): - self.logger = self.setup_logger(name, path, **kwargs) - self.name = self.logger.name - self.level = 30 if test_mode else getattr(logging, level) - - def setup_logger( - self, name_logfile, path_logfile, print_console=False - ) -> logging.Logger: - """ - Sets up a logger for that outputs to a file, and optionally, the console - - Parameters - ---------- - name_logfile : str - name of the logfile to use - path_logfile : str - path to the file that should be used as the file handler - print_console : bool, default-False - if True, prints to console as well as log file. - - Returns - ------- - logger : logging.Logger - the logger object with specified handlers - """ - - logger = logging.getLogger(name_logfile) - # check to see if handlers already exist for this logger - if logger.handlers: - for handler in logger.handlers: - # if it's a file handler - # type is used instead of isinstance, - # which doesn't work properly with logging.StreamHandler - if type(handler) == logging.FileHandler: - # if paths don't match, change file handler path - if not os.path.samefile(handler.baseFilename, path_logfile): - handler.close() - logger.removeHandler(handler) - file_handler = self._get_file_handler(path_logfile) - logger.addHandler(file_handler) - # if a stream handler exists and - # if print_console is False remove streamHandler - if type(handler) == logging.StreamHandler: - if not print_console: - handler.close() - logger.removeHandler(handler) - if print_console and not any( - type(handler) == logging.StreamHandler - for handler in logger.handlers - ): - logger.addHandler(self._get_stream_handler()) - - else: - file_handler = self._get_file_handler(path_logfile) + def decorator(func): + def wrapper(self, *args, **kwargs): + if not (log_path := getattr(self, "log_path", None)): + self.log_path = f"temp_{self.__class__.__name__}.log" + file_handler = logging.FileHandler(log_path, mode="a") + file_fmt = logging.Formatter( + "[%(asctime)s][%(levelname)s] Spyglass " + + "%(filename)s:%(lineno)d: %(message)s", + datefmt="%y-%m-%d %H:%M:%S", + ) + file_handler.setFormatter(file_fmt) logger.addHandler(file_handler) - if print_console: - logger.addHandler(self._get_stream_handler()) - logger.setLevel(logging.INFO) - return logger - - def _get_file_handler(self, path): - output_dir = pathlib.Path(os.path.dirname(path)) - if not os.path.exists(output_dir): - output_dir.mkdir(parents=True, exist_ok=True) - file_handler = logging.FileHandler(path, mode="a") - file_handler.setFormatter(self._get_formatter()) - return file_handler - - def _get_stream_handler(self): - stream_handler = logging.StreamHandler() - stream_handler.setFormatter(self._get_formatter()) - return stream_handler - - def _get_formatter(self): - return logging.Formatter( - "[%(asctime)s] in %(pathname)s, line %(lineno)d: %(message)s", - datefmt="%d-%b-%y %H:%M:%S", - ) - - def write(self, msg): - if msg and not msg.isspace(): - self.logger.log(self.level, msg) - - def flush(self): - pass + if not console: + logger.removeHandler(logger.handlers[0]) + try: + return func(self, *args, **kwargs) + finally: + if not console: + logger.addHandler(stream_handler) + logger.removeHandler(file_handler) + file_handler.close() - def __enter__(self): - self._redirector = redirect_stdout(self) - self._redirector.__enter__() - return self + return wrapper - def __exit__(self, exc_type, exc_value, traceback): - # let contextlib do any exception handling here - self._redirector.__exit__(exc_type, exc_value, traceback) + return decorator def get_dlc_root_data_dir(): + ActivityLog().deprecate_log("dlc_utils: get_dlc_root_data_dir") if "custom" in dj.config: if "dlc_root_data_dir" in dj.config["custom"]: dlc_root_dirs = dj.config.get("custom", {}).get("dlc_root_data_dir") @@ -307,13 +226,14 @@ def get_dlc_root_data_dir(): def get_dlc_processed_data_dir() -> str: """Returns session_dir relative to custom 'dlc_output_dir' root""" + ActivityLog().deprecate_log("dlc_utils: get_dlc_processed_data_dir") if "custom" in dj.config: if "dlc_output_dir" in dj.config["custom"]: dlc_output_dir = dj.config.get("custom", {}).get("dlc_output_dir") if dlc_output_dir: - return pathlib.Path(dlc_output_dir) + return Path(dlc_output_dir) else: - return pathlib.Path("/nimbus/deeplabcut/output/") + return Path("/nimbus/deeplabcut/output/") def find_full_path(root_directories, relative_path): @@ -323,15 +243,16 @@ def find_full_path(root_directories, relative_path): from provided potential root directories (in the given order) :param root_directories: potential root directories :param relative_path: the relative path to find the valid root directory - :return: full-path (pathlib.Path object) + :return: full-path (Path object) """ + ActivityLog().deprecate_log("dlc_utils: find_full_path") relative_path = _to_Path(relative_path) if relative_path.exists(): return relative_path # Turn to list if only a single root directory is provided - if isinstance(root_directories, (str, pathlib.Path)): + if isinstance(root_directories, (str, Path)): root_directories = [_to_Path(root_directories)] for root_dir in root_directories: @@ -351,15 +272,16 @@ def find_root_directory(root_directories, full_path): search and return one directory that is the parent of the given path :param root_directories: potential root directories :param full_path: the full path to search the root directory - :return: root_directory (pathlib.Path object) + :return: root_directory (Path object) """ + ActivityLog().deprecate_log("dlc_utils: find_full_path") full_path = _to_Path(full_path) if not full_path.exists(): raise FileNotFoundError(f"{full_path} does not exist!") # Turn to list if only a single root directory is provided - if isinstance(root_directories, (str, pathlib.Path)): + if isinstance(root_directories, (str, Path)): root_directories = [_to_Path(root_directories)] try: @@ -383,8 +305,6 @@ def infer_output_dir(key, makedir=True): ---------- key: DataJoint key specifying a pairing of VideoFile and Model. """ - # TODO: add check to make sure interval_list_name refers to a single epoch - # Or make key include epoch in and of itself instead of interval_list_name file_name = key.get("nwb_file_name") dlc_model_name = key.get("dlc_model_name") @@ -395,29 +315,29 @@ def infer_output_dir(key, makedir=True): "Key must contain 'nwb_file_name', 'dlc_model_name', and 'epoch'" ) - nwb_file_name = file_name.split("_.")[0] - output_dir = pathlib.Path(dlc_output_dir) / pathlib.Path( + nwb_file_name = key["nwb_file_name"].split("_.")[0] + output_dir = Path(dlc_output_dir) / Path( f"{nwb_file_name}/{nwb_file_name}_{key['epoch']:02}" f"_model_" + key["dlc_model_name"].replace(" ", "-") ) - if makedir is True: - if not os.path.exists(output_dir): - output_dir.mkdir(parents=True, exist_ok=True) + if makedir: + output_dir.mkdir(parents=True, exist_ok=True) return output_dir def _to_Path(path): """ - Convert the input "path" into a pathlib.Path object + Convert the input "path" into a Path object Handles one odd Windows/Linux incompatibility of the "\\" """ - return pathlib.Path(str(path).replace("\\", "/")) + return Path(str(path).replace("\\", "/")) -def get_video_path(key): - """ +def get_video_info(key): + """Returns video path for a given key. + Given nwb_file_name and interval_list_name returns specified - video file filename and path + video file filename, path, meters_per_pixel, and timestamps. Parameters ---------- @@ -430,16 +350,21 @@ def get_video_path(key): path to the video file, including video filename video_filename : str filename of the video + meters_per_pixel : float + meters per pixel conversion factor + timestamps : np.array + timestamps of the video """ import pynwb - vf_key = {k: val for k, val in key.items() if k in VideoFile.heading.names} - if not VideoFile & vf_key: - VideoFile()._no_transaction_make(vf_key, verbose=False) + vf_key = {k: val for k, val in key.items() if k in VideoFile.heading} video_query = VideoFile & vf_key + if not video_query: + VideoFile()._no_transaction_make(vf_key, verbose=False) + if len(video_query) != 1: - print(f"Found {len(video_query)} videos for {vf_key}") + logger.warning(f"Found {len(video_query)} videos for {vf_key}") return None, None, None, None video_info = video_query.fetch1() @@ -457,15 +382,13 @@ def get_video_path(key): return video_dir, video_filename, meters_per_pixel, timestamps -def check_videofile( - video_path: Union[str, pathlib.PosixPath], - output_path: Union[str, pathlib.PosixPath] = dlc_video_dir, +def find_mp4( + video_path: Union[str, PosixPath], + output_path: Union[str, PosixPath] = dlc_video_dir, video_filename: str = None, video_filetype: str = "h264", ): - """ - Checks the file extension of a video file to make sure it is .mp4 for - DeepLabCut processes. Converts to MP4 if not already. + """Check for video file and convert to .mp4 if necessary. Parameters ---------- @@ -474,44 +397,50 @@ def check_videofile( output_path : str or PosixPath object path to directory where converted video will be saved video_filename : str, Optional - filename of the video to convert, if not provided, video_filetype must be - and all video files of video_filetype in the directory will be converted + filename of the video to convert, if not provided, video_filetype must + be and all video files of video_filetype in the directory will be + converted video_filetype : str or List, Default 'h264', Optional If video_filename is not provided, all videos of this filetype will be converted to .mp4 Returns ------- - output_files : List of PosixPath objects - paths to converted video file(s) + PosixPath object + path to converted video file """ - if not video_filename: - video_files = pathlib.Path(video_path).glob(f"*.{video_filetype}") - else: - video_files = [pathlib.Path(f"{video_path}/{video_filename}")] - output_files = [] - for video_filepath in video_files: - if video_filepath.exists(): - if video_filepath.suffix == ".mp4": - output_files.append(video_filepath) - continue - video_file = ( - video_filepath.as_posix() - .rsplit(video_filepath.parent.as_posix(), maxsplit=1)[-1] - .split("/")[-1] - ) - output_files.append( - _convert_mp4(video_file, video_path, output_path, videotype="mp4") + if not video_path or not Path(video_path).exists(): + raise FileNotFoundError(f"Video path does not exist: {video_path}") + + video_files = ( + [Path(video_path) / video_filename] + if video_filename + else Path(video_path).glob(f"*.{video_filetype}") + ) + + if len(video_files) != 1: + raise FileNotFoundError( + f"Found {len(video_files)} video files in {video_path}" ) - return output_files + video_filepath = video_files[0] + + if video_filepath.exists() and video_filepath.suffix == ".mp4": + return video_filepath + + video_file = ( + video_filepath.as_posix() + .rsplit(video_filepath.parent.as_posix(), maxsplit=1)[-1] + .split("/")[-1] + ) + return _convert_mp4(video_file, video_path, output_path, videotype="mp4") def _convert_mp4( filename: str, video_path: str, dest_path: str, - videotype: str, + videotype: str = "mp4", count_frames=False, return_output=True, ): @@ -531,107 +460,85 @@ def _convert_mp4( return_output: bool if True returns the destination filename """ + if videotype not in ["mp4"]: + raise NotImplementedError(f"videotype {videotype} not implemented") orig_filename = filename - video_path = pathlib.PurePath( - pathlib.Path(video_path), pathlib.Path(filename) - ) - if videotype not in ["mp4"]: - raise NotImplementedError + video_path = Path(video_path) / filename + dest_filename = os.path.splitext(filename)[0] if ".1" in dest_filename: dest_filename = os.path.splitext(dest_filename)[0] - dest_path = pathlib.Path(f"{dest_path}/{dest_filename}.{videotype}") - convert_command = [ - "ffmpeg", - "-vsync", - "passthrough", - "-i", - f"{video_path.as_posix()}", - "-codec", - "copy", - f"{dest_path.as_posix()}", - ] + dest_path = Path(f"{dest_path}/{dest_filename}.{videotype}") if dest_path.exists(): logger.info(f"{dest_path} already exists, skipping conversion") - else: - try: - sys.stdout.flush() - convert_process = subprocess.Popen( - convert_command, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) - except subprocess.CalledProcessError as err: - raise RuntimeError( - f"command {err.cmd} return with error (code {err.returncode}): {err.output}" - ) from err - out, _ = convert_process.communicate() - logger.info(out.decode("utf-8")) - logger.info(f"finished converting {filename}") - logger.info( - f"Checking that number of packets match between {orig_filename} and {dest_filename}" - ) - num_packets = [] - for file in [video_path, dest_path]: - packets_command = [ - "ffprobe", - "-v", - "error", - "-select_streams", - "v:0", - "-count_packets", - "-show_entries", - "stream=nb_read_packets", - "-of", - "csv=p=0", - file.as_posix(), - ] - frames_command = [ - "ffprobe", - "-v", - "error", - "-select_streams", - "v:0", - "-count_frames", - "-show_entries", - "stream=nb_read_frames", - "-of", - "csv=p=0", - file.as_posix(), - ] - if count_frames: - try: - check_process = subprocess.Popen( - frames_command, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) - except subprocess.CalledProcessError as err: - raise RuntimeError( - f"command {err.cmd} return with error (code {err.returncode}): {err.output}" - ) from err - else: - try: - check_process = subprocess.Popen( - packets_command, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) - except subprocess.CalledProcessError as err: - raise RuntimeError( - f"command {err.cmd} return with error (code {err.returncode}): {err.output}" - ) from err - out, _ = check_process.communicate() - num_packets.append(int(out.decode("utf-8").split("\n")[0])) - print( - f"Number of packets in {orig_filename}: {num_packets[0]}, {dest_filename}: {num_packets[1]}" - ) - assert num_packets[0] == num_packets[1] + return dest_path + + try: + sys.stdout.flush() + convert_process = subprocess.Popen( + [ + "ffmpeg", + "-vsync", + "passthrough", + "-i", + f"{video_path.as_posix()}", + "-codec", + "copy", + f"{dest_path.as_posix()}", + ], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + except subprocess.CalledProcessError as err: + raise RuntimeError( + f"Video convert errored: Code {err.returncode}, {err.output}" + ) from err + out, _ = convert_process.communicate() + logger.info(f"Finished converting {filename}") + + # check packets match orig file + logger.info(f"Checking packets match orig file: {dest_filename}") + orig_packets = _check_packets(video_path, count_frames=count_frames) + dest_packets = _check_packets(dest_path, count_frames=count_frames) + if orig_packets != dest_packets: + logger.warning(f"Conversion error: {orig_filename} -> {dest_filename}") + if return_output: return dest_path +def _check_packets(file, count_frames=False): + checked = "frames" if count_frames else "packets" + try: + check_process = subprocess.Popen( + [ + "ffprobe", + "-v", + "error", + "-select_streams", + "v:0", + f"-count_{checked}", + "-show_entries", + f"stream=nb_read_{checked}", + "-of", + "csv=p=0", + file.as_posix(), + ], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + except subprocess.CalledProcessError as err: + raise RuntimeError( + f"Check packets error: Code {err.returncode}, {err.output}" + ) from err + out, _ = check_process.communicate() + decoded_out = out.decode("utf-8").split("\n")[0] + if decoded_out.isnumeric(): + return int(decoded_out) + raise ValueError(f"Check packets error: {out}") + + def get_gpu_memory(): """Queries the gpu cluster and returns the memory use for each core. This is used to evaluate which GPU cores are available to run jobs on @@ -658,8 +565,7 @@ def output_to_list(x): )[1:] except subprocess.CalledProcessError as err: raise RuntimeError( - f"command {err.cmd} return with error (code {err.returncode}): " - + f"{err.output}" + f"Get GPU memory errored: Code {err.returncode}, {err.output}" ) from err memory_use_values = { i: int(x.split()[0]) for i, x in enumerate(memory_use_info) @@ -668,18 +574,6 @@ def output_to_list(x): def get_span_start_stop(indices): - """_summary_ - - Parameters - ---------- - indices : _type_ - _description_ - - Returns - ------- - _type_ - _description_ - """ span_inds = [] # Get start and stop index of spans of consecutive indices for k, g in groupby(enumerate(indices), lambda x: x[1] - x[0]): @@ -690,62 +584,83 @@ def get_span_start_stop(indices): def interp_pos(dlc_df, spans_to_interp, **kwargs): idx = pd.IndexSlice + + no_x_msg = "Index {ind} has no {coord}point with which to interpolate" + no_interp_msg = "Index {start} to {stop} not interpolated" + max_pts_to_interp = kwargs.get("max_pts_to_interp", float("inf")) + max_cm_to_interp = kwargs.get("max_cm_to_interp", float("inf")) + + def _get_new_dim(dim, span_start, span_stop, start_time, stop_time): + return np.interp( + x=dlc_df.index[span_start : span_stop + 1], + xp=[start_time, stop_time], + fp=[dim[0], dim[-1]], + ) + for ind, (span_start, span_stop) in enumerate(spans_to_interp): + idx_span = idx[span_start:span_stop] + if (span_stop + 1) >= len(dlc_df): - dlc_df.loc[idx[span_start:span_stop], idx["x"]] = np.nan - dlc_df.loc[idx[span_start:span_stop], idx["y"]] = np.nan - print(f"ind: {ind} has no endpoint with which to interpolate") + dlc_df.loc[idx_span, idx[["x", "y"]]] = np.nan + logger.info(no_x_msg.format(ind=ind, coord="end")) continue if span_start < 1: - dlc_df.loc[idx[span_start:span_stop], idx["x"]] = np.nan - dlc_df.loc[idx[span_start:span_stop], idx["y"]] = np.nan - print(f"ind: {ind} has no startpoint with which to interpolate") + dlc_df.loc[idx_span, idx[["x", "y"]]] = np.nan + logger.info(no_x_msg.format(ind=ind, coord="start")) continue + x = [dlc_df["x"].iloc[span_start - 1], dlc_df["x"].iloc[span_stop + 1]] y = [dlc_df["y"].iloc[span_start - 1], dlc_df["y"].iloc[span_stop + 1]] + span_len = int(span_stop - span_start + 1) start_time = dlc_df.index[span_start] stop_time = dlc_df.index[span_stop] - if "max_pts_to_interp" in kwargs: - if span_len > kwargs["max_pts_to_interp"]: - dlc_df.loc[idx[span_start:span_stop], idx["x"]] = np.nan - dlc_df.loc[idx[span_start:span_stop], idx["y"]] = np.nan - print( - f"inds {span_start} to {span_stop} " - f"length: {span_len} not interpolated" - ) - if "max_cm_to_interp" in kwargs: - if ( - np.linalg.norm(np.array([x[0], y[0]]) - np.array([x[1], y[1]])) - > kwargs["max_cm_to_interp"] - ): - dlc_df.loc[idx[start_time:stop_time], idx["x"]] = np.nan - dlc_df.loc[idx[start_time:stop_time], idx["y"]] = np.nan - change = np.linalg.norm( - np.array([x[0], y[0]]) - np.array([x[1], y[1]]) - ) - print( - f"inds {span_start} to {span_stop + 1} " - f"with change in position: {change:.2f} not interpolated" - ) + change = np.linalg.norm(np.array([x[0], y[0]]) - np.array([x[1], y[1]])) + + if span_len > max_pts_to_interp or change > max_cm_to_interp: + dlc_df.loc[idx_span, idx[["x", "y"]]] = np.nan + logger.info(no_interp_msg.format(start=span_start, stop=span_stop)) + if change > max_cm_to_interp: continue - xnew = np.interp( - x=dlc_df.index[span_start : span_stop + 1], - xp=[start_time, stop_time], - fp=[x[0], x[-1]], - ) - ynew = np.interp( - x=dlc_df.index[span_start : span_stop + 1], - xp=[start_time, stop_time], - fp=[y[0], y[-1]], - ) + xnew = _get_new_dim(x, span_start, span_stop, start_time, stop_time) + ynew = _get_new_dim(y, span_start, span_stop, start_time, stop_time) + dlc_df.loc[idx[start_time:stop_time], idx["x"]] = xnew dlc_df.loc[idx[start_time:stop_time], idx["y"]] = ynew return dlc_df +def interp_orientation(df, spans_to_interp, **kwargs): + idx = pd.IndexSlice + no_x_msg = "Index {ind} has no {x}point with which to interpolate" + df_orient = df["orientation"] + + for ind, (span_start, span_stop) in enumerate(spans_to_interp): + idx_span = idx[span_start:span_stop] + if (span_stop + 1) >= len(df): + df.loc[idx_span, idx["orientation"]] = np.nan + logger.info(no_x_msg.format(ind=ind, x="stop")) + continue + if span_start < 1: + df.loc[idx_span, idx["orientation"]] = np.nan + logger.info(no_x_msg.format(ind=ind, x="start")) + continue + + orient = [df_orient.iloc[span_start - 1], df_orient.iloc[span_stop + 1]] + + start_time = df.index[span_start] + stop_time = df.index[span_stop] + orientnew = np.interp( + x=df.index[span_start : span_stop + 1], + xp=[start_time, stop_time], + fp=[orient[0], orient[-1]], + ) + df.loc[idx[start_time:stop_time], idx["orientation"]] = orientnew + return df + + def smooth_moving_avg( interp_df, smoothing_duration: float, sampling_rate: int, **kwargs ): @@ -753,6 +668,7 @@ def smooth_moving_avg( idx = pd.IndexSlice moving_avg_window = int(np.round(smoothing_duration * sampling_rate)) + xy_arr = interp_df.loc[:, idx[("x", "y")]].values smoothed_xy_arr = bn.move_mean( xy_arr, window=moving_avg_window, axis=0, min_count=1 @@ -768,6 +684,71 @@ def smooth_moving_avg( } +def two_pt_head_orientation(pos_df: pd.DataFrame, **params): + """Determines orientation based on vector between two points""" + BP1 = params.pop("bodypart1", None) + BP2 = params.pop("bodypart2", None) + orientation = np.arctan2( + (pos_df[BP1]["y"] - pos_df[BP2]["y"]), + (pos_df[BP1]["x"] - pos_df[BP2]["x"]), + ) + return orientation + + +def no_orientation(pos_df: pd.DataFrame, **params): + fill_value = params.pop("fill_with", np.nan) + n_frames = len(pos_df) + orientation = np.full( + shape=(n_frames), fill_value=fill_value, dtype=np.float16 + ) + return orientation + + +def red_led_bisector_orientation(pos_df: pd.DataFrame, **params): + """Determines orientation based on 2 equally-spaced identifiers + + Identifiers are assumed to be perpendicular to the orientation direction. + A third object is needed to determine forward/backward + """ # timeit reported 3500x improvement for vectorized implementation + LED1 = params.pop("led1", None) + LED2 = params.pop("led2", None) + LED3 = params.pop("led3", None) + + x_vec = pos_df[[LED1, LED2]].diff(axis=1).iloc[:, 0] + y_vec = pos_df[[LED1, LED2]].diff(axis=1).iloc[:, 1] + + y_is_zero = y_vec.eq(0) + perp_direction = pos_df[[LED3]].diff(axis=1) + + # Handling the special case where y_vec is zero all Ys are the same + special_case = ( + y_is_zero + & (pos_df[LED3]["y"] == pos_df[LED1]["y"]) + & (pos_df[LED3]["y"] == pos_df[LED2]["y"]) + ) + if special_case.any(): + raise Exception("Cannot determine head direction from bisector") + + orientation = np.zeros(len(pos_df)) + orientation[y_is_zero & perp_direction.iloc[:, 0].gt(0)] = np.pi / 2 + orientation[y_is_zero & perp_direction.iloc[:, 0].lt(0)] = -np.pi / 2 + + orientation[~y_is_zero & ~x_vec.eq(0)] = np.arctan2( + y_vec[~y_is_zero], x_vec[~x_vec.eq(0)] + ) + + return orientation + + +# Add new functions for orientation calculation here + +_key_to_func_dict = { + "none": no_orientation, + "red_green_orientation": two_pt_head_orientation, + "red_led_bisector": red_led_bisector_orientation, +} + + def fill_nan(variable, video_time, variable_time): video_ind = np.digitize(variable_time, video_time[1:]) @@ -782,8 +763,9 @@ def fill_nan(variable, video_time, variable_time): return filled_variable -def convert_to_pixels(data, frame_size, cm_to_pixels=1.0): +def convert_to_pixels(data, frame_size=None, cm_to_pixels=1.0): """Converts from cm to pixels and flips the y-axis. + Parameters ---------- data : ndarray, shape (n_time, 2) @@ -797,503 +779,157 @@ def convert_to_pixels(data, frame_size, cm_to_pixels=1.0): return data / cm_to_pixels -def make_video( - video_filename, - video_frame_inds, - position_mean, - orientation_mean, - centroids, - likelihoods, - position_time, - video_time=None, - processor="opencv", - frames=None, - percent_frames=1, - output_video_filename="output.mp4", - cm_to_pixels=1.0, - disable_progressbar=False, - crop=None, - arrow_radius=15, - circle_radius=8, -): - import cv2 - - RGB_PINK = (234, 82, 111) - RGB_YELLOW = (253, 231, 76) - # RGB_WHITE = (255, 255, 255) - RGB_BLUE = (30, 144, 255) - RGB_ORANGE = (255, 127, 80) - # "#29ff3e", - # "#ff0073", - # "#ff291a", - # "#1e2cff", - # "#b045f3", - # "#ffe91a", - # ] - if processor == "opencv": - video = cv2.VideoCapture(video_filename) - fourcc = cv2.VideoWriter_fourcc(*"mp4v") - frame_size = (int(video.get(3)), int(video.get(4))) - frame_rate = video.get(5) - if frames is not None: - n_frames = len(frames) - else: - n_frames = int(len(video_frame_inds) * percent_frames) - frames = np.arange(0, n_frames) - print( - f"video save path: {output_video_filename}\n{n_frames} frames in total." - ) - if crop: - crop_offset_x = crop[0] - crop_offset_y = crop[2] - frame_size = (crop[1] - crop[0], crop[3] - crop[2]) - out = cv2.VideoWriter( - output_video_filename, fourcc, frame_rate, frame_size, True +class Centroid: + def __init__(self, pos_df, points, max_LED_separation=None): + if max_LED_separation is None and len(points) != 1: + raise ValueError("max_LED_separation must be provided") + if len(points) not in [1, 2, 4]: + raise ValueError("Invalid number of points") + + self.pos_df = pos_df + self.max_LED_separation = max_LED_separation + self.points_dict = points + self.point_names = list(points.values()) + self.idx = pd.IndexSlice + self.centroid = np.zeros(shape=(len(pos_df), 2)) + self.coords = { + p: pos_df.loc[:, self.idx[p, ("x", "y")]].to_numpy() + for p in self.point_names + } + self.nans = { + p: np.isnan(coord).any(axis=1) for p, coord in self.coords.items() + } + + if len(points) == 1: + self.get_1pt_centroid() + return + if len(points) in [2, 4]: # 4 also requires 2 + self.get_2pt_centroid() + if len(points) == 4: + self.get_4pt_centroid() + + def calc_centroid( + self, + mask: tuple, + points: list = None, + replace: bool = False, + midpoint: bool = False, + logical_or: bool = False, + ): + """Calculate the centroid of the points in the mask + + Parameters + ---------- + mask : Union[tuple, list] + Tuple of masks to apply to the points. Default is np.logical_and + over a tuple. If a list is passed, then np.logical_or is used. + List cannoot be used with logical_or=True + points : list, optional + List of points to calculate the centroid of. For replace, not needed + replace : bool, optional + Special case for replacing mask with nans, by default False + logical_or : bool, optional + Whether to use logical_and or logical_or to combine mask tuple. + """ + if isinstance(mask, list): + mask = [reduce(np.logical_and, m) for m in mask] + + if points is not None: # Check that combinations of points close enough + for pair in combinations(points, 2): + mask = (*mask, ~self.too_sep(pair[0], pair[1])) + + func = np.logical_or if logical_or else np.logical_and + mask = reduce(func, mask) + + if not np.any(mask): + return + if replace: + self.centroid[mask] = np.nan + return + if len(points) == 1: # only one point + self.centroid[mask] = self.coords[points[0]][mask] + return + elif len(points) == 3: + self.coords["midpoint"] = ( + self.coords[points[0]] + self.coords[points[1]] + ) / 2 + points = ["midpoint", points[2]] + coord_arrays = np.array([self.coords[point][mask] for point in points]) + self.centroid[mask] = np.nanmean(coord_arrays, axis=0) + + def too_sep(self, point1, point2): + """Check if points are too far apart""" + return ( + get_distance(self.coords[point1], self.coords[point2]) + >= self.max_LED_separation ) - print(f"video_output: {output_video_filename}") - - # centroids = { - # color: self.fill_nan(data, video_time, position_time) - # for color, data in centroids.items() - # } - if video_time: - position_mean = { - key: fill_nan( - position_mean[key]["position"], video_time, position_time - ) - for key in position_mean.keys() - } - orientation_mean = { - key: fill_nan( - position_mean[key]["orientation"], video_time, position_time - ) - for key in position_mean.keys() - # CBroz: Bug was here, using nonexistent orientation_mean dict - } - print( - f"frames start: {frames[0]}\nvideo_frames start: " - + f"{video_frame_inds[0]}\ncv2 frame ind start: {int(video.get(1))}" + + def get_1pt_centroid(self): + """Passthrough. If point is NaN, then centroid is NaN.""" + PT1 = self.points_dict.get("point1", None) + self.calc_centroid( + mask=(~self.nans[PT1],), + points=[PT1], ) - for time_ind in tqdm( - frames, desc="frames", disable=disable_progressbar - ): - if time_ind == 0: - video.set(1, time_ind + 1) - elif int(video.get(1)) != time_ind - 1: - video.set(1, time_ind - 1) - is_grabbed, frame = video.read() - - if is_grabbed: - frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - if crop: - frame = frame[crop[2] : crop[3], crop[0] : crop[1]].copy() - if time_ind < video_frame_inds[0] - 1: - cv2.putText( - img=frame, - text=f"time_ind: {int(time_ind)} video frame: {int(video.get(1))}", - org=(10, 10), - fontFace=cv2.FONT_HERSHEY_SIMPLEX, - fontScale=0.5, - color=RGB_YELLOW, - thickness=1, - ) - frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR) - out.write(frame) - continue - cv2.putText( - img=frame, - text=f"time_ind: {int(time_ind)} video frame: {int(video.get(1))}", - org=(10, 10), - fontFace=cv2.FONT_HERSHEY_SIMPLEX, - fontScale=0.5, - color=RGB_YELLOW, - thickness=1, - ) - pos_ind = time_ind - video_frame_inds[0] - # red_centroid = centroids["red"][time_ind] - # green_centroid = centroids["green"][time_ind] - for key in position_mean.keys(): - position = position_mean[key][pos_ind] - # if crop: - # position = np.hstack( - # ( - # convert_to_pixels( - # position[0, np.newaxis], - # frame_size, - # cm_to_pixels, - # ) - # - crop_offset_x, - # convert_to_pixels( - # position[1, np.newaxis], - # frame_size, - # cm_to_pixels, - # ) - # - crop_offset_y, - # ) - # ) - # else: - # position = convert_to_pixels(position, frame_size, cm_to_pixels) - position = convert_to_pixels( - position, frame_size, cm_to_pixels - ) - orientation = orientation_mean[key][pos_ind] - if key == "DLC": - color = RGB_BLUE - if key == "Trodes": - color = RGB_ORANGE - if key == "Common": - color = RGB_PINK - if np.all(~np.isnan(position)) & np.all( - ~np.isnan(orientation) - ): - arrow_tip = ( - int( - position[0] + arrow_radius * np.cos(orientation) - ), - int( - position[1] + arrow_radius * np.sin(orientation) - ), - ) - cv2.arrowedLine( - img=frame, - pt1=tuple(position.astype(int)), - pt2=arrow_tip, - color=color, - thickness=4, - line_type=8, - shift=cv2.CV_8U, - tipLength=0.25, - ) - - if np.all(~np.isnan(position)): - cv2.circle( - img=frame, - center=tuple(position.astype(int)), - radius=circle_radius, - color=color, - thickness=-1, - shift=cv2.CV_8U, - ) - # if np.all(~np.isnan(red_centroid)): - # cv2.circle( - # img=frame, - # center=tuple(red_centroid.astype(int)), - # radius=circle_radius, - # color=RGB_YELLOW, - # thickness=-1, - # shift=cv2.CV_8U, - # ) - - # if np.all(~np.isnan(green_centroid)): - # cv2.circle( - # img=frame, - # center=tuple(green_centroid.astype(int)), - # radius=circle_radius, - # color=RGB_PINK, - # thickness=-1, - # shift=cv2.CV_8U, - # ) - - # if np.all(~np.isnan(head_position)) & np.all( - # ~np.isnan(head_orientation) - # ): - # arrow_tip = ( - # int(head_position[0] + arrow_radius * np.cos(head_orientation)), - # int(head_position[1] + arrow_radius * np.sin(head_orientation)), - # ) - # cv2.arrowedLine( - # img=frame, - # pt1=tuple(head_position.astype(int)), - # pt2=arrow_tip, - # color=RGB_WHITE, - # thickness=4, - # line_type=8, - # shift=cv2.CV_8U, - # tipLength=0.25, - # ) - - # if np.all(~np.isnan(head_position)): - # cv2.circle( - # img=frame, - # center=tuple(head_position.astype(int)), - # radius=circle_radius, - # color=RGB_WHITE, - # thickness=-1, - # shift=cv2.CV_8U, - # ) - - frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR) - out.write(frame) - else: - print("not grabbed") - break - print("releasing video") - video.release() - out.release() - print("destroying cv2 windows") - try: - cv2.destroyAllWindows() - except cv2.error: # if cv is already closed or does not have func - pass - print("finished making video with opencv") - return - elif processor == "matplotlib": - import matplotlib.animation as animation - import matplotlib.font_manager as fm - - position_mean = position_mean["DLC"] - orientation_mean = orientation_mean["DLC"] - video_slowdown = 1 - - # Set up formatting for the movie files - window_size = 501 - if likelihoods: - plot_likelihood = True - elif likelihoods is None: - plot_likelihood = False - - window_ind = np.arange(window_size) - window_size // 2 - # Get video frames - assert pathlib.Path( - video_filename - ).exists(), f"Path to video: {video_filename} does not exist" - color_swatch = [ - "#29ff3e", - "#ff0073", - "#ff291a", - "#1e2cff", - "#b045f3", - "#ffe91a", - ] - video = cv2.VideoCapture(video_filename) - fourcc = cv2.VideoWriter_fourcc(*"mp4v") - frame_size = (int(video.get(3)), int(video.get(4))) - frame_rate = video.get(5) - Writer = animation.writers["ffmpeg"] - if frames is not None: - n_frames = len(frames) - else: - n_frames = int(len(video_frame_inds) * percent_frames) - frames = np.arange(0, n_frames) - print( - f"video save path: {output_video_filename}\n{n_frames} frames in total." + def get_2pt_centroid(self): + self.calc_centroid( # Good points + points=self.point_names, + mask=(~self.nans[p] for p in self.point_names), ) - fps = int(np.round(frame_rate / video_slowdown)) - writer = Writer(fps=fps, bitrate=-1) - ret, frame = video.read() - frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - if crop: - frame = frame[crop[2] : crop[3], crop[0] : crop[1]].copy() - crop_offset_x = crop[0] - crop_offset_y = crop[2] - frame_ind = 0 - with plt.style.context("dark_background"): - # Set up plots - fig, axes = plt.subplots( - 2, - 1, - figsize=(8, 6), - gridspec_kw={"height_ratios": [8, 1]}, - constrained_layout=False, + self.calc_centroid(mask=self.nans.values(), replace=True) # All bad + for point in self.point_names: # only one point + self.calc_centroid( + points=[point], + mask=( + ~self.nans[point], + *[self.nans[p] for p in self.point_names if p != point], + ), ) - axes[0].tick_params(colors="white", which="both") - axes[0].spines["bottom"].set_color("white") - axes[0].spines["left"].set_color("white") - image = axes[0].imshow(frame, animated=True) - print(f"frame after init plot: {video.get(1)}") - centroid_plot_objs = { - bodypart: axes[0].scatter( - [], - [], - s=2, - zorder=102, - color=color, - label=f"{bodypart} position", - animated=True, - alpha=0.6, - ) - for color, bodypart in zip(color_swatch, centroids.keys()) - } - centroid_position_dot = axes[0].scatter( - [], - [], - s=5, - zorder=102, - color="#b045f3", - label="centroid position", - animated=True, - alpha=0.6, - ) - (orientation_line,) = axes[0].plot( - [], - [], - color="cyan", - linewidth=1, - animated=True, - label="Orientation", - ) - axes[0].set_xlabel("") - axes[0].set_ylabel("") - ratio = frame_size[1] / frame_size[0] - if crop: - ratio = (crop[3] - crop[2]) / (crop[1] - crop[0]) - x_left, x_right = axes[0].get_xlim() - y_low, y_high = axes[0].get_ylim() - axes[0].set_aspect( - abs((x_right - x_left) / (y_low - y_high)) * ratio - ) - axes[0].spines["top"].set_color("black") - axes[0].spines["right"].set_color("black") - time_delta = pd.Timedelta( - position_time[0] - position_time[0] - ).total_seconds() - axes[0].legend(loc="lower right", fontsize=4) - title = axes[0].set_title( - f"time = {time_delta:3.4f}s\n frame = {frame_ind}", - fontsize=8, - ) - _ = fm.FontProperties(size=12) - axes[0].axis("off") - if plot_likelihood: - likelihood_objs = { - bodypart: axes[1].plot( - [], - [], - color=color, - linewidth=1, - animated=True, - clip_on=False, - label=bodypart, - )[0] - for color, bodypart in zip(color_swatch, likelihoods.keys()) - } - axes[1].set_ylim((0.0, 1)) - print(f"frame_rate: {frame_rate}") - axes[1].set_xlim( - ( - window_ind[0] / frame_rate, - window_ind[-1] / frame_rate, - ) - ) - axes[1].set_xlabel("Time [s]") - axes[1].set_ylabel("Likelihood") - axes[1].set_facecolor("black") - axes[1].spines["top"].set_color("black") - axes[1].spines["right"].set_color("black") - axes[1].legend(loc="upper right", fontsize=4) - progress_bar = tqdm(leave=True, position=0) - progress_bar.reset(total=n_frames) - - def _update_plot(time_ind): - if time_ind == 0: - video.set(1, time_ind + 1) - else: - video.set(1, time_ind - 1) - ret, frame = video.read() - if ret: - frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - if crop: - frame = frame[ - crop[2] : crop[3], crop[0] : crop[1] - ].copy() - image.set_array(frame) - pos_ind = np.where(video_frame_inds == time_ind)[0] - if len(pos_ind) == 0: - centroid_position_dot.set_offsets((np.NaN, np.NaN)) - for bodypart in centroid_plot_objs.keys(): - centroid_plot_objs[bodypart].set_offsets( - (np.NaN, np.NaN) - ) - orientation_line.set_data((np.NaN, np.NaN)) - title.set_text(f"time = {0:3.4f}s\n frame = {time_ind}") - else: - pos_ind = pos_ind[0] - dlc_centroid_data = convert_to_pixels( - position_mean[pos_ind], frame, cm_to_pixels - ) - if crop: - dlc_centroid_data = np.hstack( - ( - convert_to_pixels( - position_mean[pos_ind, 0, np.newaxis], - frame, - cm_to_pixels, - ) - - crop_offset_x, - convert_to_pixels( - position_mean[pos_ind, 1, np.newaxis], - frame, - cm_to_pixels, - ) - - crop_offset_y, - ) - ) - for bodypart in centroid_plot_objs.keys(): - centroid_plot_objs[bodypart].set_offsets( - convert_to_pixels( - centroids[bodypart][pos_ind], - frame, - cm_to_pixels, - ) - ) - centroid_position_dot.set_offsets(dlc_centroid_data) - r = 30 - orientation_line.set_data( - [ - dlc_centroid_data[0], - dlc_centroid_data[0] - + r * np.cos(orientation_mean[pos_ind]), - ], - [ - dlc_centroid_data[1], - dlc_centroid_data[1] - + r * np.sin(orientation_mean[pos_ind]), - ], - ) - # Need to convert times to datetime object probably. - - time_delta = pd.Timedelta( - pd.to_datetime(position_time[pos_ind] * 1e9, unit="ns") - - pd.to_datetime(position_time[0] * 1e9, unit="ns") - ).total_seconds() - title.set_text( - f"time = {time_delta:3.4f}s\n frame = {time_ind}" - ) - likelihood_inds = pos_ind + window_ind - neg_inds = np.where(likelihood_inds < 0)[0] - over_inds = np.where( - likelihood_inds - > (len(likelihoods[list(likelihood_objs.keys())[0]])) - - 1 - )[0] - if len(neg_inds) > 0: - likelihood_inds[neg_inds] = 0 - if len(over_inds) > 0: - likelihood_inds[neg_inds] = -1 - for bodypart in likelihood_objs.keys(): - likelihood_objs[bodypart].set_data( - window_ind / frame_rate, - np.asarray(likelihoods[bodypart][likelihood_inds]), - ) - progress_bar.update() - - return ( - image, - centroid_position_dot, - orientation_line, - title, - ) - - movie = animation.FuncAnimation( - fig, - _update_plot, - frames=frames, - interval=1000 / fps, - blit=True, + def get_4pt_centroid(self): + green = self.points_dict.get("greenLED", None) + red_C = self.points_dict.get("redLED_C", None) + red_L = self.points_dict.get("redLED_L", None) + red_R = self.points_dict.get("redLED_R", None) + + self.calc_centroid( # Good green and center + points=[green, red_C], + mask=(~self.nans[green], ~self.nans[red_C]), + ) + + self.calc_centroid( # green, left/right - average left/right + points=[red_L, red_R, green], + mask=( + ~self.nans[green], + self.nans[red_C], + ~self.nans[red_L], + ~self.nans[red_R], + ), + ) + + self.calc_centroid( # only left/right + points=[red_L, red_R], + mask=( + self.nans[green], + self.nans[red_C], + ~self.nans[red_L], + ~self.nans[red_R], + ), + ) + + for side, other in [red_L, red_R], [red_R, red_L]: + self.calc_centroid( # green and one side are good, others are NaN + points=[side, green], + mask=( + ~self.nans[green], + self.nans[red_C], + ~self.nans[side], + self.nans[other], + ), ) - movie.save(output_video_filename, writer=writer, dpi=400) - video.release() - print("finished making video with matplotlib") - return + + self.calc_centroid( # green is NaN, red center is good + points=[red_C], + mask=(self.nans[green], ~self.nans[red_C]), + ) diff --git a/src/spyglass/position/v1/dlc_utils_makevid.py b/src/spyglass/position/v1/dlc_utils_makevid.py new file mode 100644 index 000000000..0a80254e1 --- /dev/null +++ b/src/spyglass/position/v1/dlc_utils_makevid.py @@ -0,0 +1,562 @@ +# Convenience functions +# some DLC-utils copied from datajoint element-interface utils.py +from pathlib import Path + +import cv2 +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +from tqdm import tqdm as tqdm + +from spyglass.position.v1.dlc_utils import convert_to_pixels as _to_px +from spyglass.position.v1.dlc_utils import fill_nan +from spyglass.utils import logger + +RGB_PINK = (234, 82, 111) +RGB_YELLOW = (253, 231, 76) +RGB_BLUE = (30, 144, 255) +RGB_ORANGE = (255, 127, 80) +RGB_WHITE = (255, 255, 255) +COLOR_SWATCH = [ + "#29ff3e", + "#ff0073", + "#ff291a", + "#1e2cff", + "#b045f3", + "#ffe91a", +] + + +class VideoMaker: + def __init__( + self, + video_filename, + position_mean, + orientation_mean, + centroids, + position_time, + video_frame_inds=None, + likelihoods=None, + processor="opencv", # opencv, opencv-trodes, matplotlib + video_time=None, + frames=None, + percent_frames=1, + output_video_filename="output.mp4", + cm_to_pixels=1.0, + disable_progressbar=False, + crop=None, + arrow_radius=15, + circle_radius=8, + ): + self.video_filename = video_filename + self.video_frame_inds = video_frame_inds + self.position_mean = position_mean + self.orientation_mean = orientation_mean + self.centroids = centroids + self.likelihoods = likelihoods + self.position_time = position_time + self.processor = processor + self.video_time = video_time + self.frames = frames + self.percent_frames = percent_frames + self.output_video_filename = output_video_filename + self.cm_to_pixels = cm_to_pixels + self.disable_progressbar = disable_progressbar + self.crop = crop + self.arrow_radius = arrow_radius + self.circle_radius = circle_radius + + if not Path(self.video_filename).exists(): + raise FileNotFoundError(f"Video not found: {self.video_filename}") + + if frames is None: + self.n_frames = ( + int(self.orientation_mean.shape[0]) + if processor == "opencv-trodes" + else int(len(video_frame_inds) * percent_frames) + ) + self.frames = np.arange(0, self.n_frames) + else: + self.n_frames = len(frames) + + self.tqdm_kwargs = { + "iterable": ( + range(self.n_frames - 1) + if self.processor == "opencv-trodes" + else self.frames + ), + "desc": "frames", + "disable": self.disable_progressbar, + } + + # init for cv + self.video, self.frame_size = None, None + self.frame_rate, self.out = None, None + self.source_map = { + "DLC": RGB_BLUE, + "Trodes": RGB_ORANGE, + "Common": RGB_PINK, + } + + # intit for matplotlib + self.image, self.title, self.progress_bar = None, None, None + self.crop_offset_x = crop[0] if crop else 0 + self.crop_offset_y = crop[2] if crop else 0 + self.centroid_plot_objs, self.centroid_position_dot = None, None + self.orientation_line = None + self.likelihood_objs = None + self.window_ind = np.arange(501) - 501 // 2 + + self.make_video() + + def make_video(self): + if self.processor == "opencv": + self.make_video_opencv() + elif self.processor == "opencv-trodes": + self.make_trodes_video() + elif self.processor == "matplotlib": + self.make_video_matplotlib() + + def _init_video(self): + logger.info(f"Making video: {self.output_video_filename}") + self.video = cv2.VideoCapture(self.video_filename) + self.frame_size = ( + (int(self.video.get(3)), int(self.video.get(4))) + if not self.crop + else ( + self.crop[1] - self.crop[0], + self.crop[3] - self.crop[2], + ) + ) + self.frame_rate = self.video.get(5) + + def _init_cv_video(self): + _ = self._init_video() + self.out = cv2.VideoWriter( + filename=self.output_video_filename, + fourcc=cv2.VideoWriter_fourcc(*"mp4v"), + fps=self.frame_rate, + frameSize=self.frame_size, + isColor=True, + ) + frames_log = ( + f"\tFrames start: {self.frames[0]}\n" if np.any(self.frames) else "" + ) + inds_log = ( + f"\tVideo frame inds: {self.video_frame_inds[0]}\n" + if np.any(self.video_frame_inds) + else "" + ) + logger.info( + f"\n{frames_log}{inds_log}\tcv2 ind start: {int(self.video.get(1))}" + ) + + def _close_cv_video(self): + self.video.release() + self.out.release() + try: + cv2.destroyAllWindows() + except cv2.error: # if cv is already closed or does not have func + pass + logger.info(f"Finished video: {self.output_video_filename}") + + def _get_frame(self, frame, init_only=False, crop_order=(0, 1, 2, 3)): + frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + if init_only or not self.crop: + return frame + x1, x2, y1, y2 = self.crop_order + return frame[ + self.crop[x1] : self.crop[x2], self.crop[y1] : self.crop[y2] + ].copy() + + def _video_set_by_ind(self, time_ind): + if time_ind == 0: + self.video.set(1, time_ind + 1) + elif int(self.video.get(1)) != time_ind - 1: + self.video.set(1, time_ind - 1) + + def _all_num(self, *args): + return all(np.all(~np.isnan(data)) for data in args) + + def _make_arrow( + self, + position, + orientation, + color, + img, + thickness=4, + line_type=8, + tipLength=0.25, + shift=cv2.CV_8U, + ): + if not self._all_num(position, orientation): + return + arrow_tip = ( + int(position[0] + self.arrow_radius * np.cos(orientation)), + int(position[1] + self.arrow_radius * np.sin(orientation)), + ) + cv2.arrowedLine( + img=img, + pt1=tuple(position.astype(int)), + pt2=arrow_tip, + color=color, + thickness=thickness, + line_type=line_type, + tipLength=tipLength, + shift=shift, + ) + + def _make_circle( + self, + data, + color, + img, + radius=None, + thickness=-1, + shift=cv2.CV_8U, + **kwargs, + ): + if not self._all_num(data): + return + cv2.circle( + img=img, + center=tuple(data.astype(int)), + radius=radius or self.circle_radius, + color=color, + thickness=thickness, + shift=shift, + ) + + def make_video_opencv(self): + _ = self._init_cv_video() + + if self.video_time: + self.position_mean = { + key: fill_nan( + self.position_mean[key]["position"], + self.video_time, + self.position_time, + ) + for key in self.position_mean.keys() + } + self.orientation_mean = { + key: fill_nan( + self.position_mean[key]["orientation"], + self.video_time, + self.position_time, + ) + for key in self.position_mean.keys() + } + + for time_ind in tqdm(**self.tqdm_kwargs): + _ = self._video_set_by_ind(time_ind) + + is_grabbed, frame = self.video.read() + + if not is_grabbed: + break + + frame = self._get_frame(frame) + + cv2.putText( + img=frame, + text=f"time_ind: {int(time_ind)} video frame: {int(self.video.get(1))}", + org=(10, 10), + fontFace=cv2.FONT_HERSHEY_SIMPLEX, + fontScale=0.5, + color=RGB_YELLOW, + thickness=1, + ) + + if time_ind < self.video_frame_inds[0] - 1: + self.out.write(self._get_frame(frame, init_only=True)) + continue + + pos_ind = time_ind - self.video_frame_inds[0] + + for key in self.position_mean: + position = _to_px( + data=self.position_mean[key][pos_ind], + cm_to_pixels=self.cm_to_pixels, + ) + orientation = self.orientation_mean[key][pos_ind] + cv_kwargs = { + "img": frame, + "color": self.source_map[key], + } + self._make_arrow(position, orientation, **cv_kwargs) + self._make_circle(data=position, **cv_kwargs) + + self._get_frame(frame, init_only=True) + self.out.write(frame) + self._close_cv_video() + return + + def make_trodes_video(self): + _ = self._init_cv_video() + + if np.any(self.video_time): + centroids = { + color: fill_nan( + variable=data, + video_time=self.video_time, + variable_time=self.position_time, + ) + for color, data in self.centroids.items() + } + position_mean = fill_nan( + self.position_mean, self.video_time, self.position_time + ) + orientation_mean = fill_nan( + self.orientation_mean, self.video_time, self.position_time + ) + + for time_ind in tqdm(**self.tqdm_kwargs): + is_grabbed, frame = self.video.read() + if not is_grabbed: + break + + frame = self._get_frame(frame) + + red_centroid = centroids["red"][time_ind] + green_centroid = centroids["green"][time_ind] + position = position_mean[time_ind] + position = _to_px(data=position, cm_to_pixels=self.cm_to_pixels) + orientation = orientation_mean[time_ind] + + self._make_circle(data=red_centroid, img=frame, color=RGB_YELLOW) + self._make_circle(data=green_centroid, img=frame, color=RGB_PINK) + self._make_arrow( + position=position, + orientation=orientation, + color=RGB_WHITE, + img=frame, + ) + self._make_circle(data=position, img=frame, color=RGB_WHITE) + self._get_frame(frame, init_only=True) + self.out.write(frame) + + self._close_cv_video() + + def make_video_matplotlib(self): + import matplotlib.animation as animation + + self.position_mean = self.position_mean["DLC"] + self.orientation_mean = self.orientation_mean["DLC"] + + _ = self._init_video() + + video_slowdown = 1 + fps = int(np.round(self.frame_rate / video_slowdown)) + Writer = animation.writers["ffmpeg"] + writer = Writer(fps=fps, bitrate=-1) + + ret, frame = self.video.read() + frame = self._get_frame(frame, crop_order=(2, 3, 0, 1)) + + frame_ind = 0 + plt.style.use("dark_background") + fig, axes = plt.subplots( + 2, + 1, + figsize=(8, 6), + gridspec_kw={"height_ratios": [8, 1]}, + constrained_layout=False, + ) + + axes[0].tick_params(colors="white", which="both") + axes[0].spines["bottom"].set_color("white") + axes[0].spines["left"].set_color("white") + self.image = axes[0].imshow(frame, animated=True) + + logger.info(f"frame after init plot: {self.video.get(1)}") + + self.centroid_plot_objs = { + bodypart: axes[0].scatter( + [], + [], + s=2, + zorder=102, + color=color, + label=f"{bodypart} position", + animated=True, + alpha=0.6, + ) + for color, bodypart in zip(COLOR_SWATCH, self.centroids.keys()) + } + self.centroid_position_dot = axes[0].scatter( + [], + [], + s=5, + zorder=102, + color="#b045f3", + label="centroid position", + animated=True, + alpha=0.6, + ) + (self.orientation_line,) = axes[0].plot( + [], + [], + color="cyan", + linewidth=1, + animated=True, + label="Orientation", + ) + + axes[0].set_xlabel("") + axes[0].set_ylabel("") + + ratio = ( + (self.crop[3] - self.crop[2]) / (self.crop[1] - self.crop[0]) + if self.crop + else self.frame_size[1] / self.frame_size[0] + ) + + x_left, x_right = axes[0].get_xlim() + y_low, y_high = axes[0].get_ylim() + + axes[0].set_aspect(abs((x_right - x_left) / (y_low - y_high)) * ratio) + axes[0].spines["top"].set_color("black") + axes[0].spines["right"].set_color("black") + + time_delta = pd.Timedelta( + self.position_time[0] - self.position_time[-1] + ).total_seconds() + + axes[0].legend(loc="lower right", fontsize=4) + self.title = axes[0].set_title( + f"time = {time_delta:3.4f}s\n frame = {frame_ind}", + fontsize=8, + ) + axes[0].axis("off") + + if self.likelihoods: + self.likelihood_objs = { + bodypart: axes[1].plot( + [], + [], + color=color, + linewidth=1, + animated=True, + clip_on=False, + label=bodypart, + )[0] + for color, bodypart in zip( + COLOR_SWATCH, self.likelihoods.keys() + ) + } + axes[1].set_ylim((0.0, 1)) + axes[1].set_xlim( + ( + self.window_ind[0] / self.frame_rate, + self.window_ind[-1] / self.frame_rate, + ) + ) + axes[1].set_xlabel("Time [s]") + axes[1].set_ylabel("Likelihood") + axes[1].set_facecolor("black") + axes[1].spines["top"].set_color("black") + axes[1].spines["right"].set_color("black") + axes[1].legend(loc="upper right", fontsize=4) + + self.progress_bar = tqdm(leave=True, position=0) + self.progress_bar.reset(total=self.n_frames) + + movie = animation.FuncAnimation( + fig, + self._update_plot, + frames=self.frames, + interval=1000 / fps, + blit=True, + ) + movie.save(self.output_video_filename, writer=writer, dpi=400) + self.video.release() + plt.style.use("default") + logger.info("finished making video with matplotlib") + return + + def _get_centroid_data(self, pos_ind): + def centroid_to_px(*idx): + return _to_px( + data=self.position_mean[idx], cm_to_pixels=self.cm_to_pixels + ) + + if not self.crop: + return centroid_to_px(pos_ind) + return np.hstack( + ( + centroid_to_px((pos_ind, 0, np.newaxis)) - self.crop_offset_x, + centroid_to_px((pos_ind, 1, np.newaxis)) - self.crop_offset_y, + ) + ) + + def _set_orient_line(self, frame, pos_ind): + def orient_list(c): + return [c, c + 30 * np.cos(self.orientation_mean[pos_ind])] + + if np.all(np.isnan(self.orientation_mean[pos_ind])): + self.orientation_line.set_data((np.NaN, np.NaN)) + else: + c0, c1 = self._get_centroid_data(pos_ind) + self.orientation_line.set_data(orient_list(c0), orient_list(c1)) + + def _update_plot(self, time_ind, *args): + _ = self._video_set_by_ind(time_ind) + + ret, frame = self.video.read() + if ret: + frame = self._get_frame(frame, crop_order=(2, 3, 0, 1)) + self.image.set_array(frame) + + pos_ind = np.where(self.video_frame_inds == time_ind)[0] + + if len(pos_ind) == 0: + self.centroid_position_dot.set_offsets((np.NaN, np.NaN)) + for bodypart in self.centroid_plot_objs.keys(): + self.centroid_plot_objs[bodypart].set_offsets((np.NaN, np.NaN)) + self.orientation_line.set_data((np.NaN, np.NaN)) + self.title.set_text(f"time = {0:3.4f}s\n frame = {time_ind}") + self.progress_bar.update() + return + + pos_ind = pos_ind[0] + likelihood_inds = pos_ind + self.window_ind + # initial implementation did not cover case of both neg and over < 0 + neg_inds = np.where(likelihood_inds < 0)[0] + likelihood_inds[neg_inds] = 0 if len(neg_inds) > 0 else -1 + + dlc_centroid_data = self._get_centroid_data(pos_ind) + + for bodypart in self.centroid_plot_objs: + self.centroid_plot_objs[bodypart].set_offsets( + _to_px( + data=self.centroids[bodypart][pos_ind], + cm_to_pixels=self.cm_to_pixels, + ) + ) + self.centroid_position_dot.set_offsets(dlc_centroid_data) + _ = self._set_orient_line(frame, pos_ind) + + time_delta = pd.Timedelta( + pd.to_datetime(self.position_time[pos_ind] * 1e9, unit="ns") + - pd.to_datetime(self.position_time[0] * 1e9, unit="ns") + ).total_seconds() + + self.title.set_text(f"time = {time_delta:3.4f}s\n frame = {time_ind}") + for bodypart in self.likelihood_objs.keys(): + self.likelihood_objs[bodypart].set_data( + self.window_ind / self.frame_rate, + np.asarray(self.likelihoods[bodypart][likelihood_inds]), + ) + self.progress_bar.update() + + return ( + self.image, + self.centroid_position_dot, + self.orientation_line, + self.title, + ) + + +def make_video(**kwargs): + VideoMaker(**kwargs) diff --git a/src/spyglass/position/v1/position_dlc_centroid.py b/src/spyglass/position/v1/position_dlc_centroid.py index 70a1c1252..8c8f43258 100644 --- a/src/spyglass/position/v1/position_dlc_centroid.py +++ b/src/spyglass/position/v1/position_dlc_centroid.py @@ -1,4 +1,4 @@ -from functools import reduce +from pathlib import Path import datajoint as dj import numpy as np @@ -9,8 +9,11 @@ from spyglass.common.common_behav import RawPosition from spyglass.common.common_nwbfile import AnalysisNwbfile from spyglass.position.v1.dlc_utils import ( + Centroid, _key_to_smooth_func_dict, + file_log, get_span_start_stop, + infer_output_dir, interp_pos, validate_list, validate_option, @@ -18,7 +21,7 @@ ) from spyglass.position.v1.position_dlc_cohort import DLCSmoothInterpCohort from spyglass.position.v1.position_dlc_position import DLCSmoothInterpParams -from spyglass.utils.dj_mixin import SpyglassMixin +from spyglass.utils import SpyglassMixin, logger schema = dj.schema("position_v1_dlc_centroid") @@ -29,8 +32,6 @@ class DLCCentroidParams(SpyglassMixin, dj.Manual): Parameters for calculating the centroid """ - # TODO: whether to keep all params in a params dict - # or break out into individual secondary keys definition = """ dlc_centroid_params_name: varchar(80) # name for this set of parameters --- @@ -113,7 +114,6 @@ class DLCCentroidSelection(SpyglassMixin, dj.Manual): definition = """ -> DLCSmoothInterpCohort -> DLCCentroidParams - --- """ @@ -130,201 +130,170 @@ class DLCCentroid(SpyglassMixin, dj.Computed): dlc_position_object_id : varchar(80) dlc_velocity_object_id : varchar(80) """ + log_path = None def make(self, key): - from .dlc_utils import OutputLogger, infer_output_dir + output_dir = infer_output_dir(key=key, makedir=False) + self.log_path = Path(output_dir, "log.log") + self._logged_make(key) + logger.info("inserted entry into DLCCentroid") + + def _fetch_pos_df(self, key, bodyparts_to_use): + return pd.concat( + { + bodypart: ( + DLCSmoothInterpCohort.BodyPart + & {**key, **{"bodypart": bodypart}} + ).fetch1_dataframe() + for bodypart in bodyparts_to_use + }, + axis=1, + ) + def _available_bodyparts(self, key): + return (DLCSmoothInterpCohort.BodyPart & key).fetch("bodypart") + + @file_log(logger) + def _logged_make(self, key): + METERS_PER_CM = 0.01 idx = pd.IndexSlice - output_dir = infer_output_dir(key=key, makedir=False) - with OutputLogger( - name=f"{key['nwb_file_name']}_{key['epoch']}_{key['dlc_model_name']}_log", - path=f"{output_dir.as_posix()}/log.log", - print_console=False, - ) as logger: - # Add to Analysis NWB file - analysis_file_name = AnalysisNwbfile().create( # logged - key["nwb_file_name"] - ) - logger.logger.info("-----------------------") - logger.logger.info("Centroid Calculation") - - # Get labels to smooth from Parameters table - cohort_entries = DLCSmoothInterpCohort.BodyPart & key - params = (DLCCentroidParams() & key).fetch1("params") - centroid_method = params.pop("centroid_method") - bodyparts_avail = cohort_entries.fetch("bodypart") - speed_smoothing_std_dev = params.pop("speed_smoothing_std_dev") - - if not centroid_method: - raise ValueError("Please specify a centroid method to use.") - validate_option(option=centroid_method, options=_key_to_func_dict) - - points = params.get("points") - required_points = _key_to_points.get(centroid_method) - validate_list( - required_items=required_points, - option_list=points, - name="params points", - condition=centroid_method, - ) - for point in required_points: - bodypart = points[point] - if bodypart not in bodyparts_avail: - raise ValueError( # TODO: migrate to input validation - "Bodypart in points not in model." - f"\tBodypart {bodypart}" - f"\tIn Model {bodyparts_avail}" - ) - bodyparts_to_use = [points[point] for point in required_points] - - pos_df = pd.concat( - { - bodypart: ( - DLCSmoothInterpCohort.BodyPart - & {**key, **{"bodypart": bodypart}} - ).fetch1_dataframe() - for bodypart in bodyparts_to_use - }, - axis=1, - ) - dt = np.median(np.diff(pos_df.index.to_numpy())) - sampling_rate = 1 / dt - logger.logger.info( - "Calculating centroid with %s", str(centroid_method) - ) - centroid_func = _key_to_func_dict.get(centroid_method) - centroid = centroid_func(pos_df, **params) - centroid_df = pd.DataFrame( - centroid, - columns=["x", "y"], - index=pos_df.index.to_numpy(), - ) - if params["interpolate"]: - if np.any(np.isnan(centroid)): - logger.logger.info("interpolating over NaNs") - nan_inds = ( - pd.isnull(centroid_df.loc[:, idx[("x", "y")]]) - .any(axis=1) - .to_numpy() - .nonzero()[0] - ) - nan_spans = get_span_start_stop(nan_inds) - interp_df = interp_pos( - centroid_df.copy(), nan_spans, **params["interp_params"] - ) - else: - logger.logger.info("no NaNs to interpolate over") - interp_df = centroid_df.copy() - else: - interp_df = centroid_df.copy() - if params["smooth"]: - smoothing_duration = params["smoothing_params"].get( - "smoothing_duration" + logger.info("Centroid Calculation") + + # Get labels to smooth from Parameters table + params = (DLCCentroidParams() & key).fetch1("params") + + points = params.get("points") + centroid_method = params.get("centroid_method") + required_points = _key_to_points.get(centroid_method) + for point in required_points: + if points[point] not in self._available_bodyparts(key): + raise ValueError( + "Bodypart in points not in model." + f"\tBodypart {points[point]}" + f"\tIn Model {self._available_bodyparts(key)}" ) - if not smoothing_duration: - # TODO: remove - validated with `validate_smooth_params` - raise KeyError( - "smoothing_duration needs to be passed within smoothing_params" - ) - dt = np.median(np.diff(pos_df.index.to_numpy())) - sampling_rate = 1 / dt - logger.logger.info("smoothing position") - smooth_func = _key_to_smooth_func_dict[ - params["smoothing_params"]["smooth_method"] - ] - logger.logger.info( - "Smoothing using method: %s", - str(params["smoothing_params"]["smooth_method"]), + bodyparts_to_use = [points[point] for point in required_points] + + pos_df = self._fetch_pos_df(key=key, bodyparts_to_use=bodyparts_to_use) + + logger.info("Calculating centroid") # now done using number of points + centroid = Centroid( + pos_df=pos_df, + points=params.get("points"), + max_LED_separation=params.get("max_LED_separation"), + ).centroid + centroid_df = pd.DataFrame( + centroid, + columns=["x", "y"], + index=pos_df.index.to_numpy(), + ) + + if params.get("interpolate"): + if np.any(np.isnan(centroid)): + logger.info("interpolating over NaNs") + nan_inds = ( + pd.isnull(centroid_df.loc[:, idx[("x", "y")]]) + .any(axis=1) + .to_numpy() + .nonzero()[0] ) - final_df = smooth_func( - interp_df, - sampling_rate=sampling_rate, - **params["smoothing_params"], + nan_spans = get_span_start_stop(nan_inds) + interp_df = interp_pos( + centroid_df.copy(), nan_spans, **params["interp_params"] ) else: - final_df = interp_df.copy() - logger.logger.info("getting velocity") - velocity = get_velocity( - final_df.loc[:, idx[("x", "y")]].to_numpy(), - time=pos_df.index.to_numpy(), - sigma=speed_smoothing_std_dev, - sampling_frequency=sampling_rate, - ) # cm/s - speed = np.sqrt(np.sum(velocity**2, axis=1)) # cm/s - # Create dataframe - velocity_df = pd.DataFrame( - np.concatenate((velocity, speed[:, np.newaxis]), axis=1), - columns=["velocity_x", "velocity_y", "speed"], - index=pos_df.index.to_numpy(), - ) - total_nan = np.sum( - final_df.loc[:, idx[("x", "y")]].isna().any(axis=1) - ) - pretrack_nan = np.sum( - final_df.iloc[:1000].loc[:, idx[("x", "y")]].isna().any(axis=1) - ) - logger.logger.info("total NaNs in centroid dataset: %d", total_nan) - logger.logger.info( - "NaNs in centroid dataset before ind 1000: %d", pretrack_nan - ) - position = pynwb.behavior.Position() - velocity = pynwb.behavior.BehavioralTimeSeries() - if query := (RawPosition & key): - spatial_series = query.fetch_nwb()[0]["raw_position"] - else: - spatial_series = None - - METERS_PER_CM = 0.01 - position.create_spatial_series( - name="position", - timestamps=final_df.index.to_numpy(), - conversion=METERS_PER_CM, - data=final_df.loc[:, idx[("x", "y")]].to_numpy(), - reference_frame=getattr(spatial_series, "reference_frame", ""), - comments=getattr(spatial_series, "comments", "no comments"), - description="x_position, y_position", - ) - velocity.create_timeseries( - name="velocity", - timestamps=velocity_df.index.to_numpy(), - conversion=METERS_PER_CM, - unit="m/s", - data=velocity_df.loc[ - :, idx[("velocity_x", "velocity_y", "speed")] - ].to_numpy(), - comments=getattr(spatial_series, "comments", "no comments"), - description="x_velocity, y_velocity, speed", - ) - velocity.create_timeseries( - name="video_frame_ind", - unit="index", - timestamps=final_df.index.to_numpy(), - data=pos_df[ - pos_df.columns.levels[0][0] - ].video_frame_ind.to_numpy(), - description="video_frame_ind", - comments="no comments", + interp_df = centroid_df.copy() + else: + interp_df = centroid_df.copy() + + sampling_rate = 1 / np.median(np.diff(pos_df.index.to_numpy())) + if params.get("smooth"): + smooth_params = params["smoothing_params"] + dt = np.median(np.diff(pos_df.index.to_numpy())) + sampling_rate = 1 / dt + smooth_func = _key_to_smooth_func_dict[ + smooth_params["smooth_method"] + ] + logger.info( + f"Smoothing using method: {smooth_func.__name__}", ) - nwb_analysis_file = AnalysisNwbfile() - key.update( - { - "analysis_file_name": analysis_file_name, - "dlc_position_object_id": nwb_analysis_file.add_nwb_object( - analysis_file_name, position - ), - "dlc_velocity_object_id": nwb_analysis_file.add_nwb_object( - analysis_file_name, velocity - ), - } + final_df = smooth_func( + interp_df, sampling_rate=sampling_rate, **smooth_params ) + else: + final_df = interp_df.copy() + + logger.info("getting velocity") + velocity = get_velocity( + final_df.loc[:, idx[("x", "y")]].to_numpy(), + time=pos_df.index.to_numpy(), + sigma=params.pop("speed_smoothing_std_dev"), + sampling_frequency=sampling_rate, + ) + speed = np.sqrt(np.sum(velocity**2, axis=1)) # cm/s + velocity_df = pd.DataFrame( + np.concatenate((velocity, speed[:, np.newaxis]), axis=1), + columns=["velocity_x", "velocity_y", "speed"], + index=pos_df.index.to_numpy(), + ) + total_nan = np.sum(final_df.loc[:, idx[("x", "y")]].isna().any(axis=1)) - nwb_analysis_file.add( - nwb_file_name=key["nwb_file_name"], - analysis_file_name=key["analysis_file_name"], - ) - self.insert1(key) - logger.logger.info("inserted entry into DLCCentroid") - AnalysisNwbfile().log(key, table=self.full_table_name) + logger.info(f"total NaNs in centroid dataset: {total_nan}") + spatial_series = (RawPosition() & key).fetch_nwb()[0]["raw_position"] + position = pynwb.behavior.Position() + velocity = pynwb.behavior.BehavioralTimeSeries() + + common_attrs = { + "conversion": METERS_PER_CM, + "comments": spatial_series.comments, + } + position.create_spatial_series( + name="position", + timestamps=final_df.index.to_numpy(), + data=final_df.loc[:, idx[("x", "y")]].to_numpy(), + reference_frame=spatial_series.reference_frame, + description="x_position, y_position", + **common_attrs, + ) + velocity.create_timeseries( + name="velocity", + timestamps=velocity_df.index.to_numpy(), + unit="m/s", + data=velocity_df.loc[ + :, idx[("velocity_x", "velocity_y", "speed")] + ].to_numpy(), + description="x_velocity, y_velocity, speed", + **common_attrs, + ) + velocity.create_timeseries( + name="video_frame_ind", + unit="index", + timestamps=final_df.index.to_numpy(), + data=pos_df[pos_df.columns.levels[0][0]].video_frame_ind.to_numpy(), + description="video_frame_ind", + comments="no comments", + ) + + # Add to Analysis NWB file + analysis_file_name = AnalysisNwbfile().create(key["nwb_file_name"]) + nwb_analysis_file = AnalysisNwbfile() + nwb_analysis_file.add( + nwb_file_name=key["nwb_file_name"], + analysis_file_name=analysis_file_name, + ) + + self.insert1( + { + **key, + "analysis_file_name": analysis_file_name, + "dlc_position_object_id": nwb_analysis_file.add_nwb_object( + analysis_file_name, position + ), + "dlc_velocity_object_id": nwb_analysis_file.add_nwb_object( + analysis_file_name, velocity + ), + } + ) def fetch1_dataframe(self): nwb_data = self.fetch_nwb()[0] @@ -365,458 +334,6 @@ def fetch1_dataframe(self): ) -def four_led_centroid(pos_df: pd.DataFrame, **params): - """Determines the centroid of 4 LEDS on an implant LED ring. - Assumed to be the Green LED, and 3 red LEDs called: redLED_C, redLED_L, redLED_R - By default, uses (greenled + redLED_C) / 2 to calculate centroid - - If Green LED is NaN, but red center LED is not, - then the red center LED is called the centroid - If green and red center LEDs are NaN, but red left and red right LEDs are not, - then the centroid is (redLED_L + redLED_R) / 2 - If red center LED is NaN, but the other 3 LEDS are not, - then the centroid is (greenled + (redLED_L + redLED_R) / 2) / 2 - If red center and left LEDs are NaN, but green and red right LEDs are not, - then the centroid is (greenled + redLED_R) / 2 - If red center and right LEDs are NaN, but green and red left LEDs are not, - then the centroid is (greenled + redLED_L) / 2 - If all red LEDs are NaN, but green LED is not, - then the green LED is called the centroid - If all LEDs are NaN, then the centroid is NaN - - Parameters - ---------- - pos_df : pd.DataFrame - dataframe containing x and y position for each LED of interest, - index is timestamps. Column names specified by params - **params : dict - contains 'greenLED' and 'redLED_C', 'redLED_R', 'redLED_L' keys, - whose values specify the column names in `pos_df` - - Returns - ------- - centroid : np.ndarray - numpy array with shape (n_time, 2) - centroid[0] is the x coord and centroid[1] is the y coord - """ - if not (params.get("max_LED_separation") and params.get("points")): - raise KeyError("max_LED_separation/points need to be passed in params") - - centroid = np.zeros(shape=(len(pos_df), 2)) - idx = pd.IndexSlice - # TODO: this feels messy, clean-up - green_led = params["points"].pop("greenLED", None) - red_led_C = params["points"].pop("redLED_C", None) - red_led_L = params["points"].pop("redLED_L", None) - red_led_R = params["points"].pop("redLED_R", None) - green_nans = pos_df.loc[:, idx[green_led, ("x", "y")]].isna().any(axis=1) - red_C_nans = pos_df.loc[:, idx[red_led_C, ("x", "y")]].isna().any(axis=1) - red_L_nans = pos_df.loc[:, idx[red_led_L, ("x", "y")]].isna().any(axis=1) - red_R_nans = pos_df.loc[:, idx[red_led_R, ("x", "y")]].isna().any(axis=1) - # TODO: implement checks to make sure not rewriting previously set index in centroid - # If all given LEDs are not NaN - dist_between_green_red = get_distance( - pos_df.loc[:, idx[red_led_C, ("x", "y")]].to_numpy(), - pos_df.loc[:, idx[green_led, ("x", "y")]].to_numpy(), - ) - g_c_is_too_separated = ( - dist_between_green_red >= params["max_LED_separation"] - ) - all_good_mask = reduce( - np.logical_and, - ( - ~green_nans, - ~red_C_nans, - ~red_L_nans, - ~red_R_nans, - ~g_c_is_too_separated, - ), - ) - centroid[all_good_mask] = [ - *zip( - ( - pos_df.loc[idx[all_good_mask], idx[red_led_C, "x"]] - + pos_df.loc[idx[all_good_mask], idx[green_led, "x"]] - ) - / 2, - ( - pos_df.loc[idx[all_good_mask], idx[red_led_C, "y"]] - + pos_df.loc[idx[all_good_mask], idx[green_led, "y"]] - ) - / 2, - ) - ] - # If green LED and red center LED are both not NaN - green_red_C = np.logical_and( - ~green_nans, ~red_C_nans, ~g_c_is_too_separated - ) - if np.sum(green_red_C) > 0: - centroid[green_red_C] = [ - *zip( - ( - pos_df.loc[idx[green_red_C], idx[red_led_C, "x"]] - + pos_df.loc[idx[green_red_C], idx[green_led, "x"]] - ) - / 2, - ( - pos_df.loc[idx[green_red_C], idx[red_led_C, "y"]] - + pos_df.loc[idx[green_red_C], idx[green_led, "y"]] - ) - / 2, - ) - ] - # If all given LEDs are NaN - all_bad_mask = reduce( - np.logical_and, (green_nans, red_C_nans, red_L_nans, red_R_nans) - ) - centroid[all_bad_mask, :] = np.nan - # If green LED is NaN, but red center LED is not - no_green_red_C = np.logical_and(green_nans, ~red_C_nans) - if np.sum(no_green_red_C) > 0: - centroid[no_green_red_C] = [ - *zip( - pos_df.loc[idx[no_green_red_C], idx[red_led_C, "x"]], - pos_df.loc[idx[no_green_red_C], idx[red_led_C, "y"]], - ) - ] - # If green and red center LEDs are NaN, but red left and red right LEDs are not - dist_between_left_right = get_distance( - pos_df.loc[:, idx[red_led_L, ("x", "y")]].to_numpy(), - pos_df.loc[:, idx[red_led_R, ("x", "y")]].to_numpy(), - ) - l_r_is_too_separated = ( - dist_between_left_right >= params["max_LED_separation"] - ) - no_green_no_red_C_red_L_red_R = reduce( - np.logical_and, - ( - green_nans, - red_C_nans, - ~red_L_nans, - ~red_R_nans, - ~l_r_is_too_separated, - ), - ) - if np.sum(no_green_no_red_C_red_L_red_R) > 0: - centroid[no_green_no_red_C_red_L_red_R] = [ - *zip( - ( - pos_df.loc[ - idx[no_green_no_red_C_red_L_red_R], idx[red_led_L, "x"] - ] - + pos_df.loc[ - idx[no_green_no_red_C_red_L_red_R], idx[red_led_R, "x"] - ] - ) - / 2, - ( - pos_df.loc[ - idx[no_green_no_red_C_red_L_red_R], idx[red_led_L, "y"] - ] - + pos_df.loc[ - idx[no_green_no_red_C_red_L_red_R], idx[red_led_R, "y"] - ] - ) - / 2, - ) - ] - # If red center LED is NaN, but green, red left, and right LEDs are not - dist_between_left_green = get_distance( - pos_df.loc[:, idx[red_led_L, ("x", "y")]].to_numpy(), - pos_df.loc[:, idx[green_led, ("x", "y")]].to_numpy(), - ) - dist_between_right_green = get_distance( - pos_df.loc[:, idx[red_led_R, ("x", "y")]].to_numpy(), - pos_df.loc[:, idx[green_led, ("x", "y")]].to_numpy(), - ) - l_g_is_too_separated = ( - dist_between_left_green >= params["max_LED_separation"] - ) - r_g_is_too_separated = ( - dist_between_right_green >= params["max_LED_separation"] - ) - green_red_L_red_R_no_red_C = reduce( - np.logical_and, - ( - ~green_nans, - red_C_nans, - ~red_L_nans, - ~red_R_nans, - ~l_r_is_too_separated, - ~l_g_is_too_separated, - ~r_g_is_too_separated, - ), - ) - if np.sum(green_red_L_red_R_no_red_C) > 0: - midpoint = ( - ( - pos_df.loc[idx[green_red_L_red_R_no_red_C], idx[red_led_L, "x"]] - + pos_df.loc[ - idx[green_red_L_red_R_no_red_C], idx[red_led_R, "x"] - ] - ) - / 2, - ( - pos_df.loc[idx[green_red_L_red_R_no_red_C], idx[red_led_L, "y"]] - + pos_df.loc[ - idx[green_red_L_red_R_no_red_C], idx[red_led_R, "y"] - ] - ) - / 2, - ) - centroid[green_red_L_red_R_no_red_C] = [ - *zip( - ( - midpoint[0] - + pos_df.loc[ - idx[green_red_L_red_R_no_red_C], idx[green_led, "x"] - ] - ) - / 2, - ( - midpoint[1] - + pos_df.loc[ - idx[green_red_L_red_R_no_red_C], idx[green_led, "y"] - ] - ) - / 2, - ) - ] - # If red center and left LED is NaN, but green and red right LED are not - green_red_R_no_red_C_no_red_L = reduce( - np.logical_and, - ( - ~green_nans, - red_C_nans, - red_L_nans, - ~red_R_nans, - ~r_g_is_too_separated, - ), - ) - if np.sum(green_red_R_no_red_C_no_red_L) > 0: - centroid[green_red_R_no_red_C_no_red_L] = [ - *zip( - ( - pos_df.loc[ - idx[green_red_R_no_red_C_no_red_L], idx[red_led_R, "x"] - ] - + pos_df.loc[ - idx[green_red_R_no_red_C_no_red_L], idx[green_led, "x"] - ] - ) - / 2, - ( - pos_df.loc[ - idx[green_red_R_no_red_C_no_red_L], idx[red_led_R, "y"] - ] - + pos_df.loc[ - idx[green_red_R_no_red_C_no_red_L], idx[green_led, "y"] - ] - ) - / 2, - ) - ] - # If red center and right LED is NaN, but green and red left LED are not - green_red_L_no_red_C_no_red_R = reduce( - np.logical_and, - ( - ~green_nans, - red_C_nans, - ~red_L_nans, - red_R_nans, - ~l_g_is_too_separated, - ), - ) - if np.sum(green_red_L_no_red_C_no_red_R) > 0: - centroid[green_red_L_no_red_C_no_red_R] = [ - *zip( - ( - pos_df.loc[ - idx[green_red_L_no_red_C_no_red_R], idx[red_led_L, "x"] - ] - + pos_df.loc[ - idx[green_red_L_no_red_C_no_red_R], idx[green_led, "x"] - ] - ) - / 2, - ( - pos_df.loc[ - idx[green_red_L_no_red_C_no_red_R], idx[red_led_L, "y"] - ] - + pos_df.loc[ - idx[green_red_L_no_red_C_no_red_R], idx[green_led, "y"] - ] - ) - / 2, - ) - ] - # If all LEDS are NaN except red left LED - red_L_no_green_no_red_C_no_red_R = reduce( - np.logical_and, (green_nans, red_C_nans, ~red_L_nans, red_R_nans) - ) - if np.sum(red_L_no_green_no_red_C_no_red_R) > 0: - centroid[red_L_no_green_no_red_C_no_red_R] = [ - *zip( - pos_df.loc[ - idx[red_L_no_green_no_red_C_no_red_R], idx[red_led_L, "x"] - ], - pos_df.loc[ - idx[red_L_no_green_no_red_C_no_red_R], idx[red_led_L, "y"] - ], - ) - ] - # If all LEDS are NaN except red right LED - red_R_no_green_no_red_C_no_red_L = reduce( - np.logical_and, (green_nans, red_C_nans, red_L_nans, ~red_R_nans) - ) - if np.sum(red_R_no_green_no_red_C_no_red_L) > 0: - centroid[red_R_no_green_no_red_C_no_red_L] = [ - *zip( - pos_df.loc[ - idx[red_R_no_green_no_red_C_no_red_L], idx[red_led_R, "x"] - ], - pos_df.loc[ - idx[red_R_no_green_no_red_C_no_red_L], idx[red_led_R, "y"] - ], - ) - ] - # If all red LEDs are NaN, but green LED is not - green_no_red = reduce( - np.logical_and, (~green_nans, red_C_nans, red_L_nans, red_R_nans) - ) - if np.sum(green_no_red) > 0: - centroid[green_no_red] = [ - *zip( - pos_df.loc[idx[green_no_red], idx[green_led, "x"]], - pos_df.loc[idx[green_no_red], idx[green_led, "y"]], - ) - ] - too_separated_inds = reduce( - np.logical_or, - ( - g_c_is_too_separated, - l_r_is_too_separated, - l_g_is_too_separated, - r_g_is_too_separated, - ), - ) - if np.sum(too_separated_inds) > 0: - centroid[too_separated_inds, :] = np.nan - return centroid - - -def two_pt_centroid(pos_df: pd.DataFrame, **params): - """ - Determines the centroid of 2 points using (point1 + point2) / 2 - For a given timestamp, if one point is NaN, - then the other point is assigned as the centroid. - If both are NaN, the centroid is NaN - - Parameters - ---------- - pos_df : pd.DataFrame - dataframe containing x and y position for each point of interest, - index is timestamps. Column names specified by params - **params : dict - contains 'point1' and 'point2' keys, - whose values specify the column names in `pos_df` - - Returns - ------- - centroid : np.ndarray - numpy array with shape (n_time, 2) - centroid[0] is the x coord and centroid[1] is the y coord - """ - if not (params.get("max_LED_separation") and params.get("points")): - raise KeyError("max_LED_separation/points need to be passed in params") - - idx = pd.IndexSlice - centroid = np.zeros(shape=(len(pos_df), 2)) - PT1 = params["points"].pop("point1", None) - PT2 = params["points"].pop("point2", None) - pt1_nans = pos_df.loc[:, idx[PT1, ("x", "y")]].isna().any(axis=1) - pt2_nans = pos_df.loc[:, idx[PT2, ("x", "y")]].isna().any(axis=1) - dist_between_points = get_distance( - pos_df.loc[:, idx[PT1, ("x", "y")]].to_numpy(), - pos_df.loc[:, idx[PT2, ("x", "y")]].to_numpy(), - ) - is_too_separated = dist_between_points >= params["max_LED_separation"] - all_good_mask = np.logical_and(~pt1_nans, ~pt2_nans, ~is_too_separated) - centroid[all_good_mask] = [ - *zip( - ( - pos_df.loc[idx[all_good_mask], idx[PT1, "x"]] - + pos_df.loc[idx[all_good_mask], idx[PT2, "x"]] - ) - / 2, - ( - pos_df.loc[idx[all_good_mask], idx[PT1, "y"]] - + pos_df.loc[idx[all_good_mask], idx[PT2, "y"]] - ) - / 2, - ) - ] - # If only point1 is good - pt1_mask = np.logical_and(~pt1_nans, pt2_nans) - if np.sum(pt1_mask) > 0: - centroid[pt1_mask] = [ - *zip( - pos_df.loc[idx[pt1_mask], idx[PT1, "x"]], - pos_df.loc[idx[pt1_mask], idx[PT1, "y"]], - ) - ] - # If only point2 is good - pt2_mask = np.logical_and(pt1_nans, ~pt2_nans) - if np.sum(pt2_mask) > 0: - centroid[pt2_mask] = [ - *zip( - pos_df.loc[idx[pt2_mask], idx[PT2, "x"]], - pos_df.loc[idx[pt2_mask], idx[PT2, "y"]], - ) - ] - # If neither point is not NaN - all_bad_mask = np.logical_and(pt1_nans, pt2_nans) - centroid[all_bad_mask, :] = np.nan - # If LEDs are too far apart - centroid[is_too_separated, :] = np.nan - - return centroid - - -def one_pt_centroid(pos_df: pd.DataFrame, **params): - """ - Passes through the provided point as the centroid - For a given timestamp, if the point is NaN, - then the centroid is NaN - - Parameters - ---------- - pos_df : pd.DataFrame - dataframe containing x and y position for the point of interest, - index is timestamps. Column name specified by params - **params : dict - contains a 'point1' key, - whose value specifies the column name in `pos_df` - - Returns - ------- - centroid : np.ndarray - numpy array with shape (n_time, 2) - centroid[0] is the x coord and centroid[1] is the y coord - """ - if not params.get("points"): - raise KeyError("points need to be passed in params") - idx = pd.IndexSlice - PT1 = params["points"].pop("point1", None) - centroid = pos_df.loc[:, idx[PT1, ("x", "y")]].to_numpy() - return centroid - - -_key_to_func_dict = { - "four_led_centroid": four_led_centroid, - "two_pt_centroid": two_pt_centroid, - "one_pt_centroid": one_pt_centroid, -} _key_to_points = { "four_led_centroid": ["greenLED", "redLED_L", "redLED_C", "redLED_R"], "two_pt_centroid": ["point1", "point2"], diff --git a/src/spyglass/position/v1/position_dlc_cohort.py b/src/spyglass/position/v1/position_dlc_cohort.py index 6cf1f0eee..4883fc335 100644 --- a/src/spyglass/position/v1/position_dlc_cohort.py +++ b/src/spyglass/position/v1/position_dlc_cohort.py @@ -1,13 +1,16 @@ +from pathlib import Path + import datajoint as dj import numpy as np import pandas as pd from spyglass.common.common_nwbfile import AnalysisNwbfile +from spyglass.position.v1.dlc_utils import file_log, infer_output_dir from spyglass.position.v1.position_dlc_pose_estimation import ( # noqa: F401 DLCPoseEstimation, ) from spyglass.position.v1.position_dlc_position import DLCSmoothInterp -from spyglass.utils.dj_mixin import SpyglassMixin +from spyglass.utils import SpyglassMixin, logger schema = dj.schema("position_v1_dlc_cohort") @@ -39,6 +42,7 @@ class DLCSmoothInterpCohort(SpyglassMixin, dj.Computed): -> DLCSmoothInterpCohortSelection --- """ + log_path = None class BodyPart(SpyglassMixin, dj.Part): definition = """ @@ -87,46 +91,40 @@ def fetch1_dataframe(self): ) def make(self, key): - from .dlc_utils import OutputLogger, infer_output_dir - output_dir = infer_output_dir(key=key, makedir=False) - with OutputLogger( - name=f"{key['nwb_file_name']}_{key['epoch']}_{key['dlc_model_name']}_log", - path=f"{output_dir.as_posix()}/log.log", - print_console=False, - ) as logger: - logger.logger.info("-----------------------") - logger.logger.info("Bodypart Cohort") - # from Jen Guidera - self.insert1(key) - cohort_selection = (DLCSmoothInterpCohortSelection & key).fetch1() - table_entries = [] - bodyparts_params_dict = cohort_selection.pop( - "bodyparts_params_dict" + self.log_path = Path(output_dir) / "log.log" + self._logged_make(key) + logger.info("Inserted entry into DLCSmoothInterpCohort") + + @file_log(logger, console=False) + def _logged_make(self, key): + logger.info("-----------------------") + logger.info("Bodypart Cohort") + + cohort_selection = (DLCSmoothInterpCohortSelection & key).fetch1() + table_entries = [] + bp_params_dict = cohort_selection.pop("bodyparts_params_dict") + temp_key = cohort_selection.copy() + for bodypart, params in bp_params_dict.items(): + temp_key["bodypart"] = bodypart + temp_key["dlc_si_params_name"] = params + table_entries.append((DLCSmoothInterp & temp_key).fetch()) + + if not len(table_entries) == len(bp_params_dict): + raise ValueError( + f"Mismatch: DLCSmoothInterp {len(table_entries)} vs " + + f"bodyparts_params_dict {len(bp_params_dict)}" ) - temp_key = cohort_selection.copy() - for bodypart, params in bodyparts_params_dict.items(): - temp_key["bodypart"] = bodypart - temp_key["dlc_si_params_name"] = params - table_entries.append((DLCSmoothInterp & temp_key).fetch()) - assert len(table_entries) == len( - bodyparts_params_dict - ), "more entries found in DLCSmoothInterp than specified in bodyparts_params_dict" - table_column_names = list(table_entries[0].dtype.fields.keys()) - - if len(table_entries) == 0: - raise ValueError( - f"No entries found in DLCSmoothInterp for {temp_key}" - ) - - for table_entry in table_entries: - entry_key = { - **{ - k: v for k, v in zip(table_column_names, table_entry[0]) - }, - **key, - } - DLCSmoothInterpCohort.BodyPart.insert1( - entry_key, skip_duplicates=True - ) - logger.logger.info("Inserted entry into DLCSmoothInterpCohort") + + table_column_names = list(table_entries[0].dtype.fields.keys()) + + part_keys = [ + { + **{k: v for k, v in zip(table_column_names, table_entry[0])}, + **key, + } + for table_entry in table_entries + ] + + self.insert1(key) + self.BodyPart.insert(part_keys, skip_duplicates=True) diff --git a/src/spyglass/position/v1/position_dlc_model.py b/src/spyglass/position/v1/position_dlc_model.py index 5b1cf265b..979e4ddf1 100644 --- a/src/spyglass/position/v1/position_dlc_model.py +++ b/src/spyglass/position/v1/position_dlc_model.py @@ -1,14 +1,12 @@ -import glob import os -from pathlib import Path, PosixPath, PurePath +from pathlib import Path import datajoint as dj import ruamel.yaml as yaml -from spyglass.utils.dj_mixin import SpyglassMixin +from spyglass.utils import SpyglassMixin, logger from . import dlc_reader -from .dlc_decorators import accepts from .position_dlc_project import BodyPart, DLCProject # noqa: F401 from .position_dlc_training import DLCModelTraining # noqa: F401 @@ -31,10 +29,11 @@ class DLCModelInput(SpyglassMixin, dj.Manual): def insert1(self, key, **kwargs): # expects key from DLCProject with config_path project_path = Path(key["config_path"]).parent - assert project_path.exists(), "project path does not exist" + if not project_path.exists(): + raise FileNotFoundError(f"path does not exist: {project_path}") key["dlc_model_name"] = f'{project_path.name.split("model")[0]}model' key["project_path"] = project_path.as_posix() - del key["config_path"] + _ = key.pop("config_path") super().insert1(key, **kwargs) DLCModelSource.insert_entry( dlc_model_name=key["dlc_model_name"], @@ -75,7 +74,6 @@ class FromUpstream(SpyglassMixin, dj.Part): """ @classmethod - @accepts(None, None, ("FromUpstream", "FromImport"), None) def insert_entry( cls, dlc_model_name: str, @@ -144,7 +142,6 @@ class DLCModelSelection(SpyglassMixin, dj.Manual): definition = """ -> DLCModelSource -> DLCModelParams - --- """ @@ -178,34 +175,41 @@ def make(self, key): from deeplabcut.utils.auxiliaryfunctions import GetScorerName _, model_name, table_source = (DLCModelSource & key).fetch1().values() + SourceTable = getattr(DLCModelSource, table_source) params = (DLCModelParams & key).fetch1("params") - project_path = (SourceTable & key).fetch1("project_path") - if not isinstance(project_path, PosixPath): - project_path = Path(project_path) - config_query = PurePath(project_path, Path("*config.y*ml")) - available_config = glob.glob(config_query.as_posix()) - dj_config = [path for path in available_config if "dj_dlc" in path] - if len(dj_config) > 0: - config_path = Path(dj_config[0]) - elif len(available_config) == 1: - config_path = Path(available_config[0]) - else: - config_path = PurePath(project_path, Path("config.yaml")) + project_path = Path((SourceTable & key).fetch1("project_path")) + + available_config = list(project_path.glob("*config.y*ml")) + dj_config = [path for path in available_config if "dj_dlc" in str(path)] + config_path = ( + Path(dj_config[0]) + if len(dj_config) > 0 + else ( + Path(available_config[0]) + if len(available_config) == 1 + else project_path / "config.yaml" + ) + ) + if not config_path.exists(): - raise OSError(f"config_path {config_path} does not exist.") + raise FileNotFoundError(f"config does not exist: {config_path}") + if config_path.suffix in (".yml", ".yaml"): with open(config_path, "rb") as f: safe_yaml = yaml.YAML(typ="safe", pure=True) dlc_config = safe_yaml.load(f) - if isinstance(params["params"], dict): + if isinstance(params.get("params"), dict): dlc_config.update(params["params"]) del params["params"] + # TODO: clean-up. this feels sloppy shuffle = params.pop("shuffle", 1) trainingsetindex = params.pop("trainingsetindex", None) + if not isinstance(trainingsetindex, int): raise KeyError("no trainingsetindex specified in key") + model_prefix = params.pop("model_prefix", "") model_description = params.pop("model_description", model_name) _ = params.pop("dlc_training_params_name", None) @@ -217,10 +221,10 @@ def make(self, key): "snapshotindex", "TrainingFraction", ] - for attribute in needed_attributes: - assert ( - attribute in dlc_config - ), f"Couldn't find {attribute} in config" + if not set(needed_attributes).issubset(set(dlc_config)): + raise KeyError( + f"Missing required config attributes: {needed_attributes}" + ) scorer_legacy = str_to_bool(dlc_config.get("scorer_legacy", "f")) @@ -252,12 +256,12 @@ def make(self, key): # ---- Save DJ-managed config ---- _ = dlc_reader.save_yaml(project_path, dlc_config) - # ____ Insert into table ---- + # --- Insert into table ---- self.insert1(key) self.BodyPart.insert( {**part_key, "bodypart": bp} for bp in dlc_config["bodyparts"] ) - print( + logger.info( f"Finished inserting {model_name}, training iteration" f" {dlc_config['iteration']} into DLCModel" ) diff --git a/src/spyglass/position/v1/position_dlc_orient.py b/src/spyglass/position/v1/position_dlc_orient.py index f64802a59..d87ecf8bc 100644 --- a/src/spyglass/position/v1/position_dlc_orient.py +++ b/src/spyglass/position/v1/position_dlc_orient.py @@ -8,13 +8,26 @@ from spyglass.common.common_behav import RawPosition from spyglass.common.common_nwbfile import AnalysisNwbfile -from spyglass.position.v1.dlc_utils import get_span_start_stop -from spyglass.utils.dj_mixin import SpyglassMixin +from spyglass.position.v1.dlc_utils import ( + get_span_start_stop, + interp_orientation, + no_orientation, + red_led_bisector_orientation, + two_pt_head_orientation, +) +from spyglass.utils import SpyglassMixin, logger from .position_dlc_cohort import DLCSmoothInterpCohort schema = dj.schema("position_v1_dlc_orient") +# Add new functions for orientation calculation here +_key_to_func_dict = { + "none": no_orientation, + "red_green_orientation": two_pt_head_orientation, + "red_led_bisector": red_led_bisector_orientation, +} + @schema class DLCOrientationParams(SpyglassMixin, dj.Manual): @@ -63,8 +76,6 @@ def get_default(cls): @schema class DLCOrientationSelection(SpyglassMixin, dj.Manual): - """ """ - definition = """ -> DLCSmoothInterpCohort -> DLCOrientationParams @@ -85,9 +96,7 @@ class DLCOrientation(SpyglassMixin, dj.Computed): dlc_orientation_object_id : varchar(80) """ - def make(self, key): - # Get labels to smooth from Parameters table - AnalysisNwbfile()._creation_times["pre_create_time"] = time() + def _get_pos_df(self, key): cohort_entries = DLCSmoothInterpCohort.BodyPart & key pos_df = pd.concat( { @@ -99,14 +108,21 @@ def make(self, key): }, axis=1, ) + return pos_df + + def make(self, key): + # Get labels to smooth from Parameters table + AnalysisNwbfile()._creation_times["pre_create_time"] = time() + pos_df = self._get_pos_df(key) + params = (DLCOrientationParams() & key).fetch1("params") orientation_smoothing_std_dev = params.pop( "orientation_smoothing_std_dev", None ) - dt = np.median(np.diff(pos_df.index.to_numpy())) - sampling_rate = 1 / dt + sampling_rate = 1 / np.median(np.diff(pos_df.index.to_numpy())) orient_func = _key_to_func_dict[params["orient_method"]] orientation = orient_func(pos_df, **params) + if not params["orient_method"] == "none": # Smooth orientation is_nan = np.isnan(orientation) @@ -130,6 +146,7 @@ def make(self, key): ) # convert back to between -pi and pi orientation = np.angle(np.exp(1j * orientation)) + final_df = pd.DataFrame( orientation, columns=["orientation"], index=pos_df.index ) @@ -141,6 +158,7 @@ def make(self, key): spatial_series = query.fetch_nwb()[0]["raw_position"] else: spatial_series = None + orientation = pynwb.behavior.CompassDirection() orientation.create_spatial_series( name="orientation", @@ -172,9 +190,7 @@ def fetch1_dataframe(self): ), name="time", ) - COLUMNS = [ - "orientation", - ] + COLUMNS = ["orientation"] return pd.DataFrame( np.asarray(nwb_data["dlc_orientation"].get_spatial_series().data)[ :, np.newaxis @@ -182,97 +198,3 @@ def fetch1_dataframe(self): columns=COLUMNS, index=index, ) - - -def two_pt_head_orientation(pos_df: pd.DataFrame, **params): - """Determines orientation based on vector between two points""" - BP1 = params.pop("bodypart1", None) - BP2 = params.pop("bodypart2", None) - orientation = np.arctan2( - (pos_df[BP1]["y"] - pos_df[BP2]["y"]), - (pos_df[BP1]["x"] - pos_df[BP2]["x"]), - ) - return orientation - - -def no_orientation(pos_df: pd.DataFrame, **params): - fill_value = params.pop("fill_with", np.nan) - n_frames = len(pos_df) - orientation = np.full( - shape=(n_frames), fill_value=fill_value, dtype=np.float16 - ) - return orientation - - -def red_led_bisector_orientation(pos_df: pd.DataFrame, **params): - """Determines orientation based on 2 equally-spaced identifiers - that are assumed to be perpendicular to the orientation direction. - A third object is needed to determine forward/backward - """ - LED1 = params.pop("led1", None) - LED2 = params.pop("led2", None) - LED3 = params.pop("led3", None) - orientation = [] - for index, row in pos_df.iterrows(): - x_vec = row[LED1]["x"] - row[LED2]["x"] - y_vec = row[LED1]["y"] - row[LED2]["y"] - if y_vec == 0: - if (row[LED3]["y"] > row[LED1]["y"]) & ( - row[LED3]["y"] > row[LED2]["y"] - ): - orientation.append(np.pi / 2) - elif (row[LED3]["y"] < row[LED1]["y"]) & ( - row[LED3]["y"] < row[LED2]["y"] - ): - orientation.append(-(np.pi / 2)) - else: - raise Exception("Cannot determine head direction from bisector") - else: - length = np.sqrt(y_vec * y_vec + x_vec * x_vec) - norm = np.array([-y_vec / length, x_vec / length]) - orientation.append(np.arctan2(norm[1], norm[0])) - if index + 1 == len(pos_df): - break - return np.array(orientation) - - -# Add new functions for orientation calculation here - -_key_to_func_dict = { - "none": no_orientation, - "red_green_orientation": two_pt_head_orientation, - "red_led_bisector": red_led_bisector_orientation, -} - - -def interp_orientation(orientation, spans_to_interp, **kwargs): - idx = pd.IndexSlice - # TODO: add parameters to refine interpolation - for ind, (span_start, span_stop) in enumerate(spans_to_interp): - if (span_stop + 1) >= len(orientation): - orientation.loc[idx[span_start:span_stop], idx["orientation"]] = ( - np.nan - ) - print(f"ind: {ind} has no endpoint with which to interpolate") - continue - if span_start < 1: - orientation.loc[idx[span_start:span_stop], idx["orientation"]] = ( - np.nan - ) - print(f"ind: {ind} has no startpoint with which to interpolate") - continue - orient = [ - orientation["orientation"].iloc[span_start - 1], - orientation["orientation"].iloc[span_stop + 1], - ] - start_time = orientation.index[span_start] - stop_time = orientation.index[span_stop] - orientnew = np.interp( - x=orientation.index[span_start : span_stop + 1], - xp=[start_time, stop_time], - fp=[orient[0], orient[-1]], - ) - orientation.loc[idx[start_time:stop_time], idx["orientation"]] = ( - orientnew - ) - return orientation diff --git a/src/spyglass/position/v1/position_dlc_pose_estimation.py b/src/spyglass/position/v1/position_dlc_pose_estimation.py index 6ae7669bf..4b82918a0 100644 --- a/src/spyglass/position/v1/position_dlc_pose_estimation.py +++ b/src/spyglass/position/v1/position_dlc_pose_estimation.py @@ -1,5 +1,5 @@ -import os from datetime import datetime +from pathlib import Path import datajoint as dj import matplotlib.pyplot as plt @@ -14,10 +14,16 @@ convert_epoch_interval_name_to_position_interval_name, ) from spyglass.common.common_nwbfile import AnalysisNwbfile -from spyglass.utils.dj_mixin import SpyglassMixin +from spyglass.position.v1.dlc_utils import ( + file_log, + find_mp4, + get_video_info, + infer_output_dir, +) +from spyglass.position.v1.position_dlc_model import DLCModel +from spyglass.utils import SpyglassMixin, logger -from .dlc_utils import OutputLogger, infer_output_dir -from .position_dlc_model import DLCModel +from . import dlc_reader schema = dj.schema("position_v1_dlc_pose_estimation") @@ -33,6 +39,7 @@ class DLCPoseEstimationSelection(SpyglassMixin, dj.Manual): pose_estimation_output_dir='': varchar(255) # output dir relative to the root dir pose_estimation_params=null : longblob # analyze_videos params, if not default """ + log_path = None @classmethod def get_video_crop(cls, video_path, crop_input=None): @@ -48,6 +55,8 @@ def get_video_crop(cls, video_path, crop_input=None): ------- crop_ints : list list of 4 integers [x min, x max, y min, y max] + crop_input : str, optional + input string to determine cropping parameters. If None, user is queried """ import cv2 @@ -75,9 +84,8 @@ def get_video_crop(cls, video_path, crop_input=None): assert all(isinstance(val, int) for val in crop_ints) return crop_ints - @classmethod def insert_estimation_task( - cls, + self, key, task_mode="trigger", # load or trigger params: dict = None, @@ -98,40 +106,40 @@ def insert_estimation_task( videotype, gputouse, save_as_csv, batchsize, cropping, TFGPUinference, dynamic, robust_nframes, allow_growth, use_shelve """ - from .dlc_utils import check_videofile, get_video_path - - video_path, video_filename, _, _ = get_video_path(key) output_dir = infer_output_dir(key) + self.log_path = Path(output_dir) / "log.log" + self._insert_est_with_log( + key, task_mode, params, check_crop, skip_duplicates, output_dir + ) + logger.info("inserted entry into Pose Estimation Selection") + return {**key, "task_mode": task_mode} - if not video_path: - raise FileNotFoundError(f"Video file not found for {key}") + @file_log(logger, console=False) + def _insert_est_with_log( + self, key, task_mode, params, check_crop, skip_duplicates, output_dir + ): - with OutputLogger( - name=f"{key['nwb_file_name']}_{key['epoch']}_{key['dlc_model_name']}_log", - path=f"{output_dir.as_posix()}/log.log", - ) as logger: - logger.logger.info("Pose Estimation Selection") - video_dir = os.path.dirname(video_path) + "/" - logger.logger.info("video_dir: %s", video_dir) - video_path = check_videofile( - video_path=video_dir, video_filename=video_filename - )[0] - if check_crop is not None: - params["cropping"] = cls.get_video_crop( - video_path=video_path.as_posix() - ) - cls.insert1( - { - **key, - "task_mode": task_mode, - "pose_estimation_params": params, - "video_path": video_path, - "pose_estimation_output_dir": output_dir, - }, - skip_duplicates=skip_duplicates, + v_path, v_fname, _, _ = get_video_info(key) + if not v_path: + raise FileNotFoundError(f"Video file not found for {key}") + logger.info("Pose Estimation Selection") + v_dir = Path(v_path).parent + logger.info("video_dir: %s", v_dir) + v_path = find_mp4(video_path=v_dir, video_filename=v_fname) + if check_crop: + params["cropping"] = self.get_video_crop( + video_path=v_path.as_posix() ) - logger.logger.info("inserted entry into Pose Estimation Selection") - return {**key, "task_mode": task_mode} + self.insert1( + { + **key, + "task_mode": task_mode, + "pose_estimation_params": params, + "video_path": v_path, + "pose_estimation_output_dir": output_dir, + }, + skip_duplicates=skip_duplicates, + ) @schema @@ -154,6 +162,7 @@ class BodyPart(SpyglassMixin, dj.Part): """ _nwb_table = AnalysisNwbfile + log_path = None def fetch1_dataframe(self): nwb_data = self.fetch_nwb()[0] @@ -199,152 +208,147 @@ def fetch1_dataframe(self): def make(self, key): """.populate() method will launch training for each PoseEstimationTask""" - from . import dlc_reader - from .dlc_utils import get_video_path + self.log_path = ( + Path(infer_output_dir(key=key, makedir=False)) / "log.log" + ) + self._logged_make(key) + + @file_log(logger, console=True) + def _logged_make(self, key): METERS_PER_CM = 0.01 - output_dir = infer_output_dir(key=key, makedir=False) - with OutputLogger( - name=f"{key['nwb_file_name']}_{key['epoch']}_{key['dlc_model_name']}_log", - path=f"{output_dir.as_posix()}/log.log", - ) as logger: - logger.logger.info("----------------------") - logger.logger.info("Pose Estimation") - # ID model and directories - dlc_model = (DLCModel & key).fetch1() - bodyparts = (DLCModel.BodyPart & key).fetch("bodypart") - task_mode, analyze_video_params, video_path, output_dir = ( - DLCPoseEstimationSelection & key - ).fetch1( - "task_mode", - "pose_estimation_params", - "video_path", - "pose_estimation_output_dir", + logger.info("----------------------") + logger.info("Pose Estimation") + # ID model and directories + dlc_model = (DLCModel & key).fetch1() + bodyparts = (DLCModel.BodyPart & key).fetch("bodypart") + task_mode, analyze_video_params, video_path, output_dir = ( + DLCPoseEstimationSelection & key + ).fetch1( + "task_mode", + "pose_estimation_params", + "video_path", + "pose_estimation_output_dir", + ) + analyze_video_params = analyze_video_params or {} + + project_path = dlc_model["project_path"] + + # Trigger PoseEstimation + if task_mode == "trigger": + dlc_reader.do_pose_estimation( + video_path, + dlc_model, + project_path, + output_dir, + **analyze_video_params, ) - analyze_video_params = analyze_video_params or {} - - project_path = dlc_model["project_path"] - - # Trigger PoseEstimation - if task_mode == "trigger": - dlc_reader.do_pose_estimation( - video_path, - dlc_model, - project_path, - output_dir, - **analyze_video_params, - ) - dlc_result = dlc_reader.PoseEstimation(output_dir) - creation_time = datetime.fromtimestamp( - dlc_result.creation_time - ).strftime("%Y-%m-%d %H:%M:%S") - - # get video information - _, _, meters_per_pixel, video_time = get_video_path(key) - # check if a position interval exists for this epoch - if interval_list_name := ( - convert_epoch_interval_name_to_position_interval_name( + dlc_result = dlc_reader.PoseEstimation(output_dir) + creation_time = datetime.fromtimestamp( + dlc_result.creation_time + ).strftime("%Y-%m-%d %H:%M:%S") + + logger.info("getting raw position") + interval_list_name = ( + convert_epoch_interval_name_to_position_interval_name( + { + "nwb_file_name": key["nwb_file_name"], + "epoch": key["epoch"], + }, + populate_missing=False, + ) + ) + spatial_series = ( + RawPosition() & {**key, "interval_list_name": interval_list_name} + ).fetch_nwb()[0]["raw_position"] + _, _, _, video_time = get_video_info(key) + pos_time = spatial_series.timestamps + + # TODO: should get timestamps from VideoFile, but need the + # video_frame_ind from RawPosition, which also has timestamps + + key["meters_per_pixel"] = spatial_series.conversion + + # Insert entry into DLCPoseEstimation + logger.info( + "Inserting %s, epoch %02d into DLCPoseEsimation", + key["nwb_file_name"], + key["epoch"], + ) + self.insert1({**key, "pose_estimation_time": creation_time}) + + meters_per_pixel = key.pop("meters_per_pixel") + body_parts = dlc_result.df.columns.levels[0] + body_parts_df = {} + # Insert dlc pose estimation into analysis NWB file for + # each body part. + for body_part in bodyparts: + if body_part in body_parts: + body_parts_df[body_part] = pd.DataFrame.from_dict( { - "nwb_file_name": key["nwb_file_name"], - "epoch": key["epoch"], - }, - populate_missing=False, + c: dlc_result.df.get(body_part).get(c).values + for c in dlc_result.df.get(body_part).columns + } ) - ): - logger.logger.info("Getting raw position") - spatial_series = ( - RawPosition() - & {**key, "interval_list_name": interval_list_name} - ).fetch_nwb()[0]["raw_position"] - else: - spatial_series = None - - key["meters_per_pixel"] = meters_per_pixel - - # Insert entry into DLCPoseEstimation - logger.logger.info( - "Inserting %s, epoch %02d into DLCPoseEsimation", - key["nwb_file_name"], - key["epoch"], + idx = pd.IndexSlice + for body_part, part_df in body_parts_df.items(): + logger.info("converting to cm") + part_df = convert_to_cm(part_df, meters_per_pixel) + logger.info("adding timestamps to DataFrame") + part_df = add_timestamps( + part_df, pos_time=pos_time, video_time=video_time ) - self.insert1({**key, "pose_estimation_time": creation_time}) - meters_per_pixel = key["meters_per_pixel"] - del key["meters_per_pixel"] - body_parts = dlc_result.df.columns.levels[0] - body_parts_df = {} - # Insert dlc pose estimation into analysis NWB file for - # each body part. - for body_part in bodyparts: - if body_part in body_parts: - body_parts_df[body_part] = pd.DataFrame.from_dict( - { - c: dlc_result.df.get(body_part).get(c).values - for c in dlc_result.df.get(body_part).columns - } - ) - idx = pd.IndexSlice - for body_part, part_df in body_parts_df.items(): - logger.logger.info("converting to cm") - key["analysis_file_name"] = AnalysisNwbfile().create( # logged - key["nwb_file_name"] - ) - part_df = convert_to_cm(part_df, meters_per_pixel) - logger.logger.info("adding timestamps to DataFrame") - part_df = add_timestamps( - part_df, - pos_time=getattr(spatial_series, "timestamps", video_time), - video_time=video_time, - ) - key["bodypart"] = body_part - position = pynwb.behavior.Position() - likelihood = pynwb.behavior.BehavioralTimeSeries() - position.create_spatial_series( - name="position", - timestamps=part_df.time.to_numpy(), - conversion=METERS_PER_CM, - data=part_df.loc[:, idx[("x", "y")]].to_numpy(), - reference_frame=getattr( - spatial_series, "reference_frame", "" - ), - comments=getattr(spatial_series, "comments", "no commwnts"), - description="x_position, y_position", - ) - likelihood.create_timeseries( - name="likelihood", - timestamps=part_df.time.to_numpy(), - data=part_df.loc[:, idx["likelihood"]].to_numpy(), - unit="likelihood", - comments="no comments", - description="likelihood", - ) - likelihood.create_timeseries( - name="video_frame_ind", - timestamps=part_df.time.to_numpy(), - data=part_df.loc[:, idx["video_frame_ind"]].to_numpy(), - unit="index", - comments="no comments", - description="video_frame_ind", - ) - nwb_analysis_file = AnalysisNwbfile() - key["dlc_pose_estimation_position_object_id"] = ( - nwb_analysis_file.add_nwb_object( - analysis_file_name=key["analysis_file_name"], - nwb_object=position, - ) - ) - key["dlc_pose_estimation_likelihood_object_id"] = ( - nwb_analysis_file.add_nwb_object( - analysis_file_name=key["analysis_file_name"], - nwb_object=likelihood, - ) + key["bodypart"] = body_part + key["analysis_file_name"] = AnalysisNwbfile().create( + key["nwb_file_name"] + ) + position = pynwb.behavior.Position() + likelihood = pynwb.behavior.BehavioralTimeSeries() + position.create_spatial_series( + name="position", + timestamps=part_df.time.to_numpy(), + conversion=METERS_PER_CM, + data=part_df.loc[:, idx[("x", "y")]].to_numpy(), + reference_frame=spatial_series.reference_frame, + comments=spatial_series.comments, + description="x_position, y_position", + ) + likelihood.create_timeseries( + name="likelihood", + timestamps=part_df.time.to_numpy(), + data=part_df.loc[:, idx["likelihood"]].to_numpy(), + unit="likelihood", + comments="no comments", + description="likelihood", + ) + likelihood.create_timeseries( + name="video_frame_ind", + timestamps=part_df.time.to_numpy(), + data=part_df.loc[:, idx["video_frame_ind"]].to_numpy(), + unit="index", + comments="no comments", + description="video_frame_ind", + ) + nwb_analysis_file = AnalysisNwbfile() + key["dlc_pose_estimation_position_object_id"] = ( + nwb_analysis_file.add_nwb_object( + analysis_file_name=key["analysis_file_name"], + nwb_object=position, ) - nwb_analysis_file.add( - nwb_file_name=key["nwb_file_name"], + ) + key["dlc_pose_estimation_likelihood_object_id"] = ( + nwb_analysis_file.add_nwb_object( analysis_file_name=key["analysis_file_name"], + nwb_object=likelihood, ) - self.BodyPart.insert1(key) - AnalysisNwbfile().log(key, table=self.full_table_name) + ) + nwb_analysis_file.add( + nwb_file_name=key["nwb_file_name"], + analysis_file_name=key["analysis_file_name"], + ) + self.BodyPart.insert1(key) + AnalysisNwbfile().log(key, table=self.full_table_name) def fetch_dataframe(self, *attrs, **kwargs): entries = (self.BodyPart & self).fetch("KEY") @@ -362,12 +366,7 @@ def fetch_dataframe(self, *attrs, **kwargs): ), name="time", ) - COLUMNS = [ - "video_frame_ind", - "x", - "y", - "likelihood", - ] + COLUMNS = ["video_frame_ind", "x", "y", "likelihood"] return pd.concat( { entry["bodypart"]: pd.DataFrame( diff --git a/src/spyglass/position/v1/position_dlc_position.py b/src/spyglass/position/v1/position_dlc_position.py index c18eafd62..cfad61c15 100644 --- a/src/spyglass/position/v1/position_dlc_position.py +++ b/src/spyglass/position/v1/position_dlc_position.py @@ -1,4 +1,4 @@ -from time import time +from pathlib import Path import datajoint as dj import numpy as np @@ -8,15 +8,16 @@ from spyglass.common.common_nwbfile import AnalysisNwbfile from spyglass.position.v1.dlc_utils import ( _key_to_smooth_func_dict, + file_log, get_span_start_stop, + infer_output_dir, interp_pos, validate_option, validate_smooth_params, ) +from spyglass.position.v1.position_dlc_pose_estimation import DLCPoseEstimation from spyglass.settings import test_mode -from spyglass.utils.dj_mixin import SpyglassMixin - -from .position_dlc_pose_estimation import DLCPoseEstimation +from spyglass.utils import SpyglassMixin, logger schema = dj.schema("position_v1_dlc_position") @@ -34,11 +35,12 @@ class DLCSmoothInterpParams(SpyglassMixin, dj.Manual): whether to smooth the dataset smoothing_params : dict smoothing_duration : float, default 0.05 - number of frames to smooth over: sampling_rate*smoothing_duration = num_frames + number of frames to smooth over: + sampling_rate*smoothing_duration = num_frames interp_params : dict max_cm_to_interp : int, default 20 - maximum distance between high likelihood points on either side of a NaN span - to interpolate over + maximum distance between high likelihood points on either side of a + NaN span to interpolate over likelihood_thresh : float, default 0.95 likelihood below which to NaN and interpolate over """ @@ -127,7 +129,7 @@ def insert1(self, key, **kwargs): validate_option( params.get("likelihood_thresh"), name="likelihood_thresh", - types=(float), + types=float, val_range=(0, 1), ) @@ -139,8 +141,6 @@ class DLCSmoothInterpSelection(SpyglassMixin, dj.Manual): definition = """ -> DLCPoseEstimation.BodyPart -> DLCSmoothInterpParams - --- - """ @@ -158,125 +158,118 @@ class DLCSmoothInterp(SpyglassMixin, dj.Computed): dlc_smooth_interp_position_object_id : varchar(80) dlc_smooth_interp_info_object_id : varchar(80) """ + log_path = None def make(self, key): - from .dlc_utils import OutputLogger, infer_output_dir + self.log_path = ( + Path(infer_output_dir(key=key, makedir=False)) / "log.log" + ) + self._logged_make(key) + logger.info("inserted entry into DLCSmoothInterp") + + @file_log(logger, console=False) + def _logged_make(self, key): METERS_PER_CM = 0.01 - output_dir = infer_output_dir(key=key, makedir=False) - with OutputLogger( - name=f"{key['nwb_file_name']}_{key['epoch']}_{key['dlc_model_name']}_log", - path=f"{output_dir.as_posix()}/log.log", - print_console=False, - ) as logger: - AnalysisNwbfile()._creation_times["pre_create_time"] = time() - logger.logger.info("-----------------------") - idx = pd.IndexSlice - # Get labels to smooth from Parameters table - params = (DLCSmoothInterpParams() & key).fetch1("params") - # Get DLC output dataframe - logger.logger.info("fetching Pose Estimation Dataframe") - - bp_key = key.copy() - if test_mode: # during testing, analysis_file not in BodyPart table - bp_key.pop("analysis_file_name", None) - - dlc_df = (DLCPoseEstimation.BodyPart() & bp_key).fetch1_dataframe() + logger.info("-----------------------") + idx = pd.IndexSlice + # Get labels to smooth from Parameters table + params = (DLCSmoothInterpParams() & key).fetch1("params") + # Get DLC output dataframe + logger.info("fetching Pose Estimation Dataframe") + + bp_key = key.copy() + if test_mode: # during testing, analysis_file not in BodyPart table + bp_key.pop("analysis_file_name", None) + + dlc_df = (DLCPoseEstimation.BodyPart() & bp_key).fetch1_dataframe() + dt = np.median(np.diff(dlc_df.index.to_numpy())) + logger.info("Identifying indices to NaN") + df_w_nans, bad_inds = nan_inds( + dlc_df.copy(), + max_dist_between=params["max_cm_between_pts"], + likelihood_thresh=params.pop("likelihood_thresh"), + inds_to_span=params["num_inds_to_span"], + ) + + nan_spans = get_span_start_stop(np.where(bad_inds)[0]) + + if interp_params := params.get("interpolate"): + logger.info("interpolating across low likelihood times") + interp_df = interp_pos(df_w_nans.copy(), nan_spans, **interp_params) + else: + interp_df = df_w_nans.copy() + logger.info("skipping interpolation") + + if params.get("smooth"): + smooth_params = params.get("smoothing_params") + smooth_method = smooth_params.get("smooth_method") + smooth_func = _key_to_smooth_func_dict[smooth_method] + dt = np.median(np.diff(dlc_df.index.to_numpy())) - sampling_rate = 1 / dt - logger.logger.info("Identifying indices to NaN") - df_w_nans, bad_inds = nan_inds( - dlc_df.copy(), - params["max_cm_between_pts"], - likelihood_thresh=params.pop("likelihood_thresh"), - inds_to_span=params["num_inds_to_span"], + logger.info(f"Smoothing using method: {smooth_method}") + smooth_df = smooth_func( + interp_df, + smoothing_duration=smooth_params.get("smoothing_duration"), + sampling_rate=1 / dt, + **params["smoothing_params"], ) + else: + smooth_df = interp_df.copy() + logger.info("skipping smoothing") + + final_df = smooth_df.drop(["likelihood"], axis=1) + final_df = final_df.rename_axis("time").reset_index() + position_nwb_data = ( + (DLCPoseEstimation.BodyPart() & bp_key) + .fetch_nwb()[0]["dlc_pose_estimation_position"] + .get_spatial_series() + ) + key["analysis_file_name"] = AnalysisNwbfile().create( + key["nwb_file_name"] + ) - nan_spans = get_span_start_stop(np.where(bad_inds)[0]) - if params["interpolate"]: - logger.logger.info("interpolating across low likelihood times") - interp_df = interp_pos( - df_w_nans.copy(), nan_spans, **params["interp_params"] - ) - else: - interp_df = df_w_nans.copy() - logger.logger.info("skipping interpolation") - if params["smooth"]: - if "smoothing_duration" in params["smoothing_params"]: - smoothing_duration = params["smoothing_params"].pop( - "smoothing_duration" - ) - dt = np.median(np.diff(dlc_df.index.to_numpy())) - sampling_rate = 1 / dt - logger.logger.info("smoothing position") - smooth_func = _key_to_smooth_func_dict[ - params["smoothing_params"]["smooth_method"] - ] - logger.logger.info( - "Smoothing using method: %s", - str(params["smoothing_params"]["smooth_method"]), - ) - smooth_df = smooth_func( - interp_df, - smoothing_duration=smoothing_duration, - sampling_rate=sampling_rate, - **params["smoothing_params"], - ) - else: - smooth_df = interp_df.copy() - logger.logger.info("skipping smoothing") - final_df = smooth_df.drop(["likelihood"], axis=1) - final_df = final_df.rename_axis("time").reset_index() - position_nwb_data = ( - (DLCPoseEstimation.BodyPart() & bp_key) - .fetch_nwb()[0]["dlc_pose_estimation_position"] - .get_spatial_series() - ) - key["analysis_file_name"] = AnalysisNwbfile().create( # logged - key["nwb_file_name"] - ) - # Add dataframe to AnalysisNwbfile - nwb_analysis_file = AnalysisNwbfile() - position = pynwb.behavior.Position() - video_frame_ind = pynwb.behavior.BehavioralTimeSeries() - logger.logger.info("Creating NWB objects") - position.create_spatial_series( - name="position", - timestamps=final_df.time.to_numpy(), - conversion=METERS_PER_CM, - data=final_df.loc[:, idx[("x", "y")]].to_numpy(), - reference_frame=position_nwb_data.reference_frame, - comments=position_nwb_data.comments, - description="x_position, y_position", - ) - video_frame_ind.create_timeseries( - name="video_frame_ind", - timestamps=final_df.time.to_numpy(), - data=final_df.loc[:, idx["video_frame_ind"]].to_numpy(), - unit="index", - comments="no comments", - description="video_frame_ind", - ) - key["dlc_smooth_interp_position_object_id"] = ( - nwb_analysis_file.add_nwb_object( - analysis_file_name=key["analysis_file_name"], - nwb_object=position, - ) - ) - key["dlc_smooth_interp_info_object_id"] = ( - nwb_analysis_file.add_nwb_object( - analysis_file_name=key["analysis_file_name"], - nwb_object=video_frame_ind, - ) + # Add dataframe to AnalysisNwbfile + nwb_analysis_file = AnalysisNwbfile() + position = pynwb.behavior.Position() + video_frame_ind = pynwb.behavior.BehavioralTimeSeries() + logger.info("Creating NWB objects") + position.create_spatial_series( + name="position", + timestamps=final_df.time.to_numpy(), + conversion=METERS_PER_CM, + data=final_df.loc[:, idx[("x", "y")]].to_numpy(), + reference_frame=position_nwb_data.reference_frame, + comments=position_nwb_data.comments, + description="x_position, y_position", + ) + video_frame_ind.create_timeseries( + name="video_frame_ind", + timestamps=final_df.time.to_numpy(), + data=final_df.loc[:, idx["video_frame_ind"]].to_numpy(), + unit="index", + comments="no comments", + description="video_frame_ind", + ) + key["dlc_smooth_interp_position_object_id"] = ( + nwb_analysis_file.add_nwb_object( + analysis_file_name=key["analysis_file_name"], + nwb_object=position, ) - nwb_analysis_file.add( - nwb_file_name=key["nwb_file_name"], + ) + key["dlc_smooth_interp_info_object_id"] = ( + nwb_analysis_file.add_nwb_object( analysis_file_name=key["analysis_file_name"], + nwb_object=video_frame_ind, ) - self.insert1(key) - logger.logger.info("inserted entry into DLCSmoothInterp") - AnalysisNwbfile().log(key, table=self.full_table_name) + ) + nwb_analysis_file.add( + nwb_file_name=key["nwb_file_name"], + analysis_file_name=key["analysis_file_name"], + ) + self.insert1(key) + AnalysisNwbfile().log(key, table=self.full_table_name) def fetch1_dataframe(self): nwb_data = self.fetch_nwb()[0] @@ -356,6 +349,7 @@ def nan_inds( start_point = good_start[int(len(good_start) // 2)] else: start_point = span[0] + int(span_length(span) // 2) + for ind in range(start_point, span[0], -1): if subthresh_inds_mask[ind]: continue @@ -366,10 +360,11 @@ def nan_inds( ~subthresh_inds_mask[ind + 1 : start_point], ) )[0] - if len(previous_good_inds) >= 1: - last_good_ind = ind + 1 + np.min(previous_good_inds) - else: - last_good_ind = start_point + last_good_ind = ( + ind + 1 + np.min(previous_good_inds) + if len(previous_good_inds) > 0 + else start_point + ) good_x, good_y = dlc_df.loc[ idx[dlc_df.index[last_good_ind]], ["x", "y"] ] @@ -437,36 +432,34 @@ def get_good_spans(bad_inds_mask, inds_to_span: int = 50): modified_spans : list spans that are amended to bridge up to inds_to_span consecutive bad indices """ - good_spans = get_span_start_stop( - np.arange(len(bad_inds_mask))[~bad_inds_mask] - ) - if len(good_spans) > 1: - modified_spans = [] - for (start1, stop1), (start2, stop2) in zip( - good_spans[:-1], good_spans[1:] - ): - check_existing = [ - entry - for entry in modified_spans - if start1 - in range(entry[0] - inds_to_span, entry[1] + inds_to_span) - ] - if len(check_existing) > 0: - modify_ind = modified_spans.index(check_existing[0]) - if (start2 - stop1) <= inds_to_span: - modified_spans[modify_ind] = (check_existing[0][0], stop2) - else: - modified_spans[modify_ind] = (check_existing[0][0], stop1) - modified_spans.append((start2, stop2)) - continue + good = get_span_start_stop(np.arange(len(bad_inds_mask))[~bad_inds_mask]) + + if len(good) < 1: + return None, good + elif len(good) == 1: # if all good, no need to modify + return good, good + + modified_spans = [] + for (start1, stop1), (start2, stop2) in zip(good[:-1], good[1:]): + check_existing = [ + entry + for entry in modified_spans + if start1 in range(entry[0] - inds_to_span, entry[1] + inds_to_span) + ] + if len(check_existing) > 0: + modify_ind = modified_spans.index(check_existing[0]) if (start2 - stop1) <= inds_to_span: - modified_spans.append((start1, stop2)) + modified_spans[modify_ind] = (check_existing[0][0], stop2) else: - modified_spans.append((start1, stop1)) + modified_spans[modify_ind] = (check_existing[0][0], stop1) modified_spans.append((start2, stop2)) - return good_spans, modified_spans - else: - return None, good_spans + continue + if (start2 - stop1) <= inds_to_span: + modified_spans.append((start1, stop2)) + else: + modified_spans.append((start1, stop1)) + modified_spans.append((start2, stop2)) + return good, modified_spans def span_length(x): diff --git a/src/spyglass/position/v1/position_dlc_project.py b/src/spyglass/position/v1/position_dlc_project.py index 87ca4fab7..f2d377aef 100644 --- a/src/spyglass/position/v1/position_dlc_project.py +++ b/src/spyglass/position/v1/position_dlc_project.py @@ -1,20 +1,17 @@ import copy -import glob -import os import shutil from itertools import combinations from pathlib import Path, PosixPath from typing import Dict, List, Union import datajoint as dj -import numpy as np import pandas as pd -import ruamel.yaml +from ruamel.yaml import YAML from spyglass.common.common_lab import LabTeam -from spyglass.position.v1.dlc_utils import check_videofile, get_video_path +from spyglass.position.v1.dlc_utils import find_mp4, get_video_info from spyglass.settings import dlc_project_dir, dlc_video_dir -from spyglass.utils.dj_mixin import SpyglassMixin +from spyglass.utils import SpyglassMixin, logger schema = dj.schema("position_v1_dlc_project") @@ -51,7 +48,7 @@ def add_from_config(cls, bodyparts: List, descriptions: List = None): bodyparts_dict = [ {"bodypart": bp, "bodypart_description": bp} for bp in bodyparts ] - cls.insert(bodyparts_dict, skip_duplicates=True) + cls().insert(bodyparts_dict, skip_duplicates=True) @schema @@ -90,14 +87,20 @@ class File(SpyglassMixin, dj.Part): """ def insert1(self, key, **kwargs): - assert isinstance( - key["project_name"], str - ), "project_name must be a string" - assert isinstance( - key["frames_per_video"], int - ), "frames_per_video must be of type `int`" + if not isinstance(key["project_name"], str): + raise ValueError("project_name must be a string") + if not isinstance(key["frames_per_video"], int): + raise ValueError("frames_per_video must be of type `int`") super().insert1(key, **kwargs) + def _existing_project(self, project_name): + if project_name in self.fetch("project_name"): + logger.warning(f"project name: {project_name} is already in use.") + return (self & {"project_name": project_name}).fetch( + "project_name", "config_path", as_dict=True + )[0] + return None + @classmethod def insert_existing_project( cls, @@ -123,46 +126,43 @@ def insert_existing_project( optional list of bodyparts to label that are not already in existing config """ - - # Read config - project_names_in_use = np.unique(cls.fetch("project_name")) - if project_name in project_names_in_use: - print(f"project name: {project_name} is already in use.") - return_key = {} - return_key["project_name"], return_key["config_path"] = ( - cls & {"project_name": project_name} - ).fetch1("project_name", "config_path") - return return_key from deeplabcut.utils.auxiliaryfunctions import read_config + if (existing := cls()._existing_project(project_name)) is not None: + return existing + cfg = read_config(config_path) + all_bodyparts = cfg["bodyparts"] if bodyparts: bodyparts_to_add = [ bodypart for bodypart in bodyparts if bodypart not in cfg["bodyparts"] ] - all_bodyparts = bodyparts_to_add + cfg["bodyparts"] - else: - all_bodyparts = cfg["bodyparts"] + all_bodyparts += bodyparts_to_add + BodyPart.add_from_config(cfg["bodyparts"]) for bodypart in all_bodyparts: if not bool(BodyPart() & {"bodypart": bodypart}): raise ValueError( f"bodypart: {bodypart} not found in BodyPart table" ) + # check bodyparts are in config, if not add if len(bodyparts_to_add) > 0: add_to_config(config_path, bodyparts=bodyparts_to_add) + # Get frames per video from config. If passed as arg, check match if frames_per_video: if frames_per_video != cfg["numframes2pick"]: add_to_config( config_path, **{"numframes2pick": frames_per_video} ) + config_path = Path(config_path) project_path = config_path.parent dlc_project_path = dlc_project_dir + if dlc_project_path not in project_path.as_posix(): project_dirname = project_path.name dest_folder = Path(f"{dlc_project_path}/{project_dirname}/") @@ -179,6 +179,7 @@ def insert_existing_project( ), "config.yaml does not exist in new project directory" config_path = new_config_path add_to_config(config_path, **{"project_path": new_proj_dir}) + # TODO still need to copy videos over to video dir key = { "project_name": project_name, @@ -187,21 +188,17 @@ def insert_existing_project( "config_path": config_path.as_posix(), "frames_per_video": frames_per_video, } - cls.insert1(key, **kwargs) - cls.BodyPart.insert( + cls().insert1(key, **kwargs) + cls().BodyPart.insert( [ {"project_name": project_name, "bodypart": bp} for bp in all_bodyparts ], **kwargs, ) - if add_to_files: - del key["bodyparts"] - del key["team_name"] - del key["config_path"] - del key["frames_per_video"] - # Check for training files to add - cls.add_training_files(key, **kwargs) + if add_to_files: # Check for training files to add + cls().add_training_files(key, **kwargs) + return { "project_name": project_name, "config_path": config_path.as_posix(), @@ -245,68 +242,24 @@ def insert_new_project( target path to output converted videos (Default is '/nimbus/deeplabcut/videos/') """ - project_names_in_use = np.unique(cls.fetch("project_name")) - if project_name in project_names_in_use: - print(f"project name: {project_name} is already in use.") - return_key = {} - return_key["project_name"], return_key["config_path"] = ( - cls & {"project_name": project_name} - ).fetch1("project_name", "config_path") - return return_key + from deeplabcut import create_new_project - add_to_files = kwargs.pop("add_to_files", True) + if (existing := cls()._existing_project(project_name)) is not None: + return existing if not bool(LabTeam() & {"team_name": lab_team}): - raise ValueError(f"team_name: {lab_team} does not exist in LabTeam") + raise ValueError(f"LabTeam does not exist: {lab_team}") + + add_to_files = kwargs.pop("add_to_files", True) skeleton_node = None # If dict, assume of form {'nwb_file_name': nwb_file_name, 'epoch': epoch} # and pass to get_video_path to reference VideoFile table for path - if all(isinstance(n, Dict) for n in video_list): - videos_to_convert = [ - get_video_path(video_key) for video_key in video_list - ] - videos = [ - check_videofile( - video_path=video[0], - output_path=output_path, - video_filename=video[1], - )[0].as_posix() - for video in videos_to_convert - if video[0] is not None - ] - if len(videos) < 1: - raise ValueError( - f"no .mp4 videos found in {videos_to_convert[0][0]}" - + f" for key: {video_list[0]}" - ) - - # If not dict, assume list of video file paths that may or may not need to be converted - else: - videos = [] - if not all([Path(video).exists() for video in video_list]): - raise OSError("at least one file in video_list does not exist") - for video in video_list: - video_path = Path(video).parent - video_filename = video.rsplit( - video_path.as_posix(), maxsplit=1 - )[-1].split("/")[-1] - videos.extend( - [ - check_videofile( - video_path=video_path, - output_path=output_path, - video_filename=video_filename, - )[0].as_posix() - ] - ) - if len(videos) < 1: - raise ValueError(f"no .mp4 videos found in{video_path}") - from deeplabcut import create_new_project + videos = cls()._process_videos(video_list, output_path) config_path = create_new_project( - project_name, - lab_team, - videos, + project=project_name, + experimenter=sanitize_filename(lab_team), + videos=videos, working_directory=project_directory, copy_videos=True, multianimal=False, @@ -318,9 +271,11 @@ def insert_new_project( ) kwargs_copy = copy.deepcopy(kwargs) kwargs_copy.update({"numframes2pick": frames_per_video, "dotsize": 3}) + add_to_config( config_path, bodyparts, skeleton_node=skeleton_node, **kwargs_copy ) + key = { "project_name": project_name, "team_name": lab_team, @@ -328,133 +283,138 @@ def insert_new_project( "config_path": config_path, "frames_per_video": frames_per_video, } - cls.insert1(key, **kwargs) - cls.BodyPart.insert( + cls().insert1(key, **kwargs) + cls().BodyPart.insert( [ {"project_name": project_name, "bodypart": bp} for bp in bodyparts ], **kwargs, ) - if add_to_files: - del key["bodyparts"] - del key["team_name"] - del key["config_path"] - del key["frames_per_video"] - # Add videos to training files - cls.add_training_files(key, **kwargs) + if add_to_files: # Add videos to training files + cls().add_training_files(key, **kwargs) + if isinstance(config_path, PosixPath): config_path = config_path.as_posix() return {"project_name": project_name, "config_path": config_path} + def _process_videos(self, video_list, output_path): + # If dict, assume {'nwb_file_name': nwb_file_name, 'epoch': epoch} + if all(isinstance(n, Dict) for n in video_list): + videos_to_convert = [] + for video in video_list: + if (video_path := get_video_info(video))[0] is not None: + videos_to_convert.append(video_path) + + else: # Otherwise, assume list of video file paths + if not all([Path(video).exists() for video in video_list]): + raise FileNotFoundError(f"Couldn't find video(s): {video_list}") + videos_to_convert = [] + for video in video_list: + vp = Path(video) + videos_to_convert.append((vp.parent, vp.name)) + + videos = [ + find_mp4( + video_path=video[0], + output_path=output_path, + video_filename=video[1], + ) + for video in videos_to_convert + ] + + if len(videos) < 1: + raise ValueError(f"no .mp4 videos found from {video_list}") + + return videos + @classmethod def add_video_files( cls, video_list, config_path=None, key=None, - output_path: str = os.getenv("DLC_VIDEO_PATH"), + output_path: str = dlc_video_dir, add_new=False, add_to_files=True, **kwargs, ): has_config_or_key = bool(config_path) or bool(key) - if add_new and not has_config_or_key: raise ValueError("If add_new, must provide key or config_path") - config_path = config_path or (cls & key).fetch1("config_path") - if ( - add_to_files - and not key - and len(cls & {"config_path": config_path}) != 1 - ): + config_path = config_path or (cls & key).fetch1("config_path") + has_proj = bool(key) or len(cls & {"config_path": config_path}) == 1 + if add_to_files and not has_proj: raise ValueError("Cannot set add_to_files=True without passing key") - if all(isinstance(n, Dict) for n in video_list): - videos_to_convert = [ - get_video_path(video_key) for video_key in video_list - ] - videos = [ - check_videofile( - video_path=video[0], - output_path=output_path, - video_filename=video[1], - )[0].as_posix() - for video in videos_to_convert - ] - # If not dict, assume list of video file paths - # that may or may not need to be converted - else: - videos = [] - if not all([Path(video).exists() for video in video_list]): - raise OSError("at least one file in video_list does not exist") - for video in video_list: - video_path = Path(video).parent - video_filename = video.rsplit( - video_path.as_posix(), maxsplit=1 - )[-1].split("/")[-1] - videos.append( - check_videofile( - video_path=video_path, - output_path=output_path, - video_filename=video_filename, - )[0].as_posix() - ) - if len(videos) < 1: - raise ValueError(f"no .mp4 videos found in{video_path}") + videos = cls()._process_videos(video_list, output_path) + if add_new: from deeplabcut import add_new_videos add_new_videos(config=config_path, videos=videos, copy_videos=True) - if add_to_files: - # Add videos to training files - cls.add_training_files(key, **kwargs) + + if add_to_files: # Add videos to training files + cls().add_training_files(key, **kwargs) return videos @classmethod def add_training_files(cls, key, **kwargs): """Add training videos and labeled frames .h5 and .csv to DLCProject.File""" + from deeplabcut.utils.auxiliaryfunctions import read_config + config_path = (cls & {"project_name": key["project_name"]}).fetch1( "config_path" ) - from deeplabcut.utils.auxiliaryfunctions import read_config - if "config_path" in key: - del key["config_path"] + key = { # Remove non-essential vals from key + k: v + for k, v in key.items() + if k + not in [ + "bodyparts", + "team_name", + "config_path", + "frames_per_video", + ] + } + cfg = read_config(config_path) - video_names = list(cfg["video_sets"].keys()) + video_names = list(cfg["video_sets"]) + label_dir = Path(cfg["project_path"]) / "labeled-data" training_files = [] + + video_inserts = [] for video in video_names: - video_name = os.path.splitext( - video.split(os.path.dirname(video) + "/")[-1] - )[0] - training_files.extend( - glob.glob( - f"{cfg['project_path']}/" - f"labeled-data/{video_name}/*Collected*" - ) + vid_path_obj = Path(video) + video_name = vid_path_obj.stem + training_files.extend((label_dir / video_name).glob("*Collected*")) + key.update( + { + "file_name": video_name, + "file_ext": vid_path_obj.suffix[1:], # remove leading '.' + "file_path": video, + } + ) + cls().File.insert(video_inserts, **kwargs) + + if len(training_files) == 0: + logger.warning("No training files to add") + return + + for file in training_files: + path_obj = Path(file) + cls().File.insert1( + { + **key, + "file_name": f"{path_obj.name}_labeled_data", + "file_ext": path_obj.suffix[1:], + "file_path": file, + }, + **kwargs, ) - for video in video_names: - key["file_name"] = f'{os.path.splitext(video.split("/")[-1])[0]}' - key["file_ext"] = os.path.splitext(video.split("/")[-1])[-1].split( - "." - )[-1] - key["file_path"] = video - cls.File.insert1(key, **kwargs) - if len(training_files) > 0: - for file in training_files: - video_name = os.path.dirname(file).split("/")[-1] - file_type = os.path.splitext( - file.split(os.path.dirname(file) + "/")[-1] - )[-1].split(".")[-1] - key["file_name"] = f"{video_name}_labeled_data" - key["file_ext"] = file_type - key["file_path"] = file - cls.File.insert1(key, **kwargs) - else: - Warning("No training files to add") @classmethod def run_extract_frames(cls, key, **kwargs): @@ -474,7 +434,11 @@ def run_label_frames(cls, key): cannot be run through ssh tunnel """ config_path = (cls & key).fetch1("config_path") - from deeplabcut import label_frames + try: + from deeplabcut import label_frames + except (ModuleNotFoundError, ImportError): + logger.error("DLC loaded in light mode, cannot label frames") + return label_frames(config_path) @@ -492,7 +456,7 @@ def check_labels(cls, key, **kwargs): def import_labeled_frames( cls, key: Dict, - import_project_path: Union[str, PosixPath], + new_proj_path: Union[str, PosixPath], video_filenames: Union[str, List], **kwargs, ): @@ -503,63 +467,46 @@ def import_labeled_frames( ---------- key : Dict key to specify entry in DLCProject table to add labeled frames to - import_project_path : str + new_proj_path : Union[str, PosixPath] absolute path to project directory containing labeled frames to import video_filenames : str or List - filename or list of filenames of video(s) - from which to import frames. - without file extension + filename or list of filenames of video(s) from which to import + frames. Without file extension """ project_entry = (cls & key).fetch1() - team_name = project_entry["team_name"] - current_project_path = Path(project_entry["config_path"]).parent - current_labeled_data_path = Path( - f"{current_project_path.as_posix()}/labeled-data" + team_name = project_entry["team_name"].replace(" ", "_") + this_proj_path = Path(project_entry["config_path"]).parent + this_data_path = this_proj_path / "labeled-data" + new_proj_path = Path(new_proj_path) # If Path(Path), no change + new_data_path = new_proj_path / "labeled-data" + + if not new_data_path.exists(): + raise FileNotFoundError(f"Cannot find directory: {new_data_path}") + + videos = ( + video_filenames + if isinstance(video_filenames, List) + else [video_filenames] ) - if isinstance(import_project_path, PosixPath): - assert import_project_path.exists(), ( - "import_project_path: " - f"{import_project_path.as_posix()} does not exist" - ) - import_labeled_data_path = Path( - f"{import_project_path.as_posix()}/labeled-data" - ) - else: - assert Path( - import_project_path - ).exists(), ( - f"import_project_path: {import_project_path} does not exist" - ) - import_labeled_data_path = Path( - f"{import_project_path}/labeled-data" - ) - assert ( - import_labeled_data_path.exists() - ), "import_project has no directory 'labeled-data'" - if not isinstance(video_filenames, List): - video_filenames = [video_filenames] - for video_file in video_filenames: - h5_file = glob.glob( - f"{import_labeled_data_path.as_posix()}/{video_file}/*.h5" - )[0] + for video_file in videos: + h5_file = next((new_data_path / video_file).glob("*h5")) dlc_df = pd.read_hdf(h5_file) dlc_df.columns = dlc_df.columns.set_levels([team_name], level=0) + new_video_path = this_data_path / video_file + new_video_path.mkdir(exist_ok=True) dlc_df.to_hdf( - Path( - f"{current_labeled_data_path.as_posix()}/" - f"{video_file}/CollectedData_{team_name}.h5" - ).as_posix(), + new_video_path / f"CollectedData_{team_name}.h5", "df_with_missing", ) - cls.add_training_files(key, **kwargs) + cls().add_training_files(key, **kwargs) def add_to_config( config, bodyparts: List = None, skeleton_node: str = None, **kwargs ): - """ - Add necessary items to the config.yaml for the model + """Add necessary items to the config.yaml for the model + Parameters ---------- config : str @@ -572,28 +519,43 @@ def add_to_config( Other parameters of config to modify in key:value pairs """ - yaml = ruamel.yaml.YAML() + yaml = YAML() with open(config) as fp: data = yaml.load(fp) + if bodyparts: data["bodyparts"] = bodyparts - led_parts = [element for element in bodyparts if "LED" in element] - if skeleton_node is not None: - bodypart_skeleton = [ + led_parts = [bp for bp in bodyparts if "LED" in bp] + bodypart_skeleton = ( + [ list(link) for link in combinations(led_parts, 2) if skeleton_node in link ] - else: - bodypart_skeleton = list(combinations(led_parts, 2)) + if skeleton_node + else list(combinations(led_parts, 2)) + ) other_parts = list(set(bodyparts) - set(led_parts)) for ind, part in enumerate(other_parts): other_parts[ind] = [part, part] bodypart_skeleton.append(other_parts) data["skeleton"] = bodypart_skeleton - for kwarg, val in kwargs.items(): - if not isinstance(kwarg, str): - kwarg = str(kwarg) - data[kwarg] = val + + kwargs.update( + {str(k): v for k, v in kwargs.items() if not isinstance(k, str)} + ) + with open(config, "w") as fw: yaml.dump(data, fw) + + +def sanitize_filename(filename: str) -> str: + """Sanitize filename to remove special characters""" + char_map = { + " ": "_", + ".": "_", + ",": "-", + "&": "and", + "'": "", + } + return "".join([char_map.get(c, c) for c in filename]) diff --git a/src/spyglass/position/v1/position_dlc_selection.py b/src/spyglass/position/v1/position_dlc_selection.py index 8a283bb1d..7d90a6d2a 100644 --- a/src/spyglass/position/v1/position_dlc_selection.py +++ b/src/spyglass/position/v1/position_dlc_selection.py @@ -12,7 +12,7 @@ convert_epoch_interval_name_to_position_interval_name, ) from spyglass.common.common_nwbfile import AnalysisNwbfile -from spyglass.position.v1.dlc_utils import make_video +from spyglass.position.v1.dlc_utils_makevid import make_video from spyglass.position.v1.position_dlc_centroid import DLCCentroid from spyglass.position.v1.position_dlc_cohort import DLCSmoothInterpCohort from spyglass.position.v1.position_dlc_orient import DLCOrientation @@ -21,7 +21,7 @@ DLCPoseEstimationSelection, ) from spyglass.position.v1.position_dlc_position import DLCSmoothInterpParams -from spyglass.utils.dj_mixin import SpyglassMixin +from spyglass.utils import SpyglassMixin, logger schema = dj.schema("position_v1_dlc_selection") @@ -106,39 +106,46 @@ def make(self, key): velocity.create_timeseries( name=vid_frame_obj.name, - unit=vid_frame_obj.unit, timestamps=np.asarray(vid_frame_obj.timestamps), + unit=vid_frame_obj.unit, data=np.asarray(vid_frame_obj.data), description=vid_frame_obj.description, comments=vid_frame_obj.comments, ) - key["analysis_file_name"] = AnalysisNwbfile().create( - key["nwb_file_name"] - ) + # Add to Analysis NWB file + analysis_file_name = AnalysisNwbfile().create(key["nwb_file_name"]) + key["analysis_file_name"] = analysis_file_name nwb_analysis_file = AnalysisNwbfile() - key["orientation_object_id"] = nwb_analysis_file.add_nwb_object( - key["analysis_file_name"], orientation - ) - key["position_object_id"] = nwb_analysis_file.add_nwb_object( - key["analysis_file_name"], position - ) - key["velocity_object_id"] = nwb_analysis_file.add_nwb_object( - key["analysis_file_name"], velocity + + key.update( + { + "analysis_file_name": analysis_file_name, + "position_object_id": nwb_analysis_file.add_nwb_object( + analysis_file_name, position + ), + "orientation_object_id": nwb_analysis_file.add_nwb_object( + analysis_file_name, orientation + ), + "velocity_object_id": nwb_analysis_file.add_nwb_object( + analysis_file_name, velocity + ), + } ) nwb_analysis_file.add( nwb_file_name=key["nwb_file_name"], - analysis_file_name=key["analysis_file_name"], + analysis_file_name=analysis_file_name, ) self.insert1(key) from ..position_merge import PositionOutput - part_name = to_camel_case(self.table_name.split("__")[-1]) # TODO: The next line belongs in a merge table function PositionOutput._merge_insert( - [orig_key], part_name=part_name, skip_duplicates=True + [orig_key], + part_name=to_camel_case(self.table_name.split("__")[-1]), + skip_duplicates=True, ) AnalysisNwbfile().log(key, table=self.full_table_name) @@ -187,23 +194,24 @@ def fetch_nwb(self, **kwargs): @classmethod def evaluate_pose_estimation(cls, key): likelihood_thresh = [] - valid_fields = ( - DLCSmoothInterpCohort.BodyPart().fetch().dtype.fields.keys() - ) + + valid_fields = DLCSmoothInterpCohort.BodyPart().heading.names centroid_key = {k: val for k, val in key.items() if k in valid_fields} centroid_key["dlc_si_cohort_selection_name"] = key[ "dlc_si_cohort_centroid" ] + centroid_bodyparts, centroid_si_params = ( + DLCSmoothInterpCohort.BodyPart & centroid_key + ).fetch("bodypart", "dlc_si_params_name") + orientation_key = centroid_key.copy() orientation_key["dlc_si_cohort_selection_name"] = key[ "dlc_si_cohort_orientation" ] - centroid_bodyparts, centroid_si_params = ( - DLCSmoothInterpCohort.BodyPart & centroid_key - ).fetch("bodypart", "dlc_si_params_name") orientation_bodyparts, orientation_si_params = ( DLCSmoothInterpCohort.BodyPart & orientation_key ).fetch("bodypart", "dlc_si_params_name") + for param in np.unique( np.concatenate((centroid_si_params, orientation_si_params)) ): @@ -215,9 +223,10 @@ def evaluate_pose_estimation(cls, key): if len(np.unique(likelihood_thresh)) > 1: raise ValueError("more than one likelihood threshold used") + like_thresh = likelihood_thresh[0] bodyparts = np.unique([*centroid_bodyparts, *orientation_bodyparts]) - fields = list(DLCPoseEstimation.BodyPart.fetch().dtype.fields.keys()) + fields = DLCPoseEstimation.BodyPart.heading.names pose_estimation_key = {k: v for k, v in key.items() if k in fields} pose_estimation_df = pd.concat( { @@ -308,134 +317,108 @@ class DLCPosVideo(SpyglassMixin, dj.Computed): --- """ - # TODO: Shoultn't this keep track of the video file it creates? - def make(self, key): - from tqdm import tqdm as tqdm + M_TO_CM = 100 params = (DLCPosVideoParams & key).fetch1("params") - if "video_params" not in params: - params["video_params"] = {} - M_TO_CM = 100 - epoch = key["epoch"] - pose_estimation_key = { + + interval_name = convert_epoch_interval_name_to_position_interval_name( + { + "nwb_file_name": key["nwb_file_name"], + "epoch": key["epoch"], + }, + populate_missing=False, + ) + epoch = ( + int(interval_name.replace("pos ", "").replace(" valid times", "")) + + 1 + ) + pose_est_key = { "nwb_file_name": key["nwb_file_name"], "epoch": epoch, "dlc_model_name": key["dlc_model_name"], "dlc_model_params_name": key["dlc_model_params_name"], } - pose_estimation_params, video_filename, output_dir = ( - DLCPoseEstimationSelection() & pose_estimation_key + + pose_estimation_params, video_filename, output_dir, meters_per_pixel = ( + DLCPoseEstimationSelection * DLCPoseEstimation & pose_est_key ).fetch1( - "pose_estimation_params", "video_path", "pose_estimation_output_dir" - ) - print(f"video filename: {video_filename}") - meters_per_pixel = (DLCPoseEstimation() & pose_estimation_key).fetch1( - "meters_per_pixel" + "pose_estimation_params", + "video_path", + "pose_estimation_output_dir", + "meters_per_pixel", ) - crop = None - if "cropping" in pose_estimation_params: - crop = pose_estimation_params["cropping"] - print("Loading position data...") - position_info_df = ( - DLCPosV1() - & { - "nwb_file_name": key["nwb_file_name"], - "epoch": epoch, - "dlc_si_cohort_centroid": key["dlc_si_cohort_centroid"], - "dlc_centroid_params_name": key["dlc_centroid_params_name"], - "dlc_si_cohort_orientation": key["dlc_si_cohort_orientation"], - "dlc_orientation_params_name": key[ - "dlc_orientation_params_name" - ], - } + + logger.info(f"video filename: {video_filename}") + logger.info("Loading position data...") + + v1_key = {k: v for k, v in key.items() if k in DLCPosV1.primary_key} + pos_info_df = ( + DLCPosV1() & {"epoch": epoch, **v1_key} ).fetch1_dataframe() - pose_estimation_df = pd.concat( + pos_est_df = pd.concat( { bodypart: ( DLCPoseEstimation.BodyPart() - & {**pose_estimation_key, **{"bodypart": bodypart}} + & {**pose_est_key, **{"bodypart": bodypart}} ).fetch1_dataframe() - for bodypart in ( - DLCSmoothInterpCohort.BodyPart & pose_estimation_key - ) + for bodypart in (DLCSmoothInterpCohort.BodyPart & pose_est_key) .fetch("bodypart") .tolist() }, axis=1, ) - assert len(pose_estimation_df) == len(position_info_df), ( - f"length of pose_estimation_df: {len(pose_estimation_df)} " - f"does not match the length of position_info_df: {len(position_info_df)}." - ) + if not len(pos_est_df) == len(pos_info_df): + raise ValueError( + "Dataframes are not the same length\n" + + f"\tPose estim : {len(pos_est_df)}\n" + + f"\tPosition info: {len(pos_info_df)}" + ) - nwb_base_filename = key["nwb_file_name"].replace(".nwb", "") + output_video_filename = ( + key["nwb_file_name"].replace(".nwb", "") + + f"_{epoch:02d}_" + + f'{key["dlc_si_cohort_centroid"]}_' + + f'{key["dlc_centroid_params_name"]}' + + f'{key["dlc_orientation_params_name"]}.mp4' + ) if Path(output_dir).exists(): - output_video_filename = ( - f"{Path(output_dir).as_posix()}/" - f"{nwb_base_filename}_{epoch:02d}_" - f'{key["dlc_si_cohort_centroid"]}_' - f'{key["dlc_centroid_params_name"]}' - f'{key["dlc_orientation_params_name"]}.mp4' - ) - else: - output_video_filename = ( - f"{nwb_base_filename}_{epoch:02d}_" - f'{key["dlc_si_cohort_centroid"]}_' - f'{key["dlc_centroid_params_name"]}' - f'{key["dlc_orientation_params_name"]}.mp4' - ) + output_video_filename = Path(output_dir) / output_video_filename + idx = pd.IndexSlice - video_frame_inds = ( - position_info_df["video_frame_ind"].astype(int).to_numpy() - ) + video_frame_inds = pos_info_df["video_frame_ind"].astype(int).to_numpy() centroids = { - bodypart: pose_estimation_df.loc[ - :, idx[bodypart, ("x", "y")] - ].to_numpy() - for bodypart in pose_estimation_df.columns.levels[0] + bodypart: pos_est_df.loc[:, idx[bodypart, ("x", "y")]].to_numpy() + for bodypart in pos_est_df.columns.levels[0] } - if params.get("incl_likelihood", None): - likelihoods = { - bodypart: pose_estimation_df.loc[ + likelihoods = ( + { + bodypart: pos_est_df.loc[ :, idx[bodypart, ("likelihood")] ].to_numpy() - for bodypart in pose_estimation_df.columns.levels[0] + for bodypart in pos_est_df.columns.levels[0] } - else: - likelihoods = None - position_mean = { - "DLC": np.asarray(position_info_df[["position_x", "position_y"]]) - } - orientation_mean = { - "DLC": np.asarray(position_info_df[["orientation"]]) - } - position_time = np.asarray(position_info_df.index) - cm_per_pixel = meters_per_pixel * M_TO_CM - percent_frames = params.get("percent_frames", None) + if params.get("incl_likelihood") + else None + ) frames = params.get("frames", None) - if frames is not None: - frames_arr = np.arange(frames[0], frames[1]) - else: - frames_arr = frames - print("Making video...") make_video( video_filename=video_filename, video_frame_inds=video_frame_inds, - position_mean=position_mean, - orientation_mean=orientation_mean, + position_mean={ + "DLC": np.asarray(pos_info_df[["position_x", "position_y"]]) + }, + orientation_mean={"DLC": np.asarray(pos_info_df[["orientation"]])}, centroids=centroids, likelihoods=likelihoods, - position_time=position_time, - video_time=None, + position_time=np.asarray(pos_info_df.index), processor=params.get("processor", "opencv"), - frames=frames_arr, - percent_frames=percent_frames, + frames=np.arange(frames[0], frames[1]) if frames else None, + percent_frames=params.get("percent_frames", None), output_video_filename=output_video_filename, - cm_to_pixels=cm_per_pixel, - disable_progressbar=False, - crop=crop, - **params["video_params"], + cm_to_pixels=meters_per_pixel * M_TO_CM, + crop=pose_estimation_params.get("cropping"), + **params.get("video_params", {}), ) self.insert1(key) diff --git a/src/spyglass/position/v1/position_dlc_training.py b/src/spyglass/position/v1/position_dlc_training.py index 393eb6af9..7876754f5 100644 --- a/src/spyglass/position/v1/position_dlc_training.py +++ b/src/spyglass/position/v1/position_dlc_training.py @@ -4,10 +4,10 @@ import datajoint as dj -from spyglass.position.v1.dlc_utils import OutputLogger +from spyglass.position.v1.dlc_utils import file_log from spyglass.position.v1.position_dlc_project import DLCProject from spyglass.settings import test_mode -from spyglass.utils.dj_mixin import SpyglassMixin +from spyglass.utils import SpyglassMixin, logger schema = dj.schema("position_v1_dlc_training") @@ -22,13 +22,13 @@ class DLCModelTrainingParams(SpyglassMixin, dj.Lookup): params : longblob # dictionary of all applicable parameters """ - required_parameters = ( + required_params = ( "shuffle", "trainingsetindex", "net_type", "gputouse", ) - skipped_parameters = ("project_path", "video_sets") + skipped_params = ("project_path", "video_sets") @classmethod def insert_new_params(cls, paramset_name: str, params: dict, **kwargs): @@ -45,76 +45,52 @@ def insert_new_params(cls, paramset_name: str, params: dict, **kwargs): project_path and video_sets will be overwritten by config.yaml. Note that trainingsetindex is 0-indexed """ + if not set(cls.required_params).issubset(params): + raise ValueError(f"Missing required params: {cls.required_params}") + params = { + k: v for k, v in params.items() if k not in cls.skipped_params + } - for required_param in cls.required_parameters: - assert required_param in params, ( - "Missing required parameter: " + required_param - ) - for skipped_param in cls.skipped_parameters: - if skipped_param in params: - params.pop(skipped_param) + param_pk = {"dlc_training_params_name": paramset_name} + param_query = cls & param_pk - param_dict = { - "dlc_training_params_name": paramset_name, - "params": params, - } - param_query = cls & { - "dlc_training_params_name": param_dict["dlc_training_params_name"] - } - # If the specified param-set already exists - # Not sure we need this part, as much just a check if the name is the same if param_query: - existing_paramset_name = param_query.fetch1( - "dlc_training_params_name" + logger.info( + f"New param set not added\n" + f"A param set with name: {paramset_name} already exists" ) - if ( - existing_paramset_name == paramset_name - ): # If existing name same: - return print( - f"New param set not added\n" - f"A param set with name: {paramset_name} already exists" - ) - else: - cls.insert1( - param_dict, **kwargs - ) # if duplicate, will raise duplicate error - # if this will raise duplicate error, why is above check needed? @datajoint + return + cls.insert1({**param_pk, "params": params}, **kwargs) @classmethod def get_accepted_params(cls): from deeplabcut import create_training_dataset, train_network - return list( - set( - [ - *list(inspect.signature(train_network).parameters), - *list( - inspect.signature(create_training_dataset).parameters - ), - ] - ) + return set( + [ + *get_param_names(train_network), + *get_param_names(create_training_dataset), + ] ) @schema class DLCModelTrainingSelection(SpyglassMixin, dj.Manual): - definition = """ # Specification for a DLC model training instance + definition = """ # Specification for a DLC model training instance -> DLCProject -> DLCModelTrainingParams - training_id : int # unique integer, + training_id : int # unique integer # allows for multiple training runs for a specific parameter set and project --- model_prefix='' : varchar(32) """ - def insert1(self, key, **kwargs): - training_id = key.get("training_id") - if training_id is None: + def insert1(self, key, **kwargs): # Auto-increment training_id + if not (training_id := key.get("training_id")): training_id = ( dj.U().aggr(self & key, n="max(training_id)").fetch1("n") or 0 ) + 1 - key["training_id"] = training_id - super().insert1(key, **kwargs) + super().insert1({**key, "training_id": training_id}, **kwargs) @schema @@ -126,13 +102,20 @@ class DLCModelTraining(SpyglassMixin, dj.Computed): latest_snapshot: int unsigned # latest exact snapshot index (i.e., never -1) config_template: longblob # stored full config file """ + log_path = None - # To continue from previous training snapshot, devs suggest editing pose_cfg.yml + # To continue from previous training snapshot, + # devs suggest editing pose_cfg.yml # https://github.com/DeepLabCut/DeepLabCut/issues/70 def make(self, key): - """Launch training for each entry in DLCModelTrainingSelection via `.populate()`.""" - model_prefix = (DLCModelTrainingSelection & key).fetch1("model_prefix") + """Launch training for each entry in DLCModelTrainingSelection.""" + config_path = (DLCProject & key).fetch1("config_path") + self.log_path = Path(config_path).parent / "log.log" + self._logged_make(key) + + @file_log(logger, console=True) # THIS WORKS + def _logged_make(self, key): from deeplabcut import create_training_dataset, train_network from deeplabcut.utils.auxiliaryfunctions import read_config @@ -144,111 +127,106 @@ def make(self, key): from deeplabcut.utils.auxiliaryfunctions import ( GetModelFolder as get_model_folder, ) + + model_prefix = (DLCModelTrainingSelection & key).fetch1("model_prefix") config_path, project_name = (DLCProject() & key).fetch1( "config_path", "project_name" ) - with OutputLogger( - name="DLC_project_{project_name}_training", - path=f"{os.path.dirname(config_path)}/log.log", - print_console=True, - ) as logger: - dlc_config = read_config(config_path) - project_path = dlc_config["project_path"] - key["project_path"] = project_path - # ---- Build and save DLC configuration (yaml) file ---- - _, dlc_config = dlc_reader.read_yaml(project_path) - if not dlc_config: - dlc_config = read_config(config_path) - dlc_config.update((DLCModelTrainingParams & key).fetch1("params")) - dlc_config.update( - { - "project_path": Path(project_path).as_posix(), - "modelprefix": model_prefix, - "train_fraction": dlc_config["TrainingFraction"][ - int(dlc_config["trainingsetindex"]) - ], - "training_filelist_datajoint": [ # don't overwrite origin video_sets - Path(fp).as_posix() - for fp in (DLCProject.File & key).fetch("file_path") - ], - } - ) - # Write dlc config file to base project folder - # TODO: need to make sure this will work - dlc_cfg_filepath = dlc_reader.save_yaml(project_path, dlc_config) - # ---- create training dataset ---- - training_dataset_input_args = list( - inspect.signature(create_training_dataset).parameters - ) - training_dataset_kwargs = { - k: v - for k, v in dlc_config.items() - if k in training_dataset_input_args + + dlc_config = read_config(config_path) + project_path = dlc_config["project_path"] + key["project_path"] = project_path + + # ---- Build and save DLC configuration (yaml) file ---- + dlc_config = dlc_reader.read_yaml(project_path)[1] or read_config( + config_path + ) + dlc_config.update( + { + **(DLCModelTrainingParams & key).fetch1("params"), + "project_path": Path(project_path).as_posix(), + "modelprefix": model_prefix, + "train_fraction": dlc_config["TrainingFraction"][ + int(dlc_config.get("trainingsetindex", 0)) + ], + "training_filelist_datajoint": [ # don't overwrite origin video_sets + Path(fp).as_posix() + for fp in (DLCProject.File & key).fetch("file_path") + ], } - logger.logger.info("creating training dataset") - # err here - create_training_dataset(dlc_cfg_filepath, **training_dataset_kwargs) - # ---- Trigger DLC model training job ---- - train_network_input_args = list( - inspect.signature(train_network).parameters + ) + + # Write dlc config file to base project folder + dlc_cfg_filepath = dlc_reader.save_yaml(project_path, dlc_config) + # ---- create training dataset ---- + training_dataset_input_args = list( + inspect.signature(create_training_dataset).parameters + ) + training_dataset_kwargs = { + k: v + for k, v in dlc_config.items() + if k in training_dataset_input_args + } + logger.info("creating training dataset") + create_training_dataset(dlc_cfg_filepath, **training_dataset_kwargs) + # ---- Trigger DLC model training job ---- + train_network_kwargs = { + k: v + for k, v in dlc_config.items() + if k in get_param_names(train_network) + } + for k in ["shuffle", "trainingsetindex", "maxiters"]: + if value := train_network_kwargs.get(k): + train_network_kwargs[k] = int(value) + if test_mode: + train_network_kwargs["maxiters"] = 2 + + try: + train_network(dlc_cfg_filepath, **train_network_kwargs) + except KeyboardInterrupt: + logger.info("DLC training stopped via Keyboard Interrupt") + + snapshots = ( + project_path + / get_model_folder( + trainFraction=dlc_config["train_fraction"], + shuffle=dlc_config["shuffle"], + cfg=dlc_config, + modelprefix=dlc_config["modelprefix"], ) - train_network_kwargs = { - k: v - for k, v in dlc_config.items() - if k in train_network_input_args + / "train" + ).glob("*index*") + + # DLC goes by snapshot magnitude when judging 'latest' for + # evaluation. Here, we mean most recently generated + max_modified_time = 0 + for snapshot in snapshots: + modified_time = os.path.getmtime(snapshot) + if modified_time > max_modified_time: + latest_snapshot = int(snapshot.stem[9:]) + max_modified_time = modified_time + + self.insert1( + { + **key, + "latest_snapshot": latest_snapshot, + "config_template": dlc_config, } - for k in ["shuffle", "trainingsetindex", "maxiters"]: - if k in train_network_kwargs: - train_network_kwargs[k] = int(train_network_kwargs[k]) - if test_mode: - train_network_kwargs["maxiters"] = 2 - try: - train_network(dlc_cfg_filepath, **train_network_kwargs) - except ( - KeyboardInterrupt - ): # Instructions indicate to train until interrupt - logger.logger.info( - "DLC training stopped via Keyboard Interrupt" - ) - - snapshots = list( - ( - project_path - / get_model_folder( - trainFraction=dlc_config["train_fraction"], - shuffle=dlc_config["shuffle"], - cfg=dlc_config, - modelprefix=dlc_config["modelprefix"], - ) - / "train" - ).glob("*index*") - ) - max_modified_time = 0 - # DLC goes by snapshot magnitude when judging 'latest' for evaluation - # Here, we mean most recently generated - for snapshot in snapshots: - modified_time = os.path.getmtime(snapshot) - if modified_time > max_modified_time: - latest_snapshot = int(snapshot.stem[9:]) - max_modified_time = modified_time - - self.insert1( - { - **key, - "latest_snapshot": latest_snapshot, - "config_template": dlc_config, - } - ) - from .position_dlc_model import DLCModelSource - - dlc_model_name = f"{key['project_name']}_{key['dlc_training_params_name']}_{key['training_id']:02d}" - DLCModelSource.insert_entry( - dlc_model_name=dlc_model_name, - project_name=key["project_name"], - source="FromUpstream", - key=key, - skip_duplicates=True, - ) - print( - f"Inserted {dlc_model_name} from {key['project_name']} into DLCModelSource" ) + from .position_dlc_model import DLCModelSource + + dlc_model_name = ( + f"{key['project_name']}_" + + f"{key['dlc_training_params_name']}_{key['training_id']:02d}" + ) + DLCModelSource.insert_entry( + dlc_model_name=dlc_model_name, + project_name=key["project_name"], + source="FromUpstream", + key=key, + skip_duplicates=True, + ) + + +def get_param_names(func): + return list(inspect.signature(func).parameters) diff --git a/src/spyglass/position/v1/position_trodes_position.py b/src/spyglass/position/v1/position_trodes_position.py index 86487ad23..501407571 100644 --- a/src/spyglass/position/v1/position_trodes_position.py +++ b/src/spyglass/position/v1/position_trodes_position.py @@ -1,16 +1,15 @@ import copy import os -from pathlib import Path import datajoint as dj import numpy as np from datajoint.utils import to_camel_case -from tqdm import tqdm as tqdm from spyglass.common.common_behav import RawPosition from spyglass.common.common_nwbfile import AnalysisNwbfile -from spyglass.common.common_position import IntervalPositionInfo -from spyglass.position.v1.dlc_utils import check_videofile, get_video_path +from spyglass.common.common_position import IntervalPositionInfo, _fix_col_names +from spyglass.position.v1.dlc_utils import find_mp4, get_video_info +from spyglass.position.v1.dlc_utils_makevid import make_video from spyglass.settings import test_mode from spyglass.utils import SpyglassMixin, logger @@ -250,14 +249,14 @@ def make(self, key): M_TO_CM = 100 logger.info("Loading position data...") - raw_position_df = ( + raw_df = ( RawPosition.PosObject & { "nwb_file_name": key["nwb_file_name"], "interval_list_name": key["interval_list_name"], } ).fetch1_dataframe() - position_info_df = (TrodesPosV1() & key).fetch1_dataframe() + pos_df = (TrodesPosV1() & key).fetch1_dataframe() logger.info("Loading video data...") epoch = ( @@ -274,7 +273,7 @@ def make(self, key): video_filename, meters_per_pixel, video_time, - ) = get_video_path( + ) = get_video_info( {"nwb_file_name": key["nwb_file_name"], "epoch": epoch} ) @@ -282,222 +281,39 @@ def make(self, key): self.insert1(dict(**key, has_video=False)) return - video_dir = os.path.dirname(video_path) + "/" - video_path = check_videofile( - video_path=video_dir, video_filename=video_filename - )[0].as_posix() - nwb_base_filename = key["nwb_file_name"].replace(".nwb", "") - current_dir = Path(os.getcwd()) - output_video_filename = ( - f"{current_dir.as_posix()}/{nwb_base_filename}_" - f"{epoch:02d}_{key['trodes_pos_params_name']}.mp4" - ) - red_cols = ( - ["xloc", "yloc"] - if "xloc" in raw_position_df.columns - else ["xloc1", "yloc1"] - ) - centroids = { - "red": np.asarray(raw_position_df[red_cols]), - "green": np.asarray(raw_position_df[["xloc2", "yloc2"]]), - } - position_mean = np.asarray( - position_info_df[["position_x", "position_y"]] + video_path = find_mp4( + video_path=os.path.dirname(video_path) + "/", + video_filename=video_filename, ) - orientation_mean = np.asarray(position_info_df[["orientation"]]) - position_time = np.asarray(position_info_df.index) - cm_per_pixel = meters_per_pixel * M_TO_CM - logger.info("Making video...") - self.make_video( - video_path, - centroids, - position_mean, - orientation_mean, - video_time, - position_time, - output_video_filename=output_video_filename, - cm_to_pixels=cm_per_pixel, - disable_progressbar=False, + output_video_filename = ( + key["nwb_file_name"].replace(".nwb", "") + + f"_{epoch:02d}_" + + f'{key["trodes_pos_params_name"]}.mp4' ) - self.insert1(dict(**key, has_video=True)) - - @staticmethod - def convert_to_pixels(data, frame_size, cm_to_pixels=1.0): - """Converts from cm to pixels and flips the y-axis. - Parameters - ---------- - data : ndarray, shape (n_time, 2) - frame_size : array_like, shape (2,) - cm_to_pixels : float - Returns - ------- - converted_data : ndarray, shape (n_time, 2) - """ - return data / cm_to_pixels + adj_df = _fix_col_names(raw_df) # adjust 'xloc1' to 'xloc' - @staticmethod - def fill_nan(variable, video_time, variable_time, truncate_data=False): - """Fill in missing values in variable with nans at video_time. - - Parameters - ---------- - variable : ndarray, shape (n_time,) or (n_time, n_dims) - The variable to fill in. - video_time : ndarray, shape (n_video_time,) - The time points of the video. - variable_time : ndarray, shape (n_variable_time,) - The time points of the variable. - """ - # TODO: Reduce duplication across dlc_utils and common_position - - video_ind = np.digitize(variable_time, video_time[1:]) - n_video_time = len(video_time) - - try: - n_variable_dims = variable.shape[1] - filled_variable = np.full((n_video_time, n_variable_dims), np.nan) - except IndexError: - filled_variable = np.full((n_video_time,), np.nan) - - filled_variable[video_ind] = variable - - return filled_variable - - def make_video( - self, - video_filename, - centroids, - position_mean, - orientation_mean, - video_time, - position_time, - output_video_filename="output.mp4", - cm_to_pixels=1.0, - disable_progressbar=False, - arrow_radius=15, - circle_radius=8, - truncate_data=False, # reduce data to min length across all variables - ): - import cv2 - - RGB_PINK = (234, 82, 111) - RGB_YELLOW = (253, 231, 76) - RGB_WHITE = (255, 255, 255) - - video = cv2.VideoCapture(video_filename) - fourcc = cv2.VideoWriter_fourcc(*"mp4v") - frame_size = (int(video.get(3)), int(video.get(4))) - frame_rate = video.get(5) - n_frames = int(orientation_mean.shape[0]) - logger.info(f"video filepath: {output_video_filename}") - out = cv2.VideoWriter( - output_video_filename, fourcc, frame_rate, frame_size, True - ) - - if test_mode or truncate_data: + if test_mode: # pytest video data has mismatched shapes in some cases - # centroid (267, 2), video_time (270, 2), position_time (5193,) - min_len = min( - n_frames, - len(video_time), - len(position_time), - len(position_mean), - len(orientation_mean), - min(len(v) for v in centroids.values()), - ) - n_frames = min_len + min_len = min(len(adj_df), len(pos_df), len(video_time)) + adj_df = adj_df[:min_len] + pos_df = pos_df[:min_len] video_time = video_time[:min_len] - position_time = position_time[:min_len] - position_mean = position_mean[:min_len] - orientation_mean = orientation_mean[:min_len] - for color, data in centroids.items(): - centroids[color] = data[:min_len] - - centroids = { - color: self.fill_nan( - variable=data, - video_time=video_time, - variable_time=position_time, - ) - for color, data in centroids.items() - } - position_mean = self.fill_nan(position_mean, video_time, position_time) - orientation_mean = self.fill_nan( - orientation_mean, video_time, position_time - ) - for time_ind in tqdm( - range(n_frames - 1), desc="frames", disable=disable_progressbar - ): - is_grabbed, frame = video.read() - if is_grabbed: - frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - - red_centroid = centroids["red"][time_ind] - green_centroid = centroids["green"][time_ind] - - position = position_mean[time_ind] - position = self.convert_to_pixels( - position, frame_size, cm_to_pixels - ) - orientation = orientation_mean[time_ind] - - if np.all(~np.isnan(red_centroid)): - cv2.circle( - img=frame, - center=tuple(red_centroid.astype(int)), - radius=circle_radius, - color=RGB_YELLOW, - thickness=-1, - shift=cv2.CV_8U, - ) - - if np.all(~np.isnan(green_centroid)): - cv2.circle( - img=frame, - center=tuple(green_centroid.astype(int)), - radius=circle_radius, - color=RGB_PINK, - thickness=-1, - shift=cv2.CV_8U, - ) - - if np.all(~np.isnan(position)) & np.all(~np.isnan(orientation)): - arrow_tip = ( - int(position[0] + arrow_radius * np.cos(orientation)), - int(position[1] + arrow_radius * np.sin(orientation)), - ) - cv2.arrowedLine( - img=frame, - pt1=tuple(position.astype(int)), - pt2=arrow_tip, - color=RGB_WHITE, - thickness=4, - line_type=8, - shift=cv2.CV_8U, - tipLength=0.25, - ) - - if np.all(~np.isnan(position)): - cv2.circle( - img=frame, - center=tuple(position.astype(int)), - radius=circle_radius, - color=RGB_WHITE, - thickness=-1, - shift=cv2.CV_8U, - ) - - frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR) - out.write(frame) - else: - break - - video.release() - out.release() - try: - cv2.destroyAllWindows() - except cv2.error: # if cv is already closed or does not have func - pass + make_video( + processor="opencv-trodes", + video_filename=video_path, + centroids={ + "red": np.asarray(adj_df[["xloc", "yloc"]]), + "green": np.asarray(adj_df[["xloc2", "yloc2"]]), + }, + position_mean=np.asarray(pos_df[["position_x", "position_y"]]), + orientation_mean=np.asarray(pos_df[["orientation"]]), + video_time=video_time, + position_time=np.asarray(pos_df.index), + output_video_filename=output_video_filename, + cm_to_pixels=meters_per_pixel * M_TO_CM, + disable_progressbar=False, + ) + self.insert1(dict(**key, has_video=True)) diff --git a/src/spyglass/utils/dj_helper_fn.py b/src/spyglass/utils/dj_helper_fn.py index 85ff1922a..f42c9858a 100644 --- a/src/spyglass/utils/dj_helper_fn.py +++ b/src/spyglass/utils/dj_helper_fn.py @@ -142,7 +142,7 @@ def dj_replace(original_table, new_values, key_column, replace_column): return original_table -def get_fetching_table_from_stack(stack): +def get_all_tables_in_stack(stack): """Get all classes from a stack of tables.""" classes = set() for frame_info in stack: @@ -153,11 +153,13 @@ def get_fetching_table_from_stack(stack): if (name := obj.full_table_name) in PERIPHERAL_TABLES: continue # skip common_nwbfile tables classes.add(name) + return classes + + +def get_fetching_table_from_stack(stack): + """Get all classes from a stack of tables.""" + classes = get_all_tables_in_stack(stack) if len(classes) > 1: - logger.warn( - f"Multiple classes found in stack: {classes}. " - "Please submit a bug report with the snippet used." - ) classes = None # predict only one but not sure, so return None return next(iter(classes)) if classes else None @@ -262,7 +264,11 @@ def fetch_nwb(query_expression, nwb_master, *attrs, **kwargs): # skip the filepath checksum if streamed from Dandi rec_dict["nwb2load_filepath"] = file_path continue - rec_dict["nwb2load_filepath"] = (query_table & rec_dict).fetch1( + + # Pulled from future cbroz1/ndo + # Full dict caused issues with dlc tables using dicts in secondary keys + rec_only_pk = {k: rec_dict[k] for k in query_table.heading.primary_key} + rec_dict["nwb2load_filepath"] = (query_table & rec_only_pk).fetch1( "nwb2load_filepath" ) @@ -332,7 +338,7 @@ def update_analysis_for_dandi_standard( # edit the file with h5py.File(filepath, "a") as file: sex_value = file["/general/subject/sex"][()].decode("utf-8") - if not sex_value in ["Female", "Male", "F", "M", "O", "U"]: + if sex_value not in ["Female", "Male", "F", "M", "O", "U"]: raise ValueError(f"Unexpected value for sex: {sex_value}") if len(sex_value) > 1: @@ -347,7 +353,8 @@ def update_analysis_for_dandi_standard( if species_value == "Rat": new_species_value = "Rattus norvegicus" print( - f"Adjusting subject species from '{species_value}' to '{new_species_value}'." + f"Adjusting subject species from '{species_value}' to " + + f"'{new_species_value}'." ) file["/general/subject/species"][()] = new_species_value @@ -355,8 +362,10 @@ def update_analysis_for_dandi_standard( len(species_value.split(" ")) == 2 or "NCBITaxon" in species_value ): raise ValueError( - f"Dandi upload requires species either be in Latin binomial form (e.g., 'Mus musculus' and 'Homo sapiens')" - + "or be a NCBI taxonomy link (e.g., 'http://purl.obolibrary.org/obo/NCBITaxon_280675')." + "Dandi upload requires species either be in Latin binomial form" + + " (e.g., 'Mus musculus' and 'Homo sapiens')" + + "or be a NCBI taxonomy link (e.g., " + + "'http://purl.obolibrary.org/obo/NCBITaxon_280675')." + f"\n Please update species value of: {species_value}" ) diff --git a/src/spyglass/utils/dj_merge_tables.py b/src/spyglass/utils/dj_merge_tables.py index 37a51b674..d1176de30 100644 --- a/src/spyglass/utils/dj_merge_tables.py +++ b/src/spyglass/utils/dj_merge_tables.py @@ -822,13 +822,14 @@ def delete_downstream_merge( Passthrough to SpyglassMixin.delete_downstream_merge """ - logger.warning( - "DEPRECATED: This function will be removed in `0.6`. " - + "Use AnyTable().delete_downstream_merge() instead." - ) + from spyglass.common.common_usage import ActivityLog from spyglass.utils.dj_mixin import SpyglassMixin + ActivityLog().deprecate_log( + "delete_downstream_merge. Use Table.delete_downstream_merge" + ) + if not isinstance(table, SpyglassMixin): raise ValueError("Input must be a Spyglass Table.") table = table if isinstance(table, dj.Table) else table() diff --git a/src/spyglass/utils/dj_mixin.py b/src/spyglass/utils/dj_mixin.py index 51f398436..4cdbbbaa0 100644 --- a/src/spyglass/utils/dj_mixin.py +++ b/src/spyglass/utils/dj_mixin.py @@ -1,4 +1,3 @@ -import multiprocessing.pool from atexit import register as exit_register from atexit import unregister as exit_unregister from collections import OrderedDict @@ -129,6 +128,18 @@ def file_like(self, name=None, **kwargs): return return self & f"{attr} LIKE '%{name}%'" + def find_insert_fail(self, key): + """Find which parent table is causing an IntergrityError on insert.""" + for parent in self.parents(as_objects=True): + parent_key = { + k: v for k, v in key.items() if k in parent.heading.names + } + parent_name = to_camel_case(parent.table_name) + if query := parent & parent_key: + logger.info(f"{parent_name}:\n{query}") + else: + logger.info(f"{parent_name}: MISSING") + @classmethod def _safe_context(cls): """Return transaction if not already in one.""" diff --git a/src/spyglass/utils/nwb_helper_fn.py b/src/spyglass/utils/nwb_helper_fn.py index d5b6e4624..641b3f2da 100644 --- a/src/spyglass/utils/nwb_helper_fn.py +++ b/src/spyglass/utils/nwb_helper_fn.py @@ -101,7 +101,7 @@ def file_from_dandi(filepath): return False -def get_config(nwb_file_path): +def get_config(nwb_file_path, calling_table=None): """Return a dictionary of config settings for the given NWB file. If the file does not exist, return an empty dict. @@ -122,8 +122,14 @@ def get_config(nwb_file_path): # NOTE use p.stem[:-1] to remove the underscore that was added to the file config_path = p.parent / (p.stem[:-1] + "_spyglass_config.yaml") if not os.path.exists(config_path): - logger.info(f"No config found at file path {config_path}") - return dict() + from spyglass.settings import base_dir # noqa: F401 + + rel_path = p.relative_to(base_dir) + table = f"{calling_table}: " if calling_table else "" + logger.info(f"{table}No config found at {rel_path}") + ret = dict() + __configs[nwb_file_path] = ret # cache to avoid repeated null lookups + return ret with open(config_path, "r") as stream: d = yaml.safe_load(stream) diff --git a/tests/common/test_behav.py b/tests/common/test_behav.py index 6f2daa690..bcfd50270 100644 --- a/tests/common/test_behav.py +++ b/tests/common/test_behav.py @@ -22,18 +22,18 @@ def test_valid_epoch_num(common): assert epoch_num == 1, "PositionSource get_epoch_num failed" -def test_possource_make(common): +def test_pos_source_make(common): """Test custom populate""" common.PositionSource().make(common.Session()) -def test_possource_make_invalid(common): +def test_pos_source_make_invalid(common): """Test invalid populate""" with pytest.raises(ValueError): common.PositionSource().make(dict()) -def test_raw_position_fetchnwb(common, mini_pos, mini_pos_interval_dict): +def test_raw_position_fetch_nwb(common, mini_pos, mini_pos_interval_dict): """Test RawPosition fetch nwb""" fetched = DataFrame( (common.RawPosition & mini_pos_interval_dict) @@ -56,7 +56,7 @@ def test_raw_position_fetch1_df(common, mini_pos, mini_pos_interval_dict): assert fetched.equals(raw), "RawPosition fetch1_dataframe failed" -def test_raw_position_fetch_mult_df(common, mini_pos, mini_pos_interval_dict): +def test_raw_position_fetch_multi_df(common, mini_pos, mini_pos_interval_dict): """Test RawPosition fetch1 dataframe""" shape = common.RawPosition().fetch1_dataframe().shape assert shape == (542, 8), "RawPosition.PosObj fetch1_dataframe failed" @@ -94,7 +94,7 @@ def test_videofile_getabspath(common, video_keys): @pytest.mark.skipif(not TEARDOWN, reason="No teardown: expect no change.") -def test_posinterval_no_transaction(verbose_context, common, mini_restr): +def test_pos_interval_no_transaction(verbose_context, common, mini_restr): """Test no transaction""" before = common.PositionIntervalMap().fetch() with verbose_context: diff --git a/tests/common/test_device.py b/tests/common/test_device.py index 19103cf98..abfe60863 100644 --- a/tests/common/test_device.py +++ b/tests/common/test_device.py @@ -16,7 +16,7 @@ def test_get_device(common, mini_content): assert len(dev) == 3, "Unexpected number of devices found" -def test_spikegadets_system_alias(mini_insert, common): +def test_spike_gadgets_system_alias(mini_insert, common): assert ( common.DataAcquisitionDevice()._add_system("MCU") == "SpikeGadgets" ), "SpikeGadgets MCU alias not found" diff --git a/tests/common/test_ephys.py b/tests/common/test_ephys.py index 3887e00fc..37f298fdc 100644 --- a/tests/common/test_ephys.py +++ b/tests/common/test_ephys.py @@ -25,7 +25,7 @@ def test_electrode_populate(common_ephys): assert len(common_ephys.Electrode()) == 128, "Electrode.populate failed" -def test_egroup_populate(common_ephys): +def test_elec_group_populate(common_ephys): common_ephys.ElectrodeGroup.populate() assert ( len(common_ephys.ElectrodeGroup()) == 32 @@ -37,7 +37,7 @@ def test_raw_populate(common_ephys): assert len(common_ephys.Raw()) == 1, "Raw.populate failed" -def test_samplecount_populate(common_ephys): +def test_sample_count_populate(common_ephys): common_ephys.SampleCount.populate() assert len(common_ephys.SampleCount()) == 1, "SampleCount.populate failed" diff --git a/tests/common/test_insert.py b/tests/common/test_insert.py index 6d2fd18b3..f80967b4a 100644 --- a/tests/common/test_insert.py +++ b/tests/common/test_insert.py @@ -7,10 +7,10 @@ def test_insert_session(mini_insert, mini_content, mini_restr, common): subj_raw = mini_content.subject meta_raw = mini_content - sess_data = (common.Session & mini_restr).fetch1() + session_data = (common.Session & mini_restr).fetch1() assert ( - sess_data["subject_id"] == subj_raw.subject_id - ), "Subjuect ID not match" + session_data["subject_id"] == subj_raw.subject_id + ), "Subject ID not match" attrs = [ ("institution_name", "institution"), @@ -20,37 +20,37 @@ def test_insert_session(mini_insert, mini_content, mini_restr, common): ("experiment_description", "experiment_description"), ] - for sess_attr, meta_attr in attrs: - assert sess_data[sess_attr] == getattr( + for session_attr, meta_attr in attrs: + assert session_data[session_attr] == getattr( meta_raw, meta_attr - ), f"Session table {sess_attr} not match raw data {meta_attr}" + ), f"Session table {session_attr} not match raw data {meta_attr}" time_attrs = [ ("session_start_time", "session_start_time"), ("timestamps_reference_time", "timestamps_reference_time"), ] - for sess_attr, meta_attr in time_attrs: + for session_attr, meta_attr in time_attrs: # a. strip timezone info from meta_raw # b. convert to timestamp # c. compare precision to 1 second - assert sess_data[sess_attr].timestamp() == approx( + assert session_data[session_attr].timestamp() == approx( getattr(meta_raw, meta_attr).replace(tzinfo=None).timestamp(), abs=1 - ), f"Session table {sess_attr} not match raw data {meta_attr}" + ), f"Session table {session_attr} not match raw data {meta_attr}" def test_insert_electrode_group(mini_insert, mini_content, common): group_name = "0" - egroup_data = ( + elec_group_data = ( common.ElectrodeGroup & {"electrode_group_name": group_name} ).fetch1() - egroup_raw = mini_content.electrode_groups.get(group_name) + elec_group_raw = mini_content.electrode_groups.get(group_name) assert ( - egroup_data["description"] == egroup_raw.description + elec_group_data["description"] == elec_group_raw.description ), "ElectrodeGroup description not match" - assert egroup_data["region_id"] == ( - common.BrainRegion & {"region_name": egroup_raw.location} + assert elec_group_data["region_id"] == ( + common.BrainRegion & {"region_name": elec_group_raw.location} ).fetch1( "region_id" ), "Region ID does not match across raw data and BrainRegion table" @@ -138,7 +138,7 @@ def test_insert_pos( assert data_obj_id == raw_obj_id, "PosObject insertion error" -def test_fetch_posobj( +def test_fetch_pos_obj( mini_insert, common, mini_pos, mini_pos_series, mini_pos_tbl ): pos_key = ( diff --git a/tests/common/test_interval.py b/tests/common/test_interval.py index 8353961f8..e720b4466 100644 --- a/tests/common/test_interval.py +++ b/tests/common/test_interval.py @@ -23,5 +23,5 @@ def test_plot_epoch(mini_insert, interval_list): epoch_label = fig.get_axes()[0].get_yticklabels()[-1].get_text() assert epoch_label == "epoch", "plot_epoch failed" - epoch_interv = fig.get_axes()[0].lines[0].get_ydata() - assert array_equal(epoch_interv, [1, 1]), "plot_epoch failed" + epoch_interval = fig.get_axes()[0].lines[0].get_ydata() + assert array_equal(epoch_interval, [1, 1]), "plot_epoch failed" diff --git a/tests/common/test_interval_helpers.py b/tests/common/test_interval_helpers.py index d4e7eb1ac..3ef505f57 100644 --- a/tests/common/test_interval_helpers.py +++ b/tests/common/test_interval_helpers.py @@ -111,7 +111,7 @@ def test_interval_list_contains_ind(common, interval_list_dict): ), "Problem with common_interval.interval_list_contains_ind" -def test_insterval_list_contains(common, interval_list_dict): +def test_interval_list_contains(common, interval_list_dict): idxs = common.common_interval.interval_list_contains(**interval_list_dict) assert np.array_equal( idxs, np.array([1, 7, 8]) diff --git a/tests/common/test_lab.py b/tests/common/test_lab.py index 83ab84c10..0133c2bfe 100644 --- a/tests/common/test_lab.py +++ b/tests/common/test_lab.py @@ -71,7 +71,7 @@ def add_member_team(common_lab, add_admin): yield -def test_labmember_insert_file_str(mini_insert, common_lab, mini_copy_name): +def test_lab_member_insert_file_str(mini_insert, common_lab, mini_copy_name): before = common_lab.LabMember.fetch() common_lab.LabMember.insert_from_nwbfile(mini_copy_name) after = common_lab.LabMember.fetch() diff --git a/tests/common/test_region.py b/tests/common/test_region.py index 8241cb304..9a89ede03 100644 --- a/tests/common/test_region.py +++ b/tests/common/test_region.py @@ -29,4 +29,4 @@ def test_region_add(brain_region, region_dict): ) assert ( region_id == next_id - ), "Region.fetch_add() should autincrement region_id." + ), "Region.fetch_add() should autoincrement region_id." diff --git a/tests/common/test_session.py b/tests/common/test_session.py index 6e0a8f0ce..2276f23bd 100644 --- a/tests/common/test_session.py +++ b/tests/common/test_session.py @@ -1,5 +1,4 @@ import pytest -from datajoint.errors import DataJointError @pytest.fixture @@ -46,7 +45,7 @@ def add_session_to_group(session_group, mini_copy_name, group_name_dict): ) -def test_addremove_session_group( +def test_add_remove_session_group( common_session, session_group, session_group_dict, diff --git a/tests/conftest.py b/tests/conftest.py index fe8ce1a5b..8a58df39b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,7 +11,6 @@ from contextlib import nullcontext from pathlib import Path from shutil import rmtree as shutil_rmtree -from time import sleep as tsleep import datajoint as dj import numpy as np @@ -171,18 +170,18 @@ def server(request, teardown): @pytest.fixture(scope="session") -def server_creds(server): - yield server.creds +def server_credentials(server): + yield server.credentials @pytest.fixture(scope="session") -def dj_conn(request, server_creds, verbose, teardown): +def dj_conn(request, server_credentials, verbose, teardown): """Fixture for datajoint connection.""" config_file = "dj_local_conf.json_test" if Path(config_file).exists(): os.remove(config_file) - dj.config.update(server_creds) + dj.config.update(server_credentials) dj.config["loglevel"] = "INFO" if verbose else "ERROR" dj.config["custom"]["spyglass_dirs"] = {"base": str(BASE_DIR)} dj.config.save(config_file) @@ -210,34 +209,25 @@ def raw_dir(base_dir): @pytest.fixture(scope="session") def mini_path(raw_dir): path = raw_dir / TEST_FILE + DOWNLOADS.wait_for(TEST_FILE) # wait for wget download to finish - # wait for wget download to finish - if (nwb_download := DOWNLOADS.file_downloads.get(TEST_FILE)) is not None: - nwb_download.wait() - - # wait for download to finish - timeout, wait, found = 60, 5, False - for _ in range(timeout // wait): - if path.exists(): - found = True - break - tsleep(wait) - - if not found: + if not path.exists(): raise ConnectionError("Download failed.") yield path @pytest.fixture(scope="session") -def nodlc(request): +def no_dlc(request): yield NO_DLC @pytest.fixture(scope="session") -def skipif_nodlc(request): +def skipif_no_dlc(request): if NO_DLC: yield pytest.mark.skip(reason="Skipping DLC-dependent tests.") + else: + yield @pytest.fixture(scope="session") @@ -293,11 +283,9 @@ def mini_insert( _ = SpikeSortingOutput() - LabMember().insert1( - ["Root User", "Root", "User"], skip_duplicates=not teardown - ) + LabMember().insert1(["Root User", "Root", "User"], skip_duplicates=True) LabMember.LabMemberInfo().insert1( - ["Root User", "email", "root", 1], skip_duplicates=not teardown + ["Root User", "email", "root", 1], skip_duplicates=True ) dj_logger.info("Inserting test data.") @@ -395,14 +383,40 @@ def populate_exception(): yield PopulateException +@pytest.fixture(scope="session") +def frequent_imports(): + """Often needed for graph cascade.""" + from spyglass.common.common_ripple import RippleLFPSelection + from spyglass.decoding.v0.clusterless import UnitMarksIndicatorSelection + from spyglass.decoding.v0.sorted_spikes import ( + SortedSpikesIndicatorSelection, + ) + from spyglass.decoding.v1.core import PositionGroup + from spyglass.lfp.analysis.v1 import LFPBandSelection + from spyglass.mua.v1.mua import MuaEventsV1 + from spyglass.ripple.v1.ripple import RippleTimesV1 + from spyglass.spikesorting.v0.figurl_views import SpikeSortingRecordingView + + return ( + LFPBandSelection, + MuaEventsV1, + PositionGroup, + RippleLFPSelection, + RippleTimesV1, + SortedSpikesIndicatorSelection, + SpikeSortingRecordingView, + UnitMarksIndicatorSelection, + ) + + # -------------------------- FIXTURES, COMMON TABLES -------------------------- @pytest.fixture(scope="session") def video_keys(common, base_dir): - for file, download in DOWNLOADS.file_downloads.items(): - if file.endswith(".h264") and download is not None: - download.wait() # wait for videos to finish downloading + for file in DOWNLOADS.file_downloads: + if file.endswith(".h264"): + DOWNLOADS.wait_for(file) DOWNLOADS.rename_files() return common.VideoFile().fetch(as_dict=True) @@ -807,6 +821,7 @@ def dlc_project_name(): def insert_project( verbose_context, teardown, + video_keys, # wait for video downloads dlc_project_name, dlc_project_tbl, common, @@ -818,18 +833,32 @@ def insert_project( from deeplabcut.utils.auxiliaryfunctions import read_config, write_config + from spyglass.decoding.v1.core import PositionGroup + from spyglass.linearization.merge import LinearizedPositionOutput + from spyglass.linearization.v1 import LinearizationSelection + from spyglass.mua.v1.mua import MuaEventsV1 + from spyglass.ripple.v1 import RippleTimesV1 + + _ = ( + PositionGroup, + LinearizedPositionOutput, + LinearizationSelection, + MuaEventsV1, + RippleTimesV1, + ) + team_name = "sc_eb" common.LabTeam.insert1({"team_name": team_name}, skip_duplicates=True) + video_list = common.VideoFile().fetch( + "nwb_file_name", "epoch", as_dict=True + )[:2] with verbose_context: project_key = dlc_project_tbl.insert_new_project( project_name=dlc_project_name, bodyparts=bodyparts, lab_team=team_name, frames_per_video=100, - video_list=[ - {"nwb_file_name": mini_copy_name, "epoch": 0}, - {"nwb_file_name": mini_copy_name, "epoch": 1}, - ], + video_list=video_list, skip_duplicates=True, ) config_path = (dlc_project_tbl & project_key).fetch1("config_path") @@ -904,23 +933,8 @@ def labeled_vid_dir(extract_frames): @pytest.fixture(scope="session") -def fix_downloaded(labeled_vid_dir, project_dir): - """Grabs CollectedData and img files from project_dir, moves to labeled""" - for file in project_dir.parent.parent.glob("*"): - if file.is_dir(): - continue - dest = labeled_vid_dir / file.name - if dest.exists(): - dest.unlink() - dest.write_bytes(file.read_bytes()) - # TODO: revert to rename before merge - # file.rename(labeled_vid_dir / file.name) - - yield - - -@pytest.fixture(scope="session") -def add_training_files(dlc_project_tbl, project_key, fix_downloaded): +def add_training_files(dlc_project_tbl, project_key, labeled_vid_dir): + DOWNLOADS.move_dlc_items(labeled_vid_dir) dlc_project_tbl.add_training_files(project_key, skip_duplicates=True) yield @@ -970,11 +984,13 @@ def model_train_key(sgp, project_key, training_params_key): @pytest.fixture(scope="session") -def populate_training(sgp, fix_downloaded, model_train_key, add_training_files): +def populate_training( + sgp, model_train_key, add_training_files, labeled_vid_dir +): train_tbl = sgp.v1.DLCModelTraining if len(train_tbl & model_train_key) == 0: _ = add_training_files - _ = fix_downloaded + DOWNLOADS.move_dlc_items(labeled_vid_dir) sgp.v1.DLCModelTraining.populate(model_train_key) yield model_train_key @@ -1004,7 +1020,7 @@ def populate_model(sgp, model_key): @pytest.fixture(scope="session") def pose_estimation_key(sgp, mini_copy_name, populate_model, model_key): - yield sgp.v1.DLCPoseEstimationSelection.insert_estimation_task( + yield sgp.v1.DLCPoseEstimationSelection().insert_estimation_task( { "nwb_file_name": mini_copy_name, "epoch": 1, @@ -1094,13 +1110,10 @@ def cohort_selection(sgp, si_key, si_params_name): @pytest.fixture(scope="session") -def cohort_key(sgp, cohort_selection): - yield cohort_selection.copy() - - -@pytest.fixture(scope="session") -def populate_cohort(sgp, cohort_selection, populate_si): - sgp.v1.DLCSmoothInterpCohort.populate(cohort_selection) +def cohort_key(sgp, cohort_selection, populate_si): + cohort_tbl = sgp.v1.DLCSmoothInterpCohort() + cohort_tbl.populate(cohort_selection) + yield cohort_tbl.fetch("KEY", as_dict=True)[0] @pytest.fixture(scope="session") @@ -1130,7 +1143,7 @@ def centroid_params(sgp): @pytest.fixture(scope="session") -def centroid_selection(sgp, cohort_key, populate_cohort, centroid_params): +def centroid_selection(sgp, cohort_key, centroid_params): centroid_key = cohort_key.copy() centroid_key = { key: val @@ -1194,7 +1207,10 @@ def populate_orient(sgp, orient_selection): @pytest.fixture(scope="session") -def dlc_selection(sgp, centroid_key, orient_key, populate_orient): +def dlc_selection( + sgp, centroid_key, orient_key, populate_orient, populate_centroid +): + _ = populate_orient, populate_centroid dlc_key = { key: val for key, val in centroid_key.items() diff --git a/tests/container.py b/tests/container.py index b9d77263e..1747d76b8 100644 --- a/tests/container.py +++ b/tests/container.py @@ -190,7 +190,7 @@ def add_user(self) -> int: return None @property - def creds(self): + def credentials(self): """Datajoint credentials for this container.""" return { "database.host": "localhost", @@ -204,7 +204,7 @@ def creds(self): @property def connected(self) -> bool: self.wait() - dj.config.update(self.creds) + dj.config.update(self.credentials) return dj.conn().is_connected def stop(self, remove=True) -> None: diff --git a/tests/data_downloader.py b/tests/data_downloader.py index 98a254eda..cb58e1c71 100644 --- a/tests/data_downloader.py +++ b/tests/data_downloader.py @@ -1,10 +1,14 @@ from functools import cached_property from os import environ as os_environ from pathlib import Path +from shutil import copy as shutil_copy from subprocess import DEVNULL, Popen from sys import stderr, stdout +from time import sleep as time_sleep from typing import Dict, Union +from datajoint import logger as dj_logger + UCSF_BOX_USER = os_environ.get("UCSF_BOX_USER") UCSF_BOX_TOKEN = os_environ.get("UCSF_BOX_TOKEN") BASE_URL = "ftps://ftp.box.com/trodes_to_nwb_test_data/" @@ -87,6 +91,7 @@ def __init__( self.cmd_kwargs = dict(stdout=stdout, stderr=stderr) self.base_dir = Path(base_dir).resolve() + self.download_dlc = download_dlc self.file_paths = file_paths if download_dlc else file_paths[:NON_DLC] self.base_dir.mkdir(exist_ok=True) @@ -94,7 +99,7 @@ def __init__( _ = self.file_downloads def rename_files(self): - """Redund, but allows rerun later in startup process of conftest.""" + """Redundant, but allows rerun later in startup process of conftest.""" for path in self.file_paths: target, url = path["target_name"], path["url"] target_dir = self.base_dir / path["relative_dir"] @@ -112,28 +117,42 @@ def file_downloads(self) -> Dict[str, Union[Popen, None]]: for path in self.file_paths: target, url = path["target_name"], path["url"] target_dir = self.base_dir / path["relative_dir"] + target_dir.mkdir(exist_ok=True, parents=True) dest = target_dir / target + cmd = ( + ["echo", f"Already have {target}"] + if dest.exists() + else self.cmd + [target_dir, url] + ) + ret[target] = Popen(cmd, **self.cmd_kwargs) + return ret - if dest.exists(): - ret[target] = None - continue + def wait_for(self, target: str): + """Wait for target to finish downloading.""" + status = self.file_downloads.get(target).poll() + limit = 10 + while status is None and limit > 0: + time_sleep(5) # Some + limit -= 1 + status = self.file_downloads.get(target).poll() + if status != 0: + raise ValueError(f"Error downloading: {target}") + if limit < 1: + raise TimeoutError(f"Timeout downloading: {target}") - target_dir.mkdir(exist_ok=True, parents=True) - ret[target] = Popen(self.cmd + [target_dir, url], **self.cmd_kwargs) - return ret + def move_dlc_items(self, dest_dir: Path): + """Move completed DLC files to dest_dir.""" + if not self.download_dlc: + return + if not dest_dir.exists(): + dest_dir.mkdir(parents=True) - def check_download(self, download, info): - if download is not None: - download.wait() - if download.returncode: - return download - return None - - @property - def download_errors(self): - ret = [] - for download, item in zip(self.file_downloads, self.file_paths): - if d_status := self.check_download(download, item): - ret.append(d_status) - continue - return ret + for path in self.file_paths[NON_DLC:]: + target = path["target_name"] + self.wait_for(target) # Could be faster if moved finished first + + src_path = self.base_dir / path["relative_dir"] / target + dest_path = dest_dir / src_path.name + if not dest_path.exists(): + shutil_copy(str(src_path), str(dest_path)) + dj_logger.info(f"Moved: {src_path} -> {dest_path}") diff --git a/tests/lfp/test_lfp.py b/tests/lfp/test_lfp.py index b496ae445..b85bcc3bf 100644 --- a/tests/lfp/test_lfp.py +++ b/tests/lfp/test_lfp.py @@ -25,13 +25,13 @@ def test_lfp_dataframe(lfp, lfp_raw, lfp_merge_key): def test_lfp_band_dataframe(lfp_band_analysis_raw, lfp_band, lfp_band_key): - lfpb_raw = ( + lfp_band_raw = ( lfp_band_analysis_raw.processing["ecephys"] .fields["data_interfaces"]["LFP"] .electrical_series["filtered data"] ) - lfpb_index = Index(lfpb_raw.timestamps, name="time") - df_raw = DataFrame(lfpb_raw.data, index=lfpb_index) + lfp_band_index = Index(lfp_band_raw.timestamps, name="time") + df_raw = DataFrame(lfp_band_raw.data, index=lfp_band_index) df_fetch = (lfp_band.LFPBandV1 & lfp_band_key).fetch1_dataframe() assert df_raw.equals(df_fetch), "LFPBand dataframe not match." @@ -91,7 +91,7 @@ def test_invalid_band_selection( set_elec(**valid | {"reference_electrode_list": [3]}) -def test_artifactparam_defaults(art_params, art_param_defaults): +def test_artifact_param_defaults(art_params, art_param_defaults): assert set(art_params.fetch("artifact_params_name")).issubset( set(art_param_defaults) ), "LFPArtifactDetectionParameters missing default item." diff --git a/tests/position/conftest.py b/tests/position/conftest.py index c6c58d199..8f9e90795 100644 --- a/tests/position/conftest.py +++ b/tests/position/conftest.py @@ -30,6 +30,7 @@ def dlc_video_params(sgp): "params": { "percent_frames": 0.05, "incl_likelihood": True, + "processor": "opencv", }, }, skip_duplicates=True, diff --git a/tests/position/test_dlc_cent.py b/tests/position/test_dlc_cent.py index a3675b2ae..7980a2b30 100644 --- a/tests/position/test_dlc_cent.py +++ b/tests/position/test_dlc_cent.py @@ -47,17 +47,34 @@ def test_validate_params(params_tbl): @pytest.mark.parametrize( - "key", ["four_led_centroid", "two_pt_centroid", "one_pt_centroid"] + "key", ["one_pt_centroid", "two_pt_centroid", "four_led_centroid"] ) def test_centroid_calcs(key, sgp): + from spyglass.position.v1.dlc_utils import Centroid + points = sgp.v1.position_dlc_centroid._key_to_points[key] - func = sgp.v1.position_dlc_centroid._key_to_func_dict[key] df = generate_led_df(points) - ret = func(df, max_LED_separation=100, points={p: p for p in points}) + ret = Centroid( + df, max_LED_separation=100, points={p: p for p in points} + ).centroid assert np.all(ret[:-1] == 1), f"Centroid calculation failed for {key}" assert np.all(np.isnan(ret[-1])), f"Centroid calculation failed for {key}" - with pytest.raises(KeyError): - func(df) # Missing led separation/point names + +def test_centroid_error(sgp): + from spyglass.position.v1.dlc_utils import Centroid + + one_pt = {"point1": "point1"} + df = generate_led_df(one_pt) + Centroid(df, points=one_pt) # no sep ok on one point + + two_pt = {"point1": "point1", "point2": "point2"} + with pytest.raises(ValueError): + Centroid(df, points=two_pt) # Missing led separation for valid points + + three_pt = {"point1": "point1", "point2": "point2", "point3": "point3"} + three_pt_df = generate_led_df(three_pt) + with pytest.raises(ValueError): # invalid point number + Centroid(three_pt_df, points=three_pt, max_LED_separation=100) diff --git a/tests/position/test_dlc_model.py b/tests/position/test_dlc_model.py index 6f1ccf89d..036f98cdf 100644 --- a/tests/position/test_dlc_model.py +++ b/tests/position/test_dlc_model.py @@ -14,5 +14,5 @@ def test_model_params_default(sgp): def test_model_input_assert(sgp): - with pytest.raises(AssertionError): + with pytest.raises(FileNotFoundError): sgp.v1.DLCModelInput().insert1({"config_path": "/fake/path/"}) diff --git a/tests/position/test_dlc_pos_est.py b/tests/position/test_dlc_pos_est.py index fdf055843..f66f06616 100644 --- a/tests/position/test_dlc_pos_est.py +++ b/tests/position/test_dlc_pos_est.py @@ -6,9 +6,9 @@ def pos_est_sel(sgp): yield sgp.v1.position_dlc_pose_estimation.DLCPoseEstimationSelection() -@pytest.mark.usefixtures("skipif_nodlc") +@pytest.mark.usefixtures("skipif_no_dlc") def test_rename_non_default_columns(sgp, common, pos_est_sel, video_keys): - vid_path, vid_name, _, _ = sgp.v1.dlc_utils.get_video_path(video_keys[0]) + vid_path, vid_name, _, _ = sgp.v1.dlc_utils.get_video_info(video_keys[0]) input = "0, 10, 0, 1000" output = pos_est_sel.get_video_crop(vid_path + vid_name, input) diff --git a/tests/position/test_dlc_sel.py b/tests/position/test_dlc_sel.py index 35b33fe06..b0cd3340b 100644 --- a/tests/position/test_dlc_sel.py +++ b/tests/position/test_dlc_sel.py @@ -1,4 +1,4 @@ -def test_dlcvideo_default(sgp): +def test_dlc_video_default(sgp): expected_default = { "dlc_pos_video_params_name": "default", "params": { diff --git a/tests/position/test_dlc_train.py b/tests/position/test_dlc_train.py index eefa26f66..acd4d701d 100644 --- a/tests/position/test_dlc_train.py +++ b/tests/position/test_dlc_train.py @@ -25,9 +25,9 @@ def test_existing_params( assert len(params_query) == 1, "Existing params duplicated" -@pytest.mark.usefixtures("skipif_nodlc") -def test_get_params(nodlc, verbose_context, dlc_training_params): - if nodlc: # Decorator wasn't working here, so duplicate skipif +@pytest.mark.usefixtures("skipif_no_dlc") +def test_get_params(no_dlc, verbose_context, dlc_training_params): + if no_dlc: # Decorator wasn't working here, so duplicate skipif pytest.skip(reason="Skipping DLC-dependent tests.") params_tbl, _ = dlc_training_params diff --git a/tests/position/test_trodes.py b/tests/position/test_trodes.py index 92fdfeeb1..6d65f375c 100644 --- a/tests/position/test_trodes.py +++ b/tests/position/test_trodes.py @@ -61,7 +61,8 @@ def test_fetch_df(trodes_pos_v1, trodes_params): assert hash_df == hash_exp, "Dataframe differs from expected" -def test_trodes_video(sgp): +def test_trodes_video(sgp, trodes_pos_v1): + _ = trodes_pos_v1 # ensure table is populated vid_tbl = sgp.v1.TrodesPosVideo() _ = vid_tbl.populate() assert len(vid_tbl) == 2, "Failed to populate TrodesPosVideo" diff --git a/tests/utils/test_db_settings.py b/tests/utils/test_db_settings.py index 3b72ec885..b2435e1f1 100644 --- a/tests/utils/test_db_settings.py +++ b/tests/utils/test_db_settings.py @@ -12,10 +12,10 @@ def db_settings(user_name): return DatabaseSettings( user_name=user_name, - host_name=docker_server.creds["database.host"], + host_name=docker_server.credentials["database.host"], target_database=id, - exec_user=docker_server.creds["database.user"], - exec_pass=docker_server.creds["database.password"], + exec_user=docker_server.credentials["database.user"], + exec_pass=docker_server.credentials["database.password"], test_mode=no_docker, ) diff --git a/tests/utils/test_graph.py b/tests/utils/test_graph.py index 7d5257a36..18899e147 100644 --- a/tests/utils/test_graph.py +++ b/tests/utils/test_graph.py @@ -72,9 +72,11 @@ def test_add_leaf_restr_ft(restr_graph_new_leaf): @pytest.fixture(scope="session") -def restr_graph_root(restr_graph, common, lfp_band, lin_v1): +def restr_graph_root(restr_graph, common, lfp_band, lin_v1, frequent_imports): from spyglass.utils.dj_graph import RestrGraph + _ = lfp_band, lin_v1, frequent_imports # tables populated + yield RestrGraph( seed_table=common.Session(), table_name=common.Session.full_table_name, diff --git a/tests/utils/test_mixin.py b/tests/utils/test_mixin.py index 5b6beb4d0..93d13407a 100644 --- a/tests/utils/test_mixin.py +++ b/tests/utils/test_mixin.py @@ -21,7 +21,7 @@ class Mixin(SpyglassMixin, dj.Manual): reason="Error only on verbose or new declare.", ) def test_bad_prefix(caplog, dj_conn, Mixin): - schema_bad = dj.Schema("badprefix", {}, connection=dj_conn) + schema_bad = dj.Schema("bad_prefix", {}, connection=dj_conn) schema_bad(Mixin) assert "Schema prefix not in SHARED_MODULES" in caplog.text @@ -55,7 +55,7 @@ def test_merge_chain_join( ] end_len = [len(chain) for chain in all_chains] - assert sum(end_len) == 4, "Merge chains not joined correctly." + assert sum(end_len) >= 3, "Merge chains not joined correctly." def test_get_chain(Nwbfile, pos_merge_tables): diff --git a/tests/utils/test_nwb_helper_fn.py b/tests/utils/test_nwb_helper_fn.py index ff8175e44..bf2e8aba1 100644 --- a/tests/utils/test_nwb_helper_fn.py +++ b/tests/utils/test_nwb_helper_fn.py @@ -36,7 +36,7 @@ def custom_nwbfile(common): filtering="filtering", group=elec_group, ) - elecs_region = nwbfile.electrodes.create_region( + electrode_region = nwbfile.electrodes.create_region( name="electrodes", region=[2, 3, 4, 5], description="description", # indices @@ -46,7 +46,7 @@ def custom_nwbfile(common): name="eseries", data=[0, 1, 2], timestamps=[0.0, 1.0, 2.0], - electrodes=elecs_region, + electrodes=electrode_region, ) ) yield nwbfile From b89ea99949dd312de2cd571c60149115c36e385f Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Wed, 26 Jun 2024 17:13:39 -0500 Subject: [PATCH 08/94] Prevent delete orphans (#1002) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * WIP: No Orphans downstream * WIP: Bidirectional RestrGraph, remove TableChains * WIP: bridge up to interval list * Add tests for new delete * Update changelog * Fix typo * WIP: topological sort of deletes * ✅ : Topological sort * Revise downloads * Update src/spyglass/utils/dj_helper_fn.py Co-authored-by: Samuel Bray * Ignore unimported non-spyglass in cascade * Load part-master cache before graph * Add more automatic imports * Pin twine req for build --------- Co-authored-by: Samuel Bray --- .github/workflows/test-package-build.yml | 4 +- .gitignore | 2 + CHANGELOG.md | 2 + docs/src/misc/mixin.md | 37 +- notebooks/01_Insert_Data.ipynb | 2 +- notebooks/03_Merge_Tables.ipynb | 10 +- notebooks/py_scripts/01_Insert_Data.py | 2 +- notebooks/py_scripts/03_Merge_Tables.py | 12 +- pyproject.toml | 4 +- src/spyglass/common/common_interval.py | 7 + src/spyglass/common/common_usage.py | 21 +- src/spyglass/decoding/v0/core.py | 1 + src/spyglass/utils/dj_graph.py | 493 ++++++++++++----------- src/spyglass/utils/dj_helper_fn.py | 14 +- src/spyglass/utils/dj_merge_tables.py | 5 +- src/spyglass/utils/dj_mixin.py | 363 +++++++++-------- tests/common/test_usage.py | 89 ++++ tests/utils/conftest.py | 17 +- tests/utils/test_chains.py | 31 +- tests/utils/test_graph.py | 134 +++++- tests/utils/test_merge.py | 19 + tests/utils/test_mixin.py | 99 +++-- 22 files changed, 840 insertions(+), 528 deletions(-) create mode 100644 tests/common/test_usage.py diff --git a/.github/workflows/test-package-build.yml b/.github/workflows/test-package-build.yml index 41aace719..c93b77398 100644 --- a/.github/workflows/test-package-build.yml +++ b/.github/workflows/test-package-build.yml @@ -27,7 +27,9 @@ jobs: - uses: actions/setup-python@v5 with: python-version: 3.9 - - run: pip install --upgrade build twine + - run: | + pip install --upgrade build twine + pip install importlib_metadata==7.2.1 # twine #977 - name: Build sdist and wheel run: python -m build - run: twine check dist/* diff --git a/.gitignore b/.gitignore index 1f18f4178..0cbd43c74 100644 --- a/.gitignore +++ b/.gitignore @@ -60,6 +60,7 @@ coverage.xml .hypothesis/ .pytest_cache/ tests/_data/* +wget-log* # Translations *.mo @@ -128,6 +129,7 @@ dmypy.json .pyre/ # Test Data Files +tests/_data/* *.dat *.mda *.rec diff --git a/CHANGELOG.md b/CHANGELOG.md index 69d2d0a87..0092789c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,8 @@ PositionIntervalMap.alter() - Add pytests for position pipeline, various `test_mode` exceptions #966 - Migrate `pip` dependencies from `environment.yml`s to `pyproject.toml` #966 - Add documentation for common error messages #997 +- Expand `delete_downstream_merge` -> `delete_downstream_parts`. #1002 +- `cautious_delete` now checks `IntervalList` and externals tables. #1002 - Allow mixin tables with parallelization in `make` to run populate with `processes > 1` #1001 diff --git a/docs/src/misc/mixin.md b/docs/src/misc/mixin.md index ab49b0c49..23135d3c4 100644 --- a/docs/src/misc/mixin.md +++ b/docs/src/misc/mixin.md @@ -136,29 +136,38 @@ masters, or null entry masters without matching data. For [Merge tables](./merge_tables.md), this is a significant problem. If a user wants to delete all entries associated with a given session, she must find all -Merge entries and delete them in the correct order. The mixin provides a -function, `delete_downstream_merge`, to handle this, which is run by default -when calling `delete`. +part table entries, including Merge tables, and delete them in the correct +order. The mixin provides a function, `delete_downstream_parts`, to handle this, +which is run by default when calling `delete`. -`delete_downstream_merge`, also aliased as `ddm`, identifies all Merge tables -downstream of where it is called. If `dry_run=True`, it will return a list of -entries that would be deleted, otherwise it will delete them. +`delete_downstream_parts`, also aliased as `ddp`, identifies all part tables +with foreign key references downstream of where it is called. If `dry_run=True`, +it will return a list of entries that would be deleted, otherwise it will delete +them. -Importantly, `delete_downstream_merge` cannot properly interact with tables that +Importantly, `delete_downstream_parts` cannot properly interact with tables that have not been imported into the current namespace. If you are having trouble with part deletion errors, import the offending table and rerun the function with `reload_cache=True`. ```python +import datajoint as dj from spyglass.common import Nwbfile restricted_nwbfile = Nwbfile() & "nwb_file_name LIKE 'Name%'" -restricted_nwbfile.delete_downstream_merge(dry_run=False) -# DataJointError("Attempt to delete part table MyMerge.Part before ... + +vanilla_dj_table = dj.FreeTable(dj.conn(), Nwbfile.full_table_name) +vanilla_dj_table.delete() +# DataJointError("Attempt to delete part table MyMerge.Part before ... ") + +restricted_nwbfile.delete() +# [WARNING] Spyglass: No part deletes found w/ Nwbfile ... +# OR +# ValueError("Please import MyMerge and try again.") from spyglass.example import MyMerge -restricted_nwbfile.delete_downstream_merge(reload_cache=True, dry_run=False) +restricted_nwbfile.delete_downstream_parts(reload_cache=True, dry_run=False) ``` Because each table keeps a cache of downstream merge tables, it is important to @@ -169,13 +178,13 @@ Speed gains can also be achieved by avoiding re-instancing the table each time. # Slow from spyglass.common import Nwbfile -(Nwbfile() & "nwb_file_name LIKE 'Name%'").ddm(dry_run=False) -(Nwbfile() & "nwb_file_name LIKE 'Other%'").ddm(dry_run=False) +(Nwbfile() & "nwb_file_name LIKE 'Name%'").ddp(dry_run=False) +(Nwbfile() & "nwb_file_name LIKE 'Other%'").ddp(dry_run=False) # Faster from spyglass.common import Nwbfile nwbfile = Nwbfile() -(nwbfile & "nwb_file_name LIKE 'Name%'").ddm(dry_run=False) -(nwbfile & "nwb_file_name LIKE 'Other%'").ddm(dry_run=False) +(nwbfile & "nwb_file_name LIKE 'Name%'").ddp(dry_run=False) +(nwbfile & "nwb_file_name LIKE 'Other%'").ddp(dry_run=False) ``` diff --git a/notebooks/01_Insert_Data.ipynb b/notebooks/01_Insert_Data.ipynb index 23c208cdf..a92e14ca0 100644 --- a/notebooks/01_Insert_Data.ipynb +++ b/notebooks/01_Insert_Data.ipynb @@ -2134,7 +2134,7 @@ "```python\n", "nwbfile = sgc.Nwbfile()\n", "\n", - "(nwbfile & {\"nwb_file_name\": nwb_copy_file_name}).delete_downstream_merge(\n", + "(nwbfile & {\"nwb_file_name\": nwb_copy_file_name}).delete_downstream_parts(\n", " dry_run=False, # True will show Merge Table entries that would be deleted\n", ")\n", "```\n", diff --git a/notebooks/03_Merge_Tables.ipynb b/notebooks/03_Merge_Tables.ipynb index 6adbbd5bf..0f9b2c3c2 100644 --- a/notebooks/03_Merge_Tables.ipynb +++ b/notebooks/03_Merge_Tables.ipynb @@ -90,7 +90,7 @@ "import spyglass.common as sgc\n", "import spyglass.lfp as lfp\n", "from spyglass.utils.nwb_helper_fn import get_nwb_copy_filename\n", - "from spyglass.utils.dj_merge_tables import delete_downstream_merge, Merge\n", + "from spyglass.utils.dj_merge_tables import delete_downstream_parts, Merge\n", "from spyglass.common.common_ephys import LFP as CommonLFP # Upstream 1\n", "from spyglass.lfp.lfp_merge import LFPOutput # Merge Table\n", "from spyglass.lfp.v1.lfp import LFPV1 # Upstream 2" @@ -955,8 +955,8 @@ "2. use `merge_delete_parent` to delete from the parent sources, getting rid of\n", " the entries in the source table they came from.\n", "\n", - "3. use `delete_downstream_merge` to find Merge Tables downstream of any other\n", - " table and get rid full entries, avoiding orphaned master table entries.\n", + "3. use `delete_downstream_parts` to find downstream part tables, like Merge \n", + " Tables, and get rid full entries, avoiding orphaned master table entries.\n", "\n", "The two latter cases can be destructive, so we include an extra layer of\n", "protection with `dry_run`. When true (by default), these functions return\n", @@ -1016,7 +1016,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "`delete_downstream_merge` is available from any other table in the pipeline,\n", + "`delete_downstream_parts` is available from any other table in the pipeline,\n", "but it does take some time to find the links downstream. If you're using this,\n", "you can save time by reassigning your table to a variable, which will preserve\n", "a copy of the previous search.\n", @@ -1056,7 +1056,7 @@ "source": [ "nwbfile = sgc.Nwbfile()\n", "\n", - "(nwbfile & nwb_file_dict).delete_downstream_merge(\n", + "(nwbfile & nwb_file_dict).delete_downstream_parts(\n", " dry_run=True,\n", " reload_cache=False, # if still encountering errors, try setting this to True\n", ")" diff --git a/notebooks/py_scripts/01_Insert_Data.py b/notebooks/py_scripts/01_Insert_Data.py index 48ddae39b..f569f971f 100644 --- a/notebooks/py_scripts/01_Insert_Data.py +++ b/notebooks/py_scripts/01_Insert_Data.py @@ -378,7 +378,7 @@ # ```python # nwbfile = sgc.Nwbfile() # -# (nwbfile & {"nwb_file_name": nwb_copy_file_name}).delete_downstream_merge( +# (nwbfile & {"nwb_file_name": nwb_copy_file_name}).delete_downstream_parts( # dry_run=False, # True will show Merge Table entries that would be deleted # ) # ``` diff --git a/notebooks/py_scripts/03_Merge_Tables.py b/notebooks/py_scripts/03_Merge_Tables.py index ac3ad4e69..690bc7834 100644 --- a/notebooks/py_scripts/03_Merge_Tables.py +++ b/notebooks/py_scripts/03_Merge_Tables.py @@ -5,7 +5,7 @@ # extension: .py # format_name: light # format_version: '1.5' -# jupytext_version: 1.15.2 +# jupytext_version: 1.16.0 # kernelspec: # display_name: spy # language: python @@ -64,7 +64,7 @@ import spyglass.common as sgc import spyglass.lfp as lfp from spyglass.utils.nwb_helper_fn import get_nwb_copy_filename -from spyglass.utils.dj_merge_tables import delete_downstream_merge, Merge +from spyglass.utils.dj_merge_tables import delete_downstream_parts, Merge from spyglass.common.common_ephys import LFP as CommonLFP # Upstream 1 from spyglass.lfp.lfp_merge import LFPOutput # Merge Table from spyglass.lfp.v1.lfp import LFPV1 # Upstream 2 @@ -192,8 +192,8 @@ # 2. use `merge_delete_parent` to delete from the parent sources, getting rid of # the entries in the source table they came from. # -# 3. use `delete_downstream_merge` to find Merge Tables downstream of any other -# table and get rid full entries, avoiding orphaned master table entries. +# 3. use `delete_downstream_parts` to find downstream part tables, like Merge +# Tables, and get rid full entries, avoiding orphaned master table entries. # # The two latter cases can be destructive, so we include an extra layer of # protection with `dry_run`. When true (by default), these functions return @@ -204,7 +204,7 @@ LFPOutput.merge_delete_parent(restriction=nwb_file_dict, dry_run=True) -# `delete_downstream_merge` is available from any other table in the pipeline, +# `delete_downstream_parts` is available from any other table in the pipeline, # but it does take some time to find the links downstream. If you're using this, # you can save time by reassigning your table to a variable, which will preserve # a copy of the previous search. @@ -216,7 +216,7 @@ # + nwbfile = sgc.Nwbfile() -(nwbfile & nwb_file_dict).delete_downstream_merge( +(nwbfile & nwb_file_dict).delete_downstream_parts( dry_run=True, reload_cache=False, # if still encountering errors, try setting this to True ) diff --git a/pyproject.toml b/pyproject.toml index 2538b00dc..061947e3d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -126,8 +126,8 @@ ignore-words-list = 'nevers' minversion = "7.0" addopts = [ # "-sv", # no capture, verbose output - # "--sw", # stepwise: resume with next test after failure - # "--pdb", # drop into debugger on failure + "--sw", # stepwise: resume with next test after failure + "--pdb", # drop into debugger on failure "-p no:warnings", # "--no-teardown", # don't teardown the database after tests # "--quiet-spy", # don't show logging from spyglass diff --git a/src/spyglass/common/common_interval.py b/src/spyglass/common/common_interval.py index 66e82bda8..6e4d6b042 100644 --- a/src/spyglass/common/common_interval.py +++ b/src/spyglass/common/common_interval.py @@ -7,6 +7,7 @@ import pandas as pd from spyglass.utils import SpyglassMixin, logger +from spyglass.utils.dj_helper_fn import get_child_tables from .common_session import Session # noqa: F401 @@ -152,6 +153,12 @@ def plot_epoch_pos_raw_intervals(self, figsize=(20, 5), return_fig=False): if return_fig: return fig + def nightly_cleanup(self, dry_run=True): + orphans = self - get_child_tables(self) + if dry_run: + return orphans + orphans.super_delete(warn=False) + def intervals_by_length(interval_list, min_length=0.0, max_length=1e10): """Select intervals of certain lengths from an interval list. diff --git a/src/spyglass/common/common_usage.py b/src/spyglass/common/common_usage.py index 6616fedf6..f6f15ce76 100644 --- a/src/spyglass/common/common_usage.py +++ b/src/spyglass/common/common_usage.py @@ -15,7 +15,7 @@ from pynwb import NWBHDF5IO from spyglass.common.common_nwbfile import AnalysisNwbfile, Nwbfile -from spyglass.settings import export_dir +from spyglass.settings import export_dir, test_mode from spyglass.utils import SpyglassMixin, logger from spyglass.utils.dj_graph import RestrGraph from spyglass.utils.dj_helper_fn import ( @@ -122,7 +122,8 @@ def insert1_return_pk(self, key: dict, **kwargs) -> int: export_id = query.fetch1("export_id") export_key = {"export_id": export_id} if query := (Export & export_key): - query.super_delete(warn=False) + safemode = False if test_mode else None # No prompt in tests + query.super_delete(warn=False, safemode=safemode) logger.info(f"{status} {export_key}") return export_id @@ -193,9 +194,11 @@ def _max_export_id(self, paper_id: str, return_all=False) -> int: all_export_ids = query.fetch("export_id") return all_export_ids if return_all else max(all_export_ids) - def paper_export_id(self, paper_id: str) -> dict: + def paper_export_id(self, paper_id: str, return_all=False) -> dict: """Return the maximum export_id for a paper, used to populate Export.""" - return {"export_id": self._max_export_id(paper_id)} + if not return_all: + return {"export_id": self._max_export_id(paper_id)} + return [{"export_id": id} for id in self._max_export_id(paper_id, True)] @schema @@ -234,11 +237,11 @@ def populate_paper(self, paper_id: Union[str, dict]): self.populate(ExportSelection().paper_export_id(paper_id)) def make(self, key): - query = ExportSelection & key - paper_key = query.fetch("paper_id", as_dict=True)[0] + paper_key = (ExportSelection & key).fetch("paper_id", as_dict=True)[0] + query = ExportSelection & paper_key # Null insertion if export_id is not the maximum for the paper - all_export_ids = query._max_export_id(paper_key, return_all=True) + all_export_ids = ExportSelection()._max_export_id(paper_key, True) max_export_id = max(all_export_ids) if key.get("export_id") != max_export_id: logger.info( @@ -259,7 +262,7 @@ def make(self, key): (self.Table & id_dict).delete_quick() (self.Table & id_dict).delete_quick() - restr_graph = query.get_restr_graph(paper_key) + restr_graph = ExportSelection().get_restr_graph(paper_key) file_paths = unique_dicts( # Original plus upstream files query.list_file_paths(paper_key) + restr_graph.file_paths ) @@ -275,7 +278,7 @@ def make(self, key): # Writes but does not run mysqldump. Assumes single version per paper. version_key = query.fetch("spyglass_version", as_dict=True)[0] self.write_export( - free_tables=restr_graph.all_ft, **paper_key, **version_key + free_tables=restr_graph.restr_ft, **paper_key, **version_key ) self.insert1({**key, **paper_key}) diff --git a/src/spyglass/decoding/v0/core.py b/src/spyglass/decoding/v0/core.py index 5664c12d9..3df82f318 100644 --- a/src/spyglass/decoding/v0/core.py +++ b/src/spyglass/decoding/v0/core.py @@ -13,6 +13,7 @@ ObservationModel, ) except ImportError as e: + RandomWalk, Uniform, Environment, ObservationModel = None, None, None, None logger.warning(e) from spyglass.common.common_behav import PositionIntervalMap, RawPosition diff --git a/src/spyglass/utils/dj_graph.py b/src/spyglass/utils/dj_graph.py index 5bf3d25d0..3e90d4736 100644 --- a/src/spyglass/utils/dj_graph.py +++ b/src/spyglass/utils/dj_graph.py @@ -4,16 +4,16 @@ """ from abc import ABC, abstractmethod -from collections.abc import KeysView +from copy import deepcopy from enum import Enum from functools import cached_property from itertools import chain as iter_chain -from typing import Any, Dict, List, Set, Tuple, Union +from typing import Any, Dict, Iterable, List, Set, Tuple, Union -import datajoint as dj from datajoint import FreeTable, Table from datajoint.condition import make_condition from datajoint.dependencies import unite_master_parts +from datajoint.user_tables import TableMeta from datajoint.utils import get_master, to_camel_case from networkx import ( NetworkXNoPath, @@ -25,12 +25,12 @@ from tqdm import tqdm from spyglass.utils import logger +from spyglass.utils.database_settings import SHARED_MODULES from spyglass.utils.dj_helper_fn import ( PERIPHERAL_TABLES, fuzzy_get, unique_dicts, ) -from spyglass.utils.dj_merge_tables import is_merge_table class Direction(Enum): @@ -70,10 +70,12 @@ class AbstractGraph(ABC): ------- cascade: Abstract method implemented by child classes cascade1: Cascade a restriction up/down the graph, recursively + ft_from_list: Return non-empty FreeTable objects from list of table names Properties ---------- all_ft: Get all FreeTables for visited nodes with restrictions applied. + restr_ft: Get non-empty FreeTables for visited nodes with restrictions. as_dict: Get visited nodes as a list of dictionaries of {table_name: restriction} """ @@ -91,11 +93,16 @@ def __init__(self, seed_table: Table, verbose: bool = False, **kwargs): self.seed_table = seed_table self.connection = seed_table.connection - # Undirected graph may not be needed, but adding FT to the graph - # prevents `to_undirected` from working. If using undirected, remove - # PERIPHERAL_TABLES from the graph. - self.graph = seed_table.connection.dependencies - self.graph.load() + # Deepcopy graph to avoid seed `load()` resetting custom attributes + seed_table.connection.dependencies.load() + graph = seed_table.connection.dependencies + orig_conn = graph._conn # Cannot deepcopy connection + graph._conn = None + self.graph = deepcopy(graph) + graph._conn = orig_conn + + # undirect not needed in all cases but need to do before adding ft nodes + self.undirect_graph = self.graph.to_undirected() self.verbose = verbose self.leaves = set() @@ -111,6 +118,24 @@ def cascade(self): """Cascade restrictions through graph.""" raise NotImplementedError("Child class mut implement `cascade` method") + # --------------------------- Dunder Properties --------------------------- + + def __repr__(self): + l_str = ( + ",\n\t".join(self._camel(self.leaves)) + "\n" + if self.leaves + else "Seed: " + self._camel(self.seed_table) + "\n" + ) + casc_str = "Cascaded" if self.cascaded else "Uncascaded" + return f"{casc_str} {self.__class__.__name__}(\n\t{l_str})" + + def __getitem__(self, index: Union[int, str]): + names = [t.full_table_name for t in self.restr_ft] + return fuzzy_get(index, names, self.restr_ft) + + def __len__(self): + return len(self.restr_ft) + # ---------------------------- Logging Helpers ---------------------------- def _log_truncate(self, log_str: str, max_len: int = 80): @@ -123,34 +148,33 @@ def _log_truncate(self, log_str: str, max_len: int = 80): def _camel(self, table): """Convert table name(s) to camel case.""" - if isinstance(table, KeysView): - table = list(table) - if not isinstance(table, list): - table = [table] - ret = [to_camel_case(t.split(".")[-1].strip("`")) for t in table] - return ret[0] if len(ret) == 1 else ret - - def _print_restr(self): - """Print restrictions for debugging.""" - for table in self.visited: - if restr := self._get_restr(table): - logger.info(f"{table}: {restr}") + table = self._ensure_names(table) + if isinstance(table, str): + return to_camel_case(table.split(".")[-1].strip("`")) + if isinstance(table, Iterable) and not isinstance( + table, (Table, TableMeta) + ): + return [self._camel(t) for t in table] # ------------------------------ Graph Nodes ------------------------------ - def _ensure_name(self, table: Union[str, Table] = None) -> str: + def _ensure_names( + self, table: Union[str, Table] = None + ) -> Union[str, List[str]]: """Ensure table is a string.""" if table is None: return None if isinstance(table, str): return table - if isinstance(table, list): - return [self._ensure_name(t) for t in table] + if isinstance(table, Iterable) and not isinstance( + table, (Table, TableMeta) + ): + return [self._ensure_names(t) for t in table] return getattr(table, "full_table_name", None) def _get_node(self, table: Union[str, Table]): """Get node from graph.""" - table = self._ensure_name(table) + table = self._ensure_names(table) if not (node := self.graph.nodes.get(table)): raise ValueError( f"Table {table} not found in graph." @@ -160,6 +184,7 @@ def _get_node(self, table: Union[str, Table]): def _set_node(self, table, attr: str = "ft", value: Any = None): """Set attribute on node. General helper for various attributes.""" + table = self._ensure_names(table) _ = self._get_node(table) # Ensure node exists self.graph.nodes[table][attr] = value @@ -175,8 +200,8 @@ def _get_edge(self, child: str, parent: str) -> Tuple[bool, Dict[str, str]]: Tuple of boolean indicating direction and edge data. True if child is child of parent. """ - child = self._ensure_name(child) - parent = self._ensure_name(parent) + child = self._ensure_names(child) + parent = self._ensure_names(parent) if edge := self.graph.get_edge_data(parent, child): return False, edge @@ -196,7 +221,7 @@ def _get_edge(self, child: str, parent: str) -> Tuple[bool, Dict[str, str]]: def _get_restr(self, table): """Get restriction from graph node.""" - return self._get_node(self._ensure_name(table)).get("restr") + return self._get_node(self._ensure_names(table)).get("restr") def _set_restr(self, table, restriction, replace=False): """Add restriction to graph node. If one exists, merge with new.""" @@ -207,6 +232,7 @@ def _set_restr(self, table, restriction, replace=False): else restriction ) existing = self._get_restr(table) + if not replace and existing: if restriction == existing: return @@ -219,12 +245,13 @@ def _set_restr(self, table, restriction, replace=False): self._set_node(table, "restr", restriction) - def _get_ft(self, table, with_restr=False): + def _get_ft(self, table, with_restr=False, warn=True): """Get FreeTable from graph node. If one doesn't exist, create it.""" - table = self._ensure_name(table) + table = self._ensure_names(table) if with_restr: if not (restr := self._get_restr(table) or False): - self._log_truncate(f"No restriction for {table}") + if warn: + self._log_truncate(f"No restr for {self._camel(table)}") else: restr = True @@ -234,13 +261,14 @@ def _get_ft(self, table, with_restr=False): return ft & restr - def _and_parts(self, table): - """Return table, its master and parts.""" - ret = [table] - if master := get_master(table): - ret.append(master) - if parts := self._get_ft(table).parts(): - ret.extend(parts) + def _is_out(self, table, warn=True): + """Check if table is outside of spyglass.""" + table = self._ensure_names(table) + if self.graph.nodes.get(table): + return False + ret = table.split(".")[0].split("_")[0].strip("`") not in SHARED_MODULES + if warn and ret: # Log warning if outside + logger.warning(f"Skipping unimported: {table}") return ret # ---------------------------- Graph Traversal ----------------------------- @@ -282,15 +310,19 @@ def _bridge_restr( List[Dict[str, str]] List of dicts containing primary key fields for restricted table2. """ + if self._is_out(table2) or self._is_out(table1): # 2 more likely + return ["False"] # Stop cascade if outside, see #1002 + if not all([direction, attr_map]): dir_bool, edge = self._get_edge(table1, table2) direction = "up" if dir_bool else "down" attr_map = edge.get("attr_map") + # May return empty table if outside imported and outside spyglass ft1 = self._get_ft(table1) & restr ft2 = self._get_ft(table2) - if len(ft1) == 0: + if len(ft1) == 0 or len(ft2) == 0: return ["False"] if bool(set(attr_map.values()) - set(ft1.heading.names)): @@ -333,11 +365,16 @@ def _get_next_tables(self, table: str, direction: Direction) -> Tuple: G = self.graph dir_dict = {"direction": direction} - bonus = {} + bonus = {} # Add master and parts to next tables direction = Direction(direction) if direction == Direction.UP: next_func = G.parents - bonus.update({part: {} for part in self._get_ft(table).parts()}) + table_ft = self._get_ft(table) + for part in table_ft.parts(): # Assumes parts do not alias master + bonus[part] = { + "attr_map": {k: k for k in table_ft.primary_key}, + **dir_dict, + } elif direction == Direction.DOWN: next_func = G.children if (master_name := get_master(table)) != "": @@ -382,9 +419,12 @@ def cascade1( next_tables, next_func = self._get_next_tables(table, direction) - self._log_truncate( - f"Checking {count:>2}: {self._camel(next_tables.keys())}" - ) + if next_list := next_tables.keys(): + self._log_truncate( + f"Checking {count:>2}: {self._camel(table)}" + + f" -> {self._camel(next_list)}" + ) + for next_table, data in next_tables.items(): if next_table.isnumeric(): # Skip alias nodes next_table, data = next_func(next_table).popitem() @@ -422,21 +462,81 @@ def cascade1( # ---------------------------- Graph Properties ---------------------------- + def _topo_sort( + self, nodes: List[str], subgraph: bool = True, reverse: bool = False + ) -> List[str]: + """Return topologically sorted list of nodes. + + Parameters + ---------- + nodes : List[str] + List of table names + subgraph : bool, optional + Whether to use subgraph. Default True + reverse : bool, optional + Whether to reverse the order. Default False. If true, bottom-up. + If None, return nodes as is. + """ + if reverse is None: + return nodes + nodes = [ + node + for node in self._ensure_names(nodes) + if not self._is_out(node, warn=False) + ] + graph = self.graph.subgraph(nodes) if subgraph else self.graph + ordered = unite_master_parts(list(topological_sort(graph))) + if reverse: + ordered.reverse() + return [n for n in ordered if n in nodes] + @property def all_ft(self): """Get restricted FreeTables from all visited nodes. Topological sort logic adopted from datajoint.diagram. """ - self.cascade() + self.cascade(warn=False) nodes = [n for n in self.visited if not n.isnumeric()] - sorted_nodes = unite_master_parts( - list(topological_sort(self.graph.subgraph(nodes))) - ) - all_ft = [ - self._get_ft(table, with_restr=True) for table in sorted_nodes + return [ + self._get_ft(table, with_restr=True, warn=False) + for table in self._topo_sort(nodes, subgraph=True, reverse=False) ] - return [ft for ft in all_ft if len(ft) > 0] + + @property + def restr_ft(self): + """Get non-empty restricted FreeTables from all visited nodes.""" + return [ft for ft in self.all_ft if len(ft) > 0] + + def ft_from_list( + self, + tables: List[str], + with_restr: bool = True, + sort_reverse: bool = None, + return_empty: bool = False, + ) -> List[FreeTable]: + """Return non-empty FreeTable objects from list of table names. + + Parameters + ---------- + tables : List[str] + List of table names + with_restr : bool, optional + Restrict FreeTable to restriction. Default True. + sort_reverse : bool, optional + Sort reverse topologically. Default True. If None, no sort. + """ + + self.cascade(warn=False) + + fts = [ + self._get_ft(table, with_restr=with_restr, warn=False) + for table in self._topo_sort( + tables, subgraph=False, reverse=sort_reverse + ) + ] + + return fts if return_empty else [ft for ft in fts if len(ft) > 0] @property def as_dict(self) -> List[Dict[str, str]]: @@ -453,9 +553,8 @@ class RestrGraph(AbstractGraph): def __init__( self, seed_table: Table, - table_name: str = None, - restriction: str = None, leaves: List[Dict[str, str]] = None, + destinations: List[str] = None, direction: Direction = "up", cascade: bool = False, verbose: bool = False, @@ -473,13 +572,12 @@ def __init__( ---------- seed_table : Table Table to use to establish connection and graph - table_name : str, optional - Table name of single leaf, default None - restriction : str, optional - Restriction to apply to leaf. default None leaves : Dict[str, str], optional List of dictionaries with keys table_name and restriction. One entry per leaf node. Default None. + destinations : List[str], optional + List of endpoints of interest in the graph. Default None. Used to + ignore nodes not in the path(s) to the destination(s). direction : Direction, optional Direction to cascade. Default 'up' cascade : bool, optional @@ -490,27 +588,18 @@ def __init__( """ super().__init__(seed_table, verbose=verbose) - self.add_leaf( - table_name=table_name, restriction=restriction, direction=direction - ) self.add_leaves(leaves) - if cascade: - self.cascade(direction=direction) + dir_list = ["up", "down"] if direction == "both" else [direction] - # --------------------------- Dunder Properties --------------------------- - - def __repr__(self): - l_str = ",\n\t".join(self.leaves) + "\n" if self.leaves else "" - processed = "Cascaded" if self.cascaded else "Uncascaded" - return f"{processed} {self.__class__.__name__}(\n\t{l_str})" - - def __getitem__(self, index: Union[int, str]): - all_ft_names = [t.full_table_name for t in self.all_ft] - return fuzzy_get(index, all_ft_names, self.all_ft) - - def __len__(self): - return len(self.all_ft) + if cascade: + for dir in dir_list: + self._log_truncate(f"Start {dir:<4} : {self.leaves}") + self.cascade(direction=dir) + self.cascaded = False + self.visited -= self.leaves + self.cascaded = True + self.visited |= self.leaves # ---------------------------- Public Properties -------------------------- @@ -558,7 +647,13 @@ def add_leaf( self.cascaded = True def _process_leaves(self, leaves=None, default_restriction=True): - """Process leaves to ensure they are unique and have required keys.""" + """Process leaves to ensure they are unique and have required keys. + + Accepts ... + - [str]: table names, use default_restriction + - [{'table_name': str, 'restriction': str}]: used for export + - [{table_name: restriction}]: userd for distance restriction + """ if not leaves: return [] if not isinstance(leaves, list): @@ -568,10 +663,22 @@ def _process_leaves(self, leaves=None, default_restriction=True): {"table_name": leaf, "restriction": default_restriction} for leaf in leaves ] - if all(isinstance(leaf, dict) for leaf in leaves) and not all( - leaf.get("table_name") for leaf in leaves - ): - raise ValueError(f"All leaves must have table_name: {leaves}") + hashable = True + if all(isinstance(leaf, dict) for leaf in leaves): + new_leaves = [] + for leaf in leaves: + if "table_name" in leaf and "restriction" in leaf: + new_leaves.append(leaf) + continue + for table, restr in leaf.items(): + if not isinstance(restr, (str, dict)): + hashable = False # likely a dj.AndList + new_leaves.append( + {"table_name": table, "restriction": restr} + ) + if not hashable: + return new_leaves + leaves = new_leaves return unique_dicts(leaves) @@ -609,7 +716,7 @@ def add_leaves( # ------------------------------ Graph Traversal -------------------------- - def cascade(self, show_progress=None, direction="up") -> None: + def cascade(self, show_progress=None, direction="up", warn=True) -> None: """Cascade all restrictions up the graph. Parameters @@ -618,6 +725,8 @@ def cascade(self, show_progress=None, direction="up") -> None: Show tqdm progress bar. Default to verbose setting. """ if self.cascaded: + if warn: + self._log_truncate("Already cascaded") return to_visit = self.leaves - self.visited @@ -629,27 +738,16 @@ def cascade(self, show_progress=None, direction="up") -> None: disable=not (show_progress or self.verbose), ): restr = self._get_restr(table) - self._log_truncate(f"Start {table}: {restr}") + self._log_truncate( + f"Start {direction:<4}: {self._camel(table)}, {restr}" + ) self.cascade1(table, restr, direction=direction) - self.cascade_files() - self.cascaded = True + self.cascaded = True # Mark here so next step can use `restr_ft` + self.cascade_files() # Otherwise attempts to re-cascade, recursively # ----------------------------- File Handling ----------------------------- - def _get_files(self, table): - """Get analysis files from graph node.""" - return self._get_node(table).get("files", []) - - def cascade_files(self): - """Set node attribute for analysis files.""" - for table in self.visited: - ft = self._get_ft(table, with_restr=True) - if not set(self.analysis_pk).issubset(ft.heading.names): - continue - files = list(ft.fetch(*self.analysis_pk)) - self._set_node(table, "files", files) - @property def analysis_file_tbl(self) -> Table: """Return the analysis file table. Avoids circular import.""" @@ -657,10 +755,14 @@ def analysis_file_tbl(self) -> Table: return AnalysisNwbfile() - @property - def analysis_pk(self) -> List[str]: - """Return primary key fields from analysis file table.""" - return self.analysis_file_tbl.primary_key + def cascade_files(self): + """Set node attribute for analysis files.""" + analysis_pk = self.analysis_file_tbl.primary_key + for ft in self.restr_ft: + if not set(analysis_pk).issubset(ft.heading.names): + continue + files = list(ft.fetch(*analysis_pk)) + self._set_node(ft, "files", files) @property def file_dict(self) -> Dict[str, List[str]]: @@ -668,8 +770,8 @@ def file_dict(self) -> Dict[str, List[str]]: Included for debugging, to associate files with tables. """ - self.cascade() - return {t: self._get_node(t).get("files", []) for t in self.visited} + self.cascade(warn=False) + return {t: self._get_node(t).get("files", []) for t in self.restr_ft} @property def file_paths(self) -> List[str]: @@ -688,96 +790,16 @@ def file_paths(self) -> List[str]: ] -class TableChains: - """Class for representing chains from parent to Merge table via parts. - - Functions as a plural version of TableChain, allowing a single `cascade` - call across all chains from parent -> Merge table. - - Attributes - ---------- - parent : Table - Parent or origin of chains. - child : Table - Merge table or destination of chains. - connection : datajoint.Connection, optional - Connection to database used to create FreeTable objects. Defaults to - parent.connection. - part_names : List[str] - List of full table names of child parts. - chains : List[TableChain] - List of TableChain objects for each part in child. - has_link : bool - Cached attribute to store whether parent is linked to child via any of - child parts. False if (a) child is not in parent.descendants or (b) - nx.NetworkXNoPath is raised by nx.shortest_path for all chains. - - Methods - ------- - __init__(parent, child, connection=None) - Initialize TableChains with parent and child tables. - __repr__() - Return full representation of chains. - Multiline parent -> child for each chain. - __len__() - Return number of chains with links. - __getitem__(index: Union[int, str]) - Return TableChain object at index, or use substring of table name. - cascade(restriction: str = None) - Return list of cascade for each chain in self.chains. - """ - - def __init__(self, parent, child, direction=Direction.DOWN, verbose=False): - self.parent = parent - self.child = child - self.connection = parent.connection - self.part_names = child.parts() - self.chains = [ - TableChain(parent, part, direction=direction, verbose=verbose) - for part in self.part_names - ] - self.has_link = any([chain.has_link for chain in self.chains]) - - # --------------------------- Dunder Properties --------------------------- - - def __repr__(self): - l_str = ",\n\t".join([str(c) for c in self.chains]) + "\n" - return f"{self.__class__.__name__}(\n\t{l_str})" - - def __len__(self): - return len([c for c in self.chains if c.has_link]) - - def __getitem__(self, index: Union[int, str]): - """Return FreeTable object at index.""" - return fuzzy_get(index, self.part_names, self.chains) - - # ---------------------------- Public Properties -------------------------- - - @property - def max_len(self): - """Return length of longest chain.""" - return max([len(chain) for chain in self.chains]) - - # ------------------------------ Graph Traversal -------------------------- - - def cascade( - self, restriction: str = None, direction: Direction = Direction.DOWN - ): - """Return list of cascades for each chain in self.chains.""" - restriction = restriction or self.parent.restriction or True - cascades = [] - for chain in self.chains: - if joined := chain.cascade(restriction, direction): - cascades.append(joined) - return cascades - - class TableChain(RestrGraph): """Class for representing a chain of tables. A chain is a sequence of tables from parent to child identified by - networkx.shortest_path. Parent -> Merge should use TableChains instead to - handle multiple paths to the respective parts of the Merge table. + networkx.shortest_path from parent to child. To avoid issues with merge + tables, use the Merge table as the child, not the part table. + + Either the parent or child can be omitted if a search_restr is provided. + The missing table will be found by searching for where the restriction + can be applied. Attributes ---------- @@ -789,9 +811,6 @@ class TableChain(RestrGraph): Cached attribute to store whether parent is linked to child. path : List[str] Names of tables along the path from parent to child. - all_ft : List[dj.FreeTable] - List of FreeTable objects for each table in chain with restriction - applied. Methods ------- @@ -804,6 +823,8 @@ class TableChain(RestrGraph): Given a restriction at the beginning, return a restricted FreeTable object at the end of the chain. If direction is 'up', start at the child and move up to the parent. If direction is 'down', start at the parent. + cascade_search() + Search from the leaf node to find where a restriction can be applied. """ def __init__( @@ -814,27 +835,21 @@ def __init__( search_restr: str = None, cascade: bool = False, verbose: bool = False, - allow_merge: bool = False, banned_tables: List[str] = None, **kwargs, ): - if not allow_merge and child is not None and is_merge_table(child): - raise TypeError("Child is a merge table. Use TableChains instead.") - - self.parent = self._ensure_name(parent) - self.child = self._ensure_name(child) + self.parent = self._ensure_names(parent) + self.child = self._ensure_names(child) if not self.parent and not self.child: raise ValueError("Parent or child table required.") - if not search_restr and not (self.parent and self.child): - raise ValueError("Search restriction required to find path.") seed_table = parent if isinstance(parent, Table) else child super().__init__(seed_table=seed_table, verbose=verbose) - self.no_visit.update(PERIPHERAL_TABLES) - self.no_visit.update(self._ensure_name(banned_tables) or []) - self.no_visit.difference_update([self.parent, self.child]) + self._ignore_peripheral(except_tables=[self.parent, self.child]) + self.no_visit.update(self._ensure_names(banned_tables) or []) + self.no_visit.difference_update(set([self.parent, self.child])) self.searched_tables = set() self.found_restr = False self.link_type = None @@ -843,6 +858,8 @@ def __init__( self.search_restr = search_restr self.direction = Direction(direction) + if self.parent and self.child and not self.direction: + self.direction = Direction.DOWN self.leaf = None if search_restr and not parent: @@ -856,10 +873,20 @@ def __init__( self.add_leaf(self.leaf, True, cascade=False, direction=direction) if cascade and search_restr: - self.cascade_search() - self.cascade(restriction=search_restr) + self.cascade_search() # only cascade if found or not looking + if (search_restr and self.found_restr) or not search_restr: + self.cascade(restriction=search_restr) self.cascaded = True + # ------------------------------ Ignore Nodes ------------------------------ + + def _ignore_peripheral(self, except_tables: List[str] = None): + """Ignore peripheral tables in graph traversal.""" + except_tables = self._ensure_names(except_tables) + ignore_tables = set(PERIPHERAL_TABLES) - set(except_tables or []) + self.no_visit.update(ignore_tables) + self.undirect_graph.remove_nodes_from(ignore_tables) + # --------------------------- Dunder Properties --------------------------- def __str__(self): @@ -884,9 +911,6 @@ def __len__(self): return 0 return len(self.path) - def __getitem__(self, index: Union[int, str]): - return fuzzy_get(index, self.path, self.all_ft) - # ---------------------------- Public Properties -------------------------- @property @@ -900,26 +924,18 @@ def has_link(self) -> bool: _ = self.path return self.link_type is not None - @cached_property - def all_ft(self) -> List[dj.FreeTable]: - """Return list of FreeTable objects for each table in chain. - - Unused. Preserved for future debugging. - """ - if not self.has_link: - return None - return [ - self._get_ft(table, with_restr=False) - for table in self.path - if not table.isnumeric() - ] - @property def path_str(self) -> str: if not self.path: return "No link" return self._link_symbol.join([self._camel(t) for t in self.path]) + @property + def path_ft(self) -> List[FreeTable]: + """Return FreeTables along the path.""" + path_with_ends = set([self.parent, self.child]) | set(self.path) + return self.ft_from_list(path_with_ends, with_restr=True) + # ------------------------------ Graph Nodes ------------------------------ def _set_find_restr(self, table_name, restriction): @@ -962,6 +978,7 @@ def cascade_search(self) -> None: replace=True, ) if not self.found_restr: + self.link_type = None searched = ( "parents" if self.direction == Direction.UP else "children" ) @@ -975,7 +992,14 @@ def _set_found_vars(self, table): """Set found_restr and searched_tables.""" self._set_restr(table, self.search_restr, replace=True) self.found_restr = True - self.searched_tables.update(set(self._and_parts(table))) + + and_parts = set([table]) + if master := get_master(table): + and_parts.add(master) + if parts := self._get_ft(table).parts(): + and_parts.update(parts) + + self.searched_tables.update(and_parts) if self.direction == Direction.UP: self.parent = table @@ -1055,12 +1079,7 @@ def find_path(self, directed=True) -> List[str]: List of names in the path. """ source, target = self.parent, self.child - search_graph = self.graph - - if not directed: - self.connection.dependencies.load() - self.undirect_graph = self.connection.dependencies.to_undirected() - search_graph = self.undirect_graph + search_graph = self.graph if directed else self.undirect_graph search_graph.remove_nodes_from(self.no_visit) @@ -1077,7 +1096,6 @@ def find_path(self, directed=True) -> List[str]: ignore_nodes = self.graph.nodes - set(path) self.no_visit.update(ignore_nodes) - self._log_truncate(f"Ignore : {ignore_nodes}") return path @cached_property @@ -1095,7 +1113,9 @@ def path(self) -> list: return path - def cascade(self, restriction: str = None, direction: Direction = None): + def cascade( + self, restriction: str = None, direction: Direction = None, **kwargs + ): if not self.has_link: return @@ -1111,11 +1131,20 @@ def cascade(self, restriction: str = None, direction: Direction = None): self.cascade1( table=start, - restriction=restriction or self._get_restr(start), + restriction=restriction or self._get_restr(start) or True, direction=direction, replace=True, ) + # Cascade will stop if any restriction is empty, so set rest to None + # This would cause issues if we want a table partway through the chain + # but that's not a typical use case, were the start and end are desired + non_numeric = [t for t in self.path if not t.isnumeric()] + if any(self._get_restr(t) is None for t in non_numeric): + for table in non_numeric: + if table is not start: + self._set_restr(table, False, replace=True) + return self._get_ft(end, with_restr=True) def restrict_by(self, *args, **kwargs) -> None: diff --git a/src/spyglass/utils/dj_helper_fn.py b/src/spyglass/utils/dj_helper_fn.py index f42c9858a..da7d30a3b 100644 --- a/src/spyglass/utils/dj_helper_fn.py +++ b/src/spyglass/utils/dj_helper_fn.py @@ -265,7 +265,6 @@ def fetch_nwb(query_expression, nwb_master, *attrs, **kwargs): rec_dict["nwb2load_filepath"] = file_path continue - # Pulled from future cbroz1/ndo # Full dict caused issues with dlc tables using dicts in secondary keys rec_only_pk = {k: rec_dict[k] for k in query_table.heading.primary_key} rec_dict["nwb2load_filepath"] = (query_table & rec_only_pk).fetch1( @@ -352,7 +351,7 @@ def update_analysis_for_dandi_standard( species_value = file["/general/subject/species"][()].decode("utf-8") if species_value == "Rat": new_species_value = "Rattus norvegicus" - print( + logger.info( f"Adjusting subject species from '{species_value}' to " + f"'{new_species_value}'." ) @@ -363,10 +362,10 @@ def update_analysis_for_dandi_standard( ): raise ValueError( "Dandi upload requires species either be in Latin binomial form" - + " (e.g., 'Mus musculus' and 'Homo sapiens')" - + "or be a NCBI taxonomy link (e.g., " - + "'http://purl.obolibrary.org/obo/NCBITaxon_280675')." - + f"\n Please update species value of: {species_value}" + + " (e.g., 'Mus musculus' and 'Homo sapiens') or be a NCBI " + + "taxonomy link (e.g., " + + "'http://purl.obolibrary.org/obo/NCBITaxon_280675').\n " + + f"Please update species value of: {species_value}" ) # add subject age dataset "P4M/P8M" @@ -385,7 +384,8 @@ def update_analysis_for_dandi_standard( if experimenter_value != new_experimenter_value: new_experimenter_value = new_experimenter_value.astype(STR_DTYPE) logger.info( - f"Adjusting experimenter from {experimenter_value} to {new_experimenter_value}." + f"Adjusting experimenter from {experimenter_value} to " + + f"{new_experimenter_value}." ) file["/general/experimenter"][:] = new_experimenter_value diff --git a/src/spyglass/utils/dj_merge_tables.py b/src/spyglass/utils/dj_merge_tables.py index d1176de30..474ff91c8 100644 --- a/src/spyglass/utils/dj_merge_tables.py +++ b/src/spyglass/utils/dj_merge_tables.py @@ -820,9 +820,8 @@ def delete_downstream_merge( ) -> list: """Given a table/restriction, id or delete relevant downstream merge entries - Passthrough to SpyglassMixin.delete_downstream_merge + Passthrough to SpyglassMixin.delete_downstream_parts """ - from spyglass.common.common_usage import ActivityLog from spyglass.utils.dj_mixin import SpyglassMixin @@ -834,4 +833,4 @@ def delete_downstream_merge( raise ValueError("Input must be a Spyglass Table.") table = table if isinstance(table, dj.Table) else table() - return table.delete_downstream_merge(**kwargs) + return table.delete_downstream_parts(**kwargs) diff --git a/src/spyglass/utils/dj_mixin.py b/src/spyglass/utils/dj_mixin.py index 4cdbbbaa0..1db44078a 100644 --- a/src/spyglass/utils/dj_mixin.py +++ b/src/spyglass/utils/dj_mixin.py @@ -1,6 +1,5 @@ from atexit import register as exit_register from atexit import unregister as exit_unregister -from collections import OrderedDict from contextlib import nullcontext from functools import cached_property from inspect import stack as inspect_stack @@ -25,7 +24,6 @@ get_nwb_table, populate_pass_function, ) -from spyglass.utils.dj_merge_tables import RESERVED_PRIMARY_KEY as MERGE_PK from spyglass.utils.dj_merge_tables import Merge, is_merge_table from spyglass.utils.logging import logger @@ -56,8 +54,8 @@ class SpyglassMixin: `restriction` can be set to a string to restrict the delete. `dry_run` can be set to False to commit the delete. `reload_cache` can be set to True to reload the merge cache. - ddm(*args, **kwargs) - Alias for delete_downstream_merge. + ddp(*args, **kwargs) + Alias for delete_downstream_parts cautious_delete(force_permission=False, *args, **kwargs) Check user permissions before deleting table rows. Permission is granted to users listed as admin in LabMember table or to users on a team with @@ -117,14 +115,14 @@ def _auto_increment(self, key, pk, *args, **kwargs): def file_like(self, name=None, **kwargs): """Convenience method for wildcard search on file name fields.""" if not name: - return self & True + return self attr = None for field in self.heading.names: if "file" in field: attr = field break if not attr: - logger.error(f"No file-like field found in {self.full_table_name}") + logger.error(f"No file_like field found in {self.full_table_name}") return return self & f"{attr} LIKE '%{name}%'" @@ -257,135 +255,145 @@ def fetch_pynapple(self, *attrs, **kwargs): for file_name in nwb_files ] - # ------------------------ delete_downstream_merge ------------------------ + # ------------------------ delete_downstream_parts ------------------------ - def _import_merge_tables(self): - """Import all merge tables downstream of self.""" + def _import_part_masters(self): + """Import tables that may constrain a RestrGraph. See #1002""" + from spyglass.common.common_ripple import ( + RippleLFPSelection, + ) # noqa F401 from spyglass.decoding.decoding_merge import DecodingOutput # noqa F401 + from spyglass.decoding.v0.clusterless import ( # noqa F401 + UnitMarksIndicatorSelection, + ) + from spyglass.decoding.v0.sorted_spikes import ( # noqa F401 + SortedSpikesIndicatorSelection, + ) + from spyglass.decoding.v1.core import PositionGroup # noqa F401 + from spyglass.lfp.analysis.v1 import LFPBandSelection # noqa F401 from spyglass.lfp.lfp_merge import LFPOutput # noqa F401 - from spyglass.linearization.merge import ( + from spyglass.linearization.merge import ( # noqa F401 LinearizedPositionOutput, - ) # noqa F401 + LinearizedPositionV1, + ) + from spyglass.mua.v1.mua import MuaEventsV1 # noqa F401 from spyglass.position.position_merge import PositionOutput # noqa F401 + from spyglass.ripple.v1.ripple import RippleTimesV1 # noqa F401 + from spyglass.spikesorting.analysis.v1.group import ( # noqa F401 + SortedSpikesGroup, + ) from spyglass.spikesorting.spikesorting_merge import ( # noqa F401 SpikeSortingOutput, ) + from spyglass.spikesorting.v0.figurl_views import ( # noqa F401 + SpikeSortingRecordingView, + ) _ = ( DecodingOutput(), + LFPBandSelection(), LFPOutput(), LinearizedPositionOutput(), + LinearizedPositionV1(), + MuaEventsV1(), + PositionGroup(), PositionOutput(), + RippleLFPSelection(), + RippleTimesV1(), + SortedSpikesGroup(), + SortedSpikesIndicatorSelection(), SpikeSortingOutput(), + SpikeSortingRecordingView(), + UnitMarksIndicatorSelection(), ) @cached_property - def _merge_tables(self) -> Dict[str, dj.FreeTable]: - """Dict of merge tables downstream of self: {full_table_name: FreeTable}. + def _part_masters(self) -> set: + """Set of master tables downstream of self. - Cache of items in parents of self.descendants(as_objects=True). Both - descendant and parent must have the reserved primary key 'merge_id'. + Cache of masters in self.descendants(as_objects=True) with another + foreign key reference in the part. Used for delete_downstream_parts. """ self.connection.dependencies.load() - merge_tables = {} - visited = set() + part_masters = set() def search_descendants(parent): for desc in parent.descendants(as_objects=True): - if ( - MERGE_PK not in desc.heading.names - or not (master_name := get_master(desc.full_table_name)) - or master_name in merge_tables + if ( # Check if has master, is part + not (master := get_master(desc.full_table_name)) + # has other non-master parent + or not set(desc.parents()) - set([master]) + or master in part_masters # already in cache ): continue - master_ft = dj.FreeTable(self.connection, master_name) - if is_merge_table(master_ft): - merge_tables[master_name] = master_ft - if master_name not in visited: - visited.add(master_name) - search_descendants(master_ft) + if master not in part_masters: + part_masters.add(master) + search_descendants(dj.FreeTable(self.connection, master)) try: _ = search_descendants(self) except NetworkXError: try: # Attempt to import missing table - self._import_merge_tables() + self._import_part_masters() _ = search_descendants(self) except NetworkXError as e: table_name = "".join(e.args[0].split("`")[1:4]) raise ValueError(f"Please import {table_name} and try again.") logger.info( - f"Building merge cache for {self.camel_name}.\n\t" - + f"Found {len(merge_tables)} downstream merge tables" + f"Building part-parent cache for {self.camel_name}.\n\t" + + f"Found {len(part_masters)} downstream part tables" ) - return merge_tables - - @cached_property - def _merge_chains(self) -> OrderedDict[str, List[dj.FreeTable]]: - """Dict of chains to merges downstream of self + return part_masters - Format: {full_table_name: TableChains}. - - For each merge table found in _merge_tables, find the path from self to - merge via merge parts. If the path is valid, add it to the dict. Cache - prevents need to recompute whenever delete_downstream_merge is called - with a new restriction. To recompute, add `reload_cache=True` to - delete_downstream_merge call. + def _commit_downstream_delete(self, down_fts, start=None, **kwargs): """ - from spyglass.utils.dj_graph import TableChains # noqa F401 - - merge_chains = {} - for name, merge_table in self._merge_tables.items(): - chains = TableChains(self, merge_table) - if len(chains): - merge_chains[name] = chains - - # This is ordered by max_len of chain from self to merge, which assumes - # that the merge table with the longest chain is the most downstream. - # A more sophisticated approach would order by length from self to - # each merge part independently, but this is a good first approximation. - - return OrderedDict( - sorted( - merge_chains.items(), key=lambda x: x[1].max_len, reverse=True - ) - ) + Commit delete of downstream parts via down_fts. Logs with _log_delete. - def _get_chain(self, substring): - """Return chain from self to merge table with substring in name.""" - for name, chain in self._merge_chains.items(): - if substring.lower() in name: - return chain - raise ValueError(f"No chain found with '{substring}' in name.") + Used by both delete_downstream_parts and cautious_delete. + """ + start = start or time() - def _commit_merge_deletes( - self, merge_join_dict: Dict[str, List[QueryExpression]], **kwargs - ) -> None: - """Commit merge deletes. + safemode = ( + dj.config.get("safemode", True) + if kwargs.get("safemode") is None + else kwargs["safemode"] + ) + _ = kwargs.pop("safemode", None) + + ran_deletes = True + if down_fts: + for down_ft in down_fts: + dj_logger.info( + f"Spyglass: Deleting {len(down_ft)} rows from " + + f"{down_ft.full_table_name}" + ) + if ( + self._test_mode + or not safemode + or user_choice("Commit deletes?", default="no") == "yes" + ): + for down_ft in down_fts: # safemode off b/c already checked + down_ft.delete(safemode=False, **kwargs) + else: + logger.info("Delete aborted.") + ran_deletes = False - Parameters - ---------- - merge_join_dict : Dict[str, List[QueryExpression]] - Dictionary of merge tables and their joins. Uses 'merge_id' primary - key to restrict delete. + self._log_delete(start, del_blob=down_fts if ran_deletes else None) - Extracted for use in cautious_delete and delete_downstream_merge.""" - for table_name, part_restr in merge_join_dict.items(): - table = self._merge_tables[table_name] - keys = [part.fetch(MERGE_PK, as_dict=True) for part in part_restr] - (table & keys).delete(**kwargs) + return ran_deletes - def delete_downstream_merge( + def delete_downstream_parts( self, restriction: str = None, dry_run: bool = True, reload_cache: bool = False, disable_warning: bool = False, - return_parts: bool = True, + return_graph: bool = False, + verbose: bool = False, **kwargs, - ) -> Union[List[QueryExpression], Dict[str, List[QueryExpression]]]: + ) -> List[dj.FreeTable]: """Delete downstream merge table entries associated with restriction. Requires caching of merge tables and links, which is slow on first call. @@ -402,72 +410,83 @@ def delete_downstream_merge( If True, reload merge cache. Default False. disable_warning : bool, optional If True, do not warn if no merge tables found. Default False. - return_parts : bool, optional - If True, return list of merge part entries to be deleted. Default + return_graph: bool, optional + If True, return RestrGraph object used to identify downstream + tables. Default False, return list of part FreeTables. True. If False, return dictionary of merge tables and their joins. + verbose : bool, optional + If True, call RestrGraph with verbose=True. Default False. **kwargs : Any Passed to datajoint.table.Table.delete. """ + from spyglass.utils.dj_graph import RestrGraph # noqa F401 + + start = time() + if reload_cache: - for attr in ["_merge_tables", "_merge_chains"]: - _ = self.__dict__.pop(attr, None) + _ = self.__dict__.pop("_part_masters", None) + _ = self._part_masters # load cache before loading graph restriction = restriction or self.restriction or True - merge_join_dict = {} - for name, chain in self._merge_chains.items(): - if join := chain.cascade(restriction, direction="down"): - merge_join_dict[name] = join + restr_graph = RestrGraph( + seed_table=self, + leaves={self.full_table_name: restriction}, + direction="down", + cascade=True, + verbose=verbose, + ) + + if return_graph: + return restr_graph + + down_fts = restr_graph.ft_from_list( + self._part_masters, sort_reverse=False + ) - if not merge_join_dict and not disable_warning: + if not down_fts and not disable_warning: logger.warning( - f"No merge deletes found w/ {self.camel_name} & " + f"No part deletes found w/ {self.camel_name} & " + f"{restriction}.\n\tIf this is unexpected, try importing " + " Merge table(s) and running with `reload_cache`." ) if dry_run: - return merge_join_dict.values() if return_parts else merge_join_dict + return down_fts - self._commit_merge_deletes(merge_join_dict, **kwargs) + self._commit_downstream_delete(down_fts, start, **kwargs) - def ddm( - self, - restriction: str = None, - dry_run: bool = True, - reload_cache: bool = False, - disable_warning: bool = False, - return_parts: bool = True, - *args, - **kwargs, + def ddp( + self, *args, **kwargs ) -> Union[List[QueryExpression], Dict[str, List[QueryExpression]]]: - """Alias for delete_downstream_merge.""" - return self.delete_downstream_merge( - restriction=restriction, - dry_run=dry_run, - reload_cache=reload_cache, - disable_warning=disable_warning, - return_parts=return_parts, - *args, - **kwargs, - ) + """Alias for delete_downstream_parts.""" + return self.delete_downstream_parts(*args, **kwargs) # ---------------------------- cautious_delete ---------------------------- @cached_property def _delete_deps(self) -> List[Table]: - """List of tables required for delete permission check. + """List of tables required for delete permission and orphan checks. LabMember, LabTeam, and Session are required for delete permission. + common_nwbfile.schema.external is required for deleting orphaned + external files. IntervalList is required for deleting orphaned interval + lists. Used to delay import of tables until needed, avoiding circular imports. Each of these tables inheits SpyglassMixin. """ - from spyglass.common import LabMember, LabTeam, Session # noqa F401 + from spyglass.common import ( # noqa F401 + IntervalList, + LabMember, + LabTeam, + Session, + ) + from spyglass.common.common_nwbfile import schema # noqa F401 self._session_pk = Session.primary_key[0] self._member_pk = LabMember.primary_key[0] - return [LabMember, LabTeam, Session] + return [LabMember, LabTeam, Session, schema.external, IntervalList] def _get_exp_summary(self): """Get summary of experimenters for session(s), including NULL. @@ -483,7 +502,7 @@ def _get_exp_summary(self): Summary of experimenters for session(s). """ - Session = self._delete_deps[-1] + Session = self._delete_deps[2] SesExp = Session.Experimenter # Not called in delete permission check, only bare _get_exp_summary @@ -506,7 +525,7 @@ def _session_connection(self): """Path from Session table to self. False if no connection found.""" from spyglass.utils.dj_graph import TableChain # noqa F401 - connection = TableChain(parent=self._delete_deps[-1], child=self) + connection = TableChain(parent=self._delete_deps[2], child=self) return connection if connection.has_link else False @cached_property @@ -532,7 +551,7 @@ def _check_delete_permission(self) -> None: Permission denied because (a) Session has no experimenter, or (b) user is not on a team with Session experimenter(s). """ - LabMember, LabTeam, Session = self._delete_deps + LabMember, LabTeam, Session, _, _ = self._delete_deps dj_user = dj.config["database.user"] if dj_user in LabMember().admin: # bypass permission check for admin @@ -575,16 +594,14 @@ def _check_delete_permission(self) -> None: logger.info(f"Queueing delete for session(s):\n{sess_summary}") @cached_property - def _usage_table(self): + def _cautious_del_tbl(self): """Temporary inclusion for usage tracking.""" from spyglass.common.common_usage import CautiousDelete return CautiousDelete() - def _log_delete(self, start, merge_deletes=None, super_delete=False): + def _log_delete(self, start, del_blob=None, super_delete=False): """Log use of cautious_delete.""" - if isinstance(merge_deletes, QueryExpression): - merge_deletes = merge_deletes.fetch(as_dict=True) safe_insert = dict( duration=time() - start, dj_user=dj.config["database.user"], @@ -593,21 +610,23 @@ def _log_delete(self, start, merge_deletes=None, super_delete=False): restr_str = "Super delete: " if super_delete else "" restr_str += "".join(self.restriction) if self.restriction else "None" try: - self._usage_table.insert1( + self._cautious_del_tbl.insert1( dict( **safe_insert, restriction=restr_str[:255], - merge_deletes=merge_deletes, + merge_deletes=del_blob, ) ) except (DataJointError, DataError): - self._usage_table.insert1( + self._cautious_del_tbl.insert1( dict(**safe_insert, restriction="Unknown") ) # TODO: Intercept datajoint delete confirmation prompt for merge deletes - def cautious_delete(self, force_permission: bool = False, *args, **kwargs): - """Delete table rows after checking user permission. + def cautious_delete( + self, force_permission: bool = False, dry_run=False, *args, **kwargs + ): + """Permission check, then delete potential orphans and table rows. Permission is granted to users listed as admin in LabMember table or to users on a team with with the Session experimenter(s). If the table @@ -615,56 +634,61 @@ def cautious_delete(self, force_permission: bool = False, *args, **kwargs): continues. If the Session has no experimenter, or if the user is not on a team with the Session experimenter(s), a PermissionError is raised. + Potential downstream orphans are deleted first. These are master tables + whose parts have foreign keys to descendants of self. Then, rows from + self are deleted. Last, Nwbfile and IntervalList externals are deleted. + Parameters ---------- force_permission : bool, optional Bypass permission check. Default False. + dry_run : bool, optional + Default False. If True, return items to be deleted as + Tuple[Upstream, Downstream, externals['raw'], externals['analysis']] + If False, delete items. *args, **kwargs : Any Passed to datajoint.table.Table.delete. """ start = time() + external, IntervalList = self._delete_deps[3], self._delete_deps[4] - if not force_permission: + if not force_permission or dry_run: self._check_delete_permission() - merge_deletes = self.delete_downstream_merge( + down_fts = self.delete_downstream_parts( dry_run=True, disable_warning=True, - return_parts=False, ) - safemode = ( - dj.config.get("safemode", True) - if kwargs.get("safemode") is None - else kwargs["safemode"] - ) + if dry_run: + return ( + down_fts, + IntervalList(), # cleanup func relies on downstream deletes + external["raw"].unused(), + external["analysis"].unused(), + ) - if merge_deletes: - for table, content in merge_deletes.items(): - count = sum([len(part) for part in content]) - dj_logger.info(f"Merge: Deleting {count} rows from {table}") - if ( - not self._test_mode - or not safemode - or user_choice("Commit deletes?", default="no") == "yes" - ): - self._commit_merge_deletes(merge_deletes, **kwargs) - else: - logger.info("Delete aborted.") - self._log_delete(start) - return + if not self._commit_downstream_delete(down_fts, start=start, **kwargs): + return # Abort delete based on user input + + super().delete(*args, **kwargs) # Confirmation here - super().delete(*args, **kwargs) # Additional confirm here + for ext_type in ["raw", "analysis"]: + external[ext_type].delete( + delete_external_files=True, display_progress=False + ) - self._log_delete(start=start, merge_deletes=merge_deletes) + _ = IntervalList().nightly_cleanup(dry_run=False) - def cdel(self, force_permission=False, *args, **kwargs): + self._log_delete(start=start, del_blob=down_fts) + + def cdel(self, *args, **kwargs): """Alias for cautious_delete.""" - self.cautious_delete(force_permission=force_permission, *args, **kwargs) + return self.cautious_delete(*args, **kwargs) - def delete(self, force_permission=False, *args, **kwargs): + def delete(self, *args, **kwargs): """Alias for cautious_delete, overwrites datajoint.table.Table.delete""" - self.cautious_delete(force_permission=force_permission, *args, **kwargs) + self.cautious_delete(*args, **kwargs) def super_delete(self, warn=True, *args, **kwargs): """Alias for datajoint.table.Table.delete.""" @@ -708,7 +732,7 @@ def populate(self, *restrictions, **kwargs): @cached_property def _spyglass_version(self): - """Get Spyglass version from dj.config.""" + """Get Spyglass version.""" from spyglass import __version__ as sg_version return ".".join(sg_version.split(".")[:3]) # Major.Minor.Patch @@ -905,8 +929,8 @@ def restrict_by( Returns ------- - Union[QueryExpression, FindKeyGraph] - Restricted version of present table or FindKeyGraph object. If + Union[QueryExpression, TableChain] + Restricted version of present table or TableChain object. If return_graph, use all_ft attribute to see all tables in cascade. """ from spyglass.utils.dj_graph import TableChain # noqa: F401 @@ -917,6 +941,8 @@ def restrict_by( try: ret = self.restrict(restriction) # Save time trying first if len(ret) < len(self): + # If it actually restricts, if not it might by a dict that + # is not a valid restriction, returned as True logger.warning("Restriction valid for this table. Using as is.") return ret except DataJointError: @@ -936,21 +962,26 @@ def restrict_by( direction=direction, search_restr=restriction, banned_tables=list(self._banned_search_tables), - allow_merge=True, cascade=True, verbose=verbose, **kwargs, ) + if not graph.found_restr: + return None + if return_graph: return graph ret = self & graph._get_restr(self.full_table_name) - if len(ret) == len(self) or len(ret) == 0: - logger.warning( - f"Failed to restrict with path: {graph.path_str}\n\t" - + "See `help(YourTable.restrict_by)`" - ) + warn_text = ( + f" after restrict with path: {graph.path_str}\n\t " + + "See `help(YourTable.restrict_by)`" + ) + if len(ret) == len(self): + logger.warning("Same length" + warn_text) + elif len(ret) == 0: + logger.warning("No entries" + warn_text) return ret diff --git a/tests/common/test_usage.py b/tests/common/test_usage.py new file mode 100644 index 000000000..71449b3e3 --- /dev/null +++ b/tests/common/test_usage.py @@ -0,0 +1,89 @@ +import pytest + + +@pytest.fixture(scope="session") +def export_tbls(common): + from spyglass.common.common_usage import Export, ExportSelection + + return ExportSelection(), Export() + + +@pytest.fixture(scope="session") +def gen_export_selection( + lfp, trodes_pos_v1, track_graph, export_tbls, populate_lfp +): + ExportSelection, _ = export_tbls + _ = populate_lfp + + ExportSelection.start_export(paper_id=1, analysis_id=1) + lfp.v1.LFPV1().fetch_nwb() + trodes_pos_v1.fetch() + ExportSelection.start_export(paper_id=1, analysis_id=2) + track_graph.fetch() + ExportSelection.stop_export() + + yield dict(paper_id=1) + + ExportSelection.stop_export() + ExportSelection.super_delete(warn=False, safemode=False) + + +def test_export_selection_files(gen_export_selection, export_tbls): + ExportSelection, _ = export_tbls + paper_key = gen_export_selection + + len_fi = len(ExportSelection * ExportSelection.File & paper_key) + assert len_fi == 1, "Selection files not captured correctly" + + +def test_export_selection_tables(gen_export_selection, export_tbls): + ExportSelection, _ = export_tbls + paper_key = gen_export_selection + + paper = ExportSelection * ExportSelection.Table & paper_key + len_tbl_1 = len(paper & dict(analysis_id=1)) + len_tbl_2 = len(paper & dict(analysis_id=2)) + assert len_tbl_1 == 2, "Selection tables not captured correctly" + assert len_tbl_2 == 1, "Selection tables not captured correctly" + + +def tests_export_selection_max_id(gen_export_selection, export_tbls): + ExportSelection, _ = export_tbls + _ = gen_export_selection + + exp_id = max(ExportSelection.fetch("export_id")) + got_id = ExportSelection._max_export_id(1) + assert exp_id == got_id, "Max export id not captured correctly" + + +@pytest.fixture(scope="session") +def populate_export(export_tbls, gen_export_selection): + _, Export = export_tbls + Export.populate_paper(**gen_export_selection) + key = (Export & gen_export_selection).fetch("export_id", as_dict=True) + + yield (Export.Table & key), (Export.File & key) + + Export.super_delete(warn=False, safemode=False) + + +def test_export_populate(populate_export): + table, file = populate_export + + assert len(file) == 4, "Export tables not captured correctly" + assert len(table) == 31, "Export files not captured correctly" + + +def test_invalid_export_id(export_tbls): + ExportSelection, _ = export_tbls + ExportSelection.start_export(paper_id=2, analysis_id=1) + with pytest.raises(RuntimeError): + ExportSelection.export_id = 99 + ExportSelection.stop_export() + + +def test_del_export_id(export_tbls): + ExportSelection, _ = export_tbls + ExportSelection.start_export(paper_id=2, analysis_id=1) + del ExportSelection.export_id + assert ExportSelection.export_id == 0, "Export id not reset correctly" diff --git a/tests/utils/conftest.py b/tests/utils/conftest.py index a4bc7f900..726b6b8a7 100644 --- a/tests/utils/conftest.py +++ b/tests/utils/conftest.py @@ -30,23 +30,14 @@ def schema_test(teardown, dj_conn): @pytest.fixture(scope="module") -def chains(Nwbfile): - """Return example TableChains object from Nwbfile.""" - from spyglass.lfp.lfp_merge import LFPOutput # noqa: F401 +def chain(Nwbfile): + """Return example TableChain object from chains.""" from spyglass.linearization.merge import ( LinearizedPositionOutput, ) # noqa: F401 - from spyglass.position.position_merge import PositionOutput # noqa: F401 - - _ = LFPOutput, LinearizedPositionOutput, PositionOutput - - yield Nwbfile._get_chain("linear") - + from spyglass.utils.dj_graph import TableChain -@pytest.fixture(scope="module") -def chain(chains): - """Return example TableChain object from chains.""" - yield chains[0] + yield TableChain(Nwbfile, LinearizedPositionOutput) @pytest.fixture(scope="module") diff --git a/tests/utils/test_chains.py b/tests/utils/test_chains.py index 66d9772c3..093ed5485 100644 --- a/tests/utils/test_chains.py +++ b/tests/utils/test_chains.py @@ -13,27 +13,6 @@ def full_to_camel(t): return to_camel_case(t.split(".")[-1].strip("`")) -def test_chains_repr(chains): - """Test that the repr of a TableChains object is as expected.""" - repr_got = repr(chains) - chain_st = ",\n\t".join([str(c) for c in chains.chains]) + "\n" - repr_exp = f"TableChains(\n\t{chain_st})" - assert repr_got == repr_exp, "Unexpected repr of TableChains object." - - -def test_str_getitem(chains): - """Test getitem of TableChains object.""" - by_int = chains[0] - by_str = chains[chains.part_names[0]] - assert by_int == by_str, "Getitem by int and str not equal." - - -def test_invalid_chain(Nwbfile, pos_merge_tables, TableChain): - """Test that an invalid chain raises an error.""" - with pytest.raises(TypeError): - TableChain(Nwbfile, pos_merge_tables[0]) - - def test_chain_str(chain): """Test that the str of a TableChain object is as expected.""" chain = chain @@ -64,8 +43,8 @@ def test_chain_len(chain): def test_chain_getitem(chain): """Test getitem of TableChain object.""" - by_int = chain[0] - by_str = chain[chain.path[0]] + by_int = str(chain[0]) + by_str = str(chain[chain.restr_ft[0].full_table_name]) assert by_int == by_str, "Getitem by int and str not equal." @@ -76,3 +55,9 @@ def test_nolink_join(no_link_chain): def test_chain_str_no_link(no_link_chain): """Test that the str of a TableChain object with no link is as expected.""" assert str(no_link_chain) == "No link", "Unexpected str of no link chain." + assert repr(no_link_chain) == "No link", "Unexpected repr of no link chain." + + +def test_invalid_chain(TableChain): + with pytest.raises(ValueError): + TableChain() diff --git a/tests/utils/test_graph.py b/tests/utils/test_graph.py index 18899e147..c51427810 100644 --- a/tests/utils/test_graph.py +++ b/tests/utils/test_graph.py @@ -1,4 +1,7 @@ import pytest +from datajoint.utils import to_camel_case + +from tests.conftest import VERBOSE @pytest.fixture(scope="session") @@ -14,8 +17,7 @@ def restr_graph(leaf, verbose, lin_merge_key): yield RestrGraph( seed_table=leaf, - table_name=leaf.full_table_name, - restriction=True, + leaves={leaf.full_table_name: True}, cascade=True, verbose=verbose, ) @@ -26,13 +28,19 @@ def test_rg_repr(restr_graph, leaf): repr_got = repr(restr_graph) assert "cascade" in repr_got.lower(), "Cascade not in repr." - assert leaf.full_table_name in repr_got, "Table name not in repr." + + assert to_camel_case(leaf.table_name) in repr_got, "Table name not in repr." + + +def test_rg_len(restr_graph): + assert len(restr_graph) == len( + restr_graph.restr_ft + ), "Unexpected length of RestrGraph." def test_rg_ft(restr_graph): """Test FreeTable attribute of RestrGraph.""" assert len(restr_graph.leaf_ft) == 1, "Unexpected # of leaf tables." - assert len(restr_graph["spatial"]) == 2, "Unexpected cascaded table length." def test_rg_restr_ft(restr_graph): @@ -43,8 +51,41 @@ def test_rg_restr_ft(restr_graph): def test_rg_file_paths(restr_graph): """Test collection of upstream file paths.""" - paths = [p.get("file_path") for p in restr_graph.file_paths] - assert len(paths) == 2, "Unexpected number of file paths." + assert len(restr_graph.file_paths) == 2, "Unexpected number of file paths." + + +def test_rg_invalid_table(restr_graph): + """Test that an invalid table raises an error.""" + with pytest.raises(ValueError): + restr_graph._get_node("invalid_table") + + +def test_rg_invalid_edge(restr_graph, Nwbfile, common): + """Test that an invalid edge raises an error.""" + with pytest.raises(ValueError): + restr_graph._get_edge(Nwbfile, common.common_behav.PositionSource) + + +def test_rg_restr_subset(restr_graph, leaf): + prev_ft = restr_graph._get_ft(leaf.full_table_name, with_restr=True) + + restr_graph._set_restr(leaf, restriction=False) + + new_ft = restr_graph._get_ft(leaf.full_table_name, with_restr=True) + assert len(prev_ft) == len(new_ft), "Subset sestriction changed length." + + +@pytest.mark.skipif(not VERBOSE, reason="No logging to test when quiet-spy") +def test_rg_no_restr(caplog, restr_graph, common): + restr_graph._set_restr(common.LabTeam, restriction=False) + restr_graph._get_ft(common.LabTeam.full_table_name, with_restr=True) + assert "No restr" in caplog.text, "No warning logged on no restriction." + + +def test_rg_invalid_direction(restr_graph, leaf): + """Test that an invalid direction raises an error.""" + with pytest.raises(ValueError): + restr_graph._get_next_tables(leaf.full_table_name, "invalid_direction") @pytest.fixture(scope="session") @@ -79,8 +120,7 @@ def restr_graph_root(restr_graph, common, lfp_band, lin_v1, frequent_imports): yield RestrGraph( seed_table=common.Session(), - table_name=common.Session.full_table_name, - restriction="True", + leaves={common.Session.full_table_name: "True"}, direction="down", cascade=True, verbose=False, @@ -89,7 +129,7 @@ def restr_graph_root(restr_graph, common, lfp_band, lin_v1, frequent_imports): def test_rg_root(restr_graph_root): assert ( - len(restr_graph_root["trodes_pos_v1"]) == 2 + len(restr_graph_root["trodes_pos_v1"]) >= 1 ), "Incomplete cascade from root." @@ -125,6 +165,52 @@ def test_restr_from_downstream(graph_tables, table, restr, expect_n, msg): assert len(graph_tables[table]() << restr) == expect_n, msg +@pytest.mark.skipif(not VERBOSE, reason="No logging to test when quiet-spy.") +def test_ban_node(caplog, graph_tables): + search_restr = "sk_attr > 17" + ParentNode = graph_tables["ParentNode"]() + SkNode = graph_tables["SkNode"]() + + ParentNode.ban_search_table(SkNode) + ParentNode >> search_restr + assert "could not be applied" in caplog.text, "Found banned table." + + ParentNode.see_banned_tables() + assert "Banned tables" in caplog.text, "Banned tables not logged." + + ParentNode.unban_search_table(SkNode) + assert len(ParentNode >> search_restr) == 3, "Unban failed." + + +def test_null_restrict_by(graph_tables): + PkNode = graph_tables["PkNode"]() + assert (PkNode >> True) == PkNode, "Null restriction failed." + + +@pytest.mark.skipif(not VERBOSE, reason="No logging to test when quiet-spy.") +def test_restrict_by_this_table(caplog, graph_tables): + PkNode = graph_tables["PkNode"]() + PkNode >> "pk_id > 4" + assert "valid for" in caplog.text, "No warning logged without search." + + +def test_invalid_restr_direction(graph_tables): + PkNode = graph_tables["PkNode"]() + with pytest.raises(ValueError): + PkNode.restrict_by("bad_attr > 0", direction="invalid_direction") + + +@pytest.mark.skipif(not VERBOSE, reason="No logging to test when quiet-spy.") +def test_warn_nonrestrict(caplog, graph_tables): + ParentNode = graph_tables["ParentNode"]() + restr_parent = ParentNode & "parent_id > 4 AND parent_id < 9" + + restr_parent >> "sk_id > 0" + assert "Same length" in caplog.text, "No warning logged on non-restrict." + restr_parent >> "sk_id > 99" + assert "No entries" in caplog.text, "No warning logged on non-restrict." + + def test_restr_many_to_one(graph_tables_many_to_one): PK = graph_tables_many_to_one["PkSkNode"]() OP = graph_tables_many_to_one["OtherParentNode"]() @@ -139,7 +225,35 @@ def test_restr_many_to_one(graph_tables_many_to_one): ), "Error accepting list of dicts for `>>` for many to one." -def test_restr_invalid(graph_tables): +def test_restr_invalid_err(graph_tables): PkNode = graph_tables["PkNode"]() with pytest.raises(ValueError): len(PkNode << set(["parent_attr > 15", "parent_attr < 20"])) + + +@pytest.mark.skipif(not VERBOSE, reason="No logging to test when quiet-spy.") +def test_restr_invalid(caplog, graph_tables): + graph_tables["PkNode"]() << "invalid_restr=1" + assert ( + "could not be applied" in caplog.text + ), "No warning logged on invalid restr." + + +@pytest.fixture(scope="session") +def direction(): + from spyglass.utils.dj_graph import Direction + + yield Direction + + +def test_direction_str(direction): + assert str(direction.UP) == "up", "Direction str not as expected." + + +def test_direction_invert(direction): + assert ~direction.UP == direction("down"), "Direction inversion failed." + + +def test_direction_bool(direction): + assert bool(direction.UP), "Direction bool not as expected." + assert not direction.NONE, "Direction bool not as expected." diff --git a/tests/utils/test_merge.py b/tests/utils/test_merge.py index 9c192c20a..2876555a1 100644 --- a/tests/utils/test_merge.py +++ b/tests/utils/test_merge.py @@ -35,6 +35,25 @@ def test_nwb_table_missing(BadMerge, caplog, schema_test): assert "non-default definition" in txt, "Warning not caught." +@pytest.fixture(scope="function") +def NonMerge(): + from spyglass.utils import SpyglassMixin + + class NonMerge(SpyglassMixin, dj.Manual): + definition = """ + merge_id : uuid + --- + source : varchar(32) + """ + + yield NonMerge + + +def test_non_merge(NonMerge): + with pytest.raises(AttributeError): + NonMerge() + + def test_part_camel(merge_table): example_part = merge_table.parts(camel_case=True)[0] assert "_" not in example_part, "Camel case not applied." diff --git a/tests/utils/test_mixin.py b/tests/utils/test_mixin.py index 93d13407a..a35041013 100644 --- a/tests/utils/test_mixin.py +++ b/tests/utils/test_mixin.py @@ -8,10 +8,11 @@ def Mixin(): from spyglass.utils import SpyglassMixin - class Mixin(SpyglassMixin, dj.Manual): + class Mixin(SpyglassMixin, dj.Lookup): definition = """ id : int """ + contents = [(0,), (1,)] yield Mixin @@ -32,64 +33,92 @@ def test_nwb_table_missing(schema_test, Mixin): Mixin().fetch_nwb() -def test_merge_detect(Nwbfile, pos_merge_tables): +def test_auto_increment(schema_test, Mixin): + schema_test(Mixin) + ret = Mixin()._auto_increment(key={}, pk="id") + assert ret["id"] == 2, "Auto increment not working." + + +def test_null_file_like(schema_test, Mixin): + schema_test(Mixin) + ret = Mixin().file_like(None) + assert len(ret) == len(Mixin()), "Null file_like not working." + + +@pytest.mark.skipif(not VERBOSE, reason="No logging to test when quiet-spy.") +def test_bad_file_like(caplog, schema_test, Mixin): + schema_test(Mixin) + Mixin().file_like("BadName") + assert "No file_like field" in caplog.text, "No warning issued." + + +def test_partmaster_detect(Nwbfile, pos_merge_tables): """Test that the mixin can detect merge children of merge.""" - merges_found = set(Nwbfile._merge_chains.keys()) - merges_expected = set([t.full_table_name for t in pos_merge_tables]) - assert merges_expected.issubset( - merges_found - ), "Merges not detected by mixin." + assert len(Nwbfile._part_masters) >= 14, "Part masters not detected." -def test_merge_chain_join( - Nwbfile, pos_merge_tables, lin_v1, lfp_merge_key, populate_dlc +def test_downstream_restrict( + Nwbfile, frequent_imports, pos_merge_tables, lin_v1, lfp_merge_key ): - """Test that the mixin can join merge chains. + """Test that the mixin can join merge chains.""" - NOTE: This will change if more data is added to merge tables.""" - _ = lin_v1, lfp_merge_key, populate_dlc # merge tables populated + _ = frequent_imports # graph for cascade + _ = lin_v1, lfp_merge_key # merge tables populated - all_chains = [ - chains.cascade(True, direction="down") - for chains in Nwbfile._merge_chains.values() - ] - end_len = [len(chain) for chain in all_chains] + restr_ddp = Nwbfile.ddp(dry_run=True, reload_cache=True) + end_len = [len(ft) for ft in restr_ddp] - assert sum(end_len) >= 3, "Merge chains not joined correctly." + assert sum(end_len) >= 8, "Downstream parts not restricted correctly." -def test_get_chain(Nwbfile, pos_merge_tables): +def test_get_downstream_merge(Nwbfile, pos_merge_tables): """Test that the mixin can get the chain of a merge.""" - lin_parts = Nwbfile._get_chain("linear").part_names - lin_output = pos_merge_tables[1] - assert lin_parts == lin_output.parts(), "Chain not found." + lin_output = pos_merge_tables[1].full_table_name + assert lin_output in Nwbfile._part_masters, "Merge not found." @pytest.mark.skipif(not VERBOSE, reason="No logging to test when quiet-spy.") -def test_ddm_warning(Nwbfile, caplog): +def test_ddp_warning(Nwbfile, caplog): """Test that the mixin warns on empty delete_downstream_merge.""" - (Nwbfile.file_like("BadName")).delete_downstream_merge( + (Nwbfile.file_like("BadName")).delete_downstream_parts( reload_cache=True, disable_warnings=False ) - assert "No merge deletes found" in caplog.text, "No warning issued." + assert "No part deletes found" in caplog.text, "No warning issued." -def test_ddm_dry_run(Nwbfile, common, sgp, pos_merge_tables, lin_v1): +def test_ddp_dry_run( + Nwbfile, frequent_imports, common, sgp, pos_merge_tables, lin_v1 +): """Test that the mixin can dry run delete_downstream_merge.""" _ = lin_v1 # merge tables populated + _ = frequent_imports # graph for cascade + pos_output_name = pos_merge_tables[0].full_table_name param_field = "trodes_pos_params_name" trodes_params = sgp.v1.TrodesPosParams() - rft = (trodes_params & f'{param_field} LIKE "%ups%"').ddm( - reload_cache=True, dry_run=True, return_parts=False - )[pos_output_name][0] - assert len(rft) == 1, "ddm did not return restricted table." + rft = [ + table + for table in (trodes_params & f'{param_field} LIKE "%ups%"').ddp( + reload_cache=True, dry_run=True + ) + if table.full_table_name == pos_output_name + ] + assert len(rft) == 1, "ddp did not return restricted table." + + +def test_exp_summary(Nwbfile): + fields = Nwbfile._get_exp_summary().heading.names + expected = ["nwb_file_name", "lab_member_name"] + assert fields == expected, "Exp summary fields not as expected." - table_name = [p for p in pos_merge_tables[0].parts() if "trode" in p][0] - assert table_name == rft.full_table_name, "ddm didn't grab right table." - assert ( - rft.fetch1(param_field) == "single_led_upsampled" - ), "ddm didn't grab right row." +def test_cautious_del_dry_run(Nwbfile, frequent_imports): + _ = frequent_imports # part of cascade, need import + ret = Nwbfile.cdel(dry_run=True) + part_master_names = [t.full_table_name for t in ret[0]] + part_masters = Nwbfile._part_masters + assert all( + [pm in part_masters for pm in part_master_names] + ), "Non part masters found in cautious delete dry run." From 3ba5d0a50bee913e62ed15890b2a3515c469bc37 Mon Sep 17 00:00:00 2001 From: Samuel Bray Date: Thu, 27 Jun 2024 08:48:32 -0700 Subject: [PATCH 09/94] remove kachery_client dependency (#1014) --- CHANGELOG.md | 1 + .../v0/figurl_views/SpikeSortingRecordingView.py | 7 ++----- .../spikesorting/v0/figurl_views/SpikeSortingView.py | 4 ++-- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0092789c2..c304baf07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,7 @@ PositionIntervalMap.alter() - Remove unused `UnitInclusionParameters` table from `spikesorting.v0` #1003 - Fix bug in identification of artifact samples to be zeroed out in `spikesorting.v1.SpikeSorting` #1009 + - Remove deprecated dependencies on kachery_client #1014 ## [0.5.2] (April 22, 2024) diff --git a/src/spyglass/spikesorting/v0/figurl_views/SpikeSortingRecordingView.py b/src/spyglass/spikesorting/v0/figurl_views/SpikeSortingRecordingView.py index 0c0e4a9e1..aedb83522 100644 --- a/src/spyglass/spikesorting/v0/figurl_views/SpikeSortingRecordingView.py +++ b/src/spyglass/spikesorting/v0/figurl_views/SpikeSortingRecordingView.py @@ -2,7 +2,7 @@ from typing import List, Union import datajoint as dj -import kachery_client as kc +import kachery_cloud as kcl import numpy as np import spikeinterface as si from sortingview.SpikeSortingView import create_raw_traces_plot @@ -103,9 +103,6 @@ def create_mountain_layout( def _upload_data_and_return_sha1(data): - data_uri = kc.store_json(data) + data_uri = kcl.store_json(data) data_hash = data_uri.split("/")[2] - kc.upload_file( - data_uri, channel=os.environ["FIGURL_CHANNEL"], single_chunk=True - ) return data_hash diff --git a/src/spyglass/spikesorting/v0/figurl_views/SpikeSortingView.py b/src/spyglass/spikesorting/v0/figurl_views/SpikeSortingView.py index d05b61f1a..45d498565 100644 --- a/src/spyglass/spikesorting/v0/figurl_views/SpikeSortingView.py +++ b/src/spyglass/spikesorting/v0/figurl_views/SpikeSortingView.py @@ -1,5 +1,5 @@ import datajoint as dj -import kachery_client as kc +import kachery_cloud as kcl import spikeinterface as si from sortingview.SpikeSortingView import ( SpikeSortingView as SortingViewSpikeSortingView, @@ -38,7 +38,7 @@ def make(self, key): recording: si.BaseRecording = si.load_extractor(recording_path) sorting: si.BaseSorting = si.load_extractor(sorting_path) - with kc.TemporaryDirectory() as tmpdir: + with kcl.TemporaryDirectory() as tmpdir: fname = f"{tmpdir}/spikesortingview.h5" logger.info("Preparing spikesortingview data") prepare_spikesortingview_data( From 1445b67065df42497e478a6f6fda2063e506f941 Mon Sep 17 00:00:00 2001 From: Samuel Bray Date: Mon, 1 Jul 2024 16:15:24 -0700 Subject: [PATCH 10/94] Add upsampling option to PositionGroup (#1008) * add upsampling option to PositionGroup * ignore nan values in non-decoded columns * add upsample option to tutorial notebook * update changelog * fix spelling * add alter statement to release notes * remove todo comment --- CHANGELOG.md | 6 +- notebooks/41_Decoding_Clusterless.ipynb | 5 +- .../py_scripts/41_Decoding_Clusterless.py | 3 +- src/spyglass/decoding/v1/clusterless.py | 19 +-- src/spyglass/decoding/v1/core.py | 110 ++++++++++++++++++ src/spyglass/decoding/v1/sorted_spikes.py | 20 +--- 6 files changed, 130 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c304baf07..5d23ccba1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,10 @@ ```python from spyglass.common.common_behav import PositionIntervalMap +from spyglass.decoding.v1.core import PositionGroup PositionIntervalMap.alter() +PositionGroup.alter() ``` ### Infrastructure @@ -42,7 +44,9 @@ PositionIntervalMap.alter() - Remove redundant calls to tables in `populate_all_common` #870 - Improve logging clarity in `populate_all_common` #870 - `PositionIntervalMap` now inserts null entries for missing intervals #870 -- Decoding: Default values for classes on `ImportError` #966 +- Decoding: + - Default values for classes on `ImportError` #966 + - Add option to upsample data rate in `PositionGroup` #1008 - Position - Allow dlc without pre-existing tracking data #973, #975 - Raise `KeyError` for missing input parameters across helper funcs #966 diff --git a/notebooks/41_Decoding_Clusterless.ipynb b/notebooks/41_Decoding_Clusterless.ipynb index 69286c69a..6f8db891b 100644 --- a/notebooks/41_Decoding_Clusterless.ipynb +++ b/notebooks/41_Decoding_Clusterless.ipynb @@ -700,7 +700,7 @@ "\n", "We use the the `PositionOutput` table to figure out the `merge_id` associated with `nwb_file_name` to get the position data associated with the NWB file of interest. In this case, we only have one position to insert, but we could insert multiple positions if we wanted to decode from multiple sessions.\n", "\n", - "Note that the position data sampling frequency is what determines the time step of the decoding. In this case, the position data sampling frequency is 30 Hz, so the time step of the decoding will be 1/30 seconds. In practice, you will want to use a smaller time step such as 500 Hz. This will allow you to decode at a finer time scale. To do this, you will want to interpolate the position data to a higher sampling frequency as shown in the [position trodes notebook](./20_Position_Trodes.ipynb).\n", + "Note that we can use the `upsample_rate` parameter to define the rate to which position data will be upsampled to to for decoding in Hz. This is useful if we want to decode at a finer time scale than the position data sampling frequency. In practice, a value of 500Hz is used in many analyses. Skipping or providing a null value for this parameter will default to using the position sampling rate.\n", "\n", "You will also want to specify the name of the position variables if they are different from the default names. The default names are `position_x` and `position_y`." ] @@ -981,6 +981,7 @@ " nwb_file_name=nwb_copy_file_name,\n", " group_name=\"test_group\",\n", " keys=[{\"pos_merge_id\": merge_id} for merge_id in position_merge_ids],\n", + " upsample_rate=500,\n", ")\n", "\n", "PositionGroup & {\n", @@ -2956,7 +2957,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.16" + "version": "3.9.-1" } }, "nbformat": 4, diff --git a/notebooks/py_scripts/41_Decoding_Clusterless.py b/notebooks/py_scripts/41_Decoding_Clusterless.py index 6c286bf8d..b9eede455 100644 --- a/notebooks/py_scripts/41_Decoding_Clusterless.py +++ b/notebooks/py_scripts/41_Decoding_Clusterless.py @@ -125,7 +125,7 @@ # # We use the the `PositionOutput` table to figure out the `merge_id` associated with `nwb_file_name` to get the position data associated with the NWB file of interest. In this case, we only have one position to insert, but we could insert multiple positions if we wanted to decode from multiple sessions. # -# Note that the position data sampling frequency is what determines the time step of the decoding. In this case, the position data sampling frequency is 30 Hz, so the time step of the decoding will be 1/30 seconds. In practice, you will want to use a smaller time step such as 500 Hz. This will allow you to decode at a finer time scale. To do this, you will want to interpolate the position data to a higher sampling frequency as shown in the [position trodes notebook](./20_Position_Trodes.ipynb). +# Note that we can use the `upsample_rate` parameter to define the rate to which position data will be upsampled to to for decoding in Hz. This is useful if we want to decode at a finer time scale than the position data sampling frequency. In practice, a value of 500Hz is used in many analyses. Skipping or providing a null value for this parameter will default to using the position sampling rate. # # You will also want to specify the name of the position variables if they are different from the default names. The default names are `position_x` and `position_y`. @@ -181,6 +181,7 @@ nwb_file_name=nwb_copy_file_name, group_name="test_group", keys=[{"pos_merge_id": merge_id} for merge_id in position_merge_ids], + upsample_rate=500, ) PositionGroup & { diff --git a/src/spyglass/decoding/v1/clusterless.py b/src/spyglass/decoding/v1/clusterless.py index 9dd601651..881551922 100644 --- a/src/spyglass/decoding/v1/clusterless.py +++ b/src/spyglass/decoding/v1/clusterless.py @@ -385,22 +385,11 @@ def fetch_position_info(key): "position_group_name": key["position_group_name"], "nwb_file_name": key["nwb_file_name"], } - position_variable_names = (PositionGroup & position_group_key).fetch1( - "position_variables" - ) - - position_info = [] - for pos_merge_id in (PositionGroup.Position & position_group_key).fetch( - "pos_merge_id" - ): - position_info.append( - (PositionOutput & {"merge_id": pos_merge_id}).fetch1_dataframe() - ) min_time, max_time = ClusterlessDecodingV1._get_interval_range(key) - position_info = ( - pd.concat(position_info, axis=0).loc[min_time:max_time].dropna() - ) + position_info, position_variable_names = ( + PositionGroup & position_group_key + ).fetch_position_info(min_time=min_time, max_time=max_time) return position_info, position_variable_names @@ -441,7 +430,7 @@ def fetch_linear_position_info(key): axis=1, ) .loc[min_time:max_time] - .dropna() + .dropna(subset=position_variable_names) ) @staticmethod diff --git a/src/spyglass/decoding/v1/core.py b/src/spyglass/decoding/v1/core.py index 79828b7db..d7ed2e89c 100644 --- a/src/spyglass/decoding/v1/core.py +++ b/src/spyglass/decoding/v1/core.py @@ -1,4 +1,6 @@ import datajoint as dj +import numpy as np +import pandas as pd from non_local_detector import ( ContFragClusterlessClassifier, ContFragSortedSpikesClassifier, @@ -92,6 +94,7 @@ class PositionGroup(SpyglassMixin, dj.Manual): position_group_name: varchar(80) ---- position_variables = NULL: longblob # list of position variables to decode + upsample_rate = NULL: float # upsampling rate for position data (Hz) """ class Position(SpyglassMixinPart): @@ -106,6 +109,7 @@ def create_group( group_name: str, keys: list[dict], position_variables: list[str] = ["position_x", "position_y"], + upsample_rate: float = np.nan, ): group_key = { "nwb_file_name": nwb_file_name, @@ -115,6 +119,7 @@ def create_group( { **group_key, "position_variables": position_variables, + "upsample_rate": upsample_rate, }, skip_duplicates=True, ) @@ -126,3 +131,108 @@ def create_group( }, skip_duplicates=True, ) + + def fetch_position_info( + self, key: dict = None, min_time: float = None, max_time: float = None + ) -> tuple[pd.DataFrame, list[str]]: + """fetch position information for decoding + + Parameters + ---------- + key : dict, optional + restriction to a single entry in PositionGroup, by default None + min_time : float, optional + restrict position information to times greater than min_time, by default None + max_time : float, optional + restrict position information to times less than max_time, by default None + + Returns + ------- + tuple[pd.DataFrame, list[str]] + position information and names of position variables + """ + if key is None: + key = {} + key = (self & key).fetch1("KEY") + position_variable_names = (self & key).fetch1("position_variables") + + position_info = [] + upsample_rate = (self & key).fetch1("upsample_rate") + for pos_merge_id in (self.Position & key).fetch("pos_merge_id"): + if not np.isnan(upsample_rate): + position_info.append( + self._upsample( + ( + PositionOutput & {"merge_id": pos_merge_id} + ).fetch1_dataframe(), + upsampling_sampling_rate=upsample_rate, + ) + ) + else: + position_info.append( + ( + PositionOutput & {"merge_id": pos_merge_id} + ).fetch1_dataframe() + ) + + if min_time is None: + min_time = min([df.index.min() for df in position_info]) + if max_time is None: + max_time = max([df.index.max() for df in position_info]) + position_info = ( + pd.concat(position_info, axis=0) + .loc[min_time:max_time] + .dropna(subset=position_variable_names) + ) + + return position_info, position_variable_names + + @staticmethod + def _upsample( + position_df: pd.DataFrame, + upsampling_sampling_rate: float, + upsampling_interpolation_method: str = "linear", + ) -> pd.DataFrame: + """upsample position data to a fixed sampling rate + + Parameters + ---------- + position_df : pd.DataFrame + dataframe containing position data + upsampling_sampling_rate : float + sampling rate to upsample to + upsampling_interpolation_method : str, optional + pandas method for interpolation, by default "linear" + + Returns + ------- + pd.DataFrame + upsampled position data + """ + + upsampling_start_time = position_df.index[0] + upsampling_end_time = position_df.index[-1] + + n_samples = ( + int( + np.ceil( + (upsampling_end_time - upsampling_start_time) + * upsampling_sampling_rate + ) + ) + + 1 + ) + new_time = np.linspace( + upsampling_start_time, upsampling_end_time, n_samples + ) + new_index = pd.Index( + np.unique(np.concatenate((position_df.index, new_time))), + name="time", + ) + position_df = ( + position_df.reindex(index=new_index) + .interpolate(method=upsampling_interpolation_method) + .reindex(index=new_time) + ) + + return position_df diff --git a/src/spyglass/decoding/v1/sorted_spikes.py b/src/spyglass/decoding/v1/sorted_spikes.py index 310b6ca43..339bde314 100644 --- a/src/spyglass/decoding/v1/sorted_spikes.py +++ b/src/spyglass/decoding/v1/sorted_spikes.py @@ -349,20 +349,12 @@ def fetch_position_info(key): "position_group_name": key["position_group_name"], "nwb_file_name": key["nwb_file_name"], } - position_variable_names = (PositionGroup & position_group_key).fetch1( - "position_variables" - ) - - position_info = [] - for pos_merge_id in (PositionGroup.Position & position_group_key).fetch( - "pos_merge_id" - ): - position_info.append( - (PositionOutput & {"merge_id": pos_merge_id}).fetch1_dataframe() - ) min_time, max_time = SortedSpikesDecodingV1._get_interval_range(key) - position_info = ( - pd.concat(position_info, axis=0).loc[min_time:max_time].dropna() + position_info, position_variable_names = ( + PositionGroup & position_group_key + ).fetch_position_info( + min_time=min_time, + max_time=max_time, ) return position_info, position_variable_names @@ -402,7 +394,7 @@ def fetch_linear_position_info(key): axis=1, ) .loc[min_time:max_time] - .dropna() + .dropna(subset=position_variable_names) ) @staticmethod From 589e315443dc983301580a00ad4d353edcebf86f Mon Sep 17 00:00:00 2001 From: Stephanie Crater <70605721+calderast@users.noreply.github.com> Date: Tue, 2 Jul 2024 09:25:25 -0700 Subject: [PATCH 11/94] Implement adding data from config file (#934) * Implement adding data from config file * run black * Apply suggestions from code review Co-authored-by: Chris Brozdowski * Apply suggestions from code review * add default config to arguments --------- Co-authored-by: Chris Brozdowski Co-authored-by: Sam Bray Co-authored-by: Eric Denovellis --- src/spyglass/common/common_device.py | 25 ++++++++-- src/spyglass/common/common_lab.py | 67 +++++++++++++++++++++------ src/spyglass/common/common_session.py | 36 +++++++------- src/spyglass/common/common_subject.py | 58 ++++++++++++++++------- 4 files changed, 135 insertions(+), 51 deletions(-) diff --git a/src/spyglass/common/common_device.py b/src/spyglass/common/common_device.py index b2e764b24..ea71e4b3e 100644 --- a/src/spyglass/common/common_device.py +++ b/src/spyglass/common/common_device.py @@ -36,7 +36,7 @@ class DataAcquisitionDevice(SpyglassMixin, dj.Manual): """ @classmethod - def insert_from_nwbfile(cls, nwbf, config): + def insert_from_nwbfile(cls, nwbf, config={}): """Insert data acquisition devices from an NWB file. Note that this does not link the DataAcquisitionDevices with a Session. @@ -252,13 +252,16 @@ class CameraDevice(SpyglassMixin, dj.Manual): """ @classmethod - def insert_from_nwbfile(cls, nwbf): + def insert_from_nwbfile(cls, nwbf, config={}): """Insert camera devices from an NWB file Parameters ---------- nwbf : pynwb.NWBFile The source NWB file object. + config : dict + Dictionary read from a user-defined YAML file containing values to + replace in the NWB file. Returns ------- @@ -268,7 +271,6 @@ def insert_from_nwbfile(cls, nwbf): device_name_list = list() for device in nwbf.devices.values(): if isinstance(device, ndx_franklab_novela.CameraDevice): - device_dict = dict() # TODO ideally the ID is not encoded in the name formatted in a # particular way device.name must have the form "[any string # without a space, usually camera] [int]" @@ -282,6 +284,21 @@ def insert_from_nwbfile(cls, nwbf): } cls.insert1(device_dict, skip_duplicates=True) device_name_list.append(device_dict["camera_name"]) + # Append devices from config file + if device_list := config.get("CameraDevice"): + device_inserts = [ + { + "camera_id": device.get("camera_id", -1), + "camera_name": device.get("camera_name"), + "manufacturer": device.get("manufacturer"), + "model": device.get("model"), + "lens": device.get("lens"), + "meters_per_pixel": device.get("meters_per_pixel", 0), + } + for device in device_list + ] + cls.insert(device_inserts, skip_duplicates=True) + device_name_list.extend([d["camera_name"] for d in device_inserts]) if device_name_list: logger.info(f"Inserted camera devices {device_name_list}") else: @@ -339,7 +356,7 @@ class Electrode(SpyglassMixin, dj.Part): """ @classmethod - def insert_from_nwbfile(cls, nwbf, config): + def insert_from_nwbfile(cls, nwbf, config={}): """Insert probe devices from an NWB file. Parameters diff --git a/src/spyglass/common/common_lab.py b/src/spyglass/common/common_lab.py index 5958a4de9..be847d79c 100644 --- a/src/spyglass/common/common_lab.py +++ b/src/spyglass/common/common_lab.py @@ -42,23 +42,33 @@ class LabMemberInfo(SpyglassMixin, dj.Part): _admin = [] @classmethod - def insert_from_nwbfile(cls, nwbf): + def insert_from_nwbfile(cls, nwbf, config={}): """Insert lab member information from an NWB file. Parameters ---------- nwbf: pynwb.NWBFile The NWB file with experimenter information. + config : dict + Dictionary read from a user-defined YAML file containing values to + replace in the NWB file. """ if isinstance(nwbf, str): nwb_file_abspath = Nwbfile.get_abs_path(nwbf, new_file=True) nwbf = get_nwb_file(nwb_file_abspath) - if nwbf.experimenter is None: + if "LabMember" in config: + experimenter_list = [ + member_dict["lab_member_name"] + for member_dict in config["LabMember"] + ] + elif nwbf.experimenter is not None: + experimenter_list = nwbf.experimenter + else: logger.info("No experimenter metadata found.\n") return - for experimenter in nwbf.experimenter: + for experimenter in experimenter_list: cls.insert_from_name(experimenter) # each person is by default the member of their own LabTeam @@ -235,21 +245,36 @@ class Institution(SpyglassMixin, dj.Manual): """ @classmethod - def insert_from_nwbfile(cls, nwbf): + def insert_from_nwbfile(cls, nwbf, config={}): """Insert institution information from an NWB file. Parameters ---------- nwbf : pynwb.NWBFile The NWB file with institution information. + config : dict + Dictionary read from a user-defined YAML file containing values to + replace in the NWB file. + + Returns + ------- + institution_name : string + The name of the institution found in the NWB or config file, or None. """ - if nwbf.institution is None: + inst_list = config.get("Institution", [{}]) + if len(inst_list) > 1: + logger.info( + "Multiple institution entries not allowed. Using the first entry only.\n" + ) + inst_name = inst_list[0].get("institution_name") or getattr( + nwbf, "institution", None + ) + if not inst_name: logger.info("No institution metadata found.\n") - return + return None - cls.insert1( - dict(institution_name=nwbf.institution), skip_duplicates=True - ) + cls.insert1(dict(institution_name=inst_name), skip_duplicates=True) + return inst_name @schema @@ -259,18 +284,34 @@ class Lab(SpyglassMixin, dj.Manual): """ @classmethod - def insert_from_nwbfile(cls, nwbf): + def insert_from_nwbfile(cls, nwbf, config={}): """Insert lab name information from an NWB file. Parameters ---------- nwbf : pynwb.NWBFile The NWB file with lab name information. + config : dict + Dictionary read from a user-defined YAML file containing values to + replace in the NWB file. + + Returns + ------- + lab_name : string + The name of the lab found in the NWB or config file, or None. """ - if nwbf.lab is None: + lab_list = config.get("Lab", [{}]) + if len(lab_list) > 1: + logger.info( + "Multiple lab entries not allowed. Using the first entry only.\n" + ) + lab_name = lab_list[0].get("lab_name") or getattr(nwbf, "lab", None) + if not lab_name: logger.info("No lab metadata found.\n") - return - cls.insert1(dict(lab_name=nwbf.lab), skip_duplicates=True) + return None + + cls.insert1(dict(lab_name=lab_name), skip_duplicates=True) + return lab_name def decompose_name(full_name: str) -> tuple: diff --git a/src/spyglass/common/common_session.py b/src/spyglass/common/common_session.py index 186562444..893b727b5 100644 --- a/src/spyglass/common/common_session.py +++ b/src/spyglass/common/common_session.py @@ -78,38 +78,33 @@ def make(self, key): # tables (e.g., Experimenter, DataAcquisitionDevice). logger.info("Session populates Institution...") - Institution().insert_from_nwbfile(nwbf) + institution_name = Institution().insert_from_nwbfile(nwbf, config) logger.info("Session populates Lab...") - Lab().insert_from_nwbfile(nwbf) + lab_name = Lab().insert_from_nwbfile(nwbf, config) logger.info("Session populates LabMember...") - LabMember().insert_from_nwbfile(nwbf) + LabMember().insert_from_nwbfile(nwbf, config) logger.info("Session populates Subject...") - Subject().insert_from_nwbfile(nwbf) + subject_id = Subject().insert_from_nwbfile(nwbf, config) if not debug_mode: # TODO: remove when demo files agree on device logger.info("Session populates Populate DataAcquisitionDevice...") DataAcquisitionDevice.insert_from_nwbfile(nwbf, config) logger.info("Session populates Populate CameraDevice...") - CameraDevice.insert_from_nwbfile(nwbf) + CameraDevice.insert_from_nwbfile(nwbf, config) logger.info("Session populates Populate Probe...") Probe.insert_from_nwbfile(nwbf, config) - if nwbf.subject is not None: - subject_id = nwbf.subject.subject_id - else: - subject_id = None - Session().insert1( { "nwb_file_name": nwb_file_name, "subject_id": subject_id, - "institution_name": nwbf.institution, - "lab_name": nwbf.lab, + "institution_name": institution_name, + "lab_name": lab_name, "session_id": nwbf.session_id, "session_description": nwbf.session_description, "session_start_time": nwbf.session_start_time, @@ -133,9 +128,9 @@ def make(self, key): # Unit().insert_from_nwbfile(nwbf, nwb_file_name=nwb_file_name) self._add_data_acquisition_device_part(nwb_file_name, nwbf, config) - self._add_experimenter_part(nwb_file_name, nwbf) + self._add_experimenter_part(nwb_file_name, nwbf, config) - def _add_data_acquisition_device_part(self, nwb_file_name, nwbf, config): + def _add_data_acquisition_device_part(self, nwb_file_name, nwbf, config={}): # get device names from both the NWB file and the associated config file device_names, _, _ = DataAcquisitionDevice.get_all_device_names( nwbf, config @@ -157,11 +152,18 @@ def _add_data_acquisition_device_part(self, nwb_file_name, nwbf, config): key["data_acquisition_device_name"] = device_name Session.DataAcquisitionDevice.insert1(key) - def _add_experimenter_part(self, nwb_file_name, nwbf): - if nwbf.experimenter is None: + def _add_experimenter_part(self, nwb_file_name, nwbf, config={}): + # Use config file over nwb file + if members := config.get("LabMember"): + experimenter_list = [ + member["lab_member_name"] for member in members + ] + elif nwbf.experimenter is not None: + experimenter_list = nwbf.experimenter + else: return - for name in nwbf.experimenter: + for name in experimenter_list: # ensure that the foreign key exists and do nothing if not query = LabMember & {"lab_member_name": name} if len(query) == 0: diff --git a/src/spyglass/common/common_subject.py b/src/spyglass/common/common_subject.py index 053909374..a6320f7e5 100644 --- a/src/spyglass/common/common_subject.py +++ b/src/spyglass/common/common_subject.py @@ -18,23 +18,47 @@ class Subject(SpyglassMixin, dj.Manual): """ @classmethod - def insert_from_nwbfile(cls, nwbf): - """Get the subject info from the NWBFile, insert into the Subject.""" - sub = nwbf.subject - if sub is None: + def insert_from_nwbfile(cls, nwbf, config={}): + """Get the subject info from the NWBFile, insert into the Subject. + + Parameters + ---------- + nwbf: pynwb.NWBFile + The NWB file with subject information. + config : dict + Dictionary read from a user-defined YAML file containing values to + replace in the NWB file. + + Returns + ------- + subject_id : string + The id of the subject found in the NWB or config file, or None. + """ + if "Subject" not in config and nwbf.subject is None: logger.warn("No subject metadata found.\n") - return - subject_dict = dict() - subject_dict["subject_id"] = sub.subject_id - subject_dict["age"] = sub.age - subject_dict["description"] = sub.description - subject_dict["genotype"] = sub.genotype - if sub.sex in ("Male", "male", "M", "m"): - sex = "M" - elif sub.sex in ("Female", "female", "F", "f"): - sex = "F" + return None + + conf = config["Subject"][0] if "Subject" in config else dict() + sub = ( + nwbf.subject + if nwbf.subject is not None + else type("DefaultObject", (), {})() + ) + subject_dict = { + field: conf.get(field, getattr(sub, field, None)) + for field in [ + "subject_id", + "age", + "description", + "genotype", + "species", + "sex", + ] + } + if (sex := subject_dict["sex"][0].upper()) in ("M", "F"): + subject_dict["sex"] = sex else: - sex = "U" - subject_dict["sex"] = sex - subject_dict["species"] = sub.species + subject_dict["sex"] = "U" + cls.insert1(subject_dict, skip_duplicates=True) + return subject_dict["subject_id"] From 10645ca0ac3eeaadda01e3e94277864d8899ab6d Mon Sep 17 00:00:00 2001 From: Kyu Hyun Lee Date: Wed, 3 Jul 2024 07:54:13 -0700 Subject: [PATCH 12/94] Fix broken link to Clusterless Decoding tutorial notebook from Sorted Spikes Decoding tutorial notebook (#1018) * Save LFP as pynwb.ecephys.LFP * Fix formatting * Fix formatting * Fix typo in link --------- Co-authored-by: Eric Denovellis --- notebooks/42_Decoding_SortedSpikes.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/42_Decoding_SortedSpikes.ipynb b/notebooks/42_Decoding_SortedSpikes.ipynb index 66c3de7f0..e4448c681 100644 --- a/notebooks/42_Decoding_SortedSpikes.ipynb +++ b/notebooks/42_Decoding_SortedSpikes.ipynb @@ -6,7 +6,7 @@ "source": [ "# Sorted Spikes Decoding\n", "\n", - "The mechanics of decoding with sorted spikes are largely similar to those of decoding with unsorted spikes. You should familiarize yourself with the [clusterless decoding tutorial](./42_Decoding_Clusterless.ipynb) before proceeding with this one.\n", + "The mechanics of decoding with sorted spikes are largely similar to those of decoding with unsorted spikes. You should familiarize yourself with the [clusterless decoding tutorial](./41_Decoding_Clusterless.ipynb) before proceeding with this one.\n", "\n", "The elements we will need to decode with sorted spikes are:\n", "- `PositionGroup`\n", From 08fa17ce186d522a0f9976b4dd439be64e2c2ce9 Mon Sep 17 00:00:00 2001 From: Eric Denovellis Date: Wed, 3 Jul 2024 08:07:32 -0700 Subject: [PATCH 13/94] Hotfix: fix unsafe config kwarg --- src/spyglass/common/common_device.py | 10 +++++++--- src/spyglass/common/common_lab.py | 7 ++++--- src/spyglass/common/common_subject.py | 3 ++- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/spyglass/common/common_device.py b/src/spyglass/common/common_device.py index ea71e4b3e..b7a67bcab 100644 --- a/src/spyglass/common/common_device.py +++ b/src/spyglass/common/common_device.py @@ -36,7 +36,7 @@ class DataAcquisitionDevice(SpyglassMixin, dj.Manual): """ @classmethod - def insert_from_nwbfile(cls, nwbf, config={}): + def insert_from_nwbfile(cls, nwbf, config=None): """Insert data acquisition devices from an NWB file. Note that this does not link the DataAcquisitionDevices with a Session. @@ -50,6 +50,7 @@ def insert_from_nwbfile(cls, nwbf, config={}): Dictionary read from a user-defined YAML file containing values to replace in the NWB file. """ + config = config or dict() _, ndx_devices, _ = cls.get_all_device_names(nwbf, config) for device_name in ndx_devices: @@ -108,6 +109,7 @@ def get_all_device_names(cls, nwbf, config) -> tuple: device_name_list : tuple List of data acquisition object names found in the NWB file. """ + config = config or dict() # make a dict mapping device name to PyNWB device object for all devices # in the NWB file that are of type ndx_franklab_novela.DataAcqDevice and # thus have the required metadata @@ -252,7 +254,7 @@ class CameraDevice(SpyglassMixin, dj.Manual): """ @classmethod - def insert_from_nwbfile(cls, nwbf, config={}): + def insert_from_nwbfile(cls, nwbf, config=None): """Insert camera devices from an NWB file Parameters @@ -268,6 +270,7 @@ def insert_from_nwbfile(cls, nwbf, config={}): device_name_list : list List of camera device object names found in the NWB file. """ + config = config or dict() device_name_list = list() for device in nwbf.devices.values(): if isinstance(device, ndx_franklab_novela.CameraDevice): @@ -356,7 +359,7 @@ class Electrode(SpyglassMixin, dj.Part): """ @classmethod - def insert_from_nwbfile(cls, nwbf, config={}): + def insert_from_nwbfile(cls, nwbf, config=None): """Insert probe devices from an NWB file. Parameters @@ -372,6 +375,7 @@ def insert_from_nwbfile(cls, nwbf, config={}): device_name_list : list List of probe device types found in the NWB file. """ + config = config or dict() all_probes_types, ndx_probes, _ = cls.get_all_probe_names(nwbf, config) for probe_type in all_probes_types: diff --git a/src/spyglass/common/common_lab.py b/src/spyglass/common/common_lab.py index be847d79c..57acb780c 100644 --- a/src/spyglass/common/common_lab.py +++ b/src/spyglass/common/common_lab.py @@ -42,7 +42,7 @@ class LabMemberInfo(SpyglassMixin, dj.Part): _admin = [] @classmethod - def insert_from_nwbfile(cls, nwbf, config={}): + def insert_from_nwbfile(cls, nwbf, config=None): """Insert lab member information from an NWB file. Parameters @@ -245,7 +245,7 @@ class Institution(SpyglassMixin, dj.Manual): """ @classmethod - def insert_from_nwbfile(cls, nwbf, config={}): + def insert_from_nwbfile(cls, nwbf, config=None): """Insert institution information from an NWB file. Parameters @@ -284,7 +284,7 @@ class Lab(SpyglassMixin, dj.Manual): """ @classmethod - def insert_from_nwbfile(cls, nwbf, config={}): + def insert_from_nwbfile(cls, nwbf, config=None): """Insert lab name information from an NWB file. Parameters @@ -300,6 +300,7 @@ def insert_from_nwbfile(cls, nwbf, config={}): lab_name : string The name of the lab found in the NWB or config file, or None. """ + config = config or dict() lab_list = config.get("Lab", [{}]) if len(lab_list) > 1: logger.info( diff --git a/src/spyglass/common/common_subject.py b/src/spyglass/common/common_subject.py index a6320f7e5..b64f27b69 100644 --- a/src/spyglass/common/common_subject.py +++ b/src/spyglass/common/common_subject.py @@ -18,7 +18,7 @@ class Subject(SpyglassMixin, dj.Manual): """ @classmethod - def insert_from_nwbfile(cls, nwbf, config={}): + def insert_from_nwbfile(cls, nwbf, config=None): """Get the subject info from the NWBFile, insert into the Subject. Parameters @@ -34,6 +34,7 @@ def insert_from_nwbfile(cls, nwbf, config={}): subject_id : string The id of the subject found in the NWB or config file, or None. """ + config = config or dict() if "Subject" not in config and nwbf.subject is None: logger.warn("No subject metadata found.\n") return None From e395d0ab80bf5d2115ba8d3d68598891fd0b2b47 Mon Sep 17 00:00:00 2001 From: Eric Denovellis Date: Wed, 3 Jul 2024 18:25:24 -0700 Subject: [PATCH 14/94] Hotfix: fix missing dicts --- src/spyglass/common/common_lab.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/spyglass/common/common_lab.py b/src/spyglass/common/common_lab.py index 57acb780c..44be5f4b2 100644 --- a/src/spyglass/common/common_lab.py +++ b/src/spyglass/common/common_lab.py @@ -1,7 +1,6 @@ """Schema for institution, lab team/name/members. Session-independent.""" import datajoint as dj - from spyglass.utils import SpyglassMixin, logger from ..utils.nwb_helper_fn import get_nwb_file @@ -53,6 +52,7 @@ def insert_from_nwbfile(cls, nwbf, config=None): Dictionary read from a user-defined YAML file containing values to replace in the NWB file. """ + config = config or dict() if isinstance(nwbf, str): nwb_file_abspath = Nwbfile.get_abs_path(nwbf, new_file=True) nwbf = get_nwb_file(nwb_file_abspath) @@ -261,6 +261,7 @@ def insert_from_nwbfile(cls, nwbf, config=None): institution_name : string The name of the institution found in the NWB or config file, or None. """ + config = config or dict() inst_list = config.get("Institution", [{}]) if len(inst_list) > 1: logger.info( From 2f4b952bd975339df48a4302a63a86cad9db9cca Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Mon, 8 Jul 2024 10:54:49 -0700 Subject: [PATCH 15/94] #980 (#1021) * #980 * Add PR number --- CHANGELOG.md | 1 + src/spyglass/common/common_nwbfile.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d23ccba1..7042076c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ PositionGroup.alter() - Remove redundant calls to tables in `populate_all_common` #870 - Improve logging clarity in `populate_all_common` #870 - `PositionIntervalMap` now inserts null entries for missing intervals #870 + - `AnalysisFileLog` now truncates table names that exceed field length #1021 - Decoding: - Default values for classes on `ImportError` #966 - Add option to upsample data rate in `PositionGroup` #1008 diff --git a/src/spyglass/common/common_nwbfile.py b/src/spyglass/common/common_nwbfile.py index bcdc50c28..aa33b51e1 100644 --- a/src/spyglass/common/common_nwbfile.py +++ b/src/spyglass/common/common_nwbfile.py @@ -737,13 +737,14 @@ def log( analysis_file_name : str The name of the analysis NWB file. """ + self.insert1( { "dj_user": dj.config["database.user"], "analysis_file_name": analysis_file_name, "time_delta": time_delta, "file_size": file_size, - "table": table, + "table": table[:64], } ) From 3ccda2450a573f2ed778c9a0be335fbb97a347dc Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Tue, 9 Jul 2024 15:31:27 -0700 Subject: [PATCH 16/94] #1016 - Allow `ModuleNotFoundError` (#1023) * #1016 * Update changelog --- CHANGELOG.md | 1 + src/spyglass/__init__.py | 4 ++-- src/spyglass/cli/cli.py | 3 ++- src/spyglass/common/common_dandi.py | 2 +- src/spyglass/common/common_filter.py | 2 +- src/spyglass/common/common_position.py | 2 +- src/spyglass/decoding/v0/clusterless.py | 10 +++++++++- src/spyglass/decoding/v0/core.py | 2 +- src/spyglass/decoding/v0/dj_decoder_conversion.py | 2 +- src/spyglass/decoding/v0/sorted_spikes.py | 10 +++++++++- src/spyglass/position/v1/position_dlc_training.py | 2 +- src/spyglass/utils/dj_mixin.py | 2 +- 12 files changed, 30 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7042076c3..f4a84784c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ PositionGroup.alter() - `cautious_delete` now checks `IntervalList` and externals tables. #1002 - Allow mixin tables with parallelization in `make` to run populate with `processes > 1` #1001 +- Allow `ModuleNotFoundError` or `ImportError` for optional dependencies #1023 ### Pipelines diff --git a/src/spyglass/__init__.py b/src/spyglass/__init__.py index bfe9713c3..4e0413ec4 100644 --- a/src/spyglass/__init__.py +++ b/src/spyglass/__init__.py @@ -2,12 +2,12 @@ try: import ndx_franklab_novela -except ImportError: +except (ImportError, ModuleNotFoundError): pass try: from ._version import __version__ -except ImportError: +except (ImportError, ModuleNotFoundError): pass __all__ = ["ndx_franklab_novela", "__version__", "config"] diff --git a/src/spyglass/cli/cli.py b/src/spyglass/cli/cli.py index a07919da2..37871d490 100644 --- a/src/spyglass/cli/cli.py +++ b/src/spyglass/cli/cli.py @@ -4,7 +4,8 @@ try: import click -except ImportError: +except (ImportError, ModuleNotFoundError): + click = None raise ImportError( "spyglass.cli.cli requires the 'click' package. " "You can install it with 'pip install click'." diff --git a/src/spyglass/common/common_dandi.py b/src/spyglass/common/common_dandi.py index 8264de4cb..27d340766 100644 --- a/src/spyglass/common/common_dandi.py +++ b/src/spyglass/common/common_dandi.py @@ -21,7 +21,7 @@ from dandi.organize import OrganizeInvalid from dandi.validate_types import Severity -except ImportError as e: +except (ImportError, ModuleNotFoundError) as e: ( dandi.download, dandi.organize, diff --git a/src/spyglass/common/common_filter.py b/src/spyglass/common/common_filter.py index 59870f266..2726806e0 100644 --- a/src/spyglass/common/common_filter.py +++ b/src/spyglass/common/common_filter.py @@ -20,7 +20,7 @@ def _import_ghostipy(): import ghostipy as gsp return gsp - except ImportError as e: + except (ImportError, ModuleNotFoundError) as e: raise ImportError( "You must install ghostipy to use filtering methods. Please note " "that to install ghostipy on an Mac M1, you must first install " diff --git a/src/spyglass/common/common_position.py b/src/spyglass/common/common_position.py index f94cfff67..25ed5efeb 100644 --- a/src/spyglass/common/common_position.py +++ b/src/spyglass/common/common_position.py @@ -30,7 +30,7 @@ try: from position_tools import get_centroid -except ImportError: +except (ImportError, ModuleNotFoundError): logger.warning("Please update position_tools to >= 0.1.0") from position_tools import get_centriod as get_centroid diff --git a/src/spyglass/decoding/v0/clusterless.py b/src/spyglass/decoding/v0/clusterless.py index ee44075ee..69f8b1c62 100644 --- a/src/spyglass/decoding/v0/clusterless.py +++ b/src/spyglass/decoding/v0/clusterless.py @@ -36,8 +36,16 @@ from replay_trajectory_classification.initial_conditions import ( UniformInitialConditions, ) -except ImportError as e: +except (ImportError, ModuleNotFoundError) as e: + ( + _DEFAULT_CONTINUOUS_TRANSITIONS, + _DEFAULT_ENVIRONMENT, + _DEFAULT_SORTED_SPIKES_MODEL_KWARGS, + DiagonalDiscrete, + UniformInitialConditions, + ) = [None] * 5 logger.warning(e) + from tqdm.auto import tqdm from spyglass.common.common_behav import ( diff --git a/src/spyglass/decoding/v0/core.py b/src/spyglass/decoding/v0/core.py index 3df82f318..42cbb59d4 100644 --- a/src/spyglass/decoding/v0/core.py +++ b/src/spyglass/decoding/v0/core.py @@ -12,7 +12,7 @@ from replay_trajectory_classification.observation_model import ( ObservationModel, ) -except ImportError as e: +except (ImportError, ModuleNotFoundError) as e: RandomWalk, Uniform, Environment, ObservationModel = None, None, None, None logger.warning(e) diff --git a/src/spyglass/decoding/v0/dj_decoder_conversion.py b/src/spyglass/decoding/v0/dj_decoder_conversion.py index 1cf6d30c4..af03f541e 100644 --- a/src/spyglass/decoding/v0/dj_decoder_conversion.py +++ b/src/spyglass/decoding/v0/dj_decoder_conversion.py @@ -25,7 +25,7 @@ from replay_trajectory_classification.observation_model import ( ObservationModel, ) -except ImportError as e: +except (ImportError, ModuleNotFoundError) as e: ( Identity, RandomWalk, diff --git a/src/spyglass/decoding/v0/sorted_spikes.py b/src/spyglass/decoding/v0/sorted_spikes.py index 8a7013564..36f171219 100644 --- a/src/spyglass/decoding/v0/sorted_spikes.py +++ b/src/spyglass/decoding/v0/sorted_spikes.py @@ -28,8 +28,16 @@ from replay_trajectory_classification.initial_conditions import ( UniformInitialConditions, ) -except ImportError as e: +except (ImportError, ModuleNotFoundError) as e: + ( + _DEFAULT_CONTINUOUS_TRANSITIONS, + _DEFAULT_ENVIRONMENT, + _DEFAULT_SORTED_SPIKES_MODEL_KWARGS, + DiagonalDiscrete, + UniformInitialConditions, + ) = [None] * 5 logger.warning(e) + from spyglass.common.common_behav import ( convert_epoch_interval_name_to_position_interval_name, ) diff --git a/src/spyglass/position/v1/position_dlc_training.py b/src/spyglass/position/v1/position_dlc_training.py index 7876754f5..94548c6b1 100644 --- a/src/spyglass/position/v1/position_dlc_training.py +++ b/src/spyglass/position/v1/position_dlc_training.py @@ -123,7 +123,7 @@ def _logged_make(self, key): try: from deeplabcut.utils.auxiliaryfunctions import get_model_folder - except ImportError: + except (ImportError, ModuleNotFoundError): from deeplabcut.utils.auxiliaryfunctions import ( GetModelFolder as get_model_folder, ) diff --git a/src/spyglass/utils/dj_mixin.py b/src/spyglass/utils/dj_mixin.py index 1db44078a..ed08d6212 100644 --- a/src/spyglass/utils/dj_mixin.py +++ b/src/spyglass/utils/dj_mixin.py @@ -29,7 +29,7 @@ try: import pynapple # noqa F401 -except ImportError: +except (ImportError, ModuleNotFoundError): pynapple = None EXPORT_ENV_VAR = "SPYGLASS_EXPORT_ID" From 31ff1cc774880926b3efbe789a5249c6504e7666 Mon Sep 17 00:00:00 2001 From: Samuel Bray Date: Tue, 9 Jul 2024 15:32:46 -0700 Subject: [PATCH 17/94] Merge table efficiency (#1017) * improve efficiency of Merge.fetch_nwb * make fetch_spike_data a single fetch_nwb call * allow Merge.fetch_nwb for entries with multiple sources at once * update changelog * Apply suggestions from code review Co-authored-by: Chris Broz --------- Co-authored-by: Chris Broz --- CHANGELOG.md | 1 + .../spikesorting/analysis/v1/group.py | 5 +-- src/spyglass/utils/dj_merge_tables.py | 32 +++++++++++++++---- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4a84784c..dadf8a2b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ PositionGroup.alter() - `cautious_delete` now checks `IntervalList` and externals tables. #1002 - Allow mixin tables with parallelization in `make` to run populate with `processes > 1` #1001 +- Speed up fetch_nwb calls through merge tables #1017 - Allow `ModuleNotFoundError` or `ImportError` for optional dependencies #1023 ### Pipelines diff --git a/src/spyglass/spikesorting/analysis/v1/group.py b/src/spyglass/spikesorting/analysis/v1/group.py index ab824bebe..d6e096f3a 100644 --- a/src/spyglass/spikesorting/analysis/v1/group.py +++ b/src/spyglass/spikesorting/analysis/v1/group.py @@ -150,8 +150,9 @@ def fetch_spike_data( # get the spike times for each merge_id spike_times = [] - for merge_id in merge_ids: - nwb_file = SpikeSortingOutput().fetch_nwb({"merge_id": merge_id})[0] + merge_keys = [dict(merge_id=merge_id) for merge_id in merge_ids] + nwb_file_list = (SpikeSortingOutput & merge_keys).fetch_nwb() + for nwb_file in nwb_file_list: nwb_field_name = ( "object_id" if "object_id" in nwb_file diff --git a/src/spyglass/utils/dj_merge_tables.py b/src/spyglass/utils/dj_merge_tables.py index 474ff91c8..d17c087bd 100644 --- a/src/spyglass/utils/dj_merge_tables.py +++ b/src/spyglass/utils/dj_merge_tables.py @@ -374,7 +374,8 @@ def _ensure_dependencies_loaded(cls) -> None: Otherwise parts returns none """ - dj.conn.connection.dependencies.load() + if not dj.conn.connection.dependencies._loaded: + dj.conn.connection.dependencies.load() def insert(self, rows: list, **kwargs): """Merges table specific insert, ensuring data exists in part parents. @@ -520,12 +521,26 @@ def fetch_nwb( Restriction to apply to parents before running fetch. Default True. multi_source: bool Return from multiple parents. Default False. + + Notes + ----- + Nwb files not strictly returned in same order as self """ if isinstance(self, dict): raise ValueError("Try replacing Merge.method with Merge().method") restriction = restriction or self.restriction or True - - return self.merge_restrict_class(restriction).fetch_nwb() + sources = set((self & restriction).fetch(self._reserved_sk)) + nwb_list = [] + for source in sources: + source_restr = ( + self & {self._reserved_sk: source} & restriction + ).fetch("KEY") + nwb_list.extend( + self.merge_restrict_class( + source_restr, permit_multiple_rows=True + ).fetch_nwb() + ) + return nwb_list @classmethod def merge_get_part( @@ -713,17 +728,20 @@ def merge_get_parent_class(self, source: str) -> dj.Table: ) return ret - def merge_restrict_class(self, key: dict) -> dj.Table: + def merge_restrict_class( + self, key: dict, permit_multiple_rows: bool = False + ) -> dj.Table: """Returns native parent class, restricted with key.""" - parent_key = self.merge_get_parent(key).fetch("KEY", as_dict=True) + parent = self.merge_get_parent(key) + parent_key = parent.fetch("KEY", as_dict=True) - if len(parent_key) > 1: + if not permit_multiple_rows and len(parent_key) > 1: raise ValueError( f"Ambiguous entry. Data has mult rows in parent:\n\tData:{key}" + f"\n\t{parent_key}" ) - parent_class = self.merge_get_parent_class(key) + parent_class = self.merge_get_parent_class(parent) return parent_class & parent_key @classmethod From d0ffae45de5fb71d7c6e251d8fae11501839226a Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Wed, 10 Jul 2024 08:11:36 -0700 Subject: [PATCH 18/94] Disable AnalysisFile logging (#1024) * Disable logging * Disable stack inspection --- CHANGELOG.md | 1 + src/spyglass/common/common_nwbfile.py | 16 ++++++++++++---- src/spyglass/utils/dj_helper_fn.py | 9 +++++---- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dadf8a2b5..0aa67fa2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ PositionGroup.alter() - Improve logging clarity in `populate_all_common` #870 - `PositionIntervalMap` now inserts null entries for missing intervals #870 - `AnalysisFileLog` now truncates table names that exceed field length #1021 + - Disable logging with `AnalysisFileLog` #1024 - Decoding: - Default values for classes on `ImportError` #966 - Add option to upsample data rate in `PositionGroup` #1008 diff --git a/src/spyglass/common/common_nwbfile.py b/src/spyglass/common/common_nwbfile.py index aa33b51e1..6a9316a88 100644 --- a/src/spyglass/common/common_nwbfile.py +++ b/src/spyglass/common/common_nwbfile.py @@ -188,7 +188,7 @@ def create(self, nwb_file_name): The name of the new NWB file. """ # To allow some times to occur before create - creation_time = self._creation_times.pop("pre_create_time", time()) + # creation_time = self._creation_times.pop("pre_create_time", time()) nwb_file_abspath = Nwbfile.get_abs_path(nwb_file_name) alter_source_script = False @@ -232,7 +232,7 @@ def create(self, nwb_file_name): permissions = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH os.chmod(analysis_file_abs_path, permissions) - self._creation_times[analysis_file_name] = creation_time + # self._creation_times[analysis_file_name] = creation_time return analysis_file_name @@ -685,7 +685,11 @@ def nightly_cleanup(): # during times when no other transactions are in progress. AnalysisNwbfile.cleanup(True) - def log(self, analysis_file_name, table=None): + def log(self, *args, **kwargs): + """Null log method. Revert to _disabled_log to turn back on.""" + logger.debug("Logging disabled.") + + def _disabled_log(self, analysis_file_name, table=None): """Passthrough to the AnalysisNwbfileLog table. Avoid new imports.""" if isinstance(analysis_file_name, dict): analysis_file_name = analysis_file_name["analysis_file_name"] @@ -699,7 +703,11 @@ def log(self, analysis_file_name, table=None): table=table, ) - def increment_access(self, keys, table=None): + def increment_access(self, *args, **kwargs): + """Null method. Revert to _disabled_increment_access to turn back on.""" + logger.debug("Incrementing access disabled.") + + def _disabled_increment_access(self, keys, table=None): """Passthrough to the AnalysisNwbfileLog table. Avoid new imports.""" if not isinstance(keys, list): key = [keys] diff --git a/src/spyglass/utils/dj_helper_fn.py b/src/spyglass/utils/dj_helper_fn.py index da7d30a3b..d9465fffa 100644 --- a/src/spyglass/utils/dj_helper_fn.py +++ b/src/spyglass/utils/dj_helper_fn.py @@ -204,10 +204,11 @@ def get_nwb_table(query_expression, tbl, attr_name, *attrs, **kwargs): query_expression * tbl.proj(nwb2load_filepath=attr_name) ).fetch(file_name_str) - if which == "analysis": # log access of analysis files to log table - AnalysisNwbfile().increment_access( - nwb_files, table=get_fetching_table_from_stack(inspect.stack()) - ) + # Disabled #1024 + # if which == "analysis": # log access of analysis files to log table + # AnalysisNwbfile().increment_access( + # nwb_files, table=get_fetching_table_from_stack(inspect.stack()) + # ) return nwb_files, file_path_fn From 2c7c98854a7449659d7015fb19c84fd1ac470ce4 Mon Sep 17 00:00:00 2001 From: Samuel Bray Date: Mon, 15 Jul 2024 08:13:17 -0700 Subject: [PATCH 19/94] Group integrity (#1026) * restrict populate calls to sort interval * prevent adding new pieces to existing groups in create functions * update changelog * fix lint * fix lint version --- CHANGELOG.md | 1 + src/spyglass/decoding/v1/core.py | 5 +++++ src/spyglass/spikesorting/analysis/v1/group.py | 6 ++++++ src/spyglass/spikesorting/v0/spikesorting_populator.py | 6 +++--- src/spyglass/spikesorting/v1/recording.py | 2 +- 5 files changed, 16 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0aa67fa2c..4f987bf95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ PositionGroup.alter() `processes > 1` #1001 - Speed up fetch_nwb calls through merge tables #1017 - Allow `ModuleNotFoundError` or `ImportError` for optional dependencies #1023 +- Ensure integrity of group tables #1026 ### Pipelines diff --git a/src/spyglass/decoding/v1/core.py b/src/spyglass/decoding/v1/core.py index d7ed2e89c..8771cc2d4 100644 --- a/src/spyglass/decoding/v1/core.py +++ b/src/spyglass/decoding/v1/core.py @@ -115,6 +115,11 @@ def create_group( "nwb_file_name": nwb_file_name, "position_group_name": group_name, } + if self & group_key: + raise ValueError( + f"Group {nwb_file_name}: {position_group_name} already exists", + "please delete the group before creating a new one", + ) self.insert1( { **group_key, diff --git a/src/spyglass/spikesorting/analysis/v1/group.py b/src/spyglass/spikesorting/analysis/v1/group.py index d6e096f3a..d47b3c980 100644 --- a/src/spyglass/spikesorting/analysis/v1/group.py +++ b/src/spyglass/spikesorting/analysis/v1/group.py @@ -69,6 +69,12 @@ def create_group( "nwb_file_name": nwb_file_name, "unit_filter_params_name": unit_filter_params_name, } + if self & group_key: + raise ValueError( + f"Group {nwb_file_name}: {group_name} already exists", + "please delete the group before creating a new one", + ) + parts_insert = [{**key, **group_key} for key in keys] self.insert1( diff --git a/src/spyglass/spikesorting/v0/spikesorting_populator.py b/src/spyglass/spikesorting/v0/spikesorting_populator.py index df8926a8c..519c8ad08 100644 --- a/src/spyglass/spikesorting/v0/spikesorting_populator.py +++ b/src/spyglass/spikesorting/v0/spikesorting_populator.py @@ -194,16 +194,16 @@ def spikesorting_pipeline_populator( ) SpikeSortingRecordingSelection.insert1(ssr_key, skip_duplicates=True) - SpikeSortingRecording.populate(interval_dict) + SpikeSortingRecording.populate(sort_dict) # Artifact detection logger.info("Running artifact detection") artifact_keys = [ {**k, "artifact_params_name": artifact_parameters} - for k in (SpikeSortingRecordingSelection() & interval_dict).fetch("KEY") + for k in (SpikeSortingRecordingSelection() & sort_dict).fetch("KEY") ] ArtifactDetectionSelection().insert(artifact_keys, skip_duplicates=True) - ArtifactDetection.populate(interval_dict) + ArtifactDetection.populate(sort_dict) # Spike sorting logger.info("Running spike sorting") diff --git a/src/spyglass/spikesorting/v1/recording.py b/src/spyglass/spikesorting/v1/recording.py index 5fa069a28..a9873afe0 100644 --- a/src/spyglass/spikesorting/v1/recording.py +++ b/src/spyglass/spikesorting/v1/recording.py @@ -73,7 +73,7 @@ def set_group_by_shank( Optional. If True, no sort groups are defined for unitrodes. """ # delete any current groups - # (SortGroup & {"nwb_file_name": nwb_file_name}).delete() + (SortGroup & {"nwb_file_name": nwb_file_name}).delete() # get the electrodes from this NWB file electrodes = ( Electrode() From 734e3aa9b2500a9f7379f80bb8a626114bfdebbf Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Mon, 15 Jul 2024 15:09:03 -0700 Subject: [PATCH 20/94] Bugfixes from 870 (#1034) * Bugfixes from 870 * Update changelog * Declare y_eq0 var. Use over --- CHANGELOG.md | 2 + src/spyglass/position/v1/dlc_utils.py | 54 ++++++++++++--------------- 2 files changed, 26 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f987bf95..7a003d5ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,6 +63,8 @@ PositionGroup.alter() - Replace `OutputLogger` context manager with decorator #870 - Rename `check_videofile` -> `find_mp4` and `get_video_path` -> `get_video_info` to reflect actual use #870 + - Fix `red_led_bisector` `np.nan` handling issue from #870. Fixed in #1034 + - Fix `one_pt_centoid` `np.nan` handling issue from #870. Fixed in #1034 - Spikesorting - Allow user to set smoothing timescale in `SortedSpikesGroup.get_firing_rate` #994 diff --git a/src/spyglass/position/v1/dlc_utils.py b/src/spyglass/position/v1/dlc_utils.py index 1523e01b4..f8a911148 100644 --- a/src/spyglass/position/v1/dlc_utils.py +++ b/src/spyglass/position/v1/dlc_utils.py @@ -714,30 +714,27 @@ def red_led_bisector_orientation(pos_df: pd.DataFrame, **params): LED2 = params.pop("led2", None) LED3 = params.pop("led3", None) - x_vec = pos_df[[LED1, LED2]].diff(axis=1).iloc[:, 0] - y_vec = pos_df[[LED1, LED2]].diff(axis=1).iloc[:, 1] - - y_is_zero = y_vec.eq(0) - perp_direction = pos_df[[LED3]].diff(axis=1) - - # Handling the special case where y_vec is zero all Ys are the same - special_case = ( - y_is_zero - & (pos_df[LED3]["y"] == pos_df[LED1]["y"]) - & (pos_df[LED3]["y"] == pos_df[LED2]["y"]) - ) - if special_case.any(): + orient = np.full(len(pos_df), np.nan) # Initialize with NaNs + x_vec = pos_df[LED1]["x"] - pos_df[LED2]["x"] + y_vec = pos_df[LED1]["y"] - pos_df[LED2]["y"] + y_eq0 = np.isclose(y_vec, 0) + + # when y_vec is zero, 1&2 are equal. Compare to 3, determine if up or down + orient[y_eq0 & pos_df[LED3]["y"].gt(pos_df[LED1]["y"])] = np.pi / 2 + orient[y_eq0 & pos_df[LED3]["y"].lt(pos_df[LED1]["y"])] = -np.pi / 2 + + # Handling error case where y_vec is zero and all Ys are the same + y_1, y_2, y_3 = pos_df[LED1]["y"], pos_df[LED2]["y"], pos_df[LED3]["y"] + if np.any(y_eq0 & np.isclose(y_1, y_2) & np.isclose(y_2, y_3)): raise Exception("Cannot determine head direction from bisector") - orientation = np.zeros(len(pos_df)) - orientation[y_is_zero & perp_direction.iloc[:, 0].gt(0)] = np.pi / 2 - orientation[y_is_zero & perp_direction.iloc[:, 0].lt(0)] = -np.pi / 2 - - orientation[~y_is_zero & ~x_vec.eq(0)] = np.arctan2( - y_vec[~y_is_zero], x_vec[~x_vec.eq(0)] - ) + # General case where y_vec is not zero. Use arctan2 to determine orientation + length = np.sqrt(x_vec**2 + y_vec**2) + norm_x = (-y_vec / length)[~y_eq0] + norm_y = (x_vec / length)[~y_eq0] + orient[~y_eq0] = np.arctan2(norm_y, norm_x) - return orientation + return orient # Add new functions for orientation calculation here @@ -834,7 +831,8 @@ def calc_centroid( if isinstance(mask, list): mask = [reduce(np.logical_and, m) for m in mask] - if points is not None: # Check that combinations of points close enough + # Check that combinations of points close enough + if points is not None and len(points) > 1: for pair in combinations(points, 2): mask = (*mask, ~self.too_sep(pair[0], pair[1])) @@ -846,10 +844,7 @@ def calc_centroid( if replace: self.centroid[mask] = np.nan return - if len(points) == 1: # only one point - self.centroid[mask] = self.coords[points[0]][mask] - return - elif len(points) == 3: + if len(points) == 3: self.coords["midpoint"] = ( self.coords[points[0]] + self.coords[points[1]] ) / 2 @@ -867,10 +862,9 @@ def too_sep(self, point1, point2): def get_1pt_centroid(self): """Passthrough. If point is NaN, then centroid is NaN.""" PT1 = self.points_dict.get("point1", None) - self.calc_centroid( - mask=(~self.nans[PT1],), - points=[PT1], - ) + mask = ~self.nans[PT1] # For good points, centroid is the point + self.centroid[mask] = self.coords[PT1][mask] + self.centroid[~mask] = np.nan # For bad points, centroid is NaN def get_2pt_centroid(self): self.calc_centroid( # Good points From f6f76a79a160c62da1e46724851208a84d0d51a7 Mon Sep 17 00:00:00 2001 From: Samuel Bray Date: Tue, 30 Jul 2024 08:10:24 -0700 Subject: [PATCH 21/94] Spike Unit Annotation (#1027) * add option to get unit_id from fetch_spike_data * linting * Move annotations into a part table * lint * lint * lint * fix key bug * add spiksorting analysis tutorial * update changelog * fix spelling * Apply suggestions from code review Co-authored-by: Chris Broz * update doc references * change unit identification to dictionary * fix function doc * consolidate spike obj finding code * fix spelling --------- Co-authored-by: Eric Denovellis Co-authored-by: Chris Broz --- CHANGELOG.md | 1 + docs/mkdocs.yml | 1 + notebooks/11_Spike_Sorting_Analysis.ipynb | 974 ++++++++++++++++++ notebooks/README.md | 3 + .../py_scripts/11_Spike_Sorting_Analysis.py | 192 ++++ src/spyglass/decoding/v1/sorted_spikes.py | 14 +- .../spikesorting/analysis/v1/group.py | 41 +- .../analysis/v1/unit_annotation.py | 139 +++ src/spyglass/utils/dj_merge_tables.py | 8 + 9 files changed, 1362 insertions(+), 11 deletions(-) create mode 100644 notebooks/11_Spike_Sorting_Analysis.ipynb create mode 100644 notebooks/py_scripts/11_Spike_Sorting_Analysis.py create mode 100644 src/spyglass/spikesorting/analysis/v1/unit_annotation.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a003d5ad..66e0a36d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,6 +73,7 @@ PositionGroup.alter() - Fix bug in identification of artifact samples to be zeroed out in `spikesorting.v1.SpikeSorting` #1009 - Remove deprecated dependencies on kachery_client #1014 + - Add `UnitAnnotation` table and naming convention for units #1027 ## [0.5.2] (April 22, 2024) diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index c6394657e..88734f3a0 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -55,6 +55,7 @@ nav: - Spikes: - Spike Sorting V0: notebooks/10_Spike_SortingV0.ipynb - Spike Sorting V1: notebooks/10_Spike_SortingV1.ipynb + - Spike Sorting Analysis: notebooks/11_Spike_Sorting_Analysis.ipynb - Position: - Position Trodes: notebooks/20_Position_Trodes.ipynb - DLC Models: notebooks/21_DLC.ipynb diff --git a/notebooks/11_Spike_Sorting_Analysis.ipynb b/notebooks/11_Spike_Sorting_Analysis.ipynb new file mode 100644 index 000000000..3e8ea436b --- /dev/null +++ b/notebooks/11_Spike_Sorting_Analysis.ipynb @@ -0,0 +1,974 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Spike Sorting Analysis\n", + "\n", + "Sorted spike times are a starting point of many analysis pipelines. Spyglass provides\n", + "several tools to aid in organizing spikesorting results and tracking annotations \n", + "across multiple analyses depending on this data.\n", + "\n", + "For practical examples see [Sorted Spikes Decoding](./42_Decoding_SortedSpikes.ipynb)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## SortedSpikesGroup\n", + "\n", + "In practice, downstream analyses of spikesorting will often need to combine results\n", + "from multiple sorts (e.g. across tetrodes groups in a single interval). To make \n", + "this simple with spyglass's relational database, we use the `SortedSpikesGroup` table.\n", + "\n", + "`SortedSpikesGroup` is a child table of `SpikeSortingOutput` in the spikesorting pipeline.\n", + "It allows us to group the spikesorting results from multiple sources into a single\n", + "entry for downstream reference, and provides tools for easily\n", + "accessing the compiled data. Here we will group together the spiking of multiple \n", + "tetrode groups.\n", + "\n", + "\n", + "This table allows us filter units by their annotation labels from curation (e.g only\n", + "include units labeled \"good\", exclude units labeled \"noise\") by defining parameters\n", + "from `UnitSelectionParams`. When accessing data through `SortedSpikesGroup` the table\n", + "will include only units with at least one label in `include_labels` and no labels in \n", + "`exclude_labels`. We can look at those here:\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'unit_filter_params_name': 'default_exclusion', 'include_labels': [], 'exclude_labels': ['noise', 'mua']}\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "

unit_filter_params_name

\n", + " \n", + "
\n", + "

include_labels

\n", + " \n", + "
\n", + "

exclude_labels

\n", + " \n", + "
all_units=BLOB==BLOB=
default_exclusion=BLOB==BLOB=
exclude_noise=BLOB==BLOB=
MS2220180629=BLOB==BLOB=
\n", + " \n", + "

Total: 4

\n", + " " + ], + "text/plain": [ + "*unit_filter_p include_la exclude_la\n", + "+------------+ +--------+ +--------+\n", + "all_units =BLOB= =BLOB= \n", + "default_exclus =BLOB= =BLOB= \n", + "exclude_noise =BLOB= =BLOB= \n", + "MS2220180629 =BLOB= =BLOB= \n", + " (Total: 4)" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from spyglass.spikesorting.analysis.v1.group import UnitSelectionParams\n", + "\n", + "UnitSelectionParams().insert_default()\n", + "\n", + "# look at the filter set we'll use here\n", + "unit_filter_params_name = \"default_exclusion\"\n", + "print(\n", + " (\n", + " UnitSelectionParams()\n", + " & {\"unit_filter_params_name\": unit_filter_params_name}\n", + " ).fetch1()\n", + ")\n", + "# look at full table\n", + "UnitSelectionParams()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We then define the set of curated sorting results to include in the group\n", + "\n", + "Finding the merge id's corresponding to an interpretable restriction such as `merge_id` or `interval_list` can require several join steps with upstream tables. To simplify this process we can use the included helper function `SpikeSortingOutput().get_restricted_merge_ids()` to perform the necessary joins and return the matching merge id's" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[12:11:56][WARNING] Spyglass: V0 requires artifact restrict. Ignoring \"restrict_by_artifact\" flag.\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "

merge_id

\n", + " \n", + "
\n", + "

sorting_id

\n", + " \n", + "
\n", + "

curation_id

\n", + " \n", + "
143dff79-3779-c0d2-46fe-7c5040404219a4b5a94d-ba41-4634-92d0-1d31c9daa9131
2249c566-cc17-bdda-4074-d772ee40b772874775be-df0f-4850-8f88-59ba1bbead891
4a191cc4-945b-3ad8-592a-a95e874b2507a4b5a94d-ba41-4634-92d0-1d31c9daa9131
75286bf3-f876-4550-f235-321f2a7badef642242ff-5f0e-45a2-bcc1-ca681f37b4a31
76ec4894-300d-4ed3-ce26-0327e7ed3345642242ff-5f0e-45a2-bcc1-ca681f37b4a31
a900c1c8-909d-e583-c377-e98c4f0deebf874775be-df0f-4850-8f88-59ba1bbead891
\n", + " \n", + "

Total: 6

\n", + " " + ], + "text/plain": [ + "*merge_id sorting_id curation_id \n", + "+------------+ +------------+ +------------+\n", + "143dff79-3779- a4b5a94d-ba41- 1 \n", + "2249c566-cc17- 874775be-df0f- 1 \n", + "4a191cc4-945b- a4b5a94d-ba41- 1 \n", + "75286bf3-f876- 642242ff-5f0e- 1 \n", + "76ec4894-300d- 642242ff-5f0e- 1 \n", + "a900c1c8-909d- 874775be-df0f- 1 \n", + " (Total: 6)" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from spyglass.spikesorting.spikesorting_merge import SpikeSortingOutput\n", + "\n", + "nwb_file_name = \"mediumnwb20230802_.nwb\"\n", + "\n", + "sorter_keys = {\n", + " \"nwb_file_name\": nwb_file_name,\n", + " \"sorter\": \"mountainsort4\",\n", + " \"curation_id\": 1,\n", + "}\n", + "\n", + "# get the merge_ids for the selected sorting\n", + "spikesorting_merge_ids = SpikeSortingOutput().get_restricted_merge_ids(\n", + " sorter_keys, restrict_by_artifact=True\n", + ")\n", + "\n", + "keys = [{\"merge_id\": merge_id} for merge_id in spikesorting_merge_ids]\n", + "(SpikeSortingOutput.CurationV1 & keys)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can now combine this information to make a spike sorting group" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "
\n", + "

nwb_file_name

\n", + " name of the NWB file\n", + "
\n", + "

unit_filter_params_name

\n", + " \n", + "
\n", + "

sorted_spikes_group_name

\n", + " \n", + "
mediumnwb20230802_.nwbdefault_exclusiondemo_group
\n", + " \n", + "

Total: 1

\n", + " " + ], + "text/plain": [ + "*nwb_file_name *unit_filter_p *sorted_spikes\n", + "+------------+ +------------+ +------------+\n", + "mediumnwb20230 default_exclus demo_group \n", + " (Total: 1)" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from spyglass.spikesorting.analysis.v1.group import SortedSpikesGroup\n", + "\n", + "# create a new sorted spikes group\n", + "unit_filter_params_name = \"default_exclusion\"\n", + "SortedSpikesGroup().create_group(\n", + " group_name=\"demo_group\",\n", + " nwb_file_name=nwb_file_name,\n", + " keys=[\n", + " {\"spikesorting_merge_id\": merge_id}\n", + " for merge_id in spikesorting_merge_ids\n", + " ],\n", + " unit_filter_params_name=unit_filter_params_name,\n", + ")\n", + "# check the new group\n", + "group_key = {\n", + " \"nwb_file_name\": nwb_file_name,\n", + " \"sorted_spikes_group_name\": \"demo_group\",\n", + "}\n", + "SortedSpikesGroup & group_key" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "

nwb_file_name

\n", + " name of the NWB file\n", + "
\n", + "

unit_filter_params_name

\n", + " \n", + "
\n", + "

sorted_spikes_group_name

\n", + " \n", + "
\n", + "

spikesorting_merge_id

\n", + " \n", + "
mediumnwb20230802_.nwbdefault_exclusiondemo_group143dff79-3779-c0d2-46fe-7c5040404219
mediumnwb20230802_.nwbdefault_exclusiondemo_group2249c566-cc17-bdda-4074-d772ee40b772
mediumnwb20230802_.nwbdefault_exclusiondemo_group4a191cc4-945b-3ad8-592a-a95e874b2507
mediumnwb20230802_.nwbdefault_exclusiondemo_group75286bf3-f876-4550-f235-321f2a7badef
mediumnwb20230802_.nwbdefault_exclusiondemo_group76ec4894-300d-4ed3-ce26-0327e7ed3345
mediumnwb20230802_.nwbdefault_exclusiondemo_groupa900c1c8-909d-e583-c377-e98c4f0deebf
\n", + " \n", + "

Total: 6

\n", + " " + ], + "text/plain": [ + "*nwb_file_name *unit_filter_p *sorted_spikes *spikesorting_\n", + "+------------+ +------------+ +------------+ +------------+\n", + "mediumnwb20230 default_exclus demo_group 143dff79-3779-\n", + "mediumnwb20230 default_exclus demo_group 2249c566-cc17-\n", + "mediumnwb20230 default_exclus demo_group 4a191cc4-945b-\n", + "mediumnwb20230 default_exclus demo_group 75286bf3-f876-\n", + "mediumnwb20230 default_exclus demo_group 76ec4894-300d-\n", + "mediumnwb20230 default_exclus demo_group a900c1c8-909d-\n", + " (Total: 6)" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "SortedSpikesGroup.Units & group_key" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can access the spikesorting results for this data using `SortedSpikesGroup.fetch_spike_data()`\n" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[array([1.62593570e+09, 1.62593570e+09, 1.62593570e+09, ...,\n", + " 1.62593718e+09, 1.62593718e+09, 1.62593718e+09]),\n", + " array([1.62593570e+09, 1.62593570e+09, 1.62593570e+09, ...,\n", + " 1.62593718e+09, 1.62593718e+09, 1.62593718e+09]),\n", + " array([1.62593570e+09, 1.62593570e+09, 1.62593570e+09, ...,\n", + " 1.62593718e+09, 1.62593718e+09, 1.62593718e+09]),\n", + " array([1.62593570e+09, 1.62593570e+09, 1.62593570e+09, ...,\n", + " 1.62593718e+09, 1.62593718e+09, 1.62593718e+09]),\n", + " array([1.62593570e+09, 1.62593570e+09, 1.62593570e+09, ...,\n", + " 1.62593718e+09, 1.62593718e+09, 1.62593718e+09]),\n", + " array([1.62593570e+09, 1.62593570e+09, 1.62593570e+09, ...,\n", + " 1.62593718e+09, 1.62593718e+09, 1.62593718e+09]),\n", + " array([1.62593570e+09, 1.62593570e+09, 1.62593570e+09, ...,\n", + " 1.62593717e+09, 1.62593717e+09, 1.62593718e+09]),\n", + " array([1.62593570e+09, 1.62593570e+09, 1.62593572e+09, ...,\n", + " 1.62593717e+09, 1.62593717e+09, 1.62593717e+09]),\n", + " array([1.62593571e+09, 1.62593571e+09, 1.62593571e+09, ...,\n", + " 1.62593717e+09, 1.62593717e+09, 1.62593717e+09]),\n", + " array([1.62593571e+09, 1.62593572e+09, 1.62593574e+09, ...,\n", + " 1.62593714e+09, 1.62593714e+09, 1.62593715e+09]),\n", + " array([1.62593570e+09, 1.62593570e+09, 1.62593570e+09, ...,\n", + " 1.62593718e+09, 1.62593718e+09, 1.62593718e+09]),\n", + " array([1.62593570e+09, 1.62593570e+09, 1.62593570e+09, ...,\n", + " 1.62593718e+09, 1.62593718e+09, 1.62593718e+09]),\n", + " array([1.62593570e+09, 1.62593570e+09, 1.62593570e+09, ...,\n", + " 1.62593718e+09, 1.62593718e+09, 1.62593718e+09]),\n", + " array([1.62593570e+09, 1.62593570e+09, 1.62593570e+09, ...,\n", + " 1.62593718e+09, 1.62593718e+09, 1.62593718e+09]),\n", + " array([1.62593570e+09, 1.62593570e+09, 1.62593570e+09, ...,\n", + " 1.62593718e+09, 1.62593718e+09, 1.62593718e+09]),\n", + " array([1.62593570e+09, 1.62593570e+09, 1.62593570e+09, ...,\n", + " 1.62593718e+09, 1.62593718e+09, 1.62593718e+09]),\n", + " array([1.62593570e+09, 1.62593570e+09, 1.62593570e+09, ...,\n", + " 1.62593718e+09, 1.62593718e+09, 1.62593718e+09]),\n", + " array([1.62593577e+09, 1.62593577e+09, 1.62593577e+09, 1.62593577e+09,\n", + " 1.62593577e+09, 1.62593577e+09, 1.62593577e+09, 1.62593577e+09,\n", + " 1.62593577e+09, 1.62593577e+09, 1.62593579e+09, 1.62593579e+09,\n", + " 1.62593579e+09, 1.62593579e+09, 1.62593579e+09, 1.62593583e+09,\n", + " 1.62593583e+09, 1.62593583e+09, 1.62593583e+09, 1.62593583e+09,\n", + " 1.62593583e+09, 1.62593583e+09, 1.62593583e+09, 1.62593583e+09,\n", + " 1.62593583e+09, 1.62593583e+09, 1.62593583e+09, 1.62593594e+09,\n", + " 1.62593594e+09, 1.62593594e+09, 1.62593594e+09, 1.62593594e+09,\n", + " 1.62593594e+09, 1.62593594e+09, 1.62593594e+09, 1.62593594e+09,\n", + " 1.62593595e+09, 1.62593595e+09, 1.62593595e+09, 1.62593595e+09,\n", + " 1.62593595e+09, 1.62593595e+09, 1.62593595e+09, 1.62593595e+09,\n", + " 1.62593595e+09, 1.62593595e+09, 1.62593599e+09, 1.62593599e+09,\n", + " 1.62593599e+09, 1.62593599e+09, 1.62593599e+09, 1.62593599e+09,\n", + " 1.62593599e+09, 1.62593599e+09, 1.62593599e+09, 1.62593599e+09,\n", + " 1.62593625e+09, 1.62593625e+09, 1.62593625e+09, 1.62593626e+09,\n", + " 1.62593643e+09, 1.62593643e+09, 1.62593643e+09, 1.62593643e+09,\n", + " 1.62593643e+09, 1.62593643e+09, 1.62593643e+09, 1.62593643e+09,\n", + " 1.62593643e+09, 1.62593643e+09, 1.62593643e+09, 1.62593647e+09,\n", + " 1.62593647e+09, 1.62593651e+09, 1.62593651e+09, 1.62593651e+09,\n", + " 1.62593653e+09, 1.62593653e+09, 1.62593653e+09, 1.62593653e+09,\n", + " 1.62593653e+09, 1.62593654e+09, 1.62593654e+09, 1.62593654e+09,\n", + " 1.62593654e+09, 1.62593654e+09, 1.62593654e+09, 1.62593654e+09,\n", + " 1.62593654e+09, 1.62593654e+09, 1.62593654e+09, 1.62593654e+09,\n", + " 1.62593654e+09, 1.62593654e+09, 1.62593654e+09, 1.62593657e+09,\n", + " 1.62593657e+09, 1.62593657e+09, 1.62593657e+09, 1.62593657e+09,\n", + " 1.62593657e+09, 1.62593657e+09, 1.62593657e+09, 1.62593657e+09,\n", + " 1.62593657e+09, 1.62593657e+09, 1.62593657e+09, 1.62593657e+09,\n", + " 1.62593658e+09, 1.62593658e+09, 1.62593658e+09, 1.62593658e+09,\n", + " 1.62593658e+09, 1.62593658e+09, 1.62593658e+09, 1.62593658e+09,\n", + " 1.62593658e+09, 1.62593658e+09, 1.62593658e+09, 1.62593658e+09,\n", + " 1.62593659e+09, 1.62593659e+09, 1.62593659e+09, 1.62593659e+09,\n", + " 1.62593659e+09, 1.62593659e+09, 1.62593659e+09, 1.62593659e+09,\n", + " 1.62593659e+09, 1.62593659e+09, 1.62593660e+09, 1.62593660e+09,\n", + " 1.62593661e+09, 1.62593661e+09, 1.62593661e+09, 1.62593661e+09,\n", + " 1.62593661e+09, 1.62593661e+09, 1.62593661e+09, 1.62593661e+09,\n", + " 1.62593671e+09, 1.62593674e+09, 1.62593678e+09, 1.62593695e+09,\n", + " 1.62593712e+09, 1.62593712e+09]),\n", + " array([1.62593570e+09, 1.62593570e+09, 1.62593570e+09, ...,\n", + " 1.62593718e+09, 1.62593718e+09, 1.62593718e+09]),\n", + " array([1.62593570e+09, 1.62593570e+09, 1.62593570e+09, ...,\n", + " 1.62593715e+09, 1.62593715e+09, 1.62593715e+09]),\n", + " array([1.62593570e+09, 1.62593572e+09, 1.62593572e+09, ...,\n", + " 1.62593717e+09, 1.62593717e+09, 1.62593717e+09]),\n", + " array([1.62593570e+09, 1.62593570e+09, 1.62593570e+09, ...,\n", + " 1.62593715e+09, 1.62593715e+09, 1.62593716e+09]),\n", + " array([1.62593570e+09, 1.62593570e+09, 1.62593570e+09, ...,\n", + " 1.62593715e+09, 1.62593715e+09, 1.62593715e+09]),\n", + " array([1.62593570e+09, 1.62593570e+09, 1.62593570e+09, ...,\n", + " 1.62593718e+09, 1.62593718e+09, 1.62593718e+09]),\n", + " array([1.62593570e+09, 1.62593570e+09, 1.62593570e+09, ...,\n", + " 1.62593718e+09, 1.62593718e+09, 1.62593718e+09]),\n", + " array([1.62593570e+09, 1.62593570e+09, 1.62593570e+09, ...,\n", + " 1.62593718e+09, 1.62593718e+09, 1.62593718e+09]),\n", + " array([1.62593570e+09, 1.62593570e+09, 1.62593570e+09, ...,\n", + " 1.62593718e+09, 1.62593718e+09, 1.62593718e+09])]" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# get the complete key\n", + "group_key = (SortedSpikesGroup & group_key).fetch1(\"KEY\")\n", + "# get the spike data, returns a list of unit spike times\n", + "SortedSpikesGroup().fetch_spike_data(group_key)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Unit Annotation\n", + "\n", + "Many neuroscience applications are interested in the properties of individual neurons\n", + "or units. For example, one set of custom analysis may classify each unit as a cell type\n", + "based on firing properties, and a second analysis step may want to compare additional\n", + "features based on this classification.\n", + "\n", + "Doing so requires a consistent manner of identifying a unit, and a location to track annotations\n", + "\n", + "Spyglass uses the unit identification system: \n", + "`{\"spikesorting_merge_id\" : merge_id, \"unit_id\" : unit_id}\"`,\n", + "where `unit_id` is the index of a units in the saved nwb file. `fetch_spike_data`\n", + "can return these identifications by setting `return_unit_ids = True`" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'spikesorting_merge_id': UUID('143dff79-3779-c0d2-46fe-7c5040404219'), 'unit_id': 0}\n", + "[1.62593570e+09 1.62593570e+09 1.62593570e+09 ... 1.62593718e+09\n", + " 1.62593718e+09 1.62593718e+09]\n" + ] + } + ], + "source": [ + "spike_times, unit_ids = SortedSpikesGroup().fetch_spike_data(\n", + " group_key, return_unit_ids=True\n", + ")\n", + "print(unit_ids[0])\n", + "print(spike_times[0])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Further analysis may assign annotations to individual units. These can either be a \n", + "string `label` (e.g. \"pyridimal_cell\", \"thirst_sensitive\"), or a float `quantification`\n", + "(e.g. firing_rate, signal_correlation). \n", + "\n", + "The `UnitAnnotation` table can be used to track and cross reference these annotations\n", + "between analysis pipelines. Each unit has a single entry in `UnitAnnotation`, which \n", + "can be connected to multiple entries in the `UnitAnnotation.Annotation` part table.\n", + "\n", + "An `Annotation` entry should include an `annotation` describing the originating analysis,\n", + "along with a `label` and/or `quantification` with the analysis result.\n", + "\n", + "Here, we demonstrate adding quantification and label annotations to the units in \n", + "the spike group we created using the `add_annotation` function." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "

spikesorting_merge_id

\n", + " \n", + "
\n", + "

unit_id

\n", + " \n", + "
\n", + "

annotation

\n", + " the kind of annotation (e.g. a table name, \"cell_type\", \"firing_rate\", etc.)\n", + "
\n", + "

label

\n", + " text labels from analysis\n", + "
\n", + "

quantification

\n", + " quantification label from analysis\n", + "
143dff79-3779-c0d2-46fe-7c50404042190cell_typeinterneuronnan
143dff79-3779-c0d2-46fe-7c50404042190spike_countNone40509.0
143dff79-3779-c0d2-46fe-7c50404042191cell_typeinterneuronnan
143dff79-3779-c0d2-46fe-7c50404042191spike_countNone40181.0
143dff79-3779-c0d2-46fe-7c50404042192cell_typeinterneuronnan
143dff79-3779-c0d2-46fe-7c50404042192spike_countNone18233.0
143dff79-3779-c0d2-46fe-7c50404042193cell_typeinterneuronnan
143dff79-3779-c0d2-46fe-7c50404042193spike_countNone36711.0
2249c566-cc17-bdda-4074-d772ee40b7720cell_typeinterneuronnan
2249c566-cc17-bdda-4074-d772ee40b7720spike_countNone48076.0
2249c566-cc17-bdda-4074-d772ee40b7721cell_typeinterneuronnan
2249c566-cc17-bdda-4074-d772ee40b7721spike_countNone97667.0
\n", + "

...

\n", + "

Total: 54

\n", + " " + ], + "text/plain": [ + "*spikesorting_ *unit_id *annotation label quantification\n", + "+------------+ +---------+ +------------+ +------------+ +------------+\n", + "143dff79-3779- 0 cell_type interneuron nan \n", + "143dff79-3779- 0 spike_count None 40509.0 \n", + "143dff79-3779- 1 cell_type interneuron nan \n", + "143dff79-3779- 1 spike_count None 40181.0 \n", + "143dff79-3779- 2 cell_type interneuron nan \n", + "143dff79-3779- 2 spike_count None 18233.0 \n", + "143dff79-3779- 3 cell_type interneuron nan \n", + "143dff79-3779- 3 spike_count None 36711.0 \n", + "2249c566-cc17- 0 cell_type interneuron nan \n", + "2249c566-cc17- 0 spike_count None 48076.0 \n", + "2249c566-cc17- 1 cell_type interneuron nan \n", + "2249c566-cc17- 1 spike_count None 97667.0 \n", + " ...\n", + " (Total: 54)" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from spyglass.spikesorting.analysis.v1.unit_annotation import UnitAnnotation\n", + "\n", + "for spikes, unit_key in zip(spike_times, unit_ids):\n", + " # add a quantification annotation for the number of spikes\n", + " annotation_key = {\n", + " **unit_key,\n", + " \"annotation\": \"spike_count\",\n", + " \"quantification\": len(spikes),\n", + " }\n", + " UnitAnnotation().add_annotation(annotation_key, skip_duplicates=True)\n", + " # add a label annotation for the unit id\n", + " annotation_key = {\n", + " **unit_key,\n", + " \"annotation\": \"cell_type\",\n", + " \"label\": \"pyridimal\" if len(spikes) < 1000 else \"interneuron\",\n", + " }\n", + " UnitAnnotation().add_annotation(annotation_key, skip_duplicates=True)\n", + "\n", + "annotations = UnitAnnotation().Annotation() & unit_ids\n", + "annotations" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Subsets of the the spikesorting data can then be accessed by calling `fetch_unit_spikes`\n", + "on a restricted instance of the table. This allows the user to perform further analysis\n", + "based on these labels. \n", + "\n", + "*Note:* This function will return the spike times for all units in the restricted table" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'spikesorting_merge_id': UUID('143dff79-3779-c0d2-46fe-7c5040404219'), 'unit_id': 0}\n", + "[1.62593570e+09 1.62593570e+09 1.62593570e+09 ... 1.62593718e+09\n", + " 1.62593718e+09 1.62593718e+09]\n" + ] + } + ], + "source": [ + "# restrict to units from our sorted spikes group\n", + "annotations = UnitAnnotation.Annotation & (SortedSpikesGroup.Units & group_key)\n", + "# restrict to units with more than 3000 spikes\n", + "annotations = annotations & {\"annotation\": \"spike_count\"}\n", + "annotations = annotations & \"quantification > 3000\"\n", + "\n", + "selected_spike_times, selected_unit_ids = annotations.fetch_unit_spikes(\n", + " return_unit_ids=True\n", + ")\n", + "print(selected_unit_ids[0])\n", + "print(selected_spike_times[0])" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "spyglass", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/README.md b/notebooks/README.md index aec471d19..62b136240 100644 --- a/notebooks/README.md +++ b/notebooks/README.md @@ -23,6 +23,9 @@ This series of notebooks covers the process of spike sorting, from automated spike sorting to optional manual curation of the output of the automated sorting. +Spikesorting results from any pipeline can then be organized and tracked using +tools in [Spikesorting Analysis](./11_Spike_Sorting_Agit analysis.ipynb) + ## 2. Position Pipeline This series of notebooks covers tracking the position(s) of the animal. The user diff --git a/notebooks/py_scripts/11_Spike_Sorting_Analysis.py b/notebooks/py_scripts/11_Spike_Sorting_Analysis.py new file mode 100644 index 000000000..1af3b70f9 --- /dev/null +++ b/notebooks/py_scripts/11_Spike_Sorting_Analysis.py @@ -0,0 +1,192 @@ +# --- +# jupyter: +# jupytext: +# text_representation: +# extension: .py +# format_name: light +# format_version: '1.5' +# jupytext_version: 1.15.2 +# kernelspec: +# display_name: spyglass +# language: python +# name: python3 +# --- + +# # Spike Sorting Analysis +# +# Sorted spike times are a starting point of many analysis pipelines. Spyglass provides +# several tools to aid in organizing spikesorting results and tracking annotations +# across multiple analyses depending on this data. +# +# For practical examples see [Sorted Spikes Decoding](./42_Decoding_SortedSpikes.ipynb) + +# ## SortedSpikesGroup +# +# In practice, downstream analyses of spikesorting will often need to combine results +# from multiple sorts (e.g. across tetrodes groups in a single interval). To make +# this simple with spyglass's relational database, we use the `SortedSpikesGroup` table. +# +# `SortedSpikesGroup` is a child table of `SpikeSortingOutput` in the spikesorting pipeline. +# It allows us to group the spikesorting results from multiple sources into a single +# entry for downstream reference, and provides tools for easily +# accessing the compiled data. Here we will group together the spiking of multiple +# tetrode groups. +# +# +# This table allows us filter units by their annotation labels from curation (e.g only +# include units labeled "good", exclude units labeled "noise") by defining parameters +# from `UnitSelectionParams`. When accessing data through `SortedSpikesGroup` the table +# will include only units with at least one label in `include_labels` and no labels in +# `exclude_labels`. We can look at those here: +# + +# + +from spyglass.spikesorting.analysis.v1.group import UnitSelectionParams + +UnitSelectionParams().insert_default() + +# look at the filter set we'll use here +unit_filter_params_name = "default_exclusion" +print( + ( + UnitSelectionParams() + & {"unit_filter_params_name": unit_filter_params_name} + ).fetch1() +) +# look at full table +UnitSelectionParams() +# - + +# We then define the set of curated sorting results to include in the group +# +# Finding the merge id's corresponding to an interpretable restriction such as `merge_id` or `interval_list` can require several join steps with upstream tables. To simplify this process we can use the included helper function `SpikeSortingOutput().get_restricted_merge_ids()` to perform the necessary joins and return the matching merge id's + +# + +from spyglass.spikesorting.spikesorting_merge import SpikeSortingOutput + +nwb_file_name = "mediumnwb20230802_.nwb" + +sorter_keys = { + "nwb_file_name": nwb_file_name, + "sorter": "mountainsort4", + "curation_id": 1, +} + +# get the merge_ids for the selected sorting +spikesorting_merge_ids = SpikeSortingOutput().get_restricted_merge_ids( + sorter_keys, restrict_by_artifact=True +) + +keys = [{"merge_id": merge_id} for merge_id in spikesorting_merge_ids] +(SpikeSortingOutput.CurationV1 & keys) +# - + +# We can now combine this information to make a spike sorting group + +# + +from spyglass.spikesorting.analysis.v1.group import SortedSpikesGroup + +# create a new sorted spikes group +unit_filter_params_name = "default_exclusion" +SortedSpikesGroup().create_group( + group_name="demo_group", + nwb_file_name=nwb_file_name, + keys=[ + {"spikesorting_merge_id": merge_id} + for merge_id in spikesorting_merge_ids + ], + unit_filter_params_name=unit_filter_params_name, +) +# check the new group +group_key = { + "nwb_file_name": nwb_file_name, + "sorted_spikes_group_name": "demo_group", +} +SortedSpikesGroup & group_key +# - + +SortedSpikesGroup.Units & group_key + +# We can access the spikesorting results for this data using `SortedSpikesGroup.fetch_spike_data()` +# + +# get the complete key +group_key = (SortedSpikesGroup & group_key).fetch1("KEY") +# get the spike data, returns a list of unit spike times +SortedSpikesGroup().fetch_spike_data(group_key) + +# ## Unit Annotation +# +# Many neuroscience applications are interested in the properties of individual neurons +# or units. For example, one set of custom analysis may classify each unit as a cell type +# based on firing properties, and a second analysis step may want to compare additional +# features based on this classification. +# +# Doing so requires a consistent manner of identifying a unit, and a location to track annotations +# +# Spyglass uses the unit identification system: +# `{"spikesorting_merge_id" : merge_id, "unit_id" : unit_id}"`, +# where `unit_id` is the index of a units in the saved nwb file. `fetch_spike_data` +# can return these identifications by setting `return_unit_ids = True` + +spike_times, unit_ids = SortedSpikesGroup().fetch_spike_data( + group_key, return_unit_ids=True +) +print(unit_ids[0]) +print(spike_times[0]) + +# Further analysis may assign annotations to individual units. These can either be a +# string `label` (e.g. "pyridimal_cell", "thirst_sensitive"), or a float `quantification` +# (e.g. firing_rate, signal_correlation). +# +# The `UnitAnnotation` table can be used to track and cross reference these annotations +# between analysis pipelines. Each unit has a single entry in `UnitAnnotation`, which +# can be connected to multiple entries in the `UnitAnnotation.Annotation` part table. +# +# An `Annotation` entry should include an `annotation` describing the originating analysis, +# along with a `label` and/or `quantification` with the analysis result. +# +# Here, we demonstrate adding quantification and label annotations to the units in +# the spike group we created using the `add_annotation` function. + +# + +from spyglass.spikesorting.analysis.v1.unit_annotation import UnitAnnotation + +for spikes, unit_key in zip(spike_times, unit_ids): + # add a quantification annotation for the number of spikes + annotation_key = { + **unit_key, + "annotation": "spike_count", + "quantification": len(spikes), + } + UnitAnnotation().add_annotation(annotation_key, skip_duplicates=True) + # add a label annotation for the unit id + annotation_key = { + **unit_key, + "annotation": "cell_type", + "label": "pyridimal" if len(spikes) < 1000 else "interneuron", + } + UnitAnnotation().add_annotation(annotation_key, skip_duplicates=True) + +annotations = UnitAnnotation().Annotation() & unit_ids +annotations +# - + +# Subsets of the the spikesorting data can then be accessed by calling `fetch_unit_spikes` +# on a restricted instance of the table. This allows the user to perform further analysis +# based on these labels. +# +# *Note:* This function will return the spike times for all units in the restricted table + +# + +# restrict to units from our sorted spikes group +annotations = UnitAnnotation.Annotation & (SortedSpikesGroup.Units & group_key) +# restrict to units with more than 3000 spikes +annotations = annotations & {"annotation": "spike_count"} +annotations = annotations & "quantification > 3000" + +selected_spike_times, selected_unit_ids = annotations.fetch_unit_spikes( + return_unit_ids=True +) +print(selected_unit_ids[0]) +print(selected_spike_times[0]) diff --git a/src/spyglass/decoding/v1/sorted_spikes.py b/src/spyglass/decoding/v1/sorted_spikes.py index 339bde314..e4d378cc4 100644 --- a/src/spyglass/decoding/v1/sorted_spikes.py +++ b/src/spyglass/decoding/v1/sorted_spikes.py @@ -11,6 +11,7 @@ import copy import uuid from pathlib import Path +from typing import Optional, Union import datajoint as dj import non_local_detector.analysis as analysis @@ -398,7 +399,9 @@ def fetch_linear_position_info(key): ) @staticmethod - def fetch_spike_data(key, filter_by_interval=True, time_slice=None): + def fetch_spike_data( + key, filter_by_interval=True, time_slice=None, return_unit_ids=False + ) -> Union[list[np.ndarray], Optional[list[dict]]]: """Fetch the spike times for the decoding model Parameters @@ -409,13 +412,18 @@ def fetch_spike_data(key, filter_by_interval=True, time_slice=None): Whether to filter for spike times in the model interval, by default True time_slice : Slice, optional User provided slice of time to restrict spikes to, by default None + return_unit_ids : bool, optional + if True, return the unit_ids along with the spike times, by default False + Unit ids defined as a list of dictionaries with keys 'spikesorting_merge_id' and 'unit_number' Returns ------- list[np.ndarray] List of spike times for each unit in the model's spike group """ - spike_times = SortedSpikesGroup.fetch_spike_data(key) + spike_times, unit_ids = SortedSpikesGroup.fetch_spike_data( + key, return_unit_ids=True + ) if not filter_by_interval: return spike_times @@ -431,6 +439,8 @@ def fetch_spike_data(key, filter_by_interval=True, time_slice=None): ) new_spike_times.append(elec_spike_times[is_in_interval]) + if return_unit_ids: + return new_spike_times, unit_ids return new_spike_times def spike_times_sorted_by_place_field_peak(self, time_slice=None): diff --git a/src/spyglass/spikesorting/analysis/v1/group.py b/src/spyglass/spikesorting/analysis/v1/group.py index d47b3c980..208120cbf 100644 --- a/src/spyglass/spikesorting/analysis/v1/group.py +++ b/src/spyglass/spikesorting/analysis/v1/group.py @@ -1,4 +1,5 @@ from itertools import compress +from typing import Optional, Union import datajoint as dj import numpy as np @@ -122,8 +123,8 @@ def filter_units( @staticmethod def fetch_spike_data( - key: dict, time_slice: list[float] = None - ) -> list[np.ndarray]: + key: dict, time_slice: list[float] = None, return_unit_ids: bool = False + ) -> Union[list[np.ndarray], Optional[list[dict]]]: """fetch spike times for units in the group Parameters @@ -132,6 +133,9 @@ def fetch_spike_data( dictionary containing the group key time_slice : list of float, optional if provided, filter for spikes occurring in the interval [start, stop], by default None + return_unit_ids : bool, optional + if True, return the unit_ids along with the spike times, by default False + Unit ids defined as a list of dictionaries with keys 'spikesorting_merge_id' and 'unit_number' Returns ------- @@ -156,20 +160,23 @@ def fetch_spike_data( # get the spike times for each merge_id spike_times = [] + unit_ids = [] merge_keys = [dict(merge_id=merge_id) for merge_id in merge_ids] - nwb_file_list = (SpikeSortingOutput & merge_keys).fetch_nwb() - for nwb_file in nwb_file_list: - nwb_field_name = ( - "object_id" - if "object_id" in nwb_file - else "units" if "units" in nwb_file else None - ) + nwb_file_list, merge_ids = (SpikeSortingOutput & merge_keys).fetch_nwb( + return_merge_ids=True + ) + for nwb_file, merge_id in zip(nwb_file_list, merge_ids): + nwb_field_name = _get_spike_obj_name(nwb_file, allow_empty=True) if nwb_field_name is None: # case where no units found or curation removed all units continue sorting_spike_times = nwb_file[nwb_field_name][ "spike_times" ].to_list() + file_unit_ids = [ + {"spikesorting_merge_id": merge_id, "unit_id": unit_id} + for unit_id in range(len(sorting_spike_times)) + ] # filter the spike times based on the labels if present if "label" in nwb_file[nwb_field_name]: @@ -181,6 +188,7 @@ def fetch_spike_data( sorting_spike_times = list( compress(sorting_spike_times, include_unit) ) + file_unit_ids = list(compress(file_unit_ids, include_unit)) # filter the spike times based on the time slice if provided if time_slice is not None: @@ -195,7 +203,10 @@ def fetch_spike_data( # append the approved spike times to the list spike_times.extend(sorting_spike_times) + unit_ids.extend(file_unit_ids) + if return_unit_ids: + return spike_times, unit_ids return spike_times @classmethod @@ -276,3 +287,15 @@ def get_firing_rate( ], axis=1, ) + + +@staticmethod +def _get_spike_obj_name(nwb_file, allow_empty=False): + nwb_field_name = ( + "object_id" + if "object_id" in nwb_file + else "units" if "units" in nwb_file else None + ) + if nwb_field_name is None and not allow_empty: + raise ValueError("NWB file does not have 'object_id' or 'units' field") + return nwb_field_name diff --git a/src/spyglass/spikesorting/analysis/v1/unit_annotation.py b/src/spyglass/spikesorting/analysis/v1/unit_annotation.py new file mode 100644 index 000000000..4e1328979 --- /dev/null +++ b/src/spyglass/spikesorting/analysis/v1/unit_annotation.py @@ -0,0 +1,139 @@ +from typing import Optional, Union + +import datajoint as dj +import numpy as np + +from spyglass.spikesorting.analysis.v1.group import _get_spike_obj_name +from spyglass.spikesorting.spikesorting_merge import SpikeSortingOutput +from spyglass.utils import logger +from spyglass.utils.dj_mixin import SpyglassMixin + +schema = dj.schema("spikesorting_unit_annotation_v1") + + +@schema +class UnitAnnotation(SpyglassMixin, dj.Manual): + definition = """ + -> SpikeSortingOutput.proj(spikesorting_merge_id='merge_id') + unit_id: int + """ + + class Annotation(SpyglassMixin, dj.Part): + definition = """ + -> master + annotation: varchar(128) # the kind of annotation (e.g. a table name, "cell_type", "firing_rate", etc.) + --- + label = NULL: varchar(128) # text labels from analysis + quantification = NULL: float # quantification label from analysis + """ + + def fetch_unit_spikes(self, return_unit_ids=False): + """Fetch the spike times for a restricted set of units + + Parameters + ---------- + return_unit_ids : bool, optional + whether to return unit ids with spike times, by default False + + Returns + ------- + list of np.ndarray + list of spike times for each unit in the group, + if return_unit_ids is False + tuple of list of np.ndarray, list of str + list of spike times for each unit in the group and the unit ids, + if return_unit_ids is True + """ + return (UnitAnnotation & self).fetch_unit_spikes(return_unit_ids) + + def add_annotation(self, key, **kwargs): + """Add an annotation to a unit. Creates the unit if it does not exist. + + Parameters + ---------- + key : dict + dictionary with key for Annotation + + Raises + ------ + ValueError + if unit_id is not valid for the sorting + """ + # validate new units + unit_key = { + k: v + for k, v in key.items() + if k in ["spikesorting_merge_id", "unit_id"] + } + if not self & unit_key: + nwb_file = ( + SpikeSortingOutput & {"merge_id": key["spikesorting_merge_id"]} + ).fetch_nwb()[0] + nwb_field_name = _get_spike_obj_name(nwb_file) + spikes = nwb_file[nwb_field_name]["spike_times"].to_list() + if key["unit_id"] > len(spikes): + raise ValueError( + f"unit_id {key['unit_id']} is greater than ", + f"the number of units in {key['spikesorting_merge_id']}", + ) + self.insert1(unit_key) + # add annotation + self.Annotation().insert1(key, **kwargs) + + def fetch_unit_spikes( + self, return_unit_ids=False + ) -> Union[list[np.ndarray], Optional[list[dict]]]: + """Fetch the spike times for a restricted set of units + + Parameters + ---------- + return_unit_ids : bool, optional + whether to return unit ids with spike times, by default False + + Returns + ------- + list of np.ndarray + list of spike times for each unit in the group, + if return_unit_ids is False + tuple of list of np.ndarray, list of str + list of spike times for each unit in the group and the unit ids, + if return_unit_ids is True + """ + if len(self) == len(UnitAnnotation()): + logger.warning( + "fetching all unit spikes", + "if this is unintended, please call as: ", + "(UnitAnnotation & key).fetch_unit_spikes()", + ) + # get the set of nwb files to load + merge_keys = [ + {"merge_id": merge_id} + for merge_id in list(set(self.fetch("spikesorting_merge_id"))) + ] + nwb_file_list, merge_ids = (SpikeSortingOutput & merge_keys).fetch_nwb( + return_merge_ids=True + ) + + spikes = [] + unit_ids = [] + for nwb_file, merge_id in zip(nwb_file_list, merge_ids): + nwb_field_name = _get_spike_obj_name(nwb_file) + sorting_spike_times = nwb_file[nwb_field_name][ + "spike_times" + ].to_list() + include_unit = np.unique( + (self & {"spikesorting_merge_id": merge_id}).fetch("unit_id") + ) + spikes.extend( + [sorting_spike_times[unit_id] for unit_id in include_unit] + ) + unit_ids.extend( + [ + {"spikesorting_merge_id": merge_id, "unit_id": unit_id} + for unit_id in include_unit + ] + ) + + if return_unit_ids: + return spikes, unit_ids + return spikes diff --git a/src/spyglass/utils/dj_merge_tables.py b/src/spyglass/utils/dj_merge_tables.py index d17c087bd..4ffbf38f4 100644 --- a/src/spyglass/utils/dj_merge_tables.py +++ b/src/spyglass/utils/dj_merge_tables.py @@ -507,6 +507,7 @@ def fetch_nwb( restriction: str = None, multi_source=False, disable_warning=False, + return_merge_ids=False, *attrs, **kwargs, ): @@ -521,6 +522,8 @@ def fetch_nwb( Restriction to apply to parents before running fetch. Default True. multi_source: bool Return from multiple parents. Default False. + return_merge_ids: bool + Default False. Return merge_ids with nwb files. Notes ----- @@ -531,6 +534,7 @@ def fetch_nwb( restriction = restriction or self.restriction or True sources = set((self & restriction).fetch(self._reserved_sk)) nwb_list = [] + merge_ids = [] for source in sources: source_restr = ( self & {self._reserved_sk: source} & restriction @@ -540,6 +544,10 @@ def fetch_nwb( source_restr, permit_multiple_rows=True ).fetch_nwb() ) + if return_merge_ids: + merge_ids.extend([k[self._reserved_pk] for k in source_restr]) + if return_merge_ids: + return nwb_list, merge_ids return nwb_list @classmethod From 68e8de359caecfbde1be497635533b5a398268c9 Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Fri, 2 Aug 2024 10:03:58 -0500 Subject: [PATCH 22/94] Reduce duplication 1 (#1050) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * WIP: Merge duplicate funcs 1 * WIP: Merge duplicate funcs 2 * WIP: Fix typo * WIP: Merge duplicate funcs 3 * ✅ : Update changelog * 🧹 : Tidy common * 🧹 : Line length * Review suggestions from @samuelbray32 * Update src/spyglass/decoding/v0/clusterless.py Co-authored-by: Samuel Bray * Collapse cpu clusterlees args, spike_inticator dim check * Fix error * Fix typo * Fix failing tests --------- Co-authored-by: Samuel Bray --- CHANGELOG.md | 1 + src/spyglass/common/common_behav.py | 9 +- src/spyglass/common/common_dandi.py | 20 +- src/spyglass/common/common_device.py | 78 +++-- src/spyglass/common/common_ephys.py | 176 ++++++----- src/spyglass/common/common_filter.py | 1 - src/spyglass/common/common_interval.py | 21 +- src/spyglass/common/common_lab.py | 13 +- src/spyglass/common/common_nwbfile.py | 286 +++++++++--------- src/spyglass/common/common_position.py | 98 ++---- src/spyglass/common/common_region.py | 5 +- src/spyglass/common/common_ripple.py | 27 +- src/spyglass/common/common_session.py | 25 +- src/spyglass/common/common_subject.py | 5 +- src/spyglass/common/common_task.py | 38 +-- .../common/prepopulate/prepopulate.py | 5 +- src/spyglass/common/signal_processing.py | 4 + src/spyglass/data_import/insert_sessions.py | 5 +- src/spyglass/decoding/decoding_merge.py | 1 - src/spyglass/decoding/utils.py | 43 +++ src/spyglass/decoding/v0/clusterless.py | 172 ++--------- src/spyglass/decoding/v0/core.py | 11 +- src/spyglass/decoding/v0/sorted_spikes.py | 97 +----- src/spyglass/decoding/v0/utils.py | 124 ++++++++ .../decoding/v0/visualization_1D_view.py | 12 +- .../decoding/v0/visualization_2D_view.py | 33 +- src/spyglass/decoding/v1/clusterless.py | 100 +++--- src/spyglass/decoding/v1/core.py | 29 +- src/spyglass/decoding/v1/sorted_spikes.py | 64 ++-- src/spyglass/decoding/v1/waveform_features.py | 64 +--- src/spyglass/lfp/analysis/v1/lfp_band.py | 21 +- src/spyglass/lfp/lfp_electrode.py | 4 +- src/spyglass/lfp/lfp_imported.py | 2 +- src/spyglass/lfp/v1/lfp.py | 30 +- .../lfp/v1/lfp_artifact_MAD_detection.py | 15 +- .../v1/lfp_artifact_difference_detection.py | 3 +- src/spyglass/linearization/utils.py | 0 src/spyglass/linearization/v0/main.py | 6 +- src/spyglass/linearization/v1/main.py | 16 +- src/spyglass/lock/file_lock.py | 6 +- src/spyglass/position/v1/dlc_utils.py | 30 -- src/spyglass/position/v1/dlc_utils_makevid.py | 4 +- src/spyglass/ripple/v1/ripple.py | 34 +-- .../spikesorting/analysis/v1/group.py | 2 +- .../spikesorting/spikesorting_merge.py | 37 ++- src/spyglass/spikesorting/utils.py | 130 ++++++++ .../spikesorting/v0/curation_figurl.py | 13 +- .../spikesorting/v0/spikesorting_artifact.py | 20 +- .../spikesorting/v0/spikesorting_curation.py | 24 +- .../spikesorting/v0/spikesorting_recording.py | 110 +------ src/spyglass/spikesorting/v1/artifact.py | 11 +- src/spyglass/spikesorting/v1/curation.py | 6 +- src/spyglass/spikesorting/v1/recording.py | 144 ++------- src/spyglass/utils/position.py | 30 ++ tests/common/test_nwbfile.py | 2 +- tests/common/test_position.py | 5 +- tests/conftest.py | 4 +- 57 files changed, 1120 insertions(+), 1156 deletions(-) create mode 100644 src/spyglass/decoding/utils.py create mode 100644 src/spyglass/decoding/v0/utils.py create mode 100644 src/spyglass/linearization/utils.py create mode 100644 src/spyglass/spikesorting/utils.py create mode 100644 src/spyglass/utils/position.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 66e0a36d4..f03c2c298 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ PositionGroup.alter() - Speed up fetch_nwb calls through merge tables #1017 - Allow `ModuleNotFoundError` or `ImportError` for optional dependencies #1023 - Ensure integrity of group tables #1026 +- Merge duplicate functions in decoding and spikesorting #1050 ### Pipelines diff --git a/src/spyglass/common/common_behav.py b/src/spyglass/common/common_behav.py index a1397769b..efff2a408 100644 --- a/src/spyglass/common/common_behav.py +++ b/src/spyglass/common/common_behav.py @@ -505,7 +505,7 @@ class PositionIntervalMap(SpyglassMixin, dj.Computed): definition = """ -> IntervalList --- - position_interval_name="": varchar(200) # name of the corresponding interval + position_interval_name="": varchar(200) # corresponding interval name """ # #849 - Insert null to avoid rerun @@ -519,7 +519,7 @@ def _no_transaction_make(self, key): # epoch/pos intervals if not self.connection.in_transaction: - # if not called in the context of a make function, call its own make function + # if called w/o transaction, call add via `populate` self.populate(key) return if self & key: @@ -574,8 +574,9 @@ def _no_transaction_make(self, key): # Check that each pos interval was matched to only one epoch if len(matching_pos_intervals) != 1: logger.warning( - f"{no_pop_msg}. Found {len(matching_pos_intervals)} pos intervals for " - + f"\n\t{key}\n\tMatching intervals: {matching_pos_intervals}" + f"{no_pop_msg}. Found {len(matching_pos_intervals)} pos " + + f"intervals for\n\t{key}\n\t" + + f"Matching intervals: {matching_pos_intervals}" ) self.insert1(null_key, **insert_opts) return diff --git a/src/spyglass/common/common_dandi.py b/src/spyglass/common/common_dandi.py index 27d340766..3c721368f 100644 --- a/src/spyglass/common/common_dandi.py +++ b/src/spyglass/common/common_dandi.py @@ -75,7 +75,7 @@ def fetch_file_from_dandi(self, key: dict): # create a cache to save downloaded data to disk (optional) fsspec_file = CachingFileSystem( fs=fs, - cache_storage=f"{export_dir}/nwb-cache", # Local folder for the cache + cache_storage=f"{export_dir}/nwb-cache", # Local folder for cache ) # Open and return the file @@ -101,7 +101,7 @@ def compile_dandiset( dandi_api_key : str, optional API key for the dandi server. Optional if the environment variable DANDI_API_KEY is set. - dandi_instance : What instance of Dandi the dandiset is on. Defaults to the dev server + dandi_instance : dandiset's Dandi instance. Defaults to the dev server """ key = (Export & key).fetch1("KEY") paper_id = (Export & key).fetch1("paper_id") @@ -118,7 +118,8 @@ def compile_dandiset( destination_dir = f"{paper_dir}/dandiset_{paper_id}" dandiset_dir = f"{paper_dir}/{dandiset_id}" - # check if pre-existing directories for dandi export exist. Remove if so to continue + # check if pre-existing directories for dandi export exist. + # Remove if so to continue for dandi_dir in destination_dir, dandiset_dir: if os.path.exists(dandi_dir): from datajoint.utils import user_choice @@ -134,7 +135,8 @@ def compile_dandiset( shutil.rmtree(dandi_dir) continue raise RuntimeError( - f"Directory must be removed prior to dandi export to ensure dandi-compatability: {dandi_dir}" + "Directory must be removed prior to dandi export to ensure " + + f"dandi-compatability: {dandi_dir}" ) os.makedirs(destination_dir, exist_ok=False) @@ -148,7 +150,10 @@ def compile_dandiset( validate_dandiset(destination_dir, ignore_external_files=True) # given dandiset_id, download the dandiset to the export_dir - url = f"{known_instances[dandi_instance].gui}/dandiset/{dandiset_id}/draft" + url = ( + f"{known_instances[dandi_instance].gui}" + + f"/dandiset/{dandiset_id}/draft" + ) dandi.download.download(url, output_dir=paper_dir) # organize the files in the dandiset directory @@ -196,7 +201,7 @@ def _get_metadata(path): def translate_name_to_dandi(folder): """Uses dandi.organize to translate filenames to dandi paths - *Note* The name for a given file is dependent on that of all files in the folder + NOTE: The name for a given file depends on all files in the folder Parameters ---------- @@ -232,7 +237,8 @@ def validate_dandiset( folder : str location of dandiset to be validated min_severity : str - minimum severity level for errors to be reported, threshold for failed Dandi upload is "ERROR" + minimum severity level for errors to be reported, threshold for failed + Dandi upload is "ERROR" ignore_external_files : bool whether to ignore external file errors. Used if validating before the organize step diff --git a/src/spyglass/common/common_device.py b/src/spyglass/common/common_device.py index b7a67bcab..19ab7ff2a 100644 --- a/src/spyglass/common/common_device.py +++ b/src/spyglass/common/common_device.py @@ -54,8 +54,6 @@ def insert_from_nwbfile(cls, nwbf, config=None): _, ndx_devices, _ = cls.get_all_device_names(nwbf, config) for device_name in ndx_devices: - new_device_dict = dict() - # read device properties into new_device_dict from PyNWB extension # device object nwb_device_obj = ndx_devices[device_name] @@ -75,12 +73,14 @@ def insert_from_nwbfile(cls, nwbf, config=None): if adc_circuit.title() == "Intan": adc_circuit = "Intan" - new_device_dict["data_acquisition_device_name"] = name - new_device_dict["data_acquisition_device_system"] = system - new_device_dict["data_acquisition_device_amplifier"] = amplifier - new_device_dict["adc_circuit"] = adc_circuit - - cls._add_device(new_device_dict) + cls._add_device( + dict( + data_acquisition_device_name=name, + data_acquisition_device_system=system, + data_acquisition_device_amplifier=amplifier, + adc_circuit=adc_circuit, + ) + ) if ndx_devices: logger.info( @@ -638,41 +638,37 @@ def create_from_nwbfile( # only look at electrodes where the associated device is the one # specified - if eg_device_name == nwb_device_name: - device_found = True - - # if a Shank has not yet been created from the electrode group, - # then create it - if electrode_group.name not in created_shanks: - shank_index = len(created_shanks) - created_shanks[electrode_group.name] = shank_index - - # build the dictionary of Probe.Shank data - shank_dict[shank_index] = { - "probe_id": new_probe_dict["probe_id"], - "probe_shank": shank_index, - } - - # get the probe shank index associated with this Electrode - probe_shank = created_shanks[electrode_group.name] - - # build the dictionary of Probe.Electrode data - elect_dict[elec_index] = { + if eg_device_name != nwb_device_name: + continue + + device_found = True + + # if a Shank has not yet been created from the electrode group, + # then create it + if electrode_group.name not in created_shanks: + shank_index = len(created_shanks) + created_shanks[electrode_group.name] = shank_index + + # build the dictionary of Probe.Shank data + shank_dict[shank_index] = { "probe_id": new_probe_dict["probe_id"], - "probe_shank": probe_shank, - "probe_electrode": elec_index, + "probe_shank": shank_index, } - if "rel_x" in nwbfile.electrodes[elec_index]: - elect_dict[elec_index]["rel_x"] = nwbfile.electrodes[ - elec_index, "rel_x" - ] - if "rel_y" in nwbfile.electrodes[elec_index]: - elect_dict[elec_index]["rel_y"] = nwbfile.electrodes[ - elec_index, "rel_y" - ] - if "rel_z" in nwbfile.electrodes[elec_index]: - elect_dict[elec_index]["rel_z"] = nwbfile.electrodes[ - elec_index, "rel_z" + + # get the probe shank index associated with this Electrode + probe_shank = created_shanks[electrode_group.name] + + # build the dictionary of Probe.Electrode data + elect_dict[elec_index] = { + "probe_id": new_probe_dict["probe_id"], + "probe_shank": probe_shank, + "probe_electrode": elec_index, + } + + for dim in ["rel_x", "rel_y", "rel_z"]: + if dim in nwbfile.electrodes[elec_index]: + elect_dict[elec_index][dim] = nwbfile.electrodes[ + elec_index, dim ] if not device_found: diff --git a/src/spyglass/common/common_ephys.py b/src/spyglass/common/common_ephys.py index 146efeea1..c3bb210f6 100644 --- a/src/spyglass/common/common_ephys.py +++ b/src/spyglass/common/common_ephys.py @@ -53,7 +53,7 @@ def make(self, key): nwbf = get_nwb_file(nwb_file_abspath) for electrode_group in nwbf.electrode_groups.values(): key["electrode_group_name"] = electrode_group.name - # add electrode group location if it does not exist, and fetch the row + # add electrode group location if it not exist, and fetch the row key["region_id"] = BrainRegion.fetch_add( region_name=electrode_group.location ) @@ -189,7 +189,7 @@ def make(self, key): @classmethod def create_from_config(cls, nwb_file_name: str): - """Create or update Electrode entries from what is specified in the config YAML file. + """Create/update Electrode entries using config YAML file. Parameters ---------- @@ -396,7 +396,7 @@ class LFPElectrode(SpyglassMixin, dj.Part): """ def set_lfp_electrodes(self, nwb_file_name, electrode_list): - """Removes all electrodes for the specified nwb file and then adds back the electrodes in the list + """Replaces all electrodes for an nwb file with the given list Parameters ---------- @@ -406,31 +406,23 @@ def set_lfp_electrodes(self, nwb_file_name, electrode_list): list of electrodes to be used for LFP """ + nwb_dict = dict(nwb_file_name=nwb_file_name) + # remove the session and then recreate the session and Electrode list - (LFPSelection() & {"nwb_file_name": nwb_file_name}).delete( - safemode=not test_mode - ) - # check to see if the user allowed the deletion - if ( - len((LFPSelection() & {"nwb_file_name": nwb_file_name}).fetch()) - == 0 - ): - LFPSelection().insert1({"nwb_file_name": nwb_file_name}) - - # TODO: do this in a better way - all_electrodes = ( - Electrode() & {"nwb_file_name": nwb_file_name} - ).fetch(as_dict=True) - primary_key = Electrode.primary_key - for e in all_electrodes: - # create a dictionary so we can insert new elects - if e["electrode_id"] in electrode_list: - lfpelectdict = { - k: v for k, v in e.items() if k in primary_key - } - LFPSelection().LFPElectrode.insert1( - lfpelectdict, replace=True - ) + (LFPSelection() & nwb_dict).delete(safemode=not test_mode) + + # check to see if the deletion occurred + if len((LFPSelection() & nwb_dict).fetch()) != 0: + return + + insert_list = [ + {k: v for k, v in e.items() if k in Electrode.primary_key} + for e in (Electrode() & nwb_dict).fetch(as_dict=True) + if e["electrode_id"] in electrode_list + ] + + LFPSelection().insert1(nwb_dict) + LFPSelection().LFPElectrode.insert(insert_list, replace=True) @schema @@ -439,9 +431,9 @@ class LFP(SpyglassMixin, dj.Imported): -> LFPSelection --- -> IntervalList # the valid intervals for the data - -> FirFilterParameters # the filter used for the data + -> FirFilterParameters # the filter used for the data -> AnalysisNwbfile # the name of the nwb file with the lfp data - lfp_object_id: varchar(40) # the NWB object ID for loading this object from the file + lfp_object_id: varchar(40) # the ID for loading this object from the file lfp_sampling_rate: float # the sampling rate, in HZ """ @@ -525,7 +517,7 @@ def make(self, key): key["lfp_object_id"] = lfp_object_id key["lfp_sampling_rate"] = sampling_rate // decimation - # finally, we need to censor the valid times to account for the downsampling + # finally, censor the valid times to account for the downsampling lfp_valid_times = interval_list_censor(valid_times, timestamp_interval) # add an interval list for the LFP valid times, skipping duplicates key["interval_list_name"] = "lfp valid times" @@ -578,40 +570,55 @@ class LFPBandElectrode(SpyglassMixin, dj.Part): -> LFPBandSelection -> LFPSelection.LFPElectrode # the LFP electrode to be filtered reference_elect_id = -1: int # the reference electrode to use; -1 for no reference - --- """ def set_lfp_band_electrodes( self, - nwb_file_name, - electrode_list, - filter_name, - interval_list_name, - reference_electrode_list, - lfp_band_sampling_rate, - ): - """ - Adds an entry for each electrode in the electrode_list with the specified filter, interval_list, and - reference electrode. - Also removes any entries that have the same filter, interval list and reference electrode but are not - in the electrode_list. - :param nwb_file_name: string - the name of the nwb file for the desired session - :param electrode_list: list of LFP electrodes to be filtered - :param filter_name: the name of the filter (from the FirFilterParameters schema) - :param interval_name: the name of the interval list (from the IntervalList schema) - :param reference_electrode_list: A single electrode id corresponding to the reference to use for all - electrodes or a list with one element per entry in the electrode_list - :param lfp_band_sampling_rate: The output sampling rate to be used for the filtered data; must be an - integer divisor of the LFP sampling rate - :return: none + nwb_file_name: str, + electrode_list: list, + filter_name: str, + interval_list_name: str, + reference_electrode_list: list, + lfp_band_sampling_rate: int, + ) -> None: + """Add entry for each electrode with specified filter, interval, ref. + + Adds an entry for each electrode in the electrode_list with the + specified filter, interval_list, and reference electrode. Also removes + any entries that have the same filter, interval list and reference + electrode but are not in the electrode_list. + + Parameters + ---------- + nwb_file_name : str + The name of the nwb file for the desired Session. + electrode_list : list + List of LFP electrodes to be filtered. + filter_name : str + The name of the filter (from the FirFilterParameters table). + interval_list_name : str + The name of the interval list (from the IntervalList table). + reference_electrode_list : list + A single electrode id corresponding to the reference to use for all + electrodes. Or a list with one element per entry in the + electrode_list + lfp_band_sampling_rate : int + The output sampling rate to be used for the filtered data; must be + an integer divisor of the LFP sampling rate. + + Returns + ------- + None """ # Error checks on parameters # electrode_list + query = LFPSelection().LFPElectrode() & {"nwb_file_name": nwb_file_name} available_electrodes = query.fetch("electrode_id") if not np.all(np.isin(electrode_list, available_electrodes)): raise ValueError( - "All elements in electrode_list must be valid electrode_ids in the LFPSelection table" + "All elements in electrode_list must be valid electrode_ids in " + + "the LFPSelection table" ) # sampling rate lfp_sampling_rate = (LFP() & {"nwb_file_name": nwb_file_name}).fetch1( @@ -620,8 +627,8 @@ def set_lfp_band_electrodes( decimation = lfp_sampling_rate // lfp_band_sampling_rate if lfp_sampling_rate // decimation != lfp_band_sampling_rate: raise ValueError( - f"lfp_band_sampling rate {lfp_band_sampling_rate} is not an integer divisor of lfp " - f"sampling rate {lfp_sampling_rate}" + f"lfp_band_sampling rate {lfp_band_sampling_rate} is not an " + f"integer divisor of lfp sampling rate {lfp_sampling_rate}" ) # filter query = FirFilterParameters() & { @@ -630,7 +637,8 @@ def set_lfp_band_electrodes( } if not query: raise ValueError( - f"filter {filter_name}, sampling rate {lfp_sampling_rate} is not in the FirFilterParameters table" + f"filter {filter_name}, sampling rate {lfp_sampling_rate} is " + + "not in the FirFilterParameters table" ) # interval_list query = IntervalList() & { @@ -639,34 +647,36 @@ def set_lfp_band_electrodes( } if not query: raise ValueError( - f"interval list {interval_list_name} is not in the IntervalList table; the list must be " - "added before this function is called" + f"Item not in IntervalList: {interval_list_name}\n" + + "Item must be added before this function is called." ) # reference_electrode_list if len(reference_electrode_list) != 1 and len( reference_electrode_list ) != len(electrode_list): raise ValueError( - "reference_electrode_list must contain either 1 or len(electrode_list) elements" + "reference_electrode_list must contain either 1 or " + + "len(electrode_list) elements" ) # add a -1 element to the list to allow for the no reference option available_electrodes = np.append(available_electrodes, [-1]) if not np.all(np.isin(reference_electrode_list, available_electrodes)): raise ValueError( - "All elements in reference_electrode_list must be valid electrode_ids in the LFPSelection " - "table" + "All elements in reference_electrode_list must be valid " + + "electrode_ids in the LFPSelection table" ) # make a list of all the references ref_list = np.zeros((len(electrode_list),)) ref_list[:] = reference_electrode_list - key = dict() - key["nwb_file_name"] = nwb_file_name - key["filter_name"] = filter_name - key["filter_sampling_rate"] = lfp_sampling_rate - key["target_interval_list_name"] = interval_list_name - key["lfp_band_sampling_rate"] = lfp_sampling_rate // decimation + key = dict( + nwb_file_name=nwb_file_name, + filter_name=filter_name, + filter_sampling_rate=lfp_sampling_rate, + target_interval_list_name=interval_list_name, + lfp_band_sampling_rate=lfp_sampling_rate // decimation, + ) # insert an entry into the main LFPBandSelectionTable self.insert1(key, skip_duplicates=True) @@ -707,7 +717,9 @@ def make(self, key): lfp_band_file_name = AnalysisNwbfile().create( # logged key["nwb_file_name"] ) - # get the NWB object with the lfp data; FIX: change to fetch with additional infrastructure + + # get the NWB object with the lfp data; + # FIX: change to fetch with additional infrastructure lfp_object = ( LFP() & {"nwb_file_name": key["nwb_file_name"]} ).fetch_nwb()[0]["lfp"] @@ -737,8 +749,9 @@ def make(self, key): "interval_list_name": interval_list_name, } ).fetch1("valid_times") - # the valid_times for this interval may be slightly beyond the valid times for the lfp itself, - # so we have to intersect the two + + # the valid_times for this interval may be slightly beyond the valid + # times for the lfp itself, so we have to intersect the two lfp_interval_list = ( LFP() & {"nwb_file_name": key["nwb_file_name"]} ).fetch1("interval_list_name") @@ -764,7 +777,9 @@ def make(self, key): # load in the timestamps timestamps = np.asarray(lfp_object.timestamps) - # get the indices of the first timestamp and the last timestamp that are within the valid times + + # get the indices of the first timestamp and the last timestamp that + # are within the valid times included_indices = interval_list_contains_ind( lfp_band_valid_times, timestamps ) @@ -804,14 +819,16 @@ def make(self, key): ).fetch(as_dict=True) if len(filter) == 0: raise ValueError( - f"Filter {filter_name} and sampling_rate {lfp_band_sampling_rate} does not exit in the " - "FirFilterParameters table" + f"Filter {filter_name} and sampling_rate " + + f"{lfp_band_sampling_rate} does not exit in the " + + "FirFilterParameters table" ) filter_coeff = filter[0]["filter_coeff"] if len(filter_coeff) == 0: logger.info( - f"Error in LFPBand: no filter found with data sampling rate of {lfp_band_sampling_rate}" + "Error in LFPBand: no filter found with data sampling rate of " + + f"{lfp_band_sampling_rate}" ) return None @@ -828,12 +845,15 @@ def make(self, key): decimation, ) - # now that the LFP is filtered, we create an electrical series for it and add it to the file + # now that the LFP is filtered, we create an electrical series for it + # and add it to the file with pynwb.NWBHDF5IO( path=lfp_band_file_abspath, mode="a", load_namespaces=True ) as io: nwbf = io.read() - # get the indices of the electrodes in the electrode table of the file to get the right values + + # get the indices of the electrodes in the electrode table of the + # file to get the right values elect_index = get_electrode_indices(nwbf, lfp_band_elect_id) electrode_table_region = nwbf.create_electrode_table_region( elect_index, "filtered electrode table" @@ -856,8 +876,8 @@ def make(self, key): key["analysis_file_name"] = lfp_band_file_name key["filtered_data_object_id"] = filtered_data_object_id - # finally, we need to censor the valid times to account for the downsampling if this is the first time we've - # downsampled these data + # finally, we need to censor the valid times to account for the + # downsampling if this is the first time we've downsampled these data key["interval_list_name"] = ( interval_list_name + " lfp band " diff --git a/src/spyglass/common/common_filter.py b/src/spyglass/common/common_filter.py index 2726806e0..8deb667c0 100644 --- a/src/spyglass/common/common_filter.py +++ b/src/spyglass/common/common_filter.py @@ -345,7 +345,6 @@ def filter_data_nwb( io.write(nwbf) # Reload NWB file to get h5py objects for data/timestamps - # NOTE: CBroz - why io context within io context? Unindenting with pynwb.NWBHDF5IO( path=analysis_file_abs_path, mode="a", load_namespaces=True ) as io: diff --git a/src/spyglass/common/common_interval.py b/src/spyglass/common/common_interval.py index 6e4d6b042..1b31b612b 100644 --- a/src/spyglass/common/common_interval.py +++ b/src/spyglass/common/common_interval.py @@ -5,12 +5,12 @@ import matplotlib.pyplot as plt import numpy as np import pandas as pd +from pynwb import NWBFile +from spyglass.common.common_session import Session # noqa: F401 from spyglass.utils import SpyglassMixin, logger from spyglass.utils.dj_helper_fn import get_child_tables -from .common_session import Session # noqa: F401 - schema = dj.schema("common_interval") # TODO: ADD export to NWB function to save relevant intervals in an NWB file @@ -24,13 +24,13 @@ class IntervalList(SpyglassMixin, dj.Manual): interval_list_name: varchar(170) # descriptive name of this interval list --- valid_times: longblob # numpy array with start/end times for each interval - pipeline = "": varchar(64) # type of interval list (e.g. 'position', 'spikesorting_recording_v1') + pipeline = "": varchar(64) # type of interval list """ # See #630, #664. Excessive key length. @classmethod - def insert_from_nwbfile(cls, nwbf, *, nwb_file_name): + def insert_from_nwbfile(cls, nwbf: NWBFile, *, nwb_file_name: str): """Add each entry in the NWB file epochs table to the IntervalList. The interval list name for each epoch is set to the first tag for the @@ -54,20 +54,23 @@ def insert_from_nwbfile(cls, nwbf, *, nwb_file_name): epochs = nwbf.epochs.to_dataframe() - for _, epoch_data in epochs.iterrows(): - epoch_dict = { + # Create a list of dictionaries to insert + epoch_inserts = epochs.apply( + lambda epoch_data: { "nwb_file_name": nwb_file_name, "interval_list_name": ( epoch_data.tags[0] if epoch_data.tags - else f"interval_{epoch_data[0]}" + else f"interval_{epoch_data.name}" ), "valid_times": np.asarray( [[epoch_data.start_time, epoch_data.stop_time]] ), - } + }, + axis=1, + ).tolist() - cls.insert1(epoch_dict, skip_duplicates=True) + cls.insert(epoch_inserts, skip_duplicates=True) def plot_intervals(self, figsize=(20, 5), return_fig=False): interval_list = pd.DataFrame(self) diff --git a/src/spyglass/common/common_lab.py b/src/spyglass/common/common_lab.py index 44be5f4b2..486041abc 100644 --- a/src/spyglass/common/common_lab.py +++ b/src/spyglass/common/common_lab.py @@ -1,6 +1,7 @@ """Schema for institution, lab team/name/members. Session-independent.""" import datajoint as dj + from spyglass.utils import SpyglassMixin, logger from ..utils.nwb_helper_fn import get_nwb_file @@ -223,8 +224,8 @@ def create_new_team( ) if not query: logger.info( - "To help manage permissions in LabMemberInfo, please add Google " - + f"user ID for {team_member}" + "To help manage permissions in LabMemberInfo, please add " + + f"Google user ID for {team_member}" ) labteammember_dict = { "team_name": team_name, @@ -259,13 +260,15 @@ def insert_from_nwbfile(cls, nwbf, config=None): Returns ------- institution_name : string - The name of the institution found in the NWB or config file, or None. + The name of the institution found in the NWB or config file, + or None. """ config = config or dict() inst_list = config.get("Institution", [{}]) if len(inst_list) > 1: logger.info( - "Multiple institution entries not allowed. Using the first entry only.\n" + "Multiple institution entries not allowed. " + + "Using the first entry only.\n" ) inst_name = inst_list[0].get("institution_name") or getattr( nwbf, "institution", None @@ -305,7 +308,7 @@ def insert_from_nwbfile(cls, nwbf, config=None): lab_list = config.get("Lab", [{}]) if len(lab_list) > 1: logger.info( - "Multiple lab entries not allowed. Using the first entry only.\n" + "Multiple lab entries not allowed. Using the first entry only." ) lab_name = lab_list[0].get("lab_name") or getattr(nwbf, "lab", None) if not lab_name: diff --git a/src/spyglass/common/common_nwbfile.py b/src/spyglass/common/common_nwbfile.py index 6a9316a88..9d5fce015 100644 --- a/src/spyglass/common/common_nwbfile.py +++ b/src/spyglass/common/common_nwbfile.py @@ -51,13 +51,13 @@ class Nwbfile(SpyglassMixin, dj.Manual): nwb_file_abs_path: filepath@raw INDEX (nwb_file_abs_path) """ - # NOTE the INDEX above is implicit from filepath@... above but needs to be explicit - # so that alter() can work + # NOTE the INDEX above is implicit from filepath@... above but needs to be + # explicit so that alter() can work # NOTE: See #630, #664. Excessive key length. @classmethod - def insert_from_relative_file_name(cls, nwb_file_name): + def insert_from_relative_file_name(cls, nwb_file_name: str) -> None: """Insert a new session from an existing NWB file. Parameters @@ -95,7 +95,7 @@ def get_file_key(cls, nwb_file_name: str) -> dict: return {"nwb_file_name": cls._get_file_name(nwb_file_name)} @classmethod - def get_abs_path(cls, nwb_file_name, new_file=False) -> str: + def get_abs_path(cls, nwb_file_name: str, new_file: bool = False) -> str: """Return absolute path for a stored raw NWB file given file name. The SPYGLASS_BASE_DIR must be set, either as an environment or part of @@ -120,42 +120,44 @@ def get_abs_path(cls, nwb_file_name, new_file=False) -> str: return raw_dir + "/" + cls._get_file_name(nwb_file_name) @staticmethod - def add_to_lock(nwb_file_name): - """Add the specified NWB file to the file with the list of NWB files to be locked. + def add_to_lock(nwb_file_name: str) -> None: + """Add the specified NWB file to the list of locked items. - The NWB_LOCK_FILE environment variable must be set. + The NWB_LOCK_FILE environment variable must be set to the path of the + lock file, listing locked NWB files. Parameters ---------- nwb_file_name : str - The name of an NWB file that has been inserted into the Nwbfile() schema. + The name of an NWB file in the Nwbfile table. """ - key = {"nwb_file_name": nwb_file_name} - # check to make sure the file exists - assert ( - len((Nwbfile() & key).fetch()) > 0 - ), f"Error adding {nwb_file_name} to lock file, not in Nwbfile() schema" + if not (Nwbfile() & {"nwb_file_name": nwb_file_name}): + raise FileNotFoundError( + f"File not found in Nwbfile table. Cannot lock {nwb_file_name}" + ) - lock_file = open(os.getenv("NWB_LOCK_FILE"), "a+") - lock_file.write(f"{nwb_file_name}\n") - lock_file.close() + with open(os.getenv("NWB_LOCK_FILE"), "a+") as lock_file: + lock_file.write(f"{nwb_file_name}\n") @staticmethod - def cleanup(delete_files=False): + def cleanup(delete_files: bool = False) -> None: """Remove the filepath entries for NWB files that are not in use. - This does not delete the files themselves unless delete_files=True is specified - Run this after deleting the Nwbfile() entries themselves. + This does not delete the files themselves unless delete_files=True is + specified. Run this after deleting the Nwbfile() entries themselves. """ schema.external["raw"].delete(delete_external_files=delete_files) -# TODO: add_to_kachery will not work because we can't update the entry after it's been used in another table. -# We therefore need another way to keep track of the +# TODO: add_to_kachery will not work because we can't update the entry after +# it's been used in another table. We therefore need another way to keep track +# of the file here + + @schema class AnalysisNwbfile(SpyglassMixin, dj.Manual): definition = """ - # Table for holding the NWB files that contain results of analysis, such as spike sorting. + # Table for NWB files that contain results of analysis. analysis_file_name: varchar(64) # name of the file --- -> Nwbfile # name of the parent NWB file. Used for naming and metadata copy @@ -172,10 +174,11 @@ class AnalysisNwbfile(SpyglassMixin, dj.Manual): _creation_times = {} - def create(self, nwb_file_name): - """Open the NWB file, create a copy, write the copy to disk and return the name of the new file. + def create(self, nwb_file_name: str) -> str: + """Open the NWB file, create copy, write to disk and return new name. - Note that this does NOT add the file to the schema; that needs to be done after data are written to it. + Note that this does NOT add the file to the schema; that needs to be + done after data are written to it. Parameters ---------- @@ -237,13 +240,13 @@ def create(self, nwb_file_name): return analysis_file_name @staticmethod - def _alter_spyglass_version(nwb_file_path): + def _alter_spyglass_version(nwb_file_path: str) -> None: """Change the source script to the current version of spyglass""" with h5py.File(nwb_file_path, "a") as f: f["/general/source_script"][()] = f"spyglass={sg_version}" @classmethod - def __get_new_file_name(cls, nwb_file_name): + def __get_new_file_name(cls, nwb_file_name: str) -> str: # each file ends with a random string of 10 digits, so we generate that # string and redo if by some miracle it's already there file_in_table = True @@ -262,15 +265,16 @@ def __get_new_file_name(cls, nwb_file_name): return analysis_file_name @classmethod - def __get_analysis_file_dir(cls, analysis_file_name: str): - # strip off everything after and including the final underscore and return the result + def __get_analysis_file_dir(cls, analysis_file_name: str) -> str: + """Strip off final underscore and remaining chars, return the result.""" return analysis_file_name[0 : analysis_file_name.rfind("_")] @classmethod - def copy(cls, nwb_file_name): + def copy(cls, nwb_file_name: str): """Make a copy of an analysis NWB file. - Note that this does NOT add the file to the schema; that needs to be done after data are written to it. + Note that this does NOT add the file to the schema; that needs to be + done after data are written to it. Parameters ---------- @@ -304,7 +308,7 @@ def copy(cls, nwb_file_name): return analysis_file_name - def add(self, nwb_file_name, analysis_file_name): + def add(self, nwb_file_name: str, analysis_file_name: str) -> None: """Add the specified file to AnalysisNWBfile table. Parameters @@ -325,15 +329,15 @@ def add(self, nwb_file_name, analysis_file_name): self.insert1(key) @classmethod - def get_abs_path(cls, analysis_nwb_file_name): - """Return the absolute path for a stored analysis NWB file given just the file name. + def get_abs_path(cls, analysis_nwb_file_name: str) -> str: + """Return the absolute path for an analysis NWB file given the name. The spyglass config from settings.py must be set. Parameters ---------- analysis_nwb_file_name : str - The name of the NWB file that has been inserted into the AnalysisNwbfile() schema + The name of the NWB file in AnalysisNwbfile. Returns ------- @@ -367,11 +371,16 @@ def get_abs_path(cls, analysis_nwb_file_name): return str(analysis_file_base_path / analysis_nwb_file_name) def add_nwb_object( - self, analysis_file_name, nwb_object, table_name="pandas_table" + self, + analysis_file_name: str, + nwb_object: pynwb.core.NWBDataInterface, + table_name: str = "pandas_table", ): # TODO: change to add_object with checks for object type and a name # parameter, which should be specified if it is not an NWB container - """Add an NWB object to the analysis file in the scratch area and returns the NWB object ID + """Add an NWB object to the analysis file and return the NWB object ID + + Adds object to the scratch space of the NWB file. Parameters ---------- @@ -379,8 +388,9 @@ def add_nwb_object( The name of the analysis NWB file. nwb_object : pynwb.core.NWBDataInterface The NWB object created by PyNWB. - table_name : str (optional, defaults to 'pandas_table') - The name of the DynamicTable made from a dataframe. + table_name : str, optional + The name of the DynamicTable made from a dataframe. Defaults to + 'pandas_table'. Returns ------- @@ -394,26 +404,22 @@ def add_nwb_object( ) as io: nwbf = io.read() if isinstance(nwb_object, pd.DataFrame): - dt_object = DynamicTable.from_dataframe( + nwb_object = DynamicTable.from_dataframe( name=table_name, df=nwb_object ) - nwbf.add_scratch(dt_object) - io.write(nwbf) - return dt_object.object_id - else: - nwbf.add_scratch(nwb_object) - io.write(nwbf) - return nwb_object.object_id + nwbf.add_scratch(nwb_object) + io.write(nwbf) + return nwb_object.object_id def add_units( self, - analysis_file_name, - units, - units_valid_times, - units_sort_interval, - metrics=None, - units_waveforms=None, - labels=None, + analysis_file_name: str, + units: dict, + units_valid_times: dict, + units_sort_interval: dict, + metrics: dict = None, + units_waveforms: dict = None, + labels: dict = None, ): """Add units to analysis NWB file @@ -437,7 +443,8 @@ def add_units( Returns ------- units_object_id, waveforms_object_id : str, str - The NWB object id of the Units object and the object id of the waveforms object ('' if None) + The NWB object id of the Units object and the object id of the + waveforms object ('' if None) """ with pynwb.NWBHDF5IO( path=self.get_abs_path(analysis_file_name), @@ -446,82 +453,83 @@ def add_units( ) as io: nwbf = io.read() sort_intervals = list() - if len(units.keys()): - # Add spike times and valid time range for the sort - for id in units.keys(): - nwbf.add_unit( - spike_times=units[id], - id=id, - # waveform_mean = units_templates[id], - obs_intervals=units_valid_times[id], - ) - sort_intervals.append(units_sort_interval[id]) - # Add a column for the sort interval (subset of valid time) + + if not len(units.keys()): + return "" + + # Add spike times and valid time range for the sort + for id in units.keys(): + nwbf.add_unit( + spike_times=units[id], + id=id, + # waveform_mean = units_templates[id], + obs_intervals=units_valid_times[id], + ) + sort_intervals.append(units_sort_interval[id]) + + # Add a column for the sort interval (subset of valid time) + nwbf.add_unit_column( + name="sort_interval", + description="the interval used for spike sorting", + data=sort_intervals, + ) + + # If metrics were specified, add one column per metric + metrics = metrics or [] # do nothing if metrics is None + for metric in metrics: + if not metrics.get(metric): + continue + + unit_ids = np.array(list(metrics[metric].keys())) + metric_values = np.array(list(metrics[metric].values())) + + # sort by unit_ids and apply that sorting to values + # to ensure that things go in the right order + + metric_values = metric_values[np.argsort(unit_ids)] + logger.info(f"Adding metric {metric} : {metric_values}") nwbf.add_unit_column( - name="sort_interval", - description="the interval used for spike sorting", - data=sort_intervals, + name=metric, + description=f"{metric} metric", + data=metric_values, ) - # If metrics were specified, add one column per metric - if metrics is not None: - for metric in metrics: - if metrics[metric]: - unit_ids = np.array(list(metrics[metric].keys())) - metric_values = np.array( - list(metrics[metric].values()) - ) - - # sort by unit_ids and apply that sorting to values - # to ensure that things go in the right order - - metric_values = metric_values[np.argsort(unit_ids)] - logger.info( - f"Adding metric {metric} : {metric_values}" - ) - nwbf.add_unit_column( - name=metric, - description=f"{metric} metric", - data=metric_values, - ) - if labels is not None: - unit_ids = np.array(list(units.keys())) - for unit in unit_ids: - if unit not in labels: - labels[unit] = "" - label_values = np.array(list(labels.values())) - label_values = label_values[np.argsort(unit_ids)].tolist() - nwbf.add_unit_column( - name="label", - description="label given during curation", - data=label_values, - ) - # If the waveforms were specified, add them as a dataframe to scratch - waveforms_object_id = "" - if units_waveforms is not None: - waveforms_df = pd.DataFrame.from_dict( - units_waveforms, orient="index" - ) - waveforms_df.columns = ["waveforms"] - nwbf.add_scratch( - waveforms_df, - name="units_waveforms", - notes="spike waveforms for each unit", - ) - waveforms_object_id = nwbf.scratch[ - "units_waveforms" - ].object_id - io.write(nwbf) - return nwbf.units.object_id, waveforms_object_id - else: - return "" + if labels is not None: + unit_ids = np.array(list(units.keys())) + labels.update( + {unit: "" for unit in unit_ids if unit not in labels} + ) + label_values = np.array(list(labels.values())) + label_values = label_values[np.argsort(unit_ids)].tolist() + nwbf.add_unit_column( + name="label", + description="label given during curation", + data=label_values, + ) + + # If the waveforms were specified, add them as a df to scratch + waveforms_object_id = "" + if units_waveforms is not None: + waveforms_df = pd.DataFrame.from_dict( + units_waveforms, orient="index" + ) + waveforms_df.columns = ["waveforms"] + nwbf.add_scratch( + waveforms_df, + name="units_waveforms", + notes="spike waveforms for each unit", + ) + waveforms_object_id = nwbf.scratch["units_waveforms"].object_id + + io.write(nwbf) + return nwbf.units.object_id, waveforms_object_id def add_units_waveforms( self, - analysis_file_name, + analysis_file_name: str, waveform_extractor: si.WaveformExtractor, - metrics=None, - labels=None, + metrics: dict = None, + labels: dict = None, ): """Add units to analysis NWB file along with the waveforms @@ -562,9 +570,10 @@ def add_units_waveforms( ) # The following is a rough sketch of AnalysisNwbfile().add_waveforms - # analysis_file_name = AnalysisNwbfile().create(key['nwb_file_name']) - # or - # nwbfile = pynwb.NWBFile(...) + # analysis_file_name = + # AnalysisNwbfile().create(key['nwb_file_name']) + # or + # nwbfile = pynwb.NWBFile(...) # (channels, spikes, samples) # wfs = [ # [ # elec 1 @@ -584,8 +593,10 @@ def add_units_waveforms( # [1, 2, 3] # spike 4 # ] # ] - # elecs = ... # DynamicTableRegion referring to three electrodes (rows) of the electrodes table - # nwbfile.add_unit(spike_times=[1, 2, 3], electrodes=elecs, waveforms=wfs) + # elecs = ... # DynamicTableRegion referring to three electrodes + # (rows) of the electrodes table + # nwbfile.add_unit(spike_times=[1, 2, 3], electrodes=elecs, + # waveforms=wfs) # If metrics were specified, add one column per metric if metrics is not None: @@ -607,14 +618,14 @@ def add_units_waveforms( io.write(nwbf) return nwbf.units.object_id - def add_units_metrics(self, analysis_file_name, metrics): + def add_units_metrics(self, analysis_file_name: str, metrics: dict): """Add units to analysis NWB file along with the waveforms Parameters ---------- analysis_file_name : str The name of the analysis NWB file. - metrics : dict, optional + metrics : dict Cluster metrics. Returns @@ -644,8 +655,10 @@ def add_units_metrics(self, analysis_file_name, metrics): return nwbf.units.object_id @classmethod - def get_electrode_indices(cls, analysis_file_name, electrode_ids): - """Given an analysis NWB file name, returns the indices of the specified electrode_ids. + def get_electrode_indices( + cls, analysis_file_name: str, electrode_ids: np.array + ): + """Returns indices of the specified electrode_ids for an analysis file. Parameters ---------- @@ -657,7 +670,8 @@ def get_electrode_indices(cls, analysis_file_name, electrode_ids): Returns ------- electrode_indices : numpy array - Array of indices in the electrodes table for the given electrode IDs. + Array of indices in the electrodes table for the given electrode + IDs. """ nwbf = get_nwb_file(cls.get_abs_path(analysis_file_name)) return get_electrode_indices(nwbf.electrodes, electrode_ids) @@ -666,8 +680,8 @@ def get_electrode_indices(cls, analysis_file_name, electrode_ids): def cleanup(delete_files=False): """Remove the filepath entries for NWB files that are not in use. - Does not delete the files themselves unless delete_files=True is specified. - Run this after deleting the Nwbfile() entries themselves. + Does not delete the files themselves unless delete_files=True is + specified. Run this after deleting the Nwbfile() entries themselves. Parameters ---------- diff --git a/src/spyglass/common/common_position.py b/src/spyglass/common/common_position.py index 25ed5efeb..3331db555 100644 --- a/src/spyglass/common/common_position.py +++ b/src/spyglass/common/common_position.py @@ -14,12 +14,6 @@ ) from position_tools.core import gaussian_smooth from tqdm import tqdm_notebook as tqdm -from track_linearization import ( - get_linearized_position, - make_track_graph, - plot_graph_as_1D, - plot_track_graph, -) from spyglass.common.common_behav import RawPosition, VideoFile from spyglass.common.common_interval import IntervalList # noqa F401 @@ -27,6 +21,7 @@ from spyglass.settings import raw_dir, test_mode, video_dir from spyglass.utils import SpyglassMixin, logger from spyglass.utils.dj_helper_fn import deprecated_factory +from spyglass.utils.position import convert_to_pixels, fill_nan try: from position_tools import get_centroid @@ -70,7 +65,6 @@ class IntervalPositionInfoSelection(SpyglassMixin, dj.Lookup): definition = """ -> PositionInfoParameters -> IntervalList - --- """ @@ -128,10 +122,10 @@ def make(self, key): def generate_pos_components( spatial_series, position_info, - analysis_fname, - prefix="head_", - add_frame_ind=False, - video_frame_ind=None, + analysis_fname: str, + prefix: str = "head_", + add_frame_ind: bool = False, + video_frame_ind: int = None, ): """Generate position, orientation and velocity components.""" METERS_PER_CM = 0.01 @@ -140,9 +134,6 @@ def generate_pos_components( orientation = pynwb.behavior.CompassDirection() velocity = pynwb.behavior.BehavioralTimeSeries() - # NOTE: CBroz1 removed a try/except ValueError that surrounded all - # .create_X_series methods. dpeg22 could not recall purpose - time_comments = dict( comments=spatial_series.comments, timestamps=position_info["time"], @@ -317,8 +308,8 @@ def calculate_position_info( spatial_df: pd.DataFrame, meters_to_pixels: float, position_smoothing_duration, - led1_is_front, - is_upsampled, + led1_is_front: bool, + is_upsampled: bool, upsampling_sampling_rate, upsampling_interpolation_method, orient_smoothing_std_dev=None, @@ -466,7 +457,9 @@ def fetch1_dataframe(self): return self._data_to_df(self.fetch_nwb()[0]) @staticmethod - def _data_to_df(data, prefix="head_", add_frame_ind=False): + def _data_to_df( + data: pd.DataFrame, prefix: str = "head_", add_frame_ind: bool = False + ): pos, ori, vel = [ prefix + c for c in ["position", "orientation", "velocity"] ] @@ -526,17 +519,18 @@ def make(self, key): M_TO_CM = 100 logger.info("Loading position data...") + + nwb_dict = dict(nwb_file_name=key["nwb_file_name"]) + raw_position_df = ( RawPosition() - & { - "nwb_file_name": key["nwb_file_name"], - "interval_list_name": key["interval_list_name"], - } + & nwb_dict + & {"interval_list_name": key["interval_list_name"]} ).fetch1_dataframe() position_info_df = ( IntervalPositionInfo() & { - "nwb_file_name": key["nwb_file_name"], + **nwb_dict, "interval_list_name": key["interval_list_name"], "position_info_param_name": key["position_info_param_name"], } @@ -551,10 +545,7 @@ def make(self, key): ) + 1 ) - video_info = ( - VideoFile() - & {"nwb_file_name": key["nwb_file_name"], "epoch": epoch} - ).fetch1() + video_info = (VideoFile() & {**nwb_dict, "epoch": epoch}).fetch1() io = pynwb.NWBHDF5IO(raw_dir + "/" + video_info["nwb_file_name"], "r") nwb_file = io.read() nwb_video = nwb_file.objects[video_info["video_file_object_id"]] @@ -603,49 +594,20 @@ def make(self, key): ) self.insert1(key) - @staticmethod - def convert_to_pixels(data, frame_size, cm_to_pixels=1.0): - """Converts from cm to pixels and flips the y-axis. - Parameters - ---------- - data : ndarray, shape (n_time, 2) - frame_size : array_like, shape (2,) - cm_to_pixels : float - - Returns - ------- - converted_data : ndarray, shape (n_time, 2) - """ - return data / cm_to_pixels - - @staticmethod - def fill_nan(variable, video_time, variable_time): - video_ind = np.digitize(variable_time, video_time[1:]) - - n_video_time = len(video_time) - try: - n_variable_dims = variable.shape[1] - filled_variable = np.full((n_video_time, n_variable_dims), np.nan) - except IndexError: - filled_variable = np.full((n_video_time,), np.nan) - filled_variable[video_ind] = variable - - return filled_variable - def make_video( self, - video_filename, + video_filename: str, centroids, - head_position_mean, - head_orientation_mean, + head_position_mean: np.ndarray, + head_orientation_mean: np.ndarray, video_time, position_time, - output_video_filename="output.mp4", - cm_to_pixels=1.0, - disable_progressbar=False, - arrow_radius=15, - circle_radius=8, - truncate_data=False, # reduce data to min length across all variables + output_video_filename: str = "output.mp4", + cm_to_pixels: float = 1.0, + disable_progressbar: bool = False, + arrow_radius: int = 15, + circle_radius: int = 8, + truncate_data: bool = False, # reduce data to min len across all vars ): import cv2 # noqa: F401 @@ -683,13 +645,13 @@ def make_video( ) centroids = { - color: self.fill_nan(data, video_time, position_time) + color: fill_nan(data, video_time, position_time) for color, data in centroids.items() } - head_position_mean = self.fill_nan( + head_position_mean = fill_nan( head_position_mean, video_time, position_time ) - head_orientation_mean = self.fill_nan( + head_orientation_mean = fill_nan( head_orientation_mean, video_time, position_time ) @@ -704,7 +666,7 @@ def make_video( green_centroid = centroids["green"][time_ind] head_position = head_position_mean[time_ind] - head_position = self.convert_to_pixels( + head_position = convert_to_pixels( data=head_position, cm_to_pixels=cm_to_pixels ) head_orientation = head_orientation_mean[time_ind] diff --git a/src/spyglass/common/common_region.py b/src/spyglass/common/common_region.py index e2abfff5a..759537c4f 100644 --- a/src/spyglass/common/common_region.py +++ b/src/spyglass/common/common_region.py @@ -21,7 +21,10 @@ class BrainRegion(SpyglassMixin, dj.Lookup): @classmethod def fetch_add( - cls, region_name, subregion_name=None, subsubregion_name=None + cls, + region_name: str, + subregion_name: str = None, + subsubregion_name: str = None, ): """Return the region ID for names. If no match, add to the BrainRegion. diff --git a/src/spyglass/common/common_ripple.py b/src/spyglass/common/common_ripple.py index 154c937af..3bef3fa9a 100644 --- a/src/spyglass/common/common_ripple.py +++ b/src/spyglass/common/common_ripple.py @@ -2,6 +2,7 @@ import matplotlib.pyplot as plt import numpy as np import pandas as pd +from matplotlib.axes import Axes from ripple_detection import Karlsson_ripple_detector, Kay_ripple_detector from ripple_detection.core import gaussian_smooth, get_envelope @@ -54,16 +55,17 @@ def insert1(self, key, **kwargs): @staticmethod def set_lfp_electrodes( key, - electrode_list=None, - group_name="CA1", + electrode_list: list = None, + group_name: str = "CA1", **kwargs, ): - """Removes all electrodes for the specified nwb file and then adds back the electrodes in the list + """Replaces all electrodes for an nwb file with specified electrodes. Parameters ---------- key : dict - dictionary corresponding to the LFPBand entry to use for ripple detection + dictionary corresponding to the LFPBand entry to use for ripple + detection electrode_list : list list of electrodes from LFPBandSelection.LFPBandElectrode to be used as the ripple LFP during detection @@ -267,7 +269,7 @@ def get_ripple_lfps_and_position_info(key): @staticmethod def get_Kay_ripple_consensus_trace( - ripple_filtered_lfps, sampling_frequency, smoothing_sigma=0.004 + ripple_filtered_lfps, sampling_frequency, smoothing_sigma: float = 0.004 ): ripple_consensus_trace = np.full_like(ripple_filtered_lfps, np.nan) not_null = np.all(pd.notnull(ripple_filtered_lfps), axis=1) @@ -289,10 +291,10 @@ def get_Kay_ripple_consensus_trace( def plot_ripple_consensus_trace( ripple_consensus_trace, ripple_times, - ripple_label=1, - offset=0.100, - relative=True, - ax=None, + ripple_label: int = 1, + offset: float = 0.100, + relative: bool = True, + ax: Axes = None, ): ripple_start = ripple_times.loc[ripple_label].start_time ripple_end = ripple_times.loc[ripple_label].end_time @@ -319,7 +321,12 @@ def plot_ripple_consensus_trace( @staticmethod def plot_ripple( - lfps, ripple_times, ripple_label=1, offset=0.100, relative=True, ax=None + lfps, + ripple_times, + ripple_label: int = 1, + offset: float = 0.100, + relative: bool = True, + ax: Axes = None, ): lfp_labels = lfps.columns n_lfps = len(lfp_labels) diff --git a/src/spyglass/common/common_session.py b/src/spyglass/common/common_session.py index 893b727b5..7ea12e0d3 100644 --- a/src/spyglass/common/common_session.py +++ b/src/spyglass/common/common_session.py @@ -19,8 +19,8 @@ class Session(SpyglassMixin, dj.Imported): definition = """ # Table for holding experimental sessions. - # Note that each session can have multiple experimenters and data acquisition devices. See DataAcquisitionDevice - # and Experimenter part tables below. + # Note that each session can have multiple experimenters and data acquisition + # devices. See DataAcquisitionDevice and Experimenter part tables below. -> Nwbfile --- -> [nullable] Subject @@ -35,26 +35,23 @@ class Session(SpyglassMixin, dj.Imported): class DataAcquisitionDevice(SpyglassMixin, dj.Part): definition = """ - # Part table that allows a Session to be associated with multiple DataAcquisitionDevice entries. + # Part table linking Session to multiple DataAcquisitionDevice entries. -> Session -> DataAcquisitionDevice """ - # NOTE: as a Part table, it is generally advised not to delete entries directly - # (see https://docs.datajoint.org/python/computation/03-master-part.html), + # NOTE: as a Part table, it is ill advised to delete entries directly + # (https://docs.datajoint.org/python/computation/03-master-part.html), # but you can use `delete(force=True)`. class Experimenter(SpyglassMixin, dj.Part): definition = """ - # Part table that allows a Session to be associated with multiple LabMember entries. + # Part table linking Session to multiple LabMember entries. -> Session -> LabMember """ def make(self, key): - """Make without transaction - - Allows populate_all_common to work within a single transaction.""" # These imports must go here to avoid cyclic dependencies # from .common_task import Task, TaskEpoch from .common_interval import IntervalList @@ -118,8 +115,9 @@ def make(self, key): logger.info("Skipping Apparatus for now...") # Apparatus().insert_from_nwbfile(nwbf) - # interval lists depend on Session (as a primary key) but users may want to add these manually so this is - # a manual table that is also populated from NWB files + # interval lists depend on Session (as a primary key) but users may + # want to add these manually so this is a manual table that is also + # populated from NWB files logger.info("Session populates IntervalList...") IntervalList().insert_from_nwbfile(nwbf, nwb_file_name=nwb_file_name) @@ -152,8 +150,11 @@ def _add_data_acquisition_device_part(self, nwb_file_name, nwbf, config={}): key["data_acquisition_device_name"] = device_name Session.DataAcquisitionDevice.insert1(key) - def _add_experimenter_part(self, nwb_file_name, nwbf, config={}): + def _add_experimenter_part( + self, nwb_file_name: str, nwbf, config: dict = None + ): # Use config file over nwb file + config = config or dict() if members := config.get("LabMember"): experimenter_list = [ member["lab_member_name"] for member in members diff --git a/src/spyglass/common/common_subject.py b/src/spyglass/common/common_subject.py index b64f27b69..2b8dc071a 100644 --- a/src/spyglass/common/common_subject.py +++ b/src/spyglass/common/common_subject.py @@ -1,4 +1,5 @@ import datajoint as dj +from pynwb import NWBFile from spyglass.utils import SpyglassMixin, logger @@ -18,14 +19,14 @@ class Subject(SpyglassMixin, dj.Manual): """ @classmethod - def insert_from_nwbfile(cls, nwbf, config=None): + def insert_from_nwbfile(cls, nwbf: NWBFile, config: dict = None): """Get the subject info from the NWBFile, insert into the Subject. Parameters ---------- nwbf: pynwb.NWBFile The NWB file with subject information. - config : dict + config : dict, optional Dictionary read from a user-defined YAML file containing values to replace in the NWB file. diff --git a/src/spyglass/common/common_task.py b/src/spyglass/common/common_task.py index d63901ec2..94ec34c58 100644 --- a/src/spyglass/common/common_task.py +++ b/src/spyglass/common/common_task.py @@ -23,7 +23,7 @@ class Task(SpyglassMixin, dj.Manual): """ @classmethod - def insert_from_nwbfile(cls, nwbf): + def insert_from_nwbfile(cls, nwbf: pynwb.NWBFile): """Insert tasks from an NWB file. Parameters @@ -40,7 +40,7 @@ def insert_from_nwbfile(cls, nwbf): cls.insert_from_task_table(task) @classmethod - def insert_from_task_table(cls, task_table): + def insert_from_task_table(cls, task_table: pynwb.core.DynamicTable): """Insert tasks from a pynwb DynamicTable containing task metadata. Duplicate tasks will not be added. @@ -51,19 +51,23 @@ def insert_from_task_table(cls, task_table): The table representing task metadata. """ taskdf = task_table.to_dataframe() - for task_entry in taskdf.iterrows(): - task_dict = dict() - task_dict["task_name"] = task_entry[1].task_name - task_dict["task_description"] = task_entry[1].task_description - cls.insert1(task_dict, skip_duplicates=True) - @classmethod - def check_task_table(cls, task_table): - """Check whether the pynwb DynamicTable containing task metadata conforms to the expected format. + task_dicts = taskdf.apply( + lambda row: dict( + task_name=row.task_name, + task_description=row.task_description, + ), + axis=1, + ).tolist() + + cls.insert(task_dicts, skip_duplicates=True) + @classmethod + def check_task_table(cls, task_table: pynwb.core.DynamicTable) -> bool: + """Check format of pynwb DynamicTable containing task metadata. - The table should be an instance of pynwb.core.DynamicTable and contain the columns 'task_name' and - 'task_description'. + The table should be an instance of pynwb.core.DynamicTable and contain + the columns 'task_name' and 'task_description'. Parameters ---------- @@ -73,7 +77,8 @@ def check_task_table(cls, task_table): Returns ------- bool - Whether the DynamicTable conforms to the expected format for loading data into the Task table. + Whether the DynamicTable conforms to the expected format for loading + data into the Task table. """ return ( isinstance(task_table, pynwb.core.DynamicTable) @@ -174,7 +179,7 @@ def make(self, key): self.insert(task_inserts, allow_direct_insert=True) @classmethod - def update_entries(cls, restrict={}): + def update_entries(cls, restrict=True): existing_entries = (cls & restrict).fetch("KEY") for row in existing_entries: if (cls & row).fetch1("camera_names"): @@ -185,9 +190,8 @@ def update_entries(cls, restrict={}): cls.update1(row=row) @classmethod - def check_task_table(cls, task_table) -> bool: - """Check whether the pynwb DynamicTable containing task metadata - conforms to the expected format. + def check_task_table(cls, task_table: pynwb.core.DynamicTable) -> bool: + """Check format of pynwb DynamicTable containing task metadata. The table should be an instance of pynwb.core.DynamicTable and contain the columns 'task_name', 'task_description', 'camera_id', 'and diff --git a/src/spyglass/common/prepopulate/prepopulate.py b/src/spyglass/common/prepopulate/prepopulate.py index e77e68ef5..ecad63a25 100644 --- a/src/spyglass/common/prepopulate/prepopulate.py +++ b/src/spyglass/common/prepopulate/prepopulate.py @@ -77,10 +77,11 @@ def populate_from_yaml(yaml_path: str): ) -def _get_table_cls(table_name): +def _get_table_cls(table_name: str): """Get the spyglass.common class associated with a given table name. - Also works for part tables one level deep.""" + Also works for part tables one level deep. + """ if "." in table_name: # part table master_table_name = table_name[0 : table_name.index(".")] diff --git a/src/spyglass/common/signal_processing.py b/src/spyglass/common/signal_processing.py index 4603dc321..1d3c89673 100644 --- a/src/spyglass/common/signal_processing.py +++ b/src/spyglass/common/signal_processing.py @@ -2,6 +2,8 @@ import pynwb import scipy.signal as signal +from spyglass.common.common_usage import ActivityLog + def hilbert_decomp(lfp_band_object, sampling_rate=1): """Generates analytical decomposition of signals in the lfp_band_object @@ -21,6 +23,8 @@ def hilbert_decomp(lfp_band_object, sampling_rate=1): envelope : pynwb.ecephys.ElectricalSeries envelope of the signal """ + ActivityLog().deprecate_log("common.signal_processing.hilbert_decomp") + analytical_signal = signal.hilbert(lfp_band_object.data, axis=0) eseries_name = "envelope" diff --git a/src/spyglass/data_import/insert_sessions.py b/src/spyglass/data_import/insert_sessions.py index b4dc1d406..e43645d70 100644 --- a/src/spyglass/data_import/insert_sessions.py +++ b/src/spyglass/data_import/insert_sessions.py @@ -138,8 +138,9 @@ def copy_nwb_link_raw_ephys(nwb_file_name, out_nwb_file_name): ) as export_io: export_io.export(input_io, nwbf) - # add link from new file back to raw ephys data in raw data file using fresh build manager and container cache - # where the acquisition electricalseries objects have not been removed + # add link from new file back to raw ephys data in raw data file using + # fresh build manager and container cache where the acquisition + # electricalseries objects have not been removed with pynwb.NWBHDF5IO( path=nwb_file_abs_path, mode="r", load_namespaces=True ) as input_io: diff --git a/src/spyglass/decoding/decoding_merge.py b/src/spyglass/decoding/decoding_merge.py index de890590f..15483c0c5 100644 --- a/src/spyglass/decoding/decoding_merge.py +++ b/src/spyglass/decoding/decoding_merge.py @@ -3,7 +3,6 @@ import datajoint as dj import numpy as np -from datajoint.utils import to_camel_case from non_local_detector.visualization.figurl_1D import create_1D_decode_view from non_local_detector.visualization.figurl_2D import create_2D_decode_view diff --git a/src/spyglass/decoding/utils.py b/src/spyglass/decoding/utils.py new file mode 100644 index 000000000..c8a26e9b7 --- /dev/null +++ b/src/spyglass/decoding/utils.py @@ -0,0 +1,43 @@ +import numpy as np +import spikeinterface as si + + +def _get_peak_amplitude( + waveform_extractor: si.WaveformExtractor, + unit_id: int, + peak_sign: str = "neg", + estimate_peak_time: bool = False, +) -> np.ndarray: + """Returns the amplitudes of all channels at the time of the peak + amplitude across channels. + + Parameters + ---------- + waveform : array-like, shape (n_spikes, n_time, n_channels) + peak_sign : ('pos', 'neg', 'both'), optional + Direction of the peak in the waveform + estimate_peak_time : bool, optional + Find the peak times for each spike because some spikesorters do not + align the spike time (at index n_time // 2) to the peak + + Returns + ------- + peak_amplitudes : array-like, shape (n_spikes, n_channels) + + """ + waveforms = waveform_extractor.get_waveforms(unit_id) + if estimate_peak_time: + if peak_sign == "neg": + peak_inds = np.argmin(np.min(waveforms, axis=2), axis=1) + elif peak_sign == "pos": + peak_inds = np.argmax(np.max(waveforms, axis=2), axis=1) + elif peak_sign == "both": + peak_inds = np.argmax(np.max(np.abs(waveforms), axis=2), axis=1) + + # Get mode of peaks to find the peak time + values, counts = np.unique(peak_inds, return_counts=True) + spike_peak_ind = values[counts.argmax()] + else: + spike_peak_ind = waveforms.shape[1] // 2 + + return waveforms[:, spike_peak_ind] diff --git a/src/spyglass/decoding/v0/clusterless.py b/src/spyglass/decoding/v0/clusterless.py index 69f8b1c62..c7d3e4b5a 100644 --- a/src/spyglass/decoding/v0/clusterless.py +++ b/src/spyglass/decoding/v0/clusterless.py @@ -1,5 +1,6 @@ -"""Pipeline for decoding the animal's mental position and some category of interest -from unclustered spikes and spike waveform features. See [1] for details. +"""Pipeline for decoding the animal's mental position and some category of +interest from unclustered spikes and spike waveform features. See [1] for +details. References ---------- @@ -54,6 +55,7 @@ from spyglass.common.common_interval import IntervalList from spyglass.common.common_nwbfile import AnalysisNwbfile from spyglass.common.common_position import IntervalPositionInfo +from spyglass.decoding.utils import _get_peak_amplitude from spyglass.decoding.v0.core import ( convert_valid_times_to_slice, get_valid_ephys_position_times_by_epoch, @@ -62,6 +64,10 @@ convert_classes_to_dict, restore_classes, ) +from spyglass.decoding.v0.utils import ( + get_time_bins_from_interval, + make_default_decoding_params, +) from spyglass.spikesorting.v0.spikesorting_curation import ( CuratedSpikeSorting, CuratedSpikeSortingSelection, @@ -199,9 +205,10 @@ def make(self, key): marks = np.concatenate( [ - UnitMarks._get_peak_amplitude( - waveform=waveform_extractor.get_waveforms(unit_id), + _get_peak_amplitude( + waveform_extractor=waveform_extractor, peak_sign=peak_sign, + unit_id=unit_id, estimate_peak_time=estimate_peak_time, ) for unit_id in nwb_units.index @@ -251,48 +258,6 @@ def _convert_to_dataframe(nwb_data) -> pd.DataFrame: columns=columns, ) - @staticmethod - def _get_peak_amplitude( - waveform: np.array, - peak_sign: str = "neg", - estimate_peak_time: bool = False, - ) -> np.array: - """Returns the amplitudes of all channels at the time of the peak. - - Amplitude across channels. - - Parameters - ---------- - waveform : np.array - array-like, shape (n_spikes, n_time, n_channels) - peak_sign : str, optional - One of 'pos', 'neg', 'both'. Direction of the peak in the waveform - estimate_peak_time : bool, optional - Find the peak times for each spike because some spikesorters do not - align the spike time (at index n_time // 2) to the peak - - Returns - ------- - peak_amplitudes : np.array - array-like, shape (n_spikes, n_channels) - - """ - if estimate_peak_time: - if peak_sign == "neg": - peak_inds = np.argmin(np.min(waveform, axis=2), axis=1) - elif peak_sign == "pos": - peak_inds = np.argmax(np.max(waveform, axis=2), axis=1) - elif peak_sign == "both": - peak_inds = np.argmax(np.max(np.abs(waveform), axis=2), axis=1) - - # Get mode of peaks to find the peak time - values, counts = np.unique(peak_inds, return_counts=True) - spike_peak_ind = values[counts.argmax()] - else: - spike_peak_ind = waveform.shape[1] // 2 - - return waveform[:, spike_peak_ind] - @staticmethod def _threshold( timestamps: np.array, marks: np.array, mark_param_dict: dict @@ -368,7 +333,7 @@ def make(self, key): marks_df = (UnitMarks & key).fetch1_dataframe() - time = self.get_time_bins_from_interval(interval_times, sampling_rate) + time = get_time_bins_from_interval(interval_times, sampling_rate) # Bin marks into time bins. No spike bins will have NaN marks_df = marks_df.loc[time.min() : time.max()] @@ -397,16 +362,6 @@ def make(self, key): self.insert1(key) - @staticmethod - def get_time_bins_from_interval( - interval_times: np.array, sampling_rate: int - ) -> np.array: - """Picks the superset of the interval""" - start_time, end_time = interval_times[0][0], interval_times[-1][-1] - n_samples = int(np.ceil((end_time - start_time) * sampling_rate)) + 1 - - return np.linspace(start_time, end_time, n_samples) - @staticmethod def plot_all_marks( marks_indicators: xr.DataArray, @@ -509,72 +464,6 @@ def reformat_name(name): ) -def make_default_decoding_parameters_cpu() -> tuple[dict, dict, dict]: - """Default parameters for decoding on CPU - - Returns - ------- - classifier_parameters : dict - fit_parameters : dict - predict_parameters : dict - """ - - classifier_parameters = dict( - environments=[_DEFAULT_ENVIRONMENT], - observation_models=None, - continuous_transition_types=_DEFAULT_CONTINUOUS_TRANSITIONS, - discrete_transition_type=DiagonalDiscrete(0.98), - initial_conditions_type=UniformInitialConditions(), - infer_track_interior=True, - clusterless_algorithm="multiunit_likelihood_integer", - clusterless_algorithm_params=_DEFAULT_CLUSTERLESS_MODEL_KWARGS, - ) - - predict_parameters = { - "is_compute_acausal": True, - "use_gpu": False, - "state_names": ["Continuous", "Fragmented"], - } - fit_parameters = dict() - - return classifier_parameters, fit_parameters, predict_parameters - - -def make_default_decoding_parameters_gpu() -> tuple[dict, dict, dict]: - """Default parameters for decoding on GPU - - Returns - ------- - classifier_parameters : dict - fit_parameters : dict - predict_parameters : dict - """ - - classifier_parameters = dict( - environments=[_DEFAULT_ENVIRONMENT], - observation_models=None, - continuous_transition_types=_DEFAULT_CONTINUOUS_TRANSITIONS, - discrete_transition_type=DiagonalDiscrete(0.98), - initial_conditions_type=UniformInitialConditions(), - infer_track_interior=True, - clusterless_algorithm="multiunit_likelihood_integer_gpu", - clusterless_algorithm_params={ - "mark_std": 24.0, - "position_std": 6.0, - }, - ) - - predict_parameters = { - "is_compute_acausal": True, - "use_gpu": True, - "state_names": ["Continuous", "Fragmented"], - } - - fit_parameters = dict() - - return classifier_parameters, fit_parameters, predict_parameters - - @schema class ClusterlessClassifierParameters(SpyglassMixin, dj.Manual): """Decodes animal's mental position. @@ -592,34 +481,12 @@ class ClusterlessClassifierParameters(SpyglassMixin, dj.Manual): """ def insert_default(self) -> None: - """Insert the default parameter set""" - ( - classifier_parameters, - fit_parameters, - predict_parameters, - ) = make_default_decoding_parameters_cpu() - self.insert1( - { - "classifier_param_name": "default_decoding_cpu", - "classifier_params": classifier_parameters, - "fit_params": fit_parameters, - "predict_params": predict_parameters, - }, - skip_duplicates=True, - ) - - ( - classifier_parameters, - fit_parameters, - predict_parameters, - ) = make_default_decoding_parameters_gpu() - self.insert1( - { - "classifier_param_name": "default_decoding_gpu", - "classifier_params": classifier_parameters, - "fit_params": fit_parameters, - "predict_params": predict_parameters, - }, + """Insert the default parameter sets""" + self.insert( + [ + make_default_decoding_params(clusterless=True), + make_default_decoding_params(clusterless=True, use_gpu=True), + ], skip_duplicates=True, ) @@ -661,6 +528,8 @@ def get_decoding_data_for_epoch( interval_list_name ] valid_slices = convert_valid_times_to_slice(valid_ephys_position_times) + + # position interval position_interval_name = ( convert_epoch_interval_name_to_position_interval_name( { @@ -727,6 +596,7 @@ def get_data_for_multiple_epochs( environment_labels = [] for epoch in epoch_names: + logger.info(epoch) data.append( get_decoding_data_for_epoch( nwb_file_name, diff --git a/src/spyglass/decoding/v0/core.py b/src/spyglass/decoding/v0/core.py index 42cbb59d4..0751598cd 100644 --- a/src/spyglass/decoding/v0/core.py +++ b/src/spyglass/decoding/v0/core.py @@ -26,8 +26,10 @@ def get_valid_ephys_position_times_from_interval( interval_list_name: str, nwb_file_name: str ) -> np.ndarray: - """Finds the intersection of the valid times for the interval list, the valid times for the ephys data, - and the valid times for the position data. + """Returns the intersection of valid times across ephys and position data. + + Finds the intersection of the valid times for the interval list, the + valid times for the ephys data, and the valid times for the position data. Parameters ---------- @@ -135,7 +137,7 @@ def get_valid_ephys_position_times_by_epoch( def convert_valid_times_to_slice(valid_times: np.ndarray) -> list[slice]: - """Converts the valid times to a list of slices so that arrays can be indexed easily. + """Converts valid times to a list of slices for easy indexing. Parameters ---------- @@ -153,7 +155,8 @@ def convert_valid_times_to_slice(valid_times: np.ndarray) -> list[slice]: def create_model_for_multiple_epochs( epoch_names: list[str], env_kwargs: dict ) -> tuple[list[ObservationModel], list[Environment], list[list[object]]]: - """Creates the observation model, environment, and continuous transition types for multiple epochs for decoding + """Creates the observation model, environment, and continuous transition + types for multiple epochs for decoding Parameters ---------- diff --git a/src/spyglass/decoding/v0/sorted_spikes.py b/src/spyglass/decoding/v0/sorted_spikes.py index 36f171219..7cdfaa810 100644 --- a/src/spyglass/decoding/v0/sorted_spikes.py +++ b/src/spyglass/decoding/v0/sorted_spikes.py @@ -1,5 +1,5 @@ -"""Pipeline for decoding the animal's mental position and some category of interest -from clustered spikes times. See [1] for details. +"""Pipeline for decoding the animal's mental position and some category of +interest from clustered spikes times. See [1] for details. References ---------- @@ -52,6 +52,10 @@ convert_classes_to_dict, restore_classes, ) +from spyglass.decoding.v0.utils import ( + get_time_bins_from_interval, + make_default_decoding_params, +) from spyglass.spikesorting.v0.spikesorting_curation import CuratedSpikeSorting from spyglass.utils import SpyglassMixin, logger @@ -95,7 +99,7 @@ def make(self, key): "sampling_rate" ) - time = self.get_time_bins_from_interval(interval_times, sampling_rate) + time = get_time_bins_from_interval(interval_times, sampling_rate) spikes_nwb = (CuratedSpikeSorting & key).fetch_nwb() # restrict to cases with units @@ -153,14 +157,6 @@ def make(self, key): self.insert1(key) - @staticmethod - def get_time_bins_from_interval(interval_times, sampling_rate): - """Gets the superset of the interval.""" - start_time, end_time = interval_times[0][0], interval_times[-1][-1] - n_samples = int(np.ceil((end_time - start_time) * sampling_rate)) + 1 - - return np.linspace(start_time, end_time, n_samples) - def fetch1_dataframe(self): return self.fetch_dataframe()[0] @@ -174,51 +170,6 @@ def fetch_dataframe(self): ) -def make_default_decoding_parameters_cpu(): - classifier_parameters = dict( - environments=[_DEFAULT_ENVIRONMENT], - observation_models=None, - continuous_transition_types=_DEFAULT_CONTINUOUS_TRANSITIONS, - discrete_transition_type=DiagonalDiscrete(0.98), - initial_conditions_type=UniformInitialConditions(), - infer_track_interior=True, - knot_spacing=10, - spike_model_penalty=1e1, - ) - - predict_parameters = { - "is_compute_acausal": True, - "use_gpu": False, - "state_names": ["Continuous", "Fragmented"], - } - fit_parameters = dict() - - return classifier_parameters, fit_parameters, predict_parameters - - -def make_default_decoding_parameters_gpu(): - classifier_parameters = dict( - environments=[_DEFAULT_ENVIRONMENT], - observation_models=None, - continuous_transition_types=_DEFAULT_CONTINUOUS_TRANSITIONS, - discrete_transition_type=DiagonalDiscrete(0.98), - initial_conditions_type=UniformInitialConditions(), - infer_track_interior=True, - sorted_spikes_algorithm="spiking_likelihood_kde", - sorted_spikes_algorithm_params=_DEFAULT_SORTED_SPIKES_MODEL_KWARGS, - ) - - predict_parameters = { - "is_compute_acausal": True, - "use_gpu": True, - "state_names": ["Continuous", "Fragmented"], - } - - fit_parameters = dict() - - return classifier_parameters, fit_parameters, predict_parameters - - @schema class SortedSpikesClassifierParameters(SpyglassMixin, dj.Manual): """Stores parameters for decoding with sorted spikes""" @@ -232,33 +183,11 @@ class SortedSpikesClassifierParameters(SpyglassMixin, dj.Manual): """ def insert_default(self): - ( - classifier_parameters, - fit_parameters, - predict_parameters, - ) = make_default_decoding_parameters_cpu() - self.insert1( - { - "classifier_param_name": "default_decoding_cpu", - "classifier_params": classifier_parameters, - "fit_params": fit_parameters, - "predict_params": predict_parameters, - }, - skip_duplicates=True, - ) - - ( - classifier_parameters, - fit_parameters, - predict_parameters, - ) = make_default_decoding_parameters_gpu() - self.insert1( - { - "classifier_param_name": "default_decoding_gpu", - "classifier_params": classifier_parameters, - "fit_params": fit_parameters, - "predict_params": predict_parameters, - }, + self.insert( + [ + make_default_decoding_params(), + make_default_decoding_params(use_gpu=True), + ], skip_duplicates=True, ) @@ -272,7 +201,7 @@ def fetch1(self, *args, **kwargs): def get_spike_indicator( key: dict, time_range: tuple[float, float], sampling_rate: float = 500.0 ) -> pd.DataFrame: - """For a given key, returns a dataframe with the spike indicator for each unit + """Returns a dataframe with the spike indicator for each unit Parameters ---------- diff --git a/src/spyglass/decoding/v0/utils.py b/src/spyglass/decoding/v0/utils.py new file mode 100644 index 000000000..7f1ae1c15 --- /dev/null +++ b/src/spyglass/decoding/v0/utils.py @@ -0,0 +1,124 @@ +import numpy as np +import xarray as xr + +from spyglass.utils import logger + +try: + from replay_trajectory_classification.classifier import ( + _DEFAULT_CLUSTERLESS_MODEL_KWARGS, + _DEFAULT_CONTINUOUS_TRANSITIONS, + _DEFAULT_ENVIRONMENT, + _DEFAULT_SORTED_SPIKES_MODEL_KWARGS, + ) + from replay_trajectory_classification.discrete_state_transitions import ( + DiagonalDiscrete, + ) + from replay_trajectory_classification.initial_conditions import ( + UniformInitialConditions, + ) +except (ImportError, ModuleNotFoundError) as e: + ( + _DEFAULT_CLUSTERLESS_MODEL_KWARGS, + _DEFAULT_CONTINUOUS_TRANSITIONS, + _DEFAULT_ENVIRONMENT, + _DEFAULT_SORTED_SPIKES_MODEL_KWARGS, + DiagonalDiscrete, + UniformInitialConditions, + ) = [None] * 6 + logger.warning(e) + + +def get_time_bins_from_interval(interval_times: np.array, sampling_rate: int): + """Gets the superset of the interval.""" + start_time, end_time = interval_times[0][0], interval_times[-1][-1] + n_samples = int(np.ceil((end_time - start_time) * sampling_rate)) + 1 + + return np.linspace(start_time, end_time, n_samples) + + +def discretize_and_trim(series: xr.DataArray, ndims=1) -> xr.DataArray: + """Discretizes a continuous series and trims the zeros. + + Parameters + ---------- + series : xr.DataArray, shape (n_time, n_position_bins) + Continuous series to be discretized + ndims : int, optional + Number of spatial dimensions of the series. Default is 1 for 1D series + (time, position), 2 for 2D series (time, y_position, x_position) + + Returns + ------- + discretized : xr.DataArray, shape (n_time, n_position_bins) + Discretized and trimmed series + """ + if ndims not in [1, 2]: + raise ValueError("ndims must be 1 or 2 spatial dimensions.") + + index = ( + ["time", "position"] + if ndims == 1 + else ["time", "y_position", "x_position"] + ) + discretized = np.multiply(series, 255).astype(np.uint8) # type: ignore + stacked = discretized.stack(unified_index=index) + return stacked.where(stacked > 0, drop=True).astype(np.uint8) + + +def make_default_decoding_params(clusterless=False, use_gpu=False): + """Default parameters for decoding + + Returns + ------- + classifier_parameters : dict + fit_parameters : dict + predict_parameters : dict + """ + + classifier_params = dict( + environments=[_DEFAULT_ENVIRONMENT], + observation_models=None, + continuous_transition_types=_DEFAULT_CONTINUOUS_TRANSITIONS, + discrete_transition_type=DiagonalDiscrete(0.98), + initial_conditions_type=UniformInitialConditions(), + infer_track_interior=True, + ) + + if clusterless: + clusterless_algorithm = ( + "multiunit_likelihood_integer_gpu" + if use_gpu + else "multiunit_likelihood_integer" + ) + classifier_params.update( + dict( + clusterless_algorithm=clusterless_algorithm, + clusterless_algorithm_params=_DEFAULT_CLUSTERLESS_MODEL_KWARGS, + ) + ) + else: + extra_params = ( + dict( + sorted_spikes_algorithm="spiking_likelihood_kde", + sorted_spikes_algorithm_params=_DEFAULT_SORTED_SPIKES_MODEL_KWARGS, + ) + if use_gpu + else dict(knot_spacing=10, spike_model_penalty=1e1) + ) + classifier_params.update(extra_params) + + predict_params = { + "is_compute_acausal": True, + "use_gpu": use_gpu, + "state_names": ["Continuous", "Fragmented"], + } + fit_params = dict() + + return dict( + classifier_params_name=( + "default_decoding_gpu" if use_gpu else "default_decoding_cpu" + ), + classifier_params=classifier_params, + fit_params=fit_params, + predict_params=predict_params, + ) diff --git a/src/spyglass/decoding/v0/visualization_1D_view.py b/src/spyglass/decoding/v0/visualization_1D_view.py index c118dfe90..fd69b3a9e 100644 --- a/src/spyglass/decoding/v0/visualization_1D_view.py +++ b/src/spyglass/decoding/v0/visualization_1D_view.py @@ -4,11 +4,7 @@ import sortingview.views.franklab as vvf import xarray as xr - -def discretize_and_trim(series: xr.DataArray) -> xr.DataArray: - discretized = np.multiply(series, 255).astype(np.uint8) # type: ignore - stacked = discretized.stack(unified_index=["time", "position"]) - return stacked.where(stacked > 0, drop=True).astype(np.uint8) +from spyglass.decoding.v0.utils import discretize_and_trim def get_observations_per_time( @@ -16,7 +12,9 @@ def get_observations_per_time( ) -> np.ndarray: times, counts = np.unique(trimmed_posterior.time.values, return_counts=True) indexed_counts = xr.DataArray(counts, coords={"time": times}) - _, good_counts = xr.align(base_data.time, indexed_counts, join="left", fill_value=0) # type: ignore + _, good_counts = xr.align( + base_data.time, indexed_counts, join="left", fill_value=0 + ) # type: ignore return good_counts.values.astype(np.uint8) @@ -57,7 +55,7 @@ def create_1D_decode_view( if linear_position is not None: linear_position = np.asarray(linear_position).squeeze() - trimmed_posterior = discretize_and_trim(posterior) + trimmed_posterior = discretize_and_trim(posterior, ndims=1) observations_per_time = get_observations_per_time( trimmed_posterior, posterior ) diff --git a/src/spyglass/decoding/v0/visualization_2D_view.py b/src/spyglass/decoding/v0/visualization_2D_view.py index f0264f102..948a20dd1 100644 --- a/src/spyglass/decoding/v0/visualization_2D_view.py +++ b/src/spyglass/decoding/v0/visualization_2D_view.py @@ -4,6 +4,8 @@ import sortingview.views.franklab as vvf import xarray as xr +from spyglass.decoding.v0.utils import discretize_and_trim + def create_static_track_animation( *, @@ -109,13 +111,6 @@ def inner(t: Tuple[float, float, float]): return inner -def discretize_and_trim(base_slice: xr.DataArray): - i = np.multiply(base_slice, 255).astype(np.uint8) - i_stack = i.stack(unified_index=["time", "y_position", "x_position"]) - - return i_stack.where(i_stack > 0, drop=True).astype(np.uint8) - - def get_positions( i_trim: xr.Dataset, linearization_fn: Callable[[Tuple[float, float]], int] ): @@ -137,7 +132,7 @@ def get_observations_per_frame(i_trim: xr.DataArray, base_slice: xr.DataArray): def extract_slice_data( base_slice: xr.DataArray, location_fn: Callable[[Tuple[float, float]], int] ): - i_trim = discretize_and_trim(base_slice) + i_trim = discretize_and_trim(base_slice, ndims=2) positions = get_positions(i_trim, location_fn) observations_per_frame = get_observations_per_frame(i_trim, base_slice) @@ -163,10 +158,14 @@ def process_decoded_data(posterior: xr.DataArray): total_frame_count = len(posterior.time) final_frame_bounds = np.zeros(total_frame_count, dtype=np.uint8) # intentionally oversized preallocation--will trim later - # Note: By definition there can't be more than 255 observations per frame (since we drop any observation - # lower than 1/255 and the probabilities for any frame sum to 1). However, this preallocation may be way - # too big for memory for long recordings. We could use a smaller one, but would need to include logic - # to expand the length of the array if its actual allocated bounds are exceeded. + + # Note: By definition there can't be more than 255 observations per frame + # (since we drop any observation lower than 1/255 and the probabilities for + # any frame sum to 1). However, this preallocation may be way too big for + # memory for long recordings. We could use a smaller one, but would need to + # include logic to expand the length of the array if its actual allocated + # bounds are exceeded. + final_values = np.zeros(total_frame_count * 255, dtype=np.uint8) final_locations = np.zeros(total_frame_count * 255, dtype=np.uint16) @@ -190,7 +189,10 @@ def process_decoded_data(posterior: xr.DataArray): ] = positions total_observations += len(observations) frames_done += frame_step_size - # These were intentionally oversized in preallocation; trim to the number of actual values. + + # These were intentionally oversized in preallocation; trim to the number + # of actual values. + final_values.resize(total_observations) final_locations.resize(total_observations) @@ -301,8 +303,11 @@ def create_2D_decode_view( track_bin_width = place_bin_size[0] track_bin_height = place_bin_size[1] + # NOTE: We expect caller to have converted from fortran ordering already - # i.e. somewhere upstream, centers = env.place_bin_centers_[env.is_track_interior_.ravel(order="F")] + # i.e. somewhere upstream, centers = + # env.place_bin_centers_[env.is_track_interior_.ravel(order="F")] + upper_left_points = get_ul_corners( track_bin_width, track_bin_height, interior_place_bin_centers ) diff --git a/src/spyglass/decoding/v1/clusterless.py b/src/spyglass/decoding/v1/clusterless.py index 881551922..3694a550f 100644 --- a/src/spyglass/decoding/v1/clusterless.py +++ b/src/spyglass/decoding/v1/clusterless.py @@ -1,5 +1,6 @@ -"""Pipeline for decoding the animal's mental position and some category of interest -from unclustered spikes and spike waveform features. See [1] for details. +"""Pipeline for decoding the animal's mental position and some category of +interest from unclustered spikes and spike waveform features. See [1] for +details. References ---------- @@ -23,10 +24,8 @@ from spyglass.common.common_interval import IntervalList # noqa: F401 from spyglass.common.common_session import Session # noqa: F401 -from spyglass.decoding.v1.core import ( - DecodingParameters, - PositionGroup, -) # noqa: F401 +from spyglass.decoding.v1.core import DecodingParameters # noqa: F401 +from spyglass.decoding.v1.core import PositionGroup from spyglass.decoding.v1.waveform_features import ( UnitWaveformFeatures, ) # noqa: F401 @@ -76,7 +75,7 @@ class ClusterlessDecodingSelection(SpyglassMixin, dj.Manual): -> DecodingParameters -> IntervalList.proj(encoding_interval='interval_list_name') -> IntervalList.proj(decoding_interval='interval_list_name') - estimate_decoding_params = 1 : bool # whether to estimate the decoding parameters + estimate_decoding_params = 1 : bool # 1 to estimate the decoding parameters """ @@ -109,8 +108,9 @@ def make(self, key): position_variable_names, ) = self.fetch_position_info(key) - # Get the waveform features for the selected units - # Don't need to filter by interval since the non_local_detector code will do that + # Get the waveform features for the selected units. Don't need to filter + # by interval since the non_local_detector code will do that + ( spike_times, spike_waveform_features, @@ -147,10 +147,13 @@ def make(self, key): classifier = ClusterlessDetector(**decoding_params) if key["estimate_decoding_params"]: - # if estimating parameters, then we need to treat times outside decoding interval as missing - # this means that times outside the decoding interval will not use the spiking data - # a better approach would be to treat the intervals as multiple sequences - # (see https://en.wikipedia.org/wiki/Baum%E2%80%93Welch_algorithm#Multiple_sequences) + + # if estimating parameters, then we need to treat times outside + # decoding interval as missing this means that times outside the + # decoding interval will not use the spiking data a better approach + # would be to treat the intervals as multiple sequences (see + # https://en.wikipedia.org/wiki/Baum%E2%80%93Welch_algorithm#Multiple_sequences) + is_missing = np.ones(len(position_info), dtype=bool) for interval_start, interval_end in decoding_interval: is_missing[ @@ -328,7 +331,7 @@ def fetch_environments(key): @staticmethod def _get_interval_range(key): - """Get the maximum range of model times in the encoding and decoding intervals + """Return max range of model times in the encoding/decoding intervals Parameters ---------- @@ -442,9 +445,10 @@ def fetch_spike_data(key, filter_by_interval=True): key : dict The decoding selection key filter_by_interval : bool, optional - Whether to filter for spike times in the model interval, by default True + Whether to filter for spike times in the model interval. + Default True time_slice : Slice, optional - User provided slice of time to restrict spikes to, by default None + User provided slice of time to restrict spikes to. Default None Returns ------- @@ -515,7 +519,13 @@ def get_spike_indicator(cls, key, time): return spike_indicator @classmethod - def get_firing_rate(cls, key, time, multiunit=False): + def get_firing_rate( + cls, + key: dict, + time: np.ndarray, + multiunit: bool = False, + smoothing_sigma: float = 0.015, + ) -> np.ndarray: """get time-dependent firing rate for units in the group Parameters @@ -525,14 +535,16 @@ def get_firing_rate(cls, key, time, multiunit=False): time : np.ndarray time vector for which to calculate the firing rate multiunit : bool, optional - if True, return the multiunit firing rate for units in the group, by default False + if True, return the multiunit firing rate for units in the group. + Default False smoothing_sigma : float, optional - standard deviation of gaussian filter to smooth firing rates in seconds, by default 0.015 + standard deviation of gaussian filter to smooth firing rates in + seconds. Default 0.015 Returns ------- np.ndarray - _description_ + time-dependent firing rate with shape (len(time), n_units) """ spike_indicator = cls.get_spike_indicator(key, time) if spike_indicator.ndim == 1: @@ -545,14 +557,21 @@ def get_firing_rate(cls, key, time, multiunit=False): return np.stack( [ get_multiunit_population_firing_rate( - indicator[:, np.newaxis], sampling_frequency + indicator[:, np.newaxis], + sampling_frequency, + smoothing_sigma, ) for indicator in spike_indicator.T ], axis=1, ) - def get_ahead_behind_distance(self): + def get_orientation_col(self, df): + """Examine columns of a input df and return orientation col name""" + cols = df.columns + return "orientation" if "orientation" in cols else "head_orientation" + + def get_ahead_behind_distance(self, track_graph=None, time_slice=None): """get the ahead-behind distance for the decoding model Returns @@ -564,24 +583,31 @@ def get_ahead_behind_distance(self): # TODO: allow specification of track graph # TODO: Handle decode intervals, store in table + if time_slice is None: + time_slice = slice(-np.inf, np.inf) + classifier = self.fetch_model() - results = self.fetch_results().squeeze() - posterior = results.acausal_posterior.unstack("state_bins").sum("state") + posterior = ( + self.fetch_results() + .acausal_posterior(time=time_slice) + .squeeze() + .unstack("state_bins") + .sum("state") + ) + + if track_graph is None: + track_graph = classifier.environments[0].track_graph - if getattr(classifier.environments[0], "track_graph") is not None: + if track_graph is not None: linear_position_info = self.fetch_linear_position_info( self.fetch1("KEY") - ) + ).loc[time_slice] - orientation_name = ( - "orientation" - if "orientation" in linear_position_info.columns - else "head_orientation" - ) + orientation_name = self.get_orientation_col(linear_position_info) traj_data = analysis.get_trajectory_data( posterior=posterior, - track_graph=classifier.environments[0].track_graph, + track_graph=track_graph, decoder=classifier, actual_projected_position=linear_position_info[ ["projected_x_position", "projected_y_position"] @@ -594,14 +620,12 @@ def get_ahead_behind_distance(self): classifier.environments[0].track_graph, *traj_data ) else: - position_info = self.fetch_position_info(self.fetch1("KEY")) + position_info = self.fetch_position_info(self.fetch1("KEY")).loc[ + time_slice + ] map_position = analysis.maximum_a_posteriori_estimate(posterior) - orientation_name = ( - "orientation" - if "orientation" in position_info.columns - else "head_orientation" - ) + orientation_name = self.get_orientation_col(position_info) position_variable_names = ( PositionGroup & self.fetch1("KEY") ).fetch1("position_variables") diff --git a/src/spyglass/decoding/v1/core.py b/src/spyglass/decoding/v1/core.py index 8771cc2d4..de733bb01 100644 --- a/src/spyglass/decoding/v1/core.py +++ b/src/spyglass/decoding/v1/core.py @@ -22,7 +22,7 @@ @schema class DecodingParameters(SpyglassMixin, dj.Lookup): - """Parameters for decoding the animal's mental position and some category of interest""" + """Params for decoding mental position and some category of interest""" definition = """ decoding_param_name : varchar(80) # a name for this set of parameters @@ -31,22 +31,25 @@ class DecodingParameters(SpyglassMixin, dj.Lookup): decoding_kwargs = NULL : BLOB # additional keyword arguments """ + pk = "decoding_param_name" + sk = "decoding_params" + contents = [ { - "decoding_param_name": f"contfrag_clusterless_{non_local_detector_version}", - "decoding_params": ContFragClusterlessClassifier(), + pk: f"contfrag_clusterless_{non_local_detector_version}", + sk: ContFragClusterlessClassifier(), }, { - "decoding_param_name": f"nonlocal_clusterless_{non_local_detector_version}", - "decoding_params": NonLocalClusterlessDetector(), + pk: f"nonlocal_clusterless_{non_local_detector_version}", + sk: NonLocalClusterlessDetector(), }, { - "decoding_param_name": f"contfrag_sorted_{non_local_detector_version}", - "decoding_params": ContFragSortedSpikesClassifier(), + pk: f"contfrag_sorted_{non_local_detector_version}", + sk: ContFragSortedSpikesClassifier(), }, { - "decoding_param_name": f"nonlocal_sorted_{non_local_detector_version}", - "decoding_params": NonLocalSortedSpikesDetector(), + pk: f"nonlocal_sorted_{non_local_detector_version}", + sk: NonLocalSortedSpikesDetector(), }, ] @@ -117,7 +120,7 @@ def create_group( } if self & group_key: raise ValueError( - f"Group {nwb_file_name}: {position_group_name} already exists", + f"Group {nwb_file_name}: {group_name} already exists", "please delete the group before creating a new one", ) self.insert1( @@ -147,9 +150,11 @@ def fetch_position_info( key : dict, optional restriction to a single entry in PositionGroup, by default None min_time : float, optional - restrict position information to times greater than min_time, by default None + restrict position information to times greater than min_time, + by default None max_time : float, optional - restrict position information to times less than max_time, by default None + restrict position information to times less than max_time, + by default None Returns ------- diff --git a/src/spyglass/decoding/v1/sorted_spikes.py b/src/spyglass/decoding/v1/sorted_spikes.py index e4d378cc4..bd44f8eb6 100644 --- a/src/spyglass/decoding/v1/sorted_spikes.py +++ b/src/spyglass/decoding/v1/sorted_spikes.py @@ -1,5 +1,5 @@ -"""Pipeline for decoding the animal's mental position and some category of interest -from clustered spikes times. See [1] for details. +"""Pipeline for decoding the animal's mental position and some category of +interest from clustered spikes times. See [1] for details. References ---------- @@ -23,10 +23,8 @@ from spyglass.common.common_interval import IntervalList # noqa: F401 from spyglass.common.common_session import Session # noqa: F401 -from spyglass.decoding.v1.core import ( - DecodingParameters, - PositionGroup, -) # noqa: F401 +from spyglass.decoding.v1.core import DecodingParameters # noqa: F401 +from spyglass.decoding.v1.core import PositionGroup from spyglass.position.position_merge import PositionOutput # noqa: F401 from spyglass.settings import config from spyglass.spikesorting.analysis.v1.group import SortedSpikesGroup @@ -46,7 +44,7 @@ class SortedSpikesDecodingSelection(SpyglassMixin, dj.Manual): -> DecodingParameters -> IntervalList.proj(encoding_interval='interval_list_name') -> IntervalList.proj(decoding_interval='interval_list_name') - estimate_decoding_params = 1 : bool # whether to estimate the decoding parameters + estimate_decoding_params = 1 : bool # 1 to estimate the decoding parameters """ # NOTE: Excessive key length fixed by reducing UnitSelectionParams.unit_filter_params_name @@ -80,8 +78,9 @@ def make(self, key): position_variable_names, ) = self.fetch_position_info(key) - # Get the spike times for the selected units - # Don't need to filter by interval since the non_local_detector code will do that + # Get the spike times for the selected units. Don't need to filter by + # interval since the non_local_detector code will do that + spike_times = self.fetch_spike_data(key, filter_by_interval=False) # Get the encoding and decoding intervals @@ -115,10 +114,13 @@ def make(self, key): classifier = SortedSpikesDetector(**decoding_params) if key["estimate_decoding_params"]: - # if estimating parameters, then we need to treat times outside decoding interval as missing - # this means that times outside the decoding interval will not use the spiking data - # a better approach would be to treat the intervals as multiple sequences - # (see https://en.wikipedia.org/wiki/Baum%E2%80%93Welch_algorithm#Multiple_sequences) + + # if estimating parameters, then we need to treat times outside + # decoding interval as missing this means that times outside the + # decoding interval will not use the spiking data a better approach + # would be to treat the intervals as multiple sequences (see + # https://en.wikipedia.org/wiki/Baum%E2%80%93Welch_algorithm#Multiple_sequences) + is_missing = np.ones(len(position_info), dtype=bool) for interval_start, interval_end in decoding_interval: is_missing[ @@ -293,7 +295,7 @@ def fetch_environments(key): @staticmethod def _get_interval_range(key): - """Get the maximum range of model times in the encoding and decoding intervals + """Return maximum range of model times in encoding/decoding intervals Parameters ---------- @@ -353,10 +355,7 @@ def fetch_position_info(key): min_time, max_time = SortedSpikesDecodingV1._get_interval_range(key) position_info, position_variable_names = ( PositionGroup & position_group_key - ).fetch_position_info( - min_time=min_time, - max_time=max_time, - ) + ).fetch_position_info(min_time=min_time, max_time=max_time) return position_info, position_variable_names @@ -389,6 +388,7 @@ def fetch_linear_position_info(key): edge_spacing=environment.edge_spacing, ) min_time, max_time = SortedSpikesDecodingV1._get_interval_range(key) + return ( pd.concat( [linear_position_df.set_index(position_df.index), position_df], @@ -409,12 +409,14 @@ def fetch_spike_data( key : dict The decoding selection key filter_by_interval : bool, optional - Whether to filter for spike times in the model interval, by default True + Whether to filter for spike times in the model interval, + by default True time_slice : Slice, optional User provided slice of time to restrict spikes to, by default None return_unit_ids : bool, optional - if True, return the unit_ids along with the spike times, by default False - Unit ids defined as a list of dictionaries with keys 'spikesorting_merge_id' and 'unit_number' + if True, return the unit_ids along with the spike times, by default + False Unit ids defined as a list of dictionaries with keys + 'spikesorting_merge_id' and 'unit_number' Returns ------- @@ -477,8 +479,13 @@ def spike_times_sorted_by_place_field_peak(self, time_slice=None): ] return new_spike_times + def get_orientation_col(self, df): + """Examine columns of a input df and return orientation col name""" + cols = df.columns + return "orientation" if "orientation" in cols else "head_orientation" + def get_ahead_behind_distance(self, track_graph=None, time_slice=None): - """Get the ahead-behind distance of the decoded position from the animal's actual position + """Get relative decoded position from the animal's actual position Parameters ---------- @@ -514,11 +521,7 @@ def get_ahead_behind_distance(self, track_graph=None, time_slice=None): self.fetch1("KEY") ).loc[time_slice] - orientation_name = ( - "orientation" - if "orientation" in linear_position_info.columns - else "head_orientation" - ) + orientation_name = self.get_orientation_col(linear_position_info) traj_data = analysis.get_trajectory_data( posterior=posterior, @@ -538,11 +541,8 @@ def get_ahead_behind_distance(self, track_graph=None, time_slice=None): ] map_position = analysis.maximum_a_posteriori_estimate(posterior) - orientation_name = ( - "orientation" - if "orientation" in position_info.columns - else "head_orientation" - ) + orientation_name = self.get_orientation_col(position_info) + position_variable_names = ( PositionGroup & self.fetch1("KEY") ).fetch1("position_variables") diff --git a/src/spyglass/decoding/v1/waveform_features.py b/src/spyglass/decoding/v1/waveform_features.py index 67f7ab692..1208c53fd 100644 --- a/src/spyglass/decoding/v1/waveform_features.py +++ b/src/spyglass/decoding/v1/waveform_features.py @@ -9,6 +9,7 @@ import spikeinterface as si from spyglass.common.common_nwbfile import AnalysisNwbfile +from spyglass.decoding.utils import _get_peak_amplitude from spyglass.settings import temp_dir from spyglass.spikesorting.spikesorting_merge import SpikeSortingOutput from spyglass.spikesorting.v1 import SpikeSortingSelection @@ -19,8 +20,7 @@ @schema class WaveformFeaturesParams(SpyglassMixin, dj.Lookup): - """Defines the types of spike waveform features computed for a given spike - time.""" + """Defines types of waveform features computed for a given spike time.""" definition = """ features_param_name : varchar(80) # a name for this set of parameters @@ -33,7 +33,7 @@ class WaveformFeaturesParams(SpyglassMixin, dj.Lookup): "estimate_peak_time": False, } } - _default_waveform_extraction_params = { + _default_waveform_extract_params = { "ms_before": 0.5, "ms_after": 0.5, "max_spikes_per_unit": None, @@ -45,7 +45,7 @@ class WaveformFeaturesParams(SpyglassMixin, dj.Lookup): "amplitude", { "waveform_features_params": _default_waveform_feature_params, - "waveform_extraction_params": _default_waveform_extraction_params, + "waveform_extraction_params": _default_waveform_extract_params, }, ], [ @@ -55,7 +55,7 @@ class WaveformFeaturesParams(SpyglassMixin, dj.Lookup): "amplitude": _default_waveform_feature_params["amplitude"], "spike_location": {}, }, - "waveform_extraction_params": _default_waveform_extraction_params, + "waveform_extraction_params": _default_waveform_extract_params, }, ], ] @@ -91,8 +91,9 @@ class UnitWaveformFeaturesSelection(SpyglassMixin, dj.Manual): @schema class UnitWaveformFeatures(SpyglassMixin, dj.Computed): - """For each spike time, compute a spike waveform feature associated with that - spike. Used for clusterless decoding. + """For each spike time, compute waveform feature associated with that spike. + + Used for clusterless decoding. """ definition = """ @@ -114,7 +115,8 @@ def make(self, key): params["waveform_features_params"] ): raise NotImplementedError( - f"Features {set(params['waveform_features_params'])} are not supported" + f"Features {set(params['waveform_features_params'])} are " + + "not supported" ) merge_key = {"merge_id": key["spikesorting_merge_id"]} @@ -248,47 +250,6 @@ def _convert_data(nwb_data) -> list[tuple[np.ndarray, np.ndarray]]: ] -def _get_peak_amplitude( - waveform_extractor: si.WaveformExtractor, - unit_id: int, - peak_sign: str = "neg", - estimate_peak_time: bool = False, -) -> np.ndarray: - """Returns the amplitudes of all channels at the time of the peak - amplitude across channels. - - Parameters - ---------- - waveform : array-like, shape (n_spikes, n_time, n_channels) - peak_sign : ('pos', 'neg', 'both'), optional - Direction of the peak in the waveform - estimate_peak_time : bool, optional - Find the peak times for each spike because some spikesorters do not - align the spike time (at index n_time // 2) to the peak - - Returns - ------- - peak_amplitudes : array-like, shape (n_spikes, n_channels) - - """ - waveforms = waveform_extractor.get_waveforms(unit_id) - if estimate_peak_time: - if peak_sign == "neg": - peak_inds = np.argmin(np.min(waveforms, axis=2), axis=1) - elif peak_sign == "pos": - peak_inds = np.argmax(np.max(waveforms, axis=2), axis=1) - elif peak_sign == "both": - peak_inds = np.argmax(np.max(np.abs(waveforms), axis=2), axis=1) - - # Get mode of peaks to find the peak time - values, counts = np.unique(peak_inds, return_counts=True) - spike_peak_ind = values[counts.argmax()] - else: - spike_peak_ind = waveforms.shape[1] // 2 - - return waveforms[:, spike_peak_ind] - - def _get_full_waveform( waveform_extractor: si.WaveformExtractor, unit_id: int, **kwargs ) -> np.ndarray: @@ -343,7 +304,7 @@ def _write_waveform_features_to_nwb( spike_times: pd.DataFrame, waveform_features: dict, ) -> tuple[str, str]: - """Save waveforms, metrics, labels, and merge groups to NWB in the units table. + """Save waveforms, metrics, labels, and merge groups to NWB units table. Parameters ---------- @@ -358,7 +319,8 @@ def _write_waveform_features_to_nwb( Returns ------- analysis_nwb_file : str - name of analysis NWB file containing the sorting and curation information + name of analysis NWB file containing the sorting and curation + information object_id : str object_id of the units table in the analysis NWB file """ diff --git a/src/spyglass/lfp/analysis/v1/lfp_band.py b/src/spyglass/lfp/analysis/v1/lfp_band.py index d77148610..985cf84d7 100644 --- a/src/spyglass/lfp/analysis/v1/lfp_band.py +++ b/src/spyglass/lfp/analysis/v1/lfp_band.py @@ -39,7 +39,6 @@ class LFPBandElectrode(SpyglassMixin, dj.Part): -> LFPBandSelection # the LFP band selection -> LFPElectrodeGroup.LFPElectrode # the LFP electrode to be filtered reference_elect_id = -1: int # the reference electrode to use; -1 for no reference - --- """ def set_lfp_band_electrodes( @@ -178,7 +177,8 @@ def make(self, key): lfp_band_file_name = AnalysisNwbfile().create( # logged key["nwb_file_name"] ) - # get the NWB object with the lfp data; FIX: change to fetch with additional infrastructure + # get the NWB object with the lfp data; + # FIX: change to fetch with additional infrastructure lfp_key = {"merge_id": key["lfp_merge_id"]} lfp_object = (LFPOutput & lfp_key).fetch_nwb()[0]["lfp"] @@ -293,12 +293,15 @@ def make(self, key): decimation, ) - # now that the LFP is filtered, we create an electrical series for it and add it to the file + # now that the LFP is filtered, we create an electrical series for it + # and add it to the file with pynwb.NWBHDF5IO( path=lfp_band_file_abspath, mode="a", load_namespaces=True ) as io: nwbf = io.read() - # get the indices of the electrodes in the electrode table of the file to get the right values + + # get the indices of the electrodes in the electrode table of the + # file to get the right values elect_index = get_electrode_indices(nwbf, lfp_band_elect_id) electrode_table_region = nwbf.create_electrode_table_region( elect_index, "filtered electrode table" @@ -325,8 +328,8 @@ def make(self, key): key["analysis_file_name"] = lfp_band_file_name key["lfp_band_object_id"] = lfp_band_object_id - # finally, we need to censor the valid times to account for the downsampling if this is the first time we've - # downsampled these data + # finally, censor the valid times to account for the downsampling if + # this is the first time we've downsampled these data key["interval_list_name"] = ( interval_list_name + " lfp band " @@ -378,7 +381,7 @@ def fetch1_dataframe(self, *attrs, **kwargs): def compute_analytic_signal(self, electrode_list: list[int], **kwargs): """Computes the hilbert transform of a given LFPBand signal - Uses scipy.signal.hilbert to compute the hilbert transform of the signal + Uses scipy.signal.hilbert to compute the hilbert transform Parameters ---------- @@ -415,7 +418,7 @@ def compute_analytic_signal(self, electrode_list: list[int], **kwargs): def compute_signal_phase( self, electrode_list: list[int] = None, **kwargs ) -> pd.DataFrame: - """Computes the phase of a given LFPBand signals using the hilbert transform + """Computes phase of LFPBand signals using the hilbert transform Parameters ---------- @@ -443,7 +446,7 @@ def compute_signal_phase( def compute_signal_power( self, electrode_list: list[int] = None, **kwargs ) -> pd.DataFrame: - """Computes the power of a given LFPBand signals using the hilbert transform + """Computes power LFPBand signals using the hilbert transform Parameters ---------- diff --git a/src/spyglass/lfp/lfp_electrode.py b/src/spyglass/lfp/lfp_electrode.py index 0b683a3da..edb6f18a0 100644 --- a/src/spyglass/lfp/lfp_electrode.py +++ b/src/spyglass/lfp/lfp_electrode.py @@ -11,8 +11,8 @@ @schema class LFPElectrodeGroup(SpyglassMixin, dj.Manual): definition = """ - -> Session # the session to which this LFP belongs - lfp_electrode_group_name: varchar(200) # the name of this group of electrodes + -> Session # the session for this LFP + lfp_electrode_group_name: varchar(200) # name for this group of electrodes """ class LFPElectrode(SpyglassMixin, dj.Part): diff --git a/src/spyglass/lfp/lfp_imported.py b/src/spyglass/lfp/lfp_imported.py index 38a0878fc..6219b8a40 100644 --- a/src/spyglass/lfp/lfp_imported.py +++ b/src/spyglass/lfp/lfp_imported.py @@ -15,7 +15,7 @@ class ImportedLFP(SpyglassMixin, dj.Imported): -> Session # the session to which this LFP belongs -> LFPElectrodeGroup # the group of electrodes to be filtered -> IntervalList # the original set of times to be filtered - lfp_object_id: varchar(40) # the NWB object ID for loading this object from the file + lfp_object_id: varchar(40) # object ID for loading from the NWB file --- lfp_sampling_rate: float # the sampling rate, in samples/sec -> AnalysisNwbfile diff --git a/src/spyglass/lfp/v1/lfp.py b/src/spyglass/lfp/v1/lfp.py index 097e67c23..8d12582cd 100644 --- a/src/spyglass/lfp/v1/lfp.py +++ b/src/spyglass/lfp/v1/lfp.py @@ -25,14 +25,15 @@ class LFPSelection(SpyglassMixin, dj.Manual): """The user's selection of LFP data to be filtered - This table is used to select the LFP data to be filtered. The user can select - the LFP data by specifying the electrode group and the interval list to be used. - The interval list is used to select the times from the raw data that will be - filtered. The user can also specify the filter to be used. - - The LFP data is filtered and downsampled to the user-defined sampling rate, specified - as lfp_sampling_rate. The filtered data is stored in the AnalysisNwbfile table. - The valid times for the filtered data are stored in the IntervalList table. + This table is used to select the LFP data to be filtered. The user can + select the LFP data by specifying the electrode group and the interval list + to be used. The interval list is used to select the times from the raw data + that will be filtered. The user can also specify the filter to be used. + + The LFP data is filtered and downsampled to the user-defined sampling rate, + specified as lfp_sampling_rate. The filtered data is stored in the + AnalysisNwbfile table. The valid times for the filtered data are stored in + the IntervalList table. """ definition = """ @@ -40,7 +41,7 @@ class LFPSelection(SpyglassMixin, dj.Manual): -> IntervalList.proj(target_interval_list_name='interval_list_name') # the original set of times to be filtered -> FirFilterParameters # the filter to be used --- - target_sampling_rate = 1000 : float # the desired output sampling rate, in HZ + target_sampling_rate = 1000 : float # the desired output sampling rate, in HZ """ @@ -49,11 +50,11 @@ class LFPV1(SpyglassMixin, dj.Computed): """The filtered LFP data""" definition = """ - -> LFPSelection # the user's selection of LFP data to be filtered + -> LFPSelection # the user's selection of data to be filtered --- -> AnalysisNwbfile # the name of the nwb file with the lfp data - -> IntervalList # the final interval list of valid times for the data - lfp_object_id: varchar(40) # the NWB object ID for loading this object from the file + -> IntervalList # final interval list of times for the data + lfp_object_id: varchar(40) # object ID for loading from the NWB file lfp_sampling_rate: float # the sampling rate, in HZ """ @@ -78,7 +79,7 @@ def make(self, key): orig_key = copy.deepcopy(key) orig_key["interval_list_name"] = key["target_interval_list_name"] user_valid_times = (IntervalList() & orig_key).fetch1("valid_times") - # we remove the extra entry so we can insert this into the LFPOutput table. + # remove the extra entry so we can insert into the LFPOutput table. del orig_key["interval_list_name"] raw_valid_times = ( @@ -153,7 +154,8 @@ def make(self, key): # need to censor the valid times to account for the downsampling lfp_valid_times = interval_list_censor(valid_times, timestamp_interval) - # add an interval list for the LFP valid times, or check that it matches the existing one + # add an interval list for the LFP valid times, or check that it + # matches the existing one key["interval_list_name"] = "_".join( ( "lfp", diff --git a/src/spyglass/lfp/v1/lfp_artifact_MAD_detection.py b/src/spyglass/lfp/v1/lfp_artifact_MAD_detection.py index eac1943a5..c8bf3219f 100644 --- a/src/spyglass/lfp/v1/lfp_artifact_MAD_detection.py +++ b/src/spyglass/lfp/v1/lfp_artifact_MAD_detection.py @@ -21,7 +21,8 @@ def mad_artifact_detector( mad_thresh : float, optional Threshold on the median absolute deviation scaled LFPs, defaults to 6.0 proportion_above_thresh : float, optional - Proportion of electrodes that need to be above the threshold, defaults to 1.0 + Proportion of electrodes that need to be above the threshold, defaults + to 1.0 removal_window_ms : float, optional Width of the window in milliseconds to mask out per artifact (window/2 removed on each side of threshold crossing), defaults to 1 ms @@ -31,10 +32,11 @@ def mad_artifact_detector( Returns ------- artifact_removed_valid_times : np.ndarray - Intervals of valid times where artifacts were not detected, unit: seconds + Intervals of valid times where artifacts were not detected, + unit: seconds artifact_intervals : np.ndarray - Intervals in which artifacts are detected (including removal windows), unit: seconds - + Intervals in which artifacts are detected (including removal windows), + unit: seconds """ timestamps = np.asarray(recording.timestamps) @@ -82,7 +84,7 @@ def _is_above_proportion_thresh( mad_thresh: np.ndarray, proportion_above_thresh: float = 1.0, ) -> np.ndarray: - """Return whether each sample is above the threshold on the proportion of electrodes. + """Return array of bools for samples > thresh on proportion of electodes. Parameters ---------- @@ -96,7 +98,8 @@ def _is_above_proportion_thresh( Returns ------- is_above_thresh : np.ndarray, shape (n_samples,) - Whether each sample is above the threshold on the proportion of electrodes + Whether each sample is above the threshold on the proportion of + electrodes """ return ( diff --git a/src/spyglass/lfp/v1/lfp_artifact_difference_detection.py b/src/spyglass/lfp/v1/lfp_artifact_difference_detection.py index 72d1be70b..7b289a657 100644 --- a/src/spyglass/lfp/v1/lfp_artifact_difference_detection.py +++ b/src/spyglass/lfp/v1/lfp_artifact_difference_detection.py @@ -53,7 +53,8 @@ def difference_artifact_detector( Width of the window in milliseconds to mask out per artifact (window/2 removed on each side of threshold crossing), defaults to 1 ms referencing : bool, optional - Whether or not the data passed to this function is referenced, defaults to False + Whether or not the data passed to this function is referenced, defaults + to False Returns ------- diff --git a/src/spyglass/linearization/utils.py b/src/spyglass/linearization/utils.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/spyglass/linearization/v0/main.py b/src/spyglass/linearization/v0/main.py index cfe3a02db..2884d8525 100644 --- a/src/spyglass/linearization/v0/main.py +++ b/src/spyglass/linearization/v0/main.py @@ -57,6 +57,8 @@ class TrackGraph(SpyglassMixin, dj.Manual): """ def get_networkx_track_graph(self, track_graph_parameters=None): + # CB: Does this need to be a public method? Can other methods inherit + # the logic? It's a pretty simple wrapper around make_track_graph. if track_graph_parameters is None: track_graph_parameters = self.fetch1() return make_track_graph( @@ -66,7 +68,9 @@ def get_networkx_track_graph(self, track_graph_parameters=None): def plot_track_graph(self, ax=None, draw_edge_labels=False, **kwds): """Plot the track graph in 2D position space.""" - track_graph = self.get_networkx_track_graph() + track_graph = self.get_networkx_track_graph( + track_graph_parameters=self.fetch1() + ) plot_track_graph( track_graph, ax=ax, draw_edge_labels=draw_edge_labels, **kwds ) diff --git a/src/spyglass/linearization/v1/main.py b/src/spyglass/linearization/v1/main.py index b9ca2347d..3e7023ca2 100644 --- a/src/spyglass/linearization/v1/main.py +++ b/src/spyglass/linearization/v1/main.py @@ -43,14 +43,16 @@ class TrackGraph(SpyglassMixin, dj.Manual): definition = """ track_graph_name : varchar(80) ---- - environment : varchar(80) # Type of Environment - node_positions : blob # 2D position of track_graph nodes, shape (n_nodes, 2) - edges: blob # shape (n_edges, 2) - linear_edge_order : blob # order of track graph edges in the linear space, shape (n_edges, 2) - linear_edge_spacing : blob # amount of space between edges in the linear space, shape (n_edges,) + environment : varchar(80) # Type of Environment + node_positions : blob # 2D position of nodes, (n_nodes, 2) + edges: blob # shape (n_edges, 2) + linear_edge_order : blob # order of edges in linear space, (n_edges, 2) + linear_edge_spacing : blob # space btwn edges in linear space, (n_edges,) """ def get_networkx_track_graph(self, track_graph_parameters=None): + # CB: Does this need to be a public method? Can other methods inherit + # the logic? It's a pretty simple wrapper around make_track_graph. if track_graph_parameters is None: track_graph_parameters = self.fetch1() return make_track_graph( @@ -60,7 +62,9 @@ def get_networkx_track_graph(self, track_graph_parameters=None): def plot_track_graph(self, ax=None, draw_edge_labels=False, **kwds): """Plot the track graph in 2D position space.""" - track_graph = self.get_networkx_track_graph() + track_graph = self.get_networkx_track_graph( + track_graph_parameters=self.fetch1() + ) plot_track_graph( track_graph, ax=ax, draw_edge_labels=draw_edge_labels, **kwds ) diff --git a/src/spyglass/lock/file_lock.py b/src/spyglass/lock/file_lock.py index f2207dd20..2dc051920 100644 --- a/src/spyglass/lock/file_lock.py +++ b/src/spyglass/lock/file_lock.py @@ -37,9 +37,11 @@ class AnalysisNwbfileLock(SpyglassMixin, dj.Manual): """ def populate_from_lock_file(self): - """Reads from the ANALYSIS_LOCK_FILE (defined by an environment variable), adds the entries to this schema, and - then removes the file + """Reads/inserts from lock file, then removes lock file. + + Requires ANALYSIS_LOCK_FILE environment variable. """ + if os.path.exists(os.getenv("ANALYSIS_LOCK_FILE")): lock_file = open(os.getenv("ANALYSIS_LOCK_FILE"), "r") for line in lock_file: diff --git a/src/spyglass/position/v1/dlc_utils.py b/src/spyglass/position/v1/dlc_utils.py index f8a911148..df5c16427 100644 --- a/src/spyglass/position/v1/dlc_utils.py +++ b/src/spyglass/position/v1/dlc_utils.py @@ -746,36 +746,6 @@ def red_led_bisector_orientation(pos_df: pd.DataFrame, **params): } -def fill_nan(variable, video_time, variable_time): - video_ind = np.digitize(variable_time, video_time[1:]) - - n_video_time = len(video_time) - try: - n_variable_dims = variable.shape[1] - filled_variable = np.full((n_video_time, n_variable_dims), np.nan) - except IndexError: - filled_variable = np.full((n_video_time,), np.nan) - filled_variable[video_ind] = variable - - return filled_variable - - -def convert_to_pixels(data, frame_size=None, cm_to_pixels=1.0): - """Converts from cm to pixels and flips the y-axis. - - Parameters - ---------- - data : ndarray, shape (n_time, 2) - frame_size : array_like, shape (2,) - cm_to_pixels : float - - Returns - ------- - converted_data : ndarray, shape (n_time, 2) - """ - return data / cm_to_pixels - - class Centroid: def __init__(self, pos_df, points, max_LED_separation=None): if max_LED_separation is None and len(points) != 1: diff --git a/src/spyglass/position/v1/dlc_utils_makevid.py b/src/spyglass/position/v1/dlc_utils_makevid.py index 0a80254e1..cc4cadc0d 100644 --- a/src/spyglass/position/v1/dlc_utils_makevid.py +++ b/src/spyglass/position/v1/dlc_utils_makevid.py @@ -8,9 +8,9 @@ import pandas as pd from tqdm import tqdm as tqdm -from spyglass.position.v1.dlc_utils import convert_to_pixels as _to_px -from spyglass.position.v1.dlc_utils import fill_nan from spyglass.utils import logger +from spyglass.utils.position import convert_to_pixels as _to_px +from spyglass.utils.position import fill_nan RGB_PINK = (234, 82, 111) RGB_YELLOW = (253, 231, 76) diff --git a/src/spyglass/ripple/v1/ripple.py b/src/spyglass/ripple/v1/ripple.py index 563b220d8..bac24dc5c 100644 --- a/src/spyglass/ripple/v1/ripple.py +++ b/src/spyglass/ripple/v1/ripple.py @@ -4,7 +4,6 @@ import pandas as pd import sortingview.views as vv from ripple_detection import Karlsson_ripple_detector, Kay_ripple_detector -from ripple_detection.core import gaussian_smooth, get_envelope from scipy.stats import zscore from spyglass.common.common_interval import ( @@ -12,6 +11,7 @@ interval_list_intersect, ) from spyglass.common.common_nwbfile import AnalysisNwbfile +from spyglass.common.common_ripple import RippleTimes, interpolate_to_new_time from spyglass.lfp.analysis.v1.lfp_band import LFPBandSelection, LFPBandV1 from spyglass.lfp.lfp_merge import LFPOutput from spyglass.position import PositionOutput @@ -29,20 +29,6 @@ UPSTREAM_ACCEPTED_VERSIONS = ["LFPBandV1"] -def interpolate_to_new_time( - df, new_time, upsampling_interpolation_method="linear" -): - old_time = df.index - new_index = pd.Index( - np.unique(np.concatenate((old_time, new_time))), name="time" - ) - return ( - df.reindex(index=new_index) - .interpolate(method=upsampling_interpolation_method) - .reindex(index=new_time) - ) - - @schema class RippleLFPSelection(SpyglassMixin, dj.Manual): definition = """ @@ -296,20 +282,10 @@ def get_ripple_lfps_and_position_info(key): def get_Kay_ripple_consensus_trace( ripple_filtered_lfps, sampling_frequency, smoothing_sigma=0.004 ): - ripple_consensus_trace = np.full_like(ripple_filtered_lfps, np.nan) - not_null = np.all(pd.notnull(ripple_filtered_lfps), axis=1) - - ripple_consensus_trace[not_null] = get_envelope( - np.asarray(ripple_filtered_lfps)[not_null] - ) - ripple_consensus_trace = np.sum(ripple_consensus_trace**2, axis=1) - ripple_consensus_trace[not_null] = gaussian_smooth( - ripple_consensus_trace[not_null], - smoothing_sigma, - sampling_frequency, - ) - return pd.DataFrame( - np.sqrt(ripple_consensus_trace), index=ripple_filtered_lfps.index + return RippleTimes.get_Kay_ripple_consensus_trace( + ripple_filtered_lfps=ripple_filtered_lfps, + sampling_frequency=sampling_frequency, + smoothing_sigma=smoothing_sigma, ) @staticmethod diff --git a/src/spyglass/spikesorting/analysis/v1/group.py b/src/spyglass/spikesorting/analysis/v1/group.py index 208120cbf..45b155576 100644 --- a/src/spyglass/spikesorting/analysis/v1/group.py +++ b/src/spyglass/spikesorting/analysis/v1/group.py @@ -211,7 +211,7 @@ def fetch_spike_data( @classmethod def get_spike_indicator(cls, key: dict, time: np.ndarray) -> np.ndarray: - """get spike indicator matrix for the group + """Get spike indicator matrix for the group Parameters ---------- diff --git a/src/spyglass/spikesorting/spikesorting_merge.py b/src/spyglass/spikesorting/spikesorting_merge.py index fda83cdd7..9886958a0 100644 --- a/src/spyglass/spikesorting/spikesorting_merge.py +++ b/src/spyglass/spikesorting/spikesorting_merge.py @@ -4,11 +4,11 @@ from ripple_detection import get_multiunit_population_firing_rate from spyglass.spikesorting.imported import ImportedSpikeSorting # noqa: F401 -from spyglass.spikesorting.v0.spikesorting_curation import ( # noqa: F401 +from spyglass.spikesorting.v0.spikesorting_curation import ( CuratedSpikeSorting, -) -from spyglass.spikesorting.v1 import ( # noqa: F401 - ArtifactDetectionSelection, +) # noqa: F401 +from spyglass.spikesorting.v1 import ArtifactDetectionSelection # noqa: F401 +from spyglass.spikesorting.v1 import ( CurationV1, MetricCurationSelection, SpikeSortingRecordingSelection, @@ -193,6 +193,20 @@ def get_spike_times(self, key): @classmethod def get_spike_indicator(cls, key, time): + """Get spike indicator matrix for the group + + Parameters + ---------- + key : dict + key to identify the group + time : np.ndarray + time vector for which to calculate the spike indicator matrix + + Returns + ------- + np.ndarray + spike indicator matrix with shape (len(time), n_units) + """ time = np.asarray(time) min_time, max_time = time[[0, -1]] spike_times = cls.fetch_spike_data(key) @@ -205,10 +219,19 @@ def get_spike_indicator(cls, key, time): minlength=time.shape[0], ) + if spike_indicator.ndim == 1: + spike_indicator = spike_indicator[:, np.newaxis] + return spike_indicator @classmethod - def get_firing_rate(cls, key, time, multiunit=False): + def get_firing_rate( + cls, + key: dict, + time: np.array, + multiunit: bool = False, + smoothing_sigma: float = 0.015, + ): spike_indicator = cls.get_spike_indicator(key, time) if spike_indicator.ndim == 1: spike_indicator = spike_indicator[:, np.newaxis] @@ -220,7 +243,9 @@ def get_firing_rate(cls, key, time, multiunit=False): return np.stack( [ get_multiunit_population_firing_rate( - indicator[:, np.newaxis], sampling_frequency + indicator[:, np.newaxis], + sampling_frequency, + smoothing_sigma, ) for indicator in spike_indicator.T ], diff --git a/src/spyglass/spikesorting/utils.py b/src/spyglass/spikesorting/utils.py new file mode 100644 index 000000000..a2e8f2f54 --- /dev/null +++ b/src/spyglass/spikesorting/utils.py @@ -0,0 +1,130 @@ +import numpy as np +import spikeinterface as si + +from spyglass.common.common_ephys import Electrode +from spyglass.utils import logger + + +def get_group_by_shank( + nwb_file_name: str, + references: dict = None, + omit_ref_electrode_group=False, + omit_unitrode=True, +): + """Divides electrodes into groups based on their shank position. + + * Electrodes from probes with 1 shank (e.g. tetrodes) are placed in a + single group + * Electrodes from probes with multiple shanks (e.g. polymer probes) are + placed in one group per shank + * Bad channels are omitted + + Parameters + ---------- + nwb_file_name : str + the name of the NWB file whose electrodes should be put into + sorting groups + references : dict, optional + If passed, used to set references. Otherwise, references set using + original reference electrodes from config. Keys: electrode groups. + Values: reference electrode. + omit_ref_electrode_group : bool + Optional. If True, no sort group is defined for electrode group of + reference. + omit_unitrode : bool + Optional. If True, no sort groups are defined for unitrodes. + """ + # get the electrodes from this NWB file + electrodes = ( + Electrode() + & {"nwb_file_name": nwb_file_name} + & {"bad_channel": "False"} + ).fetch() + + e_groups = list(np.unique(electrodes["electrode_group_name"])) + e_groups.sort(key=int) # sort electrode groups numerically + + sort_group = 0 + sg_keys = list() + sge_keys = list() + for e_group in e_groups: + sg_key = dict() + sge_key = dict() + sg_key["nwb_file_name"] = sge_key["nwb_file_name"] = nwb_file_name + # for each electrode group, get a list of the unique shank numbers + shank_list = np.unique( + electrodes["probe_shank"][ + electrodes["electrode_group_name"] == e_group + ] + ) + sge_key["electrode_group_name"] = e_group + # get the indices of all electrodes in this group / shank and set their sorting group + for shank in shank_list: + sg_key["sort_group_id"] = sge_key["sort_group_id"] = sort_group + # specify reference electrode. Use 'references' if passed, otherwise use reference from config + if not references: + shank_elect_ref = electrodes["original_reference_electrode"][ + np.logical_and( + electrodes["electrode_group_name"] == e_group, + electrodes["probe_shank"] == shank, + ) + ] + if np.max(shank_elect_ref) == np.min(shank_elect_ref): + sg_key["sort_reference_electrode_id"] = shank_elect_ref[0] + else: + ValueError( + f"Error in electrode group {e_group}: reference " + + "electrodes are not all the same" + ) + else: + if e_group not in references.keys(): + raise Exception( + f"electrode group {e_group} not a key in " + + "references, so cannot set reference" + ) + else: + sg_key["sort_reference_electrode_id"] = references[e_group] + # Insert sort group and sort group electrodes + reference_electrode_group = electrodes[ + electrodes["electrode_id"] + == sg_key["sort_reference_electrode_id"] + ][ + "electrode_group_name" + ] # reference for this electrode group + if len(reference_electrode_group) == 1: # unpack single reference + reference_electrode_group = reference_electrode_group[0] + elif (int(sg_key["sort_reference_electrode_id"]) > 0) and ( + len(reference_electrode_group) != 1 + ): + raise Exception( + "Should have found exactly one electrode group for " + + "reference electrode, but found " + + f"{len(reference_electrode_group)}." + ) + if omit_ref_electrode_group and ( + str(e_group) == str(reference_electrode_group) + ): + logger.warn( + f"Omitting electrode group {e_group} from sort groups " + + "because contains reference." + ) + continue + shank_elect = electrodes["electrode_id"][ + np.logical_and( + electrodes["electrode_group_name"] == e_group, + electrodes["probe_shank"] == shank, + ) + ] + if ( + omit_unitrode and len(shank_elect) == 1 + ): # omit unitrodes if indicated + logger.warn( + f"Omitting electrode group {e_group}, shank {shank} " + + "from sort groups because unitrode." + ) + continue + sg_keys.append(sg_key) + for elect in shank_elect: + sge_key["electrode_id"] = elect + sge_keys.append(sge_key.copy()) + sort_group += 1 diff --git a/src/spyglass/spikesorting/v0/curation_figurl.py b/src/spyglass/spikesorting/v0/curation_figurl.py index caab594b7..c33903173 100644 --- a/src/spyglass/spikesorting/v0/curation_figurl.py +++ b/src/spyglass/spikesorting/v0/curation_figurl.py @@ -194,18 +194,15 @@ def _generate_the_figurl( def _reformat_metrics(metrics: Dict[str, Dict[str, float]]) -> List[Dict]: - for metric_name in metrics: - metrics[metric_name] = { - str(unit_id): metric_value - for unit_id, metric_value in metrics[metric_name].items() - } - new_external_metrics = [ + return [ { "name": metric_name, "label": metric_name, "tooltip": metric_name, - "data": metric, + "data": { + str(unit_id): metric_value + for unit_id, metric_value in metric.items() + }, } for metric_name, metric in metrics.items() ] - return new_external_metrics diff --git a/src/spyglass/spikesorting/v0/spikesorting_artifact.py b/src/spyglass/spikesorting/v0/spikesorting_artifact.py index 4ea90e092..c2962ba2e 100644 --- a/src/spyglass/spikesorting/v0/spikesorting_artifact.py +++ b/src/spyglass/spikesorting/v0/spikesorting_artifact.py @@ -164,10 +164,12 @@ def _get_artifact_times( **job_kwargs, ): """Detects times during which artifacts do and do not occur. - Artifacts are defined as periods where the absolute value of the recording signal exceeds one - or both specified amplitude or zscore thresholds on the proportion of channels specified, - with the period extended by the removal_window_ms/2 on each side. Z-score and amplitude - threshold values of None are ignored. + + Artifacts are defined as periods where the absolute value of the recording + signal exceeds one or both specified amplitude or z-score thresholds on the + proportion of channels specified, with the period extended by the + removal_window_ms/2 on each side. Z-score and amplitude threshold values of + None are ignored. Parameters ---------- @@ -224,8 +226,6 @@ def _get_artifact_times( n_jobs = ensure_n_jobs(recording, n_jobs=job_kwargs.get("n_jobs", 1)) logger.info(f"using {n_jobs} jobs...") - func = _compute_artifact_chunk - init_func = _init_artifact_worker if n_jobs == 1: init_args = ( recording, @@ -242,10 +242,10 @@ def _get_artifact_times( ) executor = ChunkRecordingExecutor( - recording, - func, - init_func, - init_args, + recording=recording, + func=_compute_artifact_chunk, + init_func=_init_artifact_worker, + init_args=init_args, verbose=verbose, handle_returns=True, job_name="detect_artifact_frames", diff --git a/src/spyglass/spikesorting/v0/spikesorting_curation.py b/src/spyglass/spikesorting/v0/spikesorting_curation.py index 78ed93bba..edf99b8f4 100644 --- a/src/spyglass/spikesorting/v0/spikesorting_curation.py +++ b/src/spyglass/spikesorting/v0/spikesorting_curation.py @@ -129,13 +129,17 @@ def insert_curation( # convert unit_ids in labels to integers for labels from sortingview. new_labels = {int(unit_id): labels[unit_id] for unit_id in labels} - sorting_key["curation_id"] = curation_id - sorting_key["parent_curation_id"] = parent_curation_id - sorting_key["description"] = description - sorting_key["curation_labels"] = new_labels - sorting_key["merge_groups"] = merge_groups - sorting_key["quality_metrics"] = metrics - sorting_key["time_of_creation"] = int(time.time()) + sorting_key.update( + { + "curation_id": curation_id, + "parent_curation_id": parent_curation_id, + "description": description, + "curation_labels": new_labels, + "merge_groups": merge_groups, + "quality_metrics": metrics, + "time_of_creation": int(time.time()), + } + ) # mike: added skip duplicates Curation.insert1(sorting_key, skip_duplicates=True) @@ -1062,8 +1066,10 @@ def get_sort_group_info(cls, key): sort_group_info : Table Table with information about the sort groups """ + table = cls & key + electrode_restrict_list = [] - for entry in cls & key: + for entry in table: # Just take one electrode entry per sort group electrode_restrict_list.extend( ((SortGroup.SortGroupElectrode() & entry) * Electrode).fetch( @@ -1073,7 +1079,7 @@ def get_sort_group_info(cls, key): # Run joins with the tables with info and return sort_group_info = ( (Electrode & electrode_restrict_list) - * (cls & key) + * table * SortGroup.SortGroupElectrode() ) * BrainRegion() return sort_group_info diff --git a/src/spyglass/spikesorting/v0/spikesorting_recording.py b/src/spyglass/spikesorting/v0/spikesorting_recording.py index 26016cf91..a7a2cb312 100644 --- a/src/spyglass/spikesorting/v0/spikesorting_recording.py +++ b/src/spyglass/spikesorting/v0/spikesorting_recording.py @@ -21,7 +21,8 @@ from spyglass.common.common_nwbfile import Nwbfile from spyglass.common.common_session import Session # noqa: F401 from spyglass.settings import recording_dir -from spyglass.utils import SpyglassMixin, logger +from spyglass.spikesorting.utils import get_group_by_shank +from spyglass.utils import SpyglassMixin from spyglass.utils.dj_helper_fn import dj_replace schema = dj.schema("spikesorting_recording") @@ -75,104 +76,15 @@ def set_group_by_shank( """ # delete any current groups (SortGroup & {"nwb_file_name": nwb_file_name}).delete() - # get the electrodes from this NWB file - electrodes = ( - Electrode() - & {"nwb_file_name": nwb_file_name} - & {"bad_channel": "False"} - ).fetch() - e_groups = list(np.unique(electrodes["electrode_group_name"])) - e_groups.sort(key=int) # sort electrode groups numerically - sort_group = 0 - sg_key = dict() - sge_key = dict() - sg_key["nwb_file_name"] = sge_key["nwb_file_name"] = nwb_file_name - for e_group in e_groups: - # for each electrode group, get a list of the unique shank numbers - shank_list = np.unique( - electrodes["probe_shank"][ - electrodes["electrode_group_name"] == e_group - ] - ) - sge_key["electrode_group_name"] = e_group - # get the indices of all electrodes in this group / shank and set their sorting group - for shank in shank_list: - sg_key["sort_group_id"] = sge_key["sort_group_id"] = sort_group - # specify reference electrode. Use 'references' if passed, otherwise use reference from config - if not references: - shank_elect_ref = electrodes[ - "original_reference_electrode" - ][ - np.logical_and( - electrodes["electrode_group_name"] == e_group, - electrodes["probe_shank"] == shank, - ) - ] - if np.max(shank_elect_ref) == np.min(shank_elect_ref): - sg_key["sort_reference_electrode_id"] = shank_elect_ref[ - 0 - ] - else: - ValueError( - f"Error in electrode group {e_group}: reference " - + "electrodes are not all the same" - ) - else: - if e_group not in references.keys(): - raise Exception( - f"electrode group {e_group} not a key in " - + "references, so cannot set reference" - ) - else: - sg_key["sort_reference_electrode_id"] = references[ - e_group - ] - # Insert sort group and sort group electrodes - reference_electrode_group = electrodes[ - electrodes["electrode_id"] - == sg_key["sort_reference_electrode_id"] - ][ - "electrode_group_name" - ] # reference for this electrode group - if ( - len(reference_electrode_group) == 1 - ): # unpack single reference - reference_electrode_group = reference_electrode_group[0] - elif (int(sg_key["sort_reference_electrode_id"]) > 0) and ( - len(reference_electrode_group) != 1 - ): - raise Exception( - "Should have found exactly one electrode group for " - + "reference electrode, but found " - + f"{len(reference_electrode_group)}." - ) - if omit_ref_electrode_group and ( - str(e_group) == str(reference_electrode_group) - ): - logger.warn( - f"Omitting electrode group {e_group} from sort groups " - + "because contains reference." - ) - continue - shank_elect = electrodes["electrode_id"][ - np.logical_and( - electrodes["electrode_group_name"] == e_group, - electrodes["probe_shank"] == shank, - ) - ] - if ( - omit_unitrode and len(shank_elect) == 1 - ): # omit unitrodes if indicated - logger.warn( - f"Omitting electrode group {e_group}, shank {shank} " - + "from sort groups because unitrode." - ) - continue - self.insert1(sg_key) - for elect in shank_elect: - sge_key["electrode_id"] = elect - self.SortGroupElectrode().insert1(sge_key) - sort_group += 1 + + sg_keys, sge_keys = get_group_by_shank( + nwb_file_name=nwb_file_name, + references=references, + omit_ref_electrode_group=omit_ref_electrode_group, + omit_unitrode=omit_unitrode, + ) + self.insert(sg_keys, skip_duplicates=False) + self.SortGroupElectrode().insert(sge_keys, skip_duplicates=False) def set_group_by_electrode_group(self, nwb_file_name: str): """Assign groups to all non-bad channel electrodes based on their electrode group diff --git a/src/spyglass/spikesorting/v1/artifact.py b/src/spyglass/spikesorting/v1/artifact.py index 6f241578e..f17651a8b 100644 --- a/src/spyglass/spikesorting/v1/artifact.py +++ b/src/spyglass/spikesorting/v1/artifact.py @@ -229,8 +229,7 @@ def _get_artifact_times( # detect frames that are above threshold in parallel n_jobs = ensure_n_jobs(recording, n_jobs=job_kwargs.get("n_jobs", 1)) logger.info(f"Using {n_jobs} jobs...") - func = _compute_artifact_chunk - init_func = _init_artifact_worker + if n_jobs == 1: init_args = ( recording, @@ -247,10 +246,10 @@ def _get_artifact_times( ) executor = ChunkRecordingExecutor( - recording, - func, - init_func, - init_args, + recording=recording, + func=_compute_artifact_chunk, + init_func=_init_artifact_worker, + init_args=init_args, verbose=verbose, handle_returns=True, job_name="detect_artifact_frames", diff --git a/src/spyglass/spikesorting/v1/curation.py b/src/spyglass/spikesorting/v1/curation.py index 078076b51..00b1ef81e 100644 --- a/src/spyglass/spikesorting/v1/curation.py +++ b/src/spyglass/spikesorting/v1/curation.py @@ -85,7 +85,6 @@ def insert_curation( sort_query = cls & {"sorting_id": sorting_id} parent_curation_id = max(parent_curation_id, -1) if parent_curation_id == -1: - parent_curation_id = -1 # check to see if this sorting with a parent of -1 # has already been inserted and if so, warn the user query = sort_query & {"parent_curation_id": -1} @@ -124,10 +123,7 @@ def insert_curation( "merges_applied": apply_merge, "description": description, } - cls.insert1( - key, - skip_duplicates=True, - ) + cls.insert1(key, skip_duplicates=True) AnalysisNwbfile().log(analysis_file_name, table=cls.full_table_name) return key diff --git a/src/spyglass/spikesorting/v1/recording.py b/src/spyglass/spikesorting/v1/recording.py index a9873afe0..06b7a4489 100644 --- a/src/spyglass/spikesorting/v1/recording.py +++ b/src/spyglass/spikesorting/v1/recording.py @@ -19,6 +19,7 @@ ) from spyglass.common.common_lab import LabTeam from spyglass.common.common_nwbfile import AnalysisNwbfile, Nwbfile +from spyglass.spikesorting.utils import get_group_by_shank from spyglass.utils import SpyglassMixin, logger schema = dj.schema("spikesorting_v1_recording") @@ -74,106 +75,15 @@ def set_group_by_shank( """ # delete any current groups (SortGroup & {"nwb_file_name": nwb_file_name}).delete() - # get the electrodes from this NWB file - electrodes = ( - Electrode() - & {"nwb_file_name": nwb_file_name} - & {"bad_channel": "False"} - ).fetch() - e_groups = list(np.unique(electrodes["electrode_group_name"])) - e_groups.sort(key=int) # sort electrode groups numerically - sort_group = 0 - sg_key = dict() - sge_key = dict() - sg_key["nwb_file_name"] = sge_key["nwb_file_name"] = nwb_file_name - for e_group in e_groups: - # for each electrode group, get a list of the unique shank numbers - shank_list = np.unique( - electrodes["probe_shank"][ - electrodes["electrode_group_name"] == e_group - ] - ) - sge_key["electrode_group_name"] = e_group - # get the indices of all electrodes in this group / shank and set their sorting group - for shank in shank_list: - sg_key["sort_group_id"] = sge_key["sort_group_id"] = sort_group - # specify reference electrode. Use 'references' if passed, otherwise use reference from config - if not references: - shank_elect_ref = electrodes[ - "original_reference_electrode" - ][ - np.logical_and( - electrodes["electrode_group_name"] == e_group, - electrodes["probe_shank"] == shank, - ) - ] - if np.max(shank_elect_ref) == np.min(shank_elect_ref): - sg_key["sort_reference_electrode_id"] = shank_elect_ref[ - 0 - ] - else: - ValueError( - f"Error in electrode group {e_group}: reference " - + "electrodes are not all the same" - ) - else: - if e_group not in references.keys(): - raise Exception( - f"electrode group {e_group} not a key in " - + "references, so cannot set reference" - ) - else: - sg_key["sort_reference_electrode_id"] = references[ - e_group - ] - # Insert sort group and sort group electrodes - reference_electrode_group = electrodes[ - electrodes["electrode_id"] - == sg_key["sort_reference_electrode_id"] - ][ - "electrode_group_name" - ] # reference for this electrode group - if ( - len(reference_electrode_group) == 1 - ): # unpack single reference - reference_electrode_group = reference_electrode_group[0] - elif (int(sg_key["sort_reference_electrode_id"]) > 0) and ( - len(reference_electrode_group) != 1 - ): - raise Exception( - "Should have found exactly one electrode group for " - + "reference electrode, but found " - + f"{len(reference_electrode_group)}." - ) - if omit_ref_electrode_group and ( - str(e_group) == str(reference_electrode_group) - ): - logger.warn( - f"Omitting electrode group {e_group} from sort groups " - + "because contains reference." - ) - continue - shank_elect = electrodes["electrode_id"][ - np.logical_and( - electrodes["electrode_group_name"] == e_group, - electrodes["probe_shank"] == shank, - ) - ] - if ( - omit_unitrode and len(shank_elect) == 1 - ): # omit unitrodes if indicated - logger.warn( - f"Omitting electrode group {e_group}, shank {shank} " - + "from sort groups because unitrode." - ) - continue - cls.insert1(sg_key, skip_duplicates=True) - for elect in shank_elect: - sge_key["electrode_id"] = elect - cls.SortGroupElectrode().insert1( - sge_key, skip_duplicates=True - ) - sort_group += 1 + + sg_keys, sge_keys = get_group_by_shank( + nwb_file_name=nwb_file_name, + references=references, + omit_ref_electrode_group=omit_ref_electrode_group, + omit_unitrode=omit_unitrode, + ) + cls.insert(sg_keys, skip_duplicates=True) + cls.SortGroupElectrode().insert(sge_keys, skip_duplicates=True) @schema @@ -313,23 +223,27 @@ def get_recording(cls, key: dict) -> si.BaseRecording: @staticmethod def _get_recording_timestamps(recording): - if recording.get_num_segments() > 1: - frames_per_segment = [0] - for i in range(recording.get_num_segments()): - frames_per_segment.append( - recording.get_num_frames(segment_index=i) - ) + num_segments = recording.get_num_segments() - cumsum_frames = np.cumsum(frames_per_segment) - total_frames = np.sum(frames_per_segment) + if num_segments <= 1: + return recording.get_times() + + frames_per_segment = [0] + [ + recording.get_num_frames(segment_index=i) + for i in range(num_segments) + ] + + cumsum_frames = np.cumsum(frames_per_segment) + total_frames = np.sum(frames_per_segment) + + timestamps = np.zeros((total_frames,)) + for i in range(num_segments): + start_index = cumsum_frames[i] + end_index = cumsum_frames[i + 1] + timestamps[start_index:end_index] = recording.get_times( + segment_index=i + ) - timestamps = np.zeros((total_frames,)) - for i in range(recording.get_num_segments()): - timestamps[cumsum_frames[i] : cumsum_frames[i + 1]] = ( - recording.get_times(segment_index=i) - ) - else: - timestamps = recording.get_times() return timestamps def _get_sort_interval_valid_times(self, key: dict): diff --git a/src/spyglass/utils/position.py b/src/spyglass/utils/position.py new file mode 100644 index 000000000..e3636bbe9 --- /dev/null +++ b/src/spyglass/utils/position.py @@ -0,0 +1,30 @@ +import numpy as np + + +def convert_to_pixels(data, frame_size=None, cm_to_pixels=1.0): + """Converts from cm to pixels and flips the y-axis. + Parameters + ---------- + data : ndarray, shape (n_time, 2) + frame_size : array_like, shape (2,) + cm_to_pixels : float + + Returns + ------- + converted_data : ndarray, shape (n_time, 2) + """ + return data / cm_to_pixels + + +def fill_nan(variable, video_time, variable_time): + video_ind = np.digitize(variable_time, video_time[1:]) + + n_video_time = len(video_time) + try: + n_variable_dims = variable.shape[1] + filled_variable = np.full((n_video_time, n_variable_dims), np.nan) + except IndexError: + filled_variable = np.full((n_video_time,), np.nan) + filled_variable[video_ind] = variable + + return filled_variable diff --git a/tests/common/test_nwbfile.py b/tests/common/test_nwbfile.py index a8671b7ce..a808ad96d 100644 --- a/tests/common/test_nwbfile.py +++ b/tests/common/test_nwbfile.py @@ -30,7 +30,7 @@ def test_add_to_lock(common_nwbfile, lockfile, mini_copy_name): with lockfile.open("r") as f: assert mini_copy_name in f.read() - with pytest.raises(AssertionError): + with pytest.raises(FileNotFoundError): common_nwbfile.Nwbfile.add_to_lock("non-existent-file.nwb") diff --git a/tests/common/test_position.py b/tests/common/test_position.py index bb74e213c..f2d882cba 100644 --- a/tests/common/test_position.py +++ b/tests/common/test_position.py @@ -159,11 +159,12 @@ def test_position_video(position_video, upsample_position): assert len(position_video) == 1, "Failed to populate PositionVideo table." -def test_convert_to_pixels(position_video): +def test_convert_to_pixels(): + from spyglass.utils.position import convert_to_pixels data = np.array([[2, 4], [6, 8]]) expect = np.array([[1, 2], [3, 4]]) - output = position_video.convert_to_pixels(data, "junk", 2) + output = convert_to_pixels(data, "junk", 2) assert np.array_equal(output, expect), "Failed to convert to pixels." diff --git a/tests/conftest.py b/tests/conftest.py index 8a58df39b..0bff200ef 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -276,9 +276,9 @@ def mini_insert( ): from spyglass.common import LabMember, Nwbfile, Session # noqa: E402 from spyglass.data_import import insert_sessions # noqa: E402 - from spyglass.spikesorting.spikesorting_merge import ( # noqa: E402 + from spyglass.spikesorting.spikesorting_merge import ( SpikeSortingOutput, - ) + ) # noqa: E402 from spyglass.utils.nwb_helper_fn import close_nwb_files # noqa: E402 _ = SpikeSortingOutput() From 5145ff6b234c27377c226f57f30806dbd1ea5a50 Mon Sep 17 00:00:00 2001 From: MichaelCoulter <37707865+MichaelCoulter@users.noreply.github.com> Date: Fri, 2 Aug 2024 08:04:55 -0700 Subject: [PATCH 23/94] quick fix for issue 1045 (#1046) * quick fix for issue 1045 the issue about lists vs array for lfp artifact removed intervals * Update lfp_artifact_MAD_detection.py black format * Update lfp_artifact_MAD_detection.py black format * Update CHANGELOG.md --------- Co-authored-by: Eric Denovellis --- CHANGELOG.md | 1 + src/spyglass/lfp/v1/lfp_artifact_MAD_detection.py | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f03c2c298..a1146ef4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ PositionGroup.alter() - Speed up fetch_nwb calls through merge tables #1017 - Allow `ModuleNotFoundError` or `ImportError` for optional dependencies #1023 - Ensure integrity of group tables #1026 +- Convert list of LFP artifact removed interval list to array #1046 - Merge duplicate functions in decoding and spikesorting #1050 ### Pipelines diff --git a/src/spyglass/lfp/v1/lfp_artifact_MAD_detection.py b/src/spyglass/lfp/v1/lfp_artifact_MAD_detection.py index c8bf3219f..7c5906e60 100644 --- a/src/spyglass/lfp/v1/lfp_artifact_MAD_detection.py +++ b/src/spyglass/lfp/v1/lfp_artifact_MAD_detection.py @@ -52,11 +52,13 @@ def mad_artifact_detector( half_removal_window_idx = int(half_removal_window_s * sampling_frequency) is_artifact = _extend_array_by_window(is_artifact, half_removal_window_idx) - artifact_intervals_s = _get_time_intervals_from_bool_array( - is_artifact, timestamps + artifact_intervals_s = np.array( + _get_time_intervals_from_bool_array(is_artifact, timestamps) ) - valid_times = _get_time_intervals_from_bool_array(~is_artifact, timestamps) + valid_times = np.array( + _get_time_intervals_from_bool_array(~is_artifact, timestamps) + ) return valid_times, artifact_intervals_s From 24793f545ca82e4dbf3bbf5168e4371291e0e9a7 Mon Sep 17 00:00:00 2001 From: Samuel Bray Date: Fri, 2 Aug 2024 11:34:49 -0700 Subject: [PATCH 24/94] Quick fix for parralel populate and spike unit naming (#1052) * remove staticmethod decorator * fix bug preventing parallel populate * update changelog --- CHANGELOG.md | 4 ++-- src/spyglass/spikesorting/analysis/v1/group.py | 1 - src/spyglass/utils/dj_mixin.py | 1 + 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1146ef4a..8509f24b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,7 +30,7 @@ PositionGroup.alter() - Expand `delete_downstream_merge` -> `delete_downstream_parts`. #1002 - `cautious_delete` now checks `IntervalList` and externals tables. #1002 - Allow mixin tables with parallelization in `make` to run populate with - `processes > 1` #1001 + `processes > 1` #1001, #1052 - Speed up fetch_nwb calls through merge tables #1017 - Allow `ModuleNotFoundError` or `ImportError` for optional dependencies #1023 - Ensure integrity of group tables #1026 @@ -75,7 +75,7 @@ PositionGroup.alter() - Fix bug in identification of artifact samples to be zeroed out in `spikesorting.v1.SpikeSorting` #1009 - Remove deprecated dependencies on kachery_client #1014 - - Add `UnitAnnotation` table and naming convention for units #1027 + - Add `UnitAnnotation` table and naming convention for units #1027, #1052 ## [0.5.2] (April 22, 2024) diff --git a/src/spyglass/spikesorting/analysis/v1/group.py b/src/spyglass/spikesorting/analysis/v1/group.py index 45b155576..e2e9d0765 100644 --- a/src/spyglass/spikesorting/analysis/v1/group.py +++ b/src/spyglass/spikesorting/analysis/v1/group.py @@ -289,7 +289,6 @@ def get_firing_rate( ) -@staticmethod def _get_spike_obj_name(nwb_file, allow_empty=False): nwb_field_name = ( "object_id" diff --git a/src/spyglass/utils/dj_mixin.py b/src/spyglass/utils/dj_mixin.py index ed08d6212..6eee80e40 100644 --- a/src/spyglass/utils/dj_mixin.py +++ b/src/spyglass/utils/dj_mixin.py @@ -708,6 +708,7 @@ def populate(self, *restrictions, **kwargs): # Pass through to super if not parallel in the make function or only a single process processes = kwargs.pop("processes", 1) if processes == 1 or not self._parallel_make: + kwargs["processes"] = processes return super().populate(*restrictions, **kwargs) # If parallel in both make and populate, use non-daemon processes From 907098a5bff85f0340c31180105a7dad4a151afe Mon Sep 17 00:00:00 2001 From: Kyu Hyun Lee Date: Mon, 5 Aug 2024 07:03:33 -0700 Subject: [PATCH 25/94] Set `sparse=False` during waveform extraction (#1039) * Save LFP as pynwb.ecephys.LFP * Fix formatting * Fix formatting * Set sparse False during wf extraciton * Black * Update Changelog * Black --------- Co-authored-by: Eric Denovellis Co-authored-by: Eric Denovellis --- CHANGELOG.md | 3 ++- src/spyglass/decoding/v1/clusterless.py | 1 - src/spyglass/decoding/v1/sorted_spikes.py | 1 - src/spyglass/position/v1/position_dlc_pose_estimation.py | 2 -- src/spyglass/position/v1/position_dlc_position.py | 1 - src/spyglass/spikesorting/v1/metric_curation.py | 1 + tests/common/test_position.py | 1 - tests/utils/conftest.py | 1 - 8 files changed, 3 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8509f24b0..449ccd8de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,7 +76,8 @@ PositionGroup.alter() `spikesorting.v1.SpikeSorting` #1009 - Remove deprecated dependencies on kachery_client #1014 - Add `UnitAnnotation` table and naming convention for units #1027, #1052 - + - Set `sparse` parameter to waveform extraction step in `spikesorting.v1` + #1039 ## [0.5.2] (April 22, 2024) ### Infrastructure diff --git a/src/spyglass/decoding/v1/clusterless.py b/src/spyglass/decoding/v1/clusterless.py index 3694a550f..f9e149df1 100644 --- a/src/spyglass/decoding/v1/clusterless.py +++ b/src/spyglass/decoding/v1/clusterless.py @@ -147,7 +147,6 @@ def make(self, key): classifier = ClusterlessDetector(**decoding_params) if key["estimate_decoding_params"]: - # if estimating parameters, then we need to treat times outside # decoding interval as missing this means that times outside the # decoding interval will not use the spiking data a better approach diff --git a/src/spyglass/decoding/v1/sorted_spikes.py b/src/spyglass/decoding/v1/sorted_spikes.py index bd44f8eb6..9dbdd015b 100644 --- a/src/spyglass/decoding/v1/sorted_spikes.py +++ b/src/spyglass/decoding/v1/sorted_spikes.py @@ -114,7 +114,6 @@ def make(self, key): classifier = SortedSpikesDetector(**decoding_params) if key["estimate_decoding_params"]: - # if estimating parameters, then we need to treat times outside # decoding interval as missing this means that times outside the # decoding interval will not use the spiking data a better approach diff --git a/src/spyglass/position/v1/position_dlc_pose_estimation.py b/src/spyglass/position/v1/position_dlc_pose_estimation.py index 4b82918a0..8fa3df922 100644 --- a/src/spyglass/position/v1/position_dlc_pose_estimation.py +++ b/src/spyglass/position/v1/position_dlc_pose_estimation.py @@ -118,7 +118,6 @@ def insert_estimation_task( def _insert_est_with_log( self, key, task_mode, params, check_crop, skip_duplicates, output_dir ): - v_path, v_fname, _, _ = get_video_info(key) if not v_path: raise FileNotFoundError(f"Video file not found for {key}") @@ -215,7 +214,6 @@ def make(self, key): @file_log(logger, console=True) def _logged_make(self, key): - METERS_PER_CM = 0.01 logger.info("----------------------") diff --git a/src/spyglass/position/v1/position_dlc_position.py b/src/spyglass/position/v1/position_dlc_position.py index cfad61c15..01b732612 100644 --- a/src/spyglass/position/v1/position_dlc_position.py +++ b/src/spyglass/position/v1/position_dlc_position.py @@ -169,7 +169,6 @@ def make(self, key): @file_log(logger, console=False) def _logged_make(self, key): - METERS_PER_CM = 0.01 logger.info("-----------------------") diff --git a/src/spyglass/spikesorting/v1/metric_curation.py b/src/spyglass/spikesorting/v1/metric_curation.py index 836de018d..b03c7fa9c 100644 --- a/src/spyglass/spikesorting/v1/metric_curation.py +++ b/src/spyglass/spikesorting/v1/metric_curation.py @@ -242,6 +242,7 @@ def make(self, key): waveforms = si.extract_waveforms( recording=recording, sorting=sorting, + sparse=waveform_params.get("sparse", False), folder=waveforms_dir, overwrite=True, **waveform_params, diff --git a/tests/common/test_position.py b/tests/common/test_position.py index f2d882cba..e5f39c20c 100644 --- a/tests/common/test_position.py +++ b/tests/common/test_position.py @@ -183,7 +183,6 @@ def rename_default_cols(common_position): ], ) def test_rename_columns(rename_default_cols, col_type, cols): - _fix_col_names, defaults = rename_default_cols df = pd.DataFrame([range(len(cols) + 1)], columns=["junk"] + cols) result = _fix_col_names(df).columns.tolist() diff --git a/tests/utils/conftest.py b/tests/utils/conftest.py index 726b6b8a7..3723f191c 100644 --- a/tests/utils/conftest.py +++ b/tests/utils/conftest.py @@ -233,7 +233,6 @@ class MergeChild(SpyglassMixin, dj.Manual): @pytest.fixture(scope="module") def graph_tables(dj_conn, graph_schema): - schema = dj.Schema(context=graph_schema) for table in graph_schema.values(): From 1071910ac29f081d280cb50d96185b88436d35f7 Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Mon, 5 Aug 2024 15:47:07 -0500 Subject: [PATCH 26/94] Re-organize docs (#1029) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Docs Overhaul. Ignore pre-commit * Note pre-commit exception * Use most recent tag to deploy docs on manual dispatch * Use marketplace git describe for tag * Fix typo * Upgrade workflow deps * Test tag fetch * Minor fixes * Fix failing tests * 🐛 : Debug tests * Update C * Add params edit snippet * Mention optional dependencies in 00_Setup * SpikesortV0: mention version * Add nwb-obj/tbl mappings from @samuelbray32 * Revert version bump --- .github/workflows/publish-docs.yml | 14 +- .github/workflows/test-package-build.yml | 2 +- .markdownlint.yaml | 6 +- CHANGELOG.md | 12 +- CITATION.cff | 14 +- README.md | 40 +- docs/README.md | 10 +- docs/build-docs.sh | 21 +- docs/mkdocs.yml | 35 +- .../{misc/export.md => Features/Export.md} | 0 .../figurl_views.md => Features/FigURL.md} | 0 .../merge_tables.md => Features/Merge.md} | 0 docs/src/{misc/mixin.md => Features/Mixin.md} | 18 +- .../SessionGroups.md} | 0 docs/src/Features/index.md | 12 + docs/src/ForDevelopers/Contribute.md | 57 +++ .../Management.md} | 0 docs/src/ForDevelopers/Reuse.md | 359 +++++++++++++ docs/src/ForDevelopers/Schema.md | 483 ++++++++++++++++++ docs/src/ForDevelopers/TableTypes.md | 108 ++++ docs/src/ForDevelopers/UsingNWB.md | 269 ++++++++++ docs/src/ForDevelopers/index.md | 37 ++ docs/src/api/index.md | 35 +- docs/src/contribute.md | 244 --------- docs/src/images/merge_diagram.png | Bin 98482 -> 63402 bytes docs/src/images/merge_diagram_large.png | Bin 0 -> 98482 bytes docs/src/index.md | 34 +- docs/src/installation.md | 127 ----- docs/src/misc/common_errs.md | 111 ---- docs/src/misc/index.md | 12 - docs/src/misc/insert_data.md | 101 ---- notebooks/00_Setup.ipynb | 331 ++++++++++-- notebooks/01_Concepts.ipynb | 218 ++++++++ ...Insert_Data.ipynb => 02_Insert_Data.ipynb} | 232 +++++++++ ...{02_Data_Sync.ipynb => 03_Data_Sync.ipynb} | 0 ...rge_Tables.ipynb => 04_Merge_Tables.ipynb} | 0 notebooks/04_PopulateConfigFile.ipynb | 273 ---------- notebooks/10_Spike_SortingV0.ipynb | 6 +- notebooks/21_DLC.ipynb | 35 ++ notebooks/README.md | 22 +- notebooks/py_scripts/00_Setup.py | 295 +++++++++-- notebooks/py_scripts/01_Concepts.py | 170 ++++++ .../{01_Insert_Data.py => 02_Insert_Data.py} | 113 +++- .../{02_Data_Sync.py => 03_Data_Sync.py} | 0 ...{03_Merge_Tables.py => 04_Merge_Tables.py} | 0 notebooks/py_scripts/04_PopulateConfigFile.py | 194 ------- notebooks/py_scripts/10_Spike_SortingV0.py | 8 +- notebooks/py_scripts/21_DLC.py | 29 ++ pyproject.toml | 2 +- src/spyglass/decoding/v1/clusterless.py | 2 - src/spyglass/position/v1/dlc_utils_makevid.py | 2 +- src/spyglass/utils/dj_helper_fn.py | 2 +- src/spyglass/utils/dj_mixin.py | 14 +- tests/position/test_dlc_cent.py | 5 +- tests/position/test_dlc_proj.py | 5 +- 55 files changed, 2861 insertions(+), 1258 deletions(-) rename docs/src/{misc/export.md => Features/Export.md} (100%) rename docs/src/{misc/figurl_views.md => Features/FigURL.md} (100%) rename docs/src/{misc/merge_tables.md => Features/Merge.md} (100%) rename docs/src/{misc/mixin.md => Features/Mixin.md} (91%) rename docs/src/{misc/session_groups.md => Features/SessionGroups.md} (100%) create mode 100644 docs/src/Features/index.md create mode 100644 docs/src/ForDevelopers/Contribute.md rename docs/src/{misc/database_management.md => ForDevelopers/Management.md} (100%) create mode 100644 docs/src/ForDevelopers/Reuse.md create mode 100644 docs/src/ForDevelopers/Schema.md create mode 100644 docs/src/ForDevelopers/TableTypes.md create mode 100644 docs/src/ForDevelopers/UsingNWB.md create mode 100644 docs/src/ForDevelopers/index.md delete mode 100644 docs/src/contribute.md create mode 100644 docs/src/images/merge_diagram_large.png delete mode 100644 docs/src/installation.md delete mode 100644 docs/src/misc/common_errs.md delete mode 100644 docs/src/misc/index.md delete mode 100644 docs/src/misc/insert_data.md create mode 100644 notebooks/01_Concepts.ipynb rename notebooks/{01_Insert_Data.ipynb => 02_Insert_Data.ipynb} (94%) rename notebooks/{02_Data_Sync.ipynb => 03_Data_Sync.ipynb} (100%) rename notebooks/{03_Merge_Tables.ipynb => 04_Merge_Tables.ipynb} (100%) delete mode 100644 notebooks/04_PopulateConfigFile.ipynb create mode 100644 notebooks/py_scripts/01_Concepts.py rename notebooks/py_scripts/{01_Insert_Data.py => 02_Insert_Data.py} (81%) rename notebooks/py_scripts/{02_Data_Sync.py => 03_Data_Sync.py} (100%) rename notebooks/py_scripts/{03_Merge_Tables.py => 04_Merge_Tables.py} (100%) delete mode 100644 notebooks/py_scripts/04_PopulateConfigFile.py diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index db1cf6224..28c566665 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -2,7 +2,7 @@ name: Publish docs on: push: tags: # See PEP 440 for valid version format - - "*.*.*" # For docs bump, use X.X.XaX + - "*.*.*" # For docs bump, use workflow_dispatch branches: - test_branch workflow_dispatch: # Manually trigger with 'Run workflow' button @@ -23,13 +23,17 @@ jobs: issues: write steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: token: ${{ secrets.GITHUB_TOKEN }} fetch-depth: 0 + - name: Git describe # Get tags + id: ghd # see Deploy below. Will fail if no tags on branch + uses: proudust/gh-describe@v2 + - name: Set up Python runtime - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.9 token: ${{ secrets.GITHUB_TOKEN }} @@ -40,8 +44,8 @@ jobs: git config user.name 'github-actions[bot]' && git config user.email 'github-actions[bot]@users.noreply.github.com' - name: Deploy - run: | - FULL_VERSION=${{ github.ref_name }} + run: | # github.ref_name is branch name if dispatch + FULL_VERSION=${{ steps.ghd.outputs.tag }} export MAJOR_VERSION=${FULL_VERSION:0:3} echo "OWNER: ${REPO_OWNER}. BUILD: ${MAJOR_VERSION}" bash ./docs/build-docs.sh push $REPO_OWNER diff --git a/.github/workflows/test-package-build.yml b/.github/workflows/test-package-build.yml index c93b77398..3513bb664 100644 --- a/.github/workflows/test-package-build.yml +++ b/.github/workflows/test-package-build.yml @@ -64,7 +64,7 @@ jobs: with: name: archive path: archive/ - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: 3.9 - name: Display Python version diff --git a/.markdownlint.yaml b/.markdownlint.yaml index 5c9c3712b..f57fbf732 100644 --- a/.markdownlint.yaml +++ b/.markdownlint.yaml @@ -1,8 +1,10 @@ # https://github.com/DavidAnson/markdownlint # https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md -MD007: false # permit indenting 4 spaces instead of 2 +MD007: # permit indenting 4 spaces instead of 2 + indent: 4 + start_indent: 4 MD013: - line_length: "80" # Line length limits + line_length: 80 # Line length limits tables: false # disable for tables code_blocks: false # disable for code blocks MD025: false # permit adjacent headings diff --git a/CHANGELOG.md b/CHANGELOG.md index 449ccd8de..ba21ee169 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## [0.5.3] (Unreleased) -### Release Notes +## Release Notes @@ -36,10 +36,14 @@ PositionGroup.alter() - Ensure integrity of group tables #1026 - Convert list of LFP artifact removed interval list to array #1046 - Merge duplicate functions in decoding and spikesorting #1050 +- Revise docs organization. + - Misc -> Features/ForDevelopers. #1029 + - Installation instructions -> Setup notebook. #1029 ### Pipelines - Common + - `PositionVideo` table now inserts into self after `make` #966 - Don't insert lab member when creating lab team #983 - Files created by `AnalysisNwbfile.create()` receive new object_id #999 @@ -51,10 +55,14 @@ PositionGroup.alter() - `PositionIntervalMap` now inserts null entries for missing intervals #870 - `AnalysisFileLog` now truncates table names that exceed field length #1021 - Disable logging with `AnalysisFileLog` #1024 + - Decoding: + - Default values for classes on `ImportError` #966 - Add option to upsample data rate in `PositionGroup` #1008 + - Position + - Allow dlc without pre-existing tracking data #973, #975 - Raise `KeyError` for missing input parameters across helper funcs #966 - `DLCPosVideo` table now inserts into self after `make` #966 @@ -67,7 +75,9 @@ PositionGroup.alter() `get_video_info` to reflect actual use #870 - Fix `red_led_bisector` `np.nan` handling issue from #870. Fixed in #1034 - Fix `one_pt_centoid` `np.nan` handling issue from #870. Fixed in #1034 + - Spikesorting + - Allow user to set smoothing timescale in `SortedSpikesGroup.get_firing_rate` #994 - Update docstrings #996 diff --git a/CITATION.cff b/CITATION.cff index 6fc0e83aa..ed9dd3cc5 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -2,7 +2,7 @@ # Visit https://bit.ly/cffinit to generate yours today! cff-version: 1.2.0 -title: spyglass +title: 'Spyglass: a data analysis framework for reproducible and shareable neuroscience research' message: 'If you use this software, please cite it as below.' type: software authors: @@ -84,18 +84,18 @@ authors: email: emrey.broyles@ucsf.edu affiliation: 'University of California, San Francisco' orcid: 'https://orcid.org/0000-0001-5559-2910' - - given-names: Shin - family-names: Donghoon + - given-names: Donghoon + family-names: Shin email: donghoon.shin@ucsf.edu affiliation: 'University of California, San Francisco' orcid: 'https://orcid.org/0009-0000-8916-7314' - - given-names: Chiang - family-names: Sharon + - given-names: Sharon + family-names: Chiang email: sharon.chiang@ucsf.edu affiliation: 'University of California, San Francisco' orcid: 'https://orcid.org/0000-0002-4548-4550' - - given-names: Holobetz - family-names: Cristofer + - given-names: Cristofer + family-names: Holobetz email: cristofer.holobetz.23@ucl.ac.uk affiliation: 'University College London' orcid: 'https://orcid.org/0009-0009-8567-3290' diff --git a/README.md b/README.md index 42c0e0357..d2e4e5e47 100644 --- a/README.md +++ b/README.md @@ -5,14 +5,20 @@ ![Spyglass Figure](docs/src/images/fig1.png) -[Demo](https://spyglass.hhmi.2i2c.cloud/hub/user-redirect/git-pull?repo=https%3A%2F%2Fgithub.com%2FLorenFrankLab%2Fspyglass-demo&urlpath=lab%2Ftree%2Fspyglass-demo%2Fnotebooks%2F01_Insert_Data.ipynb&branch=main) | [Installation](https://lorenfranklab.github.io/spyglass/latest/installation/) | [Docs](https://lorenfranklab.github.io/spyglass/) | [Tutorials](https://github.com/LorenFrankLab/spyglass/tree/master/notebooks) | [Citation](#citation) +[Demo](https://spyglass.hhmi.2i2c.cloud/hub/user-redirect/git-pull?repo=https%3A%2F%2Fgithub.com%2FLorenFrankLab%2Fspyglass-demo&urlpath=lab%2Ftree%2Fspyglass-demo%2Fnotebooks%2F02_Insert_Data.ipynb&branch=main) +| +[Installation](https://lorenfranklab.github.io/spyglass/latest/notebooks/00_Setup/) +| [Docs](https://lorenfranklab.github.io/spyglass/) | +[Tutorials](https://github.com/LorenFrankLab/spyglass/tree/master/notebooks) | +[Citation](#citation) `spyglass` is a data analysis framework that facilitates the storage, analysis, visualization, and sharing of neuroscience data to support reproducible research. It is designed to be interoperable with the NWB format and integrates open-source tools into a coherent framework. -Try out a demo [here](https://spyglass.hhmi.2i2c.cloud/hub/user-redirect/git-pull?repo=https%3A%2F%2Fgithub.com%2FLorenFrankLab%2Fspyglass-demo&urlpath=lab%2Ftree%2Fspyglass-demo%2Fnotebooks%2F01_Insert_Data.ipynb&branch=main)! +Try out a demo +[here](https://spyglass.hhmi.2i2c.cloud/hub/user-redirect/git-pull?repo=https%3A%2F%2Fgithub.com%2FLorenFrankLab%2Fspyglass-demo&urlpath=lab%2Ftree%2Fspyglass-demo%2Fnotebooks%2F02_Insert_Data.ipynb&branch=main)! Features of Spyglass include: @@ -60,16 +66,16 @@ Documentation can be found at - ## Installation For installation instructions see - -[https://lorenfranklab.github.io/spyglass/latest/installation/](https://lorenfranklab.github.io/spyglass/latest/installation/) +[https://lorenfranklab.github.io/spyglass/latest/notebooks/00_Setup/](https://lorenfranklab.github.io/spyglass/latest/notebooks/00_Setup/) Typical installation time is: 5-10 minutes ## Tutorials -The tutorials for `spyglass` is currently in the form of Jupyter Notebooks and +The tutorials for `spyglass` are currently in the form of Jupyter Notebooks and can be found in the [notebooks](https://github.com/LorenFrankLab/spyglass/tree/master/notebooks) -directory. We strongly recommend opening them in the context of `jupyterlab`. +directory. We strongly recommend running the notebooks yourself. ## Contributing @@ -85,17 +91,31 @@ License and Copyright notice can be found at ## System requirements -Spyglass has been tested on Linux Ubuntu 20.04 and MacOS 10.15. It has not been tested on Windows and likely will not work. +Spyglass has been tested on Linux Ubuntu 20.04 and MacOS 10.15. It has not been +tested on Windows and likely will not work. -No specific hardware requirements are needed to run spyglass. However, the amount of data that can be stored and analyzed is limited by the available disk space and memory. GPUs are required for some of the analysis tools, such as DeepLabCut. +No specific hardware requirements are needed to run spyglass. However, the +amount of data that can be stored and analyzed is limited by the available disk +space and memory. GPUs are required for some of the analysis tools, such as +DeepLabCut. -See [pyproject.toml](pyproject.toml), [environment.yml](environment.yml), or [environment_dlc.yml](environment_dlc.yml) for software dependencies. +See [pyproject.toml](pyproject.toml), [environment.yml](environment.yml), or +[environment_dlc.yml](environment_dlc.yml) for software dependencies. -See [spec-file.txt](https://github.com/LorenFrankLab/spyglass-demo/blob/main/spec-file/spec-file.txt) for the conda environment used in the demo. +See +[spec-file.txt](https://github.com/LorenFrankLab/spyglass-demo/blob/main/spec-file/spec-file.txt) +for the conda environment used in the demo. ## Citation -> Lee, K.H.\*, Denovellis, E.L.\*, Ly, R., Magland, J., Soules, J., Comrie, A.E., Gramling, D.P., Guidera, J.A., Nevers, R., Adenekan, P., Brozdowski, C., Bray, S., Monroe, E., Bak, J.H., Coulter, M.E., Sun, X., Broyles, E., Shin, D., Chiang, S., Holobetz, C., Tritt, A., Rübel, O., Nguyen, T., Yatsenko, D., Chu, J., Kemere, C., Garcia, S., Buccino, A., Frank, L.M., 2024. Spyglass: a data analysis framework for reproducible and shareable neuroscience research. bioRxiv. [10.1101/2024.01.25.577295](https://doi.org/10.1101/2024.01.25.577295). +> Lee, K.H.\*, Denovellis, E.L.\*, Ly, R., Magland, J., Soules, J., Comrie, +> A.E., Gramling, D.P., Guidera, J.A., Nevers, R., Adenekan, P., Brozdowski, C., +> Bray, S., Monroe, E., Bak, J.H., Coulter, M.E., Sun, X., Broyles, E., Shin, +> D., Chiang, S., Holobetz, C., Tritt, A., Rübel, O., Nguyen, T., Yatsenko, D., +> Chu, J., Kemere, C., Garcia, S., Buccino, A., Frank, L.M., 2024. Spyglass: a +> data analysis framework for reproducible and shareable neuroscience research. +> bioRxiv. +> [10.1101/2024.01.25.577295](https://doi.org/10.1101/2024.01.25.577295). *\* Equal contribution* diff --git a/docs/README.md b/docs/README.md index 8eee3f1a4..0ae399532 100644 --- a/docs/README.md +++ b/docs/README.md @@ -16,11 +16,9 @@ The remainder of `mkdocs.yml` specifies the site's ## GitHub Whenever a new tag is pushed, GitHub actions will run -`.github/workflows/publish-docs.yml`. Progress can be monitored in the 'Actions' -tab within the repo. - -Releases should be tagged with `X.Y.Z`. A tag to redeploy docs should use the -current version, with an alpha release suffix, e.g. `X.Y.Za1`. +`.github/workflows/publish-docs.yml`. From the repository, select the Actions +tab, and then the 'Publish Docs' workflow on the left to monitor progress. The +process can also be manually triggered by selecting 'Run workflow' on the right. To deploy on your own fork without a tag, follow turn on github pages in settings, following a `documentation` branch, and then push to `test_branch`. @@ -47,7 +45,7 @@ the root notebooks directory may not be reflected when rebuilding. Use a browser to navigate to `localhost:8000/` to inspect the site. For auto-reload of markdown files during development, use `mkdocs serve -f ./docs/mkdosc.yaml`. The `mike` package used in the build -script manages versioning, but does not support dynamic versioning. +script manages versioning, but does not support dynamic reloading. The following items can be commented out in `mkdocs.yml` to reduce build time: diff --git a/docs/build-docs.sh b/docs/build-docs.sh index 50d44f511..44b383853 100755 --- a/docs/build-docs.sh +++ b/docs/build-docs.sh @@ -14,12 +14,25 @@ mv ./docs/src/notebooks/README.md ./docs/src/notebooks/index.md cp -r ./notebook-images ./docs/src/notebooks/ cp -r ./notebook-images ./docs/src/ -if [ -z "$MAJOR_VERSION" ]; then # Get version from file - version_line=$(grep "__version__ =" ./src/spyglass/_version.py) - version_string=$(echo "$version_line" | awk -F"[\"']" '{print $2}') +# Function for checking major version format: #.# +check_format() { + local version="$1" + if [[ $version =~ ^[0-9]+\.[0-9]+$ ]]; then + return 0 + else + return 1 + fi +} + +# Check if the MAJOR_VERSION not defined or does not meet format criteria +if [ -z "$MAJOR_VERSION" ] || ! check_format "$MAJOR_VERSION"; then + full_version=$(git describe --tags --abbrev=0) export MAJOR_VERSION="${version_string:0:3}" fi -echo "$MAJOR_VERSION" # May be available as env var +if ! check_format "$MAJOR_VERSION"; then + export MAJOR_VERSION="dev" # Fallback to dev if still not valid +fi +echo "$MAJOR_VERSION" # Get ahead of errors export JUPYTER_PLATFORM_DIRS=1 diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 88734f3a0..30d2bd79d 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -42,15 +42,14 @@ theme: nav: - Home: index.md - - Installation: installation.md - Tutorials: - Overview: notebooks/index.md - Intro: - Setup: notebooks/00_Setup.ipynb - - Insert Data: notebooks/01_Insert_Data.ipynb - - Data Sync: notebooks/02_Data_Sync.ipynb - - Merge Tables: notebooks/03_Merge_Tables.ipynb - - Config Populate: notebooks/04_PopulateConfigFile.ipynb + - Concepts: notebooks/01_Concepts.ipynb + - Insert Data: notebooks/02_Insert_Data.ipynb + - Data Sync: notebooks/03_Data_Sync.ipynb + - Merge Tables: notebooks/04_Merge_Tables.ipynb - Export: notebooks/05_Export.ipynb - Spikes: - Spike Sorting V0: notebooks/10_Spike_SortingV0.ipynb @@ -70,18 +69,22 @@ nav: - Decoding Clusterless: notebooks/41_Decoding_Clusterless.ipynb - Decoding Sorted Spikes: notebooks/42_Decoding_SortedSpikes.ipynb - MUA Detection: notebooks/50_MUA_Detection.ipynb - - Miscellaneous: - - Overview: misc/index.md - - Common Errors: misc/common_errs.md - - Database Management: misc/database_management.md - - Export: misc/export.md - - FigURL: misc/figurl_views.md - - Insert Data: misc/insert_data.md - - Merge Tables: misc/merge_tables.md - - Mixin: misc/mixin.md - - Session Groups: misc/session_groups.md + - Features: + - Overview: Features/index.md + - FigURL: Features/FigURL.md + - Merge Tables: Features/Merge.md + - Export: Features/Export.md + - Session Groups: Features/SessionGroups.md + - Centralized Code: Features/Mixin.md + - For Developers: + - Overview: ForDevelopers/index.md + - How to Contribute: ForDevelopers/Contribute.md + - Database Management: ForDevelopers/Management.md + - Code Reuse: ForDevelopers/Reuse.md + - Table Types: ForDevelopers/TableTypes.md + - Understanding a Schema: ForDevelopers/Schema.md + - Using NWB: ForDevelopers/UsingNWB.md - API Reference: api/ # defer to gen-files + literate-nav - - How to Contribute: contribute.md - Change Log: CHANGELOG.md - Copyright: LICENSE.md diff --git a/docs/src/misc/export.md b/docs/src/Features/Export.md similarity index 100% rename from docs/src/misc/export.md rename to docs/src/Features/Export.md diff --git a/docs/src/misc/figurl_views.md b/docs/src/Features/FigURL.md similarity index 100% rename from docs/src/misc/figurl_views.md rename to docs/src/Features/FigURL.md diff --git a/docs/src/misc/merge_tables.md b/docs/src/Features/Merge.md similarity index 100% rename from docs/src/misc/merge_tables.md rename to docs/src/Features/Merge.md diff --git a/docs/src/misc/mixin.md b/docs/src/Features/Mixin.md similarity index 91% rename from docs/src/misc/mixin.md rename to docs/src/Features/Mixin.md index 23135d3c4..ac227a7be 100644 --- a/docs/src/misc/mixin.md +++ b/docs/src/Features/Mixin.md @@ -6,7 +6,7 @@ functionalities that have been added to DataJoint tables. This includes... - Fetching NWB files - Long-distance restrictions. - Delete functionality, including permission checks and part/master pairs -- Export logging. See [export doc](export.md) for more information. +- Export logging. See [export doc](./Export.md) for more information. To add this functionality to your own tables, simply inherit from the mixin: @@ -53,8 +53,8 @@ to a `_nwb_table` attribute. In complicated pipelines like Spyglass, there are often tables that 'bury' their foreign keys as secondary keys. This is done to avoid having to pass a long list of foreign keys through the pipeline, potentially hitting SQL limits (see also -[Merge Tables](./merge_tables.md)). This burrying makes it difficult to restrict -a given table by familiar attributes. +[Merge Tables](./Merge.md)). This burrying makes it difficult to restrict a +given table by familiar attributes. Spyglass provides a function, `restrict_by`, to handle this. The function takes your restriction and checks parents/children until the restriction can be @@ -122,7 +122,7 @@ If the user shares a lab team with the session experimenter, the deletion is permitted. This is not secure system and is not a replacement for database backups (see -[database management](./database_management.md)). A user could readily +[database management](../ForDevelopers/Management.md)). A user could readily curcumvent the default permission checks by adding themselves to the relevant team or removing the mixin from the class declaration. However, it provides a reasonable level of security for the average user. @@ -134,11 +134,11 @@ entry without deleting the corresponding master. This is useful for enforcing the custom of adding/removing all parts of a master at once and avoids orphaned masters, or null entry masters without matching data. -For [Merge tables](./merge_tables.md), this is a significant problem. If a user -wants to delete all entries associated with a given session, she must find all -part table entries, including Merge tables, and delete them in the correct -order. The mixin provides a function, `delete_downstream_parts`, to handle this, -which is run by default when calling `delete`. +For [Merge tables](./Merge.md), this is a significant problem. If a user wants +to delete all entries associated with a given session, she must find all part +table entries, including Merge tables, and delete them in the correct order. The +mixin provides a function, `delete_downstream_parts`, to handle this, which is +run by default when calling `delete`. `delete_downstream_parts`, also aliased as `ddp`, identifies all part tables with foreign key references downstream of where it is called. If `dry_run=True`, diff --git a/docs/src/misc/session_groups.md b/docs/src/Features/SessionGroups.md similarity index 100% rename from docs/src/misc/session_groups.md rename to docs/src/Features/SessionGroups.md diff --git a/docs/src/Features/index.md b/docs/src/Features/index.md new file mode 100644 index 000000000..e8399f84a --- /dev/null +++ b/docs/src/Features/index.md @@ -0,0 +1,12 @@ +# Features + +This directory contains a series of explainers on tools that have been added to +Spyglass. + +- [Export](./Export.md) - How to export an analysis. +- [FigURL](./FigURL.md) - How to use FigURL to share figures. +- [Merge Tables](./Merge.md) - Tables for pipeline versioning. +- [Mixin](./Mixin.md) - Spyglass-specific functionalities to DataJoint tables, + including fetching NWB files, long-distance restrictions, and permission + checks on delete operations. +- [Session Groups](./SessionGroups.md) - How to operate on sets of sessions. diff --git a/docs/src/ForDevelopers/Contribute.md b/docs/src/ForDevelopers/Contribute.md new file mode 100644 index 000000000..6a58ea792 --- /dev/null +++ b/docs/src/ForDevelopers/Contribute.md @@ -0,0 +1,57 @@ +# Contributing to Spyglass + +This document provides an overview of the Spyglass development, and provides +guidance for folks looking to contribute to the project itself. For information +on setting up custom tables, skip to Code Organization. + +## Development workflow + +New contributors should follow the +[Fork-and-Branch workflow](https://www.atlassian.com/git/tutorials/comparing-workflows/forking-workflow). +See GitHub instructions +[here](https://docs.github.com/en/get-started/quickstart/contributing-to-projects). + +Regular contributors may choose to follow the +[Feature Branch Workflow](https://www.atlassian.com/git/tutorials/comparing-workflows/feature-branch-workflow) +for features that will involve multiple contributors. + +## Code organization + +- Tables are grouped into schemas by topic (e.g., `common_metrics`) +- Schemas + - Are defined in a `py` pile. + - Correspond to MySQL 'databases'. + - Are organized into modules (e.g., `common`) by folders. +- The _common_ module + - In principle, contains schema that are shared across all projects. + - In practice, contains shared tables (e.g., Session) and the first draft of + schemas that have since been split into their own + modality-specific\ + modules (e.g., `lfp`) + - Should not be added to without discussion. +- A pipeline + - Refers to a set of tables used for processing data of a particular modality + (e.g., LFP, spike sorting, position tracking). + - May span multiple schema. +- For analysis that will be only useful to you, create your own schema. + +## Misc + +- During development, we suggest using a Docker container. See + [example](../notebooks/00_Setup.ipynb). +- `numpy` style docstrings will be interpreted by API docs. To check for + compliance, monitor the output when building docs (see `docs/README.md`) + +## Making a release + +Spyglass follows [Semantic Versioning](https://semver.org/) with versioning of +the form `X.Y.Z` (e.g., `0.4.2`). + +1. In `CITATION.cff`, update the `version` key. +2. Make a pull request with changes. +3. After the pull request is merged, pull this merge commit and tag it with + `git tag {version}` +4. Publish the new release tag. Run `git push origin {version}`. This will + rebuild docs and push updates to PyPI. +5. Make a new + [release on GitHub](https://docs.github.com/en/repositories/releasing-projects-on-github/managing-releases-in-a-repository). diff --git a/docs/src/misc/database_management.md b/docs/src/ForDevelopers/Management.md similarity index 100% rename from docs/src/misc/database_management.md rename to docs/src/ForDevelopers/Management.md diff --git a/docs/src/ForDevelopers/Reuse.md b/docs/src/ForDevelopers/Reuse.md new file mode 100644 index 000000000..baf969ecc --- /dev/null +++ b/docs/src/ForDevelopers/Reuse.md @@ -0,0 +1,359 @@ +# Coding for Reuse + + + +*Reusing code requires that it be faster to read and change than it would be to +start from scratch.* + +We can speed up that process by ... + +1. Making reading predictable. +2. Atomizing - separating pieces into the smallest meaningful chunks. +3. Leaving notes via type hints, docstrings, and comments +4. Getting ahead of errors +5. Automating as much of the above as possible. + +This document pulls from resources like +[Tidy First](https://www.oreilly.com/library/view/tidy-first/9781098151232/) and +[SOLID Principles](https://arjancodes.com/blog/solid-principles-in-python-programming/). +Experienced object-oriented developers may find these principles familiar. + +## Predictable Formatting + +- Many programming languages offer flexibility in how they are written. +- Tools like `black` and `isort` take away stylistic preferences in favor of one + norm. +- Strict line limits (e.g., 80) make it easier to do side by side comparisons in + git interfaces. +- `black` is also useful for detecting an error on save - if it doesn't run on + what you wrote, there's an error somewhere. + +Let's look at a few examples of the same code block formatted different ways... + +### Original + +```python +def get_data_interface(nwbfile, data_interface_name, data_interface_class=None, unused_other_arg=None): + ret = { 'centroid_method': "two_pt_centroid", 'points': {'point1': 'greenLED', "point2": 'redLED_C'}, 'interpolate': True} + for module in nwbfile.processing.values(): + match = module.data_interfaces.get(data_interface_name, None) + if match is not None: + if data_interface_class is not None and not isinstance(match, data_interface_class): + continue + ret.append(match) + if len(ret) > 1: + print(f"Multiple data interfaces with name '{data_interface_name}' found with identifier {nwbfile.identifier}.") + if len(ret) >= 1: + return ret[0] + return None +``` + +### Black formatted + +With `black`, we have a limited line length and indents reflect embedding. + +```python +def get_data_interface( # Each arg gets its own line + nwbfile, + data_interface_name, + data_interface_class=None, + unused_other_arg=None, +): + ret = { # dictionaries show embedding + "centroid_method": "two_pt_centroid", + "points": { + "point1": "greenLED", + "point2": "redLED_C", + }, + "interpolate": True, + } + for module in nwbfile.processing.values(): + match = module.data_interfaces.get(data_interface_name, None) + if match is not None: + if data_interface_class is not None and not isinstance( + match, data_interface_class + ): # long lines broken up + continue + ret.append(match) + if len(ret) > 1: + print( # long strings need to be broken up manually + f"Multiple data interfaces with name '{data_interface_name}' " + f"found in NWBFile with identifier {nwbfile.identifier}. " + ) + if len(ret) >= 1: + return ret[0] + return None +``` + +### Control flow adjustments + +Although subjective, we can do even better by adjusting the logic to follow how +we read. + +```python +from typing import Type +def get_data_interface(...): + ret = {...} + # decide no input early + data_interface_class = data_interface_class or Type + for match in [ # generate via list comprehension + module.get_data_interface(data_interface_name) + for module in nwbfile.processing.values() + ]: # only process good case, no `continue` + if match and isinstance(match, data_interface_class): + ret.append(match) + if len(ret) > 1: + print(...) + return ret[0] if len(ret) >= 1 else None # fits on one line +``` + +## Atomizing + +Working memory limits our ability to understand long code blocks. + +We can extract pieces into separate places to give them a name and make 'one' +memory chunk out of a set of functions. + +Depending on the scope, chunks can be separated with ... + +1. Paragraph breaks - to group instructions together. +2. Conditional assignment - for data maps local to a function. +3. Methods of a class - for functions that deserve a separate name. +4. Helpers in a script - for functions used multiple times in a schema. +5. Util scripts in a package - for functions used throughout a project. + +### Atomizing example + +- Let's read the next function as if we're revisiting old code. +- This example was taken from an existing project and adjusted for + demonstration. +- Please review without commentary and make mental notes ow what each part line + is doing and how they relate to other lines. + +
No commentary + +```python +class MyTable(dj.Computed): + ... + + def make(self, key): + rat_name = key["rat_name"] + ron_all_dict = {"some_data": 1} + tonks_all_dict = {"other_data": 2} + try: + if len((OtherTable & key).fetch("cluster_id")[0]) > 0: + if rat_name == "ron": + data_dict = ron_all_dict + elif rat_name == "tonks": + data_dict = tonks_all_dict + else: + raise ValueError(f"Unsupported rat {rat_name}") + for data_key, data_value in data_dict.items(): + try: + if data_value == 1: + cluster_spike_times = (OtherTable & key).fetch_nwb()[ + 0 + ]["units"]["spike_times"] + else: + cluster_spike_times = (OtherTable & key).fetch_nwb()[ + data_value - 1 + ]["units"]["spike_times"][data_key] + self.insert1(cluster_spike_times) + except KeyError: + print("cluster missing", key["nwb_file_name"]) + else: + print("no spikes") + except IndexError: + print("no data") +``` + +
+ +
With Commentary + +Note how the numbers correspond to their counterparts - 1Q, 1A, 2Q, 2A ... + +```python +class MyTable(dj.Computed): + ... + def make(self, key): + rat_name = key["rat_name"] # 1Q. Can this function handle others? + ron_all_dict = {"some_data": 1} # 2Q. Are these parameters? + tonks_all_dict = {"other_data": 2} + try: # 3Q. What error could be thrown? And by what? + if len((OtherTable & key).fetch("cluster_id")[0]) > 0: # 4Q. What happens if none? + if rat_name == "ron": + data_dict = ron_all_dict # 2A. ok, we decide the data here + elif rat_name == "tonks": + data_dict = tonks_all_dict + else: # 1Q. Ok, we can only do these two + raise ValueError(f"Unsupported rat {rat_name}") + for data_key, data_value in data_dict.items(): # 2A. Maybe parameter? + try: # 5Q. What could throw an error? + if data_value == 1: + cluster_spike_times = (OtherTable & key).fetch_nwb()[ + 0 + ]["units"]["spike_times"] # 6Q. What do we need this for? + else: + cluster_spike_times = (OtherTable & key).fetch_nwb()[ + data_value - 1 + ]["units"]["spike_times"][data_key] + self.insert1(cluster_spike_times) # 6A. Ok, insertion + except KeyError: # 5A. Mayble this fetch is unreliable? + print("cluster missing", key["nwb_file_name"]) + else: + print("no spikes") # 4A. Ok we bail if no clusters + except IndexError: # 3A. What could have thrown this? Are we sure nothing else? + print("no data") +``` + +
+ +### Embedding + +- The process of stream of consciousness coding often generates an embedding + trail from core out +- Our mental model of A -> B -> C -> D may actually read like `D( C( B( A )))` + or ... + +1. Prepare for D +2. Open a loop for C +3. Add caveat B +4. Do core process A +5. Check other condition B +6. Close D + +Let's contrast with an approach that reduces embedding. + +```python +class MyTable(dj.Computed): + ... + def _get_cluster_times(self, key, nth_file, index): # We will need times + clust = (OtherTable & key).fetch_nwb()[nth_file]["units"]["spike_times"] + try: # Looks like this indexing may not return the data + return clust[index] if nth_file == 0 else clust # if/then handled here + except KeyError: # Show as err, keep moving + logger.error("Cluster missing", key["nwb_file_name"]) + + def make(self, key): + rat_paramsets = {"ron": {"some_data": 1}, "tonks": {"other_data": 2}} # informative variable name + if (rat_name := key["rat_name"]) not in rat_paramsets: # walrus operator `:=` can assign within `if` + raise ValueError(f"Unsupported rat {rat_name}") # we can only handle a subset a rats + rat_params = rat_paramsets[rat_name] # conditional assignment + + if not len((OtherTable & key).fetch("cluster_id")[0]): # paragraph breaks separate chunks conceptually + logger.info(f"No spikes for {key}") # log level can be adjusted at run + + insertion_list = [] # We're gonna insert something + for file_index, file_n in rat_params.items(): + insertion_list.append( + self._get_cluster_times(key, file_n - 1, file_index) # there it is, clusters + ) + self.insert(insertion_list) # separate inserts to happen all at once +``` + +## Comments, Type hints and docstrings + +It's tempting to leave comments in code, but they can become outdated and +confusing. Instead try Atomizing and using Type hints and docstrings. + +Type hints are not enforced, but make it much easier to tell the design intent +when reread. Docstrings are similarly optional, but make it easy to get prompts +without looking at the code again via `help(myfunc)` + +### Type hints + +```python +def get_data_interface( + nwbfile: pynwb.Nwbfile, + data_interface_name: Union[str, list], # one or the other + other_arg: Dict[str, Dict[str, dj.FreeTable]] = None, # show embedding +) -> NWBDataInterface: # What it returns. `None` if no return + pass +``` + +### Docstrings + +- Spyglass uses the NumPy docstring style, as opposed to Google. +- These are rendered in the + [API documentation](https://lorenfranklab.github.io/spyglass/latest/api/utils/nwb_helper_fn/#src.spyglass.utils.nwb_helper_fn.get_data_interface) + +```python +def get_data_interface(*args, **kwargs): + """One-line description. + + Additional notes or further description in case the one line above is + not enough. + + Parameters + ---------- + nwbfile : pynwb.NWBFile + Description of the arg. e.g., The NWB file object to search in. + data_interface_name : Union[str, list] + More here. + data_interface_class : Dict[str, Dict[str, dj.FreeTable]], optional + more here + + Warns + ----- + LoggerWarning + Why warn. + + Raises + ------ + ValueError + Why it would hit this error. + + Returns + ------- + data_interface : NWBDataInterface + + Example + ------- + > data_interface = get_data_interface(mynwb, "interface_name") + """ + pass +``` + +## Error detection with linting + +- Packages like `ruff` can show you bad code 'smells' while you write and fix + some for you. +- PEP8, Flake8 and other standards will flag issues like ... + - F401: Module imported but unused + - E402: Module level import not at top of file + - E713: Test for membership should be 'not in' +- `black` will fix a subset of Flake8 issues, but not all. `ruff` identifies or + fixes these rules and [many others](https://docs.astral.sh/ruff/rules/). + +## Automation + +- `black`, `isort`, and `ruff` can be run on save in most IDEs by searching + their extensions. +- `pre-commit` is a tool that can be used to run these checks before each + commit, ensuring that all your code is formatted, as defined in a `yaml` + file. + +```yaml +default_stages: [commit, push] +exclude: (^.github/|^docs/site/|^images/) + +repos: + - repo: https://github.com/ambv/black + rev: 24.1.1 + hooks: + - id: black + language_version: python3.9 + + - repo: https://github.com/codespell-project/codespell + rev: v2.2.6 + hooks: + - id: codespell + args: [--toml, pyproject.toml] + additional_dependencies: + - tomli +``` diff --git a/docs/src/ForDevelopers/Schema.md b/docs/src/ForDevelopers/Schema.md new file mode 100644 index 000000000..65df014ba --- /dev/null +++ b/docs/src/ForDevelopers/Schema.md @@ -0,0 +1,483 @@ +# Schema Design + +This document gives a detailed overview of how to read a schema script, +including explations of the different components that define a pipeline. + +1. Goals of a schema +2. Front matter + 1. Imports + 2. Schema declaration +3. Table syntax + 1. Class inheritance + 2. Explicit table types + 3. Definitions + 4. Methods +4. Conceptual table types + +Some of this will be redundant with general Python best practices and DataJoint +documentation, but it is important be able to read a schema, espically if you +plan to write your own. + +Later sections will depend on information presented in the article on +[Table Types](./TableTypes.md). + +## Goals of a schema + +- At its core, DataJoint is just a mapping between Python and SQL. +- SQL is a language for managing relational databases. +- DataJoint is opinionated about how to structure the database, and limits SQL's + potential options in way that promotes good practices. +- Python stores ... + - A copy of table definitions, that may be out of sync with the database. + - Methods for processing data, that may be out of sync with existing data. + +Good data provenance requires good version control and documentation to keep +these in sync. + +## Example schema + +This is the full example schema referenced in subsections below. + +
Full Schema + +```python +"""Schema example for custom pipelines + +Note: `noqa: F401` is a comment that tells linters to ignore the fact that +`Subject` seems unused in the file. If this table is only used in a table +definition string, the linter will not recognize it as being used. +""" + +import random # Package import +from typing import Union # Individual class import +from uuid import UUID + +import datajoint as dj # Aliased package import +from custom_package.utils import process_df, schema_prefix # custom functions +from spyglass.common import RawPosition, Subject # noqa: F401 +from spyglass.utils import SpyglassMixin # Additional Spyglass features + +schema = dj.schema(schema_prefix + "_example") # schema name from string + + +# Model to demonstrate DataJoint syntax +@schema # Decorator to define a table in the schema on the server +class ExampleTable(SpyglassMixin, dj.Manual): # Inherit SpyglassMixin class + """Table Description""" # Table docstring, one-line if possible + + definition = """ # Table comment + primary_key1 : uuid # randomized string + primary_key2 : int # integer + --- + secondary_key1 : varchar(32) # string of max length 32 + -> Subject # Foreign key reference, inherit primary key of this table + """ + + +# Model to demonstrate field aliasing with `proj` +@schema +class SubjBlinded(SpyglassMixin, dj.Manual): + """Blinded subject table.""" # Class docstring for `help()` + + definition = """ + subject_id: uuid # id + --- + -> Subject.proj(actual_id='subject_id') + """ + + @property # Static information, Table.property + def pk(self): + """Return the primary key""" # Function docstring for `help()` + return self.heading.primary_key + + @staticmethod # Basic func with no reference to self instance + def _subj_dict(subj_uuid: UUID): # Type hint for argument + """Return the subject dict""" + return {"subject_id": subj_uuid} + + @classmethod # Class, not instance. Table.func(), not Table().func() + def hash(cls, argument: Union[str, dict] = None): # Default value + """Example class method""" + return dj.hash.key_hash(argument) + + def blind_subjects(self, restriction: Union[str, dict]): # Union is "or" + """Import all subjects selected by the restriction""" + insert_keys = [ + { + **self._subj_dict(self.hash(key)), + "actual_id": key["subject_id"], + } + for key in (Subject & restriction).fetch("KEY") + ] + self.insert(insert_keys, skip_duplicates=True) + + def return_subj(self, key: str): + """Return the entry in subject table""" + if isinstance(key, dict): # get rid of extra values + key = key["subject_id"] + key = self._subj_dict(key) + actual_ids = (self & key).fetch("actual_id") + ret = [{"subject_id": actual_id} for actual_id in actual_ids] + return ret[0] if len(ret) == 1 else ret + + +@schema +class MyParams(SpyglassMixin, dj.Lookup): # Lookup allows for default values + """Parameter table.""" + + definition = """ + param_name: varchar(32) + --- + params: blob + """ + contents = [ # Default values as list of tuples + ["example1", {"A": 1, "B": 2}], + ["example2", {"A": 3, "B": 4}], + ] + + @classmethod + def insert_default(cls): # Not req for dj.Lookup, but Spyglass convention + """Insert default values.""" # skip_duplicates prevents errors + cls().insert(rows=cls.contents, skip_duplicates=True) + + +@schema +class MyAnalysisSelection(SpyglassMixin, dj.Manual): + """Selection table.""" # Pair subjects and params for computation + + definition = """ + -> SubjBlinded + -> MyParams + """ + + def insert_all(self, param_name="example1"): # Optional helper function + """Insert all subjects with given param name""" + self.insert( + [ + {**subj_key, "param_name": param_name} + for subj_key in SubjBlinded.fetch("KEY") + ], + skip_duplicates=True, + ) + + +@schema +class MyAnalysis(SpyglassMixin, dj.Computed): + """Analysis table.""" + + # One or more foreign keys, no manual input + definition = """ + -> MyAnalysisSelection + """ + + class MyPart(SpyglassMixin, dj.Part): + """Part table.""" + + definition = """ + -> MyAnalysis + --- + result: int + """ + + def make(self, key): + # Prepare for computation + this_subj = SubjBlinded().return_subj(key["subject_id"]) + param_key = {"param_name": key["param_name"]} + these_param = (MyParams & param_key).fetch1("params") + + # Perform computation. + # Ideally, all data is linked with foreign keys, but not enforced + for pos_obj in RawPosition.PosObject * (Subject & this_subj): + dataframe = (RawPosition.PosObject & pos_obj).fetch1_dataframe() + result = process_df(dataframe, **these_param) + + part_inserts = [] # Prepare inserts, to minimize insert calls + for _ in range(10): + result += random.randint(0, 100) + part_inserts.append(dict(key, result=result)) + + self.insert1(key) # Insert into 'master' first, then all parts + self.MyPart().insert(rows=part_inserts, skip_duplicates=True) +``` + +
+ +## Front matter + +At the beginning of the schema file, you'll find ... + +- Script docstring +- Imports + - Aliased imports + - Package imports + - Individual imports + - Relative imports +- Schema declaration + +```python +"""Schema example for custom pipelines + +Note: `noqa: F401` is a comment that tells linters to ignore the fact that +`Subject` seems unused in the file. If this table is only used in a table +definition string, the linter will not recognize it as being used. +""" + +import random # Package import +from typing import Union # Individual class import +from uuid import UUID + +import datajoint as dj # Aliased package import +from custom_package.utils import process_df, schema_prefix # custom functions +from spyglass.common import RawPosition, Subject # noqa: F401 +from spyglass.utils import SpyglassMixin # Additional Spyglass features + +schema = dj.schema(schema_prefix + "_example") # schema name from string +``` + +- The `schema` variable determines the name of the schema in the database. +- Existing schema prefixes (e.g., `common`) should not be added to without + discussion with the Spyglass team. +- Database admins may be interested in limiting privileges on a per-prefix + basis. For example, Frank Lab members use ... +- Their respective usernames for solo work +- Project-specific prefixes for shared work. + +## Table syntax + +Each table is defined as a Python class, with a `definition` attribute that +contains the SQL-like table definition. + +### Class inheritance + +The parentheses in the class definition indicate that the class inherits from. + +This table is ... + +- A `SpyglassMixin` class, which provides a number of useful methods specific to + Spyglass as discussed in the [mixin article](../Features/Mixin.md). +- A DataJoint `Manual` table, which is a table that is manually populated. + +```python +@schema # Decorator to define a table in the schema on the server +class ExampleTable(SpyglassMixin, dj.Manual): # Inherit SpyglassMixin class + pass +``` + +### Table types + +- [DataJoint types](https://datajoint.com/docs/core/datajoint-python/0.14/design/tables/tiers/): + - `Manual` tables are manually populated. + - `Lookup` tables can be populated on declaration, and rarely change. + - `Computed` tables are populated by a method runs computations on upstream + entries. + - `Imported` tables are populated by a method that imports data from another + source. + - `Part` tables are used to store data that is conceptually part of another + table. +- [Spyglass conceptual types](./TableTypes.md): + - Optional upstream Data tables from a previous pipeline. + - Parameter tables (often `dj.Lookup`) store parameters for analysis. + - Selection tables store pairings of parameters and data to be analyzed. + - Compute tables (often `dj.Computed`) store the results of analysis. + - Merge tables combine data from multiple pipeline versions. + +### Definitions + +Each table can have a docstring that describes the table, and must have a +`definition` attribute that contains the SQL-like table definition. + +- `#` comments are used to describe the table and its columns. + +- `---` separates the primary key columns from the data columns. + +- `field : datatype` defines a column using a + [SQL datatype](https://datajoint.com/docs/core/datajoint-python/0.14/design/tables/attributes/) + + +- `->` indicates a foreign key reference to another table. + +```python +@schema # Decorator to define a table in the schema on the server +class ExampleTable(SpyglassMixin, dj.Manual): # Inherit SpyglassMixin class + """Table Description""" # Table docstring, one-line if possible + + definition = """ # Table comment + primary_key1 : uuid # randomized string + primary_key2 : int # integer + --- + secondary_key1 : varchar(32) # string of max length 32 + -> Subject # Foreign key reference, inherit primary key of this table + """ +``` + +### Methods + +Many Spyglss tables have methods that provide functionality for the pipeline. + +Check out our [API documentation](../api/index.md) for a full list of available +methods. + +This example models subject blinding to demonstrate ... + +- An aliased foreign key in the definition, using `proj` to rename the field. +- A static property that returns the primary key. +- A static method that returns a dictionary of subject information. +- A class method that hashes an argument. +- An instance method that self-inserts subjects based on a restriction. +- An instance method that returns the unblinded subject information. + +```python +# Model to demonstrate field aliasing with `proj` +@schema +class SubjBlinded(SpyglassMixin, dj.Manual): + """Blinded subject table.""" # Class docstring for `help()` + + definition = """ + subject_id: uuid # id + --- + -> Subject.proj(actual_id='subject_id') + """ + + @property # Static information, Table.property + def pk(self): + """Return the primary key""" # Function docstring for `help()` + return self.heading.primary_key + + @staticmethod # Basic func with no reference to self instance + def _subj_dict(subj_uuid: UUID): # Type hint for argument + """Return the subject dict""" + return {"subject_id": subj_uuid} + + @classmethod # Class, not instance. Table.func(), not Table().func() + def hash(cls, argument: Union[str, dict] = None): # Default value + """Example class method""" + return dj.hash.key_hash(argument) + + def blind_subjects(self, restriction: Union[str, dict]): # Union is "or" + """Import all subjects selected by the restriction""" + insert_keys = [ + { + **self._subj_dict(self.hash(key)), + "actual_id": key["subject_id"], + } + for key in (Subject & restriction).fetch("KEY") + ] + self.insert(insert_keys, skip_duplicates=True) + + def return_subj(self, key: str): + """Return the entry in subject table""" + if isinstance(key, dict): # get rid of extra values + key = key["subject_id"] + key = self._subj_dict(key) + actual_ids = (self & key).fetch("actual_id") + ret = [{"subject_id": actual_id} for actual_id in actual_ids] + return ret[0] if len(ret) == 1 else ret +``` + +### Example Table Types + +#### Params Table + +This stores the set of values that may be used in an analysis. For analyses that +are unlikely to change, consider specifying all parameters in the table's +secondary keys. For analyses that may have different parameters, of when +depending on outside packages, consider a `blob` datatype that can store a +python dictionary. + +```python +@schema +class MyParams(SpyglassMixin, dj.Lookup): # Lookup allows for default values + """Parameter table.""" + + definition = """ + param_name: varchar(32) + --- + params: blob + """ + contents = [ # Default values as list of tuples + ["example1", {"A": 1, "B": 2}], + ["example2", {"A": 3, "B": 4}], + ] + + @classmethod + def insert_default(cls): # Not req for dj.Lookup, but Spyglass convention + """Insert default values.""" # skip_duplicates prevents errors + cls().insert(rows=cls.contents, skip_duplicates=True) +``` + +#### Selection Table + +This is the staging area to pair sessions with parameter sets. Depending on what +is inserted, you might pair the same subject with different parameter sets, or +different subjects with the same parameter set. + +```python +@schema +class MyAnalysisSelection(SpyglassMixin, dj.Manual): + """Selection table.""" # Pair subjects and params for computation + + definition = """ + -> SubjBlinded + -> MyParams + """ + + def insert_all(self, param_name="example1"): # Optional helper function + """Insert all subjects with given param name""" + self.insert( + [ + {**subj_key, "param_name": param_name} + for subj_key in SubjBlinded.fetch("KEY") + ], + skip_duplicates=True, + ) +``` + +#### Compute Table + +This is how processing steps are paired with data entry. By running +`MyAnalysis().populate()`, the `make` method is called for each foreign key +pairing in the selection table. The `make` method should end in one or one +inserts into the compute table. + +```python +@schema +class MyAnalysis(SpyglassMixin, dj.Computed): + """Analysis table.""" + + # One or more foreign keys, no manual input + definition = """ + -> MyAnalysisSelection + """ + + class MyPart(SpyglassMixin, dj.Part): + """Part table.""" + + definition = """ + -> MyAnalysis + --- + result: int + """ + + def make(self, key): + # Prepare for computation + this_subj = SubjBlinded().return_subj(key["subject_id"]) + param_key = {"param_name": key["param_name"]} + these_param = (MyParams & param_key).fetch1("params") + + # Perform computation. + # Ideally, all data is linked with foreign keys, but not enforced + for pos_obj in RawPosition.PosObject * (Subject & this_subj): + dataframe = (RawPosition.PosObject & pos_obj).fetch1_dataframe() + result = process_df(dataframe, **these_param) + + part_inserts = [] # Prepare inserts, to minimize insert calls + for _ in range(10): + result += random.randint(0, 100) + part_inserts.append(dict(key, result=result)) + + self.insert1(key) # Insert into 'master' first, then all parts + self.MyPart().insert(rows=part_inserts, skip_duplicates=True) +``` + +To see how tables of a given schema relate to one another, use a +[schema diagram](https://datajoint.com/docs/core/datajoint-python/0.14/design/diagrams/) diff --git a/docs/src/ForDevelopers/TableTypes.md b/docs/src/ForDevelopers/TableTypes.md new file mode 100644 index 000000000..5a040f94f --- /dev/null +++ b/docs/src/ForDevelopers/TableTypes.md @@ -0,0 +1,108 @@ +# Table Types + +Spyglass uses DataJoint's default +[table tiers](https://datajoint.com/docs/core/datajoint-python/0.14/design/tables/tiers/). + +By convention, an individual pipeline has one or more the following table types: + +- Common/Multi-pipeline table +- NWB ingestion table +- Parameters table +- Selection table +- Data table +- Merge Table (see also [stand-alone doc](../Features/Merge.md)) + +## Common/Multi-pipeline + +Tables shared across multiple pipelines for shared data types. + +- Naming convention: None +- Data tier: `dj.Manual` +- Examples: `IntervalList` (time interval for any analysis), `AnalysisNwbfile` + (analysis NWB files) + +_Note_: Because these are stand-alone tables not part of the dependency +structure, developers should include enough information to link entries back to +the pipeline where the data is used. + +## NWB ingestion + +Automatically populated when an NWB file is ingested (i.e., `dj.Imported`) to +keep track of object hashes (i.e., `object_id`) in the NWB file. All such tables +should be included in the `make` method of `Session`. + +- Naming convention: None +- Data tier: `dj.Imported` +- Primary key: foreign key from `Session` +- Non-primary key: `object_id`, the unique hash of an object in the NWB file. +- Examples: `Raw`, `Institution`, etc. +- Required methods: + - `make`: must read information from an NWB file and insert it to the table. + - `fetch_nwb`: retrieve the data specified by the object ID. + +## Parameters + +Stores the set of values that may be used in an analysis. + +- Naming convention: end with `Parameters` or `Params` +- Data tier: `dj.Manual`, or `dj.Lookup` +- Primary key: `{pipeline}_params_name`, `varchar` +- Non-primary key: `{pipeline}_params`, `blob` - dict of parameters +- Examples: `RippleParameters`, `DLCModelParams` +- Possible method: if `dj.Manual`, include `insert_default` + +_Notes_: Some early instances of Parameter tables (a) used non-primary keys for +each individual parameter, and (b) use the Manual rather than Lookup tier, +requiring a class method to insert defaults. + +## Selection + +A staging area to pair sessions with parameter sets, allowing us to be selective +in the analyses we run. It may not make sense to pair every paramset with every +session. + +- Naming convention: end with `Selection` +- Data tier: `dj.Manual` +- Primary key(s): Foreign key references to + - one or more NWB or data tables + - optionally, one or more parameter tables +- Non-primary key: None +- Examples: `MetricSelection`, `LFPSelection` + +It is possible for a Selection table to collect information from more than one +Parameter table. For example, the Selection table for spike sorting holds +information about both the interval (`SortInterval`) and the group of electrodes +(`SortGroup`) to be sorted. + +## Data + +The output of processing steps associated with a selection table. Has a `make` +method that carries out the computation specified in the Selection table when +`populate` is called. + +- Naming convention: None +- Data tier: `dj.Computed` +- Primary key: Foreign key reference to a Selection table. +- Non-primary key: `analysis_file_name` inherited from `AnalysisNwbfile` table + (i.e., name of the analysis NWB file that will hold the output of the + computation). +- Required method, `make`: carries out the computation and insert a new entry; + must also create an analysis NWB file and insert it to the `AnalysisNwbfile` + table. Note that this method is never called directly; it is called via + `populate`. Multiple entries can be run in parallel when called with + `reserve_jobs=True`. +- Example: `QualityMetrics`, `LFPV1` + +## Merge + +Following a convention outlined in [a dedicated doc](../Features/Merge.md), +merges the output of different pipelines dedicated to the same modality as part +tables (e.g., common LFP, LFP v1, imported LFP) to permit unified downstream +processing. + +- Naming convention: `{Pipeline}Output` +- Data tier: custom `_Merge` class +- Primary key: `merge_id`, `uuid` +- Non-primary key: `source`, `varchar` table name associated with that entry +- Required methods: None - see custom class methods with `merge_` prefix +- Example: `LFPOutput`, `PositionOutput` diff --git a/docs/src/ForDevelopers/UsingNWB.md b/docs/src/ForDevelopers/UsingNWB.md new file mode 100644 index 000000000..3f68f930e --- /dev/null +++ b/docs/src/ForDevelopers/UsingNWB.md @@ -0,0 +1,269 @@ +# Using NWB + +This article explains how to use the NWB format in Spyglass. It covers the +naming conventions, storage locations, and the relationships between NWB files +and other tables in the database. + +## NWB files + +NWB files contain everything about the experiment and form the starting point of +all analyses. + +- Naming: `{animal name}YYYYMMDD.nwb` +- Storage: + - On disk, directory identified by `settings.py` as `raw_dir` (e.g., + `/stelmo/nwb/raw`) + - In database, in the `Nwbfile` table +- Copies: + - made with an underscore `{animal name}YYYYMMDD_.nwb` + - stored in the same `raw_dir` + - contain pointers to objects in original file + - permit adding new parts to the NWB file without risk of corrupting the + original data + +## Analysis files + +Hold the results of intermediate steps in the analysis. + +- Naming: `{animal name}YYYYMMDD_{10-character random string}.nwb` +- Storage: + - On disk, directory identified by `settings.py` as `analysis_dir` (e.g., + `/stelmo/nwb/analysis`). Items are further sorted into folders matching + original NWB file name + - In database, in the `AnalysisNwbfile` table. +- Examples: filtered recordings, spike times of putative units after sorting, or + waveform snippets. + +_Note_: Because NWB files and analysis files exist both on disk and listed in +tables, these can become out of sync. You can 'equalize' the database table +lists and the set of files on disk by running `cleanup` method, which deletes +any files not listed in the table from disk. + +## Reading and writing recordings + +Recordings start out as an NWB file, which is opened as a +`NwbRecordingExtractor`, a class in `spikeinterface`. When using `sortingview` +for visualizing the results of spike sorting, this recording is saved again in +HDF5 format. This duplication should be resolved in the future. + +## Naming convention + +The following objects should be uniquely named. + +- _Recordings_: Underscore-separated concatenations of uniquely defining + features, + `NWBFileName_IntervalName_ElectrodeGroupName_PreprocessingParamsName`. +- _SpikeSorting_: Adds `SpikeSorter_SorterParamName` to the name of the + recording. +- _Waveforms_: Adds `_WaveformParamName` to the name of the sorting. +- _Quality metrics_: Adds `_MetricParamName` to the name of the waveform. +- _Analysis NWB files_: + `NWBFileName_IntervalName_ElectrodeGroupName_PreprocessingParamsName.nwb` +- Each recording and sorting is given truncated UUID strings as part of + concatenations. + +Following broader Python conventions, methods a method that will not be +explicitly called by the user should start with `_` + +## Time + +The `IntervalList` table stores all time intervals in the following format: +`[start_time, stop_time]`, which represents a contiguous time of valid data. +These are used to exclude any invalid timepoints, such as missing data from a +faulty connection. + +- Intervals can be nested for a set of disjoint intervals. +- Some recordings have explicit + [PTP timestamps](https://en.wikipedia.org/wiki/Precision_Time_Protocol) + associated with each sample. Some older recordings are missing PTP times, + and times must be inferred from the TTL pulses from the camera. + +## Object-Table mappings + +The following tables highlight the correspondence between NWB objects and +Spyglass tables/fields and should be a useful reference for developers looking +to adapt existing NWB files for Spyglass injestion. + +Please contact the developers if you have any questions or need help with +adapting your NWB files for use with Spyglass, especially items marked with +'TODO' in the tables below. + + NWBfile Location: nwbf
Object type: pynwb.file.NWBFile + +| Spyglass Table | Key | NWBfile Location | Config option | Notes | +| :------------------- | :-----------------------: | -----------------------------: | --------------------------------------------: | ---------------------------: | +| Institution | institution_name | nwbf.institution | config\["Institution"\]\["institution_name"\] | str | +| Session | institution_name | nwbf.institution | config\["Institution"\]\["institution_name"\] | str | +| Lab | lab_name | nwbf.lab | config\["Lab"\]\["lab_name"\] | str | +| Session | lab_name | nwbf.lab | config\["Lab"\]\["lab_name"\] | str | +| LabMember | lab_member_name | nwbf.experimenter | config\["LabMember"\]\["lab_member_name"\] | str("last_name, first_name") | +| Session.Experimenter | lab_member_name | nwbf.experimenter | config\["LabMember"\]\["lab_member_name"\] | str("last_name, first_name") | +| Session | session_id | nwbf.session_id | XXX | | +| Session | session_description | nwbf.session_description | XXX | | +| Session | session_start_time | nwbf.session_start_time | XXX | | +| Session | timestamps_reference_time | nwbf.timestamps_reference_time | XXX | | +| Session | experiment_description | nwbf.experiment_description | XXX | | + + NWBfile Location: nwbf.subject
Object type: pynwb.file.Subject + +| Spyglass Table | Key | NWBfile Location | Config option | Notes | +| :------------- | :---------: | -----------------------: | -----------------------------------: | -----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | +| Subject | subject_id | nwbf.subject.subject_id | config\["Subject"\]\["subject_id"\] | | +| Subject | age | nwbf.subject.age | config\["Subject"\]\["age"\] | Dandi requires age must be in ISO 8601 format, e.g. "P70D" for 70 days, or, if it is a range, must be "\[lower\]/\[upper\]", e.g. "P10W/P12W", which means "between 10 and 12 weeks" | +| Subject | description | nwbf.subject.description | config\["Subject"\]\["description"\] | | +| Subject | genotype | nwbf.subject.genotype | config\["Subject"\]\["genotype"\] | | +| Subject | species | nwbf.subject.species | config\["Subject"\]\["species"\] | Dandi upload requires species either be in Latin binomial form (e.g., 'Mus musculus' and 'Homo sapiens') or be a NCBI taxonomy link | +| Subject | sex | nwbf.subject.sex | config\["Subject"\]\["sex"\] | single character identifier (e.g. "F", "M", "U","O") | +| Session | subject_id | nwbf.subject.subject_id | config\["Subject"\]\["subject_id"\] | str("animal_name") | + + NWBfile Location: nwbf.devices
Object type: +ndx_franklab_novela.DataAcqDevice + +| Spyglass Table | Key | NWBfile Location | Config option | Notes | +| :----------------------------- | :-------------------------------: | -------------------------------------: | -----------------------------------------------------------------------: | ----: | +| DataAcquisitionDevice | data_acquisition_device_name | nwbf.devices.\<\*DataAcqDevice>.name | config\["DataAcquisitionDevice"\]\["data_acquisition_device_name"\] | | +| DataAcquisitionDevice | adc_circuit | nwbf.devices.\<\*DataAcqDevice>.name | config\["DataAcquisitionDevice"\]\["data_acquisition_device_name"\] | | +| DataAcquisitionDeviceSystem | data_acquisition_device_system | nwbf.devices.\<\*DataAcqDevice>.system | config\["DataAcquisitionDevice"\]\["data_acquisition_device_system"\] | | +| DataAcquisitionDeviceAmplifier | data_acquisition_device_amplifier | nwbf.devices.\<\*DataAcqDevice>.system | config\["DataAcquisitionDevice"\]\["data_acquisition_device_amplifier"\] | | + + NWBfile Location: nwbf.devices
Object type: +ndx_franklab_novela.CameraDevice + +| Spyglass Table | Key | NWBfile Location | Config option | Notes | +| :------------- | :-----------------: | ----------------------------------------------: | ------------------------------------------------------: | ----: | +| CameraDevice | camera_id | nwbf.devices.\<\*CameraDevice>.camera_id | config\["CameraDevice"\]\[index\]\["camera_id"\] | int | +| CameraDevice | camera_name | nwbf.devices.\<\*CameraDevice>.camera_name | config\["CameraDevice"\]\[index\]\["camera_name"\] | str | +| CameraDevice | camera_manufacturer | nwbf.devices.\<\*CameraDevice>.manufacturer | config\["CameraDevice"\]\[index\]\["manufacturer"\] | str | +| CameraDevice | model | nwbf.devices.\<\*CameraDevice>.model | config\["CameraDevice"\]\[index\]\["model"\] | str | +| CameraDevice | lens | nwbf.devices.\<\*CameraDevice>.lens | config\["CameraDevice"\]\[index\]\["lens"\] | str | +| CameraDevice | meters_per_pixel | nwbf.devices.\<\*CameraDevice>.meters_per_pixel | config\["CameraDevice"\]\[index\]\["meters_per_pixel"\] | str | + + NWBfile Location: nwbf.devices
Object type: ndx_franklab_novela.Probe + + +| Spyglass Table | Key | NWBfile Location | Config option | Notes | +| :------------- | :---------------: | ----------------------------------------: | -----------------------------------------: | ----: | +| Probe | probe_type | nwbf.devices.\<\*Probe>.probe_type | config\["Probe"\]\[index\]\["probe_type"\] | str | +| Probe | probe_id | nwbf.devices.\<\*Probe>.probe_type | XXX | str | +| Probe | manufacturer | nwbf.devices.\<\*Probe>.manufacturer | XXX | str | +| Probe | probe_description | nwbf.devices.\<\*Probe>.probe_description | XXX | str | +| Probe | num_shanks | nwbf.devices.\<\*Probe>.num_shanks | XXX | int | + + NWBfile Location: nwbf.devices.\<\*Probe>.\<\*Shank>
Object type: +ndx_franklab_novela.Shank + +| Spyglass Table | Key | NWBfile Location | Config option | Notes | +| :------------- | :---------: | ---------------------------------------------: | ------------: | ----: | +| Probe.Shank | probe_shank | nwbf.devices.\<\*Probe>.\<\*Shank>.probe_shank | XXX | int | + + NWBfile Location: nwbf.devices.\<\*Probe>.\<\*Shank>.\<\*Electrode>
+Object type: ndx_franklab_novela.Electrode + +| Spyglass Table | Key | NWBfile Location | Config option | Notes | +| :-------------- | :----------: | -------------------------------------------------------------: | ------------: | ----: | +| Probe.Electrode | probe_shank | nwbf.devices.\<\*Probe>.\<\*Shank>.probe_shank | XXX | int | +| Probe.Electrode | contact_size | nwbf.devices.\<\*Probe>.\<\*Shank>.\<\*Electrode>.contact_size | XXX | float | +| Probe.Electrode | rel_x | nwbf.devices.\<\*Probe>.\<\*Shank>.\<\*Electrode>.rel_x | XXX | float | + + NWBfile Location: nwbf.epochs
Object type: pynwb.epoch.TimeIntervals + + +| Spyglass Table | Key | NWBfile Location | Config option | Notes | +| :-------------------- | :----------------: | ------------------------------------------------------------------: | ------------: | ----: | +| IntervalList (epochs) | interval_list_name | nwbf.epochs.\[index\].tags\[0\] | | str | +| IntervalList (epochs) | valid_times | \[nwbf.epoch.\[index\].start_time, nwbf.epoch.\[index\].stop_time\] | | float | + + NWBfile Location: nwbf.electrode_groups + +| Spyglass Table | Key | NWBfile Location | Config option | Notes | +| :------------- | :---------------: | ------------------------------------------------: | ------------: | ----------------------------------------------------------------------------------------------------------------------------------: | +| BrainRegion | region_name | nwbf.electrode_groups.\[index\].location | | str | +| ElectrodeGroup | description | nwbf.electrode_groups.\[index\].description | | str | +| ElectrodeGroup | probe_id | nwbf.electrode_groups.\[index\].device.probe_type | | + device must be of type ndx_franklab_novela.Probe | +| ElectrodeGroup | target_hemisphere | nwbf.electrode_groups.\[index\].targeted_x | | + electrode group must be of type ndx_franklab_novela.NwbElectrodeGroup. target_hemisphere = "Right" if targeted_x >= 0 else "Left" | + + NWBfile Location: nwbf.acquisition
Object type: +pynwb.ecephys.ElectricalSeries + +| Spyglass Table | Key | NWBfile Location | Config option | Notes | +| :----------------- | :----------------: | ---------------------------------------------------: | ------------: | ----: | +| Raw | sampling_rate | eseries.rate else, estimated from eseries.timestamps | | float | +| IntervalList (raw) | interval_list_name | "raw data valid times" | | str | +| IntervalList (raw) | valid_times | get_valid_intervals(eseries.timestamps, ...) | | | + + NWBfile Location: nwbf.processing.sample_count
Object type: +pynwb.base.TimeSeries + +| Spyglass Table | Key | NWBfile Location | Config option | Notes | +| :------------- | :-----------------: | ---------------------------: | ------------: | ----: | +| SampleCount | sample_count_obj_id | nwbf.processing.sample_count | | | + + NWBfile Location: nwbf.processing.behavior.behavioralEvents
Object +type: pynwb.base.TimeSeries + +| Spyglass Table | Key | NWBfile Location | Config option | Notes | +| :------------- | :------------: | --------------------------------------------------: | ------------: | ----: | +| DIOEvents | dio_event_name | nwbf.processing.behavior.behavioralEvents.name | | | +| DIOEvents | dio_obj_id | nwbf.processing.behavior.behavioralEvents.object_id | | | + + NWBfile Location: nwbf.processing.tasks
Object type: +hdmf.common.table.DynamicTable + +| Spyglass Table | Key | NWBfile Location | Config option | Notes | +| :------------- | :--------------: | -----------------------------------------------: | ------------: | ----: | +| Task | task_name | nwbf.processing.tasks.\[index\].name | | | +| Task | task_description | nwbf.processing.\[index\].tasks.description | | | +| TaskEpoch | task_name | nwbf.processing.\[index\].tasks.name | | | +| TaskEpoch | camera_names | nwbf.processing.\[index\].tasks.camera_id | | | +| TaskEpoch | task_environment | nwbf.processing.\[index\].tasks.task_environment | | | + + NWBfile Location: nwbf.units
Object type: pynwb.misc.Units + +| Spyglass Table | Key | NWBfile Location | Config option | Notes | +| :------------------- | :-------: | -------------------: | ------------: | ----: | +| ImportedSpikeSorting | object_id | nwbf.units.object_id | | | + + NWBfile Location: nwbf.electrodes
Object type: +hdmf.common.table.DynamicTable + +| Spyglass Table | Key | NWBfile Location | Config option | Notes | +| :------------- | :--------------------------: | -------------------------------------------------------------------------------------: | ---------------------------------------------------------------: | ---------------------------------------------------------------------------: | +| Electrode | electrode_id | nwbf.electrodes.\[index\] (the enumerated index number) | config\["Electrode"\]\[index\]\["electrode_id"\] | int | +| Electrode | name | str(nwbf.electrodes.\[index\]) nwbf.electrodes.\[index\] (the enumerated index number) | config\["Electrode"\]\[index\]\["name"\] | str | +| Electrode | group_name | nwbf.electrodes.\[index\].group_name | config\["Electrode"\]\[index\]\["group_name"\] | int | +| Electrode | x | nwbf.electrodes.\[index\].x | config\["Electrode"\]\[index\]\["x"\] | int | +| Electrode | y | nwbf.electrodes.\[index\].y | config\["Electrode"\]\[index\]\["y"\] | int | +| Electrode | z | nwbf.electrodes.\[index\].z | config\["Electrode"\]\[index\]\["z"\] | int | +| Electrode | filtering | nwbf.electrodes.\[index\].filtering | config\["Electrode"\]\[index\]\["filtering"\] | int | +| Electrode | impedance | nwbf.electrodes.\[index\].impedance | config\["Electrode"\]\[index\]\["impedance"\] | int | +| Electrode | probe_id | nwbf.electrodes.\[index\].group.device.probe_type | config\["Electrode"\]\[index\]\["probe_id"\] | if type(nwbf.electrodes.\[index\].group.device) is ndx_franklab_novela.Probe | +| Electrode | probe_shank | nwbf.electrodes.\[index\].group.device.probe_shank | config\["Electrode"\]\[index\]\["probe_shank"\] | if type(nwbf.electrodes.\[index\].group.device) is ndx_franklab_novela.Probe | +| Electrode | probe_electrode | nwbf.electrodes.\[index\].group.device.probe_electrode | config\["Electrode"\]\[index\]\["probe_electrode"\] | if type(nwbf.electrodes.\[index\].group.device) is ndx_franklab_novela.Probe | +| Electrode | bad_channel | nwbf.electrodes.\[index\].group.device.bad_channel | config\["Electrode"\]\[index\]\["bad_channel"\] | if type(nwbf.electrodes.\[index\].group.device) is ndx_franklab_novela.Probe | +| Electrode | original_reference_electrode | nwbf.electrodes.\[index\].group.device.ref_elect_id | config\["Electrode"\]\[index\]\["original_reference_electrode"\] | if type(nwbf.electrodes.\[index\].group.device) is ndx_franklab_novela.Probe | + + NWBfile Location: nwbf.processing.behavior.position
Object type: +(pynwb.behavior.Position).(pynwb.behavior.SpatialSeries) + +| Spyglass Table | Key | NWBfile Location | Config option | Notes | +| :--------------------------- | :--------------------: | -------------------------------------------------------------------------------: | ------------: | --------------------: | +| IntervalList (position) | interval_list_name | "pos {index} valid times" | | | +| IntervalList (position) | valid_times | get_valid_intervals(nwbf.processing.behavior.position.\[index\].timestamps, ...) | | | +| PositionSource | source | "trodes" | | TODO: infer from file | +| PositionSource | interval_list_name | See: IntervalList (position) | | | +| PositionSource.SpatialSeries | id | int(nwbf.processing.behavior.position.\[index\]) (the enumerated index number) | | | +| RawPosition.PosObject | raw_position_object_id | nwbf.processing.behavior.position.\[index\].object_id | | | + + NWBfile Location: nwbf.processing.video_files.video
Object type: +pynwb.image.ImageSeries + +| Spyglass Table | Key | NWBfile Location | Config option | Notes | +| :------------- | :---------: | ------------------------------------------------------: | ------------: | ----: | +| VideoFile | camera_name | nwbf.processing.video_files.video.\[index\].camera_name | | | + + NWBfile Location: nwbf.processing.associated_files
Object type: +ndx_franklab_novela.AssociatedFiles + +| Spyglass Table | Key | NWBfile Location | Config option | Notes | +| :-------------- | :---: | -----------------------------------------------------: | ------------: | --------------------------------------------------------------------------------------: | +| StateScriptFile | epoch | nwbf.processing.associated_files.\[index\].task_epochs | | type(nwbf.processing.associated_files.\[index\]) == ndx_franklab_novela.AssociatedFiles | diff --git a/docs/src/ForDevelopers/index.md b/docs/src/ForDevelopers/index.md new file mode 100644 index 000000000..6128aa10e --- /dev/null +++ b/docs/src/ForDevelopers/index.md @@ -0,0 +1,37 @@ +# For Developers + +This folder covers the process of developing new pipelines and features to be +used with Spyglass. + +## Contributing + +If you're looking to contribute to the project itself, either by adding a new +pipeline or improving an existing one, please review the article on +[contributing](./Contribute.md). + +Any computation that might be useful for more that one project is a good +candidate for contribution. If you're note sure, feel free to +[open an issue](https://github.com/LorenFrankLab/spyglass/issues/new) to +discuss. + +## Management + +If you're looking to declare and manage your own instance of Spyglass, please +review the article on [database management](./Management.md). + +## Custom + +This folder also contains a number of articles on understanding pipelines in +order to develop your own. + +- [Code for Reuse](./Reuse.md) discusses good practice for writing readable and + reusable code in Python. +- [Table Types](./TableTypes.md) explains the different table motifs in Spyglass + and how to use them. +- [Schema design](./Schema.md) explains the anatomy of a Spyglass schema and + gives a model for writing your version of each of the types of tables. +- [Using NWB](./UsingNWB.md) explains how to use the NWB format in Spyglass. + +If you'd like help in developing a new pipeline, please reach out to the +Spyglass team via our +[discussion board](https://github.com/LorenFrankLab/spyglass/discussions). diff --git a/docs/src/api/index.md b/docs/src/api/index.md index d616c0757..1d23e35b4 100644 --- a/docs/src/api/index.md +++ b/docs/src/api/index.md @@ -4,24 +4,19 @@ The files in this directory are automatically generated from the docstrings in the source code. They include descriptions of each of the DataJoint tables and other classes/methods within Spyglass. -These docs are updated any time a new release is made or a tag is pushed to the -repository. +## Directories - +- `cli`: See README.md at `spyglass/examples/cli/README.md` +- `common`: Data insertion point for all pipelines. +- `data_import`: Data insertion tools. +- `decoding`: Decoding animal position from spiking data. +- `figurl_views`: Tools for visualizing data. +- `lfp`: Local field potential processing. +- `lock`: Tables for locking files, preventing deletion. +- `position`: Tracking animal posisiton via LEDs ('Trodes') or DeepLabCut. +- `linearization`: Linearizing position data for decoding. +- `ripple`: Detecting ripples in LFP data. +- `sharing`: Tables for data sharing via Kachery. +- `spikesorting`: Sorting spikes from raw electrophysiology data. +- `utils`: Utilities for working with DataJoint and Neurodata Without Borders + (NWB) data. diff --git a/docs/src/contribute.md b/docs/src/contribute.md deleted file mode 100644 index d34698a39..000000000 --- a/docs/src/contribute.md +++ /dev/null @@ -1,244 +0,0 @@ -# Developer notes - -Notes on how the repo / database is organized, intended for a new developer. - -## Development workflow - -New contributors should follow the -[Fork-and-Branch workflow](https://www.atlassian.com/git/tutorials/comparing-workflows/forking-workflow). -See GitHub instructions -[here](https://docs.github.com/en/get-started/quickstart/contributing-to-projects). - -Regular contributors may choose to follow the -[Feature Branch Workflow](https://www.atlassian.com/git/tutorials/comparing-workflows/feature-branch-workflow) -for features that will involve multiple contributors. - -## Code organization - -- Tables are grouped into schemas by topic (e.g., `common_metrics`) -- Schemas - - Are defined in a `py` pile. - - Correspond to MySQL 'databases'. - - Are organized into modules (e.g., `common`) by folders. -- The _common_ module - - In principle, contains schema that are shared across all projects. - - In practice, contains shared tables (e.g., Session) and the first draft of - schemas that have since been split into their own - modality-specific\ - modules (e.g., `lfp`) - - Should not be added to without discussion. -- A pipeline - - Refers to a set of tables used for processing data of a particular modality - (e.g., LFP, spike sorting, position tracking). - - May span multiple schema. -- For analysis that will be only useful to you, create your own schema. - -## Types of tables - -Spyglass uses DataJoint's default -[table tiers](https://datajoint.com/docs/core/datajoint-python/0.14/design/tables/tiers/). - -By convention, an individual pipeline has one or more the following table types: - -- Common/Multi-pipeline table -- NWB ingestion table -- Parameters table -- Selection table -- Data table -- Merge Table (see also [doc](./misc/merge_tables.md)) - -### Common/Multi-pipeline - -Tables shared across multiple pipelines for shared data types. - -- Naming convention: None -- Data tier: `dj.Manual` -- Examples: `IntervalList` (time interval for any analysis), `AnalysisNwbfile` - (analysis NWB files) - -_Note_: Because these are stand-alone tables not part of the dependency -structure, developers should include enough information to link entries back to -the pipeline where the data is used. - -### NWB ingestion - -Automatically populated when an NWB file is ingested (i.e., `dj.Imported`) to -keep track of object hashes (i.e., `object_id`) in the NWB file. All such tables -should be included in the `make` method of `Session`. - -- Naming convention: None -- Data tier: `dj.Imported` -- Primary key: foreign key from `Session` -- Non-primary key: `object_id`, the unique hash of an object in the NWB file. -- Examples: `Raw`, `Institution`, etc. -- Required methods: - - `make`: must read information from an NWB file and insert it to the table. - - `fetch_nwb`: retrieve the data specified by the object ID. - -### Parameters - -Stores the set of values that may be used in an analysis. - -- Naming convention: end with `Parameters` or `Params` -- Data tier: `dj.Manual`, or `dj.Lookup` -- Primary key: `{pipeline}_params_name`, `varchar` -- Non-primary key: `{pipeline}_params`, `blob` - dict of parameters -- Examples: `RippleParameters`, `DLCModelParams` -- Possible method: if `dj.Manual`, include `insert_default` - -_Notes_: Some early instances of Parameter tables (a) used non-primary keys for -each individual parameter, and (b) use the Manual rather than Lookup tier, -requiring a class method to insert defaults. - -### Selection - -A staging area to pair sessions with parameter sets, allowing us to be selective -in the analyses we run. It may not make sense to pair every paramset with every -session. - -- Naming convention: end with `Selection` -- Data tier: `dj.Manual` -- Primary key(s): Foreign key references to - - one or more NWB or data tables - - optionally, one or more parameter tables -- Non-primary key: None -- Examples: `MetricSelection`, `LFPSelection` - -It is possible for a Selection table to collect information from more than one -Parameter table. For example, the Selection table for spike sorting holds -information about both the interval (`SortInterval`) and the group of electrodes -(`SortGroup`) to be sorted. - -### Data - -The output of processing steps associated with a selection table. Has a `make` -method that carries out the computation specified in the Selection table when -`populate` is called. - -- Naming convention: None -- Data tier: `dj.Computed` -- Primary key: Foreign key reference to a Selection table. -- Non-primary key: `analysis_file_name` inherited from `AnalysisNwbfile` table - (i.e., name of the analysis NWB file that will hold the output of the - computation). -- Required methods: - - `make`: carries out the computation and insert a new entry; must also create - an analysis NWB file and insert it to the `AnalysisNwbfile` table. Note - that this method is never called directly; it is called via `populate`. - Multiple entries can be run in parallel when called with - `reserve_jobs=True`. - - `delete`: extension of the `delete` method that checks user privilege before - deleting entries as a way to prevent accidental deletion of computations - that take a long time (see below). -- Example: `QualityMetrics`, `LFPV1` - -### Merge - -Following a convention outlined in [the dedicated doc](./misc/merge_tables.md), -merges the output of different pipelines dedicated to the same modality as part -tables (e.g., common LFP, LFP v1, imported LFP) to permit unified downstream -processing. - -- Naming convention: `{Pipeline}Output` -- Data tier: custom `_Merge` class -- Primary key: `merge_id`, `uuid` -- Non-primary key: `source`, `varchar` table name associated with that entry -- Required methods: None - see custom class methods with `merge_` prefix -- Example: `LFPOutput`, `PositionOutput` - -## Integration with NWB - -### NWB files - -NWB files contain everything about the experiment and form the starting point of -all analyses. - -- Naming: `{animal name}YYYYMMDD.nwb` -- Storage: - - On disk, directory identified by `settings.py` as `raw_dir` (e.g., - `/stelmo/nwb/raw`) - - In database, in the `Nwbfile` table -- Copies: - - made with an underscore `{animal name}YYYYMMDD_.nwb` - - stored in the same `raw_dir` - - contain pointers to objects in original file - - permit adding new parts to the NWB file without risk of corrupting the - original data - -### Analysis files - -Hold the results of intermediate steps in the analysis. - -- Naming: `{animal name}YYYYMMDD_{10-character random string}.nwb` -- Storage: - - On disk, directory identified by `settings.py` as `analysis_dir` (e.g., - `/stelmo/nwb/analysis`). Items are further sorted into folders matching - original NWB file name - - In database, in the `AnalysisNwbfile` table. -- Examples: filtered recordings, spike times of putative units after sorting, or - waveform snippets. - -_Note_: Because NWB files and analysis files exist both on disk and listed in -tables, these can become out of sync. You can 'equalize' the database table -lists and the set of files on disk by running `cleanup` method, which deletes -any files not listed in the table from disk. - -## Reading and writing recordings - -Recordings start out as an NWB file, which is opened as a -`NwbRecordingExtractor`, a class in `spikeinterface`. When using `sortingview` -for visualizing the results of spike sorting, this recording is saved again in -HDF5 format. This duplication should be resolved in the future. - -## Naming convention - -The following objects should be uniquely named. - -- _Recordings_: Underscore-separated concatenations of uniquely defining - features, - `NWBFileName_IntervalName_ElectrodeGroupName_PreprocessingParamsName`. -- _SpikeSorting_: Adds `SpikeSorter_SorterParamName` to the name of the - recording. -- _Waveforms_: Adds `_WaveformParamName` to the name of the sorting. -- _Quality metrics_: Adds `_MetricParamName` to the name of the waveform. -- _Analysis NWB files_: - `NWBFileName_IntervalName_ElectrodeGroupName_PreprocessingParamsName.nwb` -- Each recording and sorting is given truncated UUID strings as part of - concatenations. - -Following broader Python conventions, methods a method that will not be -explicitly called by the user should start with `_` - -## Time - -The `IntervalList` table stores all time intervals in the following format: -`[start_time, stop_time]`, which represents a contiguous time of valid data. -These are used to exclude any invalid timepoints, such as missing data from a -faulty connection. - -- Intervals can be nested for a set of disjoint intervals. -- Some recordings have explicit - [PTP timestamps](https://en.wikipedia.org/wiki/Precision_Time_Protocol) - associated with each sample. Some older recordings are missing PTP times, - and times must be inferred from the TTL pulses from the camera. - -## Misc - -- During development, we suggest using a Docker container. See - [example](./notebooks/00_Setup.ipynb). -- `numpy` style docstrings will be interpreted by API docs. To check for - compliance, monitor the std out when building docs (see `docs/README.md`) - -## Making a release - -Spyglass follows [Semantic Versioning](https://semver.org/) with versioning of -the form `X.Y.Z` (e.g., `0.4.2`). - -1. In `CITATION.cff`, update the `version` key. -2. Make a pull request with changes. -3. After the pull request is merged, pull this merge commit and tag it with - `git tag {version}` -4. Publish the new release tag. Run `git push origin {version}`. This will - rebuild docs and push updates to PyPI. -5. Make a new - [release on GitHub](https://docs.github.com/en/repositories/releasing-projects-on-github/managing-releases-in-a-repository). diff --git a/docs/src/images/merge_diagram.png b/docs/src/images/merge_diagram.png index 72b0fc8c08afea29f838129425922bbda2ada54d..0829cc0464cafba692e07013cff1cb622a628987 100644 GIT binary patch literal 63402 zcmce81zVI+w>Bm!2#5+uiXtd2EvYop-6=J6H;h4wfPgUch)7Gz07FTaHP{Go;nh74BU;-WzFBbhL(1CJ!mpC`J4z)OcN4ejN8b@)OcPw!n~uT~S-B~bgQ9lJ$3o?x7_d@VgJ_qPrLm&H^pG_IhdLU0H@I~96dJ2pC{-RMt z>BzF}46MRVj5#nIb-RuHJKp{m<5tkSu%@9EOUKSuM$*uoDWPe39*UOrqql$P1Mz@& zex&p8a0L-gdoK!UuxJKgcj3!$=mmV0fMxq%m*<8CE#8=*<=+-Pau6mWzS)XAf5pobHH{OP_C6K*(kw#}9_9QCH;O!8 zS$U8zMHWvkYfie7q@@2VV%VC6eX^YXb$|a0to;rgwg7xU3{`}|@jR8th*pZs1#1xx=SpOMAg6Fr~m-RtWQ=_?;n6LtJj z(*T2?ab>foJ<;yWJ@>svy(RY7YWiGv%70^WB`GqpXJz9#?x+aIrZ~%p?e4`;jsLJ8 z9bH*rwIn+QVYGl>gn94vB+~RzWvWB4V93cy6W#LeIc*VB*V@XLmbR10`ZGhK1KOGF z>2uV@;V~{rySMgji9z$K(*Sz7%Ze65NasHy<19285g;E#qWu^E&|UL)C^Fmm$aEnJ z_@m|C`u$~d0^KRs3lTRyPtu*6PBSW~J4vn8MD+xB?I#Cx??x<7`=#Md;&8IU_m)Td zUeZ>tzf|a27;Jx0w|Af|Amkj6TZdzf2|bCf?erW7`Rx|fwKq3jKc=mO`5T@Wmv~#v z(WLp$>-I##7movWkG*DQuHmJ?U-gpLA4s(>w+>;dhcLa0Y4fM*y{;Z}thg=Q@wV3p z{ZO4*W_$a_ttzv~Jl%o{S5c8(=vgOpZgm8n^UofIaq;6&u4f|(2Ab@k+JwK z5o)gha+C7!qSLOj=z%f?oNl?Bz;O>Bio^F?%q`-a5c!*4Cqca-(hj}>8_l_%4&R^_ z6jYa2zu4u+C3tc5h@JL{&B8^o$`B&{ckj5XdAjKQ=cBy#7MV`>w=Z7dmc>rRc9FHh z{(BHL_MP%3%AQq3rrj6cgOi zxK}h_oeoFlNi-dXUd>Wr-vM_#!PKAoU;1oxq&6z5etrJe1t(F+r^J>=M2B>9^%wiN z#^?`E1!$C(V9Tc*&~sz2qlGfRI_@v@u zvJBeh>L@A$qGOTYcnN&9E;{#9LzYHoUW)Ghk?{gS>t#4~cF;xOSof!@!*#-U$Ea?n zpQG=Abz`HrL`Hok-1uyiV|QsKSs1spp8^-{rIGODtQRJJ?OK`mRZIik1G!tVM?i2iJNHpZ1UB zL)vx6)#X@XXw0&bh>Y+frr6%WY?nY1bQHi5yK8)W*X*xd;&TMt8`tXuf;PKNX}@Pd z=*!>8;;(u$x!P%SnJA9ku{O~z95xmw^v02-iOUPqxNgz_Ol+Mb%>^^?H@VyRfwQ)t zw#H@Y;Z&Mma&_>Q8MHT`Z+07g?)!Q12{CQJSB%Ix00;m5++h;ii-t5jGjqZ|R#K0>vWvX(;~v z_~;5!Nm-t_maW!rXNow6%~Ds#Xl8KbMcwh*vDe|I_GzcOae8%TGTeK^>bZ6&D=WwQ zo62ckF|gjcnpM}ojFt_-0KX!-L$=OIn0nz0Rs3o5UDFsYC~jgJH9Xkb{~=?-cgE;) zDU?7x4lOOE626X8WLYEO+ocWhje)Snp0h=X9 z?in}ejYEq${$dL;HOG<1d+@q5_z9L}dO0^%c{g_ZOpdVY59iRHUVU3s@BF*6{jIt} zf4tw4dd0=o;fP&FY)Ad<>^tK9CY0F1#)+;axm999+DvtkA1u9)9(&q`dnM-FtNi|^ zeKa5Tuoq){iS{)o!{Id|yL^D17%dr3w!*f#246Lcou7!!Upo#kL9ITds;+~5LCs2z z@K4W-&pOSx3A|*x-=tS>r!(T_rU?5Z9!m2MsoG1~uyaDthWi|14NTMfa@eR|EZurt zy~miHeq{vlB#yogH-0Q<8DoCuky(1e+mykXY`SxZ=b+;g28=#)&%9dsHh7bI{%#v z;NqR0L7|#gds#@_!7b=XgqS&aACVjF=*~s?Vx_gpbMoifUCEJp-#VY|EeadK$cxbE zl?s_#&f%;4IG~KF7W=3H<;q$OT&(B>qHmrY*6*a$TESu*EpTss5_g`4$Gc9nGCpz#y@Sx(9QCT9VcIG zaa^z)78ok@D096Od(-l3r!g_^oFo67_&4!R^!XsB9w?lF$>Hw7#s~AR^RP)(M0F+= ze5B35C2}lN_dp&=cQ+#J^fu}0b$~8Foxr;9-4owX_$PlTDYr$>9LDv=D>?l?h+&4| zT&W4t5=|vbQNgW2CpYR&=lB~w3WaPlP=1;F(;y*3+qmkn*Qn3oWV0}cjIE07iQ>3O zfa3UZ{|L2xVU8ioVlQ(W8a92tx4jE?H~i^30RfW|2zW0MBe#%9pjSBWV$aT@-P``B z$0s3lI0;M%8!NSdb$I!#yRY*KkwH)Q@u!kviZ$vvm2LW^_@zs7&?sOQ^ud&~lgFP9 zN7V(Xz7*i|`VyH%l=$|Z8SJuRplolPGnsx9aKsc4FLnxtHsUU3aSIcJuV`!MRfDTT zc6~1v28S>bExuH)j=Xy-dt{ z%s_qStAQ`*RYvXh=$`Dc=~}okc93Jl*2wQ-lFo`pwDZ20&qwI-2rhETiQ55|Z17S> z5N26vn_zr8Sky;rudVwW5n=i$AwDl#RX=sV;apUfIeCr)ej$&AGhykx~g|1)I9;>%PO$&u#5gQFpn)$NE5NtBpBqO6 z@w(%Xww6OjZbvjKE% zf;swiAaI@cw8-|m@BXjL?^g%l$ftdxwWqz~yts_&aexzRa9hXRed$|85Xv(7GQas2 zN0NgZr-QztQloT*^-3tNSKI-?%Tr#F(PQ`SEMA>`o$&5jo7eQ@c1C?IbaKD-zH=S6 zapd;dN58#UWzyS{S}di7emE))ziag)z1dUyFOnJ+*N?tb-P6qb%dN~mNxj5r#E`2Q zs{aOvKZifvH)$v-$IRkKdh3h#wp+Ts zI9rhrhm)~C4M9?;TP6OW zWR|?vaa-+H2~0Ni|Ga4b|J4^LK|xrc&0F-_*(4PqAe$#1-@SX;F>#)$%xz=-c}Kv2 zNcf6!hVgKrsw_F)j?BdfrfjK`jvn+uK-t4DE5q~}@BV#)^Nhx<~Dy~<`ujGLIv=J9tcXXU+&=+Tqij4s08Bo*H*(TK?v>02czjog^W zAHuwwBzA)#oINvtr!syCZzQw_D}3ypLqCLZ4zJWw>NcPv+axTLTAph^#e&Ng1>;SN6X%+a7IQ3 z18LZqJ;Hu|*K1{aJ3AxeTFCs8i{ud(iYKgXUO`DojUCp{!%J#`=1W%zZD@Ftl$=a8 zsX%|k094Ybc&^6h=;(V~iGPu?rjJQXtgS=isy=jQwQY_?Ewa3Gd&(uaQ6GenrOCmPJBB;>52`n%{LK zPb*(RQSqgaQ1$FFU3d!xHTAdm%AssefKw|}%k?bG%s!5d>9X3o>lPHQyScmfPftg6 zJ!5(5z~5D>qoZTu=jS&mt8Q;&Yhv<#czD?TN*Kx9aO`)9Q1nk7RxUxo&bc`=3_?%o zjj=J^T1L4Hz|Md3HHl1Q<-x;l9ra{ zOPTN!r30&V(VTqUVip}1cAJchY&b#`k|YhbaqZf*6HMtZtx)topef1Bdhf`nac`fs zwzdKT=8SyIdB${fbToiMJ(bEqRK3k?Y;L~3>*Z&dhM%LF2ZHEhgGom6i{?nC%S5A3 z?&jj+5@(Lfo7SRAqobz&SzBvJD(x9MelWVQz>uj1nJgVQ;nd-Pvx8LTSGojX6|O{c zbMrt0_6k#IRC+pnQj$3ek!tPjomf)xk}FXGgBVv^2FpWMS6eK+4W(iL4;tw}*e@+D zfvhq1lp@S)T-@0yCT!Vu3KIWNQc|MWm0eL$fda8GvTlEAMCZ$02_|H93=Td}S68P9 zRwWkTQOu~VO$8em>v{*{B+N*o_Jf=rSo3uD^?m)OOg<-g*?i<>hIM_3isI(yuh?1} znRV(zp(Gp~i!{n6EbP#QFlA$X&lZIJMSvIvgNdnl+UyP*rlGc9dzT+QdNfx2YhZqt zlauqcl~s0%mq!VRD^QRH2b#jVq&z+OMMOkOLn28=UIQZ(nt|1+IAo2AXF<4xgt`_M z7J~E2{*&CgSeIt@pnK}pmc!ZEnK{Q!Rnd zC7!*lpYHAJBPl2-h;aBZ8qC^d77!31QBtW_x;r!Tn2n9C6e)KLTygb%ON*4WjLgW8 z8Q!gsrb~DUlcb!KEa^5Z@bl5uh7CI3^G&->W2Rij8V(SFk&%(_mozYb|@*i}}A>;7bVfksC%tB$GsWkI3yw4^xKyG%pze)$~z{9bZ>_dwX!quqZt0!-p$l@)8qVV|MYt;qfrof0MAfIvGT{ zZUEcSgWH15X)z$Lf#G3Y1q7@g8y=q@Z9b)p;6sL)!Ct_+6ciLVK}2Kb=Y3<@9 z6PUO<&6MI2YehsNOexmiCV>nBFcjK04{_seq5TTL;;B7v`Q&HYMbtBx?1m+&3+d^4S;APfAOp_3`!1^!PZccIuY~(w8)>8QJ?2K(m`$A295j;?eTUM`;18 zZAYl8SLabvP<$^dQzw;{k^u1Ub9x{T8)`Lsc@rQ1c1RP5Y?6u;;8krsBO?$nfp&b= zg{MY zYaU7qFRuiZ=YuPjcT{0Y*_TnBsx;!5KlEjFb2GELT0aY~r4Wp1MtT2(#~mFV?h?aE zS^eYV|1jY@8CtDG$K1@!OkzuQvrGsWr+4`hqbnhoPfJ(Jw_w2=D1CU7M z&xzlo_W-FvAYlN_z{YBg9Ba)A{nA>LfSeiaSIgD#7xlA$k~Y0<1*iZJn>4>_=z4s& zCD<&wc&~HSZD43f%GekV(po467*M9vKKXw#$O8L`@-rq zWUfJ;h@~Y<5EZv>e!8^$h}1`-l6jqYO=XH&x2@`UPG&hzGydXBChBQ!Zbt5{+QOjx zyu2NydL{2J_wvoAyc3W^Sy|b7A=b?>yR0l01p6ZovT7v&@pg7!?D+KS3*svV0px&` zy*`S=j&en?#pwaSxWf;EPDQV#N9@J7u3rVffEpYeTw%OZQn2Xq3UEH(hQxxK z;OfT4&x#7|tY=aZzzUQ-|0UaO4dwy@0kaZ{rluyfE!|LC4G=#mDGanx?~;~~+q}HI zFL`(v+G4FKZVO6E3?cT@l9KeaJzt}E#>dAm#oI?2aRa8V(uCyXcOZ`pAA=h7!|V5e zg!T0F%=6UjUV-bHG?*U(V*w1KB9*SLu3lYVk9`Iep9oxBv;Zq?H|IJ;+bp7deSKHg z)?(f%*uA)X*uuUy;0OM{BEG`3AWojGDu(J`YCHf>b8~aZ86&U;H2M)h{(i|43t$U< z_PDn%fmUIR9pl^(0{#Vjn34RvxzLQ_CJKcDN$dC=P8%i(tp0>Fj0c%$Mto`b#I88U zn?t*IiHW=F?ukz@md`u-`K6eZRBm~IK>YIuZ*y}qW|`iB1z>o5&O^?mgoIJ2%%Oi6 zLLv$;x`gSGE^{v6Z~L5}d0^|+)%5Iad{`Hl_I);d`;o{Qt(mV02<(zOwexgG+yNp0 zJr7e-3T4w^hXES10wg*4^7h&j6DKE?&g^hBD8rofABJBtq^^O%uW7BWwC(lv3=o^7 z(m~dE|BL&hDlZ@0c_|?9{KjcB8xU-j_FRdCPoF;hiCR6gFF&0J>I*mu@GAXLjo#^j z0lRj+{pQx$(fxs{n2j?tQM#f-MhTLh_cNF?Bgsz z?FNW+WQ1uE)p9xbuFPcZdO{1D?=MSC+Ywb&iQqX`KfDIvA6O}wsb=cqla!d42ypIQ z2?dY|f#T`X{OTJ{;Bop{dIem(yq5(+u<776qJIU1iHQkb1$q+TI6Zq)DJ#JE#DoN$ zieI%_G9Y(>&ezh_VsS{gPBRz({C^Xxa4j7O-HHkr;HX4$GP4lbOf`Ty4tk=s+1Vig za9+UK9YVo1Sr%owK&FFVAENEht?P$em-Y$?9R2XQFYgE`K<%}an9NO#$o#uoSJ?-COvXhb4N z1R&=m8m(E$-T`_t;T*l(mO~7HBsR8}e0@uJx8iAG7+54S@xNMjomASk zS_{a!&OJeTk3^s}Mn@?+%u56G{|O^ogV7_GJ#;)sXRotRRbAsDAue zsXbMf+Oh7r#~`+fk47BD<7VhzfTeOXY?|$J7SO8C4p5rc%2FTfWa{niF0jo9#0G2y zM2Z~eeKQ!4#z0n*Mz!&OQ+5PO%gJd6!4@`MBUWC*nm+~04HfCZ z^{U}#F&Dp+>EE;|=G-92f~XcGY2=f|eGW6#`POwa?z3Fxk2)&~vZ1^n(}aYq2-Crj zQS!tI<8R8(gl)y<^YH@(DKx&OV>jK_Dh-=spOST2MjL@+dYf?{`=o-UMam#>PhQ-YUYrhy@-{ z%VhPCp?E+$5Bq!AbNCE3ZlA|th~ZMq51{vi!1c;>;Yhx|9K+(Ft9L^rrhh6AIakvF zBINbYJxfF78oM+w@jZRh#+xHG1U6D-r^256AYAja)F-q4?*@F&8|-*^lQHA{&;lgI8Ga9=!4xirE?MfaoH!bbAz{6xt` z;rHRnHyFW7*{BbNMNup2xfasWgQQ6Lku5-EsELV*l?n(tPlFi{ncb+Nww(w_!zzQ{ z6%}0?e-z(>0v^M{Q?Vv9G(U1SI3ARS(p}d+GlM_icJs+QyM z)Zh@U3D1+~EER8U;7}#teEaO7?83h0-sznxgCOzrx9Q(M-hJqv&(USp%@Yz?_z9I; zokTZ>AyYj{@r|oiqLBtb?*4gLP?v97d8)Cf2iaXhGXrisEGce0Ek>bq3Ztkx9?dgf_Fz<6On99;?%+YvJQ(FFt<44JYNTqvHQqqo3 z?IpAR{vBwa|M)uq6F$CRQ;LJb!*>+r*xzs2v(Hd^#(D%i!;xmAeCt^e?uLizqVTeS zd+rdZ&+0+j;+gvBFXc%J>%p`3ah?c^f}`wUQ}u$p?<)hML#u<%)kEzeZS#M&Jqy;= zuU!F-vR$0({ZP2>%T6-3|0cvDmsllT%kiZ~M7&F1I^?Ve?hxb$! z=n@x_UrO%3Zr-CZ?5!NIJT`=KLSU!Zr6^4y0h_94wjpftnWP;*&(aFj>BN8CphIpk z#Fdt8J`sf{Z*37BZPUfOpNVgn2Gqkp;S_!_+Mma%Y%p8Rmsk$E`q*j^n}$ul!T*{07-gKh@_Natxn!;wfL` zMlg(A{i@O~@qB^t!QRYqzKLW+D9@Q;ZS9dkpK3DquLz@`YR?8%NfQ+k4GK$1qmx$z zStP89u3q6+RJ6es^7VRV&LX?9^>IbpE;ibCm6)vc>_<`RQ7g8nH_W$Gn^HhzjE%7p zg+!Wq=@=RPsI3i>WoRGN_~10kK;c$o5Y$dxMdBNVv_|cm>pKf{DkvM9d%p^qo<7!o0$yYn#nPf2eXad?(08 zm8?HKvehimN`%APIKN2qOMQR4FR#=Vk#G1r6SWwJ)iA7Fba@!lVX=1#P=52okJe9= z$*3qlO&j4cXvT4n?X)ObuJcQcITzw%V)3?B@!v9QC7(>4J@0c!pq(gx6Jnq5-Ry=` z;L>G?e*RSB-Jc{6;LfaDz z`;&XbB?(bWmUQ56r_oR7+KOX?=a42uXaE|!K3T7>-3D%BLc#cbcK_Aw5D^~U(>3GSV*Dtp<(rSI*;k&!6hUOj z%ikakyBa1LZKu`!OpuyV6yB+-BA>?4Ua49>oeUWV@Za(ysf;a-&@R~rZ(w30yMXPr zdTf+QUoA@rL6P*w$W5t_k+$ko{j!YVtn7JMo;YR@pjVu#HG8x-e{|34h}?b)^}!6^ zq|a04UGvYg81oV)YBc5O?iSV4yWrtJU%%3>RLb%Z__o86g@N=28QC5yJDF_e=Wkt% zO4)Bu2(KGlhR?-p4|E7Ezv-z*@Y<#7PRoyy(qF?;>alPHpIAY`m84{Uui8QpZp@Q6 zZAaj1af}b3dpsy3D`E5ZyF2!&=1UOQWc6v%3 z9)1JZJ_|B`D5$Na^Xn$%0L-Ca2v0Keu6fy_&!a$Tnm0dGw-IRrQ_*i^ zs_cq?YnVH*SCiU30C`Yj#QSsmxQJb3+`GuZGq)DM{MD>NmO{DBJwl{4WEa(c??=8o z>ya!|>}gXz-O!*z_ch`K3_)*+6FI)gbYQ+3HNxTKUg4u4Wu$qFXY_-2qOhb^KWz zfY|KyPtpF&XOWZ-rz`S2XH%I}{rCJz9TQ7NdEqn*Z{30qL^mfTpBM?#G9lUV+dMKM z%xWAD@_wS8eU`cw)%M2L_I+U?$>e0Z;UhoUzLHwmzH}uW5NL^s3L>L98h_7Pkn`n1cf9H4y9Js?f~mk zP1Yncf;cQC|3rjF$YpmaC}t*tr;EXh;NL)QvyU8gUm+RE=KGP-t)*kcZbm_-s3_{= z(|B+c46(n<WSO`m+*TXy`8bZbq%xtXSCxpkZDr-w z7E)53-l5!O%*2JOX~jCmOE|+w2o6pS!5AOWC`P}17$&LlP(E8dEAv<|N+!a^oXa9U{>~xS>b^kP{Oz3z*>J z3B=v-aPOW}eKS8{#d4cM`N!WrnLD~x)3|wt=o+z;Mp?Ze8&ZlmKjOWAr{h)v2`}O< z0fBc9+!(xHD$}fg?1g`>P7&hg@UrVmG5CCA(DfkkILvuIh#==GhizFGiOaxAOzqz5 z5t3I>_~~5b2BHH1=d0I}sfD%xHFv#I{`M{l_WRLnI8%9rqNi-WCdN$?Gtcnfas*x3zT_pcx) z#>cO&AQiGhyMx3X3<^1SE0Y{v5{5sO(F1_Mi-d^te6bi)P_z(mwfrzP_Uj;XE@>4h zt*?(4)z;7`ex<3HlQ0}}p1gbNpzPK6^-In8Eg7~M5IMgq@I4e?K6myV^{`js`eNVI z5-!)1X2=l7{O_iLcLWAfUX}g%s}QX7x7xyW-|RS4vc}E*28gAa{T@pi{~F3Xfba*+~21!0HxJ`Z{GsMpueZg(SN+#H}oFR1KksnX|WK zlyEH@-!ed|IX?xmCpDG1<6xrd`qQUf8u?3X&@?t;LA6}c<(_8=DSt`sR4qQPy1}C; zF;Xsa=<6ov4w70YzPr7hi=2ZoSHl5wr8IREp;)drxnn_bGYbNa;u3`nmX5IiTDZJL zVh2gY8Kr1<4n)uYX{n9oU6g)&MAl|aogBE~E4>G|DpxiLW8OY|1r-IkK-q;XcXiWG zLa8!8GN?%~ci{>`7aWa`ekRT^)kzWLttfreQ;zv!FCiZeZwf(EA`X2@bQZh$mtT1##d-YJ-vnb&KWhm6VmzqHo9*aQW?eD*mmVt$} z)PQ`Tn*61&G~y0A4X1G2K6K7a5MpDK2kk}JGsoQlb5k(Q7X?8?_4$X8C5{`BJK zQcUjX;F+I)@j4FSFnI7gt6{<*5@wWS28`!2_Gfi)UG5iv#3>)!w0JVH<0*7Y$-B+U z$%s7iAJIxMuhT_Hp5}h(Q_lFz$QMN;9e-YbJFfG5A5Vr$TUCFv==JR_BR99MI6ZO0 z_mI~!!x}r2rQe^XvU7{3nE_Dp^)=~=cNxknnRHcVsxq*cY^bU+SlFz9_3>1$7e>0H zapMR@*D^YcUdA`{Yn0a0ZQrfcpBYapw>-0y_!z^(nV+O4$b$0RZ%sV}SipDgaf$t{ z5RnIWGy(R{&x?xF_ZG4y8lP{}EsJc=8spZXwOhWyraNo-Md0x44*tb44m({nA1UOu z?K>TAp_?q)TK?r@&aBIM>2!5kTh)Yr(E+tB^hghG%b}9lt-j#$5Lqp8H@`0*!YPtE z+rh-aB?f;=y6du>N^@{5b1O(VA*-eFV_pN#%jmh>Y-vEJHRqH56}4o|apS)rGl$4Z z9M++6_E4V%WKga-r%%y1=N4vl^+!M%0&I4Zzj+6- zP@$tQA<7SH+4j75#~X1FxZ<`p;qN(uM7tRl1PDMyKQyJgPWw*+dCLxg`J=~k&(t=P z(j-QB0sBx3kA5ld$oX!en<56E@tv-22R`0khI#EY82zbh+@G2nsTcy%5kM>m%Ym^+ zKk`{XuXXZr`aU~5>s|p#YW;8xU&p{;p&nC#YX#ZT_B2ZPYK|%?isuDc_=7t(*-Nh< zzwcev_Z;9?ypQ5>uAcnXW#Ljq-C#lz)#ln1^$yJk8o@4A3Y-+p>zWc92z&MC55R=^ z1QT!(fD0aZm&6DLF=i#9BkRvdZq2w<@m*K*l`FToOk-1eWrn$iZO6aZmq}Ud3~+DD zFMzxJ{qNEG+I`*H+Ci?aOikSiAu8V+x8<(c>*Qfl`b!e#RRtm_Uldz@6txar8vp#g z%0LjGm=6WW%L!Vp9oEqi>zwB6kWx}&k(JF&EzsvTb`eYk+4fj=TlC(kBka?;eze?5QR&?n9uZ97to9z~oL|Eh&U^hWsvsKP45lJ-C81GPF!lwU!SZJA_#K4xQc!R1wGOASyXKstaGdTtUWt(6t; zHm|!0zH(W(g%=(kK8lDaTsNrkiY|g;)@v^Ix_+V{cIYh7$R`fi`Q*364X3A!xVnSc zhQjGCX7K&y?!#x2r!N>CYY)jhC*8^G5B2M%WMuF`i|_(7jT7Ohzu?N%xPwN3(Z=jG zx)VG&X2UA1#SBmQe51_o96h~c1wFFzJ{;0MojR_QVTZi~ovdCcc7MWmyVh?%d3s^p z7$+De`4NHO6Fsi#-QA6XQfH&WvLIkExWfdED zCS8ByZmdFHZv|w$*({=KzSv&C-{(W_i*lQ#Pju9UOdiHQW=H3S!UrlGKr@UO!xtAi zbZ_4Lao9gP-YYwD`Vqt11SHwMu|NYyf>(K8M)>yeR!Ol zO7&;e&1pf_w)Ut46eWPRIlLY2qJBSz-39s`_1me6`UF$x{;$Bz*;%-1aRsD3!gh5(W6?Jsza#r@9gKuAYX9_tG>ht@rc27N|)gmE!H3tI^ z0E%waRmWz?px5Y>6hD<&!Mresx}X%_hT=#*yeB|wO?NHl3u6wR$9?WlzGzmdwXtN^ zo;_H-!ET^(9;y>>SQG>WI0Lk#qdLjruzA$t|6%9);NT!74-R9eBP~9i_n7)WuQ%7kWaB`v8Vl^%pQ4DR>kZ)hUafGJy8~x3|wCX@sOR z)n>=6;1*zGJ6udA+Q!uh8Y9Ktf#fY+n!+)nf$#zMq3L z0ie6TzYweJGS_??yF-loI^PzxU3HJR{F>(n^HN!X4TqHnQo!gyiX!Y?53H#+HygSl zuZ|)*Usa!PDM?Gmu(B^bqEo&X?*0R^vy|+5DcFFyX57|lj>ZYY!-IVK`lsm@(|nQE z4n|_61(2^D-@Ny`z1fWvV!AYo?JuKc#^j68_QvQA_lVMsH%i$=gC8%3@JsFEv|X3} ziu?QC>?(B@bO-fO2yulASb|YeY=1ukN4>CwZ9cq>mMZ5MRo*;2UNvX9y=H2i12H5y zI9qp~^0tZwMVir6rf&#e4__Md;jdHK;-uNSTX)NFGJ_7=rScfSlC4%mQ?j zWug!T)Zdt8^Vv!SU4GgxcADExTf^inQnS^hNW#G&fTG<+ z4rt=*$A17teawRVL#d6ivWsCy44D{iU(6W3=e1craE=~8uoC050ochWv6E{`se(XuwNYaD?5f1m zJLk;O-nRU#73upEZ_Ua9lV!Mb;>Yge3zFTVB{MGG0yVLbk#m!j?wS?sJcpbyo4t_ zr~NOYaJyn&E9oI#$JNt!b6%uZZz029KURU0=J`iNL|l%dL3S7w!QiKf@LBK(a&Y{~ zmn)YinJrcc=2Y!2oYopp2!n-d15{?o@n-+7J6-)@3Wzh z^;e!l(Rpo^aT6^W}LPJ^g>UH?n0!H;F|y0$q5%|YwGD=v+a4PtAG+8;RSb!`&`Av zn^96@U;@c~EguejcKq`t+&e_#DTxzspKRt^5Hk)odCLfVnp3|xgTjyK0)E!6GwB@f zZ9LG(y6oVxJzSIH0X>h>F)lW?0Io!?U9U)+p=MB3tq)B)FaZtgW~gaW>qP&dcX_T~ z<{b30hKuezxyP~XtxVU|l}FK)$W1`iNs*k@O~CPGf3Hg&94S~1wHbN>F9Pi{a8Fuq za}$fWaGwkDH#csAPK)!*YkB`Pbq!%}#UCq0cZ&PGtoWsRDclCl8aRD(G}yLoy#dV#?F;J)cmJqbR~Wo5ODd@M8K zo}y$W^7T^Q5{q4M0vTZLuaThzU0CVQIKf~=_{*=_ynPBUnma^N8Z}g@*P>hiG!==QcwiTZ9XtZ^Y!OM$4F{bo0C}pf!>R zAMcHl4?d!|b5vuFYqFz2kug6HM#ovN+;S`wsdzzOaXa9+5lJ0BE-PMcQ=lP$Sl!el z51TEvvIvyQP|2CwfAxsgVGh9D?dubV?-?jH_&+6)JZmt+}55BJG zrv)oWpOmU9)_=3LH&0XmHUQllAps{^v>7HhN;2PA5)O5)z7)29a?u$lV`8DEJm#;>6!Pq~-1&G9QnOB#91% z1>K0F>K`Wiu9Cya56b{!5@X{)I*j#aFhB$03F<(wV8W&!MI0Z|n|9V3h3k|pW^pcK z3|{Lt<8}l9s|r19!=bosoIi$75#>}pdEzJPW@y~8s+zhFPQ^km+Ms{SdPU{=l3NyC z`uqFeK%ru@PF_vtKOy5;EOJ@d*{88$r}we5bXe1-dC4cyiqno2=-Sne{#hpjkKA7t zy1x;Ky%vsu=pz43+nSw50)Pu^ND!Rw*JVLTExdqIl@CH9?4atq44pI zXH$0WkgGBDn%4bcendTJ%?oT8aw}%IFf^LR)#w!Hv2^rPs;jRB{`wqwnxob29z_n? zTx)k;3MHF*XcY+S`qFDB`ylMylOcM*w!x;e9zqnf4rVwkV{ow9;d=>8iXxboDEqJN zbnaKKM_K9GRM`s}7EWv5kn0;E2Xx5AoiR>xt5TbN;`8ChV5^>-+;5=yvskK^x!|@2 zILArgx4zM%1D`2wO?|0|N^Rz0MZ>?7!{3`qHE3?fnUw7p3b-3)WXsqSiwGrW8t`ce z&JcnIQbPm9f7{nmq@+RD8BV`Q7d+vXWy`F78j zH=V47_hiLBeR|2-RoF{YcQg(ZG76PeFBQjPA0KaIdJ$BRI#At_fK(2K5_0cg4?6P(Y-yY z`dunN>^|HIoY?1|^7@nw4@kN!l|do3jl*uw?{B6~1`RRaKAalf%c1O8cOTSt)L>rX zt1x!MiEm|9)n>VL1|?|+u{Hq0Fbl%2f4^P!AVZ8n8%5%(H%LtMT?81RR&eGI)fkc z$Lj>Tx|pX~DGoS=^0U-|?z2v6T}2W6?Oqh{`5j@W74dz9S&?7xPvKB_p2pv}dwEpG zdH!e_!9lqg8r`IEggsF7U@~;}?Q<3clrs=p0+R6D<{Sd@7yyK4XHUSG##!KZ+|c8- z@4wi>@-)|~4jtQb{En;1Pe$8_`ugwb z9q%ueg62vY=-tW)77&Sp@fw9&|HK9pBcqrQVxU4KXY?W zkcHq>r$cZSq~>HUT=;NM-EYekK7X=1|GT_=_s=!rompaN0*Bu$L5h%Pq&b(IqvLj3 zqGfI>g{kjBW=R4uOg>wO2NWz|=vU}5k(?#PrQ3|KH+#7qJhQ4NCKPY+j$*j;R)G|s z{?tK0UXP>+D)+YG2oS0)>aW%idbq<)NBI2T#2cV&Cx&`WBcmh7eVu0;NkN6z0=hs= z#ZCflU@f-Q_FniEijrq*eGq^+63@A;f`9?j zYT)x9s>Zb|UyyrR_dLpN?9YNBNqWOJNo2H9wkV~=0YMr>fwT;*zuS9_yk^iiTn7L7 zz&ko29pgHJ!Pwjh2Yk=Tp*N02otM-6z$RKCs%GX|7J`@8HwcGftx>DslO2gZ9nPRN z?psZHTlMMVMPApa2t(2W!DYRj{z?xXy#nbFaO6#gwX~*2!ODv1-$zAG!s|~60GL0b zrZ(Ugbg60shxH!Zxp8@p^pvW@Jf{AltKQtjB?gPdvWWJSt92pBPe55EnX55(856fd zG|l!dE&#d&gJWRg>3xwK>JAzOf-mht>NGTM9c=cv`C%4d4y4>07u&|GSTLi`vN8B= zXZ-B-$Z@p_xl+7^>xxPj_?USYt_L*zGl>SC70tx}TXc77kzNDwz6^T$9Z>;1E$7o> z|Bt7y46Ew-+7?AoL_h^aQUpOM>6AvgQ@XnwNfS^Sq(MqrK;=d0s`+k z{NDfLb@{>L-m@pxtXXlddsOklha}xWPZ+$nL3H(nCk)%!9o)TMdHvcXgrZwoVn*r| zstfy3B(n?c+@+;so+)G@o@?!Y!Agv5f7EUgv0}+_d2Hl^U%G0vRKAvMrsnh8G5m&8ym8OPAMykr-*=?r2Jmx}T3H{lncpL# zP>AS`fwbK5ar#i+4o7H+r51PcF@ngyKC{Ti^msu_e$(o6;X8Ej@!wd==>t%%4L_Rj z1cNOTYPVZRpwJ~vGyw3UrmtQ-kQ-e6m}D)%0R9ftyWsgOi~d_lIeXf0l-SVjHML95 z&^Z^vH3iTpr@wy;K{7`8+}pNkFg-S!7>jB z*}%#a$Hl&RLzEvKQKVo?#iKjYW=4GY;);S^2CEC_{R1cwcapTRG!5ByJ#`4w=9(?c*`jw(l_i?>A7cOyrGR>_OjO}zBJYBWLI4LB(&3#IrNxvg(Uo!=HwD&dyL~qD)`Tt)86>-oR21 zqamtql1xu?%5|gR8^j~QadkXceT`)DcE>y*)4gV; zB9Py^;+Fq3W_z0eBoVxCZjbMoUvj6W3dKE#Vs@@EGGb_HpN=3mQ9sy`efcA6I=K-g zyP7dC(i9&OU~i-KD#Y9DUw)~E#r~UpzLzq$@A(RXcHo0ALA4Cp`}95@O!+<69Q1tq z*Ig)HgA_ttUETAuzw=@lN={We{M>!)At@(SDNXa+e;{qQ4SxaSTe+fE+Ak-}p zO&Y80{I}g|X=g_R5u$9DpR>$if@j=NUIrj+p%5E5{{{ZM!E=DK}RCiP{9!gnbB9^T4cKvr&ZZz#Z&quA-jckpt`>t&%NqYp_4Af=(5 z=fk1IkZ0iWt_-A~0E+@j;9`6p0r@H^>O>`^?j7z0ihLA&ccM&6HNW^CuNa#j#b!G zkdcw?x8A_K3`vMoOX8G&3$*QvPH0nL|9phpy#W1UBI5iV#lMHC(m5kvw%Q`7_72g@ z@5r0{UW8IQ^G>h83B#;I9K@gf6dp%iduR90m$uocT;Rj#6RVH;u@I(RKqm)5H%C2n zZYA45kM$@iE=!J1Z6Vn`2A#08)B?%*G3o9;H6-0K-hSNp#tc?)KHj_q@B>qNuEnv^ zgs&jKYGyX%cK!5JNvP)cQhpr*i(;KJf?MJFmwAX_paWW@xXCYNLrNfO7ndMCIQAe< zBV7P;C0tX1SMsZ9Im)A-fDa0gA4+_OBGgHydZnY~)3CxdNLNB~<5r4RKYKYga&Z8X z=iJFC56`PZZ@>bLOYBr)O2oVzGCu6VmM@b~I7AaGgDeCwYmR=LDLk=>#Cw-;$1%K9 z4RY-T)Z_&MsZD9yGw=8TZU&f!nUAzRcY}@`cwIr^K)?l}KM*3Gk)Dl1au(peavpQb zvy8z%oEU5_?G1ID6?w}wowRgzQUjSECcv4{dzIwy{IWuV=MdPL>3leu3~_cLI<@qd z-wGKpN;M9Uf9?jofU`hw$pT!K7*7aEj$qf6FokOwy7-uk)7wA1l#Nm@TwW@ZCUAZ-NL zft>>TI#Kq#;~^V_ET?u_)mZDv6j!C4YNopHjMeF9TND~`Guu>9d*+#>lKmT?NDhAt zZbG=9g|(z>-NxVEch^=+>+*iOA)tlcTKZhHuKt4{aGl)j)}=&>|ornrpfE9cKO}Pah>uS0&#ne5}XBki#%tzE4Xm zQ*gqG6NErK6csJ*{PLAO#4{W`_Jy_x&bL31j1m0gw8%Uuz%if^FSe5}Nks@JKMl(` zYvj{pS>FBV!Q)#SB0;{AUBg$q$s;$IrOFDp7&dOAnz=3c!Izpm=?G8vDpD7h_ZIee zg#?L8oiNpRE40l$D&r+6$Yy4Kn8^;dY`Ft6kD5Ts2RA?zg0~RUKu=1e(6=uOd674f z0Yyd6I%b;~Zwa<*$8~jm8=$cb`MUJz?rgKN6J!*~EoLC6psf4x7Ye?t9cyccrt3Z9TNUL6zSB$2g#=Wpr+Vdi9c=GKthS<{_b)5n>u-?f z#|%_dV1chTx371`Hsi3&&XFq;F#Da(GF3cCO$f3<0HVt^Nwr*^+nwCnTm*puB@o%NZGRkg5ifI9bBU;WB0nr zq;c>5O|)?_ku-k!6`$J2+Aka7MK06jSRK)8|MI_%J82eZxYv-;4_+EKr8>2?onGA3 z*H+MHAl~ne9Tr#$sV}sC)vcZBmI*PTr9ZRxHtQUl)6k_*DNX^C8 z-oABloSb%J4HDnKe|^*}8QYO=o^MI{Y*21r%HVoOT}Df7MyZCVY9RNwRYrI0$r&JP z0sAXv5-U@LCCG)!)+-FtuBj=|V$Gl3A9ib+B=fQAhV-LR|4d;GFSpyb{VWMy#8-__ zael}IY3itY84Y#R+F_0Cw&<RiLLLc7^D1NHiM964d*4x^(&lVt5P0j|isJVGyPtOcvd;3JBlyGdP zu!2I|u)3emlywp68_pj)@1BB4>1a^Pg@w#yWHsLos_^ND3JVR5SjrcFx_zsz(K|>ioHH}TJ;o{a|i*7 z2i4_z(~%Dy8{7&5If9+;ws=Endl4B$J?YAK`pxsMN-z`?iN9^pdPs3Bas z)wtIU&wa}Wt?1OW#%_uQk{F}h>drc63_kz(@vp1pY;^dDHEqKu|1#vD~%fyLvHDh3@F5k@?-K>t@6Ou z(%0R89N4h%5EC&E5tM7#x=hJwlxp^0C#K9!10MydX&*(YA4f!|^TdP3yeD zw1Ar920-UPv*f7-{WhEaLT}Gd2ob`yw6m=+BW34qhv_OU@z!_Y!;ATV=*SpaPTlVh!9lWipKhhVL~lj>~(M z1&Y%Xv;WB2uO(+zo_Iqm>5WP{=cu10s+s*JC1M~H7Ee0r zedK-_d6~N_Q5JqkK!gHZe)~ihqTlgJU46s9W0QD_L$%tJ01OA4uW-G9a57T5=N2BX z(BoJ)Fm60`0XPXL-L4+9k77fGUaYIdT+z~2A5=1p3T-$EjKOZ~rG0QLVv{u-dwja- zsn5$53xRup(nl$U@%NP209y5<*ljQ)&rYpAbQI7z`Z6L+nkv=+76_dH2gJ;d3GlA~ zas$LGt6-G`s8~8RKyo8H1@O>#+0p7N^XieerV+8pwX>*w0P8^t5|T%E!^~N55}}F! zQ2zmYkz;U4QJ`2k&T6bJ*o_Rf8&dF7k{r%es09#-ffD69unG9te zki4(1=i_2Tbpl?wo}{>S9+Jf9`1_YH4x+n{D_r|9R)wGd15EC?gbvua5XicYsrm<& zx}TlLz%4=RJD{px>Z$(_=Uz|Cppw*2`SjCW#^C9HAnwqF@trovPk@h#E~D}G0Qq3u zD0zqMc?T&AUR|0*M#wamA+CWE8bp!M+}TcF3n7~tKMHmqVa3H#_j~`~T>wZ5J^Wsy&L{;)`Sr!rD^~ZN2qEe2nA9x{ZX`Y6B~*%|?WPvxh=~%}v1BQyQ^MBVH-E%l0X$+!TO!p|=oa zuF`Q`e7%j2Q|(wq%6eQJVv*1>_%v3I^#S1<>jZJ?xim;rxF2rfa9U9>kLn$GUcCK# z$r;2M6m3MiY9BiUb0EA0yRMq)zB5X0TxY3_JOmC%^BNYJ64TflcjynpI zWFN6rmYUI0qB<}OM<8T+{;%5Mum&KCf4!~tL@$RIR2-zXE#=c=efS86MsB-=mlwKd z0)+10+2rJr%tggO@fTZ2X#!*)aCg6A2^m?VCUAmt30#Xv@9L|=*NB_bg_$%cQ;G~wGlw-6P*E|S zcZ~dPR$lE~QzSXQE!?Vw%mc`cnPYL|fbtu$bTIR?bT}Qj*+H6p8*TzvYcO|YI%#s3 zWK^sC3cw5K2!o7@(OO)GvyXlMBvibvtSskUdTwsa(}wP_PujBo731WWGDK5_%ihy? z8}oJT zrWibqk#;#mx*3yQhbh^h5308)@9h?Nzq)~`<7IZdaIqDrmn$8SdBpv)nd&D4<2zkW zV-W04k|(c&!jB9B$yNjG0&hd?Ft<*_(ao-`2SRIk0_;2SH%<7HCI zjUx!Mf;yv2aY5L$;1i1cXe?p4q{Ae^e@T*VraR;`j?Z|$%LzMtBU0D;Aoe=l?SfJc(`ySN9i9H8DGCCq-(sdiL z^ft8Is*_u7Amh_8TYN`{u!`nK0ku6`FwXWL{#sPlm(K>sxjpx5E8F6s;Nox#2k zScU*MQD(_iW#}-?m{PJxK(O9gq{er+3!5|mIh__xQZ!Od|7Z&(_n&a0pp<7-Sdvp) zb48&OlnC|*tJAsW5*v0o-BBddvJMkjtW4{Q%Z!6XW6N1tzmH}2Z=XEf-X=w#yz+tU zg#8(rL7|cT{Jq?sXaUzlw>&T2<;u^dx%B^(+o@ADG=Bn?`mu4Pba77Mu{ABQf)(4A zmu5gED=9ZFtX+WURjTU3vqHwntNVuI>ujzCDW%<0EUt-lJG`(avV(d@C94?X8?Tvh zoELeJQM}2X{0+@Z&QQ~H9dk%N+UjaJxWvV4>6--)|En__&e5HWm2Q#t(<(U3E`KPt zpZ!}}zj{2Sa6@L~AR9M8eM?2-L}r6?Kl@UiTukQi0Mb|ORJS03{xtXa@~VoIe0IiE z_VC+aEkBT%RjEq5!>Lu3$IF$U#<67NI&-|Q?VBn-$?=Qr{9T6v_$vk$+;UXuVsG&>`GhrZECJEM@^W};28PueNvLUA5gj?qhYH-` zExlYp-3^uU<7Jxyg@S#r*n|AlB@GJm{mV|}`UO{sV#+;HnPISXxsyYWo8wwb$Fq;U zd!nC>NT;mk9Ol-%BAC~b>~nl4JD4X?X}wEfg(X{MGnCDs%5GznU?pw5fmMBv;oE8B zJKiVKnUAVB8|O7A4a?n_k+`h7`Xr~;`}FU|v1;9zhkh%oefiX(25>rh(S$oX6Y~Cs z;quvQMB>syMe1sP z)ou%J8&AY+KOYVRl{i(sx6JF;jW(WpF+#Rq@3PT6J>LIWD`gb~-_o^0yLH~tuJU!h z++7~hfo#2xq^?|&J`cGR626g%CEjXB;(i_>ZtZ_&trqq(&R$PF+!eVBR5Ju!068GuP{uC>q*n++Q-U8k`pk8Wbmpj;+R`Bww8JfixS$~CQ$GQo;3;)qV0m2@a{P`1^UAV*H1DcWW zQKs_8?QKVZ;R2`|$Pn<0Mo<@>L7=aGs1Ul~t9kD&pa3W@sE^?$^|Gd-r39Z7XtXMC zlso+f))vS(|9FRGN&&YqT##*73i}%}b8wUwCcSz>V1@3y5h@ZG#z+)_&I(B1A|(5! zrXcHkUOoBx1Ki($p0N?j|ER<|7E3;}-4_X5LleJ`v;j}X1z%e0qLUPlGqf*{9?z|> zNmckF6VK6o_@u+s3UPiYXMvb6@Wwb)>4ezDz{t7LPhPd|jNd|4KoJhECNBMZ;8cVa z;2|Wi!Er=pRCIWy@G8e-PfyMTcFGxB9DUr{Pf+G0=G+P2%N(zAi zWmF{(Ne8SRzvdm&`=5Y_Dfj5WEge7vV$|EVvenH9A%#3r1frx zPDw1~X;_>;)Da&R5@Lu#Tr8HY7!221N2|vhc%2-$FZ%iZC7P6q@M3yx{9w>zMe04Z zFBvX6HrfK?gXFj-(2@YpV(8$?LSNed9|Q~ZIaC$O(36r(+}zYJW(pVGmg7ZRXkOz$ zGE-ZgU7&FR;3Bmj-1Xsh9sQC)0u7`PfX?t@pnRXv97Xx6bZ90#LBVH>T_5m`uwW%^ ztp6i1D;Z;eY|`G|o+Ns%M$^6?-a`#!E_NNl_R-c(ycijpzq)vX=zpwgn3pmBw~*@~ zGJ&?4K&l2@JwVX}OinmBIR7<;zK8_X0!`IaS37W%UdUshHcV2#LyRf#B{hP;LP`R? zs_+Lr{qHNA;hkIP>idKQBQrDci(xatQd`9R$F&7w0_>H*5p*F&>p)={=#Qqj3=UQ- zQ2+0@+wdEsX#$P2voqu}&_xv9f0-|6yY9f9E?40L+Ga3s0Ez)+_uphvqF*USB)w8l zz=O%S5TgU4mHlR0pxXp$Az+~})Qq{93VxUhN7Oe)b8?((4}jCj={GbQ00LqcRKD1y zt*s5o`Ogy9sAOe98{kex%z!h~JLe?P0i&ujlNhZ)$}g~BWVAY}XG^uN`y1yV;xT}UuO zVF9Fxpu=baC-iVA3qK|?{^HOEhN4)uX~-7==COoS9lp*1!lV;znXvEl8;C4G953W< zt{R442u1`85QqZ81Xuz=FlP2WNhTTKHX1%4L_|R;kuOIex?k%asMKeK`QT^DGu)KD zYeSN!_9S{pqv{(jk*@w}b%@nsS8Ho)zpE}H1eHtbU>nCo0La8f2CU>aGC9l2>&aeT;#BLe#y?}G@MSr26X z*MYfa-nkAcTRneyk+oovRTy=x(J;Z|9^myZWvlFG*l~G0U$uYGU01-4(tAbwYxujZ zs}KQpzJB?YUnPa4jwr-&eN(Y#IOgSbWfBu|koTyfuAj0UjV4-XXJC#))k9|ANyGoKOPCsEs=sJ?6U?u8XUXjhI zTFIHfCFiOihV@qX`iG%u@?KXi`~N#w;pByb6|GKw9jGyhqB&7?X!ABaaT*8-3wyR1maVYyCQ+z5B#y&m7ngrSABy_*yr7ld-UIQ`v`+p z?Xy*Mf4z9Lx^|7}_b>4~E)()GnYf>TjpDT`9bMm_BfEc%qrtZMtxN<%7%PYQja2tk z&g+ALlN7i`w>@;wD~o6lF)SA>STw=BygIj`PV&chWm}il_`B#|ly!mZ*J%Ee!2~^cx&~f7E>guBRaWmbHapLBU%`u2ybuHS6} zUT0+K_{(1FT%x*3GOTga;h# zuXZZ8%UNfuB4ipd_A|>o(}XIunl+(P`vvHmS5*^McYL3e`CP_T^zoujKR(tn$~Wbx zR^#($(5NIV(cGVD$VZ~iZ5Hs;SV1!GAvW{iQ^5>zNVsCwXDo_9ISyS|_Cj!2-|hqtSVIHLP%D4(Z(1%1XF z&mTUG)Oal1+6@W^t3X-P@Ac5?Km9e0?nT7Q`e^NEe95b;l!z``?%r`aCNern2EM!_rgz2MgC zt_jkcOFg$O?6T(jD9eDgrQ?UWdOmkj&l=ydw?H{2W0cZvWLjS|%2M-}-#=sQkC+>+ zVRu*8YwH@cDSu6=JjoGgM~s9KTNB3^hJQ(T#KJ3WW)~`*_IlH0F09#NbuGiv0pFf` zA}+#z+&{T|_svMnse}cwg*FzM%u>(xZ>YLijoGAMMY7o@x>B&9T2}Vw>}~5#g(Wi6 zal@_DHk{Lm{gs%U&sMVAnV8ZsG22Eykx6)4=F<*7Mkx%g@(=F_;4Lj@h+?Q zIm-`pU8QboXg|WQ0h`%|BtzBcID{)_VRzkcy?8DEN#5FODieXavr?hrWG^d@IN(g1 zOU)h62cE%U^1ia%O6~K_t4J>7F_j!uL)ow9r7y<6ySbj4meOF7y9)~vPFqtUUC)-_87V4Z!DegUKk z<<(j*2HSqkCmkP;Xx4h%c7G_eHR(PGpQ)F`M6pXX&vzyBh&p*E!Rmi7iS!8*_*9QP zk*v~L)luYo2HvtJig($9$0I!E9TvEXF`c!?*?!VyZZQZd3GRccUR$3gpw!0tDP%BM zV^J+X?Q_`gettSqCN1V(#eEPZ_|cTeie554oUaL!DG#DS|o6?Kv7Yp-mYRbkv+tTsWqPVi^2BXi$}V;q(^NM z7J?r5KHb1p6}vh2p->F`r?UgPb~r+(5=Fo{H9QwkbfFHlFu}UeYM017^1Ay}dZquDU=d59dsDms)>j&4Ob#PH74lBGpMoN=Q*fJ(X<{hHI zPB2o^C2804ld5@BMnK$=;zIU}53dV9Fp%9DgO;s>To& z=Dh7S-9!@Nw0t+EvD_@#{c+0DhQ0SYWx5yVUQ?p2y6y!hV}I9**tX|_T&7&_pN0uw z3zaJx94tu96*b8FI^Kpks?lMWc(g|*D8GD9ZZXdP$5D0)O~v=dKU86T?A(>x@(JRZ ziZdHqK4F;=x7h>U{Td~rt>1f`&d-nDbH^VA;Mt+}D2CABoUin{w0#32hso{(uNJQ3 zhDqGIG0v&Z)XH~0eJ;TgYdk$4#^yYS>6|T}#+U3O_~+8QF#eWiD4XsYA`pc-Xhu+B z53Vhxc)SH(-vEIlsgiB(yS;)6zJ?g;S;o~{HDQ}r6gvDu8c;(sH^D=r=kaE5h#) z`~Dh5I39cBG%WEmBnc{by2X6hU0U(`BizTH1nn0vxa|2Lt-?YgTr#?A?lK z1=O1>d(+@@KfIb#VSG?f5tt$%XZY#?3bE}kQbzV%a*O|PCP-YvuVq$I1&DHpug0bt zs{kdI!;v?2Pjb_f=;c>47)GL^`&*_!kbOLh2kN@=)OUQVGDCphP%U4!?jP%R6zz)6 z4RPsQ0{hd3^Ji?d@-q%q*WKLa9IBqk%0+mby3Q^2IJPRB64(ad>Iz5?mv!QmjvZK0 z+a)mQRSFq-C-uy?T2@a;23tmIYoVnW1F8J6>)-QDp-WzisXD|DX1ncuqxQOR>R;mc zG5DJj!36aFL)o+a=@TV=_0KU$?-@7g%jQQx_H{2jEpOy28~$F9vKjovDHqLdg<_~X z<-BR$nra+{9&sJ16I!K`{qnh2j1g=54+U?dN%p=W znP-m2*7_Mq@^{yl8}H((NJ4c=zJ8|Vw1Bd)@n3E2vBSUf5I!gd0@PLN4TquFhTlj~GO=!R58Bth$yzg+bCe!q zIEyx*2%9{=w$aN=l1v;`lK8r(A;UhJ8k}Det@4C^3Gi^@gW%2iH!Q{;%Xx48PwY;q?DEFiY+5fQG!tvjPcND&Npm4ip>acut5 z69Sc|=>8`pS_(K^LX~iz|GFy+1>0htp4>a}y?x$)9)gQH5znUcw}mT?*V(_KexH#% zu=#pL|{UFPQPne^wUteG0shc@RPG`L9_MDSnl2At-fk| zidMnx8%cIt@?ilYkAAG!`z7wJGx*>=Z#+A!j}>xegCm-VWWLM2zG)fHFHrEF@pML5 zUR$lQu;^lESKSef51ydxNZnyxYHI1-$rZ6d*FBlxChO`@%>vfb4*I>iC4QXxEgz2z zB|7ll1pi_f_daj>yW2Z>+IZ!Cb@e65kg(kYu6kjPSmSdj_~<*C4x=T_`Yg?|Go%vC zf68uFE)Q4aNpY#J(*mJBdl?>&ZhvPWwc{dd4K<9X zDe~FRmrddp2E{q&T%{@>U<#*7aT;Z)&!+R*TgqMRMISh8jv6sSeGf|aIbVrT;(G68 zFnr+=^|kys@vjz}*CsLkE^S+H?;zA#ksdG5uBkkGgWwDW+%Li18(9iT7JIFd*NfF~ zM12b4L!T5a^z!I1O^mRNuiW_1M~WM4bh}jfNBKY)+z0WD^!I41fcV# z#VlF!U#Q8#28}gS*S?yW#re+Ps{Qe>6$+d!a|v}6lC#9NWY%`bi-`ohMV;E|KQ=i9s_XKd->GH7Ig8-`Ue-LOrM-?(v@ zpFqN8)LkZfdY&_5F#GfBC0<+#b&P1sysaW?0}edgJ2sY-6_t`SW&C!477wb{L+mEvv zU$`j#ZMpSnOqMPRyN)m+EB9Cup_^h?$gl7E)_E0m72m*e9f zcz1VyrKNqErHnox`_Ft^QoKTkMu`xnN{j@OI+d?!3DOrEF)o3v9%FgO#$Mk~q#*eZ zM}eM7Mx49~WAv&L$Lmdp@8`v>%k}ecU;+9~?sug!{s%U0cz#e-{Cbz52Gf}M^Mp1sd&egsuIAB`U?k*h|C+jG}i&VJXD z3p-}7G>y>L`iWpDxp}kpth~rJj;8;$%X8cWamonA=rrs6eRkVHHBOPSXyJgtwampn z%HIkpG>+SkHrywE>?}Wdc@K8OPkw$6s@uwU@2WcSfMHNo#glNbMGLO5ttQbTD?~kp zb3=$P;Z~4L&Nk+$a^7Ni{^nABRu;XP?zFU7`KNLxBHQrelUb0Zj9|aZRpo8!T+jJ_3@r0m~Z~5)_9JB|ZTybNAtcoRBlGU$L6bWKV1aNcA%}z)K{{JA}F^uf};W%JKgbNFw4!f52Md} z<`Ppw$|Bs8)7c41=+FD6_gwDedrl5CqQ`#MYWO-b_zfEO0w`>!BY00SWzZW67l8%b|yYR=OPnYSN z4(={1303@J33d`ShD>1H?7+ATLhgHcw1G2Q?wIBsV`FxEb&;9K)+brY)U>qyF3DME zXPy-G2aZL#JoG+K5ORlP?#%z%s1gq(%6*4c+=`0)%EqkkxYlqkUq9T#UtOnWW=cC-)qO>D&sB zm=|MXa$=;#s%zAIf2xZk61`KC^`1>R#7+FXjIg}3xj8jTGpO41R&g3q6=ClAuYMYO z%gHU=zbC1nFj(ey?tcOvSMEJ*?L#jD5%%HCXaQRiIAK~NsL-m3_f=$}aqEz;ss<6q zm*efQ_O9;PAaS(^??^TVUC8v=KV_|{2E7p3*jSTbQwQarOMqQ@dggla1}R|+;jE!T zS?a55w@Vjigy9UF@8y@^NTKEAeyN}rd<%EZp(fsEgCp|(vpbqMl&!#o^GH{Uk&uuo zB-0yob>Pi4DgC98(t=HYRKOnxtoG!&w`R`v&gr3Yb(HZn4`PSiiMJ;>7YebC8XvFH92k`Hn!?d#tKQv2AC zfzCzJooi)Gv*-CX=7-G3NCjjhhlbU|F?x*F=A{&@&h*1CUV(+*VrrDSuUyIdJ@}Wq zeR~^uP*7GJ*}c;;($Ev+cbwgo#9My+t~@nkSibnj=4pWiaW@ujDjzAs7yh6uOZHoM zBMoGm8_oBFBEBSmhm)&DVlDkg{N!Y%_QVe6#T+;`z_hyuN5-|C6?S{U6Na1jnec6F zGY>awDme3yWu6?aIk}A9c#s{$q{zNHU^(K^afY!uoRs=tDeq zNv2aj%MZR}^1OrV0A79R-;$lH07`pMjs5IkT3A8MbHgG4Q(3Gpo- zVLv$H+TM$#*ou5yl7`@{O~db0Q9%ofR9QvriSL2=4R=?^y$p)Q0(+mJZ4Es`OOy8o z*7OL|i1S3<*D^iubGz3k zl&fKf&`dtb;H;p+;p%18-B)1mC%aE~Pe|_z-QamYdL>DI-;`+0zQUF_jQoW=x5v`S zG%XKL`_2OOlq6=wbeL(v=K)Jl_Mp$w0w{jZDEz~mk~9&OdPBbtH{Fe#tdF0 zU2=OUB_^yo6y_`TFNEs}P1Gp;#|%-R(eOE(5MBQ>>nb?!z>7YAC&oWqbG)6SQx^x%wDsN*ILi^|~{Y zQ-(fTHN~a8R67tLqP_6^QWODkvCDPM&em zLv$zBW#!lP+?uYi1?2!Xc|+#a094&^f(CCw`+fa|a=}xTYEW}nzDKcKax-^C)*ty} zu(5sro{D%sZ8LTzeGVr4XJ?)+r>c5q1Zqp;O$VVkYoSYh<=mk)J)z9wj;bWBg5~|W z8`qF8`ZHe;$HxBH3YgNX41E;j)gDT+@_Oxgy#CYF%9vEs4})uJotB2K8ozYxI7;Bg zaxU&zsC%xKCP^k&@QL5zrN@e(B~X?z9+}dsv4@DIdJ5hKUDKQYh9J`b*O$!q3Um&2 zJ}cc4r7)Tf5UkzgNQ}n5nXPaI&dARIrLSw8y_BMGs+(&=?(%I<6F2+rc_vl9##0%G z^4(^_hK8NWecuD=UP~gnXU%Jq|2VE21Y9G#1)(&H;aSBW5*Z4i8pPkKCk}12%O{sq z^tG7@a#na}zmQ9Jb_PTJe5EL5!oh~9J8$lSlPs{xu@%+%H)MVQ6V|%#nacHI0NCye z>pcUE*!vmjQ>Bb+OGO8@nUNiG+b76rBbA-Rs@n-!xHt_IJ4bR$lofT-FUny`u}07)#0Xi_2l%aK2kI3Nh2t9?{_#{xN$AbDoKEIc zHqcaLD*AGtI%U4Z%|yNxVL-&pm0f=sgD`iCI`=a#vnLRmH)RwUeU@fO0$~Q12sqMN zaVFke>QAGr-mFh_u(!|f;}Z!iFhPaO{@g1&J~TVwu{Cc_$ByIW9f}Qh(TTw%uX z^UhqN2wxjT@9l6=;lCa)8I?mIrekchUo61*SI)2dT_*GN4uKvIk|OL`E;Ej+Ls?l{ z*(ZD>FXtLHedN&B<3d@plVA^^r~FKH=gqKsyn2;<P-LnVU-5AIuDI_*YZW^G zG+ekPC0Rp3?0PZRk#}BiiLE5Hs{TD(E;nMT6b)dDx6-D5)c$uw>TRfYZp<_Q5~o+n zcxMAdwEa?5^H(1t*G0i{OmVIbQ)H`5qhr%o z?$HfJk`-usQ30@~t?IJYLstF^yH;OV?x_d|1UrELmS?22pWVJ-Rkh7H&h|a(d1Oz3`{BpXd{tc{h8kV|17tbQpK&KW%J-w~kmie}iONeEbS=l; za?|ZXS~+~~%p130tU_+t@7WtkL_q`ja?T9T@*AwRDZI|lHzwV^Nf*oip7(YgAoIRe zGiV|wuaN}XGS|7^k$f{mGH{deJ;ee|)k+Bd=B+<+*YUMPmctd_IyMuKybHl_*G5#E z?5k5u+$lp zoD2OU<`dVh1z-AM!ul37ANlv3>K1;w()@N>-x<=#mXoR`B7((Jlt<)x1Wqw!SiQi! zHM0dKY4LpSnD>G$275PmmcK~Lu2@FPym6$kN?YR^|o^j|F(x!g$&{^QB*3`10#wUp~kk?zm#7^T< zXtvqz8RgcfBhn}$sN(h`VTqEJ6u;)@ z;EPZbyF3rUBCnRWSHMGZaHxKcj)+Q3^tgSW?~8OQ%iO{a_ZWexA3nzWIz4YrlIp^Y zHQvI#z0=(xKxvU`*+?$F9$rw2`Y5DL(!Xw#5%O9R`v$LaaF*qAb!f@6Hbpg#vva^p z2oSsK+xk-)XZ6Z*kjQ zsAH4*mAhtS1Ja*Ft^j_@Sn_S?LS!n{hvjOfc(@Lj2qQ1A<>uatowf&z%{aKc)PsPe zFnS(w@r1zFa8*_^Xlq+aPQogjP3u!c0=&b0psLoq;zB$r`C_1LN`X=t{SH-=T?L!h zfek~6vW>D-Uxd-uEPxM%^1o2}j@oFS3cjJhl%dk*zyXoC>+soU)XS1ASi>%De?H-h zczahD&L5t@4S(%ZpKr+ZSUeRQEPJ0SI;;zWGo~WIF7`QbgNvvl>z>|CfMHT2Mzwhyad9Brt-upf5_|Z72cyQdyDU;FnpVm;akGxYG{C z3IDYKGWX%Xf5}+!M#{j$P6mC3tnn8O7z-&8km6!-CNhpCvZl4awSQ{z=d)(`iZ1l` zTU@tUmTdf^2Mg5bHb^VK)iE5QOWbMCbotch%1^nhNC3C-!BU9BwWV2E{>R6j@joKPB3C3Sv3&7OJYK-mhd|Fm z^V=VL3BG{E=78y95iuf@dg0Wr$@oE14m~)A70CphJ1$?ZUrb8`o5u#>jpvaUgTHv* z$z&sXA*xjCQ@AYpaexJ04A#-0Ljhar1OD9TM>2p9OEU4~XI|Gd1muo5+_~BQ*oRkH z?0@SZX|q9Si4%kVO5hLruGMNL^t6&11D&Y&h-+@F%p+Ky_c%p>`e`fCDrld2e?!5F zSNYeQ4`2V26nl`A+x-^ohToJfMvMS@fj2@zpQNHdlnE%C=wVzXfN>oI=Em^Hw7-@? zc3+SZxx~rR0E!WRj_8vqugNu;9a!K;7t&276PnvjD=hH|u5W`LtFOn=ageoB!4KRH z!Ae@GH$gn@7rJ5|T?=FBFtUeLTbIhBIzo5jVhcq)7M9p)k7vBvR&oH}-ZAO~{TA^Y zJ-{p34pL~b7YFY6b}DlOLF!ud5Xk$l>!n%+qZ@AV58}j1X0$(rqqh-}=F!oSiMhFP z=@?qD!Hju2)0luOgx+zE!SUT@U1mHRkE3+tvjeXLX#BS^Ya`gSvBSsEW8O*c&-qpt zX>#%1{M^UrP1vfgp>ZFDumyVHhah~0=w6Ct^uKS@{%A@ zprsl8)ZYN15m!~U=vhFUas!M|Qh=v4^Yv)SK{4KYv>b|n{JDI6{j%>iV8dVuz4G!p z#_6SKpuLNk{3Z4f-8Azub37mkVbClzFs~P%WB#AK5}_p=Een_0(NDCofzBnxjVFU? z*^$HkZ(3f#4R~D%95?fk3F~t4-=QNIO>*&s=-nPKOGAn)QNm>`5>={=&F!G+gPQsKZ%77(@60H%H_gw`f6UI^og8|_J&G4g!9<+;MbYo`Cw8}V#j}ATcr><4 z^SyJ<(&_kAo_TZ@=G*`R`C8jUMd zHHycLc;Pj}idJQA93P*)L1Rqp+3yk24!j{?&hY-lZ=c!1 z>oHkAK1+*-mxoa|A3O+;HpQf1Z3fMSD>LFMl7^p`2gg3*D;R+<`&_9?XSazk-b~w@ zu(Ps-7|hQP89n*5O?ofrb}i<+7DYL)n%hw1B>u;+g4XNVbxJYmn*mMh6ID@NU9ue= z9g%fBci|dP3Jq!_Ok!PK@fcCHkcW*{{0Bvuc`f)slXjS#de1N~w=mMS5_v`#send^ zDO>|YGjyZ6P^LGO8hSB*Js`v&qja!MfIGRxp;hvnlK#`SmU{ke4;L4rQlp)n;Pp>? zM{sOW61<4uT!!7)DnW?-;D@+*Hp@apyk!^b139zO?doD)JUKQyy9!Klm~VU#w=n40 z3vL@spk8kIBdxB!V;1@36U#Rv)(1^|#Vtm6JYYPR!F~X-l%T&eCiwNvO>X5hd;caw z%!kF*ccR+c-nPYM!iNbWE}M6e6829JhI=V6kvxYb0gEN^ikI*nMYWI;eKrRL?C z@J2rQ&=Y6;dc@j<$GiS?&`>30(31V$+6oJ{9AivGyc88IUR;-+6z&b)P*rZ5z1ih8 zHu?1oU3O3Qfq?`2q9%;0D%LL6pF_eoIoD3kpm6vruj#a(#G1wQ`OHW^_s-QjYN`YT z>}8I{4bA-#YGzRiZ$qil)=v0B@~l?zG;{L$=_cn{?22=1=i*cp{~XtSo30|)Z^Xiu zrH^kJn#&D%o@-g%a-6(oxn(J#rfR0qmBr9tBb#!akf8dwX7s!9mD`L?s$t2RyiuRh zKT-LTg^e`Tpr)JcN=prnFOwpuv=xl!H$&LnIoa~P?-72GIPf&`E*`V_e=J=ER8?!! zwh)n0q*J7&yFpMwI;2Zly1Nubln&_z=?3W%kZuqG>6Y%Uf6o2>yO!&^@SM%w@1A&O z=9!t{V2-bp|9G&PQy<&@TIUI=+&XP^QW1}8Gce*kzX*QcH6J#?K=DE(Eu-qAu;2v^ zAwQvre;}St??BVOLofA#2-(P=>2?zN>E`4WLGWtU7gJW>R%1ffQeTQ?7FC4kU0=nV z`Zj(FlJNXuh{FM{;;5OMFT@nSkhJo_aWp? z3q1U^iW>d8TPQpPS-s)H&*|wERd7^7{9nUL#`#~Gi5?B_@}xSF1wVNTt~&JB{e4O6&@Bt929-R zqz_@6&J+{3X&-;!Tb^p+e+qA14y?_!%)9z~SAONvP~W5Em7^qNWqU3G zJx`k0>EcI>qJB5bOS~N`p#PS&Vpq8CxX^(!?=wf_7@afiTI~K z+T6*`ZzM(POyOcAuO4(FIa%tWr5^c*S^8pukFktp>PwksL88EhQFMLzrwXkx`@Qr) zf3(t^91bVWJ@2Cn8Mkc zv1F6KOJ>AsZp5LCK0%+4?*}*p)~qC2#OiPny`T1QDPKyWvdo?K75+!V%^p*}lGJu- zcHnll74y|xv50q-aEi%A;L(s{aNkjaMx#D&9ihuS>bay#mV@@Pop$w{Xg6wzEMs@k zicbA?^M+;q?B|XRSog4CUmL3_5QU1X{j+yC?qwFw^8nxM+17PT)pC;_!g_D!iM0vD z8}3i=A%(w6{IGiX(5q2sX8iEJMP)%+VIBJW^u%ZVu^#ucNn8lcDmhRD+IeJ0a*B&- zu)eZ*ojc}kNZvu=!=NJOD!W%pn({24RQ*+69xb)9fJT1a^MHvUHrH6=-4FXe=jIfI zi7pTB`efy$nti(Rc9dSD!hf_%3Cq~|FTdou`+LSId z;9Sm?3%{KY^4t5F+BecZlAp3tEwxxNd%bI=Fn9L=Ig$57CWM(kxq|=CzCX?NPq1Ls z>{Hb`)?h0%>Wmo!O&AR2_SZ^d9>+7nCB8W|zW zWW7wuLB84}JvA*5%lKWaC$A@s8X3YLQg1eLJgcYpZ>ros3sdN4a=6H7Znl9P-=c8n zy^otvpc3^KR>@BK%+62r{vD5ugcGMnm37~!CS#Zc~T?$f)*rT90; z7s2iNSdKyXb))?Yynz~(-VHubjmw)HW`4s2_{+h0a)l(DmBC21fLC_g8-KKj!H1LeNqM78;Aym>S8 z_Z-(n&;>8e_KS_qF(Z?v#GLd8XoVt zX9rC;GVp@+cy>^=-Oi(yjFujSe4ZGs4s2>XRO8zE`^&*bpyR8u{4bNgF1J&299{W# zk*_=#hiz>4S_19Hi7ksJdi8WqQS-TrsD%+e(4Cma zi}(slodMUwhg$2Ewl~rz2bV+xk@LNCwe-bA&hI3H(94dc_FoU$vD8}2groZfkq3m1 zYfE#+?Rp5jfz=j6lAosX@=g^3NrQ(EuP+d<&T?%HTPERiUHz1+l}cuDMb!G3UPpLb z*KhXzTVmt_EI&h*dvZNiNz&(8LJAmUOxV2&3pc0B{8li9m#TZ1X|l`gedZpc#vANN ztmK3eldQLyTlxH^B3;Bg?=t2?wX$~Nve zQJdcsJ@@po{9FFD{aF0m>uP)i8U69|-HCFco(eo2qFH&<0YReju3jJE-Pr~9re(5(#`#`ur}6Ru0Qn};Jqf+cE+okaiM2!x?;m)st(i0EDzzU z#>#z(NNY*YyabHU`?G`hb^i03sHkyh0W&jy_V0-Bc2N4G2|Hno?&GB?06*Zj}|L&m|{#H9a*~1 zBh{0TT->D+W4)7YJFfm+E(l+CEH+U%Zt;1jq0N{s*QjN{4Z|ap-9IZ;owg<=w(TwC zp(={GR)xk|S6usL2i0$zk;dOF<_Y1d8%>n+?EE@-;fK!8J-D+oD+~kiyX=ka_BSJ0 z^`A*$izz-36Rj`bA{m^oKU-A{Q+e@?mXD8+kl+~BIv8?xz+%zo=vL^Kh0{{bs$6gy zkLt|j$D`4-?yvRXE6vF$IMOYvctc z^wguTc(CMR*D}@!-89<0+45IhV-(SYpJXR>ZU#&^9yylQm>h`c=}DMQS31FvaNIua zUy0qkd*u1*`rA`^HNCm9FSD}`!Um|?KdTRte~_@3%vyX4nc&wO=R@3giDbsQ5;?y9 zn9{z_$G_+7_;#TCPHIyFiJl8LsD!8xS#rDhq2#?bqi2KR$}bn&gL~`ee6J<}>S=`7p3=^~=h5V9YtMC% zGX^(NFkKccjjQ83YLuB)_(7I4Ay%2yA5go#hu9D2iedgi>1g`Ej%Z==c)?OvMU^*z zGdLf{di>ox^swa*dQwreVU%qex!fHfEH0Afk{a`8TNOJ}boqNX`G*f7a$o?r9bH-$ z|JF|a+dH1KuXQyTeNs(aL_RsZV)jxbt-LC-sFA5GQA0xKAS^EWwX-JI-L%`zRhwC( zBT^PF^nFQv96vZq^6l^-uhm$%xAn(qzkHvrn@Q?Yhz}$GW0^18%*-Lzk=4F@c{#V*eVo*PqcIr?Fo)3C~Irrp=ToD1D%d_ z`*3BA*Ye6wbex=)g5EPZu=G#$3K6DYl^?daWY!@%zfVwUlAWOW{iH!RcuZg zX-`v{Yw|>vL{^3_vE^cPXJmA&4cmz95UYmWgr7uIrNNAwD)r+u-90M6VkavP_LMk-v-N zDWcc%ZLV1~{Lyuo@XSvJ-1o$iirCsrc#4eAQT>AL{+Ep+mML&LW96x)rG5fd<9Crx zV0PMvcPwAh^BhM<^(!Z6gk;jZ&9ROpG65swp4G)~d zVSW8o^RL^(2M47*#HRwJE#6Ib!s9nW91s*d&Pr2EROHhCOGNctJP?sBgOO=JD7+mX z!w?hw)RJ~eNd54kNjZVe7#%(B;(2~i$FePY)lg1Az=Tm2Pt|EoIq~!78~I}gg~w9 zJaPRT47Fr$s?}uI?Gl>r>S5uL;UCkxO-&gfV^{+rExr!7+~CDJv(UCfjkJ`liRsA5 zluT+d=BQ;>b`61=E-3*LQZT|h${#Vr=W{25NQ1FZ2$2=b`%E~9 z_-!*^>2z3@>$4?%{Ve)#D5rF*^eM%0@~ETi$+z*6s_zwZ|1nf`klZyp{p8XZv`p!FCqG9PTFtqW{sFtnEWwktCACJ?+ zg2$_W4dp~p;oX*XX2)dYqa^?vZ>uIyKHYtTtD>aB;^-W}yp+*5A|6k%l}h0M62(Fv z0ddAB8)Mkz1tQ73_9T{!SM*hn2TpUX3qml9DJ`qg2e}6&imO~CF@tS6h{>?%`su}0 zev?mT85&9LvmFGDDZRm!(PFWDUsh$x46{rN4huqm1Qba!B9buJ5u{R^udeLY_>OyJ z%A}=H4v&WFT$sH)PO$!MQRRQXXW`DL5lKVzp|cT9!bOK9BnYjnr~B9{z}K7$~!B^~1Bb5gm1!A_4S;0KCy0Lh;C>QF|BcM#1{%UA&L64IoS^J1mujYz=>S#*mUdxBp^DGp0 zG^Ea9o||pIyZ^R12npL5CyiF>lGE2c%*CZXuxRaFZ8S@px*|5f_whI7C{Hdnih5yo zRdr;clNQjXOr6g*U8_d!=H~P!uwk|plvq#l0b0EMW%~j%wW2DD>}IRObFts11cECR zzIL9#Hs3yzi&J?@AuI@zOl&_d;@>09YZJ9r!Kuv}0=dgBeauM6>HAQOygcWDhY589 zBjb;qzc|IyK*$zSo;etGm}lCudEj*P#$)nr)b6xZ800xXm*&OqK*SuPTvb zNFJp7;ib_&yivc#jBc#rXk<*2{4z>bw30%9^V-FSTR9*bjm}*#Q9Y|UY5V?ktlZS{ zRsiy=t84FMuPY5*_Y10@>Tj5_f`7@M2c$njyYKd{>t%h_YJ=+wrG~2brp8k>zFK~s z#Z&fJMLr7B7igw?iEX`1DSgRVf|qBrK3Q3X?KzV#(h94B{bK}rKPM5D#4^alaX_;y zYti@bpVmgQ4bOR=-1{X#-Aa#~K`VZHtyD8wcdUzvz7;~8jz$f-${!qM0fE6XlH&K{ z{QQ2Xsg0EA-yH&8XWnxP+i@2bn77&vE*;(rxF)xeY$Ct3i%M?72ZOa%M4FVMs&=fE z%}!*c^siK+elFsVIFZPX1mVaam*(KgB{@7tq_1x8+%oMKj9jQ+_(c9tcI3rm%C?Ha z>0w+-R%2nZ+KiiDiqNIv@_t*)!PmAFN!NxPT#{i+v|}Xp z39%{LQS_2U!UiOZG<SX`S^Kf8nJ z*Y=L_MSO9M)fSnYxpz!dCm7&=?ISzQi$|$T^r=K5l5(mT%?e>!H}3QxX{`p8C7I5s zM}up;gPOF{NJKlBc1R$+jg||v0~GU;mlYoE_knsGSs5F%$ycuWiA`4 zn*;>k5^e7UOw&R%LNi%Yk*9RD>3p8!ds z&+yr}Oy%ioX%&^{0nZ0MIo5e5($lyM)TqmT%i6ZX zNZNMmIMqqk930r#+8B7{JtpkDK5V$<#+{?pzAatsaw{P=v(5QCjon%CvaVOUD>!|N zEs;#W^j&OwNHZKvH~0AXCd?)-=hn$s@-1<}oN>yv98Iz$^nP23v7fm-I9C7Y9W07}e@`!ZD(gWgm$?|Nw>&t_)-*G7hL%m{q zemmcBY9EmWr%Q|V?K+bD{hL}tkSJ%UaXO##e2d?Cx1S>PeY)+RVD1QAb=Sf8!1Xa* z!-GKP^bJ1te*Vyj7#-Z57lac3&a%v*;p)og!aRXT0&D*+==C7 zc+uyRe|I^!3`!^C?3$6@FzF^%cWTRrK9-D%BG@RFvGu;RH05Xfp&&`V68gyJ;-hz7 zLO#m9yZ0zWph#w2O2hChbAzwhmPH8Z@F(uy4+^(rM@lpcQ0Rx-pKo4xUHx$TyNl#C zUg!*Ir9HauABAfb8Kqc=U#YyY#GMSIOKJ`)MR!gZnu947X={Wgkp%vI%~^s8ry~wN z>FFiDL_$scx{Mn!(q3!WFX z?1GIeGL{zlner?!eBibSjUK==$(y5=_qHQcjp&kRZDS%;{lD1}&^)6(nfXRqF6M!l9Xpk}w5 zJ5?W`3DKs+f~=rHg{JYk0~R7A!bU1kU3^7?K_>8T(5cs%Kd!9}QqWzNLPjQ(Re1qm z7Jo?odUwwG&y6i0pfiCl>ch5mfXRRy8qe`GO+)JYGc`Tg=~|~B)|29^$%5^^He5tv z<)+Gw?;f9xWKrEGF1;qk@c;2b_%pT`t1p@umF$E|1nG*}&JTXf^YU@*(T_Sj{FWFG z%TO2Mkafl|-Dgj~{kiK?^m8A*$Ia;~Kaq2Xp-5=MT-#Yq;O?WZ;^W$N1cK?xnhNGJ zs3g>e^Ci5UE~{iimBfc$q$>@$CU_VZm(KZKor@#H7bi8-<}@@T2NW3s7HGz6`ch=pE)zAMEHP$hV#>ae9+<;-<#^f9 z@GkrTI2y2Cu?2t9LIYnN7k{G{>0mXi`WwgUCSJz9Sn}qLW}TMXcJL~qgkwtTV*+Vg zyu-bAf148h)%&o;C`Di^vNu1L#tGH`X)C><7AH)ge4JCrj;BQdjJB^`RY_}lP ziK3;b(kr2!_?soK=>%ARJ9)MrjsWY5UHLdmqK4Tj#@`4Wmd(&`gK-%cj(RbyGQTbr?wido%(l z^fUWMdilBtht-rG-!)Wl*!DQg?VKn)2-3z&MQllF_A(Ohro9`(t?wjS8h^HgTy+BQ zPYgbE;ACq}-`oooRSAyt5cv>(2K7|yE?Opd zG~V>mS5?JWO;!%g&ZYocB8+lmPccsB!h*WjB%_@Ii&gG4qf}hl zQWmFp%=M~Xz)(_*4GzUl0W4d}fCS|;R3d61j*9I0AhS8Q&a?Q2Tzeau7q;qS$B_mSP}W)-DP3M) z(a)@`5Q@l)f9_QR{uaektHDSRK*Kf@cHdjsBlzZP8|?0Gr=H^P-KHvbVA@3A01iloaKs|UN2Ly>2UW0U=xsf0-M#hn=PZKNJ!U2Jv)AR#A1E;5X z=2HeDUy6u6h(nPbzciuC=pLTj>b29`O=mMJm?b9h`fn;t46rj0d`8Udkxr%4t9Hgm zEY{&)_Pk_Y28w?b&xnV@XIG2Fu_^BIC}Tl2tkJU_P=jsh^YkMYBA@$F5F0^ z{p6T56M~WM<|eL4EBPL-P4Vh~EIbW^4}R&XIqTPHz`1!u0oY8xeBwKiSp+<{QRzFa zb^@5JwZuz|VKR^$%o**3n*nuDV2&XuMxnrOTyv!MV5es2Vy4otTn7RXa0J`=ff92I zJL1V9-N)-E859#2+TLSb2sivT>>US!Q4?n5O4S8_cny%fNs^w5q*Jcm!bTB$fOb!^ zM=G-EgCgLByA5~8iMbdm71a~%@E*SY!C=)wJumXnpi3kwDhA3j8o8$y7wA}Ys| zAuSP#Oq%2vvTo6#J2tkH(NLN6^TqquBQ)XRNvvFQulU*@CY8#n{Iuj^$7Pzs!9IuH z;2q2Ml^@}H;qsc|e;cbfYcF)gq6#Fi-o<|&SOvl#70~%8!9K_+Vi&0912t7fj4H`7 zFNg4G><>q9(J_S+pJ&Zm#=-4(fI=}cs-++Ffl z$%pI3PqMf2TcrDU&mP^n6Q&p~V(o3Ml%=3qn%+%&k`uXTwU0P$US8g6Z6?byhO3-1 z10cjJHq@#pJh*q4gM;G*{gTA~i7E0qc|K#{{{FtvPJC-4a>`2U&e7t**)3b~M46$i zxnFVnaq;KJ?)K*nor?BBHm5IgRas6F=#sPprme}2nozo&gP6n=7dDE&3;`)BV#1OO zEb~(W%3yz1rF1d(a z7|h(`kiCm^m?LN5K<{0g5)kmP%pmDQTQzX8z)}OWZX${+w<7S$o)<0dq7Md&*|@7beo;!Lr$VADd5iGGMLW)?`(O4;i)cI`aF4f53XH zY5${WO&lPe0Yp?724>f8*ST||tfvQ45c%V?s4hdw%Yx>yt0t2+`w|sosGk6I z74ShC&sifjMFoWy8X5%7b%GNKeEK!h*1)GkM@NT^l32mwir7f9BkwFMT2~Jg6%{{B zZbhT|+KH>Ez@&1@$f=x=9%qmOXUU)4!g2xkvdg=pR%&B zywK4hNlHq3`FGOP6ez%oBI4ozF_h5^*y3V2%#=KC4Mbw4=|G6U$XHFkI^m}J% zX|yxb@TqnF)BHD>ws`vWIYD3cuc1=S!Z)g_3-&coP!2tLc6-;o0YO!Q5pq&eQk3tB zGqA-7p2(iI)5GN$qJ3lNvrBWEno@vu`{|qKriHaN)AI6iLFH@00SK%qko_2jKlosS=56s+R5E#ZP>e!p;_i_c z%Cq1H$(IKOUvwB587b)M$~ik%Ug~z-4s63>ep(1mCGVq(jRaoJ$%zw2n8_pyXk-b8 z0(JD;be+qhEe|a-bIiaRE#xk{jLa@dGeO)Do2%##np^3Bi8-ubP8@(9m6@4|Zgz+p zEDr0e+Bc4*P>)ze$BsVr|1@v}R zIbFI}@Hs#=akS5$2b>(@7x}sS_%t0_ye9w1{Pfp9c-XJN5N>QF1tvjGPS{V1E`zP2 zjg5`Ws2ZN<{^n^j6BA!&cVckY>0SXh6iBcUZVfX=Hoh<5vY|VYnmH#&N33r9Mkw0K z?+Z8*z!`r*k%xaXr1~Q&$W>>&`$&fpD-9;o&5XSK{4WAfW`U|j{izy)bsa&AYX-~& zpUc;_)v7CPZEbKC(vMSl%t4}M61ajvhRo8E4c@3^*4GPoZ{lPCN}!Bf;pieS>}`__ z{O{T2W1R)7{oG+jD!drBE%$F-*K(wAX_!7f#H~$#puQ6fUpCvM3?Tr!+11s&=7=#V z+K`#)2iv#J(wL~x5Fd`NBoi$Cc=>^`z_744)2{-bt`v8jDaJyQ-peda==@pugZQl1 zA~skEN1!Aq$cHi3tzE>?CdnQr12X#yFN7%3EE0D;BfqrtamqkP5CJ=7Cy_C{D#e1Z zH)X^Ws|tm^eJ1QkNJwm1OyZ;AKNMo()5CGl?(dy>+7HFtgHM7e0L7};q?}k=93ejZ zNpf+5Pb1oaHE-Pg?PI0FY^kAtZ!)3CFfjvy)cH(BC63KF;QPscpI%*B0*Pnx!g{5Y zn~RaZWaHQu6t*)rN?$9|F;a7v}g zq#w0%FwxLTEXImlH}*aME?oThAycK1M`D7}@m4em5h8F^+JXb0-&1ir!js zY1QKoSVjvwv2qa?`PG79#uMpj>z1HsrSGDUSfCu{2ySc?%9XqM0o0(C5I31VPF9cYPa+uF@e+(TOyc}GsJa5CjE6YH$R_I% z*@P|QZ{(B1FBJ;R%*-8I&NLnl9T0pmRIt8=-VZEjqNu?Vwjb`+mf?^X5!LzpI8+9X zzow=p!dULk8LI4UfTsjtbY8yv2nzm0uE4LhXPK}}U7r>^JuB;e@fbbh@6fPO^C%gK z2oSGVY{{-hjyl;82><6HY+6o0aQyuI5az1AvaLEMDyj|0w7JE_*@cCSLKom8K#be( zff!5v(dDflf75(7{}j|WHph~SNJT|uSoF&9i+Bd~urS~T68lJiseDbzN#eh9?yLANXKyv%hK6A5VC3>AT`rJ6PjkX+w@qz^3*BUD#JgV4ZA^!K;V+oDa1dV!zZOD26% z>gT@cN*@OXWv}0n&gkvi_euON?RBVf*x)+=LRH$#ChyLG<;c&=i{-M4zeg^hA@1Yaw6e5^#dTyDuPV}_mT*Mc5S+IzzF;v8yguK0=wpik0dcH z+5nyvfFueC6fnM_--0qgB}V>}JCd0((<}Qn&`k3jI93~26$LU4{B$|F{%T?B+`NEM zbCQ!Cb{v>pC$4STt*Z?E;BJo*XDU89hr136@lrEqXR3yM#?wZiO2MTLmd(l4biM}S zSOjhK4AB6M7UEZ!E2=~R)PNDWpRLkn1_yt_f7RQ8hFl5854|Hgb>-9d1d?gtZY8bY zIb~&K0Z2%qI!I9$@O?aakLaE=G6qBGArgM4vhql^bsFi{SAb-OQz%iE(87QY z7ZP}wjJP>ZmV`b4dw_S^olu>i%|!rVt;NOfvzzqdkF=u07RtYWH39Oy|E0PD-v3)v z)N4~k^7dE?tF+{(C{Ai->h=Y$P&|Zq!kL%1X@D$5N8doo-Ch7uu{xVRx_90iY_ zsDJ>tyYm5%MrqB012c3Qe-m9PpImwr^GwMXaXSXk69L{YGCG>e67>jvbG`0PtW_G* z|C>b)S(rp8Lp>5Jc|uYYOam83 zr=0cOa~ffH3=D{-LL(W{g9V2goD&5x{Xh zJz+4L>9l*QE8r2jW@I2};D3X9BuU9;qM0Kt`ya-5l{*7ToWkC~z`%}Pm0(1_PlWQ(ii1Ksi7Dda zWqLIO_LAG5}UkN=3~_ zZEtUrBY#z=Kk)Uv1AtuFTdMfcGfwbG?v~YU;3T$A@gDk(*ErCCBan&bpqr?(6oqb# zuMs4Upp1K^-(UP>P#Q@C_;h?+2F+{K^2v!maMyv+z{2uW`6}`SLgfqD5`umL^nVZC zTM!5M|6l`s-D^wyB$>MLD44LytH*bv)Mumi-m_=h3XH#7=WU0zayr;*|3jj$+}zxZ zOiV)79oclo);2d~Vi#~P8bK6L3~r0_prwWFaT1Fkopl~wlp=k+HI9Q+QHY)>SE3%2 zFlz3}8AtgjZq9)5HxUuhM4gLGHvpDY3SMlTFljY{jtoNXasosG*O8FBZu-HCU!S6+`tE;heoE$jN)EyF8x<8Q zk1-kOI*fe^v$YYfsIE>6kmJ1AGZvOxS6A(7aNXqe&4IVEeW`h|Z=8JYzZ_d^eh~h} zZexSJ(T3||>un~y@7XUk__8{7k0WJWrFD9^ybqRH zJZ=gVTSW2D(2xkYd7%^pMor^DhhCC3);kzYv!{)HmBNxde>c?_ZlT-{UOjeCtHNVr zd&)lYK$J3riyYlLZDx2_6yn5pX{4wE;%pm){21o((VNq(^I(oBYl}LvLse8&LnM)0 zSn!G2LI^1edy(FvB^@5(a~F?w>VD58xZTmgJC~oQIHq4gIDm&n826-Uwlm$xn4pUC z#NttxT@LQLP>RhP&72-PW?qCP1&HeI;UPi!cN^RzjLl9n5IE*!W)3YYV>2@r3SX-Y zAN{v|KeVG$3YtF@I!haC->59QK^Jc4S7YTnw>$f}KI<&qW0@y@jp|f9CT(m?MLJ2mz6fZD2FZBOdN?$4Hz}z-%)jl9^-1TLW=`JYapdKR(EP_@#E30;t*=4rR zf)QOivt*ap*!ZoC5qq(Nm)^ooPEHUwLM)m5tzF+>2|A7Yk?!0EDD?d?4WAA-8!>!$ z5K?vP*mn2tv|~>~uMn(&%WOCF4Serw&b+8R00=&D}E8 zGVm*;j#M;!CTK{_0eH~Kbn)AzsvrK&o!kG`*#g83+21Gyx9a~+ID|#>X~|L_!XGWf zhBYgbx0jF0z@{vo@a%n#HFXNM(2j8vvnk47)rhXS#fhKq?e2uW3Thd8Qxv6uZAe7% z*f0T^QIhr(J{fe68q^PS7)rKnaDG;4Z^5sbu(Y%&uyUu2*n~O5#4io!XemGOe3N|o z0%j04prhYA;rvuyt8HXt9DZBfP`0N1p?_ALy#C-I|TqNQ&2pyg0MG12KrihDh(VXFAgIkM$0{Z|8#h=Hmm z|BlyHzV;<>ms?NQwmLr9Vs(2I@3hE_6~1($XniAt_nLkIgu#F{En34`$8uD`n$A2-m z(0K79=I|Ks*Sv-N999>m#i=o8*rkghfWSUY;HBE`L*ou$AD9#49NH-Qdv3NNcF~ zAqC`CI(vE+^={7bvJ>sQOHQV%?I^}{Tu5LU0?aegGuc~cqa$-H(5{J?w60?mO3@y3 zN*|)FEdmI)xPc78fSkbID>yJRN_}Uts2{HM&Q}lk zBp;@ypR#=X^XIe8xnm`hp4>r^&YPxVLSBfhrnXD%B0>t4Hcp6|J8V;k#KNOQF=60w zG?8`n?c%YX0*!Llv*j2%dU~ScM(JpV`N`R9t`9HXr@(tQeD)8qOWbX@8M#!Q;pg=t zGxYIG@m#~I=#-@&BXazkyj1vT_5*?^CE!-&;P~Kqh`6nnJZ3LQg>TLVLfr<3CMLS3 zVR#V*3~dICg5iI+d_v~wqo-i7Z0AL->$bQM@@Lo>)GkRll=^ba2|9{vi$?zZY447r z$0T$tP%rhZtmOLnPn;W3XiYa&MsEi;_Hkk$WmjMpAClIridNn z00R0g>OiXN2J%hkj3SChFHr9f;j~rX%vs?f0zlI;bO%om+Ee=^fFc!~zZZ2kPj{jP~HtPSj z$|wDpZ5<-qiw_xR7QjahIn-)L6?qZUNiZ z{dpZ$e%+8@_-m9Ip^T1>dM-RkdE4xZM0VyyE8&X6?b>y2SULcaGGhDqlg5+tjbewL zh7%=4#pbcb;vuBa`uc6o;8=%oME>G}<} zCn=^Z5%)rGf7aV`$7AaWQAAtZ-P@Z?cy1$EiY#7yE8~V3b6fvXoi0NW zF=cFw-52jC$nNmZ5Zmt^?W>OMX~#EeiyLbqw}ZPS98?MHCM%@xk2fbAI#;DK z7k6^Ale^&$;f)KV=0gScA9#6rd2TzT7((s`*-K7#1S6j0OM@y~IkL~i8r^q7MQLOGcLy99}QZM!xcfs4)Zm(O|?Xn#%GH1&ZhnORs+u(L;k(gr*9^U@?= zL-HnyUTt`0raxUQ(DnLk)y&dz30BK51<)7$G_M$*oa~99k_Is(0izFgGg+Fs;oHfq zPYPUfIYy7vhKLkiH2gEYS`i}24S8VO_1tXx?V8mH@38a=NlL6xSBh4dqm8vSeK5ZT zdQHi&+6Bg;K%PX~wi&56Sg_+J3OYHPAbN!M}v%adosYR_=Lb zAD@t5SECJeh&tOp+Xn1vvK#Nyl2?}nr9GKDtop~KF-EwyRe_w0G1Y&V)ugPBz53})- z4_ybV>>Cl%B{kY8%`bXZ`Yb(;RL&#G=Oz>uOm~ClmAR4Zaxp z_*h3}rnK4LYM?^4EjaoXjg%RqI>8LE<=kwxqs?;GNz1*TIaQf{vdr}_)O1MlCWEQi zBSQ+c%ygTQ>*LrXujuigaL6Kr0-~O;+%v8p0k(Gc^gP)<0L^*=cHtL|5Fqer*nsWc zfH>g#qd!`Zk_=0Mh0v?XJuq9jbQ{(6?uT2yW!RULMHQkpO^RFTd->?B;DzPoLoEA>woZ7eM4=~WBnS}}#c1Dvg!?gz(H{VzT6^X3y^JN^6j_H2LI za=hd={0l;Rknpr^2N!SNLcywjznk82wvpsL_-nf^su;?9n8+iLCP8=cpT`83dJ;e!&xM)neVgzZDh`uJ$JKMgm;& z?DRmtcL#AShzA0=GiA+fn(Rhxn&kB73j-G4^Kvx?T-4PRA0g)Y-8=9Ak`MS`EHd0H zLfipR9UP9x(+D_&5DQ<LQSqEG-R}W~%uFWk>6pW!z+sZC-nOd&r8d_~Gf!b8*q)IjX= zsIIOKq1x9MJF7DuJIi2K?gt5<+Vb@7c%Se)Z~Ya6auNW2e=_e__5lIJwG;C4@;)}I zvfwX)&0p}jIq`w0A|e6{;v)nWr~zA5vlzmU;f7=T9Z-MOtT#X84@|()f8W}0u5sSd zw631N9}pNA$8AHZoF#Mn?aB7^l=GxjqSGG@ZKqKc2s2=4yYTL+@C|{A3LO(u77!;c zcA8Qg1_Wb-ulGXtIs#tdW=EE>_orqTg+!msWEAIq1)1+E%6rBxNa8=>&q;9?s`}(# z^~-Pqx?agUu+Q|%WhL9)&qT$2Tc){|m#aUSU#tY?sq)a#MAtR#@gw?&Ngl@~YkqhS zj!%@`y6I$+g~>d`?_VdAuKy{_&|ehiYS_Jl+IT7k-r&U;kl#1e8x#Ll%~F@=e91LU z!oif{voIg8FvsCnEBa+w#eUSwE~5Crt-(Z+ybbj-yTFj>~p#I^0VsS$;q?5P#-YA0v{*0 zsD*`78az(yVYR|u15EbUvT4BaZ~ZW^x_AQv=OqA2Lugrlbv(KE5IodepKl=W zEEINNhDWm3vBpY&64;>xUT_49e#uZMy&Sl?#FQ60ql82bWN{IC4O0d?s+z8>yiZ%n z+Cw=d)<0qj&1EH68;oShLP8A^c2L?~Y`oYugm){j`}1z^6kg}^08ywX1N>&MbJ@v) zjmof*j?{JTc2VQGxpmXk@k?A=CQv!J=_tshK*9!uhbGz87a?E>;e}^B2%9_ySuFK9 zu`a4ze-4BSufYo#vLbY~CVzl{#$fdkB2=JCk12e~0$_LrT+T|xm~QKu&Kg>pM{oB}BRBJlN!4y1v1dHU-)1N^Or{?DR;%px#l@9n^ACOZ#AsfqC z3I&4+&jJaNQh+p&C|nTU0OQjZfRpHbz6N=JUeet5qH{8zOS|prMGQINE5~RCjTV4A z8a4J*5CV*ybqHTK3!Y4U%re`%fCQGJj!vJB%XHfa%q0Z_7}PpHhfuS<4zg8;lndf~ z)k1^OJQabxUr$c2yq!14!^3pkz)zk|?|65Be)rNB*Z+5U5Uknj<0`lPE*b<7;IO3h z$~o~thyZBpK4PdS7!HUTlyrTroUSjLZW18DesI!slLVgDhYYIR>+9x-dsZ#dhPi0@ z)qCwk8jYU3E>m{MX$tQ;H-3>`?D*Uy=v}R=^(OJh#OQg);n1s=n2$cUu3e+PzS{9Q z@E5-Pu-nLb6q->`5af2SvN!KfHatH5U^B%F+zA5s0M^*MumOj6eC*uMyGjf+0|=>j znr^Ok;)KCUHeK3Zp4eXA?^6Z)hZDs{E^5xewpe#gUM9h{K^NK${Z z+gebwN;-4hAw2%W2&IqA^F`-|;5+nBq`^v+Z*Pwb2&X`&8qk@sQ*7^)GuPZJpOyn3 zIy5_jktfB0oo>=c2u}GMqjqN^v~Sj>-!XOO;#E;0zWFMzz~R9KErJ}4C%OXp!#Zjb zt?j|kQxNB|9QBP!`wKlYrkc0Va&}@1e6z!-LJ7v%yYlnBP%)dW2s~alXWKuB^O*O; zmPrX=XAYerV2{1XWDe)I`Wu^bA%z3Z0T*-#{0hVTcm>GtpJKg%HatB6%DaTY;%7}a z7s4>ZRIM`iU(|4{37KJp zHFsMK=B@^xlK5LTZDr%?up*}Yf-$H$?Wx&)8fx=3o7%konqRD zkd6jeu=_6~w9KIGVKlQ|zlxep8{*DhVhTDAj^z~7NFvhnEwMU*!&p@s??Y#k9c z0ql2qkaF`KR-WmmEG}#DFT0x9qviMz&8g#-oryf&2X5vUO*ZM zO2o!1y>S3kO2EsX3aK}^b3%qZ(R05G=phI2vLlj``igz7EN~grU#P0$B97>vgz!Ro zAk#u-jIIG3EX55npLnhb`?6_L8y)xbuRVZRR5U3;?A$RBLAkuVTL;tX(L;s%>uyew zN$Q-@z3%9AB*(pmXf3DIva?a^`Um9j@Yn~ZgR>h+vHd>u%hj&014;(1{X!S*K7BG# zZe+rC_Uj2a2~fP254ZmDS@FAI5#pypY-ev6m#@y6ldiyGcz?Sq5;|7r%q;K8X22AS z^za8r?4}XPk%nk^TXhesOikF__7+%GN>gOwvsj(h-{3N7cP7|0zJ{vYr72ip(J?15 zSc&8p4g<-82oW$sh{6J1Q)*dR1Plo>=}+up_BoRV`EpwdmjJ}Pw+Is&7Lj4s#+bGP zI^s_UiHr7S-UX=565_|;Y?qdncGu*6;)!hPS+Cvm@|4unIt*cL4rHl*BCZDWG%1S&}dF2A6F0j9nnf^Z;5L6P#wfAlJy zF&JBO3213)%3*+to<0x?O}hvv0fm%g&;8f19Q3nSAtHnVkemch95>*(!oF8)8un-f0BF{UgvPY}TzG6dU1y`+;vT z@w;pXnZ5=hI8BnHQj0|g>BSi&xVk>(Uw#k^#8m__9f$^#Q{u2HM=cxw3u4G|sF_@H zI_UskU<5Et2nl}(&yh`qX3bhw0?e$2@-!$lYzhQoPpS1ZANYJos_nTy(iuZUl@JG_ z3@kd=@h%3jgQjwiS6WS${(aW8}-m&Tatwu$Z&9Nz8aB3e5;bKIPi#c zLp2`UAB#W0z?al~R52Bl&AQ?VbC`~RBc`}}a;(%~j?=2c?dK^3G>E3cV9BR%1u(-j zv@JRA!_Y^oURtL1#DM(etkm(nC6Ia}2z>U)ae;LM4hj8HJ zkz<9ZJtNQjQXT)nJ&0qnYHQ=6Y#$I10Po`R(ldmAyZY1W8GyV!R{=yk@G>hE;OYce zz>X9bxE7EE5u+;+=dl|PsfW!?OYkbN{I~US_SLjCchmJz@el`k)2uv<*8^4!ln!Zx z(1cwp1Bx~faWLSp|Eudtu5hDyp%nWxP25SoNC=aeEk zM-FljwhWmvr;I5>#4$uN?lLC4>+b*my}$R%`=L*H_I~#B-1l{@Ypr!%x5}U&%3y*= zPR1Yl8w5>RS=mx6hbEkRx#l%Ml@4X~uCpBd-Lycou46o%;HL7MS51!{l)WYDBZHoQuBY zmq4q?CgP!}rG`n1i#v6*8eF=>2bDZ0C&%2BxI3{8z(s+z1{<*bmO6}(zGr{9HG}oS zd;AK-I!NS?r# z6lE1|8GnPxQgs=lY?p`G|oFs9pSglH%2{n zbm1e1W)=HZF_s5+?ecphdZ#37-8ov6ziA_O;7xx5UIyGx0w&LbR5v*BH2W;$ z=XU^ud|tH%s`4X$BK_637s?ZE&ymql_kO=IH-IlHn0ADGQkP*v^0==X-Z|bOt*bJB z7Pa~d!Ev>?#^6RPi-hddC+jFmCzy@~Gn5q(V+{<60fLkJ2M0 z7SH^ojc$M>BS-)cf{(;=VYTHGAOT1wI20e=igS%LM*Ub@0(h1#WM;BliQy_+MM)=A z1mz;8WE`IlgB-=Op0Zq?mYxoNacmao={5|^RFDB=FRBJAhI5J!0q^XYsa#hDa;vkU zx_z)u-&h~2Zt(tQ=Y1SbFRi0w_xJbUvnrpz2b+X?Z=W!uckGc-MNuJLum;- z;vqIGZc-fc6D3t2n;zZAXU@wh^#;XwzrHra~4`rQO!sR?@B7 z_wGEk2P29JuUb>64ui{s=26T3ImdThmlU9)m5^i%QXXlL#K>D)cd?W_&Y1u%9d<9l z(4a?nXBtAEp{|4`CnwiUr(+-YFQM zSyqf;wgZ%4-^Qr0I@huD%r&#r1&G+ub|1Q86{iY1idUvfL2aGy2D9xk*0NttE}jri zMZLT;7bSfUoHREV1k6rI;D&iP-*PNR_AkZNsvGsVoKWLuzxfC-aWS8>%JAG#UG%O^NWWN&94G8dMSRdAwoxQz6=5B?XyrAmf z_x~1s^~cTn`ue3mJ_Es~=_b8Kwe~ViCy~A0({g1Q4BGJw9;R+^`Ormq(7;f`m`dkv z+fsY?zE;=t$AWE#3Ju-|*R{{NK8X5qx!^d1t+%&&>vq5~Z?E>=w-(CIZ60$)MVQ_Z z@qWDaqVX24$g&G1CEyf4XXMBxAHx8%Yvd6Ow_deDA2B-%Bf7sKZi%eeq%!#Xv-77< zFsyv`{#*n?p7J6d~1eTt14`8+nPHC6tiQ!<+sx<7suNT=iDP z4GK@v0|){f1{g9&yeP%j#G;g28@sv|iu`18{M31U{TipDkM+pg#Y1;QizoBnM17gT zY-LnhM#Zjz&oE1jkXF^La*o|EK`_3-U&}6poM%yfI65-oip5C7p~fNb)EtVK-_TvO zS6U}LJ%E>m33|(3qIGwgxL9K$iW%l!gS^G(H?qTfx~~x_Qwn)+I66F#w+Xd*hmIT# zmH8}Kw#NqloI8EyOf`ApC{}U}#b{2j^<22RI!~hd`r6Do0nLcfLo{FCx%Oia8@Cct zoYnX!cW+1)$;*hZE)~lMD!;#fCO`JbgA}W*_uqU)6NvvQEfu}msTWS z{O25H^^r$J6NYo(@Y2OSE^RLYm0BhpA34j3Y$1K!>(iy4o$pRj&FKx|ge_$sw#h^; zh{G&9y#48zJl)MbHrL%Y7TUE7B?o=CN8b#4r-S0`5{B^Af)Q@=>blqOivdxAv@cwldXDs0zqK=T4|@xiOJT3`u*tx8&aO*W#6i$1zzzzXECk`k*IVvf zm_JHQoHiN|I)m!rXQ2r&0jA*FOM2Ee2)5@dzTPvR&|u`>%9N<&a|pqQ=)eeCQZD_< z&=~x%kA`kNMwbh-4x7$_op?$7P>`?zAHwJ54)ILV{_Q>jB$sAiUthsyIW(+~lUGS3 zL!knynmx3p3S{_C5}dHt`{OUL4-`Ik=)zpKpnF8+`1dAjy_7tW%!n>TG&r2BT2ZxA zPZ_-XR#(L&kahVW_{c6HIg5bZ;D{#Iu|FSFjWwd%w*p);)$f}PVWnkV`kTu9zM;fF z%&NL10M_is+G3@e-L?7CFzA@eCjKfe{&z-(h@g&l)uvsuKfbl1(c6HeAJcF%w58!_%jd=qWgIvOyZa{o@EC&6lkDtBpsRc;RRMaz^6T>z z)r1-*9xQ2YH0C_2gu=+|^x8?bHzrUPMwO)nwdIjGA}-K|GEBi{1uT{hM9%E}X%r02 zN(x)O#L@|HZUCWH`BDH)ll+q37ccI;W?##7m^tm!r(VyzzXl$_9iEX1?9Eze6-X>I=SWz#E>VCb9LMFtnV1Io*_svb4h zzoWLK*ey)1PUK8l7WT|a%h09=)^))Hie&z7tFt}%yMI2=t%Kazd#~%l#~qyh#ssuZ zwYB1QwVJ5|Z8*U>iWl>`w%9hO;N;rK$=f-YQrij{-VDhuf2>&!a|7?{Uy8acIcV>g zYPskv#=_JA)eyRTnAVBIB4*!9#`2e!$9ibrCp=lV#VA-ZUQg!xAnT$flNhP~*0OB0 z&esC{{23X?4aWe*|vBgpfDhe-)GH%1|dm7wnQ(pGK72+}WkIX)k~cHv--XiYW9 zQ_yo&>A4!N;WatR_U39Z5v81vW1#cZAQ1nCXN`)qVm{l2hMg)rSUr>ZPfoR!q#HIH zuK4IYHWH^dN-8Eo?tv{q zO%A%Lv9%Quzh#h+kcg@MUIxHC>-KMJviZCO%(*{v?^g2cQ4=RlgV_ z<8t+`iWmv%(bF8DMzP;`-J|X#MQ`1*WvJo()3y6o@9!G@117$`q&cZQ_ z4h5XgfMP>LOicUL-MNK@dfTVl*7j=&?LeYxL}M8`MZ3B})9i88+rGXg0t>5eZS}*} zW&cpYvTfTbG`*UbnEar>aQ3Wco9T_530j0E`Zi+JQm3b#frinTq=pVLghmuVNS~kt zKmr1?&&|)*#Y;?2PNu>h51fvN5c19ruEH0kr66K8zUaXn0h$l(uNWr+2G!|X+ZZa) z^#yOCzu5c*t_^)N&og|Lm6hsPMTco4{R80|2IGC@iO;bJbc+(PVeI>gV=?^>Q-m#w z;^dsT^;?xYUb$~mJu0#VwejVFpu|y~M9dNVb0+bw(@DK_yB3~IU8U`|j^-tSp!hKS z?Q&;MT}K-mKR3ll%t|-mXx&hqc7l|!>Q1JJPZNtdCJoa*{eQWbZO}O_VU#`6oS;cd z(LB38re@gDp)j5}Abt6_qc=bagM33r@L)HlSz&3Sv4yZ11B>YQg})>@SvA$Bd6lyLG1w83Db}#G&R$znifcsc*b`KhST5kMAgrJj1>w`Be4Z25kBxGDuJnk` zSE^~GcLzNhc>EYdmY$#gHKqYA$3wg0(a)5YWD`xz)qs94dWkD+3+vMRnDR7VmLlu+v2&(A(OM#e6%&m2_ePeYc-KhcTTw*6X!$ z962fWAPDB6t-LCTr_AFiBw+V>cIXhL?8o=6O5KIDEwtERM#fnmuN{~wrzmmlDRpOP z=1^9~%QBawxZ)#6jv4E+gjcS4kqNa{WnP z|72{At5df#)jt^iX8+brX}J)Mi3Cs75&&-A!`cx?g@{$758eQh69UFhn*P3!cIULj z7wY-Df+{LW0H*E`LS`Xg$BX6^)5X83L-peeZ2k~*(c{%g+qm+F>H$`r1=3me_EtCC zzB=1qFTWB=*R;1dCpRrUZck#nwB3iP+yxo7d|PXCer27R&3v|n#*HEeh4!9P;wYVQ zjb;u#(vdql5~E^&#v)I&KE+399v)wc$Efxi%94}`|A5GQM8PDvoZ70I2paZqPvWmy zjlr805ow4X0YwQ8ru(S*hxC|8z0y0(J1H2BUXfIhT0$+)>N<@D$2 zkS1-BUpFpCMESwQnwFNfDfjPA?PMFnxQmD4r^0~UdqHnoY zz1i#ulde3)B#svN7*#%Id^|2w^=E9JTS-J#B-b~XC7>`cIXQ|P2w}cA&Ss%LJ`-B6 zHM=aJq+}kZ-X)!~(z!=+%E2R9Qo8w-yL^&_U;YkdX2JSRyG4GnD07z=IOpj18`CjO z1Bywu*8RQr=vX8|LSA-z8=W{u8d_Nd>qU_`Zy(o_(;eZ@YhKN`b2MVZuPVHl2(u^< z%q2%G^U5brO6m4&Ga0JI?+H%M{pHv16qo;e_Bwc7=leL+0sPWoUww*_e zxkpC19|o@lJ@J(`xFfw9wD(PHthwWGg=~*{80{GD^8lSDC4X~*O`pX)Z(~m4>~h4l znol<{aAl9alPo%?9-zTC>^8=^_}}hcU!33lkTu?6PH*moGsB*s%GkkIZ{ig3AsaBe zlnw9-G1dtqa~-*rGgEneM*wCkg`qFy zQMV%|uKcqz&v<_~8);k!mJhb$s2;2}*_U5Az_gEX4`;X=)p*fX3nygplVgC6O}`X7 zrY+VhNIezI7Aio_D8iVOTYp63Kqoefv*aT%iRZN7fB&Oyowd3iLmxhLWo_87LEKZ* z&{MQ5fpD%fpeVYf@>?YUfyV!X8 on7dnVuyuBEvKIHi4O%;U*tvMl&{wMBk2ajp)IFM`Zg%5;05V?b`Tzg` literal 98482 zcmdqJbyQVr_b?&o~`cp2I>L zjTveUE5^VzonBofTIw~-+^k2%Zf`dtDJ8&cwp-G*z|f;V$LYpoWd4XuD9WY_9*|mS ze|e|F(MBu6RIBOWtS#cjmsgHGKW%$fSqV#`mkwc*i3d}cT zVKv_XHria2>x}|al#J^_xw=vCH4bn`1WU#{ajv>iQCE&6F_{z|ZoV8b>cD-Iqo}&0L60RC*Ntdaw`{xm zLvAh`JIyb_JOKjD9=WHu2c`x-KVtCiIKp7Qq;COGdiORX%nw#vFKbT&Pxr_LxP-C; zy)VMOf6R(}t+)5Anc45=nNXwxh>8(a0C<;Q6By3rxok^}C2;7Cm8Crx!24Apd>9f& zpDbUZ=mOU95}JXlQ?4xzOP#L~UU25F_|Zi3b?=uS#;#>=v8p+VRje0mEYcs*#Y;25 z*$d9w&JI*(Rp2HA)CnIHmOq^L-E`T{yUEap+>LvO%m4>KOAj6$ZGzOh3&*%iY@f&)Zn88wRtNC60W0F~v&r z1z$1jPtqFfN}3Kk$I3g*l{rp#@tO+5rcNWGncT(TeLeaWqqHEF*Car2S^|#1L9k*H zeogfGnDNcU?Ff~)hyB92NHPxFbbr#k|UjI9G?jd9IQUue47)6EZ9}e&nXud zsNU3!5PtB=z=tm&6wh+*hp)VB%GdqgN~^}I0z|ROM!$M(&otwO_nNb|mzdW#aQVAs zH=_l5)HKv`rwO-5W4j=9Xj51$n!M+GK-Fl zMzrf>65EHcST=p$JzkVmo`rAfz>m8EM_lvl2c42|b6K!-b3{HZzTPaloz%5}(->g*@6OT?$F_@qFesoRZYWY*J&oVif2A8Vz?pq1)p&uuyi+L4OP z$;ovd2-_ZPzTBTBh=0e#&;5xyZ?^gL$Ytw&v6_@r(9+R$80 z0V71Y1%KF9>(ul(7Jg(to9wh1V{JFMW&g=*#QH>Q)%^5xW9rwzvdowc-I-Ul7u#VM z@`(&Q$$L`HOeA+b*s6!CxSHeg^3?`V5*Yp+I4Dl;h82vI5@`j~{xbWEF`fhez^1gd z#r=YjpIaTxMItA^QWpsvB%P@Gic%ZY{5sGF;4@Lv3hRxoO*|zq(MYpCEw3dEgF%st zNs;2UgFP!XHp&KR5yJOOQIprNz4P5XA#+f|AnkLOj9{bqIrP1O2kLY2z>w~%E zBQ+G4)sgwg`Id8$S_7VjeRMQd?Bdgd*ahR+<2drmg9W}0=@8}VACV-^eS0N?&nH-3_PEn&n88KI8?#D1jQah#yLjt4jQ0Ek%|PNw2q6MMB$chW zvnPdJ61^)fd1O*5@*Sh}i9FL;tZQXsISn%OCSMgvu0D3NQEa|+Z7w->7WA-jIxzEE zPEJ+oC=R9VIPYM+6!SSdXzp`A^*Pup9?hLSSwma#2Jju3o!P6I7}Mdg9T~ga-Ka}j zenVR_>ryIXf0kPVc)V+~Ohfk&(`%h9Rr_E==7d>;uYz0(kgt+!!x^q#A z)h3GBT+}v<7*bajS7A4E`IdPX50|4Y*Ho^#@T?8iW_=2eo({**^W!xN>z&t)SpDZq zO$N&i-aUv}$5DV{mKpc58K(sg$52R_Ws$lE8!OcD((Bthpq_1Hw(fZyTzb27`HCf+ z@%%^`YmnIabV0_~kt(!XNd9ZW&wXUh{>O%a{pA=fDz4NOqBitB_4>;$+RoisHLk&6 zxwk>7IY?*cD!cm3*)=Ylx^`+U3$C01S>H#_{!&dn9nIaZ3u}-cJ0Ed5-E68B~AZ(J>aMq7-+601MG1tK1ia(+1vi&eSa$kV;g6w`k zdfhs|-jHWd@oK}1!4#5L1|k%*XG=%-uA@;j9kdl7s=2LKi<LUUoPbdUX7neIsH!@k`QLlfbQ-XvF0q=6%s0W88Fcfdi zI?Pkf;BtQKpqsEkdBl!sD*!Q*mU^+AJ258W zzWgXbULG-}l_I3Rl;3@S@7%!4Kts=QgfDi|ZCd|)eXp6{eAW2qVDD$e*8xX;3|8#* z-O(QAFdk{_^FhR=Fn|U*tOKHy19LGH^-n6&$c%n)11T^2DU*wY6HC-5zj1KQ5sd!`MD;;d3>KFroJ? zJv&v8&&j{sI-5;0F7X~j>^1evOd1pr+spNr<-$|7zi;Yp5mxDVQ)GTt)KlBWj`DPU z1N{4MSm3sjJ#E~ot65|8xgZewIq}GOEOjQkrs0%%eQLyg^vMHm0wl)866)#v#C0-NOguMlBn`2>X?Re99i5JTcAq(^k z+4#u3CdzZ%%L{=^$`_98RPAPU?qrk3uHvd%BAqpQ zk^B76kvLR(NJykX0#7)?AWBAx{|;Z|9^BcJbH!x=Ed&l~E`wF*S2mh41}>fjupCVn zZ6!TR&THi{J^PW>v8$$JtjNRDt(ckF?F}857rUR|7}r-XQIJXvJ#Tuyzt<=-&_td{ zTEW@BakzXSEH1Up@rb(#6=rw>wO+hwWZO-%&mhZP`-YM&TlT#A5*ZdVzP!c9N*O!vS+ z2{~M77GHz4VE+8*;XGMYv)1EtxP&I6DRR-f5d4gqujAoDRQ1T6_Z|008R_W<&+OsN zO^2}t$R2+Ehi$w^wohi$g}t@~QjrHfE{#p6of@&&tm+G=>PdW2?Mu$d6(?)T{n(6m zy545Lq}84|13kfBhb&}0?}bPJ7ejX^#KOi=Hsl}#bNfch7}sRNx2s)L%%0dB##tu7 zb1v`ec}e-4Fdm&KCQ?=pb^NTM8ux?o{1@dnCqkR#f zOQ%ta?+eJ6SNr<0S}N?;gE4SA^UC1TW~!g#P0)So*3ZgZeXeFZ*f5x)8%(O%~eeWiMCJQ_HwD{20Iz(*BGgA;K0 zM;}@zofCQ5*J59pYaW)e30FSD-PUnd{{ndfP*oeJJtYQv8>~6y4_t$SEED+Sd20)4 z8O{4EwSs=!gu%w&FEIdbTGUi0yy9{^@#Bv9a>vFY%lI2-e!k?rx^|rkL;5dYe@;=F zRPmC@HyR#c?QfIJ7I|E(lb;`_dKcoIw+7yure93FJm`B<@7?~wO9+=-=u@FJ8If@K z9$aU-+Efn%1}h{7Gk}(WS3g7FJL%(*Q4HB1UM@G3Bqqo)$K6y^i6?=BTGCsrr&}4n zJiOR(Z$7HkoM^8Cu6XQxbYs7$`631gIMa$XY9NL)`%A~5h(i$6SGJQ?hFV=serLN) z^8Cr_M~1HbWgM;@fIR#I!Statz46o57pzdPpJxVwI~&_;6Pa30+$-ZRyx3U|3=;NE zgJuOIDmdN~l;y`2|B7OCImaj&6S!C-J6#Gl??6v+zn*;|13$@CCbPCNzt|b*|Z#8)flV;;y5yshk3dOQlvXGF+J)8Sy4KXXsh=*Wa z0>4VPwuef60!e~XC{#DoXRD1pP0NLR2v)&;|0LrBt^=#-iMqKd1WO~s%E9z#iRCpzl4>g`irn!kgE9BN+rwXU#o~4X})pZ#4guh@u-0u%6etZ%{ zygK)6{52H2(%@Zpxz-R%HNBVV;i#|iiDCa&%U-PBUBLZhSDndDGEak3Yng)28S-?$ zMx-$2kT|y^BUF0hJ#%+s-9%LnSApwLF=`1+z`?sjf*`- zz7BAvk(uEXFR%Q|p`_%W5;V>U+<`OWB-7{ zTQBo&rg>vK6rtlaRQOh)0YV}b&beY8LddxuQsTA{X*}z-Q17JQk-GrjFrR_<>)#2K zz}pZ?^%{6{L9pZA2O{CoEmU{-d~BJt9<&VyjRxL3g=4YHW>1s&5c9p(6JF<@?%hK0 zJ%nRy3n{PT6Z^~DnpCHNE-?`wNs7zm4yRBx_k#WES@%iz5zqjsW9W2V1MPgK&#@92 zxlm!s3fwW2fUqEUNM)7O#g@08qQ`4pQx-XSIV(Z#Vh~3}Q@Suf5!zUh zgW4)`=h(S_H-baYy({{38sV)m>-}qT!Z;%6-OPR$&ssq2R2pR2KxHF_>7r?bcy^O6 z_PQ>IdA#>S&`>(6Sew*=11g&nf<_ubH+QzMWE>) zjoK$hXW%(EP+h&Ac)$;tj^X^tqo4zDp6h)5T%6TTWeI3;hihJT*;3NV%tgQ?(6+bU z`^_TVcuz4-p# zhYHDA(Lr%`U+N7V?rCT;TkC@H_PKHWX1&YZ3!g#o_bpfTb*-9l7$Lbg&2p$KEiJu&xG_^qqVMR} zwErV^e^n*S>igp7$Q1!;vbu~t%<0o^oe}SdDz+tp*hbMT>LCA*=W0`_q$|RBO1|npt^$_0|wcG2GA2 zvn#}0?**tHOcg138QNzcNONG4d;2+Ofs*a0*+{-!Q4fFaGxqf2Ru(d1zAR(o4t$x< zACsgChUwJ6Ob{oLWVpQX`D-jxc4>9H$gFto5zvmEi=9KQF6bXr_0@~5?5lp^cvo%P zsQG)$A0pKI5-Ps=xHZG^4y;T(pFPTzXE4oKba5X66BNRo22Hu9oHvaZjx^?x@VZM& z9CDK3psu@ss3OOu<2rK<&*a2Gp=|Cs9XG!|QgyoCK8@ni1)5+yPXlaU~7?OSZ8ayVX5o zb-*3K3fW$1fJ}m~|4W1P|7S9Oi3b019}`oaxyDt=iXeLoa4F#rJvH%sc^8dHHBwa_PTbCdf4I$ zv0&qI$JejJY={Pg&kF16>XF}7xkBu&Q5j6S$zkFk}t z^~lVOd@5wLp8!6hlcuJoUf3F9kW2H5fSF~Yv0OE--kq_ zz)Gtj&U$XlCvrS8I$BUw6^)*$85EH=-xru##TPrdkv=e;c6LGv@|_FCOvV8 zHu=Kd3I2`iHiej_1~hk+e0+TLdo;k#32t1EYrGDlS zcCvaxlb{rSC+IC44(AjWwsZf`voy3lT|Z%V52mOJ-DBh1@$sj><{k8L#Qj>+*4Ni< zKzQ^3%q}m^`8|(Ip|Ey>)UWJKXx;~-!mIwk4hIhpQBpFnd%_YcV{2=>3vc9BM=LD) z4I<5LBo+N+uAi+L3T%T6dj6rezOys15#(ZOn$0gDV96GY^BVdg3BlD5rLvX5eTT-! z_NH%Z=oiWnS1YZ zLcJ_3)`qX`1-^foCgh-FU~uvBTDX4z8Mm=fRZ)q80&j>1by9V=Lc|F${h4@(xUj`q z4Hd!dxHZu6o0jZ2NU+hDocV1Fv0=W^&mkDNK`>ZfFZXOhuBl2$VBmUpH^EZTP+-!xx3?Qkooyz{pTBsaS08fF>7a$N75DWB&K%MDBaN57-OP3zCgKDp>)taoft*B*(0lZpR(3Q}d{09fL zOh8ys*8>IPTAM;2_4m5~uC7;Tr&I`ft#&Wy+3@V_0Cxo_A3x|01)vy5#>UJ~JY##Z zLql=h1<675OFu5zaBRG)ANlr;p=8Y3*c3bne+HeD0A#wk*+3$cv`jFkyj(dsIT@UI zblj2nI!ORk^DJxXfH|~!5sLaaOSE)80wH5;Y#h@81+CQ(pcb}h?dGP5ii+xP%dxR{ z0+k5CBwUGr)xD#l4_Es#0YKuatC@bDF1vOI{0VIbg80OqBU+|1%) zd5|e+BZa(IA9*aN0(PoGa|Z;asY$=pFr4C*uL~2PCwxrsfwrwzh>D6R2rmRst!duw zGEq)flq}Qi-Hijt+xk}31L9H`1J&h?>y@TG_aHFZ&_H0^y9a6*H!xcEA#<`lQGm9# zcBRXPdLkog>9>G@08YT8C!TF=!(YEXnY#(DeDx7_cJ_>>CgC1|JisJ+j$O1MN5Pu^ zgGe=-uEBU&q4lO7+qEmsb1f)6D;|ydfO&q3VV9!N+_?!Pvb2v0W~c@MYqvV7>d3{# z1?0eymlTLjS~%dW^RuIss>w(rqTRFe-4jnpi|Ev$i#7nj3+?rCxN{NNT3Ce7?$neq z!2E6gXX@Q=!omqb5PeO{G-c-<#&`KWz-gwY3L!Svz3(eplVoeg$jAtg*nNiwECbhu z$%soxj3ALw8h(S-)!d__sGaO`DjrAMBSCi`F@?>0+=Nk5KznfNc9oF~>&OI){@|J% zP!dQu4lb@XxiL`yF$u}Y@URJllNXQ}YDR*B1o@AkmS76YFDz6wH-Cg9j*_GXVrOq( z0$6A1F^G?qy?w3%C~7I_CeO1$Ww*^YRQv3ocmnMMfK7`7(g{MfG}1Z#@7zC!qImCJ z#{yu(w-hm?6+^<77yUSBeaO$zN+!mG42YmRj?r@0~DAfRF6qRv$G{_ zfp|K?spy%o&2UMYVJL_ql%Vvd>s>2As(r!Tx%rWt0C8O0+-B~wUS5rFS}6rcVWs|H z+<7=*?y~xos>c|+O%W&!2p^;%imPBdhNv>2b_3how`@wCR^+#)YLQJDu(>}@YY|kz zm8rV>*R>XOs0c_@0m1W=z$29asqqJE5bkj{&q$?37v*Ex3&+Gtw+B`WC zCUfCWUSC~JYo+t$0yFcv`g)gN$1g$Yp4^0SKLijOH^e0u%=5#*(-xlHI4CG83VD

+2VwAdIe0-^)!sY#l!{HM<_WCqZ9a+`#V49cf+{>SV+GTcCZetuOoHB*rIQug1L zsRi&h3<8~E<-cklz#Gsu@rP@guD}UHa=nebedGFZ3fM#dw25BF90wqzh{ncbkoD0b z6ZZV#VwR9F;F=a3Yb!mD>_7>O1Gi(yb+rrNPoOkTFP?b{gJ=Nm^BpOhvNdzv98U*f zx>{Lv3+y-Izd{*?V7(J$G27(jez2AYJTDtj7ATcMCurB#+dDb{vi7Xlh_TWCfVz|$ zxC}`Bz4$@+fL~fe0v|;Ll3wW4*474`X@zG~&yuo^P9hj&1JP{xrnAjm@yE>ox+%qi z)*7Fm|2A-Y5Lrx2NckzjNCGSqCTz`$*s;YzLPD&ptja+yeaRrOl1cwJ7Y3u#jgWi> zt^@(^Dutu*|J8ZH(tglh(c@xc-{u!WgTjIW28gl?@c|AsC4RRgOX!(}%ox79|B%df zHtdsg?-NhqtHsx$#gD|r#fL{mOiw&>k@>$f+46whi+^{*Z1k#Ppt8hggO*|(?#z~m z1>$O5;TxD3@Gb*{K`)(t?uzK>XkaC?L9FrVz+yT>U=lzUf}cHz=@=dxW4v{v)DIJ; z87`fT4>GWhoOkAT`T2zqk(1Nj4qB=n$jG2E{0rfrwWxo!RSr0tP8eLt!9mf_&o8C} zk^_9G*yx`8;cI*=z}LWi7wOb|)4dPFFZr*%2Hh01dvqUYY-2knOImz0*mms0b6J%hN*`GZr<7&DKqY13Nr%Mjt(2TCM?ED zOixD#F*>RaiojQHciL=}?ZYPIaH^m)Su_2Yt^!8ccd)h!T)Op%w5@FsaF`XF;PoJY z>K5W|*R2}hrS0rSt8FJidjJcM_)V06Q2*^KH6bA?6@L)%4iM3Q0NR%WXbh0WjUV%E zF&$rliC3q95c+z4-Uzaw!m~?``5<}ZVI3fOuoXzGQ&Px#y!dHM&zzwEkB5P}Ro4Q7 z6H3bW;^bF%w=`(Kh79pxnAd@$d@>Kn+*d79Ive07yQ~a!h}AX|8HQ{Yp!M`!V*7^_ zzT_nZ_B9iLVjODQ)So_xP^80G79_6`%z^A=jXk8wEhpeJAibcqtP}(g15&5J8J0?87{18hk|PCnYw0)y5OL-b7WQT8Edc6Jkrr^d(g^y*)MW)Nmr1@;ap z@pLXVr~pqiSU)P1jBwx`K|^K2Zn*}zr+?$ukomXovGDYq+9mq8Ff!eRKIZ$+HexkW zh9$j~4u)U*A5DbscgQKPtktfhAYY?m{O*%RN86oEVB=5z-9$h&NrXKF;IRuKzNa#Y z<@C3^0%pw@hwg9l2=$eeA`sZH^Hrq>1ovctn(@)C-QAebQdw;A;)dXE^bH`;mW+SRO28DLs7US)nl~Ve;!-~iwqY8`0@J(>qL$?M zUriNkr7}r;^q(?V2p!Ym9ylgI3-A^dg6%7N^tWz|gG8ngfhHWZjg%#}W?sUnzr01D z#M47@c?&37kuJ+k(9T^JI-&in9aIP}V(gu)E$t2s0b;%QVNal?dH-%1z_VU;>HpC) zSUTevwm|{lAQghE3JD3t47k?nLKTEU3N!oytis>ku?_GGlr}!_7M08K52KzHBq&cq z0Md-@ZMz;0ELtG4f4YQHDxj;r%PDb?OC97Mw0AD0R(ixCJFHX;)Y0?>0q4bc%`R}Ho{ zHqhWL?VZ>FpE!z3h2e`s{V$FAmy3e&ia=i0z#w&LDK>KPcT-4({bUfZN-5Rn-w57_ z{b#=q+-x)GQFWA*(m{_F^WqvE1K3FL%<5_$INnF;Y^LNL2i{>&cL}Co5b^?W$o4ei z?jw0k@-}mF-xpw(k(ReqKa$e1paiUiAbA+D7GP}hkDd?|jInWMLP7$xcXbS~_l(LU z^i0r(LjE5VzTx!yXAgiRsR0iwm;?I%t*tE}VWq_&-*o%{#`6xmCAJ6uYWHEKb$=!* zR-d*&a}8Vyn5%5ugRXoh401*UVPH~d38V)Q9`N!&!pf53Lx3AG6auhRS+mjJeb4mRLY0^0gO zJc?>h@4v@Zn6Pn&-?Oycb)BW(RuQx)r7S=uHEV!N1!j};?nT@=N!S(@Iw^1N|@m}(f#{M3JMCq`hghzE@?VVkjByE zzw0y)bOz{t1tldlVC(@Kjdc?&R;2r=k%|j8qSy&toAG~}TY)z2&Hpx6*;GVNn}asa z^S?C-asW$hj%Uu`EPrJ0if=Jz98=i9XylJ`g1P^P;cFFIcA22d4$&0OaB# z=oW#o0kZ-oJhakfC@)}y2MuUXEJ4To?*T2$_d5_dxQB^9IY#}2GOgCJf3 z=suPvCeOUQykcAxfue&ZNKs8K8njAa#{chOS|JID9A^L5ui@zE=s|XJFemRvv|6wcsRH$aj^>-qw1VRQFJ2rEh8Er@Cs&790_u1H zbuIpM10!PaXbkF4P+$838}tHzj+-6C%sK;1B%wLgKLPy&<*^(`eP5y4KL=-qnZRb0 z7W{WW!;-=Pr(axnL&%3xSeWo%KGFbETrpbO?u&_eHdS?XhQPLPq!vS)=HmB_&ll%sS8e}_hPrw| zW24^hz60hv9aGU~1Ly)k=zXz*#PSShG&`4oz|l(Y`SRxGUmi$~9R-e~Y(^I!`0`G6 zcI{DN{Mq2p(9~G`8M0T1LGj%UzEYMrGwq;f(nusdZB#`3A)YYK&Th|F z%v8&cLHW1kH`19|G;zB7Y|J#pB8g_+s*z{z#ONcq0e zxRK4xDX*2aiO*}BJMkXMNY%pukd~99g7;lV7tY&9Q)dIUXKMLbi~Hv*zdS~IKZN#f z^vd31WFLFUuP_^B_9i?SGgp^ILVpakA50h!4*~4DqPkQlh$@0R%sIKZGV}9?`NTdr z^Lw@ThScVJJ3S4EPwtC2Nu%T56ES4Bbk~}g8vd)iyf)u2UW;r6^BFq@TXgi&u3L9G z%Db@*-3Yu`K$F1I0ek*)$1E>9`?*eyozTuvdpnpMF@edr|ARY0c{Md-yFza`Yx-e@=zUdH*lgyB3;UM>rZ2@wei$ z`1bm^qpVWpv!ij)NkfM=H+r!@Oj%nG-ZIKp^?ex;>*19q5SUd-3DRJ|LBfoSZd#*XY$&= zC2bkH!KWw*oCEaSYO<^T>R}v9y@CX<zk>FOdKOgJAC~4Lp6<3HM^91`Gls!+OuP zEZ3&aO_y6$&Cuca>%aJhql*Q>FR2_D=TCt<21%mYbX}Hdm%5=uW0n z&9=Wx+Xa$_B40}`A53g3$;ikoI`XM1&G*h^!UawD(ONS=v~1#q?>|y_$i)}0$`qr@ z$=|rhb1obHSSXpliqupc)Uphf_%ex)+5Prk*`%hX0pNNSB)eMQ%JfZgcoT9sg_653 z`1Z2q_Aki>$ljRPa@ta~ih@%`1j(BpC ziWu@zK2q9Tv+)=+^Kbn^x{O_iIi0Rr9q`EbzQ~(Tb8=NC;z#4}GsmB)GroRj^wC5N z4{Yh|YyY}X!K4u38&$PvLfJ5W=)RgUC#7*ovQ{r1lr08|gSV81Riz&(^cNW?B_t?4 zczKBTzL%$}chj#n9wVE9(J9I45XI3nX3ykJSZ@>twE5< zH#Zx7%qu9+jgFRg7j$ZsV0QG2wyy(%YXy5ca47Z@h3OvHy}vV9$`*sYJW5)n9M$!LdGCe2z)#pHA+$HIYg?2^ zW)KV}hBSTt1o4W-`81y1X)mksLO600_>!|Zv9&SSZ+a|CN#wkDGov`-=(|+p<2dGZ zTnq=3hNSZvV5W)zqp>trsuAjc@~XPD8i(EB^Uc2PnD+J0l+TkQHASADSdM{Qfq_`A zAm6g4VWwbn3`~db1Wjyu$t)jbZ_DWwE32t~G(UUdU|;8LvaQ5{r0wS@ zU8^gDK@mSeJ~FZdfqKKGxZMB);Tvx3$KF@@AmY=*M^VAgzGI1o^kPUby*h0(Ewg`s zD-k3}zS2-6y^^w@>1L7={CR;R7hc-Sbds*DZq~Ws(!KC<_4wyg+Nh|=tRdw<@WMlO zLV}!QJ=wR)uA+hh$C1&x+SZm}FbRke?|gkgc`tFa#AiAi6$71c%9WdQb2vv%N1h8W zS&6CIIqmV7t4*K%|3z4}O=Q6n~G0i6Af0=D-5GJan+!Pi}DOV9%PQo%HiH*Jw z&X8X{`h2yl|F)wIWtpaOX68#W5pTyqMe2JpW|ulPIvQlb}af)GK*6< z0|mu9%{| z>;3tKRA28R`iuA?dk&FFChke`^z(0tX!oyp0cKQmi z|DtR$qerE?M@CKQaZRH5!`m1!ky6M#0%K4NF820T7W50@H56m(ZeoqEPw z`c+l2jErgA%4h|)^NNnH@0~R=bH8_k!?)AB78fh!lj4Tw3`@Xgad4P(w`ry2w^AOv z8fa@trJ-yn({q@g&3!SWH8-R2Xp~pesCV2JM?a|Acy;5{>IeBwk2noXJz1{Y9 zS&^6I_r5e4bK=3fD31t&WG$UJz1u$wR6cx|Qz4K=6ICmNEiFCdST6${e|mSfBYCya zb5rtTvuzdQGJlyp4i3JC8GjLy`{F^|wvLG6{yx9jYtK5Bu7Ce|KjcG_ZYehH-uz=O zt{Q>v_zff!J+-CCjs-=v==X}K)je;?<1hQMg}S@30UXt|_+m3+KPFx9ti2Mbbgz)m zsdm$%4nq4~afV!g%7{r4hmPGuGc$jJUxrY~b$kQb&E&-kcx#}PscEg1)x8UkzU$<|2H61LRdi%%e`t0y<|cP2X}s=_7F$R5 z{I0I7hY#=YRdBHjl-hnKGIo~U$m^~DB0fCzKmv%VD+vE!n@fNw5dr-ps8ItAAqxLK~)7^eKyYcvF^~77# zFfw#kdU+5~(2$V+=E`v@V5<3v!87T>B-)`piciV4GSZg$>FeveDsmm}a{&j`93*_i z$)m?^4_2O*w;7p`tPNbc3i$+J$Y`q*Qc2`(RjdE~F2L)==9rfzizbI|K(d1FV)=#0 zs^laP*AyUnPSWu<)FDeDm5`b;fG22WHN{6k!^Ks+i4kPvuuutgGBVVQEz*3GUTex6 zK^%Bf1qi*~(I-lsy}O;)$@k1h22z7`RTjEg8dH2y>rCIf7D>;t-QdGi1!A0@o+mD` zFp=-IC-S5pn>!|96c67I80@=ukqJpjSc2?43Nl{dAEWQr&Aq`q@r>Tz8K?XWi8)UQ z667hvot<4tXG7rE6XaxR?%{B=3n>0*!TG|nXK3f?Y1Jyb2m-QMc>1B2g?TFXTVQym zFX$|&FGW0DiD{*@ma%?OM4UB^HN9ePRO)*mgXc_b5!OwI8Eny`I}dMH3hRG~-of!M z<-T+OjZml11;-?64*dppUP%NcU9d076gshm1w6ande$+2ii7&--5J$~ykOme@&W`y z1$*i&Vcw>82EPDx%CC++;S!OcuC5!(`CkYnA4UZZ#H#i(g>Rb)kis#!B~|G0ufG+g zcRw=@;#~T+#z_}r7B5S@zdxQ=pV_L^`Bc&r-!Z&u&K*VH{u&ZF7+L3RMCC6@TevgtW!#&(U za`Ld-{A7)M)LEqY3-sgzovtyOF|XUGa+ZhC#MX~jH!EA_*@kz+yZAH0{ldHH7AZ%5 zoZPu_o%c(1Nxmw%w@-#id}Uj@gv|S(b!E{4)qIXwGi2++WBPT~gz1;_bc=Z?sjRQ# z9*>PdI{BUE!^Hn!P_hF3Q#kdYf)Y2n6EAM4-YV}5_Tu1H7A zsuHoyUQ-#Dw-pwS^|5R5_6>&{fk;QI;=xh}-p^xkm4^{54|FK}N#Cl9Q(@4z?_FD7 z{vH<})a8=z-1k8o=_rxq);HX?l>=gV@Q~U3=Jm*6mc{Q!Z}^ijlatk*oi#1&B%zIRby+*)kApO@>s>AK078*X4Lo3 zktBEB$)lof_x86h5m!zrTjrld)N$XKaEo47O_U3axmWQ*(b7B@t+DYHCYi<)st)vg zd^>!DXQa62soyU5tN2w#6EHiLZrz|+8Z)Y``l4)(!E8Li53-WQ4zMwR6_*eUvdj=j zQl^RCYA9_NZ@`b`Yi4^(nPh2(RlA6vHyQop{6P=9kl73AZQD5 zbPc~rm57+y7&1eq7xl>+b=-SKg+Udq)H>rr**li>;!fJvc3xK&Iz0Sr2fy&sM~(8CgV=8RbZ7ik znw3Pvj;&TBBgYm}wD^vdWucj1IC-@$eSyP|Q$chbe>;CTG})8IF&H=ua<9Nya89-6`};i1B# zAK4gLT`le8na4)wsudbD)=XQt-lnvkh`<5*)`I1B^MCm?WQ-lktZkQJNgv@yE%%$#+Tes5o{HOt2VP zaI?w6v!wRW{fH-P>MboClGA=yriG0ltD&dhhqW~#SeoK3HXcGTHo5>ZrXFyB43N-y zbSfNCOaYK^Of>zb+L;*^I}Sz{N<5s}gxwrce^SK9fjwRy;p-r~PduqP`R+&Sz`j&d zaTKb<7sMo~=RPI9h^5I?@@o;L1gqx)_Tc?qo{7FtAfACF9-KBOK=U3Wu zj`vaC@hqWZB=2_gy%7yn10mz};C|)q^iMLXl;M-(hWhyTfPM#ZC7H3dX z1@c;ek;R+}+22PJJP`~7Vb~q|sLBwww)Q2_?qL)fkTiMBMGzgE^GO#S6tX2d7Zr~D zt3-vl@5BV}y?eZ{?{YPv zxLC~DG+9#qZS=hZkHY)zE!u$>GZ={iUh9{%e^qpHc2Vpsdy`$k6U<5j7y@jVC?tWH zf|kmJ$y7Ds*)a&jI%JGQ41Cff?O6oy= zxWYvH=V#aN-hDBs>q%v4&5@byU%f(=r6q-$y1cS5=l=9Iyqw8z+m7+{jI2tyH|Oc^ z;w?L1c18ACsAkqrveabX6}>^(7y7|>Vb|2YduwVfq9)ar6W{TpQbYesZ(-b2g`~!( zgm*+cl|gJXGmZFQ2W}sBZNR(5VaWnQl@H-}6@TlTjV-7i0PAZ{c&2YqA*gR$yM}_R zZ=bO#SDQ{{F4q4sGd#0%D60X$knjFerWh31r)hC#7cFbgw{OGYDpU6?$}9$_ZT-{{ zZ(7sZx2KsM-&0Ny{RxK7-2p6K_aRmT9dOZ#itpTIAckIExOHpdH}a)_%GHAKPuacX zmas>F{Y(Xxg_Lx)3%dzf8SJrKe;35t~bpcEs*M;KLONOVo|22tBEP7s7w&_P4wn9 z8-tV9QL!buo-cLUHk(ALI#QWz(Xi3^WEI{^)eh#hSKEdN#rfYV_b#8fmXnhqNmCx& z*lj7_(f&s}$F*OSVb8d=1$w4PFwhp~E9CAe=4Gg}Gnniu>mm_hse9mFRx35pk~W)F3}R678&`({CCYGY=dou#Vea$id|U z{0`pTxqglG`qP$06Z{BCVy2!nL=R>zQgK6!=B*2^OTgj(Q*w*ZlNlARFy z&^JYE3hR*H%4Tf+1AZ5Zzi&BW+-@wrMx(*|mS`tji{Hm_EeWFINoz6tozX@+&Mo2( z*(eg}lyjRCw;n`r0vqDzCy&W{gNm?si9Mra06jYRIa5sQxF(VR8(#qqHb@ywjjK*<}Hx6~2jZEfGRJKO}{4-Vb}X+tA>drK5X zxAK>qym)D8%WKp7z~FryeA<(9lZqtJh^Vqg=PS|}wTB&u0B{brCuy1VVxf*GfuQi` z((dx>8Owg8eLIYoEFbb31p_b~FH2xq*}Dg;ly7ehB>_+b`%;tMi121!GYEG_kTyn_B@v3@mzGx^o zp=T4#^{4k8uyOg*l?SyYL_6E>#D1<#8$(3MuYgK@eI;%>?6i;G-YVzz_EAo0r9vZR zAb6inMaZ1^@a^!2k%EqDTU_^)sr|}NBb%!hM6aKej|OT~<=g~)=3BwfmY#wSSe5P* zIViR~y)nUQ#l`6A44PEt`;hrfSB=*PQ1A908m4BRl5y%u0;)by0BBot;4JK@G2Ma5S)Y+uM zpO|90oA`pvTXQ02kN0$X7uD z(vhX?4$KJ(Oiyn#K(1#X8S6n}pMCeh`likHdv*a=%+>cyLW1n&{XtB!YqL9W$x$530cBCTiGj!&B`x?c( z2(Kig-81*{k(1L)&=TQeUTg1Y1D%hZtFeJ>iZ>j;Ow&xodo+Xxm&c{^c2L%j&v(to zw$2`7qt@#I0>NNu`h-(dwCr*=ljU8jEAF6++iHjjbQT-0e(4}=Yb!>Zf?O-}QK~0M z7nbz}JX+cx6$5v!Uts6+j~_qy$?sET-2>ENX8tA77JLjJQ?S>JnV-LqOSt?Lq*J-u}cH}~cKsvx=B z9M`YiSo4UUk;?=g$N9RtLN?-&P7}+o^sdfUk!)s)XA!fiQzFc?LG7RXjd=!wJS zdC6S7xwvKR3VJ$mRqbPJ;KS+#IeqNpNuR+lJoD@foNy+)ULM-q0+tDsnT6_945EHg zf1~X|mQ_dC*{z_gpXqpd&x+!3LXC)=T_)K0iN*gHS6>}f_4<4bihv4;l9D2bNJ~qH zgfvJC5<16N3mc6WFCEOMRp-SNM%U3VZ0lA=wZ zpRr=aW0&@diHqzHn3?@Hb*?O5b3VpFL2kU<;?nkVb!zQX&QsgvZ8}(ALDVD*Z*ftNp57KhwAlMGF;O*|6$6nJ;^0&2`HG}I z(lr*-#F?HjHy?XJB6WDo6Dg@&$>YcMLAe_9_il0CdWumw$BtsGq?nj`p7lH(t_w`m z%g1X1Rurl=PX_-w;5MF5b6s8Zp^co&ze;*e#BOTAYT%?YB{^bf-?Ae0Eu&SbvcM;u zn=AtDp1_5za*75GF05C%xf%Bz|5+xiqX#I+Iu{EBdw&PzMDm8Mmr@@HkEqedMaAd9 zxu4`_U3nj0>Mc_NHVU%9_36Xv7Ncitg?n>f?|OQ2eCknmE%R~Uz6T*vNeQ!T@?XlR zcG?CG!lm8cA~tg(9+U z%j|P-UwmYI&iC)1%srG67Xyd`a<8UlubeQ%!)XQ&jvKv4-N+SRgW1(@!#YlAa5AP2 z$Vwse?~9k`_4>O?0suLtwe#BxVrygTO5}NFv~V1>C(e#)&!6887#F7nH?g~V?;InQ{{H_NK#1BlHLPTO##xgtbIOs`WpXPb(=JqsJYa@;CbWXet`^*yp1MCnZ2#HQE z7kaATKmtopFcO&g+xAk_SMWiHAr`D&|63)?@o_nk1>k}NM|k~x@b`%Jy8h0M-Tkh& z?A)}ap9uq^0)P7pKU+HBUD%_);cFNc5o%fC>x+CbtaBN=P@;3S9}l>jaJ%};$7b*c z?9&b866&q>QnpK$ScA4YE~oA+mr$dJn%yX|3ni4xTJcDX@sl&+ZR?X z-0}Fsn$iqJ2hP7P$=+A49xFVv$VeENnDZ&urzy*e7tQy)d_X%v_bwQyRUVrLJ0dwKlk17+;Mlg*lWXU(IUk{v##3{^EfI?@gJ$gh=^&f`0xL>kQP6l8smQ&#R{NU}fM+C9EFc>RPYP|Mqs92Jep z(l)g}ySayR=jT1IT_XGH*6lD#abw$#Csr~7H;w8MYm1Z={|g`S``MKDlU6TzlSQeb z1E#gfg{(#VNe&M5j{{~?Q2OrLa$~|FQO-}V_!N%58@?!9jNp(2_GkNcq>H-OJ|vRy zyyP)6dmf*!EJIzESL1{Jj_{&&@w0%8W1p|y8`E<7d=*k&GMrGc9yfAsH8j8v0REe| zC4vwa9suH4nk7>f-aa~~2h2oWea5b98S%TPecoh)gI%*Ff8FX`HcQ%e5pz~CY26av zcwv;Sl(E8}(ia%7?RMLTQ03&(i-gxjp6vad57Q5UzSB$@!K?VdKL2=+o}RwFEvCaT zQZ~!(c-ncfU!0ub+!J!bA+_5zAc)+(akVvjg=t<+B*(jlvR$*mC|X5vJFw+s{+{=T ztQ0RIFwD%r4ea!q1sdSoBKb8}owiNuteJxYZdzKJ%WgB;UXGmSos{#bmAFfuM`2wn zI6KzCe!H%33TiTU=>Jragp;c>zBaRBU)=BF8CgEB)fCfwD|655`uz53iDr0PUQzF2 z&BoctU%p#Hn&wo~62qo%K1r)XL@~n771YKGfG1Uqnqe6)Ktj^gnBZzWNx~_wAY-Q$ zbX-?4VKdh)UkAgsr0(d(ElJVlrZG`Zu~i~tgU->5BUcxclaqb17fxzjceypGa=|Tx zz3Y9|X94KdzClY%VJx6&1L&o8|a&@Yi5@4H_~ zP%+P7W7hKf_XH|SLFGcN-!L3ZCcX4?z+24B)R)MLpEb)WAAZm0J^_ns3vU$NCU&K! zAC8RsSRvCSaGZc8gx!0`I;J5*Jb(T?b=ApbH8t?}#6)o8X=0<=9@lomxwU9h$q+;3ROwpV0b+?Xp0qM8MKIvs z)np{7(`Fi2Yh`bS!?4c|WD04uwZz`%mELq{Vgm@pw&?0}0fC44NxSb;aesR(S<7h)#-2};m9j4XMYIiA^5EC+5<{)5kb`6oP4zoNk#tF~?adNNy4 zvHnMztrQU)x-Rb*3!Il(i?$8>*Iew&)?Mt=M|ZTIt%PybjWsq_S|%O9F|;g8T%OSU zl;7?a(xZ5~xr}(@-3beAs+jTUqMJWg{=&K_c6aVd@PgnRu^i@yo;8vL;{jj?0if-2 ztk&vIQN%+(ivk7T!=l$YVg&_p_&5;G|Cq<+=B5XjxqaodW=y3@PZ-Ngqfl2~x;sqF zaq#IM;+*>F(3qzLAv2t_n%V2=3&&5T0%($X2=dp{SvU#We*9P%quz)?-(h>8t#AdcZhA(-sEn@L-R0j=-gv%Dm``Fn#P^Hl7)Sw z=r>+Uc4shRP1xHRU}DiSNtV3H%#Bg0Mh;fIQ5Ai#5=+n~NgxJGVKpslZ?nuGFFBWw zhq&eXvazTuRgK~2;^*jjJ~~V~Vt5E~0XG9>g7AzV^K4!hi%?WeL$R;VF|BN}l`~+# zRc2jzy8*aNkzT-Iq^7A*Zz9J!k*yChkv?1Ua=6T-$ z)UC-S3PHHG2!GIJJtu3oy3Z3U<+R&m7jc+EOk#rDCB?DmlFswN`!wD==zOP92a5|y zFmHIF==!W^&-b2(pcnmvAP5*VU6*6I99FS`y-kOPGcdb0+IZ#I+UEj_Rb29{K{MXN zq1XFu-q*+85yuV3^}#+OXI|u2yMfD}DSS?jk2lNu7|&tLQ(Bch8_raUxdD*m!9k~Q ztEXoL=D3;himY6R!7!HS-}4t2tKv5KgtqkTFj5iT9^cj}eu+o)&Oa4+QLW#kU};Go z3fC=97`b7Qqu?8shuFOg0eVX-+PTUbM&eBYrjG5#cO^1@zPL;Bw1V%I4OaWF;>p8Z z7wKT{b^wv^n3fh7m!Gc7NxxK?>bkgaU4G^0Qgty9t#f_m(UEQvXY`z9;9U~Wv~6`L z9MJ&#efsoau6*>*vM>rZ8VxD8!-L9MuZ~Ot>UWRkMjQD9g@qpWDxGN|Kf;$!D$)N3orh*bwq`9gX+(pKeoU^zWAFZ(3)4&kbCk46qRw^!6#7Og5la*PVOFT zk*Wnd53n)6`n<2+o-S5b$5vjw?=c z!@}t88@pvGNrKdr$wg{$tw&pomXm*crVGVWn}*{*UJ)J~tbz0=3b8r-k29Hg3|U*c@D+&H?DoTwsA ztD5u1WA0{cu~lN)(pyJv_Dv_gfB?-Z8p`Va|8W5>_OUKb+MeR(oU9Fck#H3kEcJ-+ z@sIWh78oycG`a8s%n~8^DCo#(g58|%b5c_4+}vD3qt{Ar(#Zlr9Lh8>hjp?&g=~YB*^TLL+_25~#0+mHD+`j75*iZ1)p5#!6wX)dDNS1w+ zKiaODtbfi)^)SE}0^Q9`ebUn6;{9s>71x)K^_@k<18;H8RMjtG+UTU*jMtv4sgj4Kl1F%hL}0tE$jebSnGQEBb8jhT z#rB|wNA)6Uq)XjGNq2EWDObTls=zoE0dkJ27EEBP>gMo?nV+*H(iN>d*)TC}&p<~7 zWR?c^eK0Zr2GS)S@AU{z7yiXV@piiLDt}>OaVuQp03LmPe4U9Kp;o|sv8=Q0+o5zi zi_%qEbu__WOz*dnqH}QuIBroRf8%CQ=^M6$!gXupN6Ch;haX4b0vJn39zo-z7)iqQ zTc-q*ISJupEDJW}6>T%R&(n;|ST}3TP6TEpb2AD2vU5f8Bbl;ZPH1y;FH8)Ywnl1e zZMJ(cwzUvWSKeS7*>g})S&w;rwR4SC-|&~jcHAg_zO;)`WB8bEMk_r5+1|#rOC$Ge zCee}C{b*C|s4qk$2$bSQCwwxQt#7sYi!1|=TzFU4)|!Vtx_Bad`?-Tt+w-kT1E{BB zbd%r?SAr@V?Z`Jp8TFq0cgBw{>*>p*CTqA!+yIo7T#oEm%3JpW6UpUb+gDnfpy}m` z=F#w8zmm`_D^jv?bjMpD)upUxFvoUxu02=_jgpc9B4ggGlZU4z;0QHMX=0F)4mUQ> zY{FLj^3PU`;f}~qXD8|zs#CH0kXOvcEMb0EvuBRH)Cn@|fTU18(AO|#Y*QuKL_u0y z40(zgXH;U9sAYY+P{oUnuej^dnX($9(bBo~&i)}*L7u(K@%AVTgM_U_OM}rD5{ZL(gflH<(z>?55b2Nk%g6VnWn~wps^4;Xp0S0K@dbk; zM8}_%kbw(&WDEX)Fs|gIqfX~>q>s2+SZ$rHeQNV?2mElw-x3_c(yJIPvB*|{R z^{R+7M^#f0iR%#U2$>9?~J@Q}Z zsh_qO$;sxiFI9Y=jiIT9g~nkYy1%f?e#g_j^}M2mi_?Qo zU%q@ak=CS1v^rdA4C5sWF>-VcjGHXTP)EB*_VY8V$!-hzmHeIyV|)vg39$q5AHu1G zOSK@Jo5f$mHb3!HHSL-O;w|}dKE4T2l*#@6y*JWw$zC@s(;-N{`YiKPPiE1ZN!$Y$ zX>cP&ReklTO7AL$D5dOwBeELrawbDN@*j>xPKT6 zlr5~f0!e~i;`1^+in)K+J1jtJ^s=I3KOk^AIHvQ&g>Y0y$89uX(HJ}IG~rTb)Pq)35r z|ENh@p%%kfNsF@b@rUom2g2D(ogv~GJUNH^nw$^_#WL+G<}tJ_o2t#=>$4F_IfEI`R4 zBZ0P!!-Zxv@}Ce}d;klxNT+k!nH#?wSrB4A(8=L`Wt}R2b~DeFk()dBp29;h51fYY zAyor~Uahr9m>68Dmo{lxZIJVe<)MT${vDSD{8~4vvm`y1Gap#sBdnUh^GYQQd=*HyEI;vjF7hcx~i1m8&9&hpHSH%IYZ0dsisaa9(db`omg#s};QjzEJ z7LG0QA&n;0%;MPcrNkldRImSGW79H!J<=d=D7$I>pW{ysa)&8>lu5lJ zdCJ3eD%1NEfkS6cF8wm@cK%V!A^dyHV|%pu?qUp9Ac59U?nxcwzekwE#QejtSP27! zRO9gs4D`Q7cHqt(QcI!^OL${_@ZWC|TzloeC^_m@Gi@<>HN3^~Oc*OXuVHyT!NRTB zKz#YAUOLlfv8~?a>eMWKQ2+TyfP+sKZS;3$WH)DLYpn^Q;B)y8B5VmlRMEm*U&LsB zQeaDT2)MJaFIQWOLAXW|AI~&1{Cz6=*KsS8m%@D-NIgse@-|j!Lk?PDZKpS8G`O^` z`3eyxlu&dAGv#9G>0XDN*Ml6TJ4}riELSIO0t=r_lAA%#ci}p2iSIn^NZ@r`J3E|z z)8CRT_Z1x-CnaUlOjY5}1t-@4z1)QHtkEK;fZ$SNx>2J@$z{pHNv3+Xv{l}075X?8 zk1x_HI(@^h!)Pz-4s#JvM7>*{a-~N*#pNx@EfI5eEqRW~-;rgj=?|X0$f4J|9+|d^ zot%Od8XWU$$?&C};@;_lV2-9_GK=xM@@uUazs*NSgjMQPV$Q-!wz|X|-^JC=Q zZ@&!~Yn(A}-{S7P8Kg$4PX8oXG<=Yj6SrGG&-ztjKJ8*Yh~8ew{&EE#w>&w`*Zz>D4WaUgc*^_Iq_H! z{IRxXR`+*6YN)3l1y)-38h!m9ZitbT(_0Ou)JXdk>JU+l;emJNsNsbw)rHY7AJg6I zti#sG5xd1vA)7%|Ua?*P3+irA@J1?a{D&wL$AI{Zii18arZvG0SN>Q?l4m;Nqk5}KzKO-aI;B?}M zcwg(!PY&oSVw`bmIW1?ZJKvZkcir`6Em&C_ znmgw)PcI?VM4(i;JFI-9t#V%W_TzU+TbPUAeBX5mHMJ=FS~eIAtR6NZyoSB778)hj zuzYy^&#wr=)w34UkDwgc!lHSF>#{3ZAQq`2Z^eM^?*cd8jWvWxzP{+YMnd}Q#>aoT zCf8!g!F0|J#TA1iD)j^JP;BHB^kCF$7p$7ocu`hs8yo9CDXLCRp4fV)V4X9uTkNh} z8>%!gQS&^Bc|$qvMxtt#VwgJldd5HI<0H|l4y}UMX~yg@-J8}d{A6)*vv{Ioj>jh6n8gOEW_)x4?#`K-w* zr@>y?yM^@?XpPsGp#EtI%mT`YxLn_UOL}w%J(=fUhYtT1QzZk_cK=>kVj$d2SsSz# z8<&?m7o@;GNE%I7nG7?{dZHnEmDAHLE_+5bWLqyT7EtwC1RL*J+7ax(_ARMSp2 zEGq`|TsPKo^J=814FDQGSv2V^T-02LP!e}?`bN`}`0w+tQ)(V?PBDk_xvTfjSgZZ{ zL6PtFLiV4bkl~5HV>(e9s_8ha92^1pSQ}N%2T_t4yRk(c8JmhBhAiWeZDq+bM-`RN zxLF&{r*{^7(a5sLlJ*j3FHmRCMk<#goVO-KDa2#GCOEi)>mD3&3z^H_&{)e`j4GP3 zdSiR<6NlxIIADxculR6-@(aYKEeRB1jp!iO2+yHPp7p70cq?;+Orgvt-cf7BaTW6IYN3}v|xH!E-MtklEQQ7OPj|>aIva35t@>A}k z3~DRO+g{2(d^;VQ{+V^)pvIx+nVBqvLCug}+Jn^64J1gg&FqCc&-syGJOc6FoCEhf zggCk1d)* z6F5OGtlj2@1K-z34dn)M-N`}-m(%8_i*m8F`;hO@!Hh_@q|SX6KRRilI&=?d_ZP9Y zrjy!ajXyR_j?1emhgQ6#js5yj$pYhf`-dgP#e@s8GNL8!&KoQ`%H$*JX*bGq5XTLv z*-4Ps?Q49k`8aIE_#2LYoQOMunb?l6gVzSa1-ho*6*?4%~1oXyvKV8^@2t!Sh zA1d>DT~ra&I@d}K7$JRlsraNN|7g=pYq)9n^XGw7o#83h+-1ZVA0+hCB<}&;s+}^4xuI%phyI z4cOEXd`_s&Q+8{&W=}o0%3WKxiG0SwYAxbd-ZkvFt!`J)3Wp!8hyXaReAx>*-RnJa zaVWQ=TehZGW@gtfT0pFwRhS`m%lsefMm!K^yvFBR5dU{b6|V$UEDe>u3nG+M#gg`w zTk1E}C|hF!K*sbbL7n&cn*#2!t>t(xdKo!a?WXw_)?ge1_Sd0-fw3Ly2oNElkO-3V zJCDuFb4$&3#9X>GxPSNSF}Pj;0;uW~X}(ucEqH3g)Ahp5p>V0h+=&yhUZwBcLarvc z7iYBekaghJd$vj_+kse-Ct#c~k_tVZr>f-bjZ`+^fvx}H+;Bh&C>$*MosZYNxOe8) zvQn(u7u9Skr|j=~op$Lg1Q1*MsA!yOZrE2NLi@$a#!CjMPf-~pE@3W{`%0NMz{ziaROYe`^$=h8Qv9t4X@YpI%CxBd=z?H2`T|Hdx~d)0)a+Z)#Z^??gYj1 z*z+R&2umcwhh~%Cn_pnuoU(1f2I@`rX^(w3_gZN;2rK~UXUdA1i+#s&^a53hh$KZ9>8HF zZO!K8-|Fj05%`HN%d^R*S+Enj9v*cI$8}p3leUvqmBT%I2zI+aQy(Pz3ol!coC4a%|o(;kepncmkh0|)V6jSftt0qOH4a{i|kP-qz-Fi#haC9 z&m*>(3P&U3x)ZLEKME1|$> zX3LSsNxhV0ee&%JH!&z>+2drfqCVJ5UfDk_pV8(4S_NA5s}>8SJ^z7mwFEMPLDM&z z(~}vWZqyey&3j+1pjcNM#ZU38_X5Rkcj)yBUc}lV5#&*w{fov3K}h0$>@>ftx9q>U z9B)!$5sy|-5Hg9+ofHtgHKy%mhFa_78h5F6OQ;Ev*sN@1s9!|>nu{z~?p2uhin~BF zS9qo3>~A||#9Wj(SR0;opdN4Rt-q9B2WWkG8vmzU`qKEk~KR>y*35`eMf zm#bXxLZhX6kZ=b#@e7UGmBUu~u+3r(naXRhxrju;w8!C_JPwKWRR2-B3PA>XTR`{A z`O`DO?a3bV8XZ+VP1k46?jO*9*B4K7TGlM|G*_#^d02V+)0a@%ilBtVkol`~9mr)l z=>hEwpj-&2`1=g(S>6G@`Pzg35vDqk<`Gq$LSc0#0c3L@;YxNy?A7@^P2mj$r1+(F zS35I?(~dO&`izVvkcox`!XE%4(BHE*s`>8SE3QCn{Sif3_9;4L4~UYCEmA!@+eAbs z`;7Mv^-bHux)?X)>6;~}%OELl0Nb^-U4o9AhrxaeA1ljtKb>4WygLqs1cFS(7{*wH z-KhA-ZqPlrf6X`a8o*hKz&3*))o&@lO$JESp~gp8D7!XBgB93uahvuxKJ@iV`U>f! z)Nj5r0&Dva)n&)sc@8o?LF41CIEKLC26BL}z?Iz%TD-eq0R>fucwMC~qqV3O9DHgS z`nD~YA}9S3;dHX={5m(_VHUv2T55yAC$?#LD39U}D02j_{S6?T%VyoD#q)FEj*zEO zX2~}@-o}4J&kaYrKJCtNJU`m<&{?%L>bRwlm$eM_-p`lQ(hgr5m9(!V=dY|$X`ti1Ds@?47!`}Ef`hZ&^ zh@;3Y%zy)d6rh9;Z=BbhU84fivUn@m>e(#>dVR?g)*HjqRxQbV3{N5it}oJg4C7oSg>;cHqAOEBYBD ziC7vD=L}aUSTAiBBuh7L_2KAPH{D?>P=ONa`T zBH-u$Va9$E-}M2QU5c8{TI7Gv%ANa;{*LYTM`*vRtS@M^zKJD>aeL-`1_J#e@fq{C z&xEjTx?}HhfkSQ-U%u!^UYDKOqLc@z_qI}AcWsndqp{`#wx7%gma~;Gj<&acW$`=S!XhE0sSOc- z0V7GEGZg@%p%3_xJBJneioZS~>iN;(0#1s{UB>Ekt5(v%H*w7XOJ5XSm!s{twCQTS z=K^R^aBCfs8=QgP*-MA&74B(~AT^;2@6j(5JWJ|bY$Y&xgnmQz3;I|2x#zr_xgFBC z6urS=Vr5kT%C4;@83bmqydd}n*$Wp**xsM#@pQV@j0ov~5-nOX0m4nr=zo0`r;wYgP=zbg7Pnd-m_8EPGi+3Z3|n+fqf;= z`62{1WGDy8{sozzU$3%F%6PY84lf9h@bQ1TQOC@uDn5r)UGeyLQ?rt$Ee?*Kp(sK~ zb=oCPi&Y38W5^sI)ejZU6~O;j9WbBk`K%A2;)G;8=fL+BYtcWpE~zz{ZAh*`e@{dW zmEUllUVt@QB8X^7EpkhI3`76)^=o~amAy~D$IX0UK2BM3-9Yz;khGPY$|l`u+L8OQ zBc~6qT&5P%ECj6P5Rg^$fbam+b8t@x?!mDCDAY<;C&%|eRfle7TU!g!cpy%TGr|rt zj_mlDQZI;3aY!4}(IvI5uF+bt(%W0mA0UNJfwH=a-8T;8X)r_KUH#kM?;`^sZF@h& z=&NGM{)76$!n1Mx%2Z*jz!s#KS2scO7ASHrnSfbq}rqteC8p?ZG zQ9g|EkCq~1=?iwL(NBFdU_paXt>7r(DQHZY=)MiYmMC+@h&B}X+T*BL=f{uWYP7hL z6jANMX=Z^aORdNQK-sXTMvk#iRqn z@ZOTtsrE{)OBr8eqq>^#j(@KfJtCazx4->Wx+gU z-^KmibF&D0VF}LzHymI@ApqNAq4QE4;r5Yot3(&)!7UDucRh-E_Pd8uxIKAQdeKUj(f*Tso2DK36A0yUC2P8$!a_?$!nNRXJ>mforN{4gJG zwkC)#0Ql{iYU3=z2j^3%(&j`93lic&OCW-;jPLT|02I12x6XeWNJkrs-THz_K0ZQO8FUOGb38tV53_ z&~unTU}x zp9a^3FpTJXWr&91&e%L^7YlIW_lauWSzEBSEVKCN{d@t6z(<7A6ogCWo~G}yZ$Mx- z0I-*?8c1lcz&`PkrSYbRgLY>G1W(IGbe^70@DNlnpmGwoK>VUKe{C(hF0r%8XYA$g zX{&;bC&QSeP<|CuT(FxFi3pEyNB~*UNulSw2=CG}m!!9?1zJ4e5MO4nMXq=c)? zfKCmdFG!>O{rf@79f*i`@3N7kU@N8wr{cbA#zku9T!a-z;xv>OaPI}!KT(2mAESq$ zU_VJe5p)NEZUt2w0tQ%hVCXj(Koe zM_4TB9x8?0Hup8=k1fbgaBR6S#_A_;BAMD$wrl<34c029;AJX8b>L&-Y;K{e=8{ zk)MKys-S?{)fk)NedMndz#$9*xx91oq)6tT-Xbj_WYD5`KLk~CTUSlrU~#20=sR!P z!ipL>S@oE3t?#kmsTp3~nP+@x806a+KQH0Uq5<42{BYP&DmfXoR@rjqrjot0@Nxv@RC#B9&1{QT?hM z{EZRwAWW!DCv+=|;w^wBn(WXp~G=tqEg& zWm_dAbNT_hC6QsShtqF}S^`~-v*ZoYN?#!0QOrco=0KnIZ3qbM5MVK5_#HWw@rF7W zQ-`0Px)j`tz#fR46^2d7Y-a7qG=5kCdd$pok)tBA(l6(aU%Sbo zS2KeXX<%^Ql^}LrFP_)W?*^7IIz@Cc88Hlv_Z@C}KqWxz1n4pN(DF3s<@_&&CH(HA z*kS$taRUvMa)SmUz=rVQ@)J3`F8FXCB{ion;r~f{|AEDP`ktbafIJb_C^N?EMTFGn z$-76oP8HbaF4a=cfew-*(>+ z2@;X`UAk`E!J6TLvROX>?^G|y{YM*z1bnN-#rrUG*-G?h=UcFO!A8_NyH^ZF_S+}$ zFGYe>5sqTiN7}l^CLlmX{#Tk?lZ`E$3p`DbH3`wi-s}pt$WM~JiBz&>LzlI12-OP^ z+I|CzAkc?9(t&I7SPHG&md~skM`LLPS*&LdBsHQ#QYxXLdSv1uFhvhrtMg;pk1W zcHzxbC)aQJpAORvybI7_qJ?(}VgD3LpD*TO?OkL0TARoncAFya^^}<&x-g~!JYKjHAl-~}NRX0j^^G#{kY9t6=N>LsLREz`Y^e9%gO2?|LFrC~3* z)e}LOGk~4r!Uwes5Tz5w5+K2uXv5gvl^yRywBbMt(ZT8jEaA=p;uCth-m5*ApM}LE zDmXy*k9{SoybU2)@hoF9Bx_9{YmKHPysclP(wCE4A>Z%NIvSfZks7e)p8>3ed`~87LW|X z19*UL`2a>r&;RNS5C+E=f&LaDfhD#DaylugO16^2&G1NLm5XH1Kc;4x?D1Iu*Zn4x zuiIwJj7+=JN)U^%vTKC!ya-prr}~A_@9}Z7*qdR-Ri;C9|EchQ{VV)es>p+*Hnn~l zBd|ptv2BKgncJIJpuu#OB1Ve@&TWIen6ZPXj{ze*r%$-pD7ek*i}stG3WuLA~T z{gJ?>hTWa^g#JAn_}CGjKhs3NvNH4GcJ92MUnK zSAJq_*qf1_CYA!HSFE)RzWZ^C2n7jv88EuAS4*J@hO0~cZ4mv4rd8b<)0H{5@hU26axZs*3u; zlgTs~?#1Ay{#Vfe3pN{^9*_*3%3j#`EY|X4@p*fYcv6 zJfxB?7u({~xXh1;dKZ#ZrBFQKy~Ill&d%xk72^(YQp=ACNhHx6b2DH)0ZagP^>+qK z(*KNsDL*e$MX8*OXp}5d04X?owqLkD$!F-pW}e^sM6>xQT~_%H=fp&}y;kmM9`HHh z(H3y-yci+vySsb(GbH%qSJgW_@bT}p)o^Bta`}rI6v@d;3{==&6OsgBZsPx|-I1O# z3=YZC!2)smKczb;0!atD!-2d${hl&i76bt&9{Ne02Md#q;2wlyg^*Z-e+j8LaL;63 zI2(Xju#B|7?N(h3&L#~15B*f0pYJVE!@D3^&p25vH2-TvZrHXOa|n2ArO@on?ery5%x>?zdKM3$kM}Dz1XA~9nXegf$AO8g=j5^;xJ+$ zhX5y%Uq@IWYhud{JpQF#?>n5i;uLMvQ9r`h)>@6|L0krFQ1H>SqLOcMLc$|*w6Wqe zAHfrh)-izX2jQA^r9y+Cd{xQe{|fr(!^vyS2$J)$QqFlcB1Fj?YtG?}_xY95)DfUg z*NxOTs{i_3IwP-3vP&`dCEN|R3+Ux@a%e!D=F?i+4P&BBej7<8Pu5!I8*WnMx`dev zTSAnEt#50sB37V~SGVE7SEw2?FLv|4x)YCLKi1x^RMavrU(eb26v>{el(p_a3-Ku4 z&iN|!)PzF2f(XQ@FcPjCYPuyx%=lfM&uS!}WUk6)3C%qIfc*`s-+(mWei-A7Z|>0g ziW4ym)i;I;BJ}@?->BX0L#V8w5QnawZwz_-pPMh!9=e{SI%IuD)bUZ>=F_g`#zO*M z(zY4{*IE+^3Ka=*GgP4ULI$rVc%wLd>)j7Mqu|299H{g$??7bAfN$moEDlr>vPB3B z39=>p`XT%P7$fBI+w`uyox&mNmkBrj_qHL+>jQ#5KsAY5K}T93<_r{haS$A7{CW-b z3m|9Hh2WLaXk@wZQYTcFghSDy*NVYp7@4@!qZP_>bnaMC6vs5@*$(c_0CBzBow_fCKRRk&+vaW}i1 zUE-*t%?KfydQTUTZU4hIR#JG8kn2+55KlfyK6Tv zaht))>hG@L;v4l&3v26FZ{Mn^w*C?e!dw`d7L85|H!O3VZb<4MQkqca_|%tJ^Y&?; zG;0J$J;$tNlYH4|?=;jg6Ixz6l$zmv0I2A;&c}nJLYYx5T z%3T}m4s(LgCdYgF7MDAnqc|Gy6L4OIa~pBeB_&8&*x3~zf2=R=?lT#E!n;WjTu|Uu zQLZcZM!Oe261{u33-4NctAJ*?jfRu+d}#b(W)KYt)eE1ns3pPjae@WfC%pIssw$P= ztK3}(1H_xZs?HbdL`eunNqohBjYdn0AJS=jHz84D;Nqgxf;iYvN1}q3qx0HHLp{Oa z;sm+8lJhNcp_qg{n~$`yXr+yb!^X(BydNzMEXk6~-;0Z{1WaDX%x}h0C5BH&_9e=e z|N8NxG*4YbU4!|DUZc9k^j)#HWpd&XVh*eFAM920MWSzx5x&COq~@7_&QqH{KK^91 zZL=UdXQ(tTZhwQ(`hAPnN?18rZHKHmDESa?xaP1N(#pUf6a4Z(L)DG2g&8APgolbSB%M_m{GOERr zWju836=<8LXIhDsTm}RAu_Hepnrvgf=`5RzKI9zjzpv=-9(n8Lr0{B2~B>lb`{dv(ojnoA~C3 zmRK$6=Pvf&Ba5oUny5H$8;)v)#%w45jigHHD|74r87!5Un3(-CVBhui=TPO;s2==} z9aY5s6Ou7R3=CI}m#q0Qz6m)a*&gpF8wCtPdZSbb%R+!oK^`CFubZOo&8V(^1v|*s zC*qcI1RZ{ey4v&D9`%ry9EqybWu?{;`8_MFLjwz{Qc^J&UPX_inrd2q$eMqz|GfTZ zrsI!r1=i1a1^J)U)k|~x-d6vpq188IxWe$ixa04UM_@R5t$}t!8Lm$f=G9!KSQC>J~L!E^V`8*co7};^OjKn+pfVXg*Vz z9nM-Yq~6Cmr=zCZ#Bj05b5!9v$1BnJZ+xe&Qrn z?_l>T>n$D*xzmf(!*s*guw{Gy;0yx}_o&;bqxhc(&UtTJB_;hVo>ftiW`M~*k7b$R zNa>S*G^b3nq*GsxA0~OOu{9`{U$#}S^~|tIf$l$RN6FIjEoQSg_k{{AHGg{f7+Yew zHnWDEd8$5%MVY~1CUwy}p%k1%^^$1ElES~=F)a(<&_id)I4YJqvfbvhv(m1i=8cUC zq2{U&ChO-vC>#?+lT?|+J|GVXi9K$9WuH8jzlD7|Y<`38wMNeO;*WYRhJ@-`x+A%Q z&=%yNVRR#Ebn?^fD6sAx4Rce4`JeI{D;LZkQ|^RLhv~WCVw~Zky9`fFEoqjcqt+8M zj>yOaJdW~#o&A7lEaty~zWl)!QWzW|ZEVyMVjs=Jp{1=|EBLYJ52uXJm2 zHHo6Zm0lUqrl}~U`FMQ+&nY9Ly^<@kR`Io`COn=A_vk$A6PK_7#lJ`OQLTHJ%)#lq zn2TcCd-;>Gu|0Gj>7HXm;Mlm;tJweCC|{6C>9bu>H9I&NBI4{M)4($rk=)B+w=N8k z>)J`oJaj!OxI8beM8Uh$DwF3&cX{DtgA3U)jR$#doHB+I+l1`c#TBwRY5J6~`YiGn zzo@9lS(N0yz{ihMEo)39Oz(7aA zUyePu0xF5OZ)4OoZbElBq7MPWn*ySj|9KnoYWy~|i9HsuBYtKc1-;V|# zceQ(7Sz~I+r=eZ5I+j$q^?Z=NuuD=(y+=APegE6)6jP-F!HOiXFTMn0*S`DF+g{W+ z<1Wc^WRD-b#h>F|?}TQz(d|;3ST!6OBzhxkO*@9fd3!2~s=Tst=kG@!K0h#$o=%PE zJZpM3<=UEc?nTv-o08~*A7L}gN~_bGm%BMLpcWW=&~^j~jnc-Cwq<1u_hsPGI9b+A zGjJUlsB2_;_KujPQ*`*&=wt#CNY0q@`~fwp$x^K7Crmwd3r_ z_vUa%HG!g1>Rht&TmF{XU#qIG$hNg^hq*JZE2Pllj_>MZ5t_G@CqF>?D*MUUZaT;@ zqlZlup5{rL6VBtwl49m8C3CNqh{_5wc`C&uDic>c@keuXL8_Q0=ijHR@WvQOu)=C| z3CaA0Rg8X2e`Ms&R2$m2pLtkTx|;cRmQnOO>w4^r<)I0PSXU98p80$vq_AGu2NDzgg95+W zLWsz9#&;L___Ws7a=eD07stZ^g0y*eaBz?aTAc8wIRAoHEYPV}r$r4J8ihQKf{)qQ zpg9o-M<%qIH0fr>fW}^;Y5hdK7)VG+JYiJOZ&_~{Uh={1(GEK3j%d>RMS%u4cfK1w zmdB)mexaDppc5W6lt}F1Hh_jC&|Yu<^vxA~Omts?4C(8d>-1|%xC>`eD3(C`yn*;`1ji7;AJ7nJF|Qag-%8A zH!CzaMzDWudFhdmn)t=2(2=s~{vB1P!`apKbw%hUj%jiWZd>nzcr5GC&`?u*`+;ic ztqX7Ut>5&&Pr>Iv((TI|@}?xjpe2hK^u7rXgoSujK8F8)9RGP(pS%4O@QWJPgG@&4 zx>cf$wKYU@BP(hYxEUW}f^_?!FBS;aYwQ*SSpPtW^N?0((WB4;>~ADpSWL)BW3Nlc z^WjYf*1od>NC3+e4+GWW-(hw{%%Qs&ZR`NF_}Oq7gBD3|=&4(n|DA=QoBuqTp1wZR z%}(p}gw|h^lX=juny42VtZPMp=@2WE3c>+u{DM-WOE`e z&kpr9KEf<^7=A&_;$Rlo;^WbOzaak47oPL+A^J_JUWY-g%EM#$5Nb>RA6;(&Rn@lr zj~_+l3IbO}lvY7H43G{3X%5|`ARW>j>P1R8(j_I`(jAJl(w$05OE>)I27T}Q{=fGe z>PKuIqSth>8KaB!8J6;_7j2FaW9-)Sjt;*0R6>51HanQrp*~pN!rvq`J({%PBDSs>)np47^boTee@H5FpQGAr3 zf^NPF2nY0P(Puht8V}v~PL^&Uk4&BinpGPb7&O}Ony``d)YPC3IN)7r_Q#RXGL#)k zSinxrK%-iu-{shscYUqD?c0J!c<|!_*W*-_yU%~h zIrM!x21n>WgkP`PQ%g^2{wbt;q89Cq)XjFEL6DSjZAa+f_G0vzTa6s$LbJ*Ck z)xaO_d>HsiFwlB$(^R9vUS`L6l$ghHJrx>Z8F4Zq^qxl+1+A^JS5_>JcX58c3oe?U z@Q-#B{93~w+BO4SwxNH~{9EW31j6wzlx2~CioVI~;?E_}S~ zF(EE$uYG;J0P;394mkcVa-+69VP*?d&_dP3L;{fwWL3}>7?jY-s)hxQ$A%^=2zScT z>oXDJ&>uH{>^YXZn%3*E+k1i_8qmCxCCsWGy34}CejYFU%>!Lr6rS0M)c+@P3)%pW zxu1I;narY5mI#%&^xqzm0xb-5VgKSz!7~1c1N7^K{;r?2Kfbw#fLuB`HK zt9?5ay0*RGR6oW!R5>zwcd0dZ7WC0Fxj+9_=;lSf)kdllTDo}ryo#FZ|L?Rcd;oGW z2W?>Ck`mHg?VjGXH^2fXthYx56HC2UsxAc(V`XL4s!WG8I#v3k1KkG^sj;Cx7K*Z6 z^T!D?&@`|87}obFF5W?PdwYAI_w)BJtA{J<-6JC!3B*z}6-Ho{I|$-8D7t7ixbCQ>8b;9}lJuIH292 zxjE}G>bOfHTgL}(H)Di^k*#enG48q7&@vpZv7SKM$1M;RcfOlJO>_YwbU{s=I?!7U z!@K%dLoL89N=>l)0?Mi8zf*AoQODi?NyTRvcF3Q~ZWg*vX)T8Pc`M9K0AHl5<=qELchzXRR`iM`-uYQc-s6BAl zP$lSNjo8`qA4W#M5KR0#KlCp{j7UHmCK5j*ZEZn*N^~xKw#Mw{X>4~V>$Pb~z`73h z-mb2M7`SmI@fZBqKZ8*9K6D_4PPHQ?7PbEmdLJOOHhFYE3huv1b4T0OfBx|idP}|f zqEj^2Co6q;`-!6h*E_gCPf?y@jp?aRrA zb8vk+G9tpFe&4kk#)P(TC$3=gWPz>NUqOQPo4`X}#{vNl4j0|zO z_WpbLZ-_M91;YtCMOZJ9zKIW_Mb9+`Q2qY79Cl)|EUcH z*3z4F7_ds%49)Ng4f!=H!C572V*1oGTRAigFz0)5-Xs4oG<{}X9kyp&F% z<~siugQ^1d$7O?UAs)tJXk^rUEENqC(AhWR*rq{slK{1CsG`0x1c;P?a`;cAL2aG} z7MlAEmtGK?3EV9Gg`XYBR;D739hf$^KmryVpa3HYC8Zs*m*7K@^VnwE1|fL-aRxZ} zqdwxEo%o>n>mesASfgimPykAVR_TUYGgAM+?m{tM$-lvC?}hx2~KwCTSG z`l-J^oQ9S*`cCa_Hcrmee=(^bAHt;Q#wkmH=*XxjtNMrW??5^q#!H__e4HlCPTC6k z)dI+us<#qDL*ajMsXOyT50-mmWO#Tx7=Xr^IXP%V?vEkd-4P&CQ1bsr*1BiM?hxt) z8kv~QF?&9owH_pfX1QR5%HO`0`J3kVfT*_6vGOg8ddYWgilY2{xm@MUV;dNC>=E>{ z|4%z3P6nB@%`txdH;)3vb_qsLhYmZ8pdk^L_dhgbzf8qvFz)}WXNZcEL3RUC`Tr&< zI3t99PkzIhADl(GlZSwWh)}a}aLl~*cz6S`7azu-;DTTR_&eGSCkPI%61ZwmSM&r* zQ@J&5Q2iJCa{er;Ne-QKl5xu=<|2EmaLIZ3fUuYMf@S9P_VA)VB<==E7ogjl}~u#zKJuA{Ab3?*JH!-R16J8 zo2N3*PO{(^7Uzu66J{36`uZ9#JMs2@zH~v={M)zQ1Ma*3)gQ$(@?9}(obQS6?Zu;{ znOsxBP_e5!hL`S;L^_jC6!fBB)p^qk2n~h})XDn|kvv{7 ziUl=S`!>YyGq*cC2bKBFN7{6ISxIbnVk)&dX7T}9%D3J$OpG2GEQb`B<9ZZU22Eki zSs8x(QZ;J*xH8I7T*!Xj9u=8bbl=!HY^1_L?kcOMwcp12 zI<)`R@0$T|YnMg_g-BR^kd&IQXLLLZVWxxwjSgSDcyEv$oi5)?&xi9{)QUkGt|bM! zBazdC?^RV3MRb{+s(gLD!Imc`?(ie!cz@?#x;5R+4HWYmA!nQgl0Y&8vYiUHd-@z9}y>6avR856rWy)-t}P?Ly^O7h3gc>59OzsX0)@_Cr; zFPN={Fr3r?BTvs{EtKd!kC~Ynfr-ByZ9#wD*;jp{rCJ?s&)eJ9K3bK4tNq6k`W

n|)Qx!1Z8=X__yapYS}WMtwaPg`B{ z>HkMBZ5{Lj-|Sslp_i7U=8~?Kr8l(gbrgs#9&MY-+YaC`9T|W?s0~oT%xbeqpG}OqaKij@DiP2 zuq!YaXyrL^7FtQP8ag%E@Oi_cgr&Hz!u>uk7iy54_<}8qbFUS9hxu76^R2{{&O%s$ zotJlEuDYYACkc8R8+{`~q2h8c#;R^D-{C)N3wpL2Gxglis9BZQlFRrMtWap;`-6G! z!j9+pozXnKxjOu@h40Rs?tNyt{VRFzy70d(Xj*hkcL0^JtlHvd?4n4i(oENaH$;m$nUJ&{WkI6a+*MqDaz|IQqH z_{Weu%sAe~KJ*s-(&%Q>%SzVhP_wb3Q4qUGE{Hq7aTZ`;CSV%Do?Q{?NX?(QcM?=F z4wDc=`fxK51z}bn0J}OpaxNPsnvjbd6tUDcS5q&zI~iE;QJcK_mddv%RW)OdoXVRa zd7}&Wn$^^zJtp-Sk|2r}v1#!<5W-m9U0rhJJ0MKmJmX#8OfSU-{;V;(4wErc3+%kAW}e$W~^1<;4!|c@^(<9<-;N zVF$E#l4H?lz7BiWvOJtyuOi5xrW&ZHm$^ki_U`7zDcrM_wi9ohGY&XjG0_2CxBR!Q zofy8PtlB5)G-P%uxr1r`vXVA7EL*FX=v^nDKhyQ7d^aTu%tpD9nd1G}u$*!={!6gQ z$eZh<@?Pkv4KZPiWd&1u@+kt7CmM`ISyuIuT@2U-SKn>g61^)w?~%#mko)s}8^!w0 zWK>jsJ-qwT1=ub7p>z7fPey}c&OXN#ImR{j8&TDKfbyJT)t;_g)5TfY*l8JxSvm^h8%0IMB>t&5y1m^kpX4~5C@7NDMAW^mp+07AeL+H15=&&+Fv9PIg`gZyB@AfW7n?Uf- zopfb(=?1@02pmkC65{j@^mh)L8Z(ZhXVgvPx>g77ZAldYN7OuYE-Uv!%iQYL81W=E zuZRyZu~n*45_YRr{m`MYqgnpBHIS8AW$#yoCOu*LcA8#%oHxf<=AgWPG(U6f*fL2} znY~PluW#!CQgQZI{G%(?u9gNO9wAp|Tnlp0->~uJ_4Br<2=kgzN{X3dPFDFg)s-#{ zRdwC%??M+VGXn8rIOAXF&-nB4Lkzmdgo=NpG1|D#+F19oUABhY?xra-D^u#bmL>xu zS-IP((oW@rWo6bMQ-q@l1elgx1+t!B9hX}WpypbtRnyvj6}r&cqOjRdD=@$*>i_U* zmUmF)%$-XE+g(;%m4|h#il*G2j!u2IxLO5{DyDO^?3Q8=Iw(hOH?2&ERcES;KAha+ zU-4$9nJIXO0c+7Zo5F~7yjp?@&CC^T8~lDzz(#bN0To7+aZ)zLzOfE0uv zPzx>qP8wQec^_7kG&}l>tF!9-G)X5do0(aREcBifKM_u{nwi8s)4kD1<+x2nDPvH5 z|Ne-_HsKxt@OV|7(7acwi7m7EXs7JF%3``YO~RVnhEmz;T*hu=&VKU}H5c{#R>eo{ zgVB5y`fCM;d_^T*@L9G*r?+Txnjpt{*29Oykf?HE^hynOTv@juMQcvejlrG2WvJ#k&zBvF;5z>}`>;#jZ!6;zF49 zF9Nsi4cwpT`e4YgvXNuFt~ORS|H#ROA31s^U5~ttd)BXV`$Mv)=f+a(pcvsjjz~vEa!tX6!i}c%VH#AWbEC0DQG$ao~tx2kl8mmZTTSQ zD5)tv-iC6`p4SIF%4r5nsp9%GbPG3FxMX>P+?dDkP`T{S_XN7(ff_?oOn>U}0Eh)14n9ez@ zHmRcH%T!e}g>)@C9NjD|gv*q-pzILOZ3F`}3jDUM_d}Gdfh9C_C~Dik=eMOFVNv$4 z7PD1oZa^hq?n}P2<*`QS+5mr3QSuna%uCMR}ugy#e&#sM8@_FL8ajAJGJvYPOj7XMIqnmuJ}LM zSg+ny@(GT95?1CQSIb)Q0*z5A_Cn`A0v){7pJYFfX+!$gLe!fCC#cHCOx-y+o?9Sa zz5~$M{rUcQm+#p28)IW{YCP8k9d66HZWqZLf8)NLm-+H+hedk75v z+4jmpN!_gBH?#y@@yEE&%|53Cyk24Br!o`f9ex+PdT}@`sgx@zIXPHES^BAi%AvH# zOOeOe7|O7|<*|KVtW(B68uj{`v7i{nudXmT`jIMY(SaQY_Ts{{iAmA3s%m!Iw0VTS zLP-EjMwYUr)~cG-TA#46t-o(uurIfBt+u2EFR52$E)2iNVPrHiGnNzAPr>K?RpZYK z;FgND>HD+%OG?XZK{k|`HqOk-5T}I}I>N~pxtx^=+}0DgkM`GEm7hKrT87RBB+i=y zPL5w839j3ZRBWUoO`V3yTrTKUfEPX)Px5q85qU(hb?k~kscC(RAcQvNYK+}}VVAkA ze6XsXA6f35jooi5ch$&M?#4Y@5iqJ7)uNmIse_pfh}}&qck6m41fk&D1u5CX0s#bR) z$7uWh#{3n5hF?3@Kfj#cfWi&j&a1jp*hXfd)6>0orv@@x4`z-slX|R1qf}Mbe~fM8 z0o-Wp&ndC5?yoTf2VZc!@JP5drYDW^K&CMJe!t^vv6~t$9NEmg-(8{XTS>Q&S+XhQ>%auO~QuPfM9B^%RSnhpJH> z=FX4q)z)4SpjcYyenh$Z4tZkr+M8_G86Gj$S@G6}y;v7TTK?S*Q#UQ(dF+7@;?2c# zE3-YzF6g>EDs42#{1&0F$>f(VG=i3$c`k3W5}_L4hkk;6_qRBH(pvkk1*9DvRRgc= z@vRwG(}*_DSd#2abFd$-wq!rqO&TmKw;A87)*|2qvL0OEA;N@a{2iOFoVZI0nTiSc z1$^u8vMcxVtlUZ`M6*Hu=d-ESN=zg}XmZ_^5-h4MMFEvbzIEr)DcuYqw zC{#~Bx}oTZkJWjKv249CCu@&FsH@u#cKipF%4o`cFUBQ$Nt7Wt#CkAvW+uAro^x+k z`)~-OU~^o0f3K*bBsIcrRHmTXZDb~_K0}I^ohwc0a1tFn@cl!AsoArKIZz^W ze$}ozBZYtEJiF<{ZFh1%;T8oibV!TLDSy*=T48cdD7u%uC!U|Ls$7Xv6`^)T^D77p zgGOhi0~N=pa~GedU8V8J456f4&i>N!+#;*1#o8H9T|ODn-qSY?H|z0S*79R}7VSn$ z=hd|xW|G3zSC6(Sheo5gKxI|!474%0%=hjRgf2TVfwi$WE`;ZHFi*-~^kZ%I%SI2(g#5{aSpSx`HIwRMtat|D+s=9r z7%9f{W(Y4E9NRISrqfDnl9f9tZgYdrbnGiv?JK1Un*#Fni`7MQ+$ETcL!%zZGTt_zEy_+e#_W?vZKIyydUK#+s4S)4mopnsMTQP z&`{6TYk}0(u9TkP(d^;z4|)yt);k_ot@k=_iFS5=ef#8aE@x?Z1`Ii;AeVsFs*&#{ zy--;~nnpQEE4 z+fw+G8gq5BSCXYg0>Jtqh~~@iZR1N3pK0aVn_F#ZahyUQ$x}jEdeTj@7h1PdV`yW8 zbxm#Pw8+Zf)NZ>y9PD=M!x0&I)G!`)h)-`Y52)$Y}ObzP`;yVc$A| z)^6a}p1HSQ7cxh;JHyr|(MQ%zlZvsY7jXOo#r#7g>Svoe2S>8Hjuv8PrI~0jPwA{+ z{jPgMbJwo*@3KC_0zc)ekz#dau@VMqBj$*ICZ`~iU-slr%mth`cU$qz{?amaRkNS1 z>CV2ATM|zgYk9qY)x3t|eo$<1&ZLiDPN_=#BhT)G$sm%wq3WLYt5w&p_+Hsd1*_`Z zyS8$3mk=!yX8F`t*}Z4E;q#W0UV04;eN}D)9+6^SAAU;vZXn>QjyYO$t756*bzV($ zjCB;#89OW*J#4AAZjfWGt}qNtNC*ABx*e>en=1(#{0p?pvDpwR0UZu8paC}}Gd=tM zLXR&KE1YQEVitAjT;<`9P3_-`M=eJLEH^VV1~3tA^W#d@bGz@SUt@!%b+mr;W#W%;qJY?&D3mkX*TqO?! zef&Toa@|(ZODnYLnT6$Q=%ePG#`30^mdkCt38fiIHXC(UqjS5z`~w<-F{Ug~OSQ96 z>!{F}iNtrd$!*VE|W!VMc}C2vt6|gv}7hH z&!!dzd|({Nj4Wy|sNVP~!*_sN{c$#p2COJmtG=B1xvi?1w@GHh?VZEJZv>PSCJ@$7 zi~i?ykyi|v&4a^OywN<>wewJEEy|H7IbD!oQ~OJuEB zFOh#M4&Fyw)t~J*gmiM5U6x)hE!n6=SzdWXAR1{ub*1k>2P2R@ZTkLamm6lL6AO|g zvU&MO61@$Q&h0(9QmnGaKuy1@W>7`UZ7&TfirEE&CG6j*rg$-ivGlfl`Wy(2*b2Q1 z*GI^EoD*>ns#`zjCj|Do658))n%umj(Qws^`0>6`F&Yiwa0-5x!HrjW29c2}rp*RU z-&rJm-e3K;nbnp;l^dQhkW4*7K{~W=u^GSGrSG4)`K8|r3(r9|ddh^QN zNpcq#9>>GIClIR0-9T^JPnw(B;3=q5zu4Vho7XJ1RD4Z&%x0^2tmw6q-$x9yKzo2<;3S5M35 zapj#@HnSS29C(eKzMql|ffBSSffEM2*;Hy3t6HLyzkWC!4r<6TWzV~gIEG!eVOY7( zo;LOUfztW$h3yjjjm>P=hbqYxljbuuPn>SqDceU0gylG^iFr@hLKlA(tu7y^6Vm%-BJ-rb94X4y%) z^-ZJn!)D&-wItSQz#!q~A_gW_rpzcMW2L_7%dBlR*))o(7&41nT$aARQ36ulMN3Pj zVNE?zQxbX_hU17+xU%eOH-rf(wQ4x%Pdf6CVw!uIcfqYte$c5bVs$Q1OusyglGxb3 z@=8?DsN{CzML32;u|?vIcZo$Ze4#Iuc=Cw>Mt^>lkdLBmS~xp<)%=H5Gi5PYK}LP! z9UDPbV<~}HLxO<`tz-`C9pyA!W+CfCKRF#_>y2pGU@tQ;4OzOKA(P?dmUTVS1v=W= z8nsO}l?1#L4&a%UR*=2e5Y5k^%f#f+Xk90=6O{%|Y$ZNhU6KXKBxVx95>9l29# z3e*{=6N6d*E^%smgV%hjMSODg{mVt!DrN)PQf}*^BiE@Tsm@TM?#guMjAq*{|H1SD zWyopnZoFTVAup!)RkKRnPsQ-Lq3CQLweUO<`cB##WEK!5KW+EumT>fnXDxSs-wYXL zl$vt;eh8yf&db@niD>+=_iru1c|LpdVhWpyp1rbO{C9cB;H?Qdb8CjI>T%5y%$F}> zDzj=yQk{D$NoQ(vyHbj+pX`p1(TzNlukvE^T5a~rqNr5Nj`Zb$#F{GF!bH62mcWJZoyp8p5n(q#W2+QRo@vr?i@%Cb^Yo*lhfKG0&&v-oq(# zDOWX9JRG;^YUuZ5IN;wE_;reWcvdgYb|>~=IUhGAZQ-2Tz|Mw;x|D~y*-+AZQgSrp zBhK}oTQR%^8ZW?;vKQRq7xiXZRheMzJltwuHom8%II%7pz13lI1|OKI#kT z{lf0}Ra0u)XSGa(>P;8}rXn*&QN!n+)RH-y%i)me`CNq2R3V^XiTbLjYKeu;+W8?X zx~P`V-NEyRQmmIYSpLS(+1nBytEyZZORe~!_?nvQW`lngDI=e%L&nmSF@%FDiXS+a z1x#QBmMG|m*e#ca{iO^oY!x}0)AQ6Y6j`dvn5=%FB=`biAOSU6;^a7Xw$PZ_y6bDB z7fwq{WV!P|&p}BAws6v&a?x4EqI412#0=w+kYAI$#ZcqtuTD zETWe!!d~0#?{9TWU}hUm7yi*Ib+)-IrtR%plleP1rSGR({3QW!@U){+>>ls!^|4>C zMB~Y2PW=_{5;1#x8nU9m=hsYAC}*Xg&Y=>`2**Pa9lQ z3Z#MEm;BOik=QJ?`TNlqm!zuQQke*?_3RlO(OB6d(aafaY0a;)Ev8Cj2*w&EA?~mo zxbKeF)?{|w&a_iCIbp$TEUGCH=Q$qxGEMBax%s16fmw6c`Qr6X%ES24ZGoePN=$+C zMZ-8<;hEVk(2P}U(WWj3Uovt=mB!#8;iIsrPPvb6-HTkx{Re%@H)8uTz}5fu z?83Ry?Z5h}rspuZ9g0_mavLbDMq-I6dZWL^Br1|daTo7YJZWj(1Lg9;#U;vh?2Yv7 zdB-3|Y~cxuzJn=i)rp*nyexCB&FE!=TR}1qZkHq=m~YO=X`W{^Y>=ZDU)B#HvqAr8rcb>;80^7L*9bsH1p3q{APy;wX?6t>R#E zs}I!r4*ZJJ`yC<1YJ(@Db(A1rX&PcdulBfJ%S}TrjHQ%+>Jx2DYV*LTlb_6wCga+XSk!W1F6I7MmMyFCVtD_d=TSAcj(l^yAfw~e+=LJ@VDWz zspjiSa}6@Pxt*q$lhEledwXqIaatU(PDStcH>m62g87=%=KQ23P2Pj`m#X=jct-Q; zERI;WQH9Fy1RiWqGSO_8kJRJGMmCR+v891QSC02V4MEA$Ojy{)`0`QJl6ra3Y#(!L z>m1LOKy-?jIP;pr{MhH2{B((h;&Yap`mRqXc-#YQRGW+4);JKOypOztM#aw>36>S9 zuOJ7L(xT1`Iek8!MelwZm>G8k2^@6IkC*Pu_ib>*?!1MTA!T{ExKaF1I@s84Rz7HM zvmObVyMg?rI}&WJ|LmaL%UwK@P>}CAh6ke~o>OY_d%Lv5`K|f-?BGNy?Dy-5;=6HX zVl`llmzqOtD0wV57eX~BW1Kft5COH(U0mo7Kb%%_Q#cSCT~Ro-K{{SgcgKmjv6 z0^YUf<#=lf@-XyY$DUU<6-*i)TaW65?N7t zUBXoJgxoV$6FHr#4+FFnu5NWcezj5Z7M!tiH=O2h&s`a*9~A(`OMx@BjqiJ-N-J|k z5p8Pc@eM~-(DA*Bm0Llv-LKKSxl_x1bkD$V738e=(ZEP9q-S60IG*AvMj)^zF?DsI z?gN~Z`QSp9cgG`WQI_JmG}9ZR$Q@x>PAT>p7Hjw&$b&vuWo0bzNbrdwpy+nu-=N=s z!0>!g0VIW?;vV!HmyuJNc59ho1++@Je3y&ugd>?|t==3n9{gy|c!K?Mx0G#xU16vD zpKd7Jp^_2=nFOx8UAvyFE}~tv zwey;fQ#cH4XIJbAou-NTw!d61TeovdfdjTeYT2Z=Ek_&c7t+wUrW&7rTYCzfcXds> zy=!US`Uv9e zF*p`HOjijRZAP({5D9@`z1Ubq+VJOB5P33!JO^>M^ddpQADgzb^?1fDV)`>xZC_bf zS&V*6OJ*@KH~rcn^mAHv4wt+3{YX_c)w{)LsB+q9-Nb#tsgN1MB1mo0 zBx8W{d^g-2C2q&=^dc_TiXSAH(VXhT^^}n4D61ubjh5=n*1W~Xu%q}_d?BmO_)Z6Y zT?#l3R+ld$hcyuFgIV3J&}3)l$SAGUFxAj7T3j%B8$Ry}Q3*&SI3;hXE$k2#X$_Ty z#%-@jqQ&7T-%mp|(6ZDecJi-+0y=5i(cMt+=>%Mz*CvYZ%h8Lso*sWjgag1SVs2)h z(_sSUn*XYOCZ~btgq#~)Vl8A`|7r^2Ve8;q`12LWcXK*@%J|~SUX~DbRLEvW8iP}7 zXP}MovRwtHGSR8WBga{&t10m_RIqoR^x{UxDh}Njd$?S&=KnR2MwcQ-=|`T`_$i1( zz`?xQCLANr84t6(?stle8YymRQB=8ZtCd^AFK5O|gQsU}Ew7~IEYma-(^UsWb=9<- zAlRwCHRR-Iz>9+tq`JXB!m}v@LqHDCNLROR^yrT@6|^#v^^&^>`E+A$*R>ZN)*btZ zg3ejSWg6)i8Ho6jve`1*P55!Jt&A*})t8N}#gIpVj_$|uLT~xIT#Eg$iyS1{^9H(D zr$;l7C6eAtH?Zg^D-=hO7BJ=CqX^5=<7Lbe}u#VjQH)P!PyOwcmjIecy6&6e27wthKY^2 zmDnQ(2l|LJsJNBCV|z#Web6#4ut(*P=y9n4p!fzLvbCy_Y1YQqO^o$mnW(nu2SY;<2*F%jwk5nzrmH)s!*E86Y^e8u+0^0d@gN0p748b0aN6xA|dz}12HugS3 zvcah@OS$Gv4Mig(a=BG0xgh<4y^540Qz$W^mI2$uIDd3A?Sr8axR!eStXkK<8_7b> z|0eNx*Wh*jI((gS93v{SqPzk~Stx{*8g7OJ?gggOHpr!-z}WOWqg=c8W}lfEut@L~ zeEI2!&FT6@>qjb{PcGM&zG?+SuX4gy^c$wvxs(Y0`5O+`fwk~O8N zGpJuYPbyk7VuJ*Rn>R%l7kd|9V(G#?2vjq$89pQkL4H#=v_&lp-#$=uP|c+qDV2le z6YXsgdn@mWLk$NZ;B=JPmy{l1NVfv18C>0W+g#$6%FCLmaTeE z?-NP@uAP~9C6ln?E&i9$i#>XS_G=0C%y%Bdcrp4c{Dt%S!e`XI zuk{!5H>>RWG&lApmbw&D%E~g*Q_6eWoHP&8xl@YD(GU7cbIP*kZAF-wryX??yhq>E zvN=Su>gY_{m#H@mcJ?Y-4Z~_~lh35!`mV__-F`(zS5?L>xAoJSrmN@8o3E4tUo|*Q z{1bHjGrA1@^V^h$E#UcJ?pE~Gb2BM)D0m*v)U@eV`Mzrq5$A0|j`>obhHooD5|PhM zv$(3Vvg*jMLJYL@`@XfSg{#l$8;vhJzgq$&ryZ^G(_X%P66E$*Kx`Gi)qcgOZrI$W zt=uwPuyJ4D6o}S(ANxV@FUvUVZRaMlUS?*RJkkp^eQi(7&S6V3XdCHPB#i!FF$U#i0AS-pJbI3O1&S8I1> zy?^Wzo_+ETE2-|s?G21s@y_ks!)-r*vCEh1$ko!K!as6Yrl;kms6f8pb+`Aitw18W z)397Vv*H4Ki@YIAD=SMEOMa2NQIPHBSyT}2|6MAjyW>GLSVA@4csEri+OQn?R45J?BpRV7D&+W9qe23tLys)nPdyK5JbAK;4^E zKwXYIH#RmSMI4L<)ik2h7Ogmac&eU>NMfuxzgm2}IhmC;J%n5!VY<+YPMO#zgj%H7 z&5y2M-&hY9fQDJ!W9HdTyvY}XTND;95=k-X^%7$q2U3z07EbSu*D=*089aZzkEMaH zU3&NqYl`X`n;UO}On&QTjeBVf34OM;qUW6bFq?^yDLZANR7!#Fd3JyJoX3|PdLSUz z87^tfn-}Ghk{CW&^+Z@wO#9~3!y0^ivG{kTS;_U84H~V5?*hDLrN}Qtn&$L5YnP}u5sGlz zL!2TQzZK0TAk+a2`b8!+($IV@>N1o)%KTY9WJC?VCm^ChNJsLDER`(4OO)WknXk5NM*Qls0usCwYfoj>)JSYzdH+Ma+#f#M$b3tH zmxMHE_AP&`=vk%|F9CSH5cZIc6JF<4Oh^&=V$$%ei6F>+c)^PEE4HX<)()0NBLN|! z=^a9`Cv{nORAzm9Jv|%b_*yU^#_|lU=$EfRB%5^);wgwl?>psv=)8QnNl!FzXJT<9 zGInrqvTSsXEA&Nwh2HJ({TKO~L%Y}4b^;AuDKKeP>w7ciNJc6h1v1Qa80P9<{pEt< zG>&F2;mVYE(}tHOF&pvYwowABy_hb;o6CyP&w0&UKgg`@2Y8j5u}9=78+!A{db_9C z_HQ~aO1_7|q#b~YL>C_5%VP|>6LI|g1`>L_|Jk{I{+#6Fp>)3YtoWWgaldxHdOnu) z2AMnUX)bLOL6%|u@R|wyw!aeK};@mtxJOD^eJ*~TZuqQk6Cn-)TJ{~gq5G)c-5c)#d=W%zXT~2*N{lbQyqF+K+nv%&NwYi4IUD+ke zh@Zs`L2rZe%{XC9%aE!4?$B>{9Lz|3p7it#F>*y48JS3IF2Zyb-dx9i*J@Ou`&^%m zLtkn(YEr%EYd0gn6$0TfcTwI5yAOa z$eyF5Ny4K~u+@D3~dY)beZ5KK`^}Lj;{SXkjutTnnjaXw zu_$7la~G?>fEO-Q6azJxGX$w%wkVYDBjh7|U^c;5oNqtEf_VqMzK*;mRySB^d`m{w zZw`6wXy7gvF9rJL6J#q;jj&C(ahny8|34Y~ z!GsL*7dlEHv&uWfnE104Wv2V{i@LuT1H}63&V&y?1_vGNZ82W*rlY_Z_oOSVI#$+E zT#|kPuM_LeH~g?H#bLVyk6kZ^+fQ)aP&_b{RZHp}mV$TAhnK{sI5a{IN( z6vO%i@=5nj!0&LVHXX5Hd-r7N1-Heq^xiQC)lRGs8c@SCg0^W9^#fHOu^6sAz}G&P z30)Q7&(z64e!{aSlTY%1C1U;FS54rjdm_}A{_Lxms8L&qiB|uq5FhwF*6*!ycp7@{ z>vgsDp()EN$6rFZt0VmK4)}!zp*t#=NYAk50h^)Fj=$P8g zxcO|_=b%$V?@$wX=6{xS%1;=fX#(Z3Cqjh^rqnYMJlO)iKP*4_#`20gVAI88 zA*AC-n@NTHhF(;sd^{saU{&=Ae+IfOhjmxm2D#geqC1}2CGRhp#X-Nh3VHgJcq1bD z!-@AB+Cn2P$TPs2Np>xeHv}D?6`f0L3a$H>0H6ff{PM6(lc-|s?j^QE1$Fq=5mu*v+ZF*_+7#_duTc=kUE%BRb zun@w5JWlH0lAjJ^(B~zsC$Tgklt$4p1Gf<#gN-1U_&4s>TjIp2Y{K%QuwV9?fu zu#CjU%?E-4C&Oxd?RO_U7V7iAk?>jlxx~iMrXMTzzk$vzZK`LWss`(LRS1;-ktF2F zKMbXXeCb#`KmhCB;)z`R_vC}`hPANUMBmbt@q>xbA4>$xkx%Q^vGjBb`k43$%TXaX zDVWB!Ga1w~ILJ^ul=T}65Xr+ih7Lh4V>2^FDA|Bu0p7j)*Ux60Z21m|P~wSt`;BKT zdL}Sz1Q`V;<+s|I^aZAQ)$?05+|j|3)ivf7L;ofNGX39z;*9)8eaTzJJmrLmGgDH8 zp|F(ClLX{;l#`QFuICv-LfGQp3aS+fB6Rok+!8f1GGahjMW_CpM6+W0_#a(?szKnE z<~b;bI>0V>*|5P!EJVInET{!7ZX09KJZDshpQS@~G59_JAN3!R)9F1Y;d89Cf3NEJ zR}##zkS4|C>+w{dMRrB|($q27NsopXo(jqnDx*IE+D>q|!oSQRztP#H4Kxh2n!gkS z&I%7UCATw9I0z@tfpC-iK*{Y%b zy)LNXl%-Lw2u1h-5Bh6Q;rOeN-u6D0f?%GUzKqt^)@KA*bR>GFzh41tf=PNED{oZl znXkjB=ga@r(?ENGq+q^(X*zyAGxxl(jCRbiRuiLqfG@ayg3)3%eO>N}BGh;17yrdJ z(=1S@VywyqLfDp^WU9qx8oxJR@K_4s?n22G8yg$ZK-ha9UXr+9z~Mlybm6{90aU83 zIL*;16a(%(4#9!fWGC8#ON7t-o2@^t{7slh;xTaPTE=y*XK1<1F>c;Q#eE;_PF5@= za=M0cKl}MB-rvWdj1s)1J$(++vQjmj6VAt}zWa0z)$=lfG;ZMmeprjb3CdW7?M|r| z1)9WNf+asb*2N5ujNVZVaj{5y!^D~oPn~FJR4_t?lcoNVW`u?@h#{9s7MvLu2_qnO zCH%b^N(P0Ofr-=e2KinD*>9YkMcz08VR>-X-Scv8oEVqQAQ#f#!b?v*I|gBSFf~*4 zz79)oD61Sp6$2Ld8+^Z+{5g0s>0y5i*XHw645@m~LVnF*A|e4^gnr;Z68c8Wd;dd- zVdTGi)5b2!bRO%|Y%m#q(~o!gRhlFep~GfP6L-SSz>NWv^asUk^)!24mXt2~oIbUM zAE>iJ@32B>NkU#dp#S^b=ckvyhkNwRcCWTd)m=U>lprIdb4OGbsi_knUD-vChfhQK z{e=iin7o>?t22K-Qs61reRFg38NsPzr}0977u(%uuSjPf_&xr7hW~?%m6|Gzu<6bC z5SFlkP5b1#oRcrY-{OUdlaRBAY37tlP$%ZzS)+8NsQl8o zU2>i+akEE+TH+%WSu5FkxFid^FTUq_qwe0wIAKXYhl7BNYZJCn+QKfC=ObY@;G-iE z1oU<7SLa^}T57Qw717cpW3y+jUdo7lGh|v6-Kx8cH-DFcE0nP#|>j>Sp81)jRwwczCy7<9h#d-b7h|$g}ngn{$sgGXs7nokCR|x$joLjFcAeO}rUjO=E9*Hp&YrNmS>494ef#3&!PYTYGlHU7- zmqC4t=$^`@%a>`sUyrDHq7i-mc6U$2qR04)@ETD`KREYb!kj-7_y8l_INY|(u@#Uw zs=4O$_X0AE`&m$vNsJ^RyL&+@Bps=SC~p3k3QV7|*kk7^4?L5v^*?UsEU!hlH+=c$ z{8!}&@`(wWU!DC1OJhUEo5e+ucOm)(i>lCm`0ilQ!YrgKLsh@DEaqLGQ7#fgx>&R~1)JbZ>TQvtp2dYSq~sSiiJdg+L)xG%A}(MPIl68EVJNm7dn zbJv+*u<1G|Y6o?fs70Y{n-B$_v@!%oxSu>pe0j=>kA%Ii_G!L`oX%YeF=i{Ot7)J7 z<8NG^ymsyOEk<9Wh&^kYh&|;49%MZz2gX`rDh4TDvpkN>QBuVOnWi&;k&rU#oL4L! z(lwEg2zQs_G*Q&dB?;KviW(XmY^IV!1npK+MBX#);byOM=@8H%3}f%WJdymUqg|uR zEuuF**3IUk(r2lpq#qfKj~B5T^6A0l?-0;ax3z)!PDK=gmT~`p7`Td0^M!*$q#GrE zCUoe+D%l?R72Ur#n^S=DglJxMY)Ik73y~(F(v6wTmir~8#~QU!#iRAQ%(nwXdY{jV~(dr($|LsG3$evQFmx}Yj2udjo{ZXMj+Pl%;@!~4pLL5RBnX}Ufj8)=f-?}j5`^k6Q6^_a zNysI$lYuD|ucj#~?uh9moHx6G7crga{SUp)jdn!;f@dMf4lDb7-Xrjo|Bt6@RJX#E zJh<>BC230T+s}@Nt(Lv^L{AMh9H%a%T$=$$oMy{^4M} z5{35g-lO1{Vo)UT7xMk zUH3KpMxeZwx#FbPu?syOIncf<(MU>v*H6hU}8|kT(SWxF8n=UCGNuJcV z=OpI*AaHDEOZf9j=GW8b-8#|b zlSp|Qj?$X_txnCcYS+iQSAeoX`o2j|%sxUIcF9@XcwdS8B({?Y<^ytr;?3Lht7+~F zHBmoE?nwq%Sux9d$l`H=d} zbIMn*=g%>GYd%~;`Gi|;@D>H*cZ-P)y>e4mOILHW-tmqeL~fO|(oz*&q}sn(QNRfO z9853O~fs0N8@3CQL@>#*-uhS%W@7vcmlYTpl*Bu?dKLfsb zPeMm00n*n1!K=v(3#~8r2?8Zj!xtxqj0~uAvzZBf8_D!qy~hlsC*<9}_I{@D-kEZ9 zEAU9HHqlMQm_j;mu?{?@Kx%yGv{&L%P|KX|7DkFQL>&FuOx_+)nnh+vzk^MUjO+d} z(`%ctR_+eTXUBI3yxfl{YmVO}rn+RQ`TRipx{(lEo~quSH`C`RPcPb~L?IgmeXY=} zLd(D6^(-XX!#v+XdMF7zJN}D0^N8WVQfqI|9SQfHOktIJO;P8U2@c~m218aAb4DaM_*@;Ws z@#e%y$tt=pNblqN-8103Ez%e1tv$z>pOIXpE9N<**X0i(+dG`U zR4%%#mX|b&E7s^f6DuaW!0U4~gUl=_=Ppm`aR4(-<{+4<_b)Eu$&;#^G`y`p#Y@&Y zSFkw3@hG4A2~12#%Mf%O~JxiAyjQw5UB%&q)zEh%4b#8OK+t`WoVuk3=a@`7}j5^u-r5{ z*a)f3Lr~q7aQB=alTuYI=Al^(U{PDRuzRAf+YIXBi#~|2^v##`)zhz+mS%W9?;BCm zOW0Hr+}%vFn55ou+!A(2qvj1FB0Fc=F>D5#;Mk*97WOdo^8>~>Q3>~MU2Zj5Pe|~=XxEwb)`XlFB zrnS6I=OODj_XED<)vEGbpESKynN>cEZ4GlP%g)8ZBkH43H``2iUk{?-H-i%c1+(Ps zr3W*En|z{DcSG8r1knE)^>M7L#Hr+96H1x#f5uBLyKix{q(1OzkRCad@^1gi=L=8H zth$={R$izgx_V)7XduHVFxqUp@2y*sy#nJpt-KYS)9Lv8C*~YXNHY@Lm$EV1-UDo2 z+vjgbl32burqe*7M%?yRl)Bp0CL+RNzTkj>8cTqY4!d6+1^r{1~@G=YeUP^7DsEMZfH0iG<``!F+@P zZQ4|59LD*}Z&)WQSMS6RFAY&n@Rimwo83v;;j#}z8t$MV1vTst#VW@YMZSzG=@abF{2O;N~Amq`O#lg_{ zDx5yOM+ zgD?(avUWaWv#AiNPb*-$+&;U;h*`i>I&GE0egFmUlLozm15;qg=p6Mb(*In;T)nH>O-zN!qzLrNp){tD0 zm;GtVv=HK8&)wLc?+hi?Ua$>MShle-tF}rCc^{z-g@dm#ahZ<%GIAhk$D=&>B=ma_ z&oJRI9X-BT*T0=qqkG?_E`HZSs1(h3)N=pCpcy4-9a8Y)Io{|N-tO{!;mv4`b7a}h zHPuq}%h&1Y01@c1LaU&9 zlIM)tXA-4wL?H{2d(~jLFR9djh)2foxt&|DAahqhu+(c^X?N+G*5rqN0~g8(tGJSk z3)uEfyEj~x9ZrsTQ9_xG+3j3<8N&%^zI=Y8ge9+S5;F3E;3Iy=VFQ=NRvkIn@m=wC zf|_}$^_M`%nb7Y#k$O^~Bsm zek5AhZ9YG0GV=Mf$(@MYnR5=3e!oz0)V3 z8h{9A93NV@8)3TNzZ9;PTT!cl%gezblqTrAyHLBjyEt5TQu^!9-hcs)=!LEABrly4 z+i~HUtx6YRyWN3pO`H6Jsx3a9owoLpJg?YSQ^~rn`{$2-f1+G+={~Vu@-B3hrbN%R zlz9HJPpCcIa@$)CH^_dod;P_fcqNCcpU4|$&2ir zKbhZDqBq-*$V)58c`ox~W7~eW{h-6gtGNuBpDj~fi&jmr8@*k2l#rjWWVt#ptHnCp z?Y+zHwd9ec$V?G#7*%*Qvn!N+zZMBJ)bqf&BA%6eSvVs} zXh$ibc!vySQ0vj{vi-H;rX?n-KOf&1F8|@qJAhttQe2j(^lYE4oNnOXddpv%>`G;Q zCCgALZ@LJ-cBda{CJ>{2sbhL)p`F`XLyOxw!C9}a8<}n;wY^lVmk+2X%k8}{szuj~ z^f|Hc+8f1dU+fUn*og>M+8W&TQ2|i1^hmhU{Y4%e_LKKk9OWYW$#!gZQ9hug9GL9^ zGUP-}7s(zj>3;vrt!y7sj0h%Q@T#Jau4n4AM%ln@QGSl1X=i&B;{ySKZ3oAe;1#?u z4$qYs(O}AQ!JWO$?)j}BFQ58y^bK-u^JJm~4PJnfWjbl&o$4WcWVY(|^X2VxWMPCM zqJzaIA!S~>RqivndV{V$KC;uX?s(51_D46m`v6At^`xj3c+Czta}WC*KoBj&!}h2E~SA6DZaki>bi?6rN+>|6WLY9o3rNNX}n>-c~eReK@P`%!nzmbNak zuE-(1cH5^eeUmGmzPz0UM#^E9D=~K>)nPb+->I&!KBjhed3}VU%5Bxy>(97ee5uB4 zlHSJr!FMg8u;G}rUzE}NY-|tNto)})bzPn|V~JJDWLo#3kGH(ayk4ii=Ie2~Ge2x& zWjV-wavWC%zvkB4Gxk~-#T(gKQ)AfpU{Q8jAb&b># z_43o*pq;G_+x4^w!Tze|3-8`>3=Kzb7q+;pm4rNdmluc-D6pDDCnlmtx(NeCi-l}* zUP6QB@740$_4H_ak4Nr$!Sbea=(9#M8=V*yb>y{^hz<81AG}Q&H<~Dw-PT*1+D{j@ z?z4$@GB<%1Y@Nc{%NYJ$<~3Qji*^f|^0M(xB3ULWo>-4Yd7HH}{C2NCn9=i6WHs5r z+8Vs6m!D5sB)7zpZmT`lZj*%0|JrN#=!e>Lzn0$SP&}niD-&yM8PbQR0Ovb?i2D{-USxtkJ6R9tFDF+&-yH)|C=IhutT6 z!=8JMeA6;EiZJ_jwRY-{o!~L$$*116wLF85?v#0VE=axm^hDczC%vropiBo<#JU<= znG?{y*k{r0W;Fp{Iqib#^VA~I+{(Bz5i#tby}4vziNfSNPo~Dse%P)(h*#?q@6-%U z8?OGv37#K6HPNr*)3nVyS^lHNNGPFniP>w7JgGlOZ9GfZZ7^wpCAwC`TW_tsq>Q5^ ztJz>(g6QJ>(B3}ypW`AEYv*DuuJ6;~+5=5gtyNXo?BD{&siVE(d8zi zJ3UtI?9e6kf*Ad!gFd(J+cjj}<0Mx!{ni9G9FsMlwLMyVJGF3(j&R@jrjGp%Z5baL z*r3U-aNMv%T5j6E9Ot8eR`?>7iM@Eglq#IRqc%pOhjVlR_eYzUBK`Fm^iJpK3#7#` zD{3cfdpVPH?|sX2_dm60OSETaXXVdk(o7?ZH2X?zO`q9Fz9Tk|*WY^AtjS%e+QU8! zq4m!{^&3llCC#J}YjnF4VtD?@>T z6s^5p%L>z$53=u&qfE4JBf^G|>cWjR><&C+yn-2o!BUJoLq5{G0&dW5u_Hrr?ML*B z&eE3_?)ws4wIoSV_jRpN(Z{+oui^gW{R2+6`P<|K5)VQ4nW^=Y5n};}%`PicXdnpx zN)d<{IO&rz*Q?>1K2mDe0AChP<)x=ukkZ0b@@63yJY_d{D!1^4K9x1Z#FqSy9d zj@5T7vl=I)SHAnsN^gC`YbErK;D(>7_4vJc=@!%?$GRA^@dRb7cGuHgujC)PUPHM~ zf#|60*2D0s#DU*?pRF#bNSQB+24#xgOmecDKynLkSz0tRNKBq=?>Y_JthJ#lf_Ev) zt#+>&t)rpF#BB=!bX+!KevlDer8fj{ z#5d}p3}j^i1c_Dt%KH`-%B=RQ9--zt+_j#sYX{gt01#vF8XX3scLDV)h|9k>>mx=q*U5=w?{diq@@#u+c8ogM0 zUGyVPgG(C9?6ngj$BOLOX<-X3&m_9DVzTdxnOaIy>*y$NlvB&LU zFS&8`n@nqrpS)+LjS_}wRtBGhe|#7?11)%~q^KhH_I=?+q)<22%R9^D91Uv0H zT8kaB3PO1j#ZIwLm8zeZ>D6djBh`9a6Yn;*gfQ{&!4QBnk6ml6&aGdZ76St#j4m=k zDdFt}J8P3jgaE6UW_C+Sx|oT^AlQO!nL_ z%O|rer-5WwOUPEs^M32eDWvne_KIbD&)hw-YbX-uIV!K&sCSiz#gCaI_rO|VM}J?c z8=v8mH=e3ZkU=-C+UOg$@Tti9z+cWdr2mVd|0-wWv1%p5%&RU=Bfv3)}}u;Gl4gAUQ4`lyR&Rno+eG7C&neC z{!RXc~){KWgl&a#nCNz!=w_@(c&Bp2CB;|Rs8F8Pa z7ndJuwYFhMhE);Yup&__Hf~?EjbNXL;X99;Y8g0D9wBR;w%sGM8!?!h*gNR1vu3gh zOT$pE@uQI{r_F8d<(y4q-vQCG$u(Q-#)kLhM(ojvlox_g(L#ECVHJ;^S?{gZV1J0H z9pjLfxoCgKhgQ3QN+Mx-7$qs)C#57>S(%}#AG-1~XzPxc`$^y$m?S@YUJg|(^9N;X zq|2zkVZoR#y1Yk~vFm-_umt9s?WS(u9HonwtKFMXKG!s$qC(8c+*h&Om9V)y;BHY` z)ITw^n>{YqxxY``x6#x-*w^zMwYySlvgsbI&yywSn|a}Fe`LR1D;?RjWi`3uwlbT| z(awWCJx|Z(WYNmC2J5C9JNotqJJLJk?PF*&I)j%F$=3ueQ7+A9CNtY)+4mM}u=T~{ zaFy9%Ly>2ql27X-@b})AcOeuBqq5yA>hVQvlKm3&BP3oslsw6a)camj^uu=5+uZG;rLXL)@uR^&1omQmi7xwYEd+~Y?$m*j(Kahk zIGBW?;#O$x?qlN|O7Vi`vj&xW=WZ?sQTn7#IH9VxTGzc4QM;DHpX`i^K)^uljLVHB z3{S6nEY-%dUvFFd(lMt#`7r3h%7~qsj;)IB<3*DlL3S&>f*S!={N=NUd|{4|M327O zv5wXvm2ExbjmVXByMG4jGhE(r-uKT=8n1e%wv3a<8D9ns5UEvr*OBn^XO@xU`i`pR ze;f^d*GMAZC?2+0{16J2uW4?!(zg#=B43xMMaR(m`jh=GV?O|UMP_cSRKt8Ff0z2m z>+C&o_trLy#$?bfoZ&BDq`b1h5}ovWx%yy54OMnQ^n&AXs>|#)nTwEPCddd*U*6L?!Ksg(x*@&akyo@f_RMYhF=>s;!&{+&|JveUPF z&X;)gs&izv2yJydU7}d&TC9Z(*+szB*J?5y`9+y?;)WJgHK&mvdTrJ|Dhuu)pptR61Um_0$B($KU^Sv&gAYHD9x` zt!$mi#mh)j?@cV(8gHJ>$r7r}D>tg3FGGnt;rEX|OdiZg0_q_e4ie4oy(+6zaGv zI#pbAB*q9XG1jZ2rK6vpz2@rp;0C2t#=_tAZs$!XOb%tdYYNOPL*r2>Gm{PVgi@C! z5-$q!u&ow<43TA0iKYGC+tYhDx}}!+XU|k4`|kdR<*W21IjN^_Qwu$XcNc@>mIj|>qiVx{$3ucdJln~kW%(z_DJ zQN;U7MYK4$mZnbdAn*XqBhZJ7lu3YE{oU^iAl;vOKI2s@HF@e+HAEw0BITW*= z)I~36*i5Ej6xF1Yr0(U_B#)}&t)%0DAyc-RZ}p#IPrM`g?PC2Q7h67A*e>%Dd@6X5 z+kF!0z58`Nk3X*wkub=P#J6w`p%h;yrLmqwiEW=YN;m>*Py=_bE<9np=3taE}_DPLct3Zq&FgG-V%Al+#ZivCKMZ8*g-R zS1aKO9KCS&bvYc)ahwP`}Tta!*W)XlNCzmF5t;pxfn926sDQDuAHpeSfMcMr)oLEmVah( zhw}IV`ugW^Tx(Ru+W5|mStTx;#4`JKjv@53$aYubq%t58c@lq9T*=iUpQ6?jJqjUWd%t|!UdP5Sg-Dd z`akE&uv${yiB!%aZF)0p_DZ@m9D-Nv{VzrqeHQNRn#Z+^sT&`M4;p2JA<)PiN9E<{;;M`{hyD+WGJLVGY*R96CsBr7~ z6t5i&=qSyeU%a8N7j(a02^E2|Yc<^N8?4-%<9F(88a6yih-T5vAVpRx2%l_qXG-dH z>9`aC?fFs>x1Q z)G{<2YGsI3`qpmI0)OVSI$_i*TOb#S}|x3{9kkQHgFDhxz$X zq!u7TwKQHXholf}?80J)AuM0RB3`!hy!MPLvA=9hH?<~nTI!Chw6djX@O!%SjwJbgwwZXxtt$j5 zM9#)4#U-#tcO4W9x347AvW10Qf-+>>geXs$Nh;?C)vG(~Zf+#0-5mB`TN zH%CVilI!zjvTXh5pVz$ieLohAg-DrLWb$e8TcI&KFjcw}rOC*utY@JtOlc;c#qe;2 zc-pCnC;Y3UF}cO)$D+K;I9T-p&vD*>qTH=o=PNJ15j|1P%NimEd2TL~)&6kP$VZp9 zt=S*w-;HHnYadBUP1Hk;xPa5@{9bcej-@;j_w_zOn~fh#_h#jj(W&T21$I^JXqF~8 zYg*qo77Q6If9*}ipn%!vSQM1I>2Zt5sO6Y^iDX65KqnQp-5fz;FKe~us(D%i6}z&& zN@54W3w!l{-ri!(`PSAI36+{uBXvG?XT$YU@->P@CoH|jUtEUWVFx88gSxAot8ukY zc+94o%^nAj9N4LA$_c8-mZOMYs8>7ZYdd@oh8)hliF0_NWV3rXaCba&?WSIQlulTV zUF-Gyu}YQpa?95B?Y!NFd*nqEl*!*L!Qnv2XW4mhuWi_SgVSqs%_avn`N9VJex%@? z?8AQ5VN31yL)6YYCGG`koBi+A!30?HJWzHE!-t}2^3@BW?kxugWJ4L5AT!fN=eTzD zjtI&4M|1TV+|`fcCVszjBTKVui(=t3YLBX>oZoQWqEAAPRR^apriDAMZ4WaM2j!fk zM{^9Jav^C*nffeHTDnnQmg6?UHdx*{rG-@QFtk?>dPWZAijRmisyy?p&>-vdhfYxQ zRUFOag?KIC;VmZ8uNSUX*|$e-@@ZirxQ>vCeG z2D?@E=#4POz|@ODOIU<0Zlqn|bwkKM$@@dB*Mk%rpekW8yxy&kmp9cc7^IQThn@~JQ#$u*H6@p6btON0$-viR7F*gCY zU|r-TTkO|O6e>=au{-{NT)Q)qHLInbJjz2ZJHAyG=Li|?d}&btjgD>e`BV4>@s+t} z0qP=%DKI;gzE5+I(W0W#T$oK~HHL78>ej9irIy5^yU972P3}r~ChouS@Rp`bD2BSC zU6xEI6|^R=`Q+;{@vE6{R(4iCZzs~M;XvmoVTxFS?_7|Te){^7{lK6?ks@=UdXaL| zJbU!^b`vSyi!k}={NBiU-E5|A&fK(b({@4UTL}FT%34ZHJ}eFLP-p6P(f;Q*SAh%! z6(QTpVE;>ImeS^y%sbz6T7JY-Z0$?iW@o(fCd10G<4Z8lo3>%cQobVBmgC!NS^}LJ zmjjU~yI*}$G<0;*)Y8n=P(fzH7+dqVb?e1Fs92>h_|#yC?^-iQPa-2*UHBPZSP>AZ zFzu+A166rg$>x|M69+ySw}8fHXDPxs_%z2Jssx4-IG9CV;N>#XCikW}FG@Ge(uF96_rO{u=xdfhtSB~0xCEh*f*AmEI zAW0MwH)xq=NDQT?GxkI_((rODc|+LO@cH;r`a|P9`_bK$UPIpVbHfW+t-ruDr`3F) zVoJ@jFt1F`x@fO|{AIj#f=ouncyjLxp+8iAMbcfD*b%@K8FF!kVoKq$l9%9C4$ZB( z8gP-r5pp!w`+n*dim7PiO^@{GA0)vCBm>jHy5nx2?2dV*Jx^K`u|rK^ui?!%jU0Cq z{@GkTK@Y9AUogdWk0PP6Zy(AU5A1moTI~Zq-DT%(8GG9}$0IrSg)d=_2km*kCUm3{rwG7d!vYp zp9l3bIyv_Ep4KQDgUkoVCw;t%?c11#+xH{ps^j}$Vh_jkCQX0QFcK2!7mZU7&fp@; z|D*Kr?eu2sd)!uR-n>UGzaI$lF*i2S{ic?Xk(m=?qZQ`lY$81^(2Y2=F`=NYnw(lb zjLkb{9bWo??Rk{APRVye7Uj5s4RR&1qO|4{bc!=+F%?iSY zJfeW9keD&3!^4?(b7KYN&%?Ga4-`7z=p5DTM>eiTQ4wV4=DL%F$x4flkI(Ivk(3M# zeWXiG0HeV?L@awK0&G&!io5Cvu30&0Bgu_}BfT#wpu3{LdNo(TNWcCtSk*bO#UNPB zgkDAz^PW3$TQiUB+ywgi(iMH`IJk`6635s-%cT>FTFovn9Kj{OJ_^+8l}Pe zu^#ywKrQ#|pBN&w<`x+`--s2hXXp8$kpCh?-Q?7=A#D81w||S%bf`bMi|#AO6)UQ4 zzHIF(Fz1q$CYl2uzyO+BvL@8=E*49w0*>yBF&MgQa&X4x!DQV24tD$p!cb9K_ZA2z z1x5EEj^Oo6H?%L|bjs0$Rd-%4+w+`jx%c#sYe&q{@jfZ0{_(->bg4c$TlIsAl5*ybvm+2_=tVeOs!=TPc|qrkvyEOTFV|8KEZsJ>si&NQ;b3 z5EMMKLt}Qh5R56q=3H7@VXAkn*`Z6t-13X_(6Tt&(iY|?D#O_RBW7f>N8cN+>gPoq znPw55cwU}|YS1HKxaUddr=VoeP58%E`awo*Vj|}33oQ9Z9rg0H&Fs< z!!OOj*drU3Iddb$={^u#jxB-$?}%9SGXYmC-93oji_SXom3}EzDdprG(d(Xmc$-XB zRe~{Z1%pSJhM^qjVO6GJN`A({~rP;+o7HrHXjQSzCM zSDW<)6x@f!lBqsDZPtC%3RI-cgcGMz*_evadnKl|`NNEM?bucy<_@tU$n957W5-vh z*)WH2iTU9@OcOiP-{v^1WGZM{Rk=?RB&*}n{rT(__`SR=^b*1ZZhB}9>AqUC_(JmA z`_oGGd+$-JsxY}4(^E$Fw~q?m4+89HP$u3iJOLZS{ZOiKUGF zukZWYdkRkX658ChpBMfZa4L^G0Sy65{lL(rU~E{rr{<>mFL#bry5@< z!qsme7qnHL(*N?K5ldcQP*8Abd09zMPp{DTEa6o>t#glvmg5f9f<)a#9WA>bHl^B; z=kw=8rL#cg;^#X!!B?}EXIXi1`3EX0l{ynT;ioUo+Er*PX|(s6M&ubOpT~4ks?ZY1 zWm(un2d@aExDKTKc3xjs)u)l42|crUpNCj^S86=0!w22sOxa&J-V)(QA0WU6ybI@( zfO4rR2cOe8^bJF=+OtyG<6N#OeRVVxe4fPUdGCjFf1@3SnuUa7Nmg#3D|4vd^`Av? zM2)L!3ey#2SY*zVDLDCw7>w>8$|jHrk9arQF|rZ%jxKychMWCpSTZpY-je4AYV$HtlqCNAAryh8WgjbEKSqLvK z?$aj?Zw$5%4vnb(gi7+ywg7WEP$mi4LJF+rW#iVFI`iv6qNT85UU8tqr=oH8*0`dW zgD>=yUSa=ouQ$(lFx$Err6zscc(}N@*ek#Lf!M6K!zDm!l*J84K(or3rS#y@dHbqy zK=Hw;I(ubKNDhohVZ2{g0zc7J94ND^^+FF!Omz9*X;T%T7>3PDhgu)#9)>~pCC;mw zoZS5pgjn(un9kKKaj{^s-hMgZ1hp5bWo4=+NnTuMa3eUO?Rk57IJc~fYrX~h?3vXc zg-gxJOcbcqZbarFk$TCYXWkZ@_EK4m6p28S z39m5ZuFJF0dPBwCLUC%g_onpEF_?lJOOVhZWU`u}bTg!-)aS&T9Ohsk zTLuRQAx8|#$&IUEq?2Eh|uyyL>(p?>IlLOVBsbV3R8S~3S`R%za zfWA#%W680SEulDdAv-g%jj4XhxvJExpAI?(qp&XI1gO*&J!^b-l2pSVVx3#N*?h87 zy7E1YnJJV=O!pPHhQbW(rEGqdqcdmTS?9fFe30(((o7CzHs9fxxX6~C;Kt*SvHB^2 zt@~U;ALCOBK8~KCm*=i&yig2y7)(tzQZz#&Y1v*yG*U>UReNw`N3CIBJ+{0zsyTsl zu%}`?kfdx4-l|DLA9>b(@9vHkTpO!!Ff}uKdbs0wtfw0dj6G;F38DBOt+$4=LSA&^&VcXr&#if0%>TUp)00rKYPxxw&2+~8p zC6W~v7n9z))O+wO>~+y-N5@r|Z+H!>XL(!N9zXqMb~fXeaqFg8Mk6Dm6_!Vip|Y7z z4GgFcy@g?T^@`ogM_3K|_%bpw(!|#Z6eOR+M|@-m_|q=HBoc9OfJRTjVI#-u1vJ9K zl+IP;#>E;<-~IYR?p(~u$$8al`et}Jay-J(-JOJmh2`^zr2y|E1cK350=LZ^T5|ui zW3He;oaQWa?wVStV?BtK&O##Rhdv|PAH;RS6)r3-DQefUv9X=?ZEQF(Kp-O6T$SPO z8CeJ~e}eV~g;!7Uv4P(`A1=@l*rN)2oI99OSV&M>S{e{`48vkw(W~V~^kROdUbAy^ z&G(r!&Pzme@oZ94r9=#(P*-5ors;dtToErX!K*m0B^3ya=pN(C>+0gB_FjZb%JOWQ zwTBb8&#I`X+!K>PNYc=;#LgsJHqcx!D*c9s{wB>$V3ijed)v{;iIG*VP@VZRYE`@D zvE%LcqSP>982X_f@bEBLyk`jbjXt{g_6hvjQc^O4s?26GI4tZs)*}UlUL3KfB|%(A%tG z^3CMB@8o*4AS?dGJGZW;WM}(JNWxpMoxZgIjF)J22n%a}e;=L;*xNl(QH1@aS1)pM z?%fI!b*<4I@#2Otp1pm2XX83?atCwlBftQIVrG)_n(c{Fwr}_&e!4TidGya(0OzWQ zTwEEZV{@HxQ;vL}dVezoG7QFa8e<$2V<39BZ{L2|IQ8?yii!wE5$hWoVq?iGDxM8& zDrz7g$PN*6iWrQUxuZ$>N&X@XHxIZ@_UcY@Ss5`npTI>b$!D7R!~lOV(@|whhmVgh zpnTF>%h~>oUqoDd{I~DlFG5TCmwp1HYh5J%e&Q-MH8sCsC}y*1sU~DEjrtF=2#E;t zY5&@Hc6Va9Y4%3A7(MN)hBj9{u# zeuDmkFH0MQJCk=LZT^~XL4j%~gWK7#wK5%kPa;CNLyCY<;ZJX~@=K$Y$ku3t?tsYI+goeZzSHMJ=Wr zB(T{84;K6~C5V_wkCB!2-SBb~4DimggE97i-!D6jU&ANzN~Ux~good#>j1>KLdOd( zHprSO@YclCzbLTj`C~Bl%F0T^-En*o0*a6epXB|RYW@7giBDgt0&WDM>LDj*x~aB? z#%)WkI6x?cd;R96|EF!YDu6E%V{AK3-N#25#>%IpBmWvVv)tV_doZS)oXeuA1ke-a ztkk!-XcmRUXN2&8G3h&0R3lWO7lH4l1g;DT~_o&*i;W|r72<6^Z zJ`oW)pOcfL931B7*LOY~;vrc2176-saH0|GT*Iy2voPg5AS$dyotgN@s&m1_9jXyW zKJx!J03#59SkA!68tUtz54k@>2I3IyMV!|Q3k#2Eu7?VW;$thTsijs|YeO_+P`dOA z%eV1mN(z-oXHZK^OYg)45G&_;L$+)(T7?5~fsy1Hbo^hFVX@mtOCye92=`C%OH8EL z+TLa)QBzmH=-YTEK{iv-5D~%he3l7lS5bL3Xb@o2pEdKdva_dl`3MxEeq{<9al*q7 zd)xS|q_mV#);1zC^4ad%xU?;P@5kOnM{q`D%Ib3w1%#d+C0qnuRrv)10wZFIK>`8- zEPXIi0FFOGoc3U&92^|=#>Ed6#1R&<_}H*8@B@aW@`n!}zKn_@2GU5BN-gO%_9Uu^ z00wXQ6^+eCWCF|MI-sT+jQuBSwN7xU5{iJwym@k$?m43c?eRo+XQ!1u4PqIhI8_QW z7d(e@6id~dU`AfvRjxRh^ejI=*7QO1-XA{zB=F#f9uMt1_>UgkLV#g}7ezR2xeEfq zV`aArDRAjy7|F_tFb^gN!)vJd}MKy?X7Zn0Oj^*(D_<4MPY-2)yek{~38x23lw?5Gde~LZ`TtRN$|T z2#i%OIu;4M_3L|g=b|m_>$QUff-%^x4I-+Ps|_Pe@cQc_1duW=K+4Eb@RvcNn6E30 z21Fu(2ujP!wvpUm(knbQON9uUyHqIyUoqeF4vsJa2l*ga5yHsC^t#hn3EVYhJGFoD zvq?7vgiA0f4Ff=NFwIvDcSuM`Vph`MBxOr2|b9b+y(9zK;LR&+y`WJP;tOCdB-F&?A_6e}d zuLA=SDR!6;n4PVFxgY2Z{4dk>i$*Ru@)1$fD2n}(G@{KF@b9~gL0PlBx;jdsB)k-c zHj30i^Fo6b+!$#(hzV4ZEdRt>1%>^#tt}2sEiDBlrPoD8Y>}xnR4MIEumYSrGs%ub zKp~u~h^($%!GcHbT$NC_j)*AjFlG-32q5%&2doD<92go(0Nxw5M?jGNpv#^ySI4f2 z2m(e5L(xQ3pC|+2E7WR~7xDEEUHYp30xiG3*P^M9f!E^T;1K4&Bc-Gi^YGwTYs7p8 zCPrJq`6Q`N?-~x;)FHDv3YwR z_`J2N>n%tTPC1W0@LI40a6W?kNBG$A<#19C<2^lcxCt#SQc*E6mG{AZei1)AqWu^| z{tIX``&-!q5s1r8ZEBFQ1K8cQ=)j~r@jL{2ye$-Z2n1^;$MH=~;{%z3%$0myJjnnZZmzP%!9UNUI3ZoZdSYG9MgfwYyZ31Q3(KQ2u48 z5Pgc8|7ohPzwLb9LV z#lO@Lk_`A;)N=mp8Azl7%dMwm5GBDF!b%%Jb(-~o8ybbZ=90Ftxi1SNeN8WAtL8RA zAkk-wRet~8&kt+SP+Ua?q&n)qZr9)64|w-fM&=r@Yly(6cCYT}TU+B|wW~1jW1f7; zQpIC#$0g<%F1uLx@7;d_kY_labKmuv_cyf^Yyg6$i2JW?i!jl z{{_8})Lw$%axJCT6u6m!np&pm@dF{D;GG@kks3i@QBFKk8F}DERuSwbT|Z}9f#&_? zxy1!>ehlX?zIxHurwHB%RB>dMY0CE2|KPbPE8hUndiem*0{FICA|fInP}*(d5GVvd zjQxLD?9h3*Wt45RG@=Ce-Me=^VXUCh;zPso($P3>Z`P1e~E#t?e5b z@dxc$!vGfh)RYTbHZ&B^xjav`Y?M=^qKX7s{4c^V)y9N0gwVmz3AnW~QKQxCT$sW@ zg@5s9e)&Y$Y3PcAe2XCTGUh%;M*~`0B}NQ5NUi~7V-hHg9SqQ1JPTM0KOv;e%{)zl zUOl>#{M-?Nhz+sW^7kA1zz`5X($j>&TZZDlS&qoHOoPe0z+z{az?EQ)jaaPjN?Aaz z-mvP-q-(7W>Fg@1JQorX0xLoYm(9(hdq>{zcVznPODqT)Kn%)p>8TX`z)v*)p%C=U zF0HPkNJ)nae0gfPDt^ zDu(;_)qsgmNs5pJ;o$n7i{rL=*F7(FFF~j4Zz@x95TZKU5z7uKKGqAk!$wF$Y!Qeh zj9Q}l>l+m}x63)|PmYcUTW0Y>UYh@;3PWl)2#g;7Cr{v4puqKoc9-8D0gZvRx3o9$%e*CxvrlXeL7Y4`!_W86;6CjWG z>(m2yLWJ}@fF!oFk!oxWj}eHX|HDKT2e843!|iCnen0?dYm>y%rxw>70J82WL`fCu zxAL&CYyYp*C6Lb`8Tw^Z2haaU ztydMKUSL&_s{F6q>+6&)q@6jp?xsVA$AH3sqN4)fD<%PZ2*Awmyz-*eP!Y%lRh#O+ zh=JX{Bm`tQ0gP~a?+#==kbx)+1tO?ytHy7Uk}7d@{R>N}6eCA0MHLhXU`ES|m4S!? zh-yuLlQAhqe+G-s>LGqbn*TEid)vQ=K>)v@x1y;p`xg;^$?Np=w5g3vGzI|}#_gPC z5gN-|i~T%a2PLN$y1l(^sVM~kGUZ?Sm-D_P581o_NWnfR%KW+%TTxMg3H@G}d)1&P z!)01hr&IzZ1H_p@QZR~7f736bRcyKdDo_$(G27Mr7*g#rWHp#>U2Vp*m!Z2sDrW5e z>C;`%EUB;if2W|nc$N(J0yA+SCXN!c-b7p9`wOBq7R}&g5O1HAqxNe=5x4y@;LxD| zKsQHTL!*^O?A?w3MQBZ`{rnhc|APZSQt%ke6clzqxYH$)Q+a!#oWN zs;ci`kaQRX9x6#Qa3pBd5ET)5y8^W3Kd`e;$=cfbuqZ4t53v})#0xhCQcnH+CgXwP z@P86mM2RyX6vRzwx*M2K`mG7W12r|_>-9@DA%*r6_kt{RNK#-yro&dr7ziDx-nS?y zR5^k%+$o|9Sm2g>O$rVQXw&qcJEsQf*GB==Gs5n#b*9XggFO`$T}PGMoB|Zpwppnr z2NxG6C83u~jQEH0^)Cx02z7^4KjnSjzqL$u+83PzPKZJhJBnTsspfIN9 z38|0-*@> zjMuM$(|mKLAtfb!J#KAs$>bww7l*eMRi8DG8|6a=AW$#!e}sNvAMg~@e+fpQ9{kli z+t*fsXhFJDY|?#?1emvio*v%ohGa!$1Okzr`Tt|PehV7_Dg^22wmk2F zN;F-L>xSy^#2FU}o`+qQ)`KP!EqAbYS2lxFV-&IhEuU;4tLmv$AmJl)qjYcAn7usV_cNcMqmF?uf67@(T6PnR+e13w{8QP9o)b^1Ijjh^KbVW zNR<8yIn7EzT(#Ypk-&QN`0*J)DlQ%>Dkso=5R6iUB96|^&s|;BdwHTV@d*eRO}OGd z4KMe?iI!EKBL5T&qL67>S8lTY6*`rbOB#CdvqLb>3mX!D+DWdRFr|G2Nf)EE2UY=( z%FFwJOx%*XTO-nc3Od2e+|0Qc85EkE0?N#HHB>e&^ z$>V+aC1o%sEpcpYY;X44_<4UejMyA;9h?rs`nkC1PFY`mJ_co50jxqVL$ca9_2|D* zb|8Gu0i#Zy3AxN*5g8GIk=3xUYIN%wdJXRbv0oZsj2HA!?dAEmazI55n?5oBCa+8! z5H_`0Zvqy|tEi;gzd<0bLzJgS|f~3`#X2zih-{FJf6dv;RN+@2^%1 z%Um_ymC)@c?j-Q!PRg~192{>zSqFY}%KQL%JB*7@k+)Y71QH0Yy}Nt=fZbETOm2^M zRTOWn4rI{5n830mp~%+ORx%PP*Zbw`*C&u{B%YxHgmiBHf9q@5r?4jd5lXV4n%k*KMv;>2}koO*8w3HTsr-2XF?;o#)_GjpCG zpBIuZQ=O})aC^#k7+wR>|F>E%Bk`B0{#&h&kijUMDv%=HpqJHmbX4x;0p&Y^--Sw8 zSTFs%A7m2w|D>)>5LjEL{iBheKYii*@|M|%)H@76)ch~ft`zgV`2Xs9%djk?Zfg_- zMM)`Vdq`Olh9(1)x9|stf-j06*lxvX>#)wqKtfy_u)n z2mAbHFOTcj#6+I)dlae2jQ_!jsHnIA--5dWZiHY79_j)U|K-aU2yUYR=1_cqSU@o| zTkVsRWY96YCQ&+s3-2##@Rw{zY}J1U6pNAyyb&fOaX%*~F#+vpY!pF2F+hNX7OuW} zp+u#L1rT9>{4JlS{|NzsBm)bC$g*(_Z;RZXwdQ>eiPj&3fqyZr;n890Z`!(}eq(zS z>ie3M0_peCyTP5yC-M9IwY=GbE6>65{)08a>G#XX%Ia}^nzgMoh%NjF(*t^$_W85K z@0*WEU13D#WMJj+j0lZ+w`BoU%U?f0`!llFx;{X~5 zMCTua025~qu@3;;uQ*BO4qOb6rs9&5CH^olB$EZW4pwd}pt|eq5cGf)VyEs1%$-;eE z;d0=FO8DROiaG|7#y=($Yzu_AfJr?R2Y)R#miwPj;SfF8nin&+`+DAR*Tx#_u9HAq zQqpxKfSrmM1PnQon;ua8C&312=)Xe-Dy3lgY%P@&nSWq>+b*FL<6BAKCOA0WfBN(( z7~4-u1uTIo^kd%X9@Ei57+nAZ5D+UxM!)gTpFfenDBDuPv?;yJcM~@;S+BHP{y8x4 zU#WJKO!3*D5c;66|4sGdYkpPzYqtFFKsuWW4tHvbxq8bN_+=*n1iuUnyNGDi9IV@} zrvMTm=33PX=?Sn$W?BJY?L=4X`5`_87_NEisGEZ5ChBd>8~sB=EsJ2OsIb99&$h`PzUEjQ@egO!WtU1AmEkV;cAHkXf{mmXftUFgW@J^k8Je48Lc(rGmX6tc z)gKiNt7HtKWCYZ(%EAIWDlbn_N2fsFKrHGP74AK<8=te^eIjOPEV}v9k|i*>jh0d( zyI%@gIxqM2U5UzvL>>P$%@V$Tec&xsI_f__?a$U$duJq_VWH~(xgGjlIs+s!`h&6G z`wtEN-}Z#!czn|Jr}r5cir8{+gJl9jrNmClM`c2c?L7!3*R4dtBJLIe%1X<&ySt=j zzqllG6(f~_Y|sCs5RKbHvwq~6n2BB~ox^|meSHJcwb7#2ypEgT1afk2L7X{_fYO_z z$;60Fjf`#p)E(IUg6R%MtR5BXxAfVhW42`6 zO~)j_yn)~*A3dUW#q56J{|9eyP>Dm&#pSY0N~}tV>#d0rwWGlMLHBk^~+03uC70NdxaJB2dK*i)+hNG+Roilzes8RhHUcLU%;1ukdVQ&Oz0ov zw?10r4TzEnVtry#{f{|MY<$%KlB%?YMr2&%f!yF%)!XB21Aj#F@vlb7ks8IhF^#URSOk?rj4)k!l2+3?r@7=-`Md*2V6 zibD>E$}Rjr<}t9;QTP9sH%J^d^#Oko%Zn==;Qh+FF1|+F;J>v1G=`S)nzZa(w6tVM z3`J>!(zk9TZ4M+8n+8U=%PGiuw)gffElNoTH#MEKWy(3+?iVkbQeKQHY=JLtoU=9% zI|&313vgy6gz6(R)4-dWN(l$TP~!h$Oae`rxJa=66Spa9`ipt>_n)Ycgx_Ri53d=N z_NY^)QhNTaE+9BdEJJD6(3YbzQ9K*vW7~;=qzLjWZdpSXv8Z8RKR@s}gt2?$=G}P%XHji_>p=9IT*WVrz>Bq#07CQtCq|BgynIBR{I*8dZ`hqo1!Y0WB}!5*8NB z4jbdq+`3I-nGv6eQ|onQto^CLZ>7xsTzmIb>k)NhxKuWZj7;VTQSh~eg@Lu?gve>X zckfPY$stqzA0-MW9T~8X2(=mqmFCxdzA9=>(Uthc(J~9tGV?L2wK}-e^>AIp*}1Yy z5m&LR+FI|@OQtA5J zK3*#8P!Buhsj{;27cyepuQ$rWvhoRTqR#G}Kv_;!7I1fCv;K<-pFWA0nXMI1q<|Xk zzk~oLtxenCN?cyMXlnL=dx`mGWakiVCPp=|R&OmL;}O^6rONnnx=Ll-G6q1jd@@68 zYwON41bfcILoF=qNlslY04kEWI`v$2b@h&=oTz?qt}>zsZP{{?BTMDWzfu7^AP=M; z&}HpnK3NsPWpo@|$y`bpqbG^(vhPiajUX)&BIo1tYK?HFeJob;u$#--%Gmf7wPQK~ z;P8s^5N}AArXmmlUb3Gy@8!k3>~9MTFuhEU3oG-xp1G7RvC!5U{qX7i6$SZ_lEJ|K z4ly)b+E4NE@4J7?jJs$3#RDu*GN{s{A>EJqET-53Y$O{yJ96GIdO}Z(>V>$!lz0&Z zS^t-)$+);MXolj0wjFRs_HuDisz;_WD#1TYyH20&E`rGIE%#R<14CnC4z1^b{?X=; z7TYs2%0%^}m&xO6YtNeaG^;+jUs96A1gDuu=keRxdJQPuW3>X&V0sk0fWQx(J7Om9 zK^#DpE{7W+HPI5NZ;ONdfH#K7wb0f!^C&m9r3m#bl&!fg7yd-(eR|E1*g2|-rdrw4-)k}*2y7+FWp z2(rW^IUjP-CB={bLF9r`5B~i^gNCEtc43n5m4oUkb&+9rTU&I9z91hbNzUT(WaUYV zB296BKO*hN1ke^9^u1rv>klWcQjiIyEoBG0qN`J*ZVk`7_qfW(s{E&h&pfl_Iti;5Vwy$6-3L~Wq zgyvwd#25fEMWh*zc<8=PBV`((-Lvq_?Kjt!`$W&Gph=7$+#8GH^K{@$;eTC=KXLHd zR7SHit^b>xIJ3C;6U%~qA%$nsTaVT35FgF-Z>%~e$=KNMzYMvs;I!o-EQL?rVhua5 zv*SX1g3NrCDm<5f&109wS3a@fIOhegFz1I)0k_1AjKUIP{c7G{`x6)hWj((7d4yWs zU8$cEB2UE@m#*C0pDCwo$~Cp+KJj>{>AnIn8u#O(U| z@#Jk>#a{{W@lAjqutC-^HhRQm;a{TD3EW|jg6HXA?4;%QvF!~j&$D8Rvmo7bs`o_} zCoHEE^%q)TFV~7GVrd}$ueiEAQ*mC;E2u(GxjJ7S*=N^v-QW^99DTjBCVbq81M0)# z2cZ`OvaEBRbSWQoJ@@-1^`z3WmRAnHQ^~yjJ~-&KASv9mezGTBjzNM|#>JJES%llR zYik@77*=h94}<-ExsTfZlv6@e^XJ4b-?1D9lb@~3uNAIV6n|K@=_T(KFo@?v$>j%0g{8_!)-c(z)h%J_z8*xR10A$bjwu zD|J4WazVUVvnN04v=H3U-96D@{9RV_Xc71?dV2bb#lxB!0RV+wb;mI0xlDp6RwS8V=0=%i(x2SuUom}f^@x`XZKCw-pDoH_6;5Ust8Y`>ht z@$@f*L%ny_nwy(JS*8=&u7mM!An35ko*eD<(M^?QmEBRBcBI7Yes-_D&FlHi@FI45 zfiougN^NPhk@phbq?empsXPh8=t)Sg48$I)L0IrmkKtj&lvk7>-yd6>&4juE3!&&9 zyP4>k!*R3Y@sT=T;foc8;&yFNbsR78)CcsrEj_wjK^nixu}hwul+;@2d9kJvdbM?R zxyAJn_PrDb&*PvMk|8uxMpg6ZHCtx7T0)J;QWum+##FfLr?Y6RE6w*MsQw7q8$Vmf z6902BFjExGi=n!B+XA)MJz$6^Kx`YnokC65y)+#a4GDnmNyi!Yip{DuYgLn+hYT^l zen0Hyvz|o~g3g)+rt7SpC*52jzIO(UIV+ap%~WgthKK-njm#VrIEs`vsRm&CEfgU3qfSjf3e2rw_06{lE-Mz*zoTuD?CR@Zs9KHoHV}!+Bp)%O@ayDYKlavMjuxV4z&tbugkE zIytG0G+>)PbH2nANVz}+2W6k{LQXWo#0`eGW8Z$@Z4XuHK~P|@tduR1*YHZsVnJkc zb6$v*ktw?9)YNu)+5F;K;lhszE9%h^zc2I!r|d+J$T9VQE(8cGY!$t}dw9ae^XU8M zzKn6ZR+lvGbLS6eXt=x%nAss@jh)SY>_4YNa>B#8{F9BH6VLE+wqVRDJdUboS__E? z3Fj)8lbb_5_A|us$pUtFcek@x-EVQOCDB?%Vy%0b7-nqEP!=C0o5_UP+4GLk@D&nup~cVr?BL!)=)Yqqal z13pF8_#^Rm>B9Cy$6Aas_p@&dzapQeRZ5}KU2UT!T9`FgUS5iCQ5$p87v#L#h&TIo zz(Hc0d#Qc>`V%&Pk64U4d^$Fl)BW?q;#+|m4x?6e#9C`>YC8}8Rog0A!>f9e1%s^I z7D#v2fH8Gi`t-)S8HgFysF{QXz0@&aj|DZrvJ zv9QvB5quCS{VNW19f^E)H$YWL&&>RCFiX+2wzrfZSOSn^Ul8QV$;kn6tq-Jx*{@D# zdwT+RcXzO;fEJAfmPVQ}kWXLPbMW%|&&}xp`45js2Wb}{A0H|C9mAWi#l;5>zR;(i zd0wJn9Zi1UPqn`5FOfBHQxVnHE=!n*Ysw*EeYu{E{oP7XWM15*YOfFmb=YXdtU$p4JaA*Sw4<*j87#N^hWB-De}yBUg* z06+f*HDi!OgPv#>Kjjz|)DEg{iHIqy08?+XQ93d&t9y!v1oYnQYF?oPpmGjd)4`xz zn2k3psLVu`2bQ|=b*P}PW9`~Ygg|9*rDb!f8UXS4@$n=A$FtsXRij8-6El|%=O?hM zD8EjuDZhZ<;428co3N@UZ*6S_$K^X^*$y>jAe%x9YJCsXg4_85C5R`3=G#LhGLBai zJOihy9WyE_?m}e&h?hhG76;zV4-DS5c@oy05NW?7Pg>o}-PnbCJIbr$cC{L4?}Rpg zRFC+K?b$}dEy`et9q^aD9#_Bl+w1pwA0My35e|Y!nNO5~ZWhZJPByDgLHw4`vHyXl z+)fC2hZda8T)U4q*(X$OeG~I%r={kL+xgMB%WCgxl*G0Y*SwCQ0}bpC$X5UO$;9|(g}Z3zO&>| zy#s0~ZQFWpG@Le1VM?djcbOAb)f?PE#~?iN==V1`fWf^gQr`|xa081$9yl-UH?ua} zQ4viBNc)Jcr=X6@3N6w=ac!UbrX6EQ0>Wi`955m&QGY_1KHf}g#yZcFSkFesx;^p_ zvAm?*01wlpxZyz@((ZmV<%bxi4&8J`4Np(cb0Q(UfP1%Zz5;#bj``zz?mufzSMxw4 zUa`|g8rgQ{CNyc&P7WgBAfZb@CU=sy+|U$+PG~@u#w0`Pa1^K|tzHZ<$xOKWz! z7?1a}g@a#Y%QL&ygSB0ssi#CdRQF<7cP?C#B0T!Kpy#E4WzCL6&G~M3?0EvPE?{Xw zCJ?j=Pv@|uh zn=n{g3R$_|L2lXveS%cZVVE^`Z2)CjsHb+EdQ0SSI+)Uwy8VMgQO;6_!kqH_9;}B1 z`|9P&W$V*FistpJ52Ltvnw3?!>fh5l++a9#)R~TN9o$`vyuG{xwK^_>WLuuX4*^k_ zZWZe~DPm7_xp;f5-;;~>HG6150}S_4Z95mKi|8F2*!7!3*BfvCZN*NwX^pz=p%M!`oKy9MDDgC ztJ{HYMGxNTneYEL-QvRs`B_Tp0y4JKKtzK>nM(re%}LS<3(n@^ zUvTv!AOW$O4ps#?9z^kxORc{|#?G0*>7NX6Kdt3; z2VV(O7Lr{z1L}Jd6UiV=!GMxO1jG(t#eC@5;e^$~xLIMVSz&F=khw9}!0(ILwO?9R zt=>37ck3_DX09kwj$7#N)NTP48bZbwVXtLwvZ(}%?~=}=5_}$5GwCaj%X4RKSZT3| zorvP4qSt7)c6LCM1Ydq|xy(-SAOYgc>Ew5;nmMO<3BiP)(1&*m?d(f?v{N2>`q!q3 zbUz%m5*2kCe=uB@CqR^i$jRRymr%`Pv`1zjey2_1tr> zhu(&FyKN!#^`?G0?ZZH0*He?pZAVYPMB12Qd2F18<5rX}cfa=gJHpV9+O`r(mlK&! zf86a!P^>c}>MbclEZd#MZU{@pvB?^$V~^d``<8cM6Yh*+^ASZpH^fFOIG-fZF3X`veUo%YZLhyhWM0K-R_Id0?GR=C<%sQ4YD1?*tBJ+~`79&!)I0l#xB zo=LBQ0+vKxn`?-h?kuz;^bu~0)Rf5gl5&I`)<=EodrPfhdfhhdQi41?mVf`AoAaY= z1E)Rhab{7oKOoC%|K~Q8zJk&@Nf{qaD(@Cv{k<5xGlhDq86l`)0fp?}h!k8U7U+@J z3aVK=WjXCbWn=X-tatTSa&eebKc z>K-#RBGmlL$*$PYYs%?hSj`4DBO*xunI>xeV4o7qa|QgbGNX9<54y#r`C89Co^DHW z*ZTxdw^A-Y9MUB>r1?=a?af?rPdlwL9BXQ_g|mIqz(@d}bI^biOT&w&VQN-%#CpZC zQr0=Am#ATMYb}6C$Dd9SDQ9K%jKXO)BD4AF#L;GJwBt;Yy;gm1EU+$ zEwrugz2*baYt~}40A)dA8u-Q~d3zR8s39ia1-X)6yhpz$c?Os2a<-@IFHhH`M-VYe zVdsJ>D9ZEmnLMUI4eQdlX%-MdDNLbYt?jx|>I8yKgaR8hBcP_8gW+ibS1_H)u6uPl z66;o@>|`|@6E2M%*%?*xVWZ(4O82xd8FqX5WLI~0Gr8mB{j*BYn}6DuRR#?OaG;YO zXB)AcjP|@_2rCj%Y$|-xsjv;(pZSce$WZBM+LaYZ_wBx`^S-}HF?1&{2W_dGrhb_x z#0n_ok2fn&SnH0d)*W02Uh9CTb~+q;v(w7z!EOkq>3FQq!*_am`uM=}tlYC1QTi^P za36{u$U}uMc`rHi=7pd#;Bjw)Wz{WK2ujSkwFWl0xVW5lT8VZVaXi>OCelRx44(1E zM6hhtEAh;@*`4n;dZR`5ve0PeUbQLZ@#()I)Q3VnuxFK-(G&==AW-&}RV$w}Z^%NB zj7X=q+eocu-M3tI&krHoXV#j3S`s0Ugnc!0b*ejRGhs1+ysHF!CrK*+EO)n^wtKr- z7k*QoBe4qZ-$sBT#brUkCRG;>#%kkJ*Pob(qLeX@>g?|0)tdH(2H&@y7aOoRfWDFF zvN8s(%5|L93Y64`Q{nyE|ym?lDgU@!2lkl!zHCygXS>X@|reY4&2v zvp!ujkMeTcPnz5jq6UU2rbk6xJw2@pp~A^Sb0=WfF9Z8tLxeT@I`pN|qio%ucQ^fn z&-KoWq5K#wsxwcTYQ_=%__EsJ>+$*80Y9tSyr0S2w_Ka`SC@qC2=GFBq^~W8iuE-V z{X5ME(1n$^G)n8}z&Ui^QySx}jrM2Y;2br#$*j!ov%(?Buu19#09AuiTE(|wE4mg0 zx11Huca@|=T2Ot*%;jaCBQ-eIpG_Sbf>Q&Z;WBo01n9ie=}LAaf7R7m{UzfP3yTe% zW-ytE*FxEZ^}gBVQmcYt$Kv0kLc#hnTJ|g+xI|*(L1>H&4Y8=?FF(Z5#sdfHBOAX3wi{3wvp_dj31JfIS4aQI---X zZ&;z;DML>Xn34v?>&LsX^<-D&SrKFbSq~B0ZNMNKq<@HN#x?NfBeZX5?!3^{jOS$F zeY-l8%O8o)oXPd++a5J#<5~Tc`?~A42(M*5`BD}C&45}Sed&Xf86?3dtf+Q=_Xz>jzfBorPZ6>M3KJtDEXk$qF|R#%9l9!hp=**R4e+2<~q^@;;k z?TdrLh0j-ep4P)^b=S@gMqYOxZghLdOE&4%z#8|Nrd1VXHrzuOWW3Xbi+Wd9HqsJ< z*&Oq3u>SdvPniiH(Md?!7MFgD)W3Qv=y1C4E1-5f$@hbW_a<9(OfMauJ|ka@#+1q{ z?$?nYi|?fm&rinM&B(=_eeQCtw=OM6BsY#yKYpZcG7{l)>tidEc6gEXNR+a()rhFX zW&xhv$VAFLmuD`JL4gvnad{5}B@4$p3jjXzMD(trs28(} z$$5BqC{cI)W;!Q^c;#uo!L^S3BTx47Nm~q$lWyH>VI-Y*UO)#SNiwb0B_qBQVN|coSmI_ z3_Xe-Xp5P4^z^l>R+DS>7y(?s#>4BM_6`{{Q7}j+79;$6Wb9}N-RI)s6OKSa0_N`) zdrJF8c1qV~&)+{1C;I$3BHruTKmm>H!IpaO(EA5@M0XPTGz*vc>pM8~y<2TfoV(Cx zD1EZBDrR@+LxQDPdfoYsUeZrNh_E|NL_qK)qrV0J31f65Jy+fA_jr?F0;b)L_dQ1*PJsVdoK71qbmQ9 zTAE3v5|U@d=g-6T-2%2mVNOo2Ga*33>5EQwxY`$agH?eB(AR)~? zzd83@YR8V*Ua#?W;fP%i1Y0+d;-4ifIVwvZ^D)y|iLfH4mjW=jT(=^0^epSh1w3NFIS#R|Me(k+2@PVQ+H&~Gftf_9> zhxOIfFP*9X6Sx}e15SWZq9A+UHay2O!nuB_eI1O z@%We&^QoWTeVh3H?H5%M$y|#!PvTbPY&5mj*@qd@Rj)HM{2HQc;!dG6Ea|#4XQ6g4WEACt&Hf<))7J4* zM`cY-lFu+y!q}VLHizpPAiXE^`m(27Kt(`6aJ-elENo(BZ@x`JLr3TMg%TNQn2@!0 zgj$L0IX8WLTSnUl)JaaWJD&v)qMVAz7YBG|QcIJ@b#}67?oLRDVzsA^{W;|{=^am% zbQ~}$-2e63Bqf7JTK{3l?wRPBdtk%RvxL{_QWZzR6i0RaYdy7v$-jhn8O37^qhewl z4x*6!BZ_93GC}Xmo#1(G>IF+icIk$RO;RVae~I94qH_S;RZLu=x*P(3q!zcGkW1U4 zU-HZ^t4gMnGo(J7@>e@bSbr$2UEn*|ixB(7Cn}>AE35nTn<6oy8T% zu1-xwil7mRF;rGjL6qr2+Xv7PZ^srvBa>kX1>BmP{$CFzs4IMx5@??e-BEjiuer9? zKmOC}m+uW491>J04YnMQjEVG4z(9YFOE4MfE~L%pP9o|p(dk3o&0&y&qlCsIKfZh0 zB9&%RYBKG0+Xn+Ldx_i!FU!j*WJyQE4T*KiN!ze*@Qq(Mrdu}DTG>||4KGitDVTYJ zX+Du;Mwwq`IBwE@voZEva(IurVn$qHU~v4fqyonl5x_}FON)Sjphk!F65@$jfIm=? z%%QcsXMM=$HRPKsY&A|h`f_n>X=Zej*Bdk3p_1k~&>v z2eKZ3P^y=i6QcvvWAr2L8>>udj@Cy1ub-vj30=eCxEA3;l6^3UV1AnI`UKIC5Rey4 zp&x(}0AKcarp(@JPZjYFOGybWEDW}v1PlSu^+E%V4SFUDiF3#6>sNA+PJ(1d@&avi z?I5@AD(S>S>8kW7TJ5>(X~`9@>#l*EdU584f)dFB!&Y;YC}s+WIN>7F(=!9Hc}o)! zeKcA6k}-l?@p)-;#e@>Qi2?iA+V}*?y(d) z-xfuh_tUDh6)^sJZy4i?v58rBllXJj$g+3(6=NN9&=X?T{%4z(tX1Em24~;i^HRn!wl|FF|?WA?fBa&@A&6ujrxWc)%wTVbD%vwqO_W- zA}nS)^OQh-8xVM?&Y&??Y#4M^W}WpAN%@%!@)wD|(q+h1`IDMrHrgENrGHjFnK!q5IEa`3vdGNGL{vsRl z3(2={{}|6WyeI2fUl6VJa-BJ}*~**wyJx|s4L4juAtpT&)B3=OfW!LTz@VUu_MDs? zCjk1Z_qj5%{%&2$MWodDks7;YdF@Ku>#%b#In&T6-M+oRt%Sr8OpwbRae_V%BWHXtcndIg+4dc^|U(#4u zQ~%g9O&P%`e^5;|zWx#V>V7|T5~Sc%R>`rH9XvND=`qSQHVrc4%t)@5wGvc~mOxcmISf^lt%I8LbL&7<=65xVW9Mu|HXlDYQ?EO&#OT64v|npHWGI5abahM z#>I&bL&+4My$kTo&bm%nzkN-}^Tcr7c>#;;cot>oL#M||aC-#Jtxxex-?qBDMZL)^ z3?`LLeuaFYv)LjgKB_yNK3lcefB-z;I>w#jP%|?#cB^TDtIS$sX8{U?L?d@8OG86L zz0_1CwvS!YMQb8?2s*pox^*je=%9Ss1p~mUYe<-++;Rs!oAgN z%x1sx8zB=^DHk97SX@z|t}>EK+s%>eacVN`-;bcyH$0}K^z|;PP3KzzE^P`R(>q?% zy;{90w`M5q?Ci|VLuot+4wq3u>i@ua?5e1t5dhh9)s%7%tmp-?pDIV*o+28MuD8v?qcO`nOC4;i0ts6f8AuBO{~*XV;Dtn~9$neh1>PLhY)S40(1za-@W<<{7DwDKtjQf`a!#Vefk9j9*= z6WnqsLgZ^Bg`ul!s~gU~k6yQ74)4JFwc&U=UJMK%kOia>?{A0+-?87BZ#`ZXz9P1* zZk}tHZw)H4S&+E0IGVImXxR-g-d{WgK;$fprt_-IxIbM!<&BzD#0NgRrQ=Xg#9%x* z$Muh5$4#7@yXCSoq6%FVlihZuKT-&ed!9qqsyEly&^)=8(jVJjGE5u@^psFV(+NUHj zQOGqvzeYfvON|ES4pGo0T~|~r+n^58?Ad^lcF4l`n*q8__sS!zJF>1b$vS5to9@u> z(jU0fFEZ}{?+uoYd@x5Rd~yrWhCAG(OS6sMPJlt#K+@(LC3*Fm)pKS0=JMukjH|+G z(Mbh0MF=bJfuea)T<*=_0Rc401Aqs13stY~gmFwv3>1h_emClAf?^n`fv{N_pf#VW z;>slMgPo*e;xIM0<$3iOIx)-@COx*9|8Rv956x;sMZI(HnE%Sx7^Ns_#8At~X#a9@yPprdck2eIZ_c0+UtOvr>A8I)TA_k9}}!2?H3Rwkwv&#N$ zh=jPUrt?p0!=s`;q^GAlEPulHd1Qa?gh?(;LDb!)T%K~i{>bfWSgk_u&>S;?gZ5|U zkg+lve~nJbuaSlMaBAMN*OF3q+Pr|#@Lm~wY%h=4k_~8-v%k}WeNgnBW_?`RFv-kp?*6H$TbYhjqr_-)y`$aU~ z$j-v2xH8dLbxav`x#(wsfovzA-n=T-Y&NphW?_~5v~)zGXf-<>>pG)8b+&4=0j(l} z05hCx7rtZ%BpdsQK_S7NJL%B!NHvdeY;3HfqvHv*Zeh`>x$E#SxK+Oq=!*jM9|sQv z3oI1=@Q47I{Q+$A7o8_l_JhyvW=jiie7rB}3u=5+aLjwz%wVVyt3v;UsCT26N}lRp z&>@ymx7)OIK2kxb@ll_EHx?=Pu~n=g-=g2&^mn|}Oa;q|sz+{QVh?lq;%KHFz`1ks z?^;?~D((a&6fS@6*Fdrg@#`ngub>XVO6fxr{lJ(85e&pMErLgrDA0-w1r2)^Fm}DI zgxFX=1>uV!UEhAfqHmWkRh~abf<}ch2^b@xqU<*FVo-n#A>Of-8EcRzyXg@9`b0U7 z-O`Vpt0GgVVs`a;8lI341kjW`R~W6*C%W5md5 z_HB%axd9!#w1b1KU)FtW>yqId8WF|bJL`;!^Op`4A}Pzv5PTymtLQp+&9d&y5|fw< zUE(`2nDe~l-nqrisq`zcFw{K4a>4!Q3A%R19zqgwNr5j!dR8?_NQ>5hL%H~HtK$*s zO(P~|)A|}_NPGa#+7BV;t#)-xC6kXx?~1>_iv2fwmd{^zKd3l_5Vs9*{388U^n9&y z`RH2c)Mx{al3Vx+Am{tP3SN?`Y(*t!wwG>LUVPTR{ycjFnd)U>wSl3b9)Jo6&9`y} zTPYODBfJC<>h+RWkub>ly7d&aw6q9~PZ-QeNeL&M)Bt@yh!EnwBIqb&KKkwcB2oOX zY4GvegF}JRg6d6vD36>2bI!+fM|SiKssSfK7IwVb%~}NAX)`~6*7tah?hHP*da5#8 z?$dj4QfaBF_spaCA3efHkQUHKBkXHV;}&{tp{E*1-iN=i+;bDg_USZ0mmdDe-GS`i zqXSgkC1?P|&6UxS5LyK|AZ8{bJ^ej0_9};+65ghE9oeT037-o`jhKFLeJmhdX~|BR zueO-5UhQVJnM<&G?s>wo$kro?kA~Thq##7BRMww7+ew2J%QBaTMf&{weDC;}r@nEC zkn!7D>B!S-WS?8awrBu{GqpI;AzvlUV#fLVRW$f);y z3?cM-wK|lOMAGw1(M9JEzcJy+-~>gso9vfhd3l` zU8ADs*a??>wbnq$uMx8|wVbjr8bZGP_SI?tjdVv6$My0b2}DI7KNnJTuQ@Et`eO0w zs@|ZXL3$ABloEFP@G^&wFYa_rcVV`AYhLN+aNi1}8kHy=sZM%9V3y#mqq>)aYLE*Q zTp-AQkWE6ijp(o|EiDb-AO^YuiBJlx7jzshJuot^IK7w7WCk zq=mh0>_-WM-@o-1KN3sfkR0V#x98@I4HvyWKKyxKThI|(N(Oyjp|Z`qRR8#;qu$W@ z30s`p=v8?=dxywSPsI@}74r?gc{v#wbSSJUE-pssS;PpVJvJ+sl_RNPX2+Dn{lDNi zy<{0m;zMcw-ECawLb$nPbga#CcXdsVGa;O*TdL>Bhl?03DkH2CTBR^iMb8zyf@D7L zm{XTy17c}z(g9oBSAD&kN_z%`8WI~Pdtd79Z+(6C%=&Q>TfCf`Bz=Yv@a2uomaLfO4_F&u z&hB!V5>yuAKw9zF-PcVr=3zQm>60iBw1H_1rdqw^>@iM zyt@4f1-I=&yh$#^@?Lj3oV|lmTrAev6?UI0!Q0=3{HRVWGwVvVaOgfXqK1n5zuGFO zRKIrHE@6TMgH{?3fz8i{dY-?wN_v2c`}V^a(>0PV0-RV`13NbF)FXyxuki83y_$^; zyu82Q{C#C6wE$6s%Lu2I{N{3Cy129?QIi6Z!u#tO2-zi~ZJdD-FD~xN2O8uq|ItjE zox0$h`$OJQHuW-IIrz1T{$E@XH)>Qb7{E#7e~Yf`dBIhnU3m?1(5NVak9XOxfnePy zS`9hCU*CHI%k9ejRk0sUfNcP_`G7{jfOyw9O2f}I{S80tjiF3Zs#U(pGrR*UCbbnb ziH~>R;ZjTNADBR`fLdHnx>$OtiUBTfl2&at zN9#PAP&v_p<>h+1R&yQ&!{L4OYKk1Dn7&WGL60GSdv8HRrH#Jy%Ooc=%NqsSxcF+m zm@8}F13hC|0`5sl_|mHtO|+6i$;=8mqSdIsey0~QRZxRIuEuo=g;(=v2bi=$%?5bL z*wT0!Tu_9P`4|&XS<{LaDIJxKTL0719M!F^PO#AF_g(Mu9PDPyro;)waT9pV6rV66IFT&{&O%sy?w_CaE4K4n}irNhDG_FU` zT%+)!*RZKRJFJ-xo?wPk8Ny;7OlN=Q2!d~tYX5grafj-g4m33WySFrMn3|oP=FbNe zCo?dlUj=j$+Tm|n_u0GdNDK{K(svM%o3|E`6NHD@!l^!h8(NDAQ+zU)_aKZ875L*HZ~=0 zK8>ia+O=IKj7jkI2Pk61VXaiZc&Ib}U7OzY@7hNs?|#Vb6iu*7)wQ$JANsNuU+GnD^-g{pQ~wML zfk5=%=cAdJq?a@q;Jt&1Sx_J)IfYBToA1Y$$wb_=z8m~RUq7{y5k{dE_3xeM-_ZnI zl0AsL=|IogOG=8I{oShX*;b~hvr^Ky zh(&`O^5H|BAP;=wDuHy2VP&Dhdl<$l4ZBCyC&kAVY4?97G#lYtUA%O6zxO9H{^5Hv zuba1N>5itH;gu9)+5S#pVWNC$m8O9$YOJC9*Yu+gGzS|e2R^gAns0$vLkoh}*H{eAyQ&5A1#n)r? z72P@ez))Ig)SEXje>%>fTUd+oS!MP4E9FMSj|;ai!MGhz|9gz1WjVdBiRGPkiqM-L zsA!LW7~!seQ{V4ze;U(z!cV5Jk9qs1<*4~P+k{LS#aujDV(6hIHe~Ym$Kk1*RBx+W zi$+LqIvl&rb4$a54iZPb?eb_gEo!)=>_#znl$}zSS=6lW*raFM0z5XH^6#q>q@v=H zkX-ALiq5~4{pjZp{M(H_A5eFUiu@a%I^Ga8&@eD5De>o_Jx&)W!!wqaL61rUON=9F zYkQOgllb4uoxWzt=pYL3iGd@1BR1_#ydMVWH4Gv+kS zhoI2t2Vc;$%pWjF8yKT1cu7k==OKm)vUHCC5yUK;CbUz_XpmvQ<@R9;dLk(s)KT%= z<+VK#eAUNH8r&8sIpsHjaXnP8UpF8fPJ%tBLi#%jijo!D=?ao&X2#D&W5gf$b`y4V zJ)e%+@Qo-^|;v_+rX9{ zOB`JY>m_IQBqQ8a+9zvB~@xOogcQYX_ wrf2!Pdl46l*}VHZK!~gVYb*S_^|7Zd+v%ZGjz$igt%D>jE-zN_^ws?&o~`cp2I>L zjTveUE5^VzonBofTIw~-+^k2%Zf`dtDJ8&cwp-G*z|f;V$LYpoWd4XuD9WY_9*|mS ze|e|F(MBu6RIBOWtS#cjmsgHGKW%$fSqV#`mkwc*i3d}cT zVKv_XHria2>x}|al#J^_xw=vCH4bn`1WU#{ajv>iQCE&6F_{z|ZoV8b>cD-Iqo}&0L60RC*Ntdaw`{xm zLvAh`JIyb_JOKjD9=WHu2c`x-KVtCiIKp7Qq;COGdiORX%nw#vFKbT&Pxr_LxP-C; zy)VMOf6R(}t+)5Anc45=nNXwxh>8(a0C<;Q6By3rxok^}C2;7Cm8Crx!24Apd>9f& zpDbUZ=mOU95}JXlQ?4xzOP#L~UU25F_|Zi3b?=uS#;#>=v8p+VRje0mEYcs*#Y;25 z*$d9w&JI*(Rp2HA)CnIHmOq^L-E`T{yUEap+>LvO%m4>KOAj6$ZGzOh3&*%iY@f&)Zn88wRtNC60W0F~v&r z1z$1jPtqFfN}3Kk$I3g*l{rp#@tO+5rcNWGncT(TeLeaWqqHEF*Car2S^|#1L9k*H zeogfGnDNcU?Ff~)hyB92NHPxFbbr#k|UjI9G?jd9IQUue47)6EZ9}e&nXud zsNU3!5PtB=z=tm&6wh+*hp)VB%GdqgN~^}I0z|ROM!$M(&otwO_nNb|mzdW#aQVAs zH=_l5)HKv`rwO-5W4j=9Xj51$n!M+GK-Fl zMzrf>65EHcST=p$JzkVmo`rAfz>m8EM_lvl2c42|b6K!-b3{HZzTPaloz%5}(->g*@6OT?$F_@qFesoRZYWY*J&oVif2A8Vz?pq1)p&uuyi+L4OP z$;ovd2-_ZPzTBTBh=0e#&;5xyZ?^gL$Ytw&v6_@r(9+R$80 z0V71Y1%KF9>(ul(7Jg(to9wh1V{JFMW&g=*#QH>Q)%^5xW9rwzvdowc-I-Ul7u#VM z@`(&Q$$L`HOeA+b*s6!CxSHeg^3?`V5*Yp+I4Dl;h82vI5@`j~{xbWEF`fhez^1gd z#r=YjpIaTxMItA^QWpsvB%P@Gic%ZY{5sGF;4@Lv3hRxoO*|zq(MYpCEw3dEgF%st zNs;2UgFP!XHp&KR5yJOOQIprNz4P5XA#+f|AnkLOj9{bqIrP1O2kLY2z>w~%E zBQ+G4)sgwg`Id8$S_7VjeRMQd?Bdgd*ahR+<2drmg9W}0=@8}VACV-^eS0N?&nH-3_PEn&n88KI8?#D1jQah#yLjt4jQ0Ek%|PNw2q6MMB$chW zvnPdJ61^)fd1O*5@*Sh}i9FL;tZQXsISn%OCSMgvu0D3NQEa|+Z7w->7WA-jIxzEE zPEJ+oC=R9VIPYM+6!SSdXzp`A^*Pup9?hLSSwma#2Jju3o!P6I7}Mdg9T~ga-Ka}j zenVR_>ryIXf0kPVc)V+~Ohfk&(`%h9Rr_E==7d>;uYz0(kgt+!!x^q#A z)h3GBT+}v<7*bajS7A4E`IdPX50|4Y*Ho^#@T?8iW_=2eo({**^W!xN>z&t)SpDZq zO$N&i-aUv}$5DV{mKpc58K(sg$52R_Ws$lE8!OcD((Bthpq_1Hw(fZyTzb27`HCf+ z@%%^`YmnIabV0_~kt(!XNd9ZW&wXUh{>O%a{pA=fDz4NOqBitB_4>;$+RoisHLk&6 zxwk>7IY?*cD!cm3*)=Ylx^`+U3$C01S>H#_{!&dn9nIaZ3u}-cJ0Ed5-E68B~AZ(J>aMq7-+601MG1tK1ia(+1vi&eSa$kV;g6w`k zdfhs|-jHWd@oK}1!4#5L1|k%*XG=%-uA@;j9kdl7s=2LKi<LUUoPbdUX7neIsH!@k`QLlfbQ-XvF0q=6%s0W88Fcfdi zI?Pkf;BtQKpqsEkdBl!sD*!Q*mU^+AJ258W zzWgXbULG-}l_I3Rl;3@S@7%!4Kts=QgfDi|ZCd|)eXp6{eAW2qVDD$e*8xX;3|8#* z-O(QAFdk{_^FhR=Fn|U*tOKHy19LGH^-n6&$c%n)11T^2DU*wY6HC-5zj1KQ5sd!`MD;;d3>KFroJ? zJv&v8&&j{sI-5;0F7X~j>^1evOd1pr+spNr<-$|7zi;Yp5mxDVQ)GTt)KlBWj`DPU z1N{4MSm3sjJ#E~ot65|8xgZewIq}GOEOjQkrs0%%eQLyg^vMHm0wl)866)#v#C0-NOguMlBn`2>X?Re99i5JTcAq(^k z+4#u3CdzZ%%L{=^$`_98RPAPU?qrk3uHvd%BAqpQ zk^B76kvLR(NJykX0#7)?AWBAx{|;Z|9^BcJbH!x=Ed&l~E`wF*S2mh41}>fjupCVn zZ6!TR&THi{J^PW>v8$$JtjNRDt(ckF?F}857rUR|7}r-XQIJXvJ#Tuyzt<=-&_td{ zTEW@BakzXSEH1Up@rb(#6=rw>wO+hwWZO-%&mhZP`-YM&TlT#A5*ZdVzP!c9N*O!vS+ z2{~M77GHz4VE+8*;XGMYv)1EtxP&I6DRR-f5d4gqujAoDRQ1T6_Z|008R_W<&+OsN zO^2}t$R2+Ehi$w^wohi$g}t@~QjrHfE{#p6of@&&tm+G=>PdW2?Mu$d6(?)T{n(6m zy545Lq}84|13kfBhb&}0?}bPJ7ejX^#KOi=Hsl}#bNfch7}sRNx2s)L%%0dB##tu7 zb1v`ec}e-4Fdm&KCQ?=pb^NTM8ux?o{1@dnCqkR#f zOQ%ta?+eJ6SNr<0S}N?;gE4SA^UC1TW~!g#P0)So*3ZgZeXeFZ*f5x)8%(O%~eeWiMCJQ_HwD{20Iz(*BGgA;K0 zM;}@zofCQ5*J59pYaW)e30FSD-PUnd{{ndfP*oeJJtYQv8>~6y4_t$SEED+Sd20)4 z8O{4EwSs=!gu%w&FEIdbTGUi0yy9{^@#Bv9a>vFY%lI2-e!k?rx^|rkL;5dYe@;=F zRPmC@HyR#c?QfIJ7I|E(lb;`_dKcoIw+7yure93FJm`B<@7?~wO9+=-=u@FJ8If@K z9$aU-+Efn%1}h{7Gk}(WS3g7FJL%(*Q4HB1UM@G3Bqqo)$K6y^i6?=BTGCsrr&}4n zJiOR(Z$7HkoM^8Cu6XQxbYs7$`631gIMa$XY9NL)`%A~5h(i$6SGJQ?hFV=serLN) z^8Cr_M~1HbWgM;@fIR#I!Statz46o57pzdPpJxVwI~&_;6Pa30+$-ZRyx3U|3=;NE zgJuOIDmdN~l;y`2|B7OCImaj&6S!C-J6#Gl??6v+zn*;|13$@CCbPCNzt|b*|Z#8)flV;;y5yshk3dOQlvXGF+J)8Sy4KXXsh=*Wa z0>4VPwuef60!e~XC{#DoXRD1pP0NLR2v)&;|0LrBt^=#-iMqKd1WO~s%E9z#iRCpzl4>g`irn!kgE9BN+rwXU#o~4X})pZ#4guh@u-0u%6etZ%{ zygK)6{52H2(%@Zpxz-R%HNBVV;i#|iiDCa&%U-PBUBLZhSDndDGEak3Yng)28S-?$ zMx-$2kT|y^BUF0hJ#%+s-9%LnSApwLF=`1+z`?sjf*`- zz7BAvk(uEXFR%Q|p`_%W5;V>U+<`OWB-7{ zTQBo&rg>vK6rtlaRQOh)0YV}b&beY8LddxuQsTA{X*}z-Q17JQk-GrjFrR_<>)#2K zz}pZ?^%{6{L9pZA2O{CoEmU{-d~BJt9<&VyjRxL3g=4YHW>1s&5c9p(6JF<@?%hK0 zJ%nRy3n{PT6Z^~DnpCHNE-?`wNs7zm4yRBx_k#WES@%iz5zqjsW9W2V1MPgK&#@92 zxlm!s3fwW2fUqEUNM)7O#g@08qQ`4pQx-XSIV(Z#Vh~3}Q@Suf5!zUh zgW4)`=h(S_H-baYy({{38sV)m>-}qT!Z;%6-OPR$&ssq2R2pR2KxHF_>7r?bcy^O6 z_PQ>IdA#>S&`>(6Sew*=11g&nf<_ubH+QzMWE>) zjoK$hXW%(EP+h&Ac)$;tj^X^tqo4zDp6h)5T%6TTWeI3;hihJT*;3NV%tgQ?(6+bU z`^_TVcuz4-p# zhYHDA(Lr%`U+N7V?rCT;TkC@H_PKHWX1&YZ3!g#o_bpfTb*-9l7$Lbg&2p$KEiJu&xG_^qqVMR} zwErV^e^n*S>igp7$Q1!;vbu~t%<0o^oe}SdDz+tp*hbMT>LCA*=W0`_q$|RBO1|npt^$_0|wcG2GA2 zvn#}0?**tHOcg138QNzcNONG4d;2+Ofs*a0*+{-!Q4fFaGxqf2Ru(d1zAR(o4t$x< zACsgChUwJ6Ob{oLWVpQX`D-jxc4>9H$gFto5zvmEi=9KQF6bXr_0@~5?5lp^cvo%P zsQG)$A0pKI5-Ps=xHZG^4y;T(pFPTzXE4oKba5X66BNRo22Hu9oHvaZjx^?x@VZM& z9CDK3psu@ss3OOu<2rK<&*a2Gp=|Cs9XG!|QgyoCK8@ni1)5+yPXlaU~7?OSZ8ayVX5o zb-*3K3fW$1fJ}m~|4W1P|7S9Oi3b019}`oaxyDt=iXeLoa4F#rJvH%sc^8dHHBwa_PTbCdf4I$ zv0&qI$JejJY={Pg&kF16>XF}7xkBu&Q5j6S$zkFk}t z^~lVOd@5wLp8!6hlcuJoUf3F9kW2H5fSF~Yv0OE--kq_ zz)Gtj&U$XlCvrS8I$BUw6^)*$85EH=-xru##TPrdkv=e;c6LGv@|_FCOvV8 zHu=Kd3I2`iHiej_1~hk+e0+TLdo;k#32t1EYrGDlS zcCvaxlb{rSC+IC44(AjWwsZf`voy3lT|Z%V52mOJ-DBh1@$sj><{k8L#Qj>+*4Ni< zKzQ^3%q}m^`8|(Ip|Ey>)UWJKXx;~-!mIwk4hIhpQBpFnd%_YcV{2=>3vc9BM=LD) z4I<5LBo+N+uAi+L3T%T6dj6rezOys15#(ZOn$0gDV96GY^BVdg3BlD5rLvX5eTT-! z_NH%Z=oiWnS1YZ zLcJ_3)`qX`1-^foCgh-FU~uvBTDX4z8Mm=fRZ)q80&j>1by9V=Lc|F${h4@(xUj`q z4Hd!dxHZu6o0jZ2NU+hDocV1Fv0=W^&mkDNK`>ZfFZXOhuBl2$VBmUpH^EZTP+-!xx3?Qkooyz{pTBsaS08fF>7a$N75DWB&K%MDBaN57-OP3zCgKDp>)taoft*B*(0lZpR(3Q}d{09fL zOh8ys*8>IPTAM;2_4m5~uC7;Tr&I`ft#&Wy+3@V_0Cxo_A3x|01)vy5#>UJ~JY##Z zLql=h1<675OFu5zaBRG)ANlr;p=8Y3*c3bne+HeD0A#wk*+3$cv`jFkyj(dsIT@UI zblj2nI!ORk^DJxXfH|~!5sLaaOSE)80wH5;Y#h@81+CQ(pcb}h?dGP5ii+xP%dxR{ z0+k5CBwUGr)xD#l4_Es#0YKuatC@bDF1vOI{0VIbg80OqBU+|1%) zd5|e+BZa(IA9*aN0(PoGa|Z;asY$=pFr4C*uL~2PCwxrsfwrwzh>D6R2rmRst!duw zGEq)flq}Qi-Hijt+xk}31L9H`1J&h?>y@TG_aHFZ&_H0^y9a6*H!xcEA#<`lQGm9# zcBRXPdLkog>9>G@08YT8C!TF=!(YEXnY#(DeDx7_cJ_>>CgC1|JisJ+j$O1MN5Pu^ zgGe=-uEBU&q4lO7+qEmsb1f)6D;|ydfO&q3VV9!N+_?!Pvb2v0W~c@MYqvV7>d3{# z1?0eymlTLjS~%dW^RuIss>w(rqTRFe-4jnpi|Ev$i#7nj3+?rCxN{NNT3Ce7?$neq z!2E6gXX@Q=!omqb5PeO{G-c-<#&`KWz-gwY3L!Svz3(eplVoeg$jAtg*nNiwECbhu z$%soxj3ALw8h(S-)!d__sGaO`DjrAMBSCi`F@?>0+=Nk5KznfNc9oF~>&OI){@|J% zP!dQu4lb@XxiL`yF$u}Y@URJllNXQ}YDR*B1o@AkmS76YFDz6wH-Cg9j*_GXVrOq( z0$6A1F^G?qy?w3%C~7I_CeO1$Ww*^YRQv3ocmnMMfK7`7(g{MfG}1Z#@7zC!qImCJ z#{yu(w-hm?6+^<77yUSBeaO$zN+!mG42YmRj?r@0~DAfRF6qRv$G{_ zfp|K?spy%o&2UMYVJL_ql%Vvd>s>2As(r!Tx%rWt0C8O0+-B~wUS5rFS}6rcVWs|H z+<7=*?y~xos>c|+O%W&!2p^;%imPBdhNv>2b_3how`@wCR^+#)YLQJDu(>}@YY|kz zm8rV>*R>XOs0c_@0m1W=z$29asqqJE5bkj{&q$?37v*Ex3&+Gtw+B`WC zCUfCWUSC~JYo+t$0yFcv`g)gN$1g$Yp4^0SKLijOH^e0u%=5#*(-xlHI4CG83VD

+2VwAdIe0-^)!sY#l!{HM<_WCqZ9a+`#V49cf+{>SV+GTcCZetuOoHB*rIQug1L zsRi&h3<8~E<-cklz#Gsu@rP@guD}UHa=nebedGFZ3fM#dw25BF90wqzh{ncbkoD0b z6ZZV#VwR9F;F=a3Yb!mD>_7>O1Gi(yb+rrNPoOkTFP?b{gJ=Nm^BpOhvNdzv98U*f zx>{Lv3+y-Izd{*?V7(J$G27(jez2AYJTDtj7ATcMCurB#+dDb{vi7Xlh_TWCfVz|$ zxC}`Bz4$@+fL~fe0v|;Ll3wW4*474`X@zG~&yuo^P9hj&1JP{xrnAjm@yE>ox+%qi z)*7Fm|2A-Y5Lrx2NckzjNCGSqCTz`$*s;YzLPD&ptja+yeaRrOl1cwJ7Y3u#jgWi> zt^@(^Dutu*|J8ZH(tglh(c@xc-{u!WgTjIW28gl?@c|AsC4RRgOX!(}%ox79|B%df zHtdsg?-NhqtHsx$#gD|r#fL{mOiw&>k@>$f+46whi+^{*Z1k#Ppt8hggO*|(?#z~m z1>$O5;TxD3@Gb*{K`)(t?uzK>XkaC?L9FrVz+yT>U=lzUf}cHz=@=dxW4v{v)DIJ; z87`fT4>GWhoOkAT`T2zqk(1Nj4qB=n$jG2E{0rfrwWxo!RSr0tP8eLt!9mf_&o8C} zk^_9G*yx`8;cI*=z}LWi7wOb|)4dPFFZr*%2Hh01dvqUYY-2knOImz0*mms0b6J%hN*`GZr<7&DKqY13Nr%Mjt(2TCM?ED zOixD#F*>RaiojQHciL=}?ZYPIaH^m)Su_2Yt^!8ccd)h!T)Op%w5@FsaF`XF;PoJY z>K5W|*R2}hrS0rSt8FJidjJcM_)V06Q2*^KH6bA?6@L)%4iM3Q0NR%WXbh0WjUV%E zF&$rliC3q95c+z4-Uzaw!m~?``5<}ZVI3fOuoXzGQ&Px#y!dHM&zzwEkB5P}Ro4Q7 z6H3bW;^bF%w=`(Kh79pxnAd@$d@>Kn+*d79Ive07yQ~a!h}AX|8HQ{Yp!M`!V*7^_ zzT_nZ_B9iLVjODQ)So_xP^80G79_6`%z^A=jXk8wEhpeJAibcqtP}(g15&5J8J0?87{18hk|PCnYw0)y5OL-b7WQT8Edc6Jkrr^d(g^y*)MW)Nmr1@;ap z@pLXVr~pqiSU)P1jBwx`K|^K2Zn*}zr+?$ukomXovGDYq+9mq8Ff!eRKIZ$+HexkW zh9$j~4u)U*A5DbscgQKPtktfhAYY?m{O*%RN86oEVB=5z-9$h&NrXKF;IRuKzNa#Y z<@C3^0%pw@hwg9l2=$eeA`sZH^Hrq>1ovctn(@)C-QAebQdw;A;)dXE^bH`;mW+SRO28DLs7US)nl~Ve;!-~iwqY8`0@J(>qL$?M zUriNkr7}r;^q(?V2p!Ym9ylgI3-A^dg6%7N^tWz|gG8ngfhHWZjg%#}W?sUnzr01D z#M47@c?&37kuJ+k(9T^JI-&in9aIP}V(gu)E$t2s0b;%QVNal?dH-%1z_VU;>HpC) zSUTevwm|{lAQghE3JD3t47k?nLKTEU3N!oytis>ku?_GGlr}!_7M08K52KzHBq&cq z0Md-@ZMz;0ELtG4f4YQHDxj;r%PDb?OC97Mw0AD0R(ixCJFHX;)Y0?>0q4bc%`R}Ho{ zHqhWL?VZ>FpE!z3h2e`s{V$FAmy3e&ia=i0z#w&LDK>KPcT-4({bUfZN-5Rn-w57_ z{b#=q+-x)GQFWA*(m{_F^WqvE1K3FL%<5_$INnF;Y^LNL2i{>&cL}Co5b^?W$o4ei z?jw0k@-}mF-xpw(k(ReqKa$e1paiUiAbA+D7GP}hkDd?|jInWMLP7$xcXbS~_l(LU z^i0r(LjE5VzTx!yXAgiRsR0iwm;?I%t*tE}VWq_&-*o%{#`6xmCAJ6uYWHEKb$=!* zR-d*&a}8Vyn5%5ugRXoh401*UVPH~d38V)Q9`N!&!pf53Lx3AG6auhRS+mjJeb4mRLY0^0gO zJc?>h@4v@Zn6Pn&-?Oycb)BW(RuQx)r7S=uHEV!N1!j};?nT@=N!S(@Iw^1N|@m}(f#{M3JMCq`hghzE@?VVkjByE zzw0y)bOz{t1tldlVC(@Kjdc?&R;2r=k%|j8qSy&toAG~}TY)z2&Hpx6*;GVNn}asa z^S?C-asW$hj%Uu`EPrJ0if=Jz98=i9XylJ`g1P^P;cFFIcA22d4$&0OaB# z=oW#o0kZ-oJhakfC@)}y2MuUXEJ4To?*T2$_d5_dxQB^9IY#}2GOgCJf3 z=suPvCeOUQykcAxfue&ZNKs8K8njAa#{chOS|JID9A^L5ui@zE=s|XJFemRvv|6wcsRH$aj^>-qw1VRQFJ2rEh8Er@Cs&790_u1H zbuIpM10!PaXbkF4P+$838}tHzj+-6C%sK;1B%wLgKLPy&<*^(`eP5y4KL=-qnZRb0 z7W{WW!;-=Pr(axnL&%3xSeWo%KGFbETrpbO?u&_eHdS?XhQPLPq!vS)=HmB_&ll%sS8e}_hPrw| zW24^hz60hv9aGU~1Ly)k=zXz*#PSShG&`4oz|l(Y`SRxGUmi$~9R-e~Y(^I!`0`G6 zcI{DN{Mq2p(9~G`8M0T1LGj%UzEYMrGwq;f(nusdZB#`3A)YYK&Th|F z%v8&cLHW1kH`19|G;zB7Y|J#pB8g_+s*z{z#ONcq0e zxRK4xDX*2aiO*}BJMkXMNY%pukd~99g7;lV7tY&9Q)dIUXKMLbi~Hv*zdS~IKZN#f z^vd31WFLFUuP_^B_9i?SGgp^ILVpakA50h!4*~4DqPkQlh$@0R%sIKZGV}9?`NTdr z^Lw@ThScVJJ3S4EPwtC2Nu%T56ES4Bbk~}g8vd)iyf)u2UW;r6^BFq@TXgi&u3L9G z%Db@*-3Yu`K$F1I0ek*)$1E>9`?*eyozTuvdpnpMF@edr|ARY0c{Md-yFza`Yx-e@=zUdH*lgyB3;UM>rZ2@wei$ z`1bm^qpVWpv!ij)NkfM=H+r!@Oj%nG-ZIKp^?ex;>*19q5SUd-3DRJ|LBfoSZd#*XY$&= zC2bkH!KWw*oCEaSYO<^T>R}v9y@CX<zk>FOdKOgJAC~4Lp6<3HM^91`Gls!+OuP zEZ3&aO_y6$&Cuca>%aJhql*Q>FR2_D=TCt<21%mYbX}Hdm%5=uW0n z&9=Wx+Xa$_B40}`A53g3$;ikoI`XM1&G*h^!UawD(ONS=v~1#q?>|y_$i)}0$`qr@ z$=|rhb1obHSSXpliqupc)Uphf_%ex)+5Prk*`%hX0pNNSB)eMQ%JfZgcoT9sg_653 z`1Z2q_Aki>$ljRPa@ta~ih@%`1j(BpC ziWu@zK2q9Tv+)=+^Kbn^x{O_iIi0Rr9q`EbzQ~(Tb8=NC;z#4}GsmB)GroRj^wC5N z4{Yh|YyY}X!K4u38&$PvLfJ5W=)RgUC#7*ovQ{r1lr08|gSV81Riz&(^cNW?B_t?4 zczKBTzL%$}chj#n9wVE9(J9I45XI3nX3ykJSZ@>twE5< zH#Zx7%qu9+jgFRg7j$ZsV0QG2wyy(%YXy5ca47Z@h3OvHy}vV9$`*sYJW5)n9M$!LdGCe2z)#pHA+$HIYg?2^ zW)KV}hBSTt1o4W-`81y1X)mksLO600_>!|Zv9&SSZ+a|CN#wkDGov`-=(|+p<2dGZ zTnq=3hNSZvV5W)zqp>trsuAjc@~XPD8i(EB^Uc2PnD+J0l+TkQHASADSdM{Qfq_`A zAm6g4VWwbn3`~db1Wjyu$t)jbZ_DWwE32t~G(UUdU|;8LvaQ5{r0wS@ zU8^gDK@mSeJ~FZdfqKKGxZMB);Tvx3$KF@@AmY=*M^VAgzGI1o^kPUby*h0(Ewg`s zD-k3}zS2-6y^^w@>1L7={CR;R7hc-Sbds*DZq~Ws(!KC<_4wyg+Nh|=tRdw<@WMlO zLV}!QJ=wR)uA+hh$C1&x+SZm}FbRke?|gkgc`tFa#AiAi6$71c%9WdQb2vv%N1h8W zS&6CIIqmV7t4*K%|3z4}O=Q6n~G0i6Af0=D-5GJan+!Pi}DOV9%PQo%HiH*Jw z&X8X{`h2yl|F)wIWtpaOX68#W5pTyqMe2JpW|ulPIvQlb}af)GK*6< z0|mu9%{| z>;3tKRA28R`iuA?dk&FFChke`^z(0tX!oyp0cKQmi z|DtR$qerE?M@CKQaZRH5!`m1!ky6M#0%K4NF820T7W50@H56m(ZeoqEPw z`c+l2jErgA%4h|)^NNnH@0~R=bH8_k!?)AB78fh!lj4Tw3`@Xgad4P(w`ry2w^AOv z8fa@trJ-yn({q@g&3!SWH8-R2Xp~pesCV2JM?a|Acy;5{>IeBwk2noXJz1{Y9 zS&^6I_r5e4bK=3fD31t&WG$UJz1u$wR6cx|Qz4K=6ICmNEiFCdST6${e|mSfBYCya zb5rtTvuzdQGJlyp4i3JC8GjLy`{F^|wvLG6{yx9jYtK5Bu7Ce|KjcG_ZYehH-uz=O zt{Q>v_zff!J+-CCjs-=v==X}K)je;?<1hQMg}S@30UXt|_+m3+KPFx9ti2Mbbgz)m zsdm$%4nq4~afV!g%7{r4hmPGuGc$jJUxrY~b$kQb&E&-kcx#}PscEg1)x8UkzU$<|2H61LRdi%%e`t0y<|cP2X}s=_7F$R5 z{I0I7hY#=YRdBHjl-hnKGIo~U$m^~DB0fCzKmv%VD+vE!n@fNw5dr-ps8ItAAqxLK~)7^eKyYcvF^~77# zFfw#kdU+5~(2$V+=E`v@V5<3v!87T>B-)`piciV4GSZg$>FeveDsmm}a{&j`93*_i z$)m?^4_2O*w;7p`tPNbc3i$+J$Y`q*Qc2`(RjdE~F2L)==9rfzizbI|K(d1FV)=#0 zs^laP*AyUnPSWu<)FDeDm5`b;fG22WHN{6k!^Ks+i4kPvuuutgGBVVQEz*3GUTex6 zK^%Bf1qi*~(I-lsy}O;)$@k1h22z7`RTjEg8dH2y>rCIf7D>;t-QdGi1!A0@o+mD` zFp=-IC-S5pn>!|96c67I80@=ukqJpjSc2?43Nl{dAEWQr&Aq`q@r>Tz8K?XWi8)UQ z667hvot<4tXG7rE6XaxR?%{B=3n>0*!TG|nXK3f?Y1Jyb2m-QMc>1B2g?TFXTVQym zFX$|&FGW0DiD{*@ma%?OM4UB^HN9ePRO)*mgXc_b5!OwI8Eny`I}dMH3hRG~-of!M z<-T+OjZml11;-?64*dppUP%NcU9d076gshm1w6ande$+2ii7&--5J$~ykOme@&W`y z1$*i&Vcw>82EPDx%CC++;S!OcuC5!(`CkYnA4UZZ#H#i(g>Rb)kis#!B~|G0ufG+g zcRw=@;#~T+#z_}r7B5S@zdxQ=pV_L^`Bc&r-!Z&u&K*VH{u&ZF7+L3RMCC6@TevgtW!#&(U za`Ld-{A7)M)LEqY3-sgzovtyOF|XUGa+ZhC#MX~jH!EA_*@kz+yZAH0{ldHH7AZ%5 zoZPu_o%c(1Nxmw%w@-#id}Uj@gv|S(b!E{4)qIXwGi2++WBPT~gz1;_bc=Z?sjRQ# z9*>PdI{BUE!^Hn!P_hF3Q#kdYf)Y2n6EAM4-YV}5_Tu1H7A zsuHoyUQ-#Dw-pwS^|5R5_6>&{fk;QI;=xh}-p^xkm4^{54|FK}N#Cl9Q(@4z?_FD7 z{vH<})a8=z-1k8o=_rxq);HX?l>=gV@Q~U3=Jm*6mc{Q!Z}^ijlatk*oi#1&B%zIRby+*)kApO@>s>AK078*X4Lo3 zktBEB$)lof_x86h5m!zrTjrld)N$XKaEo47O_U3axmWQ*(b7B@t+DYHCYi<)st)vg zd^>!DXQa62soyU5tN2w#6EHiLZrz|+8Z)Y``l4)(!E8Li53-WQ4zMwR6_*eUvdj=j zQl^RCYA9_NZ@`b`Yi4^(nPh2(RlA6vHyQop{6P=9kl73AZQD5 zbPc~rm57+y7&1eq7xl>+b=-SKg+Udq)H>rr**li>;!fJvc3xK&Iz0Sr2fy&sM~(8CgV=8RbZ7ik znw3Pvj;&TBBgYm}wD^vdWucj1IC-@$eSyP|Q$chbe>;CTG})8IF&H=ua<9Nya89-6`};i1B# zAK4gLT`le8na4)wsudbD)=XQt-lnvkh`<5*)`I1B^MCm?WQ-lktZkQJNgv@yE%%$#+Tes5o{HOt2VP zaI?w6v!wRW{fH-P>MboClGA=yriG0ltD&dhhqW~#SeoK3HXcGTHo5>ZrXFyB43N-y zbSfNCOaYK^Of>zb+L;*^I}Sz{N<5s}gxwrce^SK9fjwRy;p-r~PduqP`R+&Sz`j&d zaTKb<7sMo~=RPI9h^5I?@@o;L1gqx)_Tc?qo{7FtAfACF9-KBOK=U3Wu zj`vaC@hqWZB=2_gy%7yn10mz};C|)q^iMLXl;M-(hWhyTfPM#ZC7H3dX z1@c;ek;R+}+22PJJP`~7Vb~q|sLBwww)Q2_?qL)fkTiMBMGzgE^GO#S6tX2d7Zr~D zt3-vl@5BV}y?eZ{?{YPv zxLC~DG+9#qZS=hZkHY)zE!u$>GZ={iUh9{%e^qpHc2Vpsdy`$k6U<5j7y@jVC?tWH zf|kmJ$y7Ds*)a&jI%JGQ41Cff?O6oy= zxWYvH=V#aN-hDBs>q%v4&5@byU%f(=r6q-$y1cS5=l=9Iyqw8z+m7+{jI2tyH|Oc^ z;w?L1c18ACsAkqrveabX6}>^(7y7|>Vb|2YduwVfq9)ar6W{TpQbYesZ(-b2g`~!( zgm*+cl|gJXGmZFQ2W}sBZNR(5VaWnQl@H-}6@TlTjV-7i0PAZ{c&2YqA*gR$yM}_R zZ=bO#SDQ{{F4q4sGd#0%D60X$knjFerWh31r)hC#7cFbgw{OGYDpU6?$}9$_ZT-{{ zZ(7sZx2KsM-&0Ny{RxK7-2p6K_aRmT9dOZ#itpTIAckIExOHpdH}a)_%GHAKPuacX zmas>F{Y(Xxg_Lx)3%dzf8SJrKe;35t~bpcEs*M;KLONOVo|22tBEP7s7w&_P4wn9 z8-tV9QL!buo-cLUHk(ALI#QWz(Xi3^WEI{^)eh#hSKEdN#rfYV_b#8fmXnhqNmCx& z*lj7_(f&s}$F*OSVb8d=1$w4PFwhp~E9CAe=4Gg}Gnniu>mm_hse9mFRx35pk~W)F3}R678&`({CCYGY=dou#Vea$id|U z{0`pTxqglG`qP$06Z{BCVy2!nL=R>zQgK6!=B*2^OTgj(Q*w*ZlNlARFy z&^JYE3hR*H%4Tf+1AZ5Zzi&BW+-@wrMx(*|mS`tji{Hm_EeWFINoz6tozX@+&Mo2( z*(eg}lyjRCw;n`r0vqDzCy&W{gNm?si9Mra06jYRIa5sQxF(VR8(#qqHb@ywjjK*<}Hx6~2jZEfGRJKO}{4-Vb}X+tA>drK5X zxAK>qym)D8%WKp7z~FryeA<(9lZqtJh^Vqg=PS|}wTB&u0B{brCuy1VVxf*GfuQi` z((dx>8Owg8eLIYoEFbb31p_b~FH2xq*}Dg;ly7ehB>_+b`%;tMi121!GYEG_kTyn_B@v3@mzGx^o zp=T4#^{4k8uyOg*l?SyYL_6E>#D1<#8$(3MuYgK@eI;%>?6i;G-YVzz_EAo0r9vZR zAb6inMaZ1^@a^!2k%EqDTU_^)sr|}NBb%!hM6aKej|OT~<=g~)=3BwfmY#wSSe5P* zIViR~y)nUQ#l`6A44PEt`;hrfSB=*PQ1A908m4BRl5y%u0;)by0BBot;4JK@G2Ma5S)Y+uM zpO|90oA`pvTXQ02kN0$X7uD z(vhX?4$KJ(Oiyn#K(1#X8S6n}pMCeh`likHdv*a=%+>cyLW1n&{XtB!YqL9W$x$530cBCTiGj!&B`x?c( z2(Kig-81*{k(1L)&=TQeUTg1Y1D%hZtFeJ>iZ>j;Ow&xodo+Xxm&c{^c2L%j&v(to zw$2`7qt@#I0>NNu`h-(dwCr*=ljU8jEAF6++iHjjbQT-0e(4}=Yb!>Zf?O-}QK~0M z7nbz}JX+cx6$5v!Uts6+j~_qy$?sET-2>ENX8tA77JLjJQ?S>JnV-LqOSt?Lq*J-u}cH}~cKsvx=B z9M`YiSo4UUk;?=g$N9RtLN?-&P7}+o^sdfUk!)s)XA!fiQzFc?LG7RXjd=!wJS zdC6S7xwvKR3VJ$mRqbPJ;KS+#IeqNpNuR+lJoD@foNy+)ULM-q0+tDsnT6_945EHg zf1~X|mQ_dC*{z_gpXqpd&x+!3LXC)=T_)K0iN*gHS6>}f_4<4bihv4;l9D2bNJ~qH zgfvJC5<16N3mc6WFCEOMRp-SNM%U3VZ0lA=wZ zpRr=aW0&@diHqzHn3?@Hb*?O5b3VpFL2kU<;?nkVb!zQX&QsgvZ8}(ALDVD*Z*ftNp57KhwAlMGF;O*|6$6nJ;^0&2`HG}I z(lr*-#F?HjHy?XJB6WDo6Dg@&$>YcMLAe_9_il0CdWumw$BtsGq?nj`p7lH(t_w`m z%g1X1Rurl=PX_-w;5MF5b6s8Zp^co&ze;*e#BOTAYT%?YB{^bf-?Ae0Eu&SbvcM;u zn=AtDp1_5za*75GF05C%xf%Bz|5+xiqX#I+Iu{EBdw&PzMDm8Mmr@@HkEqedMaAd9 zxu4`_U3nj0>Mc_NHVU%9_36Xv7Ncitg?n>f?|OQ2eCknmE%R~Uz6T*vNeQ!T@?XlR zcG?CG!lm8cA~tg(9+U z%j|P-UwmYI&iC)1%srG67Xyd`a<8UlubeQ%!)XQ&jvKv4-N+SRgW1(@!#YlAa5AP2 z$Vwse?~9k`_4>O?0suLtwe#BxVrygTO5}NFv~V1>C(e#)&!6887#F7nH?g~V?;InQ{{H_NK#1BlHLPTO##xgtbIOs`WpXPb(=JqsJYa@;CbWXet`^*yp1MCnZ2#HQE z7kaATKmtopFcO&g+xAk_SMWiHAr`D&|63)?@o_nk1>k}NM|k~x@b`%Jy8h0M-Tkh& z?A)}ap9uq^0)P7pKU+HBUD%_);cFNc5o%fC>x+CbtaBN=P@;3S9}l>jaJ%};$7b*c z?9&b866&q>QnpK$ScA4YE~oA+mr$dJn%yX|3ni4xTJcDX@sl&+ZR?X z-0}Fsn$iqJ2hP7P$=+A49xFVv$VeENnDZ&urzy*e7tQy)d_X%v_bwQyRUVrLJ0dwKlk17+;Mlg*lWXU(IUk{v##3{^EfI?@gJ$gh=^&f`0xL>kQP6l8smQ&#R{NU}fM+C9EFc>RPYP|Mqs92Jep z(l)g}ySayR=jT1IT_XGH*6lD#abw$#Csr~7H;w8MYm1Z={|g`S``MKDlU6TzlSQeb z1E#gfg{(#VNe&M5j{{~?Q2OrLa$~|FQO-}V_!N%58@?!9jNp(2_GkNcq>H-OJ|vRy zyyP)6dmf*!EJIzESL1{Jj_{&&@w0%8W1p|y8`E<7d=*k&GMrGc9yfAsH8j8v0REe| zC4vwa9suH4nk7>f-aa~~2h2oWea5b98S%TPecoh)gI%*Ff8FX`HcQ%e5pz~CY26av zcwv;Sl(E8}(ia%7?RMLTQ03&(i-gxjp6vad57Q5UzSB$@!K?VdKL2=+o}RwFEvCaT zQZ~!(c-ncfU!0ub+!J!bA+_5zAc)+(akVvjg=t<+B*(jlvR$*mC|X5vJFw+s{+{=T ztQ0RIFwD%r4ea!q1sdSoBKb8}owiNuteJxYZdzKJ%WgB;UXGmSos{#bmAFfuM`2wn zI6KzCe!H%33TiTU=>Jragp;c>zBaRBU)=BF8CgEB)fCfwD|655`uz53iDr0PUQzF2 z&BoctU%p#Hn&wo~62qo%K1r)XL@~n771YKGfG1Uqnqe6)Ktj^gnBZzWNx~_wAY-Q$ zbX-?4VKdh)UkAgsr0(d(ElJVlrZG`Zu~i~tgU->5BUcxclaqb17fxzjceypGa=|Tx zz3Y9|X94KdzClY%VJx6&1L&o8|a&@Yi5@4H_~ zP%+P7W7hKf_XH|SLFGcN-!L3ZCcX4?z+24B)R)MLpEb)WAAZm0J^_ns3vU$NCU&K! zAC8RsSRvCSaGZc8gx!0`I;J5*Jb(T?b=ApbH8t?}#6)o8X=0<=9@lomxwU9h$q+;3ROwpV0b+?Xp0qM8MKIvs z)np{7(`Fi2Yh`bS!?4c|WD04uwZz`%mELq{Vgm@pw&?0}0fC44NxSb;aesR(S<7h)#-2};m9j4XMYIiA^5EC+5<{)5kb`6oP4zoNk#tF~?adNNy4 zvHnMztrQU)x-Rb*3!Il(i?$8>*Iew&)?Mt=M|ZTIt%PybjWsq_S|%O9F|;g8T%OSU zl;7?a(xZ5~xr}(@-3beAs+jTUqMJWg{=&K_c6aVd@PgnRu^i@yo;8vL;{jj?0if-2 ztk&vIQN%+(ivk7T!=l$YVg&_p_&5;G|Cq<+=B5XjxqaodW=y3@PZ-Ngqfl2~x;sqF zaq#IM;+*>F(3qzLAv2t_n%V2=3&&5T0%($X2=dp{SvU#We*9P%quz)?-(h>8t#AdcZhA(-sEn@L-R0j=-gv%Dm``Fn#P^Hl7)Sw z=r>+Uc4shRP1xHRU}DiSNtV3H%#Bg0Mh;fIQ5Ai#5=+n~NgxJGVKpslZ?nuGFFBWw zhq&eXvazTuRgK~2;^*jjJ~~V~Vt5E~0XG9>g7AzV^K4!hi%?WeL$R;VF|BN}l`~+# zRc2jzy8*aNkzT-Iq^7A*Zz9J!k*yChkv?1Ua=6T-$ z)UC-S3PHHG2!GIJJtu3oy3Z3U<+R&m7jc+EOk#rDCB?DmlFswN`!wD==zOP92a5|y zFmHIF==!W^&-b2(pcnmvAP5*VU6*6I99FS`y-kOPGcdb0+IZ#I+UEj_Rb29{K{MXN zq1XFu-q*+85yuV3^}#+OXI|u2yMfD}DSS?jk2lNu7|&tLQ(Bch8_raUxdD*m!9k~Q ztEXoL=D3;himY6R!7!HS-}4t2tKv5KgtqkTFj5iT9^cj}eu+o)&Oa4+QLW#kU};Go z3fC=97`b7Qqu?8shuFOg0eVX-+PTUbM&eBYrjG5#cO^1@zPL;Bw1V%I4OaWF;>p8Z z7wKT{b^wv^n3fh7m!Gc7NxxK?>bkgaU4G^0Qgty9t#f_m(UEQvXY`z9;9U~Wv~6`L z9MJ&#efsoau6*>*vM>rZ8VxD8!-L9MuZ~Ot>UWRkMjQD9g@qpWDxGN|Kf;$!D$)N3orh*bwq`9gX+(pKeoU^zWAFZ(3)4&kbCk46qRw^!6#7Og5la*PVOFT zk*Wnd53n)6`n<2+o-S5b$5vjw?=c z!@}t88@pvGNrKdr$wg{$tw&pomXm*crVGVWn}*{*UJ)J~tbz0=3b8r-k29Hg3|U*c@D+&H?DoTwsA ztD5u1WA0{cu~lN)(pyJv_Dv_gfB?-Z8p`Va|8W5>_OUKb+MeR(oU9Fck#H3kEcJ-+ z@sIWh78oycG`a8s%n~8^DCo#(g58|%b5c_4+}vD3qt{Ar(#Zlr9Lh8>hjp?&g=~YB*^TLL+_25~#0+mHD+`j75*iZ1)p5#!6wX)dDNS1w+ zKiaODtbfi)^)SE}0^Q9`ebUn6;{9s>71x)K^_@k<18;H8RMjtG+UTU*jMtv4sgj4Kl1F%hL}0tE$jebSnGQEBb8jhT z#rB|wNA)6Uq)XjGNq2EWDObTls=zoE0dkJ27EEBP>gMo?nV+*H(iN>d*)TC}&p<~7 zWR?c^eK0Zr2GS)S@AU{z7yiXV@piiLDt}>OaVuQp03LmPe4U9Kp;o|sv8=Q0+o5zi zi_%qEbu__WOz*dnqH}QuIBroRf8%CQ=^M6$!gXupN6Ch;haX4b0vJn39zo-z7)iqQ zTc-q*ISJupEDJW}6>T%R&(n;|ST}3TP6TEpb2AD2vU5f8Bbl;ZPH1y;FH8)Ywnl1e zZMJ(cwzUvWSKeS7*>g})S&w;rwR4SC-|&~jcHAg_zO;)`WB8bEMk_r5+1|#rOC$Ge zCee}C{b*C|s4qk$2$bSQCwwxQt#7sYi!1|=TzFU4)|!Vtx_Bad`?-Tt+w-kT1E{BB zbd%r?SAr@V?Z`Jp8TFq0cgBw{>*>p*CTqA!+yIo7T#oEm%3JpW6UpUb+gDnfpy}m` z=F#w8zmm`_D^jv?bjMpD)upUxFvoUxu02=_jgpc9B4ggGlZU4z;0QHMX=0F)4mUQ> zY{FLj^3PU`;f}~qXD8|zs#CH0kXOvcEMb0EvuBRH)Cn@|fTU18(AO|#Y*QuKL_u0y z40(zgXH;U9sAYY+P{oUnuej^dnX($9(bBo~&i)}*L7u(K@%AVTgM_U_OM}rD5{ZL(gflH<(z>?55b2Nk%g6VnWn~wps^4;Xp0S0K@dbk; zM8}_%kbw(&WDEX)Fs|gIqfX~>q>s2+SZ$rHeQNV?2mElw-x3_c(yJIPvB*|{R z^{R+7M^#f0iR%#U2$>9?~J@Q}Z zsh_qO$;sxiFI9Y=jiIT9g~nkYy1%f?e#g_j^}M2mi_?Qo zU%q@ak=CS1v^rdA4C5sWF>-VcjGHXTP)EB*_VY8V$!-hzmHeIyV|)vg39$q5AHu1G zOSK@Jo5f$mHb3!HHSL-O;w|}dKE4T2l*#@6y*JWw$zC@s(;-N{`YiKPPiE1ZN!$Y$ zX>cP&ReklTO7AL$D5dOwBeELrawbDN@*j>xPKT6 zlr5~f0!e~i;`1^+in)K+J1jtJ^s=I3KOk^AIHvQ&g>Y0y$89uX(HJ}IG~rTb)Pq)35r z|ENh@p%%kfNsF@b@rUom2g2D(ogv~GJUNH^nw$^_#WL+G<}tJ_o2t#=>$4F_IfEI`R4 zBZ0P!!-Zxv@}Ce}d;klxNT+k!nH#?wSrB4A(8=L`Wt}R2b~DeFk()dBp29;h51fYY zAyor~Uahr9m>68Dmo{lxZIJVe<)MT${vDSD{8~4vvm`y1Gap#sBdnUh^GYQQd=*HyEI;vjF7hcx~i1m8&9&hpHSH%IYZ0dsisaa9(db`omg#s};QjzEJ z7LG0QA&n;0%;MPcrNkldRImSGW79H!J<=d=D7$I>pW{ysa)&8>lu5lJ zdCJ3eD%1NEfkS6cF8wm@cK%V!A^dyHV|%pu?qUp9Ac59U?nxcwzekwE#QejtSP27! zRO9gs4D`Q7cHqt(QcI!^OL${_@ZWC|TzloeC^_m@Gi@<>HN3^~Oc*OXuVHyT!NRTB zKz#YAUOLlfv8~?a>eMWKQ2+TyfP+sKZS;3$WH)DLYpn^Q;B)y8B5VmlRMEm*U&LsB zQeaDT2)MJaFIQWOLAXW|AI~&1{Cz6=*KsS8m%@D-NIgse@-|j!Lk?PDZKpS8G`O^` z`3eyxlu&dAGv#9G>0XDN*Ml6TJ4}riELSIO0t=r_lAA%#ci}p2iSIn^NZ@r`J3E|z z)8CRT_Z1x-CnaUlOjY5}1t-@4z1)QHtkEK;fZ$SNx>2J@$z{pHNv3+Xv{l}075X?8 zk1x_HI(@^h!)Pz-4s#JvM7>*{a-~N*#pNx@EfI5eEqRW~-;rgj=?|X0$f4J|9+|d^ zot%Od8XWU$$?&C};@;_lV2-9_GK=xM@@uUazs*NSgjMQPV$Q-!wz|X|-^JC=Q zZ@&!~Yn(A}-{S7P8Kg$4PX8oXG<=Yj6SrGG&-ztjKJ8*Yh~8ew{&EE#w>&w`*Zz>D4WaUgc*^_Iq_H! z{IRxXR`+*6YN)3l1y)-38h!m9ZitbT(_0Ou)JXdk>JU+l;emJNsNsbw)rHY7AJg6I zti#sG5xd1vA)7%|Ua?*P3+irA@J1?a{D&wL$AI{Zii18arZvG0SN>Q?l4m;Nqk5}KzKO-aI;B?}M zcwg(!PY&oSVw`bmIW1?ZJKvZkcir`6Em&C_ znmgw)PcI?VM4(i;JFI-9t#V%W_TzU+TbPUAeBX5mHMJ=FS~eIAtR6NZyoSB778)hj zuzYy^&#wr=)w34UkDwgc!lHSF>#{3ZAQq`2Z^eM^?*cd8jWvWxzP{+YMnd}Q#>aoT zCf8!g!F0|J#TA1iD)j^JP;BHB^kCF$7p$7ocu`hs8yo9CDXLCRp4fV)V4X9uTkNh} z8>%!gQS&^Bc|$qvMxtt#VwgJldd5HI<0H|l4y}UMX~yg@-J8}d{A6)*vv{Ioj>jh6n8gOEW_)x4?#`K-w* zr@>y?yM^@?XpPsGp#EtI%mT`YxLn_UOL}w%J(=fUhYtT1QzZk_cK=>kVj$d2SsSz# z8<&?m7o@;GNE%I7nG7?{dZHnEmDAHLE_+5bWLqyT7EtwC1RL*J+7ax(_ARMSp2 zEGq`|TsPKo^J=814FDQGSv2V^T-02LP!e}?`bN`}`0w+tQ)(V?PBDk_xvTfjSgZZ{ zL6PtFLiV4bkl~5HV>(e9s_8ha92^1pSQ}N%2T_t4yRk(c8JmhBhAiWeZDq+bM-`RN zxLF&{r*{^7(a5sLlJ*j3FHmRCMk<#goVO-KDa2#GCOEi)>mD3&3z^H_&{)e`j4GP3 zdSiR<6NlxIIADxculR6-@(aYKEeRB1jp!iO2+yHPp7p70cq?;+Orgvt-cf7BaTW6IYN3}v|xH!E-MtklEQQ7OPj|>aIva35t@>A}k z3~DRO+g{2(d^;VQ{+V^)pvIx+nVBqvLCug}+Jn^64J1gg&FqCc&-syGJOc6FoCEhf zggCk1d)* z6F5OGtlj2@1K-z34dn)M-N`}-m(%8_i*m8F`;hO@!Hh_@q|SX6KRRilI&=?d_ZP9Y zrjy!ajXyR_j?1emhgQ6#js5yj$pYhf`-dgP#e@s8GNL8!&KoQ`%H$*JX*bGq5XTLv z*-4Ps?Q49k`8aIE_#2LYoQOMunb?l6gVzSa1-ho*6*?4%~1oXyvKV8^@2t!Sh zA1d>DT~ra&I@d}K7$JRlsraNN|7g=pYq)9n^XGw7o#83h+-1ZVA0+hCB<}&;s+}^4xuI%phyI z4cOEXd`_s&Q+8{&W=}o0%3WKxiG0SwYAxbd-ZkvFt!`J)3Wp!8hyXaReAx>*-RnJa zaVWQ=TehZGW@gtfT0pFwRhS`m%lsefMm!K^yvFBR5dU{b6|V$UEDe>u3nG+M#gg`w zTk1E}C|hF!K*sbbL7n&cn*#2!t>t(xdKo!a?WXw_)?ge1_Sd0-fw3Ly2oNElkO-3V zJCDuFb4$&3#9X>GxPSNSF}Pj;0;uW~X}(ucEqH3g)Ahp5p>V0h+=&yhUZwBcLarvc z7iYBekaghJd$vj_+kse-Ct#c~k_tVZr>f-bjZ`+^fvx}H+;Bh&C>$*MosZYNxOe8) zvQn(u7u9Skr|j=~op$Lg1Q1*MsA!yOZrE2NLi@$a#!CjMPf-~pE@3W{`%0NMz{ziaROYe`^$=h8Qv9t4X@YpI%CxBd=z?H2`T|Hdx~d)0)a+Z)#Z^??gYj1 z*z+R&2umcwhh~%Cn_pnuoU(1f2I@`rX^(w3_gZN;2rK~UXUdA1i+#s&^a53hh$KZ9>8HF zZO!K8-|Fj05%`HN%d^R*S+Enj9v*cI$8}p3leUvqmBT%I2zI+aQy(Pz3ol!coC4a%|o(;kepncmkh0|)V6jSftt0qOH4a{i|kP-qz-Fi#haC9 z&m*>(3P&U3x)ZLEKME1|$> zX3LSsNxhV0ee&%JH!&z>+2drfqCVJ5UfDk_pV8(4S_NA5s}>8SJ^z7mwFEMPLDM&z z(~}vWZqyey&3j+1pjcNM#ZU38_X5Rkcj)yBUc}lV5#&*w{fov3K}h0$>@>ftx9q>U z9B)!$5sy|-5Hg9+ofHtgHKy%mhFa_78h5F6OQ;Ev*sN@1s9!|>nu{z~?p2uhin~BF zS9qo3>~A||#9Wj(SR0;opdN4Rt-q9B2WWkG8vmzU`qKEk~KR>y*35`eMf zm#bXxLZhX6kZ=b#@e7UGmBUu~u+3r(naXRhxrju;w8!C_JPwKWRR2-B3PA>XTR`{A z`O`DO?a3bV8XZ+VP1k46?jO*9*B4K7TGlM|G*_#^d02V+)0a@%ilBtVkol`~9mr)l z=>hEwpj-&2`1=g(S>6G@`Pzg35vDqk<`Gq$LSc0#0c3L@;YxNy?A7@^P2mj$r1+(F zS35I?(~dO&`izVvkcox`!XE%4(BHE*s`>8SE3QCn{Sif3_9;4L4~UYCEmA!@+eAbs z`;7Mv^-bHux)?X)>6;~}%OELl0Nb^-U4o9AhrxaeA1ljtKb>4WygLqs1cFS(7{*wH z-KhA-ZqPlrf6X`a8o*hKz&3*))o&@lO$JESp~gp8D7!XBgB93uahvuxKJ@iV`U>f! z)Nj5r0&Dva)n&)sc@8o?LF41CIEKLC26BL}z?Iz%TD-eq0R>fucwMC~qqV3O9DHgS z`nD~YA}9S3;dHX={5m(_VHUv2T55yAC$?#LD39U}D02j_{S6?T%VyoD#q)FEj*zEO zX2~}@-o}4J&kaYrKJCtNJU`m<&{?%L>bRwlm$eM_-p`lQ(hgr5m9(!V=dY|$X`ti1Ds@?47!`}Ef`hZ&^ zh@;3Y%zy)d6rh9;Z=BbhU84fivUn@m>e(#>dVR?g)*HjqRxQbV3{N5it}oJg4C7oSg>;cHqAOEBYBD ziC7vD=L}aUSTAiBBuh7L_2KAPH{D?>P=ONa`T zBH-u$Va9$E-}M2QU5c8{TI7Gv%ANa;{*LYTM`*vRtS@M^zKJD>aeL-`1_J#e@fq{C z&xEjTx?}HhfkSQ-U%u!^UYDKOqLc@z_qI}AcWsndqp{`#wx7%gma~;Gj<&acW$`=S!XhE0sSOc- z0V7GEGZg@%p%3_xJBJneioZS~>iN;(0#1s{UB>Ekt5(v%H*w7XOJ5XSm!s{twCQTS z=K^R^aBCfs8=QgP*-MA&74B(~AT^;2@6j(5JWJ|bY$Y&xgnmQz3;I|2x#zr_xgFBC z6urS=Vr5kT%C4;@83bmqydd}n*$Wp**xsM#@pQV@j0ov~5-nOX0m4nr=zo0`r;wYgP=zbg7Pnd-m_8EPGi+3Z3|n+fqf;= z`62{1WGDy8{sozzU$3%F%6PY84lf9h@bQ1TQOC@uDn5r)UGeyLQ?rt$Ee?*Kp(sK~ zb=oCPi&Y38W5^sI)ejZU6~O;j9WbBk`K%A2;)G;8=fL+BYtcWpE~zz{ZAh*`e@{dW zmEUllUVt@QB8X^7EpkhI3`76)^=o~amAy~D$IX0UK2BM3-9Yz;khGPY$|l`u+L8OQ zBc~6qT&5P%ECj6P5Rg^$fbam+b8t@x?!mDCDAY<;C&%|eRfle7TU!g!cpy%TGr|rt zj_mlDQZI;3aY!4}(IvI5uF+bt(%W0mA0UNJfwH=a-8T;8X)r_KUH#kM?;`^sZF@h& z=&NGM{)76$!n1Mx%2Z*jz!s#KS2scO7ASHrnSfbq}rqteC8p?ZG zQ9g|EkCq~1=?iwL(NBFdU_paXt>7r(DQHZY=)MiYmMC+@h&B}X+T*BL=f{uWYP7hL z6jANMX=Z^aORdNQK-sXTMvk#iRqn z@ZOTtsrE{)OBr8eqq>^#j(@KfJtCazx4->Wx+gU z-^KmibF&D0VF}LzHymI@ApqNAq4QE4;r5Yot3(&)!7UDucRh-E_Pd8uxIKAQdeKUj(f*Tso2DK36A0yUC2P8$!a_?$!nNRXJ>mforN{4gJG zwkC)#0Ql{iYU3=z2j^3%(&j`93lic&OCW-;jPLT|02I12x6XeWNJkrs-THz_K0ZQO8FUOGb38tV53_ z&~unTU}x zp9a^3FpTJXWr&91&e%L^7YlIW_lauWSzEBSEVKCN{d@t6z(<7A6ogCWo~G}yZ$Mx- z0I-*?8c1lcz&`PkrSYbRgLY>G1W(IGbe^70@DNlnpmGwoK>VUKe{C(hF0r%8XYA$g zX{&;bC&QSeP<|CuT(FxFi3pEyNB~*UNulSw2=CG}m!!9?1zJ4e5MO4nMXq=c)? zfKCmdFG!>O{rf@79f*i`@3N7kU@N8wr{cbA#zku9T!a-z;xv>OaPI}!KT(2mAESq$ zU_VJe5p)NEZUt2w0tQ%hVCXj(Koe zM_4TB9x8?0Hup8=k1fbgaBR6S#_A_;BAMD$wrl<34c029;AJX8b>L&-Y;K{e=8{ zk)MKys-S?{)fk)NedMndz#$9*xx91oq)6tT-Xbj_WYD5`KLk~CTUSlrU~#20=sR!P z!ipL>S@oE3t?#kmsTp3~nP+@x806a+KQH0Uq5<42{BYP&DmfXoR@rjqrjot0@Nxv@RC#B9&1{QT?hM z{EZRwAWW!DCv+=|;w^wBn(WXp~G=tqEg& zWm_dAbNT_hC6QsShtqF}S^`~-v*ZoYN?#!0QOrco=0KnIZ3qbM5MVK5_#HWw@rF7W zQ-`0Px)j`tz#fR46^2d7Y-a7qG=5kCdd$pok)tBA(l6(aU%Sbo zS2KeXX<%^Ql^}LrFP_)W?*^7IIz@Cc88Hlv_Z@C}KqWxz1n4pN(DF3s<@_&&CH(HA z*kS$taRUvMa)SmUz=rVQ@)J3`F8FXCB{ion;r~f{|AEDP`ktbafIJb_C^N?EMTFGn z$-76oP8HbaF4a=cfew-*(>+ z2@;X`UAk`E!J6TLvROX>?^G|y{YM*z1bnN-#rrUG*-G?h=UcFO!A8_NyH^ZF_S+}$ zFGYe>5sqTiN7}l^CLlmX{#Tk?lZ`E$3p`DbH3`wi-s}pt$WM~JiBz&>LzlI12-OP^ z+I|CzAkc?9(t&I7SPHG&md~skM`LLPS*&LdBsHQ#QYxXLdSv1uFhvhrtMg;pk1W zcHzxbC)aQJpAORvybI7_qJ?(}VgD3LpD*TO?OkL0TARoncAFya^^}<&x-g~!JYKjHAl-~}NRX0j^^G#{kY9t6=N>LsLREz`Y^e9%gO2?|LFrC~3* z)e}LOGk~4r!Uwes5Tz5w5+K2uXv5gvl^yRywBbMt(ZT8jEaA=p;uCth-m5*ApM}LE zDmXy*k9{SoybU2)@hoF9Bx_9{YmKHPysclP(wCE4A>Z%NIvSfZks7e)p8>3ed`~87LW|X z19*UL`2a>r&;RNS5C+E=f&LaDfhD#DaylugO16^2&G1NLm5XH1Kc;4x?D1Iu*Zn4x zuiIwJj7+=JN)U^%vTKC!ya-prr}~A_@9}Z7*qdR-Ri;C9|EchQ{VV)es>p+*Hnn~l zBd|ptv2BKgncJIJpuu#OB1Ve@&TWIen6ZPXj{ze*r%$-pD7ek*i}stG3WuLA~T z{gJ?>hTWa^g#JAn_}CGjKhs3NvNH4GcJ92MUnK zSAJq_*qf1_CYA!HSFE)RzWZ^C2n7jv88EuAS4*J@hO0~cZ4mv4rd8b<)0H{5@hU26axZs*3u; zlgTs~?#1Ay{#Vfe3pN{^9*_*3%3j#`EY|X4@p*fYcv6 zJfxB?7u({~xXh1;dKZ#ZrBFQKy~Ill&d%xk72^(YQp=ACNhHx6b2DH)0ZagP^>+qK z(*KNsDL*e$MX8*OXp}5d04X?owqLkD$!F-pW}e^sM6>xQT~_%H=fp&}y;kmM9`HHh z(H3y-yci+vySsb(GbH%qSJgW_@bT}p)o^Bta`}rI6v@d;3{==&6OsgBZsPx|-I1O# z3=YZC!2)smKczb;0!atD!-2d${hl&i76bt&9{Ne02Md#q;2wlyg^*Z-e+j8LaL;63 zI2(Xju#B|7?N(h3&L#~15B*f0pYJVE!@D3^&p25vH2-TvZrHXOa|n2ArO@on?ery5%x>?zdKM3$kM}Dz1XA~9nXegf$AO8g=j5^;xJ+$ zhX5y%Uq@IWYhud{JpQF#?>n5i;uLMvQ9r`h)>@6|L0krFQ1H>SqLOcMLc$|*w6Wqe zAHfrh)-izX2jQA^r9y+Cd{xQe{|fr(!^vyS2$J)$QqFlcB1Fj?YtG?}_xY95)DfUg z*NxOTs{i_3IwP-3vP&`dCEN|R3+Ux@a%e!D=F?i+4P&BBej7<8Pu5!I8*WnMx`dev zTSAnEt#50sB37V~SGVE7SEw2?FLv|4x)YCLKi1x^RMavrU(eb26v>{el(p_a3-Ku4 z&iN|!)PzF2f(XQ@FcPjCYPuyx%=lfM&uS!}WUk6)3C%qIfc*`s-+(mWei-A7Z|>0g ziW4ym)i;I;BJ}@?->BX0L#V8w5QnawZwz_-pPMh!9=e{SI%IuD)bUZ>=F_g`#zO*M z(zY4{*IE+^3Ka=*GgP4ULI$rVc%wLd>)j7Mqu|299H{g$??7bAfN$moEDlr>vPB3B z39=>p`XT%P7$fBI+w`uyox&mNmkBrj_qHL+>jQ#5KsAY5K}T93<_r{haS$A7{CW-b z3m|9Hh2WLaXk@wZQYTcFghSDy*NVYp7@4@!qZP_>bnaMC6vs5@*$(c_0CBzBow_fCKRRk&+vaW}i1 zUE-*t%?KfydQTUTZU4hIR#JG8kn2+55KlfyK6Tv zaht))>hG@L;v4l&3v26FZ{Mn^w*C?e!dw`d7L85|H!O3VZb<4MQkqca_|%tJ^Y&?; zG;0J$J;$tNlYH4|?=;jg6Ixz6l$zmv0I2A;&c}nJLYYx5T z%3T}m4s(LgCdYgF7MDAnqc|Gy6L4OIa~pBeB_&8&*x3~zf2=R=?lT#E!n;WjTu|Uu zQLZcZM!Oe261{u33-4NctAJ*?jfRu+d}#b(W)KYt)eE1ns3pPjae@WfC%pIssw$P= ztK3}(1H_xZs?HbdL`eunNqohBjYdn0AJS=jHz84D;Nqgxf;iYvN1}q3qx0HHLp{Oa z;sm+8lJhNcp_qg{n~$`yXr+yb!^X(BydNzMEXk6~-;0Z{1WaDX%x}h0C5BH&_9e=e z|N8NxG*4YbU4!|DUZc9k^j)#HWpd&XVh*eFAM920MWSzx5x&COq~@7_&QqH{KK^91 zZL=UdXQ(tTZhwQ(`hAPnN?18rZHKHmDESa?xaP1N(#pUf6a4Z(L)DG2g&8APgolbSB%M_m{GOERr zWju836=<8LXIhDsTm}RAu_Hepnrvgf=`5RzKI9zjzpv=-9(n8Lr0{B2~B>lb`{dv(ojnoA~C3 zmRK$6=Pvf&Ba5oUny5H$8;)v)#%w45jigHHD|74r87!5Un3(-CVBhui=TPO;s2==} z9aY5s6Ou7R3=CI}m#q0Qz6m)a*&gpF8wCtPdZSbb%R+!oK^`CFubZOo&8V(^1v|*s zC*qcI1RZ{ey4v&D9`%ry9EqybWu?{;`8_MFLjwz{Qc^J&UPX_inrd2q$eMqz|GfTZ zrsI!r1=i1a1^J)U)k|~x-d6vpq188IxWe$ixa04UM_@R5t$}t!8Lm$f=G9!KSQC>J~L!E^V`8*co7};^OjKn+pfVXg*Vz z9nM-Yq~6Cmr=zCZ#Bj05b5!9v$1BnJZ+xe&Qrn z?_l>T>n$D*xzmf(!*s*guw{Gy;0yx}_o&;bqxhc(&UtTJB_;hVo>ftiW`M~*k7b$R zNa>S*G^b3nq*GsxA0~OOu{9`{U$#}S^~|tIf$l$RN6FIjEoQSg_k{{AHGg{f7+Yew zHnWDEd8$5%MVY~1CUwy}p%k1%^^$1ElES~=F)a(<&_id)I4YJqvfbvhv(m1i=8cUC zq2{U&ChO-vC>#?+lT?|+J|GVXi9K$9WuH8jzlD7|Y<`38wMNeO;*WYRhJ@-`x+A%Q z&=%yNVRR#Ebn?^fD6sAx4Rce4`JeI{D;LZkQ|^RLhv~WCVw~Zky9`fFEoqjcqt+8M zj>yOaJdW~#o&A7lEaty~zWl)!QWzW|ZEVyMVjs=Jp{1=|EBLYJ52uXJm2 zHHo6Zm0lUqrl}~U`FMQ+&nY9Ly^<@kR`Io`COn=A_vk$A6PK_7#lJ`OQLTHJ%)#lq zn2TcCd-;>Gu|0Gj>7HXm;Mlm;tJweCC|{6C>9bu>H9I&NBI4{M)4($rk=)B+w=N8k z>)J`oJaj!OxI8beM8Uh$DwF3&cX{DtgA3U)jR$#doHB+I+l1`c#TBwRY5J6~`YiGn zzo@9lS(N0yz{ihMEo)39Oz(7aA zUyePu0xF5OZ)4OoZbElBq7MPWn*ySj|9KnoYWy~|i9HsuBYtKc1-;V|# zceQ(7Sz~I+r=eZ5I+j$q^?Z=NuuD=(y+=APegE6)6jP-F!HOiXFTMn0*S`DF+g{W+ z<1Wc^WRD-b#h>F|?}TQz(d|;3ST!6OBzhxkO*@9fd3!2~s=Tst=kG@!K0h#$o=%PE zJZpM3<=UEc?nTv-o08~*A7L}gN~_bGm%BMLpcWW=&~^j~jnc-Cwq<1u_hsPGI9b+A zGjJUlsB2_;_KujPQ*`*&=wt#CNY0q@`~fwp$x^K7Crmwd3r_ z_vUa%HG!g1>Rht&TmF{XU#qIG$hNg^hq*JZE2Pllj_>MZ5t_G@CqF>?D*MUUZaT;@ zqlZlup5{rL6VBtwl49m8C3CNqh{_5wc`C&uDic>c@keuXL8_Q0=ijHR@WvQOu)=C| z3CaA0Rg8X2e`Ms&R2$m2pLtkTx|;cRmQnOO>w4^r<)I0PSXU98p80$vq_AGu2NDzgg95+W zLWsz9#&;L___Ws7a=eD07stZ^g0y*eaBz?aTAc8wIRAoHEYPV}r$r4J8ihQKf{)qQ zpg9o-M<%qIH0fr>fW}^;Y5hdK7)VG+JYiJOZ&_~{Uh={1(GEK3j%d>RMS%u4cfK1w zmdB)mexaDppc5W6lt}F1Hh_jC&|Yu<^vxA~Omts?4C(8d>-1|%xC>`eD3(C`yn*;`1ji7;AJ7nJF|Qag-%8A zH!CzaMzDWudFhdmn)t=2(2=s~{vB1P!`apKbw%hUj%jiWZd>nzcr5GC&`?u*`+;ic ztqX7Ut>5&&Pr>Iv((TI|@}?xjpe2hK^u7rXgoSujK8F8)9RGP(pS%4O@QWJPgG@&4 zx>cf$wKYU@BP(hYxEUW}f^_?!FBS;aYwQ*SSpPtW^N?0((WB4;>~ADpSWL)BW3Nlc z^WjYf*1od>NC3+e4+GWW-(hw{%%Qs&ZR`NF_}Oq7gBD3|=&4(n|DA=QoBuqTp1wZR z%}(p}gw|h^lX=juny42VtZPMp=@2WE3c>+u{DM-WOE`e z&kpr9KEf<^7=A&_;$Rlo;^WbOzaak47oPL+A^J_JUWY-g%EM#$5Nb>RA6;(&Rn@lr zj~_+l3IbO}lvY7H43G{3X%5|`ARW>j>P1R8(j_I`(jAJl(w$05OE>)I27T}Q{=fGe z>PKuIqSth>8KaB!8J6;_7j2FaW9-)Sjt;*0R6>51HanQrp*~pN!rvq`J({%PBDSs>)np47^boTee@H5FpQGAr3 zf^NPF2nY0P(Puht8V}v~PL^&Uk4&BinpGPb7&O}Ony``d)YPC3IN)7r_Q#RXGL#)k zSinxrK%-iu-{shscYUqD?c0J!c<|!_*W*-_yU%~h zIrM!x21n>WgkP`PQ%g^2{wbt;q89Cq)XjFEL6DSjZAa+f_G0vzTa6s$LbJ*Ck z)xaO_d>HsiFwlB$(^R9vUS`L6l$ghHJrx>Z8F4Zq^qxl+1+A^JS5_>JcX58c3oe?U z@Q-#B{93~w+BO4SwxNH~{9EW31j6wzlx2~CioVI~;?E_}S~ zF(EE$uYG;J0P;394mkcVa-+69VP*?d&_dP3L;{fwWL3}>7?jY-s)hxQ$A%^=2zScT z>oXDJ&>uH{>^YXZn%3*E+k1i_8qmCxCCsWGy34}CejYFU%>!Lr6rS0M)c+@P3)%pW zxu1I;narY5mI#%&^xqzm0xb-5VgKSz!7~1c1N7^K{;r?2Kfbw#fLuB`HK zt9?5ay0*RGR6oW!R5>zwcd0dZ7WC0Fxj+9_=;lSf)kdllTDo}ryo#FZ|L?Rcd;oGW z2W?>Ck`mHg?VjGXH^2fXthYx56HC2UsxAc(V`XL4s!WG8I#v3k1KkG^sj;Cx7K*Z6 z^T!D?&@`|87}obFF5W?PdwYAI_w)BJtA{J<-6JC!3B*z}6-Ho{I|$-8D7t7ixbCQ>8b;9}lJuIH292 zxjE}G>bOfHTgL}(H)Di^k*#enG48q7&@vpZv7SKM$1M;RcfOlJO>_YwbU{s=I?!7U z!@K%dLoL89N=>l)0?Mi8zf*AoQODi?NyTRvcF3Q~ZWg*vX)T8Pc`M9K0AHl5<=qELchzXRR`iM`-uYQc-s6BAl zP$lSNjo8`qA4W#M5KR0#KlCp{j7UHmCK5j*ZEZn*N^~xKw#Mw{X>4~V>$Pb~z`73h z-mb2M7`SmI@fZBqKZ8*9K6D_4PPHQ?7PbEmdLJOOHhFYE3huv1b4T0OfBx|idP}|f zqEj^2Co6q;`-!6h*E_gCPf?y@jp?aRrA zb8vk+G9tpFe&4kk#)P(TC$3=gWPz>NUqOQPo4`X}#{vNl4j0|zO z_WpbLZ-_M91;YtCMOZJ9zKIW_Mb9+`Q2qY79Cl)|EUcH z*3z4F7_ds%49)Ng4f!=H!C572V*1oGTRAigFz0)5-Xs4oG<{}X9kyp&F% z<~siugQ^1d$7O?UAs)tJXk^rUEENqC(AhWR*rq{slK{1CsG`0x1c;P?a`;cAL2aG} z7MlAEmtGK?3EV9Gg`XYBR;D739hf$^KmryVpa3HYC8Zs*m*7K@^VnwE1|fL-aRxZ} zqdwxEo%o>n>mesASfgimPykAVR_TUYGgAM+?m{tM$-lvC?}hx2~KwCTSG z`l-J^oQ9S*`cCa_Hcrmee=(^bAHt;Q#wkmH=*XxjtNMrW??5^q#!H__e4HlCPTC6k z)dI+us<#qDL*ajMsXOyT50-mmWO#Tx7=Xr^IXP%V?vEkd-4P&CQ1bsr*1BiM?hxt) z8kv~QF?&9owH_pfX1QR5%HO`0`J3kVfT*_6vGOg8ddYWgilY2{xm@MUV;dNC>=E>{ z|4%z3P6nB@%`txdH;)3vb_qsLhYmZ8pdk^L_dhgbzf8qvFz)}WXNZcEL3RUC`Tr&< zI3t99PkzIhADl(GlZSwWh)}a}aLl~*cz6S`7azu-;DTTR_&eGSCkPI%61ZwmSM&r* zQ@J&5Q2iJCa{er;Ne-QKl5xu=<|2EmaLIZ3fUuYMf@S9P_VA)VB<==E7ogjl}~u#zKJuA{Ab3?*JH!-R16J8 zo2N3*PO{(^7Uzu66J{36`uZ9#JMs2@zH~v={M)zQ1Ma*3)gQ$(@?9}(obQS6?Zu;{ znOsxBP_e5!hL`S;L^_jC6!fBB)p^qk2n~h})XDn|kvv{7 ziUl=S`!>YyGq*cC2bKBFN7{6ISxIbnVk)&dX7T}9%D3J$OpG2GEQb`B<9ZZU22Eki zSs8x(QZ;J*xH8I7T*!Xj9u=8bbl=!HY^1_L?kcOMwcp12 zI<)`R@0$T|YnMg_g-BR^kd&IQXLLLZVWxxwjSgSDcyEv$oi5)?&xi9{)QUkGt|bM! zBazdC?^RV3MRb{+s(gLD!Imc`?(ie!cz@?#x;5R+4HWYmA!nQgl0Y&8vYiUHd-@z9}y>6avR856rWy)-t}P?Ly^O7h3gc>59OzsX0)@_Cr; zFPN={Fr3r?BTvs{EtKd!kC~Ynfr-ByZ9#wD*;jp{rCJ?s&)eJ9K3bK4tNq6k`W

n|)Qx!1Z8=X__yapYS}WMtwaPg`B{ z>HkMBZ5{Lj-|Sslp_i7U=8~?Kr8l(gbrgs#9&MY-+YaC`9T|W?s0~oT%xbeqpG}OqaKij@DiP2 zuq!YaXyrL^7FtQP8ag%E@Oi_cgr&Hz!u>uk7iy54_<}8qbFUS9hxu76^R2{{&O%s$ zotJlEuDYYACkc8R8+{`~q2h8c#;R^D-{C)N3wpL2Gxglis9BZQlFRrMtWap;`-6G! z!j9+pozXnKxjOu@h40Rs?tNyt{VRFzy70d(Xj*hkcL0^JtlHvd?4n4i(oENaH$;m$nUJ&{WkI6a+*MqDaz|IQqH z_{Weu%sAe~KJ*s-(&%Q>%SzVhP_wb3Q4qUGE{Hq7aTZ`;CSV%Do?Q{?NX?(QcM?=F z4wDc=`fxK51z}bn0J}OpaxNPsnvjbd6tUDcS5q&zI~iE;QJcK_mddv%RW)OdoXVRa zd7}&Wn$^^zJtp-Sk|2r}v1#!<5W-m9U0rhJJ0MKmJmX#8OfSU-{;V;(4wErc3+%kAW}e$W~^1<;4!|c@^(<9<-;N zVF$E#l4H?lz7BiWvOJtyuOi5xrW&ZHm$^ki_U`7zDcrM_wi9ohGY&XjG0_2CxBR!Q zofy8PtlB5)G-P%uxr1r`vXVA7EL*FX=v^nDKhyQ7d^aTu%tpD9nd1G}u$*!={!6gQ z$eZh<@?Pkv4KZPiWd&1u@+kt7CmM`ISyuIuT@2U-SKn>g61^)w?~%#mko)s}8^!w0 zWK>jsJ-qwT1=ub7p>z7fPey}c&OXN#ImR{j8&TDKfbyJT)t;_g)5TfY*l8JxSvm^h8%0IMB>t&5y1m^kpX4~5C@7NDMAW^mp+07AeL+H15=&&+Fv9PIg`gZyB@AfW7n?Uf- zopfb(=?1@02pmkC65{j@^mh)L8Z(ZhXVgvPx>g77ZAldYN7OuYE-Uv!%iQYL81W=E zuZRyZu~n*45_YRr{m`MYqgnpBHIS8AW$#yoCOu*LcA8#%oHxf<=AgWPG(U6f*fL2} znY~PluW#!CQgQZI{G%(?u9gNO9wAp|Tnlp0->~uJ_4Br<2=kgzN{X3dPFDFg)s-#{ zRdwC%??M+VGXn8rIOAXF&-nB4Lkzmdgo=NpG1|D#+F19oUABhY?xra-D^u#bmL>xu zS-IP((oW@rWo6bMQ-q@l1elgx1+t!B9hX}WpypbtRnyvj6}r&cqOjRdD=@$*>i_U* zmUmF)%$-XE+g(;%m4|h#il*G2j!u2IxLO5{DyDO^?3Q8=Iw(hOH?2&ERcES;KAha+ zU-4$9nJIXO0c+7Zo5F~7yjp?@&CC^T8~lDzz(#bN0To7+aZ)zLzOfE0uv zPzx>qP8wQec^_7kG&}l>tF!9-G)X5do0(aREcBifKM_u{nwi8s)4kD1<+x2nDPvH5 z|Ne-_HsKxt@OV|7(7acwi7m7EXs7JF%3``YO~RVnhEmz;T*hu=&VKU}H5c{#R>eo{ zgVB5y`fCM;d_^T*@L9G*r?+Txnjpt{*29Oykf?HE^hynOTv@juMQcvejlrG2WvJ#k&zBvF;5z>}`>;#jZ!6;zF49 zF9Nsi4cwpT`e4YgvXNuFt~ORS|H#ROA31s^U5~ttd)BXV`$Mv)=f+a(pcvsjjz~vEa!tX6!i}c%VH#AWbEC0DQG$ao~tx2kl8mmZTTSQ zD5)tv-iC6`p4SIF%4r5nsp9%GbPG3FxMX>P+?dDkP`T{S_XN7(ff_?oOn>U}0Eh)14n9ez@ zHmRcH%T!e}g>)@C9NjD|gv*q-pzILOZ3F`}3jDUM_d}Gdfh9C_C~Dik=eMOFVNv$4 z7PD1oZa^hq?n}P2<*`QS+5mr3QSuna%uCMR}ugy#e&#sM8@_FL8ajAJGJvYPOj7XMIqnmuJ}LM zSg+ny@(GT95?1CQSIb)Q0*z5A_Cn`A0v){7pJYFfX+!$gLe!fCC#cHCOx-y+o?9Sa zz5~$M{rUcQm+#p28)IW{YCP8k9d66HZWqZLf8)NLm-+H+hedk75v z+4jmpN!_gBH?#y@@yEE&%|53Cyk24Br!o`f9ex+PdT}@`sgx@zIXPHES^BAi%AvH# zOOeOe7|O7|<*|KVtW(B68uj{`v7i{nudXmT`jIMY(SaQY_Ts{{iAmA3s%m!Iw0VTS zLP-EjMwYUr)~cG-TA#46t-o(uurIfBt+u2EFR52$E)2iNVPrHiGnNzAPr>K?RpZYK z;FgND>HD+%OG?XZK{k|`HqOk-5T}I}I>N~pxtx^=+}0DgkM`GEm7hKrT87RBB+i=y zPL5w839j3ZRBWUoO`V3yTrTKUfEPX)Px5q85qU(hb?k~kscC(RAcQvNYK+}}VVAkA ze6XsXA6f35jooi5ch$&M?#4Y@5iqJ7)uNmIse_pfh}}&qck6m41fk&D1u5CX0s#bR) z$7uWh#{3n5hF?3@Kfj#cfWi&j&a1jp*hXfd)6>0orv@@x4`z-slX|R1qf}Mbe~fM8 z0o-Wp&ndC5?yoTf2VZc!@JP5drYDW^K&CMJe!t^vv6~t$9NEmg-(8{XTS>Q&S+XhQ>%auO~QuPfM9B^%RSnhpJH> z=FX4q)z)4SpjcYyenh$Z4tZkr+M8_G86Gj$S@G6}y;v7TTK?S*Q#UQ(dF+7@;?2c# zE3-YzF6g>EDs42#{1&0F$>f(VG=i3$c`k3W5}_L4hkk;6_qRBH(pvkk1*9DvRRgc= z@vRwG(}*_DSd#2abFd$-wq!rqO&TmKw;A87)*|2qvL0OEA;N@a{2iOFoVZI0nTiSc z1$^u8vMcxVtlUZ`M6*Hu=d-ESN=zg}XmZ_^5-h4MMFEvbzIEr)DcuYqw zC{#~Bx}oTZkJWjKv249CCu@&FsH@u#cKipF%4o`cFUBQ$Nt7Wt#CkAvW+uAro^x+k z`)~-OU~^o0f3K*bBsIcrRHmTXZDb~_K0}I^ohwc0a1tFn@cl!AsoArKIZz^W ze$}ozBZYtEJiF<{ZFh1%;T8oibV!TLDSy*=T48cdD7u%uC!U|Ls$7Xv6`^)T^D77p zgGOhi0~N=pa~GedU8V8J456f4&i>N!+#;*1#o8H9T|ODn-qSY?H|z0S*79R}7VSn$ z=hd|xW|G3zSC6(Sheo5gKxI|!474%0%=hjRgf2TVfwi$WE`;ZHFi*-~^kZ%I%SI2(g#5{aSpSx`HIwRMtat|D+s=9r z7%9f{W(Y4E9NRISrqfDnl9f9tZgYdrbnGiv?JK1Un*#Fni`7MQ+$ETcL!%zZGTt_zEy_+e#_W?vZKIyydUK#+s4S)4mopnsMTQP z&`{6TYk}0(u9TkP(d^;z4|)yt);k_ot@k=_iFS5=ef#8aE@x?Z1`Ii;AeVsFs*&#{ zy--;~nnpQEE4 z+fw+G8gq5BSCXYg0>Jtqh~~@iZR1N3pK0aVn_F#ZahyUQ$x}jEdeTj@7h1PdV`yW8 zbxm#Pw8+Zf)NZ>y9PD=M!x0&I)G!`)h)-`Y52)$Y}ObzP`;yVc$A| z)^6a}p1HSQ7cxh;JHyr|(MQ%zlZvsY7jXOo#r#7g>Svoe2S>8Hjuv8PrI~0jPwA{+ z{jPgMbJwo*@3KC_0zc)ekz#dau@VMqBj$*ICZ`~iU-slr%mth`cU$qz{?amaRkNS1 z>CV2ATM|zgYk9qY)x3t|eo$<1&ZLiDPN_=#BhT)G$sm%wq3WLYt5w&p_+Hsd1*_`Z zyS8$3mk=!yX8F`t*}Z4E;q#W0UV04;eN}D)9+6^SAAU;vZXn>QjyYO$t756*bzV($ zjCB;#89OW*J#4AAZjfWGt}qNtNC*ABx*e>en=1(#{0p?pvDpwR0UZu8paC}}Gd=tM zLXR&KE1YQEVitAjT;<`9P3_-`M=eJLEH^VV1~3tA^W#d@bGz@SUt@!%b+mr;W#W%;qJY?&D3mkX*TqO?! zef&Toa@|(ZODnYLnT6$Q=%ePG#`30^mdkCt38fiIHXC(UqjS5z`~w<-F{Ug~OSQ96 z>!{F}iNtrd$!*VE|W!VMc}C2vt6|gv}7hH z&!!dzd|({Nj4Wy|sNVP~!*_sN{c$#p2COJmtG=B1xvi?1w@GHh?VZEJZv>PSCJ@$7 zi~i?ykyi|v&4a^OywN<>wewJEEy|H7IbD!oQ~OJuEB zFOh#M4&Fyw)t~J*gmiM5U6x)hE!n6=SzdWXAR1{ub*1k>2P2R@ZTkLamm6lL6AO|g zvU&MO61@$Q&h0(9QmnGaKuy1@W>7`UZ7&TfirEE&CG6j*rg$-ivGlfl`Wy(2*b2Q1 z*GI^EoD*>ns#`zjCj|Do658))n%umj(Qws^`0>6`F&Yiwa0-5x!HrjW29c2}rp*RU z-&rJm-e3K;nbnp;l^dQhkW4*7K{~W=u^GSGrSG4)`K8|r3(r9|ddh^QN zNpcq#9>>GIClIR0-9T^JPnw(B;3=q5zu4Vho7XJ1RD4Z&%x0^2tmw6q-$x9yKzo2<;3S5M35 zapj#@HnSS29C(eKzMql|ffBSSffEM2*;Hy3t6HLyzkWC!4r<6TWzV~gIEG!eVOY7( zo;LOUfztW$h3yjjjm>P=hbqYxljbuuPn>SqDceU0gylG^iFr@hLKlA(tu7y^6Vm%-BJ-rb94X4y%) z^-ZJn!)D&-wItSQz#!q~A_gW_rpzcMW2L_7%dBlR*))o(7&41nT$aARQ36ulMN3Pj zVNE?zQxbX_hU17+xU%eOH-rf(wQ4x%Pdf6CVw!uIcfqYte$c5bVs$Q1OusyglGxb3 z@=8?DsN{CzML32;u|?vIcZo$Ze4#Iuc=Cw>Mt^>lkdLBmS~xp<)%=H5Gi5PYK}LP! z9UDPbV<~}HLxO<`tz-`C9pyA!W+CfCKRF#_>y2pGU@tQ;4OzOKA(P?dmUTVS1v=W= z8nsO}l?1#L4&a%UR*=2e5Y5k^%f#f+Xk90=6O{%|Y$ZNhU6KXKBxVx95>9l29# z3e*{=6N6d*E^%smgV%hjMSODg{mVt!DrN)PQf}*^BiE@Tsm@TM?#guMjAq*{|H1SD zWyopnZoFTVAup!)RkKRnPsQ-Lq3CQLweUO<`cB##WEK!5KW+EumT>fnXDxSs-wYXL zl$vt;eh8yf&db@niD>+=_iru1c|LpdVhWpyp1rbO{C9cB;H?Qdb8CjI>T%5y%$F}> zDzj=yQk{D$NoQ(vyHbj+pX`p1(TzNlukvE^T5a~rqNr5Nj`Zb$#F{GF!bH62mcWJZoyp8p5n(q#W2+QRo@vr?i@%Cb^Yo*lhfKG0&&v-oq(# zDOWX9JRG;^YUuZ5IN;wE_;reWcvdgYb|>~=IUhGAZQ-2Tz|Mw;x|D~y*-+AZQgSrp zBhK}oTQR%^8ZW?;vKQRq7xiXZRheMzJltwuHom8%II%7pz13lI1|OKI#kT z{lf0}Ra0u)XSGa(>P;8}rXn*&QN!n+)RH-y%i)me`CNq2R3V^XiTbLjYKeu;+W8?X zx~P`V-NEyRQmmIYSpLS(+1nBytEyZZORe~!_?nvQW`lngDI=e%L&nmSF@%FDiXS+a z1x#QBmMG|m*e#ca{iO^oY!x}0)AQ6Y6j`dvn5=%FB=`biAOSU6;^a7Xw$PZ_y6bDB z7fwq{WV!P|&p}BAws6v&a?x4EqI412#0=w+kYAI$#ZcqtuTD zETWe!!d~0#?{9TWU}hUm7yi*Ib+)-IrtR%plleP1rSGR({3QW!@U){+>>ls!^|4>C zMB~Y2PW=_{5;1#x8nU9m=hsYAC}*Xg&Y=>`2**Pa9lQ z3Z#MEm;BOik=QJ?`TNlqm!zuQQke*?_3RlO(OB6d(aafaY0a;)Ev8Cj2*w&EA?~mo zxbKeF)?{|w&a_iCIbp$TEUGCH=Q$qxGEMBax%s16fmw6c`Qr6X%ES24ZGoePN=$+C zMZ-8<;hEVk(2P}U(WWj3Uovt=mB!#8;iIsrPPvb6-HTkx{Re%@H)8uTz}5fu z?83Ry?Z5h}rspuZ9g0_mavLbDMq-I6dZWL^Br1|daTo7YJZWj(1Lg9;#U;vh?2Yv7 zdB-3|Y~cxuzJn=i)rp*nyexCB&FE!=TR}1qZkHq=m~YO=X`W{^Y>=ZDU)B#HvqAr8rcb>;80^7L*9bsH1p3q{APy;wX?6t>R#E zs}I!r4*ZJJ`yC<1YJ(@Db(A1rX&PcdulBfJ%S}TrjHQ%+>Jx2DYV*LTlb_6wCga+XSk!W1F6I7MmMyFCVtD_d=TSAcj(l^yAfw~e+=LJ@VDWz zspjiSa}6@Pxt*q$lhEledwXqIaatU(PDStcH>m62g87=%=KQ23P2Pj`m#X=jct-Q; zERI;WQH9Fy1RiWqGSO_8kJRJGMmCR+v891QSC02V4MEA$Ojy{)`0`QJl6ra3Y#(!L z>m1LOKy-?jIP;pr{MhH2{B((h;&Yap`mRqXc-#YQRGW+4);JKOypOztM#aw>36>S9 zuOJ7L(xT1`Iek8!MelwZm>G8k2^@6IkC*Pu_ib>*?!1MTA!T{ExKaF1I@s84Rz7HM zvmObVyMg?rI}&WJ|LmaL%UwK@P>}CAh6ke~o>OY_d%Lv5`K|f-?BGNy?Dy-5;=6HX zVl`llmzqOtD0wV57eX~BW1Kft5COH(U0mo7Kb%%_Q#cSCT~Ro-K{{SgcgKmjv6 z0^YUf<#=lf@-XyY$DUU<6-*i)TaW65?N7t zUBXoJgxoV$6FHr#4+FFnu5NWcezj5Z7M!tiH=O2h&s`a*9~A(`OMx@BjqiJ-N-J|k z5p8Pc@eM~-(DA*Bm0Llv-LKKSxl_x1bkD$V738e=(ZEP9q-S60IG*AvMj)^zF?DsI z?gN~Z`QSp9cgG`WQI_JmG}9ZR$Q@x>PAT>p7Hjw&$b&vuWo0bzNbrdwpy+nu-=N=s z!0>!g0VIW?;vV!HmyuJNc59ho1++@Je3y&ugd>?|t==3n9{gy|c!K?Mx0G#xU16vD zpKd7Jp^_2=nFOx8UAvyFE}~tv zwey;fQ#cH4XIJbAou-NTw!d61TeovdfdjTeYT2Z=Ek_&c7t+wUrW&7rTYCzfcXds> zy=!US`Uv9e zF*p`HOjijRZAP({5D9@`z1Ubq+VJOB5P33!JO^>M^ddpQADgzb^?1fDV)`>xZC_bf zS&V*6OJ*@KH~rcn^mAHv4wt+3{YX_c)w{)LsB+q9-Nb#tsgN1MB1mo0 zBx8W{d^g-2C2q&=^dc_TiXSAH(VXhT^^}n4D61ubjh5=n*1W~Xu%q}_d?BmO_)Z6Y zT?#l3R+ld$hcyuFgIV3J&}3)l$SAGUFxAj7T3j%B8$Ry}Q3*&SI3;hXE$k2#X$_Ty z#%-@jqQ&7T-%mp|(6ZDecJi-+0y=5i(cMt+=>%Mz*CvYZ%h8Lso*sWjgag1SVs2)h z(_sSUn*XYOCZ~btgq#~)Vl8A`|7r^2Ve8;q`12LWcXK*@%J|~SUX~DbRLEvW8iP}7 zXP}MovRwtHGSR8WBga{&t10m_RIqoR^x{UxDh}Njd$?S&=KnR2MwcQ-=|`T`_$i1( zz`?xQCLANr84t6(?stle8YymRQB=8ZtCd^AFK5O|gQsU}Ew7~IEYma-(^UsWb=9<- zAlRwCHRR-Iz>9+tq`JXB!m}v@LqHDCNLROR^yrT@6|^#v^^&^>`E+A$*R>ZN)*btZ zg3ejSWg6)i8Ho6jve`1*P55!Jt&A*})t8N}#gIpVj_$|uLT~xIT#Eg$iyS1{^9H(D zr$;l7C6eAtH?Zg^D-=hO7BJ=CqX^5=<7Lbe}u#VjQH)P!PyOwcmjIecy6&6e27wthKY^2 zmDnQ(2l|LJsJNBCV|z#Web6#4ut(*P=y9n4p!fzLvbCy_Y1YQqO^o$mnW(nu2SY;<2*F%jwk5nzrmH)s!*E86Y^e8u+0^0d@gN0p748b0aN6xA|dz}12HugS3 zvcah@OS$Gv4Mig(a=BG0xgh<4y^540Qz$W^mI2$uIDd3A?Sr8axR!eStXkK<8_7b> z|0eNx*Wh*jI((gS93v{SqPzk~Stx{*8g7OJ?gggOHpr!-z}WOWqg=c8W}lfEut@L~ zeEI2!&FT6@>qjb{PcGM&zG?+SuX4gy^c$wvxs(Y0`5O+`fwk~O8N zGpJuYPbyk7VuJ*Rn>R%l7kd|9V(G#?2vjq$89pQkL4H#=v_&lp-#$=uP|c+qDV2le z6YXsgdn@mWLk$NZ;B=JPmy{l1NVfv18C>0W+g#$6%FCLmaTeE z?-NP@uAP~9C6ln?E&i9$i#>XS_G=0C%y%Bdcrp4c{Dt%S!e`XI zuk{!5H>>RWG&lApmbw&D%E~g*Q_6eWoHP&8xl@YD(GU7cbIP*kZAF-wryX??yhq>E zvN=Su>gY_{m#H@mcJ?Y-4Z~_~lh35!`mV__-F`(zS5?L>xAoJSrmN@8o3E4tUo|*Q z{1bHjGrA1@^V^h$E#UcJ?pE~Gb2BM)D0m*v)U@eV`Mzrq5$A0|j`>obhHooD5|PhM zv$(3Vvg*jMLJYL@`@XfSg{#l$8;vhJzgq$&ryZ^G(_X%P66E$*Kx`Gi)qcgOZrI$W zt=uwPuyJ4D6o}S(ANxV@FUvUVZRaMlUS?*RJkkp^eQi(7&S6V3XdCHPB#i!FF$U#i0AS-pJbI3O1&S8I1> zy?^Wzo_+ETE2-|s?G21s@y_ks!)-r*vCEh1$ko!K!as6Yrl;kms6f8pb+`Aitw18W z)397Vv*H4Ki@YIAD=SMEOMa2NQIPHBSyT}2|6MAjyW>GLSVA@4csEri+OQn?R45J?BpRV7D&+W9qe23tLys)nPdyK5JbAK;4^E zKwXYIH#RmSMI4L<)ik2h7Ogmac&eU>NMfuxzgm2}IhmC;J%n5!VY<+YPMO#zgj%H7 z&5y2M-&hY9fQDJ!W9HdTyvY}XTND;95=k-X^%7$q2U3z07EbSu*D=*089aZzkEMaH zU3&NqYl`X`n;UO}On&QTjeBVf34OM;qUW6bFq?^yDLZANR7!#Fd3JyJoX3|PdLSUz z87^tfn-}Ghk{CW&^+Z@wO#9~3!y0^ivG{kTS;_U84H~V5?*hDLrN}Qtn&$L5YnP}u5sGlz zL!2TQzZK0TAk+a2`b8!+($IV@>N1o)%KTY9WJC?VCm^ChNJsLDER`(4OO)WknXk5NM*Qls0usCwYfoj>)JSYzdH+Ma+#f#M$b3tH zmxMHE_AP&`=vk%|F9CSH5cZIc6JF<4Oh^&=V$$%ei6F>+c)^PEE4HX<)()0NBLN|! z=^a9`Cv{nORAzm9Jv|%b_*yU^#_|lU=$EfRB%5^);wgwl?>psv=)8QnNl!FzXJT<9 zGInrqvTSsXEA&Nwh2HJ({TKO~L%Y}4b^;AuDKKeP>w7ciNJc6h1v1Qa80P9<{pEt< zG>&F2;mVYE(}tHOF&pvYwowABy_hb;o6CyP&w0&UKgg`@2Y8j5u}9=78+!A{db_9C z_HQ~aO1_7|q#b~YL>C_5%VP|>6LI|g1`>L_|Jk{I{+#6Fp>)3YtoWWgaldxHdOnu) z2AMnUX)bLOL6%|u@R|wyw!aeK};@mtxJOD^eJ*~TZuqQk6Cn-)TJ{~gq5G)c-5c)#d=W%zXT~2*N{lbQyqF+K+nv%&NwYi4IUD+ke zh@Zs`L2rZe%{XC9%aE!4?$B>{9Lz|3p7it#F>*y48JS3IF2Zyb-dx9i*J@Ou`&^%m zLtkn(YEr%EYd0gn6$0TfcTwI5yAOa z$eyF5Ny4K~u+@D3~dY)beZ5KK`^}Lj;{SXkjutTnnjaXw zu_$7la~G?>fEO-Q6azJxGX$w%wkVYDBjh7|U^c;5oNqtEf_VqMzK*;mRySB^d`m{w zZw`6wXy7gvF9rJL6J#q;jj&C(ahny8|34Y~ z!GsL*7dlEHv&uWfnE104Wv2V{i@LuT1H}63&V&y?1_vGNZ82W*rlY_Z_oOSVI#$+E zT#|kPuM_LeH~g?H#bLVyk6kZ^+fQ)aP&_b{RZHp}mV$TAhnK{sI5a{IN( z6vO%i@=5nj!0&LVHXX5Hd-r7N1-Heq^xiQC)lRGs8c@SCg0^W9^#fHOu^6sAz}G&P z30)Q7&(z64e!{aSlTY%1C1U;FS54rjdm_}A{_Lxms8L&qiB|uq5FhwF*6*!ycp7@{ z>vgsDp()EN$6rFZt0VmK4)}!zp*t#=NYAk50h^)Fj=$P8g zxcO|_=b%$V?@$wX=6{xS%1;=fX#(Z3Cqjh^rqnYMJlO)iKP*4_#`20gVAI88 zA*AC-n@NTHhF(;sd^{saU{&=Ae+IfOhjmxm2D#geqC1}2CGRhp#X-Nh3VHgJcq1bD z!-@AB+Cn2P$TPs2Np>xeHv}D?6`f0L3a$H>0H6ff{PM6(lc-|s?j^QE1$Fq=5mu*v+ZF*_+7#_duTc=kUE%BRb zun@w5JWlH0lAjJ^(B~zsC$Tgklt$4p1Gf<#gN-1U_&4s>TjIp2Y{K%QuwV9?fu zu#CjU%?E-4C&Oxd?RO_U7V7iAk?>jlxx~iMrXMTzzk$vzZK`LWss`(LRS1;-ktF2F zKMbXXeCb#`KmhCB;)z`R_vC}`hPANUMBmbt@q>xbA4>$xkx%Q^vGjBb`k43$%TXaX zDVWB!Ga1w~ILJ^ul=T}65Xr+ih7Lh4V>2^FDA|Bu0p7j)*Ux60Z21m|P~wSt`;BKT zdL}Sz1Q`V;<+s|I^aZAQ)$?05+|j|3)ivf7L;ofNGX39z;*9)8eaTzJJmrLmGgDH8 zp|F(ClLX{;l#`QFuICv-LfGQp3aS+fB6Rok+!8f1GGahjMW_CpM6+W0_#a(?szKnE z<~b;bI>0V>*|5P!EJVInET{!7ZX09KJZDshpQS@~G59_JAN3!R)9F1Y;d89Cf3NEJ zR}##zkS4|C>+w{dMRrB|($q27NsopXo(jqnDx*IE+D>q|!oSQRztP#H4Kxh2n!gkS z&I%7UCATw9I0z@tfpC-iK*{Y%b zy)LNXl%-Lw2u1h-5Bh6Q;rOeN-u6D0f?%GUzKqt^)@KA*bR>GFzh41tf=PNED{oZl znXkjB=ga@r(?ENGq+q^(X*zyAGxxl(jCRbiRuiLqfG@ayg3)3%eO>N}BGh;17yrdJ z(=1S@VywyqLfDp^WU9qx8oxJR@K_4s?n22G8yg$ZK-ha9UXr+9z~Mlybm6{90aU83 zIL*;16a(%(4#9!fWGC8#ON7t-o2@^t{7slh;xTaPTE=y*XK1<1F>c;Q#eE;_PF5@= za=M0cKl}MB-rvWdj1s)1J$(++vQjmj6VAt}zWa0z)$=lfG;ZMmeprjb3CdW7?M|r| z1)9WNf+asb*2N5ujNVZVaj{5y!^D~oPn~FJR4_t?lcoNVW`u?@h#{9s7MvLu2_qnO zCH%b^N(P0Ofr-=e2KinD*>9YkMcz08VR>-X-Scv8oEVqQAQ#f#!b?v*I|gBSFf~*4 zz79)oD61Sp6$2Ld8+^Z+{5g0s>0y5i*XHw645@m~LVnF*A|e4^gnr;Z68c8Wd;dd- zVdTGi)5b2!bRO%|Y%m#q(~o!gRhlFep~GfP6L-SSz>NWv^asUk^)!24mXt2~oIbUM zAE>iJ@32B>NkU#dp#S^b=ckvyhkNwRcCWTd)m=U>lprIdb4OGbsi_knUD-vChfhQK z{e=iin7o>?t22K-Qs61reRFg38NsPzr}0977u(%uuSjPf_&xr7hW~?%m6|Gzu<6bC z5SFlkP5b1#oRcrY-{OUdlaRBAY37tlP$%ZzS)+8NsQl8o zU2>i+akEE+TH+%WSu5FkxFid^FTUq_qwe0wIAKXYhl7BNYZJCn+QKfC=ObY@;G-iE z1oU<7SLa^}T57Qw717cpW3y+jUdo7lGh|v6-Kx8cH-DFcE0nP#|>j>Sp81)jRwwczCy7<9h#d-b7h|$g}ngn{$sgGXs7nokCR|x$joLjFcAeO}rUjO=E9*Hp&YrNmS>494ef#3&!PYTYGlHU7- zmqC4t=$^`@%a>`sUyrDHq7i-mc6U$2qR04)@ETD`KREYb!kj-7_y8l_INY|(u@#Uw zs=4O$_X0AE`&m$vNsJ^RyL&+@Bps=SC~p3k3QV7|*kk7^4?L5v^*?UsEU!hlH+=c$ z{8!}&@`(wWU!DC1OJhUEo5e+ucOm)(i>lCm`0ilQ!YrgKLsh@DEaqLGQ7#fgx>&R~1)JbZ>TQvtp2dYSq~sSiiJdg+L)xG%A}(MPIl68EVJNm7dn zbJv+*u<1G|Y6o?fs70Y{n-B$_v@!%oxSu>pe0j=>kA%Ii_G!L`oX%YeF=i{Ot7)J7 z<8NG^ymsyOEk<9Wh&^kYh&|;49%MZz2gX`rDh4TDvpkN>QBuVOnWi&;k&rU#oL4L! z(lwEg2zQs_G*Q&dB?;KviW(XmY^IV!1npK+MBX#);byOM=@8H%3}f%WJdymUqg|uR zEuuF**3IUk(r2lpq#qfKj~B5T^6A0l?-0;ax3z)!PDK=gmT~`p7`Td0^M!*$q#GrE zCUoe+D%l?R72Ur#n^S=DglJxMY)Ik73y~(F(v6wTmir~8#~QU!#iRAQ%(nwXdY{jV~(dr($|LsG3$evQFmx}Yj2udjo{ZXMj+Pl%;@!~4pLL5RBnX}Ufj8)=f-?}j5`^k6Q6^_a zNysI$lYuD|ucj#~?uh9moHx6G7crga{SUp)jdn!;f@dMf4lDb7-Xrjo|Bt6@RJX#E zJh<>BC230T+s}@Nt(Lv^L{AMh9H%a%T$=$$oMy{^4M} z5{35g-lO1{Vo)UT7xMk zUH3KpMxeZwx#FbPu?syOIncf<(MU>v*H6hU}8|kT(SWxF8n=UCGNuJcV z=OpI*AaHDEOZf9j=GW8b-8#|b zlSp|Qj?$X_txnCcYS+iQSAeoX`o2j|%sxUIcF9@XcwdS8B({?Y<^ytr;?3Lht7+~F zHBmoE?nwq%Sux9d$l`H=d} zbIMn*=g%>GYd%~;`Gi|;@D>H*cZ-P)y>e4mOILHW-tmqeL~fO|(oz*&q}sn(QNRfO z9853O~fs0N8@3CQL@>#*-uhS%W@7vcmlYTpl*Bu?dKLfsb zPeMm00n*n1!K=v(3#~8r2?8Zj!xtxqj0~uAvzZBf8_D!qy~hlsC*<9}_I{@D-kEZ9 zEAU9HHqlMQm_j;mu?{?@Kx%yGv{&L%P|KX|7DkFQL>&FuOx_+)nnh+vzk^MUjO+d} z(`%ctR_+eTXUBI3yxfl{YmVO}rn+RQ`TRipx{(lEo~quSH`C`RPcPb~L?IgmeXY=} zLd(D6^(-XX!#v+XdMF7zJN}D0^N8WVQfqI|9SQfHOktIJO;P8U2@c~m218aAb4DaM_*@;Ws z@#e%y$tt=pNblqN-8103Ez%e1tv$z>pOIXpE9N<**X0i(+dG`U zR4%%#mX|b&E7s^f6DuaW!0U4~gUl=_=Ppm`aR4(-<{+4<_b)Eu$&;#^G`y`p#Y@&Y zSFkw3@hG4A2~12#%Mf%O~JxiAyjQw5UB%&q)zEh%4b#8OK+t`WoVuk3=a@`7}j5^u-r5{ z*a)f3Lr~q7aQB=alTuYI=Al^(U{PDRuzRAf+YIXBi#~|2^v##`)zhz+mS%W9?;BCm zOW0Hr+}%vFn55ou+!A(2qvj1FB0Fc=F>D5#;Mk*97WOdo^8>~>Q3>~MU2Zj5Pe|~=XxEwb)`XlFB zrnS6I=OODj_XED<)vEGbpESKynN>cEZ4GlP%g)8ZBkH43H``2iUk{?-H-i%c1+(Ps zr3W*En|z{DcSG8r1knE)^>M7L#Hr+96H1x#f5uBLyKix{q(1OzkRCad@^1gi=L=8H zth$={R$izgx_V)7XduHVFxqUp@2y*sy#nJpt-KYS)9Lv8C*~YXNHY@Lm$EV1-UDo2 z+vjgbl32burqe*7M%?yRl)Bp0CL+RNzTkj>8cTqY4!d6+1^r{1~@G=YeUP^7DsEMZfH0iG<``!F+@P zZQ4|59LD*}Z&)WQSMS6RFAY&n@Rimwo83v;;j#}z8t$MV1vTst#VW@YMZSzG=@abF{2O;N~Amq`O#lg_{ zDx5yOM+ zgD?(avUWaWv#AiNPb*-$+&;U;h*`i>I&GE0egFmUlLozm15;qg=p6Mb(*In;T)nH>O-zN!qzLrNp){tD0 zm;GtVv=HK8&)wLc?+hi?Ua$>MShle-tF}rCc^{z-g@dm#ahZ<%GIAhk$D=&>B=ma_ z&oJRI9X-BT*T0=qqkG?_E`HZSs1(h3)N=pCpcy4-9a8Y)Io{|N-tO{!;mv4`b7a}h zHPuq}%h&1Y01@c1LaU&9 zlIM)tXA-4wL?H{2d(~jLFR9djh)2foxt&|DAahqhu+(c^X?N+G*5rqN0~g8(tGJSk z3)uEfyEj~x9ZrsTQ9_xG+3j3<8N&%^zI=Y8ge9+S5;F3E;3Iy=VFQ=NRvkIn@m=wC zf|_}$^_M`%nb7Y#k$O^~Bsm zek5AhZ9YG0GV=Mf$(@MYnR5=3e!oz0)V3 z8h{9A93NV@8)3TNzZ9;PTT!cl%gezblqTrAyHLBjyEt5TQu^!9-hcs)=!LEABrly4 z+i~HUtx6YRyWN3pO`H6Jsx3a9owoLpJg?YSQ^~rn`{$2-f1+G+={~Vu@-B3hrbN%R zlz9HJPpCcIa@$)CH^_dod;P_fcqNCcpU4|$&2ir zKbhZDqBq-*$V)58c`ox~W7~eW{h-6gtGNuBpDj~fi&jmr8@*k2l#rjWWVt#ptHnCp z?Y+zHwd9ec$V?G#7*%*Qvn!N+zZMBJ)bqf&BA%6eSvVs} zXh$ibc!vySQ0vj{vi-H;rX?n-KOf&1F8|@qJAhttQe2j(^lYE4oNnOXddpv%>`G;Q zCCgALZ@LJ-cBda{CJ>{2sbhL)p`F`XLyOxw!C9}a8<}n;wY^lVmk+2X%k8}{szuj~ z^f|Hc+8f1dU+fUn*og>M+8W&TQ2|i1^hmhU{Y4%e_LKKk9OWYW$#!gZQ9hug9GL9^ zGUP-}7s(zj>3;vrt!y7sj0h%Q@T#Jau4n4AM%ln@QGSl1X=i&B;{ySKZ3oAe;1#?u z4$qYs(O}AQ!JWO$?)j}BFQ58y^bK-u^JJm~4PJnfWjbl&o$4WcWVY(|^X2VxWMPCM zqJzaIA!S~>RqivndV{V$KC;uX?s(51_D46m`v6At^`xj3c+Czta}WC*KoBj&!}h2E~SA6DZaki>bi?6rN+>|6WLY9o3rNNX}n>-c~eReK@P`%!nzmbNak zuE-(1cH5^eeUmGmzPz0UM#^E9D=~K>)nPb+->I&!KBjhed3}VU%5Bxy>(97ee5uB4 zlHSJr!FMg8u;G}rUzE}NY-|tNto)})bzPn|V~JJDWLo#3kGH(ayk4ii=Ie2~Ge2x& zWjV-wavWC%zvkB4Gxk~-#T(gKQ)AfpU{Q8jAb&b># z_43o*pq;G_+x4^w!Tze|3-8`>3=Kzb7q+;pm4rNdmluc-D6pDDCnlmtx(NeCi-l}* zUP6QB@740$_4H_ak4Nr$!Sbea=(9#M8=V*yb>y{^hz<81AG}Q&H<~Dw-PT*1+D{j@ z?z4$@GB<%1Y@Nc{%NYJ$<~3Qji*^f|^0M(xB3ULWo>-4Yd7HH}{C2NCn9=i6WHs5r z+8Vs6m!D5sB)7zpZmT`lZj*%0|JrN#=!e>Lzn0$SP&}niD-&yM8PbQR0Ovb?i2D{-USxtkJ6R9tFDF+&-yH)|C=IhutT6 z!=8JMeA6;EiZJ_jwRY-{o!~L$$*116wLF85?v#0VE=axm^hDczC%vropiBo<#JU<= znG?{y*k{r0W;Fp{Iqib#^VA~I+{(Bz5i#tby}4vziNfSNPo~Dse%P)(h*#?q@6-%U z8?OGv37#K6HPNr*)3nVyS^lHNNGPFniP>w7JgGlOZ9GfZZ7^wpCAwC`TW_tsq>Q5^ ztJz>(g6QJ>(B3}ypW`AEYv*DuuJ6;~+5=5gtyNXo?BD{&siVE(d8zi zJ3UtI?9e6kf*Ad!gFd(J+cjj}<0Mx!{ni9G9FsMlwLMyVJGF3(j&R@jrjGp%Z5baL z*r3U-aNMv%T5j6E9Ot8eR`?>7iM@Eglq#IRqc%pOhjVlR_eYzUBK`Fm^iJpK3#7#` zD{3cfdpVPH?|sX2_dm60OSETaXXVdk(o7?ZH2X?zO`q9Fz9Tk|*WY^AtjS%e+QU8! zq4m!{^&3llCC#J}YjnF4VtD?@>T z6s^5p%L>z$53=u&qfE4JBf^G|>cWjR><&C+yn-2o!BUJoLq5{G0&dW5u_Hrr?ML*B z&eE3_?)ws4wIoSV_jRpN(Z{+oui^gW{R2+6`P<|K5)VQ4nW^=Y5n};}%`PicXdnpx zN)d<{IO&rz*Q?>1K2mDe0AChP<)x=ukkZ0b@@63yJY_d{D!1^4K9x1Z#FqSy9d zj@5T7vl=I)SHAnsN^gC`YbErK;D(>7_4vJc=@!%?$GRA^@dRb7cGuHgujC)PUPHM~ zf#|60*2D0s#DU*?pRF#bNSQB+24#xgOmecDKynLkSz0tRNKBq=?>Y_JthJ#lf_Ev) zt#+>&t)rpF#BB=!bX+!KevlDer8fj{ z#5d}p3}j^i1c_Dt%KH`-%B=RQ9--zt+_j#sYX{gt01#vF8XX3scLDV)h|9k>>mx=q*U5=w?{diq@@#u+c8ogM0 zUGyVPgG(C9?6ngj$BOLOX<-X3&m_9DVzTdxnOaIy>*y$NlvB&LU zFS&8`n@nqrpS)+LjS_}wRtBGhe|#7?11)%~q^KhH_I=?+q)<22%R9^D91Uv0H zT8kaB3PO1j#ZIwLm8zeZ>D6djBh`9a6Yn;*gfQ{&!4QBnk6ml6&aGdZ76St#j4m=k zDdFt}J8P3jgaE6UW_C+Sx|oT^AlQO!nL_ z%O|rer-5WwOUPEs^M32eDWvne_KIbD&)hw-YbX-uIV!K&sCSiz#gCaI_rO|VM}J?c z8=v8mH=e3ZkU=-C+UOg$@Tti9z+cWdr2mVd|0-wWv1%p5%&RU=Bfv3)}}u;Gl4gAUQ4`lyR&Rno+eG7C&neC z{!RXc~){KWgl&a#nCNz!=w_@(c&Bp2CB;|Rs8F8Pa z7ndJuwYFhMhE);Yup&__Hf~?EjbNXL;X99;Y8g0D9wBR;w%sGM8!?!h*gNR1vu3gh zOT$pE@uQI{r_F8d<(y4q-vQCG$u(Q-#)kLhM(ojvlox_g(L#ECVHJ;^S?{gZV1J0H z9pjLfxoCgKhgQ3QN+Mx-7$qs)C#57>S(%}#AG-1~XzPxc`$^y$m?S@YUJg|(^9N;X zq|2zkVZoR#y1Yk~vFm-_umt9s?WS(u9HonwtKFMXKG!s$qC(8c+*h&Om9V)y;BHY` z)ITw^n>{YqxxY``x6#x-*w^zMwYySlvgsbI&yywSn|a}Fe`LR1D;?RjWi`3uwlbT| z(awWCJx|Z(WYNmC2J5C9JNotqJJLJk?PF*&I)j%F$=3ueQ7+A9CNtY)+4mM}u=T~{ zaFy9%Ly>2ql27X-@b})AcOeuBqq5yA>hVQvlKm3&BP3oslsw6a)camj^uu=5+uZG;rLXL)@uR^&1omQmi7xwYEd+~Y?$m*j(Kahk zIGBW?;#O$x?qlN|O7Vi`vj&xW=WZ?sQTn7#IH9VxTGzc4QM;DHpX`i^K)^uljLVHB z3{S6nEY-%dUvFFd(lMt#`7r3h%7~qsj;)IB<3*DlL3S&>f*S!={N=NUd|{4|M327O zv5wXvm2ExbjmVXByMG4jGhE(r-uKT=8n1e%wv3a<8D9ns5UEvr*OBn^XO@xU`i`pR ze;f^d*GMAZC?2+0{16J2uW4?!(zg#=B43xMMaR(m`jh=GV?O|UMP_cSRKt8Ff0z2m z>+C&o_trLy#$?bfoZ&BDq`b1h5}ovWx%yy54OMnQ^n&AXs>|#)nTwEPCddd*U*6L?!Ksg(x*@&akyo@f_RMYhF=>s;!&{+&|JveUPF z&X;)gs&izv2yJydU7}d&TC9Z(*+szB*J?5y`9+y?;)WJgHK&mvdTrJ|Dhuu)pptR61Um_0$B($KU^Sv&gAYHD9x` zt!$mi#mh)j?@cV(8gHJ>$r7r}D>tg3FGGnt;rEX|OdiZg0_q_e4ie4oy(+6zaGv zI#pbAB*q9XG1jZ2rK6vpz2@rp;0C2t#=_tAZs$!XOb%tdYYNOPL*r2>Gm{PVgi@C! z5-$q!u&ow<43TA0iKYGC+tYhDx}}!+XU|k4`|kdR<*W21IjN^_Qwu$XcNc@>mIj|>qiVx{$3ucdJln~kW%(z_DJ zQN;U7MYK4$mZnbdAn*XqBhZJ7lu3YE{oU^iAl;vOKI2s@HF@e+HAEw0BITW*= z)I~36*i5Ej6xF1Yr0(U_B#)}&t)%0DAyc-RZ}p#IPrM`g?PC2Q7h67A*e>%Dd@6X5 z+kF!0z58`Nk3X*wkub=P#J6w`p%h;yrLmqwiEW=YN;m>*Py=_bE<9np=3taE}_DPLct3Zq&FgG-V%Al+#ZivCKMZ8*g-R zS1aKO9KCS&bvYc)ahwP`}Tta!*W)XlNCzmF5t;pxfn926sDQDuAHpeSfMcMr)oLEmVah( zhw}IV`ugW^Tx(Ru+W5|mStTx;#4`JKjv@53$aYubq%t58c@lq9T*=iUpQ6?jJqjUWd%t|!UdP5Sg-Dd z`akE&uv${yiB!%aZF)0p_DZ@m9D-Nv{VzrqeHQNRn#Z+^sT&`M4;p2JA<)PiN9E<{;;M`{hyD+WGJLVGY*R96CsBr7~ z6t5i&=qSyeU%a8N7j(a02^E2|Yc<^N8?4-%<9F(88a6yih-T5vAVpRx2%l_qXG-dH z>9`aC?fFs>x1Q z)G{<2YGsI3`qpmI0)OVSI$_i*TOb#S}|x3{9kkQHgFDhxz$X zq!u7TwKQHXholf}?80J)AuM0RB3`!hy!MPLvA=9hH?<~nTI!Chw6djX@O!%SjwJbgwwZXxtt$j5 zM9#)4#U-#tcO4W9x347AvW10Qf-+>>geXs$Nh;?C)vG(~Zf+#0-5mB`TN zH%CVilI!zjvTXh5pVz$ieLohAg-DrLWb$e8TcI&KFjcw}rOC*utY@JtOlc;c#qe;2 zc-pCnC;Y3UF}cO)$D+K;I9T-p&vD*>qTH=o=PNJ15j|1P%NimEd2TL~)&6kP$VZp9 zt=S*w-;HHnYadBUP1Hk;xPa5@{9bcej-@;j_w_zOn~fh#_h#jj(W&T21$I^JXqF~8 zYg*qo77Q6If9*}ipn%!vSQM1I>2Zt5sO6Y^iDX65KqnQp-5fz;FKe~us(D%i6}z&& zN@54W3w!l{-ri!(`PSAI36+{uBXvG?XT$YU@->P@CoH|jUtEUWVFx88gSxAot8ukY zc+94o%^nAj9N4LA$_c8-mZOMYs8>7ZYdd@oh8)hliF0_NWV3rXaCba&?WSIQlulTV zUF-Gyu}YQpa?95B?Y!NFd*nqEl*!*L!Qnv2XW4mhuWi_SgVSqs%_avn`N9VJex%@? z?8AQ5VN31yL)6YYCGG`koBi+A!30?HJWzHE!-t}2^3@BW?kxugWJ4L5AT!fN=eTzD zjtI&4M|1TV+|`fcCVszjBTKVui(=t3YLBX>oZoQWqEAAPRR^apriDAMZ4WaM2j!fk zM{^9Jav^C*nffeHTDnnQmg6?UHdx*{rG-@QFtk?>dPWZAijRmisyy?p&>-vdhfYxQ zRUFOag?KIC;VmZ8uNSUX*|$e-@@ZirxQ>vCeG z2D?@E=#4POz|@ODOIU<0Zlqn|bwkKM$@@dB*Mk%rpekW8yxy&kmp9cc7^IQThn@~JQ#$u*H6@p6btON0$-viR7F*gCY zU|r-TTkO|O6e>=au{-{NT)Q)qHLInbJjz2ZJHAyG=Li|?d}&btjgD>e`BV4>@s+t} z0qP=%DKI;gzE5+I(W0W#T$oK~HHL78>ej9irIy5^yU972P3}r~ChouS@Rp`bD2BSC zU6xEI6|^R=`Q+;{@vE6{R(4iCZzs~M;XvmoVTxFS?_7|Te){^7{lK6?ks@=UdXaL| zJbU!^b`vSyi!k}={NBiU-E5|A&fK(b({@4UTL}FT%34ZHJ}eFLP-p6P(f;Q*SAh%! z6(QTpVE;>ImeS^y%sbz6T7JY-Z0$?iW@o(fCd10G<4Z8lo3>%cQobVBmgC!NS^}LJ zmjjU~yI*}$G<0;*)Y8n=P(fzH7+dqVb?e1Fs92>h_|#yC?^-iQPa-2*UHBPZSP>AZ zFzu+A166rg$>x|M69+ySw}8fHXDPxs_%z2Jssx4-IG9CV;N>#XCikW}FG@Ge(uF96_rO{u=xdfhtSB~0xCEh*f*AmEI zAW0MwH)xq=NDQT?GxkI_((rODc|+LO@cH;r`a|P9`_bK$UPIpVbHfW+t-ruDr`3F) zVoJ@jFt1F`x@fO|{AIj#f=ouncyjLxp+8iAMbcfD*b%@K8FF!kVoKq$l9%9C4$ZB( z8gP-r5pp!w`+n*dim7PiO^@{GA0)vCBm>jHy5nx2?2dV*Jx^K`u|rK^ui?!%jU0Cq z{@GkTK@Y9AUogdWk0PP6Zy(AU5A1moTI~Zq-DT%(8GG9}$0IrSg)d=_2km*kCUm3{rwG7d!vYp zp9l3bIyv_Ep4KQDgUkoVCw;t%?c11#+xH{ps^j}$Vh_jkCQX0QFcK2!7mZU7&fp@; z|D*Kr?eu2sd)!uR-n>UGzaI$lF*i2S{ic?Xk(m=?qZQ`lY$81^(2Y2=F`=NYnw(lb zjLkb{9bWo??Rk{APRVye7Uj5s4RR&1qO|4{bc!=+F%?iSY zJfeW9keD&3!^4?(b7KYN&%?Ga4-`7z=p5DTM>eiTQ4wV4=DL%F$x4flkI(Ivk(3M# zeWXiG0HeV?L@awK0&G&!io5Cvu30&0Bgu_}BfT#wpu3{LdNo(TNWcCtSk*bO#UNPB zgkDAz^PW3$TQiUB+ywgi(iMH`IJk`6635s-%cT>FTFovn9Kj{OJ_^+8l}Pe zu^#ywKrQ#|pBN&w<`x+`--s2hXXp8$kpCh?-Q?7=A#D81w||S%bf`bMi|#AO6)UQ4 zzHIF(Fz1q$CYl2uzyO+BvL@8=E*49w0*>yBF&MgQa&X4x!DQV24tD$p!cb9K_ZA2z z1x5EEj^Oo6H?%L|bjs0$Rd-%4+w+`jx%c#sYe&q{@jfZ0{_(->bg4c$TlIsAl5*ybvm+2_=tVeOs!=TPc|qrkvyEOTFV|8KEZsJ>si&NQ;b3 z5EMMKLt}Qh5R56q=3H7@VXAkn*`Z6t-13X_(6Tt&(iY|?D#O_RBW7f>N8cN+>gPoq znPw55cwU}|YS1HKxaUddr=VoeP58%E`awo*Vj|}33oQ9Z9rg0H&Fs< z!!OOj*drU3Iddb$={^u#jxB-$?}%9SGXYmC-93oji_SXom3}EzDdprG(d(Xmc$-XB zRe~{Z1%pSJhM^qjVO6GJN`A({~rP;+o7HrHXjQSzCM zSDW<)6x@f!lBqsDZPtC%3RI-cgcGMz*_evadnKl|`NNEM?bucy<_@tU$n957W5-vh z*)WH2iTU9@OcOiP-{v^1WGZM{Rk=?RB&*}n{rT(__`SR=^b*1ZZhB}9>AqUC_(JmA z`_oGGd+$-JsxY}4(^E$Fw~q?m4+89HP$u3iJOLZS{ZOiKUGF zukZWYdkRkX658ChpBMfZa4L^G0Sy65{lL(rU~E{rr{<>mFL#bry5@< z!qsme7qnHL(*N?K5ldcQP*8Abd09zMPp{DTEa6o>t#glvmg5f9f<)a#9WA>bHl^B; z=kw=8rL#cg;^#X!!B?}EXIXi1`3EX0l{ynT;ioUo+Er*PX|(s6M&ubOpT~4ks?ZY1 zWm(un2d@aExDKTKc3xjs)u)l42|crUpNCj^S86=0!w22sOxa&J-V)(QA0WU6ybI@( zfO4rR2cOe8^bJF=+OtyG<6N#OeRVVxe4fPUdGCjFf1@3SnuUa7Nmg#3D|4vd^`Av? zM2)L!3ey#2SY*zVDLDCw7>w>8$|jHrk9arQF|rZ%jxKychMWCpSTZpY-je4AYV$HtlqCNAAryh8WgjbEKSqLvK z?$aj?Zw$5%4vnb(gi7+ywg7WEP$mi4LJF+rW#iVFI`iv6qNT85UU8tqr=oH8*0`dW zgD>=yUSa=ouQ$(lFx$Err6zscc(}N@*ek#Lf!M6K!zDm!l*J84K(or3rS#y@dHbqy zK=Hw;I(ubKNDhohVZ2{g0zc7J94ND^^+FF!Omz9*X;T%T7>3PDhgu)#9)>~pCC;mw zoZS5pgjn(un9kKKaj{^s-hMgZ1hp5bWo4=+NnTuMa3eUO?Rk57IJc~fYrX~h?3vXc zg-gxJOcbcqZbarFk$TCYXWkZ@_EK4m6p28S z39m5ZuFJF0dPBwCLUC%g_onpEF_?lJOOVhZWU`u}bTg!-)aS&T9Ohsk zTLuRQAx8|#$&IUEq?2Eh|uyyL>(p?>IlLOVBsbV3R8S~3S`R%za zfWA#%W680SEulDdAv-g%jj4XhxvJExpAI?(qp&XI1gO*&J!^b-l2pSVVx3#N*?h87 zy7E1YnJJV=O!pPHhQbW(rEGqdqcdmTS?9fFe30(((o7CzHs9fxxX6~C;Kt*SvHB^2 zt@~U;ALCOBK8~KCm*=i&yig2y7)(tzQZz#&Y1v*yG*U>UReNw`N3CIBJ+{0zsyTsl zu%}`?kfdx4-l|DLA9>b(@9vHkTpO!!Ff}uKdbs0wtfw0dj6G;F38DBOt+$4=LSA&^&VcXr&#if0%>TUp)00rKYPxxw&2+~8p zC6W~v7n9z))O+wO>~+y-N5@r|Z+H!>XL(!N9zXqMb~fXeaqFg8Mk6Dm6_!Vip|Y7z z4GgFcy@g?T^@`ogM_3K|_%bpw(!|#Z6eOR+M|@-m_|q=HBoc9OfJRTjVI#-u1vJ9K zl+IP;#>E;<-~IYR?p(~u$$8al`et}Jay-J(-JOJmh2`^zr2y|E1cK350=LZ^T5|ui zW3He;oaQWa?wVStV?BtK&O##Rhdv|PAH;RS6)r3-DQefUv9X=?ZEQF(Kp-O6T$SPO z8CeJ~e}eV~g;!7Uv4P(`A1=@l*rN)2oI99OSV&M>S{e{`48vkw(W~V~^kROdUbAy^ z&G(r!&Pzme@oZ94r9=#(P*-5ors;dtToErX!K*m0B^3ya=pN(C>+0gB_FjZb%JOWQ zwTBb8&#I`X+!K>PNYc=;#LgsJHqcx!D*c9s{wB>$V3ijed)v{;iIG*VP@VZRYE`@D zvE%LcqSP>982X_f@bEBLyk`jbjXt{g_6hvjQc^O4s?26GI4tZs)*}UlUL3KfB|%(A%tG z^3CMB@8o*4AS?dGJGZW;WM}(JNWxpMoxZgIjF)J22n%a}e;=L;*xNl(QH1@aS1)pM z?%fI!b*<4I@#2Otp1pm2XX83?atCwlBftQIVrG)_n(c{Fwr}_&e!4TidGya(0OzWQ zTwEEZV{@HxQ;vL}dVezoG7QFa8e<$2V<39BZ{L2|IQ8?yii!wE5$hWoVq?iGDxM8& zDrz7g$PN*6iWrQUxuZ$>N&X@XHxIZ@_UcY@Ss5`npTI>b$!D7R!~lOV(@|whhmVgh zpnTF>%h~>oUqoDd{I~DlFG5TCmwp1HYh5J%e&Q-MH8sCsC}y*1sU~DEjrtF=2#E;t zY5&@Hc6Va9Y4%3A7(MN)hBj9{u# zeuDmkFH0MQJCk=LZT^~XL4j%~gWK7#wK5%kPa;CNLyCY<;ZJX~@=K$Y$ku3t?tsYI+goeZzSHMJ=Wr zB(T{84;K6~C5V_wkCB!2-SBb~4DimggE97i-!D6jU&ANzN~Ux~good#>j1>KLdOd( zHprSO@YclCzbLTj`C~Bl%F0T^-En*o0*a6epXB|RYW@7giBDgt0&WDM>LDj*x~aB? z#%)WkI6x?cd;R96|EF!YDu6E%V{AK3-N#25#>%IpBmWvVv)tV_doZS)oXeuA1ke-a ztkk!-XcmRUXN2&8G3h&0R3lWO7lH4l1g;DT~_o&*i;W|r72<6^Z zJ`oW)pOcfL931B7*LOY~;vrc2176-saH0|GT*Iy2voPg5AS$dyotgN@s&m1_9jXyW zKJx!J03#59SkA!68tUtz54k@>2I3IyMV!|Q3k#2Eu7?VW;$thTsijs|YeO_+P`dOA z%eV1mN(z-oXHZK^OYg)45G&_;L$+)(T7?5~fsy1Hbo^hFVX@mtOCye92=`C%OH8EL z+TLa)QBzmH=-YTEK{iv-5D~%he3l7lS5bL3Xb@o2pEdKdva_dl`3MxEeq{<9al*q7 zd)xS|q_mV#);1zC^4ad%xU?;P@5kOnM{q`D%Ib3w1%#d+C0qnuRrv)10wZFIK>`8- zEPXIi0FFOGoc3U&92^|=#>Ed6#1R&<_}H*8@B@aW@`n!}zKn_@2GU5BN-gO%_9Uu^ z00wXQ6^+eCWCF|MI-sT+jQuBSwN7xU5{iJwym@k$?m43c?eRo+XQ!1u4PqIhI8_QW z7d(e@6id~dU`AfvRjxRh^ejI=*7QO1-XA{zB=F#f9uMt1_>UgkLV#g}7ezR2xeEfq zV`aArDRAjy7|F_tFb^gN!)vJd}MKy?X7Zn0Oj^*(D_<4MPY-2)yek{~38x23lw?5Gde~LZ`TtRN$|T z2#i%OIu;4M_3L|g=b|m_>$QUff-%^x4I-+Ps|_Pe@cQc_1duW=K+4Eb@RvcNn6E30 z21Fu(2ujP!wvpUm(knbQON9uUyHqIyUoqeF4vsJa2l*ga5yHsC^t#hn3EVYhJGFoD zvq?7vgiA0f4Ff=NFwIvDcSuM`Vph`MBxOr2|b9b+y(9zK;LR&+y`WJP;tOCdB-F&?A_6e}d zuLA=SDR!6;n4PVFxgY2Z{4dk>i$*Ru@)1$fD2n}(G@{KF@b9~gL0PlBx;jdsB)k-c zHj30i^Fo6b+!$#(hzV4ZEdRt>1%>^#tt}2sEiDBlrPoD8Y>}xnR4MIEumYSrGs%ub zKp~u~h^($%!GcHbT$NC_j)*AjFlG-32q5%&2doD<92go(0Nxw5M?jGNpv#^ySI4f2 z2m(e5L(xQ3pC|+2E7WR~7xDEEUHYp30xiG3*P^M9f!E^T;1K4&Bc-Gi^YGwTYs7p8 zCPrJq`6Q`N?-~x;)FHDv3YwR z_`J2N>n%tTPC1W0@LI40a6W?kNBG$A<#19C<2^lcxCt#SQc*E6mG{AZei1)AqWu^| z{tIX``&-!q5s1r8ZEBFQ1K8cQ=)j~r@jL{2ye$-Z2n1^;$MH=~;{%z3%$0myJjnnZZmzP%!9UNUI3ZoZdSYG9MgfwYyZ31Q3(KQ2u48 z5Pgc8|7ohPzwLb9LV z#lO@Lk_`A;)N=mp8Azl7%dMwm5GBDF!b%%Jb(-~o8ybbZ=90Ftxi1SNeN8WAtL8RA zAkk-wRet~8&kt+SP+Ua?q&n)qZr9)64|w-fM&=r@Yly(6cCYT}TU+B|wW~1jW1f7; zQpIC#$0g<%F1uLx@7;d_kY_labKmuv_cyf^Yyg6$i2JW?i!jl z{{_8})Lw$%axJCT6u6m!np&pm@dF{D;GG@kks3i@QBFKk8F}DERuSwbT|Z}9f#&_? zxy1!>ehlX?zIxHurwHB%RB>dMY0CE2|KPbPE8hUndiem*0{FICA|fInP}*(d5GVvd zjQxLD?9h3*Wt45RG@=Ce-Me=^VXUCh;zPso($P3>Z`P1e~E#t?e5b z@dxc$!vGfh)RYTbHZ&B^xjav`Y?M=^qKX7s{4c^V)y9N0gwVmz3AnW~QKQxCT$sW@ zg@5s9e)&Y$Y3PcAe2XCTGUh%;M*~`0B}NQ5NUi~7V-hHg9SqQ1JPTM0KOv;e%{)zl zUOl>#{M-?Nhz+sW^7kA1zz`5X($j>&TZZDlS&qoHOoPe0z+z{az?EQ)jaaPjN?Aaz z-mvP-q-(7W>Fg@1JQorX0xLoYm(9(hdq>{zcVznPODqT)Kn%)p>8TX`z)v*)p%C=U zF0HPkNJ)nae0gfPDt^ zDu(;_)qsgmNs5pJ;o$n7i{rL=*F7(FFF~j4Zz@x95TZKU5z7uKKGqAk!$wF$Y!Qeh zj9Q}l>l+m}x63)|PmYcUTW0Y>UYh@;3PWl)2#g;7Cr{v4puqKoc9-8D0gZvRx3o9$%e*CxvrlXeL7Y4`!_W86;6CjWG z>(m2yLWJ}@fF!oFk!oxWj}eHX|HDKT2e843!|iCnen0?dYm>y%rxw>70J82WL`fCu zxAL&CYyYp*C6Lb`8Tw^Z2haaU ztydMKUSL&_s{F6q>+6&)q@6jp?xsVA$AH3sqN4)fD<%PZ2*Awmyz-*eP!Y%lRh#O+ zh=JX{Bm`tQ0gP~a?+#==kbx)+1tO?ytHy7Uk}7d@{R>N}6eCA0MHLhXU`ES|m4S!? zh-yuLlQAhqe+G-s>LGqbn*TEid)vQ=K>)v@x1y;p`xg;^$?Np=w5g3vGzI|}#_gPC z5gN-|i~T%a2PLN$y1l(^sVM~kGUZ?Sm-D_P581o_NWnfR%KW+%TTxMg3H@G}d)1&P z!)01hr&IzZ1H_p@QZR~7f736bRcyKdDo_$(G27Mr7*g#rWHp#>U2Vp*m!Z2sDrW5e z>C;`%EUB;if2W|nc$N(J0yA+SCXN!c-b7p9`wOBq7R}&g5O1HAqxNe=5x4y@;LxD| zKsQHTL!*^O?A?w3MQBZ`{rnhc|APZSQt%ke6clzqxYH$)Q+a!#oWN zs;ci`kaQRX9x6#Qa3pBd5ET)5y8^W3Kd`e;$=cfbuqZ4t53v})#0xhCQcnH+CgXwP z@P86mM2RyX6vRzwx*M2K`mG7W12r|_>-9@DA%*r6_kt{RNK#-yro&dr7ziDx-nS?y zR5^k%+$o|9Sm2g>O$rVQXw&qcJEsQf*GB==Gs5n#b*9XggFO`$T}PGMoB|Zpwppnr z2NxG6C83u~jQEH0^)Cx02z7^4KjnSjzqL$u+83PzPKZJhJBnTsspfIN9 z38|0-*@> zjMuM$(|mKLAtfb!J#KAs$>bww7l*eMRi8DG8|6a=AW$#!e}sNvAMg~@e+fpQ9{kli z+t*fsXhFJDY|?#?1emvio*v%ohGa!$1Okzr`Tt|PehV7_Dg^22wmk2F zN;F-L>xSy^#2FU}o`+qQ)`KP!EqAbYS2lxFV-&IhEuU;4tLmv$AmJl)qjYcAn7usV_cNcMqmF?uf67@(T6PnR+e13w{8QP9o)b^1Ijjh^KbVW zNR<8yIn7EzT(#Ypk-&QN`0*J)DlQ%>Dkso=5R6iUB96|^&s|;BdwHTV@d*eRO}OGd z4KMe?iI!EKBL5T&qL67>S8lTY6*`rbOB#CdvqLb>3mX!D+DWdRFr|G2Nf)EE2UY=( z%FFwJOx%*XTO-nc3Od2e+|0Qc85EkE0?N#HHB>e&^ z$>V+aC1o%sEpcpYY;X44_<4UejMyA;9h?rs`nkC1PFY`mJ_co50jxqVL$ca9_2|D* zb|8Gu0i#Zy3AxN*5g8GIk=3xUYIN%wdJXRbv0oZsj2HA!?dAEmazI55n?5oBCa+8! z5H_`0Zvqy|tEi;gzd<0bLzJgS|f~3`#X2zih-{FJf6dv;RN+@2^%1 z%Um_ymC)@c?j-Q!PRg~192{>zSqFY}%KQL%JB*7@k+)Y71QH0Yy}Nt=fZbETOm2^M zRTOWn4rI{5n830mp~%+ORx%PP*Zbw`*C&u{B%YxHgmiBHf9q@5r?4jd5lXV4n%k*KMv;>2}koO*8w3HTsr-2XF?;o#)_GjpCG zpBIuZQ=O})aC^#k7+wR>|F>E%Bk`B0{#&h&kijUMDv%=HpqJHmbX4x;0p&Y^--Sw8 zSTFs%A7m2w|D>)>5LjEL{iBheKYii*@|M|%)H@76)ch~ft`zgV`2Xs9%djk?Zfg_- zMM)`Vdq`Olh9(1)x9|stf-j06*lxvX>#)wqKtfy_u)n z2mAbHFOTcj#6+I)dlae2jQ_!jsHnIA--5dWZiHY79_j)U|K-aU2yUYR=1_cqSU@o| zTkVsRWY96YCQ&+s3-2##@Rw{zY}J1U6pNAyyb&fOaX%*~F#+vpY!pF2F+hNX7OuW} zp+u#L1rT9>{4JlS{|NzsBm)bC$g*(_Z;RZXwdQ>eiPj&3fqyZr;n890Z`!(}eq(zS z>ie3M0_peCyTP5yC-M9IwY=GbE6>65{)08a>G#XX%Ia}^nzgMoh%NjF(*t^$_W85K z@0*WEU13D#WMJj+j0lZ+w`BoU%U?f0`!llFx;{X~5 zMCTua025~qu@3;;uQ*BO4qOb6rs9&5CH^olB$EZW4pwd}pt|eq5cGf)VyEs1%$-;eE z;d0=FO8DROiaG|7#y=($Yzu_AfJr?R2Y)R#miwPj;SfF8nin&+`+DAR*Tx#_u9HAq zQqpxKfSrmM1PnQon;ua8C&312=)Xe-Dy3lgY%P@&nSWq>+b*FL<6BAKCOA0WfBN(( z7~4-u1uTIo^kd%X9@Ei57+nAZ5D+UxM!)gTpFfenDBDuPv?;yJcM~@;S+BHP{y8x4 zU#WJKO!3*D5c;66|4sGdYkpPzYqtFFKsuWW4tHvbxq8bN_+=*n1iuUnyNGDi9IV@} zrvMTm=33PX=?Sn$W?BJY?L=4X`5`_87_NEisGEZ5ChBd>8~sB=EsJ2OsIb99&$h`PzUEjQ@egO!WtU1AmEkV;cAHkXf{mmXftUFgW@J^k8Je48Lc(rGmX6tc z)gKiNt7HtKWCYZ(%EAIWDlbn_N2fsFKrHGP74AK<8=te^eIjOPEV}v9k|i*>jh0d( zyI%@gIxqM2U5UzvL>>P$%@V$Tec&xsI_f__?a$U$duJq_VWH~(xgGjlIs+s!`h&6G z`wtEN-}Z#!czn|Jr}r5cir8{+gJl9jrNmClM`c2c?L7!3*R4dtBJLIe%1X<&ySt=j zzqllG6(f~_Y|sCs5RKbHvwq~6n2BB~ox^|meSHJcwb7#2ypEgT1afk2L7X{_fYO_z z$;60Fjf`#p)E(IUg6R%MtR5BXxAfVhW42`6 zO~)j_yn)~*A3dUW#q56J{|9eyP>Dm&#pSY0N~}tV>#d0rwWGlMLHBk^~+03uC70NdxaJB2dK*i)+hNG+Roilzes8RhHUcLU%;1ukdVQ&Oz0ov zw?10r4TzEnVtry#{f{|MY<$%KlB%?YMr2&%f!yF%)!XB21Aj#F@vlb7ks8IhF^#URSOk?rj4)k!l2+3?r@7=-`Md*2V6 zibD>E$}Rjr<}t9;QTP9sH%J^d^#Oko%Zn==;Qh+FF1|+F;J>v1G=`S)nzZa(w6tVM z3`J>!(zk9TZ4M+8n+8U=%PGiuw)gffElNoTH#MEKWy(3+?iVkbQeKQHY=JLtoU=9% zI|&313vgy6gz6(R)4-dWN(l$TP~!h$Oae`rxJa=66Spa9`ipt>_n)Ycgx_Ri53d=N z_NY^)QhNTaE+9BdEJJD6(3YbzQ9K*vW7~;=qzLjWZdpSXv8Z8RKR@s}gt2?$=G}P%XHji_>p=9IT*WVrz>Bq#07CQtCq|BgynIBR{I*8dZ`hqo1!Y0WB}!5*8NB z4jbdq+`3I-nGv6eQ|onQto^CLZ>7xsTzmIb>k)NhxKuWZj7;VTQSh~eg@Lu?gve>X zckfPY$stqzA0-MW9T~8X2(=mqmFCxdzA9=>(Uthc(J~9tGV?L2wK}-e^>AIp*}1Yy z5m&LR+FI|@OQtA5J zK3*#8P!Buhsj{;27cyepuQ$rWvhoRTqR#G}Kv_;!7I1fCv;K<-pFWA0nXMI1q<|Xk zzk~oLtxenCN?cyMXlnL=dx`mGWakiVCPp=|R&OmL;}O^6rONnnx=Ll-G6q1jd@@68 zYwON41bfcILoF=qNlslY04kEWI`v$2b@h&=oTz?qt}>zsZP{{?BTMDWzfu7^AP=M; z&}HpnK3NsPWpo@|$y`bpqbG^(vhPiajUX)&BIo1tYK?HFeJob;u$#--%Gmf7wPQK~ z;P8s^5N}AArXmmlUb3Gy@8!k3>~9MTFuhEU3oG-xp1G7RvC!5U{qX7i6$SZ_lEJ|K z4ly)b+E4NE@4J7?jJs$3#RDu*GN{s{A>EJqET-53Y$O{yJ96GIdO}Z(>V>$!lz0&Z zS^t-)$+);MXolj0wjFRs_HuDisz;_WD#1TYyH20&E`rGIE%#R<14CnC4z1^b{?X=; z7TYs2%0%^}m&xO6YtNeaG^;+jUs96A1gDuu=keRxdJQPuW3>X&V0sk0fWQx(J7Om9 zK^#DpE{7W+HPI5NZ;ONdfH#K7wb0f!^C&m9r3m#bl&!fg7yd-(eR|E1*g2|-rdrw4-)k}*2y7+FWp z2(rW^IUjP-CB={bLF9r`5B~i^gNCEtc43n5m4oUkb&+9rTU&I9z91hbNzUT(WaUYV zB296BKO*hN1ke^9^u1rv>klWcQjiIyEoBG0qN`J*ZVk`7_qfW(s{E&h&pfl_Iti;5Vwy$6-3L~Wq zgyvwd#25fEMWh*zc<8=PBV`((-Lvq_?Kjt!`$W&Gph=7$+#8GH^K{@$;eTC=KXLHd zR7SHit^b>xIJ3C;6U%~qA%$nsTaVT35FgF-Z>%~e$=KNMzYMvs;I!o-EQL?rVhua5 zv*SX1g3NrCDm<5f&109wS3a@fIOhegFz1I)0k_1AjKUIP{c7G{`x6)hWj((7d4yWs zU8$cEB2UE@m#*C0pDCwo$~Cp+KJj>{>AnIn8u#O(U| z@#Jk>#a{{W@lAjqutC-^HhRQm;a{TD3EW|jg6HXA?4;%QvF!~j&$D8Rvmo7bs`o_} zCoHEE^%q)TFV~7GVrd}$ueiEAQ*mC;E2u(GxjJ7S*=N^v-QW^99DTjBCVbq81M0)# z2cZ`OvaEBRbSWQoJ@@-1^`z3WmRAnHQ^~yjJ~-&KASv9mezGTBjzNM|#>JJES%llR zYik@77*=h94}<-ExsTfZlv6@e^XJ4b-?1D9lb@~3uNAIV6n|K@=_T(KFo@?v$>j%0g{8_!)-c(z)h%J_z8*xR10A$bjwu zD|J4WazVUVvnN04v=H3U-96D@{9RV_Xc71?dV2bb#lxB!0RV+wb;mI0xlDp6RwS8V=0=%i(x2SuUom}f^@x`XZKCw-pDoH_6;5Ust8Y`>ht z@$@f*L%ny_nwy(JS*8=&u7mM!An35ko*eD<(M^?QmEBRBcBI7Yes-_D&FlHi@FI45 zfiougN^NPhk@phbq?empsXPh8=t)Sg48$I)L0IrmkKtj&lvk7>-yd6>&4juE3!&&9 zyP4>k!*R3Y@sT=T;foc8;&yFNbsR78)CcsrEj_wjK^nixu}hwul+;@2d9kJvdbM?R zxyAJn_PrDb&*PvMk|8uxMpg6ZHCtx7T0)J;QWum+##FfLr?Y6RE6w*MsQw7q8$Vmf z6902BFjExGi=n!B+XA)MJz$6^Kx`YnokC65y)+#a4GDnmNyi!Yip{DuYgLn+hYT^l zen0Hyvz|o~g3g)+rt7SpC*52jzIO(UIV+ap%~WgthKK-njm#VrIEs`vsRm&CEfgU3qfSjf3e2rw_06{lE-Mz*zoTuD?CR@Zs9KHoHV}!+Bp)%O@ayDYKlavMjuxV4z&tbugkE zIytG0G+>)PbH2nANVz}+2W6k{LQXWo#0`eGW8Z$@Z4XuHK~P|@tduR1*YHZsVnJkc zb6$v*ktw?9)YNu)+5F;K;lhszE9%h^zc2I!r|d+J$T9VQE(8cGY!$t}dw9ae^XU8M zzKn6ZR+lvGbLS6eXt=x%nAss@jh)SY>_4YNa>B#8{F9BH6VLE+wqVRDJdUboS__E? z3Fj)8lbb_5_A|us$pUtFcek@x-EVQOCDB?%Vy%0b7-nqEP!=C0o5_UP+4GLk@D&nup~cVr?BL!)=)Yqqal z13pF8_#^Rm>B9Cy$6Aas_p@&dzapQeRZ5}KU2UT!T9`FgUS5iCQ5$p87v#L#h&TIo zz(Hc0d#Qc>`V%&Pk64U4d^$Fl)BW?q;#+|m4x?6e#9C`>YC8}8Rog0A!>f9e1%s^I z7D#v2fH8Gi`t-)S8HgFysF{QXz0@&aj|DZrvJ zv9QvB5quCS{VNW19f^E)H$YWL&&>RCFiX+2wzrfZSOSn^Ul8QV$;kn6tq-Jx*{@D# zdwT+RcXzO;fEJAfmPVQ}kWXLPbMW%|&&}xp`45js2Wb}{A0H|C9mAWi#l;5>zR;(i zd0wJn9Zi1UPqn`5FOfBHQxVnHE=!n*Ysw*EeYu{E{oP7XWM15*YOfFmb=YXdtU$p4JaA*Sw4<*j87#N^hWB-De}yBUg* z06+f*HDi!OgPv#>Kjjz|)DEg{iHIqy08?+XQ93d&t9y!v1oYnQYF?oPpmGjd)4`xz zn2k3psLVu`2bQ|=b*P}PW9`~Ygg|9*rDb!f8UXS4@$n=A$FtsXRij8-6El|%=O?hM zD8EjuDZhZ<;428co3N@UZ*6S_$K^X^*$y>jAe%x9YJCsXg4_85C5R`3=G#LhGLBai zJOihy9WyE_?m}e&h?hhG76;zV4-DS5c@oy05NW?7Pg>o}-PnbCJIbr$cC{L4?}Rpg zRFC+K?b$}dEy`et9q^aD9#_Bl+w1pwA0My35e|Y!nNO5~ZWhZJPByDgLHw4`vHyXl z+)fC2hZda8T)U4q*(X$OeG~I%r={kL+xgMB%WCgxl*G0Y*SwCQ0}bpC$X5UO$;9|(g}Z3zO&>| zy#s0~ZQFWpG@Le1VM?djcbOAb)f?PE#~?iN==V1`fWf^gQr`|xa081$9yl-UH?ua} zQ4viBNc)Jcr=X6@3N6w=ac!UbrX6EQ0>Wi`955m&QGY_1KHf}g#yZcFSkFesx;^p_ zvAm?*01wlpxZyz@((ZmV<%bxi4&8J`4Np(cb0Q(UfP1%Zz5;#bj``zz?mufzSMxw4 zUa`|g8rgQ{CNyc&P7WgBAfZb@CU=sy+|U$+PG~@u#w0`Pa1^K|tzHZ<$xOKWz! z7?1a}g@a#Y%QL&ygSB0ssi#CdRQF<7cP?C#B0T!Kpy#E4WzCL6&G~M3?0EvPE?{Xw zCJ?j=Pv@|uh zn=n{g3R$_|L2lXveS%cZVVE^`Z2)CjsHb+EdQ0SSI+)Uwy8VMgQO;6_!kqH_9;}B1 z`|9P&W$V*FistpJ52Ltvnw3?!>fh5l++a9#)R~TN9o$`vyuG{xwK^_>WLuuX4*^k_ zZWZe~DPm7_xp;f5-;;~>HG6150}S_4Z95mKi|8F2*!7!3*BfvCZN*NwX^pz=p%M!`oKy9MDDgC ztJ{HYMGxNTneYEL-QvRs`B_Tp0y4JKKtzK>nM(re%}LS<3(n@^ zUvTv!AOW$O4ps#?9z^kxORc{|#?G0*>7NX6Kdt3; z2VV(O7Lr{z1L}Jd6UiV=!GMxO1jG(t#eC@5;e^$~xLIMVSz&F=khw9}!0(ILwO?9R zt=>37ck3_DX09kwj$7#N)NTP48bZbwVXtLwvZ(}%?~=}=5_}$5GwCaj%X4RKSZT3| zorvP4qSt7)c6LCM1Ydq|xy(-SAOYgc>Ew5;nmMO<3BiP)(1&*m?d(f?v{N2>`q!q3 zbUz%m5*2kCe=uB@CqR^i$jRRymr%`Pv`1zjey2_1tr> zhu(&FyKN!#^`?G0?ZZH0*He?pZAVYPMB12Qd2F18<5rX}cfa=gJHpV9+O`r(mlK&! zf86a!P^>c}>MbclEZd#MZU{@pvB?^$V~^d``<8cM6Yh*+^ASZpH^fFOIG-fZF3X`veUo%YZLhyhWM0K-R_Id0?GR=C<%sQ4YD1?*tBJ+~`79&!)I0l#xB zo=LBQ0+vKxn`?-h?kuz;^bu~0)Rf5gl5&I`)<=EodrPfhdfhhdQi41?mVf`AoAaY= z1E)Rhab{7oKOoC%|K~Q8zJk&@Nf{qaD(@Cv{k<5xGlhDq86l`)0fp?}h!k8U7U+@J z3aVK=WjXCbWn=X-tatTSa&eebKc z>K-#RBGmlL$*$PYYs%?hSj`4DBO*xunI>xeV4o7qa|QgbGNX9<54y#r`C89Co^DHW z*ZTxdw^A-Y9MUB>r1?=a?af?rPdlwL9BXQ_g|mIqz(@d}bI^biOT&w&VQN-%#CpZC zQr0=Am#ATMYb}6C$Dd9SDQ9K%jKXO)BD4AF#L;GJwBt;Yy;gm1EU+$ zEwrugz2*baYt~}40A)dA8u-Q~d3zR8s39ia1-X)6yhpz$c?Os2a<-@IFHhH`M-VYe zVdsJ>D9ZEmnLMUI4eQdlX%-MdDNLbYt?jx|>I8yKgaR8hBcP_8gW+ibS1_H)u6uPl z66;o@>|`|@6E2M%*%?*xVWZ(4O82xd8FqX5WLI~0Gr8mB{j*BYn}6DuRR#?OaG;YO zXB)AcjP|@_2rCj%Y$|-xsjv;(pZSce$WZBM+LaYZ_wBx`^S-}HF?1&{2W_dGrhb_x z#0n_ok2fn&SnH0d)*W02Uh9CTb~+q;v(w7z!EOkq>3FQq!*_am`uM=}tlYC1QTi^P za36{u$U}uMc`rHi=7pd#;Bjw)Wz{WK2ujSkwFWl0xVW5lT8VZVaXi>OCelRx44(1E zM6hhtEAh;@*`4n;dZR`5ve0PeUbQLZ@#()I)Q3VnuxFK-(G&==AW-&}RV$w}Z^%NB zj7X=q+eocu-M3tI&krHoXV#j3S`s0Ugnc!0b*ejRGhs1+ysHF!CrK*+EO)n^wtKr- z7k*QoBe4qZ-$sBT#brUkCRG;>#%kkJ*Pob(qLeX@>g?|0)tdH(2H&@y7aOoRfWDFF zvN8s(%5|L93Y64`Q{nyE|ym?lDgU@!2lkl!zHCygXS>X@|reY4&2v zvp!ujkMeTcPnz5jq6UU2rbk6xJw2@pp~A^Sb0=WfF9Z8tLxeT@I`pN|qio%ucQ^fn z&-KoWq5K#wsxwcTYQ_=%__EsJ>+$*80Y9tSyr0S2w_Ka`SC@qC2=GFBq^~W8iuE-V z{X5ME(1n$^G)n8}z&Ui^QySx}jrM2Y;2br#$*j!ov%(?Buu19#09AuiTE(|wE4mg0 zx11Huca@|=T2Ot*%;jaCBQ-eIpG_Sbf>Q&Z;WBo01n9ie=}LAaf7R7m{UzfP3yTe% zW-ytE*FxEZ^}gBVQmcYt$Kv0kLc#hnTJ|g+xI|*(L1>H&4Y8=?FF(Z5#sdfHBOAX3wi{3wvp_dj31JfIS4aQI---X zZ&;z;DML>Xn34v?>&LsX^<-D&SrKFbSq~B0ZNMNKq<@HN#x?NfBeZX5?!3^{jOS$F zeY-l8%O8o)oXPd++a5J#<5~Tc`?~A42(M*5`BD}C&45}Sed&Xf86?3dtf+Q=_Xz>jzfBorPZ6>M3KJtDEXk$qF|R#%9l9!hp=**R4e+2<~q^@;;k z?TdrLh0j-ep4P)^b=S@gMqYOxZghLdOE&4%z#8|Nrd1VXHrzuOWW3Xbi+Wd9HqsJ< z*&Oq3u>SdvPniiH(Md?!7MFgD)W3Qv=y1C4E1-5f$@hbW_a<9(OfMauJ|ka@#+1q{ z?$?nYi|?fm&rinM&B(=_eeQCtw=OM6BsY#yKYpZcG7{l)>tidEc6gEXNR+a()rhFX zW&xhv$VAFLmuD`JL4gvnad{5}B@4$p3jjXzMD(trs28(} z$$5BqC{cI)W;!Q^c;#uo!L^S3BTx47Nm~q$lWyH>VI-Y*UO)#SNiwb0B_qBQVN|coSmI_ z3_Xe-Xp5P4^z^l>R+DS>7y(?s#>4BM_6`{{Q7}j+79;$6Wb9}N-RI)s6OKSa0_N`) zdrJF8c1qV~&)+{1C;I$3BHruTKmm>H!IpaO(EA5@M0XPTGz*vc>pM8~y<2TfoV(Cx zD1EZBDrR@+LxQDPdfoYsUeZrNh_E|NL_qK)qrV0J31f65Jy+fA_jr?F0;b)L_dQ1*PJsVdoK71qbmQ9 zTAE3v5|U@d=g-6T-2%2mVNOo2Ga*33>5EQwxY`$agH?eB(AR)~? zzd83@YR8V*Ua#?W;fP%i1Y0+d;-4ifIVwvZ^D)y|iLfH4mjW=jT(=^0^epSh1w3NFIS#R|Me(k+2@PVQ+H&~Gftf_9> zhxOIfFP*9X6Sx}e15SWZq9A+UHay2O!nuB_eI1O z@%We&^QoWTeVh3H?H5%M$y|#!PvTbPY&5mj*@qd@Rj)HM{2HQc;!dG6Ea|#4XQ6g4WEACt&Hf<))7J4* zM`cY-lFu+y!q}VLHizpPAiXE^`m(27Kt(`6aJ-elENo(BZ@x`JLr3TMg%TNQn2@!0 zgj$L0IX8WLTSnUl)JaaWJD&v)qMVAz7YBG|QcIJ@b#}67?oLRDVzsA^{W;|{=^am% zbQ~}$-2e63Bqf7JTK{3l?wRPBdtk%RvxL{_QWZzR6i0RaYdy7v$-jhn8O37^qhewl z4x*6!BZ_93GC}Xmo#1(G>IF+icIk$RO;RVae~I94qH_S;RZLu=x*P(3q!zcGkW1U4 zU-HZ^t4gMnGo(J7@>e@bSbr$2UEn*|ixB(7Cn}>AE35nTn<6oy8T% zu1-xwil7mRF;rGjL6qr2+Xv7PZ^srvBa>kX1>BmP{$CFzs4IMx5@??e-BEjiuer9? zKmOC}m+uW491>J04YnMQjEVG4z(9YFOE4MfE~L%pP9o|p(dk3o&0&y&qlCsIKfZh0 zB9&%RYBKG0+Xn+Ldx_i!FU!j*WJyQE4T*KiN!ze*@Qq(Mrdu}DTG>||4KGitDVTYJ zX+Du;Mwwq`IBwE@voZEva(IurVn$qHU~v4fqyonl5x_}FON)Sjphk!F65@$jfIm=? z%%QcsXMM=$HRPKsY&A|h`f_n>X=Zej*Bdk3p_1k~&>v z2eKZ3P^y=i6QcvvWAr2L8>>udj@Cy1ub-vj30=eCxEA3;l6^3UV1AnI`UKIC5Rey4 zp&x(}0AKcarp(@JPZjYFOGybWEDW}v1PlSu^+E%V4SFUDiF3#6>sNA+PJ(1d@&avi z?I5@AD(S>S>8kW7TJ5>(X~`9@>#l*EdU584f)dFB!&Y;YC}s+WIN>7F(=!9Hc}o)! zeKcA6k}-l?@p)-;#e@>Qi2?iA+V}*?y(d) z-xfuh_tUDh6)^sJZy4i?v58rBllXJj$g+3(6=NN9&=X?T{%4z(tX1Em24~;i^HRn!wl|FF|?WA?fBa&@A&6ujrxWc)%wTVbD%vwqO_W- zA}nS)^OQh-8xVM?&Y&??Y#4M^W}WpAN%@%!@)wD|(q+h1`IDMrHrgENrGHjFnK!q5IEa`3vdGNGL{vsRl z3(2={{}|6WyeI2fUl6VJa-BJ}*~**wyJx|s4L4juAtpT&)B3=OfW!LTz@VUu_MDs? zCjk1Z_qj5%{%&2$MWodDks7;YdF@Ku>#%b#In&T6-M+oRt%Sr8OpwbRae_V%BWHXtcndIg+4dc^|U(#4u zQ~%g9O&P%`e^5;|zWx#V>V7|T5~Sc%R>`rH9XvND=`qSQHVrc4%t)@5wGvc~mOxcmISf^lt%I8LbL&7<=65xVW9Mu|HXlDYQ?EO&#OT64v|npHWGI5abahM z#>I&bL&+4My$kTo&bm%nzkN-}^Tcr7c>#;;cot>oL#M||aC-#Jtxxex-?qBDMZL)^ z3?`LLeuaFYv)LjgKB_yNK3lcefB-z;I>w#jP%|?#cB^TDtIS$sX8{U?L?d@8OG86L zz0_1CwvS!YMQb8?2s*pox^*je=%9Ss1p~mUYe<-++;Rs!oAgN z%x1sx8zB=^DHk97SX@z|t}>EK+s%>eacVN`-;bcyH$0}K^z|;PP3KzzE^P`R(>q?% zy;{90w`M5q?Ci|VLuot+4wq3u>i@ua?5e1t5dhh9)s%7%tmp-?pDIV*o+28MuD8v?qcO`nOC4;i0ts6f8AuBO{~*XV;Dtn~9$neh1>PLhY)S40(1za-@W<<{7DwDKtjQf`a!#Vefk9j9*= z6WnqsLgZ^Bg`ul!s~gU~k6yQ74)4JFwc&U=UJMK%kOia>?{A0+-?87BZ#`ZXz9P1* zZk}tHZw)H4S&+E0IGVImXxR-g-d{WgK;$fprt_-IxIbM!<&BzD#0NgRrQ=Xg#9%x* z$Muh5$4#7@yXCSoq6%FVlihZuKT-&ed!9qqsyEly&^)=8(jVJjGE5u@^psFV(+NUHj zQOGqvzeYfvON|ES4pGo0T~|~r+n^58?Ad^lcF4l`n*q8__sS!zJF>1b$vS5to9@u> z(jU0fFEZ}{?+uoYd@x5Rd~yrWhCAG(OS6sMPJlt#K+@(LC3*Fm)pKS0=JMukjH|+G z(Mbh0MF=bJfuea)T<*=_0Rc401Aqs13stY~gmFwv3>1h_emClAf?^n`fv{N_pf#VW z;>slMgPo*e;xIM0<$3iOIx)-@COx*9|8Rv956x;sMZI(HnE%Sx7^Ns_#8At~X#a9@yPprdck2eIZ_c0+UtOvr>A8I)TA_k9}}!2?H3Rwkwv&#N$ zh=jPUrt?p0!=s`;q^GAlEPulHd1Qa?gh?(;LDb!)T%K~i{>bfWSgk_u&>S;?gZ5|U zkg+lve~nJbuaSlMaBAMN*OF3q+Pr|#@Lm~wY%h=4k_~8-v%k}WeNgnBW_?`RFv-kp?*6H$TbYhjqr_-)y`$aU~ z$j-v2xH8dLbxav`x#(wsfovzA-n=T-Y&NphW?_~5v~)zGXf-<>>pG)8b+&4=0j(l} z05hCx7rtZ%BpdsQK_S7NJL%B!NHvdeY;3HfqvHv*Zeh`>x$E#SxK+Oq=!*jM9|sQv z3oI1=@Q47I{Q+$A7o8_l_JhyvW=jiie7rB}3u=5+aLjwz%wVVyt3v;UsCT26N}lRp z&>@ymx7)OIK2kxb@ll_EHx?=Pu~n=g-=g2&^mn|}Oa;q|sz+{QVh?lq;%KHFz`1ks z?^;?~D((a&6fS@6*Fdrg@#`ngub>XVO6fxr{lJ(85e&pMErLgrDA0-w1r2)^Fm}DI zgxFX=1>uV!UEhAfqHmWkRh~abf<}ch2^b@xqU<*FVo-n#A>Of-8EcRzyXg@9`b0U7 z-O`Vpt0GgVVs`a;8lI341kjW`R~W6*C%W5md5 z_HB%axd9!#w1b1KU)FtW>yqId8WF|bJL`;!^Op`4A}Pzv5PTymtLQp+&9d&y5|fw< zUE(`2nDe~l-nqrisq`zcFw{K4a>4!Q3A%R19zqgwNr5j!dR8?_NQ>5hL%H~HtK$*s zO(P~|)A|}_NPGa#+7BV;t#)-xC6kXx?~1>_iv2fwmd{^zKd3l_5Vs9*{388U^n9&y z`RH2c)Mx{al3Vx+Am{tP3SN?`Y(*t!wwG>LUVPTR{ycjFnd)U>wSl3b9)Jo6&9`y} zTPYODBfJC<>h+RWkub>ly7d&aw6q9~PZ-QeNeL&M)Bt@yh!EnwBIqb&KKkwcB2oOX zY4GvegF}JRg6d6vD36>2bI!+fM|SiKssSfK7IwVb%~}NAX)`~6*7tah?hHP*da5#8 z?$dj4QfaBF_spaCA3efHkQUHKBkXHV;}&{tp{E*1-iN=i+;bDg_USZ0mmdDe-GS`i zqXSgkC1?P|&6UxS5LyK|AZ8{bJ^ej0_9};+65ghE9oeT037-o`jhKFLeJmhdX~|BR zueO-5UhQVJnM<&G?s>wo$kro?kA~Thq##7BRMww7+ew2J%QBaTMf&{weDC;}r@nEC zkn!7D>B!S-WS?8awrBu{GqpI;AzvlUV#fLVRW$f);y z3?cM-wK|lOMAGw1(M9JEzcJy+-~>gso9vfhd3l` zU8ADs*a??>wbnq$uMx8|wVbjr8bZGP_SI?tjdVv6$My0b2}DI7KNnJTuQ@Et`eO0w zs@|ZXL3$ABloEFP@G^&wFYa_rcVV`AYhLN+aNi1}8kHy=sZM%9V3y#mqq>)aYLE*Q zTp-AQkWE6ijp(o|EiDb-AO^YuiBJlx7jzshJuot^IK7w7WCk zq=mh0>_-WM-@o-1KN3sfkR0V#x98@I4HvyWKKyxKThI|(N(Oyjp|Z`qRR8#;qu$W@ z30s`p=v8?=dxywSPsI@}74r?gc{v#wbSSJUE-pssS;PpVJvJ+sl_RNPX2+Dn{lDNi zy<{0m;zMcw-ECawLb$nPbga#CcXdsVGa;O*TdL>Bhl?03DkH2CTBR^iMb8zyf@D7L zm{XTy17c}z(g9oBSAD&kN_z%`8WI~Pdtd79Z+(6C%=&Q>TfCf`Bz=Yv@a2uomaLfO4_F&u z&hB!V5>yuAKw9zF-PcVr=3zQm>60iBw1H_1rdqw^>@iM zyt@4f1-I=&yh$#^@?Lj3oV|lmTrAev6?UI0!Q0=3{HRVWGwVvVaOgfXqK1n5zuGFO zRKIrHE@6TMgH{?3fz8i{dY-?wN_v2c`}V^a(>0PV0-RV`13NbF)FXyxuki83y_$^; zyu82Q{C#C6wE$6s%Lu2I{N{3Cy129?QIi6Z!u#tO2-zi~ZJdD-FD~xN2O8uq|ItjE zox0$h`$OJQHuW-IIrz1T{$E@XH)>Qb7{E#7e~Yf`dBIhnU3m?1(5NVak9XOxfnePy zS`9hCU*CHI%k9ejRk0sUfNcP_`G7{jfOyw9O2f}I{S80tjiF3Zs#U(pGrR*UCbbnb ziH~>R;ZjTNADBR`fLdHnx>$OtiUBTfl2&at zN9#PAP&v_p<>h+1R&yQ&!{L4OYKk1Dn7&WGL60GSdv8HRrH#Jy%Ooc=%NqsSxcF+m zm@8}F13hC|0`5sl_|mHtO|+6i$;=8mqSdIsey0~QRZxRIuEuo=g;(=v2bi=$%?5bL z*wT0!Tu_9P`4|&XS<{LaDIJxKTL0719M!F^PO#AF_g(Mu9PDPyro;)waT9pV6rV66IFT&{&O%sy?w_CaE4K4n}irNhDG_FU` zT%+)!*RZKRJFJ-xo?wPk8Ny;7OlN=Q2!d~tYX5grafj-g4m33WySFrMn3|oP=FbNe zCo?dlUj=j$+Tm|n_u0GdNDK{K(svM%o3|E`6NHD@!l^!h8(NDAQ+zU)_aKZ875L*HZ~=0 zK8>ia+O=IKj7jkI2Pk61VXaiZc&Ib}U7OzY@7hNs?|#Vb6iu*7)wQ$JANsNuU+GnD^-g{pQ~wML zfk5=%=cAdJq?a@q;Jt&1Sx_J)IfYBToA1Y$$wb_=z8m~RUq7{y5k{dE_3xeM-_ZnI zl0AsL=|IogOG=8I{oShX*;b~hvr^Ky zh(&`O^5H|BAP;=wDuHy2VP&Dhdl<$l4ZBCyC&kAVY4?97G#lYtUA%O6zxO9H{^5Hv zuba1N>5itH;gu9)+5S#pVWNC$m8O9$YOJC9*Yu+gGzS|e2R^gAns0$vLkoh}*H{eAyQ&5A1#n)r? z72P@ez))Ig)SEXje>%>fTUd+oS!MP4E9FMSj|;ai!MGhz|9gz1WjVdBiRGPkiqM-L zsA!LW7~!seQ{V4ze;U(z!cV5Jk9qs1<*4~P+k{LS#aujDV(6hIHe~Ym$Kk1*RBx+W zi$+LqIvl&rb4$a54iZPb?eb_gEo!)=>_#znl$}zSS=6lW*raFM0z5XH^6#q>q@v=H zkX-ALiq5~4{pjZp{M(H_A5eFUiu@a%I^Ga8&@eD5De>o_Jx&)W!!wqaL61rUON=9F zYkQOgllb4uoxWzt=pYL3iGd@1BR1_#ydMVWH4Gv+kS zhoI2t2Vc;$%pWjF8yKT1cu7k==OKm)vUHCC5yUK;CbUz_XpmvQ<@R9;dLk(s)KT%= z<+VK#eAUNH8r&8sIpsHjaXnP8UpF8fPJ%tBLi#%jijo!D=?ao&X2#D&W5gf$b`y4V zJ)e%+@Qo-^|;v_+rX9{ zOB`JY>m_IQBqQ8a+9zvB~@xOogcQYX_ wrf2!Pdl46l*}VHZK!~gVYb*S_^|7Zd+v%ZGjz$igt%D>jE-zN_^ws Lee, K.H.\*, Denovellis, E.L.\*, Ly, R., Magland, J., Soules, J., Comrie, A.E., Gramling, D.P., Guidera, J.A., Nevers, R., Adenekan, P., Brozdowski, C., Bray, S., Monroe, E., Bak, J.H., Coulter, M.E., Sun, X., Tritt, A., Rübel, O., Nguyen, T., Yatsenko, D., Chu, J., Kemere, C., Garcia, S., Buccino, A., Frank, L.M., 2024. Spyglass: a data analysis framework for reproducible and shareable neuroscience research. bioRxiv. [10.1101/2024.01.25.577295](https://doi.org/10.1101/2024.01.25.577295 ). +> Lee, K.H.\*, Denovellis, E.L.\*, Ly, R., Magland, J., Soules, J., Comrie, +> A.E., Gramling, D.P., Guidera, J.A., Nevers, R., Adenekan, P., Brozdowski, C., +> Bray, S., Monroe, E., Bak, J.H., Coulter, M.E., Sun, X., Tritt, A., Rübel, O., +> Nguyen, T., Yatsenko, D., Chu, J., Kemere, C., Garcia, S., Buccino, A., Frank, +> L.M., 2024. Spyglass: a data analysis framework for reproducible and shareable +> neuroscience research. bioRxiv. +> [10.1101/2024.01.25.577295](https://doi.org/10.1101/2024.01.25.577295). *\* Equal contribution* See paper related code [here](https://github.com/LorenFrankLab/spyglass-paper). - - diff --git a/docs/src/installation.md b/docs/src/installation.md deleted file mode 100644 index d588d2daf..000000000 --- a/docs/src/installation.md +++ /dev/null @@ -1,127 +0,0 @@ -# Installation - -_Note:_ Developers, or those who wish to add features or otherwise work on the -codebase should follow the same steps below, but install Spyglass as editable -with the `-e` flag: `pip install -e /path/to/spyglass` - -## Basic Installation - -For basic installation steps, see the -[Setup notebook](./notebooks/00_Setup.ipynb) 'local installation' section, -including python, mamba (for managing a -[virtual environment](https://en.wikipedia.org/wiki/Virtual_environment_software)), -VSCode, Jupyter, and git. This notebook also covers -[database access](#database-access). - -## Additional Packages - -Some pipelines require installation of additional packages. - -The spike sorting pipeline relies on `spikeinterface` and optionally -`mountainsort4`. - -```bash -pip install spikeinterface[full,widgets] -pip install mountainsort4 -``` - -__WARNING:__ If you are on an M1 Mac, you need to install `pyfftw` via `conda` -BEFORE installing `ghostipy`: - -```bash -conda install -c conda-forge pyfftw -``` - -The LFP pipeline uses `ghostipy`: - -```bash -pip install ghostipy -``` - -## Database access - -For basic installation steps, see the -[Setup notebook](./notebooks/00_Setup.ipynb) 'database connection' section. For -additional details, see the -[DataJoint documentation](https://datajoint.com/docs/elements/user-guide/#relational-databases). - -### Config - -#### Via File (Recommended) - -A `dj_local_conf.json` file in your current directory when launching python can -hold all the specifics needed to connect to a database. This can include -different directories for different pipelines. If only the Spyglass `base` is -specified, other subfolder names are assumed from defaults. See -`dj_local_conf_example.json` for the full set of options. This example can be -copied and saved as `dj_local_conf.json` to set the configuration for a given -folder. Alternatively, it can be saved as `.datajoint_config.json` in a user's -home directory to be accessed globally. See -[DataJoint docs](https://datajoint.com/docs/core/datajoint-python/0.14/quick-start/#connection) -for more details. - -Note that raw and analysis folder locations should be specified under both -`stores` and `custom` sections of the config file. The `stores` section is used -by DataJoint to store the location of files referenced in database, while the -`custom` section is used by Spyglass. Spyglass will check that these sections -match on startup. - -#### Via Environment Variables - -Older versions of Spyglass relied exclusively on environment for config. If -`spyglass_dirs` is not found in the config file, Spyglass will look for -environment variables. These can be set either once in a terminal session, or -permanently in a unix settings file (e.g., `.bashrc` or `.bash_profile`) in your -home directory. - -```bash -export SPYGLASS_BASE_DIR="/stelmo/nwb" -export SPYGLASS_RECORDING_DIR="$SPYGLASS_BASE_DIR/recording" -export SPYGLASS_SORTING_DIR="$SPYGLASS_BASE_DIR/sorting" -export SPYGLASS_VIDEO_DIR="$SPYGLASS_BASE_DIR/video" -export SPYGLASS_WAVEFORMS_DIR="$SPYGLASS_BASE_DIR/waveforms" -export SPYGLASS_TEMP_DIR="$SPYGLASS_BASE_DIR/tmp" -export DJ_SUPPORT_FILEPATH_MANAGEMENT="TRUE" -``` - -To load variables from a `.bashrc` file, run `source ~/.bashrc` in a terminal. - -#### Temporary directory - -A temporary directory will speed up spike sorting. If unspecified by either -method above, it will be assumed as a `tmp` subfolder relative to the base path. -Be sure it has enough free space (ideally at least 500GB). - -#### Subfolders - -If subfolders do not exist, they will be created automatically. If unspecified -by either method above, they will be assumed as `recording`, `sorting`, `video`, -etc. subfolders relative to the base path. - -## File manager - -[`kachery-cloud`](https://github.com/flatironinstitute/kachery-cloud) is a file -manager for Frank Lab collaborators who do not have access to the lab's -production database. - -To customize `kachery` file paths, see `dj_local_conf_example.json` or set the -following variables in your unix settings file (e.g., `.bashrc`). If -unspecified, the defaults below are assumed. - -```bash -export KACHERY_CLOUD_DIR="$SPYGLASS_BASE_DIR/.kachery-cloud" -export KACHERY_TEMP_DIR="$SPYGLASS_BASE_DIR/tmp" -``` - -Be sure to load these with `source ~/.bashrc` to persist changes. - -## Test connection - -Finally, open up a python console (e.g., run `ipython` from terminal) and import -`spyglass` to check that the installation has worked. - -```python -from spyglass.common import Nwbfile - -Nwbfile() -``` diff --git a/docs/src/misc/common_errs.md b/docs/src/misc/common_errs.md deleted file mode 100644 index 34143b0f5..000000000 --- a/docs/src/misc/common_errs.md +++ /dev/null @@ -1,111 +0,0 @@ -# Common Errors - -## Debug Mode - -To enter into debug mode, you can add the following line to your code ... - -```python -__import__("pdb").set_trace() -``` - -This will set a breakpoint in your code at that line. When you run your code, it -will pause at that line and you can explore the variables in the current frame. -Commands in this mode include ... - -- `u` and `d` to move up and down the stack -- `l` to list the code around the current line -- `q` to quit the debugger -- `c` to continue running the code -- `h` for help, which will list all the commands - -`ipython` and jupyter notebooks can launch a debugger automatically at the last -error by running `%debug`. - -## Integrity - -```console -IntegrityError: Cannot add or update a child row: a foreign key constraint fails (`schema`.`_table`, CONSTRAINT `_table_ibfk_1` FOREIGN KEY (`parent_field`) REFERENCES `other_schema`.`parent_name` (`parent_field`) ON DELETE RESTRICT ON UPDATE CASCADE) -``` - -`IntegrityError` during `insert` means that some part of the key you're -inserting doesn't exist in the parent of the table you're inserting into. You -can explore which that may be by doing the following... - -```python -my_key = dict(value=key) # whatever you're inserting -MyTable.insert1(my_key) # error here -parents = MyTable.parents(as_objects=True) # get the parents as FreeTables -for parent in parents: # iterate through the parents, with only relevant fields - parent_key = {k: v for k, v in my_key.items() if k in parent.heading.names} - print(parent & parent_key) # restricted parent -``` - -If any of the printed tables are empty, you know you need to insert into that -table (or another ancestor up the pipeline) first. This code will not work if -there are aliases in the table (i.e., `proj` in the definition). In that case, -you'll need to modify your `parent_key` to reflect the renaming. - -The error message itself will tell you which table is the limiting parent. After -`REFERENCES` in the error message, you'll see the parent table and the column -that is causing the error. - -## Permission - -```console -('Insufficient privileges.', "INSERT command denied to user 'username'@'127.0.0.1' for table '_table_name'", 'INSERT INTO `schema_name`.`table_name`(`field1`,`field2`) VALUES (%s,%s)') -``` - -This is a MySQL error that means that either ... - -- You don't have access to the command you're trying to run (e.g., `INSERT`) -- You don't have access to this command on the schema you're trying to run it on - -To see what permissions you have, you can run the following ... - -```python -dj.conn().query("SHOW GRANTS FOR CURRENT_USER();").fetchall() -``` - -If you think you should have access to the command, you contact your database -administrator (e.g., Chris in the Frank Lab). Please share the output of the -above command with them. - -## Type - -```console -TypeError: example_function() got an unexpected keyword argument 'this_arg' -``` - -This means that you're calling a function with an argument that it doesn't -expect (e.g., `example_function(this_arg=5)`). You can check the function's -accepted arguments by running `help(example_function)`. - -```console -TypeError: 'NoneType' object is not iterable -``` - -This means that some function is trying to do something with an object of an -unexpected type. For example, if might by running `for item in variable: ...` -when `variable` is `None`. You can check the type of the variable by going into -debug mode and running `type(variable)`. - -## KeyError - -```console -KeyError: 'field_name' -``` - -This means that you're trying to access a key in a dictionary that doesn't -exist. You can check the keys of the dictionary by running `variable.keys()`. If -this is in your custom code, you can get a key and supply a default value if it -doesn't exist by running `variable.get('field_name', default_value)`. - -## DataJoint - -```console -DataJointError("Attempt to delete part table {part} before deleting from its master {master} first.") -``` - -This means that DataJoint's delete process found a part table with a foreign key -reference to the data you're trying to delete. You need to find the master table -listed and delete from that table first. diff --git a/docs/src/misc/index.md b/docs/src/misc/index.md deleted file mode 100644 index 51ef0007d..000000000 --- a/docs/src/misc/index.md +++ /dev/null @@ -1,12 +0,0 @@ -# Misc Docs - -This folder contains miscellaneous supporting files documentation. - -- [Common Errors](./common_errs.md) -- [Database Management](./database_management.md) -- [Export](./export.md) -- [Insert Data](./insert_data.md) -- [Merge Tables](./merge_tables.md) -- [Mixin Class](./mixin.md) -- [Session Groups](./session_groups.md) -- [figurl Views](./figurl_views.md) diff --git a/docs/src/misc/insert_data.md b/docs/src/misc/insert_data.md deleted file mode 100644 index 1706c7f73..000000000 --- a/docs/src/misc/insert_data.md +++ /dev/null @@ -1,101 +0,0 @@ -# How to insert data into `spyglass` - -In `spyglass`, every table corresponds to an object. An experimental session is -defined as a collection of such objects. When an NWB file is ingested into -`spyglass`, the information about these objects is first read and inserted into -tables in the `common` module (e.g. `Institution`, `Lab`, `Electrode`, etc). -However, not every NWB file has all the information required by `spyglass`. For -example, many NWB files do not contain any information about the -`DataAcquisitionDevice` or `Probe` because NWB does not yet have an official -standard for specifying them. In addition, one might find that the information -contained in the NWB file is incorrect and would like to modify it before -inserting it into `spyglass` without having to go through the time-consuming -process of re-generating the NWB file. For these cases, we provide an -alternative approach to inserting data to `spyglass`. - -This alternate approach consists of two steps. First, the user must identify -entries that they would like to add to the `spyglass` database that exist -independently of any particular NWB file. For example, information about a -particular probe is stored in the `ProbeType` and `Probe` tables of -`spyglass.common`. The user can either: - -1. create these entries programmatically using DataJoint `insert` commands, for - example: - - ```python - sgc.ProbeType.insert1( - { - "probe_type": "128c-4s6mm6cm-15um-26um-sl", - "probe_description": "A Livermore flexible probe with 128 channels, 4 shanks, 6 mm shank length, 6 cm ribbon length. 15 um contact diameter, 26 um center-to-center distance (pitch), single-line configuration.", - "manufacturer": "Lawrence Livermore National Lab", - "num_shanks": 4, - }, - skip_duplicates=True, - ) - ``` - -2. define these entries in a special YAML file called `entries.yaml` that is - processed when `spyglass` is imported. One can think of `entries.yaml` as a - place to define information that the database should come pre-equipped - prior to ingesting any NWB files. The `entries.yaml` file should be placed - in the `spyglass` base directory. An example can be found in - `examples/config_yaml/entries.yaml`. It has the following structure: - - ```yaml - TableName: - - TableEntry1Field1: Value - - TableEntry1Field2: - - TableEntry2Field1: Value - - TableEntry2Field2: Value - ``` - - For example, - - ```yaml - ProbeType: - - probe_type: 128c-4s6mm6cm-15um-26um-sl - probe_description: A Livermore flexible probe with 128 channels, 4 shanks, 6 mm shank - length, 6 cm ribbon length. 15 um contact diameter, 26 um center-to-center distance - (pitch), single-line configuration. - manufacturer: Lawrence Livermore National Lab - num_shanks: 4 - ``` - -Using a YAML file over programmatically creating these entries in a notebook or -script has the advantages that the YAML file maintains a record of what entries -have been added that is easy to access, and the file is portable and can be -shared alongside an NWB file or set of NWB files from a given experiment. - -Next, the user must associate the NWB file with entries defined in the database. -This is done by cresqating a _configuration file_, which must: be in the same -directory as the NWB file that it configures be in YAML format have the -following naming convention: `_spyglass_config.yaml`. - -Users can programmatically generate this configuration file. It is then read by -spyglass when calling `insert_session` on the associated NWB file. - -An example of this can be found at -`examples/config_yaml/​​sub-AppleBottom_ses-AppleBottom-DY20-g3_behavior+ecephys_spyglass_config.yaml`. -This file is associated with the NWB file -`sub-AppleBottom_ses-AppleBottom-DY20-g3_behavior+ecephys.nwb`. - -This is the general format for the config entry: - -```yaml -TableName: - - primary_key1: value1 -``` - -For example: - -```yaml -DataAcquisitionDevice: - - data_acquisition_device_name: Neuropixels Recording Device -``` - -In this example, the NWB file that corresponds to this config YAML will become -associated with the DataAcquisitionDevice with primary key -data_acquisition_device_name: Neuropixels Recording Device. This entry must -exist. diff --git a/notebooks/00_Setup.ipynb b/notebooks/00_Setup.ipynb index 9bfeff14b..578c6c179 100644 --- a/notebooks/00_Setup.ipynb +++ b/notebooks/00_Setup.ipynb @@ -47,8 +47,18 @@ "id": "aa6bddcb", "metadata": {}, "source": [ - "JupyterHub users can skip this step. Frank Lab members should first follow\n", - "'rec to nwb overview' steps on Google Drive to set up an ssh connection.\n", + "Skip this step if you're ...\n", + "\n", + "1. Running the tutorials on [JupyterHub](https://spyglass.hhmi.2i2c.cloud/) \n", + "2. A member of the Frank Lab members. Instead, ssh to a shared machine.\n" + ] + }, + { + "cell_type": "markdown", + "id": "520ea38f", + "metadata": {}, + "source": [ + "### Tools\n", "\n", "For local use, download and install ...\n", "\n", @@ -68,15 +78,147 @@ "4. [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) for\n", " downloading the repository, including notebooks.\n", "\n", + "See [this DataJoint guide](https://datajoint.com/docs/elements/user-guide/) for \n", + "additional details on each of these programs and the role they play in using the \n", + "pipeline.\n", + "\n", + "
Suggested VSCode settings\n", + "\n", + "Within the Spyglass repository, there is a `.vscode` folder with `json` files\n", + "that specify limited settings and extensions intended for developers. The average\n", + "user may benefit from the following fuller sets.\n", + "\n", + "We recommending these incrementally so you get a feel for what each one does\n", + "before adding the next, and to avoid being overwhelmed by changes.\n", + "\n", + "1. `extensions.json`. By updating this file, you'll add to the 'Recommended'\n", + "section of the extensions tab. Each extension page will provide more information \n", + "on the uses and benefits. Some relevant concepts include...\n", + " - Linting: Warning of potential problems\n", + " - Formatting: Auto-adjusting optional coding styles to align across users\n", + " - Debugger: Progressive running of code. Please search for tutorials\n", + " - Autocompletion: Prompting for potential options when coding\n", + "\n", + "```json\n", + "{\n", + " \"recommendations\": [\n", + " // Python Extensions\n", + " \"charliermarsh.ruff\", // Fast linter\n", + " \"donjayamanne.python-environment-manager\", // Environment manager\n", + " \"kevinrose.vsc-python-indent\", // Auto-indent when coding\n", + " \"ms-python.black-formatter\", // Opinionated formatting\n", + " \"ms-python.debugpy\", // Debugger\n", + " \"ms-python.isort\", // Opinionated formatter for imports\n", + " \"ms-python.pylint\", // Linter to support a DataJoint-specific linter\n", + " \"ms-python.python\", // Language support for Python\n", + " \"ms-python.vscode-pylance\", // Additional language support\n", + " // Jupyter\n", + " \"ms-toolsai.jupyter\", // Run notebooks in VSCode\n", + " \"ms-toolsai.jupyter-keymap\", // Allow key-bindings\n", + " \"ms-toolsai.jupyter-renderers\", // Display images\n", + " // Autocompletion/Markdown\n", + " \"github.copilot\", // Auto-suggest with copilot LLM\n", + " \"github.copilot-chat\", // Add chat-box for questions to LLM\n", + " \"visualstudioexptteam.intellicode-api-usage-examples\", // Prompt package options\n", + " \"visualstudioexptteam.vscodeintellicode\", // Prompt Python-general options\n", + " \"davidanson.vscode-markdownlint\", // Linter for markdown\n", + " \"streetsidesoftware.code-spell-checker\", // Spell checker\n", + " // SSH - Work on remote servers - Required for Frank Lab members\n", + " \"ms-vscode-remote.remote-ssh\",\n", + " \"ms-vscode-remote.remote-ssh-edit\",\n", + " \"ms-vscode.remote-explorer\",\n", + " ],\n", + " \"unwantedRecommendations\": []\n", + "}\n", + "```\n", + "\n", + "2. `settings.json`. These can be places just in Spyglass, or added to your user\n", + "settings file. Search settings in the command panel (cmd/ctrl+shift+P) to open\n", + "this file directly.\n", + "\n", + "```json\n", + "{\n", + " // GENERAL\n", + " \"editor.insertSpaces\": true, // tab -> spaces\n", + " \"editor.rulers\": [ 80 ], // vertical line at 80\n", + " \"editor.stickyScroll.enabled\": true, // Show scope at top\n", + " \"files.associations\": { \"*.json\": \"jsonc\" }, // Load JSON with comments\n", + " \"files.autoSave\": \"onFocusChange\", // Save on focus change\n", + " \"files.exclude\": { // Hide these in the file viewer\n", + " \"**/__pycache*\": true, // Add others with wildcards\n", + " \"**/.ipynb_ch*\": true, \n", + " },\n", + " \"files.trimTrailingWhitespace\": true, // Remove extra spaces in lines\n", + " \"git.enabled\": true, // use git \n", + " \"workbench.editorAssociations\": { // open file extension as given type\n", + " \"*.ipynb\": \"jupyter-notebook\", \n", + " },\n", + " // PYTHON\n", + " \"editor.defaultFormatter\": \"ms-python.black-formatter\", // use black\n", + " \"[python]\": {\n", + " \"editor.formatOnSave\": true,\n", + " \"editor.defaultFormatter\": \"ms-python.black-formatter\",\n", + " \"editor.codeActionsOnSave\": { \"source.organizeImports\": \"always\"},\n", + " },\n", + " \"python.analysis.autoImportCompletions\": false, // Disable auto-import\n", + " \"python.languageServer\": \"Pylance\", // Use Pylance\n", + " \"pylint.args\": [ // DataJoint linter optional\n", + " // \"--load-plugins=datajoint_linter\", // Requires pip installing\n", + " // \"--permit-dj-filepath=y\", // Specific to datajoint_linter\n", + " \"--disable=E0401,E0102,W0621,W0401,W0611,W0614\"\n", + " ],\n", + " // NOTEBOOKS\n", + " \"jupyter.askForKernelRestart\": false, // Prevent dialog box on restart\n", + " \"jupyter.widgetScriptSources\": [\"jsdelivr.com\", \"unpkg.com\"], // IPyWidgets\n", + " \"notebook.output.textLineLimit\": 15, // Limit output\n", + " \"notebook.lineNumbers\": \"on\", // Number lines in cells\n", + " \"notebook.formatOnSave.enabled\": true, // blackify cells\n", + " // AUTOCOMPLETION\n", + " \"editor.tabCompletion\": \"on\", // tab over suggestions\n", + " \"github.copilot.editor.enableAutoCompletions\": true, // Copilot\n", + " \"cSpell.enabled\": true, // Spellcheck\n", + " \"cSpell.language\": \"en,en-US,companies,python,python-common\",\n", + " \"cSpell.maxDuplicateProblems\": 2, // Only mention a problem twice\n", + " \"cSpell.spellCheckDelayMs\": 500, // Wait 0.5s after save\n", + " \"cSpell.userWords\": [ \"datajoint\", \"longblob\", ], // Add words\n", + " \"cSpell.enableFiletypes\": [ \n", + " \"!json\", \"markdown\", \"yaml\", \"python\" // disable (!) json, check others\n", + " ],\n", + " \"cSpell.logLevel\": \"Warning\", // Only show warnings, can turn off\n", + " // MARKDOWN\n", + " \"[markdown]\": { // Use linter and format on save\n", + " \"editor.defaultFormatter\": \"DavidAnson.vscode-markdownlint\",\n", + " \"editor.formatOnSave\": true,\n", + " },\n", + " \"editor.codeActionsOnSave\": { \"source.fixAll.markdownlint\": \"explicit\" },\n", + " \"rewrap.reformat\": true, // allows context-aware rewrapping\n", + " \"rewrap.wrappingColumn\": 80, // Align with Black formatter\n", + "}\n", + "```\n", + "\n", + "The DataJoint linter is available at \n", + "[this repository](https://github.com/CBroz1/datajoint_linter).\n", + "\n", + "
" + ] + }, + { + "cell_type": "markdown", + "id": "df7554fc", + "metadata": {}, + "source": [ + "\n", + "### Installation\n", + "\n", "In a terminal, ...\n", "\n", - "1. navigate to your project directory.\n", - "2. use `git` to download the Spyglass repository.\n", - "3. navigate to the newly downloaded directory.\n", - "4. create a `mamba` environment with either the standard `environment.yml` or\n", + "1. Navigate to your project directory.\n", + "2. Use `git` to download the Spyglass repository.\n", + "3. Navigate to the newly downloaded directory.\n", + "4. Create a `mamba` environment with either the standard `environment.yml` or\n", " the `environment_position.yml`, if you intend to use the full position\n", " pipeline. The latter will take longer to install.\n", - "5. open this notebook with VSCode\n", + "5. Open this notebook with VSCode\n", "\n", "Commands for the steps above ...\n", "\n", @@ -88,19 +230,61 @@ "code notebooks/00_Setup.ipynb # 5\n", "```\n", "\n", - "_Note:_ Spyglass is also installable via\n", - "[pip]()\n", - "and [pypi](https://pypi.org/project/spyglass-neuro/) with\n", - "`pip install spyglass-neuro`, but downloading from GitHub will also download\n", - "other files.\n", - "\n", "Next, within VSCode,\n", "[select the kernel](https://code.visualstudio.com/docs/datascience/jupyter-kernel-management)\n", "that matches your spyglass environment created with `mamba`. To use other Python\n", "interfaces, be sure to activate the environment: `conda activate spyglass`\n", "\n", - "See [this guide](https://datajoint.com/docs/elements/user-guide/) for additional\n", - "details on each of these programs and the role they play in using the pipeline.\n" + "\n", + "### Considerations\n", + "\n", + "1. Spyglass is also installable via\n", + "[pip]()\n", + "and [pypi](https://pypi.org/project/spyglass-neuro/) with\n", + "`pip install spyglass-neuro`, but downloading from GitHub will also download\n", + "other files, like this tutorial.\n", + "2. Developers who wish to work on the code base may want to do an editable\n", + "install from within their conda environment: `pip install -e /path/to/spyglass/`\n" + ] + }, + { + "cell_type": "markdown", + "id": "aa254c25", + "metadata": {}, + "source": [ + "### Optional Dependencies\n", + "\n", + "Some pipelines require installation of additional packages.\n", + "\n", + "#### Spike Sorting\n", + "\n", + "The spike sorting pipeline relies on `spikeinterface` and optionally\n", + "`mountainsort4`.\n", + "\n", + "```bash\n", + "conda activate \n", + "pip install spikeinterface[full,widgets]\n", + "pip install mountainsort4\n", + "```\n", + "\n", + "#### LFP\n", + "\n", + "The LFP pipeline uses `ghostipy`.\n", + "\n", + "__WARNING:__ If you are on an M1 Mac, you need to install `pyfftw` via `conda`\n", + "BEFORE installing `ghostipy`:\n", + "\n", + "```bash\n", + "conda install -c conda-forge pyfftw # for M1 Macs\n", + "pip install ghostipy\n", + "```\n", + "\n", + "#### Decoding\n", + "\n", + "The Decoding pipeline relies on `jax` to process data with GPUs. Please see\n", + "their conda installation steps\n", + "[here](https://jax.readthedocs.io/en/latest/installation.html#conda-installation).\n", + "\n" ] }, { @@ -120,7 +304,7 @@ "\n", "1. Connect to an existing database.\n", "2. Run your own database with [Docker](#running-your-own-database)\n", - "3. JupyterHub (coming soon...)\n", + "3. JupyterHub (database pre-configured, skip this step)\n", "\n", "Your choice above should result in a set of credentials, including host name,\n", "host port, user name, and password. Note these for the next step.\n", @@ -155,7 +339,9 @@ "Connecting to an existing database will require a user name and password.\n", "Please contact your database administrator for this information.\n", "\n", - "Frank Lab members should contact Chris.\n" + "For persistent databases with backups, administrators should review our \n", + "documentation on \n", + "[database management](https://lorenfranklab.github.io/spyglass/latest/ForDevelopers/Database).\n" ] }, { @@ -202,10 +388,10 @@ "\n", "Docker credentials are as follows:\n", "\n", - "- Host: localhost\n", - "- Password: tutorial\n", - "- User: root\n", - "- Port: 3306\n" + "- Host: `localhost`\n", + "- User: `root`\n", + "- Password: `tutorial`\n", + "- Port: `3306`\n" ] }, { @@ -213,7 +399,7 @@ "id": "706d0ed5", "metadata": {}, "source": [ - "### Config and Connecting to the database\n" + "### Config\n" ] }, { @@ -221,20 +407,27 @@ "id": "22d3b72d", "metadata": {}, "source": [ - "Spyglass can load settings from either a DataJoint config file (recommended) or\n", - "environmental variables. The code below will generate a config file, but we\n", - "first need to decide a 'base path'. This is generally the parent directory\n", - "where the data will be stored, with subdirectories for `raw`, `analysis`, and\n", - "other data folders. If they don't exist already, they will be created.\n", + "Spyglass will load settings the 'custom' section of your DataJoint config file.\n", + "The code below will generate a config\n", + "file, but we first need to decide a 'base path'. This is generally the parent\n", + "directory where the data will be stored, with subdirectories for `raw`,\n", + "`analysis`, and other data folders. If they don't exist already, they will be\n", + "created relative to the base path specified with their default names. \n", + "\n", + "A temporary directory is one such subfolder (default `base-dir/tmp`) to speed\n", + "up spike sorting. Ideally, this folder should have ~500GB free.\n", "\n", "The function below will create a config file (`~/.datajoint.config` if global,\n", - "`./dj_local_conf.json` if local). Local is recommended for the notebooks, as\n", + "`./dj_local_conf.json` if local).\n", + "See also [DataJoint docs](https://datajoint.com/docs/core/datajoint-python/0.14/quick-start/#connection).\n", + "Local is recommended for the notebooks, as\n", "each will start by loading this file. Custom json configs can be saved elsewhere, but will need to be loaded in startup with\n", "`dj.config.load('your-path')`.\n", "\n", - "To point spyglass to a folder elsewhere (e.g., an external drive for waveform\n", - "data), simply edit the json file. Note that the `raw` and `analysis` paths\n", - "appear under both `stores` and `custom`.\n" + "To point Spyglass to a folder elsewhere (e.g., an external drive for waveform\n", + "data), simply edit the resulting json file. Note that the `raw` and `analysis` paths\n", + "appear under both `stores` and `custom`. Spyglass will check that these match\n", + "on startup and log a warning if not.\n" ] }, { @@ -256,12 +449,67 @@ " base_dir=\"/path/like/stelmo/nwb/\",\n", " database_user=\"your username\",\n", " database_password=\"your password\", # remove this line for shared machines\n", - " database_host=\"localhost or lmf-db.cin.ucsf.edu\",\n", + " database_host=\"localhost or lmf-db.cin.ucsf.edu\", # only list one\n", " database_port=3306,\n", " set_password=False,\n", ")" ] }, + { + "cell_type": "markdown", + "id": "a1c20b5b", + "metadata": {}, + "source": [ + "
Legacy config\n", + "\n", + "Older versions of Spyglass relied exclusively on environment variables for\n", + "config. If `spyglass_dirs` is not found in the config file, Spyglass will look\n", + "for environment variables. These can be set either once in a terminal session,\n", + "or permanently in a unix settings file (e.g., `.bashrc` or `.bash_profile`) in\n", + "your home directory.\n", + "\n", + "```bash\n", + "export SPYGLASS_BASE_DIR=\"/stelmo/nwb\"\n", + "export SPYGLASS_RECORDING_DIR=\"$SPYGLASS_BASE_DIR/recording\"\n", + "export SPYGLASS_SORTING_DIR=\"$SPYGLASS_BASE_DIR/sorting\"\n", + "export SPYGLASS_VIDEO_DIR=\"$SPYGLASS_BASE_DIR/video\"\n", + "export SPYGLASS_WAVEFORMS_DIR=\"$SPYGLASS_BASE_DIR/waveforms\"\n", + "export SPYGLASS_TEMP_DIR=\"$SPYGLASS_BASE_DIR/tmp\"\n", + "export KACHERY_CLOUD_DIR=\"$SPYGLASS_BASE_DIR/.kachery-cloud\"\n", + "export KACHERY_TEMP_DIR=\"$SPYGLASS_BASE_DIR/tmp\"\n", + "export DJ_SUPPORT_FILEPATH_MANAGEMENT=\"TRUE\"\n", + "```\n", + "\n", + "To load variables from a `.bashrc` file, run `source ~/.bashrc` in a terminal.\n", + "\n", + "
" + ] + }, + { + "cell_type": "markdown", + "id": "f2ef434f", + "metadata": {}, + "source": [ + "### Managing Files\n", + "\n", + "[`kachery-cloud`](https://github.com/flatironinstitute/kachery-cloud) is a file\n", + "manager for collaborators to share files. This is an optional dependency for\n", + "collaborating teams who don't have direct access to one another's disk space, \n", + "but want to share a MySQL database instance.\n", + "To customize `kachery` file paths, see `dj_local_conf_example.json`. \n", + "\n", + "To set up a new `kachery` instance for your project, contact maintainers\n", + "of this package." + ] + }, + { + "cell_type": "markdown", + "id": "38679c3a", + "metadata": {}, + "source": [ + "### Connecting" + ] + }, { "cell_type": "markdown", "id": "06eef771", @@ -294,16 +542,17 @@ "metadata": {}, "source": [ "If you see an error saying `Could not find SPYGLASS_BASE_DIR`, try loading your\n", - "config before importing Spyglass, try setting this as an environmental variable\n", - "before importing Spyglass.\n", + "config before importing Spyglass. \n", "\n", "```python\n", - "os.environ['SPYGLASS_BASE_DIR'] = '/your/base/path'\n", - "\n", - "import spyglass\n", - "from spyglass.settings import SpyglassConfig\n", "import datajoint as dj\n", - "print(SpyglassConfig().config)\n", + "dj.config.load('/your/config/path')\n", + "\n", + "from spyglass.common import Session\n", + "\n", + "Session()\n", + "\n", + "# If successful...\n", "dj.config.save_local() # or global\n", "```\n" ] @@ -321,7 +570,7 @@ "id": "c6850095", "metadata": {}, "source": [ - "Next, we'll try [inserting data](./01_Insert_Data.ipynb)\n" + "Next, we'll try [introduce some concepts](./01_Concepts.ipynb)\n" ] } ], @@ -341,7 +590,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.16" + "version": "3.9.19" } }, "nbformat": 4, diff --git a/notebooks/01_Concepts.ipynb b/notebooks/01_Concepts.ipynb new file mode 100644 index 000000000..2c3d535d1 --- /dev/null +++ b/notebooks/01_Concepts.ipynb @@ -0,0 +1,218 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "# Concepts\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Intro\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "_Developer Note:_ if you may make a PR in the future, be sure to copy this\n", + "notebook, and use the `gitignore` prefix `temp` to avoid future conflicts.\n", + "\n", + "This is one notebook in a multi-part series on Spyglass. To set up your Spyglass environment and database, see\n", + "[the Setup notebook](./00_Setup.ipynb)\n", + "\n", + "This notebook will introduce foundational concepts that will help in\n", + "understanding how to work with Spyglass pipelines.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Other materials\n", + "\n", + "DataJoint is an \"Object-relational mapping\" tool, which means that it gives us\n", + "a Python object for tables that exist on a shared SQL server. Many Spyglass\n", + "imports are DataJoint tables like this.\n", + "\n", + "Any 'introduction to SQL' will give an overview of relational data models as\n", + "a primer on how DataJoint tables within Spyglass will interact with one-another,\n", + "and the ways we can interact with them. A quick primer may help with the\n", + "specifics ahead.\n", + "\n", + "For an overview of DataJoint, including table definitions and inserts, see\n", + "[DataJoint tutorials](https://github.com/datajoint/datajoint-tutorials)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Common Errors\n", + "\n", + "Skip this for now, but refer back if you hit issues.\n", + "\n", + "\n", + "### Integrity\n", + "\n", + "```console\n", + "IntegrityError: Cannot add or update a child row: a foreign key constraint fails (`schema`.`_table`, CONSTRAINT `_table_ibfk_1` FOREIGN KEY (`parent_field`) REFERENCES `other_schema`.`parent_name` (`parent_field`) ON DELETE RESTRICT ON UPDATE CASCADE)\n", + "```\n", + "\n", + "`IntegrityError` during `insert` means that some part of the key you're\n", + "inserting doesn't exist in the parent of the table you're inserting into. You\n", + "can explore which that may be by doing the following...\n", + "\n", + "```python\n", + "my_key = dict(value=key) # whatever you're inserting\n", + "MyTable.insert1(my_key) # error here\n", + "parents = MyTable.parents(as_objects=True) # get the parents as FreeTables\n", + "for parent in parents: # iterate through the parents, with only relevant fields\n", + " parent_key = {k: v for k, v in my_key.items() if k in parent.heading.names}\n", + " print(parent & parent_key) # restricted parent\n", + "```\n", + "\n", + "If any of the printed tables are empty, you know you need to insert into that\n", + "table (or another ancestor up the pipeline) first. This code will not work if\n", + "there are aliases in the table (i.e., `proj` in the definition). In that case,\n", + "you'll need to modify your `parent_key` to reflect the renaming.\n", + "\n", + "The error message itself will tell you which table is the limiting parent. After\n", + "`REFERENCES` in the error message, you'll see the parent table and the column\n", + "that is causing the error.\n", + "\n", + "### Permission\n", + "\n", + "```console\n", + "('Insufficient privileges.', \"INSERT command denied to user 'username'@'127.0.0.1' for table '_table_name'\", 'INSERT INTO `schema_name`.`table_name`(`field1`,`field2`) VALUES (%s,%s)')\n", + "```\n", + "\n", + "This is a MySQL error that means that either ...\n", + "\n", + "- You don't have access to the command you're trying to run (e.g., `INSERT`)\n", + "- You don't have access to this command on the schema you're trying to run it on\n", + "\n", + "To see what permissions you have, you can run the following ...\n", + "\n", + "```python\n", + "dj.conn().query(\"SHOW GRANTS FOR CURRENT_USER();\").fetchall()\n", + "```\n", + "\n", + "If you think you should have access to the command, you contact your database\n", + "administrator (e.g., Chris in the Frank Lab). Please share the output of the\n", + "above command with them.\n", + "\n", + "### Type\n", + "\n", + "```console\n", + "TypeError: example_function() got an unexpected keyword argument 'this_arg'\n", + "```\n", + "\n", + "This means that you're calling a function with an argument that it doesn't\n", + "expect (e.g., `example_function(this_arg=5)`). You can check the function's\n", + "accepted arguments by running `help(example_function)`.\n", + "\n", + "```console\n", + "TypeError: 'NoneType' object is not iterable\n", + "```\n", + "\n", + "This means that some function is trying to do something with an object of an\n", + "unexpected type. For example, if might by running `for item in variable: ...`\n", + "when `variable` is `None`. You can check the type of the variable by going into\n", + "debug mode and running `type(variable)`.\n", + "\n", + "### KeyError\n", + "\n", + "```console\n", + "KeyError: 'field_name'\n", + "```\n", + "\n", + "This means that you're trying to access a key in a dictionary that doesn't\n", + "exist. You can check the keys of the dictionary by running `variable.keys()`. If\n", + "this is in your custom code, you can get a key and supply a default value if it\n", + "doesn't exist by running `variable.get('field_name', default_value)`.\n", + "\n", + "### DataJoint\n", + "\n", + "```console\n", + "DataJointError(\"Attempt to delete part table {part} before deleting from its master {master} first.\")\n", + "```\n", + "\n", + "This means that DataJoint's delete process found a part table with a foreign key\n", + "reference to the data you're trying to delete. You need to find the master table\n", + "listed and delete from that table first.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Debug Mode\n", + "\n", + "To fix an error, you may want to enter 'debug mode'. VSCode has a dedicated\n", + "featureful [extension](https://code.visualstudio.com/docs/python/debugging)\n", + "for making use of the UI, but you can choose to use Python's built-in tool.\n", + "\n", + "To enter into debug mode, you can add the following line to your code ...\n", + "\n", + "```python\n", + "__import__(\"pdb\").set_trace()\n", + "```\n", + "\n", + "This will set a breakpoint in your code at that line. When you run your code, it\n", + "will pause at that line and you can explore the variables in the current frame.\n", + "Commands in this mode include ...\n", + "\n", + "- `u` and `d` to move up and down the stack\n", + "- `l` to list the code around the current line\n", + "- `q` to quit the debugger\n", + "- `c` to continue running the code\n", + "- `h` for help, which will list all the commands\n", + "\n", + "`ipython` and jupyter notebooks can launch a debugger automatically at the last\n", + "error by running `%debug`.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Up Next\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, we'll try [inserting data](./01_Insert_Data.ipynb)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "spy", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/notebooks/01_Insert_Data.ipynb b/notebooks/02_Insert_Data.ipynb similarity index 94% rename from notebooks/01_Insert_Data.ipynb rename to notebooks/02_Insert_Data.ipynb index a92e14ca0..d8ef86233 100644 --- a/notebooks/01_Insert_Data.ipynb +++ b/notebooks/02_Insert_Data.ipynb @@ -1062,6 +1062,48 @@ "sgc.LabTeam.LabTeamMember()" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In general, we can insert into any table in this say, by supplying \n", + "a dictionary (or list of dictionaries) with all the fields mentioned in \n", + "`Table.heading.names` so long as the data types match what is described in\n", + "`Table.heading`\n", + "\n", + "```python\n", + "Table.insert1({'a': 1, 'b': 'other'}) # only one entry\n", + "Table.insert([{'a':1, 'b': 'other'}, {'a':1, 'b': 'next'}]) # multiple\n", + "```\n", + "\n", + "For example ..." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sgc.ProbeType.insert1(\n", + " {\n", + " \"probe_type\": \"128c-4s6mm6cm-15um-26um-sl\",\n", + " \"probe_description\": \"A Livermore flexible probe with 128 channels ...\",\n", + " \"manufacturer\": \"Lawrence Livermore National Lab\",\n", + " \"num_shanks\": 4,\n", + " },\n", + " skip_duplicates=True,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `skip_duplicates` flag tells DataJoint not to raise an error if the data\n", + "is already in the table. This should only be used in special cases." + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -2734,6 +2776,196 @@ "!ls $SPYGLASS_BASE_DIR/raw" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## YAML Inserts\n", + "\n", + "The following step is an optional feature, and not required for the remaining\n", + "notebooks.\n", + "\n", + "Not every NWB file has all the information required by Spyglass. For example,\n", + "many NWB files do not contain any information about the `DataAcquisitionDevice`\n", + "or `Probe` because NWB does not yet have an official standard for specifying\n", + "them. Or, information in the NWB file may need correcting. For example,\n", + "the NWB file specifies the lab name as the \"Loren Frank Lab\", but your lab table expects \"Frank Lab\".\n", + "\n", + "Manual inserts can either be done on tables directly (e.g., \n", + "`Table.insert1(my_dict)`), or done in batch with `yaml` files. This is done in\n", + "two steps: \n", + "\n", + "1. Generate data to be entered.\n", + "2. Associate data with one or more NWB files.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Batch Insert\n", + "\n", + "First, Spyglass will check for an `entries.yaml` file at the base directory\n", + "(see [Setup](./00_Setup.ipynb)) and run all corresponding inserts. \n", + "This is a great place to define entries that the database should auto-insert\n", + "prior to ingesting any NWB files. An example can be found in\n", + "`examples/config_yaml/entries.yaml`. It has the following structure:\n", + "\n", + "```yaml\n", + "TableName:\n", + " - TableEntry1Field1: Value\n", + "\n", + "TableEntry1Field2:\n", + " - TableEntry2Field1: Value\n", + "\n", + "TableEntry2Field2: Value\n", + "```\n", + "\n", + "For example,\n", + "\n", + "```yaml\n", + "ProbeType:\n", + " - probe_type: 128c-4s6mm6cm-15um-26um-sl\n", + " probe_description: A Livermore flexible probe with 128 channels, 4 shanks, \n", + " 6 mm shank length, 6 cm ribbon length. 15 um contact diameter, 26 um \n", + " center-to-center distance (pitch), single-line configuration.\n", + " manufacturer: Lawrence Livermore National Lab\n", + " num_shanks: 4\n", + "```\n", + "\n", + "Using a YAML file over data stored in Python scripts helps maintain records\n", + "of data entries in a human-readable file. For ways to share a state of the\n", + "database, see our [export tutorial](./05_Export.ipynb).\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Pairing with NWBs\n", + "\n", + "Next, we'll create a _configuration file_ to override values in a given NWB\n", + "(e.g., \"Loren Frank Lab\" -> \"Frank Lab\"). This must be done in the same\n", + "directory as the NWB file that it configures and have the following naming\n", + "convention: `_spyglass_config.yaml`. This file is then read by\n", + "Spyglass when calling `insert_session` on the associated NWB file.\n", + "\n", + "An example of this can be found at\n", + "`examples/config_yaml/​​sub-AppleBottom_ses-AppleBottom-DY20-g3_behavior+ecephys_spyglass_config.yaml`.\n", + "\n", + "This file is associated with the NWB file\n", + "`sub-AppleBottom_ses-AppleBottom-DY20-g3_behavior+ecephys.nwb`.\n", + "\n", + "This is the general format for the config entry:\n", + "\n", + "```yaml\n", + "TableName:\n", + " - primary_key1: value1\n", + "```\n", + "\n", + "For example:\n", + "\n", + "```yaml\n", + "Lab:\n", + " - lab_name: Frank Lab\n", + "DataAcquisitionDevice:\n", + " - data_acquisition_device_name: Neuropixels Recording Device\n", + "```\n", + "\n", + "In this example, the NWB file that corresponds to this config YAML will become\n", + "associated with the Lab primary key 'Frank Lab' and the DataAcquisitionDevice\n", + "with primary key 'Neuropixels Recording Device'. This entry must already exist." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Example Ingestion with Real Data\n", + "\n", + "For this example, you will need to download the 5 GB NWB file \n", + "`sub-JDS-NFN-AM2_behavior+ecephys.nwb`\n", + "from dandiset 000447 here: \n", + "https://dandiarchive.org/dandiset/000447/0.230316.2133/files?location=sub-JDS-NFN-AM2&page=1\n", + "\n", + "Click the download arrow button to download the file to your computer. Add it to\n", + " the folder containing your raw NWB data to be ingested into Spyglass.\n", + "\n", + "This file does not specify a data acquisition device. Let's say that the\n", + "data was collected from a SpikeGadgets system with an Intan amplifier. This\n", + "matches an existing entry in the `DataAcquisitionDevice` table with name\n", + "\"data_acq_device0\". We will create a configuration YAML file to associate\n", + "this entry with the NWB file.\n", + "\n", + "If you are connected to the Frank lab database, please rename any downloaded\n", + "files (e.g., `example20200101_yourname.nwb`) to avoid naming collisions, as the\n", + "file name acts as the primary key across key tables." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nwb_file_name = \"sub-JDS-NFN-AM2_behavior+ecephys_rly.nwb\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# this configuration yaml file should be placed next to the downloaded NWB file\n", + "yaml_config_path = \"sub-JDS-NFN-AM2_behavior+ecephys_rly_spyglass_config.yaml\"\n", + "with open(yaml_config_path, \"w\") as config_file:\n", + " lines = [\n", + " \"DataAcquisitionDevice\",\n", + " \"- data_acquisition_device_name: data_acq_device0\",\n", + " ]\n", + " config_file.writelines(line + \"\\n\" for line in lines)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then call `insert_sessions` as usual." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import spyglass.data_import as sgi\n", + "\n", + "sgi.insert_sessions(nwb_file_name)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Confirm the session was inserted with the correct `DataAcquisitionDevice`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import spyglass.common as sgc\n", + "from spyglass.utils.nwb_helper_fn import get_nwb_copy_filename\n", + "\n", + "nwb_copy_file_name = get_nwb_copy_filename(nwb_file_name)\n", + "\n", + "sgc.Session.DataAcquisitionDevice & {\"nwb_file_name\": nwb_copy_file_name}" + ] + }, { "cell_type": "markdown", "metadata": {}, diff --git a/notebooks/02_Data_Sync.ipynb b/notebooks/03_Data_Sync.ipynb similarity index 100% rename from notebooks/02_Data_Sync.ipynb rename to notebooks/03_Data_Sync.ipynb diff --git a/notebooks/03_Merge_Tables.ipynb b/notebooks/04_Merge_Tables.ipynb similarity index 100% rename from notebooks/03_Merge_Tables.ipynb rename to notebooks/04_Merge_Tables.ipynb diff --git a/notebooks/04_PopulateConfigFile.ipynb b/notebooks/04_PopulateConfigFile.ipynb deleted file mode 100644 index 4ead237fb..000000000 --- a/notebooks/04_PopulateConfigFile.ipynb +++ /dev/null @@ -1,273 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "68303e8a", - "metadata": {}, - "source": [ - "# Customizing Data Insertion into Spyglass\n", - "\n", - "If you would like to insert data into Spyglass that does not\n", - "follow the naming or organizational format expected by Spyglass, \n", - "or you would like to override what values are ingested into Spyglass from \n", - "your NWB files, including missing values, follow this guide.\n", - "\n", - "## General Approach\n", - "\n", - "When an NWB file is ingested into Spyglass, metadata about the session\n", - "is first read from the NWB file and inserted into\n", - "tables in the `common` module (e.g. `Institution`, `Lab`, `Electrode`, etc).\n", - "However, not every NWB file has all the information required by Spyglass or\n", - "the information in the NWB file is not in a format that Spyglass expects. For\n", - "example, many NWB files do not contain information about the\n", - "`DataAcquisitionDevice` or `Probe` because the NWB data standard does not yet\n", - "have an official\n", - "standard for specifying them. For these cases, we provide a way to customize\n", - "how data is ingested into Spyglass.\n", - "\n", - "Let's say that you want to ingest an NWB file into Spyglass where the lab name\n", - "in the file is written as \"Loren Frank Lab\" or it is not specified, but you \n", - "know the data comes from the Loren Frank Lab. Let's say that in Spyglass,\n", - "the lab name that is associated with sessions from the Loren Frank Lab is \n", - "\"Frank Lab\" and you would like to use the same name in order to facilitate\n", - "data search in Spyglass. To change the lab name when you insert your new data \n", - "to Spyglass, you could either 1) edit the NWB file directly and then \n", - "insert it into Spyglass, or 2) define an override value \"Frank Lab\" to be \n", - "used instead of the value specified in the NWB file (or lack thereof).\n", - "\n", - "Note that if this is your own data and you want to make changes to\n", - "information about how the data is interpreted, e.g., the units of measurement\n", - "are incorrect, we recommend that you edit the NWB file directly because the \n", - "file or derivatives of it might eventually be shared outside of Spyglass and \n", - "they will not reflect any modifications that you have made to \n", - "the data only in Spyglass." - ] - }, - { - "cell_type": "markdown", - "id": "bcc87f67", - "metadata": {}, - "source": [ - "## Define a Configuration YAML File\n", - "\n", - "To override values in the NWB file during insertion into Spyglass, \n", - "you will need to create a configuration \n", - "YAML file that lives in the same directory as your NWB file, named: \n", - "`_spyglass_config.yaml`\n", - "\n", - "An example configuration YAML file can be found at\n", - "`examples/config_yaml/​​sub-AppleBottom_ses-AppleBottom-DY20-g3_behavior+ecephys_spyglass_config.yaml`.\n", - "This file is associated with the NWB file\n", - "`sub-AppleBottom_ses-AppleBottom-DY20-g3_behavior+ecephys.nwb`.\n", - "\n", - "This is the general format for entries in this configuration file:\n", - "\n", - "```yaml\n", - "TableName:\n", - "- primary_key1: value1\n", - "```\n", - "\n", - "For example:\n", - "\n", - "```yaml\n", - "Lab:\n", - "- lab_name: Frank Lab\n", - "```\n", - "\n", - "In this example, the NWB file that corresponds to this config YAML will become\n", - "associated with the entry in the `Lab` table with the value `Frank Lab` for \n", - "the primary key `lab_name`. This entry must already exist. More specifically,\n", - "when the NWB file is ingested into Spyglass, \n", - "a new `Session` entry will be created for the NWB file that has a foreign key to\n", - "the `Lab` entry with `lab_name` = \"Frank Lab\", ignoring whatever lab value is \n", - "in the NWB file, even if one does not exist.\n", - "\n", - "TODO implement this for `Lab`.\n" - ] - }, - { - "cell_type": "markdown", - "id": "fc6d0986", - "metadata": {}, - "source": [ - "## Create Entries to Reference in the Configuration YAML\n", - "\n", - "As mentioned earlier, the table entry that you want to associate with your NWB\n", - "file must already exist in the database. This entry would typically be a value\n", - "that is independent of any particular NWB file, such as\n", - "`DataAcquisitionDevice`, `Lab`, `Probe`, and `BrainRegion`. \n", - "\n", - "If the entry does not already exist, you can either:\n", - "1) create the entry programmatically using DataJoint `insert` commands, or\n", - "2) define the entry in a YAML file called `entries.yaml` that is automatically\n", - " processed when Spyglass is imported. You can think of `entries.yaml` as a\n", - " place to define information that the database should come pre-equipped prior\n", - " to ingesting your NWB files. The `entries.yaml` file should be placed in the\n", - " `spyglass` base directory (next to `README.md`). An example can be found in\n", - " `examples/config_yaml/entries.yaml`. This file should have the following\n", - " structure:\n", - "\n", - " ```yaml\n", - " TableName:\n", - " - TableEntry1Field1: Value\n", - " TableEntry1Field2: Value\n", - " - TableEntry2Field1: Value\n", - " TableEntry2Field2: Value\n", - " ```\n", - "\n", - " For example,\n", - "\n", - " ```yaml\n", - " DataAcquisitionDeviceSystem:\n", - " data_acquisition_device_system: SpikeGLX\n", - " DataAcquisitionDevice:\n", - " - data_acquisition_device_name: Neuropixels_SpikeGLX\n", - " data_acquisition_device_system: SpikeGLX\n", - " data_acquisition_device_amplifier: Intan\n", - " ```\n", - "\n", - " Only `dj.Manual`, `dj.Lookup`, and `dj.Part` tables can be populated\n", - " using this approach.\n", - "\n", - "Once the entry that you want to associate with your NWB file exists in the\n", - "database, you can write the configuration YAML file and then ingest your\n", - "NWB file. As an another example, let's say that you want to associate your NWB\n", - "file with the `DataAcquisitionDevice` entry with `data_acquisition_device_name`\n", - "= \"Neuropixels_SpikeGLX\" that was defined above. You would write the following\n", - "configuration YAML file:\n", - "\n", - "```yaml\n", - "DataAcquisitionDevice:\n", - "- data_acquisition_device_name: Neuropixels_SpikeGLX\n", - "```\n", - "\n", - "The example in\n", - "`examples/config_yaml/​​sub-AppleBottom_ses-AppleBottom-DY20-g3_behavior+ecephys_spyglass_config.yaml`\n", - "includes additional examples." - ] - }, - { - "cell_type": "markdown", - "id": "9d641e00", - "metadata": {}, - "source": [ - "## Example Ingestion with Real Data\n", - "\n", - "For this example, you will need to download the 5 GB NWB file \n", - "`sub-JDS-NFN-AM2_behavior+ecephys.nwb`\n", - "from dandiset 000447 here: \n", - "https://dandiarchive.org/dandiset/000447/0.230316.2133/files?location=sub-JDS-NFN-AM2&page=1\n", - "\n", - "Click the download arrow button to download the file to your computer. Add it to the folder\n", - "containing your raw NWB data to be ingested into Spyglass.\n", - "\n", - "This file does not specify a data acquisition device. Let's say that the\n", - "data was collected from a SpikeGadgets system with an Intan amplifier. This\n", - "matches an existing entry in the `DataAcquisitionDevice` table with name\n", - "\"data_acq_device0\". We will create a configuration YAML file to associate\n", - "this entry with the NWB file.\n", - "\n", - "If you are connected to the Frank lab database, please rename any downloaded\n", - "files (e.g., `example20200101_yourname.nwb`) to avoid naming collisions, as the\n", - "file name acts as the primary key across key tables." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "37aa5182", - "metadata": {}, - "outputs": [], - "source": [ - "nwb_file_name = \"sub-JDS-NFN-AM2_behavior+ecephys_rly.nwb\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "aab5b47a", - "metadata": {}, - "outputs": [], - "source": [ - "# this configuration yaml file should be placed next to the downloaded NWB file\n", - "yaml_config_path = \"sub-JDS-NFN-AM2_behavior+ecephys_rly_spyglass_config.yaml\"\n", - "with open(yaml_config_path, \"w\") as config_file:\n", - " lines = [\n", - " \"DataAcquisitionDevice\",\n", - " \"- data_acquisition_device_name: data_acq_device0\",\n", - " ]\n", - " config_file.writelines(line + \"\\n\" for line in lines)" - ] - }, - { - "cell_type": "markdown", - "id": "d132e797", - "metadata": {}, - "source": [ - "Then call `insert_sessions` as usual." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bed5c6e1", - "metadata": {}, - "outputs": [], - "source": [ - "import spyglass.data_import as sgi\n", - "\n", - "sgi.insert_sessions(nwb_file_name)" - ] - }, - { - "cell_type": "markdown", - "id": "d875b158", - "metadata": {}, - "source": [ - "Confirm the session was inserted with the correct `DataAcquisitionDevice`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8411cb43", - "metadata": {}, - "outputs": [], - "source": [ - "import spyglass.common as sgc\n", - "from spyglass.utils.nwb_helper_fn import get_nwb_copy_filename\n", - "\n", - "nwb_copy_file_name = get_nwb_copy_filename(nwb_file_name)\n", - "\n", - "sgc.Session.DataAcquisitionDevice & {\"nwb_file_name\": nwb_copy_file_name}" - ] - }, - { - "cell_type": "markdown", - "id": "d85b1416", - "metadata": {}, - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.18" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/10_Spike_SortingV0.ipynb b/notebooks/10_Spike_SortingV0.ipynb index d376db4b3..3497a461c 100644 --- a/notebooks/10_Spike_SortingV0.ipynb +++ b/notebooks/10_Spike_SortingV0.ipynb @@ -4,7 +4,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Spike Sorting\n" + "# Spike Sorting V0\n", + "\n", + "_Note_: This notebook explains the first version of the spike sorting pipeline\n", + "and is preserved for using existing data. New users should use\n", + "[V1](./10_Spike_SortingV1.ipynb).\n" ] }, { diff --git a/notebooks/21_DLC.ipynb b/notebooks/21_DLC.ipynb index b976ae5eb..c2e151b89 100644 --- a/notebooks/21_DLC.ipynb +++ b/notebooks/21_DLC.ipynb @@ -2042,6 +2042,41 @@ "sgp.DLCPosVideo().populate(dlc_key)" ] }, + { + "cell_type": "markdown", + "id": "04d1dca8", + "metadata": {}, + "source": [ + "
On editing parameters\n", + "\n", + "The presence of existing parameters in many tables makes it easy to tweak them \n", + "for your needs. You can fetch, edit, and re-insert new params - but the process\n", + "will look a little different if the table has a `=BLOB=` field.\n", + "\n", + "(These example assumes only one primary key. If multiple, `{'primary_key': 'x'}`\n", + "and `['primary_key']` will need to be adjusted accordingly.)\n", + "\n", + "No blob means that all parameters are fields in the table.\n", + "\n", + "```python\n", + "existing_params = (MyParamsTable & {'primary_key':'x'}).fetch1()\n", + "new_params = {**existing_params, 'primary_key': 'y', 'my_variable': 'a', 'other_variable':'b'}\n", + "MyParamsTable.insert1(new_params)\n", + "```\n", + "\n", + "A blob means that the params are stored as an embedded dictionary. We'll assume\n", + "this column is called `params`\n", + "\n", + "```python\n", + "existing_params = (MyParamsTable & {'primary_key':'x'}).fetch1()\n", + "new_params = {**existing_params, 'primary_key': 'y'}\n", + "print(existing_params['params']) # check existing values\n", + "new_params['params'] = {**existing_params['params'], 'my_variable': 'a', 'other_variable':'b'}\n", + "```\n", + "\n", + "
" + ] + }, { "cell_type": "markdown", "id": "5a68bba8-9871-40ac-84c9-51ac0e76d44e", diff --git a/notebooks/README.md b/notebooks/README.md index 62b136240..0982c464f 100644 --- a/notebooks/README.md +++ b/notebooks/README.md @@ -1,21 +1,25 @@ # Tutorial Notebooks There are several paths one can take to these notebooks. The notebooks have -two-digits in their names, the first of which indicates it's 'batch', as +two-digits in their names, the first of which indicates its 'batch', as described in the categories below. ## 0. Intro Everyone should complete the [Setup](./00_Setup.ipynb) and -[Insert Data](./01_Insert_Data.ipynb) notebooks. +[Insert Data](./02_Insert_Data.ipynb) notebooks. The +[Concepts](./01_Concepts.ipynb) notebook offers additional information that will +help users understand the data structure and how to interact with it. -[Data Sync](./02_Data_Sync.ipynb) is an optional additional tool for +[Data Sync](./03_Data_Sync.ipynb) is an optional additional tool for collaborators that want to share analysis files. -The [Merge Tables notebook](./03_Merge_Tables.ipynb) explains details on a new -table tier unique to Spyglass that allows the user to use different versions of -pipelines on the same data. This is important for understanding the later -notebooks. +The [Merge Tables notebook](./04_Merge_Tables.ipynb) explains details on a +pipeline versioning technique unique to Spyglass. This is important for +understanding the later notebooks. + +The [Export notebook](./05_Export.ipynb) shows how to export data from the +database. ## 1. Spike Sorting Pipeline @@ -24,14 +28,14 @@ spike sorting to optional manual curation of the output of the automated sorting. Spikesorting results from any pipeline can then be organized and tracked using -tools in [Spikesorting Analysis](./11_Spike_Sorting_Agit analysis.ipynb) +tools in [Spikesorting Analysis](./11_Spikesorting_Analysis.ipynb). ## 2. Position Pipeline This series of notebooks covers tracking the position(s) of the animal. The user can employ two different methods: -1. the simple [Trodes](20_Position_Trodes.ipynb) methods of tracking LEDs on the +1. The simple [Trodes](20_Position_Trodes.ipynb) methods of tracking LEDs on the animal's headstage 2. [DLC (DeepLabCut)](./21_DLC.ipynb) which uses a neural network to track the animal's body parts. diff --git a/notebooks/py_scripts/00_Setup.py b/notebooks/py_scripts/00_Setup.py index 2ea726aa8..a9a7fe269 100644 --- a/notebooks/py_scripts/00_Setup.py +++ b/notebooks/py_scripts/00_Setup.py @@ -34,8 +34,13 @@ # ## Local environment # -# JupyterHub users can skip this step. Frank Lab members should first follow -# 'rec to nwb overview' steps on Google Drive to set up an ssh connection. +# Skip this step if you're ... +# +# 1. Running the tutorials on [JupyterHub](https://spyglass.hhmi.2i2c.cloud/) +# 2. A member of the Frank Lab members. Instead, ssh to a shared machine. +# + +# ### Tools # # For local use, download and install ... # @@ -55,15 +60,141 @@ # 4. [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) for # downloading the repository, including notebooks. # +# See [this DataJoint guide](https://datajoint.com/docs/elements/user-guide/) for +# additional details on each of these programs and the role they play in using the +# pipeline. +# +#
Suggested VSCode settings +# +# Within the Spyglass repository, there is a `.vscode` folder with `json` files +# that specify limited settings and extensions intended for developers. The average +# user may benefit from the following fuller sets. +# +# We recommending these incrementally so you get a feel for what each one does +# before adding the next, and to avoid being overwhelmed by changes. +# +# 1. `extensions.json`. By updating this file, you'll add to the 'Recommended' +# section of the extensions tab. Each extension page will provide more information +# on the uses and benefits. Some relevant concepts include... +# - Linting: Warning of potential problems +# - Formatting: Auto-adjusting optional coding styles to align across users +# - Debugger: Progressive running of code. Please search for tutorials +# - Autocompletion: Prompting for potential options when coding +# +# ```json +# { +# "recommendations": [ +# // Python Extensions +# "charliermarsh.ruff", // Fast linter +# "donjayamanne.python-environment-manager", // Environment manager +# "kevinrose.vsc-python-indent", // Auto-indent when coding +# "ms-python.black-formatter", // Opinionated formatting +# "ms-python.debugpy", // Debugger +# "ms-python.isort", // Opinionated formatter for imports +# "ms-python.pylint", // Linter to support a DataJoint-specific linter +# "ms-python.python", // Language support for Python +# "ms-python.vscode-pylance", // Additional language support +# // Jupyter +# "ms-toolsai.jupyter", // Run notebooks in VSCode +# "ms-toolsai.jupyter-keymap", // Allow key-bindings +# "ms-toolsai.jupyter-renderers", // Display images +# // Autocompletion/Markdown +# "github.copilot", // Auto-suggest with copilot LLM +# "github.copilot-chat", // Add chat-box for questions to LLM +# "visualstudioexptteam.intellicode-api-usage-examples", // Prompt package options +# "visualstudioexptteam.vscodeintellicode", // Prompt Python-general options +# "davidanson.vscode-markdownlint", // Linter for markdown +# "streetsidesoftware.code-spell-checker", // Spell checker +# // SSH - Work on remote servers - Required for Frank Lab members +# "ms-vscode-remote.remote-ssh", +# "ms-vscode-remote.remote-ssh-edit", +# "ms-vscode.remote-explorer", +# ], +# "unwantedRecommendations": [] +# } +# ``` +# +# 2. `settings.json`. These can be places just in Spyglass, or added to your user +# settings file. Search settings in the command panel (cmd/ctrl+shift+P) to open +# this file directly. +# +# ```json +# { +# // GENERAL +# "editor.insertSpaces": true, // tab -> spaces +# "editor.rulers": [ 80 ], // vertical line at 80 +# "editor.stickyScroll.enabled": true, // Show scope at top +# "files.associations": { "*.json": "jsonc" }, // Load JSON with comments +# "files.autoSave": "onFocusChange", // Save on focus change +# "files.exclude": { // Hide these in the file viewer +# "**/__pycache*": true, // Add others with wildcards +# "**/.ipynb_ch*": true, +# }, +# "files.trimTrailingWhitespace": true, // Remove extra spaces in lines +# "git.enabled": true, // use git +# "workbench.editorAssociations": { // open file extension as given type +# "*.ipynb": "jupyter-notebook", +# }, +# // PYTHON +# "editor.defaultFormatter": "ms-python.black-formatter", // use black +# "[python]": { +# "editor.formatOnSave": true, +# "editor.defaultFormatter": "ms-python.black-formatter", +# "editor.codeActionsOnSave": { "source.organizeImports": "always"}, +# }, +# "python.analysis.autoImportCompletions": false, // Disable auto-import +# "python.languageServer": "Pylance", // Use Pylance +# "pylint.args": [ // DataJoint linter optional +# // "--load-plugins=datajoint_linter", // Requires pip installing +# // "--permit-dj-filepath=y", // Specific to datajoint_linter +# "--disable=E0401,E0102,W0621,W0401,W0611,W0614" +# ], +# // NOTEBOOKS +# "jupyter.askForKernelRestart": false, // Prevent dialog box on restart +# "jupyter.widgetScriptSources": ["jsdelivr.com", "unpkg.com"], // IPyWidgets +# "notebook.output.textLineLimit": 15, // Limit output +# "notebook.lineNumbers": "on", // Number lines in cells +# "notebook.formatOnSave.enabled": true, // blackify cells +# // AUTOCOMPLETION +# "editor.tabCompletion": "on", // tab over suggestions +# "github.copilot.editor.enableAutoCompletions": true, // Copilot +# "cSpell.enabled": true, // Spellcheck +# "cSpell.language": "en,en-US,companies,python,python-common", +# "cSpell.maxDuplicateProblems": 2, // Only mention a problem twice +# "cSpell.spellCheckDelayMs": 500, // Wait 0.5s after save +# "cSpell.userWords": [ "datajoint", "longblob", ], // Add words +# "cSpell.enableFiletypes": [ +# "!json", "markdown", "yaml", "python" // disable (!) json, check others +# ], +# "cSpell.logLevel": "Warning", // Only show warnings, can turn off +# // MARKDOWN +# "[markdown]": { // Use linter and format on save +# "editor.defaultFormatter": "DavidAnson.vscode-markdownlint", +# "editor.formatOnSave": true, +# }, +# "editor.codeActionsOnSave": { "source.fixAll.markdownlint": "explicit" }, +# "rewrap.reformat": true, // allows context-aware rewrapping +# "rewrap.wrappingColumn": 80, // Align with Black formatter +# } +# ``` +# +# The DataJoint linter is available at +# [this repository](https://github.com/CBroz1/datajoint_linter). +# +#
+ +# +# ### Installation +# # In a terminal, ... # -# 1. navigate to your project directory. -# 2. use `git` to download the Spyglass repository. -# 3. navigate to the newly downloaded directory. -# 4. create a `mamba` environment with either the standard `environment.yml` or +# 1. Navigate to your project directory. +# 2. Use `git` to download the Spyglass repository. +# 3. Navigate to the newly downloaded directory. +# 4. Create a `mamba` environment with either the standard `environment.yml` or # the `environment_position.yml`, if you intend to use the full position # pipeline. The latter will take longer to install. -# 5. open this notebook with VSCode +# 5. Open this notebook with VSCode # # Commands for the steps above ... # @@ -75,19 +206,56 @@ # code notebooks/00_Setup.ipynb # 5 # ``` # -# _Note:_ Spyglass is also installable via -# [pip]() -# and [pypi](https://pypi.org/project/spyglass-neuro/) with -# `pip install spyglass-neuro`, but downloading from GitHub will also download -# other files. -# # Next, within VSCode, # [select the kernel](https://code.visualstudio.com/docs/datascience/jupyter-kernel-management) # that matches your spyglass environment created with `mamba`. To use other Python # interfaces, be sure to activate the environment: `conda activate spyglass` # -# See [this guide](https://datajoint.com/docs/elements/user-guide/) for additional -# details on each of these programs and the role they play in using the pipeline. +# +# ### Considerations +# +# 1. Spyglass is also installable via +# [pip]() +# and [pypi](https://pypi.org/project/spyglass-neuro/) with +# `pip install spyglass-neuro`, but downloading from GitHub will also download +# other files, like this tutorial. +# 2. Developers who wish to work on the code base may want to do an editable +# install from within their conda environment: `pip install -e /path/to/spyglass/` +# + +# ### Optional Dependencies +# +# Some pipelines require installation of additional packages. +# +# #### Spike Sorting +# +# The spike sorting pipeline relies on `spikeinterface` and optionally +# `mountainsort4`. +# +# ```bash +# conda activate +# pip install spikeinterface[full,widgets] +# pip install mountainsort4 +# ``` +# +# #### LFP +# +# The LFP pipeline uses `ghostipy`. +# +# __WARNING:__ If you are on an M1 Mac, you need to install `pyfftw` via `conda` +# BEFORE installing `ghostipy`: +# +# ```bash +# conda install -c conda-forge pyfftw # for M1 Macs +# pip install ghostipy +# ``` +# +# #### Decoding +# +# The Decoding pipeline relies on `jax` to process data with GPUs. Please see +# their conda installation steps +# [here](https://jax.readthedocs.io/en/latest/installation.html#conda-installation). +# # # ## Database @@ -97,7 +265,7 @@ # # 1. Connect to an existing database. # 2. Run your own database with [Docker](#running-your-own-database) -# 3. JupyterHub (coming soon...) +# 3. JupyterHub (database pre-configured, skip this step) # # Your choice above should result in a set of credentials, including host name, # host port, user name, and password. Note these for the next step. @@ -122,7 +290,9 @@ # Connecting to an existing database will require a user name and password. # Please contact your database administrator for this information. # -# Frank Lab members should contact Chris. +# For persistent databases with backups, administrators should review our +# documentation on +# [database management](https://lorenfranklab.github.io/spyglass/latest/ForDevelopers/Database). # # ### Running your own database with Docker @@ -159,29 +329,36 @@ # # Docker credentials are as follows: # -# - Host: localhost -# - Password: tutorial -# - User: root -# - Port: 3306 +# - Host: `localhost` +# - User: `root` +# - Password: `tutorial` +# - Port: `3306` # -# ### Config and Connecting to the database +# ### Config # -# Spyglass can load settings from either a DataJoint config file (recommended) or -# environmental variables. The code below will generate a config file, but we -# first need to decide a 'base path'. This is generally the parent directory -# where the data will be stored, with subdirectories for `raw`, `analysis`, and -# other data folders. If they don't exist already, they will be created. +# Spyglass will load settings the 'custom' section of your DataJoint config file. +# The code below will generate a config +# file, but we first need to decide a 'base path'. This is generally the parent +# directory where the data will be stored, with subdirectories for `raw`, +# `analysis`, and other data folders. If they don't exist already, they will be +# created relative to the base path specified with their default names. +# +# A temporary directory is one such subfolder (default `base-dir/tmp`) to speed +# up spike sorting. Ideally, this folder should have ~500GB free. # # The function below will create a config file (`~/.datajoint.config` if global, -# `./dj_local_conf.json` if local). Local is recommended for the notebooks, as +# `./dj_local_conf.json` if local). +# See also [DataJoint docs](https://datajoint.com/docs/core/datajoint-python/0.14/quick-start/#connection). +# Local is recommended for the notebooks, as # each will start by loading this file. Custom json configs can be saved elsewhere, but will need to be loaded in startup with # `dj.config.load('your-path')`. # -# To point spyglass to a folder elsewhere (e.g., an external drive for waveform -# data), simply edit the json file. Note that the `raw` and `analysis` paths -# appear under both `stores` and `custom`. +# To point Spyglass to a folder elsewhere (e.g., an external drive for waveform +# data), simply edit the resulting json file. Note that the `raw` and `analysis` paths +# appear under both `stores` and `custom`. Spyglass will check that these match +# on startup and log a warning if not. # # + @@ -197,12 +374,49 @@ base_dir="/path/like/stelmo/nwb/", database_user="your username", database_password="your password", # remove this line for shared machines - database_host="localhost or lmf-db.cin.ucsf.edu", + database_host="localhost or lmf-db.cin.ucsf.edu", # only list one database_port=3306, set_password=False, ) # - +#
Legacy config +# +# Older versions of Spyglass relied exclusively on environment variables for +# config. If `spyglass_dirs` is not found in the config file, Spyglass will look +# for environment variables. These can be set either once in a terminal session, +# or permanently in a unix settings file (e.g., `.bashrc` or `.bash_profile`) in +# your home directory. +# +# ```bash +# export SPYGLASS_BASE_DIR="/stelmo/nwb" +# export SPYGLASS_RECORDING_DIR="$SPYGLASS_BASE_DIR/recording" +# export SPYGLASS_SORTING_DIR="$SPYGLASS_BASE_DIR/sorting" +# export SPYGLASS_VIDEO_DIR="$SPYGLASS_BASE_DIR/video" +# export SPYGLASS_WAVEFORMS_DIR="$SPYGLASS_BASE_DIR/waveforms" +# export SPYGLASS_TEMP_DIR="$SPYGLASS_BASE_DIR/tmp" +# export KACHERY_CLOUD_DIR="$SPYGLASS_BASE_DIR/.kachery-cloud" +# export KACHERY_TEMP_DIR="$SPYGLASS_BASE_DIR/tmp" +# export DJ_SUPPORT_FILEPATH_MANAGEMENT="TRUE" +# ``` +# +# To load variables from a `.bashrc` file, run `source ~/.bashrc` in a terminal. +# +#
+ +# ### Managing Files +# +# [`kachery-cloud`](https://github.com/flatironinstitute/kachery-cloud) is a file +# manager for collaborators to share files. This is an optional dependency for +# collaborating teams who don't have direct access to one another's disk space, +# but want to share a MySQL database instance. +# To customize `kachery` file paths, see `dj_local_conf_example.json`. +# +# To set up a new `kachery` instance for your project, contact maintainers +# of this package. + +# ### Connecting + # If you used either a local or global save method, we can check the connection # to the database with ... # @@ -219,16 +433,17 @@ # - # If you see an error saying `Could not find SPYGLASS_BASE_DIR`, try loading your -# config before importing Spyglass, try setting this as an environmental variable -# before importing Spyglass. +# config before importing Spyglass. # # ```python -# os.environ['SPYGLASS_BASE_DIR'] = '/your/base/path' -# -# import spyglass -# from spyglass.settings import SpyglassConfig # import datajoint as dj -# print(SpyglassConfig().config) +# dj.config.load('/your/config/path') +# +# from spyglass.common import Session +# +# Session() +# +# # If successful... # dj.config.save_local() # or global # ``` # @@ -236,5 +451,5 @@ # # Up Next # -# Next, we'll try [inserting data](./01_Insert_Data.ipynb) +# Next, we'll try [introduce some concepts](./01_Concepts.ipynb) # diff --git a/notebooks/py_scripts/01_Concepts.py b/notebooks/py_scripts/01_Concepts.py new file mode 100644 index 000000000..f7f8ca190 --- /dev/null +++ b/notebooks/py_scripts/01_Concepts.py @@ -0,0 +1,170 @@ +# --- +# jupyter: +# jupytext: +# text_representation: +# extension: .py +# format_name: light +# format_version: '1.5' +# jupytext_version: 1.16.0 +# kernelspec: +# display_name: spy +# language: python +# name: python3 +# --- + +# # Concepts +# + +# ## Intro +# + +# _Developer Note:_ if you may make a PR in the future, be sure to copy this +# notebook, and use the `gitignore` prefix `temp` to avoid future conflicts. +# +# This is one notebook in a multi-part series on Spyglass. To set up your Spyglass environment and database, see +# [the Setup notebook](./00_Setup.ipynb) +# +# This notebook will introduce foundational concepts that will help in +# understanding how to work with Spyglass pipelines. +# + +# ## Other materials +# +# DataJoint is an "Object-relational mapping" tool, which means that it gives us +# a Python object for tables that exist on a shared SQL server. Many Spyglass +# imports are DataJoint tables like this. +# +# Any 'introduction to SQL' will give an overview of relational data models as +# a primer on how DataJoint tables within Spyglass will interact with one-another, +# and the ways we can interact with them. A quick primer may help with the +# specifics ahead. +# +# For an overview of DataJoint, including table definitions and inserts, see +# [DataJoint tutorials](https://github.com/datajoint/datajoint-tutorials). + +# ## Common Errors +# +# Skip this for now, but refer back if you hit issues. +# +# +# ### Integrity +# +# ```console +# IntegrityError: Cannot add or update a child row: a foreign key constraint fails (`schema`.`_table`, CONSTRAINT `_table_ibfk_1` FOREIGN KEY (`parent_field`) REFERENCES `other_schema`.`parent_name` (`parent_field`) ON DELETE RESTRICT ON UPDATE CASCADE) +# ``` +# +# `IntegrityError` during `insert` means that some part of the key you're +# inserting doesn't exist in the parent of the table you're inserting into. You +# can explore which that may be by doing the following... +# +# ```python +# my_key = dict(value=key) # whatever you're inserting +# MyTable.insert1(my_key) # error here +# parents = MyTable.parents(as_objects=True) # get the parents as FreeTables +# for parent in parents: # iterate through the parents, with only relevant fields +# parent_key = {k: v for k, v in my_key.items() if k in parent.heading.names} +# print(parent & parent_key) # restricted parent +# ``` +# +# If any of the printed tables are empty, you know you need to insert into that +# table (or another ancestor up the pipeline) first. This code will not work if +# there are aliases in the table (i.e., `proj` in the definition). In that case, +# you'll need to modify your `parent_key` to reflect the renaming. +# +# The error message itself will tell you which table is the limiting parent. After +# `REFERENCES` in the error message, you'll see the parent table and the column +# that is causing the error. +# +# ### Permission +# +# ```console +# ('Insufficient privileges.', "INSERT command denied to user 'username'@'127.0.0.1' for table '_table_name'", 'INSERT INTO `schema_name`.`table_name`(`field1`,`field2`) VALUES (%s,%s)') +# ``` +# +# This is a MySQL error that means that either ... +# +# - You don't have access to the command you're trying to run (e.g., `INSERT`) +# - You don't have access to this command on the schema you're trying to run it on +# +# To see what permissions you have, you can run the following ... +# +# ```python +# dj.conn().query("SHOW GRANTS FOR CURRENT_USER();").fetchall() +# ``` +# +# If you think you should have access to the command, you contact your database +# administrator (e.g., Chris in the Frank Lab). Please share the output of the +# above command with them. +# +# ### Type +# +# ```console +# TypeError: example_function() got an unexpected keyword argument 'this_arg' +# ``` +# +# This means that you're calling a function with an argument that it doesn't +# expect (e.g., `example_function(this_arg=5)`). You can check the function's +# accepted arguments by running `help(example_function)`. +# +# ```console +# TypeError: 'NoneType' object is not iterable +# ``` +# +# This means that some function is trying to do something with an object of an +# unexpected type. For example, if might by running `for item in variable: ...` +# when `variable` is `None`. You can check the type of the variable by going into +# debug mode and running `type(variable)`. +# +# ### KeyError +# +# ```console +# KeyError: 'field_name' +# ``` +# +# This means that you're trying to access a key in a dictionary that doesn't +# exist. You can check the keys of the dictionary by running `variable.keys()`. If +# this is in your custom code, you can get a key and supply a default value if it +# doesn't exist by running `variable.get('field_name', default_value)`. +# +# ### DataJoint +# +# ```console +# DataJointError("Attempt to delete part table {part} before deleting from its master {master} first.") +# ``` +# +# This means that DataJoint's delete process found a part table with a foreign key +# reference to the data you're trying to delete. You need to find the master table +# listed and delete from that table first. +# + +# ## Debug Mode +# +# To fix an error, you may want to enter 'debug mode'. VSCode has a dedicated +# featureful [extension](https://code.visualstudio.com/docs/python/debugging) +# for making use of the UI, but you can choose to use Python's built-in tool. +# +# To enter into debug mode, you can add the following line to your code ... +# +# ```python +# __import__("pdb").set_trace() +# ``` +# +# This will set a breakpoint in your code at that line. When you run your code, it +# will pause at that line and you can explore the variables in the current frame. +# Commands in this mode include ... +# +# - `u` and `d` to move up and down the stack +# - `l` to list the code around the current line +# - `q` to quit the debugger +# - `c` to continue running the code +# - `h` for help, which will list all the commands +# +# `ipython` and jupyter notebooks can launch a debugger automatically at the last +# error by running `%debug`. +# +# + +# ## Up Next +# + +# Next, we'll try [inserting data](./01_Insert_Data.ipynb) diff --git a/notebooks/py_scripts/01_Insert_Data.py b/notebooks/py_scripts/02_Insert_Data.py similarity index 81% rename from notebooks/py_scripts/01_Insert_Data.py rename to notebooks/py_scripts/02_Insert_Data.py index f569f971f..fd1e43505 100644 --- a/notebooks/py_scripts/01_Insert_Data.py +++ b/notebooks/py_scripts/02_Insert_Data.py @@ -5,7 +5,7 @@ # extension: .py # format_name: light # format_version: '1.5' -# jupytext_version: 1.15.2 +# jupytext_version: 1.16.0 # kernelspec: # display_name: spy # language: python @@ -186,6 +186,31 @@ sgc.LabTeam.LabTeamMember() +# In general, we can insert into any table in this say, by supplying +# a dictionary (or list of dictionaries) with all the fields mentioned in +# `Table.heading.names` so long as the data types match what is described in +# `Table.heading` +# +# ```python +# Table.insert1({'a': 1, 'b': 'other'}) # only one entry +# Table.insert([{'a':1, 'b': 'other'}, {'a':1, 'b': 'next'}]) # multiple +# ``` +# +# For example ... + +sgc.ProbeType.insert1( + { + "probe_type": "128c-4s6mm6cm-15um-26um-sl", + "probe_description": "A Livermore flexible probe with 128 channels ...", + "manufacturer": "Lawrence Livermore National Lab", + "num_shanks": 4, + }, + skip_duplicates=True, +) + +# The `skip_duplicates` flag tells DataJoint not to raise an error if the data +# is already in the table. This should only be used in special cases. + # ## Inserting from NWB # @@ -470,6 +495,92 @@ # !ls $SPYGLASS_BASE_DIR/raw +# ## YAML Inserts +# +# The following step is an optional feature, and not required for the remaining +# notebooks. +# +# Not every NWB file has all the information required by Spyglass. For example, +# many NWB files do not contain any information about the `DataAcquisitionDevice` +# or `Probe` because NWB does not yet have an official standard for specifying +# them. Or, information in the NWB file may need correcting. +# +# Manual inserts can either be done on tables directly (e.g., +# `Table.insert1(my_dict)`), or done in batch with `yaml` files. This is done in +# two steps: +# +# 1. Generate data to be entered. +# 2. Associate data with one or more NWB files. +# + +# ### Batch Insert +# +# First, Spyglass will check for an `entries.yaml` file at the base directory +# (see [Setup](./00_Setup.ipynb)) and run all corresponding inserts. +# This is a great place to define entries that the database should auto-insert +# prior to ingesting any NWB files. An example can be found in +# `examples/config_yaml/entries.yaml`. It has the following structure: +# +# ```yaml +# TableName: +# - TableEntry1Field1: Value +# +# TableEntry1Field2: +# - TableEntry2Field1: Value +# +# TableEntry2Field2: Value +# ``` +# +# For example, +# +# ```yaml +# ProbeType: +# - probe_type: 128c-4s6mm6cm-15um-26um-sl +# probe_description: A Livermore flexible probe with 128 channels, 4 shanks, +# 6 mm shank length, 6 cm ribbon length. 15 um contact diameter, 26 um +# center-to-center distance (pitch), single-line configuration. +# manufacturer: Lawrence Livermore National Lab +# num_shanks: 4 +# ``` +# +# Using a YAML file over data stored in Python scripts helps maintain records +# of data entries in a human-readable file. For ways to share a state of the +# database, see our [export tutorial](./05_Export.ipynb). +# + +# ### Pairing with NWBs +# +# Next, we'll need to create a _configuration file_ to associate the above entries +# with session(s). This must be done in the same directory as the NWB file that it +# configures and have the following naming convention: +# `_spyglass_config.yaml`. This file is then read by Spyglass +# when calling `insert_session` on the associated NWB file. +# +# An example of this can be found at +# `examples/config_yaml/​​sub-AppleBottom_ses-AppleBottom-DY20-g3_behavior+ecephys_spyglass_config.yaml`. +# +# This file is associated with the NWB file +# `sub-AppleBottom_ses-AppleBottom-DY20-g3_behavior+ecephys.nwb`. +# +# This is the general format for the config entry: +# +# ```yaml +# TableName: +# - primary_key1: value1 +# ``` +# +# For example: +# +# ```yaml +# DataAcquisitionDevice: +# - data_acquisition_device_name: Neuropixels Recording Device +# ``` +# +# In this example, the NWB file that corresponds to this config YAML will become +# associated with the DataAcquisitionDevice with primary key +# data_acquisition_device_name: Neuropixels Recording Device. This entry must +# already exist. + # ## Up Next # diff --git a/notebooks/py_scripts/02_Data_Sync.py b/notebooks/py_scripts/03_Data_Sync.py similarity index 100% rename from notebooks/py_scripts/02_Data_Sync.py rename to notebooks/py_scripts/03_Data_Sync.py diff --git a/notebooks/py_scripts/03_Merge_Tables.py b/notebooks/py_scripts/04_Merge_Tables.py similarity index 100% rename from notebooks/py_scripts/03_Merge_Tables.py rename to notebooks/py_scripts/04_Merge_Tables.py diff --git a/notebooks/py_scripts/04_PopulateConfigFile.py b/notebooks/py_scripts/04_PopulateConfigFile.py deleted file mode 100644 index 74ec39571..000000000 --- a/notebooks/py_scripts/04_PopulateConfigFile.py +++ /dev/null @@ -1,194 +0,0 @@ -# --- -# jupyter: -# jupytext: -# text_representation: -# extension: .py -# format_name: light -# format_version: '1.5' -# jupytext_version: 1.16.0 -# kernelspec: -# display_name: Python 3 (ipykernel) -# language: python -# name: python3 -# --- - -# # Customizing Data Insertion into Spyglass -# -# If you would like to insert data into Spyglass that does not -# follow the naming or organizational format expected by Spyglass, -# or you would like to override what values are ingested into Spyglass from -# your NWB files, including missing values, follow this guide. -# -# ## General Approach -# -# When an NWB file is ingested into Spyglass, metadata about the session -# is first read from the NWB file and inserted into -# tables in the `common` module (e.g. `Institution`, `Lab`, `Electrode`, etc). -# However, not every NWB file has all the information required by Spyglass or -# the information in the NWB file is not in a format that Spyglass expects. For -# example, many NWB files do not contain information about the -# `DataAcquisitionDevice` or `Probe` because the NWB data standard does not yet -# have an official -# standard for specifying them. For these cases, we provide a way to customize -# how data is ingested into Spyglass. -# -# Let's say that you want to ingest an NWB file into Spyglass where the lab name -# in the file is written as "Loren Frank Lab" or it is not specified, but you -# know the data comes from the Loren Frank Lab. Let's say that in Spyglass, -# the lab name that is associated with sessions from the Loren Frank Lab is -# "Frank Lab" and you would like to use the same name in order to facilitate -# data search in Spyglass. To change the lab name when you insert your new data -# to Spyglass, you could either 1) edit the NWB file directly and then -# insert it into Spyglass, or 2) define an override value "Frank Lab" to be -# used instead of the value specified in the NWB file (or lack thereof). -# -# Note that if this is your own data and you want to make changes to -# information about how the data is interpreted, e.g., the units of measurement -# are incorrect, we recommend that you edit the NWB file directly because the -# file or derivatives of it might eventually be shared outside of Spyglass and -# they will not reflect any modifications that you have made to -# the data only in Spyglass. - -# ## Define a Configuration YAML File -# -# To override values in the NWB file during insertion into Spyglass, -# you will need to create a configuration -# YAML file that lives in the same directory as your NWB file, named: -# `_spyglass_config.yaml` -# -# An example configuration YAML file can be found at -# `examples/config_yaml/​​sub-AppleBottom_ses-AppleBottom-DY20-g3_behavior+ecephys_spyglass_config.yaml`. -# This file is associated with the NWB file -# `sub-AppleBottom_ses-AppleBottom-DY20-g3_behavior+ecephys.nwb`. -# -# This is the general format for entries in this configuration file: -# -# ```yaml -# TableName: -# - primary_key1: value1 -# ``` -# -# For example: -# -# ```yaml -# Lab: -# - lab_name: Frank Lab -# ``` -# -# In this example, the NWB file that corresponds to this config YAML will become -# associated with the entry in the `Lab` table with the value `Frank Lab` for -# the primary key `lab_name`. This entry must already exist. More specifically, -# when the NWB file is ingested into Spyglass, -# a new `Session` entry will be created for the NWB file that has a foreign key to -# the `Lab` entry with `lab_name` = "Frank Lab", ignoring whatever lab value is -# in the NWB file, even if one does not exist. -# -# TODO implement this for `Lab`. -# - -# ## Create Entries to Reference in the Configuration YAML -# -# As mentioned earlier, the table entry that you want to associate with your NWB -# file must already exist in the database. This entry would typically be a value -# that is independent of any particular NWB file, such as -# `DataAcquisitionDevice`, `Lab`, `Probe`, and `BrainRegion`. -# -# If the entry does not already exist, you can either: -# 1) create the entry programmatically using DataJoint `insert` commands, or -# 2) define the entry in a YAML file called `entries.yaml` that is automatically -# processed when Spyglass is imported. You can think of `entries.yaml` as a -# place to define information that the database should come pre-equipped prior -# to ingesting your NWB files. The `entries.yaml` file should be placed in the -# `spyglass` base directory (next to `README.md`). An example can be found in -# `examples/config_yaml/entries.yaml`. This file should have the following -# structure: -# -# ```yaml -# TableName: -# - TableEntry1Field1: Value -# TableEntry1Field2: Value -# - TableEntry2Field1: Value -# TableEntry2Field2: Value -# ``` -# -# For example, -# -# ```yaml -# DataAcquisitionDeviceSystem: -# data_acquisition_device_system: SpikeGLX -# DataAcquisitionDevice: -# - data_acquisition_device_name: Neuropixels_SpikeGLX -# data_acquisition_device_system: SpikeGLX -# data_acquisition_device_amplifier: Intan -# ``` -# -# Only `dj.Manual`, `dj.Lookup`, and `dj.Part` tables can be populated -# using this approach. -# -# Once the entry that you want to associate with your NWB file exists in the -# database, you can write the configuration YAML file and then ingest your -# NWB file. As an another example, let's say that you want to associate your NWB -# file with the `DataAcquisitionDevice` entry with `data_acquisition_device_name` -# = "Neuropixels_SpikeGLX" that was defined above. You would write the following -# configuration YAML file: -# -# ```yaml -# DataAcquisitionDevice: -# - data_acquisition_device_name: Neuropixels_SpikeGLX -# ``` -# -# The example in -# `examples/config_yaml/​​sub-AppleBottom_ses-AppleBottom-DY20-g3_behavior+ecephys_spyglass_config.yaml` -# includes additional examples. - -# ## Example Ingestion with Real Data -# -# For this example, you will need to download the 5 GB NWB file -# `sub-JDS-NFN-AM2_behavior+ecephys.nwb` -# from dandiset 000447 here: -# https://dandiarchive.org/dandiset/000447/0.230316.2133/files?location=sub-JDS-NFN-AM2&page=1 -# -# Click the download arrow button to download the file to your computer. Add it to the folder -# containing your raw NWB data to be ingested into Spyglass. -# -# This file does not specify a data acquisition device. Let's say that the -# data was collected from a SpikeGadgets system with an Intan amplifier. This -# matches an existing entry in the `DataAcquisitionDevice` table with name -# "data_acq_device0". We will create a configuration YAML file to associate -# this entry with the NWB file. -# -# If you are connected to the Frank lab database, please rename any downloaded -# files (e.g., `example20200101_yourname.nwb`) to avoid naming collisions, as the -# file name acts as the primary key across key tables. - -nwb_file_name = "sub-JDS-NFN-AM2_behavior+ecephys_rly.nwb" - -# this configuration yaml file should be placed next to the downloaded NWB file -yaml_config_path = "sub-JDS-NFN-AM2_behavior+ecephys_rly_spyglass_config.yaml" -with open(yaml_config_path, "w") as config_file: - lines = [ - "DataAcquisitionDevice", - "- data_acquisition_device_name: data_acq_device0", - ] - config_file.writelines(line + "\n" for line in lines) - -# Then call `insert_sessions` as usual. - -# + -import spyglass.data_import as sgi - -sgi.insert_sessions(nwb_file_name) -# - - -# Confirm the session was inserted with the correct `DataAcquisitionDevice` - -# + -import spyglass.common as sgc -from spyglass.utils.nwb_helper_fn import get_nwb_copy_filename - -nwb_copy_file_name = get_nwb_copy_filename(nwb_file_name) - -sgc.Session.DataAcquisitionDevice & {"nwb_file_name": nwb_copy_file_name} -# - - -# diff --git a/notebooks/py_scripts/10_Spike_SortingV0.py b/notebooks/py_scripts/10_Spike_SortingV0.py index 2675799db..670f46559 100644 --- a/notebooks/py_scripts/10_Spike_SortingV0.py +++ b/notebooks/py_scripts/10_Spike_SortingV0.py @@ -5,14 +5,18 @@ # extension: .py # format_name: light # format_version: '1.5' -# jupytext_version: 1.15.2 +# jupytext_version: 1.16.0 # kernelspec: # display_name: Python 3.10.5 64-bit # language: python # name: python3 # --- -# # Spike Sorting +# # Spike Sorting V0 +# +# _Note_: This notebook explains the first version of the spike sorting pipeline +# and is preserved for using existing data. New users should use +# [V1](./10_Spike_SortingV1.ipynb). # # ## Overview diff --git a/notebooks/py_scripts/21_DLC.py b/notebooks/py_scripts/21_DLC.py index 8a55441e8..5366c38ca 100644 --- a/notebooks/py_scripts/21_DLC.py +++ b/notebooks/py_scripts/21_DLC.py @@ -756,6 +756,35 @@ sgp.DLCPosVideo().populate(dlc_key) +#
On editing parameters +# +# The presence of existing parameters in many tables makes it easy to tweak them +# for your needs. You can fetch, edit, and re-insert new params - but the process +# will look a little different if the table has a `=BLOB=` field. +# +# (These example assumes only one primary key. If multiple, `{'primary_key': 'x'}` +# and `['primary_key']` will need to be adjusted accordingly.) +# +# No blob means that all parameters are fields in the table. +# +# ```python +# existing_params = (MyParamsTable & {'primary_key':'x'}).fetch1() +# new_params = {**existing_params, 'primary_key': 'y', 'my_variable': 'a', 'other_variable':'b'} +# MyParamsTable.insert1(new_params) +# ``` +# +# A blob means that the params are stored as an embedded dictionary. We'll assume +# this column is called `params` +# +# ```python +# existing_params = (MyParamsTable & {'primary_key':'x'}).fetch1() +# new_params = {**existing_params, 'primary_key': 'y'} +# print(existing_params['params']) # check existing values +# new_params['params'] = {**existing_params['params'], 'my_variable': 'a', 'other_variable':'b'} +# ``` +# +#
+ # #### [PositionOutput](#TableOfContents) # diff --git a/pyproject.toml b/pyproject.toml index 061947e3d..2b877597d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -127,7 +127,7 @@ minversion = "7.0" addopts = [ # "-sv", # no capture, verbose output "--sw", # stepwise: resume with next test after failure - "--pdb", # drop into debugger on failure + # "--pdb", # drop into debugger on failure "-p no:warnings", # "--no-teardown", # don't teardown the database after tests # "--quiet-spy", # don't show logging from spyglass diff --git a/src/spyglass/decoding/v1/clusterless.py b/src/spyglass/decoding/v1/clusterless.py index f9e149df1..5b128685e 100644 --- a/src/spyglass/decoding/v1/clusterless.py +++ b/src/spyglass/decoding/v1/clusterless.py @@ -446,8 +446,6 @@ def fetch_spike_data(key, filter_by_interval=True): filter_by_interval : bool, optional Whether to filter for spike times in the model interval. Default True - time_slice : Slice, optional - User provided slice of time to restrict spikes to. Default None Returns ------- diff --git a/src/spyglass/position/v1/dlc_utils_makevid.py b/src/spyglass/position/v1/dlc_utils_makevid.py index cc4cadc0d..12e9baeb0 100644 --- a/src/spyglass/position/v1/dlc_utils_makevid.py +++ b/src/spyglass/position/v1/dlc_utils_makevid.py @@ -119,7 +119,7 @@ def make_video(self): def _init_video(self): logger.info(f"Making video: {self.output_video_filename}") - self.video = cv2.VideoCapture(self.video_filename) + self.video = cv2.VideoCapture(str(self.video_filename)) self.frame_size = ( (int(self.video.get(3)), int(self.video.get(4))) if not self.crop diff --git a/src/spyglass/utils/dj_helper_fn.py b/src/spyglass/utils/dj_helper_fn.py index d9465fffa..9dfa6f02d 100644 --- a/src/spyglass/utils/dj_helper_fn.py +++ b/src/spyglass/utils/dj_helper_fn.py @@ -487,7 +487,7 @@ def populate_pass_function(value): Parameters ---------- value : (table, key, kwargs) - Class of table to populate, key to populate, and kwargs for populate + Class of table to populate, key to populate, and kwargs for populate """ table, key, kwargs = value return table.populate(key, **kwargs) diff --git a/src/spyglass/utils/dj_mixin.py b/src/spyglass/utils/dj_mixin.py index 6eee80e40..8481c4d30 100644 --- a/src/spyglass/utils/dj_mixin.py +++ b/src/spyglass/utils/dj_mixin.py @@ -4,6 +4,7 @@ from functools import cached_property from inspect import stack as inspect_stack from os import environ +from re import match as re_match from time import time from typing import Dict, List, Union @@ -736,7 +737,18 @@ def _spyglass_version(self): """Get Spyglass version.""" from spyglass import __version__ as sg_version - return ".".join(sg_version.split(".")[:3]) # Major.Minor.Patch + ret = ".".join(sg_version.split(".")[:3]) # Ditch commit info + + if self._test_mode: + return ret[:16] if len(ret) > 16 else ret + + if not bool(re_match(r"^\d+\.\d+\.\d+", ret)): # Major.Minor.Patch + raise ValueError( + f"Spyglass version issues. Expected #.#.#, Got {ret}." + + "Please try running `hatch build` from your spyglass dir." + ) + + return ret @cached_property def _export_table(self): diff --git a/tests/position/test_dlc_cent.py b/tests/position/test_dlc_cent.py index 7980a2b30..fb3687cef 100644 --- a/tests/position/test_dlc_cent.py +++ b/tests/position/test_dlc_cent.py @@ -59,8 +59,9 @@ def test_centroid_calcs(key, sgp): df, max_LED_separation=100, points={p: p for p in points} ).centroid - assert np.all(ret[:-1] == 1), f"Centroid calculation failed for {key}" - assert np.all(np.isnan(ret[-1])), f"Centroid calculation failed for {key}" + fail_msg = f"Centroid calculation failed for {key}" + assert np.all(ret[:-1] == 1), fail_msg + assert np.all(np.isnan(ret[-1])), fail_msg def test_centroid_error(sgp): diff --git a/tests/position/test_dlc_proj.py b/tests/position/test_dlc_proj.py index 7eaba196d..0ca4bd1bb 100644 --- a/tests/position/test_dlc_proj.py +++ b/tests/position/test_dlc_proj.py @@ -57,7 +57,10 @@ def test_failed_name_insert( ), "Project re-insert did not return expected key" -def test_failed_group_insert(dlc_project_tbl, new_project_key): +@pytest.mark.usefixtures("skipif_no_dlc") +def test_failed_group_insert(no_dlc, dlc_project_tbl, new_project_key): + if no_dlc: # Decorator wasn't working here, so duplicate skipif + pytest.skip(reason="Skipping DLC-dependent tests.") with pytest.raises(ValueError): dlc_project_tbl.insert_new_project(**new_project_key) From 9cd32e79ba19e7b9eac4dded6a1535274876e491 Mon Sep 17 00:00:00 2001 From: Samuel Bray Date: Tue, 6 Aug 2024 09:31:42 -0700 Subject: [PATCH 27/94] Fix interpolation of nans in decoding position (#1033) * fix interpolation of nans in decoding position * fix interpolation of nans in PositionGroup * lint * disable ruff * cleanup * update changelog * restrict lack of nan interpolation to position_variable_names * update docstring * update df.values to df.to_numpy --------- Co-authored-by: Eric Denovellis --- CHANGELOG.md | 1 + src/spyglass/decoding/v1/clusterless.py | 15 +++++----- src/spyglass/decoding/v1/core.py | 34 +++++++++++++++++++---- src/spyglass/decoding/v1/sorted_spikes.py | 17 ++++++------ 4 files changed, 45 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba21ee169..fecef73e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,7 @@ PositionGroup.alter() - Default values for classes on `ImportError` #966 - Add option to upsample data rate in `PositionGroup` #1008 + - Avoid interpolating over large `nan` intervals in position #1033 - Position diff --git a/src/spyglass/decoding/v1/clusterless.py b/src/spyglass/decoding/v1/clusterless.py index 5b128685e..bbb3b3418 100644 --- a/src/spyglass/decoding/v1/clusterless.py +++ b/src/spyglass/decoding/v1/clusterless.py @@ -132,6 +132,9 @@ def make(self, key): position_info.index <= interval_end, ) ] = True + is_training[ + position_info[position_variable_names].isna().values.max(axis=1) + ] = False if "is_training" not in decoding_kwargs: decoding_kwargs["is_training"] = is_training @@ -426,14 +429,10 @@ def fetch_linear_position_info(key): min_time, max_time = ClusterlessDecodingV1._get_interval_range(key) - return ( - pd.concat( - [linear_position_df.set_index(position_df.index), position_df], - axis=1, - ) - .loc[min_time:max_time] - .dropna(subset=position_variable_names) - ) + return pd.concat( + [linear_position_df.set_index(position_df.index), position_df], + axis=1, + ).loc[min_time:max_time] @staticmethod def fetch_spike_data(key, filter_by_interval=True): diff --git a/src/spyglass/decoding/v1/core.py b/src/spyglass/decoding/v1/core.py index de733bb01..a5236d30e 100644 --- a/src/spyglass/decoding/v1/core.py +++ b/src/spyglass/decoding/v1/core.py @@ -176,6 +176,7 @@ def fetch_position_info( PositionOutput & {"merge_id": pos_merge_id} ).fetch1_dataframe(), upsampling_sampling_rate=upsample_rate, + position_variable_names=position_variable_names, ) ) else: @@ -189,11 +190,7 @@ def fetch_position_info( min_time = min([df.index.min() for df in position_info]) if max_time is None: max_time = max([df.index.max() for df in position_info]) - position_info = ( - pd.concat(position_info, axis=0) - .loc[min_time:max_time] - .dropna(subset=position_variable_names) - ) + position_info = pd.concat(position_info, axis=0).loc[min_time:max_time] return position_info, position_variable_names @@ -202,6 +199,7 @@ def _upsample( position_df: pd.DataFrame, upsampling_sampling_rate: float, upsampling_interpolation_method: str = "linear", + position_variable_names: list[str] = None, ) -> pd.DataFrame: """upsample position data to a fixed sampling rate @@ -213,6 +211,9 @@ def _upsample( sampling rate to upsample to upsampling_interpolation_method : str, optional pandas method for interpolation, by default "linear" + position_variable_names : list[str], optional + names of position variables of focus, for which nan values will not be + interpolated, by default None includes all columns Returns ------- @@ -239,10 +240,33 @@ def _upsample( np.unique(np.concatenate((position_df.index, new_time))), name="time", ) + + # Find NaN intervals + nan_intervals = {} + if position_variable_names is None: + position_variable_names = position_df.columns + for column in position_variable_names: + is_nan = position_df[column].isna().to_numpy().astype(int) + st = np.where(np.diff(is_nan) == 1)[0] + 1 + en = np.where(np.diff(is_nan) == -1)[0] + if is_nan[0]: + st = np.insert(st, 0, 0) + if is_nan[-1]: + en = np.append(en, len(is_nan) - 1) + st = position_df.index[st].to_numpy() + en = position_df.index[en].to_numpy() + nan_intervals[column] = list(zip(st, en)) + + # upsample and interpolate position_df = ( position_df.reindex(index=new_index) .interpolate(method=upsampling_interpolation_method) .reindex(index=new_time) ) + # Fill NaN intervals + for column, intervals in nan_intervals.items(): + for st, en in intervals: + position_df[column][st:en] = np.nan + return position_df diff --git a/src/spyglass/decoding/v1/sorted_spikes.py b/src/spyglass/decoding/v1/sorted_spikes.py index 9dbdd015b..c70c6617b 100644 --- a/src/spyglass/decoding/v1/sorted_spikes.py +++ b/src/spyglass/decoding/v1/sorted_spikes.py @@ -99,6 +99,10 @@ def make(self, key): position_info.index <= interval_end, ) ] = True + is_training[ + position_info[position_variable_names].isna().values.max(axis=1) + ] = False + if "is_training" not in decoding_kwargs: decoding_kwargs["is_training"] = is_training @@ -387,15 +391,10 @@ def fetch_linear_position_info(key): edge_spacing=environment.edge_spacing, ) min_time, max_time = SortedSpikesDecodingV1._get_interval_range(key) - - return ( - pd.concat( - [linear_position_df.set_index(position_df.index), position_df], - axis=1, - ) - .loc[min_time:max_time] - .dropna(subset=position_variable_names) - ) + return pd.concat( + [linear_position_df.set_index(position_df.index), position_df], + axis=1, + ).loc[min_time:max_time] @staticmethod def fetch_spike_data( From 73816e3eb2acf095945385ae915898ba1b1e87db Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Tue, 6 Aug 2024 11:32:18 -0500 Subject: [PATCH 28/94] Export updates (#1048) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * WIP: Export revision * WIP: Export Revision 2 * ✅ : Export revision * Update changelog * Add to export notebook --- CHANGELOG.md | 2 + notebooks/05_Export.ipynb | 153 ++++++++++++++++----- notebooks/py_scripts/05_Export.py | 132 +++++++++++++----- pyproject.toml | 2 + src/spyglass/common/common_dandi.py | 43 ++++-- src/spyglass/common/common_usage.py | 169 ++--------------------- src/spyglass/settings.py | 1 + src/spyglass/utils/dj_mixin.py | 27 +++- src/spyglass/utils/sql_helper_fn.py | 205 ++++++++++++++++++++++++++++ 9 files changed, 491 insertions(+), 243 deletions(-) create mode 100644 src/spyglass/utils/sql_helper_fn.py diff --git a/CHANGELOG.md b/CHANGELOG.md index fecef73e9..f892611a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ PositionGroup.alter() - Revise docs organization. - Misc -> Features/ForDevelopers. #1029 - Installation instructions -> Setup notebook. #1029 +- Migrate SQL export tools to `utils` to support exporting `DandiPath` #1048 ### Pipelines @@ -89,6 +90,7 @@ PositionGroup.alter() - Add `UnitAnnotation` table and naming convention for units #1027, #1052 - Set `sparse` parameter to waveform extraction step in `spikesorting.v1` #1039 + ## [0.5.2] (April 22, 2024) ### Infrastructure diff --git a/notebooks/05_Export.ipynb b/notebooks/05_Export.ipynb index 592dc3e27..6290d3596 100644 --- a/notebooks/05_Export.ipynb +++ b/notebooks/05_Export.ipynb @@ -40,6 +40,41 @@ "- Inherit `SpyglassMixin` for all custom tables.\n", "- Run only one export at a time.\n", "- Start and stop each export logging process.\n", + "- Do not update Spyglass until the export is complete.\n", + "\n", + "
How to inherit SpyglassMixin\n", + "\n", + "DataJoint tables all inherit from one of the built-in table types.\n", + "\n", + "```python\n", + "class MyTable(dj.Manual):\n", + " ...\n", + "```\n", + "\n", + "To inherit the mixin, simply add it to the `()` of the class before the\n", + "DataJoint class. This can be done for existing tables without dropping them,\n", + "so long as the change has been made prior to export logging.\n", + "\n", + "```python\n", + "from spyglass.utils import SpyglassMixin\n", + "class MyTable(SpyglassMixin, dj.Manual):\n", + " ...\n", + "```\n", + "\n", + "
\n", + "\n", + "
Why these limitations?\n", + "\n", + "`SpyglassMixin` is what makes this process possible. It uses an environmental\n", + "variable to make sure all tables are on the same page about the export ID.\n", + "We get this feature by inheriting, but cannot set more that one value for the \n", + "environmental variable.\n", + "\n", + "The export process was designed with reproducibility in mind, and will export\n", + "your conda environment to match. We want to be sure that the analysis you run\n", + "is replicable using the same conda environment.\n", + "\n", + "
\n", "\n", "**NOTE:** For demonstration purposes, this notebook relies on a more populated\n", "database to highlight restriction merging capabilities of the export process.\n", @@ -419,29 +454,8 @@ "\n", "There are a few restrictions to keep in mind when export logging:\n", "\n", - "- You can only run _ONE_ export at a time.\n", - "- All tables must inherit `SpyglassMixin`\n", - "\n", - "
How to inherit SpyglassMixin\n", - "\n", - "DataJoint tables all inherit from one of the built-in table types.\n", - "\n", - "```python\n", - "class MyTable(dj.Manual):\n", - " ...\n", - "```\n", - "\n", - "To inherit the mixin, simply add it to the `()` of the class before the\n", - "DataJoint class. This can be done for existing tables without dropping them,\n", - "so long as the change has been made prior to export logging.\n", - "\n", - "```python\n", - "from spyglass.utils import SpyglassMixin\n", - "class MyTable(SpyglassMixin, dj.Manual):\n", - " ...\n", - "```\n", - "\n", - "
\n", + "- _ONE_ export at a time. \n", + "- All tables must inherit `SpyglassMixin`. \n", "\n", "Let's start logging for 'paper1'.\n" ] @@ -747,28 +761,35 @@ "By default the export script will be located in an `export` folder within your\n", "`SPYGLASS_BASE_DIR`. This default can be changed by adjusting your `dj.config`.\n", "\n", - "Frank Lab members will need the help of a database admin (e.g., Chris) to\n", - "run the resulting bash script. The result will be a `.sql` file that anyone\n", - "can use to replicate the database entries you used in your analysis.\n" + "Depending on your database's configuration, you may need an admin on your team\n", + "to run the resulting bash script. This is true of the Frank Lab. Doing so will\n", + "result will be a `.sql` file that anyone can use to replicate the database\n", + "entries you used in your analysis.\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "# Dandiset Upload" + "## Dandi" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "One benefit of the `Export` table is it provides a list of all raw data, intermediate analysis files, \n", - "and final analysis files needed to generate a set of figures in a work. To aid in data-sharing standards,\n", - "we have implemented tools to compile and upload this set of files as a Dandi dataset, which can then be used\n", - "by spyglass to directly read the data from the Dandi database if not available locally. \n", + "One benefit of the `Export` table is it provides a list of all raw data,\n", + "intermediate analysis files, and final analysis files needed to generate a set\n", + "of figures in a work. To aid in data-sharing standards, we have implemented\n", + "an optional additional export step with \n", + "tools to compile and upload this set of files as a Dandi dataset, which can then\n", + "be used by Spyglass to directly read the data from the Dandi database if not\n", + "available locally. \n", "\n", - "We will walk through the steps to do so here:" + "We will walk through the steps to do so here:\n", + "1. Upload the data\n", + "2. Export this table alongside the previous export\n", + "3. Generate a sharable docker container (Coming soon!)" ] }, { @@ -776,7 +797,7 @@ "metadata": {}, "source": [ "
\n", - " Dandi data compliance (admins)\n", + " Dandi data compliance (admins)\n", "\n", " >__WARNING__: The following describes spyglass utilities that require database admin privileges to run. It involves altering database values to correct for metadata format errors generated prior to spyglass insert. As such it has the potential to violate data integrity and should be used with caution.\n", " >\n", @@ -793,6 +814,13 @@ "\n" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Dandiset Upload" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -947,9 +975,62 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "When fetching data with spyglass, if a file is not available locally, syglass will automatically use \n", - "this information to stream the file from Dandi's server if available, providing an additional method\n", - " for sharing data with collaborators post-publication." + "When fetching data with Spyglass, if a file is not available locally, Syglass\n", + "will automatically use this information to stream the file from Dandi's server\n", + " if available, providing an additional method for sharing data with\n", + " collaborators post-publication." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Export Dandi Table\n", + "\n", + "Because we generated new entries in this process we may want to share alongside\n", + "our export, we'll run the additional step of exporting this table as well.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "DandiPath().write_mysqldump(paper_key)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Sharing the export\n", + "\n", + "The steps above will generate several files in this paper's export directory.\n", + "By default, this is relative to your Spyglass base directory:\n", + "`{BASE_DIR}/export/{PAPER_ID}`. \n", + "\n", + "The `.sh` files should be run by a database administrator who is familiar with\n", + "running `mysqldump` commands.\n", + "\n", + "
Note to administrators\n", + "\n", + "The dump process saves the exporter's credentials as a `.my.cnf` file \n", + "([about these files](https://dev.mysql.com/doc/refman/8.4/en/option-files.html))\n", + "to allow running `mysqldump` without additional flags for user, password, etc.\n", + "\n", + "If database permissions permit running exports from the instance that runs the\n", + "exports, you can esure you have a similar `.my.cnf` config in place and run the\n", + "export shell scripts as-is. Some databases, like the one used by the Frank Lab\n", + "have protections in place that would require these script(s) to be run from the\n", + "database instance. Resulting `.sql` files should be placed in the same export\n", + "directory mentioned above.\n", + "\n", + "
\n", + "\n", + "Then, visit the dockerization repository\n", + "[here](https://github.com/LorenFrankLab/spyglass-export-docker)\n", + "and follow the instructions in 'Quick Start'." ] }, { diff --git a/notebooks/py_scripts/05_Export.py b/notebooks/py_scripts/05_Export.py index 2b8d588d3..4f8c376d0 100644 --- a/notebooks/py_scripts/05_Export.py +++ b/notebooks/py_scripts/05_Export.py @@ -5,7 +5,7 @@ # extension: .py # format_name: light # format_version: '1.5' -# jupytext_version: 1.15.2 +# jupytext_version: 1.16.0 # kernelspec: # display_name: spy # language: python @@ -38,6 +38,41 @@ # - Inherit `SpyglassMixin` for all custom tables. # - Run only one export at a time. # - Start and stop each export logging process. +# - Do not update Spyglass until the export is complete. +# +#
How to inherit SpyglassMixin +# +# DataJoint tables all inherit from one of the built-in table types. +# +# ```python +# class MyTable(dj.Manual): +# ... +# ``` +# +# To inherit the mixin, simply add it to the `()` of the class before the +# DataJoint class. This can be done for existing tables without dropping them, +# so long as the change has been made prior to export logging. +# +# ```python +# from spyglass.utils import SpyglassMixin +# class MyTable(SpyglassMixin, dj.Manual): +# ... +# ``` +# +#
+# +#
Why these limitations? +# +# `SpyglassMixin` is what makes this process possible. It uses an environmental +# variable to make sure all tables are on the same page about the export ID. +# We get this feature by inheriting, but cannot set more that one value for the +# environmental variable. +# +# The export process was designed with reproducibility in mind, and will export +# your conda environment to match. We want to be sure that the analysis you run +# is replicable using the same conda environment. +# +#
# # **NOTE:** For demonstration purposes, this notebook relies on a more populated # database to highlight restriction merging capabilities of the export process. @@ -91,29 +126,8 @@ # # There are a few restrictions to keep in mind when export logging: # -# - You can only run _ONE_ export at a time. -# - All tables must inherit `SpyglassMixin` -# -#
How to inherit SpyglassMixin -# -# DataJoint tables all inherit from one of the built-in table types. -# -# ```python -# class MyTable(dj.Manual): -# ... -# ``` -# -# To inherit the mixin, simply add it to the `()` of the class before the -# DataJoint class. This can be done for existing tables without dropping them, -# so long as the change has been made prior to export logging. -# -# ```python -# from spyglass.utils import SpyglassMixin -# class MyTable(SpyglassMixin, dj.Manual): -# ... -# ``` -# -#
+# - _ONE_ export at a time. +# - All tables must inherit `SpyglassMixin`. # # Let's start logging for 'paper1'. # @@ -188,22 +202,29 @@ # By default the export script will be located in an `export` folder within your # `SPYGLASS_BASE_DIR`. This default can be changed by adjusting your `dj.config`. # -# Frank Lab members will need the help of a database admin (e.g., Chris) to -# run the resulting bash script. The result will be a `.sql` file that anyone -# can use to replicate the database entries you used in your analysis. +# Depending on your database's configuration, you may need an admin on your team +# to run the resulting bash script. This is true of the Frank Lab. Doing so will +# result will be a `.sql` file that anyone can use to replicate the database +# entries you used in your analysis. # -# # Dandiset Upload +# ## Dandi -# One benefit of the `Export` table is it provides a list of all raw data, intermediate analysis files, -# and final analysis files needed to generate a set of figures in a work. To aid in data-sharing standards, -# we have implemented tools to compile and upload this set of files as a Dandi dataset, which can then be used -# by spyglass to directly read the data from the Dandi database if not available locally. +# One benefit of the `Export` table is it provides a list of all raw data, +# intermediate analysis files, and final analysis files needed to generate a set +# of figures in a work. To aid in data-sharing standards, we have implemented +# an optional additional export step with +# tools to compile and upload this set of files as a Dandi dataset, which can then +# be used by Spyglass to directly read the data from the Dandi database if not +# available locally. # # We will walk through the steps to do so here: +# 1. Upload the data +# 2. Export this table alongside the previous export +# 3. Generate a sharable docker container (Coming soon!) #
-# Dandi data compliance (admins) +# Dandi data compliance (admins) # # >__WARNING__: The following describes spyglass utilities that require database admin privileges to run. It involves altering database values to correct for metadata format errors generated prior to spyglass insert. As such it has the potential to violate data integrity and should be used with caution. # > @@ -220,6 +241,8 @@ # # +# ### Dandiset Upload + # The first step you will need to do is to [create a Dandi account](https://www.dandiarchive.org/handbook/16_account/). # With this account you can then [register a new dandiset](https://dandiarchive.org/dandiset/create) by providing a name and basic metadata. # Dandi's instructions for these steps are available [here](https://www.dandiarchive.org/handbook/13_upload/). @@ -249,9 +272,46 @@ DandiPath() & {"export_id": 14} -# When fetching data with spyglass, if a file is not available locally, syglass will automatically use -# this information to stream the file from Dandi's server if available, providing an additional method -# for sharing data with collaborators post-publication. +# When fetching data with Spyglass, if a file is not available locally, Syglass +# will automatically use this information to stream the file from Dandi's server +# if available, providing an additional method for sharing data with +# collaborators post-publication. + +# ### Export Dandi Table +# +# Because we generated new entries in this process we may want to share alongside +# our export, we'll run the additional step of exporting this table as well. +# + +DandiPath().write_mysqldump(paper_key) + +# ## Sharing the export +# +# The steps above will generate several files in this paper's export directory. +# By default, this is relative to your Spyglass base directory: +# `{BASE_DIR}/export/{PAPER_ID}`. +# +# The `.sh` files should be run by a database administrator who is familiar with +# running `mysqldump` commands. +# +#
Note to administrators +# +# The dump process saves the exporter's credentials as a `.my.cnf` file +# ([about these files](https://dev.mysql.com/doc/refman/8.4/en/option-files.html)) +# to allow running `mysqldump` without additional flags for user, password, etc. +# +# If database permissions permit running exports from the instance that runs the +# exports, you can esure you have a similar `.my.cnf` config in place and run the +# export shell scripts as-is. Some databases, like the one used by the Frank Lab +# have protections in place that would require these script(s) to be run from the +# database instance. Resulting `.sql` files should be placed in the same export +# directory mentioned above. +# +#
+# +# Then, visit the dockerization repository +# [here](https://github.com/LorenFrankLab/spyglass-export-docker) +# and follow the instructions in 'Quick Start'. # ## Up Next # diff --git a/pyproject.toml b/pyproject.toml index 2b877597d..a3f96c3e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -172,4 +172,6 @@ omit = [ # which submodules have no tests [tool.ruff] # CB: Propose replacing flake8 with ruff to delete setup.cfg line-length = 80 + +[tool.ruff.lint] ignore = ["F401" , "E402", "E501"] diff --git a/src/spyglass/common/common_dandi.py b/src/spyglass/common/common_dandi.py index 3c721368f..5cd4ee872 100644 --- a/src/spyglass/common/common_dandi.py +++ b/src/spyglass/common/common_dandi.py @@ -8,7 +8,10 @@ import pynwb from fsspec.implementations.cached import CachingFileSystem -from spyglass.utils import logger +from spyglass.common.common_usage import Export, ExportSelection +from spyglass.settings import export_dir +from spyglass.utils import SpyglassMixin, logger +from spyglass.utils.sql_helper_fn import SQLDumpHelper try: import dandi.download @@ -23,23 +26,16 @@ except (ImportError, ModuleNotFoundError) as e: ( - dandi.download, - dandi.organize, - dandi.upload, - dandi.validate, + dandi, known_instances, DandiAPIClient, get_metadata, OrganizeInvalid, Severity, - ) = [None] * 9 + ) = [None] * 6 logger.warning(e) -from spyglass.common.common_usage import Export -from spyglass.settings import export_dir -from spyglass.utils import SpyglassMixin, logger - schema = dj.schema("common_dandi") @@ -101,7 +97,8 @@ def compile_dandiset( dandi_api_key : str, optional API key for the dandi server. Optional if the environment variable DANDI_API_KEY is set. - dandi_instance : dandiset's Dandi instance. Defaults to the dev server + dandi_instance : str, optional + What instance of Dandi the dandiset is on. Defaults to dev server. """ key = (Export & key).fetch1("KEY") paper_id = (Export & key).fetch1("paper_id") @@ -122,10 +119,8 @@ def compile_dandiset( # Remove if so to continue for dandi_dir in destination_dir, dandiset_dir: if os.path.exists(dandi_dir): - from datajoint.utils import user_choice - if ( - user_choice( + dj.utils.user_choice( "Pre-existing dandi export dir exist." + f"Delete existing export folder: {dandi_dir}", default="no", @@ -186,6 +181,26 @@ def compile_dandiset( ] self.insert(translations, ignore_extra_fields=True) + def write_mysqldump(self, export_key: dict): + """Write a MySQL dump script to the paper directory for DandiPath.""" + key = (Export & export_key).fetch1("KEY") + paper_id = (Export & key).fetch1("paper_id") + spyglass_version = (ExportSelection & key).fetch( + "spyglass_version", limit=1 + )[0] + + self.compare_versions( + spyglass_version, + msg="Must use same Spyglass version for export and Dandi", + ) + + sql_dump = SQLDumpHelper( + paper_id=paper_id, + docker_id=None, + spyglass_version=spyglass_version, + ) + sql_dump.write_mysqldump(self & key, file_suffix="_dandi") + def _get_metadata(path): # taken from definition within dandi.organize.organize diff --git a/src/spyglass/common/common_usage.py b/src/spyglass/common/common_usage.py index f6f15ce76..354fb92c4 100644 --- a/src/spyglass/common/common_usage.py +++ b/src/spyglass/common/common_usage.py @@ -6,7 +6,6 @@ plan future development of Spyglass. """ -from pathlib import Path from typing import List, Union import datajoint as dj @@ -23,6 +22,7 @@ unique_dicts, update_analysis_for_dandi_standard, ) +from spyglass.utils.sql_helper_fn import SQLDumpHelper schema = dj.schema("common_usage") @@ -275,167 +275,24 @@ def make(self, key): {**key, **fp, "file_id": i} for i, fp in enumerate(file_paths) ] - # Writes but does not run mysqldump. Assumes single version per paper. - version_key = query.fetch("spyglass_version", as_dict=True)[0] - self.write_export( - free_tables=restr_graph.restr_ft, **paper_key, **version_key + version_ids = query.fetch("spyglass_version") + if len(set(version_ids)) > 1: + raise ValueError( + "Multiple versions in ExportSelection\n" + + "Please rerun all analyses with the same version" + ) + self.compare_versions( + version_ids[0], + msg="Must use same Spyglass version for analysis and export", ) + sql_helper = SQLDumpHelper(**paper_key, spyglass_version=version_ids[0]) + sql_helper.write_mysqldump(free_tables=restr_graph.restr_ft) + self.insert1({**key, **paper_key}) self.Table().insert(table_inserts) self.File().insert(file_inserts) - def _get_credentials(self): - """Get credentials for database connection.""" - return { - "user": dj_config["database.user"], - "password": dj_config["database.password"], - "host": dj_config["database.host"], - } - - def _write_sql_cnf(self): - """Write SQL cnf file to avoid password prompt.""" - cnf_path = Path("~/.my.cnf").expanduser() - - if cnf_path.exists(): - return - - template = "[client]\nuser={user}\npassword={password}\nhost={host}\n" - - with open(str(cnf_path), "w") as file: - file.write(template.format(**self._get_credentials())) - cnf_path.chmod(0o600) - - def _bash_escape(self, s): - """Escape restriction string for bash.""" - s = s.strip() - - replace_map = { - "WHERE ": "", # Remove preceding WHERE of dj.where_clause - " ": " ", # Squash double spaces - "( (": "((", # Squash double parens - ") )": ")", - '"': "'", # Replace double quotes with single - "`": "", # Remove backticks - " AND ": " \\\n\tAND ", # Add newline and tab for readability - " OR ": " \\\n\tOR ", # OR extra space to align with AND - ")AND(": ") \\\n\tAND (", - ")OR(": ") \\\n\tOR (", - "#": "\\#", - } - for old, new in replace_map.items(): - s = s.replace(old, new) - if s.startswith("(((") and s.endswith(")))"): - s = s[2:-2] # Remove extra parens for readability - return s - - def _cmd_prefix(self, docker_id=None): - """Get prefix for mysqldump command. Includes docker exec if needed.""" - if not docker_id: - return "mysqldump " - return ( - f"docker exec -i {docker_id} \\\n\tmysqldump " - + "-u {user} --password={password} \\\n\t".format( - **self._get_credentials() - ) - ) - - def _write_mysqldump( - self, - free_tables: List[FreeTable], - paper_id: str, - docker_id=None, - spyglass_version=None, - ): - """Write mysqlmdump.sh script to export data. - - Parameters - ---------- - paper_id : str - Paper ID to use for export file names - docker_id : str, optional - Docker container ID to export from. Default None - spyglass_version : str, optional - Spyglass version to include in export. Default None - """ - paper_dir = Path(export_dir) / paper_id if not docker_id else Path(".") - paper_dir.mkdir(exist_ok=True) - - dump_script = paper_dir / f"_ExportSQL_{paper_id}.sh" - dump_content = paper_dir / f"_Populate_{paper_id}.sql" - - prefix = self._cmd_prefix(docker_id) - version = ( # Include spyglass version as comment in dump - "echo '--'\n" - + f"echo '-- SPYGLASS VERSION: {spyglass_version} --'\n" - + "echo '--'\n\n" - if spyglass_version - else "" - ) - create_cmd = ( - "echo 'CREATE DATABASE IF NOT EXISTS {database}; " - + "USE {database};'\n\n" - ) - dump_cmd = prefix + '{database} {table} --where="\\\n\t{where}"\n\n' - - tables_by_db = sorted(free_tables, key=lambda x: x.full_table_name) - - with open(dump_script, "w") as file: - file.write( - "#!/bin/bash\n\n" - + f"exec > {dump_content}\n\n" # Redirect output to sql file - + f"{version}" # Include spyglass version as comment - ) - - prev_db = None - for table in tables_by_db: - if not (where := table.where_clause()): - continue - where = self._bash_escape(where) - database, table_name = ( - table.full_table_name.replace("`", "") - .replace("#", "\\#") - .split(".") - ) - if database != prev_db: - file.write(create_cmd.format(database=database)) - prev_db = database - file.write( - dump_cmd.format( - database=database, table=table_name, where=where - ) - ) - logger.info(f"Export script written to {dump_script}") - - def write_export( - self, - free_tables: List[FreeTable], - paper_id: str, - docker_id=None, - spyglass_version=None, - ): - """Write export bash script for all tables in graph. - - Also writes a user-specific .my.cnf file to avoid password prompt. - - Parameters - ---------- - free_tables : List[FreeTable] - List of restricted FreeTables to export - paper_id : str - Paper ID to use for export file names - docker_id : str, optional - Docker container ID to export from. Default None - spyglass_version : str, optional - Spyglass version to include in export. Default None - """ - self._write_sql_cnf() - self._write_mysqldump( - free_tables, paper_id, docker_id, spyglass_version - ) - - # TODO: export conda env - def prepare_files_for_export(self, key, **kwargs): """Resolve common known errors to make a set of analysis files dandi compliant diff --git a/src/spyglass/settings.py b/src/spyglass/settings.py index d9d469bba..fceb0beac 100644 --- a/src/spyglass/settings.py +++ b/src/spyglass/settings.py @@ -232,6 +232,7 @@ def _load_env_vars(self) -> dict: def _set_env_with_dict(self, env_dict) -> None: # NOTE: Kept for backwards compatibility. Should be removed in future # for custom paths. Keep self.env_defaults. + # SPYGLASS_BASE_DIR may be used for docker assembly of export for var, val in env_dict.items(): os.environ[var] = str(val) diff --git a/src/spyglass/utils/dj_mixin.py b/src/spyglass/utils/dj_mixin.py index 8481c4d30..c6959219f 100644 --- a/src/spyglass/utils/dj_mixin.py +++ b/src/spyglass/utils/dj_mixin.py @@ -16,10 +16,11 @@ from datajoint.table import Table from datajoint.utils import get_master, to_camel_case, user_choice from networkx import NetworkXError +from packaging.version import parse as version_parse from pymysql.err import DataError from spyglass.utils.database_settings import SHARED_MODULES -from spyglass.utils.dj_helper_fn import ( # NonDaemonPool, +from spyglass.utils.dj_helper_fn import ( NonDaemonPool, fetch_nwb, get_nwb_table, @@ -750,6 +751,30 @@ def _spyglass_version(self): return ret + def compare_versions( + self, version: str, other: str = None, msg: str = None + ) -> None: + """Compare two versions. Raise error if not equal. + + Parameters + ---------- + version : str + Version to compare. + other : str, optional + Other version to compare. Default None. Use self._spyglass_version. + msg : str, optional + Additional error message info. Default None. + """ + if self._test_mode: + return + + other = other or self._spyglass_version + + if version_parse(version) != version_parse(other): + raise RuntimeError( + f"Found mismatched versions: {version} vs {other}\n{msg}" + ) + @cached_property def _export_table(self): """Lazy load export selection table.""" diff --git a/src/spyglass/utils/sql_helper_fn.py b/src/spyglass/utils/sql_helper_fn.py new file mode 100644 index 000000000..4735125d5 --- /dev/null +++ b/src/spyglass/utils/sql_helper_fn.py @@ -0,0 +1,205 @@ +from os import system as os_system +from pathlib import Path +from typing import List + +import yaml +from datajoint import FreeTable +from datajoint import config as dj_config + +from spyglass.settings import export_dir +from spyglass.utils import logger + + +class SQLDumpHelper: + """Write a series of export files to export_dir/paper_id. + + Includes.. + - .my.cnf file to avoid future password prompt + - bash script to export data from MySQL database + - environment.yml file to export conda environment + + Parameters + ---------- + free_tables : List[FreeTable] + List of FreeTables to export + paper_id : str + Paper ID to use for export file names + docker_id : str, optional + Docker container ID to export from. Default None + spyglass_version : str, optional + Spyglass version to include in export. Default None + """ + + def __init__( + self, + paper_id: str, + docker_id=None, + spyglass_version=None, + ): + self.paper_id = paper_id + self.docker_id = docker_id + self.spyglass_version = spyglass_version + + def _get_credentials(self): + """Get credentials for database connection.""" + return { + "user": dj_config["database.user"], + "password": dj_config["database.password"], + "host": dj_config["database.host"], + } + + def _write_sql_cnf(self): + """Write SQL cnf file to avoid password prompt.""" + cnf_path = Path("~/.my.cnf").expanduser() + + if cnf_path.exists(): + return + + template = "[client]\nuser={user}\npassword={password}\nhost={host}\n" + + with open(str(cnf_path), "w") as file: + file.write(template.format(**self._get_credentials())) + cnf_path.chmod(0o600) + + def _bash_escape(self, s): + """Escape restriction string for bash.""" + s = s.strip() + + replace_map = { + "WHERE ": "", # Remove preceding WHERE of dj.where_clause + " ": " ", # Squash double spaces + "( (": "((", # Squash double parens + ") )": ")", + '"': "'", # Replace double quotes with single + "`": "", # Remove backticks + " AND ": " \\\n\tAND ", # Add newline and tab for readability + " OR ": " \\\n\tOR ", # OR extra space to align with AND + ")AND(": ") \\\n\tAND (", + ")OR(": ") \\\n\tOR (", + "#": "\\#", + } + for old, new in replace_map.items(): + s = s.replace(old, new) + if s.startswith("(((") and s.endswith(")))"): + s = s[2:-2] # Remove extra parens for readability + return s + + def _cmd_prefix(self, docker_id=None): + """Get prefix for mysqldump command. Includes docker exec if needed.""" + if not docker_id: + return "mysqldump " + return ( + f"docker exec -i {docker_id} \\\n\tmysqldump " + + "-u {user} --password={password} \\\n\t".format( + **self._get_credentials() + ) + ) + + def write_mysqldump( + self, + free_tables: List[FreeTable], + file_suffix: str = "", + ): + """Write mysqlmdump.sh script to export data. + + Parameters + ---------- + free_tables : List[FreeTable] + List of FreeTables to export + file_suffix : str, optional + Suffix to append to export file names. Default "" + """ + self._write_sql_cnf() + + paper_dir = ( + Path(export_dir) / self.paper_id + if not self.docker_id + else Path(".") + ) + paper_dir.mkdir(exist_ok=True) + + dump_script = paper_dir / f"_ExportSQL_{self.paper_id}{file_suffix}.sh" + dump_content = paper_dir / f"_Populate_{self.paper_id}{file_suffix}.sql" + + prefix = self._cmd_prefix(self.docker_id) + version = ( # Include spyglass version as comment in dump + "echo '--'\n" + + f"echo '-- SPYGLASS VERSION: {self.spyglass_version} --'\n" + + "echo '--'\n\n" + if self.spyglass_version + else "" + ) + create_cmd = ( + "echo 'CREATE DATABASE IF NOT EXISTS {database}; " + + "USE {database};'\n\n" + ) + dump_cmd = prefix + '{database} {table} --where="\\\n\t{where}"\n\n' + + tables_by_db = sorted(free_tables, key=lambda x: x.full_table_name) + + with open(dump_script, "w") as file: + file.write( + "#!/bin/bash\n\n" + + f"exec > {dump_content}\n\n" # Redirect output to sql file + + f"{version}" # Include spyglass version as comment + ) + + prev_db = None + for table in tables_by_db: + if not (where := table.where_clause()): + continue + where = self._bash_escape(where) + database, table_name = ( + table.full_table_name.replace("`", "") + .replace("#", "\\#") + .split(".") + ) + if database != prev_db: + file.write(create_cmd.format(database=database)) + prev_db = database + file.write( + dump_cmd.format( + database=database, table=table_name, where=where + ) + ) + + self._remove_encoding(dump_script) + self._write_version_file() + + logger.info(f"Export script written to {dump_script}") + + self._export_conda_env() + + def _remove_encoding(self, dump_script): + """Remove encoding from dump_content.""" + charset_sed = r"sed -i 's/ DEFAULT CHARSET=[^ ]\w*//g' " + charset_sed = r"sed -i 's/ DEFAULT COLLATE [^ ]\w*//g' " + os_system(f"{charset_sed} {dump_script}") + + def _write_version_file(self): + """Write spyglass version to paper directory.""" + version_file = Path(export_dir) / self.paper_id / "spyglass_version" + if version_file.exists(): + return + with version_file.open("w") as file: + file.write(self.spyglass_version) + + def _export_conda_env(self): + """Export conda environment to paper directory. + + Renames environment name to paper_id. + """ + yml_path = Path(export_dir) / self.paper_id / "environment.yml" + if yml_path.exists(): + return + command = f"conda env export > {yml_path}" + os_system(command) + + # RENAME ENVIRONMENT NAME TO PAPER ID + with yml_path.open("r") as file: + yml = yaml.safe_load(file) + yml["name"] = self.paper_id + with yml_path.open("w") as file: + yaml.dump(yml, file) + + logger.info(f"Conda environment exported to {yml_path}") From 94aea2696c8d956a09ed6100c4193051e16df87a Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Tue, 6 Aug 2024 13:19:16 -0500 Subject: [PATCH 29/94] Revise get_group_by_shank (#1055) * Revise get_group_by_shank * Exit early on empty delete --- CHANGELOG.md | 6 +- src/spyglass/spikesorting/utils.py | 95 +++++++++++++++--------------- src/spyglass/utils/dj_mixin.py | 5 ++ 3 files changed, 58 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f892611a0..4b2b5ffa0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,14 +28,16 @@ PositionGroup.alter() - Migrate `pip` dependencies from `environment.yml`s to `pyproject.toml` #966 - Add documentation for common error messages #997 - Expand `delete_downstream_merge` -> `delete_downstream_parts`. #1002 -- `cautious_delete` now checks `IntervalList` and externals tables. #1002 +- `cautious_delete` now ... + - Checks `IntervalList` and externals tables. #1002 + - Ends early if called on empty table. #1055 - Allow mixin tables with parallelization in `make` to run populate with `processes > 1` #1001, #1052 - Speed up fetch_nwb calls through merge tables #1017 - Allow `ModuleNotFoundError` or `ImportError` for optional dependencies #1023 - Ensure integrity of group tables #1026 - Convert list of LFP artifact removed interval list to array #1046 -- Merge duplicate functions in decoding and spikesorting #1050 +- Merge duplicate functions in decoding and spikesorting #1050, #1052 - Revise docs organization. - Misc -> Features/ForDevelopers. #1029 - Installation instructions -> Setup notebook. #1029 diff --git a/src/spyglass/spikesorting/utils.py b/src/spyglass/spikesorting/utils.py index a2e8f2f54..42a9814b9 100644 --- a/src/spyglass/spikesorting/utils.py +++ b/src/spyglass/spikesorting/utils.py @@ -45,12 +45,11 @@ def get_group_by_shank( e_groups.sort(key=int) # sort electrode groups numerically sort_group = 0 - sg_keys = list() - sge_keys = list() + sg_keys, sge_keys = list(), list() for e_group in e_groups: - sg_key = dict() - sge_key = dict() + sg_key, sge_key = dict(), dict() sg_key["nwb_file_name"] = sge_key["nwb_file_name"] = nwb_file_name + # for each electrode group, get a list of the unique shank numbers shank_list = np.unique( electrodes["probe_shank"][ @@ -58,73 +57,77 @@ def get_group_by_shank( ] ) sge_key["electrode_group_name"] = e_group - # get the indices of all electrodes in this group / shank and set their sorting group + + # get the indices of all electrodes in this group / shank and set their + # sorting group for shank in shank_list: sg_key["sort_group_id"] = sge_key["sort_group_id"] = sort_group - # specify reference electrode. Use 'references' if passed, otherwise use reference from config - if not references: - shank_elect_ref = electrodes["original_reference_electrode"][ - np.logical_and( - electrodes["electrode_group_name"] == e_group, - electrodes["probe_shank"] == shank, + + match_names_bool = np.logical_and( + electrodes["electrode_group_name"] == e_group, + electrodes["probe_shank"] == shank, + ) + + if references: # Use 'references' if passed + sort_ref_id = references.get(e_group, None) + if not sort_ref_id: + raise Exception( + f"electrode group {e_group} not a key in " + + "references, so cannot set reference" ) + else: # otherwise use reference from config + shank_elect_ref = electrodes["original_reference_electrode"][ + match_names_bool ] if np.max(shank_elect_ref) == np.min(shank_elect_ref): - sg_key["sort_reference_electrode_id"] = shank_elect_ref[0] + sort_ref_id = shank_elect_ref[0] else: ValueError( f"Error in electrode group {e_group}: reference " + "electrodes are not all the same" ) - else: - if e_group not in references.keys(): - raise Exception( - f"electrode group {e_group} not a key in " - + "references, so cannot set reference" - ) - else: - sg_key["sort_reference_electrode_id"] = references[e_group] + sg_key["sort_reference_electrode_id"] = sort_ref_id + # Insert sort group and sort group electrodes - reference_electrode_group = electrodes[ - electrodes["electrode_id"] - == sg_key["sort_reference_electrode_id"] - ][ - "electrode_group_name" - ] # reference for this electrode group - if len(reference_electrode_group) == 1: # unpack single reference - reference_electrode_group = reference_electrode_group[0] - elif (int(sg_key["sort_reference_electrode_id"]) > 0) and ( - len(reference_electrode_group) != 1 - ): + match_elec = electrodes[electrodes["electrode_id"] == sort_ref_id] + ref_elec_group = match_elec["electrode_group_name"] # group ref + + n_ref_groups = len(ref_elec_group) + if n_ref_groups == 1: # unpack single reference + ref_elec_group = ref_elec_group[0] + elif int(sort_ref_id) > 0: # multiple references raise Exception( "Should have found exactly one electrode group for " - + "reference electrode, but found " - + f"{len(reference_electrode_group)}." + + f"reference electrode, but found {n_ref_groups}." ) + if omit_ref_electrode_group and ( - str(e_group) == str(reference_electrode_group) + str(e_group) == str(ref_elec_group) ): logger.warn( f"Omitting electrode group {e_group} from sort groups " + "because contains reference." ) continue - shank_elect = electrodes["electrode_id"][ - np.logical_and( - electrodes["electrode_group_name"] == e_group, - electrodes["probe_shank"] == shank, - ) - ] - if ( - omit_unitrode and len(shank_elect) == 1 - ): # omit unitrodes if indicated + shank_elect = electrodes["electrode_id"][match_names_bool] + + # omit unitrodes if indicated + if omit_unitrode and len(shank_elect) == 1: logger.warn( f"Omitting electrode group {e_group}, shank {shank} " + "from sort groups because unitrode." ) continue + sg_keys.append(sg_key) - for elect in shank_elect: - sge_key["electrode_id"] = elect - sge_keys.append(sge_key.copy()) + sge_keys.extend( + [ + { + **sge_key, + "electrode_id": elect, + } + for elect in shank_elect + ] + ) sort_group += 1 + return sg_keys, sge_keys diff --git a/src/spyglass/utils/dj_mixin.py b/src/spyglass/utils/dj_mixin.py index c6959219f..a3904a87b 100644 --- a/src/spyglass/utils/dj_mixin.py +++ b/src/spyglass/utils/dj_mixin.py @@ -652,6 +652,11 @@ def cautious_delete( Passed to datajoint.table.Table.delete. """ start = time() + + if len(self) == 0: + logger.warning(f"Table is empty. No need to delete.\n{self}") + return + external, IntervalList = self._delete_deps[3], self._delete_deps[4] if not force_permission or dry_run: From 197428f354084874eb46bddcdfcac93040c74523 Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Thu, 8 Aug 2024 13:22:28 -0500 Subject: [PATCH 30/94] Remove `common_ripple` (#1061) * Remove common_ripple * Edit tests/changelog --- CHANGELOG.md | 3 + src/spyglass/common/common_ripple.py | 361 --------------------------- src/spyglass/ripple/v1/ripple.py | 36 ++- src/spyglass/utils/dj_mixin.py | 4 - tests/common/test_ripple.py | 6 - tests/conftest.py | 2 - 6 files changed, 33 insertions(+), 379 deletions(-) delete mode 100644 src/spyglass/common/common_ripple.py delete mode 100644 tests/common/test_ripple.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b2b5ffa0..c7f977c6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,11 @@ ```python +import datajoint as dj from spyglass.common.common_behav import PositionIntervalMap from spyglass.decoding.v1.core import PositionGroup +dj.schema("common_ripple").drop() PositionIntervalMap.alter() PositionGroup.alter() ``` @@ -58,6 +60,7 @@ PositionGroup.alter() - `PositionIntervalMap` now inserts null entries for missing intervals #870 - `AnalysisFileLog` now truncates table names that exceed field length #1021 - Disable logging with `AnalysisFileLog` #1024 + - Remove `common_ripple` schema #1061 - Decoding: diff --git a/src/spyglass/common/common_ripple.py b/src/spyglass/common/common_ripple.py deleted file mode 100644 index 3bef3fa9a..000000000 --- a/src/spyglass/common/common_ripple.py +++ /dev/null @@ -1,361 +0,0 @@ -import datajoint as dj -import matplotlib.pyplot as plt -import numpy as np -import pandas as pd -from matplotlib.axes import Axes -from ripple_detection import Karlsson_ripple_detector, Kay_ripple_detector -from ripple_detection.core import gaussian_smooth, get_envelope - -from spyglass.common import IntervalList # noqa -from spyglass.common import IntervalPositionInfo, LFPBand, LFPBandSelection -from spyglass.common.common_nwbfile import AnalysisNwbfile -from spyglass.utils import SpyglassMixin, logger - -schema = dj.schema("common_ripple") - -RIPPLE_DETECTION_ALGORITHMS = { - "Kay_ripple_detector": Kay_ripple_detector, - "Karlsson_ripple_detector": Karlsson_ripple_detector, -} - - -def interpolate_to_new_time( - df, new_time, upsampling_interpolation_method="linear" -): - old_time = df.index - new_index = pd.Index( - np.unique(np.concatenate((old_time, new_time))), name="time" - ) - return ( - df.reindex(index=new_index) - .interpolate(method=upsampling_interpolation_method) - .reindex(index=new_time) - ) - - -@schema -class RippleLFPSelection(SpyglassMixin, dj.Manual): - definition = """ - -> LFPBand - group_name = 'CA1' : varchar(80) - """ - - class RippleLFPElectrode(SpyglassMixin, dj.Part): - definition = """ - -> RippleLFPSelection - -> LFPBandSelection.LFPBandElectrode - """ - - def insert1(self, key, **kwargs): - filter_name = (LFPBand & key).fetch1("filter_name") - if "ripple" not in filter_name.lower(): - logger.warning("Please use a ripple filter") - super().insert1(key, **kwargs) - - @staticmethod - def set_lfp_electrodes( - key, - electrode_list: list = None, - group_name: str = "CA1", - **kwargs, - ): - """Replaces all electrodes for an nwb file with specified electrodes. - - Parameters - ---------- - key : dict - dictionary corresponding to the LFPBand entry to use for ripple - detection - electrode_list : list - list of electrodes from LFPBandSelection.LFPBandElectrode - to be used as the ripple LFP during detection - group_name : str, optional - description of the electrode group, by default "CA1" - """ - - RippleLFPSelection().insert1( - {**key, "group_name": group_name}, - skip_duplicates=True, - **kwargs, - ) - if not electrode_list: - electrode_list = ( - (LFPBandSelection.LFPBandElectrode() & key) - .fetch("electrode_id") - .tolist() - ) - electrode_list.sort() - electrode_keys = ( - pd.DataFrame(LFPBandSelection.LFPBandElectrode() & key) - .set_index("electrode_id") - .loc[electrode_list] - .reset_index() - .loc[:, LFPBandSelection.LFPBandElectrode.primary_key] - ) - electrode_keys["group_name"] = group_name - electrode_keys = electrode_keys.sort_values(by=["electrode_id"]) - RippleLFPSelection().RippleLFPElectrode.insert( - electrode_keys.to_dict(orient="records"), - replace=True, - **kwargs, - ) - - -@schema -class RippleParameters(SpyglassMixin, dj.Lookup): - definition = """ - ripple_param_name : varchar(80) # a name for this set of parameters - ---- - ripple_param_dict : BLOB # dictionary of parameters - """ - - def insert_default(self): - """Insert the default parameter set""" - default_dict = { - "speed_name": "head_speed", - "ripple_detection_algorithm": "Kay_ripple_detector", - "ripple_detection_params": dict( - speed_threshold=4.0, # cm/s - minimum_duration=0.015, # sec - zscore_threshold=2.0, # std - smoothing_sigma=0.004, # sec - close_ripple_threshold=0.0, # sec - ), - } - self.insert1( - {"ripple_param_name": "default", "ripple_param_dict": default_dict}, - skip_duplicates=True, - ) - - -@schema -class RippleTimes(SpyglassMixin, dj.Computed): - definition = """ - -> RippleParameters - -> RippleLFPSelection - -> IntervalPositionInfo - --- - -> AnalysisNwbfile - ripple_times_object_id : varchar(40) - """ - - def make(self, key): - logger.info(f"Computing ripple times for: {key}") - ripple_params = ( - RippleParameters & {"ripple_param_name": key["ripple_param_name"]} - ).fetch1("ripple_param_dict") - - ripple_detection_algorithm = ripple_params["ripple_detection_algorithm"] - ripple_detection_params = ripple_params["ripple_detection_params"] - - ( - speed, - interval_ripple_lfps, - sampling_frequency, - ) = self.get_ripple_lfps_and_position_info(key) - - ripple_times = RIPPLE_DETECTION_ALGORITHMS[ripple_detection_algorithm]( - time=np.asarray(interval_ripple_lfps.index), - filtered_lfps=np.asarray(interval_ripple_lfps), - speed=np.asarray(speed), - sampling_frequency=sampling_frequency, - **ripple_detection_params, - ) - - # Insert into analysis nwb file - nwb_analysis_file = AnalysisNwbfile() - key["analysis_file_name"] = nwb_analysis_file.create( - key["nwb_file_name"] - ) - key["ripple_times_object_id"] = nwb_analysis_file.add_nwb_object( - analysis_file_name=key["analysis_file_name"], - nwb_object=ripple_times, - ) - nwb_analysis_file.add( - nwb_file_name=key["nwb_file_name"], - analysis_file_name=key["analysis_file_name"], - ) - - self.insert1(key) - - def fetch1_dataframe(self): - """Convenience function for returning the marks in a readable format""" - return self.fetch_dataframe()[0] - - def fetch_dataframe(self): - return [data["ripple_times"] for data in self.fetch_nwb()] - - @staticmethod - def get_ripple_lfps_and_position_info(key): - nwb_file_name = key["nwb_file_name"] - interval_list_name = key["target_interval_list_name"] - position_info_param_name = key["position_info_param_name"] - ripple_params = ( - RippleParameters & {"ripple_param_name": key["ripple_param_name"]} - ).fetch1("ripple_param_dict") - - speed_name = ripple_params["speed_name"] - - electrode_keys = (RippleLFPSelection.RippleLFPElectrode() & key).fetch( - "electrode_id" - ) - - # warn/validate that there is only one wire per electrode - lfp_key = key.copy() - del lfp_key["interval_list_name"] - ripple_lfp_nwb = (LFPBand & lfp_key).fetch_nwb()[0] - ripple_lfp_electrodes = ripple_lfp_nwb["filtered_data"].electrodes.data[ - : - ] - elec_mask = np.full_like(ripple_lfp_electrodes, 0, dtype=bool) - elec_mask[ - [ - ind - for ind, elec in enumerate(ripple_lfp_electrodes) - if elec in electrode_keys - ] - ] = True - ripple_lfp = pd.DataFrame( - ripple_lfp_nwb["filtered_data"].data, - index=pd.Index( - ripple_lfp_nwb["filtered_data"].timestamps, name="time" - ), - ) - sampling_frequency = ripple_lfp_nwb["lfp_band_sampling_rate"] - - ripple_lfp = ripple_lfp.loc[:, elec_mask] - - position_valid_times = ( - IntervalList - & { - "nwb_file_name": nwb_file_name, - "interval_list_name": interval_list_name, - } - ).fetch1("valid_times") - - position_info = ( - IntervalPositionInfo() - & { - "nwb_file_name": nwb_file_name, - "interval_list_name": interval_list_name, - "position_info_param_name": position_info_param_name, - } - ).fetch1_dataframe() - - position_info = pd.concat( - [ - position_info.loc[slice(valid_time[0], valid_time[1])] - for valid_time in position_valid_times - ], - axis=1, - ) - interval_ripple_lfps = pd.concat( - [ - ripple_lfp.loc[slice(valid_time[0], valid_time[1])] - for valid_time in position_valid_times - ], - axis=1, - ) - - position_info = interpolate_to_new_time( - position_info, interval_ripple_lfps.index - ) - - return ( - position_info[speed_name], - interval_ripple_lfps, - sampling_frequency, - ) - - @staticmethod - def get_Kay_ripple_consensus_trace( - ripple_filtered_lfps, sampling_frequency, smoothing_sigma: float = 0.004 - ): - ripple_consensus_trace = np.full_like(ripple_filtered_lfps, np.nan) - not_null = np.all(pd.notnull(ripple_filtered_lfps), axis=1) - - ripple_consensus_trace[not_null] = get_envelope( - np.asarray(ripple_filtered_lfps)[not_null] - ) - ripple_consensus_trace = np.sum(ripple_consensus_trace**2, axis=1) - ripple_consensus_trace[not_null] = gaussian_smooth( - ripple_consensus_trace[not_null], - smoothing_sigma, - sampling_frequency, - ) - return pd.DataFrame( - np.sqrt(ripple_consensus_trace), index=ripple_filtered_lfps.index - ) - - @staticmethod - def plot_ripple_consensus_trace( - ripple_consensus_trace, - ripple_times, - ripple_label: int = 1, - offset: float = 0.100, - relative: bool = True, - ax: Axes = None, - ): - ripple_start = ripple_times.loc[ripple_label].start_time - ripple_end = ripple_times.loc[ripple_label].end_time - time_slice = slice(ripple_start - offset, ripple_end + offset) - - start_offset = ripple_start if relative else 0 - if ax is None: - fig, ax = plt.subplots(1, 1, figsize=(12, 1)) - ax.plot( - ripple_consensus_trace.loc[time_slice].index - start_offset, - ripple_consensus_trace.loc[time_slice], - ) - ax.axvspan( - ripple_start - start_offset, - ripple_end - start_offset, - zorder=-1, - alpha=0.5, - color="lightgrey", - ) - ax.set_xlabel("Time [s]") - ax.set_xlim( - (time_slice.start - start_offset, time_slice.stop - start_offset) - ) - - @staticmethod - def plot_ripple( - lfps, - ripple_times, - ripple_label: int = 1, - offset: float = 0.100, - relative: bool = True, - ax: Axes = None, - ): - lfp_labels = lfps.columns - n_lfps = len(lfp_labels) - ripple_start = ripple_times.loc[ripple_label].start_time - ripple_end = ripple_times.loc[ripple_label].end_time - time_slice = slice(ripple_start - offset, ripple_end + offset) - if ax is None: - fig, ax = plt.subplots(1, 1, figsize=(12, n_lfps * 0.20)) - - start_offset = ripple_start if relative else 0 - - for lfp_ind, lfp_label in enumerate(lfp_labels): - lfp = lfps.loc[time_slice, lfp_label] - ax.plot( - lfp.index - start_offset, - lfp_ind + (lfp - lfp.mean()) / (lfp.max() - lfp.min()), - color="black", - ) - - ax.axvspan( - ripple_start - start_offset, - ripple_end - start_offset, - zorder=-1, - alpha=0.5, - color="lightgrey", - ) - ax.set_ylim((-1, n_lfps)) - ax.set_xlim( - (time_slice.start - start_offset, time_slice.stop - start_offset) - ) - ax.set_ylabel("LFPs") - ax.set_xlabel("Time [s]") diff --git a/src/spyglass/ripple/v1/ripple.py b/src/spyglass/ripple/v1/ripple.py index bac24dc5c..bbacfd1d5 100644 --- a/src/spyglass/ripple/v1/ripple.py +++ b/src/spyglass/ripple/v1/ripple.py @@ -4,6 +4,7 @@ import pandas as pd import sortingview.views as vv from ripple_detection import Karlsson_ripple_detector, Kay_ripple_detector +from ripple_detection.core import gaussian_smooth, get_envelope from scipy.stats import zscore from spyglass.common.common_interval import ( @@ -11,7 +12,6 @@ interval_list_intersect, ) from spyglass.common.common_nwbfile import AnalysisNwbfile -from spyglass.common.common_ripple import RippleTimes, interpolate_to_new_time from spyglass.lfp.analysis.v1.lfp_band import LFPBandSelection, LFPBandV1 from spyglass.lfp.lfp_merge import LFPOutput from spyglass.position import PositionOutput @@ -280,12 +280,22 @@ def get_ripple_lfps_and_position_info(key): @staticmethod def get_Kay_ripple_consensus_trace( - ripple_filtered_lfps, sampling_frequency, smoothing_sigma=0.004 + ripple_filtered_lfps, sampling_frequency, smoothing_sigma: float = 0.004 ): - return RippleTimes.get_Kay_ripple_consensus_trace( - ripple_filtered_lfps=ripple_filtered_lfps, - sampling_frequency=sampling_frequency, - smoothing_sigma=smoothing_sigma, + ripple_consensus_trace = np.full_like(ripple_filtered_lfps, np.nan) + not_null = np.all(pd.notnull(ripple_filtered_lfps), axis=1) + + ripple_consensus_trace[not_null] = get_envelope( + np.asarray(ripple_filtered_lfps)[not_null] + ) + ripple_consensus_trace = np.sum(ripple_consensus_trace**2, axis=1) + ripple_consensus_trace[not_null] = gaussian_smooth( + ripple_consensus_trace[not_null], + smoothing_sigma, + sampling_frequency, + ) + return pd.DataFrame( + np.sqrt(ripple_consensus_trace), index=ripple_filtered_lfps.index ) @staticmethod @@ -481,3 +491,17 @@ def _add_ripple_times( ) return view.url(label="Ripple Detection") + + +def interpolate_to_new_time( + df, new_time, upsampling_interpolation_method="linear" +): + old_time = df.index + new_index = pd.Index( + np.unique(np.concatenate((old_time, new_time))), name="time" + ) + return ( + df.reindex(index=new_index) + .interpolate(method=upsampling_interpolation_method) + .reindex(index=new_time) + ) diff --git a/src/spyglass/utils/dj_mixin.py b/src/spyglass/utils/dj_mixin.py index a3904a87b..b039add59 100644 --- a/src/spyglass/utils/dj_mixin.py +++ b/src/spyglass/utils/dj_mixin.py @@ -261,9 +261,6 @@ def fetch_pynapple(self, *attrs, **kwargs): def _import_part_masters(self): """Import tables that may constrain a RestrGraph. See #1002""" - from spyglass.common.common_ripple import ( - RippleLFPSelection, - ) # noqa F401 from spyglass.decoding.decoding_merge import DecodingOutput # noqa F401 from spyglass.decoding.v0.clusterless import ( # noqa F401 UnitMarksIndicatorSelection, @@ -300,7 +297,6 @@ def _import_part_masters(self): MuaEventsV1(), PositionGroup(), PositionOutput(), - RippleLFPSelection(), RippleTimesV1(), SortedSpikesGroup(), SortedSpikesIndicatorSelection(), diff --git a/tests/common/test_ripple.py b/tests/common/test_ripple.py deleted file mode 100644 index 71a57d022..000000000 --- a/tests/common/test_ripple.py +++ /dev/null @@ -1,6 +0,0 @@ -import pytest - - -@pytest.mark.skip(reason="Not testing V0: common_ripple") -def test_common_ripple(common): - pass diff --git a/tests/conftest.py b/tests/conftest.py index 0bff200ef..04420ffe2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -386,7 +386,6 @@ def populate_exception(): @pytest.fixture(scope="session") def frequent_imports(): """Often needed for graph cascade.""" - from spyglass.common.common_ripple import RippleLFPSelection from spyglass.decoding.v0.clusterless import UnitMarksIndicatorSelection from spyglass.decoding.v0.sorted_spikes import ( SortedSpikesIndicatorSelection, @@ -401,7 +400,6 @@ def frequent_imports(): LFPBandSelection, MuaEventsV1, PositionGroup, - RippleLFPSelection, RippleTimesV1, SortedSpikesIndicatorSelection, SpikeSortingRecordingView, From 04b4995002e01dfaae1f69870685caa7321e4298 Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Fri, 9 Aug 2024 17:05:43 -0500 Subject: [PATCH 31/94] Reduce duplication 2 (#1053) * Reduce duplication 2: spikesort utils * More combining of utils * Update changelog * Remove comment from decoding.v1.clusterless * Remove other comments --- CHANGELOG.md | 2 +- src/spyglass/decoding/v0/clusterless.py | 32 ++-- src/spyglass/decoding/v0/sorted_spikes.py | 39 +++-- src/spyglass/decoding/v1/clusterless.py | 76 ++------- src/spyglass/decoding/v1/sorted_spikes.py | 49 +----- src/spyglass/decoding/v1/utils.py | 44 +++++ src/spyglass/linearization/v0/main.py | 2 - src/spyglass/linearization/v1/main.py | 2 - src/spyglass/ripple/v1/ripple.py | 8 +- .../spikesorting/analysis/v1/group.py | 32 ++-- .../spikesorting/spikesorting_merge.py | 43 +++-- src/spyglass/spikesorting/utils.py | 160 ++++++++++++++++++ .../spikesorting/v0/curation_figurl.py | 18 +- .../spikesorting/v0/spikesorting_artifact.py | 151 +++-------------- .../spikesorting/v0/spikesorting_curation.py | 60 +++---- .../spikesorting/v0/spikesorting_recording.py | 49 ++---- src/spyglass/spikesorting/v1/artifact.py | 128 ++------------ .../spikesorting/v1/figurl_curation.py | 16 +- src/spyglass/spikesorting/v1/recording.py | 28 +-- src/spyglass/utils/dj_graph.py | 29 ++-- src/spyglass/utils/dj_helper_fn.py | 20 ++- src/spyglass/utils/dj_mixin.py | 12 +- src/spyglass/utils/spikesorting.py | 28 +++ 23 files changed, 452 insertions(+), 576 deletions(-) create mode 100644 src/spyglass/decoding/v1/utils.py create mode 100644 src/spyglass/utils/spikesorting.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c7f977c6f..0e0ebe5c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,7 +39,7 @@ PositionGroup.alter() - Allow `ModuleNotFoundError` or `ImportError` for optional dependencies #1023 - Ensure integrity of group tables #1026 - Convert list of LFP artifact removed interval list to array #1046 -- Merge duplicate functions in decoding and spikesorting #1050, #1052 +- Merge duplicate functions in decoding and spikesorting #1050, #1053 - Revise docs organization. - Misc -> Features/ForDevelopers. #1029 - Installation instructions -> Setup notebook. #1029 diff --git a/src/spyglass/decoding/v0/clusterless.py b/src/spyglass/decoding/v0/clusterless.py index c7d3e4b5a..c72d4b80b 100644 --- a/src/spyglass/decoding/v0/clusterless.py +++ b/src/spyglass/decoding/v0/clusterless.py @@ -521,29 +521,28 @@ def get_decoding_data_for_epoch( valid_slices : list[slice] """ - valid_ephys_position_times_by_epoch = ( - get_valid_ephys_position_times_by_epoch(nwb_file_name) + valid_slices = convert_valid_times_to_slice( + get_valid_ephys_position_times_by_epoch(nwb_file_name)[ + interval_list_name + ] ) - valid_ephys_position_times = valid_ephys_position_times_by_epoch[ - interval_list_name - ] - valid_slices = convert_valid_times_to_slice(valid_ephys_position_times) # position interval - position_interval_name = ( - convert_epoch_interval_name_to_position_interval_name( + nwb_dict = dict(nwb_file_name=nwb_file_name) + pos_interval_dict = dict( + nwb_dict, + interval_list_name=convert_epoch_interval_name_to_position_interval_name( { - "nwb_file_name": nwb_file_name, + **nwb_dict, "interval_list_name": interval_list_name, } - ) + ), ) position_info = ( IntervalPositionInfo() & { - "nwb_file_name": nwb_file_name, - "interval_list_name": position_interval_name, + **pos_interval_dict, "position_info_param_name": position_info_param_name, } ).fetch1_dataframe() @@ -553,14 +552,7 @@ def get_decoding_data_for_epoch( ) marks = ( - ( - UnitMarksIndicator() - & { - "nwb_file_name": nwb_file_name, - "interval_list_name": position_interval_name, - **additional_mark_keys, - } - ) + (UnitMarksIndicator() & {**pos_interval_dict, **additional_mark_keys}) ).fetch_xarray() marks = xr.concat( diff --git a/src/spyglass/decoding/v0/sorted_spikes.py b/src/spyglass/decoding/v0/sorted_spikes.py index 7cdfaa810..adf1e393e 100644 --- a/src/spyglass/decoding/v0/sorted_spikes.py +++ b/src/spyglass/decoding/v0/sorted_spikes.py @@ -267,25 +267,33 @@ def get_decoding_data_for_epoch( valid_slices : list[slice] """ - # valid slices - valid_ephys_position_times_by_epoch = ( - get_valid_ephys_position_times_by_epoch(nwb_file_name) + + valid_slices = convert_valid_times_to_slice( + get_valid_ephys_position_times_by_epoch(nwb_file_name)[ + interval_list_name + ] ) - valid_ephys_position_times = valid_ephys_position_times_by_epoch[ - interval_list_name - ] - valid_slices = convert_valid_times_to_slice(valid_ephys_position_times) # position interval - position_interval_name = ( - convert_epoch_interval_name_to_position_interval_name( + nwb_dict = dict(nwb_file_name=nwb_file_name) + pos_interval_dict = dict( + nwb_dict, + interval_list_name=convert_epoch_interval_name_to_position_interval_name( { - "nwb_file_name": nwb_file_name, + **nwb_dict, "interval_list_name": interval_list_name, } - ) + ), ) + position_info = ( + IntervalPositionInfo() + & { + **pos_interval_dict, + "position_info_param_name": position_info_param_name, + } + ).fetch1_dataframe() + # spikes valid_times = np.asarray( [(times.start, times.stop) for times in valid_slices] @@ -302,15 +310,6 @@ def get_decoding_data_for_epoch( ) spikes = pd.concat([spikes.loc[times] for times in valid_slices]) - # position - position_info = ( - IntervalPositionInfo() - & { - "nwb_file_name": nwb_file_name, - "interval_list_name": position_interval_name, - "position_info_param_name": position_info_param_name, - } - ).fetch1_dataframe() new_time = spikes.index.to_numpy() new_index = pd.Index( np.unique(np.concatenate((position_info.index, new_time))), diff --git a/src/spyglass/decoding/v1/clusterless.py b/src/spyglass/decoding/v1/clusterless.py index bbb3b3418..e5676b1f9 100644 --- a/src/spyglass/decoding/v1/clusterless.py +++ b/src/spyglass/decoding/v1/clusterless.py @@ -19,19 +19,20 @@ import pandas as pd import xarray as xr from non_local_detector.models.base import ClusterlessDetector -from ripple_detection import get_multiunit_population_firing_rate from track_linearization import get_linearized_position from spyglass.common.common_interval import IntervalList # noqa: F401 from spyglass.common.common_session import Session # noqa: F401 from spyglass.decoding.v1.core import DecodingParameters # noqa: F401 from spyglass.decoding.v1.core import PositionGroup +from spyglass.decoding.v1.utils import _get_interval_range from spyglass.decoding.v1.waveform_features import ( UnitWaveformFeatures, ) # noqa: F401 from spyglass.position.position_merge import PositionOutput # noqa: F401 from spyglass.settings import config from spyglass.utils import SpyglassMixin, SpyglassMixinPart, logger +from spyglass.utils.spikesorting import firing_rate_from_spike_indicator schema = dj.schema("decoding_clusterless_v1") @@ -331,47 +332,6 @@ def fetch_environments(key): return classifier.environments - @staticmethod - def _get_interval_range(key): - """Return max range of model times in the encoding/decoding intervals - - Parameters - ---------- - key : dict - The decoding selection key - - Returns - ------- - Tuple[float, float] - The minimum and maximum times for the model - """ - encoding_interval = ( - IntervalList - & { - "nwb_file_name": key["nwb_file_name"], - "interval_list_name": key["encoding_interval"], - } - ).fetch1("valid_times") - - decoding_interval = ( - IntervalList - & { - "nwb_file_name": key["nwb_file_name"], - "interval_list_name": key["decoding_interval"], - } - ).fetch1("valid_times") - - return ( - min( - np.asarray(encoding_interval).min(), - np.asarray(decoding_interval).min(), - ), - max( - np.asarray(encoding_interval).max(), - np.asarray(decoding_interval).max(), - ), - ) - @staticmethod def fetch_position_info(key): """Fetch the position information for the decoding model @@ -391,7 +351,7 @@ def fetch_position_info(key): "nwb_file_name": key["nwb_file_name"], } - min_time, max_time = ClusterlessDecodingV1._get_interval_range(key) + min_time, max_time = _get_interval_range(key) position_info, position_variable_names = ( PositionGroup & position_group_key ).fetch_position_info(min_time=min_time, max_time=max_time) @@ -427,7 +387,7 @@ def fetch_linear_position_info(key): edge_spacing=environment.edge_spacing, ) - min_time, max_time = ClusterlessDecodingV1._get_interval_range(key) + min_time, max_time = _get_interval_range(key) return pd.concat( [linear_position_df.set_index(position_df.index), position_df], @@ -469,7 +429,7 @@ def fetch_spike_data(key, filter_by_interval=True): if not filter_by_interval: return spike_times, spike_waveform_features - min_time, max_time = ClusterlessDecodingV1._get_interval_range(key) + min_time, max_time = _get_interval_range(key) new_spike_times = [] new_waveform_features = [] @@ -522,7 +482,8 @@ def get_firing_rate( multiunit: bool = False, smoothing_sigma: float = 0.015, ) -> np.ndarray: - """get time-dependent firing rate for units in the group + """Get time-dependent firing rate for units in the group + Parameters ---------- @@ -542,24 +503,11 @@ def get_firing_rate( np.ndarray time-dependent firing rate with shape (len(time), n_units) """ - spike_indicator = cls.get_spike_indicator(key, time) - if spike_indicator.ndim == 1: - spike_indicator = spike_indicator[:, np.newaxis] - - sampling_frequency = 1 / np.median(np.diff(time)) - - if multiunit: - spike_indicator = spike_indicator.sum(axis=1, keepdims=True) - return np.stack( - [ - get_multiunit_population_firing_rate( - indicator[:, np.newaxis], - sampling_frequency, - smoothing_sigma, - ) - for indicator in spike_indicator.T - ], - axis=1, + return firing_rate_from_spike_indicator( + spike_indicator=cls.get_spike_indicator(key, time), + time=time, + multiunit=multiunit, + smoothing_sigma=smoothing_sigma, ) def get_orientation_col(self, df): diff --git a/src/spyglass/decoding/v1/sorted_spikes.py b/src/spyglass/decoding/v1/sorted_spikes.py index c70c6617b..bb44c843f 100644 --- a/src/spyglass/decoding/v1/sorted_spikes.py +++ b/src/spyglass/decoding/v1/sorted_spikes.py @@ -25,6 +25,7 @@ from spyglass.common.common_session import Session # noqa: F401 from spyglass.decoding.v1.core import DecodingParameters # noqa: F401 from spyglass.decoding.v1.core import PositionGroup +from spyglass.decoding.v1.utils import _get_interval_range from spyglass.position.position_merge import PositionOutput # noqa: F401 from spyglass.settings import config from spyglass.spikesorting.analysis.v1.group import SortedSpikesGroup @@ -296,47 +297,6 @@ def fetch_environments(key): return classifier.environments - @staticmethod - def _get_interval_range(key): - """Return maximum range of model times in encoding/decoding intervals - - Parameters - ---------- - key : dict - The decoding selection key - - Returns - ------- - Tuple[float, float] - The minimum and maximum times for the model - """ - encoding_interval = ( - IntervalList - & { - "nwb_file_name": key["nwb_file_name"], - "interval_list_name": key["encoding_interval"], - } - ).fetch1("valid_times") - - decoding_interval = ( - IntervalList - & { - "nwb_file_name": key["nwb_file_name"], - "interval_list_name": key["decoding_interval"], - } - ).fetch1("valid_times") - - return ( - min( - np.asarray(encoding_interval).min(), - np.asarray(decoding_interval).min(), - ), - max( - np.asarray(encoding_interval).max(), - np.asarray(decoding_interval).max(), - ), - ) - @staticmethod def fetch_position_info(key): """Fetch the position information for the decoding model @@ -355,7 +315,7 @@ def fetch_position_info(key): "position_group_name": key["position_group_name"], "nwb_file_name": key["nwb_file_name"], } - min_time, max_time = SortedSpikesDecodingV1._get_interval_range(key) + min_time, max_time = _get_interval_range(key) position_info, position_variable_names = ( PositionGroup & position_group_key ).fetch_position_info(min_time=min_time, max_time=max_time) @@ -390,7 +350,8 @@ def fetch_linear_position_info(key): edge_order=environment.edge_order, edge_spacing=environment.edge_spacing, ) - min_time, max_time = SortedSpikesDecodingV1._get_interval_range(key) + min_time, max_time = _get_interval_range(key) + return pd.concat( [linear_position_df.set_index(position_df.index), position_df], axis=1, @@ -428,7 +389,7 @@ def fetch_spike_data( return spike_times if time_slice is None: - min_time, max_time = SortedSpikesDecodingV1._get_interval_range(key) + min_time, max_time = _get_interval_range(key) else: min_time, max_time = time_slice.start, time_slice.stop diff --git a/src/spyglass/decoding/v1/utils.py b/src/spyglass/decoding/v1/utils.py new file mode 100644 index 000000000..20f37ff62 --- /dev/null +++ b/src/spyglass/decoding/v1/utils.py @@ -0,0 +1,44 @@ +import numpy as np + +from spyglass.common.common_interval import IntervalList + + +def _get_interval_range(key): + """Return maximum range of model times in encoding/decoding intervals + + Parameters + ---------- + key : dict + The decoding selection key + + Returns + ------- + Tuple[float, float] + The minimum and maximum times for the model + """ + encoding_interval = ( + IntervalList + & { + "nwb_file_name": key["nwb_file_name"], + "interval_list_name": key["encoding_interval"], + } + ).fetch1("valid_times") + + decoding_interval = ( + IntervalList + & { + "nwb_file_name": key["nwb_file_name"], + "interval_list_name": key["decoding_interval"], + } + ).fetch1("valid_times") + + return ( + min( + np.asarray(encoding_interval).min(), + np.asarray(decoding_interval).min(), + ), + max( + np.asarray(encoding_interval).max(), + np.asarray(decoding_interval).max(), + ), + ) diff --git a/src/spyglass/linearization/v0/main.py b/src/spyglass/linearization/v0/main.py index 2884d8525..30749f8a2 100644 --- a/src/spyglass/linearization/v0/main.py +++ b/src/spyglass/linearization/v0/main.py @@ -57,8 +57,6 @@ class TrackGraph(SpyglassMixin, dj.Manual): """ def get_networkx_track_graph(self, track_graph_parameters=None): - # CB: Does this need to be a public method? Can other methods inherit - # the logic? It's a pretty simple wrapper around make_track_graph. if track_graph_parameters is None: track_graph_parameters = self.fetch1() return make_track_graph( diff --git a/src/spyglass/linearization/v1/main.py b/src/spyglass/linearization/v1/main.py index 3e7023ca2..91c1726b1 100644 --- a/src/spyglass/linearization/v1/main.py +++ b/src/spyglass/linearization/v1/main.py @@ -51,8 +51,6 @@ class TrackGraph(SpyglassMixin, dj.Manual): """ def get_networkx_track_graph(self, track_graph_parameters=None): - # CB: Does this need to be a public method? Can other methods inherit - # the logic? It's a pretty simple wrapper around make_track_graph. if track_graph_parameters is None: track_graph_parameters = self.fetch1() return make_track_graph( diff --git a/src/spyglass/ripple/v1/ripple.py b/src/spyglass/ripple/v1/ripple.py index bbacfd1d5..c6fb92b3a 100644 --- a/src/spyglass/ripple/v1/ripple.py +++ b/src/spyglass/ripple/v1/ripple.py @@ -3,6 +3,7 @@ import numpy as np import pandas as pd import sortingview.views as vv +from matplotlib.axes import Axes from ripple_detection import Karlsson_ripple_detector, Kay_ripple_detector from ripple_detection.core import gaussian_smooth, get_envelope from scipy.stats import zscore @@ -332,7 +333,12 @@ def plot_ripple_consensus_trace( @staticmethod def plot_ripple( - lfps, ripple_times, ripple_label=1, offset=0.100, relative=True, ax=None + lfps, + ripple_times, + ripple_label: int = 1, + offset: float = 0.100, + relative: bool = True, + ax: Axes = None, ): lfp_labels = lfps.columns n_lfps = len(lfp_labels) diff --git a/src/spyglass/spikesorting/analysis/v1/group.py b/src/spyglass/spikesorting/analysis/v1/group.py index e2e9d0765..8b3138e69 100644 --- a/src/spyglass/spikesorting/analysis/v1/group.py +++ b/src/spyglass/spikesorting/analysis/v1/group.py @@ -8,6 +8,7 @@ from spyglass.common import Session # noqa: F401 from spyglass.spikesorting.spikesorting_merge import SpikeSortingOutput from spyglass.utils.dj_mixin import SpyglassMixin, SpyglassMixinPart +from spyglass.utils.spikesorting import firing_rate_from_spike_indicator schema = dj.schema("spikesorting_group_v1") @@ -250,7 +251,7 @@ def get_firing_rate( multiunit: bool = False, smoothing_sigma: float = 0.015, ) -> np.ndarray: - """get time-dependent firing rate for units in the group + """Get time-dependent firing rate for units in the group Parameters ---------- @@ -259,33 +260,22 @@ def get_firing_rate( time : np.ndarray time vector for which to calculate the firing rate multiunit : bool, optional - if True, return the multiunit firing rate for units in the group, by default False + if True, return the multiunit firing rate for units in the group, + by default False smoothing_sigma : float, optional - standard deviation of gaussian filter to smooth firing rates in seconds, by default 0.015 + standard deviation of gaussian filter to smooth firing rates in + seconds, by default 0.015 Returns ------- np.ndarray time-dependent firing rate with shape (len(time), n_units) """ - spike_indicator = cls.get_spike_indicator(key, time) - if spike_indicator.ndim == 1: - spike_indicator = spike_indicator[:, np.newaxis] - - sampling_frequency = 1 / np.median(np.diff(time)) - - if multiunit: - spike_indicator = spike_indicator.sum(axis=1, keepdims=True) - return np.stack( - [ - get_multiunit_population_firing_rate( - indicator[:, np.newaxis], - sampling_frequency, - smoothing_sigma, - ) - for indicator in spike_indicator.T - ], - axis=1, + return firing_rate_from_spike_indicator( + spike_indicator=cls.get_spike_indicator(key, time), + time=time, + multiunit=multiunit, + smoothing_sigma=smoothing_sigma, ) diff --git a/src/spyglass/spikesorting/spikesorting_merge.py b/src/spyglass/spikesorting/spikesorting_merge.py index 9886958a0..4887cb3f3 100644 --- a/src/spyglass/spikesorting/spikesorting_merge.py +++ b/src/spyglass/spikesorting/spikesorting_merge.py @@ -17,6 +17,7 @@ from spyglass.utils.dj_merge_tables import _Merge from spyglass.utils.dj_mixin import SpyglassMixin from spyglass.utils.logging import logger +from spyglass.utils.spikesorting import firing_rate_from_spike_indicator schema = dj.schema("spikesorting_merge") @@ -209,7 +210,7 @@ def get_spike_indicator(cls, key, time): """ time = np.asarray(time) min_time, max_time = time[[0, -1]] - spike_times = cls.fetch_spike_data(key) + spike_times = cls.fetch_spike_data(key) # CB: This is undefined. spike_indicator = np.zeros((len(time), len(spike_times))) for ind, times in enumerate(spike_times): @@ -232,22 +233,30 @@ def get_firing_rate( multiunit: bool = False, smoothing_sigma: float = 0.015, ): - spike_indicator = cls.get_spike_indicator(key, time) - if spike_indicator.ndim == 1: - spike_indicator = spike_indicator[:, np.newaxis] + """Get time-dependent firing rate for units in the group - sampling_frequency = 1 / np.median(np.diff(time)) - if multiunit: - spike_indicator = spike_indicator.sum(axis=1, keepdims=True) - return np.stack( - [ - get_multiunit_population_firing_rate( - indicator[:, np.newaxis], - sampling_frequency, - smoothing_sigma, - ) - for indicator in spike_indicator.T - ], - axis=1, + Parameters + ---------- + key : dict + key to identify the group + time : np.ndarray + time vector for which to calculate the firing rate + multiunit : bool, optional + if True, return the multiunit firing rate for units in the group. + Default False + smoothing_sigma : float, optional + standard deviation of gaussian filter to smooth firing rates in + seconds. Default 0.015 + + Returns + ------- + np.ndarray + time-dependent firing rate with shape (len(time), n_units) + """ + return firing_rate_from_spike_indicator( + spike_indicator=cls.get_spike_indicator(key, time), + time=time, + multiunit=multiunit, + smoothing_sigma=smoothing_sigma, ) diff --git a/src/spyglass/spikesorting/utils.py b/src/spyglass/spikesorting/utils.py index 42a9814b9..591047016 100644 --- a/src/spyglass/spikesorting/utils.py +++ b/src/spyglass/spikesorting/utils.py @@ -1,4 +1,8 @@ +import warnings +from typing import Dict, List + import numpy as np +import scipy.stats as stats import spikeinterface as si from spyglass.common.common_ephys import Electrode @@ -131,3 +135,159 @@ def get_group_by_shank( ) sort_group += 1 return sg_keys, sge_keys + + +def _init_artifact_worker( + recording, + zscore_thresh=None, + amplitude_thresh_uV=None, + proportion_above_thresh=1.0, +): + recording = ( + si.load_extractor(recording) + if isinstance(recording, dict) + else recording + ) + # create a local dict per worker + return { + "recording": recording, + "zscore_thresh": zscore_thresh, + "amplitude_thresh_uV": amplitude_thresh_uV, + "proportion_above_thresh": proportion_above_thresh, + } + + +def _compute_artifact_chunk(segment_index, start_frame, end_frame, worker_ctx): + recording = worker_ctx["recording"] + zscore_thresh = worker_ctx["zscore_thresh"] + amplitude_thresh = worker_ctx["amplitude_thresh"] + proportion_above_thresh = worker_ctx["proportion_above_thresh"] + # compute the number of electrodes that have to be above threshold + nelect_above = np.ceil( + proportion_above_thresh * len(recording.get_channel_ids()) + ) + + traces = recording.get_traces( + segment_index=segment_index, + start_frame=start_frame, + end_frame=end_frame, + ) + + # find the artifact occurrences using one or both thresholds, across channels + if (amplitude_thresh is not None) and (zscore_thresh is None): + above_a = np.abs(traces) > amplitude_thresh + above_thresh = ( + np.ravel(np.argwhere(np.sum(above_a, axis=1) >= nelect_above)) + + start_frame + ) + elif (amplitude_thresh is None) and (zscore_thresh is not None): + dataz = np.abs(stats.zscore(traces, axis=1)) + above_z = dataz > zscore_thresh + above_thresh = ( + np.ravel(np.argwhere(np.sum(above_z, axis=1) >= nelect_above)) + + start_frame + ) + else: + above_a = np.abs(traces) > amplitude_thresh + dataz = np.abs(stats.zscore(traces, axis=1)) + above_z = dataz > zscore_thresh + above_thresh = ( + np.ravel( + np.argwhere( + np.sum(np.logical_or(above_z, above_a), axis=1) + >= nelect_above + ) + ) + + start_frame + ) + + return above_thresh + + +def _check_artifact_thresholds( + amplitude_thresh, zscore_thresh, proportion_above_thresh +): + """Alerts user to likely unintended parameters. Not an exhaustive verification. + + Parameters + ---------- + zscore_thresh: float + amplitude_thresh: float + Measured in microvolts. + proportion_above_thresh: float + + Return + ------ + zscore_thresh: float + amplitude_thresh: float + proportion_above_thresh: float + + Raise + ------ + ValueError: if signal thresholds are negative + """ + # amplitude or zscore thresholds should be negative, as they are applied to + # an absolute signal + + signal_thresholds = [ + t for t in [amplitude_thresh, zscore_thresh] if t is not None + ] + for t in signal_thresholds: + if t < 0: + raise ValueError( + "Amplitude and Z-Score thresholds must be >= 0, or None" + ) + + # proportion_above_threshold should be in [0:1] inclusive + if proportion_above_thresh < 0: + warnings.warn( + "Warning: proportion_above_thresh must be a proportion >0 and <=1." + " Using proportion_above_thresh = 0.01 instead of " + + f"{str(proportion_above_thresh)}" + ) + proportion_above_thresh = 0.01 + elif proportion_above_thresh > 1: + warnings.warn( + "Warning: proportion_above_thresh must be a proportion >0 and <=1. " + "Using proportion_above_thresh = 1 instead of " + + f"{str(proportion_above_thresh)}" + ) + proportion_above_thresh = 1 + return amplitude_thresh, zscore_thresh, proportion_above_thresh + + +def _get_recording_timestamps(recording): + num_segments = recording.get_num_segments() + + if num_segments <= 1: + return recording.get_times() + + frames_per_segment = [0] + [ + recording.get_num_frames(segment_index=i) for i in range(num_segments) + ] + + cumsum_frames = np.cumsum(frames_per_segment) + total_frames = np.sum(frames_per_segment) + + timestamps = np.zeros((total_frames,)) + for i in range(num_segments): + start_index = cumsum_frames[i] + end_index = cumsum_frames[i + 1] + timestamps[start_index:end_index] = recording.get_times(segment_index=i) + + return timestamps + + +def _reformat_metrics(metrics: Dict[str, Dict[str, float]]) -> List[Dict]: + return [ + { + "name": metric_name, + "label": metric_name, + "tooltip": metric_name, + "data": { + str(unit_id): metric_value + for unit_id, metric_value in metric.items() + }, + } + for metric_name, metric in metrics.items() + ] diff --git a/src/spyglass/spikesorting/v0/curation_figurl.py b/src/spyglass/spikesorting/v0/curation_figurl.py index c33903173..66327fdbb 100644 --- a/src/spyglass/spikesorting/v0/curation_figurl.py +++ b/src/spyglass/spikesorting/v0/curation_figurl.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Union +from typing import Any, List, Union import datajoint as dj import kachery_cloud as kcl @@ -6,6 +6,7 @@ import spikeinterface as si from sortingview.SpikeSortingView import SpikeSortingView +from spyglass.spikesorting.utils import _reformat_metrics from spyglass.spikesorting.v0.spikesorting_curation import Curation from spyglass.spikesorting.v0.spikesorting_recording import ( SpikeSortingRecording, @@ -191,18 +192,3 @@ def _generate_the_figurl( label = f"{recording_label} {sorting_label}" url = view.url(label=label, state=url_state) return url - - -def _reformat_metrics(metrics: Dict[str, Dict[str, float]]) -> List[Dict]: - return [ - { - "name": metric_name, - "label": metric_name, - "tooltip": metric_name, - "data": { - str(unit_id): metric_value - for unit_id, metric_value in metric.items() - }, - } - for metric_name, metric in metrics.items() - ] diff --git a/src/spyglass/spikesorting/v0/spikesorting_artifact.py b/src/spyglass/spikesorting/v0/spikesorting_artifact.py index c2962ba2e..1e72f9ce0 100644 --- a/src/spyglass/spikesorting/v0/spikesorting_artifact.py +++ b/src/spyglass/spikesorting/v0/spikesorting_artifact.py @@ -1,10 +1,8 @@ -import warnings from functools import reduce from typing import Union import datajoint as dj import numpy as np -import scipy.stats as stats import spikeinterface as si from spikeinterface.core.job_tools import ChunkRecordingExecutor, ensure_n_jobs @@ -14,6 +12,11 @@ interval_from_inds, interval_set_difference_inds, ) +from spyglass.spikesorting.utils import ( + _check_artifact_thresholds, + _compute_artifact_chunk, + _init_artifact_worker, +) from spyglass.spikesorting.v0.spikesorting_recording import ( SpikeSortingRecording, ) @@ -192,26 +195,26 @@ def _get_artifact_times( Intervals in which artifacts are detected (including removal windows), unit: seconds """ - if recording.get_num_segments() > 1: + num_segments = recording.get_num_segments() + if num_segments > 1: valid_timestamps = np.array([]) - for segment in range(recording.get_num_segments()): + for segment in range(num_segments): valid_timestamps = np.concatenate( (valid_timestamps, recording.get_times(segment_index=segment)) ) recording = si.concatenate_recordings([recording]) - elif recording.get_num_segments() == 1: + elif num_segments == 1: valid_timestamps = recording.get_times(0) # if both thresholds are None, we skip artifract detection - if (amplitude_thresh is None) and (zscore_thresh is None): - recording_interval = np.asarray( - [[valid_timestamps[0], valid_timestamps[-1]]] - ) - artifact_times_empty = np.asarray([]) + if amplitude_thresh is zscore_thresh is None: logger.info( - "Amplitude and zscore thresholds are both None, skipping artifact detection" + "Amplitude and zscore thresholds are both None, " + + "skipping artifact detection" ) - return recording_interval, artifact_times_empty + return np.asarray( + [[valid_timestamps[0], valid_timestamps[-1]]] + ), np.asarray([]) # verify threshold parameters ( @@ -219,12 +222,14 @@ def _get_artifact_times( zscore_thresh, proportion_above_thresh, ) = _check_artifact_thresholds( - amplitude_thresh, zscore_thresh, proportion_above_thresh + amplitude_thresh=amplitude_thresh, + zscore_thresh=zscore_thresh, + proportion_above_thresh=proportion_above_thresh, ) # detect frames that are above threshold in parallel n_jobs = ensure_n_jobs(recording, n_jobs=job_kwargs.get("n_jobs", 1)) - logger.info(f"using {n_jobs} jobs...") + logger.info(f"Using {n_jobs} jobs...") if n_jobs == 1: init_args = ( @@ -251,11 +256,13 @@ def _get_artifact_times( job_name="detect_artifact_frames", **job_kwargs, ) + artifact_frames = executor.run() artifact_frames = np.concatenate(artifact_frames) - # turn ms to remove total into s to remove from either side of each detected artifact - half_removal_window_s = removal_window_ms / 1000 * 0.5 + # turn ms to remove total into s to remove from either side of each + # detected artifact + half_removal_window_s = removal_window_ms / 2 / 1000 if len(artifact_frames) == 0: recording_interval = np.asarray( @@ -308,115 +315,3 @@ def _get_artifact_times( ) return np.asarray(artifact_removed_valid_times), artifact_intervals_s - - -def _init_artifact_worker( - recording, - zscore_thresh=None, - amplitude_thresh=None, - proportion_above_thresh=1.0, -): - # create a local dict per worker - worker_ctx = {} - if isinstance(recording, dict): - worker_ctx["recording"] = si.load_extractor(recording) - else: - worker_ctx["recording"] = recording - worker_ctx["zscore_thresh"] = zscore_thresh - worker_ctx["amplitude_thresh"] = amplitude_thresh - worker_ctx["proportion_above_thresh"] = proportion_above_thresh - return worker_ctx - - -def _compute_artifact_chunk(segment_index, start_frame, end_frame, worker_ctx): - recording = worker_ctx["recording"] - zscore_thresh = worker_ctx["zscore_thresh"] - amplitude_thresh = worker_ctx["amplitude_thresh"] - proportion_above_thresh = worker_ctx["proportion_above_thresh"] - # compute the number of electrodes that have to be above threshold - nelect_above = np.ceil( - proportion_above_thresh * len(recording.get_channel_ids()) - ) - - traces = recording.get_traces( - segment_index=segment_index, - start_frame=start_frame, - end_frame=end_frame, - ) - - # find the artifact occurrences using one or both thresholds, across channels - if (amplitude_thresh is not None) and (zscore_thresh is None): - above_a = np.abs(traces) > amplitude_thresh - above_thresh = ( - np.ravel(np.argwhere(np.sum(above_a, axis=1) >= nelect_above)) - + start_frame - ) - elif (amplitude_thresh is None) and (zscore_thresh is not None): - dataz = np.abs(stats.zscore(traces, axis=1)) - above_z = dataz > zscore_thresh - above_thresh = ( - np.ravel(np.argwhere(np.sum(above_z, axis=1) >= nelect_above)) - + start_frame - ) - else: - above_a = np.abs(traces) > amplitude_thresh - dataz = np.abs(stats.zscore(traces, axis=1)) - above_z = dataz > zscore_thresh - above_thresh = ( - np.ravel( - np.argwhere( - np.sum(np.logical_or(above_z, above_a), axis=1) - >= nelect_above - ) - ) - + start_frame - ) - - return above_thresh - - -def _check_artifact_thresholds( - amplitude_thresh, zscore_thresh, proportion_above_thresh -): - """Alerts user to likely unintended parameters. Not an exhaustive verification. - - Parameters - ---------- - zscore_thresh: float - amplitude_thresh: float - proportion_above_thresh: float - - Return - ------ - zscore_thresh: float - amplitude_thresh: float - proportion_above_thresh: float - - Raise - ------ - ValueError: if signal thresholds are negative - """ - # amplitude or zscore thresholds should be negative, as they are applied to an absolute signal - signal_thresholds = [ - t for t in [amplitude_thresh, zscore_thresh] if t is not None - ] - for t in signal_thresholds: - if t < 0: - raise ValueError( - "Amplitude and Z-Score thresholds must be >= 0, or None" - ) - - # proportion_above_threshold should be in [0:1] inclusive - if proportion_above_thresh < 0: - warnings.warn( - "Warning: proportion_above_thresh must be a proportion >0 and <=1." - f" Using proportion_above_thresh = 0.01 instead of {str(proportion_above_thresh)}" - ) - proportion_above_thresh = 0.01 - elif proportion_above_thresh > 1: - warnings.warn( - "Warning: proportion_above_thresh must be a proportion >0 and <=1. " - f"Using proportion_above_thresh = 1 instead of {str(proportion_above_thresh)}" - ) - proportion_above_thresh = 1 - return amplitude_thresh, zscore_thresh, proportion_above_thresh diff --git a/src/spyglass/spikesorting/v0/spikesorting_curation.py b/src/spyglass/spikesorting/v0/spikesorting_curation.py index edf99b8f4..bd40cd8ca 100644 --- a/src/spyglass/spikesorting/v0/spikesorting_curation.py +++ b/src/spyglass/spikesorting/v0/spikesorting_curation.py @@ -553,46 +553,48 @@ def _get_quality_metrics_name(self, key): return qm_name def _compute_metric(self, waveform_extractor, metric_name, **metric_params): - peak_sign_metrics = ["snr", "peak_offset", "peak_channel"] metric_func = _metric_name_to_func[metric_name] - # TODO clean up code below + + peak_sign_metrics = ["snr", "peak_offset", "peak_channel"] if metric_name == "isi_violation": metric = metric_func(waveform_extractor, **metric_params) elif metric_name in peak_sign_metrics: - if "peak_sign" in metric_params: - metric = metric_func( - waveform_extractor, - peak_sign=metric_params.pop("peak_sign"), - **metric_params, - ) - else: + if "peak_sign" not in metric_params: raise Exception( f"{peak_sign_metrics} metrics require peak_sign", "to be defined in the metric parameters", ) - else: - metric = {} - num_spikes = sq.compute_num_spikes(waveform_extractor) - for unit_id in waveform_extractor.sorting.get_unit_ids(): - # checks to avoid bug in spikeinterface 0.98.2 - if metric_name == "nn_isolation" and num_spikes[ - unit_id - ] < metric_params.get("min_spikes", 10): + return metric_func( + waveform_extractor, + peak_sign=metric_params.pop("peak_sign"), + **metric_params, + ) + + metric = {} + num_spikes = sq.compute_num_spikes(waveform_extractor) + + is_nn_iso = metric_name == "nn_isolation" + is_nn_overlap = metric_name == "nn_noise_overlap" + min_spikes = metric_params.get("min_spikes", 10) + + for unit_id in waveform_extractor.sorting.get_unit_ids(): + # checks to avoid bug in spikeinterface 0.98.2 + if num_spikes[unit_id] < min_spikes: + if is_nn_iso: metric[str(unit_id)] = (np.nan, np.nan) - elif metric_name == "nn_noise_overlap" and num_spikes[ - unit_id - ] < metric_params.get("min_spikes", 10): + elif is_nn_overlap: metric[str(unit_id)] = np.nan - else: - metric[str(unit_id)] = metric_func( - waveform_extractor, - this_unit_id=int(unit_id), - **metric_params, - ) - # nn_isolation returns tuple with isolation and unit number. We only want isolation. - if metric_name == "nn_isolation": - metric[str(unit_id)] = metric[str(unit_id)][0] + else: + metric[str(unit_id)] = metric_func( + waveform_extractor, + this_unit_id=int(unit_id), + **metric_params, + ) + # nn_isolation returns tuple with isolation and unit number. + # We only want isolation. + if is_nn_iso: + metric[str(unit_id)] = metric[str(unit_id)][0] return metric def _dump_to_json(self, qm_dict, save_path): diff --git a/src/spyglass/spikesorting/v0/spikesorting_recording.py b/src/spyglass/spikesorting/v0/spikesorting_recording.py index a7a2cb312..add9b455c 100644 --- a/src/spyglass/spikesorting/v0/spikesorting_recording.py +++ b/src/spyglass/spikesorting/v0/spikesorting_recording.py @@ -21,7 +21,10 @@ from spyglass.common.common_nwbfile import Nwbfile from spyglass.common.common_session import Session # noqa: F401 from spyglass.settings import recording_dir -from spyglass.spikesorting.utils import get_group_by_shank +from spyglass.spikesorting.utils import ( + _get_recording_timestamps, + get_group_by_shank, +) from spyglass.utils import SpyglassMixin from spyglass.utils.dj_helper_fn import dj_replace @@ -341,28 +344,7 @@ def _get_recording_name(key): @staticmethod def _get_recording_timestamps(recording): - num_segments = recording.get_num_segments() - - if num_segments <= 1: - return recording.get_times() - - frames_per_segment = [0] + [ - recording.get_num_frames(segment_index=i) - for i in range(num_segments) - ] - - cumsum_frames = np.cumsum(frames_per_segment) - total_frames = np.sum(frames_per_segment) - - timestamps = np.zeros((total_frames,)) - for i in range(num_segments): - start_index = cumsum_frames[i] - end_index = cumsum_frames[i + 1] - timestamps[start_index:end_index] = recording.get_times( - segment_index=i - ) - - return timestamps + return _get_recording_timestamps(recording) def _get_sort_interval_valid_times(self, key): """Identifies the intersection between sort interval specified by the user @@ -379,18 +361,24 @@ def _get_sort_interval_valid_times(self, key): (start, end) times for valid stretches of the sorting interval """ + nwb_file_name, sort_interval_name, params, interval_list_name = ( + SpikeSortingPreprocessingParameters * SpikeSortingRecordingSelection + & key + ).fetch1( + "nwb_file_name", + "sort_interval", + "preproc_params", + "interval_list_name", + ) + sort_interval = ( SortInterval & { - "nwb_file_name": key["nwb_file_name"], - "sort_interval_name": key["sort_interval_name"], + "nwb_file_name": nwb_file_name, + "sort_interval_name": sort_interval_name, } ).fetch1("sort_interval") - interval_list_name = (SpikeSortingRecordingSelection & key).fetch1( - "interval_list_name" - ) - valid_interval_times = ( IntervalList & { @@ -403,9 +391,6 @@ def _get_sort_interval_valid_times(self, key): sort_interval, valid_interval_times ) # Exclude intervals shorter than specified length - params = (SpikeSortingPreprocessingParameters & key).fetch1( - "preproc_params" - ) if "min_segment_length" in params: valid_sort_times = intervals_by_length( valid_sort_times, min_length=params["min_segment_length"] diff --git a/src/spyglass/spikesorting/v1/artifact.py b/src/spyglass/spikesorting/v1/artifact.py index f17651a8b..139f30c81 100644 --- a/src/spyglass/spikesorting/v1/artifact.py +++ b/src/spyglass/spikesorting/v1/artifact.py @@ -1,11 +1,9 @@ import uuid -import warnings from functools import reduce from typing import List, Union import datajoint as dj import numpy as np -import scipy.stats as stats import spikeinterface as si import spikeinterface.extractors as se from spikeinterface.core.job_tools import ChunkRecordingExecutor, ensure_n_jobs @@ -17,6 +15,11 @@ interval_list_complement, ) from spyglass.common.common_nwbfile import AnalysisNwbfile +from spyglass.spikesorting.utils import ( + _check_artifact_thresholds, + _compute_artifact_chunk, + _init_artifact_worker, +) from spyglass.spikesorting.v1.recording import ( SpikeSortingRecording, SpikeSortingRecordingSelection, @@ -205,6 +208,8 @@ def _get_artifact_times( Intervals in which artifacts are detected (including removal windows), unit: seconds """ + # CB: V0 checks num segments. Is that no longer necessary? + valid_timestamps = recording.get_times() # if both thresholds are None, we skip artifract detection @@ -223,7 +228,9 @@ def _get_artifact_times( zscore_thresh, proportion_above_thresh, ) = _check_artifact_thresholds( - amplitude_thresh_uV, zscore_thresh, proportion_above_thresh + amplitude_thresh=amplitude_thresh_uV, + zscore_thresh=zscore_thresh, + proportion_above_thresh=proportion_above_thresh, ) # detect frames that are above threshold in parallel @@ -259,7 +266,8 @@ def _get_artifact_times( artifact_frames = executor.run() artifact_frames = np.concatenate(artifact_frames) - # turn ms to remove total into s to remove from either side of each detected artifact + # turn ms to remove total into s to remove from either side of each + # detected artifact half_removal_window_s = removal_window_ms / 2 / 1000 if len(artifact_frames) == 0: @@ -308,118 +316,6 @@ def _get_artifact_times( return artifact_removed_valid_times, artifact_intervals_s -def _init_artifact_worker( - recording, - zscore_thresh=None, - amplitude_thresh_uV=None, - proportion_above_thresh=1.0, -): - # create a local dict per worker - worker_ctx = {} - if isinstance(recording, dict): - worker_ctx["recording"] = si.load_extractor(recording) - else: - worker_ctx["recording"] = recording - worker_ctx["zscore_thresh"] = zscore_thresh - worker_ctx["amplitude_thresh_uV"] = amplitude_thresh_uV - worker_ctx["proportion_above_thresh"] = proportion_above_thresh - return worker_ctx - - -def _compute_artifact_chunk(segment_index, start_frame, end_frame, worker_ctx): - recording = worker_ctx["recording"] - zscore_thresh = worker_ctx["zscore_thresh"] - amplitude_thresh_uV = worker_ctx["amplitude_thresh_uV"] - proportion_above_thresh = worker_ctx["proportion_above_thresh"] - # compute the number of electrodes that have to be above threshold - nelect_above = np.ceil( - proportion_above_thresh * len(recording.get_channel_ids()) - ) - - traces = recording.get_traces( - segment_index=segment_index, - start_frame=start_frame, - end_frame=end_frame, - ) - - # find the artifact occurrences using one or both thresholds, across channels - if (amplitude_thresh_uV is not None) and (zscore_thresh is None): - above_a = np.abs(traces) > amplitude_thresh_uV - above_thresh = ( - np.ravel(np.argwhere(np.sum(above_a, axis=1) >= nelect_above)) - + start_frame - ) - elif (amplitude_thresh_uV is None) and (zscore_thresh is not None): - dataz = np.abs(stats.zscore(traces, axis=1)) - above_z = dataz > zscore_thresh - above_thresh = ( - np.ravel(np.argwhere(np.sum(above_z, axis=1) >= nelect_above)) - + start_frame - ) - else: - above_a = np.abs(traces) > amplitude_thresh_uV - dataz = np.abs(stats.zscore(traces, axis=1)) - above_z = dataz > zscore_thresh - above_thresh = ( - np.ravel( - np.argwhere( - np.sum(np.logical_or(above_z, above_a), axis=1) - >= nelect_above - ) - ) - + start_frame - ) - - return above_thresh - - -def _check_artifact_thresholds( - amplitude_thresh_uV, zscore_thresh, proportion_above_thresh -): - """Alerts user to likely unintended parameters. Not an exhaustive verification. - - Parameters - ---------- - zscore_thresh: float - amplitude_thresh_uV: float - proportion_above_thresh: float - - Return - ------ - zscore_thresh: float - amplitude_thresh_uV: float - proportion_above_thresh: float - - Raise - ------ - ValueError: if signal thresholds are negative - """ - # amplitude or zscore thresholds should be negative, as they are applied to an absolute signal - signal_thresholds = [ - t for t in [amplitude_thresh_uV, zscore_thresh] if t is not None - ] - for t in signal_thresholds: - if t < 0: - raise ValueError( - "Amplitude and Z-Score thresholds must be >= 0, or None" - ) - - # proportion_above_threshold should be in [0:1] inclusive - if proportion_above_thresh < 0: - warnings.warn( - "Warning: proportion_above_thresh must be a proportion >0 and <=1." - f" Using proportion_above_thresh = 0.01 instead of {str(proportion_above_thresh)}" - ) - proportion_above_thresh = 0.01 - elif proportion_above_thresh > 1: - warnings.warn( - "Warning: proportion_above_thresh must be a proportion >0 and <=1. " - f"Using proportion_above_thresh = 1 instead of {str(proportion_above_thresh)}" - ) - proportion_above_thresh = 1 - return amplitude_thresh_uV, zscore_thresh, proportion_above_thresh - - def merge_intervals(intervals): """Takes a list of intervals each of which is [start_time, stop_time] and takes union over intervals that are intersecting diff --git a/src/spyglass/spikesorting/v1/figurl_curation.py b/src/spyglass/spikesorting/v1/figurl_curation.py index 3aa480451..fca4fb26b 100644 --- a/src/spyglass/spikesorting/v1/figurl_curation.py +++ b/src/spyglass/spikesorting/v1/figurl_curation.py @@ -9,6 +9,7 @@ from sortingview.SpikeSortingView import SpikeSortingView from spyglass.common.common_nwbfile import AnalysisNwbfile +from spyglass.spikesorting.utils import _reformat_metrics from spyglass.spikesorting.v1.curation import CurationV1, _merge_dict_to_list from spyglass.spikesorting.v1.sorting import SpikeSortingSelection from spyglass.utils import SpyglassMixin, logger @@ -270,18 +271,3 @@ def _generate_figurl( "sortingCuration": initial_curation_uri, }, ) - - -def _reformat_metrics(metrics: Dict[str, Dict[str, float]]) -> List[Dict]: - return [ - { - "name": metric_name, - "label": metric_name, - "tooltip": metric_name, - "data": { - str(unit_id): metric_value - for unit_id, metric_value in metric.items() - }, - } - for metric_name, metric in metrics.items() - ] diff --git a/src/spyglass/spikesorting/v1/recording.py b/src/spyglass/spikesorting/v1/recording.py index 06b7a4489..72f099a18 100644 --- a/src/spyglass/spikesorting/v1/recording.py +++ b/src/spyglass/spikesorting/v1/recording.py @@ -19,7 +19,10 @@ ) from spyglass.common.common_lab import LabTeam from spyglass.common.common_nwbfile import AnalysisNwbfile, Nwbfile -from spyglass.spikesorting.utils import get_group_by_shank +from spyglass.spikesorting.utils import ( + _get_recording_timestamps, + get_group_by_shank, +) from spyglass.utils import SpyglassMixin, logger schema = dj.schema("spikesorting_v1_recording") @@ -223,28 +226,7 @@ def get_recording(cls, key: dict) -> si.BaseRecording: @staticmethod def _get_recording_timestamps(recording): - num_segments = recording.get_num_segments() - - if num_segments <= 1: - return recording.get_times() - - frames_per_segment = [0] + [ - recording.get_num_frames(segment_index=i) - for i in range(num_segments) - ] - - cumsum_frames = np.cumsum(frames_per_segment) - total_frames = np.sum(frames_per_segment) - - timestamps = np.zeros((total_frames,)) - for i in range(num_segments): - start_index = cumsum_frames[i] - end_index = cumsum_frames[i + 1] - timestamps[start_index:end_index] = recording.get_times( - segment_index=i - ) - - return timestamps + return _get_recording_timestamps(recording) def _get_sort_interval_valid_times(self, key: dict): """Identifies the intersection between sort interval specified by the user diff --git a/src/spyglass/utils/dj_graph.py b/src/spyglass/utils/dj_graph.py index 3e90d4736..f437f6276 100644 --- a/src/spyglass/utils/dj_graph.py +++ b/src/spyglass/utils/dj_graph.py @@ -28,6 +28,7 @@ from spyglass.utils.database_settings import SHARED_MODULES from spyglass.utils.dj_helper_fn import ( PERIPHERAL_TABLES, + ensure_names, fuzzy_get, unique_dicts, ) @@ -148,7 +149,7 @@ def _log_truncate(self, log_str: str, max_len: int = 80): def _camel(self, table): """Convert table name(s) to camel case.""" - table = self._ensure_names(table) + table = ensure_names(table) if isinstance(table, str): return to_camel_case(table.split(".")[-1].strip("`")) if isinstance(table, Iterable) and not isinstance( @@ -169,12 +170,12 @@ def _ensure_names( if isinstance(table, Iterable) and not isinstance( table, (Table, TableMeta) ): - return [self._ensure_names(t) for t in table] + return [ensure_names(t) for t in table] return getattr(table, "full_table_name", None) def _get_node(self, table: Union[str, Table]): """Get node from graph.""" - table = self._ensure_names(table) + table = ensure_names(table) if not (node := self.graph.nodes.get(table)): raise ValueError( f"Table {table} not found in graph." @@ -184,7 +185,7 @@ def _get_node(self, table: Union[str, Table]): def _set_node(self, table, attr: str = "ft", value: Any = None): """Set attribute on node. General helper for various attributes.""" - table = self._ensure_names(table) + table = ensure_names(table) _ = self._get_node(table) # Ensure node exists self.graph.nodes[table][attr] = value @@ -200,8 +201,8 @@ def _get_edge(self, child: str, parent: str) -> Tuple[bool, Dict[str, str]]: Tuple of boolean indicating direction and edge data. True if child is child of parent. """ - child = self._ensure_names(child) - parent = self._ensure_names(parent) + child = ensure_names(child) + parent = ensure_names(parent) if edge := self.graph.get_edge_data(parent, child): return False, edge @@ -221,7 +222,7 @@ def _get_edge(self, child: str, parent: str) -> Tuple[bool, Dict[str, str]]: def _get_restr(self, table): """Get restriction from graph node.""" - return self._get_node(self._ensure_names(table)).get("restr") + return self._get_node(ensure_names(table)).get("restr") def _set_restr(self, table, restriction, replace=False): """Add restriction to graph node. If one exists, merge with new.""" @@ -247,7 +248,7 @@ def _set_restr(self, table, restriction, replace=False): def _get_ft(self, table, with_restr=False, warn=True): """Get FreeTable from graph node. If one doesn't exist, create it.""" - table = self._ensure_names(table) + table = ensure_names(table) if with_restr: if not (restr := self._get_restr(table) or False): if warn: @@ -263,7 +264,7 @@ def _get_ft(self, table, with_restr=False, warn=True): def _is_out(self, table, warn=True): """Check if table is outside of spyglass.""" - table = self._ensure_names(table) + table = ensure_names(table) if self.graph.nodes.get(table): return False ret = table.split(".")[0].split("_")[0].strip("`") not in SHARED_MODULES @@ -481,7 +482,7 @@ def _topo_sort( return nodes nodes = [ node - for node in self._ensure_names(nodes) + for node in ensure_names(nodes) if not self._is_out(node, warn=False) ] graph = self.graph.subgraph(nodes) if subgraph else self.graph @@ -838,8 +839,8 @@ def __init__( banned_tables: List[str] = None, **kwargs, ): - self.parent = self._ensure_names(parent) - self.child = self._ensure_names(child) + self.parent = ensure_names(parent) + self.child = ensure_names(child) if not self.parent and not self.child: raise ValueError("Parent or child table required.") @@ -848,7 +849,7 @@ def __init__( super().__init__(seed_table=seed_table, verbose=verbose) self._ignore_peripheral(except_tables=[self.parent, self.child]) - self.no_visit.update(self._ensure_names(banned_tables) or []) + self.no_visit.update(ensure_names(banned_tables) or []) self.no_visit.difference_update(set([self.parent, self.child])) self.searched_tables = set() self.found_restr = False @@ -882,7 +883,7 @@ def __init__( def _ignore_peripheral(self, except_tables: List[str] = None): """Ignore peripheral tables in graph traversal.""" - except_tables = self._ensure_names(except_tables) + except_tables = ensure_names(except_tables) ignore_tables = set(PERIPHERAL_TABLES) - set(except_tables or []) self.no_visit.update(ignore_tables) self.undirect_graph.remove_nodes_from(ignore_tables) diff --git a/src/spyglass/utils/dj_helper_fn.py b/src/spyglass/utils/dj_helper_fn.py index 9dfa6f02d..ae95deab1 100644 --- a/src/spyglass/utils/dj_helper_fn.py +++ b/src/spyglass/utils/dj_helper_fn.py @@ -4,13 +4,14 @@ import multiprocessing.pool import os from pathlib import Path -from typing import List, Type, Union +from typing import Iterable, List, Type, Union from uuid import uuid4 import datajoint as dj import h5py import numpy as np -from datajoint.user_tables import UserTable +from datajoint.table import Table +from datajoint.user_tables import TableMeta, UserTable from spyglass.utils.logging import logger from spyglass.utils.nwb_helper_fn import file_from_dandi, get_nwb_file @@ -33,6 +34,21 @@ ] +def ensure_names( + self, table: Union[str, Table, Iterable] = None +) -> Union[str, List[str], None]: + """Ensure table is a string.""" + if table is None: + return None + if isinstance(table, str): + return table + if isinstance(table, Iterable) and not isinstance( + table, (Table, TableMeta) + ): + return [ensure_names(t) for t in table] + return getattr(table, "full_table_name", None) + + def fuzzy_get(index: Union[int, str], names: List[str], sources: List[str]): """Given lists of items/names, return item at index or by substring.""" if isinstance(index, int): diff --git a/src/spyglass/utils/dj_mixin.py b/src/spyglass/utils/dj_mixin.py index b039add59..3a8540f35 100644 --- a/src/spyglass/utils/dj_mixin.py +++ b/src/spyglass/utils/dj_mixin.py @@ -22,6 +22,7 @@ from spyglass.utils.database_settings import SHARED_MODULES from spyglass.utils.dj_helper_fn import ( NonDaemonPool, + ensure_names, fetch_nwb, get_nwb_table, populate_pass_function, @@ -914,20 +915,13 @@ def __rshift__(self, restriction) -> QueryExpression: """ return self.restrict_by(restriction, direction="down") - def _ensure_names(self, tables) -> List[str]: - """Ensure table is a string in a list.""" - if not isinstance(tables, (list, tuple, set)): - tables = [tables] - for table in tables: - return [getattr(table, "full_table_name", table) for t in tables] - def ban_search_table(self, table): """Ban table from search in restrict_by.""" - self._banned_search_tables.update(self._ensure_names(table)) + self._banned_search_tables.update(ensure_names(table)) def unban_search_table(self, table): """Unban table from search in restrict_by.""" - self._banned_search_tables.difference_update(self._ensure_names(table)) + self._banned_search_tables.difference_update(ensure_names(table)) def see_banned_tables(self): """Print banned tables.""" diff --git a/src/spyglass/utils/spikesorting.py b/src/spyglass/utils/spikesorting.py new file mode 100644 index 000000000..3343a62ff --- /dev/null +++ b/src/spyglass/utils/spikesorting.py @@ -0,0 +1,28 @@ +import numpy as np +from ripple_detection import get_multiunit_population_firing_rate + + +def firing_rate_from_spike_indicator( + spike_indicator: np.ndarray, + time: np.array, + multiunit: bool = False, + smoothing_sigma: float = 0.015, +): + if spike_indicator.ndim == 1: + spike_indicator = spike_indicator[:, np.newaxis] + + sampling_frequency = 1 / np.median(np.diff(time)) + + if multiunit: + spike_indicator = spike_indicator.sum(axis=1, keepdims=True) + return np.stack( + [ + get_multiunit_population_firing_rate( + indicator[:, np.newaxis], + sampling_frequency, + smoothing_sigma, + ) + for indicator in spike_indicator.T + ], + axis=1, + ) From 96d11a3df089b68baaf7f70846b24ff27388f35a Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Mon, 12 Aug 2024 14:42:45 -0500 Subject: [PATCH 32/94] #1053 bugfix (#1062) --- CHANGELOG.md | 2 +- src/spyglass/utils/dj_helper_fn.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e0ebe5c1..27c681c4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,7 +39,7 @@ PositionGroup.alter() - Allow `ModuleNotFoundError` or `ImportError` for optional dependencies #1023 - Ensure integrity of group tables #1026 - Convert list of LFP artifact removed interval list to array #1046 -- Merge duplicate functions in decoding and spikesorting #1050, #1053 +- Merge duplicate functions in decoding and spikesorting #1050, #1053, #1058 - Revise docs organization. - Misc -> Features/ForDevelopers. #1029 - Installation instructions -> Setup notebook. #1029 diff --git a/src/spyglass/utils/dj_helper_fn.py b/src/spyglass/utils/dj_helper_fn.py index ae95deab1..90fd47cc0 100644 --- a/src/spyglass/utils/dj_helper_fn.py +++ b/src/spyglass/utils/dj_helper_fn.py @@ -35,7 +35,7 @@ def ensure_names( - self, table: Union[str, Table, Iterable] = None + table: Union[str, Table, Iterable] = None ) -> Union[str, List[str], None]: """Ensure table is a string.""" if table is None: From fa968bbbb3afd808af7a78e604f620018888684e Mon Sep 17 00:00:00 2001 From: Samuel Bray Date: Thu, 15 Aug 2024 08:56:38 -0700 Subject: [PATCH 33/94] Spikesort artifact fixes (#1069) * spikesort_artifact fixes * update changelog * update changelog --- CHANGELOG.md | 4 ++-- src/spyglass/spikesorting/utils.py | 2 +- src/spyglass/spikesorting/v0/spikesorting_artifact.py | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 27c681c4e..6c2158e80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,12 +34,12 @@ PositionGroup.alter() - Checks `IntervalList` and externals tables. #1002 - Ends early if called on empty table. #1055 - Allow mixin tables with parallelization in `make` to run populate with - `processes > 1` #1001, #1052 + `processes > 1` #1001, #1052, #1068 - Speed up fetch_nwb calls through merge tables #1017 - Allow `ModuleNotFoundError` or `ImportError` for optional dependencies #1023 - Ensure integrity of group tables #1026 - Convert list of LFP artifact removed interval list to array #1046 -- Merge duplicate functions in decoding and spikesorting #1050, #1053, #1058 +- Merge duplicate functions in decoding and spikesorting #1050, #1053, #1058, #1069 - Revise docs organization. - Misc -> Features/ForDevelopers. #1029 - Installation instructions -> Setup notebook. #1029 diff --git a/src/spyglass/spikesorting/utils.py b/src/spyglass/spikesorting/utils.py index 591047016..bb1c516c2 100644 --- a/src/spyglass/spikesorting/utils.py +++ b/src/spyglass/spikesorting/utils.py @@ -152,7 +152,7 @@ def _init_artifact_worker( return { "recording": recording, "zscore_thresh": zscore_thresh, - "amplitude_thresh_uV": amplitude_thresh_uV, + "amplitude_thresh": amplitude_thresh_uV, "proportion_above_thresh": proportion_above_thresh, } diff --git a/src/spyglass/spikesorting/v0/spikesorting_artifact.py b/src/spyglass/spikesorting/v0/spikesorting_artifact.py index 1e72f9ce0..f78cf0d06 100644 --- a/src/spyglass/spikesorting/v0/spikesorting_artifact.py +++ b/src/spyglass/spikesorting/v0/spikesorting_artifact.py @@ -73,6 +73,8 @@ class ArtifactDetection(SpyglassMixin, dj.Computed): artifact_removed_interval_list_name: varchar(200) # name of the array of no-artifact valid time intervals """ + _parallel_make = True + def make(self, key): if not (ArtifactDetectionSelection & key).fetch1( "custom_artifact_detection" From f5b624906f5b3472c7f1e20fe0e5b921ff06c6a5 Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Thu, 15 Aug 2024 16:19:14 -0500 Subject: [PATCH 34/94] Add check threads util (#1063) * Add check threads util * Convert to classmethod * Adjust pr #s in changelog * Remove classmethod decorator --------- Co-authored-by: Samuel Bray --- CHANGELOG.md | 3 +- src/spyglass/utils/dj_mixin.py | 136 ++++++++++++++++++++++++++++----- 2 files changed, 120 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c2158e80..dc891c2f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,11 +39,12 @@ PositionGroup.alter() - Allow `ModuleNotFoundError` or `ImportError` for optional dependencies #1023 - Ensure integrity of group tables #1026 - Convert list of LFP artifact removed interval list to array #1046 -- Merge duplicate functions in decoding and spikesorting #1050, #1053, #1058, #1069 +- Merge duplicate functions in decoding and spikesorting #1050, #1053, #1062, #1069 - Revise docs organization. - Misc -> Features/ForDevelopers. #1029 - Installation instructions -> Setup notebook. #1029 - Migrate SQL export tools to `utils` to support exporting `DandiPath` #1048 +- Add tool for checking threads for metadata locks on a table #1063 ### Pipelines diff --git a/src/spyglass/utils/dj_mixin.py b/src/spyglass/utils/dj_mixin.py index 3a8540f35..533976329 100644 --- a/src/spyglass/utils/dj_mixin.py +++ b/src/spyglass/utils/dj_mixin.py @@ -17,6 +17,7 @@ from datajoint.utils import get_master, to_camel_case, user_choice from networkx import NetworkXError from packaging.version import parse as version_parse +from pandas import DataFrame from pymysql.err import DataError from spyglass.utils.database_settings import SHARED_MODULES @@ -158,10 +159,10 @@ def _nwb_table_tuple(self) -> tuple: Used to determine fetch_nwb behavior. Also used in Merge.fetch_nwb. Implemented as a cached_property to avoid circular imports.""" - from spyglass.common.common_nwbfile import ( + from spyglass.common.common_nwbfile import ( # noqa F401 AnalysisNwbfile, Nwbfile, - ) # noqa F401 + ) table_dict = { AnalysisNwbfile: "analysis_file_abs_path", @@ -263,12 +264,12 @@ def fetch_pynapple(self, *attrs, **kwargs): def _import_part_masters(self): """Import tables that may constrain a RestrGraph. See #1002""" from spyglass.decoding.decoding_merge import DecodingOutput # noqa F401 - from spyglass.decoding.v0.clusterless import ( # noqa F401 + from spyglass.decoding.v0.clusterless import ( UnitMarksIndicatorSelection, - ) - from spyglass.decoding.v0.sorted_spikes import ( # noqa F401 + ) # noqa F401 + from spyglass.decoding.v0.sorted_spikes import ( SortedSpikesIndicatorSelection, - ) + ) # noqa F401 from spyglass.decoding.v1.core import PositionGroup # noqa F401 from spyglass.lfp.analysis.v1 import LFPBandSelection # noqa F401 from spyglass.lfp.lfp_merge import LFPOutput # noqa F401 @@ -279,15 +280,15 @@ def _import_part_masters(self): from spyglass.mua.v1.mua import MuaEventsV1 # noqa F401 from spyglass.position.position_merge import PositionOutput # noqa F401 from spyglass.ripple.v1.ripple import RippleTimesV1 # noqa F401 - from spyglass.spikesorting.analysis.v1.group import ( # noqa F401 + from spyglass.spikesorting.analysis.v1.group import ( SortedSpikesGroup, - ) - from spyglass.spikesorting.spikesorting_merge import ( # noqa F401 + ) # noqa F401 + from spyglass.spikesorting.spikesorting_merge import ( SpikeSortingOutput, - ) - from spyglass.spikesorting.v0.figurl_views import ( # noqa F401 + ) # noqa F401 + from spyglass.spikesorting.v0.figurl_views import ( SpikeSortingRecordingView, - ) + ) # noqa F401 _ = ( DecodingOutput(), @@ -475,12 +476,8 @@ def _delete_deps(self) -> List[Table]: Used to delay import of tables until needed, avoiding circular imports. Each of these tables inheits SpyglassMixin. """ - from spyglass.common import ( # noqa F401 - IntervalList, - LabMember, - LabTeam, - Session, - ) + from spyglass.common import LabMember # noqa F401 + from spyglass.common import IntervalList, LabTeam, Session from spyglass.common.common_nwbfile import schema # noqa F401 self._session_pk = Session.primary_key[0] @@ -1018,6 +1015,109 @@ def restrict_by( return ret + # ------------------------------ Check locks ------------------------------ + + def exec_sql_fetchall(self, query): + """ + Execute the given query and fetch the results. Parameters + ---------- + query : str + The SQL query to execute. Returns + ------- + list of tuples + The results of the query. + """ + results = dj.conn().query(query).fetchall() + return results # Check if performance schema is enabled + + def check_threads(self, detailed=False, all_threads=False) -> DataFrame: + """Check for locked threads in the database. + + Parameters + ---------- + detailed : bool, optional + Show all columns in the metadata_locks table. Default False, show + summary. + all_threads : bool, optional + Show all threads, not just those related to this table. + Default False. + + + Returns + ------- + DataFrame + A DataFrame containing the metadata locks. + """ + performance__status = self.exec_sql_fetchall( + "SHOW VARIABLES LIKE 'performance_schema';" + ) + if performance__status[0][1] == "OFF": + raise RuntimeError( + "Database does not monitor threads. " + + "Please ask you administrator to enable performance schema." + ) + + metadata_locks_query = """ + SELECT + ml.OBJECT_SCHEMA, -- Table schema + ml.OBJECT_NAME, -- Table name + ml.OBJECT_TYPE, -- What is locked + ml.LOCK_TYPE, -- Type of lock + ml.LOCK_STATUS, -- Lock status + ml.OWNER_THREAD_ID, -- Thread ID of the lock owner + t.PROCESSLIST_ID, -- User connection ID + t.PROCESSLIST_USER, -- User + t.PROCESSLIST_HOST, -- User machine + t.PROCESSLIST_TIME, -- Time in seconds + t.PROCESSLIST_DB, -- Thread database + t.PROCESSLIST_COMMAND, -- Likely Query + t.PROCESSLIST_STATE, -- Waiting for lock, sending data, or locked + t.PROCESSLIST_INFO -- Actual query + FROM performance_schema.metadata_locks AS ml + JOIN performance_schema.threads AS t + ON ml.OWNER_THREAD_ID = t.THREAD_ID + """ + + where_clause = ( + f"WHERE ml.OBJECT_SCHEMA = '{self.database}' " + + f"AND ml.OBJECT_NAME = '{self.table_name}'" + ) + metadata_locks_query += ";" if all_threads else where_clause + + df = DataFrame( + self.exec_sql_fetchall(metadata_locks_query), + columns=[ + "Schema", # ml.OBJECT_SCHEMA -- Table schema + "Table Name", # ml.OBJECT_NAME -- Table name + "Locked", # ml.OBJECT_TYPE -- What is locked + "Lock Type", # ml.LOCK_TYPE -- Type of lock + "Lock Status", # ml.LOCK_STATUS -- Lock status + "Thread ID", # ml.OWNER_THREAD_ID -- Thread ID of the lock owner + "Connection ID", # t.PROCESSLIST_ID -- User connection ID + "User", # t.PROCESSLIST_USER -- User + "Host", # t.PROCESSLIST_HOST -- User machine + "Process Database", # t.PROCESSLIST_DB -- Thread database + "Time (s)", # t.PROCESSLIST_TIME -- Time in seconds + "Process", # t.PROCESSLIST_COMMAND -- Likely Query + "State", # t.PROCESSLIST_STATE + "Query", # t.PROCESSLIST_INFO -- Actual query + ], + ) + + df["Name"] = df["User"].apply(self._delete_deps[0]().get_djuser_name) + + keep_cols = [] + if all_threads: + keep_cols.append("Table") + df["Table"] = df["Schema"] + "." + df["Table Name"] + df = df.drop(columns=["Schema", "Table Name"]) + + if not detailed: + keep_cols.extend(["Locked", "Name", "Time (s)", "Process", "State"]) + df = df[keep_cols] + + return df + class SpyglassMixinPart(SpyglassMixin, dj.Part): """ From 36a8bdaff7dc51c532f99201ee643da8f5d4294f Mon Sep 17 00:00:00 2001 From: Samuel Bray Date: Tue, 20 Aug 2024 06:50:58 -0700 Subject: [PATCH 35/94] fix fetch efficiency in insert_curation (#1072) * fix fetch efficiency in insert_curation * update changelog --- CHANGELOG.md | 1 + src/spyglass/spikesorting/v0/spikesorting_curation.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc891c2f6..57e315635 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -96,6 +96,7 @@ PositionGroup.alter() - Add `UnitAnnotation` table and naming convention for units #1027, #1052 - Set `sparse` parameter to waveform extraction step in `spikesorting.v1` #1039 + - Efficiency improvement to `v0.Curation.insert_curation` #1072 ## [0.5.2] (April 22, 2024) diff --git a/src/spyglass/spikesorting/v0/spikesorting_curation.py b/src/spyglass/spikesorting/v0/spikesorting_curation.py index bd40cd8ca..57fb90fa5 100644 --- a/src/spyglass/spikesorting/v0/spikesorting_curation.py +++ b/src/spyglass/spikesorting/v0/spikesorting_curation.py @@ -145,7 +145,7 @@ def insert_curation( Curation.insert1(sorting_key, skip_duplicates=True) # get the primary key for this curation - c_key = Curation.fetch("KEY")[0] + c_key = (Curation & sorting_key).fetch1("KEY") curation_key = {item: sorting_key[item] for item in c_key} return curation_key From 7b3eae9c9c40a51b5f96587020f6ef256737fc77 Mon Sep 17 00:00:00 2001 From: Samuel Bray Date: Tue, 20 Aug 2024 06:51:21 -0700 Subject: [PATCH 36/94] Decode fixes (#1073) * fix time slicing in get_ahead_behind_distance * fix fetched attribute name in _get_sort_interval_valid_times * update dtype to np.int32 * ensure unit wavform group integrity * update changelog --- CHANGELOG.md | 1 + src/spyglass/decoding/v1/clusterless.py | 7 ++++++- src/spyglass/spikesorting/v0/spikesorting_recording.py | 2 +- src/spyglass/spikesorting/v0/spikesorting_sorting.py | 2 +- src/spyglass/spikesorting/v1/sorting.py | 2 +- 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57e315635..b00777da2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,6 +68,7 @@ PositionGroup.alter() - Default values for classes on `ImportError` #966 - Add option to upsample data rate in `PositionGroup` #1008 - Avoid interpolating over large `nan` intervals in position #1033 + - Minor code calling corrections #1073 - Position diff --git a/src/spyglass/decoding/v1/clusterless.py b/src/spyglass/decoding/v1/clusterless.py index e5676b1f9..bfa83e6ac 100644 --- a/src/spyglass/decoding/v1/clusterless.py +++ b/src/spyglass/decoding/v1/clusterless.py @@ -57,6 +57,11 @@ def create_group( "nwb_file_name": nwb_file_name, "waveform_features_group_name": group_name, } + if self & group_key: + raise ValueError( + f"Group {nwb_file_name}: {group_name} already exists", + "please delete the group before creating a new one", + ) self.insert1( group_key, skip_duplicates=True, @@ -533,7 +538,7 @@ def get_ahead_behind_distance(self, track_graph=None, time_slice=None): classifier = self.fetch_model() posterior = ( self.fetch_results() - .acausal_posterior(time=time_slice) + .acausal_posterior.sel(time=time_slice) .squeeze() .unstack("state_bins") .sum("state") diff --git a/src/spyglass/spikesorting/v0/spikesorting_recording.py b/src/spyglass/spikesorting/v0/spikesorting_recording.py index add9b455c..04369b15e 100644 --- a/src/spyglass/spikesorting/v0/spikesorting_recording.py +++ b/src/spyglass/spikesorting/v0/spikesorting_recording.py @@ -366,7 +366,7 @@ def _get_sort_interval_valid_times(self, key): & key ).fetch1( "nwb_file_name", - "sort_interval", + "sort_interval_name", "preproc_params", "interval_list_name", ) diff --git a/src/spyglass/spikesorting/v0/spikesorting_sorting.py b/src/spyglass/spikesorting/v0/spikesorting_sorting.py index 9e4d84a61..ac5648552 100644 --- a/src/spyglass/spikesorting/v0/spikesorting_sorting.py +++ b/src/spyglass/spikesorting/v0/spikesorting_sorting.py @@ -213,7 +213,7 @@ def make(self, key: dict): detected_spikes = detect_peaks(recording, **sorter_params) sorting = si.NumpySorting.from_times_labels( times_list=detected_spikes["sample_index"], - labels_list=np.zeros(len(detected_spikes), dtype=np.int), + labels_list=np.zeros(len(detected_spikes), dtype=np.int32), sampling_frequency=recording.get_sampling_frequency(), ) else: diff --git a/src/spyglass/spikesorting/v1/sorting.py b/src/spyglass/spikesorting/v1/sorting.py index f738c5ff0..9196eb627 100644 --- a/src/spyglass/spikesorting/v1/sorting.py +++ b/src/spyglass/spikesorting/v1/sorting.py @@ -240,7 +240,7 @@ def make(self, key: dict): detected_spikes = detect_peaks(recording, **sorter_params) sorting = si.NumpySorting.from_times_labels( times_list=detected_spikes["sample_index"], - labels_list=np.zeros(len(detected_spikes), dtype=np.int), + labels_list=np.zeros(len(detected_spikes), dtype=np.int32), sampling_frequency=recording.get_sampling_frequency(), ) else: From 012ea30f05ee6f711acdc93bd673834993a05cc4 Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Tue, 20 Aug 2024 09:19:35 -0500 Subject: [PATCH 37/94] Ban tables in distance restrict bugfix (#1066) * Ban tables in distance restrict bugfix * Update changelog --- CHANGELOG.md | 3 ++- src/spyglass/utils/dj_graph.py | 14 -------------- src/spyglass/utils/dj_helper_fn.py | 29 ++++++++++++++++++++++++----- src/spyglass/utils/dj_mixin.py | 6 ++++-- 4 files changed, 30 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b00777da2..a5d467ec1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,7 +39,8 @@ PositionGroup.alter() - Allow `ModuleNotFoundError` or `ImportError` for optional dependencies #1023 - Ensure integrity of group tables #1026 - Convert list of LFP artifact removed interval list to array #1046 -- Merge duplicate functions in decoding and spikesorting #1050, #1053, #1062, #1069 +- Merge duplicate functions in decoding and spikesorting #1050, #1053, #1058, + #1066 - Revise docs organization. - Misc -> Features/ForDevelopers. #1029 - Installation instructions -> Setup notebook. #1029 diff --git a/src/spyglass/utils/dj_graph.py b/src/spyglass/utils/dj_graph.py index f437f6276..0ab4ab477 100644 --- a/src/spyglass/utils/dj_graph.py +++ b/src/spyglass/utils/dj_graph.py @@ -159,20 +159,6 @@ def _camel(self, table): # ------------------------------ Graph Nodes ------------------------------ - def _ensure_names( - self, table: Union[str, Table] = None - ) -> Union[str, List[str]]: - """Ensure table is a string.""" - if table is None: - return None - if isinstance(table, str): - return table - if isinstance(table, Iterable) and not isinstance( - table, (Table, TableMeta) - ): - return [ensure_names(t) for t in table] - return getattr(table, "full_table_name", None) - def _get_node(self, table: Union[str, Table]): """Get node from graph.""" table = ensure_names(table) diff --git a/src/spyglass/utils/dj_helper_fn.py b/src/spyglass/utils/dj_helper_fn.py index 90fd47cc0..0bf61b734 100644 --- a/src/spyglass/utils/dj_helper_fn.py +++ b/src/spyglass/utils/dj_helper_fn.py @@ -35,16 +35,35 @@ def ensure_names( - table: Union[str, Table, Iterable] = None + table: Union[str, Table, Iterable] = None, force_list: bool = False ) -> Union[str, List[str], None]: - """Ensure table is a string.""" + """Ensure table is a string. + + Parameters + ---------- + table : Union[str, Table, Iterable], optional + Table to ensure is a string, by default None. If passed as iterable, + will ensure all elements are strings. + force_list : bool, optional + Force the return to be a list, by default False, only used if input is + iterable. + + Returns + ------- + Union[str, List[str], None] + Table as a string or list of strings. + """ + # is iterable (list, set, set) but not a table/string + is_collection = isinstance(table, Iterable) and not isinstance( + table, (Table, TableMeta, str) + ) + if force_list and not is_collection: + return [ensure_names(table)] if table is None: return None if isinstance(table, str): return table - if isinstance(table, Iterable) and not isinstance( - table, (Table, TableMeta) - ): + if is_collection: return [ensure_names(t) for t in table] return getattr(table, "full_table_name", None) diff --git a/src/spyglass/utils/dj_mixin.py b/src/spyglass/utils/dj_mixin.py index 533976329..ff3922087 100644 --- a/src/spyglass/utils/dj_mixin.py +++ b/src/spyglass/utils/dj_mixin.py @@ -914,11 +914,13 @@ def __rshift__(self, restriction) -> QueryExpression: def ban_search_table(self, table): """Ban table from search in restrict_by.""" - self._banned_search_tables.update(ensure_names(table)) + self._banned_search_tables.update(ensure_names(table, force_list=True)) def unban_search_table(self, table): """Unban table from search in restrict_by.""" - self._banned_search_tables.difference_update(ensure_names(table)) + self._banned_search_tables.difference_update( + ensure_names(table, force_list=True) + ) def see_banned_tables(self): """Print banned tables.""" From ecf468e2c7ff77652d6b646f7f07d52ed19e84e3 Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Tue, 27 Aug 2024 11:06:59 -0500 Subject: [PATCH 38/94] Periph table fallback on TableChain for experimenter summary (#1035) * Periph table fallback on TableChain * Update Changelog * Rely on search to remove no_visit, not id step * Include generic load_shared_schemas * Update changelog for release * Allow add custom prefix for load schemas * Fix merge error --- CHANGELOG.md | 18 +---- src/spyglass/utils/dj_graph.py | 43 +++++++++--- src/spyglass/utils/dj_mixin.py | 117 ++++++++++++++++----------------- 3 files changed, 95 insertions(+), 83 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5d467ec1..57fd495d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,20 +1,6 @@ # Change Log -## [0.5.3] (Unreleased) - -## Release Notes - - - -```python -import datajoint as dj -from spyglass.common.common_behav import PositionIntervalMap -from spyglass.decoding.v1.core import PositionGroup - -dj.schema("common_ripple").drop() -PositionIntervalMap.alter() -PositionGroup.alter() -``` +## [0.5.3] (August 27, 2024) ### Infrastructure @@ -46,6 +32,8 @@ PositionGroup.alter() - Installation instructions -> Setup notebook. #1029 - Migrate SQL export tools to `utils` to support exporting `DandiPath` #1048 - Add tool for checking threads for metadata locks on a table #1063 +- Use peripheral tables as fallback in `TableChains` #1035 +- Ignore non-Spyglass tables during descendant check for `part_masters` #1035 ### Pipelines diff --git a/src/spyglass/utils/dj_graph.py b/src/spyglass/utils/dj_graph.py index 0ab4ab477..6b3928042 100644 --- a/src/spyglass/utils/dj_graph.py +++ b/src/spyglass/utils/dj_graph.py @@ -248,7 +248,7 @@ def _get_ft(self, table, with_restr=False, warn=True): return ft & restr - def _is_out(self, table, warn=True): + def _is_out(self, table, warn=True, keep_alias=False): """Check if table is outside of spyglass.""" table = ensure_names(table) if self.graph.nodes.get(table): @@ -805,7 +805,8 @@ class TableChain(RestrGraph): Returns path OrderedDict of full table names in chain. If directed is True, uses directed graph. If False, uses undirected graph. Undirected excludes PERIPHERAL_TABLES like interval_list, nwbfile, etc. to maintain - valid joins. + valid joins by default. If no path is found, another search is attempted + with PERIPHERAL_TABLES included. cascade(restriction: str = None, direction: str = "up") Given a restriction at the beginning, return a restricted FreeTable object at the end of the chain. If direction is 'up', start at the child @@ -835,8 +836,12 @@ def __init__( super().__init__(seed_table=seed_table, verbose=verbose) self._ignore_peripheral(except_tables=[self.parent, self.child]) + self._ignore_outside_spy(except_tables=[self.parent, self.child]) + self.no_visit.update(ensure_names(banned_tables) or []) + self.no_visit.difference_update(set([self.parent, self.child])) + self.searched_tables = set() self.found_restr = False self.link_type = None @@ -872,7 +877,19 @@ def _ignore_peripheral(self, except_tables: List[str] = None): except_tables = ensure_names(except_tables) ignore_tables = set(PERIPHERAL_TABLES) - set(except_tables or []) self.no_visit.update(ignore_tables) - self.undirect_graph.remove_nodes_from(ignore_tables) + + def _ignore_outside_spy(self, except_tables: List[str] = None): + """Ignore tables not shared on shared prefixes.""" + except_tables = ensure_names(except_tables) + ignore_tables = set( # Ignore tables not in shared modules + [ + t + for t in self.undirect_graph.nodes + if t not in except_tables + and self._is_out(t, warn=False, keep_alias=True) + ] + ) + self.no_visit.update(ignore_tables) # --------------------------- Dunder Properties --------------------------- @@ -1066,9 +1083,9 @@ def find_path(self, directed=True) -> List[str]: List of names in the path. """ source, target = self.parent, self.child - search_graph = self.graph if directed else self.undirect_graph - - search_graph.remove_nodes_from(self.no_visit) + search_graph = ( # Copy to ensure orig not modified by no_visit + self.graph.copy() if directed else self.undirect_graph.copy() + ) try: path = shortest_path(search_graph, source, target) @@ -1096,6 +1113,12 @@ def path(self) -> list: self.link_type = "directed" elif path := self.find_path(directed=False): self.link_type = "undirected" + else: # Search with peripheral + self.no_visit.difference_update(PERIPHERAL_TABLES) + if path := self.find_path(directed=True): + self.link_type = "directed with peripheral" + elif path := self.find_path(directed=False): + self.link_type = "undirected with peripheral" self.searched_path = True return path @@ -1126,9 +1149,11 @@ def cascade( # Cascade will stop if any restriction is empty, so set rest to None # This would cause issues if we want a table partway through the chain # but that's not a typical use case, were the start and end are desired - non_numeric = [t for t in self.path if not t.isnumeric()] - if any(self._get_restr(t) is None for t in non_numeric): - for table in non_numeric: + safe_tbls = [ + t for t in self.path if not t.isnumeric() and not self._is_out(t) + ] + if any(self._get_restr(t) is None for t in safe_tbls): + for table in safe_tbls: if table is not start: self._set_restr(table, False, replace=True) diff --git a/src/spyglass/utils/dj_mixin.py b/src/spyglass/utils/dj_mixin.py index ff3922087..04b873740 100644 --- a/src/spyglass/utils/dj_mixin.py +++ b/src/spyglass/utils/dj_mixin.py @@ -261,52 +261,41 @@ def fetch_pynapple(self, *attrs, **kwargs): # ------------------------ delete_downstream_parts ------------------------ - def _import_part_masters(self): - """Import tables that may constrain a RestrGraph. See #1002""" - from spyglass.decoding.decoding_merge import DecodingOutput # noqa F401 - from spyglass.decoding.v0.clusterless import ( - UnitMarksIndicatorSelection, - ) # noqa F401 - from spyglass.decoding.v0.sorted_spikes import ( - SortedSpikesIndicatorSelection, - ) # noqa F401 - from spyglass.decoding.v1.core import PositionGroup # noqa F401 - from spyglass.lfp.analysis.v1 import LFPBandSelection # noqa F401 - from spyglass.lfp.lfp_merge import LFPOutput # noqa F401 - from spyglass.linearization.merge import ( # noqa F401 - LinearizedPositionOutput, - LinearizedPositionV1, - ) - from spyglass.mua.v1.mua import MuaEventsV1 # noqa F401 - from spyglass.position.position_merge import PositionOutput # noqa F401 - from spyglass.ripple.v1.ripple import RippleTimesV1 # noqa F401 - from spyglass.spikesorting.analysis.v1.group import ( - SortedSpikesGroup, - ) # noqa F401 - from spyglass.spikesorting.spikesorting_merge import ( - SpikeSortingOutput, - ) # noqa F401 - from spyglass.spikesorting.v0.figurl_views import ( - SpikeSortingRecordingView, - ) # noqa F401 - - _ = ( - DecodingOutput(), - LFPBandSelection(), - LFPOutput(), - LinearizedPositionOutput(), - LinearizedPositionV1(), - MuaEventsV1(), - PositionGroup(), - PositionOutput(), - RippleTimesV1(), - SortedSpikesGroup(), - SortedSpikesIndicatorSelection(), - SpikeSortingOutput(), - SpikeSortingRecordingView(), - UnitMarksIndicatorSelection(), + def load_shared_schemas(self, additional_prefixes: list = None) -> None: + """Load shared schemas to include in graph traversal. + + Parameters + ---------- + additional_prefixes : list, optional + Additional prefixes to load. Default None. + """ + all_shared = [ + *SHARED_MODULES, + dj.config["database.user"], + "file", + "sharing", + ] + + if additional_prefixes: + all_shared.extend(additional_prefixes) + + # Get a list of all shared schemas in spyglass + schemas = dj.conn().query( + "SELECT DISTINCT table_schema " # Unique schemas + + "FROM information_schema.key_column_usage " + + "WHERE" + + ' table_name not LIKE "~%%"' # Exclude hidden + + " AND constraint_name='PRIMARY'" # Only primary keys + + "AND (" # Only shared schemas + + " OR ".join([f"table_schema LIKE '{s}_%%'" for s in all_shared]) + + ") " + + "ORDER BY table_schema;" ) + # Load the dependencies for all shared schemas + for schema in schemas: + dj.schema(schema[0]).connection.dependencies.load() + @cached_property def _part_masters(self) -> set: """Set of master tables downstream of self. @@ -318,23 +307,25 @@ def _part_masters(self) -> set: part_masters = set() def search_descendants(parent): - for desc in parent.descendants(as_objects=True): + for desc_name in parent.descendants(): if ( # Check if has master, is part - not (master := get_master(desc.full_table_name)) - # has other non-master parent - or not set(desc.parents()) - set([master]) + not (master := get_master(desc_name)) or master in part_masters # already in cache + or desc_name.replace("`", "").split("_")[0] + not in SHARED_MODULES ): continue - if master not in part_masters: - part_masters.add(master) - search_descendants(dj.FreeTable(self.connection, master)) + desc = dj.FreeTable(self.connection, desc_name) + if not set(desc.parents()) - set([master]): # no other parent + continue + part_masters.add(master) + search_descendants(dj.FreeTable(self.connection, master)) try: _ = search_descendants(self) except NetworkXError: - try: # Attempt to import missing table - self._import_part_masters() + try: # Attempt to import failing schema + self.load_shared_schemas() _ = search_descendants(self) except NetworkXError as e: table_name = "".join(e.args[0].split("`")[1:4]) @@ -484,7 +475,7 @@ def _delete_deps(self) -> List[Table]: self._member_pk = LabMember.primary_key[0] return [LabMember, LabTeam, Session, schema.external, IntervalList] - def _get_exp_summary(self): + def _get_exp_summary(self) -> Union[QueryExpression, None]: """Get summary of experimenters for session(s), including NULL. Parameters @@ -494,9 +485,12 @@ def _get_exp_summary(self): Returns ------- - str - Summary of experimenters for session(s). + Union[QueryExpression, None] + dj.Union object Summary of experimenters for session(s). If no link + to Session, return None. """ + if not self._session_connection.has_link: + return None Session = self._delete_deps[2] SesExp = Session.Experimenter @@ -521,8 +515,7 @@ def _session_connection(self): """Path from Session table to self. False if no connection found.""" from spyglass.utils.dj_graph import TableChain # noqa F401 - connection = TableChain(parent=self._delete_deps[2], child=self) - return connection if connection.has_link else False + return TableChain(parent=self._delete_deps[2], child=self, verbose=True) @cached_property def _test_mode(self) -> bool: @@ -564,7 +557,13 @@ def _check_delete_permission(self) -> None: ) return - sess_summary = self._get_exp_summary() + if not (sess_summary := self._get_exp_summary()): + logger.warn( + f"Could not find a connection from {self.camel_name} " + + "to Session.\n Be careful not to delete others' data." + ) + return + experimenters = sess_summary.fetch(self._member_pk) if None in experimenters: raise PermissionError( From adfed75f60ffd0a770369d87c27043a7e8c306f8 Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Thu, 29 Aug 2024 11:24:11 -0500 Subject: [PATCH 39/94] Allow disable transaction for select populates (#1067) * Allow disable transaction for select populates * WIP: hash for data integrity * WIP: hash for data integrity 2 * WIP: hash for data integrity 3 * Add docs * Delete on hash mismatch * Incorporate feedback --- CHANGELOG.md | 17 ++- docs/src/Features/Mixin.md | 30 +++++ .../position/v1/position_dlc_training.py | 2 + .../spikesorting/v1/figurl_curation.py | 2 + .../spikesorting/v1/metric_curation.py | 2 + src/spyglass/spikesorting/v1/sorting.py | 2 + src/spyglass/utils/dj_graph.py | 9 ++ src/spyglass/utils/dj_helper_fn.py | 3 +- src/spyglass/utils/dj_mixin.py | 118 +++++++++++++++--- 9 files changed, 166 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57fd495d2..23311bf14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Change Log +## [0.5.4] (Unreleased) + +### Release Notes + + + +### Infrastructure + +- Disable populate transaction protection for long-populating tables #1066 + ## [0.5.3] (August 27, 2024) ### Infrastructure @@ -25,9 +35,9 @@ - Allow `ModuleNotFoundError` or `ImportError` for optional dependencies #1023 - Ensure integrity of group tables #1026 - Convert list of LFP artifact removed interval list to array #1046 -- Merge duplicate functions in decoding and spikesorting #1050, #1053, #1058, - #1066 -- Revise docs organization. +- Merge duplicate functions in decoding and spikesorting #1050, #1053, #1062, + #1066, #1069 +- Reivise docs organization. - Misc -> Features/ForDevelopers. #1029 - Installation instructions -> Setup notebook. #1029 - Migrate SQL export tools to `utils` to support exporting `DandiPath` #1048 @@ -320,3 +330,4 @@ [0.5.1]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.5.1 [0.5.2]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.5.2 [0.5.3]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.5.3 +[0.5.4]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.5.4 diff --git a/docs/src/Features/Mixin.md b/docs/src/Features/Mixin.md index ac227a7be..bc02087ce 100644 --- a/docs/src/Features/Mixin.md +++ b/docs/src/Features/Mixin.md @@ -188,3 +188,33 @@ nwbfile = Nwbfile() (nwbfile & "nwb_file_name LIKE 'Name%'").ddp(dry_run=False) (nwbfile & "nwb_file_name LIKE 'Other%'").ddp(dry_run=False) ``` + +## Populate Calls + +The mixin also overrides the default `populate` function to provide additional +functionality for non-daemon process pools and disabling transaction protection. + +### Non-Daemon Process Pools + +To allow the `make` function to spawn a new process pool, the mixin overrides +the default `populate` function for tables with `_parallel_make` set to `True`. +See [issue #1000](https://github.com/LorenFrankLab/spyglass/issues/1000) and +[PR #1001](https://github.com/LorenFrankLab/spyglass/pull/1001) for more +information. + +### Disable Transaction Protection + +By default, DataJoint wraps the `populate` function in a transaction to ensure +data integrity (see +[Transactions](https://docs.datajoint.io/python/definition/05-Transactions.html)). + +This can cause issues when populating large tables if another user attempts to +declare/modify a table while the transaction is open (see +[issue #1030](https://github.com/LorenFrankLab/spyglass/issues/1030) and +[DataJoint issue #1170](https://github.com/datajoint/datajoint-python/issues/1170)). + +Tables with `_use_transaction` set to `False` will not be wrapped in a +transaction when calling `populate`. Transaction protection is replaced by a +hash of upstream data to ensure no changes are made to the table during the +unprotected populate. The additional time required to hash the data is a +trade-off for already time-consuming populates, but avoids blocking other users. diff --git a/src/spyglass/position/v1/position_dlc_training.py b/src/spyglass/position/v1/position_dlc_training.py index 94548c6b1..85e86b1c0 100644 --- a/src/spyglass/position/v1/position_dlc_training.py +++ b/src/spyglass/position/v1/position_dlc_training.py @@ -102,7 +102,9 @@ class DLCModelTraining(SpyglassMixin, dj.Computed): latest_snapshot: int unsigned # latest exact snapshot index (i.e., never -1) config_template: longblob # stored full config file """ + log_path = None + _use_transaction, _allow_insert = False, True # To continue from previous training snapshot, # devs suggest editing pose_cfg.yml diff --git a/src/spyglass/spikesorting/v1/figurl_curation.py b/src/spyglass/spikesorting/v1/figurl_curation.py index fca4fb26b..03b0313c7 100644 --- a/src/spyglass/spikesorting/v1/figurl_curation.py +++ b/src/spyglass/spikesorting/v1/figurl_curation.py @@ -117,6 +117,8 @@ class FigURLCuration(SpyglassMixin, dj.Computed): url: varchar(1000) """ + _use_transaction, _allow_insert = False, True + def make(self, key: dict): # FETCH query = ( diff --git a/src/spyglass/spikesorting/v1/metric_curation.py b/src/spyglass/spikesorting/v1/metric_curation.py index b03c7fa9c..6ef520947 100644 --- a/src/spyglass/spikesorting/v1/metric_curation.py +++ b/src/spyglass/spikesorting/v1/metric_curation.py @@ -203,6 +203,8 @@ class MetricCuration(SpyglassMixin, dj.Computed): object_id: varchar(40) # Object ID for the metrics in NWB file """ + _use_transaction, _allow_insert = False, True + def make(self, key): AnalysisNwbfile()._creation_times["pre_create_time"] = time() # FETCH diff --git a/src/spyglass/spikesorting/v1/sorting.py b/src/spyglass/spikesorting/v1/sorting.py index 9196eb627..47e8b6b68 100644 --- a/src/spyglass/spikesorting/v1/sorting.py +++ b/src/spyglass/spikesorting/v1/sorting.py @@ -144,6 +144,8 @@ class SpikeSorting(SpyglassMixin, dj.Computed): time_of_sort: int # in Unix time, to the nearest second """ + _use_transaction, _allow_insert = False, True + def make(self, key: dict): """Runs spike sorting on the data and parameters specified by the SpikeSortingSelection table and inserts a new entry to SpikeSorting table. diff --git a/src/spyglass/utils/dj_graph.py b/src/spyglass/utils/dj_graph.py index 6b3928042..354b492ab 100644 --- a/src/spyglass/utils/dj_graph.py +++ b/src/spyglass/utils/dj_graph.py @@ -7,6 +7,7 @@ from copy import deepcopy from enum import Enum from functools import cached_property +from hashlib import md5 as hash_md5 from itertools import chain as iter_chain from typing import Any, Dict, Iterable, List, Set, Tuple, Union @@ -595,6 +596,14 @@ def leaf_ft(self): """Get restricted FreeTables from graph leaves.""" return [self._get_ft(table, with_restr=True) for table in self.leaves] + @property + def hash(self): + """Return hash of all visited nodes.""" + initial = hash_md5(b"") + for table in self.all_ft: + initial.update(table.fetch()) + return initial.hexdigest() + # ------------------------------- Add Nodes ------------------------------- def add_leaf( diff --git a/src/spyglass/utils/dj_helper_fn.py b/src/spyglass/utils/dj_helper_fn.py index 0bf61b734..caf6ea57c 100644 --- a/src/spyglass/utils/dj_helper_fn.py +++ b/src/spyglass/utils/dj_helper_fn.py @@ -516,7 +516,8 @@ def make_file_obj_id_unique(nwb_path: str): def populate_pass_function(value): """Pass function for parallel populate. - Note: To avoid pickling errors, the table must be passed by class, NOT by instance. + Note: To avoid pickling errors, the table must be passed by class, + NOT by instance. Note: This function must be defined in the global namespace. Parameters diff --git a/src/spyglass/utils/dj_mixin.py b/src/spyglass/utils/dj_mixin.py index 04b873740..6ce94fbf0 100644 --- a/src/spyglass/utils/dj_mixin.py +++ b/src/spyglass/utils/dj_mixin.py @@ -81,6 +81,8 @@ class SpyglassMixin: _banned_search_tables = set() # Tables to avoid in restrict_by _parallel_make = False # Tables that use parallel processing in make + _use_transaction = True # Use transaction in populate. + def __init__(self, *args, **kwargs): """Initialize SpyglassMixin. @@ -410,7 +412,7 @@ def delete_downstream_parts( **kwargs : Any Passed to datajoint.table.Table.delete. """ - from spyglass.utils.dj_graph import RestrGraph # noqa F401 + RestrGraph = self._graph_deps[1] start = time() @@ -475,7 +477,14 @@ def _delete_deps(self) -> List[Table]: self._member_pk = LabMember.primary_key[0] return [LabMember, LabTeam, Session, schema.external, IntervalList] - def _get_exp_summary(self) -> Union[QueryExpression, None]: + @cached_property + def _graph_deps(self) -> list: + from spyglass.utils.dj_graph import RestrGraph # noqa #F401 + from spyglass.utils.dj_graph import TableChain + + return [TableChain, RestrGraph] + + def _get_exp_summary(self): """Get summary of experimenters for session(s), including NULL. Parameters @@ -513,7 +522,7 @@ def _get_exp_summary(self) -> Union[QueryExpression, None]: @cached_property def _session_connection(self): """Path from Session table to self. False if no connection found.""" - from spyglass.utils.dj_graph import TableChain # noqa F401 + TableChain = self._graph_deps[0] return TableChain(parent=self._delete_deps[2], child=self, verbose=True) @@ -697,25 +706,104 @@ def super_delete(self, warn=True, *args, **kwargs): self._log_delete(start=time(), super_delete=True) super().delete(*args, **kwargs) - # -------------------------- non-daemon populate -------------------------- + # -------------------------------- populate -------------------------------- + + def _hash_upstream(self, keys): + """Hash upstream table keys for no transaction populate. + + Uses a RestrGraph to capture all upstream tables, restrict them to + relevant entries, and hash the results. This is used to check if + upstream tables have changed during a no-transaction populate and avoid + the following data-integrity error: + + 1. User A starts no-transaction populate. + 2. User B deletes and repopulates an upstream table, changing contents. + 3. User A finishes populate, inserting data that is now invalid. + + Parameters + ---------- + keys : list + List of keys for populating table. + """ + RestrGraph = self._graph_deps[1] + + if not (parents := self.parents(as_objects=True, primary=True)): + raise RuntimeError("No upstream tables found for upstream hash.") + + leaves = { # Restriction on each primary parent + p.full_table_name: [ + {k: v for k, v in key.items() if k in p.heading.names} + for key in keys + ] + for p in parents + } + + return RestrGraph(seed_table=self, leaves=leaves, cascade=True).hash + def populate(self, *restrictions, **kwargs): - """Populate table in parallel. + """Populate table in parallel, with or without transaction protection. Supersedes datajoint.table.Table.populate for classes with that - spawn processes in their make function + spawn processes in their make function and always use transactions. + + `_use_transaction` class attribute can be set to False to disable + transaction protection for a table. This is not recommended for tables + with short processing times. A before-and-after hash check is performed + to ensure upstream tables have not changed during populate, and may + be a more time-consuming process. To permit the `make` to insert without + populate, set `_allow_insert` to True. """ - - # Pass through to super if not parallel in the make function or only a single process processes = kwargs.pop("processes", 1) + + # Decide if using transaction protection + use_transact = kwargs.pop("use_transation", None) + if use_transact is None: # if user does not specify, use class default + use_transact = self._use_transaction + if self._use_transaction is False: # If class default is off, warn + logger.warning( + "Turning off transaction protection this table by default. " + + "Use use_transation=True to re-enable.\n" + + "Read more about transactions:\n" + + "https://docs.datajoint.io/python/definition/05-Transactions.html\n" + + "https://github.com/LorenFrankLab/spyglass/issues/1030" + ) + if use_transact is False and processes > 1: + raise RuntimeError( + "Must use transaction protection with parallel processing.\n" + + "Call with use_transation=True.\n" + + f"Table default transaction use: {self._use_transaction}" + ) + + # Get keys, needed for no-transact or multi-process w/_parallel_make + keys = [True] + if use_transact is False or (processes > 1 and self._parallel_make): + keys = (self._jobs_to_do(restrictions) - self.target).fetch( + "KEY", limit=kwargs.get("limit", None) + ) + + if use_transact is False: + upstream_hash = self._hash_upstream(keys) + if kwargs: # Warn of ignoring populate kwargs, bc using `make` + logger.warning( + "Ignoring kwargs when not using transaction protection." + ) + if processes == 1 or not self._parallel_make: - kwargs["processes"] = processes - return super().populate(*restrictions, **kwargs) + if use_transact: # Pass single-process populate to super + kwargs["processes"] = processes + return super().populate(*restrictions, **kwargs) + else: # No transaction protection, use bare make + for key in keys: + self.make(key) + if upstream_hash != self._hash_upstream(keys): + (self & keys).delete(force=True) + logger.error( + "Upstream tables changed during non-transaction " + + "populate. Please try again." + ) + return # If parallel in both make and populate, use non-daemon processes - # Get keys to populate - keys = (self._jobs_to_do(restrictions) - self.target).fetch( - "KEY", limit=kwargs.get("limit", None) - ) # package the call list call_list = [(type(self), key, kwargs) for key in keys] @@ -964,7 +1052,7 @@ def restrict_by( Restricted version of present table or TableChain object. If return_graph, use all_ft attribute to see all tables in cascade. """ - from spyglass.utils.dj_graph import TableChain # noqa: F401 + TableChain = self._graph_deps[0] if restriction is True: return self From d4dbc232dcb474f037493d8bfbc28afa22ba9eaf Mon Sep 17 00:00:00 2001 From: Samuel Bray Date: Thu, 29 Aug 2024 09:29:23 -0700 Subject: [PATCH 40/94] Prevent error from unitless spike group (#1083) * prevent error from unitless spike group * fix 1077 * update changelog --- CHANGELOG.md | 5 +++++ src/spyglass/decoding/v1/waveform_features.py | 11 ++++++++--- src/spyglass/spikesorting/spikesorting_merge.py | 6 +++--- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 23311bf14..e1afbe680 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,11 @@ - Disable populate transaction protection for long-populating tables #1066 +### Pipelines + +- Decoding + - Fix edge case errors in spike time loading #1083 + ## [0.5.3] (August 27, 2024) ### Infrastructure diff --git a/src/spyglass/decoding/v1/waveform_features.py b/src/spyglass/decoding/v1/waveform_features.py index 1208c53fd..56176484d 100644 --- a/src/spyglass/decoding/v1/waveform_features.py +++ b/src/spyglass/decoding/v1/waveform_features.py @@ -152,9 +152,12 @@ def make(self, key): sorter, ) - spike_times = SpikeSortingOutput().fetch_nwb(merge_key)[0][ - analysis_nwb_key - ]["spike_times"] + nwb = SpikeSortingOutput().fetch_nwb(merge_key)[0] + spike_times = ( + nwb[analysis_nwb_key]["spike_times"] + if analysis_nwb_key in nwb + else pd.DataFrame() + ) ( key["analysis_file_name"], @@ -349,6 +352,8 @@ def _write_waveform_features_to_nwb( metric_dict[unit_id] if unit_id in metric_dict else [] for unit_id in unit_ids ] + if not metric_values: + metric_values = np.array([]).astype(np.float32) nwbf.add_unit_column( name=metric, description=metric, diff --git a/src/spyglass/spikesorting/spikesorting_merge.py b/src/spyglass/spikesorting/spikesorting_merge.py index 4887cb3f3..7d12601e2 100644 --- a/src/spyglass/spikesorting/spikesorting_merge.py +++ b/src/spyglass/spikesorting/spikesorting_merge.py @@ -4,9 +4,9 @@ from ripple_detection import get_multiunit_population_firing_rate from spyglass.spikesorting.imported import ImportedSpikeSorting # noqa: F401 -from spyglass.spikesorting.v0.spikesorting_curation import ( +from spyglass.spikesorting.v0.spikesorting_curation import ( # noqa: F401 CuratedSpikeSorting, -) # noqa: F401 +) from spyglass.spikesorting.v1 import ArtifactDetectionSelection # noqa: F401 from spyglass.spikesorting.v1 import ( CurationV1, @@ -210,7 +210,7 @@ def get_spike_indicator(cls, key, time): """ time = np.asarray(time) min_time, max_time = time[[0, -1]] - spike_times = cls.fetch_spike_data(key) # CB: This is undefined. + spike_times = (cls & key).get_spike_times(key) spike_indicator = np.zeros((len(time), len(spike_times))) for ind, times in enumerate(spike_times): From f1826767cc7a17a2f9d88d897bdf977ea0ad05aa Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Thu, 5 Sep 2024 13:11:22 -0500 Subject: [PATCH 41/94] Add tests for spikesorting (#1078) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * WIP: Add tests for spikesorting * ✅ : Add tests for spikesorting 2 * Update changelog * ✅ : Add tests of utils --- CHANGELOG.md | 1 + pyproject.toml | 3 +- .../spikesorting/analysis/v1/group.py | 3 + .../analysis/v1/unit_annotation.py | 5 +- .../spikesorting/spikesorting_merge.py | 2 + src/spyglass/spikesorting/v1/artifact.py | 2 + src/spyglass/spikesorting/v1/recording.py | 9 +- src/spyglass/spikesorting/v1/utils.py | 34 +-- tests/conftest.py | 10 +- tests/spikesorting/__init__.py | 0 tests/spikesorting/conftest.py | 262 ++++++++++++++++++ tests/spikesorting/test_analysis.py | 9 + tests/spikesorting/test_artifact.py | 28 ++ tests/spikesorting/test_curation.py | 51 ++++ tests/spikesorting/test_figurl.py | 11 + tests/spikesorting/test_merge.py | 63 +++++ tests/spikesorting/test_metric_curation.py | 3 + tests/spikesorting/test_recording.py | 10 + tests/spikesorting/test_sorting.py | 3 + tests/spikesorting/test_utils.py | 20 ++ 20 files changed, 505 insertions(+), 24 deletions(-) create mode 100644 tests/spikesorting/__init__.py create mode 100644 tests/spikesorting/conftest.py create mode 100644 tests/spikesorting/test_analysis.py create mode 100644 tests/spikesorting/test_artifact.py create mode 100644 tests/spikesorting/test_curation.py create mode 100644 tests/spikesorting/test_figurl.py create mode 100644 tests/spikesorting/test_merge.py create mode 100644 tests/spikesorting/test_metric_curation.py create mode 100644 tests/spikesorting/test_recording.py create mode 100644 tests/spikesorting/test_sorting.py create mode 100644 tests/spikesorting/test_utils.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e1afbe680..9930bcbf1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -102,6 +102,7 @@ - Set `sparse` parameter to waveform extraction step in `spikesorting.v1` #1039 - Efficiency improvement to `v0.Curation.insert_curation` #1072 + - Add pytests for `spikesorting.v1` #1078 ## [0.5.2] (April 22, 2024) diff --git a/pyproject.toml b/pyproject.toml index a3f96c3e0..8db231c8d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -165,7 +165,8 @@ omit = [ # which submodules have no tests # "*/position/*", "*/ripple/*", "*/sharing/*", - "*/spikesorting/*", + "*/spikesorting/v0/*", + # "*/spikesorting/*", # "*/utils/*", "settings.py", ] diff --git a/src/spyglass/spikesorting/analysis/v1/group.py b/src/spyglass/spikesorting/analysis/v1/group.py index 8b3138e69..ad6517558 100644 --- a/src/spyglass/spikesorting/analysis/v1/group.py +++ b/src/spyglass/spikesorting/analysis/v1/group.py @@ -6,6 +6,7 @@ from ripple_detection import get_multiunit_population_firing_rate from spyglass.common import Session # noqa: F401 +from spyglass.settings import test_mode from spyglass.spikesorting.spikesorting_merge import SpikeSortingOutput from spyglass.utils.dj_mixin import SpyglassMixin, SpyglassMixinPart from spyglass.utils.spikesorting import firing_rate_from_spike_indicator @@ -72,6 +73,8 @@ def create_group( "unit_filter_params_name": unit_filter_params_name, } if self & group_key: + if test_mode: + return raise ValueError( f"Group {nwb_file_name}: {group_name} already exists", "please delete the group before creating a new one", diff --git a/src/spyglass/spikesorting/analysis/v1/unit_annotation.py b/src/spyglass/spikesorting/analysis/v1/unit_annotation.py index 4e1328979..d1ac26a11 100644 --- a/src/spyglass/spikesorting/analysis/v1/unit_annotation.py +++ b/src/spyglass/spikesorting/analysis/v1/unit_annotation.py @@ -101,9 +101,8 @@ def fetch_unit_spikes( """ if len(self) == len(UnitAnnotation()): logger.warning( - "fetching all unit spikes", - "if this is unintended, please call as: ", - "(UnitAnnotation & key).fetch_unit_spikes()", + "fetching all unit spikes if this is unintended, please call as" + + ": (UnitAnnotation & key).fetch_unit_spikes()" ) # get the set of nwb files to load merge_keys = [ diff --git a/src/spyglass/spikesorting/spikesorting_merge.py b/src/spyglass/spikesorting/spikesorting_merge.py index 7d12601e2..e7a27bae0 100644 --- a/src/spyglass/spikesorting/spikesorting_merge.py +++ b/src/spyglass/spikesorting/spikesorting_merge.py @@ -83,6 +83,8 @@ def get_restricted_merge_ids( merge_ids : list list of merge ids from the restricted sources """ + # TODO: replace with long-distance restrictions + merge_ids = [] if "v1" in sources: diff --git a/src/spyglass/spikesorting/v1/artifact.py b/src/spyglass/spikesorting/v1/artifact.py index 139f30c81..04a7dd463 100644 --- a/src/spyglass/spikesorting/v1/artifact.py +++ b/src/spyglass/spikesorting/v1/artifact.py @@ -330,6 +330,8 @@ def merge_intervals(intervals): _type_ _description_ """ + # TODO: Migrate to common_interval.py + if len(intervals) == 0: return [] diff --git a/src/spyglass/spikesorting/v1/recording.py b/src/spyglass/spikesorting/v1/recording.py index 72f099a18..fd5214e40 100644 --- a/src/spyglass/spikesorting/v1/recording.py +++ b/src/spyglass/spikesorting/v1/recording.py @@ -19,6 +19,7 @@ ) from spyglass.common.common_lab import LabTeam from spyglass.common.common_nwbfile import AnalysisNwbfile, Nwbfile +from spyglass.settings import test_mode from spyglass.spikesorting.utils import ( _get_recording_timestamps, get_group_by_shank, @@ -76,8 +77,12 @@ def set_group_by_shank( omit_unitrode : bool Optional. If True, no sort groups are defined for unitrodes. """ - # delete any current groups - (SortGroup & {"nwb_file_name": nwb_file_name}).delete() + existing_entries = SortGroup & {"nwb_file_name": nwb_file_name} + if existing_entries and test_mode: + return + elif existing_entries: + # delete any current groups + (SortGroup & {"nwb_file_name": nwb_file_name}).delete() sg_keys, sge_keys = get_group_by_shank( nwb_file_name=nwb_file_name, diff --git a/src/spyglass/spikesorting/v1/utils.py b/src/spyglass/spikesorting/v1/utils.py index 6a511c43e..66cea0b41 100644 --- a/src/spyglass/spikesorting/v1/utils.py +++ b/src/spyglass/spikesorting/v1/utils.py @@ -9,17 +9,25 @@ from spyglass.spikesorting.v1.sorting import SpikeSortingSelection -def generate_nwb_uuid(nwb_file_name: str, initial: str, len_uuid: int = 6): +def generate_nwb_uuid( + nwb_file_name: str, initial: str, len_uuid: int = 6 +) -> str: """Generates a unique identifier related to an NWB file. Parameters ---------- nwb_file_name : str - _description_ + Nwb file name, first part of resulting string. initial : str R if recording; A if artifact; S if sorting etc len_uuid : int how many digits of uuid4 to keep + + Returns + ------- + str + A unique identifier for the NWB file. + "{nwbf}_{initial}_{uuid4[:len_uuid]}" """ uuid4 = str(uuid.uuid4()) nwb_uuid = nwb_file_name + "_" + initial + "_" + uuid4[:len_uuid] @@ -44,6 +52,7 @@ def get_spiking_sorting_v1_merge_ids(restriction: dict): name of the artifact parameter curation_id : int, optional id of the curation (if not specified, uses the latest curation) + Returns ------- merge_id_list : list @@ -62,29 +71,22 @@ def get_spiking_sorting_v1_merge_ids(restriction: dict): ] # list of sorting ids for each recording sorting_restriction = restriction.copy() - del sorting_restriction["interval_list_name"] + _ = sorting_restriction.pop("interval_list_name", None) sorting_id_list = [] for r_id, a_id in zip(recording_id_list, artifact_id_list): + rec_dict = {"recording_id": str(r_id), "interval_list_name": str(a_id)} # if sorted with artifact detection - if ( - SpikeSortingSelection() - & sorting_restriction - & {"recording_id": r_id, "interval_list_name": a_id} - ): + if SpikeSortingSelection() & sorting_restriction & rec_dict: sorting_id_list.append( ( - SpikeSortingSelection() - & sorting_restriction - & {"recording_id": r_id, "interval_list_name": a_id} + SpikeSortingSelection() & sorting_restriction & rec_dict ).fetch1("sorting_id") ) # if sorted without artifact detection else: sorting_id_list.append( ( - SpikeSortingSelection() - & sorting_restriction - & {"recording_id": r_id, "interval_list_name": r_id} + SpikeSortingSelection() & sorting_restriction & rec_dict ).fetch1("sorting_id") ) # if curation_id is specified, use that id for each sorting_id @@ -100,8 +102,8 @@ def get_spiking_sorting_v1_merge_ids(restriction: dict): merge_id_list = [ ( SpikeSortingOutput.CurationV1() - & {"sorting_id": id, "curation_id": c_id} + & {"sorting_id": s_id, "curation_id": c_id} ).fetch1("merge_id") - for id, c_id in zip(sorting_id_list, curation_id) + for s_id, c_id in zip(sorting_id_list, curation_id) ] return merge_id_list diff --git a/tests/conftest.py b/tests/conftest.py index 04420ffe2..8a9bc1a79 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -815,6 +815,13 @@ def dlc_project_name(): yield "pytest_proj" +@pytest.fixture(scope="session") +def team_name(common): + team_name = "sc_eb" + common.LabTeam.insert1({"team_name": team_name}, skip_duplicates=True) + yield team_name + + @pytest.fixture(scope="session") def insert_project( verbose_context, @@ -823,6 +830,7 @@ def insert_project( dlc_project_name, dlc_project_tbl, common, + team_name, bodyparts, mini_copy_name, ): @@ -845,8 +853,6 @@ def insert_project( RippleTimesV1, ) - team_name = "sc_eb" - common.LabTeam.insert1({"team_name": team_name}, skip_duplicates=True) video_list = common.VideoFile().fetch( "nwb_file_name", "epoch", as_dict=True )[:2] diff --git a/tests/spikesorting/__init__.py b/tests/spikesorting/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/spikesorting/conftest.py b/tests/spikesorting/conftest.py new file mode 100644 index 000000000..d287d35f4 --- /dev/null +++ b/tests/spikesorting/conftest.py @@ -0,0 +1,262 @@ +import re + +import pytest +from datajoint.hash import key_hash + + +@pytest.fixture(scope="session") +def spike_v1(common): + from spyglass.spikesorting import v1 + + yield v1 + + +@pytest.fixture(scope="session") +def pop_rec(spike_v1, mini_dict, team_name): + spike_v1.SortGroup.set_group_by_shank(**mini_dict) + key = { + **mini_dict, + "sort_group_id": 0, + "preproc_param_name": "default", + "interval_list_name": "01_s1", + "team_name": team_name, + } + spike_v1.SpikeSortingRecordingSelection.insert_selection(key) + ssr_pk = ( + (spike_v1.SpikeSortingRecordingSelection & key).proj().fetch1("KEY") + ) + spike_v1.SpikeSortingRecording.populate(ssr_pk) + + yield ssr_pk + + +@pytest.fixture(scope="session") +def pop_art(spike_v1, mini_dict, pop_rec): + key = { + "recording_id": pop_rec["recording_id"], + "artifact_param_name": "default", + } + spike_v1.ArtifactDetectionSelection.insert_selection(key) + spike_v1.ArtifactDetection.populate() + + yield spike_v1.ArtifactDetection().fetch("KEY", as_dict=True)[0] + + +@pytest.fixture(scope="session") +def sorter_dict(): + return {"sorter": "mountainsort4"} + + +@pytest.fixture(scope="session") +def pop_sort(spike_v1, pop_rec, pop_art, mini_dict, sorter_dict): + key = { + **mini_dict, + **sorter_dict, + "recording_id": pop_rec["recording_id"], + "interval_list_name": str(pop_art["artifact_id"]), + "sorter_param_name": "franklab_tetrode_hippocampus_30KHz", + } + spike_v1.SpikeSortingSelection.insert_selection(key) + spike_v1.SpikeSorting.populate() + + yield spike_v1.SpikeSorting().fetch("KEY", as_dict=True)[0] + + +@pytest.fixture(scope="session") +def sorting_objs(spike_v1, pop_sort): + sort_nwb = (spike_v1.SpikeSorting & pop_sort).fetch_nwb() + sort_si = spike_v1.SpikeSorting.get_sorting(pop_sort) + yield sort_nwb, sort_si + + +@pytest.fixture(scope="session") +def pop_curation(spike_v1, pop_sort): + spike_v1.CurationV1.insert_curation( + sorting_id=pop_sort["sorting_id"], + description="testing sort", + ) + + yield spike_v1.CurationV1().fetch("KEY", as_dict=True)[0] + + +@pytest.fixture(scope="session") +def pop_metric(spike_v1, pop_sort, pop_curation): + _ = pop_curation # make sure this happens first + key = { + "sorting_id": pop_sort["sorting_id"], + "curation_id": 0, + "waveform_param_name": "default_not_whitened", + "metric_param_name": "franklab_default", + "metric_curation_param_name": "default", + } + + spike_v1.MetricCurationSelection.insert_selection(key) + spike_v1.MetricCuration.populate(key) + + yield spike_v1.MetricCuration().fetch("KEY", as_dict=True)[0] + + +@pytest.fixture(scope="session") +def metric_objs(spike_v1, pop_metric): + key = {"metric_curation_id": pop_metric["metric_curation_id"]} + labels = spike_v1.MetricCuration.get_labels(key) + merge_groups = spike_v1.MetricCuration.get_merge_groups(key) + metrics = spike_v1.MetricCuration.get_metrics(key) + yield labels, merge_groups, metrics + + +@pytest.fixture(scope="session") +def pop_curation_metric(spike_v1, pop_metric, metric_objs): + labels, merge_groups, metrics = metric_objs + parent_dict = {"parent_curation_id": 0} + spike_v1.CurationV1.insert_curation( + sorting_id=( + spike_v1.MetricCurationSelection + & {"metric_curation_id": pop_metric["metric_curation_id"]} + ).fetch1("sorting_id"), + **parent_dict, + labels=labels, + merge_groups=merge_groups, + metrics=metrics, + description="after metric curation", + ) + + yield (spike_v1.CurationV1 & parent_dict).fetch("KEY", as_dict=True)[0] + + +@pytest.fixture(scope="session") +def pop_figurl(spike_v1, pop_sort, metric_objs): + # WON'T WORK UNTIL CI/CD KACHERY_CLOUD INIT + sort_dict = {"sorting_id": pop_sort["sorting_id"], "curation_id": 1} + curation_uri = spike_v1.FigURLCurationSelection.generate_curation_uri( + sort_dict + ) + _, _, metrics = metric_objs + key = { + **sort_dict, + "curation_uri": curation_uri, + "metrics_figurl": list(metrics.keys()), + } + spike_v1.FigURLCurationSelection.insert_selection(key) + spike_v1.FigURLCuration.populate() + + yield spike_v1.FigURLCuration().fetch("KEY", as_dict=True)[0] + + +@pytest.fixture(scope="session") +def pop_figurl_json(spike_v1, pop_metric): + # WON'T WORK UNTIL CI/CD KACHERY_CLOUD INIT + gh_curation_uri = ( + "gh://LorenFrankLab/sorting-curations/main/khl02007/test/curation.json" + ) + key = { + "sorting_id": pop_metric["sorting_id"], + "curation_id": 1, + "curation_uri": gh_curation_uri, + "metrics_figurl": [], + } + spike_v1.FigURLCurationSelection.insert_selection(key) + spike_v1.FigURLCuration.populate() + + labels = spike_v1.FigURLCuration.get_labels(gh_curation_uri) + merge_groups = spike_v1.FigURLCuration.get_merge_groups(gh_curation_uri) + _, _, metrics = metric_objs + spike_v1.CurationV1.insert_curation( + sorting_id=pop_sort["sorting_id"], + parent_curation_id=1, + labels=labels, + merge_groups=merge_groups, + metrics=metrics, + description="after figurl curation", + ) + yield spike_v1.CurationV1().fetch("KEY", as_dict=True) # list of dicts + + +@pytest.fixture(scope="session") +def spike_merge(spike_v1): + from spyglass.spikesorting.spikesorting_merge import SpikeSortingOutput + + yield SpikeSortingOutput() + + +@pytest.fixture(scope="session") +def pop_merge( + spike_v1, pop_curation_metric, spike_merge, mini_dict, sorter_dict +): + # TODO: add figurl fixtures when kachery_cloud is initialized + + spike_merge.insert([pop_curation_metric], part_name="CurationV1") + yield spike_merge.fetch("KEY", as_dict=True)[0] + + +def is_uuid(text): + uuid_pattern = re.compile( + r"\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b" + ) + return uuid_pattern.fullmatch(str(text)) is not None + + +def hash_sort_info(sort_info): + """Hashes attributes of a dj.Table object that are not randomly assigned.""" + no_str_uuid = { + k: v + for k, v in sort_info.fetch(as_dict=True)[0].items() + if not is_uuid(v) and k != "analysis_file_name" + } + return key_hash(no_str_uuid) + + +@pytest.fixture(scope="session") +def spike_v1_group(): + from spyglass.spikesorting.analysis.v1 import group + + yield group + + +@pytest.fixture(scope="session") +def pop_group(spike_v1_group, spike_merge, mini_dict, pop_merge): + + _ = pop_merge # make sure this happens first + + spike_v1_group.UnitSelectionParams().insert_default() + spike_v1_group.SortedSpikesGroup().create_group( + **mini_dict, + group_name="demo_group", + keys=spike_merge.proj(spikesorting_merge_id="merge_id").fetch("KEY"), + unit_filter_params_name="default_exclusion", + ) + yield spike_v1_group.SortedSpikesGroup().fetch("KEY", as_dict=True)[0] + + +@pytest.fixture(scope="session") +def spike_v1_ua(): + from spyglass.spikesorting.analysis.v1.unit_annotation import UnitAnnotation + + yield UnitAnnotation() + + +@pytest.fixture(scope="session") +def pop_annotations(spike_v1_group, spike_v1_ua, pop_group): + spike_times, unit_ids = spike_v1_group.SortedSpikesGroup().fetch_spike_data( + pop_group, return_unit_ids=True + ) + for spikes, unit_key in zip(spike_times, unit_ids): + quant_key = { + **unit_key, + "annotation": "spike_count", + "quantification": len(spikes), + } + label_key = { + **unit_key, + "annotation": "cell_type", + "label": "pyridimal" if len(spikes) < 1000 else "interneuron", + } + + spike_v1_ua.add_annotation(quant_key, skip_duplicates=True) + spike_v1_ua.add_annotation(label_key, skip_duplicates=True) + + yield ( + spike_v1_ua.Annotation + # * (spike_v1_group.SortedSpikesGroup.Units & pop_group) + & {"annotation": "spike_count"} + ) diff --git a/tests/spikesorting/test_analysis.py b/tests/spikesorting/test_analysis.py new file mode 100644 index 000000000..aa95e24b2 --- /dev/null +++ b/tests/spikesorting/test_analysis.py @@ -0,0 +1,9 @@ +def test_analysis_units(pop_annotations): + selected_spike_times, selected_unit_ids = pop_annotations.fetch_unit_spikes( + return_unit_ids=True + ) + + assert selected_spike_times[0].shape[0] == 243, "Unuxpected spike count" + + units = [d["unit_id"] for d in selected_unit_ids] + assert units == [0, 1, 2], "Unexpected unit ids" diff --git a/tests/spikesorting/test_artifact.py b/tests/spikesorting/test_artifact.py new file mode 100644 index 000000000..5466f9571 --- /dev/null +++ b/tests/spikesorting/test_artifact.py @@ -0,0 +1,28 @@ +import numpy as np +import pytest + + +@pytest.fixture +def art_interval(common, spike_v1, pop_art): + id = str((spike_v1.ArtifactDetection & pop_art).fetch1("artifact_id")) + yield (common.IntervalList & {"interval_list_name": id}).fetch1() + + +def test_artifact_detection(art_interval): + assert ( + art_interval["pipeline"] == "spikesorting_artifact_v1" + ), "Artifact detection failed to populate interval list" + + +def test_null_artifact_detection(spike_v1, art_interval): + from spyglass.spikesorting.v1.artifact import _get_artifact_times + + rec_key = spike_v1.SpikeSortingRecording.fetch("KEY")[0] + rec = spike_v1.SpikeSortingRecording.get_recording(rec_key) + + input_times = art_interval["valid_times"] + null_times = _get_artifact_times(rec, input_times) + + assert np.array_equal( + input_times[0], null_times[0] + ), "Null artifact detection failed" diff --git a/tests/spikesorting/test_curation.py b/tests/spikesorting/test_curation.py new file mode 100644 index 000000000..43df0fed5 --- /dev/null +++ b/tests/spikesorting/test_curation.py @@ -0,0 +1,51 @@ +import numpy as np +from datajoint.hash import key_hash +from spikeinterface import BaseSorting +from spikeinterface.extractors.nwbextractors import NwbRecordingExtractor + +from .conftest import hash_sort_info + + +def test_curation_rec(spike_v1, pop_curation): + rec = spike_v1.CurationV1.get_recording(pop_curation) + assert isinstance( + rec, NwbRecordingExtractor + ), "CurationV1.get_recording failed to return a RecordingExtractor" + + sample_freq = rec.get_sampling_frequency() + assert np.isclose( + 29_959.3, sample_freq + ), "CurqtionV1.get_sampling_frequency unexpected value" + + times = rec.get_times() + assert np.isclose( + 1687474805.4, np.mean((times[0], times[-1])) + ), "CurationV1.get_times unexpected value" + + +def test_curation_sort(spike_v1, pop_curation): + sort = spike_v1.CurationV1.get_sorting(pop_curation) + sort_dict = sort.to_dict() + assert isinstance( + sort, BaseSorting + ), "CurationV1.get_sorting failed to return a BaseSorting" + assert ( + key_hash(sort_dict) == "612983fbf4958f6b2c7abe7ced86ab73" + ), "CurationV1.get_sorting unexpected value" + assert ( + sort_dict["kwargs"]["spikes"].shape[0] == 918 + ), "CurationV1.get_sorting unexpected shape" + + +def test_curation_sort_info(spike_v1, pop_curation): + sort_info = spike_v1.CurationV1.get_sort_group_info(pop_curation) + assert ( + hash_sort_info(sort_info) == "be874e806a482ed2677fd0d0b449f965" + ), "CurationV1.get_sort_group_info unexpected value" + + +def test_curation_metric(spike_v1, pop_curation_metric): + sort_info = spike_v1.CurationV1.get_sort_group_info(pop_curation_metric) + assert ( + hash_sort_info(sort_info) == "48e437bc116900fe64e492d74595b56d" + ), "CurationV1.get_sort_group_info unexpected value" diff --git a/tests/spikesorting/test_figurl.py b/tests/spikesorting/test_figurl.py new file mode 100644 index 000000000..cf8a98e8b --- /dev/null +++ b/tests/spikesorting/test_figurl.py @@ -0,0 +1,11 @@ +import pytest + + +@pytest.mark.skip(reason="Not testing kachery") +def test_figurl(spike_v1): + pass + + +@pytest.mark.skip(reason="Not testing kachery") +def test_figurl_json(spike_v1): + pass diff --git a/tests/spikesorting/test_merge.py b/tests/spikesorting/test_merge.py new file mode 100644 index 000000000..25751684c --- /dev/null +++ b/tests/spikesorting/test_merge.py @@ -0,0 +1,63 @@ +import pytest +from spikeinterface import BaseSorting +from spikeinterface.extractors.nwbextractors import NwbRecordingExtractor + +from .conftest import hash_sort_info + + +def test_merge_get_restr(spike_merge, pop_merge, pop_curation_metric): + restr_id = spike_merge.get_restricted_merge_ids( + pop_curation_metric, sources=["v1"] + )[0] + assert ( + restr_id == pop_merge["merge_id"] + ), "SpikeSortingOutput merge_id mismatch" + + non_artifact = spike_merge.get_restricted_merge_ids( + pop_curation_metric, sources=["v1"], restrict_by_artifact=False + )[0] + assert restr_id == non_artifact, "SpikeSortingOutput merge_id mismatch" + + +def test_merge_get_recording(spike_merge, pop_merge): + rec = spike_merge.get_recording(pop_merge) + assert isinstance( + rec, NwbRecordingExtractor + ), "SpikeSortingOutput.get_recording failed to return a RecordingExtractor" + + +def test_merge_get_sorting(spike_merge, pop_merge): + sort = spike_merge.get_sorting(pop_merge) + assert isinstance( + sort, BaseSorting + ), "SpikeSortingOutput.get_sorting failed to return a BaseSorting" + + +def test_merge_get_sort_group_info(spike_merge, pop_merge): + hash = hash_sort_info(spike_merge.get_sort_group_info(pop_merge)) + assert ( + hash == "48e437bc116900fe64e492d74595b56d" + ), "SpikeSortingOutput.get_sort_group_info unexpected value" + + +@pytest.fixture(scope="session") +def merge_times(spike_merge, pop_merge): + yield spike_merge.get_spike_times(pop_merge) + + +def test_merge_get_spike_times(merge_times): + assert ( + merge_times[0].shape[0] == 243 + ), "SpikeSortingOutput.get_spike_times unexpected shape" + + +@pytest.mark.skip(reason="Not testing bc #1077") +def test_merge_get_spike_indicators(spike_merge, pop_merge, merge_times): + ret = spike_merge.get_spike_indicator(pop_merge, time=merge_times) + raise NotImplementedError(ret) + + +@pytest.mark.skip(reason="Not testing bc #1077") +def test_merge_get_firing_rate(spike_merge, pop_merge, merge_times): + ret = spike_merge.get_firing_rate(pop_merge, time=merge_times) + raise NotImplementedError(ret) diff --git a/tests/spikesorting/test_metric_curation.py b/tests/spikesorting/test_metric_curation.py new file mode 100644 index 000000000..0f7dc7a9a --- /dev/null +++ b/tests/spikesorting/test_metric_curation.py @@ -0,0 +1,3 @@ +def test_metric_curation(spike_v1, pop_curation_metric): + ret = spike_v1.CurationV1 & pop_curation_metric & "description LIKE 'a%'" + assert len(ret) == 1, "CurationV1.insert_curation failed to insert a record" diff --git a/tests/spikesorting/test_recording.py b/tests/spikesorting/test_recording.py new file mode 100644 index 000000000..780cbc46c --- /dev/null +++ b/tests/spikesorting/test_recording.py @@ -0,0 +1,10 @@ +def test_sort_group(spike_v1, pop_rec): + max_id = max(spike_v1.SortGroup.fetch("sort_group_id")) + assert ( + max_id == 31 + ), "SortGroup.insert_sort_group failed to insert all records" + + +def test_spike_sorting(spike_v1, pop_rec): + n_records = len(spike_v1.SpikeSortingRecording()) + assert n_records == 1, "SpikeSortingRecording failed to insert a record" diff --git a/tests/spikesorting/test_sorting.py b/tests/spikesorting/test_sorting.py new file mode 100644 index 000000000..e908fed07 --- /dev/null +++ b/tests/spikesorting/test_sorting.py @@ -0,0 +1,3 @@ +def test_sorting(spike_v1, pop_sort): + n_sorts = len(spike_v1.SpikeSorting & pop_sort) + assert n_sorts >= 1, "SpikeSorting population failed" diff --git a/tests/spikesorting/test_utils.py b/tests/spikesorting/test_utils.py new file mode 100644 index 000000000..47638f993 --- /dev/null +++ b/tests/spikesorting/test_utils.py @@ -0,0 +1,20 @@ +from uuid import UUID + + +def test_uuid_generator(): + + from spyglass.spikesorting.v1.utils import generate_nwb_uuid + + nwb_file_name, initial = "test.nwb", "R" + ret_parts = generate_nwb_uuid(nwb_file_name, initial).split("_") + assert ret_parts[0] == nwb_file_name, "Unexpected nwb file name" + assert ret_parts[1] == initial, "Unexpected initial" + assert len(ret_parts[2]) == 6, "Unexpected uuid length" + + +def test_get_merge_ids(pop_merge, mini_dict): + from spyglass.spikesorting.v1.utils import get_spiking_sorting_v1_merge_ids + + ret = get_spiking_sorting_v1_merge_ids(dict(mini_dict, curation_id=1)) + assert isinstance(ret[0], UUID), "Unexpected type from util" + assert ret[0] == pop_merge["merge_id"], "Unexpected merge_id from util" From 736d47b3c0611a67f8444ca45b4d7778c700ee82 Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Thu, 5 Sep 2024 13:11:42 -0500 Subject: [PATCH 42/94] Add docstrings (#1076) * Add docstrings 1 * Remove unrelated file * Add docstrings 2 --- CHANGELOG.md | 1 + src/spyglass/common/common_behav.py | 7 +++ src/spyglass/common/common_dandi.py | 1 + src/spyglass/common/common_ephys.py | 57 +++++++++++++++---- src/spyglass/common/common_filter.py | 3 + src/spyglass/common/common_interval.py | 4 ++ src/spyglass/common/common_nwbfile.py | 1 + src/spyglass/common/common_position.py | 17 +++++- src/spyglass/common/common_sensors.py | 2 + src/spyglass/common/common_session.py | 19 +++++++ src/spyglass/common/common_task.py | 2 + src/spyglass/common/common_usage.py | 5 ++ src/spyglass/decoding/decoding_merge.py | 7 +++ src/spyglass/decoding/v0/clusterless.py | 15 +++++ src/spyglass/decoding/v0/sorted_spikes.py | 18 +++++- src/spyglass/decoding/v0/visualization.py | 5 ++ .../decoding/v0/visualization_1D_view.py | 3 + .../decoding/v0/visualization_2D_view.py | 11 ++++ src/spyglass/decoding/v1/clusterless.py | 17 ++++++ src/spyglass/decoding/v1/core.py | 5 ++ src/spyglass/decoding/v1/sorted_spikes.py | 17 ++++++ src/spyglass/decoding/v1/waveform_features.py | 2 + src/spyglass/lfp/analysis/v1/lfp_band.py | 1 + src/spyglass/lfp/lfp_imported.py | 3 +- src/spyglass/lfp/lfp_merge.py | 1 + src/spyglass/lfp/v1/lfp.py | 10 +++- src/spyglass/lfp/v1/lfp_artifact.py | 9 +++ src/spyglass/linearization/merge.py | 5 +- src/spyglass/linearization/v0/main.py | 6 +- src/spyglass/linearization/v1/main.py | 15 ++++- src/spyglass/mua/v1/mua.py | 17 +++++- src/spyglass/position/position_merge.py | 4 +- src/spyglass/position/v1/__init__.py | 3 +- src/spyglass/position/v1/dlc_reader.py | 17 ++++-- src/spyglass/position/v1/dlc_utils.py | 15 ++++- src/spyglass/position/v1/dlc_utils_makevid.py | 5 ++ .../position/v1/position_dlc_centroid.py | 20 ++++++- .../position/v1/position_dlc_cohort.py | 17 ++++-- .../position/v1/position_dlc_model.py | 5 ++ .../position/v1/position_dlc_orient.py | 15 ++++- .../v1/position_dlc_pose_estimation.py | 7 ++- .../position/v1/position_dlc_position.py | 31 ++++++++-- .../position/v1/position_dlc_project.py | 2 + .../position/v1/position_dlc_selection.py | 23 +++++++- .../position/v1/position_dlc_training.py | 3 + .../position/v1/position_trodes_position.py | 40 ++++++++++--- src/spyglass/ripple/v1/ripple.py | 38 +++++++++++-- src/spyglass/settings.py | 13 +++++ src/spyglass/sharing/sharing_kachery.py | 4 +- .../spikesorting/analysis/v1/group.py | 2 + src/spyglass/spikesorting/imported.py | 6 +- .../spikesorting/spikesorting_merge.py | 1 + .../figurl_views/SpikeSortingRecordingView.py | 8 ++- .../v0/figurl_views/SpikeSortingView.py | 15 +++++ .../prepare_spikesortingview_data.py | 13 ++++- .../v0/merged_sorting_extractor.py | 5 +- src/spyglass/spikesorting/v0/sortingview.py | 1 + .../spikesorting/v0/spikesorting_artifact.py | 26 +++------ .../spikesorting/v0/spikesorting_curation.py | 50 ++++++++++++++-- .../spikesorting/v0/spikesorting_recording.py | 14 ++++- .../spikesorting/v0/spikesorting_sorting.py | 2 +- src/spyglass/spikesorting/v1/artifact.py | 23 ++++++-- .../spikesorting/v1/figurl_curation.py | 8 ++- .../spikesorting/v1/metric_curation.py | 23 ++++++++ src/spyglass/spikesorting/v1/recording.py | 13 +++++ src/spyglass/spikesorting/v1/sorting.py | 3 +- src/spyglass/utils/database_settings.py | 1 + src/spyglass/utils/dj_graph.py | 4 ++ src/spyglass/utils/dj_helper_fn.py | 8 ++- src/spyglass/utils/dj_merge_tables.py | 1 + src/spyglass/utils/logging.py | 1 + src/spyglass/utils/nwb_helper_fn.py | 2 + src/spyglass/utils/position.py | 1 + src/spyglass/utils/spikesorting.py | 1 + 74 files changed, 683 insertions(+), 97 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9930bcbf1..7bc6c50c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ ### Infrastructure - Disable populate transaction protection for long-populating tables #1066 +- Add docstrings to all public methods #1076 ### Pipelines diff --git a/src/spyglass/common/common_behav.py b/src/spyglass/common/common_behav.py index efff2a408..22afaf8c7 100644 --- a/src/spyglass/common/common_behav.py +++ b/src/spyglass/common/common_behav.py @@ -44,6 +44,7 @@ class SpatialSeries(SpyglassMixin, dj.Part): """ def populate(self, *args, **kwargs): + """Method for populate_all_common.""" logger.warning( "PositionSource is a manual table with a custom `make`." + " Use `make` instead." @@ -192,6 +193,7 @@ class PosObject(SpyglassMixin, dj.Part): _nwb_table = Nwbfile def fetch1_dataframe(self): + """Return a dataframe with all RawPosition.PosObject items.""" id_rp = [(n["id"], n["raw_position"]) for n in self.fetch_nwb()] if len(set(rp.interval for _, rp in id_rp)) > 1: @@ -375,6 +377,7 @@ class VideoFile(SpyglassMixin, dj.Imported): _nwb_table = Nwbfile def make(self, key): + """Make without transaction""" self._no_transaction_make(key) def _no_transaction_make(self, key, verbose=True, skip_duplicates=False): @@ -454,6 +457,7 @@ def _no_transaction_make(self, key, verbose=True, skip_duplicates=False): @classmethod def update_entries(cls, restrict=True): + """Update the camera_name field for all entries in the table.""" existing_entries = (cls & restrict).fetch("KEY") for row in existing_entries: if (cls & row).fetch1("camera_name"): @@ -511,6 +515,7 @@ class PositionIntervalMap(SpyglassMixin, dj.Computed): # #849 - Insert null to avoid rerun def make(self, key): + """Make without transaction""" self._no_transaction_make(key) def _no_transaction_make(self, key): @@ -593,6 +598,7 @@ def _no_transaction_make(self, key): def get_pos_interval_list_names(nwb_file_name) -> list: + """Return a list of position interval list names for a given NWB file.""" return [ interval_list_name for interval_list_name in ( @@ -679,6 +685,7 @@ def get_interval_list_name_from_epoch(nwb_file_name: str, epoch: int) -> str: def populate_position_interval_map_session(nwb_file_name: str): + """Populate PositionIntervalMap for all epochs in a given NWB file.""" # 1. remove redundancy in interval names # 2. let PositionIntervalMap handle transaction context nwb_dict = dict(nwb_file_name=nwb_file_name) diff --git a/src/spyglass/common/common_dandi.py b/src/spyglass/common/common_dandi.py index 5cd4ee872..6dfbb56e3 100644 --- a/src/spyglass/common/common_dandi.py +++ b/src/spyglass/common/common_dandi.py @@ -51,6 +51,7 @@ class DandiPath(SpyglassMixin, dj.Manual): """ def fetch_file_from_dandi(self, key: dict): + """Fetch the file from Dandi and return the NWB file object.""" dandiset_id, dandi_path, dandi_instance = (self & key).fetch1( "dandiset_id", "dandi_path", "dandi_instance" ) diff --git a/src/spyglass/common/common_ephys.py b/src/spyglass/common/common_ephys.py index c3bb210f6..37c4361c5 100644 --- a/src/spyglass/common/common_ephys.py +++ b/src/spyglass/common/common_ephys.py @@ -98,6 +98,13 @@ class Electrode(SpyglassMixin, dj.Imported): """ def make(self, key): + """Populate the Electrode table with data from the NWB file. + + - Uses the electrode table from the NWB file. + - Adds the region_id from the BrainRegion table. + - Uses novela Probe.Electrode if available. + - Overrides with information from the config YAML based on primary key + """ nwb_file_name = key["nwb_file_name"] nwb_file_abspath = Nwbfile.get_abs_path(nwb_file_name) nwbf = get_nwb_file(nwb_file_abspath) @@ -145,13 +152,15 @@ def make(self, key): # TODO this could be better resolved by making an extension for the # electrodes table - if ( - isinstance(elect_data.group.device, ndx_franklab_novela.Probe) - and "probe_shank" in elect_data - and "probe_electrode" in elect_data - and "bad_channel" in elect_data - and "ref_elect_id" in elect_data - ): + extra_cols = [ + "probe_shank", + "probe_electrode", + "bad_channel", + "ref_elect_id", + ] + if isinstance( + elect_data.group.device, ndx_franklab_novela.Probe + ) and all(col in elect_data for col in extra_cols): key.update( { "probe_id": elect_data.group.device.probe_type, @@ -163,6 +172,11 @@ def make(self, key): "original_reference_electrode": elect_data.ref_elect_id, } ) + else: + logger.warning( + "Electrode did not match extected novela format.\nPlease " + + f"ensure the following in YAML config: {extra_cols}." + ) # override with information from the config YAML based on primary # key (electrode id) @@ -339,6 +353,7 @@ def make(self, key): ) def nwb_object(self, key): + """Return the NWB object in the raw NWB file.""" # TODO return the nwb_object; FIX: this should be replaced with a fetch # call. Note that we're using the raw file so we can modify the other # one. @@ -438,6 +453,14 @@ class LFP(SpyglassMixin, dj.Imported): """ def make(self, key): + """Populate the LFP table with data from the NWB file. + + 1. Fetches the raw data and sampling rate from the Raw table. + 2. Ignores intervals < 1 second long. + 3. Decimates the data to 1 KHz + 4. Applies LFP 0-400 Hz filter from FirFilterParameters table. + 5. Generates a new analysis NWB file with the LFP data. + """ # get the NWB object with the data; FIX: change to fetch with # additional infrastructure lfp_file_name = AnalysisNwbfile().create(key["nwb_file_name"]) # logged @@ -534,7 +557,7 @@ def make(self, key): self.insert1(key) def nwb_object(self, key): - # return the NWB object in the raw NWB file + """Return the NWB object in the raw NWB file.""" lfp_file_name = ( LFP() & {"nwb_file_name": key["nwb_file_name"]} ).fetch1("analysis_file_name") @@ -546,7 +569,8 @@ def nwb_object(self, key): ) return lfp_nwbf.objects[nwb_object_id] - def fetch1_dataframe(self, *attrs, **kwargs): + def fetch1_dataframe(self, *attrs, **kwargs) -> pd.DataFrame: + """Fetch the LFP data as a pandas DataFrame.""" nwb_lfp = self.fetch_nwb()[0] return pd.DataFrame( nwb_lfp["lfp"].data, @@ -709,10 +733,20 @@ class LFPBand(SpyglassMixin, dj.Computed): --- -> AnalysisNwbfile -> IntervalList - filtered_data_object_id: varchar(40) # the NWB object ID for loading this object from the file + filtered_data_object_id: varchar(40) # the NWB object ID for this object """ def make(self, key): + """Populate the LFPBand table. + + 1. Fetches the LFP data and sampling rate from the LFP table. + 2. Fetches electrode and reference electrode ids from LFPBandSelection. + 3. Fetches interval list and filter from LFPBandSelection. + 4. Applies filter using FirFilterParameters `filter_data` method. + 5. Generates a new analysis NWB file with the filtered data as an + ElectricalSeries. + 6. Adds resulting interval list to IntervalList table. + """ # create the analysis nwb file to store the results. lfp_band_file_name = AnalysisNwbfile().create( # logged key["nwb_file_name"] @@ -915,7 +949,8 @@ def make(self, key): AnalysisNwbfile().log(lfp_band_file_name, table=self.full_table_name) self.insert1(key) - def fetch1_dataframe(self, *attrs, **kwargs): + def fetch1_dataframe(self, *attrs, **kwargs) -> pd.DataFrame: + """Fetch the LFP band data as a pandas DataFrame.""" filtered_nwb = self.fetch_nwb()[0] return pd.DataFrame( filtered_nwb["filtered_data"].data, diff --git a/src/spyglass/common/common_filter.py b/src/spyglass/common/common_filter.py index 8deb667c0..faab24f4f 100644 --- a/src/spyglass/common/common_filter.py +++ b/src/spyglass/common/common_filter.py @@ -170,6 +170,7 @@ def _filter_restrict(self, filter_name, fs): ).fetch1() def plot_magnitude(self, filter_name, fs, return_fig=False): + """Plot the magnitude of the frequency response of the filter.""" filter_dict = self._filter_restrict(filter_name, fs) plt.figure() w, h = signal.freqz(filter_dict["filter_coeff"], worN=65536) @@ -185,6 +186,7 @@ def plot_magnitude(self, filter_name, fs, return_fig=False): return plt.gcf() def plot_fir_filter(self, filter_name, fs, return_fig=False): + """Plot the filter.""" filter_dict = self._filter_restrict(filter_name, fs) plt.figure() plt.clf() @@ -197,6 +199,7 @@ def plot_fir_filter(self, filter_name, fs, return_fig=False): return plt.gcf() def filter_delay(self, filter_name, fs): + """Return the filter delay for the specified filter.""" return self.calc_filter_delay( self._filter_restrict(filter_name, fs)["filter_coeff"] ) diff --git a/src/spyglass/common/common_interval.py b/src/spyglass/common/common_interval.py index 1b31b612b..25670f03c 100644 --- a/src/spyglass/common/common_interval.py +++ b/src/spyglass/common/common_interval.py @@ -73,6 +73,7 @@ def insert_from_nwbfile(cls, nwbf: NWBFile, *, nwb_file_name: str): cls.insert(epoch_inserts, skip_duplicates=True) def plot_intervals(self, figsize=(20, 5), return_fig=False): + """Plot the intervals in the interval list.""" interval_list = pd.DataFrame(self) fig, ax = plt.subplots(figsize=figsize) interval_count = 0 @@ -94,6 +95,7 @@ def plot_intervals(self, figsize=(20, 5), return_fig=False): return fig def plot_epoch_pos_raw_intervals(self, figsize=(20, 5), return_fig=False): + """Plot an epoch's position, raw data, and valid times intervals.""" interval_list = pd.DataFrame(self) fig, ax = plt.subplots(figsize=(30, 3)) @@ -157,6 +159,7 @@ def plot_epoch_pos_raw_intervals(self, figsize=(20, 5), return_fig=False): return fig def nightly_cleanup(self, dry_run=True): + """Clean up orphaned IntervalList entries.""" orphans = self - get_child_tables(self) if dry_run: return orphans @@ -251,6 +254,7 @@ def interval_list_excludes(interval_list, timestamps): def consolidate_intervals(interval_list): + """Consolidate overlapping intervals in an interval list.""" if interval_list.ndim == 1: interval_list = np.expand_dims(interval_list, 0) else: diff --git a/src/spyglass/common/common_nwbfile.py b/src/spyglass/common/common_nwbfile.py index 9d5fce015..2166b2c04 100644 --- a/src/spyglass/common/common_nwbfile.py +++ b/src/spyglass/common/common_nwbfile.py @@ -692,6 +692,7 @@ def cleanup(delete_files=False): @staticmethod def nightly_cleanup(): + """Clean up orphaned AnalysisNwbfile entries and external files.""" child_tables = get_child_tables(AnalysisNwbfile) (AnalysisNwbfile - child_tables).delete_quick() diff --git a/src/spyglass/common/common_position.py b/src/spyglass/common/common_position.py index 3331db555..9026eae7f 100644 --- a/src/spyglass/common/common_position.py +++ b/src/spyglass/common/common_position.py @@ -70,8 +70,10 @@ class IntervalPositionInfoSelection(SpyglassMixin, dj.Lookup): @schema class IntervalPositionInfo(SpyglassMixin, dj.Computed): - """Computes the smoothed head position, orientation and velocity for a given - interval.""" + """Computes the smoothed data for a given interval. + + Data includes head position, orientation and velocity + """ definition = """ -> IntervalPositionInfoSelection @@ -83,6 +85,7 @@ class IntervalPositionInfo(SpyglassMixin, dj.Computed): """ def make(self, key): + """Insert smoothed head position, orientation and velocity.""" logger.info(f"Computing position for: {key}") analysis_file_name = AnalysisNwbfile().create( # logged @@ -318,6 +321,7 @@ def calculate_position_info( max_plausible_speed=None, **kwargs, ): + """Calculates the smoothed position, orientation and velocity.""" CM_TO_METERS = 100 ( @@ -453,7 +457,8 @@ def calculate_position_info( "speed": speed, } - def fetch1_dataframe(self): + def fetch1_dataframe(self) -> pd.DataFrame: + """Fetches the position data as a pandas dataframe.""" return self._data_to_df(self.fetch_nwb()[0]) @staticmethod @@ -516,6 +521,11 @@ class PositionVideo(SpyglassMixin, dj.Computed): """ def make(self, key): + """Populates the PositionVideo table. + + The video is created by overlaying the head position and orientation + on the video of the animal. + """ M_TO_CM = 100 logger.info("Loading position data...") @@ -609,6 +619,7 @@ def make_video( circle_radius: int = 8, truncate_data: bool = False, # reduce data to min len across all vars ): + """Generates a video with the head position and orientation overlaid.""" import cv2 # noqa: F401 RGB_PINK = (234, 82, 111) diff --git a/src/spyglass/common/common_sensors.py b/src/spyglass/common/common_sensors.py index 829d72da4..ee6dacff0 100644 --- a/src/spyglass/common/common_sensors.py +++ b/src/spyglass/common/common_sensors.py @@ -25,6 +25,8 @@ class SensorData(SpyglassMixin, dj.Imported): _nwb_table = Nwbfile def make(self, key): + """Populate SensorData using the analog BehavioralEvents from the NWB.""" + nwb_file_name = key["nwb_file_name"] nwb_file_abspath = Nwbfile().get_abs_path(nwb_file_name) nwbf = get_nwb_file(nwb_file_abspath) diff --git a/src/spyglass/common/common_session.py b/src/spyglass/common/common_session.py index 7ea12e0d3..a069e9f38 100644 --- a/src/spyglass/common/common_session.py +++ b/src/spyglass/common/common_session.py @@ -52,6 +52,18 @@ class Experimenter(SpyglassMixin, dj.Part): """ def make(self, key): + """Populate the Session table and others from an nwb file. + + Calls the insert_from_nwbfile method for each of the following tables: + - Institution + - Lab + - LabMember + - Subject + - DataAcquisitionDevice + - CameraDevice + - Probe + - IntervalList + """ # These imports must go here to avoid cyclic dependencies # from .common_task import Task, TaskEpoch from .common_interval import IntervalList @@ -195,6 +207,7 @@ def add_group( *, skip_duplicates: bool = False, ): + """Add a new session group.""" SessionGroup.insert1( { "session_group_name": session_group_name, @@ -207,6 +220,7 @@ def add_group( def update_session_group_description( session_group_name: str, session_group_description ): + """Update the description of a session group.""" SessionGroup.update1( { "session_group_name": session_group_name, @@ -221,6 +235,7 @@ def add_session_to_group( *, skip_duplicates: bool = False, ): + """Add a session to an existing session group.""" if test_mode: skip_duplicates = True SessionGroupSession.insert1( @@ -235,6 +250,7 @@ def add_session_to_group( def remove_session_from_group( nwb_file_name: str, session_group_name: str, *args, **kwargs ): + """Remove a session from a session group.""" query = { "session_group_name": session_group_name, "nwb_file_name": nwb_file_name, @@ -245,6 +261,7 @@ def remove_session_from_group( @staticmethod def delete_group(session_group_name: str, *args, **kwargs): + """Delete a session group.""" query = {"session_group_name": session_group_name} (SessionGroup & query).delete( force_permission=test_mode, *args, **kwargs @@ -252,6 +269,7 @@ def delete_group(session_group_name: str, *args, **kwargs): @staticmethod def get_group_sessions(session_group_name: str): + """Get the NWB file names of all sessions in a session group.""" results = ( SessionGroupSession & {"session_group_name": session_group_name} ).fetch(as_dict=True) @@ -261,6 +279,7 @@ def get_group_sessions(session_group_name: str): @staticmethod def create_spyglass_view(session_group_name: str): + """Create a FigURL view for a session group.""" import figurl as fig FIGURL_CHANNEL = config.get("FIGURL_CHANNEL") diff --git a/src/spyglass/common/common_task.py b/src/spyglass/common/common_task.py index 94ec34c58..b4f87eb97 100644 --- a/src/spyglass/common/common_task.py +++ b/src/spyglass/common/common_task.py @@ -102,6 +102,7 @@ class TaskEpoch(SpyglassMixin, dj.Imported): """ def make(self, key): + """Populate TaskEpoch from the processing module in the NWB file.""" nwb_file_name = key["nwb_file_name"] nwb_file_abspath = Nwbfile().get_abs_path(nwb_file_name) nwbf = get_nwb_file(nwb_file_abspath) @@ -180,6 +181,7 @@ def make(self, key): @classmethod def update_entries(cls, restrict=True): + """Update entries in the TaskEpoch table based on a restriction.""" existing_entries = (cls & restrict).fetch("KEY") for row in existing_entries: if (cls & row).fetch1("camera_names"): diff --git a/src/spyglass/common/common_usage.py b/src/spyglass/common/common_usage.py index 354fb92c4..ad49d82a1 100644 --- a/src/spyglass/common/common_usage.py +++ b/src/spyglass/common/common_usage.py @@ -69,6 +69,7 @@ class ActivityLog(dj.Manual): @classmethod def deprecate_log(cls, name, warning=True) -> None: + """Log a deprecation warning for a feature.""" if warning: logger.warning(f"DEPRECATION scheduled for version 0.6: {name}") cls.insert1(dict(dj_user=dj.config["database.user"], function=name)) @@ -96,10 +97,12 @@ class Table(SpyglassMixin, dj.Part): """ def insert1(self, key, **kwargs): + """Override insert1 to auto-increment table_id.""" key = self._auto_increment(key, pk="table_id") super().insert1(key, **kwargs) def insert(self, keys: List[dict], **kwargs): + """Override insert to auto-increment table_id.""" if not isinstance(keys[0], dict): raise TypeError("Pass Table Keys as list of dict") keys = [self._auto_increment(k, pk="table_id") for k in keys] @@ -232,11 +235,13 @@ class File(SpyglassMixin, dj.Part): """ def populate_paper(self, paper_id: Union[str, dict]): + """Populate Export for a given paper_id.""" if isinstance(paper_id, dict): paper_id = paper_id.get("paper_id") self.populate(ExportSelection().paper_export_id(paper_id)) def make(self, key): + """Populate Export table with the latest export for a given paper.""" paper_key = (ExportSelection & key).fetch("paper_id", as_dict=True)[0] query = ExportSelection & paper_key diff --git a/src/spyglass/decoding/decoding_merge.py b/src/spyglass/decoding/decoding_merge.py index 15483c0c5..5ba8a0ea4 100644 --- a/src/spyglass/decoding/decoding_merge.py +++ b/src/spyglass/decoding/decoding_merge.py @@ -84,14 +84,17 @@ def cleanup(self, dry_run=False): @classmethod def fetch_results(cls, key): + """Fetch the decoding results for a given key.""" return cls().merge_get_parent_class(key).fetch_results() @classmethod def fetch_model(cls, key): + """Fetch the decoding model for a given key.""" return cls().merge_get_parent_class(key).fetch_model() @classmethod def fetch_environments(cls, key): + """Fetch the decoding environments for a given key.""" decoding_selection_key = cls.merge_get_parent(key).fetch1("KEY") return ( cls() @@ -101,6 +104,7 @@ def fetch_environments(cls, key): @classmethod def fetch_position_info(cls, key): + """Fetch the decoding position info for a given key.""" decoding_selection_key = cls.merge_get_parent(key).fetch1("KEY") return ( cls() @@ -110,6 +114,7 @@ def fetch_position_info(cls, key): @classmethod def fetch_linear_position_info(cls, key): + """Fetch the decoding linear position info for a given key.""" decoding_selection_key = cls.merge_get_parent(key).fetch1("KEY") return ( cls() @@ -119,6 +124,7 @@ def fetch_linear_position_info(cls, key): @classmethod def fetch_spike_data(cls, key, filter_by_interval=True): + """Fetch the decoding spike data for a given key.""" decoding_selection_key = cls.merge_get_parent(key).fetch1("KEY") return ( cls() @@ -130,6 +136,7 @@ def fetch_spike_data(cls, key, filter_by_interval=True): @classmethod def create_decoding_view(cls, key, head_direction_name="head_orientation"): + """Create a decoding view for a given key.""" results = cls.fetch_results(key) posterior = ( results.squeeze() diff --git a/src/spyglass/decoding/v0/clusterless.py b/src/spyglass/decoding/v0/clusterless.py index c72d4b80b..cc73fb5eb 100644 --- a/src/spyglass/decoding/v0/clusterless.py +++ b/src/spyglass/decoding/v0/clusterless.py @@ -149,6 +149,15 @@ class UnitMarks(SpyglassMixin, dj.Computed): """ def make(self, key): + """Populate the UnitMarks table. + + 1. Fetch parameters, units, and recording from MarkParameters, + CuratedSpikeSorting, and CuratedRecording tables respectively. + 2. Uses spikeinterface to extract waveforms for each unit. + 3. Optionally calculates the peak amplitude of the waveform and + thresholds the waveform. + 4. Saves the marks as a TimeSeries object in a new AnalysisNwbfile. + """ # create a new AnalysisNwbfile and a timeseries for the marks and save key["analysis_file_name"] = AnalysisNwbfile().create( # logged key["nwb_file_name"] @@ -245,6 +254,7 @@ def fetch1_dataframe(self) -> pd.DataFrame: return self.fetch_dataframe()[0] def fetch_dataframe(self) -> list[pd.DataFrame]: + """Fetches the marks as a list of pandas dataframes""" return [self._convert_to_dataframe(data) for data in self.fetch_nwb()] @staticmethod @@ -324,6 +334,11 @@ class UnitMarksIndicator(SpyglassMixin, dj.Computed): """ def make(self, key): + """Populate the UnitMarksIndicator table. + + Bin spike times and associated spike waveform features according to the + sampling rate. + """ # TODO: intersection of sort interval and interval list interval_times = (IntervalList & key).fetch1("valid_times") diff --git a/src/spyglass/decoding/v0/sorted_spikes.py b/src/spyglass/decoding/v0/sorted_spikes.py index adf1e393e..d087b588b 100644 --- a/src/spyglass/decoding/v0/sorted_spikes.py +++ b/src/spyglass/decoding/v0/sorted_spikes.py @@ -65,6 +65,7 @@ @schema class SortedSpikesIndicatorSelection(SpyglassMixin, dj.Lookup): """Bins spike times into regular intervals given by the sampling rate. + Start and stop time of the interval are defined by the interval list. """ @@ -79,8 +80,8 @@ class SortedSpikesIndicatorSelection(SpyglassMixin, dj.Lookup): @schema class SortedSpikesIndicator(SpyglassMixin, dj.Computed): """Bins spike times into regular intervals given by the sampling rate. - Useful for GLMs and for decoding. + Useful for GLMs and for decoding. """ definition = """ @@ -91,6 +92,12 @@ class SortedSpikesIndicator(SpyglassMixin, dj.Computed): """ def make(self, key): + """Populate the SortedSpikesIndicator table. + + Fetches the spike times from the CuratedSpikeSorting table and bins + them into regular intervals given by the sampling rate. The spike + indicator is stored in an AnalysisNwbfile. + """ pprint.pprint(key) # TODO: intersection of sort interval and interval list interval_times = (IntervalList & key).fetch1("valid_times") @@ -157,10 +164,12 @@ def make(self, key): self.insert1(key) - def fetch1_dataframe(self): + def fetch1_dataframe(self) -> pd.DataFrame: + """Return the first spike indicator as a dataframe.""" return self.fetch_dataframe()[0] - def fetch_dataframe(self): + def fetch_dataframe(self) -> list[pd.DataFrame]: + """Return all spike indicators as a list of dataframes.""" return pd.concat( [ data["spike_indicator"].set_index("time") @@ -183,6 +192,7 @@ class SortedSpikesClassifierParameters(SpyglassMixin, dj.Manual): """ def insert_default(self): + """Insert default parameters for decoding with sorted spikes""" self.insert( [ make_default_decoding_params(), @@ -192,9 +202,11 @@ def insert_default(self): ) def insert1(self, key, **kwargs): + """Override insert1 to convert classes to dict""" super().insert1(convert_classes_to_dict(key), **kwargs) def fetch1(self, *args, **kwargs): + """Override fetch1 to restore classes""" return restore_classes(super().fetch1(*args, **kwargs)) diff --git a/src/spyglass/decoding/v0/visualization.py b/src/spyglass/decoding/v0/visualization.py index aa5347c17..4492109c1 100644 --- a/src/spyglass/decoding/v0/visualization.py +++ b/src/spyglass/decoding/v0/visualization.py @@ -28,6 +28,7 @@ def make_single_environment_movie( direction_name="head_orientation", vmax=0.07, ): + """Generate a movie of the decoding results for a single environment.""" if marks.ndim > 2: multiunit_spikes = (np.any(~np.isnan(marks), axis=1)).astype(float) else: @@ -235,6 +236,7 @@ def _update_plot(time_ind): def setup_subplots( classifier, window_ind=None, rate=None, sampling_frequency=None ): + """Set up subplots for decoding movies.""" env_names = [env.environment_name for env in classifier.environments] mosaic = [] @@ -305,6 +307,7 @@ def make_multi_environment_movie( direction_name="head_orientation", vmax=0.07, ): + """Generate a movie of the decoding results for multiple environments.""" # Set up formatting for the movie files Writer = animation.writers["ffmpeg"] fps = sampling_frequency // video_slowdown @@ -485,6 +488,7 @@ def create_interactive_1D_decoding_figurl( sampling_frequency: float = 500.0, view_height: int = 800, ): + """Create an interactive decoding visualization with FigURL.""" decode_view = create_1D_decode_view( posterior=results[posterior_type].sum("state"), linear_position=linear_position_info[position_name], @@ -575,6 +579,7 @@ def create_interactive_2D_decoding_figurl( sampling_frequency: float = 500.0, view_height: int = 800, ) -> vv.Box: + """Create an interactive 2D decoding visualization with FigURL.""" decode_view = create_2D_decode_view( position_time=position_info.index, position=position_info[position_name], diff --git a/src/spyglass/decoding/v0/visualization_1D_view.py b/src/spyglass/decoding/v0/visualization_1D_view.py index fd69b3a9e..c657a99d3 100644 --- a/src/spyglass/decoding/v0/visualization_1D_view.py +++ b/src/spyglass/decoding/v0/visualization_1D_view.py @@ -10,6 +10,7 @@ def get_observations_per_time( trimmed_posterior: xr.DataArray, base_data: xr.Dataset ) -> np.ndarray: + """Get the number of observations per time bin.""" times, counts = np.unique(trimmed_posterior.time.values, return_counts=True) indexed_counts = xr.DataArray(counts, coords={"time": times}) _, good_counts = xr.align( @@ -20,6 +21,7 @@ def get_observations_per_time( def get_sampling_freq(times: np.ndarray) -> float: + """Get the sampling frequency of the data.""" round_times = np.floor(1000 * times) median_delta_t_ms = np.median(np.diff(round_times)).item() return 1000 / median_delta_t_ms # from time-delta to Hz @@ -28,6 +30,7 @@ def get_sampling_freq(times: np.ndarray) -> float: def get_trimmed_bin_center_index( place_bin_centers: np.ndarray, trimmed_place_bin_centers: np.ndarray ) -> np.ndarray: + """Get the index of the trimmed bin centers in the full array.""" return np.searchsorted( place_bin_centers, trimmed_place_bin_centers, side="left" ).astype(np.uint16) diff --git a/src/spyglass/decoding/v0/visualization_2D_view.py b/src/spyglass/decoding/v0/visualization_2D_view.py index 948a20dd1..e0b7b40de 100644 --- a/src/spyglass/decoding/v0/visualization_2D_view.py +++ b/src/spyglass/decoding/v0/visualization_2D_view.py @@ -17,6 +17,7 @@ def create_static_track_animation( compute_real_time_rate: bool = False, head_dir=None, ): + """Create a static track animation object.""" # float32 gives about 7 digits of decimal precision; we want 3 digits right # of the decimal. So need to compress-store the timestamp if the start is # greater than say 5000. @@ -53,6 +54,7 @@ def create_static_track_animation( def get_base_track_information(base_probabilities: xr.Dataset): + """Get the base track information.""" x_count = len(base_probabilities.x_position) y_count = len(base_probabilities.y_position) x_min = np.min(base_probabilities.x_position).item() @@ -78,6 +80,7 @@ def memo_linearize( y_min: float, y_width: float, ): + """Memoized linearize function.""" (_, y, x) = t my_tuple = (x, y) if my_tuple not in location_lookup: @@ -96,6 +99,7 @@ def generate_linearization_function( y_min: float, y_width: float, ): + """Generate a linearization function.""" args = { "location_lookup": location_lookup, "x_count": x_count, @@ -114,11 +118,13 @@ def inner(t: Tuple[float, float, float]): def get_positions( i_trim: xr.Dataset, linearization_fn: Callable[[Tuple[float, float]], int] ): + """Get the positions from a linearization func.""" linearizer_map = map(linearization_fn, i_trim.unified_index.values) return np.array(list(linearizer_map), dtype=np.uint16) def get_observations_per_frame(i_trim: xr.DataArray, base_slice: xr.DataArray): + """Get the observations per frame.""" (times, time_counts_np) = np.unique(i_trim.time.values, return_counts=True) time_counts = xr.DataArray(time_counts_np, coords={"time": times}) raw_times = base_slice.time @@ -132,6 +138,7 @@ def get_observations_per_frame(i_trim: xr.DataArray, base_slice: xr.DataArray): def extract_slice_data( base_slice: xr.DataArray, location_fn: Callable[[Tuple[float, float]], int] ): + """Extract slice data from a location function.""" i_trim = discretize_and_trim(base_slice, ndims=2) positions = get_positions(i_trim, location_fn) @@ -140,6 +147,8 @@ def extract_slice_data( def process_decoded_data(posterior: xr.DataArray): + """Process the decoded data.""" + frame_step_size = 100_000 location_lookup = {} @@ -212,6 +221,7 @@ def process_decoded_data(posterior: xr.DataArray): def create_track_animation_object(*, static_track_animation: any): + """Create a track animation object.""" if "decodedData" in static_track_animation: decoded_data = static_track_animation["decodedData"] decoded_data_obj = vvf.DecodedPositionData( @@ -259,6 +269,7 @@ def create_track_animation_object(*, static_track_animation: any): def get_ul_corners(width: float, height: float, centers): + """Get the upper left corners.""" ul = np.subtract(centers, (width / 2, -height / 2)) # Reshape so we have an x array and a y array diff --git a/src/spyglass/decoding/v1/clusterless.py b/src/spyglass/decoding/v1/clusterless.py index bfa83e6ac..7e0711ad9 100644 --- a/src/spyglass/decoding/v1/clusterless.py +++ b/src/spyglass/decoding/v1/clusterless.py @@ -53,6 +53,7 @@ class UnitFeatures(SpyglassMixinPart): def create_group( self, nwb_file_name: str, group_name: str, keys: list[dict] ): + """Create a group of waveform features for a given session""" group_key = { "nwb_file_name": nwb_file_name, "waveform_features_group_name": group_name, @@ -95,6 +96,21 @@ class ClusterlessDecodingV1(SpyglassMixin, dj.Computed): """ def make(self, key): + """Populate the ClusterlessDecoding table. + + 1. Fetches... + position data from PositionGroup table + waveform features and spike times from UnitWaveformFeatures table + decoding parameters from DecodingParameters table + encoding/decoding intervals from IntervalList table + 2. Decodes via ClusterlessDetector from non_local_detector package + 3. Optionally estimates decoding parameters + 4. Saves the decoding results (initial conditions, discrete state + transitions) and classifier to disk. May include discrete transition + coefficients if available. + 5. Inserts into ClusterlessDecodingV1 table and DecodingOutput merge + table. + """ orig_key = copy.deepcopy(key) # Get model parameters @@ -296,6 +312,7 @@ def fetch_results(self) -> xr.Dataset: return ClusterlessDetector.load_results(self.fetch1("results_path")) def fetch_model(self): + """Retrieve the decoding model""" return ClusterlessDetector.load_model(self.fetch1("classifier_path")) @staticmethod diff --git a/src/spyglass/decoding/v1/core.py b/src/spyglass/decoding/v1/core.py index a5236d30e..087d958a3 100644 --- a/src/spyglass/decoding/v1/core.py +++ b/src/spyglass/decoding/v1/core.py @@ -55,9 +55,11 @@ class DecodingParameters(SpyglassMixin, dj.Lookup): @classmethod def insert_default(cls): + """Insert default decoding parameters""" cls.insert(cls.contents, skip_duplicates=True) def insert(self, rows, *args, **kwargs): + """Override insert to convert classes to dict before inserting""" for row in rows: row["decoding_params"] = convert_classes_to_dict( vars(row["decoding_params"]) @@ -65,6 +67,7 @@ def insert(self, rows, *args, **kwargs): super().insert(rows, *args, **kwargs) def fetch(self, *args, **kwargs): + """Return decoding parameters as a list of classes.""" rows = super().fetch(*args, **kwargs) if len(rows) > 0 and len(rows[0]) > 1: content = [] @@ -85,6 +88,7 @@ def fetch(self, *args, **kwargs): return content def fetch1(self, *args, **kwargs): + """Return one decoding paramset as a class.""" row = super().fetch1(*args, **kwargs) row["decoding_params"] = restore_classes(row["decoding_params"]) return row @@ -114,6 +118,7 @@ def create_group( position_variables: list[str] = ["position_x", "position_y"], upsample_rate: float = np.nan, ): + """Create a new position group.""" group_key = { "nwb_file_name": nwb_file_name, "position_group_name": group_name, diff --git a/src/spyglass/decoding/v1/sorted_spikes.py b/src/spyglass/decoding/v1/sorted_spikes.py index bb44c843f..9e4c2c3ba 100644 --- a/src/spyglass/decoding/v1/sorted_spikes.py +++ b/src/spyglass/decoding/v1/sorted_spikes.py @@ -60,6 +60,22 @@ class SortedSpikesDecodingV1(SpyglassMixin, dj.Computed): """ def make(self, key): + """Populate the decoding model. + + 1. Fetches parameters and position data from DecodingParameters and + PositionGroup tables. + 2. Decomposes instervals into encoding and decoding. + 3. Optionally estimates decoding parameters, otherwise uses the provided + parameters. + 4. Uses SortedSpikesDetector from non_local_detector package to decode + the animal's mental position, including initial and discrete state + transition information. + 5. Optionally includes the discrete transition coefficients. + 6. Saves the results and model to disk in the analysis directory, under + the nwb file name's folder. + 7. Inserts the results and model paths into SortedSpikesDecodingV1 and + DecodingOutput tables. + """ orig_key = copy.deepcopy(key) # Get model parameters @@ -256,6 +272,7 @@ def fetch_results(self) -> xr.Dataset: return SortedSpikesDetector.load_results(self.fetch1("results_path")) def fetch_model(self): + """Retrieve the decoding model""" return SortedSpikesDetector.load_model(self.fetch1("classifier_path")) @staticmethod diff --git a/src/spyglass/decoding/v1/waveform_features.py b/src/spyglass/decoding/v1/waveform_features.py index 56176484d..818d9bf43 100644 --- a/src/spyglass/decoding/v1/waveform_features.py +++ b/src/spyglass/decoding/v1/waveform_features.py @@ -62,6 +62,7 @@ class WaveformFeaturesParams(SpyglassMixin, dj.Lookup): @classmethod def insert_default(cls): + """Insert default waveform features parameters""" cls.insert(cls.contents, skip_duplicates=True) @staticmethod @@ -106,6 +107,7 @@ class UnitWaveformFeatures(SpyglassMixin, dj.Computed): _parallel_make = True def make(self, key): + """Populate UnitWaveformFeatures table.""" AnalysisNwbfile()._creation_times["pre_create_time"] = time() # get the list of feature parameters params = (WaveformFeaturesParams & key).fetch1("params") diff --git a/src/spyglass/lfp/analysis/v1/lfp_band.py b/src/spyglass/lfp/analysis/v1/lfp_band.py index 985cf84d7..074da4b38 100644 --- a/src/spyglass/lfp/analysis/v1/lfp_band.py +++ b/src/spyglass/lfp/analysis/v1/lfp_band.py @@ -173,6 +173,7 @@ class LFPBandV1(SpyglassMixin, dj.Computed): """ def make(self, key): + """Populate LFPBandV1""" # create the analysis nwb file to store the results. lfp_band_file_name = AnalysisNwbfile().create( # logged key["nwb_file_name"] diff --git a/src/spyglass/lfp/lfp_imported.py b/src/spyglass/lfp/lfp_imported.py index 6219b8a40..a10e918c8 100644 --- a/src/spyglass/lfp/lfp_imported.py +++ b/src/spyglass/lfp/lfp_imported.py @@ -14,7 +14,7 @@ class ImportedLFP(SpyglassMixin, dj.Imported): definition = """ -> Session # the session to which this LFP belongs -> LFPElectrodeGroup # the group of electrodes to be filtered - -> IntervalList # the original set of times to be filtered + -> IntervalList # the original set of times to be filtered lfp_object_id: varchar(40) # object ID for loading from the NWB file --- lfp_sampling_rate: float # the sampling rate, in samples/sec @@ -22,6 +22,7 @@ class ImportedLFP(SpyglassMixin, dj.Imported): """ def make(self, key): + """Placeholder for importing LFP.""" raise NotImplementedError( "For `insert`, use `allow_direct_insert=True`" ) diff --git a/src/spyglass/lfp/lfp_merge.py b/src/spyglass/lfp/lfp_merge.py index 5e55d5f94..0c3c8eac4 100644 --- a/src/spyglass/lfp/lfp_merge.py +++ b/src/spyglass/lfp/lfp_merge.py @@ -44,6 +44,7 @@ class CommonLFP(SpyglassMixin, dj.Part): # noqa: F811 """ def fetch1_dataframe(self, *attrs, **kwargs): + """Fetch a single dataframe from the merged table.""" # Note: `proj` below facilitates operator syntax eg Table & restrict nwb_lfp = self.fetch_nwb(self.proj())[0] return pd.DataFrame( diff --git a/src/spyglass/lfp/v1/lfp.py b/src/spyglass/lfp/v1/lfp.py index 8d12582cd..9c529e24a 100644 --- a/src/spyglass/lfp/v1/lfp.py +++ b/src/spyglass/lfp/v1/lfp.py @@ -59,6 +59,13 @@ class LFPV1(SpyglassMixin, dj.Computed): """ def make(self, key): + """Populate LFPV1 table with the filtered LFP data. + + The LFP data is filtered and downsampled to the user-defined sampling + rate, specified as lfp_sampling_rate. The filtered data is stored in + the AnalysisNwbfile table. The valid times for the filtered data are + stored in the IntervalList table. + """ lfp_file_name = AnalysisNwbfile().create(key["nwb_file_name"]) # logged # get the NWB object with the data nwbf_key = {"nwb_file_name": key["nwb_file_name"]} @@ -196,7 +203,8 @@ def make(self, key): LFPOutput.insert1(orig_key) AnalysisNwbfile().log(key, table=self.full_table_name) - def fetch1_dataframe(self, *attrs, **kwargs): + def fetch1_dataframe(self, *attrs, **kwargs) -> pd.DataFrame: + """Fetch a single dataframe.""" nwb_lfp = self.fetch_nwb()[0] return pd.DataFrame( nwb_lfp["lfp"].data, diff --git a/src/spyglass/lfp/v1/lfp_artifact.py b/src/spyglass/lfp/v1/lfp_artifact.py index 70b8f2f2b..d400f9525 100644 --- a/src/spyglass/lfp/v1/lfp_artifact.py +++ b/src/spyglass/lfp/v1/lfp_artifact.py @@ -127,6 +127,15 @@ class LFPArtifactDetection(SpyglassMixin, dj.Computed): """ def make(self, key): + """Populate the LFPArtifactDetection table with artifact times. + + 1. Fetch parameters and LFP data from LFPArtifactDetectionParameters + and LFPV1, respectively. + 2. Optionally reference the LFP data. + 3. Pass data to chosen artifact detection algorithm. + 3. Insert into LFPArtifactRemovedIntervalList, IntervalList, and + LFPArtifactDetection. + """ artifact_params = ( LFPArtifactDetectionParameters & {"artifact_params_name": key["artifact_params_name"]} diff --git a/src/spyglass/linearization/merge.py b/src/spyglass/linearization/merge.py index ee4feae78..8ad91eaff 100644 --- a/src/spyglass/linearization/merge.py +++ b/src/spyglass/linearization/merge.py @@ -1,8 +1,8 @@ import datajoint as dj -from spyglass.linearization.v0.main import ( # noqa F401 +from spyglass.linearization.v0.main import ( IntervalLinearizedPosition as LinearizedPositionV0, -) +) # noqa F401 from spyglass.linearization.v1.main import LinearizedPositionV1 # noqa F401 from spyglass.utils import SpyglassMixin, _Merge @@ -32,6 +32,7 @@ class LinearizedPositionV1(SpyglassMixin, dj.Part): # noqa: F811 """ def fetch1_dataframe(self): + """Fetch a single dataframe from the merged table.""" return self.fetch_nwb(self.proj())[0]["linearized_position"].set_index( "time" ) diff --git a/src/spyglass/linearization/v0/main.py b/src/spyglass/linearization/v0/main.py index 30749f8a2..c802b583c 100644 --- a/src/spyglass/linearization/v0/main.py +++ b/src/spyglass/linearization/v0/main.py @@ -1,5 +1,6 @@ import datajoint as dj import numpy as np +from pandas import DataFrame from track_linearization import ( get_linearized_position, make_track_graph, @@ -57,6 +58,7 @@ class TrackGraph(SpyglassMixin, dj.Manual): """ def get_networkx_track_graph(self, track_graph_parameters=None): + """Get the track graph as a networkx graph.""" if track_graph_parameters is None: track_graph_parameters = self.fetch1() return make_track_graph( @@ -121,6 +123,7 @@ class IntervalLinearizedPosition(SpyglassMixin, dj.Computed): """ def make(self, key): + """Compute linearized position for a given key.""" logger.info(f"Computing linear position for: {key}") key["analysis_file_name"] = AnalysisNwbfile().create( # logged @@ -188,5 +191,6 @@ def make(self, key): AnalysisNwbfile().log(key, table=self.full_table_name) - def fetch1_dataframe(self): + def fetch1_dataframe(self) -> DataFrame: + """Fetch a single dataframe""" return self.fetch_nwb()[0]["linearized_position"].set_index("time") diff --git a/src/spyglass/linearization/v1/main.py b/src/spyglass/linearization/v1/main.py index 91c1726b1..76ec85aaf 100644 --- a/src/spyglass/linearization/v1/main.py +++ b/src/spyglass/linearization/v1/main.py @@ -3,6 +3,7 @@ import datajoint as dj import numpy as np from datajoint.utils import to_camel_case +from pandas import DataFrame from track_linearization import ( get_linearized_position, make_track_graph, @@ -51,6 +52,7 @@ class TrackGraph(SpyglassMixin, dj.Manual): """ def get_networkx_track_graph(self, track_graph_parameters=None): + """Get the track graph as a networkx graph.""" if track_graph_parameters is None: track_graph_parameters = self.fetch1() return make_track_graph( @@ -116,6 +118,16 @@ class LinearizedPositionV1(SpyglassMixin, dj.Computed): """ def make(self, key): + """Populate LinearizedPositionV1 table with the linearized position. + + The linearized position is computed from the position data in the + PositionOutput table. Parameters for linearization are specified in + LinearizationParameters and the track graph is specified in TrackGraph. + The linearization function is defined by the track_linearization + package. The resulting linearized position is stored in an + AnalysisNwbfile and added as an entry in the LinearizedPositionV1 and + LinearizedPositionOutput (Merge) tables. + """ orig_key = copy.deepcopy(key) logger.info(f"Computing linear position for: {key}") @@ -185,5 +197,6 @@ def make(self, key): AnalysisNwbfile().log(key, table=self.full_table_name) - def fetch1_dataframe(self): + def fetch1_dataframe(self) -> DataFrame: + """Fetch a single dataframe.""" return self.fetch_nwb()[0]["linearized_position"].set_index("time") diff --git a/src/spyglass/mua/v1/mua.py b/src/spyglass/mua/v1/mua.py index c31320015..d8975fd49 100644 --- a/src/spyglass/mua/v1/mua.py +++ b/src/spyglass/mua/v1/mua.py @@ -1,6 +1,7 @@ import datajoint as dj import numpy as np import sortingview.views as vv +from pandas import DataFrame from ripple_detection import multiunit_HSE_detector from scipy.stats import zscore @@ -55,6 +56,16 @@ class MuaEventsV1(SpyglassMixin, dj.Computed): """ def make(self, key): + """Populates the MuaEventsV1 table. + + Fetches... + - Speed from PositionOutput + - Spike indicator from SortedSpikesGroup + - Valid times from IntervalList + - Parameters from MuaEventsParameters + Uses multiunit_HSE_detector from ripple_detection package to detect + multiunit activity. + """ speed = self.get_speed(key) time = speed.index.to_numpy() speed = speed.to_numpy() @@ -103,15 +114,18 @@ def fetch1_dataframe(self): """Convenience function for returning the marks in a readable format""" return self.fetch_dataframe()[0] - def fetch_dataframe(self): + def fetch_dataframe(self) -> list[DataFrame]: + """Fetch the MUA times as a list of dataframes""" return [data["mua_times"] for data in self.fetch_nwb()] @classmethod def get_firing_rate(cls, key, time): + """Get the firing rate of the multiunit activity""" return SortedSpikesGroup.get_firing_rate(key, time, multiunit=True) @staticmethod def get_speed(key): + """Get the speed of the animal during the recording.""" position_info = ( PositionOutput & {"merge_id": key["pos_merge_id"]} ).fetch1_dataframe() @@ -128,6 +142,7 @@ def create_figurl( mua_color="black", view_height=800, ): + """Create a FigURL for the MUA detection.""" key = self.fetch1("KEY") speed = self.get_speed(key) time = speed.index.to_numpy() diff --git a/src/spyglass/position/position_merge.py b/src/spyglass/position/position_merge.py index b6346b938..76d9b40f8 100644 --- a/src/spyglass/position/position_merge.py +++ b/src/spyglass/position/position_merge.py @@ -1,5 +1,6 @@ import datajoint as dj from datajoint.utils import to_camel_case +from pandas import DataFrame from spyglass.common.common_position import IntervalPositionInfo as CommonPos from spyglass.position.v1.position_dlc_selection import DLCPosV1 @@ -62,7 +63,8 @@ class CommonPos(SpyglassMixin, dj.Part): -> CommonPos """ - def fetch1_dataframe(self): + def fetch1_dataframe(self) -> DataFrame: + """Fetch a single dataframe from the merged table.""" # proj replaces operator restriction to enable # (TableName & restriction).fetch1_dataframe() key = self.merge_restrict(self.proj()).proj() diff --git a/src/spyglass/position/v1/__init__.py b/src/spyglass/position/v1/__init__.py index 9fc821cb6..2ca8d2ac9 100644 --- a/src/spyglass/position/v1/__init__.py +++ b/src/spyglass/position/v1/__init__.py @@ -60,7 +60,8 @@ ) -def schemas(): +def schemas() -> list[str]: + """Return list of schemas in position/v1.""" return _schemas diff --git a/src/spyglass/position/v1/dlc_reader.py b/src/spyglass/position/v1/dlc_reader.py index c3d8e8af8..e840b6d82 100644 --- a/src/spyglass/position/v1/dlc_reader.py +++ b/src/spyglass/position/v1/dlc_reader.py @@ -113,13 +113,15 @@ def __init__( @property def pkl(self): + """Pickle object with metadata about the DLC run.""" if self._pkl is None: with open(self.pkl_path, "rb") as f: self._pkl = pickle.load(f) return self._pkl["data"] @property # DLC aux_func has a read_config option, but it rewrites the proj path - def yml(self): + def yml(self) -> dict: + """Dictionary of the yaml file DLC metadata.""" if self._yml is None: with open(self.yml_path, "rb") as f: safe_yaml = yaml.YAML(typ="safe", pure=True) @@ -128,26 +130,31 @@ def yml(self): @property def rawdata(self): + """Pandas dataframe of the DLC output from the h5 file.""" if self._rawdata is None: self._rawdata = pd.read_hdf(self.h5_path) return self._rawdata @property - def data(self): + def data(self) -> dict: + """Dictionary of the bodyparts and corresponding dataframe data.""" if self._data is None: self._data = self.reformat_rawdata() return self._data @property - def df(self): + def df(self) -> pd.DataFrame: + """Pandas dataframe of the DLC output from the h5 file.""" top_level = self.rawdata.columns.levels[0][0] return self.rawdata.get(top_level) @property - def body_parts(self): + def body_parts(self) -> list[str]: + """List of body parts in the DLC output.""" return self.df.columns.levels[0] - def reformat_rawdata(self): + def reformat_rawdata(self) -> dict: + """Reformat the rawdata from the h5 file to a more useful dictionary.""" error_message = ( f"Total frames from .h5 file ({len(self.rawdata)}) differs " + f'from .pickle ({self.pkl["nframes"]})' diff --git a/src/spyglass/position/v1/dlc_utils.py b/src/spyglass/position/v1/dlc_utils.py index df5c16427..7ea82fa70 100644 --- a/src/spyglass/position/v1/dlc_utils.py +++ b/src/spyglass/position/v1/dlc_utils.py @@ -208,6 +208,7 @@ def wrapper(self, *args, **kwargs): def get_dlc_root_data_dir(): + """Returns list of potential root directories for DLC data""" ActivityLog().deprecate_log("dlc_utils: get_dlc_root_data_dir") if "custom" in dj.config: if "dlc_root_data_dir" in dj.config["custom"]: @@ -574,8 +575,8 @@ def output_to_list(x): def get_span_start_stop(indices): + """Get start and stop indices of spans of consecutive indices""" span_inds = [] - # Get start and stop index of spans of consecutive indices for k, g in groupby(enumerate(indices), lambda x: x[1] - x[0]): group = list(map(itemgetter(1), g)) span_inds.append((group[0], group[-1])) @@ -583,6 +584,7 @@ def get_span_start_stop(indices): def interp_pos(dlc_df, spans_to_interp, **kwargs): + """Interpolate x and y positions in DLC dataframe""" idx = pd.IndexSlice no_x_msg = "Index {ind} has no {coord}point with which to interpolate" @@ -633,6 +635,7 @@ def _get_new_dim(dim, span_start, span_stop, start_time, stop_time): def interp_orientation(df, spans_to_interp, **kwargs): + """Interpolate orientation in DLC dataframe""" idx = pd.IndexSlice no_x_msg = "Index {ind} has no {x}point with which to interpolate" df_orient = df["orientation"] @@ -664,6 +667,7 @@ def interp_orientation(df, spans_to_interp, **kwargs): def smooth_moving_avg( interp_df, smoothing_duration: float, sampling_rate: int, **kwargs ): + """Smooths x and y positions in DLC dataframe""" import bottleneck as bn idx = pd.IndexSlice @@ -696,6 +700,7 @@ def two_pt_head_orientation(pos_df: pd.DataFrame, **params): def no_orientation(pos_df: pd.DataFrame, **params): + """Returns an array of NaNs for orientation""" fill_value = params.pop("fill_with", np.nan) n_frames = len(pos_df) orientation = np.full( @@ -837,6 +842,7 @@ def get_1pt_centroid(self): self.centroid[~mask] = np.nan # For bad points, centroid is NaN def get_2pt_centroid(self): + """Calculate centroid for two points""" self.calc_centroid( # Good points points=self.point_names, mask=(~self.nans[p] for p in self.point_names), @@ -852,6 +858,13 @@ def get_2pt_centroid(self): ) def get_4pt_centroid(self): + """Calculate centroid for four points. + + If green and center are good, then centroid is average. + If green and left/right are good, then centroid is average. + If only left/right are good, then centroid is the average of left/right. + If only the center is good, then centroid is the center. + """ green = self.points_dict.get("greenLED", None) red_C = self.points_dict.get("redLED_C", None) red_L = self.points_dict.get("redLED_L", None) diff --git a/src/spyglass/position/v1/dlc_utils_makevid.py b/src/spyglass/position/v1/dlc_utils_makevid.py index 12e9baeb0..5ee7c096a 100644 --- a/src/spyglass/position/v1/dlc_utils_makevid.py +++ b/src/spyglass/position/v1/dlc_utils_makevid.py @@ -110,6 +110,7 @@ def __init__( self.make_video() def make_video(self): + """Make video based on processor chosen at init.""" if self.processor == "opencv": self.make_video_opencv() elif self.processor == "opencv-trodes": @@ -228,6 +229,7 @@ def _make_circle( ) def make_video_opencv(self): + """Make video using opencv.""" _ = self._init_cv_video() if self.video_time: @@ -293,6 +295,7 @@ def make_video_opencv(self): return def make_trodes_video(self): + """Make video using opencv with trodes data.""" _ = self._init_cv_video() if np.any(self.video_time): @@ -339,6 +342,7 @@ def make_trodes_video(self): self._close_cv_video() def make_video_matplotlib(self): + """Make video using matplotlib.""" import matplotlib.animation as animation self.position_mean = self.position_mean["DLC"] @@ -559,4 +563,5 @@ def _update_plot(self, time_ind, *args): def make_video(**kwargs): + """Passthrough for VideoMaker class for backwards compatibility.""" VideoMaker(**kwargs) diff --git a/src/spyglass/position/v1/position_dlc_centroid.py b/src/spyglass/position/v1/position_dlc_centroid.py index 8c8f43258..04815ae53 100644 --- a/src/spyglass/position/v1/position_dlc_centroid.py +++ b/src/spyglass/position/v1/position_dlc_centroid.py @@ -64,7 +64,8 @@ def insert_default(cls, **kwargs): ) @classmethod - def get_default(cls): + def get_default(cls) -> dict: + """Get the default centroid parameters""" query = cls & {"dlc_centroid_params_name": "default"} if not len(query) > 0: cls().insert_default(skip_duplicates=True) @@ -133,6 +134,20 @@ class DLCCentroid(SpyglassMixin, dj.Computed): log_path = None def make(self, key): + """Populate the DLCCentroid table with the centroid of the bodyparts + + Uses a decorator around the _logged_make method to log the process + to a file. + + 1. Fetch parameters and centroid method. + 2. Fetch a concatenated dataframe of all bodyparts from + DLCSmoothInterpCohort. + 3. Use the Centroid class to calculate the centroid. + 4. Optionally, interpolate over NaNs and smooth the centroid. + 5. Create a Position and Velocity objects for the centroid and video + frame indices. + 5. Add these objects to the Analysis NWB file. + """ output_dir = infer_output_dir(key=key, makedir=False) self.log_path = Path(output_dir, "log.log") self._logged_make(key) @@ -295,7 +310,8 @@ def _logged_make(self, key): } ) - def fetch1_dataframe(self): + def fetch1_dataframe(self) -> pd.DataFrame: + """Fetch a single dataframe.""" nwb_data = self.fetch_nwb()[0] index = pd.Index( np.asarray( diff --git a/src/spyglass/position/v1/position_dlc_cohort.py b/src/spyglass/position/v1/position_dlc_cohort.py index 4883fc335..04a8de197 100644 --- a/src/spyglass/position/v1/position_dlc_cohort.py +++ b/src/spyglass/position/v1/position_dlc_cohort.py @@ -6,9 +6,9 @@ from spyglass.common.common_nwbfile import AnalysisNwbfile from spyglass.position.v1.dlc_utils import file_log, infer_output_dir -from spyglass.position.v1.position_dlc_pose_estimation import ( # noqa: F401 +from spyglass.position.v1.position_dlc_pose_estimation import ( DLCPoseEstimation, -) +) # noqa: F401 from spyglass.position.v1.position_dlc_position import DLCSmoothInterp from spyglass.utils import SpyglassMixin, logger @@ -40,7 +40,6 @@ class DLCSmoothInterpCohort(SpyglassMixin, dj.Computed): # Need to ensure that nwb_file_name/epoch/interval list name endure as primary keys definition = """ -> DLCSmoothInterpCohortSelection - --- """ log_path = None @@ -54,7 +53,8 @@ class BodyPart(SpyglassMixin, dj.Part): dlc_smooth_interp_info_object_id : varchar(80) """ - def fetch1_dataframe(self): + def fetch1_dataframe(self) -> pd.DataFrame: + """Fetch a single dataframe.""" nwb_data = self.fetch_nwb()[0] index = pd.Index( np.asarray( @@ -91,6 +91,14 @@ def fetch1_dataframe(self): ) def make(self, key): + """Populate DLCSmoothInterpCohort table with the combined bodyparts. + + Calls _logged_make to log the process to a log.log file while... + 1. Fetching the cohort selection and smooted interpolated data for each + bodypart. + 2. Ensuring the number of bodyparts match across data and parameters. + 3. Inserting the combined bodyparts into DLCSmoothInterpCohort. + """ output_dir = infer_output_dir(key=key, makedir=False) self.log_path = Path(output_dir) / "log.log" self._logged_make(key) @@ -116,6 +124,7 @@ def _logged_make(self, key): + f"bodyparts_params_dict {len(bp_params_dict)}" ) + # TODO: change to DLCSmoothInterp.heading.names table_column_names = list(table_entries[0].dtype.fields.keys()) part_keys = [ diff --git a/src/spyglass/position/v1/position_dlc_model.py b/src/spyglass/position/v1/position_dlc_model.py index 979e4ddf1..ce9310d5b 100644 --- a/src/spyglass/position/v1/position_dlc_model.py +++ b/src/spyglass/position/v1/position_dlc_model.py @@ -27,6 +27,7 @@ class DLCModelInput(SpyglassMixin, dj.Manual): """ def insert1(self, key, **kwargs): + """Override insert1 to add dlc_model_name from project_path""" # expects key from DLCProject with config_path project_path = Path(key["config_path"]).parent if not project_path.exists(): @@ -82,6 +83,7 @@ def insert_entry( key: dict = None, **kwargs, ): + """Insert entry into DLCModelSource and corresponding Part table""" cls.insert1( { "dlc_model_name": dlc_model_name, @@ -116,6 +118,7 @@ class DLCModelParams(SpyglassMixin, dj.Manual): @classmethod def insert_default(cls, **kwargs): + """Insert the default parameter set""" params = { "params": {}, "shuffle": 1, @@ -128,6 +131,7 @@ def insert_default(cls, **kwargs): @classmethod def get_default(cls): + """Return the default parameter set. If it doesn't exist, insert it.""" query = cls & {"dlc_model_params_name": "default"} if not len(query) > 0: cls().insert_default(skip_duplicates=True) @@ -172,6 +176,7 @@ class BodyPart(SpyglassMixin, dj.Part): # noqa: F811 """ def make(self, key): + """Populate DLCModel table with model information.""" from deeplabcut.utils.auxiliaryfunctions import GetScorerName _, model_name, table_source = (DLCModelSource & key).fetch1().values() diff --git a/src/spyglass/position/v1/position_dlc_orient.py b/src/spyglass/position/v1/position_dlc_orient.py index d87ecf8bc..118ddffc8 100644 --- a/src/spyglass/position/v1/position_dlc_orient.py +++ b/src/spyglass/position/v1/position_dlc_orient.py @@ -43,6 +43,7 @@ class DLCOrientationParams(SpyglassMixin, dj.Manual): @classmethod def insert_params(cls, params_name: str, params: dict, **kwargs): + """Insert a set of parameters for orientation calculation""" cls.insert1( {"dlc_orientation_params_name": params_name, "params": params}, **kwargs, @@ -50,6 +51,7 @@ def insert_params(cls, params_name: str, params: dict, **kwargs): @classmethod def insert_default(cls, **kwargs): + """Insert the default set of parameters for orientation calculation""" params = { "orient_method": "red_green_orientation", "bodypart1": "greenLED", @@ -63,6 +65,7 @@ def insert_default(cls, **kwargs): @classmethod def get_default(cls): + """Return the default set of parameters for orientation calculation""" query = cls & {"dlc_orientation_params_name": "default"} if not len(query) > 0: cls().insert_default(skip_duplicates=True) @@ -111,6 +114,14 @@ def _get_pos_df(self, key): return pos_df def make(self, key): + """Populate the DLCOrientation table. + + 1. Fetch parameters and position data from DLCOrientationParams and + DLCSmoothInterpCohort.BodyPart tables, respectively. + 2. Apply chosen orientation method to position data. + 3. Generate a CompassDirection object and add it to the AnalysisNwbfile. + 4. Insert the key into the DLCOrientation table. + """ # Get labels to smooth from Parameters table AnalysisNwbfile()._creation_times["pre_create_time"] = time() pos_df = self._get_pos_df(key) @@ -123,6 +134,7 @@ def make(self, key): orient_func = _key_to_func_dict[params["orient_method"]] orientation = orient_func(pos_df, **params) + # TODO: Absorb this into the `no_orientation` function if not params["orient_method"] == "none": # Smooth orientation is_nan = np.isnan(orientation) @@ -182,7 +194,8 @@ def make(self, key): self.insert1(key) AnalysisNwbfile().log(key, table=self.full_table_name) - def fetch1_dataframe(self): + def fetch1_dataframe(self) -> pd.DataFrame: + """Fetch a single dataframe""" nwb_data = self.fetch_nwb()[0] index = pd.Index( np.asarray( diff --git a/src/spyglass/position/v1/position_dlc_pose_estimation.py b/src/spyglass/position/v1/position_dlc_pose_estimation.py index 8fa3df922..2ff376837 100644 --- a/src/spyglass/position/v1/position_dlc_pose_estimation.py +++ b/src/spyglass/position/v1/position_dlc_pose_estimation.py @@ -163,7 +163,8 @@ class BodyPart(SpyglassMixin, dj.Part): _nwb_table = AnalysisNwbfile log_path = None - def fetch1_dataframe(self): + def fetch1_dataframe(self) -> pd.DataFrame: + """Fetch a single bodypart dataframe.""" nwb_data = self.fetch_nwb()[0] index = pd.Index( np.asarray( @@ -348,7 +349,8 @@ def _logged_make(self, key): self.BodyPart.insert1(key) AnalysisNwbfile().log(key, table=self.full_table_name) - def fetch_dataframe(self, *attrs, **kwargs): + def fetch_dataframe(self, *attrs, **kwargs) -> pd.DataFrame: + """Fetch a concatenated dataframe of all bodyparts.""" entries = (self.BodyPart & self).fetch("KEY") nwb_data_dict = { entry["bodypart"]: (self.BodyPart() & entry).fetch_nwb()[0] @@ -405,6 +407,7 @@ def fetch_dataframe(self, *attrs, **kwargs): def convert_to_cm(df, meters_to_pixels): + """Converts x and y columns from pixels to cm""" CM_TO_METERS = 100 idx = pd.IndexSlice df.loc[:, idx[("x", "y")]] *= meters_to_pixels * CM_TO_METERS diff --git a/src/spyglass/position/v1/position_dlc_position.py b/src/spyglass/position/v1/position_dlc_position.py index 01b732612..58dce0a89 100644 --- a/src/spyglass/position/v1/position_dlc_position.py +++ b/src/spyglass/position/v1/position_dlc_position.py @@ -53,6 +53,7 @@ class DLCSmoothInterpParams(SpyglassMixin, dj.Manual): @classmethod def insert_params(cls, params_name: str, params: dict, **kwargs): + """Insert parameters for smoothing and interpolation.""" cls.insert1( {"dlc_si_params_name": params_name, "params": params}, **kwargs, @@ -60,6 +61,8 @@ def insert_params(cls, params_name: str, params: dict, **kwargs): @classmethod def insert_default(cls, **kwargs): + """Insert the default set of parameters.""" + default_params = { "smooth": True, "smoothing_params": { @@ -80,7 +83,8 @@ def insert_default(cls, **kwargs): ) @classmethod - def insert_nan_params(cls, **kwargs): + def insert_nan_params(cls, **kwargs) -> None: + """Insert parameters that only NaN the data.""" nan_params = { "smooth": False, "interpolate": False, @@ -93,7 +97,8 @@ def insert_nan_params(cls, **kwargs): ) @classmethod - def get_default(cls): + def get_default(cls) -> dict: + """Return the default set of parameters for smoothing calculation.""" query = cls & {"dlc_si_params_name": "default"} if not len(query) > 0: cls().insert_default(skip_duplicates=True) @@ -103,7 +108,8 @@ def get_default(cls): return default @classmethod - def get_nan_params(cls): + def get_nan_params(cls) -> dict: + """Return the parameters that NaN the data.""" query = cls & {"dlc_si_params_name": "just_nan"} if not len(query) > 0: cls().insert_nan_params(skip_duplicates=True) @@ -114,9 +120,11 @@ def get_nan_params(cls): @staticmethod def get_available_methods(): + """Return the available smoothing methods.""" return _key_to_smooth_func_dict.keys() def insert1(self, key, **kwargs): + """Override insert1 to validate params.""" params = key.get("params") if not isinstance(params, dict): raise KeyError("'params' must be a dict in key") @@ -161,6 +169,17 @@ class DLCSmoothInterp(SpyglassMixin, dj.Computed): log_path = None def make(self, key): + """Populate the DLCSmoothInterp table. + + Uses a decorator to log the output to a file. + + 1. Fetches the DLC output dataframe from DLCPoseEstimation + 2. NaNs low likelihood points and interpolates across them + 3. Optionally smooths and interpolates the data + 4. Create position and video frame index NWB objects + 5. Add NWB objects to AnalysisNwbfile + 6. Insert the key into DLCSmoothInterp. + """ self.log_path = ( Path(infer_output_dir(key=key, makedir=False)) / "log.log" ) @@ -270,7 +289,8 @@ def _logged_make(self, key): self.insert1(key) AnalysisNwbfile().log(key, table=self.full_table_name) - def fetch1_dataframe(self): + def fetch1_dataframe(self) -> pd.DataFrame: + """Fetch a single dataframe.""" nwb_data = self.fetch_nwb()[0] index = pd.Index( np.asarray( @@ -313,6 +333,7 @@ def nan_inds( likelihood_thresh: float, inds_to_span: int, ): + """Replace low likelihood points with NaNs and interpolate over them.""" idx = pd.IndexSlice # Could either NaN sub-likelihood threshold inds here and then not consider @@ -462,10 +483,12 @@ def get_good_spans(bad_inds_mask, inds_to_span: int = 50): def span_length(x): + """Return the length of a span.""" return x[-1] - x[0] def get_subthresh_inds(dlc_df: pd.DataFrame, likelihood_thresh: float): + """Return indices of subthresh points.""" df_filter = dlc_df["likelihood"] < likelihood_thresh sub_thresh_inds = np.where( ~np.isnan(dlc_df["likelihood"].where(df_filter)) diff --git a/src/spyglass/position/v1/position_dlc_project.py b/src/spyglass/position/v1/position_dlc_project.py index f2d377aef..2f19b1664 100644 --- a/src/spyglass/position/v1/position_dlc_project.py +++ b/src/spyglass/position/v1/position_dlc_project.py @@ -87,6 +87,7 @@ class File(SpyglassMixin, dj.Part): """ def insert1(self, key, **kwargs): + """Override insert1 to check types of key values.""" if not isinstance(key["project_name"], str): raise ValueError("project_name must be a string") if not isinstance(key["frames_per_video"], int): @@ -339,6 +340,7 @@ def add_video_files( add_to_files=True, **kwargs, ): + """Add videos to existing project or create new project""" has_config_or_key = bool(config_path) or bool(key) if add_new and not has_config_or_key: raise ValueError("If add_new, must provide key or config_path") diff --git a/src/spyglass/position/v1/position_dlc_selection.py b/src/spyglass/position/v1/position_dlc_selection.py index 7d90a6d2a..e0bd0359e 100644 --- a/src/spyglass/position/v1/position_dlc_selection.py +++ b/src/spyglass/position/v1/position_dlc_selection.py @@ -57,6 +57,14 @@ class DLCPosV1(SpyglassMixin, dj.Computed): """ def make(self, key): + """Populate the table with the combined position information. + + 1. Fetches position and orientation data from the DLCCentroid and + DLCOrientation tables. + 2. Creates NWB objects for position, orientation, and velocity. + 3. Generates an AnalysisNwbfile and adds the NWB objects to it. + 4. Inserts the key into the table, and the PositionOutput Merge table. + """ orig_key = copy.deepcopy(key) # Add to Analysis NWB file AnalysisNwbfile()._creation_times["pre_create_time"] = time() @@ -149,7 +157,8 @@ def make(self, key): ) AnalysisNwbfile().log(key, table=self.full_table_name) - def fetch1_dataframe(self): + def fetch1_dataframe(self) -> pd.DataFrame: + """Return the position data as a DataFrame.""" nwb_data = self.fetch_nwb()[0] index = pd.Index( np.asarray(nwb_data["position"].get_spatial_series().timestamps), @@ -188,11 +197,13 @@ def fetch1_dataframe(self): ) def fetch_nwb(self, **kwargs): + """Fetch the NWB file.""" attrs = [a for a in self.heading.names if not a == "pose_eval_result"] return super().fetch_nwb(*attrs, **kwargs) @classmethod def evaluate_pose_estimation(cls, key): + """Evaluate the pose estimation.""" likelihood_thresh = [] valid_fields = DLCSmoothInterpCohort.BodyPart().heading.names @@ -272,6 +283,7 @@ class DLCPosVideoParams(SpyglassMixin, dj.Manual): @classmethod def insert_default(cls): + """Insert the default parameters.""" params = { "percent_frames": 1, "incl_likelihood": True, @@ -287,6 +299,7 @@ def insert_default(cls): @classmethod def get_default(cls): + """Return the default parameters.""" query = cls & {"dlc_pos_video_params_name": "default"} if not len(query) > 0: cls().insert_default() @@ -318,6 +331,14 @@ class DLCPosVideo(SpyglassMixin, dj.Computed): """ def make(self, key): + """Populate the DLCPosVideo table. + + 1. Fetches parameters from the DLCPosVideoParams table. + 2. Fetches position interval name from epoch name. + 3. Fetches pose estimation data and video information. + 4. Fetches centroid and likelihood data for each bodypart. + 5. Calls make_video to create the video with the above data. + """ M_TO_CM = 100 params = (DLCPosVideoParams & key).fetch1("params") diff --git a/src/spyglass/position/v1/position_dlc_training.py b/src/spyglass/position/v1/position_dlc_training.py index 85e86b1c0..04c45dd3d 100644 --- a/src/spyglass/position/v1/position_dlc_training.py +++ b/src/spyglass/position/v1/position_dlc_training.py @@ -64,6 +64,7 @@ def insert_new_params(cls, paramset_name: str, params: dict, **kwargs): @classmethod def get_accepted_params(cls): + """Return all accepted parameters for DLC model training.""" from deeplabcut import create_training_dataset, train_network return set( @@ -86,6 +87,7 @@ class DLCModelTrainingSelection(SpyglassMixin, dj.Manual): """ def insert1(self, key, **kwargs): # Auto-increment training_id + """Override insert1 to auto-increment training_id if not provided.""" if not (training_id := key.get("training_id")): training_id = ( dj.U().aggr(self & key, n="max(training_id)").fetch1("n") or 0 @@ -231,4 +233,5 @@ def _logged_make(self, key): def get_param_names(func): + """Get parameter names for a function signature.""" return list(inspect.signature(func).parameters) diff --git a/src/spyglass/position/v1/position_trodes_position.py b/src/spyglass/position/v1/position_trodes_position.py index 501407571..72adada46 100644 --- a/src/spyglass/position/v1/position_trodes_position.py +++ b/src/spyglass/position/v1/position_trodes_position.py @@ -4,6 +4,7 @@ import datajoint as dj import numpy as np from datajoint.utils import to_camel_case +from pandas import DataFrame from spyglass.common.common_behav import RawPosition from spyglass.common.common_nwbfile import AnalysisNwbfile @@ -29,11 +30,13 @@ class TrodesPosParams(SpyglassMixin, dj.Manual): """ @property - def default_pk(self): + def default_pk(self) -> dict: + """Return the default primary key for this table.""" return {"trodes_pos_params_name": "default"} @property - def default_params(self): + def default_params(self) -> dict: + """Return the default parameters for this table.""" return { "max_LED_separation": 9.0, "max_plausible_speed": 300.0, @@ -47,7 +50,7 @@ def default_params(self): } @classmethod - def insert_default(cls, **kwargs): + def insert_default(cls, **kwargs) -> None: """ Insert default parameter set for position determination """ @@ -57,7 +60,8 @@ def insert_default(cls, **kwargs): ) @classmethod - def get_default(cls): + def get_default(cls) -> dict: + """Return the default set of parameters for position calculation""" query = cls & cls().default_pk if not len(query) > 0: cls().insert_default(skip_duplicates=True) @@ -66,7 +70,8 @@ def get_default(cls): return query.fetch1() @classmethod - def get_accepted_params(cls): + def get_accepted_params(cls) -> list: + """Return a list of accepted parameters for position calculation""" return [k for k in cls().default_params.keys()] @@ -157,6 +162,15 @@ class TrodesPosV1(SpyglassMixin, dj.Computed): """ def make(self, key): + """Populate the table with position data. + + 1. Fetch the raw position data and parameters from the RawPosition + table and TrodesPosParams table, respectively. + 2. Inherit methods from IntervalPositionInfo to calculate the position + and generate position components (position, orientation, velocity). + 3. Generate AnalysisNwbfile and insert the key into the table. + 4. Insert the key into the PositionOutput Merge table. + """ logger.info(f"Computing position for: {key}") orig_key = copy.deepcopy(key) @@ -197,6 +211,7 @@ def make(self, key): from ..position_merge import PositionOutput + # TODO: change to mixin camelize function part_name = to_camel_case(self.table_name.split("__")[-1]) # TODO: The next line belongs in a merge table function @@ -207,6 +222,7 @@ def make(self, key): @staticmethod def generate_pos_components(*args, **kwargs): + """Generate position components from 2D spatial series.""" return IntervalPositionInfo().generate_pos_components(*args, **kwargs) @staticmethod @@ -214,7 +230,8 @@ def calculate_position_info(*args, **kwargs): """Calculate position info from 2D spatial series.""" return IntervalPositionInfo().calculate_position_info(*args, **kwargs) - def fetch1_dataframe(self, add_frame_ind=True): + def fetch1_dataframe(self, add_frame_ind=True) -> DataFrame: + """Fetch the position data as a pandas DataFrame.""" pos_params = self.fetch1("trodes_pos_params_name") if ( add_frame_ind @@ -237,7 +254,8 @@ class TrodesPosVideo(SpyglassMixin, dj.Computed): """Creates a video of the computed head position and orientation as well as the original LED positions overlaid on the video of the animal. - Use for debugging the effect of position extraction parameters.""" + Use for debugging the effect of position extraction parameters. + """ definition = """ -> TrodesPosV1 @@ -246,6 +264,14 @@ class TrodesPosVideo(SpyglassMixin, dj.Computed): """ def make(self, key): + """Generate a video with overlaid position data. + + Fetches... + - Raw position data from the RawPosition table + - Position data from the TrodesPosV1 table + - Video data from the VideoFile table + Generates a video using opencv and the VideoMaker class. + """ M_TO_CM = 100 logger.info("Loading position data...") diff --git a/src/spyglass/ripple/v1/ripple.py b/src/spyglass/ripple/v1/ripple.py index c6fb92b3a..c0d70ca7b 100644 --- a/src/spyglass/ripple/v1/ripple.py +++ b/src/spyglass/ripple/v1/ripple.py @@ -1,3 +1,5 @@ +from typing import List + import datajoint as dj import matplotlib.pyplot as plt import numpy as np @@ -45,6 +47,7 @@ class RippleLFPElectrode(SpyglassMixin, dj.Part): @staticmethod def validate_key(key): + """Validates that the filter_name is a ripple filter""" filter_name = (LFPBandV1 & key).fetch1("filter_name") if "ripple" not in filter_name.lower(): raise ValueError("Please use a ripple filter") @@ -163,6 +166,17 @@ class RippleTimesV1(SpyglassMixin, dj.Computed): """ def make(self, key): + """Populate RippleTimesV1 table. + + Fetches... + - Nwb file name from LFPBandV1 + - Parameters for ripple detection from RippleParameters + - Ripple LFPs and position info from PositionOutput and LFPBandV1 + Runs she specified ripple detection algorithm (Karlsson or Kay from + ripple_detection package), inserts the results into the analysis nwb + file, and inserts the key into the RippleTimesV1 table. + + """ nwb_file_name = (LFPBandV1 & key).fetch1("nwb_file_name") logger.info(f"Computing ripple times for: {key}") @@ -199,15 +213,26 @@ def make(self, key): self.insert1(key) - def fetch1_dataframe(self): + def fetch1_dataframe(self) -> pd.DataFrame: """Convenience function for returning the marks in a readable format""" return self.fetch_dataframe()[0] - def fetch_dataframe(self): + def fetch_dataframe(self) -> List[pd.DataFrame]: + """Convenience function for returning all marks in a readable format""" return [data["ripple_times"] for data in self.fetch_nwb()] @staticmethod - def get_ripple_lfps_and_position_info(key): + def get_ripple_lfps_and_position_info(key) -> tuple: + """Return the ripple LFPs and position info for the specified key. + + Fetches... + - Ripple parameters from RippleParameters + - Electrode keys from RippleLFPSelection + - LFP data from LFPBandV1 + - Position data from PositionOutput merge table + Interpolates the position data to the LFP timestamps. + """ + # TODO: Pass parameters from make func, instead of fetching again ripple_params = ( RippleParameters & {"ripple_param_name": key["ripple_param_name"]} ).fetch1("ripple_param_dict") @@ -282,7 +307,8 @@ def get_ripple_lfps_and_position_info(key): @staticmethod def get_Kay_ripple_consensus_trace( ripple_filtered_lfps, sampling_frequency, smoothing_sigma: float = 0.004 - ): + ) -> pd.DataFrame: + """Calculate the consensus trace for the ripple filtered LFPs""" ripple_consensus_trace = np.full_like(ripple_filtered_lfps, np.nan) not_null = np.all(pd.notnull(ripple_filtered_lfps), axis=1) @@ -308,6 +334,7 @@ def plot_ripple_consensus_trace( relative=True, ax=None, ): + """Plot the consensus trace for a ripple event""" ripple_start = ripple_times.loc[ripple_label].start_time ripple_end = ripple_times.loc[ripple_label].end_time time_slice = slice(ripple_start - offset, ripple_end + offset) @@ -340,6 +367,7 @@ def plot_ripple( relative: bool = True, ax: Axes = None, ): + """Plot the LFPs for a ripple event""" lfp_labels = lfps.columns n_lfps = len(lfp_labels) ripple_start = ripple_times.loc[ripple_label].start_time @@ -383,6 +411,7 @@ def create_figurl( lfp_offset=1, lfp_channel_ind=None, ): + """Generate a FigURL for the ripple detection""" ripple_times = self.fetch1_dataframe() def _add_ripple_times( @@ -502,6 +531,7 @@ def _add_ripple_times( def interpolate_to_new_time( df, new_time, upsampling_interpolation_method="linear" ): + """Upsample a dataframe to a new time index""" old_time = df.index new_index = pd.Index( np.unique(np.concatenate((old_time, new_time))), name="time" diff --git a/src/spyglass/settings.py b/src/spyglass/settings.py index fceb0beac..ecfd8d540 100644 --- a/src/spyglass/settings.py +++ b/src/spyglass/settings.py @@ -485,43 +485,53 @@ def _dj_custom(self) -> dict: @property def config(self) -> dict: + """Dictionary of config settings.""" self.load_config() return self._config @property def base_dir(self) -> str: + """Base directory as a string.""" return self.config.get(self.dir_to_var("base")) @property def raw_dir(self) -> str: + """Raw data directory as a string.""" return self.config.get(self.dir_to_var("raw")) @property def analysis_dir(self) -> str: + """Analysis directory as a string.""" return self.config.get(self.dir_to_var("analysis")) @property def recording_dir(self) -> str: + """Recording directory as a string.""" return self.config.get(self.dir_to_var("recording")) @property def sorting_dir(self) -> str: + """Sorting directory as a string.""" return self.config.get(self.dir_to_var("sorting")) @property def waveforms_dir(self) -> str: + """Waveforms directory as a string.""" return self.config.get(self.dir_to_var("waveforms")) @property def temp_dir(self) -> str: + """Temp directory as a string.""" return self.config.get(self.dir_to_var("temp")) @property def video_dir(self) -> str: + """Video directory as a string.""" return self.config.get(self.dir_to_var("video")) @property def export_dir(self) -> str: + """Export directory as a string.""" return self.config.get(self.dir_to_var("export")) @property @@ -541,14 +551,17 @@ def test_mode(self) -> bool: @property def dlc_project_dir(self) -> str: + """DLC project directory as a string.""" return self.config.get(self.dir_to_var("project", "dlc")) @property def dlc_video_dir(self) -> str: + """DLC video directory as a string.""" return self.config.get(self.dir_to_var("video", "dlc")) @property def dlc_output_dir(self) -> str: + """DLC output directory as a string.""" return self.config.get(self.dir_to_var("output", "dlc")) diff --git a/src/spyglass/sharing/sharing_kachery.py b/src/spyglass/sharing/sharing_kachery.py index aa3c22747..7b1a7781e 100644 --- a/src/spyglass/sharing/sharing_kachery.py +++ b/src/spyglass/sharing/sharing_kachery.py @@ -29,7 +29,7 @@ def kachery_download_file(uri: str, dest: str, kachery_zone_name: str) -> str: - """set the kachery resource url and attempt to down load the uri into the destination path""" + """Set the kachery resource url and attempt to download.""" KacheryZone.set_resource_url({"kachery_zone_name": kachery_zone_name}) return kcl.load_file(uri, dest=dest) @@ -102,6 +102,7 @@ def set_resource_url(key: dict): @staticmethod def reset_resource_url(): + """Resets the KACHERY_RESOURCE_URL to the default value.""" KacheryZone.reset_zone() if default_kachery_resource_url is not None: os.environ[kachery_resource_url_envar] = ( @@ -134,6 +135,7 @@ class LinkedFile(SpyglassMixin, dj.Part): """ def make(self, key): + """Populate with the uri of the analysis file""" # note that we're assuming that the user has initialized a kachery-cloud # client with kachery-cloud-init. Uncomment the line below once we are # sharing linked files as well. diff --git a/src/spyglass/spikesorting/analysis/v1/group.py b/src/spyglass/spikesorting/analysis/v1/group.py index ad6517558..2f862c4fb 100644 --- a/src/spyglass/spikesorting/analysis/v1/group.py +++ b/src/spyglass/spikesorting/analysis/v1/group.py @@ -43,6 +43,7 @@ class UnitSelectionParams(SpyglassMixin, dj.Manual): @classmethod def insert_default(cls): + """Insert default unit selection parameters""" cls.insert(cls.contents, skip_duplicates=True) @@ -67,6 +68,7 @@ def create_group( unit_filter_params_name: str = "all_units", keys: list[dict] = [], ): + """Create a new group of sorted spikes""" group_key = { "sorted_spikes_group_name": group_name, "nwb_file_name": nwb_file_name, diff --git a/src/spyglass/spikesorting/imported.py b/src/spyglass/spikesorting/imported.py index 7e518d6d8..604718e98 100644 --- a/src/spyglass/spikesorting/imported.py +++ b/src/spyglass/spikesorting/imported.py @@ -46,9 +46,9 @@ def make(self, key): logger.warn("No units found in NWB file") return - from spyglass.spikesorting.spikesorting_merge import ( # noqa: F401 + from spyglass.spikesorting.spikesorting_merge import ( SpikeSortingOutput, - ) + ) # noqa: F401 key["object_id"] = nwbfile.units.object_id @@ -61,12 +61,14 @@ def make(self, key): @classmethod def get_recording(cls, key): + """Placeholder for merge table to call on all sources.""" raise NotImplementedError( "Imported spike sorting does not have a `get_recording` method" ) @classmethod def get_sorting(cls, key): + """Placeholder for merge table to call on all sources.""" raise NotImplementedError( "Imported spike sorting does not have a `get_sorting` method" ) diff --git a/src/spyglass/spikesorting/spikesorting_merge.py b/src/spyglass/spikesorting/spikesorting_merge.py index e7a27bae0..d285ead7e 100644 --- a/src/spyglass/spikesorting/spikesorting_merge.py +++ b/src/spyglass/spikesorting/spikesorting_merge.py @@ -187,6 +187,7 @@ def get_sort_group_info(cls, key): return part_table * sort_group_info # join the info with merge id's def get_spike_times(self, key): + """Get spike times for the group""" spike_times = [] for nwb_file in self.fetch_nwb(key): # V1 uses 'object_id', V0 uses 'units' diff --git a/src/spyglass/spikesorting/v0/figurl_views/SpikeSortingRecordingView.py b/src/spyglass/spikesorting/v0/figurl_views/SpikeSortingRecordingView.py index aedb83522..d28aa0d92 100644 --- a/src/spyglass/spikesorting/v0/figurl_views/SpikeSortingRecordingView.py +++ b/src/spyglass/spikesorting/v0/figurl_views/SpikeSortingRecordingView.py @@ -1,4 +1,3 @@ -import os from typing import List, Union import datajoint as dj @@ -26,6 +25,11 @@ class SpikeSortingRecordingView(SpyglassMixin, dj.Computed): """ def make(self, key): + """Populates SpikeSortingRecordingView. + + Fetches the recording from SpikeSortingRecording and extracts traces + and electrode geometry. + """ # Get the SpikeSortingRecording row rec = (SpikeSortingRecording & key).fetch1() nwb_file_name = rec["nwb_file_name"] @@ -66,6 +70,7 @@ def make(self, key): def create_electrode_geometry(recording: si.BaseRecording): + """Create a figure for the electrode geometry of a recording.""" channel_locations = { str(channel_id): location.astype(np.float32) for location, channel_id in zip( @@ -81,6 +86,7 @@ def create_mountain_layout( label: Union[str, None] = None, sorting_curation_uri: Union[str, None] = None, ) -> Figure: + """Create a figure for a mountain layout of multiple figures""" if label is None: label = "SpikeSortingView" diff --git a/src/spyglass/spikesorting/v0/figurl_views/SpikeSortingView.py b/src/spyglass/spikesorting/v0/figurl_views/SpikeSortingView.py index 45d498565..aef2aa1a7 100644 --- a/src/spyglass/spikesorting/v0/figurl_views/SpikeSortingView.py +++ b/src/spyglass/spikesorting/v0/figurl_views/SpikeSortingView.py @@ -24,6 +24,21 @@ class SpikeSortingView(SpyglassMixin, dj.Computed): """ def make(self, key): + """Populates SpikeSortingView. + + 1. Fetches... + - the recording from SpikeSortingRecording + - the sorting from SpikeSorting + 2. Loads each with spikeinterface and passes to SpikeSortingView from + sortingview package. + 3. Creates... + - Summary + - Autocorrelograms + - Average waveforms + - Spike amplitudes + - Electrode geometry + 4. Creates a mountain layout with the above figures and generates a URL. + """ recording_record = ( SpikeSortingRecording & {"recording_id": key["recording_id"]} ).fetch1() diff --git a/src/spyglass/spikesorting/v0/figurl_views/prepare_spikesortingview_data.py b/src/spyglass/spikesorting/v0/figurl_views/prepare_spikesortingview_data.py index c43031225..ef2d174fc 100644 --- a/src/spyglass/spikesorting/v0/figurl_views/prepare_spikesortingview_data.py +++ b/src/spyglass/spikesorting/v0/figurl_views/prepare_spikesortingview_data.py @@ -19,6 +19,7 @@ def prepare_spikesortingview_data( channel_neighborhood_size: int, output_file_name: str, ) -> str: + """Prepare data for the SpikeSortingView.""" unit_ids = np.array(sorting.get_unit_ids()).astype(np.int32) channel_ids = np.array(recording.get_channel_ids()).astype(np.int32) sampling_frequency = recording.get_sampling_frequency() @@ -27,6 +28,7 @@ def prepare_spikesortingview_data( segment_duration_sec * sampling_frequency ) num_segments = math.ceil(num_frames / num_frames_per_segment) + with h5py.File(output_file_name, "w") as f: f.create_dataset("unit_ids", data=unit_ids) f.create_dataset( @@ -120,7 +122,9 @@ def prepare_spikesortingview_data( ) if peak_channel_id is None: raise Exception( - f"Peak channel not found for unit {unit_id}. This is probably because no spikes were found in any segment for this unit." + f"Peak channel not found for unit {unit_id}. " + + "This is probably because no spikes were found in any " + + "segment for this unit." ) channel_neighborhood = unit_channel_neighborhoods[str(unit_id)] f.create_dataset( @@ -161,7 +165,9 @@ def prepare_spikesortingview_data( ) if peak_channel_id is None: raise Exception( - f"Peak channel not found for unit {unit_id}. This is probably because no spikes were found in any segment for this unit." + f"Peak channel not found for unit {unit_id}. " + + "This is probably because no spikes were found in any" + + " segment for this unit." ) spike_train = sorting.get_unit_spike_train( unit_id=unit_id, @@ -230,6 +236,7 @@ def get_channel_neighborhood( peak_channel_id: int, channel_neighborhood_size: int, ): + """Return the channel neighborhood for a peak channel.""" channel_locations_by_id = {} for ii, channel_id in enumerate(channel_ids): channel_locations_by_id[channel_id] = channel_locations[ii] @@ -247,6 +254,7 @@ def get_channel_neighborhood( def subsample(x: np.array, num: int): + """Subsample an array.""" if num >= len(x): return x stride = math.floor(len(x) / num) @@ -256,6 +264,7 @@ def subsample(x: np.array, num: int): def extract_spike_snippets( *, traces: np.ndarray, times: np.array, snippet_len: Tuple[int] ): + """Extract spike snippets.""" a = snippet_len[0] b = snippet_len[1] T = a + b diff --git a/src/spyglass/spikesorting/v0/merged_sorting_extractor.py b/src/spyglass/spikesorting/v0/merged_sorting_extractor.py index 2ce11d8d1..361d6a076 100644 --- a/src/spyglass/spikesorting/v0/merged_sorting_extractor.py +++ b/src/spyglass/spikesorting/v0/merged_sorting_extractor.py @@ -84,12 +84,13 @@ def __init__( class MergedSortingSegment(si.BaseSortingSegment): def __init__(self): + """Store all the unit spike trains in RAM.""" si.BaseSortingSegment.__init__(self) # Store all the unit spike trains in RAM self._unit_spike_trains: Dict[int, np.array] = {} def add_unit(self, unit_id: int, spike_times: np.array): - # Add a unit spike train + """Add a unit spike train.""" self._unit_spike_trains[unit_id] = spike_times def get_unit_spike_train( @@ -98,7 +99,7 @@ def get_unit_spike_train( start_frame: Union[int, None] = None, end_frame: Union[int, None] = None, ) -> np.ndarray: - # Get a unit spike train + """Get a unit spike train.""" spike_times = self._unit_spike_trains[unit_id] if start_frame is not None: spike_times = spike_times[spike_times >= start_frame] diff --git a/src/spyglass/spikesorting/v0/sortingview.py b/src/spyglass/spikesorting/v0/sortingview.py index b9e3529be..ff62f09b8 100644 --- a/src/spyglass/spikesorting/v0/sortingview.py +++ b/src/spyglass/spikesorting/v0/sortingview.py @@ -116,6 +116,7 @@ def make(self, key: dict): SortingviewWorkspace.URL.insert1(key) def remove_sorting_from_workspace(self, key): + """Remove a sorting from the workspace. NOT IMPLEMENTED YET""" return NotImplementedError def url_trythis(self, key: dict, sortingview_sorting_id: str = None): diff --git a/src/spyglass/spikesorting/v0/spikesorting_artifact.py b/src/spyglass/spikesorting/v0/spikesorting_artifact.py index f78cf0d06..b665b2314 100644 --- a/src/spyglass/spikesorting/v0/spikesorting_artifact.py +++ b/src/spyglass/spikesorting/v0/spikesorting_artifact.py @@ -76,6 +76,15 @@ class ArtifactDetection(SpyglassMixin, dj.Computed): _parallel_make = True def make(self, key): + """Populate the ArtifactDetection table. + + If custom_artifact_detection is set in selection table, do nothing. + + Fetches... + - Parameters from ArtifactDetectionParameters + - Recording from SpikeSortingRecording (loads with spikeinterface) + Uses module-level function _get_artifact_times to detect artifacts. + """ if not (ArtifactDetectionSelection & key).fetch1( "custom_artifact_detection" ): @@ -100,23 +109,6 @@ def make(self, key): recording, **artifact_params, **job_kwargs ) - # NOTE: decided not to do this but to just create a single long segment; keep for now - # get artifact times by segment - # if AppendSegmentRecording, get artifact times for each segment - # if isinstance(recording, AppendSegmentRecording): - # artifact_removed_valid_times = [] - # artifact_times = [] - # for rec in recording.recording_list: - # rec_valid_times, rec_artifact_times = _get_artifact_times(rec, **artifact_params) - # for valid_times in rec_valid_times: - # artifact_removed_valid_times.append(valid_times) - # for artifact_times in rec_artifact_times: - # artifact_times.append(artifact_times) - # artifact_removed_valid_times = np.asarray(artifact_removed_valid_times) - # artifact_times = np.asarray(artifact_times) - # else: - # artifact_removed_valid_times, artifact_times = _get_artifact_times(recording, **artifact_params) - key["artifact_times"] = artifact_times key["artifact_removed_valid_times"] = artifact_removed_valid_times diff --git a/src/spyglass/spikesorting/v0/spikesorting_curation.py b/src/spyglass/spikesorting/v0/spikesorting_curation.py index 57fb90fa5..52a2dff73 100644 --- a/src/spyglass/spikesorting/v0/spikesorting_curation.py +++ b/src/spyglass/spikesorting/v0/spikesorting_curation.py @@ -44,6 +44,7 @@ def apply_merge_groups_to_sorting( sorting: si.BaseSorting, merge_groups: List[List[int]] ): + """Apply merge groups to a sorting extractor.""" # return a new sorting where the units are merged according to merge_groups # merge_groups is a list of lists of unit_ids. # for example: merge_groups = [[1, 2], [5, 8, 4]]] @@ -284,6 +285,7 @@ class WaveformParameters(SpyglassMixin, dj.Manual): """ def insert_default(self): + """Inserts default waveform parameters""" waveform_params_name = "default_not_whitened" waveform_params = { "ms_before": 0.5, @@ -330,6 +332,15 @@ class Waveforms(SpyglassMixin, dj.Computed): """ def make(self, key): + """Populate Waveforms table with waveform extraction results + + 1. Fetches ... + - Recording and sorting from Curation table + - Parameters from WaveformParameters table + 2. Uses spikeinterface to extract waveforms + 3. Generates an analysis NWB file with the waveforms + 4. Inserts the key into Waveforms table + """ key["analysis_file_name"] = AnalysisNwbfile().create( # logged key["nwb_file_name"] ) @@ -385,6 +396,7 @@ def load_waveforms(self, key: dict): return we def fetch_nwb(self, key): + """Fetches the NWB file path for the waveforms. NOT YET IMPLEMENTED.""" # TODO: implement fetching waveforms from NWB return NotImplementedError @@ -454,13 +466,15 @@ def get_metric_default_params(self, metric: str): "Returns default params for the given metric" return self.metric_default_params(metric) - def insert_default(self): + def insert_default(self) -> None: + """Inserts default metric parameters""" self.insert1( ["franklab_default3", self.metric_default_params], skip_duplicates=True, ) def get_available_metrics(self): + """Log available metrics and their descriptions""" for metric in _metric_name_to_func: if metric in self.available_metrics: metric_doc = _metric_name_to_func[metric].__doc__.split("\n")[0] @@ -471,7 +485,7 @@ def get_available_metrics(self): # TODO def _validate_metrics_list(self, key): - """Checks whether a row to be inserted contains only the available metrics""" + """Checks whether a row to be inserted contains only available metrics""" # get available metrics list # get metric list from key # compare @@ -483,10 +497,10 @@ class MetricSelection(SpyglassMixin, dj.Manual): definition = """ -> Waveforms -> MetricParameters - --- """ def insert1(self, key, **kwargs): + """Overriding insert1 to add warnings for peak_offset and peak_channel""" waveform_params = (WaveformParameters & key).fetch1("waveform_params") metric_params = (MetricParameters & key).fetch1("metric_params") if "peak_offset" in metric_params: @@ -517,6 +531,16 @@ class QualityMetrics(SpyglassMixin, dj.Computed): """ def make(self, key): + """Populate QualityMetrics table with quality metric results. + + 1. Fetches ... + - Waveform extractor from Waveforms table + - Parameters from MetricParameters table + 2. Computes metrics, including SNR, ISI violation, NN isolation, + NN noise overlap, peak offset, peak channel, and number of spikes. + 3. Generates an analysis NWB file with the metrics. + 4. Inserts the key into QualityMetrics table + """ analysis_file_name = AnalysisNwbfile().create( # logged key["nwb_file_name"] ) @@ -693,6 +717,7 @@ class AutomaticCurationParameters(SpyglassMixin, dj.Manual): # NOTE: No existing entries impacted by this change def insert1(self, key, **kwargs): + """Overriding insert1 to validats label_params and merge_params""" # validate the labels and then insert # TODO: add validation for merge_params for metric in key["label_params"]: @@ -718,6 +743,7 @@ def insert1(self, key, **kwargs): super().insert1(key, **kwargs) def insert_default(self): + """Inserts default automatic curation parameters""" # label_params parsing: Each key is the name of a metric, # the contents are a three value list with the comparison, a value, # and a list of labels to apply if the comparison is true @@ -766,6 +792,15 @@ class AutomaticCuration(SpyglassMixin, dj.Computed): """ def make(self, key): + """Populate AutomaticCuration table with automatic curation results. + + 1. Fetches ... + - Quality metrics from QualityMetrics table + - Parameters from AutomaticCurationParameters table + - Parent curation/sorting from Curation table + 2. Curates the sorting based on provided merge and label parameters + 3. Inserts IDs into AutomaticCuration and Curation tables + """ metrics_path = (QualityMetrics & key).fetch1("quality_metrics_path") with open(metrics_path) as f: quality_metrics = json.load(f) @@ -939,6 +974,12 @@ class Unit(SpyglassMixin, dj.Part): """ def make(self, key): + """Populate CuratedSpikeSorting table with curated sorting results. + + 1. Fetches metrics and sorting from the Curation table + 2. Saves the sorting in an analysis NWB file + 3. Inserts key into CuratedSpikeSorting table and units into part table. + """ AnalysisNwbfile()._creation_times["pre_create_time"] = time.time() unit_labels_to_remove = ["reject"] # check that the Curation has metrics @@ -1028,7 +1069,8 @@ def make(self, key): key[field] = final_metrics[field][unit_id] else: Warning( - f"No metric named {field} in computed unit quality metrics; skipping" + f"No metric named {field} in computed unit quality " + + "metrics; skipping" ) CuratedSpikeSorting.Unit.insert1(key) diff --git a/src/spyglass/spikesorting/v0/spikesorting_recording.py b/src/spyglass/spikesorting/v0/spikesorting_recording.py index 04369b15e..a6a356a33 100644 --- a/src/spyglass/spikesorting/v0/spikesorting_recording.py +++ b/src/spyglass/spikesorting/v0/spikesorting_recording.py @@ -229,7 +229,8 @@ def get_geometry(self, sort_group_id, nwb_file_name): n_found += 1 else: Warning( - "Relative electrode locations have three coordinates; only two are currently supported" + "Relative electrode locations have three coordinates; " + + "only two are currently supported" ) return np.ndarray.tolist(geometry) @@ -257,6 +258,7 @@ class SpikeSortingPreprocessingParameters(SpyglassMixin, dj.Manual): # All existing entries are below 48 def insert_default(self): + """Inserts the default preprocessing parameters for spike sorting.""" # set up the default filter parameters freq_min = 300 # high pass filter value freq_max = 6000 # low pass filter value @@ -299,6 +301,16 @@ class SpikeSortingRecording(SpyglassMixin, dj.Computed): _parallel_make = True def make(self, key): + """Populates the SpikeSortingRecording table with the recording data. + + 1. Fetches ... + - Sort interval and parameters from SpikeSortingRecordingSelection + and SpikeSortingPreprocessingParameters + - Channel IDs and reference electrode from SortGroup, filtered by + filtereing parameters + 2. Saves the recording data to the recording directory + 3. Inserts the path to the recording data into SpikeSortingRecording + """ sort_interval_valid_times = self._get_sort_interval_valid_times(key) recording = self._get_filtered_recording(key) recording_name = self._get_recording_name(key) diff --git a/src/spyglass/spikesorting/v0/spikesorting_sorting.py b/src/spyglass/spikesorting/v0/spikesorting_sorting.py index ac5648552..5a03649e7 100644 --- a/src/spyglass/spikesorting/v0/spikesorting_sorting.py +++ b/src/spyglass/spikesorting/v0/spikesorting_sorting.py @@ -245,8 +245,8 @@ def make(self, key: dict): self.insert1(key) def fetch_nwb(self, *attrs, **kwargs): + """Placeholder to override mixin method""" raise NotImplementedError - return None def nightly_cleanup(self): """Clean up spike sorting directories that are not in the SpikeSorting table. diff --git a/src/spyglass/spikesorting/v1/artifact.py b/src/spyglass/spikesorting/v1/artifact.py index 04a7dd463..9a4fbeaef 100644 --- a/src/spyglass/spikesorting/v1/artifact.py +++ b/src/spyglass/spikesorting/v1/artifact.py @@ -65,13 +65,14 @@ class ArtifactDetectionParameters(SpyglassMixin, dj.Lookup): @classmethod def insert_default(cls): + """Insert default parameters into ArtifactDetectionParameters.""" cls.insert(cls.contents, skip_duplicates=True) @schema class ArtifactDetectionSelection(SpyglassMixin, dj.Manual): definition = """ - # Processed recording and artifact detection parameters. Use `insert_selection` method to insert new rows. + # Processed recording/artifact detection parameters. See `insert_selection`. artifact_id: uuid --- -> SpikeSortingRecording @@ -80,8 +81,9 @@ class ArtifactDetectionSelection(SpyglassMixin, dj.Manual): @classmethod def insert_selection(cls, key: dict): - """Insert a row into ArtifactDetectionSelection with an - automatically generated unique artifact ID as the sole primary key. + """Insert a row into ArtifactDetectionSelection. + + Automatically generates a unique artifact ID as the sole primary key. Parameters ---------- @@ -91,7 +93,8 @@ def insert_selection(cls, key: dict): Returns ------- artifact_id : str - the unique artifact ID serving as primary key for ArtifactDetectionSelection + the unique artifact ID serving as primary key for + ArtifactDetectionSelection """ query = cls & key if query: @@ -111,6 +114,17 @@ class ArtifactDetection(SpyglassMixin, dj.Computed): """ def make(self, key): + """Populate ArtifactDetection with detected artifacts. + + 1. Fetches... + - Artifact parameters from ArtifactDetectionParameters + - Recording analysis NWB file from SpikeSortingRecording + - Valid times from IntervalList + 2. Load the recording from the NWB file with spikeinterface + 3. Detect artifacts using module-level `_get_artifact_times` + 4. Insert result into IntervalList with `artifact_id` as + `interval_list_name` + """ # FETCH: # - artifact parameters # - recording analysis nwb file @@ -133,6 +147,7 @@ def make(self, key): ).fetch1("interval_list_name"), } ).fetch1("valid_times") + # DO: # - load recording recording_analysis_nwb_file_abs_path = AnalysisNwbfile.get_abs_path( diff --git a/src/spyglass/spikesorting/v1/figurl_curation.py b/src/spyglass/spikesorting/v1/figurl_curation.py index 03b0313c7..5be842d15 100644 --- a/src/spyglass/spikesorting/v1/figurl_curation.py +++ b/src/spyglass/spikesorting/v1/figurl_curation.py @@ -120,6 +120,7 @@ class FigURLCuration(SpyglassMixin, dj.Computed): _use_transaction, _allow_insert = False, True def make(self, key: dict): + """Generate a FigURL for manual curation of a spike sorting.""" # FETCH query = ( FigURLCurationSelection * CurationV1 * SpikeSortingSelection & key @@ -168,7 +169,9 @@ def make(self, key: dict): self.insert1(key, skip_duplicates=True) @classmethod - def get_labels(cls, curation_json): + def get_labels(cls, curation_json) -> Dict[int, List[str]]: + """Uses kachery cloud to load curation json. Returns labelsByUnit.""" + labels_by_unit = kcl.load_json(curation_json).get("labelsByUnit") return ( { @@ -180,7 +183,8 @@ def get_labels(cls, curation_json): ) @classmethod - def get_merge_groups(cls, curation_json): + def get_merge_groups(cls, curation_json) -> Dict: + """Uses kachery cloud to load curation json. Returns mergeGroups.""" return kcl.load_json(curation_json).get("mergeGroups", {}) diff --git a/src/spyglass/spikesorting/v1/metric_curation.py b/src/spyglass/spikesorting/v1/metric_curation.py index 6ef520947..b9d1fb66f 100644 --- a/src/spyglass/spikesorting/v1/metric_curation.py +++ b/src/spyglass/spikesorting/v1/metric_curation.py @@ -84,6 +84,7 @@ class WaveformParameters(SpyglassMixin, dj.Lookup): @classmethod def insert_default(cls): + """Insert default waveform parameters.""" cls.insert(cls.contents, skip_duplicates=True) @@ -129,10 +130,12 @@ class MetricParameters(SpyglassMixin, dj.Lookup): @classmethod def insert_default(cls): + """Insert default metric parameters.""" cls.insert(cls.contents, skip_duplicates=True) @classmethod def show_available_metrics(self): + """Prints the available metrics and their descriptions.""" for metric in _metric_name_to_func: metric_doc = _metric_name_to_func[metric].__doc__.split("\n")[0] logger.info(f"{metric} : {metric_doc}\n") @@ -155,6 +158,7 @@ class MetricCurationParameters(SpyglassMixin, dj.Lookup): @classmethod def insert_default(cls): + """Insert default metric curation parameters.""" cls.insert(cls.contents, skip_duplicates=True) @@ -206,12 +210,30 @@ class MetricCuration(SpyglassMixin, dj.Computed): _use_transaction, _allow_insert = False, True def make(self, key): + """Populate MetricCuration table. + + 1. Fetches... + - Waveform parameters from WaveformParameters + - Metric parameters from MetricParameters + - Label and merge parameters from MetricCurationParameters + - Sorting ID and curation ID from MetricCurationSelection + 2. Loads the recording and sorting from CurationV1. + 3. Optionally whitens the recording with spikeinterface + 4. Extracts waveforms from the recording based on the sorting. + 5. Optionally computes quality metrics for the units. + 6. Applies curation based on the metrics, computing labels and merge + groups. + 7. Saves the waveforms, metrics, labels, and merge groups to an + analysis NWB file and inserts into MetricCuration table. + """ + AnalysisNwbfile()._creation_times["pre_create_time"] = time() # FETCH nwb_file_name = ( SpikeSortingSelection * MetricCurationSelection & key ).fetch1("nwb_file_name") + # TODO: reduce fetch calls on same tables waveform_params = ( WaveformParameters * MetricCurationSelection & key ).fetch1("waveform_params") @@ -284,6 +306,7 @@ def make(self, key): @classmethod def get_waveforms(cls): + """Returns waveforms identified by metric curation. Not implemented.""" return NotImplementedError @classmethod diff --git a/src/spyglass/spikesorting/v1/recording.py b/src/spyglass/spikesorting/v1/recording.py index fd5214e40..f4e150837 100644 --- a/src/spyglass/spikesorting/v1/recording.py +++ b/src/spyglass/spikesorting/v1/recording.py @@ -118,6 +118,7 @@ class SpikeSortingPreprocessingParameters(SpyglassMixin, dj.Lookup): @classmethod def insert_default(cls): + """Insert default parameters.""" cls.insert(cls.contents, skip_duplicates=True) @@ -171,6 +172,16 @@ class SpikeSortingRecording(SpyglassMixin, dj.Computed): """ def make(self, key): + """Populate SpikeSortingRecording. + + 1. Get valid times for sort interval from IntervalList + 2. Use spikeinterface to preprocess recording + 3. Write processed recording to NWB file + 4. Insert resulting ... + - Interval to IntervalList + - NWB file to AnalysisNwbfile + - Recording ids to SpikeSortingRecording + """ AnalysisNwbfile()._creation_times["pre_create_time"] = time() # DO: # - get valid times for sort interval @@ -675,6 +686,7 @@ def __init__(self, timestamps, sampling_frequency, t_start, dtype): self._timeseries = timestamps def get_num_samples(self) -> int: + """Return the number of samples in the segment.""" return self._timeseries.shape[0] def get_traces( @@ -683,6 +695,7 @@ def get_traces( end_frame: Union[int, None] = None, channel_indices: Union[List, None] = None, ) -> np.ndarray: + """Return the traces for the segment for given start/end frames.""" return np.squeeze(self._timeseries[start_frame:end_frame]) diff --git a/src/spyglass/spikesorting/v1/sorting.py b/src/spyglass/spikesorting/v1/sorting.py index 47e8b6b68..06c7c6ede 100644 --- a/src/spyglass/spikesorting/v1/sorting.py +++ b/src/spyglass/spikesorting/v1/sorting.py @@ -96,13 +96,14 @@ class SpikeSorterParameters(SpyglassMixin, dj.Lookup): @classmethod def insert_default(cls): + """Insert default sorter parameters into SpikeSorterParameters table.""" cls.insert(cls.contents, skip_duplicates=True) @schema class SpikeSortingSelection(SpyglassMixin, dj.Manual): definition = """ - # Processed recording and spike sorting parameters. Use `insert_selection` method to insert rows. + # Processed recording and spike sorting parameters. See `insert_selection`. sorting_id: uuid --- -> SpikeSortingRecording diff --git a/src/spyglass/utils/database_settings.py b/src/spyglass/utils/database_settings.py index 1ad6efaa4..e7f36479e 100755 --- a/src/spyglass/utils/database_settings.py +++ b/src/spyglass/utils/database_settings.py @@ -185,6 +185,7 @@ def check_user_exists(self): ) def add_user_by_role(self, role, check_exists=False): + """Add a user to the database with the specified role""" add_func = { "guest": self.add_guest, "user": self.add_user, diff --git a/src/spyglass/utils/dj_graph.py b/src/spyglass/utils/dj_graph.py index 354b492ab..a9c3a7603 100644 --- a/src/spyglass/utils/dj_graph.py +++ b/src/spyglass/utils/dj_graph.py @@ -939,6 +939,7 @@ def has_link(self) -> bool: @property def path_str(self) -> str: + """Return string representation of path: parent -> {links} -> child.""" if not self.path: return "No link" return self._link_symbol.join([self._camel(t) for t in self.path]) @@ -981,6 +982,7 @@ def _get_find_restr(self, table) -> Tuple[str, Set[str]]: # ---------------------------- Graph Traversal ---------------------------- def cascade_search(self) -> None: + """Cascade restriction through graph to search for applicable table.""" if self.cascaded: return restriction, restr_attrs = self._get_find_restr(self.leaf) @@ -1033,6 +1035,7 @@ def cascade1_search( limit: int = 100, **kwargs, ): + """Search parents/children for a match of the provided restriction.""" if ( self.found_restr or not table @@ -1135,6 +1138,7 @@ def path(self) -> list: def cascade( self, restriction: str = None, direction: Direction = None, **kwargs ): + """Cascade restriction up or down the chain.""" if not self.has_link: return diff --git a/src/spyglass/utils/dj_helper_fn.py b/src/spyglass/utils/dj_helper_fn.py index caf6ea57c..889e64294 100644 --- a/src/spyglass/utils/dj_helper_fn.py +++ b/src/spyglass/utils/dj_helper_fn.py @@ -337,6 +337,7 @@ def _get_nwb_object(objects, object_id): def get_child_tables(table): + """Get all child tables of a given table.""" table = table() if inspect.isclass(table) else table return [ dj.FreeTable( @@ -530,8 +531,10 @@ def populate_pass_function(value): class NonDaemonPool(multiprocessing.pool.Pool): - """NonDaemonPool. Used to create a pool of non-daemonized processes, - which are required for parallel populate operations in DataJoint. + """Non-daemonized pool for multiprocessing. + + Used to create a pool of non-daemonized processes, which are required for + parallel populate operations in DataJoint. """ # Explicitly set the start method to 'fork' @@ -539,6 +542,7 @@ class NonDaemonPool(multiprocessing.pool.Pool): multiprocessing.set_start_method("fork", force=True) def Process(self, *args, **kwds): + """Return a non-daemonized process.""" proc = super(NonDaemonPool, self).Process(*args, **kwds) class NonDaemonProcess(proc.__class__): diff --git a/src/spyglass/utils/dj_merge_tables.py b/src/spyglass/utils/dj_merge_tables.py index 4ffbf38f4..f3c9e2d7b 100644 --- a/src/spyglass/utils/dj_merge_tables.py +++ b/src/spyglass/utils/dj_merge_tables.py @@ -684,6 +684,7 @@ def merge_get_parent( @property def source_class_dict(self) -> dict: + """Dictionary of part names and their respective classes.""" # NOTE: fails if table is aliased in dj.Part but not merge script # i.e., must import aliased table as part name if not self._source_class_dict: diff --git a/src/spyglass/utils/logging.py b/src/spyglass/utils/logging.py index 1771a160f..d0cdff63b 100644 --- a/src/spyglass/utils/logging.py +++ b/src/spyglass/utils/logging.py @@ -22,6 +22,7 @@ def excepthook(exc_type, exc_value, exc_traceback): + """Accommodate KeyboardInterrupt exception.""" if issubclass(exc_type, KeyboardInterrupt): sys.__excepthook__(exc_type, exc_value, exc_traceback) return diff --git a/src/spyglass/utils/nwb_helper_fn.py b/src/spyglass/utils/nwb_helper_fn.py index 641b3f2da..af25ec987 100644 --- a/src/spyglass/utils/nwb_helper_fn.py +++ b/src/spyglass/utils/nwb_helper_fn.py @@ -139,6 +139,7 @@ def get_config(nwb_file_path, calling_table=None): def close_nwb_files(): + """Close all open NWB files.""" for io, _ in __open_nwb_files.values(): io.close() __open_nwb_files.clear() @@ -548,6 +549,7 @@ def get_nwb_copy_filename(nwb_file_name): def change_group_permissions( subject_ids, set_group_name, analysis_dir="/stelmo/nwb/analysis" ): + """Change group permissions for specified subject ids in analysis dir.""" logger.warning("DEPRECATED: This function will be removed in `0.6`.") # Change to directory with analysis nwb files os.chdir(analysis_dir) diff --git a/src/spyglass/utils/position.py b/src/spyglass/utils/position.py index e3636bbe9..a66c54d8b 100644 --- a/src/spyglass/utils/position.py +++ b/src/spyglass/utils/position.py @@ -17,6 +17,7 @@ def convert_to_pixels(data, frame_size=None, cm_to_pixels=1.0): def fill_nan(variable, video_time, variable_time): + """Fill in missing values in variable with nans at video_time points.""" video_ind = np.digitize(variable_time, video_time[1:]) n_video_time = len(video_time) diff --git a/src/spyglass/utils/spikesorting.py b/src/spyglass/utils/spikesorting.py index 3343a62ff..5cafc8f5e 100644 --- a/src/spyglass/utils/spikesorting.py +++ b/src/spyglass/utils/spikesorting.py @@ -8,6 +8,7 @@ def firing_rate_from_spike_indicator( multiunit: bool = False, smoothing_sigma: float = 0.015, ): + """Calculate firing rate from spike indicator.""" if spike_indicator.ndim == 1: spike_indicator = spike_indicator[:, np.newaxis] From 54440973255d5e9e741439e056cf80f93ee6e366 Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Thu, 5 Sep 2024 13:59:12 -0500 Subject: [PATCH 43/94] Bump dj dependency to 0.14.2 (#1081) * Remove delete_downstream. Update tests * Revert pyproject * Update changelog * Fix tests * Bump CITATION.cff * Revert no-teardown --- CHANGELOG.md | 1 + CITATION.cff | 2 +- docs/src/Features/Mixin.md | 110 ++++++------ notebooks/01_Concepts.ipynb | 5 +- notebooks/py_scripts/01_Concepts.py | 5 +- pyproject.toml | 2 +- src/spyglass/utils/dj_merge_tables.py | 2 +- src/spyglass/utils/dj_mixin.py | 232 +++----------------------- tests/common/test_position.py | 6 - tests/conftest.py | 13 -- tests/container.py | 6 +- tests/position/test_trodes.py | 2 - tests/utils/test_merge.py | 6 +- tests/utils/test_mixin.py | 126 ++++++++------ 14 files changed, 166 insertions(+), 352 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bc6c50c7..eb98147e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - Disable populate transaction protection for long-populating tables #1066 - Add docstrings to all public methods #1076 +- Update DataJoint to 0.14.2 #1081 ### Pipelines diff --git a/CITATION.cff b/CITATION.cff index ed9dd3cc5..0af038cb5 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -166,5 +166,5 @@ keywords: - spike sorting - kachery license: MIT -version: 0.5.2 +version: 0.5.3 date-released: '2024-04-22' diff --git a/docs/src/Features/Mixin.md b/docs/src/Features/Mixin.md index bc02087ce..8ec47a2de 100644 --- a/docs/src/Features/Mixin.md +++ b/docs/src/Features/Mixin.md @@ -5,8 +5,9 @@ functionalities that have been added to DataJoint tables. This includes... - Fetching NWB files - Long-distance restrictions. -- Delete functionality, including permission checks and part/master pairs +- Permission checks on delete - Export logging. See [export doc](./Export.md) for more information. +- Miscellaneous helper functions To add this functionality to your own tables, simply inherit from the mixin: @@ -102,12 +103,7 @@ my_table << upstream_restriction >> downstream_restriction When providing a restriction of the parent, use 'up' direction. When providing a restriction of the child, use 'down' direction. -## Delete Functionality - -The mixin overrides the default `delete` function to provide two additional -features. - -### Permission Checks +## Delete Permission Checks By default, DataJoint is unable to set delete permissions on a per-table basis. If a user is able to delete entries in a given table, she can delete entries in @@ -127,66 +123,76 @@ curcumvent the default permission checks by adding themselves to the relevant team or removing the mixin from the class declaration. However, it provides a reasonable level of security for the average user. -### Master/Part Pairs +Because parts of this process rely on caching, this process will be faster if +you assign the instanced table to a variable. -By default, DataJoint has protections in place to prevent deletion of a part -entry without deleting the corresponding master. This is useful for enforcing -the custom of adding/removing all parts of a master at once and avoids orphaned -masters, or null entry masters without matching data. +```python +# Slower +YourTable().delete() +YourTable().delete() -For [Merge tables](./Merge.md), this is a significant problem. If a user wants -to delete all entries associated with a given session, she must find all part -table entries, including Merge tables, and delete them in the correct order. The -mixin provides a function, `delete_downstream_parts`, to handle this, which is -run by default when calling `delete`. +# Faster +nwbfile = YourTable() +nwbfile.delete() +nwbfile.delete() +``` -`delete_downstream_parts`, also aliased as `ddp`, identifies all part tables -with foreign key references downstream of where it is called. If `dry_run=True`, -it will return a list of entries that would be deleted, otherwise it will delete -them. +
Deprecated delete feature -Importantly, `delete_downstream_parts` cannot properly interact with tables that -have not been imported into the current namespace. If you are having trouble -with part deletion errors, import the offending table and rerun the function -with `reload_cache=True`. +Previous versions of Spyglass also deleted masters of parts with foreign key +references. This functionality has been migrated to DataJoint in version 0.14.2 +via the `force_masters` delete argument. This argument is `True` by default in +Spyglass tables. -```python -import datajoint as dj -from spyglass.common import Nwbfile +
-restricted_nwbfile = Nwbfile() & "nwb_file_name LIKE 'Name%'" +## Populate Calls -vanilla_dj_table = dj.FreeTable(dj.conn(), Nwbfile.full_table_name) -vanilla_dj_table.delete() -# DataJointError("Attempt to delete part table MyMerge.Part before ... ") +The mixin also overrides the default `populate` function to provide additional +functionality for non-daemon process pools and disabling transaction protection. -restricted_nwbfile.delete() -# [WARNING] Spyglass: No part deletes found w/ Nwbfile ... -# OR -# ValueError("Please import MyMerge and try again.") +### Non-Daemon Process Pools -from spyglass.example import MyMerge +To allow the `make` function to spawn a new process pool, the mixin overrides +the default `populate` function for tables with `_parallel_make` set to `True`. +See [issue #1000](https://github.com/LorenFrankLab/spyglass/issues/1000) and +[PR #1001](https://github.com/LorenFrankLab/spyglass/pull/1001) for more +information. -restricted_nwbfile.delete_downstream_parts(reload_cache=True, dry_run=False) -``` +### Disable Transaction Protection + +By default, DataJoint wraps the `populate` function in a transaction to ensure +data integrity (see +[Transactions](https://docs.datajoint.io/python/definition/05-Transactions.html)). -Because each table keeps a cache of downstream merge tables, it is important to -reload the cache if the table has been imported after the cache was created. -Speed gains can also be achieved by avoiding re-instancing the table each time. +This can cause issues when populating large tables if another user attempts to +declare/modify a table while the transaction is open (see +[issue #1030](https://github.com/LorenFrankLab/spyglass/issues/1030) and +[DataJoint issue #1170](https://github.com/datajoint/datajoint-python/issues/1170)). -```python -# Slow -from spyglass.common import Nwbfile +Tables with `_use_transaction` set to `False` will not be wrapped in a +transaction when calling `populate`. Transaction protection is replaced by a +hash of upstream data to ensure no changes are made to the table during the +unprotected populate. The additional time required to hash the data is a +trade-off for already time-consuming populates, but avoids blocking other users. -(Nwbfile() & "nwb_file_name LIKE 'Name%'").ddp(dry_run=False) -(Nwbfile() & "nwb_file_name LIKE 'Other%'").ddp(dry_run=False) +## Miscellaneous Helper functions -# Faster -from spyglass.common import Nwbfile +`file_like` allows you to restrict a table using a substring of a file name. +This is equivalent to the following: -nwbfile = Nwbfile() -(nwbfile & "nwb_file_name LIKE 'Name%'").ddp(dry_run=False) -(nwbfile & "nwb_file_name LIKE 'Other%'").ddp(dry_run=False) +```python +MyTable().file_like("eg") +MyTable() & ('nwb_file_name LIKE "%eg%" OR analysis_file_name LIKE "%eg%"') +``` + +`find_insert_fail` is a helper function to find the cause of an `IntegrityError` +when inserting into a table. This checks parent tables for required keys. + +```python +my_key = {"key": "value"} +MyTable().insert1(my_key) # Raises IntegrityError +MyTable().find_insert_fail(my_key) # Shows the parent(s) missing the key ``` ## Populate Calls diff --git a/notebooks/01_Concepts.ipynb b/notebooks/01_Concepts.ipynb index 2c3d535d1..fcb74632a 100644 --- a/notebooks/01_Concepts.ipynb +++ b/notebooks/01_Concepts.ipynb @@ -71,10 +71,7 @@ "```python\n", "my_key = dict(value=key) # whatever you're inserting\n", "MyTable.insert1(my_key) # error here\n", - "parents = MyTable.parents(as_objects=True) # get the parents as FreeTables\n", - "for parent in parents: # iterate through the parents, with only relevant fields\n", - " parent_key = {k: v for k, v in my_key.items() if k in parent.heading.names}\n", - " print(parent & parent_key) # restricted parent\n", + "parents = MyTable().find_insert_fail(my_key)\n", "```\n", "\n", "If any of the printed tables are empty, you know you need to insert into that\n", diff --git a/notebooks/py_scripts/01_Concepts.py b/notebooks/py_scripts/01_Concepts.py index f7f8ca190..70e5f6625 100644 --- a/notebooks/py_scripts/01_Concepts.py +++ b/notebooks/py_scripts/01_Concepts.py @@ -60,10 +60,7 @@ # ```python # my_key = dict(value=key) # whatever you're inserting # MyTable.insert1(my_key) # error here -# parents = MyTable.parents(as_objects=True) # get the parents as FreeTables -# for parent in parents: # iterate through the parents, with only relevant fields -# parent_key = {k: v for k, v in my_key.items() if k in parent.heading.names} -# print(parent & parent_key) # restricted parent +# parents = MyTable().find_insert_fail(my_key) # ``` # # If any of the printed tables are empty, you know you need to insert into that diff --git a/pyproject.toml b/pyproject.toml index 8db231c8d..e5f8dae5b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ dependencies = [ "black[jupyter]", "bottleneck", "dask", - "datajoint>=0.13.6", + "datajoint>=0.14.2", # "ghostipy", # removed from list bc M1 users need to install pyfftw first "hdmf>=3.4.6", "ipympl", diff --git a/src/spyglass/utils/dj_merge_tables.py b/src/spyglass/utils/dj_merge_tables.py index f3c9e2d7b..884e88f83 100644 --- a/src/spyglass/utils/dj_merge_tables.py +++ b/src/spyglass/utils/dj_merge_tables.py @@ -29,7 +29,7 @@ def is_merge_table(table): def trim_def(definition): return re_sub( r"\n\s*\n", "\n", re_sub(r"#.*\n", "\n", definition.strip()) - ) + ).replace(" ", "") if isinstance(table, str): table = dj.FreeTable(dj.conn(), table) diff --git a/src/spyglass/utils/dj_mixin.py b/src/spyglass/utils/dj_mixin.py index 6ce94fbf0..9ac836f5a 100644 --- a/src/spyglass/utils/dj_mixin.py +++ b/src/spyglass/utils/dj_mixin.py @@ -6,16 +6,14 @@ from os import environ from re import match as re_match from time import time -from typing import Dict, List, Union +from typing import List import datajoint as dj from datajoint.condition import make_condition from datajoint.errors import DataJointError from datajoint.expression import QueryExpression -from datajoint.logging import logger as dj_logger from datajoint.table import Table -from datajoint.utils import get_master, to_camel_case, user_choice -from networkx import NetworkXError +from datajoint.utils import to_camel_case from packaging.version import parse as version_parse from pandas import DataFrame from pymysql.err import DataError @@ -52,14 +50,6 @@ class SpyglassMixin: Fetch NWBFile object from relevant table. Uses either a foreign key to a NWBFile table (including AnalysisNwbfile) or a _nwb_table attribute to determine which table to use. - delte_downstream_merge(restriction=None, dry_run=True, reload_cache=False) - Delete downstream merge table entries associated with restriction. - Requires caching of merge tables and links, which is slow on first call. - `restriction` can be set to a string to restrict the delete. `dry_run` - can be set to False to commit the delete. `reload_cache` can be set to - True to reload the merge cache. - ddp(*args, **kwargs) - Alias for delete_downstream_parts cautious_delete(force_permission=False, *args, **kwargs) Check user permissions before deleting table rows. Permission is granted to users listed as admin in LabMember table or to users on a team with @@ -68,8 +58,6 @@ class SpyglassMixin: delete continues. If the Session has no experimenter, or if the user is not on a team with the Session experimenter(s), a PermissionError is raised. `force_permission` can be set to True to bypass permission check. - cdel(*args, **kwargs) - Alias for cautious_delete. """ # _nwb_table = None # NWBFile table class, defined at the table level @@ -134,15 +122,17 @@ def file_like(self, name=None, **kwargs): def find_insert_fail(self, key): """Find which parent table is causing an IntergrityError on insert.""" + rets = [] for parent in self.parents(as_objects=True): parent_key = { k: v for k, v in key.items() if k in parent.heading.names } parent_name = to_camel_case(parent.table_name) if query := parent & parent_key: - logger.info(f"{parent_name}:\n{query}") + rets.append(f"{parent_name}:\n{query}") else: - logger.info(f"{parent_name}: MISSING") + rets.append(f"{parent_name}: MISSING") + logger.info("\n".join(rets)) @classmethod def _safe_context(cls): @@ -298,163 +288,6 @@ def load_shared_schemas(self, additional_prefixes: list = None) -> None: for schema in schemas: dj.schema(schema[0]).connection.dependencies.load() - @cached_property - def _part_masters(self) -> set: - """Set of master tables downstream of self. - - Cache of masters in self.descendants(as_objects=True) with another - foreign key reference in the part. Used for delete_downstream_parts. - """ - self.connection.dependencies.load() - part_masters = set() - - def search_descendants(parent): - for desc_name in parent.descendants(): - if ( # Check if has master, is part - not (master := get_master(desc_name)) - or master in part_masters # already in cache - or desc_name.replace("`", "").split("_")[0] - not in SHARED_MODULES - ): - continue - desc = dj.FreeTable(self.connection, desc_name) - if not set(desc.parents()) - set([master]): # no other parent - continue - part_masters.add(master) - search_descendants(dj.FreeTable(self.connection, master)) - - try: - _ = search_descendants(self) - except NetworkXError: - try: # Attempt to import failing schema - self.load_shared_schemas() - _ = search_descendants(self) - except NetworkXError as e: - table_name = "".join(e.args[0].split("`")[1:4]) - raise ValueError(f"Please import {table_name} and try again.") - - logger.info( - f"Building part-parent cache for {self.camel_name}.\n\t" - + f"Found {len(part_masters)} downstream part tables" - ) - - return part_masters - - def _commit_downstream_delete(self, down_fts, start=None, **kwargs): - """ - Commit delete of downstream parts via down_fts. Logs with _log_delete. - - Used by both delete_downstream_parts and cautious_delete. - """ - start = start or time() - - safemode = ( - dj.config.get("safemode", True) - if kwargs.get("safemode") is None - else kwargs["safemode"] - ) - _ = kwargs.pop("safemode", None) - - ran_deletes = True - if down_fts: - for down_ft in down_fts: - dj_logger.info( - f"Spyglass: Deleting {len(down_ft)} rows from " - + f"{down_ft.full_table_name}" - ) - if ( - self._test_mode - or not safemode - or user_choice("Commit deletes?", default="no") == "yes" - ): - for down_ft in down_fts: # safemode off b/c already checked - down_ft.delete(safemode=False, **kwargs) - else: - logger.info("Delete aborted.") - ran_deletes = False - - self._log_delete(start, del_blob=down_fts if ran_deletes else None) - - return ran_deletes - - def delete_downstream_parts( - self, - restriction: str = None, - dry_run: bool = True, - reload_cache: bool = False, - disable_warning: bool = False, - return_graph: bool = False, - verbose: bool = False, - **kwargs, - ) -> List[dj.FreeTable]: - """Delete downstream merge table entries associated with restriction. - - Requires caching of merge tables and links, which is slow on first call. - - Parameters - ---------- - restriction : str, optional - Restriction to apply to merge tables. Default None. Will attempt to - use table restriction if None. - dry_run : bool, optional - If True, return list of merge part entries to be deleted. Default - True. - reload_cache : bool, optional - If True, reload merge cache. Default False. - disable_warning : bool, optional - If True, do not warn if no merge tables found. Default False. - return_graph: bool, optional - If True, return RestrGraph object used to identify downstream - tables. Default False, return list of part FreeTables. - True. If False, return dictionary of merge tables and their joins. - verbose : bool, optional - If True, call RestrGraph with verbose=True. Default False. - **kwargs : Any - Passed to datajoint.table.Table.delete. - """ - RestrGraph = self._graph_deps[1] - - start = time() - - if reload_cache: - _ = self.__dict__.pop("_part_masters", None) - - _ = self._part_masters # load cache before loading graph - restriction = restriction or self.restriction or True - - restr_graph = RestrGraph( - seed_table=self, - leaves={self.full_table_name: restriction}, - direction="down", - cascade=True, - verbose=verbose, - ) - - if return_graph: - return restr_graph - - down_fts = restr_graph.ft_from_list( - self._part_masters, sort_reverse=False - ) - - if not down_fts and not disable_warning: - logger.warning( - f"No part deletes found w/ {self.camel_name} & " - + f"{restriction}.\n\tIf this is unexpected, try importing " - + " Merge table(s) and running with `reload_cache`." - ) - - if dry_run: - return down_fts - - self._commit_downstream_delete(down_fts, start, **kwargs) - - def ddp( - self, *args, **kwargs - ) -> Union[List[QueryExpression], Dict[str, List[QueryExpression]]]: - """Alias for delete_downstream_parts.""" - return self.delete_downstream_parts(*args, **kwargs) - # ---------------------------- cautious_delete ---------------------------- @cached_property @@ -597,15 +430,10 @@ def _check_delete_permission(self) -> None: ) logger.info(f"Queueing delete for session(s):\n{sess_summary}") - @cached_property - def _cautious_del_tbl(self): - """Temporary inclusion for usage tracking.""" + def _log_delete(self, start, del_blob=None, super_delete=False): + """Log use of super_delete.""" from spyglass.common.common_usage import CautiousDelete - return CautiousDelete() - - def _log_delete(self, start, del_blob=None, super_delete=False): - """Log use of cautious_delete.""" safe_insert = dict( duration=time() - start, dj_user=dj.config["database.user"], @@ -614,7 +442,7 @@ def _log_delete(self, start, del_blob=None, super_delete=False): restr_str = "Super delete: " if super_delete else "" restr_str += "".join(self.restriction) if self.restriction else "None" try: - self._cautious_del_tbl.insert1( + CautiousDelete().insert1( dict( **safe_insert, restriction=restr_str[:255], @@ -622,11 +450,17 @@ def _log_delete(self, start, del_blob=None, super_delete=False): ) ) except (DataJointError, DataError): - self._cautious_del_tbl.insert1( - dict(**safe_insert, restriction="Unknown") - ) + CautiousDelete().insert1(dict(**safe_insert, restriction="Unknown")) + + @cached_property + def _has_updated_dj_version(self): + """Return True if DataJoint version is up to date.""" + target_dj = version_parse("0.14.2") + ret = version_parse(dj.__version__) >= target_dj + if not ret: + logger.warning(f"Please update DataJoint to {target_dj} or later.") + return ret - # TODO: Intercept datajoint delete confirmation prompt for merge deletes def cautious_delete( self, force_permission: bool = False, dry_run=False, *args, **kwargs ): @@ -638,10 +472,6 @@ def cautious_delete( continues. If the Session has no experimenter, or if the user is not on a team with the Session experimenter(s), a PermissionError is raised. - Potential downstream orphans are deleted first. These are master tables - whose parts have foreign keys to descendants of self. Then, rows from - self are deleted. Last, Nwbfile and IntervalList externals are deleted. - Parameters ---------- force_permission : bool, optional @@ -653,33 +483,25 @@ def cautious_delete( *args, **kwargs : Any Passed to datajoint.table.Table.delete. """ - start = time() - if len(self) == 0: logger.warning(f"Table is empty. No need to delete.\n{self}") return + if self._has_updated_dj_version: + kwargs["force_masters"] = True + external, IntervalList = self._delete_deps[3], self._delete_deps[4] if not force_permission or dry_run: self._check_delete_permission() - down_fts = self.delete_downstream_parts( - dry_run=True, - disable_warning=True, - ) - if dry_run: return ( - down_fts, IntervalList(), # cleanup func relies on downstream deletes external["raw"].unused(), external["analysis"].unused(), ) - if not self._commit_downstream_delete(down_fts, start=start, **kwargs): - return # Abort delete based on user input - super().delete(*args, **kwargs) # Confirmation here for ext_type in ["raw", "analysis"]: @@ -687,13 +509,8 @@ def cautious_delete( delete_external_files=True, display_progress=False ) - _ = IntervalList().nightly_cleanup(dry_run=False) - - self._log_delete(start=start, del_blob=down_fts) - - def cdel(self, *args, **kwargs): - """Alias for cautious_delete.""" - return self.cautious_delete(*args, **kwargs) + if not self._test_mode: + _ = IntervalList().nightly_cleanup(dry_run=False) def delete(self, *args, **kwargs): """Alias for cautious_delete, overwrites datajoint.table.Table.delete""" @@ -728,6 +545,7 @@ def _hash_upstream(self, keys): RestrGraph = self._graph_deps[1] if not (parents := self.parents(as_objects=True, primary=True)): + # Should not happen, as this is only called from populated tables raise RuntimeError("No upstream tables found for upstream hash.") leaves = { # Restriction on each primary parent diff --git a/tests/common/test_position.py b/tests/common/test_position.py index e5f39c20c..889dafa60 100644 --- a/tests/common/test_position.py +++ b/tests/common/test_position.py @@ -30,8 +30,6 @@ def param_table(common_position, default_param_key, teardown): param_table = common_position.PositionInfoParameters() param_table.insert1(default_param_key, skip_duplicates=True) yield param_table - if teardown: - param_table.delete(safemode=False) @pytest.fixture(scope="session") @@ -61,8 +59,6 @@ def upsample_position( ) common_position.IntervalPositionInfo.populate(interval_pos_key) yield interval_pos_key - if teardown: - (param_table & upsample_param_key).delete(safemode=False) @pytest.fixture(scope="session") @@ -101,8 +97,6 @@ def upsample_position_error( interval_pos_key, skip_duplicates=not teardown ) yield interval_pos_key - if teardown: - (param_table & upsample_param_key).delete(safemode=False) def test_interval_position_info_insert_error( diff --git a/tests/conftest.py b/tests/conftest.py index 8a9bc1a79..59016af15 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -465,8 +465,6 @@ def trodes_params(trodes_params_table, teardown): [v for k, v in paramsets.items()], skip_duplicates=True ) yield paramsets - if teardown: - trodes_params_table.delete(safemode=False) @pytest.fixture(scope="session") @@ -488,8 +486,6 @@ def trodes_sel_keys( ] trodes_sel_table.insert(keys, skip_duplicates=True) yield keys - if teardown: - trodes_sel_table.delete(safemode=False) @pytest.fixture(scope="session") @@ -497,8 +493,6 @@ def trodes_pos_v1(teardown, sgp, trodes_sel_keys): v1 = sgp.v1.TrodesPosV1() v1.populate(trodes_sel_keys) yield v1 - if teardown: - v1.delete(safemode=False) @pytest.fixture(scope="session") @@ -609,8 +603,6 @@ def track_graph(teardown, sgpl, track_graph_key): ) yield sgpl.TrackGraph & {"track_graph_name": "6 arm"} - if teardown: - sgpl.TrackGraph().delete(safemode=False) @pytest.fixture(scope="session") @@ -645,8 +637,6 @@ def lin_sel(teardown, sgpl, lin_sel_key): sel_table = sgpl.LinearizationSelection() sel_table.insert1(lin_sel_key, skip_duplicates=True) yield sel_table - if teardown: - sel_table.delete(safemode=False) @pytest.fixture(scope="session") @@ -654,8 +644,6 @@ def lin_v1(teardown, sgpl, lin_sel): v1 = sgpl.LinearizedPositionV1() v1.populate() yield v1 - if teardown: - v1.delete(safemode=False) @pytest.fixture(scope="session") @@ -888,7 +876,6 @@ def insert_project( yield project_key, cfg, config_path if teardown: - (dlc_project_tbl & project_key).delete(safemode=False) shutil_rmtree(str(Path(config_path).parent)) diff --git a/tests/container.py b/tests/container.py index 1747d76b8..fb960dc07 100644 --- a/tests/container.py +++ b/tests/container.py @@ -215,9 +215,9 @@ def stop(self, remove=True) -> None: return container_name = self.container_name - self.container.stop() - self.logger.info(f"Container {container_name} stopped.") + self.container.stop() # Logger I/O operations close during teardown + print(f"Container {container_name} stopped.") if remove: self.container.remove() - self.logger.info(f"Container {container_name} removed.") + print(f"Container {container_name} removed.") diff --git a/tests/position/test_trodes.py b/tests/position/test_trodes.py index 6d65f375c..2608d70c6 100644 --- a/tests/position/test_trodes.py +++ b/tests/position/test_trodes.py @@ -30,8 +30,6 @@ def sel_table(teardown, params_table, trodes_sel_table, pos_interval_key): edit_name=new_name, ) yield trodes_sel_table & restr_dict - if teardown: - (trodes_sel_table & restr_dict).delete(safemode=False) def test_sel_default(sel_table): diff --git a/tests/utils/test_merge.py b/tests/utils/test_merge.py index 2876555a1..fc225cf21 100644 --- a/tests/utils/test_merge.py +++ b/tests/utils/test_merge.py @@ -49,9 +49,9 @@ class NonMerge(SpyglassMixin, dj.Manual): yield NonMerge -def test_non_merge(NonMerge): - with pytest.raises(AttributeError): - NonMerge() +def test_non_merge(schema_test, NonMerge): + with pytest.raises(TypeError): + schema_test(NonMerge) def test_part_camel(merge_table): diff --git a/tests/utils/test_mixin.py b/tests/utils/test_mixin.py index a35041013..2d71f7ff1 100644 --- a/tests/utils/test_mixin.py +++ b/tests/utils/test_mixin.py @@ -39,6 +39,11 @@ def test_auto_increment(schema_test, Mixin): assert ret["id"] == 2, "Auto increment not working." +def test_good_file_like(common): + common.Session().file_like("min") + assert len(common.Session()) > 0, "file_like not working." + + def test_null_file_like(schema_test, Mixin): schema_test(Mixin) ret = Mixin().file_like(None) @@ -52,73 +57,84 @@ def test_bad_file_like(caplog, schema_test, Mixin): assert "No file_like field" in caplog.text, "No warning issued." -def test_partmaster_detect(Nwbfile, pos_merge_tables): - """Test that the mixin can detect merge children of merge.""" - assert len(Nwbfile._part_masters) >= 14, "Part masters not detected." - - -def test_downstream_restrict( - Nwbfile, frequent_imports, pos_merge_tables, lin_v1, lfp_merge_key -): - """Test that the mixin can join merge chains.""" - - _ = frequent_imports # graph for cascade - _ = lin_v1, lfp_merge_key # merge tables populated +@pytest.mark.skipif(not VERBOSE, reason="No logging to test when quiet-spy.") +def test_insert_fail(caplog, common, mini_dict): + this_key = dict(mini_dict, interval_list_name="BadName") + common.PositionSource().find_insert_fail(this_key) + assert "IntervalList: MISSING" in caplog.text, "No warning issued." - restr_ddp = Nwbfile.ddp(dry_run=True, reload_cache=True) - end_len = [len(ft) for ft in restr_ddp] - assert sum(end_len) >= 8, "Downstream parts not restricted correctly." +def test_exp_summary(Nwbfile): + fields = Nwbfile._get_exp_summary().heading.names + expected = ["nwb_file_name", "lab_member_name"] + assert fields == expected, "Exp summary fields not as expected." -def test_get_downstream_merge(Nwbfile, pos_merge_tables): - """Test that the mixin can get the chain of a merge.""" - lin_output = pos_merge_tables[1].full_table_name - assert lin_output in Nwbfile._part_masters, "Merge not found." +def test_exp_summary_no_link(schema_test, Mixin): + schema_test(Mixin) + assert Mixin()._get_exp_summary() is None, "Exp summary not None." -@pytest.mark.skipif(not VERBOSE, reason="No logging to test when quiet-spy.") -def test_ddp_warning(Nwbfile, caplog): - """Test that the mixin warns on empty delete_downstream_merge.""" - (Nwbfile.file_like("BadName")).delete_downstream_parts( - reload_cache=True, disable_warnings=False - ) - assert "No part deletes found" in caplog.text, "No warning issued." +def test_exp_summary_auto_link(common): + lab_member = common.LabMember() + summary_names = lab_member._get_exp_summary().heading.names + join_names = (lab_member * common.Session.Experimenter).heading.names + assert summary_names == join_names, "Auto link not working." -def test_ddp_dry_run( - Nwbfile, frequent_imports, common, sgp, pos_merge_tables, lin_v1 -): - """Test that the mixin can dry run delete_downstream_merge.""" - _ = lin_v1 # merge tables populated - _ = frequent_imports # graph for cascade +def test_cautious_del_dry_run(Nwbfile, frequent_imports): + _ = frequent_imports # part of cascade, need import + ret = Nwbfile.cautious_delete(dry_run=True)[1].full_table_name + assert ( + ret == "`common_nwbfile`.`~external_raw`" + ), "Dry run delete not working." - pos_output_name = pos_merge_tables[0].full_table_name - param_field = "trodes_pos_params_name" - trodes_params = sgp.v1.TrodesPosParams() +@pytest.mark.skipif(not VERBOSE, reason="No logging to test when quiet-spy.") +def test_empty_cautious_del(caplog, schema_test, Mixin): + schema_test(Mixin) + Mixin().cautious_delete(safemode=False) + Mixin().cautious_delete(safemode=False) + assert "empty" in caplog.text, "No warning issued." - rft = [ - table - for table in (trodes_params & f'{param_field} LIKE "%ups%"').ddp( - reload_cache=True, dry_run=True - ) - if table.full_table_name == pos_output_name - ] - assert len(rft) == 1, "ddp did not return restricted table." +def test_super_delete(schema_test, Mixin, common): + schema_test(Mixin) + Mixin().insert1((0,), skip_duplicates=True) + Mixin().super_delete(safemode=False) + assert len(Mixin()) == 0, "Super delete not working." + + logged_dels = common.common_usage.CautiousDelete & 'restriction LIKE "Sup%"' + assert len(logged_dels) > 0, "Super delete not logged." + + +def test_compare_versions(common): + # Does nothing in test_mode + compare_func = common.Nwbfile().compare_versions + compare_func("0.1.0", "0.1.1") + + +@pytest.fixture +def custom_table(): + """Custom table on user prefix for testing load_shared_schemas.""" + db, table = dj.config["database.user"] + "_test", "custom" + dj.conn().query(f"CREATE DATABASE IF NOT EXISTS {db};") + dj.conn().query(f"USE {db};") + dj.conn().query( + f"CREATE TABLE IF NOT EXISTS {table} ( " + + "`merge_id` binary(16) NOT NULL COMMENT ':uuid:', " + + "`unit_id` int NOT NULL, " + + "PRIMARY KEY (`merge_id`), " + + "CONSTRAINT `unit_annotation_ibfk_1` FOREIGN KEY (`merge_id`) " + + "REFERENCES `spikesorting_merge`.`spike_sorting_output` (`merge_id`) " + + "ON DELETE RESTRICT ON UPDATE CASCADE);" + ) + yield f"`{db}`.`{table}`" -def test_exp_summary(Nwbfile): - fields = Nwbfile._get_exp_summary().heading.names - expected = ["nwb_file_name", "lab_member_name"] - assert fields == expected, "Exp summary fields not as expected." +def test_load_shared_schemas(common, custom_table): + # from spyglass.common import Nwbfile -def test_cautious_del_dry_run(Nwbfile, frequent_imports): - _ = frequent_imports # part of cascade, need import - ret = Nwbfile.cdel(dry_run=True) - part_master_names = [t.full_table_name for t in ret[0]] - part_masters = Nwbfile._part_masters - assert all( - [pm in part_masters for pm in part_master_names] - ), "Non part masters found in cautious delete dry run." + common.Nwbfile().load_shared_schemas(additional_prefixes=["test"]) + nodes = common.Nwbfile().connection.dependencies.nodes + assert custom_table in nodes, "Custom table not loaded." From b9e363f7c5ad10a8df1f6d74f3ea4a136b81db56 Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Mon, 9 Sep 2024 16:12:10 -0500 Subject: [PATCH 44/94] Fix #1094 (#1096) * Fix #1094 * Update changelog --- CHANGELOG.md | 5 +++++ src/spyglass/spikesorting/utils.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb98147e8..546f19fd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,8 +15,13 @@ ### Pipelines - Decoding + - Fix edge case errors in spike time loading #1083 +- Spike Sorting + + - Fix bug in `get_group_by_shank` #1096 + ## [0.5.3] (August 27, 2024) ### Infrastructure diff --git a/src/spyglass/spikesorting/utils.py b/src/spyglass/spikesorting/utils.py index bb1c516c2..d99a71ff2 100644 --- a/src/spyglass/spikesorting/utils.py +++ b/src/spyglass/spikesorting/utils.py @@ -123,7 +123,7 @@ def get_group_by_shank( ) continue - sg_keys.append(sg_key) + sg_keys.append(sg_key.copy()) sge_keys.extend( [ { From 991fc7637e03085be8ce265e370a9748607d9f73 Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Tue, 10 Sep 2024 12:50:02 -0500 Subject: [PATCH 45/94] Fix 1080 (#1099) * Fix 1080 * Revert debug print --- CHANGELOG.md | 1 + src/spyglass/spikesorting/v0/spikesorting_curation.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 546f19fd3..02e5e5fb0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ - Spike Sorting - Fix bug in `get_group_by_shank` #1096 + - Fix bug in `_compute_metric` #1099 ## [0.5.3] (August 27, 2024) diff --git a/src/spyglass/spikesorting/v0/spikesorting_curation.py b/src/spyglass/spikesorting/v0/spikesorting_curation.py index 52a2dff73..33469d67c 100644 --- a/src/spyglass/spikesorting/v0/spikesorting_curation.py +++ b/src/spyglass/spikesorting/v0/spikesorting_curation.py @@ -581,7 +581,7 @@ def _compute_metric(self, waveform_extractor, metric_name, **metric_params): peak_sign_metrics = ["snr", "peak_offset", "peak_channel"] if metric_name == "isi_violation": - metric = metric_func(waveform_extractor, **metric_params) + return metric_func(waveform_extractor, **metric_params) elif metric_name in peak_sign_metrics: if "peak_sign" not in metric_params: raise Exception( From d9115f6b34a61c040f9a6454864a58462fce70c3 Mon Sep 17 00:00:00 2001 From: Eric Denovellis Date: Tue, 10 Sep 2024 12:12:41 -0700 Subject: [PATCH 46/94] Add edge map parameter to track linearization (#1091) * Add edge map parameter * Update CHANGELOG.md * Fix alter tables --------- Co-authored-by: Chris Broz --- CHANGELOG.md | 7 +++++++ src/spyglass/linearization/v1/main.py | 2 ++ 2 files changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02e5e5fb0..f4fc91759 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ +```python +from spyglass.linearization.v1.main import TrackGraph +TrackGraph.alter() # Comment regarding the change +``` + ### Infrastructure - Disable populate transaction protection for long-populating tables #1066 @@ -17,6 +22,8 @@ - Decoding - Fix edge case errors in spike time loading #1083 +- Linearization + - Add edge_map parameter to LinearizedPositionV1 #1091 - Spike Sorting diff --git a/src/spyglass/linearization/v1/main.py b/src/spyglass/linearization/v1/main.py index 76ec85aaf..ff6d51733 100644 --- a/src/spyglass/linearization/v1/main.py +++ b/src/spyglass/linearization/v1/main.py @@ -49,6 +49,7 @@ class TrackGraph(SpyglassMixin, dj.Manual): edges: blob # shape (n_edges, 2) linear_edge_order : blob # order of edges in linear space, (n_edges, 2) linear_edge_spacing : blob # space btwn edges in linear space, (n_edges,) + edge_map = NULL : blob # Maps one edge to another before linearization """ def get_networkx_track_graph(self, track_graph_parameters=None): @@ -58,6 +59,7 @@ def get_networkx_track_graph(self, track_graph_parameters=None): return make_track_graph( node_positions=track_graph_parameters["node_positions"], edges=track_graph_parameters["edges"], + edge_map=track_graph_parameters["edge_map"], ) def plot_track_graph(self, ax=None, draw_edge_labels=False, **kwds): From a7b13fa1636f0b219fc7d640e25f8269f41ac93e Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Wed, 11 Sep 2024 11:16:52 -0500 Subject: [PATCH 47/94] Fix #1098 (#1100) * Fix #1098 * Update changelog --- CHANGELOG.md | 4 +++ .../position/v1/position_dlc_centroid.py | 9 ++++--- .../position/v1/position_dlc_orient.py | 2 +- .../v1/position_dlc_pose_estimation.py | 25 +++++++++++-------- src/spyglass/utils/dj_mixin.py | 2 +- 5 files changed, 27 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4fc91759..81fee573f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,10 @@ TrackGraph.alter() # Comment regarding the change - Linearization - Add edge_map parameter to LinearizedPositionV1 #1091 +- Position + + - Restore #973, allow DLC without position tracking #1100 + - Spike Sorting - Fix bug in `get_group_by_shank` #1096 diff --git a/src/spyglass/position/v1/position_dlc_centroid.py b/src/spyglass/position/v1/position_dlc_centroid.py index 04815ae53..c1353cc9b 100644 --- a/src/spyglass/position/v1/position_dlc_centroid.py +++ b/src/spyglass/position/v1/position_dlc_centroid.py @@ -254,19 +254,22 @@ def _logged_make(self, key): total_nan = np.sum(final_df.loc[:, idx[("x", "y")]].isna().any(axis=1)) logger.info(f"total NaNs in centroid dataset: {total_nan}") - spatial_series = (RawPosition() & key).fetch_nwb()[0]["raw_position"] position = pynwb.behavior.Position() velocity = pynwb.behavior.BehavioralTimeSeries() + if query := (RawPosition() & key): + spatial_series = query.fetch_nwb()[0]["raw_position"] + else: + spatial_series = None common_attrs = { "conversion": METERS_PER_CM, - "comments": spatial_series.comments, + "comments": getattr(spatial_series, "comments", ""), } position.create_spatial_series( name="position", timestamps=final_df.index.to_numpy(), data=final_df.loc[:, idx[("x", "y")]].to_numpy(), - reference_frame=spatial_series.reference_frame, + reference_frame=getattr(spatial_series, "reference_frame", ""), description="x_position, y_position", **common_attrs, ) diff --git a/src/spyglass/position/v1/position_dlc_orient.py b/src/spyglass/position/v1/position_dlc_orient.py index 118ddffc8..0f93f8264 100644 --- a/src/spyglass/position/v1/position_dlc_orient.py +++ b/src/spyglass/position/v1/position_dlc_orient.py @@ -15,7 +15,7 @@ red_led_bisector_orientation, two_pt_head_orientation, ) -from spyglass.utils import SpyglassMixin, logger +from spyglass.utils import SpyglassMixin from .position_dlc_cohort import DLCSmoothInterpCohort diff --git a/src/spyglass/position/v1/position_dlc_pose_estimation.py b/src/spyglass/position/v1/position_dlc_pose_estimation.py index 2ff376837..b5be422a3 100644 --- a/src/spyglass/position/v1/position_dlc_pose_estimation.py +++ b/src/spyglass/position/v1/position_dlc_pose_estimation.py @@ -258,17 +258,20 @@ def _logged_make(self, key): populate_missing=False, ) ) - spatial_series = ( - RawPosition() & {**key, "interval_list_name": interval_list_name} - ).fetch_nwb()[0]["raw_position"] - _, _, _, video_time = get_video_info(key) - pos_time = spatial_series.timestamps + if interval_list_name: + spatial_series = ( + RawPosition() + & {**key, "interval_list_name": interval_list_name} + ).fetch_nwb()[0]["raw_position"] + else: + spatial_series = None + + _, _, meters_per_pixel, video_time = get_video_info(key) + key["meters_per_pixel"] = meters_per_pixel # TODO: should get timestamps from VideoFile, but need the # video_frame_ind from RawPosition, which also has timestamps - key["meters_per_pixel"] = spatial_series.conversion - # Insert entry into DLCPoseEstimation logger.info( "Inserting %s, epoch %02d into DLCPoseEsimation", @@ -296,7 +299,9 @@ def _logged_make(self, key): part_df = convert_to_cm(part_df, meters_per_pixel) logger.info("adding timestamps to DataFrame") part_df = add_timestamps( - part_df, pos_time=pos_time, video_time=video_time + part_df, + pos_time=getattr(spatial_series, "timestamps", video_time), + video_time=video_time, ) key["bodypart"] = body_part key["analysis_file_name"] = AnalysisNwbfile().create( @@ -309,8 +314,8 @@ def _logged_make(self, key): timestamps=part_df.time.to_numpy(), conversion=METERS_PER_CM, data=part_df.loc[:, idx[("x", "y")]].to_numpy(), - reference_frame=spatial_series.reference_frame, - comments=spatial_series.comments, + reference_frame=getattr(spatial_series, "reference_frame", ""), + comments=getattr(spatial_series, "comments", "no comments"), description="x_position, y_position", ) likelihood.create_timeseries( diff --git a/src/spyglass/utils/dj_mixin.py b/src/spyglass/utils/dj_mixin.py index 9ac836f5a..723fec869 100644 --- a/src/spyglass/utils/dj_mixin.py +++ b/src/spyglass/utils/dj_mixin.py @@ -614,7 +614,7 @@ def populate(self, *restrictions, **kwargs): for key in keys: self.make(key) if upstream_hash != self._hash_upstream(keys): - (self & keys).delete(force=True) + (self & keys).delete(safemode=False) logger.error( "Upstream tables changed during non-transaction " + "populate. Please try again." From 6c8a566f9ad6a56b24af4a44b3eaa3a75bcbe8cc Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Thu, 12 Sep 2024 13:13:32 -0500 Subject: [PATCH 48/94] Fix #1101 (#1103) --- CHANGELOG.md | 6 +++++- src/spyglass/position/v1/position_dlc_model.py | 7 ++++++- src/spyglass/position/v1/position_dlc_pose_estimation.py | 5 ++--- tests/conftest.py | 7 +++++-- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81fee573f..167039575 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,8 @@ ```python from spyglass.linearization.v1.main import TrackGraph -TrackGraph.alter() # Comment regarding the change + +TrackGraph.alter() # Comment regarding the change ``` ### Infrastructure @@ -22,11 +23,14 @@ TrackGraph.alter() # Comment regarding the change - Decoding - Fix edge case errors in spike time loading #1083 + - Linearization + - Add edge_map parameter to LinearizedPositionV1 #1091 - Position + - Fix video directory bug in `DLCPoseEstimationSelection` #1103 - Restore #973, allow DLC without position tracking #1100 - Spike Sorting diff --git a/src/spyglass/position/v1/position_dlc_model.py b/src/spyglass/position/v1/position_dlc_model.py index ce9310d5b..243d51a69 100644 --- a/src/spyglass/position/v1/position_dlc_model.py +++ b/src/spyglass/position/v1/position_dlc_model.py @@ -96,7 +96,12 @@ def insert_entry( table_query = dj.FreeTable( dj.conn(), full_table_name=part_table.parents()[-1] ) & {"project_name": project_name} - project_path = table_query.fetch1("project_path") + + if cls._test_mode: # temporary fix for #1105 + project_path = table_query.fetch(limit=1)[0] + else: + project_path = table_query.fetch1("project_path") + part_table.insert1( { "dlc_model_name": dlc_model_name, diff --git a/src/spyglass/position/v1/position_dlc_pose_estimation.py b/src/spyglass/position/v1/position_dlc_pose_estimation.py index b5be422a3..69a325a4f 100644 --- a/src/spyglass/position/v1/position_dlc_pose_estimation.py +++ b/src/spyglass/position/v1/position_dlc_pose_estimation.py @@ -122,9 +122,8 @@ def _insert_est_with_log( if not v_path: raise FileNotFoundError(f"Video file not found for {key}") logger.info("Pose Estimation Selection") - v_dir = Path(v_path).parent - logger.info("video_dir: %s", v_dir) - v_path = find_mp4(video_path=v_dir, video_filename=v_fname) + logger.info(f"video_dir: {v_path}") + v_path = find_mp4(video_path=Path(v_path), video_filename=v_fname) if check_crop: params["cropping"] = self.get_video_crop( video_path=v_path.as_posix() diff --git a/tests/conftest.py b/tests/conftest.py index 59016af15..adb3ccc08 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -982,13 +982,16 @@ def populate_training( if len(train_tbl & model_train_key) == 0: _ = add_training_files DOWNLOADS.move_dlc_items(labeled_vid_dir) - sgp.v1.DLCModelTraining.populate(model_train_key) + sgp.v1.DLCModelTraining().populate(model_train_key) yield model_train_key @pytest.fixture(scope="session") def model_source_key(sgp, model_train_key, populate_training): - yield (sgp.v1.DLCModelSource & model_train_key).fetch1("KEY") + + _ = populate_training + + yield (sgp.v1.DLCModelSource & model_train_key).fetch("KEY")[0] @pytest.fixture(scope="session") From 39c55dba5904ee62cb470af94a0227b668169474 Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Fri, 13 Sep 2024 13:20:16 -0500 Subject: [PATCH 49/94] Remove SessionGroup (#1106) * Remove SessionGroup * Remove SessionGroup doct --- CHANGELOG.md | 8 +- docs/mkdocs.yml | 1 - docs/src/Features/SessionGroups.md | 26 ---- docs/src/Features/index.md | 1 - examples/cli_examples/create_session_group.py | 11 -- examples/cli_examples/readme.md | 7 -- src/spyglass/cli/cli.py | 11 -- src/spyglass/common/__init__.py | 2 +- src/spyglass/common/common_session.py | 119 +----------------- 9 files changed, 9 insertions(+), 177 deletions(-) delete mode 100644 docs/src/Features/SessionGroups.md delete mode 100755 examples/cli_examples/create_session_group.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 167039575..7e0711630 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,11 @@ ```python +import datajoint as dj from spyglass.linearization.v1.main import TrackGraph -TrackGraph.alter() # Comment regarding the change +TrackGraph.alter() # Add edge map parameter +dj.FreeTable(dj.conn(), "common_session.session_group").drop() ``` ### Infrastructure @@ -20,6 +22,10 @@ TrackGraph.alter() # Comment regarding the change ### Pipelines +- Common + + - Drop `SessionGroup` table #1106 + - Decoding - Fix edge case errors in spike time loading #1083 diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 30d2bd79d..3cda1b344 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -74,7 +74,6 @@ nav: - FigURL: Features/FigURL.md - Merge Tables: Features/Merge.md - Export: Features/Export.md - - Session Groups: Features/SessionGroups.md - Centralized Code: Features/Mixin.md - For Developers: - Overview: ForDevelopers/index.md diff --git a/docs/src/Features/SessionGroups.md b/docs/src/Features/SessionGroups.md deleted file mode 100644 index e339f871b..000000000 --- a/docs/src/Features/SessionGroups.md +++ /dev/null @@ -1,26 +0,0 @@ -# Session groups - -A session group is a collection of sessions. Each group has a name (primary key) -and a description. - -```python -from spyglass.common import SessionGroup - -# Create a new session group -SessionGroup.add_group("test_group_1", "Description of test group 1") - -# Get the table of session groups -SessionGroup() - -# Add a session to the group -SessionGroup.add_session_to_group("RN2_20191110_.nwb", "test_group_1") - -# Remove a session from a group -# SessionGroup.remove_session_from_group('RN2_20191110_.nwb', 'test_group_1') - -# Get all sessions in group -SessionGroup.get_group_sessions("test_group_1") - -# Update the description of a session group -SessionGroup.update_session_group_description("test_group_1", "Test description") -``` diff --git a/docs/src/Features/index.md b/docs/src/Features/index.md index e8399f84a..59122a8c4 100644 --- a/docs/src/Features/index.md +++ b/docs/src/Features/index.md @@ -9,4 +9,3 @@ Spyglass. - [Mixin](./Mixin.md) - Spyglass-specific functionalities to DataJoint tables, including fetching NWB files, long-distance restrictions, and permission checks on delete operations. -- [Session Groups](./SessionGroups.md) - How to operate on sets of sessions. diff --git a/examples/cli_examples/create_session_group.py b/examples/cli_examples/create_session_group.py deleted file mode 100755 index 7218a9174..000000000 --- a/examples/cli_examples/create_session_group.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python3 - -import spyglass.common as sgc - -nwb_file_name = "RN2_20191110_.nwb" - -sgc.SessionGroup.add_group("group1", "Group1", skip_duplicates=True) -sgc.SessionGroup.add_session_to_group( - nwb_file_name, "group1", skip_duplicates=True -) -print(sgc.SessionGroup.get_group_sessions("group1")) diff --git a/examples/cli_examples/readme.md b/examples/cli_examples/readme.md index 3919a1cbf..4730555a2 100644 --- a/examples/cli_examples/readme.md +++ b/examples/cli_examples/readme.md @@ -109,13 +109,6 @@ spyglass insert-spike-sorter-parameters parameters.yaml spyglass list-spike-sorter-parameters ``` -## Create a session group - -A session group is a collection of sessions that can be viewed via spyglassview. - -See [session_groups.md](../../docs/session_groups.md) and -[create_session_group.py](./create_session_group.py). - ## Create spyglass view ```bash diff --git a/src/spyglass/cli/cli.py b/src/spyglass/cli/cli.py index 37871d490..ec953dd8c 100644 --- a/src/spyglass/cli/cli.py +++ b/src/spyglass/cli/cli.py @@ -447,16 +447,6 @@ def list_spike_sortings(nwb_file_name: str): print(results) -@click.command(help="Create spyglass view of a session group.") -@click.argument("session_group_name") -def create_spyglass_view(session_group_name: str): - import spyglass.common as sgc - - F = sgc.SessionGroup.create_spyglass_view(session_group_name) - url = F.url(label=session_group_name) - print(url) - - cli.add_command(insert_session) cli.add_command(list_sessions) cli.add_command(insert_lab_team) @@ -480,4 +470,3 @@ def create_spyglass_view(session_group_name: str): cli.add_command(list_spike_sorter_parameters) cli.add_command(run_spike_sorting) cli.add_command(list_spike_sortings) -cli.add_command(create_spyglass_view) diff --git a/src/spyglass/common/__init__.py b/src/spyglass/common/__init__.py index a91ceb3f3..270acdf3a 100644 --- a/src/spyglass/common/__init__.py +++ b/src/spyglass/common/__init__.py @@ -56,7 +56,7 @@ ) from spyglass.common.common_region import BrainRegion from spyglass.common.common_sensors import SensorData -from spyglass.common.common_session import Session, SessionGroup +from spyglass.common.common_session import Session from spyglass.common.common_subject import Subject from spyglass.common.common_task import Task, TaskEpoch from spyglass.common.populate_all_common import populate_all_common diff --git a/src/spyglass/common/common_session.py b/src/spyglass/common/common_session.py index a069e9f38..5095ed6ed 100644 --- a/src/spyglass/common/common_session.py +++ b/src/spyglass/common/common_session.py @@ -8,7 +8,7 @@ from spyglass.common.common_lab import Institution, Lab, LabMember from spyglass.common.common_nwbfile import Nwbfile from spyglass.common.common_subject import Subject -from spyglass.settings import config, debug_mode, test_mode +from spyglass.settings import debug_mode from spyglass.utils import SpyglassMixin, logger from spyglass.utils.nwb_helper_fn import get_config, get_nwb_file @@ -190,120 +190,3 @@ def _add_experimenter_part( key["nwb_file_name"] = nwb_file_name key["lab_member_name"] = name Session.Experimenter.insert1(key) - - -@schema -class SessionGroup(SpyglassMixin, dj.Manual): - definition = """ - session_group_name: varchar(200) - --- - session_group_description: varchar(2000) - """ - - @staticmethod - def add_group( - session_group_name: str, - session_group_description: str, - *, - skip_duplicates: bool = False, - ): - """Add a new session group.""" - SessionGroup.insert1( - { - "session_group_name": session_group_name, - "session_group_description": session_group_description, - }, - skip_duplicates=skip_duplicates, - ) - - @staticmethod - def update_session_group_description( - session_group_name: str, session_group_description - ): - """Update the description of a session group.""" - SessionGroup.update1( - { - "session_group_name": session_group_name, - "session_group_description": session_group_description, - } - ) - - @staticmethod - def add_session_to_group( - nwb_file_name: str, - session_group_name: str, - *, - skip_duplicates: bool = False, - ): - """Add a session to an existing session group.""" - if test_mode: - skip_duplicates = True - SessionGroupSession.insert1( - { - "session_group_name": session_group_name, - "nwb_file_name": nwb_file_name, - }, - skip_duplicates=skip_duplicates, - ) - - @staticmethod - def remove_session_from_group( - nwb_file_name: str, session_group_name: str, *args, **kwargs - ): - """Remove a session from a session group.""" - query = { - "session_group_name": session_group_name, - "nwb_file_name": nwb_file_name, - } - (SessionGroupSession & query).delete( - force_permission=test_mode, *args, **kwargs - ) - - @staticmethod - def delete_group(session_group_name: str, *args, **kwargs): - """Delete a session group.""" - query = {"session_group_name": session_group_name} - (SessionGroup & query).delete( - force_permission=test_mode, *args, **kwargs - ) - - @staticmethod - def get_group_sessions(session_group_name: str): - """Get the NWB file names of all sessions in a session group.""" - results = ( - SessionGroupSession & {"session_group_name": session_group_name} - ).fetch(as_dict=True) - return [ - {"nwb_file_name": result["nwb_file_name"]} for result in results - ] - - @staticmethod - def create_spyglass_view(session_group_name: str): - """Create a FigURL view for a session group.""" - import figurl as fig - - FIGURL_CHANNEL = config.get("FIGURL_CHANNEL") - if not FIGURL_CHANNEL: - raise ValueError("FIGURL_CHANNEL config/env variable not set") - - return fig.Figure( - view_url="gs://figurl/spyglassview-1", - data={ - "type": "spyglassview", - "sessionGroupName": session_group_name, - }, - ) - - -# The reason this is not implemented as a dj.Part is that -# datajoint prohibits deleting from a subtable without -# also deleting the parent table. -# See: https://docs.datajoint.org/python/computation/03-master-part.html - - -@schema -class SessionGroupSession(SpyglassMixin, dj.Manual): - definition = """ - -> SessionGroup - -> Session - """ From 975ad2576eddffb6b26414de83700c15d681605d Mon Sep 17 00:00:00 2001 From: Eric Denovellis Date: Fri, 13 Sep 2024 13:13:28 -0700 Subject: [PATCH 50/94] Hotfix: Passing track graph parameters --- src/spyglass/linearization/v1/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/spyglass/linearization/v1/main.py b/src/spyglass/linearization/v1/main.py index ff6d51733..dfc6f7a6e 100644 --- a/src/spyglass/linearization/v1/main.py +++ b/src/spyglass/linearization/v1/main.py @@ -154,10 +154,9 @@ def make(self, key): TrackGraph() & {"track_graph_name": key["track_graph_name"]} ).fetch1() - track_graph = make_track_graph( - node_positions=track_graph_info["node_positions"], - edges=track_graph_info["edges"], - ) + track_graph = ( + TrackGraph & {"track_graph_name": key["track_graph_name"]} + ).get_networkx_track_graph() linear_position_df = get_linearized_position( position=position, @@ -170,6 +169,7 @@ def make(self, key): ], sensor_std_dev=linearization_parameters["sensor_std_dev"], diagonal_bias=linearization_parameters["diagonal_bias"], + edge_map=track_graph_info["edge_map"], ) linear_position_df["time"] = time From c5ee8622a1be88dc42704da9e00623dcc0f0e5d0 Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Fri, 13 Sep 2024 15:16:05 -0500 Subject: [PATCH 51/94] Adjust no-transact pop hashing mechanism (#1108) * Adjust no-transact pop hashing mechanism * Update changelog --- CHANGELOG.md | 3 +- src/spyglass/utils/dj_graph.py | 4 +- tests/common/test_session.py | 80 ----------------------------- tests/spikesorting/conftest.py | 4 +- tests/spikesorting/test_curation.py | 9 ++-- 5 files changed, 12 insertions(+), 88 deletions(-) delete mode 100644 tests/common/test_session.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e0711630..607b78c5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,8 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() ### Infrastructure -- Disable populate transaction protection for long-populating tables #1066 +- Disable populate transaction protection for long-populating tables #1066, + #1108 - Add docstrings to all public methods #1076 - Update DataJoint to 0.14.2 #1081 diff --git a/src/spyglass/utils/dj_graph.py b/src/spyglass/utils/dj_graph.py index a9c3a7603..6269ecaba 100644 --- a/src/spyglass/utils/dj_graph.py +++ b/src/spyglass/utils/dj_graph.py @@ -14,6 +14,7 @@ from datajoint import FreeTable, Table from datajoint.condition import make_condition from datajoint.dependencies import unite_master_parts +from datajoint.hash import key_hash from datajoint.user_tables import TableMeta from datajoint.utils import get_master, to_camel_case from networkx import ( @@ -601,7 +602,8 @@ def hash(self): """Return hash of all visited nodes.""" initial = hash_md5(b"") for table in self.all_ft: - initial.update(table.fetch()) + for row in table.fetch(as_dict=True): + initial.update(key_hash(row).encode("utf-8")) return initial.hexdigest() # ------------------------------- Add Nodes ------------------------------- diff --git a/tests/common/test_session.py b/tests/common/test_session.py deleted file mode 100644 index 2276f23bd..000000000 --- a/tests/common/test_session.py +++ /dev/null @@ -1,80 +0,0 @@ -import pytest - - -@pytest.fixture -def common_session(common): - return common.common_session - - -@pytest.fixture -def group_name_dict(): - return {"session_group_name": "group1"} - - -@pytest.fixture -def add_session_group(common_session, group_name_dict): - session_group = common_session.SessionGroup() - session_group_dict = { - **group_name_dict, - "session_group_description": "group1 description", - } - session_group.add_group(**session_group_dict, skip_duplicates=True) - session_group_dict["session_group_description"] = "updated description" - session_group.update_session_group_description(**session_group_dict) - yield session_group, session_group_dict - - -@pytest.fixture -def session_group(add_session_group): - yield add_session_group[0] - - -@pytest.fixture -def session_group_dict(add_session_group): - yield add_session_group[1] - - -def test_session_group_add(session_group, session_group_dict): - assert session_group & session_group_dict, "Session group not added" - - -@pytest.fixture -def add_session_to_group(session_group, mini_copy_name, group_name_dict): - session_group.add_session_to_group( - nwb_file_name=mini_copy_name, **group_name_dict - ) - - -def test_add_remove_session_group( - common_session, - session_group, - session_group_dict, - group_name_dict, - mini_copy_name, - add_session_to_group, - add_session_group, -): - assert session_group & session_group_dict, "Session not added to group" - - session_group.remove_session_from_group( - nwb_file_name=mini_copy_name, - safemode=False, - **group_name_dict, - ) - assert ( - len(common_session.SessionGroupSession & session_group_dict) == 0 - ), "SessionGroupSession not removed from by helper function" - - -def test_get_group_sessions( - session_group, group_name_dict, add_session_to_group -): - ret = session_group.get_group_sessions(**group_name_dict) - assert len(ret) == 1, "Incorrect number of sessions returned" - - -def test_delete_group_error(session_group, group_name_dict): - session_group.delete_group(**group_name_dict, safemode=False) - assert ( - len(session_group & group_name_dict) == 0 - ), "Group not deleted by helper function" diff --git a/tests/spikesorting/conftest.py b/tests/spikesorting/conftest.py index d287d35f4..b60c57b16 100644 --- a/tests/spikesorting/conftest.py +++ b/tests/spikesorting/conftest.py @@ -76,7 +76,9 @@ def pop_curation(spike_v1, pop_sort): description="testing sort", ) - yield spike_v1.CurationV1().fetch("KEY", as_dict=True)[0] + yield (spike_v1.CurationV1() & {"parent_curation_id": -1}).fetch( + "KEY", as_dict=True + )[0] @pytest.fixture(scope="session") diff --git a/tests/spikesorting/test_curation.py b/tests/spikesorting/test_curation.py index 43df0fed5..4048e9297 100644 --- a/tests/spikesorting/test_curation.py +++ b/tests/spikesorting/test_curation.py @@ -37,15 +37,14 @@ def test_curation_sort(spike_v1, pop_curation): ), "CurationV1.get_sorting unexpected shape" -def test_curation_sort_info(spike_v1, pop_curation): +def test_curation_sort_info(spike_v1, pop_curation, pop_curation_metric): sort_info = spike_v1.CurationV1.get_sort_group_info(pop_curation) + sort_metric = spike_v1.CurationV1.get_sort_group_info(pop_curation_metric) + assert ( hash_sort_info(sort_info) == "be874e806a482ed2677fd0d0b449f965" ), "CurationV1.get_sort_group_info unexpected value" - -def test_curation_metric(spike_v1, pop_curation_metric): - sort_info = spike_v1.CurationV1.get_sort_group_info(pop_curation_metric) assert ( - hash_sort_info(sort_info) == "48e437bc116900fe64e492d74595b56d" + hash_sort_info(sort_metric) == "48e437bc116900fe64e492d74595b56d" ), "CurationV1.get_sort_group_info unexpected value" From a711874296ec09632215ec89c37959bae968a861 Mon Sep 17 00:00:00 2001 From: Samuel Bray Date: Mon, 16 Sep 2024 11:43:07 -0700 Subject: [PATCH 52/94] Add objects to analysis file before insert (#1112) * add objects to analysis file before insert * update changelog --- CHANGELOG.md | 1 + src/spyglass/position/v1/position_dlc_centroid.py | 13 +++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 607b78c5b..d75fe4313 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Fix video directory bug in `DLCPoseEstimationSelection` #1103 - Restore #973, allow DLC without position tracking #1100 + - Minor fix to `DLCCentroid` make function order #1112 - Spike Sorting diff --git a/src/spyglass/position/v1/position_dlc_centroid.py b/src/spyglass/position/v1/position_dlc_centroid.py index c1353cc9b..2e66397c3 100644 --- a/src/spyglass/position/v1/position_dlc_centroid.py +++ b/src/spyglass/position/v1/position_dlc_centroid.py @@ -295,6 +295,13 @@ def _logged_make(self, key): # Add to Analysis NWB file analysis_file_name = AnalysisNwbfile().create(key["nwb_file_name"]) nwb_analysis_file = AnalysisNwbfile() + key["dlc_position_object_id"] = nwb_analysis_file.add_nwb_object( + nwb_analysis_file.add_nwb_object(analysis_file_name, position) + ) + key["dlc_velocity_object_id"] = nwb_analysis_file.add_nwb_object( + nwb_analysis_file.add_nwb_object(analysis_file_name, velocity) + ) + nwb_analysis_file.add( nwb_file_name=key["nwb_file_name"], analysis_file_name=analysis_file_name, @@ -304,12 +311,6 @@ def _logged_make(self, key): { **key, "analysis_file_name": analysis_file_name, - "dlc_position_object_id": nwb_analysis_file.add_nwb_object( - analysis_file_name, position - ), - "dlc_velocity_object_id": nwb_analysis_file.add_nwb_object( - analysis_file_name, velocity - ), } ) From 1480c2418d49b9691f6a4d08547341c990cce3c1 Mon Sep 17 00:00:00 2001 From: Samuel Bray Date: Mon, 16 Sep 2024 12:15:09 -0700 Subject: [PATCH 53/94] Apply restrictions on parent tables in fetch_nwb (#1086) * include upstream restriction in fetch_nwb * update changelog * lint * resolve case of string restrictions * move extract_merge_id to class method * Apply suggestions from code review Co-authored-by: Chris Broz * fix lint --------- Co-authored-by: Chris Broz --- CHANGELOG.md | 1 + src/spyglass/utils/dj_merge_tables.py | 68 +++++++++++++++++++++++---- 2 files changed, 61 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d75fe4313..8f6ced3dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() #1108 - Add docstrings to all public methods #1076 - Update DataJoint to 0.14.2 #1081 +- Allow restriction based on parent keys in `Merge.fetch_nwb()` #1086 ### Pipelines diff --git a/src/spyglass/utils/dj_merge_tables.py b/src/spyglass/utils/dj_merge_tables.py index 884e88f83..b2ddbff93 100644 --- a/src/spyglass/utils/dj_merge_tables.py +++ b/src/spyglass/utils/dj_merge_tables.py @@ -3,7 +3,7 @@ from pprint import pprint from re import sub as re_sub from time import time -from typing import Union +from typing import Union, List import datajoint as dj from datajoint.condition import make_condition @@ -532,17 +532,22 @@ def fetch_nwb( if isinstance(self, dict): raise ValueError("Try replacing Merge.method with Merge().method") restriction = restriction or self.restriction or True - sources = set((self & restriction).fetch(self._reserved_sk)) + merge_restriction = self.extract_merge_id(restriction) + sources = set((self & merge_restriction).fetch(self._reserved_sk)) nwb_list = [] merge_ids = [] for source in sources: source_restr = ( - self & {self._reserved_sk: source} & restriction + self & {self._reserved_sk: source} & merge_restriction ).fetch("KEY") nwb_list.extend( - self.merge_restrict_class( - source_restr, permit_multiple_rows=True - ).fetch_nwb() + (self & source_restr) + .merge_restrict_class( + restriction, + permit_multiple_rows=True, + add_invalid_restrict=False, + ) + .fetch_nwb() ) if return_merge_ids: merge_ids.extend([k[self._reserved_pk] for k in source_restr]) @@ -738,10 +743,15 @@ def merge_get_parent_class(self, source: str) -> dj.Table: return ret def merge_restrict_class( - self, key: dict, permit_multiple_rows: bool = False + self, + key: dict, + permit_multiple_rows: bool = False, + add_invalid_restrict=True, ) -> dj.Table: """Returns native parent class, restricted with key.""" - parent = self.merge_get_parent(key) + parent = self.merge_get_parent( + key, add_invalid_restrict=add_invalid_restrict + ) parent_key = parent.fetch("KEY", as_dict=True) if not permit_multiple_rows and len(parent_key) > 1: @@ -834,6 +844,48 @@ def super_delete(self, warn=True, *args, **kwargs): self._log_delete(start=time(), super_delete=True) super().delete(*args, **kwargs) + @classmethod + def extract_merge_id(cls, restriction) -> Union[dict, list]: + """Utility function to extract merge_id from a restriction + + Removes all other restricted attributes, and defaults to a + universal set (either empty dict or True) when there is no + merge_id present in the input, relying on parent func to + restrict on secondary or part-parent key(s). + + Parameters + ---------- + restriction : str, dict, or dj.condition.AndList + A datajoint restriction + + Returns + ------- + restriction + A restriction containing only the merge_id key + """ + if restriction is None: + return None + if isinstance(restriction, dict): + if merge_id := restriction.get("merge_id"): + return {"merge_id": merge_id} + else: + return {} + merge_restr = [] + if isinstance(restriction, dj.condition.AndList) or isinstance( + restriction, List + ): + merge_id_list = [cls.extract_merge_id(r) for r in restriction] + merge_restr = [x for x in merge_id_list if x is not None] + elif isinstance(restriction, str): + parsed = [x.split(")")[0] for x in restriction.split("(") if x] + merge_restr = dj.condition.AndList( + [x for x in parsed if "merge_id" in x] + ) + + if len(merge_restr) == 0: + return True + return merge_restr + _Merge = Merge From b68aea002703e9e74dbb24cd662d254acac6bded Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Tue, 17 Sep 2024 10:07:08 -0500 Subject: [PATCH 54/94] Revise datajoint import name (#1116) * Revise datajoint import name * Update changelog --- CHANGELOG.md | 1 + src/spyglass/utils/dj_graph.py | 8 ++++++-- tests/conftest.py | 2 ++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f6ced3dd..f3e1139bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Add docstrings to all public methods #1076 - Update DataJoint to 0.14.2 #1081 - Allow restriction based on parent keys in `Merge.fetch_nwb()` #1086 +- Import `datajoint.dependencies.unite_master_parts` -> `topo_sort` #1116 ### Pipelines diff --git a/src/spyglass/utils/dj_graph.py b/src/spyglass/utils/dj_graph.py index 6269ecaba..26a944e20 100644 --- a/src/spyglass/utils/dj_graph.py +++ b/src/spyglass/utils/dj_graph.py @@ -13,7 +13,6 @@ from datajoint import FreeTable, Table from datajoint.condition import make_condition -from datajoint.dependencies import unite_master_parts from datajoint.hash import key_hash from datajoint.user_tables import TableMeta from datajoint.utils import get_master, to_camel_case @@ -35,6 +34,11 @@ unique_dicts, ) +try: # Datajoint 0.14.2+ uses topo_sort instead of unite_master_parts + from datajoint.dependencies import topo_sort as dj_topo_sort +except ImportError: + from datajoint.dependencies import unite_master_parts as dj_topo_sort + class Direction(Enum): """Cascade direction enum. Calling Up returns True. Inverting flips.""" @@ -474,7 +478,7 @@ def _topo_sort( if not self._is_out(node, warn=False) ] graph = self.graph.subgraph(nodes) if subgraph else self.graph - ordered = unite_master_parts(list(topological_sort(graph))) + ordered = dj_topo_sort(list(topological_sort(graph))) if reverse: ordered.reverse() return [n for n in ordered if n in nodes] diff --git a/tests/conftest.py b/tests/conftest.py index adb3ccc08..954dad204 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -394,6 +394,7 @@ def frequent_imports(): from spyglass.lfp.analysis.v1 import LFPBandSelection from spyglass.mua.v1.mua import MuaEventsV1 from spyglass.ripple.v1.ripple import RippleTimesV1 + from spyglass.spikesorting.analysis.v1.unit_annotation import UnitAnnotation from spyglass.spikesorting.v0.figurl_views import SpikeSortingRecordingView return ( @@ -403,6 +404,7 @@ def frequent_imports(): RippleTimesV1, SortedSpikesIndicatorSelection, SpikeSortingRecordingView, + UnitAnnotation, UnitMarksIndicatorSelection, ) From 8be2f56247dd23647282ad43ca41d984192163d2 Mon Sep 17 00:00:00 2001 From: Samuel Bray Date: Wed, 18 Sep 2024 07:52:08 -0700 Subject: [PATCH 55/94] Fix Curation primary key creation from spikesorting key (#1114) * fix primary key creation * update changelog --- CHANGELOG.md | 1 + src/spyglass/spikesorting/v0/spikesorting_curation.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3e1139bb..fbf2fa72c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Fix bug in `get_group_by_shank` #1096 - Fix bug in `_compute_metric` #1099 + - Fix bug in `insert_curation` returned key #1114 ## [0.5.3] (August 27, 2024) diff --git a/src/spyglass/spikesorting/v0/spikesorting_curation.py b/src/spyglass/spikesorting/v0/spikesorting_curation.py index 33469d67c..e8c2f149e 100644 --- a/src/spyglass/spikesorting/v0/spikesorting_curation.py +++ b/src/spyglass/spikesorting/v0/spikesorting_curation.py @@ -146,8 +146,9 @@ def insert_curation( Curation.insert1(sorting_key, skip_duplicates=True) # get the primary key for this curation - c_key = (Curation & sorting_key).fetch1("KEY") - curation_key = {item: sorting_key[item] for item in c_key} + curation_key = { + item: sorting_key[item] for item in Curation.primary_key + } return curation_key From c82f23edfbcf3e9fd7f587e1436918166bd02b14 Mon Sep 17 00:00:00 2001 From: Samuel Bray Date: Wed, 18 Sep 2024 08:07:21 -0700 Subject: [PATCH 56/94] NWB ingestion fixes (#1074) * check nwb probe geometry agains existing probe entry on insertion * allow insertion of new probe type from config * fix shank and electrode dicts * update docs for config probe insert * add option to add tasks from config during ingestion * avoid default use of last interval_list_name for task * update docs for config TaskEpoch * cleanup read of probe data from config * improve query efficiency * Apply suggestions from code review Co-authored-by: Chris Broz * suggestions from code review * spelling fix --------- Co-authored-by: Chris Broz --- docs/src/ForDevelopers/UsingNWB.md | 22 +++--- src/spyglass/common/common_device.py | 101 ++++++++++++++++++++++++++- src/spyglass/common/common_task.py | 92 ++++++++++++++++++++---- 3 files changed, 190 insertions(+), 25 deletions(-) diff --git a/docs/src/ForDevelopers/UsingNWB.md b/docs/src/ForDevelopers/UsingNWB.md index 3f68f930e..f64870930 100644 --- a/docs/src/ForDevelopers/UsingNWB.md +++ b/docs/src/ForDevelopers/UsingNWB.md @@ -144,9 +144,9 @@ ndx_franklab_novela.CameraDevice | Spyglass Table | Key | NWBfile Location | Config option | Notes | | :------------- | :---------------: | ----------------------------------------: | -----------------------------------------: | ----: | | Probe | probe_type | nwbf.devices.\<\*Probe>.probe_type | config\["Probe"\]\[index\]\["probe_type"\] | str | -| Probe | probe_id | nwbf.devices.\<\*Probe>.probe_type | XXX | str | -| Probe | manufacturer | nwbf.devices.\<\*Probe>.manufacturer | XXX | str | -| Probe | probe_description | nwbf.devices.\<\*Probe>.probe_description | XXX | str | +| Probe | probe_id | nwbf.devices.\<\*Probe>.probe_type | config\["Probe"\]\[index\]\["probe_type"\] | str | +| Probe | manufacturer | nwbf.devices.\<\*Probe>.manufacturer | config\["Probe"\]\[index\]\["manufacturer"\] | str | +| Probe | probe_description | nwbf.devices.\<\*Probe>.probe_description | config\["Probe"\]\[index\]\["description"\] | str | | Probe | num_shanks | nwbf.devices.\<\*Probe>.num_shanks | XXX | int | NWBfile Location: nwbf.devices.\<\*Probe>.\<\*Shank>
Object type: @@ -154,16 +154,18 @@ ndx_franklab_novela.Shank | Spyglass Table | Key | NWBfile Location | Config option | Notes | | :------------- | :---------: | ---------------------------------------------: | ------------: | ----: | -| Probe.Shank | probe_shank | nwbf.devices.\<\*Probe>.\<\*Shank>.probe_shank | XXX | int | +| Probe.Shank | probe_shank | nwbf.devices.\<\*Probe>.\<\*Shank>.probe_shank | config\["Probe"\]\[Shank\]\ | int | In the config, a list of ints | NWBfile Location: nwbf.devices.\<\*Probe>.\<\*Shank>.\<\*Electrode>
Object type: ndx_franklab_novela.Electrode | Spyglass Table | Key | NWBfile Location | Config option | Notes | | :-------------- | :----------: | -------------------------------------------------------------: | ------------: | ----: | -| Probe.Electrode | probe_shank | nwbf.devices.\<\*Probe>.\<\*Shank>.probe_shank | XXX | int | -| Probe.Electrode | contact_size | nwbf.devices.\<\*Probe>.\<\*Shank>.\<\*Electrode>.contact_size | XXX | float | -| Probe.Electrode | rel_x | nwbf.devices.\<\*Probe>.\<\*Shank>.\<\*Electrode>.rel_x | XXX | float | +| Probe.Electrode | probe_shank | nwbf.devices.\<\*Probe>.\<\*Shank>.probe_shank | config\["Probe"]\["Electrode"]\[index]\["probe_shank"] | int | +| Probe.Electrode | contact_size | nwbf.devices.\<\*Probe>.\<\*Shank>.\<\*Electrode>.contact_size | config\["Probe"]\["Electrode"]\[index]\["contact_size"] | float | +| Probe.Electrode | rel_x | nwbf.devices.\<\*Probe>.\<\*Shank>.\<\*Electrode>.rel_x | config\["Probe"]\["Electrode"]\[index]\["rel_x"] | float | +| Probe.Electrode | rel_y | nwbf.devices.\<\*Probe>.\<\*Shank>.\<\*Electrode>.rel_y | config\["Probe"]\["Electrode"]\[index]\["rel_y"] | float | +| Probe.Electrode | rel_z | nwbf.devices.\<\*Probe>.\<\*Shank>.\<\*Electrode>.rel_z | config\["Probe"]\["Electrode"]\[index]\["rel_z"] | float | NWBfile Location: nwbf.epochs
Object type: pynwb.epoch.TimeIntervals @@ -213,9 +215,9 @@ hdmf.common.table.DynamicTable | :------------- | :--------------: | -----------------------------------------------: | ------------: | ----: | | Task | task_name | nwbf.processing.tasks.\[index\].name | | | | Task | task_description | nwbf.processing.\[index\].tasks.description | | | -| TaskEpoch | task_name | nwbf.processing.\[index\].tasks.name | | | -| TaskEpoch | camera_names | nwbf.processing.\[index\].tasks.camera_id | | | -| TaskEpoch | task_environment | nwbf.processing.\[index\].tasks.task_environment | | | +| TaskEpoch | task_name | nwbf.processing.\[index\].tasks.name | config\["Tasks"\]\[index\]\["task_name"\]| | +| TaskEpoch | camera_names | nwbf.processing.\[index\].tasks.camera_id | config\["Tasks"\]\[index\]\["camera_id"\] | | +| TaskEpoch | task_environment | nwbf.processing.\[index\].tasks.task_environment | config\["Tasks"\]\[index\]\["task_environment"\] | | NWBfile Location: nwbf.units
Object type: pynwb.misc.Units diff --git a/src/spyglass/common/common_device.py b/src/spyglass/common/common_device.py index 19ab7ff2a..1995c2303 100644 --- a/src/spyglass/common/common_device.py +++ b/src/spyglass/common/common_device.py @@ -376,7 +376,9 @@ def insert_from_nwbfile(cls, nwbf, config=None): List of probe device types found in the NWB file. """ config = config or dict() - all_probes_types, ndx_probes, _ = cls.get_all_probe_names(nwbf, config) + all_probes_types, ndx_probes, config_probes = cls.get_all_probe_names( + nwbf, config + ) for probe_type in all_probes_types: new_probe_type_dict = dict() @@ -397,6 +399,16 @@ def insert_from_nwbfile(cls, nwbf, config=None): elect_dict, ) + elif probe_type in config_probes: + cls._read_config_probe_data( + config, + probe_type, + new_probe_type_dict, + new_probe_dict, + shank_dict, + elect_dict, + ) + # check that number of shanks is consistent num_shanks = new_probe_type_dict["num_shanks"] assert num_shanks == 0 or num_shanks == len( @@ -405,8 +417,6 @@ def insert_from_nwbfile(cls, nwbf, config=None): # if probe id already exists, do not overwrite anything or create # new Shanks and Electrodes - # TODO: test whether the Shanks and Electrodes in the NWB file match - # the ones in the database query = Probe & {"probe_id": new_probe_dict["probe_id"]} if len(query) > 0: logger.info( @@ -414,6 +424,31 @@ def insert_from_nwbfile(cls, nwbf, config=None): " the database. Spyglass will use that and not create a new" " Probe, Shanks, or Electrodes." ) + # Test whether the Shanks and Electrodes in the NWB file match + # the existing database entries + existing_shanks = query * cls.Shank() + bad_shanks = [ + shank + for shank in shank_dict.values() + if len(existing_shanks & shank) != 1 + ] + if bad_shanks: + raise ValueError( + "Mismatch between nwb file and existing database " + + f"entry for shanks: {bad_shanks}" + ) + + existing_electrodes = query * cls.Electrode() + bad_electrodes = [ + electrode + for electrode in elect_dict.values() + if len(existing_electrodes & electrode) != 1 + ] + if bad_electrodes: + raise ValueError( + f"Mismatch between nwb file and existing database " + f"entry for electrodes: {bad_electrodes}" + ) continue cls.insert1(new_probe_dict, skip_duplicates=True) @@ -523,6 +558,66 @@ def __read_ndx_probe_data( "rel_z": electrode.rel_z, } + @classmethod + def _read_config_probe_data( + cls, + config, + probe_type, + new_probe_type_dict, + new_probe_dict, + shank_dict, + elect_dict, + ): + + # get the list of shank keys for the probe + shank_list = config["Probe"][config_probes.index(probe_type)].get( + "Shank", [] + ) + for i in shank_list: + shank_dict[str(i)] = {"probe_id": probe_type, "probe_shank": int(i)} + + # get the list of electrode keys for the probe + elect_dict_list = config["Probe"][config_probes.index(probe_type)].get( + "Electrode", [] + ) + for i, e in enumerate(elect_dict_list): + elect_dict[str(i)] = { + "probe_id": probe_type, + "probe_shank": e["probe_shank"], + "probe_electrode": e["probe_electrode"], + "contact_size": e.get("contact_size"), + "rel_x": e.get("rel_x"), + "rel_y": e.get("rel_y"), + "rel_z": e.get("rel_z"), + } + + # make the probe type if not in database + new_probe_type_dict.update( + { + "manufacturer": config["Probe"][ + config_probes.index(probe_type) + ].get("manufacturer"), + "probe_type": probe_type, + "probe_description": config["Probe"][ + config_probes.index(probe_type) + ].get("probe_description"), + "num_shanks": len(shank_list), + } + ) + + cls._add_probe_type(new_probe_type_dict) + + # make the probe dictionary + new_probe_dict.update( + { + "probe_type": probe_type, + "probe_id": probe_type, + "contact_side_numbering": config["Probe"][ + config_probes.index(probe_type) + ].get("contact_side_numbering"), + } + ) + @classmethod def _add_probe_type(cls, new_probe_type_dict): """Check the probe type value against the values in the database. diff --git a/src/spyglass/common/common_task.py b/src/spyglass/common/common_task.py index b4f87eb97..9b8368192 100644 --- a/src/spyglass/common/common_task.py +++ b/src/spyglass/common/common_task.py @@ -7,7 +7,7 @@ from spyglass.common.common_nwbfile import Nwbfile from spyglass.common.common_session import Session # noqa: F401 from spyglass.utils import SpyglassMixin, logger -from spyglass.utils.nwb_helper_fn import get_nwb_file +from spyglass.utils.nwb_helper_fn import get_config, get_nwb_file schema = dj.schema("common_task") @@ -106,6 +106,7 @@ def make(self, key): nwb_file_name = key["nwb_file_name"] nwb_file_abspath = Nwbfile().get_abs_path(nwb_file_name) nwbf = get_nwb_file(nwb_file_abspath) + config = get_config(nwb_file_abspath, calling_table=self.camel_name) camera_names = dict() # the tasks refer to the camera_id which is unique for the NWB file but @@ -117,13 +118,27 @@ def make(self, key): # get the camera ID camera_id = int(str.split(device.name)[1]) camera_names[camera_id] = device.camera_name + if device_list := config.get("CameraDevice"): + for device in device_list: + camera_names.update( + { + name: id + for name, id in zip( + device.get("camera_name"), + device.get("camera_id", -1), + ) + } + ) # find the task modules and for each one, add the task to the Task # schema if it isn't there and then add an entry for each epoch tasks_mod = nwbf.processing.get("tasks") - if tasks_mod is None: - logger.warn(f"No tasks processing module found in {nwbf}\n") + config_tasks = config.get("Tasks") + if tasks_mod is None and config_tasks is None: + logger.warn( + f"No tasks processing module found in {nwbf} or config\n" + ) return task_inserts = [] @@ -166,19 +181,72 @@ def make(self, key): for epoch in task.task_epochs[0]: # TODO in beans file, task_epochs[0] is 1x2 dset of ints, # so epoch would be an int - key["epoch"] = epoch - target_interval = str(epoch).zfill(2) - for interval in session_intervals: - if ( - target_interval in interval - ): # TODO this is not true for the beans file - break - # TODO case when interval is not found is not handled - key["interval_list_name"] = interval + target_interval = self.get_epoch_interval_name( + epoch, session_intervals + ) + if target_interval is None: + logger.warn("Skipping epoch.") + continue + key["interval_list_name"] = target_interval task_inserts.append(key.copy()) + + # Add tasks from config + for task in config_tasks: + new_key = { + **key, + "task_name": task.get("task_name"), + "task_environment": task.get("task_environment", None), + } + # add cameras + camera_ids = task.get("camera_id", []) + valid_camera_ids = [ + camera_id + for camera_id in camera_ids + if camera_id in camera_names.keys() + ] + if valid_camera_ids: + new_key["camera_names"] = [ + {"camera_name": camera_names[camera_id]} + for camera_id in valid_camera_ids + ] + session_intervals = ( + IntervalList() & {"nwb_file_name": nwb_file_name} + ).fetch("interval_list_name") + for epoch in task.get("task_epochs", []): + new_key["epoch"] = epoch + target_interval = self.get_epoch_interval_name( + epoch, session_intervals + ) + if target_interval is None: + logger.warn("Skipping epoch.") + continue + new_key["interval_list_name"] = target_interval + task_inserts.append(key.copy()) + self.insert(task_inserts, allow_direct_insert=True) + @classmethod + def get_epoch_interval_name(cls, epoch, session_intervals): + """Get the interval name for a given epoch based on matching number""" + target_interval = str(epoch).zfill(2) + possible_targets = [ + interval + for interval in session_intervals + if target_interval in interval + ] + if not possible_targets: + logger.warn( + f"Interval not found for epoch {epoch} in {nwb_file_name}." + ) + elif len(possible_targets) > 1: + logger.warn( + f"Multiple intervals found for epoch {epoch} in {nwb_file_name}. " + + f"matches are {possible_targets}." + ) + else: + return possible_targets[0] + @classmethod def update_entries(cls, restrict=True): """Update entries in the TaskEpoch table based on a restriction.""" From ddfd01f437645db11902821b015c5e8e7dc4341e Mon Sep 17 00:00:00 2001 From: Eric Denovellis Date: Wed, 18 Sep 2024 08:12:26 -0700 Subject: [PATCH 57/94] Hotfix: remove keyword argument --- src/spyglass/linearization/v1/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/spyglass/linearization/v1/main.py b/src/spyglass/linearization/v1/main.py index dfc6f7a6e..8cee6b289 100644 --- a/src/spyglass/linearization/v1/main.py +++ b/src/spyglass/linearization/v1/main.py @@ -59,7 +59,6 @@ def get_networkx_track_graph(self, track_graph_parameters=None): return make_track_graph( node_positions=track_graph_parameters["node_positions"], edges=track_graph_parameters["edges"], - edge_map=track_graph_parameters["edge_map"], ) def plot_track_graph(self, ax=None, draw_edge_labels=False, **kwds): From 860d47066bf5b85692781d1fdb9fd2b2219e411a Mon Sep 17 00:00:00 2001 From: Jeremy Magland Date: Fri, 20 Sep 2024 10:31:54 -0400 Subject: [PATCH 58/94] speed up electrodes import (#1125) * speed up electrodes import * update changelog --- CHANGELOG.md | 1 + src/spyglass/common/common_ephys.py | 14 +++++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fbf2fa72c..39c54b9a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Common - Drop `SessionGroup` table #1106 + - Improve electrodes import efficiency #1125 - Decoding diff --git a/src/spyglass/common/common_ephys.py b/src/spyglass/common/common_ephys.py index 37c4361c5..2e56d47fa 100644 --- a/src/spyglass/common/common_ephys.py +++ b/src/spyglass/common/common_ephys.py @@ -127,15 +127,23 @@ def make(self, key): electrode_inserts = [] electrodes = nwbf.electrodes.to_dataframe() + + # Keep a dict of region IDs to avoid multiple fetches + region_ids_dict = dict() + for elect_id, elect_data in electrodes.iterrows(): + region_name = elect_data.group.location + if region_name not in region_ids_dict: + # Only fetch if not already fetched + region_ids_dict[region_name] = BrainRegion.fetch_add( + region_name=region_name + ) key.update( { "electrode_id": elect_id, "name": str(elect_id), "electrode_group_name": elect_data.group_name, - "region_id": BrainRegion.fetch_add( - region_name=elect_data.group.location - ), + "region_id": region_ids_dict[region_name], "x": elect_data.get("x"), "y": elect_data.get("y"), "z": elect_data.get("z"), From a9f6a7fa346ee527eb4a49fad7193cccff32817d Mon Sep 17 00:00:00 2001 From: Samuel Bray Date: Fri, 20 Sep 2024 15:32:19 -0700 Subject: [PATCH 59/94] Switch to or logic when multiple merge ids in restriction (#1126) * switch to or logic when multiple merge ids in restriction * update_changelog --- CHANGELOG.md | 2 +- src/spyglass/utils/dj_merge_tables.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39c54b9a1..8e6997e68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() #1108 - Add docstrings to all public methods #1076 - Update DataJoint to 0.14.2 #1081 -- Allow restriction based on parent keys in `Merge.fetch_nwb()` #1086 +- Allow restriction based on parent keys in `Merge.fetch_nwb()` #1086, #1126 - Import `datajoint.dependencies.unite_master_parts` -> `topo_sort` #1116 ### Pipelines diff --git a/src/spyglass/utils/dj_merge_tables.py b/src/spyglass/utils/dj_merge_tables.py index b2ddbff93..dbe626408 100644 --- a/src/spyglass/utils/dj_merge_tables.py +++ b/src/spyglass/utils/dj_merge_tables.py @@ -853,6 +853,9 @@ def extract_merge_id(cls, restriction) -> Union[dict, list]: merge_id present in the input, relying on parent func to restrict on secondary or part-parent key(s). + Assumes that a valid set of merge_id keys should have OR logic + to allow selection of an entries. + Parameters ---------- restriction : str, dict, or dj.condition.AndList @@ -878,9 +881,7 @@ def extract_merge_id(cls, restriction) -> Union[dict, list]: merge_restr = [x for x in merge_id_list if x is not None] elif isinstance(restriction, str): parsed = [x.split(")")[0] for x in restriction.split("(") if x] - merge_restr = dj.condition.AndList( - [x for x in parsed if "merge_id" in x] - ) + merge_restr = [x for x in parsed if "merge_id" in x] if len(merge_restr) == 0: return True From 2668ddc5a5a9d722df316002ac0f3ffc08c97c2a Mon Sep 17 00:00:00 2001 From: Samuel Bray Date: Fri, 20 Sep 2024 15:32:52 -0700 Subject: [PATCH 60/94] Fix null config_tasks value (#1120) * fix null config_tasks value * fix warning message * update changelog * fix test for null position interval map insert --- CHANGELOG.md | 2 ++ src/spyglass/common/common_task.py | 10 ++++------ tests/common/test_behav.py | 5 ++++- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e6997e68..df0022b40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,8 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Update DataJoint to 0.14.2 #1081 - Allow restriction based on parent keys in `Merge.fetch_nwb()` #1086, #1126 - Import `datajoint.dependencies.unite_master_parts` -> `topo_sort` #1116 +- Allow definition of tasks and new probe entries from config #1074, #1120 +- Enforce match between ingested nwb probe geometry and existing table entry #1074 ### Pipelines diff --git a/src/spyglass/common/common_task.py b/src/spyglass/common/common_task.py index 9b8368192..744d5d773 100644 --- a/src/spyglass/common/common_task.py +++ b/src/spyglass/common/common_task.py @@ -134,8 +134,8 @@ def make(self, key): # schema if it isn't there and then add an entry for each epoch tasks_mod = nwbf.processing.get("tasks") - config_tasks = config.get("Tasks") - if tasks_mod is None and config_tasks is None: + config_tasks = config.get("Tasks", []) + if tasks_mod is None and (not config_tasks): logger.warn( f"No tasks processing module found in {nwbf} or config\n" ) @@ -236,12 +236,10 @@ def get_epoch_interval_name(cls, epoch, session_intervals): if target_interval in interval ] if not possible_targets: - logger.warn( - f"Interval not found for epoch {epoch} in {nwb_file_name}." - ) + logger.warn(f"Interval not found for epoch {epoch}.") elif len(possible_targets) > 1: logger.warn( - f"Multiple intervals found for epoch {epoch} in {nwb_file_name}. " + f"Multiple intervals found for epoch {epoch}. " + f"matches are {possible_targets}." ) else: diff --git a/tests/common/test_behav.py b/tests/common/test_behav.py index bcfd50270..0295023df 100644 --- a/tests/common/test_behav.py +++ b/tests/common/test_behav.py @@ -101,8 +101,11 @@ def test_pos_interval_no_transaction(verbose_context, common, mini_restr): common.PositionIntervalMap()._no_transaction_make(mini_restr) after = common.PositionIntervalMap().fetch() assert ( - len(after) == len(before) + 2 + len(after) == len(before) + 3 ), "PositionIntervalMap no_transaction had unexpected effect" + assert ( + "" in after["position_interval_name"] + ), "PositionIntervalMap null insert failed" def test_get_pos_interval_name(pos_src, pos_interval_01): From a88d55cf971696832e019d99a4f19d397985a82e Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Wed, 25 Sep 2024 13:34:00 -0500 Subject: [PATCH 61/94] Fix #1128 (#1131) --- CHANGELOG.md | 4 +++- notebooks/00_Setup.ipynb | 18 +++++++++++++++++- notebooks/py_scripts/00_Setup.py | 18 +++++++++++++++++- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df0022b40..fcd5e57f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,9 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Allow restriction based on parent keys in `Merge.fetch_nwb()` #1086, #1126 - Import `datajoint.dependencies.unite_master_parts` -> `topo_sort` #1116 - Allow definition of tasks and new probe entries from config #1074, #1120 -- Enforce match between ingested nwb probe geometry and existing table entry #1074 +- Enforce match between ingested nwb probe geometry and existing table entry + #1074 +- Update DataJoint install and password instructions #1131 ### Pipelines diff --git a/notebooks/00_Setup.ipynb b/notebooks/00_Setup.ipynb index 578c6c179..76c801572 100644 --- a/notebooks/00_Setup.ipynb +++ b/notebooks/00_Setup.ipynb @@ -312,7 +312,8 @@ "
Note for MySQL 8 users, including Frank Lab members\n", "\n", "Using a MySQL 8 server, like the server hosted by the Frank Lab, will\n", - "require the pre-release version of DataJoint to change one's password.\n", + "require DataJoint >= 0.14.2. To keep up to data with the latest DataJoint \n", + "features, install from GitHub\n", "\n", "```bash\n", "cd /location/for/datajoint/source/files/\n", @@ -320,6 +321,13 @@ "pip install ./datajoint-python\n", "```\n", "\n", + "You can then periodically fetch updates with the following commands...\n", + "\n", + "```bash\n", + "cd /location/for/datajoint/source/files/datajoint-python\n", + "git pull origin master\n", + "```\n", + "\n", "
\n" ] }, @@ -438,12 +446,20 @@ "outputs": [], "source": [ "import os\n", + "import datajoint as dj\n", "from spyglass.settings import SpyglassConfig\n", "\n", "# change to the root directory of the project\n", "if os.path.basename(os.getcwd()) == \"notebooks\":\n", " os.chdir(\"..\")\n", "\n", + "# connect to the database\n", + "dj.conn()\n", + "\n", + "# change your password\n", + "dj.admin.set_password()\n", + "\n", + "# save the configuration\n", "SpyglassConfig().save_dj_config(\n", " save_method=\"local\", # global or local\n", " base_dir=\"/path/like/stelmo/nwb/\",\n", diff --git a/notebooks/py_scripts/00_Setup.py b/notebooks/py_scripts/00_Setup.py index a9a7fe269..8c2a1a157 100644 --- a/notebooks/py_scripts/00_Setup.py +++ b/notebooks/py_scripts/00_Setup.py @@ -273,7 +273,8 @@ #
Note for MySQL 8 users, including Frank Lab members # # Using a MySQL 8 server, like the server hosted by the Frank Lab, will -# require the pre-release version of DataJoint to change one's password. +# require DataJoint >= 0.14.2. To keep up to data with the latest DataJoint +# features, install from GitHub # # ```bash # # cd /location/for/datajoint/source/files/ @@ -281,6 +282,13 @@ # pip install ./datajoint-python # ``` # +# You can then periodically fetch updates with the following commands... +# +# ```bash +# # cd /location/for/datajoint/source/files/datajoint-python +# git pull origin master +# ``` +# #
# @@ -363,12 +371,20 @@ # + import os +import datajoint as dj from spyglass.settings import SpyglassConfig # change to the root directory of the project if os.path.basename(os.getcwd()) == "notebooks": os.chdir("..") +# connect to the database +dj.conn() + +# change your password +dj.admin.set_password() + +# save the configuration SpyglassConfig().save_dj_config( save_method="local", # global or local base_dir="/path/like/stelmo/nwb/", From 32dc077ee3fb9c0b7abfe6d4135b4a2ba8133974 Mon Sep 17 00:00:00 2001 From: Eric Denovellis Date: Wed, 25 Sep 2024 14:19:07 -0700 Subject: [PATCH 62/94] Fix waveform params in metric curation (#1132) * Fix for case where waveform_paramse sparse is defined. * Update deprecated method `warn` -> `warning` * Update changelog --- CHANGELOG.md | 2 ++ src/spyglass/common/common_task.py | 4 ++-- src/spyglass/spikesorting/v1/metric_curation.py | 5 ++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fcd5e57f0..fe345954f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Drop `SessionGroup` table #1106 - Improve electrodes import efficiency #1125 + - Fix logger method call in `common_task` #1132 - Decoding @@ -53,6 +54,7 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Fix bug in `get_group_by_shank` #1096 - Fix bug in `_compute_metric` #1099 - Fix bug in `insert_curation` returned key #1114 + - Fix handling of waveform extraction sparse parameter #1132 ## [0.5.3] (August 27, 2024) diff --git a/src/spyglass/common/common_task.py b/src/spyglass/common/common_task.py index 744d5d773..60fba08d3 100644 --- a/src/spyglass/common/common_task.py +++ b/src/spyglass/common/common_task.py @@ -236,9 +236,9 @@ def get_epoch_interval_name(cls, epoch, session_intervals): if target_interval in interval ] if not possible_targets: - logger.warn(f"Interval not found for epoch {epoch}.") + logger.warning(f"Interval not found for epoch {epoch}.") elif len(possible_targets) > 1: - logger.warn( + logger.warning( f"Multiple intervals found for epoch {epoch}. " + f"matches are {possible_targets}." ) diff --git a/src/spyglass/spikesorting/v1/metric_curation.py b/src/spyglass/spikesorting/v1/metric_curation.py index b9d1fb66f..8346f8ddd 100644 --- a/src/spyglass/spikesorting/v1/metric_curation.py +++ b/src/spyglass/spikesorting/v1/metric_curation.py @@ -263,10 +263,13 @@ def make(self, key): os.makedirs(waveforms_dir, exist_ok=True) logger.info("Extracting waveforms...") + + # Extract non-sparse waveforms by default + waveform_params.setdefault("sparse", False) + waveforms = si.extract_waveforms( recording=recording, sorting=sorting, - sparse=waveform_params.get("sparse", False), folder=waveforms_dir, overwrite=True, **waveform_params, From 05444bbe1481aaf9c6513e04e37fa66ec6583f87 Mon Sep 17 00:00:00 2001 From: Samuel Bray Date: Thu, 26 Sep 2024 15:40:57 -0700 Subject: [PATCH 63/94] Convert string config settings to bool (#1117) * convert string config settings to bool * update changelog * move str_to_bool and use in settings * fix lint * fix lint --------- Co-authored-by: Eric Denovellis --- CHANGELOG.md | 1 + src/spyglass/position/v1/position_dlc_model.py | 10 +--------- src/spyglass/settings.py | 3 +++ src/spyglass/utils/dj_helper_fn.py | 9 +++++++++ 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe345954f..344aae634 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Update DataJoint to 0.14.2 #1081 - Allow restriction based on parent keys in `Merge.fetch_nwb()` #1086, #1126 - Import `datajoint.dependencies.unite_master_parts` -> `topo_sort` #1116 +- Fix bool settings imported from dj config file #1117 - Allow definition of tasks and new probe entries from config #1074, #1120 - Enforce match between ingested nwb probe geometry and existing table entry #1074 diff --git a/src/spyglass/position/v1/position_dlc_model.py b/src/spyglass/position/v1/position_dlc_model.py index 243d51a69..2a64c05c4 100644 --- a/src/spyglass/position/v1/position_dlc_model.py +++ b/src/spyglass/position/v1/position_dlc_model.py @@ -5,6 +5,7 @@ import ruamel.yaml as yaml from spyglass.utils import SpyglassMixin, logger +from spyglass.utils.dj_helper_fn import str_to_bool from . import dlc_reader from .position_dlc_project import BodyPart, DLCProject # noqa: F401 @@ -347,12 +348,3 @@ def make(self, key): test_error_p=results["Test error with p-cutoff"], ) ) - - -def str_to_bool(value) -> bool: - """Return whether the provided string represents true. Otherwise false.""" - # Due to distutils equivalent depreciation in 3.10 - # Adopted from github.com/PostHog/posthog/blob/master/posthog/utils.py - if not value: - return False - return str(value).lower() in ("y", "yes", "t", "true", "on", "1") diff --git a/src/spyglass/settings.py b/src/spyglass/settings.py index ecfd8d540..15bcf9d9c 100644 --- a/src/spyglass/settings.py +++ b/src/spyglass/settings.py @@ -7,6 +7,7 @@ import yaml from pymysql.err import OperationalError +from spyglass.utils.dj_helper_fn import str_to_bool from spyglass.utils.logging import logger @@ -143,6 +144,8 @@ def load_config( self._test_mode = kwargs.get("test_mode") or dj_custom.get( "test_mode", False ) + self._test_mode = str_to_bool(self._test_mode) + self._debug_mode = str_to_bool(self._debug_mode) resolved_base = ( base_dir diff --git a/src/spyglass/utils/dj_helper_fn.py b/src/spyglass/utils/dj_helper_fn.py index 889e64294..35d77ef3c 100644 --- a/src/spyglass/utils/dj_helper_fn.py +++ b/src/spyglass/utils/dj_helper_fn.py @@ -558,3 +558,12 @@ def daemon(self, val): proc.__class__ = NonDaemonProcess return proc + + +def str_to_bool(value) -> bool: + """Return whether the provided string represents true. Otherwise false.""" + # Due to distutils equivalent depreciation in 3.10 + # Adopted from github.com/PostHog/posthog/blob/master/posthog/utils.py + if not value: + return False + return str(value).lower() in ("y", "yes", "t", "true", "on", "1") From e8511a52fe07cbc7ee2549ea3face3071ff7acd7 Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Tue, 1 Oct 2024 13:27:14 -0500 Subject: [PATCH 64/94] Long distance restr fix (#1137) * Fix 1136 tests * Update changelog --- CHANGELOG.md | 2 +- pyproject.toml | 2 +- src/spyglass/utils/database_settings.py | 2 +- src/spyglass/utils/dj_graph.py | 7 +++---- src/spyglass/utils/dj_merge_tables.py | 4 ++-- tests/utils/test_graph.py | 2 +- 6 files changed, 9 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 344aae634..9e87a2bd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,7 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Add docstrings to all public methods #1076 - Update DataJoint to 0.14.2 #1081 - Allow restriction based on parent keys in `Merge.fetch_nwb()` #1086, #1126 -- Import `datajoint.dependencies.unite_master_parts` -> `topo_sort` #1116 +- Import `datajoint.dependencies.unite_master_parts` -> `topo_sort` #1116, #1137 - Fix bool settings imported from dj config file #1117 - Allow definition of tasks and new probe entries from config #1074, #1120 - Enforce match between ingested nwb probe geometry and existing table entry diff --git a/pyproject.toml b/pyproject.toml index e5f8dae5b..ed3a570c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -126,7 +126,7 @@ ignore-words-list = 'nevers' minversion = "7.0" addopts = [ # "-sv", # no capture, verbose output - "--sw", # stepwise: resume with next test after failure + # "--sw", # stepwise: resume with next test after failure # "--pdb", # drop into debugger on failure "-p no:warnings", # "--no-teardown", # don't teardown the database after tests diff --git a/src/spyglass/utils/database_settings.py b/src/spyglass/utils/database_settings.py index e7f36479e..56e2aa28f 100755 --- a/src/spyglass/utils/database_settings.py +++ b/src/spyglass/utils/database_settings.py @@ -15,7 +15,7 @@ "spikesorting", "decoding", "position", - "position_linearization", + "linearization", "ripple", "lfp", "waveform", diff --git a/src/spyglass/utils/dj_graph.py b/src/spyglass/utils/dj_graph.py index 26a944e20..48847f61b 100644 --- a/src/spyglass/utils/dj_graph.py +++ b/src/spyglass/utils/dj_graph.py @@ -22,7 +22,6 @@ all_simple_paths, shortest_path, ) -from networkx.algorithms.dag import topological_sort from tqdm import tqdm from spyglass.utils import logger @@ -478,7 +477,7 @@ def _topo_sort( if not self._is_out(node, warn=False) ] graph = self.graph.subgraph(nodes) if subgraph else self.graph - ordered = dj_topo_sort(list(topological_sort(graph))) + ordered = dj_topo_sort(graph) if reverse: ordered.reverse() return [n for n in ordered if n in nodes] @@ -869,10 +868,10 @@ def __init__( self.direction = Direction.DOWN self.leaf = None - if search_restr and not parent: + if search_restr and not self.parent: # using `parent` fails on empty self.direction = Direction.UP self.leaf = self.child - if search_restr and not child: + if search_restr and not self.child: self.direction = Direction.DOWN self.leaf = self.parent if self.leaf: diff --git a/src/spyglass/utils/dj_merge_tables.py b/src/spyglass/utils/dj_merge_tables.py index dbe626408..bf13aa254 100644 --- a/src/spyglass/utils/dj_merge_tables.py +++ b/src/spyglass/utils/dj_merge_tables.py @@ -3,7 +3,7 @@ from pprint import pprint from re import sub as re_sub from time import time -from typing import Union, List +from typing import List, Union import datajoint as dj from datajoint.condition import make_condition @@ -348,7 +348,7 @@ def _merge_insert(cls, rows: list, part_name: str = None, **kwargs) -> None: ) key = keys[0] if part & key: - print(f"Key already in part {part_name}: {key}") + logger.info(f"Key already in part {part_name}: {key}") continue master_sk = {cls()._reserved_sk: part_name} uuid = dj.hash.key_hash(key | master_sk) diff --git a/tests/utils/test_graph.py b/tests/utils/test_graph.py index c51427810..4acbc2b1d 100644 --- a/tests/utils/test_graph.py +++ b/tests/utils/test_graph.py @@ -157,7 +157,7 @@ def test_restr_from_upstream(graph_tables, restr, expect_n, msg): ("PkAliasNode", "parent_attr > 17", 2, "pk pk alias"), ("SkAliasNode", "parent_attr > 18", 2, "sk sk alias"), ("MergeChild", "parent_attr > 18", 2, "merge child"), - ("MergeChild", {"parent_attr": 18}, 1, "dict restr"), + ("MergeChild", {"parent_attr": 19}, 1, "dict restr"), ], ) def test_restr_from_downstream(graph_tables, table, restr, expect_n, msg): From 4498444737e359a5820e847898e838a859ba0732 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Wed, 2 Oct 2024 10:58:34 -0400 Subject: [PATCH 65/94] Update README.md to fix link (#1140) --- notebooks/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/README.md b/notebooks/README.md index 0982c464f..565031d36 100644 --- a/notebooks/README.md +++ b/notebooks/README.md @@ -28,7 +28,7 @@ spike sorting to optional manual curation of the output of the automated sorting. Spikesorting results from any pipeline can then be organized and tracked using -tools in [Spikesorting Analysis](./11_Spikesorting_Analysis.ipynb). +tools in [Spikesorting Analysis](./11_Spike_Sorting_Analysis.ipynb). ## 2. Position Pipeline From 0ccc6ed213a2cc4db89697dbdbe41c62100eb28f Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Wed, 2 Oct 2024 10:58:50 -0400 Subject: [PATCH 66/94] Update UsingNWB.md to fix typo (#1141) --- docs/src/ForDevelopers/UsingNWB.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/ForDevelopers/UsingNWB.md b/docs/src/ForDevelopers/UsingNWB.md index f64870930..f4ff7d997 100644 --- a/docs/src/ForDevelopers/UsingNWB.md +++ b/docs/src/ForDevelopers/UsingNWB.md @@ -62,7 +62,7 @@ The following objects should be uniquely named. - Each recording and sorting is given truncated UUID strings as part of concatenations. -Following broader Python conventions, methods a method that will not be +Following broader Python conventions, a method that will not be explicitly called by the user should start with `_` ## Time From 460c24290b33524c35052d8ce6e7cf846d96866e Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Wed, 2 Oct 2024 16:41:08 -0500 Subject: [PATCH 67/94] #1138, #1139 (#1145) * #1138, #1139 * Update changelog --- CHANGELOG.md | 1 + notebooks/00_Setup.ipynb | 3 ++- notebooks/01_Concepts.ipynb | 4 ++-- notebooks/py_scripts/00_Setup.py | 3 ++- notebooks/py_scripts/01_Concepts.py | 4 ++-- 5 files changed, 9 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e87a2bd1..8515f40f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Enforce match between ingested nwb probe geometry and existing table entry #1074 - Update DataJoint install and password instructions #1131 +- Minor docs fixes #1145 ### Pipelines diff --git a/notebooks/00_Setup.ipynb b/notebooks/00_Setup.ipynb index 76c801572..9188bafa0 100644 --- a/notebooks/00_Setup.ipynb +++ b/notebooks/00_Setup.ipynb @@ -63,7 +63,8 @@ "For local use, download and install ...\n", "\n", "1. [Python 3.9](https://wiki.python.org/moin/BeginnersGuide/Download).\n", - "2. [mamba](https://mamba.readthedocs.io/en/latest/installation.html) as a\n", + "2. [mamba](https://mamba.readthedocs.io/en/latest/installation/mamba-installation.html)\n", + " as a\n", " replacement for conda. Spyglass installation is significantly faster with\n", " mamba.\n", " ```bash\n", diff --git a/notebooks/01_Concepts.ipynb b/notebooks/01_Concepts.ipynb index fcb74632a..b0ebb07aa 100644 --- a/notebooks/01_Concepts.ipynb +++ b/notebooks/01_Concepts.ipynb @@ -36,8 +36,8 @@ "source": [ "## Other materials\n", "\n", - "DataJoint is an \"Object-relational mapping\" tool, which means that it gives us\n", - "a Python object for tables that exist on a shared SQL server. Many Spyglass\n", + "DataJoint is an tool that helps us create Python classes for\n", + "tables that exist on a shared SQL server. Many Spyglass\n", "imports are DataJoint tables like this.\n", "\n", "Any 'introduction to SQL' will give an overview of relational data models as\n", diff --git a/notebooks/py_scripts/00_Setup.py b/notebooks/py_scripts/00_Setup.py index 8c2a1a157..1169999a4 100644 --- a/notebooks/py_scripts/00_Setup.py +++ b/notebooks/py_scripts/00_Setup.py @@ -45,7 +45,8 @@ # For local use, download and install ... # # 1. [Python 3.9](https://wiki.python.org/moin/BeginnersGuide/Download). -# 2. [mamba](https://mamba.readthedocs.io/en/latest/installation.html) as a +# 2. [mamba](https://mamba.readthedocs.io/en/latest/installation/mamba-installation.html) +# as a # replacement for conda. Spyglass installation is significantly faster with # mamba. # ```bash diff --git a/notebooks/py_scripts/01_Concepts.py b/notebooks/py_scripts/01_Concepts.py index 70e5f6625..1275b59a9 100644 --- a/notebooks/py_scripts/01_Concepts.py +++ b/notebooks/py_scripts/01_Concepts.py @@ -30,8 +30,8 @@ # ## Other materials # -# DataJoint is an "Object-relational mapping" tool, which means that it gives us -# a Python object for tables that exist on a shared SQL server. Many Spyglass +# DataJoint is an tool that helps us create Python classes for +# tables that exist on a shared SQL server. Many Spyglass # imports are DataJoint tables like this. # # Any 'introduction to SQL' will give an overview of relational data models as From 3e0fc215856bab84a1c93017bc9c8e9deaa0c41a Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Wed, 2 Oct 2024 19:07:30 -0400 Subject: [PATCH 68/94] Update UsingNWB.md to fix task location in NWB file (#1143) --- docs/src/ForDevelopers/UsingNWB.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/src/ForDevelopers/UsingNWB.md b/docs/src/ForDevelopers/UsingNWB.md index f4ff7d997..b5e3d89ae 100644 --- a/docs/src/ForDevelopers/UsingNWB.md +++ b/docs/src/ForDevelopers/UsingNWB.md @@ -214,10 +214,10 @@ hdmf.common.table.DynamicTable | Spyglass Table | Key | NWBfile Location | Config option | Notes | | :------------- | :--------------: | -----------------------------------------------: | ------------: | ----: | | Task | task_name | nwbf.processing.tasks.\[index\].name | | | -| Task | task_description | nwbf.processing.\[index\].tasks.description | | | -| TaskEpoch | task_name | nwbf.processing.\[index\].tasks.name | config\["Tasks"\]\[index\]\["task_name"\]| | -| TaskEpoch | camera_names | nwbf.processing.\[index\].tasks.camera_id | config\["Tasks"\]\[index\]\["camera_id"\] | | -| TaskEpoch | task_environment | nwbf.processing.\[index\].tasks.task_environment | config\["Tasks"\]\[index\]\["task_environment"\] | | +| Task | task_description | nwbf.processing.tasks.\[index\].description | | | +| TaskEpoch | task_name | nwbf.processing.tasks.\[index\].name | config\["Tasks"\]\[index\]\["task_name"\]| | +| TaskEpoch | camera_names | nwbf.processing.tasks.\[index\].camera_id | config\["Tasks"\]\[index\]\["camera_id"\] | | +| TaskEpoch | task_environment | nwbf.processing.tasks.\[index\].task_environment | config\["Tasks"\]\[index\]\["task_environment"\] | | NWBfile Location: nwbf.units
Object type: pynwb.misc.Units From fcc3bc16475ac918f10cf549b4403f6b62a91745 Mon Sep 17 00:00:00 2001 From: Samuel Bray Date: Wed, 2 Oct 2024 19:24:18 -0700 Subject: [PATCH 69/94] Quickfix: typo in adding objects to nwb (#1148) * fix typo in adding objects to nwb * update changelog --- CHANGELOG.md | 2 +- src/spyglass/position/v1/position_dlc_centroid.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8515f40f8..ee17173f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,7 +49,7 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Fix video directory bug in `DLCPoseEstimationSelection` #1103 - Restore #973, allow DLC without position tracking #1100 - - Minor fix to `DLCCentroid` make function order #1112 + - Minor fix to `DLCCentroid` make function order #1112, #1148 - Spike Sorting diff --git a/src/spyglass/position/v1/position_dlc_centroid.py b/src/spyglass/position/v1/position_dlc_centroid.py index 2e66397c3..daa55a0af 100644 --- a/src/spyglass/position/v1/position_dlc_centroid.py +++ b/src/spyglass/position/v1/position_dlc_centroid.py @@ -296,10 +296,10 @@ def _logged_make(self, key): analysis_file_name = AnalysisNwbfile().create(key["nwb_file_name"]) nwb_analysis_file = AnalysisNwbfile() key["dlc_position_object_id"] = nwb_analysis_file.add_nwb_object( - nwb_analysis_file.add_nwb_object(analysis_file_name, position) + analysis_file_name, position ) key["dlc_velocity_object_id"] = nwb_analysis_file.add_nwb_object( - nwb_analysis_file.add_nwb_object(analysis_file_name, velocity) + analysis_file_name, velocity ) nwb_analysis_file.add( From f77483d522c440157ff88f65adcb85f05505cfbc Mon Sep 17 00:00:00 2001 From: Samuel Bray Date: Thu, 3 Oct 2024 07:34:49 -0700 Subject: [PATCH 70/94] Dandi export fixes (#1095) * adjust external table for raw files * prevent error after adjusting rat species name * copy files with external filepath and allow dandi to edit copy * fix organize call functions and add temp option to skip raw files pending discussion * add faster dandi path translation via lookup * make free tables iterable in write_mysqldump call * make dandi file update tool usable for orignal nwb file * exclude linked files from export and include linked source * import shared schema during populate paper * fall back to dandi original file when fetching copied raw nwb * improve warning message * update changelog * fix lint * suggestions from code review * implement style suggestions from review * use lookup translation * style fix --------- Co-authored-by: Samuel Bray --- CHANGELOG.md | 1 + src/spyglass/common/common_dandi.py | 70 +++++++++++++++++++++++++---- src/spyglass/common/common_usage.py | 17 +++++++ src/spyglass/utils/dj_helper_fn.py | 15 +++++-- src/spyglass/utils/nwb_helper_fn.py | 18 ++++++++ 5 files changed, 109 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee17173f0..11ef7e09d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Enforce match between ingested nwb probe geometry and existing table entry #1074 - Update DataJoint install and password instructions #1131 +- Fix dandi upload process for nwb's with video or linked objects #1095 - Minor docs fixes #1145 ### Pipelines diff --git a/src/spyglass/common/common_dandi.py b/src/spyglass/common/common_dandi.py index 6dfbb56e3..8a9be9677 100644 --- a/src/spyglass/common/common_dandi.py +++ b/src/spyglass/common/common_dandi.py @@ -1,6 +1,7 @@ import os import shutil from pathlib import Path +from typing import Optional import datajoint as dj import fsspec @@ -9,7 +10,7 @@ from fsspec.implementations.cached import CachingFileSystem from spyglass.common.common_usage import Export, ExportSelection -from spyglass.settings import export_dir +from spyglass.settings import export_dir, raw_dir from spyglass.utils import SpyglassMixin, logger from spyglass.utils.sql_helper_fn import SQLDumpHelper @@ -21,7 +22,8 @@ from dandi.consts import known_instances from dandi.dandiapi import DandiAPIClient from dandi.metadata.nwb import get_metadata - from dandi.organize import OrganizeInvalid + from dandi.organize import CopyMode, FileOperationMode, OrganizeInvalid + from dandi.pynwb_utils import nwb_has_external_links from dandi.validate_types import Severity except (ImportError, ModuleNotFoundError) as e: @@ -31,8 +33,11 @@ DandiAPIClient, get_metadata, OrganizeInvalid, + CopyMode, + FileOperationMode, Severity, - ) = [None] * 6 + nwb_has_external_links, + ) = [None] * 9 logger.warning(e) @@ -87,6 +92,7 @@ def compile_dandiset( dandiset_id: str, dandi_api_key: str = None, dandi_instance: str = "dandi", + skip_raw_files: bool = False, ): """Compile a Dandiset from the export. Parameters @@ -100,6 +106,8 @@ def compile_dandiset( DANDI_API_KEY is set. dandi_instance : str, optional What instance of Dandi the dandiset is on. Defaults to dev server. + skip_raw_files : bool, optional + Dev tool to skip raw files in the export. Defaults to False. """ key = (Export & key).fetch1("KEY") paper_id = (Export & key).fetch1("paper_id") @@ -137,9 +145,14 @@ def compile_dandiset( os.makedirs(destination_dir, exist_ok=False) for file in source_files: - if not os.path.exists( - f"{destination_dir}/{os.path.basename(file)}" - ): + if os.path.exists(f"{destination_dir}/{os.path.basename(file)}"): + continue + if skip_raw_files and raw_dir in file: + continue + # copy the file if it has external links so can be safely edited + if nwb_has_external_links(file): + shutil.copy(file, f"{destination_dir}/{os.path.basename(file)}") + else: os.symlink(file, f"{destination_dir}/{os.path.basename(file)}") # validate the dandiset @@ -154,11 +167,16 @@ def compile_dandiset( # organize the files in the dandiset directory dandi.organize.organize( - destination_dir, dandiset_dir, invalid=OrganizeInvalid.WARN + destination_dir, + dandiset_dir, + update_external_file_paths=True, + invalid=OrganizeInvalid.FAIL, + media_files_mode=CopyMode.SYMLINK, + files_mode=FileOperationMode.COPY, ) # get the dandi name translations - translations = translate_name_to_dandi(destination_dir) + translations = lookup_dandi_translation(destination_dir, dandiset_dir) # upload the dandiset to the dandi server if dandi_api_key: @@ -200,7 +218,7 @@ def write_mysqldump(self, export_key: dict): docker_id=None, spyglass_version=spyglass_version, ) - sql_dump.write_mysqldump(self & key, file_suffix="_dandi") + sql_dump.write_mysqldump([self & key], file_suffix="_dandi") def _get_metadata(path): @@ -229,6 +247,7 @@ def translate_name_to_dandi(folder): dict dictionary of filename to dandi_path translations """ + files = Path(folder).glob("*") metadata = list(map(_get_metadata, files)) metadata, skip_invalid = dandi.organize.filter_invalid_metadata_rows( @@ -243,6 +262,39 @@ def translate_name_to_dandi(folder): ] +def lookup_dandi_translation(source_dir: str, dandiset_dir: str): + """Get the dandi_path for each nwb file in the source_dir from + the organized dandi directory + + Parameters + ---------- + source_dir : str + location of the source files + dandiset_dir : str + location of the organized dandiset directory + + Returns + ------- + dict + dictionary of filename to dandi_path translations + """ + # get the obj_id and dandipath for each nwb file in the dandiset + dandi_name_dict = {} + for dandi_file in Path(dandiset_dir).rglob("*.nwb"): + dandi_path = dandi_file.relative_to(dandiset_dir).as_posix() + with pynwb.NWBHDF5IO(dandi_file, "r") as io: + nwb = io.read() + dandi_name_dict[nwb.object_id] = dandi_path + # for each file in the source_dir, lookup the dandipath based on the obj_id + name_translation = {} + for file in Path(source_dir).glob("*"): + with pynwb.NWBHDF5IO(file, "r") as io: + nwb = io.read() + dandi_path = dandi_name_dict[nwb.object_id] + name_translation[file.name] = dandi_path + return name_translation + + def validate_dandiset( folder, min_severity="ERROR", ignore_external_files=False ): diff --git a/src/spyglass/common/common_usage.py b/src/spyglass/common/common_usage.py index ad49d82a1..23d9d04be 100644 --- a/src/spyglass/common/common_usage.py +++ b/src/spyglass/common/common_usage.py @@ -22,6 +22,7 @@ unique_dicts, update_analysis_for_dandi_standard, ) +from spyglass.utils.nwb_helper_fn import get_linked_nwbs from spyglass.utils.sql_helper_fn import SQLDumpHelper schema = dj.schema("common_usage") @@ -236,6 +237,7 @@ class File(SpyglassMixin, dj.Part): def populate_paper(self, paper_id: Union[str, dict]): """Populate Export for a given paper_id.""" + self.load_shared_schemas() if isinstance(paper_id, dict): paper_id = paper_id.get("paper_id") self.populate(ExportSelection().paper_export_id(paper_id)) @@ -272,6 +274,21 @@ def make(self, key): query.list_file_paths(paper_key) + restr_graph.file_paths ) + # Check for linked nwb objects and add them to the export + unlinked_files = set() + for file in file_paths: + if not (links := get_linked_nwbs(file["file_path"])): + unlinked_files.add(file) + continue + logger.warning( + "Dandi not yet supported for linked nwb objects " + + f"excluding {file['file_path']} from export " + + f" and including {links} instead" + ) + for link in links: + unlinked_files.add(link) + file_paths = {"file_path": link for link in unlinked_files} + table_inserts = [ {**key, **rd, "table_id": i} for i, rd in enumerate(restr_graph.as_dict) diff --git a/src/spyglass/utils/dj_helper_fn.py b/src/spyglass/utils/dj_helper_fn.py index 35d77ef3c..ca8b99f2b 100644 --- a/src/spyglass/utils/dj_helper_fn.py +++ b/src/spyglass/utils/dj_helper_fn.py @@ -355,6 +355,7 @@ def get_child_tables(table): def update_analysis_for_dandi_standard( filepath: str, age: str = "P4M/P8M", + resolve_external_table: bool = True, ): """Function to resolve common nwb file format errors within the database @@ -364,6 +365,9 @@ def update_analysis_for_dandi_standard( abs path to the file to edit age : str, optional age to assign animal if missing, by default "P4M/P8M" + resolve_external_table : bool, optional + whether to update the external table. Set False if editing file + outside the database, by default True """ from spyglass.common import LabMember @@ -394,7 +398,7 @@ def update_analysis_for_dandi_standard( ) file["/general/subject/species"][()] = new_species_value - if not ( + elif not ( len(species_value.split(" ")) == 2 or "NCBITaxon" in species_value ): raise ValueError( @@ -427,7 +431,9 @@ def update_analysis_for_dandi_standard( file["/general/experimenter"][:] = new_experimenter_value # update the datajoint external store table to reflect the changes - _resolve_external_table(filepath, file_name) + if resolve_external_table: + location = "raw" if filepath.endswith("_.nwb") else "analysis" + _resolve_external_table(filepath, file_name, location) def dandi_format_names(experimenter: List) -> List: @@ -510,7 +516,10 @@ def make_file_obj_id_unique(nwb_path: str): new_id = str(uuid4()) with h5py.File(nwb_path, "a") as f: f.attrs["object_id"] = new_id - _resolve_external_table(nwb_path, nwb_path.split("/")[-1]) + location = "raw" if nwb_path.endswith("_.nwb") else "analysis" + _resolve_external_table( + nwb_path, nwb_path.split("/")[-1], location=location + ) return new_id diff --git a/src/spyglass/utils/nwb_helper_fn.py b/src/spyglass/utils/nwb_helper_fn.py index af25ec987..7e930de04 100644 --- a/src/spyglass/utils/nwb_helper_fn.py +++ b/src/spyglass/utils/nwb_helper_fn.py @@ -69,6 +69,14 @@ def get_nwb_file(nwb_file_path): from ..common.common_dandi import DandiPath dandi_key = {"filename": os.path.basename(nwb_file_path)} + if not DandiPath() & dandi_key: + # Check if non-copied raw file is in Dandi + dandi_key = { + "filename": Path(nwb_file_path).name.replace( + "_.nwb", ".nwb" + ) + } + if not DandiPath & dandi_key: # If not in Dandi, then we can't find the file raise FileNotFoundError( @@ -101,6 +109,16 @@ def file_from_dandi(filepath): return False +def get_linked_nwbs(path): + """Return a list of paths to NWB files that are linked by objects in + the file at the given path.""" + with pynwb.NWBHDF5IO(path, "r") as io: + # open the nwb file (opens externally linked files as well) + nwb = io.read() + # get the linked files + return [x for x in io._HDF5IO__built if x != path] + + def get_config(nwb_file_path, calling_table=None): """Return a dictionary of config settings for the given NWB file. If the file does not exist, return an empty dict. From fd06b11bf50eef2cefb90d843b4fb77a737bc0b4 Mon Sep 17 00:00:00 2001 From: Eric Denovellis Date: Thu, 3 Oct 2024 07:35:38 -0700 Subject: [PATCH 71/94] Fix test badge (#1149) Badge is pointing to the wrong tests --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d2e4e5e47..79478c361 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # spyglass -[![Import test](https://github.com/LorenFrankLab/spyglass/actions/workflows/workflow.yml/badge.svg)](https://github.com/LorenFrankLab/spyglass/actions/workflows/workflow.yml) +[![Tests](https://github.com/LorenFrankLab/spyglass/actions/workflows/test-conda.yml/badge.svg)](https://github.com/LorenFrankLab/spyglass/actions/workflows/test-conda.yml) [![PyPI version](https://badge.fury.io/py/spyglass-neuro.svg)](https://badge.fury.io/py/spyglass-neuro) ![Spyglass Figure](docs/src/images/fig1.png) From 5446d074a37fbe48a345b567dd0501f30b4cb2a0 Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Thu, 3 Oct 2024 12:49:47 -0500 Subject: [PATCH 72/94] Additional export fixes (#1151) * Fix dict/list handling in Export * Update changelog --- CHANGELOG.md | 2 +- src/spyglass/common/common_usage.py | 33 +++++++++++++++++++---------- src/spyglass/utils/dj_graph.py | 2 +- src/spyglass/utils/nwb_helper_fn.py | 11 ++++++---- 4 files changed, 31 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11ef7e09d..0657f993b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,7 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Enforce match between ingested nwb probe geometry and existing table entry #1074 - Update DataJoint install and password instructions #1131 -- Fix dandi upload process for nwb's with video or linked objects #1095 +- Fix dandi upload process for nwb's with video or linked objects #1095, #1151 - Minor docs fixes #1145 ### Pipelines diff --git a/src/spyglass/common/common_usage.py b/src/spyglass/common/common_usage.py index 23d9d04be..ced518521 100644 --- a/src/spyglass/common/common_usage.py +++ b/src/spyglass/common/common_usage.py @@ -144,11 +144,19 @@ def stop_export(self, **kwargs) -> None: # before actually exporting anything, which is more associated with # Selection - def list_file_paths(self, key: dict) -> list[str]: + def list_file_paths(self, key: dict, as_dict=True) -> list[str]: """Return a list of unique file paths for a given restriction/key. Note: This list reflects files fetched during the export process. For upstream files, use RestrGraph.file_paths. + + Parameters + ---------- + key : dict + Any valid restriction key for ExportSelection.Table + as_dict : bool, optional + Return as a list of dicts: [{'file_path': x}]. Default True. + If False, returns a list of strings without key. """ file_table = self * self.File & key analysis_fp = [ @@ -159,7 +167,8 @@ def list_file_paths(self, key: dict) -> list[str]: Nwbfile().get_abs_path(fname) for fname in (AnalysisNwbfile * file_table).fetch("nwb_file_name") ] - return [{"file_path": p} for p in list({*analysis_fp, *nwbfile_fp})] + unique_ft = list({*analysis_fp, *nwbfile_fp}) + return [{"file_path": p} for p in unique_ft] if as_dict else unique_ft def get_restr_graph(self, key: dict, verbose=False) -> RestrGraph: """Return a RestrGraph for a restriction/key's tables/restrictions. @@ -270,31 +279,33 @@ def make(self, key): (self.Table & id_dict).delete_quick() restr_graph = ExportSelection().get_restr_graph(paper_key) - file_paths = unique_dicts( # Original plus upstream files - query.list_file_paths(paper_key) + restr_graph.file_paths - ) + # Original plus upstream files + file_paths = { + *query.list_file_paths(paper_key, as_dict=False), + *restr_graph.file_paths, + } # Check for linked nwb objects and add them to the export unlinked_files = set() for file in file_paths: - if not (links := get_linked_nwbs(file["file_path"])): + if not (links := get_linked_nwbs(file)): unlinked_files.add(file) continue logger.warning( "Dandi not yet supported for linked nwb objects " - + f"excluding {file['file_path']} from export " + + f"excluding {file} from export " + f" and including {links} instead" ) - for link in links: - unlinked_files.add(link) - file_paths = {"file_path": link for link in unlinked_files} + unlinked_files.update(links) + file_paths = unlinked_files # TODO: what if linked items have links? table_inserts = [ {**key, **rd, "table_id": i} for i, rd in enumerate(restr_graph.as_dict) ] file_inserts = [ - {**key, **fp, "file_id": i} for i, fp in enumerate(file_paths) + {**key, "file_path": fp, "file_id": i} + for i, fp in enumerate(file_paths) ] version_ids = query.fetch("spyglass_version") diff --git a/src/spyglass/utils/dj_graph.py b/src/spyglass/utils/dj_graph.py index 48847f61b..a66a6ac5d 100644 --- a/src/spyglass/utils/dj_graph.py +++ b/src/spyglass/utils/dj_graph.py @@ -783,7 +783,7 @@ def file_paths(self) -> List[str]: """ self.cascade() return [ - {"file_path": self.analysis_file_tbl.get_abs_path(file)} + self.analysis_file_tbl.get_abs_path(file) for file in set( [f for files in self.file_dict.values() for f in files] ) diff --git a/src/spyglass/utils/nwb_helper_fn.py b/src/spyglass/utils/nwb_helper_fn.py index 7e930de04..5d5fdaca4 100644 --- a/src/spyglass/utils/nwb_helper_fn.py +++ b/src/spyglass/utils/nwb_helper_fn.py @@ -4,6 +4,7 @@ import os.path from itertools import groupby from pathlib import Path +from typing import List import numpy as np import pynwb @@ -109,12 +110,14 @@ def file_from_dandi(filepath): return False -def get_linked_nwbs(path): - """Return a list of paths to NWB files that are linked by objects in - the file at the given path.""" +def get_linked_nwbs(path: str) -> List[str]: + """Return a paths linked in the given NWB file. + + Given a NWB file path, open & read the file to find any linked NWB objects. + """ with pynwb.NWBHDF5IO(path, "r") as io: # open the nwb file (opens externally linked files as well) - nwb = io.read() + _ = io.read() # get the linked files return [x for x in io._HDF5IO__built if x != path] From 368e771560b11d7b454b4ddb12f3c29660891ca5 Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Thu, 3 Oct 2024 16:12:51 -0500 Subject: [PATCH 73/94] Remove stored hashes from tests (#1152) * Remove stored hashes from tests * Remove failing test --- CHANGELOG.md | 1 + tests/conftest.py | 3 + tests/linearization/test_lin.py | 19 ++++-- tests/position/test_trodes.py | 26 +++++--- tests/spikesorting/conftest.py | 10 --- tests/spikesorting/test_curation.py | 100 +++++++++++++++++++++++----- tests/spikesorting/test_merge.py | 42 ++++++++++-- 7 files changed, 154 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0657f993b..a3946d5e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Update DataJoint install and password instructions #1131 - Fix dandi upload process for nwb's with video or linked objects #1095, #1151 - Minor docs fixes #1145 +- Remove stored hashes from pytests #1152 ### Pipelines diff --git a/tests/conftest.py b/tests/conftest.py index 954dad204..8cd07755a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -116,6 +116,9 @@ def pytest_configure(config): def pytest_unconfigure(config): + from spyglass.utils.nwb_helper_fn import close_nwb_files + + close_nwb_files() if TEARDOWN: SERVER.stop() diff --git a/tests/linearization/test_lin.py b/tests/linearization/test_lin.py index a5db28d9a..0a6d12104 100644 --- a/tests/linearization/test_lin.py +++ b/tests/linearization/test_lin.py @@ -1,12 +1,19 @@ -from datajoint.hash import key_hash +import pytest def test_fetch1_dataframe(lin_v1, lin_merge, lin_merge_key): - hash_df = key_hash( - (lin_merge & lin_merge_key).fetch1_dataframe().round(3).to_dict() - ) - hash_exp = "883a7b8aa47931ae7b265660ca27b462" - assert hash_df == hash_exp, "Dataframe differs from expected" + df = (lin_merge & lin_merge_key).fetch1_dataframe().round(3).sum().to_dict() + exp = { + "linear_position": 3249449.258, + "projected_x_position": 472245.797, + "projected_y_position": 317857.473, + "track_segment_id": 31158.0, + } + + for k in exp: + assert ( + pytest.approx(df[k], rel=1e-3) == exp[k] + ), f"Value differs from expected: {k}" # TODO: Add more tests of this pipeline, not just the fetch1_dataframe method diff --git a/tests/position/test_trodes.py b/tests/position/test_trodes.py index 2608d70c6..af6be75c7 100644 --- a/tests/position/test_trodes.py +++ b/tests/position/test_trodes.py @@ -48,15 +48,25 @@ def test_sel_insert_error(trodes_sel_table, pos_interval_key): def test_fetch_df(trodes_pos_v1, trodes_params): upsampled = {"trodes_pos_params_name": "single_led_upsampled"} - hash_df = key_hash( - ( - (trodes_pos_v1 & upsampled) - .fetch1_dataframe(add_frame_ind=True) - .round(3) # float precision - ).to_dict() + df = ( + (trodes_pos_v1 & upsampled) + .fetch1_dataframe(add_frame_ind=True) + .round(3) + .sum() + .to_dict() ) - hash_exp = "5296e74dea2e5e68d39f81bc81723a12" - assert hash_df == hash_exp, "Dataframe differs from expected" + exp = { + "position_x": 230389.335, + "position_y": 295368.260, + "orientation": 4716.906, + "velocity_x": 1726.304, + "velocity_y": -1675.276, + "speed": 6257.273, + } + for k in exp: + assert ( + pytest.approx(df[k], rel=1e-3) == exp[k] + ), f"Value differs from expected: {k}" def test_trodes_video(sgp, trodes_pos_v1): diff --git a/tests/spikesorting/conftest.py b/tests/spikesorting/conftest.py index b60c57b16..ffecbd890 100644 --- a/tests/spikesorting/conftest.py +++ b/tests/spikesorting/conftest.py @@ -198,16 +198,6 @@ def is_uuid(text): return uuid_pattern.fullmatch(str(text)) is not None -def hash_sort_info(sort_info): - """Hashes attributes of a dj.Table object that are not randomly assigned.""" - no_str_uuid = { - k: v - for k, v in sort_info.fetch(as_dict=True)[0].items() - if not is_uuid(v) and k != "analysis_file_name" - } - return key_hash(no_str_uuid) - - @pytest.fixture(scope="session") def spike_v1_group(): from spyglass.spikesorting.analysis.v1 import group diff --git a/tests/spikesorting/test_curation.py b/tests/spikesorting/test_curation.py index 4048e9297..7d08c3f29 100644 --- a/tests/spikesorting/test_curation.py +++ b/tests/spikesorting/test_curation.py @@ -3,8 +3,6 @@ from spikeinterface import BaseSorting from spikeinterface.extractors.nwbextractors import NwbRecordingExtractor -from .conftest import hash_sort_info - def test_curation_rec(spike_v1, pop_curation): rec = spike_v1.CurationV1.get_recording(pop_curation) @@ -29,22 +27,90 @@ def test_curation_sort(spike_v1, pop_curation): assert isinstance( sort, BaseSorting ), "CurationV1.get_sorting failed to return a BaseSorting" - assert ( - key_hash(sort_dict) == "612983fbf4958f6b2c7abe7ced86ab73" - ), "CurationV1.get_sorting unexpected value" - assert ( - sort_dict["kwargs"]["spikes"].shape[0] == 918 - ), "CurationV1.get_sorting unexpected shape" + + expected = { + "class": "spikeinterface.core.numpyextractors.NumpySorting", + "module": "spikeinterface", + "relative_paths": False, + } + for k in expected: + assert ( + sort_dict[k] == expected[k] + ), f"CurationV1.get_sorting unexpected value: {k}" -def test_curation_sort_info(spike_v1, pop_curation, pop_curation_metric): - sort_info = spike_v1.CurationV1.get_sort_group_info(pop_curation) - sort_metric = spike_v1.CurationV1.get_sort_group_info(pop_curation_metric) +def test_curation_sort_info(spike_v1, pop_curation): + sort_info = spike_v1.CurationV1.get_sort_group_info(pop_curation).fetch1() + exp = { + "bad_channel": "False", + "curation_id": 0, + "description": "testing sort", + "electrode_group_name": "0", + "electrode_id": 0, + "filtering": "None", + "impedance": 0.0, + "merges_applied": 0, + "name": "0", + "nwb_file_name": "minirec20230622_.nwb", + "original_reference_electrode": 0, + "parent_curation_id": -1, + "probe_electrode": 0, + "probe_id": "tetrode_12.5", + "probe_shank": 0, + "region_id": 1, + "sort_group_id": 0, + "sorter": "mountainsort4", + "sorter_param_name": "franklab_tetrode_hippocampus_30KHz", + "subregion_name": None, + "subsubregion_name": None, + "x": 0.0, + "x_warped": 0.0, + "y": 0.0, + "y_warped": 0.0, + "z": 0.0, + "z_warped": 0.0, + } + for k in exp: + assert ( + sort_info[k] == exp[k] + ), f"CurationV1.get_sort_group_info unexpected value: {k}" - assert ( - hash_sort_info(sort_info) == "be874e806a482ed2677fd0d0b449f965" - ), "CurationV1.get_sort_group_info unexpected value" - assert ( - hash_sort_info(sort_metric) == "48e437bc116900fe64e492d74595b56d" - ), "CurationV1.get_sort_group_info unexpected value" +def test_curation_sort_metric(spike_v1, pop_curation, pop_curation_metric): + sort_metric = spike_v1.CurationV1.get_sort_group_info( + pop_curation_metric + ).fetch1() + expected = { + "bad_channel": "False", + "contacts": "", + "curation_id": 1, + "description": "after metric curation", + "electrode_group_name": "0", + "electrode_id": 0, + "filtering": "None", + "impedance": 0.0, + "merges_applied": 0, + "name": "0", + "nwb_file_name": "minirec20230622_.nwb", + "original_reference_electrode": 0, + "parent_curation_id": 0, + "probe_electrode": 0, + "probe_id": "tetrode_12.5", + "probe_shank": 0, + "region_id": 1, + "sort_group_id": 0, + "sorter": "mountainsort4", + "sorter_param_name": "franklab_tetrode_hippocampus_30KHz", + "subregion_name": None, + "subsubregion_name": None, + "x": 0.0, + "x_warped": 0.0, + "y": 0.0, + "y_warped": 0.0, + "z": 0.0, + "z_warped": 0.0, + } + for k in expected: + assert ( + sort_metric[k] == expected[k] + ), f"CurationV1.get_sort_group_info unexpected value: {k}" diff --git a/tests/spikesorting/test_merge.py b/tests/spikesorting/test_merge.py index 25751684c..5e0458789 100644 --- a/tests/spikesorting/test_merge.py +++ b/tests/spikesorting/test_merge.py @@ -2,8 +2,6 @@ from spikeinterface import BaseSorting from spikeinterface.extractors.nwbextractors import NwbRecordingExtractor -from .conftest import hash_sort_info - def test_merge_get_restr(spike_merge, pop_merge, pop_curation_metric): restr_id = spike_merge.get_restricted_merge_ids( @@ -34,10 +32,42 @@ def test_merge_get_sorting(spike_merge, pop_merge): def test_merge_get_sort_group_info(spike_merge, pop_merge): - hash = hash_sort_info(spike_merge.get_sort_group_info(pop_merge)) - assert ( - hash == "48e437bc116900fe64e492d74595b56d" - ), "SpikeSortingOutput.get_sort_group_info unexpected value" + sort_info = spike_merge.get_sort_group_info(pop_merge).fetch1() + expected = { + "bad_channel": "False", + "contacts": "", + "curation_id": 1, + "description": "after metric curation", + "electrode_group_name": "0", + "electrode_id": 0, + "filtering": "None", + "impedance": 0.0, + "merges_applied": 0, + "name": "0", + "nwb_file_name": "minirec20230622_.nwb", + "original_reference_electrode": 0, + "parent_curation_id": 0, + "probe_electrode": 0, + "probe_id": "tetrode_12.5", + "probe_shank": 0, + "region_id": 1, + "sort_group_id": 0, + "sorter": "mountainsort4", + "sorter_param_name": "franklab_tetrode_hippocampus_30KHz", + "subregion_name": None, + "subsubregion_name": None, + "x": 0.0, + "x_warped": 0.0, + "y": 0.0, + "y_warped": 0.0, + "z": 0.0, + "z_warped": 0.0, + } + + for k in expected: + assert ( + sort_info[k] == expected[k] + ), f"SpikeSortingOutput.get_sort_group_info unexpected value: {k}" @pytest.fixture(scope="session") From 448fffd8b916786c593dc6946ee8b4d9a3794bc5 Mon Sep 17 00:00:00 2001 From: Eric Denovellis Date: Thu, 3 Oct 2024 14:59:53 -0700 Subject: [PATCH 74/94] Remove mambaforge from tests (#1153) * Update test-conda.yml Mambaforge is being sunset (see https://conda-forge.org/news/2024/07/29/sunsetting-mambaforge/) because miniforge is now essentially the same. This PR removes the mambaforge variant and switches to using miniforge for the tests. * Update CHANGELOG.md --- .github/workflows/test-conda.yml | 1 - CHANGELOG.md | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-conda.yml b/.github/workflows/test-conda.yml index fd9245c8e..808417967 100644 --- a/.github/workflows/test-conda.yml +++ b/.github/workflows/test-conda.yml @@ -45,7 +45,6 @@ jobs: with: activate-environment: spyglass environment-file: environment.yml - miniforge-variant: Mambaforge miniforge-version: latest use-mamba: true - name: Install apt dependencies diff --git a/CHANGELOG.md b/CHANGELOG.md index a3946d5e6..9151ebb3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Fix dandi upload process for nwb's with video or linked objects #1095, #1151 - Minor docs fixes #1145 - Remove stored hashes from pytests #1152 +- Remove mambaforge from tests #1153 ### Pipelines From 03e39960e31bb358ba3f112220a5fdbb2e6acc76 Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Mon, 7 Oct 2024 09:08:28 -0500 Subject: [PATCH 75/94] DLC `make_video` minor fixes (#1150) * Pass output file name as str * Update changelog --- CHANGELOG.md | 1 + src/spyglass/position/v1/dlc_utils_makevid.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9151ebb3c..63aef8498 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,7 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Fix video directory bug in `DLCPoseEstimationSelection` #1103 - Restore #973, allow DLC without position tracking #1100 - Minor fix to `DLCCentroid` make function order #1112, #1148 + - Pass output path as string to `cv2.VideoWriter` #1150 - Spike Sorting diff --git a/src/spyglass/position/v1/dlc_utils_makevid.py b/src/spyglass/position/v1/dlc_utils_makevid.py index 5ee7c096a..994c027f0 100644 --- a/src/spyglass/position/v1/dlc_utils_makevid.py +++ b/src/spyglass/position/v1/dlc_utils_makevid.py @@ -134,7 +134,7 @@ def _init_video(self): def _init_cv_video(self): _ = self._init_video() self.out = cv2.VideoWriter( - filename=self.output_video_filename, + filename=str(self.output_video_filename), fourcc=cv2.VideoWriter_fourcc(*"mp4v"), fps=self.frame_rate, frameSize=self.frame_size, From 398927ada98775b76d1282b233ddb932718a2b2a Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Thu, 10 Oct 2024 16:01:07 -0500 Subject: [PATCH 76/94] Fix topological sort logic (#1162) * Fix topo sort * Update changelog --- CHANGELOG.md | 3 ++- src/spyglass/utils/dj_graph.py | 35 ++++++++++++++++++++++++++++++---- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63aef8498..0140d231c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,8 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Add docstrings to all public methods #1076 - Update DataJoint to 0.14.2 #1081 - Allow restriction based on parent keys in `Merge.fetch_nwb()` #1086, #1126 -- Import `datajoint.dependencies.unite_master_parts` -> `topo_sort` #1116, #1137 +- Import `datajoint.dependencies.unite_master_parts` -> `topo_sort` #1116, + #1137, #1162 - Fix bool settings imported from dj config file #1117 - Allow definition of tasks and new probe entries from config #1074, #1120 - Enforce match between ingested nwb probe geometry and existing table entry diff --git a/src/spyglass/utils/dj_graph.py b/src/spyglass/utils/dj_graph.py index a66a6ac5d..625202569 100644 --- a/src/spyglass/utils/dj_graph.py +++ b/src/spyglass/utils/dj_graph.py @@ -17,6 +17,7 @@ from datajoint.user_tables import TableMeta from datajoint.utils import get_master, to_camel_case from networkx import ( + DiGraph, NetworkXNoPath, NodeNotFound, all_simple_paths, @@ -33,10 +34,36 @@ unique_dicts, ) -try: # Datajoint 0.14.2+ uses topo_sort instead of unite_master_parts - from datajoint.dependencies import topo_sort as dj_topo_sort -except ImportError: - from datajoint.dependencies import unite_master_parts as dj_topo_sort + +def dj_topo_sort(graph: DiGraph) -> List[str]: + """Topologically sort graph. + + Uses datajoint's topo_sort if available, otherwise uses networkx's + topological_sort, combined with datajoint's unite_master_parts. + + NOTE: This ordering will impact _hash_upstream, but usage should be + consistent before/after a no-transaction populate. + + Parameters + ---------- + graph : nx.DiGraph + Directed graph to sort + + Returns + ------- + List[str] + List of table names in topological order + """ + __import__("pdb").set_trace() + try: # Datajoint 0.14.2+ uses topo_sort instead of unite_master_parts + from datajoint.dependencies import topo_sort + + return topo_sort(graph) + except ImportError: + from datajoint.dependencies import unite_master_parts + from networkx.algorithms.dag import topological_sort + + return unite_master_parts(list(topological_sort(graph))) class Direction(Enum): From 6be6b71ce9068c9853844a1a369893b2956905bb Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Tue, 15 Oct 2024 14:09:15 -0500 Subject: [PATCH 77/94] Fix tests (#1165) * Fix tests, run test on PR * Update changelog * Inherit secrets in PR tests run * Revert run on PR --- .github/workflows/test-conda.yml | 10 +++++----- CHANGELOG.md | 6 ++++-- src/spyglass/utils/dj_graph.py | 1 - 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-conda.yml b/.github/workflows/test-conda.yml index 808417967..d34e83c01 100644 --- a/.github/workflows/test-conda.yml +++ b/.github/workflows/test-conda.yml @@ -2,7 +2,7 @@ name: Tests on: push: - branches: + branches: - '!test_branch' - '!documentation' schedule: # once a day at midnight UTC @@ -53,7 +53,7 @@ jobs: sudo apt-get install mysql-client libmysqlclient-dev libgirepository1.0-dev -y sudo apt-get install ffmpeg libsm6 libxext6 -y # non-dlc position deps - name: Run pip install for test deps - run: | + run: | pip install --quiet .[test] - name: Download data env: @@ -61,8 +61,8 @@ jobs: NWBFILE: minirec20230622.nwb # Relative to Base URL VID_ONE: 20230622_sample_01_a1/20230622_sample_01_a1.1.h264 VID_TWO: 20230622_sample_02_a1/20230622_sample_02_a1.1.h264 - RAW_DIR: /home/runner/work/spyglass/spyglass/tests/_data/raw/ - VID_DIR: /home/runner/work/spyglass/spyglass/tests/_data/video/ + RAW_DIR: /home/runner/work/spyglass/spyglass/tests/_data/raw/ + VID_DIR: /home/runner/work/spyglass/spyglass/tests/_data/video/ run: | mkdir -p $RAW_DIR $VID_DIR wget_opts() { # Declare func with download options @@ -76,4 +76,4 @@ jobs: wget_opts $VID_DIR $VID_TWO - name: Run tests run: | - pytest --no-docker --no-dlc + pytest --no-docker --no-dlc diff --git a/CHANGELOG.md b/CHANGELOG.md index 0140d231c..2c3fbe4de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,8 +30,10 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Update DataJoint install and password instructions #1131 - Fix dandi upload process for nwb's with video or linked objects #1095, #1151 - Minor docs fixes #1145 -- Remove stored hashes from pytests #1152 -- Remove mambaforge from tests #1153 +- Test fixes + - Remove stored hashes from pytests #1152 + - Remove mambaforge from tests #1153 + - Remove debug statement #1164 ### Pipelines diff --git a/src/spyglass/utils/dj_graph.py b/src/spyglass/utils/dj_graph.py index 625202569..435d37d38 100644 --- a/src/spyglass/utils/dj_graph.py +++ b/src/spyglass/utils/dj_graph.py @@ -54,7 +54,6 @@ def dj_topo_sort(graph: DiGraph) -> List[str]: List[str] List of table names in topological order """ - __import__("pdb").set_trace() try: # Datajoint 0.14.2+ uses topo_sort instead of unite_master_parts from datajoint.dependencies import topo_sort From e57638e3c928c816a8f9361100b49021f77cb832 Mon Sep 17 00:00:00 2001 From: Eric Denovellis Date: Wed, 23 Oct 2024 10:10:52 -0700 Subject: [PATCH 78/94] Allow python <3.13 and remove numpy pin (#1169) * Increase upper bound of python to include 3.12 * Add tests for different python versions * Minor formatting * Use quotes for version to avoid parsing errors * Ensure build dependencies are installed distutils was removed for python 3.12 * Add more build dependencies * Try different resources * Temp test on <3.12 * Remove importlib_metadata pin Fixed in twine https://github.com/pypa/twine/issues/1125 * Revert "Temp test on <3.12" This reverts commit 5b2736651928e01e38816a4a7f479415ea41fc74. * Change install order of build dependencies * Remove numpy pin * Update CHANGELOG.md --- .github/workflows/test-package-build.yml | 10 ++++--- CHANGELOG.md | 3 ++ environment.yml | 4 +-- environment_dlc.yml | 4 +-- pyproject.toml | 36 ++++++++++++------------ 5 files changed, 31 insertions(+), 26 deletions(-) diff --git a/.github/workflows/test-package-build.yml b/.github/workflows/test-package-build.yml index 3513bb664..e07017a31 100644 --- a/.github/workflows/test-package-build.yml +++ b/.github/workflows/test-package-build.yml @@ -29,7 +29,6 @@ jobs: python-version: 3.9 - run: | pip install --upgrade build twine - pip install importlib_metadata==7.2.1 # twine #977 - name: Build sdist and wheel run: python -m build - run: twine check dist/* @@ -50,6 +49,7 @@ jobs: needs: [build] strategy: matrix: + python-version: ['3.9', '3.10', '3.11', '3.12'] package: ['wheel', 'sdist', 'archive'] steps: - name: Download sdist and wheel artifacts @@ -66,11 +66,13 @@ jobs: path: archive/ - uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: ${{ matrix.python-version }} - name: Display Python version run: python -c "import sys; print(sys.version)" - - name: Update pip - run: pip install --upgrade pip + - name: Install build dependencies + run: | + pip install --upgrade setuptools wheel + pip install --upgrade pip - name: Install wheel if: matrix.package == 'wheel' run: pip install dist/*.whl diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c3fbe4de..84d6d2c11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,9 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Remove stored hashes from pytests #1152 - Remove mambaforge from tests #1153 - Remove debug statement #1164 +- Allow python < 3.13 #1169 +- Remove numpy version restriction #1169 +- Add testing for python versions 3.9, 3.10, 3.11, 3.12 #1169 ### Pipelines diff --git a/environment.yml b/environment.yml index 7fa1b51ea..a5229b8a8 100644 --- a/environment.yml +++ b/environment.yml @@ -23,13 +23,13 @@ dependencies: # - libgcc # dlc-only - matplotlib - non_local_detector - - numpy<1.24 + - numpy - pip - position_tools - pybind11 # req by mountainsort4 -> isosplit5 - pydotplus - pyfftw<=0.12.0 # ghostipy req. install from conda-forge for Mac ARM - - python>=3.9,<3.10 + - python>=3.9,<3.13 - pytorch<1.12.0 - ripple_detection - seaborn diff --git a/environment_dlc.yml b/environment_dlc.yml index 9870a0424..156bed793 100644 --- a/environment_dlc.yml +++ b/environment_dlc.yml @@ -23,13 +23,13 @@ dependencies: - libgcc # dlc-only - matplotlib - non_local_detector - - numpy<1.24 + - numpy - pip>=20.2.* - position_tools - pybind11 # req by mountainsort4 -> isosplit5 - pydotplus>=2.0.* - pyfftw<=0.12.0 # ghostipy req. install from conda-forge for Mac ARM - - python>=3.9,<3.10 + - python>=3.9,<3.13 - pytorch<1.12.0 - ripple_detection - seaborn diff --git a/pyproject.toml b/pyproject.toml index ed3a570c8..0bd9164cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "hatchling.build" name = "spyglass-neuro" description = "Neuroscience data analysis framework for reproducible research" readme = "README.md" -requires-python = ">=3.9,<3.10" +requires-python = ">=3.9,<3.13" license = { file = "LICENSE" } authors = [ { name = "Loren Frank", email = "loren.frank@ucsf.edu" }, @@ -46,11 +46,11 @@ dependencies = [ "matplotlib", "ndx_franklab_novela>=0.1.0", "non_local_detector", - "numpy<1.24", + "numpy", "opencv-python", - "panel>=1.4.0", # panel #6325 resolved + "panel>=1.4.0", # panel #6325 resolved "position_tools>=0.1.0", - "pubnub<6.4.0", # TODO: remove this when sortingview is updated + "pubnub<6.4.0", # TODO: remove this when sortingview is updated "pydotplus", "pynwb>=2.2.0,<3", "ripple_detection", @@ -62,21 +62,21 @@ dependencies = [ [project.optional-dependencies] dlc = [ - "ffmpeg", - "deeplabcut[tf]", # removing dlc pin removes need to pin tf/numba + "ffmpeg", + "deeplabcut[tf]", # removing dlc pin removes need to pin tf/numba ] test = [ - "click", # for CLI subpackage only - "docker", # for tests in a container + "click", # for CLI subpackage only + "docker", # for tests in a container "ghostipy", - "kachery", # database access + "kachery", # database access "kachery-client", "kachery-cloud>=0.4.0", "opencv-python-headless", # for headless testing of Qt - "pre-commit", # linting - "pytest", # unit testing - "pytest-cov", # code coverage - "pytest-xvfb", # for headless testing of Qt + "pre-commit", # linting + "pytest", # unit testing + "pytest-cov", # code coverage + "pytest-xvfb", # for headless testing of Qt ] docs = [ "hatch", # Get version from env @@ -134,7 +134,7 @@ addopts = [ # "--no-dlc", # don't run DLC tests "--show-capture=no", "--pdbcls=IPython.terminal.debugger:TerminalPdb", # use ipython debugger - "--doctest-modules", # run doctests in all modules + "--doctest-modules", # run doctests in all modules "--cov=spyglass", "--cov-report=term-missing", "--no-cov-on-fail", @@ -143,9 +143,9 @@ testpaths = ["tests"] log_level = "INFO" env = [ "QT_QPA_PLATFORM = offscreen", # QT fails headless without this - "DISPLAY = :0", # QT fails headless without this - "TF_ENABLE_ONEDNN_OPTS = 0", # TF disable approx calcs - "TF_CPP_MIN_LOG_LEVEL = 2", # Disable TF warnings + "DISPLAY = :0", # QT fails headless without this + "TF_ENABLE_ONEDNN_OPTS = 0", # TF disable approx calcs + "TF_CPP_MIN_LOG_LEVEL = 2", # Disable TF warnings ] [tool.coverage.run] @@ -175,4 +175,4 @@ omit = [ # which submodules have no tests line-length = 80 [tool.ruff.lint] -ignore = ["F401" , "E402", "E501"] +ignore = ["F401", "E402", "E501"] From 97f1a215691b5f194d3ad6753cb77926cce1a8e0 Mon Sep 17 00:00:00 2001 From: Samuel Bray Date: Fri, 25 Oct 2024 14:38:31 -0700 Subject: [PATCH 79/94] No transact insert v0 lfp (#1172) * no transact insert v0 lfp * update changelog --- CHANGELOG.md | 2 +- src/spyglass/common/common_ephys.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 84d6d2c11..9530fa9de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() ### Infrastructure - Disable populate transaction protection for long-populating tables #1066, - #1108 + #1108, #1172 - Add docstrings to all public methods #1076 - Update DataJoint to 0.14.2 #1081 - Allow restriction based on parent keys in `Merge.fetch_nwb()` #1086, #1126 diff --git a/src/spyglass/common/common_ephys.py b/src/spyglass/common/common_ephys.py index 2e56d47fa..c25fa09ef 100644 --- a/src/spyglass/common/common_ephys.py +++ b/src/spyglass/common/common_ephys.py @@ -460,6 +460,8 @@ class LFP(SpyglassMixin, dj.Imported): lfp_sampling_rate: float # the sampling rate, in HZ """ + _use_transaction, _allow_insert = False, True + def make(self, key): """Populate the LFP table with data from the NWB file. From 6b4ff10b3b6976cab15ba7df29e23a7339fc9179 Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Tue, 29 Oct 2024 15:33:52 -0700 Subject: [PATCH 80/94] `DLCPosVideo`: Use provided epoch, multithread matplotlib (#1168) * Use provided epoch * Save video to temp dir * Remove open-cv support * WIP: Multithread, RAM hungry * Limit number of workers * Save file images in batches * Reduce RAM cost, remove cv2 dep * Update changelog * Get debug arg from params * Revert merge error #870, #975 * Adjust for final frame. Resume from existing * Resume from fail * except IndexError for final frame * Delay delete temp files until complete * Explicit error messages * Return video object for debugging --- .gitignore | 1 + CHANGELOG.md | 6 +- src/spyglass/common/common_behav.py | 2 +- src/spyglass/position/v1/dlc_utils_makevid.py | 690 +++++++++--------- .../position/v1/position_dlc_selection.py | 30 +- 5 files changed, 351 insertions(+), 378 deletions(-) diff --git a/.gitignore b/.gitignore index 0cbd43c74..032ec4f7c 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ spyglass.code-workspace mountainsort4_output/ .idea/ mysql_config +memray* # Notebooks *.ipynb diff --git a/CHANGELOG.md b/CHANGELOG.md index 9530fa9de..3abcc5861 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,7 +59,11 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Fix video directory bug in `DLCPoseEstimationSelection` #1103 - Restore #973, allow DLC without position tracking #1100 - Minor fix to `DLCCentroid` make function order #1112, #1148 - - Pass output path as string to `cv2.VideoWriter` #1150 + - Video creator tools: + - Pass output path as string to `cv2.VideoWriter` #1150 + - Set `DLCPosVideo` default processor to `matplotlib`, remove support + for `open-cv` #1168 + - `VideoMaker` class to process frames in multithreaded batches #1168 - Spike Sorting diff --git a/src/spyglass/common/common_behav.py b/src/spyglass/common/common_behav.py index 22afaf8c7..bab4ba075 100644 --- a/src/spyglass/common/common_behav.py +++ b/src/spyglass/common/common_behav.py @@ -541,7 +541,7 @@ def _no_transaction_make(self, key): # Skip populating if no pos interval list names if len(pos_intervals) == 0: - logger.error(f"NO POS INTERVALS FOR {key}; {no_pop_msg}") + logger.error(f"NO POS INTERVALS FOR {key};\n{no_pop_msg}") self.insert1(null_key, **insert_opts) return diff --git a/src/spyglass/position/v1/dlc_utils_makevid.py b/src/spyglass/position/v1/dlc_utils_makevid.py index 994c027f0..51c209134 100644 --- a/src/spyglass/position/v1/dlc_utils_makevid.py +++ b/src/spyglass/position/v1/dlc_utils_makevid.py @@ -1,16 +1,21 @@ # Convenience functions + # some DLC-utils copied from datajoint element-interface utils.py +import shutil +import subprocess +from concurrent.futures import ProcessPoolExecutor, as_completed +from os import system as os_system from pathlib import Path +from typing import Tuple -import cv2 import matplotlib.pyplot as plt import numpy as np import pandas as pd -from tqdm import tqdm as tqdm +from tqdm import tqdm +from spyglass.settings import temp_dir from spyglass.utils import logger from spyglass.utils.position import convert_to_pixels as _to_px -from spyglass.utils.position import fill_nan RGB_PINK = (234, 82, 111) RGB_YELLOW = (253, 231, 76) @@ -37,328 +42,145 @@ def __init__( position_time, video_frame_inds=None, likelihoods=None, - processor="opencv", # opencv, opencv-trodes, matplotlib - video_time=None, + processor="matplotlib", frames=None, percent_frames=1, output_video_filename="output.mp4", cm_to_pixels=1.0, disable_progressbar=False, crop=None, - arrow_radius=15, - circle_radius=8, + batch_size=500, + max_workers=25, + max_jobs_in_queue=250, + debug=False, + key_hash=None, + *args, + **kwargs, ): + """Create a video from a set of position data. + + Uses batch size as frame count for processing steps. All in temp_dir. + 1. Extract frames from original video to 'orig_XXXX.png' + 2. Multithread pool frames to matplotlib 'plot_XXXX.png' + 3. Stitch frames into partial video 'partial_XXXX.mp4' + 4. Concatenate partial videos into final video output + + """ + if processor != "matplotlib": + raise ValueError( + "open-cv processors are no longer supported. \n" + + "Use matplotlib or submit a feature request via GitHub." + ) + + # key_hash supports resume from previous run + self.temp_dir = Path(temp_dir) / f"dlc_vid_{key_hash}" + self.temp_dir.mkdir(parents=True, exist_ok=True) + logger.debug(f"Temporary directory: {self.temp_dir}") + + if not Path(video_filename).exists(): + raise FileNotFoundError(f"Video not found: {video_filename}") + self.video_filename = video_filename self.video_frame_inds = video_frame_inds - self.position_mean = position_mean - self.orientation_mean = orientation_mean + self.position_mean = position_mean["DLC"] + self.orientation_mean = orientation_mean["DLC"] self.centroids = centroids self.likelihoods = likelihoods self.position_time = position_time - self.processor = processor - self.video_time = video_time - self.frames = frames self.percent_frames = percent_frames + self.frames = frames self.output_video_filename = output_video_filename self.cm_to_pixels = cm_to_pixels - self.disable_progressbar = disable_progressbar self.crop = crop - self.arrow_radius = arrow_radius - self.circle_radius = circle_radius + self.window_ind = np.arange(501) - 501 // 2 + self.debug = debug - if not Path(self.video_filename).exists(): - raise FileNotFoundError(f"Video not found: {self.video_filename}") + self.dropped_frames = set() - if frames is None: - self.n_frames = ( - int(self.orientation_mean.shape[0]) - if processor == "opencv-trodes" - else int(len(video_frame_inds) * percent_frames) - ) - self.frames = np.arange(0, self.n_frames) - else: - self.n_frames = len(frames) - - self.tqdm_kwargs = { - "iterable": ( - range(self.n_frames - 1) - if self.processor == "opencv-trodes" - else self.frames - ), - "desc": "frames", - "disable": self.disable_progressbar, - } + self.batch_size = batch_size + self.max_workers = max_workers + self.max_jobs_in_queue = max_jobs_in_queue - # init for cv - self.video, self.frame_size = None, None - self.frame_rate, self.out = None, None - self.source_map = { - "DLC": RGB_BLUE, - "Trodes": RGB_ORANGE, - "Common": RGB_PINK, - } + self.ffmpeg_log_args = ["-hide_banner", "-loglevel", "error"] + self.ffmpeg_fmt_args = ["-c:v", "libx264", "-pix_fmt", "yuv420p"] - # intit for matplotlib - self.image, self.title, self.progress_bar = None, None, None - self.crop_offset_x = crop[0] if crop else 0 - self.crop_offset_y = crop[2] if crop else 0 - self.centroid_plot_objs, self.centroid_position_dot = None, None - self.orientation_line = None - self.likelihood_objs = None - self.window_ind = np.arange(501) - 501 // 2 + _ = self._set_frame_info() + _ = self._set_plot_bases() - self.make_video() + logger.info( + f"Making video: {self.output_video_filename} " + + f"in batches of {self.batch_size}" + ) + self.process_frames() + plt.close(self.fig) + logger.info(f"Finished video: {self.output_video_filename}") + logger.debug(f"Dropped frames: {self.dropped_frames}") + + shutil.rmtree(self.temp_dir) # Clean up temp directory - def make_video(self): - """Make video based on processor chosen at init.""" - if self.processor == "opencv": - self.make_video_opencv() - elif self.processor == "opencv-trodes": - self.make_trodes_video() - elif self.processor == "matplotlib": - self.make_video_matplotlib() + def _set_frame_info(self): + """Set the frame information for the video.""" + logger.debug("Setting frame information") - def _init_video(self): - logger.info(f"Making video: {self.output_video_filename}") - self.video = cv2.VideoCapture(str(self.video_filename)) + width, height, self.frame_rate = self._get_input_stats() self.frame_size = ( - (int(self.video.get(3)), int(self.video.get(4))) + (width, height) if not self.crop else ( self.crop[1] - self.crop[0], self.crop[3] - self.crop[2], ) ) - self.frame_rate = self.video.get(5) - - def _init_cv_video(self): - _ = self._init_video() - self.out = cv2.VideoWriter( - filename=str(self.output_video_filename), - fourcc=cv2.VideoWriter_fourcc(*"mp4v"), - fps=self.frame_rate, - frameSize=self.frame_size, - isColor=True, - ) - frames_log = ( - f"\tFrames start: {self.frames[0]}\n" if np.any(self.frames) else "" - ) - inds_log = ( - f"\tVideo frame inds: {self.video_frame_inds[0]}\n" - if np.any(self.video_frame_inds) - else "" - ) - logger.info( - f"\n{frames_log}{inds_log}\tcv2 ind start: {int(self.video.get(1))}" - ) - - def _close_cv_video(self): - self.video.release() - self.out.release() - try: - cv2.destroyAllWindows() - except cv2.error: # if cv is already closed or does not have func - pass - logger.info(f"Finished video: {self.output_video_filename}") - - def _get_frame(self, frame, init_only=False, crop_order=(0, 1, 2, 3)): - frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - if init_only or not self.crop: - return frame - x1, x2, y1, y2 = self.crop_order - return frame[ - self.crop[x1] : self.crop[x2], self.crop[y1] : self.crop[y2] - ].copy() - - def _video_set_by_ind(self, time_ind): - if time_ind == 0: - self.video.set(1, time_ind + 1) - elif int(self.video.get(1)) != time_ind - 1: - self.video.set(1, time_ind - 1) - - def _all_num(self, *args): - return all(np.all(~np.isnan(data)) for data in args) - - def _make_arrow( - self, - position, - orientation, - color, - img, - thickness=4, - line_type=8, - tipLength=0.25, - shift=cv2.CV_8U, - ): - if not self._all_num(position, orientation): - return - arrow_tip = ( - int(position[0] + self.arrow_radius * np.cos(orientation)), - int(position[1] + self.arrow_radius * np.sin(orientation)), - ) - cv2.arrowedLine( - img=img, - pt1=tuple(position.astype(int)), - pt2=arrow_tip, - color=color, - thickness=thickness, - line_type=line_type, - tipLength=tipLength, - shift=shift, - ) - - def _make_circle( - self, - data, - color, - img, - radius=None, - thickness=-1, - shift=cv2.CV_8U, - **kwargs, - ): - if not self._all_num(data): - return - cv2.circle( - img=img, - center=tuple(data.astype(int)), - radius=radius or self.circle_radius, - color=color, - thickness=thickness, - shift=shift, + self.ratio = ( + (self.crop[3] - self.crop[2]) / (self.crop[1] - self.crop[0]) + if self.crop + else self.frame_size[1] / self.frame_size[0] ) + self.fps = int(np.round(self.frame_rate)) - def make_video_opencv(self): - """Make video using opencv.""" - _ = self._init_cv_video() - - if self.video_time: - self.position_mean = { - key: fill_nan( - self.position_mean[key]["position"], - self.video_time, - self.position_time, - ) - for key in self.position_mean.keys() - } - self.orientation_mean = { - key: fill_nan( - self.position_mean[key]["orientation"], - self.video_time, - self.position_time, - ) - for key in self.position_mean.keys() - } - - for time_ind in tqdm(**self.tqdm_kwargs): - _ = self._video_set_by_ind(time_ind) - - is_grabbed, frame = self.video.read() - - if not is_grabbed: - break - - frame = self._get_frame(frame) - - cv2.putText( - img=frame, - text=f"time_ind: {int(time_ind)} video frame: {int(self.video.get(1))}", - org=(10, 10), - fontFace=cv2.FONT_HERSHEY_SIMPLEX, - fontScale=0.5, - color=RGB_YELLOW, - thickness=1, - ) - - if time_ind < self.video_frame_inds[0] - 1: - self.out.write(self._get_frame(frame, init_only=True)) - continue - - pos_ind = time_ind - self.video_frame_inds[0] - - for key in self.position_mean: - position = _to_px( - data=self.position_mean[key][pos_ind], - cm_to_pixels=self.cm_to_pixels, - ) - orientation = self.orientation_mean[key][pos_ind] - cv_kwargs = { - "img": frame, - "color": self.source_map[key], - } - self._make_arrow(position, orientation, **cv_kwargs) - self._make_circle(data=position, **cv_kwargs) - - self._get_frame(frame, init_only=True) - self.out.write(frame) - self._close_cv_video() - return - - def make_trodes_video(self): - """Make video using opencv with trodes data.""" - _ = self._init_cv_video() - - if np.any(self.video_time): - centroids = { - color: fill_nan( - variable=data, - video_time=self.video_time, - variable_time=self.position_time, - ) - for color, data in self.centroids.items() - } - position_mean = fill_nan( - self.position_mean, self.video_time, self.position_time - ) - orientation_mean = fill_nan( - self.orientation_mean, self.video_time, self.position_time - ) - - for time_ind in tqdm(**self.tqdm_kwargs): - is_grabbed, frame = self.video.read() - if not is_grabbed: - break - - frame = self._get_frame(frame) - - red_centroid = centroids["red"][time_ind] - green_centroid = centroids["green"][time_ind] - position = position_mean[time_ind] - position = _to_px(data=position, cm_to_pixels=self.cm_to_pixels) - orientation = orientation_mean[time_ind] - - self._make_circle(data=red_centroid, img=frame, color=RGB_YELLOW) - self._make_circle(data=green_centroid, img=frame, color=RGB_PINK) - self._make_arrow( - position=position, - orientation=orientation, - color=RGB_WHITE, - img=frame, + if self.frames is None: + self.n_frames = int( + len(self.video_frame_inds) * self.percent_frames ) - self._make_circle(data=position, img=frame, color=RGB_WHITE) - self._get_frame(frame, init_only=True) - self.out.write(frame) - - self._close_cv_video() - - def make_video_matplotlib(self): - """Make video using matplotlib.""" - import matplotlib.animation as animation - - self.position_mean = self.position_mean["DLC"] - self.orientation_mean = self.orientation_mean["DLC"] - - _ = self._init_video() + self.frames = np.arange(0, self.n_frames) + else: + self.n_frames = len(self.frames) + self.pad_len = len(str(self.n_frames)) + + def _get_input_stats(self, video_filename=None) -> Tuple[int, int]: + """Get the width and height of the video.""" + logger.debug("Getting video dimensions") + + video_filename = video_filename or self.video_filename + ret = subprocess.run( + [ + "ffprobe", + "-v", + "error", + "-select_streams", + "v", + "-show_entries", + "stream=width,height,r_frame_rate", + "-of", + "csv=p=0:s=x", + video_filename, + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if ret.returncode != 0: + raise ValueError(f"Error getting video dimensions: {ret.stderr}") - video_slowdown = 1 - fps = int(np.round(self.frame_rate / video_slowdown)) - Writer = animation.writers["ffmpeg"] - writer = Writer(fps=fps, bitrate=-1) + stats = ret.stdout.strip().split("x") + width, height = tuple(map(int, stats[:-1])) + frame_rate = eval(stats[-1]) - ret, frame = self.video.read() - frame = self._get_frame(frame, crop_order=(2, 3, 0, 1)) + return width, height, frame_rate - frame_ind = 0 + def _set_plot_bases(self): + """Create the figure and axes for the video.""" + logger.debug("Setting plot bases") plt.style.use("dark_background") fig, axes = plt.subplots( 2, @@ -371,9 +193,6 @@ def make_video_matplotlib(self): axes[0].tick_params(colors="white", which="both") axes[0].spines["bottom"].set_color("white") axes[0].spines["left"].set_color("white") - self.image = axes[0].imshow(frame, animated=True) - - logger.info(f"frame after init plot: {self.video.get(1)}") self.centroid_plot_objs = { bodypart: axes[0].scatter( @@ -383,7 +202,7 @@ def make_video_matplotlib(self): zorder=102, color=color, label=f"{bodypart} position", - animated=True, + # animated=True, alpha=0.6, ) for color, bodypart in zip(COLOR_SWATCH, self.centroids.keys()) @@ -395,7 +214,6 @@ def make_video_matplotlib(self): zorder=102, color="#b045f3", label="centroid position", - animated=True, alpha=0.6, ) (self.orientation_line,) = axes[0].plot( @@ -403,23 +221,18 @@ def make_video_matplotlib(self): [], color="cyan", linewidth=1, - animated=True, label="Orientation", ) axes[0].set_xlabel("") axes[0].set_ylabel("") - ratio = ( - (self.crop[3] - self.crop[2]) / (self.crop[1] - self.crop[0]) - if self.crop - else self.frame_size[1] / self.frame_size[0] - ) - x_left, x_right = axes[0].get_xlim() y_low, y_high = axes[0].get_ylim() - axes[0].set_aspect(abs((x_right - x_left) / (y_low - y_high)) * ratio) + axes[0].set_aspect( + abs((x_right - x_left) / (y_low - y_high)) * self.ratio + ) axes[0].spines["top"].set_color("black") axes[0].spines["right"].set_color("black") @@ -429,7 +242,7 @@ def make_video_matplotlib(self): axes[0].legend(loc="lower right", fontsize=4) self.title = axes[0].set_title( - f"time = {time_delta:3.4f}s\n frame = {frame_ind}", + f"time = {time_delta:3.4f}s\n frame = {0}", fontsize=8, ) axes[0].axis("off") @@ -441,7 +254,6 @@ def make_video_matplotlib(self): [], color=color, linewidth=1, - animated=True, clip_on=False, label=bodypart, )[0] @@ -463,21 +275,8 @@ def make_video_matplotlib(self): axes[1].spines["right"].set_color("black") axes[1].legend(loc="upper right", fontsize=4) - self.progress_bar = tqdm(leave=True, position=0) - self.progress_bar.reset(total=self.n_frames) - - movie = animation.FuncAnimation( - fig, - self._update_plot, - frames=self.frames, - interval=1000 / fps, - blit=True, - ) - movie.save(self.output_video_filename, writer=writer, dpi=400) - self.video.release() - plt.style.use("default") - logger.info("finished making video with matplotlib") - return + self.fig = fig + self.axes = axes def _get_centroid_data(self, pos_ind): def centroid_to_px(*idx): @@ -504,64 +303,231 @@ def orient_list(c): c0, c1 = self._get_centroid_data(pos_ind) self.orientation_line.set_data(orient_list(c0), orient_list(c1)) - def _update_plot(self, time_ind, *args): - _ = self._video_set_by_ind(time_ind) - - ret, frame = self.video.read() - if ret: - frame = self._get_frame(frame, crop_order=(2, 3, 0, 1)) - self.image.set_array(frame) + def _generate_single_frame(self, frame_ind): + """Generate a single frame and save it as an image.""" + padded = self._pad(frame_ind) + frame_file = self.temp_dir / f"orig_{padded}.png" + if not frame_file.exists(): + self.dropped_frames.add(frame_ind) + print(f"\rFrame not found: {frame_file}", end="") + return + frame = plt.imread(frame_file) + _ = self.axes[0].imshow(frame) - pos_ind = np.where(self.video_frame_inds == time_ind)[0] + pos_ind = np.where(self.video_frame_inds == frame_ind)[0] if len(pos_ind) == 0: self.centroid_position_dot.set_offsets((np.NaN, np.NaN)) for bodypart in self.centroid_plot_objs.keys(): self.centroid_plot_objs[bodypart].set_offsets((np.NaN, np.NaN)) self.orientation_line.set_data((np.NaN, np.NaN)) - self.title.set_text(f"time = {0:3.4f}s\n frame = {time_ind}") - self.progress_bar.update() - return - - pos_ind = pos_ind[0] - likelihood_inds = pos_ind + self.window_ind - # initial implementation did not cover case of both neg and over < 0 - neg_inds = np.where(likelihood_inds < 0)[0] - likelihood_inds[neg_inds] = 0 if len(neg_inds) > 0 else -1 + self.title.set_text(f"time = {0:3.4f}s\n frame = {frame_ind}") + else: + pos_ind = pos_ind[0] + likelihood_inds = pos_ind + self.window_ind + neg_inds = np.where(likelihood_inds < 0)[0] + likelihood_inds[neg_inds] = 0 if len(neg_inds) > 0 else -1 + + dlc_centroid_data = self._get_centroid_data(pos_ind) + + for bodypart in self.centroid_plot_objs: + self.centroid_plot_objs[bodypart].set_offsets( + _to_px( + data=self.centroids[bodypart][pos_ind], + cm_to_pixels=self.cm_to_pixels, + ) + ) + self.centroid_position_dot.set_offsets(dlc_centroid_data) + _ = self._set_orient_line(frame, pos_ind) - dlc_centroid_data = self._get_centroid_data(pos_ind) + time_delta = pd.Timedelta( + pd.to_datetime(self.position_time[pos_ind] * 1e9, unit="ns") + - pd.to_datetime(self.position_time[0] * 1e9, unit="ns") + ).total_seconds() - for bodypart in self.centroid_plot_objs: - self.centroid_plot_objs[bodypart].set_offsets( - _to_px( - data=self.centroids[bodypart][pos_ind], - cm_to_pixels=self.cm_to_pixels, - ) + self.title.set_text( + f"time = {time_delta:3.4f}s\n frame = {frame_ind}" ) - self.centroid_position_dot.set_offsets(dlc_centroid_data) - _ = self._set_orient_line(frame, pos_ind) + if self.likelihoods: + for bodypart in self.likelihood_objs.keys(): + self.likelihood_objs[bodypart].set_data( + self.window_ind / self.frame_rate, + np.asarray(self.likelihoods[bodypart][likelihood_inds]), + ) - time_delta = pd.Timedelta( - pd.to_datetime(self.position_time[pos_ind] * 1e9, unit="ns") - - pd.to_datetime(self.position_time[0] * 1e9, unit="ns") - ).total_seconds() + # Zero-padded filename based on the dynamic padding length + frame_path = self.temp_dir / f"plot_{padded}.png" + self.fig.savefig(frame_path, dpi=400) + plt.cla() # clear the current axes - self.title.set_text(f"time = {time_delta:3.4f}s\n frame = {time_ind}") - for bodypart in self.likelihood_objs.keys(): - self.likelihood_objs[bodypart].set_data( - self.window_ind / self.frame_rate, - np.asarray(self.likelihoods[bodypart][likelihood_inds]), + return frame_ind + + def process_frames(self): + """Process video frames in batches and generate matplotlib frames.""" + + progress_bar = tqdm(leave=True, position=0, disable=self.debug) + progress_bar.reset(total=self.n_frames) + + for start_frame in range(0, self.n_frames, self.batch_size): + if start_frame >= self.n_frames: # Skip if no frames left + break + end_frame = min(start_frame + self.batch_size, self.n_frames) - 1 + logger.debug(f"Processing frames: {start_frame} - {end_frame}") + + output_partial_video = ( + self.temp_dir / f"partial_{self._pad(start_frame)}.mp4" ) - self.progress_bar.update() + if output_partial_video.exists(): + logger.debug(f"Skipping existing video: {output_partial_video}") + progress_bar.update(end_frame - start_frame) + continue - return ( - self.image, - self.centroid_position_dot, - self.orientation_line, - self.title, - ) + self.ffmpeg_extract(start_frame, end_frame) + self.plot_frames(start_frame, end_frame, progress_bar) + self.ffmpeg_stitch_partial(start_frame, str(output_partial_video)) + + for frame_file in self.temp_dir.glob("*.png"): + frame_file.unlink() # Delete orig and plot frames + + progress_bar.close() + + logger.info("Concatenating partial videos") + self.concat_partial_videos() + + def _debug_print(self, msg=None, end=""): + """Print a self-overwiting message if debug is enabled.""" + if self.debug: + print(f"\r{msg}", end=end) + + def plot_frames(self, start_frame, end_frame, progress_bar=None): + logger.debug(f"Plotting frames: {start_frame} - {end_frame}") + with ProcessPoolExecutor(max_workers=self.max_workers) as executor: + jobs = {} # dict of jobs + + frames_left = end_frame - start_frame + frames_iter = iter(range(start_frame, end_frame)) + + while frames_left: + while len(jobs) < self.max_jobs_in_queue: + try: + this_frame = next(frames_iter) + self._debug_print(f"Submit: {this_frame}") + job = executor.submit( + self._generate_single_frame, this_frame + ) + jobs[job] = this_frame + except StopIteration: + break # No more frames to submit + + for job in as_completed(jobs): + frames_left -= 1 + try: + ret = job.result() + except IndexError: + ret = "IndexError" + self._debug_print(f"Finish: {ret}") + progress_bar.update() + del jobs[job] + self._debug_print(end="\n") + + def ffmpeg_extract(self, start_frame, end_frame): + """Use ffmpeg to extract a batch of frames.""" + logger.debug(f"Extracting frames: {start_frame} - {end_frame}") + output_pattern = str(self.temp_dir / f"orig_%0{self.pad_len}d.png") + + # Use ffmpeg to extract frames + ffmpeg_cmd = [ + "ffmpeg", + "-y", # overwrite + "-i", + self.video_filename, + "-vf", + f"select=between(n\\,{start_frame}\\,{end_frame})", + "-vsync", + "vfr", + "-start_number", + str(start_frame), + "-n", # no overwrite + output_pattern, + *self.ffmpeg_log_args, + ] + ret = subprocess.run(ffmpeg_cmd, stderr=subprocess.PIPE) + + extracted = len(list(self.temp_dir.glob("orig_*.png"))) + logger.debug(f"Extracted frames: {start_frame}, len: {extracted}") + if extracted < self.batch_size - 1: + logger.warning( + f"Could not extract frames: {extracted} / {self.batch_size-1}" + ) + one_err = "\n".join(str(ret.stderr).split("\\")[-3:-1]) + logger.debug(f"\nExtract Error: {one_err}") + + def _pad(self, frame_ind): + return f"{frame_ind:0{self.pad_len}d}" + + def ffmpeg_stitch_partial(self, start_frame, output_partial_video): + """Stitch a partial movie from processed frames.""" + logger.debug(f"Stitch part vid : {start_frame}") + frame_pattern = str(self.temp_dir / f"plot_%0{self.pad_len}d.png") + + ffmpeg_cmd = [ + "ffmpeg", + "-y", # overwrite + "-r", + str(self.fps), + "-start_number", + str(start_frame), + "-i", + frame_pattern, + *self.ffmpeg_fmt_args, + output_partial_video, + *self.ffmpeg_log_args, + ] + try: + ret = subprocess.run( + ffmpeg_cmd, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + check=True, + text=True, + ) + except subprocess.CalledProcessError as e: + logger.error(f"Error stitching partial video: {e.stderr}") + + def concat_partial_videos(self): + """Concatenate all the partial videos into one final video.""" + partial_vids = sorted(self.temp_dir.glob("partial_*.mp4")) + logger.debug(f"Concat part vids: {len(partial_vids)}") + concat_list_path = self.temp_dir / "concat_list.txt" + with open(concat_list_path, "w") as f: + for partial_video in partial_vids: + f.write(f"file '{partial_video}'\n") + + ffmpeg_cmd = [ + "ffmpeg", + "-y", # overwrite + "-f", + "concat", + "-safe", + "0", + "-i", + str(concat_list_path), + *self.ffmpeg_fmt_args, + str(self.output_video_filename), + *self.ffmpeg_log_args, + ] + try: + ret = subprocess.run( + ffmpeg_cmd, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + text=True, + check=True, + ) + except subprocess.CalledProcessError as e: + logger.error(f"Error stitching partial video: {e.stderr}") def make_video(**kwargs): """Passthrough for VideoMaker class for backwards compatibility.""" - VideoMaker(**kwargs) + return VideoMaker(**kwargs) diff --git a/src/spyglass/position/v1/position_dlc_selection.py b/src/spyglass/position/v1/position_dlc_selection.py index e0bd0359e..581140797 100644 --- a/src/spyglass/position/v1/position_dlc_selection.py +++ b/src/spyglass/position/v1/position_dlc_selection.py @@ -342,18 +342,8 @@ def make(self, key): M_TO_CM = 100 params = (DLCPosVideoParams & key).fetch1("params") + epoch = key["epoch"] - interval_name = convert_epoch_interval_name_to_position_interval_name( - { - "nwb_file_name": key["nwb_file_name"], - "epoch": key["epoch"], - }, - populate_missing=False, - ) - epoch = ( - int(interval_name.replace("pos ", "").replace(" valid times", "")) - + 1 - ) pose_est_key = { "nwb_file_name": key["nwb_file_name"], "epoch": epoch, @@ -424,7 +414,12 @@ def make(self, key): ) frames = params.get("frames", None) - make_video( + if limit := params.get("limit", None): # new int param for debugging + output_video_filename = Path(".") / f"TEST_VID_{limit}.mp4" + video_frame_inds = video_frame_inds[:limit] + pos_info_df = pos_info_df.head(limit) + + video_maker = make_video( video_filename=video_filename, video_frame_inds=video_frame_inds, position_mean={ @@ -434,12 +429,19 @@ def make(self, key): centroids=centroids, likelihoods=likelihoods, position_time=np.asarray(pos_info_df.index), - processor=params.get("processor", "opencv"), + processor=params.get("processor", "matplotlib"), frames=np.arange(frames[0], frames[1]) if frames else None, percent_frames=params.get("percent_frames", None), output_video_filename=output_video_filename, cm_to_pixels=meters_per_pixel * M_TO_CM, crop=pose_estimation_params.get("cropping"), + key_hash=dj.hash.key_hash(key), + debug=params.get("debug", False), **params.get("video_params", {}), ) - self.insert1(key) + + if limit: # don't insert if we're just debugging + return video_maker + + if output_video_filename.exists(): + self.insert1(key) From 92d9c35c52832a4dd416aa09362fc855ab3d388e Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Wed, 6 Nov 2024 08:49:51 -0800 Subject: [PATCH 81/94] Expand Export logging abilities (#1164) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * WIP: export fixes, mro issues * WIP: Add one-time log of fetch/restr. Join issues * ✅ : Export logging on join, update doc * pre-commit mdformat * Add flags to permit disable export log in fetch call>< * Use cache to avoid dupe entries. Mod Notebook * Incorporate feedback from @samuelbray32 * Balance parens * Remove redundant parens * Init table in join * Update src/spyglass/utils/mixins/export.py Co-authored-by: Samuel Bray * Include externals * #1173 * Add hex-blob arg to mysqldump * Revert pytest defaults --------- Co-authored-by: Samuel Bray --- CHANGELOG.md | 8 +- docs/src/Features/Export.md | 66 +++-- notebooks/05_Export.ipynb | 207 ++++++++++++---- notebooks/py_scripts/05_Export.py | 35 ++- pyproject.toml | 3 +- src/spyglass/common/common_dandi.py | 1 - src/spyglass/common/common_nwbfile.py | 4 +- src/spyglass/common/common_usage.py | 105 ++++++-- src/spyglass/utils/dj_graph.py | 15 +- src/spyglass/utils/dj_helper_fn.py | 8 +- src/spyglass/utils/dj_merge_tables.py | 55 ++++- src/spyglass/utils/dj_mixin.py | 199 +-------------- src/spyglass/utils/mixins/__init__.py | 0 src/spyglass/utils/mixins/export.py | 338 ++++++++++++++++++++++++++ src/spyglass/utils/sql_helper_fn.py | 139 ++++++++--- tests/common/conftest.py | 6 + tests/common/test_ephys.py | 5 +- tests/common/test_position.py | 73 +----- tests/common/test_usage.py | 64 ++++- tests/conftest.py | 59 +++++ 20 files changed, 979 insertions(+), 411 deletions(-) create mode 100644 src/spyglass/utils/mixins/__init__.py create mode 100644 src/spyglass/utils/mixins/export.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3abcc5861..6bfe55b55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,9 +34,10 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Remove stored hashes from pytests #1152 - Remove mambaforge from tests #1153 - Remove debug statement #1164 -- Allow python < 3.13 #1169 + - Add testing for python versions 3.9, 3.10, 3.11, 3.12 #1169 +- Allow python \< 3.13 #1169 - Remove numpy version restriction #1169 -- Add testing for python versions 3.9, 3.10, 3.11, 3.12 #1169 +- Merge table delete removes orphaned master entries #1164 ### Pipelines @@ -45,6 +46,9 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Drop `SessionGroup` table #1106 - Improve electrodes import efficiency #1125 - Fix logger method call in `common_task` #1132 + - Export fixes #1164 + - Allow `get_abs_path` to add selection entry. + - Log restrictions and joins. - Decoding diff --git a/docs/src/Features/Export.md b/docs/src/Features/Export.md index ca3884dd7..b108d7501 100644 --- a/docs/src/Features/Export.md +++ b/docs/src/Features/Export.md @@ -11,9 +11,9 @@ from only one project be shared during publication. To export data with the current implementation, you must do the following: -- All custom tables must inherit from `SpyglassMixin` (e.g., - `class MyTable(SpyglassMixin, dj.ManualOrOther):`) -- Only one export can be active at a time. +- All custom tables must inherit from either `SpyglassMixin` or `ExportMixin` + (e.g., `class MyTable(SpyglassMixin, dj.ManualOrOther):`) +- Only one export can be active at a time for a given Python instance. - Start the export process with `ExportSelection.start_export()`, run all functions associated with a given analysis, and end the export process with `ExportSelection.end_export()`. @@ -21,30 +21,66 @@ To export data with the current implementation, you must do the following: ## How The current implementation relies on two classes in the Spyglass package -(`SpyglassMixin` and `RestrGraph`) and the `Export` tables. +(`ExportMixin` and `RestrGraph`) and the `Export` tables. -- `SpyglassMixin`: See `spyglass/utils/dj_mixin.py` +- `ExportMixin`: See `spyglass/utils/mixins/export.py` - `RestrGraph`: See `spyglass/utils/dj_graph.py` - `Export`: See `spyglass/common/common_usage.py` ### Mixin -The `SpyglassMixin` class adds functionality to DataJoint tables. A subset of +The `ExportMixin` class adds functionality to DataJoint tables. A subset of methods are used to set an environment variable, `SPYGLASS_EXPORT_ID`, and, -while active, intercept all `fetch`/`fetch_nwb` calls to tables. When `fetch` is -called, the mixin grabs the table name and the restriction applied to the table -and stores them in the `ExportSelection` part tables. +while active, intercept all `fetch`, `fetch_nwb`, `restrict` and `join` calls to +tables. When these functions are called, the mixin grabs the table name and the +restriction applied to the table and stores them in the `ExportSelection` part +tables. + + - `fetch_nwb` is specific to Spyglass and logs all analysis nwb files that are fetched. - `fetch` is a DataJoint method that retrieves data from a table. +- `restrict` is a DataJoint method that restricts a table to a subset of data, + typically using the `&` operator. +- `join` is a DataJoint method that joins two tables together, typically using + the `*` operator. + +This is designed to capture any way that Spyglass is accessed, including +restricting one table via a join with another table. If this process seems to be +missing a way that Spyglass is accessed in your pipeline, please let us know. + +Note that logging all restrictions may log more than is necessary. For example, +`MyTable & restr1 & restr2` will log `MyTable & restr1` and `MyTable & restr2`, +despite returning the combined restriction. Logging will treat compound +restrictions as 'OR' instead of 'AND' statements. This can be avoided by +combining restrictions before using the `&` operator. + +```python +MyTable & "a = b" & "c > 5" # Will capture 'a = b' OR 'c > 5' +MyTable & "a = b AND c > 5" # Will capture 'a = b AND c > 5' +MyTable & dj.AndList(["a = b", "c > 5"]) # Will capture 'a = b AND c > 5' +``` + +If this process captures too much, you can either run a process with logging +disabled, or delete these entries from `ExportSelection` after the export is +logged. + +Disabling logging with the `log_export` flag: + +```python +MyTable().fetch(log_export=False) +MyTable().fetch_nwb(log_export=False) +MyTable().restrict(restr, log_export=False) # Instead of MyTable & restr +MyTable().join(Other, log_export=False) # Instead of MyTable * Other +``` ### Graph The `RestrGraph` class uses DataJoint's networkx graph to store each of the -tables and restrictions intercepted by the `SpyglassMixin`'s `fetch` as -'leaves'. The class then cascades these restrictions up from each leaf to all -ancestors. Use is modeled in the methods of `ExportSelection`. +tables and restrictions intercepted by the `ExportMixin`'s `fetch` as 'leaves'. +The class then cascades these restrictions up from each leaf to all ancestors. +Use is modeled in the methods of `ExportSelection`. ```python from spyglass.utils.dj_graph import RestrGraph @@ -117,7 +153,7 @@ paper. Each shell script one `mysqldump` command per table. To implement an export for a non-Spyglass database, you will need to ... -- Create a modified version of `SpyglassMixin`, including ... +- Create a modified version of `ExportMixin`, including ... - `_export_table` method to lazy load an export table like `ExportSelection` - `export_id` attribute, plus setter and deleter methods, to manage the status of the export. @@ -126,6 +162,6 @@ To implement an export for a non-Spyglass database, you will need to ... `spyglass_version` to match the new database. Or, optionally, you can use the `RestrGraph` class to cascade hand-picked tables -and restrictions without the background logging of `SpyglassMixin`. The -assembled list of restricted free tables, `RestrGraph.all_ft`, can be passed to +and restrictions without the background logging of `ExportMixin`. The assembled +list of restricted free tables, `RestrGraph.all_ft`, can be passed to `Export.write_export` to generate a shell script for exporting the data. diff --git a/notebooks/05_Export.ipynb b/notebooks/05_Export.ipynb index 6290d3596..2f540f849 100644 --- a/notebooks/05_Export.ipynb +++ b/notebooks/05_Export.ipynb @@ -107,14 +107,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-05-29 14:56:01,787][INFO]: Connecting sambray@lmf-db.cin.ucsf.edu:3306\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[2024-05-29 14:56:01,872][INFO]: Connected sambray@lmf-db.cin.ucsf.edu:3306\n" + "[2024-10-14 16:47:00,520][INFO]: Connecting root@localhost:3308\n", + "[2024-10-14 16:47:00,529][INFO]: Connected root@localhost:3308\n" ] } ], @@ -128,7 +122,7 @@ "dj.config.load(\"dj_local_conf.json\") # load config for database connection info\n", "\n", "from spyglass.common.common_usage import Export, ExportSelection\n", - "from spyglass.lfp.analysis.v1 import LFPBandV1\n", + "from spyglass.common.common_ephys import Electrode\n", "from spyglass.position.v1 import TrodesPosV1\n", "from spyglass.spikesorting.v1.curation import CurationV1" ] @@ -223,17 +217,26 @@ "

time

\n", " \n", " \n", - " \n", + " 1\n", + "paper1\n", + "analysis1\n", + "0.5.1\n", + "2024-10-14 21:45:222\n", + "paper1\n", + "analysis2\n", + "0.5.1\n", + "2024-10-14 21:46:18 \n", " \n", " \n", - "

Total: 0

\n", + "

Total: 2

\n", " " ], "text/plain": [ - "*export_id paper_id analysis_id spyglass_versi time \n", - "+-----------+ +----------+ +------------+ +------------+ +------+\n", - "\n", - " (Total: 0)" + "*export_id paper_id analysis_id spyglass_versi time \n", + "+-----------+ +----------+ +------------+ +------------+ +------------+\n", + "1 paper1 analysis1 0.5.1 2024-10-14 21:\n", + "2 paper1 analysis2 0.5.1 2024-10-14 21:\n", + " (Total: 2)" ] }, "execution_count": 2, @@ -321,17 +324,44 @@ "

restriction

\n", " \n", " \n", - " \n", + " 1\n", + "1\n", + "`common_ephys`.`_electrode`\n", + "((nwb_file_name LIKE 'min%%'))AND((electrode_id=1))1\n", + "2\n", + "`common_ephys`.`_electrode`\n", + "(electrode_id > 125)2\n", + "3\n", + "`spikesorting_v1_curation`.`curation_v1`\n", + "(curation_id = 1)2\n", + "4\n", + "`common_nwbfile`.`analysis_nwbfile`\n", + "(analysis_file_name in ('minirec20230622_JKTSSFUIL3.nwb'))2\n", + "5\n", + "`position_v1_trodes_position`.`__trodes_pos_v1`\n", + "(trodes_pos_params_name = 'single_led')2\n", + "6\n", + "`common_nwbfile`.`analysis_nwbfile`\n", + "(analysis_file_name='minirec20230622_JKTSSFUIL3.nwb')2\n", + "7\n", + "`common_nwbfile`.`nwbfile`\n", + "(nwb_file_name LIKE '%%minirec20230622_.nwb%%') \n", " \n", " \n", - "

Total: 0

\n", + "

Total: 7

\n", " " ], "text/plain": [ "*export_id *table_id table_name restriction \n", "+-----------+ +----------+ +------------+ +------------+\n", - "\n", - " (Total: 0)" + "1 1 `common_ephys` ((nwb_file_nam\n", + "1 2 `common_ephys` (electrode_id \n", + "2 3 `spikesorting_ (curation_id =\n", + "2 4 `common_nwbfil (analysis_file\n", + "2 5 `position_v1_t (trodes_pos_pa\n", + "2 6 `common_nwbfil (analysis_file\n", + "2 7 `common_nwbfil (nwb_file_name\n", + " (Total: 7)" ] }, "execution_count": 3, @@ -413,17 +443,18 @@ "

analysis_file_name

\n", " name of the file\n", " \n", - " \n", + " 2\n", + "minirec20230622_JKTSSFUIL3.nwb \n", " \n", " \n", - "

Total: 0

\n", + "

Total: 1

\n", " " ], "text/plain": [ "*export_id *analysis_file\n", "+-----------+ +------------+\n", - "\n", - " (Total: 0)" + "2 minirec2023062\n", + " (Total: 1)" ] }, "execution_count": 4, @@ -469,7 +500,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "[16:32:51][INFO] Spyglass: Starting {'export_id': 1}\n" + "[16:47:01][INFO] Spyglass: Resuming {'export_id': 1}\n" ] } ], @@ -478,12 +509,25 @@ "\n", "ExportSelection().start_export(**paper_key, analysis_id=\"analysis1\")\n", "my_lfp_data = (\n", - " LFPBandV1 # Logging this table\n", - " & \"nwb_file_name LIKE 'med%'\" # using a string restriction\n", - " & {\"filter_name\": \"Theta 5-11 Hz\"} # and a dictionary restriction\n", + " Electrode # Logging this table\n", + " & dj.AndList(\n", + " [\n", + " \"nwb_file_name LIKE 'min%'\", # using a string restrictionshared\n", + " {\"electrode_id\": 1}, # and a dictionary restriction\n", + " ]\n", + " )\n", ").fetch()" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Note**: Compound resrictions (e.g., `Table & a & b`) will be logged separately\n", + "as `Table & a` or `Table & b`. To fully restrict, use strings (e.g.,\n", + "`Table & 'a AND b'`) or `dj.AndList([a,b])`." + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -571,18 +615,46 @@ " \n", " 1\n", "1\n", - "`lfp_band_v1`.`__l_f_p_band_v1`\n", - " (( ((nwb_file_name LIKE 'med%%%%%%%%')))AND( ((`filter_name`=\"Theta 5-11 Hz\")))) \n", + "`common_ephys`.`_electrode`\n", + "((nwb_file_name LIKE 'min%%'))AND((electrode_id=1))1\n", + "2\n", + "`common_ephys`.`_electrode`\n", + "(electrode_id > 125)1\n", + "8\n", + "`common_ephys`.`_electrode`\n", + "((nwb_file_name LIKE 'min%%'))AND((electrode_id=1))2\n", + "3\n", + "`spikesorting_v1_curation`.`curation_v1`\n", + "(curation_id = 1)2\n", + "4\n", + "`common_nwbfile`.`analysis_nwbfile`\n", + "(analysis_file_name in ('minirec20230622_JKTSSFUIL3.nwb'))2\n", + "5\n", + "`position_v1_trodes_position`.`__trodes_pos_v1`\n", + "(trodes_pos_params_name = 'single_led')2\n", + "6\n", + "`common_nwbfile`.`analysis_nwbfile`\n", + "(analysis_file_name='minirec20230622_JKTSSFUIL3.nwb')2\n", + "7\n", + "`common_nwbfile`.`nwbfile`\n", + "(nwb_file_name LIKE '%%minirec20230622_.nwb%%') \n", " \n", " \n", - "

Total: 1

\n", + "

Total: 8

\n", " " ], "text/plain": [ "*export_id *table_id table_name restriction \n", "+-----------+ +----------+ +------------+ +------------+\n", - "1 1 `lfp_band_v1`. (( ((nwb_file\n", - " (Total: 1)" + "1 1 `common_ephys` ((nwb_file_nam\n", + "1 2 `common_ephys` (electrode_id \n", + "1 8 `common_ephys` ((nwb_file_nam\n", + "2 3 `spikesorting_ (curation_id =\n", + "2 4 `common_nwbfil (analysis_file\n", + "2 5 `position_v1_t (trodes_pos_pa\n", + "2 6 `common_nwbfil (analysis_file\n", + "2 7 `common_nwbfil (nwb_file_name\n", + " (Total: 8)" ] }, "execution_count": 6, @@ -598,7 +670,10 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "And log more under the same analysis ...\n" + "If there seem to be redundant entries in this part table, we can ignore them. \n", + "They'll be merged later during `Export.populate`.\n", + "\n", + "Let's log more under the same analysis ...\n" ] }, { @@ -607,13 +682,7 @@ "metadata": {}, "outputs": [], "source": [ - "my_other_lfp_data = (\n", - " LFPBandV1\n", - " & {\n", - " \"nwb_file_name\": \"mediumnwb20230802_.nwb\",\n", - " \"filter_name\": \"Theta 5-10 Hz\",\n", - " }\n", - ").fetch()" + "my_other_lfp_data = (Electrode & \"electrode_id > 125\").fetch()" ] }, { @@ -621,7 +690,7 @@ "metadata": {}, "source": [ "Since these restrictions are mutually exclusive, we can check that the will\n", - "be combined appropriately by priviewing the logged tables...\n" + "be combined appropriately by previewing the logged tables...\n" ] }, { @@ -632,12 +701,37 @@ { "data": { "text/plain": [ - "[FreeTable(`lfp_band_v1`.`__l_f_p_band_v1`)\n", - " *lfp_merge_id *filter_name *filter_sampli *nwb_file_name *target_interv *lfp_band_samp analysis_file_ interval_list_ lfp_band_objec\n", - " +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+\n", - " 0f3bb01e-0ef6- Theta 5-10 Hz 1000 mediumnwb20230 pos 0 valid ti 100 mediumnwb20230 pos 0 valid ti 44e38dc1-3779-\n", - " 0f3bb01e-0ef6- Theta 5-11 Hz 1000 mediumnwb20230 pos 0 valid ti 100 mediumnwb20230 pos 0 valid ti c9b93111-decb-\n", - " (Total: 2)]" + "[FreeTable(`common_ephys`.`_electrode`)\n", + " *nwb_file_name *electrode_gro *electrode_id probe_id probe_shank probe_electrod region_id name original_refer x y z filtering impedance bad_channel x_warped y_warped z_warped contacts \n", + " +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +-----------+ +------+ +------------+ +-----+ +-----+ +-----+ +-----------+ +-----------+ +------------+ +----------+ +----------+ +----------+ +----------+\n", + " minirec2023062 0 0 tetrode_12.5 0 0 1 0 0 0.0 0.0 0.0 None 0.0 False 0.0 0.0 0.0 \n", + " minirec2023062 0 1 tetrode_12.5 0 1 1 1 0 0.0 0.0 0.0 None 0.0 False 0.0 0.0 0.0 \n", + " minirec2023062 0 2 tetrode_12.5 0 2 1 2 0 0.0 0.0 0.0 None 0.0 False 0.0 0.0 0.0 \n", + " minirec2023062 0 3 tetrode_12.5 0 3 1 3 0 0.0 0.0 0.0 None 0.0 False 0.0 0.0 0.0 \n", + " minirec2023062 31 126 tetrode_12.5 0 2 1 126 0 0.0 0.0 0.0 None 0.0 False 0.0 0.0 0.0 \n", + " minirec2023062 31 127 tetrode_12.5 0 3 1 127 0 0.0 0.0 0.0 None 0.0 False 0.0 0.0 0.0 \n", + " (Total: 6),\n", + " FreeTable(`spikesorting_v1_curation`.`curation_v1`)\n", + " *sorting_id *curation_id parent_curatio analysis_file_ object_id merges_applied description \n", + " +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+\n", + " 3b909e6b-9cec- 1 0 minirec2023062 9bcad28e-3b7f- 0 after metric c\n", + " (Total: 1),\n", + " FreeTable(`common_nwbfile`.`analysis_nwbfile`)\n", + " *analysis_file nwb_file_name analysis_f analysis_file_ analysis_p\n", + " +------------+ +------------+ +--------+ +------------+ +--------+\n", + " minirec2023062 minirec2023062 =BLOB= =BLOB= \n", + " minirec2023062 minirec2023062 =BLOB= =BLOB= \n", + " (Total: 2),\n", + " FreeTable(`position_v1_trodes_position`.`__trodes_pos_v1`)\n", + " *nwb_file_name *interval_list *trodes_pos_pa analysis_file_ position_objec orientation_ob velocity_objec\n", + " +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+\n", + " minirec2023062 pos 0 valid ti single_led minirec2023062 5f48f897-2d13- 06d39e03-51cc- 210c92b5-5719-\n", + " (Total: 1),\n", + " FreeTable(`common_nwbfile`.`nwbfile`)\n", + " *nwb_file_name nwb_file_a\n", + " +------------+ +--------+\n", + " minirec2023062 =BLOB= \n", + " (Total: 1)]" ] }, "execution_count": 8, @@ -649,6 +743,15 @@ "ExportSelection().preview_tables(**paper_key)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For a more comprehensive view of what would be in the export, you can look at\n", + "`ExportSelection().show_all_tables(**paper_key)`. This may take some time to\n", + "generate." + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -666,8 +769,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "[16:32:51][INFO] Spyglass: Export 1 in progress. Starting new.\n", - "[16:32:51][INFO] Spyglass: Starting {'export_id': 2}\n" + "[16:47:02][INFO] Spyglass: Export 1 in progress. Starting new.\n", + "[16:47:02][INFO] Spyglass: Resuming {'export_id': 2}\n" ] } ], @@ -692,8 +795,8 @@ { "data": { "text/plain": [ - "[{'file_path': '/home/cb/wrk/alt/data/raw/mediumnwb20230802_.nwb'},\n", - " {'file_path': '/home/cb/wrk/alt/data/analysis/mediumnwb20230802/mediumnwb20230802_ALNN6TZ4L7.nwb'}]" + "[{'file_path': '/home/cb/wrk/spyglass/tests/_data/raw/minirec20230622_.nwb'},\n", + " {'file_path': '/home/cb/wrk/spyglass/tests/_data/analysis/minirec20230622/minirec20230622_JKTSSFUIL3.nwb'}]" ] }, "execution_count": 10, @@ -1065,7 +1168,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.16" + "version": "3.9.20" } }, "nbformat": 4, diff --git a/notebooks/py_scripts/05_Export.py b/notebooks/py_scripts/05_Export.py index 4f8c376d0..7cf0183c0 100644 --- a/notebooks/py_scripts/05_Export.py +++ b/notebooks/py_scripts/05_Export.py @@ -96,7 +96,7 @@ dj.config.load("dj_local_conf.json") # load config for database connection info from spyglass.common.common_usage import Export, ExportSelection -from spyglass.lfp.analysis.v1 import LFPBandV1 +from spyglass.common.common_ephys import Electrode from spyglass.position.v1 import TrodesPosV1 from spyglass.spikesorting.v1.curation import CurationV1 @@ -137,12 +137,20 @@ ExportSelection().start_export(**paper_key, analysis_id="analysis1") my_lfp_data = ( - LFPBandV1 # Logging this table - & "nwb_file_name LIKE 'med%'" # using a string restriction - & {"filter_name": "Theta 5-11 Hz"} # and a dictionary restriction + Electrode # Logging this table + & dj.AndList( + [ + "nwb_file_name LIKE 'min%'", # using a string restrictionshared + {"electrode_id": 1}, # and a dictionary restriction + ] + ) ).fetch() # - +# **Note**: Compound resrictions (e.g., `Table & a & b`) will be logged separately +# as `Table & a` or `Table & b`. To fully restrict, use strings (e.g., +# `Table & 'a AND b'`) or `dj.AndList([a,b])`. + # We can check that it was logged. The syntax of the restriction will look # different from what we see in python, but the `preview_tables` will look # familiar. @@ -150,23 +158,24 @@ ExportSelection.Table() -# And log more under the same analysis ... +# If there seem to be redundant entries in this part table, we can ignore them. +# They'll be merged later during `Export.populate`. +# +# Let's log more under the same analysis ... # -my_other_lfp_data = ( - LFPBandV1 - & { - "nwb_file_name": "mediumnwb20230802_.nwb", - "filter_name": "Theta 5-10 Hz", - } -).fetch() +my_other_lfp_data = (Electrode & "electrode_id > 125").fetch() # Since these restrictions are mutually exclusive, we can check that the will -# be combined appropriately by priviewing the logged tables... +# be combined appropriately by previewing the logged tables... # ExportSelection().preview_tables(**paper_key) +# For a more comprehensive view of what would be in the export, you can look at +# `ExportSelection().show_all_tables(**paper_key)`. This may take some time to +# generate. + # Let's try adding a new analysis with a fetched nwb file. Starting a new export # will stop the previous one. # diff --git a/pyproject.toml b/pyproject.toml index 0bd9164cb..39a932919 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -125,7 +125,8 @@ ignore-words-list = 'nevers' [tool.pytest.ini_options] minversion = "7.0" addopts = [ - # "-sv", # no capture, verbose output + "-s", # no capture + # "-v", # verbose output # "--sw", # stepwise: resume with next test after failure # "--pdb", # drop into debugger on failure "-p no:warnings", diff --git a/src/spyglass/common/common_dandi.py b/src/spyglass/common/common_dandi.py index 8a9be9677..19bee97db 100644 --- a/src/spyglass/common/common_dandi.py +++ b/src/spyglass/common/common_dandi.py @@ -1,7 +1,6 @@ import os import shutil from pathlib import Path -from typing import Optional import datajoint as dj import fsspec diff --git a/src/spyglass/common/common_nwbfile.py b/src/spyglass/common/common_nwbfile.py index 2166b2c04..ad4facd17 100644 --- a/src/spyglass/common/common_nwbfile.py +++ b/src/spyglass/common/common_nwbfile.py @@ -349,7 +349,9 @@ def get_abs_path(cls, analysis_nwb_file_name: str) -> str: if cls & file_key: try: # runs if file exists locally - return (cls & file_key).fetch1("analysis_file_abs_path") + return (cls & file_key).fetch1( + "analysis_file_abs_path", log_export=False + ) except FileNotFoundError as e: # file exists in database but not locally # parse the intended path from the error message diff --git a/src/spyglass/common/common_usage.py b/src/spyglass/common/common_usage.py index ced518521..ccbf7c909 100644 --- a/src/spyglass/common/common_usage.py +++ b/src/spyglass/common/common_usage.py @@ -15,7 +15,7 @@ from spyglass.common.common_nwbfile import AnalysisNwbfile, Nwbfile from spyglass.settings import export_dir, test_mode -from spyglass.utils import SpyglassMixin, logger +from spyglass.utils import SpyglassMixin, SpyglassMixinPart, logger from spyglass.utils.dj_graph import RestrGraph from spyglass.utils.dj_helper_fn import ( make_file_obj_id_unique, @@ -88,7 +88,7 @@ class ExportSelection(SpyglassMixin, dj.Manual): unique index (paper_id, analysis_id) """ - class Table(SpyglassMixin, dj.Part): + class Table(SpyglassMixinPart): definition = """ -> master table_id: int @@ -144,6 +144,22 @@ def stop_export(self, **kwargs) -> None: # before actually exporting anything, which is more associated with # Selection + def _list_raw_files(self, key: dict) -> list[str]: + """Return a list of unique nwb file names for a given restriction/key.""" + file_table = self * self.File & key + return list( + { + *AnalysisNwbfile.join(file_table, log_export=False).fetch( + "nwb_file_name" + ) + } + ) + + def _list_analysis_files(self, key: dict) -> list[str]: + """Return a list of unique analysis file names for a given restriction/key.""" + file_table = self * self.File & key + return list(file_table.fetch("analysis_file_name")) + def list_file_paths(self, key: dict, as_dict=True) -> list[str]: """Return a list of unique file paths for a given restriction/key. @@ -159,18 +175,64 @@ def list_file_paths(self, key: dict, as_dict=True) -> list[str]: If False, returns a list of strings without key. """ file_table = self * self.File & key - analysis_fp = [ - AnalysisNwbfile().get_abs_path(fname) - for fname in file_table.fetch("analysis_file_name") - ] - nwbfile_fp = [ - Nwbfile().get_abs_path(fname) - for fname in (AnalysisNwbfile * file_table).fetch("nwb_file_name") - ] - unique_ft = list({*analysis_fp, *nwbfile_fp}) - return [{"file_path": p} for p in unique_ft] if as_dict else unique_ft + unique_fp = { + *[ + AnalysisNwbfile().get_abs_path(p) + for p in self._list_analysis_files(key) + ], + *[Nwbfile().get_abs_path(p) for p in self._list_raw_files(key)], + } + + return [{"file_path": p} for p in unique_fp] if as_dict else unique_fp + + @property + def _externals(self) -> dj.external.ExternalMapping: + """Return the external mapping for the common_n schema.""" + return dj.external.ExternalMapping(schema=AnalysisNwbfile) + + def _add_externals_to_restr_graph( + self, restr_graph: RestrGraph, key: dict + ) -> RestrGraph: + """Add external tables to a RestrGraph for a given restriction/key. + + Tables added as nodes with restrictions based on file paths. Names + added to visited set to appear in restr_ft obj bassed to SQLDumpHelper. + + Parameters + ---------- + restr_graph : RestrGraph + A RestrGraph object to add external tables to. + key : dict + Any valid restriction key for ExportSelection.Table + + Returns + ------- + restr_graph : RestrGraph + The updated RestrGraph + """ + raw_tbl = self._externals["raw"] + raw_name = raw_tbl.full_table_name + raw_restr = ( + "filepath in ('" + "','".join(self._list_raw_files(key)) + "')" + ) + restr_graph.graph.add_node(raw_name, ft=raw_tbl, restr=raw_restr) + + analysis_tbl = self._externals["analysis"] + analysis_name = analysis_tbl.full_table_name + analysis_restr = ( # filepaths have analysis subdir. regexp substrings + "filepath REGEXP '" + "|".join(self._list_analysis_files(key)) + "'" + ) # regexp is slow, but we're only doing this once, and future-proof + restr_graph.graph.add_node( + analysis_name, ft=analysis_tbl, restr=analysis_restr + ) + + restr_graph.visited.update({raw_name, analysis_name}) - def get_restr_graph(self, key: dict, verbose=False) -> RestrGraph: + return restr_graph + + def get_restr_graph( + self, key: dict, verbose=False, cascade=True + ) -> RestrGraph: """Return a RestrGraph for a restriction/key's tables/restrictions. Ignores duplicate entries. @@ -181,21 +243,36 @@ def get_restr_graph(self, key: dict, verbose=False) -> RestrGraph: Any valid restriction key for ExportSelection.Table verbose : bool, optional Turn on RestrGraph verbosity. Default False. + cascade : bool, optional + Propagate restrictions to upstream tables. Default True. """ leaves = unique_dicts( (self * self.Table & key).fetch( "table_name", "restriction", as_dict=True ) ) - return RestrGraph(seed_table=self, leaves=leaves, verbose=verbose) + + restr_graph = RestrGraph( + seed_table=self, leaves=leaves, verbose=verbose, cascade=cascade + ) + return self._add_externals_to_restr_graph(restr_graph, key) def preview_tables(self, **kwargs) -> list[dj.FreeTable]: """Return a list of restricted FreeTables for a given restriction/key. Useful for checking what will be exported. """ + kwargs["cascade"] = False return self.get_restr_graph(kwargs).leaf_ft + def show_all_tables(self, **kwargs) -> list[dj.FreeTable]: + """Return a list of all FreeTables for a given restriction/key. + + Useful for checking what will be exported. + """ + kwargs["cascade"] = True + return self.get_restr_graph(kwargs).restr_ft + def _max_export_id(self, paper_id: str, return_all=False) -> int: """Return last export associated with a given paper id. diff --git a/src/spyglass/utils/dj_graph.py b/src/spyglass/utils/dj_graph.py index 435d37d38..1e6137a02 100644 --- a/src/spyglass/utils/dj_graph.py +++ b/src/spyglass/utils/dj_graph.py @@ -808,13 +808,14 @@ def file_paths(self) -> List[str]: directly by the user. """ self.cascade() - return [ - self.analysis_file_tbl.get_abs_path(file) - for file in set( - [f for files in self.file_dict.values() for f in files] - ) - if file is not None - ] + + files = { + file + for table in self.visited + for file in self._get_node(table).get("files", []) + } + + return [self.analysis_file_tbl.get_abs_path(file) for file in files] class TableChain(RestrGraph): diff --git a/src/spyglass/utils/dj_helper_fn.py b/src/spyglass/utils/dj_helper_fn.py index ca8b99f2b..d77416487 100644 --- a/src/spyglass/utils/dj_helper_fn.py +++ b/src/spyglass/utils/dj_helper_fn.py @@ -236,7 +236,9 @@ def get_nwb_table(query_expression, tbl, attr_name, *attrs, **kwargs): # TODO: check that the query_expression restricts tbl - CBroz nwb_files = ( - query_expression * tbl.proj(nwb2load_filepath=attr_name) + query_expression.join( + tbl.proj(nwb2load_filepath=attr_name), log_export=False + ) ).fetch(file_name_str) # Disabled #1024 @@ -291,7 +293,9 @@ def fetch_nwb(query_expression, nwb_master, *attrs, **kwargs): # This also opens the file and stores the file object get_nwb_file(file_path) - query_table = query_expression * tbl.proj(nwb2load_filepath=attr_name) + query_table = query_expression.join( + tbl.proj(nwb2load_filepath=attr_name), log_export=False + ) rec_dicts = query_table.fetch(*attrs, **kwargs) # get filepath for each. Use datajoint for checksum if local for rec_dict in rec_dicts: diff --git a/src/spyglass/utils/dj_merge_tables.py b/src/spyglass/utils/dj_merge_tables.py index bf13aa254..58532fa9f 100644 --- a/src/spyglass/utils/dj_merge_tables.py +++ b/src/spyglass/utils/dj_merge_tables.py @@ -13,6 +13,7 @@ from IPython.core.display import HTML from spyglass.utils.logging import logger +from spyglass.utils.mixins.export import ExportMixin RESERVED_PRIMARY_KEY = "merge_id" RESERVED_SECONDARY_KEY = "source" @@ -49,11 +50,11 @@ def trim_def(definition): ] and table.heading.secondary_attributes == [RESERVED_SECONDARY_KEY] -class Merge(dj.Manual): +class Merge(ExportMixin, dj.Manual): """Adds funcs to support standard Merge table operations. Many methods have the @classmethod decorator to permit MergeTable.method() - symtax. This makes access to instance attributes (e.g., (MergeTable & + syntax. This makes access to instance attributes (e.g., (MergeTable & "example='restriction'").restriction) harder, but these attributes have limited utility when the user wants to, for example, restrict the merged view rather than the master table itself. @@ -508,6 +509,7 @@ def fetch_nwb( multi_source=False, disable_warning=False, return_merge_ids=False, + log_export=True, *attrs, **kwargs, ): @@ -524,6 +526,8 @@ def fetch_nwb( Return from multiple parents. Default False. return_merge_ids: bool Default False. Return merge_ids with nwb files. + log_export: bool + Default True. During export, log this fetch an export event. Notes ----- @@ -533,13 +537,19 @@ def fetch_nwb( raise ValueError("Try replacing Merge.method with Merge().method") restriction = restriction or self.restriction or True merge_restriction = self.extract_merge_id(restriction) - sources = set((self & merge_restriction).fetch(self._reserved_sk)) + + sources = set( + (self & merge_restriction).fetch( + self._reserved_sk, log_export=False + ) + ) nwb_list = [] merge_ids = [] for source in sources: source_restr = ( - self & {self._reserved_sk: source} & merge_restriction - ).fetch("KEY") + self + & dj.AndList([{self._reserved_sk: source}, merge_restriction]) + ).fetch("KEY", log_export=False) nwb_list.extend( (self & source_restr) .merge_restrict_class( @@ -763,8 +773,9 @@ def merge_restrict_class( parent_class = self.merge_get_parent_class(parent) return parent_class & parent_key - @classmethod - def merge_fetch(self, restriction: str = True, *attrs, **kwargs) -> list: + def merge_fetch( + self, restriction: str = True, log_export=True, *attrs, **kwargs + ) -> list: """Perform a fetch across all parts. If >1 result, return as a list. Parameters @@ -772,6 +783,8 @@ def merge_fetch(self, restriction: str = True, *attrs, **kwargs) -> list: restriction: str Optional restriction to apply before determining parent to return. Default True. + log_export: bool + Default True. During export, log this fetch an export event. attrs, kwargs arguments passed to DataJoint `fetch` call @@ -780,8 +793,17 @@ def merge_fetch(self, restriction: str = True, *attrs, **kwargs) -> list: Union[ List[np.array], List[dict], List[pd.DataFrame] ] Table contents, with type determined by kwargs """ + restriction = self.restriction or restriction + + if log_export and self.export_id: + self._log_fetch( # Transforming restriction to merge_id + restriction=self.merge_restrict(restriction).fetch( + RESERVED_PRIMARY_KEY, as_dict=True + ) + ) + results = [] - parts = self()._merge_restrict_parts( + parts = self._merge_restrict_parts( restriction=restriction, as_objects=True, return_empties=False, @@ -821,7 +843,10 @@ def merge_populate(self, source: str, keys=None): self.insert(successes) def delete(self, force_permission=False, *args, **kwargs): - """Alias for cautious_delete, overwrites datajoint.table.Table.delete""" + """Alias for cautious_delete, overwrites datajoint.table.Table.delete + + Delete all relevant part entries from self.restriction. + """ if not ( parts := self.merge_get_part( restriction=self.restriction, @@ -831,8 +856,18 @@ def delete(self, force_permission=False, *args, **kwargs): ): return + _ = kwargs.pop("force_masters", None) # Part not accept this kwarg for part in parts: - part.delete(force_permission=force_permission, *args, **kwargs) + part.delete( + force_permission=force_permission, + force_parts=True, + *args, + **kwargs, + ) + + # Delete orphaned master entries, no prompt + kwargs["safemode"] = False + (self - self.parts(as_objects=True)).super_delete(*args, **kwargs) def super_delete(self, warn=True, *args, **kwargs): """Alias for datajoint.table.Table.delete. diff --git a/src/spyglass/utils/dj_mixin.py b/src/spyglass/utils/dj_mixin.py index 723fec869..090b1a3ee 100644 --- a/src/spyglass/utils/dj_mixin.py +++ b/src/spyglass/utils/dj_mixin.py @@ -1,10 +1,5 @@ -from atexit import register as exit_register -from atexit import unregister as exit_unregister from contextlib import nullcontext from functools import cached_property -from inspect import stack as inspect_stack -from os import environ -from re import match as re_match from time import time from typing import List @@ -28,16 +23,15 @@ ) from spyglass.utils.dj_merge_tables import Merge, is_merge_table from spyglass.utils.logging import logger +from spyglass.utils.mixins.export import ExportMixin try: import pynapple # noqa F401 except (ImportError, ModuleNotFoundError): pynapple = None -EXPORT_ENV_VAR = "SPYGLASS_EXPORT_ID" - -class SpyglassMixin: +class SpyglassMixin(ExportMixin): """Mixin for Spyglass DataJoint tables. Provides methods for fetching NWBFile objects and checking user permission @@ -190,27 +184,9 @@ def fetch_nwb(self, *attrs, **kwargs): """ table, tbl_attr = self._nwb_table_tuple - if self.export_id and "analysis" in tbl_attr: - tbl_pk = "analysis_file_name" - fnames = (self * table).fetch(tbl_pk) - logger.debug( - f"Export {self.export_id}: fetch_nwb {self.table_name}, {fnames}" - ) - self._export_table.File.insert( - [ - {"export_id": self.export_id, tbl_pk: fname} - for fname in fnames - ], - skip_duplicates=True, - ) - self._export_table.Table.insert1( - dict( - export_id=self.export_id, - table_name=self.full_table_name, - restriction=make_condition(self, self.restriction, set()), - ), - skip_duplicates=True, - ) + log_export = kwargs.pop("log_export", True) + if log_export and self.export_id and "analysis" in tbl_attr: + self._log_fetch_nwb(table, tbl_attr) return fetch_nwb(self, self._nwb_table_tuple, *attrs, **kwargs) @@ -487,7 +463,7 @@ def cautious_delete( logger.warning(f"Table is empty. No need to delete.\n{self}") return - if self._has_updated_dj_version: + if self._has_updated_dj_version and not isinstance(self, dj.Part): kwargs["force_masters"] = True external, IntervalList = self._delete_deps[3], self._delete_deps[4] @@ -635,164 +611,6 @@ def populate(self, *restrictions, **kwargs): pool.close() pool.terminate() - # ------------------------------- Export Log ------------------------------- - - @cached_property - def _spyglass_version(self): - """Get Spyglass version.""" - from spyglass import __version__ as sg_version - - ret = ".".join(sg_version.split(".")[:3]) # Ditch commit info - - if self._test_mode: - return ret[:16] if len(ret) > 16 else ret - - if not bool(re_match(r"^\d+\.\d+\.\d+", ret)): # Major.Minor.Patch - raise ValueError( - f"Spyglass version issues. Expected #.#.#, Got {ret}." - + "Please try running `hatch build` from your spyglass dir." - ) - - return ret - - def compare_versions( - self, version: str, other: str = None, msg: str = None - ) -> None: - """Compare two versions. Raise error if not equal. - - Parameters - ---------- - version : str - Version to compare. - other : str, optional - Other version to compare. Default None. Use self._spyglass_version. - msg : str, optional - Additional error message info. Default None. - """ - if self._test_mode: - return - - other = other or self._spyglass_version - - if version_parse(version) != version_parse(other): - raise RuntimeError( - f"Found mismatched versions: {version} vs {other}\n{msg}" - ) - - @cached_property - def _export_table(self): - """Lazy load export selection table.""" - from spyglass.common.common_usage import ExportSelection - - return ExportSelection() - - @property - def export_id(self): - """ID of export in progress. - - NOTE: User of an env variable to store export_id may not be thread safe. - Exports must be run in sequence, not parallel. - """ - - return int(environ.get(EXPORT_ENV_VAR, 0)) - - @export_id.setter - def export_id(self, value): - """Set ID of export using `table.export_id = X` notation.""" - if self.export_id != 0 and self.export_id != value: - raise RuntimeError("Export already in progress.") - environ[EXPORT_ENV_VAR] = str(value) - exit_register(self._export_id_cleanup) # End export on exit - - @export_id.deleter - def export_id(self): - """Delete ID of export using `del table.export_id` notation.""" - self._export_id_cleanup() - - def _export_id_cleanup(self): - """Cleanup export ID.""" - if environ.get(EXPORT_ENV_VAR): - del environ[EXPORT_ENV_VAR] - exit_unregister(self._export_id_cleanup) # Remove exit hook - - def _start_export(self, paper_id, analysis_id): - """Start export process.""" - if self.export_id: - logger.info(f"Export {self.export_id} in progress. Starting new.") - self._stop_export(warn=False) - - self.export_id = self._export_table.insert1_return_pk( - dict( - paper_id=paper_id, - analysis_id=analysis_id, - spyglass_version=self._spyglass_version, - ) - ) - - def _stop_export(self, warn=True): - """End export process.""" - if not self.export_id and warn: - logger.warning("Export not in progress.") - del self.export_id - - def _log_fetch(self, *args, **kwargs): - """Log fetch for export.""" - if not self.export_id or self.database == "common_usage": - return - - banned = [ - "head", # Prevents on Table().head() call - "tail", # Prevents on Table().tail() call - "preview", # Prevents on Table() call - "_repr_html_", # Prevents on Table() call in notebook - "cautious_delete", # Prevents add on permission check during delete - "get_abs_path", # Assumes that fetch_nwb will catch file/table - ] - called = [i.function for i in inspect_stack()] - if set(banned) & set(called): # if called by any in banned, return - return - - logger.debug(f"Export {self.export_id}: fetch() {self.table_name}") - - restr = self.restriction or True - limit = kwargs.get("limit") - offset = kwargs.get("offset") - if limit or offset: # Use result as restr if limit/offset - restr = self.restrict(restr).fetch( - log_fetch=False, as_dict=True, limit=limit, offset=offset - ) - - restr_str = make_condition(self, restr, set()) - - if isinstance(restr_str, str) and len(restr_str) > 2048: - raise RuntimeError( - "Export cannot handle restrictions > 2048.\n\t" - + "If required, please open an issue on GitHub.\n\t" - + f"Restriction: {restr_str}" - ) - self._export_table.Table.insert1( - dict( - export_id=self.export_id, - table_name=self.full_table_name, - restriction=restr_str, - ), - skip_duplicates=True, - ) - - def fetch(self, *args, log_fetch=True, **kwargs): - """Log fetch for export.""" - ret = super().fetch(*args, **kwargs) - if log_fetch: - self._log_fetch(*args, **kwargs) - return ret - - def fetch1(self, *args, log_fetch=True, **kwargs): - """Log fetch1 for export.""" - ret = super().fetch1(*args, **kwargs) - if log_fetch: - self._log_fetch(*args, **kwargs) - return ret - # ------------------------------ Restrict by ------------------------------ def __lshift__(self, restriction) -> QueryExpression: @@ -1029,10 +847,13 @@ def check_threads(self, detailed=False, all_threads=False) -> DataFrame: class SpyglassMixinPart(SpyglassMixin, dj.Part): """ A part table for Spyglass Group tables. Assists in propagating - delete calls from upstreeam tables to downstream tables. + delete calls from upstream tables to downstream tables. """ + # TODO: See #1163 + def delete(self, *args, **kwargs): """Delete master and part entries.""" restriction = self.restriction or True # for (tbl & restr).delete() + (self.master & restriction).delete(*args, **kwargs) diff --git a/src/spyglass/utils/mixins/__init__.py b/src/spyglass/utils/mixins/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/spyglass/utils/mixins/export.py b/src/spyglass/utils/mixins/export.py new file mode 100644 index 000000000..ba05d14ce --- /dev/null +++ b/src/spyglass/utils/mixins/export.py @@ -0,0 +1,338 @@ +from atexit import register as exit_register +from atexit import unregister as exit_unregister +from collections import defaultdict +from contextvars import ContextVar +from functools import cached_property +from inspect import stack as inspect_stack +from os import environ +from re import match as re_match + +from datajoint.condition import make_condition +from datajoint.table import Table +from packaging.version import parse as version_parse + +from spyglass.utils.logging import logger +from spyglass.utils.sql_helper_fn import bash_escape_sql + +EXPORT_ENV_VAR = "SPYGLASS_EXPORT_ID" +FETCH_LOG_FLAG = ContextVar("FETCH_LOG_FLAG", default=True) + + +class ExportMixin: + + _export_cache = defaultdict(set) + + # ------------------------------ Version Info ----------------------------- + + @cached_property + def _spyglass_version(self): + """Get Spyglass version.""" + from spyglass import __version__ as sg_version + + ret = ".".join(sg_version.split(".")[:3]) # Ditch commit info + + if self._test_mode: + return ret[:16] if len(ret) > 16 else ret + + if not bool(re_match(r"^\d+\.\d+\.\d+", ret)): # Major.Minor.Patch + raise ValueError( + f"Spyglass version issues. Expected #.#.#, Got {ret}." + + "Please try running `hatch build` from your spyglass dir." + ) + + return ret + + def compare_versions( + self, version: str, other: str = None, msg: str = None + ) -> None: + """Compare two versions. Raise error if not equal. + + Parameters + ---------- + version : str + Version to compare. + other : str, optional + Other version to compare. Default None. Use self._spyglass_version. + msg : str, optional + Additional error message info. Default None. + """ + if self._test_mode: + return + + other = other or self._spyglass_version + + if version_parse(version) != version_parse(other): + raise RuntimeError( + f"Found mismatched versions: {version} vs {other}\n{msg}" + ) + + # ------------------------------- Dependency ------------------------------- + + @cached_property + def _export_table(self): + """Lazy load export selection table.""" + from spyglass.common.common_usage import ExportSelection + + return ExportSelection() + + # ------------------------------ ID Property ------------------------------ + + @property + def export_id(self): + """ID of export in progress. + + NOTE: User of an env variable to store export_id may not be thread safe. + Exports must be run in sequence, not parallel. + """ + + return int(environ.get(EXPORT_ENV_VAR, 0)) + + @export_id.setter + def export_id(self, value): + """Set ID of export using `table.export_id = X` notation.""" + if self.export_id != 0 and self.export_id != value: + raise RuntimeError("Export already in progress.") + environ[EXPORT_ENV_VAR] = str(value) + exit_register(self._export_id_cleanup) # End export on exit + + @export_id.deleter + def export_id(self): + """Delete ID of export using `del table.export_id` notation.""" + self._export_id_cleanup() + + def _export_id_cleanup(self): + """Cleanup export ID.""" + self._export_cache = dict() + if environ.get(EXPORT_ENV_VAR): + del environ[EXPORT_ENV_VAR] + exit_unregister(self._export_id_cleanup) # Remove exit hook + + # ------------------------------- Export API ------------------------------- + + def _start_export(self, paper_id, analysis_id): + """Start export process.""" + if self.export_id: + logger.info(f"Export {self.export_id} in progress. Starting new.") + self._stop_export(warn=False) + + self.export_id = self._export_table.insert1_return_pk( + dict( + paper_id=paper_id, + analysis_id=analysis_id, + spyglass_version=self._spyglass_version, + ) + ) + + def _stop_export(self, warn=True): + """End export process.""" + if not self.export_id and warn: + logger.warning("Export not in progress.") + del self.export_id + + # ------------------------------- Log Fetch ------------------------------- + + def _called_funcs(self): + """Get stack trace functions.""" + ignore = { + "__and__", # caught by restrict + "__mul__", # caught by join + "_called_funcs", # run here + "_log_fetch", # run here + "_log_fetch_nwb", # run here + "", + "_exec_file", + "_pseudo_sync_runner", + "_run_cell", + "_run_cmd_line_code", + "_run_with_log", + "execfile", + "init_code", + "initialize", + "inner", + "interact", + "launch_instance", + "mainloop", + "run", + "run_ast_nodes", + "run_cell", + "run_cell_async", + "run_code", + "run_line_magic", + "safe_execfile", + "start", + "start_ipython", + } + + ret = {i.function for i in inspect_stack()} - ignore + return ret + + def _log_fetch(self, restriction=None, *args, **kwargs): + """Log fetch for export.""" + if ( + not self.export_id + or self.database == "common_usage" + or not FETCH_LOG_FLAG.get() + ): + return + + banned = [ + "head", # Prevents on Table().head() call + "tail", # Prevents on Table().tail() call + "preview", # Prevents on Table() call + "_repr_html_", # Prevents on Table() call in notebook + "cautious_delete", # Prevents add on permission check during delete + # "get_abs_path", # Assumes that fetch_nwb will catch file/table + "_check_delete_permission", # Prevents on Table().delete() + "delete", # Prevents on Table().delete() + "_load_admin", # Prevents on permission check + ] # if called by any in banned, return + if set(banned) & self._called_funcs(): + return + + restr = restriction or self.restriction or True + limit = kwargs.get("limit") + offset = kwargs.get("offset") + if limit or offset: # Use result as restr if limit/offset + restr = self.restrict(restr).fetch( + log_export=False, as_dict=True, limit=limit, offset=offset + ) + + restr_str = make_condition(self, restr, set()) + + if restr_str is True: + restr_str = "True" # otherwise stored in table as '1' + + if isinstance(restr_str, str) and "SELECT" in restr_str: + raise RuntimeError( + "Export cannot handle subquery restrictions. Please submit a " + + "bug report on GitHub with the code you ran and this" + + f"restriction:\n\t{restr_str}" + ) + + if isinstance(restr_str, str) and len(restr_str) > 2048: + raise RuntimeError( + "Export cannot handle restrictions > 2048.\n\t" + + "If required, please open an issue on GitHub.\n\t" + + f"Restriction: {restr_str}" + ) + + if isinstance(restr_str, str): + restr_str = bash_escape_sql(restr_str, add_newline=False) + + if restr_str in self._export_cache[self.full_table_name]: + return + self._export_cache[self.full_table_name].add(restr_str) + + self._export_table.Table.insert1( + dict( + export_id=self.export_id, + table_name=self.full_table_name, + restriction=restr_str, + ) + ) + restr_logline = restr_str.replace("AND", "\n\tAND").replace( + "OR", "\n\tOR" + ) + logger.debug(f"\nTable: {self.full_table_name}\nRestr: {restr_logline}") + + def _log_fetch_nwb(self, table, table_attr): + """Log fetch_nwb for export table.""" + tbl_pk = "analysis_file_name" + fnames = self.fetch(tbl_pk, log_export=True) + logger.debug( + f"Export: fetch_nwb\nTable:{self.full_table_name},\nFiles: {fnames}" + ) + self._export_table.File.insert( + [{"export_id": self.export_id, tbl_pk: fname} for fname in fnames], + skip_duplicates=True, + ) + fnames_str = "('" + "', ".join(fnames) + "')" # log AnalysisFile table + table()._log_fetch(restriction=f"{tbl_pk} in {fnames_str}") + + def _run_join(self, **kwargs): + """Log join for export. + + Special case to log primary keys of each table in join, avoiding + long restriction strings. + """ + table_list = [self] + other = kwargs.get("other") + + if hasattr(other, "_log_fetch"): # Check if other has mixin + table_list.append(other) # can other._log_fetch + else: + logger.warning(f"Cannot export log join for\n{other}") + + joined = self.proj().join(other.proj(), log_export=False) + for table in table_list: # log separate for unique pks + if isinstance(table, type) and issubclass(table, Table): + table = table() # adapted from dj.declare.compile_foreign_key + for r in joined.fetch(*table.primary_key, as_dict=True): + table._log_fetch(restriction=r) + + def _run_with_log(self, method, *args, log_export=True, **kwargs): + """Run method, log fetch, and return result. + + Uses FETCH_LOG_FLAG to prevent multiple logs in one user call. + """ + log_this_call = FETCH_LOG_FLAG.get() # One log per fetch call + + if log_this_call and not self.database == "common_usage": + FETCH_LOG_FLAG.set(False) + + try: + ret = method(*args, **kwargs) + finally: + if log_this_call: + FETCH_LOG_FLAG.set(True) + + if log_export and self.export_id and log_this_call: + if getattr(method, "__name__", None) == "join": # special case + self._run_join(**kwargs) + else: + self._log_fetch(restriction=kwargs.get("restriction")) + logger.debug(f"Export: {self._called_funcs()}") + + return ret + + # -------------------------- Intercept DJ methods -------------------------- + + def fetch(self, *args, log_export=True, **kwargs): + """Log fetch for export.""" + if not self.export_id: + return super().fetch(*args, **kwargs) + return self._run_with_log( + super().fetch, *args, log_export=log_export, **kwargs + ) + + def fetch1(self, *args, log_export=True, **kwargs): + """Log fetch1 for export.""" + if not self.export_id: + return super().fetch1(*args, **kwargs) + return self._run_with_log( + super().fetch1, *args, log_export=log_export, **kwargs + ) + + def restrict(self, restriction): + """Log restrict for export.""" + if not self.export_id: + return super().restrict(restriction) + log_export = "fetch_nwb" not in self._called_funcs() + return self._run_with_log( + super().restrict, + restriction=dj.AndList([restriction, self.restriction]), + log_export=log_export, + ) + + def join(self, other, log_export=True, *args, **kwargs): + """Log join for export. + + Join in dj_helper_func related to fetch_nwb have `log_export=False` + because these entries are caught on the file cascade in RestrGraph. + """ + if not self.export_id: + return super().join(other=other, *args, **kwargs) + + return self._run_with_log( + super().join, other=other, log_export=log_export, *args, **kwargs + ) diff --git a/src/spyglass/utils/sql_helper_fn.py b/src/spyglass/utils/sql_helper_fn.py index 4735125d5..99b2fa52e 100644 --- a/src/spyglass/utils/sql_helper_fn.py +++ b/src/spyglass/utils/sql_helper_fn.py @@ -1,3 +1,5 @@ +import re +from functools import cached_property from os import system as os_system from pathlib import Path from typing import List @@ -6,9 +8,6 @@ from datajoint import FreeTable from datajoint import config as dj_config -from spyglass.settings import export_dir -from spyglass.utils import logger - class SQLDumpHelper: """Write a series of export files to export_dir/paper_id. @@ -40,6 +39,20 @@ def __init__( self.docker_id = docker_id self.spyglass_version = spyglass_version + @cached_property + def _export_dir(self): + """Lazy load export directory.""" + from spyglass.settings import export_dir + + return export_dir + + @cached_property + def _logger(self): + """Lazy load logger.""" + from spyglass.utils import logger + + return logger + def _get_credentials(self): """Get credentials for database connection.""" return { @@ -61,35 +74,13 @@ def _write_sql_cnf(self): file.write(template.format(**self._get_credentials())) cnf_path.chmod(0o600) - def _bash_escape(self, s): - """Escape restriction string for bash.""" - s = s.strip() - - replace_map = { - "WHERE ": "", # Remove preceding WHERE of dj.where_clause - " ": " ", # Squash double spaces - "( (": "((", # Squash double parens - ") )": ")", - '"': "'", # Replace double quotes with single - "`": "", # Remove backticks - " AND ": " \\\n\tAND ", # Add newline and tab for readability - " OR ": " \\\n\tOR ", # OR extra space to align with AND - ")AND(": ") \\\n\tAND (", - ")OR(": ") \\\n\tOR (", - "#": "\\#", - } - for old, new in replace_map.items(): - s = s.replace(old, new) - if s.startswith("(((") and s.endswith(")))"): - s = s[2:-2] # Remove extra parens for readability - return s - def _cmd_prefix(self, docker_id=None): """Get prefix for mysqldump command. Includes docker exec if needed.""" + default = "mysqldump --hex-blob " if not docker_id: - return "mysqldump " + return default return ( - f"docker exec -i {docker_id} \\\n\tmysqldump " + f"docker exec -i {docker_id} \\\n\t{default}" + "-u {user} --password={password} \\\n\t".format( **self._get_credentials() ) @@ -112,7 +103,7 @@ def write_mysqldump( self._write_sql_cnf() paper_dir = ( - Path(export_dir) / self.paper_id + Path(self._export_dir) / self.paper_id if not self.docker_id else Path(".") ) @@ -148,7 +139,7 @@ def write_mysqldump( for table in tables_by_db: if not (where := table.where_clause()): continue - where = self._bash_escape(where) + where = bash_escape_sql(where) database, table_name = ( table.full_table_name.replace("`", "") .replace("#", "\\#") @@ -166,7 +157,7 @@ def write_mysqldump( self._remove_encoding(dump_script) self._write_version_file() - logger.info(f"Export script written to {dump_script}") + self._logger.info(f"Export script written to {dump_script}") self._export_conda_env() @@ -178,7 +169,9 @@ def _remove_encoding(self, dump_script): def _write_version_file(self): """Write spyglass version to paper directory.""" - version_file = Path(export_dir) / self.paper_id / "spyglass_version" + version_file = ( + Path(self._export_dir) / self.paper_id / "spyglass_version" + ) if version_file.exists(): return with version_file.open("w") as file: @@ -189,7 +182,7 @@ def _export_conda_env(self): Renames environment name to paper_id. """ - yml_path = Path(export_dir) / self.paper_id / "environment.yml" + yml_path = Path(self._export_dir) / self.paper_id / "environment.yml" if yml_path.exists(): return command = f"conda env export > {yml_path}" @@ -202,4 +195,82 @@ def _export_conda_env(self): with yml_path.open("w") as file: yaml.dump(yml, file) - logger.info(f"Conda environment exported to {yml_path}") + self._logger.info(f"Conda environment exported to {yml_path}") + + +def remove_redundant(s): + """Remove redundant parentheses from a string. + + '((a=b)OR((c=d)AND((e=f))))' -> '(a=b) OR ((c=d) AND (e=f))' + + Full solve would require content parsing, this removes duplicates. + https://codegolf.stackexchange.com/questions/250596/remove-redundant-parentheses + """ + + def is_list(x): # Check if element is a list + return isinstance(x, list) + + def list_to_str(x): # Convert list to string + return "(%s)" % "".join(map(list_to_str, x)) if is_list(x) else x + + def flatten_list(nested): + ret = [flatten_list(e) if is_list(e) else e for e in nested if e] + return ret[0] if ret == [[*ret[0]]] else ret # first if all same + + tokens = repr("\"'" + s)[3:] # Quote to safely eval the string + as_list = tokens.translate({40: "',['", 41: "'],'"}) # parens -> square + flattened = flatten_list(eval(as_list)) # Flatten the nested list + as_str = list_to_str(flattened) # back to str + + # space out AND and OR for readability + return re.sub(r"\b(and|or)\b", r" \1 ", as_str, flags=re.IGNORECASE) + + +def bash_escape_sql(s, add_newline=True): + """Escape restriction string for bash. + + Parameters + ---------- + s : str + SQL restriction string + add_newline : bool, optional + Add newlines for readability around AND & OR. Default True + """ + s = s.strip() + if s.startswith("WHERE"): + s = s[5:].strip() + + # Balance parentheses - because make_condition may unbalance outside parens + n_open = s.count("(") + n_close = s.count(")") + add_open = max(0, n_close - n_open) + add_close = max(0, n_open - n_close) + balanced = "(" * add_open + s + ")" * add_close + + s = remove_redundant(balanced) + + replace_map = { + " ": " ", # Squash double spaces + "( (": "((", # Squash double parens + ") )": "))", + '"': "'", # Replace double quotes with single + "`": "", # Remove backticks + } + + if add_newline: + replace_map.update( + { + " AND ": " \\\n\tAND ", # Add newline and tab for readability + " OR ": " \\\n\tOR ", # OR extra space to align with AND + ")AND(": ") \\\n\tAND (", + ")OR(": ") \\\n\tOR (", + "#": "\\#", + } + ) + else: # Used in ExportMixin + replace_map.update({"%%%%": "%%"}) # Remove extra percent signs + + for old, new in replace_map.items(): + s = re.sub(re.escape(old), new, s) + + return s diff --git a/tests/common/conftest.py b/tests/common/conftest.py index 4dc40317c..83c0e87b7 100644 --- a/tests/common/conftest.py +++ b/tests/common/conftest.py @@ -51,3 +51,9 @@ def pos_interval_01(pos_src): @pytest.fixture(scope="session") def common_ephys(common): yield common.common_ephys + + +@pytest.fixture(scope="session") +def pop_common_electrode_group(common_ephys): + common_ephys.ElectrodeGroup.populate() + yield common_ephys.ElectrodeGroup diff --git a/tests/common/test_ephys.py b/tests/common/test_ephys.py index 37f298fdc..0de388b61 100644 --- a/tests/common/test_ephys.py +++ b/tests/common/test_ephys.py @@ -25,10 +25,9 @@ def test_electrode_populate(common_ephys): assert len(common_ephys.Electrode()) == 128, "Electrode.populate failed" -def test_elec_group_populate(common_ephys): - common_ephys.ElectrodeGroup.populate() +def test_elec_group_populate(pop_common_electrode_group): assert ( - len(common_ephys.ElectrodeGroup()) == 32 + len(pop_common_electrode_group) == 32 ), "ElectrodeGroup.populate failed" diff --git a/tests/common/test_position.py b/tests/common/test_position.py index 889dafa60..23db2091f 100644 --- a/tests/common/test_position.py +++ b/tests/common/test_position.py @@ -3,64 +3,6 @@ import pytest -@pytest.fixture(scope="session") -def common_position(common): - yield common.common_position - - -@pytest.fixture(scope="session") -def interval_position_info(common_position): - yield common_position.IntervalPositionInfo - - -@pytest.fixture(scope="session") -def default_param_key(): - yield {"position_info_param_name": "default"} - - -@pytest.fixture(scope="session") -def interval_key(common): - yield (common.IntervalList & "interval_list_name LIKE 'pos 0%'").fetch1( - "KEY" - ) - - -@pytest.fixture(scope="session") -def param_table(common_position, default_param_key, teardown): - param_table = common_position.PositionInfoParameters() - param_table.insert1(default_param_key, skip_duplicates=True) - yield param_table - - -@pytest.fixture(scope="session") -def upsample_position( - common, - common_position, - param_table, - default_param_key, - teardown, - interval_key, -): - params = (param_table & default_param_key).fetch1() - upsample_param_key = {"position_info_param_name": "upsampled"} - param_table.insert1( - { - **params, - **upsample_param_key, - "is_upsampled": 1, - "max_separation": 80, - "upsampling_sampling_rate": 500, - }, - skip_duplicates=True, - ) - interval_pos_key = {**interval_key, **upsample_param_key} - common_position.IntervalPositionInfoSelection.insert1( - interval_pos_key, skip_duplicates=True - ) - common_position.IntervalPositionInfo.populate(interval_pos_key) - yield interval_pos_key - - @pytest.fixture(scope="session") def interval_pos_key(upsample_position): yield upsample_position @@ -73,16 +15,17 @@ def test_interval_position_info_insert(common_position, interval_pos_key): @pytest.fixture(scope="session") def upsample_position_error( upsample_position, - default_param_key, - param_table, + default_interval_pos_param_key, + pos_info_param, common, common_position, teardown, - interval_key, + interval_keys, ): - params = (param_table & default_param_key).fetch1() + interval_key = interval_keys[0] + params = (pos_info_param & default_interval_pos_param_key).fetch1() upsample_param_key = {"position_info_param_name": "upsampled error"} - param_table.insert1( + pos_info_param.insert1( { **params, **upsample_param_key, @@ -98,6 +41,10 @@ def upsample_position_error( ) yield interval_pos_key + (common_position.IntervalPositionInfoSelection & interval_pos_key).delete( + safemode=False + ) + def test_interval_position_info_insert_error( interval_position_info, upsample_position_error diff --git a/tests/common/test_usage.py b/tests/common/test_usage.py index 71449b3e3..a3c7f6d70 100644 --- a/tests/common/test_usage.py +++ b/tests/common/test_usage.py @@ -10,9 +10,17 @@ def export_tbls(common): @pytest.fixture(scope="session") def gen_export_selection( - lfp, trodes_pos_v1, track_graph, export_tbls, populate_lfp + lfp, + trodes_pos_v1, + track_graph, + export_tbls, + populate_lfp, + pos_merge_tables, + pop_common_electrode_group, + common, ): ExportSelection, _ = export_tbls + pos_merge, lin_merge = pos_merge_tables _ = populate_lfp ExportSelection.start_export(paper_id=1, analysis_id=1) @@ -20,6 +28,20 @@ def gen_export_selection( trodes_pos_v1.fetch() ExportSelection.start_export(paper_id=1, analysis_id=2) track_graph.fetch() + ExportSelection.start_export(paper_id=1, analysis_id=3) + + _ = pop_common_electrode_group & "electrode_group_name = 1" + _ = common.IntervalPositionInfoSelection * ( + common.IntervalList & "interval_list_name = 'pos 1 valid times'" + ) + + ExportSelection.start_export(paper_id=1, analysis_id=4) + + merge_key = ( + pos_merge.TrodesPosV1 & "trodes_pos_params_name LIKE '%ups%'" + ).fetch1("KEY") + (pos_merge & merge_key).fetch_nwb() + ExportSelection.stop_export() yield dict(paper_id=1) @@ -33,7 +55,7 @@ def test_export_selection_files(gen_export_selection, export_tbls): paper_key = gen_export_selection len_fi = len(ExportSelection * ExportSelection.File & paper_key) - assert len_fi == 1, "Selection files not captured correctly" + assert len_fi == 2, "Selection files not captured correctly" def test_export_selection_tables(gen_export_selection, export_tbls): @@ -43,10 +65,44 @@ def test_export_selection_tables(gen_export_selection, export_tbls): paper = ExportSelection * ExportSelection.Table & paper_key len_tbl_1 = len(paper & dict(analysis_id=1)) len_tbl_2 = len(paper & dict(analysis_id=2)) - assert len_tbl_1 == 2, "Selection tables not captured correctly" + assert len_tbl_1 == 3, "Selection tables not captured correctly" assert len_tbl_2 == 1, "Selection tables not captured correctly" +def test_export_selection_joins(gen_export_selection, export_tbls, common): + ExportSelection, _ = export_tbls + paper_key = gen_export_selection + + restr = ( + ExportSelection * ExportSelection.Table + & paper_key + & dict(analysis_id=3) + ) + + assert "electrode_group_name = 1" in ( + restr & {"table_name": common.ElectrodeGroup.full_table_name} + ).fetch1("restriction"), "Export restriction not captured correctly" + + assert "pos 1 valid times" in ( + restr + & {"table_name": common.IntervalPositionInfoSelection.full_table_name} + ).fetch1("restriction"), "Export join not captured correctly" + + +def test_export_selection_merge_fetch( + gen_export_selection, export_tbls, trodes_pos_v1 +): + ExportSelection, _ = export_tbls + paper_key = gen_export_selection + + paper = ExportSelection * ExportSelection.Table & paper_key + restr = paper & dict(analysis_id=4) + + assert trodes_pos_v1.full_table_name in restr.fetch( + "table_name" + ), "Export merge not captured correctly" + + def tests_export_selection_max_id(gen_export_selection, export_tbls): ExportSelection, _ = export_tbls _ = gen_export_selection @@ -71,7 +127,7 @@ def test_export_populate(populate_export): table, file = populate_export assert len(file) == 4, "Export tables not captured correctly" - assert len(table) == 31, "Export files not captured correctly" + assert len(table) == 35, "Export files not captured correctly" def test_invalid_export_id(export_tbls): diff --git a/tests/conftest.py b/tests/conftest.py index 8cd07755a..de513d3a9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -482,6 +482,65 @@ def pos_interval_key(sgp, mini_copy_name, pos_interval): yield {"nwb_file_name": mini_copy_name, "interval_list_name": pos_interval} +@pytest.fixture(scope="session") +def common_position(common): + yield common.common_position + + +@pytest.fixture(scope="session") +def interval_position_info(common_position): + yield common_position.IntervalPositionInfo + + +@pytest.fixture(scope="session") +def default_interval_pos_param_key(): + yield {"position_info_param_name": "default"} + + +@pytest.fixture(scope="session") +def interval_keys(common): + yield (common.IntervalList & "interval_list_name LIKE 'pos %'").fetch("KEY") + + +@pytest.fixture(scope="session") +def pos_info_param(common_position, default_interval_pos_param_key, teardown): + pos_info_param = common_position.PositionInfoParameters() + pos_info_param.insert1(default_interval_pos_param_key, skip_duplicates=True) + yield pos_info_param + + +@pytest.fixture(scope="session") +def upsample_position( + common, + common_position, + pos_info_param, + default_interval_pos_param_key, + teardown, + interval_keys, +): + params = (pos_info_param & default_interval_pos_param_key).fetch1() + upsample_param_key = {"position_info_param_name": "upsampled"} + pos_info_param.insert1( + { + **params, + **upsample_param_key, + "is_upsampled": 1, + "max_separation": 80, + "upsampling_sampling_rate": 500, + }, + skip_duplicates=True, + ) + interval_pos_keys = [ + {**interval_key, **upsample_param_key} for interval_key in interval_keys + ] + common_position.IntervalPositionInfoSelection.insert( + interval_pos_keys, skip_duplicates=True + ) + common_position.IntervalPositionInfo.populate(interval_pos_keys) + + yield interval_pos_keys[0] + + @pytest.fixture(scope="session") def trodes_sel_keys( teardown, trodes_sel_table, pos_interval_key, trodes_params From dc08c2552bae7612e62ffe9ca2ddf84087f9a3d1 Mon Sep 17 00:00:00 2001 From: Samuel Bray Date: Wed, 6 Nov 2024 12:04:13 -0800 Subject: [PATCH 82/94] No transact bug (#1178) * check for single entry population case * fix typo * typo fix --- src/spyglass/utils/dj_mixin.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/spyglass/utils/dj_mixin.py b/src/spyglass/utils/dj_mixin.py index 090b1a3ee..5c0b7dcf3 100644 --- a/src/spyglass/utils/dj_mixin.py +++ b/src/spyglass/utils/dj_mixin.py @@ -145,10 +145,10 @@ def _nwb_table_tuple(self) -> tuple: Used to determine fetch_nwb behavior. Also used in Merge.fetch_nwb. Implemented as a cached_property to avoid circular imports.""" - from spyglass.common.common_nwbfile import ( # noqa F401 + from spyglass.common.common_nwbfile import ( AnalysisNwbfile, Nwbfile, - ) + ) # noqa F401 table_dict = { AnalysisNwbfile: "analysis_file_abs_path", @@ -519,11 +519,12 @@ def _hash_upstream(self, keys): List of keys for populating table. """ RestrGraph = self._graph_deps[1] - if not (parents := self.parents(as_objects=True, primary=True)): # Should not happen, as this is only called from populated tables raise RuntimeError("No upstream tables found for upstream hash.") + if isinstance(keys, dict): + keys = [keys] # case for single population key leaves = { # Restriction on each primary parent p.full_table_name: [ {k: v for k, v in key.items() if k in p.heading.names} @@ -550,7 +551,7 @@ def populate(self, *restrictions, **kwargs): processes = kwargs.pop("processes", 1) # Decide if using transaction protection - use_transact = kwargs.pop("use_transation", None) + use_transact = kwargs.pop("use_transaction", None) if use_transact is None: # if user does not specify, use class default use_transact = self._use_transaction if self._use_transaction is False: # If class default is off, warn From 0c5a903b319fcadaa054944058dbada6ad25461a Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Thu, 7 Nov 2024 07:32:16 -0800 Subject: [PATCH 83/94] `VideoMaker` trodes fixes and resume from fail (#1174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Additional 'start from fail' measures * WIP: fix TrodesPosVideo * WIP: get n_frames on trodes vid * WIP: error on upscaled data * ✅ : Add missing index * blackify * More test fixes for headless VideoMaker * pre-commit lint * revert matplotlib backend --- CHANGELOG.md | 7 +- dj_local_conf_example.json | 2 +- ...acting_Clusterless_Waveform_Features.ipynb | 2 +- ...xtracting_Clusterless_Waveform_Features.py | 2 +- src/spyglass/position/v1/dlc_utils.py | 4 +- src/spyglass/position/v1/dlc_utils_makevid.py | 207 ++++++++++-------- .../position/v1/position_dlc_selection.py | 5 +- .../position/v1/position_trodes_position.py | 94 ++++++-- tests/position/conftest.py | 1 - 9 files changed, 209 insertions(+), 115 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bfe55b55..bbcaac88d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,9 +65,10 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Minor fix to `DLCCentroid` make function order #1112, #1148 - Video creator tools: - Pass output path as string to `cv2.VideoWriter` #1150 - - Set `DLCPosVideo` default processor to `matplotlib`, remove support - for `open-cv` #1168 - - `VideoMaker` class to process frames in multithreaded batches #1168 + - Set `DLCPosVideo` default processor to `matplotlib`, remove support for + `open-cv` #1168 + - `VideoMaker` class to process frames in multithreaded batches #1168, #1174 + - `TrodesPosVideo` updates for `matplotlib` processor #1174 - Spike Sorting diff --git a/dj_local_conf_example.json b/dj_local_conf_example.json index b9b5e725e..bde731751 100644 --- a/dj_local_conf_example.json +++ b/dj_local_conf_example.json @@ -53,4 +53,4 @@ }, "kachery_zone": "franklab.default" } -} \ No newline at end of file +} diff --git a/notebooks/40_Extracting_Clusterless_Waveform_Features.ipynb b/notebooks/40_Extracting_Clusterless_Waveform_Features.ipynb index 07b3130a5..51ebbf36d 100644 --- a/notebooks/40_Extracting_Clusterless_Waveform_Features.ipynb +++ b/notebooks/40_Extracting_Clusterless_Waveform_Features.ipynb @@ -404,7 +404,7 @@ "for sorting_id in sorting_ids:\n", " try:\n", " sgs.CurationV1.insert_curation(sorting_id=sorting_id)\n", - " except KeyError as e:\n", + " except KeyError:\n", " pass\n", "\n", "SpikeSortingOutput.insert(\n", diff --git a/notebooks/py_scripts/40_Extracting_Clusterless_Waveform_Features.py b/notebooks/py_scripts/40_Extracting_Clusterless_Waveform_Features.py index ad17a7c6f..5449183a8 100644 --- a/notebooks/py_scripts/40_Extracting_Clusterless_Waveform_Features.py +++ b/notebooks/py_scripts/40_Extracting_Clusterless_Waveform_Features.py @@ -176,7 +176,7 @@ for sorting_id in sorting_ids: try: sgs.CurationV1.insert_curation(sorting_id=sorting_id) - except KeyError as e: + except KeyError: pass SpikeSortingOutput.insert( diff --git a/src/spyglass/position/v1/dlc_utils.py b/src/spyglass/position/v1/dlc_utils.py index 7ea82fa70..592e02964 100644 --- a/src/spyglass/position/v1/dlc_utils.py +++ b/src/spyglass/position/v1/dlc_utils.py @@ -434,7 +434,9 @@ def find_mp4( .rsplit(video_filepath.parent.as_posix(), maxsplit=1)[-1] .split("/")[-1] ) - return _convert_mp4(video_file, video_path, output_path, videotype="mp4") + return _convert_mp4( + video_file, video_path, output_path, videotype="mp4", count_frames=True + ) def _convert_mp4( diff --git a/src/spyglass/position/v1/dlc_utils_makevid.py b/src/spyglass/position/v1/dlc_utils_makevid.py index 51c209134..2763a7898 100644 --- a/src/spyglass/position/v1/dlc_utils_makevid.py +++ b/src/spyglass/position/v1/dlc_utils_makevid.py @@ -3,17 +3,17 @@ # some DLC-utils copied from datajoint element-interface utils.py import shutil import subprocess -from concurrent.futures import ProcessPoolExecutor, as_completed -from os import system as os_system +from concurrent.futures import ProcessPoolExecutor, TimeoutError, as_completed from pathlib import Path from typing import Tuple +import matplotlib import matplotlib.pyplot as plt import numpy as np import pandas as pd from tqdm import tqdm -from spyglass.settings import temp_dir +from spyglass.settings import temp_dir, test_mode from spyglass.utils import logger from spyglass.utils.position import convert_to_pixels as _to_px @@ -49,9 +49,9 @@ def __init__( cm_to_pixels=1.0, disable_progressbar=False, crop=None, - batch_size=500, - max_workers=25, - max_jobs_in_queue=250, + batch_size=512, + max_workers=256, + max_jobs_in_queue=128, debug=False, key_hash=None, *args, @@ -80,10 +80,16 @@ def __init__( if not Path(video_filename).exists(): raise FileNotFoundError(f"Video not found: {video_filename}") + try: + position_mean = position_mean["DLC"] + orientation_mean = orientation_mean["DLC"] + except IndexError: + pass # trodes data provides bare arrays + self.video_filename = video_filename self.video_frame_inds = video_frame_inds - self.position_mean = position_mean["DLC"] - self.orientation_mean = orientation_mean["DLC"] + self.position_mean = position_mean + self.orientation_mean = orientation_mean self.centroids = centroids self.likelihoods = likelihoods self.position_time = position_time @@ -94,16 +100,21 @@ def __init__( self.crop = crop self.window_ind = np.arange(501) - 501 // 2 self.debug = debug + self.start_time = pd.to_datetime(position_time[0] * 1e9, unit="ns") self.dropped_frames = set() self.batch_size = batch_size self.max_workers = max_workers self.max_jobs_in_queue = max_jobs_in_queue + self.timeout = 30 if test_mode else 300 self.ffmpeg_log_args = ["-hide_banner", "-loglevel", "error"] self.ffmpeg_fmt_args = ["-c:v", "libx264", "-pix_fmt", "yuv420p"] + prev_backend = matplotlib.get_backend() + matplotlib.use("Agg") # Use non-interactive backend + _ = self._set_frame_info() _ = self._set_plot_bases() @@ -116,15 +127,41 @@ def __init__( logger.info(f"Finished video: {self.output_video_filename}") logger.debug(f"Dropped frames: {self.dropped_frames}") - shutil.rmtree(self.temp_dir) # Clean up temp directory + if not debug: + shutil.rmtree(self.temp_dir) # Clean up temp directory + + matplotlib.use(prev_backend) # Reset to previous backend def _set_frame_info(self): """Set the frame information for the video.""" logger.debug("Setting frame information") - width, height, self.frame_rate = self._get_input_stats() + ret = subprocess.run( + [ + "ffprobe", + "-v", + "error", + "-select_streams", + "v", + "-show_entries", + "stream=width,height,r_frame_rate,nb_frames", + "-of", + "csv=p=0:s=x", + str(self.video_filename), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if ret.returncode != 0: + raise ValueError(f"Error getting video dimensions: {ret.stderr}") + + stats = ret.stdout.strip().split("x") + self.width, self.height = tuple(map(int, stats[:2])) + self.frame_rate = eval(stats[2]) + self.frame_size = ( - (width, height) + (self.width, self.height) if not self.crop else ( self.crop[1] - self.crop[0], @@ -138,45 +175,24 @@ def _set_frame_info(self): ) self.fps = int(np.round(self.frame_rate)) - if self.frames is None: + if self.frames is None and self.video_frame_inds is not None: self.n_frames = int( len(self.video_frame_inds) * self.percent_frames ) self.frames = np.arange(0, self.n_frames) - else: + elif self.frames is not None: self.n_frames = len(self.frames) - self.pad_len = len(str(self.n_frames)) + else: + self.n_frames = int(stats[3]) - def _get_input_stats(self, video_filename=None) -> Tuple[int, int]: - """Get the width and height of the video.""" - logger.debug("Getting video dimensions") + if self.debug: # If debugging, limit frames to available data + self.n_frames = min(len(self.position_mean), self.n_frames) - video_filename = video_filename or self.video_filename - ret = subprocess.run( - [ - "ffprobe", - "-v", - "error", - "-select_streams", - "v", - "-show_entries", - "stream=width,height,r_frame_rate", - "-of", - "csv=p=0:s=x", - video_filename, - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - if ret.returncode != 0: - raise ValueError(f"Error getting video dimensions: {ret.stderr}") - - stats = ret.stdout.strip().split("x") - width, height = tuple(map(int, stats[:-1])) - frame_rate = eval(stats[-1]) + self.pad_len = len(str(self.n_frames)) - return width, height, frame_rate + def _set_input_stats(self, video_filename=None) -> Tuple[int, int]: + """Get the width and height of the video.""" + logger.debug("Getting video stats with ffprobe") def _set_plot_bases(self): """Create the figure and axes for the video.""" @@ -202,7 +218,6 @@ def _set_plot_bases(self): zorder=102, color=color, label=f"{bodypart} position", - # animated=True, alpha=0.6, ) for color, bodypart in zip(COLOR_SWATCH, self.centroids.keys()) @@ -240,6 +255,7 @@ def _set_plot_bases(self): self.position_time[0] - self.position_time[-1] ).total_seconds() + # TODO: Update legend location based on centroid position axes[0].legend(loc="lower right", fontsize=4) self.title = axes[0].set_title( f"time = {time_delta:3.4f}s\n frame = {0}", @@ -305,12 +321,18 @@ def orient_list(c): def _generate_single_frame(self, frame_ind): """Generate a single frame and save it as an image.""" + # Zero-padded filename based on the dynamic padding length padded = self._pad(frame_ind) + frame_out_path = self.temp_dir / f"plot_{padded}.png" + if frame_out_path.exists() and not self.debug: + return frame_ind # Skip if frame already exists + frame_file = self.temp_dir / f"orig_{padded}.png" - if not frame_file.exists(): + if not frame_file.exists(): # Skip if input frame not found self.dropped_frames.add(frame_ind) - print(f"\rFrame not found: {frame_file}", end="") + self._debug_print(f"Frame not found: {frame_file}", end="") return + frame = plt.imread(frame_file) _ = self.axes[0].imshow(frame) @@ -322,42 +344,42 @@ def _generate_single_frame(self, frame_ind): self.centroid_plot_objs[bodypart].set_offsets((np.NaN, np.NaN)) self.orientation_line.set_data((np.NaN, np.NaN)) self.title.set_text(f"time = {0:3.4f}s\n frame = {frame_ind}") - else: - pos_ind = pos_ind[0] - likelihood_inds = pos_ind + self.window_ind - neg_inds = np.where(likelihood_inds < 0)[0] - likelihood_inds[neg_inds] = 0 if len(neg_inds) > 0 else -1 - - dlc_centroid_data = self._get_centroid_data(pos_ind) - - for bodypart in self.centroid_plot_objs: - self.centroid_plot_objs[bodypart].set_offsets( - _to_px( - data=self.centroids[bodypart][pos_ind], - cm_to_pixels=self.cm_to_pixels, - ) - ) - self.centroid_position_dot.set_offsets(dlc_centroid_data) - _ = self._set_orient_line(frame, pos_ind) - time_delta = pd.Timedelta( - pd.to_datetime(self.position_time[pos_ind] * 1e9, unit="ns") - - pd.to_datetime(self.position_time[0] * 1e9, unit="ns") - ).total_seconds() + self.fig.savefig(frame_out_path, dpi=400) + plt.cla() # clear the current axes + return frame_ind + + pos_ind = pos_ind[0] + likelihood_inds = pos_ind + self.window_ind + neg_inds = np.where(likelihood_inds < 0)[0] + likelihood_inds[neg_inds] = 0 if len(neg_inds) > 0 else -1 - self.title.set_text( - f"time = {time_delta:3.4f}s\n frame = {frame_ind}" + dlc_centroid_data = self._get_centroid_data(pos_ind) + + for bodypart in self.centroid_plot_objs: + self.centroid_plot_objs[bodypart].set_offsets( + _to_px( + data=self.centroids[bodypart][pos_ind], + cm_to_pixels=self.cm_to_pixels, + ) ) - if self.likelihoods: - for bodypart in self.likelihood_objs.keys(): - self.likelihood_objs[bodypart].set_data( - self.window_ind / self.frame_rate, - np.asarray(self.likelihoods[bodypart][likelihood_inds]), - ) + self.centroid_position_dot.set_offsets(dlc_centroid_data) + _ = self._set_orient_line(frame, pos_ind) - # Zero-padded filename based on the dynamic padding length - frame_path = self.temp_dir / f"plot_{padded}.png" - self.fig.savefig(frame_path, dpi=400) + time_delta = pd.Timedelta( + pd.to_datetime(self.position_time[pos_ind] * 1e9, unit="ns") + - self.start_time + ).total_seconds() + + self.title.set_text(f"time = {time_delta:3.4f}s\n frame = {frame_ind}") + if self.likelihoods: + for bodypart in self.likelihood_objs.keys(): + self.likelihood_objs[bodypart].set_data( + self.window_ind / self.frame_rate, + np.asarray(self.likelihoods[bodypart][likelihood_inds]), + ) + + self.fig.savefig(frame_out_path, dpi=400) plt.cla() # clear the current axes return frame_ind @@ -394,7 +416,7 @@ def process_frames(self): logger.info("Concatenating partial videos") self.concat_partial_videos() - def _debug_print(self, msg=None, end=""): + def _debug_print(self, msg=" ", end=""): """Print a self-overwiting message if debug is enabled.""" if self.debug: print(f"\r{msg}", end=end) @@ -411,7 +433,7 @@ def plot_frames(self, start_frame, end_frame, progress_bar=None): while len(jobs) < self.max_jobs_in_queue: try: this_frame = next(frames_iter) - self._debug_print(f"Submit: {this_frame}") + self._debug_print(f"Submit: {self._pad(this_frame)}") job = executor.submit( self._generate_single_frame, this_frame ) @@ -422,23 +444,28 @@ def plot_frames(self, start_frame, end_frame, progress_bar=None): for job in as_completed(jobs): frames_left -= 1 try: - ret = job.result() - except IndexError: - ret = "IndexError" - self._debug_print(f"Finish: {ret}") + ret = job.result(timeout=self.timeout) + except (IndexError, TimeoutError) as e: + ret = type(e).__name__ + self._debug_print(f"Finish: {self._pad(ret)}") progress_bar.update() del jobs[job] - self._debug_print(end="\n") + self._debug_print(msg="", end="\n") def ffmpeg_extract(self, start_frame, end_frame): """Use ffmpeg to extract a batch of frames.""" logger.debug(f"Extracting frames: {start_frame} - {end_frame}") + last_frame = self.temp_dir / f"orig_{self._pad(end_frame)}.png" + if last_frame.exists(): # assumes all frames previously extracted + logger.debug(f"Skipping existing frames: {last_frame}") + return + output_pattern = str(self.temp_dir / f"orig_%0{self.pad_len}d.png") # Use ffmpeg to extract frames ffmpeg_cmd = [ "ffmpeg", - "-y", # overwrite + "-n", # no overwrite "-i", self.video_filename, "-vf", @@ -447,7 +474,6 @@ def ffmpeg_extract(self, start_frame, end_frame): "vfr", "-start_number", str(start_frame), - "-n", # no overwrite output_pattern, *self.ffmpeg_log_args, ] @@ -462,7 +488,12 @@ def ffmpeg_extract(self, start_frame, end_frame): one_err = "\n".join(str(ret.stderr).split("\\")[-3:-1]) logger.debug(f"\nExtract Error: {one_err}") - def _pad(self, frame_ind): + def _pad(self, frame_ind=None): + """Pad a frame index with leading zeros.""" + if frame_ind is None: + return "?" * self.pad_len + elif not isinstance(frame_ind, int): + return frame_ind return f"{frame_ind:0{self.pad_len}d}" def ffmpeg_stitch_partial(self, start_frame, output_partial_video): @@ -493,6 +524,7 @@ def ffmpeg_stitch_partial(self, start_frame, output_partial_video): ) except subprocess.CalledProcessError as e: logger.error(f"Error stitching partial video: {e.stderr}") + logger.debug(f"stderr: {ret.stderr}") def concat_partial_videos(self): """Concatenate all the partial videos into one final video.""" @@ -526,6 +558,7 @@ def concat_partial_videos(self): ) except subprocess.CalledProcessError as e: logger.error(f"Error stitching partial video: {e.stderr}") + logger.debug(f"stderr: {ret.stderr}") def make_video(**kwargs): diff --git a/src/spyglass/position/v1/position_dlc_selection.py b/src/spyglass/position/v1/position_dlc_selection.py index 581140797..627a55cf7 100644 --- a/src/spyglass/position/v1/position_dlc_selection.py +++ b/src/spyglass/position/v1/position_dlc_selection.py @@ -8,9 +8,6 @@ import pynwb from datajoint.utils import to_camel_case -from spyglass.common.common_behav import ( - convert_epoch_interval_name_to_position_interval_name, -) from spyglass.common.common_nwbfile import AnalysisNwbfile from spyglass.position.v1.dlc_utils_makevid import make_video from spyglass.position.v1.position_dlc_centroid import DLCCentroid @@ -436,7 +433,7 @@ def make(self, key): cm_to_pixels=meters_per_pixel * M_TO_CM, crop=pose_estimation_params.get("cropping"), key_hash=dj.hash.key_hash(key), - debug=params.get("debug", False), + debug=params.get("debug", True), # REVERT TO FALSE **params.get("video_params", {}), ) diff --git a/src/spyglass/position/v1/position_trodes_position.py b/src/spyglass/position/v1/position_trodes_position.py index 72adada46..031c0e6bb 100644 --- a/src/spyglass/position/v1/position_trodes_position.py +++ b/src/spyglass/position/v1/position_trodes_position.py @@ -1,5 +1,6 @@ import copy import os +from pathlib import Path import datajoint as dj import numpy as np @@ -13,6 +14,7 @@ from spyglass.position.v1.dlc_utils_makevid import make_video from spyglass.settings import test_mode from spyglass.utils import SpyglassMixin, logger +from spyglass.utils.position import fill_nan schema = dj.schema("position_v1_trodes_position") @@ -270,7 +272,7 @@ def make(self, key): - Raw position data from the RawPosition table - Position data from the TrodesPosV1 table - Video data from the VideoFile table - Generates a video using opencv and the VideoMaker class. + Generates a video using VideoMaker class. """ M_TO_CM = 100 @@ -303,10 +305,31 @@ def make(self, key): {"nwb_file_name": key["nwb_file_name"], "epoch": epoch} ) + # Check if video exists if not video_path: self.insert1(dict(**key, has_video=False)) return + # Check timepoints overlap + if not set(video_time).intersection(set(pos_df.index)): + raise ValueError( + "No overlapping time points between video and position data" + ) + + params_pk = "trodes_pos_params_name" + params = (TrodesPosParams() & {params_pk: key[params_pk]}).fetch1( + "params" + ) + + # Check if upsampled + if params["is_upsampled"]: + logger.error( + "Upsampled position data not supported for video creation\n" + + "Please submit a feature request via GitHub if needed." + ) + self.insert1(dict(**key, has_video=False)) # Null insert + return + video_path = find_mp4( video_path=os.path.dirname(video_path) + "/", video_filename=video_filename, @@ -315,31 +338,70 @@ def make(self, key): output_video_filename = ( key["nwb_file_name"].replace(".nwb", "") + f"_{epoch:02d}_" - + f'{key["trodes_pos_params_name"]}.mp4' + + f"{key[params_pk]}.mp4" ) adj_df = _fix_col_names(raw_df) # adjust 'xloc1' to 'xloc' - if test_mode: + limit = params.get("limit", None) + if limit or test_mode: + params["debug"] = True + output_video_filename = Path(".") / f"TEST_VID_{limit}.mp4" # pytest video data has mismatched shapes in some cases - min_len = min(len(adj_df), len(pos_df), len(video_time)) - adj_df = adj_df[:min_len] - pos_df = pos_df[:min_len] + min_len = limit or min(len(adj_df), len(pos_df), len(video_time)) + adj_df = adj_df.head(min_len) + pos_df = pos_df.head(min_len) video_time = video_time[:min_len] - make_video( - processor="opencv-trodes", + centroids = { + "red": np.asarray(adj_df[["xloc", "yloc"]]), + "green": np.asarray(adj_df[["xloc2", "yloc2"]]), + } + position_mean = np.asarray(pos_df[["position_x", "position_y"]]) + orientation_mean = np.asarray(pos_df[["orientation"]]) + position_time = np.asarray(pos_df.index) + + ind_col = ( + pos_df["video_frame_ind"] + if "video_frame_ind" in pos_df.columns + else pos_df.index + ) + video_frame_inds = ind_col.astype(int).to_numpy() + + centroids = { + color: fill_nan( + variable=data, + video_time=video_time, + variable_time=position_time, + ) + for color, data in centroids.items() + } + position_mean = fill_nan( + variable=position_mean, + video_time=video_time, + variable_time=position_time, + ) + orientation_mean = fill_nan( + variable=orientation_mean, + video_time=video_time, + variable_time=position_time, + ) + + vid_maker = make_video( video_filename=video_path, - centroids={ - "red": np.asarray(adj_df[["xloc", "yloc"]]), - "green": np.asarray(adj_df[["xloc2", "yloc2"]]), - }, - position_mean=np.asarray(pos_df[["position_x", "position_y"]]), - orientation_mean=np.asarray(pos_df[["orientation"]]), + video_frame_inds=video_frame_inds, + centroids=centroids, video_time=video_time, - position_time=np.asarray(pos_df.index), + position_mean=position_mean, + orientation_mean=orientation_mean, + position_time=position_time, output_video_filename=output_video_filename, cm_to_pixels=meters_per_pixel * M_TO_CM, - disable_progressbar=False, + key_hash=dj.hash.key_hash(key), + **params, ) + + if limit: + return vid_maker + self.insert1(dict(**key, has_video=True)) diff --git a/tests/position/conftest.py b/tests/position/conftest.py index 8f9e90795..c6c58d199 100644 --- a/tests/position/conftest.py +++ b/tests/position/conftest.py @@ -30,7 +30,6 @@ def dlc_video_params(sgp): "params": { "percent_frames": 0.05, "incl_likelihood": True, - "processor": "opencv", }, }, skip_duplicates=True, From 42556d645d986930f183b23455fab484946577e3 Mon Sep 17 00:00:00 2001 From: Samuel Bray Date: Thu, 7 Nov 2024 13:26:43 -0800 Subject: [PATCH 84/94] Quickfix: pass config_probe (#1179) * pass config_probe * update changelog --- CHANGELOG.md | 2 +- src/spyglass/common/common_device.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bbcaac88d..db3a85074 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,7 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Import `datajoint.dependencies.unite_master_parts` -> `topo_sort` #1116, #1137, #1162 - Fix bool settings imported from dj config file #1117 -- Allow definition of tasks and new probe entries from config #1074, #1120 +- Allow definition of tasks and new probe entries from config #1074, #1120, #1179 - Enforce match between ingested nwb probe geometry and existing table entry #1074 - Update DataJoint install and password instructions #1131 diff --git a/src/spyglass/common/common_device.py b/src/spyglass/common/common_device.py index 1995c2303..1d20d8fb6 100644 --- a/src/spyglass/common/common_device.py +++ b/src/spyglass/common/common_device.py @@ -402,6 +402,7 @@ def insert_from_nwbfile(cls, nwbf, config=None): elif probe_type in config_probes: cls._read_config_probe_data( config, + config_probes, probe_type, new_probe_type_dict, new_probe_dict, @@ -562,6 +563,7 @@ def __read_ndx_probe_data( def _read_config_probe_data( cls, config, + config_probes, probe_type, new_probe_type_dict, new_probe_dict, From fa1114eb19edbb540bf4cbb30d1d002b7ba99748 Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Fri, 8 Nov 2024 12:24:19 -0800 Subject: [PATCH 85/94] Fix failing tests (#1181) --- CHANGELOG.md | 5 ++++- src/spyglass/utils/dj_merge_tables.py | 2 +- src/spyglass/utils/mixins/export.py | 4 ++-- tests/common/conftest.py | 2 +- tests/common/test_position.py | 2 +- tests/common/test_usage.py | 2 +- tests/utils/conftest.py | 8 +++++--- 7 files changed, 15 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index db3a85074..6e91b59ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,8 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Import `datajoint.dependencies.unite_master_parts` -> `topo_sort` #1116, #1137, #1162 - Fix bool settings imported from dj config file #1117 -- Allow definition of tasks and new probe entries from config #1074, #1120, #1179 +- Allow definition of tasks and new probe entries from config #1074, #1120, + #1179 - Enforce match between ingested nwb probe geometry and existing table entry #1074 - Update DataJoint install and password instructions #1131 @@ -35,9 +36,11 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Remove mambaforge from tests #1153 - Remove debug statement #1164 - Add testing for python versions 3.9, 3.10, 3.11, 3.12 #1169 + - Initialize tables in pytests #1181 - Allow python \< 3.13 #1169 - Remove numpy version restriction #1169 - Merge table delete removes orphaned master entries #1164 +- Edit `merge_fetch` to expect positional before keyword arguments #1181 ### Pipelines diff --git a/src/spyglass/utils/dj_merge_tables.py b/src/spyglass/utils/dj_merge_tables.py index 58532fa9f..7d6bf46ba 100644 --- a/src/spyglass/utils/dj_merge_tables.py +++ b/src/spyglass/utils/dj_merge_tables.py @@ -774,7 +774,7 @@ def merge_restrict_class( return parent_class & parent_key def merge_fetch( - self, restriction: str = True, log_export=True, *attrs, **kwargs + self, *attrs, restriction: str = True, log_export=True, **kwargs ) -> list: """Perform a fetch across all parts. If >1 result, return as a list. diff --git a/src/spyglass/utils/mixins/export.py b/src/spyglass/utils/mixins/export.py index ba05d14ce..222963ebb 100644 --- a/src/spyglass/utils/mixins/export.py +++ b/src/spyglass/utils/mixins/export.py @@ -7,7 +7,7 @@ from os import environ from re import match as re_match -from datajoint.condition import make_condition +from datajoint.condition import AndList, make_condition from datajoint.table import Table from packaging.version import parse as version_parse @@ -320,7 +320,7 @@ def restrict(self, restriction): log_export = "fetch_nwb" not in self._called_funcs() return self._run_with_log( super().restrict, - restriction=dj.AndList([restriction, self.restriction]), + restriction=AndList([restriction, self.restriction]), log_export=log_export, ) diff --git a/tests/common/conftest.py b/tests/common/conftest.py index 83c0e87b7..ebae0e004 100644 --- a/tests/common/conftest.py +++ b/tests/common/conftest.py @@ -56,4 +56,4 @@ def common_ephys(common): @pytest.fixture(scope="session") def pop_common_electrode_group(common_ephys): common_ephys.ElectrodeGroup.populate() - yield common_ephys.ElectrodeGroup + yield common_ephys.ElectrodeGroup() diff --git a/tests/common/test_position.py b/tests/common/test_position.py index 23db2091f..43a979c18 100644 --- a/tests/common/test_position.py +++ b/tests/common/test_position.py @@ -97,7 +97,7 @@ def position_video(common_position): def test_position_video(position_video, upsample_position): _ = position_video.populate() - assert len(position_video) == 1, "Failed to populate PositionVideo table." + assert len(position_video) == 2, "Failed to populate PositionVideo table." def test_convert_to_pixels(): diff --git a/tests/common/test_usage.py b/tests/common/test_usage.py index a3c7f6d70..8e50be14e 100644 --- a/tests/common/test_usage.py +++ b/tests/common/test_usage.py @@ -127,7 +127,7 @@ def test_export_populate(populate_export): table, file = populate_export assert len(file) == 4, "Export tables not captured correctly" - assert len(table) == 35, "Export files not captured correctly" + assert len(table) == 37, "Export files not captured correctly" def test_invalid_export_id(export_tbls): diff --git a/tests/utils/conftest.py b/tests/utils/conftest.py index 3723f191c..de5a80c4d 100644 --- a/tests/utils/conftest.py +++ b/tests/utils/conftest.py @@ -243,14 +243,16 @@ def graph_tables(dj_conn, graph_schema): # Merge inserts after declaring tables merge_keys = graph_schema["PkNode"].fetch("KEY", offset=1, as_dict=True) graph_schema["MergeOutput"].insert(merge_keys, skip_duplicates=True) - merge_child_keys = graph_schema["MergeOutput"].merge_fetch( - True, "merge_id", offset=1 + merge_child_keys = graph_schema["MergeOutput"]().merge_fetch( + "merge_id", restriction=True, offset=1 ) merge_child_inserts = [ (i, j, k + 10) for i, j, k in zip(merge_child_keys, range(4), range(10, 15)) ] - graph_schema["MergeChild"].insert(merge_child_inserts, skip_duplicates=True) + graph_schema["MergeChild"]().insert( + merge_child_inserts, skip_duplicates=True + ) yield graph_schema From 239f2a5912db3a7ae07aab8576b13a097ca7e699 Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Tue, 12 Nov 2024 13:51:01 -0600 Subject: [PATCH 86/94] No-credential data download (#1180) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * No-credential data download * Update Changelog * Fix typo * Run tests on each commit for labeled PR * Remove runif conditional on label * logger.warn -> warning * 🐛 : debug 1 * 🐛 : debug 2 * 🐛 : debug 3 * 🐛 : debug 4 --- .github/workflows/test-conda.yml | 30 +++---- CHANGELOG.md | 1 + environment.yml | 2 +- environment_dlc.yml | 2 +- src/spyglass/common/common_behav.py | 6 +- src/spyglass/common/common_device.py | 12 +-- src/spyglass/common/common_dio.py | 2 +- src/spyglass/common/common_session.py | 4 +- src/spyglass/common/common_subject.py | 2 +- src/spyglass/common/common_task.py | 10 +-- .../common/prepopulate/prepopulate.py | 2 +- src/spyglass/position/v1/dlc_utils_makevid.py | 26 +++--- .../position/v1/position_trodes_position.py | 9 +- src/spyglass/spikesorting/utils.py | 4 +- src/spyglass/spikesorting/v0/sortingview.py | 2 +- .../spikesorting/v0/sortingview_helper_fn.py | 2 +- .../spikesorting/v0/spikesorting_artifact.py | 2 +- .../spikesorting/v0/spikesorting_curation.py | 2 +- src/spyglass/spikesorting/v1/artifact.py | 4 +- .../spikesorting/v1/figurl_curation.py | 2 +- .../spikesorting/v1/metric_curation.py | 2 +- src/spyglass/spikesorting/v1/recording.py | 2 +- src/spyglass/utils/database_settings.py | 1 + src/spyglass/utils/dj_helper_fn.py | 2 +- src/spyglass/utils/dj_merge_tables.py | 8 +- src/spyglass/utils/dj_mixin.py | 4 +- src/spyglass/utils/nwb_helper_fn.py | 2 +- tests/README.md | 6 -- tests/conftest.py | 3 +- tests/data_downloader.py | 89 ++++++------------- 30 files changed, 109 insertions(+), 136 deletions(-) diff --git a/.github/workflows/test-conda.yml b/.github/workflows/test-conda.yml index d34e83c01..6ff4437e4 100644 --- a/.github/workflows/test-conda.yml +++ b/.github/workflows/test-conda.yml @@ -7,6 +7,8 @@ on: - '!documentation' schedule: # once a day at midnight UTC - cron: '0 0 * * *' + pull_request: # requires approval for first-time contributors + types: [synchronize, opened, reopened, labeled] workflow_dispatch: # Manually trigger with 'Run workflow' button concurrency: # Replace Cancel Workflow Action @@ -22,8 +24,6 @@ jobs: env: OS: ubuntu-latest PYTHON: '3.9' - UCSF_BOX_TOKEN: ${{ secrets.UCSF_BOX_TOKEN }} # for download and testing - UCSF_BOX_USER: ${{ secrets.UCSF_BOX_USER }} services: mysql: image: datajoint/mysql:8.0 @@ -57,23 +57,23 @@ jobs: pip install --quiet .[test] - name: Download data env: - BASEURL: ftps://ftp.box.com/trodes_to_nwb_test_data/ - NWBFILE: minirec20230622.nwb # Relative to Base URL - VID_ONE: 20230622_sample_01_a1/20230622_sample_01_a1.1.h264 - VID_TWO: 20230622_sample_02_a1/20230622_sample_02_a1.1.h264 + BASEURL: https://ucsf.box.com/shared/static/ + NWB_URL: k3sgql6z475oia848q1rgms4zdh4rkjn.nwb + VID1URL: ykep8ek4ogad20wz4p0vuyuqfo60cv3w.h264 + VID2URL: d2jjk0y565ru75xqojio3hymmehzr5he.h264 + NWBFILE: minirec20230622.nwb + VID_ONE: 20230622_minirec_01_s1.1.h264 + VID_TWO: 20230622_minirec_02_s2.1.h264 RAW_DIR: /home/runner/work/spyglass/spyglass/tests/_data/raw/ VID_DIR: /home/runner/work/spyglass/spyglass/tests/_data/video/ run: | mkdir -p $RAW_DIR $VID_DIR - wget_opts() { # Declare func with download options - wget \ - --recursive --no-verbose --no-host-directories --no-directories \ - --user "$UCSF_BOX_USER" --password "$UCSF_BOX_TOKEN" \ - -P "$1" "$BASEURL""$2" + curl_opts() { # Declare func with download options + curl -L --output "$1""$2" "$BASEURL""$3" } - wget_opts $RAW_DIR $NWBFILE - wget_opts $VID_DIR $VID_ONE - wget_opts $VID_DIR $VID_TWO + curl_opts $RAW_DIR $NWBFILE $NWB_URL + curl_opts $VID_DIR $VID_ONE $VID1URL + curl_opts $VID_DIR $VID_TWO $VID2URL - name: Run tests run: | - pytest --no-docker --no-dlc + pytest --no-docker --no-dlc tests/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e91b59ed..dd3bbeeb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Remove debug statement #1164 - Add testing for python versions 3.9, 3.10, 3.11, 3.12 #1169 - Initialize tables in pytests #1181 + - Download test data without credentials, trigger on approved PRs #1180 - Allow python \< 3.13 #1169 - Remove numpy version restriction #1169 - Merge table delete removes orphaned master entries #1164 diff --git a/environment.yml b/environment.yml index a5229b8a8..26155a7cc 100644 --- a/environment.yml +++ b/environment.yml @@ -9,7 +9,7 @@ name: spyglass channels: - conda-forge - - defaults + # - defaults # deprecated - franklab - edeno # - pytorch # dlc-only diff --git a/environment_dlc.yml b/environment_dlc.yml index 156bed793..df6026c18 100644 --- a/environment_dlc.yml +++ b/environment_dlc.yml @@ -9,7 +9,7 @@ name: spyglass-dlc channels: - conda-forge - - defaults + # - defaults # deprecated - franklab - edeno - pytorch # dlc-only diff --git a/src/spyglass/common/common_behav.py b/src/spyglass/common/common_behav.py index bab4ba075..53f8b7dcd 100644 --- a/src/spyglass/common/common_behav.py +++ b/src/spyglass/common/common_behav.py @@ -197,7 +197,7 @@ def fetch1_dataframe(self): id_rp = [(n["id"], n["raw_position"]) for n in self.fetch_nwb()] if len(set(rp.interval for _, rp in id_rp)) > 1: - logger.warn("Loading DataFrame with multiple intervals.") + logger.warning("Loading DataFrame with multiple intervals.") df_list = [ pd.DataFrame( @@ -395,7 +395,9 @@ def _no_transaction_make(self, key, verbose=True, skip_duplicates=False): ) if videos is None: - logger.warn(f"No video data interface found in {nwb_file_name}\n") + logger.warning( + f"No video data interface found in {nwb_file_name}\n" + ) return else: videos = videos.time_series diff --git a/src/spyglass/common/common_device.py b/src/spyglass/common/common_device.py index 1d20d8fb6..3fe14ec55 100644 --- a/src/spyglass/common/common_device.py +++ b/src/spyglass/common/common_device.py @@ -88,7 +88,9 @@ def insert_from_nwbfile(cls, nwbf, config=None): + f"{ndx_devices.keys()}" ) else: - logger.warn("No conforming data acquisition device metadata found.") + logger.warning( + "No conforming data acquisition device metadata found." + ) @classmethod def get_all_device_names(cls, nwbf, config) -> tuple: @@ -305,7 +307,7 @@ def insert_from_nwbfile(cls, nwbf, config=None): if device_name_list: logger.info(f"Inserted camera devices {device_name_list}") else: - logger.warn("No conforming camera device metadata found.") + logger.warning("No conforming camera device metadata found.") return device_name_list @@ -462,7 +464,7 @@ def insert_from_nwbfile(cls, nwbf, config=None): if all_probes_types: logger.info(f"Inserted probes {all_probes_types}") else: - logger.warn("No conforming probe metadata found.") + logger.warning("No conforming probe metadata found.") return all_probes_types @@ -709,7 +711,7 @@ def create_from_nwbfile( query = ProbeType & {"probe_type": probe_type} if len(query) == 0: - logger.warn( + logger.warning( f"No ProbeType found with probe_type '{probe_type}'. Aborting." ) return @@ -769,7 +771,7 @@ def create_from_nwbfile( ] if not device_found: - logger.warn( + logger.warning( "No electrodes in the NWB file were associated with a device " + f"named '{nwb_device_name}'." ) diff --git a/src/spyglass/common/common_dio.py b/src/spyglass/common/common_dio.py index 228e9caf9..6b8f29c17 100644 --- a/src/spyglass/common/common_dio.py +++ b/src/spyglass/common/common_dio.py @@ -38,7 +38,7 @@ def make(self, key): nwbf, "behavioral_events", pynwb.behavior.BehavioralEvents ) if behav_events is None: - logger.warn( + logger.warning( "No conforming behavioral events data interface found in " + f"{nwb_file_name}\n" ) diff --git a/src/spyglass/common/common_session.py b/src/spyglass/common/common_session.py index 5095ed6ed..3328857e5 100644 --- a/src/spyglass/common/common_session.py +++ b/src/spyglass/common/common_session.py @@ -152,7 +152,7 @@ def _add_data_acquisition_device_part(self, nwb_file_name, nwbf, config={}): "data_acquisition_device_name": device_name } if len(query) == 0: - logger.warn( + logger.warning( "Cannot link Session with DataAcquisitionDevice.\n" + f"DataAcquisitionDevice does not exist: {device_name}" ) @@ -180,7 +180,7 @@ def _add_experimenter_part( # ensure that the foreign key exists and do nothing if not query = LabMember & {"lab_member_name": name} if len(query) == 0: - logger.warn( + logger.warning( "Cannot link Session with LabMember. " + f"LabMember does not exist: {name}" ) diff --git a/src/spyglass/common/common_subject.py b/src/spyglass/common/common_subject.py index 2b8dc071a..c7757e087 100644 --- a/src/spyglass/common/common_subject.py +++ b/src/spyglass/common/common_subject.py @@ -37,7 +37,7 @@ def insert_from_nwbfile(cls, nwbf: NWBFile, config: dict = None): """ config = config or dict() if "Subject" not in config and nwbf.subject is None: - logger.warn("No subject metadata found.\n") + logger.warning("No subject metadata found.\n") return None conf = config["Subject"][0] if "Subject" in config else dict() diff --git a/src/spyglass/common/common_task.py b/src/spyglass/common/common_task.py index 60fba08d3..aabf381ac 100644 --- a/src/spyglass/common/common_task.py +++ b/src/spyglass/common/common_task.py @@ -33,7 +33,7 @@ def insert_from_nwbfile(cls, nwbf: pynwb.NWBFile): """ tasks_mod = nwbf.processing.get("tasks") if tasks_mod is None: - logger.warn(f"No tasks processing module found in {nwbf}\n") + logger.warning(f"No tasks processing module found in {nwbf}\n") return for task in tasks_mod.data_interfaces.values(): if cls.check_task_table(task): @@ -136,7 +136,7 @@ def make(self, key): tasks_mod = nwbf.processing.get("tasks") config_tasks = config.get("Tasks", []) if tasks_mod is None and (not config_tasks): - logger.warn( + logger.warning( f"No tasks processing module found in {nwbf} or config\n" ) return @@ -163,7 +163,7 @@ def make(self, key): for camera_id in valid_camera_ids ] else: - logger.warn( + logger.warning( f"No camera device found with ID {camera_ids} in NWB " + f"file {nwbf}\n" ) @@ -186,7 +186,7 @@ def make(self, key): epoch, session_intervals ) if target_interval is None: - logger.warn("Skipping epoch.") + logger.warning("Skipping epoch.") continue key["interval_list_name"] = target_interval task_inserts.append(key.copy()) @@ -219,7 +219,7 @@ def make(self, key): epoch, session_intervals ) if target_interval is None: - logger.warn("Skipping epoch.") + logger.warning("Skipping epoch.") continue new_key["interval_list_name"] = target_interval task_inserts.append(key.copy()) diff --git a/src/spyglass/common/prepopulate/prepopulate.py b/src/spyglass/common/prepopulate/prepopulate.py index ecad63a25..9ba703cd5 100644 --- a/src/spyglass/common/prepopulate/prepopulate.py +++ b/src/spyglass/common/prepopulate/prepopulate.py @@ -57,7 +57,7 @@ def populate_from_yaml(yaml_path: str): if k in table_cls.primary_key } if not primary_key_values: - logger.warn( + logger.warning( f"Populate: No primary key provided in data {entry_dict} " + f"for table {table_cls.__name__}" ) diff --git a/src/spyglass/position/v1/dlc_utils_makevid.py b/src/spyglass/position/v1/dlc_utils_makevid.py index 2763a7898..6f0df6289 100644 --- a/src/spyglass/position/v1/dlc_utils_makevid.py +++ b/src/spyglass/position/v1/dlc_utils_makevid.py @@ -5,7 +5,6 @@ import subprocess from concurrent.futures import ProcessPoolExecutor, TimeoutError, as_completed from pathlib import Path -from typing import Tuple import matplotlib import matplotlib.pyplot as plt @@ -190,10 +189,6 @@ def _set_frame_info(self): self.pad_len = len(str(self.n_frames)) - def _set_input_stats(self, video_filename=None) -> Tuple[int, int]: - """Get the width and height of the video.""" - logger.debug("Getting video stats with ffprobe") - def _set_plot_bases(self): """Create the figure and axes for the video.""" logger.debug("Setting plot bases") @@ -309,15 +304,20 @@ def centroid_to_px(*idx): ) ) - def _set_orient_line(self, frame, pos_ind): - def orient_list(c): - return [c, c + 30 * np.cos(self.orientation_mean[pos_ind])] + def _get_orient_line(self, pos_ind): + orient = self.orientation_mean[pos_ind] + if isinstance(orient, np.ndarray): + orient = orient[0] # Trodes passes orientation as a 1D array - if np.all(np.isnan(self.orientation_mean[pos_ind])): - self.orientation_line.set_data((np.NaN, np.NaN)) + def orient_list(c, axis="x"): + func = np.cos if axis == "x" else np.sin + return [c, c + 30 * func(orient)] + + if np.all(np.isnan(orient)): + return ([np.NaN], [np.NaN]) else: - c0, c1 = self._get_centroid_data(pos_ind) - self.orientation_line.set_data(orient_list(c0), orient_list(c1)) + x, y = self._get_centroid_data(pos_ind) + return (orient_list(x), orient_list(y, axis="y")) def _generate_single_frame(self, frame_ind): """Generate a single frame and save it as an image.""" @@ -364,7 +364,7 @@ def _generate_single_frame(self, frame_ind): ) ) self.centroid_position_dot.set_offsets(dlc_centroid_data) - _ = self._set_orient_line(frame, pos_ind) + self.orientation_line.set_data(self._get_orient_line(pos_ind)) time_delta = pd.Timedelta( pd.to_datetime(self.position_time[pos_ind] * 1e9, unit="ns") diff --git a/src/spyglass/position/v1/position_trodes_position.py b/src/spyglass/position/v1/position_trodes_position.py index 031c0e6bb..f7684450e 100644 --- a/src/spyglass/position/v1/position_trodes_position.py +++ b/src/spyglass/position/v1/position_trodes_position.py @@ -344,9 +344,14 @@ def make(self, key): adj_df = _fix_col_names(raw_df) # adjust 'xloc1' to 'xloc' limit = params.get("limit", None) - if limit or test_mode: + + if limit and not test_mode: params["debug"] = True output_video_filename = Path(".") / f"TEST_VID_{limit}.mp4" + elif test_mode: + limit = 10 + + if limit: # pytest video data has mismatched shapes in some cases min_len = limit or min(len(adj_df), len(pos_df), len(video_time)) adj_df = adj_df.head(min_len) @@ -401,7 +406,7 @@ def make(self, key): **params, ) - if limit: + if limit and not test_mode: return vid_maker self.insert1(dict(**key, has_video=True)) diff --git a/src/spyglass/spikesorting/utils.py b/src/spyglass/spikesorting/utils.py index d99a71ff2..f69359838 100644 --- a/src/spyglass/spikesorting/utils.py +++ b/src/spyglass/spikesorting/utils.py @@ -108,7 +108,7 @@ def get_group_by_shank( if omit_ref_electrode_group and ( str(e_group) == str(ref_elec_group) ): - logger.warn( + logger.warning( f"Omitting electrode group {e_group} from sort groups " + "because contains reference." ) @@ -117,7 +117,7 @@ def get_group_by_shank( # omit unitrodes if indicated if omit_unitrode and len(shank_elect) == 1: - logger.warn( + logger.warning( f"Omitting electrode group {e_group}, shank {shank} " + "from sort groups because unitrode." ) diff --git a/src/spyglass/spikesorting/v0/sortingview.py b/src/spyglass/spikesorting/v0/sortingview.py index ff62f09b8..baa99131c 100644 --- a/src/spyglass/spikesorting/v0/sortingview.py +++ b/src/spyglass/spikesorting/v0/sortingview.py @@ -72,7 +72,7 @@ def make(self, key: dict): LabMember.LabMemberInfo & {"lab_member_name": team_member} ).fetch("google_user_name") if len(google_user_id) != 1: - logger.warn( + logger.warning( f"Google user ID for {team_member} does not exist or more than one ID detected;\ permission not given to {team_member}, skipping..." ) diff --git a/src/spyglass/spikesorting/v0/sortingview_helper_fn.py b/src/spyglass/spikesorting/v0/sortingview_helper_fn.py index 431d99e88..3ecdd5eae 100644 --- a/src/spyglass/spikesorting/v0/sortingview_helper_fn.py +++ b/src/spyglass/spikesorting/v0/sortingview_helper_fn.py @@ -136,7 +136,7 @@ def _generate_url( ) if initial_curation is not None: - logger.warn("found initial curation") + logger.warning("found initial curation") sorting_curation_uri = kcl.store_json(initial_curation) else: sorting_curation_uri = None diff --git a/src/spyglass/spikesorting/v0/spikesorting_artifact.py b/src/spyglass/spikesorting/v0/spikesorting_artifact.py index b665b2314..71edc5de1 100644 --- a/src/spyglass/spikesorting/v0/spikesorting_artifact.py +++ b/src/spyglass/spikesorting/v0/spikesorting_artifact.py @@ -263,7 +263,7 @@ def _get_artifact_times( [[valid_timestamps[0], valid_timestamps[-1]]] ) artifact_times_empty = np.asarray([]) - logger.warn("No artifacts detected.") + logger.warning("No artifacts detected.") return recording_interval, artifact_times_empty # convert indices to intervals diff --git a/src/spyglass/spikesorting/v0/spikesorting_curation.py b/src/spyglass/spikesorting/v0/spikesorting_curation.py index e8c2f149e..27aa074d3 100644 --- a/src/spyglass/spikesorting/v0/spikesorting_curation.py +++ b/src/spyglass/spikesorting/v0/spikesorting_curation.py @@ -266,7 +266,7 @@ def save_sorting_nwb( AnalysisNwbfile().add(key["nwb_file_name"], analysis_file_name) if object_ids == "": - logger.warn( + logger.warning( "Sorting contains no units." "Created an empty analysis nwb file anyway." ) diff --git a/src/spyglass/spikesorting/v1/artifact.py b/src/spyglass/spikesorting/v1/artifact.py index 9a4fbeaef..14c82dba6 100644 --- a/src/spyglass/spikesorting/v1/artifact.py +++ b/src/spyglass/spikesorting/v1/artifact.py @@ -98,7 +98,7 @@ def insert_selection(cls, key: dict): """ query = cls & key if query: - logger.warn("Similar row(s) already inserted.") + logger.warning("Similar row(s) already inserted.") return query.fetch(as_dict=True) key["artifact_id"] = uuid.uuid4() cls.insert1(key, skip_duplicates=True) @@ -290,7 +290,7 @@ def _get_artifact_times( [[valid_timestamps[0], valid_timestamps[-1]]] ) artifact_times_empty = np.asarray([]) - logger.warn("No artifacts detected.") + logger.warning("No artifacts detected.") return recording_interval, artifact_times_empty # convert indices to intervals diff --git a/src/spyglass/spikesorting/v1/figurl_curation.py b/src/spyglass/spikesorting/v1/figurl_curation.py index 5be842d15..a79ee6246 100644 --- a/src/spyglass/spikesorting/v1/figurl_curation.py +++ b/src/spyglass/spikesorting/v1/figurl_curation.py @@ -51,7 +51,7 @@ def insert_selection(cls, key: dict): if "figurl_curation_id" in key: query = cls & {"figurl_curation_id": key["figurl_curation_id"]} if query: - logger.warn("Similar row(s) already inserted.") + logger.warning("Similar row(s) already inserted.") return query.fetch(as_dict=True) key["figurl_curation_id"] = uuid.uuid4() cls.insert1(key, skip_duplicates=True) diff --git a/src/spyglass/spikesorting/v1/metric_curation.py b/src/spyglass/spikesorting/v1/metric_curation.py index 8346f8ddd..43e09a8de 100644 --- a/src/spyglass/spikesorting/v1/metric_curation.py +++ b/src/spyglass/spikesorting/v1/metric_curation.py @@ -190,7 +190,7 @@ def insert_selection(cls, key: dict): key for the inserted row """ if cls & key: - logger.warn("This row has already been inserted.") + logger.warning("This row has already been inserted.") return (cls & key).fetch1() key["metric_curation_id"] = uuid.uuid4() cls.insert1(key, skip_duplicates=True) diff --git a/src/spyglass/spikesorting/v1/recording.py b/src/spyglass/spikesorting/v1/recording.py index f4e150837..d1257d4f3 100644 --- a/src/spyglass/spikesorting/v1/recording.py +++ b/src/spyglass/spikesorting/v1/recording.py @@ -154,7 +154,7 @@ def insert_selection(cls, key: dict): """ query = cls & key if query: - logger.warn("Similar row(s) already inserted.") + logger.warning("Similar row(s) already inserted.") return query.fetch(as_dict=True) key["recording_id"] = uuid.uuid4() cls.insert1(key, skip_duplicates=True) diff --git a/src/spyglass/utils/database_settings.py b/src/spyglass/utils/database_settings.py index 56e2aa28f..0f08fbfb4 100755 --- a/src/spyglass/utils/database_settings.py +++ b/src/spyglass/utils/database_settings.py @@ -20,6 +20,7 @@ "lfp", "waveform", "mua", + "sharing", ] GRANT_ALL = "GRANT ALL PRIVILEGES ON " GRANT_SEL = "GRANT SELECT ON " diff --git a/src/spyglass/utils/dj_helper_fn.py b/src/spyglass/utils/dj_helper_fn.py index d77416487..42cf67ba0 100644 --- a/src/spyglass/utils/dj_helper_fn.py +++ b/src/spyglass/utils/dj_helper_fn.py @@ -124,7 +124,7 @@ def _subclass_factory( # Define the __call__ method for the new class def init_override(self, *args, **kwargs): - logger.warn( + logger.warning( "Deprecation: this class has been moved out of " + f"{old_module}\n" + f"\t{old_name} -> {new_module}.{new_class.__name__}" diff --git a/src/spyglass/utils/dj_merge_tables.py b/src/spyglass/utils/dj_merge_tables.py index 7d6bf46ba..146b9873a 100644 --- a/src/spyglass/utils/dj_merge_tables.py +++ b/src/spyglass/utils/dj_merge_tables.py @@ -66,14 +66,14 @@ def __init__(self): self._reserved_sk = RESERVED_SECONDARY_KEY if not self.is_declared: if not is_merge_table(self): # Check definition - logger.warn( + logger.warning( "Merge table with non-default definition\n" + f"Expected:\n{MERGE_DEFINITION.strip()}\n" + f"Actual :\n{self.definition.strip()}" ) for part in self.parts(as_objects=True): if part.primary_key != self.primary_key: - logger.warn( # PK is only 'merge_id' in parts, no others + logger.warning( # PK is only 'merge_id' in parts, no others f"Unexpected primary key in {part.table_name}" + f"\n\tExpected: {self.primary_key}" + f"\n\tActual : {part.primary_key}" @@ -721,7 +721,7 @@ def _normalize_source( raise ValueError(f"Unable to find source for {source}") source = fetched_source[0] if len(fetched_source) > 1: - logger.warn(f"Multiple sources. Selecting first: {source}.") + logger.warning(f"Multiple sources. Selecting first: {source}.") if isinstance(source, dj.Table): source = self._part_name(source) if isinstance(source, dict): @@ -814,7 +814,7 @@ def merge_fetch( try: results.extend(part.fetch(*attrs, **kwargs)) except DataJointError as e: - logger.warn( + logger.warning( f"{e.args[0]} Skipping " + to_camel_case(part.table_name.split("__")[-1]) ) diff --git a/src/spyglass/utils/dj_mixin.py b/src/spyglass/utils/dj_mixin.py index 5c0b7dcf3..72e34c04f 100644 --- a/src/spyglass/utils/dj_mixin.py +++ b/src/spyglass/utils/dj_mixin.py @@ -368,7 +368,7 @@ def _check_delete_permission(self) -> None: not self._session_connection # Table has no session or self._member_pk in self.heading.names # Table has experimenter ): - logger.warn( # Permit delete if no session connection + logger.warning( # Permit delete if no session connection "Could not find lab team associated with " + f"{self.__class__.__name__}." + "\nBe careful not to delete others' data." @@ -376,7 +376,7 @@ def _check_delete_permission(self) -> None: return if not (sess_summary := self._get_exp_summary()): - logger.warn( + logger.warning( f"Could not find a connection from {self.camel_name} " + "to Session.\n Be careful not to delete others' data." ) diff --git a/src/spyglass/utils/nwb_helper_fn.py b/src/spyglass/utils/nwb_helper_fn.py index 5d5fdaca4..82af8a626 100644 --- a/src/spyglass/utils/nwb_helper_fn.py +++ b/src/spyglass/utils/nwb_helper_fn.py @@ -364,7 +364,7 @@ def get_valid_intervals( if total_time < min_valid_len: half_total_time = total_time / 2 - logger.warn(f"Setting minimum valid interval to {half_total_time}") + logger.warning(f"Setting minimum valid interval to {half_total_time}") min_valid_len = half_total_time # get rid of NaN elements diff --git a/tests/README.md b/tests/README.md index 36b6ab71f..d1873505c 100644 --- a/tests/README.md +++ b/tests/README.md @@ -2,12 +2,6 @@ ## Environment -To allow pytest helpers to automatically dowlnoad requisite data, you'll need to -set credentials for Box. Consider adding these to a private `.env` file. - -- `UCSF_BOX_USER`: UCSF email address -- `UCSF_BOX_TOKEN`: Token generated from UCSF Box account - To facilitate headless testing of various Qt-based tools as well as Tensorflow, `pyproject.toml` includes some environment variables associated with the display. These are... diff --git a/tests/conftest.py b/tests/conftest.py index de513d3a9..a7354a383 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,6 +28,7 @@ warnings.filterwarnings("ignore", category=FutureWarning, module="sklearn") warnings.filterwarnings("ignore", category=PerformanceWarning, module="pandas") warnings.filterwarnings("ignore", category=NumbaWarning, module="numba") +warnings.filterwarnings("ignore", category=ResourceWarning, module="datajoint") # ------------------------------- TESTS CONFIG ------------------------------- @@ -108,7 +109,6 @@ def pytest_configure(config): ) DOWNLOADS = DataDownloader( - nwb_file_name=TEST_FILE, base_dir=BASE_DIR, verbose=VERBOSE, download_dlc=not NO_DLC, @@ -420,7 +420,6 @@ def video_keys(common, base_dir): for file in DOWNLOADS.file_downloads: if file.endswith(".h264"): DOWNLOADS.wait_for(file) - DOWNLOADS.rename_files() return common.VideoFile().fetch(as_dict=True) diff --git a/tests/data_downloader.py b/tests/data_downloader.py index cb58e1c71..40af1ea88 100644 --- a/tests/data_downloader.py +++ b/tests/data_downloader.py @@ -1,5 +1,4 @@ from functools import cached_property -from os import environ as os_environ from pathlib import Path from shutil import copy as shutil_copy from subprocess import DEVNULL, Popen @@ -9,46 +8,44 @@ from datajoint import logger as dj_logger -UCSF_BOX_USER = os_environ.get("UCSF_BOX_USER") -UCSF_BOX_TOKEN = os_environ.get("UCSF_BOX_TOKEN") -BASE_URL = "ftps://ftp.box.com/trodes_to_nwb_test_data/" +BASE_URL = "https://ucsf.box.com/shared/static/" NON_DLC = 3 # First N items below are not for DeepLabCut FILE_PATHS = [ { "relative_dir": "raw", "target_name": "minirec20230622.nwb", - "url": BASE_URL + "minirec20230622.nwb", + "url": BASE_URL + "k3sgql6z475oia848q1rgms4zdh4rkjn.nwb", }, { "relative_dir": "video", "target_name": "20230622_minirec_01_s1.1.h264", - "url": BASE_URL + "20230622_sample_01_a1/20230622_sample_01_a1.1.h264", + "url": BASE_URL + "ykep8ek4ogad20wz4p0vuyuqfo60cv3w.h264", }, { "relative_dir": "video", "target_name": "20230622_minirec_02_s2.1.h264", - "url": BASE_URL + "20230622_sample_02_a1/20230622_sample_02_a1.1.h264", + "url": BASE_URL + "d2jjk0y565ru75xqojio3hymmehzr5he.h264", }, { "relative_dir": "deeplabcut", "target_name": "CollectedData_sc_eb.csv", - "url": BASE_URL + "minirec_dlc_items/CollectedData_sc_eb.csv", + "url": BASE_URL + "3nzqdfty51vrga7470rn2vayrtoor3ot.csv", }, { "relative_dir": "deeplabcut", "target_name": "CollectedData_sc_eb.h5", - "url": BASE_URL + "minirec_dlc_items/CollectedData_sc_eb.h5", + "url": BASE_URL + "sx30rqljppeisi4jdyu53y51na0q9rff.h5", }, { "relative_dir": "deeplabcut", "target_name": "img000.png", - "url": BASE_URL + "minirec_dlc_items/img000.png", + "url": BASE_URL + "wrvgncfbpjuzfhopkfaizzs069tb1ruu.png", }, { "relative_dir": "deeplabcut", "target_name": "img001.png", - "url": BASE_URL + "minirec_dlc_items/img001.png", + "url": BASE_URL + "czbkxeinemat7jj7j0877pcosfqo9psh.png", }, ] @@ -56,41 +53,18 @@ class DataDownloader: def __init__( self, - nwb_file_name, file_paths=FILE_PATHS, base_dir=".", download_dlc=True, verbose=True, ): - if not all([UCSF_BOX_USER, UCSF_BOX_TOKEN]): - raise ValueError( - "Missing os.environ credentials: UCSF_BOX_USER, UCSF_BOX_TOKEN." - ) - if nwb_file_name != file_paths[0]["target_name"]: - raise ValueError( - f"Please adjust data_downloader.py to match: {nwb_file_name}" - ) - - self.cmd = [ - "wget", - "--recursive", - "--no-host-directories", - "--no-directories", - "--user", - UCSF_BOX_USER, - "--password", - UCSF_BOX_TOKEN, - "-P", # Then need relative path, then url - ] - - self.verbose = verbose - if not verbose: - self.cmd.insert(self.cmd.index("--recursive") + 1, "--no-verbose") - self.cmd_kwargs = dict(stdout=DEVNULL, stderr=DEVNULL) - else: + if verbose: self.cmd_kwargs = dict(stdout=stdout, stderr=stderr) + else: + self.cmd_kwargs = dict(stdout=DEVNULL, stderr=DEVNULL) - self.base_dir = Path(base_dir).resolve() + self.verbose = verbose + self.base_dir = Path(base_dir).expanduser().resolve() self.download_dlc = download_dlc self.file_paths = file_paths if download_dlc else file_paths[:NON_DLC] self.base_dir.mkdir(exist_ok=True) @@ -98,46 +72,41 @@ def __init__( # Start downloads _ = self.file_downloads - def rename_files(self): - """Redundant, but allows rerun later in startup process of conftest.""" - for path in self.file_paths: - target, url = path["target_name"], path["url"] - target_dir = self.base_dir / path["relative_dir"] - orig = target_dir / url.split("/")[-1] - dest = target_dir / target - - if orig.exists(): - orig.rename(dest) - @cached_property # Only make list of processes once def file_downloads(self) -> Dict[str, Union[Popen, None]]: """{File: POpen/None} for each file. If exists/finished, None.""" ret = dict() - self.rename_files() for path in self.file_paths: - target, url = path["target_name"], path["url"] target_dir = self.base_dir / path["relative_dir"] target_dir.mkdir(exist_ok=True, parents=True) + + target = path["target_name"] dest = target_dir / target - cmd = ( - ["echo", f"Already have {target}"] - if dest.exists() - else self.cmd + [target_dir, url] - ) + + if dest.exists(): + cmd = ["echo", f"Already have {target}"] + else: + cmd = ["curl", "-L", "--output", str(dest), f"{path['url']}"] + + print(f"cmd: {cmd}") + ret[target] = Popen(cmd, **self.cmd_kwargs) + return ret def wait_for(self, target: str): """Wait for target to finish downloading.""" status = self.file_downloads.get(target).poll() + limit = 10 while status is None and limit > 0: - time_sleep(5) # Some + time_sleep(5) limit -= 1 status = self.file_downloads.get(target).poll() - if status != 0: + + if status != 0: # Error downloading raise ValueError(f"Error downloading: {target}") - if limit < 1: + if limit < 1: # Reached attempt limit raise TimeoutError(f"Timeout downloading: {target}") def move_dlc_items(self, dest_dir: Path): From 4231e51838680c7cd00204c58b820fbdb4cd6faf Mon Sep 17 00:00:00 2001 From: Samuel Bray Date: Thu, 14 Nov 2024 13:25:38 -0800 Subject: [PATCH 87/94] No transactions v0 spikesorting pipeline (#1187) * no transactions v0 spikesorting * update changelog * no transact on v0 recording --- CHANGELOG.md | 2 +- src/spyglass/spikesorting/v0/spikesorting_curation.py | 4 ++++ src/spyglass/spikesorting/v0/spikesorting_recording.py | 2 ++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd3bbeeb3..86058203b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() ### Infrastructure - Disable populate transaction protection for long-populating tables #1066, - #1108, #1172 + #1108, #1172, #1187 - Add docstrings to all public methods #1076 - Update DataJoint to 0.14.2 #1081 - Allow restriction based on parent keys in `Merge.fetch_nwb()` #1086, #1126 diff --git a/src/spyglass/spikesorting/v0/spikesorting_curation.py b/src/spyglass/spikesorting/v0/spikesorting_curation.py index 27aa074d3..77a4e8edb 100644 --- a/src/spyglass/spikesorting/v0/spikesorting_curation.py +++ b/src/spyglass/spikesorting/v0/spikesorting_curation.py @@ -324,6 +324,8 @@ class WaveformSelection(SpyglassMixin, dj.Manual): @schema class Waveforms(SpyglassMixin, dj.Computed): + use_transaction, _allow_insert = False, True + definition = """ -> WaveformSelection --- @@ -523,6 +525,8 @@ def insert1(self, key, **kwargs): @schema class QualityMetrics(SpyglassMixin, dj.Computed): + use_transaction, _allow_insert = False, True + definition = """ -> MetricSelection --- diff --git a/src/spyglass/spikesorting/v0/spikesorting_recording.py b/src/spyglass/spikesorting/v0/spikesorting_recording.py index a6a356a33..16fbf3544 100644 --- a/src/spyglass/spikesorting/v0/spikesorting_recording.py +++ b/src/spyglass/spikesorting/v0/spikesorting_recording.py @@ -291,6 +291,8 @@ class SpikeSortingRecordingSelection(SpyglassMixin, dj.Manual): @schema class SpikeSortingRecording(SpyglassMixin, dj.Computed): + use_transaction, _allow_insert = False, True + definition = """ -> SpikeSortingRecordingSelection --- From 37ddfc1ba163a740e2c44a710611f41c9fa7241f Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Tue, 26 Nov 2024 17:43:27 -0600 Subject: [PATCH 88/94] Add pytests for decoding pipeline (#1155) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * WIP: Add decoding pytests 1 * WIP: add decoding tests 2 * WIP: coverage for v1 schemas * WIP: fixing impacted tests elsewhere * ✅ : fix impacted tests * Revert merge edits --- CHANGELOG.md | 1 + pyproject.toml | 3 +- src/spyglass/decoding/decoding_merge.py | 48 +- src/spyglass/decoding/v1/clusterless.py | 10 +- src/spyglass/decoding/v1/core.py | 18 +- .../analysis/v1/unit_annotation.py | 2 +- src/spyglass/utils/dj_merge_tables.py | 2 + tests/common/test_interval.py | 7 +- tests/conftest.py | 173 +++++++ tests/decoding/__init__.py | 0 tests/decoding/conftest.py | 474 ++++++++++++++++++ tests/decoding/test_clusterless.py | 76 +++ tests/decoding/test_core.py | 22 + tests/decoding/test_merge.py | 82 +++ tests/decoding/test_spikes.py | 61 +++ tests/decoding/test_wave.py | 21 + tests/spikesorting/conftest.py | 183 +------ tests/spikesorting/test_analysis.py | 4 +- tests/spikesorting/test_artifact.py | 6 +- tests/spikesorting/test_curation.py | 7 +- tests/spikesorting/test_merge.py | 36 +- tests/spikesorting/test_utils.py | 11 +- 22 files changed, 988 insertions(+), 259 deletions(-) create mode 100644 tests/decoding/__init__.py create mode 100644 tests/decoding/conftest.py create mode 100644 tests/decoding/test_clusterless.py create mode 100644 tests/decoding/test_core.py create mode 100644 tests/decoding/test_merge.py create mode 100644 tests/decoding/test_spikes.py create mode 100644 tests/decoding/test_wave.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 86058203b..23b6e84cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Add testing for python versions 3.9, 3.10, 3.11, 3.12 #1169 - Initialize tables in pytests #1181 - Download test data without credentials, trigger on approved PRs #1180 + - Add coverage of decoding pipeline to pytests #1155 - Allow python \< 3.13 #1169 - Remove numpy version restriction #1169 - Merge table delete removes orphaned master entries #1164 diff --git a/pyproject.toml b/pyproject.toml index 39a932919..a5d8d032d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -157,7 +157,8 @@ omit = [ # which submodules have no tests "*/cli/*", # "*/common/*", "*/data_import/*", - "*/decoding/*", + "*/decoding/v0/*", + # "*/decoding/*", "*/figurl_views/*", # "*/lfp/*", # "*/linearization/*", diff --git a/src/spyglass/decoding/decoding_merge.py b/src/spyglass/decoding/decoding_merge.py index 5ba8a0ea4..e8b2e334b 100644 --- a/src/spyglass/decoding/decoding_merge.py +++ b/src/spyglass/decoding/decoding_merge.py @@ -85,53 +85,41 @@ def cleanup(self, dry_run=False): @classmethod def fetch_results(cls, key): """Fetch the decoding results for a given key.""" - return cls().merge_get_parent_class(key).fetch_results() + return cls().merge_restrict_class(key).fetch_results() @classmethod def fetch_model(cls, key): """Fetch the decoding model for a given key.""" - return cls().merge_get_parent_class(key).fetch_model() + return cls().merge_restrict_class(key).fetch_model() @classmethod def fetch_environments(cls, key): """Fetch the decoding environments for a given key.""" - decoding_selection_key = cls.merge_get_parent(key).fetch1("KEY") - return ( - cls() - .merge_get_parent_class(key) - .fetch_environments(decoding_selection_key) - ) + restr_parent = cls().merge_restrict_class(key) + decoding_selection_key = restr_parent.fetch1("KEY") + return restr_parent.fetch_environments(decoding_selection_key) @classmethod def fetch_position_info(cls, key): """Fetch the decoding position info for a given key.""" - decoding_selection_key = cls.merge_get_parent(key).fetch1("KEY") - return ( - cls() - .merge_get_parent_class(key) - .fetch_position_info(decoding_selection_key) - ) + restr_parent = cls().merge_restrict_class(key) + decoding_selection_key = restr_parent.fetch1("KEY") + return restr_parent.fetch_position_info(decoding_selection_key) @classmethod def fetch_linear_position_info(cls, key): """Fetch the decoding linear position info for a given key.""" - decoding_selection_key = cls.merge_get_parent(key).fetch1("KEY") - return ( - cls() - .merge_get_parent_class(key) - .fetch_linear_position_info(decoding_selection_key) - ) + restr_parent = cls().merge_restrict_class(key) + decoding_selection_key = restr_parent.fetch1("KEY") + return restr_parent.fetch_linear_position_info(decoding_selection_key) @classmethod def fetch_spike_data(cls, key, filter_by_interval=True): """Fetch the decoding spike data for a given key.""" - decoding_selection_key = cls.merge_get_parent(key).fetch1("KEY") - return ( - cls() - .merge_get_parent_class(key) - .fetch_linear_position_info( - decoding_selection_key, filter_by_interval=filter_by_interval - ) + restr_parent = cls().merge_restrict_class(key) + decoding_selection_key = restr_parent.fetch1("KEY") + return restr_parent.fetch_spike_data( + decoding_selection_key, filter_by_interval=filter_by_interval ) @classmethod @@ -167,11 +155,7 @@ def create_decoding_view(cls, key, head_direction_name="head_orientation"): head_dir=position_info[head_direction_name], ) else: - ( - position_info, - position_variable_names, - ) = cls.fetch_linear_position_info(key) return create_1D_decode_view( posterior=posterior, - linear_position=position_info["linear_position"], + linear_position=cls.fetch_linear_position_info(key), ) diff --git a/src/spyglass/decoding/v1/clusterless.py b/src/spyglass/decoding/v1/clusterless.py index 7e0711ad9..fbfba2183 100644 --- a/src/spyglass/decoding/v1/clusterless.py +++ b/src/spyglass/decoding/v1/clusterless.py @@ -59,10 +59,11 @@ def create_group( "waveform_features_group_name": group_name, } if self & group_key: - raise ValueError( - f"Group {nwb_file_name}: {group_name} already exists", - "please delete the group before creating a new one", + logger.error( # No error on duplicate helps with pytests + f"Group {nwb_file_name}: {group_name} already exists" + + "please delete the group before creating a new one", ) + return self.insert1( group_key, skip_duplicates=True, @@ -586,7 +587,8 @@ def get_ahead_behind_distance(self, track_graph=None, time_slice=None): classifier.environments[0].track_graph, *traj_data ) else: - position_info = self.fetch_position_info(self.fetch1("KEY")).loc[ + # `fetch_position_info` returns a tuple + position_info = self.fetch_position_info(self.fetch1("KEY"))[0].loc[ time_slice ] map_position = analysis.maximum_a_posteriori_estimate(posterior) diff --git a/src/spyglass/decoding/v1/core.py b/src/spyglass/decoding/v1/core.py index 087d958a3..d58af1643 100644 --- a/src/spyglass/decoding/v1/core.py +++ b/src/spyglass/decoding/v1/core.py @@ -15,7 +15,7 @@ restore_classes, ) from spyglass.position.position_merge import PositionOutput # noqa: F401 -from spyglass.utils import SpyglassMixin, SpyglassMixinPart +from spyglass.utils import SpyglassMixin, SpyglassMixinPart, logger schema = dj.schema("decoding_core_v1") @@ -56,14 +56,15 @@ class DecodingParameters(SpyglassMixin, dj.Lookup): @classmethod def insert_default(cls): """Insert default decoding parameters""" - cls.insert(cls.contents, skip_duplicates=True) + cls.super().insert(cls.contents, skip_duplicates=True) def insert(self, rows, *args, **kwargs): """Override insert to convert classes to dict before inserting""" for row in rows: - row["decoding_params"] = convert_classes_to_dict( - vars(row["decoding_params"]) - ) + params = row["decoding_params"] + if hasattr(params, "__dict__"): + params = vars(params) + row["decoding_params"] = convert_classes_to_dict(params) super().insert(rows, *args, **kwargs) def fetch(self, *args, **kwargs): @@ -124,10 +125,11 @@ def create_group( "position_group_name": group_name, } if self & group_key: - raise ValueError( - f"Group {nwb_file_name}: {group_name} already exists", - "please delete the group before creating a new one", + logger.error( # Easier for pytests to not raise error on duplicate + f"Group {nwb_file_name}: {group_name} already exists" + + "please delete the group before creating a new one" ) + return self.insert1( { **group_key, diff --git a/src/spyglass/spikesorting/analysis/v1/unit_annotation.py b/src/spyglass/spikesorting/analysis/v1/unit_annotation.py index d1ac26a11..5e0da16e7 100644 --- a/src/spyglass/spikesorting/analysis/v1/unit_annotation.py +++ b/src/spyglass/spikesorting/analysis/v1/unit_annotation.py @@ -71,7 +71,7 @@ def add_annotation(self, key, **kwargs): ).fetch_nwb()[0] nwb_field_name = _get_spike_obj_name(nwb_file) spikes = nwb_file[nwb_field_name]["spike_times"].to_list() - if key["unit_id"] > len(spikes): + if key["unit_id"] > len(spikes) and not self._test_mode: raise ValueError( f"unit_id {key['unit_id']} is greater than ", f"the number of units in {key['spikesorting_merge_id']}", diff --git a/src/spyglass/utils/dj_merge_tables.py b/src/spyglass/utils/dj_merge_tables.py index 146b9873a..e5e30c848 100644 --- a/src/spyglass/utils/dj_merge_tables.py +++ b/src/spyglass/utils/dj_merge_tables.py @@ -737,6 +737,8 @@ def merge_get_parent_class(self, source: str) -> dj.Table: source: Union[str, dict, dj.Table] Accepts a CamelCase name of the source, or key as a dict, or a part table. + init: bool, optional + Default False. If True, returns an instance of the class. Returns ------- diff --git a/tests/common/test_interval.py b/tests/common/test_interval.py index e720b4466..e822f07c4 100644 --- a/tests/common/test_interval.py +++ b/tests/common/test_interval.py @@ -8,7 +8,9 @@ def interval_list(common): def test_plot_intervals(mini_insert, interval_list): - fig = interval_list.plot_intervals(return_fig=True) + fig = (interval_list & 'interval_list_name LIKE "raw%"').plot_intervals( + return_fig=True + ) interval_list_name = fig.get_axes()[0].get_yticklabels()[0].get_text() times_fetch = ( interval_list & {"interval_list_name": interval_list_name} @@ -19,7 +21,8 @@ def test_plot_intervals(mini_insert, interval_list): def test_plot_epoch(mini_insert, interval_list): - fig = interval_list.plot_epoch_pos_raw_intervals(return_fig=True) + restr_interval = interval_list & "interval_list_name like 'raw%'" + fig = restr_interval.plot_epoch_pos_raw_intervals(return_fig=True) epoch_label = fig.get_axes()[0].get_yticklabels()[-1].get_text() assert epoch_label == "epoch", "plot_epoch failed" diff --git a/tests/conftest.py b/tests/conftest.py index a7354a383..22ffa5d2b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1299,3 +1299,176 @@ def dlc_key(sgp, dlc_selection): def populate_dlc(sgp, dlc_key): sgp.v1.DLCPosV1().populate(dlc_key) yield + + +# ----------------------- FIXTURES, SPIKESORTING TABLES ----------------------- +# ------------------------ Note: Used in decoding tests ------------------------ + + +@pytest.fixture(scope="session") +def spike_v1(common): + from spyglass.spikesorting import v1 + + yield v1 + + +@pytest.fixture(scope="session") +def pop_rec(spike_v1, mini_dict, team_name): + spike_v1.SortGroup.set_group_by_shank(**mini_dict) + key = { + **mini_dict, + "sort_group_id": 0, + "preproc_param_name": "default", + "interval_list_name": "01_s1", + "team_name": team_name, + } + spike_v1.SpikeSortingRecordingSelection.insert_selection(key) + ssr_pk = ( + (spike_v1.SpikeSortingRecordingSelection & key).proj().fetch1("KEY") + ) + spike_v1.SpikeSortingRecording.populate(ssr_pk) + + yield ssr_pk + + +@pytest.fixture(scope="session") +def pop_art(spike_v1, mini_dict, pop_rec): + key = { + "recording_id": pop_rec["recording_id"], + "artifact_param_name": "default", + } + spike_v1.ArtifactDetectionSelection.insert_selection(key) + spike_v1.ArtifactDetection.populate() + + yield spike_v1.ArtifactDetection().fetch("KEY", as_dict=True)[0] + + +@pytest.fixture(scope="session") +def spike_merge(spike_v1): + from spyglass.spikesorting.spikesorting_merge import SpikeSortingOutput + + yield SpikeSortingOutput() + + +@pytest.fixture(scope="session") +def sorter_dict(): + return {"sorter": "mountainsort4"} + + +@pytest.fixture(scope="session") +def pop_sort(spike_v1, pop_rec, pop_art, mini_dict, sorter_dict): + key = { + **mini_dict, + **sorter_dict, + "recording_id": pop_rec["recording_id"], + "interval_list_name": str(pop_art["artifact_id"]), + "sorter_param_name": "franklab_tetrode_hippocampus_30KHz", + } + spike_v1.SpikeSortingSelection.insert_selection(key) + spike_v1.SpikeSorting.populate() + + yield spike_v1.SpikeSorting().fetch("KEY", as_dict=True)[0] + + +@pytest.fixture(scope="session") +def sorting_objs(spike_v1, pop_sort): + sort_nwb = (spike_v1.SpikeSorting & pop_sort).fetch_nwb() + sort_si = spike_v1.SpikeSorting.get_sorting(pop_sort) + yield sort_nwb, sort_si + + +@pytest.fixture(scope="session") +def pop_curation(spike_v1, pop_sort): + spike_v1.CurationV1.insert_curation( + sorting_id=pop_sort["sorting_id"], + description="testing sort", + ) + + yield (spike_v1.CurationV1() & {"parent_curation_id": -1}).fetch( + "KEY", as_dict=True + )[0] + + +@pytest.fixture(scope="session") +def pop_metric(spike_v1, pop_sort, pop_curation): + _ = pop_curation # make sure this happens first + key = { + "sorting_id": pop_sort["sorting_id"], + "curation_id": 0, + "waveform_param_name": "default_not_whitened", + "metric_param_name": "franklab_default", + "metric_curation_param_name": "default", + } + + spike_v1.MetricCurationSelection.insert_selection(key) + spike_v1.MetricCuration.populate(key) + + yield spike_v1.MetricCuration().fetch("KEY", as_dict=True)[0] + + +@pytest.fixture(scope="session") +def metric_objs(spike_v1, pop_metric): + key = {"metric_curation_id": pop_metric["metric_curation_id"]} + labels = spike_v1.MetricCuration.get_labels(key) + merge_groups = spike_v1.MetricCuration.get_merge_groups(key) + metrics = spike_v1.MetricCuration.get_metrics(key) + yield labels, merge_groups, metrics + + +@pytest.fixture(scope="session") +def pop_curation_metric(spike_v1, pop_metric, metric_objs): + labels, merge_groups, metrics = metric_objs + parent_dict = {"parent_curation_id": 0} + spike_v1.CurationV1.insert_curation( + sorting_id=( + spike_v1.MetricCurationSelection + & {"metric_curation_id": pop_metric["metric_curation_id"]} + ).fetch1("sorting_id"), + **parent_dict, + labels=labels, + merge_groups=merge_groups, + metrics=metrics, + description="after metric curation", + ) + + yield (spike_v1.CurationV1 & parent_dict).fetch("KEY", as_dict=True)[0] + + +@pytest.fixture(scope="session") +def pop_spike_merge( + spike_v1, pop_curation_metric, spike_merge, mini_dict, sorter_dict +): + # TODO: add figurl fixtures when kachery_cloud is initialized + + spike_merge.insert([pop_curation_metric], part_name="CurationV1") + + yield (spike_merge << pop_curation_metric).fetch1("KEY") + + +@pytest.fixture(scope="session") +def spike_v1_group(): + from spyglass.spikesorting.analysis.v1 import group + + yield group + + +@pytest.fixture(scope="session") +def group_name(): + yield "test_group" + + +@pytest.fixture(scope="session") +def pop_spikes_group( + group_name, spike_v1_group, spike_merge, mini_dict, pop_spike_merge +): + + _ = pop_spike_merge # make sure this happens first + + spike_v1_group.UnitSelectionParams().insert_default() + spike_v1_group.SortedSpikesGroup().create_group( + **mini_dict, + group_name=group_name, + keys=spike_merge.proj(spikesorting_merge_id="merge_id").fetch("KEY"), + unit_filter_params_name="default_exclusion", + ) + yield spike_v1_group.SortedSpikesGroup().fetch("KEY", as_dict=True)[0] diff --git a/tests/decoding/__init__.py b/tests/decoding/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/decoding/conftest.py b/tests/decoding/conftest.py new file mode 100644 index 000000000..5e714f680 --- /dev/null +++ b/tests/decoding/conftest.py @@ -0,0 +1,474 @@ +import numpy as np +import pytest + + +@pytest.fixture(scope="session") +def result_coordinates(): + return { + "encoding_groups", + "states", + "state", + "state_bins", + "state_ind", + "time", + "environments", + } + + +@pytest.fixture(scope="session") +def decode_v1(common, trodes_pos_v1): + from spyglass.decoding import v1 + + yield v1 + + +@pytest.fixture(scope="session") +def recording_ids(spike_v1, mini_dict, pop_rec, pop_art): + _ = pop_rec # set group by shank + + recording_ids = (spike_v1.SpikeSortingRecordingSelection & mini_dict).fetch( + "recording_id" + ) + group_keys = [] + for recording_id in recording_ids: + key = { + "recording_id": recording_id, + "artifact_param_name": "none", + } + group_keys.append(key) + spike_v1.ArtifactDetectionSelection.insert_selection(key) + spike_v1.ArtifactDetection.populate(group_keys) + + yield recording_ids + + +@pytest.fixture(scope="session") +def clusterless_params_insert(spike_v1): + """Low threshold for testing, otherwise no spikes with default.""" + clusterless_params = { + "sorter": "clusterless_thresholder", + "sorter_param_name": "low_thresh", + } + spike_v1.SpikeSorterParameters.insert1( + { + **clusterless_params, + "sorter_params": { + "detect_threshold": 10.0, # was 100 + # Locally exclusive means one unit per spike detected + "method": "locally_exclusive", + "peak_sign": "neg", + "exclude_sweep_ms": 0.1, + "local_radius_um": 1000, # was 100 + # noise levels needs to be 1.0 so the units are in uV and not MAD + "noise_levels": np.asarray([1.0]), + "random_chunk_kwargs": {}, + # output needs to be set to sorting for the rest of the pipeline + "outputs": "sorting", + }, + }, + skip_duplicates=True, + ) + yield clusterless_params + + +@pytest.fixture(scope="session") +def clusterless_spikesort( + spike_v1, recording_ids, mini_dict, clusterless_params_insert +): + group_keys = [] + for recording_id in recording_ids: + key = { + **clusterless_params_insert, + **mini_dict, + "recording_id": recording_id, + "interval_list_name": str( + ( + spike_v1.ArtifactDetectionSelection + & { + "recording_id": recording_id, + "artifact_param_name": "none", + } + ).fetch1("artifact_id") + ), + } + group_keys.append(key) + spike_v1.SpikeSortingSelection.insert_selection(key) + spike_v1.SpikeSorting.populate() + yield clusterless_params_insert + + +@pytest.fixture(scope="session") +def clusterless_params(clusterless_spikesort): + yield clusterless_spikesort + + +@pytest.fixture(scope="session") +def clusterless_curate(spike_v1, clusterless_params, spike_merge): + + sorting_ids = (spike_v1.SpikeSortingSelection & clusterless_params).fetch( + "sorting_id" + ) + + fails = [] + for sorting_id in sorting_ids: + try: + spike_v1.CurationV1.insert_curation(sorting_id=sorting_id) + except KeyError: + fails.append(sorting_id) + + if len(fails) == len(sorting_ids): + (spike_v1.SpikeSorterParameters & clusterless_params).delete( + safemode=False + ) + raise ValueError("All curation insertions failed.") + + spike_merge.insert( + spike_v1.CurationV1().fetch("KEY"), + part_name="CurationV1", + skip_duplicates=True, + ) + yield + + +@pytest.fixture(scope="session") +def waveform_params_tbl(decode_v1): + params_tbl = decode_v1.waveform_features.WaveformFeaturesParams + params_tbl.insert_default() + yield params_tbl + + +@pytest.fixture(scope="session") +def waveform_params(waveform_params_tbl): + param_pk = {"features_param_name": "low_thresh_amplitude"} + waveform_params_tbl.insert1( + { + **param_pk, + "params": { + "waveform_extraction_params": { + "ms_before": 0.2, # previously 0.5 + "ms_after": 0.2, # previously 0.5 + "max_spikes_per_unit": None, + "n_jobs": 1, # previously 5 + "total_memory": "1G", # previously "5G" + }, + "waveform_features_params": { + "amplitude": { + "peak_sign": "neg", + "estimate_peak_time": False, # was False + } + }, + }, + }, + skip_duplicates=True, + ) + yield param_pk + + +@pytest.fixture(scope="session") +def clusterless_mergeids( + spike_merge, mini_dict, clusterless_curate, clusterless_params +): + _ = clusterless_curate # ensure populated + yield spike_merge.get_restricted_merge_ids( + { + **mini_dict, + **clusterless_params, + }, + sources=["v1"], + ) + + +@pytest.fixture(scope="session") +def pop_unitwave(decode_v1, waveform_params, clusterless_mergeids): + sel_keys = [ + { + "spikesorting_merge_id": merge_id, + **waveform_params, + } + for merge_id in clusterless_mergeids + ] + + wave = decode_v1.waveform_features + wave.UnitWaveformFeaturesSelection.insert(sel_keys, skip_duplicates=True) + wave.UnitWaveformFeatures.populate(sel_keys) + + yield wave.UnitWaveformFeatures & sel_keys + + +@pytest.fixture(scope="session") +def group_unitwave( + decode_v1, + mini_dict, + clusterless_mergeids, + pop_unitwave, + waveform_params, + group_name, +): + wave = decode_v1.waveform_features + waveform_selection_keys = ( + wave.UnitWaveformFeaturesSelection() & waveform_params + ).fetch("KEY", as_dict=True) + decode_v1.clusterless.UnitWaveformFeaturesGroup().create_group( + **mini_dict, + group_name="test_group", + keys=waveform_selection_keys, + ) + yield decode_v1.clusterless.UnitWaveformFeaturesGroup & { + "waveform_features_group_name": group_name, + } + + +@pytest.fixture(scope="session") +def pos_merge_keys(pos_merge): + return ( + ( + pos_merge.TrodesPosV1 + & 'trodes_pos_params_name = "single_led_upsampled"' + ) + .proj(pos_merge_id="merge_id") + .fetch(as_dict=True) + ) + + +@pytest.fixture(scope="session") +def pop_pos_group(decode_v1, pos_merge_keys, group_name, mini_dict): + + decode_v1.core.PositionGroup().create_group( + **mini_dict, + group_name=group_name, + keys=pos_merge_keys, + ) + + yield decode_v1.core.PositionGroup & { + **mini_dict, + "position_group_name": group_name, + } + + +@pytest.fixture(scope="session") +def pop_pos_group_upsampled(decode_v1, pos_merge_keys, group_name, mini_dict): + name = group_name + "_upsampled" + decode_v1.core.PositionGroup().create_group( + **mini_dict, + group_name=name, + keys=pos_merge_keys, + upsample_rate=250, + ) + + yield decode_v1.core.PositionGroup & { + **mini_dict, + "position_group_name": name, + } + + +@pytest.fixture(scope="session") +def decode_clusterless_params_insert(decode_v1, track_graph): + from non_local_detector.environment import Environment + from non_local_detector.models import ContFragClusterlessClassifier + + graph_entry = track_graph.fetch1() # Restricted table + class_kwargs = dict( + clusterless_algorithm_params={ + "block_size": 10000, + "position_std": 12.0, + "waveform_std": 24.0, + }, + environments=[ + Environment( + # environment_name=graph_entry["track_graph_name"], + track_graph=track_graph.get_networkx_track_graph(), + edge_order=graph_entry["linear_edge_order"], + edge_spacing=graph_entry["linear_edge_spacing"], + ) + ], + ) + params_pk = {"decoding_param_name": "contfrag_clusterless"} + # decode_v1.core.DecodingParameters.insert_default() + decode_v1.core.DecodingParameters.insert1( + { + **params_pk, + "decoding_params": ContFragClusterlessClassifier(**class_kwargs), + "decoding_kwargs": dict(), + }, + skip_duplicates=True, + ) + model_params = (decode_v1.core.DecodingParameters & params_pk).fetch1() + ContFragClusterlessClassifier(**model_params["decoding_params"]) + + yield params_pk + + +@pytest.fixture(scope="session") +def decode_interval(common, mini_dict): + decode_interval_name = "decode" + raw_begin = (common.IntervalList & 'interval_list_name LIKE "raw%"').fetch1( + "valid_times" + )[0][0] + common.IntervalList.insert1( + { + **mini_dict, + "interval_list_name": decode_interval_name, + "valid_times": [[raw_begin, raw_begin + 15]], + }, + skip_duplicates=True, + ) + yield decode_interval_name + + +@pytest.fixture(scope="session") +def decode_merge(common): + from spyglass.decoding import DecodingOutput + + yield DecodingOutput() + + +@pytest.fixture(scope="session") +def decode_sel_key(mini_dict, group_name, pos_interval, decode_interval): + return { + **mini_dict, + "position_group_name": group_name, + "encoding_interval": pos_interval, + "decoding_interval": decode_interval, + } + + +@pytest.fixture(scope="session") +def clusterless_pop( + decode_v1, + decode_sel_key, + group_name, + decode_clusterless_params_insert, + pop_pos_group, + group_unitwave, + teardown, + decode_merge, +): + _ = pop_pos_group, group_unitwave # ensure populated + selection_key = { + **decode_sel_key, + **decode_clusterless_params_insert, + "waveform_features_group_name": group_name, + "estimate_decoding_params": False, + } + + decode_v1.clusterless.ClusterlessDecodingSelection.insert1( + selection_key, + skip_duplicates=True, + ) + decode_v1.clusterless.ClusterlessDecodingV1.populate(selection_key) + + yield decode_v1.clusterless.ClusterlessDecodingV1 & selection_key + + if teardown: + decode_merge.cleanup() + + +@pytest.fixture(scope="session") +def clusterless_key(clusterless_pop): + yield clusterless_pop.fetch("KEY")[0] + + +@pytest.fixture(scope="session") +def clusterless_pop_estimated( + decode_v1, + decode_sel_key, + decode_clusterless_params_insert, + pop_pos_group, + group_unitwave, + group_name, + teardown, + decode_merge, +): + _ = pop_pos_group, group_unitwave + selection_key = { + **decode_sel_key, + **decode_clusterless_params_insert, + "waveform_features_group_name": group_name, + "estimate_decoding_params": True, + } + + decode_v1.clusterless.ClusterlessDecodingSelection.insert1( + selection_key, + skip_duplicates=True, + ) + decode_v1.clusterless.ClusterlessDecodingV1.populate(selection_key) + + yield decode_v1.clusterless.ClusterlessDecodingV1 & selection_key + + if teardown: + decode_merge.cleanup() + + +@pytest.fixture(scope="session") +def decode_spike_params_insert(decode_v1): + from non_local_detector.models import ContFragSortedSpikesClassifier + + params_pk = {"decoding_param_name": "contfrag_sorted"} + decode_v1.core.DecodingParameters.insert1( + { + **params_pk, + "decoding_params": ContFragSortedSpikesClassifier(), + "decoding_kwargs": dict(), + }, + skip_duplicates=True, + ) + yield params_pk + + +@pytest.fixture(scope="session") +def spikes_decoding( + decode_spike_params_insert, + decode_v1, + decode_sel_key, + group_name, + pop_spikes_group, + pop_pos_group, +): + _ = pop_spikes_group, pop_pos_group # ensure populated + spikes = decode_v1.sorted_spikes + selection_key = { + **decode_sel_key, + **decode_spike_params_insert, + "sorted_spikes_group_name": group_name, + "unit_filter_params_name": "default_exclusion", + "estimate_decoding_params": False, + } + spikes.SortedSpikesDecodingSelection.insert1( + selection_key, + skip_duplicates=True, + ) + spikes.SortedSpikesDecodingV1.populate(selection_key) + + yield spikes.SortedSpikesDecodingV1 & selection_key + + +@pytest.fixture(scope="session") +def spikes_decoding_key(spikes_decoding): + yield spikes_decoding.fetch("KEY")[0] + + +@pytest.fixture(scope="session") +def spikes_decoding_estimated( + decode_spike_params_insert, + decode_v1, + decode_sel_key, + group_name, + pop_spikes_group, + pop_pos_group, +): + _ = pop_spikes_group, pop_pos_group # ensure populated + spikes = decode_v1.sorted_spikes + selection_key = { + **decode_sel_key, + **decode_spike_params_insert, + "sorted_spikes_group_name": group_name, + "unit_filter_params_name": "default_exclusion", + "estimate_decoding_params": True, + } + spikes.SortedSpikesDecodingSelection.insert1( + selection_key, + skip_duplicates=True, + ) + spikes.SortedSpikesDecodingV1.populate(selection_key) + + yield spikes.SortedSpikesDecodingV1 & selection_key diff --git a/tests/decoding/test_clusterless.py b/tests/decoding/test_clusterless.py new file mode 100644 index 000000000..fc8967454 --- /dev/null +++ b/tests/decoding/test_clusterless.py @@ -0,0 +1,76 @@ +import numpy as np +import pandas as pd +import pytest + + +def test_fetch_results(clusterless_pop, result_coordinates): + results = clusterless_pop.fetch_results() + assert result_coordinates.issubset( + results.coords._names + ), "Incorrect coordinates in results" + + +def test_fetch_model(clusterless_pop): + from non_local_detector.models.base import ClusterlessDetector + + assert isinstance( + clusterless_pop.fetch_model(), ClusterlessDetector + ), "Model is not ClusterlessDetector" + + +def test_fetch_environments(clusterless_pop, clusterless_key): + from non_local_detector.environment import Environment + + env = clusterless_pop.fetch_environments(clusterless_key)[0] + assert isinstance(env, Environment), "Fetched obj not Environment type" + + +@pytest.mark.skip(reason="Need track graph") +def test_fetch_linearized_position(clusterless_pop, clusterless_key): + lin_pos = clusterless_pop.fetch_linear_position_info(clusterless_key) + assert lin_pos is not None, "Linearized position is None" + + +def test_fetch_spike_by_interval(decode_v1, clusterless_pop, clusterless_key): + begin, end = decode_v1.clusterless._get_interval_range(clusterless_key) + spikes = clusterless_pop.fetch_spike_data( + clusterless_key, filter_by_interval=True + )[0][0] + assert np.all((spikes >= begin) & (spikes <= end)), "Spikes not in interval" + + +def test_get_orientation_col(clusterless_pop): + df = pd.DataFrame(columns=["orientation"]) + ret = clusterless_pop.get_orientation_col(df) + assert ret == "orientation", "Orientation column not found" + + +def test_get_firing_rate( + common, decode_interval, clusterless_pop, clusterless_key +): + interval = ( + common.IntervalList & {"interval_list_name": decode_interval} + ).fetch1("valid_times")[0] + rate = clusterless_pop.get_firing_rate(key=clusterless_key, time=interval) + assert rate.shape == (2, 1), "Incorrect firing rate shape" + + +def test_clusterless_estimated(clusterless_pop_estimated, result_coordinates): + results = clusterless_pop_estimated.fetch_results() + assert result_coordinates.issubset( + results.coords._names + ), "Incorrect coordinates in estimated" + + +@pytest.mark.skip(reason="Need track graph") +def test_get_ahead(clusterless_pop): + dist = clusterless_pop.get_ahead_behind_distance() + assert dist is not None, "Distance is None" + + +def test_insert_existing_group(caplog, group_unitwave): + file, group = group_unitwave.fetch1( + "nwb_file_name", "waveform_features_group_name" + ) + group_unitwave.create_group(file, group, ["dummy_data"]) + assert "already exists" in caplog.text, "No warning issued." diff --git a/tests/decoding/test_core.py b/tests/decoding/test_core.py new file mode 100644 index 000000000..a1b5b42cf --- /dev/null +++ b/tests/decoding/test_core.py @@ -0,0 +1,22 @@ +import pytest + + +def test_decode_param_fetch(decode_v1, decode_clusterless_params_insert): + from non_local_detector.environment import Environment + + key = decode_clusterless_params_insert + ret = (decode_v1.core.DecodingParameters & key).fetch1()["decoding_params"] + env = ret["environments"][0] + assert isinstance(env, Environment), "fetch failed to restore class" + + +def test_null_pos_group(caplog, decode_v1, pop_pos_group): + file, group = pop_pos_group.fetch1("nwb_file_name", "position_group_name") + pop_pos_group.create_group(file, group, ["dummy_pos"]) + assert "already exists" in caplog.text + + +def test_upsampled_pos_group(pop_pos_group_upsampled): + ret = pop_pos_group_upsampled.fetch_position_info()[0] + sample_freq = ret.index.to_series().diff().mode().iloc[0] + pytest.approx(sample_freq, 0.001) == 1 / 250, "Upsampled data not at 250 Hz" diff --git a/tests/decoding/test_merge.py b/tests/decoding/test_merge.py new file mode 100644 index 000000000..c5709037b --- /dev/null +++ b/tests/decoding/test_merge.py @@ -0,0 +1,82 @@ +import pytest + + +@pytest.fixture(scope="session") +def decode_merge_key(decode_merge, clusterless_pop): + _ = clusterless_pop # ensure population is created + return decode_merge.fetch("KEY")[0] + + +@pytest.fixture(scope="session") +def decode_merge_class(): + from spyglass.decoding.decoding_merge import DecodingOutput + + return DecodingOutput + + +@pytest.fixture(scope="session") +def decode_merge_restr(decode_merge, decode_merge_key): + return decode_merge & decode_merge_key + + +def test_decode_merge_fetch_results( + decode_merge_restr, decode_merge_key, result_coordinates +): + + results = decode_merge_restr.fetch_results(decode_merge_key) + assert result_coordinates.issubset( + results.coords._names + ), "Incorrect coordinates in results" + + +def test_decode_merge_fetch_model(decode_merge_restr, decode_merge_key): + from non_local_detector.models.base import ClusterlessDetector + + assert isinstance( + decode_merge_restr.fetch_model(decode_merge_key), ClusterlessDetector + ), "Model is not ClusterlessDetector" + + +def test_decode_merge_fetch_env(decode_merge_restr, decode_merge_key): + from non_local_detector.environment import Environment + + env = decode_merge_restr.fetch_environments(decode_merge_key)[0] + + assert isinstance(env, Environment), "Fetched obj not Environment type" + + +def test_decode_merge_fetch_pos(decode_merge_restr, decode_merge_key): + ret = decode_merge_restr.fetch_position_info(decode_merge_key)[0] + cols = set(ret.columns) + assert cols == { + "position_x", + "velocity_x", + "orientation", + "position_y", + "speed", + "velocity_y", + }, "Incorrect columns in position info" + + +def test_decode_linear_position(decode_merge_restr, decode_merge_key): + + ret = decode_merge_restr.fetch_linear_position_info(decode_merge_key) + cols = set(ret.columns) + assert cols == { + "projected_y_position", + "speed", + "velocity_x", + "orientation", + "linear_position", + "position_x", + "velocity_y", + "position_y", + "track_segment_id", + "projected_x_position", + } + + +# @pytest.mark.skip("Errors on unpacking mult from fetch") +def test_decode_view(decode_merge_restr, decode_merge_key): + ret = decode_merge_restr.create_decoding_view(decode_merge_key) + assert ret is not None, "Failed to create decoding view" diff --git a/tests/decoding/test_spikes.py b/tests/decoding/test_spikes.py new file mode 100644 index 000000000..45e971e43 --- /dev/null +++ b/tests/decoding/test_spikes.py @@ -0,0 +1,61 @@ +import numpy as np +import pandas as pd +import pytest + + +def test_spikes_decoding(spikes_decoding, result_coordinates): + results = spikes_decoding.fetch_results() + assert result_coordinates.issubset( + results.coords._names + ), "Incorrect coordinates in results" + + +def test_fetch_model(spikes_decoding): + from non_local_detector.models.base import SortedSpikesDetector + + assert isinstance( + spikes_decoding.fetch_model(), SortedSpikesDetector + ), "Model is not ClusterlessDetector" + + +def test_fetch_environments(spikes_decoding, spikes_decoding_key): + from non_local_detector.environment import Environment + + env = spikes_decoding.fetch_environments(spikes_decoding_key)[0] + assert isinstance(env, Environment), "Fetched obj not Environment type" + + +@pytest.mark.skip(reason="Need track graph") +def test_fetch_linearized_position(spikes_decoding, spikes_decoding_key): + lin_pos = spikes_decoding.fetch_linear_position_info(spikes_decoding_key) + assert lin_pos is not None, "Linearized position is None" + + +def test_fetch_spike_by_interval( + decode_v1, spikes_decoding, spikes_decoding_key +): + begin, end = decode_v1.clusterless._get_interval_range(spikes_decoding_key) + spikes = spikes_decoding.fetch_spike_data( + spikes_decoding_key, filter_by_interval=True + )[0][0] + assert np.all((spikes >= begin) & (spikes <= end)), "Spikes not in interval" + + +def test_spikes_by_place(spikes_decoding, spikes_decoding_key): + spikes = spikes_decoding.spike_times_sorted_by_place_field_peak() + eg = next(iter(spikes.values()))[0] # get first value + assert eg.shape[0] > 0, "Spikes by place failed" + + +def test_get_orientation_col(spikes_decoding): + df = pd.DataFrame(columns=["orientation"]) + ret = spikes_decoding.get_orientation_col(df) + assert ret == "orientation", "Orientation column not found" + + +def test_spikes_decoding_estimated( + spikes_decoding_estimated, result_coordinates +): + assert result_coordinates.issubset( + spikes_decoding_estimated.fetch_results().coords._names + ), "Incorrect coordinates in estimated" diff --git a/tests/decoding/test_wave.py b/tests/decoding/test_wave.py new file mode 100644 index 000000000..20bd8cd27 --- /dev/null +++ b/tests/decoding/test_wave.py @@ -0,0 +1,21 @@ +def test_unitwave_data(pop_unitwave): + spike_times, spike_waveform_features = pop_unitwave.fetch_data() + feat_shape = spike_waveform_features[0].shape + assert ( + len(spike_times[0]) == feat_shape[0] + ), "Spike times and waveform features do not match in length" + assert feat_shape[1] == 3, "Waveform features should have 3 dimensions" + + +def test_waveform_param_default(waveform_params_tbl): + names = waveform_params_tbl.fetch("features_param_name") + assert "amplitude" in names, "Amplitude not found in waveform parameters" + + +def test_waveform_param_map(waveform_params_tbl): + funcs = waveform_params_tbl().supported_waveform_features + assert "amplitude" in funcs, "Amplitude not found in supported funcs" + + +def test_pos_group(pop_pos_group): + assert len(pop_pos_group) > 0, "No position data found" diff --git a/tests/spikesorting/conftest.py b/tests/spikesorting/conftest.py index ffecbd890..5d9ffb717 100644 --- a/tests/spikesorting/conftest.py +++ b/tests/spikesorting/conftest.py @@ -1,129 +1,4 @@ -import re - import pytest -from datajoint.hash import key_hash - - -@pytest.fixture(scope="session") -def spike_v1(common): - from spyglass.spikesorting import v1 - - yield v1 - - -@pytest.fixture(scope="session") -def pop_rec(spike_v1, mini_dict, team_name): - spike_v1.SortGroup.set_group_by_shank(**mini_dict) - key = { - **mini_dict, - "sort_group_id": 0, - "preproc_param_name": "default", - "interval_list_name": "01_s1", - "team_name": team_name, - } - spike_v1.SpikeSortingRecordingSelection.insert_selection(key) - ssr_pk = ( - (spike_v1.SpikeSortingRecordingSelection & key).proj().fetch1("KEY") - ) - spike_v1.SpikeSortingRecording.populate(ssr_pk) - - yield ssr_pk - - -@pytest.fixture(scope="session") -def pop_art(spike_v1, mini_dict, pop_rec): - key = { - "recording_id": pop_rec["recording_id"], - "artifact_param_name": "default", - } - spike_v1.ArtifactDetectionSelection.insert_selection(key) - spike_v1.ArtifactDetection.populate() - - yield spike_v1.ArtifactDetection().fetch("KEY", as_dict=True)[0] - - -@pytest.fixture(scope="session") -def sorter_dict(): - return {"sorter": "mountainsort4"} - - -@pytest.fixture(scope="session") -def pop_sort(spike_v1, pop_rec, pop_art, mini_dict, sorter_dict): - key = { - **mini_dict, - **sorter_dict, - "recording_id": pop_rec["recording_id"], - "interval_list_name": str(pop_art["artifact_id"]), - "sorter_param_name": "franklab_tetrode_hippocampus_30KHz", - } - spike_v1.SpikeSortingSelection.insert_selection(key) - spike_v1.SpikeSorting.populate() - - yield spike_v1.SpikeSorting().fetch("KEY", as_dict=True)[0] - - -@pytest.fixture(scope="session") -def sorting_objs(spike_v1, pop_sort): - sort_nwb = (spike_v1.SpikeSorting & pop_sort).fetch_nwb() - sort_si = spike_v1.SpikeSorting.get_sorting(pop_sort) - yield sort_nwb, sort_si - - -@pytest.fixture(scope="session") -def pop_curation(spike_v1, pop_sort): - spike_v1.CurationV1.insert_curation( - sorting_id=pop_sort["sorting_id"], - description="testing sort", - ) - - yield (spike_v1.CurationV1() & {"parent_curation_id": -1}).fetch( - "KEY", as_dict=True - )[0] - - -@pytest.fixture(scope="session") -def pop_metric(spike_v1, pop_sort, pop_curation): - _ = pop_curation # make sure this happens first - key = { - "sorting_id": pop_sort["sorting_id"], - "curation_id": 0, - "waveform_param_name": "default_not_whitened", - "metric_param_name": "franklab_default", - "metric_curation_param_name": "default", - } - - spike_v1.MetricCurationSelection.insert_selection(key) - spike_v1.MetricCuration.populate(key) - - yield spike_v1.MetricCuration().fetch("KEY", as_dict=True)[0] - - -@pytest.fixture(scope="session") -def metric_objs(spike_v1, pop_metric): - key = {"metric_curation_id": pop_metric["metric_curation_id"]} - labels = spike_v1.MetricCuration.get_labels(key) - merge_groups = spike_v1.MetricCuration.get_merge_groups(key) - metrics = spike_v1.MetricCuration.get_metrics(key) - yield labels, merge_groups, metrics - - -@pytest.fixture(scope="session") -def pop_curation_metric(spike_v1, pop_metric, metric_objs): - labels, merge_groups, metrics = metric_objs - parent_dict = {"parent_curation_id": 0} - spike_v1.CurationV1.insert_curation( - sorting_id=( - spike_v1.MetricCurationSelection - & {"metric_curation_id": pop_metric["metric_curation_id"]} - ).fetch1("sorting_id"), - **parent_dict, - labels=labels, - merge_groups=merge_groups, - metrics=metrics, - description="after metric curation", - ) - - yield (spike_v1.CurationV1 & parent_dict).fetch("KEY", as_dict=True)[0] @pytest.fixture(scope="session") @@ -146,7 +21,7 @@ def pop_figurl(spike_v1, pop_sort, metric_objs): @pytest.fixture(scope="session") -def pop_figurl_json(spike_v1, pop_metric): +def pop_figurl_json(spike_v1, pop_metric, metric_objs, pop_sort): # WON'T WORK UNTIL CI/CD KACHERY_CLOUD INIT gh_curation_uri = ( "gh://LorenFrankLab/sorting-curations/main/khl02007/test/curation.json" @@ -174,52 +49,6 @@ def pop_figurl_json(spike_v1, pop_metric): yield spike_v1.CurationV1().fetch("KEY", as_dict=True) # list of dicts -@pytest.fixture(scope="session") -def spike_merge(spike_v1): - from spyglass.spikesorting.spikesorting_merge import SpikeSortingOutput - - yield SpikeSortingOutput() - - -@pytest.fixture(scope="session") -def pop_merge( - spike_v1, pop_curation_metric, spike_merge, mini_dict, sorter_dict -): - # TODO: add figurl fixtures when kachery_cloud is initialized - - spike_merge.insert([pop_curation_metric], part_name="CurationV1") - yield spike_merge.fetch("KEY", as_dict=True)[0] - - -def is_uuid(text): - uuid_pattern = re.compile( - r"\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b" - ) - return uuid_pattern.fullmatch(str(text)) is not None - - -@pytest.fixture(scope="session") -def spike_v1_group(): - from spyglass.spikesorting.analysis.v1 import group - - yield group - - -@pytest.fixture(scope="session") -def pop_group(spike_v1_group, spike_merge, mini_dict, pop_merge): - - _ = pop_merge # make sure this happens first - - spike_v1_group.UnitSelectionParams().insert_default() - spike_v1_group.SortedSpikesGroup().create_group( - **mini_dict, - group_name="demo_group", - keys=spike_merge.proj(spikesorting_merge_id="merge_id").fetch("KEY"), - unit_filter_params_name="default_exclusion", - ) - yield spike_v1_group.SortedSpikesGroup().fetch("KEY", as_dict=True)[0] - - @pytest.fixture(scope="session") def spike_v1_ua(): from spyglass.spikesorting.analysis.v1.unit_annotation import UnitAnnotation @@ -228,9 +57,9 @@ def spike_v1_ua(): @pytest.fixture(scope="session") -def pop_annotations(spike_v1_group, spike_v1_ua, pop_group): +def pop_annotations(spike_v1_group, spike_v1_ua, pop_spikes_group): spike_times, unit_ids = spike_v1_group.SortedSpikesGroup().fetch_spike_data( - pop_group, return_unit_ids=True + pop_spikes_group, return_unit_ids=True ) for spikes, unit_key in zip(spike_times, unit_ids): quant_key = { @@ -247,8 +76,4 @@ def pop_annotations(spike_v1_group, spike_v1_ua, pop_group): spike_v1_ua.add_annotation(quant_key, skip_duplicates=True) spike_v1_ua.add_annotation(label_key, skip_duplicates=True) - yield ( - spike_v1_ua.Annotation - # * (spike_v1_group.SortedSpikesGroup.Units & pop_group) - & {"annotation": "spike_count"} - ) + yield (spike_v1_ua.Annotation & {"annotation": "spike_count"}) diff --git a/tests/spikesorting/test_analysis.py b/tests/spikesorting/test_analysis.py index aa95e24b2..f4578df02 100644 --- a/tests/spikesorting/test_analysis.py +++ b/tests/spikesorting/test_analysis.py @@ -3,7 +3,7 @@ def test_analysis_units(pop_annotations): return_unit_ids=True ) - assert selected_spike_times[0].shape[0] == 243, "Unuxpected spike count" + assert selected_spike_times[0].shape[0] > 0, "Found no spike times" units = [d["unit_id"] for d in selected_unit_ids] - assert units == [0, 1, 2], "Unexpected unit ids" + assert 0 in units, "Unexpected unit ids" diff --git a/tests/spikesorting/test_artifact.py b/tests/spikesorting/test_artifact.py index 5466f9571..1fda3be14 100644 --- a/tests/spikesorting/test_artifact.py +++ b/tests/spikesorting/test_artifact.py @@ -21,8 +21,10 @@ def test_null_artifact_detection(spike_v1, art_interval): rec = spike_v1.SpikeSortingRecording.get_recording(rec_key) input_times = art_interval["valid_times"] - null_times = _get_artifact_times(rec, input_times) + if len(input_times) == 1: + input_times = input_times[0] + null_times = np.concatenate(_get_artifact_times(rec, input_times)) assert np.array_equal( - input_times[0], null_times[0] + input_times, null_times ), "Null artifact detection failed" diff --git a/tests/spikesorting/test_curation.py b/tests/spikesorting/test_curation.py index 7d08c3f29..eac00ab0e 100644 --- a/tests/spikesorting/test_curation.py +++ b/tests/spikesorting/test_curation.py @@ -1,5 +1,4 @@ import numpy as np -from datajoint.hash import key_hash from spikeinterface import BaseSorting from spikeinterface.extractors.nwbextractors import NwbRecordingExtractor @@ -44,7 +43,6 @@ def test_curation_sort_info(spike_v1, pop_curation): exp = { "bad_channel": "False", "curation_id": 0, - "description": "testing sort", "electrode_group_name": "0", "electrode_id": 0, "filtering": "None", @@ -59,8 +57,6 @@ def test_curation_sort_info(spike_v1, pop_curation): "probe_shank": 0, "region_id": 1, "sort_group_id": 0, - "sorter": "mountainsort4", - "sorter_param_name": "franklab_tetrode_hippocampus_30KHz", "subregion_name": None, "subsubregion_name": None, "x": 0.0, @@ -70,6 +66,7 @@ def test_curation_sort_info(spike_v1, pop_curation): "z": 0.0, "z_warped": 0.0, } + for k in exp: assert ( sort_info[k] == exp[k] @@ -99,8 +96,6 @@ def test_curation_sort_metric(spike_v1, pop_curation, pop_curation_metric): "probe_shank": 0, "region_id": 1, "sort_group_id": 0, - "sorter": "mountainsort4", - "sorter_param_name": "franklab_tetrode_hippocampus_30KHz", "subregion_name": None, "subsubregion_name": None, "x": 0.0, diff --git a/tests/spikesorting/test_merge.py b/tests/spikesorting/test_merge.py index 5e0458789..ad7ba2510 100644 --- a/tests/spikesorting/test_merge.py +++ b/tests/spikesorting/test_merge.py @@ -3,12 +3,12 @@ from spikeinterface.extractors.nwbextractors import NwbRecordingExtractor -def test_merge_get_restr(spike_merge, pop_merge, pop_curation_metric): +def test_merge_get_restr(spike_merge, pop_spike_merge, pop_curation_metric): restr_id = spike_merge.get_restricted_merge_ids( pop_curation_metric, sources=["v1"] )[0] - assert ( - restr_id == pop_merge["merge_id"] + assert restr_id == (spike_merge >> pop_curation_metric).fetch1( + "merge_id" ), "SpikeSortingOutput merge_id mismatch" non_artifact = spike_merge.get_restricted_merge_ids( @@ -17,27 +17,25 @@ def test_merge_get_restr(spike_merge, pop_merge, pop_curation_metric): assert restr_id == non_artifact, "SpikeSortingOutput merge_id mismatch" -def test_merge_get_recording(spike_merge, pop_merge): - rec = spike_merge.get_recording(pop_merge) +def test_merge_get_recording(spike_merge, pop_spike_merge): + rec = spike_merge.get_recording(pop_spike_merge) assert isinstance( rec, NwbRecordingExtractor ), "SpikeSortingOutput.get_recording failed to return a RecordingExtractor" -def test_merge_get_sorting(spike_merge, pop_merge): - sort = spike_merge.get_sorting(pop_merge) +def test_merge_get_sorting(spike_merge, pop_spike_merge): + sort = spike_merge.get_sorting(pop_spike_merge) assert isinstance( sort, BaseSorting ), "SpikeSortingOutput.get_sorting failed to return a BaseSorting" -def test_merge_get_sort_group_info(spike_merge, pop_merge): - sort_info = spike_merge.get_sort_group_info(pop_merge).fetch1() +def test_merge_get_sort_group_info(spike_merge, pop_spike_merge): + sort_info = spike_merge.get_sort_group_info(pop_spike_merge).fetch1() expected = { "bad_channel": "False", "contacts": "", - "curation_id": 1, - "description": "after metric curation", "electrode_group_name": "0", "electrode_id": 0, "filtering": "None", @@ -52,8 +50,6 @@ def test_merge_get_sort_group_info(spike_merge, pop_merge): "probe_shank": 0, "region_id": 1, "sort_group_id": 0, - "sorter": "mountainsort4", - "sorter_param_name": "franklab_tetrode_hippocampus_30KHz", "subregion_name": None, "subsubregion_name": None, "x": 0.0, @@ -71,23 +67,23 @@ def test_merge_get_sort_group_info(spike_merge, pop_merge): @pytest.fixture(scope="session") -def merge_times(spike_merge, pop_merge): - yield spike_merge.get_spike_times(pop_merge) +def merge_times(spike_merge, pop_spike_merge): + yield spike_merge.get_spike_times(pop_spike_merge) def test_merge_get_spike_times(merge_times): assert ( - merge_times[0].shape[0] == 243 + merge_times[0].shape[0] == 23908 ), "SpikeSortingOutput.get_spike_times unexpected shape" @pytest.mark.skip(reason="Not testing bc #1077") -def test_merge_get_spike_indicators(spike_merge, pop_merge, merge_times): - ret = spike_merge.get_spike_indicator(pop_merge, time=merge_times) +def test_merge_get_spike_indicators(spike_merge, pop_spike_merge, merge_times): + ret = spike_merge.get_spike_indicator(pop_spike_merge, time=merge_times) raise NotImplementedError(ret) @pytest.mark.skip(reason="Not testing bc #1077") -def test_merge_get_firing_rate(spike_merge, pop_merge, merge_times): - ret = spike_merge.get_firing_rate(pop_merge, time=merge_times) +def test_merge_get_firing_rate(spike_merge, pop_spike_merge, merge_times): + ret = spike_merge.get_firing_rate(pop_spike_merge, time=merge_times) raise NotImplementedError(ret) diff --git a/tests/spikesorting/test_utils.py b/tests/spikesorting/test_utils.py index 47638f993..60c10814e 100644 --- a/tests/spikesorting/test_utils.py +++ b/tests/spikesorting/test_utils.py @@ -1,5 +1,7 @@ from uuid import UUID +import pytest + def test_uuid_generator(): @@ -12,9 +14,14 @@ def test_uuid_generator(): assert len(ret_parts[2]) == 6, "Unexpected uuid length" -def test_get_merge_ids(pop_merge, mini_dict): +@pytest.mark.skip(reason="Issue #1159") +def test_get_merge_ids(pop_spike_merge, mini_dict): from spyglass.spikesorting.v1.utils import get_spiking_sorting_v1_merge_ids + # Doesn't work with new decoding entries in ArtifactDetection + # many-to-one of SpikeSortingRecording to ArtifactDetection ret = get_spiking_sorting_v1_merge_ids(dict(mini_dict, curation_id=1)) assert isinstance(ret[0], UUID), "Unexpected type from util" - assert ret[0] == pop_merge["merge_id"], "Unexpected merge_id from util" + assert ( + ret[0] == pop_spike_merge["merge_id"] + ), "Unexpected merge_id from util" From 6faed4c9983a7502988e49d7480189fae6c7e3f9 Mon Sep 17 00:00:00 2001 From: Samuel Bray Date: Tue, 26 Nov 2024 15:46:23 -0800 Subject: [PATCH 89/94] Limit spikesorting artifacts to valid timestamps (#1196) * limit artifacts to valid timestamps * update changelog --- CHANGELOG.md | 1 + src/spyglass/spikesorting/v0/spikesorting_artifact.py | 5 ++++- src/spyglass/spikesorting/v1/artifact.py | 5 ++++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 23b6e84cc..4d2fa2741 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -81,6 +81,7 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Fix bug in `_compute_metric` #1099 - Fix bug in `insert_curation` returned key #1114 - Fix handling of waveform extraction sparse parameter #1132 + - Limit Artifact detection intervals to valid times #1196 ## [0.5.3] (August 27, 2024) diff --git a/src/spyglass/spikesorting/v0/spikesorting_artifact.py b/src/spyglass/spikesorting/v0/spikesorting_artifact.py index 71edc5de1..da46be139 100644 --- a/src/spyglass/spikesorting/v0/spikesorting_artifact.py +++ b/src/spyglass/spikesorting/v0/spikesorting_artifact.py @@ -276,7 +276,10 @@ def _get_artifact_times( for interval_idx, interval in enumerate(artifact_intervals): artifact_intervals_s[interval_idx] = [ valid_timestamps[interval[0]] - half_removal_window_s, - valid_timestamps[interval[1]] + half_removal_window_s, + np.minimum( + valid_timestamps[interval[1]] + half_removal_window_s, + valid_timestamps[-1], + ), ] # make the artifact intervals disjoint if len(artifact_intervals_s) > 1: diff --git a/src/spyglass/spikesorting/v1/artifact.py b/src/spyglass/spikesorting/v1/artifact.py index 14c82dba6..d795d94c9 100644 --- a/src/spyglass/spikesorting/v1/artifact.py +++ b/src/spyglass/spikesorting/v1/artifact.py @@ -308,7 +308,10 @@ def _get_artifact_times( ), np.searchsorted( valid_timestamps, - valid_timestamps[interval[1]] + half_removal_window_s, + np.minimum( + valid_timestamps[interval[1]] + half_removal_window_s, + valid_timestamps[-1], + ), ), ] artifact_intervals_s[interval_idx] = [ From f56aba0de288e1a9bac10b64830fdfbfcaca2de2 Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Thu, 5 Dec 2024 14:16:48 -0600 Subject: [PATCH 90/94] Misc fixes (#1192) * #1175 * #1185 * #1183 * Fix circular import * #1163 * #1105 * Fix failing tests, close download subprocesses * WIP: fix decode changes spikesort tests * Fix fickle test * Revert typo --- CHANGELOG.md | 9 ++- pyproject.toml | 8 ++- src/spyglass/common/common_lab.py | 6 +- src/spyglass/common/common_usage.py | 40 +++++++------ src/spyglass/decoding/v1/core.py | 4 +- .../position/v1/position_dlc_model.py | 18 ++++-- .../position/v1/position_dlc_project.py | 2 - src/spyglass/spikesorting/v1/curation.py | 13 ++--- src/spyglass/utils/dj_helper_fn.py | 14 ++++- src/spyglass/utils/dj_mixin.py | 11 +++- tests/conftest.py | 45 ++++++++++---- tests/data_downloader.py | 58 +++++++++++++------ tests/decoding/test_clusterless.py | 1 + tests/spikesorting/test_curation.py | 1 - tests/spikesorting/test_merge.py | 16 ++--- 15 files changed, 162 insertions(+), 84 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d2fa2741..9ca5b47e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,7 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Remove numpy version restriction #1169 - Merge table delete removes orphaned master entries #1164 - Edit `merge_fetch` to expect positional before keyword arguments #1181 +- Allow part restriction `SpyglassMixinPart.delete` #1192 ### Pipelines @@ -52,8 +53,11 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Improve electrodes import efficiency #1125 - Fix logger method call in `common_task` #1132 - Export fixes #1164 - - Allow `get_abs_path` to add selection entry. - - Log restrictions and joins. + - Allow `get_abs_path` to add selection entry. #1164 + - Log restrictions and joins. #1164 + - Check if querying table inherits mixin in `fetch_nwb`. #1192 + - Ensure externals entries before adding to export. #1192 + - Error specificity in `LabMemberInfo` #1192 - Decoding @@ -74,6 +78,7 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() `open-cv` #1168 - `VideoMaker` class to process frames in multithreaded batches #1168, #1174 - `TrodesPosVideo` updates for `matplotlib` processor #1174 + - User prompt if ambiguous insert in `DLCModelSource` #1192 - Spike Sorting diff --git a/pyproject.toml b/pyproject.toml index a5d8d032d..0a1cd627f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -131,7 +131,7 @@ addopts = [ # "--pdb", # drop into debugger on failure "-p no:warnings", # "--no-teardown", # don't teardown the database after tests - # "--quiet-spy", # don't show logging from spyglass + "--quiet-spy", # don't show logging from spyglass # "--no-dlc", # don't run DLC tests "--show-capture=no", "--pdbcls=IPython.terminal.debugger:TerminalPdb", # use ipython debugger @@ -148,6 +148,12 @@ env = [ "TF_ENABLE_ONEDNN_OPTS = 0", # TF disable approx calcs "TF_CPP_MIN_LOG_LEVEL = 2", # Disable TF warnings ] +filterwarnings = [ + "ignore::ResourceWarning:.*", + "ignore::DeprecationWarning:.*", + "ignore::UserWarning:.*", + "ignore::MissingRequiredBuildWarning:.*", +] [tool.coverage.run] source = ["*/src/spyglass/*"] diff --git a/src/spyglass/common/common_lab.py b/src/spyglass/common/common_lab.py index 486041abc..72bdafa6b 100644 --- a/src/spyglass/common/common_lab.py +++ b/src/spyglass/common/common_lab.py @@ -133,9 +133,11 @@ def get_djuser_name(cls, dj_user) -> str: ) if len(query) != 1: + remedy = f"delete {len(query)-1}" if len(query) > 1 else "add one" raise ValueError( - f"Could not find name for datajoint user {dj_user}" - + f" in common.LabMember.LabMemberInfo: {query}" + f"Could not find exactly 1 datajoint user {dj_user}" + + " in common.LabMember.LabMemberInfo. " + + f"Please {remedy}: {query}" ) return query[0] diff --git a/src/spyglass/common/common_usage.py b/src/spyglass/common/common_usage.py index ccbf7c909..58b28c0f6 100644 --- a/src/spyglass/common/common_usage.py +++ b/src/spyglass/common/common_usage.py @@ -9,12 +9,10 @@ from typing import List, Union import datajoint as dj -from datajoint import FreeTable -from datajoint import config as dj_config from pynwb import NWBHDF5IO from spyglass.common.common_nwbfile import AnalysisNwbfile, Nwbfile -from spyglass.settings import export_dir, test_mode +from spyglass.settings import test_mode from spyglass.utils import SpyglassMixin, SpyglassMixinPart, logger from spyglass.utils.dj_graph import RestrGraph from spyglass.utils.dj_helper_fn import ( @@ -174,7 +172,6 @@ def list_file_paths(self, key: dict, as_dict=True) -> list[str]: Return as a list of dicts: [{'file_path': x}]. Default True. If False, returns a list of strings without key. """ - file_table = self * self.File & key unique_fp = { *[ AnalysisNwbfile().get_abs_path(p) @@ -210,21 +207,26 @@ def _add_externals_to_restr_graph( restr_graph : RestrGraph The updated RestrGraph """ - raw_tbl = self._externals["raw"] - raw_name = raw_tbl.full_table_name - raw_restr = ( - "filepath in ('" + "','".join(self._list_raw_files(key)) + "')" - ) - restr_graph.graph.add_node(raw_name, ft=raw_tbl, restr=raw_restr) - - analysis_tbl = self._externals["analysis"] - analysis_name = analysis_tbl.full_table_name - analysis_restr = ( # filepaths have analysis subdir. regexp substrings - "filepath REGEXP '" + "|".join(self._list_analysis_files(key)) + "'" - ) # regexp is slow, but we're only doing this once, and future-proof - restr_graph.graph.add_node( - analysis_name, ft=analysis_tbl, restr=analysis_restr - ) + + if raw_files := self._list_raw_files(key): + raw_tbl = self._externals["raw"] + raw_name = raw_tbl.full_table_name + raw_restr = "filepath in ('" + "','".join(raw_files) + "')" + restr_graph.graph.add_node(raw_name, ft=raw_tbl, restr=raw_restr) + restr_graph.visited.add(raw_name) + + if analysis_files := self._list_analysis_files(key): + analysis_tbl = self._externals["analysis"] + analysis_name = analysis_tbl.full_table_name + # to avoid issues with analysis subdir, we use REGEXP + # this is slow, but we're only doing this once, and future-proof + analysis_restr = ( + "filepath REGEXP '" + "|".join(analysis_files) + "'" + ) + restr_graph.graph.add_node( + analysis_name, ft=analysis_tbl, restr=analysis_restr + ) + restr_graph.visited.add(analysis_name) restr_graph.visited.update({raw_name, analysis_name}) diff --git a/src/spyglass/decoding/v1/core.py b/src/spyglass/decoding/v1/core.py index d58af1643..177a87d22 100644 --- a/src/spyglass/decoding/v1/core.py +++ b/src/spyglass/decoding/v1/core.py @@ -126,8 +126,8 @@ def create_group( } if self & group_key: logger.error( # Easier for pytests to not raise error on duplicate - f"Group {nwb_file_name}: {group_name} already exists" - + "please delete the group before creating a new one" + f"Group {nwb_file_name}: {group_name} already exists. " + + "Please delete the group before creating a new one" ) return self.insert1( diff --git a/src/spyglass/position/v1/position_dlc_model.py b/src/spyglass/position/v1/position_dlc_model.py index 2a64c05c4..78325b306 100644 --- a/src/spyglass/position/v1/position_dlc_model.py +++ b/src/spyglass/position/v1/position_dlc_model.py @@ -98,16 +98,24 @@ def insert_entry( dj.conn(), full_table_name=part_table.parents()[-1] ) & {"project_name": project_name} - if cls._test_mode: # temporary fix for #1105 - project_path = table_query.fetch(limit=1)[0] - else: - project_path = table_query.fetch1("project_path") + n_found = len(table_query) + if n_found != 1: + logger.warning( + f"Found {len(table_query)} entries found for project " + + f"{project_name}:\n{table_query}" + ) + + choice = "y" + if n_found > 1 and not cls._test_mode: + choice = dj.utils.user_choice("Use first entry?")[0] + if n_found == 0 or choice != "y": + return part_table.insert1( { "dlc_model_name": dlc_model_name, "project_name": project_name, - "project_path": project_path, + "project_path": table_query.fetch("project_path", limit=1)[0], **key, }, **kwargs, diff --git a/src/spyglass/position/v1/position_dlc_project.py b/src/spyglass/position/v1/position_dlc_project.py index 2f19b1664..86617a526 100644 --- a/src/spyglass/position/v1/position_dlc_project.py +++ b/src/spyglass/position/v1/position_dlc_project.py @@ -57,8 +57,6 @@ class DLCProject(SpyglassMixin, dj.Manual): With ability to edit config, extract frames, label frames """ - # Add more parameters as secondary keys... - # TODO: collapse params into blob dict definition = """ project_name : varchar(100) # name of DLC project --- diff --git a/src/spyglass/spikesorting/v1/curation.py b/src/spyglass/spikesorting/v1/curation.py index 00b1ef81e..593d2c1de 100644 --- a/src/spyglass/spikesorting/v1/curation.py +++ b/src/spyglass/spikesorting/v1/curation.py @@ -9,7 +9,6 @@ import spikeinterface.extractors as se from spyglass.common import BrainRegion, Electrode -from spyglass.common.common_ephys import Raw from spyglass.common.common_nwbfile import AnalysisNwbfile from spyglass.spikesorting.v1.recording import ( SortGroup, @@ -17,7 +16,7 @@ SpikeSortingRecordingSelection, ) from spyglass.spikesorting.v1.sorting import SpikeSorting, SpikeSortingSelection -from spyglass.utils.dj_mixin import SpyglassMixin +from spyglass.utils import SpyglassMixin, logger schema = dj.schema("spikesorting_v1_curation") @@ -84,13 +83,13 @@ def insert_curation( sort_query = cls & {"sorting_id": sorting_id} parent_curation_id = max(parent_curation_id, -1) - if parent_curation_id == -1: + + parent_query = sort_query & {"curation_id": parent_curation_id} + if parent_curation_id == -1 and len(parent_query): # check to see if this sorting with a parent of -1 # has already been inserted and if so, warn the user - query = sort_query & {"parent_curation_id": -1} - if query: - Warning("Sorting has already been inserted.") - return query.fetch("KEY") + logger.warning("Sorting has already been inserted.") + return parent_query.fetch("KEY") # generate curation ID existing_curation_ids = sort_query.fetch("curation_id") diff --git a/src/spyglass/utils/dj_helper_fn.py b/src/spyglass/utils/dj_helper_fn.py index 42cf67ba0..de07de85b 100644 --- a/src/spyglass/utils/dj_helper_fn.py +++ b/src/spyglass/utils/dj_helper_fn.py @@ -223,6 +223,7 @@ def get_nwb_table(query_expression, tbl, attr_name, *attrs, **kwargs): Function to get the absolute path to the NWB file. """ from spyglass.common.common_nwbfile import AnalysisNwbfile, Nwbfile + from spyglass.utils.dj_mixin import SpyglassMixin kwargs["as_dict"] = True # force return as dictionary attrs = attrs or query_expression.heading.names # if none, all @@ -234,11 +235,18 @@ def get_nwb_table(query_expression, tbl, attr_name, *attrs, **kwargs): } file_name_str, file_path_fn = tbl_map[which] + # logging arg only if instanced table inherits Mixin + inst = ( # instancing may not be necessary + query_expression() + if isinstance(query_expression, type) + and issubclass(query_expression, dj.Table) + else query_expression + ) + arg = dict(log_export=False) if isinstance(inst, SpyglassMixin) else dict() + # TODO: check that the query_expression restricts tbl - CBroz nwb_files = ( - query_expression.join( - tbl.proj(nwb2load_filepath=attr_name), log_export=False - ) + query_expression.join(tbl.proj(nwb2load_filepath=attr_name), **arg) ).fetch(file_name_str) # Disabled #1024 diff --git a/src/spyglass/utils/dj_mixin.py b/src/spyglass/utils/dj_mixin.py index 72e34c04f..91cf35870 100644 --- a/src/spyglass/utils/dj_mixin.py +++ b/src/spyglass/utils/dj_mixin.py @@ -145,10 +145,10 @@ def _nwb_table_tuple(self) -> tuple: Used to determine fetch_nwb behavior. Also used in Merge.fetch_nwb. Implemented as a cached_property to avoid circular imports.""" - from spyglass.common.common_nwbfile import ( + from spyglass.common.common_nwbfile import ( # noqa F401 AnalysisNwbfile, Nwbfile, - ) # noqa F401 + ) table_dict = { AnalysisNwbfile: "analysis_file_abs_path", @@ -857,4 +857,9 @@ def delete(self, *args, **kwargs): """Delete master and part entries.""" restriction = self.restriction or True # for (tbl & restr).delete() - (self.master & restriction).delete(*args, **kwargs) + try: # try restriction on master + restricted = self.master & restriction + except DataJointError: # if error, assume restr of self + restricted = self & restriction + + restricted.delete(*args, **kwargs) diff --git a/tests/conftest.py b/tests/conftest.py index 22ffa5d2b..1cb54cbd1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,19 +17,13 @@ import pynwb import pytest from datajoint.logging import logger as dj_logger +from hdmf.build.warnings import MissingRequiredBuildWarning from numba import NumbaWarning from pandas.errors import PerformanceWarning from .container import DockerMySQLManager from .data_downloader import DataDownloader -warnings.filterwarnings("ignore", category=UserWarning, module="hdmf") -warnings.filterwarnings("ignore", module="tensorflow") -warnings.filterwarnings("ignore", category=FutureWarning, module="sklearn") -warnings.filterwarnings("ignore", category=PerformanceWarning, module="pandas") -warnings.filterwarnings("ignore", category=NumbaWarning, module="numba") -warnings.filterwarnings("ignore", category=ResourceWarning, module="datajoint") - # ------------------------------- TESTS CONFIG ------------------------------- # globals in pytest_configure: @@ -114,6 +108,19 @@ def pytest_configure(config): download_dlc=not NO_DLC, ) + warnings.filterwarnings("ignore", module="tensorflow") + warnings.filterwarnings("ignore", category=UserWarning, module="hdmf") + warnings.filterwarnings( + "ignore", category=MissingRequiredBuildWarning, module="hdmf" + ) + warnings.filterwarnings("ignore", category=FutureWarning, module="sklearn") + warnings.filterwarnings( + "ignore", category=PerformanceWarning, module="pandas" + ) + warnings.filterwarnings("ignore", category=NumbaWarning, module="numba") + warnings.simplefilter("ignore", category=ResourceWarning) + warnings.simplefilter("ignore", category=DeprecationWarning) + def pytest_unconfigure(config): from spyglass.utils.nwb_helper_fn import close_nwb_files @@ -121,6 +128,9 @@ def pytest_unconfigure(config): close_nwb_files() if TEARDOWN: SERVER.stop() + analysis_dir = BASE_DIR / "analysis" + for file in analysis_dir.glob("*.nwb"): + file.unlink() # ---------------------------- FIXTURES, TEST ENV ---------------------------- @@ -1357,6 +1367,8 @@ def sorter_dict(): @pytest.fixture(scope="session") def pop_sort(spike_v1, pop_rec, pop_art, mini_dict, sorter_dict): + pre = spike_v1.SpikeSorting().fetch("KEY", as_dict=True) + key = { **mini_dict, **sorter_dict, @@ -1367,7 +1379,9 @@ def pop_sort(spike_v1, pop_rec, pop_art, mini_dict, sorter_dict): spike_v1.SpikeSortingSelection.insert_selection(key) spike_v1.SpikeSorting.populate() - yield spike_v1.SpikeSorting().fetch("KEY", as_dict=True)[0] + yield (spike_v1.SpikeSorting() - pre).fetch( + "KEY", as_dict=True, order_by="time_of_sort desc" + )[0] @pytest.fixture(scope="session") @@ -1379,9 +1393,16 @@ def sorting_objs(spike_v1, pop_sort): @pytest.fixture(scope="session") def pop_curation(spike_v1, pop_sort): + + parent_curation_id = -1 + has_sort = spike_v1.CurationV1 & {"sorting_id": pop_sort["sorting_id"]} + if has_sort: + parent_curation_id = has_sort.fetch1("curation_id") + spike_v1.CurationV1.insert_curation( sorting_id=pop_sort["sorting_id"], description="testing sort", + parent_curation_id=parent_curation_id, ) yield (spike_v1.CurationV1() & {"parent_curation_id": -1}).fetch( @@ -1418,20 +1439,20 @@ def metric_objs(spike_v1, pop_metric): @pytest.fixture(scope="session") def pop_curation_metric(spike_v1, pop_metric, metric_objs): labels, merge_groups, metrics = metric_objs - parent_dict = {"parent_curation_id": 0} + desc_dict = dict(description="after metric curation") spike_v1.CurationV1.insert_curation( sorting_id=( spike_v1.MetricCurationSelection & {"metric_curation_id": pop_metric["metric_curation_id"]} ).fetch1("sorting_id"), - **parent_dict, + parent_curation_id=0, labels=labels, merge_groups=merge_groups, metrics=metrics, - description="after metric curation", + **desc_dict, ) - yield (spike_v1.CurationV1 & parent_dict).fetch("KEY", as_dict=True)[0] + yield (spike_v1.CurationV1 & desc_dict).fetch("KEY", as_dict=True)[0] @pytest.fixture(scope="session") diff --git a/tests/data_downloader.py b/tests/data_downloader.py index 40af1ea88..99f1c3ee4 100644 --- a/tests/data_downloader.py +++ b/tests/data_downloader.py @@ -85,29 +85,53 @@ def file_downloads(self) -> Dict[str, Union[Popen, None]]: if dest.exists(): cmd = ["echo", f"Already have {target}"] + ret[target] = "Done" else: cmd = ["curl", "-L", "--output", str(dest), f"{path['url']}"] - - print(f"cmd: {cmd}") - - ret[target] = Popen(cmd, **self.cmd_kwargs) + ret[target] = Popen(cmd, **self.cmd_kwargs) return ret - def wait_for(self, target: str): - """Wait for target to finish downloading.""" - status = self.file_downloads.get(target).poll() - - limit = 10 - while status is None and limit > 0: - time_sleep(5) - limit -= 1 - status = self.file_downloads.get(target).poll() + def wait_for(self, target: str, timeout: int = 50, interval=5): + """Wait for target to finish downloading, and clean up if needed. + + Parameters + ---------- + target : str + Name of file to wait for. + timeout : int, optional + Maximum time to wait for download to finish. + interval : int, optional + Time between checks for download completion. + + Raises + ------ + ValueError + If download failed or target not being downloaded. + TimeoutError + If download took too long. + """ + process = self.file_downloads.get(target) + if not process: + raise ValueError(f"No active download process for target: {target}") + if process == "Done": + return - if status != 0: # Error downloading - raise ValueError(f"Error downloading: {target}") - if limit < 1: # Reached attempt limit - raise TimeoutError(f"Timeout downloading: {target}") + elapsed_time = 0 + try: # Refactored to clean up process streams + while (status := process.poll()) is None: + if elapsed_time >= timeout: + process.terminate() # Terminate on timeout + process.wait() + raise TimeoutError(f"Timeout waiting for {target}.") + time_sleep(interval) + elapsed_time += interval + if status != 0: + raise ValueError(f"Error occurred during download of {target}.") + finally: # Ensure process streams are closed and cleaned up + process.stdout and process.stdout.close() + process.stderr and process.stderr.close() + self.file_downloads[target] = "Done" # Remove target from dict def move_dlc_items(self, dest_dir: Path): """Move completed DLC files to dest_dir.""" diff --git a/tests/decoding/test_clusterless.py b/tests/decoding/test_clusterless.py index fc8967454..66d80c8d0 100644 --- a/tests/decoding/test_clusterless.py +++ b/tests/decoding/test_clusterless.py @@ -31,6 +31,7 @@ def test_fetch_linearized_position(clusterless_pop, clusterless_key): assert lin_pos is not None, "Linearized position is None" +# NOTE: Impacts spikesorting merge tests def test_fetch_spike_by_interval(decode_v1, clusterless_pop, clusterless_key): begin, end = decode_v1.clusterless._get_interval_range(clusterless_key) spikes = clusterless_pop.fetch_spike_data( diff --git a/tests/spikesorting/test_curation.py b/tests/spikesorting/test_curation.py index eac00ab0e..dccff0f69 100644 --- a/tests/spikesorting/test_curation.py +++ b/tests/spikesorting/test_curation.py @@ -80,7 +80,6 @@ def test_curation_sort_metric(spike_v1, pop_curation, pop_curation_metric): expected = { "bad_channel": "False", "contacts": "", - "curation_id": 1, "description": "after metric curation", "electrode_group_name": "0", "electrode_id": 0, diff --git a/tests/spikesorting/test_merge.py b/tests/spikesorting/test_merge.py index ad7ba2510..638179271 100644 --- a/tests/spikesorting/test_merge.py +++ b/tests/spikesorting/test_merge.py @@ -68,22 +68,22 @@ def test_merge_get_sort_group_info(spike_merge, pop_spike_merge): @pytest.fixture(scope="session") def merge_times(spike_merge, pop_spike_merge): - yield spike_merge.get_spike_times(pop_spike_merge) + yield spike_merge.get_spike_times(pop_spike_merge)[0] + + +def assert_shape(df, expected: tuple, msg: str = None): + assert df.shape == expected, f"Unexpected shape: {msg}" def test_merge_get_spike_times(merge_times): - assert ( - merge_times[0].shape[0] == 23908 - ), "SpikeSortingOutput.get_spike_times unexpected shape" + assert_shape(merge_times, (243,), "SpikeSortingOutput.get_spike_times") -@pytest.mark.skip(reason="Not testing bc #1077") def test_merge_get_spike_indicators(spike_merge, pop_spike_merge, merge_times): ret = spike_merge.get_spike_indicator(pop_spike_merge, time=merge_times) - raise NotImplementedError(ret) + assert_shape(ret, (243, 3), "SpikeSortingOutput.get_spike_indicator") -@pytest.mark.skip(reason="Not testing bc #1077") def test_merge_get_firing_rate(spike_merge, pop_spike_merge, merge_times): ret = spike_merge.get_firing_rate(pop_spike_merge, time=merge_times) - raise NotImplementedError(ret) + assert_shape(ret, (243, 3), "SpikeSortingOutput.get_firing_rate") From 692b281c09b00792ae4338af691383b9c16368a3 Mon Sep 17 00:00:00 2001 From: Samuel Bray Date: Thu, 5 Dec 2024 14:51:15 -0800 Subject: [PATCH 91/94] Decoding qol updates (#1198) * generalize DecodingParameters fetch * generalize DecodingParameters fetch1 * initial key decorator * implement full_key_decorator within clusterless pipeline * update changelog * move decorator to mixin class method * remove unused import * update changelog * Update src/spyglass/utils/dj_mixin.py Co-authored-by: Chris Broz --------- Co-authored-by: Chris Broz --- CHANGELOG.md | 3 ++ src/spyglass/decoding/v1/clusterless.py | 46 +++++++++++++--- src/spyglass/decoding/v1/core.py | 48 ++++++++++++----- src/spyglass/decoding/v1/sorted_spikes.py | 52 ++++++++++++++++--- .../spikesorting/analysis/v1/group.py | 10 ++-- src/spyglass/utils/dj_mixin.py | 22 ++++++++ 6 files changed, 148 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ca5b47e1..781f66fe9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Merge table delete removes orphaned master entries #1164 - Edit `merge_fetch` to expect positional before keyword arguments #1181 - Allow part restriction `SpyglassMixinPart.delete` #1192 +- Add mixin method `get_fully_defined_key` #1198 ### Pipelines @@ -62,6 +63,8 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Decoding - Fix edge case errors in spike time loading #1083 + - Allow fetch of partial key from `DecodingParameters` #1198 + - Allow data fetching with partial but unique key #1198 - Linearization diff --git a/src/spyglass/decoding/v1/clusterless.py b/src/spyglass/decoding/v1/clusterless.py index fbfba2183..ae49f388e 100644 --- a/src/spyglass/decoding/v1/clusterless.py +++ b/src/spyglass/decoding/v1/clusterless.py @@ -316,8 +316,8 @@ def fetch_model(self): """Retrieve the decoding model""" return ClusterlessDetector.load_model(self.fetch1("classifier_path")) - @staticmethod - def fetch_environments(key): + @classmethod + def fetch_environments(cls, key): """Fetch the environments for the decoding model Parameters @@ -330,6 +330,9 @@ def fetch_environments(key): List[TrackGraph] list of track graphs in the trained model """ + key = cls.get_fully_defined_key( + key, required_fields=["decoding_param_name"] + ) model_params = ( DecodingParameters & {"decoding_param_name": key["decoding_param_name"]} @@ -355,8 +358,8 @@ def fetch_environments(key): return classifier.environments - @staticmethod - def fetch_position_info(key): + @classmethod + def fetch_position_info(cls, key): """Fetch the position information for the decoding model Parameters @@ -369,6 +372,15 @@ def fetch_position_info(key): Tuple[pd.DataFrame, List[str]] The position information and the names of the position variables """ + key = cls.get_fully_defined_key( + key, + required_fields=[ + "nwb_file_name", + "position_group_name", + "encoding_interval", + "decoding_interval", + ], + ) position_group_key = { "position_group_name": key["position_group_name"], "nwb_file_name": key["nwb_file_name"], @@ -381,8 +393,8 @@ def fetch_position_info(key): return position_info, position_variable_names - @staticmethod - def fetch_linear_position_info(key): + @classmethod + def fetch_linear_position_info(cls, key): """Fetch the position information and project it onto the track graph Parameters @@ -395,6 +407,16 @@ def fetch_linear_position_info(key): pd.DataFrame The linearized position information """ + key = cls.get_fully_defined_key( + key, + required_fields=[ + "nwb_file_name", + "position_group_name", + "encoding_interval", + "decoding_interval", + ], + ) + environment = ClusterlessDecodingV1.fetch_environments(key)[0] position_df = ClusterlessDecodingV1.fetch_position_info(key)[0] @@ -417,8 +439,8 @@ def fetch_linear_position_info(key): axis=1, ).loc[min_time:max_time] - @staticmethod - def fetch_spike_data(key, filter_by_interval=True): + @classmethod + def fetch_spike_data(cls, key, filter_by_interval=True): """Fetch the spike times for the decoding model Parameters @@ -434,6 +456,14 @@ def fetch_spike_data(key, filter_by_interval=True): list[np.ndarray] List of spike times for each unit in the model's spike group """ + key = cls.get_fully_defined_key( + key, + required_fields=[ + "nwb_file_name", + "waveform_features_group_name", + ], + ) + waveform_keys = ( ( UnitWaveformFeaturesGroup.UnitFeatures diff --git a/src/spyglass/decoding/v1/core.py b/src/spyglass/decoding/v1/core.py index 177a87d22..0e3d0fee4 100644 --- a/src/spyglass/decoding/v1/core.py +++ b/src/spyglass/decoding/v1/core.py @@ -70,20 +70,27 @@ def insert(self, rows, *args, **kwargs): def fetch(self, *args, **kwargs): """Return decoding parameters as a list of classes.""" rows = super().fetch(*args, **kwargs) - if len(rows) > 0 and len(rows[0]) > 1: + if kwargs.get("format", None) == "array": + # case when recalled by dj.fetch(), class conversion performed later in stack + return rows + + if not len(args): + # infer args from table heading + args = tuple(self.heading) + + if "decoding_params" not in args: + return rows + + params_index = args.index("decoding_params") + if len(args) == 1: + # only fetching decoding_params + content = [restore_classes(r) for r in rows] + elif len(rows): content = [] - for ( - decoding_param_name, - decoding_params, - decoding_kwargs, - ) in rows: - content.append( - ( - decoding_param_name, - restore_classes(decoding_params), - decoding_kwargs, - ) - ) + for row in zip(*rows): + row = list(row) + row[params_index] = restore_classes(row[params_index]) + content.append(tuple(row)) else: content = rows return content @@ -91,7 +98,20 @@ def fetch(self, *args, **kwargs): def fetch1(self, *args, **kwargs): """Return one decoding paramset as a class.""" row = super().fetch1(*args, **kwargs) - row["decoding_params"] = restore_classes(row["decoding_params"]) + + if len(args) == 0: + row["decoding_params"] = restore_classes(row["decoding_params"]) + return row + + if "decoding_params" in args: + if len(args) == 1: + return restore_classes(row) + row = list(row) + row[args.index("decoding_params")] = restore_classes( + row[args.index("decoding_params")] + ) + return tuple(row) + return row diff --git a/src/spyglass/decoding/v1/sorted_spikes.py b/src/spyglass/decoding/v1/sorted_spikes.py index 9e4c2c3ba..7b4ede194 100644 --- a/src/spyglass/decoding/v1/sorted_spikes.py +++ b/src/spyglass/decoding/v1/sorted_spikes.py @@ -275,8 +275,8 @@ def fetch_model(self): """Retrieve the decoding model""" return SortedSpikesDetector.load_model(self.fetch1("classifier_path")) - @staticmethod - def fetch_environments(key): + @classmethod + def fetch_environments(cls, key): """Fetch the environments for the decoding model Parameters @@ -289,6 +289,10 @@ def fetch_environments(key): List[TrackGraph] list of track graphs in the trained model """ + key = cls.get_fully_defined_key( + key, required_fields=["decoding_param_name"] + ) + model_params = ( DecodingParameters & {"decoding_param_name": key["decoding_param_name"]} @@ -314,8 +318,8 @@ def fetch_environments(key): return classifier.environments - @staticmethod - def fetch_position_info(key): + @classmethod + def fetch_position_info(cls, key): """Fetch the position information for the decoding model Parameters @@ -328,6 +332,16 @@ def fetch_position_info(key): Tuple[pd.DataFrame, List[str]] The position information and the names of the position variables """ + key = cls.get_fully_defined_key( + key, + required_fields=[ + "position_group_name", + "nwb_file_name", + "encoding_interval", + "decoding_interval", + ], + ) + position_group_key = { "position_group_name": key["position_group_name"], "nwb_file_name": key["nwb_file_name"], @@ -339,8 +353,8 @@ def fetch_position_info(key): return position_info, position_variable_names - @staticmethod - def fetch_linear_position_info(key): + @classmethod + def fetch_linear_position_info(cls, key): """Fetch the position information and project it onto the track graph Parameters @@ -353,6 +367,16 @@ def fetch_linear_position_info(key): pd.DataFrame The linearized position information """ + key = cls.get_fully_defined_key( + key, + required_fields=[ + "position_group_name", + "nwb_file_name", + "encoding_interval", + "decoding_interval", + ], + ) + environment = SortedSpikesDecodingV1.fetch_environments(key)[0] position_df = SortedSpikesDecodingV1.fetch_position_info(key)[0] @@ -374,9 +398,13 @@ def fetch_linear_position_info(key): axis=1, ).loc[min_time:max_time] - @staticmethod + @classmethod def fetch_spike_data( - key, filter_by_interval=True, time_slice=None, return_unit_ids=False + cls, + key, + filter_by_interval=True, + time_slice=None, + return_unit_ids=False, ) -> Union[list[np.ndarray], Optional[list[dict]]]: """Fetch the spike times for the decoding model @@ -399,6 +427,14 @@ def fetch_spike_data( list[np.ndarray] List of spike times for each unit in the model's spike group """ + key = cls.get_fully_defined_key( + key, + required_fields=[ + "encoding_interval", + "decoding_interval", + ], + ) + spike_times, unit_ids = SortedSpikesGroup.fetch_spike_data( key, return_unit_ids=True ) diff --git a/src/spyglass/spikesorting/analysis/v1/group.py b/src/spyglass/spikesorting/analysis/v1/group.py index 2f862c4fb..34041117b 100644 --- a/src/spyglass/spikesorting/analysis/v1/group.py +++ b/src/spyglass/spikesorting/analysis/v1/group.py @@ -3,7 +3,6 @@ import datajoint as dj import numpy as np -from ripple_detection import get_multiunit_population_firing_rate from spyglass.common import Session # noqa: F401 from spyglass.settings import test_mode @@ -127,9 +126,12 @@ def filter_units( include_mask[ind] = True return include_mask - @staticmethod + @classmethod def fetch_spike_data( - key: dict, time_slice: list[float] = None, return_unit_ids: bool = False + cls, + key: dict, + time_slice: list[float] = None, + return_unit_ids: bool = False, ) -> Union[list[np.ndarray], Optional[list[dict]]]: """fetch spike times for units in the group @@ -148,6 +150,8 @@ def fetch_spike_data( list of np.ndarray list of spike times for each unit in the group """ + key = cls.get_fully_defined_key(key) + # get merge_ids for SpikeSortingOutput merge_ids = ( ( diff --git a/src/spyglass/utils/dj_mixin.py b/src/spyglass/utils/dj_mixin.py index 91cf35870..2df4844a4 100644 --- a/src/spyglass/utils/dj_mixin.py +++ b/src/spyglass/utils/dj_mixin.py @@ -137,6 +137,28 @@ def _safe_context(cls): else nullcontext() ) + @classmethod + def get_fully_defined_key( + cls, key: dict = None, required_fields: list[str] = None + ) -> dict: + if key is None: + key = dict() + + required_fields = required_fields or cls.primary_key + if isinstance(key, (str, dict)): # check is either keys or substrings + if not all( + field in key for field in required_fields + ): # check if all required fields are in key + if not len(query := cls() & key) == 1: # check if key is unique + raise KeyError( + f"Key is neither fully specified nor a unique entry in" + + f"table.\n\tTable: {cls.full_table_name}\n\tKey: {key}" + + f"Required fields: {required_fields}\n\tResult: {query}" + ) + key = query.fetch1("KEY") + + return key + # ------------------------------- fetch_nwb ------------------------------- @cached_property From 11f7cfc7ec82acb17264a689d7ebaa524584710c Mon Sep 17 00:00:00 2001 From: Samuel Bray Date: Fri, 6 Dec 2024 19:05:59 -0800 Subject: [PATCH 92/94] Cleanup IntervalList orphans in weekly job only (#1195) * cleanup interval orphans in nightly job only * update changelog * Update docs/src/ForDevelopers/Management.md Co-authored-by: Chris Broz * suggest less frequent running of IntervalList cleanup --------- Co-authored-by: Chris Broz --- CHANGELOG.md | 2 ++ docs/src/ForDevelopers/Management.md | 11 +++++++++-- src/spyglass/common/common_interval.py | 2 +- src/spyglass/utils/dj_mixin.py | 3 --- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 781f66fe9..6f3ce4cc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,8 +44,10 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Merge table delete removes orphaned master entries #1164 - Edit `merge_fetch` to expect positional before keyword arguments #1181 - Allow part restriction `SpyglassMixinPart.delete` #1192 +- Move cleanup of `IntervalList` orphan entries to cron job cleanup process #1195 - Add mixin method `get_fully_defined_key` #1198 + ### Pipelines - Common diff --git a/docs/src/ForDevelopers/Management.md b/docs/src/ForDevelopers/Management.md index 5c00d2688..df0caae81 100644 --- a/docs/src/ForDevelopers/Management.md +++ b/docs/src/ForDevelopers/Management.md @@ -228,10 +228,16 @@ disk. There are several tables that retain lists of files that have been generated during analyses. If someone deletes analysis entries, files will still be on disk. -To remove orphaned files, we run the following commands in our cron jobs: +Additionally, there are periphery tables such as `IntervalList` which are used +to store entries created by downstream tables. These entries are not +automatically deleted when the downstream entry is removed. To minimize interference +with ongoing user entry creation, we recommend running these cleanups on a less frequent +basis (e.g. weekly). + +To remove orphaned files and entries, we run the following commands in our cron jobs: ```python -from spyglass.common import AnalysisNwbfile +from spyglass.common import AnalysisNwbfile, IntervalList from spyglass.spikesorting import SpikeSorting from spyglass.common.common_nwbfile import schema as nwbfile_schema from spyglass.decoding.v1.sorted_spikes import schema as spikes_schema @@ -241,6 +247,7 @@ from spyglass.decoding.v1.clusterless import schema as clusterless_schema def main(): AnalysisNwbfile().nightly_cleanup() SpikeSorting().nightly_cleanup() + IntervalList().cleanup() nwbfile_schema.external['analysis'].delete(delete_external_files=True)) nwbfile_schema.external['raw'].delete(delete_external_files=True)) spikes_schema.external['analysis'].delete(delete_external_files=True)) diff --git a/src/spyglass/common/common_interval.py b/src/spyglass/common/common_interval.py index 25670f03c..2021c5f69 100644 --- a/src/spyglass/common/common_interval.py +++ b/src/spyglass/common/common_interval.py @@ -158,7 +158,7 @@ def plot_epoch_pos_raw_intervals(self, figsize=(20, 5), return_fig=False): if return_fig: return fig - def nightly_cleanup(self, dry_run=True): + def cleanup(self, dry_run=True): """Clean up orphaned IntervalList entries.""" orphans = self - get_child_tables(self) if dry_run: diff --git a/src/spyglass/utils/dj_mixin.py b/src/spyglass/utils/dj_mixin.py index 2df4844a4..3fcafb71d 100644 --- a/src/spyglass/utils/dj_mixin.py +++ b/src/spyglass/utils/dj_mixin.py @@ -507,9 +507,6 @@ def cautious_delete( delete_external_files=True, display_progress=False ) - if not self._test_mode: - _ = IntervalList().nightly_cleanup(dry_run=False) - def delete(self, *args, **kwargs): """Alias for cautious_delete, overwrites datajoint.table.Table.delete""" self.cautious_delete(*args, **kwargs) From 36bd1321ec07c3b6d54cb24079cc1580f8085075 Mon Sep 17 00:00:00 2001 From: Samuel Bray Date: Thu, 19 Dec 2024 09:54:24 -0800 Subject: [PATCH 93/94] fetch_nwb non-mixin compatability (#1201) * check if mixin clas in fetch_nwb * update changelog --- CHANGELOG.md | 3 +-- src/spyglass/utils/dj_helper_fn.py | 12 +++++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f3ce4cc5..01774e0f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,7 +47,6 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Move cleanup of `IntervalList` orphan entries to cron job cleanup process #1195 - Add mixin method `get_fully_defined_key` #1198 - ### Pipelines - Common @@ -58,7 +57,7 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Export fixes #1164 - Allow `get_abs_path` to add selection entry. #1164 - Log restrictions and joins. #1164 - - Check if querying table inherits mixin in `fetch_nwb`. #1192 + - Check if querying table inherits mixin in `fetch_nwb`. #1192, #1201 - Ensure externals entries before adding to export. #1192 - Error specificity in `LabMemberInfo` #1192 diff --git a/src/spyglass/utils/dj_helper_fn.py b/src/spyglass/utils/dj_helper_fn.py index de07de85b..890ac496e 100644 --- a/src/spyglass/utils/dj_helper_fn.py +++ b/src/spyglass/utils/dj_helper_fn.py @@ -280,6 +280,8 @@ def fetch_nwb(query_expression, nwb_master, *attrs, **kwargs): nwb_objects : list List of dicts containing fetch results and NWB objects. """ + from spyglass.utils.dj_mixin import SpyglassMixin + kwargs["as_dict"] = True # force return as dictionary tbl, attr_name = nwb_master @@ -301,8 +303,16 @@ def fetch_nwb(query_expression, nwb_master, *attrs, **kwargs): # This also opens the file and stores the file object get_nwb_file(file_path) + # logging arg only if instanced table inherits Mixin + inst = ( # instancing may not be necessary + query_expression() + if isinstance(query_expression, type) + and issubclass(query_expression, dj.Table) + else query_expression + ) + arg = dict(log_export=False) if isinstance(inst, SpyglassMixin) else dict() query_table = query_expression.join( - tbl.proj(nwb2load_filepath=attr_name), log_export=False + tbl.proj(nwb2load_filepath=attr_name), **arg ) rec_dicts = query_table.fetch(*attrs, **kwargs) # get filepath for each. Use datajoint for checksum if local From 75ad067c1254350038507767cfba23a685abdc17 Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Fri, 20 Dec 2024 09:20:44 -0600 Subject: [PATCH 94/94] Release 0.5.4 (#1199) * Remove release notes from changelog * Update cff file * Update release date --- CHANGELOG.md | 17 +++-------------- CITATION.cff | 4 ++-- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 01774e0f2..4b85fda6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,18 +1,6 @@ # Change Log -## [0.5.4] (Unreleased) - -### Release Notes - - - -```python -import datajoint as dj -from spyglass.linearization.v1.main import TrackGraph - -TrackGraph.alter() # Add edge map parameter -dj.FreeTable(dj.conn(), "common_session.session_group").drop() -``` +## [0.5.4] (December 20, 2024) ### Infrastructure @@ -44,7 +32,8 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Merge table delete removes orphaned master entries #1164 - Edit `merge_fetch` to expect positional before keyword arguments #1181 - Allow part restriction `SpyglassMixinPart.delete` #1192 -- Move cleanup of `IntervalList` orphan entries to cron job cleanup process #1195 +- Move cleanup of `IntervalList` orphan entries to cron job cleanup process + #1195 - Add mixin method `get_fully_defined_key` #1198 ### Pipelines diff --git a/CITATION.cff b/CITATION.cff index 0af038cb5..64eceaa94 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -166,5 +166,5 @@ keywords: - spike sorting - kachery license: MIT -version: 0.5.3 -date-released: '2024-04-22' +version: 0.5.4 +date-released: '2024-12-20'