From 98e779e6e728fd8371b3cc8dcb0664b973758161 Mon Sep 17 00:00:00 2001 From: Eric Denovellis Date: Wed, 27 Mar 2024 09:53:10 -0700 Subject: [PATCH 1/5] Preliminary code --- src/spyglass/utils/dj_mixin.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/spyglass/utils/dj_mixin.py b/src/spyglass/utils/dj_mixin.py index 29978ae88..2627f535a 100644 --- a/src/spyglass/utils/dj_mixin.py +++ b/src/spyglass/utils/dj_mixin.py @@ -17,6 +17,11 @@ from spyglass.utils.dj_merge_tables import RESERVED_PRIMARY_KEY as MERGE_PK from spyglass.utils.logging import logger +try: + import pynapple # noqa F401 +except ImportError: + pynapple = None + class SpyglassMixin: """Mixin for Spyglass DataJoint tables. @@ -122,6 +127,14 @@ def fetch_nwb(self, *attrs, **kwargs): """ return fetch_nwb(self, self._nwb_table_tuple, *attrs, **kwargs) + def fetch_pynapple(self, *attrs, **kwargs): + """Fetch Pynapple object from relevant table.""" + if pynapple is None: + raise ImportError("Pynapple not installed.") + + pynapple.load_file("A2929-200711.nwb") + return fetch_nwb(self, self._nwb_table_tuple, *attrs, **kwargs) + # ------------------------ delete_downstream_merge ------------------------ @cached_property From c08c7b4ccbed907ccf91bd24f18979b878e8ed92 Mon Sep 17 00:00:00 2001 From: Eric Denovellis Date: Thu, 28 Mar 2024 16:54:57 -0700 Subject: [PATCH 2/5] Add retrieval of file names --- src/spyglass/utils/dj_mixin.py | 59 +++++++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/src/spyglass/utils/dj_mixin.py b/src/spyglass/utils/dj_mixin.py index 2627f535a..93e9695a0 100644 --- a/src/spyglass/utils/dj_mixin.py +++ b/src/spyglass/utils/dj_mixin.py @@ -128,12 +128,63 @@ def fetch_nwb(self, *attrs, **kwargs): return fetch_nwb(self, self._nwb_table_tuple, *attrs, **kwargs) def fetch_pynapple(self, *attrs, **kwargs): - """Fetch Pynapple object from relevant table.""" + """Get a pynapple object from the given DataJoint query. + + Parameters + ---------- + query_expression : query + A DataJoint query expression (e.g., join, restrict) or a table to call fetch on. + nwb_master : tuple + Tuple (table, attr) to get the NWB filepath from. + i.e. absolute path to NWB file can be obtained by looking up attr column of table + table is usually Nwbfile or AnalysisNwbfile; + attr is usually 'nwb_file_abs_path' or 'analysis_file_abs_path' + *attrs : list + Attributes from normal DataJoint fetch call. + **kwargs : dict + Keyword arguments from normal DataJoint fetch call. + + Returns + ------- + nwb_objects : list + List of dicts containing fetch results and NWB objects. + """ if pynapple is None: - raise ImportError("Pynapple not installed.") + raise ImportError("Pynapple is not installed.") - pynapple.load_file("A2929-200711.nwb") - return fetch_nwb(self, self._nwb_table_tuple, *attrs, **kwargs) + query_expression, nwb_master = self, self._nwb_table_tuple + tbl, attr_name = nwb_master + kwargs["as_dict"] = True # force return as dictionary + + if not attrs: + attrs = query_expression.heading.names + + # get the list of analysis or nwb files + file_name_str = ( + "analysis_file_name" + if "analysis" in nwb_master[1] + else "nwb_file_name" + ) + # TODO: avoid this import? + from spyglass.common.common_nwbfile import AnalysisNwbfile, Nwbfile + + file_path_fn = ( + AnalysisNwbfile.get_abs_path + if "analysis" in nwb_master[1] + else Nwbfile.get_abs_path + ) + + # TODO: check that the query_expression restricts tbl - CBroz + nwb_files = ( + query_expression * tbl.proj(nwb2load_filepath=attr_name) + ).fetch(file_name_str) + + py_objs = [] + for file_name in nwb_files: + file_path = file_path_fn(file_name) + py_objs.append(pynapple.load_file(file_path)) + + return py_objs # ------------------------ delete_downstream_merge ------------------------ From 530dbcf5aa0661c5b462efd8110998bcff9ec35b Mon Sep 17 00:00:00 2001 From: Eric Denovellis Date: Fri, 29 Mar 2024 09:18:13 -0700 Subject: [PATCH 3/5] Add get_nwb_table function --- src/spyglass/utils/dj_helper_fn.py | 44 ++++++++++++++++-------------- src/spyglass/utils/dj_mixin.py | 43 ++++++++--------------------- 2 files changed, 34 insertions(+), 53 deletions(-) diff --git a/src/spyglass/utils/dj_helper_fn.py b/src/spyglass/utils/dj_helper_fn.py index 4a0495778..c4f6febbd 100644 --- a/src/spyglass/utils/dj_helper_fn.py +++ b/src/spyglass/utils/dj_helper_fn.py @@ -105,6 +105,27 @@ def dj_replace(original_table, new_values, key_column, replace_column): return original_table +def get_nwb_table(query_expression, tbl, attr_name, *attrs, **kwargs): + from spyglass.common.common_nwbfile import AnalysisNwbfile, Nwbfile + + kwargs["as_dict"] = True # force return as dictionary + attrs = attrs or query_expression.heading.names # if none, all + + which = "analysis" if "analysis" in attr_name else "nwb" + tbl_map = { # map to file_name_str and file_path_fn + "analysis": ["analysis_file_name", AnalysisNwbfile.get_abs_path], + "nwb": ["nwb_file_name", Nwbfile.get_abs_path], + } + file_name_str, file_path_fn = tbl_map[which] + + # TODO: check that the query_expression restricts tbl - CBroz + nwb_files = ( + query_expression * tbl.proj(nwb2load_filepath=attr_name) + ).fetch(file_name_str) + + return nwb_files, file_path_fn + + def fetch_nwb(query_expression, nwb_master, *attrs, **kwargs): """Get an NWB object from the given DataJoint query. @@ -127,29 +148,10 @@ def fetch_nwb(query_expression, nwb_master, *attrs, **kwargs): nwb_objects : list List of dicts containing fetch results and NWB objects. """ - kwargs["as_dict"] = True # force return as dictionary tbl, attr_name = nwb_master - - if not attrs: - attrs = query_expression.heading.names - - # get the list of analysis or nwb files - file_name_str = ( - "analysis_file_name" if "analysis" in nwb_master[1] else "nwb_file_name" - ) - # TODO: avoid this import? - from ..common.common_nwbfile import AnalysisNwbfile, Nwbfile - - file_path_fn = ( - AnalysisNwbfile.get_abs_path - if "analysis" in nwb_master[1] - else Nwbfile.get_abs_path + nwb_files, file_path_fn = get_nwb_table( + query_expression, tbl, attr_name, *attrs, **kwargs ) - - # TODO: check that the query_expression restricts tbl - CBroz - nwb_files = ( - query_expression * tbl.proj(nwb2load_filepath=attr_name) - ).fetch(file_name_str) for file_name in nwb_files: file_path = file_path_fn(file_name) if not os.path.exists(file_path): diff --git a/src/spyglass/utils/dj_mixin.py b/src/spyglass/utils/dj_mixin.py index 93e9695a0..64812ea56 100644 --- a/src/spyglass/utils/dj_mixin.py +++ b/src/spyglass/utils/dj_mixin.py @@ -13,7 +13,7 @@ from spyglass.utils.database_settings import SHARED_MODULES from spyglass.utils.dj_chains import TableChain, TableChains -from spyglass.utils.dj_helper_fn import fetch_nwb +from spyglass.utils.dj_helper_fn import fetch_nwb, get_nwb_table from spyglass.utils.dj_merge_tables import RESERVED_PRIMARY_KEY as MERGE_PK from spyglass.utils.logging import logger @@ -152,39 +152,18 @@ def fetch_pynapple(self, *attrs, **kwargs): if pynapple is None: raise ImportError("Pynapple is not installed.") - query_expression, nwb_master = self, self._nwb_table_tuple - tbl, attr_name = nwb_master - kwargs["as_dict"] = True # force return as dictionary - - if not attrs: - attrs = query_expression.heading.names - - # get the list of analysis or nwb files - file_name_str = ( - "analysis_file_name" - if "analysis" in nwb_master[1] - else "nwb_file_name" - ) - # TODO: avoid this import? - from spyglass.common.common_nwbfile import AnalysisNwbfile, Nwbfile - - file_path_fn = ( - AnalysisNwbfile.get_abs_path - if "analysis" in nwb_master[1] - else Nwbfile.get_abs_path + nwb_files, file_path_fn = get_nwb_table( + self, + self._nwb_table_tuple[0], + self._nwb_table_tuple[1], + *attrs, + **kwargs, ) - # TODO: check that the query_expression restricts tbl - CBroz - nwb_files = ( - query_expression * tbl.proj(nwb2load_filepath=attr_name) - ).fetch(file_name_str) - - py_objs = [] - for file_name in nwb_files: - file_path = file_path_fn(file_name) - py_objs.append(pynapple.load_file(file_path)) - - return py_objs + return [ + pynapple.load_file(file_path_fn(file_name)) + for file_name in nwb_files + ] # ------------------------ delete_downstream_merge ------------------------ From ba5a68d83f229b84348bd7bab150f641113d3fe4 Mon Sep 17 00:00:00 2001 From: Eric Denovellis Date: Fri, 29 Mar 2024 09:22:14 -0700 Subject: [PATCH 4/5] Update docstrings --- src/spyglass/utils/dj_helper_fn.py | 22 ++++++++++++++++++++++ src/spyglass/utils/dj_mixin.py | 17 ++++++++--------- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/src/spyglass/utils/dj_helper_fn.py b/src/spyglass/utils/dj_helper_fn.py index c4f6febbd..c5fd82276 100644 --- a/src/spyglass/utils/dj_helper_fn.py +++ b/src/spyglass/utils/dj_helper_fn.py @@ -106,6 +106,28 @@ def dj_replace(original_table, new_values, key_column, replace_column): def get_nwb_table(query_expression, tbl, attr_name, *attrs, **kwargs): + """Get the NWB file name and path from the given DataJoint query. + + Parameters + ---------- + query_expression : query + A DataJoint query expression (e.g., join, restrict) or a table to call fetch on. + tbl : table + DataJoint table to fetch from. + attr_name : str + Attribute name to fetch from the table. + *attrs : list + Attributes from normal DataJoint fetch call. + **kwargs : dict + Keyword arguments from normal DataJoint fetch call. + + Returns + ------- + nwb_files : list + List of NWB file names. + file_path_fn : function + Function to get the absolute path to the NWB file. + """ from spyglass.common.common_nwbfile import AnalysisNwbfile, Nwbfile kwargs["as_dict"] = True # force return as dictionary diff --git a/src/spyglass/utils/dj_mixin.py b/src/spyglass/utils/dj_mixin.py index 64812ea56..082116bf6 100644 --- a/src/spyglass/utils/dj_mixin.py +++ b/src/spyglass/utils/dj_mixin.py @@ -132,13 +132,6 @@ def fetch_pynapple(self, *attrs, **kwargs): Parameters ---------- - query_expression : query - A DataJoint query expression (e.g., join, restrict) or a table to call fetch on. - nwb_master : tuple - Tuple (table, attr) to get the NWB filepath from. - i.e. absolute path to NWB file can be obtained by looking up attr column of table - table is usually Nwbfile or AnalysisNwbfile; - attr is usually 'nwb_file_abs_path' or 'analysis_file_abs_path' *attrs : list Attributes from normal DataJoint fetch call. **kwargs : dict @@ -146,8 +139,14 @@ def fetch_pynapple(self, *attrs, **kwargs): Returns ------- - nwb_objects : list - List of dicts containing fetch results and NWB objects. + pynapple_objects : list of pynapple objects + List of dicts containing pynapple objects. + + Raises + ------ + ImportError + If pynapple is not installed. + """ if pynapple is None: raise ImportError("Pynapple is not installed.") From a4a703be6d7317e36eb4651478d3607078ab84ab Mon Sep 17 00:00:00 2001 From: Eric Denovellis Date: Fri, 29 Mar 2024 09:23:00 -0700 Subject: [PATCH 5/5] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index da03d58c1..9af02f4e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - Refactor `TableChain` to include `_searched` attribute. #867 - Fix errors in config import #882 - Save current spyglass version in analysis nwb files to aid diagnosis #897 +- Add pynapple support #898 ### Pipelines