From 0620446b35a3e6c3bba5b27e511569aa0666fc20 Mon Sep 17 00:00:00 2001 From: Christopher Morrison Date: Wed, 1 Nov 2023 16:10:26 -0700 Subject: [PATCH] isort+black --- .../behavior_project_cache.py | 223 ++++++++++-------- .../data_io/behavior_project_cloud_api.py | 26 +- .../data_io/behavior_project_lims_api.py | 4 +- .../behavior/stimulus_processing.py | 6 +- .../ophys/project_constants.py | 62 +++-- 5 files changed, 182 insertions(+), 139 deletions(-) diff --git a/allensdk/brain_observatory/behavior/behavior_project_cache/behavior_project_cache.py b/allensdk/brain_observatory/behavior/behavior_project_cache/behavior_project_cache.py index 417a9fc3d..cea0a7125 100644 --- a/allensdk/brain_observatory/behavior/behavior_project_cache/behavior_project_cache.py +++ b/allensdk/brain_observatory/behavior/behavior_project_cache/behavior_project_cache.py @@ -1,29 +1,35 @@ -from typing import Optional, List, Union -from pathlib import Path -import pandas as pd -import numpy as np import warnings +from pathlib import Path +from typing import List, Optional, Union +import numpy as np +import pandas as pd from allensdk.api.warehouse_cache.cache import Cache -from allensdk.brain_observatory.behavior.behavior_ophys_experiment import \ - BehaviorOphysExperiment -from allensdk.brain_observatory.behavior.behavior_project_cache.tables \ - .experiments_table import \ - ExperimentsTable -from allensdk.brain_observatory.behavior.behavior_project_cache.tables \ - .sessions_table import \ - SessionsTable +from allensdk.api.warehouse_cache.caching_utilities import ( + one_file_call_caching, +) +from allensdk.brain_observatory.behavior.behavior_ophys_experiment import ( + BehaviorOphysExperiment, +) from allensdk.brain_observatory.behavior.behavior_project_cache.project_apis.data_io import ( # noqa: E501 - BehaviorProjectLimsApi, BehaviorProjectCloudApi) -from allensdk.api.warehouse_cache.caching_utilities import \ - one_file_call_caching -from allensdk.brain_observatory.behavior.behavior_project_cache.tables \ - .ophys_sessions_table import \ - BehaviorOphysSessionsTable -from allensdk.brain_observatory.behavior.behavior_session import \ - BehaviorSession -from allensdk.brain_observatory.behavior.behavior_project_cache \ - .project_cache_base import ProjectCacheBase + BehaviorProjectCloudApi, + BehaviorProjectLimsApi, +) +from allensdk.brain_observatory.behavior.behavior_project_cache.project_cache_base import ( # noqa: E501 + ProjectCacheBase, +) +from allensdk.brain_observatory.behavior.behavior_project_cache.tables.experiments_table import ( # noqa: E501 + ExperimentsTable, +) +from allensdk.brain_observatory.behavior.behavior_project_cache.tables.ophys_sessions_table import ( # noqa: E501 + BehaviorOphysSessionsTable, +) +from allensdk.brain_observatory.behavior.behavior_project_cache.tables.sessions_table import ( # noqa: E501 + SessionsTable, +) +from allensdk.brain_observatory.behavior.behavior_session import ( + BehaviorSession, +) class UpdatedStimulusPresentationTableWarning(UserWarning): @@ -46,41 +52,41 @@ class VBOLimsCache(Cache): OPHYS_SESSIONS_KEY: { "spec": f"{OPHYS_SESSIONS_KEY}.csv", "parent_key": "BASEDIR", - "typename": "file" + "typename": "file", }, BEHAVIOR_SESSIONS_KEY: { "spec": f"{BEHAVIOR_SESSIONS_KEY}.csv", "parent_key": "BASEDIR", - "typename": "file" + "typename": "file", }, OPHYS_EXPERIMENTS_KEY: { "spec": f"{OPHYS_EXPERIMENTS_KEY}.csv", "parent_key": "BASEDIR", - "typename": "file" + "typename": "file", }, OPHYS_CELLS_KEY: { "spec": f"{OPHYS_CELLS_KEY}.csv", "parent_key": "BASEDIR", - "typename": "file" - } + "typename": "file", + }, } class VisualBehaviorOphysProjectCache(ProjectCacheBase): - PROJECT_NAME = "visual-behavior-ophys" BUCKET_NAME = "staging.visual-behavior-ophys-data" def __init__( - self, - fetch_api: Optional[Union[BehaviorProjectLimsApi, - BehaviorProjectCloudApi]] = None, - fetch_tries: int = 2, - manifest: Optional[Union[str, Path]] = None, - version: Optional[str] = None, - cache: bool = True + self, + fetch_api: Optional[ + Union[BehaviorProjectLimsApi, BehaviorProjectCloudApi] + ] = None, + fetch_tries: int = 2, + manifest: Optional[Union[str, Path]] = None, + version: Optional[str] = None, + cache: bool = True, ): - """ Entrypoint for accessing visual behavior data. Supports + """Entrypoint for accessing visual behavior data. Supports access to summaries of session data and provides tools for downloading detailed session data (such as dff traces). @@ -122,25 +128,25 @@ def __init__( if not isinstance(self.fetch_api, BehaviorProjectCloudApi): if cache: - self.cache = VBOLimsCache(manifest=manifest_, - version=version, - cache=cache) + self.cache = VBOLimsCache( + manifest=manifest_, version=version, cache=cache + ) warnings.warn( message="\n\tAs of AllenSDK version 2.16.0, the latest Visual " - "Behavior Ophys data has been significantly updated from " - "previous releases. Specifically the user will need to " - "update all processing of the stimulus_presentations " - "tables. These tables now include multiple stimulus types " - "delineated by the columns `stimulus_block` and " - "`stimulus_block_name`.\n\nThe data that was available in " - "previous releases are stored in the " - "block name containing 'change_detection' and can be " - "accessed in the pandas table by using: \n\t" - "`stimulus_presentations[" - "stimulus_presentations.stimulus_block_name.str.contains" - "('change_detection')]`", - category=UpdatedStimulusPresentationTableWarning + "Behavior Ophys data has been significantly updated from " + "previous releases. Specifically the user will need to " + "update all processing of the stimulus_presentations " + "tables. These tables now include multiple stimulus types " + "delineated by the columns `stimulus_block` and " + "`stimulus_block_name`.\n\nThe data that was available in " + "previous releases are stored in the " + "block name containing 'change_detection' and can be " + "accessed in the pandas table by using: \n\t" + "`stimulus_presentations[" + "stimulus_presentations.stimulus_block_name.str.contains" + "('change_detection')]`", + category=UpdatedStimulusPresentationTableWarning, ) @classmethod @@ -152,11 +158,11 @@ def lims_api_class(cls): return BehaviorProjectLimsApi def get_ophys_session_table( - self, - suppress: Optional[List[str]] = None, - index_column: str = "ophys_session_id", - as_df=True, - include_behavior_data=True + self, + suppress: Optional[List[str]] = None, + index_column: str = "ophys_session_id", + as_df=True, + include_behavior_data=True, ) -> Union[pd.DataFrame, BehaviorOphysSessionsTable]: """ Return summary table of all ophys_session_ids in the database. @@ -177,13 +183,15 @@ def get_ophys_session_table( if isinstance(self.fetch_api, BehaviorProjectCloudApi): return self.fetch_api.get_ophys_session_table() if self.cache is not None: - path = self.cache.get_cache_path(None, - self.cache.OPHYS_SESSIONS_KEY) + path = self.cache.get_cache_path( + None, self.cache.OPHYS_SESSIONS_KEY + ) ophys_sessions = one_file_call_caching( path, self.fetch_api.get_ophys_session_table, _write_json, - lambda path: _read_json(path, index_name='ophys_session_id')) + lambda path: _read_json(path, index_name="ophys_session_id"), + ) else: ophys_sessions = self.fetch_api.get_ophys_session_table() @@ -193,24 +201,23 @@ def get_ophys_session_table( suppress=suppress, as_df=True, include_ophys_data=False, - include_trial_metrics=False + include_trial_metrics=False, ) ophys_sessions = behavior_sessions_table.merge( ophys_sessions, left_index=True, - right_on='behavior_session_id', - suffixes=('_behavior', '_ophys')) + right_on="behavior_session_id", + suffixes=("_behavior", "_ophys"), + ) - sessions = BehaviorOphysSessionsTable(df=ophys_sessions, - suppress=suppress, - index_column=index_column) + sessions = BehaviorOphysSessionsTable( + df=ophys_sessions, suppress=suppress, index_column=index_column + ) return sessions.table if as_df else sessions def get_ophys_experiment_table( - self, - suppress: Optional[List[str]] = None, - as_df=True + self, suppress: Optional[List[str]] = None, as_df=True ) -> Union[pd.DataFrame, SessionsTable]: """ Return summary table of all ophys_experiment_ids in the database. @@ -223,27 +230,34 @@ def get_ophys_experiment_table( if isinstance(self.fetch_api, BehaviorProjectCloudApi): return self.fetch_api.get_ophys_experiment_table() if self.cache is not None: - path = self.cache.get_cache_path(None, - self.cache.OPHYS_EXPERIMENTS_KEY) + path = self.cache.get_cache_path( + None, self.cache.OPHYS_EXPERIMENTS_KEY + ) experiments = one_file_call_caching( path, self.fetch_api.get_ophys_experiment_table, _write_json, - lambda path: _read_json(path, - index_name='ophys_experiment_id')) + lambda path: _read_json( + path, index_name="ophys_experiment_id" + ), + ) else: experiments = self.fetch_api.get_ophys_experiment_table() # Merge behavior data in behavior_sessions_table = self.get_behavior_session_table( - suppress=suppress, as_df=True, include_ophys_data=False, - include_trial_metrics=False + suppress=suppress, + as_df=True, + include_ophys_data=False, + include_trial_metrics=False, ) experiments = behavior_sessions_table.merge( - experiments, left_index=True, right_on='behavior_session_id', - suffixes=('_behavior', '_ophys')) - experiments = ExperimentsTable(df=experiments, - suppress=suppress) + experiments, + left_index=True, + right_on="behavior_session_id", + suffixes=("_behavior", "_ophys"), + ) + experiments = ExperimentsTable(df=experiments, suppress=suppress) return experiments.table if as_df else experiments def get_ophys_cells_table(self) -> pd.DataFrame: @@ -254,25 +268,24 @@ def get_ophys_cells_table(self) -> pd.DataFrame: if isinstance(self.fetch_api, BehaviorProjectCloudApi): return self.fetch_api.get_ophys_cells_table() if self.cache is not None: - path = self.cache.get_cache_path(None, - self.cache.OPHyS_CELLS_KEY) + path = self.cache.get_cache_path(None, self.cache.OPHyS_CELLS_KEY) ophys_cells_table = one_file_call_caching( path, self.fetch_api.get_ophys_cells_table, _write_json, - lambda path: _read_json(path, - index_name='cell_roi_id')) + lambda path: _read_json(path, index_name="cell_roi_id"), + ) else: ophys_cells_table = self.fetch_api.get_ophys_cells_table() return ophys_cells_table def get_behavior_session_table( - self, - suppress: Optional[List[str]] = None, - as_df=True, - include_ophys_data=True, - include_trial_metrics: bool = False + self, + suppress: Optional[List[str]] = None, + as_df=True, + include_ophys_data=True, + include_trial_metrics: bool = False, ) -> Union[pd.DataFrame, SessionsTable]: """ Return summary table of all behavior_session_ids in the database. @@ -291,22 +304,23 @@ def get_behavior_session_table( if isinstance(self.fetch_api, BehaviorProjectCloudApi): return self.fetch_api.get_behavior_session_table() if self.cache is not None: - path = self.cache.get_cache_path(None, - self.cache.BEHAVIOR_SESSIONS_KEY) + path = self.cache.get_cache_path( + None, self.cache.BEHAVIOR_SESSIONS_KEY + ) sessions = one_file_call_caching( path, self.fetch_api.get_behavior_session_table, _write_json, - lambda path: _read_json(path, - index_name='behavior_session_id')) + lambda path: _read_json( + path, index_name="behavior_session_id" + ), + ) else: sessions = self.fetch_api.get_behavior_session_table() if include_ophys_data: ophys_session_table = self.get_ophys_session_table( - suppress=suppress, - as_df=False, - include_behavior_data=False + suppress=suppress, as_df=False, include_behavior_data=False ) else: ophys_session_table = None @@ -315,12 +329,13 @@ def get_behavior_session_table( suppress=suppress, fetch_api=self.fetch_api, ophys_session_table=ophys_session_table, - include_trial_metrics=include_trial_metrics) + include_trial_metrics=include_trial_metrics, + ) return sessions.table if as_df else sessions def get_behavior_ophys_experiment( - self, ophys_experiment_id: int + self, ophys_experiment_id: int ) -> BehaviorOphysExperiment: """ Gets `BehaviorOphysExperiment` for `ophys_experiment_id` @@ -333,11 +348,11 @@ def get_behavior_ophys_experiment( BehaviorOphysExperiment """ return self.fetch_api.get_behavior_ophys_experiment( - ophys_experiment_id=ophys_experiment_id) + ophys_experiment_id=ophys_experiment_id + ) def get_behavior_session( - self, - behavior_session_id: int + self, behavior_session_id: int ) -> BehaviorSession: """ Gets `BehaviorSession` for `behavior_session_id` @@ -403,8 +418,12 @@ def _write_json(path, df): def _read_json(path, index_name: Optional[str] = None): """Reads a dataframe file written to the cache by _write_json.""" - df = pd.read_json(path, date_unit="s", orient="split", - convert_dates=["date_of_acquisition"]) + df = pd.read_json( + path, + date_unit="s", + orient="split", + convert_dates=["date_of_acquisition"], + ) if index_name: df = df.rename_axis(index=index_name) return df diff --git a/allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_cloud_api.py b/allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_cloud_api.py index d8c887010..3c6c74783 100644 --- a/allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_cloud_api.py +++ b/allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_cloud_api.py @@ -1,20 +1,25 @@ -from typing import Iterable, Union -import numpy as np import pathlib +from typing import Iterable, Union +import numpy as np import pandas as pd +from allensdk.api.cloud_cache.cloud_cache import ( + LocalCache, + S3CloudCache, + StaticLocalCache, +) from allensdk.brain_observatory.behavior.behavior_ophys_experiment import ( BehaviorOphysExperiment, ) from allensdk.brain_observatory.behavior.behavior_project_cache.project_apis.abcs import ( # noqa: E501 BehaviorProjectBase, ) -from allensdk.brain_observatory.behavior.behavior_project_cache.project_apis.data_io.project_cloud_api_base import ( # noqa: E501 - ProjectCloudApiBase, -) from allensdk.brain_observatory.behavior.behavior_project_cache.project_apis.data_io.natural_movie_one_cache import ( # noqa: E501 NaturalMovieOneCache, ) +from allensdk.brain_observatory.behavior.behavior_project_cache.project_apis.data_io.project_cloud_api_base import ( # noqa: E501 + ProjectCloudApiBase, +) from allensdk.brain_observatory.behavior.behavior_session import ( BehaviorSession, ) @@ -23,9 +28,6 @@ return_one_dataframe_row_only, ) from allensdk.core.utilities import literal_col_eval -from allensdk.api.cloud_cache.cloud_cache import ( - S3CloudCache, LocalCache, StaticLocalCache) - COL_EVAL_LIST = ["ophys_experiment_id", "ophys_container_id", "driver_line"] INTEGER_COLUMNS = [ @@ -71,11 +73,11 @@ def __init__( self, cache: Union[S3CloudCache, LocalCache, StaticLocalCache], skip_version_check: bool = False, - local: bool = False + local: bool = False, ): - super().__init__(cache=cache, - skip_version_check=skip_version_check, - local=local) + super().__init__( + cache=cache, skip_version_check=skip_version_check, local=local + ) self._load_manifest_tables() if isinstance(cache, S3CloudCache): diff --git a/allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_lims_api.py b/allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_lims_api.py index c517ff6ae..d7451df04 100644 --- a/allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_lims_api.py +++ b/allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_lims_api.py @@ -499,9 +499,7 @@ def get_ophys_session_table(self) -> pd.DataFrame: # Fill NaN values of imaging_plane_group_count with zero to match # the behavior of the BehaviorOphysExperiment object. - im_plane_count = ( - table["imaging_plane_group_count"].astype("Int64") - ) + im_plane_count = table["imaging_plane_group_count"].astype("Int64") table["imaging_plane_group_count"] = im_plane_count return table diff --git a/allensdk/brain_observatory/behavior/stimulus_processing.py b/allensdk/brain_observatory/behavior/stimulus_processing.py index 022368e96..a699224af 100644 --- a/allensdk/brain_observatory/behavior/stimulus_processing.py +++ b/allensdk/brain_observatory/behavior/stimulus_processing.py @@ -750,9 +750,9 @@ def compute_trials_id_for_stimulus( max_trials_stop = trials_table.stop_time.max() for idx, trial in trials_table.iterrows(): stim_mask = ( - (stim_pres_table.start_time > trial.start_time) - & (stim_pres_table.start_time < max_trials_stop) - & (~stim_pres_table.image_name.isna()) + (stim_pres_table.start_time > trial.start_time) + & (stim_pres_table.start_time < max_trials_stop) + & (~stim_pres_table.image_name.isna()) ) trials_ids[stim_mask] = idx diff --git a/allensdk/brain_observatory/ophys/project_constants.py b/allensdk/brain_observatory/ophys/project_constants.py index 7b11ca690..5648f4f7a 100644 --- a/allensdk/brain_observatory/ophys/project_constants.py +++ b/allensdk/brain_observatory/ophys/project_constants.py @@ -20,7 +20,7 @@ "VisualBehaviorMultiscope4areasx2d", "DevelopmentMultiscope4areasx2d", "MesoscopeDevelopment", - "VisBIntTestDatacube" + "VisBIntTestDatacube", ] NUM_STRUCTURES_DICT = { @@ -37,24 +37,48 @@ "VisualBehaviorMultiscope4areasx2d": 2, } -VBO_ACTIVE_MAP = {0: 'initial_gray_screen_5min', - 1: 'change_detection_behavior', - 2: 'post_behavior_gray_screen_5min', - 3: 'natural_movie_one'} +VBO_ACTIVE_MAP = { + 0: "initial_gray_screen_5min", + 1: "change_detection_behavior", + 2: "post_behavior_gray_screen_5min", + 3: "natural_movie_one", +} -VBO_PASSIVE_MAP = {0: 'initial_gray_screen_5min', - 1: 'change_detection_passive', - 2: 'post_behavior_gray_screen_5min', - 3: 'natural_movie_one'} +VBO_PASSIVE_MAP = { + 0: "initial_gray_screen_5min", + 1: "change_detection_passive", + 2: "post_behavior_gray_screen_5min", + 3: "natural_movie_one", +} VBO_METADATA_COLUMN_ORDER = [ - 'behavior_session_id', 'ophys_session_id', 'ophys_container_id', - 'mouse_id', 'indicator', 'full_genotype', 'driver_line', - 'cre_line', 'reporter_line', 'sex', 'age_in_days', - 'imaging_depth', 'targeted_structure', 'targeted_imaging_depth', - 'imaging_plane_group', 'project_code', 'session_type', - 'session_number', 'image_set', 'behavior_type', 'passive', - 'experience_level', 'prior_exposures_to_session_type', - 'prior_exposures_to_image_set', 'prior_exposures_to_omissions', - 'date_of_acquisition', 'equipment_name', 'published_at', - 'isi_experiment_id'] + "behavior_session_id", + "ophys_session_id", + "ophys_container_id", + "mouse_id", + "indicator", + "full_genotype", + "driver_line", + "cre_line", + "reporter_line", + "sex", + "age_in_days", + "imaging_depth", + "targeted_structure", + "targeted_imaging_depth", + "imaging_plane_group", + "project_code", + "session_type", + "session_number", + "image_set", + "behavior_type", + "passive", + "experience_level", + "prior_exposures_to_session_type", + "prior_exposures_to_image_set", + "prior_exposures_to_omissions", + "date_of_acquisition", + "equipment_name", + "published_at", + "isi_experiment_id", +]