Skip to content

Commit

Permalink
Merge branch 'ticket/PSB-249/dev' into rc/2.16.0
Browse files Browse the repository at this point in the history
  • Loading branch information
@morriscb
morriscb committed Nov 1, 2023
2 parents fe87c87 + 0620446 commit f8ea31b
Show file tree
Hide file tree
Showing 11 changed files with 133,493 additions and 133,307 deletions.
221 changes: 131 additions & 90 deletions allensdk/brain_observatory/behavior/behavior_project_cache/behavior_project_cache.py
View file
Original file line number Diff line number Diff line change
@@ -1,28 +1,39 @@
from typing import Optional, List, Union
import warnings
from pathlib import Path
import pandas as pd
import numpy as np
from typing import List, Optional, Union

import numpy as np
import pandas as pd
from allensdk.api.warehouse_cache.cache import Cache
from allensdk.brain_observatory.behavior.behavior_ophys_experiment import \
BehaviorOphysExperiment
from allensdk.brain_observatory.behavior.behavior_project_cache.tables \
.experiments_table import \
ExperimentsTable
from allensdk.brain_observatory.behavior.behavior_project_cache.tables \
.sessions_table import \
SessionsTable
from allensdk.api.warehouse_cache.caching_utilities import (
one_file_call_caching,
)
from allensdk.brain_observatory.behavior.behavior_ophys_experiment import (
BehaviorOphysExperiment,
)
from allensdk.brain_observatory.behavior.behavior_project_cache.project_apis.data_io import ( # noqa: E501
BehaviorProjectLimsApi, BehaviorProjectCloudApi)
from allensdk.api.warehouse_cache.caching_utilities import \
one_file_call_caching
from allensdk.brain_observatory.behavior.behavior_project_cache.tables \
.ophys_sessions_table import \
BehaviorOphysSessionsTable
from allensdk.brain_observatory.behavior.behavior_session import \
BehaviorSession
from allensdk.brain_observatory.behavior.behavior_project_cache \
.project_cache_base import ProjectCacheBase
BehaviorProjectCloudApi,
BehaviorProjectLimsApi,
)
from allensdk.brain_observatory.behavior.behavior_project_cache.project_cache_base import ( # noqa: E501
ProjectCacheBase,
)
from allensdk.brain_observatory.behavior.behavior_project_cache.tables.experiments_table import ( # noqa: E501
ExperimentsTable,
)
from allensdk.brain_observatory.behavior.behavior_project_cache.tables.ophys_sessions_table import ( # noqa: E501
BehaviorOphysSessionsTable,
)
from allensdk.brain_observatory.behavior.behavior_project_cache.tables.sessions_table import ( # noqa: E501
SessionsTable,
)
from allensdk.brain_observatory.behavior.behavior_session import (
BehaviorSession,
)


class UpdatedStimulusPresentationTableWarning(UserWarning):
pass


class VBOLimsCache(Cache):
Expand All @@ -41,41 +52,41 @@ class VBOLimsCache(Cache):
OPHYS_SESSIONS_KEY: {
"spec": f"{OPHYS_SESSIONS_KEY}.csv",
"parent_key": "BASEDIR",
"typename": "file"
"typename": "file",
},
BEHAVIOR_SESSIONS_KEY: {
"spec": f"{BEHAVIOR_SESSIONS_KEY}.csv",
"parent_key": "BASEDIR",
"typename": "file"
"typename": "file",
},
OPHYS_EXPERIMENTS_KEY: {
"spec": f"{OPHYS_EXPERIMENTS_KEY}.csv",
"parent_key": "BASEDIR",
"typename": "file"
"typename": "file",
},
OPHYS_CELLS_KEY: {
"spec": f"{OPHYS_CELLS_KEY}.csv",
"parent_key": "BASEDIR",
"typename": "file"
}
"typename": "file",
},
}


class VisualBehaviorOphysProjectCache(ProjectCacheBase):

PROJECT_NAME = "visual-behavior-ophys"
BUCKET_NAME = "visual-behavior-ophys-data"
BUCKET_NAME = "staging.visual-behavior-ophys-data"

def __init__(
self,
fetch_api: Optional[Union[BehaviorProjectLimsApi,
BehaviorProjectCloudApi]] = None,
fetch_tries: int = 2,
manifest: Optional[Union[str, Path]] = None,
version: Optional[str] = None,
cache: bool = True
self,
fetch_api: Optional[
Union[BehaviorProjectLimsApi, BehaviorProjectCloudApi]
] = None,
fetch_tries: int = 2,
manifest: Optional[Union[str, Path]] = None,
version: Optional[str] = None,
cache: bool = True,
):
""" Entrypoint for accessing visual behavior data. Supports
"""Entrypoint for accessing visual behavior data. Supports
access to summaries of session data and provides tools for
downloading detailed session data (such as dff traces).
Expand Down Expand Up @@ -117,9 +128,26 @@ def __init__(

if not isinstance(self.fetch_api, BehaviorProjectCloudApi):
if cache:
self.cache = VBOLimsCache(manifest=manifest_,
version=version,
cache=cache)
self.cache = VBOLimsCache(
manifest=manifest_, version=version, cache=cache
)

warnings.warn(
message="\n\tAs of AllenSDK version 2.16.0, the latest Visual "
"Behavior Ophys data has been significantly updated from "
"previous releases. Specifically the user will need to "
"update all processing of the stimulus_presentations "
"tables. These tables now include multiple stimulus types "
"delineated by the columns `stimulus_block` and "
"`stimulus_block_name`.\n\nThe data that was available in "
"previous releases are stored in the "
"block name containing 'change_detection' and can be "
"accessed in the pandas table by using: \n\t"
"`stimulus_presentations["
"stimulus_presentations.stimulus_block_name.str.contains"
"('change_detection')]`",
category=UpdatedStimulusPresentationTableWarning,
)

@classmethod
def cloud_api_class(cls):
Expand All @@ -130,11 +158,11 @@ def lims_api_class(cls):
return BehaviorProjectLimsApi

def get_ophys_session_table(
self,
suppress: Optional[List[str]] = None,
index_column: str = "ophys_session_id",
as_df=True,
include_behavior_data=True
self,
suppress: Optional[List[str]] = None,
index_column: str = "ophys_session_id",
as_df=True,
include_behavior_data=True,
) -> Union[pd.DataFrame, BehaviorOphysSessionsTable]:
"""
Return summary table of all ophys_session_ids in the database.
Expand All @@ -155,13 +183,15 @@ def get_ophys_session_table(
if isinstance(self.fetch_api, BehaviorProjectCloudApi):
return self.fetch_api.get_ophys_session_table()
if self.cache is not None:
path = self.cache.get_cache_path(None,
self.cache.OPHYS_SESSIONS_KEY)
path = self.cache.get_cache_path(
None, self.cache.OPHYS_SESSIONS_KEY
)
ophys_sessions = one_file_call_caching(
path,
self.fetch_api.get_ophys_session_table,
_write_json,
lambda path: _read_json(path, index_name='ophys_session_id'))
lambda path: _read_json(path, index_name="ophys_session_id"),
)
else:
ophys_sessions = self.fetch_api.get_ophys_session_table()

Expand All @@ -171,24 +201,23 @@ def get_ophys_session_table(
suppress=suppress,
as_df=True,
include_ophys_data=False,
include_trial_metrics=False
include_trial_metrics=False,
)
ophys_sessions = behavior_sessions_table.merge(
ophys_sessions,
left_index=True,
right_on='behavior_session_id',
suffixes=('_behavior', '_ophys'))
right_on="behavior_session_id",
suffixes=("_behavior", "_ophys"),
)

sessions = BehaviorOphysSessionsTable(df=ophys_sessions,
suppress=suppress,
index_column=index_column)
sessions = BehaviorOphysSessionsTable(
df=ophys_sessions, suppress=suppress, index_column=index_column
)

return sessions.table if as_df else sessions

def get_ophys_experiment_table(
self,
suppress: Optional[List[str]] = None,
as_df=True
self, suppress: Optional[List[str]] = None, as_df=True
) -> Union[pd.DataFrame, SessionsTable]:
"""
Return summary table of all ophys_experiment_ids in the database.
Expand All @@ -201,27 +230,34 @@ def get_ophys_experiment_table(
if isinstance(self.fetch_api, BehaviorProjectCloudApi):
return self.fetch_api.get_ophys_experiment_table()
if self.cache is not None:
path = self.cache.get_cache_path(None,
self.cache.OPHYS_EXPERIMENTS_KEY)
path = self.cache.get_cache_path(
None, self.cache.OPHYS_EXPERIMENTS_KEY
)
experiments = one_file_call_caching(
path,
self.fetch_api.get_ophys_experiment_table,
_write_json,
lambda path: _read_json(path,
index_name='ophys_experiment_id'))
lambda path: _read_json(
path, index_name="ophys_experiment_id"
),
)
else:
experiments = self.fetch_api.get_ophys_experiment_table()

# Merge behavior data in
behavior_sessions_table = self.get_behavior_session_table(
suppress=suppress, as_df=True, include_ophys_data=False,
include_trial_metrics=False
suppress=suppress,
as_df=True,
include_ophys_data=False,
include_trial_metrics=False,
)
experiments = behavior_sessions_table.merge(
experiments, left_index=True, right_on='behavior_session_id',
suffixes=('_behavior', '_ophys'))
experiments = ExperimentsTable(df=experiments,
suppress=suppress)
experiments,
left_index=True,
right_on="behavior_session_id",
suffixes=("_behavior", "_ophys"),
)
experiments = ExperimentsTable(df=experiments, suppress=suppress)
return experiments.table if as_df else experiments

def get_ophys_cells_table(self) -> pd.DataFrame:
Expand All @@ -232,25 +268,24 @@ def get_ophys_cells_table(self) -> pd.DataFrame:
if isinstance(self.fetch_api, BehaviorProjectCloudApi):
return self.fetch_api.get_ophys_cells_table()
if self.cache is not None:
path = self.cache.get_cache_path(None,
self.cache.OPHyS_CELLS_KEY)
path = self.cache.get_cache_path(None, self.cache.OPHyS_CELLS_KEY)
ophys_cells_table = one_file_call_caching(
path,
self.fetch_api.get_ophys_cells_table,
_write_json,
lambda path: _read_json(path,
index_name='cell_roi_id'))
lambda path: _read_json(path, index_name="cell_roi_id"),
)
else:
ophys_cells_table = self.fetch_api.get_ophys_cells_table()

return ophys_cells_table

def get_behavior_session_table(
self,
suppress: Optional[List[str]] = None,
as_df=True,
include_ophys_data=True,
include_trial_metrics: bool = False
self,
suppress: Optional[List[str]] = None,
as_df=True,
include_ophys_data=True,
include_trial_metrics: bool = False,
) -> Union[pd.DataFrame, SessionsTable]:
"""
Return summary table of all behavior_session_ids in the database.
Expand All @@ -269,22 +304,23 @@ def get_behavior_session_table(
if isinstance(self.fetch_api, BehaviorProjectCloudApi):
return self.fetch_api.get_behavior_session_table()
if self.cache is not None:
path = self.cache.get_cache_path(None,
self.cache.BEHAVIOR_SESSIONS_KEY)
path = self.cache.get_cache_path(
None, self.cache.BEHAVIOR_SESSIONS_KEY
)
sessions = one_file_call_caching(
path,
self.fetch_api.get_behavior_session_table,
_write_json,
lambda path: _read_json(path,
index_name='behavior_session_id'))
lambda path: _read_json(
path, index_name="behavior_session_id"
),
)
else:
sessions = self.fetch_api.get_behavior_session_table()

if include_ophys_data:
ophys_session_table = self.get_ophys_session_table(
suppress=suppress,
as_df=False,
include_behavior_data=False
suppress=suppress, as_df=False, include_behavior_data=False
)
else:
ophys_session_table = None
Expand All @@ -293,12 +329,13 @@ def get_behavior_session_table(
suppress=suppress,
fetch_api=self.fetch_api,
ophys_session_table=ophys_session_table,
include_trial_metrics=include_trial_metrics)
include_trial_metrics=include_trial_metrics,
)

return sessions.table if as_df else sessions

def get_behavior_ophys_experiment(
self, ophys_experiment_id: int
self, ophys_experiment_id: int
) -> BehaviorOphysExperiment:
"""
Gets `BehaviorOphysExperiment` for `ophys_experiment_id`
Expand All @@ -311,11 +348,11 @@ def get_behavior_ophys_experiment(
BehaviorOphysExperiment
"""
return self.fetch_api.get_behavior_ophys_experiment(
ophys_experiment_id=ophys_experiment_id)
ophys_experiment_id=ophys_experiment_id
)

def get_behavior_session(
self,
behavior_session_id: int
self, behavior_session_id: int
) -> BehaviorSession:
"""
Gets `BehaviorSession` for `behavior_session_id`
Expand Down Expand Up @@ -346,7 +383,7 @@ def get_natural_movie_template(self, n_workers=None) -> pd.DataFrame:
frames as presented to the mouse. The DataFrame is indexed with the
same frame index as shown in the stimulus presentation table.
The processing of the movie requires signicant processing and its
The processing of the movie requires significant processing and its
return size is very large so take care in requesting this data.
Parameters
Expand Down Expand Up @@ -381,8 +418,12 @@ def _write_json(path, df):

def _read_json(path, index_name: Optional[str] = None):
"""Reads a dataframe file written to the cache by _write_json."""
df = pd.read_json(path, date_unit="s", orient="split",
convert_dates=["date_of_acquisition"])
df = pd.read_json(
path,
date_unit="s",
orient="split",
convert_dates=["date_of_acquisition"],
)
if index_name:
df = df.rename_axis(index=index_name)
return df
Loading

0 comments on commit f8ea31b

Please sign in to comment.