diff --git a/allensdk/brain_observatory/behavior/behavior_ophys_experiment.py b/allensdk/brain_observatory/behavior/behavior_ophys_experiment.py index 62ee674c8..d7ea61295 100644 --- a/allensdk/brain_observatory/behavior/behavior_ophys_experiment.py +++ b/allensdk/brain_observatory/behavior/behavior_ophys_experiment.py @@ -611,13 +611,17 @@ def cell_specimen_table(self) -> pd.DataFrame: which image plane an ROI resides on. Overlapping ROIs are stored on different mask image planes max_corretion_down: (float) - max motion correction in down direction in pixels + max motion correction in down direction in pixels. Defines + the motion border at the top of the image. max_correction_left: (float) - max motion correction in left direction in pixels + max motion correction in left direction in pixels. Defines + the motion border at the right of the image. max_correction_right: (float) - max motion correction in right direction in pixels + max motion correction in right direction in pixels. Defines + the motion border at the left of the image. max_correction_up: (float) - max motion correction in up direction in pixels + max motion correction in up direction in pixels. Defines + the motion border at the bottom of the image. roi_mask: (array of bool) an image array that displays the location of the roi mask in the field of view diff --git a/allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_cloud_api.py b/allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_cloud_api.py index efee354d4..2d9f89450 100644 --- a/allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_cloud_api.py +++ b/allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_cloud_api.py @@ -163,7 +163,7 @@ def _get_ophys_session_table(self): df["date_of_acquisition"] = pd.to_datetime( df["date_of_acquisition"], utc="True" ) - df = enforce_df_int_typing(df, INTEGER_COLUMNS) + df = enforce_df_int_typing(df, INTEGER_COLUMNS, True) self._ophys_session_table = df.set_index("ophys_session_id") def get_ophys_session_table(self) -> pd.DataFrame: @@ -188,7 +188,7 @@ def _get_behavior_session_table(self): df["date_of_acquisition"] = pd.to_datetime( df["date_of_acquisition"], utc="True" ) - df = enforce_df_int_typing(df, INTEGER_COLUMNS) + df = enforce_df_int_typing(df, INTEGER_COLUMNS, True) self._behavior_session_table = df.set_index("behavior_session_id") @@ -218,7 +218,7 @@ def _get_ophys_experiment_table(self): df["date_of_acquisition"] = pd.to_datetime( df["date_of_acquisition"], utc="True" ) - df = enforce_df_int_typing(df, INTEGER_COLUMNS) + df = enforce_df_int_typing(df, INTEGER_COLUMNS, True) self._ophys_experiment_table = df.set_index("ophys_experiment_id") def _get_ophys_cells_table(self): diff --git a/allensdk/brain_observatory/behavior/behavior_session.py b/allensdk/brain_observatory/behavior/behavior_session.py index 4d9281c68..43b286347 100644 --- a/allensdk/brain_observatory/behavior/behavior_session.py +++ b/allensdk/brain_observatory/behavior/behavior_session.py @@ -1149,7 +1149,7 @@ def stimulus_templates(self) -> Optional[pd.DataFrame]: return None @property - def stimulus_fingerprint_movie_template(self) -> Optional[pd.DataFrame]: + def stimulus_natural_movie_template(self) -> Optional[pd.DataFrame]: """Get stimulus templates movie for the behavior session. Returns None if no stimulus movie is available. diff --git a/allensdk/brain_observatory/behavior/data_objects/stimuli/fingerprint_stimulus.py b/allensdk/brain_observatory/behavior/data_objects/stimuli/fingerprint_stimulus.py index d263ed6ef..13414d461 100644 --- a/allensdk/brain_observatory/behavior/data_objects/stimuli/fingerprint_stimulus.py +++ b/allensdk/brain_observatory/behavior/data_objects/stimuli/fingerprint_stimulus.py @@ -85,7 +85,7 @@ def from_stimulus_file( 'stop_time': stop_time, 'start_frame': start_frame, 'end_frame': end_frame, - 'repeat': repeat, + 'movie_repeat': repeat, 'duration': stop_time - start_time }) table = pd.DataFrame(res) diff --git a/allensdk/brain_observatory/behavior/data_objects/stimuli/presentations.py b/allensdk/brain_observatory/behavior/data_objects/stimuli/presentations.py index b349be722..eead6a477 100644 --- a/allensdk/brain_observatory/behavior/data_objects/stimuli/presentations.py +++ b/allensdk/brain_observatory/behavior/data_objects/stimuli/presentations.py @@ -92,7 +92,7 @@ def __init__( "is_change", "is_image_novel", "omitted", - "repeat", + "movie_repeat", "flashes_since_change", "trials_id", ], @@ -104,7 +104,7 @@ def __init__( "flashes_since_change", "image_index", "movie_frame_index", - "repeat", + "movie_repeat", "stimulus_index", ], ) @@ -290,6 +290,8 @@ def from_stimulus_file( data, stimulus_timestamps.value ) raw_stim_pres_df = raw_stim_pres_df.drop(columns=["index"]) + raw_stim_pres_df = cls._check_for_errant_omitted_stimulus( + input_df=raw_stim_pres_df) # Fill in nulls for image_name # This makes two assumptions: @@ -540,17 +542,37 @@ def _postprocess( ) } ) - # Check if the first entry in the DataFrame is an omitted stimulus. - # This shouldn't happen and likely reflects some sort of camstim error - # with appending frames to the omitted flash frame log. See - # explanation here: - # https://github.com/AllenInstitute/AllenSDK/issues/2577 - if "omitted" in df.columns and len(df) > 0: - first_row = df.iloc[0] + df = cls._check_for_errant_omitted_stimulus(input_df=df) + return df + + @staticmethod + def _check_for_errant_omitted_stimulus( + input_df: pd.DataFrame + ) -> pd.DataFrame: + """Check if the first entry in the DataFrame is an omitted stimulus. + + This shouldn't happen and likely reflects some sort of camstim error + with appending frames to the omitted flash frame log. See + explanation here: + https://github.com/AllenInstitute/AllenSDK/issues/2577 + + Parameters + ----------/ + input_df : DataFrame + Input stimulus table to check for "omitted" stimulus. + + Returns + ------- + modified_df : DataFrame + Dataframe with omitted stimulus removed from first row or if not + found, return input_df unmodified. + """ + if "omitted" in input_df.columns and len(input_df) > 0: + first_row = input_df.iloc[0] if not pd.isna(first_row["omitted"]): if first_row["omitted"]: - df = df.drop(first_row.name, axis=0) - return df + input_df = input_df.drop(first_row.name, axis=0) + return input_df @staticmethod def _fill_missing_values_for_omitted_flashes( diff --git a/allensdk/core/dataframe_utils.py b/allensdk/core/dataframe_utils.py index bcf90a975..b3501e4f9 100644 --- a/allensdk/core/dataframe_utils.py +++ b/allensdk/core/dataframe_utils.py @@ -126,7 +126,8 @@ def enforce_df_column_order( def enforce_df_int_typing(input_df: pd.DataFrame, - int_columns: List[str]) -> pd.DataFrame: + int_columns: List[str], + use_pandas_type=False) -> pd.DataFrame: """Enforce integer typing for columns that may have lost int typing when combined into the final DataFrame. @@ -138,6 +139,10 @@ def enforce_df_int_typing(input_df: pd.DataFrame, Columns to enforce int typing and fill any NaN/None values with the value set in INT_NULL in this file. Requested columns not in the dataframe are ignored. + use_pandas_type : bool + Instead of filling with the value INT_NULL to enforce integer typing, + use the pandas type Int64. This type can have issues converting to + numpy/array type values. Returns ------- @@ -147,6 +152,9 @@ def enforce_df_int_typing(input_df: pd.DataFrame, """ for col in int_columns: if col in input_df.columns: - input_df[col] = \ - input_df[col].fillna(INT_NULL).astype(int) + if use_pandas_type: + input_df[col] = input_df[col].astype('Int64') + else: + input_df[col] = \ + input_df[col].fillna(INT_NULL).astype(int) return input_df diff --git a/allensdk/test/brain_observatory/behavior/behavior_project_cache/test_behavior_project_cloud_api.py b/allensdk/test/brain_observatory/behavior/behavior_project_cache/test_behavior_project_cloud_api.py index 08b6c6ad6..287352fc0 100644 --- a/allensdk/test/brain_observatory/behavior/behavior_project_cache/test_behavior_project_cloud_api.py +++ b/allensdk/test/brain_observatory/behavior/behavior_project_cache/test_behavior_project_cloud_api.py @@ -108,7 +108,7 @@ def mock_cache(request, tmpdir): "ophys_session_table": pd.DataFrame( { "ophys_session_id": pd.Series([10, 11, 12, 13], - dtype='int64'), + dtype='Int64'), "mouse_id": ["1"] * 4, "date_of_acquisition": pd.to_datetime(["2021-01-01"] * 4), "ophys_experiment_id": [4, 5, 6, [7, 8, 9]], diff --git a/allensdk/test/brain_observatory/behavior/data_objects/test_data/fingerprint_stimulus.pkl b/allensdk/test/brain_observatory/behavior/data_objects/test_data/fingerprint_stimulus.pkl index 4652b830d..84d9f4b5f 100644 Binary files a/allensdk/test/brain_observatory/behavior/data_objects/test_data/fingerprint_stimulus.pkl and b/allensdk/test/brain_observatory/behavior/data_objects/test_data/fingerprint_stimulus.pkl differ diff --git a/allensdk/test/brain_observatory/behavior/data_objects/test_data/presentations.pkl b/allensdk/test/brain_observatory/behavior/data_objects/test_data/presentations.pkl index b18f0d870..204e537fd 100644 Binary files a/allensdk/test/brain_observatory/behavior/data_objects/test_data/presentations.pkl and b/allensdk/test/brain_observatory/behavior/data_objects/test_data/presentations.pkl differ diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_experiment.py b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_experiment.py index 6276f2e81..79825b93a 100644 --- a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_experiment.py +++ b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_experiment.py @@ -277,7 +277,7 @@ def test_behavior_ophys_experiment_list_data_attributes_and_methods( "segmentation_mask_image", "stimulus_presentations", "stimulus_templates", - 'stimulus_fingerprint_movie_template', + 'stimulus_natural_movie_template', "stimulus_timestamps", "task_parameters", "trials", diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_session.py b/allensdk/test/brain_observatory/behavior/test_behavior_session.py index 8e25c9e85..97ccccbf4 100644 --- a/allensdk/test/brain_observatory/behavior/test_behavior_session.py +++ b/allensdk/test/brain_observatory/behavior/test_behavior_session.py @@ -88,7 +88,7 @@ def dummy_init(self): 'running_speed', 'stimulus_presentations', 'stimulus_templates', - 'stimulus_fingerprint_movie_template', + 'stimulus_natural_movie_template', 'stimulus_timestamps', 'task_parameters', 'trials',