Ticket/psb-171: Various fixes for VBO release

allensdk/brain_observatory/behavior/behavior_ophys_experiment.py

@@ -611,13 +611,17 @@ def cell_specimen_table(self) -> pd.DataFrame:
  which image plane an ROI resides on. Overlapping
  ROIs are stored on different mask image planes
  max_corretion_down: (float)
- max motion correction in down direction in pixels
+ max motion correction in down direction in pixels. Defines
+ the motion border at the top of the image.
  max_correction_left: (float)
- max motion correction in left direction in pixels
+ max motion correction in left direction in pixels. Defines
+ the motion border at the right of the image.
  max_correction_right: (float)
- max motion correction in right direction in pixels
+ max motion correction in right direction in pixels. Defines
+ the motion border at the left of the image.
  max_correction_up: (float)
- max motion correction in up direction in pixels
+ max motion correction in up direction in pixels. Defines
+ the motion border at the bottom of the image.
  roi_mask: (array of bool)
  an image array that displays the location of the
  roi mask in the field of view

allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_cloud_api.py

@@ -163,7 +163,7 @@ def _get_ophys_session_table(self):
  df["date_of_acquisition"] = pd.to_datetime(
  df["date_of_acquisition"], utc="True"
  )
- df = enforce_df_int_typing(df, INTEGER_COLUMNS)
+ df = enforce_df_int_typing(df, INTEGER_COLUMNS, True)
  self._ophys_session_table = df.set_index("ophys_session_id")
 
  def get_ophys_session_table(self) -> pd.DataFrame:
@@ -188,7 +188,7 @@ def _get_behavior_session_table(self):
  df["date_of_acquisition"] = pd.to_datetime(
  df["date_of_acquisition"], utc="True"
  )
- df = enforce_df_int_typing(df, INTEGER_COLUMNS)
+ df = enforce_df_int_typing(df, INTEGER_COLUMNS, True)
 
  self._behavior_session_table = df.set_index("behavior_session_id")
 
@@ -218,7 +218,7 @@ def _get_ophys_experiment_table(self):
  df["date_of_acquisition"] = pd.to_datetime(
  df["date_of_acquisition"], utc="True"
  )
- df = enforce_df_int_typing(df, INTEGER_COLUMNS)
+ df = enforce_df_int_typing(df, INTEGER_COLUMNS, True)
  self._ophys_experiment_table = df.set_index("ophys_experiment_id")
 
  def _get_ophys_cells_table(self):

allensdk/brain_observatory/behavior/behavior_session.py

@@ -1149,7 +1149,7 @@ def stimulus_templates(self) -> Optional[pd.DataFrame]:
  return None
 
  @property
- def stimulus_fingerprint_movie_template(self) -> Optional[pd.DataFrame]:
+ def stimulus_natural_movie_template(self) -> Optional[pd.DataFrame]:
  """Get stimulus templates movie for the behavior session.
 
  Returns None if no stimulus movie is available.

allensdk/brain_observatory/behavior/data_objects/stimuli/fingerprint_stimulus.py

@@ -85,7 +85,7 @@ def from_stimulus_file(
  'stop_time': stop_time,
  'start_frame': start_frame,
  'end_frame': end_frame,
- 'repeat': repeat,
+ 'movie_repeat': repeat,
  'duration': stop_time - start_time
  })
  table = pd.DataFrame(res)

allensdk/brain_observatory/behavior/data_objects/stimuli/presentations.py

@@ -92,7 +92,7 @@ def __init__(
  "is_change",
  "is_image_novel",
  "omitted",
- "repeat",
+ "movie_repeat",
  "flashes_since_change",
  "trials_id",
  ],
@@ -104,7 +104,7 @@ def __init__(
  "flashes_since_change",
  "image_index",
  "movie_frame_index",
- "repeat",
+ "movie_repeat",
  "stimulus_index",
  ],
  )
@@ -290,6 +290,8 @@ def from_stimulus_file(
  data, stimulus_timestamps.value
  )
  raw_stim_pres_df = raw_stim_pres_df.drop(columns=["index"])
+ raw_stim_pres_df = cls._check_for_errant_omitted_stimulus(
+ input_df=raw_stim_pres_df)
 
  # Fill in nulls for image_name
  # This makes two assumptions:
@@ -540,17 +542,37 @@ def _postprocess(
  )
  }
  )
- # Check if the first entry in the DataFrame is an omitted stimulus.
- # This shouldn't happen and likely reflects some sort of camstim error
- # with appending frames to the omitted flash frame log. See
- # explanation here:
- # https://github.com/AllenInstitute/AllenSDK/issues/2577
- if "omitted" in df.columns and len(df) > 0:
- first_row = df.iloc[0]
+ df = cls._check_for_errant_omitted_stimulus(input_df=df)
+ return df
+
+ @staticmethod
+ def _check_for_errant_omitted_stimulus(
+ input_df: pd.DataFrame
+ ) -> pd.DataFrame:
+ """Check if the first entry in the DataFrame is an omitted stimulus.
+
+ This shouldn't happen and likely reflects some sort of camstim error
+ with appending frames to the omitted flash frame log. See
+ explanation here:
+ https://github.com/AllenInstitute/AllenSDK/issues/2577
+
+ Parameters
+ ----------/
+ input_df : DataFrame
+ Input stimulus table to check for "omitted" stimulus.
+
+ Returns
+ -------
+ modified_df : DataFrame
+ Dataframe with omitted stimulus removed from first row or if not
+ found, return input_df unmodified.
+ """
+ if "omitted" in input_df.columns and len(input_df) > 0:
+ first_row = input_df.iloc[0]
  if not pd.isna(first_row["omitted"]):
  if first_row["omitted"]:
- df = df.drop(first_row.name, axis=0)
- return df
+ input_df = input_df.drop(first_row.name, axis=0)
+ return input_df
 
  @staticmethod
  def _fill_missing_values_for_omitted_flashes(

allensdk/core/dataframe_utils.py

@@ -126,7 +126,8 @@ def enforce_df_column_order(
 
 
 def enforce_df_int_typing(input_df: pd.DataFrame,
- int_columns: List[str]) -> pd.DataFrame:
+ int_columns: List[str],
+ use_pandas_type=False) -> pd.DataFrame:
  """Enforce integer typing for columns that may have lost int typing when
  combined into the final DataFrame.
 
@@ -138,6 +139,10 @@ def enforce_df_int_typing(input_df: pd.DataFrame,
  Columns to enforce int typing and fill any NaN/None values with the
  value set in INT_NULL in this file. Requested columns not in the
  dataframe are ignored.
+ use_pandas_type : bool
+ Instead of filling with the value INT_NULL to enforce integer typing,
+ use the pandas type Int64. This type can have issues converting to
+ numpy/array type values.
 
  Returns
  -------
@@ -147,6 +152,9 @@ def enforce_df_int_typing(input_df: pd.DataFrame,
  """
  for col in int_columns:
  if col in input_df.columns:
- input_df[col] = \
- input_df[col].fillna(INT_NULL).astype(int)
+ if use_pandas_type:
+ input_df[col] = input_df[col].astype('Int64')
+ else:
+ input_df[col] = \
+ input_df[col].fillna(INT_NULL).astype(int)
  return input_df

allensdk/test/brain_observatory/behavior/behavior_project_cache/test_behavior_project_cloud_api.py

@@ -108,7 +108,7 @@ def mock_cache(request, tmpdir):
  "ophys_session_table": pd.DataFrame(
  {
  "ophys_session_id": pd.Series([10, 11, 12, 13],
- dtype='int64'),
+ dtype='Int64'),
  "mouse_id": ["1"] * 4,
  "date_of_acquisition": pd.to_datetime(["2021-01-01"] * 4),
  "ophys_experiment_id": [4, 5, 6, [7, 8, 9]],

allensdk/test/brain_observatory/behavior/test_behavior_ophys_experiment.py

@@ -277,7 +277,7 @@ def test_behavior_ophys_experiment_list_data_attributes_and_methods(
  "segmentation_mask_image",
  "stimulus_presentations",
  "stimulus_templates",
- 'stimulus_fingerprint_movie_template',
+ 'stimulus_natural_movie_template',
  "stimulus_timestamps",
  "task_parameters",
  "trials",

allensdk/test/brain_observatory/behavior/test_behavior_session.py

@@ -88,7 +88,7 @@ def dummy_init(self):
  'running_speed',
  'stimulus_presentations',
  'stimulus_templates',
- 'stimulus_fingerprint_movie_template',
+ 'stimulus_natural_movie_template',
  'stimulus_timestamps',
  'task_parameters',
  'trials',