From 4690626c3da52e52380dc7a4896b7e75a214902a Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Wed, 10 Jan 2024 14:54:59 +0100 Subject: [PATCH 01/41] =?UTF-8?q?integrate=20=C2=96https://github.com/brai?= =?UTF-8?q?n-score/model-tools/pull/75=20to=20brain-score/vision?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../model_helpers/activations/core.py | 245 +++++++++++++++--- .../brain_transformation/neural.py | 35 ++- pyproject.toml | 1 + .../activations/test___init__.py | 54 ++++ 4 files changed, 298 insertions(+), 37 deletions(-) diff --git a/brainscore_vision/model_helpers/activations/core.py b/brainscore_vision/model_helpers/activations/core.py index 3d982b5e2..b7b7ce460 100644 --- a/brainscore_vision/model_helpers/activations/core.py +++ b/brainscore_vision/model_helpers/activations/core.py @@ -1,5 +1,7 @@ import copy import os +import cv2 +import tempfile import functools import logging @@ -8,6 +10,7 @@ import numpy as np from tqdm.auto import tqdm +import xarray as xr from brainio.assemblies import NeuroidAssembly, walk_coords from brainio.stimuli import StimulusSet @@ -33,16 +36,41 @@ def __init__(self, get_activations, preprocessing, identifier=False, batch_size= self._stimulus_set_hooks = {} self._batch_activations_hooks = {} - def __call__(self, stimuli, layers, stimuli_identifier=None): + def __call__(self, stimuli, layers, stimuli_identifier=None, model_requirements=None): """ :param stimuli_identifier: a stimuli identifier for the stored results file. False to disable saving. + :param model_requirements: a dictionary containing any requirements a benchmark might have for models, e.g. + microsaccades for getting variable responses from non-stochastic models to the same stimuli. + + model_requirements['microsaccades']: list of tuples of x and y shifts to apply to each image to model + microsaccades. Note that the shifts happen in pixel space of the original input image, not the preprocessed + image. + Human microsaccade amplitude varies by who you ask, an estimate might be <0.1 deg = 360 arcsec = 6arcmin. + The goal of microsaccades is to obtain multiple different neural activities to the same input stimulus + from non-stochastic models. This is to improve estimates of e.g. psychophysical functions, but also other + things. Note that microsaccades are also applied to stochastic models to make them comparable within- + benchmark to non-stochastic models. + Example usage: + model_requirements = {'microsaccades': [(0, 0), (0, 1), (1, 0), (1, 1)]} + More information: + --> Rolfs 2009 "Microsaccades: Small steps on a long way" Vision Research, Volume 49, Issue 20, 15 + October 2009, Pages 2415-2441. + --> Haddad & Steinmann 1973 "The smallest voluntary saccade: Implications for fixation" Vision + Research Volume 13, Issue 6, June 1973, Pages 1075-1086, IN5-IN6. + Huge thanks to Johannes Mehrer for the implementation of microsaccades in the Brain-Score core.py and + neural.py files. + """ if isinstance(stimuli, StimulusSet): - return self.from_stimulus_set(stimulus_set=stimuli, layers=layers, stimuli_identifier=stimuli_identifier) + return self.from_stimulus_set(stimulus_set=stimuli, layers=layers, stimuli_identifier=stimuli_identifier, + model_requirements=model_requirements) else: - return self.from_paths(stimuli_paths=stimuli, layers=layers, stimuli_identifier=stimuli_identifier) + return self.from_paths(stimuli_paths=stimuli, + layers=layers, + stimuli_identifier=stimuli_identifier, + model_requirements=model_requirements) - def from_stimulus_set(self, stimulus_set, layers, stimuli_identifier=None): + def from_stimulus_set(self, stimulus_set, layers, stimuli_identifier=None, model_requirements=None): """ :param stimuli_identifier: a stimuli identifier for the stored results file. False to disable saving. None to use `stimulus_set.identifier` @@ -53,15 +81,17 @@ def from_stimulus_set(self, stimulus_set, layers, stimuli_identifier=None): stimulus_set = hook(stimulus_set) stimuli_paths = [str(stimulus_set.get_stimulus(stimulus_id)) for stimulus_id in stimulus_set['stimulus_id']] activations = self.from_paths(stimuli_paths=stimuli_paths, layers=layers, stimuli_identifier=stimuli_identifier) - activations = attach_stimulus_set_meta(activations, stimulus_set) + activations = attach_stimulus_set_meta(activations, stimulus_set, model_requirements=model_requirements) return activations - def from_paths(self, stimuli_paths, layers, stimuli_identifier=None): + def from_paths(self, stimuli_paths, layers, stimuli_identifier=None, model_requirements=None): if layers is None: layers = ['logits'] if self.identifier and stimuli_identifier: fnc = functools.partial(self._from_paths_stored, - identifier=self.identifier, stimuli_identifier=stimuli_identifier) + identifier=self.identifier, + stimuli_identifier=stimuli_identifier, + model_requirements=model_requirements) else: self._logger.debug(f"self.identifier `{self.identifier}` or stimuli_identifier {stimuli_identifier} " f"are not set, will not store") @@ -69,22 +99,26 @@ def from_paths(self, stimuli_paths, layers, stimuli_identifier=None): # In case stimuli paths are duplicates (e.g. multiple trials), we first reduce them to only the paths that need # to be run individually, compute activations for those, and then expand the activations to all paths again. # This is done here, before storing, so that we only store the reduced activations. - reduced_paths = self._reduce_paths(stimuli_paths) - activations = fnc(layers=layers, stimuli_paths=reduced_paths) - activations = self._expand_paths(activations, original_paths=stimuli_paths) + if model_requirements is not None and 'microsaccades' in model_requirements.keys(): + activations = fnc(layers=layers, stimuli_paths=stimuli_paths, model_requirements=model_requirements) + else: + reduced_paths = self._reduce_paths(stimuli_paths) + activations = fnc(layers=layers, stimuli_paths=reduced_paths, model_requirements=model_requirements) + activations = self._expand_paths(activations, original_paths=stimuli_paths) return activations @store_xarray(identifier_ignore=['stimuli_paths', 'layers'], combine_fields={'layers': 'layer'}) - def _from_paths_stored(self, identifier, layers, stimuli_identifier, stimuli_paths): + def _from_paths_stored(self, identifier, layers, stimuli_identifier, stimuli_paths, model_requirements): return self._from_paths(layers=layers, stimuli_paths=stimuli_paths) - def _from_paths(self, layers, stimuli_paths): + def _from_paths(self, layers, stimuli_paths, model_requirements=None): if len(layers) == 0: raise ValueError("No layers passed to retrieve activations from") self._logger.info('Running stimuli') - layer_activations = self._get_activations_batched(stimuli_paths, layers=layers, batch_size=self._batch_size) + layer_activations = self._get_activations_with_model_requirements(stimuli_paths, layers=layers, + model_requirements=model_requirements) self._logger.info('Packaging into assembly') - return self._package(layer_activations, stimuli_paths) + return self._package(layer_activations, stimuli_paths, model_requirements) def _reduce_paths(self, stimuli_paths): return list(set(stimuli_paths)) @@ -95,7 +129,7 @@ def _expand_paths(self, activations, original_paths): sorted_x = activations_paths[argsort_indices] sorted_index = np.searchsorted(sorted_x, original_paths) index = [argsort_indices[i] for i in sorted_index] - return activations[{'stimulus_path': index}] + return activations[{'presentation': index}] def register_batch_activations_hook(self, hook): r""" @@ -125,6 +159,13 @@ def register_stimulus_set_hook(self, hook): self._stimulus_set_hooks[handle.id] = hook return handle + def _get_activations_with_model_requirements(self, paths, layers, model_requirements): + if model_requirements is not None and 'microsaccades' in model_requirements.keys(): + assert len(model_requirements['microsaccades']) > 0 + return self._get_microsaccade_activations_batched(paths, layers=layers, batch_size=self._batch_size, + shifts=model_requirements['microsaccades']) + return self._get_activations_batched(paths, layers=layers, batch_size=self._batch_size) + def _get_activations_batched(self, paths, layers, batch_size): layer_activations = None for batch_start in tqdm(range(0, len(paths), batch_size), unit_scale=batch_size, desc="activations"): @@ -142,6 +183,42 @@ def _get_activations_batched(self, paths, layers, batch_size): return layer_activations + def _get_microsaccade_activations_batched(self, paths, layers, batch_size, shifts): + """ + :param shifts: a list of tuples containing the pixel shifts to apply to the paths. + """ + layer_activations = OrderedDict() + for batch_start in tqdm(range(0, len(paths), batch_size), unit_scale=batch_size, desc="activations"): + batch_end = min(batch_start + batch_size, len(paths)) + batch_inputs = paths[batch_start:batch_end] + + batch_activations = OrderedDict() + # compute activations on the entire batch one shift at a time + for shift in shifts: + assert type(shift) == tuple + shifted_batch, num_padding = self.translate_and_preprocess(batch_inputs, shift, batch_size) + activations = self.get_activations(shifted_batch, layers) + activations = self._unpad(activations, num_padding) + for layer_name, layer_output in activations.items(): + batch_activations.setdefault(layer_name, []).append(layer_output) + + # concatenate all shifts into this batch + for layer_name, layer_outputs in batch_activations.items(): + batch_activations[layer_name] = np.concatenate(layer_outputs) + + for hook in self._batch_activations_hooks.copy().values(): + batch_activations = hook(batch_activations) + + # add this batch to layer_activations + for layer_name, layer_output in batch_activations.items(): + layer_activations.setdefault(layer_name, []).append(layer_output) + + # fast concat all batches + for layer_name, layer_outputs in layer_activations.items(): + layer_activations[layer_name] = np.concatenate(layer_outputs) + + return layer_activations # this is all batches + def _get_batch_activations(self, inputs, layer_names, batch_size): inputs, num_padding = self._pad(inputs, batch_size) preprocessed_inputs = self.preprocess(inputs) @@ -150,6 +227,27 @@ def _get_batch_activations(self, inputs, layer_names, batch_size): activations = self._unpad(activations, num_padding) return activations + def translate_and_preprocess(self, images, shift, batch_size): + assert type(images) == list + temp_file_paths = [] + for image_path in images: + fp = self.translate_image(image_path, shift) + temp_file_paths.append(fp) + # having to pad here causes a considerable slowdown for small datasets (n_images << batch_size), + # but is necessary to accommodate tensorflow models + padded_images, num_padding = self._pad(temp_file_paths, batch_size) + preprocessed_images = self.preprocess(padded_images) + for temp_file_path in temp_file_paths: + os.remove(temp_file_path) + return preprocessed_images, num_padding + + def translate_image(self, image_path: str, shift: np.array) -> str: + """Translates and saves a temporary image to temporary_fp.""" + translated_image = self.translate(cv2.imread(image_path), shift) + temporary_fp = tempfile.mkstemp(suffix=".png")[1] + cv2.imwrite(temporary_fp, translated_image) + return temporary_fp + def _pad(self, batch_images, batch_size): num_images = len(batch_images) if num_images % batch_size == 0: @@ -161,11 +259,12 @@ def _pad(self, batch_images, batch_size): def _unpad(self, layer_activations, num_padding): return change_dict(layer_activations, lambda values: values[:-num_padding or None]) - def _package(self, layer_activations, stimuli_paths): + def _package(self, layer_activations, stimuli_paths, model_requirements): shapes = [a.shape for a in layer_activations.values()] self._logger.debug(f"Activations shapes: {shapes}") self._logger.debug("Packaging individual layers") - layer_assemblies = [self._package_layer(single_layer_activations, layer=layer, stimuli_paths=stimuli_paths) for + layer_assemblies = [self._package_layer(single_layer_activations, layer=layer, + stimuli_paths=stimuli_paths, model_requirements=model_requirements) for layer, single_layer_activations in tqdm(layer_activations.items(), desc='layer packaging')] # merge manually instead of using merge_data_arrays since `xarray.merge` is very slow with these large arrays # complication: (non)neuroid_coords are taken from the structure of layer_assemblies[0] i.e. the 1st assembly; @@ -182,17 +281,24 @@ def _package(self, layer_activations, stimuli_paths): for coord in neuroid_coords: neuroid_coords[coord][1] = np.concatenate((neuroid_coords[coord][1], layer_assembly[coord].values)) assert layer_assemblies[0].dims == layer_assembly.dims - for dim in set(layer_assembly.dims) - {'neuroid'}: - for coord in layer_assembly[dim].coords: - assert (layer_assembly[coord].values == nonneuroid_coords[coord][1]).all() + for coord, dims, values in walk_coords(layer_assembly): + if set(dims) == {'neuroid'}: + continue + assert (values == nonneuroid_coords[coord][1]).all() + neuroid_coords = {coord: (dims_values[0], dims_values[1]) # re-package as tuple instead of list for xarray for coord, dims_values in neuroid_coords.items()} model_assembly = type(layer_assemblies[0])(model_assembly, coords={**nonneuroid_coords, **neuroid_coords}, dims=layer_assemblies[0].dims) return model_assembly - def _package_layer(self, layer_activations, layer, stimuli_paths): - assert layer_activations.shape[0] == len(stimuli_paths) + def _package_layer(self, layer_activations, layer, stimuli_paths, model_requirements): + if model_requirements is not None and 'microsaccades' in model_requirements.keys(): + runs_per_image = len(model_requirements['microsaccades']) + else: + runs_per_image = 1 + assert layer_activations.shape[0] == len(stimuli_paths) * runs_per_image + stimuli_paths = np.repeat(stimuli_paths, runs_per_image) activations, flatten_indices = flatten(layer_activations, return_index=True) # collapse for single neuroid dim flatten_coord_names = None if flatten_indices.shape[1] == 1: # fully connected, e.g. classifier @@ -209,28 +315,54 @@ def _package_layer(self, layer_activations, layer, stimuli_paths): self._logger.debug(f"Unknown layer activations shape {layer_activations.shape}, not inferring channels") # build assembly - coords = {'stimulus_path': stimuli_paths, - 'neuroid_num': ('neuroid', list(range(activations.shape[1]))), - 'model': ('neuroid', [self.identifier] * activations.shape[1]), - 'layer': ('neuroid', [layer] * activations.shape[1]), - } + if model_requirements is not None and 'microsaccades' in model_requirements.keys(): + coords = self.build_microsaccade_coords(activations, layer, stimuli_paths, model_requirements) + else: + coords = {'stimulus_path': ('presentation', stimuli_paths), + 'stimulus_path2': ('presentation', stimuli_paths), # to avoid DataAssembly dim collapse + 'neuroid_num': ('neuroid', list(range(activations.shape[1]))), + 'model': ('neuroid', [self.identifier] * activations.shape[1]), + 'layer': ('neuroid', [layer] * activations.shape[1]), + } + if flatten_coord_names: flatten_coords = {flatten_coord_names[i]: [sample_index[i] if i < flatten_indices.shape[1] else np.nan for sample_index in flatten_indices] for i in range(len(flatten_coord_names))} coords = {**coords, **{coord: ('neuroid', values) for coord, values in flatten_coords.items()}} - layer_assembly = NeuroidAssembly(activations, coords=coords, dims=['stimulus_path', 'neuroid']) + layer_assembly = NeuroidAssembly(activations, coords=coords, dims=['presentation', 'neuroid']) neuroid_id = [".".join([f"{value}" for value in values]) for values in zip(*[ layer_assembly[coord].values for coord in ['model', 'layer', 'neuroid_num']])] layer_assembly['neuroid_id'] = 'neuroid', neuroid_id return layer_assembly + def build_microsaccade_coords(self, activations, layer, stimuli_paths, model_requirements): + coords = { + 'stimulus_path': ('presentation', stimuli_paths), + 'shift_x': ('presentation', [shift[0] for shift in model_requirements['microsaccades']]), + 'shift_y': ('presentation', [shift[1] for shift in model_requirements['microsaccades']]), + 'neuroid_num': ('neuroid', list(range(activations.shape[1]))), + 'model': ('neuroid', [self.identifier] * activations.shape[1]), + 'layer': ('neuroid', [layer] * activations.shape[1]), + } + return coords + def insert_attrs(self, wrapper): wrapper.from_stimulus_set = self.from_stimulus_set wrapper.from_paths = self.from_paths wrapper.register_batch_activations_hook = self.register_batch_activations_hook wrapper.register_stimulus_set_hook = self.register_stimulus_set_hook + @staticmethod + def translate(image, shift): + rows, cols, _ = image.shape + # translation matrix + M = np.float32([[1, 0, shift[0]], [0, 1, shift[1]]]) + + # Apply translation, filling new line(s) with line(s) closest to it(them). + translated_image = cv2.warpAffine(image, M, (cols, rows), borderMode=cv2.BORDER_REPLICATE) + return translated_image + def change_dict(d, change_function, keep_name=False, multithread=False): if not multithread: @@ -261,19 +393,68 @@ def lstrip_local(path): return path -def attach_stimulus_set_meta(assembly, stimulus_set): +def attach_stimulus_set_meta(assembly, stimulus_set, model_requirements): + if model_requirements is not None and 'microsaccades' in model_requirements.keys(): + return attach_stimulus_set_meta_with_microsaccades(assembly, stimulus_set, model_requirements) stimulus_paths = [str(stimulus_set.get_stimulus(stimulus_id)) for stimulus_id in stimulus_set['stimulus_id']] stimulus_paths = [lstrip_local(path) for path in stimulus_paths] assembly_paths = [lstrip_local(path) for path in assembly['stimulus_path'].values] + assert (np.array(assembly_paths) == np.array(stimulus_paths)).all() - assembly['stimulus_path'] = stimulus_set['stimulus_id'].values + assembly = assembly.reset_index('presentation') + assembly.assign_coords(stimulus_path=('presentation', stimulus_set['stimulus_id'].values)) + assembly = assembly.rename({'stimulus_path': 'stimulus_id'}) + + all_columns = [] for column in stimulus_set.columns: - assembly[column] = 'stimulus_id', stimulus_set[column].values - assembly = assembly.stack(presentation=('stimulus_id',)) + assembly = assembly.assign_coords({column: ('presentation', stimulus_set[column].values)}) + all_columns.append(column) + if 'stimulus_id' in all_columns: + all_columns.remove('stimulus_id') + if 'stimulus_path2' in all_columns: + all_columns.remove('stimulus_path2') + assembly = assembly.set_index(presentation=['stimulus_id', 'stimulus_path2'] + all_columns) return assembly +def attach_stimulus_set_meta_with_microsaccades(assembly, stimulus_set, model_requirements): + stimulus_paths = [str(stimulus_set.get_stimulus(stimulus_id)) for stimulus_id in stimulus_set['stimulus_id']] + stimulus_paths = [lstrip_local(path) for path in stimulus_paths] + assembly_paths = [lstrip_local(path) for path in assembly['stimulus_path'].values] + + replication_factor = len(model_requirements['microsaccades']) + repeated_stimulus_paths = np.repeat(stimulus_paths, replication_factor) + assert (np.array(assembly_paths) == np.array(repeated_stimulus_paths)).all() + repeated_stimulus_ids = np.repeat(stimulus_set['stimulus_id'].values, replication_factor) + + # repeat over the presentation dimension to accommodate multiple runs per stimulus + repeated_assembly = xr.concat([assembly for _ in range(replication_factor)], dim='presentation') + repeated_assembly = repeated_assembly.reset_index('presentation') + repeated_assembly.assign_coords(stimulus_path=('presentation', repeated_stimulus_ids)) + repeated_assembly = repeated_assembly.rename({'stimulus_path': 'stimulus_id'}) + + # hack to capture columns + all_columns = [] + for column in stimulus_set.columns: + repeated_values = np.repeat(stimulus_set[column].values, replication_factor) + repeated_assembly = repeated_assembly.assign_coords({column: ('presentation', repeated_values)}) + all_columns.append(column) + if 'stimulus_id' in all_columns: + all_columns.remove('stimulus_id') + if 'stimulus_path2' in all_columns: + all_columns.remove('stimulus_path2') + + repeated_assembly.coords['shift_x'] = ('presentation', [shift[0] for shift in model_requirements['microsaccades']]) + repeated_assembly.coords['shift_y'] = ('presentation', [shift[1] for shift in model_requirements['microsaccades']]) + + # Set MultiIndex + index = ['stimulus_id', 'stimulus_path2'] + all_columns + ['shift_x', 'shift_y'] + repeated_assembly = repeated_assembly.set_index(presentation=index) + + return repeated_assembly + + class HookHandle: next_id = 0 diff --git a/brainscore_vision/model_helpers/brain_transformation/neural.py b/brainscore_vision/model_helpers/brain_transformation/neural.py index 51cff1eb3..07a6ba700 100644 --- a/brainscore_vision/model_helpers/brain_transformation/neural.py +++ b/brainscore_vision/model_helpers/brain_transformation/neural.py @@ -2,6 +2,7 @@ from result_caching import store_xarray, store from tqdm import tqdm +from typing import Optional, Dict, List from brainscore_vision.metrics import Score from brainscore_vision.model_helpers.activations.pca import LayerPCA @@ -23,7 +24,29 @@ def __init__(self, identifier, activations_model, region_layer_map, visual_degre def identifier(self): return self._identifier - def look_at(self, stimuli, number_of_trials=1): + def look_at(self, stimuli, number_of_trials=1, model_requirements: Optional[Dict[str, List]] = None): + """ + :param model_requirements: a dictionary containing any requirements a benchmark might have for models, e.g. + microsaccades for getting variable responses from non-stochastic models to the same stimuli. + + model_requirements['microsaccades']: list of tuples of x and y shifts to apply to each image to model + microsaccades. Note that the shifts happen in pixel space of the original input image, not the preprocessed + image. + Human microsaccade amplitude varies by who you ask, an estimate might be <0.1 deg = 360 arcsec = 6arcmin. + The goal of microsaccades is to obtain multiple different neural activities to the same input stimulus + from non-stochastic models. This is to improve estimates of e.g. psychophysical functions, but also other + things. Note that microsaccades are also applied to stochastic models to make them comparable within- + benchmark to non-stochastic models. + Example usage: + model_requirements = {'microsaccades': [(0, 0), (0, 1), (1, 0), (1, 1)]} + More information: + --> Rolfs 2009 "Microsaccades: Small steps on a long way" Vision Research, Volume 49, Issue 20, 15 + October 2009, Pages 2415-2441. + --> Haddad & Steinmann 1973 "The smallest voluntary saccade: Implications for fixation" Vision + Research Volume 13, Issue 6, June 1973, Pages 1075-1086, IN5-IN6. + Huge thanks to Johannes Mehrer for the implementation of microsaccades in the Brain-Score core.py and + neural.py files. + """ layer_regions = {} for region in self.recorded_regions: layers = self.region_layer_map[region] @@ -31,13 +54,15 @@ def look_at(self, stimuli, number_of_trials=1): for layer in layers: assert layer not in layer_regions, f"layer {layer} has already been assigned for {layer_regions[layer]}" layer_regions[layer] = region - activations = self.run_activations( - stimuli, layers=list(layer_regions.keys()), number_of_trials=number_of_trials) + activations = self.run_activations(stimuli, + layers=list(layer_regions.keys()), + number_of_trials=number_of_trials, + model_requirements=model_requirements) activations['region'] = 'neuroid', [layer_regions[layer] for layer in activations['layer'].values] return activations - def run_activations(self, stimuli, layers, number_of_trials=1): - activations = self.activations_model(stimuli, layers=layers) + def run_activations(self, stimuli, layers, number_of_trials=1, model_requirements=None): + activations = self.activations_model(stimuli, layers=layers, model_requirements=model_requirements) return activations def start_task(self, task): diff --git a/pyproject.toml b/pyproject.toml index 5d26fe82e..8d148a9fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ dependencies = [ "importlib-metadata<5", # workaround to https://github.com/brain-score/brainio/issues/28 "scikit-learn", # for metric_helpers/transformations.py cross-validation "scipy", # for benchmark_helpers/properties_common.py + "opencv-python", # for microsaccades "h5py", "tqdm", "gitpython", diff --git a/tests/test_model_helpers/activations/test___init__.py b/tests/test_model_helpers/activations/test___init__.py index 86a1ee788..160a6852b 100644 --- a/tests/test_model_helpers/activations/test___init__.py +++ b/tests/test_model_helpers/activations/test___init__.py @@ -189,6 +189,60 @@ def test_from_image_path(model_ctr, layers, image_name, pca_components, logits): return activations +@pytest.mark.parametrize("image_name", ['rgb.jpg', 'grayscale.png', 'grayscale2.jpg', 'grayscale_alpha.png', + 'palletized.png']) +@pytest.mark.parametrize(["model_ctr", "layers"], models_layers) +@pytest.mark.parametrize("shifts", [[(2, 2), (0, 0), (1, -1), (0, 1)], [(0, 1), (2, 3)]]) +def test_microsaccades_from_image_path(model_ctr, layers, image_name, shifts): + stimulus_paths = [os.path.join(os.path.dirname(__file__), image_name)] + activations_extractor = model_ctr() + + model_requirements = {'microsaccades': shifts} + activations = activations_extractor(stimuli=stimulus_paths, layers=layers, model_requirements=model_requirements) + + assert activations is not None + assert list(activations['shift_x'].values) == [shift[0] for shift in shifts] + assert list(activations['shift_y'].values) == [shift[1] for shift in shifts] + assert len(activations['shift_x']) == len(shifts) * len(stimulus_paths) + assert len(activations['shift_y']) == len(shifts) * len(stimulus_paths) + + +@pytest.mark.parametrize("image_name", ['rgb.jpg', 'grayscale.png', 'grayscale2.jpg', 'grayscale_alpha.png', + 'palletized.png']) +@pytest.mark.parametrize(["model_ctr", "layers"], models_layers) +@pytest.mark.parametrize("model_requirements", [None, {'microsaccades': [(0, 0), (1, -1)]}]) +def test_model_requirements(model_ctr, layers, image_name, model_requirements): + stimulus_paths = [os.path.join(os.path.dirname(__file__), image_name)] + activations_extractor = model_ctr() + + activations_with_req = activations_extractor(stimuli=stimulus_paths, layers=layers, + model_requirements=model_requirements) + activations_without_req = activations_extractor(stimuli=stimulus_paths, layers=layers, + model_requirements=None) + + assert activations_with_req is not None + assert activations_without_req is not None + if model_requirements: + assert len(activations_with_req['presentation']) > len(activations_without_req['presentation']) + assert len(activations_with_req['neuroid']) == len(activations_without_req['neuroid']) + + +@pytest.mark.parametrize("image_name", ['rgb.jpg', 'grayscale.png', 'grayscale2.jpg', 'grayscale_alpha.png', + 'palletized.png']) +@pytest.mark.parametrize(["model_ctr", "layers"], models_layers) +def test_temporary_file_handling(model_ctr, layers, image_name): + import tempfile + stimulus_paths = [os.path.join(os.path.dirname(__file__), image_name)] + activations_extractor = model_ctr() + model_requirements = {'microsaccades': [(2, 2)]} + + activations = activations_extractor(stimuli=stimulus_paths, layers=layers, model_requirements=model_requirements) + temp_files = [f for f in os.listdir(tempfile.gettempdir()) if f.startswith('temp') and f.endswith('.png')] + + assert activations is not None + assert len(temp_files) == 0 + + def _build_stimulus_set(image_names): stimulus_set = StimulusSet([{'stimulus_id': image_name, 'some_meta': image_name[::-1]} for image_name in image_names]) From 1fee011d8b88b7f5829c683bce8fdee9c43c327d Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Mon, 15 Jan 2024 09:18:38 +0100 Subject: [PATCH 02/41] retrigger From 26313aa67f83a666c946e709cca78dd7a20a5afb Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Tue, 16 Jan 2024 09:57:07 +0100 Subject: [PATCH 03/41] retrigger From 71ba207296917fd7d783cecca479c5a1f0709ba2 Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Wed, 17 Jan 2024 15:54:20 +0100 Subject: [PATCH 04/41] add exception for when temp file write fails --- brainscore_vision/model_helpers/activations/core.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/brainscore_vision/model_helpers/activations/core.py b/brainscore_vision/model_helpers/activations/core.py index b7b7ce460..1a21f6404 100644 --- a/brainscore_vision/model_helpers/activations/core.py +++ b/brainscore_vision/model_helpers/activations/core.py @@ -238,14 +238,17 @@ def translate_and_preprocess(self, images, shift, batch_size): padded_images, num_padding = self._pad(temp_file_paths, batch_size) preprocessed_images = self.preprocess(padded_images) for temp_file_path in temp_file_paths: - os.remove(temp_file_path) + # os.remove(temp_file_path) + pass return preprocessed_images, num_padding def translate_image(self, image_path: str, shift: np.array) -> str: """Translates and saves a temporary image to temporary_fp.""" translated_image = self.translate(cv2.imread(image_path), shift) - temporary_fp = tempfile.mkstemp(suffix=".png")[1] - cv2.imwrite(temporary_fp, translated_image) + temp_file_descriptor, temporary_fp = tempfile.mkstemp(suffix=".png") + os.close(temp_file_descriptor) + if not cv2.imwrite(temporary_fp, translated_image): + raise Exception(f"cv2.imwrite failed: {temporary_fp}") return temporary_fp def _pad(self, batch_images, batch_size): From 3ea9a9c79270359fb973ef5cd662411b2bf002ec Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Wed, 17 Jan 2024 23:09:12 +0100 Subject: [PATCH 05/41] fix error with tf temp file management --- .../model_helpers/activations/core.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/brainscore_vision/model_helpers/activations/core.py b/brainscore_vision/model_helpers/activations/core.py index 1a21f6404..7742eda20 100644 --- a/brainscore_vision/model_helpers/activations/core.py +++ b/brainscore_vision/model_helpers/activations/core.py @@ -196,9 +196,16 @@ def _get_microsaccade_activations_batched(self, paths, layers, batch_size, shift # compute activations on the entire batch one shift at a time for shift in shifts: assert type(shift) == tuple - shifted_batch, num_padding = self.translate_and_preprocess(batch_inputs, shift, batch_size) + shifted_batch, num_padding, temp_file_paths = self.translate_and_preprocess(batch_inputs, + shift, + batch_size) activations = self.get_activations(shifted_batch, layers) activations = self._unpad(activations, num_padding) + for temp_file_path in temp_file_paths: + try: + os.remove(temp_file_path) + except FileNotFoundError: + pass for layer_name, layer_output in activations.items(): batch_activations.setdefault(layer_name, []).append(layer_output) @@ -237,10 +244,7 @@ def translate_and_preprocess(self, images, shift, batch_size): # but is necessary to accommodate tensorflow models padded_images, num_padding = self._pad(temp_file_paths, batch_size) preprocessed_images = self.preprocess(padded_images) - for temp_file_path in temp_file_paths: - # os.remove(temp_file_path) - pass - return preprocessed_images, num_padding + return preprocessed_images, num_padding, temp_file_paths def translate_image(self, image_path: str, shift: np.array) -> str: """Translates and saves a temporary image to temporary_fp.""" From 3203e7bb178493891365b916239f9b5ae72a988f Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Thu, 18 Jan 2024 13:15:31 +0100 Subject: [PATCH 06/41] add bandaid to a DataAssembly index problem --- tests/test_model_helpers/activations/test___init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_model_helpers/activations/test___init__.py b/tests/test_model_helpers/activations/test___init__.py index 160a6852b..835446410 100644 --- a/tests/test_model_helpers/activations/test___init__.py +++ b/tests/test_model_helpers/activations/test___init__.py @@ -277,6 +277,13 @@ def test_exact_activations(pca_components): image_name='rgb.jpg', pca_components=pca_components, logits=False) path_to_expected = Path(__file__).parent / f'alexnet-rgb-{pca_components}.nc' expected = xr.load_dataarray(path_to_expected) + + # TODO: this is probably bad. For now, this is required since we decided to change the `stimulus_path` index to the + # `presentation` index for all assemblies as a part of the PR. Should not cause issues for models that are run + # after the PR, but loading any extant .nc files could result in issues. Is there a canonical way to get around + # this? + expected = expected.rename({'stimulus_path': 'presentation'}) + assert (activations == expected).all() From a2e0a1577c750a9e0c14c0033d080b4bed42ab35 Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Wed, 24 Jan 2024 09:32:16 +0100 Subject: [PATCH 07/41] retrigger From c1abd95e8a9049eab65bb758b1b3624edc73226b Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Tue, 6 Feb 2024 14:04:54 +0100 Subject: [PATCH 08/41] move microsaccades to the model side --- .../model_helpers/activations/core.py | 145 ++++++++++-------- .../brain_transformation/neural.py | 22 ++- .../activations/test___init__.py | 36 ++--- 3 files changed, 116 insertions(+), 87 deletions(-) diff --git a/brainscore_vision/model_helpers/activations/core.py b/brainscore_vision/model_helpers/activations/core.py index 7742eda20..e20094a4a 100644 --- a/brainscore_vision/model_helpers/activations/core.py +++ b/brainscore_vision/model_helpers/activations/core.py @@ -2,6 +2,7 @@ import os import cv2 import tempfile +from typing import List, Tuple import functools import logging @@ -33,16 +34,18 @@ def __init__(self, get_activations, preprocessing, identifier=False, batch_size= self.identifier = identifier self.get_activations = get_activations self.preprocess = preprocessing or (lambda x: x) + self.shifts = None # for use with microsaccades self._stimulus_set_hooks = {} self._batch_activations_hooks = {} - def __call__(self, stimuli, layers, stimuli_identifier=None, model_requirements=None): + def __call__(self, stimuli, layers, stimuli_identifier=None, number_of_trials=1, require_variance=None): """ + TODO: number_of_trials needs to traverse down from here :param stimuli_identifier: a stimuli identifier for the stored results file. False to disable saving. - :param model_requirements: a dictionary containing any requirements a benchmark might have for models, e.g. + :param require_variance: a dictionary containing any requirements a benchmark might have for models, e.g. microsaccades for getting variable responses from non-stochastic models to the same stimuli. - model_requirements['microsaccades']: list of tuples of x and y shifts to apply to each image to model + require_variance['microsaccades']: list of tuples of x and y shifts to apply to each image to model microsaccades. Note that the shifts happen in pixel space of the original input image, not the preprocessed image. Human microsaccade amplitude varies by who you ask, an estimate might be <0.1 deg = 360 arcsec = 6arcmin. @@ -51,7 +54,7 @@ def __call__(self, stimuli, layers, stimuli_identifier=None, model_requirements= things. Note that microsaccades are also applied to stochastic models to make them comparable within- benchmark to non-stochastic models. Example usage: - model_requirements = {'microsaccades': [(0, 0), (0, 1), (1, 0), (1, 1)]} + require_variance = {'microsaccades': [(0, 0), (0, 1), (1, 0), (1, 1)]} More information: --> Rolfs 2009 "Microsaccades: Small steps on a long way" Vision Research, Volume 49, Issue 20, 15 October 2009, Pages 2415-2441. @@ -63,14 +66,16 @@ def __call__(self, stimuli, layers, stimuli_identifier=None, model_requirements= """ if isinstance(stimuli, StimulusSet): return self.from_stimulus_set(stimulus_set=stimuli, layers=layers, stimuli_identifier=stimuli_identifier, - model_requirements=model_requirements) + number_of_trials=number_of_trials, require_variance=require_variance) else: return self.from_paths(stimuli_paths=stimuli, layers=layers, stimuli_identifier=stimuli_identifier, - model_requirements=model_requirements) + number_of_trials=number_of_trials, + require_variance=require_variance) - def from_stimulus_set(self, stimulus_set, layers, stimuli_identifier=None, model_requirements=None): + def from_stimulus_set(self, stimulus_set, layers, stimuli_identifier=None, number_of_trials=1, + require_variance=None): """ :param stimuli_identifier: a stimuli identifier for the stored results file. False to disable saving. None to use `stimulus_set.identifier` @@ -81,44 +86,58 @@ def from_stimulus_set(self, stimulus_set, layers, stimuli_identifier=None, model stimulus_set = hook(stimulus_set) stimuli_paths = [str(stimulus_set.get_stimulus(stimulus_id)) for stimulus_id in stimulus_set['stimulus_id']] activations = self.from_paths(stimuli_paths=stimuli_paths, layers=layers, stimuli_identifier=stimuli_identifier) - activations = attach_stimulus_set_meta(activations, stimulus_set, model_requirements=model_requirements) + if require_variance: + self.shifts = self.select_microsaccades(number_of_trials=number_of_trials) + activations = attach_stimulus_set_meta_with_microsaccades(activations, + stimulus_set, + number_of_trials=number_of_trials, + shifts=self.shifts) + activations = attach_stimulus_set_meta(activations, stimulus_set) return activations - def from_paths(self, stimuli_paths, layers, stimuli_identifier=None, model_requirements=None): + def from_paths(self, stimuli_paths, layers, stimuli_identifier=None, number_of_trials=1, require_variance=None): if layers is None: layers = ['logits'] if self.identifier and stimuli_identifier: fnc = functools.partial(self._from_paths_stored, identifier=self.identifier, stimuli_identifier=stimuli_identifier, - model_requirements=model_requirements) + require_variance=require_variance) else: self._logger.debug(f"self.identifier `{self.identifier}` or stimuli_identifier {stimuli_identifier} " f"are not set, will not store") fnc = self._from_paths + if require_variance: + activations = fnc(layers=layers, stimuli_paths=stimuli_paths, number_of_trials=number_of_trials, + require_variance=require_variance) # In case stimuli paths are duplicates (e.g. multiple trials), we first reduce them to only the paths that need # to be run individually, compute activations for those, and then expand the activations to all paths again. # This is done here, before storing, so that we only store the reduced activations. - if model_requirements is not None and 'microsaccades' in model_requirements.keys(): - activations = fnc(layers=layers, stimuli_paths=stimuli_paths, model_requirements=model_requirements) else: reduced_paths = self._reduce_paths(stimuli_paths) - activations = fnc(layers=layers, stimuli_paths=reduced_paths, model_requirements=model_requirements) + activations = fnc(layers=layers, stimuli_paths=reduced_paths, number_of_trials=number_of_trials, + require_variance=require_variance) activations = self._expand_paths(activations, original_paths=stimuli_paths) return activations @store_xarray(identifier_ignore=['stimuli_paths', 'layers'], combine_fields={'layers': 'layer'}) - def _from_paths_stored(self, identifier, layers, stimuli_identifier, stimuli_paths, model_requirements): + def _from_paths_stored(self, identifier, layers, stimuli_identifier, + stimuli_paths, number_of_trials, require_variance): return self._from_paths(layers=layers, stimuli_paths=stimuli_paths) - def _from_paths(self, layers, stimuli_paths, model_requirements=None): + def _from_paths(self, layers, stimuli_paths, number_of_trials=1, require_variance=None): if len(layers) == 0: raise ValueError("No layers passed to retrieve activations from") self._logger.info('Running stimuli') - layer_activations = self._get_activations_with_model_requirements(stimuli_paths, layers=layers, - model_requirements=model_requirements) + if require_variance: + layer_activations = self._get_activations_batched_with_variance(stimuli_paths, + layers=layers, + batch_size=self._batch_size, + number_of_trials=number_of_trials) + else: + layer_activations = self._get_activations_batched(stimuli_paths, layers=layers, batch_size=self._batch_size) self._logger.info('Packaging into assembly') - return self._package(layer_activations, stimuli_paths, model_requirements) + return self._package(layer_activations, stimuli_paths, number_of_trials, require_variance) def _reduce_paths(self, stimuli_paths): return list(set(stimuli_paths)) @@ -159,13 +178,6 @@ def register_stimulus_set_hook(self, hook): self._stimulus_set_hooks[handle.id] = hook return handle - def _get_activations_with_model_requirements(self, paths, layers, model_requirements): - if model_requirements is not None and 'microsaccades' in model_requirements.keys(): - assert len(model_requirements['microsaccades']) > 0 - return self._get_microsaccade_activations_batched(paths, layers=layers, batch_size=self._batch_size, - shifts=model_requirements['microsaccades']) - return self._get_activations_batched(paths, layers=layers, batch_size=self._batch_size) - def _get_activations_batched(self, paths, layers, batch_size): layer_activations = None for batch_start in tqdm(range(0, len(paths), batch_size), unit_scale=batch_size, desc="activations"): @@ -183,10 +195,11 @@ def _get_activations_batched(self, paths, layers, batch_size): return layer_activations - def _get_microsaccade_activations_batched(self, paths, layers, batch_size, shifts): + def _get_activations_batched_with_variance(self, paths, layers, batch_size, number_of_trials): """ :param shifts: a list of tuples containing the pixel shifts to apply to the paths. """ + self.shifts = self.select_microsaccades(number_of_trials=number_of_trials) layer_activations = OrderedDict() for batch_start in tqdm(range(0, len(paths), batch_size), unit_scale=batch_size, desc="activations"): batch_end = min(batch_start + batch_size, len(paths)) @@ -194,13 +207,10 @@ def _get_microsaccade_activations_batched(self, paths, layers, batch_size, shift batch_activations = OrderedDict() # compute activations on the entire batch one shift at a time - for shift in shifts: + for shift in self.shifts: assert type(shift) == tuple - shifted_batch, num_padding, temp_file_paths = self.translate_and_preprocess(batch_inputs, - shift, - batch_size) - activations = self.get_activations(shifted_batch, layers) - activations = self._unpad(activations, num_padding) + temp_file_paths = self.translate_images(batch_inputs, shift) + activations = self._get_batch_activations(temp_file_paths, layers, batch_size) for temp_file_path in temp_file_paths: try: os.remove(temp_file_path) @@ -234,17 +244,13 @@ def _get_batch_activations(self, inputs, layer_names, batch_size): activations = self._unpad(activations, num_padding) return activations - def translate_and_preprocess(self, images, shift, batch_size): + def translate_images(self, images, shift): assert type(images) == list temp_file_paths = [] for image_path in images: fp = self.translate_image(image_path, shift) temp_file_paths.append(fp) - # having to pad here causes a considerable slowdown for small datasets (n_images << batch_size), - # but is necessary to accommodate tensorflow models - padded_images, num_padding = self._pad(temp_file_paths, batch_size) - preprocessed_images = self.preprocess(padded_images) - return preprocessed_images, num_padding, temp_file_paths + return temp_file_paths def translate_image(self, image_path: str, shift: np.array) -> str: """Translates and saves a temporary image to temporary_fp.""" @@ -266,12 +272,15 @@ def _pad(self, batch_images, batch_size): def _unpad(self, layer_activations, num_padding): return change_dict(layer_activations, lambda values: values[:-num_padding or None]) - def _package(self, layer_activations, stimuli_paths, model_requirements): + def _package(self, layer_activations, stimuli_paths, number_of_trials, require_variance): shapes = [a.shape for a in layer_activations.values()] self._logger.debug(f"Activations shapes: {shapes}") self._logger.debug("Packaging individual layers") - layer_assemblies = [self._package_layer(single_layer_activations, layer=layer, - stimuli_paths=stimuli_paths, model_requirements=model_requirements) for + layer_assemblies = [self._package_layer(single_layer_activations, + layer=layer, + stimuli_paths=stimuli_paths, + number_of_trials=number_of_trials, + require_variance=require_variance) for layer, single_layer_activations in tqdm(layer_activations.items(), desc='layer packaging')] # merge manually instead of using merge_data_arrays since `xarray.merge` is very slow with these large arrays # complication: (non)neuroid_coords are taken from the structure of layer_assemblies[0] i.e. the 1st assembly; @@ -299,13 +308,9 @@ def _package(self, layer_activations, stimuli_paths, model_requirements): dims=layer_assemblies[0].dims) return model_assembly - def _package_layer(self, layer_activations, layer, stimuli_paths, model_requirements): - if model_requirements is not None and 'microsaccades' in model_requirements.keys(): - runs_per_image = len(model_requirements['microsaccades']) - else: - runs_per_image = 1 - assert layer_activations.shape[0] == len(stimuli_paths) * runs_per_image - stimuli_paths = np.repeat(stimuli_paths, runs_per_image) + def _package_layer(self, layer_activations, layer, stimuli_paths, number_of_trials=1, require_variance=False): + assert layer_activations.shape[0] == len(stimuli_paths) * number_of_trials + stimuli_paths = np.repeat(stimuli_paths, number_of_trials) activations, flatten_indices = flatten(layer_activations, return_index=True) # collapse for single neuroid dim flatten_coord_names = None if flatten_indices.shape[1] == 1: # fully connected, e.g. classifier @@ -322,8 +327,8 @@ def _package_layer(self, layer_activations, layer, stimuli_paths, model_requirem self._logger.debug(f"Unknown layer activations shape {layer_activations.shape}, not inferring channels") # build assembly - if model_requirements is not None and 'microsaccades' in model_requirements.keys(): - coords = self.build_microsaccade_coords(activations, layer, stimuli_paths, model_requirements) + if require_variance: + coords = self.build_microsaccade_coords(activations, layer, stimuli_paths) else: coords = {'stimulus_path': ('presentation', stimuli_paths), 'stimulus_path2': ('presentation', stimuli_paths), # to avoid DataAssembly dim collapse @@ -343,11 +348,11 @@ def _package_layer(self, layer_activations, layer, stimuli_paths, model_requirem layer_assembly['neuroid_id'] = 'neuroid', neuroid_id return layer_assembly - def build_microsaccade_coords(self, activations, layer, stimuli_paths, model_requirements): + def build_microsaccade_coords(self, activations, layer, stimuli_paths): coords = { 'stimulus_path': ('presentation', stimuli_paths), - 'shift_x': ('presentation', [shift[0] for shift in model_requirements['microsaccades']]), - 'shift_y': ('presentation', [shift[1] for shift in model_requirements['microsaccades']]), + 'shift_x': ('presentation', [shift[0] for shift in self.shifts]), + 'shift_y': ('presentation', [shift[1] for shift in self.shifts]), 'neuroid_num': ('neuroid', list(range(activations.shape[1]))), 'model': ('neuroid', [self.identifier] * activations.shape[1]), 'layer': ('neuroid', [layer] * activations.shape[1]), @@ -360,6 +365,28 @@ def insert_attrs(self, wrapper): wrapper.register_batch_activations_hook = self.register_batch_activations_hook wrapper.register_stimulus_set_hook = self.register_stimulus_set_hook + @staticmethod + def select_microsaccades(number_of_trials): + # Determine the size of the grid needed to ensure we have at least number_of_trials coordinates + # This is an approximation to avoid overly large initial grids. + n = int(np.ceil(np.sqrt(number_of_trials))) # Basic estimation of required grid size + + # Generate grid coordinates + xv, yv = np.meshgrid(range(-n, n + 1), range(-n, n + 1)) + # Combine number_of_trials and y into a single array of tuples + coords = np.vstack([xv.ravel(), yv.ravel()]).T + + # Sort key equivalent: prioritize by maximum absolute value, then by total absolute value + sort_criteria = np.maximum(np.abs(coords[:, 0]), + np.abs(coords[:, 1])) + \ + np.abs(coords).sum(axis=1) / 1000 + sorted_indices = np.argsort(sort_criteria) # Get sorted indices based on our criteria + + # Select the first number_of_trials coordinates based on sorted indices + selected_coords = [tuple(coord) for coord in coords[sorted_indices][:number_of_trials]] + + return selected_coords + @staticmethod def translate(image, shift): rows, cols, _ = image.shape @@ -400,9 +427,7 @@ def lstrip_local(path): return path -def attach_stimulus_set_meta(assembly, stimulus_set, model_requirements): - if model_requirements is not None and 'microsaccades' in model_requirements.keys(): - return attach_stimulus_set_meta_with_microsaccades(assembly, stimulus_set, model_requirements) +def attach_stimulus_set_meta(assembly, stimulus_set): stimulus_paths = [str(stimulus_set.get_stimulus(stimulus_id)) for stimulus_id in stimulus_set['stimulus_id']] stimulus_paths = [lstrip_local(path) for path in stimulus_paths] assembly_paths = [lstrip_local(path) for path in assembly['stimulus_path'].values] @@ -425,12 +450,12 @@ def attach_stimulus_set_meta(assembly, stimulus_set, model_requirements): return assembly -def attach_stimulus_set_meta_with_microsaccades(assembly, stimulus_set, model_requirements): +def attach_stimulus_set_meta_with_microsaccades(assembly, stimulus_set, number_of_trials, shifts): stimulus_paths = [str(stimulus_set.get_stimulus(stimulus_id)) for stimulus_id in stimulus_set['stimulus_id']] stimulus_paths = [lstrip_local(path) for path in stimulus_paths] assembly_paths = [lstrip_local(path) for path in assembly['stimulus_path'].values] - replication_factor = len(model_requirements['microsaccades']) + replication_factor = number_of_trials repeated_stimulus_paths = np.repeat(stimulus_paths, replication_factor) assert (np.array(assembly_paths) == np.array(repeated_stimulus_paths)).all() repeated_stimulus_ids = np.repeat(stimulus_set['stimulus_id'].values, replication_factor) @@ -452,8 +477,8 @@ def attach_stimulus_set_meta_with_microsaccades(assembly, stimulus_set, model_re if 'stimulus_path2' in all_columns: all_columns.remove('stimulus_path2') - repeated_assembly.coords['shift_x'] = ('presentation', [shift[0] for shift in model_requirements['microsaccades']]) - repeated_assembly.coords['shift_y'] = ('presentation', [shift[1] for shift in model_requirements['microsaccades']]) + repeated_assembly.coords['shift_x'] = ('presentation', [shift[0] for shift in shifts]) + repeated_assembly.coords['shift_y'] = ('presentation', [shift[1] for shift in shifts]) # Set MultiIndex index = ['stimulus_id', 'stimulus_path2'] + all_columns + ['shift_x', 'shift_y'] diff --git a/brainscore_vision/model_helpers/brain_transformation/neural.py b/brainscore_vision/model_helpers/brain_transformation/neural.py index 07a6ba700..64130e98e 100644 --- a/brainscore_vision/model_helpers/brain_transformation/neural.py +++ b/brainscore_vision/model_helpers/brain_transformation/neural.py @@ -24,21 +24,26 @@ def __init__(self, identifier, activations_model, region_layer_map, visual_degre def identifier(self): return self._identifier - def look_at(self, stimuli, number_of_trials=1, model_requirements: Optional[Dict[str, List]] = None): + def look_at(self, stimuli, number_of_trials=1, require_variance: bool = False): """ - :param model_requirements: a dictionary containing any requirements a benchmark might have for models, e.g. + TODO + :param require_variance: a dictionary containing any requirements a benchmark might have for models, e.g. microsaccades for getting variable responses from non-stochastic models to the same stimuli. - model_requirements['microsaccades']: list of tuples of x and y shifts to apply to each image to model + require_variance['microsaccades']: list of tuples of x and y shifts to apply to each image to model microsaccades. Note that the shifts happen in pixel space of the original input image, not the preprocessed - image. + image. Ideally, a model will define its microsaccade behavior itself, and thus compute its microsaccade + behavior on the basis of its visual angle. Currently the base behavior is that the microsaccade extent is + fixed in the visual angle, i.e. models that see 3deg vs 8deg will have the same microsaccades in terms + of visual angle. Models that wish to bypass this behavior should reimplement the `select_microsaccades` + function in activations/core.py. Human microsaccade amplitude varies by who you ask, an estimate might be <0.1 deg = 360 arcsec = 6arcmin. The goal of microsaccades is to obtain multiple different neural activities to the same input stimulus from non-stochastic models. This is to improve estimates of e.g. psychophysical functions, but also other things. Note that microsaccades are also applied to stochastic models to make them comparable within- benchmark to non-stochastic models. Example usage: - model_requirements = {'microsaccades': [(0, 0), (0, 1), (1, 0), (1, 1)]} + require_variance = {'microsaccades': [(0, 0), (0, 1), (1, 0), (1, 1)]} More information: --> Rolfs 2009 "Microsaccades: Small steps on a long way" Vision Research, Volume 49, Issue 20, 15 October 2009, Pages 2415-2441. @@ -57,12 +62,13 @@ def look_at(self, stimuli, number_of_trials=1, model_requirements: Optional[Dict activations = self.run_activations(stimuli, layers=list(layer_regions.keys()), number_of_trials=number_of_trials, - model_requirements=model_requirements) + require_variance=require_variance) activations['region'] = 'neuroid', [layer_regions[layer] for layer in activations['layer'].values] return activations - def run_activations(self, stimuli, layers, number_of_trials=1, model_requirements=None): - activations = self.activations_model(stimuli, layers=layers, model_requirements=model_requirements) + def run_activations(self, stimuli, layers, number_of_trials=1, require_variance=None): + activations = self.activations_model(stimuli, layers=layers, number_of_trials=number_of_trials, + require_variance=require_variance) return activations def start_task(self, task): diff --git a/tests/test_model_helpers/activations/test___init__.py b/tests/test_model_helpers/activations/test___init__.py index 835446410..4f8fd3588 100644 --- a/tests/test_model_helpers/activations/test___init__.py +++ b/tests/test_model_helpers/activations/test___init__.py @@ -192,37 +192,35 @@ def test_from_image_path(model_ctr, layers, image_name, pca_components, logits): @pytest.mark.parametrize("image_name", ['rgb.jpg', 'grayscale.png', 'grayscale2.jpg', 'grayscale_alpha.png', 'palletized.png']) @pytest.mark.parametrize(["model_ctr", "layers"], models_layers) -@pytest.mark.parametrize("shifts", [[(2, 2), (0, 0), (1, -1), (0, 1)], [(0, 1), (2, 3)]]) -def test_microsaccades_from_image_path(model_ctr, layers, image_name, shifts): +@pytest.mark.parametrize("number_of_trials", [1, 5, 25]) +def test_microsaccades_from_image_path(model_ctr, layers, image_name, number_of_trials): stimulus_paths = [os.path.join(os.path.dirname(__file__), image_name)] activations_extractor = model_ctr() - model_requirements = {'microsaccades': shifts} - activations = activations_extractor(stimuli=stimulus_paths, layers=layers, model_requirements=model_requirements) + activations = activations_extractor(stimuli=stimulus_paths, layers=layers, number_of_trials=number_of_trials, + require_variance=True) assert activations is not None - assert list(activations['shift_x'].values) == [shift[0] for shift in shifts] - assert list(activations['shift_y'].values) == [shift[1] for shift in shifts] - assert len(activations['shift_x']) == len(shifts) * len(stimulus_paths) - assert len(activations['shift_y']) == len(shifts) * len(stimulus_paths) + assert len(activations['shift_x']) == number_of_trials * len(stimulus_paths) + assert len(activations['shift_y']) == number_of_trials * len(stimulus_paths) @pytest.mark.parametrize("image_name", ['rgb.jpg', 'grayscale.png', 'grayscale2.jpg', 'grayscale_alpha.png', 'palletized.png']) @pytest.mark.parametrize(["model_ctr", "layers"], models_layers) -@pytest.mark.parametrize("model_requirements", [None, {'microsaccades': [(0, 0), (1, -1)]}]) -def test_model_requirements(model_ctr, layers, image_name, model_requirements): +@pytest.mark.parametrize("require_variance", [False, True]) +def test_model_requirements(model_ctr, layers, image_name, require_variance): stimulus_paths = [os.path.join(os.path.dirname(__file__), image_name)] activations_extractor = model_ctr() activations_with_req = activations_extractor(stimuli=stimulus_paths, layers=layers, - model_requirements=model_requirements) + number_of_trials=2, require_variance=require_variance) activations_without_req = activations_extractor(stimuli=stimulus_paths, layers=layers, - model_requirements=None) + number_of_trials=1, require_variance=False) assert activations_with_req is not None assert activations_without_req is not None - if model_requirements: + if require_variance: assert len(activations_with_req['presentation']) > len(activations_without_req['presentation']) assert len(activations_with_req['neuroid']) == len(activations_without_req['neuroid']) @@ -234,9 +232,9 @@ def test_temporary_file_handling(model_ctr, layers, image_name): import tempfile stimulus_paths = [os.path.join(os.path.dirname(__file__), image_name)] activations_extractor = model_ctr() - model_requirements = {'microsaccades': [(2, 2)]} - activations = activations_extractor(stimuli=stimulus_paths, layers=layers, model_requirements=model_requirements) + activations = activations_extractor(stimuli=stimulus_paths, layers=layers, number_of_trials=2, + require_variance=True) temp_files = [f for f in os.listdir(tempfile.gettempdir()) if f.startswith('temp') and f.endswith('.png')] assert activations is not None @@ -278,10 +276,10 @@ def test_exact_activations(pca_components): path_to_expected = Path(__file__).parent / f'alexnet-rgb-{pca_components}.nc' expected = xr.load_dataarray(path_to_expected) - # TODO: this is probably bad. For now, this is required since we decided to change the `stimulus_path` index to the - # `presentation` index for all assemblies as a part of the PR. Should not cause issues for models that are run - # after the PR, but loading any extant .nc files could result in issues. Is there a canonical way to get around - # this? + # Originally, the `stimulus_path` Index was used to index into xarrays in Brain-Score, but this was changed + # as a part of PR #492 to a MultiIndex to allow metadata to be attached to multiple repetitions of the same + # `stimulus_path`. Old .nc files need to be updated to use the `presentation` index instead of `stimulus_path`, + # and instead of changing the extant activations, this test was simply modified to simulate that. expected = expected.rename({'stimulus_path': 'presentation'}) assert (activations == expected).all() From e690b4ee2642df5c0d1cb9f210b97045b6047636 Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Tue, 6 Feb 2024 14:37:17 +0100 Subject: [PATCH 09/41] update comments --- .../model_helpers/activations/core.py | 67 +++++++++---------- .../brain_transformation/neural.py | 23 +++---- 2 files changed, 43 insertions(+), 47 deletions(-) diff --git a/brainscore_vision/model_helpers/activations/core.py b/brainscore_vision/model_helpers/activations/core.py index e20094a4a..d347362fd 100644 --- a/brainscore_vision/model_helpers/activations/core.py +++ b/brainscore_vision/model_helpers/activations/core.py @@ -40,28 +40,28 @@ def __init__(self, get_activations, preprocessing, identifier=False, batch_size= def __call__(self, stimuli, layers, stimuli_identifier=None, number_of_trials=1, require_variance=None): """ - TODO: number_of_trials needs to traverse down from here :param stimuli_identifier: a stimuli identifier for the stored results file. False to disable saving. - :param require_variance: a dictionary containing any requirements a benchmark might have for models, e.g. - microsaccades for getting variable responses from non-stochastic models to the same stimuli. - - require_variance['microsaccades']: list of tuples of x and y shifts to apply to each image to model - microsaccades. Note that the shifts happen in pixel space of the original input image, not the preprocessed - image. + :param number_of_trials: An integer that determines how many repetitions of the same model performs. + :param require_variance: A bool that asks models to output different responses to the same stimuli (i.e., + allows stochastic responses to identical stimuli, even in deterministic models). The current implementation + implements this using microsaccades. Human microsaccade amplitude varies by who you ask, an estimate might be <0.1 deg = 360 arcsec = 6arcmin. The goal of microsaccades is to obtain multiple different neural activities to the same input stimulus from non-stochastic models. This is to improve estimates of e.g. psychophysical functions, but also other things. Note that microsaccades are also applied to stochastic models to make them comparable within- benchmark to non-stochastic models. + In the current implementation, if `require_variance=True`, the model selects microsaccades according to + its own microsaccade behavior (if it has implemented it), or with the base behavior of saccading in + input pixel space with 1-pixel increments from the center of the stimulus. The base behavior thus + maintains a fixed microsaccade distance as measured in visual angle, regardless of the model's visual angle. Example usage: - require_variance = {'microsaccades': [(0, 0), (0, 1), (1, 0), (1, 1)]} + require_variance = True More information: --> Rolfs 2009 "Microsaccades: Small steps on a long way" Vision Research, Volume 49, Issue 20, 15 October 2009, Pages 2415-2441. --> Haddad & Steinmann 1973 "The smallest voluntary saccade: Implications for fixation" Vision Research Volume 13, Issue 6, June 1973, Pages 1075-1086, IN5-IN6. - Huge thanks to Johannes Mehrer for the implementation of microsaccades in the Brain-Score core.py and - neural.py files. + Thanks to Johannes Mehrer for initial help in implementing microsaccades. """ if isinstance(stimuli, StimulusSet): @@ -107,17 +107,13 @@ def from_paths(self, stimuli_paths, layers, stimuli_identifier=None, number_of_t self._logger.debug(f"self.identifier `{self.identifier}` or stimuli_identifier {stimuli_identifier} " f"are not set, will not store") fnc = self._from_paths - if require_variance: - activations = fnc(layers=layers, stimuli_paths=stimuli_paths, number_of_trials=number_of_trials, - require_variance=require_variance) # In case stimuli paths are duplicates (e.g. multiple trials), we first reduce them to only the paths that need # to be run individually, compute activations for those, and then expand the activations to all paths again. # This is done here, before storing, so that we only store the reduced activations. - else: - reduced_paths = self._reduce_paths(stimuli_paths) - activations = fnc(layers=layers, stimuli_paths=reduced_paths, number_of_trials=number_of_trials, - require_variance=require_variance) - activations = self._expand_paths(activations, original_paths=stimuli_paths) + reduced_paths = self._reduce_paths(stimuli_paths) + activations = fnc(layers=layers, stimuli_paths=reduced_paths, number_of_trials=number_of_trials, + require_variance=require_variance) + activations = self._expand_paths(activations, original_paths=stimuli_paths) return activations @store_xarray(identifier_ignore=['stimuli_paths', 'layers'], combine_fields={'layers': 'layer'}) @@ -197,7 +193,11 @@ def _get_activations_batched(self, paths, layers, batch_size): def _get_activations_batched_with_variance(self, paths, layers, batch_size, number_of_trials): """ - :param shifts: a list of tuples containing the pixel shifts to apply to the paths. + This function fulfils the role of `_get_activations_batched` in the case microsaccades are needed, but + since microsaccade activations need to be computed on an entire batch at once (to accommodate TF models), + this function is implemented separately to avoid the mess that `_get_activations_batched` would otherwise be. + + :param number_of_trials: the number of trials that a model performs of each individual stimulus. """ self.shifts = self.select_microsaccades(number_of_trials=number_of_trials) layer_activations = OrderedDict() @@ -366,26 +366,25 @@ def insert_attrs(self, wrapper): wrapper.register_stimulus_set_hook = self.register_stimulus_set_hook @staticmethod - def select_microsaccades(number_of_trials): - # Determine the size of the grid needed to ensure we have at least number_of_trials coordinates - # This is an approximation to avoid overly large initial grids. - n = int(np.ceil(np.sqrt(number_of_trials))) # Basic estimation of required grid size + def select_microsaccades(number_of_trials: int): + """ + A naive function for generating microsaccade locations that span the same visual angle regardless of model + visual angle (to keep microsaccade extent constant across models). The function returns a list of + `number_of_trials` tuples that each contain a microsaccade location, expanding from the center of the image. + """ + n = int(np.ceil(np.sqrt(number_of_trials))) # upper bound on number of microsaccades needed - # Generate grid coordinates + # generate grid of potential microsaccades xv, yv = np.meshgrid(range(-n, n + 1), range(-n, n + 1)) - # Combine number_of_trials and y into a single array of tuples coords = np.vstack([xv.ravel(), yv.ravel()]).T - # Sort key equivalent: prioritize by maximum absolute value, then by total absolute value - sort_criteria = np.maximum(np.abs(coords[:, 0]), - np.abs(coords[:, 1])) + \ - np.abs(coords).sum(axis=1) / 1000 - sorted_indices = np.argsort(sort_criteria) # Get sorted indices based on our criteria - - # Select the first number_of_trials coordinates based on sorted indices - selected_coords = [tuple(coord) for coord in coords[sorted_indices][:number_of_trials]] + # sort microsaccade locations: (0, 0) should be first, and e.g. (2, 3) before (-5, 6) (absolute value). + sort_criteria = np.maximum(np.abs(coords[:, 0]), np.abs(coords[:, 1])) + np.abs(coords).sum(axis=1) / 1000 + sorted_indices = np.argsort(sort_criteria) - return selected_coords + # select the first `number_of_trials` microsaccades from the sorted upper bound + selected_microsaccades = [tuple(coord) for coord in coords[sorted_indices][:number_of_trials]] + return selected_microsaccades @staticmethod def translate(image, shift): diff --git a/brainscore_vision/model_helpers/brain_transformation/neural.py b/brainscore_vision/model_helpers/brain_transformation/neural.py index 64130e98e..568f9aa77 100644 --- a/brainscore_vision/model_helpers/brain_transformation/neural.py +++ b/brainscore_vision/model_helpers/brain_transformation/neural.py @@ -27,30 +27,27 @@ def identifier(self): def look_at(self, stimuli, number_of_trials=1, require_variance: bool = False): """ TODO - :param require_variance: a dictionary containing any requirements a benchmark might have for models, e.g. - microsaccades for getting variable responses from non-stochastic models to the same stimuli. - - require_variance['microsaccades']: list of tuples of x and y shifts to apply to each image to model - microsaccades. Note that the shifts happen in pixel space of the original input image, not the preprocessed - image. Ideally, a model will define its microsaccade behavior itself, and thus compute its microsaccade - behavior on the basis of its visual angle. Currently the base behavior is that the microsaccade extent is - fixed in the visual angle, i.e. models that see 3deg vs 8deg will have the same microsaccades in terms - of visual angle. Models that wish to bypass this behavior should reimplement the `select_microsaccades` - function in activations/core.py. + :param number_of_trials: An integer that determines how many repetitions of the same model performs. + :param require_variance: A bool that asks models to output different responses to the same stimuli (i.e., + allows stochastic responses to identical stimuli, even in deterministic models). The current implementation + implements this using microsaccades. Human microsaccade amplitude varies by who you ask, an estimate might be <0.1 deg = 360 arcsec = 6arcmin. The goal of microsaccades is to obtain multiple different neural activities to the same input stimulus from non-stochastic models. This is to improve estimates of e.g. psychophysical functions, but also other things. Note that microsaccades are also applied to stochastic models to make them comparable within- benchmark to non-stochastic models. + In the current implementation, if `require_variance=True`, the model selects microsaccades according to + its own microsaccade behavior (if it has implemented it), or with the base behavior of saccading in + input pixel space with 1-pixel increments from the center of the stimulus. The base behavior thus + maintains a fixed microsaccade distance as measured in visual angle, regardless of the model's visual angle. Example usage: - require_variance = {'microsaccades': [(0, 0), (0, 1), (1, 0), (1, 1)]} + require_variance = True More information: --> Rolfs 2009 "Microsaccades: Small steps on a long way" Vision Research, Volume 49, Issue 20, 15 October 2009, Pages 2415-2441. --> Haddad & Steinmann 1973 "The smallest voluntary saccade: Implications for fixation" Vision Research Volume 13, Issue 6, June 1973, Pages 1075-1086, IN5-IN6. - Huge thanks to Johannes Mehrer for the implementation of microsaccades in the Brain-Score core.py and - neural.py files. + Thanks to Johannes Mehrer for initial help in implementing microsaccades. """ layer_regions = {} for region in self.recorded_regions: From e05d8158b540a5ca04cf7b7114929248c66cd313 Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Wed, 7 Feb 2024 17:18:40 +0100 Subject: [PATCH 10/41] remove indexing bug --- .../model_helpers/activations/core.py | 16 ++++++++++------ .../model_helpers/brain_transformation/neural.py | 2 -- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/brainscore_vision/model_helpers/activations/core.py b/brainscore_vision/model_helpers/activations/core.py index d347362fd..dd165e5c6 100644 --- a/brainscore_vision/model_helpers/activations/core.py +++ b/brainscore_vision/model_helpers/activations/core.py @@ -107,13 +107,17 @@ def from_paths(self, stimuli_paths, layers, stimuli_identifier=None, number_of_t self._logger.debug(f"self.identifier `{self.identifier}` or stimuli_identifier {stimuli_identifier} " f"are not set, will not store") fnc = self._from_paths - # In case stimuli paths are duplicates (e.g. multiple trials), we first reduce them to only the paths that need - # to be run individually, compute activations for those, and then expand the activations to all paths again. - # This is done here, before storing, so that we only store the reduced activations. - reduced_paths = self._reduce_paths(stimuli_paths) - activations = fnc(layers=layers, stimuli_paths=reduced_paths, number_of_trials=number_of_trials, + if require_variance: + activations = fnc(layers=layers, stimuli_paths=stimuli_paths, number_of_trials=number_of_trials, require_variance=require_variance) - activations = self._expand_paths(activations, original_paths=stimuli_paths) + else: + # In case stimuli paths are duplicates (e.g. multiple trials), we first reduce them to only the paths that need + # to be run individually, compute activations for those, and then expand the activations to all paths again. + # This is done here, before storing, so that we only store the reduced activations. + reduced_paths = self._reduce_paths(stimuli_paths) + activations = fnc(layers=layers, stimuli_paths=reduced_paths, number_of_trials=number_of_trials, + require_variance=require_variance) + activations = self._expand_paths(activations, original_paths=stimuli_paths) return activations @store_xarray(identifier_ignore=['stimuli_paths', 'layers'], combine_fields={'layers': 'layer'}) diff --git a/brainscore_vision/model_helpers/brain_transformation/neural.py b/brainscore_vision/model_helpers/brain_transformation/neural.py index 568f9aa77..fbe78d10e 100644 --- a/brainscore_vision/model_helpers/brain_transformation/neural.py +++ b/brainscore_vision/model_helpers/brain_transformation/neural.py @@ -2,7 +2,6 @@ from result_caching import store_xarray, store from tqdm import tqdm -from typing import Optional, Dict, List from brainscore_vision.metrics import Score from brainscore_vision.model_helpers.activations.pca import LayerPCA @@ -26,7 +25,6 @@ def identifier(self): def look_at(self, stimuli, number_of_trials=1, require_variance: bool = False): """ - TODO :param number_of_trials: An integer that determines how many repetitions of the same model performs. :param require_variance: A bool that asks models to output different responses to the same stimuli (i.e., allows stochastic responses to identical stimuli, even in deterministic models). The current implementation From 629884ecc01215429ea75c3f0de2028bb095b3c8 Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Wed, 7 Feb 2024 18:08:00 +0100 Subject: [PATCH 11/41] fix bug with activations.shape --- brainscore_vision/data/scialom2024/__init__.py | 0 brainscore_vision/model_helpers/activations/core.py | 9 +++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 brainscore_vision/data/scialom2024/__init__.py diff --git a/brainscore_vision/data/scialom2024/__init__.py b/brainscore_vision/data/scialom2024/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/brainscore_vision/model_helpers/activations/core.py b/brainscore_vision/model_helpers/activations/core.py index dd165e5c6..84122780c 100644 --- a/brainscore_vision/model_helpers/activations/core.py +++ b/brainscore_vision/model_helpers/activations/core.py @@ -313,8 +313,13 @@ def _package(self, layer_activations, stimuli_paths, number_of_trials, require_v return model_assembly def _package_layer(self, layer_activations, layer, stimuli_paths, number_of_trials=1, require_variance=False): - assert layer_activations.shape[0] == len(stimuli_paths) * number_of_trials - stimuli_paths = np.repeat(stimuli_paths, number_of_trials) + # activation shape is larger if variance in responses is required from the model by a factor of number_of_trials + if require_variance: + runs_per_image = number_of_trials + else: + runs_per_image = 1 + assert layer_activations.shape[0] == len(stimuli_paths) * runs_per_image + stimuli_paths = np.repeat(stimuli_paths, runs_per_image) activations, flatten_indices = flatten(layer_activations, return_index=True) # collapse for single neuroid dim flatten_coord_names = None if flatten_indices.shape[1] == 1: # fully connected, e.g. classifier From 02264956dae5f6abe09e0f976075b94719bccc67 Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Mon, 12 Feb 2024 11:11:36 +0100 Subject: [PATCH 12/41] Apply suggestions from code review Co-authored-by: Martin Schrimpf --- .../model_helpers/activations/core.py | 23 ++++++++++--------- .../activations/test___init__.py | 2 +- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/brainscore_vision/model_helpers/activations/core.py b/brainscore_vision/model_helpers/activations/core.py index 84122780c..3aab1ebac 100644 --- a/brainscore_vision/model_helpers/activations/core.py +++ b/brainscore_vision/model_helpers/activations/core.py @@ -34,7 +34,7 @@ def __init__(self, get_activations, preprocessing, identifier=False, batch_size= self.identifier = identifier self.get_activations = get_activations self.preprocess = preprocessing or (lambda x: x) - self.shifts = None # for use with microsaccades + self.microsaccade_shifts = None # for use with microsaccades self._stimulus_set_hooks = {} self._batch_activations_hooks = {} @@ -43,25 +43,26 @@ def __call__(self, stimuli, layers, stimuli_identifier=None, number_of_trials=1, :param stimuli_identifier: a stimuli identifier for the stored results file. False to disable saving. :param number_of_trials: An integer that determines how many repetitions of the same model performs. :param require_variance: A bool that asks models to output different responses to the same stimuli (i.e., - allows stochastic responses to identical stimuli, even in deterministic models). The current implementation - implements this using microsaccades. + allows stochastic responses to identical stimuli, even in otherwise deterministic base models). + We here implement this using microsaccades. Human microsaccade amplitude varies by who you ask, an estimate might be <0.1 deg = 360 arcsec = 6arcmin. - The goal of microsaccades is to obtain multiple different neural activities to the same input stimulus - from non-stochastic models. This is to improve estimates of e.g. psychophysical functions, but also other - things. Note that microsaccades are also applied to stochastic models to make them comparable within- + Our motivation to make use of such microsaccades is to obtain multiple different neural activities to the same input stimulus + from non-stochastic models. This is to improve estimates of e.g. psychophysical functions. + Note that microsaccades are also applied to stochastic models to make them comparable within- benchmark to non-stochastic models. - In the current implementation, if `require_variance=True`, the model selects microsaccades according to - its own microsaccade behavior (if it has implemented it), or with the base behavior of saccading in + In the current implementation, if `require_variance=True`, the model microsaccades in input pixel space with 1-pixel increments from the center of the stimulus. The base behavior thus maintains a fixed microsaccade distance as measured in visual angle, regardless of the model's visual angle. + Example usage: - require_variance = True + `require_variance = True` + More information: --> Rolfs 2009 "Microsaccades: Small steps on a long way" Vision Research, Volume 49, Issue 20, 15 October 2009, Pages 2415-2441. --> Haddad & Steinmann 1973 "The smallest voluntary saccade: Implications for fixation" Vision Research Volume 13, Issue 6, June 1973, Pages 1075-1086, IN5-IN6. - Thanks to Johannes Mehrer for initial help in implementing microsaccades. + Implemented by Ben Lonnqvist and Johannes Mehrer. """ if isinstance(stimuli, StimulusSet): @@ -87,7 +88,7 @@ def from_stimulus_set(self, stimulus_set, layers, stimuli_identifier=None, numbe stimuli_paths = [str(stimulus_set.get_stimulus(stimulus_id)) for stimulus_id in stimulus_set['stimulus_id']] activations = self.from_paths(stimuli_paths=stimuli_paths, layers=layers, stimuli_identifier=stimuli_identifier) if require_variance: - self.shifts = self.select_microsaccades(number_of_trials=number_of_trials) + self.microsaccade_shifts = self.select_microsaccades(number_of_trials=number_of_trials) activations = attach_stimulus_set_meta_with_microsaccades(activations, stimulus_set, number_of_trials=number_of_trials, diff --git a/tests/test_model_helpers/activations/test___init__.py b/tests/test_model_helpers/activations/test___init__.py index 4f8fd3588..2032d6cbf 100644 --- a/tests/test_model_helpers/activations/test___init__.py +++ b/tests/test_model_helpers/activations/test___init__.py @@ -193,7 +193,7 @@ def test_from_image_path(model_ctr, layers, image_name, pca_components, logits): 'palletized.png']) @pytest.mark.parametrize(["model_ctr", "layers"], models_layers) @pytest.mark.parametrize("number_of_trials", [1, 5, 25]) -def test_microsaccades_from_image_path(model_ctr, layers, image_name, number_of_trials): +def test_require_variance_has_shift_coords(model_ctr, layers, image_name, number_of_trials): stimulus_paths = [os.path.join(os.path.dirname(__file__), image_name)] activations_extractor = model_ctr() From 307735d671ab3fdc9a85083c59ba8378ac82b0ba Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Mon, 12 Feb 2024 11:11:50 +0100 Subject: [PATCH 13/41] Delete brainscore_vision/data/scialom2024/__init__.py --- brainscore_vision/data/scialom2024/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 brainscore_vision/data/scialom2024/__init__.py diff --git a/brainscore_vision/data/scialom2024/__init__.py b/brainscore_vision/data/scialom2024/__init__.py deleted file mode 100644 index e69de29bb..000000000 From e13391a6aa46f2139b5783fdad4b98507b4f65a4 Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Fri, 16 Feb 2024 14:42:46 +0100 Subject: [PATCH 14/41] address review changes --- .../model_helpers/activations/core.py | 357 ++++++++++-------- .../brain_transformation/__init__.py | 2 + .../brain_transformation/neural.py | 24 +- .../activations/test___init__.py | 23 +- 4 files changed, 214 insertions(+), 192 deletions(-) diff --git a/brainscore_vision/model_helpers/activations/core.py b/brainscore_vision/model_helpers/activations/core.py index 3aab1ebac..9949dbe07 100644 --- a/brainscore_vision/model_helpers/activations/core.py +++ b/brainscore_vision/model_helpers/activations/core.py @@ -2,7 +2,7 @@ import os import cv2 import tempfile -from typing import List, Tuple +from typing import Dict, Tuple, List, Union import functools import logging @@ -34,11 +34,19 @@ def __init__(self, get_activations, preprocessing, identifier=False, batch_size= self.identifier = identifier self.get_activations = get_activations self.preprocess = preprocessing or (lambda x: x) - self.microsaccade_shifts = None # for use with microsaccades self._stimulus_set_hooks = {} self._batch_activations_hooks = {} - def __call__(self, stimuli, layers, stimuli_identifier=None, number_of_trials=1, require_variance=None): + self.number_of_trials = 1 # for use with microsaccades. + self.microsaccade_extent_degrees = 0.05 # how many degrees models microsaccade by default + + # a dict that contains the image paths and their respective microsaccades. e.g., + # {'abc.jpg': [(0, 0), (1.5, 2)]} + self.microsaccades = {} + self._visual_degrees = None # Model visual degrees. Used for computing microsaccades + + def __call__(self, stimuli, layers, stimuli_identifier=None, number_of_trials: int = 1, + require_variance: bool = False): """ :param stimuli_identifier: a stimuli identifier for the stored results file. False to disable saving. :param number_of_trials: An integer that determines how many repetitions of the same model performs. @@ -47,9 +55,7 @@ def __call__(self, stimuli, layers, stimuli_identifier=None, number_of_trials=1, We here implement this using microsaccades. Human microsaccade amplitude varies by who you ask, an estimate might be <0.1 deg = 360 arcsec = 6arcmin. Our motivation to make use of such microsaccades is to obtain multiple different neural activities to the same input stimulus - from non-stochastic models. This is to improve estimates of e.g. psychophysical functions. - Note that microsaccades are also applied to stochastic models to make them comparable within- - benchmark to non-stochastic models. + from non-stochastic models. This is to improve estimates of e.g. psychophysical functions. In the current implementation, if `require_variance=True`, the model microsaccades in input pixel space with 1-pixel increments from the center of the stimulus. The base behavior thus maintains a fixed microsaccade distance as measured in visual angle, regardless of the model's visual angle. @@ -65,18 +71,22 @@ def __call__(self, stimuli, layers, stimuli_identifier=None, number_of_trials=1, Implemented by Ben Lonnqvist and Johannes Mehrer. """ + if require_variance: + self.number_of_trials = number_of_trials # for use with microsaccades + if (self._visual_degrees is None) and require_variance: + self._logger.debug("When using microsaccades for model commitments other than ModelCommitment, you should " + "set self.activations_model.set_visual_degrees(visual_degrees). Not doing so risks " + "breaking microsaccades.") if isinstance(stimuli, StimulusSet): return self.from_stimulus_set(stimulus_set=stimuli, layers=layers, stimuli_identifier=stimuli_identifier, - number_of_trials=number_of_trials, require_variance=require_variance) + require_variance=require_variance) else: return self.from_paths(stimuli_paths=stimuli, layers=layers, stimuli_identifier=stimuli_identifier, - number_of_trials=number_of_trials, require_variance=require_variance) - def from_stimulus_set(self, stimulus_set, layers, stimuli_identifier=None, number_of_trials=1, - require_variance=None): + def from_stimulus_set(self, stimulus_set, layers, stimuli_identifier=None, require_variance: bool = False): """ :param stimuli_identifier: a stimuli identifier for the stored results file. False to disable saving. None to use `stimulus_set.identifier` @@ -87,16 +97,11 @@ def from_stimulus_set(self, stimulus_set, layers, stimuli_identifier=None, numbe stimulus_set = hook(stimulus_set) stimuli_paths = [str(stimulus_set.get_stimulus(stimulus_id)) for stimulus_id in stimulus_set['stimulus_id']] activations = self.from_paths(stimuli_paths=stimuli_paths, layers=layers, stimuli_identifier=stimuli_identifier) - if require_variance: - self.microsaccade_shifts = self.select_microsaccades(number_of_trials=number_of_trials) - activations = attach_stimulus_set_meta_with_microsaccades(activations, - stimulus_set, - number_of_trials=number_of_trials, - shifts=self.shifts) - activations = attach_stimulus_set_meta(activations, stimulus_set) + activations = attach_stimulus_set_meta(activations, stimulus_set, number_of_trials=self.number_of_trials, + microsaccades=self.microsaccades, require_variance=require_variance) return activations - def from_paths(self, stimuli_paths, layers, stimuli_identifier=None, number_of_trials=1, require_variance=None): + def from_paths(self, stimuli_paths, layers, stimuli_identifier=None, require_variance=None): if layers is None: layers = ['logits'] if self.identifier and stimuli_identifier: @@ -109,15 +114,13 @@ def from_paths(self, stimuli_paths, layers, stimuli_identifier=None, number_of_t f"are not set, will not store") fnc = self._from_paths if require_variance: - activations = fnc(layers=layers, stimuli_paths=stimuli_paths, number_of_trials=number_of_trials, - require_variance=require_variance) + activations = fnc(layers=layers, stimuli_paths=stimuli_paths, require_variance=require_variance) else: # In case stimuli paths are duplicates (e.g. multiple trials), we first reduce them to only the paths that need # to be run individually, compute activations for those, and then expand the activations to all paths again. # This is done here, before storing, so that we only store the reduced activations. reduced_paths = self._reduce_paths(stimuli_paths) - activations = fnc(layers=layers, stimuli_paths=reduced_paths, number_of_trials=number_of_trials, - require_variance=require_variance) + activations = fnc(layers=layers, stimuli_paths=reduced_paths, require_variance=require_variance) activations = self._expand_paths(activations, original_paths=stimuli_paths) return activations @@ -126,19 +129,14 @@ def _from_paths_stored(self, identifier, layers, stimuli_identifier, stimuli_paths, number_of_trials, require_variance): return self._from_paths(layers=layers, stimuli_paths=stimuli_paths) - def _from_paths(self, layers, stimuli_paths, number_of_trials=1, require_variance=None): + def _from_paths(self, layers, stimuli_paths, require_variance: bool = False): if len(layers) == 0: raise ValueError("No layers passed to retrieve activations from") self._logger.info('Running stimuli') - if require_variance: - layer_activations = self._get_activations_batched_with_variance(stimuli_paths, - layers=layers, - batch_size=self._batch_size, - number_of_trials=number_of_trials) - else: - layer_activations = self._get_activations_batched(stimuli_paths, layers=layers, batch_size=self._batch_size) + layer_activations = self._get_activations_batched(stimuli_paths, layers=layers, batch_size=self._batch_size, + require_variance=require_variance) self._logger.info('Packaging into assembly') - return self._package(layer_activations, stimuli_paths, number_of_trials, require_variance) + return self._package(layer_activations, stimuli_paths, require_variance) def _reduce_paths(self, stimuli_paths): return list(set(stimuli_paths)) @@ -179,32 +177,7 @@ def register_stimulus_set_hook(self, hook): self._stimulus_set_hooks[handle.id] = hook return handle - def _get_activations_batched(self, paths, layers, batch_size): - layer_activations = None - for batch_start in tqdm(range(0, len(paths), batch_size), unit_scale=batch_size, desc="activations"): - batch_end = min(batch_start + batch_size, len(paths)) - batch_inputs = paths[batch_start:batch_end] - batch_activations = self._get_batch_activations(batch_inputs, layer_names=layers, batch_size=batch_size) - for hook in self._batch_activations_hooks.copy().values(): # copy to avoid handle re-enabling messing with the loop - batch_activations = hook(batch_activations) - - if layer_activations is None: - layer_activations = copy.copy(batch_activations) - else: - for layer_name, layer_output in batch_activations.items(): - layer_activations[layer_name] = np.concatenate((layer_activations[layer_name], layer_output)) - - return layer_activations - - def _get_activations_batched_with_variance(self, paths, layers, batch_size, number_of_trials): - """ - This function fulfils the role of `_get_activations_batched` in the case microsaccades are needed, but - since microsaccade activations need to be computed on an entire batch at once (to accommodate TF models), - this function is implemented separately to avoid the mess that `_get_activations_batched` would otherwise be. - - :param number_of_trials: the number of trials that a model performs of each individual stimulus. - """ - self.shifts = self.select_microsaccades(number_of_trials=number_of_trials) + def _get_activations_batched(self, paths, layers, batch_size, require_variance: bool): layer_activations = OrderedDict() for batch_start in tqdm(range(0, len(paths), batch_size), unit_scale=batch_size, desc="activations"): batch_end = min(batch_start + batch_size, len(paths)) @@ -212,15 +185,11 @@ def _get_activations_batched_with_variance(self, paths, layers, batch_size, numb batch_activations = OrderedDict() # compute activations on the entire batch one shift at a time - for shift in self.shifts: - assert type(shift) == tuple - temp_file_paths = self.translate_images(batch_inputs, shift) - activations = self._get_batch_activations(temp_file_paths, layers, batch_size) - for temp_file_path in temp_file_paths: - try: - os.remove(temp_file_path) - except FileNotFoundError: - pass + for shift_number in range(self.number_of_trials): + + activations = self._get_batch_activations(batch_inputs, layers, batch_size, require_variance, + shift_number) + for layer_name, layer_output in activations.items(): batch_activations.setdefault(layer_name, []).append(layer_output) @@ -241,29 +210,62 @@ def _get_activations_batched_with_variance(self, paths, layers, batch_size, numb return layer_activations # this is all batches - def _get_batch_activations(self, inputs, layer_names, batch_size): + def _get_batch_activations(self, inputs, layer_names, batch_size, require_variance: bool = False, + trial_number: int = 1): inputs, num_padding = self._pad(inputs, batch_size) preprocessed_inputs = self.preprocess(inputs) + preprocessed_inputs = self.translate_images(preprocessed_inputs, inputs, trial_number, require_variance) activations = self.get_activations(preprocessed_inputs, layer_names) assert isinstance(activations, OrderedDict) activations = self._unpad(activations, num_padding) + if require_variance: + self.remove_temporary_files(preprocessed_inputs) return activations - def translate_images(self, images, shift): - assert type(images) == list - temp_file_paths = [] - for image_path in images: - fp = self.translate_image(image_path, shift) - temp_file_paths.append(fp) - return temp_file_paths - - def translate_image(self, image_path: str, shift: np.array) -> str: + def set_visual_degrees(self, visual_degrees: float) -> None: + """ + A method used by ModelCommitments to give the ActivationsExtractorHelper their visual degrees for + performing microsaccades. + """ + self._visual_degrees = visual_degrees + + def translate_images(self, images: List[Union[str, np.ndarray]], image_paths: List[str], trial_number: int, + require_variance: bool) -> List[str]: + """A method that translates images according to selected microsaccades, if microsaccades are required.""" + output_images = [] + for index, image_path in enumerate(image_paths): + # When microsaccades are not used, skip computing them and return the base images. + # This iteration could be entirely skipped, but recording microsaccades for all images regardless + # of whether variance is required or not is convenient for adding an extra presentation dimension + # in the layer assembly later to avoid collapse, or otherwise extraneous mock dims. + # The method could further be streamlined by calling `self.get_image_with_shape()` and + # `self.select_microsaccade` for all images regardless of require_variance, but it seems like a bad + # idea to introduce cv2 image loading for all models and images, regardless of whether they are actually + # microsaccading. + if not require_variance: + self.microsaccades[image_path] = [(0., 0.)] + output_images.append(images[index]) + else: + # translate images according to microsaccades if we are using microsaccades + image, image_shape = self.get_image_with_shape(images[index]) + microsaccade_location = self.select_microsaccade(image_path=image_path, + trial_number=trial_number, + image_shape=image_shape) + return_string = True if isinstance(images[index], str) else False + output_images.append(self.translate_image(image, microsaccade_location, image_shape, return_string)) + return self.reshape_microsaccaded_images(output_images) + + def translate_image(self, image: str, microsaccade_location: Tuple[float, float], image_shape: Tuple[int, int], + return_string: bool) -> str: """Translates and saves a temporary image to temporary_fp.""" - translated_image = self.translate(cv2.imread(image_path), shift) - temp_file_descriptor, temporary_fp = tempfile.mkstemp(suffix=".png") - os.close(temp_file_descriptor) - if not cv2.imwrite(temporary_fp, translated_image): - raise Exception(f"cv2.imwrite failed: {temporary_fp}") + translated_image = self.translate(image, microsaccade_location, image_shape) + if not return_string: # if the model accepts ndarrays after preprocessing, return one + return translated_image + else: # if the model accepts strings after preprocessing, write temp file + temp_file_descriptor, temporary_fp = tempfile.mkstemp(suffix=".png") + os.close(temp_file_descriptor) + if not cv2.imwrite(temporary_fp, translated_image): + raise Exception(f"cv2.imwrite failed: {temporary_fp}") return temporary_fp def _pad(self, batch_images, batch_size): @@ -277,14 +279,13 @@ def _pad(self, batch_images, batch_size): def _unpad(self, layer_activations, num_padding): return change_dict(layer_activations, lambda values: values[:-num_padding or None]) - def _package(self, layer_activations, stimuli_paths, number_of_trials, require_variance): + def _package(self, layer_activations, stimuli_paths, require_variance): shapes = [a.shape for a in layer_activations.values()] self._logger.debug(f"Activations shapes: {shapes}") self._logger.debug("Packaging individual layers") layer_assemblies = [self._package_layer(single_layer_activations, layer=layer, stimuli_paths=stimuli_paths, - number_of_trials=number_of_trials, require_variance=require_variance) for layer, single_layer_activations in tqdm(layer_activations.items(), desc='layer packaging')] # merge manually instead of using merge_data_arrays since `xarray.merge` is very slow with these large arrays @@ -313,10 +314,10 @@ def _package(self, layer_activations, stimuli_paths, number_of_trials, require_v dims=layer_assemblies[0].dims) return model_assembly - def _package_layer(self, layer_activations, layer, stimuli_paths, number_of_trials=1, require_variance=False): + def _package_layer(self, layer_activations, layer, stimuli_paths, require_variance=False): # activation shape is larger if variance in responses is required from the model by a factor of number_of_trials if require_variance: - runs_per_image = number_of_trials + runs_per_image = self.number_of_trials else: runs_per_image = 1 assert layer_activations.shape[0] == len(stimuli_paths) * runs_per_image @@ -337,15 +338,15 @@ def _package_layer(self, layer_activations, layer, stimuli_paths, number_of_tria self._logger.debug(f"Unknown layer activations shape {layer_activations.shape}, not inferring channels") # build assembly - if require_variance: - coords = self.build_microsaccade_coords(activations, layer, stimuli_paths) - else: - coords = {'stimulus_path': ('presentation', stimuli_paths), - 'stimulus_path2': ('presentation', stimuli_paths), # to avoid DataAssembly dim collapse - 'neuroid_num': ('neuroid', list(range(activations.shape[1]))), - 'model': ('neuroid', [self.identifier] * activations.shape[1]), - 'layer': ('neuroid', [layer] * activations.shape[1]), - } + coords = {'stimulus_path': ('presentation', stimuli_paths), + 'shift_x': ('presentation', unpack_microsaccade_coords(self.microsaccades, + np.unique(stimuli_paths), dim=0)), + 'shift_y': ('presentation', unpack_microsaccade_coords(self.microsaccades, + np.unique(stimuli_paths), dim=1)), + 'neuroid_num': ('neuroid', list(range(activations.shape[1]))), + 'model': ('neuroid', [self.identifier] * activations.shape[1]), + 'layer': ('neuroid', [layer] * activations.shape[1]), + } if flatten_coord_names: flatten_coords = {flatten_coord_names[i]: [sample_index[i] if i < flatten_indices.shape[1] else np.nan @@ -358,54 +359,94 @@ def _package_layer(self, layer_activations, layer, stimuli_paths, number_of_tria layer_assembly['neuroid_id'] = 'neuroid', neuroid_id return layer_assembly - def build_microsaccade_coords(self, activations, layer, stimuli_paths): - coords = { - 'stimulus_path': ('presentation', stimuli_paths), - 'shift_x': ('presentation', [shift[0] for shift in self.shifts]), - 'shift_y': ('presentation', [shift[1] for shift in self.shifts]), - 'neuroid_num': ('neuroid', list(range(activations.shape[1]))), - 'model': ('neuroid', [self.identifier] * activations.shape[1]), - 'layer': ('neuroid', [layer] * activations.shape[1]), - } - return coords - def insert_attrs(self, wrapper): wrapper.from_stimulus_set = self.from_stimulus_set wrapper.from_paths = self.from_paths wrapper.register_batch_activations_hook = self.register_batch_activations_hook wrapper.register_stimulus_set_hook = self.register_stimulus_set_hook - @staticmethod - def select_microsaccades(number_of_trials: int): + def select_microsaccade(self, image_path: str, trial_number: int, image_shape: Tuple[int, int] + ) -> Tuple[float, float]: """ A naive function for generating microsaccade locations that span the same visual angle regardless of model visual angle (to keep microsaccade extent constant across models). The function returns a list of `number_of_trials` tuples that each contain a microsaccade location, expanding from the center of the image. - """ - n = int(np.ceil(np.sqrt(number_of_trials))) # upper bound on number of microsaccades needed - - # generate grid of potential microsaccades - xv, yv = np.meshgrid(range(-n, n + 1), range(-n, n + 1)) - coords = np.vstack([xv.ravel(), yv.ravel()]).T - # sort microsaccade locations: (0, 0) should be first, and e.g. (2, 3) before (-5, 6) (absolute value). - sort_criteria = np.maximum(np.abs(coords[:, 0]), np.abs(coords[:, 1])) + np.abs(coords).sum(axis=1) / 1000 - sorted_indices = np.argsort(sort_criteria) + Microsaccade locations are placed within a circle, evenly distributed across the entire area in a spiral, + from the center to the circumference. + """ + # avoid duplicating this computation repeatedly + if image_path in self.microsaccades.keys(): + return self.microsaccades[image_path][trial_number] + + if image_shape[0] != image_shape[1]: + self._logger.debug('Warning: input image is not a square. Image dimension 0 is used to calculate the ' + 'extent of microsaccades.') + + # compute the maximum radius of microsaccade extent in pixel space + if self._visual_degrees is None: + raise AssertionError('self._visual_degrees is not set by the ModelCommitment, but microsaccades ' + 'are in use. Set activations_model visual degrees in your commitment after defining ' + 'your activations_model. For example, self.activations_model.set_visual_degrees' + '(visual_degrees). For more information, see https://github.com/brain-score/vision/' + 'blob/master/brainscore_vision/model_helpers/brain_transformation/__init__.py#L26') + radius_ratio = self.microsaccade_extent_degrees / self._visual_degrees + max_radius = 0.5 * radius_ratio * image_shape[0] # half width of the image as the maximum radius + + selected_microsaccades = [] + # microsaccades are placed in a spiral at sub-pixel increments + a = max_radius / np.sqrt(self.number_of_trials) # spiral coefficient to space microsaccades evenly + for i in range(self.number_of_trials): + r = np.sqrt(i / self.number_of_trials) * max_radius # compute radial distance for the i-th point + theta = a * np.sqrt(i) * 2 * np.pi / max_radius # compute angle for the i-th point + + # convert polar coordinates to Cartesian, centered on the image + x = r * np.cos(theta) + y = r * np.sin(theta) + selected_microsaccades.append((x, y)) + + # to keep consistent with number_of_trials, we count trial_number from 1 instead of from 0 + self.microsaccades[image_path] = selected_microsaccades + return selected_microsaccades[trial_number - 1] - # select the first `number_of_trials` microsaccades from the sorted upper bound - selected_microsaccades = [tuple(coord) for coord in coords[sorted_indices][:number_of_trials]] - return selected_microsaccades + @staticmethod + def remove_temporary_files(temporary_file_paths: List[str]) -> None: + """ + This function is used to manually remove all temporary file paths. We do this instead of using implicit + python garbage collection to 1) ensure that tensorflow models have access to temporary files when needed; + 2) to make the point at which temporary files are removed explicit. + """ + for temporary_file_path in temporary_file_paths: + if isinstance(temporary_file_path, str): # do not try to remove loaded images + try: + os.remove(temporary_file_path) + except FileNotFoundError: + pass @staticmethod - def translate(image, shift): - rows, cols, _ = image.shape + def translate(image, shift: Tuple[float, float], image_shape: Tuple[int, int]): + rows, cols = image_shape # translation matrix M = np.float32([[1, 0, shift[0]], [0, 1, shift[1]]]) # Apply translation, filling new line(s) with line(s) closest to it(them). - translated_image = cv2.warpAffine(image, M, (cols, rows), borderMode=cv2.BORDER_REPLICATE) + translated_image = cv2.warpAffine(image, M, (cols, rows), flags=cv2.INTER_LINEAR, # for sub-pixel shifts + borderMode=cv2.BORDER_REPLICATE) return translated_image + @staticmethod + def get_image_with_shape(image: Union[str, np.ndarray]): + if isinstance(image, str): # tf models return strings after preprocessing + image = cv2.imread(image) + rows, cols, _ = image.shape + return image, (rows, cols) + + @staticmethod + def reshape_microsaccaded_images(images: List) -> Union[List[str], np.ndarray]: + if any(isinstance(image, str) for image in images): + return images + return np.stack(images, axis=0) + def change_dict(d, change_function, keep_name=False, multithread=False): if not multithread: @@ -436,64 +477,54 @@ def lstrip_local(path): return path -def attach_stimulus_set_meta(assembly, stimulus_set): - stimulus_paths = [str(stimulus_set.get_stimulus(stimulus_id)) for stimulus_id in stimulus_set['stimulus_id']] - stimulus_paths = [lstrip_local(path) for path in stimulus_paths] - assembly_paths = [lstrip_local(path) for path in assembly['stimulus_path'].values] - - assert (np.array(assembly_paths) == np.array(stimulus_paths)).all() - assembly = assembly.reset_index('presentation') - assembly.assign_coords(stimulus_path=('presentation', stimulus_set['stimulus_id'].values)) - - assembly = assembly.rename({'stimulus_path': 'stimulus_id'}) - - all_columns = [] - for column in stimulus_set.columns: - assembly = assembly.assign_coords({column: ('presentation', stimulus_set[column].values)}) - all_columns.append(column) - if 'stimulus_id' in all_columns: - all_columns.remove('stimulus_id') - if 'stimulus_path2' in all_columns: - all_columns.remove('stimulus_path2') - assembly = assembly.set_index(presentation=['stimulus_id', 'stimulus_path2'] + all_columns) - return assembly +def unpack_microsaccade_coords(microsaccades: Dict[str, List], stimuli_paths: np.ndarray, dim: int): + """Unpacks microsaccades from stimuli_paths into a single list to conform with coord requirements.""" + unpacked_microsaccades = [] + for stimulus_path in stimuli_paths: + for microsaccade in microsaccades[stimulus_path]: + unpacked_microsaccades.append(microsaccade[dim]) + return unpacked_microsaccades -def attach_stimulus_set_meta_with_microsaccades(assembly, stimulus_set, number_of_trials, shifts): +def attach_stimulus_set_meta(assembly, stimulus_set, number_of_trials: int, microsaccades: Dict[str, List], + require_variance: bool = False): stimulus_paths = [str(stimulus_set.get_stimulus(stimulus_id)) for stimulus_id in stimulus_set['stimulus_id']] stimulus_paths = [lstrip_local(path) for path in stimulus_paths] assembly_paths = [lstrip_local(path) for path in assembly['stimulus_path'].values] - replication_factor = number_of_trials + # when microsaccades are used, we repeat stimulus_paths number_of_trials times to correctly populate the dim + if require_variance: + replication_factor = number_of_trials + else: + replication_factor = 1 repeated_stimulus_paths = np.repeat(stimulus_paths, replication_factor) assert (np.array(assembly_paths) == np.array(repeated_stimulus_paths)).all() repeated_stimulus_ids = np.repeat(stimulus_set['stimulus_id'].values, replication_factor) - # repeat over the presentation dimension to accommodate multiple runs per stimulus - repeated_assembly = xr.concat([assembly for _ in range(replication_factor)], dim='presentation') - repeated_assembly = repeated_assembly.reset_index('presentation') - repeated_assembly.assign_coords(stimulus_path=('presentation', repeated_stimulus_ids)) - repeated_assembly = repeated_assembly.rename({'stimulus_path': 'stimulus_id'}) + if replication_factor > 1: + # repeat over the presentation dimension to accommodate multiple runs per stimulus + assembly = xr.concat([assembly for _ in range(replication_factor)], dim='presentation') + assembly = assembly.reset_index('presentation') + assembly['stimulus_path'] = ('presentation', repeated_stimulus_ids) + assembly = assembly.rename({'stimulus_path': 'stimulus_id'}) + + assert (np.array(assembly_paths) == np.array(stimulus_paths)).all() - # hack to capture columns all_columns = [] for column in stimulus_set.columns: repeated_values = np.repeat(stimulus_set[column].values, replication_factor) - repeated_assembly = repeated_assembly.assign_coords({column: ('presentation', repeated_values)}) + assembly = assembly.assign_coords({column: ('presentation', repeated_values)}) # assign multiple coords at once all_columns.append(column) - if 'stimulus_id' in all_columns: - all_columns.remove('stimulus_id') - if 'stimulus_path2' in all_columns: - all_columns.remove('stimulus_path2') - - repeated_assembly.coords['shift_x'] = ('presentation', [shift[0] for shift in shifts]) - repeated_assembly.coords['shift_y'] = ('presentation', [shift[1] for shift in shifts]) - # Set MultiIndex - index = ['stimulus_id', 'stimulus_path2'] + all_columns + ['shift_x', 'shift_y'] - repeated_assembly = repeated_assembly.set_index(presentation=index) + stimuli_paths = list(microsaccades.keys()) + assembly.coords['shift_x'] = ('presentation', unpack_microsaccade_coords(microsaccades, + np.unique(stimuli_paths), dim=0)) + assembly.coords['shift_y'] = ('presentation', unpack_microsaccade_coords(microsaccades, + np.unique(stimuli_paths), dim=1)) - return repeated_assembly + index = all_columns + ['shift_x', 'shift_y'] + assembly = assembly.set_index(presentation=index) # assign MultiIndex + return assembly class HookHandle: diff --git a/brainscore_vision/model_helpers/brain_transformation/__init__.py b/brainscore_vision/model_helpers/brain_transformation/__init__.py index 47938c6d3..4da9050aa 100644 --- a/brainscore_vision/model_helpers/brain_transformation/__init__.py +++ b/brainscore_vision/model_helpers/brain_transformation/__init__.py @@ -24,6 +24,8 @@ def __init__(self, identifier, visual_degrees=8): self.layers = layers self.activations_model = activations_model + # We set the visual degrees of the ActivationsExtractorHelper here to avoid changing its signature + self.activations_model.set_visual_degrees(visual_degrees) # for microsaccades self._visual_degrees = visual_degrees # region-layer mapping if region_layer_map is None: diff --git a/brainscore_vision/model_helpers/brain_transformation/neural.py b/brainscore_vision/model_helpers/brain_transformation/neural.py index fbe78d10e..520cb32a4 100644 --- a/brainscore_vision/model_helpers/brain_transformation/neural.py +++ b/brainscore_vision/model_helpers/brain_transformation/neural.py @@ -1,5 +1,6 @@ import logging +import numpy as np from result_caching import store_xarray, store from tqdm import tqdm @@ -26,26 +27,9 @@ def identifier(self): def look_at(self, stimuli, number_of_trials=1, require_variance: bool = False): """ :param number_of_trials: An integer that determines how many repetitions of the same model performs. - :param require_variance: A bool that asks models to output different responses to the same stimuli (i.e., - allows stochastic responses to identical stimuli, even in deterministic models). The current implementation - implements this using microsaccades. - Human microsaccade amplitude varies by who you ask, an estimate might be <0.1 deg = 360 arcsec = 6arcmin. - The goal of microsaccades is to obtain multiple different neural activities to the same input stimulus - from non-stochastic models. This is to improve estimates of e.g. psychophysical functions, but also other - things. Note that microsaccades are also applied to stochastic models to make them comparable within- - benchmark to non-stochastic models. - In the current implementation, if `require_variance=True`, the model selects microsaccades according to - its own microsaccade behavior (if it has implemented it), or with the base behavior of saccading in - input pixel space with 1-pixel increments from the center of the stimulus. The base behavior thus - maintains a fixed microsaccade distance as measured in visual angle, regardless of the model's visual angle. - Example usage: - require_variance = True - More information: - --> Rolfs 2009 "Microsaccades: Small steps on a long way" Vision Research, Volume 49, Issue 20, 15 - October 2009, Pages 2415-2441. - --> Haddad & Steinmann 1973 "The smallest voluntary saccade: Implications for fixation" Vision - Research Volume 13, Issue 6, June 1973, Pages 1075-1086, IN5-IN6. - Thanks to Johannes Mehrer for initial help in implementing microsaccades. + :param require_variance: Whether to require models to return different activations for the same stimuli or not. + For detailed information, see https://github.com/brain-score/vision/blob/master/ + brainscore_vision/model_helpers/activations/core.py#L26 """ layer_regions = {} for region in self.recorded_regions: diff --git a/tests/test_model_helpers/activations/test___init__.py b/tests/test_model_helpers/activations/test___init__.py index 2032d6cbf..c71debd0c 100644 --- a/tests/test_model_helpers/activations/test___init__.py +++ b/tests/test_model_helpers/activations/test___init__.py @@ -196,6 +196,9 @@ def test_from_image_path(model_ctr, layers, image_name, pca_components, logits): def test_require_variance_has_shift_coords(model_ctr, layers, image_name, number_of_trials): stimulus_paths = [os.path.join(os.path.dirname(__file__), image_name)] activations_extractor = model_ctr() + # when using microsaccades, the ModelCommitment sets its visual angle. Since this test skips the ModelCommitment, + # we set it here manually. + activations_extractor._extractor.set_visual_degrees(8.) activations = activations_extractor(stimuli=stimulus_paths, layers=layers, number_of_trials=number_of_trials, require_variance=True) @@ -209,20 +212,22 @@ def test_require_variance_has_shift_coords(model_ctr, layers, image_name, number 'palletized.png']) @pytest.mark.parametrize(["model_ctr", "layers"], models_layers) @pytest.mark.parametrize("require_variance", [False, True]) -def test_model_requirements(model_ctr, layers, image_name, require_variance): +@pytest.mark.parametrize("number_of_trials", [1, 2, 10]) +def test_model_requirements(model_ctr, layers, image_name, require_variance, number_of_trials): stimulus_paths = [os.path.join(os.path.dirname(__file__), image_name)] activations_extractor = model_ctr() + # when using microsaccades, the ModelCommitment sets its visual angle. Since this test skips the ModelCommitment, + # we set it here manually. + activations_extractor._extractor.set_visual_degrees(8.) - activations_with_req = activations_extractor(stimuli=stimulus_paths, layers=layers, - number_of_trials=2, require_variance=require_variance) - activations_without_req = activations_extractor(stimuli=stimulus_paths, layers=layers, - number_of_trials=1, require_variance=False) + activations = activations_extractor(stimuli=stimulus_paths, layers=layers, + number_of_trials=number_of_trials, require_variance=require_variance) - assert activations_with_req is not None - assert activations_without_req is not None + assert activations is not None if require_variance: - assert len(activations_with_req['presentation']) > len(activations_without_req['presentation']) - assert len(activations_with_req['neuroid']) == len(activations_without_req['neuroid']) + assert len(activations['presentation']) == number_of_trials + else: + assert len(activations['presentation']) == 1 @pytest.mark.parametrize("image_name", ['rgb.jpg', 'grayscale.png', 'grayscale2.jpg', 'grayscale_alpha.png', From e79fbd24fb151f2e5c62e7edc7dbbe561dfa43fc Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Fri, 16 Feb 2024 14:44:10 +0100 Subject: [PATCH 15/41] remove needless import --- brainscore_vision/model_helpers/brain_transformation/neural.py | 1 - 1 file changed, 1 deletion(-) diff --git a/brainscore_vision/model_helpers/brain_transformation/neural.py b/brainscore_vision/model_helpers/brain_transformation/neural.py index 520cb32a4..05a6f3592 100644 --- a/brainscore_vision/model_helpers/brain_transformation/neural.py +++ b/brainscore_vision/model_helpers/brain_transformation/neural.py @@ -1,6 +1,5 @@ import logging -import numpy as np from result_caching import store_xarray, store from tqdm import tqdm From a17c653f6da133296d4ea1f3ce427db42e9d804c Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Mon, 19 Feb 2024 10:24:51 +0100 Subject: [PATCH 16/41] fix bug with temporary file handling test --- tests/test_model_helpers/activations/test___init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_model_helpers/activations/test___init__.py b/tests/test_model_helpers/activations/test___init__.py index c71debd0c..07ff113e5 100644 --- a/tests/test_model_helpers/activations/test___init__.py +++ b/tests/test_model_helpers/activations/test___init__.py @@ -237,6 +237,9 @@ def test_temporary_file_handling(model_ctr, layers, image_name): import tempfile stimulus_paths = [os.path.join(os.path.dirname(__file__), image_name)] activations_extractor = model_ctr() + # when using microsaccades, the ModelCommitment sets its visual angle. Since this test skips the ModelCommitment, + # we set it here manually. + activations_extractor._extractor.set_visual_degrees(8.) activations = activations_extractor(stimuli=stimulus_paths, layers=layers, number_of_trials=2, require_variance=True) From 4887728b69d7c0d7bc39660e0006a24fc633673f Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Mon, 19 Feb 2024 10:39:45 +0100 Subject: [PATCH 17/41] assume number_of_trials=1 and require_variance=False when getting stored activations --- brainscore_vision/model_helpers/activations/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brainscore_vision/model_helpers/activations/core.py b/brainscore_vision/model_helpers/activations/core.py index 9949dbe07..d19150e51 100644 --- a/brainscore_vision/model_helpers/activations/core.py +++ b/brainscore_vision/model_helpers/activations/core.py @@ -126,7 +126,7 @@ def from_paths(self, stimuli_paths, layers, stimuli_identifier=None, require_var @store_xarray(identifier_ignore=['stimuli_paths', 'layers'], combine_fields={'layers': 'layer'}) def _from_paths_stored(self, identifier, layers, stimuli_identifier, - stimuli_paths, number_of_trials, require_variance): + stimuli_paths, number_of_trials: int = 1, require_variance: bool = False): return self._from_paths(layers=layers, stimuli_paths=stimuli_paths) def _from_paths(self, layers, stimuli_paths, require_variance: bool = False): From 9a7368e72cbe6a30adcaf2d5b0ccb0b6b634db5d Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Mon, 19 Feb 2024 10:40:34 +0100 Subject: [PATCH 18/41] fix bug with access to ActivationsExtractorHelper.set_visual_degrees --- brainscore_vision/model_helpers/activations/keras.py | 4 ++++ brainscore_vision/model_helpers/activations/pytorch.py | 4 ++++ brainscore_vision/model_helpers/activations/tensorflow.py | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/brainscore_vision/model_helpers/activations/keras.py b/brainscore_vision/model_helpers/activations/keras.py index 8d1acf4d7..d86a3b77e 100644 --- a/brainscore_vision/model_helpers/activations/keras.py +++ b/brainscore_vision/model_helpers/activations/keras.py @@ -29,6 +29,10 @@ def identifier(self, value): def __call__(self, *args, **kwargs): # cannot assign __call__ as attribute due to Python convention return self._extractor(*args, **kwargs) + def set_visual_degrees(self, visual_degrees): + """A method to allow the ModelCommitment to set the visual angle of the ActivationsExtractorHelper.""" + self._extractor.set_visual_degrees(visual_degrees) + def get_activations(self, images, layer_names): from keras import backend as K input_tensor = self._model.input diff --git a/brainscore_vision/model_helpers/activations/pytorch.py b/brainscore_vision/model_helpers/activations/pytorch.py index 92ba6a857..4cab21e2c 100644 --- a/brainscore_vision/model_helpers/activations/pytorch.py +++ b/brainscore_vision/model_helpers/activations/pytorch.py @@ -40,6 +40,10 @@ def identifier(self, value): def __call__(self, *args, **kwargs): # cannot assign __call__ as attribute due to Python convention return self._extractor(*args, **kwargs) + def set_visual_degrees(self, visual_degrees): + """A method to allow the ModelCommitment to set the visual angle of the ActivationsExtractorHelper.""" + self._extractor.set_visual_degrees(visual_degrees) + def get_activations(self, images, layer_names): import torch from torch.autograd import Variable diff --git a/brainscore_vision/model_helpers/activations/tensorflow.py b/brainscore_vision/model_helpers/activations/tensorflow.py index d5e4864d5..eb3bfd03e 100644 --- a/brainscore_vision/model_helpers/activations/tensorflow.py +++ b/brainscore_vision/model_helpers/activations/tensorflow.py @@ -24,6 +24,10 @@ def identifier(self, value): def __call__(self, *args, **kwargs): # cannot assign __call__ as attribute due to Python convention return self._extractor(*args, **kwargs) + def set_visual_degrees(self, visual_degrees): + """A method to allow the ModelCommitment to set the visual angle of the ActivationsExtractorHelper.""" + self._extractor.set_visual_degrees(visual_degrees) + def get_activations(self, images, layer_names): layer_tensors = OrderedDict((layer, self._endpoints[ layer if (layer != 'logits' or layer in self._endpoints) else next(reversed(self._endpoints))]) From b3e08fbac9260e86f70e7f1e340408d40cb5a3da Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Mon, 19 Feb 2024 13:24:05 +0100 Subject: [PATCH 19/41] move extractor calls to ModelCommitment generic --- brainscore_vision/model_helpers/activations/keras.py | 4 ---- brainscore_vision/model_helpers/activations/pytorch.py | 4 ---- brainscore_vision/model_helpers/activations/tensorflow.py | 4 ---- .../model_helpers/brain_transformation/__init__.py | 2 +- 4 files changed, 1 insertion(+), 13 deletions(-) diff --git a/brainscore_vision/model_helpers/activations/keras.py b/brainscore_vision/model_helpers/activations/keras.py index d86a3b77e..8d1acf4d7 100644 --- a/brainscore_vision/model_helpers/activations/keras.py +++ b/brainscore_vision/model_helpers/activations/keras.py @@ -29,10 +29,6 @@ def identifier(self, value): def __call__(self, *args, **kwargs): # cannot assign __call__ as attribute due to Python convention return self._extractor(*args, **kwargs) - def set_visual_degrees(self, visual_degrees): - """A method to allow the ModelCommitment to set the visual angle of the ActivationsExtractorHelper.""" - self._extractor.set_visual_degrees(visual_degrees) - def get_activations(self, images, layer_names): from keras import backend as K input_tensor = self._model.input diff --git a/brainscore_vision/model_helpers/activations/pytorch.py b/brainscore_vision/model_helpers/activations/pytorch.py index 4cab21e2c..92ba6a857 100644 --- a/brainscore_vision/model_helpers/activations/pytorch.py +++ b/brainscore_vision/model_helpers/activations/pytorch.py @@ -40,10 +40,6 @@ def identifier(self, value): def __call__(self, *args, **kwargs): # cannot assign __call__ as attribute due to Python convention return self._extractor(*args, **kwargs) - def set_visual_degrees(self, visual_degrees): - """A method to allow the ModelCommitment to set the visual angle of the ActivationsExtractorHelper.""" - self._extractor.set_visual_degrees(visual_degrees) - def get_activations(self, images, layer_names): import torch from torch.autograd import Variable diff --git a/brainscore_vision/model_helpers/activations/tensorflow.py b/brainscore_vision/model_helpers/activations/tensorflow.py index eb3bfd03e..d5e4864d5 100644 --- a/brainscore_vision/model_helpers/activations/tensorflow.py +++ b/brainscore_vision/model_helpers/activations/tensorflow.py @@ -24,10 +24,6 @@ def identifier(self, value): def __call__(self, *args, **kwargs): # cannot assign __call__ as attribute due to Python convention return self._extractor(*args, **kwargs) - def set_visual_degrees(self, visual_degrees): - """A method to allow the ModelCommitment to set the visual angle of the ActivationsExtractorHelper.""" - self._extractor.set_visual_degrees(visual_degrees) - def get_activations(self, images, layer_names): layer_tensors = OrderedDict((layer, self._endpoints[ layer if (layer != 'logits' or layer in self._endpoints) else next(reversed(self._endpoints))]) diff --git a/brainscore_vision/model_helpers/brain_transformation/__init__.py b/brainscore_vision/model_helpers/brain_transformation/__init__.py index 4da9050aa..f67cc6a58 100644 --- a/brainscore_vision/model_helpers/brain_transformation/__init__.py +++ b/brainscore_vision/model_helpers/brain_transformation/__init__.py @@ -25,7 +25,7 @@ def __init__(self, identifier, self.layers = layers self.activations_model = activations_model # We set the visual degrees of the ActivationsExtractorHelper here to avoid changing its signature - self.activations_model.set_visual_degrees(visual_degrees) # for microsaccades + self.activations_model._extractor.set_visual_degrees(visual_degrees) # for microsaccades self._visual_degrees = visual_degrees # region-layer mapping if region_layer_map is None: From 849cc7afb66a2b8f952bdc7763741cc7f242c1bc Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Mon, 19 Feb 2024 13:28:54 +0100 Subject: [PATCH 20/41] add check for whether activations_model exists --- .../model_helpers/brain_transformation/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/brainscore_vision/model_helpers/brain_transformation/__init__.py b/brainscore_vision/model_helpers/brain_transformation/__init__.py index f67cc6a58..9bf02f56c 100644 --- a/brainscore_vision/model_helpers/brain_transformation/__init__.py +++ b/brainscore_vision/model_helpers/brain_transformation/__init__.py @@ -25,7 +25,8 @@ def __init__(self, identifier, self.layers = layers self.activations_model = activations_model # We set the visual degrees of the ActivationsExtractorHelper here to avoid changing its signature - self.activations_model._extractor.set_visual_degrees(visual_degrees) # for microsaccades + if activations_model is not None: + self.activations_model._extractor.set_visual_degrees(visual_degrees) # for microsaccades self._visual_degrees = visual_degrees # region-layer mapping if region_layer_map is None: From 114110c203d03760ec8d1409121a0a643ecde6bb Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Mon, 19 Feb 2024 15:21:13 +0100 Subject: [PATCH 21/41] fix bug with TestVisualDegrees --- .../brain_transformation/test___init__.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/test_model_helpers/brain_transformation/test___init__.py b/tests/test_model_helpers/brain_transformation/test___init__.py index 49fb55b1c..b61a201b8 100644 --- a/tests/test_model_helpers/brain_transformation/test___init__.py +++ b/tests/test_model_helpers/brain_transformation/test___init__.py @@ -1,9 +1,18 @@ +from unittest.mock import Mock + from brainscore_vision.model_helpers.brain_transformation import ModelCommitment from brainscore_vision.model_helpers.utils import fullname class TestVisualDegrees: def test_standard_commitment(self): - brain_model = ModelCommitment(identifier=fullname(self), activations_model=None, + # create mock ActivationsExtractorHelper with a mock set_visual_degrees to avoid failing set_visual_degrees() + mock_extractor = Mock() + mock_extractor.set_visual_degrees = Mock() + mock_activations_model = Mock() + mock_activations_model._extractor = mock_extractor + + # Initialize ModelCommitment with the mock activations_model + brain_model = ModelCommitment(identifier=fullname(self), activations_model=mock_activations_model, layers=['dummy']) assert brain_model.visual_degrees() == 8 From 218deadf15b2f3d760f097ce64eb50035b1f032c Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Tue, 20 Feb 2024 10:45:22 +0100 Subject: [PATCH 22/41] add link to BrainModel issue --- .../model_helpers/brain_transformation/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/brainscore_vision/model_helpers/brain_transformation/__init__.py b/brainscore_vision/model_helpers/brain_transformation/__init__.py index 9bf02f56c..90df82460 100644 --- a/brainscore_vision/model_helpers/brain_transformation/__init__.py +++ b/brainscore_vision/model_helpers/brain_transformation/__init__.py @@ -24,9 +24,10 @@ def __init__(self, identifier, visual_degrees=8): self.layers = layers self.activations_model = activations_model - # We set the visual degrees of the ActivationsExtractorHelper here to avoid changing its signature - if activations_model is not None: - self.activations_model._extractor.set_visual_degrees(visual_degrees) # for microsaccades + # We set the visual degrees of the ActivationsExtractorHelper here to avoid changing its signature. + # The ideal solution would be to not expose the _extractor of the activations_model here, but to change + # the signature of the ActivationsExtractorHelper. See https://github.com/brain-score/vision/issues/554 + self.activations_model._extractor.set_visual_degrees(visual_degrees) # for microsaccades self._visual_degrees = visual_degrees # region-layer mapping if region_layer_map is None: From ce56473bf3be50ece2f422643e5eff3ba9e2e261 Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Tue, 20 Feb 2024 11:00:28 +0100 Subject: [PATCH 23/41] remove shifts from stimulus set packaging --- brainscore_vision/model_helpers/activations/core.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/brainscore_vision/model_helpers/activations/core.py b/brainscore_vision/model_helpers/activations/core.py index d19150e51..aca67bf45 100644 --- a/brainscore_vision/model_helpers/activations/core.py +++ b/brainscore_vision/model_helpers/activations/core.py @@ -516,12 +516,6 @@ def attach_stimulus_set_meta(assembly, stimulus_set, number_of_trials: int, micr assembly = assembly.assign_coords({column: ('presentation', repeated_values)}) # assign multiple coords at once all_columns.append(column) - stimuli_paths = list(microsaccades.keys()) - assembly.coords['shift_x'] = ('presentation', unpack_microsaccade_coords(microsaccades, - np.unique(stimuli_paths), dim=0)) - assembly.coords['shift_y'] = ('presentation', unpack_microsaccade_coords(microsaccades, - np.unique(stimuli_paths), dim=1)) - index = all_columns + ['shift_x', 'shift_y'] assembly = assembly.set_index(presentation=index) # assign MultiIndex return assembly From 65c658bbbd01d46f327e508cc9a7551641355f29 Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Tue, 20 Feb 2024 11:01:27 +0100 Subject: [PATCH 24/41] change link signatures --- brainscore_vision/model_helpers/activations/core.py | 5 +++-- .../model_helpers/brain_transformation/neural.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/brainscore_vision/model_helpers/activations/core.py b/brainscore_vision/model_helpers/activations/core.py index aca67bf45..67833388a 100644 --- a/brainscore_vision/model_helpers/activations/core.py +++ b/brainscore_vision/model_helpers/activations/core.py @@ -388,8 +388,9 @@ def select_microsaccade(self, image_path: str, trial_number: int, image_shape: T raise AssertionError('self._visual_degrees is not set by the ModelCommitment, but microsaccades ' 'are in use. Set activations_model visual degrees in your commitment after defining ' 'your activations_model. For example, self.activations_model.set_visual_degrees' - '(visual_degrees). For more information, see https://github.com/brain-score/vision/' - 'blob/master/brainscore_vision/model_helpers/brain_transformation/__init__.py#L26') + '(visual_degrees). For detailed information, see ' + ':meth:`~brainscore_vision.model_helpers.activations.ActivationsExtractorHelper.' + '__call__`,') radius_ratio = self.microsaccade_extent_degrees / self._visual_degrees max_radius = 0.5 * radius_ratio * image_shape[0] # half width of the image as the maximum radius diff --git a/brainscore_vision/model_helpers/brain_transformation/neural.py b/brainscore_vision/model_helpers/brain_transformation/neural.py index 05a6f3592..5148bb9b7 100644 --- a/brainscore_vision/model_helpers/brain_transformation/neural.py +++ b/brainscore_vision/model_helpers/brain_transformation/neural.py @@ -27,8 +27,8 @@ def look_at(self, stimuli, number_of_trials=1, require_variance: bool = False): """ :param number_of_trials: An integer that determines how many repetitions of the same model performs. :param require_variance: Whether to require models to return different activations for the same stimuli or not. - For detailed information, see https://github.com/brain-score/vision/blob/master/ - brainscore_vision/model_helpers/activations/core.py#L26 + For detailed information, see + :meth:`~brainscore_vision.model_helpers.activations.ActivationsExtractorHelper.__call__`, """ layer_regions = {} for region in self.recorded_regions: From 22d2ecb8b5c84a1e8ff1842b97965e4375a6d991 Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Tue, 20 Feb 2024 11:14:00 +0100 Subject: [PATCH 25/41] change microsaccade call signature --- .../model_helpers/activations/core.py | 15 ++++++++------- .../activations/test___init__.py | 4 ++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/brainscore_vision/model_helpers/activations/core.py b/brainscore_vision/model_helpers/activations/core.py index 67833388a..fac1e90f9 100644 --- a/brainscore_vision/model_helpers/activations/core.py +++ b/brainscore_vision/model_helpers/activations/core.py @@ -339,10 +339,12 @@ def _package_layer(self, layer_activations, layer, stimuli_paths, require_varian # build assembly coords = {'stimulus_path': ('presentation', stimuli_paths), - 'shift_x': ('presentation', unpack_microsaccade_coords(self.microsaccades, - np.unique(stimuli_paths), dim=0)), - 'shift_y': ('presentation', unpack_microsaccade_coords(self.microsaccades, - np.unique(stimuli_paths), dim=1)), + 'microsaccade_shift_x_pixels': ('presentation', unpack_microsaccade_coords(self.microsaccades, + np.unique(stimuli_paths), + dim=0)), + 'microsaccade_shift_y_pixels': ('presentation', unpack_microsaccade_coords(self.microsaccades, + np.unique(stimuli_paths), + dim=1)), 'neuroid_num': ('neuroid', list(range(activations.shape[1]))), 'model': ('neuroid', [self.identifier] * activations.shape[1]), 'layer': ('neuroid', [layer] * activations.shape[1]), @@ -487,8 +489,7 @@ def unpack_microsaccade_coords(microsaccades: Dict[str, List], stimuli_paths: np return unpacked_microsaccades -def attach_stimulus_set_meta(assembly, stimulus_set, number_of_trials: int, microsaccades: Dict[str, List], - require_variance: bool = False): +def attach_stimulus_set_meta(assembly, stimulus_set, number_of_trials: int, require_variance: bool = False): stimulus_paths = [str(stimulus_set.get_stimulus(stimulus_id)) for stimulus_id in stimulus_set['stimulus_id']] stimulus_paths = [lstrip_local(path) for path in stimulus_paths] assembly_paths = [lstrip_local(path) for path in assembly['stimulus_path'].values] @@ -517,7 +518,7 @@ def attach_stimulus_set_meta(assembly, stimulus_set, number_of_trials: int, micr assembly = assembly.assign_coords({column: ('presentation', repeated_values)}) # assign multiple coords at once all_columns.append(column) - index = all_columns + ['shift_x', 'shift_y'] + index = all_columns + ['microsaccade_shift_x_pixels', 'microsaccade_shift_y_pixels'] assembly = assembly.set_index(presentation=index) # assign MultiIndex return assembly diff --git a/tests/test_model_helpers/activations/test___init__.py b/tests/test_model_helpers/activations/test___init__.py index 07ff113e5..dba05c676 100644 --- a/tests/test_model_helpers/activations/test___init__.py +++ b/tests/test_model_helpers/activations/test___init__.py @@ -204,8 +204,8 @@ def test_require_variance_has_shift_coords(model_ctr, layers, image_name, number require_variance=True) assert activations is not None - assert len(activations['shift_x']) == number_of_trials * len(stimulus_paths) - assert len(activations['shift_y']) == number_of_trials * len(stimulus_paths) + assert len(activations['microsaccade_shift_x_pixels']) == number_of_trials * len(stimulus_paths) + assert len(activations['microsaccade_shift_y_pixels']) == number_of_trials * len(stimulus_paths) @pytest.mark.parametrize("image_name", ['rgb.jpg', 'grayscale.png', 'grayscale2.jpg', 'grayscale_alpha.png', From 2e96e73cddffd8d4460a761d268048f39becc448 Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Tue, 20 Feb 2024 12:09:30 +0100 Subject: [PATCH 26/41] microsaccades are now computed on both a pixel and degree basis --- .../model_helpers/activations/core.py | 96 ++++++++++++------- .../activations/test___init__.py | 2 + 2 files changed, 65 insertions(+), 33 deletions(-) diff --git a/brainscore_vision/model_helpers/activations/core.py b/brainscore_vision/model_helpers/activations/core.py index fac1e90f9..200511931 100644 --- a/brainscore_vision/model_helpers/activations/core.py +++ b/brainscore_vision/model_helpers/activations/core.py @@ -42,7 +42,7 @@ def __init__(self, get_activations, preprocessing, identifier=False, batch_size= # a dict that contains the image paths and their respective microsaccades. e.g., # {'abc.jpg': [(0, 0), (1.5, 2)]} - self.microsaccades = {} + self.microsaccades = {'pixels': {}, 'degrees': {}} self._visual_degrees = None # Model visual degrees. Used for computing microsaccades def __call__(self, stimuli, layers, stimuli_identifier=None, number_of_trials: int = 1, @@ -98,7 +98,7 @@ def from_stimulus_set(self, stimulus_set, layers, stimuli_identifier=None, requi stimuli_paths = [str(stimulus_set.get_stimulus(stimulus_id)) for stimulus_id in stimulus_set['stimulus_id']] activations = self.from_paths(stimuli_paths=stimuli_paths, layers=layers, stimuli_identifier=stimuli_identifier) activations = attach_stimulus_set_meta(activations, stimulus_set, number_of_trials=self.number_of_trials, - microsaccades=self.microsaccades, require_variance=require_variance) + require_variance=require_variance) return activations def from_paths(self, stimuli_paths, layers, stimuli_identifier=None, require_variance=None): @@ -243,16 +243,18 @@ def translate_images(self, images: List[Union[str, np.ndarray]], image_paths: Li # idea to introduce cv2 image loading for all models and images, regardless of whether they are actually # microsaccading. if not require_variance: - self.microsaccades[image_path] = [(0., 0.)] + self.microsaccades['pixels'][image_path] = [(0., 0.)] + self.microsaccades['degrees'][image_path] = [(0., 0.)] output_images.append(images[index]) else: # translate images according to microsaccades if we are using microsaccades image, image_shape = self.get_image_with_shape(images[index]) - microsaccade_location = self.select_microsaccade(image_path=image_path, - trial_number=trial_number, - image_shape=image_shape) + microsaccade_location_pixels = self.select_microsaccade(image_path=image_path, + trial_number=trial_number, + image_shape=image_shape) return_string = True if isinstance(images[index], str) else False - output_images.append(self.translate_image(image, microsaccade_location, image_shape, return_string)) + output_images.append(self.translate_image(image, microsaccade_location_pixels, image_shape, + return_string)) return self.reshape_microsaccaded_images(output_images) def translate_image(self, image: str, microsaccade_location: Tuple[float, float], image_shape: Tuple[int, int], @@ -339,12 +341,22 @@ def _package_layer(self, layer_activations, layer, stimuli_paths, require_varian # build assembly coords = {'stimulus_path': ('presentation', stimuli_paths), - 'microsaccade_shift_x_pixels': ('presentation', unpack_microsaccade_coords(self.microsaccades, - np.unique(stimuli_paths), - dim=0)), - 'microsaccade_shift_y_pixels': ('presentation', unpack_microsaccade_coords(self.microsaccades, - np.unique(stimuli_paths), - dim=1)), + 'microsaccade_shift_x_pixels': ('presentation', self.unpack_microsaccade_coords( + np.unique(stimuli_paths), + pixels_or_degrees='pixels', + dim=0)), + 'microsaccade_shift_y_pixels': ('presentation', self.unpack_microsaccade_coords( + np.unique(stimuli_paths), + pixels_or_degrees='pixels', + dim=1)), + 'microsaccade_shift_x_degrees': ('presentation', self.unpack_microsaccade_coords( + np.unique(stimuli_paths), + pixels_or_degrees='degrees', + dim=0)), + 'microsaccade_shift_y_degrees': ('presentation', self.unpack_microsaccade_coords( + np.unique(stimuli_paths), + pixels_or_degrees='degrees', + dim=1)), 'neuroid_num': ('neuroid', list(range(activations.shape[1]))), 'model': ('neuroid', [self.identifier] * activations.shape[1]), 'layer': ('neuroid', [layer] * activations.shape[1]), @@ -370,16 +382,16 @@ def insert_attrs(self, wrapper): def select_microsaccade(self, image_path: str, trial_number: int, image_shape: Tuple[int, int] ) -> Tuple[float, float]: """ - A naive function for generating microsaccade locations that span the same visual angle regardless of model - visual angle (to keep microsaccade extent constant across models). The function returns a list of - `number_of_trials` tuples that each contain a microsaccade location, expanding from the center of the image. + A function for generating microsaccade locations. The function returns a tuple of pixel shifts expanding from + the center of the image. Microsaccade locations are placed within a circle, evenly distributed across the entire area in a spiral, - from the center to the circumference. + from the center to the circumference. We keep track of microsaccades both on a pixel and visual angle basis, + but only pixel values are returned. This is because shifting the image using cv2 requires pixel representation. """ # avoid duplicating this computation repeatedly if image_path in self.microsaccades.keys(): - return self.microsaccades[image_path][trial_number] + return self.microsaccades['pixels'][image_path][trial_number] if image_shape[0] != image_shape[1]: self._logger.debug('Warning: input image is not a square. Image dimension 0 is used to calculate the ' @@ -394,9 +406,9 @@ def select_microsaccade(self, image_path: str, trial_number: int, image_shape: T ':meth:`~brainscore_vision.model_helpers.activations.ActivationsExtractorHelper.' '__call__`,') radius_ratio = self.microsaccade_extent_degrees / self._visual_degrees - max_radius = 0.5 * radius_ratio * image_shape[0] # half width of the image as the maximum radius + max_radius = radius_ratio * image_shape[0] # maximum radius in pixels, set in self.microsaccade_extent_degrees - selected_microsaccades = [] + selected_microsaccades = {'pixels': [], 'degrees': []} # microsaccades are placed in a spiral at sub-pixel increments a = max_radius / np.sqrt(self.number_of_trials) # spiral coefficient to space microsaccades evenly for i in range(self.number_of_trials): @@ -406,11 +418,37 @@ def select_microsaccade(self, image_path: str, trial_number: int, image_shape: T # convert polar coordinates to Cartesian, centered on the image x = r * np.cos(theta) y = r * np.sin(theta) - selected_microsaccades.append((x, y)) + + pixels_per_degree = self.calculate_pixels_per_degree_in_image(image_shape[0]) + selected_microsaccades['pixels'].append((x, y)) + selected_microsaccades['degrees'].append(self.convert_pixels_to_degrees((x, y), pixels_per_degree)) # to keep consistent with number_of_trials, we count trial_number from 1 instead of from 0 - self.microsaccades[image_path] = selected_microsaccades - return selected_microsaccades[trial_number - 1] + self.microsaccades['pixels'][image_path] = selected_microsaccades['pixels'] + self.microsaccades['degrees'][image_path] = selected_microsaccades['degrees'] + return selected_microsaccades['pixels'][trial_number - 1] + + def unpack_microsaccade_coords(self, stimuli_paths: np.ndarray, pixels_or_degrees: str, + dim: int): + """Unpacks microsaccades from stimuli_paths into a single list to conform with coord requirements.""" + # Python 3.7 does not support typing.Literal + assert pixels_or_degrees == 'pixels' or pixels_or_degrees == 'degrees' + unpacked_microsaccades = [] + for stimulus_path in stimuli_paths: + for microsaccade in self.microsaccades[pixels_or_degrees][stimulus_path]: + unpacked_microsaccades.append(microsaccade[dim]) + return unpacked_microsaccades + + def calculate_pixels_per_degree_in_image(self, image_width_pixels: int) -> float: + """Calculates the pixels per degree in the image, assuming the calculation based on image width.""" + pixels_per_degree = image_width_pixels / self._visual_degrees + return pixels_per_degree + + @staticmethod + def convert_pixels_to_degrees(pixel_coords: Tuple[float, float], pixels_per_degree: float) -> Tuple[float, float]: + degrees_x = pixel_coords[0] / pixels_per_degree + degrees_y = pixel_coords[1] / pixels_per_degree + return degrees_x, degrees_y @staticmethod def remove_temporary_files(temporary_file_paths: List[str]) -> None: @@ -480,15 +518,6 @@ def lstrip_local(path): return path -def unpack_microsaccade_coords(microsaccades: Dict[str, List], stimuli_paths: np.ndarray, dim: int): - """Unpacks microsaccades from stimuli_paths into a single list to conform with coord requirements.""" - unpacked_microsaccades = [] - for stimulus_path in stimuli_paths: - for microsaccade in microsaccades[stimulus_path]: - unpacked_microsaccades.append(microsaccade[dim]) - return unpacked_microsaccades - - def attach_stimulus_set_meta(assembly, stimulus_set, number_of_trials: int, require_variance: bool = False): stimulus_paths = [str(stimulus_set.get_stimulus(stimulus_id)) for stimulus_id in stimulus_set['stimulus_id']] stimulus_paths = [lstrip_local(path) for path in stimulus_paths] @@ -518,7 +547,8 @@ def attach_stimulus_set_meta(assembly, stimulus_set, number_of_trials: int, requ assembly = assembly.assign_coords({column: ('presentation', repeated_values)}) # assign multiple coords at once all_columns.append(column) - index = all_columns + ['microsaccade_shift_x_pixels', 'microsaccade_shift_y_pixels'] + index = all_columns + ['microsaccade_shift_x_pixels', 'microsaccade_shift_y_pixels', + 'microsaccade_shift_x_degrees', 'microsaccade_shift_y_degrees'] assembly = assembly.set_index(presentation=index) # assign MultiIndex return assembly diff --git a/tests/test_model_helpers/activations/test___init__.py b/tests/test_model_helpers/activations/test___init__.py index dba05c676..551c5cd7a 100644 --- a/tests/test_model_helpers/activations/test___init__.py +++ b/tests/test_model_helpers/activations/test___init__.py @@ -206,6 +206,8 @@ def test_require_variance_has_shift_coords(model_ctr, layers, image_name, number assert activations is not None assert len(activations['microsaccade_shift_x_pixels']) == number_of_trials * len(stimulus_paths) assert len(activations['microsaccade_shift_y_pixels']) == number_of_trials * len(stimulus_paths) + assert len(activations['microsaccade_shift_x_degrees']) == number_of_trials * len(stimulus_paths) + assert len(activations['microsaccade_shift_y_degrees']) == number_of_trials * len(stimulus_paths) @pytest.mark.parametrize("image_name", ['rgb.jpg', 'grayscale.png', 'grayscale2.jpg', 'grayscale_alpha.png', From 926778c5653fe546211bcacb9a608c788a639b6c Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Thu, 29 Feb 2024 13:40:16 +0100 Subject: [PATCH 27/41] Apply suggestions from code review Co-authored-by: Martin Schrimpf --- .../model_helpers/activations/core.py | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/brainscore_vision/model_helpers/activations/core.py b/brainscore_vision/model_helpers/activations/core.py index 200511931..3860e2700 100644 --- a/brainscore_vision/model_helpers/activations/core.py +++ b/brainscore_vision/model_helpers/activations/core.py @@ -43,7 +43,7 @@ def __init__(self, get_activations, preprocessing, identifier=False, batch_size= # a dict that contains the image paths and their respective microsaccades. e.g., # {'abc.jpg': [(0, 0), (1.5, 2)]} self.microsaccades = {'pixels': {}, 'degrees': {}} - self._visual_degrees = None # Model visual degrees. Used for computing microsaccades + self._visual_degrees = None # Model visual degrees. Used for computing microsaccades in the space of degrees rather than pixels def __call__(self, stimuli, layers, stimuli_identifier=None, number_of_trials: int = 1, require_variance: bool = False): @@ -55,7 +55,7 @@ def __call__(self, stimuli, layers, stimuli_identifier=None, number_of_trials: i We here implement this using microsaccades. Human microsaccade amplitude varies by who you ask, an estimate might be <0.1 deg = 360 arcsec = 6arcmin. Our motivation to make use of such microsaccades is to obtain multiple different neural activities to the same input stimulus - from non-stochastic models. This is to improve estimates of e.g. psychophysical functions. + from non-stochastic models. This enables models to engage on e.g. psychophysical functions which often require variance for the same stimulus. In the current implementation, if `require_variance=True`, the model microsaccades in input pixel space with 1-pixel increments from the center of the stimulus. The base behavior thus maintains a fixed microsaccade distance as measured in visual angle, regardless of the model's visual angle. @@ -116,7 +116,7 @@ def from_paths(self, stimuli_paths, layers, stimuli_identifier=None, require_var if require_variance: activations = fnc(layers=layers, stimuli_paths=stimuli_paths, require_variance=require_variance) else: - # In case stimuli paths are duplicates (e.g. multiple trials), we first reduce them to only the paths that need + # When we are not asked for varying responses but receive `stimuli_paths` duplicates (e.g. multiple trials), we first reduce them to only the paths that need # to be run individually, compute activations for those, and then expand the activations to all paths again. # This is done here, before storing, so that we only store the reduced activations. reduced_paths = self._reduce_paths(stimuli_paths) @@ -136,7 +136,7 @@ def _from_paths(self, layers, stimuli_paths, require_variance: bool = False): layer_activations = self._get_activations_batched(stimuli_paths, layers=layers, batch_size=self._batch_size, require_variance=require_variance) self._logger.info('Packaging into assembly') - return self._package(layer_activations, stimuli_paths, require_variance) + return self._package(layer_activations=layer_activations, stimuli_paths=stimuli_paths, require_variance=require_variance) def _reduce_paths(self, stimuli_paths): return list(set(stimuli_paths)) @@ -177,7 +177,7 @@ def register_stimulus_set_hook(self, hook): self._stimulus_set_hooks[handle.id] = hook return handle - def _get_activations_batched(self, paths, layers, batch_size, require_variance: bool): + def _get_activations_batched(self, paths, layers, batch_size: int, require_variance: bool): layer_activations = OrderedDict() for batch_start in tqdm(range(0, len(paths), batch_size), unit_scale=batch_size, desc="activations"): batch_end = min(batch_start + batch_size, len(paths)) @@ -187,13 +187,13 @@ def _get_activations_batched(self, paths, layers, batch_size, require_variance: # compute activations on the entire batch one shift at a time for shift_number in range(self.number_of_trials): - activations = self._get_batch_activations(batch_inputs, layers, batch_size, require_variance, - shift_number) + activations = self._get_batch_activations(inputs=batch_inputs, layer_names=layers, batch_size=batch_size, require_variance=require_variance, + trial_number=shift_number) for layer_name, layer_output in activations.items(): batch_activations.setdefault(layer_name, []).append(layer_output) - # concatenate all shifts into this batch + # concatenate all shifts in this batch for layer_name, layer_outputs in batch_activations.items(): batch_activations[layer_name] = np.concatenate(layer_outputs) @@ -210,7 +210,7 @@ def _get_activations_batched(self, paths, layers, batch_size, require_variance: return layer_activations # this is all batches - def _get_batch_activations(self, inputs, layer_names, batch_size, require_variance: bool = False, + def _get_batch_activations(self, inputs, layer_names, batch_size: int, require_variance: bool = False, trial_number: int = 1): inputs, num_padding = self._pad(inputs, batch_size) preprocessed_inputs = self.preprocess(inputs) @@ -222,7 +222,7 @@ def _get_batch_activations(self, inputs, layer_names, batch_size, require_varian self.remove_temporary_files(preprocessed_inputs) return activations - def set_visual_degrees(self, visual_degrees: float) -> None: + def set_visual_degrees(self, visual_degrees: float): """ A method used by ModelCommitments to give the ActivationsExtractorHelper their visual degrees for performing microsaccades. @@ -231,7 +231,7 @@ def set_visual_degrees(self, visual_degrees: float) -> None: def translate_images(self, images: List[Union[str, np.ndarray]], image_paths: List[str], trial_number: int, require_variance: bool) -> List[str]: - """A method that translates images according to selected microsaccades, if microsaccades are required.""" + """Translate images according to selected microsaccades, if microsaccades are required.""" output_images = [] for index, image_path in enumerate(image_paths): # When microsaccades are not used, skip computing them and return the base images. @@ -281,7 +281,7 @@ def _pad(self, batch_images, batch_size): def _unpad(self, layer_activations, num_padding): return change_dict(layer_activations, lambda values: values[:-num_padding or None]) - def _package(self, layer_activations, stimuli_paths, require_variance): + def _package(self, layer_activations, stimuli_paths, require_variance: bool): shapes = [a.shape for a in layer_activations.values()] self._logger.debug(f"Activations shapes: {shapes}") self._logger.debug("Packaging individual layers") @@ -316,7 +316,7 @@ def _package(self, layer_activations, stimuli_paths, require_variance): dims=layer_assemblies[0].dims) return model_assembly - def _package_layer(self, layer_activations, layer, stimuli_paths, require_variance=False): + def _package_layer(self, layer_activations: np.ndarray, layer: str, stimuli_paths: List[str], require_variance: bool = False): # activation shape is larger if variance in responses is required from the model by a factor of number_of_trials if require_variance: runs_per_image = self.number_of_trials @@ -382,14 +382,14 @@ def insert_attrs(self, wrapper): def select_microsaccade(self, image_path: str, trial_number: int, image_shape: Tuple[int, int] ) -> Tuple[float, float]: """ - A function for generating microsaccade locations. The function returns a tuple of pixel shifts expanding from + A function for generating a microsaccade location. The function returns a tuple of pixel shifts expanding from the center of the image. Microsaccade locations are placed within a circle, evenly distributed across the entire area in a spiral, from the center to the circumference. We keep track of microsaccades both on a pixel and visual angle basis, but only pixel values are returned. This is because shifting the image using cv2 requires pixel representation. """ - # avoid duplicating this computation repeatedly + # if we already computed `self.microsaccades`, we can just index into it if image_path in self.microsaccades.keys(): return self.microsaccades['pixels'][image_path][trial_number] @@ -398,8 +398,7 @@ def select_microsaccade(self, image_path: str, trial_number: int, image_shape: T 'extent of microsaccades.') # compute the maximum radius of microsaccade extent in pixel space - if self._visual_degrees is None: - raise AssertionError('self._visual_degrees is not set by the ModelCommitment, but microsaccades ' + assert self._visual_degrees is not None, ('self._visual_degrees is not set by the ModelCommitment, but microsaccades ' 'are in use. Set activations_model visual degrees in your commitment after defining ' 'your activations_model. For example, self.activations_model.set_visual_degrees' '(visual_degrees). For detailed information, see ' From f7a03409923313c0df55c49fac3bcc76654da031 Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Thu, 29 Feb 2024 14:58:06 +0100 Subject: [PATCH 28/41] fix outdated comments, type hints, etc. --- .../model_helpers/activations/core.py | 48 ++++++++++++------- .../brain_transformation/neural.py | 2 +- 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/brainscore_vision/model_helpers/activations/core.py b/brainscore_vision/model_helpers/activations/core.py index 3860e2700..1950c561b 100644 --- a/brainscore_vision/model_helpers/activations/core.py +++ b/brainscore_vision/model_helpers/activations/core.py @@ -40,10 +40,12 @@ def __init__(self, get_activations, preprocessing, identifier=False, batch_size= self.number_of_trials = 1 # for use with microsaccades. self.microsaccade_extent_degrees = 0.05 # how many degrees models microsaccade by default - # a dict that contains the image paths and their respective microsaccades. e.g., - # {'abc.jpg': [(0, 0), (1.5, 2)]} + # a dict that contains two dicts, one for representing microsaccades in pixels, and one in degrees. + # Each dict inside contain image paths and their respective microsaccades. For example + # {'pixels': {'abc.jpg': [(0, 0), (1.5, 2)]}, 'degrees': {'abc.jpg': [(0., 0.), (0.0075, 0.001)]}} self.microsaccades = {'pixels': {}, 'degrees': {}} - self._visual_degrees = None # Model visual degrees. Used for computing microsaccades in the space of degrees rather than pixels + # Model visual degrees. Used for computing microsaccades in the space of degrees rather than pixels + self._visual_degrees = None def __call__(self, stimuli, layers, stimuli_identifier=None, number_of_trials: int = 1, require_variance: bool = False): @@ -54,11 +56,12 @@ def __call__(self, stimuli, layers, stimuli_identifier=None, number_of_trials: i allows stochastic responses to identical stimuli, even in otherwise deterministic base models). We here implement this using microsaccades. Human microsaccade amplitude varies by who you ask, an estimate might be <0.1 deg = 360 arcsec = 6arcmin. - Our motivation to make use of such microsaccades is to obtain multiple different neural activities to the same input stimulus - from non-stochastic models. This enables models to engage on e.g. psychophysical functions which often require variance for the same stimulus. - In the current implementation, if `require_variance=True`, the model microsaccades in - input pixel space with 1-pixel increments from the center of the stimulus. The base behavior thus - maintains a fixed microsaccade distance as measured in visual angle, regardless of the model's visual angle. + Our motivation to make use of such microsaccades is to obtain multiple different neural activities to the + same input stimulus from non-stochastic models. This enables models to engage on e.g. psychophysical + functions which often require variance for the same stimulus. In the current implementation, + if `require_variance=True`, the model microsaccades in the preprocessed input space in sub-pixel increments, + the extent and position of which are determined by `self._visual_degrees`, and + `self.microsaccade_extent_degrees`. Example usage: `require_variance = True` @@ -184,16 +187,16 @@ def _get_activations_batched(self, paths, layers, batch_size: int, require_varia batch_inputs = paths[batch_start:batch_end] batch_activations = OrderedDict() - # compute activations on the entire batch one shift at a time + # compute activations on the entire batch one microsaccade shift at a time. for shift_number in range(self.number_of_trials): - activations = self._get_batch_activations(inputs=batch_inputs, layer_names=layers, batch_size=batch_size, require_variance=require_variance, trial_number=shift_number) for layer_name, layer_output in activations.items(): batch_activations.setdefault(layer_name, []).append(layer_output) - # concatenate all shifts in this batch + # concatenate all microsaccade shifts in this batch (for example, if the model microsaccaded 15 times, + # the 15 microsaccaded layer_outputs are concatenated to the batch here. for layer_name, layer_outputs in batch_activations.items(): batch_activations[layer_name] = np.concatenate(layer_outputs) @@ -204,7 +207,7 @@ def _get_activations_batched(self, paths, layers, batch_size: int, require_varia for layer_name, layer_output in batch_activations.items(): layer_activations.setdefault(layer_name, []).append(layer_output) - # fast concat all batches + # concat all batches for layer_name, layer_outputs in layer_activations.items(): layer_activations[layer_name] = np.concatenate(layer_outputs) @@ -231,13 +234,23 @@ def set_visual_degrees(self, visual_degrees: float): def translate_images(self, images: List[Union[str, np.ndarray]], image_paths: List[str], trial_number: int, require_variance: bool) -> List[str]: - """Translate images according to selected microsaccades, if microsaccades are required.""" + """ + Translate images according to selected microsaccades, if microsaccades are required. + + :param images: A list of images (in the case of tensorflow models), or a list of arrays (non-tf models). + :param image_paths: A list of image paths. Both `image_paths` and `images` are needed since while both tf and + non-tf models preprocess images before this point, non-tf models' preprocessed images + are fixed as arrays when fed into here. As such, simply returning `image_paths` for + non-tf models would require double-loading of the images, which does not seem like a + good idea. + """ output_images = [] for index, image_path in enumerate(image_paths): # When microsaccades are not used, skip computing them and return the base images. # This iteration could be entirely skipped, but recording microsaccades for all images regardless # of whether variance is required or not is convenient for adding an extra presentation dimension - # in the layer assembly later to avoid collapse, or otherwise extraneous mock dims. + # in the layer assembly later to keep track of as much metadata as possible, to avoid layer assembly + # collapse, or to avoid otherwise extraneous mock dims. # The method could further be streamlined by calling `self.get_image_with_shape()` and # `self.select_microsaccade` for all images regardless of require_variance, but it seems like a bad # idea to introduce cv2 image loading for all models and images, regardless of whether they are actually @@ -394,7 +407,7 @@ def select_microsaccade(self, image_path: str, trial_number: int, image_shape: T return self.microsaccades['pixels'][image_path][trial_number] if image_shape[0] != image_shape[1]: - self._logger.debug('Warning: input image is not a square. Image dimension 0 is used to calculate the ' + self._logger.debug('Input image is not a square. Image dimension 0 is used to calculate the ' 'extent of microsaccades.') # compute the maximum radius of microsaccade extent in pixel space @@ -430,7 +443,6 @@ def select_microsaccade(self, image_path: str, trial_number: int, image_shape: T def unpack_microsaccade_coords(self, stimuli_paths: np.ndarray, pixels_or_degrees: str, dim: int): """Unpacks microsaccades from stimuli_paths into a single list to conform with coord requirements.""" - # Python 3.7 does not support typing.Literal assert pixels_or_degrees == 'pixels' or pixels_or_degrees == 'degrees' unpacked_microsaccades = [] for stimulus_path in stimuli_paths: @@ -464,7 +476,7 @@ def remove_temporary_files(temporary_file_paths: List[str]) -> None: pass @staticmethod - def translate(image, shift: Tuple[float, float], image_shape: Tuple[int, int]): + def translate(image, shift: Tuple[float, float], image_shape: Tuple[int, int]) -> np.array: rows, cols = image_shape # translation matrix M = np.float32([[1, 0, shift[0]], [0, 1, shift[1]]]) @@ -475,7 +487,7 @@ def translate(image, shift: Tuple[float, float], image_shape: Tuple[int, int]): return translated_image @staticmethod - def get_image_with_shape(image: Union[str, np.ndarray]): + def get_image_with_shape(image: Union[str, np.ndarray]) -> Tuple[np.array, Tuple[int, int]]: if isinstance(image, str): # tf models return strings after preprocessing image = cv2.imread(image) rows, cols, _ = image.shape diff --git a/brainscore_vision/model_helpers/brain_transformation/neural.py b/brainscore_vision/model_helpers/brain_transformation/neural.py index 5148bb9b7..f1cb04356 100644 --- a/brainscore_vision/model_helpers/brain_transformation/neural.py +++ b/brainscore_vision/model_helpers/brain_transformation/neural.py @@ -25,7 +25,7 @@ def identifier(self): def look_at(self, stimuli, number_of_trials=1, require_variance: bool = False): """ - :param number_of_trials: An integer that determines how many repetitions of the same model performs. + :param number_of_trials: An integer that determines how many repetitions of the same image the model performs. :param require_variance: Whether to require models to return different activations for the same stimuli or not. For detailed information, see :meth:`~brainscore_vision.model_helpers.activations.ActivationsExtractorHelper.__call__`, From 5a8881d5d0452806f3961a0ad99d5927d2436926 Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Fri, 1 Mar 2024 10:22:38 +0100 Subject: [PATCH 29/41] change function call to reduce repetition --- brainscore_vision/model_helpers/activations/core.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/brainscore_vision/model_helpers/activations/core.py b/brainscore_vision/model_helpers/activations/core.py index 1950c561b..eb13435a7 100644 --- a/brainscore_vision/model_helpers/activations/core.py +++ b/brainscore_vision/model_helpers/activations/core.py @@ -81,13 +81,13 @@ def __call__(self, stimuli, layers, stimuli_identifier=None, number_of_trials: i "set self.activations_model.set_visual_degrees(visual_degrees). Not doing so risks " "breaking microsaccades.") if isinstance(stimuli, StimulusSet): - return self.from_stimulus_set(stimulus_set=stimuli, layers=layers, stimuli_identifier=stimuli_identifier, - require_variance=require_variance) + function_call = functools.partial(self.from_stimulus_set, stimulus_set=stimuli) else: - return self.from_paths(stimuli_paths=stimuli, - layers=layers, - stimuli_identifier=stimuli_identifier, - require_variance=require_variance) + function_call = functools.partial(self.from_paths, stimuli_paths=stimuli) + return function_call( + layers=layers, + stimuli_identifier=stimuli_identifier, + require_variance=require_variance) def from_stimulus_set(self, stimulus_set, layers, stimuli_identifier=None, require_variance: bool = False): """ From 2655ecffd26246b027888e57d76416f241d9de5f Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Fri, 1 Mar 2024 10:32:52 +0100 Subject: [PATCH 30/41] add kwargs to microsaccade helpers --- brainscore_vision/model_helpers/activations/core.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/brainscore_vision/model_helpers/activations/core.py b/brainscore_vision/model_helpers/activations/core.py index eb13435a7..3e30418b8 100644 --- a/brainscore_vision/model_helpers/activations/core.py +++ b/brainscore_vision/model_helpers/activations/core.py @@ -217,7 +217,10 @@ def _get_batch_activations(self, inputs, layer_names, batch_size: int, require_v trial_number: int = 1): inputs, num_padding = self._pad(inputs, batch_size) preprocessed_inputs = self.preprocess(inputs) - preprocessed_inputs = self.translate_images(preprocessed_inputs, inputs, trial_number, require_variance) + preprocessed_inputs = self.translate_images(images=preprocessed_inputs, + image_paths=inputs, + trial_number=trial_number, + require_variance=require_variance) activations = self.get_activations(preprocessed_inputs, layer_names) assert isinstance(activations, OrderedDict) activations = self._unpad(activations, num_padding) @@ -266,8 +269,10 @@ def translate_images(self, images: List[Union[str, np.ndarray]], image_paths: Li trial_number=trial_number, image_shape=image_shape) return_string = True if isinstance(images[index], str) else False - output_images.append(self.translate_image(image, microsaccade_location_pixels, image_shape, - return_string)) + output_images.append(self.translate_image(image=image, + microsaccade_location=microsaccade_location_pixels, + image_shape=image_shape, + return_string=return_string)) return self.reshape_microsaccaded_images(output_images) def translate_image(self, image: str, microsaccade_location: Tuple[float, float], image_shape: Tuple[int, int], From 232f4d732cbd59f807869e28de9f52b24fbf5da5 Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Fri, 1 Mar 2024 10:45:19 +0100 Subject: [PATCH 31/41] refactor microsaccade usage to their own class to improve readability --- .../model_helpers/activations/core.py | 179 +++++++++--------- 1 file changed, 94 insertions(+), 85 deletions(-) diff --git a/brainscore_vision/model_helpers/activations/core.py b/brainscore_vision/model_helpers/activations/core.py index 3e30418b8..86cd696c6 100644 --- a/brainscore_vision/model_helpers/activations/core.py +++ b/brainscore_vision/model_helpers/activations/core.py @@ -36,16 +36,7 @@ def __init__(self, get_activations, preprocessing, identifier=False, batch_size= self.preprocess = preprocessing or (lambda x: x) self._stimulus_set_hooks = {} self._batch_activations_hooks = {} - - self.number_of_trials = 1 # for use with microsaccades. - self.microsaccade_extent_degrees = 0.05 # how many degrees models microsaccade by default - - # a dict that contains two dicts, one for representing microsaccades in pixels, and one in degrees. - # Each dict inside contain image paths and their respective microsaccades. For example - # {'pixels': {'abc.jpg': [(0, 0), (1.5, 2)]}, 'degrees': {'abc.jpg': [(0., 0.), (0.0075, 0.001)]}} - self.microsaccades = {'pixels': {}, 'degrees': {}} - # Model visual degrees. Used for computing microsaccades in the space of degrees rather than pixels - self._visual_degrees = None + self._microsaccade_helper = MicrosaccadeHelper() def __call__(self, stimuli, layers, stimuli_identifier=None, number_of_trials: int = 1, require_variance: bool = False): @@ -75,8 +66,8 @@ def __call__(self, stimuli, layers, stimuli_identifier=None, number_of_trials: i """ if require_variance: - self.number_of_trials = number_of_trials # for use with microsaccades - if (self._visual_degrees is None) and require_variance: + self._microsaccade_helper.number_of_trials = number_of_trials # for use with microsaccades + if (self._microsaccade_helper.visual_degrees is None) and require_variance: self._logger.debug("When using microsaccades for model commitments other than ModelCommitment, you should " "set self.activations_model.set_visual_degrees(visual_degrees). Not doing so risks " "breaking microsaccades.") @@ -100,7 +91,9 @@ def from_stimulus_set(self, stimulus_set, layers, stimuli_identifier=None, requi stimulus_set = hook(stimulus_set) stimuli_paths = [str(stimulus_set.get_stimulus(stimulus_id)) for stimulus_id in stimulus_set['stimulus_id']] activations = self.from_paths(stimuli_paths=stimuli_paths, layers=layers, stimuli_identifier=stimuli_identifier) - activations = attach_stimulus_set_meta(activations, stimulus_set, number_of_trials=self.number_of_trials, + activations = attach_stimulus_set_meta(activations, + stimulus_set, + number_of_trials=self._microsaccade_helper.number_of_trials, require_variance=require_variance) return activations @@ -119,9 +112,10 @@ def from_paths(self, stimuli_paths, layers, stimuli_identifier=None, require_var if require_variance: activations = fnc(layers=layers, stimuli_paths=stimuli_paths, require_variance=require_variance) else: - # When we are not asked for varying responses but receive `stimuli_paths` duplicates (e.g. multiple trials), we first reduce them to only the paths that need - # to be run individually, compute activations for those, and then expand the activations to all paths again. - # This is done here, before storing, so that we only store the reduced activations. + # When we are not asked for varying responses but receive `stimuli_paths` duplicates (e.g. multiple trials), + # we first reduce them to only the paths that need to be run individually, compute activations for those, + # and then expand the activations to all paths again. This is done here, before storing, so that we only + # store the reduced activations. reduced_paths = self._reduce_paths(stimuli_paths) activations = fnc(layers=layers, stimuli_paths=reduced_paths, require_variance=require_variance) activations = self._expand_paths(activations, original_paths=stimuli_paths) @@ -188,7 +182,7 @@ def _get_activations_batched(self, paths, layers, batch_size: int, require_varia batch_activations = OrderedDict() # compute activations on the entire batch one microsaccade shift at a time. - for shift_number in range(self.number_of_trials): + for shift_number in range(self._microsaccade_helper.number_of_trials): activations = self._get_batch_activations(inputs=batch_inputs, layer_names=layers, batch_size=batch_size, require_variance=require_variance, trial_number=shift_number) @@ -217,76 +211,24 @@ def _get_batch_activations(self, inputs, layer_names, batch_size: int, require_v trial_number: int = 1): inputs, num_padding = self._pad(inputs, batch_size) preprocessed_inputs = self.preprocess(inputs) - preprocessed_inputs = self.translate_images(images=preprocessed_inputs, - image_paths=inputs, - trial_number=trial_number, - require_variance=require_variance) + preprocessed_inputs = self._microsaccade_helper.translate_images(images=preprocessed_inputs, + image_paths=inputs, + trial_number=trial_number, + require_variance=require_variance) activations = self.get_activations(preprocessed_inputs, layer_names) assert isinstance(activations, OrderedDict) activations = self._unpad(activations, num_padding) if require_variance: - self.remove_temporary_files(preprocessed_inputs) + self._microsaccade_helper.remove_temporary_files(preprocessed_inputs) return activations def set_visual_degrees(self, visual_degrees: float): """ - A method used by ModelCommitments to give the ActivationsExtractorHelper their visual degrees for - performing microsaccades. + A method used by ModelCommitments to give the ActivationsExtractorHelper.MicrosaccadeHelper their visual + degrees for performing microsaccades. """ - self._visual_degrees = visual_degrees + self._microsaccade_helper.visual_degrees = visual_degrees - def translate_images(self, images: List[Union[str, np.ndarray]], image_paths: List[str], trial_number: int, - require_variance: bool) -> List[str]: - """ - Translate images according to selected microsaccades, if microsaccades are required. - - :param images: A list of images (in the case of tensorflow models), or a list of arrays (non-tf models). - :param image_paths: A list of image paths. Both `image_paths` and `images` are needed since while both tf and - non-tf models preprocess images before this point, non-tf models' preprocessed images - are fixed as arrays when fed into here. As such, simply returning `image_paths` for - non-tf models would require double-loading of the images, which does not seem like a - good idea. - """ - output_images = [] - for index, image_path in enumerate(image_paths): - # When microsaccades are not used, skip computing them and return the base images. - # This iteration could be entirely skipped, but recording microsaccades for all images regardless - # of whether variance is required or not is convenient for adding an extra presentation dimension - # in the layer assembly later to keep track of as much metadata as possible, to avoid layer assembly - # collapse, or to avoid otherwise extraneous mock dims. - # The method could further be streamlined by calling `self.get_image_with_shape()` and - # `self.select_microsaccade` for all images regardless of require_variance, but it seems like a bad - # idea to introduce cv2 image loading for all models and images, regardless of whether they are actually - # microsaccading. - if not require_variance: - self.microsaccades['pixels'][image_path] = [(0., 0.)] - self.microsaccades['degrees'][image_path] = [(0., 0.)] - output_images.append(images[index]) - else: - # translate images according to microsaccades if we are using microsaccades - image, image_shape = self.get_image_with_shape(images[index]) - microsaccade_location_pixels = self.select_microsaccade(image_path=image_path, - trial_number=trial_number, - image_shape=image_shape) - return_string = True if isinstance(images[index], str) else False - output_images.append(self.translate_image(image=image, - microsaccade_location=microsaccade_location_pixels, - image_shape=image_shape, - return_string=return_string)) - return self.reshape_microsaccaded_images(output_images) - - def translate_image(self, image: str, microsaccade_location: Tuple[float, float], image_shape: Tuple[int, int], - return_string: bool) -> str: - """Translates and saves a temporary image to temporary_fp.""" - translated_image = self.translate(image, microsaccade_location, image_shape) - if not return_string: # if the model accepts ndarrays after preprocessing, return one - return translated_image - else: # if the model accepts strings after preprocessing, write temp file - temp_file_descriptor, temporary_fp = tempfile.mkstemp(suffix=".png") - os.close(temp_file_descriptor) - if not cv2.imwrite(temporary_fp, translated_image): - raise Exception(f"cv2.imwrite failed: {temporary_fp}") - return temporary_fp def _pad(self, batch_images, batch_size): num_images = len(batch_images) @@ -337,7 +279,7 @@ def _package(self, layer_activations, stimuli_paths, require_variance: bool): def _package_layer(self, layer_activations: np.ndarray, layer: str, stimuli_paths: List[str], require_variance: bool = False): # activation shape is larger if variance in responses is required from the model by a factor of number_of_trials if require_variance: - runs_per_image = self.number_of_trials + runs_per_image = self._microsaccade_helper.number_of_trials else: runs_per_image = 1 assert layer_activations.shape[0] == len(stimuli_paths) * runs_per_image @@ -359,19 +301,19 @@ def _package_layer(self, layer_activations: np.ndarray, layer: str, stimuli_path # build assembly coords = {'stimulus_path': ('presentation', stimuli_paths), - 'microsaccade_shift_x_pixels': ('presentation', self.unpack_microsaccade_coords( + 'microsaccade_shift_x_pixels': ('presentation', self._microsaccade_helper.unpack_microsaccade_coords( np.unique(stimuli_paths), pixels_or_degrees='pixels', dim=0)), - 'microsaccade_shift_y_pixels': ('presentation', self.unpack_microsaccade_coords( + 'microsaccade_shift_y_pixels': ('presentation', self._microsaccade_helper.unpack_microsaccade_coords( np.unique(stimuli_paths), pixels_or_degrees='pixels', dim=1)), - 'microsaccade_shift_x_degrees': ('presentation', self.unpack_microsaccade_coords( + 'microsaccade_shift_x_degrees': ('presentation', self._microsaccade_helper.unpack_microsaccade_coords( np.unique(stimuli_paths), pixels_or_degrees='degrees', dim=0)), - 'microsaccade_shift_y_degrees': ('presentation', self.unpack_microsaccade_coords( + 'microsaccade_shift_y_degrees': ('presentation', self._microsaccade_helper.unpack_microsaccade_coords( np.unique(stimuli_paths), pixels_or_degrees='degrees', dim=1)), @@ -397,6 +339,73 @@ def insert_attrs(self, wrapper): wrapper.register_batch_activations_hook = self.register_batch_activations_hook wrapper.register_stimulus_set_hook = self.register_stimulus_set_hook + +class MicrosaccadeHelper: + def __init__(self): + self._logger = logging.getLogger(fullname(self)) + self.number_of_trials = 1 # for use with microsaccades. + self.microsaccade_extent_degrees = 0.05 # how many degrees models microsaccade by default + + # a dict that contains two dicts, one for representing microsaccades in pixels, and one in degrees. + # Each dict inside contain image paths and their respective microsaccades. For example + # {'pixels': {'abc.jpg': [(0, 0), (1.5, 2)]}, 'degrees': {'abc.jpg': [(0., 0.), (0.0075, 0.001)]}} + self.microsaccades = {'pixels': {}, 'degrees': {}} + # Model visual degrees. Used for computing microsaccades in the space of degrees rather than pixels + self.visual_degrees = None + + def translate_images(self, images: List[Union[str, np.ndarray]], image_paths: List[str], trial_number: int, + require_variance: bool) -> List[str]: + """ + Translate images according to selected microsaccades, if microsaccades are required. + + :param images: A list of images (in the case of tensorflow models), or a list of arrays (non-tf models). + :param image_paths: A list of image paths. Both `image_paths` and `images` are needed since while both tf and + non-tf models preprocess images before this point, non-tf models' preprocessed images + are fixed as arrays when fed into here. As such, simply returning `image_paths` for + non-tf models would require double-loading of the images, which does not seem like a + good idea. + """ + output_images = [] + for index, image_path in enumerate(image_paths): + # When microsaccades are not used, skip computing them and return the base images. + # This iteration could be entirely skipped, but recording microsaccades for all images regardless + # of whether variance is required or not is convenient for adding an extra presentation dimension + # in the layer assembly later to keep track of as much metadata as possible, to avoid layer assembly + # collapse, or to avoid otherwise extraneous mock dims. + # The method could further be streamlined by calling `self.get_image_with_shape()` and + # `self.select_microsaccade` for all images regardless of require_variance, but it seems like a bad + # idea to introduce cv2 image loading for all models and images, regardless of whether they are actually + # microsaccading. + if not require_variance: + self.microsaccades['pixels'][image_path] = [(0., 0.)] + self.microsaccades['degrees'][image_path] = [(0., 0.)] + output_images.append(images[index]) + else: + # translate images according to microsaccades if we are using microsaccades + image, image_shape = self.get_image_with_shape(images[index]) + microsaccade_location_pixels = self.select_microsaccade(image_path=image_path, + trial_number=trial_number, + image_shape=image_shape) + return_string = True if isinstance(images[index], str) else False + output_images.append(self.translate_image(image=image, + microsaccade_location=microsaccade_location_pixels, + image_shape=image_shape, + return_string=return_string)) + return self.reshape_microsaccaded_images(output_images) + + def translate_image(self, image: str, microsaccade_location: Tuple[float, float], image_shape: Tuple[int, int], + return_string: bool) -> str: + """Translates and saves a temporary image to temporary_fp.""" + translated_image = self.translate(image, microsaccade_location, image_shape) + if not return_string: # if the model accepts ndarrays after preprocessing, return one + return translated_image + else: # if the model accepts strings after preprocessing, write temp file + temp_file_descriptor, temporary_fp = tempfile.mkstemp(suffix=".png") + os.close(temp_file_descriptor) + if not cv2.imwrite(temporary_fp, translated_image): + raise Exception(f"cv2.imwrite failed: {temporary_fp}") + return temporary_fp + def select_microsaccade(self, image_path: str, trial_number: int, image_shape: Tuple[int, int] ) -> Tuple[float, float]: """ @@ -416,13 +425,13 @@ def select_microsaccade(self, image_path: str, trial_number: int, image_shape: T 'extent of microsaccades.') # compute the maximum radius of microsaccade extent in pixel space - assert self._visual_degrees is not None, ('self._visual_degrees is not set by the ModelCommitment, but microsaccades ' + assert self.visual_degrees is not None, ('self._visual_degrees is not set by the ModelCommitment, but microsaccades ' 'are in use. Set activations_model visual degrees in your commitment after defining ' 'your activations_model. For example, self.activations_model.set_visual_degrees' '(visual_degrees). For detailed information, see ' ':meth:`~brainscore_vision.model_helpers.activations.ActivationsExtractorHelper.' '__call__`,') - radius_ratio = self.microsaccade_extent_degrees / self._visual_degrees + radius_ratio = self.microsaccade_extent_degrees / self.visual_degrees max_radius = radius_ratio * image_shape[0] # maximum radius in pixels, set in self.microsaccade_extent_degrees selected_microsaccades = {'pixels': [], 'degrees': []} @@ -457,7 +466,7 @@ def unpack_microsaccade_coords(self, stimuli_paths: np.ndarray, pixels_or_degree def calculate_pixels_per_degree_in_image(self, image_width_pixels: int) -> float: """Calculates the pixels per degree in the image, assuming the calculation based on image width.""" - pixels_per_degree = image_width_pixels / self._visual_degrees + pixels_per_degree = image_width_pixels / self.visual_degrees return pixels_per_degree @staticmethod From 06e4b4cde4324003d4ae61ccddbaf137fc294b5c Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Fri, 1 Mar 2024 10:50:55 +0100 Subject: [PATCH 32/41] refactor microsaccade coords into MicrosaccadeHelper --- .../model_helpers/activations/core.py | 37 +++++++++++-------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/brainscore_vision/model_helpers/activations/core.py b/brainscore_vision/model_helpers/activations/core.py index 86cd696c6..2406fd62e 100644 --- a/brainscore_vision/model_helpers/activations/core.py +++ b/brainscore_vision/model_helpers/activations/core.py @@ -301,22 +301,7 @@ def _package_layer(self, layer_activations: np.ndarray, layer: str, stimuli_path # build assembly coords = {'stimulus_path': ('presentation', stimuli_paths), - 'microsaccade_shift_x_pixels': ('presentation', self._microsaccade_helper.unpack_microsaccade_coords( - np.unique(stimuli_paths), - pixels_or_degrees='pixels', - dim=0)), - 'microsaccade_shift_y_pixels': ('presentation', self._microsaccade_helper.unpack_microsaccade_coords( - np.unique(stimuli_paths), - pixels_or_degrees='pixels', - dim=1)), - 'microsaccade_shift_x_degrees': ('presentation', self._microsaccade_helper.unpack_microsaccade_coords( - np.unique(stimuli_paths), - pixels_or_degrees='degrees', - dim=0)), - 'microsaccade_shift_y_degrees': ('presentation', self._microsaccade_helper.unpack_microsaccade_coords( - np.unique(stimuli_paths), - pixels_or_degrees='degrees', - dim=1)), + **self._microsaccade_helper.build_microsaccade_coords(stimuli_paths), 'neuroid_num': ('neuroid', list(range(activations.shape[1]))), 'model': ('neuroid', [self.identifier] * activations.shape[1]), 'layer': ('neuroid', [layer] * activations.shape[1]), @@ -513,6 +498,26 @@ def reshape_microsaccaded_images(images: List) -> Union[List[str], np.ndarray]: return images return np.stack(images, axis=0) + def build_microsaccade_coords(self, stimuli_paths: np.array) -> Dict: + return { + 'microsaccade_shift_x_pixels': ('presentation', self.unpack_microsaccade_coords( + np.unique(stimuli_paths), + pixels_or_degrees='pixels', + dim=0)), + 'microsaccade_shift_y_pixels': ('presentation', self.unpack_microsaccade_coords( + np.unique(stimuli_paths), + pixels_or_degrees='pixels', + dim=1)), + 'microsaccade_shift_x_degrees': ('presentation', self.unpack_microsaccade_coords( + np.unique(stimuli_paths), + pixels_or_degrees='degrees', + dim=0)), + 'microsaccade_shift_y_degrees': ('presentation', self.unpack_microsaccade_coords( + np.unique(stimuli_paths), + pixels_or_degrees='degrees', + dim=1)) + } + def change_dict(d, change_function, keep_name=False, multithread=False): if not multithread: From 18e48aea52cda40bdee53fa6097bb3e329e511ea Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Fri, 1 Mar 2024 10:58:13 +0100 Subject: [PATCH 33/41] refactor microsaccade building --- .../model_helpers/activations/core.py | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/brainscore_vision/model_helpers/activations/core.py b/brainscore_vision/model_helpers/activations/core.py index 2406fd62e..735c442c0 100644 --- a/brainscore_vision/model_helpers/activations/core.py +++ b/brainscore_vision/model_helpers/activations/core.py @@ -401,21 +401,24 @@ def select_microsaccade(self, image_path: str, trial_number: int, image_shape: T from the center to the circumference. We keep track of microsaccades both on a pixel and visual angle basis, but only pixel values are returned. This is because shifting the image using cv2 requires pixel representation. """ - # if we already computed `self.microsaccades`, we can just index into it - if image_path in self.microsaccades.keys(): - return self.microsaccades['pixels'][image_path][trial_number] + # if we did not already compute `self.microsaccades`, we build them first. + if image_path not in self.microsaccades.keys(): + self.build_microsaccades(image_path=image_path, image_shape=image_shape) + return self.microsaccades['pixels'][image_path][trial_number - 1] + def build_microsaccades(self, image_path: str, image_shape: Tuple[int, int]): if image_shape[0] != image_shape[1]: self._logger.debug('Input image is not a square. Image dimension 0 is used to calculate the ' 'extent of microsaccades.') + assert self.visual_degrees is not None, ( + 'self._visual_degrees is not set by the ModelCommitment, but microsaccades ' + 'are in use. Set activations_model visual degrees in your commitment after defining ' + 'your activations_model. For example, self.activations_model.set_visual_degrees' + '(visual_degrees). For detailed information, see ' + ':meth:`~brainscore_vision.model_helpers.activations.ActivationsExtractorHelper.' + '__call__`,') # compute the maximum radius of microsaccade extent in pixel space - assert self.visual_degrees is not None, ('self._visual_degrees is not set by the ModelCommitment, but microsaccades ' - 'are in use. Set activations_model visual degrees in your commitment after defining ' - 'your activations_model. For example, self.activations_model.set_visual_degrees' - '(visual_degrees). For detailed information, see ' - ':meth:`~brainscore_vision.model_helpers.activations.ActivationsExtractorHelper.' - '__call__`,') radius_ratio = self.microsaccade_extent_degrees / self.visual_degrees max_radius = radius_ratio * image_shape[0] # maximum radius in pixels, set in self.microsaccade_extent_degrees @@ -437,7 +440,6 @@ def select_microsaccade(self, image_path: str, trial_number: int, image_shape: T # to keep consistent with number_of_trials, we count trial_number from 1 instead of from 0 self.microsaccades['pixels'][image_path] = selected_microsaccades['pixels'] self.microsaccades['degrees'][image_path] = selected_microsaccades['degrees'] - return selected_microsaccades['pixels'][trial_number - 1] def unpack_microsaccade_coords(self, stimuli_paths: np.ndarray, pixels_or_degrees: str, dim: int): From 954b4cc16cbb747dfa8e64f98e6965f6e7916701 Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Fri, 1 Mar 2024 11:00:50 +0100 Subject: [PATCH 34/41] change the way MultiIndex is set --- brainscore_vision/model_helpers/activations/core.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/brainscore_vision/model_helpers/activations/core.py b/brainscore_vision/model_helpers/activations/core.py index 735c442c0..9aa386c29 100644 --- a/brainscore_vision/model_helpers/activations/core.py +++ b/brainscore_vision/model_helpers/activations/core.py @@ -579,9 +579,8 @@ def attach_stimulus_set_meta(assembly, stimulus_set, number_of_trials: int, requ assembly = assembly.assign_coords({column: ('presentation', repeated_values)}) # assign multiple coords at once all_columns.append(column) - index = all_columns + ['microsaccade_shift_x_pixels', 'microsaccade_shift_y_pixels', - 'microsaccade_shift_x_degrees', 'microsaccade_shift_y_degrees'] - assembly = assembly.set_index(presentation=index) # assign MultiIndex + presentation_coords = all_columns + [coord for coord, dims, values in walk_coords(assembly['presentation'])] + assembly = assembly.set_index(presentation=list(set(presentation_coords))) # assign MultiIndex return assembly From 52596ac432bd171f46fc2b016e069fab0e64ef17 Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Fri, 1 Mar 2024 11:35:05 +0100 Subject: [PATCH 35/41] fix tf/pytorch/keras bug with image shape calculation --- .../model_helpers/activations/core.py | 84 ++++++++++--------- 1 file changed, 43 insertions(+), 41 deletions(-) diff --git a/brainscore_vision/model_helpers/activations/core.py b/brainscore_vision/model_helpers/activations/core.py index 9aa386c29..4446ac9eb 100644 --- a/brainscore_vision/model_helpers/activations/core.py +++ b/brainscore_vision/model_helpers/activations/core.py @@ -45,24 +45,7 @@ def __call__(self, stimuli, layers, stimuli_identifier=None, number_of_trials: i :param number_of_trials: An integer that determines how many repetitions of the same model performs. :param require_variance: A bool that asks models to output different responses to the same stimuli (i.e., allows stochastic responses to identical stimuli, even in otherwise deterministic base models). - We here implement this using microsaccades. - Human microsaccade amplitude varies by who you ask, an estimate might be <0.1 deg = 360 arcsec = 6arcmin. - Our motivation to make use of such microsaccades is to obtain multiple different neural activities to the - same input stimulus from non-stochastic models. This enables models to engage on e.g. psychophysical - functions which often require variance for the same stimulus. In the current implementation, - if `require_variance=True`, the model microsaccades in the preprocessed input space in sub-pixel increments, - the extent and position of which are determined by `self._visual_degrees`, and - `self.microsaccade_extent_degrees`. - - Example usage: - `require_variance = True` - - More information: - --> Rolfs 2009 "Microsaccades: Small steps on a long way" Vision Research, Volume 49, Issue 20, 15 - October 2009, Pages 2415-2441. - --> Haddad & Steinmann 1973 "The smallest voluntary saccade: Implications for fixation" Vision - Research Volume 13, Issue 6, June 1973, Pages 1075-1086, IN5-IN6. - Implemented by Ben Lonnqvist and Johannes Mehrer. + We here implement this using microsaccades. For more, see ... """ if require_variance: @@ -326,6 +309,24 @@ def insert_attrs(self, wrapper): class MicrosaccadeHelper: + """ + A class that allows ActivationsExtractorHelper to implement microsaccades. + + Human microsaccade amplitude varies by who you ask, an estimate might be <0.1 deg = 360 arcsec = 6arcmin. + Our motivation to make use of such microsaccades is to obtain multiple different neural activities to the + same input stimulus from non-stochastic models. This enables models to engage on e.g. psychophysical + functions which often require variance for the same stimulus. In the current implementation, + if `require_variance=True`, the model microsaccades in the preprocessed input space in sub-pixel increments, + the extent and position of which are determined by `self._visual_degrees`, and + `self.microsaccade_extent_degrees`. + + More information: + --> Rolfs 2009 "Microsaccades: Small steps on a long way" Vision Research, Volume 49, Issue 20, 15 + October 2009, Pages 2415-2441. + --> Haddad & Steinmann 1973 "The smallest voluntary saccade: Implications for fixation" Vision + Research Volume 13, Issue 6, June 1973, Pages 1075-1086, IN5-IN6. + Implemented by Ben Lonnqvist and Johannes Mehrer. + """ def __init__(self): self._logger = logging.getLogger(fullname(self)) self.number_of_trials = 1 # for use with microsaccades. @@ -441,8 +442,7 @@ def build_microsaccades(self, image_path: str, image_shape: Tuple[int, int]): self.microsaccades['pixels'][image_path] = selected_microsaccades['pixels'] self.microsaccades['degrees'][image_path] = selected_microsaccades['degrees'] - def unpack_microsaccade_coords(self, stimuli_paths: np.ndarray, pixels_or_degrees: str, - dim: int): + def unpack_microsaccade_coords(self, stimuli_paths: np.ndarray, pixels_or_degrees: str, dim: int): """Unpacks microsaccades from stimuli_paths into a single list to conform with coord requirements.""" assert pixels_or_degrees == 'pixels' or pixels_or_degrees == 'degrees' unpacked_microsaccades = [] @@ -456,6 +456,26 @@ def calculate_pixels_per_degree_in_image(self, image_width_pixels: int) -> float pixels_per_degree = image_width_pixels / self.visual_degrees return pixels_per_degree + def build_microsaccade_coords(self, stimuli_paths: np.array) -> Dict: + return { + 'microsaccade_shift_x_pixels': ('presentation', self.unpack_microsaccade_coords( + np.unique(stimuli_paths), + pixels_or_degrees='pixels', + dim=0)), + 'microsaccade_shift_y_pixels': ('presentation', self.unpack_microsaccade_coords( + np.unique(stimuli_paths), + pixels_or_degrees='pixels', + dim=1)), + 'microsaccade_shift_x_degrees': ('presentation', self.unpack_microsaccade_coords( + np.unique(stimuli_paths), + pixels_or_degrees='degrees', + dim=0)), + 'microsaccade_shift_y_degrees': ('presentation', self.unpack_microsaccade_coords( + np.unique(stimuli_paths), + pixels_or_degrees='degrees', + dim=1)) + } + @staticmethod def convert_pixels_to_degrees(pixel_coords: Tuple[float, float], pixels_per_degree: float) -> Tuple[float, float]: degrees_x = pixel_coords[0] / pixels_per_degree @@ -491,7 +511,9 @@ def translate(image, shift: Tuple[float, float], image_shape: Tuple[int, int]) - def get_image_with_shape(image: Union[str, np.ndarray]) -> Tuple[np.array, Tuple[int, int]]: if isinstance(image, str): # tf models return strings after preprocessing image = cv2.imread(image) - rows, cols, _ = image.shape + rows, cols, _ = image.shape # cv2 uses height, width, channels + else: + _, rows, cols, = image.shape # pytorch and keras use channels, height, width return image, (rows, cols) @staticmethod @@ -500,26 +522,6 @@ def reshape_microsaccaded_images(images: List) -> Union[List[str], np.ndarray]: return images return np.stack(images, axis=0) - def build_microsaccade_coords(self, stimuli_paths: np.array) -> Dict: - return { - 'microsaccade_shift_x_pixels': ('presentation', self.unpack_microsaccade_coords( - np.unique(stimuli_paths), - pixels_or_degrees='pixels', - dim=0)), - 'microsaccade_shift_y_pixels': ('presentation', self.unpack_microsaccade_coords( - np.unique(stimuli_paths), - pixels_or_degrees='pixels', - dim=1)), - 'microsaccade_shift_x_degrees': ('presentation', self.unpack_microsaccade_coords( - np.unique(stimuli_paths), - pixels_or_degrees='degrees', - dim=0)), - 'microsaccade_shift_y_degrees': ('presentation', self.unpack_microsaccade_coords( - np.unique(stimuli_paths), - pixels_or_degrees='degrees', - dim=1)) - } - def change_dict(d, change_function, keep_name=False, multithread=False): if not multithread: From 9245cb4a9541eed8fa43cd4b89c08cc89241c208 Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Fri, 1 Mar 2024 11:54:56 +0100 Subject: [PATCH 36/41] cv2 reshaping in translate --- .../model_helpers/activations/core.py | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/brainscore_vision/model_helpers/activations/core.py b/brainscore_vision/model_helpers/activations/core.py index 4446ac9eb..21a3cce8f 100644 --- a/brainscore_vision/model_helpers/activations/core.py +++ b/brainscore_vision/model_helpers/activations/core.py @@ -368,7 +368,7 @@ def translate_images(self, images: List[Union[str, np.ndarray]], image_paths: Li output_images.append(images[index]) else: # translate images according to microsaccades if we are using microsaccades - image, image_shape = self.get_image_with_shape(images[index]) + image, image_shape, image_is_channels_first = self.get_image_with_shape(images[index]) microsaccade_location_pixels = self.select_microsaccade(image_path=image_path, trial_number=trial_number, image_shape=image_shape) @@ -376,13 +376,15 @@ def translate_images(self, images: List[Union[str, np.ndarray]], image_paths: Li output_images.append(self.translate_image(image=image, microsaccade_location=microsaccade_location_pixels, image_shape=image_shape, - return_string=return_string)) + return_string=return_string, + image_is_channels_first=image_is_channels_first)) return self.reshape_microsaccaded_images(output_images) def translate_image(self, image: str, microsaccade_location: Tuple[float, float], image_shape: Tuple[int, int], - return_string: bool) -> str: + return_string: bool, image_is_channels_first: bool) -> str: """Translates and saves a temporary image to temporary_fp.""" - translated_image = self.translate(image, microsaccade_location, image_shape) + translated_image = self.translate(image=image, shift=microsaccade_location, image_shape=image_shape, + image_is_channels_first=image_is_channels_first) if not return_string: # if the model accepts ndarrays after preprocessing, return one return translated_image else: # if the model accepts strings after preprocessing, write temp file @@ -497,24 +499,31 @@ def remove_temporary_files(temporary_file_paths: List[str]) -> None: pass @staticmethod - def translate(image, shift: Tuple[float, float], image_shape: Tuple[int, int]) -> np.array: + def translate(image: np.array, shift: Tuple[float, float], image_shape: Tuple[int, int], + image_is_channels_first: bool) -> np.array: rows, cols = image_shape # translation matrix M = np.float32([[1, 0, shift[0]], [0, 1, shift[1]]]) + if image_is_channels_first: + image = np.transpose(image, (1, 2, 0)) # cv2 expects channels last # Apply translation, filling new line(s) with line(s) closest to it(them). translated_image = cv2.warpAffine(image, M, (cols, rows), flags=cv2.INTER_LINEAR, # for sub-pixel shifts borderMode=cv2.BORDER_REPLICATE) + if image_is_channels_first: + translated_image = np.transpose(translated_image, (2, 0, 1)) # convert the image back to channels-first return translated_image @staticmethod - def get_image_with_shape(image: Union[str, np.ndarray]) -> Tuple[np.array, Tuple[int, int]]: + def get_image_with_shape(image: Union[str, np.ndarray]) -> Tuple[np.array, Tuple[int, int], bool]: if isinstance(image, str): # tf models return strings after preprocessing image = cv2.imread(image) rows, cols, _ = image.shape # cv2 uses height, width, channels + image_is_channels_first = False else: _, rows, cols, = image.shape # pytorch and keras use channels, height, width - return image, (rows, cols) + image_is_channels_first = True + return image, (rows, cols), image_is_channels_first @staticmethod def reshape_microsaccaded_images(images: List) -> Union[List[str], np.ndarray]: From ead4cee55175b3da9988b8b3e9a765b32144de95 Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Fri, 1 Mar 2024 12:16:07 +0100 Subject: [PATCH 37/41] add test for exact microsaccades --- .../activations/test___init__.py | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/tests/test_model_helpers/activations/test___init__.py b/tests/test_model_helpers/activations/test___init__.py index 551c5cd7a..b72e77510 100644 --- a/tests/test_model_helpers/activations/test___init__.py +++ b/tests/test_model_helpers/activations/test___init__.py @@ -163,6 +163,20 @@ def tfslim_vgg16(): pytest.param(tfslim_vgg16, ['vgg_16/pool5'], marks=pytest.mark.memory_intense), ] +# exact microsaccades for pytorch_alexnet, grayscale.png, for 1 and 10 number_of_trials +exact_microsaccades = {"x_degrees": {1: np.array([0.]), + 10: np.array([0., -0.00639121, -0.02114204, -0.02616418, -0.02128906, + -0.00941355, 0.00596172, 0.02166913, 0.03523793, 0.04498976])}, + "y_degrees": {1: np.array([0.]), + 10: np.array([0., 0.0144621, 0.00728107, -0.00808922, -0.02338324, -0.0340791, + -0.03826824, -0.03578336, -0.02753704, -0.01503068])}, + "x_pixels": {1: np.array([0.]), + 10: np.array([0., -0.17895397, -0.59197722, -0.73259714, -0.59609364, -0.26357934, + 0.16692818, 0.60673569, 0.98666196, 1.25971335])}, + "y_pixels": {1: np.array([0.]), + 10: np.array([0., 0.40493885, 0.20386999, -0.22649819, -0.65473077, -0.95421482, + -1.07151061, -1.00193403, -0.77103707, -0.42085896])}} + @pytest.mark.parametrize("image_name", ['rgb.jpg', 'grayscale.png', 'grayscale2.jpg', 'grayscale_alpha.png', 'palletized.png']) @@ -295,6 +309,41 @@ def test_exact_activations(pca_components): assert (activations == expected).all() +@pytest.mark.memory_intense +@pytest.mark.parametrize("number_of_trials", [1, 10]) +def test_exact_microsaccades(number_of_trials): + image_name = 'grayscale.png' + stimulus_paths = [os.path.join(os.path.dirname(__file__), image_name)] + activations_extractor = pytorch_alexnet() + # when using microsaccades, the ModelCommitment sets its visual angle. Since this test skips the ModelCommitment, + # we set it here manually. + activations_extractor._extractor.set_visual_degrees(8.) + # the exact microsaccades were computed at this extent + assert activations_extractor._extractor._microsaccade_helper.microsaccade_extent_degrees == 0.05 + + activations = activations_extractor(stimuli=stimulus_paths, layers=['features.12'], + number_of_trials=number_of_trials, require_variance=True) + + assert activations is not None + # test with np.isclose instead of == since while the arrays are visually equal, == often fails due to float errors + assert np.isclose(activations['microsaccade_shift_x_degrees'].values, + exact_microsaccades['x_degrees'][number_of_trials], + rtol=1e-05, + atol=1e-08).all() + assert np.isclose(activations['microsaccade_shift_y_degrees'].values, + exact_microsaccades['y_degrees'][number_of_trials], + rtol=1e-05, + atol=1e-08).all() + assert np.isclose(activations['microsaccade_shift_x_pixels'].values, + exact_microsaccades['x_pixels'][number_of_trials], + rtol=1e-05, + atol=1e-08).all() + assert np.isclose(activations['microsaccade_shift_y_pixels'].values, + exact_microsaccades['y_pixels'][number_of_trials], + rtol=1e-05, + atol=1e-08).all() + + @pytest.mark.memory_intense @pytest.mark.parametrize(["model_ctr", "internal_layers"], [ (pytorch_alexnet, ['features.12', 'classifier.5']), From 63990cfba5ba07ef334f272c910b27ad3b40a245 Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Fri, 1 Mar 2024 13:17:46 +0100 Subject: [PATCH 38/41] fix microsaccade indexing --- brainscore_vision/model_helpers/activations/core.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/brainscore_vision/model_helpers/activations/core.py b/brainscore_vision/model_helpers/activations/core.py index 21a3cce8f..3c3f5e10d 100644 --- a/brainscore_vision/model_helpers/activations/core.py +++ b/brainscore_vision/model_helpers/activations/core.py @@ -166,7 +166,10 @@ def _get_activations_batched(self, paths, layers, batch_size: int, require_varia batch_activations = OrderedDict() # compute activations on the entire batch one microsaccade shift at a time. for shift_number in range(self._microsaccade_helper.number_of_trials): - activations = self._get_batch_activations(inputs=batch_inputs, layer_names=layers, batch_size=batch_size, require_variance=require_variance, + activations = self._get_batch_activations(inputs=batch_inputs, + layer_names=layers, + batch_size=batch_size, + require_variance=require_variance, trial_number=shift_number) for layer_name, layer_output in activations.items(): @@ -407,7 +410,7 @@ def select_microsaccade(self, image_path: str, trial_number: int, image_shape: T # if we did not already compute `self.microsaccades`, we build them first. if image_path not in self.microsaccades.keys(): self.build_microsaccades(image_path=image_path, image_shape=image_shape) - return self.microsaccades['pixels'][image_path][trial_number - 1] + return self.microsaccades['pixels'][image_path][trial_number] def build_microsaccades(self, image_path: str, image_shape: Tuple[int, int]): if image_shape[0] != image_shape[1]: From 453a2a6306114f39538a1a10ef4c128bb7a8d3c7 Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Mon, 4 Mar 2024 09:27:57 +0100 Subject: [PATCH 39/41] rename test to be in line with current naming --- tests/test_model_helpers/activations/test___init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_model_helpers/activations/test___init__.py b/tests/test_model_helpers/activations/test___init__.py index b72e77510..199b735c5 100644 --- a/tests/test_model_helpers/activations/test___init__.py +++ b/tests/test_model_helpers/activations/test___init__.py @@ -229,7 +229,7 @@ def test_require_variance_has_shift_coords(model_ctr, layers, image_name, number @pytest.mark.parametrize(["model_ctr", "layers"], models_layers) @pytest.mark.parametrize("require_variance", [False, True]) @pytest.mark.parametrize("number_of_trials", [1, 2, 10]) -def test_model_requirements(model_ctr, layers, image_name, require_variance, number_of_trials): +def test_require_variance_presentation_length(model_ctr, layers, image_name, require_variance, number_of_trials): stimulus_paths = [os.path.join(os.path.dirname(__file__), image_name)] activations_extractor = model_ctr() # when using microsaccades, the ModelCommitment sets its visual angle. Since this test skips the ModelCommitment, From 6e9287898769af088745ec2d5594a3c9e56ad7a3 Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Mon, 4 Mar 2024 09:28:44 +0100 Subject: [PATCH 40/41] add require_variance to _from_paths_stored --- brainscore_vision/model_helpers/activations/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brainscore_vision/model_helpers/activations/core.py b/brainscore_vision/model_helpers/activations/core.py index 3c3f5e10d..f37631dbb 100644 --- a/brainscore_vision/model_helpers/activations/core.py +++ b/brainscore_vision/model_helpers/activations/core.py @@ -107,7 +107,7 @@ def from_paths(self, stimuli_paths, layers, stimuli_identifier=None, require_var @store_xarray(identifier_ignore=['stimuli_paths', 'layers'], combine_fields={'layers': 'layer'}) def _from_paths_stored(self, identifier, layers, stimuli_identifier, stimuli_paths, number_of_trials: int = 1, require_variance: bool = False): - return self._from_paths(layers=layers, stimuli_paths=stimuli_paths) + return self._from_paths(layers=layers, stimuli_paths=stimuli_paths, require_variance=require_variance) def _from_paths(self, layers, stimuli_paths, require_variance: bool = False): if len(layers) == 0: From 9a67b152dd8f1950e70f648977f532a8a7233cd6 Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Tue, 5 Mar 2024 10:09:30 +0100 Subject: [PATCH 41/41] reduce unnecessarily long test times by reducing the number of trials tests run for, while keeping the test the same --- tests/test_model_helpers/activations/test___init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_model_helpers/activations/test___init__.py b/tests/test_model_helpers/activations/test___init__.py index 199b735c5..3d1306329 100644 --- a/tests/test_model_helpers/activations/test___init__.py +++ b/tests/test_model_helpers/activations/test___init__.py @@ -206,7 +206,7 @@ def test_from_image_path(model_ctr, layers, image_name, pca_components, logits): @pytest.mark.parametrize("image_name", ['rgb.jpg', 'grayscale.png', 'grayscale2.jpg', 'grayscale_alpha.png', 'palletized.png']) @pytest.mark.parametrize(["model_ctr", "layers"], models_layers) -@pytest.mark.parametrize("number_of_trials", [1, 5, 25]) +@pytest.mark.parametrize("number_of_trials", [1, 3, 10]) def test_require_variance_has_shift_coords(model_ctr, layers, image_name, number_of_trials): stimulus_paths = [os.path.join(os.path.dirname(__file__), image_name)] activations_extractor = model_ctr() @@ -228,7 +228,7 @@ def test_require_variance_has_shift_coords(model_ctr, layers, image_name, number 'palletized.png']) @pytest.mark.parametrize(["model_ctr", "layers"], models_layers) @pytest.mark.parametrize("require_variance", [False, True]) -@pytest.mark.parametrize("number_of_trials", [1, 2, 10]) +@pytest.mark.parametrize("number_of_trials", [1, 3, 10]) def test_require_variance_presentation_length(model_ctr, layers, image_name, require_variance, number_of_trials): stimulus_paths = [os.path.join(os.path.dirname(__file__), image_name)] activations_extractor = model_ctr()