Add sparse objects benchmark

brainscore_vision/benchmarks/scialom2024/__init__.py

@@ -0,0 +1,50 @@
+from brainscore_vision import benchmark_registry
+from . import benchmark
+
+# behavioral benchmarks
+benchmark_registry['Scialom2024_rgbBehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('rgb')
+benchmark_registry['Scialom2024_contoursBehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('contours')
+benchmark_registry['Scialom2024_phosphenes-12BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('phosphenes-12')
+benchmark_registry['Scialom2024_phosphenes-16BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('phosphenes-16')
+benchmark_registry['Scialom2024_phosphenes-21BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('phosphenes-21')
+benchmark_registry['Scialom2024_phosphenes-27BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('phosphenes-27')
+benchmark_registry['Scialom2024_phosphenes-35BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('phosphenes-35')
+benchmark_registry['Scialom2024_phosphenes-46BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('phosphenes-46')
+benchmark_registry['Scialom2024_phosphenes-59BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('phosphenes-59')
+benchmark_registry['Scialom2024_phosphenes-77BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('phosphenes-77')
+benchmark_registry['Scialom2024_phosphenes-100BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('phosphenes-100')
+benchmark_registry['Scialom2024_segments-12BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('segments-12')
+benchmark_registry['Scialom2024_segments-16BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('segments-16')
+benchmark_registry['Scialom2024_segments-21BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('segments-21')
+benchmark_registry['Scialom2024_segments-27BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('segments-27')
+benchmark_registry['Scialom2024_segments-35BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('segments-35')
+benchmark_registry['Scialom2024_segments-46BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('segments-46')
+benchmark_registry['Scialom2024_segments-59BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('segments-59')
+benchmark_registry['Scialom2024_segments-77BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('segments-77')
+benchmark_registry['Scialom2024_segments-100BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('segments-100')
+
+# composites
+benchmark_registry['Scialom2024_phosphenes-allBehavioralErrorConsistency'] = lambda: benchmark._Scialom2024BehavioralErrorConsistency('phosphenes-all')
+benchmark_registry['Scialom2024_segments-allBehavioralErrorConsistency'] = lambda: benchmark._Scialom2024BehavioralErrorConsistency('segments-all')
+
+# engineering benchmarks
+benchmark_registry['Scialom2024_rgbEngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('rgb')
+benchmark_registry['Scialom2024_contoursEngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('contours')
+benchmark_registry['Scialom2024_phosphenes-12EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('phosphenes-12')
+benchmark_registry['Scialom2024_phosphenes-16EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('phosphenes-16')
+benchmark_registry['Scialom2024_phosphenes-21EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('phosphenes-21')
+benchmark_registry['Scialom2024_phosphenes-27EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('phosphenes-27')
+benchmark_registry['Scialom2024_phosphenes-35EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('phosphenes-35')
+benchmark_registry['Scialom2024_phosphenes-46EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('phosphenes-46')
+benchmark_registry['Scialom2024_phosphenes-59EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('phosphenes-59')
+benchmark_registry['Scialom2024_phosphenes-77EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('phosphenes-77')
+benchmark_registry['Scialom2024_phosphenes-100EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('phosphenes-100')
+benchmark_registry['Scialom2024_segments-12EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('segments-12')
+benchmark_registry['Scialom2024_segments-16EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('segments-16')
+benchmark_registry['Scialom2024_segments-21EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('segments-21')
+benchmark_registry['Scialom2024_segments-27EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('segments-27')
+benchmark_registry['Scialom2024_segments-35EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('segments-35')
+benchmark_registry['Scialom2024_segments-46EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('segments-46')
+benchmark_registry['Scialom2024_segments-59EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('segments-59')
+benchmark_registry['Scialom2024_segments-77EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('segments-77')
+benchmark_registry['Scialom2024_segments-100EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('segments-100')

brainscore_vision/benchmarks/scialom2024/benchmark.py

@@ -0,0 +1,97 @@
+import numpy as np
+
+import brainscore_vision
+from brainio.assemblies import BehavioralAssembly
+from brainscore_vision import load_dataset, load_metric
+from brainscore_vision.benchmark_helpers.screen import place_on_screen
+from brainscore_vision.benchmarks import BenchmarkBase
+from brainscore_vision.metrics import Score
+from brainscore_vision.model_interface import BrainModel
+from brainscore_vision.utils import LazyLoad
+
+BIBTEX = ""  # to appear in a future article
+
+
+class _Scialom2024BehavioralErrorConsistency(BenchmarkBase):
+    def __init__(self, dataset):
+        self._metric = load_metric('error_consistency')
+        self._assembly = LazyLoad(lambda: load_assembly(dataset))
+        self._visual_degrees = 8
+        self._number_of_trials = 1
+
+        super(_Scialom2024BehavioralErrorConsistency, self).__init__(
+            identifier=f'Scialom2024_{dataset}-error_consistency', version=1,
+            ceiling_func=lambda: self._metric.ceiling(self._assembly),
+            parent='Scialom2024',
+            bibtex=BIBTEX)
+
+    def __call__(self, candidate: BrainModel):
+        choice_labels = set(self._assembly['truth'].values)
+        choice_labels = list(sorted(choice_labels))
+        candidate.start_task(BrainModel.Task.label, choice_labels)
+        stimulus_set = place_on_screen(self._assembly.stimulus_set, target_visual_degrees=candidate.visual_degrees(),
+                                       source_visual_degrees=self._visual_degrees)
+        labels = candidate.look_at(stimulus_set, number_of_trials=self._number_of_trials)
+        raw_score = self._metric(labels, self._assembly)
+        ceiling = self.ceiling
+        score = raw_score / ceiling
+        score.attrs['raw'] = raw_score
+        score.attrs['ceiling'] = ceiling
+        return score
+
+
+class _Scialom2024BehavioralAccuracyDistance(BenchmarkBase):
+    # behavioral benchmark
+    def __init__(self, dataset):
+        self._metric = load_metric('accuracy_distance')
+        self._assembly = LazyLoad(lambda: load_assembly(dataset))
+        super(_Scialom2024BehavioralAccuracyDistance, self).__init__(
+            identifier=f'Scialom2024_{dataset}-behavioral_accuracy', version=1,
+            ceiling_func=lambda: self._metric.ceiling(self._assembly),
+            parent='Scialom2024',
+            bibtex=BIBTEX)
+
+    def __call__(self, candidate: BrainModel):
+        choice_labels = set(self._assembly.stimulus_set['truth'].values)
+        choice_labels = list(sorted(choice_labels))
+        candidate.start_task(BrainModel.Task.label, choice_labels)
+        labels = candidate.look_at(self._assembly.stimulus_set, number_of_trials=1)
+        raw_score = self._metric(labels, target=self._assembly)
+        ceiling = self.ceiling
+        score = raw_score / ceiling
+        # ensure score <= 1.0
+        if score.values > 1:
+            score = Score(np.array(1.))
+        score.attrs['raw'] = raw_score
+        score.attrs['ceiling'] = ceiling
+        return score
+
+
+class _Scialom2024EngineeringAccuracy(BenchmarkBase):
+    # engineering/ML benchmark
+    def __init__(self, dataset):
+        self._metric = load_metric('accuracy')
+        # no lazyload because needed in candidate.look_at()
+        self._stimulus_set = load_assembly(dataset).stimulus_set
+        super(_Scialom2024EngineeringAccuracy, self).__init__(
+            identifier=f'Scialom2024_{dataset}-engineering_accuracy', version=1,
+            ceiling_func=lambda: Score(1),
+            parent='Scialom2024-top1',
+            bibtex=BIBTEX)
+
+    def __call__(self, candidate: BrainModel):
+        choice_labels = set(self._stimulus_set['truth'].values)
+        choice_labels = list(sorted(choice_labels))
+        candidate.start_task(BrainModel.Task.label, choice_labels)
+        labels = candidate.look_at(self._stimulus_set, number_of_trials=10)
+        raw_score = self._metric(labels, target=self._stimulus_set['truth'].values)
+        ceiling = Score(np.array(1.))
+        score = raw_score / ceiling
+        score.attrs['raw'] = raw_score
+        score.attrs['ceiling'] = ceiling
+        return score
+
+
+def load_assembly(dataset: str) -> BehavioralAssembly:
+    assembly = brainscore_vision.load_dataset(f'Scialom2024_{dataset}')
+    return assembly

brainscore_vision/benchmarks/scialom2024/test.py

@@ -0,0 +1,158 @@
+from pathlib import Path
+
+import numpy as np
+import pytest
+from pytest import approx
+
+from brainio.assemblies import BehavioralAssembly
+from brainscore_vision import benchmark_registry, load_benchmark
+from brainscore_vision.benchmark_helpers import PrecomputedFeatures
+from brainscore_vision.data_helpers import s3
+
+
+@pytest.mark.parametrize('benchmark', [
+    'Scialom2024_rgbBehavioralAccuracyDistance',
+    'Scialom2024_contoursBehavioralAccuracyDistance',
+    'Scialom2024_phosphenes-12BehavioralAccuracyDistance',
+    'Scialom2024_phosphenes-16BehavioralAccuracyDistance',
+    'Scialom2024_phosphenes-21BehavioralAccuracyDistance',
+    'Scialom2024_phosphenes-27BehavioralAccuracyDistance',
+    'Scialom2024_phosphenes-35BehavioralAccuracyDistance',
+    'Scialom2024_phosphenes-46BehavioralAccuracyDistance',
+    'Scialom2024_phosphenes-59BehavioralAccuracyDistance',
+    'Scialom2024_phosphenes-77BehavioralAccuracyDistance',
+    'Scialom2024_phosphenes-100BehavioralAccuracyDistance',
+    'Scialom2024_segments-12BehavioralAccuracyDistance',
+    'Scialom2024_segments-16BehavioralAccuracyDistance',
+    'Scialom2024_segments-21BehavioralAccuracyDistance',
+    'Scialom2024_segments-27BehavioralAccuracyDistance',
+    'Scialom2024_segments-35BehavioralAccuracyDistance',
+    'Scialom2024_segments-46BehavioralAccuracyDistance',
+    'Scialom2024_segments-59BehavioralAccuracyDistance',
+    'Scialom2024_segments-77BehavioralAccuracyDistance',
+    'Scialom2024_segments-100BehavioralAccuracyDistance',
+    'Scialom2024_phosphenes-allBehavioralErrorConsistency',
+    'Scialom2024_segments-allBehavioralErrorConsistency'
+])
+def test_benchmark_registry(benchmark):
+    assert benchmark in benchmark_registry
+
+
+class TestBehavioral:
+    @pytest.mark.private_access
+    @pytest.mark.parametrize('dataset, expected_ceiling', [
+        ('rgb', approx(0.98513, abs=0.001)),
+        ('contours', approx(0.97848, abs=0.001)),
+        ('phosphenes-12', approx(0.95416, abs=0.001)),
+        ('phosphenes-16', approx(0.92583, abs=0.001)),
+        ('phosphenes-21', approx(0.92166, abs=0.001)),
+        ('phosphenes-27', approx(0.86888, abs=0.001)),
+        ('phosphenes-35', approx(0.87277, abs=0.001)),
+        ('phosphenes-46', approx(0.87125, abs=0.001)),
+        ('phosphenes-59', approx(0.87625, abs=0.001)),
+        ('phosphenes-77', approx(0.89277, abs=0.001)),
+        ('phosphenes-100', approx(0.89930, abs=0.001)),
+        ('segments-12', approx(0.89847, abs=0.001)),
+        ('segments-16', approx(0.89055, abs=0.001)),
+        ('segments-21', approx(0.88083, abs=0.001)),
+        ('segments-27', approx(0.87083, abs=0.001)),
+        ('segments-35', approx(0.86333, abs=0.001)),
+        ('segments-46', approx(0.90250, abs=0.001)),
+        ('segments-59', approx(0.87847, abs=0.001)),
+        ('segments-77', approx(0.89013, abs=0.001)),
+        ('segments-100', approx(0.93236, abs=0.001)),  # all of the above are AccuracyDistance
+        ('phosphenes-all', approx(0.45755, abs=0.01)),  # alls are ErrorConsistency
+        ('segments-all', approx(0.42529, abs=0.01)),
+    ])
+    def test_dataset_ceiling(self, dataset, expected_ceiling):
+        if 'all' in dataset:
+            benchmark = f"Scialom2024_{dataset}BehavioralErrorConsistency"
+        else:
+            benchmark = f"Scialom2024_{dataset}BehavioralAccuracyDistance"
+        benchmark = load_benchmark(benchmark)
+        ceiling = benchmark.ceiling
+        assert ceiling == expected_ceiling
+
+    @pytest.mark.private_access
+    @pytest.mark.parametrize('dataset, expected_raw_score', [
+        ('rgb', approx(0.92666, abs=0.001)),
+        ('contours', approx(0.26708, abs=0.001)),
+        ('phosphenes-12', approx(0.87666, abs=0.001)),
+        ('phosphenes-16', approx(0.83666, abs=0.001)),
+        ('phosphenes-21', approx(0.83166, abs=0.001)),
+        ('phosphenes-27', approx(0.73666, abs=0.001)),
+        ('phosphenes-35', approx(0.72416, abs=0.001)),
+        ('phosphenes-46', approx(0.59500, abs=0.001)),
+        ('phosphenes-59', approx(0.50666, abs=0.001)),
+        ('phosphenes-77', approx(0.42083, abs=0.001)),
+        ('phosphenes-100', approx(0.33166, abs=0.001)),
+        ('segments-12', approx(0.81500, abs=0.001)),
+        ('segments-16', approx(0.73750, abs=0.001)),
+        ('segments-21', approx(0.69666, abs=0.001)),
+        ('segments-27', approx(0.59500, abs=0.001)),
+        ('segments-35', approx(0.52666, abs=0.001)),
+        ('segments-46', approx(0.42166, abs=0.001)),
+        ('segments-59', approx(0.34583, abs=0.001)),
+        ('segments-77', approx(0.28916, abs=0.001)),
+        ('segments-100', approx(0.19750, abs=0.001)),  # all of the above are AccuracyDistance
+        ('phosphenes-all', approx(0.18057, abs=0.01)),  # alls are ErrorConsistency
+        ('segments-all', approx(0.15181, abs=0.01)),
+    ])
+    def test_model_8_degrees(self, dataset, expected_raw_score):
+        if 'all' in dataset:
+            benchmark = f"Scialom2024_{dataset}BehavioralErrorConsistency"
+        else:
+            benchmark = f"Scialom2024_{dataset}BehavioralAccuracyDistance"
+        benchmark = load_benchmark(benchmark)
+        nc_filename = dataset.upper() if dataset == 'rgb' else dataset
+        filename = f"resnet50_julios_Scialom2024_{nc_filename}.nc"
+        precomputed_features = Path(__file__).parent / filename
+        s3.download_file_if_not_exists(precomputed_features,
+                                       bucket='brainscore-vision', remote_filepath=f'benchmarks/Scialom2024/{filename}')
+        precomputed_features = BehavioralAssembly.from_files(file_path=precomputed_features)
+        precomputed_features = PrecomputedFeatures(precomputed_features, visual_degrees=8)
+        score = benchmark(precomputed_features)
+        raw_score = score.raw
+        # division by ceiling <= 1 should result in higher score
+        assert score >= raw_score
+        assert raw_score == expected_raw_score
+
+
+class TestEngineering:
+    @pytest.mark.private_access
+    @pytest.mark.parametrize('dataset, expected_accuracy', [
+        ('rgb', approx(0.91666, abs=0.001)),
+        ('contours', approx(0.25000, abs=0.001)),
+        ('phosphenes-12', approx(0.08333, abs=0.001)),
+        ('phosphenes-16', approx(0.08333, abs=0.001)),
+        ('phosphenes-21', approx(0.08333, abs=0.001)),
+        ('phosphenes-27', approx(0.08333, abs=0.001)),
+        ('phosphenes-35', approx(0.12500, abs=0.001)),
+        ('phosphenes-46', approx(0.08333, abs=0.001)),
+        ('phosphenes-59', approx(0.08333, abs=0.001)),
+        ('phosphenes-77', approx(0.08333, abs=0.001)),
+        ('phosphenes-100', approx(0.08333, abs=0.001)),
+        ('segments-12', approx(0.08333, abs=0.001)),
+        ('segments-16', approx(0.08333, abs=0.001)),
+        ('segments-21', approx(0.08333, abs=0.001)),
+        ('segments-27', approx(0.08333, abs=0.001)),
+        ('segments-35', approx(0.08333, abs=0.001)),
+        ('segments-46', approx(0.08333, abs=0.001)),
+        ('segments-59', approx(0.08333, abs=0.001)),
+        ('segments-77', approx(0.10416, abs=0.001)),
+        ('segments-100', approx(0.10416, abs=0.001))
+    ])
+    def test_accuracy(self, dataset, expected_accuracy):
+        benchmark = load_benchmark(f"Scialom2024_{dataset}EngineeringAccuracy")
+        nc_filename = dataset.upper() if dataset == 'rgb' else dataset
+        filename = f"resnet50_julios_Scialom2024_{nc_filename}.nc"
+        precomputed_features = Path(__file__).parent / filename
+        s3.download_file_if_not_exists(precomputed_features,
+                                       bucket='brainio-brainscore', remote_filepath=f'/benchmarks/Scialom2024/{filename}')
+        precomputed_features = BehavioralAssembly.from_files(file_path=precomputed_features)
+        precomputed_features = PrecomputedFeatures(precomputed_features, visual_degrees=8)
+        score = benchmark(precomputed_features)
+        raw_score = score.raw
+        # division by ceiling <= 1 should result in higher score
+        assert score >= raw_score
+        assert raw_score == expected_accuracy