Leaderboard

mteb/__init__.py

@@ -10,8 +10,8 @@
     CoIR,
 )
 from mteb.evaluation import *
-from mteb.load_results import load_results
-from mteb.models import get_model, get_model_meta
+from mteb.load_results import BenchmarkResults, load_results
+from mteb.models import get_model, get_model_meta, get_model_metas
 from mteb.overview import TASKS_REGISTRY, get_task, get_tasks
 
 from .benchmarks.benchmarks import Benchmark
@@ -31,8 +31,10 @@
     "get_task",
     "get_model",
     "get_model_meta",
+    "get_model_metas",
     "load_results",
     "Benchmark",
     "get_benchmark",
     "get_benchmarks",
+    "BenchmarkResults",
 ]

mteb/abstasks/AbsTask.py

@@ -310,3 +310,6 @@ def __repr__(self) -> str:
         return (
             f"{self.__class__.__name__}(name='{self.metadata.name}', languages={langs})"
         )
+
+    def __hash__(self) -> int:
+        return hash(self.metadata)

mteb/abstasks/AbsTaskBitextMining.py

@@ -8,7 +8,7 @@
 from mteb.encoder_interface import Encoder
 
 from ..evaluation.evaluators import BitextMiningEvaluator
-from ..load_results.mteb_results import HFSubset, ScoresDict
+from ..load_results.task_results import HFSubset, ScoresDict
 from .AbsTask import AbsTask, DescriptiveStatistics
 
 logger = logging.getLogger(__name__)

mteb/abstasks/AbsTaskClassification.py

@@ -14,7 +14,7 @@
     kNNClassificationEvaluatorPytorch,
     logRegClassificationEvaluator,
 )
-from ..load_results.mteb_results import HFSubset, ScoresDict
+from ..load_results.task_results import HFSubset, ScoresDict
 from .AbsTask import AbsTask, DescriptiveStatistics
 
 logger = logging.getLogger(__name__)

mteb/abstasks/AbsTaskClustering.py

@@ -9,7 +9,7 @@
 from datasets import Dataset
 
 from mteb.encoder_interface import Encoder, EncoderWithQueryCorpusEncode
-from mteb.load_results.mteb_results import ScoresDict
+from mteb.load_results.task_results import ScoresDict
 
 from ..evaluation.evaluators import ClusteringEvaluator
 from .AbsTask import AbsTask, DescriptiveStatistics

mteb/abstasks/AbsTaskClusteringFast.py

@@ -15,7 +15,7 @@
 from mteb.encoder_interface import Encoder
 
 from ..evaluation.evaluators.model_encode import model_encode
-from ..load_results.mteb_results import HFSubset
+from ..load_results.task_results import HFSubset
 from .AbsTask import AbsTask, DescriptiveStatistics
 
 logger = logging.getLogger(__name__)

mteb/abstasks/AbsTaskMultilabelClassification.py

@@ -15,7 +15,7 @@
 from mteb.encoder_interface import Encoder
 
 from ..evaluation.evaluators.model_encode import model_encode
-from ..load_results.mteb_results import HFSubset, ScoresDict
+from ..load_results.task_results import HFSubset, ScoresDict
 from .AbsTask import AbsTask, DescriptiveStatistics
 
 logger = logging.getLogger(__name__)

mteb/abstasks/AbsTaskPairClassification.py

@@ -7,7 +7,7 @@
 
 from ..encoder_interface import Encoder, EncoderWithQueryCorpusEncode
 from ..evaluation.evaluators import PairClassificationEvaluator
-from ..load_results.mteb_results import ScoresDict
+from ..load_results.task_results import ScoresDict
 from .AbsTask import AbsTask, DescriptiveStatistics
 
 logger = logging.getLogger(__name__)

mteb/abstasks/AbsTaskReranking.py

@@ -5,7 +5,7 @@
 from datasets import Dataset
 
 from mteb.encoder_interface import Encoder, EncoderWithQueryCorpusEncode
-from mteb.load_results.mteb_results import ScoresDict
+from mteb.load_results.task_results import ScoresDict
 
 from ..evaluation.evaluators import RerankingEvaluator
 from .AbsTask import AbsTask, DescriptiveStatistics

mteb/abstasks/AbsTaskRetrieval.py

@@ -13,7 +13,7 @@
 from mteb.abstasks.TaskMetadata import HFSubset
 
 from ..evaluation.evaluators import RetrievalEvaluator
-from ..load_results.mteb_results import ScoresDict
+from ..load_results.task_results import ScoresDict
 from .AbsTask import AbsTask, DescriptiveStatistics
 
 logger = logging.getLogger(__name__)

mteb/abstasks/AbsTaskSTS.py

@@ -4,7 +4,7 @@
 from typing import Any
 
 from ..evaluation.evaluators import STSEvaluator
-from ..load_results.mteb_results import ScoresDict
+from ..load_results.task_results import ScoresDict
 from .AbsTask import AbsTask, DescriptiveStatistics
 
 logger = logging.getLogger(__name__)

mteb/abstasks/AbsTaskSpeedTask.py

@@ -8,7 +8,7 @@
 import numpy as np
 
 from mteb.encoder_interface import Encoder, EncoderWithQueryCorpusEncode
-from mteb.load_results.mteb_results import ScoresDict
+from mteb.load_results.task_results import ScoresDict
 
 from .AbsTask import AbsTask
 

mteb/abstasks/AbsTaskSummarization.py

@@ -6,7 +6,7 @@
 import numpy as np
 
 from mteb.encoder_interface import Encoder
-from mteb.load_results.mteb_results import ScoresDict
+from mteb.load_results.task_results import ScoresDict
 
 from ..evaluation.evaluators import SummarizationEvaluator
 from .AbsTask import AbsTask, DescriptiveStatistics

mteb/abstasks/TaskMetadata.py

@@ -6,7 +6,7 @@
 from typing import Annotated, Any, Union
 
 from pydantic import AnyUrl, BaseModel, BeforeValidator, TypeAdapter, field_validator
-from typing_extensions import Literal
+from typing_extensions import Annotated, Literal
 
 from ..languages import (
     ISO_LANGUAGE_SCRIPT,
@@ -352,3 +352,6 @@ def intext_citation(self, include_cite: bool = True) -> str:
                 )
             return f"\\cite{{{cite}}}"
         return cite
+
+    def __hash__(self) -> int:
+        return hash(self.model_dump_json())

mteb/benchmarks/__init__.py

@@ -1,3 +1,4 @@
 from __future__ import annotations
 
 from mteb.benchmarks.benchmarks import *
+from mteb.benchmarks.get_benchmark import *

mteb/benchmarks/benchmarks.py

@@ -7,6 +7,12 @@
 from pydantic import AnyUrl, BeforeValidator, TypeAdapter
 
 from mteb.abstasks.AbsTask import AbsTask
+from mteb.load_results.benchmark_results import (
+    BenchmarkResults,
+    ModelResult,
+    TaskResult,
+)
+from mteb.load_results.load_results import load_results
 from mteb.overview import get_tasks
 
 http_url_adapter = TypeAdapter(AnyUrl)
@@ -52,6 +58,15 @@ def __len__(self) -> int:
     def __getitem__(self, index):
         return self.tasks[index]
 
+    def load_results(
+        self, base_results: None | BenchmarkResults = None
+    ) -> BenchmarkResults:
+        if base_results is None:
+            base_results = load_results()
+        return base_results.select_tasks(self.tasks)
+
+
+MTEB_MAIN_MULTILINGUAL = Benchmark(name="MTEB(multilingual)", tasks=get_tasks())
 
 MTEB_MAIN_EN = Benchmark(
     name="MTEB(eng)",

mteb/benchmarks/get_benchmark.py

@@ -3,7 +3,7 @@
 import difflib
 
 import mteb.benchmarks.benchmarks as benchmark_module
-from mteb.benchmarks import Benchmark
+from mteb.benchmarks.benchmarks import Benchmark
 
 BENCHMARK_REGISTRY = {
     inst.name: inst

mteb/create_meta.py

@@ -7,8 +7,8 @@
 import yaml
 
 import mteb
-from mteb import MTEBResults
-from mteb.load_results.mteb_results import CQADupstackRetrievalDummy
+from mteb import TaskResult
+from mteb.load_results.task_results import CQADupstackRetrievalDummy
 
 
 def generate_readme(results_folder: Path, from_existing: Path | None = None) -> str:
@@ -45,7 +45,7 @@ def load_model_name(results_folder: Path) -> str:
     return "PLACEHOLDER"
 
 
-def process_task_result(task_result: MTEBResults) -> list[dict[str, Any]]:
+def process_task_result(task_result: TaskResult) -> list[dict[str, Any]]:
     # CQADupstackRetrieval is a combined dataset (special case atm.)
     task = (
         CQADupstackRetrievalDummy()
@@ -84,13 +84,13 @@ def process_task_result(task_result: MTEBResults) -> list[dict[str, Any]]:
     return yaml_results
 
 
-def get_task_results(results_folder: Path) -> list[MTEBResults]:
+def get_task_results(results_folder: Path) -> list[TaskResult]:
     json_files = [
         r
         for r in results_folder.glob("*.json")
         if r.is_file() and r.name != "model_meta.json"
     ]
-    task_results = [MTEBResults.from_disk(path) for path in json_files]
+    task_results = [TaskResult.from_disk(path) for path in json_files]
     task_results = [
         results
         for results in task_results
@@ -102,8 +102,8 @@ def get_task_results(results_folder: Path) -> list[MTEBResults]:
 
 
 def potentially_add_cqadupstack_to_results(
-    results: list[MTEBResults],
-) -> list[MTEBResults]:
+    results: list[TaskResult],
+) -> list[TaskResult]:
     task_list_cqa = {
         "CQADupstackAndroidRetrieval",
         "CQADupstackEnglishRetrieval",
@@ -128,7 +128,7 @@ def potentially_add_cqadupstack_to_results(
     main_scores = [r.get_score(splits=["test"]) for r in cqa_results]
     main_score = float(sum(main_scores) / len(main_scores))
 
-    combined_result = MTEBResults(
+    combined_result = TaskResult(
         task_name="CQADupstackRetrieval",
         dataset_revision="CQADupstackRetrieval_is_a_combined_dataset",
         mteb_version="NA",

mteb/evaluation/MTEB.py

@@ -21,7 +21,7 @@
 
 from ..abstasks import *
 from ..abstasks import AbsTask
-from ..load_results.mteb_results import MTEBResults
+from ..load_results.task_results import TaskResult
 from ..tasks import *
 from . import LangMapping
 
@@ -317,7 +317,7 @@ def run(
         co2_tracker: bool = False,
         encode_kwargs: dict[str, Any] = {},
         **kwargs,
-    ) -> list[MTEBResults]:
+    ) -> list[TaskResult]:
         """Run the evaluation pipeline on the selected tasks.
 
         Args:
@@ -336,7 +336,7 @@ def run(
             kwargs: Additional arguments to be passed to `_run_eval` method and task.load_data.
 
         Returns:
-            A list of MTEBResults objects, one for each task evaluated.
+            A list of TaskResult objects, one for each task evaluated.
         """
         if "batch_size" in kwargs:
             logger.warning(
@@ -376,7 +376,7 @@ def run(
                     logger.info(
                         f"{task.metadata.name} results already exists. Loading results from disk. Set overwrite_results=True to overwrite."
                     )
-                    mteb_results = MTEBResults.from_disk(save_path)
+                    mteb_results = TaskResult.from_disk(save_path)
                     evaluation_results.append(mteb_results)
                     del self.tasks[0]  # empty memory
                     continue
@@ -437,7 +437,7 @@ def run(
                     if verbosity >= 1:
                         logger.info(f"Scores: {results}")
 
-                mteb_task_result = MTEBResults.from_task_results(
+                mteb_task_result = TaskResult.from_task_results(
                     task,
                     task_results,
                     evaluation_time=evaluation_time,

mteb/leaderboard/__init__.py

@@ -0,0 +1 @@
+from mteb.leaderboard.app import demo