Merge branch 'main' of https://github.com/KennethEnevoldsen/scandinav…

…ian-embedding-benchmark into historical_task

CHANGELOG.md

@@ -2,6 +2,29 @@
 
 
 
+## v0.13.7 (2024-11-14)
+
+### Fix
+
+* fix: Add Jina
+
+* fix: Add Jina
+
+I am still running the model
+
+* Add results for Jina
+
+* Add results for Jina
+
+* format fixes
+
+* Update mkdocs to legacy python
+
+---------
+
+Co-authored-by: Your Name <you@example.com> ([`84c2f91`](https://github.com/KennethEnevoldsen/scandinavian-embedding-benchmark/commit/84c2f912588d16d346b838b8e0e87a1df1733f4f))
+
+
 ## v0.13.6 (2024-07-23)
 
 ### Documentation

docs/create_desc_stats.py

@@ -1,9 +1,10 @@
 from pathlib import Path
 
 import pandas as pd
-from seb import registries
 from tqdm import tqdm
 
+from seb import registries
+
 
 def insert_table(file: Path, table: str) -> None:
     # Read the original Markdown file

docs/update_benchmark_tables.py

@@ -13,8 +13,9 @@
 
 import numpy as np
 import pandas as pd
-import seb
 from datawrapper import Datawrapper
+
+import seb
 from seb.full_benchmark import BENCHMARKS
 from seb.registered_tasks.speed import CPUSpeedTask
 

makefile

@@ -1,7 +1,8 @@
 install:
 	@echo "--- 🚀 Installing project ---"
 	pip install pip --upgrade
-	pip install -e ".[dev, docs, openai, cohere, tests, mistral, fasttext]" 
+	pip install -e ".[dev, docs, openai, cohere, tests, mistral, fasttext, jina]" 
+	# uv sync --extra jina --extra dev --extra docs --extra tests
 
 static-type-check:
 	@echo "--- 🔍 Running static type check ---"

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "seb"
-version = "0.13.6"
+version = "0.13.7"
 authors = [
   { name = "Kenneth Enevoldsen", email = "Kennethcenevoldsen@gmail.com" },
 ]
@@ -25,6 +25,7 @@ dependencies = [
   "catalogue>=2.0.8",
   "radicli>=0.0.25",
   "datasets<2.20.0",   # will need to update the mteb version (current implementation of MTEB does not pass the "trust_remote_code" argument.
+  "fsspec<=2023.9.2",  # can be removed updated along with the datasets limit when MTEB is updated
 
 ]
 
@@ -40,7 +41,7 @@ docs = [
   "mkdocs==1.5.1",
   "mkdocs-jupyter==0.24.6",
   "mkdocs-material==9.1.21",
-  "mkdocstrings[python]==0.22.0",
+  "mkdocstrings[python-legacy]==0.22.0",
   # for managing tables
   "datawrapper>=0.5.3",
   # for tutorials
@@ -58,6 +59,7 @@ sonar = [
 ] # fairseq2 only works for linux at the moment
 
 fasttext = ["pybind11>=2.11.0", "fasttext-wheel>=0.9.0"]
+jina = ["einops", "numpy<2.0.0"]
 
 
 [project.readme]
@@ -121,7 +123,6 @@ lint.ignore = [
   "RET504",
   "COM812",
 ]
-lint.ignore-init-module-imports = true
 # Allow autofix for all enabled rules (when `--fix`) is provided.
 lint.unfixable = ["ERA"]
 # Exclude a variety of commonly ignored directories.

src/seb/cache/jinaai__jina-embeddings-v3/Angry_Tweets.json

@@ -0,0 +1 @@
+{"task_name":"Angry Tweets","task_description":"A sentiment dataset with 3 classes (positiv, negativ, neutral) for Danish tweets","task_version":"1.1.1","time_of_run":"2024-11-13T21:33:16.042746","scores":{"da":{"accuracy":0.5680993314231136,"f1":0.5594053621774726,"accuracy_stderr":0.024346122584687404,"f1_stderr":0.022681854105695665,"main_score":0.5680993314231136}},"main_score":"accuracy"}

src/seb/cache/jinaai__jina-embeddings-v3/Bornholm_Parallel.json

@@ -0,0 +1 @@
+{"task_name":"Bornholm Parallel","task_description":"Danish Bornholmsk Parallel Corpus. Bornholmsk is a Danish dialect spoken on the island of Bornholm, Denmark. Historically it is a part of east Danish which was also spoken in Scania and Halland, Sweden.","task_version":"1.1.1","time_of_run":"2024-11-13T21:33:55.587815","scores":{"da":{"precision":0.3206724089635854,"recall":0.436,"f1":0.35174285714285713,"accuracy":0.436,"main_score":0.35174285714285713},"da-bornholm":{"precision":0.3206724089635854,"recall":0.436,"f1":0.35174285714285713,"accuracy":0.436,"main_score":0.35174285714285713}},"main_score":"f1"}

src/seb/cache/jinaai__jina-embeddings-v3/DKHate.json

@@ -0,0 +1 @@
+{"task_name":"DKHate","task_description":"Danish Tweets annotated for Hate Speech either being Offensive or not","task_version":"1.1.1","time_of_run":"2024-11-13T22:19:50.826703","scores":{"da":{"accuracy":0.6477203647416412,"f1":0.5366895373219578,"ap":0.1795654122069819,"accuracy_stderr":0.06684788272628261,"f1_stderr":0.04571944298710895,"ap_stderr":0.0282077813963096,"main_score":0.6477203647416412}},"main_score":"accuracy"}

src/seb/cache/jinaai__jina-embeddings-v3/DaLAJ.json

@@ -0,0 +1 @@
+{"task_name":"DaLAJ","task_description":"A Swedish dataset for linguistic acceptability. Available as a part of Superlim.","task_version":"1.1.1","time_of_run":"2024-11-13T21:55:46.102795","scores":{"sv":{"accuracy":0.5009009009009009,"f1":0.4979056245942316,"ap":0.5004671834488945,"accuracy_stderr":0.00392693598517179,"f1_stderr":0.0054452878888052595,"ap_stderr":0.001969818405074579,"main_score":0.5009009009009009}},"main_score":"accuracy"}

src/seb/cache/jinaai__jina-embeddings-v3/Da_Political_Comments.json

@@ -0,0 +1 @@
+{"task_name":"Da Political Comments","task_description":"A dataset of Danish political comments rated for sentiment","task_version":"1.1.1","time_of_run":"2024-11-13T21:34:35.344272","scores":{"da":{"accuracy":0.4206437291897891,"f1":0.38642142217868036,"accuracy_stderr":0.027264394356405246,"f1_stderr":0.017817279657788544,"main_score":0.4206437291897891}},"main_score":"accuracy"}

src/seb/cache/jinaai__jina-embeddings-v3/DanFEVER.json

@@ -0,0 +1 @@
+{"task_name":"DanFEVER","task_description":"A Danish dataset intended for misinformation research. It follows the same format as the English FEVER dataset.","task_version":"1.1.1","time_of_run":"2024-11-13T21:36:36.608562","scores":{"da":{"ndcg_at_1":0.25859,"ndcg_at_3":0.34764,"ndcg_at_5":0.35958,"ndcg_at_10":0.36608,"ndcg_at_100":0.37129,"ndcg_at_1000":0.37169,"map_at_1":0.25851,"map_at_3":0.32666,"map_at_5":0.33333,"map_at_10":0.33608,"map_at_100":0.33722,"map_at_1000":0.33724,"recall_at_1":0.25851,"recall_at_3":0.40797,"recall_at_5":0.43676,"recall_at_10":0.45646,"recall_at_100":0.48015,"recall_at_1000":0.48313,"precision_at_1":0.25859,"precision_at_3":0.13604,"precision_at_5":0.0874,"precision_at_10":0.04568,"precision_at_100":0.00481,"precision_at_1000":0.00048,"mrr_at_1":0.25875,"mrr_at_3":0.32674,"mrr_at_5":0.33343,"mrr_at_10":0.33619,"mrr_at_100":0.3373,"mrr_at_1000":0.33732}},"main_score":"ndcg_at_10"}

src/seb/cache/jinaai__jina-embeddings-v3/LCC.json

@@ -0,0 +1 @@
+{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.1.1","time_of_run":"2024-11-13T21:33:38.301520","scores":{"da":{"accuracy":0.5946666666666667,"f1":0.5872722607515735,"accuracy_stderr":0.03222145592958552,"f1_stderr":0.0278698114421992,"main_score":0.5946666666666667}},"main_score":"accuracy"}

src/seb/cache/jinaai__jina-embeddings-v3/Language_Identification.json

@@ -0,0 +1 @@
+{"task_name":"Language Identification","task_description":"A dataset for Nordic language identification.","task_version":"1.1.1","time_of_run":"2024-11-13T21:50:33.466847","scores":{"da":{"accuracy":0.48129999999999995,"f1":0.4716944543998669,"accuracy_stderr":0.012093570376214148,"f1_stderr":0.013172338407872259,"main_score":0.48129999999999995},"sv":{"accuracy":0.48129999999999995,"f1":0.4716944543998669,"accuracy_stderr":0.012093570376214148,"f1_stderr":0.013172338407872259,"main_score":0.48129999999999995},"nb":{"accuracy":0.48129999999999995,"f1":0.4716944543998669,"accuracy_stderr":0.012093570376214148,"f1_stderr":0.013172338407872259,"main_score":0.48129999999999995},"nn":{"accuracy":0.48129999999999995,"f1":0.4716944543998669,"accuracy_stderr":0.012093570376214148,"f1_stderr":0.013172338407872259,"main_score":0.48129999999999995},"is":{"accuracy":0.48129999999999995,"f1":0.4716944543998669,"accuracy_stderr":0.012093570376214148,"f1_stderr":0.013172338407872259,"main_score":0.48129999999999995},"fo":{"accuracy":0.48129999999999995,"f1":0.4716944543998669,"accuracy_stderr":0.012093570376214148,"f1_stderr":0.013172338407872259,"main_score":0.48129999999999995}},"main_score":"accuracy"}

src/seb/cache/jinaai__jina-embeddings-v3/Massive_Intent.json

@@ -0,0 +1 @@
+{"task_name":"Massive Intent","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.1.1","time_of_run":"2024-11-13T21:43:34.445063","scores":{"da":{"accuracy":0.6379959650302622,"f1":0.6031210839216415,"accuracy_stderr":0.018687738234453372,"f1_stderr":0.01601335989279753,"main_score":0.6379959650302622},"nb":{"accuracy":0.6341627437794217,"f1":0.6000839733610837,"accuracy_stderr":0.016888194867408664,"f1_stderr":0.017814875436374125,"main_score":0.6341627437794217},"sv":{"accuracy":0.6594821788836583,"f1":0.6323874307279661,"accuracy_stderr":0.022018459492548024,"f1_stderr":0.017138250345814364,"main_score":0.6594821788836583}},"main_score":"accuracy"}

src/seb/cache/jinaai__jina-embeddings-v3/Massive_Scenario.json

@@ -0,0 +1 @@
+{"task_name":"Massive Scenario","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.1.1","time_of_run":"2024-11-13T21:46:44.729375","scores":{"da":{"accuracy":0.7350706119704102,"f1":0.7264571146442774,"accuracy_stderr":0.007798404730966382,"f1_stderr":0.009457832203659417,"main_score":0.7350706119704102},"nb":{"accuracy":0.7190988567585743,"f1":0.7119164347268657,"accuracy_stderr":0.01014353370687806,"f1_stderr":0.010327527565859801,"main_score":0.7190988567585743},"sv":{"accuracy":0.7415265635507734,"f1":0.7321277292640845,"accuracy_stderr":0.009021276077818262,"f1_stderr":0.009842616330101132,"main_score":0.7415265635507734}},"main_score":"accuracy"}

src/seb/cache/jinaai__jina-embeddings-v3/NoReC.json

@@ -0,0 +1 @@
+{"task_name":"NoReC","task_description":"A Norwegian dataset for sentiment classification on review","task_version":"1.1.1","time_of_run":"2024-11-13T21:51:14.423683","scores":{"nb":{"accuracy":0.5984375,"f1":0.5772198167649724,"accuracy_stderr":0.018554944488351894,"f1_stderr":0.01755204684778349,"main_score":0.5984375}},"main_score":"accuracy"}

src/seb/cache/jinaai__jina-embeddings-v3/NorQuad.json

@@ -0,0 +1 @@
+{"task_name":"NorQuad","task_description":"Human-created question for Norwegian wikipedia passages.","task_version":"0.0.1","time_of_run":"2024-11-13T21:54:45.566159","scores":{"nb":{"ndcg_at_1":0.24805,"ndcg_at_3":0.20567,"ndcg_at_5":0.22425,"ndcg_at_10":0.24207,"ndcg_at_100":0.28795,"ndcg_at_1000":0.32916,"map_at_1":0.12402,"map_at_3":0.16471,"map_at_5":0.17546,"map_at_10":0.18267,"map_at_100":0.19102,"map_at_1000":0.19243,"recall_at_1":0.12402,"recall_at_3":0.19824,"recall_at_5":0.23486,"recall_at_10":0.27979,"recall_at_100":0.46436,"recall_at_1000":0.74707,"precision_at_1":0.24805,"precision_at_3":0.13216,"precision_at_5":0.09395,"precision_at_10":0.05596,"precision_at_100":0.00929,"precision_at_1000":0.00149,"mrr_at_1":0.24805,"mrr_at_3":0.29915,"mrr_at_5":0.31146,"mrr_at_10":0.32102,"mrr_at_100":0.33193,"mrr_at_1000":0.33295}},"main_score":"ndcg_at_10"}

src/seb/cache/jinaai__jina-embeddings-v3/Norwegian_courts.json

@@ -0,0 +1 @@
+{"task_name":"Norwegian courts","task_description":"Nynorsk and Bokmål parallel corpus from Norwegian courts. Norway has two standardised written languages. Bokmål is a variant closer to Danish, while Nynorsk was created to resemble regional dialects of Norwegian.","task_version":"1.1.1","time_of_run":"2024-11-13T21:52:11.201687","scores":{"nb":{"precision":0.9049707602339182,"recall":0.9298245614035088,"f1":0.9130116959064327,"accuracy":0.9298245614035088,"main_score":0.9130116959064327},"nn":{"precision":0.9049707602339182,"recall":0.9298245614035088,"f1":0.9130116959064327,"accuracy":0.9298245614035088,"main_score":0.9130116959064327}},"main_score":"f1"}

src/seb/cache/jinaai__jina-embeddings-v3/Norwegian_parliament.json

@@ -0,0 +1 @@
+{"task_name":"Norwegian parliament","task_description":"Norwegian parliament speeches annotated with the party of the speaker (`Sosialistisk Venstreparti` vs `Fremskrittspartiet`)","task_version":"1.1.1","time_of_run":"2024-11-13T21:52:00.593689","scores":{"nb":{"accuracy":0.6006666666666666,"f1":0.598941034068567,"ap":0.5607351566727996,"accuracy_stderr":0.018195695461656135,"f1_stderr":0.019486204000061156,"ap_stderr":0.013099108893732402,"main_score":0.6006666666666666}},"main_score":"accuracy"}

src/seb/cache/jinaai__jina-embeddings-v3/SNL_Clustering.json

@@ -0,0 +1 @@
+{"task_name":"SNL Clustering","task_description":"Webscrabed articles from the Norwegian lexicon 'Det Store Norske Leksikon'. Uses articles categories as clusters.","task_version":"0.0.1","time_of_run":"2024-11-13T21:54:10.329920","scores":{"nb":{"v_measure":0.5957282278588,"v_measure_std":0.014688046062978624}},"main_score":"v_measure"}

src/seb/cache/jinaai__jina-embeddings-v3/SNL_Retrieval.json

@@ -0,0 +1 @@
+{"task_name":"SNL Retrieval","task_description":"Webscrabed articles and ingresses from the Norwegian lexicon 'Det Store Norske Leksikon'.","task_version":"0.0.1","time_of_run":"2024-11-13T21:32:05.916886","scores":{"nb":{"ndcg_at_1":0.96154,"ndcg_at_3":0.97191,"ndcg_at_5":0.9741,"ndcg_at_10":0.97508,"ndcg_at_100":0.97711,"ndcg_at_1000":0.97746,"map_at_1":0.96154,"map_at_3":0.96962,"map_at_5":0.97081,"map_at_10":0.97121,"map_at_100":0.97158,"map_at_1000":0.97159,"recall_at_1":0.96154,"recall_at_3":0.97846,"recall_at_5":0.98385,"recall_at_10":0.98692,"recall_at_100":0.99692,"recall_at_1000":1.0,"precision_at_1":0.96154,"precision_at_3":0.32615,"precision_at_5":0.19677,"precision_at_10":0.09869,"precision_at_100":0.00997,"precision_at_1000":0.001,"mrr_at_1":0.96154,"mrr_at_3":0.96962,"mrr_at_5":0.97081,"mrr_at_10":0.97121,"mrr_at_100":0.97158,"mrr_at_1000":0.97159}},"main_score":"ndcg_at_10"}

src/seb/cache/jinaai__jina-embeddings-v3/ScaLA.json

@@ -0,0 +1 @@
+{"task_name":"ScaLA","task_description":"A linguistic acceptability task for Danish, Norwegian Bokmål Norwegian Nynorsk and Swedish.","task_version":"1.1.1","time_of_run":"2024-11-13T21:49:22.408633","scores":{"da":{"accuracy":0.50224609375,"f1":0.4992941088752125,"ap":0.5011438581747162,"accuracy_stderr":0.003858348879909846,"f1_stderr":0.0052233518938709315,"ap_stderr":0.0019490072295206748,"main_score":0.50224609375},"nb":{"accuracy":0.504248046875,"f1":0.5019808251239641,"ap":0.50216298178004,"accuracy_stderr":0.004754415367279045,"f1_stderr":0.004887920058285869,"ap_stderr":0.0024141142550112176,"main_score":0.504248046875},"sv":{"accuracy":0.50302734375,"f1":0.49853968956171374,"ap":0.5015564574921725,"accuracy_stderr":0.006050748466663932,"f1_stderr":0.008325097133306763,"ap_stderr":0.0030894211378567557,"main_score":0.50302734375},"nn":{"accuracy":0.5015625,"f1":0.4991181856004777,"ap":0.5008044312421281,"accuracy_stderr":0.004366228301974249,"f1_stderr":0.005773270356524286,"ap_stderr":0.0021724434704132266,"main_score":0.5015625}},"main_score":"accuracy"}

src/seb/cache/jinaai__jina-embeddings-v3/SweFAQ.json

@@ -0,0 +1 @@
+{"task_name":"SweFAQ","task_description":"A Swedish QA dataset derived from FAQ","task_version":"0.0.1","time_of_run":"2024-11-13T21:56:01.501825","scores":{"sv":{"ndcg_at_1":0.66472,"ndcg_at_3":0.77446,"ndcg_at_5":0.79954,"ndcg_at_10":0.8148,"ndcg_at_100":0.82488,"ndcg_at_1000":0.82514,"map_at_1":0.66472,"map_at_3":0.74886,"map_at_5":0.7629,"map_at_10":0.76927,"map_at_100":0.77174,"map_at_1000":0.77175,"recall_at_1":0.66472,"recall_at_3":0.84795,"recall_at_5":0.90838,"recall_at_10":0.95517,"recall_at_100":0.99805,"recall_at_1000":1.0,"precision_at_1":0.66472,"precision_at_3":0.28265,"precision_at_5":0.18168,"precision_at_10":0.09552,"precision_at_100":0.00998,"precision_at_1000":0.001,"mrr_at_1":0.66472,"mrr_at_3":0.74886,"mrr_at_5":0.7629,"mrr_at_10":0.76927,"mrr_at_100":0.77174,"mrr_at_1000":0.77175}},"main_score":"ndcg_at_10"}

src/seb/cache/jinaai__jina-embeddings-v3/SweReC.json

@@ -0,0 +1 @@
+{"task_name":"SweReC","task_description":"A Swedish dataset for sentiment classification on review","task_version":"1.1.1","time_of_run":"2024-11-13T21:55:25.524698","scores":{"sv":{"accuracy":0.803857421875,"f1":0.7171887589191882,"accuracy_stderr":0.0206261120120739,"f1_stderr":0.018435082318465818,"main_score":0.803857421875}},"main_score":"accuracy"}

src/seb/cache/jinaai__jina-embeddings-v3/SwednClustering.json

@@ -0,0 +1 @@
+{"task_name":"SwednClustering","task_description":"The SWE-DN corpus is based on 1,963,576 news articles from the Swedish newspaper Dagens Nyheter (DN) during the years 2000--2020. The articles are filtered to resemble the CNN/DailyMail dataset both regarding textual structure. This dataset uses the category labels as clusters.","task_version":"0.0.1","time_of_run":"2024-11-13T22:17:42.551717","scores":{"sv":{"v_measure":0.05041607670215932,"v_measure_std":0.046431754822418195}},"main_score":"v_measure"}

src/seb/cache/jinaai__jina-embeddings-v3/SwednRetrieval.json

@@ -0,0 +1 @@
+{"task_name":"SwednRetrieval","task_description":"News Article Summary Semantic Similarity Estimation.","task_version":"0.0.1","time_of_run":"2024-11-13T22:16:40.335296","scores":{"sv":{"ndcg_at_1":0.78906,"ndcg_at_3":0.78071,"ndcg_at_5":0.81017,"ndcg_at_10":0.83069,"ndcg_at_100":0.84939,"ndcg_at_1000":0.85357,"map_at_1":0.39453,"map_at_3":0.74788,"map_at_5":0.77201,"map_at_10":0.78434,"map_at_100":0.7897,"map_at_1000":0.78993,"recall_at_1":0.39453,"recall_at_3":0.79053,"recall_at_5":0.84814,"recall_at_10":0.89893,"recall_at_100":0.97021,"recall_at_1000":0.99707,"precision_at_1":0.78906,"precision_at_3":0.52702,"precision_at_5":0.33926,"precision_at_10":0.17979,"precision_at_100":0.0194,"precision_at_1000":0.00199,"mrr_at_1":0.78906,"mrr_at_3":0.83496,"mrr_at_5":0.84072,"mrr_at_10":0.8444,"mrr_at_100":0.84634,"mrr_at_1000":0.84641}},"main_score":"ndcg_at_10"}

src/seb/cache/jinaai__jina-embeddings-v3/TV2Nord_Retrieval.json

@@ -0,0 +1 @@
+{"task_name":"TV2Nord Retrieval","task_description":"News Article and corresponding summaries extracted from the Danish newspaper TV2 Nord.","task_version":"1.1.1","time_of_run":"2024-11-13T21:37:48.874367","scores":{"da":{"ndcg_at_1":0.93359,"ndcg_at_3":0.95819,"ndcg_at_5":0.96202,"ndcg_at_10":0.96502,"ndcg_at_100":0.96634,"ndcg_at_1000":0.96647,"map_at_1":0.93359,"map_at_3":0.95247,"map_at_5":0.9546,"map_at_10":0.95584,"map_at_100":0.95614,"map_at_1000":0.95615,"recall_at_1":0.93359,"recall_at_3":0.97461,"recall_at_5":0.98389,"recall_at_10":0.99316,"recall_at_100":0.99902,"recall_at_1000":1.0,"precision_at_1":0.93359,"precision_at_3":0.32487,"precision_at_5":0.19678,"precision_at_10":0.09932,"precision_at_100":0.00999,"precision_at_1000":0.001,"mrr_at_1":0.93359,"mrr_at_3":0.95247,"mrr_at_5":0.9546,"mrr_at_10":0.95584,"mrr_at_100":0.95614,"mrr_at_1000":0.95615}},"main_score":"ndcg_at_10"}

src/seb/cache/jinaai__jina-embeddings-v3/Twitterhjerne.json

@@ -0,0 +1 @@
+{"task_name":"Twitterhjerne","task_description":"Danish question asked on Twitter with the Hashtag #Twitterhjerne ('Twitter brain') and their corresponding answer.","task_version":"1.1.1","time_of_run":"2024-11-13T21:37:56.999306","scores":{"da":{"ndcg_at_1":0.80769,"ndcg_at_3":0.71733,"ndcg_at_5":0.72496,"ndcg_at_10":0.77033,"ndcg_at_100":0.82115,"ndcg_at_1000":0.82486,"map_at_1":0.26026,"map_at_3":0.54334,"map_at_5":0.63438,"map_at_10":0.68197,"map_at_100":0.70354,"map_at_1000":0.70399,"recall_at_1":0.26026,"recall_at_3":0.56987,"recall_at_5":0.70342,"recall_at_10":0.80256,"recall_at_100":0.96966,"recall_at_1000":0.98718,"precision_at_1":0.80769,"precision_at_3":0.62821,"precision_at_5":0.47949,"precision_at_10":0.27821,"precision_at_100":0.03295,"precision_at_1000":0.00336,"mrr_at_1":0.80769,"mrr_at_3":0.85897,"mrr_at_5":0.87179,"mrr_at_10":0.87179,"mrr_at_100":0.87385,"mrr_at_1000":0.87385}},"main_score":"ndcg_at_10"}

src/seb/cache/jinaai__jina-embeddings-v3/VG_Clustering.json

@@ -0,0 +1 @@
+{"task_name":"VG Clustering","task_description":"Articles and their classes (e.g. sports) from VG news articles extracted from Norsk Aviskorpus.","task_version":"0.0.1","time_of_run":"2024-11-13T21:53:11.844162","scores":{"nb":{"v_measure":0.25556076014493123,"v_measure_std":0.013569083660680839}},"main_score":"v_measure"}

src/seb/registered_models/normalize_to_ndarray.py

@@ -10,5 +10,5 @@ def normalize_to_ndarray(embeddings: Union[torch.Tensor, np.ndarray, list[np.nda
             return torch.cat(embeddings).detach().cpu().numpy()  # type: ignore
         return np.concatenate(embeddings)
     if isinstance(embeddings, torch.Tensor):
-        return embeddings.detach().cpu().numpy()
+        return embeddings.detach().cpu().float().numpy()
     return embeddings