Added results for jina

src/seb/cache/jinaai__jina-embeddings-v3/Language_Identification.json

@@ -0,0 +1 @@
+{"task_name":"Language Identification","task_description":"A dataset for Nordic language identification.","task_version":"1.1.1","time_of_run":"2024-12-13T16:27:56.414291","scores":{"da":{"accuracy":0.4083333333333333,"f1":0.3925482204639472,"accuracy_stderr":0.008633268983029165,"f1_stderr":0.007916299274783293,"main_score":0.4083333333333333},"sv":{"accuracy":0.4083333333333333,"f1":0.3925482204639472,"accuracy_stderr":0.008633268983029165,"f1_stderr":0.007916299274783293,"main_score":0.4083333333333333},"nb":{"accuracy":0.4083333333333333,"f1":0.3925482204639472,"accuracy_stderr":0.008633268983029165,"f1_stderr":0.007916299274783293,"main_score":0.4083333333333333},"nn":{"accuracy":0.4083333333333333,"f1":0.3925482204639472,"accuracy_stderr":0.008633268983029165,"f1_stderr":0.007916299274783293,"main_score":0.4083333333333333},"is":{"accuracy":0.4083333333333333,"f1":0.3925482204639472,"accuracy_stderr":0.008633268983029165,"f1_stderr":0.007916299274783293,"main_score":0.4083333333333333},"fo":{"accuracy":0.4083333333333333,"f1":0.3925482204639472,"accuracy_stderr":0.008633268983029165,"f1_stderr":0.007916299274783293,"main_score":0.4083333333333333}},"main_score":"accuracy"}

src/seb/cache/jinaai__jina-embeddings-v3/Massive_Intent.json

@@ -0,0 +1 @@
+{"task_name":"Massive Intent","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.1.1","time_of_run":"2024-12-13T12:30:27.747355","scores":{"da":{"accuracy":0.7229993275050437,"f1":0.6777312864344521,"accuracy_stderr":0.021605725550200672,"f1_stderr":0.019667189317759903,"main_score":0.7229993275050437},"nb":{"accuracy":0.7127437794216543,"f1":0.6693877898238115,"accuracy_stderr":0.014615011239315112,"f1_stderr":0.012458761208978947,"main_score":0.7127437794216543},"sv":{"accuracy":0.731102891728312,"f1":0.6918924054876893,"accuracy_stderr":0.021828159697349376,"f1_stderr":0.02278756069148256,"main_score":0.731102891728312}},"main_score":"accuracy"}

src/seb/cache/jinaai__jina-embeddings-v3/Massive_Scenario.json

@@ -0,0 +1 @@
+{"task_name":"Massive Scenario","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.1.1","time_of_run":"2024-12-13T16:04:24.007081","scores":{"da":{"accuracy":0.8303967720242097,"f1":0.8144907891114421,"accuracy_stderr":0.01345679043361314,"f1_stderr":0.012214051749594796,"main_score":0.8303967720242097},"nb":{"accuracy":0.8227303295225286,"f1":0.809055319974268,"accuracy_stderr":0.01164970138337166,"f1_stderr":0.010758393028445808,"main_score":0.8227303295225286},"sv":{"accuracy":0.8377269670477471,"f1":0.8214039898309877,"accuracy_stderr":0.011274273146386207,"f1_stderr":0.010311356519461268,"main_score":0.8377269670477471}},"main_score":"accuracy"}

src/seb/cache/jinaai__jina-embeddings-v3/NoReC.json

@@ -0,0 +1 @@
+{"task_name":"NoReC","task_description":"A Norwegian dataset for sentiment classification on review","task_version":"1.1.1","time_of_run":"2024-12-13T16:29:32.719809","scores":{"nb":{"accuracy":0.61494140625,"f1":0.5977866105449843,"accuracy_stderr":0.030842373184347828,"f1_stderr":0.02791508266619411,"main_score":0.61494140625}},"main_score":"accuracy"}

src/seb/cache/jinaai__jina-embeddings-v3/Norwegian_courts.json

@@ -0,0 +1 @@
+{"task_name":"Norwegian courts","task_description":"Nynorsk and Bokmål parallel corpus from Norwegian courts. Norway has two standardised written languages. Bokmål is a variant closer to Danish, while Nynorsk was created to resemble regional dialects of Norwegian.","task_version":"1.1.1","time_of_run":"2024-12-13T16:36:58.457846","scores":{"nb":{"precision":0.9203216374269007,"recall":0.9385964912280702,"f1":0.9261695906432749,"accuracy":0.9385964912280702,"main_score":0.9261695906432749},"nn":{"precision":0.9203216374269007,"recall":0.9385964912280702,"f1":0.9261695906432749,"accuracy":0.9385964912280702,"main_score":0.9261695906432749}},"main_score":"f1"}

src/seb/cache/jinaai__jina-embeddings-v3/Norwegian_parliament.json

@@ -0,0 +1 @@
+{"task_name":"Norwegian parliament","task_description":"Norwegian parliament speeches annotated with the party of the speaker (`Sosialistisk Venstreparti` vs `Fremskrittspartiet`)","task_version":"1.1.1","time_of_run":"2024-12-14T15:36:28.812275","scores":{"nb":{"accuracy":0.5688333333333333,"f1":0.5652299805603136,"ap":0.5401466909214766,"accuracy_stderr":0.026199554703595005,"f1_stderr":0.02768317419649942,"ap_stderr":0.01711798949729587,"main_score":0.5688333333333333}},"main_score":"accuracy"}

src/seb/cache/jinaai__jina-embeddings-v3/SNL_Clustering.json

@@ -0,0 +1 @@
+{"task_name":"SNL Clustering","task_description":"Webscrabed articles from the Norwegian lexicon 'Det Store Norske Leksikon'. Uses articles categories as clusters.","task_version":"0.0.1","time_of_run":"2024-12-14T19:34:35.710300","scores":{"nb":{"v_measure":0.6868723955917873,"v_measure_std":0.009034855336828941}},"main_score":"v_measure"}

src/seb/cache/jinaai__jina-embeddings-v3/ScaLA.json

@@ -0,0 +1 @@
+{"task_name":"ScaLA","task_description":"A linguistic acceptability task for Danish, Norwegian Bokmål Norwegian Nynorsk and Swedish.","task_version":"1.1.1","time_of_run":"2024-12-13T16:21:01.748529","scores":{"da":{"accuracy":0.503857421875,"f1":0.5010150063382104,"ap":0.5019662110991986,"accuracy_stderr":0.005174168749404531,"f1_stderr":0.004400393254493083,"ap_stderr":0.0026253967285794415,"main_score":0.503857421875},"nb":{"accuracy":0.500830078125,"f1":0.49806400988951943,"ap":0.5004272465429791,"accuracy_stderr":0.0032758541245545914,"f1_stderr":0.004599162738598509,"ap_stderr":0.001644370497139994,"main_score":0.500830078125},"sv":{"accuracy":0.500341796875,"f1":0.496277876839753,"ap":0.5002269842196376,"accuracy_stderr":0.007229366274083332,"f1_stderr":0.009507866178624286,"ap_stderr":0.003675449940166358,"main_score":0.500341796875},"nn":{"accuracy":0.5017578125,"f1":0.5005416037277838,"ap":0.5009383843683527,"accuracy_stderr":0.007580792459013808,"f1_stderr":0.007002385836975437,"ap_stderr":0.0037841908762277186,"main_score":0.5017578125}},"main_score":"accuracy"}

src/seb/cache/jinaai__jina-embeddings-v3/VG_Clustering.json

@@ -0,0 +1 @@
+{"task_name":"VG Clustering","task_description":"Articles and their classes (e.g. sports) from VG news articles extracted from Norsk Aviskorpus.","task_version":"0.0.1","time_of_run":"2024-12-14T10:32:20.142395","scores":{"nb":{"v_measure":0.3335847591217076,"v_measure_std":0.015008707429290039}},"main_score":"v_measure"}

src/seb/registered_models/jina_models.py

@@ -34,7 +34,7 @@ def encode(  # type: ignore
             if task.task_type in ["Classification"]:
                 task_prompt = "classification"
             if task.task_type in ["Clustering"]:
-                task_prompt = "seperation"
+                task_prompt = "separation"
             if task.task_type in ["Retrieval"] and encode_type == "query":
                 task_prompt = "retrieval.query"
             if task.task_type in ["Retrieval"] and encode_type == "passage":
@@ -83,7 +83,9 @@ def create_jina_embeddings_v3() -> SebModel:
         release_date=date(2024, 8, 5),
     )
     return SebModel(
-        encoder=LazyLoadEncoder(partial(wrap_jina_sentence_transformer, model_name=hf_name, trust_remote_code=True)),  # type: ignore
+        encoder=LazyLoadEncoder(
+            partial(wrap_jina_sentence_transformer, model_name=hf_name, trust_remote_code=True)
+        ),
         meta=meta,
     )