diff --git a/.github/workflows/check_benchmark_is_up_to_date.yml b/.github/workflows/check_benchmark_is_up_to_date.yml new file mode 100644 index 00000000..f91e6f40 --- /dev/null +++ b/.github/workflows/check_benchmark_is_up_to_date.yml @@ -0,0 +1,28 @@ +name: Benchmark is up to date +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + bench-is-up-to-date: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: Install dependencies + shell: bash + run: | + make install + + - name: Check benchmark is up to date + shell: bash + run: | + make check-benchmark-is-up-to-date diff --git a/makefile b/makefile index bf0527d4..fa8dc733 100644 --- a/makefile +++ b/makefile @@ -22,9 +22,14 @@ pr: make test @echo "Ready to make a PR" +update-table-in-docs: + @echo "--- 🔄 Updating table in docs ---" + python src/scripts/create_desc_stats.py + build-docs: @echo "--- 📚 Building docs ---" @echo "Builds the docs and puts them in the 'site' folder" + @echo "You might need to also update the table with the desc. stats you can do this by running 'make update-table-in-docs'" mkdocs build view-docs: @@ -37,15 +42,16 @@ update-from-template: cruft update --skip-apply-ask update-benchmark: - datawrapper_api_key=$(cat datawrapper_api_key.txt) - python docs/run_benchmark.py --data-wrapper-api-token $datawrapper_api_key - -update-benchmark-on-ucloud: # set environment variables hf_api_key=$(cat hf_api_key.txt) export HF_TOKEN=hf_api_key - export SEB_CACHE_DIR=./seb_cache # run benchmark datawrapper_api_key=$(cat datawrapper_api_key.txt) - python docs/run_benchmark.py --data-wrapper-api-token $datawrapper_api_key \ No newline at end of file + python docs/run_benchmark.py --data-wrapper-api-token $datawrapper_api_key + + +check-benchmark-is-up-to-date: + @echo "--- 🔄 Checking benchmark is up to date ---" + + python src/scripts/check_benchmark_is_up_to_date.py diff --git a/pyproject.toml b/pyproject.toml index 78716e57..e32c7627 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,9 +22,9 @@ requires-python = ">=3.9" dependencies = [ "tabulate>=0.9.0", "mteb[beir]==1.1.1", - "typer>=0.7.0", "pydantic>=2.1.0", "catalogue>=2.0.8", + ] [project.license] diff --git a/src/scripts/check_benchmark_is_up_to_date.py b/src/scripts/check_benchmark_is_up_to_date.py new file mode 100644 index 00000000..73dd5f4e --- /dev/null +++ b/src/scripts/check_benchmark_is_up_to_date.py @@ -0,0 +1,9 @@ +from seb import run_benchmark + + +def main(): + run_benchmark(use_cache=True, run_models=False, raise_errors=True) + + +if __name__ == "__main__": + main() diff --git a/src/seb/benchmark.py b/src/seb/benchmark.py index 586204fb..e80b4a68 100644 --- a/src/seb/benchmark.py +++ b/src/seb/benchmark.py @@ -14,11 +14,13 @@ logger = logging.getLogger(__name__) -def get_cache_path(task: Task, model: SebModel) -> Path: +def get_cache_path( + task: Task, model: SebModel, cache_dir: Optional[Path] = None +) -> Path: """ Get the cache path for a task and model. """ - cache_path = get_cache_dir() + cache_path = cache_dir if cache_dir is not None else get_cache_dir() mdl_path_name = model.meta.get_path_name() task_path_name = name_to_path(task.name) + ".json" task_cache_path = cache_path / mdl_path_name / task_path_name @@ -29,15 +31,28 @@ def run_task( task: Task, model: SebModel, use_cache: bool, + run_model: bool, raise_errors: bool, + cache_dir: Optional[Path] = None, ) -> Union[TaskResult, TaskError]: """ Tests a model on a task """ + if run_model is False and use_cache is False: + raise ValueError("run_model and use_cache cannot both be False") + if not raise_errors and run_model is False: + raise ValueError("raise_errors cannot be False when run_model is False") if not raise_errors: try: - return run_task(task, model, use_cache, raise_errors=True) + return run_task( + task=task, + model=model, + use_cache=use_cache, + run_model=run_model, + raise_errors=True, + cache_dir=cache_dir, + ) except Exception as e: logger.error(f"Error when running {task.name} on {model.meta.name}: {e}") return TaskError( @@ -46,13 +61,19 @@ def run_task( time_of_run=datetime.now(), ) - cache_path = get_cache_path(task, model) + cache_path = get_cache_path(task, model, cache_dir) if cache_path.exists() and use_cache: logger.info(f"Loading cached result for {model.meta.name} on {task.name}") task_result = TaskResult.from_disk(cache_path) return task_result cache_path.parent.mkdir(parents=True, exist_ok=True) + + if not run_model: + raise ValueError( + f"Cache for {model.meta.name} on {task.name} does not exist. " + "Set run_model=True to run the model.", + ) with WarningIgnoreContextManager(): task_result = task.evaluate(model) task_result.to_disk(cache_path) @@ -104,7 +125,9 @@ def evaluate_model( self, model: SebModel, use_cache: bool = True, + run_model: bool = True, raise_errors: bool = True, + cache_dir: Optional[Path] = None, ) -> BenchmarkResults: """ Evaluate a model on the benchmark. @@ -112,7 +135,9 @@ def evaluate_model( Args: model: The model to evaluate. use_cache: Whether to use the cache. + run_model: Whether to run the model if the cache is not present. raise_errors: Whether to raise errors. + cache_dir: The cache directory to use. If None, the default cache directory is used. Returns: The results of the benchmark. @@ -121,7 +146,14 @@ def evaluate_model( task_results = [] pbar = tqdm(tasks, position=1, desc=f"Running {model.meta.name}", leave=False) for task in pbar: - task_result = run_task(task, model, use_cache, raise_errors) + task_result = run_task( + task, + model, + use_cache=use_cache, + run_model=run_model, + raise_errors=raise_errors, + cache_dir=cache_dir, + ) task_results.append(task_result) return BenchmarkResults(meta=model.meta, task_results=task_results) @@ -130,7 +162,9 @@ def evaluate_models( self, models: list[SebModel], use_cache: bool = True, + run_model: bool = True, raise_errors: bool = True, + cache_dir: Optional[Path] = None, ) -> list[BenchmarkResults]: """ Evaluate a list of models on the benchmark. @@ -138,7 +172,9 @@ def evaluate_models( Args: models: The models to evaluate. use_cache: Whether to use the cache. + run_model: Whether to run the model if the cache is not present. raise_errors: Whether to raise errors. + cache_dir: The cache directory to use. If None, the default cache directory is used. Returns: The results of the benchmark, once for each model. @@ -151,7 +187,9 @@ def evaluate_models( self.evaluate_model( model, use_cache=use_cache, + run_model=run_model, raise_errors=raise_errors, + cache_dir=cache_dir, ), ) return results diff --git a/src/seb/cache/KBLab__sentence-bert-swedish-cased/Angry_Tweets.json b/src/seb/cache/KBLab__sentence-bert-swedish-cased/Angry_Tweets.json new file mode 100644 index 00000000..7b854ea5 --- /dev/null +++ b/src/seb/cache/KBLab__sentence-bert-swedish-cased/Angry_Tweets.json @@ -0,0 +1 @@ +{"task_name":"Angry Tweets","task_description":"A sentiment dataset with 3 classes (positiv, negativ, neutral) for Danish tweets","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:15:07.906466","scores":{"da":{"accuracy":0.44460362941738296,"f1":0.4380942035064149,"accuracy_stderr":0.02809792891547516,"f1_stderr":0.02869393997039908,"main_score":0.44460362941738296}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KBLab__sentence-bert-swedish-cased/Bornholm_Parallel.json b/src/seb/cache/KBLab__sentence-bert-swedish-cased/Bornholm_Parallel.json new file mode 100644 index 00000000..cc836ed7 --- /dev/null +++ b/src/seb/cache/KBLab__sentence-bert-swedish-cased/Bornholm_Parallel.json @@ -0,0 +1 @@ +{"task_name":"Bornholm Parallel","task_description":"Danish Bornholmsk Parallel Corpus. Bornholmsk is a Danish dialect spoken on the island of Bornholm, Denmark. Historically it is a part of east Danish which was also spoken in Scania and Halland, Sweden.","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:15:39.189398","scores":{"da":{"precision":0.12769630591630593,"recall":0.188,"f1":0.1408286249981902,"accuracy":0.188,"main_score":0.1408286249981902},"da-bornholm":{"precision":0.12769630591630593,"recall":0.188,"f1":0.1408286249981902,"accuracy":0.188,"main_score":0.1408286249981902}},"main_score":"f1"} \ No newline at end of file diff --git a/src/seb/cache/KBLab__sentence-bert-swedish-cased/DKHate.json b/src/seb/cache/KBLab__sentence-bert-swedish-cased/DKHate.json new file mode 100644 index 00000000..6dbf7a7b --- /dev/null +++ b/src/seb/cache/KBLab__sentence-bert-swedish-cased/DKHate.json @@ -0,0 +1 @@ +{"task_name":"DKHate","task_description":"Danish Tweets annotated for Hate Speech either being Offensive or not","task_version":"1.0.3.dev0","time_of_run":"2023-07-30T13:55:26.414673","scores":{"da":{"accuracy":0.5936170212765958,"f1":0.48062030395159827,"ap":0.8897617189814045,"accuracy_stderr":0.09420580255043687,"f1_stderr":0.05292797998084632,"ap_stderr":0.006783234279065605,"main_score":0.5936170212765958}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KBLab__sentence-bert-swedish-cased/DaLAJ.json b/src/seb/cache/KBLab__sentence-bert-swedish-cased/DaLAJ.json new file mode 100644 index 00000000..4202be2d --- /dev/null +++ b/src/seb/cache/KBLab__sentence-bert-swedish-cased/DaLAJ.json @@ -0,0 +1 @@ +{"task_name":"DaLAJ","task_description":"A Swedish dataset for linguistic accebtablity. Available as a part of Superlim.","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:32:34.602207","scores":{"sv":{"accuracy":0.5011261261261262,"f1":0.4981211967409228,"ap":0.5005828726112415,"accuracy_stderr":0.004419238033862288,"f1_stderr":0.0035063858678943075,"ap_stderr":0.0022274185349420317,"main_score":0.5011261261261262}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KBLab__sentence-bert-swedish-cased/Da_Political_Comments.json b/src/seb/cache/KBLab__sentence-bert-swedish-cased/Da_Political_Comments.json new file mode 100644 index 00000000..35c873de --- /dev/null +++ b/src/seb/cache/KBLab__sentence-bert-swedish-cased/Da_Political_Comments.json @@ -0,0 +1 @@ +{"task_name":"Da Political Comments","task_description":"A dataset of Danish political comments rated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:17:00.221073","scores":{"da":{"accuracy":0.28546059933407325,"f1":0.2577317269849485,"accuracy_stderr":0.023480148626138817,"f1_stderr":0.01681654012226201,"main_score":0.28546059933407325}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KBLab__sentence-bert-swedish-cased/LCC.json b/src/seb/cache/KBLab__sentence-bert-swedish-cased/LCC.json new file mode 100644 index 00000000..814331aa --- /dev/null +++ b/src/seb/cache/KBLab__sentence-bert-swedish-cased/LCC.json @@ -0,0 +1 @@ +{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:15:26.284602","scores":{"da":{"accuracy":0.4720000000000001,"f1":0.4564433994886203,"accuracy_stderr":0.03512517299285198,"f1_stderr":0.029213561687871915,"main_score":0.4720000000000001}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KBLab__sentence-bert-swedish-cased/Language_Identification.json b/src/seb/cache/KBLab__sentence-bert-swedish-cased/Language_Identification.json new file mode 100644 index 00000000..8c7ccb71 --- /dev/null +++ b/src/seb/cache/KBLab__sentence-bert-swedish-cased/Language_Identification.json @@ -0,0 +1 @@ +{"task_name":"Language Identification","task_description":"A dataset for Nordic language identification.","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:26:29.341469","scores":{"da":{"accuracy":0.5144666666666666,"f1":0.5082242214165055,"accuracy_stderr":0.010790118936632086,"f1_stderr":0.013579053769348828,"main_score":0.5144666666666666},"sv":{"accuracy":0.5144666666666666,"f1":0.5082242214165055,"accuracy_stderr":0.010790118936632086,"f1_stderr":0.013579053769348828,"main_score":0.5144666666666666},"nb":{"accuracy":0.5144666666666666,"f1":0.5082242214165055,"accuracy_stderr":0.010790118936632086,"f1_stderr":0.013579053769348828,"main_score":0.5144666666666666},"nn":{"accuracy":0.5144666666666666,"f1":0.5082242214165055,"accuracy_stderr":0.010790118936632086,"f1_stderr":0.013579053769348828,"main_score":0.5144666666666666},"is":{"accuracy":0.5144666666666666,"f1":0.5082242214165055,"accuracy_stderr":0.010790118936632086,"f1_stderr":0.013579053769348828,"main_score":0.5144666666666666},"fo":{"accuracy":0.5144666666666666,"f1":0.5082242214165055,"accuracy_stderr":0.010790118936632086,"f1_stderr":0.013579053769348828,"main_score":0.5144666666666666}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KBLab__sentence-bert-swedish-cased/Massive_Intent.json b/src/seb/cache/KBLab__sentence-bert-swedish-cased/Massive_Intent.json new file mode 100644 index 00000000..e5545cea --- /dev/null +++ b/src/seb/cache/KBLab__sentence-bert-swedish-cased/Massive_Intent.json @@ -0,0 +1 @@ +{"task_name":"Massive Intent","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:20:45.850492","scores":{"da":{"accuracy":0.42844653665097515,"f1":0.42183077398989566,"accuracy_stderr":0.009531652284439484,"f1_stderr":0.011720697125388892,"main_score":0.42844653665097515},"nb":{"accuracy":0.42737054472091457,"f1":0.4065771240847707,"accuracy_stderr":0.012555140439491758,"f1_stderr":0.011758439799234426,"main_score":0.42737054472091457},"sv":{"accuracy":0.6910894418291863,"f1":0.6651160698998817,"accuracy_stderr":0.013941949333232515,"f1_stderr":0.012610378122239658,"main_score":0.6910894418291863}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KBLab__sentence-bert-swedish-cased/Massive_Scenario.json b/src/seb/cache/KBLab__sentence-bert-swedish-cased/Massive_Scenario.json new file mode 100644 index 00000000..8b241aff --- /dev/null +++ b/src/seb/cache/KBLab__sentence-bert-swedish-cased/Massive_Scenario.json @@ -0,0 +1 @@ +{"task_name":"Massive Scenario","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:22:36.232225","scores":{"da":{"accuracy":0.4964021519838601,"f1":0.48223918173555036,"accuracy_stderr":0.013060130811491842,"f1_stderr":0.010102691330202264,"main_score":0.4964021519838601},"nb":{"accuracy":0.4948890383322125,"f1":0.4762622480394999,"accuracy_stderr":0.012910151752994623,"f1_stderr":0.013032592130554148,"main_score":0.4948890383322125},"sv":{"accuracy":0.7595830531271015,"f1":0.7530102836662811,"accuracy_stderr":0.01613987895090787,"f1_stderr":0.013497075103297649,"main_score":0.7595830531271015}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KBLab__sentence-bert-swedish-cased/NoReC.json b/src/seb/cache/KBLab__sentence-bert-swedish-cased/NoReC.json new file mode 100644 index 00000000..83472aab --- /dev/null +++ b/src/seb/cache/KBLab__sentence-bert-swedish-cased/NoReC.json @@ -0,0 +1 @@ +{"task_name":"NoReC","task_description":"A Norwegian dataset for sentiment classification on review","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:27:09.39258","scores":{"nb":{"accuracy":0.43525390625,"f1":0.4148123251467906,"accuracy_stderr":0.01785160523791383,"f1_stderr":0.016923236820285816,"main_score":0.43525390625}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KBLab__sentence-bert-swedish-cased/Norwegian_parliament.json b/src/seb/cache/KBLab__sentence-bert-swedish-cased/Norwegian_parliament.json new file mode 100644 index 00000000..d52e178c --- /dev/null +++ b/src/seb/cache/KBLab__sentence-bert-swedish-cased/Norwegian_parliament.json @@ -0,0 +1 @@ +{"task_name":"Norwegian parliament","task_description":"Norwegian parliament speeches annotated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:30:21.169416","scores":{"nb":{"accuracy":0.5574166666666668,"f1":0.5530107857827613,"ap":0.5325918715216138,"accuracy_stderr":0.020243140127515356,"f1_stderr":0.02220607579055956,"ap_stderr":0.012785093237452189,"main_score":0.5574166666666668}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KBLab__sentence-bert-swedish-cased/ScaLA.json b/src/seb/cache/KBLab__sentence-bert-swedish-cased/ScaLA.json new file mode 100644 index 00000000..f2174446 --- /dev/null +++ b/src/seb/cache/KBLab__sentence-bert-swedish-cased/ScaLA.json @@ -0,0 +1 @@ +{"task_name":"ScaLA","task_description":"A linguistic acceptability task for Danish, Norwegian Bokmål Norwegian Nynorsk and Swedish.","task_version":"1.1.1.dev0","time_of_run":"2023-09-13T15:34:22.754407","scores":{"da":{"accuracy":0.501220703125,"f1":0.4994319866681282,"ap":0.5006382834998069,"accuracy_stderr":0.005056717572560542,"f1_stderr":0.004478704690331606,"ap_stderr":0.0025455551645878903,"main_score":0.501220703125},"nb":{"accuracy":0.50341796875,"f1":0.49599382125168273,"ap":0.5017502977252766,"accuracy_stderr":0.004784159653873394,"f1_stderr":0.008067901398966753,"ap_stderr":0.002431992168215134,"main_score":0.50341796875},"sv":{"accuracy":0.4984375,"f1":0.4943650186171043,"ap":0.4992456314269318,"accuracy_stderr":0.005148068497303595,"f1_stderr":0.006846332722567433,"ap_stderr":0.002557397210459417,"main_score":0.4984375},"nn":{"accuracy":0.50009765625,"f1":0.4977961797199092,"ap":0.5000778352453731,"accuracy_stderr":0.005493109808759856,"f1_stderr":0.00640345588489854,"ap_stderr":0.0027285475508133294,"main_score":0.50009765625}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KBLab__sentence-bert-swedish-cased/SweFAQ.json b/src/seb/cache/KBLab__sentence-bert-swedish-cased/SweFAQ.json new file mode 100644 index 00000000..5115c74a --- /dev/null +++ b/src/seb/cache/KBLab__sentence-bert-swedish-cased/SweFAQ.json @@ -0,0 +1 @@ +{"task_name":"SweFAQ","task_description":"A Swedish QA dataset derived from FAQ","task_version":"0.0.1","time_of_run":"2023-09-04T19:24:19.566584","scores":{"sv":{"ndcg_at_1":0.56725,"ndcg_at_3":0.68335,"ndcg_at_5":0.71238,"ndcg_at_10":0.73295,"ndcg_at_100":0.75403,"ndcg_at_1000":0.75533,"map_at_1":0.56725,"map_at_3":0.65595,"map_at_5":0.67212,"map_at_10":0.68084,"map_at_100":0.68553,"map_at_1000":0.68559,"recall_at_1":0.56725,"recall_at_3":0.76218,"recall_at_5":0.83236,"recall_at_10":0.89474,"recall_at_100":0.99025,"recall_at_1000":1.0,"precision_at_1":0.56725,"precision_at_3":0.25406,"precision_at_5":0.16647,"precision_at_10":0.08947,"precision_at_100":0.0099,"precision_at_1000":0.001,"mrr_at_1":0.56725,"mrr_at_3":0.65595,"mrr_at_5":0.67212,"mrr_at_10":0.68084,"mrr_at_100":0.68553,"mrr_at_1000":0.68559}},"main_score":"ndcg_at_10"} \ No newline at end of file diff --git a/src/seb/cache/KBLab__sentence-bert-swedish-cased/SweReC.json b/src/seb/cache/KBLab__sentence-bert-swedish-cased/SweReC.json new file mode 100644 index 00000000..f6501f13 --- /dev/null +++ b/src/seb/cache/KBLab__sentence-bert-swedish-cased/SweReC.json @@ -0,0 +1 @@ +{"task_name":"SweReC","task_description":"A Swedish dataset for sentiment classification on review","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:32:09.364853","scores":{"sv":{"accuracy":0.7140625,"f1":0.6458174750991554,"accuracy_stderr":0.024781608357670588,"f1_stderr":0.0193427061836479,"main_score":0.7140625}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KB__bert-base-swedish-cased/Angry_Tweets.json b/src/seb/cache/KB__bert-base-swedish-cased/Angry_Tweets.json new file mode 100644 index 00000000..4ee4c23f --- /dev/null +++ b/src/seb/cache/KB__bert-base-swedish-cased/Angry_Tweets.json @@ -0,0 +1 @@ +{"task_name":"Angry Tweets","task_description":"A sentiment dataset with 3 classes (positiv, negativ, neutral) for Danish tweets","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T23:30:08.745371","scores":{"da":{"accuracy":0.4458452722063037,"f1":0.4353629739721945,"accuracy_stderr":0.028880980134025704,"f1_stderr":0.029111483185951324,"main_score":0.4458452722063037}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KB__bert-base-swedish-cased/Bornholm_Parallel.json b/src/seb/cache/KB__bert-base-swedish-cased/Bornholm_Parallel.json new file mode 100644 index 00000000..9c334005 --- /dev/null +++ b/src/seb/cache/KB__bert-base-swedish-cased/Bornholm_Parallel.json @@ -0,0 +1 @@ +{"task_name":"Bornholm Parallel","task_description":"Danish Bornholmsk Parallel Corpus. Bornholmsk is a Danish dialect spoken on the island of Bornholm, Denmark. Historically it is a part of east Danish which was also spoken in Scania and Halland, Sweden.","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T23:30:37.205614","scores":{"da":{"precision":0.0603645764405845,"recall":0.092,"f1":0.06602699130934425,"accuracy":0.092,"main_score":0.06602699130934425},"da-bornholm":{"precision":0.0603645764405845,"recall":0.092,"f1":0.06602699130934425,"accuracy":0.092,"main_score":0.06602699130934425}},"main_score":"f1"} \ No newline at end of file diff --git a/src/seb/cache/KB__bert-base-swedish-cased/DKHate.json b/src/seb/cache/KB__bert-base-swedish-cased/DKHate.json new file mode 100644 index 00000000..c68e14e1 --- /dev/null +++ b/src/seb/cache/KB__bert-base-swedish-cased/DKHate.json @@ -0,0 +1 @@ +{"task_name":"DKHate","task_description":"Danish Tweets annotated for Hate Speech either being Offensive or not","task_version":"1.0.3.dev0","time_of_run":"2023-07-30T14:01:46.078734","scores":{"da":{"accuracy":0.5553191489361702,"f1":0.45947587346589847,"ap":0.8870741597469347,"accuracy_stderr":0.08659200521947144,"f1_stderr":0.058762937812424136,"ap_stderr":0.011945599407142385,"main_score":0.5553191489361702}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KB__bert-base-swedish-cased/DaLAJ.json b/src/seb/cache/KB__bert-base-swedish-cased/DaLAJ.json new file mode 100644 index 00000000..f6c787a0 --- /dev/null +++ b/src/seb/cache/KB__bert-base-swedish-cased/DaLAJ.json @@ -0,0 +1 @@ +{"task_name":"DaLAJ","task_description":"A Swedish dataset for linguistic accebtablity. Available as a part of Superlim.","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T11:03:11.954025","scores":{"sv":{"accuracy":0.5176801801801801,"f1":0.5152889708732695,"ap":0.5101123206773084,"accuracy_stderr":0.028431283803400806,"f1_stderr":0.029514988893198382,"ap_stderr":0.015255890577832387,"main_score":0.5176801801801801}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KB__bert-base-swedish-cased/Da_Political_Comments.json b/src/seb/cache/KB__bert-base-swedish-cased/Da_Political_Comments.json new file mode 100644 index 00000000..b8dddab3 --- /dev/null +++ b/src/seb/cache/KB__bert-base-swedish-cased/Da_Political_Comments.json @@ -0,0 +1 @@ +{"task_name":"Da Political Comments","task_description":"A dataset of Danish political comments rated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T23:31:58.639984","scores":{"da":{"accuracy":0.28546059933407325,"f1":0.2514372540209549,"accuracy_stderr":0.025401660170899606,"f1_stderr":0.01728671199739282,"main_score":0.28546059933407325}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KB__bert-base-swedish-cased/LCC.json b/src/seb/cache/KB__bert-base-swedish-cased/LCC.json new file mode 100644 index 00000000..16cc9b7c --- /dev/null +++ b/src/seb/cache/KB__bert-base-swedish-cased/LCC.json @@ -0,0 +1 @@ +{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T23:30:25.154348","scores":{"da":{"accuracy":0.41200000000000003,"f1":0.3942888799816736,"accuracy_stderr":0.06469071717711042,"f1_stderr":0.05673115864109014,"main_score":0.41200000000000003}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KB__bert-base-swedish-cased/Language_Identification.json b/src/seb/cache/KB__bert-base-swedish-cased/Language_Identification.json new file mode 100644 index 00000000..53018079 --- /dev/null +++ b/src/seb/cache/KB__bert-base-swedish-cased/Language_Identification.json @@ -0,0 +1 @@ +{"task_name":"Language Identification","task_description":"A dataset for Nordic language identification.","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T23:41:03.343376","scores":{"da":{"accuracy":0.6245,"f1":0.621903954791346,"accuracy_stderr":0.01292306293243035,"f1_stderr":0.012823268129236837,"main_score":0.6245},"sv":{"accuracy":0.6245,"f1":0.621903954791346,"accuracy_stderr":0.01292306293243035,"f1_stderr":0.012823268129236837,"main_score":0.6245},"nb":{"accuracy":0.6245,"f1":0.621903954791346,"accuracy_stderr":0.01292306293243035,"f1_stderr":0.012823268129236837,"main_score":0.6245},"nn":{"accuracy":0.6245,"f1":0.621903954791346,"accuracy_stderr":0.01292306293243035,"f1_stderr":0.012823268129236837,"main_score":0.6245},"is":{"accuracy":0.6245,"f1":0.621903954791346,"accuracy_stderr":0.01292306293243035,"f1_stderr":0.012823268129236837,"main_score":0.6245},"fo":{"accuracy":0.6245,"f1":0.621903954791346,"accuracy_stderr":0.01292306293243035,"f1_stderr":0.012823268129236837,"main_score":0.6245}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KB__bert-base-swedish-cased/Massive_Intent.json b/src/seb/cache/KB__bert-base-swedish-cased/Massive_Intent.json new file mode 100644 index 00000000..2f08076b --- /dev/null +++ b/src/seb/cache/KB__bert-base-swedish-cased/Massive_Intent.json @@ -0,0 +1 @@ +{"task_name":"Massive Intent","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T23:35:38.694124","scores":{"da":{"accuracy":0.37982515131136513,"f1":0.3448174943380284,"accuracy_stderr":0.011440520880727738,"f1_stderr":0.009748846860402905,"main_score":0.37982515131136513},"nb":{"accuracy":0.3574983187626093,"f1":0.3338735327824476,"accuracy_stderr":0.01486943985314982,"f1_stderr":0.011441579699662988,"main_score":0.3574983187626093},"sv":{"accuracy":0.5275386684599865,"f1":0.4917374109552649,"accuracy_stderr":0.00907799693547891,"f1_stderr":0.007623106603827818,"main_score":0.5275386684599865}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KB__bert-base-swedish-cased/Massive_Scenario.json b/src/seb/cache/KB__bert-base-swedish-cased/Massive_Scenario.json new file mode 100644 index 00000000..de32e22b --- /dev/null +++ b/src/seb/cache/KB__bert-base-swedish-cased/Massive_Scenario.json @@ -0,0 +1 @@ +{"task_name":"Massive Scenario","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T23:37:28.577205","scores":{"da":{"accuracy":0.40437121721587077,"f1":0.3723288747252761,"accuracy_stderr":0.0217415653668484,"f1_stderr":0.019283422341780296,"main_score":0.40437121721587077},"nb":{"accuracy":0.35763281775386685,"f1":0.3412599346756071,"accuracy_stderr":0.02580146772548029,"f1_stderr":0.0237566412933957,"main_score":0.35763281775386685},"sv":{"accuracy":0.5609280430396772,"f1":0.547979935150986,"accuracy_stderr":0.024760603110657826,"f1_stderr":0.02207147319053608,"main_score":0.5609280430396772}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KB__bert-base-swedish-cased/NoReC.json b/src/seb/cache/KB__bert-base-swedish-cased/NoReC.json new file mode 100644 index 00000000..cf565a75 --- /dev/null +++ b/src/seb/cache/KB__bert-base-swedish-cased/NoReC.json @@ -0,0 +1 @@ +{"task_name":"NoReC","task_description":"A Norwegian dataset for sentiment classification on review","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T23:41:40.416839","scores":{"nb":{"accuracy":0.4390625,"f1":0.4202061121201135,"accuracy_stderr":0.028534013227877218,"f1_stderr":0.024742648756657316,"main_score":0.4390625}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KB__bert-base-swedish-cased/Norwegian_parliament.json b/src/seb/cache/KB__bert-base-swedish-cased/Norwegian_parliament.json new file mode 100644 index 00000000..51f0481c --- /dev/null +++ b/src/seb/cache/KB__bert-base-swedish-cased/Norwegian_parliament.json @@ -0,0 +1 @@ +{"task_name":"Norwegian parliament","task_description":"Norwegian parliament speeches annotated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T00:02:20.59066","scores":{"nb":{"accuracy":0.5755833333333334,"f1":0.5723464665881223,"ap":0.5448983994542387,"accuracy_stderr":0.0293731973442154,"f1_stderr":0.029062139074027413,"ap_stderr":0.019931830281335862,"main_score":0.5755833333333334}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KB__bert-base-swedish-cased/Scala.json b/src/seb/cache/KB__bert-base-swedish-cased/Scala.json new file mode 100644 index 00000000..921c1435 --- /dev/null +++ b/src/seb/cache/KB__bert-base-swedish-cased/Scala.json @@ -0,0 +1,45 @@ +{ + "task_name": "ScaLA", + "task_description": "A linguistic acceptability task for Danish, Norwegian Bokmål Norwegian Nynorsk and Swedish.", + "task_version": "1.0.3.dev0", + "time_of_run": "2023-07-27T23:39:46.312359", + "scores": { + "da": { + "accuracy": 0.535302734375, + "f1": 0.5317564506131404, + "ap": 0.5190339482077201, + "accuracy_stderr": 0.014917851250069142, + "f1_stderr": 0.015658989073144235, + "ap_stderr": 0.008496708329137853, + "main_score": 0.535302734375 + }, + "nb": { + "accuracy": 0.536279296875, + "f1": 0.5337152569584676, + "ap": 0.5197845375798126, + "accuracy_stderr": 0.01425421682507119, + "f1_stderr": 0.01295236142071566, + "ap_stderr": 0.008407439404250613, + "main_score": 0.536279296875 + }, + "sv": { + "accuracy": 0.606884765625, + "f1": 0.6040820549557981, + "ap": 0.5669525516433389, + "accuracy_stderr": 0.03894051387383141, + "f1_stderr": 0.042546675006439924, + "ap_stderr": 0.02720295446228148, + "main_score": 0.606884765625 + }, + "nn": { + "accuracy": 0.51923828125, + "f1": 0.5160130062470971, + "ap": 0.5101820258549491, + "accuracy_stderr": 0.013660011593993868, + "f1_stderr": 0.015036008604954812, + "ap_stderr": 0.007522714100966331, + "main_score": 0.51923828125 + } + }, + "main_score": "accuracy" +} \ No newline at end of file diff --git a/src/seb/cache/KB__bert-base-swedish-cased/SweFAQ.json b/src/seb/cache/KB__bert-base-swedish-cased/SweFAQ.json new file mode 100644 index 00000000..574bf3aa --- /dev/null +++ b/src/seb/cache/KB__bert-base-swedish-cased/SweFAQ.json @@ -0,0 +1 @@ +{"task_name":"SweFAQ","task_description":"A Swedish QA dataset derived from FAQ","task_version":"0.0.1","time_of_run":"2023-09-04T19:37:50.677888","scores":{"sv":{"ndcg_at_1":0.20468,"ndcg_at_3":0.28536,"ndcg_at_5":0.32001,"ndcg_at_10":0.34008,"ndcg_at_100":0.41087,"ndcg_at_1000":0.43315,"map_at_1":0.20468,"map_at_3":0.26543,"map_at_5":0.28473,"map_at_10":0.29295,"map_at_100":0.30656,"map_at_1000":0.30753,"recall_at_1":0.20468,"recall_at_3":0.34308,"recall_at_5":0.4269,"recall_at_10":0.48928,"recall_at_100":0.83236,"recall_at_1000":1.0,"precision_at_1":0.20468,"precision_at_3":0.11436,"precision_at_5":0.08538,"precision_at_10":0.04893,"precision_at_100":0.00832,"precision_at_1000":0.001,"mrr_at_1":0.20468,"mrr_at_3":0.26543,"mrr_at_5":0.28473,"mrr_at_10":0.29295,"mrr_at_100":0.30656,"mrr_at_1000":0.30753}},"main_score":"ndcg_at_10"} \ No newline at end of file diff --git a/src/seb/cache/KB__bert-base-swedish-cased/SweReC.json b/src/seb/cache/KB__bert-base-swedish-cased/SweReC.json new file mode 100644 index 00000000..758be2e9 --- /dev/null +++ b/src/seb/cache/KB__bert-base-swedish-cased/SweReC.json @@ -0,0 +1 @@ +{"task_name":"SweReC","task_description":"A Swedish dataset for sentiment classification on review","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T11:02:47.080172","scores":{"sv":{"accuracy":0.6984375,"f1":0.6401058854724648,"accuracy_stderr":0.033370156611782956,"f1_stderr":0.031149126296143328,"main_score":0.6984375}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/Angry_Tweets.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/Angry_Tweets.json new file mode 100644 index 00000000..aa8bba7a --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/Angry_Tweets.json @@ -0,0 +1 @@ +{"task_name":"Angry Tweets","task_description":"A sentiment dataset with 3 classes (positiv, negativ, neutral) for Danish tweets","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T16:45:25.73012","scores":{"da":{"accuracy":0.5442215854823305,"f1":0.5342464349802925,"accuracy_stderr":0.01958697190236269,"f1_stderr":0.019105747094368127,"main_score":0.5442215854823305}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/Bornholm_Parallel.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/Bornholm_Parallel.json new file mode 100644 index 00000000..c6511fbb --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/Bornholm_Parallel.json @@ -0,0 +1 @@ +{"task_name":"Bornholm Parallel","task_description":"Danish Bornholmsk Parallel Corpus. Bornholmsk is a Danish dialect spoken on the island of Bornholm, Denmark. Historically it is a part of east Danish which was also spoken in Scania and Halland, Sweden.","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T16:46:26.470392","scores":{"da":{"precision":0.15114859033371691,"recall":0.19,"f1":0.15932621715352807,"accuracy":0.19,"main_score":0.15932621715352807},"da-bornholm":{"precision":0.15114859033371691,"recall":0.19,"f1":0.15932621715352807,"accuracy":0.19,"main_score":0.15932621715352807}},"main_score":"f1"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/DKHate.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/DKHate.json new file mode 100644 index 00000000..cade1fa9 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/DKHate.json @@ -0,0 +1 @@ +{"task_name":"DKHate","task_description":"Danish Tweets annotated for Hate Speech either being Offensive or not","task_version":"1.0.3.dev0","time_of_run":"2023-07-30T14:11:53.05474","scores":{"da":{"accuracy":0.6319148936170212,"f1":0.5204047139843642,"ap":0.9040917686410743,"accuracy_stderr":0.046025303308468324,"f1_stderr":0.034102688729471656,"ap_stderr":0.014895049848455449,"main_score":0.6319148936170212}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/DaLAJ.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/DaLAJ.json new file mode 100644 index 00000000..ebc469a2 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/DaLAJ.json @@ -0,0 +1 @@ +{"task_name":"DaLAJ","task_description":"A Swedish dataset for linguistic accebtablity. Available as a part of Superlim.","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T17:35:31.278543","scores":{"sv":{"accuracy":0.5003378378378378,"f1":0.4987202282686436,"ap":0.5003902780548716,"accuracy_stderr":0.014880639323292307,"f1_stderr":0.01519993053587878,"ap_stderr":0.0073418619974873756,"main_score":0.5003378378378378}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/Da_Political_Comments.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/Da_Political_Comments.json new file mode 100644 index 00000000..29e92b5c --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/Da_Political_Comments.json @@ -0,0 +1 @@ +{"task_name":"Da Political Comments","task_description":"A dataset of Danish political comments rated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T16:49:45.029022","scores":{"da":{"accuracy":0.3846836847946725,"f1":0.35985601979838255,"accuracy_stderr":0.033741748902330254,"f1_stderr":0.024193648148913893,"main_score":0.3846836847946725}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/LCC.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/LCC.json new file mode 100644 index 00000000..19446ac7 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/LCC.json @@ -0,0 +1 @@ +{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T16:45:59.23931","scores":{"da":{"accuracy":0.5806666666666667,"f1":0.5689971987949232,"accuracy_stderr":0.042941821107167766,"f1_stderr":0.03371032554439154,"main_score":0.5806666666666667}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/Language_Identification.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/Language_Identification.json new file mode 100644 index 00000000..dce9aeae --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/Language_Identification.json @@ -0,0 +1 @@ +{"task_name":"Language Identification","task_description":"A dataset for Nordic language identification.","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T17:15:15.864662","scores":{"da":{"accuracy":0.7598333333333334,"f1":0.7558976547136231,"accuracy_stderr":0.013334374959313059,"f1_stderr":0.013821655260721424,"main_score":0.7598333333333334},"sv":{"accuracy":0.7598333333333334,"f1":0.7558976547136231,"accuracy_stderr":0.013334374959313059,"f1_stderr":0.013821655260721424,"main_score":0.7598333333333334},"nb":{"accuracy":0.7598333333333334,"f1":0.7558976547136231,"accuracy_stderr":0.013334374959313059,"f1_stderr":0.013821655260721424,"main_score":0.7598333333333334},"nn":{"accuracy":0.7598333333333334,"f1":0.7558976547136231,"accuracy_stderr":0.013334374959313059,"f1_stderr":0.013821655260721424,"main_score":0.7598333333333334},"is":{"accuracy":0.7598333333333334,"f1":0.7558976547136231,"accuracy_stderr":0.013334374959313059,"f1_stderr":0.013821655260721424,"main_score":0.7598333333333334},"fo":{"accuracy":0.7598333333333334,"f1":0.7558976547136231,"accuracy_stderr":0.013334374959313059,"f1_stderr":0.013821655260721424,"main_score":0.7598333333333334}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/Massive_Intent.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/Massive_Intent.json new file mode 100644 index 00000000..08fa911f --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/Massive_Intent.json @@ -0,0 +1 @@ +{"task_name":"Massive Intent","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T17:01:33.382762","scores":{"da":{"accuracy":0.6582716879623403,"f1":0.6208842888158964,"accuracy_stderr":0.010244296914036263,"f1_stderr":0.00941057589534994,"main_score":0.6582716879623403},"nb":{"accuracy":0.5757229320780095,"f1":0.5479101324305733,"accuracy_stderr":0.014698899958770613,"f1_stderr":0.009961334138223912,"main_score":0.5757229320780095},"sv":{"accuracy":0.5539340954942839,"f1":0.5246392360657182,"accuracy_stderr":0.009796522813182882,"f1_stderr":0.008644343226026373,"main_score":0.5539340954942839}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/Massive_Scenario.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/Massive_Scenario.json new file mode 100644 index 00000000..4ea330bd --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/Massive_Scenario.json @@ -0,0 +1 @@ +{"task_name":"Massive Scenario","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T17:06:12.775757","scores":{"da":{"accuracy":0.7160726294552792,"f1":0.7041873241706947,"accuracy_stderr":0.02100360122291461,"f1_stderr":0.019159130877734326,"main_score":0.7160726294552792},"nb":{"accuracy":0.6365837256220579,"f1":0.6196857169074892,"accuracy_stderr":0.015808495581346612,"f1_stderr":0.013663579561431998,"main_score":0.6365837256220579},"sv":{"accuracy":0.5715870880968392,"f1":0.5542408157073229,"accuracy_stderr":0.018867319112281,"f1_stderr":0.014907747726350446,"main_score":0.5715870880968392}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/NoReC.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/NoReC.json new file mode 100644 index 00000000..8e0281d4 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/NoReC.json @@ -0,0 +1 @@ +{"task_name":"NoReC","task_description":"A Norwegian dataset for sentiment classification on review","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T17:16:44.474202","scores":{"nb":{"accuracy":0.504638671875,"f1":0.4843289551497828,"accuracy_stderr":0.02303761441662381,"f1_stderr":0.02645031091137531,"main_score":0.504638671875}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/Norwegian_parliament.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/Norwegian_parliament.json new file mode 100644 index 00000000..65298ee0 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/Norwegian_parliament.json @@ -0,0 +1 @@ +{"task_name":"Norwegian parliament","task_description":"Norwegian parliament speeches annotated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T17:28:01.01584","scores":{"nb":{"accuracy":0.5765833333333333,"f1":0.5727197115118734,"ap":0.5450433584591905,"accuracy_stderr":0.02711971914800495,"f1_stderr":0.030120069211795164,"ap_stderr":0.018743637349274624,"main_score":0.5765833333333333}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/ScaLA.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/ScaLA.json new file mode 100644 index 00000000..27efed2c --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/ScaLA.json @@ -0,0 +1 @@ +{"task_name":"ScaLA","task_description":"A linguistic acceptability task for Danish, Norwegian Bokmål Norwegian Nynorsk and Swedish.","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T17:12:00.914539","scores":{"da":{"accuracy":0.6609375,"f1":0.6577331070475134,"ap":0.6082074644370966,"accuracy_stderr":0.025283540713876947,"f1_stderr":0.026302978989849624,"ap_stderr":0.020631811137533373,"main_score":0.6609375},"nb":{"accuracy":0.62685546875,"f1":0.6234950635607823,"ap":0.5783958485104621,"accuracy_stderr":0.015920964058674877,"f1_stderr":0.018014238232907147,"ap_stderr":0.011821491427302796,"main_score":0.62685546875},"sv":{"accuracy":0.576806640625,"f1":0.5750900787345138,"ap":0.5459700866073529,"accuracy_stderr":0.04010632331631002,"f1_stderr":0.04076555572153807,"ap_stderr":0.024639356679992935,"main_score":0.576806640625},"nn":{"accuracy":0.59365234375,"f1":0.5914774221765096,"ap":0.5559171784545646,"accuracy_stderr":0.020033579293686044,"f1_stderr":0.021165959480845965,"ap_stderr":0.013679691929052069,"main_score":0.59365234375}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/SweFAQ.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/SweFAQ.json new file mode 100644 index 00000000..b3a8c7cc --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/SweFAQ.json @@ -0,0 +1 @@ +{"task_name":"SweFAQ","task_description":"A Swedish QA dataset derived from FAQ","task_version":"0.0.1","time_of_run":"2023-09-04T19:48:22.557389","scores":{"sv":{"ndcg_at_1":0.2924,"ndcg_at_3":0.37842,"ndcg_at_5":0.40938,"ndcg_at_10":0.43807,"ndcg_at_100":0.49433,"ndcg_at_1000":0.51052,"map_at_1":0.2924,"map_at_3":0.3564,"map_at_5":0.37385,"map_at_10":0.3855,"map_at_100":0.39671,"map_at_1000":0.39732,"recall_at_1":0.2924,"recall_at_3":0.4425,"recall_at_5":0.51657,"recall_at_10":0.60624,"recall_at_100":0.87329,"recall_at_1000":1.0,"precision_at_1":0.2924,"precision_at_3":0.1475,"precision_at_5":0.10331,"precision_at_10":0.06062,"precision_at_100":0.00873,"precision_at_1000":0.001,"mrr_at_1":0.29435,"mrr_at_3":0.35737,"mrr_at_5":0.37482,"mrr_at_10":0.38647,"mrr_at_100":0.39768,"mrr_at_1000":0.39829}},"main_score":"ndcg_at_10"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/SweReC.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/SweReC.json new file mode 100644 index 00000000..5e6cd45d --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-1/SweReC.json @@ -0,0 +1 @@ +{"task_name":"SweReC","task_description":"A Swedish dataset for sentiment classification on review","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T17:34:29.035978","scores":{"sv":{"accuracy":0.669580078125,"f1":0.6045095443781163,"accuracy_stderr":0.02740824042544494,"f1_stderr":0.025235723807146324,"main_score":0.669580078125}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/Angry_Tweets.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/Angry_Tweets.json new file mode 100644 index 00000000..2297809b --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/Angry_Tweets.json @@ -0,0 +1 @@ +{"task_name":"Angry Tweets","task_description":"A sentiment dataset with 3 classes (positiv, negativ, neutral) for Danish tweets","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T13:33:53.643601","scores":{"da":{"accuracy":0.5303724928366762,"f1":0.5253068690627222,"accuracy_stderr":0.026977584164188308,"f1_stderr":0.024889776062830975,"main_score":0.5303724928366762}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/Bornholm_Parallel.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/Bornholm_Parallel.json new file mode 100644 index 00000000..28f02ada --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/Bornholm_Parallel.json @@ -0,0 +1 @@ +{"task_name":"Bornholm Parallel","task_description":"Danish Bornholmsk Parallel Corpus. Bornholmsk is a Danish dialect spoken on the island of Bornholm, Denmark. Historically it is a part of east Danish which was also spoken in Scania and Halland, Sweden.","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T13:34:50.084207","scores":{"da":{"precision":0.685,"recall":0.754,"f1":0.7066,"accuracy":0.754,"main_score":0.7066},"da-bornholm":{"precision":0.685,"recall":0.754,"f1":0.7066,"accuracy":0.754,"main_score":0.7066}},"main_score":"f1"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/DKHate.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/DKHate.json new file mode 100644 index 00000000..1051ee7e --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/DKHate.json @@ -0,0 +1 @@ +{"task_name":"DKHate","task_description":"Danish Tweets annotated for Hate Speech either being Offensive or not","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T13:35:54.938407","scores":{"da":{"accuracy":0.6617021276595745,"f1":0.5491001096430189,"ap":0.9112454967335175,"accuracy_stderr":0.05881360790239555,"f1_stderr":0.0455047266703659,"ap_stderr":0.012126443800377697,"main_score":0.6617021276595745}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/DaLAJ.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/DaLAJ.json new file mode 100644 index 00000000..ac9f29f1 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/DaLAJ.json @@ -0,0 +1 @@ +{"task_name":"DaLAJ","task_description":"A Swedish dataset for linguistic acceptability. Available as a part of Superlim.","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T14:06:34.476335","scores":{"sv":{"accuracy":0.4978603603603604,"f1":0.49712872717740497,"ap":0.49895189387373745,"accuracy_stderr":0.004286681601503207,"f1_stderr":0.00453293691502554,"ap_stderr":0.0021166494604273017,"main_score":0.4978603603603604}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/Da_Political_Comments.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/Da_Political_Comments.json new file mode 100644 index 00000000..146de6d2 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/Da_Political_Comments.json @@ -0,0 +1 @@ +{"task_name":"Da Political Comments","task_description":"A dataset of Danish political comments rated for sentiment","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T13:37:08.425363","scores":{"da":{"accuracy":0.37669256381798005,"f1":0.34647339889444656,"accuracy_stderr":0.02333754199608934,"f1_stderr":0.01963303196288125,"main_score":0.37669256381798005}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/LCC.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/LCC.json new file mode 100644 index 00000000..7e38f960 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/LCC.json @@ -0,0 +1 @@ +{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T13:34:25.265834","scores":{"da":{"accuracy":0.5646666666666667,"f1":0.5509255123901754,"accuracy_stderr":0.04686149805543992,"f1_stderr":0.040864693722885134,"main_score":0.5646666666666667}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/Language_Identification.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/Language_Identification.json new file mode 100644 index 00000000..c118f6d9 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/Language_Identification.json @@ -0,0 +1 @@ +{"task_name":"Language Identification","task_description":"A dataset for Nordic language identification.","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T13:55:44.172167","scores":{"da":{"accuracy":0.4648333333333333,"f1":0.4616914360939985,"accuracy_stderr":0.017227400655157845,"f1_stderr":0.01515335772170534,"main_score":0.4648333333333333},"sv":{"accuracy":0.4648333333333333,"f1":0.4616914360939985,"accuracy_stderr":0.017227400655157845,"f1_stderr":0.01515335772170534,"main_score":0.4648333333333333},"nb":{"accuracy":0.4648333333333333,"f1":0.4616914360939985,"accuracy_stderr":0.017227400655157845,"f1_stderr":0.01515335772170534,"main_score":0.4648333333333333},"nn":{"accuracy":0.4648333333333333,"f1":0.4616914360939985,"accuracy_stderr":0.017227400655157845,"f1_stderr":0.01515335772170534,"main_score":0.4648333333333333},"is":{"accuracy":0.4648333333333333,"f1":0.4616914360939985,"accuracy_stderr":0.017227400655157845,"f1_stderr":0.01515335772170534,"main_score":0.4648333333333333},"fo":{"accuracy":0.4648333333333333,"f1":0.4616914360939985,"accuracy_stderr":0.017227400655157845,"f1_stderr":0.01515335772170534,"main_score":0.4648333333333333}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/Massive_Intent.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/Massive_Intent.json new file mode 100644 index 00000000..500ee4c6 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/Massive_Intent.json @@ -0,0 +1 @@ +{"task_name":"Massive Intent","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T13:43:45.727572","scores":{"da":{"accuracy":0.640114324142569,"f1":0.6132115246966137,"accuracy_stderr":0.015953906425053222,"f1_stderr":0.015421148799563949,"main_score":0.640114324142569},"nb":{"accuracy":0.6327505043712173,"f1":0.6078005115673859,"accuracy_stderr":0.013136737563782178,"f1_stderr":0.011679160911823337,"main_score":0.6327505043712173},"sv":{"accuracy":0.572158708809684,"f1":0.5423765775066416,"accuracy_stderr":0.015276296903067412,"f1_stderr":0.008607902586583675,"main_score":0.572158708809684}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/Massive_Scenario.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/Massive_Scenario.json new file mode 100644 index 00000000..a7af95dd --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/Massive_Scenario.json @@ -0,0 +1 @@ +{"task_name":"Massive Scenario","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T13:47:45.089019","scores":{"da":{"accuracy":0.7010087424344318,"f1":0.6925077323613686,"accuracy_stderr":0.011455927400521915,"f1_stderr":0.011545746744515549,"main_score":0.7010087424344318},"nb":{"accuracy":0.7010087424344318,"f1":0.6944094609008696,"accuracy_stderr":0.01235250773779178,"f1_stderr":0.011367117350983278,"main_score":0.7010087424344318},"sv":{"accuracy":0.6205447209145931,"f1":0.613576531654512,"accuracy_stderr":0.011675343345736092,"f1_stderr":0.0101436666142386,"main_score":0.6205447209145931}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/NoReC.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/NoReC.json new file mode 100644 index 00000000..408f2479 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/NoReC.json @@ -0,0 +1 @@ +{"task_name":"NoReC","task_description":"A Norwegian dataset for sentiment classification on review","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T13:57:04.088391","scores":{"nb":{"accuracy":0.536279296875,"f1":0.5147308818477775,"accuracy_stderr":0.029207260402358548,"f1_stderr":0.025725422038223048,"main_score":0.536279296875}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/Norwegian_parliament.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/Norwegian_parliament.json new file mode 100644 index 00000000..892b173a --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/Norwegian_parliament.json @@ -0,0 +1 @@ +{"task_name":"Norwegian parliament","task_description":"Norwegian parliament speeches annotated for sentiment","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T14:01:46.531047","scores":{"nb":{"accuracy":0.5544166666666668,"f1":0.5505233928368802,"ap":0.5324447154885268,"accuracy_stderr":0.03959526136070101,"f1_stderr":0.03831623567526245,"ap_stderr":0.024102999476410703,"main_score":0.5544166666666668}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/ScaLA.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/ScaLA.json new file mode 100644 index 00000000..6f51cf02 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/ScaLA.json @@ -0,0 +1 @@ +{"task_name":"ScaLA","task_description":"A linguistic acceptability task for Danish, Norwegian Bokmål Norwegian Nynorsk and Swedish.","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T13:52:53.261223","scores":{"da":{"accuracy":0.508837890625,"f1":0.5043736216956247,"ap":0.5046155216411,"accuracy_stderr":0.011822961927598985,"f1_stderr":0.011372955731495184,"ap_stderr":0.006083632107655575,"main_score":0.508837890625},"nb":{"accuracy":0.511767578125,"f1":0.5088198635934648,"ap":0.5060948273481344,"accuracy_stderr":0.007164435490307341,"f1_stderr":0.008204387465364784,"ap_stderr":0.003750344249701892,"main_score":0.511767578125},"sv":{"accuracy":0.502734375,"f1":0.49776322292447245,"ap":0.5014004881285211,"accuracy_stderr":0.004975679376116014,"f1_stderr":0.009584884832978447,"ap_stderr":0.0024993404404634705,"main_score":0.502734375},"nn":{"accuracy":0.50888671875,"f1":0.5072927816759232,"ap":0.504556743325273,"accuracy_stderr":0.006003278472384251,"f1_stderr":0.006024449005916439,"ap_stderr":0.0030937506147436433,"main_score":0.50888671875}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/SweFAQ.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/SweFAQ.json new file mode 100644 index 00000000..fcf959ae --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/SweFAQ.json @@ -0,0 +1 @@ +{"task_name":"SweFAQ","task_description":"A Swedish QA dataset derived from FAQ","task_version":"0.0.1","time_of_run":"2023-11-15T14:07:47.128684","scores":{"sv":{"ndcg_at_1":0.27875,"ndcg_at_3":0.37482,"ndcg_at_5":0.4208,"ndcg_at_10":0.45171,"ndcg_at_100":0.51532,"ndcg_at_1000":0.52261,"map_at_1":0.27875,"map_at_3":0.35023,"map_at_5":0.37586,"map_at_10":0.38826,"map_at_100":0.40194,"map_at_1000":0.40231,"recall_at_1":0.27875,"recall_at_3":0.44639,"recall_at_5":0.5575,"recall_at_10":0.65497,"recall_at_100":0.94737,"recall_at_1000":1.0,"precision_at_1":0.27875,"precision_at_3":0.1488,"precision_at_5":0.1115,"precision_at_10":0.0655,"precision_at_100":0.00947,"precision_at_1000":0.001,"mrr_at_1":0.2807,"mrr_at_3":0.3512,"mrr_at_5":0.37684,"mrr_at_10":0.38924,"mrr_at_100":0.40292,"mrr_at_1000":0.40329}},"main_score":"ndcg_at_10"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/SweReC.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/SweReC.json new file mode 100644 index 00000000..eaf7f360 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp1/SweReC.json @@ -0,0 +1 @@ +{"task_name":"SweReC","task_description":"A Swedish dataset for sentiment classification on review","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T14:05:37.351721","scores":{"sv":{"accuracy":0.682421875,"f1":0.6200398041535817,"accuracy_stderr":0.03117758944723332,"f1_stderr":0.026012060399460787,"main_score":0.682421875}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/Angry_Tweets.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/Angry_Tweets.json new file mode 100644 index 00000000..46d3d452 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/Angry_Tweets.json @@ -0,0 +1 @@ +{"task_name":"Angry Tweets","task_description":"A sentiment dataset with 3 classes (positiv, negativ, neutral) for Danish tweets","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T16:10:33.999119","scores":{"da":{"accuracy":0.5339063992359121,"f1":0.5288222170280102,"accuracy_stderr":0.025971261281550032,"f1_stderr":0.02612912167930505,"main_score":0.5339063992359121}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/Bornholm_Parallel.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/Bornholm_Parallel.json new file mode 100644 index 00000000..56d1aa87 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/Bornholm_Parallel.json @@ -0,0 +1 @@ +{"task_name":"Bornholm Parallel","task_description":"Danish Bornholmsk Parallel Corpus. Bornholmsk is a Danish dialect spoken on the island of Bornholm, Denmark. Historically it is a part of east Danish which was also spoken in Scania and Halland, Sweden.","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T16:11:34.241275","scores":{"da":{"precision":0.786,"recall":0.838,"f1":0.8026,"accuracy":0.838,"main_score":0.8026},"da-bornholm":{"precision":0.786,"recall":0.838,"f1":0.8026,"accuracy":0.838,"main_score":0.8026}},"main_score":"f1"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/DKHate.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/DKHate.json new file mode 100644 index 00000000..1666e1f4 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/DKHate.json @@ -0,0 +1 @@ +{"task_name":"DKHate","task_description":"Danish Tweets annotated for Hate Speech either being Offensive or not","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T16:12:41.593164","scores":{"da":{"accuracy":0.5924012158054711,"f1":0.4952491516050313,"ap":0.16181889535519384,"accuracy_stderr":0.05364461792127439,"f1_stderr":0.0390806907988279,"ap_stderr":0.027653041619518814,"main_score":0.5924012158054711}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/DaLAJ.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/DaLAJ.json new file mode 100644 index 00000000..fb50ee27 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/DaLAJ.json @@ -0,0 +1 @@ +{"task_name":"DaLAJ","task_description":"A Swedish dataset for linguistic acceptability. Available as a part of Superlim.","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T17:19:18.481195","scores":{"sv":{"accuracy":0.4968468468468468,"f1":0.4946605519153303,"ap":0.49855850527615964,"accuracy_stderr":0.010501390294481777,"f1_stderr":0.009437647593689131,"ap_stderr":0.00544777400090162,"main_score":0.4968468468468468}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/Da_Political_Comments.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/Da_Political_Comments.json new file mode 100644 index 00000000..795e5271 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/Da_Political_Comments.json @@ -0,0 +1 @@ +{"task_name":"Da Political Comments","task_description":"A dataset of Danish political comments rated for sentiment","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T16:13:58.639627","scores":{"da":{"accuracy":0.3849056603773585,"f1":0.3576728840501947,"accuracy_stderr":0.023503744926926327,"f1_stderr":0.02211513325389706,"main_score":0.3849056603773585}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/LCC.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/LCC.json new file mode 100644 index 00000000..70df0e53 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/LCC.json @@ -0,0 +1 @@ +{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T16:11:07.339041","scores":{"da":{"accuracy":0.5599999999999999,"f1":0.5446085372513285,"accuracy_stderr":0.03333333333333333,"f1_stderr":0.031727678118169636,"main_score":0.5599999999999999}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/Language_Identification.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/Language_Identification.json new file mode 100644 index 00000000..878d6722 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/Language_Identification.json @@ -0,0 +1 @@ +{"task_name":"Language Identification","task_description":"A dataset for Nordic language identification.","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T16:34:59.248579","scores":{"da":{"accuracy":0.6252333333333333,"f1":0.6193653504972276,"accuracy_stderr":0.015101177290382226,"f1_stderr":0.014701958418252682,"main_score":0.6252333333333333},"sv":{"accuracy":0.6252333333333333,"f1":0.6193653504972276,"accuracy_stderr":0.015101177290382226,"f1_stderr":0.014701958418252682,"main_score":0.6252333333333333},"nb":{"accuracy":0.6252333333333333,"f1":0.6193653504972276,"accuracy_stderr":0.015101177290382226,"f1_stderr":0.014701958418252682,"main_score":0.6252333333333333},"nn":{"accuracy":0.6252333333333333,"f1":0.6193653504972276,"accuracy_stderr":0.015101177290382226,"f1_stderr":0.014701958418252682,"main_score":0.6252333333333333},"is":{"accuracy":0.6252333333333333,"f1":0.6193653504972276,"accuracy_stderr":0.015101177290382226,"f1_stderr":0.014701958418252682,"main_score":0.6252333333333333},"fo":{"accuracy":0.6252333333333333,"f1":0.6193653504972276,"accuracy_stderr":0.015101177290382226,"f1_stderr":0.014701958418252682,"main_score":0.6252333333333333}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/Massive_Intent.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/Massive_Intent.json new file mode 100644 index 00000000..3b15616d --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/Massive_Intent.json @@ -0,0 +1 @@ +{"task_name":"Massive Intent","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T16:20:55.54355","scores":{"da":{"accuracy":0.6732683254875588,"f1":0.6351917795376865,"accuracy_stderr":0.009653348286521398,"f1_stderr":0.008252642158605205,"main_score":0.6732683254875588},"nb":{"accuracy":0.6429051782111634,"f1":0.6156286593697626,"accuracy_stderr":0.014413215890769491,"f1_stderr":0.012331367116296947,"main_score":0.6429051782111634},"sv":{"accuracy":0.6094821788836584,"f1":0.5712468350417064,"accuracy_stderr":0.013577238486933076,"f1_stderr":0.011368480806483552,"main_score":0.6094821788836584}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/Massive_Scenario.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/Massive_Scenario.json new file mode 100644 index 00000000..f29efb62 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/Massive_Scenario.json @@ -0,0 +1 @@ +{"task_name":"Massive Scenario","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T16:25:18.475077","scores":{"da":{"accuracy":0.7364828513786146,"f1":0.7264125769333567,"accuracy_stderr":0.015576680302857804,"f1_stderr":0.012842282118468157,"main_score":0.7364828513786146},"nb":{"accuracy":0.708238063214526,"f1":0.6970047543988354,"accuracy_stderr":0.010625686324592197,"f1_stderr":0.009124043502303395,"main_score":0.708238063214526},"sv":{"accuracy":0.6453934095494285,"f1":0.6292547297746359,"accuracy_stderr":0.008887625766883995,"f1_stderr":0.011402555574163658,"main_score":0.6453934095494285}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/NoReC.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/NoReC.json new file mode 100644 index 00000000..5511a3b4 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/NoReC.json @@ -0,0 +1 @@ +{"task_name":"NoReC","task_description":"A Norwegian dataset for sentiment classification on review","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T16:38:18.261049","scores":{"nb":{"accuracy":0.516064453125,"f1":0.4904087683600162,"accuracy_stderr":0.023646579232123693,"f1_stderr":0.022702952953719543,"main_score":0.516064453125}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/Norwegian_parliament.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/Norwegian_parliament.json new file mode 100644 index 00000000..0efac5c7 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/Norwegian_parliament.json @@ -0,0 +1 @@ +{"task_name":"Norwegian parliament","task_description":"Norwegian parliament speeches annotated for sentiment","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T17:14:20.225595","scores":{"nb":{"accuracy":0.5585,"f1":0.5560010923554946,"ap":0.5338997316953418,"accuracy_stderr":0.0341784532515248,"f1_stderr":0.03402327344682774,"ap_stderr":0.020219122654274186,"main_score":0.5585}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/ScaLA.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/ScaLA.json new file mode 100644 index 00000000..35049439 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/ScaLA.json @@ -0,0 +1 @@ +{"task_name":"ScaLA","task_description":"A linguistic acceptability task for Danish, Norwegian Bokmål Norwegian Nynorsk and Swedish.","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T16:31:58.002132","scores":{"da":{"accuracy":0.551806640625,"f1":0.5483321676008772,"ap":0.5287602312778555,"accuracy_stderr":0.018837090727898097,"f1_stderr":0.019875107739646018,"ap_stderr":0.011251095986871233,"main_score":0.551806640625},"nb":{"accuracy":0.540869140625,"f1":0.536871297585976,"ap":0.5224474391725933,"accuracy_stderr":0.01583123477614281,"f1_stderr":0.015290306640459092,"ap_stderr":0.00925803703070263,"main_score":0.540869140625},"sv":{"accuracy":0.530615234375,"f1":0.5284719501984025,"ap":0.5166866244399766,"accuracy_stderr":0.019816099110724704,"f1_stderr":0.0202551551304083,"ap_stderr":0.011209010599913208,"main_score":0.530615234375},"nn":{"accuracy":0.538134765625,"f1":0.5361146747111086,"ap":0.5207556851500014,"accuracy_stderr":0.012548828125,"f1_stderr":0.014622980691166156,"ap_stderr":0.007265564215843514,"main_score":0.538134765625}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/SweFAQ.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/SweFAQ.json new file mode 100644 index 00000000..8e16ea97 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/SweFAQ.json @@ -0,0 +1 @@ +{"task_name":"SweFAQ","task_description":"A Swedish QA dataset derived from FAQ","task_version":"0.0.1","time_of_run":"2023-11-15T17:20:28.70611","scores":{"sv":{"ndcg_at_1":0.37622,"ndcg_at_3":0.49969,"ndcg_at_5":0.53535,"ndcg_at_10":0.57175,"ndcg_at_100":0.61258,"ndcg_at_1000":0.61583,"map_at_1":0.37622,"map_at_3":0.47076,"map_at_5":0.49074,"map_at_10":0.50566,"map_at_100":0.51389,"map_at_1000":0.51401,"recall_at_1":0.37622,"recall_at_3":0.58285,"recall_at_5":0.66862,"recall_at_10":0.78168,"recall_at_100":0.97466,"recall_at_1000":1.0,"precision_at_1":0.37622,"precision_at_3":0.19428,"precision_at_5":0.13372,"precision_at_10":0.07817,"precision_at_100":0.00975,"precision_at_1000":0.001,"mrr_at_1":0.37622,"mrr_at_3":0.47076,"mrr_at_5":0.49084,"mrr_at_10":0.50576,"mrr_at_100":0.51399,"mrr_at_1000":0.51411}},"main_score":"ndcg_at_10"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/SweReC.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/SweReC.json new file mode 100644 index 00000000..5aad053f --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-large-exp2-no-lang-align/SweReC.json @@ -0,0 +1 @@ +{"task_name":"SweReC","task_description":"A Swedish dataset for sentiment classification on review","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T17:18:21.040696","scores":{"sv":{"accuracy":0.64619140625,"f1":0.5939322678768221,"accuracy_stderr":0.023688636611603496,"f1_stderr":0.019923539292951657,"main_score":0.64619140625}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/Angry_Tweets.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/Angry_Tweets.json new file mode 100644 index 00000000..edeefd70 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/Angry_Tweets.json @@ -0,0 +1 @@ +{"task_name":"Angry Tweets","task_description":"A sentiment dataset with 3 classes (positiv, negativ, neutral) for Danish tweets","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T14:40:51.901333","scores":{"da":{"accuracy":0.5573065902578798,"f1":0.5512869518635695,"accuracy_stderr":0.038145250305046126,"f1_stderr":0.03317668065362972,"main_score":0.5573065902578798}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/Bornholm_Parallel.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/Bornholm_Parallel.json new file mode 100644 index 00000000..a0e661ad --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/Bornholm_Parallel.json @@ -0,0 +1 @@ +{"task_name":"Bornholm Parallel","task_description":"Danish Bornholmsk Parallel Corpus. Bornholmsk is a Danish dialect spoken on the island of Bornholm, Denmark. Historically it is a part of east Danish which was also spoken in Scania and Halland, Sweden.","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T14:41:15.87554","scores":{"da":{"precision":0.6547999999999999,"recall":0.716,"f1":0.6736666666666667,"accuracy":0.716,"main_score":0.6736666666666667},"da-bornholm":{"precision":0.6547999999999999,"recall":0.716,"f1":0.6736666666666667,"accuracy":0.716,"main_score":0.6736666666666667}},"main_score":"f1"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/DKHate.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/DKHate.json new file mode 100644 index 00000000..48a14d39 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/DKHate.json @@ -0,0 +1 @@ +{"task_name":"DKHate","task_description":"Danish Tweets annotated for Hate Speech either being Offensive or not","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T14:41:39.837194","scores":{"da":{"accuracy":0.6203647416413375,"f1":0.5186619780914523,"ap":0.17076120663254926,"accuracy_stderr":0.04511297766782941,"f1_stderr":0.030791979307257456,"ap_stderr":0.016093233647309954,"main_score":0.6203647416413375}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/DaLAJ.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/DaLAJ.json new file mode 100644 index 00000000..7f37a0ea --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/DaLAJ.json @@ -0,0 +1 @@ +{"task_name":"DaLAJ","task_description":"A Swedish dataset for linguistic acceptability. Available as a part of Superlim.","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T15:10:03.571841","scores":{"sv":{"accuracy":0.49831081081081086,"f1":0.4964435107880174,"ap":0.4991809848466541,"accuracy_stderr":0.004784379681587388,"f1_stderr":0.005153986431977443,"ap_stderr":0.002394220261157318,"main_score":0.49831081081081086}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/Da_Political_Comments.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/Da_Political_Comments.json new file mode 100644 index 00000000..78caf456 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/Da_Political_Comments.json @@ -0,0 +1 @@ +{"task_name":"Da Political Comments","task_description":"A dataset of Danish political comments rated for sentiment","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T14:42:09.519969","scores":{"da":{"accuracy":0.4047169811320755,"f1":0.37700676839186154,"accuracy_stderr":0.03150083993485701,"f1_stderr":0.022566912460391983,"main_score":0.4047169811320755}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/LCC.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/LCC.json new file mode 100644 index 00000000..19aed4a3 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/LCC.json @@ -0,0 +1 @@ +{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T14:41:05.060876","scores":{"da":{"accuracy":0.5900000000000001,"f1":0.5776762202401808,"accuracy_stderr":0.06038763670377131,"f1_stderr":0.05325735757279493,"main_score":0.5900000000000001}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/Language_Identification.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/Language_Identification.json new file mode 100644 index 00000000..cdb01129 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/Language_Identification.json @@ -0,0 +1 @@ +{"task_name":"Language Identification","task_description":"A dataset for Nordic language identification.","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T15:05:56.710508","scores":{"da":{"accuracy":0.5110333333333335,"f1":0.5081885747117414,"accuracy_stderr":0.01796196598989715,"f1_stderr":0.01705034415740678,"main_score":0.5110333333333335},"sv":{"accuracy":0.5110333333333335,"f1":0.5081885747117414,"accuracy_stderr":0.01796196598989715,"f1_stderr":0.01705034415740678,"main_score":0.5110333333333335},"nb":{"accuracy":0.5110333333333335,"f1":0.5081885747117414,"accuracy_stderr":0.01796196598989715,"f1_stderr":0.01705034415740678,"main_score":0.5110333333333335},"nn":{"accuracy":0.5110333333333335,"f1":0.5081885747117414,"accuracy_stderr":0.01796196598989715,"f1_stderr":0.01705034415740678,"main_score":0.5110333333333335},"is":{"accuracy":0.5110333333333335,"f1":0.5081885747117414,"accuracy_stderr":0.01796196598989715,"f1_stderr":0.01705034415740678,"main_score":0.5110333333333335},"fo":{"accuracy":0.5110333333333335,"f1":0.5081885747117414,"accuracy_stderr":0.01796196598989715,"f1_stderr":0.01705034415740678,"main_score":0.5110333333333335}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/Massive_Intent.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/Massive_Intent.json new file mode 100644 index 00000000..20eb3b65 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/Massive_Intent.json @@ -0,0 +1 @@ +{"task_name":"Massive Intent","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T15:01:04.204673","scores":{"da":{"accuracy":0.6284129119031607,"f1":0.5951314833491403,"accuracy_stderr":0.013549104544463056,"f1_stderr":0.012946379201390924,"main_score":0.6284129119031607},"nb":{"accuracy":0.5870880968392737,"f1":0.5589729354026357,"accuracy_stderr":0.01383281421703477,"f1_stderr":0.011092636167197566,"main_score":0.5870880968392737},"sv":{"accuracy":0.5093476798924008,"f1":0.480164927627215,"accuracy_stderr":0.009116958341403317,"f1_stderr":0.0096102841768827,"main_score":0.5093476798924008}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/Massive_Scenario.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/Massive_Scenario.json new file mode 100644 index 00000000..35721612 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/Massive_Scenario.json @@ -0,0 +1 @@ +{"task_name":"Massive Scenario","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T15:02:33.238487","scores":{"da":{"accuracy":0.7022528581035641,"f1":0.6900395536680104,"accuracy_stderr":0.008014054986294546,"f1_stderr":0.006418081236164402,"main_score":0.7022528581035641},"nb":{"accuracy":0.6398789509078682,"f1":0.6312720024243548,"accuracy_stderr":0.01297497066682901,"f1_stderr":0.01614471079589574,"main_score":0.6398789509078682},"sv":{"accuracy":0.5345326160053799,"f1":0.5254927360969441,"accuracy_stderr":0.006009430131246728,"f1_stderr":0.010711836669943403,"main_score":0.5345326160053799}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/NoReC.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/NoReC.json new file mode 100644 index 00000000..94655aec --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/NoReC.json @@ -0,0 +1 @@ +{"task_name":"NoReC","task_description":"A Norwegian dataset for sentiment classification on review","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T15:06:28.951186","scores":{"nb":{"accuracy":0.533203125,"f1":0.5115197897627954,"accuracy_stderr":0.02834135129088174,"f1_stderr":0.025169476753664988,"main_score":0.533203125}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/Norwegian_parliament.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/Norwegian_parliament.json new file mode 100644 index 00000000..69139a4c --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/Norwegian_parliament.json @@ -0,0 +1 @@ +{"task_name":"Norwegian parliament","task_description":"Norwegian parliament speeches annotated for sentiment","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T15:08:13.649939","scores":{"nb":{"accuracy":0.5489999999999999,"f1":0.5453674845029315,"ap":0.5285569811273876,"accuracy_stderr":0.03254014614458747,"f1_stderr":0.030701546555598105,"ap_stderr":0.019866513665152682,"main_score":0.5489999999999999}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/ScaLA.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/ScaLA.json new file mode 100644 index 00000000..1e7a7a85 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/ScaLA.json @@ -0,0 +1 @@ +{"task_name":"ScaLA","task_description":"A linguistic acceptability task for Danish, Norwegian Bokmål Norwegian Nynorsk and Swedish.","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T15:04:40.552524","scores":{"da":{"accuracy":0.5119140625,"f1":0.5085168551376079,"ap":0.5062645683139095,"accuracy_stderr":0.010568102913689661,"f1_stderr":0.010563136069460142,"ap_stderr":0.005630151802961199,"main_score":0.5119140625},"nb":{"accuracy":0.50458984375,"f1":0.5031476524406753,"ap":0.5023420749792255,"accuracy_stderr":0.005250783226082057,"f1_stderr":0.0060014319400411185,"ap_stderr":0.002665397117295337,"main_score":0.50458984375},"sv":{"accuracy":0.504296875,"f1":0.5005471043617412,"ap":0.5021863752506963,"accuracy_stderr":0.004521811290152431,"f1_stderr":0.006809911687341478,"ap_stderr":0.0022974211138394054,"main_score":0.504296875},"nn":{"accuracy":0.503515625,"f1":0.5022883466781493,"ap":0.5018299519994283,"accuracy_stderr":0.007827134730278858,"f1_stderr":0.008190197956806415,"ap_stderr":0.003945196230116476,"main_score":0.503515625}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/SweFAQ.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/SweFAQ.json new file mode 100644 index 00000000..b642eae5 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/SweFAQ.json @@ -0,0 +1 @@ +{"task_name":"SweFAQ","task_description":"A Swedish QA dataset derived from FAQ","task_version":"0.0.1","time_of_run":"2023-11-15T15:10:32.984384","scores":{"sv":{"ndcg_at_1":0.26121,"ndcg_at_3":0.35953,"ndcg_at_5":0.40727,"ndcg_at_10":0.44561,"ndcg_at_100":0.49949,"ndcg_at_1000":0.50878,"map_at_1":0.26121,"map_at_3":0.33496,"map_at_5":0.36166,"map_at_10":0.37705,"map_at_100":0.38711,"map_at_1000":0.3875,"recall_at_1":0.26121,"recall_at_3":0.4308,"recall_at_5":0.54581,"recall_at_10":0.66667,"recall_at_100":0.92982,"recall_at_1000":1.0,"precision_at_1":0.26121,"precision_at_3":0.1436,"precision_at_5":0.10916,"precision_at_10":0.06667,"precision_at_100":0.0093,"precision_at_1000":0.001,"mrr_at_1":0.26121,"mrr_at_3":0.33496,"mrr_at_5":0.36166,"mrr_at_10":0.37705,"mrr_at_100":0.38711,"mrr_at_1000":0.3875}},"main_score":"ndcg_at_10"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/SweReC.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/SweReC.json new file mode 100644 index 00000000..605157a3 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-medium-v1/SweReC.json @@ -0,0 +1 @@ +{"task_name":"SweReC","task_description":"A Swedish dataset for sentiment classification on review","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T15:09:35.791551","scores":{"sv":{"accuracy":0.674951171875,"f1":0.6114249519282305,"accuracy_stderr":0.03247789805155075,"f1_stderr":0.030870464076338994,"main_score":0.674951171875}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/Angry_Tweets.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/Angry_Tweets.json new file mode 100644 index 00000000..2e9ae7ab --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/Angry_Tweets.json @@ -0,0 +1 @@ +{"task_name":"Angry Tweets","task_description":"A sentiment dataset with 3 classes (positiv, negativ, neutral) for Danish tweets","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T14:32:47.288668","scores":{"da":{"accuracy":0.5191977077363897,"f1":0.5132158564983342,"accuracy_stderr":0.01898819852220906,"f1_stderr":0.015130671688156076,"main_score":0.5191977077363897}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/Bornholm_Parallel.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/Bornholm_Parallel.json new file mode 100644 index 00000000..4b4c5995 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/Bornholm_Parallel.json @@ -0,0 +1 @@ +{"task_name":"Bornholm Parallel","task_description":"Danish Bornholmsk Parallel Corpus. Bornholmsk is a Danish dialect spoken on the island of Bornholm, Denmark. Historically it is a part of east Danish which was also spoken in Scania and Halland, Sweden.","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T14:32:57.928189","scores":{"da":{"precision":0.38051666666666667,"recall":0.482,"f1":0.4082253968253968,"accuracy":0.482,"main_score":0.4082253968253968},"da-bornholm":{"precision":0.38051666666666667,"recall":0.482,"f1":0.4082253968253968,"accuracy":0.482,"main_score":0.4082253968253968}},"main_score":"f1"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/DKHate.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/DKHate.json new file mode 100644 index 00000000..c18fb2dd --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/DKHate.json @@ -0,0 +1 @@ +{"task_name":"DKHate","task_description":"Danish Tweets annotated for Hate Speech either being Offensive or not","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T14:33:08.751075","scores":{"da":{"accuracy":0.6,"f1":0.5012566596065436,"ap":0.16270307162079187,"accuracy_stderr":0.05644336798269208,"f1_stderr":0.04805561200068095,"ap_stderr":0.02698227209450767,"main_score":0.6}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/DaLAJ.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/DaLAJ.json new file mode 100644 index 00000000..b4b8c5af --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/DaLAJ.json @@ -0,0 +1 @@ +{"task_name":"DaLAJ","task_description":"A Swedish dataset for linguistic acceptability. Available as a part of Superlim.","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T14:37:31.266277","scores":{"sv":{"accuracy":0.5001126126126126,"f1":0.4982926810519622,"ap":0.5001043315152114,"accuracy_stderr":0.006931852881311886,"f1_stderr":0.007320171394310959,"ap_stderr":0.003506539995014823,"main_score":0.5001126126126126}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/Da_Political_Comments.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/Da_Political_Comments.json new file mode 100644 index 00000000..a52c727a --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/Da_Political_Comments.json @@ -0,0 +1 @@ +{"task_name":"Da Political Comments","task_description":"A dataset of Danish political comments rated for sentiment","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T14:33:20.413257","scores":{"da":{"accuracy":0.35249722530521643,"f1":0.3187350040385473,"accuracy_stderr":0.02828752002004102,"f1_stderr":0.01838540638376595,"main_score":0.35249722530521643}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/LCC.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/LCC.json new file mode 100644 index 00000000..1640aa8e --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/LCC.json @@ -0,0 +1 @@ +{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T14:32:53.023627","scores":{"da":{"accuracy":0.5853333333333333,"f1":0.5647784502599756,"accuracy_stderr":0.034357596604600305,"f1_stderr":0.03239486829371719,"main_score":0.5853333333333333}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/Language_Identification.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/Language_Identification.json new file mode 100644 index 00000000..e3cb6bc9 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/Language_Identification.json @@ -0,0 +1 @@ +{"task_name":"Language Identification","task_description":"A dataset for Nordic language identification.","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T14:36:09.076881","scores":{"da":{"accuracy":0.505,"f1":0.5010970951703367,"accuracy_stderr":0.01820317432635185,"f1_stderr":0.01838238234795185,"main_score":0.505},"sv":{"accuracy":0.505,"f1":0.5010970951703367,"accuracy_stderr":0.01820317432635185,"f1_stderr":0.01838238234795185,"main_score":0.505},"nb":{"accuracy":0.505,"f1":0.5010970951703367,"accuracy_stderr":0.01820317432635185,"f1_stderr":0.01838238234795185,"main_score":0.505},"nn":{"accuracy":0.505,"f1":0.5010970951703367,"accuracy_stderr":0.01820317432635185,"f1_stderr":0.01838238234795185,"main_score":0.505},"is":{"accuracy":0.505,"f1":0.5010970951703367,"accuracy_stderr":0.01820317432635185,"f1_stderr":0.01838238234795185,"main_score":0.505},"fo":{"accuracy":0.505,"f1":0.5010970951703367,"accuracy_stderr":0.01820317432635185,"f1_stderr":0.01838238234795185,"main_score":0.505}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/Massive_Intent.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/Massive_Intent.json new file mode 100644 index 00000000..8b00e33b --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/Massive_Intent.json @@ -0,0 +1 @@ +{"task_name":"Massive Intent","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T14:34:18.888326","scores":{"da":{"accuracy":0.47921990585070606,"f1":0.4433799121272618,"accuracy_stderr":0.01299569315850024,"f1_stderr":0.009713059818510427,"main_score":0.47921990585070606},"nb":{"accuracy":0.4654337592468056,"f1":0.4367740153029044,"accuracy_stderr":0.01696154401725224,"f1_stderr":0.013125506633203768,"main_score":0.4654337592468056},"sv":{"accuracy":0.4917619367854741,"f1":0.4589453341490243,"accuracy_stderr":0.010569355291491143,"f1_stderr":0.011404353256540482,"main_score":0.4917619367854741}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/Massive_Scenario.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/Massive_Scenario.json new file mode 100644 index 00000000..323f4628 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/Massive_Scenario.json @@ -0,0 +1 @@ +{"task_name":"Massive Scenario","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T14:34:55.761982","scores":{"da":{"accuracy":0.5294889038332213,"f1":0.5103888060203308,"accuracy_stderr":0.022866831076512568,"f1_stderr":0.01899387062783642,"main_score":0.5294889038332213},"nb":{"accuracy":0.5093476798924007,"f1":0.49128580669338173,"accuracy_stderr":0.016608583306046667,"f1_stderr":0.016106935036285203,"main_score":0.5093476798924007},"sv":{"accuracy":0.5424008069939475,"f1":0.5210134602991238,"accuracy_stderr":0.015040789827084247,"f1_stderr":0.013810704711262651,"main_score":0.5424008069939475}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/NoReC.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/NoReC.json new file mode 100644 index 00000000..983dc79c --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/NoReC.json @@ -0,0 +1 @@ +{"task_name":"NoReC","task_description":"A Norwegian dataset for sentiment classification on review","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T14:36:21.154187","scores":{"nb":{"accuracy":0.5064453125,"f1":0.4843662060314601,"accuracy_stderr":0.035522452547786805,"f1_stderr":0.032611423378075995,"main_score":0.5064453125}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/Norwegian_parliament.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/Norwegian_parliament.json new file mode 100644 index 00000000..12f41b6a --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/Norwegian_parliament.json @@ -0,0 +1 @@ +{"task_name":"Norwegian parliament","task_description":"Norwegian parliament speeches annotated for sentiment","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T14:36:55.055365","scores":{"nb":{"accuracy":0.552,"f1":0.546225856420494,"ap":0.5295946431869756,"accuracy_stderr":0.022389233523677814,"f1_stderr":0.02402494059208828,"ap_stderr":0.014210892331042854,"main_score":0.552}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/ScaLA.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/ScaLA.json new file mode 100644 index 00000000..4a76d8d8 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/ScaLA.json @@ -0,0 +1 @@ +{"task_name":"ScaLA","task_description":"A linguistic acceptability task for Danish, Norwegian Bokmål Norwegian Nynorsk and Swedish.","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T14:35:48.418422","scores":{"da":{"accuracy":0.51357421875,"f1":0.5105653567181552,"ap":0.5072806484419583,"accuracy_stderr":0.01425212589936558,"f1_stderr":0.013073141018719283,"ap_stderr":0.007738063416857466,"main_score":0.51357421875},"nb":{"accuracy":0.50947265625,"f1":0.5073328212188307,"ap":0.5048819626581496,"accuracy_stderr":0.007727195412968104,"f1_stderr":0.008073511669183759,"ap_stderr":0.004022142466037445,"main_score":0.50947265625},"sv":{"accuracy":0.510546875,"f1":0.5060922966569086,"ap":0.5054898590217124,"accuracy_stderr":0.010879362086500277,"f1_stderr":0.014482042069988042,"ap_stderr":0.005541867643015626,"main_score":0.510546875},"nn":{"accuracy":0.51328125,"f1":0.511953080357905,"ap":0.5069194924023904,"accuracy_stderr":0.009126549994106309,"f1_stderr":0.008926499356323798,"ap_stderr":0.004804528202522355,"main_score":0.51328125}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/SweFAQ.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/SweFAQ.json new file mode 100644 index 00000000..71e8c532 --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/SweFAQ.json @@ -0,0 +1 @@ +{"task_name":"SweFAQ","task_description":"A Swedish QA dataset derived from FAQ","task_version":"0.0.1","time_of_run":"2023-11-15T14:37:39.198535","scores":{"sv":{"ndcg_at_1":0.16764,"ndcg_at_3":0.24178,"ndcg_at_5":0.27676,"ndcg_at_10":0.31399,"ndcg_at_100":0.38971,"ndcg_at_1000":0.40677,"map_at_1":0.16764,"map_at_3":0.22125,"map_at_5":0.24045,"map_at_10":0.25584,"map_at_100":0.26995,"map_at_1000":0.27072,"recall_at_1":0.16764,"recall_at_3":0.30214,"recall_at_5":0.38791,"recall_at_10":0.50292,"recall_at_100":0.87329,"recall_at_1000":1.0,"precision_at_1":0.16764,"precision_at_3":0.10071,"precision_at_5":0.07758,"precision_at_10":0.05029,"precision_at_100":0.00873,"precision_at_1000":0.001,"mrr_at_1":0.16764,"mrr_at_3":0.22125,"mrr_at_5":0.24045,"mrr_at_10":0.25584,"mrr_at_100":0.26995,"mrr_at_1000":0.27072}},"main_score":"ndcg_at_10"} \ No newline at end of file diff --git a/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/SweReC.json b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/SweReC.json new file mode 100644 index 00000000..abc44e7a --- /dev/null +++ b/src/seb/cache/KennethEnevoldsen__dfm-sentence-encoder-small-v1/SweReC.json @@ -0,0 +1 @@ +{"task_name":"SweReC","task_description":"A Swedish dataset for sentiment classification on review","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T14:37:21.852283","scores":{"sv":{"accuracy":0.6634765625,"f1":0.6013475160327366,"accuracy_stderr":0.020989052234931978,"f1_stderr":0.01742307859040556,"main_score":0.6634765625}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/NbAiLab__nb-bert-base/Angry_Tweets.json b/src/seb/cache/NbAiLab__nb-bert-base/Angry_Tweets.json new file mode 100644 index 00000000..595f3241 --- /dev/null +++ b/src/seb/cache/NbAiLab__nb-bert-base/Angry_Tweets.json @@ -0,0 +1 @@ +{"task_name":"Angry Tweets","task_description":"A sentiment dataset with 3 classes (positiv, negativ, neutral) for Danish tweets","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T23:11:17.360141","scores":{"da":{"accuracy":0.5213944603629418,"f1":0.5141558383682454,"accuracy_stderr":0.022756681979440658,"f1_stderr":0.02056420429446079,"main_score":0.5213944603629418}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/NbAiLab__nb-bert-base/Bornholm_Parallel.json b/src/seb/cache/NbAiLab__nb-bert-base/Bornholm_Parallel.json new file mode 100644 index 00000000..fbb5178c --- /dev/null +++ b/src/seb/cache/NbAiLab__nb-bert-base/Bornholm_Parallel.json @@ -0,0 +1 @@ +{"task_name":"Bornholm Parallel","task_description":"Danish Bornholmsk Parallel Corpus. Bornholmsk is a Danish dialect spoken on the island of Bornholm, Denmark. Historically it is a part of east Danish which was also spoken in Scania and Halland, Sweden.","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T23:11:44.433525","scores":{"da":{"precision":0.09182183886421494,"recall":0.132,"f1":0.09876539201539201,"accuracy":0.132,"main_score":0.09876539201539201},"da-bornholm":{"precision":0.09182183886421494,"recall":0.132,"f1":0.09876539201539201,"accuracy":0.132,"main_score":0.09876539201539201}},"main_score":"f1"} \ No newline at end of file diff --git a/src/seb/cache/NbAiLab__nb-bert-base/DKHate.json b/src/seb/cache/NbAiLab__nb-bert-base/DKHate.json new file mode 100644 index 00000000..a5209563 --- /dev/null +++ b/src/seb/cache/NbAiLab__nb-bert-base/DKHate.json @@ -0,0 +1 @@ +{"task_name":"DKHate","task_description":"Danish Tweets annotated for Hate Speech either being Offensive or not","task_version":"1.0.3.dev0","time_of_run":"2023-07-30T14:00:57.230216","scores":{"da":{"accuracy":0.6173252279635258,"f1":0.5095902843392796,"ap":0.899947008300422,"accuracy_stderr":0.0665014752317482,"f1_stderr":0.048941549826110015,"ap_stderr":0.01277243357239159,"main_score":0.6173252279635258}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/NbAiLab__nb-bert-base/DaLAJ.json b/src/seb/cache/NbAiLab__nb-bert-base/DaLAJ.json new file mode 100644 index 00000000..27430c60 --- /dev/null +++ b/src/seb/cache/NbAiLab__nb-bert-base/DaLAJ.json @@ -0,0 +1 @@ +{"task_name":"DaLAJ","task_description":"A Swedish dataset for linguistic accebtablity. Available as a part of Superlim.","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T23:29:29.88681","scores":{"sv":{"accuracy":0.5032657657657659,"f1":0.5014445068022071,"ap":0.5017048517846604,"accuracy_stderr":0.007678330600779626,"f1_stderr":0.007472513590423133,"ap_stderr":0.0037748953104772675,"main_score":0.5032657657657659}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/NbAiLab__nb-bert-base/Da_Political_Comments.json b/src/seb/cache/NbAiLab__nb-bert-base/Da_Political_Comments.json new file mode 100644 index 00000000..791f5fbc --- /dev/null +++ b/src/seb/cache/NbAiLab__nb-bert-base/Da_Political_Comments.json @@ -0,0 +1 @@ +{"task_name":"Da Political Comments","task_description":"A dataset of Danish political comments rated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T23:13:07.00127","scores":{"da":{"accuracy":0.3428412874583796,"f1":0.30895218713067385,"accuracy_stderr":0.03539170782524705,"f1_stderr":0.025817419621321905,"main_score":0.3428412874583796}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/NbAiLab__nb-bert-base/LCC.json b/src/seb/cache/NbAiLab__nb-bert-base/LCC.json new file mode 100644 index 00000000..fa82f72b --- /dev/null +++ b/src/seb/cache/NbAiLab__nb-bert-base/LCC.json @@ -0,0 +1 @@ +{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T23:11:32.822733","scores":{"da":{"accuracy":0.514,"f1":0.4979878836449485,"accuracy_stderr":0.0439646322501268,"f1_stderr":0.03742084573864153,"main_score":0.514}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/NbAiLab__nb-bert-base/Language_Identification.json b/src/seb/cache/NbAiLab__nb-bert-base/Language_Identification.json new file mode 100644 index 00000000..6f3f11e1 --- /dev/null +++ b/src/seb/cache/NbAiLab__nb-bert-base/Language_Identification.json @@ -0,0 +1 @@ +{"task_name":"Language Identification","task_description":"A dataset for Nordic language identification.","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T23:22:18.236776","scores":{"da":{"accuracy":0.8468666666666668,"f1":0.8471019434920242,"accuracy_stderr":0.0099802026254202,"f1_stderr":0.009793587373515725,"main_score":0.8468666666666668},"sv":{"accuracy":0.8468666666666668,"f1":0.8471019434920242,"accuracy_stderr":0.0099802026254202,"f1_stderr":0.009793587373515725,"main_score":0.8468666666666668},"nb":{"accuracy":0.8468666666666668,"f1":0.8471019434920242,"accuracy_stderr":0.0099802026254202,"f1_stderr":0.009793587373515725,"main_score":0.8468666666666668},"nn":{"accuracy":0.8468666666666668,"f1":0.8471019434920242,"accuracy_stderr":0.0099802026254202,"f1_stderr":0.009793587373515725,"main_score":0.8468666666666668},"is":{"accuracy":0.8468666666666668,"f1":0.8471019434920242,"accuracy_stderr":0.0099802026254202,"f1_stderr":0.009793587373515725,"main_score":0.8468666666666668},"fo":{"accuracy":0.8468666666666668,"f1":0.8471019434920242,"accuracy_stderr":0.0099802026254202,"f1_stderr":0.009793587373515725,"main_score":0.8468666666666668}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/NbAiLab__nb-bert-base/Massive_Intent.json b/src/seb/cache/NbAiLab__nb-bert-base/Massive_Intent.json new file mode 100644 index 00000000..faee4266 --- /dev/null +++ b/src/seb/cache/NbAiLab__nb-bert-base/Massive_Intent.json @@ -0,0 +1 @@ +{"task_name":"Massive Intent","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T23:16:52.223325","scores":{"da":{"accuracy":0.5669468728984534,"f1":0.5323365421837685,"accuracy_stderr":0.011940762950616142,"f1_stderr":0.011739919936215938,"main_score":0.5669468728984534},"nb":{"accuracy":0.6066913248150639,"f1":0.5711065956384204,"accuracy_stderr":0.01495379146579084,"f1_stderr":0.010880838723287366,"main_score":0.6066913248150639},"sv":{"accuracy":0.5389038332212508,"f1":0.5056225089127879,"accuracy_stderr":0.011694501665842556,"f1_stderr":0.006644765457372844,"main_score":0.5389038332212508}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/NbAiLab__nb-bert-base/Massive_Scenario.json b/src/seb/cache/NbAiLab__nb-bert-base/Massive_Scenario.json new file mode 100644 index 00000000..31798169 --- /dev/null +++ b/src/seb/cache/NbAiLab__nb-bert-base/Massive_Scenario.json @@ -0,0 +1 @@ +{"task_name":"Massive Scenario","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T23:18:39.406556","scores":{"da":{"accuracy":0.6192669804976463,"f1":0.6079566761576601,"accuracy_stderr":0.020088387922951534,"f1_stderr":0.018480298509820116,"main_score":0.6192669804976463},"nb":{"accuracy":0.6731002017484868,"f1":0.6660561469371957,"accuracy_stderr":0.016653451974664585,"f1_stderr":0.0156676777743156,"main_score":0.6731002017484868},"sv":{"accuracy":0.5537323470073974,"f1":0.5385609530945743,"accuracy_stderr":0.014555127617509366,"f1_stderr":0.013419274514718067,"main_score":0.5537323470073974}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/NbAiLab__nb-bert-base/NoReC.json b/src/seb/cache/NbAiLab__nb-bert-base/NoReC.json new file mode 100644 index 00000000..bf7de274 --- /dev/null +++ b/src/seb/cache/NbAiLab__nb-bert-base/NoReC.json @@ -0,0 +1 @@ +{"task_name":"NoReC","task_description":"A Norwegian dataset for sentiment classification on review","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T23:23:01.791703","scores":{"nb":{"accuracy":0.51318359375,"f1":0.49683955399048774,"accuracy_stderr":0.020609865126374402,"f1_stderr":0.022113970692863742,"main_score":0.51318359375}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/NbAiLab__nb-bert-base/Norwegian_parliament.json b/src/seb/cache/NbAiLab__nb-bert-base/Norwegian_parliament.json new file mode 100644 index 00000000..0e79e152 --- /dev/null +++ b/src/seb/cache/NbAiLab__nb-bert-base/Norwegian_parliament.json @@ -0,0 +1 @@ +{"task_name":"Norwegian parliament","task_description":"Norwegian parliament speeches annotated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T23:26:47.149543","scores":{"nb":{"accuracy":0.5740833333333334,"f1":0.5714626096323772,"ap":0.5446707978776629,"accuracy_stderr":0.0405867555025309,"f1_stderr":0.04141802285608337,"ap_stderr":0.02662000536137305,"main_score":0.5740833333333334}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/NbAiLab__nb-bert-base/ScaLA.json b/src/seb/cache/NbAiLab__nb-bert-base/ScaLA.json new file mode 100644 index 00000000..8119cfc1 --- /dev/null +++ b/src/seb/cache/NbAiLab__nb-bert-base/ScaLA.json @@ -0,0 +1 @@ +{"task_name":"ScaLA","task_description":"A linguistic acceptability task for Danish, Norwegian Bokmål Norwegian Nynorsk and Swedish.","task_version":"1.1.1.dev0","time_of_run":"2023-09-13T15:47:37.201993","scores":{"da":{"accuracy":0.5798828125,"f1":0.5728864530560204,"ap":0.5467524352522081,"accuracy_stderr":0.020793214953102792,"f1_stderr":0.025517024495058493,"ap_stderr":0.013834406657981496,"main_score":0.5798828125},"nb":{"accuracy":0.622509765625,"f1":0.6206591229885107,"ap":0.5768603526239042,"accuracy_stderr":0.019686213860054425,"f1_stderr":0.019748952426219338,"ap_stderr":0.01584335022223278,"main_score":0.622509765625},"sv":{"accuracy":0.557666015625,"f1":0.5552202911007053,"ap":0.5324043067146238,"accuracy_stderr":0.016469560809507233,"f1_stderr":0.018174254415336984,"ap_stderr":0.009872562820345834,"main_score":0.557666015625},"nn":{"accuracy":0.58994140625,"f1":0.5875364313281968,"ap":0.553731453333792,"accuracy_stderr":0.02931786357553683,"f1_stderr":0.028923778042822167,"ap_stderr":0.019299275776242898,"main_score":0.58994140625}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/NbAiLab__nb-bert-base/SweFAQ.json b/src/seb/cache/NbAiLab__nb-bert-base/SweFAQ.json new file mode 100644 index 00000000..81259e9c --- /dev/null +++ b/src/seb/cache/NbAiLab__nb-bert-base/SweFAQ.json @@ -0,0 +1 @@ +{"task_name":"SweFAQ","task_description":"A Swedish QA dataset derived from FAQ","task_version":"0.0.1","time_of_run":"2023-09-04T19:37:20.894304","scores":{"sv":{"ndcg_at_1":0.15789,"ndcg_at_3":0.21526,"ndcg_at_5":0.23933,"ndcg_at_10":0.25645,"ndcg_at_100":0.31387,"ndcg_at_1000":0.35666,"map_at_1":0.15789,"map_at_3":0.20078,"map_at_5":0.21413,"map_at_10":0.22125,"map_at_100":0.23141,"map_at_1000":0.23282,"recall_at_1":0.15789,"recall_at_3":0.25731,"recall_at_5":0.31579,"recall_at_10":0.36842,"recall_at_100":0.65497,"recall_at_1000":1.0,"precision_at_1":0.15789,"precision_at_3":0.08577,"precision_at_5":0.06316,"precision_at_10":0.03684,"precision_at_100":0.00655,"precision_at_1000":0.001,"mrr_at_1":0.15789,"mrr_at_3":0.20078,"mrr_at_5":0.21413,"mrr_at_10":0.22125,"mrr_at_100":0.23141,"mrr_at_1000":0.23282}},"main_score":"ndcg_at_10"} \ No newline at end of file diff --git a/src/seb/cache/NbAiLab__nb-bert-base/SweReC.json b/src/seb/cache/NbAiLab__nb-bert-base/SweReC.json new file mode 100644 index 00000000..8c6bb22e --- /dev/null +++ b/src/seb/cache/NbAiLab__nb-bert-base/SweReC.json @@ -0,0 +1 @@ +{"task_name":"SweReC","task_description":"A Swedish dataset for sentiment classification on review","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T23:29:05.209093","scores":{"sv":{"accuracy":0.639306640625,"f1":0.5771097720866216,"accuracy_stderr":0.01927697023843442,"f1_stderr":0.018947257391601732,"main_score":0.639306640625}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/NbAiLab__nb-bert-large/Angry_Tweets.json b/src/seb/cache/NbAiLab__nb-bert-large/Angry_Tweets.json new file mode 100644 index 00000000..4e6df86e --- /dev/null +++ b/src/seb/cache/NbAiLab__nb-bert-large/Angry_Tweets.json @@ -0,0 +1 @@ +{"task_name":"Angry Tweets","task_description":"A sentiment dataset with 3 classes (positiv, negativ, neutral) for Danish tweets","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T22:01:29.580994","scores":{"da":{"accuracy":0.5213944603629418,"f1":0.5083316170118218,"accuracy_stderr":0.028565449549892194,"f1_stderr":0.024100025569820143,"main_score":0.5213944603629418}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/NbAiLab__nb-bert-large/Bornholm_Parallel.json b/src/seb/cache/NbAiLab__nb-bert-large/Bornholm_Parallel.json new file mode 100644 index 00000000..fc7e6db5 --- /dev/null +++ b/src/seb/cache/NbAiLab__nb-bert-large/Bornholm_Parallel.json @@ -0,0 +1 @@ +{"task_name":"Bornholm Parallel","task_description":"Danish Bornholmsk Parallel Corpus. Bornholmsk is a Danish dialect spoken on the island of Bornholm, Denmark. Historically it is a part of east Danish which was also spoken in Scania and Halland, Sweden.","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T22:02:32.91248","scores":{"da":{"precision":0.042301087981785654,"recall":0.064,"f1":0.04530713061982502,"accuracy":0.064,"main_score":0.04530713061982502},"da-bornholm":{"precision":0.042301087981785654,"recall":0.064,"f1":0.04530713061982502,"accuracy":0.064,"main_score":0.04530713061982502}},"main_score":"f1"} \ No newline at end of file diff --git a/src/seb/cache/NbAiLab__nb-bert-large/DKHate.json b/src/seb/cache/NbAiLab__nb-bert-large/DKHate.json new file mode 100644 index 00000000..cea68d17 --- /dev/null +++ b/src/seb/cache/NbAiLab__nb-bert-large/DKHate.json @@ -0,0 +1 @@ +{"task_name":"DKHate","task_description":"Danish Tweets annotated for Hate Speech either being Offensive or not","task_version":"1.0.3.dev0","time_of_run":"2023-07-30T14:00:12.038661","scores":{"da":{"accuracy":0.6212765957446809,"f1":0.518060959729912,"ap":0.9047845814803217,"accuracy_stderr":0.06244259010903199,"f1_stderr":0.046140711086432924,"ap_stderr":0.012850529711304355,"main_score":0.6212765957446809}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/NbAiLab__nb-bert-large/DaLAJ.json b/src/seb/cache/NbAiLab__nb-bert-large/DaLAJ.json new file mode 100644 index 00000000..980815ca --- /dev/null +++ b/src/seb/cache/NbAiLab__nb-bert-large/DaLAJ.json @@ -0,0 +1 @@ +{"task_name":"DaLAJ","task_description":"A Swedish dataset for linguistic accebtablity. Available as a part of Superlim.","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T23:10:36.869608","scores":{"sv":{"accuracy":0.509009009009009,"f1":0.5063862294190298,"ap":0.5050366322658202,"accuracy_stderr":0.01911762869631622,"f1_stderr":0.018863739406908855,"ap_stderr":0.010422086596047622,"main_score":0.509009009009009}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/NbAiLab__nb-bert-large/Da_Political_Comments.json b/src/seb/cache/NbAiLab__nb-bert-large/Da_Political_Comments.json new file mode 100644 index 00000000..7a3f640f --- /dev/null +++ b/src/seb/cache/NbAiLab__nb-bert-large/Da_Political_Comments.json @@ -0,0 +1 @@ +{"task_name":"Da Political Comments","task_description":"A dataset of Danish political comments rated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T22:05:52.56754","scores":{"da":{"accuracy":0.35571587125416204,"f1":0.32757824470546887,"accuracy_stderr":0.03438114442273317,"f1_stderr":0.02413341398437584,"main_score":0.35571587125416204}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/NbAiLab__nb-bert-large/LCC.json b/src/seb/cache/NbAiLab__nb-bert-large/LCC.json new file mode 100644 index 00000000..48dd562b --- /dev/null +++ b/src/seb/cache/NbAiLab__nb-bert-large/LCC.json @@ -0,0 +1 @@ +{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T22:02:05.116923","scores":{"da":{"accuracy":0.5626666666666666,"f1":0.5531944743435006,"accuracy_stderr":0.03968766950969924,"f1_stderr":0.03592239064328234,"main_score":0.5626666666666666}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/NbAiLab__nb-bert-large/Language_Identification.json b/src/seb/cache/NbAiLab__nb-bert-large/Language_Identification.json new file mode 100644 index 00000000..f358b8d0 --- /dev/null +++ b/src/seb/cache/NbAiLab__nb-bert-large/Language_Identification.json @@ -0,0 +1 @@ +{"task_name":"Language Identification","task_description":"A dataset for Nordic language identification.","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T22:27:38.28355","scores":{"da":{"accuracy":0.8527333333333333,"f1":0.8524516422189619,"accuracy_stderr":0.008871677781945555,"f1_stderr":0.008822889805474025,"main_score":0.8527333333333333},"sv":{"accuracy":0.8527333333333333,"f1":0.8524516422189619,"accuracy_stderr":0.008871677781945555,"f1_stderr":0.008822889805474025,"main_score":0.8527333333333333},"nb":{"accuracy":0.8527333333333333,"f1":0.8524516422189619,"accuracy_stderr":0.008871677781945555,"f1_stderr":0.008822889805474025,"main_score":0.8527333333333333},"nn":{"accuracy":0.8527333333333333,"f1":0.8524516422189619,"accuracy_stderr":0.008871677781945555,"f1_stderr":0.008822889805474025,"main_score":0.8527333333333333},"is":{"accuracy":0.8527333333333333,"f1":0.8524516422189619,"accuracy_stderr":0.008871677781945555,"f1_stderr":0.008822889805474025,"main_score":0.8527333333333333},"fo":{"accuracy":0.8527333333333333,"f1":0.8524516422189619,"accuracy_stderr":0.008871677781945555,"f1_stderr":0.008822889805474025,"main_score":0.8527333333333333}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/NbAiLab__nb-bert-large/Massive_Intent.json b/src/seb/cache/NbAiLab__nb-bert-large/Massive_Intent.json new file mode 100644 index 00000000..fe326ff4 --- /dev/null +++ b/src/seb/cache/NbAiLab__nb-bert-large/Massive_Intent.json @@ -0,0 +1 @@ +{"task_name":"Massive Intent","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T22:13:47.616011","scores":{"da":{"accuracy":0.5703093476798924,"f1":0.5240749176338441,"accuracy_stderr":0.01205103643759604,"f1_stderr":0.010180325034852665,"main_score":0.5703093476798924},"nb":{"accuracy":0.6268325487558843,"f1":0.5873743585657378,"accuracy_stderr":0.013013950307628905,"f1_stderr":0.008665088175154967,"main_score":0.6268325487558843},"sv":{"accuracy":0.5502353732347007,"f1":0.5117239842319402,"accuracy_stderr":0.007667917129501097,"f1_stderr":0.006629028437789528,"main_score":0.5502353732347007}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/NbAiLab__nb-bert-large/Massive_Scenario.json b/src/seb/cache/NbAiLab__nb-bert-large/Massive_Scenario.json new file mode 100644 index 00000000..ebbe80c3 --- /dev/null +++ b/src/seb/cache/NbAiLab__nb-bert-large/Massive_Scenario.json @@ -0,0 +1 @@ +{"task_name":"Massive Scenario","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T22:18:25.646096","scores":{"da":{"accuracy":0.6043039677202422,"f1":0.5929666103807454,"accuracy_stderr":0.028896083939013734,"f1_stderr":0.024498119074388865,"main_score":0.6043039677202422},"nb":{"accuracy":0.6744451916610626,"f1":0.6615213677979109,"accuracy_stderr":0.018218160191781674,"f1_stderr":0.017939032267758108,"main_score":0.6744451916610626},"sv":{"accuracy":0.5711835911230666,"f1":0.5635949260739122,"accuracy_stderr":0.013201343205764519,"f1_stderr":0.012873335207392875,"main_score":0.5711835911230666}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/NbAiLab__nb-bert-large/NoReC.json b/src/seb/cache/NbAiLab__nb-bert-large/NoReC.json new file mode 100644 index 00000000..37f46a52 --- /dev/null +++ b/src/seb/cache/NbAiLab__nb-bert-large/NoReC.json @@ -0,0 +1 @@ +{"task_name":"NoReC","task_description":"A Norwegian dataset for sentiment classification on review","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T22:29:03.338059","scores":{"nb":{"accuracy":0.55458984375,"f1":0.5340063774183045,"accuracy_stderr":0.029595483425011938,"f1_stderr":0.03013105829172929,"main_score":0.55458984375}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/NbAiLab__nb-bert-large/Norwegian_parliament.json b/src/seb/cache/NbAiLab__nb-bert-large/Norwegian_parliament.json new file mode 100644 index 00000000..cd3e8171 --- /dev/null +++ b/src/seb/cache/NbAiLab__nb-bert-large/Norwegian_parliament.json @@ -0,0 +1 @@ +{"task_name":"Norwegian parliament","task_description":"Norwegian parliament speeches annotated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T22:57:35.743766","scores":{"nb":{"accuracy":0.6258333333333332,"f1":0.6213354614690039,"ap":0.5803720024987109,"accuracy_stderr":0.03899073964132738,"f1_stderr":0.041873284893048375,"ap_stderr":0.03170063435583393,"main_score":0.6258333333333332}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/NbAiLab__nb-bert-large/Scala.json b/src/seb/cache/NbAiLab__nb-bert-large/Scala.json new file mode 100644 index 00000000..cfec63c6 --- /dev/null +++ b/src/seb/cache/NbAiLab__nb-bert-large/Scala.json @@ -0,0 +1,45 @@ +{ + "task_name": "ScaLA", + "task_description": "A linguistic acceptability task for Danish, Norwegian Bokmål Norwegian Nynorsk and Swedish.", + "task_version": "1.0.3.dev0", + "time_of_run": "2023-07-27T22:24:17.761314", + "scores": { + "da": { + "accuracy": 0.628466796875, + "f1": 0.6238542066339783, + "ap": 0.5842781884511643, + "accuracy_stderr": 0.027491968889551587, + "f1_stderr": 0.03043629054817742, + "ap_stderr": 0.020819715140096985, + "main_score": 0.628466796875 + }, + "nb": { + "accuracy": 0.669677734375, + "f1": 0.667487547472665, + "ap": 0.6143789601460962, + "accuracy_stderr": 0.018740397219524366, + "f1_stderr": 0.019748617254076787, + "ap_stderr": 0.013558678026780121, + "main_score": 0.669677734375 + }, + "sv": { + "accuracy": 0.62099609375, + "f1": 0.6188218236384682, + "ap": 0.5753698942702191, + "accuracy_stderr": 0.026309072239523997, + "f1_stderr": 0.027920711529225564, + "ap_stderr": 0.020384598111657177, + "main_score": 0.62099609375 + }, + "nn": { + "accuracy": 0.64775390625, + "f1": 0.6460266516045423, + "ap": 0.5987709855704623, + "accuracy_stderr": 0.04226733530360049, + "f1_stderr": 0.04334958966713283, + "ap_stderr": 0.03405750688686972, + "main_score": 0.64775390625 + } + }, + "main_score": "accuracy" +} \ No newline at end of file diff --git a/src/seb/cache/NbAiLab__nb-bert-large/SweFAQ.json b/src/seb/cache/NbAiLab__nb-bert-large/SweFAQ.json new file mode 100644 index 00000000..07c15fdb --- /dev/null +++ b/src/seb/cache/NbAiLab__nb-bert-large/SweFAQ.json @@ -0,0 +1 @@ +{"task_name":"SweFAQ","task_description":"A Swedish QA dataset derived from FAQ","task_version":"0.0.1","time_of_run":"2023-09-04T19:36:36.621194","scores":{"sv":{"ndcg_at_1":0.18129,"ndcg_at_3":0.24875,"ndcg_at_5":0.27601,"ndcg_at_10":0.30685,"ndcg_at_100":0.37951,"ndcg_at_1000":0.40403,"map_at_1":0.18129,"map_at_3":0.23229,"map_at_5":0.2474,"map_at_10":0.26047,"map_at_100":0.27392,"map_at_1000":0.27489,"recall_at_1":0.18129,"recall_at_3":0.2963,"recall_at_5":0.36257,"recall_at_10":0.45614,"recall_at_100":0.81092,"recall_at_1000":1.0,"precision_at_1":0.18129,"precision_at_3":0.09877,"precision_at_5":0.07251,"precision_at_10":0.04561,"precision_at_100":0.00811,"precision_at_1000":0.001,"mrr_at_1":0.18129,"mrr_at_3":0.23229,"mrr_at_5":0.2474,"mrr_at_10":0.26047,"mrr_at_100":0.27392,"mrr_at_1000":0.27489}},"main_score":"ndcg_at_10"} \ No newline at end of file diff --git a/src/seb/cache/NbAiLab__nb-bert-large/SweReC.json b/src/seb/cache/NbAiLab__nb-bert-large/SweReC.json new file mode 100644 index 00000000..94a6a8d1 --- /dev/null +++ b/src/seb/cache/NbAiLab__nb-bert-large/SweReC.json @@ -0,0 +1 @@ +{"task_name":"SweReC","task_description":"A Swedish dataset for sentiment classification on review","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T23:09:29.056119","scores":{"sv":{"accuracy":0.677001953125,"f1":0.6103765260432408,"accuracy_stderr":0.031463087282697526,"f1_stderr":0.024909439520548547,"main_score":0.677001953125}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/chcaa__dfm-encoder-large-v1/Angry_Tweets.json b/src/seb/cache/chcaa__dfm-encoder-large-v1/Angry_Tweets.json new file mode 100644 index 00000000..73f524e9 --- /dev/null +++ b/src/seb/cache/chcaa__dfm-encoder-large-v1/Angry_Tweets.json @@ -0,0 +1 @@ +{"task_name":"Angry Tweets","task_description":"A sentiment dataset with 3 classes (positiv, negativ, neutral) for Danish tweets","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T21:13:43.267024","scores":{"da":{"accuracy":0.5380133715377269,"f1":0.5278713940222668,"accuracy_stderr":0.028188040634955128,"f1_stderr":0.025453796294134995,"main_score":0.5380133715377269}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/chcaa__dfm-encoder-large-v1/Bornholm_Parallel.json b/src/seb/cache/chcaa__dfm-encoder-large-v1/Bornholm_Parallel.json new file mode 100644 index 00000000..2630f3a1 --- /dev/null +++ b/src/seb/cache/chcaa__dfm-encoder-large-v1/Bornholm_Parallel.json @@ -0,0 +1 @@ +{"task_name":"Bornholm Parallel","task_description":"Danish Bornholmsk Parallel Corpus. Bornholmsk is a Danish dialect spoken on the island of Bornholm, Denmark. Historically it is a part of east Danish which was also spoken in Scania and Halland, Sweden.","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T21:14:46.620979","scores":{"da":{"precision":0.1041448634080987,"recall":0.166,"f1":0.11648071969709954,"accuracy":0.166,"main_score":0.11648071969709954},"da-bornholm":{"precision":0.1041448634080987,"recall":0.166,"f1":0.11648071969709954,"accuracy":0.166,"main_score":0.11648071969709954}},"main_score":"f1"} \ No newline at end of file diff --git a/src/seb/cache/chcaa__dfm-encoder-large-v1/DKHate.json b/src/seb/cache/chcaa__dfm-encoder-large-v1/DKHate.json new file mode 100644 index 00000000..cbd5c6f2 --- /dev/null +++ b/src/seb/cache/chcaa__dfm-encoder-large-v1/DKHate.json @@ -0,0 +1 @@ +{"task_name":"DKHate","task_description":"Danish Tweets annotated for Hate Speech either being Offensive or not","task_version":"1.0.3.dev0","time_of_run":"2023-07-30T13:58:14.096478","scores":{"da":{"accuracy":0.6009118541033435,"f1":0.4905126907046687,"ap":0.8945184860452674,"accuracy_stderr":0.06147111139432738,"f1_stderr":0.0307782133210074,"ap_stderr":0.009291158016442577,"main_score":0.6009118541033435}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/chcaa__dfm-encoder-large-v1/DaLAJ.json b/src/seb/cache/chcaa__dfm-encoder-large-v1/DaLAJ.json new file mode 100644 index 00000000..9b007f05 --- /dev/null +++ b/src/seb/cache/chcaa__dfm-encoder-large-v1/DaLAJ.json @@ -0,0 +1 @@ +{"task_name":"DaLAJ","task_description":"A Swedish dataset for linguistic accebtablity. Available as a part of Superlim.","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T21:59:47.651123","scores":{"sv":{"accuracy":0.5041666666666667,"f1":0.50277036118561,"ap":0.5022338415041723,"accuracy_stderr":0.011340375252504836,"f1_stderr":0.010877471827967751,"ap_stderr":0.005819630084571685,"main_score":0.5041666666666667}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/chcaa__dfm-encoder-large-v1/Da_Political_Comments.json b/src/seb/cache/chcaa__dfm-encoder-large-v1/Da_Political_Comments.json new file mode 100644 index 00000000..e6cdbc92 --- /dev/null +++ b/src/seb/cache/chcaa__dfm-encoder-large-v1/Da_Political_Comments.json @@ -0,0 +1 @@ +{"task_name":"Da Political Comments","task_description":"A dataset of Danish political comments rated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T21:18:02.308812","scores":{"da":{"accuracy":0.3714206437291898,"f1":0.33478650503406837,"accuracy_stderr":0.021452607942069716,"f1_stderr":0.017175596739954026,"main_score":0.3714206437291898}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/chcaa__dfm-encoder-large-v1/LCC.json b/src/seb/cache/chcaa__dfm-encoder-large-v1/LCC.json new file mode 100644 index 00000000..aaafa8bc --- /dev/null +++ b/src/seb/cache/chcaa__dfm-encoder-large-v1/LCC.json @@ -0,0 +1 @@ +{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T21:14:18.646555","scores":{"da":{"accuracy":0.5733333333333334,"f1":0.5589306138422592,"accuracy_stderr":0.024944382578492928,"f1_stderr":0.022088522359918823,"main_score":0.5733333333333334}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/chcaa__dfm-encoder-large-v1/Language_Identification.json b/src/seb/cache/chcaa__dfm-encoder-large-v1/Language_Identification.json new file mode 100644 index 00000000..15f8d0b1 --- /dev/null +++ b/src/seb/cache/chcaa__dfm-encoder-large-v1/Language_Identification.json @@ -0,0 +1 @@ +{"task_name":"Language Identification","task_description":"A dataset for Nordic language identification.","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T21:39:37.355786","scores":{"da":{"accuracy":0.7767666666666666,"f1":0.7738031422395183,"accuracy_stderr":0.01610869330516909,"f1_stderr":0.015915869484821918,"main_score":0.7767666666666666},"sv":{"accuracy":0.7767666666666666,"f1":0.7738031422395183,"accuracy_stderr":0.01610869330516909,"f1_stderr":0.015915869484821918,"main_score":0.7767666666666666},"nb":{"accuracy":0.7767666666666666,"f1":0.7738031422395183,"accuracy_stderr":0.01610869330516909,"f1_stderr":0.015915869484821918,"main_score":0.7767666666666666},"nn":{"accuracy":0.7767666666666666,"f1":0.7738031422395183,"accuracy_stderr":0.01610869330516909,"f1_stderr":0.015915869484821918,"main_score":0.7767666666666666},"is":{"accuracy":0.7767666666666666,"f1":0.7738031422395183,"accuracy_stderr":0.01610869330516909,"f1_stderr":0.015915869484821918,"main_score":0.7767666666666666},"fo":{"accuracy":0.7767666666666666,"f1":0.7738031422395183,"accuracy_stderr":0.01610869330516909,"f1_stderr":0.015915869484821918,"main_score":0.7767666666666666}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/chcaa__dfm-encoder-large-v1/Massive_Intent.json b/src/seb/cache/chcaa__dfm-encoder-large-v1/Massive_Intent.json new file mode 100644 index 00000000..bdb6708a --- /dev/null +++ b/src/seb/cache/chcaa__dfm-encoder-large-v1/Massive_Intent.json @@ -0,0 +1 @@ +{"task_name":"Massive Intent","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T21:25:52.794636","scores":{"da":{"accuracy":0.6055480833893746,"f1":0.5575931847636777,"accuracy_stderr":0.010974707671310587,"f1_stderr":0.005463323455870516,"main_score":0.6055480833893746},"nb":{"accuracy":0.5248823133826497,"f1":0.4916043573899421,"accuracy_stderr":0.012975842025794888,"f1_stderr":0.00929974027333486,"main_score":0.5248823133826497},"sv":{"accuracy":0.4974445191661063,"f1":0.46779266847135725,"accuracy_stderr":0.011126895967691426,"f1_stderr":0.009957825087612915,"main_score":0.4974445191661063}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/chcaa__dfm-encoder-large-v1/Massive_Scenario.json b/src/seb/cache/chcaa__dfm-encoder-large-v1/Massive_Scenario.json new file mode 100644 index 00000000..e8d4ce51 --- /dev/null +++ b/src/seb/cache/chcaa__dfm-encoder-large-v1/Massive_Scenario.json @@ -0,0 +1 @@ +{"task_name":"Massive Scenario","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T21:30:36.585779","scores":{"da":{"accuracy":0.6415601882985877,"f1":0.623984081357014,"accuracy_stderr":0.01681304638870262,"f1_stderr":0.014082572901473661,"main_score":0.6415601882985877},"nb":{"accuracy":0.5459314055144586,"f1":0.5225983817325339,"accuracy_stderr":0.019140841546015294,"f1_stderr":0.021011670214319638,"main_score":0.5459314055144586},"sv":{"accuracy":0.5009751176866174,"f1":0.47320217900892514,"accuracy_stderr":0.016466825531738845,"f1_stderr":0.014959598967962843,"main_score":0.5009751176866174}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/chcaa__dfm-encoder-large-v1/NoReC.json b/src/seb/cache/chcaa__dfm-encoder-large-v1/NoReC.json new file mode 100644 index 00000000..c863c710 --- /dev/null +++ b/src/seb/cache/chcaa__dfm-encoder-large-v1/NoReC.json @@ -0,0 +1 @@ +{"task_name":"NoReC","task_description":"A Norwegian dataset for sentiment classification on review","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T21:41:03.369038","scores":{"nb":{"accuracy":0.4830078125,"f1":0.46143106496173225,"accuracy_stderr":0.01712022968316421,"f1_stderr":0.01809566501225381,"main_score":0.4830078125}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/chcaa__dfm-encoder-large-v1/Norwegian_parliament.json b/src/seb/cache/chcaa__dfm-encoder-large-v1/Norwegian_parliament.json new file mode 100644 index 00000000..02da4848 --- /dev/null +++ b/src/seb/cache/chcaa__dfm-encoder-large-v1/Norwegian_parliament.json @@ -0,0 +1 @@ +{"task_name":"Norwegian parliament","task_description":"Norwegian parliament speeches annotated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T21:51:41.927343","scores":{"nb":{"accuracy":0.58775,"f1":0.5844203893671156,"ap":0.5528065041528007,"accuracy_stderr":0.03341666666666667,"f1_stderr":0.03534225442463607,"ap_stderr":0.02428619847072626,"main_score":0.58775}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/chcaa__dfm-encoder-large-v1/ScaLA.json b/src/seb/cache/chcaa__dfm-encoder-large-v1/ScaLA.json new file mode 100644 index 00000000..6c836fd1 --- /dev/null +++ b/src/seb/cache/chcaa__dfm-encoder-large-v1/ScaLA.json @@ -0,0 +1 @@ +{"task_name":"ScaLA","task_description":"A linguistic acceptability task for Danish, Norwegian Bokmål Norwegian Nynorsk and Swedish.","task_version":"1.1.1.dev0","time_of_run":"2023-09-13T15:40:30.815061","scores":{"da":{"accuracy":0.630810546875,"f1":0.6260740455694338,"ap":0.5837260452649996,"accuracy_stderr":0.023466422990415216,"f1_stderr":0.024513023713813444,"ap_stderr":0.019280573415297236,"main_score":0.630810546875},"nb":{"accuracy":0.589453125,"f1":0.5882174583564697,"ap":0.5535172133296042,"accuracy_stderr":0.0210944281575015,"f1_stderr":0.020389746733680672,"ap_stderr":0.014096413182041974,"main_score":0.589453125},"sv":{"accuracy":0.55703125,"f1":0.5558733679549084,"ap":0.5322501816325099,"accuracy_stderr":0.020064497898719877,"f1_stderr":0.019885357969932186,"ap_stderr":0.012602462290892777,"main_score":0.55703125},"nn":{"accuracy":0.566943359375,"f1":0.5653136782151329,"ap":0.5386721128900174,"accuracy_stderr":0.028983883095240436,"f1_stderr":0.029240441372741073,"ap_stderr":0.01758497140989902,"main_score":0.566943359375}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/chcaa__dfm-encoder-large-v1/SweFAQ.json b/src/seb/cache/chcaa__dfm-encoder-large-v1/SweFAQ.json new file mode 100644 index 00000000..c1d679e7 --- /dev/null +++ b/src/seb/cache/chcaa__dfm-encoder-large-v1/SweFAQ.json @@ -0,0 +1 @@ +{"task_name":"SweFAQ","task_description":"A Swedish QA dataset derived from FAQ","task_version":"0.0.1","time_of_run":"2023-09-04T19:27:36.602064","scores":{"sv":{"ndcg_at_1":0.10721,"ndcg_at_3":0.14761,"ndcg_at_5":0.16346,"ndcg_at_10":0.19129,"ndcg_at_100":0.26475,"ndcg_at_1000":0.30559,"map_at_1":0.10721,"map_at_3":0.13678,"map_at_5":0.14545,"map_at_10":0.15699,"map_at_100":0.16908,"map_at_1000":0.17063,"recall_at_1":0.10721,"recall_at_3":0.17934,"recall_at_5":0.21832,"recall_at_10":0.30409,"recall_at_100":0.68226,"recall_at_1000":1.0,"precision_at_1":0.10721,"precision_at_3":0.05978,"precision_at_5":0.04366,"precision_at_10":0.03041,"precision_at_100":0.00682,"precision_at_1000":0.001,"mrr_at_1":0.10721,"mrr_at_3":0.13678,"mrr_at_5":0.14545,"mrr_at_10":0.15699,"mrr_at_100":0.16908,"mrr_at_1000":0.17063}},"main_score":"ndcg_at_10"} \ No newline at end of file diff --git a/src/seb/cache/chcaa__dfm-encoder-large-v1/SweReC.json b/src/seb/cache/chcaa__dfm-encoder-large-v1/SweReC.json new file mode 100644 index 00000000..c8de791b --- /dev/null +++ b/src/seb/cache/chcaa__dfm-encoder-large-v1/SweReC.json @@ -0,0 +1 @@ +{"task_name":"SweReC","task_description":"A Swedish dataset for sentiment classification on review","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T21:58:40.68154","scores":{"sv":{"accuracy":0.652001953125,"f1":0.5870284778032546,"accuracy_stderr":0.02672469045176074,"f1_stderr":0.0230883220645814,"main_score":0.652001953125}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/embed-multilingual-v3.0/Angry_Tweets.json b/src/seb/cache/embed-multilingual-v3.0/Angry_Tweets.json new file mode 100644 index 00000000..1f98df21 --- /dev/null +++ b/src/seb/cache/embed-multilingual-v3.0/Angry_Tweets.json @@ -0,0 +1 @@ +{"task_name":"Angry Tweets","task_description":"A sentiment dataset with 3 classes (positiv, negativ, neutral) for Danish tweets","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T15:49:12.771515","scores":{"da":{"accuracy":0.589111747851003,"f1":0.5800442049443755,"accuracy_stderr":0.02208679883291171,"f1_stderr":0.0205122012161316,"main_score":0.589111747851003}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/embed-multilingual-v3.0/Bornholm_Parallel.json b/src/seb/cache/embed-multilingual-v3.0/Bornholm_Parallel.json new file mode 100644 index 00000000..8b654b79 --- /dev/null +++ b/src/seb/cache/embed-multilingual-v3.0/Bornholm_Parallel.json @@ -0,0 +1 @@ +{"task_name":"Bornholm Parallel","task_description":"Danish Bornholmsk Parallel Corpus. Bornholmsk is a Danish dialect spoken on the island of Bornholm, Denmark. Historically it is a part of east Danish which was also spoken in Scania and Halland, Sweden.","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T15:49:36.527514","scores":{"da":{"precision":0.32936800976800973,"recall":0.44,"f1":0.35662395382395384,"accuracy":0.44,"main_score":0.35662395382395384},"da-bornholm":{"precision":0.32936800976800973,"recall":0.44,"f1":0.35662395382395384,"accuracy":0.44,"main_score":0.35662395382395384}},"main_score":"f1"} \ No newline at end of file diff --git a/src/seb/cache/embed-multilingual-v3.0/DKHate.json b/src/seb/cache/embed-multilingual-v3.0/DKHate.json new file mode 100644 index 00000000..a7d776ae --- /dev/null +++ b/src/seb/cache/embed-multilingual-v3.0/DKHate.json @@ -0,0 +1 @@ +{"task_name":"DKHate","task_description":"Danish Tweets annotated for Hate Speech either being Offensive or not","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T15:49:50.255921","scores":{"da":{"accuracy":0.6878419452887539,"f1":0.5754292687298207,"ap":0.20997905156829208,"accuracy_stderr":0.06866946902787877,"f1_stderr":0.05597856609791259,"ap_stderr":0.041374474998038195,"main_score":0.6878419452887539}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/embed-multilingual-v3.0/DaLAJ.json b/src/seb/cache/embed-multilingual-v3.0/DaLAJ.json new file mode 100644 index 00000000..8007b1a8 --- /dev/null +++ b/src/seb/cache/embed-multilingual-v3.0/DaLAJ.json @@ -0,0 +1 @@ +{"task_name":"DaLAJ","task_description":"A Swedish dataset for linguistic acceptability. Available as a part of Superlim.","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T15:59:10.156497","scores":{"sv":{"accuracy":0.4996621621621622,"f1":0.4973466706921844,"ap":0.499855758478619,"accuracy_stderr":0.004779075489622282,"f1_stderr":0.0062521755097949515,"ap_stderr":0.0023683552238971985,"main_score":0.4996621621621622}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/embed-multilingual-v3.0/Da_Political_Comments.json b/src/seb/cache/embed-multilingual-v3.0/Da_Political_Comments.json new file mode 100644 index 00000000..47ba6a45 --- /dev/null +++ b/src/seb/cache/embed-multilingual-v3.0/Da_Political_Comments.json @@ -0,0 +1 @@ +{"task_name":"Da Political Comments","task_description":"A dataset of Danish political comments rated for sentiment","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T15:50:10.324853","scores":{"da":{"accuracy":0.4340732519422864,"f1":0.4053270285804955,"accuracy_stderr":0.032140309062267135,"f1_stderr":0.023638097975834493,"main_score":0.4340732519422864}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/embed-multilingual-v3.0/LCC.json b/src/seb/cache/embed-multilingual-v3.0/LCC.json new file mode 100644 index 00000000..aab44a14 --- /dev/null +++ b/src/seb/cache/embed-multilingual-v3.0/LCC.json @@ -0,0 +1 @@ +{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T15:49:26.932464","scores":{"da":{"accuracy":0.604,"f1":0.6045645057913338,"accuracy_stderr":0.034794635601866374,"f1_stderr":0.03469599049990604,"main_score":0.604}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/embed-multilingual-v3.0/Language_Identification.json b/src/seb/cache/embed-multilingual-v3.0/Language_Identification.json new file mode 100644 index 00000000..22dae117 --- /dev/null +++ b/src/seb/cache/embed-multilingual-v3.0/Language_Identification.json @@ -0,0 +1 @@ +{"task_name":"Language Identification","task_description":"A dataset for Nordic language identification.","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T15:57:45.658096","scores":{"da":{"accuracy":0.7872333333333332,"f1":0.7806507474390829,"accuracy_stderr":0.0072419303749453184,"f1_stderr":0.007780108614777823,"main_score":0.7872333333333332},"sv":{"accuracy":0.7872333333333332,"f1":0.7806507474390829,"accuracy_stderr":0.0072419303749453184,"f1_stderr":0.007780108614777823,"main_score":0.7872333333333332},"nb":{"accuracy":0.7872333333333332,"f1":0.7806507474390829,"accuracy_stderr":0.0072419303749453184,"f1_stderr":0.007780108614777823,"main_score":0.7872333333333332},"nn":{"accuracy":0.7872333333333332,"f1":0.7806507474390829,"accuracy_stderr":0.0072419303749453184,"f1_stderr":0.007780108614777823,"main_score":0.7872333333333332},"is":{"accuracy":0.7872333333333332,"f1":0.7806507474390829,"accuracy_stderr":0.0072419303749453184,"f1_stderr":0.007780108614777823,"main_score":0.7872333333333332},"fo":{"accuracy":0.7872333333333332,"f1":0.7806507474390829,"accuracy_stderr":0.0072419303749453184,"f1_stderr":0.007780108614777823,"main_score":0.7872333333333332}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/embed-multilingual-v3.0/Massive_Intent.json b/src/seb/cache/embed-multilingual-v3.0/Massive_Intent.json new file mode 100644 index 00000000..004c6fed --- /dev/null +++ b/src/seb/cache/embed-multilingual-v3.0/Massive_Intent.json @@ -0,0 +1 @@ +{"task_name":"Massive Intent","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T15:53:09.380602","scores":{"da":{"accuracy":0.6654673839946199,"f1":0.637776757848086,"accuracy_stderr":0.013775850305668075,"f1_stderr":0.013697069516200004,"main_score":0.6654673839946199},"nb":{"accuracy":0.6719233355749832,"f1":0.6446330966343174,"accuracy_stderr":0.014987021986641317,"f1_stderr":0.015961831197456297,"main_score":0.6719233355749832},"sv":{"accuracy":0.6960659045057163,"f1":0.6766171113444953,"accuracy_stderr":0.018051479607875766,"f1_stderr":0.015799973996323373,"main_score":0.6960659045057163}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/embed-multilingual-v3.0/Massive_Scenario.json b/src/seb/cache/embed-multilingual-v3.0/Massive_Scenario.json new file mode 100644 index 00000000..9dc4c748 --- /dev/null +++ b/src/seb/cache/embed-multilingual-v3.0/Massive_Scenario.json @@ -0,0 +1 @@ +{"task_name":"Massive Scenario","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T15:55:22.745539","scores":{"da":{"accuracy":0.7438466711499665,"f1":0.7366895282457374,"accuracy_stderr":0.009173827073322146,"f1_stderr":0.009689167533545422,"main_score":0.7438466711499665},"nb":{"accuracy":0.7384330867518494,"f1":0.7322937369822327,"accuracy_stderr":0.010778239699265587,"f1_stderr":0.009692505530198426,"main_score":0.7384330867518494},"sv":{"accuracy":0.7574310692669805,"f1":0.7517381645229646,"accuracy_stderr":0.012045171284048639,"f1_stderr":0.01166614751961022,"main_score":0.7574310692669805}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/embed-multilingual-v3.0/NoReC.json b/src/seb/cache/embed-multilingual-v3.0/NoReC.json new file mode 100644 index 00000000..e5bc1baa --- /dev/null +++ b/src/seb/cache/embed-multilingual-v3.0/NoReC.json @@ -0,0 +1 @@ +{"task_name":"NoReC","task_description":"A Norwegian dataset for sentiment classification on review","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T15:58:05.614937","scores":{"nb":{"accuracy":0.66064453125,"f1":0.6496479700114184,"accuracy_stderr":0.016785799855301105,"f1_stderr":0.015430913489577222,"main_score":0.66064453125}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/embed-multilingual-v3.0/Norwegian_parliament.json b/src/seb/cache/embed-multilingual-v3.0/Norwegian_parliament.json new file mode 100644 index 00000000..37a39070 --- /dev/null +++ b/src/seb/cache/embed-multilingual-v3.0/Norwegian_parliament.json @@ -0,0 +1 @@ +{"task_name":"Norwegian parliament","task_description":"Norwegian parliament speeches annotated for sentiment","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T15:58:29.889565","scores":{"nb":{"accuracy":0.5998333333333333,"f1":0.5973067990191325,"ap":0.5608034931676839,"accuracy_stderr":0.027610384519838403,"f1_stderr":0.028539401198880567,"ap_stderr":0.019753617006441477,"main_score":0.5998333333333333}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/embed-multilingual-v3.0/ScaLA.json b/src/seb/cache/embed-multilingual-v3.0/ScaLA.json new file mode 100644 index 00000000..3b7766d9 --- /dev/null +++ b/src/seb/cache/embed-multilingual-v3.0/ScaLA.json @@ -0,0 +1 @@ +{"task_name":"ScaLA","task_description":"A linguistic acceptability task for Danish, Norwegian Bokmål Norwegian Nynorsk and Swedish.","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T15:56:53.574324","scores":{"da":{"accuracy":0.508349609375,"f1":0.5056089560837547,"ap":0.504270609280779,"accuracy_stderr":0.005552039093165773,"f1_stderr":0.0061853850329536605,"ap_stderr":0.002852190710808234,"main_score":0.508349609375},"nb":{"accuracy":0.5068359375,"f1":0.5028082737136484,"ap":0.5035329742210839,"accuracy_stderr":0.007542325435410011,"f1_stderr":0.00918042108637033,"ap_stderr":0.003932270904497431,"main_score":0.5068359375},"sv":{"accuracy":0.5068359375,"f1":0.5043076304129289,"ap":0.5035176962385777,"accuracy_stderr":0.007069125418302288,"f1_stderr":0.008946547886637759,"ap_stderr":0.003617227768456735,"main_score":0.5068359375},"nn":{"accuracy":0.5056640625,"f1":0.5036120708948217,"ap":0.5029426709291045,"accuracy_stderr":0.00923148617038787,"f1_stderr":0.008517687177028652,"ap_stderr":0.004577040920590477,"main_score":0.5056640625}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/embed-multilingual-v3.0/SweFAQ.json b/src/seb/cache/embed-multilingual-v3.0/SweFAQ.json new file mode 100644 index 00000000..82b80b7b --- /dev/null +++ b/src/seb/cache/embed-multilingual-v3.0/SweFAQ.json @@ -0,0 +1 @@ +{"task_name":"SweFAQ","task_description":"A Swedish QA dataset derived from FAQ","task_version":"0.0.1","time_of_run":"2023-11-15T15:59:20.656254","scores":{"sv":{"ndcg_at_1":0.5809,"ndcg_at_3":0.71038,"ndcg_at_5":0.7374,"ndcg_at_10":0.75234,"ndcg_at_100":0.76935,"ndcg_at_1000":0.77073,"map_at_1":0.5809,"map_at_3":0.67836,"map_at_5":0.69366,"map_at_10":0.69972,"map_at_100":0.70353,"map_at_1000":0.70361,"recall_at_1":0.5809,"recall_at_3":0.80312,"recall_at_5":0.86745,"recall_at_10":0.91423,"recall_at_100":0.99025,"recall_at_1000":1.0,"precision_at_1":0.5809,"precision_at_3":0.26771,"precision_at_5":0.17349,"precision_at_10":0.09142,"precision_at_100":0.0099,"precision_at_1000":0.001,"mrr_at_1":0.5809,"mrr_at_3":0.67836,"mrr_at_5":0.69366,"mrr_at_10":0.69972,"mrr_at_100":0.70353,"mrr_at_1000":0.70361}},"main_score":"ndcg_at_10"} \ No newline at end of file diff --git a/src/seb/cache/embed-multilingual-v3.0/SweReC.json b/src/seb/cache/embed-multilingual-v3.0/SweReC.json new file mode 100644 index 00000000..79e94863 --- /dev/null +++ b/src/seb/cache/embed-multilingual-v3.0/SweReC.json @@ -0,0 +1 @@ +{"task_name":"SweReC","task_description":"A Swedish dataset for sentiment classification on review","task_version":"1.1.1.dev0","time_of_run":"2023-11-15T15:58:54.160842","scores":{"sv":{"accuracy":0.841796875,"f1":0.7577270299740745,"accuracy_stderr":0.009037820821425667,"f1_stderr":0.006868062554294348,"main_score":0.841796875}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-base/Angry_Tweets.json b/src/seb/cache/intfloat__e5-base/Angry_Tweets.json new file mode 100644 index 00000000..e5d5268b --- /dev/null +++ b/src/seb/cache/intfloat__e5-base/Angry_Tweets.json @@ -0,0 +1 @@ +{"task_name":"Angry Tweets","task_description":"A sentiment dataset with 3 classes (positiv, negativ, neutral) for Danish tweets","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T12:12:43.050686","scores":{"da":{"accuracy":0.45062082139446036,"f1":0.4444174436926957,"accuracy_stderr":0.018014900863957715,"f1_stderr":0.0178439514512202,"main_score":0.45062082139446036}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-base/Bornholm_Parallel.json b/src/seb/cache/intfloat__e5-base/Bornholm_Parallel.json new file mode 100644 index 00000000..f3b4ff3d --- /dev/null +++ b/src/seb/cache/intfloat__e5-base/Bornholm_Parallel.json @@ -0,0 +1 @@ +{"task_name":"Bornholm Parallel","task_description":"Danish Bornholmsk Parallel Corpus. Bornholmsk is a Danish dialect spoken on the island of Bornholm, Denmark. Historically it is a part of east Danish which was also spoken in Scania and Halland, Sweden.","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T12:13:10.71302","scores":{"da":{"precision":0.3756453102453103,"recall":0.476,"f1":0.40085997335997336,"accuracy":0.476,"main_score":0.40085997335997336},"da-bornholm":{"precision":0.3756453102453103,"recall":0.476,"f1":0.40085997335997336,"accuracy":0.476,"main_score":0.40085997335997336}},"main_score":"f1"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-base/DKHate.json b/src/seb/cache/intfloat__e5-base/DKHate.json new file mode 100644 index 00000000..81f762cb --- /dev/null +++ b/src/seb/cache/intfloat__e5-base/DKHate.json @@ -0,0 +1 @@ +{"task_name":"DKHate","task_description":"Danish Tweets annotated for Hate Speech either being Offensive or not","task_version":"1.0.3.dev0","time_of_run":"2023-07-30T14:04:01.90744","scores":{"da":{"accuracy":0.5851063829787233,"f1":0.48251157398731886,"ap":0.8934379089500915,"accuracy_stderr":0.08442245820326019,"f1_stderr":0.05216259611447489,"ap_stderr":0.011387211965949523,"main_score":0.5851063829787233}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-base/DaLAJ.json b/src/seb/cache/intfloat__e5-base/DaLAJ.json new file mode 100644 index 00000000..c1f47a8e --- /dev/null +++ b/src/seb/cache/intfloat__e5-base/DaLAJ.json @@ -0,0 +1 @@ +{"task_name":"DaLAJ","task_description":"A Swedish dataset for linguistic accebtablity. Available as a part of Superlim.","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T12:47:51.12742","scores":{"sv":{"accuracy":0.4961711711711712,"f1":0.4911100751198404,"ap":0.4981458216185105,"accuracy_stderr":0.006047622334278188,"f1_stderr":0.007501621108791459,"ap_stderr":0.0029592715632036624,"main_score":0.4961711711711712}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-base/Da_Political_Comments.json b/src/seb/cache/intfloat__e5-base/Da_Political_Comments.json new file mode 100644 index 00000000..b2ce983a --- /dev/null +++ b/src/seb/cache/intfloat__e5-base/Da_Political_Comments.json @@ -0,0 +1 @@ +{"task_name":"Da Political Comments","task_description":"A dataset of Danish political comments rated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T12:14:45.041122","scores":{"da":{"accuracy":0.27780244173140956,"f1":0.24995185992134927,"accuracy_stderr":0.01814098759778861,"f1_stderr":0.014719657591458882,"main_score":0.27780244173140956}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-base/LCC.json b/src/seb/cache/intfloat__e5-base/LCC.json new file mode 100644 index 00000000..6019ae7a --- /dev/null +++ b/src/seb/cache/intfloat__e5-base/LCC.json @@ -0,0 +1 @@ +{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T12:12:58.744619","scores":{"da":{"accuracy":0.3746666666666667,"f1":0.34892402884941426,"accuracy_stderr":0.04568491119736484,"f1_stderr":0.03473805734862492,"main_score":0.3746666666666667}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-base/Language_Identification.json b/src/seb/cache/intfloat__e5-base/Language_Identification.json new file mode 100644 index 00000000..f6bfb33d --- /dev/null +++ b/src/seb/cache/intfloat__e5-base/Language_Identification.json @@ -0,0 +1 @@ +{"task_name":"Language Identification","task_description":"A dataset for Nordic language identification.","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T12:39:44.457932","scores":{"da":{"accuracy":0.5933666666666666,"f1":0.5686537021938836,"accuracy_stderr":0.01949555504895068,"f1_stderr":0.025468676414749182,"main_score":0.5933666666666666},"sv":{"accuracy":0.5933666666666666,"f1":0.5686537021938836,"accuracy_stderr":0.01949555504895068,"f1_stderr":0.025468676414749182,"main_score":0.5933666666666666},"nb":{"accuracy":0.5933666666666666,"f1":0.5686537021938836,"accuracy_stderr":0.01949555504895068,"f1_stderr":0.025468676414749182,"main_score":0.5933666666666666},"nn":{"accuracy":0.5933666666666666,"f1":0.5686537021938836,"accuracy_stderr":0.01949555504895068,"f1_stderr":0.025468676414749182,"main_score":0.5933666666666666},"is":{"accuracy":0.5933666666666666,"f1":0.5686537021938836,"accuracy_stderr":0.01949555504895068,"f1_stderr":0.025468676414749182,"main_score":0.5933666666666666},"fo":{"accuracy":0.5933666666666666,"f1":0.5686537021938836,"accuracy_stderr":0.01949555504895068,"f1_stderr":0.025468676414749182,"main_score":0.5933666666666666}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-base/Massive_Intent.json b/src/seb/cache/intfloat__e5-base/Massive_Intent.json new file mode 100644 index 00000000..7c004d10 --- /dev/null +++ b/src/seb/cache/intfloat__e5-base/Massive_Intent.json @@ -0,0 +1 @@ +{"task_name":"Massive Intent","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T12:18:14.922391","scores":{"da":{"accuracy":0.4425353059852051,"f1":0.42176846706873306,"accuracy_stderr":0.011987069326364846,"f1_stderr":0.011356143085800888,"main_score":0.4425353059852051},"nb":{"accuracy":0.4156691324815064,"f1":0.38586961267500197,"accuracy_stderr":0.01598275910181979,"f1_stderr":0.015427786451280444,"main_score":0.4156691324815064},"sv":{"accuracy":0.4134498991257566,"f1":0.3929727867759662,"accuracy_stderr":0.018471099583417097,"f1_stderr":0.013898327460181962,"main_score":0.4134498991257566}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-base/Massive_Scenario.json b/src/seb/cache/intfloat__e5-base/Massive_Scenario.json new file mode 100644 index 00000000..4656026f --- /dev/null +++ b/src/seb/cache/intfloat__e5-base/Massive_Scenario.json @@ -0,0 +1 @@ +{"task_name":"Massive Scenario","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T12:20:51.074698","scores":{"da":{"accuracy":0.5299260255548084,"f1":0.498560032028262,"accuracy_stderr":0.00994860772595831,"f1_stderr":0.011851321957858672,"main_score":0.5299260255548084},"nb":{"accuracy":0.503261600537996,"f1":0.4690655880152623,"accuracy_stderr":0.023111260517713856,"f1_stderr":0.02016718494227476,"main_score":0.503261600537996},"sv":{"accuracy":0.5000336247478143,"f1":0.4690590587313784,"accuracy_stderr":0.016636232640655457,"f1_stderr":0.015175993334931649,"main_score":0.5000336247478143}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-base/NoReC.json b/src/seb/cache/intfloat__e5-base/NoReC.json new file mode 100644 index 00000000..f408b1fd --- /dev/null +++ b/src/seb/cache/intfloat__e5-base/NoReC.json @@ -0,0 +1 @@ +{"task_name":"NoReC","task_description":"A Norwegian dataset for sentiment classification on review","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T12:40:25.650181","scores":{"nb":{"accuracy":0.42001953125,"f1":0.39420842567274017,"accuracy_stderr":0.017598187335909503,"f1_stderr":0.013027788332644633,"main_score":0.42001953125}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-base/Norwegian_parliament.json b/src/seb/cache/intfloat__e5-base/Norwegian_parliament.json new file mode 100644 index 00000000..f8671bab --- /dev/null +++ b/src/seb/cache/intfloat__e5-base/Norwegian_parliament.json @@ -0,0 +1 @@ +{"task_name":"Norwegian parliament","task_description":"Norwegian parliament speeches annotated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T12:44:44.485459","scores":{"nb":{"accuracy":0.5742499999999999,"f1":0.5701556209363587,"ap":0.5430478170562347,"accuracy_stderr":0.0184030869994985,"f1_stderr":0.021749204878184724,"ap_stderr":0.012016851444700091,"main_score":0.5742499999999999}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-base/ScaLA.json b/src/seb/cache/intfloat__e5-base/ScaLA.json new file mode 100644 index 00000000..6ea1ac24 --- /dev/null +++ b/src/seb/cache/intfloat__e5-base/ScaLA.json @@ -0,0 +1 @@ +{"task_name":"ScaLA","task_description":"A linguistic acceptability task for Danish, Norwegian Bokmål Norwegian Nynorsk and Swedish.","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T12:38:25.571788","scores":{"da":{"accuracy":0.500830078125,"f1":0.4973482849517068,"ap":0.5004512186779815,"accuracy_stderr":0.005389041391598447,"f1_stderr":0.004568697226028063,"ap_stderr":0.0027327139411046967,"main_score":0.500830078125},"nb":{"accuracy":0.501806640625,"f1":0.4981610113006016,"ap":0.5009193289813187,"accuracy_stderr":0.0035348999604957547,"f1_stderr":0.007048761194173441,"ap_stderr":0.0017779198690836895,"main_score":0.501806640625},"sv":{"accuracy":0.49951171875,"f1":0.49540605303641233,"ap":0.4997630461906465,"accuracy_stderr":0.0025929527436233894,"f1_stderr":0.003819030502792098,"ap_stderr":0.001293738054598032,"main_score":0.49951171875},"nn":{"accuracy":0.501171875,"f1":0.4997391539249885,"ap":0.5006107227948361,"accuracy_stderr":0.00471993983343538,"f1_stderr":0.005327797489679061,"ap_stderr":0.002382782710554753,"main_score":0.501171875}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-base/SweFAQ.json b/src/seb/cache/intfloat__e5-base/SweFAQ.json new file mode 100644 index 00000000..a0eefdc1 --- /dev/null +++ b/src/seb/cache/intfloat__e5-base/SweFAQ.json @@ -0,0 +1 @@ +{"task_name":"SweFAQ","task_description":"A Swedish QA dataset derived from FAQ","task_version":"0.0.1","time_of_run":"2023-09-04T19:40:13.341176","scores":{"sv":{"ndcg_at_1":0.37427,"ndcg_at_3":0.47684,"ndcg_at_5":0.5041,"ndcg_at_10":0.53411,"ndcg_at_100":0.57982,"ndcg_at_1000":0.5901,"map_at_1":0.37427,"map_at_3":0.45224,"map_at_5":0.46735,"map_at_10":0.4796,"map_at_100":0.48893,"map_at_1000":0.48936,"recall_at_1":0.37427,"recall_at_3":0.54776,"recall_at_5":0.61404,"recall_at_10":0.7076,"recall_at_100":0.92203,"recall_at_1000":1.0,"precision_at_1":0.37427,"precision_at_3":0.18259,"precision_at_5":0.12281,"precision_at_10":0.07076,"precision_at_100":0.00922,"precision_at_1000":0.001,"mrr_at_1":0.37622,"mrr_at_3":0.45322,"mrr_at_5":0.46832,"mrr_at_10":0.48057,"mrr_at_100":0.48989,"mrr_at_1000":0.49033}},"main_score":"ndcg_at_10"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-base/SweReC.json b/src/seb/cache/intfloat__e5-base/SweReC.json new file mode 100644 index 00000000..0ed1b150 --- /dev/null +++ b/src/seb/cache/intfloat__e5-base/SweReC.json @@ -0,0 +1 @@ +{"task_name":"SweReC","task_description":"A Swedish dataset for sentiment classification on review","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T12:47:22.748591","scores":{"sv":{"accuracy":0.61044921875,"f1":0.5308090059465818,"accuracy_stderr":0.02752260839343394,"f1_stderr":0.019609990731894015,"main_score":0.61044921875}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-large/Angry_Tweets.json b/src/seb/cache/intfloat__e5-large/Angry_Tweets.json new file mode 100644 index 00000000..a7c13c16 --- /dev/null +++ b/src/seb/cache/intfloat__e5-large/Angry_Tweets.json @@ -0,0 +1 @@ +{"task_name":"Angry Tweets","task_description":"A sentiment dataset with 3 classes (positiv, negativ, neutral) for Danish tweets","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T12:49:55.846642","scores":{"da":{"accuracy":0.46141356255969435,"f1":0.45213682091348817,"accuracy_stderr":0.01975692661833229,"f1_stderr":0.018483239580143,"main_score":0.46141356255969435}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-large/Bornholm_Parallel.json b/src/seb/cache/intfloat__e5-large/Bornholm_Parallel.json new file mode 100644 index 00000000..f4c037a4 --- /dev/null +++ b/src/seb/cache/intfloat__e5-large/Bornholm_Parallel.json @@ -0,0 +1 @@ +{"task_name":"Bornholm Parallel","task_description":"Danish Bornholmsk Parallel Corpus. Bornholmsk is a Danish dialect spoken on the island of Bornholm, Denmark. Historically it is a part of east Danish which was also spoken in Scania and Halland, Sweden.","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T12:51:12.671843","scores":{"da":{"precision":0.3737246031746032,"recall":0.48,"f1":0.4015300588300588,"accuracy":0.48,"main_score":0.4015300588300588},"da-bornholm":{"precision":0.3737246031746032,"recall":0.48,"f1":0.4015300588300588,"accuracy":0.48,"main_score":0.4015300588300588}},"main_score":"f1"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-large/DKHate.json b/src/seb/cache/intfloat__e5-large/DKHate.json new file mode 100644 index 00000000..11d233ef --- /dev/null +++ b/src/seb/cache/intfloat__e5-large/DKHate.json @@ -0,0 +1 @@ +{"task_name":"DKHate","task_description":"Danish Tweets annotated for Hate Speech either being Offensive or not","task_version":"1.0.3.dev0","time_of_run":"2023-07-30T14:06:33.363868","scores":{"da":{"accuracy":0.5872340425531914,"f1":0.48395133167666576,"ap":0.8936550246115663,"accuracy_stderr":0.07910681065370229,"f1_stderr":0.047771795718712304,"ap_stderr":0.00981794770014744,"main_score":0.5872340425531914}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-large/DaLAJ.json b/src/seb/cache/intfloat__e5-large/DaLAJ.json new file mode 100644 index 00000000..8585f2e5 --- /dev/null +++ b/src/seb/cache/intfloat__e5-large/DaLAJ.json @@ -0,0 +1 @@ +{"task_name":"DaLAJ","task_description":"A Swedish dataset for linguistic accebtablity. Available as a part of Superlim.","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T14:32:18.336197","scores":{"sv":{"accuracy":0.4976351351351352,"f1":0.4950559049528488,"ap":0.4988653813237763,"accuracy_stderr":0.006399131822151077,"f1_stderr":0.007482085016856324,"ap_stderr":0.0032253782282889304,"main_score":0.4976351351351352}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-large/Da_Political_Comments.json b/src/seb/cache/intfloat__e5-large/Da_Political_Comments.json new file mode 100644 index 00000000..f7c337c1 --- /dev/null +++ b/src/seb/cache/intfloat__e5-large/Da_Political_Comments.json @@ -0,0 +1 @@ +{"task_name":"Da Political Comments","task_description":"A dataset of Danish political comments rated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T12:55:51.693582","scores":{"da":{"accuracy":0.279023307436182,"f1":0.2531537291315898,"accuracy_stderr":0.016660742569460263,"f1_stderr":0.014417751080696962,"main_score":0.279023307436182}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-large/LCC.json b/src/seb/cache/intfloat__e5-large/LCC.json new file mode 100644 index 00000000..07d39dc0 --- /dev/null +++ b/src/seb/cache/intfloat__e5-large/LCC.json @@ -0,0 +1 @@ +{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T12:50:41.016728","scores":{"da":{"accuracy":0.4213333333333334,"f1":0.3986438588836284,"accuracy_stderr":0.025957229778575713,"f1_stderr":0.022316523622632742,"main_score":0.4213333333333334}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-large/Language_Identification.json b/src/seb/cache/intfloat__e5-large/Language_Identification.json new file mode 100644 index 00000000..f4321eae --- /dev/null +++ b/src/seb/cache/intfloat__e5-large/Language_Identification.json @@ -0,0 +1 @@ +{"task_name":"Language Identification","task_description":"A dataset for Nordic language identification.","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T13:20:54.957511","scores":{"da":{"accuracy":0.5829666666666667,"f1":0.556416612372918,"accuracy_stderr":0.01689605476632537,"f1_stderr":0.024345884708486788,"main_score":0.5829666666666667},"sv":{"accuracy":0.5829666666666667,"f1":0.556416612372918,"accuracy_stderr":0.01689605476632537,"f1_stderr":0.024345884708486788,"main_score":0.5829666666666667},"nb":{"accuracy":0.5829666666666667,"f1":0.556416612372918,"accuracy_stderr":0.01689605476632537,"f1_stderr":0.024345884708486788,"main_score":0.5829666666666667},"nn":{"accuracy":0.5829666666666667,"f1":0.556416612372918,"accuracy_stderr":0.01689605476632537,"f1_stderr":0.024345884708486788,"main_score":0.5829666666666667},"is":{"accuracy":0.5829666666666667,"f1":0.556416612372918,"accuracy_stderr":0.01689605476632537,"f1_stderr":0.024345884708486788,"main_score":0.5829666666666667},"fo":{"accuracy":0.5829666666666667,"f1":0.556416612372918,"accuracy_stderr":0.01689605476632537,"f1_stderr":0.024345884708486788,"main_score":0.5829666666666667}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-large/Massive_Intent.json b/src/seb/cache/intfloat__e5-large/Massive_Intent.json new file mode 100644 index 00000000..58b382e8 --- /dev/null +++ b/src/seb/cache/intfloat__e5-large/Massive_Intent.json @@ -0,0 +1 @@ +{"task_name":"Massive Intent","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T13:04:17.998107","scores":{"da":{"accuracy":0.42286482851378615,"f1":0.4045450712997618,"accuracy_stderr":0.012383040984379165,"f1_stderr":0.011450412901048949,"main_score":0.42286482851378615},"nb":{"accuracy":0.4063214525891056,"f1":0.3789828222208245,"accuracy_stderr":0.014827865675936818,"f1_stderr":0.012769924322664489,"main_score":0.4063214525891056},"sv":{"accuracy":0.4068594485541358,"f1":0.3868424150540931,"accuracy_stderr":0.016741613025560997,"f1_stderr":0.014421268110823617,"main_score":0.4068594485541358}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-large/Massive_Scenario.json b/src/seb/cache/intfloat__e5-large/Massive_Scenario.json new file mode 100644 index 00000000..1d94eaed --- /dev/null +++ b/src/seb/cache/intfloat__e5-large/Massive_Scenario.json @@ -0,0 +1 @@ +{"task_name":"Massive Scenario","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T13:09:35.662108","scores":{"da":{"accuracy":0.5294889038332213,"f1":0.497844188181838,"accuracy_stderr":0.01179845398050312,"f1_stderr":0.015948599545340024,"main_score":0.5294889038332213},"nb":{"accuracy":0.51906523201076,"f1":0.48469253168791776,"accuracy_stderr":0.020893959423191263,"f1_stderr":0.018431620456520796,"main_score":0.51906523201076},"sv":{"accuracy":0.5096503026227303,"f1":0.47970318432652237,"accuracy_stderr":0.01508477542287528,"f1_stderr":0.013314843525331407,"main_score":0.5096503026227303}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-large/NoReC.json b/src/seb/cache/intfloat__e5-large/NoReC.json new file mode 100644 index 00000000..49d833cb --- /dev/null +++ b/src/seb/cache/intfloat__e5-large/NoReC.json @@ -0,0 +1 @@ +{"task_name":"NoReC","task_description":"A Norwegian dataset for sentiment classification on review","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T13:22:49.408683","scores":{"nb":{"accuracy":0.418310546875,"f1":0.3960993062098541,"accuracy_stderr":0.016873702953082853,"f1_stderr":0.014283859855597512,"main_score":0.418310546875}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-large/Norwegian_parliament.json b/src/seb/cache/intfloat__e5-large/Norwegian_parliament.json new file mode 100644 index 00000000..f6203333 --- /dev/null +++ b/src/seb/cache/intfloat__e5-large/Norwegian_parliament.json @@ -0,0 +1 @@ +{"task_name":"Norwegian parliament","task_description":"Norwegian parliament speeches annotated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T14:02:23.243931","scores":{"nb":{"accuracy":0.5725833333333334,"f1":0.5690528302549381,"ap":0.5421311550575515,"accuracy_stderr":0.022473595617969102,"f1_stderr":0.022504018792512858,"ap_stderr":0.014799630712362313,"main_score":0.5725833333333334}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-large/ScaLA.json b/src/seb/cache/intfloat__e5-large/ScaLA.json new file mode 100644 index 00000000..278f01f6 --- /dev/null +++ b/src/seb/cache/intfloat__e5-large/ScaLA.json @@ -0,0 +1 @@ +{"task_name":"ScaLA","task_description":"A linguistic acceptability task for Danish, Norwegian Bokmål Norwegian Nynorsk and Swedish.","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T13:17:34.836663","scores":{"da":{"accuracy":0.498974609375,"f1":0.4973243378375657,"ap":0.4994975402181119,"accuracy_stderr":0.0030293119861808785,"f1_stderr":0.003381182397789181,"ap_stderr":0.0015025263112233393,"main_score":0.498974609375},"nb":{"accuracy":0.50126953125,"f1":0.49493851513228365,"ap":0.5006549217251715,"accuracy_stderr":0.0045919210724643545,"f1_stderr":0.008711845834406649,"ap_stderr":0.0022812677266345655,"main_score":0.50126953125},"sv":{"accuracy":0.49833984375,"f1":0.49411065609244,"ap":0.4992063849005556,"accuracy_stderr":0.005916872835820746,"f1_stderr":0.0050753509990883235,"ap_stderr":0.0029618573072430895,"main_score":0.49833984375},"nn":{"accuracy":0.497900390625,"f1":0.4950763565792915,"ap":0.4989615171014935,"accuracy_stderr":0.0025749603268428144,"f1_stderr":0.0028533289445563394,"ap_stderr":0.0012755367059780374,"main_score":0.497900390625}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-large/SweFAQ.json b/src/seb/cache/intfloat__e5-large/SweFAQ.json new file mode 100644 index 00000000..56dfdec3 --- /dev/null +++ b/src/seb/cache/intfloat__e5-large/SweFAQ.json @@ -0,0 +1 @@ +{"task_name":"SweFAQ","task_description":"A Swedish QA dataset derived from FAQ","task_version":"0.0.1","time_of_run":"2023-09-04T19:42:39.124202","scores":{"sv":{"ndcg_at_1":0.37817,"ndcg_at_3":0.48879,"ndcg_at_5":0.52243,"ndcg_at_10":0.55326,"ndcg_at_100":0.59303,"ndcg_at_1000":0.60256,"map_at_1":0.37817,"map_at_3":0.46166,"map_at_5":0.48028,"map_at_10":0.49297,"map_at_100":0.50166,"map_at_1000":0.50202,"recall_at_1":0.37817,"recall_at_3":0.56725,"recall_at_5":0.64912,"recall_at_10":0.74464,"recall_at_100":0.92593,"recall_at_1000":1.0,"precision_at_1":0.37817,"precision_at_3":0.18908,"precision_at_5":0.12982,"precision_at_10":0.07446,"precision_at_100":0.00926,"precision_at_1000":0.001,"mrr_at_1":0.38012,"mrr_at_3":0.46264,"mrr_at_5":0.48125,"mrr_at_10":0.49375,"mrr_at_100":0.50261,"mrr_at_1000":0.50298}},"main_score":"ndcg_at_10"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-large/SweReC.json b/src/seb/cache/intfloat__e5-large/SweReC.json new file mode 100644 index 00000000..ec621b43 --- /dev/null +++ b/src/seb/cache/intfloat__e5-large/SweReC.json @@ -0,0 +1 @@ +{"task_name":"SweReC","task_description":"A Swedish dataset for sentiment classification on review","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T14:31:01.551071","scores":{"sv":{"accuracy":0.60087890625,"f1":0.5301196731263103,"accuracy_stderr":0.026912404135228996,"f1_stderr":0.012139795101073433,"main_score":0.60087890625}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-small/Angry_Tweets.json b/src/seb/cache/intfloat__e5-small/Angry_Tweets.json new file mode 100644 index 00000000..ce863dd7 --- /dev/null +++ b/src/seb/cache/intfloat__e5-small/Angry_Tweets.json @@ -0,0 +1 @@ +{"task_name":"Angry Tweets","task_description":"A sentiment dataset with 3 classes (positiv, negativ, neutral) for Danish tweets","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T12:00:30.558944","scores":{"da":{"accuracy":0.436007640878701,"f1":0.42911866353028233,"accuracy_stderr":0.021755758478771595,"f1_stderr":0.018644890106638213,"main_score":0.436007640878701}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-small/Bornholm_Parallel.json b/src/seb/cache/intfloat__e5-small/Bornholm_Parallel.json new file mode 100644 index 00000000..5e2bb55f --- /dev/null +++ b/src/seb/cache/intfloat__e5-small/Bornholm_Parallel.json @@ -0,0 +1 @@ +{"task_name":"Bornholm Parallel","task_description":"Danish Bornholmsk Parallel Corpus. Bornholmsk is a Danish dialect spoken on the island of Bornholm, Denmark. Historically it is a part of east Danish which was also spoken in Scania and Halland, Sweden.","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T12:00:49.008261","scores":{"da":{"precision":0.37982467532467534,"recall":0.47,"f1":0.4027064935064935,"accuracy":0.47,"main_score":0.4027064935064935},"da-bornholm":{"precision":0.37982467532467534,"recall":0.47,"f1":0.4027064935064935,"accuracy":0.47,"main_score":0.4027064935064935}},"main_score":"f1"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-small/DKHate.json b/src/seb/cache/intfloat__e5-small/DKHate.json new file mode 100644 index 00000000..799bf794 --- /dev/null +++ b/src/seb/cache/intfloat__e5-small/DKHate.json @@ -0,0 +1 @@ +{"task_name":"DKHate","task_description":"Danish Tweets annotated for Hate Speech either being Offensive or not","task_version":"1.0.3.dev0","time_of_run":"2023-07-30T14:03:12.250963","scores":{"da":{"accuracy":0.5756838905775077,"f1":0.47109747610810365,"ap":0.8879650182552059,"accuracy_stderr":0.07542894769238224,"f1_stderr":0.04666878188830251,"ap_stderr":0.00896554793947715,"main_score":0.5756838905775077}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-small/DaLAJ.json b/src/seb/cache/intfloat__e5-small/DaLAJ.json new file mode 100644 index 00000000..cb5b2d27 --- /dev/null +++ b/src/seb/cache/intfloat__e5-small/DaLAJ.json @@ -0,0 +1 @@ +{"task_name":"DaLAJ","task_description":"A Swedish dataset for linguistic accebtablity. Available as a part of Superlim.","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T12:12:02.949471","scores":{"sv":{"accuracy":0.49853603603603613,"f1":0.4944401568286577,"ap":0.4993237308818935,"accuracy_stderr":0.006979256951507278,"f1_stderr":0.009348893113348399,"ap_stderr":0.0034992260984685357,"main_score":0.49853603603603613}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-small/Da_Political_Comments.json b/src/seb/cache/intfloat__e5-small/Da_Political_Comments.json new file mode 100644 index 00000000..f7b74e37 --- /dev/null +++ b/src/seb/cache/intfloat__e5-small/Da_Political_Comments.json @@ -0,0 +1 @@ +{"task_name":"Da Political Comments","task_description":"A dataset of Danish political comments rated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T12:01:44.107344","scores":{"da":{"accuracy":0.27652608213096563,"f1":0.24257917155509343,"accuracy_stderr":0.023423486257928235,"f1_stderr":0.01535952783226172,"main_score":0.27652608213096563}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-small/LCC.json b/src/seb/cache/intfloat__e5-small/LCC.json new file mode 100644 index 00000000..b5bd62cf --- /dev/null +++ b/src/seb/cache/intfloat__e5-small/LCC.json @@ -0,0 +1 @@ +{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T12:00:41.266001","scores":{"da":{"accuracy":0.4026666666666666,"f1":0.3834992022087647,"accuracy_stderr":0.04697043987682277,"f1_stderr":0.038043224748645146,"main_score":0.4026666666666666}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-small/Language_Identification.json b/src/seb/cache/intfloat__e5-small/Language_Identification.json new file mode 100644 index 00000000..445737a8 --- /dev/null +++ b/src/seb/cache/intfloat__e5-small/Language_Identification.json @@ -0,0 +1 @@ +{"task_name":"Language Identification","task_description":"A dataset for Nordic language identification.","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T12:07:09.843358","scores":{"da":{"accuracy":0.5347333333333333,"f1":0.5137160081212816,"accuracy_stderr":0.015105407420302617,"f1_stderr":0.019860823827651492,"main_score":0.5347333333333333},"sv":{"accuracy":0.5347333333333333,"f1":0.5137160081212816,"accuracy_stderr":0.015105407420302617,"f1_stderr":0.019860823827651492,"main_score":0.5347333333333333},"nb":{"accuracy":0.5347333333333333,"f1":0.5137160081212816,"accuracy_stderr":0.015105407420302617,"f1_stderr":0.019860823827651492,"main_score":0.5347333333333333},"nn":{"accuracy":0.5347333333333333,"f1":0.5137160081212816,"accuracy_stderr":0.015105407420302617,"f1_stderr":0.019860823827651492,"main_score":0.5347333333333333},"is":{"accuracy":0.5347333333333333,"f1":0.5137160081212816,"accuracy_stderr":0.015105407420302617,"f1_stderr":0.019860823827651492,"main_score":0.5347333333333333},"fo":{"accuracy":0.5347333333333333,"f1":0.5137160081212816,"accuracy_stderr":0.015105407420302617,"f1_stderr":0.019860823827651492,"main_score":0.5347333333333333}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-small/Massive_Intent.json b/src/seb/cache/intfloat__e5-small/Massive_Intent.json new file mode 100644 index 00000000..d6cc4f22 --- /dev/null +++ b/src/seb/cache/intfloat__e5-small/Massive_Intent.json @@ -0,0 +1 @@ +{"task_name":"Massive Intent","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T12:03:54.357718","scores":{"da":{"accuracy":0.41893073301950234,"f1":0.3916826783567854,"accuracy_stderr":0.011477274617836132,"f1_stderr":0.012505345549790286,"main_score":0.41893073301950234},"nb":{"accuracy":0.40248823133826495,"f1":0.37594036649527024,"accuracy_stderr":0.01653777085051505,"f1_stderr":0.01620910514389266,"main_score":0.40248823133826495},"sv":{"accuracy":0.4007061197041022,"f1":0.37856957810370473,"accuracy_stderr":0.01818788065442142,"f1_stderr":0.014384271559521303,"main_score":0.4007061197041022}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-small/Massive_Scenario.json b/src/seb/cache/intfloat__e5-small/Massive_Scenario.json new file mode 100644 index 00000000..6e1fbedc --- /dev/null +++ b/src/seb/cache/intfloat__e5-small/Massive_Scenario.json @@ -0,0 +1 @@ +{"task_name":"Massive Scenario","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T12:04:59.054786","scores":{"da":{"accuracy":0.49926025554808345,"f1":0.47224689228546657,"accuracy_stderr":0.016129227599473626,"f1_stderr":0.018442568597481064,"main_score":0.49926025554808345},"nb":{"accuracy":0.4857767316745125,"f1":0.45651634870157637,"accuracy_stderr":0.019317146922155894,"f1_stderr":0.01685437076119252,"main_score":0.4857767316745125},"sv":{"accuracy":0.4706455951580363,"f1":0.44281581197784325,"accuracy_stderr":0.01540369478672511,"f1_stderr":0.01354462829679692,"main_score":0.4706455951580363}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-small/NoReC.json b/src/seb/cache/intfloat__e5-small/NoReC.json new file mode 100644 index 00000000..22f937ba --- /dev/null +++ b/src/seb/cache/intfloat__e5-small/NoReC.json @@ -0,0 +1 @@ +{"task_name":"NoReC","task_description":"A Norwegian dataset for sentiment classification on review","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T12:07:30.484166","scores":{"nb":{"accuracy":0.418359375,"f1":0.3910501878638593,"accuracy_stderr":0.01863392824555122,"f1_stderr":0.012803012731477874,"main_score":0.418359375}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-small/Norwegian_parliament.json b/src/seb/cache/intfloat__e5-small/Norwegian_parliament.json new file mode 100644 index 00000000..ee26319f --- /dev/null +++ b/src/seb/cache/intfloat__e5-small/Norwegian_parliament.json @@ -0,0 +1 @@ +{"task_name":"Norwegian parliament","task_description":"Norwegian parliament speeches annotated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T12:10:18.182722","scores":{"nb":{"accuracy":0.5656666666666667,"f1":0.5621976821470727,"ap":0.5375258937556727,"accuracy_stderr":0.02344852518461092,"f1_stderr":0.024627476178317884,"ap_stderr":0.014876487957062229,"main_score":0.5656666666666667}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-small/Scala.json b/src/seb/cache/intfloat__e5-small/Scala.json new file mode 100644 index 00000000..40129b3c --- /dev/null +++ b/src/seb/cache/intfloat__e5-small/Scala.json @@ -0,0 +1 @@ +{"task_name":"ScaLA","task_description":"A linguistic acceptability task for Danish, Norwegian Bokmål Norwegian Nynorsk and Swedish.","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T12:06:33.602892","scores":{"da":{"accuracy":0.50146484375,"f1":0.49714680612943984,"ap":0.5007564535242134,"accuracy_stderr":0.0051164792568084904,"f1_stderr":0.005960010796020731,"ap_stderr":0.0025841563282887636,"main_score":0.50146484375},"nb":{"accuracy":0.500341796875,"f1":0.4975895573014273,"ap":0.5001974054036913,"accuracy_stderr":0.005362430869059241,"f1_stderr":0.007651356339793306,"ap_stderr":0.002687674959550601,"main_score":0.500341796875},"sv":{"accuracy":0.497900390625,"f1":0.4944976634934668,"ap":0.49896998064152215,"accuracy_stderr":0.0038388347677736244,"f1_stderr":0.004516986774483215,"ap_stderr":0.0019135700844428744,"main_score":0.497900390625},"nn":{"accuracy":0.498876953125,"f1":0.49752134555861655,"ap":0.4994545410122539,"accuracy_stderr":0.003820157225404033,"f1_stderr":0.0035504584689772375,"ap_stderr":0.0019138256560218567,"main_score":0.498876953125}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-small/SweFAQ.json b/src/seb/cache/intfloat__e5-small/SweFAQ.json new file mode 100644 index 00000000..1f68601c --- /dev/null +++ b/src/seb/cache/intfloat__e5-small/SweFAQ.json @@ -0,0 +1 @@ +{"task_name":"SweFAQ","task_description":"A Swedish QA dataset derived from FAQ","task_version":"0.0.1","time_of_run":"2023-09-04T19:39:18.196596","scores":{"sv":{"ndcg_at_1":0.37232,"ndcg_at_3":0.4793,"ndcg_at_5":0.50899,"ndcg_at_10":0.53963,"ndcg_at_100":0.58258,"ndcg_at_1000":0.5939,"map_at_1":0.37232,"map_at_3":0.45354,"map_at_5":0.47001,"map_at_10":0.48288,"map_at_100":0.4924,"map_at_1000":0.49284,"recall_at_1":0.37232,"recall_at_3":0.55361,"recall_at_5":0.62573,"recall_at_10":0.7193,"recall_at_100":0.91228,"recall_at_1000":1.0,"precision_at_1":0.37232,"precision_at_3":0.18454,"precision_at_5":0.12515,"precision_at_10":0.07193,"precision_at_100":0.00912,"precision_at_1000":0.001,"mrr_at_1":0.37427,"mrr_at_3":0.45452,"mrr_at_5":0.47099,"mrr_at_10":0.48385,"mrr_at_100":0.49337,"mrr_at_1000":0.49382}},"main_score":"ndcg_at_10"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__e5-small/SweReC.json b/src/seb/cache/intfloat__e5-small/SweReC.json new file mode 100644 index 00000000..373fa0b0 --- /dev/null +++ b/src/seb/cache/intfloat__e5-small/SweReC.json @@ -0,0 +1 @@ +{"task_name":"SweReC","task_description":"A Swedish dataset for sentiment classification on review","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T12:11:47.951676","scores":{"sv":{"accuracy":0.609130859375,"f1":0.5303730149483024,"accuracy_stderr":0.02853856666232368,"f1_stderr":0.020179238627503652,"main_score":0.609130859375}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-base/Angry_Tweets.json b/src/seb/cache/intfloat__multilingual-e5-base/Angry_Tweets.json new file mode 100644 index 00000000..dd1d9c4f --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-base/Angry_Tweets.json @@ -0,0 +1 @@ +{"task_name":"Angry Tweets","task_description":"A sentiment dataset with 3 classes (positiv, negativ, neutral) for Danish tweets","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T16:36:53.648305","scores":{"da":{"accuracy":0.5629417382999045,"f1":0.5513172707177706,"accuracy_stderr":0.024987396579874234,"f1_stderr":0.021507910333896468,"main_score":0.5629417382999045}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-base/Bornholm_Parallel.json b/src/seb/cache/intfloat__multilingual-e5-base/Bornholm_Parallel.json new file mode 100644 index 00000000..5f1b8779 --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-base/Bornholm_Parallel.json @@ -0,0 +1 @@ +{"task_name":"Bornholm Parallel","task_description":"Danish Bornholmsk Parallel Corpus. Bornholmsk is a Danish dialect spoken on the island of Bornholm, Denmark. Historically it is a part of east Danish which was also spoken in Scania and Halland, Sweden.","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T16:37:18.80355","scores":{"da":{"precision":0.3094267638793955,"recall":0.408,"f1":0.3321808524808525,"accuracy":0.408,"main_score":0.3321808524808525},"da-bornholm":{"precision":0.3094267638793955,"recall":0.408,"f1":0.3321808524808525,"accuracy":0.408,"main_score":0.3321808524808525}},"main_score":"f1"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-base/DKHate.json b/src/seb/cache/intfloat__multilingual-e5-base/DKHate.json new file mode 100644 index 00000000..0dd9b6fa --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-base/DKHate.json @@ -0,0 +1 @@ +{"task_name":"DKHate","task_description":"Danish Tweets annotated for Hate Speech either being Offensive or not","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T16:37:54.103629","scores":{"da":{"accuracy":0.6382978723404256,"f1":0.5228565797183726,"ap":0.902024699314798,"accuracy_stderr":0.07190232851096628,"f1_stderr":0.05163702678835975,"ap_stderr":0.01326660366720444,"main_score":0.6382978723404256}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-base/DaLAJ.json b/src/seb/cache/intfloat__multilingual-e5-base/DaLAJ.json new file mode 100644 index 00000000..fc49c16d --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-base/DaLAJ.json @@ -0,0 +1 @@ +{"task_name":"DaLAJ","task_description":"A Swedish dataset for linguistic acceptability. Available as a part of Superlim.","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T17:56:06.54485","scores":{"sv":{"accuracy":0.4981981981981982,"f1":0.4940233228832834,"ap":0.4991502620553585,"accuracy_stderr":0.006213114515375547,"f1_stderr":0.00785886246257008,"ap_stderr":0.0030460807497184206,"main_score":0.4981981981981982}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-base/Da_Political_Comments.json b/src/seb/cache/intfloat__multilingual-e5-base/Da_Political_Comments.json new file mode 100644 index 00000000..e69b938a --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-base/Da_Political_Comments.json @@ -0,0 +1 @@ +{"task_name":"Da Political Comments","task_description":"A dataset of Danish political comments rated for sentiment","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T16:38:24.726421","scores":{"da":{"accuracy":0.3634850166481687,"f1":0.32742197410406404,"accuracy_stderr":0.03325654299734052,"f1_stderr":0.023934278827848342,"main_score":0.3634850166481687}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-base/LCC.json b/src/seb/cache/intfloat__multilingual-e5-base/LCC.json new file mode 100644 index 00000000..a71cec3b --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-base/LCC.json @@ -0,0 +1 @@ +{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T16:37:07.227512","scores":{"da":{"accuracy":0.6033333333333334,"f1":0.5999104895067112,"accuracy_stderr":0.03144660377352199,"f1_stderr":0.030428553203369315,"main_score":0.6033333333333334}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-base/Language_Identification.json b/src/seb/cache/intfloat__multilingual-e5-base/Language_Identification.json new file mode 100644 index 00000000..d7d8eb0f --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-base/Language_Identification.json @@ -0,0 +1 @@ +{"task_name":"Language Identification","task_description":"A dataset for Nordic language identification.","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T16:46:11.767498","scores":{"da":{"accuracy":0.7585999999999999,"f1":0.7523494474978634,"accuracy_stderr":0.011863857345362482,"f1_stderr":0.0128638527317893,"main_score":0.7585999999999999},"sv":{"accuracy":0.7585999999999999,"f1":0.7523494474978634,"accuracy_stderr":0.011863857345362482,"f1_stderr":0.0128638527317893,"main_score":0.7585999999999999},"nb":{"accuracy":0.7585999999999999,"f1":0.7523494474978634,"accuracy_stderr":0.011863857345362482,"f1_stderr":0.0128638527317893,"main_score":0.7585999999999999},"nn":{"accuracy":0.7585999999999999,"f1":0.7523494474978634,"accuracy_stderr":0.011863857345362482,"f1_stderr":0.0128638527317893,"main_score":0.7585999999999999},"is":{"accuracy":0.7585999999999999,"f1":0.7523494474978634,"accuracy_stderr":0.011863857345362482,"f1_stderr":0.0128638527317893,"main_score":0.7585999999999999},"fo":{"accuracy":0.7585999999999999,"f1":0.7523494474978634,"accuracy_stderr":0.011863857345362482,"f1_stderr":0.0128638527317893,"main_score":0.7585999999999999}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-base/Massive_Intent.json b/src/seb/cache/intfloat__multilingual-e5-base/Massive_Intent.json new file mode 100644 index 00000000..21309f88 --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-base/Massive_Intent.json @@ -0,0 +1 @@ +{"task_name":"Massive Intent","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T16:41:29.304607","scores":{"da":{"accuracy":0.6062205783456623,"f1":0.5689865700059744,"accuracy_stderr":0.017785838882924798,"f1_stderr":0.014917350460261347,"main_score":0.6062205783456623},"nb":{"accuracy":0.5997646267652993,"f1":0.5658332162535938,"accuracy_stderr":0.015078028296126048,"f1_stderr":0.014893568396365962,"main_score":0.5997646267652993},"sv":{"accuracy":0.6242434431741761,"f1":0.5880553337086173,"accuracy_stderr":0.01832116827487102,"f1_stderr":0.017015923001762353,"main_score":0.6242434431741761}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-base/Massive_Scenario.json b/src/seb/cache/intfloat__multilingual-e5-base/Massive_Scenario.json new file mode 100644 index 00000000..e8e61389 --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-base/Massive_Scenario.json @@ -0,0 +1 @@ +{"task_name":"Massive Scenario","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T16:43:00.762179","scores":{"da":{"accuracy":0.6796570275722933,"f1":0.6641354513881321,"accuracy_stderr":0.016332111492628056,"f1_stderr":0.018540512366687925,"main_score":0.6796570275722933},"nb":{"accuracy":0.6650638870208473,"f1":0.6521994400105137,"accuracy_stderr":0.01802013574776764,"f1_stderr":0.015684749070903847,"main_score":0.6650638870208473},"sv":{"accuracy":0.6930060524546067,"f1":0.6763732947857418,"accuracy_stderr":0.01817354630196441,"f1_stderr":0.019492714112551015,"main_score":0.6930060524546067}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-base/NoReC.json b/src/seb/cache/intfloat__multilingual-e5-base/NoReC.json new file mode 100644 index 00000000..1f1a2f9c --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-base/NoReC.json @@ -0,0 +1 @@ +{"task_name":"NoReC","task_description":"A Norwegian dataset for sentiment classification on review","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T16:46:45.708846","scores":{"nb":{"accuracy":0.589892578125,"f1":0.5766678720243498,"accuracy_stderr":0.01891414004801972,"f1_stderr":0.017278312203201952,"main_score":0.589892578125}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-base/Norwegian_parliament.json b/src/seb/cache/intfloat__multilingual-e5-base/Norwegian_parliament.json new file mode 100644 index 00000000..acebc681 --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-base/Norwegian_parliament.json @@ -0,0 +1 @@ +{"task_name":"Norwegian parliament","task_description":"Norwegian parliament speeches annotated for sentiment","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T16:50:35.467123","scores":{"nb":{"accuracy":0.5958333333333333,"f1":0.5938705573261827,"ap":0.5581136181374919,"accuracy_stderr":0.024324199198877357,"f1_stderr":0.024753645615771615,"ap_stderr":0.01771348601229892,"main_score":0.5958333333333333}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-base/ScaLA.json b/src/seb/cache/intfloat__multilingual-e5-base/ScaLA.json new file mode 100644 index 00000000..2816694d --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-base/ScaLA.json @@ -0,0 +1 @@ +{"task_name":"ScaLA","task_description":"A linguistic acceptability task for Danish, Norwegian Bokmål Norwegian Nynorsk and Swedish.","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T16:45:07.312295","scores":{"da":{"accuracy":0.50673828125,"f1":0.5035767046866525,"ap":0.5035243081073488,"accuracy_stderr":0.009595192468411427,"f1_stderr":0.009243292175432646,"ap_stderr":0.004909569318594445,"main_score":0.50673828125},"nb":{"accuracy":0.502392578125,"f1":0.4982194666611958,"ap":0.501254074929416,"accuracy_stderr":0.0074707031249999995,"f1_stderr":0.008043441095830797,"ap_stderr":0.0037513686612729163,"main_score":0.502392578125},"sv":{"accuracy":0.503759765625,"f1":0.5009915979641681,"ap":0.5019932178718128,"accuracy_stderr":0.009743749913869568,"f1_stderr":0.011107709893542246,"ap_stderr":0.004932863893043515,"main_score":0.503759765625},"nn":{"accuracy":0.505419921875,"f1":0.5033920658634355,"ap":0.5028718352203354,"accuracy_stderr":0.010954162983816948,"f1_stderr":0.012034642067722754,"ap_stderr":0.005576471535713489,"main_score":0.505419921875}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-base/SweFAQ.json b/src/seb/cache/intfloat__multilingual-e5-base/SweFAQ.json new file mode 100644 index 00000000..f77cb1a9 --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-base/SweFAQ.json @@ -0,0 +1 @@ +{"task_name":"SweFAQ","task_description":"A Swedish QA dataset derived from FAQ","task_version":"0.0.1","time_of_run":"2023-09-25T17:56:48.579561","scores":{"sv":{"ndcg_at_1":0.51072,"ndcg_at_3":0.64072,"ndcg_at_5":0.66522,"ndcg_at_10":0.69491,"ndcg_at_100":0.71858,"ndcg_at_1000":0.71934,"map_at_1":0.51072,"map_at_3":0.60884,"map_at_5":0.62268,"map_at_10":0.63495,"map_at_100":0.63977,"map_at_1000":0.6398,"recall_at_1":0.51072,"recall_at_3":0.73294,"recall_at_5":0.79142,"recall_at_10":0.88304,"recall_at_100":0.99415,"recall_at_1000":1.0,"precision_at_1":0.51072,"precision_at_3":0.24431,"precision_at_5":0.15828,"precision_at_10":0.0883,"precision_at_100":0.00994,"precision_at_1000":0.001,"mrr_at_1":0.51072,"mrr_at_3":0.60884,"mrr_at_5":0.62268,"mrr_at_10":0.63495,"mrr_at_100":0.63977,"mrr_at_1000":0.6398}},"main_score":"ndcg_at_10"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-base/SweReC.json b/src/seb/cache/intfloat__multilingual-e5-base/SweReC.json new file mode 100644 index 00000000..1e03b4f8 --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-base/SweReC.json @@ -0,0 +1 @@ +{"task_name":"SweReC","task_description":"A Swedish dataset for sentiment classification on review","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T17:55:47.079738","scores":{"sv":{"accuracy":0.80166015625,"f1":0.7060055269481745,"accuracy_stderr":0.016791196351134238,"f1_stderr":0.01604693868094686,"main_score":0.80166015625}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-large/Angry_Tweets.json b/src/seb/cache/intfloat__multilingual-e5-large/Angry_Tweets.json new file mode 100644 index 00000000..1455a2bf --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-large/Angry_Tweets.json @@ -0,0 +1 @@ +{"task_name":"Angry Tweets","task_description":"A sentiment dataset with 3 classes (positiv, negativ, neutral) for Danish tweets","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T17:58:19.03926","scores":{"da":{"accuracy":0.5768863419293219,"f1":0.5647943761418569,"accuracy_stderr":0.02172953597029208,"f1_stderr":0.020317847788255346,"main_score":0.5768863419293219}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-large/Bornholm_Parallel.json b/src/seb/cache/intfloat__multilingual-e5-large/Bornholm_Parallel.json new file mode 100644 index 00000000..e5dfa183 --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-large/Bornholm_Parallel.json @@ -0,0 +1 @@ +{"task_name":"Bornholm Parallel","task_description":"Danish Bornholmsk Parallel Corpus. Bornholmsk is a Danish dialect spoken on the island of Bornholm, Denmark. Historically it is a part of east Danish which was also spoken in Scania and Halland, Sweden.","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T17:59:22.23214","scores":{"da":{"precision":0.2760605422355422,"recall":0.368,"f1":0.29607949436184733,"accuracy":0.368,"main_score":0.29607949436184733},"da-bornholm":{"precision":0.2760605422355422,"recall":0.368,"f1":0.29607949436184733,"accuracy":0.368,"main_score":0.29607949436184733}},"main_score":"f1"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-large/DKHate.json b/src/seb/cache/intfloat__multilingual-e5-large/DKHate.json new file mode 100644 index 00000000..d3e4d5f1 --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-large/DKHate.json @@ -0,0 +1 @@ +{"task_name":"DKHate","task_description":"Danish Tweets annotated for Hate Speech either being Offensive or not","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T18:00:59.068556","scores":{"da":{"accuracy":0.6620060790273558,"f1":0.5449710348807391,"ap":0.9085362159340036,"accuracy_stderr":0.07768083625694755,"f1_stderr":0.05859252718657472,"ap_stderr":0.014254769906843061,"main_score":0.6620060790273558}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-large/DaLAJ.json b/src/seb/cache/intfloat__multilingual-e5-large/DaLAJ.json new file mode 100644 index 00000000..55ed8026 --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-large/DaLAJ.json @@ -0,0 +1 @@ +{"task_name":"DaLAJ","task_description":"A Swedish dataset for linguistic acceptability. Available as a part of Superlim.","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T19:05:50.728747","scores":{"sv":{"accuracy":0.49864864864864866,"f1":0.49597763331842126,"ap":0.49934757586394285,"accuracy_stderr":0.004664935850722515,"f1_stderr":0.005890028014980534,"ap_stderr":0.002302457357180006,"main_score":0.49864864864864866}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-large/Da_Political_Comments.json b/src/seb/cache/intfloat__multilingual-e5-large/Da_Political_Comments.json new file mode 100644 index 00000000..9f736bc0 --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-large/Da_Political_Comments.json @@ -0,0 +1 @@ +{"task_name":"Da Political Comments","task_description":"A dataset of Danish political comments rated for sentiment","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T18:02:22.96372","scores":{"da":{"accuracy":0.39650388457269703,"f1":0.36524945371981243,"accuracy_stderr":0.035201025149546365,"f1_stderr":0.024136133587464075,"main_score":0.39650388457269703}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-large/LCC.json b/src/seb/cache/intfloat__multilingual-e5-large/LCC.json new file mode 100644 index 00000000..beb72f2e --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-large/LCC.json @@ -0,0 +1 @@ +{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T17:58:52.979095","scores":{"da":{"accuracy":0.6166666666666666,"f1":0.6164836495996562,"accuracy_stderr":0.04716401641553062,"f1_stderr":0.04790241759343871,"main_score":0.6166666666666666}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-large/Language_Identification.json b/src/seb/cache/intfloat__multilingual-e5-large/Language_Identification.json new file mode 100644 index 00000000..d0a9e274 --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-large/Language_Identification.json @@ -0,0 +1 @@ +{"task_name":"Language Identification","task_description":"A dataset for Nordic language identification.","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T18:48:10.764992","scores":{"da":{"accuracy":0.8015333333333332,"f1":0.796288598455154,"accuracy_stderr":0.007533923280734938,"f1_stderr":0.009056161875406527,"main_score":0.8015333333333332},"sv":{"accuracy":0.8015333333333332,"f1":0.796288598455154,"accuracy_stderr":0.007533923280734938,"f1_stderr":0.009056161875406527,"main_score":0.8015333333333332},"nb":{"accuracy":0.8015333333333332,"f1":0.796288598455154,"accuracy_stderr":0.007533923280734938,"f1_stderr":0.009056161875406527,"main_score":0.8015333333333332},"nn":{"accuracy":0.8015333333333332,"f1":0.796288598455154,"accuracy_stderr":0.007533923280734938,"f1_stderr":0.009056161875406527,"main_score":0.8015333333333332},"is":{"accuracy":0.8015333333333332,"f1":0.796288598455154,"accuracy_stderr":0.007533923280734938,"f1_stderr":0.009056161875406527,"main_score":0.8015333333333332},"fo":{"accuracy":0.8015333333333332,"f1":0.796288598455154,"accuracy_stderr":0.007533923280734938,"f1_stderr":0.009056161875406527,"main_score":0.8015333333333332}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-large/Massive_Intent.json b/src/seb/cache/intfloat__multilingual-e5-large/Massive_Intent.json new file mode 100644 index 00000000..a22872dd --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-large/Massive_Intent.json @@ -0,0 +1 @@ +{"task_name":"Massive Intent","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T18:09:37.713047","scores":{"da":{"accuracy":0.6374915938130465,"f1":0.6095209823689112,"accuracy_stderr":0.013080372650990131,"f1_stderr":0.0123105848109145,"main_score":0.6374915938130465},"nb":{"accuracy":0.6451244115669132,"f1":0.6151567689298606,"accuracy_stderr":0.017947679649994,"f1_stderr":0.01787152266902865,"main_score":0.6451244115669132},"sv":{"accuracy":0.665366509751177,"f1":0.6404515575021981,"accuracy_stderr":0.018444515132656283,"f1_stderr":0.016067270041325344,"main_score":0.665366509751177}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-large/Massive_Scenario.json b/src/seb/cache/intfloat__multilingual-e5-large/Massive_Scenario.json new file mode 100644 index 00000000..e8b20aa5 --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-large/Massive_Scenario.json @@ -0,0 +1 @@ +{"task_name":"Massive Scenario","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T18:14:08.314072","scores":{"da":{"accuracy":0.7114660390047074,"f1":0.7023253062945793,"accuracy_stderr":0.01512399891431203,"f1_stderr":0.016115474706742145,"main_score":0.7114660390047074},"nb":{"accuracy":0.7039677202420982,"f1":0.6941514719389039,"accuracy_stderr":0.01835751105836139,"f1_stderr":0.01624822992478041,"main_score":0.7039677202420982},"sv":{"accuracy":0.7270342972427708,"f1":0.7202597856696572,"accuracy_stderr":0.013923893984056305,"f1_stderr":0.013290618471430102,"main_score":0.7270342972427708}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-large/NoReC.json b/src/seb/cache/intfloat__multilingual-e5-large/NoReC.json new file mode 100644 index 00000000..6eaf00e9 --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-large/NoReC.json @@ -0,0 +1 @@ +{"task_name":"NoReC","task_description":"A Norwegian dataset for sentiment classification on review","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T18:49:43.239486","scores":{"nb":{"accuracy":0.635302734375,"f1":0.624589982402441,"accuracy_stderr":0.018167015573959912,"f1_stderr":0.016979462240002116,"main_score":0.635302734375}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-large/Norwegian_parliament.json b/src/seb/cache/intfloat__multilingual-e5-large/Norwegian_parliament.json new file mode 100644 index 00000000..209761b1 --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-large/Norwegian_parliament.json @@ -0,0 +1 @@ +{"task_name":"Norwegian parliament","task_description":"Norwegian parliament speeches annotated for sentiment","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T18:59:38.929172","scores":{"nb":{"accuracy":0.6029166666666668,"f1":0.5993592454121626,"ap":0.5635563826522597,"accuracy_stderr":0.033712942618525595,"f1_stderr":0.035908065389925005,"ap_stderr":0.024613513455923747,"main_score":0.6029166666666668}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-large/ScaLA.json b/src/seb/cache/intfloat__multilingual-e5-large/ScaLA.json new file mode 100644 index 00000000..08f5ba95 --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-large/ScaLA.json @@ -0,0 +1 @@ +{"task_name":"ScaLA","task_description":"A linguistic acceptability task for Danish, Norwegian Bokmål Norwegian Nynorsk and Swedish.","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T18:45:30.505102","scores":{"da":{"accuracy":0.5140625,"f1":0.5110096350178809,"ap":0.5074025241027542,"accuracy_stderr":0.013737634820944719,"f1_stderr":0.013200959370182407,"ap_stderr":0.007105907107278635,"main_score":0.5140625},"nb":{"accuracy":0.509326171875,"f1":0.5060070471323789,"ap":0.504796826492099,"accuracy_stderr":0.006717905390680548,"f1_stderr":0.008867768768255003,"ap_stderr":0.003495226892618272,"main_score":0.509326171875},"sv":{"accuracy":0.511669921875,"f1":0.5091068656627107,"ap":0.506073277397398,"accuracy_stderr":0.009245035230983122,"f1_stderr":0.009297444141790184,"ap_stderr":0.00484964434326875,"main_score":0.511669921875},"nn":{"accuracy":0.51279296875,"f1":0.5103750460870845,"ap":0.5067093378763262,"accuracy_stderr":0.012877671640474563,"f1_stderr":0.01229717065979964,"ap_stderr":0.006523427651546832,"main_score":0.51279296875}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-large/SweFAQ.json b/src/seb/cache/intfloat__multilingual-e5-large/SweFAQ.json new file mode 100644 index 00000000..3b928f78 --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-large/SweFAQ.json @@ -0,0 +1 @@ +{"task_name":"SweFAQ","task_description":"A Swedish QA dataset derived from FAQ","task_version":"0.0.1","time_of_run":"2023-09-25T19:07:51.256832","scores":{"sv":{"ndcg_at_1":0.54386,"ndcg_at_3":0.68179,"ndcg_at_5":0.71627,"ndcg_at_10":0.733,"ndcg_at_100":0.75143,"ndcg_at_1000":0.75255,"map_at_1":0.54386,"map_at_3":0.64912,"map_at_5":0.66823,"map_at_10":0.67534,"map_at_100":0.67955,"map_at_1000":0.67961,"recall_at_1":0.54386,"recall_at_3":0.77583,"recall_at_5":0.85965,"recall_at_10":0.91033,"recall_at_100":0.9922,"recall_at_1000":1.0,"precision_at_1":0.54386,"precision_at_3":0.25861,"precision_at_5":0.17193,"precision_at_10":0.09103,"precision_at_100":0.00992,"precision_at_1000":0.001,"mrr_at_1":0.54386,"mrr_at_3":0.64912,"mrr_at_5":0.66823,"mrr_at_10":0.67534,"mrr_at_100":0.67955,"mrr_at_1000":0.67961}},"main_score":"ndcg_at_10"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-large/SweReC.json b/src/seb/cache/intfloat__multilingual-e5-large/SweReC.json new file mode 100644 index 00000000..a625052c --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-large/SweReC.json @@ -0,0 +1 @@ +{"task_name":"SweReC","task_description":"A Swedish dataset for sentiment classification on review","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T19:04:50.226041","scores":{"sv":{"accuracy":0.8337890625,"f1":0.7464490374037602,"accuracy_stderr":0.014170591521263324,"f1_stderr":0.013660066633469786,"main_score":0.8337890625}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-small/Angry_Tweets.json b/src/seb/cache/intfloat__multilingual-e5-small/Angry_Tweets.json new file mode 100644 index 00000000..6c7d56b3 --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-small/Angry_Tweets.json @@ -0,0 +1 @@ +{"task_name":"Angry Tweets","task_description":"A sentiment dataset with 3 classes (positiv, negativ, neutral) for Danish tweets","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T16:21:17.31627","scores":{"da":{"accuracy":0.5622731614135625,"f1":0.552564695046545,"accuracy_stderr":0.021267231534685977,"f1_stderr":0.0187870312606918,"main_score":0.5622731614135625}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-small/Bornholm_Parallel.json b/src/seb/cache/intfloat__multilingual-e5-small/Bornholm_Parallel.json new file mode 100644 index 00000000..734b3946 --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-small/Bornholm_Parallel.json @@ -0,0 +1 @@ +{"task_name":"Bornholm Parallel","task_description":"Danish Bornholmsk Parallel Corpus. Bornholmsk is a Danish dialect spoken on the island of Bornholm, Denmark. Historically it is a part of east Danish which was also spoken in Scania and Halland, Sweden.","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T16:21:33.675592","scores":{"da":{"precision":0.3450174603174603,"recall":0.446,"f1":0.37145728715728715,"accuracy":0.446,"main_score":0.37145728715728715},"da-bornholm":{"precision":0.3450174603174603,"recall":0.446,"f1":0.37145728715728715,"accuracy":0.446,"main_score":0.37145728715728715}},"main_score":"f1"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-small/DKHate.json b/src/seb/cache/intfloat__multilingual-e5-small/DKHate.json new file mode 100644 index 00000000..434e2fbf --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-small/DKHate.json @@ -0,0 +1 @@ +{"task_name":"DKHate","task_description":"Danish Tweets annotated for Hate Speech either being Offensive or not","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T16:21:57.740156","scores":{"da":{"accuracy":0.6240121580547113,"f1":0.5134789106104648,"ap":0.1654729011202576,"accuracy_stderr":0.05892346370719707,"f1_stderr":0.05043451072105682,"ap_stderr":0.031277058821955694,"main_score":0.6240121580547113}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-small/DaLAJ.json b/src/seb/cache/intfloat__multilingual-e5-small/DaLAJ.json new file mode 100644 index 00000000..90c959c0 --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-small/DaLAJ.json @@ -0,0 +1 @@ +{"task_name":"DaLAJ","task_description":"A Swedish dataset for linguistic acceptability. Available as a part of Superlim.","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T16:30:35.329722","scores":{"sv":{"accuracy":0.49988738738738736,"f1":0.4963700769394328,"ap":0.4999738889848798,"accuracy_stderr":0.005147024721633538,"f1_stderr":0.007611287661401983,"ap_stderr":0.00256661112462305,"main_score":0.49988738738738736}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-small/Da_Political_Comments.json b/src/seb/cache/intfloat__multilingual-e5-small/Da_Political_Comments.json new file mode 100644 index 00000000..e5558923 --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-small/Da_Political_Comments.json @@ -0,0 +1 @@ +{"task_name":"Da Political Comments","task_description":"A dataset of Danish political comments rated for sentiment","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T16:22:16.037763","scores":{"da":{"accuracy":0.3469478357380688,"f1":0.3118942408677866,"accuracy_stderr":0.03210100025238275,"f1_stderr":0.02125984767681479,"main_score":0.3469478357380688}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-small/LCC.json b/src/seb/cache/intfloat__multilingual-e5-small/LCC.json new file mode 100644 index 00000000..f59b5f25 --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-small/LCC.json @@ -0,0 +1 @@ +{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T16:21:26.061052","scores":{"da":{"accuracy":0.5853333333333334,"f1":0.5748383064284881,"accuracy_stderr":0.03303869784897031,"f1_stderr":0.029575180263160184,"main_score":0.5853333333333334}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-small/Language_Identification.json b/src/seb/cache/intfloat__multilingual-e5-small/Language_Identification.json new file mode 100644 index 00000000..c623ec33 --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-small/Language_Identification.json @@ -0,0 +1 @@ +{"task_name":"Language Identification","task_description":"A dataset for Nordic language identification.","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T16:26:54.975504","scores":{"da":{"accuracy":0.7213666666666667,"f1":0.7074068687868917,"accuracy_stderr":0.010001610981348075,"f1_stderr":0.011658643942785296,"main_score":0.7213666666666667},"sv":{"accuracy":0.7213666666666667,"f1":0.7074068687868917,"accuracy_stderr":0.010001610981348075,"f1_stderr":0.011658643942785296,"main_score":0.7213666666666667},"nb":{"accuracy":0.7213666666666667,"f1":0.7074068687868917,"accuracy_stderr":0.010001610981348075,"f1_stderr":0.011658643942785296,"main_score":0.7213666666666667},"nn":{"accuracy":0.7213666666666667,"f1":0.7074068687868917,"accuracy_stderr":0.010001610981348075,"f1_stderr":0.011658643942785296,"main_score":0.7213666666666667},"is":{"accuracy":0.7213666666666667,"f1":0.7074068687868917,"accuracy_stderr":0.010001610981348075,"f1_stderr":0.011658643942785296,"main_score":0.7213666666666667},"fo":{"accuracy":0.7213666666666667,"f1":0.7074068687868917,"accuracy_stderr":0.010001610981348075,"f1_stderr":0.011658643942785296,"main_score":0.7213666666666667}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-small/Massive_Intent.json b/src/seb/cache/intfloat__multilingual-e5-small/Massive_Intent.json new file mode 100644 index 00000000..da81f7ad --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-small/Massive_Intent.json @@ -0,0 +1 @@ +{"task_name":"Massive Intent","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T16:24:17.218706","scores":{"da":{"accuracy":0.5618022864828515,"f1":0.5265687606969173,"accuracy_stderr":0.016711059897682973,"f1_stderr":0.014590491597714576,"main_score":0.5618022864828515},"nb":{"accuracy":0.5528581035642233,"f1":0.5133709916110052,"accuracy_stderr":0.017036037463567855,"f1_stderr":0.018540015613397447,"main_score":0.5528581035642233},"sv":{"accuracy":0.5820107599193006,"f1":0.5505317590609262,"accuracy_stderr":0.01788220234514548,"f1_stderr":0.01399218317021721,"main_score":0.5820107599193006}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-small/Massive_Scenario.json b/src/seb/cache/intfloat__multilingual-e5-small/Massive_Scenario.json new file mode 100644 index 00000000..a602276c --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-small/Massive_Scenario.json @@ -0,0 +1 @@ +{"task_name":"Massive Scenario","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T16:25:12.317164","scores":{"da":{"accuracy":0.6396435776731674,"f1":0.6208377992895038,"accuracy_stderr":0.020534262111544202,"f1_stderr":0.02176003783576135,"main_score":0.6396435776731674},"nb":{"accuracy":0.6193678547410896,"f1":0.6034263632375385,"accuracy_stderr":0.026066879570464923,"f1_stderr":0.021518488323420674,"main_score":0.6193678547410896},"sv":{"accuracy":0.6729657027572294,"f1":0.6595949478851747,"accuracy_stderr":0.020559656760484404,"f1_stderr":0.018001949274946347,"main_score":0.6729657027572294}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-small/NoReC.json b/src/seb/cache/intfloat__multilingual-e5-small/NoReC.json new file mode 100644 index 00000000..1fed4e24 --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-small/NoReC.json @@ -0,0 +1 @@ +{"task_name":"NoReC","task_description":"A Norwegian dataset for sentiment classification on review","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T16:27:15.303383","scores":{"nb":{"accuracy":0.544970703125,"f1":0.525990808528716,"accuracy_stderr":0.022488996146167874,"f1_stderr":0.02002790132721357,"main_score":0.544970703125}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-small/Norwegian_parliament.json b/src/seb/cache/intfloat__multilingual-e5-small/Norwegian_parliament.json new file mode 100644 index 00000000..0561ccba --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-small/Norwegian_parliament.json @@ -0,0 +1 @@ +{"task_name":"Norwegian parliament","task_description":"Norwegian parliament speeches annotated for sentiment","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T16:29:18.991097","scores":{"nb":{"accuracy":0.5998333333333333,"f1":0.5978983661692413,"ap":0.5603565023880613,"accuracy_stderr":0.02947880594596733,"f1_stderr":0.029200965335298316,"ap_stderr":0.02017863154673315,"main_score":0.5998333333333333}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-small/ScaLA.json b/src/seb/cache/intfloat__multilingual-e5-small/ScaLA.json new file mode 100644 index 00000000..b288c862 --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-small/ScaLA.json @@ -0,0 +1 @@ +{"task_name":"ScaLA","task_description":"A linguistic acceptability task for Danish, Norwegian Bokmål Norwegian Nynorsk and Swedish.","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T16:26:25.438391","scores":{"da":{"accuracy":0.50576171875,"f1":0.5010609456401955,"ap":0.503060996904327,"accuracy_stderr":0.01037824538613931,"f1_stderr":0.009451854744021831,"ap_stderr":0.005370986336965475,"main_score":0.50576171875},"nb":{"accuracy":0.503759765625,"f1":0.49844455758079026,"ap":0.5019331997328252,"accuracy_stderr":0.006503678967969452,"f1_stderr":0.009730920884692986,"ap_stderr":0.003327293677784496,"main_score":0.503759765625},"sv":{"accuracy":0.502294921875,"f1":0.49781050318128484,"ap":0.5012304738997291,"accuracy_stderr":0.00840652431338268,"f1_stderr":0.010033053608518689,"ap_stderr":0.004238295990141352,"main_score":0.502294921875},"nn":{"accuracy":0.50166015625,"f1":0.49900675779022236,"ap":0.5008715417815609,"accuracy_stderr":0.006157766707200956,"f1_stderr":0.0073065604950670274,"ap_stderr":0.0030813173899733644,"main_score":0.50166015625}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-small/SweFAQ.json b/src/seb/cache/intfloat__multilingual-e5-small/SweFAQ.json new file mode 100644 index 00000000..83fecb8c --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-small/SweFAQ.json @@ -0,0 +1 @@ +{"task_name":"SweFAQ","task_description":"A Swedish QA dataset derived from FAQ","task_version":"0.0.1","time_of_run":"2023-09-25T16:30:58.406932","scores":{"sv":{"ndcg_at_1":0.49708,"ndcg_at_3":0.64204,"ndcg_at_5":0.66577,"ndcg_at_10":0.68724,"ndcg_at_100":0.7119,"ndcg_at_1000":0.71418,"map_at_1":0.49708,"map_at_3":0.60656,"map_at_5":0.61953,"map_at_10":0.6284,"map_at_100":0.63369,"map_at_1000":0.63378,"recall_at_1":0.49708,"recall_at_3":0.74464,"recall_at_5":0.80312,"recall_at_10":0.8694,"recall_at_100":0.98246,"recall_at_1000":1.0,"precision_at_1":0.49708,"precision_at_3":0.24821,"precision_at_5":0.16062,"precision_at_10":0.08694,"precision_at_100":0.00982,"precision_at_1000":0.001,"mrr_at_1":0.49708,"mrr_at_3":0.60656,"mrr_at_5":0.61953,"mrr_at_10":0.6284,"mrr_at_100":0.63369,"mrr_at_1000":0.63378}},"main_score":"ndcg_at_10"} \ No newline at end of file diff --git a/src/seb/cache/intfloat__multilingual-e5-small/SweReC.json b/src/seb/cache/intfloat__multilingual-e5-small/SweReC.json new file mode 100644 index 00000000..0bde1fdc --- /dev/null +++ b/src/seb/cache/intfloat__multilingual-e5-small/SweReC.json @@ -0,0 +1 @@ +{"task_name":"SweReC","task_description":"A Swedish dataset for sentiment classification on review","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T16:30:22.373024","scores":{"sv":{"accuracy":0.7736328125,"f1":0.6685307663136847,"accuracy_stderr":0.013814476734790018,"f1_stderr":0.013415545434145465,"main_score":0.7736328125}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/jonfd__electra-small-nordic/Angry_Tweets.json b/src/seb/cache/jonfd__electra-small-nordic/Angry_Tweets.json new file mode 100644 index 00000000..fad50c32 --- /dev/null +++ b/src/seb/cache/jonfd__electra-small-nordic/Angry_Tweets.json @@ -0,0 +1 @@ +{"task_name":"Angry Tweets","task_description":"A sentiment dataset with 3 classes (positiv, negativ, neutral) for Danish tweets","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:32:48.850097","scores":{"da":{"accuracy":0.47908309455587395,"f1":0.468231508172157,"accuracy_stderr":0.02190598080918979,"f1_stderr":0.019450397227134677,"main_score":0.47908309455587395}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/jonfd__electra-small-nordic/Bornholm_Parallel.json b/src/seb/cache/jonfd__electra-small-nordic/Bornholm_Parallel.json new file mode 100644 index 00000000..53afcd3f --- /dev/null +++ b/src/seb/cache/jonfd__electra-small-nordic/Bornholm_Parallel.json @@ -0,0 +1 @@ +{"task_name":"Bornholm Parallel","task_description":"Danish Bornholmsk Parallel Corpus. Bornholmsk is a Danish dialect spoken on the island of Bornholm, Denmark. Historically it is a part of east Danish which was also spoken in Scania and Halland, Sweden.","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:32:58.712297","scores":{"da":{"precision":0.01242835966922654,"recall":0.028,"f1":0.01438003170260352,"accuracy":0.028,"main_score":0.01438003170260352},"da-bornholm":{"precision":0.01242835966922654,"recall":0.028,"f1":0.01438003170260352,"accuracy":0.028,"main_score":0.01438003170260352}},"main_score":"f1"} \ No newline at end of file diff --git a/src/seb/cache/jonfd__electra-small-nordic/DKHate.json b/src/seb/cache/jonfd__electra-small-nordic/DKHate.json new file mode 100644 index 00000000..80fd35e7 --- /dev/null +++ b/src/seb/cache/jonfd__electra-small-nordic/DKHate.json @@ -0,0 +1 @@ +{"task_name":"DKHate","task_description":"Danish Tweets annotated for Hate Speech either being Offensive or not","task_version":"1.0.3.dev0","time_of_run":"2023-07-30T13:55:38.480327","scores":{"da":{"accuracy":0.5945288753799393,"f1":0.4912211182797449,"ap":0.8950480900418238,"accuracy_stderr":0.07818347662767612,"f1_stderr":0.05511334661624392,"ap_stderr":0.013877821318913264,"main_score":0.5945288753799393}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/jonfd__electra-small-nordic/DaLAJ.json b/src/seb/cache/jonfd__electra-small-nordic/DaLAJ.json new file mode 100644 index 00000000..68fc4176 --- /dev/null +++ b/src/seb/cache/jonfd__electra-small-nordic/DaLAJ.json @@ -0,0 +1 @@ +{"task_name":"DaLAJ","task_description":"A Swedish dataset for linguistic accebtablity. Available as a part of Superlim.","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:37:18.27137","scores":{"sv":{"accuracy":0.558445945945946,"f1":0.5562253573135036,"ap":0.5346527347661278,"accuracy_stderr":0.04445688652339418,"f1_stderr":0.0462439502665578,"ap_stderr":0.027366913086196886,"main_score":0.558445945945946}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/jonfd__electra-small-nordic/Da_Political_Comments.json b/src/seb/cache/jonfd__electra-small-nordic/Da_Political_Comments.json new file mode 100644 index 00000000..2f6ace9b --- /dev/null +++ b/src/seb/cache/jonfd__electra-small-nordic/Da_Political_Comments.json @@ -0,0 +1 @@ +{"task_name":"Da Political Comments","task_description":"A dataset of Danish political comments rated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:33:10.658513","scores":{"da":{"accuracy":0.3147058823529411,"f1":0.27953578094645204,"accuracy_stderr":0.024836788827231376,"f1_stderr":0.015763450167311996,"main_score":0.3147058823529411}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/jonfd__electra-small-nordic/LCC.json b/src/seb/cache/jonfd__electra-small-nordic/LCC.json new file mode 100644 index 00000000..5c412e62 --- /dev/null +++ b/src/seb/cache/jonfd__electra-small-nordic/LCC.json @@ -0,0 +1 @@ +{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:32:53.734705","scores":{"da":{"accuracy":0.4793333333333334,"f1":0.4561594764023352,"accuracy_stderr":0.03405224743765902,"f1_stderr":0.03850197007453869,"main_score":0.4793333333333334}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/jonfd__electra-small-nordic/Language_Identification.json b/src/seb/cache/jonfd__electra-small-nordic/Language_Identification.json new file mode 100644 index 00000000..9a71f99f --- /dev/null +++ b/src/seb/cache/jonfd__electra-small-nordic/Language_Identification.json @@ -0,0 +1 @@ +{"task_name":"Language Identification","task_description":"A dataset for Nordic language identification.","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:36:16.178463","scores":{"da":{"accuracy":0.5781999999999999,"f1":0.5741478053962179,"accuracy_stderr":0.017301380805524676,"f1_stderr":0.01943559028700142,"main_score":0.5781999999999999},"sv":{"accuracy":0.5781999999999999,"f1":0.5741478053962179,"accuracy_stderr":0.017301380805524676,"f1_stderr":0.01943559028700142,"main_score":0.5781999999999999},"nb":{"accuracy":0.5781999999999999,"f1":0.5741478053962179,"accuracy_stderr":0.017301380805524676,"f1_stderr":0.01943559028700142,"main_score":0.5781999999999999},"nn":{"accuracy":0.5781999999999999,"f1":0.5741478053962179,"accuracy_stderr":0.017301380805524676,"f1_stderr":0.01943559028700142,"main_score":0.5781999999999999},"is":{"accuracy":0.5781999999999999,"f1":0.5741478053962179,"accuracy_stderr":0.017301380805524676,"f1_stderr":0.01943559028700142,"main_score":0.5781999999999999},"fo":{"accuracy":0.5781999999999999,"f1":0.5741478053962179,"accuracy_stderr":0.017301380805524676,"f1_stderr":0.01943559028700142,"main_score":0.5781999999999999}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/jonfd__electra-small-nordic/Massive_Intent.json b/src/seb/cache/jonfd__electra-small-nordic/Massive_Intent.json new file mode 100644 index 00000000..bc99697b --- /dev/null +++ b/src/seb/cache/jonfd__electra-small-nordic/Massive_Intent.json @@ -0,0 +1 @@ +{"task_name":"Massive Intent","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:34:20.033507","scores":{"da":{"accuracy":0.2629791526563551,"f1":0.22980369734119951,"accuracy_stderr":0.008387942541855315,"f1_stderr":0.007579204651192388,"main_score":0.2629791526563551},"nb":{"accuracy":0.24603227975790182,"f1":0.21966728945303612,"accuracy_stderr":0.011219838214051958,"f1_stderr":0.011543763578443968,"main_score":0.24603227975790182},"sv":{"accuracy":0.27575655682582384,"f1":0.24809328854347754,"accuracy_stderr":0.015714273262737038,"f1_stderr":0.014263974098066344,"main_score":0.27575655682582384}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/jonfd__electra-small-nordic/Massive_Scenario.json b/src/seb/cache/jonfd__electra-small-nordic/Massive_Scenario.json new file mode 100644 index 00000000..9f67725b --- /dev/null +++ b/src/seb/cache/jonfd__electra-small-nordic/Massive_Scenario.json @@ -0,0 +1 @@ +{"task_name":"Massive Scenario","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:35:04.712217","scores":{"da":{"accuracy":0.28934095494283796,"f1":0.2721739007115139,"accuracy_stderr":0.012764345785374599,"f1_stderr":0.010201933258922631,"main_score":0.28934095494283796},"nb":{"accuracy":0.2729657027572293,"f1":0.2550361435799058,"accuracy_stderr":0.012971310319977877,"f1_stderr":0.008997820373006915,"main_score":0.2729657027572293},"sv":{"accuracy":0.2992938802958978,"f1":0.28544183384003097,"accuracy_stderr":0.018575017991964805,"f1_stderr":0.017308765160599954,"main_score":0.2992938802958978}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/jonfd__electra-small-nordic/NoReC.json b/src/seb/cache/jonfd__electra-small-nordic/NoReC.json new file mode 100644 index 00000000..251d389c --- /dev/null +++ b/src/seb/cache/jonfd__electra-small-nordic/NoReC.json @@ -0,0 +1 @@ +{"task_name":"NoReC","task_description":"A Norwegian dataset for sentiment classification on review","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:36:28.704226","scores":{"nb":{"accuracy":0.45439453125,"f1":0.43581076230549776,"accuracy_stderr":0.02059852519265165,"f1_stderr":0.020224458055483393,"main_score":0.45439453125}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/jonfd__electra-small-nordic/Norwegian_parliament.json b/src/seb/cache/jonfd__electra-small-nordic/Norwegian_parliament.json new file mode 100644 index 00000000..e7734ea9 --- /dev/null +++ b/src/seb/cache/jonfd__electra-small-nordic/Norwegian_parliament.json @@ -0,0 +1 @@ +{"task_name":"Norwegian parliament","task_description":"Norwegian parliament speeches annotated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:36:50.051194","scores":{"nb":{"accuracy":0.5325,"f1":0.5293381445159899,"ap":0.518486003899672,"accuracy_stderr":0.027913557040740365,"f1_stderr":0.027110902221909646,"ap_stderr":0.016792731518067185,"main_score":0.5325}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/jonfd__electra-small-nordic/Scala.json b/src/seb/cache/jonfd__electra-small-nordic/Scala.json new file mode 100644 index 00000000..cdd91a80 --- /dev/null +++ b/src/seb/cache/jonfd__electra-small-nordic/Scala.json @@ -0,0 +1,45 @@ +{ + "task_name": "ScaLA", + "task_description": "A linguistic acceptability task for Danish, Norwegian Bokmål Norwegian Nynorsk and Swedish.", + "task_version": "1.0.3.dev0", + "time_of_run": "2023-07-27T20:35:54.008956", + "scores": { + "da": { + "accuracy": 0.704052734375, + "f1": 0.7016426788632224, + "ap": 0.6489189767992944, + "accuracy_stderr": 0.022459292144219167, + "f1_stderr": 0.025962579679041423, + "ap_stderr": 0.017125658069639487, + "main_score": 0.704052734375 + }, + "nb": { + "accuracy": 0.75283203125, + "f1": 0.752373309494691, + "ap": 0.6921163261018733, + "accuracy_stderr": 0.02621465574176361, + "f1_stderr": 0.026451034005481124, + "ap_stderr": 0.025567288727820582, + "main_score": 0.75283203125 + }, + "sv": { + "accuracy": 0.719384765625, + "f1": 0.7178199229809393, + "ap": 0.6569343597290781, + "accuracy_stderr": 0.01647187685733988, + "f1_stderr": 0.016268252909828418, + "ap_stderr": 0.015231690971978113, + "main_score": 0.719384765625 + }, + "nn": { + "accuracy": 0.69345703125, + "f1": 0.6925467164872301, + "ap": 0.6345195789841289, + "accuracy_stderr": 0.01846297234077574, + "f1_stderr": 0.0191971625171775, + "ap_stderr": 0.01802347858060941, + "main_score": 0.69345703125 + } + }, + "main_score": "accuracy" +} \ No newline at end of file diff --git a/src/seb/cache/jonfd__electra-small-nordic/SweFAQ.json b/src/seb/cache/jonfd__electra-small-nordic/SweFAQ.json new file mode 100644 index 00000000..32790dbf --- /dev/null +++ b/src/seb/cache/jonfd__electra-small-nordic/SweFAQ.json @@ -0,0 +1 @@ +{"task_name":"SweFAQ","task_description":"A Swedish QA dataset derived from FAQ","task_version":"0.0.1","time_of_run":"2023-09-04T19:24:27.873872","scores":{"sv":{"ndcg_at_1":0.03314,"ndcg_at_3":0.04344,"ndcg_at_5":0.04999,"ndcg_at_10":0.05488,"ndcg_at_100":0.10303,"ndcg_at_1000":0.18607,"map_at_1":0.03314,"map_at_3":0.04094,"map_at_5":0.04464,"map_at_10":0.04657,"map_at_100":0.0538,"map_at_1000":0.05661,"recall_at_1":0.03314,"recall_at_3":0.05068,"recall_at_5":0.06628,"recall_at_10":0.08187,"recall_at_100":0.33723,"recall_at_1000":1.0,"precision_at_1":0.03314,"precision_at_3":0.01689,"precision_at_5":0.01326,"precision_at_10":0.00819,"precision_at_100":0.00337,"precision_at_1000":0.001,"mrr_at_1":0.03314,"mrr_at_3":0.04094,"mrr_at_5":0.04464,"mrr_at_10":0.04657,"mrr_at_100":0.0538,"mrr_at_1000":0.05661}},"main_score":"ndcg_at_10"} \ No newline at end of file diff --git a/src/seb/cache/jonfd__electra-small-nordic/SweReC.json b/src/seb/cache/jonfd__electra-small-nordic/SweReC.json new file mode 100644 index 00000000..0f6518fc --- /dev/null +++ b/src/seb/cache/jonfd__electra-small-nordic/SweReC.json @@ -0,0 +1 @@ +{"task_name":"SweReC","task_description":"A Swedish dataset for sentiment classification on review","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:37:09.916482","scores":{"sv":{"accuracy":0.589794921875,"f1":0.5304933657673652,"accuracy_stderr":0.050827920448320824,"f1_stderr":0.041407801963220546,"main_score":0.589794921875}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/kb__electra-small-swedish-cased-discriminator/Angry_Tweets.json b/src/seb/cache/kb__electra-small-swedish-cased-discriminator/Angry_Tweets.json new file mode 100644 index 00000000..c1dbbfef --- /dev/null +++ b/src/seb/cache/kb__electra-small-swedish-cased-discriminator/Angry_Tweets.json @@ -0,0 +1 @@ +{"task_name":"Angry Tweets","task_description":"A sentiment dataset with 3 classes (positiv, negativ, neutral) for Danish tweets","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T11:03:29.742921","scores":{"da":{"accuracy":0.4051575931232092,"f1":0.39175653093962015,"accuracy_stderr":0.04497299760378002,"f1_stderr":0.039227248494350914,"main_score":0.4051575931232092}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/kb__electra-small-swedish-cased-discriminator/Bornholm_Parallel.json b/src/seb/cache/kb__electra-small-swedish-cased-discriminator/Bornholm_Parallel.json new file mode 100644 index 00000000..f3a4728c --- /dev/null +++ b/src/seb/cache/kb__electra-small-swedish-cased-discriminator/Bornholm_Parallel.json @@ -0,0 +1 @@ +{"task_name":"Bornholm Parallel","task_description":"Danish Bornholmsk Parallel Corpus. Bornholmsk is a Danish dialect spoken on the island of Bornholm, Denmark. Historically it is a part of east Danish which was also spoken in Scania and Halland, Sweden.","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T11:03:44.053892","scores":{"da":{"precision":0.007566666666666667,"recall":0.012,"f1":0.008466666666666667,"accuracy":0.012,"main_score":0.008466666666666667},"da-bornholm":{"precision":0.007566666666666667,"recall":0.012,"f1":0.008466666666666667,"accuracy":0.012,"main_score":0.008466666666666667}},"main_score":"f1"} \ No newline at end of file diff --git a/src/seb/cache/kb__electra-small-swedish-cased-discriminator/DKHate.json b/src/seb/cache/kb__electra-small-swedish-cased-discriminator/DKHate.json new file mode 100644 index 00000000..cb75d0d9 --- /dev/null +++ b/src/seb/cache/kb__electra-small-swedish-cased-discriminator/DKHate.json @@ -0,0 +1 @@ +{"task_name":"DKHate","task_description":"Danish Tweets annotated for Hate Speech either being Offensive or not","task_version":"1.0.3.dev0","time_of_run":"2023-07-30T14:02:01.734684","scores":{"da":{"accuracy":0.5227963525835866,"f1":0.43278888167711116,"ap":0.8796166943005271,"accuracy_stderr":0.07200504617549441,"f1_stderr":0.04968889181524428,"ap_stderr":0.01436712606484303,"main_score":0.5227963525835866}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/kb__electra-small-swedish-cased-discriminator/DaLAJ.json b/src/seb/cache/kb__electra-small-swedish-cased-discriminator/DaLAJ.json new file mode 100644 index 00000000..0315b26b --- /dev/null +++ b/src/seb/cache/kb__electra-small-swedish-cased-discriminator/DaLAJ.json @@ -0,0 +1 @@ +{"task_name":"DaLAJ","task_description":"A Swedish dataset for linguistic accebtablity. Available as a part of Superlim.","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T11:14:18.010746","scores":{"sv":{"accuracy":0.5564189189189189,"f1":0.5545516091706424,"ap":0.532391543149427,"accuracy_stderr":0.03467097102241357,"f1_stderr":0.03486640642416263,"ap_stderr":0.02117454851875759,"main_score":0.5564189189189189}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/kb__electra-small-swedish-cased-discriminator/Da_Political_Comments.json b/src/seb/cache/kb__electra-small-swedish-cased-discriminator/Da_Political_Comments.json new file mode 100644 index 00000000..044292dd --- /dev/null +++ b/src/seb/cache/kb__electra-small-swedish-cased-discriminator/Da_Political_Comments.json @@ -0,0 +1 @@ +{"task_name":"Da Political Comments","task_description":"A dataset of Danish political comments rated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T11:04:15.079119","scores":{"da":{"accuracy":0.2404550499445061,"f1":0.20785107656341495,"accuracy_stderr":0.024615843600652224,"f1_stderr":0.018349360476823503,"main_score":0.2404550499445061}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/kb__electra-small-swedish-cased-discriminator/LCC.json b/src/seb/cache/kb__electra-small-swedish-cased-discriminator/LCC.json new file mode 100644 index 00000000..5b17b467 --- /dev/null +++ b/src/seb/cache/kb__electra-small-swedish-cased-discriminator/LCC.json @@ -0,0 +1 @@ +{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T11:03:36.950352","scores":{"da":{"accuracy":0.3666666666666667,"f1":0.32644576196905895,"accuracy_stderr":0.026666666666666675,"f1_stderr":0.03544236459188725,"main_score":0.3666666666666667}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/kb__electra-small-swedish-cased-discriminator/Language_Identification.json b/src/seb/cache/kb__electra-small-swedish-cased-discriminator/Language_Identification.json new file mode 100644 index 00000000..c0c3abbf --- /dev/null +++ b/src/seb/cache/kb__electra-small-swedish-cased-discriminator/Language_Identification.json @@ -0,0 +1 @@ +{"task_name":"Language Identification","task_description":"A dataset for Nordic language identification.","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T11:07:53.799564","scores":{"da":{"accuracy":0.4453333333333333,"f1":0.4394329377426039,"accuracy_stderr":0.01224654146188947,"f1_stderr":0.013718232970041364,"main_score":0.4453333333333333},"sv":{"accuracy":0.4453333333333333,"f1":0.4394329377426039,"accuracy_stderr":0.01224654146188947,"f1_stderr":0.013718232970041364,"main_score":0.4453333333333333},"nb":{"accuracy":0.4453333333333333,"f1":0.4394329377426039,"accuracy_stderr":0.01224654146188947,"f1_stderr":0.013718232970041364,"main_score":0.4453333333333333},"nn":{"accuracy":0.4453333333333333,"f1":0.4394329377426039,"accuracy_stderr":0.01224654146188947,"f1_stderr":0.013718232970041364,"main_score":0.4453333333333333},"is":{"accuracy":0.4453333333333333,"f1":0.4394329377426039,"accuracy_stderr":0.01224654146188947,"f1_stderr":0.013718232970041364,"main_score":0.4453333333333333},"fo":{"accuracy":0.4453333333333333,"f1":0.4394329377426039,"accuracy_stderr":0.01224654146188947,"f1_stderr":0.013718232970041364,"main_score":0.4453333333333333}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/kb__electra-small-swedish-cased-discriminator/Massive_Intent.json b/src/seb/cache/kb__electra-small-swedish-cased-discriminator/Massive_Intent.json new file mode 100644 index 00000000..3a88c076 --- /dev/null +++ b/src/seb/cache/kb__electra-small-swedish-cased-discriminator/Massive_Intent.json @@ -0,0 +1 @@ +{"task_name":"Massive Intent","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T11:05:34.775412","scores":{"da":{"accuracy":0.06513113651647613,"f1":0.05043467599564514,"accuracy_stderr":0.011214798579686689,"f1_stderr":0.0035046004441874334,"main_score":0.06513113651647613},"nb":{"accuracy":0.05655682582380632,"f1":0.04546233598378012,"accuracy_stderr":0.006670594050204399,"f1_stderr":0.004960950317012453,"main_score":0.05655682582380632},"sv":{"accuracy":0.0660053799596503,"f1":0.05218747326664214,"accuracy_stderr":0.005903130694545474,"f1_stderr":0.0038648945970980028,"main_score":0.0660053799596503}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/kb__electra-small-swedish-cased-discriminator/Massive_Scenario.json b/src/seb/cache/kb__electra-small-swedish-cased-discriminator/Massive_Scenario.json new file mode 100644 index 00000000..efe57eb7 --- /dev/null +++ b/src/seb/cache/kb__electra-small-swedish-cased-discriminator/Massive_Scenario.json @@ -0,0 +1 @@ +{"task_name":"Massive Scenario","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T11:06:23.501961","scores":{"da":{"accuracy":0.11499663752521858,"f1":0.1002213125938091,"accuracy_stderr":0.006917224409212533,"f1_stderr":0.006521016859703459,"main_score":0.11499663752521858},"nb":{"accuracy":0.11264290517821118,"f1":0.09663541948337996,"accuracy_stderr":0.0071975812851916175,"f1_stderr":0.006912988810159833,"main_score":0.11264290517821118},"sv":{"accuracy":0.12155346334902488,"f1":0.10615364153403199,"accuracy_stderr":0.009327155941023088,"f1_stderr":0.00862797693736984,"main_score":0.12155346334902488}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/kb__electra-small-swedish-cased-discriminator/NoReC.json b/src/seb/cache/kb__electra-small-swedish-cased-discriminator/NoReC.json new file mode 100644 index 00000000..349b0f21 --- /dev/null +++ b/src/seb/cache/kb__electra-small-swedish-cased-discriminator/NoReC.json @@ -0,0 +1 @@ +{"task_name":"NoReC","task_description":"A Norwegian dataset for sentiment classification on review","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T11:08:08.454676","scores":{"nb":{"accuracy":0.397216796875,"f1":0.3795056315066193,"accuracy_stderr":0.02702295958997128,"f1_stderr":0.02046195703623087,"main_score":0.397216796875}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/kb__electra-small-swedish-cased-discriminator/Norwegian_parliament.json b/src/seb/cache/kb__electra-small-swedish-cased-discriminator/Norwegian_parliament.json new file mode 100644 index 00000000..f7252a1c --- /dev/null +++ b/src/seb/cache/kb__electra-small-swedish-cased-discriminator/Norwegian_parliament.json @@ -0,0 +1 @@ +{"task_name":"Norwegian parliament","task_description":"Norwegian parliament speeches annotated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T11:13:35.230167","scores":{"nb":{"accuracy":0.5244166666666666,"f1":0.5209059822096618,"ap":0.5134130030901037,"accuracy_stderr":0.02614609063618412,"f1_stderr":0.02426586990161433,"ap_stderr":0.01420872550714793,"main_score":0.5244166666666666}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/kb__electra-small-swedish-cased-discriminator/ScaLA.json b/src/seb/cache/kb__electra-small-swedish-cased-discriminator/ScaLA.json new file mode 100644 index 00000000..30c71df9 --- /dev/null +++ b/src/seb/cache/kb__electra-small-swedish-cased-discriminator/ScaLA.json @@ -0,0 +1 @@ +{"task_name":"ScaLA","task_description":"A linguistic acceptability task for Danish, Norwegian Bokmål Norwegian Nynorsk and Swedish.","task_version":"1.1.1.dev0","time_of_run":"2023-09-13T15:49:40.394759","scores":{"da":{"accuracy":0.51669921875,"f1":0.515062187079973,"ap":0.5090642458395526,"accuracy_stderr":0.021386272826401365,"f1_stderr":0.022655896014508747,"ap_stderr":0.011367379192702396,"main_score":0.51669921875},"nb":{"accuracy":0.52412109375,"f1":0.5218775073234838,"ap":0.5129596203593526,"accuracy_stderr":0.016143142311866417,"f1_stderr":0.016662789571752212,"ap_stderr":0.00902197530405059,"main_score":0.52412109375},"sv":{"accuracy":0.708544921875,"f1":0.7078190534267823,"ap":0.6496661850879124,"accuracy_stderr":0.03287148396650812,"f1_stderr":0.033380149881587225,"ap_stderr":0.028335766796377563,"main_score":0.708544921875},"nn":{"accuracy":0.510302734375,"f1":0.5068111395849514,"ap":0.5055763387271746,"accuracy_stderr":0.017376845117751893,"f1_stderr":0.018871570815152693,"ap_stderr":0.008955982269814214,"main_score":0.510302734375}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/kb__electra-small-swedish-cased-discriminator/SweFAQ.json b/src/seb/cache/kb__electra-small-swedish-cased-discriminator/SweFAQ.json new file mode 100644 index 00000000..f832bd1e --- /dev/null +++ b/src/seb/cache/kb__electra-small-swedish-cased-discriminator/SweFAQ.json @@ -0,0 +1 @@ +{"task_name":"SweFAQ","task_description":"A Swedish QA dataset derived from FAQ","task_version":"0.0.1","time_of_run":"2023-09-04T19:38:02.728295","scores":{"sv":{"ndcg_at_1":0.00195,"ndcg_at_3":0.00907,"ndcg_at_5":0.00983,"ndcg_at_10":0.01683,"ndcg_at_100":0.06125,"ndcg_at_1000":0.15204,"map_at_1":0.00195,"map_at_3":0.00747,"map_at_5":0.00786,"map_at_10":0.01079,"map_at_100":0.01741,"map_at_1000":0.02045,"recall_at_1":0.00195,"recall_at_3":0.01365,"recall_at_5":0.01559,"recall_at_10":0.03704,"recall_at_100":0.2729,"recall_at_1000":1.0,"precision_at_1":0.00195,"precision_at_3":0.00455,"precision_at_5":0.00312,"precision_at_10":0.0037,"precision_at_100":0.00273,"precision_at_1000":0.001,"mrr_at_1":0.00195,"mrr_at_3":0.00747,"mrr_at_5":0.00786,"mrr_at_10":0.01079,"mrr_at_100":0.01741,"mrr_at_1000":0.02045}},"main_score":"ndcg_at_10"} \ No newline at end of file diff --git a/src/seb/cache/kb__electra-small-swedish-cased-discriminator/SweReC.json b/src/seb/cache/kb__electra-small-swedish-cased-discriminator/SweReC.json new file mode 100644 index 00000000..612ed6a8 --- /dev/null +++ b/src/seb/cache/kb__electra-small-swedish-cased-discriminator/SweReC.json @@ -0,0 +1 @@ +{"task_name":"SweReC","task_description":"A Swedish dataset for sentiment classification on review","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T11:14:10.554136","scores":{"sv":{"accuracy":0.549560546875,"f1":0.49387662903535984,"accuracy_stderr":0.02222323481583929,"f1_stderr":0.015706046226432843,"main_score":0.549560546875}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/Angry_Tweets.json b/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/Angry_Tweets.json new file mode 100644 index 00000000..a64ec592 --- /dev/null +++ b/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/Angry_Tweets.json @@ -0,0 +1 @@ +{"task_name":"Angry Tweets","task_description":"A sentiment dataset with 3 classes (positiv, negativ, neutral) for Danish tweets","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T18:26:30.430087","scores":{"da":{"accuracy":0.42492836676217766,"f1":0.4146959958482899,"accuracy_stderr":0.025562172565760207,"f1_stderr":0.018608050832297794,"main_score":0.42492836676217766}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/Bornholm_Parallel.json b/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/Bornholm_Parallel.json new file mode 100644 index 00000000..7fecb9c8 --- /dev/null +++ b/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/Bornholm_Parallel.json @@ -0,0 +1 @@ +{"task_name":"Bornholm Parallel","task_description":"Danish Bornholmsk Parallel Corpus. Bornholmsk is a Danish dialect spoken on the island of Bornholm, Denmark. Historically it is a part of east Danish which was also spoken in Scania and Halland, Sweden.","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T18:26:56.265976","scores":{"da":{"precision":0.27690919913419915,"recall":0.36,"f1":0.2968132161955691,"accuracy":0.36,"main_score":0.2968132161955691},"da-bornholm":{"precision":0.27690919913419915,"recall":0.36,"f1":0.2968132161955691,"accuracy":0.36,"main_score":0.2968132161955691}},"main_score":"f1"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/DKHate.json b/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/DKHate.json new file mode 100644 index 00000000..64b35a5f --- /dev/null +++ b/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/DKHate.json @@ -0,0 +1 @@ +{"task_name":"DKHate","task_description":"Danish Tweets annotated for Hate Speech either being Offensive or not","task_version":"1.1.0","time_of_run":"2023-07-31T15:19:48.879189","scores":{"da":{"accuracy":0.5504559270516718,"f1":0.4487544754943351,"ap":0.8825715897823836,"accuracy_stderr":0.08179003177509295,"f1_stderr":0.04439449341359171,"ap_stderr":0.008146255235874632,"main_score":0.5504559270516718}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/DaLAJ.json b/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/DaLAJ.json new file mode 100644 index 00000000..bab1abb7 --- /dev/null +++ b/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/DaLAJ.json @@ -0,0 +1 @@ +{"task_name":"DaLAJ","task_description":"A Swedish dataset for linguistic accebtablity. Available as a part of Superlim.","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:14:26.377559","scores":{"sv":{"accuracy":0.49898648648648647,"f1":0.49645260794205,"ap":0.4995893380614416,"accuracy_stderr":0.009521594515282868,"f1_stderr":0.010845998499157633,"ap_stderr":0.004686992032945654,"main_score":0.49898648648648647}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/Da_Political_Comments.json b/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/Da_Political_Comments.json new file mode 100644 index 00000000..f85c213c --- /dev/null +++ b/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/Da_Political_Comments.json @@ -0,0 +1 @@ +{"task_name":"Da Political Comments","task_description":"A dataset of Danish political comments rated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T18:27:24.316668","scores":{"da":{"accuracy":0.26159822419533857,"f1":0.23486652223054044,"accuracy_stderr":0.019093771385596345,"f1_stderr":0.017237286849500585,"main_score":0.26159822419533857}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/LCC.json b/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/LCC.json new file mode 100644 index 00000000..64ce3486 --- /dev/null +++ b/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/LCC.json @@ -0,0 +1 @@ +{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T18:26:51.592244","scores":{"da":{"accuracy":0.3846666666666666,"f1":0.3650136884557438,"accuracy_stderr":0.03664241622309678,"f1_stderr":0.03540233062350939,"main_score":0.3846666666666666}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/Language_Identification.json b/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/Language_Identification.json new file mode 100644 index 00000000..354e9df9 --- /dev/null +++ b/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/Language_Identification.json @@ -0,0 +1 @@ +{"task_name":"Language Identification","task_description":"A dataset for Nordic language identification.","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:12:49.666188","scores":{"da":{"accuracy":0.5471,"f1":0.5337340133025357,"accuracy_stderr":0.019781051539288805,"f1_stderr":0.021836739801343485,"main_score":0.5471},"sv":{"accuracy":0.5471,"f1":0.5337340133025357,"accuracy_stderr":0.019781051539288805,"f1_stderr":0.021836739801343485,"main_score":0.5471},"nb":{"accuracy":0.5471,"f1":0.5337340133025357,"accuracy_stderr":0.019781051539288805,"f1_stderr":0.021836739801343485,"main_score":0.5471},"nn":{"accuracy":0.5471,"f1":0.5337340133025357,"accuracy_stderr":0.019781051539288805,"f1_stderr":0.021836739801343485,"main_score":0.5471},"is":{"accuracy":0.5471,"f1":0.5337340133025357,"accuracy_stderr":0.019781051539288805,"f1_stderr":0.021836739801343485,"main_score":0.5471},"fo":{"accuracy":0.5471,"f1":0.5337340133025357,"accuracy_stderr":0.019781051539288805,"f1_stderr":0.021836739801343485,"main_score":0.5471}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/Massive_Intent.json b/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/Massive_Intent.json new file mode 100644 index 00000000..c74b2c2a --- /dev/null +++ b/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/Massive_Intent.json @@ -0,0 +1 @@ +{"task_name":"Massive Intent","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:10:25.70716","scores":{"da":{"accuracy":0.4098520511096167,"f1":0.39031952711786977,"accuracy_stderr":0.013546600921886436,"f1_stderr":0.01465861207914092,"main_score":0.4098520511096167},"nb":{"accuracy":0.3934431741761937,"f1":0.3725650363380114,"accuracy_stderr":0.010632494048802496,"f1_stderr":0.0047186150764588165,"main_score":0.3934431741761937},"sv":{"accuracy":0.38103564223268327,"f1":0.36129998730373103,"accuracy_stderr":0.015896221776346243,"f1_stderr":0.009758357654155333,"main_score":0.38103564223268327}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/Massive_Scenario.json b/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/Massive_Scenario.json new file mode 100644 index 00000000..cbc99fe3 --- /dev/null +++ b/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/Massive_Scenario.json @@ -0,0 +1 @@ +{"task_name":"Massive Scenario","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:11:25.752928","scores":{"da":{"accuracy":0.47007397444519167,"f1":0.44591837506202986,"accuracy_stderr":0.016525460362257268,"f1_stderr":0.011715003849608418,"main_score":0.47007397444519167},"nb":{"accuracy":0.44670477471418957,"f1":0.42175654729028544,"accuracy_stderr":0.01725133080004754,"f1_stderr":0.015659543471027913,"main_score":0.44670477471418957},"sv":{"accuracy":0.4293207800941493,"f1":0.3990548911517426,"accuracy_stderr":0.018548430093723933,"f1_stderr":0.015416056576756836,"main_score":0.4293207800941493}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/NoReC.json b/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/NoReC.json new file mode 100644 index 00000000..fb89a2a5 --- /dev/null +++ b/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/NoReC.json @@ -0,0 +1 @@ +{"task_name":"NoReC","task_description":"A Norwegian dataset for sentiment classification on review","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:13:05.003898","scores":{"nb":{"accuracy":0.4001953125,"f1":0.3800296115219944,"accuracy_stderr":0.02129286985736529,"f1_stderr":0.016433278119896217,"main_score":0.4001953125}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/Norwegian_parliament.json b/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/Norwegian_parliament.json new file mode 100644 index 00000000..a8ffd463 --- /dev/null +++ b/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/Norwegian_parliament.json @@ -0,0 +1 @@ +{"task_name":"Norwegian parliament","task_description":"Norwegian parliament speeches annotated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:13:44.497941","scores":{"nb":{"accuracy":0.548,"f1":0.5465938348735246,"ap":0.5271332062355649,"accuracy_stderr":0.024218909049656952,"f1_stderr":0.022992567784370865,"ap_stderr":0.014648312153288252,"main_score":0.548}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/ScaLA.json b/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/ScaLA.json new file mode 100644 index 00000000..7169187c --- /dev/null +++ b/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/ScaLA.json @@ -0,0 +1 @@ +{"task_name":"ScaLA","task_description":"A linguistic acceptability task for Danish, Norwegian Bokmål Norwegian Nynorsk and Swedish.","task_version":"1.1.1.dev0","time_of_run":"2023-09-13T15:31:42.627564","scores":{"da":{"accuracy":0.500341796875,"f1":0.49707836995043886,"ap":0.5002094933987072,"accuracy_stderr":0.00619193763738963,"f1_stderr":0.0056844458165261,"ap_stderr":0.003086975974967957,"main_score":0.500341796875},"nb":{"accuracy":0.501708984375,"f1":0.4975764910012903,"ap":0.5008866627584277,"accuracy_stderr":0.005376639537486631,"f1_stderr":0.0057738834379346465,"ap_stderr":0.0027086373278729326,"main_score":0.501708984375},"sv":{"accuracy":0.4994140625,"f1":0.49683291347295633,"ap":0.49972805126929065,"accuracy_stderr":0.004620905532937882,"f1_stderr":0.004218877367923389,"ap_stderr":0.002309284713600242,"main_score":0.4994140625},"nn":{"accuracy":0.500341796875,"f1":0.4961039513494653,"ap":0.5002008066815936,"accuracy_stderr":0.005393463714180493,"f1_stderr":0.007401911092391309,"ap_stderr":0.002724109383421351,"main_score":0.500341796875}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/SweFAQ.json b/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/SweFAQ.json new file mode 100644 index 00000000..74a4884c --- /dev/null +++ b/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/SweFAQ.json @@ -0,0 +1 @@ +{"task_name":"SweFAQ","task_description":"A Swedish QA dataset derived from FAQ","task_version":"0.0.1","time_of_run":"2023-09-04T19:23:39.098869","scores":{"sv":{"ndcg_at_1":0.2768,"ndcg_at_3":0.34786,"ndcg_at_5":0.38158,"ndcg_at_10":0.41075,"ndcg_at_100":0.46199,"ndcg_at_1000":0.485,"map_at_1":0.2768,"map_at_3":0.32943,"map_at_5":0.34815,"map_at_10":0.36064,"map_at_100":0.37011,"map_at_1000":0.37101,"recall_at_1":0.2768,"recall_at_3":0.40156,"recall_at_5":0.48343,"recall_at_10":0.57115,"recall_at_100":0.82261,"recall_at_1000":1.0,"precision_at_1":0.2768,"precision_at_3":0.13385,"precision_at_5":0.09669,"precision_at_10":0.05712,"precision_at_100":0.00823,"precision_at_1000":0.001,"mrr_at_1":0.2768,"mrr_at_3":0.32943,"mrr_at_5":0.34776,"mrr_at_10":0.36058,"mrr_at_100":0.37004,"mrr_at_1000":0.37094}},"main_score":"ndcg_at_10"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/SweReC.json b/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/SweReC.json new file mode 100644 index 00000000..30b02020 --- /dev/null +++ b/src/seb/cache/sentence-transformers__all-MiniLM-L6-v2/SweReC.json @@ -0,0 +1 @@ +{"task_name":"SweReC","task_description":"A Swedish dataset for sentiment classification on review","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:14:15.648062","scores":{"sv":{"accuracy":0.6083984375,"f1":0.5383846800461662,"accuracy_stderr":0.0200147669635808,"f1_stderr":0.01947712920967888,"main_score":0.6083984375}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/Angry_Tweets.json b/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/Angry_Tweets.json new file mode 100644 index 00000000..1a975782 --- /dev/null +++ b/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/Angry_Tweets.json @@ -0,0 +1 @@ +{"task_name":"Angry Tweets","task_description":"A sentiment dataset with 3 classes (positiv, negativ, neutral) for Danish tweets","task_version":"1.0.3.dev0","time_of_run":"2023-07-31T11:33:16.683863","scores":{"da":{"accuracy":0.5093600764087871,"f1":0.5048502157125114,"accuracy_stderr":0.028824860449880765,"f1_stderr":0.025917063627089995,"main_score":0.5093600764087871}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/Bornholm_Parallel.json b/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/Bornholm_Parallel.json new file mode 100644 index 00000000..f865fa11 --- /dev/null +++ b/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/Bornholm_Parallel.json @@ -0,0 +1 @@ +{"task_name":"Bornholm Parallel","task_description":"Danish Bornholmsk Parallel Corpus. Bornholmsk is a Danish dialect spoken on the island of Bornholm, Denmark. Historically it is a part of east Danish which was also spoken in Scania and Halland, Sweden.","task_version":"1.0.3.dev0","time_of_run":"2023-07-31T11:33:30.498759","scores":{"da":{"precision":0.17640476190476193,"recall":0.256,"f1":0.19671528822055137,"accuracy":0.256,"main_score":0.19671528822055137},"da-bornholm":{"precision":0.17640476190476193,"recall":0.256,"f1":0.19671528822055137,"accuracy":0.256,"main_score":0.19671528822055137}},"main_score":"f1"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/DKHate.json b/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/DKHate.json new file mode 100644 index 00000000..8d54dc5d --- /dev/null +++ b/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/DKHate.json @@ -0,0 +1 @@ +{"task_name":"DKHate","task_description":"Danish Tweets annotated for Hate Speech either being Offensive or not","task_version":"1.0.3.dev0","time_of_run":"2023-07-31T11:33:41.19396","scores":{"da":{"accuracy":0.5905775075987841,"f1":0.5012328623078618,"ap":0.17069754794987557,"accuracy_stderr":0.08564797279638994,"f1_stderr":0.06385487550431072,"ap_stderr":0.029794817561981823,"main_score":0.5905775075987841}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/DaLAJ.json b/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/DaLAJ.json new file mode 100644 index 00000000..a8080aaf --- /dev/null +++ b/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/DaLAJ.json @@ -0,0 +1 @@ +{"task_name":"DaLAJ","task_description":"A Swedish dataset for linguistic accebtablity. Available as a part of Superlim.","task_version":"1.0.3.dev0","time_of_run":"2023-07-31T11:40:10.550133","scores":{"sv":{"accuracy":0.5011261261261262,"f1":0.49729121130311,"ap":0.5005806001914621,"accuracy_stderr":0.003666401035157583,"f1_stderr":0.00456638933382744,"ap_stderr":0.0018378809094670475,"main_score":0.5011261261261262}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/Da_Political_Comments.json b/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/Da_Political_Comments.json new file mode 100644 index 00000000..7a42eb9f --- /dev/null +++ b/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/Da_Political_Comments.json @@ -0,0 +1 @@ +{"task_name":"Da Political Comments","task_description":"A dataset of Danish political comments rated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-31T11:34:00.756118","scores":{"da":{"accuracy":0.374472807991121,"f1":0.34917527075979476,"accuracy_stderr":0.02507046908500865,"f1_stderr":0.023278188351939278,"main_score":0.374472807991121}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/LCC.json b/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/LCC.json new file mode 100644 index 00000000..4ebbbfdb --- /dev/null +++ b/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/LCC.json @@ -0,0 +1 @@ +{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-31T11:33:23.98691","scores":{"da":{"accuracy":0.5433333333333332,"f1":0.5345356103584374,"accuracy_stderr":0.03714835124201342,"f1_stderr":0.03635866013750846,"main_score":0.5433333333333332}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/Language_Identification.json b/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/Language_Identification.json new file mode 100644 index 00000000..24dcfbcc --- /dev/null +++ b/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/Language_Identification.json @@ -0,0 +1 @@ +{"task_name":"Language Identification","task_description":"A dataset for Nordic language identification.","task_version":"1.0.3.dev0","time_of_run":"2023-07-31T11:38:34.547835","scores":{"da":{"accuracy":0.4253333333333333,"f1":0.42104006055518506,"accuracy_stderr":0.010717586793054982,"f1_stderr":0.010497866908361114,"main_score":0.4253333333333333},"sv":{"accuracy":0.4253333333333333,"f1":0.42104006055518506,"accuracy_stderr":0.010717586793054982,"f1_stderr":0.010497866908361114,"main_score":0.4253333333333333},"nb":{"accuracy":0.4253333333333333,"f1":0.42104006055518506,"accuracy_stderr":0.010717586793054982,"f1_stderr":0.010497866908361114,"main_score":0.4253333333333333},"nn":{"accuracy":0.4253333333333333,"f1":0.42104006055518506,"accuracy_stderr":0.010717586793054982,"f1_stderr":0.010497866908361114,"main_score":0.4253333333333333},"is":{"accuracy":0.4253333333333333,"f1":0.42104006055518506,"accuracy_stderr":0.010717586793054982,"f1_stderr":0.010497866908361114,"main_score":0.4253333333333333},"fo":{"accuracy":0.4253333333333333,"f1":0.42104006055518506,"accuracy_stderr":0.010717586793054982,"f1_stderr":0.010497866908361114,"main_score":0.4253333333333333}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/Massive_Intent.json b/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/Massive_Intent.json new file mode 100644 index 00000000..a6794ecc --- /dev/null +++ b/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/Massive_Intent.json @@ -0,0 +1 @@ +{"task_name":"Massive Intent","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.0.3.dev0","time_of_run":"2023-07-31T11:35:58.387057","scores":{"da":{"accuracy":0.5772024209818427,"f1":0.5527844433247312,"accuracy_stderr":0.0101991126931784,"f1_stderr":0.013982726023235116,"main_score":0.5772024209818427},"nb":{"accuracy":0.5550100874243443,"f1":0.5377361727083334,"accuracy_stderr":0.012446789833231229,"f1_stderr":0.01362732032794397,"main_score":0.5550100874243443},"sv":{"accuracy":0.5942501681237391,"f1":0.5718810835950179,"accuracy_stderr":0.01199612065539809,"f1_stderr":0.012591093748023804,"main_score":0.5942501681237391}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/Massive_Scenario.json b/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/Massive_Scenario.json new file mode 100644 index 00000000..5786c89e --- /dev/null +++ b/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/Massive_Scenario.json @@ -0,0 +1 @@ +{"task_name":"Massive Scenario","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.0.3.dev0","time_of_run":"2023-07-31T11:36:51.26379","scores":{"da":{"accuracy":0.6686953597848017,"f1":0.6625246153454784,"accuracy_stderr":0.013017642091446834,"f1_stderr":0.011739866130355563,"main_score":0.6686953597848017},"nb":{"accuracy":0.6425016812373907,"f1":0.6393095258431323,"accuracy_stderr":0.019436035003297384,"f1_stderr":0.015519757775818235,"main_score":0.6425016812373907},"sv":{"accuracy":0.6713517148621385,"f1":0.6637867671764992,"accuracy_stderr":0.007875946723312635,"f1_stderr":0.008424364811900611,"main_score":0.6713517148621385}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/NoReC.json b/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/NoReC.json new file mode 100644 index 00000000..266c5787 --- /dev/null +++ b/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/NoReC.json @@ -0,0 +1 @@ +{"task_name":"NoReC","task_description":"A Norwegian dataset for sentiment classification on review","task_version":"1.0.3.dev0","time_of_run":"2023-07-31T11:38:54.204105","scores":{"nb":{"accuracy":0.498876953125,"f1":0.4861519782809852,"accuracy_stderr":0.023909881914294717,"f1_stderr":0.021786268832168113,"main_score":0.498876953125}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/Norwegian_parliament.json b/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/Norwegian_parliament.json new file mode 100644 index 00000000..4104c4eb --- /dev/null +++ b/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/Norwegian_parliament.json @@ -0,0 +1 @@ +{"task_name":"Norwegian parliament","task_description":"Norwegian parliament speeches annotated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-31T11:39:27.012525","scores":{"nb":{"accuracy":0.5656666666666667,"f1":0.5625237053609615,"ap":0.5376425771073513,"accuracy_stderr":0.023885723304471605,"f1_stderr":0.027082881727603685,"ap_stderr":0.014827255520924457,"main_score":0.5656666666666667}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/Scala.json b/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/Scala.json new file mode 100644 index 00000000..aa564b6f --- /dev/null +++ b/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/Scala.json @@ -0,0 +1,45 @@ +{ + "task_name": "ScaLA", + "task_description": "A linguistic acceptability task for Danish, Norwegian Bokmål Norwegian Nynorsk and Swedish.", + "task_version": "1.0.3.dev0", + "time_of_run": "2023-07-31T11:38:04.551647", + "scores": { + "da": { + "accuracy": 0.499169921875, + "f1": 0.4941248680131377, + "ap": 0.4995928806833481, + "accuracy_stderr": 0.002736553958828526, + "f1_stderr": 0.004677527075963332, + "ap_stderr": 0.00135958840907401, + "main_score": 0.499169921875 + }, + "nb": { + "accuracy": 0.50185546875, + "f1": 0.49709017306024006, + "ap": 0.5009454476875702, + "accuracy_stderr": 0.003749287855557122, + "f1_stderr": 0.0060526189643126055, + "ap_stderr": 0.00188996728561891, + "main_score": 0.50185546875 + }, + "sv": { + "accuracy": 0.499560546875, + "f1": 0.4961635072926544, + "ap": 0.49979020803267254, + "accuracy_stderr": 0.0032052233266164666, + "f1_stderr": 0.0042516182801659265, + "ap_stderr": 0.0015925475134142984, + "main_score": 0.499560546875 + }, + "nn": { + "accuracy": 0.500390625, + "f1": 0.49778813816297623, + "ap": 0.500213558270544, + "accuracy_stderr": 0.0043607643675324425, + "f1_stderr": 0.004148006783726859, + "ap_stderr": 0.0021832183309978227, + "main_score": 0.500390625 + } + }, + "main_score": "accuracy" +} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/SweFAQ.json b/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/SweFAQ.json new file mode 100644 index 00000000..f321612e --- /dev/null +++ b/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/SweFAQ.json @@ -0,0 +1 @@ +{"task_name":"SweFAQ","task_description":"A Swedish QA dataset derived from FAQ","task_version":"0.0.1","time_of_run":"2023-09-04T19:23:50.254128","scores":{"sv":{"ndcg_at_1":0.39571,"ndcg_at_3":0.49679,"ndcg_at_5":0.53907,"ndcg_at_10":0.56936,"ndcg_at_100":0.60773,"ndcg_at_1000":0.61477,"map_at_1":0.39571,"map_at_3":0.47238,"map_at_5":0.49568,"map_at_10":0.50782,"map_at_100":0.51553,"map_at_1000":0.51585,"recall_at_1":0.39571,"recall_at_3":0.56725,"recall_at_5":0.67057,"recall_at_10":0.76608,"recall_at_100":0.94737,"recall_at_1000":1.0,"precision_at_1":0.39571,"precision_at_3":0.18908,"precision_at_5":0.13411,"precision_at_10":0.07661,"precision_at_100":0.00947,"precision_at_1000":0.001,"mrr_at_1":0.39571,"mrr_at_3":0.47238,"mrr_at_5":0.49568,"mrr_at_10":0.50782,"mrr_at_100":0.51553,"mrr_at_1000":0.51585}},"main_score":"ndcg_at_10"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/SweReC.json b/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/SweReC.json new file mode 100644 index 00000000..ab164967 --- /dev/null +++ b/src/seb/cache/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/SweReC.json @@ -0,0 +1 @@ +{"task_name":"SweReC","task_description":"A Swedish dataset for sentiment classification on review","task_version":"1.0.3.dev0","time_of_run":"2023-07-31T11:39:57.219154","scores":{"sv":{"accuracy":0.699853515625,"f1":0.6279867802236269,"accuracy_stderr":0.022311888699746006,"f1_stderr":0.020297264889115377,"main_score":0.699853515625}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/Angry_Tweets.json b/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/Angry_Tweets.json new file mode 100644 index 00000000..cba36e81 --- /dev/null +++ b/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/Angry_Tweets.json @@ -0,0 +1 @@ +{"task_name":"Angry Tweets","task_description":"A sentiment dataset with 3 classes (positiv, negativ, neutral) for Danish tweets","task_version":"1.1.1.dev0","time_of_run":"2023-10-24T10:33:45.843745","scores":{"da":{"accuracy":0.5487106017191976,"f1":0.541952814980804,"accuracy_stderr":0.033133502265756086,"f1_stderr":0.030774728096590768,"main_score":0.5487106017191976}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/Bornholm_Parallel.json b/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/Bornholm_Parallel.json new file mode 100644 index 00000000..70416611 --- /dev/null +++ b/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/Bornholm_Parallel.json @@ -0,0 +1 @@ +{"task_name":"Bornholm Parallel","task_description":"Danish Bornholmsk Parallel Corpus. Bornholmsk is a Danish dialect spoken on the island of Bornholm, Denmark. Historically it is a part of east Danish which was also spoken in Scania and Halland, Sweden.","task_version":"1.1.1.dev0","time_of_run":"2023-10-24T10:34:08.879529","scores":{"da":{"precision":0.16436095238095236,"recall":0.232,"f1":0.18182931512931513,"accuracy":0.232,"main_score":0.18182931512931513},"da-bornholm":{"precision":0.16436095238095236,"recall":0.232,"f1":0.18182931512931513,"accuracy":0.232,"main_score":0.18182931512931513}},"main_score":"f1"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/DKHate.json b/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/DKHate.json new file mode 100644 index 00000000..b6457c88 --- /dev/null +++ b/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/DKHate.json @@ -0,0 +1 @@ +{"task_name":"DKHate","task_description":"Danish Tweets annotated for Hate Speech either being Offensive or not","task_version":"1.1.1.dev0","time_of_run":"2023-10-24T10:34:25.98135","scores":{"da":{"accuracy":0.6379939209726444,"f1":0.5360215022364951,"ap":0.9111072259687687,"accuracy_stderr":0.067767523538305,"f1_stderr":0.050101459576658555,"ap_stderr":0.011687917559401089,"main_score":0.6379939209726444}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/DaLAJ.json b/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/DaLAJ.json new file mode 100644 index 00000000..fac7e499 --- /dev/null +++ b/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/DaLAJ.json @@ -0,0 +1 @@ +{"task_name":"DaLAJ","task_description":"A Swedish dataset for linguistic acceptability. Available as a part of Superlim.","task_version":"1.1.1.dev0","time_of_run":"2023-10-24T10:44:55.292474","scores":{"sv":{"accuracy":0.5003378378378379,"f1":0.4960343544120457,"ap":0.5002025149164775,"accuracy_stderr":0.005540952522586734,"f1_stderr":0.005653068963986722,"ap_stderr":0.0027620575839472678,"main_score":0.5003378378378379}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/Da_Political_Comments.json b/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/Da_Political_Comments.json new file mode 100644 index 00000000..14cda029 --- /dev/null +++ b/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/Da_Political_Comments.json @@ -0,0 +1 @@ +{"task_name":"Da Political Comments","task_description":"A dataset of Danish political comments rated for sentiment","task_version":"1.1.1.dev0","time_of_run":"2023-10-24T10:34:54.952404","scores":{"da":{"accuracy":0.4133185349611542,"f1":0.389051497727286,"accuracy_stderr":0.022650484608778228,"f1_stderr":0.022647246804913045,"main_score":0.4133185349611542}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/LCC.json b/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/LCC.json new file mode 100644 index 00000000..7f38401f --- /dev/null +++ b/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/LCC.json @@ -0,0 +1 @@ +{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.1.1.dev0","time_of_run":"2023-10-24T10:33:59.702788","scores":{"da":{"accuracy":0.584,"f1":0.5844888377683077,"accuracy_stderr":0.033888706739036885,"f1_stderr":0.03243288494020961,"main_score":0.584}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/Language_Identification.json b/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/Language_Identification.json new file mode 100644 index 00000000..3c2ce286 --- /dev/null +++ b/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/Language_Identification.json @@ -0,0 +1 @@ +{"task_name":"Language Identification","task_description":"A dataset for Nordic language identification.","task_version":"1.1.1.dev0","time_of_run":"2023-10-24T10:42:19.076906","scores":{"da":{"accuracy":0.4155666666666667,"f1":0.40955919231034316,"accuracy_stderr":0.009022256431243296,"f1_stderr":0.007603572801138439,"main_score":0.4155666666666667},"sv":{"accuracy":0.4155666666666667,"f1":0.40955919231034316,"accuracy_stderr":0.009022256431243296,"f1_stderr":0.007603572801138439,"main_score":0.4155666666666667},"nb":{"accuracy":0.4155666666666667,"f1":0.40955919231034316,"accuracy_stderr":0.009022256431243296,"f1_stderr":0.007603572801138439,"main_score":0.4155666666666667},"nn":{"accuracy":0.4155666666666667,"f1":0.40955919231034316,"accuracy_stderr":0.009022256431243296,"f1_stderr":0.007603572801138439,"main_score":0.4155666666666667},"is":{"accuracy":0.4155666666666667,"f1":0.40955919231034316,"accuracy_stderr":0.009022256431243296,"f1_stderr":0.007603572801138439,"main_score":0.4155666666666667},"fo":{"accuracy":0.4155666666666667,"f1":0.40955919231034316,"accuracy_stderr":0.009022256431243296,"f1_stderr":0.007603572801138439,"main_score":0.4155666666666667}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/Massive_Intent.json b/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/Massive_Intent.json new file mode 100644 index 00000000..8926bda9 --- /dev/null +++ b/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/Massive_Intent.json @@ -0,0 +1 @@ +{"task_name":"Massive Intent","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.1.1.dev0","time_of_run":"2023-10-24T10:37:41.998563","scores":{"da":{"accuracy":0.6276731674512441,"f1":0.6045950683900405,"accuracy_stderr":0.012883193887038194,"f1_stderr":0.011787993931648488,"main_score":0.6276731674512441},"nb":{"accuracy":0.6260255548083389,"f1":0.6060235982782182,"accuracy_stderr":0.012410037599896246,"f1_stderr":0.012665171614848545,"main_score":0.6260255548083389},"sv":{"accuracy":0.6473100201748487,"f1":0.6208160110958849,"accuracy_stderr":0.012241855915225917,"f1_stderr":0.011143143573587698,"main_score":0.6473100201748487}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/Massive_Scenario.json b/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/Massive_Scenario.json new file mode 100644 index 00000000..985c7e99 --- /dev/null +++ b/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/Massive_Scenario.json @@ -0,0 +1 @@ +{"task_name":"Massive Scenario","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.1.1.dev0","time_of_run":"2023-10-24T10:39:04.549074","scores":{"da":{"accuracy":0.7099865501008743,"f1":0.7048020593632474,"accuracy_stderr":0.01779587991215053,"f1_stderr":0.01616953224023678,"main_score":0.7099865501008743},"nb":{"accuracy":0.7023873570948218,"f1":0.7009435905461785,"accuracy_stderr":0.012134248610647588,"f1_stderr":0.011191173293935064,"main_score":0.7023873570948218},"sv":{"accuracy":0.7159717552118359,"f1":0.7126344894382457,"accuracy_stderr":0.014963617255042654,"f1_stderr":0.016797115175110268,"main_score":0.7159717552118359}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/NoReC.json b/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/NoReC.json new file mode 100644 index 00000000..72e12ca3 --- /dev/null +++ b/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/NoReC.json @@ -0,0 +1 @@ +{"task_name":"NoReC","task_description":"A Norwegian dataset for sentiment classification on review","task_version":"1.1.1.dev0","time_of_run":"2023-10-24T10:42:50.473153","scores":{"nb":{"accuracy":0.5607421875,"f1":0.5443227049974069,"accuracy_stderr":0.022685065525053623,"f1_stderr":0.020403561502383604,"main_score":0.5607421875}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/Norwegian_parliament.json b/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/Norwegian_parliament.json new file mode 100644 index 00000000..ca67d344 --- /dev/null +++ b/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/Norwegian_parliament.json @@ -0,0 +1 @@ +{"task_name":"Norwegian parliament","task_description":"Norwegian parliament speeches annotated for sentiment","task_version":"1.1.1.dev0","time_of_run":"2023-10-24T10:43:45.417955","scores":{"nb":{"accuracy":0.546,"f1":0.5430762836336488,"ap":0.5258502861083085,"accuracy_stderr":0.02883814988671932,"f1_stderr":0.029867246016325278,"ap_stderr":0.016626629507821497,"main_score":0.546}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/ScaLA.json b/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/ScaLA.json new file mode 100644 index 00000000..27f22de4 --- /dev/null +++ b/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/ScaLA.json @@ -0,0 +1 @@ +{"task_name":"ScaLA","task_description":"A linguistic acceptability task for Danish, Norwegian Bokmål Norwegian Nynorsk and Swedish.","task_version":"1.1.1.dev0","time_of_run":"2023-10-24T10:41:15.057446","scores":{"da":{"accuracy":0.503076171875,"f1":0.49962521526322085,"ap":0.5015674787360893,"accuracy_stderr":0.004698930155960577,"f1_stderr":0.006942996835906564,"ap_stderr":0.0023977372533521652,"main_score":0.503076171875},"nb":{"accuracy":0.498779296875,"f1":0.4954829756909468,"ap":0.4994059087693648,"accuracy_stderr":0.0039922777975661275,"f1_stderr":0.006173156887437416,"ap_stderr":0.002003611070762503,"main_score":0.498779296875},"sv":{"accuracy":0.499169921875,"f1":0.49447185439806873,"ap":0.49961566876803165,"accuracy_stderr":0.005571757638736133,"f1_stderr":0.009058207123069583,"ap_stderr":0.002795551249647806,"main_score":0.499169921875},"nn":{"accuracy":0.49970703125,"f1":0.497603845109563,"ap":0.4998654048339211,"accuracy_stderr":0.003262361931345716,"f1_stderr":0.005205692277914849,"ap_stderr":0.0016373420445338525,"main_score":0.49970703125}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/SweFAQ.json b/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/SweFAQ.json new file mode 100644 index 00000000..5fe887bc --- /dev/null +++ b/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/SweFAQ.json @@ -0,0 +1 @@ +{"task_name":"SweFAQ","task_description":"A Swedish QA dataset derived from FAQ","task_version":"0.0.1","time_of_run":"2023-10-24T10:45:12.224967","scores":{"sv":{"ndcg_at_1":0.42885,"ndcg_at_3":0.54121,"ndcg_at_5":0.57912,"ndcg_at_10":0.60365,"ndcg_at_100":0.64027,"ndcg_at_1000":0.64629,"map_at_1":0.42885,"map_at_3":0.51397,"map_at_5":0.53463,"map_at_10":0.54473,"map_at_100":0.55274,"map_at_1000":0.55301,"recall_at_1":0.42885,"recall_at_3":0.61988,"recall_at_5":0.71345,"recall_at_10":0.78947,"recall_at_100":0.95517,"recall_at_1000":1.0,"precision_at_1":0.42885,"precision_at_3":0.20663,"precision_at_5":0.14269,"precision_at_10":0.07895,"precision_at_100":0.00955,"precision_at_1000":0.001,"mrr_at_1":0.42885,"mrr_at_3":0.51397,"mrr_at_5":0.53463,"mrr_at_10":0.54473,"mrr_at_100":0.55274,"mrr_at_1000":0.55301}},"main_score":"ndcg_at_10"} \ No newline at end of file diff --git a/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/SweReC.json b/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/SweReC.json new file mode 100644 index 00000000..0a28d1b7 --- /dev/null +++ b/src/seb/cache/sentence-transformers__paraphrase-multilingual-mpnet-base-v2/SweReC.json @@ -0,0 +1 @@ +{"task_name":"SweReC","task_description":"A Swedish dataset for sentiment classification on review","task_version":"1.1.1.dev0","time_of_run":"2023-10-24T10:44:35.730553","scores":{"sv":{"accuracy":0.733740234375,"f1":0.6657565173000352,"accuracy_stderr":0.019849758890450868,"f1_stderr":0.016911517656779413,"main_score":0.733740234375}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-dan/Angry_Tweets.json b/src/seb/cache/sonar-dan/Angry_Tweets.json new file mode 100644 index 00000000..c5f3c3f4 --- /dev/null +++ b/src/seb/cache/sonar-dan/Angry_Tweets.json @@ -0,0 +1 @@ +{"task_name":"Angry Tweets","task_description":"A sentiment dataset with 3 classes (positiv, negativ, neutral) for Danish tweets","task_version":"1.1.0","time_of_run":"2023-08-25T10:28:07.260199","scores":{"da":{"accuracy":0.48223495702005736,"f1":0.471627609210049,"accuracy_stderr":0.02476461861197014,"f1_stderr":0.02742719908401787,"main_score":0.48223495702005736}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-dan/Bornholm_Parallel.json b/src/seb/cache/sonar-dan/Bornholm_Parallel.json new file mode 100644 index 00000000..ab7488e9 --- /dev/null +++ b/src/seb/cache/sonar-dan/Bornholm_Parallel.json @@ -0,0 +1 @@ +{"task_name":"Bornholm Parallel","task_description":"Danish Bornholmsk Parallel Corpus. Bornholmsk is a Danish dialect spoken on the island of Bornholm, Denmark. Historically it is a part of east Danish which was also spoken in Scania and Halland, Sweden.","task_version":"1.1.0","time_of_run":"2023-08-25T10:32:17.829353","scores":{"da":{"precision":0.4480674603174603,"recall":0.538,"f1":0.47103601953601953,"accuracy":0.538,"main_score":0.47103601953601953},"da-bornholm":{"precision":0.4480674603174603,"recall":0.538,"f1":0.47103601953601953,"accuracy":0.538,"main_score":0.47103601953601953}},"main_score":"f1"} \ No newline at end of file diff --git a/src/seb/cache/sonar-dan/DKHate.json b/src/seb/cache/sonar-dan/DKHate.json new file mode 100644 index 00000000..be66305d --- /dev/null +++ b/src/seb/cache/sonar-dan/DKHate.json @@ -0,0 +1 @@ +{"task_name":"DKHate","task_description":"Danish Tweets annotated for Hate Speech either being Offensive or not","task_version":"1.1.0","time_of_run":"2023-08-25T10:41:02.216551","scores":{"da":{"accuracy":0.7042553191489362,"f1":0.5582951993211343,"ap":0.17980244246096472,"accuracy_stderr":0.09634874816180004,"f1_stderr":0.06285181232707737,"ap_stderr":0.036989815343104505,"main_score":0.7042553191489362}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-dan/DaLAJ.json b/src/seb/cache/sonar-dan/DaLAJ.json new file mode 100644 index 00000000..6d2d2856 --- /dev/null +++ b/src/seb/cache/sonar-dan/DaLAJ.json @@ -0,0 +1 @@ +{"task_name":"DaLAJ","task_description":"A Swedish dataset for linguistic acceptability. Available as a part of Superlim.","task_version":"1.1.0","time_of_run":"2023-08-25T14:35:56.267582","scores":{"sv":{"accuracy":0.5001126126126126,"f1":0.4944094794301179,"ap":0.5000897722208921,"accuracy_stderr":0.0054579373045282225,"f1_stderr":0.008874404820241408,"ap_stderr":0.0027178282096810497,"main_score":0.5001126126126126}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-dan/Da_Political_Comments.json b/src/seb/cache/sonar-dan/Da_Political_Comments.json new file mode 100644 index 00000000..120f8a3c --- /dev/null +++ b/src/seb/cache/sonar-dan/Da_Political_Comments.json @@ -0,0 +1 @@ +{"task_name":"Da Political Comments","task_description":"A dataset of Danish political comments rated for sentiment","task_version":"1.1.0","time_of_run":"2023-08-25T10:49:16.948165","scores":{"da":{"accuracy":0.33706992230854604,"f1":0.29236712540348914,"accuracy_stderr":0.041607249117834165,"f1_stderr":0.026096375170624454,"main_score":0.33706992230854604}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-dan/LCC.json b/src/seb/cache/sonar-dan/LCC.json new file mode 100644 index 00000000..b3e207dc --- /dev/null +++ b/src/seb/cache/sonar-dan/LCC.json @@ -0,0 +1 @@ +{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.1.0","time_of_run":"2023-08-25T10:29:57.232240","scores":{"da":{"accuracy":0.5313333333333333,"f1":0.5109906324617105,"accuracy_stderr":0.044427218884123036,"f1_stderr":0.03642549622521004,"main_score":0.5313333333333333}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-dan/Language_Identification.json b/src/seb/cache/sonar-dan/Language_Identification.json new file mode 100644 index 00000000..2f39c033 --- /dev/null +++ b/src/seb/cache/sonar-dan/Language_Identification.json @@ -0,0 +1 @@ +{"task_name":"Language Identification","task_description":"A dataset for Nordic language identification.","task_version":"1.1.0","time_of_run":"2023-08-25T12:06:15.790286","scores":{"da":{"accuracy":0.46626666666666666,"f1":0.44771861545457475,"accuracy_stderr":0.012030332035687506,"f1_stderr":0.016963477761452003,"main_score":0.46626666666666666},"sv":{"accuracy":0.46626666666666666,"f1":0.44771861545457475,"accuracy_stderr":0.012030332035687506,"f1_stderr":0.016963477761452003,"main_score":0.46626666666666666},"nb":{"accuracy":0.46626666666666666,"f1":0.44771861545457475,"accuracy_stderr":0.012030332035687506,"f1_stderr":0.016963477761452003,"main_score":0.46626666666666666},"nn":{"accuracy":0.46626666666666666,"f1":0.44771861545457475,"accuracy_stderr":0.012030332035687506,"f1_stderr":0.016963477761452003,"main_score":0.46626666666666666},"is":{"accuracy":0.46626666666666666,"f1":0.44771861545457475,"accuracy_stderr":0.012030332035687506,"f1_stderr":0.016963477761452003,"main_score":0.46626666666666666},"fo":{"accuracy":0.46626666666666666,"f1":0.44771861545457475,"accuracy_stderr":0.012030332035687506,"f1_stderr":0.016963477761452003,"main_score":0.46626666666666666}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-dan/Massive_Intent.json b/src/seb/cache/sonar-dan/Massive_Intent.json new file mode 100644 index 00000000..b367f185 --- /dev/null +++ b/src/seb/cache/sonar-dan/Massive_Intent.json @@ -0,0 +1 @@ +{"task_name":"Massive Intent","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.1.0","time_of_run":"2023-08-25T11:11:07.020456","scores":{"da":{"accuracy":0.5555480833893746,"f1":0.5284120572778492,"accuracy_stderr":0.015421850466044745,"f1_stderr":0.009994939264145613,"main_score":0.5555480833893746},"nb":{"accuracy":0.5447209145931405,"f1":0.5243954360807268,"accuracy_stderr":0.014515613191241795,"f1_stderr":0.01492930026562576,"main_score":0.5447209145931405},"sv":{"accuracy":0.5454270342972427,"f1":0.514006863037079,"accuracy_stderr":0.016491524857694145,"f1_stderr":0.012958066844970926,"main_score":0.5454270342972427}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-dan/Massive_Scenario.json b/src/seb/cache/sonar-dan/Massive_Scenario.json new file mode 100644 index 00000000..d81ba4be --- /dev/null +++ b/src/seb/cache/sonar-dan/Massive_Scenario.json @@ -0,0 +1 @@ +{"task_name":"Massive Scenario","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.1.0","time_of_run":"2023-08-25T11:23:51.662759","scores":{"da":{"accuracy":0.6298251513113653,"f1":0.6181962359822365,"accuracy_stderr":0.022603960053256895,"f1_stderr":0.018562862736360885,"main_score":0.6298251513113653},"nb":{"accuracy":0.6221250840618695,"f1":0.6135479578888746,"accuracy_stderr":0.027533638658030456,"f1_stderr":0.023901383650081473,"main_score":0.6221250840618695},"sv":{"accuracy":0.6299596503026227,"f1":0.614029727732946,"accuracy_stderr":0.019362279002013556,"f1_stderr":0.01713349232253477,"main_score":0.6299596503026227}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-dan/NoReC.json b/src/seb/cache/sonar-dan/NoReC.json new file mode 100644 index 00000000..f77522fb --- /dev/null +++ b/src/seb/cache/sonar-dan/NoReC.json @@ -0,0 +1 @@ +{"task_name":"NoReC","task_description":"A Norwegian dataset for sentiment classification on review","task_version":"1.1.0","time_of_run":"2023-08-25T12:14:28.144891","scores":{"nb":{"accuracy":0.505517578125,"f1":0.46910924223778877,"accuracy_stderr":0.011151749416354637,"f1_stderr":0.014127209939908765,"main_score":0.505517578125}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-dan/Norwegian_parliament.json b/src/seb/cache/sonar-dan/Norwegian_parliament.json new file mode 100644 index 00000000..35c5ff26 --- /dev/null +++ b/src/seb/cache/sonar-dan/Norwegian_parliament.json @@ -0,0 +1 @@ +{"task_name":"Norwegian parliament","task_description":"Norwegian parliament speeches annotated for sentiment","task_version":"1.1.0","time_of_run":"2023-08-25T13:18:39.042706","scores":{"nb":{"accuracy":0.5409166666666667,"f1":0.5376868553564627,"ap":0.5226309585751945,"accuracy_stderr":0.02455563411249375,"f1_stderr":0.022404472828535485,"ap_stderr":0.014111003962086057,"main_score":0.5409166666666667}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-dan/Scala.json b/src/seb/cache/sonar-dan/Scala.json new file mode 100644 index 00000000..ddeeff2f --- /dev/null +++ b/src/seb/cache/sonar-dan/Scala.json @@ -0,0 +1,45 @@ +{ + "task_name": "ScaLA", + "task_description": "A linguistic acceptability task for Danish, Norwegian Bokmål Norwegian Nynorsk and Swedish.", + "task_version": "1.1.0", + "time_of_run": "2023-08-25T11:53:29.468075", + "scores": { + "da": { + "accuracy": 0.5068359375, + "f1": 0.5038855675524828, + "ap": 0.5034959088698512, + "accuracy_stderr": 0.004813967790572749, + "f1_stderr": 0.00605463176030832, + "ap_stderr": 0.0024940701697088135, + "main_score": 0.5068359375 + }, + "nb": { + "accuracy": 0.507373046875, + "f1": 0.5015009381421695, + "ap": 0.5037708525615003, + "accuracy_stderr": 0.006072970442972755, + "f1_stderr": 0.007898122211372672, + "ap_stderr": 0.0031388674252631767, + "main_score": 0.507373046875 + }, + "sv": { + "accuracy": 0.50361328125, + "f1": 0.4971987735169524, + "ap": 0.5018543764248932, + "accuracy_stderr": 0.0061422598412173394, + "f1_stderr": 0.011003668237774279, + "ap_stderr": 0.003096863937270873, + "main_score": 0.50361328125 + }, + "nn": { + "accuracy": 0.502734375, + "f1": 0.49859863230306845, + "ap": 0.5014019861852337, + "accuracy_stderr": 0.004994809317400165, + "f1_stderr": 0.006654980108361048, + "ap_stderr": 0.00250088478407295, + "main_score": 0.502734375 + } + }, + "main_score": "accuracy" +} \ No newline at end of file diff --git a/src/seb/cache/sonar-dan/SweFAQ.json b/src/seb/cache/sonar-dan/SweFAQ.json new file mode 100644 index 00000000..f8ff17d3 --- /dev/null +++ b/src/seb/cache/sonar-dan/SweFAQ.json @@ -0,0 +1 @@ +{"task_name":"SweFAQ","task_description":"A Swedish QA dataset derived from FAQ","task_version":"0.0.1","time_of_run":"2023-09-13T14:53:44.155685","scores":{"sv":{"ndcg_at_1":0.15205,"ndcg_at_3":0.23545,"ndcg_at_5":0.26036,"ndcg_at_10":0.28883,"ndcg_at_100":0.36383,"ndcg_at_1000":0.38829,"map_at_1":0.15205,"map_at_3":0.21507,"map_at_5":0.22891,"map_at_10":0.24035,"map_at_100":0.25366,"map_at_1000":0.25477,"recall_at_1":0.15205,"recall_at_3":0.29435,"recall_at_5":0.35478,"recall_at_10":0.44444,"recall_at_100":0.81871,"recall_at_1000":1.0,"precision_at_1":0.15205,"precision_at_3":0.09812,"precision_at_5":0.07096,"precision_at_10":0.04444,"precision_at_100":0.00819,"precision_at_1000":0.001,"mrr_at_1":0.15205,"mrr_at_3":0.21507,"mrr_at_5":0.22891,"mrr_at_10":0.24035,"mrr_at_100":0.25366,"mrr_at_1000":0.25477}},"main_score":"ndcg_at_10"} \ No newline at end of file diff --git a/src/seb/cache/sonar-dan/SweReC.json b/src/seb/cache/sonar-dan/SweReC.json new file mode 100644 index 00000000..c6b1552b --- /dev/null +++ b/src/seb/cache/sonar-dan/SweReC.json @@ -0,0 +1 @@ +{"task_name":"SweReC","task_description":"A Swedish dataset for sentiment classification on review","task_version":"1.1.0","time_of_run":"2023-08-25T14:32:37.099899","scores":{"sv":{"accuracy":0.668310546875,"f1":0.5479754339119112,"accuracy_stderr":0.040803517075955864,"f1_stderr":0.020328618963022172,"main_score":0.668310546875}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-nno/Angry_Tweets.json b/src/seb/cache/sonar-nno/Angry_Tweets.json new file mode 100644 index 00000000..842dcb9e --- /dev/null +++ b/src/seb/cache/sonar-nno/Angry_Tweets.json @@ -0,0 +1 @@ +{"task_name":"Angry Tweets","task_description":"A sentiment dataset with 3 classes (positiv, negativ, neutral) for Danish tweets","task_version":"1.1.0","time_of_run":"2023-08-25T23:13:00.449195","scores":{"da":{"accuracy":0.4817574021012416,"f1":0.468662927949872,"accuracy_stderr":0.028159385278064015,"f1_stderr":0.028250428553180475,"main_score":0.4817574021012416}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-nno/Bornholm_Parallel.json b/src/seb/cache/sonar-nno/Bornholm_Parallel.json new file mode 100644 index 00000000..075ef217 --- /dev/null +++ b/src/seb/cache/sonar-nno/Bornholm_Parallel.json @@ -0,0 +1 @@ +{"task_name":"Bornholm Parallel","task_description":"Danish Bornholmsk Parallel Corpus. Bornholmsk is a Danish dialect spoken on the island of Bornholm, Denmark. Historically it is a part of east Danish which was also spoken in Scania and Halland, Sweden.","task_version":"1.1.0","time_of_run":"2023-08-25T23:17:18.612137","scores":{"da":{"precision":0.35111670084301666,"recall":0.418,"f1":0.36624463121419637,"accuracy":0.418,"main_score":0.36624463121419637},"da-bornholm":{"precision":0.35111670084301666,"recall":0.418,"f1":0.36624463121419637,"accuracy":0.418,"main_score":0.36624463121419637}},"main_score":"f1"} \ No newline at end of file diff --git a/src/seb/cache/sonar-nno/DKHate.json b/src/seb/cache/sonar-nno/DKHate.json new file mode 100644 index 00000000..d4808dab --- /dev/null +++ b/src/seb/cache/sonar-nno/DKHate.json @@ -0,0 +1 @@ +{"task_name":"DKHate","task_description":"Danish Tweets annotated for Hate Speech either being Offensive or not","task_version":"1.1.0","time_of_run":"2023-08-25T23:25:35.337810","scores":{"da":{"accuracy":0.688145896656535,"f1":0.5471650574183772,"ap":0.1733226610967633,"accuracy_stderr":0.08739011487688383,"f1_stderr":0.05050518474979556,"ap_stderr":0.028009507247693247,"main_score":0.688145896656535}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-nno/DaLAJ.json b/src/seb/cache/sonar-nno/DaLAJ.json new file mode 100644 index 00000000..bacbdb1f --- /dev/null +++ b/src/seb/cache/sonar-nno/DaLAJ.json @@ -0,0 +1 @@ +{"task_name":"DaLAJ","task_description":"A Swedish dataset for linguistic acceptability. Available as a part of Superlim.","task_version":"1.1.0","time_of_run":"2023-08-26T03:22:36.291702","scores":{"sv":{"accuracy":0.4999999999999999,"f1":0.49280955355011924,"ap":0.5000510832066688,"accuracy_stderr":0.006429775866495582,"f1_stderr":0.010075721661433466,"ap_stderr":0.0032309515456516967,"main_score":0.4999999999999999}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-nno/Da_Political_Comments.json b/src/seb/cache/sonar-nno/Da_Political_Comments.json new file mode 100644 index 00000000..e6ff6bb7 --- /dev/null +++ b/src/seb/cache/sonar-nno/Da_Political_Comments.json @@ -0,0 +1 @@ +{"task_name":"Da Political Comments","task_description":"A dataset of Danish political comments rated for sentiment","task_version":"1.1.0","time_of_run":"2023-08-25T23:32:52.924084","scores":{"da":{"accuracy":0.3242508324084351,"f1":0.2842143557182969,"accuracy_stderr":0.033838583162694585,"f1_stderr":0.021369372633368973,"main_score":0.3242508324084351}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-nno/LCC.json b/src/seb/cache/sonar-nno/LCC.json new file mode 100644 index 00000000..3a2c98a8 --- /dev/null +++ b/src/seb/cache/sonar-nno/LCC.json @@ -0,0 +1 @@ +{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.1.0","time_of_run":"2023-08-25T23:14:54.369870","scores":{"da":{"accuracy":0.4833333333333333,"f1":0.45427812223502195,"accuracy_stderr":0.04697517074654083,"f1_stderr":0.04174167041669261,"main_score":0.4833333333333333}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-nno/Language_Identification.json b/src/seb/cache/sonar-nno/Language_Identification.json new file mode 100644 index 00000000..f305d9b0 --- /dev/null +++ b/src/seb/cache/sonar-nno/Language_Identification.json @@ -0,0 +1 @@ +{"task_name":"Language Identification","task_description":"A dataset for Nordic language identification.","task_version":"1.1.0","time_of_run":"2023-08-26T00:47:53.159326","scores":{"da":{"accuracy":0.4469,"f1":0.42873489618887983,"accuracy_stderr":0.012840084804150553,"f1_stderr":0.01471256195768591,"main_score":0.4469},"sv":{"accuracy":0.4469,"f1":0.42873489618887983,"accuracy_stderr":0.012840084804150553,"f1_stderr":0.01471256195768591,"main_score":0.4469},"nb":{"accuracy":0.4469,"f1":0.42873489618887983,"accuracy_stderr":0.012840084804150553,"f1_stderr":0.01471256195768591,"main_score":0.4469},"nn":{"accuracy":0.4469,"f1":0.42873489618887983,"accuracy_stderr":0.012840084804150553,"f1_stderr":0.01471256195768591,"main_score":0.4469},"is":{"accuracy":0.4469,"f1":0.42873489618887983,"accuracy_stderr":0.012840084804150553,"f1_stderr":0.01471256195768591,"main_score":0.4469},"fo":{"accuracy":0.4469,"f1":0.42873489618887983,"accuracy_stderr":0.012840084804150553,"f1_stderr":0.01471256195768591,"main_score":0.4469}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-nno/Massive_Intent.json b/src/seb/cache/sonar-nno/Massive_Intent.json new file mode 100644 index 00000000..9039c319 --- /dev/null +++ b/src/seb/cache/sonar-nno/Massive_Intent.json @@ -0,0 +1 @@ +{"task_name":"Massive Intent","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.1.0","time_of_run":"2023-08-25T23:53:53.192244","scores":{"da":{"accuracy":0.5566240753194351,"f1":0.5323839242957359,"accuracy_stderr":0.012040477104147974,"f1_stderr":0.007248649729570686,"main_score":0.5566240753194351},"nb":{"accuracy":0.5715198386012104,"f1":0.5468272457718947,"accuracy_stderr":0.017190140542183617,"f1_stderr":0.013427902990435272,"main_score":0.5715198386012104},"sv":{"accuracy":0.560659045057162,"f1":0.5361206287714774,"accuracy_stderr":0.013848498425843233,"f1_stderr":0.010785529609859719,"main_score":0.560659045057162}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-nno/Massive_Scenario.json b/src/seb/cache/sonar-nno/Massive_Scenario.json new file mode 100644 index 00000000..705df8f8 --- /dev/null +++ b/src/seb/cache/sonar-nno/Massive_Scenario.json @@ -0,0 +1 @@ +{"task_name":"Massive Scenario","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.1.0","time_of_run":"2023-08-26T00:06:24.467900","scores":{"da":{"accuracy":0.6148957632817754,"f1":0.6050717765397428,"accuracy_stderr":0.019750096082025912,"f1_stderr":0.01908259759987581,"main_score":0.6148957632817754},"nb":{"accuracy":0.6304976462676529,"f1":0.621009335440376,"accuracy_stderr":0.023930546634583553,"f1_stderr":0.02227536705619115,"main_score":0.6304976462676529},"sv":{"accuracy":0.63093476798924,"f1":0.6162597403303389,"accuracy_stderr":0.021703049130845226,"f1_stderr":0.020338232370328325,"main_score":0.63093476798924}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-nno/NoReC.json b/src/seb/cache/sonar-nno/NoReC.json new file mode 100644 index 00000000..0c2a4fc1 --- /dev/null +++ b/src/seb/cache/sonar-nno/NoReC.json @@ -0,0 +1 @@ +{"task_name":"NoReC","task_description":"A Norwegian dataset for sentiment classification on review","task_version":"1.1.0","time_of_run":"2023-08-26T00:55:58.756507","scores":{"nb":{"accuracy":0.48515625,"f1":0.445067234426291,"accuracy_stderr":0.017438136154110533,"f1_stderr":0.01671198333351763,"main_score":0.48515625}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-nno/Norwegian_parliament.json b/src/seb/cache/sonar-nno/Norwegian_parliament.json new file mode 100644 index 00000000..d90b24ee --- /dev/null +++ b/src/seb/cache/sonar-nno/Norwegian_parliament.json @@ -0,0 +1 @@ +{"task_name":"Norwegian parliament","task_description":"Norwegian parliament speeches annotated for sentiment","task_version":"1.1.0","time_of_run":"2023-08-26T02:02:23.467269","scores":{"nb":{"accuracy":0.5459166666666666,"f1":0.5423885866897666,"ap":0.5251987670245654,"accuracy_stderr":0.01817144432588917,"f1_stderr":0.017685037272977367,"ap_stderr":0.0106863545027924,"main_score":0.5459166666666666}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-nno/ScaLA.json b/src/seb/cache/sonar-nno/ScaLA.json new file mode 100644 index 00000000..c3e02d8d --- /dev/null +++ b/src/seb/cache/sonar-nno/ScaLA.json @@ -0,0 +1 @@ +{"task_name":"ScaLA","task_description":"A linguistic acceptability task for Danish, Norwegian Bokmål Norwegian Nynorsk and Swedish.","task_version":"1.1.1.dev0","time_of_run":"2023-09-13T15:18:49.324936","scores":{"da":{"accuracy":0.509228515625,"f1":0.5049006250810197,"ap":0.5047600627546273,"accuracy_stderr":0.007023277000825346,"f1_stderr":0.009062331563372343,"ap_stderr":0.0036683455662138895,"main_score":0.509228515625},"nb":{"accuracy":0.505859375,"f1":0.4996950764593847,"ap":0.5030306722777176,"accuracy_stderr":0.0066735740481523,"f1_stderr":0.006865216157865124,"ap_stderr":0.003465930937682291,"main_score":0.505859375},"sv":{"accuracy":0.507958984375,"f1":0.5007519308349113,"ap":0.504086575283085,"accuracy_stderr":0.006245611557470303,"f1_stderr":0.010041421021678596,"ap_stderr":0.0031985415954774647,"main_score":0.507958984375},"nn":{"accuracy":0.507568359375,"f1":0.5019852110629762,"ap":0.5038729566144591,"accuracy_stderr":0.004291600544738024,"f1_stderr":0.006015342402785477,"ap_stderr":0.002226380696089599,"main_score":0.507568359375}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-nno/SweFAQ.json b/src/seb/cache/sonar-nno/SweFAQ.json new file mode 100644 index 00000000..f9660b1e --- /dev/null +++ b/src/seb/cache/sonar-nno/SweFAQ.json @@ -0,0 +1 @@ +{"task_name":"SweFAQ","task_description":"A Swedish QA dataset derived from FAQ","task_version":"0.0.1","time_of_run":"2023-09-13T15:20:18.140717","scores":{"sv":{"ndcg_at_1":0.1501,"ndcg_at_3":0.22315,"ndcg_at_5":0.2463,"ndcg_at_10":0.28099,"ndcg_at_100":0.35724,"ndcg_at_1000":0.3814,"map_at_1":0.1501,"map_at_3":0.20468,"map_at_5":0.21745,"map_at_10":0.23178,"map_at_100":0.24558,"map_at_1000":0.24663,"recall_at_1":0.1501,"recall_at_3":0.2768,"recall_at_5":0.33333,"recall_at_10":0.44055,"recall_at_100":0.81871,"recall_at_1000":1.0,"precision_at_1":0.1501,"precision_at_3":0.09227,"precision_at_5":0.06667,"precision_at_10":0.04405,"precision_at_100":0.00819,"precision_at_1000":0.001,"mrr_at_1":0.1501,"mrr_at_3":0.20468,"mrr_at_5":0.21745,"mrr_at_10":0.23178,"mrr_at_100":0.24558,"mrr_at_1000":0.24663}},"main_score":"ndcg_at_10"} \ No newline at end of file diff --git a/src/seb/cache/sonar-nno/SweReC.json b/src/seb/cache/sonar-nno/SweReC.json new file mode 100644 index 00000000..03c58484 --- /dev/null +++ b/src/seb/cache/sonar-nno/SweReC.json @@ -0,0 +1 @@ +{"task_name":"SweReC","task_description":"A Swedish dataset for sentiment classification on review","task_version":"1.1.0","time_of_run":"2023-08-26T03:19:01.365855","scores":{"sv":{"accuracy":0.672412109375,"f1":0.5493298218499844,"accuracy_stderr":0.03952150367941831,"f1_stderr":0.02897219073150573,"main_score":0.672412109375}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-nob/Angry_Tweets.json b/src/seb/cache/sonar-nob/Angry_Tweets.json new file mode 100644 index 00000000..818264aa --- /dev/null +++ b/src/seb/cache/sonar-nob/Angry_Tweets.json @@ -0,0 +1 @@ +{"task_name":"Angry Tweets","task_description":"A sentiment dataset with 3 classes (positiv, negativ, neutral) for Danish tweets","task_version":"1.1.0","time_of_run":"2023-08-25T18:59:18.043964","scores":{"da":{"accuracy":0.4815663801337154,"f1":0.46970080266148867,"accuracy_stderr":0.0251859403611836,"f1_stderr":0.026202748525101894,"main_score":0.4815663801337154}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-nob/Bornholm_Parallel.json b/src/seb/cache/sonar-nob/Bornholm_Parallel.json new file mode 100644 index 00000000..c153da2c --- /dev/null +++ b/src/seb/cache/sonar-nob/Bornholm_Parallel.json @@ -0,0 +1 @@ +{"task_name":"Bornholm Parallel","task_description":"Danish Bornholmsk Parallel Corpus. Bornholmsk is a Danish dialect spoken on the island of Bornholm, Denmark. Historically it is a part of east Danish which was also spoken in Scania and Halland, Sweden.","task_version":"1.1.0","time_of_run":"2023-08-25T19:03:42.815894","scores":{"da":{"precision":0.3140888888888889,"recall":0.388,"f1":0.33129538083538085,"accuracy":0.388,"main_score":0.33129538083538085},"da-bornholm":{"precision":0.3140888888888889,"recall":0.388,"f1":0.33129538083538085,"accuracy":0.388,"main_score":0.33129538083538085}},"main_score":"f1"} \ No newline at end of file diff --git a/src/seb/cache/sonar-nob/DKHate.json b/src/seb/cache/sonar-nob/DKHate.json new file mode 100644 index 00000000..4207b046 --- /dev/null +++ b/src/seb/cache/sonar-nob/DKHate.json @@ -0,0 +1 @@ +{"task_name":"DKHate","task_description":"Danish Tweets annotated for Hate Speech either being Offensive or not","task_version":"1.1.0","time_of_run":"2023-08-25T19:12:11.172422","scores":{"da":{"accuracy":0.6966565349544073,"f1":0.5498840348700053,"ap":0.1751076764405522,"accuracy_stderr":0.11254248332242581,"f1_stderr":0.07146710018434102,"ap_stderr":0.04033052874518164,"main_score":0.6966565349544073}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-nob/DaLAJ.json b/src/seb/cache/sonar-nob/DaLAJ.json new file mode 100644 index 00000000..7d14de95 --- /dev/null +++ b/src/seb/cache/sonar-nob/DaLAJ.json @@ -0,0 +1 @@ +{"task_name":"DaLAJ","task_description":"A Swedish dataset for linguistic acceptability. Available as a part of Superlim.","task_version":"1.1.0","time_of_run":"2023-08-25T23:06:48.746773","scores":{"sv":{"accuracy":0.5016891891891893,"f1":0.49259387126683213,"ap":0.5008784116596001,"accuracy_stderr":0.005747657727556241,"f1_stderr":0.010073282243860279,"ap_stderr":0.0028908912345401412,"main_score":0.5016891891891893}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-nob/Da_Political_Comments.json b/src/seb/cache/sonar-nob/Da_Political_Comments.json new file mode 100644 index 00000000..a7d885b2 --- /dev/null +++ b/src/seb/cache/sonar-nob/Da_Political_Comments.json @@ -0,0 +1 @@ +{"task_name":"Da Political Comments","task_description":"A dataset of Danish political comments rated for sentiment","task_version":"1.1.0","time_of_run":"2023-08-25T19:19:43.822956","scores":{"da":{"accuracy":0.3245283018867925,"f1":0.2842529174666799,"accuracy_stderr":0.03912992598037314,"f1_stderr":0.022607450442775085,"main_score":0.3245283018867925}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-nob/LCC.json b/src/seb/cache/sonar-nob/LCC.json new file mode 100644 index 00000000..936e3268 --- /dev/null +++ b/src/seb/cache/sonar-nob/LCC.json @@ -0,0 +1 @@ +{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.1.0","time_of_run":"2023-08-25T19:01:13.830221","scores":{"da":{"accuracy":0.4693333333333333,"f1":0.44149149076283756,"accuracy_stderr":0.0556337227867335,"f1_stderr":0.059331267854567105,"main_score":0.4693333333333333}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-nob/Language_Identification.json b/src/seb/cache/sonar-nob/Language_Identification.json new file mode 100644 index 00000000..2f8adfa0 --- /dev/null +++ b/src/seb/cache/sonar-nob/Language_Identification.json @@ -0,0 +1 @@ +{"task_name":"Language Identification","task_description":"A dataset for Nordic language identification.","task_version":"1.1.0","time_of_run":"2023-08-25T20:34:39.759602","scores":{"da":{"accuracy":0.4918666666666667,"f1":0.4773382827352514,"accuracy_stderr":0.013443875102729057,"f1_stderr":0.01722138526244745,"main_score":0.4918666666666667},"sv":{"accuracy":0.4918666666666667,"f1":0.4773382827352514,"accuracy_stderr":0.013443875102729057,"f1_stderr":0.01722138526244745,"main_score":0.4918666666666667},"nb":{"accuracy":0.4918666666666667,"f1":0.4773382827352514,"accuracy_stderr":0.013443875102729057,"f1_stderr":0.01722138526244745,"main_score":0.4918666666666667},"nn":{"accuracy":0.4918666666666667,"f1":0.4773382827352514,"accuracy_stderr":0.013443875102729057,"f1_stderr":0.01722138526244745,"main_score":0.4918666666666667},"is":{"accuracy":0.4918666666666667,"f1":0.4773382827352514,"accuracy_stderr":0.013443875102729057,"f1_stderr":0.01722138526244745,"main_score":0.4918666666666667},"fo":{"accuracy":0.4918666666666667,"f1":0.4773382827352514,"accuracy_stderr":0.013443875102729057,"f1_stderr":0.01722138526244745,"main_score":0.4918666666666667}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-nob/Massive_Intent.json b/src/seb/cache/sonar-nob/Massive_Intent.json new file mode 100644 index 00000000..d6895fd2 --- /dev/null +++ b/src/seb/cache/sonar-nob/Massive_Intent.json @@ -0,0 +1 @@ +{"task_name":"Massive Intent","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.1.0","time_of_run":"2023-08-25T19:40:50.371002","scores":{"da":{"accuracy":0.5478816408876934,"f1":0.5255079395173758,"accuracy_stderr":0.011808176528751716,"f1_stderr":0.012305301753709167,"main_score":0.5478816408876934},"nb":{"accuracy":0.5505716207128446,"f1":0.5312820524166937,"accuracy_stderr":0.009970403962356815,"f1_stderr":0.010044542252980362,"main_score":0.5505716207128446},"sv":{"accuracy":0.5341963685272361,"f1":0.5109539366858403,"accuracy_stderr":0.016188875696151288,"f1_stderr":0.013650068066477075,"main_score":0.5341963685272361}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-nob/Massive_Scenario.json b/src/seb/cache/sonar-nob/Massive_Scenario.json new file mode 100644 index 00000000..5b01e7f4 --- /dev/null +++ b/src/seb/cache/sonar-nob/Massive_Scenario.json @@ -0,0 +1 @@ +{"task_name":"Massive Scenario","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.1.0","time_of_run":"2023-08-25T19:53:24.436671","scores":{"da":{"accuracy":0.6176529926025556,"f1":0.6038965359995132,"accuracy_stderr":0.019397982666207497,"f1_stderr":0.01685601474320412,"main_score":0.6176529926025556},"nb":{"accuracy":0.6199058507061197,"f1":0.6080631394200557,"accuracy_stderr":0.021997473466064194,"f1_stderr":0.01871154463626231,"main_score":0.6199058507061197},"sv":{"accuracy":0.6209482178883657,"f1":0.5999037279010286,"accuracy_stderr":0.014220565370250168,"f1_stderr":0.013959514394638824,"main_score":0.6209482178883657}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-nob/NoReC.json b/src/seb/cache/sonar-nob/NoReC.json new file mode 100644 index 00000000..31d2b4c6 --- /dev/null +++ b/src/seb/cache/sonar-nob/NoReC.json @@ -0,0 +1 @@ +{"task_name":"NoReC","task_description":"A Norwegian dataset for sentiment classification on review","task_version":"1.1.0","time_of_run":"2023-08-25T20:42:28.131227","scores":{"nb":{"accuracy":0.48720703125,"f1":0.44401944146100975,"accuracy_stderr":0.014648111975549686,"f1_stderr":0.015111728312145809,"main_score":0.48720703125}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-nob/Norwegian_parliament.json b/src/seb/cache/sonar-nob/Norwegian_parliament.json new file mode 100644 index 00000000..12b50fe3 --- /dev/null +++ b/src/seb/cache/sonar-nob/Norwegian_parliament.json @@ -0,0 +1 @@ +{"task_name":"Norwegian parliament","task_description":"Norwegian parliament speeches annotated for sentiment","task_version":"1.1.0","time_of_run":"2023-08-25T21:47:36.021930","scores":{"nb":{"accuracy":0.5540833333333333,"f1":0.550030558323002,"ap":0.5303584049725228,"accuracy_stderr":0.022941501694527328,"f1_stderr":0.021830460008297994,"ap_stderr":0.01375103542445196,"main_score":0.5540833333333333}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-nob/Scala.json b/src/seb/cache/sonar-nob/Scala.json new file mode 100644 index 00000000..2ce890a9 --- /dev/null +++ b/src/seb/cache/sonar-nob/Scala.json @@ -0,0 +1,45 @@ +{ + "task_name": "ScaLA", + "task_description": "A linguistic acceptability task for Danish, Norwegian Bokmål Norwegian Nynorsk and Swedish.", + "task_version": "1.1.0", + "time_of_run": "2023-08-25T20:22:16.198541", + "scores": { + "da": { + "accuracy": 0.51025390625, + "f1": 0.5046687698263934, + "ap": 0.5052973250024836, + "accuracy_stderr": 0.007462880146305782, + "f1_stderr": 0.01174318988986479, + "ap_stderr": 0.003902661675853376, + "main_score": 0.51025390625 + }, + "nb": { + "accuracy": 0.509033203125, + "f1": 0.5012338348445107, + "ap": 0.5046583486592466, + "accuracy_stderr": 0.007059844471458393, + "f1_stderr": 0.01353050930168835, + "ap_stderr": 0.0036397114939476123, + "main_score": 0.509033203125 + }, + "sv": { + "accuracy": 0.504248046875, + "f1": 0.49730799632788686, + "ap": 0.5021744712292657, + "accuracy_stderr": 0.006149436627147143, + "f1_stderr": 0.00734213870640468, + "ap_stderr": 0.003092939592085986, + "main_score": 0.504248046875 + }, + "nn": { + "accuracy": 0.51025390625, + "f1": 0.5022360666316321, + "ap": 0.5052925673800392, + "accuracy_stderr": 0.006525454637612544, + "f1_stderr": 0.010887357278712824, + "ap_stderr": 0.0034355817047167485, + "main_score": 0.51025390625 + } + }, + "main_score": "accuracy" +} \ No newline at end of file diff --git a/src/seb/cache/sonar-nob/SweFAQ.json b/src/seb/cache/sonar-nob/SweFAQ.json new file mode 100644 index 00000000..1d438f5e --- /dev/null +++ b/src/seb/cache/sonar-nob/SweFAQ.json @@ -0,0 +1 @@ +{"task_name":"SweFAQ","task_description":"A Swedish QA dataset derived from FAQ","task_version":"0.0.1","time_of_run":"2023-09-13T15:09:16.561743","scores":{"sv":{"ndcg_at_1":0.14815,"ndcg_at_3":0.22658,"ndcg_at_5":0.24529,"ndcg_at_10":0.27498,"ndcg_at_100":0.35356,"ndcg_at_1000":0.37834,"map_at_1":0.14815,"map_at_3":0.20663,"map_at_5":0.21715,"map_at_10":0.22906,"map_at_100":0.24318,"map_at_1000":0.24428,"recall_at_1":0.14815,"recall_at_3":0.2846,"recall_at_5":0.32943,"recall_at_10":0.423,"recall_at_100":0.81481,"recall_at_1000":1.0,"precision_at_1":0.14815,"precision_at_3":0.09487,"precision_at_5":0.06589,"precision_at_10":0.0423,"precision_at_100":0.00815,"precision_at_1000":0.001,"mrr_at_1":0.14815,"mrr_at_3":0.20663,"mrr_at_5":0.21715,"mrr_at_10":0.22906,"mrr_at_100":0.24318,"mrr_at_1000":0.24428}},"main_score":"ndcg_at_10"} \ No newline at end of file diff --git a/src/seb/cache/sonar-nob/SweReC.json b/src/seb/cache/sonar-nob/SweReC.json new file mode 100644 index 00000000..137db95d --- /dev/null +++ b/src/seb/cache/sonar-nob/SweReC.json @@ -0,0 +1 @@ +{"task_name":"SweReC","task_description":"A Swedish dataset for sentiment classification on review","task_version":"1.1.0","time_of_run":"2023-08-25T23:03:23.875113","scores":{"sv":{"accuracy":0.668994140625,"f1":0.548086404471206,"accuracy_stderr":0.036535348836847406,"f1_stderr":0.024885277806070808,"main_score":0.668994140625}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-swe/Angry_Tweets.json b/src/seb/cache/sonar-swe/Angry_Tweets.json new file mode 100644 index 00000000..1802e6c7 --- /dev/null +++ b/src/seb/cache/sonar-swe/Angry_Tweets.json @@ -0,0 +1 @@ +{"task_name":"Angry Tweets","task_description":"A sentiment dataset with 3 classes (positiv, negativ, neutral) for Danish tweets","task_version":"1.1.0","time_of_run":"2023-08-25T14:42:08.067347","scores":{"da":{"accuracy":0.47803247373447955,"f1":0.466732460453582,"accuracy_stderr":0.024459950835037294,"f1_stderr":0.02363685721527473,"main_score":0.47803247373447955}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-swe/Bornholm_Parallel.json b/src/seb/cache/sonar-swe/Bornholm_Parallel.json new file mode 100644 index 00000000..df3f0ee1 --- /dev/null +++ b/src/seb/cache/sonar-swe/Bornholm_Parallel.json @@ -0,0 +1 @@ +{"task_name":"Bornholm Parallel","task_description":"Danish Bornholmsk Parallel Corpus. Bornholmsk is a Danish dialect spoken on the island of Bornholm, Denmark. Historically it is a part of east Danish which was also spoken in Scania and Halland, Sweden.","task_version":"1.1.0","time_of_run":"2023-08-25T14:46:28.941172","scores":{"da":{"precision":0.45882844932844935,"recall":0.546,"f1":0.4813063936063936,"accuracy":0.546,"main_score":0.4813063936063936},"da-bornholm":{"precision":0.45882844932844935,"recall":0.546,"f1":0.4813063936063936,"accuracy":0.546,"main_score":0.4813063936063936}},"main_score":"f1"} \ No newline at end of file diff --git a/src/seb/cache/sonar-swe/DKHate.json b/src/seb/cache/sonar-swe/DKHate.json new file mode 100644 index 00000000..6da71ccd --- /dev/null +++ b/src/seb/cache/sonar-swe/DKHate.json @@ -0,0 +1 @@ +{"task_name":"DKHate","task_description":"Danish Tweets annotated for Hate Speech either being Offensive or not","task_version":"1.1.0","time_of_run":"2023-08-25T14:55:10.611295","scores":{"da":{"accuracy":0.6996960486322188,"f1":0.5450336734711327,"ap":0.16756852219954438,"accuracy_stderr":0.09607748402799435,"f1_stderr":0.05698815412616313,"ap_stderr":0.030320902299273256,"main_score":0.6996960486322188}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-swe/DaLAJ.json b/src/seb/cache/sonar-swe/DaLAJ.json new file mode 100644 index 00000000..ce39d83f --- /dev/null +++ b/src/seb/cache/sonar-swe/DaLAJ.json @@ -0,0 +1 @@ +{"task_name":"DaLAJ","task_description":"A Swedish dataset for linguistic acceptability. Available as a part of Superlim.","task_version":"1.1.0","time_of_run":"2023-08-25T18:53:19.514484","scores":{"sv":{"accuracy":0.5004504504504504,"f1":0.4948578760810561,"ap":0.5002574261238442,"accuracy_stderr":0.005165470694225098,"f1_stderr":0.005315320742576052,"ap_stderr":0.002572447096981639,"main_score":0.5004504504504504}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-swe/Da_Political_Comments.json b/src/seb/cache/sonar-swe/Da_Political_Comments.json new file mode 100644 index 00000000..38ee3941 --- /dev/null +++ b/src/seb/cache/sonar-swe/Da_Political_Comments.json @@ -0,0 +1 @@ +{"task_name":"Da Political Comments","task_description":"A dataset of Danish political comments rated for sentiment","task_version":"1.1.0","time_of_run":"2023-08-25T15:02:50.664985","scores":{"da":{"accuracy":0.3169256381798003,"f1":0.27971012346301705,"accuracy_stderr":0.03867606042330829,"f1_stderr":0.023056115586375558,"main_score":0.3169256381798003}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-swe/LCC.json b/src/seb/cache/sonar-swe/LCC.json new file mode 100644 index 00000000..ae5bc356 --- /dev/null +++ b/src/seb/cache/sonar-swe/LCC.json @@ -0,0 +1 @@ +{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.1.0","time_of_run":"2023-08-25T14:44:00.205628","scores":{"da":{"accuracy":0.5293333333333334,"f1":0.5098465056493404,"accuracy_stderr":0.04513682901282867,"f1_stderr":0.03998789721907398,"main_score":0.5293333333333334}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-swe/Language_Identification.json b/src/seb/cache/sonar-swe/Language_Identification.json new file mode 100644 index 00000000..6d11ab20 --- /dev/null +++ b/src/seb/cache/sonar-swe/Language_Identification.json @@ -0,0 +1 @@ +{"task_name":"Language Identification","task_description":"A dataset for Nordic language identification.","task_version":"1.1.0","time_of_run":"2023-08-25T16:19:04.900942","scores":{"da":{"accuracy":0.45920000000000005,"f1":0.4333159560719455,"accuracy_stderr":0.01363997393285225,"f1_stderr":0.01344601089991843,"main_score":0.45920000000000005},"sv":{"accuracy":0.45920000000000005,"f1":0.4333159560719455,"accuracy_stderr":0.01363997393285225,"f1_stderr":0.01344601089991843,"main_score":0.45920000000000005},"nb":{"accuracy":0.45920000000000005,"f1":0.4333159560719455,"accuracy_stderr":0.01363997393285225,"f1_stderr":0.01344601089991843,"main_score":0.45920000000000005},"nn":{"accuracy":0.45920000000000005,"f1":0.4333159560719455,"accuracy_stderr":0.01363997393285225,"f1_stderr":0.01344601089991843,"main_score":0.45920000000000005},"is":{"accuracy":0.45920000000000005,"f1":0.4333159560719455,"accuracy_stderr":0.01363997393285225,"f1_stderr":0.01344601089991843,"main_score":0.45920000000000005},"fo":{"accuracy":0.45920000000000005,"f1":0.4333159560719455,"accuracy_stderr":0.01363997393285225,"f1_stderr":0.01344601089991843,"main_score":0.45920000000000005}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-swe/Massive_Intent.json b/src/seb/cache/sonar-swe/Massive_Intent.json new file mode 100644 index 00000000..eff35653 --- /dev/null +++ b/src/seb/cache/sonar-swe/Massive_Intent.json @@ -0,0 +1 @@ +{"task_name":"Massive Intent","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.1.0","time_of_run":"2023-08-25T15:24:52.893683","scores":{"da":{"accuracy":0.531607262945528,"f1":0.5001509970641951,"accuracy_stderr":0.011762896080211447,"f1_stderr":0.009606754098565996,"main_score":0.531607262945528},"nb":{"accuracy":0.5185272360457296,"f1":0.4913413045084699,"accuracy_stderr":0.011555813710775893,"f1_stderr":0.011544809864478837,"main_score":0.5185272360457296},"sv":{"accuracy":0.5752185608607936,"f1":0.5465171004538218,"accuracy_stderr":0.013005476961443814,"f1_stderr":0.010464756001748642,"main_score":0.5752185608607936}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-swe/Massive_Scenario.json b/src/seb/cache/sonar-swe/Massive_Scenario.json new file mode 100644 index 00000000..bcee4b9e --- /dev/null +++ b/src/seb/cache/sonar-swe/Massive_Scenario.json @@ -0,0 +1 @@ +{"task_name":"Massive Scenario","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.1.0","time_of_run":"2023-08-25T15:37:39.841901","scores":{"da":{"accuracy":0.5999663752521855,"f1":0.5827803147643623,"accuracy_stderr":0.02068129742818657,"f1_stderr":0.019310926556921647,"main_score":0.5999663752521855},"nb":{"accuracy":0.5878950907868191,"f1":0.5773294342564773,"accuracy_stderr":0.025093255156122443,"f1_stderr":0.020854809459798913,"main_score":0.5878950907868191},"sv":{"accuracy":0.647848016139879,"f1":0.6336081856889688,"accuracy_stderr":0.019368000681285337,"f1_stderr":0.01740611220723243,"main_score":0.647848016139879}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-swe/NoReC.json b/src/seb/cache/sonar-swe/NoReC.json new file mode 100644 index 00000000..e2ba5b37 --- /dev/null +++ b/src/seb/cache/sonar-swe/NoReC.json @@ -0,0 +1 @@ +{"task_name":"NoReC","task_description":"A Norwegian dataset for sentiment classification on review","task_version":"1.1.0","time_of_run":"2023-08-25T16:27:10.084740","scores":{"nb":{"accuracy":0.49912109375,"f1":0.46646270381718374,"accuracy_stderr":0.011404895440759132,"f1_stderr":0.011266772234324993,"main_score":0.49912109375}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-swe/Norwegian_parliament.json b/src/seb/cache/sonar-swe/Norwegian_parliament.json new file mode 100644 index 00000000..9b6bf481 --- /dev/null +++ b/src/seb/cache/sonar-swe/Norwegian_parliament.json @@ -0,0 +1 @@ +{"task_name":"Norwegian parliament","task_description":"Norwegian parliament speeches annotated for sentiment","task_version":"1.1.0","time_of_run":"2023-08-25T17:33:02.139653","scores":{"nb":{"accuracy":0.5436666666666665,"f1":0.5400112256293337,"ap":0.5240383118358639,"accuracy_stderr":0.017982244329090624,"f1_stderr":0.017032604548552623,"ap_stderr":0.01031673036190653,"main_score":0.5436666666666665}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/sonar-swe/Scala.json b/src/seb/cache/sonar-swe/Scala.json new file mode 100644 index 00000000..72a892a2 --- /dev/null +++ b/src/seb/cache/sonar-swe/Scala.json @@ -0,0 +1,45 @@ +{ + "task_name": "ScaLA", + "task_description": "A linguistic acceptability task for Danish, Norwegian Bokmål Norwegian Nynorsk and Swedish.", + "task_version": "1.1.0", + "time_of_run": "2023-08-25T16:06:36.552819", + "scores": { + "da": { + "accuracy": 0.505029296875, + "f1": 0.5022281135095892, + "ap": 0.5026128321313533, + "accuracy_stderr": 0.007945792844762866, + "f1_stderr": 0.008062920450712349, + "ap_stderr": 0.004003441094047779, + "main_score": 0.505029296875 + }, + "nb": { + "accuracy": 0.50322265625, + "f1": 0.49649018627643027, + "ap": 0.5016432924317262, + "accuracy_stderr": 0.00478515625, + "f1_stderr": 0.007727822642397125, + "ap_stderr": 0.0024123895074655657, + "main_score": 0.50322265625 + }, + "sv": { + "accuracy": 0.509619140625, + "f1": 0.5026727387729569, + "ap": 0.5049469083709516, + "accuracy_stderr": 0.007739065270179824, + "f1_stderr": 0.010959255102129518, + "ap_stderr": 0.004018014966811578, + "main_score": 0.509619140625 + }, + "nn": { + "accuracy": 0.50390625, + "f1": 0.49991574405235023, + "ap": 0.502037754938025, + "accuracy_stderr": 0.007437278423695223, + "f1_stderr": 0.007768645432116215, + "ap_stderr": 0.0038227742208500172, + "main_score": 0.50390625 + } + }, + "main_score": "accuracy" +} \ No newline at end of file diff --git a/src/seb/cache/sonar-swe/SweFAQ.json b/src/seb/cache/sonar-swe/SweFAQ.json new file mode 100644 index 00000000..e1468abb --- /dev/null +++ b/src/seb/cache/sonar-swe/SweFAQ.json @@ -0,0 +1 @@ +{"task_name":"SweFAQ","task_description":"A Swedish QA dataset derived from FAQ","task_version":"0.0.1","time_of_run":"2023-09-13T14:55:19.888245","scores":{"sv":{"ndcg_at_1":0.16764,"ndcg_at_3":0.25592,"ndcg_at_5":0.2784,"ndcg_at_10":0.31201,"ndcg_at_100":0.38962,"ndcg_at_1000":0.4083,"map_at_1":0.16764,"map_at_3":0.23392,"map_at_5":0.24639,"map_at_10":0.26001,"map_at_100":0.27428,"map_at_1000":0.27513,"recall_at_1":0.16764,"recall_at_3":0.31969,"recall_at_5":0.37427,"recall_at_10":0.47953,"recall_at_100":0.8616,"recall_at_1000":1.0,"precision_at_1":0.16764,"precision_at_3":0.10656,"precision_at_5":0.07485,"precision_at_10":0.04795,"precision_at_100":0.00862,"precision_at_1000":0.001,"mrr_at_1":0.16764,"mrr_at_3":0.23392,"mrr_at_5":0.24639,"mrr_at_10":0.26001,"mrr_at_100":0.27428,"mrr_at_1000":0.27513}},"main_score":"ndcg_at_10"} \ No newline at end of file diff --git a/src/seb/cache/sonar-swe/SweReC.json b/src/seb/cache/sonar-swe/SweReC.json new file mode 100644 index 00000000..4a338527 --- /dev/null +++ b/src/seb/cache/sonar-swe/SweReC.json @@ -0,0 +1 @@ +{"task_name":"SweReC","task_description":"A Swedish dataset for sentiment classification on review","task_version":"1.1.0","time_of_run":"2023-08-25T18:49:41.900384","scores":{"sv":{"accuracy":0.668701171875,"f1":0.5475017096649799,"accuracy_stderr":0.03840092228934247,"f1_stderr":0.0225870709584527,"main_score":0.668701171875}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/text-embedding-ada-002/Angry_Tweets.json b/src/seb/cache/text-embedding-ada-002/Angry_Tweets.json new file mode 100644 index 00000000..8524b056 --- /dev/null +++ b/src/seb/cache/text-embedding-ada-002/Angry_Tweets.json @@ -0,0 +1 @@ +{"task_name":"Angry Tweets","task_description":"A sentiment dataset with 3 classes (positiv, negativ, neutral) for Danish tweets","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T13:57:01.409821","scores":{"da":{"accuracy":0.5663801337153772,"f1":0.556615577703742,"accuracy_stderr":0.02611836872568494,"f1_stderr":0.025872948893508194,"main_score":0.5663801337153772}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/text-embedding-ada-002/Bornholm_Parallel.json b/src/seb/cache/text-embedding-ada-002/Bornholm_Parallel.json new file mode 100644 index 00000000..dbe8df24 --- /dev/null +++ b/src/seb/cache/text-embedding-ada-002/Bornholm_Parallel.json @@ -0,0 +1 @@ +{"task_name":"Bornholm Parallel","task_description":"Danish Bornholmsk Parallel Corpus. Bornholmsk is a Danish dialect spoken on the island of Bornholm, Denmark. Historically it is a part of east Danish which was also spoken in Scania and Halland, Sweden.","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T13:57:22.744047","scores":{"da":{"precision":0.38812896825396825,"recall":0.486,"f1":0.4119671844495374,"accuracy":0.486,"main_score":0.4119671844495374},"da-bornholm":{"precision":0.38812896825396825,"recall":0.486,"f1":0.4119671844495374,"accuracy":0.486,"main_score":0.4119671844495374}},"main_score":"f1"} \ No newline at end of file diff --git a/src/seb/cache/text-embedding-ada-002/DKHate.json b/src/seb/cache/text-embedding-ada-002/DKHate.json new file mode 100644 index 00000000..5cdb804e --- /dev/null +++ b/src/seb/cache/text-embedding-ada-002/DKHate.json @@ -0,0 +1 @@ +{"task_name":"DKHate","task_description":"Danish Tweets annotated for Hate Speech either being Offensive or not","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T13:57:37.623388","scores":{"da":{"accuracy":0.6556231003039514,"f1":0.5329371638763648,"ap":0.16937533683906875,"accuracy_stderr":0.06536085291277825,"f1_stderr":0.042492520098797205,"ap_stderr":0.021055847287488553,"main_score":0.6556231003039514}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/text-embedding-ada-002/DaLAJ.json b/src/seb/cache/text-embedding-ada-002/DaLAJ.json new file mode 100644 index 00000000..8d0d87a5 --- /dev/null +++ b/src/seb/cache/text-embedding-ada-002/DaLAJ.json @@ -0,0 +1 @@ +{"task_name":"DaLAJ","task_description":"A Swedish dataset for linguistic acceptability. Available as a part of Superlim.","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T15:24:40.690636","scores":{"sv":{"accuracy":0.5009009009009009,"f1":0.4983667766559349,"ap":0.5004757427091461,"accuracy_stderr":0.004903500238080445,"f1_stderr":0.005425037134598474,"ap_stderr":0.0024776425075315814,"main_score":0.5009009009009009}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/text-embedding-ada-002/Da_Political_Comments.json b/src/seb/cache/text-embedding-ada-002/Da_Political_Comments.json new file mode 100644 index 00000000..662c0ec9 --- /dev/null +++ b/src/seb/cache/text-embedding-ada-002/Da_Political_Comments.json @@ -0,0 +1 @@ +{"task_name":"Da Political Comments","task_description":"A dataset of Danish political comments rated for sentiment","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T13:57:58.279151","scores":{"da":{"accuracy":0.4032741398446171,"f1":0.3605467733846693,"accuracy_stderr":0.02337479037033965,"f1_stderr":0.021171582141671797,"main_score":0.4032741398446171}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/text-embedding-ada-002/LCC.json b/src/seb/cache/text-embedding-ada-002/LCC.json new file mode 100644 index 00000000..3f9df414 --- /dev/null +++ b/src/seb/cache/text-embedding-ada-002/LCC.json @@ -0,0 +1 @@ +{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T13:57:16.96858","scores":{"da":{"accuracy":0.6193333333333333,"f1":0.6156940649480169,"accuracy_stderr":0.029431653406192162,"f1_stderr":0.027473715475941973,"main_score":0.6193333333333333}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/text-embedding-ada-002/Language_Identification.json b/src/seb/cache/text-embedding-ada-002/Language_Identification.json new file mode 100644 index 00000000..95fb7ed5 --- /dev/null +++ b/src/seb/cache/text-embedding-ada-002/Language_Identification.json @@ -0,0 +1 @@ +{"task_name":"Language Identification","task_description":"A dataset for Nordic language identification.","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T14:38:10.405058","scores":{"da":{"accuracy":0.6979666666666666,"f1":0.678327826645535,"accuracy_stderr":0.013505924625881795,"f1_stderr":0.020176751143484856,"main_score":0.6979666666666666},"sv":{"accuracy":0.6979666666666666,"f1":0.678327826645535,"accuracy_stderr":0.013505924625881795,"f1_stderr":0.020176751143484856,"main_score":0.6979666666666666},"nb":{"accuracy":0.6979666666666666,"f1":0.678327826645535,"accuracy_stderr":0.013505924625881795,"f1_stderr":0.020176751143484856,"main_score":0.6979666666666666},"nn":{"accuracy":0.6979666666666666,"f1":0.678327826645535,"accuracy_stderr":0.013505924625881795,"f1_stderr":0.020176751143484856,"main_score":0.6979666666666666},"is":{"accuracy":0.6979666666666666,"f1":0.678327826645535,"accuracy_stderr":0.013505924625881795,"f1_stderr":0.020176751143484856,"main_score":0.6979666666666666},"fo":{"accuracy":0.6979666666666666,"f1":0.678327826645535,"accuracy_stderr":0.013505924625881795,"f1_stderr":0.020176751143484856,"main_score":0.6979666666666666}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/text-embedding-ada-002/Massive_Intent.json b/src/seb/cache/text-embedding-ada-002/Massive_Intent.json new file mode 100644 index 00000000..f79b6ab1 --- /dev/null +++ b/src/seb/cache/text-embedding-ada-002/Massive_Intent.json @@ -0,0 +1 @@ +{"task_name":"Massive Intent","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T14:29:25.012835","scores":{"da":{"accuracy":0.6121721587088096,"f1":0.577040892283424,"accuracy_stderr":0.016140019053415677,"f1_stderr":0.015470824849175313,"main_score":0.6121721587088096},"nb":{"accuracy":0.6167451244115669,"f1":0.5726733728422938,"accuracy_stderr":0.015554708031324825,"f1_stderr":0.012798053555369936,"main_score":0.6167451244115669},"sv":{"accuracy":0.6248823133826497,"f1":0.5889053268742211,"accuracy_stderr":0.018948492650780892,"f1_stderr":0.02256084461506101,"main_score":0.6248823133826497}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/text-embedding-ada-002/Massive_Scenario.json b/src/seb/cache/text-embedding-ada-002/Massive_Scenario.json new file mode 100644 index 00000000..024b9d2a --- /dev/null +++ b/src/seb/cache/text-embedding-ada-002/Massive_Scenario.json @@ -0,0 +1 @@ +{"task_name":"Massive Scenario","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T14:33:41.851913","scores":{"da":{"accuracy":0.6973772696704774,"f1":0.6873177613903919,"accuracy_stderr":0.010731510917672457,"f1_stderr":0.011862237531415644,"main_score":0.6973772696704774},"nb":{"accuracy":0.6894082044384667,"f1":0.6774224144029255,"accuracy_stderr":0.01205253745804449,"f1_stderr":0.011087076809301585,"main_score":0.6894082044384667},"sv":{"accuracy":0.6959986550100874,"f1":0.6856409803455563,"accuracy_stderr":0.013342247393978608,"f1_stderr":0.012806850519009748,"main_score":0.6959986550100874}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/text-embedding-ada-002/NoReC.json b/src/seb/cache/text-embedding-ada-002/NoReC.json new file mode 100644 index 00000000..6d457b10 --- /dev/null +++ b/src/seb/cache/text-embedding-ada-002/NoReC.json @@ -0,0 +1 @@ +{"task_name":"NoReC","task_description":"A Norwegian dataset for sentiment classification on review","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T14:38:56.064979","scores":{"nb":{"accuracy":0.583642578125,"f1":0.5642185585919139,"accuracy_stderr":0.02510169327028359,"f1_stderr":0.021900935726049505,"main_score":0.583642578125}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/text-embedding-ada-002/Norwegian_parliament.json b/src/seb/cache/text-embedding-ada-002/Norwegian_parliament.json new file mode 100644 index 00000000..2b7010d3 --- /dev/null +++ b/src/seb/cache/text-embedding-ada-002/Norwegian_parliament.json @@ -0,0 +1 @@ +{"task_name":"Norwegian parliament","task_description":"Norwegian parliament speeches annotated for sentiment","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T15:23:25.273753","scores":{"nb":{"accuracy":0.5771666666666666,"f1":0.5745307683135964,"ap":0.5456244805295153,"accuracy_stderr":0.028265113951064597,"f1_stderr":0.028910297340516485,"ap_stderr":0.01800783514140835,"main_score":0.5771666666666666}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/text-embedding-ada-002/ScaLA.json b/src/seb/cache/text-embedding-ada-002/ScaLA.json new file mode 100644 index 00000000..33ea95d5 --- /dev/null +++ b/src/seb/cache/text-embedding-ada-002/ScaLA.json @@ -0,0 +1 @@ +{"task_name":"ScaLA","task_description":"A linguistic acceptability task for Danish, Norwegian Bokmål Norwegian Nynorsk and Swedish.","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T14:36:37.342832","scores":{"da":{"accuracy":0.5044921875,"f1":0.4979940070321585,"ap":0.5022811631258827,"accuracy_stderr":0.004532344291356364,"f1_stderr":0.006412155172784267,"ap_stderr":0.002296298420339464,"main_score":0.5044921875},"nb":{"accuracy":0.50322265625,"f1":0.49919133524792947,"ap":0.5016708668939369,"accuracy_stderr":0.0073702977594192216,"f1_stderr":0.008489857856462987,"ap_stderr":0.003701108868008933,"main_score":0.50322265625},"sv":{"accuracy":0.50068359375,"f1":0.4962123787313528,"ap":0.5003736548375419,"accuracy_stderr":0.005615022074791036,"f1_stderr":0.008823609598588298,"ap_stderr":0.002815574011291658,"main_score":0.50068359375},"nn":{"accuracy":0.500927734375,"f1":0.4998816306858206,"ap":0.5005224905843788,"accuracy_stderr":0.007634851564014916,"f1_stderr":0.008013784591981389,"ap_stderr":0.0038935528640665846,"main_score":0.500927734375}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/text-embedding-ada-002/SweFAQ.json b/src/seb/cache/text-embedding-ada-002/SweFAQ.json new file mode 100644 index 00000000..8d39f05c --- /dev/null +++ b/src/seb/cache/text-embedding-ada-002/SweFAQ.json @@ -0,0 +1 @@ +{"task_name":"SweFAQ","task_description":"A Swedish QA dataset derived from FAQ","task_version":"0.0.1","time_of_run":"2023-09-25T15:25:37.299751","scores":{"sv":{"ndcg_at_1":0.5653,"ndcg_at_3":0.67766,"ndcg_at_5":0.70057,"ndcg_at_10":0.7226,"ndcg_at_100":0.74435,"ndcg_at_1000":0.74766,"map_at_1":0.5653,"map_at_3":0.65042,"map_at_5":0.66339,"map_at_10":0.67245,"map_at_100":0.67748,"map_at_1000":0.67761,"recall_at_1":0.5653,"recall_at_3":0.75634,"recall_at_5":0.81092,"recall_at_10":0.87914,"recall_at_100":0.97466,"recall_at_1000":1.0,"precision_at_1":0.5653,"precision_at_3":0.25211,"precision_at_5":0.16218,"precision_at_10":0.08791,"precision_at_100":0.00975,"precision_at_1000":0.001,"mrr_at_1":0.5653,"mrr_at_3":0.65042,"mrr_at_5":0.66339,"mrr_at_10":0.67245,"mrr_at_100":0.67748,"mrr_at_1000":0.67761}},"main_score":"ndcg_at_10"} \ No newline at end of file diff --git a/src/seb/cache/text-embedding-ada-002/SweReC.json b/src/seb/cache/text-embedding-ada-002/SweReC.json new file mode 100644 index 00000000..f69b4b8e --- /dev/null +++ b/src/seb/cache/text-embedding-ada-002/SweReC.json @@ -0,0 +1 @@ +{"task_name":"SweReC","task_description":"A Swedish dataset for sentiment classification on review","task_version":"1.1.1.dev0","time_of_run":"2023-09-25T15:24:15.919024","scores":{"sv":{"accuracy":0.7955078125,"f1":0.6944839591682725,"accuracy_stderr":0.01435380783333838,"f1_stderr":0.015842222266049637,"main_score":0.7955078125}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/vesteinn__DanskBERT/Angry_Tweets.json b/src/seb/cache/vesteinn__DanskBERT/Angry_Tweets.json new file mode 100644 index 00000000..062f7056 --- /dev/null +++ b/src/seb/cache/vesteinn__DanskBERT/Angry_Tweets.json @@ -0,0 +1 @@ +{"task_name":"Angry Tweets","task_description":"A sentiment dataset with 3 classes (positiv, negativ, neutral) for Danish tweets","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:37:54.652194","scores":{"da":{"accuracy":0.5427889207258835,"f1":0.5339328890266668,"accuracy_stderr":0.01993435732781835,"f1_stderr":0.017878917919632555,"main_score":0.5427889207258835}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/vesteinn__DanskBERT/Bornholm_Parallel.json b/src/seb/cache/vesteinn__DanskBERT/Bornholm_Parallel.json new file mode 100644 index 00000000..f6d956fc --- /dev/null +++ b/src/seb/cache/vesteinn__DanskBERT/Bornholm_Parallel.json @@ -0,0 +1 @@ +{"task_name":"Bornholm Parallel","task_description":"Danish Bornholmsk Parallel Corpus. Bornholmsk is a Danish dialect spoken on the island of Bornholm, Denmark. Historically it is a part of east Danish which was also spoken in Scania and Halland, Sweden.","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:38:18.996132","scores":{"da":{"precision":0.059330365960386246,"recall":0.09,"f1":0.06338898363570435,"accuracy":0.09,"main_score":0.06338898363570435},"da-bornholm":{"precision":0.059330365960386246,"recall":0.09,"f1":0.06338898363570435,"accuracy":0.09,"main_score":0.06338898363570435}},"main_score":"f1"} \ No newline at end of file diff --git a/src/seb/cache/vesteinn__DanskBERT/DKHate.json b/src/seb/cache/vesteinn__DanskBERT/DKHate.json new file mode 100644 index 00000000..88a8e04b --- /dev/null +++ b/src/seb/cache/vesteinn__DanskBERT/DKHate.json @@ -0,0 +1 @@ +{"task_name":"DKHate","task_description":"Danish Tweets annotated for Hate Speech either being Offensive or not","task_version":"1.0.3.dev0","time_of_run":"2023-07-30T13:56:16.712352","scores":{"da":{"accuracy":0.5930091185410334,"f1":0.48968337481112795,"ap":0.8946529290076611,"accuracy_stderr":0.09222758041789648,"f1_stderr":0.06219457537723159,"ap_stderr":0.011813322596830038,"main_score":0.5930091185410334}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/vesteinn__DanskBERT/DaLAJ.json b/src/seb/cache/vesteinn__DanskBERT/DaLAJ.json new file mode 100644 index 00000000..8a0aae0b --- /dev/null +++ b/src/seb/cache/vesteinn__DanskBERT/DaLAJ.json @@ -0,0 +1 @@ +{"task_name":"DaLAJ","task_description":"A Swedish dataset for linguistic accebtablity. Available as a part of Superlim.","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T21:12:07.93813","scores":{"sv":{"accuracy":0.5031531531531531,"f1":0.5006833169570167,"ap":0.5016233522210309,"accuracy_stderr":0.005868835234592509,"f1_stderr":0.0058928047682858435,"ap_stderr":0.002961416663478842,"main_score":0.5031531531531531}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/vesteinn__DanskBERT/Da_Political_Comments.json b/src/seb/cache/vesteinn__DanskBERT/Da_Political_Comments.json new file mode 100644 index 00000000..652130d2 --- /dev/null +++ b/src/seb/cache/vesteinn__DanskBERT/Da_Political_Comments.json @@ -0,0 +1 @@ +{"task_name":"Da Political Comments","task_description":"A dataset of Danish political comments rated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:39:30.806916","scores":{"da":{"accuracy":0.3985571587125416,"f1":0.3630685592020103,"accuracy_stderr":0.035815538427674945,"f1_stderr":0.027671931289537103,"main_score":0.3985571587125416}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/vesteinn__DanskBERT/LCC.json b/src/seb/cache/vesteinn__DanskBERT/LCC.json new file mode 100644 index 00000000..190b81a8 --- /dev/null +++ b/src/seb/cache/vesteinn__DanskBERT/LCC.json @@ -0,0 +1 @@ +{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:38:08.431677","scores":{"da":{"accuracy":0.58,"f1":0.5646156675447656,"accuracy_stderr":0.040879225911349036,"f1_stderr":0.03460749732787966,"main_score":0.58}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/vesteinn__DanskBERT/Language_Identification.json b/src/seb/cache/vesteinn__DanskBERT/Language_Identification.json new file mode 100644 index 00000000..d342d3a7 --- /dev/null +++ b/src/seb/cache/vesteinn__DanskBERT/Language_Identification.json @@ -0,0 +1 @@ +{"task_name":"Language Identification","task_description":"A dataset for Nordic language identification.","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T21:04:59.487773","scores":{"da":{"accuracy":0.7424666666666666,"f1":0.7379219364810606,"accuracy_stderr":0.012513281832605807,"f1_stderr":0.012082505150569824,"main_score":0.7424666666666666},"sv":{"accuracy":0.7424666666666666,"f1":0.7379219364810606,"accuracy_stderr":0.012513281832605807,"f1_stderr":0.012082505150569824,"main_score":0.7424666666666666},"nb":{"accuracy":0.7424666666666666,"f1":0.7379219364810606,"accuracy_stderr":0.012513281832605807,"f1_stderr":0.012082505150569824,"main_score":0.7424666666666666},"nn":{"accuracy":0.7424666666666666,"f1":0.7379219364810606,"accuracy_stderr":0.012513281832605807,"f1_stderr":0.012082505150569824,"main_score":0.7424666666666666},"is":{"accuracy":0.7424666666666666,"f1":0.7379219364810606,"accuracy_stderr":0.012513281832605807,"f1_stderr":0.012082505150569824,"main_score":0.7424666666666666},"fo":{"accuracy":0.7424666666666666,"f1":0.7379219364810606,"accuracy_stderr":0.012513281832605807,"f1_stderr":0.012082505150569824,"main_score":0.7424666666666666}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/vesteinn__DanskBERT/Massive_Intent.json b/src/seb/cache/vesteinn__DanskBERT/Massive_Intent.json new file mode 100644 index 00000000..09b3ebfa --- /dev/null +++ b/src/seb/cache/vesteinn__DanskBERT/Massive_Intent.json @@ -0,0 +1 @@ +{"task_name":"Massive Intent","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T20:59:26.15118","scores":{"da":{"accuracy":0.5468392737054472,"f1":0.5039000994078626,"accuracy_stderr":0.011536817088604236,"f1_stderr":0.006389390451655079,"main_score":0.5468392737054472},"nb":{"accuracy":0.45379959650302626,"f1":0.4271217484445252,"accuracy_stderr":0.012549015365232539,"f1_stderr":0.011229470448672847,"main_score":0.45379959650302626},"sv":{"accuracy":0.40817081371889713,"f1":0.3841954212578219,"accuracy_stderr":0.006134927155520256,"f1_stderr":0.008971093419384164,"main_score":0.40817081371889713}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/vesteinn__DanskBERT/Massive_Scenario.json b/src/seb/cache/vesteinn__DanskBERT/Massive_Scenario.json new file mode 100644 index 00000000..c1a5ef62 --- /dev/null +++ b/src/seb/cache/vesteinn__DanskBERT/Massive_Scenario.json @@ -0,0 +1 @@ +{"task_name":"Massive Scenario","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T21:01:11.378911","scores":{"da":{"accuracy":0.5955615332885004,"f1":0.5794034655925369,"accuracy_stderr":0.01577699539582055,"f1_stderr":0.013384379870283205,"main_score":0.5955615332885004},"nb":{"accuracy":0.47552118359112316,"f1":0.4640926498110063,"accuracy_stderr":0.018872921265933037,"f1_stderr":0.01476824680252287,"main_score":0.47552118359112316},"sv":{"accuracy":0.40141223940820436,"f1":0.3822692785842296,"accuracy_stderr":0.01808423841391776,"f1_stderr":0.013452966848637938,"main_score":0.40141223940820436}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/vesteinn__DanskBERT/NoReC.json b/src/seb/cache/vesteinn__DanskBERT/NoReC.json new file mode 100644 index 00000000..5262df15 --- /dev/null +++ b/src/seb/cache/vesteinn__DanskBERT/NoReC.json @@ -0,0 +1 @@ +{"task_name":"NoReC","task_description":"A Norwegian dataset for sentiment classification on review","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T21:05:34.968397","scores":{"nb":{"accuracy":0.460595703125,"f1":0.43854924377503063,"accuracy_stderr":0.02193578721014052,"f1_stderr":0.016858535482613627,"main_score":0.460595703125}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/vesteinn__DanskBERT/Norwegian_parliament.json b/src/seb/cache/vesteinn__DanskBERT/Norwegian_parliament.json new file mode 100644 index 00000000..60f8af37 --- /dev/null +++ b/src/seb/cache/vesteinn__DanskBERT/Norwegian_parliament.json @@ -0,0 +1 @@ +{"task_name":"Norwegian parliament","task_description":"Norwegian parliament speeches annotated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T21:09:18.297013","scores":{"nb":{"accuracy":0.5679166666666666,"f1":0.5666235749808163,"ap":0.539603032753131,"accuracy_stderr":0.029674741410461815,"f1_stderr":0.0296227793144478,"ap_stderr":0.01869489598457436,"main_score":0.5679166666666666}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/vesteinn__DanskBERT/Scala.json b/src/seb/cache/vesteinn__DanskBERT/Scala.json new file mode 100644 index 00000000..ad757d4b --- /dev/null +++ b/src/seb/cache/vesteinn__DanskBERT/Scala.json @@ -0,0 +1,45 @@ +{ + "task_name": "ScaLA", + "task_description": "A linguistic acceptability task for Danish, Norwegian Bokmål Norwegian Nynorsk and Swedish.", + "task_version": "1.0.3.dev0", + "time_of_run": "2023-07-27T21:03:34.879581", + "scores": { + "da": { + "accuracy": 0.66591796875, + "f1": 0.6616198709006833, + "ap": 0.6179104217471408, + "accuracy_stderr": 0.019314964556917027, + "f1_stderr": 0.017917908374838355, + "ap_stderr": 0.021368618593014095, + "main_score": 0.66591796875 + }, + "nb": { + "accuracy": 0.59990234375, + "f1": 0.5990010371256689, + "ap": 0.5608231549243188, + "accuracy_stderr": 0.02391900414221448, + "f1_stderr": 0.024566392985160047, + "ap_stderr": 0.016255913124253032, + "main_score": 0.59990234375 + }, + "sv": { + "accuracy": 0.537939453125, + "f1": 0.5357569438251415, + "ap": 0.5213109065272618, + "accuracy_stderr": 0.03139997732805539, + "f1_stderr": 0.03126635766907699, + "ap_stderr": 0.018150283855429068, + "main_score": 0.537939453125 + }, + "nn": { + "accuracy": 0.549951171875, + "f1": 0.5491603482683816, + "ap": 0.5280226741033301, + "accuracy_stderr": 0.021599533785980857, + "f1_stderr": 0.02169568820747518, + "ap_stderr": 0.01274915013058807, + "main_score": 0.549951171875 + } + }, + "main_score": "accuracy" +} \ No newline at end of file diff --git a/src/seb/cache/vesteinn__DanskBERT/SweFAQ.json b/src/seb/cache/vesteinn__DanskBERT/SweFAQ.json new file mode 100644 index 00000000..bdb4de54 --- /dev/null +++ b/src/seb/cache/vesteinn__DanskBERT/SweFAQ.json @@ -0,0 +1 @@ +{"task_name":"SweFAQ","task_description":"A Swedish QA dataset derived from FAQ","task_version":"0.0.1","time_of_run":"2023-09-04T19:25:20.531296","scores":{"sv":{"ndcg_at_1":0.06823,"ndcg_at_3":0.08807,"ndcg_at_5":0.10006,"ndcg_at_10":0.11629,"ndcg_at_100":0.18353,"ndcg_at_1000":0.24534,"map_at_1":0.06823,"map_at_3":0.08285,"map_at_5":0.08947,"map_at_10":0.09608,"map_at_100":0.10792,"map_at_1000":0.11029,"recall_at_1":0.06823,"recall_at_3":0.10331,"recall_at_5":0.13255,"recall_at_10":0.18324,"recall_at_100":0.52047,"recall_at_1000":1.0,"precision_at_1":0.06823,"precision_at_3":0.03444,"precision_at_5":0.02651,"precision_at_10":0.01832,"precision_at_100":0.0052,"precision_at_1000":0.001,"mrr_at_1":0.06823,"mrr_at_3":0.08285,"mrr_at_5":0.08947,"mrr_at_10":0.09608,"mrr_at_100":0.10792,"mrr_at_1000":0.11029}},"main_score":"ndcg_at_10"} \ No newline at end of file diff --git a/src/seb/cache/vesteinn__DanskBERT/SweReC.json b/src/seb/cache/vesteinn__DanskBERT/SweReC.json new file mode 100644 index 00000000..8254de5a --- /dev/null +++ b/src/seb/cache/vesteinn__DanskBERT/SweReC.json @@ -0,0 +1 @@ +{"task_name":"SweReC","task_description":"A Swedish dataset for sentiment classification on review","task_version":"1.0.3.dev0","time_of_run":"2023-07-27T21:11:41.782914","scores":{"sv":{"accuracy":0.619482421875,"f1":0.5558447995104121,"accuracy_stderr":0.020710321291315156,"f1_stderr":0.021773450519163632,"main_score":0.619482421875}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/xlm-roberta-base/Angry_Tweets.json b/src/seb/cache/xlm-roberta-base/Angry_Tweets.json new file mode 100644 index 00000000..8b799849 --- /dev/null +++ b/src/seb/cache/xlm-roberta-base/Angry_Tweets.json @@ -0,0 +1 @@ +{"task_name":"Angry Tweets","task_description":"A sentiment dataset with 3 classes (positiv, negativ, neutral) for Danish tweets","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T11:14:58.903598","scores":{"da":{"accuracy":0.5240687679083095,"f1":0.5092211937200768,"accuracy_stderr":0.03304749048310259,"f1_stderr":0.02751105245577153,"main_score":0.5240687679083095}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/xlm-roberta-base/Bornholm_Parallel.json b/src/seb/cache/xlm-roberta-base/Bornholm_Parallel.json new file mode 100644 index 00000000..420676a7 --- /dev/null +++ b/src/seb/cache/xlm-roberta-base/Bornholm_Parallel.json @@ -0,0 +1 @@ +{"task_name":"Bornholm Parallel","task_description":"Danish Bornholmsk Parallel Corpus. Bornholmsk is a Danish dialect spoken on the island of Bornholm, Denmark. Historically it is a part of east Danish which was also spoken in Scania and Halland, Sweden.","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T11:15:23.276666","scores":{"da":{"precision":0.03856726388344035,"recall":0.07,"f1":0.044201958647684864,"accuracy":0.07,"main_score":0.044201958647684864},"da-bornholm":{"precision":0.03856726388344035,"recall":0.07,"f1":0.044201958647684864,"accuracy":0.07,"main_score":0.044201958647684864}},"main_score":"f1"} \ No newline at end of file diff --git a/src/seb/cache/xlm-roberta-base/DKHate.json b/src/seb/cache/xlm-roberta-base/DKHate.json new file mode 100644 index 00000000..9faaeb0c --- /dev/null +++ b/src/seb/cache/xlm-roberta-base/DKHate.json @@ -0,0 +1 @@ +{"task_name":"DKHate","task_description":"Danish Tweets annotated for Hate Speech either being Offensive or not","task_version":"1.0.3.dev0","time_of_run":"2023-07-30T14:02:43.70996","scores":{"da":{"accuracy":0.5677811550151975,"f1":0.47583466207028763,"ap":0.8940789213094907,"accuracy_stderr":0.08064512016344795,"f1_stderr":0.05930921494392278,"ap_stderr":0.017016508149669225,"main_score":0.5677811550151975}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/xlm-roberta-base/DaLAJ.json b/src/seb/cache/xlm-roberta-base/DaLAJ.json new file mode 100644 index 00000000..da1e41e7 --- /dev/null +++ b/src/seb/cache/xlm-roberta-base/DaLAJ.json @@ -0,0 +1 @@ +{"task_name":"DaLAJ","task_description":"A Swedish dataset for linguistic accebtablity. Available as a part of Superlim.","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T12:00:08.637044","scores":{"sv":{"accuracy":0.5115990990990992,"f1":0.5088630658829845,"ap":0.5063588334137684,"accuracy_stderr":0.019686604417079024,"f1_stderr":0.021030307311759724,"ap_stderr":0.010378991032277352,"main_score":0.5115990990990992}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/xlm-roberta-base/Da_Political_Comments.json b/src/seb/cache/xlm-roberta-base/Da_Political_Comments.json new file mode 100644 index 00000000..4ff7ca2b --- /dev/null +++ b/src/seb/cache/xlm-roberta-base/Da_Political_Comments.json @@ -0,0 +1 @@ +{"task_name":"Da Political Comments","task_description":"A dataset of Danish political comments rated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T11:16:38.783441","scores":{"da":{"accuracy":0.33745837957824637,"f1":0.29300187702513114,"accuracy_stderr":0.03334296587071571,"f1_stderr":0.01943691614850412,"main_score":0.33745837957824637}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/xlm-roberta-base/LCC.json b/src/seb/cache/xlm-roberta-base/LCC.json new file mode 100644 index 00000000..7ab8b211 --- /dev/null +++ b/src/seb/cache/xlm-roberta-base/LCC.json @@ -0,0 +1 @@ +{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T11:15:12.248792","scores":{"da":{"accuracy":0.5226666666666667,"f1":0.49497455570746823,"accuracy_stderr":0.03427989368581985,"f1_stderr":0.02679743546046411,"main_score":0.5226666666666667}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/xlm-roberta-base/Language_Identification.json b/src/seb/cache/xlm-roberta-base/Language_Identification.json new file mode 100644 index 00000000..65dffa68 --- /dev/null +++ b/src/seb/cache/xlm-roberta-base/Language_Identification.json @@ -0,0 +1 @@ +{"task_name":"Language Identification","task_description":"A dataset for Nordic language identification.","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T11:30:56.58552","scores":{"da":{"accuracy":0.7939333333333333,"f1":0.794387036082508,"accuracy_stderr":0.007444311773278839,"f1_stderr":0.008042982825948577,"main_score":0.7939333333333333},"sv":{"accuracy":0.7939333333333333,"f1":0.794387036082508,"accuracy_stderr":0.007444311773278839,"f1_stderr":0.008042982825948577,"main_score":0.7939333333333333},"nb":{"accuracy":0.7939333333333333,"f1":0.794387036082508,"accuracy_stderr":0.007444311773278839,"f1_stderr":0.008042982825948577,"main_score":0.7939333333333333},"nn":{"accuracy":0.7939333333333333,"f1":0.794387036082508,"accuracy_stderr":0.007444311773278839,"f1_stderr":0.008042982825948577,"main_score":0.7939333333333333},"is":{"accuracy":0.7939333333333333,"f1":0.794387036082508,"accuracy_stderr":0.007444311773278839,"f1_stderr":0.008042982825948577,"main_score":0.7939333333333333},"fo":{"accuracy":0.7939333333333333,"f1":0.794387036082508,"accuracy_stderr":0.007444311773278839,"f1_stderr":0.008042982825948577,"main_score":0.7939333333333333}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/xlm-roberta-base/Massive_Intent.json b/src/seb/cache/xlm-roberta-base/Massive_Intent.json new file mode 100644 index 00000000..e0887317 --- /dev/null +++ b/src/seb/cache/xlm-roberta-base/Massive_Intent.json @@ -0,0 +1 @@ +{"task_name":"Massive Intent","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T11:20:00.265468","scores":{"da":{"accuracy":0.4106254203093477,"f1":0.3795648689253743,"accuracy_stderr":0.015077840832839742,"f1_stderr":0.012582701200982243,"main_score":0.4106254203093477},"nb":{"accuracy":0.404640215198386,"f1":0.37341048610165817,"accuracy_stderr":0.014068009415622156,"f1_stderr":0.01144099071939448,"main_score":0.404640215198386},"sv":{"accuracy":0.45117686617350367,"f1":0.4202125805260968,"accuracy_stderr":0.014194901450353853,"f1_stderr":0.010769282318607538,"main_score":0.45117686617350367}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/xlm-roberta-base/Massive_Scenario.json b/src/seb/cache/xlm-roberta-base/Massive_Scenario.json new file mode 100644 index 00000000..635af5b9 --- /dev/null +++ b/src/seb/cache/xlm-roberta-base/Massive_Scenario.json @@ -0,0 +1 @@ +{"task_name":"Massive Scenario","task_description":"MASSIVE: A 1M-Example Multilingual Natural Language Understanding Dataset with 51 Typologically-Diverse Languages","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T11:25:25.719702","scores":{"da":{"accuracy":0.43913920645595156,"f1":0.4224444767774179,"accuracy_stderr":0.02403781459559167,"f1_stderr":0.019128246487203907,"main_score":0.43913920645595156},"nb":{"accuracy":0.44828513786146607,"f1":0.4280187540493935,"accuracy_stderr":0.03030513291601151,"f1_stderr":0.02025670836250579,"main_score":0.44828513786146607},"sv":{"accuracy":0.4735036987222595,"f1":0.46097644872017873,"accuracy_stderr":0.03037481874129291,"f1_stderr":0.022149804025228082,"main_score":0.4735036987222595}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/xlm-roberta-base/NoReC.json b/src/seb/cache/xlm-roberta-base/NoReC.json new file mode 100644 index 00000000..8878e472 --- /dev/null +++ b/src/seb/cache/xlm-roberta-base/NoReC.json @@ -0,0 +1 @@ +{"task_name":"NoReC","task_description":"A Norwegian dataset for sentiment classification on review","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T11:48:47.082912","scores":{"nb":{"accuracy":0.46279296875,"f1":0.4447190592125052,"accuracy_stderr":0.0231032893361403,"f1_stderr":0.01961653942909908,"main_score":0.46279296875}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/xlm-roberta-base/Norwegian_parliament.json b/src/seb/cache/xlm-roberta-base/Norwegian_parliament.json new file mode 100644 index 00000000..6e9726d4 --- /dev/null +++ b/src/seb/cache/xlm-roberta-base/Norwegian_parliament.json @@ -0,0 +1 @@ +{"task_name":"Norwegian parliament","task_description":"Norwegian parliament speeches annotated for sentiment","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T11:56:16.675737","scores":{"nb":{"accuracy":0.5674999999999999,"f1":0.5626654592692376,"ap":0.5401314577638446,"accuracy_stderr":0.03321228019205479,"f1_stderr":0.03413593526569537,"ap_stderr":0.02161389508438745,"main_score":0.5674999999999999}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/xlm-roberta-base/ScaLA.json b/src/seb/cache/xlm-roberta-base/ScaLA.json new file mode 100644 index 00000000..47e519f7 --- /dev/null +++ b/src/seb/cache/xlm-roberta-base/ScaLA.json @@ -0,0 +1 @@ +{"task_name":"ScaLA","task_description":"A linguistic acceptability task for Danish, Norwegian Bokmål Norwegian Nynorsk and Swedish.","task_version":"1.1.1.dev0","time_of_run":"2023-09-13T15:51:24.271240","scores":{"da":{"accuracy":0.573095703125,"f1":0.5672374962367553,"ap":0.542176016030716,"accuracy_stderr":0.022787673191201296,"f1_stderr":0.02649911130479038,"ap_stderr":0.014713655172726818,"main_score":0.573095703125},"nb":{"accuracy":0.5833984375,"f1":0.5804317746794289,"ap":0.549269722172648,"accuracy_stderr":0.029797000718714203,"f1_stderr":0.029898785736251494,"ap_stderr":0.019317688439180285,"main_score":0.5833984375},"sv":{"accuracy":0.59990234375,"f1":0.5983969041239451,"ap":0.5607959074664981,"accuracy_stderr":0.025954525426921577,"f1_stderr":0.025959747531727584,"ap_stderr":0.017882408251096812,"main_score":0.59990234375},"nn":{"accuracy":0.551123046875,"f1":0.5479409186834777,"ap":0.5289897893476994,"accuracy_stderr":0.025671755306919123,"f1_stderr":0.02796589485461358,"ap_stderr":0.015404023155278153,"main_score":0.551123046875}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cache/xlm-roberta-base/SweFAQ.json b/src/seb/cache/xlm-roberta-base/SweFAQ.json new file mode 100644 index 00000000..f47f0f4b --- /dev/null +++ b/src/seb/cache/xlm-roberta-base/SweFAQ.json @@ -0,0 +1 @@ +{"task_name":"SweFAQ","task_description":"A Swedish QA dataset derived from FAQ","task_version":"0.0.1","time_of_run":"2023-09-04T19:38:48.821422","scores":{"sv":{"ndcg_at_1":0.06043,"ndcg_at_3":0.07319,"ndcg_at_5":0.0773,"ndcg_at_10":0.08612,"ndcg_at_100":0.13224,"ndcg_at_1000":0.21347,"map_at_1":0.06043,"map_at_3":0.07018,"map_at_5":0.07251,"map_at_10":0.07615,"map_at_100":0.083,"map_at_1000":0.086,"recall_at_1":0.06043,"recall_at_3":0.08187,"recall_at_5":0.09162,"recall_at_10":0.11891,"recall_at_100":0.36452,"recall_at_1000":1.0,"precision_at_1":0.06043,"precision_at_3":0.02729,"precision_at_5":0.01832,"precision_at_10":0.01189,"precision_at_100":0.00365,"precision_at_1000":0.001,"mrr_at_1":0.06043,"mrr_at_3":0.07018,"mrr_at_5":0.07251,"mrr_at_10":0.07615,"mrr_at_100":0.083,"mrr_at_1000":0.086}},"main_score":"ndcg_at_10"} \ No newline at end of file diff --git a/src/seb/cache/xlm-roberta-base/SweReC.json b/src/seb/cache/xlm-roberta-base/SweReC.json new file mode 100644 index 00000000..4e5db742 --- /dev/null +++ b/src/seb/cache/xlm-roberta-base/SweReC.json @@ -0,0 +1 @@ +{"task_name":"SweReC","task_description":"A Swedish dataset for sentiment classification on review","task_version":"1.0.3.dev0","time_of_run":"2023-07-28T11:59:18.486646","scores":{"sv":{"accuracy":0.66201171875,"f1":0.5912368951480952,"accuracy_stderr":0.03303597056167941,"f1_stderr":0.025418121637137903,"main_score":0.66201171875}},"main_score":"accuracy"} \ No newline at end of file diff --git a/src/seb/cli.py b/src/seb/cli.py index 7d605442..134fad98 100644 --- a/src/seb/cli.py +++ b/src/seb/cli.py @@ -46,6 +46,9 @@ def pretty_print(results: seb.BenchmarkResults): def run_benchmark( model_name_or_path: str, languages: Optional[list[str]], + use_cache: bool = True, + raise_errors: bool = True, + cache_dir: Optional[str] = None, ) -> seb.BenchmarkResults: """Runs benchmark on a given model and languages.""" meta = seb.ModelMeta( @@ -56,7 +59,11 @@ def run_benchmark( loader=partial(SentenceTransformer, model_name_or_path=model_name_or_path), # type: ignore ) benchmark = seb.Benchmark(languages) - res = benchmark.evaluate_model(model, raise_errors=False) + + cache_dir_path = Path(cache_dir) if cache_dir else None + res = benchmark.evaluate_model( + model, use_cache, raise_errors, cache_dir=cache_dir_path + ) return res @@ -68,22 +75,47 @@ def main(): help="Name of the model on HuggingFace hub, or path to the model.", ) parser.add_argument( - "languages", + "--languages", + "-l", nargs="*", help="List of language codes to evaluate the model on.", ) parser.add_argument( - "--save_path", + "--output_path", "-o", default="benchmark_results.json", help="File to store benchmark results in.", ) + parser.add_argument( + "--ignore_cache", + action="store_true", + default=False, + help="Ignore cached results.", + ) + parser.add_argument( + "--ignore_errors", + action="store_true", + default=False, + help="Ignore errors on specific tasks during evaluation.", + ) + parser.add_argument( + "--cache_dir", + default=None, + help="Directory to store cached results in.", + ) args = parser.parse_args() logging.info(f"Running benchmark with {args.model_name_or_path}...") if not args.languages: args.languages = None - results = run_benchmark(args.model_name_or_path, args.languages) + + results = run_benchmark( + args.model_name_or_path, + args.languages, + use_cache=not args.ignore_cache, + raise_errors=not args.ignore_errors, + cache_dir=args.cache_dir, + ) logging.info("Saving results...") save_path = Path(args.save_path) with save_path.open("w") as save_file: diff --git a/src/seb/full_benchmark.py b/src/seb/full_benchmark.py index b05f5f5e..205b83f2 100644 --- a/src/seb/full_benchmark.py +++ b/src/seb/full_benchmark.py @@ -3,6 +3,9 @@ """ +from pathlib import Path +from typing import Optional + from seb.model_interface import SebModel from .benchmark import Benchmark @@ -19,7 +22,9 @@ def run_benchmark( use_cache: bool = True, + run_models: bool = True, raise_errors: bool = True, + cache_dir: Optional[Path] = None, ) -> dict[str, list[BenchmarkResults]]: """ Run the full SEB benchmark. @@ -32,7 +37,9 @@ def run_benchmark( bm_results = benchmark.evaluate_models( models=models, use_cache=use_cache, + run_model=run_models, raise_errors=raise_errors, + cache_dir=cache_dir, ) results[subset] = bm_results diff --git a/src/seb/utils.py b/src/seb/utils.py index b9025e4a..70d92c74 100644 --- a/src/seb/utils.py +++ b/src/seb/utils.py @@ -1,7 +1,8 @@ import os from pathlib import Path -CACHE_DIR = Path.home() / ".cache" / "seb" +package_dir = Path(__file__).parent +CACHE_DIR = package_dir / "cache" def get_cache_dir() -> Path: diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index 822de9bc..d2c0eb13 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -37,6 +37,7 @@ def test_run_benchmark( model_names: list[str], languages: Optional[list[str]], tasks: Optional[list[str]], + tmp_path: Path, ): """ Test that the benchmark runs without errors. @@ -51,8 +52,7 @@ def test_run_benchmark( tasks=tasks, ) bm_results: list[seb.BenchmarkResults] = benchmark.evaluate_models( - models=models, - use_cache=False, + models=models, use_cache=False, cache_dir=tmp_path ) assert len(bm_results) == len(models) @@ -91,6 +91,7 @@ def test_cache_dir_is_reused( model_name: str, languages: Optional[list[str]], tasks: Optional[list[str]], + tmp_path: Path, ): """ Check that the cache dir is reused. @@ -102,7 +103,9 @@ def test_cache_dir_is_reused( ) before_run = datetime.now() - bm_result: seb.BenchmarkResults = benchmark.evaluate_model(model, use_cache=False) + bm_result: seb.BenchmarkResults = benchmark.evaluate_model( + model, use_cache=False, cache_dir=tmp_path + ) after_run = datetime.now() assert len(bm_result) == 1 @@ -113,6 +116,7 @@ def test_cache_dir_is_reused( bm_result: seb.BenchmarkResults = benchmark.evaluate_model( model=model, use_cache=True, + cache_dir=tmp_path, ) assert isinstance(bm_result, seb.BenchmarkResults) @@ -124,17 +128,10 @@ def test_cache_dir_is_reused( assert task_result_1 == task_result_2, "The two task results should be equal" -def set_cache_dir(): - new_cache_dir = Path(__file__).parent / "tmp_cache" - os.environ["SEB_CACHE_DIR"] = str(new_cache_dir) - new_cache_dir.mkdir(exist_ok=True) - - -def test_benchmark_skip_on_error_raised(): +def test_benchmark_skip_on_error_raised(tmp_path: Path): """ Test that the benchmark skips a model if an error is raised. """ - set_cache_dir() model = seb.get_model("test_model") benchmark: seb.Benchmark = seb.Benchmark( languages=None, @@ -145,6 +142,7 @@ def test_benchmark_skip_on_error_raised(): model, use_cache=False, raise_errors=False, + cache_dir=tmp_path, ) assert len(bm_result) == 1 @@ -154,4 +152,6 @@ def test_benchmark_skip_on_error_raised(): # test that the benchmark raises an error if raise_errors is True with pytest.raises(ValueError): # noqa: PT011 - benchmark.evaluate_model(model, use_cache=False, raise_errors=True) + benchmark.evaluate_model( + model, use_cache=False, raise_errors=True, cache_dir=tmp_path + )