diff --git a/src/main/python/run_regression.py b/src/main/python/run_regression.py index 0595ba94a2..27007b7479 100644 --- a/src/main/python/run_regression.py +++ b/src/main/python/run_regression.py @@ -204,8 +204,10 @@ def evaluate_and_verify(yaml_data, dry_run): expected = round(model['results'][metric['metric']][i], metric['metric_precision']) actual = round(float(eval_out), metric['metric_precision']) + using_hnsw = True if 'VectorQueryGenerator' in model['params'] or '-encoder' in model['params'] else False + # For HNSW, we only print to third digit - if 'VectorQueryGenerator' in model['params']: + if using_hnsw: result_str = 'expected: {0:.3f} actual: {1:.3f} - metric: {2:<8} model: {3} topics: {4}'.format( expected, actual, metric['metric'], model['name'], topic_set['id']) else: @@ -216,8 +218,8 @@ def evaluate_and_verify(yaml_data, dry_run): # For HNSW, be more tolerant, but as long as the actual score is higher than the expected score, # let the test pass. if is_close(expected, actual) or \ - ('VectorQueryGenerator' in model['params'] and is_close(expected, actual, abs_tol=0.007)) or \ - ('VectorQueryGenerator' in model['params'] and actual > expected): + (using_hnsw and is_close(expected, actual, abs_tol=0.007)) or \ + (using_hnsw and actual > expected): logger.info(ok_str + result_str) # For ONNX runs, increase tolerance a bit because we observe some minor differences across OSes. elif '-encoder' in model['params'] and is_close(expected, actual, abs_tol=0.001): diff --git a/src/main/resources/regression/beir-v1.0.0-arguana-splade-pp-ed-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-arguana-splade-pp-ed-onnx.yaml new file mode 100644 index 0000000000..5dd7a09793 --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-arguana-splade-pp-ed-onnx.yaml @@ -0,0 +1,55 @@ +--- +corpus: beir-v1.0.0-arguana-splade-pp-ed +corpus_path: collections/beir-v1.0.0/splade-pp-ed/arguana + +index_path: indexes/lucene-index.beir-v1.0.0-arguana-splade-pp-ed/ +collection_class: JsonVectorCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 16 +index_options: -impact -pretokenized +index_stats: + documents: 8674 + documents (non-empty): 8674 + total terms: 71992355 + +metrics: + - metric: nDCG@10 + command: target/appassembler/bin/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: target/appassembler/bin/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: target/appassembler/bin/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topics: + - name: "BEIR (v1.0.0): ArguAna" + id: test + path: topics.beir-v1.0.0-arguana.test.tsv.gz + qrel: qrels.beir-v1.0.0-arguana.test.txt + +models: + - name: splade-pp-ed + display: SPLADE++ (CoCondenser-EnsembleDistil) + params: -impact -pretokenized -removeQuery -hits 1000 -encoder SpladePlusPlusEnsembleDistil + results: + nDCG@10: + - 0.5203 + R@100: + - 0.9744 + R@1000: + - 0.9950 diff --git a/src/main/resources/regression/beir-v1.0.0-bioasq-splade-pp-ed-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-bioasq-splade-pp-ed-onnx.yaml new file mode 100644 index 0000000000..913f731e06 --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-bioasq-splade-pp-ed-onnx.yaml @@ -0,0 +1,55 @@ +--- +corpus: beir-v1.0.0-bioasq-splade-pp-ed +corpus_path: collections/beir-v1.0.0/splade-pp-ed/bioasq + +index_path: indexes/lucene-index.beir-v1.0.0-bioasq-splade-pp-ed/ +collection_class: JsonVectorCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 16 +index_options: -impact -pretokenized +index_stats: + documents: 14914603 + documents (non-empty): 14914603 + total terms: 127381306317 + +metrics: + - metric: nDCG@10 + command: target/appassembler/bin/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: target/appassembler/bin/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: target/appassembler/bin/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topics: + - name: "BEIR (v1.0.0): BioASQ" + id: test + path: topics.beir-v1.0.0-bioasq.test.tsv.gz + qrel: qrels.beir-v1.0.0-bioasq.test.txt + +models: + - name: splade-pp-ed + display: SPLADE++ (CoCondenser-EnsembleDistil) + params: -impact -pretokenized -removeQuery -hits 1000 -encoder SpladePlusPlusEnsembleDistil + results: + nDCG@10: + - 0.4980 + R@100: + - 0.7385 + R@1000: + - 0.8757 diff --git a/src/main/resources/regression/beir-v1.0.0-climate-fever-splade-pp-ed-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-climate-fever-splade-pp-ed-onnx.yaml new file mode 100644 index 0000000000..db973ad323 --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-climate-fever-splade-pp-ed-onnx.yaml @@ -0,0 +1,55 @@ +--- +corpus: beir-v1.0.0-climate-fever-splade-pp-ed +corpus_path: collections/beir-v1.0.0/splade-pp-ed/climate-fever + +index_path: indexes/lucene-index.beir-v1.0.0-climate-fever-splade-pp-ed/ +collection_class: JsonVectorCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 16 +index_options: -impact -pretokenized +index_stats: + documents: 5416593 + documents (non-empty): 5416593 + total terms: 28498465299 + +metrics: + - metric: nDCG@10 + command: target/appassembler/bin/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: target/appassembler/bin/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: target/appassembler/bin/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topics: + - name: "BEIR (v1.0.0): Climate-FEVER" + id: test + path: topics.beir-v1.0.0-climate-fever.test.tsv.gz + qrel: qrels.beir-v1.0.0-climate-fever.test.txt + +models: + - name: splade-pp-ed + display: SPLADE++ (CoCondenser-EnsembleDistil) + params: -impact -pretokenized -removeQuery -hits 1000 -encoder SpladePlusPlusEnsembleDistil + results: + nDCG@10: + - 0.2297 + R@100: + - 0.5211 + R@1000: + - 0.7183 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-android-splade-pp-ed-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-android-splade-pp-ed-onnx.yaml new file mode 100644 index 0000000000..41338cbfc0 --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-android-splade-pp-ed-onnx.yaml @@ -0,0 +1,55 @@ +--- +corpus: beir-v1.0.0-cqadupstack-android-splade-pp-ed +corpus_path: collections/beir-v1.0.0/splade-pp-ed/cqadupstack-android + +index_path: indexes/lucene-index.beir-v1.0.0-cqadupstack-android-splade-pp-ed/ +collection_class: JsonVectorCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 16 +index_options: -impact -pretokenized +index_stats: + documents: 22998 + documents (non-empty): 22998 + total terms: 108476959 + +metrics: + - metric: nDCG@10 + command: target/appassembler/bin/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: target/appassembler/bin/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: target/appassembler/bin/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topics: + - name: "BEIR (v1.0.0): CQADupStack-android" + id: test + path: topics.beir-v1.0.0-cqadupstack-android.test.tsv.gz + qrel: qrels.beir-v1.0.0-cqadupstack-android.test.txt + +models: + - name: splade-pp-ed + display: SPLADE++ (CoCondenser-EnsembleDistil) + params: -impact -pretokenized -removeQuery -hits 1000 -encoder SpladePlusPlusEnsembleDistil + results: + nDCG@10: + - 0.3904 + R@100: + - 0.7404 + R@1000: + - 0.9064 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-english-splade-pp-ed-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-english-splade-pp-ed-onnx.yaml new file mode 100644 index 0000000000..da9991fcc8 --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-english-splade-pp-ed-onnx.yaml @@ -0,0 +1,55 @@ +--- +corpus: beir-v1.0.0-cqadupstack-english-splade-pp-ed +corpus_path: collections/beir-v1.0.0/splade-pp-ed/cqadupstack-english + +index_path: indexes/lucene-index.beir-v1.0.0-cqadupstack-english-splade-pp-ed/ +collection_class: JsonVectorCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 16 +index_options: -impact -pretokenized +index_stats: + documents: 40221 + documents (non-empty): 40221 + total terms: 158861979 + +metrics: + - metric: nDCG@10 + command: target/appassembler/bin/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: target/appassembler/bin/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: target/appassembler/bin/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topics: + - name: "BEIR (v1.0.0): CQADupStack-english" + id: test + path: topics.beir-v1.0.0-cqadupstack-english.test.tsv.gz + qrel: qrels.beir-v1.0.0-cqadupstack-english.test.txt + +models: + - name: splade-pp-ed + display: SPLADE++ (CoCondenser-EnsembleDistil) + params: -impact -pretokenized -removeQuery -hits 1000 -encoder SpladePlusPlusEnsembleDistil + results: + nDCG@10: + - 0.4079 + R@100: + - 0.6946 + R@1000: + - 0.8454 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-gaming-splade-pp-ed-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-gaming-splade-pp-ed-onnx.yaml new file mode 100644 index 0000000000..8403e4c4b6 --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-gaming-splade-pp-ed-onnx.yaml @@ -0,0 +1,55 @@ +--- +corpus: beir-v1.0.0-cqadupstack-gaming-splade-pp-ed +corpus_path: collections/beir-v1.0.0/splade-pp-ed/cqadupstack-gaming + +index_path: indexes/lucene-index.beir-v1.0.0-cqadupstack-gaming-splade-pp-ed/ +collection_class: JsonVectorCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 16 +index_options: -impact -pretokenized +index_stats: + documents: 45301 + documents (non-empty): 45301 + total terms: 197713644 + +metrics: + - metric: nDCG@10 + command: target/appassembler/bin/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: target/appassembler/bin/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: target/appassembler/bin/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topics: + - name: "BEIR (v1.0.0): CQADupStack-gaming" + id: test + path: topics.beir-v1.0.0-cqadupstack-gaming.test.tsv.gz + qrel: qrels.beir-v1.0.0-cqadupstack-gaming.test.txt + +models: + - name: splade-pp-ed + display: SPLADE++ (CoCondenser-EnsembleDistil) + params: -impact -pretokenized -removeQuery -hits 1000 -encoder SpladePlusPlusEnsembleDistil + results: + nDCG@10: + - 0.4957 + R@100: + - 0.8131 + R@1000: + - 0.9221 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-gis-splade-pp-ed-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-gis-splade-pp-ed-onnx.yaml new file mode 100644 index 0000000000..2302f514e7 --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-gis-splade-pp-ed-onnx.yaml @@ -0,0 +1,55 @@ +--- +corpus: beir-v1.0.0-cqadupstack-gis-splade-pp-ed +corpus_path: collections/beir-v1.0.0/splade-pp-ed/cqadupstack-gis + +index_path: indexes/lucene-index.beir-v1.0.0-cqadupstack-gis-splade-pp-ed/ +collection_class: JsonVectorCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 16 +index_options: -impact -pretokenized +index_stats: + documents: 37637 + documents (non-empty): 37637 + total terms: 214744014 + +metrics: + - metric: nDCG@10 + command: target/appassembler/bin/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: target/appassembler/bin/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: target/appassembler/bin/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topics: + - name: "BEIR (v1.0.0): CQADupStack-gis" + id: test + path: topics.beir-v1.0.0-cqadupstack-gis.test.tsv.gz + qrel: qrels.beir-v1.0.0-cqadupstack-gis.test.txt + +models: + - name: splade-pp-ed + display: SPLADE++ (CoCondenser-EnsembleDistil) + params: -impact -pretokenized -removeQuery -hits 1000 -encoder SpladePlusPlusEnsembleDistil + results: + nDCG@10: + - 0.3150 + R@100: + - 0.6320 + R@1000: + - 0.8325 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-mathematica-splade-pp-ed-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-mathematica-splade-pp-ed-onnx.yaml new file mode 100644 index 0000000000..1282a543da --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-mathematica-splade-pp-ed-onnx.yaml @@ -0,0 +1,55 @@ +--- +corpus: beir-v1.0.0-cqadupstack-mathematica-splade-pp-ed +corpus_path: collections/beir-v1.0.0/splade-pp-ed/cqadupstack-mathematica + +index_path: indexes/lucene-index.beir-v1.0.0-cqadupstack-mathematica-splade-pp-ed/ +collection_class: JsonVectorCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 16 +index_options: -impact -pretokenized +index_stats: + documents: 16705 + documents (non-empty): 16705 + total terms: 90452420 + +metrics: + - metric: nDCG@10 + command: target/appassembler/bin/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: target/appassembler/bin/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: target/appassembler/bin/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topics: + - name: "BEIR (v1.0.0): CQADupStack-mathematica" + id: test + path: topics.beir-v1.0.0-cqadupstack-mathematica.test.tsv.gz + qrel: qrels.beir-v1.0.0-cqadupstack-mathematica.test.txt + +models: + - name: splade-pp-ed + display: SPLADE++ (CoCondenser-EnsembleDistil) + params: -impact -pretokenized -removeQuery -hits 1000 -encoder SpladePlusPlusEnsembleDistil + results: + nDCG@10: + - 0.2377 + R@100: + - 0.5797 + R@1000: + - 0.8007 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-physics-splade-pp-ed-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-physics-splade-pp-ed-onnx.yaml new file mode 100644 index 0000000000..e10b8ea672 --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-physics-splade-pp-ed-onnx.yaml @@ -0,0 +1,55 @@ +--- +corpus: beir-v1.0.0-cqadupstack-physics-splade-pp-ed +corpus_path: collections/beir-v1.0.0/splade-pp-ed/cqadupstack-physics + +index_path: indexes/lucene-index.beir-v1.0.0-cqadupstack-physics-splade-pp-ed/ +collection_class: JsonVectorCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 16 +index_options: -impact -pretokenized +index_stats: + documents: 38316 + documents (non-empty): 38316 + total terms: 199892911 + +metrics: + - metric: nDCG@10 + command: target/appassembler/bin/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: target/appassembler/bin/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: target/appassembler/bin/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topics: + - name: "BEIR (v1.0.0): CQADupStack-physics" + id: test + path: topics.beir-v1.0.0-cqadupstack-physics.test.tsv.gz + qrel: qrels.beir-v1.0.0-cqadupstack-physics.test.txt + +models: + - name: splade-pp-ed + display: SPLADE++ (CoCondenser-EnsembleDistil) + params: -impact -pretokenized -removeQuery -hits 1000 -encoder SpladePlusPlusEnsembleDistil + results: + nDCG@10: + - 0.3599 + R@100: + - 0.7196 + R@1000: + - 0.9010 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-programmers-splade-pp-ed-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-programmers-splade-pp-ed-onnx.yaml new file mode 100644 index 0000000000..d39c76aaeb --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-programmers-splade-pp-ed-onnx.yaml @@ -0,0 +1,55 @@ +--- +corpus: beir-v1.0.0-cqadupstack-programmers-splade-pp-ed +corpus_path: collections/beir-v1.0.0/splade-pp-ed/cqadupstack-programmers + +index_path: indexes/lucene-index.beir-v1.0.0-cqadupstack-programmers-splade-pp-ed/ +collection_class: JsonVectorCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 16 +index_options: -impact -pretokenized +index_stats: + documents: 32176 + documents (non-empty): 32176 + total terms: 182133939 + +metrics: + - metric: nDCG@10 + command: target/appassembler/bin/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: target/appassembler/bin/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: target/appassembler/bin/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topics: + - name: "BEIR (v1.0.0): CQADupStack-programmers" + id: test + path: topics.beir-v1.0.0-cqadupstack-programmers.test.tsv.gz + qrel: qrels.beir-v1.0.0-cqadupstack-programmers.test.txt + +models: + - name: splade-pp-ed + display: SPLADE++ (CoCondenser-EnsembleDistil) + params: -impact -pretokenized -removeQuery -hits 1000 -encoder SpladePlusPlusEnsembleDistil + results: + nDCG@10: + - 0.3401 + R@100: + - 0.6585 + R@1000: + - 0.8603 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-stats-splade-pp-ed-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-stats-splade-pp-ed-onnx.yaml new file mode 100644 index 0000000000..947b5e3b96 --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-stats-splade-pp-ed-onnx.yaml @@ -0,0 +1,55 @@ +--- +corpus: beir-v1.0.0-cqadupstack-stats-splade-pp-ed +corpus_path: collections/beir-v1.0.0/splade-pp-ed/cqadupstack-stats + +index_path: indexes/lucene-index.beir-v1.0.0-cqadupstack-stats-splade-pp-ed/ +collection_class: JsonVectorCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 16 +index_options: -impact -pretokenized +index_stats: + documents: 42269 + documents (non-empty): 42269 + total terms: 236361350 + +metrics: + - metric: nDCG@10 + command: target/appassembler/bin/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: target/appassembler/bin/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: target/appassembler/bin/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topics: + - name: "BEIR (v1.0.0): CQADupStack-stats" + id: test + path: topics.beir-v1.0.0-cqadupstack-stats.test.tsv.gz + qrel: qrels.beir-v1.0.0-cqadupstack-stats.test.txt + +models: + - name: splade-pp-ed + display: SPLADE++ (CoCondenser-EnsembleDistil) + params: -impact -pretokenized -removeQuery -hits 1000 -encoder SpladePlusPlusEnsembleDistil + results: + nDCG@10: + - 0.2990 + R@100: + - 0.5894 + R@1000: + - 0.7776 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-tex-splade-pp-ed-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-tex-splade-pp-ed-onnx.yaml new file mode 100644 index 0000000000..0f8202ab0b --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-tex-splade-pp-ed-onnx.yaml @@ -0,0 +1,55 @@ +--- +corpus: beir-v1.0.0-cqadupstack-tex-splade-pp-ed +corpus_path: collections/beir-v1.0.0/splade-pp-ed/cqadupstack-tex + +index_path: indexes/lucene-index.beir-v1.0.0-cqadupstack-tex-splade-pp-ed/ +collection_class: JsonVectorCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 16 +index_options: -impact -pretokenized +index_stats: + documents: 68184 + documents (non-empty): 68184 + total terms: 433864313 + +metrics: + - metric: nDCG@10 + command: target/appassembler/bin/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: target/appassembler/bin/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: target/appassembler/bin/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topics: + - name: "BEIR (v1.0.0): CQADupStack-tex" + id: test + path: topics.beir-v1.0.0-cqadupstack-tex.test.tsv.gz + qrel: qrels.beir-v1.0.0-cqadupstack-tex.test.txt + +models: + - name: splade-pp-ed + display: SPLADE++ (CoCondenser-EnsembleDistil) + params: -impact -pretokenized -removeQuery -hits 1000 -encoder SpladePlusPlusEnsembleDistil + results: + nDCG@10: + - 0.2530 + R@100: + - 0.5161 + R@1000: + - 0.7341 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-unix-splade-pp-ed-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-unix-splade-pp-ed-onnx.yaml new file mode 100644 index 0000000000..91309ebd7c --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-unix-splade-pp-ed-onnx.yaml @@ -0,0 +1,55 @@ +--- +corpus: beir-v1.0.0-cqadupstack-unix-splade-pp-ed +corpus_path: collections/beir-v1.0.0/splade-pp-ed/cqadupstack-unix + +index_path: indexes/lucene-index.beir-v1.0.0-cqadupstack-unix-splade-pp-ed/ +collection_class: JsonVectorCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 16 +index_options: -impact -pretokenized +index_stats: + documents: 47382 + documents (non-empty): 47382 + total terms: 260688145 + +metrics: + - metric: nDCG@10 + command: target/appassembler/bin/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: target/appassembler/bin/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: target/appassembler/bin/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topics: + - name: "BEIR (v1.0.0): CQADupStack-unix" + id: test + path: topics.beir-v1.0.0-cqadupstack-unix.test.tsv.gz + qrel: qrels.beir-v1.0.0-cqadupstack-unix.test.txt + +models: + - name: splade-pp-ed + display: SPLADE++ (CoCondenser-EnsembleDistil) + params: -impact -pretokenized -removeQuery -hits 1000 -encoder SpladePlusPlusEnsembleDistil + results: + nDCG@10: + - 0.3167 + R@100: + - 0.6214 + R@1000: + - 0.8257 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-webmasters-splade-pp-ed-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-webmasters-splade-pp-ed-onnx.yaml new file mode 100644 index 0000000000..6d3178ade7 --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-webmasters-splade-pp-ed-onnx.yaml @@ -0,0 +1,55 @@ +--- +corpus: beir-v1.0.0-cqadupstack-webmasters-splade-pp-ed +corpus_path: collections/beir-v1.0.0/splade-pp-ed/cqadupstack-webmasters + +index_path: indexes/lucene-index.beir-v1.0.0-cqadupstack-webmasters-splade-pp-ed/ +collection_class: JsonVectorCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 16 +index_options: -impact -pretokenized +index_stats: + documents: 17405 + documents (non-empty): 17405 + total terms: 89755810 + +metrics: + - metric: nDCG@10 + command: target/appassembler/bin/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: target/appassembler/bin/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: target/appassembler/bin/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topics: + - name: "BEIR (v1.0.0): CQADupStack-webmasters" + id: test + path: topics.beir-v1.0.0-cqadupstack-webmasters.test.tsv.gz + qrel: qrels.beir-v1.0.0-cqadupstack-webmasters.test.txt + +models: + - name: splade-pp-ed + display: SPLADE++ (CoCondenser-EnsembleDistil) + params: -impact -pretokenized -removeQuery -hits 1000 -encoder SpladePlusPlusEnsembleDistil + results: + nDCG@10: + - 0.3167 + R@100: + - 0.6360 + R@1000: + - 0.8710 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-wordpress-splade-pp-ed-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-wordpress-splade-pp-ed-onnx.yaml new file mode 100644 index 0000000000..2700cf00c6 --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-wordpress-splade-pp-ed-onnx.yaml @@ -0,0 +1,55 @@ +--- +corpus: beir-v1.0.0-cqadupstack-wordpress-splade-pp-ed +corpus_path: collections/beir-v1.0.0/splade-pp-ed/cqadupstack-wordpress + +index_path: indexes/lucene-index.beir-v1.0.0-cqadupstack-wordpress-splade-pp-ed/ +collection_class: JsonVectorCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 16 +index_options: -impact -pretokenized +index_stats: + documents: 48605 + documents (non-empty): 48605 + total terms: 257594340 + +metrics: + - metric: nDCG@10 + command: target/appassembler/bin/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: target/appassembler/bin/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: target/appassembler/bin/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topics: + - name: "BEIR (v1.0.0): CQADupStack-wordpress" + id: test + path: topics.beir-v1.0.0-cqadupstack-wordpress.test.tsv.gz + qrel: qrels.beir-v1.0.0-cqadupstack-wordpress.test.txt + +models: + - name: splade-pp-ed + display: SPLADE++ (CoCondenser-EnsembleDistil) + params: -impact -pretokenized -removeQuery -hits 1000 -encoder SpladePlusPlusEnsembleDistil + results: + nDCG@10: + - 0.2733 + R@100: + - 0.5945 + R@1000: + - 0.7924 diff --git a/src/main/resources/regression/beir-v1.0.0-dbpedia-entity-splade-pp-ed-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-dbpedia-entity-splade-pp-ed-onnx.yaml new file mode 100644 index 0000000000..1f6ec7115c --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-dbpedia-entity-splade-pp-ed-onnx.yaml @@ -0,0 +1,55 @@ +--- +corpus: beir-v1.0.0-dbpedia-entity-splade-pp-ed +corpus_path: collections/beir-v1.0.0/splade-pp-ed/dbpedia-entity + +index_path: indexes/lucene-index.beir-v1.0.0-dbpedia-entity-splade-pp-ed/ +collection_class: JsonVectorCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 16 +index_options: -impact -pretokenized +index_stats: + documents: 4635922 + documents (non-empty): 4635922 + total terms: 22302972729 + +metrics: + - metric: nDCG@10 + command: target/appassembler/bin/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: target/appassembler/bin/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: target/appassembler/bin/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topics: + - name: "BEIR (v1.0.0): DBPedia" + id: test + path: topics.beir-v1.0.0-dbpedia-entity.test.tsv.gz + qrel: qrels.beir-v1.0.0-dbpedia-entity.test.txt + +models: + - name: splade-pp-ed + display: SPLADE++ (CoCondenser-EnsembleDistil) + params: -impact -pretokenized -removeQuery -hits 1000 -encoder SpladePlusPlusEnsembleDistil + results: + nDCG@10: + - 0.4366 + R@100: + - 0.5624 + R@1000: + - 0.7838 diff --git a/src/main/resources/regression/beir-v1.0.0-fever-splade-pp-ed-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-fever-splade-pp-ed-onnx.yaml new file mode 100644 index 0000000000..0c6f5f8cce --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-fever-splade-pp-ed-onnx.yaml @@ -0,0 +1,55 @@ +--- +corpus: beir-v1.0.0-fever-splade-pp-ed +corpus_path: collections/beir-v1.0.0/splade-pp-ed/fever + +index_path: indexes/lucene-index.beir-v1.0.0-fever-splade-pp-ed/ +collection_class: JsonVectorCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 16 +index_options: -impact -pretokenized +index_stats: + documents: 5416593 + documents (non-empty): 5416593 + total terms: 28498465299 + +metrics: + - metric: nDCG@10 + command: target/appassembler/bin/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: target/appassembler/bin/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: target/appassembler/bin/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topics: + - name: "BEIR (v1.0.0): FEVER" + id: test + path: topics.beir-v1.0.0-fever.test.tsv.gz + qrel: qrels.beir-v1.0.0-fever.test.txt + +models: + - name: splade-pp-ed + display: SPLADE++ (CoCondenser-EnsembleDistil) + params: -impact -pretokenized -removeQuery -hits 1000 -encoder SpladePlusPlusEnsembleDistil + results: + nDCG@10: + - 0.7882 + R@100: + - 0.9459 + R@1000: + - 0.9660 diff --git a/src/main/resources/regression/beir-v1.0.0-fiqa-splade-pp-ed-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-fiqa-splade-pp-ed-onnx.yaml new file mode 100644 index 0000000000..f51052113a --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-fiqa-splade-pp-ed-onnx.yaml @@ -0,0 +1,55 @@ +--- +corpus: beir-v1.0.0-fiqa-splade-pp-ed +corpus_path: collections/beir-v1.0.0/splade-pp-ed/fiqa + +index_path: indexes/lucene-index.beir-v1.0.0-fiqa-splade-pp-ed/ +collection_class: JsonVectorCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 16 +index_options: -impact -pretokenized +index_stats: + documents: 57638 + documents (non-empty): 57638 + total terms: 342348959 + +metrics: + - metric: nDCG@10 + command: target/appassembler/bin/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: target/appassembler/bin/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: target/appassembler/bin/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topics: + - name: "BEIR (v1.0.0): FiQA-2018" + id: test + path: topics.beir-v1.0.0-fiqa.test.tsv.gz + qrel: qrels.beir-v1.0.0-fiqa.test.txt + +models: + - name: splade-pp-ed + display: SPLADE++ (CoCondenser-EnsembleDistil) + params: -impact -pretokenized -removeQuery -hits 1000 -encoder SpladePlusPlusEnsembleDistil + results: + nDCG@10: + - 0.3475 + R@100: + - 0.6314 + R@1000: + - 0.8392 diff --git a/src/main/resources/regression/beir-v1.0.0-hotpotqa-splade-pp-ed-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-hotpotqa-splade-pp-ed-onnx.yaml new file mode 100644 index 0000000000..9edb22d487 --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-hotpotqa-splade-pp-ed-onnx.yaml @@ -0,0 +1,55 @@ +--- +corpus: beir-v1.0.0-hotpotqa-splade-pp-ed +corpus_path: collections/beir-v1.0.0/splade-pp-ed/hotpotqa + +index_path: indexes/lucene-index.beir-v1.0.0-hotpotqa-splade-pp-ed/ +collection_class: JsonVectorCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 16 +index_options: -impact -pretokenized +index_stats: + documents: 5233329 + documents (non-empty): 5233329 + total terms: 23736328387 + +metrics: + - metric: nDCG@10 + command: target/appassembler/bin/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: target/appassembler/bin/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: target/appassembler/bin/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topics: + - name: "BEIR (v1.0.0): HotpotQA" + id: test + path: topics.beir-v1.0.0-hotpotqa.test.tsv.gz + qrel: qrels.beir-v1.0.0-hotpotqa.test.txt + +models: + - name: splade-pp-ed + display: SPLADE++ (CoCondenser-EnsembleDistil) + params: -impact -pretokenized -removeQuery -hits 1000 -encoder SpladePlusPlusEnsembleDistil + results: + nDCG@10: + - 0.6868 + R@100: + - 0.8177 + R@1000: + - 0.8952 diff --git a/src/main/resources/regression/beir-v1.0.0-nfcorpus-splade-pp-ed-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-nfcorpus-splade-pp-ed-onnx.yaml new file mode 100644 index 0000000000..2098eddae6 --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-nfcorpus-splade-pp-ed-onnx.yaml @@ -0,0 +1,55 @@ +--- +corpus: beir-v1.0.0-nfcorpus-splade-pp-ed +corpus_path: collections/beir-v1.0.0/splade-pp-ed/nfcorpus + +index_path: indexes/lucene-index.beir-v1.0.0-nfcorpus-splade-pp-ed/ +collection_class: JsonVectorCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 16 +index_options: -impact -pretokenized +index_stats: + documents: 3633 + documents (non-empty): 3633 + total terms: 30711150 + +metrics: + - metric: nDCG@10 + command: target/appassembler/bin/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: target/appassembler/bin/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: target/appassembler/bin/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topics: + - name: "BEIR (v1.0.0): NFCorpus" + id: test + path: topics.beir-v1.0.0-nfcorpus.test.tsv.gz + qrel: qrels.beir-v1.0.0-nfcorpus.test.txt + +models: + - name: splade-pp-ed + display: SPLADE++ (CoCondenser-EnsembleDistil) + params: -impact -pretokenized -removeQuery -hits 1000 -encoder SpladePlusPlusEnsembleDistil + results: + nDCG@10: + - 0.3470 + R@100: + - 0.2844 + R@1000: + - 0.5925 diff --git a/src/main/resources/regression/beir-v1.0.0-nq-splade-pp-ed-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-nq-splade-pp-ed-onnx.yaml new file mode 100644 index 0000000000..557e4060d0 --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-nq-splade-pp-ed-onnx.yaml @@ -0,0 +1,55 @@ +--- +corpus: beir-v1.0.0-nq-splade-pp-ed +corpus_path: collections/beir-v1.0.0/splade-pp-ed/nq + +index_path: indexes/lucene-index.beir-v1.0.0-nq-splade-pp-ed/ +collection_class: JsonVectorCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 16 +index_options: -impact -pretokenized +index_stats: + documents: 2681468 + documents (non-empty): 2681468 + total terms: 15061905296 + +metrics: + - metric: nDCG@10 + command: target/appassembler/bin/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: target/appassembler/bin/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: target/appassembler/bin/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topics: + - name: "BEIR (v1.0.0): NQ" + id: test + path: topics.beir-v1.0.0-nq.test.tsv.gz + qrel: qrels.beir-v1.0.0-nq.test.txt + +models: + - name: splade-pp-ed + display: SPLADE++ (CoCondenser-EnsembleDistil) + params: -impact -pretokenized -removeQuery -hits 1000 -encoder SpladePlusPlusEnsembleDistil + results: + nDCG@10: + - 0.5378 + R@100: + - 0.9296 + R@1000: + - 0.9839 diff --git a/src/main/resources/regression/beir-v1.0.0-quora-splade-pp-ed-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-quora-splade-pp-ed-onnx.yaml new file mode 100644 index 0000000000..7ce7082891 --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-quora-splade-pp-ed-onnx.yaml @@ -0,0 +1,55 @@ +--- +corpus: beir-v1.0.0-quora-splade-pp-ed +corpus_path: collections/beir-v1.0.0/splade-pp-ed/quora + +index_path: indexes/lucene-index.beir-v1.0.0-quora-splade-pp-ed/ +collection_class: JsonVectorCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 16 +index_options: -impact -pretokenized +index_stats: + documents: 522931 + documents (non-empty): 522931 + total terms: 1064938611 + +metrics: + - metric: nDCG@10 + command: target/appassembler/bin/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: target/appassembler/bin/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: target/appassembler/bin/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topics: + - name: "BEIR (v1.0.0): Quora" + id: test + path: topics.beir-v1.0.0-quora.test.tsv.gz + qrel: qrels.beir-v1.0.0-quora.test.txt + +models: + - name: splade-pp-ed + display: SPLADE++ (CoCondenser-EnsembleDistil) + params: -impact -pretokenized -removeQuery -hits 1000 -encoder SpladePlusPlusEnsembleDistil + results: + nDCG@10: + - 0.8343 + R@100: + - 0.9863 + R@1000: + - 0.9989 diff --git a/src/main/resources/regression/beir-v1.0.0-robust04-splade-pp-ed-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-robust04-splade-pp-ed-onnx.yaml new file mode 100644 index 0000000000..8b5e3f6e6e --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-robust04-splade-pp-ed-onnx.yaml @@ -0,0 +1,55 @@ +--- +corpus: beir-v1.0.0-robust04-splade-pp-ed +corpus_path: collections/beir-v1.0.0/splade-pp-ed/robust04 + +index_path: indexes/lucene-index.beir-v1.0.0-robust04-splade-pp-ed/ +collection_class: JsonVectorCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 16 +index_options: -impact -pretokenized +index_stats: + documents: 528155 + documents (non-empty): 528155 + total terms: 4818025575 + +metrics: + - metric: nDCG@10 + command: target/appassembler/bin/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: target/appassembler/bin/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: target/appassembler/bin/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topics: + - name: "BEIR (v1.0.0): Robust04" + id: test + path: topics.beir-v1.0.0-robust04.test.tsv.gz + qrel: qrels.beir-v1.0.0-robust04.test.txt + +models: + - name: splade-pp-ed + display: SPLADE++ (CoCondenser-EnsembleDistil) + params: -impact -pretokenized -removeQuery -hits 1000 -encoder SpladePlusPlusEnsembleDistil + results: + nDCG@10: + - 0.4679 + R@100: + - 0.3850 + R@1000: + - 0.6228 diff --git a/src/main/resources/regression/beir-v1.0.0-scidocs-splade-pp-ed-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-scidocs-splade-pp-ed-onnx.yaml new file mode 100644 index 0000000000..1ed220574d --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-scidocs-splade-pp-ed-onnx.yaml @@ -0,0 +1,55 @@ +--- +corpus: beir-v1.0.0-scidocs-splade-pp-ed +corpus_path: collections/beir-v1.0.0/splade-pp-ed/scidocs + +index_path: indexes/lucene-index.beir-v1.0.0-scidocs-splade-pp-ed/ +collection_class: JsonVectorCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 16 +index_options: -impact -pretokenized +index_stats: + documents: 25657 + documents (non-empty): 25657 + total terms: 192911913 + +metrics: + - metric: nDCG@10 + command: target/appassembler/bin/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: target/appassembler/bin/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: target/appassembler/bin/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topics: + - name: "BEIR (v1.0.0): SCIDOCS" + id: test + path: topics.beir-v1.0.0-scidocs.test.tsv.gz + qrel: qrels.beir-v1.0.0-scidocs.test.txt + +models: + - name: splade-pp-ed + display: SPLADE++ (CoCondenser-EnsembleDistil) + params: -impact -pretokenized -removeQuery -hits 1000 -encoder SpladePlusPlusEnsembleDistil + results: + nDCG@10: + - 0.1591 + R@100: + - 0.3730 + R@1000: + - 0.6016 diff --git a/src/main/resources/regression/beir-v1.0.0-scifact-splade-pp-ed-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-scifact-splade-pp-ed-onnx.yaml new file mode 100644 index 0000000000..734ee472bf --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-scifact-splade-pp-ed-onnx.yaml @@ -0,0 +1,55 @@ +--- +corpus: beir-v1.0.0-scifact-splade-pp-ed +corpus_path: collections/beir-v1.0.0/splade-pp-ed/scifact + +index_path: indexes/lucene-index.beir-v1.0.0-scifact-splade-pp-ed/ +collection_class: JsonVectorCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 16 +index_options: -impact -pretokenized +index_stats: + documents: 5183 + documents (non-empty): 5183 + total terms: 47317323 + +metrics: + - metric: nDCG@10 + command: target/appassembler/bin/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: target/appassembler/bin/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: target/appassembler/bin/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topics: + - name: "BEIR (v1.0.0): SciFact" + id: test + path: topics.beir-v1.0.0-scifact.test.tsv.gz + qrel: qrels.beir-v1.0.0-scifact.test.txt + +models: + - name: splade-pp-ed + display: SPLADE++ (CoCondenser-EnsembleDistil) + params: -impact -pretokenized -removeQuery -hits 1000 -encoder SpladePlusPlusEnsembleDistil + results: + nDCG@10: + - 0.7041 + R@100: + - 0.9353 + R@1000: + - 0.9867 diff --git a/src/main/resources/regression/beir-v1.0.0-signal1m-splade-pp-ed-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-signal1m-splade-pp-ed-onnx.yaml new file mode 100644 index 0000000000..2e574314b7 --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-signal1m-splade-pp-ed-onnx.yaml @@ -0,0 +1,55 @@ +--- +corpus: beir-v1.0.0-signal1m-splade-pp-ed +corpus_path: collections/beir-v1.0.0/splade-pp-ed/signal1m + +index_path: indexes/lucene-index.beir-v1.0.0-signal1m-splade-pp-ed/ +collection_class: JsonVectorCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 16 +index_options: -impact -pretokenized +index_stats: + documents: 2866316 + documents (non-empty): 2866316 + total terms: 8237410263 + +metrics: + - metric: nDCG@10 + command: target/appassembler/bin/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: target/appassembler/bin/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: target/appassembler/bin/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topics: + - name: "BEIR (v1.0.0): Signal-1M" + id: test + path: topics.beir-v1.0.0-signal1m.test.tsv.gz + qrel: qrels.beir-v1.0.0-signal1m.test.txt + +models: + - name: splade-pp-ed + display: SPLADE++ (CoCondenser-EnsembleDistil) + params: -impact -pretokenized -removeQuery -hits 1000 -encoder SpladePlusPlusEnsembleDistil + results: + nDCG@10: + - 0.3008 + R@100: + - 0.3398 + R@1000: + - 0.5492 diff --git a/src/main/resources/regression/beir-v1.0.0-trec-covid-splade-pp-ed-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-trec-covid-splade-pp-ed-onnx.yaml new file mode 100644 index 0000000000..78ea3a29bc --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-trec-covid-splade-pp-ed-onnx.yaml @@ -0,0 +1,55 @@ +--- +corpus: beir-v1.0.0-trec-covid-splade-pp-ed +corpus_path: collections/beir-v1.0.0/splade-pp-ed/trec-covid + +index_path: indexes/lucene-index.beir-v1.0.0-trec-covid-splade-pp-ed/ +collection_class: JsonVectorCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 16 +index_options: -impact -pretokenized +index_stats: + documents: 171332 + documents (non-empty): 171332 + total terms: 1206882333 + +metrics: + - metric: nDCG@10 + command: target/appassembler/bin/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: target/appassembler/bin/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: target/appassembler/bin/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topics: + - name: "BEIR (v1.0.0): TREC-COVID" + id: test + path: topics.beir-v1.0.0-trec-covid.test.tsv.gz + qrel: qrels.beir-v1.0.0-trec-covid.test.txt + +models: + - name: splade-pp-ed + display: SPLADE++ (CoCondenser-EnsembleDistil) + params: -impact -pretokenized -removeQuery -hits 1000 -encoder SpladePlusPlusEnsembleDistil + results: + nDCG@10: + - 0.7274 + R@100: + - 0.1282 + R@1000: + - 0.4441 diff --git a/src/main/resources/regression/beir-v1.0.0-trec-news-splade-pp-ed-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-trec-news-splade-pp-ed-onnx.yaml new file mode 100644 index 0000000000..4b9a2b42a0 --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-trec-news-splade-pp-ed-onnx.yaml @@ -0,0 +1,55 @@ +--- +corpus: beir-v1.0.0-trec-news-splade-pp-ed +corpus_path: collections/beir-v1.0.0/splade-pp-ed/trec-news + +index_path: indexes/lucene-index.beir-v1.0.0-trec-news-splade-pp-ed/ +collection_class: JsonVectorCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 16 +index_options: -impact -pretokenized +index_stats: + documents: 594977 + documents (non-empty): 594977 + total terms: 5190619991 + +metrics: + - metric: nDCG@10 + command: target/appassembler/bin/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: target/appassembler/bin/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: target/appassembler/bin/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topics: + - name: "BEIR (v1.0.0): TREC-NEWS" + id: test + path: topics.beir-v1.0.0-trec-news.test.tsv.gz + qrel: qrels.beir-v1.0.0-trec-news.test.txt + +models: + - name: splade-pp-ed + display: SPLADE++ (CoCondenser-EnsembleDistil) + params: -impact -pretokenized -removeQuery -hits 1000 -encoder SpladePlusPlusEnsembleDistil + results: + nDCG@10: + - 0.4152 + R@100: + - 0.4414 + R@1000: + - 0.7060 diff --git a/src/main/resources/regression/beir-v1.0.0-webis-touche2020-splade-pp-ed-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-webis-touche2020-splade-pp-ed-onnx.yaml new file mode 100644 index 0000000000..9ba018617d --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-webis-touche2020-splade-pp-ed-onnx.yaml @@ -0,0 +1,55 @@ +--- +corpus: beir-v1.0.0-webis-touche2020-splade-pp-ed +corpus_path: collections/beir-v1.0.0/splade-pp-ed/webis-touche2020 + +index_path: indexes/lucene-index.beir-v1.0.0-webis-touche2020-splade-pp-ed/ +collection_class: JsonVectorCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 16 +index_options: -impact -pretokenized +index_stats: + documents: 382545 + documents (non-empty): 382545 + total terms: 2275005818 + +metrics: + - metric: nDCG@10 + command: target/appassembler/bin/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: target/appassembler/bin/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: target/appassembler/bin/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topics: + - name: "BEIR (v1.0.0): Webis-Touche2020" + id: test + path: topics.beir-v1.0.0-webis-touche2020.test.tsv.gz + qrel: qrels.beir-v1.0.0-webis-touche2020.test.txt + +models: + - name: splade-pp-ed + display: SPLADE++ (CoCondenser-EnsembleDistil) + params: -impact -pretokenized -removeQuery -hits 1000 -encoder SpladePlusPlusEnsembleDistil + results: + nDCG@10: + - 0.2468 + R@100: + - 0.4715 + R@1000: + - 0.8191