From d457c8815d64f8df51b2bebac46bc14aae5153a8 Mon Sep 17 00:00:00 2001 From: Yuqing Xie Date: Sat, 21 May 2022 15:04:52 -0400 Subject: [PATCH] Yuqingxie/add beir word piece (#1880) * add beir word piece tests and documents Co-authored-by: Jimmy Lin --- docs/regressions-beir-v1.0.0-arguana-wp.md | 69 +++++++++++++++++++ ...egressions-beir-v1.0.0-climate-fever-wp.md | 69 +++++++++++++++++++ ...ions-beir-v1.0.0-cqadupstack-android-wp.md | 69 +++++++++++++++++++ ...ions-beir-v1.0.0-cqadupstack-english-wp.md | 69 +++++++++++++++++++ ...sions-beir-v1.0.0-cqadupstack-gaming-wp.md | 69 +++++++++++++++++++ ...ressions-beir-v1.0.0-cqadupstack-gis-wp.md | 69 +++++++++++++++++++ ...-beir-v1.0.0-cqadupstack-mathematica-wp.md | 69 +++++++++++++++++++ ...ions-beir-v1.0.0-cqadupstack-physics-wp.md | 69 +++++++++++++++++++ ...-beir-v1.0.0-cqadupstack-programmers-wp.md | 69 +++++++++++++++++++ ...ssions-beir-v1.0.0-cqadupstack-stats-wp.md | 69 +++++++++++++++++++ ...ressions-beir-v1.0.0-cqadupstack-tex-wp.md | 69 +++++++++++++++++++ ...essions-beir-v1.0.0-cqadupstack-unix-wp.md | 69 +++++++++++++++++++ ...s-beir-v1.0.0-cqadupstack-webmasters-wp.md | 69 +++++++++++++++++++ ...ns-beir-v1.0.0-cqadupstack-wordpress-wp.md | 69 +++++++++++++++++++ ...gressions-beir-v1.0.0-dbpedia-entity-wp.md | 69 +++++++++++++++++++ docs/regressions-beir-v1.0.0-fever-wp.md | 69 +++++++++++++++++++ docs/regressions-beir-v1.0.0-fiqa-wp.md | 69 +++++++++++++++++++ docs/regressions-beir-v1.0.0-hotpotqa-wp.md | 69 +++++++++++++++++++ docs/regressions-beir-v1.0.0-nfcorpus-wp.md | 69 +++++++++++++++++++ docs/regressions-beir-v1.0.0-nq-wp.md | 69 +++++++++++++++++++ docs/regressions-beir-v1.0.0-quora-wp.md | 69 +++++++++++++++++++ docs/regressions-beir-v1.0.0-scidocs-wp.md | 69 +++++++++++++++++++ docs/regressions-beir-v1.0.0-scifact-wp.md | 69 +++++++++++++++++++ docs/regressions-beir-v1.0.0-trec-covid-wp.md | 69 +++++++++++++++++++ ...essions-beir-v1.0.0-webis-touche2020-wp.md | 69 +++++++++++++++++++ .../templates/beir-v1.0.0-arguana-wp.template | 44 ++++++++++++ .../beir-v1.0.0-climate-fever-wp.template | 44 ++++++++++++ ...eir-v1.0.0-cqadupstack-android-wp.template | 44 ++++++++++++ ...eir-v1.0.0-cqadupstack-english-wp.template | 44 ++++++++++++ ...beir-v1.0.0-cqadupstack-gaming-wp.template | 44 ++++++++++++ .../beir-v1.0.0-cqadupstack-gis-wp.template | 44 ++++++++++++ ...v1.0.0-cqadupstack-mathematica-wp.template | 44 ++++++++++++ ...eir-v1.0.0-cqadupstack-physics-wp.template | 44 ++++++++++++ ...v1.0.0-cqadupstack-programmers-wp.template | 44 ++++++++++++ .../beir-v1.0.0-cqadupstack-stats-wp.template | 44 ++++++++++++ .../beir-v1.0.0-cqadupstack-tex-wp.template | 44 ++++++++++++ .../beir-v1.0.0-cqadupstack-unix-wp.template | 44 ++++++++++++ ...-v1.0.0-cqadupstack-webmasters-wp.template | 44 ++++++++++++ ...r-v1.0.0-cqadupstack-wordpress-wp.template | 44 ++++++++++++ .../beir-v1.0.0-dbpedia-entity-wp.template | 44 ++++++++++++ .../templates/beir-v1.0.0-fever-wp.template | 44 ++++++++++++ .../templates/beir-v1.0.0-fiqa-wp.template | 44 ++++++++++++ .../beir-v1.0.0-hotpotqa-wp.template | 44 ++++++++++++ .../beir-v1.0.0-nfcorpus-wp.template | 44 ++++++++++++ .../templates/beir-v1.0.0-nq-wp.template | 44 ++++++++++++ .../templates/beir-v1.0.0-quora-wp.template | 44 ++++++++++++ .../templates/beir-v1.0.0-scidocs-wp.template | 44 ++++++++++++ .../templates/beir-v1.0.0-scifact-wp.template | 44 ++++++++++++ .../beir-v1.0.0-trec-covid-wp.template | 44 ++++++++++++ .../beir-v1.0.0-webis-touche2020-wp.template | 44 ++++++++++++ .../regression/beir-v1.0.0-arguana-wp.yaml | 57 +++++++++++++++ .../beir-v1.0.0-climate-fever-wp.yaml | 57 +++++++++++++++ .../beir-v1.0.0-cqadupstack-android-wp.yaml | 57 +++++++++++++++ .../beir-v1.0.0-cqadupstack-english-wp.yaml | 57 +++++++++++++++ .../beir-v1.0.0-cqadupstack-gaming-wp.yaml | 57 +++++++++++++++ .../beir-v1.0.0-cqadupstack-gis-wp.yaml | 57 +++++++++++++++ ...eir-v1.0.0-cqadupstack-mathematica-wp.yaml | 57 +++++++++++++++ .../beir-v1.0.0-cqadupstack-physics-wp.yaml | 57 +++++++++++++++ ...eir-v1.0.0-cqadupstack-programmers-wp.yaml | 57 +++++++++++++++ .../beir-v1.0.0-cqadupstack-stats-wp.yaml | 57 +++++++++++++++ .../beir-v1.0.0-cqadupstack-tex-wp.yaml | 57 +++++++++++++++ .../beir-v1.0.0-cqadupstack-unix-wp.yaml | 57 +++++++++++++++ ...beir-v1.0.0-cqadupstack-webmasters-wp.yaml | 57 +++++++++++++++ .../beir-v1.0.0-cqadupstack-wordpress-wp.yaml | 57 +++++++++++++++ .../beir-v1.0.0-dbpedia-entity-wp.yaml | 57 +++++++++++++++ .../regression/beir-v1.0.0-fever-wp.yaml | 57 +++++++++++++++ .../regression/beir-v1.0.0-fiqa-wp.yaml | 57 +++++++++++++++ .../regression/beir-v1.0.0-hotpotqa-wp.yaml | 57 +++++++++++++++ .../regression/beir-v1.0.0-nfcorpus-wp.yaml | 57 +++++++++++++++ .../regression/beir-v1.0.0-nq-wp.yaml | 57 +++++++++++++++ .../regression/beir-v1.0.0-quora-wp.yaml | 57 +++++++++++++++ .../regression/beir-v1.0.0-scidocs-wp.yaml | 57 +++++++++++++++ .../regression/beir-v1.0.0-scifact-wp.yaml | 57 +++++++++++++++ .../regression/beir-v1.0.0-trec-covid-wp.yaml | 57 +++++++++++++++ .../beir-v1.0.0-webis-touche2020-wp.yaml | 57 +++++++++++++++ 75 files changed, 4250 insertions(+) create mode 100644 docs/regressions-beir-v1.0.0-arguana-wp.md create mode 100644 docs/regressions-beir-v1.0.0-climate-fever-wp.md create mode 100644 docs/regressions-beir-v1.0.0-cqadupstack-android-wp.md create mode 100644 docs/regressions-beir-v1.0.0-cqadupstack-english-wp.md create mode 100644 docs/regressions-beir-v1.0.0-cqadupstack-gaming-wp.md create mode 100644 docs/regressions-beir-v1.0.0-cqadupstack-gis-wp.md create mode 100644 docs/regressions-beir-v1.0.0-cqadupstack-mathematica-wp.md create mode 100644 docs/regressions-beir-v1.0.0-cqadupstack-physics-wp.md create mode 100644 docs/regressions-beir-v1.0.0-cqadupstack-programmers-wp.md create mode 100644 docs/regressions-beir-v1.0.0-cqadupstack-stats-wp.md create mode 100644 docs/regressions-beir-v1.0.0-cqadupstack-tex-wp.md create mode 100644 docs/regressions-beir-v1.0.0-cqadupstack-unix-wp.md create mode 100644 docs/regressions-beir-v1.0.0-cqadupstack-webmasters-wp.md create mode 100644 docs/regressions-beir-v1.0.0-cqadupstack-wordpress-wp.md create mode 100644 docs/regressions-beir-v1.0.0-dbpedia-entity-wp.md create mode 100644 docs/regressions-beir-v1.0.0-fever-wp.md create mode 100644 docs/regressions-beir-v1.0.0-fiqa-wp.md create mode 100644 docs/regressions-beir-v1.0.0-hotpotqa-wp.md create mode 100644 docs/regressions-beir-v1.0.0-nfcorpus-wp.md create mode 100644 docs/regressions-beir-v1.0.0-nq-wp.md create mode 100644 docs/regressions-beir-v1.0.0-quora-wp.md create mode 100644 docs/regressions-beir-v1.0.0-scidocs-wp.md create mode 100644 docs/regressions-beir-v1.0.0-scifact-wp.md create mode 100644 docs/regressions-beir-v1.0.0-trec-covid-wp.md create mode 100644 docs/regressions-beir-v1.0.0-webis-touche2020-wp.md create mode 100644 src/main/resources/docgen/templates/beir-v1.0.0-arguana-wp.template create mode 100644 src/main/resources/docgen/templates/beir-v1.0.0-climate-fever-wp.template create mode 100644 src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-android-wp.template create mode 100644 src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-english-wp.template create mode 100644 src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-gaming-wp.template create mode 100644 src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-gis-wp.template create mode 100644 src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-mathematica-wp.template create mode 100644 src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-physics-wp.template create mode 100644 src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-programmers-wp.template create mode 100644 src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-stats-wp.template create mode 100644 src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-tex-wp.template create mode 100644 src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-unix-wp.template create mode 100644 src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-webmasters-wp.template create mode 100644 src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-wordpress-wp.template create mode 100644 src/main/resources/docgen/templates/beir-v1.0.0-dbpedia-entity-wp.template create mode 100644 src/main/resources/docgen/templates/beir-v1.0.0-fever-wp.template create mode 100644 src/main/resources/docgen/templates/beir-v1.0.0-fiqa-wp.template create mode 100644 src/main/resources/docgen/templates/beir-v1.0.0-hotpotqa-wp.template create mode 100644 src/main/resources/docgen/templates/beir-v1.0.0-nfcorpus-wp.template create mode 100644 src/main/resources/docgen/templates/beir-v1.0.0-nq-wp.template create mode 100644 src/main/resources/docgen/templates/beir-v1.0.0-quora-wp.template create mode 100644 src/main/resources/docgen/templates/beir-v1.0.0-scidocs-wp.template create mode 100644 src/main/resources/docgen/templates/beir-v1.0.0-scifact-wp.template create mode 100644 src/main/resources/docgen/templates/beir-v1.0.0-trec-covid-wp.template create mode 100644 src/main/resources/docgen/templates/beir-v1.0.0-webis-touche2020-wp.template create mode 100644 src/main/resources/regression/beir-v1.0.0-arguana-wp.yaml create mode 100644 src/main/resources/regression/beir-v1.0.0-climate-fever-wp.yaml create mode 100644 src/main/resources/regression/beir-v1.0.0-cqadupstack-android-wp.yaml create mode 100644 src/main/resources/regression/beir-v1.0.0-cqadupstack-english-wp.yaml create mode 100644 src/main/resources/regression/beir-v1.0.0-cqadupstack-gaming-wp.yaml create mode 100644 src/main/resources/regression/beir-v1.0.0-cqadupstack-gis-wp.yaml create mode 100644 src/main/resources/regression/beir-v1.0.0-cqadupstack-mathematica-wp.yaml create mode 100644 src/main/resources/regression/beir-v1.0.0-cqadupstack-physics-wp.yaml create mode 100644 src/main/resources/regression/beir-v1.0.0-cqadupstack-programmers-wp.yaml create mode 100644 src/main/resources/regression/beir-v1.0.0-cqadupstack-stats-wp.yaml create mode 100644 src/main/resources/regression/beir-v1.0.0-cqadupstack-tex-wp.yaml create mode 100644 src/main/resources/regression/beir-v1.0.0-cqadupstack-unix-wp.yaml create mode 100644 src/main/resources/regression/beir-v1.0.0-cqadupstack-webmasters-wp.yaml create mode 100644 src/main/resources/regression/beir-v1.0.0-cqadupstack-wordpress-wp.yaml create mode 100644 src/main/resources/regression/beir-v1.0.0-dbpedia-entity-wp.yaml create mode 100644 src/main/resources/regression/beir-v1.0.0-fever-wp.yaml create mode 100644 src/main/resources/regression/beir-v1.0.0-fiqa-wp.yaml create mode 100644 src/main/resources/regression/beir-v1.0.0-hotpotqa-wp.yaml create mode 100644 src/main/resources/regression/beir-v1.0.0-nfcorpus-wp.yaml create mode 100644 src/main/resources/regression/beir-v1.0.0-nq-wp.yaml create mode 100644 src/main/resources/regression/beir-v1.0.0-quora-wp.yaml create mode 100644 src/main/resources/regression/beir-v1.0.0-scidocs-wp.yaml create mode 100644 src/main/resources/regression/beir-v1.0.0-scifact-wp.yaml create mode 100644 src/main/resources/regression/beir-v1.0.0-trec-covid-wp.yaml create mode 100644 src/main/resources/regression/beir-v1.0.0-webis-touche2020-wp.yaml diff --git a/docs/regressions-beir-v1.0.0-arguana-wp.md b/docs/regressions-beir-v1.0.0-arguana-wp.md new file mode 100644 index 0000000000..deacb2e5e2 --- /dev/null +++ b/docs/regressions-beir-v1.0.0-arguana-wp.md @@ -0,0 +1,69 @@ +# Anserini Regressions: BEIR (v1.0.0) — ArguAna + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — ArguAna](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/beir-v1.0.0-arguana-wp.yaml). +Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/beir-v1.0.0-arguana-wp.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-arguana-wp +``` + +## Indexing + +Typical indexing command: + +``` +target/appassembler/bin/IndexCollection \ + -collection BeirFlatCollection \ + -input /path/to/beir-v1.0.0-arguana-wp \ + -index indexes/lucene-index.beir-v1.0.0-arguana-wp/ \ + -generator DefaultLuceneDocumentGenerator \ + -threads 1 -storePositions -storeDocvectors -storeRaw -pretokenized \ + >& logs/log.beir-v1.0.0-arguana-wp & +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.beir-v1.0.0-arguana-wp/ \ + -topics src/main/resources/topics-and-qrels/topics.beir-v1.0.0-arguana.test.wp.tsv.gz \ + -topicreader TsvString \ + -output runs/run.beir-v1.0.0-arguana-wp.bm25.topics.beir-v1.0.0-arguana.test.wp.txt \ + -bm25 -removeQuery -pretokenized & +``` + +Evaluation can be performed using `trec_eval`: + +``` +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-arguana.test.txt runs/run.beir-v1.0.0-arguana-wp.bm25.topics.beir-v1.0.0-arguana.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-arguana.test.txt runs/run.beir-v1.0.0-arguana-wp.bm25.topics.beir-v1.0.0-arguana.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-arguana.test.txt runs/run.beir-v1.0.0-arguana-wp.bm25.topics.beir-v1.0.0-arguana.test.wp.txt +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +| nDCG@10 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): ArguAna | 0.3639 | + + +| R@100 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): ArguAna | 0.8791 | + + +| R@1000 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): ArguAna | 0.9602 | diff --git a/docs/regressions-beir-v1.0.0-climate-fever-wp.md b/docs/regressions-beir-v1.0.0-climate-fever-wp.md new file mode 100644 index 0000000000..764db8cc1a --- /dev/null +++ b/docs/regressions-beir-v1.0.0-climate-fever-wp.md @@ -0,0 +1,69 @@ +# Anserini Regressions: BEIR (v1.0.0) — Climate-FEVER + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — Climate-FEVER](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/beir-v1.0.0-climate-fever-wp.yaml). +Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/beir-v1.0.0-climate-fever-wp.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-climate-fever-wp +``` + +## Indexing + +Typical indexing command: + +``` +target/appassembler/bin/IndexCollection \ + -collection BeirFlatCollection \ + -input /path/to/beir-v1.0.0-climate-fever-wp \ + -index indexes/lucene-index.beir-v1.0.0-climate-fever-wp/ \ + -generator DefaultLuceneDocumentGenerator \ + -threads 1 -storePositions -storeDocvectors -storeRaw -pretokenized \ + >& logs/log.beir-v1.0.0-climate-fever-wp & +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.beir-v1.0.0-climate-fever-wp/ \ + -topics src/main/resources/topics-and-qrels/topics.beir-v1.0.0-climate-fever.test.wp.tsv.gz \ + -topicreader TsvString \ + -output runs/run.beir-v1.0.0-climate-fever-wp.bm25.topics.beir-v1.0.0-climate-fever.test.wp.txt \ + -bm25 -removeQuery -pretokenized & +``` + +Evaluation can be performed using `trec_eval`: + +``` +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-climate-fever.test.txt runs/run.beir-v1.0.0-climate-fever-wp.bm25.topics.beir-v1.0.0-climate-fever.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-climate-fever.test.txt runs/run.beir-v1.0.0-climate-fever-wp.bm25.topics.beir-v1.0.0-climate-fever.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-climate-fever.test.txt runs/run.beir-v1.0.0-climate-fever-wp.bm25.topics.beir-v1.0.0-climate-fever.test.wp.txt +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +| nDCG@10 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): Climate-FEVER | 0.1576 | + + +| R@100 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): Climate-FEVER | 0.4077 | + + +| R@1000 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): Climate-FEVER | 0.5984 | diff --git a/docs/regressions-beir-v1.0.0-cqadupstack-android-wp.md b/docs/regressions-beir-v1.0.0-cqadupstack-android-wp.md new file mode 100644 index 0000000000..0d2f2d63e4 --- /dev/null +++ b/docs/regressions-beir-v1.0.0-cqadupstack-android-wp.md @@ -0,0 +1,69 @@ +# Anserini Regressions: BEIR (v1.0.0) — CQADupStack-android + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — CQADupStack-android](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/beir-v1.0.0-cqadupstack-android-wp.yaml). +Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-android-wp.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-android-wp +``` + +## Indexing + +Typical indexing command: + +``` +target/appassembler/bin/IndexCollection \ + -collection BeirFlatCollection \ + -input /path/to/beir-v1.0.0-cqadupstack-android-wp \ + -index indexes/lucene-index.beir-v1.0.0-cqadupstack-android-wp/ \ + -generator DefaultLuceneDocumentGenerator \ + -threads 1 -storePositions -storeDocvectors -storeRaw -pretokenized \ + >& logs/log.beir-v1.0.0-cqadupstack-android-wp & +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.beir-v1.0.0-cqadupstack-android-wp/ \ + -topics src/main/resources/topics-and-qrels/topics.beir-v1.0.0-cqadupstack-android.test.wp.tsv.gz \ + -topicreader TsvString \ + -output runs/run.beir-v1.0.0-cqadupstack-android-wp.bm25.topics.beir-v1.0.0-cqadupstack-android.test.wp.txt \ + -bm25 -removeQuery -pretokenized & +``` + +Evaluation can be performed using `trec_eval`: + +``` +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-android.test.txt runs/run.beir-v1.0.0-cqadupstack-android-wp.bm25.topics.beir-v1.0.0-cqadupstack-android.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-android.test.txt runs/run.beir-v1.0.0-cqadupstack-android-wp.bm25.topics.beir-v1.0.0-cqadupstack-android.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-android.test.txt runs/run.beir-v1.0.0-cqadupstack-android-wp.bm25.topics.beir-v1.0.0-cqadupstack-android.test.wp.txt +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +| nDCG@10 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-android | 0.3694 | + + +| R@100 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-android | 0.6394 | + + +| R@1000 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-android | 0.8447 | diff --git a/docs/regressions-beir-v1.0.0-cqadupstack-english-wp.md b/docs/regressions-beir-v1.0.0-cqadupstack-english-wp.md new file mode 100644 index 0000000000..7890f3d884 --- /dev/null +++ b/docs/regressions-beir-v1.0.0-cqadupstack-english-wp.md @@ -0,0 +1,69 @@ +# Anserini Regressions: BEIR (v1.0.0) — CQADupStack-english + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — CQADupStack-english](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/beir-v1.0.0-cqadupstack-english-wp.yaml). +Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-english-wp.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-english-wp +``` + +## Indexing + +Typical indexing command: + +``` +target/appassembler/bin/IndexCollection \ + -collection BeirFlatCollection \ + -input /path/to/beir-v1.0.0-cqadupstack-english-wp \ + -index indexes/lucene-index.beir-v1.0.0-cqadupstack-english-wp/ \ + -generator DefaultLuceneDocumentGenerator \ + -threads 1 -storePositions -storeDocvectors -storeRaw -pretokenized \ + >& logs/log.beir-v1.0.0-cqadupstack-english-wp & +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.beir-v1.0.0-cqadupstack-english-wp/ \ + -topics src/main/resources/topics-and-qrels/topics.beir-v1.0.0-cqadupstack-english.test.wp.tsv.gz \ + -topicreader TsvString \ + -output runs/run.beir-v1.0.0-cqadupstack-english-wp.bm25.topics.beir-v1.0.0-cqadupstack-english.test.wp.txt \ + -bm25 -removeQuery -pretokenized & +``` + +Evaluation can be performed using `trec_eval`: + +``` +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-english.test.txt runs/run.beir-v1.0.0-cqadupstack-english-wp.bm25.topics.beir-v1.0.0-cqadupstack-english.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-english.test.txt runs/run.beir-v1.0.0-cqadupstack-english-wp.bm25.topics.beir-v1.0.0-cqadupstack-english.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-english.test.txt runs/run.beir-v1.0.0-cqadupstack-english-wp.bm25.topics.beir-v1.0.0-cqadupstack-english.test.wp.txt +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +| nDCG@10 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-english | 0.3457 | + + +| R@100 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-english | 0.5544 | + + +| R@1000 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-english | 0.7243 | diff --git a/docs/regressions-beir-v1.0.0-cqadupstack-gaming-wp.md b/docs/regressions-beir-v1.0.0-cqadupstack-gaming-wp.md new file mode 100644 index 0000000000..a505b44157 --- /dev/null +++ b/docs/regressions-beir-v1.0.0-cqadupstack-gaming-wp.md @@ -0,0 +1,69 @@ +# Anserini Regressions: BEIR (v1.0.0) — CQADupStack-gaming + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — CQADupStack-gaming](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/beir-v1.0.0-cqadupstack-gaming-wp.yaml). +Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-gaming-wp.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-gaming-wp +``` + +## Indexing + +Typical indexing command: + +``` +target/appassembler/bin/IndexCollection \ + -collection BeirFlatCollection \ + -input /path/to/beir-v1.0.0-cqadupstack-gaming-wp \ + -index indexes/lucene-index.beir-v1.0.0-cqadupstack-gaming-wp/ \ + -generator DefaultLuceneDocumentGenerator \ + -threads 1 -storePositions -storeDocvectors -storeRaw -pretokenized \ + >& logs/log.beir-v1.0.0-cqadupstack-gaming-wp & +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.beir-v1.0.0-cqadupstack-gaming-wp/ \ + -topics src/main/resources/topics-and-qrels/topics.beir-v1.0.0-cqadupstack-gaming.test.wp.tsv.gz \ + -topicreader TsvString \ + -output runs/run.beir-v1.0.0-cqadupstack-gaming-wp.bm25.topics.beir-v1.0.0-cqadupstack-gaming.test.wp.txt \ + -bm25 -removeQuery -pretokenized & +``` + +Evaluation can be performed using `trec_eval`: + +``` +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-gaming.test.txt runs/run.beir-v1.0.0-cqadupstack-gaming-wp.bm25.topics.beir-v1.0.0-cqadupstack-gaming.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-gaming.test.txt runs/run.beir-v1.0.0-cqadupstack-gaming-wp.bm25.topics.beir-v1.0.0-cqadupstack-gaming.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-gaming.test.txt runs/run.beir-v1.0.0-cqadupstack-gaming-wp.bm25.topics.beir-v1.0.0-cqadupstack-gaming.test.wp.txt +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +| nDCG@10 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-gaming | 0.4701 | + + +| R@100 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-gaming | 0.7438 | + + +| R@1000 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-gaming | 0.8810 | diff --git a/docs/regressions-beir-v1.0.0-cqadupstack-gis-wp.md b/docs/regressions-beir-v1.0.0-cqadupstack-gis-wp.md new file mode 100644 index 0000000000..13c1fdba8a --- /dev/null +++ b/docs/regressions-beir-v1.0.0-cqadupstack-gis-wp.md @@ -0,0 +1,69 @@ +# Anserini Regressions: BEIR (v1.0.0) — CQADupStack-gis + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — CQADupStack-gis](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/beir-v1.0.0-cqadupstack-gis-wp.yaml). +Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-gis-wp.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-gis-wp +``` + +## Indexing + +Typical indexing command: + +``` +target/appassembler/bin/IndexCollection \ + -collection BeirFlatCollection \ + -input /path/to/beir-v1.0.0-cqadupstack-gis-wp \ + -index indexes/lucene-index.beir-v1.0.0-cqadupstack-gis-wp/ \ + -generator DefaultLuceneDocumentGenerator \ + -threads 1 -storePositions -storeDocvectors -storeRaw -pretokenized \ + >& logs/log.beir-v1.0.0-cqadupstack-gis-wp & +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.beir-v1.0.0-cqadupstack-gis-wp/ \ + -topics src/main/resources/topics-and-qrels/topics.beir-v1.0.0-cqadupstack-gis.test.wp.tsv.gz \ + -topicreader TsvString \ + -output runs/run.beir-v1.0.0-cqadupstack-gis-wp.bm25.topics.beir-v1.0.0-cqadupstack-gis.test.wp.txt \ + -bm25 -removeQuery -pretokenized & +``` + +Evaluation can be performed using `trec_eval`: + +``` +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-gis.test.txt runs/run.beir-v1.0.0-cqadupstack-gis-wp.bm25.topics.beir-v1.0.0-cqadupstack-gis.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-gis.test.txt runs/run.beir-v1.0.0-cqadupstack-gis-wp.bm25.topics.beir-v1.0.0-cqadupstack-gis.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-gis.test.txt runs/run.beir-v1.0.0-cqadupstack-gis-wp.bm25.topics.beir-v1.0.0-cqadupstack-gis.test.wp.txt +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +| nDCG@10 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-gis | 0.2803 | + + +| R@100 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-gis | 0.5721 | + + +| R@1000 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-gis | 0.7901 | diff --git a/docs/regressions-beir-v1.0.0-cqadupstack-mathematica-wp.md b/docs/regressions-beir-v1.0.0-cqadupstack-mathematica-wp.md new file mode 100644 index 0000000000..4ce235b373 --- /dev/null +++ b/docs/regressions-beir-v1.0.0-cqadupstack-mathematica-wp.md @@ -0,0 +1,69 @@ +# Anserini Regressions: BEIR (v1.0.0) — CQADupStack-mathematica + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — CQADupStack-mathematica](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/beir-v1.0.0-cqadupstack-mathematica-wp.yaml). +Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-mathematica-wp.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-mathematica-wp +``` + +## Indexing + +Typical indexing command: + +``` +target/appassembler/bin/IndexCollection \ + -collection BeirFlatCollection \ + -input /path/to/beir-v1.0.0-cqadupstack-mathematica-wp \ + -index indexes/lucene-index.beir-v1.0.0-cqadupstack-mathematica-wp/ \ + -generator DefaultLuceneDocumentGenerator \ + -threads 1 -storePositions -storeDocvectors -storeRaw -pretokenized \ + >& logs/log.beir-v1.0.0-cqadupstack-mathematica-wp & +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.beir-v1.0.0-cqadupstack-mathematica-wp/ \ + -topics src/main/resources/topics-and-qrels/topics.beir-v1.0.0-cqadupstack-mathematica.test.wp.tsv.gz \ + -topicreader TsvString \ + -output runs/run.beir-v1.0.0-cqadupstack-mathematica-wp.bm25.topics.beir-v1.0.0-cqadupstack-mathematica.test.wp.txt \ + -bm25 -removeQuery -pretokenized & +``` + +Evaluation can be performed using `trec_eval`: + +``` +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-mathematica.test.txt runs/run.beir-v1.0.0-cqadupstack-mathematica-wp.bm25.topics.beir-v1.0.0-cqadupstack-mathematica.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-mathematica.test.txt runs/run.beir-v1.0.0-cqadupstack-mathematica-wp.bm25.topics.beir-v1.0.0-cqadupstack-mathematica.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-mathematica.test.txt runs/run.beir-v1.0.0-cqadupstack-mathematica-wp.bm25.topics.beir-v1.0.0-cqadupstack-mathematica.test.wp.txt +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +| nDCG@10 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-mathematica | 0.1989 | + + +| R@100 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-mathematica | 0.4979 | + + +| R@1000 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-mathematica | 0.7160 | diff --git a/docs/regressions-beir-v1.0.0-cqadupstack-physics-wp.md b/docs/regressions-beir-v1.0.0-cqadupstack-physics-wp.md new file mode 100644 index 0000000000..4f4e5b3927 --- /dev/null +++ b/docs/regressions-beir-v1.0.0-cqadupstack-physics-wp.md @@ -0,0 +1,69 @@ +# Anserini Regressions: BEIR (v1.0.0) — CQADupStack-physics + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — CQADupStack-physics](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/beir-v1.0.0-cqadupstack-physics-wp.yaml). +Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-physics-wp.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-physics-wp +``` + +## Indexing + +Typical indexing command: + +``` +target/appassembler/bin/IndexCollection \ + -collection BeirFlatCollection \ + -input /path/to/beir-v1.0.0-cqadupstack-physics-wp \ + -index indexes/lucene-index.beir-v1.0.0-cqadupstack-physics-wp/ \ + -generator DefaultLuceneDocumentGenerator \ + -threads 1 -storePositions -storeDocvectors -storeRaw -pretokenized \ + >& logs/log.beir-v1.0.0-cqadupstack-physics-wp & +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.beir-v1.0.0-cqadupstack-physics-wp/ \ + -topics src/main/resources/topics-and-qrels/topics.beir-v1.0.0-cqadupstack-physics.test.wp.tsv.gz \ + -topicreader TsvString \ + -output runs/run.beir-v1.0.0-cqadupstack-physics-wp.bm25.topics.beir-v1.0.0-cqadupstack-physics.test.wp.txt \ + -bm25 -removeQuery -pretokenized & +``` + +Evaluation can be performed using `trec_eval`: + +``` +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-physics.test.txt runs/run.beir-v1.0.0-cqadupstack-physics-wp.bm25.topics.beir-v1.0.0-cqadupstack-physics.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-physics.test.txt runs/run.beir-v1.0.0-cqadupstack-physics-wp.bm25.topics.beir-v1.0.0-cqadupstack-physics.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-physics.test.txt runs/run.beir-v1.0.0-cqadupstack-physics-wp.bm25.topics.beir-v1.0.0-cqadupstack-physics.test.wp.txt +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +| nDCG@10 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-physics | 0.3012 | + + +| R@100 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-physics | 0.5979 | + + +| R@1000 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-physics | 0.7968 | diff --git a/docs/regressions-beir-v1.0.0-cqadupstack-programmers-wp.md b/docs/regressions-beir-v1.0.0-cqadupstack-programmers-wp.md new file mode 100644 index 0000000000..251e34727c --- /dev/null +++ b/docs/regressions-beir-v1.0.0-cqadupstack-programmers-wp.md @@ -0,0 +1,69 @@ +# Anserini Regressions: BEIR (v1.0.0) — CQADupStack-programmers + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — CQADupStack-programmers](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/beir-v1.0.0-cqadupstack-programmers-wp.yaml). +Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-programmers-wp.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-programmers-wp +``` + +## Indexing + +Typical indexing command: + +``` +target/appassembler/bin/IndexCollection \ + -collection BeirFlatCollection \ + -input /path/to/beir-v1.0.0-cqadupstack-programmers-wp \ + -index indexes/lucene-index.beir-v1.0.0-cqadupstack-programmers-wp/ \ + -generator DefaultLuceneDocumentGenerator \ + -threads 1 -storePositions -storeDocvectors -storeRaw -pretokenized \ + >& logs/log.beir-v1.0.0-cqadupstack-programmers-wp & +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.beir-v1.0.0-cqadupstack-programmers-wp/ \ + -topics src/main/resources/topics-and-qrels/topics.beir-v1.0.0-cqadupstack-programmers.test.wp.tsv.gz \ + -topicreader TsvString \ + -output runs/run.beir-v1.0.0-cqadupstack-programmers-wp.bm25.topics.beir-v1.0.0-cqadupstack-programmers.test.wp.txt \ + -bm25 -removeQuery -pretokenized & +``` + +Evaluation can be performed using `trec_eval`: + +``` +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-programmers.test.txt runs/run.beir-v1.0.0-cqadupstack-programmers-wp.bm25.topics.beir-v1.0.0-cqadupstack-programmers.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-programmers.test.txt runs/run.beir-v1.0.0-cqadupstack-programmers-wp.bm25.topics.beir-v1.0.0-cqadupstack-programmers.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-programmers.test.txt runs/run.beir-v1.0.0-cqadupstack-programmers-wp.bm25.topics.beir-v1.0.0-cqadupstack-programmers.test.wp.txt +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +| nDCG@10 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-programmers | 0.2750 | + + +| R@100 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-programmers | 0.5450 | + + +| R@1000 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-programmers | 0.7565 | diff --git a/docs/regressions-beir-v1.0.0-cqadupstack-stats-wp.md b/docs/regressions-beir-v1.0.0-cqadupstack-stats-wp.md new file mode 100644 index 0000000000..99460227dc --- /dev/null +++ b/docs/regressions-beir-v1.0.0-cqadupstack-stats-wp.md @@ -0,0 +1,69 @@ +# Anserini Regressions: BEIR (v1.0.0) — CQADupStack-stats + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — CQADupStack-stats](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/beir-v1.0.0-cqadupstack-stats-wp.yaml). +Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-stats-wp.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-stats-wp +``` + +## Indexing + +Typical indexing command: + +``` +target/appassembler/bin/IndexCollection \ + -collection BeirFlatCollection \ + -input /path/to/beir-v1.0.0-cqadupstack-stats-wp \ + -index indexes/lucene-index.beir-v1.0.0-cqadupstack-stats-wp/ \ + -generator DefaultLuceneDocumentGenerator \ + -threads 1 -storePositions -storeDocvectors -storeRaw -pretokenized \ + >& logs/log.beir-v1.0.0-cqadupstack-stats-wp & +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.beir-v1.0.0-cqadupstack-stats-wp/ \ + -topics src/main/resources/topics-and-qrels/topics.beir-v1.0.0-cqadupstack-stats.test.wp.tsv.gz \ + -topicreader TsvString \ + -output runs/run.beir-v1.0.0-cqadupstack-stats-wp.bm25.topics.beir-v1.0.0-cqadupstack-stats.test.wp.txt \ + -bm25 -removeQuery -pretokenized & +``` + +Evaluation can be performed using `trec_eval`: + +``` +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-stats.test.txt runs/run.beir-v1.0.0-cqadupstack-stats-wp.bm25.topics.beir-v1.0.0-cqadupstack-stats.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-stats.test.txt runs/run.beir-v1.0.0-cqadupstack-stats-wp.bm25.topics.beir-v1.0.0-cqadupstack-stats.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-stats.test.txt runs/run.beir-v1.0.0-cqadupstack-stats-wp.bm25.topics.beir-v1.0.0-cqadupstack-stats.test.wp.txt +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +| nDCG@10 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-stats | 0.2677 | + + +| R@100 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-stats | 0.5225 | + + +| R@1000 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-stats | 0.7228 | diff --git a/docs/regressions-beir-v1.0.0-cqadupstack-tex-wp.md b/docs/regressions-beir-v1.0.0-cqadupstack-tex-wp.md new file mode 100644 index 0000000000..2d4770c1c9 --- /dev/null +++ b/docs/regressions-beir-v1.0.0-cqadupstack-tex-wp.md @@ -0,0 +1,69 @@ +# Anserini Regressions: BEIR (v1.0.0) — CQADupStack-tex + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — CQADupStack-tex](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/beir-v1.0.0-cqadupstack-tex-wp.yaml). +Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-tex-wp.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-tex-wp +``` + +## Indexing + +Typical indexing command: + +``` +target/appassembler/bin/IndexCollection \ + -collection BeirFlatCollection \ + -input /path/to/beir-v1.0.0-cqadupstack-tex-wp \ + -index indexes/lucene-index.beir-v1.0.0-cqadupstack-tex-wp/ \ + -generator DefaultLuceneDocumentGenerator \ + -threads 1 -storePositions -storeDocvectors -storeRaw -pretokenized \ + >& logs/log.beir-v1.0.0-cqadupstack-tex-wp & +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.beir-v1.0.0-cqadupstack-tex-wp/ \ + -topics src/main/resources/topics-and-qrels/topics.beir-v1.0.0-cqadupstack-tex.test.wp.tsv.gz \ + -topicreader TsvString \ + -output runs/run.beir-v1.0.0-cqadupstack-tex-wp.bm25.topics.beir-v1.0.0-cqadupstack-tex.test.wp.txt \ + -bm25 -removeQuery -pretokenized & +``` + +Evaluation can be performed using `trec_eval`: + +``` +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-tex.test.txt runs/run.beir-v1.0.0-cqadupstack-tex-wp.bm25.topics.beir-v1.0.0-cqadupstack-tex.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-tex.test.txt runs/run.beir-v1.0.0-cqadupstack-tex-wp.bm25.topics.beir-v1.0.0-cqadupstack-tex.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-tex.test.txt runs/run.beir-v1.0.0-cqadupstack-tex-wp.bm25.topics.beir-v1.0.0-cqadupstack-tex.test.wp.txt +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +| nDCG@10 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-tex | 0.2173 | + + +| R@100 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-tex | 0.4475 | + + +| R@1000 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-tex | 0.6599 | diff --git a/docs/regressions-beir-v1.0.0-cqadupstack-unix-wp.md b/docs/regressions-beir-v1.0.0-cqadupstack-unix-wp.md new file mode 100644 index 0000000000..0724e0cd7a --- /dev/null +++ b/docs/regressions-beir-v1.0.0-cqadupstack-unix-wp.md @@ -0,0 +1,69 @@ +# Anserini Regressions: BEIR (v1.0.0) — CQADupStack-unix + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — CQADupStack-unix](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/beir-v1.0.0-cqadupstack-unix-wp.yaml). +Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-unix-wp.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-unix-wp +``` + +## Indexing + +Typical indexing command: + +``` +target/appassembler/bin/IndexCollection \ + -collection BeirFlatCollection \ + -input /path/to/beir-v1.0.0-cqadupstack-unix-wp \ + -index indexes/lucene-index.beir-v1.0.0-cqadupstack-unix-wp/ \ + -generator DefaultLuceneDocumentGenerator \ + -threads 1 -storePositions -storeDocvectors -storeRaw -pretokenized \ + >& logs/log.beir-v1.0.0-cqadupstack-unix-wp & +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.beir-v1.0.0-cqadupstack-unix-wp/ \ + -topics src/main/resources/topics-and-qrels/topics.beir-v1.0.0-cqadupstack-unix.test.wp.tsv.gz \ + -topicreader TsvString \ + -output runs/run.beir-v1.0.0-cqadupstack-unix-wp.bm25.topics.beir-v1.0.0-cqadupstack-unix.test.wp.txt \ + -bm25 -removeQuery -pretokenized & +``` + +Evaluation can be performed using `trec_eval`: + +``` +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-unix.test.txt runs/run.beir-v1.0.0-cqadupstack-unix-wp.bm25.topics.beir-v1.0.0-cqadupstack-unix.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-unix.test.txt runs/run.beir-v1.0.0-cqadupstack-unix-wp.bm25.topics.beir-v1.0.0-cqadupstack-unix.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-unix.test.txt runs/run.beir-v1.0.0-cqadupstack-unix-wp.bm25.topics.beir-v1.0.0-cqadupstack-unix.test.wp.txt +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +| nDCG@10 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-unix | 0.2668 | + + +| R@100 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-unix | 0.4932 | + + +| R@1000 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-unix | 0.7347 | diff --git a/docs/regressions-beir-v1.0.0-cqadupstack-webmasters-wp.md b/docs/regressions-beir-v1.0.0-cqadupstack-webmasters-wp.md new file mode 100644 index 0000000000..57e1fe3f42 --- /dev/null +++ b/docs/regressions-beir-v1.0.0-cqadupstack-webmasters-wp.md @@ -0,0 +1,69 @@ +# Anserini Regressions: BEIR (v1.0.0) — CQADupStack-webmasters + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — CQADupStack-webmasters](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/beir-v1.0.0-cqadupstack-webmasters-wp.yaml). +Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-webmasters-wp.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-webmasters-wp +``` + +## Indexing + +Typical indexing command: + +``` +target/appassembler/bin/IndexCollection \ + -collection BeirFlatCollection \ + -input /path/to/beir-v1.0.0-cqadupstack-webmasters-wp \ + -index indexes/lucene-index.beir-v1.0.0-cqadupstack-webmasters-wp/ \ + -generator DefaultLuceneDocumentGenerator \ + -threads 1 -storePositions -storeDocvectors -storeRaw -pretokenized \ + >& logs/log.beir-v1.0.0-cqadupstack-webmasters-wp & +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.beir-v1.0.0-cqadupstack-webmasters-wp/ \ + -topics src/main/resources/topics-and-qrels/topics.beir-v1.0.0-cqadupstack-webmasters.test.wp.tsv.gz \ + -topicreader TsvString \ + -output runs/run.beir-v1.0.0-cqadupstack-webmasters-wp.bm25.topics.beir-v1.0.0-cqadupstack-webmasters.test.wp.txt \ + -bm25 -removeQuery -pretokenized & +``` + +Evaluation can be performed using `trec_eval`: + +``` +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-webmasters.test.txt runs/run.beir-v1.0.0-cqadupstack-webmasters-wp.bm25.topics.beir-v1.0.0-cqadupstack-webmasters.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-webmasters.test.txt runs/run.beir-v1.0.0-cqadupstack-webmasters-wp.bm25.topics.beir-v1.0.0-cqadupstack-webmasters.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-webmasters.test.txt runs/run.beir-v1.0.0-cqadupstack-webmasters-wp.bm25.topics.beir-v1.0.0-cqadupstack-webmasters.test.wp.txt +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +| nDCG@10 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-webmasters | 0.3012 | + + +| R@100 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-webmasters | 0.5627 | + + +| R@1000 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-webmasters | 0.7793 | diff --git a/docs/regressions-beir-v1.0.0-cqadupstack-wordpress-wp.md b/docs/regressions-beir-v1.0.0-cqadupstack-wordpress-wp.md new file mode 100644 index 0000000000..da76e83d4e --- /dev/null +++ b/docs/regressions-beir-v1.0.0-cqadupstack-wordpress-wp.md @@ -0,0 +1,69 @@ +# Anserini Regressions: BEIR (v1.0.0) — CQADupStack-wordpress + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — CQADupStack-wordpress](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/beir-v1.0.0-cqadupstack-wordpress-wp.yaml). +Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-wordpress-wp.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-wordpress-wp +``` + +## Indexing + +Typical indexing command: + +``` +target/appassembler/bin/IndexCollection \ + -collection BeirFlatCollection \ + -input /path/to/beir-v1.0.0-cqadupstack-wordpress-wp \ + -index indexes/lucene-index.beir-v1.0.0-cqadupstack-wordpress-wp/ \ + -generator DefaultLuceneDocumentGenerator \ + -threads 1 -storePositions -storeDocvectors -storeRaw -pretokenized \ + >& logs/log.beir-v1.0.0-cqadupstack-wordpress-wp & +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.beir-v1.0.0-cqadupstack-wordpress-wp/ \ + -topics src/main/resources/topics-and-qrels/topics.beir-v1.0.0-cqadupstack-wordpress.test.wp.tsv.gz \ + -topicreader TsvString \ + -output runs/run.beir-v1.0.0-cqadupstack-wordpress-wp.bm25.topics.beir-v1.0.0-cqadupstack-wordpress.test.wp.txt \ + -bm25 -removeQuery -pretokenized & +``` + +Evaluation can be performed using `trec_eval`: + +``` +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-wordpress.test.txt runs/run.beir-v1.0.0-cqadupstack-wordpress-wp.bm25.topics.beir-v1.0.0-cqadupstack-wordpress.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-wordpress.test.txt runs/run.beir-v1.0.0-cqadupstack-wordpress-wp.bm25.topics.beir-v1.0.0-cqadupstack-wordpress.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-cqadupstack-wordpress.test.txt runs/run.beir-v1.0.0-cqadupstack-wordpress-wp.bm25.topics.beir-v1.0.0-cqadupstack-wordpress.test.wp.txt +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +| nDCG@10 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-wordpress | 0.2415 | + + +| R@100 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-wordpress | 0.4985 | + + +| R@1000 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): CQADupStack-wordpress | 0.7193 | diff --git a/docs/regressions-beir-v1.0.0-dbpedia-entity-wp.md b/docs/regressions-beir-v1.0.0-dbpedia-entity-wp.md new file mode 100644 index 0000000000..961e35e614 --- /dev/null +++ b/docs/regressions-beir-v1.0.0-dbpedia-entity-wp.md @@ -0,0 +1,69 @@ +# Anserini Regressions: BEIR (v1.0.0) — DBPedia + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — DBPedia](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/beir-v1.0.0-dbpedia-entity-wp.yaml). +Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/beir-v1.0.0-dbpedia-entity-wp.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-dbpedia-entity-wp +``` + +## Indexing + +Typical indexing command: + +``` +target/appassembler/bin/IndexCollection \ + -collection BeirFlatCollection \ + -input /path/to/beir-v1.0.0-dbpedia-entity-wp \ + -index indexes/lucene-index.beir-v1.0.0-dbpedia-entity-wp/ \ + -generator DefaultLuceneDocumentGenerator \ + -threads 1 -storePositions -storeDocvectors -storeRaw -pretokenized \ + >& logs/log.beir-v1.0.0-dbpedia-entity-wp & +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.beir-v1.0.0-dbpedia-entity-wp/ \ + -topics src/main/resources/topics-and-qrels/topics.beir-v1.0.0-dbpedia-entity.test.wp.tsv.gz \ + -topicreader TsvString \ + -output runs/run.beir-v1.0.0-dbpedia-entity-wp.bm25.topics.beir-v1.0.0-dbpedia-entity.test.wp.txt \ + -bm25 -removeQuery -pretokenized & +``` + +Evaluation can be performed using `trec_eval`: + +``` +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-dbpedia-entity.test.txt runs/run.beir-v1.0.0-dbpedia-entity-wp.bm25.topics.beir-v1.0.0-dbpedia-entity.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-dbpedia-entity.test.txt runs/run.beir-v1.0.0-dbpedia-entity-wp.bm25.topics.beir-v1.0.0-dbpedia-entity.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-dbpedia-entity.test.txt runs/run.beir-v1.0.0-dbpedia-entity-wp.bm25.topics.beir-v1.0.0-dbpedia-entity.test.wp.txt +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +| nDCG@10 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): DBPedia | 0.2841 | + + +| R@100 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): DBPedia | 0.4255 | + + +| R@1000 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): DBPedia | 0.6124 | diff --git a/docs/regressions-beir-v1.0.0-fever-wp.md b/docs/regressions-beir-v1.0.0-fever-wp.md new file mode 100644 index 0000000000..fb2b85ca86 --- /dev/null +++ b/docs/regressions-beir-v1.0.0-fever-wp.md @@ -0,0 +1,69 @@ +# Anserini Regressions: BEIR (v1.0.0) — FEVER + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — FEVER](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/beir-v1.0.0-fever-wp.yaml). +Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/beir-v1.0.0-fever-wp.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-fever-wp +``` + +## Indexing + +Typical indexing command: + +``` +target/appassembler/bin/IndexCollection \ + -collection BeirFlatCollection \ + -input /path/to/beir-v1.0.0-fever-wp \ + -index indexes/lucene-index.beir-v1.0.0-fever-wp/ \ + -generator DefaultLuceneDocumentGenerator \ + -threads 1 -storePositions -storeDocvectors -storeRaw -pretokenized \ + >& logs/log.beir-v1.0.0-fever-wp & +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.beir-v1.0.0-fever-wp/ \ + -topics src/main/resources/topics-and-qrels/topics.beir-v1.0.0-fever.test.wp.tsv.gz \ + -topicreader TsvString \ + -output runs/run.beir-v1.0.0-fever-wp.bm25.topics.beir-v1.0.0-fever.test.wp.txt \ + -bm25 -removeQuery -pretokenized & +``` + +Evaluation can be performed using `trec_eval`: + +``` +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-fever.test.txt runs/run.beir-v1.0.0-fever-wp.bm25.topics.beir-v1.0.0-fever.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-fever.test.txt runs/run.beir-v1.0.0-fever-wp.bm25.topics.beir-v1.0.0-fever.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-fever.test.txt runs/run.beir-v1.0.0-fever-wp.bm25.topics.beir-v1.0.0-fever.test.wp.txt +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +| nDCG@10 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): FEVER | 0.6583 | + + +| R@100 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): FEVER | 0.9212 | + + +| R@1000 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): FEVER | 0.9622 | diff --git a/docs/regressions-beir-v1.0.0-fiqa-wp.md b/docs/regressions-beir-v1.0.0-fiqa-wp.md new file mode 100644 index 0000000000..584060ed57 --- /dev/null +++ b/docs/regressions-beir-v1.0.0-fiqa-wp.md @@ -0,0 +1,69 @@ +# Anserini Regressions: BEIR (v1.0.0) — FiQA-2018 + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — FiQA-2018](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/beir-v1.0.0-fiqa-wp.yaml). +Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/beir-v1.0.0-fiqa-wp.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-fiqa-wp +``` + +## Indexing + +Typical indexing command: + +``` +target/appassembler/bin/IndexCollection \ + -collection BeirFlatCollection \ + -input /path/to/beir-v1.0.0-fiqa-wp \ + -index indexes/lucene-index.beir-v1.0.0-fiqa-wp/ \ + -generator DefaultLuceneDocumentGenerator \ + -threads 1 -storePositions -storeDocvectors -storeRaw -pretokenized \ + >& logs/log.beir-v1.0.0-fiqa-wp & +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.beir-v1.0.0-fiqa-wp/ \ + -topics src/main/resources/topics-and-qrels/topics.beir-v1.0.0-fiqa.test.wp.tsv.gz \ + -topicreader TsvString \ + -output runs/run.beir-v1.0.0-fiqa-wp.bm25.topics.beir-v1.0.0-fiqa.test.wp.txt \ + -bm25 -removeQuery -pretokenized & +``` + +Evaluation can be performed using `trec_eval`: + +``` +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-fiqa.test.txt runs/run.beir-v1.0.0-fiqa-wp.bm25.topics.beir-v1.0.0-fiqa.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-fiqa.test.txt runs/run.beir-v1.0.0-fiqa-wp.bm25.topics.beir-v1.0.0-fiqa.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-fiqa.test.txt runs/run.beir-v1.0.0-fiqa-wp.bm25.topics.beir-v1.0.0-fiqa.test.wp.txt +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +| nDCG@10 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): FiQA-2018 | 0.2179 | + + +| R@100 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): FiQA-2018 | 0.4825 | + + +| R@1000 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): FiQA-2018 | 0.7081 | diff --git a/docs/regressions-beir-v1.0.0-hotpotqa-wp.md b/docs/regressions-beir-v1.0.0-hotpotqa-wp.md new file mode 100644 index 0000000000..27b663dc1a --- /dev/null +++ b/docs/regressions-beir-v1.0.0-hotpotqa-wp.md @@ -0,0 +1,69 @@ +# Anserini Regressions: BEIR (v1.0.0) — HotpotQA + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — HotpotQA](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/beir-v1.0.0-hotpotqa-wp.yaml). +Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/beir-v1.0.0-hotpotqa-wp.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-hotpotqa-wp +``` + +## Indexing + +Typical indexing command: + +``` +target/appassembler/bin/IndexCollection \ + -collection BeirFlatCollection \ + -input /path/to/beir-v1.0.0-hotpotqa-wp \ + -index indexes/lucene-index.beir-v1.0.0-hotpotqa-wp/ \ + -generator DefaultLuceneDocumentGenerator \ + -threads 1 -storePositions -storeDocvectors -storeRaw -pretokenized \ + >& logs/log.beir-v1.0.0-hotpotqa-wp & +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.beir-v1.0.0-hotpotqa-wp/ \ + -topics src/main/resources/topics-and-qrels/topics.beir-v1.0.0-hotpotqa.test.wp.tsv.gz \ + -topicreader TsvString \ + -output runs/run.beir-v1.0.0-hotpotqa-wp.bm25.topics.beir-v1.0.0-hotpotqa.test.wp.txt \ + -bm25 -removeQuery -pretokenized & +``` + +Evaluation can be performed using `trec_eval`: + +``` +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-hotpotqa.test.txt runs/run.beir-v1.0.0-hotpotqa-wp.bm25.topics.beir-v1.0.0-hotpotqa.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-hotpotqa.test.txt runs/run.beir-v1.0.0-hotpotqa-wp.bm25.topics.beir-v1.0.0-hotpotqa.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-hotpotqa.test.txt runs/run.beir-v1.0.0-hotpotqa-wp.bm25.topics.beir-v1.0.0-hotpotqa.test.wp.txt +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +| nDCG@10 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): HotpotQA | 0.5926 | + + +| R@100 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): HotpotQA | 0.7612 | + + +| R@1000 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): HotpotQA | 0.8600 | diff --git a/docs/regressions-beir-v1.0.0-nfcorpus-wp.md b/docs/regressions-beir-v1.0.0-nfcorpus-wp.md new file mode 100644 index 0000000000..3b32462ca7 --- /dev/null +++ b/docs/regressions-beir-v1.0.0-nfcorpus-wp.md @@ -0,0 +1,69 @@ +# Anserini Regressions: BEIR (v1.0.0) — NFCorpus + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — NFCorpus](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/beir-v1.0.0-nfcorpus-wp.yaml). +Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/beir-v1.0.0-nfcorpus-wp.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-nfcorpus-wp +``` + +## Indexing + +Typical indexing command: + +``` +target/appassembler/bin/IndexCollection \ + -collection BeirFlatCollection \ + -input /path/to/beir-v1.0.0-nfcorpus-wp \ + -index indexes/lucene-index.beir-v1.0.0-nfcorpus-wp/ \ + -generator DefaultLuceneDocumentGenerator \ + -threads 1 -storePositions -storeDocvectors -storeRaw -pretokenized \ + >& logs/log.beir-v1.0.0-nfcorpus-wp & +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.beir-v1.0.0-nfcorpus-wp/ \ + -topics src/main/resources/topics-and-qrels/topics.beir-v1.0.0-nfcorpus.test.wp.tsv.gz \ + -topicreader TsvString \ + -output runs/run.beir-v1.0.0-nfcorpus-wp.bm25.topics.beir-v1.0.0-nfcorpus.test.wp.txt \ + -bm25 -removeQuery -pretokenized & +``` + +Evaluation can be performed using `trec_eval`: + +``` +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-nfcorpus.test.txt runs/run.beir-v1.0.0-nfcorpus-wp.bm25.topics.beir-v1.0.0-nfcorpus.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-nfcorpus.test.txt runs/run.beir-v1.0.0-nfcorpus-wp.bm25.topics.beir-v1.0.0-nfcorpus.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-nfcorpus.test.txt runs/run.beir-v1.0.0-nfcorpus-wp.bm25.topics.beir-v1.0.0-nfcorpus.test.wp.txt +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +| nDCG@10 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): NFCorpus | 0.3142 | + + +| R@100 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): NFCorpus | 0.2441 | + + +| R@1000 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): NFCorpus | 0.4222 | diff --git a/docs/regressions-beir-v1.0.0-nq-wp.md b/docs/regressions-beir-v1.0.0-nq-wp.md new file mode 100644 index 0000000000..8b8d27eb63 --- /dev/null +++ b/docs/regressions-beir-v1.0.0-nq-wp.md @@ -0,0 +1,69 @@ +# Anserini Regressions: BEIR (v1.0.0) — NQ + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — NQ](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/beir-v1.0.0-nq-wp.yaml). +Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/beir-v1.0.0-nq-wp.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-nq-wp +``` + +## Indexing + +Typical indexing command: + +``` +target/appassembler/bin/IndexCollection \ + -collection BeirFlatCollection \ + -input /path/to/beir-v1.0.0-nq-wp \ + -index indexes/lucene-index.beir-v1.0.0-nq-wp/ \ + -generator DefaultLuceneDocumentGenerator \ + -threads 1 -storePositions -storeDocvectors -storeRaw -pretokenized \ + >& logs/log.beir-v1.0.0-nq-wp & +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.beir-v1.0.0-nq-wp/ \ + -topics src/main/resources/topics-and-qrels/topics.beir-v1.0.0-nq.test.wp.tsv.gz \ + -topicreader TsvString \ + -output runs/run.beir-v1.0.0-nq-wp.bm25.topics.beir-v1.0.0-nq.test.wp.txt \ + -bm25 -removeQuery -pretokenized & +``` + +Evaluation can be performed using `trec_eval`: + +``` +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-nq.test.txt runs/run.beir-v1.0.0-nq-wp.bm25.topics.beir-v1.0.0-nq.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-nq.test.txt runs/run.beir-v1.0.0-nq-wp.bm25.topics.beir-v1.0.0-nq.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-nq.test.txt runs/run.beir-v1.0.0-nq-wp.bm25.topics.beir-v1.0.0-nq.test.wp.txt +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +| nDCG@10 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): NQ | 0.3046 | + + +| R@100 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): NQ | 0.7390 | + + +| R@1000 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): NQ | 0.8917 | diff --git a/docs/regressions-beir-v1.0.0-quora-wp.md b/docs/regressions-beir-v1.0.0-quora-wp.md new file mode 100644 index 0000000000..209a189e16 --- /dev/null +++ b/docs/regressions-beir-v1.0.0-quora-wp.md @@ -0,0 +1,69 @@ +# Anserini Regressions: BEIR (v1.0.0) — Quora + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — Quora](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/beir-v1.0.0-quora-wp.yaml). +Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/beir-v1.0.0-quora-wp.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-quora-wp +``` + +## Indexing + +Typical indexing command: + +``` +target/appassembler/bin/IndexCollection \ + -collection BeirFlatCollection \ + -input /path/to/beir-v1.0.0-quora-wp \ + -index indexes/lucene-index.beir-v1.0.0-quora-wp/ \ + -generator DefaultLuceneDocumentGenerator \ + -threads 1 -storePositions -storeDocvectors -storeRaw -pretokenized \ + >& logs/log.beir-v1.0.0-quora-wp & +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.beir-v1.0.0-quora-wp/ \ + -topics src/main/resources/topics-and-qrels/topics.beir-v1.0.0-quora.test.wp.tsv.gz \ + -topicreader TsvString \ + -output runs/run.beir-v1.0.0-quora-wp.bm25.topics.beir-v1.0.0-quora.test.wp.txt \ + -bm25 -removeQuery -pretokenized & +``` + +Evaluation can be performed using `trec_eval`: + +``` +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-quora.test.txt runs/run.beir-v1.0.0-quora-wp.bm25.topics.beir-v1.0.0-quora.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-quora.test.txt runs/run.beir-v1.0.0-quora-wp.bm25.topics.beir-v1.0.0-quora.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-quora.test.txt runs/run.beir-v1.0.0-quora-wp.bm25.topics.beir-v1.0.0-quora.test.wp.txt +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +| nDCG@10 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): Quora | 0.7295 | + + +| R@100 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): Quora | 0.9459 | + + +| R@1000 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): Quora | 0.9872 | diff --git a/docs/regressions-beir-v1.0.0-scidocs-wp.md b/docs/regressions-beir-v1.0.0-scidocs-wp.md new file mode 100644 index 0000000000..44e61d17c4 --- /dev/null +++ b/docs/regressions-beir-v1.0.0-scidocs-wp.md @@ -0,0 +1,69 @@ +# Anserini Regressions: BEIR (v1.0.0) — SCIDOCS + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — SCIDOCS](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/beir-v1.0.0-scidocs-wp.yaml). +Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/beir-v1.0.0-scidocs-wp.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-scidocs-wp +``` + +## Indexing + +Typical indexing command: + +``` +target/appassembler/bin/IndexCollection \ + -collection BeirFlatCollection \ + -input /path/to/beir-v1.0.0-scidocs-wp \ + -index indexes/lucene-index.beir-v1.0.0-scidocs-wp/ \ + -generator DefaultLuceneDocumentGenerator \ + -threads 1 -storePositions -storeDocvectors -storeRaw -pretokenized \ + >& logs/log.beir-v1.0.0-scidocs-wp & +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.beir-v1.0.0-scidocs-wp/ \ + -topics src/main/resources/topics-and-qrels/topics.beir-v1.0.0-scidocs.test.wp.tsv.gz \ + -topicreader TsvString \ + -output runs/run.beir-v1.0.0-scidocs-wp.bm25.topics.beir-v1.0.0-scidocs.test.wp.txt \ + -bm25 -removeQuery -pretokenized & +``` + +Evaluation can be performed using `trec_eval`: + +``` +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-scidocs.test.txt runs/run.beir-v1.0.0-scidocs-wp.bm25.topics.beir-v1.0.0-scidocs.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-scidocs.test.txt runs/run.beir-v1.0.0-scidocs-wp.bm25.topics.beir-v1.0.0-scidocs.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-scidocs.test.txt runs/run.beir-v1.0.0-scidocs-wp.bm25.topics.beir-v1.0.0-scidocs.test.wp.txt +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +| nDCG@10 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): SCIDOCS | 0.1380 | + + +| R@100 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): SCIDOCS | 0.3325 | + + +| R@1000 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): SCIDOCS | 0.5479 | diff --git a/docs/regressions-beir-v1.0.0-scifact-wp.md b/docs/regressions-beir-v1.0.0-scifact-wp.md new file mode 100644 index 0000000000..584c750c1b --- /dev/null +++ b/docs/regressions-beir-v1.0.0-scifact-wp.md @@ -0,0 +1,69 @@ +# Anserini Regressions: BEIR (v1.0.0) — SciFact + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — SciFact](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/beir-v1.0.0-scifact-wp.yaml). +Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/beir-v1.0.0-scifact-wp.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-scifact-wp +``` + +## Indexing + +Typical indexing command: + +``` +target/appassembler/bin/IndexCollection \ + -collection BeirFlatCollection \ + -input /path/to/beir-v1.0.0-scifact-wp \ + -index indexes/lucene-index.beir-v1.0.0-scifact-wp/ \ + -generator DefaultLuceneDocumentGenerator \ + -threads 1 -storePositions -storeDocvectors -storeRaw -pretokenized \ + >& logs/log.beir-v1.0.0-scifact-wp & +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.beir-v1.0.0-scifact-wp/ \ + -topics src/main/resources/topics-and-qrels/topics.beir-v1.0.0-scifact.test.wp.tsv.gz \ + -topicreader TsvString \ + -output runs/run.beir-v1.0.0-scifact-wp.bm25.topics.beir-v1.0.0-scifact.test.wp.txt \ + -bm25 -removeQuery -pretokenized & +``` + +Evaluation can be performed using `trec_eval`: + +``` +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-scifact.test.txt runs/run.beir-v1.0.0-scifact-wp.bm25.topics.beir-v1.0.0-scifact.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-scifact.test.txt runs/run.beir-v1.0.0-scifact-wp.bm25.topics.beir-v1.0.0-scifact.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-scifact.test.txt runs/run.beir-v1.0.0-scifact-wp.bm25.topics.beir-v1.0.0-scifact.test.wp.txt +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +| nDCG@10 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): SciFact | 0.6718 | + + +| R@100 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): SciFact | 0.9213 | + + +| R@1000 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): SciFact | 0.9743 | diff --git a/docs/regressions-beir-v1.0.0-trec-covid-wp.md b/docs/regressions-beir-v1.0.0-trec-covid-wp.md new file mode 100644 index 0000000000..2b9fc80e1e --- /dev/null +++ b/docs/regressions-beir-v1.0.0-trec-covid-wp.md @@ -0,0 +1,69 @@ +# Anserini Regressions: BEIR (v1.0.0) — TREC-COVID + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — TREC-COVID](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/beir-v1.0.0-trec-covid-wp.yaml). +Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/beir-v1.0.0-trec-covid-wp.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-trec-covid-wp +``` + +## Indexing + +Typical indexing command: + +``` +target/appassembler/bin/IndexCollection \ + -collection BeirFlatCollection \ + -input /path/to/beir-v1.0.0-trec-covid-wp \ + -index indexes/lucene-index.beir-v1.0.0-trec-covid-wp/ \ + -generator DefaultLuceneDocumentGenerator \ + -threads 1 -storePositions -storeDocvectors -storeRaw -pretokenized \ + >& logs/log.beir-v1.0.0-trec-covid-wp & +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.beir-v1.0.0-trec-covid-wp/ \ + -topics src/main/resources/topics-and-qrels/topics.beir-v1.0.0-trec-covid.test.wp.tsv.gz \ + -topicreader TsvString \ + -output runs/run.beir-v1.0.0-trec-covid-wp.bm25.topics.beir-v1.0.0-trec-covid.test.wp.txt \ + -bm25 -removeQuery -pretokenized & +``` + +Evaluation can be performed using `trec_eval`: + +``` +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-trec-covid.test.txt runs/run.beir-v1.0.0-trec-covid-wp.bm25.topics.beir-v1.0.0-trec-covid.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-trec-covid.test.txt runs/run.beir-v1.0.0-trec-covid-wp.bm25.topics.beir-v1.0.0-trec-covid.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-trec-covid.test.txt runs/run.beir-v1.0.0-trec-covid-wp.bm25.topics.beir-v1.0.0-trec-covid.test.wp.txt +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +| nDCG@10 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): TREC-COVID | 0.5645 | + + +| R@100 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): TREC-COVID | 0.1001 | + + +| R@1000 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): TREC-COVID | 0.3621 | diff --git a/docs/regressions-beir-v1.0.0-webis-touche2020-wp.md b/docs/regressions-beir-v1.0.0-webis-touche2020-wp.md new file mode 100644 index 0000000000..4cda32ef01 --- /dev/null +++ b/docs/regressions-beir-v1.0.0-webis-touche2020-wp.md @@ -0,0 +1,69 @@ +# Anserini Regressions: BEIR (v1.0.0) — Webis-Touche2020 + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — Webis-Touche2020](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/beir-v1.0.0-webis-touche2020-wp.yaml). +Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/beir-v1.0.0-webis-touche2020-wp.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-webis-touche2020-wp +``` + +## Indexing + +Typical indexing command: + +``` +target/appassembler/bin/IndexCollection \ + -collection BeirFlatCollection \ + -input /path/to/beir-v1.0.0-webis-touche2020-wp \ + -index indexes/lucene-index.beir-v1.0.0-webis-touche2020-wp/ \ + -generator DefaultLuceneDocumentGenerator \ + -threads 1 -storePositions -storeDocvectors -storeRaw -pretokenized \ + >& logs/log.beir-v1.0.0-webis-touche2020-wp & +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.beir-v1.0.0-webis-touche2020-wp/ \ + -topics src/main/resources/topics-and-qrels/topics.beir-v1.0.0-webis-touche2020.test.wp.tsv.gz \ + -topicreader TsvString \ + -output runs/run.beir-v1.0.0-webis-touche2020-wp.bm25.topics.beir-v1.0.0-webis-touche2020.test.wp.txt \ + -bm25 -removeQuery -pretokenized & +``` + +Evaluation can be performed using `trec_eval`: + +``` +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-webis-touche2020.test.txt runs/run.beir-v1.0.0-webis-touche2020-wp.bm25.topics.beir-v1.0.0-webis-touche2020.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-webis-touche2020.test.txt runs/run.beir-v1.0.0-webis-touche2020-wp.bm25.topics.beir-v1.0.0-webis-touche2020.test.wp.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.beir-v1.0.0-webis-touche2020.test.txt runs/run.beir-v1.0.0-webis-touche2020-wp.bm25.topics.beir-v1.0.0-webis-touche2020.test.wp.txt +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +| nDCG@10 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): Webis-Touche2020 | 0.4661 | + + +| R@100 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): Webis-Touche2020 | 0.5702 | + + +| R@1000 | BM25 | +|:-------------------------------------------------------------------------------------------------------------|-----------| +| BEIR (v1.0.0): Webis-Touche2020 | 0.8415 | diff --git a/src/main/resources/docgen/templates/beir-v1.0.0-arguana-wp.template b/src/main/resources/docgen/templates/beir-v1.0.0-arguana-wp.template new file mode 100644 index 0000000000..2b1483f2f4 --- /dev/null +++ b/src/main/resources/docgen/templates/beir-v1.0.0-arguana-wp.template @@ -0,0 +1,44 @@ +# Anserini Regressions: BEIR (v1.0.0) — ArguAna + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — ArguAna](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](${yaml}). +Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression ${test_name} +``` + +## Indexing + +Typical indexing command: + +``` +${index_cmds} +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +${ranking_cmds} +``` + +Evaluation can be performed using `trec_eval`: + +``` +${eval_cmds} +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +${effectiveness} diff --git a/src/main/resources/docgen/templates/beir-v1.0.0-climate-fever-wp.template b/src/main/resources/docgen/templates/beir-v1.0.0-climate-fever-wp.template new file mode 100644 index 0000000000..a53e67f7ba --- /dev/null +++ b/src/main/resources/docgen/templates/beir-v1.0.0-climate-fever-wp.template @@ -0,0 +1,44 @@ +# Anserini Regressions: BEIR (v1.0.0) — Climate-FEVER + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — Climate-FEVER](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](${yaml}). +Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression ${test_name} +``` + +## Indexing + +Typical indexing command: + +``` +${index_cmds} +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +${ranking_cmds} +``` + +Evaluation can be performed using `trec_eval`: + +``` +${eval_cmds} +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +${effectiveness} diff --git a/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-android-wp.template b/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-android-wp.template new file mode 100644 index 0000000000..b1c6e68ea6 --- /dev/null +++ b/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-android-wp.template @@ -0,0 +1,44 @@ +# Anserini Regressions: BEIR (v1.0.0) — CQADupStack-android + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — CQADupStack-android](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](${yaml}). +Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression ${test_name} +``` + +## Indexing + +Typical indexing command: + +``` +${index_cmds} +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +${ranking_cmds} +``` + +Evaluation can be performed using `trec_eval`: + +``` +${eval_cmds} +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +${effectiveness} diff --git a/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-english-wp.template b/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-english-wp.template new file mode 100644 index 0000000000..f3880ef499 --- /dev/null +++ b/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-english-wp.template @@ -0,0 +1,44 @@ +# Anserini Regressions: BEIR (v1.0.0) — CQADupStack-english + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — CQADupStack-english](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](${yaml}). +Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression ${test_name} +``` + +## Indexing + +Typical indexing command: + +``` +${index_cmds} +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +${ranking_cmds} +``` + +Evaluation can be performed using `trec_eval`: + +``` +${eval_cmds} +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +${effectiveness} diff --git a/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-gaming-wp.template b/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-gaming-wp.template new file mode 100644 index 0000000000..db7920a5ca --- /dev/null +++ b/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-gaming-wp.template @@ -0,0 +1,44 @@ +# Anserini Regressions: BEIR (v1.0.0) — CQADupStack-gaming + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — CQADupStack-gaming](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](${yaml}). +Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression ${test_name} +``` + +## Indexing + +Typical indexing command: + +``` +${index_cmds} +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +${ranking_cmds} +``` + +Evaluation can be performed using `trec_eval`: + +``` +${eval_cmds} +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +${effectiveness} diff --git a/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-gis-wp.template b/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-gis-wp.template new file mode 100644 index 0000000000..c4cb0da3b5 --- /dev/null +++ b/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-gis-wp.template @@ -0,0 +1,44 @@ +# Anserini Regressions: BEIR (v1.0.0) — CQADupStack-gis + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — CQADupStack-gis](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](${yaml}). +Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression ${test_name} +``` + +## Indexing + +Typical indexing command: + +``` +${index_cmds} +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +${ranking_cmds} +``` + +Evaluation can be performed using `trec_eval`: + +``` +${eval_cmds} +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +${effectiveness} diff --git a/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-mathematica-wp.template b/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-mathematica-wp.template new file mode 100644 index 0000000000..d714c02176 --- /dev/null +++ b/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-mathematica-wp.template @@ -0,0 +1,44 @@ +# Anserini Regressions: BEIR (v1.0.0) — CQADupStack-mathematica + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — CQADupStack-mathematica](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](${yaml}). +Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression ${test_name} +``` + +## Indexing + +Typical indexing command: + +``` +${index_cmds} +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +${ranking_cmds} +``` + +Evaluation can be performed using `trec_eval`: + +``` +${eval_cmds} +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +${effectiveness} diff --git a/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-physics-wp.template b/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-physics-wp.template new file mode 100644 index 0000000000..134f6c6192 --- /dev/null +++ b/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-physics-wp.template @@ -0,0 +1,44 @@ +# Anserini Regressions: BEIR (v1.0.0) — CQADupStack-physics + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — CQADupStack-physics](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](${yaml}). +Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression ${test_name} +``` + +## Indexing + +Typical indexing command: + +``` +${index_cmds} +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +${ranking_cmds} +``` + +Evaluation can be performed using `trec_eval`: + +``` +${eval_cmds} +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +${effectiveness} diff --git a/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-programmers-wp.template b/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-programmers-wp.template new file mode 100644 index 0000000000..be07a2c61e --- /dev/null +++ b/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-programmers-wp.template @@ -0,0 +1,44 @@ +# Anserini Regressions: BEIR (v1.0.0) — CQADupStack-programmers + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — CQADupStack-programmers](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](${yaml}). +Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression ${test_name} +``` + +## Indexing + +Typical indexing command: + +``` +${index_cmds} +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +${ranking_cmds} +``` + +Evaluation can be performed using `trec_eval`: + +``` +${eval_cmds} +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +${effectiveness} diff --git a/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-stats-wp.template b/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-stats-wp.template new file mode 100644 index 0000000000..c3d14d19d0 --- /dev/null +++ b/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-stats-wp.template @@ -0,0 +1,44 @@ +# Anserini Regressions: BEIR (v1.0.0) — CQADupStack-stats + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — CQADupStack-stats](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](${yaml}). +Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression ${test_name} +``` + +## Indexing + +Typical indexing command: + +``` +${index_cmds} +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +${ranking_cmds} +``` + +Evaluation can be performed using `trec_eval`: + +``` +${eval_cmds} +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +${effectiveness} diff --git a/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-tex-wp.template b/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-tex-wp.template new file mode 100644 index 0000000000..0b04d5c7ce --- /dev/null +++ b/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-tex-wp.template @@ -0,0 +1,44 @@ +# Anserini Regressions: BEIR (v1.0.0) — CQADupStack-tex + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — CQADupStack-tex](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](${yaml}). +Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression ${test_name} +``` + +## Indexing + +Typical indexing command: + +``` +${index_cmds} +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +${ranking_cmds} +``` + +Evaluation can be performed using `trec_eval`: + +``` +${eval_cmds} +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +${effectiveness} diff --git a/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-unix-wp.template b/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-unix-wp.template new file mode 100644 index 0000000000..45278a0ad6 --- /dev/null +++ b/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-unix-wp.template @@ -0,0 +1,44 @@ +# Anserini Regressions: BEIR (v1.0.0) — CQADupStack-unix + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — CQADupStack-unix](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](${yaml}). +Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression ${test_name} +``` + +## Indexing + +Typical indexing command: + +``` +${index_cmds} +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +${ranking_cmds} +``` + +Evaluation can be performed using `trec_eval`: + +``` +${eval_cmds} +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +${effectiveness} diff --git a/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-webmasters-wp.template b/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-webmasters-wp.template new file mode 100644 index 0000000000..326f6a5834 --- /dev/null +++ b/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-webmasters-wp.template @@ -0,0 +1,44 @@ +# Anserini Regressions: BEIR (v1.0.0) — CQADupStack-webmasters + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — CQADupStack-webmasters](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](${yaml}). +Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression ${test_name} +``` + +## Indexing + +Typical indexing command: + +``` +${index_cmds} +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +${ranking_cmds} +``` + +Evaluation can be performed using `trec_eval`: + +``` +${eval_cmds} +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +${effectiveness} diff --git a/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-wordpress-wp.template b/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-wordpress-wp.template new file mode 100644 index 0000000000..a975d3ac1c --- /dev/null +++ b/src/main/resources/docgen/templates/beir-v1.0.0-cqadupstack-wordpress-wp.template @@ -0,0 +1,44 @@ +# Anserini Regressions: BEIR (v1.0.0) — CQADupStack-wordpress + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — CQADupStack-wordpress](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](${yaml}). +Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression ${test_name} +``` + +## Indexing + +Typical indexing command: + +``` +${index_cmds} +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +${ranking_cmds} +``` + +Evaluation can be performed using `trec_eval`: + +``` +${eval_cmds} +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +${effectiveness} diff --git a/src/main/resources/docgen/templates/beir-v1.0.0-dbpedia-entity-wp.template b/src/main/resources/docgen/templates/beir-v1.0.0-dbpedia-entity-wp.template new file mode 100644 index 0000000000..62355e461b --- /dev/null +++ b/src/main/resources/docgen/templates/beir-v1.0.0-dbpedia-entity-wp.template @@ -0,0 +1,44 @@ +# Anserini Regressions: BEIR (v1.0.0) — DBPedia + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — DBPedia](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](${yaml}). +Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression ${test_name} +``` + +## Indexing + +Typical indexing command: + +``` +${index_cmds} +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +${ranking_cmds} +``` + +Evaluation can be performed using `trec_eval`: + +``` +${eval_cmds} +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +${effectiveness} diff --git a/src/main/resources/docgen/templates/beir-v1.0.0-fever-wp.template b/src/main/resources/docgen/templates/beir-v1.0.0-fever-wp.template new file mode 100644 index 0000000000..b921ce4043 --- /dev/null +++ b/src/main/resources/docgen/templates/beir-v1.0.0-fever-wp.template @@ -0,0 +1,44 @@ +# Anserini Regressions: BEIR (v1.0.0) — FEVER + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — FEVER](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](${yaml}). +Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression ${test_name} +``` + +## Indexing + +Typical indexing command: + +``` +${index_cmds} +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +${ranking_cmds} +``` + +Evaluation can be performed using `trec_eval`: + +``` +${eval_cmds} +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +${effectiveness} diff --git a/src/main/resources/docgen/templates/beir-v1.0.0-fiqa-wp.template b/src/main/resources/docgen/templates/beir-v1.0.0-fiqa-wp.template new file mode 100644 index 0000000000..c4c3c7e8d3 --- /dev/null +++ b/src/main/resources/docgen/templates/beir-v1.0.0-fiqa-wp.template @@ -0,0 +1,44 @@ +# Anserini Regressions: BEIR (v1.0.0) — FiQA-2018 + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — FiQA-2018](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](${yaml}). +Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression ${test_name} +``` + +## Indexing + +Typical indexing command: + +``` +${index_cmds} +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +${ranking_cmds} +``` + +Evaluation can be performed using `trec_eval`: + +``` +${eval_cmds} +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +${effectiveness} diff --git a/src/main/resources/docgen/templates/beir-v1.0.0-hotpotqa-wp.template b/src/main/resources/docgen/templates/beir-v1.0.0-hotpotqa-wp.template new file mode 100644 index 0000000000..392e7c282f --- /dev/null +++ b/src/main/resources/docgen/templates/beir-v1.0.0-hotpotqa-wp.template @@ -0,0 +1,44 @@ +# Anserini Regressions: BEIR (v1.0.0) — HotpotQA + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — HotpotQA](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](${yaml}). +Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression ${test_name} +``` + +## Indexing + +Typical indexing command: + +``` +${index_cmds} +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +${ranking_cmds} +``` + +Evaluation can be performed using `trec_eval`: + +``` +${eval_cmds} +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +${effectiveness} diff --git a/src/main/resources/docgen/templates/beir-v1.0.0-nfcorpus-wp.template b/src/main/resources/docgen/templates/beir-v1.0.0-nfcorpus-wp.template new file mode 100644 index 0000000000..571ca1ed46 --- /dev/null +++ b/src/main/resources/docgen/templates/beir-v1.0.0-nfcorpus-wp.template @@ -0,0 +1,44 @@ +# Anserini Regressions: BEIR (v1.0.0) — NFCorpus + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — NFCorpus](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](${yaml}). +Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression ${test_name} +``` + +## Indexing + +Typical indexing command: + +``` +${index_cmds} +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +${ranking_cmds} +``` + +Evaluation can be performed using `trec_eval`: + +``` +${eval_cmds} +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +${effectiveness} diff --git a/src/main/resources/docgen/templates/beir-v1.0.0-nq-wp.template b/src/main/resources/docgen/templates/beir-v1.0.0-nq-wp.template new file mode 100644 index 0000000000..96d2de5b7e --- /dev/null +++ b/src/main/resources/docgen/templates/beir-v1.0.0-nq-wp.template @@ -0,0 +1,44 @@ +# Anserini Regressions: BEIR (v1.0.0) — NQ + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — NQ](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](${yaml}). +Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression ${test_name} +``` + +## Indexing + +Typical indexing command: + +``` +${index_cmds} +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +${ranking_cmds} +``` + +Evaluation can be performed using `trec_eval`: + +``` +${eval_cmds} +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +${effectiveness} diff --git a/src/main/resources/docgen/templates/beir-v1.0.0-quora-wp.template b/src/main/resources/docgen/templates/beir-v1.0.0-quora-wp.template new file mode 100644 index 0000000000..9764aeeef6 --- /dev/null +++ b/src/main/resources/docgen/templates/beir-v1.0.0-quora-wp.template @@ -0,0 +1,44 @@ +# Anserini Regressions: BEIR (v1.0.0) — Quora + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — Quora](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](${yaml}). +Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression ${test_name} +``` + +## Indexing + +Typical indexing command: + +``` +${index_cmds} +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +${ranking_cmds} +``` + +Evaluation can be performed using `trec_eval`: + +``` +${eval_cmds} +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +${effectiveness} diff --git a/src/main/resources/docgen/templates/beir-v1.0.0-scidocs-wp.template b/src/main/resources/docgen/templates/beir-v1.0.0-scidocs-wp.template new file mode 100644 index 0000000000..e9fe4785bf --- /dev/null +++ b/src/main/resources/docgen/templates/beir-v1.0.0-scidocs-wp.template @@ -0,0 +1,44 @@ +# Anserini Regressions: BEIR (v1.0.0) — SCIDOCS + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — SCIDOCS](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](${yaml}). +Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression ${test_name} +``` + +## Indexing + +Typical indexing command: + +``` +${index_cmds} +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +${ranking_cmds} +``` + +Evaluation can be performed using `trec_eval`: + +``` +${eval_cmds} +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +${effectiveness} diff --git a/src/main/resources/docgen/templates/beir-v1.0.0-scifact-wp.template b/src/main/resources/docgen/templates/beir-v1.0.0-scifact-wp.template new file mode 100644 index 0000000000..c01cf3017c --- /dev/null +++ b/src/main/resources/docgen/templates/beir-v1.0.0-scifact-wp.template @@ -0,0 +1,44 @@ +# Anserini Regressions: BEIR (v1.0.0) — SciFact + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — SciFact](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](${yaml}). +Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression ${test_name} +``` + +## Indexing + +Typical indexing command: + +``` +${index_cmds} +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +${ranking_cmds} +``` + +Evaluation can be performed using `trec_eval`: + +``` +${eval_cmds} +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +${effectiveness} diff --git a/src/main/resources/docgen/templates/beir-v1.0.0-trec-covid-wp.template b/src/main/resources/docgen/templates/beir-v1.0.0-trec-covid-wp.template new file mode 100644 index 0000000000..06d13da188 --- /dev/null +++ b/src/main/resources/docgen/templates/beir-v1.0.0-trec-covid-wp.template @@ -0,0 +1,44 @@ +# Anserini Regressions: BEIR (v1.0.0) — TREC-COVID + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — TREC-COVID](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](${yaml}). +Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression ${test_name} +``` + +## Indexing + +Typical indexing command: + +``` +${index_cmds} +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +${ranking_cmds} +``` + +Evaluation can be performed using `trec_eval`: + +``` +${eval_cmds} +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +${effectiveness} diff --git a/src/main/resources/docgen/templates/beir-v1.0.0-webis-touche2020-wp.template b/src/main/resources/docgen/templates/beir-v1.0.0-webis-touche2020-wp.template new file mode 100644 index 0000000000..036332b46e --- /dev/null +++ b/src/main/resources/docgen/templates/beir-v1.0.0-webis-touche2020-wp.template @@ -0,0 +1,44 @@ +# Anserini Regressions: BEIR (v1.0.0) — Webis-Touche2020 + +This page documents BM25 regression experiments for [BEIR (v1.0.0) — Webis-Touche2020](http://beir.ai/). +These experiments index the corpus in a "flat" manner, by concatenating the "title" and "text" into the "contents" field. +All the documents and queries are pre-tokenized with `bert-base-uncased` tokenizer. + +The exact configurations for these regressions are stored in [this YAML file](${yaml}). +Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +From one of our Waterloo servers (e.g., `orca`), the following command will perform the complete regression, end to end: + +``` +python src/main/python/run_regression.py --index --verify --search --regression ${test_name} +``` + +## Indexing + +Typical indexing command: + +``` +${index_cmds} +``` + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +${ranking_cmds} +``` + +Evaluation can be performed using `trec_eval`: + +``` +${eval_cmds} +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +${effectiveness} diff --git a/src/main/resources/regression/beir-v1.0.0-arguana-wp.yaml b/src/main/resources/regression/beir-v1.0.0-arguana-wp.yaml new file mode 100644 index 0000000000..11c5b124cf --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-arguana-wp.yaml @@ -0,0 +1,57 @@ +--- +corpus: beir-v1.0.0-arguana-wp +corpus_path: collections/beir-v1.0.0/wp/arguana/ + +index_path: indexes/lucene-index.beir-v1.0.0-arguana-wp/ +collection_class: BeirFlatCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 1 +index_options: -storePositions -storeDocvectors -storeRaw -pretokenized +index_stats: + documents: 8674 + documents (non-empty): 8674 + total terms: 1793357 + +metrics: + - metric: nDCG@10 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topic_root: src/main/resources/topics-and-qrels/ +qrels_root: src/main/resources/topics-and-qrels/ +topics: + - name: "BEIR (v1.0.0): ArguAna" + id: test + path: topics.beir-v1.0.0-arguana.test.wp.tsv.gz + qrel: qrels.beir-v1.0.0-arguana.test.txt + +models: + - name: bm25 + display: BM25 + params: -bm25 -removeQuery -pretokenized + results: + nDCG@10: + - 0.3639 + R@100: + - 0.8791 + R@1000: + - 0.9602 diff --git a/src/main/resources/regression/beir-v1.0.0-climate-fever-wp.yaml b/src/main/resources/regression/beir-v1.0.0-climate-fever-wp.yaml new file mode 100644 index 0000000000..17699e427c --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-climate-fever-wp.yaml @@ -0,0 +1,57 @@ +--- +corpus: beir-v1.0.0-climate-fever-wp +corpus_path: collections/beir-v1.0.0/wp/climate-fever/ + +index_path: indexes/lucene-index.beir-v1.0.0-climate-fever-wp/ +collection_class: BeirFlatCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 1 +index_options: -storePositions -storeDocvectors -storeRaw -pretokenized +index_stats: + documents: 5416593 + documents (non-empty): 5416593 + total terms: 633232115 + +metrics: + - metric: nDCG@10 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topic_root: src/main/resources/topics-and-qrels/ +qrels_root: src/main/resources/topics-and-qrels/ +topics: + - name: "BEIR (v1.0.0): Climate-FEVER" + id: test + path: topics.beir-v1.0.0-climate-fever.test.wp.tsv.gz + qrel: qrels.beir-v1.0.0-climate-fever.test.txt + +models: + - name: bm25 + display: BM25 + params: -bm25 -removeQuery -pretokenized + results: + nDCG@10: + - 0.1576 + R@100: + - 0.4077 + R@1000: + - 0.5984 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-android-wp.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-android-wp.yaml new file mode 100644 index 0000000000..6eed9dda9e --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-android-wp.yaml @@ -0,0 +1,57 @@ +--- +corpus: beir-v1.0.0-cqadupstack-android-wp +corpus_path: collections/beir-v1.0.0/wp/cqadupstack-android/ + +index_path: indexes/lucene-index.beir-v1.0.0-cqadupstack-android-wp/ +collection_class: BeirFlatCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 1 +index_options: -storePositions -storeDocvectors -storeRaw -pretokenized +index_stats: + documents: 22998 + documents (non-empty): 22998 + total terms: 3430386 + +metrics: + - metric: nDCG@10 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topic_root: src/main/resources/topics-and-qrels/ +qrels_root: src/main/resources/topics-and-qrels/ +topics: + - name: "BEIR (v1.0.0): CQADupStack-android" + id: test + path: topics.beir-v1.0.0-cqadupstack-android.test.wp.tsv.gz + qrel: qrels.beir-v1.0.0-cqadupstack-android.test.txt + +models: + - name: bm25 + display: BM25 + params: -bm25 -removeQuery -pretokenized + results: + nDCG@10: + - 0.3694 + R@100: + - 0.6394 + R@1000: + - 0.8447 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-english-wp.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-english-wp.yaml new file mode 100644 index 0000000000..9a3d2bea74 --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-english-wp.yaml @@ -0,0 +1,57 @@ +--- +corpus: beir-v1.0.0-cqadupstack-english-wp +corpus_path: collections/beir-v1.0.0/wp/cqadupstack-english/ + +index_path: indexes/lucene-index.beir-v1.0.0-cqadupstack-english-wp/ +collection_class: BeirFlatCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 1 +index_options: -storePositions -storeDocvectors -storeRaw -pretokenized +index_stats: + documents: 40221 + documents (non-empty): 40221 + total terms: 4756152 + +metrics: + - metric: nDCG@10 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topic_root: src/main/resources/topics-and-qrels/ +qrels_root: src/main/resources/topics-and-qrels/ +topics: + - name: "BEIR (v1.0.0): CQADupStack-english" + id: test + path: topics.beir-v1.0.0-cqadupstack-english.test.wp.tsv.gz + qrel: qrels.beir-v1.0.0-cqadupstack-english.test.txt + +models: + - name: bm25 + display: BM25 + params: -bm25 -removeQuery -pretokenized + results: + nDCG@10: + - 0.3457 + R@100: + - 0.5544 + R@1000: + - 0.7243 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-gaming-wp.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-gaming-wp.yaml new file mode 100644 index 0000000000..8a80320716 --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-gaming-wp.yaml @@ -0,0 +1,57 @@ +--- +corpus: beir-v1.0.0-cqadupstack-gaming-wp +corpus_path: collections/beir-v1.0.0/wp/cqadupstack-gaming/ + +index_path: indexes/lucene-index.beir-v1.0.0-cqadupstack-gaming-wp/ +collection_class: BeirFlatCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 1 +index_options: -storePositions -storeDocvectors -storeRaw -pretokenized +index_stats: + documents: 45301 + documents (non-empty): 45301 + total terms: 5465343 + +metrics: + - metric: nDCG@10 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topic_root: src/main/resources/topics-and-qrels/ +qrels_root: src/main/resources/topics-and-qrels/ +topics: + - name: "BEIR (v1.0.0): CQADupStack-gaming" + id: test + path: topics.beir-v1.0.0-cqadupstack-gaming.test.wp.tsv.gz + qrel: qrels.beir-v1.0.0-cqadupstack-gaming.test.txt + +models: + - name: bm25 + display: BM25 + params: -bm25 -removeQuery -pretokenized + results: + nDCG@10: + - 0.4701 + R@100: + - 0.7438 + R@1000: + - 0.881 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-gis-wp.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-gis-wp.yaml new file mode 100644 index 0000000000..ea6165c371 --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-gis-wp.yaml @@ -0,0 +1,57 @@ +--- +corpus: beir-v1.0.0-cqadupstack-gis-wp +corpus_path: collections/beir-v1.0.0/wp/cqadupstack-gis/ + +index_path: indexes/lucene-index.beir-v1.0.0-cqadupstack-gis-wp/ +collection_class: BeirFlatCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 1 +index_options: -storePositions -storeDocvectors -storeRaw -pretokenized +index_stats: + documents: 37637 + documents (non-empty): 37637 + total terms: 10372856 + +metrics: + - metric: nDCG@10 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topic_root: src/main/resources/topics-and-qrels/ +qrels_root: src/main/resources/topics-and-qrels/ +topics: + - name: "BEIR (v1.0.0): CQADupStack-gis" + id: test + path: topics.beir-v1.0.0-cqadupstack-gis.test.wp.tsv.gz + qrel: qrels.beir-v1.0.0-cqadupstack-gis.test.txt + +models: + - name: bm25 + display: BM25 + params: -bm25 -removeQuery -pretokenized + results: + nDCG@10: + - 0.2803 + R@100: + - 0.5721 + R@1000: + - 0.7901 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-mathematica-wp.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-mathematica-wp.yaml new file mode 100644 index 0000000000..6b6e03fd4a --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-mathematica-wp.yaml @@ -0,0 +1,57 @@ +--- +corpus: beir-v1.0.0-cqadupstack-mathematica-wp +corpus_path: collections/beir-v1.0.0/wp/cqadupstack-mathematica/ + +index_path: indexes/lucene-index.beir-v1.0.0-cqadupstack-mathematica-wp/ +collection_class: BeirFlatCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 1 +index_options: -storePositions -storeDocvectors -storeRaw -pretokenized +index_stats: + documents: 16705 + documents (non-empty): 16705 + total terms: 6535155 + +metrics: + - metric: nDCG@10 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topic_root: src/main/resources/topics-and-qrels/ +qrels_root: src/main/resources/topics-and-qrels/ +topics: + - name: "BEIR (v1.0.0): CQADupStack-mathematica" + id: test + path: topics.beir-v1.0.0-cqadupstack-mathematica.test.wp.tsv.gz + qrel: qrels.beir-v1.0.0-cqadupstack-mathematica.test.txt + +models: + - name: bm25 + display: BM25 + params: -bm25 -removeQuery -pretokenized + results: + nDCG@10: + - 0.1989 + R@100: + - 0.4979 + R@1000: + - 0.716 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-physics-wp.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-physics-wp.yaml new file mode 100644 index 0000000000..dd78f70c7b --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-physics-wp.yaml @@ -0,0 +1,57 @@ +--- +corpus: beir-v1.0.0-cqadupstack-physics-wp +corpus_path: collections/beir-v1.0.0/wp/cqadupstack-physics/ + +index_path: indexes/lucene-index.beir-v1.0.0-cqadupstack-physics-wp/ +collection_class: BeirFlatCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 1 +index_options: -storePositions -storeDocvectors -storeRaw -pretokenized +index_stats: + documents: 38316 + documents (non-empty): 38316 + total terms: 8408859 + +metrics: + - metric: nDCG@10 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topic_root: src/main/resources/topics-and-qrels/ +qrels_root: src/main/resources/topics-and-qrels/ +topics: + - name: "BEIR (v1.0.0): CQADupStack-physics" + id: test + path: topics.beir-v1.0.0-cqadupstack-physics.test.wp.tsv.gz + qrel: qrels.beir-v1.0.0-cqadupstack-physics.test.txt + +models: + - name: bm25 + display: BM25 + params: -bm25 -removeQuery -pretokenized + results: + nDCG@10: + - 0.3012 + R@100: + - 0.5979 + R@1000: + - 0.7968 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-programmers-wp.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-programmers-wp.yaml new file mode 100644 index 0000000000..9ff65a6fa0 --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-programmers-wp.yaml @@ -0,0 +1,57 @@ +--- +corpus: beir-v1.0.0-cqadupstack-programmers-wp +corpus_path: collections/beir-v1.0.0/wp/cqadupstack-programmers/ + +index_path: indexes/lucene-index.beir-v1.0.0-cqadupstack-programmers-wp/ +collection_class: BeirFlatCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 1 +index_options: -storePositions -storeDocvectors -storeRaw -pretokenized +index_stats: + documents: 32176 + documents (non-empty): 32176 + total terms: 7757786 + +metrics: + - metric: nDCG@10 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topic_root: src/main/resources/topics-and-qrels/ +qrels_root: src/main/resources/topics-and-qrels/ +topics: + - name: "BEIR (v1.0.0): CQADupStack-programmers" + id: test + path: topics.beir-v1.0.0-cqadupstack-programmers.test.wp.tsv.gz + qrel: qrels.beir-v1.0.0-cqadupstack-programmers.test.txt + +models: + - name: bm25 + display: BM25 + params: -bm25 -removeQuery -pretokenized + results: + nDCG@10: + - 0.2750 + R@100: + - 0.5450 + R@1000: + - 0.7565 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-stats-wp.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-stats-wp.yaml new file mode 100644 index 0000000000..3b3c49425d --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-stats-wp.yaml @@ -0,0 +1,57 @@ +--- +corpus: beir-v1.0.0-cqadupstack-stats-wp +corpus_path: collections/beir-v1.0.0/wp/cqadupstack-stats/ + +index_path: indexes/lucene-index.beir-v1.0.0-cqadupstack-stats-wp/ +collection_class: BeirFlatCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 1 +index_options: -storePositions -storeDocvectors -storeRaw -pretokenized +index_stats: + documents: 42269 + documents (non-empty): 42269 + total terms: 12031077 + +metrics: + - metric: nDCG@10 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topic_root: src/main/resources/topics-and-qrels/ +qrels_root: src/main/resources/topics-and-qrels/ +topics: + - name: "BEIR (v1.0.0): CQADupStack-stats" + id: test + path: topics.beir-v1.0.0-cqadupstack-stats.test.wp.tsv.gz + qrel: qrels.beir-v1.0.0-cqadupstack-stats.test.txt + +models: + - name: bm25 + display: BM25 + params: -bm25 -removeQuery -pretokenized + results: + nDCG@10: + - 0.2677 + R@100: + - 0.5225 + R@1000: + - 0.7228 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-tex-wp.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-tex-wp.yaml new file mode 100644 index 0000000000..42140639c2 --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-tex-wp.yaml @@ -0,0 +1,57 @@ +--- +corpus: beir-v1.0.0-cqadupstack-tex-wp +corpus_path: collections/beir-v1.0.0/wp/cqadupstack-tex/ + +index_path: indexes/lucene-index.beir-v1.0.0-cqadupstack-tex-wp/ +collection_class: BeirFlatCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 1 +index_options: -storePositions -storeDocvectors -storeRaw -pretokenized +index_stats: + documents: 68184 + documents (non-empty): 68184 + total terms: 27136402 + +metrics: + - metric: nDCG@10 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topic_root: src/main/resources/topics-and-qrels/ +qrels_root: src/main/resources/topics-and-qrels/ +topics: + - name: "BEIR (v1.0.0): CQADupStack-tex" + id: test + path: topics.beir-v1.0.0-cqadupstack-tex.test.wp.tsv.gz + qrel: qrels.beir-v1.0.0-cqadupstack-tex.test.txt + +models: + - name: bm25 + display: BM25 + params: -bm25 -removeQuery -pretokenized + results: + nDCG@10: + - 0.2173 + R@100: + - 0.4475 + R@1000: + - 0.6599 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-unix-wp.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-unix-wp.yaml new file mode 100644 index 0000000000..3e1fb953e7 --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-unix-wp.yaml @@ -0,0 +1,57 @@ +--- +corpus: beir-v1.0.0-cqadupstack-unix-wp +corpus_path: collections/beir-v1.0.0/wp/cqadupstack-unix/ + +index_path: indexes/lucene-index.beir-v1.0.0-cqadupstack-unix-wp/ +collection_class: BeirFlatCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 1 +index_options: -storePositions -storeDocvectors -storeRaw -pretokenized +index_stats: + documents: 47382 + documents (non-empty): 47382 + total terms: 13814118 + +metrics: + - metric: nDCG@10 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topic_root: src/main/resources/topics-and-qrels/ +qrels_root: src/main/resources/topics-and-qrels/ +topics: + - name: "BEIR (v1.0.0): CQADupStack-unix" + id: test + path: topics.beir-v1.0.0-cqadupstack-unix.test.wp.tsv.gz + qrel: qrels.beir-v1.0.0-cqadupstack-unix.test.txt + +models: + - name: bm25 + display: BM25 + params: -bm25 -removeQuery -pretokenized + results: + nDCG@10: + - 0.2668 + R@100: + - 0.4932 + R@1000: + - 0.7347 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-webmasters-wp.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-webmasters-wp.yaml new file mode 100644 index 0000000000..d8af6f37e2 --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-webmasters-wp.yaml @@ -0,0 +1,57 @@ +--- +corpus: beir-v1.0.0-cqadupstack-webmasters-wp +corpus_path: collections/beir-v1.0.0/wp/cqadupstack-webmasters/ + +index_path: indexes/lucene-index.beir-v1.0.0-cqadupstack-webmasters-wp/ +collection_class: BeirFlatCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 1 +index_options: -storePositions -storeDocvectors -storeRaw -pretokenized +index_stats: + documents: 17405 + documents (non-empty): 17405 + total terms: 3129289 + +metrics: + - metric: nDCG@10 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topic_root: src/main/resources/topics-and-qrels/ +qrels_root: src/main/resources/topics-and-qrels/ +topics: + - name: "BEIR (v1.0.0): CQADupStack-webmasters" + id: test + path: topics.beir-v1.0.0-cqadupstack-webmasters.test.wp.tsv.gz + qrel: qrels.beir-v1.0.0-cqadupstack-webmasters.test.txt + +models: + - name: bm25 + display: BM25 + params: -bm25 -removeQuery -pretokenized + results: + nDCG@10: + - 0.3012 + R@100: + - 0.5627 + R@1000: + - 0.7793 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-wordpress-wp.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-wordpress-wp.yaml new file mode 100644 index 0000000000..f6712ca1bd --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-wordpress-wp.yaml @@ -0,0 +1,57 @@ +--- +corpus: beir-v1.0.0-cqadupstack-wordpress-wp +corpus_path: collections/beir-v1.0.0/wp/cqadupstack-wordpress/ + +index_path: indexes/lucene-index.beir-v1.0.0-cqadupstack-wordpress-wp/ +collection_class: BeirFlatCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 1 +index_options: -storePositions -storeDocvectors -storeRaw -pretokenized +index_stats: + documents: 48605 + documents (non-empty): 48605 + total terms: 15350711 + +metrics: + - metric: nDCG@10 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topic_root: src/main/resources/topics-and-qrels/ +qrels_root: src/main/resources/topics-and-qrels/ +topics: + - name: "BEIR (v1.0.0): CQADupStack-wordpress" + id: test + path: topics.beir-v1.0.0-cqadupstack-wordpress.test.wp.tsv.gz + qrel: qrels.beir-v1.0.0-cqadupstack-wordpress.test.txt + +models: + - name: bm25 + display: BM25 + params: -bm25 -removeQuery -pretokenized + results: + nDCG@10: + - 0.2415 + R@100: + - 0.4985 + R@1000: + - 0.7193 diff --git a/src/main/resources/regression/beir-v1.0.0-dbpedia-entity-wp.yaml b/src/main/resources/regression/beir-v1.0.0-dbpedia-entity-wp.yaml new file mode 100644 index 0000000000..39e7f35faf --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-dbpedia-entity-wp.yaml @@ -0,0 +1,57 @@ +--- +corpus: beir-v1.0.0-dbpedia-entity-wp +corpus_path: collections/beir-v1.0.0/wp/dbpedia-entity/ + +index_path: indexes/lucene-index.beir-v1.0.0-dbpedia-entity-wp/ +collection_class: BeirFlatCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 1 +index_options: -storePositions -storeDocvectors -storeRaw -pretokenized +index_stats: + documents: 4635922 + documents (non-empty): 4635922 + total terms: 328666200 + +metrics: + - metric: nDCG@10 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topic_root: src/main/resources/topics-and-qrels/ +qrels_root: src/main/resources/topics-and-qrels/ +topics: + - name: "BEIR (v1.0.0): DBPedia" + id: test + path: topics.beir-v1.0.0-dbpedia-entity.test.wp.tsv.gz + qrel: qrels.beir-v1.0.0-dbpedia-entity.test.txt + +models: + - name: bm25 + display: BM25 + params: -bm25 -removeQuery -pretokenized + results: + nDCG@10: + - 0.2841 + R@100: + - 0.4255 + R@1000: + - 0.6124 diff --git a/src/main/resources/regression/beir-v1.0.0-fever-wp.yaml b/src/main/resources/regression/beir-v1.0.0-fever-wp.yaml new file mode 100644 index 0000000000..477e4d2252 --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-fever-wp.yaml @@ -0,0 +1,57 @@ +--- +corpus: beir-v1.0.0-fever-wp +corpus_path: collections/beir-v1.0.0/wp/fever/ + +index_path: indexes/lucene-index.beir-v1.0.0-fever-wp/ +collection_class: BeirFlatCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 1 +index_options: -storePositions -storeDocvectors -storeRaw -pretokenized +index_stats: + documents: 5416568 + documents (non-empty): 5416568 + total terms: 633222046 + +metrics: + - metric: nDCG@10 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topic_root: src/main/resources/topics-and-qrels/ +qrels_root: src/main/resources/topics-and-qrels/ +topics: + - name: "BEIR (v1.0.0): FEVER" + id: test + path: topics.beir-v1.0.0-fever.test.wp.tsv.gz + qrel: qrels.beir-v1.0.0-fever.test.txt + +models: + - name: bm25 + display: BM25 + params: -bm25 -removeQuery -pretokenized + results: + nDCG@10: + - 0.6583 + R@100: + - 0.9212 + R@1000: + - 0.9622 diff --git a/src/main/resources/regression/beir-v1.0.0-fiqa-wp.yaml b/src/main/resources/regression/beir-v1.0.0-fiqa-wp.yaml new file mode 100644 index 0000000000..de7dde0c5d --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-fiqa-wp.yaml @@ -0,0 +1,57 @@ +--- +corpus: beir-v1.0.0-fiqa-wp +corpus_path: collections/beir-v1.0.0/wp/fiqa/ + +index_path: indexes/lucene-index.beir-v1.0.0-fiqa-wp/ +collection_class: BeirFlatCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 1 +index_options: -storePositions -storeDocvectors -storeRaw -pretokenized +index_stats: + documents: 57600 + documents (non-empty): 57600 + total terms: 10093406 + +metrics: + - metric: nDCG@10 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topic_root: src/main/resources/topics-and-qrels/ +qrels_root: src/main/resources/topics-and-qrels/ +topics: + - name: "BEIR (v1.0.0): FiQA-2018" + id: test + path: topics.beir-v1.0.0-fiqa.test.wp.tsv.gz + qrel: qrels.beir-v1.0.0-fiqa.test.txt + +models: + - name: bm25 + display: BM25 + params: -bm25 -removeQuery -pretokenized + results: + nDCG@10: + - 0.2179 + R@100: + - 0.4825 + R@1000: + - 0.7081 diff --git a/src/main/resources/regression/beir-v1.0.0-hotpotqa-wp.yaml b/src/main/resources/regression/beir-v1.0.0-hotpotqa-wp.yaml new file mode 100644 index 0000000000..ae81dd1d6d --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-hotpotqa-wp.yaml @@ -0,0 +1,57 @@ +--- +corpus: beir-v1.0.0-hotpotqa-wp +corpus_path: collections/beir-v1.0.0/wp/hotpotqa/ + +index_path: indexes/lucene-index.beir-v1.0.0-hotpotqa-wp/ +collection_class: BeirFlatCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 1 +index_options: -storePositions -storeDocvectors -storeRaw -pretokenized +index_stats: + documents: 5233329 + documents (non-empty): 5233329 + total terms: 343599707 + +metrics: + - metric: nDCG@10 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topic_root: src/main/resources/topics-and-qrels/ +qrels_root: src/main/resources/topics-and-qrels/ +topics: + - name: "BEIR (v1.0.0): HotpotQA" + id: test + path: topics.beir-v1.0.0-hotpotqa.test.wp.tsv.gz + qrel: qrels.beir-v1.0.0-hotpotqa.test.txt + +models: + - name: bm25 + display: BM25 + params: -bm25 -removeQuery -pretokenized + results: + nDCG@10: + - 0.5926 + R@100: + - 0.7612 + R@1000: + - 0.8600 diff --git a/src/main/resources/regression/beir-v1.0.0-nfcorpus-wp.yaml b/src/main/resources/regression/beir-v1.0.0-nfcorpus-wp.yaml new file mode 100644 index 0000000000..2803bd2af0 --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-nfcorpus-wp.yaml @@ -0,0 +1,57 @@ +--- +corpus: beir-v1.0.0-nfcorpus-wp +corpus_path: collections/beir-v1.0.0/wp/nfcorpus/ + +index_path: indexes/lucene-index.beir-v1.0.0-nfcorpus-wp/ +collection_class: BeirFlatCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 1 +index_options: -storePositions -storeDocvectors -storeRaw -pretokenized +index_stats: + documents: 3633 + documents (non-empty): 3633 + total terms: 1291082 + +metrics: + - metric: nDCG@10 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topic_root: src/main/resources/topics-and-qrels/ +qrels_root: src/main/resources/topics-and-qrels/ +topics: + - name: "BEIR (v1.0.0): NFCorpus" + id: test + path: topics.beir-v1.0.0-nfcorpus.test.wp.tsv.gz + qrel: qrels.beir-v1.0.0-nfcorpus.test.txt + +models: + - name: bm25 + display: BM25 + params: -bm25 -removeQuery -pretokenized + results: + nDCG@10: + - 0.3142 + R@100: + - 0.2441 + R@1000: + - 0.4222 diff --git a/src/main/resources/regression/beir-v1.0.0-nq-wp.yaml b/src/main/resources/regression/beir-v1.0.0-nq-wp.yaml new file mode 100644 index 0000000000..c2b6132d1d --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-nq-wp.yaml @@ -0,0 +1,57 @@ +--- +corpus: beir-v1.0.0-nq-wp +corpus_path: collections/beir-v1.0.0/wp/nq/ + +index_path: indexes/lucene-index.beir-v1.0.0-nq-wp/ +collection_class: BeirFlatCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 1 +index_options: -storePositions -storeDocvectors -storeRaw -pretokenized +index_stats: + documents: 2681468 + documents (non-empty): 2681468 + total terms: 287547771 + +metrics: + - metric: nDCG@10 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topic_root: src/main/resources/topics-and-qrels/ +qrels_root: src/main/resources/topics-and-qrels/ +topics: + - name: "BEIR (v1.0.0): NQ" + id: test + path: topics.beir-v1.0.0-nq.test.wp.tsv.gz + qrel: qrels.beir-v1.0.0-nq.test.txt + +models: + - name: bm25 + display: BM25 + params: -bm25 -removeQuery -pretokenized + results: + nDCG@10: + - 0.3046 + R@100: + - 0.7390 + R@1000: + - 0.8917 diff --git a/src/main/resources/regression/beir-v1.0.0-quora-wp.yaml b/src/main/resources/regression/beir-v1.0.0-quora-wp.yaml new file mode 100644 index 0000000000..8ff529d24d --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-quora-wp.yaml @@ -0,0 +1,57 @@ +--- +corpus: beir-v1.0.0-quora-wp +corpus_path: collections/beir-v1.0.0/wp/quora/ + +index_path: indexes/lucene-index.beir-v1.0.0-quora-wp/ +collection_class: BeirFlatCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 1 +index_options: -storePositions -storeDocvectors -storeRaw -pretokenized +index_stats: + documents: 522931 + documents (non-empty): 522931 + total terms: 7466732 + +metrics: + - metric: nDCG@10 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topic_root: src/main/resources/topics-and-qrels/ +qrels_root: src/main/resources/topics-and-qrels/ +topics: + - name: "BEIR (v1.0.0): Quora" + id: test + path: topics.beir-v1.0.0-quora.test.wp.tsv.gz + qrel: qrels.beir-v1.0.0-quora.test.txt + +models: + - name: bm25 + display: BM25 + params: -bm25 -removeQuery -pretokenized + results: + nDCG@10: + - 0.7295 + R@100: + - 0.9459 + R@1000: + - 0.9872 diff --git a/src/main/resources/regression/beir-v1.0.0-scidocs-wp.yaml b/src/main/resources/regression/beir-v1.0.0-scidocs-wp.yaml new file mode 100644 index 0000000000..997b1a1cc8 --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-scidocs-wp.yaml @@ -0,0 +1,57 @@ +--- +corpus: beir-v1.0.0-scidocs-wp +corpus_path: collections/beir-v1.0.0/wp/scidocs/ + +index_path: indexes/lucene-index.beir-v1.0.0-scidocs-wp/ +collection_class: BeirFlatCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 1 +index_options: -storePositions -storeDocvectors -storeRaw -pretokenized +index_stats: + documents: 25657 + documents (non-empty): 25657 + total terms: 6017268 + +metrics: + - metric: nDCG@10 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topic_root: src/main/resources/topics-and-qrels/ +qrels_root: src/main/resources/topics-and-qrels/ +topics: + - name: "BEIR (v1.0.0): SCIDOCS" + id: test + path: topics.beir-v1.0.0-scidocs.test.wp.tsv.gz + qrel: qrels.beir-v1.0.0-scidocs.test.txt + +models: + - name: bm25 + display: BM25 + params: -bm25 -removeQuery -pretokenized + results: + nDCG@10: + - 0.1380 + R@100: + - 0.3325 + R@1000: + - 0.5479 diff --git a/src/main/resources/regression/beir-v1.0.0-scifact-wp.yaml b/src/main/resources/regression/beir-v1.0.0-scifact-wp.yaml new file mode 100644 index 0000000000..656f4ae20a --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-scifact-wp.yaml @@ -0,0 +1,57 @@ +--- +corpus: beir-v1.0.0-scifact-wp +corpus_path: collections/beir-v1.0.0/wp/scifact/ + +index_path: indexes/lucene-index.beir-v1.0.0-scifact-wp/ +collection_class: BeirFlatCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 1 +index_options: -storePositions -storeDocvectors -storeRaw -pretokenized +index_stats: + documents: 5183 + documents (non-empty): 5183 + total terms: 1737313 + +metrics: + - metric: nDCG@10 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topic_root: src/main/resources/topics-and-qrels/ +qrels_root: src/main/resources/topics-and-qrels/ +topics: + - name: "BEIR (v1.0.0): SciFact" + id: test + path: topics.beir-v1.0.0-scifact.test.wp.tsv.gz + qrel: qrels.beir-v1.0.0-scifact.test.txt + +models: + - name: bm25 + display: BM25 + params: -bm25 -removeQuery -pretokenized + results: + nDCG@10: + - 0.6718 + R@100: + - 0.9213 + R@1000: + - 0.9743 diff --git a/src/main/resources/regression/beir-v1.0.0-trec-covid-wp.yaml b/src/main/resources/regression/beir-v1.0.0-trec-covid-wp.yaml new file mode 100644 index 0000000000..d6af17809a --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-trec-covid-wp.yaml @@ -0,0 +1,57 @@ +--- +corpus: beir-v1.0.0-trec-covid-wp +corpus_path: collections/beir-v1.0.0/wp/trec-covid/ + +index_path: indexes/lucene-index.beir-v1.0.0-trec-covid-wp/ +collection_class: BeirFlatCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 1 +index_options: -storePositions -storeDocvectors -storeRaw -pretokenized +index_stats: + documents: 171331 + documents (non-empty): 171331 + total terms: 41714921 + +metrics: + - metric: nDCG@10 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topic_root: src/main/resources/topics-and-qrels/ +qrels_root: src/main/resources/topics-and-qrels/ +topics: + - name: "BEIR (v1.0.0): TREC-COVID" + id: test + path: topics.beir-v1.0.0-trec-covid.test.wp.tsv.gz + qrel: qrels.beir-v1.0.0-trec-covid.test.txt + +models: + - name: bm25 + display: BM25 + params: -bm25 -removeQuery -pretokenized + results: + nDCG@10: + - 0.5645 + R@100: + - 0.1001 + R@1000: + - 0.3621 diff --git a/src/main/resources/regression/beir-v1.0.0-webis-touche2020-wp.yaml b/src/main/resources/regression/beir-v1.0.0-webis-touche2020-wp.yaml new file mode 100644 index 0000000000..193a5ec784 --- /dev/null +++ b/src/main/resources/regression/beir-v1.0.0-webis-touche2020-wp.yaml @@ -0,0 +1,57 @@ +--- +corpus: beir-v1.0.0-webis-touche2020-wp +corpus_path: collections/beir-v1.0.0/wp/webis-touche2020/ + +index_path: indexes/lucene-index.beir-v1.0.0-webis-touche2020-wp/ +collection_class: BeirFlatCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 1 +index_options: -storePositions -storeDocvectors -storeRaw -pretokenized +index_stats: + documents: 382545 + documents (non-empty): 382545 + total terms: 146696393 + +metrics: + - metric: nDCG@10 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvString +topic_root: src/main/resources/topics-and-qrels/ +qrels_root: src/main/resources/topics-and-qrels/ +topics: + - name: "BEIR (v1.0.0): Webis-Touche2020" + id: test + path: topics.beir-v1.0.0-webis-touche2020.test.wp.tsv.gz + qrel: qrels.beir-v1.0.0-webis-touche2020.test.txt + +models: + - name: bm25 + display: BM25 + params: -bm25 -removeQuery -pretokenized + results: + nDCG@10: + - 0.4661 + R@100: + - 0.5702 + R@1000: + - 0.8415