From 54c89f490a145cceb8ef9d2b45dba9d6baa1fa9f Mon Sep 17 00:00:00 2001 From: lintool Date: Sat, 15 Jan 2022 13:44:50 -0500 Subject: [PATCH 01/10] Add V2 d2q-t5 --- .../regression/msmarco-v2-doc-d2q-t5.yaml | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 src/main/resources/regression/msmarco-v2-doc-d2q-t5.yaml diff --git a/src/main/resources/regression/msmarco-v2-doc-d2q-t5.yaml b/src/main/resources/regression/msmarco-v2-doc-d2q-t5.yaml new file mode 100644 index 0000000000..e9a9963de5 --- /dev/null +++ b/src/main/resources/regression/msmarco-v2-doc-d2q-t5.yaml @@ -0,0 +1,74 @@ +--- +corpus: msmarco-v2-doc-d2q-t5 +corpus_path: collections/msmarco/msmarco_v2_doc_d2q-t5/ + +index_path: indexes/lucene-index.msmarco-v2-doc-d2q-t5/ +collection_class: MsMarcoV2DocCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 24 +index_options: -storePositions -storeDocvectors -storeRaw +index_stats: + documents: 11959635 + documents (non-empty): 11959635 + total terms: 14165661202 + +metrics: + - metric: MAP@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -M 100 -m map + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: true + - metric: MRR@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -M 100 -m recip_rank + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: true + - metric: R@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvInt +topic_root: src/main/resources/topics-and-qrels/ +qrels_root: src/main/resources/topics-and-qrels/ +topics: + - name: "[MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)" + id: dev + path: topics.msmarco-v2-doc.dev.txt + qrel: qrels.msmarco-v2-doc.dev.txt + - name: "[MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)" + id: dev2 + path: topics.msmarco-v2-doc.dev2.txt + qrel: qrels.msmarco-v2-doc.dev2.txt + +models: + - name: bm25-default + display: BM25 (default) + params: -bm25 + results: + MAP@100: + - 0.1552 + - 0.1639 + MRR@100: + - 0.1572 + - 0.1659 + R@100: + - 0.5956 + - 0.5970 + R@1000: + - 0.8054 + - 0.8029 From 24489a7da791eeffcac8c0a6588dbc9687dfea84 Mon Sep 17 00:00:00 2001 From: lintool Date: Sat, 15 Jan 2022 14:52:05 -0500 Subject: [PATCH 02/10] update --- docs/regressions.md | 3 + .../msmarco-v2-doc-segmented-d2q-t5.yaml | 74 +++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 src/main/resources/regression/msmarco-v2-doc-segmented-d2q-t5.yaml diff --git a/docs/regressions.md b/docs/regressions.md index 9b3070d3cf..f4de720651 100644 --- a/docs/regressions.md +++ b/docs/regressions.md @@ -84,6 +84,9 @@ nohup python src/main/python/run_regression.py --index --verify --search --regre nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-v2-passage >& logs/log.msmarco-v2-passage & nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-v2-passage-augmented >& logs/log.msmarco-v2-passage-augmented & +nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-v2-doc-d2q-t5 >& logs/log.msmarco-v2-doc-d2q-t5 & +nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-v2-doc-segmented-d2q-t5 >& logs/log.msmarco-v2-doc-segmented-d2q-t5 & + nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-v2-passage-unicoil-noexp-0shot >& logs/log.msmarco-v2-passage-unicoil-noexp-0shot & nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-v2-doc-segmented-unicoil-noexp-0shot >& logs/log.msmarco-v2-doc-segmented-unicoil-noexp-0shot & diff --git a/src/main/resources/regression/msmarco-v2-doc-segmented-d2q-t5.yaml b/src/main/resources/regression/msmarco-v2-doc-segmented-d2q-t5.yaml new file mode 100644 index 0000000000..c59f7413a9 --- /dev/null +++ b/src/main/resources/regression/msmarco-v2-doc-segmented-d2q-t5.yaml @@ -0,0 +1,74 @@ +--- +corpus: msmarco-v2-doc-segmented-d2q-t5 +corpus_path: collections/msmarco/msmarco_v2_doc_segmented_d2q-t5/ + +index_path: indexes/lucene-index.msmarco-v2-doc-segmented-d2q-t5/ +collection_class: MsMarcoV2DocCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 24 +index_options: -storePositions -storeDocvectors -storeRaw +index_stats: + documents: 124131414 + documents (non-empty): 124131414 + total terms: 24780915974 + +metrics: + - metric: MAP@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -M 100 -m map + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: true + - metric: MRR@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -M 100 -m recip_rank + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: true + - metric: R@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvInt +topic_root: src/main/resources/topics-and-qrels/ +qrels_root: src/main/resources/topics-and-qrels/ +topics: + - name: "[MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)" + id: dev + path: topics.msmarco-v2-doc.dev.txt + qrel: qrels.msmarco-v2-doc.dev.txt + - name: "[MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)" + id: dev2 + path: topics.msmarco-v2-doc.dev2.txt + qrel: qrels.msmarco-v2-doc.dev2.txt + +models: + - name: bm25-default + display: BM25 (default) + params: -bm25 -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 + results: + MAP@100: + - 0.1875 + - 0.1903 + MRR@100: + - 0.1896 + - 0.1930 + R@100: + - 0.6555 + - 0.6629 + R@1000: + - 0.8542 + - 0.8549 From 37bbf159c9a056935974ee3df42f6f4bd19d40e0 Mon Sep 17 00:00:00 2001 From: lintool Date: Sat, 15 Jan 2022 17:20:51 -0500 Subject: [PATCH 03/10] Fixed regression numbers --- .../regression/msmarco-v2-doc-d2q-t5.yaml | 18 +++++++++--------- .../msmarco-v2-doc-segmented-d2q-t5.yaml | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/main/resources/regression/msmarco-v2-doc-d2q-t5.yaml b/src/main/resources/regression/msmarco-v2-doc-d2q-t5.yaml index e9a9963de5..ed70560f1a 100644 --- a/src/main/resources/regression/msmarco-v2-doc-d2q-t5.yaml +++ b/src/main/resources/regression/msmarco-v2-doc-d2q-t5.yaml @@ -10,7 +10,7 @@ index_options: -storePositions -storeDocvectors -storeRaw index_stats: documents: 11959635 documents (non-empty): 11959635 - total terms: 14165661202 + total terms: 19760777295 metrics: - metric: MAP@100 @@ -61,14 +61,14 @@ models: params: -bm25 results: MAP@100: - - 0.1552 - - 0.1639 + - 0.1988 + - 0.1986 MRR@100: - - 0.1572 - - 0.1659 + - 0.2011 + - 0.2012 R@100: - - 0.5956 - - 0.5970 + - 0.6786 + - 0.6821 R@1000: - - 0.8054 - - 0.8029 + - 0.8614 + - 0.8568 diff --git a/src/main/resources/regression/msmarco-v2-doc-segmented-d2q-t5.yaml b/src/main/resources/regression/msmarco-v2-doc-segmented-d2q-t5.yaml index c59f7413a9..cb858b3776 100644 --- a/src/main/resources/regression/msmarco-v2-doc-segmented-d2q-t5.yaml +++ b/src/main/resources/regression/msmarco-v2-doc-segmented-d2q-t5.yaml @@ -10,7 +10,7 @@ index_options: -storePositions -storeDocvectors -storeRaw index_stats: documents: 124131414 documents (non-empty): 124131414 - total terms: 24780915974 + total terms: 30376032067 metrics: - metric: MAP@100 @@ -61,14 +61,14 @@ models: params: -bm25 -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 results: MAP@100: - - 0.1875 - - 0.1903 + - 0.2203 + - 0.2205 MRR@100: - - 0.1896 - - 0.1930 + - 0.2226 + - 0.2234 R@100: - - 0.6555 - - 0.6629 + - 0.7297 + - 0.7316 R@1000: - - 0.8542 - - 0.8549 + - 0.8982 + - 0.8952 From c42ce8d2a97dcc4fabbe6865b28136a0ad9b9d12 Mon Sep 17 00:00:00 2001 From: lintool Date: Sun, 16 Jan 2022 11:31:05 -0500 Subject: [PATCH 04/10] fixed numbers --- .../resources/regression/dl21-doc-d2q-t5.yaml | 89 +++++++++++++++++++ .../regression/dl21-doc-segmented-d2q-t5.yaml | 89 +++++++++++++++++++ 2 files changed, 178 insertions(+) create mode 100644 src/main/resources/regression/dl21-doc-d2q-t5.yaml create mode 100644 src/main/resources/regression/dl21-doc-segmented-d2q-t5.yaml diff --git a/src/main/resources/regression/dl21-doc-d2q-t5.yaml b/src/main/resources/regression/dl21-doc-d2q-t5.yaml new file mode 100644 index 0000000000..6bb1bbeba5 --- /dev/null +++ b/src/main/resources/regression/dl21-doc-d2q-t5.yaml @@ -0,0 +1,89 @@ +--- +corpus: msmarco-v2-doc-d2q-t5 +corpus_path: collections/msmarco/msmarco_v2_doc_d2q-t5/ + +index_path: indexes/lucene-index.msmarco-v2-doc-d2q-t5/ +collection_class: MsMarcoV2DocCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 24 +index_options: -storePositions -storeDocvectors -storeRaw +index_stats: + documents: 11959635 + documents (non-empty): 11959635 + total terms: 19760777295 + +metrics: + - metric: MAP@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -M 100 -m map + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: no + - metric: MRR@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -M 100 -m recip_rank + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: true + - metric: nDCG@10 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: true + - metric: R@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvInt +topic_root: src/main/resources/topics-and-qrels/ +qrels_root: src/main/resources/topics-and-qrels/ +topics: + - name: "[DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning)" + id: dl21 + path: topics.dl21.txt + qrel: qrels.dl21-doc.txt + +models: + - name: bm25-default + display: BM25 (default) + params: -hits 1000 -bm25 + results: + MAP@100: + - 0.2387 + MRR@100: + - 0.8866 + nDCG@10: + - 0.5792 + R@100: + - 0.3443 + R@1000: + - 0.7066 + - name: bm25-default+rm3 + display: +RM3 + params: -hits 1000 -bm25 -rm3 + results: + MAP@100: + - 0.2608 + MRR@100: + - 0.8342 + nDCG@10: + - 0.5392 + R@100: + - 0.3580 + R@1000: + - 0.7572 diff --git a/src/main/resources/regression/dl21-doc-segmented-d2q-t5.yaml b/src/main/resources/regression/dl21-doc-segmented-d2q-t5.yaml new file mode 100644 index 0000000000..372640716f --- /dev/null +++ b/src/main/resources/regression/dl21-doc-segmented-d2q-t5.yaml @@ -0,0 +1,89 @@ +--- +corpus: msmarco-v2-doc-segmented-d2q-t5 +corpus_path: collections/msmarco/msmarco_v2_doc_segmented_d2q-t5/ + +index_path: indexes/lucene-index.msmarco-v2-doc-segmented-d2q-t5/ +collection_class: MsMarcoV2DocCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 24 +index_options: -storePositions -storeDocvectors -storeRaw +index_stats: + documents: 124131414 + documents (non-empty): 124131414 + total terms: 30376032067 + +metrics: + - metric: MAP@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -M 100 -m map + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: no + - metric: MRR@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -M 100 -m recip_rank + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: true + - metric: nDCG@10 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: true + - metric: R@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvInt +topic_root: src/main/resources/topics-and-qrels/ +qrels_root: src/main/resources/topics-and-qrels/ +topics: + - name: "[DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning)" + id: dl21 + path: topics.dl21.txt + qrel: qrels.dl21-doc.txt + +models: + - name: bm25-default + display: BM25 (default) + params: -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 -bm25 + results: + MAP@100: + - 0.2683 + MRR@100: + - 0.9454 + nDCG@10: + - 0.6289 + R@100: + - 0.3656 + R@1000: + - 0.7202 + - name: bm25-default+rm3 + display: +RM3 + params: -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 -bm25 -rm3 + results: + MAP@100: + - 0.3192 + MRR@100: + - 0.8960 + nDCG@10: + - 0.6555 + R@100: + - 0.4119 + R@1000: + - 0.7941 From 7463a6df5ef0d3edaf9acc447afa9a3440c135ee Mon Sep 17 00:00:00 2001 From: lintool Date: Sun, 16 Jan 2022 11:43:19 -0500 Subject: [PATCH 05/10] Updated docs. --- README.md | 10 +- docs/regressions-dl21-doc-d2q-t5.md | 106 ++++++++++++++++++ docs/regressions-dl21-doc-segmented-d2q-t5.md | 106 ++++++++++++++++++ docs/regressions-msmarco-v2-doc-d2q-t5.md | 84 ++++++++++++++ ...essions-msmarco-v2-doc-segmented-d2q-t5.md | 84 ++++++++++++++ docs/regressions.md | 18 +-- .../docgen/templates/dl21-doc-d2q-t5.template | 60 ++++++++++ .../dl21-doc-segmented-d2q-t5.template | 60 ++++++++++ .../templates/msmarco-v2-doc-d2q-t5.template | 43 +++++++ .../msmarco-v2-doc-segmented-d2q-t5.template | 43 +++++++ 10 files changed, 602 insertions(+), 12 deletions(-) create mode 100644 docs/regressions-dl21-doc-d2q-t5.md create mode 100644 docs/regressions-dl21-doc-segmented-d2q-t5.md create mode 100644 docs/regressions-msmarco-v2-doc-d2q-t5.md create mode 100644 docs/regressions-msmarco-v2-doc-segmented-d2q-t5.md create mode 100644 src/main/resources/docgen/templates/dl21-doc-d2q-t5.template create mode 100644 src/main/resources/docgen/templates/dl21-doc-segmented-d2q-t5.template create mode 100644 src/main/resources/docgen/templates/msmarco-v2-doc-d2q-t5.template create mode 100644 src/main/resources/docgen/templates/msmarco-v2-doc-segmented-d2q-t5.template diff --git a/README.md b/README.md index 2c2e9342cf..729f17dde5 100644 --- a/README.md +++ b/README.md @@ -65,14 +65,18 @@ For the most part, these runs are based on [_default_ parameter settings](https: + Document ranking, complete doc[*](docs/experiments-msmarco-doc-doc2query-details.md): [baselines](docs/regressions-dl20-doc.md), [doc2query-T5](docs/regressions-dl20-doc-docTTTTTquery.md) + Document ranking, segmented doc[*](docs/experiments-msmarco-doc-doc2query-details.md): [baselines](docs/regressions-dl20-doc-segmented.md), [doc2query-T5](docs/regressions-dl20-doc-segmented-docTTTTTquery.md) + Regressions for MS MARCO (V2) Passage Ranking: - + Bag-of-words models: [baselines](docs/regressions-msmarco-v2-passage.md), [on augmented corpus](docs/regressions-msmarco-v2-passage-augmented.md) + + Bag-of-words models, default corpus: [baselines](docs/regressions-msmarco-v2-passage.md) + + Bag-of-words models, augmented corpus: [baselines](docs/regressions-msmarco-v2-passage-augmented.md) + Sparse learned models: [uniCOIL noexp zero-shot](docs/regressions-msmarco-v2-passage-unicoil-noexp-0shot.md) + Regressions for MS MARCO (V2) Document Ranking: - + Bag-of-words models: [baselines](docs/regressions-msmarco-v2-doc.md), [on segmented corpus](docs/regressions-msmarco-v2-doc-segmented.md) + + Bag-of-words, complete doc: [baselines](docs/regressions-msmarco-v2-doc.md), [doc2query-T5](docs/regressions-msmarco-v2-doc-d2q-t5.md) + + Bag-of-words, segmented doc: [baselines](docs/regressions-msmarco-v2-doc-segmented.md), [doc2query-T5](docs/regressions-msmarco-v2-doc-segmented-d2q-t5.md) + Sparse learned models: [uniCOIL noexp zero-shot](docs/regressions-msmarco-v2-doc-segmented-unicoil-noexp-0shot.md) + Regressions for TREC 2021 Deep Learning Track: + Passage ranking: [baselines](docs/regressions-dl21-passage.md), [on augmented corpus](docs/regressions-dl21-passage-augmented.md), [uniCOIL noexp zero-shot](docs/regressions-dl21-passage-unicoil-noexp-0shot.md) - + Document ranking: [baselines](docs/regressions-dl21-doc.md), [on segmented corpus](docs/regressions-dl21-doc-segmented.md), [uniCOIL noexp zero-shot](docs/regressions-dl21-doc-segmented-unicoil-noexp-0shot.md) + + Document ranking, complete doc: [baselines](docs/regressions-dl21-doc.md), [doc2query-T5](docs/regressions-dl21-doc-d2q-t5.md) + + Document ranking, segmented doc: [baselines](docs/regressions-dl21-doc-segmented.md), [doc2query-T5](docs/regressions-dl21-doc-segmented-d2q-t5.md) + + Document ranking, sparse learned models: [uniCOIL noexp zero-shot](docs/regressions-dl21-doc-segmented-unicoil-noexp-0shot.md) + Regressions for TREC News Tracks (Background Linking Task): [2018](docs/regressions-backgroundlinking18.md), [2019](docs/regressions-backgroundlinking19.md), [2020](docs/regressions-backgroundlinking20.md) + Regressions for [FEVER Fact Verification](docs/regressions-fever.md) + Regressions for [NTCIR-8 ACLIA (IR4QA subtask, Monolingual Chinese)](docs/regressions-ntcir8-zh.md) diff --git a/docs/regressions-dl21-doc-d2q-t5.md b/docs/regressions-dl21-doc-d2q-t5.md new file mode 100644 index 0000000000..a5e2ac4558 --- /dev/null +++ b/docs/regressions-dl21-doc-d2q-t5.md @@ -0,0 +1,106 @@ +# Anserini: Regressions for [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) + +This page describes experiments, integrated into Anserini's regression testing framework, for the TREC 2021 Deep Learning Track (Document Ranking Task) on the MS MARCO V2 document collection (with doc2query-T5 expansions) using relevance judgments from NIST. + +At the time this regression was created (November 2021), the qrels are only available to TREC participants. +You must download the qrels from NIST's "active participants" password-protected site and place at `src/main/resources/topics-and-qrels/qrels.dl21-doc.txt`. +The qrels will be added to Anserini when they are publicly released in Spring 2022. + +Note that the NIST relevance judgments provide far more relevant documents per topic, unlike the "sparse" judgments provided by Microsoft (these are sometimes called "dense" judgments to emphasize this contrast). +For additional instructions on working with MS MARCO document collection, refer to [this page](experiments-msmarco-v2.md). + +Note that there are four different regression conditions for this task, and this page describes the following: + ++ **Indexing Condition:** each document in the MS MARCO V2 document collection is treated as a unit of indexing ++ **Expansion Condition:** doc2query-T5 + +The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/dl21-doc-d2q-t5.yaml). +Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/dl21-doc-d2q-t5.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +## Indexing + +Typical indexing command: + +``` +target/appassembler/bin/IndexCollection \ + -collection MsMarcoV2DocCollection \ + -input /path/to/msmarco-v2-doc-d2q-t5 \ + -index indexes/lucene-index.msmarco-v2-doc-d2q-t5/ \ + -generator DefaultLuceneDocumentGenerator \ + -threads 24 -storePositions -storeDocvectors -storeRaw \ + >& logs/log.msmarco-v2-doc-d2q-t5 & +``` + +The value of `-input` should be a directory containing the compressed `jsonl` files that comprise the corpus. +See [this page](experiments-msmarco-v2.md) for additional details. + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +Topics and qrels are stored in [`src/main/resources/topics-and-qrels/`](../src/main/resources/topics-and-qrels/). +The regression experiments here evaluate on the 57 topics for which NIST has provided judgments as part of the TREC 2021 Deep Learning Track. + + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-doc-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.dl21.txt -topicreader TsvInt \ + -output runs/run.msmarco-v2-doc-d2q-t5.bm25-default.topics.dl21.txt \ + -hits 1000 -bm25 & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-doc-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.dl21.txt -topicreader TsvInt \ + -output runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rm3.topics.dl21.txt \ + -hits 1000 -bm25 -rm3 & +``` + +Evaluation can be performed using `trec_eval`: + +``` +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default.topics.dl21.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rm3.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rm3.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rm3.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rm3.topics.dl21.txt +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +MAP@100 | BM25 (default)| +RM3 | +:---------------------------------------|-----------|-----------| +[DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning)| 0.2387 | 0.2608 | + + +MRR@100 | BM25 (default)| +RM3 | +:---------------------------------------|-----------|-----------| +[DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning)| 0.8866 | 0.8342 | + + +nDCG@10 | BM25 (default)| +RM3 | +:---------------------------------------|-----------|-----------| +[DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning)| 0.5792 | 0.5392 | + + +R@100 | BM25 (default)| +RM3 | +:---------------------------------------|-----------|-----------| +[DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning)| 0.3443 | 0.3580 | + + +R@1000 | BM25 (default)| +RM3 | +:---------------------------------------|-----------|-----------| +[DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning)| 0.7066 | 0.7572 | + +Some of these regressions correspond to official TREC 2021 Deep Learning Track "baseline" submissions: + ++ `d_bm25` = BM25 (default), `k1=0.9`, `b=0.4` ++ `d_bm25rm3` = BM25 (default) + RM3, `k1=0.9`, `b=0.4` diff --git a/docs/regressions-dl21-doc-segmented-d2q-t5.md b/docs/regressions-dl21-doc-segmented-d2q-t5.md new file mode 100644 index 0000000000..3dfa87ba2e --- /dev/null +++ b/docs/regressions-dl21-doc-segmented-d2q-t5.md @@ -0,0 +1,106 @@ +# Anserini: Regressions for [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) + +This page describes experiments, integrated into Anserini's regression testing framework, for the TREC 2021 Deep Learning Track (Document Ranking Task) on the MS MARCO V2 _segmented_ document collection (with doc2query-T5 expansions) using relevance judgments from NIST. + +At the time this regression was created (November 2021), the qrels are only available to TREC participants. +You must download the qrels from NIST's "active participants" password-protected site and place at `src/main/resources/topics-and-qrels/qrels.dl21-doc.txt`. +The qrels will be added to Anserini when they are publicly released in Spring 2022. + +Note that the NIST relevance judgments provide far more relevant documents per topic, unlike the "sparse" judgments provided by Microsoft (these are sometimes called "dense" judgments to emphasize this contrast). +For additional instructions on working with MS MARCO document collection, refer to [this page](experiments-msmarco-v2.md). + +Note that there are four different regression conditions for this task, and this page describes the following: + ++ **Indexing Condition:** each segment in the MS MARCO V2 _segmented_ document collection is treated as a unit of indexing ++ **Expansion Condition:** doc2query-T5 + +The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/dl21-doc-segmented-d2q-t5.yaml). +Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/dl21-doc-segmented-d2q-t5.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +## Indexing + +Typical indexing command: + +``` +target/appassembler/bin/IndexCollection \ + -collection MsMarcoV2DocCollection \ + -input /path/to/msmarco-v2-doc-segmented-d2q-t5 \ + -index indexes/lucene-index.msmarco-v2-doc-segmented-d2q-t5/ \ + -generator DefaultLuceneDocumentGenerator \ + -threads 24 -storePositions -storeDocvectors -storeRaw \ + >& logs/log.msmarco-v2-doc-segmented-d2q-t5 & +``` + +The value of `-input` should be a directory containing the compressed `jsonl` files that comprise the corpus. +See [this page](experiments-msmarco-v2.md) for additional details. + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +Topics and qrels are stored in [`src/main/resources/topics-and-qrels/`](../src/main/resources/topics-and-qrels/). +The regression experiments here evaluate on the 57 topics for which NIST has provided judgments as part of the TREC 2021 Deep Learning Track. + + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-doc-segmented-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.dl21.txt -topicreader TsvInt \ + -output runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default.topics.dl21.txt \ + -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 -bm25 & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-doc-segmented-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.dl21.txt -topicreader TsvInt \ + -output runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rm3.topics.dl21.txt \ + -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 -bm25 -rm3 & +``` + +Evaluation can be performed using `trec_eval`: + +``` +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default.topics.dl21.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rm3.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rm3.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rm3.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rm3.topics.dl21.txt +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +MAP@100 | BM25 (default)| +RM3 | +:---------------------------------------|-----------|-----------| +[DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning)| 0.2683 | 0.3192 | + + +MRR@100 | BM25 (default)| +RM3 | +:---------------------------------------|-----------|-----------| +[DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning)| 0.9454 | 0.8960 | + + +nDCG@10 | BM25 (default)| +RM3 | +:---------------------------------------|-----------|-----------| +[DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning)| 0.6289 | 0.6555 | + + +R@100 | BM25 (default)| +RM3 | +:---------------------------------------|-----------|-----------| +[DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning)| 0.3656 | 0.4119 | + + +R@1000 | BM25 (default)| +RM3 | +:---------------------------------------|-----------|-----------| +[DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning)| 0.7202 | 0.7941 | + +Some of these regressions correspond to official TREC 2021 Deep Learning Track "baseline" submissions: + ++ `dseg_bm25` = BM25 (default), `k1=0.9`, `b=0.4` ++ `dseg_bm25rm3` = BM25 (default) + RM3, `k1=0.9`, `b=0.4` diff --git a/docs/regressions-msmarco-v2-doc-d2q-t5.md b/docs/regressions-msmarco-v2-doc-d2q-t5.md new file mode 100644 index 0000000000..345b140380 --- /dev/null +++ b/docs/regressions-msmarco-v2-doc-d2q-t5.md @@ -0,0 +1,84 @@ +# Anserini: Regressions for [MS MARCO (V2) Document Ranking](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) + +This page describes regression experiments for document ranking on the MS MARCO (V2) document corpus using the dev queries, which is integrated into Anserini's regression testing framework. +Here, we expand the document corpus with doc2query-T5. + +The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/msmarco-v2-doc-d2q-t5.yaml). +Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/msmarco-v2-doc-d2q-t5.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +## Indexing + +Typical indexing command: + +``` +target/appassembler/bin/IndexCollection \ + -collection MsMarcoV2DocCollection \ + -input /path/to/msmarco-v2-doc-d2q-t5 \ + -index indexes/lucene-index.msmarco-v2-doc-d2q-t5/ \ + -generator DefaultLuceneDocumentGenerator \ + -threads 24 -storePositions -storeDocvectors -storeRaw \ + >& logs/log.msmarco-v2-doc-d2q-t5 & +``` + +The directory `/path/to/msmarco-v2-doc-d2q-t5/` should be a directory containing the compressed `jsonl` files that comprise the corpus. +See [this page](experiments-msmarco-v2.md) for additional details. + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +Topics and qrels are stored in [`src/main/resources/topics-and-qrels/`](../src/main/resources/topics-and-qrels/). +These regression experiments use the [dev queries](../src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev.txt) and the [dev2 queries](../src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev2.txt). + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-doc-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev.txt -topicreader TsvInt \ + -output runs/run.msmarco-v2-doc-d2q-t5.bm25-default.topics.msmarco-v2-doc.dev.txt \ + -bm25 & +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-doc-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev2.txt -topicreader TsvInt \ + -output runs/run.msmarco-v2-doc-d2q-t5.bm25-default.topics.msmarco-v2-doc.dev2.txt \ + -bm25 & +``` + +Evaluation can be performed using `trec_eval`: + +``` +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default.topics.msmarco-v2-doc.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default.topics.msmarco-v2-doc.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default.topics.msmarco-v2-doc.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default.topics.msmarco-v2-doc.dev2.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default.topics.msmarco-v2-doc.dev2.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default.topics.msmarco-v2-doc.dev2.txt +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +MAP@100 | BM25 (default)| +:---------------------------------------|-----------| +[MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)| 0.1988 | +[MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)| 0.1986 | + + +MRR@100 | BM25 (default)| +:---------------------------------------|-----------| +[MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)| 0.2011 | +[MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)| 0.2012 | + + +R@100 | BM25 (default)| +:---------------------------------------|-----------| +[MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)| 0.6786 | +[MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)| 0.6821 | + + +R@1000 | BM25 (default)| +:---------------------------------------|-----------| +[MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)| 0.8614 | +[MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)| 0.8568 | diff --git a/docs/regressions-msmarco-v2-doc-segmented-d2q-t5.md b/docs/regressions-msmarco-v2-doc-segmented-d2q-t5.md new file mode 100644 index 0000000000..ce3e34cc09 --- /dev/null +++ b/docs/regressions-msmarco-v2-doc-segmented-d2q-t5.md @@ -0,0 +1,84 @@ +# Anserini: Regressions for [MS MARCO (V2) Document Ranking](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) + +This page describes regression experiments for document ranking _on the segmented version_ of the MS MARCO (V2) document corpus using the dev queries, which is integrated into Anserini's regression testing framework. +Here, we expand the segmented document corpus with doc2query-T5. + +The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/msmarco-v2-doc-segmented-d2q-t5.yaml). +Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/msmarco-v2-doc-segmented-d2q-t5.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +## Indexing + +Typical indexing command: + +``` +target/appassembler/bin/IndexCollection \ + -collection MsMarcoV2DocCollection \ + -input /path/to/msmarco-v2-doc-segmented-d2q-t5 \ + -index indexes/lucene-index.msmarco-v2-doc-segmented-d2q-t5/ \ + -generator DefaultLuceneDocumentGenerator \ + -threads 24 -storePositions -storeDocvectors -storeRaw \ + >& logs/log.msmarco-v2-doc-segmented-d2q-t5 & +``` + +The directory `/path/to/msmarco-v2-doc-segmented-d2q-t5/` should be a directory containing the compressed `jsonl` files that comprise the corpus. +See [this page](experiments-msmarco-v2.md) for additional details. + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +Topics and qrels are stored in [`src/main/resources/topics-and-qrels/`](../src/main/resources/topics-and-qrels/). +These regression experiments use the [dev queries](../src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev.txt) and the [dev2 queries](../src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev2.txt). + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-doc-segmented-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev.txt -topicreader TsvInt \ + -output runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default.topics.msmarco-v2-doc.dev.txt \ + -bm25 -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 & +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-doc-segmented-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev2.txt -topicreader TsvInt \ + -output runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default.topics.msmarco-v2-doc.dev2.txt \ + -bm25 -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 & +``` + +Evaluation can be performed using `trec_eval`: + +``` +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default.topics.msmarco-v2-doc.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default.topics.msmarco-v2-doc.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default.topics.msmarco-v2-doc.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default.topics.msmarco-v2-doc.dev2.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default.topics.msmarco-v2-doc.dev2.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default.topics.msmarco-v2-doc.dev2.txt +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +MAP@100 | BM25 (default)| +:---------------------------------------|-----------| +[MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)| 0.2203 | +[MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)| 0.2205 | + + +MRR@100 | BM25 (default)| +:---------------------------------------|-----------| +[MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)| 0.2226 | +[MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)| 0.2234 | + + +R@100 | BM25 (default)| +:---------------------------------------|-----------| +[MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)| 0.7297 | +[MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)| 0.7316 | + + +R@1000 | BM25 (default)| +:---------------------------------------|-----------| +[MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)| 0.8982 | +[MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)| 0.8952 | diff --git a/docs/regressions.md b/docs/regressions.md index f4de720651..a1338e2eca 100644 --- a/docs/regressions.md +++ b/docs/regressions.md @@ -54,17 +54,16 @@ nohup python src/main/python/run_regression.py --index --verify --search --regre nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-passage >& logs/log.msmarco-passage & nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-passage-doc2query >& logs/log.msmarco-passage-doc2query & nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-passage-docTTTTTquery >& logs/log.msmarco-passage-docTTTTTquery & +nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-passage-deepimpact >& logs/log.msmarco-passage-deepimpact & +nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-passage-unicoil >& logs/log.msmarco-passage-unicoil & +nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-passage-unicoil-tilde-expansion >& logs/log.msmarco-passage-unicoil-tilde-expansion & +nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-passage-distill-splade-max >& logs/log.msmarco-passage-distill-splade-max & nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-doc >& logs/log.msmarco-doc & nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-doc-docTTTTTquery >& logs/log.msmarco-doc-docTTTTTquery & nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-doc-segmented >& logs/log.msmarco-doc-segmented & nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-doc-segmented-docTTTTTquery >& logs/log.msmarco-doc-segmented-docTTTTTquery & -nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-passage-deepimpact >& logs/log.msmarco-passage-deepimpact & -nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-passage-unicoil >& logs/log.msmarco-passage-unicoil & -nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-passage-unicoil-tilde-expansion >& logs/log.msmarco-passage-unicoil-tilde-expansion & -nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-passage-distill-splade-max >& logs/log.msmarco-passage-distill-splade-max & - nohup python src/main/python/run_regression.py --index --verify --search --regression dl19-passage >& logs/log.dl19-passage & nohup python src/main/python/run_regression.py --index --verify --search --regression dl19-passage-docTTTTTquery >& logs/log.dl19-passage-docTTTTTquery & nohup python src/main/python/run_regression.py --index --verify --search --regression dl19-doc >& logs/log.dl19-doc & @@ -79,23 +78,24 @@ nohup python src/main/python/run_regression.py --index --verify --search --regre nohup python src/main/python/run_regression.py --index --verify --search --regression dl20-doc-segmented >& logs/log.dl20-doc-segmented & nohup python src/main/python/run_regression.py --index --verify --search --regression dl20-doc-segmented-docTTTTTquery >& logs/log.dl20-doc-segmented-docTTTTTquery & -nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-v2-doc >& logs/log.msmarco-v2-doc & -nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-v2-doc-segmented >& logs/log.msmarco-v2-doc-segmented & nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-v2-passage >& logs/log.msmarco-v2-passage & nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-v2-passage-augmented >& logs/log.msmarco-v2-passage-augmented & +nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-v2-passage-unicoil-noexp-0shot >& logs/log.msmarco-v2-passage-unicoil-noexp-0shot & +nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-v2-doc >& logs/log.msmarco-v2-doc & nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-v2-doc-d2q-t5 >& logs/log.msmarco-v2-doc-d2q-t5 & +nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-v2-doc-segmented >& logs/log.msmarco-v2-doc-segmented & nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-v2-doc-segmented-d2q-t5 >& logs/log.msmarco-v2-doc-segmented-d2q-t5 & - -nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-v2-passage-unicoil-noexp-0shot >& logs/log.msmarco-v2-passage-unicoil-noexp-0shot & nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-v2-doc-segmented-unicoil-noexp-0shot >& logs/log.msmarco-v2-doc-segmented-unicoil-noexp-0shot & nohup python src/main/python/run_regression.py --index --verify --search --regression dl21-passage >& logs/log.dl21-passage & nohup python src/main/python/run_regression.py --index --verify --search --regression dl21-passage-augmented >& logs/log.dl21-passage-augmented & nohup python src/main/python/run_regression.py --index --verify --search --regression dl21-passage-unicoil-noexp-0shot >& logs/log.dl21-passage-unicoil-noexp-0shot & nohup python src/main/python/run_regression.py --index --verify --search --regression dl21-doc >& logs/log.dl21-doc & +nohup python src/main/python/run_regression.py --index --verify --search --regression dl21-doc-d2q-t5 >& logs/log.dl21-doc-d2q-t5 & nohup python src/main/python/run_regression.py --index --verify --search --regression dl21-doc-segmented >& logs/log.dl21-doc-segmented & nohup python src/main/python/run_regression.py --index --verify --search --regression dl21-doc-segmented-unicoil-noexp-0shot >& logs/log.dl21-doc-segmented-unicoil-noexp-0shot & +nohup python src/main/python/run_regression.py --index --verify --search --regression dl21-doc-segmented-d2q-t5 >& logs/log.dl21-doc-segmented-d2q-t5 & nohup python src/main/python/run_regression.py --index --verify --search --regression backgroundlinking18 >& logs/log.backgroundlinking18 & nohup python src/main/python/run_regression.py --index --verify --search --regression backgroundlinking19 >& logs/log.backgroundlinking19 & diff --git a/src/main/resources/docgen/templates/dl21-doc-d2q-t5.template b/src/main/resources/docgen/templates/dl21-doc-d2q-t5.template new file mode 100644 index 0000000000..5d86766ea8 --- /dev/null +++ b/src/main/resources/docgen/templates/dl21-doc-d2q-t5.template @@ -0,0 +1,60 @@ +# Anserini: Regressions for [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) + +This page describes experiments, integrated into Anserini's regression testing framework, for the TREC 2021 Deep Learning Track (Document Ranking Task) on the MS MARCO V2 document collection (with doc2query-T5 expansions) using relevance judgments from NIST. + +At the time this regression was created (November 2021), the qrels are only available to TREC participants. +You must download the qrels from NIST's "active participants" password-protected site and place at `src/main/resources/topics-and-qrels/qrels.dl21-doc.txt`. +The qrels will be added to Anserini when they are publicly released in Spring 2022. + +Note that the NIST relevance judgments provide far more relevant documents per topic, unlike the "sparse" judgments provided by Microsoft (these are sometimes called "dense" judgments to emphasize this contrast). +For additional instructions on working with MS MARCO document collection, refer to [this page](experiments-msmarco-v2.md). + +Note that there are four different regression conditions for this task, and this page describes the following: + ++ **Indexing Condition:** each document in the MS MARCO V2 document collection is treated as a unit of indexing ++ **Expansion Condition:** doc2query-T5 + +The exact configurations for these regressions are stored in [this YAML file](${yaml}). +Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +## Indexing + +Typical indexing command: + +``` +${index_cmds} +``` + +The value of `-input` should be a directory containing the compressed `jsonl` files that comprise the corpus. +See [this page](experiments-msmarco-v2.md) for additional details. + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +Topics and qrels are stored in [`src/main/resources/topics-and-qrels/`](../src/main/resources/topics-and-qrels/). +The regression experiments here evaluate on the 57 topics for which NIST has provided judgments as part of the TREC 2021 Deep Learning Track. + + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +${ranking_cmds} +``` + +Evaluation can be performed using `trec_eval`: + +``` +${eval_cmds} +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +${effectiveness} + +Some of these regressions correspond to official TREC 2021 Deep Learning Track "baseline" submissions: + ++ `d_bm25` = BM25 (default), `k1=0.9`, `b=0.4` ++ `d_bm25rm3` = BM25 (default) + RM3, `k1=0.9`, `b=0.4` diff --git a/src/main/resources/docgen/templates/dl21-doc-segmented-d2q-t5.template b/src/main/resources/docgen/templates/dl21-doc-segmented-d2q-t5.template new file mode 100644 index 0000000000..22c3c30880 --- /dev/null +++ b/src/main/resources/docgen/templates/dl21-doc-segmented-d2q-t5.template @@ -0,0 +1,60 @@ +# Anserini: Regressions for [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) + +This page describes experiments, integrated into Anserini's regression testing framework, for the TREC 2021 Deep Learning Track (Document Ranking Task) on the MS MARCO V2 _segmented_ document collection (with doc2query-T5 expansions) using relevance judgments from NIST. + +At the time this regression was created (November 2021), the qrels are only available to TREC participants. +You must download the qrels from NIST's "active participants" password-protected site and place at `src/main/resources/topics-and-qrels/qrels.dl21-doc.txt`. +The qrels will be added to Anserini when they are publicly released in Spring 2022. + +Note that the NIST relevance judgments provide far more relevant documents per topic, unlike the "sparse" judgments provided by Microsoft (these are sometimes called "dense" judgments to emphasize this contrast). +For additional instructions on working with MS MARCO document collection, refer to [this page](experiments-msmarco-v2.md). + +Note that there are four different regression conditions for this task, and this page describes the following: + ++ **Indexing Condition:** each segment in the MS MARCO V2 _segmented_ document collection is treated as a unit of indexing ++ **Expansion Condition:** doc2query-T5 + +The exact configurations for these regressions are stored in [this YAML file](${yaml}). +Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +## Indexing + +Typical indexing command: + +``` +${index_cmds} +``` + +The value of `-input` should be a directory containing the compressed `jsonl` files that comprise the corpus. +See [this page](experiments-msmarco-v2.md) for additional details. + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +Topics and qrels are stored in [`src/main/resources/topics-and-qrels/`](../src/main/resources/topics-and-qrels/). +The regression experiments here evaluate on the 57 topics for which NIST has provided judgments as part of the TREC 2021 Deep Learning Track. + + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +${ranking_cmds} +``` + +Evaluation can be performed using `trec_eval`: + +``` +${eval_cmds} +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +${effectiveness} + +Some of these regressions correspond to official TREC 2021 Deep Learning Track "baseline" submissions: + ++ `dseg_bm25` = BM25 (default), `k1=0.9`, `b=0.4` ++ `dseg_bm25rm3` = BM25 (default) + RM3, `k1=0.9`, `b=0.4` diff --git a/src/main/resources/docgen/templates/msmarco-v2-doc-d2q-t5.template b/src/main/resources/docgen/templates/msmarco-v2-doc-d2q-t5.template new file mode 100644 index 0000000000..f5ae2018f3 --- /dev/null +++ b/src/main/resources/docgen/templates/msmarco-v2-doc-d2q-t5.template @@ -0,0 +1,43 @@ +# Anserini: Regressions for [MS MARCO (V2) Document Ranking](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) + +This page describes regression experiments for document ranking on the MS MARCO (V2) document corpus using the dev queries, which is integrated into Anserini's regression testing framework. +Here, we expand the document corpus with doc2query-T5. + +The exact configurations for these regressions are stored in [this YAML file](${yaml}). +Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +## Indexing + +Typical indexing command: + +``` +${index_cmds} +``` + +The directory `/path/to/msmarco-v2-doc-d2q-t5/` should be a directory containing the compressed `jsonl` files that comprise the corpus. +See [this page](experiments-msmarco-v2.md) for additional details. + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +Topics and qrels are stored in [`src/main/resources/topics-and-qrels/`](../src/main/resources/topics-and-qrels/). +These regression experiments use the [dev queries](../src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev.txt) and the [dev2 queries](../src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev2.txt). + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +${ranking_cmds} +``` + +Evaluation can be performed using `trec_eval`: + +``` +${eval_cmds} +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +${effectiveness} diff --git a/src/main/resources/docgen/templates/msmarco-v2-doc-segmented-d2q-t5.template b/src/main/resources/docgen/templates/msmarco-v2-doc-segmented-d2q-t5.template new file mode 100644 index 0000000000..d23cf1e376 --- /dev/null +++ b/src/main/resources/docgen/templates/msmarco-v2-doc-segmented-d2q-t5.template @@ -0,0 +1,43 @@ +# Anserini: Regressions for [MS MARCO (V2) Document Ranking](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) + +This page describes regression experiments for document ranking _on the segmented version_ of the MS MARCO (V2) document corpus using the dev queries, which is integrated into Anserini's regression testing framework. +Here, we expand the segmented document corpus with doc2query-T5. + +The exact configurations for these regressions are stored in [this YAML file](${yaml}). +Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +## Indexing + +Typical indexing command: + +``` +${index_cmds} +``` + +The directory `/path/to/msmarco-v2-doc-segmented-d2q-t5/` should be a directory containing the compressed `jsonl` files that comprise the corpus. +See [this page](experiments-msmarco-v2.md) for additional details. + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +Topics and qrels are stored in [`src/main/resources/topics-and-qrels/`](../src/main/resources/topics-and-qrels/). +These regression experiments use the [dev queries](../src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev.txt) and the [dev2 queries](../src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev2.txt). + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +${ranking_cmds} +``` + +Evaluation can be performed using `trec_eval`: + +``` +${eval_cmds} +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +${effectiveness} From d0953c61aeae0e78ee1545f56cedba00fa1546d0 Mon Sep 17 00:00:00 2001 From: lintool Date: Tue, 18 Jan 2022 21:10:24 -0500 Subject: [PATCH 06/10] more configs. --- .../msmarco-v2-passage-augmented-d2q-t5.yaml | 74 +++++++++++++++++++ .../regression/msmarco-v2-passage-d2q-t5.yaml | 74 +++++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100644 src/main/resources/regression/msmarco-v2-passage-augmented-d2q-t5.yaml create mode 100644 src/main/resources/regression/msmarco-v2-passage-d2q-t5.yaml diff --git a/src/main/resources/regression/msmarco-v2-passage-augmented-d2q-t5.yaml b/src/main/resources/regression/msmarco-v2-passage-augmented-d2q-t5.yaml new file mode 100644 index 0000000000..7ca8cb0688 --- /dev/null +++ b/src/main/resources/regression/msmarco-v2-passage-augmented-d2q-t5.yaml @@ -0,0 +1,74 @@ +--- +corpus: msmarco-v2-passage-augmented-d2q-t5 +corpus_path: collections/msmarco/msmarco_v2_passage_augmented_d2q-t5/ + +index_path: indexes/lucene-index.msmarco-v2-passage-augmented-d2q-t5/ +collection_class: MsMarcoV2PassageCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 18 +index_options: -storePositions -storeDocvectors -storeRaw +index_stats: + documents: 138364198 + documents (non-empty): 138364198 + total terms: 27561177420 + +metrics: + - metric: MAP@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -M 100 -m map + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: true + - metric: MRR@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -M 100 -m recip_rank + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: true + - metric: R@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvInt +topic_root: src/main/resources/topics-and-qrels/ +qrels_root: src/main/resources/topics-and-qrels/ +topics: + - name: "[MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)" + id: dev + path: topics.msmarco-v2-passage.dev.txt + qrel: qrels.msmarco-v2-passage.dev.txt + - name: "[MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)" + id: dev2 + path: topics.msmarco-v2-passage.dev2.txt + qrel: qrels.msmarco-v2-passage.dev2.txt + +models: + - name: bm25-default + display: BM25 (default) + params: -bm25 + results: + MAP@100: + - 0.1160 + - 0.1158 + MRR@100: + - 0.1172 + - 0.1170 + R@100: + - 0.5039 + - 0.5158 + R@1000: + - 0.7647 + - 0.7659 diff --git a/src/main/resources/regression/msmarco-v2-passage-d2q-t5.yaml b/src/main/resources/regression/msmarco-v2-passage-d2q-t5.yaml new file mode 100644 index 0000000000..5c1a17e9e4 --- /dev/null +++ b/src/main/resources/regression/msmarco-v2-passage-d2q-t5.yaml @@ -0,0 +1,74 @@ +--- +corpus: msmarco-v2-passage-d2q-t5 +corpus_path: collections/msmarco/msmarco_v2_passage_d2q-t5/ + +index_path: indexes/lucene-index.msmarco-v2-passage-d2q-t5/ +collection_class: MsMarcoV2PassageCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 18 +index_options: -storePositions -storeDocvectors -storeRaw +index_stats: + documents: 138364198 + documents (non-empty): 138364198 + total terms: 16961479226 + +metrics: + - metric: MAP@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -M 100 -m map + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: true + - metric: MRR@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -M 100 -m recip_rank + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: true + - metric: R@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.100 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.1000 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvInt +topic_root: src/main/resources/topics-and-qrels/ +qrels_root: src/main/resources/topics-and-qrels/ +topics: + - name: "[MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)" + id: dev + path: topics.msmarco-v2-passage.dev.txt + qrel: qrels.msmarco-v2-passage.dev.txt + - name: "[MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)" + id: dev2 + path: topics.msmarco-v2-passage.dev2.txt + qrel: qrels.msmarco-v2-passage.dev2.txt + +models: + - name: bm25-default + display: BM25 (default) + params: -bm25 + results: + MAP@100: + - 0.1057 + - 0.1112 + MRR@100: + - 0.1072 + - 0.1123 + R@100: + - 0.4670 + - 0.4803 + R@1000: + - 0.7083 + - 0.7151 From aff43e32cdd665491069e36f7503f6424afd3c94 Mon Sep 17 00:00:00 2001 From: lintool Date: Tue, 18 Jan 2022 22:46:07 -0500 Subject: [PATCH 07/10] more regressions. --- .../dl21-passage-augmented-d2q-t5.yaml | 89 +++++++++++++++++++ .../regression/dl21-passage-d2q-t5.yaml | 89 +++++++++++++++++++ 2 files changed, 178 insertions(+) create mode 100644 src/main/resources/regression/dl21-passage-augmented-d2q-t5.yaml create mode 100644 src/main/resources/regression/dl21-passage-d2q-t5.yaml diff --git a/src/main/resources/regression/dl21-passage-augmented-d2q-t5.yaml b/src/main/resources/regression/dl21-passage-augmented-d2q-t5.yaml new file mode 100644 index 0000000000..05d4301a3a --- /dev/null +++ b/src/main/resources/regression/dl21-passage-augmented-d2q-t5.yaml @@ -0,0 +1,89 @@ +--- +corpus: msmarco-v2-passage-augmented-d2q-t5 +corpus_path: collections/msmarco/msmarco_v2_passage_augmented_d2q-t5/ + +index_path: indexes/lucene-index.msmarco-v2-passage-augmented-d2q-t5/ +collection_class: MsMarcoV2PassageCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 18 +index_options: -storePositions -storeDocvectors -storeRaw +index_stats: + documents: 138364198 + documents (non-empty): 138364198 + total terms: 27561177420 + +metrics: + - metric: MAP@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -M 100 -m map -l 2 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: MRR@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -M 100 -m recip_rank -l 2 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: nDCG@10 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.100 -l 2 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.1000 -l 2 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvInt +topic_root: src/main/resources/topics-and-qrels/ +qrels_root: src/main/resources/topics-and-qrels/ +topics: + - name: "[DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning)" + id: dl21 + path: topics.dl21.txt + qrel: qrels.dl21-passage.txt + +models: + - name: bm25-default + display: BM25 (default) + params: -bm25 + results: + MAP@100: + - 0.1649 + MRR@100: + - 0.6391 + nDCG@10: + - 0.4702 + R@100: + - 0.3883 + R@1000: + - 0.6962 + - name: bm25-default+rm3 + display: +RM3 + params: -bm25 -rm3 + results: + MAP@100: + - 0.1932 + MRR@100: + - 0.5882 + nDCG@10: + - 0.4834 + R@100: + - 0.4295 + R@1000: + - 0.7668 diff --git a/src/main/resources/regression/dl21-passage-d2q-t5.yaml b/src/main/resources/regression/dl21-passage-d2q-t5.yaml new file mode 100644 index 0000000000..2b2d27aff4 --- /dev/null +++ b/src/main/resources/regression/dl21-passage-d2q-t5.yaml @@ -0,0 +1,89 @@ +--- +corpus: msmarco-v2-passage-d2q-t5 +corpus_path: collections/msmarco/msmarco_v2_passage_d2q-t5/ + +index_path: indexes/lucene-index.msmarco-v2-passage-d2q-t5/ +collection_class: MsMarcoV2PassageCollection +generator_class: DefaultLuceneDocumentGenerator +index_threads: 18 +index_options: -storePositions -storeDocvectors -storeRaw +index_stats: + documents: 138364198 + documents (non-empty): 138364198 + total terms: 16961479226 + +metrics: + - metric: MAP@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -M 100 -m map -l 2 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: MRR@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -M 100 -m recip_rank -l 2 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: nDCG@10 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m ndcg_cut.10 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@100 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.100 -l 2 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + - metric: R@1000 + command: tools/eval/trec_eval.9.0.4/trec_eval + params: -c -m recall.1000 -l 2 + separator: "\t" + parse_index: 2 + metric_precision: 4 + can_combine: false + +topic_reader: TsvInt +topic_root: src/main/resources/topics-and-qrels/ +qrels_root: src/main/resources/topics-and-qrels/ +topics: + - name: "[DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning)" + id: dl21 + path: topics.dl21.txt + qrel: qrels.dl21-passage.txt + +models: + - name: bm25-default + display: BM25 (default) + params: -bm25 + results: + MAP@100: + - 0.1874 + MRR@100: + - 0.6848 + nDCG@10: + - 0.4816 + R@100: + - 0.4076 + R@1000: + - 0.7078 + - name: bm25-default+rm3 + display: +RM3 + params: -bm25 -rm3 + results: + MAP@100: + - 0.2271 + MRR@100: + - 0.6651 + nDCG@10: + - 0.5099 + R@100: + - 0.4444 + R@1000: + - 0.7512 From a1b2309b0675b1935103398cee7fa8d241b0d476 Mon Sep 17 00:00:00 2001 From: lintool Date: Tue, 18 Jan 2022 23:37:54 -0500 Subject: [PATCH 08/10] tweaked doc. --- docs/regressions.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/regressions.md b/docs/regressions.md index a1338e2eca..496eb5bd23 100644 --- a/docs/regressions.md +++ b/docs/regressions.md @@ -79,7 +79,9 @@ nohup python src/main/python/run_regression.py --index --verify --search --regre nohup python src/main/python/run_regression.py --index --verify --search --regression dl20-doc-segmented-docTTTTTquery >& logs/log.dl20-doc-segmented-docTTTTTquery & nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-v2-passage >& logs/log.msmarco-v2-passage & +nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-v2-passage-d2q-t5 >& logs/log.msmarco-v2-passage-d2q-t5 & nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-v2-passage-augmented >& logs/log.msmarco-v2-passage-augmented & +nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-v2-passage-augmented-d2q-t5 >& logs/log.msmarco-v2-passage-augmented-d2q-t5 & nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-v2-passage-unicoil-noexp-0shot >& logs/log.msmarco-v2-passage-unicoil-noexp-0shot & nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-v2-doc >& logs/log.msmarco-v2-doc & @@ -89,7 +91,9 @@ nohup python src/main/python/run_regression.py --index --verify --search --regre nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-v2-doc-segmented-unicoil-noexp-0shot >& logs/log.msmarco-v2-doc-segmented-unicoil-noexp-0shot & nohup python src/main/python/run_regression.py --index --verify --search --regression dl21-passage >& logs/log.dl21-passage & +nohup python src/main/python/run_regression.py --index --verify --search --regression dl21-passage-d2q-t5 >& logs/log.dl21-passage-d2q-t5 & nohup python src/main/python/run_regression.py --index --verify --search --regression dl21-passage-augmented >& logs/log.dl21-passage-augmented & +nohup python src/main/python/run_regression.py --index --verify --search --regression dl21-passage-augmented-d2q-t5 >& logs/log.dl21-passage-augmented-d2q-t5 & nohup python src/main/python/run_regression.py --index --verify --search --regression dl21-passage-unicoil-noexp-0shot >& logs/log.dl21-passage-unicoil-noexp-0shot & nohup python src/main/python/run_regression.py --index --verify --search --regression dl21-doc >& logs/log.dl21-doc & nohup python src/main/python/run_regression.py --index --verify --search --regression dl21-doc-d2q-t5 >& logs/log.dl21-doc-d2q-t5 & From fb761d6a794de8fa12b9f3ba9069d1548d1ed1d3 Mon Sep 17 00:00:00 2001 From: lintool Date: Wed, 19 Jan 2022 09:53:56 -0500 Subject: [PATCH 09/10] Updated docs. --- README.md | 8 +- ...gressions-dl21-passage-augmented-d2q-t5.md | 98 +++++++++++++++++++ docs/regressions-dl21-passage-d2q-t5.md | 98 +++++++++++++++++++ ...ons-msmarco-v2-passage-augmented-d2q-t5.md | 83 ++++++++++++++++ ...egressions-msmarco-v2-passage-augmented.md | 1 + docs/regressions-msmarco-v2-passage-d2q-t5.md | 83 ++++++++++++++++ .../dl21-passage-augmented-d2q-t5.template | 50 ++++++++++ .../templates/dl21-passage-d2q-t5.template | 50 ++++++++++ ...marco-v2-passage-augmented-d2q-t5.template | 42 ++++++++ .../msmarco-v2-passage-augmented.template | 1 + .../msmarco-v2-passage-d2q-t5.template | 42 ++++++++ 11 files changed, 553 insertions(+), 3 deletions(-) create mode 100644 docs/regressions-dl21-passage-augmented-d2q-t5.md create mode 100644 docs/regressions-dl21-passage-d2q-t5.md create mode 100644 docs/regressions-msmarco-v2-passage-augmented-d2q-t5.md create mode 100644 docs/regressions-msmarco-v2-passage-d2q-t5.md create mode 100644 src/main/resources/docgen/templates/dl21-passage-augmented-d2q-t5.template create mode 100644 src/main/resources/docgen/templates/dl21-passage-d2q-t5.template create mode 100644 src/main/resources/docgen/templates/msmarco-v2-passage-augmented-d2q-t5.template create mode 100644 src/main/resources/docgen/templates/msmarco-v2-passage-d2q-t5.template diff --git a/README.md b/README.md index 729f17dde5..3387721438 100644 --- a/README.md +++ b/README.md @@ -65,15 +65,17 @@ For the most part, these runs are based on [_default_ parameter settings](https: + Document ranking, complete doc[*](docs/experiments-msmarco-doc-doc2query-details.md): [baselines](docs/regressions-dl20-doc.md), [doc2query-T5](docs/regressions-dl20-doc-docTTTTTquery.md) + Document ranking, segmented doc[*](docs/experiments-msmarco-doc-doc2query-details.md): [baselines](docs/regressions-dl20-doc-segmented.md), [doc2query-T5](docs/regressions-dl20-doc-segmented-docTTTTTquery.md) + Regressions for MS MARCO (V2) Passage Ranking: - + Bag-of-words models, default corpus: [baselines](docs/regressions-msmarco-v2-passage.md) - + Bag-of-words models, augmented corpus: [baselines](docs/regressions-msmarco-v2-passage-augmented.md) + + Bag-of-words models, original corpus: [baselines](docs/regressions-msmarco-v2-passage.md), [doc2query-T5](docs/regressions-msmarco-v2-passage-d2q-t5.md) + + Bag-of-words models, augmented corpus: [baselines](docs/regressions-msmarco-v2-passage-augmented.md), [doc2query-T5](docs/regressions-msmarco-v2-passage-augmented-d2q-t5.md) + Sparse learned models: [uniCOIL noexp zero-shot](docs/regressions-msmarco-v2-passage-unicoil-noexp-0shot.md) + Regressions for MS MARCO (V2) Document Ranking: + Bag-of-words, complete doc: [baselines](docs/regressions-msmarco-v2-doc.md), [doc2query-T5](docs/regressions-msmarco-v2-doc-d2q-t5.md) + Bag-of-words, segmented doc: [baselines](docs/regressions-msmarco-v2-doc-segmented.md), [doc2query-T5](docs/regressions-msmarco-v2-doc-segmented-d2q-t5.md) + Sparse learned models: [uniCOIL noexp zero-shot](docs/regressions-msmarco-v2-doc-segmented-unicoil-noexp-0shot.md) + Regressions for TREC 2021 Deep Learning Track: - + Passage ranking: [baselines](docs/regressions-dl21-passage.md), [on augmented corpus](docs/regressions-dl21-passage-augmented.md), [uniCOIL noexp zero-shot](docs/regressions-dl21-passage-unicoil-noexp-0shot.md) + + Passage ranking, original corpus: [baselines](docs/regressions-dl21-passage.md), [doc2query-T5](docs/regressions-dl21-passage-d2q-t5.md) + + Passage ranking, augmented corpus: [baselines](docs/regressions-dl21-passage-augmented.md), [doc2query-T5](docs/regressions-dl21-passage-augmented-d2q-t5.md) + + Passage ranking, sparse learned models: [uniCOIL noexp zero-shot](docs/regressions-dl21-passage-unicoil-noexp-0shot.md) + Document ranking, complete doc: [baselines](docs/regressions-dl21-doc.md), [doc2query-T5](docs/regressions-dl21-doc-d2q-t5.md) + Document ranking, segmented doc: [baselines](docs/regressions-dl21-doc-segmented.md), [doc2query-T5](docs/regressions-dl21-doc-segmented-d2q-t5.md) + Document ranking, sparse learned models: [uniCOIL noexp zero-shot](docs/regressions-dl21-doc-segmented-unicoil-noexp-0shot.md) diff --git a/docs/regressions-dl21-passage-augmented-d2q-t5.md b/docs/regressions-dl21-passage-augmented-d2q-t5.md new file mode 100644 index 0000000000..67ce5c8598 --- /dev/null +++ b/docs/regressions-dl21-passage-augmented-d2q-t5.md @@ -0,0 +1,98 @@ +# Anserini: Regressions for [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) + +This page describes document expansion experiments (with doc2query-T5), integrated into Anserini's regression testing framework, for the TREC 2021 Deep Learning Track (Passage Ranking Task) on the MS MARCO V2 _augmented_ passage collection using relevance judgments from NIST. + +At the time this regression was created (November 2021), the qrels are only available to TREC participants. +You must download the qrels from NIST's "active participants" password-protected site and place at `src/main/resources/topics-and-qrels/qrels.dl21-passage.txt`. +The qrels will be added to Anserini when they are publicly released in Spring 2022. + +Note that the NIST relevance judgments provide far more relevant passages per topic, unlike the "sparse" judgments provided by Microsoft (these are sometimes called "dense" judgments to emphasize this contrast). +For additional instructions on working with MS MARCO passage collection, refer to [this page](experiments-msmarco-v2.md). + +The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/dl21-passage-augmented-d2q-t5.yaml). +Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/dl21-passage-augmented-d2q-t5.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +## Indexing + +Typical indexing command: + +``` +target/appassembler/bin/IndexCollection \ + -collection MsMarcoV2PassageCollection \ + -input /path/to/msmarco-v2-passage-augmented-d2q-t5 \ + -index indexes/lucene-index.msmarco-v2-passage-augmented-d2q-t5/ \ + -generator DefaultLuceneDocumentGenerator \ + -threads 18 -storePositions -storeDocvectors -storeRaw \ + >& logs/log.msmarco-v2-passage-augmented-d2q-t5 & +``` + +The value of `-input` should be a directory containing the compressed `jsonl` files that comprise the corpus. +See [this page](experiments-msmarco-v2.md) for additional details. + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +Topics and qrels are stored in [`src/main/resources/topics-and-qrels/`](../src/main/resources/topics-and-qrels/). +The regression experiments here evaluate on the 53 topics for which NIST has provided judgments as part of the TREC 2021 Deep Learning Track. + + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-passage-augmented-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.dl21.txt -topicreader TsvInt \ + -output runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default.topics.dl21.txt \ + -bm25 & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-passage-augmented-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.dl21.txt -topicreader TsvInt \ + -output runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rm3.topics.dl21.txt \ + -bm25 -rm3 & +``` + +Evaluation can be performed using `trec_eval`: + +``` +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default.topics.dl21.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rm3.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rm3.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rm3.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rm3.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rm3.topics.dl21.txt +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +MAP@100 | BM25 (default)| +RM3 | +:---------------------------------------|-----------|-----------| +[DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning)| 0.1649 | 0.1932 | + + +MRR@100 | BM25 (default)| +RM3 | +:---------------------------------------|-----------|-----------| +[DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning)| 0.6391 | 0.5882 | + + +nDCG@10 | BM25 (default)| +RM3 | +:---------------------------------------|-----------|-----------| +[DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning)| 0.4702 | 0.4834 | + + +R@100 | BM25 (default)| +RM3 | +:---------------------------------------|-----------|-----------| +[DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning)| 0.3883 | 0.4295 | + + +R@1000 | BM25 (default)| +RM3 | +:---------------------------------------|-----------|-----------| +[DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning)| 0.6962 | 0.7668 | diff --git a/docs/regressions-dl21-passage-d2q-t5.md b/docs/regressions-dl21-passage-d2q-t5.md new file mode 100644 index 0000000000..a6d816d48f --- /dev/null +++ b/docs/regressions-dl21-passage-d2q-t5.md @@ -0,0 +1,98 @@ +# Anserini: Regressions for [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) + +This page describes document expansion experiments (with doc2query-T5), integrated into Anserini's regression testing framework, for the TREC 2021 Deep Learning Track (Passage Ranking Task) on the MS MARCO V2 passage collection using relevance judgments from NIST. + +At the time this regression was created (November 2021), the qrels are only available to TREC participants. +You must download the qrels from NIST's "active participants" password-protected site and place at `src/main/resources/topics-and-qrels/qrels.dl21-passage.txt`. +The qrels will be added to Anserini when they are publicly released in Spring 2022. + +Note that the NIST relevance judgments provide far more relevant passages per topic, unlike the "sparse" judgments provided by Microsoft (these are sometimes called "dense" judgments to emphasize this contrast). +For additional instructions on working with MS MARCO passage collection, refer to [this page](experiments-msmarco-v2.md). + +The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/dl21-passage-d2q-t5.yaml). +Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/dl21-passage-d2q-t5.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +## Indexing + +Typical indexing command: + +``` +target/appassembler/bin/IndexCollection \ + -collection MsMarcoV2PassageCollection \ + -input /path/to/msmarco-v2-passage-d2q-t5 \ + -index indexes/lucene-index.msmarco-v2-passage-d2q-t5/ \ + -generator DefaultLuceneDocumentGenerator \ + -threads 18 -storePositions -storeDocvectors -storeRaw \ + >& logs/log.msmarco-v2-passage-d2q-t5 & +``` + +The value of `-input` should be a directory containing the compressed `jsonl` files that comprise the corpus. +See [this page](experiments-msmarco-v2.md) for additional details. + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +Topics and qrels are stored in [`src/main/resources/topics-and-qrels/`](../src/main/resources/topics-and-qrels/). +The regression experiments here evaluate on the 53 topics for which NIST has provided judgments as part of the TREC 2021 Deep Learning Track. + + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-passage-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.dl21.txt -topicreader TsvInt \ + -output runs/run.msmarco-v2-passage-d2q-t5.bm25-default.topics.dl21.txt \ + -bm25 & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-passage-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.dl21.txt -topicreader TsvInt \ + -output runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rm3.topics.dl21.txt \ + -bm25 -rm3 & +``` + +Evaluation can be performed using `trec_eval`: + +``` +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default.topics.dl21.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rm3.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rm3.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rm3.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rm3.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rm3.topics.dl21.txt +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +MAP@100 | BM25 (default)| +RM3 | +:---------------------------------------|-----------|-----------| +[DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning)| 0.1874 | 0.2271 | + + +MRR@100 | BM25 (default)| +RM3 | +:---------------------------------------|-----------|-----------| +[DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning)| 0.6848 | 0.6651 | + + +nDCG@10 | BM25 (default)| +RM3 | +:---------------------------------------|-----------|-----------| +[DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning)| 0.4816 | 0.5099 | + + +R@100 | BM25 (default)| +RM3 | +:---------------------------------------|-----------|-----------| +[DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning)| 0.4076 | 0.4444 | + + +R@1000 | BM25 (default)| +RM3 | +:---------------------------------------|-----------|-----------| +[DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning)| 0.7078 | 0.7512 | diff --git a/docs/regressions-msmarco-v2-passage-augmented-d2q-t5.md b/docs/regressions-msmarco-v2-passage-augmented-d2q-t5.md new file mode 100644 index 0000000000..daf7845364 --- /dev/null +++ b/docs/regressions-msmarco-v2-passage-augmented-d2q-t5.md @@ -0,0 +1,83 @@ +# Anserini: Regressions for [MS MARCO (V2) Passage Ranking](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) + +This page describes regression experiments for passage ranking _on the augmented version_ of the MS MARCO (V2) passage corpus using the dev queries, which is integrated into Anserini's regression testing framework. +Here, we expand the augmented passage corpus with doc2query-T5. + +The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/msmarco-v2-passage-augmented-d2q-t5.yaml). +Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/msmarco-v2-passage-augmented-d2q-t5.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +## Indexing + +Typical indexing command: + +``` +target/appassembler/bin/IndexCollection \ + -collection MsMarcoV2PassageCollection \ + -input /path/to/msmarco-v2-passage-augmented-d2q-t5 \ + -index indexes/lucene-index.msmarco-v2-passage-augmented-d2q-t5/ \ + -generator DefaultLuceneDocumentGenerator \ + -threads 18 -storePositions -storeDocvectors -storeRaw \ + >& logs/log.msmarco-v2-passage-augmented-d2q-t5 & +``` + +The directory `/path/to/msmarco-v2-passage-augmented-d2q-t5/` should be a directory containing the compressed `jsonl` files that comprise the corpus. + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +Topics and qrels are stored in [`src/main/resources/topics-and-qrels/`](../src/main/resources/topics-and-qrels/). +These regression experiments use the [dev queries](../src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev.txt) and the [dev2 queries](../src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev2.txt). + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-passage-augmented-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev.txt -topicreader TsvInt \ + -output runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default.topics.msmarco-v2-passage.dev.txt \ + -bm25 & +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-passage-augmented-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev2.txt -topicreader TsvInt \ + -output runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default.topics.msmarco-v2-passage.dev2.txt \ + -bm25 & +``` + +Evaluation can be performed using `trec_eval`: + +``` +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default.topics.msmarco-v2-passage.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default.topics.msmarco-v2-passage.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default.topics.msmarco-v2-passage.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default.topics.msmarco-v2-passage.dev2.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default.topics.msmarco-v2-passage.dev2.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default.topics.msmarco-v2-passage.dev2.txt +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +MAP@100 | BM25 (default)| +:---------------------------------------|-----------| +[MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)| 0.1160 | +[MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)| 0.1158 | + + +MRR@100 | BM25 (default)| +:---------------------------------------|-----------| +[MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)| 0.1172 | +[MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)| 0.1170 | + + +R@100 | BM25 (default)| +:---------------------------------------|-----------| +[MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)| 0.5039 | +[MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)| 0.5158 | + + +R@1000 | BM25 (default)| +:---------------------------------------|-----------| +[MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)| 0.7647 | +[MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)| 0.7659 | diff --git a/docs/regressions-msmarco-v2-passage-augmented.md b/docs/regressions-msmarco-v2-passage-augmented.md index a330c7082f..cca3c12508 100644 --- a/docs/regressions-msmarco-v2-passage-augmented.md +++ b/docs/regressions-msmarco-v2-passage-augmented.md @@ -1,6 +1,7 @@ # Anserini: Regressions for [MS MARCO (V2) Passage Ranking](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) This page describes regression experiments for passage ranking _on the augmented version_ of the MS MARCO (V2) passage corpus using the dev queries, which is integrated into Anserini's regression testing framework. +Here, we cover bag-of-words baselines. For more complete instructions on how to run end-to-end experiments, refer to [this page](experiments-msmarco-v2.md). The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/msmarco-v2-passage-augmented.yaml). diff --git a/docs/regressions-msmarco-v2-passage-d2q-t5.md b/docs/regressions-msmarco-v2-passage-d2q-t5.md new file mode 100644 index 0000000000..77a929668a --- /dev/null +++ b/docs/regressions-msmarco-v2-passage-d2q-t5.md @@ -0,0 +1,83 @@ +# Anserini: Regressions for [MS MARCO (V2) Passage Ranking](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) + +This page describes regression experiments for passage ranking on the MS MARCO (V2) passage corpus using the dev queries, which is integrated into Anserini's regression testing framework. +Here, we expand the passage corpus with doc2query-T5. + +The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/msmarco-v2-passage-d2q-t5.yaml). +Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/msmarco-v2-passage-d2q-t5.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +## Indexing + +Typical indexing command: + +``` +target/appassembler/bin/IndexCollection \ + -collection MsMarcoV2PassageCollection \ + -input /path/to/msmarco-v2-passage-d2q-t5 \ + -index indexes/lucene-index.msmarco-v2-passage-d2q-t5/ \ + -generator DefaultLuceneDocumentGenerator \ + -threads 18 -storePositions -storeDocvectors -storeRaw \ + >& logs/log.msmarco-v2-passage-d2q-t5 & +``` + +The directory `/path/to/msmarco-v2-passage-d2q-t5/` should be a directory containing the compressed `jsonl` files that comprise the corpus. + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +Topics and qrels are stored in [`src/main/resources/topics-and-qrels/`](../src/main/resources/topics-and-qrels/). +These regression experiments use the [dev queries](../src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev.txt) and the [dev2 queries](../src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev2.txt). + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-passage-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev.txt -topicreader TsvInt \ + -output runs/run.msmarco-v2-passage-d2q-t5.bm25-default.topics.msmarco-v2-passage.dev.txt \ + -bm25 & +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-passage-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev2.txt -topicreader TsvInt \ + -output runs/run.msmarco-v2-passage-d2q-t5.bm25-default.topics.msmarco-v2-passage.dev2.txt \ + -bm25 & +``` + +Evaluation can be performed using `trec_eval`: + +``` +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default.topics.msmarco-v2-passage.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default.topics.msmarco-v2-passage.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default.topics.msmarco-v2-passage.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default.topics.msmarco-v2-passage.dev2.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default.topics.msmarco-v2-passage.dev2.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default.topics.msmarco-v2-passage.dev2.txt +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +MAP@100 | BM25 (default)| +:---------------------------------------|-----------| +[MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)| 0.1057 | +[MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)| 0.1112 | + + +MRR@100 | BM25 (default)| +:---------------------------------------|-----------| +[MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)| 0.1072 | +[MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)| 0.1123 | + + +R@100 | BM25 (default)| +:---------------------------------------|-----------| +[MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)| 0.4670 | +[MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)| 0.4803 | + + +R@1000 | BM25 (default)| +:---------------------------------------|-----------| +[MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)| 0.7083 | +[MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html)| 0.7151 | diff --git a/src/main/resources/docgen/templates/dl21-passage-augmented-d2q-t5.template b/src/main/resources/docgen/templates/dl21-passage-augmented-d2q-t5.template new file mode 100644 index 0000000000..bc2edb8704 --- /dev/null +++ b/src/main/resources/docgen/templates/dl21-passage-augmented-d2q-t5.template @@ -0,0 +1,50 @@ +# Anserini: Regressions for [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) + +This page describes document expansion experiments (with doc2query-T5), integrated into Anserini's regression testing framework, for the TREC 2021 Deep Learning Track (Passage Ranking Task) on the MS MARCO V2 _augmented_ passage collection using relevance judgments from NIST. + +At the time this regression was created (November 2021), the qrels are only available to TREC participants. +You must download the qrels from NIST's "active participants" password-protected site and place at `src/main/resources/topics-and-qrels/qrels.dl21-passage.txt`. +The qrels will be added to Anserini when they are publicly released in Spring 2022. + +Note that the NIST relevance judgments provide far more relevant passages per topic, unlike the "sparse" judgments provided by Microsoft (these are sometimes called "dense" judgments to emphasize this contrast). +For additional instructions on working with MS MARCO passage collection, refer to [this page](experiments-msmarco-v2.md). + +The exact configurations for these regressions are stored in [this YAML file](${yaml}). +Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +## Indexing + +Typical indexing command: + +``` +${index_cmds} +``` + +The value of `-input` should be a directory containing the compressed `jsonl` files that comprise the corpus. +See [this page](experiments-msmarco-v2.md) for additional details. + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +Topics and qrels are stored in [`src/main/resources/topics-and-qrels/`](../src/main/resources/topics-and-qrels/). +The regression experiments here evaluate on the 53 topics for which NIST has provided judgments as part of the TREC 2021 Deep Learning Track. + + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +${ranking_cmds} +``` + +Evaluation can be performed using `trec_eval`: + +``` +${eval_cmds} +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +${effectiveness} diff --git a/src/main/resources/docgen/templates/dl21-passage-d2q-t5.template b/src/main/resources/docgen/templates/dl21-passage-d2q-t5.template new file mode 100644 index 0000000000..f8ad2a4ec8 --- /dev/null +++ b/src/main/resources/docgen/templates/dl21-passage-d2q-t5.template @@ -0,0 +1,50 @@ +# Anserini: Regressions for [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) + +This page describes document expansion experiments (with doc2query-T5), integrated into Anserini's regression testing framework, for the TREC 2021 Deep Learning Track (Passage Ranking Task) on the MS MARCO V2 passage collection using relevance judgments from NIST. + +At the time this regression was created (November 2021), the qrels are only available to TREC participants. +You must download the qrels from NIST's "active participants" password-protected site and place at `src/main/resources/topics-and-qrels/qrels.dl21-passage.txt`. +The qrels will be added to Anserini when they are publicly released in Spring 2022. + +Note that the NIST relevance judgments provide far more relevant passages per topic, unlike the "sparse" judgments provided by Microsoft (these are sometimes called "dense" judgments to emphasize this contrast). +For additional instructions on working with MS MARCO passage collection, refer to [this page](experiments-msmarco-v2.md). + +The exact configurations for these regressions are stored in [this YAML file](${yaml}). +Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +## Indexing + +Typical indexing command: + +``` +${index_cmds} +``` + +The value of `-input` should be a directory containing the compressed `jsonl` files that comprise the corpus. +See [this page](experiments-msmarco-v2.md) for additional details. + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +Topics and qrels are stored in [`src/main/resources/topics-and-qrels/`](../src/main/resources/topics-and-qrels/). +The regression experiments here evaluate on the 53 topics for which NIST has provided judgments as part of the TREC 2021 Deep Learning Track. + + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +${ranking_cmds} +``` + +Evaluation can be performed using `trec_eval`: + +``` +${eval_cmds} +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +${effectiveness} diff --git a/src/main/resources/docgen/templates/msmarco-v2-passage-augmented-d2q-t5.template b/src/main/resources/docgen/templates/msmarco-v2-passage-augmented-d2q-t5.template new file mode 100644 index 0000000000..cd1dbae133 --- /dev/null +++ b/src/main/resources/docgen/templates/msmarco-v2-passage-augmented-d2q-t5.template @@ -0,0 +1,42 @@ +# Anserini: Regressions for [MS MARCO (V2) Passage Ranking](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) + +This page describes regression experiments for passage ranking _on the augmented version_ of the MS MARCO (V2) passage corpus using the dev queries, which is integrated into Anserini's regression testing framework. +Here, we expand the augmented passage corpus with doc2query-T5. + +The exact configurations for these regressions are stored in [this YAML file](${yaml}). +Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +## Indexing + +Typical indexing command: + +``` +${index_cmds} +``` + +The directory `/path/to/msmarco-v2-passage-augmented-d2q-t5/` should be a directory containing the compressed `jsonl` files that comprise the corpus. + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +Topics and qrels are stored in [`src/main/resources/topics-and-qrels/`](../src/main/resources/topics-and-qrels/). +These regression experiments use the [dev queries](../src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev.txt) and the [dev2 queries](../src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev2.txt). + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +${ranking_cmds} +``` + +Evaluation can be performed using `trec_eval`: + +``` +${eval_cmds} +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +${effectiveness} diff --git a/src/main/resources/docgen/templates/msmarco-v2-passage-augmented.template b/src/main/resources/docgen/templates/msmarco-v2-passage-augmented.template index f9b0645182..ba5188fe21 100644 --- a/src/main/resources/docgen/templates/msmarco-v2-passage-augmented.template +++ b/src/main/resources/docgen/templates/msmarco-v2-passage-augmented.template @@ -1,6 +1,7 @@ # Anserini: Regressions for [MS MARCO (V2) Passage Ranking](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) This page describes regression experiments for passage ranking _on the augmented version_ of the MS MARCO (V2) passage corpus using the dev queries, which is integrated into Anserini's regression testing framework. +Here, we cover bag-of-words baselines. For more complete instructions on how to run end-to-end experiments, refer to [this page](experiments-msmarco-v2.md). The exact configurations for these regressions are stored in [this YAML file](${yaml}). diff --git a/src/main/resources/docgen/templates/msmarco-v2-passage-d2q-t5.template b/src/main/resources/docgen/templates/msmarco-v2-passage-d2q-t5.template new file mode 100644 index 0000000000..2817a61775 --- /dev/null +++ b/src/main/resources/docgen/templates/msmarco-v2-passage-d2q-t5.template @@ -0,0 +1,42 @@ +# Anserini: Regressions for [MS MARCO (V2) Passage Ranking](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) + +This page describes regression experiments for passage ranking on the MS MARCO (V2) passage corpus using the dev queries, which is integrated into Anserini's regression testing framework. +Here, we expand the passage corpus with doc2query-T5. + +The exact configurations for these regressions are stored in [this YAML file](${yaml}). +Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. + +## Indexing + +Typical indexing command: + +``` +${index_cmds} +``` + +The directory `/path/to/msmarco-v2-passage-d2q-t5/` should be a directory containing the compressed `jsonl` files that comprise the corpus. + +For additional details, see explanation of [common indexing options](common-indexing-options.md). + +## Retrieval + +Topics and qrels are stored in [`src/main/resources/topics-and-qrels/`](../src/main/resources/topics-and-qrels/). +These regression experiments use the [dev queries](../src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev.txt) and the [dev2 queries](../src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev2.txt). + +After indexing has completed, you should be able to perform retrieval as follows: + +``` +${ranking_cmds} +``` + +Evaluation can be performed using `trec_eval`: + +``` +${eval_cmds} +``` + +## Effectiveness + +With the above commands, you should be able to reproduce the following results: + +${effectiveness} From c380d836aebeae963dda10f0d68b68078e65be8e Mon Sep 17 00:00:00 2001 From: lintool Date: Thu, 20 Jan 2022 09:20:03 -0500 Subject: [PATCH 10/10] addresses PR --- docs/regressions-dl21-doc-d2q-t5.md | 2 +- docs/regressions-dl21-doc-segmented-d2q-t5.md | 2 +- docs/regressions-dl21-doc-segmented-unicoil-noexp-0shot.md | 2 +- docs/regressions-dl21-doc-segmented.md | 2 +- docs/regressions-dl21-doc.md | 2 +- docs/regressions-dl21-passage-augmented-d2q-t5.md | 2 +- docs/regressions-dl21-passage-augmented.md | 2 +- docs/regressions-dl21-passage-d2q-t5.md | 2 +- docs/regressions-dl21-passage-unicoil-noexp-0shot.md | 2 +- docs/regressions-dl21-passage.md | 2 +- src/main/resources/docgen/templates/dl21-doc-d2q-t5.template | 2 +- .../docgen/templates/dl21-doc-segmented-d2q-t5.template | 2 +- .../templates/dl21-doc-segmented-unicoil-noexp-0shot.template | 2 +- src/main/resources/docgen/templates/dl21-doc-segmented.template | 2 +- src/main/resources/docgen/templates/dl21-doc.template | 2 +- .../docgen/templates/dl21-passage-augmented-d2q-t5.template | 2 +- .../resources/docgen/templates/dl21-passage-augmented.template | 2 +- .../resources/docgen/templates/dl21-passage-d2q-t5.template | 2 +- .../docgen/templates/dl21-passage-unicoil-noexp-0shot.template | 2 +- src/main/resources/docgen/templates/dl21-passage.template | 2 +- 20 files changed, 20 insertions(+), 20 deletions(-) diff --git a/docs/regressions-dl21-doc-d2q-t5.md b/docs/regressions-dl21-doc-d2q-t5.md index a5e2ac4558..da6735ca63 100644 --- a/docs/regressions-dl21-doc-d2q-t5.md +++ b/docs/regressions-dl21-doc-d2q-t5.md @@ -7,7 +7,7 @@ You must download the qrels from NIST's "active participants" password-protected The qrels will be added to Anserini when they are publicly released in Spring 2022. Note that the NIST relevance judgments provide far more relevant documents per topic, unlike the "sparse" judgments provided by Microsoft (these are sometimes called "dense" judgments to emphasize this contrast). -For additional instructions on working with MS MARCO document collection, refer to [this page](experiments-msmarco-v2.md). +For additional instructions on working with MS MARCO V2 document collection, refer to [this page](experiments-msmarco-v2.md). Note that there are four different regression conditions for this task, and this page describes the following: diff --git a/docs/regressions-dl21-doc-segmented-d2q-t5.md b/docs/regressions-dl21-doc-segmented-d2q-t5.md index 3dfa87ba2e..dfe74717bf 100644 --- a/docs/regressions-dl21-doc-segmented-d2q-t5.md +++ b/docs/regressions-dl21-doc-segmented-d2q-t5.md @@ -7,7 +7,7 @@ You must download the qrels from NIST's "active participants" password-protected The qrels will be added to Anserini when they are publicly released in Spring 2022. Note that the NIST relevance judgments provide far more relevant documents per topic, unlike the "sparse" judgments provided by Microsoft (these are sometimes called "dense" judgments to emphasize this contrast). -For additional instructions on working with MS MARCO document collection, refer to [this page](experiments-msmarco-v2.md). +For additional instructions on working with MS MARCO V2 document collection, refer to [this page](experiments-msmarco-v2.md). Note that there are four different regression conditions for this task, and this page describes the following: diff --git a/docs/regressions-dl21-doc-segmented-unicoil-noexp-0shot.md b/docs/regressions-dl21-doc-segmented-unicoil-noexp-0shot.md index adf55fbd98..c4c1db6ca2 100644 --- a/docs/regressions-dl21-doc-segmented-unicoil-noexp-0shot.md +++ b/docs/regressions-dl21-doc-segmented-unicoil-noexp-0shot.md @@ -8,7 +8,7 @@ You must download the qrels from NIST's "active participants" password-protected The qrels will be added to Anserini when they are publicly released in Spring 2022. Note that the NIST relevance judgments provide far more relevant documents per topic, unlike the "sparse" judgments provided by Microsoft (these are sometimes called "dense" judgments to emphasize this contrast). -For additional instructions on working with MS MARCO document collection, refer to [this page](experiments-msmarco-v2.md). +For additional instructions on working with MS MARCO V2 document collection, refer to [this page](experiments-msmarco-v2.md). The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/dl21-doc-segmented-unicoil-noexp-0shot.yaml). Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/dl21-doc-segmented-unicoil-noexp-0shot.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. diff --git a/docs/regressions-dl21-doc-segmented.md b/docs/regressions-dl21-doc-segmented.md index ae1670692d..74a502da09 100644 --- a/docs/regressions-dl21-doc-segmented.md +++ b/docs/regressions-dl21-doc-segmented.md @@ -7,7 +7,7 @@ You must download the qrels from NIST's "active participants" password-protected The qrels will be added to Anserini when they are publicly released in Spring 2022. Note that the NIST relevance judgments provide far more relevant documents per topic, unlike the "sparse" judgments provided by Microsoft (these are sometimes called "dense" judgments to emphasize this contrast). -For additional instructions on working with MS MARCO document collection, refer to [this page](experiments-msmarco-v2.md). +For additional instructions on working with MS MARCO V2 document collection, refer to [this page](experiments-msmarco-v2.md). Note that there are four different regression conditions for this task, and this page describes the following: diff --git a/docs/regressions-dl21-doc.md b/docs/regressions-dl21-doc.md index 33e15b71a1..9d65860dd4 100644 --- a/docs/regressions-dl21-doc.md +++ b/docs/regressions-dl21-doc.md @@ -7,7 +7,7 @@ You must download the qrels from NIST's "active participants" password-protected The qrels will be added to Anserini when they are publicly released in Spring 2022. Note that the NIST relevance judgments provide far more relevant documents per topic, unlike the "sparse" judgments provided by Microsoft (these are sometimes called "dense" judgments to emphasize this contrast). -For additional instructions on working with MS MARCO document collection, refer to [this page](experiments-msmarco-v2.md). +For additional instructions on working with MS MARCO V2 document collection, refer to [this page](experiments-msmarco-v2.md). Note that there are four different regression conditions for this task, and this page describes the following: diff --git a/docs/regressions-dl21-passage-augmented-d2q-t5.md b/docs/regressions-dl21-passage-augmented-d2q-t5.md index 67ce5c8598..b57c40dc05 100644 --- a/docs/regressions-dl21-passage-augmented-d2q-t5.md +++ b/docs/regressions-dl21-passage-augmented-d2q-t5.md @@ -7,7 +7,7 @@ You must download the qrels from NIST's "active participants" password-protected The qrels will be added to Anserini when they are publicly released in Spring 2022. Note that the NIST relevance judgments provide far more relevant passages per topic, unlike the "sparse" judgments provided by Microsoft (these are sometimes called "dense" judgments to emphasize this contrast). -For additional instructions on working with MS MARCO passage collection, refer to [this page](experiments-msmarco-v2.md). +For additional instructions on working with MS MARCO V2 passage collection, refer to [this page](experiments-msmarco-v2.md). The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/dl21-passage-augmented-d2q-t5.yaml). Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/dl21-passage-augmented-d2q-t5.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. diff --git a/docs/regressions-dl21-passage-augmented.md b/docs/regressions-dl21-passage-augmented.md index 7f741f8b31..2c0df50f15 100644 --- a/docs/regressions-dl21-passage-augmented.md +++ b/docs/regressions-dl21-passage-augmented.md @@ -7,7 +7,7 @@ You must download the qrels from NIST's "active participants" password-protected The qrels will be added to Anserini when they are publicly released in Spring 2022. Note that the NIST relevance judgments provide far more relevant passages per topic, unlike the "sparse" judgments provided by Microsoft (these are sometimes called "dense" judgments to emphasize this contrast). -For additional instructions on working with MS MARCO passage collection, refer to [this page](experiments-msmarco-v2.md). +For additional instructions on working with MS MARCO V2 passage collection, refer to [this page](experiments-msmarco-v2.md). The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/dl21-passage-augmented.yaml). Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/dl21-passage-augmented.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. diff --git a/docs/regressions-dl21-passage-d2q-t5.md b/docs/regressions-dl21-passage-d2q-t5.md index a6d816d48f..50be74166e 100644 --- a/docs/regressions-dl21-passage-d2q-t5.md +++ b/docs/regressions-dl21-passage-d2q-t5.md @@ -7,7 +7,7 @@ You must download the qrels from NIST's "active participants" password-protected The qrels will be added to Anserini when they are publicly released in Spring 2022. Note that the NIST relevance judgments provide far more relevant passages per topic, unlike the "sparse" judgments provided by Microsoft (these are sometimes called "dense" judgments to emphasize this contrast). -For additional instructions on working with MS MARCO passage collection, refer to [this page](experiments-msmarco-v2.md). +For additional instructions on working with MS MARCO V2 passage collection, refer to [this page](experiments-msmarco-v2.md). The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/dl21-passage-d2q-t5.yaml). Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/dl21-passage-d2q-t5.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. diff --git a/docs/regressions-dl21-passage-unicoil-noexp-0shot.md b/docs/regressions-dl21-passage-unicoil-noexp-0shot.md index 72d270a653..b0f71d56bc 100644 --- a/docs/regressions-dl21-passage-unicoil-noexp-0shot.md +++ b/docs/regressions-dl21-passage-unicoil-noexp-0shot.md @@ -8,7 +8,7 @@ You must download the qrels from NIST's "active participants" password-protected The qrels will be added to Anserini when they are publicly released in Spring 2022. Note that the NIST relevance judgments provide far more relevant passages per topic, unlike the "sparse" judgments provided by Microsoft (these are sometimes called "dense" judgments to emphasize this contrast). -For additional instructions on working with MS MARCO passage collection, refer to [this page](experiments-msmarco-v2.md). +For additional instructions on working with MS MARCO V2 passage collection, refer to [this page](experiments-msmarco-v2.md). The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/dl21-passage-unicoil-noexp-0shot.yaml). Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/dl21-passage-unicoil-noexp-0shot.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. diff --git a/docs/regressions-dl21-passage.md b/docs/regressions-dl21-passage.md index ec3bb511ac..0ebfd3043c 100644 --- a/docs/regressions-dl21-passage.md +++ b/docs/regressions-dl21-passage.md @@ -7,7 +7,7 @@ You must download the qrels from NIST's "active participants" password-protected The qrels will be added to Anserini when they are publicly released in Spring 2022. Note that the NIST relevance judgments provide far more relevant passages per topic, unlike the "sparse" judgments provided by Microsoft (these are sometimes called "dense" judgments to emphasize this contrast). -For additional instructions on working with MS MARCO passage collection, refer to [this page](experiments-msmarco-v2.md). +For additional instructions on working with MS MARCO V2 passage collection, refer to [this page](experiments-msmarco-v2.md). The exact configurations for these regressions are stored in [this YAML file](../src/main/resources/regression/dl21-passage.yaml). Note that this page is automatically generated from [this template](../src/main/resources/docgen/templates/dl21-passage.template) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. diff --git a/src/main/resources/docgen/templates/dl21-doc-d2q-t5.template b/src/main/resources/docgen/templates/dl21-doc-d2q-t5.template index 5d86766ea8..66d8b76102 100644 --- a/src/main/resources/docgen/templates/dl21-doc-d2q-t5.template +++ b/src/main/resources/docgen/templates/dl21-doc-d2q-t5.template @@ -7,7 +7,7 @@ You must download the qrels from NIST's "active participants" password-protected The qrels will be added to Anserini when they are publicly released in Spring 2022. Note that the NIST relevance judgments provide far more relevant documents per topic, unlike the "sparse" judgments provided by Microsoft (these are sometimes called "dense" judgments to emphasize this contrast). -For additional instructions on working with MS MARCO document collection, refer to [this page](experiments-msmarco-v2.md). +For additional instructions on working with MS MARCO V2 document collection, refer to [this page](experiments-msmarco-v2.md). Note that there are four different regression conditions for this task, and this page describes the following: diff --git a/src/main/resources/docgen/templates/dl21-doc-segmented-d2q-t5.template b/src/main/resources/docgen/templates/dl21-doc-segmented-d2q-t5.template index 22c3c30880..ef17ecc67d 100644 --- a/src/main/resources/docgen/templates/dl21-doc-segmented-d2q-t5.template +++ b/src/main/resources/docgen/templates/dl21-doc-segmented-d2q-t5.template @@ -7,7 +7,7 @@ You must download the qrels from NIST's "active participants" password-protected The qrels will be added to Anserini when they are publicly released in Spring 2022. Note that the NIST relevance judgments provide far more relevant documents per topic, unlike the "sparse" judgments provided by Microsoft (these are sometimes called "dense" judgments to emphasize this contrast). -For additional instructions on working with MS MARCO document collection, refer to [this page](experiments-msmarco-v2.md). +For additional instructions on working with MS MARCO V2 document collection, refer to [this page](experiments-msmarco-v2.md). Note that there are four different regression conditions for this task, and this page describes the following: diff --git a/src/main/resources/docgen/templates/dl21-doc-segmented-unicoil-noexp-0shot.template b/src/main/resources/docgen/templates/dl21-doc-segmented-unicoil-noexp-0shot.template index 42b9107b91..792ef05f42 100644 --- a/src/main/resources/docgen/templates/dl21-doc-segmented-unicoil-noexp-0shot.template +++ b/src/main/resources/docgen/templates/dl21-doc-segmented-unicoil-noexp-0shot.template @@ -8,7 +8,7 @@ You must download the qrels from NIST's "active participants" password-protected The qrels will be added to Anserini when they are publicly released in Spring 2022. Note that the NIST relevance judgments provide far more relevant documents per topic, unlike the "sparse" judgments provided by Microsoft (these are sometimes called "dense" judgments to emphasize this contrast). -For additional instructions on working with MS MARCO document collection, refer to [this page](experiments-msmarco-v2.md). +For additional instructions on working with MS MARCO V2 document collection, refer to [this page](experiments-msmarco-v2.md). The exact configurations for these regressions are stored in [this YAML file](${yaml}). Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. diff --git a/src/main/resources/docgen/templates/dl21-doc-segmented.template b/src/main/resources/docgen/templates/dl21-doc-segmented.template index 15f60305a6..70fa5d42fe 100644 --- a/src/main/resources/docgen/templates/dl21-doc-segmented.template +++ b/src/main/resources/docgen/templates/dl21-doc-segmented.template @@ -7,7 +7,7 @@ You must download the qrels from NIST's "active participants" password-protected The qrels will be added to Anserini when they are publicly released in Spring 2022. Note that the NIST relevance judgments provide far more relevant documents per topic, unlike the "sparse" judgments provided by Microsoft (these are sometimes called "dense" judgments to emphasize this contrast). -For additional instructions on working with MS MARCO document collection, refer to [this page](experiments-msmarco-v2.md). +For additional instructions on working with MS MARCO V2 document collection, refer to [this page](experiments-msmarco-v2.md). Note that there are four different regression conditions for this task, and this page describes the following: diff --git a/src/main/resources/docgen/templates/dl21-doc.template b/src/main/resources/docgen/templates/dl21-doc.template index c4de07935a..5be9e5c340 100644 --- a/src/main/resources/docgen/templates/dl21-doc.template +++ b/src/main/resources/docgen/templates/dl21-doc.template @@ -7,7 +7,7 @@ You must download the qrels from NIST's "active participants" password-protected The qrels will be added to Anserini when they are publicly released in Spring 2022. Note that the NIST relevance judgments provide far more relevant documents per topic, unlike the "sparse" judgments provided by Microsoft (these are sometimes called "dense" judgments to emphasize this contrast). -For additional instructions on working with MS MARCO document collection, refer to [this page](experiments-msmarco-v2.md). +For additional instructions on working with MS MARCO V2 document collection, refer to [this page](experiments-msmarco-v2.md). Note that there are four different regression conditions for this task, and this page describes the following: diff --git a/src/main/resources/docgen/templates/dl21-passage-augmented-d2q-t5.template b/src/main/resources/docgen/templates/dl21-passage-augmented-d2q-t5.template index bc2edb8704..2169ebdafa 100644 --- a/src/main/resources/docgen/templates/dl21-passage-augmented-d2q-t5.template +++ b/src/main/resources/docgen/templates/dl21-passage-augmented-d2q-t5.template @@ -7,7 +7,7 @@ You must download the qrels from NIST's "active participants" password-protected The qrels will be added to Anserini when they are publicly released in Spring 2022. Note that the NIST relevance judgments provide far more relevant passages per topic, unlike the "sparse" judgments provided by Microsoft (these are sometimes called "dense" judgments to emphasize this contrast). -For additional instructions on working with MS MARCO passage collection, refer to [this page](experiments-msmarco-v2.md). +For additional instructions on working with MS MARCO V2 passage collection, refer to [this page](experiments-msmarco-v2.md). The exact configurations for these regressions are stored in [this YAML file](${yaml}). Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. diff --git a/src/main/resources/docgen/templates/dl21-passage-augmented.template b/src/main/resources/docgen/templates/dl21-passage-augmented.template index 41394d91ba..10734cd6b5 100644 --- a/src/main/resources/docgen/templates/dl21-passage-augmented.template +++ b/src/main/resources/docgen/templates/dl21-passage-augmented.template @@ -7,7 +7,7 @@ You must download the qrels from NIST's "active participants" password-protected The qrels will be added to Anserini when they are publicly released in Spring 2022. Note that the NIST relevance judgments provide far more relevant passages per topic, unlike the "sparse" judgments provided by Microsoft (these are sometimes called "dense" judgments to emphasize this contrast). -For additional instructions on working with MS MARCO passage collection, refer to [this page](experiments-msmarco-v2.md). +For additional instructions on working with MS MARCO V2 passage collection, refer to [this page](experiments-msmarco-v2.md). The exact configurations for these regressions are stored in [this YAML file](${yaml}). Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. diff --git a/src/main/resources/docgen/templates/dl21-passage-d2q-t5.template b/src/main/resources/docgen/templates/dl21-passage-d2q-t5.template index f8ad2a4ec8..77888eb481 100644 --- a/src/main/resources/docgen/templates/dl21-passage-d2q-t5.template +++ b/src/main/resources/docgen/templates/dl21-passage-d2q-t5.template @@ -7,7 +7,7 @@ You must download the qrels from NIST's "active participants" password-protected The qrels will be added to Anserini when they are publicly released in Spring 2022. Note that the NIST relevance judgments provide far more relevant passages per topic, unlike the "sparse" judgments provided by Microsoft (these are sometimes called "dense" judgments to emphasize this contrast). -For additional instructions on working with MS MARCO passage collection, refer to [this page](experiments-msmarco-v2.md). +For additional instructions on working with MS MARCO V2 passage collection, refer to [this page](experiments-msmarco-v2.md). The exact configurations for these regressions are stored in [this YAML file](${yaml}). Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. diff --git a/src/main/resources/docgen/templates/dl21-passage-unicoil-noexp-0shot.template b/src/main/resources/docgen/templates/dl21-passage-unicoil-noexp-0shot.template index 319fedea5a..8582418bff 100644 --- a/src/main/resources/docgen/templates/dl21-passage-unicoil-noexp-0shot.template +++ b/src/main/resources/docgen/templates/dl21-passage-unicoil-noexp-0shot.template @@ -8,7 +8,7 @@ You must download the qrels from NIST's "active participants" password-protected The qrels will be added to Anserini when they are publicly released in Spring 2022. Note that the NIST relevance judgments provide far more relevant passages per topic, unlike the "sparse" judgments provided by Microsoft (these are sometimes called "dense" judgments to emphasize this contrast). -For additional instructions on working with MS MARCO passage collection, refer to [this page](experiments-msmarco-v2.md). +For additional instructions on working with MS MARCO V2 passage collection, refer to [this page](experiments-msmarco-v2.md). The exact configurations for these regressions are stored in [this YAML file](${yaml}). Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead. diff --git a/src/main/resources/docgen/templates/dl21-passage.template b/src/main/resources/docgen/templates/dl21-passage.template index d5500ab64d..fa5b804409 100644 --- a/src/main/resources/docgen/templates/dl21-passage.template +++ b/src/main/resources/docgen/templates/dl21-passage.template @@ -7,7 +7,7 @@ You must download the qrels from NIST's "active participants" password-protected The qrels will be added to Anserini when they are publicly released in Spring 2022. Note that the NIST relevance judgments provide far more relevant passages per topic, unlike the "sparse" judgments provided by Microsoft (these are sometimes called "dense" judgments to emphasize this contrast). -For additional instructions on working with MS MARCO passage collection, refer to [this page](experiments-msmarco-v2.md). +For additional instructions on working with MS MARCO V2 passage collection, refer to [this page](experiments-msmarco-v2.md). The exact configurations for these regressions are stored in [this YAML file](${yaml}). Note that this page is automatically generated from [this template](${template}) as part of Anserini's regression pipeline, so do not modify this page directly; modify the template instead.