diff --git a/docs/regressions-msmarco-doc-docTTTTTquery.md b/docs/regressions-msmarco-doc-docTTTTTquery.md index f60b95ed16..7ccb7f77b8 100644 --- a/docs/regressions-msmarco-doc-docTTTTTquery.md +++ b/docs/regressions-msmarco-doc-docTTTTTquery.md @@ -57,12 +57,26 @@ target/appassembler/bin/SearchCollection \ -output runs/run.msmarco-doc-docTTTTTquery.bm25-default.topics.msmarco-doc.dev.txt \ -bm25 & +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-doc-docTTTTTquery/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-doc.dev.txt \ + -topicreader TsvInt \ + -output runs/run.msmarco-doc-docTTTTTquery.bm25-default+rm3.topics.msmarco-doc.dev.txt \ + -bm25 -rm3 & + target/appassembler/bin/SearchCollection \ -index indexes/lucene-index.msmarco-doc-docTTTTTquery/ \ -topics src/main/resources/topics-and-qrels/topics.msmarco-doc.dev.txt \ -topicreader TsvInt \ -output runs/run.msmarco-doc-docTTTTTquery.bm25-tuned.topics.msmarco-doc.dev.txt \ -bm25 -bm25.k1 4.68 -bm25.b 0.87 & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-doc-docTTTTTquery/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-doc.dev.txt \ + -topicreader TsvInt \ + -output runs/run.msmarco-doc-docTTTTTquery.bm25-tuned+rm3.topics.msmarco-doc.dev.txt \ + -bm25 -bm25.k1 4.68 -bm25.b 0.87 -rm3 & ``` Evaluation can be performed using `trec_eval`: @@ -73,34 +87,44 @@ tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank src/main/resources/ tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-doc.dev.txt runs/run.msmarco-doc-docTTTTTquery.bm25-default.topics.msmarco-doc.dev.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-doc.dev.txt runs/run.msmarco-doc-docTTTTTquery.bm25-default.topics.msmarco-doc.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m map src/main/resources/topics-and-qrels/qrels.msmarco-doc.dev.txt runs/run.msmarco-doc-docTTTTTquery.bm25-default+rm3.topics.msmarco-doc.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-doc.dev.txt runs/run.msmarco-doc-docTTTTTquery.bm25-default+rm3.topics.msmarco-doc.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-doc.dev.txt runs/run.msmarco-doc-docTTTTTquery.bm25-default+rm3.topics.msmarco-doc.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-doc.dev.txt runs/run.msmarco-doc-docTTTTTquery.bm25-default+rm3.topics.msmarco-doc.dev.txt + tools/eval/trec_eval.9.0.4/trec_eval -c -m map src/main/resources/topics-and-qrels/qrels.msmarco-doc.dev.txt runs/run.msmarco-doc-docTTTTTquery.bm25-tuned.topics.msmarco-doc.dev.txt tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-doc.dev.txt runs/run.msmarco-doc-docTTTTTquery.bm25-tuned.topics.msmarco-doc.dev.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-doc.dev.txt runs/run.msmarco-doc-docTTTTTquery.bm25-tuned.topics.msmarco-doc.dev.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-doc.dev.txt runs/run.msmarco-doc-docTTTTTquery.bm25-tuned.topics.msmarco-doc.dev.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -m map src/main/resources/topics-and-qrels/qrels.msmarco-doc.dev.txt runs/run.msmarco-doc-docTTTTTquery.bm25-tuned+rm3.topics.msmarco-doc.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-doc.dev.txt runs/run.msmarco-doc-docTTTTTquery.bm25-tuned+rm3.topics.msmarco-doc.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-doc.dev.txt runs/run.msmarco-doc-docTTTTTquery.bm25-tuned+rm3.topics.msmarco-doc.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-doc.dev.txt runs/run.msmarco-doc-docTTTTTquery.bm25-tuned+rm3.topics.msmarco-doc.dev.txt ``` ## Effectiveness With the above commands, you should be able to reproduce the following results: -| AP@1000 | BM25 (default)| BM25 (tuned)| -|:-------------------------------------------------------------------------------------------------------------|-----------|-----------| -| [MS MARCO Doc: Dev](https://github.com/microsoft/MSMARCO-Document-Ranking) | 0.2886 | 0.3273 | +| AP@1000 | BM25 (default)| +RM3 | BM25 (tuned)| +RM3 | +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------|-----------| +| [MS MARCO Doc: Dev](https://github.com/microsoft/MSMARCO-Document-Ranking) | 0.2886 | 0.1839 | 0.3273 | 0.2627 | -| RR@100 | BM25 (default)| BM25 (tuned)| -|:-------------------------------------------------------------------------------------------------------------|-----------|-----------| -| [MS MARCO Doc: Dev](https://github.com/microsoft/MSMARCO-Document-Ranking) | 0.2880 | 0.3269 | +| RR@100 | BM25 (default)| +RM3 | BM25 (tuned)| +RM3 | +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------|-----------| +| [MS MARCO Doc: Dev](https://github.com/microsoft/MSMARCO-Document-Ranking) | 0.2880 | 0.1831 | 0.3269 | 0.2621 | -| R@100 | BM25 (default)| BM25 (tuned)| -|:-------------------------------------------------------------------------------------------------------------|-----------|-----------| -| [MS MARCO Doc: Dev](https://github.com/microsoft/MSMARCO-Document-Ranking) | 0.7993 | 0.8612 | +| R@100 | BM25 (default)| +RM3 | BM25 (tuned)| +RM3 | +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------|-----------| +| [MS MARCO Doc: Dev](https://github.com/microsoft/MSMARCO-Document-Ranking) | 0.7993 | 0.7420 | 0.8612 | 0.8379 | -| R@1000 | BM25 (default)| BM25 (tuned)| -|:-------------------------------------------------------------------------------------------------------------|-----------|-----------| -| [MS MARCO Doc: Dev](https://github.com/microsoft/MSMARCO-Document-Ranking) | 0.9259 | 0.9553 | +| R@1000 | BM25 (default)| +RM3 | BM25 (tuned)| +RM3 | +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------|-----------| +| [MS MARCO Doc: Dev](https://github.com/microsoft/MSMARCO-Document-Ranking) | 0.9259 | 0.9128 | 0.9553 | 0.9524 | Explanation of settings: diff --git a/docs/regressions-msmarco-doc-segmented-docTTTTTquery.md b/docs/regressions-msmarco-doc-segmented-docTTTTTquery.md index 09c1493c54..99c9d29a2b 100644 --- a/docs/regressions-msmarco-doc-segmented-docTTTTTquery.md +++ b/docs/regressions-msmarco-doc-segmented-docTTTTTquery.md @@ -58,12 +58,26 @@ target/appassembler/bin/SearchCollection \ -output runs/run.msmarco-doc-segmented-docTTTTTquery.bm25-default.topics.msmarco-doc.dev.txt \ -bm25 -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 & +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-doc-segmented-docTTTTTquery/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-doc.dev.txt \ + -topicreader TsvInt \ + -output runs/run.msmarco-doc-segmented-docTTTTTquery.bm25-default+rm3.topics.msmarco-doc.dev.txt \ + -bm25 -rm3 -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 & + target/appassembler/bin/SearchCollection \ -index indexes/lucene-index.msmarco-doc-segmented-docTTTTTquery/ \ -topics src/main/resources/topics-and-qrels/topics.msmarco-doc.dev.txt \ -topicreader TsvInt \ -output runs/run.msmarco-doc-segmented-docTTTTTquery.bm25-tuned.topics.msmarco-doc.dev.txt \ -bm25 -bm25.k1 2.56 -bm25.b 0.59 -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-doc-segmented-docTTTTTquery/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-doc.dev.txt \ + -topicreader TsvInt \ + -output runs/run.msmarco-doc-segmented-docTTTTTquery.bm25-tuned+rm3.topics.msmarco-doc.dev.txt \ + -bm25 -bm25.k1 2.56 -bm25.b 0.59 -rm3 -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 & ``` Evaluation can be performed using `trec_eval`: @@ -74,34 +88,44 @@ tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank src/main/resources/ tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-doc.dev.txt runs/run.msmarco-doc-segmented-docTTTTTquery.bm25-default.topics.msmarco-doc.dev.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-doc.dev.txt runs/run.msmarco-doc-segmented-docTTTTTquery.bm25-default.topics.msmarco-doc.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m map src/main/resources/topics-and-qrels/qrels.msmarco-doc.dev.txt runs/run.msmarco-doc-segmented-docTTTTTquery.bm25-default+rm3.topics.msmarco-doc.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-doc.dev.txt runs/run.msmarco-doc-segmented-docTTTTTquery.bm25-default+rm3.topics.msmarco-doc.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-doc.dev.txt runs/run.msmarco-doc-segmented-docTTTTTquery.bm25-default+rm3.topics.msmarco-doc.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-doc.dev.txt runs/run.msmarco-doc-segmented-docTTTTTquery.bm25-default+rm3.topics.msmarco-doc.dev.txt + tools/eval/trec_eval.9.0.4/trec_eval -c -m map src/main/resources/topics-and-qrels/qrels.msmarco-doc.dev.txt runs/run.msmarco-doc-segmented-docTTTTTquery.bm25-tuned.topics.msmarco-doc.dev.txt tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-doc.dev.txt runs/run.msmarco-doc-segmented-docTTTTTquery.bm25-tuned.topics.msmarco-doc.dev.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-doc.dev.txt runs/run.msmarco-doc-segmented-docTTTTTquery.bm25-tuned.topics.msmarco-doc.dev.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-doc.dev.txt runs/run.msmarco-doc-segmented-docTTTTTquery.bm25-tuned.topics.msmarco-doc.dev.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -m map src/main/resources/topics-and-qrels/qrels.msmarco-doc.dev.txt runs/run.msmarco-doc-segmented-docTTTTTquery.bm25-tuned+rm3.topics.msmarco-doc.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-doc.dev.txt runs/run.msmarco-doc-segmented-docTTTTTquery.bm25-tuned+rm3.topics.msmarco-doc.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-doc.dev.txt runs/run.msmarco-doc-segmented-docTTTTTquery.bm25-tuned+rm3.topics.msmarco-doc.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-doc.dev.txt runs/run.msmarco-doc-segmented-docTTTTTquery.bm25-tuned+rm3.topics.msmarco-doc.dev.txt ``` ## Effectiveness With the above commands, you should be able to reproduce the following results: -| AP@1000 | BM25 (default)| BM25 (tuned)| -|:-------------------------------------------------------------------------------------------------------------|-----------|-----------| -| [MS MARCO Doc: Dev](https://github.com/microsoft/MSMARCO-Document-Ranking) | 0.3184 | 0.3213 | +| AP@1000 | BM25 (default)| +RM3 | BM25 (tuned)| +RM3 | +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------|-----------| +| [MS MARCO Doc: Dev](https://github.com/microsoft/MSMARCO-Document-Ranking) | 0.3184 | 0.2823 | 0.3213 | 0.2989 | -| RR@100 | BM25 (default)| BM25 (tuned)| -|:-------------------------------------------------------------------------------------------------------------|-----------|-----------| -| [MS MARCO Doc: Dev](https://github.com/microsoft/MSMARCO-Document-Ranking) | 0.3179 | 0.3209 | +| RR@100 | BM25 (default)| +RM3 | BM25 (tuned)| +RM3 | +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------|-----------| +| [MS MARCO Doc: Dev](https://github.com/microsoft/MSMARCO-Document-Ranking) | 0.3179 | 0.2818 | 0.3209 | 0.2985 | -| R@100 | BM25 (default)| BM25 (tuned)| -|:-------------------------------------------------------------------------------------------------------------|-----------|-----------| -| [MS MARCO Doc: Dev](https://github.com/microsoft/MSMARCO-Document-Ranking) | 0.8479 | 0.8627 | +| R@100 | BM25 (default)| +RM3 | BM25 (tuned)| +RM3 | +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------|-----------| +| [MS MARCO Doc: Dev](https://github.com/microsoft/MSMARCO-Document-Ranking) | 0.8479 | 0.8479 | 0.8627 | 0.8556 | -| R@1000 | BM25 (default)| BM25 (tuned)| -|:-------------------------------------------------------------------------------------------------------------|-----------|-----------| -| [MS MARCO Doc: Dev](https://github.com/microsoft/MSMARCO-Document-Ranking) | 0.9490 | 0.9530 | +| R@1000 | BM25 (default)| +RM3 | BM25 (tuned)| +RM3 | +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------|-----------| +| [MS MARCO Doc: Dev](https://github.com/microsoft/MSMARCO-Document-Ranking) | 0.9490 | 0.9547 | 0.9530 | 0.9567 | Explanation of settings: diff --git a/docs/regressions-msmarco-v2-doc-d2q-t5.md b/docs/regressions-msmarco-v2-doc-d2q-t5.md index 093073ba84..86419c3cb4 100644 --- a/docs/regressions-msmarco-v2-doc-d2q-t5.md +++ b/docs/regressions-msmarco-v2-doc-d2q-t5.md @@ -53,6 +53,19 @@ target/appassembler/bin/SearchCollection \ -topicreader TsvInt \ -output runs/run.msmarco-v2-doc-d2q-t5.bm25-default.topics.msmarco-v2-doc.dev2.txt \ -bm25 & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-doc-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev.txt \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rm3.topics.msmarco-v2-doc.dev.txt \ + -bm25 -rm3 & +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-doc-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev2.txt \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rm3.topics.msmarco-v2-doc.dev2.txt \ + -bm25 -rm3 & ``` Evaluation can be performed using `trec_eval`: @@ -64,31 +77,38 @@ tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank sr tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default.topics.msmarco-v2-doc.dev2.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default.topics.msmarco-v2-doc.dev2.txt tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default.topics.msmarco-v2-doc.dev2.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rm3.topics.msmarco-v2-doc.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rm3.topics.msmarco-v2-doc.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rm3.topics.msmarco-v2-doc.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rm3.topics.msmarco-v2-doc.dev2.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rm3.topics.msmarco-v2-doc.dev2.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rm3.topics.msmarco-v2-doc.dev2.txt ``` ## Effectiveness With the above commands, you should be able to reproduce the following results: -| MAP@100 | BM25 (default)| -|:-------------------------------------------------------------------------------------------------------------|-----------| -| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1988 | -| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1986 | +| MAP@100 | BM25 (default)| +RM3 | +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------| +| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1988 | 0.1128 | +| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1986 | 0.1151 | -| MRR@100 | BM25 (default)| -|:-------------------------------------------------------------------------------------------------------------|-----------| -| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2011 | -| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2012 | +| MRR@100 | BM25 (default)| +RM3 | +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------| +| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2011 | 0.1144 | +| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2012 | 0.1169 | -| R@100 | BM25 (default)| -|:-------------------------------------------------------------------------------------------------------------|-----------| -| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.6786 | -| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.6821 | +| R@100 | BM25 (default)| +RM3 | +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------| +| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.6786 | 0.5935 | +| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.6821 | 0.5898 | -| R@1000 | BM25 (default)| -|:-------------------------------------------------------------------------------------------------------------|-----------| -| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.8614 | -| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.8568 | +| R@1000 | BM25 (default)| +RM3 | +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------| +| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.8614 | 0.8189 | +| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.8568 | 0.8245 | diff --git a/docs/regressions-msmarco-v2-doc-segmented-d2q-t5.md b/docs/regressions-msmarco-v2-doc-segmented-d2q-t5.md index 40b048e5d4..6c6b5d5332 100644 --- a/docs/regressions-msmarco-v2-doc-segmented-d2q-t5.md +++ b/docs/regressions-msmarco-v2-doc-segmented-d2q-t5.md @@ -53,6 +53,19 @@ target/appassembler/bin/SearchCollection \ -topicreader TsvInt \ -output runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default.topics.msmarco-v2-doc.dev2.txt \ -bm25 -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-doc-segmented-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev.txt \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rm3.topics.msmarco-v2-doc.dev.txt \ + -bm25 -rm3 -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 & +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-doc-segmented-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev2.txt \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rm3.topics.msmarco-v2-doc.dev2.txt \ + -bm25 -rm3 -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 & ``` Evaluation can be performed using `trec_eval`: @@ -64,31 +77,38 @@ tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank sr tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default.topics.msmarco-v2-doc.dev2.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default.topics.msmarco-v2-doc.dev2.txt tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default.topics.msmarco-v2-doc.dev2.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rm3.topics.msmarco-v2-doc.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rm3.topics.msmarco-v2-doc.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rm3.topics.msmarco-v2-doc.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rm3.topics.msmarco-v2-doc.dev2.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rm3.topics.msmarco-v2-doc.dev2.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rm3.topics.msmarco-v2-doc.dev2.txt ``` ## Effectiveness With the above commands, you should be able to reproduce the following results: -| MAP@100 | BM25 (default)| -|:-------------------------------------------------------------------------------------------------------------|-----------| -| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2203 | -| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2205 | +| MAP@100 | BM25 (default)| +RM3 | +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------| +| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2203 | 0.1955 | +| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2205 | 0.1939 | -| MRR@100 | BM25 (default)| -|:-------------------------------------------------------------------------------------------------------------|-----------| -| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2226 | -| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2234 | +| MRR@100 | BM25 (default)| +RM3 | +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------| +| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2226 | 0.1974 | +| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2234 | 0.1966 | -| R@100 | BM25 (default)| -|:-------------------------------------------------------------------------------------------------------------|-----------| -| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7297 | -| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7316 | +| R@100 | BM25 (default)| +RM3 | +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------| +| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7297 | 0.7104 | +| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7316 | 0.7164 | -| R@1000 | BM25 (default)| -|:-------------------------------------------------------------------------------------------------------------|-----------| -| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.8982 | -| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.8952 | +| R@1000 | BM25 (default)| +RM3 | +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------| +| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.8982 | 0.9000 | +| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.8952 | 0.8967 | diff --git a/docs/regressions-msmarco-v2-passage-augmented-d2q-t5.md b/docs/regressions-msmarco-v2-passage-augmented-d2q-t5.md index cc69f740b6..5913017681 100644 --- a/docs/regressions-msmarco-v2-passage-augmented-d2q-t5.md +++ b/docs/regressions-msmarco-v2-passage-augmented-d2q-t5.md @@ -52,6 +52,19 @@ target/appassembler/bin/SearchCollection \ -topicreader TsvInt \ -output runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default.topics.msmarco-v2-passage.dev2.txt \ -bm25 & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-passage-augmented-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev.txt \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rm3.topics.msmarco-v2-passage.dev.txt \ + -bm25 -rm3 & +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-passage-augmented-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev2.txt \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rm3.topics.msmarco-v2-passage.dev2.txt \ + -bm25 -rm3 & ``` Evaluation can be performed using `trec_eval`: @@ -63,31 +76,38 @@ tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank sr tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default.topics.msmarco-v2-passage.dev2.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default.topics.msmarco-v2-passage.dev2.txt tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default.topics.msmarco-v2-passage.dev2.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rm3.topics.msmarco-v2-passage.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rm3.topics.msmarco-v2-passage.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rm3.topics.msmarco-v2-passage.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rm3.topics.msmarco-v2-passage.dev2.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rm3.topics.msmarco-v2-passage.dev2.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rm3.topics.msmarco-v2-passage.dev2.txt ``` ## Effectiveness With the above commands, you should be able to reproduce the following results: -| MAP@100 | BM25 (default)| -|:-------------------------------------------------------------------------------------------------------------|-----------| -| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1160 | -| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1158 | +| MAP@100 | BM25 (default)| +RM3 | +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------| +| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1160 | 0.0870 | +| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1158 | 0.0879 | -| MRR@100 | BM25 (default)| -|:-------------------------------------------------------------------------------------------------------------|-----------| -| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1172 | -| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1170 | +| MRR@100 | BM25 (default)| +RM3 | +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------| +| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1172 | 0.0880 | +| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1170 | 0.0887 | -| R@100 | BM25 (default)| -|:-------------------------------------------------------------------------------------------------------------|-----------| -| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.5039 | -| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.5158 | +| R@100 | BM25 (default)| +RM3 | +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------| +| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.5039 | 0.4742 | +| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.5158 | 0.4905 | -| R@1000 | BM25 (default)| -|:-------------------------------------------------------------------------------------------------------------|-----------| -| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7647 | -| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7659 | +| R@1000 | BM25 (default)| +RM3 | +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------| +| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7647 | 0.7625 | +| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7659 | 0.7649 | diff --git a/docs/regressions-msmarco-v2-passage-d2q-t5.md b/docs/regressions-msmarco-v2-passage-d2q-t5.md index d0eb9b6b2f..1f8451caa4 100644 --- a/docs/regressions-msmarco-v2-passage-d2q-t5.md +++ b/docs/regressions-msmarco-v2-passage-d2q-t5.md @@ -52,6 +52,19 @@ target/appassembler/bin/SearchCollection \ -topicreader TsvInt \ -output runs/run.msmarco-v2-passage-d2q-t5.bm25-default.topics.msmarco-v2-passage.dev2.txt \ -bm25 & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-passage-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev.txt \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rm3.topics.msmarco-v2-passage.dev.txt \ + -bm25 -rm3 & +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-passage-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev2.txt \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rm3.topics.msmarco-v2-passage.dev2.txt \ + -bm25 -rm3 & ``` Evaluation can be performed using `trec_eval`: @@ -63,31 +76,38 @@ tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank sr tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default.topics.msmarco-v2-passage.dev2.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default.topics.msmarco-v2-passage.dev2.txt tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default.topics.msmarco-v2-passage.dev2.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rm3.topics.msmarco-v2-passage.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rm3.topics.msmarco-v2-passage.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rm3.topics.msmarco-v2-passage.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rm3.topics.msmarco-v2-passage.dev2.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rm3.topics.msmarco-v2-passage.dev2.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rm3.topics.msmarco-v2-passage.dev2.txt ``` ## Effectiveness With the above commands, you should be able to reproduce the following results: -| MAP@100 | BM25 (default)| -|:-------------------------------------------------------------------------------------------------------------|-----------| -| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1057 | -| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1112 | +| MAP@100 | BM25 (default)| +RM3 | +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------| +| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1057 | 0.0939 | +| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1112 | 0.0979 | -| MRR@100 | BM25 (default)| -|:-------------------------------------------------------------------------------------------------------------|-----------| -| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1072 | -| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1123 | +| MRR@100 | BM25 (default)| +RM3 | +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------| +| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1072 | 0.0948 | +| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1123 | 0.0985 | -| R@100 | BM25 (default)| -|:-------------------------------------------------------------------------------------------------------------|-----------| -| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.4670 | -| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.4803 | +| R@100 | BM25 (default)| +RM3 | +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------| +| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.4670 | 0.4723 | +| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.4803 | 0.4800 | -| R@1000 | BM25 (default)| -|:-------------------------------------------------------------------------------------------------------------|-----------| -| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7083 | -| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7151 | +| R@1000 | BM25 (default)| +RM3 | +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------| +| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7083 | 0.7174 | +| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7151 | 0.7240 | diff --git a/src/main/resources/regression/dl21-passage-unicoil-0shot.yaml b/src/main/resources/regression/dl21-passage-unicoil-0shot.yaml index 6f9e1fbcf5..8980b6debd 100644 --- a/src/main/resources/regression/dl21-passage-unicoil-0shot.yaml +++ b/src/main/resources/regression/dl21-passage-unicoil-0shot.yaml @@ -72,4 +72,4 @@ models: R@100: - 0.4731 R@1000: - - 0.7551 \ No newline at end of file + - 0.7551 diff --git a/src/main/resources/regression/msmarco-doc-docTTTTTquery.yaml b/src/main/resources/regression/msmarco-doc-docTTTTTquery.yaml index 8927f46fc7..746a221518 100644 --- a/src/main/resources/regression/msmarco-doc-docTTTTTquery.yaml +++ b/src/main/resources/regression/msmarco-doc-docTTTTTquery.yaml @@ -64,6 +64,18 @@ models: - 0.7993 R@1000: - 0.9259 + - name: bm25-default+rm3 + display: +RM3 + params: -bm25 -rm3 + results: + AP@1000: + - 0.1839 + RR@100: + - 0.1831 + R@100: + - 0.7420 + R@1000: + - 0.9128 - name: bm25-tuned display: BM25 (tuned) params: -bm25 -bm25.k1 4.68 -bm25.b 0.87 @@ -75,4 +87,16 @@ models: R@100: - 0.8612 R@1000: - - 0.9553 \ No newline at end of file + - 0.9553 + - name: bm25-tuned+rm3 + display: +RM3 + params: -bm25 -bm25.k1 4.68 -bm25.b 0.87 -rm3 + results: + AP@1000: + - 0.2627 + RR@100: + - 0.2621 + R@100: + - 0.8379 + R@1000: + - 0.9524 diff --git a/src/main/resources/regression/msmarco-doc-segmented-docTTTTTquery.yaml b/src/main/resources/regression/msmarco-doc-segmented-docTTTTTquery.yaml index fae56fafd2..449e03dad2 100644 --- a/src/main/resources/regression/msmarco-doc-segmented-docTTTTTquery.yaml +++ b/src/main/resources/regression/msmarco-doc-segmented-docTTTTTquery.yaml @@ -64,6 +64,18 @@ models: - 0.8479 R@1000: - 0.9490 + - name: bm25-default+rm3 + display: +RM3 + params: -bm25 -rm3 -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 + results: + AP@1000: + - 0.2823 + RR@100: + - 0.2818 + R@100: + - 0.8479 + R@1000: + - 0.9547 - name: bm25-tuned display: BM25 (tuned) params: -bm25 -bm25.k1 2.56 -bm25.b 0.59 -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 @@ -75,4 +87,16 @@ models: R@100: - 0.8627 R@1000: - - 0.9530 \ No newline at end of file + - 0.9530 + - name: bm25-tuned+rm3 + display: +RM3 + params: -bm25 -bm25.k1 2.56 -bm25.b 0.59 -rm3 -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 + results: + AP@1000: + - 0.2989 + RR@100: + - 0.2985 + R@100: + - 0.8556 + R@1000: + - 0.9567 diff --git a/src/main/resources/regression/msmarco-v2-doc-d2q-t5.yaml b/src/main/resources/regression/msmarco-v2-doc-d2q-t5.yaml index ed70560f1a..273ea83bc5 100644 --- a/src/main/resources/regression/msmarco-v2-doc-d2q-t5.yaml +++ b/src/main/resources/regression/msmarco-v2-doc-d2q-t5.yaml @@ -72,3 +72,19 @@ models: R@1000: - 0.8614 - 0.8568 + - name: bm25-default+rm3 + display: +RM3 + params: -bm25 -rm3 + results: + MAP@100: + - 0.1128 + - 0.1151 + MRR@100: + - 0.1144 + - 0.1169 + R@100: + - 0.5935 + - 0.5898 + R@1000: + - 0.8189 + - 0.8245 diff --git a/src/main/resources/regression/msmarco-v2-doc-segmented-d2q-t5.yaml b/src/main/resources/regression/msmarco-v2-doc-segmented-d2q-t5.yaml index cb858b3776..f67f50a216 100644 --- a/src/main/resources/regression/msmarco-v2-doc-segmented-d2q-t5.yaml +++ b/src/main/resources/regression/msmarco-v2-doc-segmented-d2q-t5.yaml @@ -72,3 +72,19 @@ models: R@1000: - 0.8982 - 0.8952 + - name: bm25-default+rm3 + display: +RM3 + params: -bm25 -rm3 -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 + results: + MAP@100: + - 0.1955 + - 0.1939 + MRR@100: + - 0.1974 + - 0.1966 + R@100: + - 0.7104 + - 0.7164 + R@1000: + - 0.9000 + - 0.8967 diff --git a/src/main/resources/regression/msmarco-v2-passage-augmented-d2q-t5.yaml b/src/main/resources/regression/msmarco-v2-passage-augmented-d2q-t5.yaml index 7ca8cb0688..42c2160b81 100644 --- a/src/main/resources/regression/msmarco-v2-passage-augmented-d2q-t5.yaml +++ b/src/main/resources/regression/msmarco-v2-passage-augmented-d2q-t5.yaml @@ -72,3 +72,19 @@ models: R@1000: - 0.7647 - 0.7659 + - name: bm25-default+rm3 + display: +RM3 + params: -bm25 -rm3 + results: + MAP@100: + - 0.0870 + - 0.0879 + MRR@100: + - 0.0880 + - 0.0887 + R@100: + - 0.4742 + - 0.4905 + R@1000: + - 0.7625 + - 0.7649 \ No newline at end of file diff --git a/src/main/resources/regression/msmarco-v2-passage-d2q-t5.yaml b/src/main/resources/regression/msmarco-v2-passage-d2q-t5.yaml index 5c1a17e9e4..546ba3d88b 100644 --- a/src/main/resources/regression/msmarco-v2-passage-d2q-t5.yaml +++ b/src/main/resources/regression/msmarco-v2-passage-d2q-t5.yaml @@ -72,3 +72,19 @@ models: R@1000: - 0.7083 - 0.7151 + - name: bm25-default+rm3 + display: +RM3 + params: -bm25 -rm3 + results: + MAP@100: + - 0.0939 + - 0.0979 + MRR@100: + - 0.0948 + - 0.0985 + R@100: + - 0.4723 + - 0.4800 + R@1000: + - 0.7174 + - 0.7240 \ No newline at end of file