From 1aa3970bd32b456025ada608389f7e4896eff19e Mon Sep 17 00:00:00 2001 From: Jimmy Lin Date: Mon, 24 Dec 2018 07:22:20 -0500 Subject: [PATCH] Changed RM3 defaults to match settings in Indri (#539) --- docs/experiments-car17.md | 4 +- docs/experiments-core17.md | 4 +- docs/experiments-core18.md | 4 +- docs/experiments-cw09b.md | 24 +++++----- docs/experiments-cw12.md | 16 +++---- docs/experiments-cw12b13.md | 16 +++---- docs/experiments-disk12.md | 12 ++--- docs/experiments-gov2.md | 12 ++--- docs/experiments-mb11.md | 8 ++-- docs/experiments-mb13.md | 8 ++-- docs/experiments-robust04.md | 4 +- docs/experiments-robust05.md | 4 +- docs/experiments-wt10g.md | 4 +- .../java/io/anserini/search/SearchArgs.java | 30 ++++++++---- src/main/resources/regression/cacm.yaml | 8 ++-- src/main/resources/regression/car17.yaml | 8 ++-- src/main/resources/regression/core17.yaml | 8 ++-- src/main/resources/regression/core18.yaml | 8 ++-- src/main/resources/regression/cw09b.yaml | 48 +++++++++---------- src/main/resources/regression/cw12.yaml | 32 ++++++------- src/main/resources/regression/cw12b13.yaml | 30 ++++++------ src/main/resources/regression/disk12.yaml | 24 +++++----- src/main/resources/regression/gov2.yaml | 22 ++++----- src/main/resources/regression/mb11.yaml | 14 +++--- src/main/resources/regression/mb13.yaml | 16 +++---- src/main/resources/regression/robust04.yaml | 8 ++-- src/main/resources/regression/robust05.yaml | 6 +-- src/main/resources/regression/wt10g.yaml | 8 ++-- 28 files changed, 202 insertions(+), 188 deletions(-) diff --git a/docs/experiments-car17.md b/docs/experiments-car17.md index b3973eb72a..495e6a8482 100644 --- a/docs/experiments-car17.md +++ b/docs/experiments-car17.md @@ -63,11 +63,11 @@ With the above commands, you should be able to replicate the following results: MAP | BM25 | BM25+RM3 | BM25+AX | QL | QL+RM3 | QL+AX | :---------------------------------------|-----------|-----------|-----------|-----------|-----------|-----------| -All Topics | 0.1689 | 0.1386 | 0.1355 | 0.1516 | 0.1198 | 0.1082 | +All Topics | 0.1689 | 0.1287 | 0.1355 | 0.1516 | 0.1173 | 0.1082 | RECIP_RANK | BM25 | BM25+RM3 | BM25+AX | QL | QL+RM3 | QL+AX | :---------------------------------------|-----------|-----------|-----------|-----------|-----------|-----------| -All Topics | 0.2321 | 0.1907 | 0.1857 | 0.2085 | 0.1653 | 0.1501 | +All Topics | 0.2321 | 0.1788 | 0.1857 | 0.2085 | 0.1573 | 0.1501 | diff --git a/docs/experiments-core17.md b/docs/experiments-core17.md index 3b0793c9e6..033b51300a 100644 --- a/docs/experiments-core17.md +++ b/docs/experiments-core17.md @@ -64,11 +64,11 @@ With the above commands, you should be able to replicate the following results: MAP | BM25 | BM25+RM3 | BM25+AX | QL | QL+RM3 | QL+AX | :---------------------------------------|-----------|-----------|-----------|-----------|-----------|-----------| -All Topics | 0.1977 | 0.2596 | 0.2700 | 0.1913 | 0.2405 | 0.2514 | +All Topics | 0.1977 | 0.2682 | 0.2700 | 0.1913 | 0.2485 | 0.2514 | P30 | BM25 | BM25+RM3 | BM25+AX | QL | QL+RM3 | QL+AX | :---------------------------------------|-----------|-----------|-----------|-----------|-----------|-----------| -All Topics | 0.4160 | 0.4820 | 0.4927 | 0.4373 | 0.4580 | 0.4827 | +All Topics | 0.4160 | 0.4940 | 0.4927 | 0.4373 | 0.4740 | 0.4827 | diff --git a/docs/experiments-core18.md b/docs/experiments-core18.md index 51145dfa75..4d4be497b4 100644 --- a/docs/experiments-core18.md +++ b/docs/experiments-core18.md @@ -63,12 +63,12 @@ With the above commands, you should be able to replicate the following results: MAP | BM25 | BM25+RM3 | BM25+AX | QL | QL+RM3 | QL+AX | :---------------------------------------|-----------|-----------|-----------|-----------|-----------|-----------| -All Topics | 0.2491 | 0.2952 | 0.2921 | 0.2522 | 0.2759 | 0.2975 | +All Topics | 0.2491 | 0.3147 | 0.2921 | 0.2522 | 0.3064 | 0.2975 | P30 | BM25 | BM25+RM3 | BM25+AX | QL | QL+RM3 | QL+AX | :---------------------------------------|-----------|-----------|-----------|-----------|-----------|-----------| -All Topics | 0.3580 | 0.4200 | 0.4007 | 0.3627 | 0.3753 | 0.4073 | +All Topics | 0.3580 | 0.4193 | 0.4007 | 0.3627 | 0.4007 | 0.4073 | diff --git a/docs/experiments-cw09b.md b/docs/experiments-cw09b.md index 53a424d4c8..9bac861ada 100644 --- a/docs/experiments-cw09b.md +++ b/docs/experiments-cw09b.md @@ -110,29 +110,29 @@ With the above commands, you should be able to replicate the following results: MAP | BM25 | BM25+RM3 | BM25+AX | QL | QL+RM3 | QL+AX | :---------------------------------------|-----------|-----------|-----------|-----------|-----------|-----------| -TREC 2010 Web Track: Topics 51-100 | 0.1126 | 0.1171 | 0.0928 | 0.1060 | 0.1117 | 0.1086 | -TREC 2011 Web Track: Topics 101-150 | 0.1094 | 0.1142 | 0.0974 | 0.0958 | 0.0964 | 0.0879 | -TREC 2012 Web Track: Topics 151-200 | 0.1106 | 0.1382 | 0.1315 | 0.1069 | 0.1167 | 0.1212 | +TREC 2010 Web Track: Topics 51-100 | 0.1126 | 0.0933 | 0.0928 | 0.1060 | 0.1019 | 0.1086 | +TREC 2011 Web Track: Topics 101-150 | 0.1094 | 0.1081 | 0.0974 | 0.0958 | 0.0837 | 0.0879 | +TREC 2012 Web Track: Topics 151-200 | 0.1106 | 0.1107 | 0.1315 | 0.1069 | 0.1059 | 0.1212 | P30 | BM25 | BM25+RM3 | BM25+AX | QL | QL+RM3 | QL+AX | :---------------------------------------|-----------|-----------|-----------|-----------|-----------|-----------| -TREC 2010 Web Track: Topics 51-100 | 0.2681 | 0.2819 | 0.2354 | 0.2431 | 0.2611 | 0.2618 | -TREC 2011 Web Track: Topics 101-150 | 0.2513 | 0.2700 | 0.2393 | 0.2147 | 0.2147 | 0.2167 | -TREC 2012 Web Track: Topics 151-200 | 0.2167 | 0.2473 | 0.2553 | 0.2080 | 0.2053 | 0.2140 | +TREC 2010 Web Track: Topics 51-100 | 0.2681 | 0.2389 | 0.2354 | 0.2431 | 0.2312 | 0.2618 | +TREC 2011 Web Track: Topics 101-150 | 0.2513 | 0.2467 | 0.2393 | 0.2147 | 0.2067 | 0.2167 | +TREC 2012 Web Track: Topics 151-200 | 0.2167 | 0.1920 | 0.2553 | 0.2080 | 0.1980 | 0.2140 | NDCG20 | BM25 | BM25+RM3 | BM25+AX | QL | QL+RM3 | QL+AX | :---------------------------------------|-----------|-----------|-----------|-----------|-----------|-----------| -TREC 2010 Web Track: Topics 51-100 | 0.1354 | 0.1545 | 0.1637 | 0.1143 | 0.1362 | 0.1454 | -TREC 2011 Web Track: Topics 101-150 | 0.1890 | 0.1823 | 0.1833 | 0.1619 | 0.1520 | 0.1509 | -TREC 2012 Web Track: Topics 151-200 | 0.1014 | 0.1329 | 0.1441 | 0.0868 | 0.1059 | 0.1030 | +TREC 2010 Web Track: Topics 51-100 | 0.1354 | 0.1369 | 0.1637 | 0.1143 | 0.1185 | 0.1454 | +TREC 2011 Web Track: Topics 101-150 | 0.1890 | 0.1916 | 0.1833 | 0.1619 | 0.1447 | 0.1509 | +TREC 2012 Web Track: Topics 151-200 | 0.1014 | 0.0917 | 0.1441 | 0.0868 | 0.0896 | 0.1030 | ERR20 | BM25 | BM25+RM3 | BM25+AX | QL | QL+RM3 | QL+AX | :---------------------------------------|-----------|-----------|-----------|-----------|-----------|-----------| -TREC 2010 Web Track: Topics 51-100 | 0.0733 | 0.0865 | 0.0981 | 0.0599 | 0.0649 | 0.0742 | -TREC 2011 Web Track: Topics 101-150 | 0.0959 | 0.1042 | 0.1091 | 0.0849 | 0.0865 | 0.0820 | -TREC 2012 Web Track: Topics 151-200 | 0.1304 | 0.2224 | 0.2355 | 0.1305 | 0.1475 | 0.1558 | +TREC 2010 Web Track: Topics 51-100 | 0.0733 | 0.0747 | 0.0981 | 0.0599 | 0.0592 | 0.0742 | +TREC 2011 Web Track: Topics 101-150 | 0.0959 | 0.0960 | 0.1091 | 0.0849 | 0.0786 | 0.0820 | +TREC 2012 Web Track: Topics 151-200 | 0.1304 | 0.1493 | 0.2355 | 0.1305 | 0.1334 | 0.1558 | diff --git a/docs/experiments-cw12.md b/docs/experiments-cw12.md index a3ac70bd39..cec82a6012 100644 --- a/docs/experiments-cw12.md +++ b/docs/experiments-cw12.md @@ -73,25 +73,25 @@ With the above commands, you should be able to replicate the following results: MAP | BM25 | BM25+RM3 | QL | QL+RM3 | :---------------------------------------|-----------|-----------|-----------|-----------| -[TREC 2013 Web Track: Topics 201-250](http://trec.nist.gov/data/web2013.html)| 0.1695 | 0.1498 | 0.1493 | 0.1280 | -[TREC 2014 Web Track: Topics 251-300](http://trec.nist.gov/data/web2014.html)| 0.2469 | 0.2496 | 0.2467 | 0.2383 | +[TREC 2013 Web Track: Topics 201-250](http://trec.nist.gov/data/web2013.html)| 0.1695 | 0.1464 | 0.1493 | 0.1291 | +[TREC 2014 Web Track: Topics 251-300](http://trec.nist.gov/data/web2014.html)| 0.2469 | 0.2325 | 0.2467 | 0.2168 | P30 | BM25 | BM25+RM3 | QL | QL+RM3 | :---------------------------------------|-----------|-----------|-----------|-----------| -[TREC 2013 Web Track: Topics 201-250](http://trec.nist.gov/data/web2013.html)| 0.2767 | 0.2407 | 0.2613 | 0.2207 | -[TREC 2014 Web Track: Topics 251-300](http://trec.nist.gov/data/web2014.html)| 0.4533 | 0.4180 | 0.4380 | 0.4107 | +[TREC 2013 Web Track: Topics 201-250](http://trec.nist.gov/data/web2013.html)| 0.2767 | 0.2387 | 0.2613 | 0.2347 | +[TREC 2014 Web Track: Topics 251-300](http://trec.nist.gov/data/web2014.html)| 0.4533 | 0.4073 | 0.4380 | 0.3793 | NDCG20 | BM25 | BM25+RM3 | QL | QL+RM3 | :---------------------------------------|-----------|-----------|-----------|-----------| -[TREC 2013 Web Track: Topics 201-250](http://trec.nist.gov/data/web2013.html)| 0.2086 | 0.1836 | 0.1993 | 0.1611 | -[TREC 2014 Web Track: Topics 251-300](http://trec.nist.gov/data/web2014.html)| 0.2578 | 0.2430 | 0.2228 | 0.2258 | +[TREC 2013 Web Track: Topics 201-250](http://trec.nist.gov/data/web2013.html)| 0.2086 | 0.2033 | 0.1993 | 0.1725 | +[TREC 2014 Web Track: Topics 251-300](http://trec.nist.gov/data/web2014.html)| 0.2578 | 0.2530 | 0.2228 | 0.2066 | ERR20 | BM25 | BM25+RM3 | QL | QL+RM3 | :---------------------------------------|-----------|-----------|-----------|-----------| -[TREC 2013 Web Track: Topics 201-250](http://trec.nist.gov/data/web2013.html)| 0.1284 | 0.0974 | 0.1232 | 0.0913 | -[TREC 2014 Web Track: Topics 251-300](http://trec.nist.gov/data/web2014.html)| 0.1630 | 0.1713 | 0.1321 | 0.1407 | +[TREC 2013 Web Track: Topics 201-250](http://trec.nist.gov/data/web2013.html)| 0.1284 | 0.1264 | 0.1232 | 0.1008 | +[TREC 2014 Web Track: Topics 251-300](http://trec.nist.gov/data/web2014.html)| 0.1630 | 0.1655 | 0.1321 | 0.1218 | diff --git a/docs/experiments-cw12b13.md b/docs/experiments-cw12b13.md index 7983908823..4dfe6f754e 100644 --- a/docs/experiments-cw12b13.md +++ b/docs/experiments-cw12b13.md @@ -88,25 +88,25 @@ With the above commands, you should be able to replicate the following results: MAP | BM25 | BM25+RM3 | BM25+AX | QL | QL+RM3 | QL+AX | :---------------------------------------|-----------|-----------|-----------|-----------|-----------|-----------| -[TREC 2013 Web Track: Topics 201-250](http://trec.nist.gov/data/web2013.html)| 0.0468 | 0.0450 | 0.0435 | 0.0397 | 0.0319 | 0.0359 | -[TREC 2014 Web Track: Topics 251-300](http://trec.nist.gov/data/web2014.html)| 0.0224 | 0.0189 | 0.0180 | 0.0235 | 0.0205 | 0.0186 | +[TREC 2013 Web Track: Topics 201-250](http://trec.nist.gov/data/web2013.html)| 0.0468 | 0.0412 | 0.0435 | 0.0397 | 0.0322 | 0.0359 | +[TREC 2014 Web Track: Topics 251-300](http://trec.nist.gov/data/web2014.html)| 0.0224 | 0.0210 | 0.0180 | 0.0235 | 0.0203 | 0.0186 | P30 | BM25 | BM25+RM3 | BM25+AX | QL | QL+RM3 | QL+AX | :---------------------------------------|-----------|-----------|-----------|-----------|-----------|-----------| -[TREC 2013 Web Track: Topics 201-250](http://trec.nist.gov/data/web2013.html)| 0.2113 | 0.1787 | 0.1840 | 0.1767 | 0.1373 | 0.1513 | -[TREC 2014 Web Track: Topics 251-300](http://trec.nist.gov/data/web2014.html)| 0.1273 | 0.1133 | 0.1107 | 0.1373 | 0.1173 | 0.1167 | +[TREC 2013 Web Track: Topics 201-250](http://trec.nist.gov/data/web2013.html)| 0.2113 | 0.1713 | 0.1840 | 0.1767 | 0.1507 | 0.1513 | +[TREC 2014 Web Track: Topics 251-300](http://trec.nist.gov/data/web2014.html)| 0.1273 | 0.1207 | 0.1107 | 0.1373 | 0.1173 | 0.1167 | NDCG20 | BM25 | BM25+RM3 | BM25+AX | QL | QL+RM3 | QL+AX | :---------------------------------------|-----------|-----------|-----------|-----------|-----------|-----------| -[TREC 2013 Web Track: Topics 201-250](http://trec.nist.gov/data/web2013.html)| 0.1286 | 0.1228 | 0.1287 | 0.1107 | 0.0880 | 0.1143 | -[TREC 2014 Web Track: Topics 251-300](http://trec.nist.gov/data/web2014.html)| 0.1185 | 0.1012 | 0.0964 | 0.1177 | 0.1024 | 0.1001 | +[TREC 2013 Web Track: Topics 201-250](http://trec.nist.gov/data/web2013.html)| 0.1286 | 0.1129 | 0.1287 | 0.1107 | 0.0920 | 0.1143 | +[TREC 2014 Web Track: Topics 251-300](http://trec.nist.gov/data/web2014.html)| 0.1185 | 0.1080 | 0.0964 | 0.1177 | 0.1003 | 0.1001 | ERR20 | BM25 | BM25+RM3 | BM25+AX | QL | QL+RM3 | QL+AX | :---------------------------------------|-----------|-----------|-----------|-----------|-----------|-----------| -[TREC 2013 Web Track: Topics 201-250](http://trec.nist.gov/data/web2013.html)| 0.0838 | 0.0879 | 0.0943 | 0.0769 | 0.0568 | 0.0780 | -[TREC 2014 Web Track: Topics 251-300](http://trec.nist.gov/data/web2014.html)| 0.1201 | 0.1039 | 0.0929 | 0.1091 | 0.1036 | 0.0896 | +[TREC 2013 Web Track: Topics 201-250](http://trec.nist.gov/data/web2013.html)| 0.0838 | 0.0763 | 0.0943 | 0.0769 | 0.0553 | 0.0780 | +[TREC 2014 Web Track: Topics 251-300](http://trec.nist.gov/data/web2014.html)| 0.1201 | 0.1065 | 0.0929 | 0.1091 | 0.0929 | 0.0896 | diff --git a/docs/experiments-disk12.md b/docs/experiments-disk12.md index e23e0d8499..6dad3cdfb8 100644 --- a/docs/experiments-disk12.md +++ b/docs/experiments-disk12.md @@ -90,15 +90,15 @@ With the above commands, you should be able to replicate the following results: MAP | BM25 | BM25+RM3 | BM25+AX | QL | QL+RM3 | QL+AX | :---------------------------------------|-----------|-----------|-----------|-----------|-----------|-----------| -TREC-1 Ad Hoc Track: Topics 51-100 | 0.2273 | 0.2617 | 0.2640 | 0.2189 | 0.2478 | 0.2501 | -TREC-2 Ad Hoc Track: Topics 101-150 | 0.2010 | 0.2600 | 0.2722 | 0.2015 | 0.2485 | 0.2593 | -TREC-3 Ad Hoc Track: Topics 151-200 | 0.2580 | 0.3227 | 0.3318 | 0.2518 | 0.2996 | 0.3103 | +TREC-1 Ad Hoc Track: Topics 51-100 | 0.2273 | 0.2634 | 0.2640 | 0.2189 | 0.2435 | 0.2501 | +TREC-2 Ad Hoc Track: Topics 101-150 | 0.2010 | 0.2587 | 0.2722 | 0.2015 | 0.2442 | 0.2593 | +TREC-3 Ad Hoc Track: Topics 151-200 | 0.2580 | 0.3390 | 0.3318 | 0.2518 | 0.3042 | 0.3103 | P30 | BM25 | BM25+RM3 | BM25+AX | QL | QL+RM3 | QL+AX | :---------------------------------------|-----------|-----------|-----------|-----------|-----------|-----------| -TREC-1 Ad Hoc Track: Topics 51-100 | 0.4533 | 0.4867 | 0.5067 | 0.4520 | 0.4653 | 0.4953 | -TREC-2 Ad Hoc Track: Topics 101-150 | 0.4280 | 0.4580 | 0.4753 | 0.4207 | 0.4453 | 0.4740 | -TREC-3 Ad Hoc Track: Topics 151-200 | 0.4740 | 0.5040 | 0.5100 | 0.4580 | 0.4933 | 0.5167 | +TREC-1 Ad Hoc Track: Topics 51-100 | 0.4533 | 0.4800 | 0.5067 | 0.4520 | 0.4627 | 0.4953 | +TREC-2 Ad Hoc Track: Topics 101-150 | 0.4280 | 0.4593 | 0.4753 | 0.4207 | 0.4420 | 0.4740 | +TREC-3 Ad Hoc Track: Topics 151-200 | 0.4740 | 0.5273 | 0.5100 | 0.4580 | 0.4913 | 0.5167 | diff --git a/docs/experiments-gov2.md b/docs/experiments-gov2.md index 9f406e8e75..859b7490e7 100644 --- a/docs/experiments-gov2.md +++ b/docs/experiments-gov2.md @@ -90,15 +90,15 @@ With the above commands, you should be able to replicate the following results: MAP | BM25 | BM25+RM3 | BM25+AX | QL | QL+RM3 | QL+AX | :---------------------------------------|-----------|-----------|-----------|-----------|-----------|-----------| -[TREC 2004 Terabyte Track: Topics 701-750](http://trec.nist.gov/data/terabyte04.html)| 0.2689 | 0.2943 | 0.2665 | 0.2681 | 0.2806 | 0.2666 | -[TREC 2005 Terabyte Track: Topics 751-800](http://trec.nist.gov/data/terabyte05.html)| 0.3390 | 0.3800 | 0.3664 | 0.3303 | 0.3628 | 0.3646 | -[TREC 2006 Terabyte Track: Topics 801-850](http://trec.nist.gov/data/terabyte06.html)| 0.3080 | 0.3356 | 0.3069 | 0.2996 | 0.3173 | 0.3084 | +[TREC 2004 Terabyte Track: Topics 701-750](http://trec.nist.gov/data/terabyte04.html)| 0.2689 | 0.2844 | 0.2665 | 0.2681 | 0.2708 | 0.2666 | +[TREC 2005 Terabyte Track: Topics 751-800](http://trec.nist.gov/data/terabyte05.html)| 0.3390 | 0.3820 | 0.3664 | 0.3303 | 0.3559 | 0.3646 | +[TREC 2006 Terabyte Track: Topics 801-850](http://trec.nist.gov/data/terabyte06.html)| 0.3080 | 0.3377 | 0.3069 | 0.2996 | 0.3154 | 0.3084 | P30 | BM25 | BM25+RM3 | BM25+AX | QL | QL+RM3 | QL+AX | :---------------------------------------|-----------|-----------|-----------|-----------|-----------|-----------| -[TREC 2004 Terabyte Track: Topics 701-750](http://trec.nist.gov/data/terabyte04.html)| 0.4864 | 0.5313 | 0.4986 | 0.4755 | 0.4952 | 0.4932 | -[TREC 2005 Terabyte Track: Topics 751-800](http://trec.nist.gov/data/terabyte05.html)| 0.5540 | 0.5873 | 0.5933 | 0.5347 | 0.5720 | 0.5840 | -[TREC 2006 Terabyte Track: Topics 801-850](http://trec.nist.gov/data/terabyte06.html)| 0.4907 | 0.5160 | 0.5033 | 0.4720 | 0.4773 | 0.4920 | +[TREC 2004 Terabyte Track: Topics 701-750](http://trec.nist.gov/data/terabyte04.html)| 0.4864 | 0.5190 | 0.4986 | 0.4755 | 0.4925 | 0.4932 | +[TREC 2005 Terabyte Track: Topics 751-800](http://trec.nist.gov/data/terabyte05.html)| 0.5540 | 0.5920 | 0.5933 | 0.5347 | 0.5620 | 0.5840 | +[TREC 2006 Terabyte Track: Topics 801-850](http://trec.nist.gov/data/terabyte06.html)| 0.4907 | 0.5160 | 0.5033 | 0.4720 | 0.4847 | 0.4920 | diff --git a/docs/experiments-mb11.md b/docs/experiments-mb11.md index da3e293a4c..9805271083 100644 --- a/docs/experiments-mb11.md +++ b/docs/experiments-mb11.md @@ -88,13 +88,13 @@ With the above commands, you should be able to replicate the following results: MAP | BM25 | BM25+RM3 | BM25+AX | QL | QL+RM3 | QL+AX | :---------------------------------------|-----------|-----------|-----------|-----------|-----------|-----------| -[TREC 2011 Microblog Track](http://trec.nist.gov/data/microblog2011.html)| 0.3384 | 0.3621 | 0.4008 | 0.3584 | 0.4097 | 0.4201 | -[TREC 2012 Microblog Track](http://trec.nist.gov/data/microblog2012.html)| 0.1948 | 0.2124 | 0.2309 | 0.2102 | 0.2397 | 0.2474 | +[TREC 2011 Microblog Track](http://trec.nist.gov/data/microblog2011.html)| 0.3384 | 0.3650 | 0.4008 | 0.3584 | 0.3923 | 0.4201 | +[TREC 2012 Microblog Track](http://trec.nist.gov/data/microblog2012.html)| 0.1948 | 0.2193 | 0.2309 | 0.2102 | 0.2389 | 0.2474 | P30 | BM25 | BM25+RM3 | BM25+AX | QL | QL+RM3 | QL+AX | :---------------------------------------|-----------|-----------|-----------|-----------|-----------|-----------| -[TREC 2011 Microblog Track](http://trec.nist.gov/data/microblog2011.html)| 0.3959 | 0.4088 | 0.4612 | 0.4061 | 0.4483 | 0.4408 | -[TREC 2012 Microblog Track](http://trec.nist.gov/data/microblog2012.html)| 0.3316 | 0.3463 | 0.3554 | 0.3333 | 0.3571 | 0.3842 | +[TREC 2011 Microblog Track](http://trec.nist.gov/data/microblog2011.html)| 0.3959 | 0.4170 | 0.4612 | 0.4061 | 0.4435 | 0.4408 | +[TREC 2012 Microblog Track](http://trec.nist.gov/data/microblog2012.html)| 0.3316 | 0.3463 | 0.3554 | 0.3333 | 0.3520 | 0.3842 | diff --git a/docs/experiments-mb13.md b/docs/experiments-mb13.md index 277f62c193..133a4c809e 100644 --- a/docs/experiments-mb13.md +++ b/docs/experiments-mb13.md @@ -88,13 +88,13 @@ With the above commands, you should be able to replicate the following results: MAP | BM25 | BM25+RM3 | BM25+AX | QL | QL+RM3 | QL+AX | :---------------------------------------|-----------|-----------|-----------|-----------|-----------|-----------| -[TREC 2013 Microblog Track](http://trec.nist.gov/data/microblog2013.html)| 0.2371 | 0.2440 | 0.2855 | 0.2602 | 0.2815 | 0.3152 | -[TREC 2014 Microblog Track](http://trec.nist.gov/data/microblog2014.html)| 0.3931 | 0.4158 | 0.4796 | 0.4181 | 0.4746 | 0.4965 | +[TREC 2013 Microblog Track](http://trec.nist.gov/data/microblog2013.html)| 0.2371 | 0.2513 | 0.2855 | 0.2602 | 0.2911 | 0.3152 | +[TREC 2014 Microblog Track](http://trec.nist.gov/data/microblog2014.html)| 0.3931 | 0.4374 | 0.4796 | 0.4181 | 0.4676 | 0.4965 | P30 | BM25 | BM25+RM3 | BM25+AX | QL | QL+RM3 | QL+AX | :---------------------------------------|-----------|-----------|-----------|-----------|-----------|-----------| -[TREC 2013 Microblog Track](http://trec.nist.gov/data/microblog2013.html)| 0.4339 | 0.4350 | 0.4728 | 0.4561 | 0.4672 | 0.5078 | -[TREC 2014 Microblog Track](http://trec.nist.gov/data/microblog2014.html)| 0.6212 | 0.6236 | 0.6648 | 0.6430 | 0.6594 | 0.6727 | +[TREC 2013 Microblog Track](http://trec.nist.gov/data/microblog2013.html)| 0.4339 | 0.4411 | 0.4728 | 0.4561 | 0.4906 | 0.5078 | +[TREC 2014 Microblog Track](http://trec.nist.gov/data/microblog2014.html)| 0.6212 | 0.6442 | 0.6648 | 0.6430 | 0.6533 | 0.6727 | diff --git a/docs/experiments-robust04.md b/docs/experiments-robust04.md index b4c38f9579..69aca1b3ab 100644 --- a/docs/experiments-robust04.md +++ b/docs/experiments-robust04.md @@ -63,12 +63,12 @@ With the above commands, you should be able to replicate the following results: MAP | BM25 | BM25+RM3 | BM25+AX | QL | QL+RM3 | QL+AX | :---------------------------------------|-----------|-----------|-----------|-----------|-----------|-----------| -All Topics | 0.2531 | 0.2778 | 0.2895 | 0.2467 | 0.2649 | 0.2774 | +All Topics | 0.2531 | 0.2903 | 0.2895 | 0.2467 | 0.2747 | 0.2774 | P30 | BM25 | BM25+RM3 | BM25+AX | QL | QL+RM3 | QL+AX | :---------------------------------------|-----------|-----------|-----------|-----------|-----------|-----------| -All Topics | 0.3102 | 0.3288 | 0.3333 | 0.3079 | 0.3171 | 0.3229 | +All Topics | 0.3102 | 0.3365 | 0.3333 | 0.3079 | 0.3232 | 0.3229 | diff --git a/docs/experiments-robust05.md b/docs/experiments-robust05.md index 5abd7d2b37..96cad0889e 100644 --- a/docs/experiments-robust05.md +++ b/docs/experiments-robust05.md @@ -62,11 +62,11 @@ With the above commands, you should be able to replicate the following results: MAP | BM25 | BM25+RM3 | BM25+AX | QL | QL+RM3 | QL+AX | :---------------------------------------|-----------|-----------|-----------|-----------|-----------|-----------| -All Topics | 0.2031 | 0.2523 | 0.2584 | 0.2028 | 0.2466 | 0.2476 | +All Topics | 0.2031 | 0.2602 | 0.2584 | 0.2028 | 0.2491 | 0.2476 | P30 | BM25 | BM25+RM3 | BM25+AX | QL | QL+RM3 | QL+AX | :---------------------------------------|-----------|-----------|-----------|-----------|-----------|-----------| -All Topics | 0.3693 | 0.4007 | 0.4120 | 0.3653 | 0.4067 | 0.4113 | +All Topics | 0.3693 | 0.4187 | 0.4120 | 0.3653 | 0.4067 | 0.4113 | diff --git a/docs/experiments-wt10g.md b/docs/experiments-wt10g.md index 75c59d60f1..cff0a0bf1d 100644 --- a/docs/experiments-wt10g.md +++ b/docs/experiments-wt10g.md @@ -64,11 +64,11 @@ With the above commands, you should be able to replicate the following results: MAP | BM25 | BM25+RM3 | BM25+AX | QL | QL+RM3 | QL+AX | :---------------------------------------|-----------|-----------|-----------|-----------|-----------|-----------| -Wt10g: Topics 451-550 | 0.1992 | 0.2163 | 0.2200 | 0.2021 | 0.2151 | 0.2275 | +Wt10g: Topics 451-550 | 0.1992 | 0.2276 | 0.2200 | 0.2021 | 0.2188 | 0.2275 | P30 | BM25 | BM25+RM3 | BM25+AX | QL | QL+RM3 | QL+AX | :---------------------------------------|-----------|-----------|-----------|-----------|-----------|-----------| -Wt10g: Topics 451-550 | 0.2218 | 0.2463 | 0.2483 | 0.2180 | 0.2276 | 0.2517 | +Wt10g: Topics 451-550 | 0.2218 | 0.2398 | 0.2483 | 0.2180 | 0.2310 | 0.2517 | diff --git a/src/main/java/io/anserini/search/SearchArgs.java b/src/main/java/io/anserini/search/SearchArgs.java index 3dd4a93276..5625118313 100644 --- a/src/main/java/io/anserini/search/SearchArgs.java +++ b/src/main/java/io/anserini/search/SearchArgs.java @@ -157,21 +157,35 @@ public class SearchArgs { @Option(name = "-sdm.uw", metaVar = "[value]", usage = "unordered window weight in sdm") public float sdm_uw = 0.05f; - @Option(name = "-rm3", usage = "use RM3 query expansion model (implies using query likelihood)") + // RM3 Options: Anserini uses the same default options as in Indri. + // As of v5.13, the defaults in Indri are, from src/RMExpander.cpp: + // + // int fbDocs = _param.get( "fbDocs" , 10 ); + // int fbTerms = _param.get( "fbTerms" , 10 ); + // double fbOrigWt = _param.get( "fbOrigWeight", 0.5 ); + // double mu = _param.get( "fbMu", 0 ); + + @Option(name = "-rm3", usage = "use RM3 query expansion model") public boolean rm3 = false; - @Option(name = "-rm3.fbTerms", handler = StringArrayOptionHandler.class, usage = "parameter to decide how many expansion terms to be picked") - public String[] rm3_fbTerms = new String[] {"20"}; + @Option(name = "-rm3.fbTerms", handler = StringArrayOptionHandler.class, + usage = "RM3 parameter: number of expansion terms") + public String[] rm3_fbTerms = new String[] {"10"}; - @Option(name = "-rm3.fbDocs", handler = StringArrayOptionHandler.class, usage = "parameter to decide how many documents to be used to find expansion terms") - public String[] rm3_fbDocs = new String[] {"50"}; + @Option(name = "-rm3.fbDocs", handler = StringArrayOptionHandler.class, + usage = "RM3 parameter: number of documents") + public String[] rm3_fbDocs = new String[] {"10"}; - @Option(name = "-rm3.originalQueryWeight", handler = StringArrayOptionHandler.class, usage = "parameter to decide how many documents to be used to find expansion terms") - public String[] rm3_originalQueryWeight = new String[] {"0.6"}; + @Option(name = "-rm3.originalQueryWeight", handler = StringArrayOptionHandler.class, + usage = "RM3 parameter: weight to assign to the original query") + public String[] rm3_originalQueryWeight = new String[] {"0.5"}; - @Option(name = "-rm3.outputQuery", usage = "output original and expanded query") + @Option(name = "-rm3.outputQuery", + usage = "RM3 parameter: print original and expanded queries") public boolean rm3_outputQuery = false; + // Axiomatic semantic matching matching options. + @Option(name = "-axiom", usage = "use Axiomatic query expansion model for the reranking") public boolean axiom = false; diff --git a/src/main/resources/regression/cacm.yaml b/src/main/resources/regression/cacm.yaml index eda72978cb..cc7b386b57 100644 --- a/src/main/resources/regression/cacm.yaml +++ b/src/main/resources/regression/cacm.yaml @@ -61,9 +61,9 @@ models: - -rm3 results: map: - - 0.3688 + - 0.3643 p30: - - 0.2295 + - 0.2237 - name: bm25+ax params: - -bm25 @@ -89,9 +89,9 @@ models: - -rm3 results: map: - - 0.3818 + - 0.3620 p30: - - 0.2237 + - 0.2218 - name: ql+ax params: - -ql diff --git a/src/main/resources/regression/car17.yaml b/src/main/resources/regression/car17.yaml index 23cbcd6e7b..c2b08c8da8 100644 --- a/src/main/resources/regression/car17.yaml +++ b/src/main/resources/regression/car17.yaml @@ -59,9 +59,9 @@ models: - -rm3 results: map: - - 0.1386 + - 0.1287 recip_rank: - - 0.1907 + - 0.1788 - name: bm25+ax params: - -bm25 @@ -87,9 +87,9 @@ models: - -rm3 results: map: - - 0.1198 + - 0.1173 recip_rank: - - 0.1653 + - 0.1573 - name: ql+ax params: - -ql diff --git a/src/main/resources/regression/core17.yaml b/src/main/resources/regression/core17.yaml index 42ca9c6f26..bd928d6f2b 100644 --- a/src/main/resources/regression/core17.yaml +++ b/src/main/resources/regression/core17.yaml @@ -59,9 +59,9 @@ models: - -rm3 results: map: - - 0.2596 + - 0.2682 p30: - - 0.4820 + - 0.4940 - name: bm25+ax params: - -bm25 @@ -87,9 +87,9 @@ models: - -rm3 results: map: - - 0.2405 + - 0.2485 p30: - - 0.4580 + - 0.4740 - name: ql+ax params: - -ql diff --git a/src/main/resources/regression/core18.yaml b/src/main/resources/regression/core18.yaml index 7ad9953fae..af8fb27aad 100644 --- a/src/main/resources/regression/core18.yaml +++ b/src/main/resources/regression/core18.yaml @@ -59,9 +59,9 @@ models: - -rm3 results: map: - - 0.2952 + - 0.3147 p30: - - 0.4200 + - 0.4193 - name: bm25+ax params: - -bm25 @@ -87,9 +87,9 @@ models: - -rm3 results: map: - - 0.2759 + - 0.3064 p30: - - 0.3753 + - 0.4007 - name: ql+ax params: - -ql diff --git a/src/main/resources/regression/cw09b.yaml b/src/main/resources/regression/cw09b.yaml index e121457ac8..f8901e6ff4 100644 --- a/src/main/resources/regression/cw09b.yaml +++ b/src/main/resources/regression/cw09b.yaml @@ -89,21 +89,21 @@ models: - -rm3 results: map: - - 0.1171 - - 0.1142 - - 0.1382 + - 0.0933 + - 0.1081 + - 0.1107 p30: - - 0.2819 - - 0.2700 - - 0.2473 + - 0.2389 + - 0.2467 + - 0.1920 ndcg20: - - 0.15446 - - 0.18227 - - 0.13294 + - 0.13690 + - 0.19164 + - 0.09170 err20: - - 0.08653 - - 0.10422 - - 0.22241 + - 0.07470 + - 0.09597 + - 0.14933 - name: bm25+ax params: - -bm25 @@ -154,21 +154,21 @@ models: - -rm3 results: map: - - 0.1117 - - 0.0964 - - 0.1167 + - 0.1019 + - 0.0837 + - 0.1059 p30: - - 0.2611 - - 0.2147 - - 0.2053 + - 0.2312 + - 0.2067 + - 0.1980 ndcg20: - - 0.13618 - - 0.15199 - - 0.10590 + - 0.11852 + - 0.14469 + - 0.08959 err20: - - 0.06486 - - 0.08655 - - 0.14750 + - 0.05920 + - 0.07861 + - 0.13336 - name: ql+ax params: - -ql diff --git a/src/main/resources/regression/cw12.yaml b/src/main/resources/regression/cw12.yaml index 219e7b7c38..2e221ee26a 100644 --- a/src/main/resources/regression/cw12.yaml +++ b/src/main/resources/regression/cw12.yaml @@ -82,17 +82,17 @@ models: - -rm3 results: map: - - 0.1498 - - 0.2496 + - 0.1464 + - 0.2325 p30: - - 0.2407 - - 0.4180 + - 0.2387 + - 0.4073 ndcg20: - - 0.18362 - - 0.24303 + - 0.20327 + - 0.25304 err20: - - 0.09742 - - 0.17134 + - 0.12637 + - 0.16550 - name: ql params: - -ql @@ -115,15 +115,15 @@ models: - -rm3 results: map: - - 0.1280 - - 0.2383 + - 0.1291 + - 0.2168 p30: - - 0.2207 - - 0.4107 + - 0.2347 + - 0.3793 ndcg20: - - 0.16115 - - 0.22580 + - 0.17253 + - 0.20662 err20: - - 0.09129 - - 0.14066 + - 0.10084 + - 0.12179 diff --git a/src/main/resources/regression/cw12b13.yaml b/src/main/resources/regression/cw12b13.yaml index de5a60229b..584fd76759 100644 --- a/src/main/resources/regression/cw12b13.yaml +++ b/src/main/resources/regression/cw12b13.yaml @@ -82,17 +82,17 @@ models: - -rm3 results: map: - - 0.0450 - - 0.0189 + - 0.0412 + - 0.0210 p30: - - 0.1787 - - 0.1133 + - 0.1713 + - 0.1207 ndcg20: - - 0.12284 - - 0.10124 + - 0.11293 + - 0.10796 err20: - - 0.08793 - - 0.10390 + - 0.07629 + - 0.10653 - name: bm25+ax params: - -bm25 @@ -135,17 +135,17 @@ models: - -rm3 results: map: - - 0.0319 - - 0.0205 + - 0.0322 + - 0.0203 p30: - - 0.1373 + - 0.1507 - 0.1173 ndcg20: - - 0.08799 - - 0.10240 + - 0.09199 + - 0.10035 err20: - - 0.05681 - - 0.10356 + - 0.05525 + - 0.09289 - name: ql+ax params: - -ql diff --git a/src/main/resources/regression/disk12.yaml b/src/main/resources/regression/disk12.yaml index e1997ecd22..9d799681ae 100644 --- a/src/main/resources/regression/disk12.yaml +++ b/src/main/resources/regression/disk12.yaml @@ -69,13 +69,13 @@ models: - -rm3 results: map: - - 0.2617 - - 0.2600 - - 0.3227 + - 0.2634 + - 0.2587 + - 0.3390 p30: - - 0.4867 - - 0.4580 - - 0.5040 + - 0.4800 + - 0.4593 + - 0.5273 - name: bm25+ax params: - -bm25 @@ -109,13 +109,13 @@ models: - -rm3 results: map: - - 0.2478 - - 0.2485 - - 0.2996 + - 0.2435 + - 0.2442 + - 0.3042 p30: - - 0.4653 - - 0.4453 - - 0.4933 + - 0.4627 + - 0.4420 + - 0.4913 - name: ql+ax params: - -ql diff --git a/src/main/resources/regression/gov2.yaml b/src/main/resources/regression/gov2.yaml index 9173dd1bfe..f0c23ec988 100644 --- a/src/main/resources/regression/gov2.yaml +++ b/src/main/resources/regression/gov2.yaml @@ -69,12 +69,12 @@ models: - -rm3 results: map: - - 0.2943 - - 0.3800 - - 0.3356 + - 0.2844 + - 0.3820 + - 0.3377 p30: - - 0.5313 - - 0.5873 + - 0.5190 + - 0.5920 - 0.5160 - name: bm25+ax params: @@ -110,13 +110,13 @@ models: - -rm3 results: map: - - 0.2806 - - 0.3628 - - 0.3173 + - 0.2708 + - 0.3559 + - 0.3154 p30: - - 0.4952 - - 0.5720 - - 0.4773 + - 0.4925 + - 0.5620 + - 0.4847 - name: ql+ax params: - -ql diff --git a/src/main/resources/regression/mb11.yaml b/src/main/resources/regression/mb11.yaml index b9d5085184..173b3b48a3 100644 --- a/src/main/resources/regression/mb11.yaml +++ b/src/main/resources/regression/mb11.yaml @@ -70,10 +70,10 @@ models: - -rm3 results: map: - - 0.3621 - - 0.2124 + - 0.3650 + - 0.2193 p30: - - 0.4088 + - 0.4170 - 0.3463 - name: bm25+ax params: @@ -108,11 +108,11 @@ models: - -rm3 results: map: - - 0.4097 - - 0.2397 + - 0.3923 + - 0.2389 p30: - - 0.4483 - - 0.3571 + - 0.4435 + - 0.3520 - name: ql+ax params: - -searchtweets diff --git a/src/main/resources/regression/mb13.yaml b/src/main/resources/regression/mb13.yaml index df6175c273..8208b0a8b2 100644 --- a/src/main/resources/regression/mb13.yaml +++ b/src/main/resources/regression/mb13.yaml @@ -71,11 +71,11 @@ models: - -rm3 results: map: - - 0.2440 - - 0.4158 + - 0.2513 + - 0.4374 p30: - - 0.4350 - - 0.6236 + - 0.4411 + - 0.6442 - name: bm25+ax params: - -searchtweets @@ -109,11 +109,11 @@ models: - -rm3 results: map: - - 0.2815 - - 0.4746 + - 0.2911 + - 0.4676 p30: - - 0.4672 - - 0.6594 + - 0.4906 + - 0.6533 - name: ql+ax params: - -searchtweets diff --git a/src/main/resources/regression/robust04.yaml b/src/main/resources/regression/robust04.yaml index 55844ce23b..5193b946cb 100644 --- a/src/main/resources/regression/robust04.yaml +++ b/src/main/resources/regression/robust04.yaml @@ -60,9 +60,9 @@ models: - -rm3 results: map: - - 0.2778 + - 0.2903 p30: - - 0.3288 + - 0.3365 - name: bm25+ax params: - -bm25 @@ -88,9 +88,9 @@ models: - -rm3 results: map: - - 0.2649 + - 0.2747 p30: - - 0.3171 + - 0.3232 - name: ql+ax params: - -ql diff --git a/src/main/resources/regression/robust05.yaml b/src/main/resources/regression/robust05.yaml index 901d84fa48..500a386f39 100644 --- a/src/main/resources/regression/robust05.yaml +++ b/src/main/resources/regression/robust05.yaml @@ -60,9 +60,9 @@ models: - -rm3 results: map: - - 0.2523 + - 0.2602 p30: - - 0.4007 + - 0.4187 - name: bm25+ax params: - -bm25 @@ -88,7 +88,7 @@ models: - -rm3 results: map: - - 0.2466 + - 0.2491 p30: - 0.4067 - name: ql+ax diff --git a/src/main/resources/regression/wt10g.yaml b/src/main/resources/regression/wt10g.yaml index 7f2197cf90..fceec4ffac 100644 --- a/src/main/resources/regression/wt10g.yaml +++ b/src/main/resources/regression/wt10g.yaml @@ -60,9 +60,9 @@ models: - -rm3 results: map: - - 0.2163 + - 0.2276 p30: - - 0.2463 + - 0.2398 - name: bm25+ax params: - -bm25 @@ -89,9 +89,9 @@ models: - -rm3 results: map: - - 0.2151 + - 0.2188 p30: - - 0.2276 + - 0.2310 - name: ql+ax params: - -ql