From 42db07ba1f4d26bb1f07707b300dd6c424817cc1 Mon Sep 17 00:00:00 2001 From: yuki617 <43072754+yuki617@users.noreply.github.com> Date: Wed, 28 Apr 2021 22:27:40 -0400 Subject: [PATCH] Added integration tests for classification-based approach to PRF (#513) --- integrations/test_simplesearcher_check_prf.py | 964 +++++++++++------- 1 file changed, 614 insertions(+), 350 deletions(-) diff --git a/integrations/test_simplesearcher_check_prf.py b/integrations/test_simplesearcher_check_prf.py index 34d99462d..12e3c36b1 100644 --- a/integrations/test_simplesearcher_check_prf.py +++ b/integrations/test_simplesearcher_check_prf.py @@ -41,39 +41,32 @@ def setUp(self): else: os.mkdir(self.tmp) + self.pyserini_search_cmd = 'python -m pyserini.search' + self.core17_index_path = os.path.join(self.anserini_root, 'indexes/lucene-index.core17.pos+docvectors+raw') + self.core17_qrels_path = os.path.join(self.pyserini_root, 'tools/topics-and-qrels/qrels.core17.txt') + + self.core18_index_path = os.path.join(self.anserini_root, 'indexes/lucene-index.core18.pos+docvectors+raw') + self.core18_qrels_path = os.path.join(self.pyserini_root, 'tools/topics-and-qrels/qrels.core18.txt') + + self.robust04_index_path = os.path.join(self.anserini_root, 'indexes/lucene-index.robust04.pos+docvectors+raw') + self.robust04_qrels_path = os.path.join(self.pyserini_root, 'tools/topics-and-qrels/qrels.robust04.txt') + + self.robust05_index_path = os.path.join(self.anserini_root, 'indexes/lucene-index.robust05.pos+docvectors+raw') + self.robust05_qrels_path = os.path.join(self.pyserini_root, 'tools/topics-and-qrels/qrels.robust05.txt') + self.core17_checker = SimpleSearcherScoreChecker( - index=os.path.join(self.anserini_root, 'indexes/lucene-index.core17.pos+docvectors+raw'), + index=self.core17_index_path, topics=os.path.join(self.pyserini_root, 'tools/topics-and-qrels/topics.core17.txt'), pyserini_topics='core17', - qrels=os.path.join(self.pyserini_root, 'tools/topics-and-qrels/qrels.core17.txt'), + qrels=self.core17_qrels_path, eval=f'{self.pyserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30') - try: - if os.path.exists(f'{self.tmp}/core17') == False: - tar = tarfile.open(f"{self.pyserini_root}/integrations/core17.tar.gz", "r:gz") - tar.extractall(path=f'{self.tmp}') - tar.close() - except: - shutil.rmtree(f'{self.tmp}') - print(f'core17.tar.gz is not saved in {self.pyserini_root}/integrations') - raise - self.core18_checker = SimpleSearcherScoreChecker( - index=os.path.join(self.anserini_root, 'indexes/lucene-index.core18.pos+docvectors+raw'), - topics=os.path.join(self.pyserini_root, 'tools/topics-and-qrels/topics.core18.txt'), - pyserini_topics='core18', - qrels=os.path.join(self.pyserini_root, 'tools/topics-and-qrels/qrels.core18.txt'), - eval=f'{self.pyserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30') - - try: - if os.path.exists(f'{self.tmp}/core18') == False: - tar = tarfile.open(f"{self.pyserini_root}/integrations/core18.tar.gz", "r:gz") - tar.extractall(path=f'{self.tmp}') - tar.close() - except: - shutil.rmtree(f'{self.tmp}') - print(f'core18.tar.gz is not saved in {self.pyserini_root}/integrations') - raise + index=os.path.join(self.anserini_root, 'indexes/lucene-index.core18.pos+docvectors+raw'), + topics=os.path.join(self.pyserini_root, 'tools/topics-and-qrels/topics.core18.txt'), + pyserini_topics='core18', + qrels=os.path.join(self.pyserini_root, 'tools/topics-and-qrels/qrels.core18.txt'), + eval=f'{self.pyserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30') self.robust04_checker = SimpleSearcherScoreChecker( index=os.path.join(self.anserini_root, 'indexes/lucene-index.robust04.pos+docvectors+raw'), @@ -82,16 +75,6 @@ def setUp(self): qrels=os.path.join(self.pyserini_root, 'tools/topics-and-qrels/qrels.robust04.txt'), eval=f'{self.pyserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30') - try: - if os.path.exists(f'{self.tmp}/robust04') == False: - tar = tarfile.open(f"{self.pyserini_root}/integrations/robust04.tar.gz", "r:gz") - tar.extractall(path=f'{self.tmp}') - tar.close() - except: - shutil.rmtree(f'{self.tmp}') - print(f'robust04.tar.gz is not saved in {self.pyserini_root}/integrations') - raise - self.robust05_checker = SimpleSearcherScoreChecker( index=os.path.join(self.anserini_root, 'indexes/lucene-index.robust05.pos+docvectors+raw'), topics=os.path.join(self.pyserini_root, 'tools/topics-and-qrels/topics.robust05.txt'), @@ -99,15 +82,31 @@ def setUp(self): qrels=os.path.join(self.pyserini_root, 'tools/topics-and-qrels/qrels.robust05.txt'), eval=f'{self.pyserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30') - try: - if os.path.exists(f'{self.tmp}/robust05') == False: - tar = tarfile.open(f"{self.pyserini_root}/integrations/robust05.tar.gz", "r:gz") - tar.extractall(path=f'{self.tmp}') - tar.close() - except: - shutil.rmtree(f'{self.tmp}') - print(f'robust05.tar.gz is not saved in {self.pyserini_root}/integrations') - raise + def test_cross_validation(self): + pyserini_topics = 'core17' + os.mkdir(f'{self.tmp}/core17') + for alpha in [x / 10.0 for x in range(0, 11)]: + run_file_cmd = f'{self.pyserini_search_cmd} --index {self.core17_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/core17/core17_lr_A{alpha}_bm25.txt ' \ + + f'--prcl lr --prcl.vectorizer TfidfVectorizer --prcl.alpha {alpha}' + + status = os.system(run_file_cmd) + self.assertEqual(status, 0) + os.system(f'python {self.pyserini_root}/scripts/classifier_prf/cross_validate.py \ + --anserini {self.anserini_root} --run_file {self.tmp} --pyserini {self.pyserini_root} \ + --collection core17 --output {self.tmp}/core17_lr.txt --classifier lr ') + + cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ + {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.core17.txt \ + {self.tmp}/core17_lr.txt' + + status = os.system(cmd) + stdout, stderr = run_command(cmd) + score = parse_score(stdout, "map") + + self.assertEqual(status, 0) + self.assertEqual(stderr, '') + self.assertAlmostEqual(score, 0.2462, delta=0.0001) def test_core17(self): self.assertTrue(self.core17_checker.run('core17_bm25', '--bm25', 0.2087)) @@ -116,139 +115,206 @@ def test_core17_rm3(self): self.assertTrue(self.core17_checker.run('core17_bm25', '--bm25 --rm3', 0.2823)) def test_core17_lr(self): - os.system(f'python {self.pyserini_root}/scripts/classifier_prf/cross_validate.py \ - --anserini {self.anserini_root} --run_file {self.tmp} --pyserini {self.pyserini_root} \ - --collection core17 --output {self.tmp}/core17_lr.txt --classifier lr ') + pyserini_topics = 'core17' - cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ - {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.core17.txt {self.tmp}/core17_lr.txt' + run_file_cmd = f'{self.pyserini_search_cmd} --index {self.core17_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/core17_lr.txt ' \ + + f'--prcl lr --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.7' - status = os.system(cmd) - stdout, stderr = run_command(cmd) + status = os.system(run_file_cmd) + self.assertEqual(status, 0) + + score_cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ + {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.core17.txt \ + {self.tmp}/core17_lr.txt' + + status = os.system(score_cmd) + stdout, stderr = run_command(score_cmd) score = parse_score(stdout, "map") self.assertEqual(status, 0) self.assertEqual(stderr, '') - self.assertAlmostEqual(score, 0.2462, delta=0.0001) + self.assertAlmostEqual(score, 0.2473, delta=0.0001) def test_core17_lr_rm3(self): - os.system(f'python {self.pyserini_root}/scripts/classifier_prf/cross_validate.py \ - --anserini {self.anserini_root} --run_file {self.tmp} --pyserini {self.pyserini_root} \ - --collection core17 --output {self.tmp}/core17_lr_rm3.txt --classifier lr -rm3') + pyserini_topics = 'core17' - cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ - {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.core17.txt \ - {self.tmp}/core17_lr_rm3.txt' + run_file_cmd = f'{self.pyserini_search_cmd} --index {self.core17_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/core17_lr_rm3.txt ' \ + + f'--prcl lr --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.4 --rm3' - status = os.system(cmd) - stdout, stderr = run_command(cmd) + status = os.system(run_file_cmd) + self.assertEqual(status, 0) + + score_cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ + {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.core17.txt \ + {self.tmp}/core17_lr_rm3.txt' + + status = os.system(score_cmd) + stdout, stderr = run_command(score_cmd) score = parse_score(stdout, "map") self.assertEqual(status, 0) self.assertEqual(stderr, '') - self.assertAlmostEqual(score, 0.2906, delta=0.0001) + self.assertAlmostEqual(score, 0.2940, delta=0.0001) def test_core17_svm(self): - os.system(f'python {self.pyserini_root}/scripts/classifier_prf/cross_validate.py \ - --anserini {self.anserini_root} --run_file {self.tmp} --pyserini {self.pyserini_root} \ - --collection core17 --output {self.tmp}/core17_svm.txt --classifier svm') + pyserini_topics = 'core17' - cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ - {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.core17.txt \ - {self.tmp}/core17_svm.txt' + run_file_cmd = f'{self.pyserini_search_cmd} --index {self.core17_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/core17_svm.txt ' \ + + f'--prcl svm --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.7' - status = os.system(cmd) - stdout, stderr = run_command(cmd) + status = os.system(run_file_cmd) + self.assertEqual(status, 0) + + score_cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ + {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.core17.txt \ + {self.tmp}/core17_svm.txt' + + status = os.system(score_cmd) + stdout, stderr = run_command(score_cmd) score = parse_score(stdout, "map") self.assertEqual(status, 0) self.assertEqual(stderr, '') - self.assertAlmostEqual(score, 0.2367, delta=0.0001) + self.assertAlmostEqual(score, 0.2385, delta=0.0001) def test_core17_svm_rm3(self): - os.system(f'python {self.pyserini_root}/scripts/classifier_prf/cross_validate.py \ - --anserini {self.anserini_root} --run_file {self.tmp} --pyserini {self.pyserini_root} \ - --collection core17 --output {self.tmp}/core17_svm_rm3.txt --classifier svm -rm3') + pyserini_topics = 'core17' - cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ - {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.core17.txt \ - {self.tmp}/core17_svm_rm3.txt' + run_file_cmd = f'{self.pyserini_search_cmd} --index {self.core17_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/core17_svm_rm3.txt ' \ + + f'--prcl svm --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.4 --rm3' - status = os.system(cmd) - stdout, stderr = run_command(cmd) + status = os.system(run_file_cmd) + self.assertEqual(status, 0) + + score_cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ + {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.core17.txt \ + {self.tmp}/core17_svm_rm3.txt' + + status = os.system(score_cmd) + stdout, stderr = run_command(score_cmd) score = parse_score(stdout, "map") self.assertEqual(status, 0) self.assertEqual(stderr, '') - self.assertAlmostEqual(score, 0.2940, delta=0.0001) + self.assertAlmostEqual(score, 0.2970, delta=0.0001) def test_core17_avg(self): - os.system(f'python {self.pyserini_root}/scripts/classifier_prf/cross_validate.py \ - --anserini {self.anserini_root} --run_file {self.tmp} --pyserini {self.pyserini_root} \ - --collection core17 --output {self.tmp}/core17_lr+svm.txt --classifier lr+svm ') + pyserini_topics = 'core17' - cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ - {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.core17.txt \ - {self.tmp}/core17_lr+svm.txt' + run_file_cmd = f'{self.pyserini_search_cmd} --index {self.core17_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/core17_avg.txt ' \ + + f'--prcl lr svm --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.6' - status = os.system(cmd) - stdout, stderr = run_command(cmd) + status = os.system(run_file_cmd) + self.assertEqual(status, 0) + + score_cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ + {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.core17.txt \ + {self.tmp}/core17_avg.txt' + + status = os.system(score_cmd) + stdout, stderr = run_command(score_cmd) score = parse_score(stdout, "map") self.assertEqual(status, 0) self.assertEqual(stderr, '') - self.assertAlmostEqual(score, 0.2426, delta=0.0001) + self.assertAlmostEqual(score, 0.2442, delta=0.0001) def test_core17_avg_rm3(self): - os.system(f'python {self.pyserini_root}/scripts/classifier_prf/cross_validate.py \ - --anserini {self.anserini_root} --run_file {self.tmp} --pyserini {self.pyserini_root} \ - --collection core17 --output {self.tmp}/core17_lr+svm_rm3.txt --classifier lr+svm -rm3') + pyserini_topics = 'core17' - cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ - {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.core17.txt \ - {self.tmp}/core17_lr+svm_rm3.txt' + run_file_cmd = f'{self.pyserini_search_cmd} --index {self.core17_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/core17_avg_rm3.txt ' \ + + f'--prcl lr svm --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.5 --rm3' - status = os.system(cmd) - stdout, stderr = run_command(cmd) + status = os.system(run_file_cmd) + self.assertEqual(status, 0) + + score_cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ + {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.core17.txt \ + {self.tmp}/core17_avg_rm3.txt' + + status = os.system(score_cmd) + stdout, stderr = run_command(score_cmd) score = parse_score(stdout, "map") self.assertEqual(status, 0) self.assertEqual(stderr, '') - self.assertAlmostEqual(score, 0.2952, delta=0.0001) + self.assertAlmostEqual(score, 0.2967, delta=0.0001) def test_core17_rrf(self): - os.system(f'python {self.pyserini_root}/scripts/classifier_prf/cross_validate.py \ - --anserini {self.anserini_root} --run_file {self.tmp} --pyserini {self.pyserini_root} \ - --collection core17 --output {self.tmp}/core17_rrf.txt --classifier rrf') + pyserini_topics = 'core17' + lr_cmd = f'{self.pyserini_search_cmd} --index {self.core17_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/core17_lr.txt ' \ + + f'--prcl lr --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.7' - cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ - {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.core17.txt \ - {self.tmp}/core17_rrf.txt' + status = os.system(lr_cmd) + self.assertEqual(status, 0) - status = os.system(cmd) - stdout, stderr = run_command(cmd) + svm_cmd = f'{self.pyserini_search_cmd} --index {self.core17_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/core17_svm.txt ' \ + + f'--prcl svm --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.7' + + status = os.system(svm_cmd) + self.assertEqual(status, 0) + + rrf_cmd = f'python {self.anserini_root}/src/main/python/fusion.py ' \ + + f'--runs {self.tmp}/core17_lr.txt {self.tmp}/core17_svm.txt ' \ + + f'--out {self.tmp}/core17_rrf.txt' + + status = os.system(rrf_cmd) + self.assertEqual(status, 0) + + score_cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ + {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.core17.txt \ + {self.tmp}/core17_rrf.txt' + + status = os.system(score_cmd) + stdout, stderr = run_command(score_cmd) score = parse_score(stdout, "map") self.assertEqual(status, 0) self.assertEqual(stderr, '') - self.assertAlmostEqual(score, 0.2433, delta=0.0001) + self.assertAlmostEqual(score, 0.2446, delta=0.0001) def test_core17_rrf_rm3(self): - os.system(f'python {self.pyserini_root}/scripts/classifier_prf/cross_validate.py \ - --anserini {self.anserini_root} --run_file {self.tmp} --pyserini {self.pyserini_root} \ - --collection core17 --output {self.tmp}/core17_rrf_rm3.txt --classifier rrf -rm3') + pyserini_topics = 'core17' + lr_cmd = f'{self.pyserini_search_cmd} --index {self.core17_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/core17_lr_rm3.txt ' \ + + f'--prcl lr --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.4 --rm3' - cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ - {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.core17.txt \ - {self.tmp}/core17_rrf_rm3.txt' + status = os.system(lr_cmd) + self.assertEqual(status, 0) - status = os.system(cmd) - stdout, stderr = run_command(cmd) + svm_cmd = f'{self.pyserini_search_cmd} --index {self.core17_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/core17_svm_rm3.txt ' \ + + f'--prcl svm --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.4 --rm3' + + status = os.system(svm_cmd) + self.assertEqual(status, 0) + + rrf_cmd = f'python {self.anserini_root}/src/main/python/fusion.py ' \ + + f'--runs {self.tmp}/core17_lr_rm3.txt {self.tmp}/core17_svm_rm3.txt ' \ + + f'--out {self.tmp}/core17_rrf_rm3.txt' + + status = os.system(rrf_cmd) + self.assertEqual(status, 0) + + score_cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ + {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.core17.txt \ + {self.tmp}/core17_rrf_rm3.txt' + + status = os.system(score_cmd) + stdout, stderr = run_command(score_cmd) score = parse_score(stdout, "map") self.assertEqual(status, 0) self.assertEqual(stderr, '') - self.assertAlmostEqual(score, 0.2951, delta=0.0001) + self.assertAlmostEqual(score, 0.2965, delta=0.0001) def test_core18(self): self.assertTrue(self.core18_checker.run('core18_bm25', '--bm25', 0.2495)) @@ -257,140 +323,206 @@ def test_core18_rm3(self): self.assertTrue(self.core18_checker.run('core18_bm25', '--bm25 --rm3', 0.3135)) def test_core18_lr(self): - os.system(f'python {self.pyserini_root}/scripts/classifier_prf/cross_validate.py \ - --anserini {self.anserini_root} --run_file {self.tmp} --pyserini {self.pyserini_root} \ - --collection core18 --output {self.tmp}/core18_lr.txt --classifier lr') + pyserini_topics = 'core18' - cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ + run_file_cmd = f'{self.pyserini_search_cmd} --index {self.core18_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/core18_lr.txt ' \ + + f'--prcl lr --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.6' + + status = os.system(run_file_cmd) + self.assertEqual(status, 0) + + score_cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.core18.txt \ {self.tmp}/core18_lr.txt' - status = os.system(cmd) - stdout, stderr = run_command(cmd) - score = parse_score(stdout, 'map') + status = os.system(score_cmd) + stdout, stderr = run_command(score_cmd) + score = parse_score(stdout, "map") self.assertEqual(status, 0) self.assertEqual(stderr, '') self.assertAlmostEqual(score, 0.2837, delta=0.0001) def test_core18_lr_rm3(self): - os.system(f'python {self.pyserini_root}/scripts/classifier_prf/cross_validate.py \ - --anserini {self.anserini_root} --run_file {self.tmp} --pyserini {self.pyserini_root} \ - --collection core18 --output {self.tmp}/core18_lr_rm3.txt --classifier lr -rm3') + pyserini_topics = 'core18' - cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ - {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.core18.txt \ - {self.tmp}/core18_lr_rm3.txt' + run_file_cmd = f'{self.pyserini_search_cmd} --index {self.core18_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/core18_lr_rm3.txt ' \ + + f'--prcl lr --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.5 --rm3' - status = os.system(cmd) - stdout, stderr = run_command(cmd) - score = parse_score(stdout, 'map') + status = os.system(run_file_cmd) + self.assertEqual(status, 0) + + score_cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ + {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.core18.txt \ + {self.tmp}/core18_lr_rm3.txt' + + status = os.system(score_cmd) + stdout, stderr = run_command(score_cmd) + score = parse_score(stdout, "map") self.assertEqual(status, 0) self.assertEqual(stderr, '') - self.assertAlmostEqual(score, 0.3195, delta=0.0001) + self.assertAlmostEqual(score, 0.3222, delta=0.0001) def test_core18_svm(self): - os.system(f'python {self.pyserini_root}/scripts/classifier_prf/cross_validate.py \ - --anserini {self.anserini_root} --run_file {self.tmp} --pyserini {self.pyserini_root} \ - --collection core18 --output {self.tmp}/core18_svm.txt --classifier svm ') + pyserini_topics = 'core18' - cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ - {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.core18.txt \ - {self.tmp}/core18_svm.txt' + run_file_cmd = f'{self.pyserini_search_cmd} --index {self.core18_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/core18_svm.txt ' \ + + f'--prcl svm --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.6' - status = os.system(cmd) - stdout, stderr = run_command(cmd) - score = parse_score(stdout, 'map') + status = os.system(run_file_cmd) + self.assertEqual(status, 0) + + score_cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ + {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.core18.txt \ + {self.tmp}/core18_svm.txt' + + status = os.system(score_cmd) + stdout, stderr = run_command(score_cmd) + score = parse_score(stdout, "map") self.assertEqual(status, 0) self.assertEqual(stderr, '') - self.assertAlmostEqual(score, 0.2786, delta=0.0001) + self.assertAlmostEqual(score, 0.2840, delta=0.0001) def test_core18_svm_rm3(self): - os.system(f'python {self.pyserini_root}/scripts/classifier_prf/cross_validate.py \ - --anserini {self.anserini_root} --run_file {self.tmp} --pyserini {self.pyserini_root} \ - --collection core18 --output {self.tmp}/core18_svm_rm3.txt --classifier svm -rm3') + pyserini_topics = 'core18' - cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ - {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.core18.txt \ - {self.tmp}/core18_svm_rm3.txt' + run_file_cmd = f'{self.pyserini_search_cmd} --index {self.core18_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/core18_svm_rm3.txt ' \ + + f'--prcl svm --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.5 --rm3' - status = os.system(cmd) - stdout, stderr = run_command(cmd) - score = parse_score(stdout, 'map') + status = os.system(run_file_cmd) + self.assertEqual(status, 0) + + score_cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ + {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.core18.txt \ + {self.tmp}/core18_svm_rm3.txt' + + status = os.system(score_cmd) + stdout, stderr = run_command(score_cmd) + score = parse_score(stdout, "map") self.assertEqual(status, 0) self.assertEqual(stderr, '') - self.assertAlmostEqual(score, 0.3220, delta=0.0001) + self.assertAlmostEqual(score, 0.3219, delta=0.0001) def test_core18_avg(self): - os.system(f'python {self.pyserini_root}/scripts/classifier_prf/cross_validate.py \ - --anserini {self.anserini_root} --run_file {self.tmp} --pyserini {self.pyserini_root} \ - --collection core18 --output {self.tmp}/core18_lr+svm.txt --classifier lr+svm') + pyserini_topics = 'core18' - cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ - {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.core18.txt \ - {self.tmp}/core18_lr+svm.txt' + run_file_cmd = f'{self.pyserini_search_cmd} --index {self.core18_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/core18_avg.txt ' \ + + f'--prcl lr svm --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.4' - status = os.system(cmd) - stdout, stderr = run_command(cmd) - score = parse_score(stdout, 'map') + status = os.system(run_file_cmd) + self.assertEqual(status, 0) + + score_cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ + {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.core18.txt \ + {self.tmp}/core18_avg.txt' + + status = os.system(score_cmd) + stdout, stderr = run_command(score_cmd) + score = parse_score(stdout, "map") self.assertEqual(status, 0) self.assertEqual(stderr, '') - self.assertAlmostEqual(score, 0.2821, delta=0.0001) + self.assertAlmostEqual(score, 0.2860, delta=0.0001) def test_core18_avg_rm3(self): - os.system(f'python {self.pyserini_root}/scripts/classifier_prf/cross_validate.py \ - --anserini {self.anserini_root} --run_file {self.tmp} --pyserini {self.pyserini_root} \ - --collection core18 --output {self.tmp}/core18_lr+svm_rm3.txt --classifier lr+svm -rm3') + pyserini_topics = 'core18' - cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ - {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.core18.txt \ - {self.tmp}/core18_lr+svm_rm3.txt' + run_file_cmd = f'{self.pyserini_search_cmd} --index {self.core18_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/core18_avg_rm3.txt ' \ + + f'--prcl lr svm --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.4 --rm3' - status = os.system(cmd) - stdout, stderr = run_command(cmd) - score = parse_score(stdout, 'map') + status = os.system(run_file_cmd) + self.assertEqual(status, 0) + + score_cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ + {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.core18.txt \ + {self.tmp}/core18_avg_rm3.txt' + + status = os.system(score_cmd) + stdout, stderr = run_command(score_cmd) + score = parse_score(stdout, "map") self.assertEqual(status, 0) self.assertEqual(stderr, '') - self.assertAlmostEqual(score, 0.3200, delta=0.0001) + self.assertAlmostEqual(score, 0.3227, delta=0.0001) def test_core18_rrf(self): - os.system(f'python {self.pyserini_root}/scripts/classifier_prf/cross_validate.py \ - --anserini {self.anserini_root} --run_file {self.tmp} --pyserini {self.pyserini_root} \ - --collection core18 --output {self.tmp}/core18_rrf.txt --classifier rrf') + pyserini_topics = 'core18' + lr_cmd = f'{self.pyserini_search_cmd} --index {self.core18_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/core18_lr.txt ' \ + + f'--prcl lr --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.6' - cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ - {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.core18.txt \ - {self.tmp}/core18_rrf.txt' + status = os.system(lr_cmd) + self.assertEqual(status, 0) - status = os.system(cmd) - stdout, stderr = run_command(cmd) - score = parse_score(stdout, 'map') + svm_cmd = f'{self.pyserini_search_cmd} --index {self.core18_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/core18_svm.txt ' \ + + f'--prcl svm --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.6' + + status = os.system(svm_cmd) + self.assertEqual(status, 0) + + rrf_cmd = f'python {self.anserini_root}/src/main/python/fusion.py ' \ + + f'--runs {self.tmp}/core18_lr.txt {self.tmp}/core18_svm.txt ' \ + + f'--out {self.tmp}/core18_rrf.txt' + + status = os.system(rrf_cmd) + self.assertEqual(status, 0) + + score_cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ + {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.core18.txt \ + {self.tmp}/core18_rrf.txt' + + status = os.system(score_cmd) + stdout, stderr = run_command(score_cmd) + score = parse_score(stdout, "map") self.assertEqual(status, 0) self.assertEqual(stderr, '') - self.assertAlmostEqual(score, 0.2871, delta=0.0001) + self.assertAlmostEqual(score, 0.2881, delta=0.0001) def test_core18_rrf_rm3(self): - os.system(f'python {self.pyserini_root}/scripts/classifier_prf/cross_validate.py \ - --anserini {self.anserini_root} --run_file {self.tmp} --pyserini {self.pyserini_root} \ - --collection core18 --output {self.tmp}/core18_rrf_rm3.txt --classifier rrf -rm3') + pyserini_topics = 'core18' + lr_cmd = f'{self.pyserini_search_cmd} --index {self.core18_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/core18_lr_rm3.txt ' \ + + f'--prcl lr --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.5 --rm3' - cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ - {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.core18.txt \ - {self.tmp}/core18_rrf_rm3.txt' + status = os.system(lr_cmd) + self.assertEqual(status, 0) - status = os.system(cmd) - stdout, stderr = run_command(cmd) - score = parse_score(stdout, 'map') + svm_cmd = f'{self.pyserini_search_cmd} --index {self.core18_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/core18_svm_rm3.txt ' \ + + f'--prcl svm --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.5 --rm3' + + status = os.system(svm_cmd) + self.assertEqual(status, 0) + + rrf_cmd = f'python {self.anserini_root}/src/main/python/fusion.py ' \ + + f'--runs {self.tmp}/core18_lr_rm3.txt {self.tmp}/core18_svm_rm3.txt ' \ + + f'--out {self.tmp}/core18_rrf_rm3.txt' + + status = os.system(rrf_cmd) + self.assertEqual(status, 0) + + score_cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ + {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.core18.txt \ + {self.tmp}/core18_rrf_rm3.txt' + + status = os.system(score_cmd) + stdout, stderr = run_command(score_cmd) + score = parse_score(stdout, "map") self.assertEqual(status, 0) self.assertEqual(stderr, '') - self.assertAlmostEqual(score, 0.3204, delta=0.0001) + self.assertAlmostEqual(score, 0.3214, delta=0.0001) def test_robust04(self): self.assertTrue(self.robust04_checker.run('robust04_bm25', '--bm25', 0.2531)) @@ -399,136 +531,202 @@ def test_robust04_rm3(self): self.assertTrue(self.robust04_checker.run('robust04_bm25_rm3', '--bm25 --rm3', 0.2903)) def test_robust04_lr(self): - os.system(f'python {self.pyserini_root}/scripts/classifier_prf/cross_validate.py \ - --anserini {self.anserini_root} --run_file {self.tmp} --pyserini {self.pyserini_root} \ - --collection robust04 --output {self.tmp}/robust04_lr.txt --classifier lr ') + pyserini_topics = 'robust04' - cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ - {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.robust04.txt \ - {self.tmp}/robust04_lr.txt' + run_file_cmd = f'{self.pyserini_search_cmd} --index {self.robust04_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/robust04_lr.txt ' \ + + f'--prcl lr --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.5' - status = os.system(cmd) - stdout, stderr = run_command(cmd) - score = parse_score(stdout, 'map') + status = os.system(run_file_cmd) + self.assertEqual(status, 0) + + score_cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ + {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.robust04.txt \ + {self.tmp}/robust04_lr.txt' + + status = os.system(score_cmd) + stdout, stderr = run_command(score_cmd) + score = parse_score(stdout, "map") self.assertEqual(status, 0) self.assertEqual(stderr, '') - self.assertAlmostEqual(score, 0.2721, delta=0.0001) + self.assertAlmostEqual(score, 0.2747, delta=0.0001) def test_robust04_lr_rm3(self): - os.system(f'python {self.pyserini_root}/scripts/classifier_prf/cross_validate.py \ - --anserini {self.anserini_root} --run_file {self.tmp} --pyserini {self.pyserini_root} \ - --collection robust04 --output {self.tmp}/robust04_lr_rm3.txt --classifier lr -rm3') + pyserini_topics = 'robust04' - cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ - {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.robust04.txt \ - {self.tmp}/robust04_lr_rm3.txt' + run_file_cmd = f'{self.pyserini_search_cmd} --index {self.robust04_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/robust04_lr_rm3.txt ' \ + + f'--prcl lr --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.3 --rm3' - status = os.system(cmd) - stdout, stderr = run_command(cmd) - score = parse_score(stdout, 'map') + status = os.system(run_file_cmd) + self.assertEqual(status, 0) + + score_cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ + {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.robust04.txt \ + {self.tmp}/robust04_lr_rm3.txt' + + status = os.system(score_cmd) + stdout, stderr = run_command(score_cmd) + score = parse_score(stdout, "map") self.assertEqual(status, 0) self.assertEqual(stderr, '') - self.assertAlmostEqual(score, 0.2961, delta=0.0001) + self.assertAlmostEqual(score, 0.2971, delta=0.0001) def test_robust04_svm(self): - os.system(f'python {self.pyserini_root}/scripts/classifier_prf/cross_validate.py \ - --anserini {self.anserini_root} --run_file {self.tmp} --pyserini {self.pyserini_root} \ - --collection robust04 --output {self.tmp}/robust04_svm.txt --classifier svm ') + pyserini_topics = 'robust04' - cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ - {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.robust04.txt \ - {self.tmp}/robust04_svm.txt' + run_file_cmd = f'{self.pyserini_search_cmd} --index {self.robust04_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/robust04_svm.txt ' \ + + f'--prcl svm --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.5' - status = os.system(cmd) - stdout, stderr = run_command(cmd) - score = parse_score(stdout, 'map') + status = os.system(run_file_cmd) + self.assertEqual(status, 0) + + score_cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ + {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.robust04.txt \ + {self.tmp}/robust04_svm.txt' + + status = os.system(score_cmd) + stdout, stderr = run_command(score_cmd) + score = parse_score(stdout, "map") self.assertEqual(status, 0) self.assertEqual(stderr, '') - self.assertAlmostEqual(score, 0.2716, delta=0.0001) + self.assertAlmostEqual(score, 0.2726, delta=0.0001) def test_robust04_svm_rm3(self): - os.system(f'python {self.pyserini_root}/scripts/classifier_prf/cross_validate.py \ - --anserini {self.anserini_root} --run_file {self.tmp} --pyserini {self.pyserini_root} \ - --collection robust04 --output {self.tmp}/robust04_svm_rm3.txt --classifier svm -rm3') + pyserini_topics = 'robust04' - cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ - {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.robust04.txt \ - {self.tmp}/robust04_svm_rm3.txt' + run_file_cmd = f'{self.pyserini_search_cmd} --index {self.robust04_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/robust04_svm_rm3.txt ' \ + + f'--prcl svm --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.3 --rm3' - status = os.system(cmd) - stdout, stderr = run_command(cmd) - score = parse_score(stdout, 'map') + status = os.system(run_file_cmd) + self.assertEqual(status, 0) + + score_cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ + {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.robust04.txt \ + {self.tmp}/robust04_svm_rm3.txt' + + status = os.system(score_cmd) + stdout, stderr = run_command(score_cmd) + score = parse_score(stdout, "map") self.assertEqual(status, 0) self.assertEqual(stderr, '') - self.assertAlmostEqual(score, 0.2963, delta=0.0001) + self.assertAlmostEqual(score, 0.2967, delta=0.0001) def test_robust04_avg(self): - os.system(f'python {self.pyserini_root}/scripts/classifier_prf/cross_validate.py \ - --anserini {self.anserini_root} --run_file {self.tmp} --pyserini {self.pyserini_root} \ - --collection robust04 --output {self.tmp}/robust04_lr+svm.txt --classifier lr+svm') + pyserini_topics = 'robust04' - cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ - {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.robust04.txt \ - {self.tmp}/robust04_lr+svm.txt' + run_file_cmd = f'{self.pyserini_search_cmd} --index {self.robust04_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/robust04_avg.txt ' \ + + f'--prcl lr svm --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.5' - status = os.system(cmd) - stdout, stderr = run_command(cmd) - score = parse_score(stdout, 'map') + status = os.system(run_file_cmd) + self.assertEqual(status, 0) + + score_cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ + {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.robust04.txt \ + {self.tmp}/robust04_avg.txt' + + status = os.system(score_cmd) + stdout, stderr = run_command(score_cmd) + score = parse_score(stdout, "map") self.assertEqual(status, 0) self.assertEqual(stderr, '') - self.assertAlmostEqual(score, 0.2745, delta=0.0001) + self.assertAlmostEqual(score, 0.276, delta=0.0001) def test_robust04_avg_rm3(self): - os.system(f'python {self.pyserini_root}/scripts/classifier_prf/cross_validate.py \ - --anserini {self.anserini_root} --run_file {self.tmp} --pyserini {self.pyserini_root} \ - --collection robust04 --output {self.tmp}/robust04_lr+svm_rm3.txt --classifier lr+svm -rm3') + pyserini_topics = 'robust04' - cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ - {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.robust04.txt \ - {self.tmp}/robust04_lr+svm_rm3.txt' + run_file_cmd = f'{self.pyserini_search_cmd} --index {self.robust04_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/robust04_avg_rm3.txt ' \ + + f'--prcl lr svm --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.3 --rm3' - status = os.system(cmd) - stdout, stderr = run_command(cmd) - score = parse_score(stdout, 'map') + status = os.system(run_file_cmd) + self.assertEqual(status, 0) + + score_cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ + {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.robust04.txt \ + {self.tmp}/robust04_avg_rm3.txt' + + status = os.system(score_cmd) + stdout, stderr = run_command(score_cmd) + score = parse_score(stdout, "map") self.assertEqual(status, 0) self.assertEqual(stderr, '') self.assertAlmostEqual(score, 0.2980, delta=0.0001) def test_robust04_rrf(self): - os.system(f'python {self.pyserini_root}/scripts/classifier_prf/cross_validate.py \ - --anserini {self.anserini_root} --run_file {self.tmp} --pyserini {self.pyserini_root} \ - --collection robust04 --output {self.tmp}/robust04_rrf.txt --classifier rrf') + pyserini_topics = 'robust04' + lr_cmd = f'{self.pyserini_search_cmd} --index {self.robust04_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/robust04_lr.txt ' \ + + f'--prcl lr --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.5' - cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ - {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.robust04.txt \ - {self.tmp}/robust04_rrf.txt' + status = os.system(lr_cmd) + self.assertEqual(status, 0) - status = os.system(cmd) - stdout, stderr = run_command(cmd) - score = parse_score(stdout, 'map') + svm_cmd = f'{self.pyserini_search_cmd} --index {self.robust04_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/robust04_svm.txt ' \ + + f'--prcl svm --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.5' + + status = os.system(svm_cmd) + self.assertEqual(status, 0) + + rrf_cmd = f'python {self.anserini_root}/src/main/python/fusion.py ' \ + + f'--runs {self.tmp}/robust04_lr.txt {self.tmp}/robust04_svm.txt ' \ + + f'--out {self.tmp}/robust04_rrf.txt' + + status = os.system(rrf_cmd) + self.assertEqual(status, 0) + + score_cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ + {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.robust04.txt \ + {self.tmp}/robust04_rrf.txt' + + status = os.system(score_cmd) + stdout, stderr = run_command(score_cmd) + score = parse_score(stdout, "map") self.assertEqual(status, 0) self.assertEqual(stderr, '') - self.assertAlmostEqual(score, 0.2740, delta=0.0001) + self.assertAlmostEqual(score, 0.275, delta=0.0001) def test_robust04_rrf_rm3(self): - os.system(f'python {self.pyserini_root}/scripts/classifier_prf/cross_validate.py \ - --anserini {self.anserini_root} --run_file {self.tmp} --pyserini {self.pyserini_root} \ - --collection robust04 --output {self.tmp}/robust04_rrf_rm3.txt --classifier rrf -rm3') + pyserini_topics = 'robust04' + lr_cmd = f'{self.pyserini_search_cmd} --index {self.robust04_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/robust04_lr_rm3.txt ' \ + + f'--prcl lr --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.3 --rm3' - cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ - {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.robust04.txt \ - {self.tmp}/robust04_rrf_rm3.txt' + status = os.system(lr_cmd) + self.assertEqual(status, 0) - status = os.system(cmd) - stdout, stderr = run_command(cmd) - score = parse_score(stdout, 'map') + svm_cmd = f'{self.pyserini_search_cmd} --index {self.robust04_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/robust04_svm_rm3.txt ' \ + + f'--prcl svm --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.3 --rm3' + + status = os.system(svm_cmd) + self.assertEqual(status, 0) + + rrf_cmd = f'python {self.anserini_root}/src/main/python/fusion.py ' \ + + f'--runs {self.tmp}/robust04_lr_rm3.txt {self.tmp}/robust04_svm_rm3.txt ' \ + + f'--out {self.tmp}/robust04_rrf_rm3.txt' + + status = os.system(rrf_cmd) + self.assertEqual(status, 0) + + score_cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ + {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.robust04.txt \ + {self.tmp}/robust04_rrf_rm3.txt' + + status = os.system(score_cmd) + stdout, stderr = run_command(score_cmd) + score = parse_score(stdout, "map") self.assertEqual(status, 0) self.assertEqual(stderr, '') @@ -541,140 +739,206 @@ def test_robust05_rm3(self): self.assertTrue(self.robust05_checker.run('robust05_bm25_rm3', '--bm25 --rm3', 0.2602)) def test_robust05_lr(self): - os.system(f'python {self.pyserini_root}/scripts/classifier_prf/cross_validate.py \ - --anserini {self.anserini_root} --run_file {self.tmp} --pyserini {self.pyserini_root} \ - --collection robust05 --output {self.tmp}/robust05_lr.txt --classifier lr ') + pyserini_topics = 'robust05' - cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ - {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.robust05.txt \ - {self.tmp}/robust05_lr.txt' + run_file_cmd = f'{self.pyserini_search_cmd} --index {self.robust05_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/robust05_lr.txt ' \ + + f'--prcl lr --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.8' - status = os.system(cmd) - stdout, stderr = run_command(cmd) - score = parse_score(stdout, 'map') + status = os.system(run_file_cmd) + self.assertEqual(status, 0) + + score_cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ + {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.robust05.txt \ + {self.tmp}/robust05_lr.txt' + + status = os.system(score_cmd) + stdout, stderr = run_command(score_cmd) + score = parse_score(stdout, "map") self.assertEqual(status, 0) self.assertEqual(stderr, '') self.assertAlmostEqual(score, 0.2476, delta=0.0001) def test_robust05_lr_rm3(self): - os.system(f'python {self.pyserini_root}/scripts/classifier_prf/cross_validate.py \ - --anserini {self.anserini_root} --run_file {self.tmp} --pyserini {self.pyserini_root} \ - --collection robust05 --output {self.tmp}/robust05_lr_rm3.txt --classifier lr -rm3') + pyserini_topics = 'robust05' - cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ - {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.robust05.txt \ - {self.tmp}/robust05_lr_rm3.txt' + run_file_cmd = f'{self.pyserini_search_cmd} --index {self.robust05_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/robust05_lr_rm3.txt ' \ + + f'--prcl lr --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.6 --rm3' - status = os.system(cmd) - stdout, stderr = run_command(cmd) - score = parse_score(stdout, 'map') + status = os.system(run_file_cmd) + self.assertEqual(status, 0) + + score_cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ + {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.robust05.txt \ + {self.tmp}/robust05_lr_rm3.txt' + + status = os.system(score_cmd) + stdout, stderr = run_command(score_cmd) + score = parse_score(stdout, "map") self.assertEqual(status, 0) self.assertEqual(stderr, '') - self.assertAlmostEqual(score, 0.2845, delta=0.0001) + self.assertAlmostEqual(score, 0.2854, delta=0.0001) def test_robust05_svm(self): - os.system(f'python {self.pyserini_root}/scripts/classifier_prf/cross_validate.py \ - --anserini {self.anserini_root} --run_file {self.tmp} --pyserini {self.pyserini_root} \ - --collection robust05 --output {self.tmp}/robust05_svm.txt --classifier svm ') + pyserini_topics = 'robust05' - cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ - {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.robust05.txt \ - {self.tmp}/robust05_svm.txt' + run_file_cmd = f'{self.pyserini_search_cmd} --index {self.robust05_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/robust05_svm.txt ' \ + + f'--prcl svm --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.8' - status = os.system(cmd) - stdout, stderr = run_command(cmd) - score = parse_score(stdout, 'map') + status = os.system(run_file_cmd) + self.assertEqual(status, 0) + + score_cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ + {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.robust05.txt \ + {self.tmp}/robust05_svm.txt' + + status = os.system(score_cmd) + stdout, stderr = run_command(score_cmd) + score = parse_score(stdout, "map") self.assertEqual(status, 0) self.assertEqual(stderr, '') - self.assertAlmostEqual(score, 0.2454, delta=0.0001) + self.assertAlmostEqual(score, 0.2486, delta=0.0001) def test_robust05_svm_rm3(self): - os.system(f'python {self.pyserini_root}/scripts/classifier_prf/cross_validate.py \ - --anserini {self.anserini_root} --run_file {self.tmp} --pyserini {self.pyserini_root} \ - --collection robust05 --output {self.tmp}/robust05_svm_rm3.txt --classifier svm -rm3') + pyserini_topics = 'robust05' - cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ - {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.robust05.txt \ - {self.tmp}/robust05_svm_rm3.txt' + run_file_cmd = f'{self.pyserini_search_cmd} --index {self.robust05_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/robust05_svm_rm3.txt ' \ + + f'--prcl svm --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.6 --rm3' - status = os.system(cmd) - stdout, stderr = run_command(cmd) - score = parse_score(stdout, 'map') + status = os.system(run_file_cmd) + self.assertEqual(status, 0) + + score_cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ + {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.robust05.txt \ + {self.tmp}/robust05_svm_rm3.txt' + + status = os.system(score_cmd) + stdout, stderr = run_command(score_cmd) + score = parse_score(stdout, "map") self.assertEqual(status, 0) self.assertEqual(stderr, '') - self.assertAlmostEqual(score, 0.2850, delta=0.0001) + self.assertAlmostEqual(score, 0.2855, delta=0.0001) def test_robust05_avg(self): - os.system(f'python {self.pyserini_root}/scripts/classifier_prf/cross_validate.py \ - --anserini {self.anserini_root} --run_file {self.tmp} --pyserini {self.pyserini_root} \ - --collection robust05 --output {self.tmp}/robust05_lr+svm.txt --classifier lr+svm ') + pyserini_topics = 'robust05' - cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ - {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.robust05.txt \ - {self.tmp}/robust05_lr+svm.txt' + run_file_cmd = f'{self.pyserini_search_cmd} --index {self.robust05_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/robust05_avg.txt ' \ + + f'--prcl lr svm --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.8' - status = os.system(cmd) - stdout, stderr = run_command(cmd) - score = parse_score(stdout, 'map') + status = os.system(run_file_cmd) + self.assertEqual(status, 0) + + score_cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ + {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.robust05.txt \ + {self.tmp}/robust05_avg.txt' + + status = os.system(score_cmd) + stdout, stderr = run_command(score_cmd) + score = parse_score(stdout, "map") self.assertEqual(status, 0) self.assertEqual(stderr, '') self.assertAlmostEqual(score, 0.2485, delta=0.0001) def test_robust05_avg_rm3(self): - os.system(f'python {self.pyserini_root}/scripts/classifier_prf/cross_validate.py \ - --anserini {self.anserini_root} --run_file {self.tmp} --pyserini {self.pyserini_root} \ - --collection robust05 --output {self.tmp}/robust05_lr+svm_rm3.txt --classifier lr+svm -rm3') + pyserini_topics = 'robust05' - cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ - {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.robust05.txt \ - {self.tmp}/robust05_lr+svm_rm3.txt' + run_file_cmd = f'{self.pyserini_search_cmd} --index {self.robust05_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/robust05_avg_rm3.txt ' \ + + f'--prcl lr svm --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.6 --rm3' - status = os.system(cmd) - stdout, stderr = run_command(cmd) - score = parse_score(stdout, 'map') + status = os.system(run_file_cmd) + self.assertEqual(status, 0) + + score_cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ + {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.robust05.txt \ + {self.tmp}/robust05_avg_rm3.txt' + + status = os.system(score_cmd) + stdout, stderr = run_command(score_cmd) + score = parse_score(stdout, "map") self.assertEqual(status, 0) self.assertEqual(stderr, '') - self.assertAlmostEqual(score, 0.2860, delta=0.0001) + self.assertAlmostEqual(score, 0.2865, delta=0.0001) def test_robust05_rrf(self): - os.system(f'python {self.pyserini_root}/scripts/classifier_prf/cross_validate.py \ - --anserini {self.anserini_root} --run_file {self.tmp} --pyserini {self.pyserini_root} \ - --collection robust05 --output {self.tmp}/robust05_rrf.txt --classifier rrf') + pyserini_topics = 'robust05' + lr_cmd = f'{self.pyserini_search_cmd} --index {self.robust05_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/robust05_lr.txt ' \ + + f'--prcl lr --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.5' - cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ - {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.robust05.txt \ - {self.tmp}/robust05_rrf.txt' + status = os.system(lr_cmd) + self.assertEqual(status, 0) - status = os.system(cmd) - stdout, stderr = run_command(cmd) - score = parse_score(stdout, 'map') + svm_cmd = f'{self.pyserini_search_cmd} --index {self.robust05_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/robust05_svm.txt ' \ + + f'--prcl svm --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.5' + + status = os.system(svm_cmd) + self.assertEqual(status, 0) + + rrf_cmd = f'python {self.anserini_root}/src/main/python/fusion.py ' \ + + f'--runs {self.tmp}/robust05_lr.txt {self.tmp}/robust05_svm.txt ' \ + + f'--out {self.tmp}/robust05_rrf.txt' + + status = os.system(rrf_cmd) + self.assertEqual(status, 0) + + score_cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ + {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.robust05.txt \ + {self.tmp}/robust05_rrf.txt' + + status = os.system(score_cmd) + stdout, stderr = run_command(score_cmd) + score = parse_score(stdout, "map") self.assertEqual(status, 0) self.assertEqual(stderr, '') - self.assertAlmostEqual(score, 0.2490, delta=0.0001) + self.assertAlmostEqual(score, 0.2401, delta=0.0001) def test_robust05_rrf_rm3(self): - os.system(f'python {self.pyserini_root}/scripts/classifier_prf/cross_validate.py \ - --anserini {self.anserini_root} --run_file {self.tmp} --pyserini {self.pyserini_root} \ - --collection robust05 --output {self.tmp}/robust05_rrf_rm3.txt --classifier rrf -rm3') + pyserini_topics = 'robust05' + lr_cmd = f'{self.pyserini_search_cmd} --index {self.robust05_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/robust05_lr_rm3.txt ' \ + + f'--prcl lr --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.3 --rm3' - cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ - {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.robust05.txt \ - {self.tmp}/robust05_rrf_rm3.txt' + status = os.system(lr_cmd) + self.assertEqual(status, 0) - status = os.system(cmd) - stdout, stderr = run_command(cmd) - score = parse_score(stdout, 'map') + svm_cmd = f'{self.pyserini_search_cmd} --index {self.robust05_index_path} ' \ + + f'--topics {pyserini_topics} --output {self.tmp}/robust05_svm_rm3.txt ' \ + + f'--prcl svm --prcl.vectorizer TfidfVectorizer --prcl.alpha 0.3 --rm3' + + status = os.system(svm_cmd) + self.assertEqual(status, 0) + + rrf_cmd = f'python {self.anserini_root}/src/main/python/fusion.py ' \ + + f'--runs {self.tmp}/robust05_lr_rm3.txt {self.tmp}/robust05_svm_rm3.txt ' \ + + f'--out {self.tmp}/robust05_rrf_rm3.txt' + + status = os.system(rrf_cmd) + self.assertEqual(status, 0) + + score_cmd = f'{self.anserini_root}/tools/eval/trec_eval.9.0.4/trec_eval -m map -m P.30 \ + {self.anserini_root}/src/main/resources/topics-and-qrels/qrels.robust05.txt \ + {self.tmp}/robust05_rrf_rm3.txt' + + status = os.system(score_cmd) + stdout, stderr = run_command(score_cmd) + score = parse_score(stdout, "map") self.assertEqual(status, 0) self.assertEqual(stderr, '') - self.assertAlmostEqual(score, 0.2870, delta=0.0001) + self.assertAlmostEqual(score, 0.2788, delta=0.0001) def tearDown(self): shutil.rmtree(f'{self.tmp}')