Skip to content
This repository has been archived by the owner on Nov 3, 2023. It is now read-only.

SeeKeR #4447

Merged
merged 15 commits into from
Mar 25, 2022
Prev Previous commit
Next Next commit
various updates
  • Loading branch information
klshuster committed Mar 23, 2022
commit 56c9924880b8bc3513c6526c8da99c9d0a2de985
8 changes: 8 additions & 0 deletions parlai/opt_presets/gen/seeker_dialogue.opt
Original file line number Diff line number Diff line change
@@ -9,7 +9,10 @@
"drm_inference": "beam",
"drm_message_mutators": null,
"drm_model": "projects.seeker.agents.seeker:ComboFidSearchQueryAgent",
"exclude_context_in_krm_context_blocking": false,
"include_knowledge_in_krm_context_blocking": true,
"inject_query_string": null,
"knowledge_response_control_token": null,
"krm_beam_block_ngram": 3,
"krm_beam_context_block_ngram": 3,
"krm_beam_min_length": 1,
@@ -33,6 +36,11 @@
"sdm_inference": "greedy",
"sdm_model": "projects.seeker.agents.seeker:ComboFidSearchQueryAgent",
"search_decision": "always",
"search_decision_control_token": "__is-search-required__",
"search_decision_do_search_reply": "__do-search__",
"search_decision_dont_search_reply": "__do-not-search__",
"search_query_control_token": "__generate-query__",
"search_server": null,
"sqm_beam_block_ngram": -1,
"sqm_beam_min_length": 2,
"sqm_beam_size": 1,
8 changes: 8 additions & 0 deletions parlai/opt_presets/gen/seeker_lm.opt
Original file line number Diff line number Diff line change
@@ -9,7 +9,10 @@
"drm_inference": "beam",
"drm_message_mutators": null,
"drm_model": "projects.seeker.agents.gpt2_seeker:GPT2ComboSearchQueryAgent",
"exclude_context_in_krm_context_blocking": false,
"include_knowledge_in_krm_context_blocking": true,
"inject_query_string": null,
"knowledge_response_control_token": null,
"krm_beam_block_ngram": 3,
"krm_beam_context_block_ngram": -1,
"krm_beam_min_length": 1,
@@ -32,6 +35,11 @@
"sdm_history_size": 1,
"sdm_inference": "greedy",
"search_decision": "always",
"search_decision_control_token": "__is-search-required__",
"search_decision_do_search_reply": "__do-search__",
"search_decision_dont_search_reply": "__do-not-search__",
"search_query_control_token": "__generate-query__",
"search_server": null,
"sqm_beam_block_ngram": -1,
"sqm_beam_min_length": 2,
"sqm_beam_size": 1,
7 changes: 6 additions & 1 deletion parlai/tasks/natural_questions/agents.py
Original file line number Diff line number Diff line change
@@ -24,7 +24,9 @@
from parlai.core.metrics import ExactMatchMetric, normalize_answer, AverageMetric
from parlai.core.params import ParlaiParser

from parlai.utils.data import DatatypeHelper
from parlai.utils.io import PathManager
import parlai.utils.logging as logging


def _count_lines_in_file(fname):
@@ -129,7 +131,10 @@ def __init__(self, opt, shared=None):
self.use_context = opt.get('use_context', False)
self.id = 'natural_questions'
self.opt = copy.deepcopy(opt)
self.dtype = self.opt['datatype'].split(':')[0]
self.dtype = DatatypeHelper.fold(self.opt['datatype'])
if self.dtype == 'test':
logging.error("No test split for this teacher; overriding to valid")
self.dtype = 'valid'
self.dpath = os.path.join(self.opt['datapath'], DATASET_NAME_LOCAL, self.dtype)
self.n_samples = None
super().__init__(self.opt, shared)
17 changes: 17 additions & 0 deletions parlai/tasks/natural_questions/test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
#!/usr/bin/env python3

# Copyright (c) Facebook, Inc. and its affiliates.
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

from parlai.utils.testing import AutoTeacherTest # noqa: F401


class TestNaturalQuestionsTeacher(AutoTeacherTest):
task = 'natural_questions' # replace with your teacher name


class TestNaturalQuestionsOpenTeacher(AutoTeacherTest):
task = (
'natural_questions:NaturalQuestionsOpenTeacher'
) # replace with your teacher name
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
acts:
- - answers: '["Wilhelm Conrad R\u00f6ntgen"]'
checked_sentence: 'The first Nobel Prize in Physics was awarded in 1901 to Wilhelm
Conrad Röntgen , of Germany , who received 150,782 SEK , which is equal to 7,731,004
SEK in December 2007 . John Bardeen is the only laureate to win the prize twice
-- in 1956 and 1972 . Maria Skłodowska - Curie also won two Nobel Prizes , for
physics in 1903 and chemistry in 1911 . William Lawrence Bragg was , until October
2014 , the youngest ever Nobel laureate ; he won the prize in 1915 at the age
of 25 . Two women have won the prize : Curie and Maria Goeppert - Mayer ( 1963
) . As of 2017 , the prize has been awarded to 206 individuals . There have
been six years in which the Nobel Prize in Physics was not awarded ( 1916 ,
1931 , 1934 , 1940 -- 1942 ) .'
episode_done: true
eval_labels:
- Wilhelm Conrad Röntgen
id: natural_questions:NaturalQuestionsOpenTeacher
text: who got the first nobel prize in physics
title: List of Nobel laureates in Physics
- - answers: '["May 18 , 2018"]'
checked_sentence: Deadpool 2 is scheduled to be released in the United States
on May 18 , 2018 . A sequel , Deadpool 3 , is in development .
episode_done: true
eval_labels:
- May 18 , 2018
id: natural_questions:NaturalQuestionsOpenTeacher
text: when is the next deadpool movie being released
title: Deadpool 2
- - answers: '["MFSK", "Olivia"]'
checked_sentence: ''
episode_done: true
eval_labels:
- MFSK
id: natural_questions:NaturalQuestionsOpenTeacher
text: which mode is used for short wave broadcast service
title: Shortwave radio
- - answers: '["till September"]'
checked_sentence: ''
episode_done: true
eval_labels:
- till September
id: natural_questions:NaturalQuestionsOpenTeacher
text: the south west wind blows across nigeria between
title: Geography of Nigeria
- - answers: '["hit points or health points"]'
checked_sentence: ''
episode_done: true
eval_labels:
- hit points or health points
id: natural_questions:NaturalQuestionsOpenTeacher
text: what does hp mean in war and order
title: Health (gaming)
num_episodes: 3610
num_examples: 3610
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
acts:
- - answers: '["2,718"]'
checked_sentence: As of June 14 , 2018 , there were 2,718 death row inmates in
the United States . The number of death row inmates changes daily with new convictions
, appellate decisions overturning conviction or sentence alone , commutations
, or deaths ( through execution or otherwise ) . Due to this fluctuation as
well as lag and inconsistencies in inmate reporting procedures across jurisdictions
, the information in this article may be out of date .
episode_done: true
id: natural_questions:NaturalQuestionsOpenTeacher
labels:
- 2,718
text: total number of death row inmates in the us
title: List of death row inmates in the United States
- - answers: '["seven"]'
checked_sentence: Despite originally being billed as a miniseries , HBO renewed
the series for a second season . Production on the second season began in March
2018 and is set to premiere in 2019 . The entire main cast will return with
the addition of Meryl Streep . All seven episodes are being written by Kelley
and directed by Andrea Arnold .
episode_done: true
id: natural_questions:NaturalQuestionsOpenTeacher
labels:
- seven
text: big little lies season 2 how many episodes
title: Big Little Lies (TV series)
- - answers: '["Foreigner"]'
checked_sentence: '`` Waiting for a Girl Like You '''' is a 1981 power ballad
by the British - American rock band Foreigner . The distinctive synthesizer
theme was performed by the then - little - known Thomas Dolby , this song also
marked a major departure from their earlier singles because their previous singles
were mid to upper tempo rock songs while this song was a softer love song with
the energy of a power ballad .'
episode_done: true
id: natural_questions:NaturalQuestionsOpenTeacher
labels:
- Foreigner
text: who sang waiting for a girl like you
title: Waiting for a Girl Like You
- - answers: '["Saltfjellet"]'
checked_sentence: The Arctic circle crosses mainland Norway at Saltfjellet , which
separates Helgeland from the northern part of Nordland county . Thus about half
of the county lies north of the Arctic circle , along with the whole of Troms
and Finnmark counties . The total area of mainland Norway above the Arctic circle
is ca. 95,000 km ( 37,000 sq mi ) . The population is about 393,000 , which
makes this the most populated arctic region in the world .
episode_done: true
id: natural_questions:NaturalQuestionsOpenTeacher
labels:
- Saltfjellet
text: where do you cross the arctic circle in norway
title: Arctic Norway
- - answers: '["Sam - I - am"]'
checked_sentence: A character named `` Sam - I - am '' pesters an unnamed character
to try a plate of green eggs and ham . The unnamed character refuses , responding
, `` I do not like green eggs and ham . I do not like them , Sam - I - am .
'' He continues to repeat this as Sam persistently follows him , asking him
to try them in eight locations ( house , box , car , tree , train , dark , rain
, boat ) and with three animals ( mouse , fox , and goat ) . Finally , he gives
into Sam 's pestering and tries the green eggs and ham , which he does like
after all and happily responds , `` I do so like green eggs and ham . Thank
you . Thank you , Sam - I - am . ''
episode_done: true
id: natural_questions:NaturalQuestionsOpenTeacher
labels:
- Sam - I - am
text: who is the main character in green eggs and ham
title: Green Eggs and Ham
num_episodes: 79168
num_examples: 79168
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
acts:
- - answers: '["Linda Davis"]'
checked_sentence: '`` Does He Love You '''' is a song written by Sandy Knox and
Billy Stritch , and recorded as a duet by American country music artists Reba
McEntire and Linda Davis . It was released in August 1993 as the first single
from Reba ''s album Greatest Hits Volume Two . It is one of country music ''s
several songs about a love triangle .'
episode_done: true
eval_labels:
- Linda Davis
id: natural_questions:NaturalQuestionsOpenTeacher
text: who sings does he love me with reba
title: Does He Love You
- - answers: '["581 ( second edition )"]'
checked_sentence: Invisible Man First edition Author Ralph Ellison Country United
States Language English Genre Bildungsroman African - American literature social
commentary Publisher Random House Publication date 1952 Media type Print ( hardcover
and paperback ) Pages 581 ( second edition ) ISBN 978 - 0 - 679 - 60139 - 5
OCLC 30780333 Dewey Decimal 813 /. 54 20 LC Class PS3555. L625 I5 1994
episode_done: true
eval_labels:
- 581 ( second edition )
id: natural_questions:NaturalQuestionsOpenTeacher
text: how many pages is invisible man by ralph ellison
title: Invisible Man
- - answers: '["the Saint Lawrence River"]'
checked_sentence: 'The Great Lakes ( French : les Grands - Lacs ) , also called
the Laurentian Great Lakes and the Great Lakes of North America , are a series
of interconnected freshwater lakes located primarily in the upper mid-east region
of North America , on the Canada -- United States border , which connect to
the Atlantic Ocean through the Saint Lawrence River . They consist of Lakes
Superior , Michigan , Huron ( or Michigan -- Huron ) , Erie , and Ontario .'
episode_done: true
eval_labels:
- the Saint Lawrence River
id: natural_questions:NaturalQuestionsOpenTeacher
text: where do the great lakes meet the ocean
title: Great Lakes
- - answers: '["July 5 , 2018"]'
checked_sentence: 'My Hero Academia : Two Heroes ( Japanese : 僕 の ヒーロー アカデミア THE
MOVIE ~ 2 人 の 英雄 ( ヒーロー ) ~ , Hepburn : Boku no Hīrō Academia THE MOVIE : Futari
no Hīrō ) is a 2018 Japanese anime superhero film based on the manga My Hero
Academia by Kōhei Horikoshi . Set between the second and third seasons of the
anime series , the film was directed by Kenji Nagasaki and produced by Bones
. Anime Expo hosted the film ''s world premiere on July 5 , 2018 , and it was
later released to theaters in Japan on August 3 , 2018 .'
episode_done: true
eval_labels:
- July 5 , 2018
id: natural_questions:NaturalQuestionsOpenTeacher
text: when does the new my hero academia movie come out
title: 'My Hero Academia: Two Heroes'
- - answers: '["31"]'
checked_sentence: 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 1 -- 20 5 7 11 13
17 19 23 29 31 37 41 43 47 53 59 61 67 71 21 -- 40 73 79 83 89 97 101 103 107
109 113 127 131 137 139 149 151 157 163 167 173 41 -- 60 179 181 191 193 197
199 211 223 227 229 233 239 241 251 257 263 269 271 277 281 61 -- 80 283 293
307 311 313 317 331 337 347 349 353 359 367 373 379 383 389 397 401 409 81 --
100 419 421 431 433 439 443 449 457 461 463 467 479 487 491 499 503 509 521
523 541 101 -- 120 547 557 563 569 571 577 587 593 599 601 607 613 617 619 631
641 643 647 653 659 121 -- 140 661 673 677 683 691 701 709 719 727 733 739 743
751 757 761 769 773 787 797 809 141 -- 160 811 821 823 827 829 839 853 857 859
863 877 881 883 887 907 911 919 929 937 941 161 -- 180 947 953 967 971 977 983
991 997 1009 1013 1019 1021 1031 1033 1039 1049 1051 1061 1063 1069 181 -- 200
1087 1091 1093 1097 1103 1109 1117 1123 1129 1151 1153 1163 1171 1181 1187 1193
1201 1213 1217 1223 201 -- 220 1229 1231 1237 1249 1259 1277 1279 1283 1289
1291 1297 1301 1303 1307 1319 1321 1327 1361 1367 1373 221 -- 240 1381 1399
1409 1423 1427 1429 1433 1439 1447 1451 1453 1459 1471 1481 1483 1487 1489 1493
1499 1511 241 -- 260 1523 1531 1543 1549 1553 1559 1567 1571 1579 1583 1597
1601 1607 1609 1613 1619 1621 1627 1637 1657 261 -- 280 1663 1667 1669 1693
1697 1699 1709 1721 1723 1733 1741 1747 1753 1759 1777 1783 1787 1789 1801 1811
281 -- 300 1823 1831 1847 1861 1867 1871 1873 1877 1879 1889 1901 1907 1913
1931 1933 1949 1951 1973 1979 301 -- 320 1999 2003 2011 2017 2027 2029 2039
2053 2063 2069 2081 2083 2087 2089 2099 2111 2113 2129 321 -- 340 2131 2137
2141 2143 2153 2161 2179 2203 2207 2213 2221 2237 2239 2243 2251 2267 2269 2273
2281 2287 341 -- 360 2293 2297 2309 2311 2333 2339 2341 2347 2351 2357 2371
2377 2381 2383 2389 2393 2399 2411 2417 2423 361 -- 380 2437 2441 2447 2459
2467 2473 2477 2503 2521 2531 2539 2543 2549 2551 2557 2579 2591 2593 2609 2617
381 -- 400 2621 2633 2647 2657 2659 2663 2671 2677 2683 2687 2689 2693 2699
2707 2711 2713 2719 2729 2731 2741 401 -- 420 2749 2753 2767 2777 2789 2791
2797 2801 2803 2819 2833 2837 2843 2851 2857 2861 2879 2887 2897 2903 421 --
440 2909 2917 2927 2939 2953 2957 2963 2969 2971 2999 3001 3011 3019 3023 3037
3041 3049 3061 3067 3079 441 -- 460 3083 3089 3109 3119 3121 3137 3163 3167
3169 3181 3187 3191 3203 3209 3217 3221 3229 3251 3253 3257 461 -- 480 3259
3271 3299 3301 3307 3313 3319 3323 3329 3331 3343 3347 3359 3361 3371 3373 3389
3391 3407 3413 481 -- 500 3433 3449 3457 3461 3463 3467 3469 3491 3499 3511
3517 3527 3529 3533 3539 3541 3547 3557 3559 3571 501 -- 520 3581 3583 3593
3607 3613 3617 3623 3631 3637 3643 3659 3671 3673 3677 3691 3697 3701 3709 3719
3727 521 -- 540 3733 3739 3761 3767 3769 3779 3793 3797 3803 3821 3823 3833
3847 3851 3853 3863 3877 3881 3889 3907 541 -- 560 3911 3917 3919 3923 3929
3931 3943 3947 3967 3989 4001 4003 4007 4013 4019 4021 4027 4049 4051 4057 561
-- 580 4073 4079 4091 4093 4099 4111 4127 4129 4133 4139 4153 4157 4159 4177
4201 4211 4217 4219 4229 4231 581 -- 600 4241 4243 4253 4259 4261 4271 4273
4283 4289 4297 4327 4337 4339 4349 4357 4363 4373 4391 4397 4409 601 -- 620
4421 4423 4441 4447 4451 4457 4463 4481 4483 4493 4507 4513 4517 4519 4523 4547
4549 4561 4567 4583 621 -- 640 4591 4597 4603 4621 4637 4639 4643 4649 4651
4657 4663 4673 4679 4691 4703 4721 4723 4729 4733 4751 641 -- 660 4759 4783
4787 4789 4793 4799 4801 4813 4817 4831 4861 4871 4877 4889 4903 4909 4919 4931
4933 4937 661 -- 680 4943 4951 4957 4967 4969 4973 4987 4993 4999 5003 5009
5011 5021 5023 5039 5051 5059 5077 5081 5087 681 -- 700 5099 5101 5107 5113
5119 5147 5153 5167 5171 5179 5189 5197 5209 5227 5231 5233 5237 5261 5273 5279
701 -- 720 5281 5297 5303 5309 5323 5333 5347 5351 5381 5387 5393 5399 5407
5413 5417 5419 5431 5437 5441 5443 721 -- 740 5449 5471 5477 5479 5483 5501
5503 5507 5519 5521 5527 5531 5557 5563 5569 5573 5581 5591 5623 5639 741 --
760 5641 5647 5651 5653 5657 5659 5669 5683 5689 5693 5701 5711 5717 5737 5741
5743 5749 5779 5783 5791 761 -- 780 5801 5807 5813 5821 5827 5839 5843 5849
5851 5857 5861 5867 5869 5879 5881 5897 5903 5923 5927 5939 781 -- 800 5953
5981 5987 6007 6011 6029 6037 6043 6047 6053 6067 6073 6079 6089 6091 6101 6113
6121 6131 6133 801 -- 820 6143 6151 6163 6173 6197 6199 6203 6211 6217 6221
6229 6247 6257 6263 6269 6271 6277 6287 6299 6301 821 -- 840 6311 6317 6323
6329 6337 6343 6353 6359 6361 6367 6373 6379 6389 6397 6421 6427 6449 6451 6469
6473 841 -- 860 6481 6491 6521 6529 6547 6551 6553 6563 6569 6571 6577 6581
6599 6607 6619 6637 6653 6659 6661 6673 861 -- 880 6679 6689 6691 6701 6703
6709 6719 6733 6737 6761 6763 6779 6781 6791 6793 6803 6823 6827 6829 6833 881
-- 900 6841 6857 6863 6869 6871 6883 6899 6907 6911 6917 6947 6949 6959 6961
6967 6971 6977 6983 6991 6997 901 -- 920 7001 7013 7019 7027 7039 7043 7057
7069 7079 7103 7109 7121 7127 7129 7151 7159 7177 7187 7193 7207 921 -- 940
7211 7213 7219 7229 7237 7243 7247 7253 7283 7297 7307 7309 7321 7331 7333 7349
7351 7369 7393 7411 941 -- 960 7417 7433 7451 7457 7459 7477 7481 7487 7489
7499 7507 7517 7523 7529 7537 7541 7547 7549 7559 7561 961 -- 980 7573 7577
7583 7589 7591 7603 7607 7621 7639 7643 7649 7669 7673 7681 7687 7691 7699 7703
7717 7723 981 -- 1000 7727 7741 7753 7757 7759 7789 7793 7817 7823 7829 7841
7853 7867 7873 7877 7879 7883 7901 7907 7919
episode_done: true
eval_labels:
- '31'
id: natural_questions:NaturalQuestionsOpenTeacher
text: what is the smallest prime number that is greater than 30
title: List of prime numbers
num_episodes: 1223
num_examples: 1223
Loading