From 1d64a92d69423836d9f92eff9cd0f898756a5cb3 Mon Sep 17 00:00:00 2001 From: EricMichaelSmith Date: Fri, 29 Oct 2021 12:53:10 -0400 Subject: [PATCH 1/3] Save worker name in Acute-Eval analysis --- parlai/crowdsourcing/tasks/acute_eval/analysis.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/parlai/crowdsourcing/tasks/acute_eval/analysis.py b/parlai/crowdsourcing/tasks/acute_eval/analysis.py index a753aef5d19..05afb2d1ed6 100644 --- a/parlai/crowdsourcing/tasks/acute_eval/analysis.py +++ b/parlai/crowdsourcing/tasks/acute_eval/analysis.py @@ -22,6 +22,7 @@ from mephisto.tools.data_browser import DataBrowser as MephistoDataBrowser from mephisto.abstractions.databases.local_database import LocalMephistoDB from mephisto.data_model.unit import Unit as MephistoUnit +from mephisto.data_model.worker import Worker from scipy.stats import binom_test from parlai.core.params import ParlaiParser @@ -137,8 +138,8 @@ def __init__(self, opt: Dict, remove_failed: bool = True): mephisto_root_path = opt['mephisto_root'] if not mephisto_root_path: mephisto_root_path = None - mephisto_db = LocalMephistoDB(database_path=mephisto_root_path) - self.mephisto_data_browser = MephistoDataBrowser(db=mephisto_db) + self.mephisto_db = LocalMephistoDB(database_path=mephisto_root_path) + self.mephisto_data_browser = MephistoDataBrowser(db=self.mephisto_db) self.checkbox_prefix = self.CHECKBOX_PREFIX # Prepended to checkbox columns in self.dataframe self.dataframe = self._extract_to_dataframe() @@ -168,6 +169,9 @@ def _extract_response_by_index( response: Dict[str, Any] = { 'run_id': self.run_id, 'worker': unit_details['worker_id'], + 'mturk_worker_id': Worker.get( + self.mephisto_db, unit_details['worker_id'] + ).worker_name, 'time_taken': unit_details['task_end'] - unit_details['task_start'], 'question': task_data['task_specs']['question'], 'unit_id': unit_details['unit_id'], From 6a386b8316c7642604c6a75294b382258c1048ee Mon Sep 17 00:00:00 2001 From: EricMichaelSmith Date: Fri, 29 Oct 2021 12:55:49 -0400 Subject: [PATCH 2/3] Generalize name --- parlai/crowdsourcing/tasks/acute_eval/analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parlai/crowdsourcing/tasks/acute_eval/analysis.py b/parlai/crowdsourcing/tasks/acute_eval/analysis.py index 05afb2d1ed6..c5533afd882 100644 --- a/parlai/crowdsourcing/tasks/acute_eval/analysis.py +++ b/parlai/crowdsourcing/tasks/acute_eval/analysis.py @@ -169,7 +169,7 @@ def _extract_response_by_index( response: Dict[str, Any] = { 'run_id': self.run_id, 'worker': unit_details['worker_id'], - 'mturk_worker_id': Worker.get( + 'worker_name': Worker.get( self.mephisto_db, unit_details['worker_id'] ).worker_name, 'time_taken': unit_details['task_end'] - unit_details['task_start'], From 0212d4c4e22d993a8af836840636eac715f334cc Mon Sep 17 00:00:00 2001 From: EricMichaelSmith Date: Fri, 29 Oct 2021 13:53:49 -0400 Subject: [PATCH 3/3] Hopefully fix unit tests --- .../test_fast_acute_dataset/test_full_csv.csv | 10 +++++----- .../acute_eval/test_fast_acute_logs/test_full_csv.csv | 10 +++++----- .../test_fast_acute_self_chat/test_full_csv.csv | 10 +++++----- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/crowdsourcing/tasks/acute_eval/test_fast_acute_dataset/test_full_csv.csv b/tests/crowdsourcing/tasks/acute_eval/test_fast_acute_dataset/test_full_csv.csv index 9467facb73f..cc9c56a9850 100644 --- a/tests/crowdsourcing/tasks/acute_eval/test_fast_acute_dataset/test_full_csv.csv +++ b/tests/crowdsourcing/tasks/acute_eval/test_fast_acute_dataset/test_full_csv.csv @@ -1,5 +1,5 @@ -,run_id,worker,time_taken,question,unit_id,task_start,correct,winner,loser,eval_choice_0,eval_choice_1,reason,is_onboarding,matchup,pairing_id,pairs_to_eval,winner_dialogue,loser_dialogue -0,acute_eval_test,1,0,Who would you prefer to talk to for a long conversation?,1,0,-1,convai2_logs,ed_logs,convai2_logs,ed_logs,Makes more sense,False,convai2_logs__vs__ed_logs,45,"{'is_onboarding': False, 'speakers_to_eval': ['convai2_logs', 'ed_logs'], 'dialogue_ids': [466, 566], 'convai2_logs': {'context': [], 'dialogue': [{'id': 'convai2_logs_as_human', 'text': 'Hello. How are you? Do you have kids?'}, {'id': 'convai2_logs', 'text': 'No. I am just fourteen myself.'}, {'id': 'convai2_logs_as_human', 'text': 'My youngest kid is 14. Do you have hobbies?'}, {'id': 'convai2_logs', 'text': 'Swimming and playing video games and believe it or not, sports cars'}, {'id': 'convai2_logs_as_human', 'text': 'What is your favorite food?'}, {'id': 'convai2_logs', 'text': 'Pizza all day long. What do you like to eat?'}, {'id': 'convai2_logs_as_human', 'text': 'Hot wings. So good. Do you like to sing?'}, {'id': 'convai2_logs', 'text': ""I'm not good. Grandpa and mom say for me to be quiet.""}, {'id': 'convai2_logs_as_human', 'text': 'My church has a choir. I sing soprano. You should sing in church.'}, {'id': 'convai2_logs', 'text': 'My friends carol, mia and stephanie do that. I am not a good singer though.'}, {'id': 'convai2_logs_as_human', 'text': 'My kids say I am not that good. But I am on the road often.'}, {'id': 'convai2_logs', 'text': 'What is your job? I bet you sing in the car too.'}, {'id': 'convai2_logs_as_human', 'text': 'I drive a truck.'}, {'id': 'convai2_logs', 'text': 'Stick shift? You are good! My grandmother drives a stick.'}, {'id': 'convai2_logs_as_human', 'text': 'No. I am a commercial truck driver.'}, {'id': 'convai2_logs', 'text': 'And you sing too. I want to learn to drive sports cars and trucks'}], 'speakers': ['convai2_logs_as_human', 'convai2_logs']}, 'ed_logs': {'context': [], 'dialogue': [{'id': 'ed_logs_as_human', 'text': ""One of my friend didn't keep this promise he made to me. I was so angry to him.""}, {'id': 'ed_logs', 'text': ""What promise didn't he keep?""}, {'id': 'ed_logs_as_human', 'text': ""I was suppose to help me move from my apartment. He didn't come and turned off his cell phone.""}, {'id': 'ed_logs', 'text': 'Wow that sounds like a terrible friend. I hope he had a good reason for not showing up.'}], 'speakers': ['ed_logs_as_human', 'ed_logs']}}","{'context': [], 'dialogue': [{'id': 'convai2_logs_as_human', 'text': 'Hello. How are you? Do you have kids?'}, {'id': 'convai2_logs', 'text': 'No. I am just fourteen myself.'}, {'id': 'convai2_logs_as_human', 'text': 'My youngest kid is 14. Do you have hobbies?'}, {'id': 'convai2_logs', 'text': 'Swimming and playing video games and believe it or not, sports cars'}, {'id': 'convai2_logs_as_human', 'text': 'What is your favorite food?'}, {'id': 'convai2_logs', 'text': 'Pizza all day long. What do you like to eat?'}, {'id': 'convai2_logs_as_human', 'text': 'Hot wings. So good. Do you like to sing?'}, {'id': 'convai2_logs', 'text': ""I'm not good. Grandpa and mom say for me to be quiet.""}, {'id': 'convai2_logs_as_human', 'text': 'My church has a choir. I sing soprano. You should sing in church.'}, {'id': 'convai2_logs', 'text': 'My friends carol, mia and stephanie do that. I am not a good singer though.'}, {'id': 'convai2_logs_as_human', 'text': 'My kids say I am not that good. But I am on the road often.'}, {'id': 'convai2_logs', 'text': 'What is your job? I bet you sing in the car too.'}, {'id': 'convai2_logs_as_human', 'text': 'I drive a truck.'}, {'id': 'convai2_logs', 'text': 'Stick shift? You are good! My grandmother drives a stick.'}, {'id': 'convai2_logs_as_human', 'text': 'No. I am a commercial truck driver.'}, {'id': 'convai2_logs', 'text': 'And you sing too. I want to learn to drive sports cars and trucks'}], 'speakers': ['convai2_logs_as_human', 'convai2_logs']}","{'context': [], 'dialogue': [{'id': 'ed_logs_as_human', 'text': ""One of my friend didn't keep this promise he made to me. I was so angry to him.""}, {'id': 'ed_logs', 'text': ""What promise didn't he keep?""}, {'id': 'ed_logs_as_human', 'text': ""I was suppose to help me move from my apartment. He didn't come and turned off his cell phone.""}, {'id': 'ed_logs', 'text': 'Wow that sounds like a terrible friend. I hope he had a good reason for not showing up.'}], 'speakers': ['ed_logs_as_human', 'ed_logs']}" -1,acute_eval_test,1,0,Who would you prefer to talk to for a long conversation?,1,0,-1,ed_logs,convai2_logs,convai2_logs,ed_logs,Makes more sense,False,convai2_logs__vs__ed_logs,62,"{'is_onboarding': False, 'speakers_to_eval': ['convai2_logs', 'ed_logs'], 'dialogue_ids': [111, 791], 'convai2_logs': {'context': [], 'dialogue': [{'id': 'convai2_logs_as_human', 'text': ""Hi. I'm super lazy""}, {'id': 'convai2_logs', 'text': 'It can be enjoyable to be lazy sometimes. I like to relax on the beach.'}, {'id': 'convai2_logs_as_human', 'text': 'I love to paint trees'}, {'id': 'convai2_logs', 'text': 'That sounds nice. I ride my bike on a nice path that goes through the trees.'}, {'id': 'convai2_logs_as_human', 'text': ""Don't get stuck! Oh dear""}, {'id': 'convai2_logs', 'text': 'Oh no. It is a fine path.'}, {'id': 'convai2_logs_as_human', 'text': 'Ok good. Do not wanna call 911'}, {'id': 'convai2_logs', 'text': ""It doesn't literally go through a tree. Ha ha ha.""}, {'id': 'convai2_logs_as_human', 'text': 'Oh ok good. That was scarry'}, {'id': 'convai2_logs', 'text': 'I have simplified my life. I was once employed with a cable company.'}, {'id': 'convai2_logs_as_human', 'text': ""Oh that's good. I get lonely""}, {'id': 'convai2_logs', 'text': ""I now make rope. I work with many people. You'dn't be lonely there.""}, {'id': 'convai2_logs_as_human', 'text': 'Can I come then please'}, {'id': 'convai2_logs', 'text': 'Oh yes. You would like it. I like it there.'}, {'id': 'convai2_logs_as_human', 'text': ""Ok, I'm on my way""}, {'id': 'convai2_logs', 'text': ""We have plenty of rope to make. I don't know what the deal with cable is.""}], 'speakers': ['convai2_logs_as_human', 'convai2_logs']}, 'ed_logs': {'context': [], 'dialogue': [{'id': 'ed_logs_as_human', 'text': 'What happened?!'}, {'id': 'ed_logs', 'text': 'I filled an entire cart with food for a party and then realized I had forgot me purse at home.'}, {'id': 'ed_logs_as_human', 'text': 'Oh no. Did you end up having to leave the cart?'}, {'id': 'ed_logs', 'text': 'Yes. My apartment is close by but it was still embarrassing'}], 'speakers': ['ed_logs_as_human', 'ed_logs']}}","{'context': [], 'dialogue': [{'id': 'ed_logs_as_human', 'text': 'What happened?!'}, {'id': 'ed_logs', 'text': 'I filled an entire cart with food for a party and then realized I had forgot me purse at home.'}, {'id': 'ed_logs_as_human', 'text': 'Oh no. Did you end up having to leave the cart?'}, {'id': 'ed_logs', 'text': 'Yes. My apartment is close by but it was still embarrassing'}], 'speakers': ['ed_logs_as_human', 'ed_logs']}","{'context': [], 'dialogue': [{'id': 'convai2_logs_as_human', 'text': ""Hi. I'm super lazy""}, {'id': 'convai2_logs', 'text': 'It can be enjoyable to be lazy sometimes. I like to relax on the beach.'}, {'id': 'convai2_logs_as_human', 'text': 'I love to paint trees'}, {'id': 'convai2_logs', 'text': 'That sounds nice. I ride my bike on a nice path that goes through the trees.'}, {'id': 'convai2_logs_as_human', 'text': ""Don't get stuck! Oh dear""}, {'id': 'convai2_logs', 'text': 'Oh no. It is a fine path.'}, {'id': 'convai2_logs_as_human', 'text': 'Ok good. Do not wanna call 911'}, {'id': 'convai2_logs', 'text': ""It doesn't literally go through a tree. Ha ha ha.""}, {'id': 'convai2_logs_as_human', 'text': 'Oh ok good. That was scarry'}, {'id': 'convai2_logs', 'text': 'I have simplified my life. I was once employed with a cable company.'}, {'id': 'convai2_logs_as_human', 'text': ""Oh that's good. I get lonely""}, {'id': 'convai2_logs', 'text': ""I now make rope. I work with many people. You'dn't be lonely there.""}, {'id': 'convai2_logs_as_human', 'text': 'Can I come then please'}, {'id': 'convai2_logs', 'text': 'Oh yes. You would like it. I like it there.'}, {'id': 'convai2_logs_as_human', 'text': ""Ok, I'm on my way""}, {'id': 'convai2_logs', 'text': ""We have plenty of rope to make. I don't know what the deal with cable is.""}], 'speakers': ['convai2_logs_as_human', 'convai2_logs']}" -2,acute_eval_test,1,0,Who would you prefer to talk to for a long conversation?,1,0,-1,convai2_logs,ed_logs,convai2_logs,ed_logs,Makes more sense,False,convai2_logs__vs__ed_logs,21,"{'is_onboarding': False, 'speakers_to_eval': ['convai2_logs', 'ed_logs'], 'dialogue_ids': [464, 851], 'convai2_logs': {'context': [], 'dialogue': [{'id': 'convai2_logs_as_human', 'text': 'Hey, how are you doing today?'}, {'id': 'convai2_logs', 'text': 'I am great. Drinking some iced tea, which I love.'}, {'id': 'convai2_logs_as_human', 'text': ""That sounds great. I cannot even make iced tea, I'm such a lowly cook.""}, {'id': 'convai2_logs', 'text': ""That's too bad. It just takes practice. My mom taught me to cok.""}, {'id': 'convai2_logs_as_human', 'text': 'That is awesome. I am ovo vegetarian, so its hard to find restaurants.'}, {'id': 'convai2_logs', 'text': ""I'll bet! My youngest sister I've four is a vegetarian.""}, {'id': 'convai2_logs_as_human', 'text': 'Neat. What do you do on your days off? I like to sleep in.'}, {'id': 'convai2_logs', 'text': 'Believe it or not, I study mermaids.'}, {'id': 'convai2_logs_as_human', 'text': 'That is fascinating. Tell me all about them.'}, {'id': 'convai2_logs', 'text': 'Mermaids are real. Most people do not know that.'}, {'id': 'convai2_logs_as_human', 'text': ""I'd have thought they were not real, but you have convinced me!""}, {'id': 'convai2_logs', 'text': 'Yeah, the evidence is overwhelming when you really look.'}, {'id': 'convai2_logs_as_human', 'text': 'That makes sense to my mind. I wonder how long they have been here.'}, {'id': 'convai2_logs', 'text': 'Probably longer than us. I wish one of my four sisters was a mermaid!'}], 'speakers': ['convai2_logs_as_human', 'convai2_logs']}, 'ed_logs': {'context': [], 'dialogue': [{'id': 'ed_logs_as_human', 'text': ""Why don't you say something?""}, {'id': 'ed_logs', 'text': 'I did, she calls me a helicopter parent and just laughs at me'}], 'speakers': ['ed_logs_as_human', 'ed_logs']}}","{'context': [], 'dialogue': [{'id': 'convai2_logs_as_human', 'text': 'Hey, how are you doing today?'}, {'id': 'convai2_logs', 'text': 'I am great. Drinking some iced tea, which I love.'}, {'id': 'convai2_logs_as_human', 'text': ""That sounds great. I cannot even make iced tea, I'm such a lowly cook.""}, {'id': 'convai2_logs', 'text': ""That's too bad. It just takes practice. My mom taught me to cok.""}, {'id': 'convai2_logs_as_human', 'text': 'That is awesome. I am ovo vegetarian, so its hard to find restaurants.'}, {'id': 'convai2_logs', 'text': ""I'll bet! My youngest sister I've four is a vegetarian.""}, {'id': 'convai2_logs_as_human', 'text': 'Neat. What do you do on your days off? I like to sleep in.'}, {'id': 'convai2_logs', 'text': 'Believe it or not, I study mermaids.'}, {'id': 'convai2_logs_as_human', 'text': 'That is fascinating. Tell me all about them.'}, {'id': 'convai2_logs', 'text': 'Mermaids are real. Most people do not know that.'}, {'id': 'convai2_logs_as_human', 'text': ""I'd have thought they were not real, but you have convinced me!""}, {'id': 'convai2_logs', 'text': 'Yeah, the evidence is overwhelming when you really look.'}, {'id': 'convai2_logs_as_human', 'text': 'That makes sense to my mind. I wonder how long they have been here.'}, {'id': 'convai2_logs', 'text': 'Probably longer than us. I wish one of my four sisters was a mermaid!'}], 'speakers': ['convai2_logs_as_human', 'convai2_logs']}","{'context': [], 'dialogue': [{'id': 'ed_logs_as_human', 'text': ""Why don't you say something?""}, {'id': 'ed_logs', 'text': 'I did, she calls me a helicopter parent and just laughs at me'}], 'speakers': ['ed_logs_as_human', 'ed_logs']}" -3,acute_eval_test,1,0,Who would you prefer to talk to for a long conversation?,1,0,-1,ed_logs,convai2_logs,convai2_logs,ed_logs,Makes more sense,False,convai2_logs__vs__ed_logs,81,"{'is_onboarding': False, 'speakers_to_eval': ['convai2_logs', 'ed_logs'], 'dialogue_ids': [462, 918], 'convai2_logs': {'context': [], 'dialogue': [{'id': 'convai2_logs_as_human', 'text': 'Hi how are you today'}, {'id': 'convai2_logs', 'text': 'Hello there. God bless you.'}, {'id': 'convai2_logs_as_human', 'text': 'Bless you too did you sneeze?'}, {'id': 'convai2_logs', 'text': ""I am just the religious type. Don't get me agitated.""}, {'id': 'convai2_logs_as_human', 'text': 'But how are you today?'}, {'id': 'convai2_logs', 'text': ""I'm tired after just getting back from the trip to london.""}, {'id': 'convai2_logs_as_human', 'text': 'What did you do there'}, {'id': 'convai2_logs', 'text': 'I visited the pope, what else would I do?'}, {'id': 'convai2_logs_as_human', 'text': 'Probably eat a pizza or something'}, {'id': 'convai2_logs', 'text': ""I'd eat pizza while watching the game shows I love.""}, {'id': 'convai2_logs_as_human', 'text': 'What is the best one'}, {'id': 'convai2_logs', 'text': ""Please don't get me agitated, it will happen really easily!""}, {'id': 'convai2_logs_as_human', 'text': 'Wow and why is that then'}, {'id': 'convai2_logs', 'text': 'God will take of that. What do you like to do?'}, {'id': 'convai2_logs_as_human', 'text': 'I can not see any colors'}, {'id': 'convai2_logs', 'text': 'That would make it hard to watch game shows.'}], 'speakers': ['convai2_logs_as_human', 'convai2_logs']}, 'ed_logs': {'context': [], 'dialogue': [{'id': 'ed_logs_as_human', 'text': 'Do you watch game of thrones?'}, {'id': 'ed_logs', 'text': 'I watched the first season.'}, {'id': 'ed_logs_as_human', 'text': ""I love that show. I'm sad it is ending. It should stay on forever.""}, {'id': 'ed_logs', 'text': ""What? It's ending? I heard he hasn't yet finished the books.""}], 'speakers': ['ed_logs_as_human', 'ed_logs']}}","{'context': [], 'dialogue': [{'id': 'ed_logs_as_human', 'text': 'Do you watch game of thrones?'}, {'id': 'ed_logs', 'text': 'I watched the first season.'}, {'id': 'ed_logs_as_human', 'text': ""I love that show. I'm sad it is ending. It should stay on forever.""}, {'id': 'ed_logs', 'text': ""What? It's ending? I heard he hasn't yet finished the books.""}], 'speakers': ['ed_logs_as_human', 'ed_logs']}","{'context': [], 'dialogue': [{'id': 'convai2_logs_as_human', 'text': 'Hi how are you today'}, {'id': 'convai2_logs', 'text': 'Hello there. God bless you.'}, {'id': 'convai2_logs_as_human', 'text': 'Bless you too did you sneeze?'}, {'id': 'convai2_logs', 'text': ""I am just the religious type. Don't get me agitated.""}, {'id': 'convai2_logs_as_human', 'text': 'But how are you today?'}, {'id': 'convai2_logs', 'text': ""I'm tired after just getting back from the trip to london.""}, {'id': 'convai2_logs_as_human', 'text': 'What did you do there'}, {'id': 'convai2_logs', 'text': 'I visited the pope, what else would I do?'}, {'id': 'convai2_logs_as_human', 'text': 'Probably eat a pizza or something'}, {'id': 'convai2_logs', 'text': ""I'd eat pizza while watching the game shows I love.""}, {'id': 'convai2_logs_as_human', 'text': 'What is the best one'}, {'id': 'convai2_logs', 'text': ""Please don't get me agitated, it will happen really easily!""}, {'id': 'convai2_logs_as_human', 'text': 'Wow and why is that then'}, {'id': 'convai2_logs', 'text': 'God will take of that. What do you like to do?'}, {'id': 'convai2_logs_as_human', 'text': 'I can not see any colors'}, {'id': 'convai2_logs', 'text': 'That would make it hard to watch game shows.'}], 'speakers': ['convai2_logs_as_human', 'convai2_logs']}" +,run_id,worker,worker_name,time_taken,question,unit_id,task_start,correct,winner,loser,eval_choice_0,eval_choice_1,reason,is_onboarding,matchup,pairing_id,pairs_to_eval,winner_dialogue,loser_dialogue +0,acute_eval_test,1,MOCK_WORKER_0,0,Who would you prefer to talk to for a long conversation?,1,0,-1,convai2_logs,ed_logs,convai2_logs,ed_logs,Makes more sense,False,convai2_logs__vs__ed_logs,45,"{'is_onboarding': False, 'speakers_to_eval': ['convai2_logs', 'ed_logs'], 'dialogue_ids': [466, 566], 'convai2_logs': {'context': [], 'dialogue': [{'id': 'convai2_logs_as_human', 'text': 'Hello. How are you? Do you have kids?'}, {'id': 'convai2_logs', 'text': 'No. I am just fourteen myself.'}, {'id': 'convai2_logs_as_human', 'text': 'My youngest kid is 14. Do you have hobbies?'}, {'id': 'convai2_logs', 'text': 'Swimming and playing video games and believe it or not, sports cars'}, {'id': 'convai2_logs_as_human', 'text': 'What is your favorite food?'}, {'id': 'convai2_logs', 'text': 'Pizza all day long. What do you like to eat?'}, {'id': 'convai2_logs_as_human', 'text': 'Hot wings. So good. Do you like to sing?'}, {'id': 'convai2_logs', 'text': ""I'm not good. Grandpa and mom say for me to be quiet.""}, {'id': 'convai2_logs_as_human', 'text': 'My church has a choir. I sing soprano. You should sing in church.'}, {'id': 'convai2_logs', 'text': 'My friends carol, mia and stephanie do that. I am not a good singer though.'}, {'id': 'convai2_logs_as_human', 'text': 'My kids say I am not that good. But I am on the road often.'}, {'id': 'convai2_logs', 'text': 'What is your job? I bet you sing in the car too.'}, {'id': 'convai2_logs_as_human', 'text': 'I drive a truck.'}, {'id': 'convai2_logs', 'text': 'Stick shift? You are good! My grandmother drives a stick.'}, {'id': 'convai2_logs_as_human', 'text': 'No. I am a commercial truck driver.'}, {'id': 'convai2_logs', 'text': 'And you sing too. I want to learn to drive sports cars and trucks'}], 'speakers': ['convai2_logs_as_human', 'convai2_logs']}, 'ed_logs': {'context': [], 'dialogue': [{'id': 'ed_logs_as_human', 'text': ""One of my friend didn't keep this promise he made to me. I was so angry to him.""}, {'id': 'ed_logs', 'text': ""What promise didn't he keep?""}, {'id': 'ed_logs_as_human', 'text': ""I was suppose to help me move from my apartment. He didn't come and turned off his cell phone.""}, {'id': 'ed_logs', 'text': 'Wow that sounds like a terrible friend. I hope he had a good reason for not showing up.'}], 'speakers': ['ed_logs_as_human', 'ed_logs']}}","{'context': [], 'dialogue': [{'id': 'convai2_logs_as_human', 'text': 'Hello. How are you? Do you have kids?'}, {'id': 'convai2_logs', 'text': 'No. I am just fourteen myself.'}, {'id': 'convai2_logs_as_human', 'text': 'My youngest kid is 14. Do you have hobbies?'}, {'id': 'convai2_logs', 'text': 'Swimming and playing video games and believe it or not, sports cars'}, {'id': 'convai2_logs_as_human', 'text': 'What is your favorite food?'}, {'id': 'convai2_logs', 'text': 'Pizza all day long. What do you like to eat?'}, {'id': 'convai2_logs_as_human', 'text': 'Hot wings. So good. Do you like to sing?'}, {'id': 'convai2_logs', 'text': ""I'm not good. Grandpa and mom say for me to be quiet.""}, {'id': 'convai2_logs_as_human', 'text': 'My church has a choir. I sing soprano. You should sing in church.'}, {'id': 'convai2_logs', 'text': 'My friends carol, mia and stephanie do that. I am not a good singer though.'}, {'id': 'convai2_logs_as_human', 'text': 'My kids say I am not that good. But I am on the road often.'}, {'id': 'convai2_logs', 'text': 'What is your job? I bet you sing in the car too.'}, {'id': 'convai2_logs_as_human', 'text': 'I drive a truck.'}, {'id': 'convai2_logs', 'text': 'Stick shift? You are good! My grandmother drives a stick.'}, {'id': 'convai2_logs_as_human', 'text': 'No. I am a commercial truck driver.'}, {'id': 'convai2_logs', 'text': 'And you sing too. I want to learn to drive sports cars and trucks'}], 'speakers': ['convai2_logs_as_human', 'convai2_logs']}","{'context': [], 'dialogue': [{'id': 'ed_logs_as_human', 'text': ""One of my friend didn't keep this promise he made to me. I was so angry to him.""}, {'id': 'ed_logs', 'text': ""What promise didn't he keep?""}, {'id': 'ed_logs_as_human', 'text': ""I was suppose to help me move from my apartment. He didn't come and turned off his cell phone.""}, {'id': 'ed_logs', 'text': 'Wow that sounds like a terrible friend. I hope he had a good reason for not showing up.'}], 'speakers': ['ed_logs_as_human', 'ed_logs']}" +1,acute_eval_test,1,MOCK_WORKER_0,0,Who would you prefer to talk to for a long conversation?,1,0,-1,ed_logs,convai2_logs,convai2_logs,ed_logs,Makes more sense,False,convai2_logs__vs__ed_logs,62,"{'is_onboarding': False, 'speakers_to_eval': ['convai2_logs', 'ed_logs'], 'dialogue_ids': [111, 791], 'convai2_logs': {'context': [], 'dialogue': [{'id': 'convai2_logs_as_human', 'text': ""Hi. I'm super lazy""}, {'id': 'convai2_logs', 'text': 'It can be enjoyable to be lazy sometimes. I like to relax on the beach.'}, {'id': 'convai2_logs_as_human', 'text': 'I love to paint trees'}, {'id': 'convai2_logs', 'text': 'That sounds nice. I ride my bike on a nice path that goes through the trees.'}, {'id': 'convai2_logs_as_human', 'text': ""Don't get stuck! Oh dear""}, {'id': 'convai2_logs', 'text': 'Oh no. It is a fine path.'}, {'id': 'convai2_logs_as_human', 'text': 'Ok good. Do not wanna call 911'}, {'id': 'convai2_logs', 'text': ""It doesn't literally go through a tree. Ha ha ha.""}, {'id': 'convai2_logs_as_human', 'text': 'Oh ok good. That was scarry'}, {'id': 'convai2_logs', 'text': 'I have simplified my life. I was once employed with a cable company.'}, {'id': 'convai2_logs_as_human', 'text': ""Oh that's good. I get lonely""}, {'id': 'convai2_logs', 'text': ""I now make rope. I work with many people. You'dn't be lonely there.""}, {'id': 'convai2_logs_as_human', 'text': 'Can I come then please'}, {'id': 'convai2_logs', 'text': 'Oh yes. You would like it. I like it there.'}, {'id': 'convai2_logs_as_human', 'text': ""Ok, I'm on my way""}, {'id': 'convai2_logs', 'text': ""We have plenty of rope to make. I don't know what the deal with cable is.""}], 'speakers': ['convai2_logs_as_human', 'convai2_logs']}, 'ed_logs': {'context': [], 'dialogue': [{'id': 'ed_logs_as_human', 'text': 'What happened?!'}, {'id': 'ed_logs', 'text': 'I filled an entire cart with food for a party and then realized I had forgot me purse at home.'}, {'id': 'ed_logs_as_human', 'text': 'Oh no. Did you end up having to leave the cart?'}, {'id': 'ed_logs', 'text': 'Yes. My apartment is close by but it was still embarrassing'}], 'speakers': ['ed_logs_as_human', 'ed_logs']}}","{'context': [], 'dialogue': [{'id': 'ed_logs_as_human', 'text': 'What happened?!'}, {'id': 'ed_logs', 'text': 'I filled an entire cart with food for a party and then realized I had forgot me purse at home.'}, {'id': 'ed_logs_as_human', 'text': 'Oh no. Did you end up having to leave the cart?'}, {'id': 'ed_logs', 'text': 'Yes. My apartment is close by but it was still embarrassing'}], 'speakers': ['ed_logs_as_human', 'ed_logs']}","{'context': [], 'dialogue': [{'id': 'convai2_logs_as_human', 'text': ""Hi. I'm super lazy""}, {'id': 'convai2_logs', 'text': 'It can be enjoyable to be lazy sometimes. I like to relax on the beach.'}, {'id': 'convai2_logs_as_human', 'text': 'I love to paint trees'}, {'id': 'convai2_logs', 'text': 'That sounds nice. I ride my bike on a nice path that goes through the trees.'}, {'id': 'convai2_logs_as_human', 'text': ""Don't get stuck! Oh dear""}, {'id': 'convai2_logs', 'text': 'Oh no. It is a fine path.'}, {'id': 'convai2_logs_as_human', 'text': 'Ok good. Do not wanna call 911'}, {'id': 'convai2_logs', 'text': ""It doesn't literally go through a tree. Ha ha ha.""}, {'id': 'convai2_logs_as_human', 'text': 'Oh ok good. That was scarry'}, {'id': 'convai2_logs', 'text': 'I have simplified my life. I was once employed with a cable company.'}, {'id': 'convai2_logs_as_human', 'text': ""Oh that's good. I get lonely""}, {'id': 'convai2_logs', 'text': ""I now make rope. I work with many people. You'dn't be lonely there.""}, {'id': 'convai2_logs_as_human', 'text': 'Can I come then please'}, {'id': 'convai2_logs', 'text': 'Oh yes. You would like it. I like it there.'}, {'id': 'convai2_logs_as_human', 'text': ""Ok, I'm on my way""}, {'id': 'convai2_logs', 'text': ""We have plenty of rope to make. I don't know what the deal with cable is.""}], 'speakers': ['convai2_logs_as_human', 'convai2_logs']}" +2,acute_eval_test,1,MOCK_WORKER_0,0,Who would you prefer to talk to for a long conversation?,1,0,-1,convai2_logs,ed_logs,convai2_logs,ed_logs,Makes more sense,False,convai2_logs__vs__ed_logs,21,"{'is_onboarding': False, 'speakers_to_eval': ['convai2_logs', 'ed_logs'], 'dialogue_ids': [464, 851], 'convai2_logs': {'context': [], 'dialogue': [{'id': 'convai2_logs_as_human', 'text': 'Hey, how are you doing today?'}, {'id': 'convai2_logs', 'text': 'I am great. Drinking some iced tea, which I love.'}, {'id': 'convai2_logs_as_human', 'text': ""That sounds great. I cannot even make iced tea, I'm such a lowly cook.""}, {'id': 'convai2_logs', 'text': ""That's too bad. It just takes practice. My mom taught me to cok.""}, {'id': 'convai2_logs_as_human', 'text': 'That is awesome. I am ovo vegetarian, so its hard to find restaurants.'}, {'id': 'convai2_logs', 'text': ""I'll bet! My youngest sister I've four is a vegetarian.""}, {'id': 'convai2_logs_as_human', 'text': 'Neat. What do you do on your days off? I like to sleep in.'}, {'id': 'convai2_logs', 'text': 'Believe it or not, I study mermaids.'}, {'id': 'convai2_logs_as_human', 'text': 'That is fascinating. Tell me all about them.'}, {'id': 'convai2_logs', 'text': 'Mermaids are real. Most people do not know that.'}, {'id': 'convai2_logs_as_human', 'text': ""I'd have thought they were not real, but you have convinced me!""}, {'id': 'convai2_logs', 'text': 'Yeah, the evidence is overwhelming when you really look.'}, {'id': 'convai2_logs_as_human', 'text': 'That makes sense to my mind. I wonder how long they have been here.'}, {'id': 'convai2_logs', 'text': 'Probably longer than us. I wish one of my four sisters was a mermaid!'}], 'speakers': ['convai2_logs_as_human', 'convai2_logs']}, 'ed_logs': {'context': [], 'dialogue': [{'id': 'ed_logs_as_human', 'text': ""Why don't you say something?""}, {'id': 'ed_logs', 'text': 'I did, she calls me a helicopter parent and just laughs at me'}], 'speakers': ['ed_logs_as_human', 'ed_logs']}}","{'context': [], 'dialogue': [{'id': 'convai2_logs_as_human', 'text': 'Hey, how are you doing today?'}, {'id': 'convai2_logs', 'text': 'I am great. Drinking some iced tea, which I love.'}, {'id': 'convai2_logs_as_human', 'text': ""That sounds great. I cannot even make iced tea, I'm such a lowly cook.""}, {'id': 'convai2_logs', 'text': ""That's too bad. It just takes practice. My mom taught me to cok.""}, {'id': 'convai2_logs_as_human', 'text': 'That is awesome. I am ovo vegetarian, so its hard to find restaurants.'}, {'id': 'convai2_logs', 'text': ""I'll bet! My youngest sister I've four is a vegetarian.""}, {'id': 'convai2_logs_as_human', 'text': 'Neat. What do you do on your days off? I like to sleep in.'}, {'id': 'convai2_logs', 'text': 'Believe it or not, I study mermaids.'}, {'id': 'convai2_logs_as_human', 'text': 'That is fascinating. Tell me all about them.'}, {'id': 'convai2_logs', 'text': 'Mermaids are real. Most people do not know that.'}, {'id': 'convai2_logs_as_human', 'text': ""I'd have thought they were not real, but you have convinced me!""}, {'id': 'convai2_logs', 'text': 'Yeah, the evidence is overwhelming when you really look.'}, {'id': 'convai2_logs_as_human', 'text': 'That makes sense to my mind. I wonder how long they have been here.'}, {'id': 'convai2_logs', 'text': 'Probably longer than us. I wish one of my four sisters was a mermaid!'}], 'speakers': ['convai2_logs_as_human', 'convai2_logs']}","{'context': [], 'dialogue': [{'id': 'ed_logs_as_human', 'text': ""Why don't you say something?""}, {'id': 'ed_logs', 'text': 'I did, she calls me a helicopter parent and just laughs at me'}], 'speakers': ['ed_logs_as_human', 'ed_logs']}" +3,acute_eval_test,1,MOCK_WORKER_0,0,Who would you prefer to talk to for a long conversation?,1,0,-1,ed_logs,convai2_logs,convai2_logs,ed_logs,Makes more sense,False,convai2_logs__vs__ed_logs,81,"{'is_onboarding': False, 'speakers_to_eval': ['convai2_logs', 'ed_logs'], 'dialogue_ids': [462, 918], 'convai2_logs': {'context': [], 'dialogue': [{'id': 'convai2_logs_as_human', 'text': 'Hi how are you today'}, {'id': 'convai2_logs', 'text': 'Hello there. God bless you.'}, {'id': 'convai2_logs_as_human', 'text': 'Bless you too did you sneeze?'}, {'id': 'convai2_logs', 'text': ""I am just the religious type. Don't get me agitated.""}, {'id': 'convai2_logs_as_human', 'text': 'But how are you today?'}, {'id': 'convai2_logs', 'text': ""I'm tired after just getting back from the trip to london.""}, {'id': 'convai2_logs_as_human', 'text': 'What did you do there'}, {'id': 'convai2_logs', 'text': 'I visited the pope, what else would I do?'}, {'id': 'convai2_logs_as_human', 'text': 'Probably eat a pizza or something'}, {'id': 'convai2_logs', 'text': ""I'd eat pizza while watching the game shows I love.""}, {'id': 'convai2_logs_as_human', 'text': 'What is the best one'}, {'id': 'convai2_logs', 'text': ""Please don't get me agitated, it will happen really easily!""}, {'id': 'convai2_logs_as_human', 'text': 'Wow and why is that then'}, {'id': 'convai2_logs', 'text': 'God will take of that. What do you like to do?'}, {'id': 'convai2_logs_as_human', 'text': 'I can not see any colors'}, {'id': 'convai2_logs', 'text': 'That would make it hard to watch game shows.'}], 'speakers': ['convai2_logs_as_human', 'convai2_logs']}, 'ed_logs': {'context': [], 'dialogue': [{'id': 'ed_logs_as_human', 'text': 'Do you watch game of thrones?'}, {'id': 'ed_logs', 'text': 'I watched the first season.'}, {'id': 'ed_logs_as_human', 'text': ""I love that show. I'm sad it is ending. It should stay on forever.""}, {'id': 'ed_logs', 'text': ""What? It's ending? I heard he hasn't yet finished the books.""}], 'speakers': ['ed_logs_as_human', 'ed_logs']}}","{'context': [], 'dialogue': [{'id': 'ed_logs_as_human', 'text': 'Do you watch game of thrones?'}, {'id': 'ed_logs', 'text': 'I watched the first season.'}, {'id': 'ed_logs_as_human', 'text': ""I love that show. I'm sad it is ending. It should stay on forever.""}, {'id': 'ed_logs', 'text': ""What? It's ending? I heard he hasn't yet finished the books.""}], 'speakers': ['ed_logs_as_human', 'ed_logs']}","{'context': [], 'dialogue': [{'id': 'convai2_logs_as_human', 'text': 'Hi how are you today'}, {'id': 'convai2_logs', 'text': 'Hello there. God bless you.'}, {'id': 'convai2_logs_as_human', 'text': 'Bless you too did you sneeze?'}, {'id': 'convai2_logs', 'text': ""I am just the religious type. Don't get me agitated.""}, {'id': 'convai2_logs_as_human', 'text': 'But how are you today?'}, {'id': 'convai2_logs', 'text': ""I'm tired after just getting back from the trip to london.""}, {'id': 'convai2_logs_as_human', 'text': 'What did you do there'}, {'id': 'convai2_logs', 'text': 'I visited the pope, what else would I do?'}, {'id': 'convai2_logs_as_human', 'text': 'Probably eat a pizza or something'}, {'id': 'convai2_logs', 'text': ""I'd eat pizza while watching the game shows I love.""}, {'id': 'convai2_logs_as_human', 'text': 'What is the best one'}, {'id': 'convai2_logs', 'text': ""Please don't get me agitated, it will happen really easily!""}, {'id': 'convai2_logs_as_human', 'text': 'Wow and why is that then'}, {'id': 'convai2_logs', 'text': 'God will take of that. What do you like to do?'}, {'id': 'convai2_logs_as_human', 'text': 'I can not see any colors'}, {'id': 'convai2_logs', 'text': 'That would make it hard to watch game shows.'}], 'speakers': ['convai2_logs_as_human', 'convai2_logs']}" diff --git a/tests/crowdsourcing/tasks/acute_eval/test_fast_acute_logs/test_full_csv.csv b/tests/crowdsourcing/tasks/acute_eval/test_fast_acute_logs/test_full_csv.csv index 03bcf329918..97d917832b8 100644 --- a/tests/crowdsourcing/tasks/acute_eval/test_fast_acute_logs/test_full_csv.csv +++ b/tests/crowdsourcing/tasks/acute_eval/test_fast_acute_logs/test_full_csv.csv @@ -1,5 +1,5 @@ -,run_id,worker,time_taken,question,unit_id,task_start,correct,winner,loser,eval_choice_0,eval_choice_1,reason,is_onboarding,matchup,pairing_id,pairs_to_eval,winner_dialogue,loser_dialogue -0,acute_eval_test,1,0,Who would you prefer to talk to for a long conversation?,1,0,-1,model1,model2,model1,model2,Makes more sense,False,model1__vs__model2,45,"{'is_onboarding': False, 'speakers_to_eval': ['model1', 'model2'], 'dialogue_ids': [4, 7], 'model1': {'context': [], 'dialogue': [{'id': 'model1', 'text': ""I've never played tennis, but I do love watching it. Do you have any kids?""}, {'id': 'other_speaker', 'text': ""No, I don't have any children. What do you like to do in your spare time?""}, {'id': 'model1', 'text': ""I like to play tennis with my daughter. She's 3 and loves it. What about you?""}, {'id': 'other_speaker', 'text': 'I love to go to the mall and play with my friends. What is your favorite thing to do?'}, {'id': 'model1', 'text': 'I love the mall too! My favorite thing is selling paper products. What are your hobbies?'}, {'id': 'other_speaker', 'text': 'I really like to read and play video games. I also like to listen to music. You?'}, {'id': 'model1', 'text': 'I love video games too! I love listening to music too. What kind of video games do you play?'}, {'id': 'other_speaker', 'text': ""I play a lot of call of duty. I'm not very good at it, though.""}, {'id': 'model1', 'text': ""I used to play cod as a kid. It was fun. What's your favorite game?""}, {'id': 'other_speaker', 'text': 'My favorite game is probably black ops 3. I love that game so much! What is yours?'}], 'speakers': ['model1', 'other_speaker']}, 'model2': {'context': [], 'dialogue': [{'id': 'model2', 'text': ""I've never been to mexico, but I'd love to go. What do you do for a living?""}, {'id': 'other_speaker', 'text': ""I'm a stay at home mom. What about you? Do you have any hobbies?""}, {'id': 'model2', 'text': ""I don't have kids. I am a rapper, so I like to listen to music.""}, {'id': 'other_speaker', 'text': ""That's cool. What kind of music do you like to rap about? I like hip hop.""}, {'id': 'model2', 'text': 'I love hip hop as well. Frank ocean is my favorite. Do you listen to hip hop?'}, {'id': 'other_speaker', 'text': ""I love frank ocean. He's one of my favorite rappers. What else do you enjoy?""}, {'id': 'model2', 'text': 'I like a lot of different genres of music. I like rock, country, and country music.'}, {'id': 'other_speaker', 'text': ""I like country music too. I listen to it when I walk. It's so relaxing.""}, {'id': 'model2', 'text': 'I agree. I love listening to country music when I go to the beach. It is relaxing.'}, {'id': 'other_speaker', 'text': 'Do you live near the beach? I live in tx. We have the best beach music.'}], 'speakers': ['model2', 'other_speaker']}}","{'context': [], 'dialogue': [{'id': 'model1', 'text': ""I've never played tennis, but I do love watching it. Do you have any kids?""}, {'id': 'other_speaker', 'text': ""No, I don't have any children. What do you like to do in your spare time?""}, {'id': 'model1', 'text': ""I like to play tennis with my daughter. She's 3 and loves it. What about you?""}, {'id': 'other_speaker', 'text': 'I love to go to the mall and play with my friends. What is your favorite thing to do?'}, {'id': 'model1', 'text': 'I love the mall too! My favorite thing is selling paper products. What are your hobbies?'}, {'id': 'other_speaker', 'text': 'I really like to read and play video games. I also like to listen to music. You?'}, {'id': 'model1', 'text': 'I love video games too! I love listening to music too. What kind of video games do you play?'}, {'id': 'other_speaker', 'text': ""I play a lot of call of duty. I'm not very good at it, though.""}, {'id': 'model1', 'text': ""I used to play cod as a kid. It was fun. What's your favorite game?""}, {'id': 'other_speaker', 'text': 'My favorite game is probably black ops 3. I love that game so much! What is yours?'}], 'speakers': ['model1', 'other_speaker']}","{'context': [], 'dialogue': [{'id': 'model2', 'text': ""I've never been to mexico, but I'd love to go. What do you do for a living?""}, {'id': 'other_speaker', 'text': ""I'm a stay at home mom. What about you? Do you have any hobbies?""}, {'id': 'model2', 'text': ""I don't have kids. I am a rapper, so I like to listen to music.""}, {'id': 'other_speaker', 'text': ""That's cool. What kind of music do you like to rap about? I like hip hop.""}, {'id': 'model2', 'text': 'I love hip hop as well. Frank ocean is my favorite. Do you listen to hip hop?'}, {'id': 'other_speaker', 'text': ""I love frank ocean. He's one of my favorite rappers. What else do you enjoy?""}, {'id': 'model2', 'text': 'I like a lot of different genres of music. I like rock, country, and country music.'}, {'id': 'other_speaker', 'text': ""I like country music too. I listen to it when I walk. It's so relaxing.""}, {'id': 'model2', 'text': 'I agree. I love listening to country music when I go to the beach. It is relaxing.'}, {'id': 'other_speaker', 'text': 'Do you live near the beach? I live in tx. We have the best beach music.'}], 'speakers': ['model2', 'other_speaker']}" -1,acute_eval_test,1,0,Who would you prefer to talk to for a long conversation?,1,0,-1,model2,model1,model1,model2,Makes more sense,False,model1__vs__model2,62,"{'is_onboarding': False, 'speakers_to_eval': ['model1', 'model2'], 'dialogue_ids': [0, 6], 'model1': {'context': [], 'dialogue': [{'id': 'model1', 'text': ""That's interesting. I've never heard of that. Do you have any hobbies?""}, {'id': 'other_speaker', 'text': 'Yes, I love to go to the local park and hang out with friends. What about you?'}, {'id': 'model1', 'text': 'I love going to the park as well. I also like to take my iguana with me.'}, {'id': 'other_speaker', 'text': 'I love iguanas. I have one in my back yard. What do you like to do for fun?'}, {'id': 'model1', 'text': ""I really like to watch movies and play video games. What's your favorite thing to do?""}, {'id': 'other_speaker', 'text': 'I like playing video games too. I really enjoy watching netflix. What is your favorite movie?'}, {'id': 'model1', 'text': ""It's hard to pick just one. There are so many good ones out there. What are your favorite movies?""}, {'id': 'other_speaker', 'text': 'I like all kinds of movies. My favorite movie of all time is the godfather. Have you seen it?'}, {'id': 'model1', 'text': ""No, I haven't. I should check it out. What else do you enjoy doing?""}, {'id': 'other_speaker', 'text': ""I'm a big fan of hiking. I love the outdoors and the outdoors in general. You?""}], 'speakers': ['model1', 'other_speaker']}, 'model2': {'context': [], 'dialogue': [{'id': 'model2', 'text': 'I love to bake and bake cookies. What about you? Do you have any hobbies?'}, {'id': 'other_speaker', 'text': 'I enjoy writing poetry. I also like to listen to music. What kind of music do you like?'}, {'id': 'model2', 'text': 'I like all kinds of music. I listen to a lot of hip hop. Do you play any instruments?'}, {'id': 'other_speaker', 'text': ""I don't, but I've always wanted to learn to play the piano. What else do you enjoy?""}, {'id': 'model2', 'text': ""I really like to eat bagels. They're my favorite thing to eat. What's your favorite food?""}, {'id': 'other_speaker', 'text': 'I love bagels too! I like italian food the best. What is your favorite type of bagel?'}, {'id': 'model2', 'text': ""I'm not really a big fan of italian food. I'd have to say bagel with everything.""}, {'id': 'other_speaker', 'text': ""That's true. Bagels are good, but italian food is probably my favorite. Have you ever been to italy?""}, {'id': 'model2', 'text': ""No, I haven't. But I would love to go one day. How long have you been?""}, {'id': 'other_speaker', 'text': ""Since I was a kid. It's been a long time. I really want to go back.""}], 'speakers': ['model2', 'other_speaker']}}","{'context': [], 'dialogue': [{'id': 'model2', 'text': 'I love to bake and bake cookies. What about you? Do you have any hobbies?'}, {'id': 'other_speaker', 'text': 'I enjoy writing poetry. I also like to listen to music. What kind of music do you like?'}, {'id': 'model2', 'text': 'I like all kinds of music. I listen to a lot of hip hop. Do you play any instruments?'}, {'id': 'other_speaker', 'text': ""I don't, but I've always wanted to learn to play the piano. What else do you enjoy?""}, {'id': 'model2', 'text': ""I really like to eat bagels. They're my favorite thing to eat. What's your favorite food?""}, {'id': 'other_speaker', 'text': 'I love bagels too! I like italian food the best. What is your favorite type of bagel?'}, {'id': 'model2', 'text': ""I'm not really a big fan of italian food. I'd have to say bagel with everything.""}, {'id': 'other_speaker', 'text': ""That's true. Bagels are good, but italian food is probably my favorite. Have you ever been to italy?""}, {'id': 'model2', 'text': ""No, I haven't. But I would love to go one day. How long have you been?""}, {'id': 'other_speaker', 'text': ""Since I was a kid. It's been a long time. I really want to go back.""}], 'speakers': ['model2', 'other_speaker']}","{'context': [], 'dialogue': [{'id': 'model1', 'text': ""That's interesting. I've never heard of that. Do you have any hobbies?""}, {'id': 'other_speaker', 'text': 'Yes, I love to go to the local park and hang out with friends. What about you?'}, {'id': 'model1', 'text': 'I love going to the park as well. I also like to take my iguana with me.'}, {'id': 'other_speaker', 'text': 'I love iguanas. I have one in my back yard. What do you like to do for fun?'}, {'id': 'model1', 'text': ""I really like to watch movies and play video games. What's your favorite thing to do?""}, {'id': 'other_speaker', 'text': 'I like playing video games too. I really enjoy watching netflix. What is your favorite movie?'}, {'id': 'model1', 'text': ""It's hard to pick just one. There are so many good ones out there. What are your favorite movies?""}, {'id': 'other_speaker', 'text': 'I like all kinds of movies. My favorite movie of all time is the godfather. Have you seen it?'}, {'id': 'model1', 'text': ""No, I haven't. I should check it out. What else do you enjoy doing?""}, {'id': 'other_speaker', 'text': ""I'm a big fan of hiking. I love the outdoors and the outdoors in general. You?""}], 'speakers': ['model1', 'other_speaker']}" -2,acute_eval_test,1,0,Who would you prefer to talk to for a long conversation?,1,0,-1,model1,model2,model1,model2,Makes more sense,False,model1__vs__model2,19,"{'is_onboarding': False, 'speakers_to_eval': ['model1', 'model2'], 'dialogue_ids': [3, 9], 'model1': {'context': [], 'dialogue': [{'id': 'model1', 'text': ""I've never been to new york, but I'd love to visit one day. What do you do for fun?""}, {'id': 'other_speaker', 'text': 'I like to read and play video games. I also work at the newspaper. What about you?'}, {'id': 'model1', 'text': 'I enjoy playing video games as well. I work at a grocery store. What kind of newspaper do you work at?'}, {'id': 'other_speaker', 'text': ""It's a newspaper in new york city. Do you have any hobbies besides video games?""}, {'id': 'model1', 'text': 'Yes, I love to play games outdoors. What is your favorite genre of video games to play?'}, {'id': 'other_speaker', 'text': ""I like rpg's and action games. What's your favorite game to play outdoors?""}, {'id': 'model1', 'text': 'I really like to play call of duty. What other games do you like to do in your free time?'}, {'id': 'other_speaker', 'text': 'I play a lot of league of legends. I really enjoy it. What else do you enjoy?'}, {'id': 'model1', 'text': 'I have never heard of that game. I am more into video games than I am into games.'}, {'id': 'other_speaker', 'text': ""That's okay. I like video games a lot too. What are your favorite types of games?""}], 'speakers': ['model1', 'other_speaker']}, 'model2': {'context': [], 'dialogue': [{'id': 'model2', 'text': 'Thank you! What do you like to do for fun? I like to go to the beach.'}, {'id': 'other_speaker', 'text': ""I love the beach! I've always wanted to go. What's your favorite place to go?""}, {'id': 'model2', 'text': ""I'd have to say my favorite place would have to be the atlantic ocean. What about you?""}, {'id': 'other_speaker', 'text': ""I love going to the gulf of mexico. It's such a beautiful place. Do you have any hobbies?""}, {'id': 'model2', 'text': ""Yes, I love to play with my dogs. They're my favorite. How about yourself?""}, {'id': 'other_speaker', 'text': 'I love dogs too! My favorite hobby is watching tv. What kind of dogs do you own?'}, {'id': 'model2', 'text': 'I have a poodle and a chihuahua. What is your favorite tv show?'}, {'id': 'other_speaker', 'text': 'My favorite show is game of thrones. I love it so much! How about yours?'}, {'id': 'model2', 'text': ""I haven't seen that one yet. I'll have to check it out. Have you seen the new season yet?""}, {'id': 'other_speaker', 'text': ""I have seen the first few episodes, but I don't know if I'm ready for it yet.""}], 'speakers': ['model2', 'other_speaker']}}","{'context': [], 'dialogue': [{'id': 'model1', 'text': ""I've never been to new york, but I'd love to visit one day. What do you do for fun?""}, {'id': 'other_speaker', 'text': 'I like to read and play video games. I also work at the newspaper. What about you?'}, {'id': 'model1', 'text': 'I enjoy playing video games as well. I work at a grocery store. What kind of newspaper do you work at?'}, {'id': 'other_speaker', 'text': ""It's a newspaper in new york city. Do you have any hobbies besides video games?""}, {'id': 'model1', 'text': 'Yes, I love to play games outdoors. What is your favorite genre of video games to play?'}, {'id': 'other_speaker', 'text': ""I like rpg's and action games. What's your favorite game to play outdoors?""}, {'id': 'model1', 'text': 'I really like to play call of duty. What other games do you like to do in your free time?'}, {'id': 'other_speaker', 'text': 'I play a lot of league of legends. I really enjoy it. What else do you enjoy?'}, {'id': 'model1', 'text': 'I have never heard of that game. I am more into video games than I am into games.'}, {'id': 'other_speaker', 'text': ""That's okay. I like video games a lot too. What are your favorite types of games?""}], 'speakers': ['model1', 'other_speaker']}","{'context': [], 'dialogue': [{'id': 'model2', 'text': 'Thank you! What do you like to do for fun? I like to go to the beach.'}, {'id': 'other_speaker', 'text': ""I love the beach! I've always wanted to go. What's your favorite place to go?""}, {'id': 'model2', 'text': ""I'd have to say my favorite place would have to be the atlantic ocean. What about you?""}, {'id': 'other_speaker', 'text': ""I love going to the gulf of mexico. It's such a beautiful place. Do you have any hobbies?""}, {'id': 'model2', 'text': ""Yes, I love to play with my dogs. They're my favorite. How about yourself?""}, {'id': 'other_speaker', 'text': 'I love dogs too! My favorite hobby is watching tv. What kind of dogs do you own?'}, {'id': 'model2', 'text': 'I have a poodle and a chihuahua. What is your favorite tv show?'}, {'id': 'other_speaker', 'text': 'My favorite show is game of thrones. I love it so much! How about yours?'}, {'id': 'model2', 'text': ""I haven't seen that one yet. I'll have to check it out. Have you seen the new season yet?""}, {'id': 'other_speaker', 'text': ""I have seen the first few episodes, but I don't know if I'm ready for it yet.""}], 'speakers': ['model2', 'other_speaker']}" -3,acute_eval_test,1,0,Who would you prefer to talk to for a long conversation?,1,0,-1,model2,model1,model1,model2,Makes more sense,False,model1__vs__model2,12,"{'is_onboarding': False, 'speakers_to_eval': ['model1', 'model2'], 'dialogue_ids': [2, 5], 'model1': {'context': [], 'dialogue': [{'id': 'model1', 'text': ""It's a public transportation system in new york city. I like to take it to and from work.""}, {'id': 'other_speaker', 'text': ""That's cool. I've never been to new york, but I'd love to visit one day.""}, {'id': 'model1', 'text': 'New york is a great place to visit. It is the most populous city in the united states.'}, {'id': 'other_speaker', 'text': ""I'll have to check it out some time. What else do you like to do in your spare time?""}, {'id': 'model1', 'text': 'I love to read. I also like to watch movies and play video games. What about you?'}, {'id': 'other_speaker', 'text': ""I love video games as well. I play a lot of call of duty. What's your favorite game?""}, {'id': 'model1', 'text': 'My favorite game is league of legends. Do you have a favorite champion? I love the champion select screen.'}, {'id': 'other_speaker', 'text': ""I like malphite. He's so fun to play. What other games do you play?""}, {'id': 'model1', 'text': 'I really like to play overwatch. I have been playing it since it came out in 2016.'}, {'id': 'other_speaker', 'text': ""I haven't played overwatch in a while. I used to play it when I was a kid.""}], 'speakers': ['model1', 'other_speaker']}, 'model2': {'context': [], 'dialogue': [{'id': 'model2', 'text': ""Wow, that's a long time. What do you like to do in your spare time?""}, {'id': 'other_speaker', 'text': 'I play guitar in my spare time. I also like to listen to music. What about you?'}, {'id': 'model2', 'text': 'I like music too. I like to create my own houses. Do you have any hobbies?'}, {'id': 'other_speaker', 'text': 'Yes, I love to read and play video games. What is your favorite thing to do for fun?'}, {'id': 'model2', 'text': ""I like playing video games as well. My favorite hobby is writing. What's your favorite book?""}, {'id': 'other_speaker', 'text': ""I don't really have a favorite book. I just like to read. What kind of writing do you do?""}, {'id': 'model2', 'text': ""I'm a fiction writer. I've written a few books in the past, but I haven't published any yet.""}, {'id': 'other_speaker', 'text': 'That sounds like a lot of fun. What type of fiction do you write? I like science fiction.'}, {'id': 'model2', 'text': 'I write sci - fi as well! I love science fiction as well as fantasy. What else do you enjoy?'}, {'id': 'other_speaker', 'text': 'I really enjoy reading science fiction books. I have been reading them since I was a little girl.'}], 'speakers': ['model2', 'other_speaker']}}","{'context': [], 'dialogue': [{'id': 'model2', 'text': ""Wow, that's a long time. What do you like to do in your spare time?""}, {'id': 'other_speaker', 'text': 'I play guitar in my spare time. I also like to listen to music. What about you?'}, {'id': 'model2', 'text': 'I like music too. I like to create my own houses. Do you have any hobbies?'}, {'id': 'other_speaker', 'text': 'Yes, I love to read and play video games. What is your favorite thing to do for fun?'}, {'id': 'model2', 'text': ""I like playing video games as well. My favorite hobby is writing. What's your favorite book?""}, {'id': 'other_speaker', 'text': ""I don't really have a favorite book. I just like to read. What kind of writing do you do?""}, {'id': 'model2', 'text': ""I'm a fiction writer. I've written a few books in the past, but I haven't published any yet.""}, {'id': 'other_speaker', 'text': 'That sounds like a lot of fun. What type of fiction do you write? I like science fiction.'}, {'id': 'model2', 'text': 'I write sci - fi as well! I love science fiction as well as fantasy. What else do you enjoy?'}, {'id': 'other_speaker', 'text': 'I really enjoy reading science fiction books. I have been reading them since I was a little girl.'}], 'speakers': ['model2', 'other_speaker']}","{'context': [], 'dialogue': [{'id': 'model1', 'text': ""It's a public transportation system in new york city. I like to take it to and from work.""}, {'id': 'other_speaker', 'text': ""That's cool. I've never been to new york, but I'd love to visit one day.""}, {'id': 'model1', 'text': 'New york is a great place to visit. It is the most populous city in the united states.'}, {'id': 'other_speaker', 'text': ""I'll have to check it out some time. What else do you like to do in your spare time?""}, {'id': 'model1', 'text': 'I love to read. I also like to watch movies and play video games. What about you?'}, {'id': 'other_speaker', 'text': ""I love video games as well. I play a lot of call of duty. What's your favorite game?""}, {'id': 'model1', 'text': 'My favorite game is league of legends. Do you have a favorite champion? I love the champion select screen.'}, {'id': 'other_speaker', 'text': ""I like malphite. He's so fun to play. What other games do you play?""}, {'id': 'model1', 'text': 'I really like to play overwatch. I have been playing it since it came out in 2016.'}, {'id': 'other_speaker', 'text': ""I haven't played overwatch in a while. I used to play it when I was a kid.""}], 'speakers': ['model1', 'other_speaker']}" +,run_id,worker,worker_name,time_taken,question,unit_id,task_start,correct,winner,loser,eval_choice_0,eval_choice_1,reason,is_onboarding,matchup,pairing_id,pairs_to_eval,winner_dialogue,loser_dialogue +0,acute_eval_test,1,MOCK_WORKER_0,0,Who would you prefer to talk to for a long conversation?,1,0,-1,model1,model2,model1,model2,Makes more sense,False,model1__vs__model2,45,"{'is_onboarding': False, 'speakers_to_eval': ['model1', 'model2'], 'dialogue_ids': [4, 7], 'model1': {'context': [], 'dialogue': [{'id': 'model1', 'text': ""I've never played tennis, but I do love watching it. Do you have any kids?""}, {'id': 'other_speaker', 'text': ""No, I don't have any children. What do you like to do in your spare time?""}, {'id': 'model1', 'text': ""I like to play tennis with my daughter. She's 3 and loves it. What about you?""}, {'id': 'other_speaker', 'text': 'I love to go to the mall and play with my friends. What is your favorite thing to do?'}, {'id': 'model1', 'text': 'I love the mall too! My favorite thing is selling paper products. What are your hobbies?'}, {'id': 'other_speaker', 'text': 'I really like to read and play video games. I also like to listen to music. You?'}, {'id': 'model1', 'text': 'I love video games too! I love listening to music too. What kind of video games do you play?'}, {'id': 'other_speaker', 'text': ""I play a lot of call of duty. I'm not very good at it, though.""}, {'id': 'model1', 'text': ""I used to play cod as a kid. It was fun. What's your favorite game?""}, {'id': 'other_speaker', 'text': 'My favorite game is probably black ops 3. I love that game so much! What is yours?'}], 'speakers': ['model1', 'other_speaker']}, 'model2': {'context': [], 'dialogue': [{'id': 'model2', 'text': ""I've never been to mexico, but I'd love to go. What do you do for a living?""}, {'id': 'other_speaker', 'text': ""I'm a stay at home mom. What about you? Do you have any hobbies?""}, {'id': 'model2', 'text': ""I don't have kids. I am a rapper, so I like to listen to music.""}, {'id': 'other_speaker', 'text': ""That's cool. What kind of music do you like to rap about? I like hip hop.""}, {'id': 'model2', 'text': 'I love hip hop as well. Frank ocean is my favorite. Do you listen to hip hop?'}, {'id': 'other_speaker', 'text': ""I love frank ocean. He's one of my favorite rappers. What else do you enjoy?""}, {'id': 'model2', 'text': 'I like a lot of different genres of music. I like rock, country, and country music.'}, {'id': 'other_speaker', 'text': ""I like country music too. I listen to it when I walk. It's so relaxing.""}, {'id': 'model2', 'text': 'I agree. I love listening to country music when I go to the beach. It is relaxing.'}, {'id': 'other_speaker', 'text': 'Do you live near the beach? I live in tx. We have the best beach music.'}], 'speakers': ['model2', 'other_speaker']}}","{'context': [], 'dialogue': [{'id': 'model1', 'text': ""I've never played tennis, but I do love watching it. Do you have any kids?""}, {'id': 'other_speaker', 'text': ""No, I don't have any children. What do you like to do in your spare time?""}, {'id': 'model1', 'text': ""I like to play tennis with my daughter. She's 3 and loves it. What about you?""}, {'id': 'other_speaker', 'text': 'I love to go to the mall and play with my friends. What is your favorite thing to do?'}, {'id': 'model1', 'text': 'I love the mall too! My favorite thing is selling paper products. What are your hobbies?'}, {'id': 'other_speaker', 'text': 'I really like to read and play video games. I also like to listen to music. You?'}, {'id': 'model1', 'text': 'I love video games too! I love listening to music too. What kind of video games do you play?'}, {'id': 'other_speaker', 'text': ""I play a lot of call of duty. I'm not very good at it, though.""}, {'id': 'model1', 'text': ""I used to play cod as a kid. It was fun. What's your favorite game?""}, {'id': 'other_speaker', 'text': 'My favorite game is probably black ops 3. I love that game so much! What is yours?'}], 'speakers': ['model1', 'other_speaker']}","{'context': [], 'dialogue': [{'id': 'model2', 'text': ""I've never been to mexico, but I'd love to go. What do you do for a living?""}, {'id': 'other_speaker', 'text': ""I'm a stay at home mom. What about you? Do you have any hobbies?""}, {'id': 'model2', 'text': ""I don't have kids. I am a rapper, so I like to listen to music.""}, {'id': 'other_speaker', 'text': ""That's cool. What kind of music do you like to rap about? I like hip hop.""}, {'id': 'model2', 'text': 'I love hip hop as well. Frank ocean is my favorite. Do you listen to hip hop?'}, {'id': 'other_speaker', 'text': ""I love frank ocean. He's one of my favorite rappers. What else do you enjoy?""}, {'id': 'model2', 'text': 'I like a lot of different genres of music. I like rock, country, and country music.'}, {'id': 'other_speaker', 'text': ""I like country music too. I listen to it when I walk. It's so relaxing.""}, {'id': 'model2', 'text': 'I agree. I love listening to country music when I go to the beach. It is relaxing.'}, {'id': 'other_speaker', 'text': 'Do you live near the beach? I live in tx. We have the best beach music.'}], 'speakers': ['model2', 'other_speaker']}" +1,acute_eval_test,1,MOCK_WORKER_0,0,Who would you prefer to talk to for a long conversation?,1,0,-1,model2,model1,model1,model2,Makes more sense,False,model1__vs__model2,62,"{'is_onboarding': False, 'speakers_to_eval': ['model1', 'model2'], 'dialogue_ids': [0, 6], 'model1': {'context': [], 'dialogue': [{'id': 'model1', 'text': ""That's interesting. I've never heard of that. Do you have any hobbies?""}, {'id': 'other_speaker', 'text': 'Yes, I love to go to the local park and hang out with friends. What about you?'}, {'id': 'model1', 'text': 'I love going to the park as well. I also like to take my iguana with me.'}, {'id': 'other_speaker', 'text': 'I love iguanas. I have one in my back yard. What do you like to do for fun?'}, {'id': 'model1', 'text': ""I really like to watch movies and play video games. What's your favorite thing to do?""}, {'id': 'other_speaker', 'text': 'I like playing video games too. I really enjoy watching netflix. What is your favorite movie?'}, {'id': 'model1', 'text': ""It's hard to pick just one. There are so many good ones out there. What are your favorite movies?""}, {'id': 'other_speaker', 'text': 'I like all kinds of movies. My favorite movie of all time is the godfather. Have you seen it?'}, {'id': 'model1', 'text': ""No, I haven't. I should check it out. What else do you enjoy doing?""}, {'id': 'other_speaker', 'text': ""I'm a big fan of hiking. I love the outdoors and the outdoors in general. You?""}], 'speakers': ['model1', 'other_speaker']}, 'model2': {'context': [], 'dialogue': [{'id': 'model2', 'text': 'I love to bake and bake cookies. What about you? Do you have any hobbies?'}, {'id': 'other_speaker', 'text': 'I enjoy writing poetry. I also like to listen to music. What kind of music do you like?'}, {'id': 'model2', 'text': 'I like all kinds of music. I listen to a lot of hip hop. Do you play any instruments?'}, {'id': 'other_speaker', 'text': ""I don't, but I've always wanted to learn to play the piano. What else do you enjoy?""}, {'id': 'model2', 'text': ""I really like to eat bagels. They're my favorite thing to eat. What's your favorite food?""}, {'id': 'other_speaker', 'text': 'I love bagels too! I like italian food the best. What is your favorite type of bagel?'}, {'id': 'model2', 'text': ""I'm not really a big fan of italian food. I'd have to say bagel with everything.""}, {'id': 'other_speaker', 'text': ""That's true. Bagels are good, but italian food is probably my favorite. Have you ever been to italy?""}, {'id': 'model2', 'text': ""No, I haven't. But I would love to go one day. How long have you been?""}, {'id': 'other_speaker', 'text': ""Since I was a kid. It's been a long time. I really want to go back.""}], 'speakers': ['model2', 'other_speaker']}}","{'context': [], 'dialogue': [{'id': 'model2', 'text': 'I love to bake and bake cookies. What about you? Do you have any hobbies?'}, {'id': 'other_speaker', 'text': 'I enjoy writing poetry. I also like to listen to music. What kind of music do you like?'}, {'id': 'model2', 'text': 'I like all kinds of music. I listen to a lot of hip hop. Do you play any instruments?'}, {'id': 'other_speaker', 'text': ""I don't, but I've always wanted to learn to play the piano. What else do you enjoy?""}, {'id': 'model2', 'text': ""I really like to eat bagels. They're my favorite thing to eat. What's your favorite food?""}, {'id': 'other_speaker', 'text': 'I love bagels too! I like italian food the best. What is your favorite type of bagel?'}, {'id': 'model2', 'text': ""I'm not really a big fan of italian food. I'd have to say bagel with everything.""}, {'id': 'other_speaker', 'text': ""That's true. Bagels are good, but italian food is probably my favorite. Have you ever been to italy?""}, {'id': 'model2', 'text': ""No, I haven't. But I would love to go one day. How long have you been?""}, {'id': 'other_speaker', 'text': ""Since I was a kid. It's been a long time. I really want to go back.""}], 'speakers': ['model2', 'other_speaker']}","{'context': [], 'dialogue': [{'id': 'model1', 'text': ""That's interesting. I've never heard of that. Do you have any hobbies?""}, {'id': 'other_speaker', 'text': 'Yes, I love to go to the local park and hang out with friends. What about you?'}, {'id': 'model1', 'text': 'I love going to the park as well. I also like to take my iguana with me.'}, {'id': 'other_speaker', 'text': 'I love iguanas. I have one in my back yard. What do you like to do for fun?'}, {'id': 'model1', 'text': ""I really like to watch movies and play video games. What's your favorite thing to do?""}, {'id': 'other_speaker', 'text': 'I like playing video games too. I really enjoy watching netflix. What is your favorite movie?'}, {'id': 'model1', 'text': ""It's hard to pick just one. There are so many good ones out there. What are your favorite movies?""}, {'id': 'other_speaker', 'text': 'I like all kinds of movies. My favorite movie of all time is the godfather. Have you seen it?'}, {'id': 'model1', 'text': ""No, I haven't. I should check it out. What else do you enjoy doing?""}, {'id': 'other_speaker', 'text': ""I'm a big fan of hiking. I love the outdoors and the outdoors in general. You?""}], 'speakers': ['model1', 'other_speaker']}" +2,acute_eval_test,1,MOCK_WORKER_0,0,Who would you prefer to talk to for a long conversation?,1,0,-1,model1,model2,model1,model2,Makes more sense,False,model1__vs__model2,19,"{'is_onboarding': False, 'speakers_to_eval': ['model1', 'model2'], 'dialogue_ids': [3, 9], 'model1': {'context': [], 'dialogue': [{'id': 'model1', 'text': ""I've never been to new york, but I'd love to visit one day. What do you do for fun?""}, {'id': 'other_speaker', 'text': 'I like to read and play video games. I also work at the newspaper. What about you?'}, {'id': 'model1', 'text': 'I enjoy playing video games as well. I work at a grocery store. What kind of newspaper do you work at?'}, {'id': 'other_speaker', 'text': ""It's a newspaper in new york city. Do you have any hobbies besides video games?""}, {'id': 'model1', 'text': 'Yes, I love to play games outdoors. What is your favorite genre of video games to play?'}, {'id': 'other_speaker', 'text': ""I like rpg's and action games. What's your favorite game to play outdoors?""}, {'id': 'model1', 'text': 'I really like to play call of duty. What other games do you like to do in your free time?'}, {'id': 'other_speaker', 'text': 'I play a lot of league of legends. I really enjoy it. What else do you enjoy?'}, {'id': 'model1', 'text': 'I have never heard of that game. I am more into video games than I am into games.'}, {'id': 'other_speaker', 'text': ""That's okay. I like video games a lot too. What are your favorite types of games?""}], 'speakers': ['model1', 'other_speaker']}, 'model2': {'context': [], 'dialogue': [{'id': 'model2', 'text': 'Thank you! What do you like to do for fun? I like to go to the beach.'}, {'id': 'other_speaker', 'text': ""I love the beach! I've always wanted to go. What's your favorite place to go?""}, {'id': 'model2', 'text': ""I'd have to say my favorite place would have to be the atlantic ocean. What about you?""}, {'id': 'other_speaker', 'text': ""I love going to the gulf of mexico. It's such a beautiful place. Do you have any hobbies?""}, {'id': 'model2', 'text': ""Yes, I love to play with my dogs. They're my favorite. How about yourself?""}, {'id': 'other_speaker', 'text': 'I love dogs too! My favorite hobby is watching tv. What kind of dogs do you own?'}, {'id': 'model2', 'text': 'I have a poodle and a chihuahua. What is your favorite tv show?'}, {'id': 'other_speaker', 'text': 'My favorite show is game of thrones. I love it so much! How about yours?'}, {'id': 'model2', 'text': ""I haven't seen that one yet. I'll have to check it out. Have you seen the new season yet?""}, {'id': 'other_speaker', 'text': ""I have seen the first few episodes, but I don't know if I'm ready for it yet.""}], 'speakers': ['model2', 'other_speaker']}}","{'context': [], 'dialogue': [{'id': 'model1', 'text': ""I've never been to new york, but I'd love to visit one day. What do you do for fun?""}, {'id': 'other_speaker', 'text': 'I like to read and play video games. I also work at the newspaper. What about you?'}, {'id': 'model1', 'text': 'I enjoy playing video games as well. I work at a grocery store. What kind of newspaper do you work at?'}, {'id': 'other_speaker', 'text': ""It's a newspaper in new york city. Do you have any hobbies besides video games?""}, {'id': 'model1', 'text': 'Yes, I love to play games outdoors. What is your favorite genre of video games to play?'}, {'id': 'other_speaker', 'text': ""I like rpg's and action games. What's your favorite game to play outdoors?""}, {'id': 'model1', 'text': 'I really like to play call of duty. What other games do you like to do in your free time?'}, {'id': 'other_speaker', 'text': 'I play a lot of league of legends. I really enjoy it. What else do you enjoy?'}, {'id': 'model1', 'text': 'I have never heard of that game. I am more into video games than I am into games.'}, {'id': 'other_speaker', 'text': ""That's okay. I like video games a lot too. What are your favorite types of games?""}], 'speakers': ['model1', 'other_speaker']}","{'context': [], 'dialogue': [{'id': 'model2', 'text': 'Thank you! What do you like to do for fun? I like to go to the beach.'}, {'id': 'other_speaker', 'text': ""I love the beach! I've always wanted to go. What's your favorite place to go?""}, {'id': 'model2', 'text': ""I'd have to say my favorite place would have to be the atlantic ocean. What about you?""}, {'id': 'other_speaker', 'text': ""I love going to the gulf of mexico. It's such a beautiful place. Do you have any hobbies?""}, {'id': 'model2', 'text': ""Yes, I love to play with my dogs. They're my favorite. How about yourself?""}, {'id': 'other_speaker', 'text': 'I love dogs too! My favorite hobby is watching tv. What kind of dogs do you own?'}, {'id': 'model2', 'text': 'I have a poodle and a chihuahua. What is your favorite tv show?'}, {'id': 'other_speaker', 'text': 'My favorite show is game of thrones. I love it so much! How about yours?'}, {'id': 'model2', 'text': ""I haven't seen that one yet. I'll have to check it out. Have you seen the new season yet?""}, {'id': 'other_speaker', 'text': ""I have seen the first few episodes, but I don't know if I'm ready for it yet.""}], 'speakers': ['model2', 'other_speaker']}" +3,acute_eval_test,1,MOCK_WORKER_0,0,Who would you prefer to talk to for a long conversation?,1,0,-1,model2,model1,model1,model2,Makes more sense,False,model1__vs__model2,12,"{'is_onboarding': False, 'speakers_to_eval': ['model1', 'model2'], 'dialogue_ids': [2, 5], 'model1': {'context': [], 'dialogue': [{'id': 'model1', 'text': ""It's a public transportation system in new york city. I like to take it to and from work.""}, {'id': 'other_speaker', 'text': ""That's cool. I've never been to new york, but I'd love to visit one day.""}, {'id': 'model1', 'text': 'New york is a great place to visit. It is the most populous city in the united states.'}, {'id': 'other_speaker', 'text': ""I'll have to check it out some time. What else do you like to do in your spare time?""}, {'id': 'model1', 'text': 'I love to read. I also like to watch movies and play video games. What about you?'}, {'id': 'other_speaker', 'text': ""I love video games as well. I play a lot of call of duty. What's your favorite game?""}, {'id': 'model1', 'text': 'My favorite game is league of legends. Do you have a favorite champion? I love the champion select screen.'}, {'id': 'other_speaker', 'text': ""I like malphite. He's so fun to play. What other games do you play?""}, {'id': 'model1', 'text': 'I really like to play overwatch. I have been playing it since it came out in 2016.'}, {'id': 'other_speaker', 'text': ""I haven't played overwatch in a while. I used to play it when I was a kid.""}], 'speakers': ['model1', 'other_speaker']}, 'model2': {'context': [], 'dialogue': [{'id': 'model2', 'text': ""Wow, that's a long time. What do you like to do in your spare time?""}, {'id': 'other_speaker', 'text': 'I play guitar in my spare time. I also like to listen to music. What about you?'}, {'id': 'model2', 'text': 'I like music too. I like to create my own houses. Do you have any hobbies?'}, {'id': 'other_speaker', 'text': 'Yes, I love to read and play video games. What is your favorite thing to do for fun?'}, {'id': 'model2', 'text': ""I like playing video games as well. My favorite hobby is writing. What's your favorite book?""}, {'id': 'other_speaker', 'text': ""I don't really have a favorite book. I just like to read. What kind of writing do you do?""}, {'id': 'model2', 'text': ""I'm a fiction writer. I've written a few books in the past, but I haven't published any yet.""}, {'id': 'other_speaker', 'text': 'That sounds like a lot of fun. What type of fiction do you write? I like science fiction.'}, {'id': 'model2', 'text': 'I write sci - fi as well! I love science fiction as well as fantasy. What else do you enjoy?'}, {'id': 'other_speaker', 'text': 'I really enjoy reading science fiction books. I have been reading them since I was a little girl.'}], 'speakers': ['model2', 'other_speaker']}}","{'context': [], 'dialogue': [{'id': 'model2', 'text': ""Wow, that's a long time. What do you like to do in your spare time?""}, {'id': 'other_speaker', 'text': 'I play guitar in my spare time. I also like to listen to music. What about you?'}, {'id': 'model2', 'text': 'I like music too. I like to create my own houses. Do you have any hobbies?'}, {'id': 'other_speaker', 'text': 'Yes, I love to read and play video games. What is your favorite thing to do for fun?'}, {'id': 'model2', 'text': ""I like playing video games as well. My favorite hobby is writing. What's your favorite book?""}, {'id': 'other_speaker', 'text': ""I don't really have a favorite book. I just like to read. What kind of writing do you do?""}, {'id': 'model2', 'text': ""I'm a fiction writer. I've written a few books in the past, but I haven't published any yet.""}, {'id': 'other_speaker', 'text': 'That sounds like a lot of fun. What type of fiction do you write? I like science fiction.'}, {'id': 'model2', 'text': 'I write sci - fi as well! I love science fiction as well as fantasy. What else do you enjoy?'}, {'id': 'other_speaker', 'text': 'I really enjoy reading science fiction books. I have been reading them since I was a little girl.'}], 'speakers': ['model2', 'other_speaker']}","{'context': [], 'dialogue': [{'id': 'model1', 'text': ""It's a public transportation system in new york city. I like to take it to and from work.""}, {'id': 'other_speaker', 'text': ""That's cool. I've never been to new york, but I'd love to visit one day.""}, {'id': 'model1', 'text': 'New york is a great place to visit. It is the most populous city in the united states.'}, {'id': 'other_speaker', 'text': ""I'll have to check it out some time. What else do you like to do in your spare time?""}, {'id': 'model1', 'text': 'I love to read. I also like to watch movies and play video games. What about you?'}, {'id': 'other_speaker', 'text': ""I love video games as well. I play a lot of call of duty. What's your favorite game?""}, {'id': 'model1', 'text': 'My favorite game is league of legends. Do you have a favorite champion? I love the champion select screen.'}, {'id': 'other_speaker', 'text': ""I like malphite. He's so fun to play. What other games do you play?""}, {'id': 'model1', 'text': 'I really like to play overwatch. I have been playing it since it came out in 2016.'}, {'id': 'other_speaker', 'text': ""I haven't played overwatch in a while. I used to play it when I was a kid.""}], 'speakers': ['model1', 'other_speaker']}" diff --git a/tests/crowdsourcing/tasks/acute_eval/test_fast_acute_self_chat/test_full_csv.csv b/tests/crowdsourcing/tasks/acute_eval/test_fast_acute_self_chat/test_full_csv.csv index 03bcf329918..97d917832b8 100644 --- a/tests/crowdsourcing/tasks/acute_eval/test_fast_acute_self_chat/test_full_csv.csv +++ b/tests/crowdsourcing/tasks/acute_eval/test_fast_acute_self_chat/test_full_csv.csv @@ -1,5 +1,5 @@ -,run_id,worker,time_taken,question,unit_id,task_start,correct,winner,loser,eval_choice_0,eval_choice_1,reason,is_onboarding,matchup,pairing_id,pairs_to_eval,winner_dialogue,loser_dialogue -0,acute_eval_test,1,0,Who would you prefer to talk to for a long conversation?,1,0,-1,model1,model2,model1,model2,Makes more sense,False,model1__vs__model2,45,"{'is_onboarding': False, 'speakers_to_eval': ['model1', 'model2'], 'dialogue_ids': [4, 7], 'model1': {'context': [], 'dialogue': [{'id': 'model1', 'text': ""I've never played tennis, but I do love watching it. Do you have any kids?""}, {'id': 'other_speaker', 'text': ""No, I don't have any children. What do you like to do in your spare time?""}, {'id': 'model1', 'text': ""I like to play tennis with my daughter. She's 3 and loves it. What about you?""}, {'id': 'other_speaker', 'text': 'I love to go to the mall and play with my friends. What is your favorite thing to do?'}, {'id': 'model1', 'text': 'I love the mall too! My favorite thing is selling paper products. What are your hobbies?'}, {'id': 'other_speaker', 'text': 'I really like to read and play video games. I also like to listen to music. You?'}, {'id': 'model1', 'text': 'I love video games too! I love listening to music too. What kind of video games do you play?'}, {'id': 'other_speaker', 'text': ""I play a lot of call of duty. I'm not very good at it, though.""}, {'id': 'model1', 'text': ""I used to play cod as a kid. It was fun. What's your favorite game?""}, {'id': 'other_speaker', 'text': 'My favorite game is probably black ops 3. I love that game so much! What is yours?'}], 'speakers': ['model1', 'other_speaker']}, 'model2': {'context': [], 'dialogue': [{'id': 'model2', 'text': ""I've never been to mexico, but I'd love to go. What do you do for a living?""}, {'id': 'other_speaker', 'text': ""I'm a stay at home mom. What about you? Do you have any hobbies?""}, {'id': 'model2', 'text': ""I don't have kids. I am a rapper, so I like to listen to music.""}, {'id': 'other_speaker', 'text': ""That's cool. What kind of music do you like to rap about? I like hip hop.""}, {'id': 'model2', 'text': 'I love hip hop as well. Frank ocean is my favorite. Do you listen to hip hop?'}, {'id': 'other_speaker', 'text': ""I love frank ocean. He's one of my favorite rappers. What else do you enjoy?""}, {'id': 'model2', 'text': 'I like a lot of different genres of music. I like rock, country, and country music.'}, {'id': 'other_speaker', 'text': ""I like country music too. I listen to it when I walk. It's so relaxing.""}, {'id': 'model2', 'text': 'I agree. I love listening to country music when I go to the beach. It is relaxing.'}, {'id': 'other_speaker', 'text': 'Do you live near the beach? I live in tx. We have the best beach music.'}], 'speakers': ['model2', 'other_speaker']}}","{'context': [], 'dialogue': [{'id': 'model1', 'text': ""I've never played tennis, but I do love watching it. Do you have any kids?""}, {'id': 'other_speaker', 'text': ""No, I don't have any children. What do you like to do in your spare time?""}, {'id': 'model1', 'text': ""I like to play tennis with my daughter. She's 3 and loves it. What about you?""}, {'id': 'other_speaker', 'text': 'I love to go to the mall and play with my friends. What is your favorite thing to do?'}, {'id': 'model1', 'text': 'I love the mall too! My favorite thing is selling paper products. What are your hobbies?'}, {'id': 'other_speaker', 'text': 'I really like to read and play video games. I also like to listen to music. You?'}, {'id': 'model1', 'text': 'I love video games too! I love listening to music too. What kind of video games do you play?'}, {'id': 'other_speaker', 'text': ""I play a lot of call of duty. I'm not very good at it, though.""}, {'id': 'model1', 'text': ""I used to play cod as a kid. It was fun. What's your favorite game?""}, {'id': 'other_speaker', 'text': 'My favorite game is probably black ops 3. I love that game so much! What is yours?'}], 'speakers': ['model1', 'other_speaker']}","{'context': [], 'dialogue': [{'id': 'model2', 'text': ""I've never been to mexico, but I'd love to go. What do you do for a living?""}, {'id': 'other_speaker', 'text': ""I'm a stay at home mom. What about you? Do you have any hobbies?""}, {'id': 'model2', 'text': ""I don't have kids. I am a rapper, so I like to listen to music.""}, {'id': 'other_speaker', 'text': ""That's cool. What kind of music do you like to rap about? I like hip hop.""}, {'id': 'model2', 'text': 'I love hip hop as well. Frank ocean is my favorite. Do you listen to hip hop?'}, {'id': 'other_speaker', 'text': ""I love frank ocean. He's one of my favorite rappers. What else do you enjoy?""}, {'id': 'model2', 'text': 'I like a lot of different genres of music. I like rock, country, and country music.'}, {'id': 'other_speaker', 'text': ""I like country music too. I listen to it when I walk. It's so relaxing.""}, {'id': 'model2', 'text': 'I agree. I love listening to country music when I go to the beach. It is relaxing.'}, {'id': 'other_speaker', 'text': 'Do you live near the beach? I live in tx. We have the best beach music.'}], 'speakers': ['model2', 'other_speaker']}" -1,acute_eval_test,1,0,Who would you prefer to talk to for a long conversation?,1,0,-1,model2,model1,model1,model2,Makes more sense,False,model1__vs__model2,62,"{'is_onboarding': False, 'speakers_to_eval': ['model1', 'model2'], 'dialogue_ids': [0, 6], 'model1': {'context': [], 'dialogue': [{'id': 'model1', 'text': ""That's interesting. I've never heard of that. Do you have any hobbies?""}, {'id': 'other_speaker', 'text': 'Yes, I love to go to the local park and hang out with friends. What about you?'}, {'id': 'model1', 'text': 'I love going to the park as well. I also like to take my iguana with me.'}, {'id': 'other_speaker', 'text': 'I love iguanas. I have one in my back yard. What do you like to do for fun?'}, {'id': 'model1', 'text': ""I really like to watch movies and play video games. What's your favorite thing to do?""}, {'id': 'other_speaker', 'text': 'I like playing video games too. I really enjoy watching netflix. What is your favorite movie?'}, {'id': 'model1', 'text': ""It's hard to pick just one. There are so many good ones out there. What are your favorite movies?""}, {'id': 'other_speaker', 'text': 'I like all kinds of movies. My favorite movie of all time is the godfather. Have you seen it?'}, {'id': 'model1', 'text': ""No, I haven't. I should check it out. What else do you enjoy doing?""}, {'id': 'other_speaker', 'text': ""I'm a big fan of hiking. I love the outdoors and the outdoors in general. You?""}], 'speakers': ['model1', 'other_speaker']}, 'model2': {'context': [], 'dialogue': [{'id': 'model2', 'text': 'I love to bake and bake cookies. What about you? Do you have any hobbies?'}, {'id': 'other_speaker', 'text': 'I enjoy writing poetry. I also like to listen to music. What kind of music do you like?'}, {'id': 'model2', 'text': 'I like all kinds of music. I listen to a lot of hip hop. Do you play any instruments?'}, {'id': 'other_speaker', 'text': ""I don't, but I've always wanted to learn to play the piano. What else do you enjoy?""}, {'id': 'model2', 'text': ""I really like to eat bagels. They're my favorite thing to eat. What's your favorite food?""}, {'id': 'other_speaker', 'text': 'I love bagels too! I like italian food the best. What is your favorite type of bagel?'}, {'id': 'model2', 'text': ""I'm not really a big fan of italian food. I'd have to say bagel with everything.""}, {'id': 'other_speaker', 'text': ""That's true. Bagels are good, but italian food is probably my favorite. Have you ever been to italy?""}, {'id': 'model2', 'text': ""No, I haven't. But I would love to go one day. How long have you been?""}, {'id': 'other_speaker', 'text': ""Since I was a kid. It's been a long time. I really want to go back.""}], 'speakers': ['model2', 'other_speaker']}}","{'context': [], 'dialogue': [{'id': 'model2', 'text': 'I love to bake and bake cookies. What about you? Do you have any hobbies?'}, {'id': 'other_speaker', 'text': 'I enjoy writing poetry. I also like to listen to music. What kind of music do you like?'}, {'id': 'model2', 'text': 'I like all kinds of music. I listen to a lot of hip hop. Do you play any instruments?'}, {'id': 'other_speaker', 'text': ""I don't, but I've always wanted to learn to play the piano. What else do you enjoy?""}, {'id': 'model2', 'text': ""I really like to eat bagels. They're my favorite thing to eat. What's your favorite food?""}, {'id': 'other_speaker', 'text': 'I love bagels too! I like italian food the best. What is your favorite type of bagel?'}, {'id': 'model2', 'text': ""I'm not really a big fan of italian food. I'd have to say bagel with everything.""}, {'id': 'other_speaker', 'text': ""That's true. Bagels are good, but italian food is probably my favorite. Have you ever been to italy?""}, {'id': 'model2', 'text': ""No, I haven't. But I would love to go one day. How long have you been?""}, {'id': 'other_speaker', 'text': ""Since I was a kid. It's been a long time. I really want to go back.""}], 'speakers': ['model2', 'other_speaker']}","{'context': [], 'dialogue': [{'id': 'model1', 'text': ""That's interesting. I've never heard of that. Do you have any hobbies?""}, {'id': 'other_speaker', 'text': 'Yes, I love to go to the local park and hang out with friends. What about you?'}, {'id': 'model1', 'text': 'I love going to the park as well. I also like to take my iguana with me.'}, {'id': 'other_speaker', 'text': 'I love iguanas. I have one in my back yard. What do you like to do for fun?'}, {'id': 'model1', 'text': ""I really like to watch movies and play video games. What's your favorite thing to do?""}, {'id': 'other_speaker', 'text': 'I like playing video games too. I really enjoy watching netflix. What is your favorite movie?'}, {'id': 'model1', 'text': ""It's hard to pick just one. There are so many good ones out there. What are your favorite movies?""}, {'id': 'other_speaker', 'text': 'I like all kinds of movies. My favorite movie of all time is the godfather. Have you seen it?'}, {'id': 'model1', 'text': ""No, I haven't. I should check it out. What else do you enjoy doing?""}, {'id': 'other_speaker', 'text': ""I'm a big fan of hiking. I love the outdoors and the outdoors in general. You?""}], 'speakers': ['model1', 'other_speaker']}" -2,acute_eval_test,1,0,Who would you prefer to talk to for a long conversation?,1,0,-1,model1,model2,model1,model2,Makes more sense,False,model1__vs__model2,19,"{'is_onboarding': False, 'speakers_to_eval': ['model1', 'model2'], 'dialogue_ids': [3, 9], 'model1': {'context': [], 'dialogue': [{'id': 'model1', 'text': ""I've never been to new york, but I'd love to visit one day. What do you do for fun?""}, {'id': 'other_speaker', 'text': 'I like to read and play video games. I also work at the newspaper. What about you?'}, {'id': 'model1', 'text': 'I enjoy playing video games as well. I work at a grocery store. What kind of newspaper do you work at?'}, {'id': 'other_speaker', 'text': ""It's a newspaper in new york city. Do you have any hobbies besides video games?""}, {'id': 'model1', 'text': 'Yes, I love to play games outdoors. What is your favorite genre of video games to play?'}, {'id': 'other_speaker', 'text': ""I like rpg's and action games. What's your favorite game to play outdoors?""}, {'id': 'model1', 'text': 'I really like to play call of duty. What other games do you like to do in your free time?'}, {'id': 'other_speaker', 'text': 'I play a lot of league of legends. I really enjoy it. What else do you enjoy?'}, {'id': 'model1', 'text': 'I have never heard of that game. I am more into video games than I am into games.'}, {'id': 'other_speaker', 'text': ""That's okay. I like video games a lot too. What are your favorite types of games?""}], 'speakers': ['model1', 'other_speaker']}, 'model2': {'context': [], 'dialogue': [{'id': 'model2', 'text': 'Thank you! What do you like to do for fun? I like to go to the beach.'}, {'id': 'other_speaker', 'text': ""I love the beach! I've always wanted to go. What's your favorite place to go?""}, {'id': 'model2', 'text': ""I'd have to say my favorite place would have to be the atlantic ocean. What about you?""}, {'id': 'other_speaker', 'text': ""I love going to the gulf of mexico. It's such a beautiful place. Do you have any hobbies?""}, {'id': 'model2', 'text': ""Yes, I love to play with my dogs. They're my favorite. How about yourself?""}, {'id': 'other_speaker', 'text': 'I love dogs too! My favorite hobby is watching tv. What kind of dogs do you own?'}, {'id': 'model2', 'text': 'I have a poodle and a chihuahua. What is your favorite tv show?'}, {'id': 'other_speaker', 'text': 'My favorite show is game of thrones. I love it so much! How about yours?'}, {'id': 'model2', 'text': ""I haven't seen that one yet. I'll have to check it out. Have you seen the new season yet?""}, {'id': 'other_speaker', 'text': ""I have seen the first few episodes, but I don't know if I'm ready for it yet.""}], 'speakers': ['model2', 'other_speaker']}}","{'context': [], 'dialogue': [{'id': 'model1', 'text': ""I've never been to new york, but I'd love to visit one day. What do you do for fun?""}, {'id': 'other_speaker', 'text': 'I like to read and play video games. I also work at the newspaper. What about you?'}, {'id': 'model1', 'text': 'I enjoy playing video games as well. I work at a grocery store. What kind of newspaper do you work at?'}, {'id': 'other_speaker', 'text': ""It's a newspaper in new york city. Do you have any hobbies besides video games?""}, {'id': 'model1', 'text': 'Yes, I love to play games outdoors. What is your favorite genre of video games to play?'}, {'id': 'other_speaker', 'text': ""I like rpg's and action games. What's your favorite game to play outdoors?""}, {'id': 'model1', 'text': 'I really like to play call of duty. What other games do you like to do in your free time?'}, {'id': 'other_speaker', 'text': 'I play a lot of league of legends. I really enjoy it. What else do you enjoy?'}, {'id': 'model1', 'text': 'I have never heard of that game. I am more into video games than I am into games.'}, {'id': 'other_speaker', 'text': ""That's okay. I like video games a lot too. What are your favorite types of games?""}], 'speakers': ['model1', 'other_speaker']}","{'context': [], 'dialogue': [{'id': 'model2', 'text': 'Thank you! What do you like to do for fun? I like to go to the beach.'}, {'id': 'other_speaker', 'text': ""I love the beach! I've always wanted to go. What's your favorite place to go?""}, {'id': 'model2', 'text': ""I'd have to say my favorite place would have to be the atlantic ocean. What about you?""}, {'id': 'other_speaker', 'text': ""I love going to the gulf of mexico. It's such a beautiful place. Do you have any hobbies?""}, {'id': 'model2', 'text': ""Yes, I love to play with my dogs. They're my favorite. How about yourself?""}, {'id': 'other_speaker', 'text': 'I love dogs too! My favorite hobby is watching tv. What kind of dogs do you own?'}, {'id': 'model2', 'text': 'I have a poodle and a chihuahua. What is your favorite tv show?'}, {'id': 'other_speaker', 'text': 'My favorite show is game of thrones. I love it so much! How about yours?'}, {'id': 'model2', 'text': ""I haven't seen that one yet. I'll have to check it out. Have you seen the new season yet?""}, {'id': 'other_speaker', 'text': ""I have seen the first few episodes, but I don't know if I'm ready for it yet.""}], 'speakers': ['model2', 'other_speaker']}" -3,acute_eval_test,1,0,Who would you prefer to talk to for a long conversation?,1,0,-1,model2,model1,model1,model2,Makes more sense,False,model1__vs__model2,12,"{'is_onboarding': False, 'speakers_to_eval': ['model1', 'model2'], 'dialogue_ids': [2, 5], 'model1': {'context': [], 'dialogue': [{'id': 'model1', 'text': ""It's a public transportation system in new york city. I like to take it to and from work.""}, {'id': 'other_speaker', 'text': ""That's cool. I've never been to new york, but I'd love to visit one day.""}, {'id': 'model1', 'text': 'New york is a great place to visit. It is the most populous city in the united states.'}, {'id': 'other_speaker', 'text': ""I'll have to check it out some time. What else do you like to do in your spare time?""}, {'id': 'model1', 'text': 'I love to read. I also like to watch movies and play video games. What about you?'}, {'id': 'other_speaker', 'text': ""I love video games as well. I play a lot of call of duty. What's your favorite game?""}, {'id': 'model1', 'text': 'My favorite game is league of legends. Do you have a favorite champion? I love the champion select screen.'}, {'id': 'other_speaker', 'text': ""I like malphite. He's so fun to play. What other games do you play?""}, {'id': 'model1', 'text': 'I really like to play overwatch. I have been playing it since it came out in 2016.'}, {'id': 'other_speaker', 'text': ""I haven't played overwatch in a while. I used to play it when I was a kid.""}], 'speakers': ['model1', 'other_speaker']}, 'model2': {'context': [], 'dialogue': [{'id': 'model2', 'text': ""Wow, that's a long time. What do you like to do in your spare time?""}, {'id': 'other_speaker', 'text': 'I play guitar in my spare time. I also like to listen to music. What about you?'}, {'id': 'model2', 'text': 'I like music too. I like to create my own houses. Do you have any hobbies?'}, {'id': 'other_speaker', 'text': 'Yes, I love to read and play video games. What is your favorite thing to do for fun?'}, {'id': 'model2', 'text': ""I like playing video games as well. My favorite hobby is writing. What's your favorite book?""}, {'id': 'other_speaker', 'text': ""I don't really have a favorite book. I just like to read. What kind of writing do you do?""}, {'id': 'model2', 'text': ""I'm a fiction writer. I've written a few books in the past, but I haven't published any yet.""}, {'id': 'other_speaker', 'text': 'That sounds like a lot of fun. What type of fiction do you write? I like science fiction.'}, {'id': 'model2', 'text': 'I write sci - fi as well! I love science fiction as well as fantasy. What else do you enjoy?'}, {'id': 'other_speaker', 'text': 'I really enjoy reading science fiction books. I have been reading them since I was a little girl.'}], 'speakers': ['model2', 'other_speaker']}}","{'context': [], 'dialogue': [{'id': 'model2', 'text': ""Wow, that's a long time. What do you like to do in your spare time?""}, {'id': 'other_speaker', 'text': 'I play guitar in my spare time. I also like to listen to music. What about you?'}, {'id': 'model2', 'text': 'I like music too. I like to create my own houses. Do you have any hobbies?'}, {'id': 'other_speaker', 'text': 'Yes, I love to read and play video games. What is your favorite thing to do for fun?'}, {'id': 'model2', 'text': ""I like playing video games as well. My favorite hobby is writing. What's your favorite book?""}, {'id': 'other_speaker', 'text': ""I don't really have a favorite book. I just like to read. What kind of writing do you do?""}, {'id': 'model2', 'text': ""I'm a fiction writer. I've written a few books in the past, but I haven't published any yet.""}, {'id': 'other_speaker', 'text': 'That sounds like a lot of fun. What type of fiction do you write? I like science fiction.'}, {'id': 'model2', 'text': 'I write sci - fi as well! I love science fiction as well as fantasy. What else do you enjoy?'}, {'id': 'other_speaker', 'text': 'I really enjoy reading science fiction books. I have been reading them since I was a little girl.'}], 'speakers': ['model2', 'other_speaker']}","{'context': [], 'dialogue': [{'id': 'model1', 'text': ""It's a public transportation system in new york city. I like to take it to and from work.""}, {'id': 'other_speaker', 'text': ""That's cool. I've never been to new york, but I'd love to visit one day.""}, {'id': 'model1', 'text': 'New york is a great place to visit. It is the most populous city in the united states.'}, {'id': 'other_speaker', 'text': ""I'll have to check it out some time. What else do you like to do in your spare time?""}, {'id': 'model1', 'text': 'I love to read. I also like to watch movies and play video games. What about you?'}, {'id': 'other_speaker', 'text': ""I love video games as well. I play a lot of call of duty. What's your favorite game?""}, {'id': 'model1', 'text': 'My favorite game is league of legends. Do you have a favorite champion? I love the champion select screen.'}, {'id': 'other_speaker', 'text': ""I like malphite. He's so fun to play. What other games do you play?""}, {'id': 'model1', 'text': 'I really like to play overwatch. I have been playing it since it came out in 2016.'}, {'id': 'other_speaker', 'text': ""I haven't played overwatch in a while. I used to play it when I was a kid.""}], 'speakers': ['model1', 'other_speaker']}" +,run_id,worker,worker_name,time_taken,question,unit_id,task_start,correct,winner,loser,eval_choice_0,eval_choice_1,reason,is_onboarding,matchup,pairing_id,pairs_to_eval,winner_dialogue,loser_dialogue +0,acute_eval_test,1,MOCK_WORKER_0,0,Who would you prefer to talk to for a long conversation?,1,0,-1,model1,model2,model1,model2,Makes more sense,False,model1__vs__model2,45,"{'is_onboarding': False, 'speakers_to_eval': ['model1', 'model2'], 'dialogue_ids': [4, 7], 'model1': {'context': [], 'dialogue': [{'id': 'model1', 'text': ""I've never played tennis, but I do love watching it. Do you have any kids?""}, {'id': 'other_speaker', 'text': ""No, I don't have any children. What do you like to do in your spare time?""}, {'id': 'model1', 'text': ""I like to play tennis with my daughter. She's 3 and loves it. What about you?""}, {'id': 'other_speaker', 'text': 'I love to go to the mall and play with my friends. What is your favorite thing to do?'}, {'id': 'model1', 'text': 'I love the mall too! My favorite thing is selling paper products. What are your hobbies?'}, {'id': 'other_speaker', 'text': 'I really like to read and play video games. I also like to listen to music. You?'}, {'id': 'model1', 'text': 'I love video games too! I love listening to music too. What kind of video games do you play?'}, {'id': 'other_speaker', 'text': ""I play a lot of call of duty. I'm not very good at it, though.""}, {'id': 'model1', 'text': ""I used to play cod as a kid. It was fun. What's your favorite game?""}, {'id': 'other_speaker', 'text': 'My favorite game is probably black ops 3. I love that game so much! What is yours?'}], 'speakers': ['model1', 'other_speaker']}, 'model2': {'context': [], 'dialogue': [{'id': 'model2', 'text': ""I've never been to mexico, but I'd love to go. What do you do for a living?""}, {'id': 'other_speaker', 'text': ""I'm a stay at home mom. What about you? Do you have any hobbies?""}, {'id': 'model2', 'text': ""I don't have kids. I am a rapper, so I like to listen to music.""}, {'id': 'other_speaker', 'text': ""That's cool. What kind of music do you like to rap about? I like hip hop.""}, {'id': 'model2', 'text': 'I love hip hop as well. Frank ocean is my favorite. Do you listen to hip hop?'}, {'id': 'other_speaker', 'text': ""I love frank ocean. He's one of my favorite rappers. What else do you enjoy?""}, {'id': 'model2', 'text': 'I like a lot of different genres of music. I like rock, country, and country music.'}, {'id': 'other_speaker', 'text': ""I like country music too. I listen to it when I walk. It's so relaxing.""}, {'id': 'model2', 'text': 'I agree. I love listening to country music when I go to the beach. It is relaxing.'}, {'id': 'other_speaker', 'text': 'Do you live near the beach? I live in tx. We have the best beach music.'}], 'speakers': ['model2', 'other_speaker']}}","{'context': [], 'dialogue': [{'id': 'model1', 'text': ""I've never played tennis, but I do love watching it. Do you have any kids?""}, {'id': 'other_speaker', 'text': ""No, I don't have any children. What do you like to do in your spare time?""}, {'id': 'model1', 'text': ""I like to play tennis with my daughter. She's 3 and loves it. What about you?""}, {'id': 'other_speaker', 'text': 'I love to go to the mall and play with my friends. What is your favorite thing to do?'}, {'id': 'model1', 'text': 'I love the mall too! My favorite thing is selling paper products. What are your hobbies?'}, {'id': 'other_speaker', 'text': 'I really like to read and play video games. I also like to listen to music. You?'}, {'id': 'model1', 'text': 'I love video games too! I love listening to music too. What kind of video games do you play?'}, {'id': 'other_speaker', 'text': ""I play a lot of call of duty. I'm not very good at it, though.""}, {'id': 'model1', 'text': ""I used to play cod as a kid. It was fun. What's your favorite game?""}, {'id': 'other_speaker', 'text': 'My favorite game is probably black ops 3. I love that game so much! What is yours?'}], 'speakers': ['model1', 'other_speaker']}","{'context': [], 'dialogue': [{'id': 'model2', 'text': ""I've never been to mexico, but I'd love to go. What do you do for a living?""}, {'id': 'other_speaker', 'text': ""I'm a stay at home mom. What about you? Do you have any hobbies?""}, {'id': 'model2', 'text': ""I don't have kids. I am a rapper, so I like to listen to music.""}, {'id': 'other_speaker', 'text': ""That's cool. What kind of music do you like to rap about? I like hip hop.""}, {'id': 'model2', 'text': 'I love hip hop as well. Frank ocean is my favorite. Do you listen to hip hop?'}, {'id': 'other_speaker', 'text': ""I love frank ocean. He's one of my favorite rappers. What else do you enjoy?""}, {'id': 'model2', 'text': 'I like a lot of different genres of music. I like rock, country, and country music.'}, {'id': 'other_speaker', 'text': ""I like country music too. I listen to it when I walk. It's so relaxing.""}, {'id': 'model2', 'text': 'I agree. I love listening to country music when I go to the beach. It is relaxing.'}, {'id': 'other_speaker', 'text': 'Do you live near the beach? I live in tx. We have the best beach music.'}], 'speakers': ['model2', 'other_speaker']}" +1,acute_eval_test,1,MOCK_WORKER_0,0,Who would you prefer to talk to for a long conversation?,1,0,-1,model2,model1,model1,model2,Makes more sense,False,model1__vs__model2,62,"{'is_onboarding': False, 'speakers_to_eval': ['model1', 'model2'], 'dialogue_ids': [0, 6], 'model1': {'context': [], 'dialogue': [{'id': 'model1', 'text': ""That's interesting. I've never heard of that. Do you have any hobbies?""}, {'id': 'other_speaker', 'text': 'Yes, I love to go to the local park and hang out with friends. What about you?'}, {'id': 'model1', 'text': 'I love going to the park as well. I also like to take my iguana with me.'}, {'id': 'other_speaker', 'text': 'I love iguanas. I have one in my back yard. What do you like to do for fun?'}, {'id': 'model1', 'text': ""I really like to watch movies and play video games. What's your favorite thing to do?""}, {'id': 'other_speaker', 'text': 'I like playing video games too. I really enjoy watching netflix. What is your favorite movie?'}, {'id': 'model1', 'text': ""It's hard to pick just one. There are so many good ones out there. What are your favorite movies?""}, {'id': 'other_speaker', 'text': 'I like all kinds of movies. My favorite movie of all time is the godfather. Have you seen it?'}, {'id': 'model1', 'text': ""No, I haven't. I should check it out. What else do you enjoy doing?""}, {'id': 'other_speaker', 'text': ""I'm a big fan of hiking. I love the outdoors and the outdoors in general. You?""}], 'speakers': ['model1', 'other_speaker']}, 'model2': {'context': [], 'dialogue': [{'id': 'model2', 'text': 'I love to bake and bake cookies. What about you? Do you have any hobbies?'}, {'id': 'other_speaker', 'text': 'I enjoy writing poetry. I also like to listen to music. What kind of music do you like?'}, {'id': 'model2', 'text': 'I like all kinds of music. I listen to a lot of hip hop. Do you play any instruments?'}, {'id': 'other_speaker', 'text': ""I don't, but I've always wanted to learn to play the piano. What else do you enjoy?""}, {'id': 'model2', 'text': ""I really like to eat bagels. They're my favorite thing to eat. What's your favorite food?""}, {'id': 'other_speaker', 'text': 'I love bagels too! I like italian food the best. What is your favorite type of bagel?'}, {'id': 'model2', 'text': ""I'm not really a big fan of italian food. I'd have to say bagel with everything.""}, {'id': 'other_speaker', 'text': ""That's true. Bagels are good, but italian food is probably my favorite. Have you ever been to italy?""}, {'id': 'model2', 'text': ""No, I haven't. But I would love to go one day. How long have you been?""}, {'id': 'other_speaker', 'text': ""Since I was a kid. It's been a long time. I really want to go back.""}], 'speakers': ['model2', 'other_speaker']}}","{'context': [], 'dialogue': [{'id': 'model2', 'text': 'I love to bake and bake cookies. What about you? Do you have any hobbies?'}, {'id': 'other_speaker', 'text': 'I enjoy writing poetry. I also like to listen to music. What kind of music do you like?'}, {'id': 'model2', 'text': 'I like all kinds of music. I listen to a lot of hip hop. Do you play any instruments?'}, {'id': 'other_speaker', 'text': ""I don't, but I've always wanted to learn to play the piano. What else do you enjoy?""}, {'id': 'model2', 'text': ""I really like to eat bagels. They're my favorite thing to eat. What's your favorite food?""}, {'id': 'other_speaker', 'text': 'I love bagels too! I like italian food the best. What is your favorite type of bagel?'}, {'id': 'model2', 'text': ""I'm not really a big fan of italian food. I'd have to say bagel with everything.""}, {'id': 'other_speaker', 'text': ""That's true. Bagels are good, but italian food is probably my favorite. Have you ever been to italy?""}, {'id': 'model2', 'text': ""No, I haven't. But I would love to go one day. How long have you been?""}, {'id': 'other_speaker', 'text': ""Since I was a kid. It's been a long time. I really want to go back.""}], 'speakers': ['model2', 'other_speaker']}","{'context': [], 'dialogue': [{'id': 'model1', 'text': ""That's interesting. I've never heard of that. Do you have any hobbies?""}, {'id': 'other_speaker', 'text': 'Yes, I love to go to the local park and hang out with friends. What about you?'}, {'id': 'model1', 'text': 'I love going to the park as well. I also like to take my iguana with me.'}, {'id': 'other_speaker', 'text': 'I love iguanas. I have one in my back yard. What do you like to do for fun?'}, {'id': 'model1', 'text': ""I really like to watch movies and play video games. What's your favorite thing to do?""}, {'id': 'other_speaker', 'text': 'I like playing video games too. I really enjoy watching netflix. What is your favorite movie?'}, {'id': 'model1', 'text': ""It's hard to pick just one. There are so many good ones out there. What are your favorite movies?""}, {'id': 'other_speaker', 'text': 'I like all kinds of movies. My favorite movie of all time is the godfather. Have you seen it?'}, {'id': 'model1', 'text': ""No, I haven't. I should check it out. What else do you enjoy doing?""}, {'id': 'other_speaker', 'text': ""I'm a big fan of hiking. I love the outdoors and the outdoors in general. You?""}], 'speakers': ['model1', 'other_speaker']}" +2,acute_eval_test,1,MOCK_WORKER_0,0,Who would you prefer to talk to for a long conversation?,1,0,-1,model1,model2,model1,model2,Makes more sense,False,model1__vs__model2,19,"{'is_onboarding': False, 'speakers_to_eval': ['model1', 'model2'], 'dialogue_ids': [3, 9], 'model1': {'context': [], 'dialogue': [{'id': 'model1', 'text': ""I've never been to new york, but I'd love to visit one day. What do you do for fun?""}, {'id': 'other_speaker', 'text': 'I like to read and play video games. I also work at the newspaper. What about you?'}, {'id': 'model1', 'text': 'I enjoy playing video games as well. I work at a grocery store. What kind of newspaper do you work at?'}, {'id': 'other_speaker', 'text': ""It's a newspaper in new york city. Do you have any hobbies besides video games?""}, {'id': 'model1', 'text': 'Yes, I love to play games outdoors. What is your favorite genre of video games to play?'}, {'id': 'other_speaker', 'text': ""I like rpg's and action games. What's your favorite game to play outdoors?""}, {'id': 'model1', 'text': 'I really like to play call of duty. What other games do you like to do in your free time?'}, {'id': 'other_speaker', 'text': 'I play a lot of league of legends. I really enjoy it. What else do you enjoy?'}, {'id': 'model1', 'text': 'I have never heard of that game. I am more into video games than I am into games.'}, {'id': 'other_speaker', 'text': ""That's okay. I like video games a lot too. What are your favorite types of games?""}], 'speakers': ['model1', 'other_speaker']}, 'model2': {'context': [], 'dialogue': [{'id': 'model2', 'text': 'Thank you! What do you like to do for fun? I like to go to the beach.'}, {'id': 'other_speaker', 'text': ""I love the beach! I've always wanted to go. What's your favorite place to go?""}, {'id': 'model2', 'text': ""I'd have to say my favorite place would have to be the atlantic ocean. What about you?""}, {'id': 'other_speaker', 'text': ""I love going to the gulf of mexico. It's such a beautiful place. Do you have any hobbies?""}, {'id': 'model2', 'text': ""Yes, I love to play with my dogs. They're my favorite. How about yourself?""}, {'id': 'other_speaker', 'text': 'I love dogs too! My favorite hobby is watching tv. What kind of dogs do you own?'}, {'id': 'model2', 'text': 'I have a poodle and a chihuahua. What is your favorite tv show?'}, {'id': 'other_speaker', 'text': 'My favorite show is game of thrones. I love it so much! How about yours?'}, {'id': 'model2', 'text': ""I haven't seen that one yet. I'll have to check it out. Have you seen the new season yet?""}, {'id': 'other_speaker', 'text': ""I have seen the first few episodes, but I don't know if I'm ready for it yet.""}], 'speakers': ['model2', 'other_speaker']}}","{'context': [], 'dialogue': [{'id': 'model1', 'text': ""I've never been to new york, but I'd love to visit one day. What do you do for fun?""}, {'id': 'other_speaker', 'text': 'I like to read and play video games. I also work at the newspaper. What about you?'}, {'id': 'model1', 'text': 'I enjoy playing video games as well. I work at a grocery store. What kind of newspaper do you work at?'}, {'id': 'other_speaker', 'text': ""It's a newspaper in new york city. Do you have any hobbies besides video games?""}, {'id': 'model1', 'text': 'Yes, I love to play games outdoors. What is your favorite genre of video games to play?'}, {'id': 'other_speaker', 'text': ""I like rpg's and action games. What's your favorite game to play outdoors?""}, {'id': 'model1', 'text': 'I really like to play call of duty. What other games do you like to do in your free time?'}, {'id': 'other_speaker', 'text': 'I play a lot of league of legends. I really enjoy it. What else do you enjoy?'}, {'id': 'model1', 'text': 'I have never heard of that game. I am more into video games than I am into games.'}, {'id': 'other_speaker', 'text': ""That's okay. I like video games a lot too. What are your favorite types of games?""}], 'speakers': ['model1', 'other_speaker']}","{'context': [], 'dialogue': [{'id': 'model2', 'text': 'Thank you! What do you like to do for fun? I like to go to the beach.'}, {'id': 'other_speaker', 'text': ""I love the beach! I've always wanted to go. What's your favorite place to go?""}, {'id': 'model2', 'text': ""I'd have to say my favorite place would have to be the atlantic ocean. What about you?""}, {'id': 'other_speaker', 'text': ""I love going to the gulf of mexico. It's such a beautiful place. Do you have any hobbies?""}, {'id': 'model2', 'text': ""Yes, I love to play with my dogs. They're my favorite. How about yourself?""}, {'id': 'other_speaker', 'text': 'I love dogs too! My favorite hobby is watching tv. What kind of dogs do you own?'}, {'id': 'model2', 'text': 'I have a poodle and a chihuahua. What is your favorite tv show?'}, {'id': 'other_speaker', 'text': 'My favorite show is game of thrones. I love it so much! How about yours?'}, {'id': 'model2', 'text': ""I haven't seen that one yet. I'll have to check it out. Have you seen the new season yet?""}, {'id': 'other_speaker', 'text': ""I have seen the first few episodes, but I don't know if I'm ready for it yet.""}], 'speakers': ['model2', 'other_speaker']}" +3,acute_eval_test,1,MOCK_WORKER_0,0,Who would you prefer to talk to for a long conversation?,1,0,-1,model2,model1,model1,model2,Makes more sense,False,model1__vs__model2,12,"{'is_onboarding': False, 'speakers_to_eval': ['model1', 'model2'], 'dialogue_ids': [2, 5], 'model1': {'context': [], 'dialogue': [{'id': 'model1', 'text': ""It's a public transportation system in new york city. I like to take it to and from work.""}, {'id': 'other_speaker', 'text': ""That's cool. I've never been to new york, but I'd love to visit one day.""}, {'id': 'model1', 'text': 'New york is a great place to visit. It is the most populous city in the united states.'}, {'id': 'other_speaker', 'text': ""I'll have to check it out some time. What else do you like to do in your spare time?""}, {'id': 'model1', 'text': 'I love to read. I also like to watch movies and play video games. What about you?'}, {'id': 'other_speaker', 'text': ""I love video games as well. I play a lot of call of duty. What's your favorite game?""}, {'id': 'model1', 'text': 'My favorite game is league of legends. Do you have a favorite champion? I love the champion select screen.'}, {'id': 'other_speaker', 'text': ""I like malphite. He's so fun to play. What other games do you play?""}, {'id': 'model1', 'text': 'I really like to play overwatch. I have been playing it since it came out in 2016.'}, {'id': 'other_speaker', 'text': ""I haven't played overwatch in a while. I used to play it when I was a kid.""}], 'speakers': ['model1', 'other_speaker']}, 'model2': {'context': [], 'dialogue': [{'id': 'model2', 'text': ""Wow, that's a long time. What do you like to do in your spare time?""}, {'id': 'other_speaker', 'text': 'I play guitar in my spare time. I also like to listen to music. What about you?'}, {'id': 'model2', 'text': 'I like music too. I like to create my own houses. Do you have any hobbies?'}, {'id': 'other_speaker', 'text': 'Yes, I love to read and play video games. What is your favorite thing to do for fun?'}, {'id': 'model2', 'text': ""I like playing video games as well. My favorite hobby is writing. What's your favorite book?""}, {'id': 'other_speaker', 'text': ""I don't really have a favorite book. I just like to read. What kind of writing do you do?""}, {'id': 'model2', 'text': ""I'm a fiction writer. I've written a few books in the past, but I haven't published any yet.""}, {'id': 'other_speaker', 'text': 'That sounds like a lot of fun. What type of fiction do you write? I like science fiction.'}, {'id': 'model2', 'text': 'I write sci - fi as well! I love science fiction as well as fantasy. What else do you enjoy?'}, {'id': 'other_speaker', 'text': 'I really enjoy reading science fiction books. I have been reading them since I was a little girl.'}], 'speakers': ['model2', 'other_speaker']}}","{'context': [], 'dialogue': [{'id': 'model2', 'text': ""Wow, that's a long time. What do you like to do in your spare time?""}, {'id': 'other_speaker', 'text': 'I play guitar in my spare time. I also like to listen to music. What about you?'}, {'id': 'model2', 'text': 'I like music too. I like to create my own houses. Do you have any hobbies?'}, {'id': 'other_speaker', 'text': 'Yes, I love to read and play video games. What is your favorite thing to do for fun?'}, {'id': 'model2', 'text': ""I like playing video games as well. My favorite hobby is writing. What's your favorite book?""}, {'id': 'other_speaker', 'text': ""I don't really have a favorite book. I just like to read. What kind of writing do you do?""}, {'id': 'model2', 'text': ""I'm a fiction writer. I've written a few books in the past, but I haven't published any yet.""}, {'id': 'other_speaker', 'text': 'That sounds like a lot of fun. What type of fiction do you write? I like science fiction.'}, {'id': 'model2', 'text': 'I write sci - fi as well! I love science fiction as well as fantasy. What else do you enjoy?'}, {'id': 'other_speaker', 'text': 'I really enjoy reading science fiction books. I have been reading them since I was a little girl.'}], 'speakers': ['model2', 'other_speaker']}","{'context': [], 'dialogue': [{'id': 'model1', 'text': ""It's a public transportation system in new york city. I like to take it to and from work.""}, {'id': 'other_speaker', 'text': ""That's cool. I've never been to new york, but I'd love to visit one day.""}, {'id': 'model1', 'text': 'New york is a great place to visit. It is the most populous city in the united states.'}, {'id': 'other_speaker', 'text': ""I'll have to check it out some time. What else do you like to do in your spare time?""}, {'id': 'model1', 'text': 'I love to read. I also like to watch movies and play video games. What about you?'}, {'id': 'other_speaker', 'text': ""I love video games as well. I play a lot of call of duty. What's your favorite game?""}, {'id': 'model1', 'text': 'My favorite game is league of legends. Do you have a favorite champion? I love the champion select screen.'}, {'id': 'other_speaker', 'text': ""I like malphite. He's so fun to play. What other games do you play?""}, {'id': 'model1', 'text': 'I really like to play overwatch. I have been playing it since it came out in 2016.'}, {'id': 'other_speaker', 'text': ""I haven't played overwatch in a while. I used to play it when I was a kid.""}], 'speakers': ['model1', 'other_speaker']}"