Skip to content
This repository has been archived by the owner on Nov 3, 2023. It is now read-only.

[teacher] MetaLWoZ #3583

Merged
merged 5 commits into from
Apr 21, 2021
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Add to task_list
  • Loading branch information
stephenroller committed Apr 12, 2021
commit fd64b0904a1129e577127b1341ddd0b95b290ce9
5 changes: 5 additions & 0 deletions parlai/tasks/metalwoz/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
#!/usr/bin/env python3

# Copyright (c) Facebook, Inc. and its affiliates.
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
72 changes: 72 additions & 0 deletions parlai/tasks/metalwoz/agents.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
#!/usr/bin/env python3

# Copyright (c) Facebook, Inc. and its affiliates.
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

from parlai.core.teachers import DialogTeacher
from parlai.utils.io import PathManager
from parlai.utils.data import DatatypeHelper
from .build import build
import os
import pandas as pd


class MetalWozTeacher(DialogTeacher):
def _path(self, opt):
fold = DatatypeHelper.fold(opt['datatype'])
folder = os.path.join(opt['datapath'], 'metalwoz')
return folder, fold

def __init__(self, opt, shared=None):
if shared is None:
build(opt)
folder, fold = self._path(opt)
self.fold = fold
opt['datafile'] = os.path.join(folder, fold)
super().__init__(opt, shared)

def setup_data(self, datapath):
folder, fold = os.path.split(datapath)
with PathManager.open(os.path.join(folder, 'tasks.txt')) as taskf:
tasks_table = pd.read_json(taskf, lines=True)

dfolder = os.path.join(folder, 'dialogues')

data = []

for filename in PathManager.ls(dfolder):
fullfn = os.path.join(dfolder, filename)
with PathManager.open(fullfn) as dataf:
data.append(pd.read_json(dataf, lines=True))

data = pd.concat(data, axis=0)
data = data.sample(frac=1.0, random_state=83741) # metal in l33t numbers
data['fold'] = data['id'].apply(lambda x: (int(x, 16) * 13) % 10)
data = data.merge(tasks_table, on='task_id')

for _, row in data.iterrows():
if fold == 'test' and row['fold'] != 9:
continue
if fold == 'valid' and row['fold'] != 8:
continue
if fold == 'train' and row['fold'] >= 8:
continue
texts = [row['bot_role']] + list(row['turns'])
prompts, labels = texts[::2], texts[1::2]
for i, (prompt, label) in enumerate(zip(prompts, labels)):
yield {
'text': prompt,
'label': label,
'bot_role': row['bot_role'],
'bot_prompt': row['bot_prompt'],
'user_role': row['user_role'],
'user_prompt': row['user_prompt'],
'utterance_id': row['id'],
'domain': row['domain_x'],
'task_id': row['task_id'],
}, i == 0


class DefaultTeacher(MetalWozTeacher):
pass
34 changes: 34 additions & 0 deletions parlai/tasks/metalwoz/build.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
#!/usr/bin/env python3

# Copyright (c) Facebook, Inc. and its affiliates.
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

import parlai.core.build_data as build_data
import os
from parlai.core.build_data import DownloadableFile

RESOURCES = [
DownloadableFile(
'https://download.microsoft.com/download/E/B/8/EB84CB1A-D57D-455F-B905-3ABDE80404E5/metalwoz-v1.zip',
'metalwoz-v1.zip',
'2a2ae3b25760aa2725e70bc6480562fa5d720c9689a508d28417631496d6764f',
)
]


def build(opt):
dpath = os.path.join(opt['datapath'], 'metalwoz')
version = '1.0'

if not build_data.built(dpath, version_string=version):
if build_data.built(dpath):
build_data.remove_dir(dpath)
build_data.make_dir(dpath)
build_data.make_dir(os.path.join(dpath, 'dialogues'))

# Download the data.
for downloadable_file in RESOURCES:
downloadable_file.download_file(dpath)

build_data.mark_done(dpath, version_string=version)
11 changes: 11 additions & 0 deletions parlai/tasks/metalwoz/test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
#!/usr/bin/env python3

# Copyright (c) Facebook, Inc. and its affiliates.
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

from parlai.utils.testing import AutoTeacherTest


class TestDefaultTeacher(AutoTeacherTest):
task = "metalwoz"
69 changes: 69 additions & 0 deletions parlai/tasks/metalwoz/test/metalwoz_test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
acts:
- - bot_prompt: Tell the user that there are no ski hills in their immediate location
bot_role: You are a bot that helps people book skiing trips
domain: SKI_BOT
episode_done: false
eval_labels:
- Hello how may I help you?
id: metalwoz
task_id: 2511cf64
text: You are a bot that helps people book skiing trips
user_prompt: You want to know if there are good ski hills an hour's drive from
your current location
user_role: You are interacting with a bot designed to help you book a skiing trip
utterance_id: f04197d9
- - bot_prompt: Tell the user that there are no ski hills in their immediate location
bot_role: You are a bot that helps people book skiing trips
domain: SKI_BOT
episode_done: false
eval_labels:
- I can Help you with that. Looking in the database, no. There are none near your
location
id: metalwoz
task_id: 2511cf64
text: I need to know if there are some skiing hills about an hour from me.
user_prompt: You want to know if there are good ski hills an hour's drive from
your current location
user_role: You are interacting with a bot designed to help you book a skiing trip
utterance_id: f04197d9
- - bot_prompt: Tell the user that there are no ski hills in their immediate location
bot_role: You are a bot that helps people book skiing trips
domain: SKI_BOT
episode_done: false
eval_labels:
- Would you like the closest one?
id: metalwoz
task_id: 2511cf64
text: Ok, dang
user_prompt: You want to know if there are good ski hills an hour's drive from
your current location
user_role: You are interacting with a bot designed to help you book a skiing trip
utterance_id: f04197d9
- - bot_prompt: Tell the user that there are no ski hills in their immediate location
bot_role: You are a bot that helps people book skiing trips
domain: SKI_BOT
episode_done: false
eval_labels:
- The closest one is 30 miles north, in Duluth.
id: metalwoz
task_id: 2511cf64
text: What is the closest one?
user_prompt: You want to know if there are good ski hills an hour's drive from
your current location
user_role: You are interacting with a bot designed to help you book a skiing trip
utterance_id: f04197d9
- - bot_prompt: Tell the user that there are no ski hills in their immediate location
bot_role: You are a bot that helps people book skiing trips
domain: SKI_BOT
episode_done: false
eval_labels:
- I can do that now.For how many?
id: metalwoz
task_id: 2511cf64
text: Ok great, I would like to book a trip to duluth then
user_prompt: You want to know if there are good ski hills an hour's drive from
your current location
user_role: You are interacting with a bot designed to help you book a skiing trip
utterance_id: f04197d9
num_episodes: 3891
num_examples: 23806
68 changes: 68 additions & 0 deletions parlai/tasks/metalwoz/test/metalwoz_train.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
acts:
- - bot_prompt: Tell the user that there are no ski hills in their immediate location
bot_role: You are a bot that helps people book skiing trips
domain: SKI_BOT
episode_done: false
id: metalwoz
labels:
- Hello how may I help you?
task_id: 2511cf64
text: You are a bot that helps people book skiing trips
user_prompt: You want to know if there are good ski hills an hour's drive from
your current location
user_role: You are interacting with a bot designed to help you book a skiing trip
utterance_id: c3ec2179
- - bot_prompt: Tell the user that there are no ski hills in their immediate location
bot_role: You are a bot that helps people book skiing trips
domain: SKI_BOT
episode_done: false
id: metalwoz
labels:
- There are no ski hills in your location
task_id: 2511cf64
text: Are there any ski resorts me?
user_prompt: You want to know if there are good ski hills an hour's drive from
your current location
user_role: You are interacting with a bot designed to help you book a skiing trip
utterance_id: c3ec2179
- - bot_prompt: Tell the user that there are no ski hills in their immediate location
bot_role: You are a bot that helps people book skiing trips
domain: SKI_BOT
episode_done: false
id: metalwoz
labels:
- In the mount le
task_id: 2511cf64
text: What's the nearest ski resort?
user_prompt: You want to know if there are good ski hills an hour's drive from
your current location
user_role: You are interacting with a bot designed to help you book a skiing trip
utterance_id: c3ec2179
- - bot_prompt: Tell the user that there are no ski hills in their immediate location
bot_role: You are a bot that helps people book skiing trips
domain: SKI_BOT
episode_done: false
id: metalwoz
labels:
- 4 hrs
task_id: 2511cf64
text: How many hours way is that from me?
user_prompt: You want to know if there are good ski hills an hour's drive from
your current location
user_role: You are interacting with a bot designed to help you book a skiing trip
utterance_id: c3ec2179
- - bot_prompt: Tell the user that there are no ski hills in their immediate location
bot_role: You are a bot that helps people book skiing trips
domain: SKI_BOT
episode_done: false
id: metalwoz
labels:
- is there anything else?
task_id: 2511cf64
text: Okay thanks
user_prompt: You want to know if there are good ski hills an hour's drive from
your current location
user_role: You are interacting with a bot designed to help you book a skiing trip
utterance_id: c3ec2179
num_episodes: 30240
num_examples: 185086
68 changes: 68 additions & 0 deletions parlai/tasks/metalwoz/test/metalwoz_valid.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
acts:
- - bot_prompt: Tell the user that there are no ski hills in their immediate location
bot_role: You are a bot that helps people book skiing trips
domain: SKI_BOT
episode_done: false
eval_labels:
- Hello how may I help you?
id: metalwoz
task_id: 2511cf64
text: You are a bot that helps people book skiing trips
user_prompt: You want to know if there are good ski hills an hour's drive from
your current location
user_role: You are interacting with a bot designed to help you book a skiing trip
utterance_id: 111daec4
- - bot_prompt: Tell the user that there are no ski hills in their immediate location
bot_role: You are a bot that helps people book skiing trips
domain: SKI_BOT
episode_done: false
eval_labels:
- 'Yes'
id: metalwoz
task_id: 2511cf64
text: can you help me about booking?
user_prompt: You want to know if there are good ski hills an hour's drive from
your current location
user_role: You are interacting with a bot designed to help you book a skiing trip
utterance_id: 111daec4
- - bot_prompt: Tell the user that there are no ski hills in their immediate location
bot_role: You are a bot that helps people book skiing trips
domain: SKI_BOT
episode_done: false
eval_labels:
- Ok, where to?
id: metalwoz
task_id: 2511cf64
text: I need to book a skii trip
user_prompt: You want to know if there are good ski hills an hour's drive from
your current location
user_role: You are interacting with a bot designed to help you book a skiing trip
utterance_id: 111daec4
- - bot_prompt: Tell the user that there are no ski hills in their immediate location
bot_role: You are a bot that helps people book skiing trips
domain: SKI_BOT
episode_done: false
eval_labels:
- There are not
id: metalwoz
task_id: 2511cf64
text: Is there any good ski hills an hour drive from my place?
user_prompt: You want to know if there are good ski hills an hour's drive from
your current location
user_role: You are interacting with a bot designed to help you book a skiing trip
utterance_id: 111daec4
- - bot_prompt: Tell the user that there are no ski hills in their immediate location
bot_role: You are a bot that helps people book skiing trips
domain: SKI_BOT
episode_done: false
eval_labels:
- Nothing in that immediate location, sorry
id: metalwoz
task_id: 2511cf64
text: I am from Salt Lake can you double check?
user_prompt: You want to know if there are good ski hills an hour's drive from
your current location
user_role: You are interacting with a bot designed to help you book a skiing trip
utterance_id: 111daec4
num_episodes: 3753
num_examples: 22977
14 changes: 14 additions & 0 deletions parlai/tasks/task_list.py
Original file line number Diff line number Diff line change
@@ -1350,4 +1350,18 @@
"description": "Task for detect whether the last utterance contradicts previous dialogue history.",
"links": {"arXiv": "https://arxiv.org/abs/2012.13391"},
},
{
"id": "metalwoz",
"display_name": "MetaLWOz",
"task": "metalwoz",
"tags": ["Goal"],
"description": (
"Meta-Learning Wizard-of-Oz (MetaLWOz) is a dataset designed to help "
"develop models capable of predicting user responses in unseen domains."
),
"links": {
"paper": "http://workshop.colips.org/dstc7/dstc8/DTSC8_multidomain_task_proposal.pdf",
"website": "https://www.microsoft.com/en-us/research/project/metalwoz/",
},
},
]