Multilight models

parlai/zoo/model_list.py

@@ -4,6 +4,8 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
+# flake8: noqa
+
 """
 The Model Zoo.
 
@@ -1424,8 +1426,15 @@
         "agent": "projects.style_gen.classifier:ClassifierAgent",
         "task": "style_gen:LabeledBlendedSkillTalk",
         "project": 'https://github.com/facebookresearch/ParlAI/tree/main/projects/style_gen',
-        "description": "Classifier trained on Image-Chat turns 2 and 3 to classify the personality of an example given that utterance as the sole context.",
-        "example": "parlai eval_model --task style_gen:CurrUttOnlyStyle --wrapper-task style_gen:LabeledBlendedSkillTalk --model-file zoo:style_gen/curr_only_classifier/model --model projects.style_gen.classifier:ClassifierAgent --classes-from-file image_chat_personalities_file",
+        "description": (
+            "Classifier trained on Image-Chat turns 2 and 3 to classify the personality"
+            " of an example given that utterance as the sole context."
+        ),
+        "example": """
+        parlai eval_model --task style_gen:CurrUttOnlyStyle --wrapper-task style_gen:LabeledBlendedSkillTalk \
+            --model-file zoo:style_gen/curr_only_classifier/model \
+            --model projects.style_gen.classifier:ClassifierAgent --classes-from-file image_chat_personalities_file
+        """,
         "result": """
 16:46:41 | Finished evaluating tasks ['style_gen:CurrUttOnlyStyle'] using datatype valid
     accuracy  bleu-4  <PER_CLASS_METRICS_SNIPPED>  clen  ctpb  ctps  ctrunc  ctrunclen  exps  exs    f1  gpu_mem  llen  loss    lr  ltpb  ltps  ltrunc  ltrunclen   tpb   tps  \
@@ -3060,4 +3069,68 @@
             [Bart]: I am an AI researcher. I love my job.
         """,
     },
+    {
+        "title": "Multi-party speaker prediction",
+        "id": "multilight",
+        "path": "zoo:multilight/speaker/model",
+        "agent": "bart",
+        "task": "light_multiparty:SpeakerPrediction",
+        "project": "https://parl.ai/projects/multilight",
+        "description": (
+            "Predicts the most plausible next speaker at any point during a three-player conversation in LIGHT."
+        ),
+        "example": (
+            """
+            parlai eval_model --model-file zoo:multilight/speaker/model --task light_multiparty:SpeakerPrediction \
+                --add-location-to-context true --add-personas-to-context true --include-speaker-in-label false
+            """
+        ),
+        "result": """
+            Report for light_multiparty:SpeakerPrediction:
+            clen  ctpb  ctps  ctrunc  ctrunclen  exps   exs  gpu_mem  llen  loss    lr  ltpb  ltps  ltrunc  ltrunclen  ppl  token_acc  token_em   tpb  tps
+            473.5 420.2  6130   .4140      54.44 14.59 11005    .1218 3.091 .3438 5e-10 3.091 45.09       0          0 1.41      .8462     .5256 423.3 6175
+            """,  # noqa: E501
+    },
+    {
+        "title": "Multi-party utterance only 3B",
+        "id": "multilight",
+        "path": "zoo:multilight/utterance_3B/model",
+        "agent": "bart",
+        "task": "light_multiparty",
+        "project": "https://parl.ai/projects/multilight",
+        "description": ("Utterance generation model for the multi-party LIGHT game."),
+        "example": (
+            """
+            parlai eval_model --model-file zoo:multilight/utterance_3B/model --task light_multiparty \
+                --add-location-to-context true --add-personas-to-context true --include-speaker-in-label false --add-speaker-to-context-end true
+            """
+        ),
+        "result": """
+            Report for light_multiparty:
+            clen  ctpb  ctps  ctrunc  ctrunclen  exps   exs  gpu_mem  llen  loss    lr  ltpb  ltps  ltrunc  ltrunclen   ppl  token_acc  token_em  tpb  tps
+            478.6 423.2  5603   .4249      56.58 13.24 11005    .7931 15.83 2.584 5e-07 15.83 209.6       0          0 13.25      .4308         0  439 5813
+            """,  # noqa: E501
+    },
+    {
+        "title": "Multi-party utterance only 400m",
+        "id": "multilight",
+        "path": "zoo:multilight/utterance_400m/model",
+        "agent": "bart",
+        "task": "light_multiparty",
+        "project": "https://parl.ai/projects/multilight",
+        "description": (
+            "Utterance generation model for the multi-party LIGHT game. This is the smaller version of the original 3B model presented in the paper."
+        ),
+        "example": (
+            """
+            parlai eval_model --model-file zoo:multilight/utterance_400m/model --task light_multiparty \
+                --add-location-to-context true --add-personas-to-context true --include-speaker-in-label false --add-speaker-to-context-end true
+            """
+        ),
+        "result": """
+            Report for light_multiparty:
+            clen  ctpb  ctps  ctrunc  ctrunclen  exps   exs  gpu_mem  llen  loss    lr  ltpb  ltps  ltrunc  ltrunclen   ppl  token_acc  token_em   tpb  tps
+            477.6 422.6  7264   .4222      56.16 17.19 11005    .1204 15.83 2.714 1e-06 15.83 272.1       0          0 15.08      .4140         0 438.4 7536
+            """,  # noqa: E501
+    },
 ]

parlai/zoo/multilight/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python3
+
+# Copyright (c) Facebook, Inc. and its affiliates.
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.

parlai/zoo/multilight/speaker.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+
+# Copyright (c) Facebook, Inc. and its affiliates.
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+"""
+The speaker prediction model for MultiLIGHT.
+"""
+
+
+from parlai.zoo.multilight.utils import maybe_download
+
+MODEL_NAME = 'speaker'
+
+
+def download(datapath):
+    maybe_download(MODEL_NAME, datapath)

parlai/zoo/multilight/utils.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+
+# Copyright (c) Facebook, Inc. and its affiliates.
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import os
+from parlai.core.build_data import built, download_models, get_model_dir
+
+PROJECT_NAME = 'multilight'
+VERSION = 'v1.0'
+
+
+def maybe_download(model_type, datapath):
+    ddir = os.path.join(get_model_dir(datapath), PROJECT_NAME)
+    if not built(os.path.join(ddir, model_type), VERSION):
+        opt = {'datapath': datapath, 'model_type': model_type}
+        fnames = ['model.tar.gz']
+        download_models(opt, fnames, PROJECT_NAME, version=VERSION, use_model_type=True)

parlai/zoo/multilight/utterance_3B.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+
+# Copyright (c) Facebook, Inc. and its affiliates.
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+"""
+Utterance generations model for multi-player light (multilight dataset).
+
+2.7B parameter
+"""
+
+
+from parlai.zoo.multilight.utils import maybe_download
+
+MODEL_NAME = 'utterance_3B'
+
+
+def download(datapath):
+    maybe_download(MODEL_NAME, datapath)

parlai/zoo/multilight/utterance_400m.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+
+# Copyright (c) Facebook, Inc. and its affiliates.
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+"""
+Utterance generations model for multi-player light (multilight dataset).
+
+400m parameter
+"""
+
+
+from parlai.zoo.multilight.utils import maybe_download
+
+MODEL_NAME = 'utterance_400m'
+
+
+def download(datapath):
+    maybe_download(MODEL_NAME, datapath)

projects/multilight/README.md

@@ -0,0 +1,60 @@
+# Multi-Party Chat: Conversational Agents in Group Settings with Humans and Models
+
+Jimmy Wei, Kurt Shuster, Arthur Szlam, Jason Weston, Jack Urbanek, Mojtaba Komeili
+
+## Abstract
+
+Current dialogue research primarily studies pairwise (two-party) conversations, and does not address the everyday setting where more than two speakers converse together. In this work, we both collect and evaluate multi-party conversations to study this more general case. We use the LIGHT environment to construct grounded conversations, where each participant has an assigned character to role-play. We thus evaluate the ability of language models to act as one or more characters in such conversations. Models require two skills that pairwise-trained models appear to lack: (1) being able to decide when to talk; (2) producing coherent utterances grounded on multiple characters. We compare models trained on our new dataset to existing pairwise-trained dialogue models, as well as large language models with few-shot prompting. We find that our new dataset, MultiLIGHT, which we will publicly release, can help bring significant improvements in the group setting.
+
+## Paper
+
+[Link](https://arxiv.org/abs/2304.13835)
+
+
+## Data
+
+There are three types of teachers in this task. Two of them focus on the utterances and conversation content, the other one on predicting the next speaker.
+
+1. Utterances from all the characters:
+```.sh
+parlai dd -t light_multiparty --add-location-to-context true --add-personas-to-context true
+```
+
+2. Utterances from a single character only:
+```.sh
+parlai dd -t light_multiparty:FirstSpeakerTeacher --add-location-to-context true --add-personas-to-context true
+```
+
+3. Predicting the speaker for the next turn:
+```.sh
+parlai dd -t light_multiparty:SpeakerPredictionTeacher --add-location-to-context true --add-personas-to-context true
+```
+
+See the [agent](https://github.com/facebookresearch/ParlAI/blob/main/parlai/tasks/light_multiparty/agents.py) for up-to-date details on available teachers and flags.
+
+### Rendered sample
+<p align="center"><img width="50%" src="DatasetExample.png" /></p>
+
+## Models
+
+We released three models from this project:
+
+* `zoo:multilight/utterance_3B/model`: the best performing *Utterance only* model trained on LIGHT, LIGHT Wild and MultiLIGHT, multi-tasked (3B parameters size).
+* `zoo:multilight/utterance_400m/model`: the best performing *Utterance only* model trained on LIGHT, LIGHT Wild and MultiLIGHT, multi-tasked (400m parameters size).
+* `zoo:multilight/speaker/model`: predicts the next speaker.
+
+`zoo:multilight/utterance_3B/model` and `zoo:multilight/speaker/model` were used in our human evals as reported in the paper.
+
+### Running models
+
+You can run these models with the existing ParlAI dataset, for example
+```.sh
+parlai eval_model -mf zoo:multilight/utterance_3B/model \
+--task light_multiparty --add-location-to-context true --add-personas-to-context true
+```
+
+Utterance models performance on all character teachers:
+| utterance model                     | PPL   |
+|-------------------------------------|-------|
+| zoo:multilight/utterance_3B/model   | 13.25 |
+| zoo:multilight/utterance_400m/model | 15.08 |