From b2debccee1cee36a535df2bc3539d1dc6bcab23f Mon Sep 17 00:00:00 2001
From: Stephen Roller <roller@fb.com>
Date: Tue, 30 Mar 2021 17:02:49 -0400
Subject: [PATCH 1/7] Opt presets

---
 .gitignore                          |  6 +-----
 docs/Makefile                       |  1 +
 docs/source/generate_opt_presets.py | 26 +++++++++++++++++++++++
 docs/source/index.md                |  1 +
 docs/source/opt_presets.md          | 25 ++++++++++++++++++++++
 parlai/core/opt.py                  |  8 ++++++--
 parlai/core/params.py               | 32 +++++++++++++++++++++++++++--
 parlai/opt_presets/README.md        | 19 +++++++++++++++++
 parlai/opt_presets/__init__.py      |  5 +++++
 parlai/opt_presets/docs.py          | 20 ++++++++++++++++++
 parlai/opt_presets/gen/meena.opt    |  5 +++++
 setup.py                            |  2 +-
 12 files changed, 140 insertions(+), 10 deletions(-)
 create mode 100755 docs/source/generate_opt_presets.py
 create mode 100644 docs/source/opt_presets.md
 create mode 100644 parlai/opt_presets/README.md
 create mode 100644 parlai/opt_presets/__init__.py
 create mode 100644 parlai/opt_presets/docs.py
 create mode 100644 parlai/opt_presets/gen/meena.opt

diff --git a/.gitignore b/.gitignore
index 124cd8c6d9c..87dfb93b432 100644
--- a/.gitignore
+++ b/.gitignore
@@ -83,11 +83,7 @@ instance/
 # Sphinx documentation
 docs/_build/
 # autogenerated files
-docs/source/task_list.inc
-docs/source/zoo_list.inc
-docs/source/cli_usage.inc
-docs/source/cli_advanced.inc
-docs/source/mutators_list.inc
+docs/source/*.inc
 docs/source/agent_refs
 
 # PyBuilder
diff --git a/docs/Makefile b/docs/Makefile
index ac53a3995c1..73e9a8725ea 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -34,4 +34,5 @@ clean:
 	cd "${SOURCEDIR}"; python generate_mutator_list.py
 	cd "${SOURCEDIR}"; python generate_metric_list.py
 	cd "${SOURCEDIR}"; python generate_cli.py
+	cd "${SOURCEDIR}"; python generate_opt_presets.py
 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
diff --git a/docs/source/generate_opt_presets.py b/docs/source/generate_opt_presets.py
new file mode 100755
index 00000000000..9314399632c
--- /dev/null
+++ b/docs/source/generate_opt_presets.py
@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+
+# Copyright (c) Facebook, Inc. and its affiliates.
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import json
+import pkg_resources
+
+from parlai.opt_presets.docs import PRESET_DESCRIPTIONS
+
+fout = open('opt_presets_list.inc', 'w')
+fout.write('Preset name | Description | Expansion\n')
+fout.write('----------- | ----------- | ---------\n')
+for alias in sorted(PRESET_DESCRIPTIONS.keys()):
+    description = PRESET_DESCRIPTIONS[alias]
+    with pkg_resources.resource_stream("parlai", f"opt_presets/{alias}.opt") as f:
+        expansion = json.load(f)
+        assert isinstance(expansion, dict)
+        expansion_str = []
+        for key, value in expansion.items():
+            key = '--' + key.replace('_', '-')
+            expansion_str.append(f'`{key} {value}`')
+        expansion_str = " ".join(expansion_str)
+        fout.write(f'`{alias}` | {description} | {expansion_str}')
+fout.close()
diff --git a/docs/source/index.md b/docs/source/index.md
index 7822ee4b984..3a13565d6a1 100644
--- a/docs/source/index.md
+++ b/docs/source/index.md
@@ -45,6 +45,7 @@ zoo
 cli_usage
 cli_advanced
 cli_custom
+opt_presets
 ```
 
 ```{toctree}
diff --git a/docs/source/opt_presets.md b/docs/source/opt_presets.md
new file mode 100644
index 00000000000..f76ecb03294
--- /dev/null
+++ b/docs/source/opt_presets.md
@@ -0,0 +1,25 @@
+# Opt Presets
+
+Opt presets are a way to provide multiple options on the command line as
+shorthand. Opt presets are bundled with ParlAI and may be used by simply
+invoking the `-o preset_name` option within any ParlAI command.
+
+You may also define your own options by placing them in `~/.parlai/opt_presets/`.
+For example, creating `~/.parlai/opt_presets/myfolder/mypreset.opt` allows you to
+invoke it via `-o myfolder/mypreset`. These preset files are simple json files
+containing a dictionary of files. For example:
+
+```js
+{
+    "inference": "beam",
+    "beam_size": 10,
+}
+```
+
+## List of presets
+
+The following is a list of all options presets bundled with the latest version
+of ParlAI.
+
+```{include} opt_presets_list.inc
+```
diff --git a/parlai/core/opt.py b/parlai/core/opt.py
index 7192d17e3ae..84a958d6847 100644
--- a/parlai/core/opt.py
+++ b/parlai/core/opt.py
@@ -12,9 +12,10 @@
 import json
 import pickle
 import traceback
+import io
 import parlai.utils.logging as logging
 
-from typing import List
+from typing import List, Union
 
 from parlai.utils.io import PathManager
 
@@ -119,10 +120,13 @@ def save(self, filename: str) -> None:
             f.write('\n')
 
     @classmethod
-    def load(cls, optfile: str) -> 'Opt':
+    def load(cls, optfile: Union[str, io.IOBase]) -> 'Opt':
         """
         Load an Opt from disk.
         """
+        if isinstance(optfile, io.IOBase):
+            with optfile:
+                return json.load(optfile)
         try:
             # try json first
             with PathManager.open(optfile, 'r', encoding='utf-8') as t_handle:
diff --git a/parlai/core/params.py b/parlai/core/params.py
index e0d35b85cf6..0b0acc18cf9 100644
--- a/parlai/core/params.py
+++ b/parlai/core/params.py
@@ -13,6 +13,7 @@
 import sys as _sys
 import datetime
 import parlai
+import pkg_resources
 
 try:
     import git
@@ -977,6 +978,33 @@ def parse_known_args(self, args=None, namespace=None, nohelp=False):
             args = [a for a in args if a != '-h' and a != '--help' and a != '--helpall']
         return super().parse_known_args(args, namespace)
 
+    def _find_opt_file(self, optfile: str):
+        """
+        Look for an opt file as an argument to --init-opt.
+
+        First looks for a particular existing file on disk, then checks the
+        folder of option aliases.
+
+        May return a filename or a streaming file of an Opt.
+        """
+        oa_filename = os.path.join("opt_presets", optfile + ".opt")
+        user_filename = os.path.join(os.path.expanduser(f"~/.parlai"), oa_filename)
+        if PathManager.exists(optfile):
+            return optfile
+        elif PathManager.exists(user_filename):
+            # use a user's custom opt preset
+            return user_filename
+        elif pkg_resources.resource_exists("parlai", oa_filename):
+            # Maybe a bundled opt preset
+            return pkg_resources.resource_stream("parlai", oa_filename)
+        else:
+            raise FileNotFoundError(
+                f"Could not find filename or opt preset {optfile}. "
+                "Opt presets should not have the '.opt' suffix. Please "
+                "check https://parl.ai/docs/opt_presets.html for a list "
+                "of available opt presets."
+            )
+
     def _load_known_opts(self, optfile, parsed):
         """
         Pull in CLI args for proper models/tasks/etc.
@@ -984,7 +1012,7 @@ def _load_known_opts(self, optfile, parsed):
         Called before args are parsed; ``_load_opts`` is used for actually overriding
         opts after they are parsed.
         """
-        new_opt = Opt.load(optfile)
+        new_opt = Opt.load(self._find_opt_file(optfile))
         for key, value in new_opt.items():
             # existing command line parameters take priority.
             if key not in parsed or parsed[key] is None:
@@ -992,7 +1020,7 @@ def _load_known_opts(self, optfile, parsed):
 
     def _load_opts(self, opt):
         optfile = opt.get('init_opt')
-        new_opt = Opt.load(optfile)
+        new_opt = Opt.load(self._find_opt_file(optfile))
         for key, value in new_opt.items():
             # existing command line parameters take priority.
             if key not in opt:
diff --git a/parlai/opt_presets/README.md b/parlai/opt_presets/README.md
new file mode 100644
index 00000000000..ac983dc8274
--- /dev/null
+++ b/parlai/opt_presets/README.md
@@ -0,0 +1,19 @@
+# Option Aliases
+
+This folder contains a set of "option aliases" that are automatically packaged
+and provided with ParlAI. They are used as shorthand for 
+
+## Adding option aliases
+
+Simply adding `.opt` file to this folder is enough to add an alias. The
+directory structure is respected, so `parlai/options/myfolder/myalias.opt` can
+be invoked using `-o myfolder/myalias`.
+
+For quality assurance purposes, we request new option aliases respect the
+following conventions:
+
+- Only include the very minimum number of options you wish to specify with this
+  alias.
+- Pipe your opt file through `jq -S .` so keys are always in alphabetical order.
+- Add an entry to "docs.py" briefly describing your option. The full expansion
+  will be automatically rendered, but a human-friendly summary should be added.
diff --git a/parlai/opt_presets/__init__.py b/parlai/opt_presets/__init__.py
new file mode 100644
index 00000000000..240697e3247
--- /dev/null
+++ b/parlai/opt_presets/__init__.py
@@ -0,0 +1,5 @@
+#!/usr/bin/env python3
+
+# Copyright (c) Facebook, Inc. and its affiliates.
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
diff --git a/parlai/opt_presets/docs.py b/parlai/opt_presets/docs.py
new file mode 100644
index 00000000000..db65dda2bae
--- /dev/null
+++ b/parlai/opt_presets/docs.py
@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+
+# Copyright (c) Facebook, Inc. and its affiliates.
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+List of all opt presets distributed with ParlAI.
+
+This file is for automatically generating docs.
+"""
+
+# PRESET_DESCRIPTIONS is a dictionary mapping alias names to human descriptions
+# for the sake of documentation
+PRESET_DESCRIPTIONS = {
+    "gen/meena": (
+        "Inference parameters for the Sample & Rank procedure of Meena. "
+        "See [Adiwardana et al. (2020)](https://arxiv.org/abs/2001.09977)."
+    )
+}
diff --git a/parlai/opt_presets/gen/meena.opt b/parlai/opt_presets/gen/meena.opt
new file mode 100644
index 00000000000..e70b5b61ca1
--- /dev/null
+++ b/parlai/opt_presets/gen/meena.opt
@@ -0,0 +1,5 @@
+{
+  "beam_size": 20,
+  "inference": "topk",
+  "topk": 40
+}
diff --git a/setup.py b/setup.py
index 0841e616026..b2b799fe62a 100644
--- a/setup.py
+++ b/setup.py
@@ -37,7 +37,7 @@
         packages=find_packages(exclude=('data', 'docs', 'tests', 'parlai_internal*')),
         install_requires=reqs,
         include_package_data=True,
-        package_data={'': ['*.txt', '*.md']},
+        package_data={'': ['*.txt', '*.md', '*.opt']},
         entry_points={
             "flake8.extension": ["PAI = parlai.utils.flake8:ParlAIChecker"],
             "console_scripts": ["parlai=parlai.__main__:main"],

From e570b89ca340aa12ff9d2ccf91c111ae7c932bc5 Mon Sep 17 00:00:00 2001
From: Stephen Roller <roller@fb.com>
Date: Tue, 30 Mar 2021 17:12:10 -0400
Subject: [PATCH 2/7] Add blenderbot generation options.

---
 parlai/opt_presets/gen/blenderbot.opt |  8 ++++++++
 tests/test_params.py                  | 12 ++++++++++++
 2 files changed, 20 insertions(+)
 create mode 100644 parlai/opt_presets/gen/blenderbot.opt

diff --git a/parlai/opt_presets/gen/blenderbot.opt b/parlai/opt_presets/gen/blenderbot.opt
new file mode 100644
index 00000000000..316201ba223
--- /dev/null
+++ b/parlai/opt_presets/gen/blenderbot.opt
@@ -0,0 +1,8 @@
+{
+  "beam_block_context_ngram": 3,
+  "beam_block_ngram": 3,
+  "beam_size": 10,
+  "inference": "beam",
+  "beam_min_length": 20,
+  "beam_block_full_context": false
+}
diff --git a/tests/test_params.py b/tests/test_params.py
index d8e5f1647fc..c90ba152cf6 100644
--- a/tests/test_params.py
+++ b/tests/test_params.py
@@ -47,6 +47,18 @@ def test_shortopt(self):
         opt = pp.parse_args(["-m", "memnn"])
         print(opt)
 
+    def test_opt_presets(self):
+        """
+        Tests whether opt presets bundled with parlai work as expected.
+        """
+        pp = ParlaiParser(True, False)
+        pp.add_argument("-m", "--model")
+        # hardcoded example
+        opt = pp.parse_args(['--model', 'transformer/generator', '-o', 'gen/meena'])
+        assert opt['beam_size'] == 20
+        assert opt['inference'] == 'topk'
+        assert opt['topk'] == 40
+
     def test_upgrade_opt(self):
         """
         Test whether upgrade_opt works.

From 5f3665a27ac2c598b22285cb01fec0a54ae7d43f Mon Sep 17 00:00:00 2001
From: Stephen Roller <roller@fb.com>
Date: Tue, 30 Mar 2021 17:15:12 -0400
Subject: [PATCH 3/7] Exception for test

---
 tests/test_code.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tests/test_code.py b/tests/test_code.py
index 6ce6d7559cd..f71fcae114a 100644
--- a/tests/test_code.py
+++ b/tests/test_code.py
@@ -22,7 +22,14 @@ class TestInit(unittest.TestCase):
 
     def test_init_everywhere(self):
         for folder_path in testing_utils.git_ls_dirs('parlai'):
-            excluded_folders = ['conf', 'frontend', 'mturk', 'task_config', 'webapp']
+            excluded_folders = [
+                'conf',
+                'frontend',
+                'mturk',
+                'task_config',
+                'webapp',
+                'opt_presets',
+            ]
             # conf: contains YAML files for Hydra
             # frontend, mturk, webapp: contains frontend code for crowdsourcing tasks
             # task_config: contains JSONs, HTML files, etc. for MTurk/Mephisto tasks

From d6d43738b1840c1c53d792185c608adcc1f0ffdf Mon Sep 17 00:00:00 2001
From: Stephen Roller <roller@fb.com>
Date: Tue, 30 Mar 2021 17:25:03 -0400
Subject: [PATCH 4/7] Double check CLI still overrides init_opt

---
 parlai/core/params.py | 1 +
 tests/test_params.py  | 7 +++++++
 2 files changed, 8 insertions(+)

diff --git a/parlai/core/params.py b/parlai/core/params.py
index 0b0acc18cf9..4d8a942c722 100644
--- a/parlai/core/params.py
+++ b/parlai/core/params.py
@@ -1159,6 +1159,7 @@ def parse_args(self, args=None, namespace=None, **kwargs):
                 "no longer supported. Use opt.log() to print the arguments"
             )
             del kwargs['print_args']
+        self.overridable.clear()
         self.add_extra_args(args)
         self.args = super().parse_args(args=args)
 
diff --git a/tests/test_params.py b/tests/test_params.py
index c90ba152cf6..a5f35b0dcd9 100644
--- a/tests/test_params.py
+++ b/tests/test_params.py
@@ -58,6 +58,13 @@ def test_opt_presets(self):
         assert opt['beam_size'] == 20
         assert opt['inference'] == 'topk'
         assert opt['topk'] == 40
+        # and preference for command line over opt presets
+        opt = pp.parse_args(
+            ['--model', 'transformer/generator', '-o', 'gen/meena', '--topk', '7']
+        )
+        assert opt['beam_size'] == 20
+        assert opt['inference'] == 'topk'
+        assert opt['topk'] == 7
 
     def test_upgrade_opt(self):
         """

From 135646058291433bebb1dced923c643fb6618549 Mon Sep 17 00:00:00 2001
From: Stephen Roller <roller@fb.com>
Date: Tue, 30 Mar 2021 17:52:36 -0400
Subject: [PATCH 5/7] Support composition

---
 parlai/core/opt.py                        | 43 +++++++++++++++++++----
 parlai/core/params.py                     | 33 ++---------------
 parlai/opt_presets/arch/blenderbot_3B.opt | 16 +++++++++
 tests/test_params.py                      | 19 ++++++++++
 4 files changed, 74 insertions(+), 37 deletions(-)
 create mode 100644 parlai/opt_presets/arch/blenderbot_3B.opt

diff --git a/parlai/core/opt.py b/parlai/core/opt.py
index 84a958d6847..b8c3b0f19d9 100644
--- a/parlai/core/opt.py
+++ b/parlai/core/opt.py
@@ -8,14 +8,17 @@
 Opt is the system for passing around options throughout ParlAI.
 """
 
+from __future__ import annotations
+
 import copy
 import json
 import pickle
 import traceback
-import io
+import os
+import pkg_resources
 import parlai.utils.logging as logging
 
-from typing import List, Union
+from typing import List
 
 from parlai.utils.io import PathManager
 
@@ -120,13 +123,10 @@ def save(self, filename: str) -> None:
             f.write('\n')
 
     @classmethod
-    def load(cls, optfile: Union[str, io.IOBase]) -> 'Opt':
+    def load(cls, optfile: str) -> Opt:
         """
         Load an Opt from disk.
         """
-        if isinstance(optfile, io.IOBase):
-            with optfile:
-                return json.load(optfile)
         try:
             # try json first
             with PathManager.open(optfile, 'r', encoding='utf-8') as t_handle:
@@ -140,6 +140,37 @@ def load(cls, optfile: Union[str, io.IOBase]) -> 'Opt':
                 del dct[key]
         return cls(dct)
 
+    @classmethod
+    def load_init(cls, optfile: str) -> Opt:
+        """
+        Like load, but also looks in opt_presets folders.
+
+        optfile may also be a comma-separated list of multiple presets/files.
+        """
+        if "," in optfile:
+            # load and combine each of the individual files
+            new_opt = cls()
+            for subopt in optfile.split(","):
+                new_opt.update(cls.load_init(subopt))
+            return new_opt
+
+        oa_filename = os.path.join("opt_presets", optfile + ".opt")
+        user_filename = os.path.join(os.path.expanduser(f"~/.parlai"), oa_filename)
+        if PathManager.exists(optfile):
+            return cls.load(optfile)
+        elif PathManager.exists(user_filename):
+            # use a user's custom opt preset
+            return cls.load(user_filename)
+        elif pkg_resources.resource_exists("parlai", oa_filename):
+            # Maybe a bundled opt preset
+            return cls.load(pkg_resources.resource_filename("parlai", oa_filename))
+        else:
+            raise FileNotFoundError(
+                f"Could not find filename '{optfile} or opt preset '{optfile}.opt'. "
+                "Please check https://parl.ai/docs/opt_presets.html for a list "
+                "of available opt presets."
+            )
+
     def log(self, header="Opt"):
         from parlai.core.params import print_git_commit
 
diff --git a/parlai/core/params.py b/parlai/core/params.py
index 4d8a942c722..7e24671e031 100644
--- a/parlai/core/params.py
+++ b/parlai/core/params.py
@@ -13,7 +13,6 @@
 import sys as _sys
 import datetime
 import parlai
-import pkg_resources
 
 try:
     import git
@@ -978,33 +977,6 @@ def parse_known_args(self, args=None, namespace=None, nohelp=False):
             args = [a for a in args if a != '-h' and a != '--help' and a != '--helpall']
         return super().parse_known_args(args, namespace)
 
-    def _find_opt_file(self, optfile: str):
-        """
-        Look for an opt file as an argument to --init-opt.
-
-        First looks for a particular existing file on disk, then checks the
-        folder of option aliases.
-
-        May return a filename or a streaming file of an Opt.
-        """
-        oa_filename = os.path.join("opt_presets", optfile + ".opt")
-        user_filename = os.path.join(os.path.expanduser(f"~/.parlai"), oa_filename)
-        if PathManager.exists(optfile):
-            return optfile
-        elif PathManager.exists(user_filename):
-            # use a user's custom opt preset
-            return user_filename
-        elif pkg_resources.resource_exists("parlai", oa_filename):
-            # Maybe a bundled opt preset
-            return pkg_resources.resource_stream("parlai", oa_filename)
-        else:
-            raise FileNotFoundError(
-                f"Could not find filename or opt preset {optfile}. "
-                "Opt presets should not have the '.opt' suffix. Please "
-                "check https://parl.ai/docs/opt_presets.html for a list "
-                "of available opt presets."
-            )
-
     def _load_known_opts(self, optfile, parsed):
         """
         Pull in CLI args for proper models/tasks/etc.
@@ -1012,7 +984,7 @@ def _load_known_opts(self, optfile, parsed):
         Called before args are parsed; ``_load_opts`` is used for actually overriding
         opts after they are parsed.
         """
-        new_opt = Opt.load(self._find_opt_file(optfile))
+        new_opt = Opt.load_init(optfile)
         for key, value in new_opt.items():
             # existing command line parameters take priority.
             if key not in parsed or parsed[key] is None:
@@ -1020,7 +992,7 @@ def _load_known_opts(self, optfile, parsed):
 
     def _load_opts(self, opt):
         optfile = opt.get('init_opt')
-        new_opt = Opt.load(self._find_opt_file(optfile))
+        new_opt = Opt.load_init(optfile)
         for key, value in new_opt.items():
             # existing command line parameters take priority.
             if key not in opt:
@@ -1159,7 +1131,6 @@ def parse_args(self, args=None, namespace=None, **kwargs):
                 "no longer supported. Use opt.log() to print the arguments"
             )
             del kwargs['print_args']
-        self.overridable.clear()
         self.add_extra_args(args)
         self.args = super().parse_args(args=args)
 
diff --git a/parlai/opt_presets/arch/blenderbot_3B.opt b/parlai/opt_presets/arch/blenderbot_3B.opt
new file mode 100644
index 00000000000..a5de45c90e9
--- /dev/null
+++ b/parlai/opt_presets/arch/blenderbot_3B.opt
@@ -0,0 +1,16 @@
+{
+  "activation": "gelu",
+  "attention_dropout": 0,
+  "embedding_size": 2560,
+  "ffn_size": 10240,
+  "label_truncate": 128,
+  "model": "transformer/generator",
+  "n_decoder_layers": 24,
+  "n_encoder_layers": 2,
+  "n_heads": 32,
+  "n_positions": 128,
+  "relu_dropout": 0,
+  "text_truncate": 128,
+  "truncate": 128,
+  "variant": "prelayernorm"
+}
diff --git a/tests/test_params.py b/tests/test_params.py
index a5f35b0dcd9..fa2e9f863cd 100644
--- a/tests/test_params.py
+++ b/tests/test_params.py
@@ -59,12 +59,31 @@ def test_opt_presets(self):
         assert opt['inference'] == 'topk'
         assert opt['topk'] == 40
         # and preference for command line over opt presets
+        pp = ParlaiParser(True, False)
+        pp.add_argument("-m", "--model")
         opt = pp.parse_args(
             ['--model', 'transformer/generator', '-o', 'gen/meena', '--topk', '7']
         )
         assert opt['beam_size'] == 20
         assert opt['inference'] == 'topk'
         assert opt['topk'] == 7
+        # double check ordering doesn't matter
+        pp = ParlaiParser(True, False)
+        pp.add_argument("-m", "--model")
+        opt = pp.parse_args(
+            ['--model', 'transformer/generator', '--topk', '8', '-o', 'gen/meena']
+        )
+        assert opt['beam_size'] == 20
+        assert opt['inference'] == 'topk'
+        assert opt['topk'] == 8
+        # check composability
+        pp = ParlaiParser(True, False)
+        pp.add_argument("-m", "--model")
+        opt = pp.parse_args(['-o', 'arch/blenderbot_3B,gen/meena'])
+        assert opt['beam_size'] == 20
+        assert opt['inference'] == 'topk'
+        assert opt['model'] == 'transformer/generator'
+        assert opt['n_encoder_layers'] == 2
 
     def test_upgrade_opt(self):
         """

From 61caa0fd71c2b17298f0c1a6670157d0aa5e1bab Mon Sep 17 00:00:00 2001
From: Stephen Roller <roller@fb.com>
Date: Fri, 16 Apr 2021 16:24:08 -0400
Subject: [PATCH 6/7] Add new consistency test.

---
 parlai/opt_presets/docs.py | 10 +++++++++-
 tests/test_code.py         | 29 +++++++++++++++++++++++++++++
 2 files changed, 38 insertions(+), 1 deletion(-)

diff --git a/parlai/opt_presets/docs.py b/parlai/opt_presets/docs.py
index db65dda2bae..dc0a11857fb 100644
--- a/parlai/opt_presets/docs.py
+++ b/parlai/opt_presets/docs.py
@@ -16,5 +16,13 @@
     "gen/meena": (
         "Inference parameters for the Sample & Rank procedure of Meena. "
         "See [Adiwardana et al. (2020)](https://arxiv.org/abs/2001.09977)."
-    )
+    ),
+    "arch/blenderbot_3B": (
+        "Architecture parameters (number layers, etc) for BlenderBot 3B. See "
+        "[Roller et al. (2020)](https://arxiv.org/abs/2004.13637)"
+    ),
+    "gen/blenderbot": (
+        "Beam search parameters for BlenderBot. See"
+        "[Roller et al. (2020)](https://arxiv.org/abs/2004.13637)"
+    ),
 }
diff --git a/tests/test_code.py b/tests/test_code.py
index f71fcae114a..c2f21707ae5 100644
--- a/tests/test_code.py
+++ b/tests/test_code.py
@@ -9,9 +9,11 @@
 """
 
 import pytest
+import glob
 import unittest
 import os
 import parlai.utils.testing as testing_utils
+import parlai.opt_presets as opt_presets
 
 
 @pytest.mark.nofbcode
@@ -45,5 +47,32 @@ def test_init_everywhere(self):
             )
 
 
+@pytest.mark.nofbcode
+class TestOptPresets(unittest.TestCase):
+    """
+    Ensure all opt presets contain descriptions.
+    """
+
+    def test_opt_preset_docs(self):
+        from parlai.opt_presets.docs import PRESET_DESCRIPTIONS
+
+        folder = os.path.dirname(opt_presets.__file__)
+        has_file = set(x[len(folder) + 1 : -4] for x in glob.glob(f'{folder}/*/*.opt'))
+        has_docs = set(PRESET_DESCRIPTIONS.keys())
+
+        file_no_docs = has_file - has_docs
+        if file_no_docs:
+            raise AssertionError(
+                "The following opt presets have files but no documentation: "
+                f"{', '.join(file_no_docs)}"
+            )
+        docs_no_file = has_docs - has_file
+        if docs_no_file:
+            raise AssertionError(
+                "The following opt presets have documentation but no files: "
+                f"{', '.join(docs_no_file)}"
+            )
+
+
 if __name__ == '__main__':
     unittest.main()

From ff0c5b0675e8f2e41b7ba2e94a9c20934bfcb2fa Mon Sep 17 00:00:00 2001
From: Stephen Roller <roller@fb.com>
Date: Fri, 16 Apr 2021 21:24:22 -0400
Subject: [PATCH 7/7] Doc formatting typos

---
 docs/source/generate_mutator_list.py | 4 +++-
 docs/source/generate_opt_presets.py  | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/docs/source/generate_mutator_list.py b/docs/source/generate_mutator_list.py
index 5cb1a56b6ef..4553a7dbef1 100755
--- a/docs/source/generate_mutator_list.py
+++ b/docs/source/generate_mutator_list.py
@@ -94,9 +94,11 @@ def _display_data(**kwargs):
     fout.write(f"```\n{output}\n```\n")
 
     for mutator_name in sorted(mutators.keys()):
-        fout.write('\n------------\n\n')
         mutator = mutators[mutator_name]
         options = _make_argparse_table(mutator)
+        if not mutator.__doc__:
+            continue
+        fout.write('\n------------\n\n')
         fout.write(f'## {mutator_name}\n\n')
         fout.write(textwrap.dedent(mutator.__doc__).strip() + '\n\n')
         fout.write(
diff --git a/docs/source/generate_opt_presets.py b/docs/source/generate_opt_presets.py
index 9314399632c..36261da4f97 100755
--- a/docs/source/generate_opt_presets.py
+++ b/docs/source/generate_opt_presets.py
@@ -22,5 +22,5 @@
             key = '--' + key.replace('_', '-')
             expansion_str.append(f'`{key} {value}`')
         expansion_str = " ".join(expansion_str)
-        fout.write(f'`{alias}` | {description} | {expansion_str}')
+        fout.write(f'`{alias}` | {description} | {expansion_str}\n')
 fout.close()