Opt presets

.gitignore

@@ -83,11 +83,7 @@ instance/
 # Sphinx documentation
 docs/_build/
 # autogenerated files
-docs/source/task_list.inc
-docs/source/zoo_list.inc
-docs/source/cli_usage.inc
-docs/source/cli_advanced.inc
-docs/source/mutators_list.inc
+docs/source/*.inc
 docs/source/agent_refs
 
 # PyBuilder

docs/Makefile

@@ -34,4 +34,5 @@ clean:
 	cd "${SOURCEDIR}"; python generate_mutator_list.py
 	cd "${SOURCEDIR}"; python generate_metric_list.py
 	cd "${SOURCEDIR}"; python generate_cli.py
+	cd "${SOURCEDIR}"; python generate_opt_presets.py
 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/source/generate_mutator_list.py

@@ -94,9 +94,11 @@ def _display_data(**kwargs):
     fout.write(f"```\n{output}\n```\n")
 
     for mutator_name in sorted(mutators.keys()):
-        fout.write('\n------------\n\n')
         mutator = mutators[mutator_name]
         options = _make_argparse_table(mutator)
+        if not mutator.__doc__:
+            continue
+        fout.write('\n------------\n\n')
         fout.write(f'## {mutator_name}\n\n')
         fout.write(textwrap.dedent(mutator.__doc__).strip() + '\n\n')
         fout.write(

docs/source/generate_opt_presets.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+
+# Copyright (c) Facebook, Inc. and its affiliates.
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import json
+import pkg_resources
+
+from parlai.opt_presets.docs import PRESET_DESCRIPTIONS
+
+fout = open('opt_presets_list.inc', 'w')
+fout.write('Preset name | Description | Expansion\n')
+fout.write('----------- | ----------- | ---------\n')
+for alias in sorted(PRESET_DESCRIPTIONS.keys()):
+    description = PRESET_DESCRIPTIONS[alias]
+    with pkg_resources.resource_stream("parlai", f"opt_presets/{alias}.opt") as f:
+        expansion = json.load(f)
+        assert isinstance(expansion, dict)
+        expansion_str = []
+        for key, value in expansion.items():
+            key = '--' + key.replace('_', '-')
+            expansion_str.append(f'`{key} {value}`')
+        expansion_str = " ".join(expansion_str)
+        fout.write(f'`{alias}` | {description} | {expansion_str}\n')
+fout.close()

docs/source/index.md

@@ -45,6 +45,7 @@ zoo
 cli_usage
 cli_advanced
 cli_custom
+opt_presets
 ```
 
 ```{toctree}

docs/source/opt_presets.md

@@ -0,0 +1,25 @@
+# Opt Presets
+
+Opt presets are a way to provide multiple options on the command line as
+shorthand. Opt presets are bundled with ParlAI and may be used by simply
+invoking the `-o preset_name` option within any ParlAI command.
+
+You may also define your own options by placing them in `~/.parlai/opt_presets/`.
+For example, creating `~/.parlai/opt_presets/myfolder/mypreset.opt` allows you to
+invoke it via `-o myfolder/mypreset`. These preset files are simple json files
+containing a dictionary of files. For example:
+
+```js
+{
+    "inference": "beam",
+    "beam_size": 10,
+}
+```
+
+## List of presets
+
+The following is a list of all options presets bundled with the latest version
+of ParlAI.
+
+```{include} opt_presets_list.inc
+```

parlai/core/opt.py

@@ -8,10 +8,14 @@
 Opt is the system for passing around options throughout ParlAI.
 """
 
+from __future__ import annotations
+
 import copy
 import json
 import pickle
 import traceback
+import os
+import pkg_resources
 import parlai.utils.logging as logging
 
 from typing import List
@@ -119,7 +123,7 @@ def save(self, filename: str) -> None:
             f.write('\n')
 
     @classmethod
-    def load(cls, optfile: str) -> 'Opt':
+    def load(cls, optfile: str) -> Opt:
         """
         Load an Opt from disk.
         """
@@ -136,6 +140,37 @@ def load(cls, optfile: str) -> 'Opt':
                 del dct[key]
         return cls(dct)
 
+    @classmethod
+    def load_init(cls, optfile: str) -> Opt:
+        """
+        Like load, but also looks in opt_presets folders.
+
+        optfile may also be a comma-separated list of multiple presets/files.
+        """
+        if "," in optfile:
+            # load and combine each of the individual files
+            new_opt = cls()
+            for subopt in optfile.split(","):
+                new_opt.update(cls.load_init(subopt))
+            return new_opt
+
+        oa_filename = os.path.join("opt_presets", optfile + ".opt")
+        user_filename = os.path.join(os.path.expanduser(f"~/.parlai"), oa_filename)
+        if PathManager.exists(optfile):
+            return cls.load(optfile)
+        elif PathManager.exists(user_filename):
+            # use a user's custom opt preset
+            return cls.load(user_filename)
+        elif pkg_resources.resource_exists("parlai", oa_filename):
+            # Maybe a bundled opt preset
+            return cls.load(pkg_resources.resource_filename("parlai", oa_filename))
+        else:
+            raise FileNotFoundError(
+                f"Could not find filename '{optfile} or opt preset '{optfile}.opt'. "
+                "Please check https://parl.ai/docs/opt_presets.html for a list "
+                "of available opt presets."
+            )
+
     def log(self, header="Opt"):
         from parlai.core.params import print_git_commit
 

parlai/core/params.py

@@ -984,15 +984,15 @@ def _load_known_opts(self, optfile, parsed):
         Called before args are parsed; ``_load_opts`` is used for actually overriding
         opts after they are parsed.
         """
-        new_opt = Opt.load(optfile)
+        new_opt = Opt.load_init(optfile)
         for key, value in new_opt.items():
             # existing command line parameters take priority.
             if key not in parsed or parsed[key] is None:
                 parsed[key] = value
 
     def _load_opts(self, opt):
         optfile = opt.get('init_opt')
-        new_opt = Opt.load(optfile)
+        new_opt = Opt.load_init(optfile)
         for key, value in new_opt.items():
             # existing command line parameters take priority.
             if key not in opt:

parlai/opt_presets/README.md

@@ -0,0 +1,19 @@
+# Option Aliases
+
+This folder contains a set of "option aliases" that are automatically packaged
+and provided with ParlAI. They are used as shorthand for 
+
+## Adding option aliases
+
+Simply adding `.opt` file to this folder is enough to add an alias. The
+directory structure is respected, so `parlai/options/myfolder/myalias.opt` can
+be invoked using `-o myfolder/myalias`.
+
+For quality assurance purposes, we request new option aliases respect the
+following conventions:
+
+- Only include the very minimum number of options you wish to specify with this
+  alias.
+- Pipe your opt file through `jq -S .` so keys are always in alphabetical order.
+- Add an entry to "docs.py" briefly describing your option. The full expansion
+  will be automatically rendered, but a human-friendly summary should be added.

parlai/opt_presets/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python3
+
+# Copyright (c) Facebook, Inc. and its affiliates.
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.

parlai/opt_presets/arch/blenderbot_3B.opt

@@ -0,0 +1,16 @@
+{
+  "activation": "gelu",
+  "attention_dropout": 0,
+  "embedding_size": 2560,
+  "ffn_size": 10240,
+  "label_truncate": 128,
+  "model": "transformer/generator",
+  "n_decoder_layers": 24,
+  "n_encoder_layers": 2,
+  "n_heads": 32,
+  "n_positions": 128,
+  "relu_dropout": 0,
+  "text_truncate": 128,
+  "truncate": 128,
+  "variant": "prelayernorm"
+}

parlai/opt_presets/docs.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+
+# Copyright (c) Facebook, Inc. and its affiliates.
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+List of all opt presets distributed with ParlAI.
+
+This file is for automatically generating docs.
+"""
+
+# PRESET_DESCRIPTIONS is a dictionary mapping alias names to human descriptions
+# for the sake of documentation
+PRESET_DESCRIPTIONS = {
+    "gen/meena": (
+        "Inference parameters for the Sample & Rank procedure of Meena. "
+        "See [Adiwardana et al. (2020)](https://arxiv.org/abs/2001.09977)."
+    ),
+    "arch/blenderbot_3B": (
+        "Architecture parameters (number layers, etc) for BlenderBot 3B. See "
+        "[Roller et al. (2020)](https://arxiv.org/abs/2004.13637)"
+    ),
+    "gen/blenderbot": (
+        "Beam search parameters for BlenderBot. See"
+        "[Roller et al. (2020)](https://arxiv.org/abs/2004.13637)"
+    ),
+}

parlai/opt_presets/gen/blenderbot.opt

@@ -0,0 +1,8 @@
+{
+  "beam_block_context_ngram": 3,
+  "beam_block_ngram": 3,
+  "beam_size": 10,
+  "inference": "beam",
+  "beam_min_length": 20,
+  "beam_block_full_context": false
+}

parlai/opt_presets/gen/meena.opt

@@ -0,0 +1,5 @@
+{
+  "beam_size": 20,
+  "inference": "topk",
+  "topk": 40
+}

setup.py

@@ -37,7 +37,7 @@
         packages=find_packages(exclude=('data', 'docs', 'tests', 'parlai_internal*')),
         install_requires=reqs,
         include_package_data=True,
-        package_data={'': ['*.txt', '*.md']},
+        package_data={'': ['*.txt', '*.md', '*.opt']},
         entry_points={
             "flake8.extension": ["PAI = parlai.utils.flake8:ParlAIChecker"],
             "console_scripts": ["parlai=parlai.__main__:main"],

tests/test_code.py

@@ -9,9 +9,11 @@
 """
 
 import pytest
+import glob
 import unittest
 import os
 import parlai.utils.testing as testing_utils
+import parlai.opt_presets as opt_presets
 
 
 @pytest.mark.nofbcode
@@ -22,7 +24,14 @@ class TestInit(unittest.TestCase):
 
     def test_init_everywhere(self):
         for folder_path in testing_utils.git_ls_dirs('parlai'):
-            excluded_folders = ['conf', 'frontend', 'mturk', 'task_config', 'webapp']
+            excluded_folders = [
+                'conf',
+                'frontend',
+                'mturk',
+                'task_config',
+                'webapp',
+                'opt_presets',
+            ]
             # conf: contains YAML files for Hydra
             # frontend, mturk, webapp: contains frontend code for crowdsourcing tasks
             # task_config: contains JSONs, HTML files, etc. for MTurk/Mephisto tasks
@@ -38,5 +47,32 @@ def test_init_everywhere(self):
             )
 
 
+@pytest.mark.nofbcode
+class TestOptPresets(unittest.TestCase):
+    """
+    Ensure all opt presets contain descriptions.
+    """
+
+    def test_opt_preset_docs(self):
+        from parlai.opt_presets.docs import PRESET_DESCRIPTIONS
+
+        folder = os.path.dirname(opt_presets.__file__)
+        has_file = set(x[len(folder) + 1 : -4] for x in glob.glob(f'{folder}/*/*.opt'))
+        has_docs = set(PRESET_DESCRIPTIONS.keys())
+
+        file_no_docs = has_file - has_docs
+        if file_no_docs:
+            raise AssertionError(
+                "The following opt presets have files but no documentation: "
+                f"{', '.join(file_no_docs)}"
+            )
+        docs_no_file = has_docs - has_file
+        if docs_no_file:
+            raise AssertionError(
+                "The following opt presets have documentation but no files: "
+                f"{', '.join(docs_no_file)}"
+            )
+
+
 if __name__ == '__main__':
     unittest.main()

tests/test_params.py

@@ -47,6 +47,44 @@ def test_shortopt(self):
         opt = pp.parse_args(["-m", "memnn"])
         print(opt)
 
+    def test_opt_presets(self):
+        """
+        Tests whether opt presets bundled with parlai work as expected.
+        """
+        pp = ParlaiParser(True, False)
+        pp.add_argument("-m", "--model")
+        # hardcoded example
+        opt = pp.parse_args(['--model', 'transformer/generator', '-o', 'gen/meena'])
+        assert opt['beam_size'] == 20
+        assert opt['inference'] == 'topk'
+        assert opt['topk'] == 40
+        # and preference for command line over opt presets
+        pp = ParlaiParser(True, False)
+        pp.add_argument("-m", "--model")
+        opt = pp.parse_args(
+            ['--model', 'transformer/generator', '-o', 'gen/meena', '--topk', '7']
+        )
+        assert opt['beam_size'] == 20
+        assert opt['inference'] == 'topk'
+        assert opt['topk'] == 7
+        # double check ordering doesn't matter
+        pp = ParlaiParser(True, False)
+        pp.add_argument("-m", "--model")
+        opt = pp.parse_args(
+            ['--model', 'transformer/generator', '--topk', '8', '-o', 'gen/meena']
+        )
+        assert opt['beam_size'] == 20
+        assert opt['inference'] == 'topk'
+        assert opt['topk'] == 8
+        # check composability
+        pp = ParlaiParser(True, False)
+        pp.add_argument("-m", "--model")
+        opt = pp.parse_args(['-o', 'arch/blenderbot_3B,gen/meena'])
+        assert opt['beam_size'] == 20
+        assert opt['inference'] == 'topk'
+        assert opt['model'] == 'transformer/generator'
+        assert opt['n_encoder_layers'] == 2
+
     def test_upgrade_opt(self):
         """
         Test whether upgrade_opt works.