Add custom metrics tutorial

.gitignore

@@ -86,6 +86,7 @@ docs/_build/
 docs/source/task_list.inc
 docs/source/zoo_list.inc
 docs/source/cli_usage.inc
+docs/source/cli_advanced.inc
 docs/source/agent_refs
 
 # PyBuilder

docs/source/_static/css/parlai_theme.css

@@ -13,6 +13,11 @@ h1, h2, .rst-content .toctree-wrapper p.caption, h3, h4, h5, h6, legend, p.capti
     font-family: "Lato","proxima-nova","Helvetica Neue",Arial,sans-serif;
 }
 
+p { margin-bottom: 16px; }
+h1 { size: 250%; }
+h2 { size: 200%; }
+h3 { size: 150%; }
+
 .wy-menu.wy-menu-vertical p.caption {
     color: #fcfcfc;
 }
@@ -26,6 +31,14 @@ h1, h2, .rst-content .toctree-wrapper p.caption, h3, h4, h5, h6, legend, p.capti
     background-color: #fff;
 }
 
+th p, td div.line-block {
+    margin-bottom: 0px !important;
+}
+
+.text-align\:right {
+    text-align: right;
+}
+
 @media screen and (min-width: 1400px) {
     .wy-nav-content-wrap {
         background-color: rgba(0, 0, 0, 0.0470588);
@@ -67,7 +80,7 @@ h1, h2, .rst-content .toctree-wrapper p.caption, h3, h4, h5, h6, legend, p.capti
 
 /* Use our red for literals (it's very similar to the original color) */
 .rst-content tt.literal, .rst-content tt.literal, .rst-content code.literal {
-    color: #CE3239;
+    color: #333;
 }
 
 .rst-content tt.xref, a .rst-content tt, .rst-content tt.xref,
@@ -141,3 +154,12 @@ nav .hidden-section {
 .wy-table-responsive table {
     width: 100%;
 }
+
+.section {
+}
+.section .section {
+    margin-bottom: 5em;
+}
+.section .section .section {
+    margin-bottom: 2em;
+}

docs/source/agents_list.md → docs/source/agents_list.rst

@@ -1,15 +1,14 @@
-# Standard Agents
+Standard Agents
+---------------
 
 ParlAI comes with a wide variety of standardized agents. Some of these are
 common baselines made available for quicker research; some can be made
 state-of-the-art with careful tuning and experimentation. Each of these contains
 a README with examples of usage and description of the agent.
 
-```eval_rst
 .. toctree::
    :maxdepth: 1
    :caption: List of Agents
    :glob:
 
    agent_refs/*
-```

docs/source/cli_advanced.rst

@@ -0,0 +1,6 @@
+Advance Scripts
+==================
+
+These are the more obscure and advanced scripts in parlai.
+
+.. include:: cli_advanced.inc

docs/source/cli_usage.rst

@@ -1,7 +1,38 @@
-Command Line Usage
+Main scripts
 ==================
 
 This contains the command line usage for each of the standard scripts we
-release. These are each included in ``parlai/scripts``.
+release. These are each included in ``parlai/scripts``, and all may
+be invoked with the "parlai" supercommand.
+
+The parlai supercommand may be invoked from the command line by running
+``parlai`` after installing ParlAI. Its default output looks like this:
+
+.. code-block::txt
+
+  usage: parlai [-h] [--helpall] COMMAND ...
+
+         _
+        /")
+       //)
+    ==//'=== ParlAI
+     /
+
+  optional arguments:
+    -h, --help               show this help message and exit
+    --helpall                show all commands, including advanced ones.
+
+  Commands:
+
+    display_data (dd)        Display data from a task
+    display_model (dm)       Display model predictions.
+    eval_model (em, eval)    Evaluate a model
+    train_model (tm, train)  Train a model
+    interactive (i)          Interactive chat with a model on the command line
+    safe_interactive         Like interactive, but adds a safety filter
+    self_chat                Generate self-chats of a model
+
+The remainder of this page describes each of the commands, their possible arguments,
+and some examples of their usage.
 
 .. include:: cli_usage.inc

docs/source/conf.py

@@ -39,12 +39,11 @@
 import git
 import sphinx_rtd_theme
 import parlai
-from recommonmark.transform import AutoStructify
 
 extensions = [
     'sphinx.ext.autodoc',
     'sphinx.ext.githubpages',
-    'recommonmark',
+    'myst_parser',
     'sphinx.ext.linkcode',
 ]
 
@@ -207,11 +206,3 @@ def find_source():
         return "https://github.com/facebookresearch/ParlAI/blob/%s/%s" % (tag, filename)
     except Exception:
         return None
-
-
-# At the bottom of conf.py
-def setup(app):
-    app.add_config_value(
-        'recommonmark_config', {'auto_toc_tree_section': 'Contents'}, True
-    )
-    app.add_transform(AutoStructify)

docs/source/core.rst

@@ -0,0 +1,11 @@
+parlai.core
+---------------
+
+This contains the full, comprehensive docstrings of all of ``parlai.core``,
+which powers the vast majority of ParlAI.
+
+.. toctree::
+   :maxdepth: 1
+   :glob:
+
+   core/*

docs/source/agents.rst → docs/source/core/agents.rst

@@ -1,4 +1,4 @@
-core.agents
+parlai.core.agents
 ===================================
 .. automodule:: parlai.core.agents
   :members:

docs/source/build_data.rst → docs/source/core/build_data.rst

@@ -1,4 +1,4 @@
-core.build_data
+parlai.core.build_data
 ===================================
 .. automodule:: parlai.core.build_data
   :members:

docs/source/dict.rst → docs/source/core/dict.rst

@@ -1,4 +1,4 @@
-core.dict
+parlai.core.dict
 ===================================
 .. automodule:: parlai.core.dict
   :members:

docs/source/core/loader.rst

@@ -0,0 +1,4 @@
+parlai.core.loader
+===================================
+.. automodule:: parlai.core.loader
+  :members:

docs/source/messages.rst → docs/source/core/messages.rst

@@ -1,7 +1,12 @@
-messages
-============
+parlai.core.message
+===================================
 
-.. image:: _static/img/act-obs-dict.png
+.. automodule:: parlai.core.message
+  :members:
+
+
+
+.. image:: /_static/img/act-obs-dict.png
     :width: 60 %
 
 The primary medium for information flow (messages between agents and the environment)

docs/source/metrics.rst → docs/source/core/metrics_api.rst

@@ -1,4 +1,4 @@
-core.metrics
+parlai.core.metrics
 ===================================
 .. automodule:: parlai.core.metrics
   :members:

docs/source/core/params.rst

@@ -0,0 +1,9 @@
+parlai.core.opt
+===================================
+.. automodule:: parlai.core.opt
+  :members:
+
+parlai.core.params
+===================================
+.. automodule:: parlai.core.params
+  :members:

docs/source/core/script.rst

@@ -0,0 +1,4 @@
+parlai.core.script
+===================================
+.. automodule:: parlai.core.script
+  :members:

docs/source/teachers.rst → docs/source/core/teachers.rst

@@ -1,4 +1,4 @@
-core.teachers
+parlai.core.teachers
 ===================================
 .. automodule:: parlai.core.teachers
   :members:

docs/source/torch_agent.rst → docs/source/core/torch_agent.rst

@@ -1,4 +1,4 @@
-core.torch_agent
+parlai.core.torch_agent
 ===================================
 
 Torch Agent implements much of the boilerplate necessary for creating
@@ -21,13 +21,26 @@ Torch Agent
   :members:
 
 
+Torch Generator Agent
+-----------------------------------
+.. automodule:: parlai.core.torch_generator_agent
+  :members:
+
+
 Torch Ranker Agent
 -----------------------------------
 .. automodule:: parlai.core.torch_ranker_agent
   :members:
 
 
-Torch Generator Agent
+Torch Classifier Agent
 -----------------------------------
-.. automodule:: parlai.core.torch_generator_agent
+.. automodule:: parlai.core.torch_classifier_agent
+  :members:
+
+
+
+Torch Image Agent
+-----------------------------------
+.. automodule:: parlai.core.torch_image_agent
   :members:

docs/source/worlds.rst → docs/source/core/worlds.rst

@@ -1,4 +1,4 @@
-core.worlds
+parlai.core.worlds
 ===================================
 .. automodule:: parlai.core.worlds
   :members:

docs/source/generate_agent_list.py

@@ -7,12 +7,11 @@
 import parlai.agents
 from parlai.core.params import ParlaiParser
 
+import argparse
 import os
 import pkgutil
 import importlib
 import inspect
-import io
-import csv
 
 
 """
@@ -35,22 +34,24 @@ def _make_argparse_table(class_):
             if hasattr(action, 'hidden') and action.hidden:
                 # some options are marked hidden
                 continue
-            action_strings = ",  ".join(f'``{a}``' for a in action.option_strings)
+            if action.dest == argparse.SUPPRESS or action.dest == 'help':
+                continue
+            action_strings = ",  ".join(f'`{a}`' for a in action.option_strings)
             description = []
-            # start with the help message
             if action.help:
                 h = action.help
                 if not h[0].isupper():
                     h = h[0].upper() + h[1:]
+                h = h.replace("%(default)s", str(action.default))
                 description += [h]
             # list choices if there are any
             if action.choices:
                 description += [
-                    "Choices: " + ", ".join(f'``{c}``' for c in action.choices) + "."
+                    "Choices: " + ", ".join(f'`{c}`' for c in action.choices) + "."
                 ]
             # list default and recommended values.
             default_value = ""
-            if action.default is not None:
+            if action.default is not None and action.default is not argparse.SUPPRESS:
                 default_value += f"Default: ``{action.default}``.  "
             if hasattr(action, 'recommended') and action.recommended:
                 default_value += f"Recommended: ``{action.recommended}``. "
@@ -68,17 +69,14 @@ def _make_argparse_table(class_):
         if not actions:
             continue
 
-        # render the table
-        readme.append(f"```eval_rst\n")
-        readme.append(f".. csv-table:: {ag.title}\n")
-        readme.append(f'   :widths: 35, 65\n\n')
-        cout = io.StringIO()
-        csvw = csv.writer(cout, csv.unix_dialect, delimiter=",")
+        readme.append(f'__{ag.title}__\n\n')
+        readme.append("| Argument | Description |\n")
+        readme.append("|----------|----------|\n")
         for row in actions:
-            cout.write("   ")
-            csvw.writerow(row)
-        readme.append(cout.getvalue())
-        readme.append("```\n\n")
+            text = "| " + " | ".join(row) + " |"
+            text = text.replace("\n", "<br>")
+            readme.append(f"{text}\n")
+        readme.append("\n\n")
     return readme