Examples page

.gitignore

@@ -2,6 +2,7 @@
 build/
 
 # Cookbook files
+source/examples/
 source/_cookbook/
 
 # Python stuff

.readthedocs.yaml

@@ -6,5 +6,12 @@ build:
   os: ubuntu-20.04
   tools:
     python: "mambaforge-4.10"
+  apt_packages:
+    - rsync
+    - git
+  jobs:
+    post_install:
+      - git clone -v --depth=1 --single-branch --branch=_cookbook_data -- https://github.com/openforcefield/openff-docs.git build/_cookbook_data
+      - rsync -a build/_cookbook_data/ ./
 conda:
   environment: devtools/conda-envs/rtd_env.yml

devtools/conda-envs/examples_env.yml

@@ -12,11 +12,14 @@ dependencies:
     - packaging
     # Examples
     - openff-toolkit-examples
-    - openff-interchange==0.3.5 # Pin Interchange 0.3.5 because 0.3.6 has a broken example
+    - openff-interchange # 0.3.6 has a broken example, but we're loading examples from the central-examples branch where it's fixed
     - openff-nagl==0.2.2
+    # - openff-fragmenter
+    # - openff-qcsubmit
     - gromacs >=2021=nompi*
     - lammps
     - rich
     - jax
     - dglteam/label/cu118::dgl
     - parmed<4
+    - nglview>=3.0.6

devtools/conda-envs/lint_env.yml

@@ -6,7 +6,13 @@ dependencies:
     # readthedocs dependencies
     - sphinx>=5,<7
     - myst-parser>=1,<2
+    # - myst-nb
     - sphinx-notfound-page
+    - ipython >=8.8
+    - sphinx-design
+    - gitpython
+    - nbconvert
+    - nbformat
     # Code example deps
     - openff-toolkit-base
     - openff-interchange-base
@@ -18,3 +24,5 @@ dependencies:
         - git+https://github.com/openforcefield/openff-sphinx-theme.git@main
         # Lints
         - sphinxawesome-codelinter
+        # Sphinx
+        - git+https://github.com/Yoshanuikabundi/MyST-NB.git@upgrade-to-1

devtools/conda-envs/rtd_env.yml

@@ -6,10 +6,19 @@ dependencies:
     # readthedocs dependencies
     - sphinx>=5,<7
     - myst-parser>=1,<2
+    # - myst-nb
     - sphinx-notfound-page
+    - ipython >=8.8
+    - sphinx-design
+    # Examples
+    - gitpython
+    - nbconvert
+    - nbformat
     # Theme
     - pip:
         # Theme
         - git+https://github.com/openforcefield/openff-sphinx-theme.git@main
         # Lints
         - sphinxawesome-codelinter
+        # Sphinx
+        - git+https://github.com/Yoshanuikabundi/MyST-NB.git@upgrade-to-1

source/_ext/cookbook/_cookbook.py

@@ -0,0 +1,123 @@
+"""Implementation of the cookbook Sphinx extension"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+import json
+from os import stat
+from pathlib import Path
+import shutil
+
+from sphinx.application import Sphinx
+from sphinx.config import Config
+
+from .github import download_dir
+from .notebook import (
+    insert_cell,
+    get_metadata,
+    find_notebooks,
+    notebook_zip,
+    set_metadata,
+)
+from .globals import (
+    EXEC_IPYNB_ROOT,
+    REPO_EXAMPLES_DIR,
+    CACHE_BRANCH,
+    COLAB_IPYNB_ROOT,
+    OPENFF_DOCS_ROOT,
+    ZIPPED_IPYNB_ROOT,
+    GITHUB_REPOS,
+)
+
+
+def inject_tags_index(notebook: dict) -> dict:
+    """Inject an `index` directive containing the notebook's metadata tags"""
+
+    tags = get_metadata(notebook, "tags", ["untagged"])
+
+    return insert_cell(
+        notebook,
+        cell_type="markdown",
+        source=[
+            f"```{{index}} {', '.join(tags)}",
+            f"```",
+        ],
+    )
+
+
+def inject_links(app: Sphinx, notebook: dict, docpath: Path) -> dict:
+    user, repo, *path = str(docpath.relative_to(EXEC_IPYNB_ROOT)).split("/")
+    path = "/".join(path)
+
+    tag = get_metadata(notebook, "src_repo_tag", "main")
+
+    github_uri = (
+        f"https://github.com/{user}/{repo}/blob/{tag}/{REPO_EXAMPLES_DIR}/{path}"
+    )
+
+    # TODO: Test colab
+    colab_path = COLAB_IPYNB_ROOT.relative_to(OPENFF_DOCS_ROOT) / user / repo / path
+    colab_uri = f"https://colab.research.google.com/github/openforcefield/openff-docs/blob/{CACHE_BRANCH}/{colab_path}"
+
+    zip_path = notebook_zip(docpath).relative_to(app.srcdir)
+
+    return insert_cell(
+        notebook,
+        cell_type="markdown",
+        source=[
+            f"[Download Notebook](path:/{zip_path})",
+            f"[View in GitHub]({github_uri})",
+            f"[Open in Google Colab]({colab_uri})",
+        ],
+    )
+
+
+def process_notebook(app: Sphinx, docname: str, source: list[str]):
+    docpath = Path(app.env.doc2path(docname))
+    if docpath.suffix != ".ipynb":
+        return
+
+    notebook = json.loads(source[0])
+
+    notebook = inject_links(app, notebook, docpath)
+    notebook = inject_tags_index(notebook)
+
+    # Tell Sphinx we don't expect this notebook to show up in a toctree
+    set_metadata(notebook, "orphan", True)
+
+    source[0] = json.dumps(notebook)
+
+
+def download_cached_notebooks(app: Sphinx, config: Config):
+    """
+    Download notebooks from the cache iff they do not exist.
+
+    Note that the cache is not checked for changes; if you want to refresh the
+    cache, you must manually delete it.
+    """
+    for directory in [
+        COLAB_IPYNB_ROOT,
+        EXEC_IPYNB_ROOT,
+        ZIPPED_IPYNB_ROOT,
+    ]:
+        for repo in GITHUB_REPOS:
+            repo, _, tag = repo.partition("#")
+            repo_directory = directory / repo
+            if not repo_directory.exists():
+                download_dir(
+                    "openforcefield/openff-docs",
+                    str(repo_directory.relative_to(OPENFF_DOCS_ROOT)),
+                    repo_directory,
+                    refspec=CACHE_BRANCH,
+                )
+
+    # Exclude notebooks from linkcheck
+    config["linkcheck_exclude_documents"].extend(
+        str(doc.relative_to(app.srcdir)) for doc in Path(EXEC_IPYNB_ROOT).glob("**/*")
+    )
+
+
+def find_notebook_docnames(app, env, docnames):
+    """Find the downloaded notebooks and make sure Sphinx sees them"""
+    for path in find_notebooks(EXEC_IPYNB_ROOT):
+        docname = env.project.path2doc(str(path))
+        docnames.append(docname)

source/_ext/cookbook/_gallery.py

@@ -0,0 +1,178 @@
+from pathlib import Path
+
+from sphinx.environment import BuildEnvironment
+from sphinx.util.docutils import SphinxDirective
+import sphinx.addnodes
+import docutils.nodes
+from sphinx.writers.html5 import HTML5Translator
+from docutils.parsers.rst.directives import choice
+from sphinx.application import Sphinx
+
+from ._cookbook import find_notebooks
+from .globals import EXEC_IPYNB_ROOT
+from .utils import flatten
+
+
+class CookbookDirective(SphinxDirective):
+    """
+    Directive to draw thumbnails of the cookbook.
+
+    The "categories" option specifies the categories of notebooks that should be
+    included in this cookbook. It is a comma-separated string listing category
+    names. Omitting this option will include all categories. The
+    names "uncategorized" and "other" are special; "uncategorized" is applied
+    to any notebook without a category, whereas "other" will include all
+    categories not rendered in a cookbook somewhere else on the same page as
+    the current directive (possibly including "uncategorized").
+    """
+
+    optional_arguments = 1
+    option_spec = {
+        "categories": lambda x: [
+            s.strip().lower().replace("-", "_") for s in str(x).split(",")
+        ],
+    }
+    has_content = False
+
+    def run(self):
+        node = CookbookNode(categories=self.options.get("categories", []))
+
+        for path in find_notebooks(EXEC_IPYNB_ROOT):
+            entry = CookbookEntryNode.from_path(self.env, path)
+            node.append(entry)
+
+        return [node]
+
+
+class CookbookNode(docutils.nodes.Element):
+    """Docutils node representing the cookbook directive"""
+
+    def __init__(
+        self,
+        categories: list[str] | None = None,
+        *args,
+        **kwargs,
+    ):
+        self.categories: list[str] = categories if categories else []
+        self.children: list[CookbookEntryNode]
+        return super().__init__(*args, **kwargs)
+
+    @staticmethod
+    def visit(translator: HTML5Translator, node: "CookbookNode"):
+        """Render the CookbookNode in HTML"""
+        translator.body.append("<div class='notebook-grid'>")
+
+    @staticmethod
+    def depart(translator: HTML5Translator, node: "CookbookNode"):
+        """Render the CookbookNode in HTML"""
+        translator.body.append("</div>")
+
+
+class CookbookEntryNode(docutils.nodes.Element):
+    def __init__(
+        self,
+        docname: str,
+        thumbnail_uri: str = "https://openforcefield.org/about/branding/img/openforcefield_v2_full-color.png",
+        *args,
+        **kwargs,
+    ):
+        """The absolute path to the notebook."""
+        self.docname: str = docname
+        """The docname of the notebook."""
+        self.uri: str | None = None
+        """URI of the built notebook."""
+        self.title: str | None = None
+        """Title of the notebook."""
+
+        super().__init__(*args, **kwargs)
+
+        self.append(
+            docutils.nodes.image(
+                "",
+                uri=thumbnail_uri,
+                alt="",
+                candidates={"?": ""},
+                classes=["output", "image_png"],
+            )
+        )
+
+    @classmethod
+    def from_path(cls, env: BuildEnvironment, path: Path) -> "CookbookEntryNode":
+        docname = env.path2doc(str(path))
+
+        if docname is None:
+            raise ValueError(
+                f"path {path} is not a document in this Sphinx environment.",
+            )
+
+        if path.with_name("thumbnail.png").is_file():
+            thumbnail_uri = str(path.with_name("thumbnail.png").relative_to(env.srcdir))
+            return cls(docname=docname, thumbnail_uri=thumbnail_uri)
+
+        return cls(docname=docname)
+
+    @staticmethod
+    def visit(translator: HTML5Translator, node: CookbookNode):
+        """Render the CookbookEntryNode in HTML"""
+        translator.body.extend(
+            [
+                f"<a class='notebook-grid-elem' href={node.uri}>",
+            ]
+        )
+
+    @staticmethod
+    def depart(translator: HTML5Translator, node: CookbookNode):
+        """Render the CookbookEntryNode in HTML"""
+        translator.body.extend(
+            [
+                "<div class='caption'>",
+                node.title,
+                "</div>",
+                "</a>",
+            ]
+        )
+
+
+def proc_cookbook_toctree(
+    app: Sphinx,
+    doctree: sphinx.addnodes.document,
+    docname: str,
+):
+    """Update the cookbook with URIs and titles"""
+    metadata = app.env.metadata
+
+    cookbook_nodes = [*doctree.findall(CookbookNode)]
+
+    # Get the categories that are in a cookbook directive on this page
+    categories_on_page = {*flatten(node.categories for node in cookbook_nodes)}
+
+    for cookbook_node in cookbook_nodes:
+        cookbook_categories = cookbook_node.categories
+
+        if cookbook_categories:
+            # "uncategorised" is a special category for notebooks whose metadata
+            # doesn't specify a category
+            cookbook_entries_with_categories = (
+                (entry, metadata[entry.docname].get("category", "uncategorized"))
+                for entry in cookbook_node.children
+            )
+            # "other" is a special category for cookbook directives that should
+            # include all notebooks not rendered in any other category on the
+            # current page.
+            # TODO: Make this all notebooks not rendered in the entire project?
+            cookbook_node.children = [
+                entry
+                for entry, entry_category in cookbook_entries_with_categories
+                if entry_category in cookbook_categories
+                or (
+                    "other" in cookbook_categories
+                    and entry_category not in categories_on_page
+                )
+            ]
+
+        for entry in cookbook_node.children:
+            entry.title = app.env.titles[entry.docname].astext()
+            if entry.title == "<no title>":
+                entry.title = Path(entry.docname).stem.replace("_", " ")
+
+            entry.uri = app.builder.get_relative_uri(docname, entry.docname)

source/_ext/cookbook/github.py

@@ -5,6 +5,7 @@
 from tempfile import TemporaryDirectory
 from importlib import import_module
 
+from git.cmd import Git
 from git.repo import Repo
 import requests
 from packaging.version import Version