docs: Tooling and automation for support matrix (#290)

* Extract the broad swath of data from
  the NGC containers.
* Create RST tables from the data, by year.
* Add tests.
* Rebase after docs-ext-toc was merged.
* Add logging for troublshooting unexpected results.
* Add get_from_pip function from Ben

.github/workflows/docs-ci.yaml

@@ -0,0 +1,51 @@
+name: docs-ci
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [ main ]
+    paths:
+      - 'docs/**.py'
+  pull_request:
+    branches: [ main ]
+    types: [opened, synchronize, reopened, closed]
+    paths:
+      - 'docs/**.py'
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python-version: [3.8]
+        os: [ubuntu-latest]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install Ubuntu packages
+      run: |
+        sudo apt-get update -y
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip setuptools==59.4.0 wheel
+        python -m pip install -r docs/requirements-doc.txt
+    - name: Lint with flake8
+      run: |
+        flake8 docs
+    - name: Lint with black
+      run: |
+        black --check docs
+    - name: Lint with isort
+      run: |
+        isort -c docs
+    - name: Lint with codespell
+      run: |
+        codespell
+    - name: Run unittests
+      # python -m pytest -rxs docs
+      run: |
+        coverage run -m pytest -v docs && coverage report -m

.github/workflows/docs-sched-rebuild.yaml

@@ -3,7 +3,7 @@ name: docs-sched-rebuild
 on:
   schedule:
     # * is a special character in YAML so you have to quote this string
-    - cron: "0 0 * * *"
+    - cron: "0 1 * * *"
   workflow_dispatch:
 
 jobs:

.github/workflows/docs-smx-data.yaml

@@ -0,0 +1,54 @@
+name: docs-smx-data
+
+on:
+  schedule:
+    # * is a special character in YAML so you have to quote this string
+    - cron: "0 0 * * *"
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python-version: [3.8]
+        os: [ubuntu-latest]
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Ubuntu packages
+        run: |
+          sudo apt-get update -y
+          sudo apt-get clean autoclean
+          sudo apt-get autoremove --yes
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip setuptools==59.4.0 wheel
+          python -m pip install -r docs/requirements-doc.txt
+      - name: Maximize disk space
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /usr/local/share/boost
+      - name: Get container data
+        run: |
+          python docs/extractor.py
+      - name: Initialize Git configuration
+        run: |
+          git config user.name docs-smx-bot
+          git config user.email do-not-send-@github.com
+      - name: Commit updates to data.json
+        run: |
+          x=$(git status --porcelain)
+          if grep -q "data.json" <<< "${x}"; then
+            git add .
+            git commit -m 'Updates from new containers'
+            git push
+          fi

.gitignore

@@ -1,3 +1,9 @@
 docs/build/
 docs/source/examples
 docs/source/README.md
+docs/source/generated/
+
+.coverage
+
+__pycache__/
+*.py[cod]

.pylintrc

@@ -0,0 +1,62 @@
+[MESSAGES CONTROL]
+disable=fixme,
+    # docstrings aren't required (yet).
+    missing-function-docstring,
+    missing-module-docstring,
+    missing-class-docstring,
+
+    # formatting checks that we're handling with black/isort
+    wrong-import-order,
+    wrong-import-position,
+    ungrouped-imports,
+    line-too-long,
+    superfluous-parens,
+    trailing-whitespace,
+
+    # we'll probably never enable these checks
+    invalid-name,
+    import-error,
+    duplicate-code,
+
+    # disable code-complexity checks for now
+    # TODO: should we configure the thresholds for these rather than just disable?
+    ; too-many-function-args,
+    ; too-many-instance-attributes,
+    ; too-many-locals,
+    ; too-many-branches,
+    ; too-many-nested-blocks,
+    ; too-many-statements,
+    ; too-many-arguments,
+    ; too-many-return-statements,
+    ; too-many-lines,
+    ; too-few-public-methods,
+
+    # many of these checks would be great to include at some point, but would
+    # require some changes to our codebase
+    ; useless-return,
+    ; protected-access,
+    ; arguments-differ,
+    ; unused-argument,
+    ; unused-variable,
+    ; abstract-method,
+    ; no-name-in-module,
+    ; attribute-defined-outside-init,
+    ; redefined-outer-name,
+    ; import-outside-toplevel,
+    ; no-else-continue,
+    ; no-else-return,
+    ; no-else-raise,
+    ; no-member,
+    ; super-with-arguments,
+    ; unsupported-assignment-operation,
+    ; inconsistent-return-statements,
+    ; duplicate-string-formatting-argument,
+    ; len-as-condition,
+    ; cyclic-import,
+
+    # producing false positives
+    ; unexpected-keyword-arg,
+    ; not-an-iterable,
+    ; unsubscriptable-object
+[SIMILARITIES]
+min-similarty-lines=20

ci/versions.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 #
 
+# pylint: skip-file
+
 import argparse
 import contextlib
 
@@ -46,7 +48,7 @@ def get_pythonpkg_version(container, pkg):
             "bash -c 'pip list | grep " + pkg + "'", stderr=False
         )
         return output[1].decode("utf-8").split()[1]
-    except:
+    except:  # noqa
         return "N/A"
 
 
@@ -56,7 +58,7 @@ def create_pr(repo, branch, filename, content, token, version):
     r.create_git_ref(ref="refs/heads/" + branch, sha=r.get_branch("main").commit.sha)
     f = r.get_contents(filename, ref=branch)
     r.update_file(f.path, "release " + version, content, branch=branch, sha=f.sha)
-    pr = r.create_pull(
+    pr = r.create_pull(  # noqa
         title="Merlin Release " + version, body="", head=branch, base="main"
     )
 

docs/README.md

@@ -25,6 +25,12 @@ Perform the following steps to build the docs.
 These steps should run Sphinx in your shell and create HTML in the `build/html/`
 directory.
 
+The build for Merlin is unique because the support matrix is generated during
+the build.
+The build reads the `docs/data.json` file and creates several RST snippet files
+in `docs/source/generated`.
+The `docs/data.json` file is updated by the `docs-smx-data` GitHub workflow.
+
 ## Preview the Changes
 
 View the docs web page by opening the HTML in your browser. First, navigate to
@@ -37,3 +43,37 @@ python -m http.server
 Afterward, open a web browser and access <https://localhost:8000>.
 
 Check that yours edits formatted correctly and read well.
+
+## Tests
+
+```shell
+python -m pytest docs
+```
+...or...
+
+```shell
+coverage run -m pytest -v docs && coverage report -m
+```
+
+## Handy notes
+
+### Remove a field from the JSON
+
+In the following case, the `cuparse` key is a mistake and should have been
+`cusparse`.  The following command removes the mistake from the JSON:
+
+```shell
+jq 'walk(if type == "object" then del(.cuparse) else . end)' < data.json > x
+```
+
+### View a container for a release
+
+```shell
+jq '.["nvcr.io/nvidia/merlin/merlin-inference"]["22.03"]' < ../docs/source/data.json
+```
+
+### List the containers and releases
+
+```shell
+jq '. | map_values(keys) ' < docs/source/data.json
+```