doc: extend linters, manage deps via bazel

.bazelignore

@@ -1,5 +1,6 @@
 bin
 doc/_build
+doc/venv
 docker/_build
 private/mgmtapi/tools/node_modules
 tools/lint/logctxcheck/testdata/src

.bazelrc

@@ -35,7 +35,7 @@ test:unit_all --config=unit //...
 test:integration --test_tag_filters=integration,-lint
 test:integration_all --config=integration //...
 
-test:lint --test_tag_filters=lint --test_summary=terse --noshow_progress --experimental_convenience_symlinks=ignore //...
+test:lint --test_tag_filters=lint --test_summary=terse --noshow_progress --experimental_convenience_symlinks=ignore
 
 # run quietly, only display errors
 common:quiet --ui_event_filters=-warning,-info,-debug,-stdout,-stderr --noshow_progress

.github/ISSUE_TEMPLATE/01-proposal.md

@@ -8,4 +8,3 @@ labels: i/proposal
 Our proposal process is documented here:
 https://docs.scion.org/en/latest/dev/contribute.html#change-proposal-process
 -->
-

.markdownlint-cli2.jsonc

@@ -0,0 +1,27 @@
+{
+ "globs": ["**/*.md"],
+ "ignores": [
+ "**/venv/**",
+ "**/_build/**",
+ "**/node_modules/**",
+ "licenses/data/**",
+ "tools/coremark/LICENSE.md"
+ ],
+ "config": {
+ "default": true,
+ "MD007": {
+ "indent": 4
+ },
+ "MD013": {
+ "line_length": 100,
+ "code_blocks": false,
+ "tables": false
+ },
+ "MD024": {
+ "siblings_only": true
+ }
+ },
+ "outputFormatters": [
+ [ "markdownlint-cli2-formatter-pretty", { "appendLink": true } ]
+ ]
+}

.readthedocs.yaml

@@ -7,7 +7,7 @@ version: 2
 build:
  os: ubuntu-22.04
  tools:
- python: "3.8"
+ python: "3.10"
 
 sphinx:
  configuration: doc/conf.py

Makefile

@@ -96,7 +96,7 @@ antlr:
 write_all_source_files:
  bazel run //:write_all_source_files
 
-.PHONY: lint lint-bazel lint-bazel-buildifier lint-doc lint-doc-mdlint lint-go lint-go-bazel lint-go-gazelle lint-go-golangci lint-go-semgrep lint-openapi lint-openapi-spectral lint-protobuf lint-protobuf-buf
+.PHONY: lint lint-bazel lint-bazel-buildifier lint-doc lint-doc-mdlint lint-doc-sphinx lint-go lint-go-bazel lint-go-gazelle lint-go-golangci lint-go-semgrep lint-openapi lint-openapi-spectral lint-protobuf lint-protobuf-buf
 
 # Enable --keep-going if all goals specified on the command line match the pattern "lint%"
 ifeq ($(filter-out lint%, $(MAKECMDGOALS)), )
@@ -113,7 +113,7 @@ lint-go-gazelle:
 
 lint-go-bazel:
  $(info ==> $@)
- @tools/quiet bazel test --config lint
+ @tools/quiet bazel test --config lint //...
 
 GO_BUILD_TAGS_ARG=$(shell bazel build --ui_event_filters=-stdout,-stderr --announce_rc --noshow_progress :dummy_setting 2>&1 | grep "'build' options" | sed -n "s/^.*--define gotags=\(\S*\).*/--build-tags \1/p" )
 
@@ -149,9 +149,13 @@ lint-openapi-spectral:
  $(info ==> $@)
  @tools/quiet bazel run --config=quiet //:spectral -- lint --ruleset ${PWD}/spec/.spectral.yml ${PWD}/spec/*.gen.yml
 
-lint-doc: lint-doc-mdlint
+lint-doc: lint-doc-mdlint lint-doc-sphinx
 
 lint-doc-mdlint:
  $(info ==> $@)
- @FILES=$$(find -type f -iname '*.md' -not -path "./private/mgmtapi/tools/node_modules/*" -not -path "./.github/**/*" | grep -vf tools/md/skipped); \
- docker run --rm -v ${PWD}:/data -v ${PWD}/tools/md/mdlintstyle.rb:/style.rb $$(docker build -q tools/md) $${FILES} -s /style.rb
+ @if [ -t 1 ]; then tty=true; else tty=false; fi; \
+ tools/quiet docker run --tty=$$tty --rm -v ${PWD}:/workdir davidanson/markdownlint-cli2:v0.12.1
+
+lint-doc-sphinx:
+ $(info ==> $@)
+ @tools/quiet bazel test --config=lint //doc:sphinx_lint_test

WORKSPACE

@@ -111,6 +111,14 @@ load("@com_github_scionproto_scion_python_deps//:requirements.bzl", install_pyth
 
 install_python_deps()
 
+load("//doc:deps.bzl", "python_doc_deps")
+
+python_doc_deps(python_interpreter)
+
+load("@com_github_scionproto_scion_python_doc_deps//:requirements.bzl", install_python_doc_deps = "install_deps")
+
+install_python_doc_deps()
+
 http_archive(
  name = "rules_pkg",
  sha256 = "8f9ee2dc10c1ae514ee599a8b42ed99fa262b757058f65ad3c384289ff70c4b8",

doc/.markdownlint-cli2.jsonc

@@ -0,0 +1,11 @@
+{
+ "extends": "../.markdownlint-cli2.jsonc",
+ "config": {
+ "MD033": {
+ "allowed_elements": [
+ "a",
+ "sub"
+ ]
+ }
+ }
+}

doc/BUILD.bazel

@@ -0,0 +1,58 @@
+load("@rules_python//python:pip.bzl", "compile_pip_requirements")
+load("@com_github_scionproto_scion_python_doc_deps//:requirements.bzl", "all_requirements", "entry_point", "requirement")
+load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary")
+load(":lint.bzl", "sphinx_lint_test")
+
+# This BUILD file helps to manage requirements for the sphinx build tools.
+# It assists in building the documentation locally, for developers that already
+# have the bazel setup, as an alternative to manually installing the dependencies from ./requirements.txt.
+#
+# Build the documentation with the accompanying ./Makefile.
+#
+# Note that for builds on the readthedocs infrastructure, this Makefile or bazel is not used!
+# See ../.readthedocs.yaml
+
+compile_pip_requirements(
+ name = "requirements",
+ requirements_in = ":requirements.in",
+ requirements_txt = ":requirements.txt",
+)
+
+# sphinx-build:
+# Needs more than "entry_point" because for the extensions, we need to include more dependencies than the default requirements of the package.
+# Run with all packages from requirements.txt so we don't have to explicitly repeat the required modules.
+py_console_script_binary(
+ name = "sphinx-build",
+ pkg = requirement("sphinx"), # note: "requirement" is just a shorthand for the :pkg label referencing the py_library
+ script = "sphinx-build",
+ deps = [r for r in all_requirements if r != requirement("sphinx")], # macro adds "pkg" to deps, filter it out to avoid duplicate value in attribute
+)
+
+# sphinx-autobuild: just like above
+py_console_script_binary(
+ name = "sphinx-autobuild",
+ pkg = requirement("sphinx-autobuild"),
+ script = "sphinx-autobuild",
+ deps = [r for r in all_requirements if r != requirement("sphinx-autobuild")],
+)
+
+# sphinx-lint
+alias(
+ name = "sphinx-lint",
+ actual = entry_point("sphinx-lint"),
+)
+
+sphinx_lint_test(
+ name = "sphinx_lint_test",
+ srcs = glob([
+ "**/*.rst",
+ "**/*.md",
+ ]),
+ args = [
+ "--enable",
+ "all",
+ "--disable",
+ "line-too-long",
+ ],
+ tags = ["lint"],
+)

doc/Makefile

@@ -1,32 +1,37 @@
-# These variables can be overridden from the command line from the environment.
+# This Makefile helps to build the sphinx documentation.
+# This runs tools using python dependencies managed via bazel/rules_python for the convenience of contributors
+# that have already set up bazel.
+# Alternatively, with dependencies from ./requirements.txt installed with pip, e.g. in a venv,
+# this Makefile can be used directly invoking the build tool (e.g "make SPHINX_BUILD=sphinx-build")
+#
+# Note that for builds on the readthedocs infrastructure, this Makefile or bazel is not used!
+# See ../.readthedocs.yaml
+
+# Variables can be overridden from the command line from the environment.
+SPHINX_BUILD ?= bazel run //doc:sphinx-build --
+SPHINX_AUTOBUILD ?= bazel run //doc:sphinx-autobuild --
 SPHINXOPTS ?= -W --keep-going # treat warnings as errors, but process all files when an error occurs
-SOURCEDIR = .
-BUILDDIR = _build
+SOURCEDIR = $(realpath .)
+BUILDDIR = $(realpath _build)
 
 # Build docs with Sphinx using the "make mode" option.
 # Explicitly list the main build targets so they auto-complete in shells.
 # The first listed name is the default goal (i.e. `make` is `make html`).
 .PHONY: html latex latexpdf linkcheck help
 html latex latexpdf linkcheck help:
- sphinx-build -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS)
+ $(SPHINX_BUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS)
 
 # Catch-all pattern to match any unknown target, too.
 .PHONY: %
 %:
- sphinx-build -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS)
+ $(SPHINX_BUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS)
 
 # Autobuild; automatically build HTML on source changes and serve on localhost
 # Note: most options are forwarded as-is to sphinx, but --keep-going is not understood and we explicitly drop it.
 .PHONY: autobuild
 autobuild:
- sphinx-autobuild "$(SOURCEDIR)" "$(BUILDDIR)" $(filter-out --keep-going,$(SPHINXOPTS))
+ $(SPHINX_AUTOBUILD) "$(SOURCEDIR)" "$(BUILDDIR)" $(filter-out --keep-going,$(SPHINXOPTS))
 
 .PHONY: clean
 clean:
  rm -rf _build
-
-# Generate commandline docs by building the tools and running their `gendocus` subcommand
-.PHONY: command-gendocs
-command-gendocs:
- bazel run //doc/command:copy_scion
- bazel run //doc/command:copy_scion-pki

doc/control-plane.rst

@@ -271,7 +271,7 @@ path-lookup on behalf of applications. This SCION daemon also caches path-segmen
 Path-Segment Combination
 ========================
 
-The last step of the path-resolution process is to combine the available up-, core- and down-
+The last step of the path-resolution process is to combine the available up, core and down
 path segments to end-to-end forwarding paths.
 This path-segment combination process is done by each endpoint separately.
 Typically, end hosts run the :doc:`SCION daemon <manuals/daemon>` which centralizes the

doc/cryptography/certificates.rst

@@ -105,7 +105,7 @@ Version
 -------
 
 The ``version`` field is always ``v3`` in SCION; this is required because
-`extensions` are mandatory.
+``extensions`` are mandatory.
 
 **Deprecation warning**: note that the X.509 ``version`` field has different
 semantics compared to the old SCION JSON format for certificates (where version
@@ -247,14 +247,14 @@ The string representation MUST follow the canonical formatting defined in `ISD
 and AS numbering
 <https://github.com/scionproto/scion/wiki/ISD-and-AS-numbering>`_.
 
-The canonical string representation of the *ISD-AS number* uses a `-`-separator
+The canonical string representation of the *ISD-AS number* uses a ``-``-separator
 between the ISD and AS numbers.
 
 The ISD numbers are formatted as decimal.
 
 The canonical string formatting of AS numbers in the BGP AS range (
 :math:`0 - 2^{32}-1`) is the decimal form.
-For larger AS numbers, it uses a 16-bit `:`-separated lower-case hex encoding
+For larger AS numbers, it uses a 16-bit ``:``-separated lower-case hex encoding
 with leading 0's omitted: 1:0:0 to ffff:ffff:ffff.
 
 For example, AS ``ff00:0:110`` in ISD ``1`` is formatted as ``1-ff00:0:110``.

doc/cryptography/drkey.rst

@@ -279,12 +279,12 @@ the CP-PKI infrastructure.
 
 The Level 1 key request message contains the ``validTime`` that specifies for what time the requested
 key must be valid (it implicitly specifies the epoch for which the key will be valid)
-and the ``protocol_id``. 
+and the ``protocol_id``.
 The Level 1 key response includes the symmetric key along with the epoch
 for which this key will be valid.
 
 The ``protocol_id`` is either set to ``GENERIC = 0`` to request Lvl1 keys that will be derived according to
-the `generic-protocol` hierarchy or to the protocol number for the `protocol-specific` derivation.
+the ``generic-protocol`` hierarchy or to the protocol number for the ``protocol-specific`` derivation.
 
 Level 0/2/3 key establishment
 -----------------------------
@@ -307,8 +307,8 @@ information (depending on the key type).
 The server responds with the symmetric key and the epoch.
 
 The ``protocol_id`` in Lvl2/3 requests is always set to the final protocol identifier.
-The key service will choose between the `protocol-specific` derivation, if it exists, or
-the `generic-protocol` derivation, otherwise.
+The key service will choose between the ``protocol-specific`` derivation, if it exists, or
+the ``generic-protocol`` derivation, otherwise.
 
 Spreading Level 1 key requests
 ==============================

doc/cryptography/trc-signing-ceremony-phases-sensitive.rst

@@ -143,7 +143,7 @@ below.
 
 Seventh, ask the *voting representatives* which voters from the predecessor TRC
 should take part in the voting process. The value will be used to fill the
-``{{.Votes}}`` variable below. Votes contains a sequence of indices of the voting 
+``{{.Votes}}`` variable below. Votes contains a sequence of indices of the voting
 certificates in the predecessor TRC. To find the indices, you can use the ``scion-pki
 trcs human`` command.
 

doc/cryptography/trc.rst

@@ -111,7 +111,7 @@ version is always ``v1``.
 
 **Deprecation warning**: note that the ``version`` field has different semantics
 compared to the old SCION JSON format for TRCs (where version was an
-incrementing counter). The JSON style `version` is moved to the `serial number`
+incrementing counter). The JSON style ``version`` is moved to the ``serial number``
 field in the :ref:`trc-id-field` sequence bellow.
 
 .. _trc-id-field:
@@ -347,7 +347,7 @@ We build on top of the rules from [RFC5652]_ and add the following restrictions:
 - The ``certificates`` field in ``SignedData`` is left empty. The certificate
  pool used to verify the TRC updates is based on the previous TRC.
 - The ``eContentType`` is set to ``id-data``. The contents of the ``eContent``
- is the DER encoded `TRCPayload`, as specified above. This has the benefit that
+ is the DER encoded ``TRCPayload``, as specified above. This has the benefit that
  the format is backwards compatible with PKCS #7, as described in [RFC5652]_,
  Section 5.2.1.
 - Because we do not include certificates in ``SignedData`` and choose

doc/deps.bzl

@@ -0,0 +1,10 @@
+""" Python dependency declaration """
+
+load("@rules_python//python:pip.bzl", "pip_parse")
+
+def python_doc_deps(interpreter):
+ pip_parse(
+ name = "com_github_scionproto_scion_python_doc_deps",
+ python_interpreter_target = interpreter,
+ requirements = "@com_github_scionproto_scion//doc:requirements.txt",
+ )

doc/dev/design/BeaconService.md

@@ -4,7 +4,8 @@
 * Status: **outdated**
 
 ---
-⚠️ **NOTE** ⚠️<br/>
+⚠️ **NOTE** ⚠️
+
 Outdated contents! This document is kept for historical purpose.
 
 ---

doc/dev/design/ColibriService.md

@@ -9,7 +9,8 @@
 * Discussion at: [#3653](https://github.com/scionproto/scion/issues/3653), [#3794](https://github.com/scionproto/scion/issues/3794)
 
 ---
-⚠️ **NOTE** ⚠️<br/>
+⚠️ **NOTE** ⚠️
+
 Outdated contents! This document is kept for historical purpose.
 
 ---
@@ -91,7 +92,7 @@ next AS after AS<sub>i</sub> in the path of the reservation.
 
 1. The *COS* store is queried to admit the segment reservation.
 1. The store decides the admission for the reservation (how much bandwidth). It uses the
- _traffic_matrix_ from the configuration package.
+ *traffic_matrix* from the configuration package.
 1. The store saves an intermediate reservation entry in the DB.
 1. If this AS is the last one in the path, the *COS* store saves the reservation as final
  and notifies the previous AS in the path with a reservation response.
@@ -105,7 +106,7 @@ ASes. Whenever that configuration changes, the service should be notified.
 1. The service triggers the creation of a new segment reservation at boot time and whenever
  the segment reservation configuration file changes.
 1. The service reads the configuration file and creates a segment reservation request per each entry.
- * The path used in the request must be obtained using the _path predicate_ in the configuration.
+ * The path used in the request must be obtained using the *path predicate* in the configuration.
 1. The store in the *COS* saves the intermediate request and sends the request to the next AS
  in the path.
 1. If there is a timeout, this store will send a cleanup request to the next AS in the path.