post-processing: re-write generate-projects-to-versions in Python

.pre-commit-config.yaml

@@ -6,9 +6,6 @@ ci:
   autoupdate_branch: ""
   autoupdate_commit_msg: "[pre-commit.ci] pre-commit autoupdate"
   autoupdate_schedule: quarterly
-  skip:
-    # requires 'jq' and 'yq', which don't come pre-installed in the pre-commit.ci image
-    - generate-projects-to-versions
   submodules: false
 
 repos:
@@ -42,9 +39,11 @@ repos:
     hooks:
       - id: generate-projects-to-versions
         name: generate-projects-to-versions
-        entry: ./ci/generate-projects-to-versions.sh
-        language: system
+        entry: ./ci/generate-projects-to-versions.py
+        language: python
         pass_filenames: false
+        additional_dependencies:
+          - pyyaml
   - repo: https://github.com/sirosen/texthooks
     rev: 0.7.1
     hooks:

ci/README.md

@@ -11,12 +11,13 @@ The steps are roughly as follows.
 
 ### Step 1: determine libraries and versions to build
 
-`get-projects-to-versions.sh` is responsible for holding all the logic relevant to the questions "What projects' API docs should be hosted? What versions?".
+`generate-projects-to-versions.py` is responsible for holding all the logic relevant to the questions "What projects' API docs should be hosted? What versions?".
 
 Run it from the root of the repo to see for yourself.
 
 ```shell
-./ci/get-projects-to-versions.sh
+pre-commit run --all-files generate-projects-to-versions
+cat ./ci/customization/projects-to-versions.json
 ```
 
 That script is reused by other automation to determine which projects and versions to build.

ci/customization/README.md

@@ -40,12 +40,12 @@ Building the API docs requires answering these questions:
 * what version types? (stable? legacy? nightly?)
 * what version numbers correspond to those version types?
 
-Logic for all of that is centralized in a script.
+Logic for all of that is centralized in a script which is run by `pre-commit`.
 
 Invoke it to see what will be built.
 
 ```shell
-./ci/get-projects-to-versions.sh > ./ci/customization/projects-to-versions.json
+pre-commit run --all-files generate-projects-to-versions
 ```
 
 ### Symlinks

ci/customization/projects-to-versions.json

@@ -1,112 +1,112 @@
 {
-  "cucim": {
+  "cudf": {
     "legacy": "25.06",
-    "nightly": "25.10",
-    "stable": "25.08"
+    "stable": "25.08",
+    "nightly": "25.10"
   },
-  "cudf": {
+  "dask-cudf": {
     "legacy": "25.06",
-    "nightly": "25.10",
-    "stable": "25.08"
+    "stable": "25.08",
+    "nightly": "25.10"
   },
-  "cudf-java": {
+  "cuml": {
     "legacy": "25.06",
-    "stable": "25.08"
+    "stable": "25.08",
+    "nightly": "25.10"
   },
   "cugraph": {
     "legacy": "25.06",
-    "nightly": "25.10",
-    "stable": "25.08"
+    "stable": "25.08",
+    "nightly": "25.10"
   },
-  "cuml": {
+  "cuxfilter": {
     "legacy": "25.06",
-    "nightly": "25.10",
-    "stable": "25.08"
+    "stable": "25.08",
+    "nightly": "25.10"
   },
-  "cuproj": {
-    "legacy": "25.02",
-    "stable": "25.04"
+  "cudf-java": {
+    "legacy": "25.06",
+    "stable": "25.08"
   },
-  "cusignal": {},
-  "cuspatial": {
-    "legacy": "25.02",
-    "stable": "25.04"
+  "cucim": {
+    "legacy": "25.06",
+    "stable": "25.08",
+    "nightly": "25.10"
   },
   "cuvs": {
     "legacy": "25.06",
-    "nightly": "25.10",
-    "stable": "25.08"
+    "stable": "25.08",
+    "nightly": "25.10"
   },
-  "cuxfilter": {
+  "kvikio": {
     "legacy": "25.06",
-    "nightly": "25.10",
-    "stable": "25.08"
+    "stable": "25.08",
+    "nightly": "25.10"
+  },
+  "raft": {
+    "legacy": "25.06",
+    "stable": "25.08",
+    "nightly": "25.10"
   },
   "dask-cuda": {
     "legacy": "25.06",
-    "nightly": "25.10",
-    "stable": "25.08"
+    "stable": "25.08",
+    "nightly": "25.10"
   },
-  "dask-cudf": {
+  "rmm": {
     "legacy": "25.06",
-    "nightly": "25.10",
-    "stable": "25.08"
+    "stable": "25.08",
+    "nightly": "25.10"
   },
-  "kvikio": {
+  "rapidsmpf": {
+    "stable": "25.08",
+    "nightly": "25.10"
+  },
+  "librmm": {
     "legacy": "25.06",
-    "nightly": "25.10",
-    "stable": "25.08"
+    "stable": "25.08",
+    "nightly": "25.10"
   },
   "libcudf": {
     "legacy": "25.06",
-    "nightly": "25.10",
-    "stable": "25.08"
+    "stable": "25.08",
+    "nightly": "25.10"
   },
   "libcuml": {
     "legacy": "25.06",
-    "nightly": "25.10",
-    "stable": "25.08"
-  },
-  "libcuproj": {
-    "legacy": "25.02",
-    "stable": "25.04"
-  },
-  "libcuspatial": {
-    "legacy": "25.02",
-    "stable": "25.04"
+    "stable": "25.08",
+    "nightly": "25.10"
   },
   "libkvikio": {
     "legacy": "25.06",
-    "nightly": "25.10",
-    "stable": "25.08"
-  },
-  "librmm": {
-    "legacy": "25.06",
-    "nightly": "25.10",
-    "stable": "25.08"
+    "stable": "25.08",
+    "nightly": "25.10"
   },
   "libucxx": {
     "legacy": "0.44",
-    "nightly": "0.46",
-    "stable": "0.45"
-  },
-  "raft": {
-    "legacy": "25.06",
-    "nightly": "25.10",
-    "stable": "25.08"
+    "stable": "0.45",
+    "nightly": "0.46"
   },
   "rapids-cmake": {
     "legacy": "25.06",
-    "nightly": "25.10",
-    "stable": "25.08"
+    "stable": "25.08",
+    "nightly": "25.10"
   },
-  "rapidsmpf": {
-    "nightly": "25.10",
-    "stable": "25.08"
+  "cuproj": {
+    "legacy": "25.02",
+    "stable": "25.04"
   },
-  "rmm": {
-    "legacy": "25.06",
-    "nightly": "25.10",
-    "stable": "25.08"
+  "cusignal": {},
+  "cuspatial": {
+    "legacy": "25.02",
+    "stable": "25.04"
+  },
+  "libcuproj": {
+    "legacy": "25.02",
+    "stable": "25.04"
+  },
+  "libcuspatial": {
+    "legacy": "25.02",
+    "stable": "25.04"
   }
 }

ci/generate-projects-to-versions.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# [description]
+#
+# Determines:
+#
+#   * which RAPIDS libraries to host docs for
+#   * what types of docs to host ('legacy', 'nightly', 'stable', etc.)
+#   * what versions to map to those types
+#
+# The libraries that should be copied are read from "_data/docs.yml".
+#
+# The versions that should be copied are read from a mix of sources:
+#
+#   - active projects: "_data/releases.json"
+#   - inactive projects: 'version-overrides' field in entries in "_data/docs.yml"
+#
+# Produces a JSON mapping of the form:
+#
+#     {
+#         "{project}": {
+#             "stable": {version_number},
+#             "legacy": {version_number},
+#             "nightly": {version_number}
+#          },
+#      }
+#
+# With keys omitted based on configuration in _data/docs.yml.
+#
+# e.g. if a project has 'stable: 0' in that file, it will not have a '{project}.stable'
+# key in the mapping produced by this script.
+#
+import json
+import sys
+import yaml
+from collections import OrderedDict
+
+with open("_data/docs.yml") as f:
+    DOCS_YML_DICT = yaml.safe_load(f)
+
+with open("_data/releases.json") as f:
+    RELEASES_JSON_DICT = json.load(f)
+
+# using OrderedDict minimizes churn in the output as projects are added and removed
+PROJECTS_TO_VERSIONS_DICT = OrderedDict()
+
+for docs_key in ["apis", "libs", "inactive-projects"]:
+    for project_name, project_details in DOCS_YML_DICT[docs_key].items():
+        print(f"Processing: {project_name}", file=sys.stderr)
+        # what entry from releases.json should be used to find version numbers?
+        version_key = "version"
+        if "ucxx" in project_name:
+            version_key = "ucxx_version"
+
+        # what versions should be built for this project?
+        versions_for_this_project = OrderedDict()
+        for version_name, should_include in project_details["versions"].items():
+            if should_include == 1:
+                version_override = project_details.get("version-overrides", dict()).get(
+                    version_name, ""
+                )
+                if version_override:
+                    versions_for_this_project[version_name] = version_override
+                else:
+                    versions_for_this_project[version_name] = RELEASES_JSON_DICT[
+                        version_name
+                    ][version_key]
+            else:
+                print(f"Skipping: {project_name} | {version_name}", file=sys.stderr)
+
+        # update overall mapping
+        PROJECTS_TO_VERSIONS_DICT[project_name] = versions_for_this_project
+
+
+with open("ci/customization/projects-to-versions.json", "w") as f:
+    json.dump(PROJECTS_TO_VERSIONS_DICT, f, indent=2)
+    f.write("\n")