post-processing: consolidate project-to-versions implementations

.gitignore

@@ -11,4 +11,5 @@ files-to-customize.txt
 .netlify
 __pycache__/
 node_modules
+projects-to-versions.json
 rapids-docs-env/

_includes/api-docs.html

@@ -8,7 +8,15 @@
 {% assign versions = api.versions | sort | where_exp: "item", "item[1] == 1" | join: "" | split: "1" | reverse %}
 ### {{ api.name }}
 {{ api.desc }}
-#### DOCS {% for version_name in versions %} {% if api.name == "libucxx" %} **[{{ version_name }} ({{ site.data.releases[version_name].ucxx_version }})](/api/{{ api.path }}/{{ version_name }})** {% else %} **[{{ version_name }} ({{ site.data.releases[version_name].version }})](/api/{{ api.path }}/{{ version_name }})** {% endif %} {% unless forloop.last %}|{% endunless %} {% endfor %}
+#### DOCS {% for version_name in versions -%}
+    {%- if api.name == "libucxx" -%}
+        **[{{ version_name }} ({{ site.data.releases[version_name].ucxx_version }})](/api/{{ api.path }}/{{ version_name }})**
+    {%- else -%}
+        **[{{ version_name }} ({{ site.data.releases[version_name].version }})](/api/{{ api.path }}/{{ version_name }})**
+    {%- endif -%}
+    {%- unless forloop.last %} | {% endunless -%}
+{%- endfor %}
+
 #### LINKS {% if api.cllink %} **[changelog]({{ api.cllink }}){:target="_blank"}** | {% endif %} **[github]({{ api.ghlink }}){:target="_blank"}**
 {: .mb-7 }
 {% endif %}

ci/customization/customize_doc.py

@@ -20,38 +20,23 @@
 FA_TAG_ID = "rapids-fa-tag"
 
 
-class r_versions(str):
-    def compare(self, other: str) -> int:
-        yearA, monthA = map(int, self.split("."))
-        yearB, monthB = map(int, other.split("."))
-
-        if yearA < yearB or (yearA == yearB and monthA < monthB):
-            return -1
-        elif yearA == yearB and monthA == monthB:
-            return 0
-        else:
-            return 1
-
-    def is_less_than(self, other: str) -> bool:
-        return self.compare(other) == -1
-
-    def is_greater_than(self, other: str) -> bool:
-        return self.compare(other) == 1
-
-
 def get_version_from_fp(*, filepath: str, versions_dict: dict):
     """
     Determines if the current HTML document is for legacy, stable, or nightly versions
     based on the file path
     """
     match = re.search(r"/(\d?\d\.\d\d)/", filepath)
-    version_number_str = r_versions(match.group(1))
-    version_name = "stable"
-    if version_number_str.is_greater_than(versions_dict["stable"]):
-        version_name = "nightly"
-    if version_number_str.is_less_than(versions_dict["stable"]):
-        version_name = "legacy"
-    return {"name": version_name, "number": version_number_str}
+    version_number_from_filepath = match.group(1)
+
+    # given a version number like "25.10", figure out the corresponding version name like "stable", "nightly", or "legacy"
+    for version_name, version_number in versions_dict.items():
+        if version_number == version_number_from_filepath:
+            return {"name": version_name, "number": version_number_from_filepath}
+
+    # if we get here, the version number wasn't found
+    raise ValueError(
+        f"Filepath implies version '{version_number_from_filepath}', no matching entry in versions_dict: {versions_dict}"
+    )
 
 
 def get_lib_from_fp(*, filepath: str, lib_path_dict: dict) -> str:
@@ -293,21 +278,20 @@ def get_theme_info(soup, *, filepath: str):
     )
 
 
-def main(*, filepath: str, lib_path_dict: dict, versions_dict: dict[str, str]) -> None:
+def main(
+    *,
+    filepath: str,
+    lib_path_dict: dict,
+    project_name: str,
+    versions_dict: dict[str, str],
+) -> None:
     """
     Given the path to a documentation HTML file, this function will
     parse the file and add library/version selectors and a Home button
     """
 
-    # parse CLI arguments
     print(f"--- {filepath} ---")
 
-    # determine project name (e.g. 'cudf')
-    project_name = get_lib_from_fp(
-        lib_path_dict=lib_path_dict,
-        filepath=filepath,
-    )
-
     with open(filepath) as fp:
         soup = BeautifulSoup(fp, "html5lib")
 
@@ -364,33 +348,32 @@ def main(*, filepath: str, lib_path_dict: dict, versions_dict: dict[str, str]) -
 
 
 if __name__ == "__main__":
-    # read in config files (doing this here so it only happens once)
+    MANIFEST_FILEPATH = sys.argv[1]
+    PROJECT_TO_VERSIONS_PATH = sys.argv[2]
     LIB_MAP_PATH = os.path.join(os.path.dirname(__file__), "lib_map.json")
-    RELEASES_PATH = os.path.join(
-        os.path.dirname(__file__), "../", "../", "_data", "releases.json"
-    )
 
+    # read in config files (doing this here so it only happens once)
     with open(LIB_MAP_PATH) as fp:
         LIB_PATH_DICT = json.load(fp)
 
-    with open(RELEASES_PATH) as fp:
-        RELEASE_DATA = json.load(fp)
+    with open(PROJECT_TO_VERSIONS_PATH) as fp:
+        PROJECT_TO_VERSIONS_DICT = json.load(fp)
 
-    MANIFEST_FILEPATH = sys.argv[1]
     with open(MANIFEST_FILEPATH) as manifest_file:
         for line in manifest_file:
             filepath = line.strip()
-            version_key = "version"
-            if "ucxx" in filepath:
-                version_key = "ucxx_version"
-
-            versions_dict = {
-                "legacy": RELEASE_DATA["legacy"][version_key],
-                "nightly": RELEASE_DATA["nightly"][version_key],
-                "stable": RELEASE_DATA["stable"][version_key],
-            }
+
+            lib_path_dict = deepcopy(LIB_PATH_DICT)
+
+            # determine project name (e.g. 'cudf')
+            project_name = get_lib_from_fp(
+                lib_path_dict=lib_path_dict,
+                filepath=filepath,
+            )
+
             main(
                 filepath=filepath,
-                lib_path_dict=deepcopy(LIB_PATH_DICT),
-                versions_dict=deepcopy(versions_dict),
+                lib_path_dict=lib_path_dict,
+                project_name=project_name,
+                versions_dict=deepcopy(PROJECT_TO_VERSIONS_DICT[project_name]),
             )

ci/customization/customize_docs_in_folder.sh

@@ -23,8 +23,9 @@ display_usage() {
 }
 
 FOLDER_TO_CUSTOMIZE=$1
+PROJECTS_TO_VERSIONS_PATH=$2
 
-if [[ $# -ne 1 ]]; then
+if [[ $# -ne 2 ]]; then
   display_usage
   exit 1
 fi
@@ -51,5 +52,8 @@ grep "${JTD_SEARCH_TERM}\|${DOXYGEN_SEARCH_TERM}\|${PYDATA_SEARCH_TERM}" -rl \
 > "${MANIFEST_FILE}"
 
 echo "Customizing $(wc -l < ${MANIFEST_FILE} | tr -d ' ') HTML files"
-python -u ${SCRIPT_SRC_FOLDER}/customize_doc.py "${MANIFEST_FILE}"
+python -u ${SCRIPT_SRC_FOLDER}/customize_doc.py \
+  "${MANIFEST_FILE}" \
+  "${PROJECTS_TO_VERSIONS_PATH}"
+
 echo "Done customizing"

ci/download_from_s3.sh

@@ -2,7 +2,8 @@
 # SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES.
 # All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
-# Copies the RAPIDS libraries' HTML files from S3 into the "_site" directory of
+#
+# Copies the RAPIDS projects' HTML files from S3 into the "_site" directory of
 # the Jekyll build.
 set -euo pipefail
 
@@ -64,50 +65,52 @@ aws_cp() {
 }
 
 # Downloads the RAPIDS libraries' documentation files from S3 and places them
-# into the "_site/api" folder. The versions that should be copied are read from
-# "_data/releases.json" and the libraries that should be copied are read from
-# "_data/docs.yml".
+# into the "_site/api" folder.
 download_lib_docs() {
-  local DST PROJECT PROJECT_MAP \
-        SRC VERSION_MAP VERSION_NAME \
-        VERSION_NUMBER
-
-  VERSION_MAP=$(jq '{
-    "legacy": { "version": .legacy.version, "ucxx_version": .legacy.ucxx_version },
-    "stable": { "version": .stable.version, "ucxx_version": .stable.ucxx_version },
-    "nightly": { "version": .nightly.version, "ucxx_version": .nightly.ucxx_version }
-  }' _data/releases.json)
-
-  PROJECT_MAP=$(yq '.apis + .libs' _data/docs.yml)
-
-  for VERSION_NAME in $(jq -r 'keys | .[]' <<< "$VERSION_MAP"); do
-    for PROJECT in $(yq -r 'keys | .[]' <<< "$PROJECT_MAP"); do
-      VERSION_NUMBER=$(jq -r --arg vn "$VERSION_NAME" --arg pr "$PROJECT" '
-        if ($pr | contains("ucxx")) then
-          .[$vn].ucxx_version
-        else
-          .[$vn].version
-        end' <<< "$VERSION_MAP")
-
-      PROJECT_MAP_JSON=$(yq -r -o json '.' <<< "$PROJECT_MAP")
-      if [ "$(jq -r --arg pr "$PROJECT" --arg vn "$VERSION_NAME" '.[$pr].versions[$vn]' <<< "$PROJECT_MAP_JSON")" == "0" ]; then
-        echo "Skipping: $PROJECT | $VERSION_NAME | $VERSION_NUMBER"
-        continue
-      fi
-
+  local DST PROJECT PROJECTS_TO_VERSIONS_JSON \
+    SRC VERSION_NAME VERSION_NUMBER
+
+  echo "--- processing RAPIDS libraries ---"
+  PROJECTS_TO_VERSIONS_JSON=$(./ci/get-projects-to-versions.sh)
+  for PROJECT in $(jq -r 'keys | .[]' <<< "${PROJECTS_TO_VERSIONS_JSON}"); do
+
+    # extract the map of versions to download for this project, which will look something like:
+    #
+    # {"stable": 25.10, "nightly": 25.12, "legacy": 25.08}
+    #
+    # With keys varying based on which types of docs we want to build for this particular project.
+    VERSIONS_FOR_THIS_PROJECT=$(
+      jq \
+        -r \
+        --arg pr "${PROJECT}" \
+        '.[$pr]' \
+      <<< "${PROJECTS_TO_VERSIONS_JSON}"
+    )
+
+    # loop over 'stable', 'nightly', etc.
+    for VERSION_NAME in $(jq -r 'keys | .[]' <<< "${VERSIONS_FOR_THIS_PROJECT}"); do
+      VERSION_NUMBER=$(
+        jq \
+          -r \
+          --arg version_name "${VERSION_NAME}" \
+          '.[$version_name]' \
+        <<< "${VERSIONS_FOR_THIS_PROJECT}"
+      )
+      # copy the relevant files from S3 to the local directory
       SRC="s3://${DOCS_BUCKET}/${PROJECT}/html/${VERSION_NUMBER}/"
       DST="$(yq -n 'env(GENERATED_DIRS)|.libs')/${PROJECT}/${VERSION_NUMBER}/"
-
       aws_cp "${SRC}" "${DST}"
-    done
-  done
+    done  # for VERSION_NAME
+
+  done  # for PROJECT
 }
 
 # Downloads the deployment docs from S3 and places them in the
 # "_site/deployment" directory.
 download_deployment_docs() {
   local DST SRC VERSION
 
+  echo "--- processing deployment docs ---"
   for VERSION in nightly stable; do
     SRC="s3://${DOCS_BUCKET}/deployment/html/${VERSION}/"
     DST="$(yq -n 'env(GENERATED_DIRS)|.deployment')/${VERSION}/"

ci/get-projects-to-versions.sh

@@ -0,0 +1,85 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# [description]
+#
+# Determines:
+#
+#   * which RAPIDS libraries to host docs for
+#   * what types of docs to host ('legacy', 'nightly', 'stable', etc.)
+#   * what versions to map to those types
+#
+# The libraries that should be copied are read from "_data/docs.yml".
+#
+# Produces a JSON mapping of the form:
+#
+#     {
+#         "{project}": {
+#             "stable": {version_number},
+#             "legacy": {version_number},
+#             "nightly": {version_number}
+#          },
+#      }
+#
+# With keys omitted based on configuration in _data/docs.yml.
+#
+# e.g. if a project has 'stable: 0' in that file, it will not have a '{project}.stable'
+# key in the mapping produced by this script.
+#
+# Only that mapping is written to stdout, so this is safe to use inline like this:
+#
+#     PROJECTS_TO_VERSIONS=$(./ci/get-projects-to-versions.sh)
+#
+# WARNING: no guarantees are made about the ordering of output in this mapping.
+#
+
+set -e -E -u -o pipefail
+
+log-stderr() {
+    echo "${1}" >&2
+}
+
+PROJECT_MAP=$(yq '.apis + .libs'  _data/docs.yml)
+
+VERSION_MAP=$(jq '{
+    "legacy": { "version": .legacy.version, "ucxx_version": .legacy.ucxx_version },
+    "stable": { "version": .stable.version, "ucxx_version": .stable.ucxx_version },
+    "nightly": { "version": .nightly.version, "ucxx_version": .nightly.ucxx_version }
+}' _data/releases.json)
+
+PROJECTS_TO_VERSIONS='{}'
+
+for PROJECT in $(yq -r 'keys | .[]' <<< "$PROJECT_MAP"); do
+    THIS_PROJECT_MAP="{\"${PROJECT}\":{}}"
+    for VERSION_NAME in $(jq -r 'keys | .[]' <<< "$VERSION_MAP"); do
+        VERSION_NUMBER=$(jq -r --arg vn "$VERSION_NAME" --arg pr "$PROJECT" '
+        if ($pr | contains("ucxx")) then
+            .[$vn].ucxx_version
+        else
+            .[$vn].version
+        end' <<< "$VERSION_MAP")
+        PROJECT_MAP_JSON=$(yq -r -o json '.' <<< "$PROJECT_MAP")
+        if [ "$(jq -r --arg pr "$PROJECT" --arg vn "$VERSION_NAME" '.[$pr].versions[$vn]' <<< "$PROJECT_MAP_JSON")" == "0" ]; then
+            log-stderr "Skipping: $PROJECT | $VERSION_NAME | $VERSION_NUMBER"
+            continue
+        fi
+        THIS_PROJECT_MAP=$(
+            jq \
+                --arg pr "${PROJECT}" \
+                --arg version_name "${VERSION_NAME}" \
+                --arg version_number "${VERSION_NUMBER}" \
+                '.[$pr] |= . + {$version_name: $version_number}' \
+            <<< "${THIS_PROJECT_MAP}"
+        )
+    done
+    # add this new entry to the mapping
+    PROJECTS_TO_VERSIONS=$(
+        jq --slurp \
+            'map(to_entries) | flatten | group_by(.key) | map({key: .[0].key, value: map(.value) | add}) | from_entries' \
+        <<< "${PROJECTS_TO_VERSIONS}${THIS_PROJECT_MAP}"
+    )
+done
+
+echo -n "${PROJECTS_TO_VERSIONS}"

ci/post-process.sh

@@ -9,8 +9,11 @@ CURRENT_DIR=$(dirname $(realpath $0))
 
 pip install -r "${CURRENT_DIR}/customization/requirements.txt"
 
-"${CURRENT_DIR}"/update_symlinks.sh
+PROJECTS_TO_VERSIONS_PATH="${CURRENT_DIR}"/customization/projects-to-versions.json
+"${CURRENT_DIR}"/get-projects-to-versions.sh > "${PROJECTS_TO_VERSIONS_PATH}"
+
+"${CURRENT_DIR}"/update_symlinks.sh "${PROJECTS_TO_VERSIONS_PATH}"
 
 "${CURRENT_DIR}"/customization/lib_map.sh
 
-"${CURRENT_DIR}"/customization/customize_docs_in_folder.sh "_site/api"
+"${CURRENT_DIR}"/customization/customize_docs_in_folder.sh "_site/api" "${PROJECTS_TO_VERSIONS_PATH}"