diff --git a/.gitignore b/.gitignore index d2e9d00015e..15e57da1a36 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,5 @@ files-to-customize.txt .netlify __pycache__/ node_modules +projects-to-versions.json rapids-docs-env/ diff --git a/_includes/api-docs.html b/_includes/api-docs.html index da5a66cc507..b022cc099cd 100644 --- a/_includes/api-docs.html +++ b/_includes/api-docs.html @@ -8,7 +8,15 @@ {% assign versions = api.versions | sort | where_exp: "item", "item[1] == 1" | join: "" | split: "1" | reverse %} ### {{ api.name }} {{ api.desc }} -#### DOCS {% for version_name in versions %} {% if api.name == "libucxx" %} **[{{ version_name }} ({{ site.data.releases[version_name].ucxx_version }})](/api/{{ api.path }}/{{ version_name }})** {% else %} **[{{ version_name }} ({{ site.data.releases[version_name].version }})](/api/{{ api.path }}/{{ version_name }})** {% endif %} {% unless forloop.last %}|{% endunless %} {% endfor %} +#### DOCS {% for version_name in versions -%} + {%- if api.name == "libucxx" -%} + **[{{ version_name }} ({{ site.data.releases[version_name].ucxx_version }})](/api/{{ api.path }}/{{ version_name }})** + {%- else -%} + **[{{ version_name }} ({{ site.data.releases[version_name].version }})](/api/{{ api.path }}/{{ version_name }})** + {%- endif -%} + {%- unless forloop.last %} | {% endunless -%} +{%- endfor %} + #### LINKS {% if api.cllink %} **[changelog]({{ api.cllink }}){:target="_blank"}** | {% endif %} **[github]({{ api.ghlink }}){:target="_blank"}** {: .mb-7 } {% endif %} diff --git a/ci/customization/customize_doc.py b/ci/customization/customize_doc.py index 1229e904bb4..49eddb4afa6 100644 --- a/ci/customization/customize_doc.py +++ b/ci/customization/customize_doc.py @@ -20,38 +20,23 @@ FA_TAG_ID = "rapids-fa-tag" -class r_versions(str): - def compare(self, other: str) -> int: - yearA, monthA = map(int, self.split(".")) - yearB, monthB = map(int, other.split(".")) - - if yearA < yearB or (yearA == yearB and monthA < monthB): - return -1 - elif yearA == yearB and monthA == monthB: - return 0 - else: - return 1 - - def is_less_than(self, other: str) -> bool: - return self.compare(other) == -1 - - def is_greater_than(self, other: str) -> bool: - return self.compare(other) == 1 - - def get_version_from_fp(*, filepath: str, versions_dict: dict): """ Determines if the current HTML document is for legacy, stable, or nightly versions based on the file path """ match = re.search(r"/(\d?\d\.\d\d)/", filepath) - version_number_str = r_versions(match.group(1)) - version_name = "stable" - if version_number_str.is_greater_than(versions_dict["stable"]): - version_name = "nightly" - if version_number_str.is_less_than(versions_dict["stable"]): - version_name = "legacy" - return {"name": version_name, "number": version_number_str} + version_number_from_filepath = match.group(1) + + # given a version number like "25.10", figure out the corresponding version name like "stable", "nightly", or "legacy" + for version_name, version_number in versions_dict.items(): + if version_number == version_number_from_filepath: + return {"name": version_name, "number": version_number_from_filepath} + + # if we get here, the version number wasn't found + raise ValueError( + f"Filepath implies version '{version_number_from_filepath}', no matching entry in versions_dict: {versions_dict}" + ) def get_lib_from_fp(*, filepath: str, lib_path_dict: dict) -> str: @@ -293,21 +278,20 @@ def get_theme_info(soup, *, filepath: str): ) -def main(*, filepath: str, lib_path_dict: dict, versions_dict: dict[str, str]) -> None: +def main( + *, + filepath: str, + lib_path_dict: dict, + project_name: str, + versions_dict: dict[str, str], +) -> None: """ Given the path to a documentation HTML file, this function will parse the file and add library/version selectors and a Home button """ - # parse CLI arguments print(f"--- {filepath} ---") - # determine project name (e.g. 'cudf') - project_name = get_lib_from_fp( - lib_path_dict=lib_path_dict, - filepath=filepath, - ) - with open(filepath) as fp: soup = BeautifulSoup(fp, "html5lib") @@ -364,33 +348,32 @@ def main(*, filepath: str, lib_path_dict: dict, versions_dict: dict[str, str]) - if __name__ == "__main__": - # read in config files (doing this here so it only happens once) + MANIFEST_FILEPATH = sys.argv[1] + PROJECT_TO_VERSIONS_PATH = sys.argv[2] LIB_MAP_PATH = os.path.join(os.path.dirname(__file__), "lib_map.json") - RELEASES_PATH = os.path.join( - os.path.dirname(__file__), "../", "../", "_data", "releases.json" - ) + # read in config files (doing this here so it only happens once) with open(LIB_MAP_PATH) as fp: LIB_PATH_DICT = json.load(fp) - with open(RELEASES_PATH) as fp: - RELEASE_DATA = json.load(fp) + with open(PROJECT_TO_VERSIONS_PATH) as fp: + PROJECT_TO_VERSIONS_DICT = json.load(fp) - MANIFEST_FILEPATH = sys.argv[1] with open(MANIFEST_FILEPATH) as manifest_file: for line in manifest_file: filepath = line.strip() - version_key = "version" - if "ucxx" in filepath: - version_key = "ucxx_version" - - versions_dict = { - "legacy": RELEASE_DATA["legacy"][version_key], - "nightly": RELEASE_DATA["nightly"][version_key], - "stable": RELEASE_DATA["stable"][version_key], - } + + lib_path_dict = deepcopy(LIB_PATH_DICT) + + # determine project name (e.g. 'cudf') + project_name = get_lib_from_fp( + lib_path_dict=lib_path_dict, + filepath=filepath, + ) + main( filepath=filepath, - lib_path_dict=deepcopy(LIB_PATH_DICT), - versions_dict=deepcopy(versions_dict), + lib_path_dict=lib_path_dict, + project_name=project_name, + versions_dict=deepcopy(PROJECT_TO_VERSIONS_DICT[project_name]), ) diff --git a/ci/customization/customize_docs_in_folder.sh b/ci/customization/customize_docs_in_folder.sh index 831891aaf72..a7456d66957 100755 --- a/ci/customization/customize_docs_in_folder.sh +++ b/ci/customization/customize_docs_in_folder.sh @@ -23,8 +23,9 @@ display_usage() { } FOLDER_TO_CUSTOMIZE=$1 +PROJECTS_TO_VERSIONS_PATH=$2 -if [[ $# -ne 1 ]]; then +if [[ $# -ne 2 ]]; then display_usage exit 1 fi @@ -51,5 +52,8 @@ grep "${JTD_SEARCH_TERM}\|${DOXYGEN_SEARCH_TERM}\|${PYDATA_SEARCH_TERM}" -rl \ > "${MANIFEST_FILE}" echo "Customizing $(wc -l < ${MANIFEST_FILE} | tr -d ' ') HTML files" -python -u ${SCRIPT_SRC_FOLDER}/customize_doc.py "${MANIFEST_FILE}" +python -u ${SCRIPT_SRC_FOLDER}/customize_doc.py \ + "${MANIFEST_FILE}" \ + "${PROJECTS_TO_VERSIONS_PATH}" + echo "Done customizing" diff --git a/ci/download_from_s3.sh b/ci/download_from_s3.sh index eaa68bf7aee..82185b3ccc5 100755 --- a/ci/download_from_s3.sh +++ b/ci/download_from_s3.sh @@ -2,7 +2,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. # All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Copies the RAPIDS libraries' HTML files from S3 into the "_site" directory of +# +# Copies the RAPIDS projects' HTML files from S3 into the "_site" directory of # the Jekyll build. set -euo pipefail @@ -64,43 +65,44 @@ aws_cp() { } # Downloads the RAPIDS libraries' documentation files from S3 and places them -# into the "_site/api" folder. The versions that should be copied are read from -# "_data/releases.json" and the libraries that should be copied are read from -# "_data/docs.yml". +# into the "_site/api" folder. download_lib_docs() { - local DST PROJECT PROJECT_MAP \ - SRC VERSION_MAP VERSION_NAME \ - VERSION_NUMBER - - VERSION_MAP=$(jq '{ - "legacy": { "version": .legacy.version, "ucxx_version": .legacy.ucxx_version }, - "stable": { "version": .stable.version, "ucxx_version": .stable.ucxx_version }, - "nightly": { "version": .nightly.version, "ucxx_version": .nightly.ucxx_version } - }' _data/releases.json) - - PROJECT_MAP=$(yq '.apis + .libs' _data/docs.yml) - - for VERSION_NAME in $(jq -r 'keys | .[]' <<< "$VERSION_MAP"); do - for PROJECT in $(yq -r 'keys | .[]' <<< "$PROJECT_MAP"); do - VERSION_NUMBER=$(jq -r --arg vn "$VERSION_NAME" --arg pr "$PROJECT" ' - if ($pr | contains("ucxx")) then - .[$vn].ucxx_version - else - .[$vn].version - end' <<< "$VERSION_MAP") - - PROJECT_MAP_JSON=$(yq -r -o json '.' <<< "$PROJECT_MAP") - if [ "$(jq -r --arg pr "$PROJECT" --arg vn "$VERSION_NAME" '.[$pr].versions[$vn]' <<< "$PROJECT_MAP_JSON")" == "0" ]; then - echo "Skipping: $PROJECT | $VERSION_NAME | $VERSION_NUMBER" - continue - fi - + local DST PROJECT PROJECTS_TO_VERSIONS_JSON \ + SRC VERSION_NAME VERSION_NUMBER + + echo "--- processing RAPIDS libraries ---" + PROJECTS_TO_VERSIONS_JSON=$(./ci/get-projects-to-versions.sh) + for PROJECT in $(jq -r 'keys | .[]' <<< "${PROJECTS_TO_VERSIONS_JSON}"); do + + # extract the map of versions to download for this project, which will look something like: + # + # {"stable": 25.10, "nightly": 25.12, "legacy": 25.08} + # + # With keys varying based on which types of docs we want to build for this particular project. + VERSIONS_FOR_THIS_PROJECT=$( + jq \ + -r \ + --arg pr "${PROJECT}" \ + '.[$pr]' \ + <<< "${PROJECTS_TO_VERSIONS_JSON}" + ) + + # loop over 'stable', 'nightly', etc. + for VERSION_NAME in $(jq -r 'keys | .[]' <<< "${VERSIONS_FOR_THIS_PROJECT}"); do + VERSION_NUMBER=$( + jq \ + -r \ + --arg version_name "${VERSION_NAME}" \ + '.[$version_name]' \ + <<< "${VERSIONS_FOR_THIS_PROJECT}" + ) + # copy the relevant files from S3 to the local directory SRC="s3://${DOCS_BUCKET}/${PROJECT}/html/${VERSION_NUMBER}/" DST="$(yq -n 'env(GENERATED_DIRS)|.libs')/${PROJECT}/${VERSION_NUMBER}/" - aws_cp "${SRC}" "${DST}" - done - done + done # for VERSION_NAME + + done # for PROJECT } # Downloads the deployment docs from S3 and places them in the @@ -108,6 +110,7 @@ download_lib_docs() { download_deployment_docs() { local DST SRC VERSION + echo "--- processing deployment docs ---" for VERSION in nightly stable; do SRC="s3://${DOCS_BUCKET}/deployment/html/${VERSION}/" DST="$(yq -n 'env(GENERATED_DIRS)|.deployment')/${VERSION}/" diff --git a/ci/get-projects-to-versions.sh b/ci/get-projects-to-versions.sh new file mode 100755 index 00000000000..81adf3e2568 --- /dev/null +++ b/ci/get-projects-to-versions.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# [description] +# +# Determines: +# +# * which RAPIDS libraries to host docs for +# * what types of docs to host ('legacy', 'nightly', 'stable', etc.) +# * what versions to map to those types +# +# The libraries that should be copied are read from "_data/docs.yml". +# +# Produces a JSON mapping of the form: +# +# { +# "{project}": { +# "stable": {version_number}, +# "legacy": {version_number}, +# "nightly": {version_number} +# }, +# } +# +# With keys omitted based on configuration in _data/docs.yml. +# +# e.g. if a project has 'stable: 0' in that file, it will not have a '{project}.stable' +# key in the mapping produced by this script. +# +# Only that mapping is written to stdout, so this is safe to use inline like this: +# +# PROJECTS_TO_VERSIONS=$(./ci/get-projects-to-versions.sh) +# +# WARNING: no guarantees are made about the ordering of output in this mapping. +# + +set -e -E -u -o pipefail + +log-stderr() { + echo "${1}" >&2 +} + +PROJECT_MAP=$(yq '.apis + .libs' _data/docs.yml) + +VERSION_MAP=$(jq '{ + "legacy": { "version": .legacy.version, "ucxx_version": .legacy.ucxx_version }, + "stable": { "version": .stable.version, "ucxx_version": .stable.ucxx_version }, + "nightly": { "version": .nightly.version, "ucxx_version": .nightly.ucxx_version } +}' _data/releases.json) + +PROJECTS_TO_VERSIONS='{}' + +for PROJECT in $(yq -r 'keys | .[]' <<< "$PROJECT_MAP"); do + THIS_PROJECT_MAP="{\"${PROJECT}\":{}}" + for VERSION_NAME in $(jq -r 'keys | .[]' <<< "$VERSION_MAP"); do + VERSION_NUMBER=$(jq -r --arg vn "$VERSION_NAME" --arg pr "$PROJECT" ' + if ($pr | contains("ucxx")) then + .[$vn].ucxx_version + else + .[$vn].version + end' <<< "$VERSION_MAP") + PROJECT_MAP_JSON=$(yq -r -o json '.' <<< "$PROJECT_MAP") + if [ "$(jq -r --arg pr "$PROJECT" --arg vn "$VERSION_NAME" '.[$pr].versions[$vn]' <<< "$PROJECT_MAP_JSON")" == "0" ]; then + log-stderr "Skipping: $PROJECT | $VERSION_NAME | $VERSION_NUMBER" + continue + fi + THIS_PROJECT_MAP=$( + jq \ + --arg pr "${PROJECT}" \ + --arg version_name "${VERSION_NAME}" \ + --arg version_number "${VERSION_NUMBER}" \ + '.[$pr] |= . + {$version_name: $version_number}' \ + <<< "${THIS_PROJECT_MAP}" + ) + done + # add this new entry to the mapping + PROJECTS_TO_VERSIONS=$( + jq --slurp \ + 'map(to_entries) | flatten | group_by(.key) | map({key: .[0].key, value: map(.value) | add}) | from_entries' \ + <<< "${PROJECTS_TO_VERSIONS}${THIS_PROJECT_MAP}" + ) +done + +echo -n "${PROJECTS_TO_VERSIONS}" diff --git a/ci/post-process.sh b/ci/post-process.sh index 740e163c590..0bbbbc4f7cd 100755 --- a/ci/post-process.sh +++ b/ci/post-process.sh @@ -9,8 +9,11 @@ CURRENT_DIR=$(dirname $(realpath $0)) pip install -r "${CURRENT_DIR}/customization/requirements.txt" -"${CURRENT_DIR}"/update_symlinks.sh +PROJECTS_TO_VERSIONS_PATH="${CURRENT_DIR}"/customization/projects-to-versions.json +"${CURRENT_DIR}"/get-projects-to-versions.sh > "${PROJECTS_TO_VERSIONS_PATH}" + +"${CURRENT_DIR}"/update_symlinks.sh "${PROJECTS_TO_VERSIONS_PATH}" "${CURRENT_DIR}"/customization/lib_map.sh -"${CURRENT_DIR}"/customization/customize_docs_in_folder.sh "_site/api" +"${CURRENT_DIR}"/customization/customize_docs_in_folder.sh "_site/api" "${PROJECTS_TO_VERSIONS_PATH}" diff --git a/ci/update_symlinks.sh b/ci/update_symlinks.sh index 8540ecb4a89..7940930fcf9 100755 --- a/ci/update_symlinks.sh +++ b/ci/update_symlinks.sh @@ -8,51 +8,61 @@ ####################################### set -euEo pipefail -PROJ_ROOT=$(realpath "$(dirname $(realpath $0))/../") -RELEASES="${PROJ_ROOT}/_data/releases.json" +PROJECTS_TO_VERSIONS_PATH="${1}" -STABLE_VERSION=$(jq -r '.stable.version' < "${RELEASES}") -LEGACY_VERSION=$(jq -r '.legacy.version' < "${RELEASES}") -NIGHTLY_VERSION=$(jq -r '.nightly.version' < "${RELEASES}") +# expect paths to be relative to the project root +PROJ_ROOT=$(realpath "$(dirname $(realpath $0))/../") +pushd "${PROJ_ROOT}" -STABLE_UCXX_VERSION=$(jq -r '.stable.ucxx_version' < "${RELEASES}") -LEGACY_UCXX_VERSION=$(jq -r '.legacy.ucxx_version' < "${RELEASES}") -NIGHTLY_UCXX_VERSION=$(jq -r '.nightly.ucxx_version' < "${RELEASES}") +PROJECTS_TO_VERSIONS_JSON=$(cat "${PROJECTS_TO_VERSIONS_PATH}") echo "Updating symlinks..." echo "" -for FOLDER in _site/api/*/ ; do - if [[ "${FOLDER}" == *"ucxx"* ]]; then - STABLE_FOLDER=$STABLE_UCXX_VERSION - LEGACY_FOLDER=$LEGACY_UCXX_VERSION - NIGHTLY_FOLDER=$NIGHTLY_UCXX_VERSION - else - STABLE_FOLDER=$STABLE_VERSION - LEGACY_FOLDER=$LEGACY_VERSION - NIGHTLY_FOLDER=$NIGHTLY_VERSION +for PROJECT in $(jq -r 'keys | .[]' <<< "${PROJECTS_TO_VERSIONS_JSON}"); do + VERSIONS_FOR_THIS_PROJECT=$( + jq \ + -r \ + --arg pr "${PROJECT}" \ + '.[$pr]' \ + <<< "${PROJECTS_TO_VERSIONS_JSON}" + ) + + if [[ "${VERSIONS_FOR_THIS_PROJECT}" == "{}" ]]; then + echo "skipping '${PROJECT}'... no API docs hosted for this project" + continue fi - cd ${FOLDER} + # expect to find a local folder, relative to the root of the repo, + # named e.g. '_site/api/cudf' + PROJECT_FOLDER="_site/api/${PROJECT}" + pushd "${PROJECT_FOLDER}" echo "" - echo "${FOLDER}--------" + echo "${PROJECT_FOLDER}/--------" - if [ -d "${STABLE_FOLDER}" ]; then - ln -s ${STABLE_FOLDER} stable - ln -s ${STABLE_FOLDER} latest - echo " - stable & latest point to ${STABLE_FOLDER}" - fi + # loop over 'stable', 'nightly', etc. + for VERSION_NAME in $(jq -r 'keys | .[]' <<< "${VERSIONS_FOR_THIS_PROJECT}"); do + # expect to find a directory matching the version number, e.g. '_site/api/cudf/25.10' + VERSION_NUMBER=$( + jq \ + -r \ + --arg version_name "${VERSION_NAME}" \ + '.[$version_name]' \ + <<< "${VERSIONS_FOR_THIS_PROJECT}" + ) + FOLDER_FOR_THIS_VERSION="${VERSION_NUMBER}" - if [ -d "${LEGACY_FOLDER}" ]; then - ln -s ${LEGACY_FOLDER} legacy - echo " - legacy points to ${LEGACY_FOLDER}" - fi - - if [ -d "${NIGHTLY_FOLDER}" ]; then - ln -s ${NIGHTLY_FOLDER} nightly - echo " - nightly points to ${NIGHTLY_FOLDER}" - fi + # map /latest to the same version as /stable + if [[ "${VERSION_NAME}" == "stable" ]]; then + ln -s "${FOLDER_FOR_THIS_VERSION}" stable + ln -s "${FOLDER_FOR_THIS_VERSION}" latest + echo " - 'stable' and 'latest' point to '${FOLDER_FOR_THIS_VERSION}'" + else + ln -s "${FOLDER_FOR_THIS_VERSION}" "${VERSION_NAME}" + echo " - '${VERSION_NAME}' points to '${FOLDER_FOR_THIS_VERSION}'" + fi + done # for VERSION + popd echo "---------------" echo "" - cd ${PROJ_ROOT} -done +done # for PROJECT