Adding support nightly cuopt-examples notebook testing

.github/workflows/pr.yaml

@@ -35,7 +35,6 @@ jobs:
       - conda-python-tests
       - docs-build
       - wheel-build-libcuopt
-      # - conda-notebook-tests
       - wheel-build-cuopt
       - wheel-tests-cuopt
       - wheel-build-cuopt-server

.github/workflows/test.yaml

@@ -69,3 +69,15 @@ jobs:
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
       script: ci/test_wheel_cuopt_server.sh
+  conda-notebook-tests:
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.10
+    with:
+      build_type: ${{ inputs.build_type }}
+      branch: ${{ inputs.branch }}
+      date: ${{ inputs.date }}
+      sha: ${{ inputs.sha }}
+      node_type: "gpu-l4-latest-1"
+      arch: "amd64"
+      container_image: "rapidsai/ci-conda:25.10-latest"
+      script: ci/test_notebooks.sh

ci/test_notebooks.sh

@@ -21,13 +21,22 @@ set -euo pipefail
 
 CUOPT_VERSION="$(rapids-version)"
 
+rapids-logger "Downloading artifacts from previous jobs"
+CPP_CHANNEL=$(rapids-download-conda-from-github cpp)
+PYTHON_CHANNEL=$(rapids-download-conda-from-github python)
+
 rapids-logger "Generate notebook testing dependencies"
+
+ENV_YAML_DIR="$(mktemp -d)"
+
 rapids-dependency-file-generator \
   --output conda \
   --file-key test_notebooks \
-  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+  --prepend-channel "${CPP_CHANNEL}" \
+  --prepend-channel "${PYTHON_CHANNEL}" \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${ENV_YAML_DIR}/env.yaml"
 
-rapids-mamba-retry env create --yes -f env.yaml -n test
+rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n test
 
 # Temporarily allow unbound variables for conda activation.
 set +u
@@ -36,46 +45,37 @@ set -u
 
 rapids-print-env
 
-rapids-logger "Downloading artifacts from previous jobs"
-CPP_CHANNEL=$(rapids-download-conda-from-github cpp)
-PYTHON_CHANNEL=$(rapids-download-conda-from-github python)
+EXAMPLES_BRANCH="branch-${CUOPT_VERSION%.*}"
 
-rapids-mamba-retry install \
-  --channel "${CPP_CHANNEL}" \
-  --channel "${PYTHON_CHANNEL}" \
-  "libcuopt=${CUOPT_VERSION}" \
-  "cuopt=${CUOPT_VERSION}" \
-  "cuopt-server=${CUOPT_VERSION}"
+# Remove any existing cuopt-examples directory
 
-pip install python/cuopt_self_hosted/
+rapids-logger "Cloning cuopt-examples repository for branch: ${EXAMPLES_BRANCH}"
+rm -rf cuopt-examples
+git clone --single-branch --branch "${EXAMPLES_BRANCH}" https://github.com/NVIDIA/cuopt-examples.git
 
 NBTEST="$(realpath "$(dirname "$0")/utils/nbtest.sh")"
 NBLIST_PATH="$(realpath "$(dirname "$0")/utils/notebook_list.py")"
-NBLIST=$(python "${NBLIST_PATH}")
-SERVER_WAIT_DELAY=10
 
-pushd notebooks
+pushd cuopt-examples
+
+NBLIST=$(python "${NBLIST_PATH}")
 
 EXITCODE=0
 trap "EXITCODE=1" ERR
 
 rapids-logger "Start cuopt-server"
 
 set +e
-#python -c "from cuopt_server.cuopt_service import run_server; run_server()" &
-
-python -m cuopt_server.cuopt_service &
-export SERVER_PID=$!
-sleep "${SERVER_WAIT_DELAY}"
-curl http://0.0.0.0:5000/cuopt/health
 
 rapids-logger "Start notebooks tests"
 for nb in ${NBLIST}; do
   nvidia-smi
   ${NBTEST} "${nb}"
+  if [ $? -ne 0 ]; then
+    echo "Notebook ${nb} failed to execute. Exiting."
+    exit 1
+  fi
 done
 
 rapids-logger "Notebook test script exiting with value: $EXITCODE"
-kill -s SIGTERM $SERVER_PID
-wait $SERVER_PID
 exit ${EXITCODE}

ci/utils/nbtest.sh

@@ -12,75 +12,70 @@
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# See the License for the specific language governing permissions and limitations.
+#
+# This script executes Jupyter notebooks directly using nbconvert.
 
 set +e           # do not abort the script on error
 set -o pipefail  # piped commands propagate their error
 set -E           # ERR traps are inherited by subcommands
 trap "EXITCODE=1" ERR
 
-# Prepend the following code to all scripts generated from nbconvert.  This
-# allows all cell and line magic code to run and update the namespace as if
-# running in jupyter, but will also tolerate failures due to running in a
-# non-jupyter env.
-# Note: depending on the assumptions of the notebook script, ignoring failures
-# may not be acceptable (meaning the converted notebook simply cannot run
-# outside of jupyter as-is), hence the warning.
-MAGIC_OVERRIDE_CODE="
-def my_run_line_magic(*args, **kwargs):
-    g=globals()
-    l={}
-    for a in args:
-        try:
-            exec(str(a),g,l)
-        except Exception as e:
-            print('WARNING: %s\n   While executing this magic function code:\n%s\n   continuing...\n' % (e, a))
-        else:
-            g.update(l)
-
-def my_run_cell_magic(*args, **kwargs):
-    my_run_line_magic(*args, **kwargs)
-
-get_ipython().run_line_magic=my_run_line_magic
-get_ipython().run_cell_magic=my_run_cell_magic
-
-"
-
-NO_COLORS=--colors=NoColor
-EXITCODE=0
+# Save the original directory
+ORIGINAL_DIR=$(pwd)
 
-# PWD is REPO_ROOT/notebooks
-NBTMPDIR="${PWD}/../tmp"
-mkdir -p "${NBTMPDIR}"
-NBUTILS="${PWD}/external"
-cp -r "${NBUTILS}/python/utils" "${NBTMPDIR}/."
-cp -r "${NBUTILS}/server/notebook_utils" "${NBTMPDIR}/."
-cp -r "${NBUTILS}/dli/helper_function" "${NBTMPDIR}/."
-cd "${NBTMPDIR}" || exit 1
+EXITCODE=0
 
 for nb in "$@"; do
     NBFILENAME=$nb
     NBNAME=${NBFILENAME%.*}
     NBNAME=${NBNAME##*/}
-    NBTESTSCRIPT=${NBTMPDIR}/${NBNAME}-test.py
-    shift
+
+    # Get the directory where the notebook is located
+    NBDIR=$(dirname "$NBFILENAME")
+
+    cd "${NBDIR}" || exit 1
+
+    # Output the executed notebook in the same folder
+    EXECUTED_NOTEBOOK="${NBNAME}-executed.ipynb"
 
     echo --------------------------------------------------------------------------------
     echo STARTING: "${NBNAME}"
     echo --------------------------------------------------------------------------------
-    jupyter nbconvert --to script ../"${NBFILENAME}" --output "${NBTMPDIR}"/"${NBNAME}"-test
-    python "${PWD}/../ci/utils/dli_nb_strip.py" "${NBTESTSCRIPT}"
-    echo "${MAGIC_OVERRIDE_CODE}" > "${NBTMPDIR}"/tmpfile
-    cat "${NBTESTSCRIPT}" >> "${NBTMPDIR}"/tmpfile
-    mv "${NBTMPDIR}"/tmpfile "${NBTESTSCRIPT}"
-
-    echo "Running \"ipython ${NO_COLORS} ${NBTESTSCRIPT}\" on $(date)"
-    echo
-    time timeout 30m bash -c "ipython ${NO_COLORS} ${NBTESTSCRIPT}; EC=\$?; echo -------------------------------------------------------------------------------- ; echo DONE: ${NBNAME}; exit \$EC"
-    NBEXITCODE=$?
-    echo EXIT CODE: ${NBEXITCODE}
-    echo
+
+    # Skip notebooks that are not yet supported
+    SKIP_NOTEBOOKS=(
+        "trnsport_cuopt"
+        "Production_Planning_Example_Pulp"
+        "Simple_LP_pulp"
+        "Simple_MIP_pulp"
+        "Sudoku_pulp"
+    )
+
+    for skip in "${SKIP_NOTEBOOKS[@]}"; do
+        if [[ "$NBNAME" == "$skip"* ]]; then
+            echo "Skipping notebook '${NBNAME}' as it matches skip pattern '${skip}'"
+            cd "$ORIGINAL_DIR" || exit 1
+            continue 2
+        fi
+    done
+
+    rapids-logger "Running commands from notebook: ${NBNAME}.ipynb"
+
+    python3 "$ORIGINAL_DIR/../ci/utils/notebook_command_extractor.py" "$NBNAME.ipynb" --verbose
+
+    rapids-logger "Executing notebook: ${NBNAME}.ipynb"
+    # Execute notebook with default kernel
+    jupyter nbconvert --execute "${NBNAME}.ipynb" --to notebook --output "${EXECUTED_NOTEBOOK}" --ExecutePreprocessor.kernel_name="python3"
+
+    if [ $? -eq 0 ]; then
+        echo "Notebook executed successfully: ${EXECUTED_NOTEBOOK}"
+    else
+        echo "ERROR: Failed to execute notebook: ${NBFILENAME}"
+        EXITCODE=1
+    fi
+
+    cd "${ORIGINAL_DIR}" || exit 1
 done
 
 exit ${EXITCODE}