Internal PR sync

.github/workflows/service_nightly.yaml

@@ -82,15 +82,16 @@ jobs:
 
       - name: Download latest artifacts from S3
         run: |
-          mkdir wheels
-
-          RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
-
-          RAPIDS_PY_WHEEL_NAME="cuopt_mps_parser" rapids-download-wheels-from-s3 ./wheels
+          # make rapids-download-wheels-from-github download everything to the same directory
+          export RAPIDS_UNZIP_DIR="$(pwd)/wheels"
+          mkdir "${RAPIDS_UNZIP_DIR}"
 
-          RAPIDS_PY_WHEEL_NAME="cuopt_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./wheels
-
-          RAPIDS_PY_WHEEL_NAME="cuopt_server_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./wheels
+          # download latest wheels built from build.yaml
+          RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
+          RAPIDS_PY_WHEEL_NAME="cuopt_mps_parser" rapids-download-wheels-from-github python
+          RAPIDS_PY_WHEEL_NAME="cuopt_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github python
+          RAPIDS_PY_WHEEL_NAME="cuopt_server_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github python
+          RAPIDS_PY_WHEEL_NAME="libcuopt_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github cpp
 
       - name: Generate git commit file for tracking the container
         run: |
@@ -165,15 +166,16 @@ jobs:
 
       - name: Download latest artifacts from S3
         run: |
-          mkdir wheels
-
-          RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
-
-          RAPIDS_PY_WHEEL_NAME="cuopt_mps_parser" rapids-download-wheels-from-s3 ./wheels
+          # make rapids-download-wheels-from-github download everything to the same directory
+          export RAPIDS_UNZIP_DIR="$(pwd)/wheels"
+          mkdir "${RAPIDS_UNZIP_DIR}"
 
-          RAPIDS_PY_WHEEL_NAME="cuopt_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./wheels
-
-          RAPIDS_PY_WHEEL_NAME="cuopt_server_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./wheels
+          # download latest wheels built from build.yaml
+          RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
+          RAPIDS_PY_WHEEL_NAME="cuopt_mps_parser" rapids-download-wheels-from-github python
+          RAPIDS_PY_WHEEL_NAME="cuopt_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github python
+          RAPIDS_PY_WHEEL_NAME="cuopt_server_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github python
+          RAPIDS_PY_WHEEL_NAME="libcuopt_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github cpp
 
       - name: Generate git commit file for tracking the container
         run: |

.gitignore

@@ -43,6 +43,7 @@ datasets/**
 
 ## Doxygen
 cpp/doxygen/html
+cpp/doxygen/xml
 
 # Jupyter Notebooks
 .ipynb_checkpoints
@@ -55,5 +56,12 @@ version_cache.json
 error_log.txt
 
 # docs
-docs/cuopt/_build
-docs/cuopt/_repo
+docs/cuopt/source/cuopt-c/lp-milp/cuopt-cli-help.txt
+docs/cuopt/source/cuopt-server/client-api/sh-cli-help.txt
+docs/cuopt/source/cuopt-server/server-api/server-cli-help.txt
+docs/cuopt/source/cuopt_spec.yaml
+python/cuopt_self_hosted/cuopt_sh_client/tests/utils/certs/*.key
+docs/cuopt/build
+
+# generated version file
+cpp/include/cuopt/semantic_version.hpp

.pre-commit-config.yaml

@@ -64,7 +64,7 @@ repos:
               CMakeLists[.]txt$|
               meta[.]yaml$
   - repo: https://github.com/pre-commit/mirrors-clang-format
-    rev: v16.0.6
+    rev: v20.1.4
     hooks:
       - id: clang-format
         files: \.(cu|cuh|h|hpp|cpp|inl)$

CONTRIBUTING.md

@@ -30,12 +30,14 @@ Contributions to NVIDIA cuOpt fall into the following categories:
 4. Create a fork of the cuopt repository and check out a branch with a name that
    describes your planned work. For example, `fix-documentation`.
 5. Write code to address the issue or implement the feature.
-6. Add unit tests and unit benchmarks.
+6. Add unit tests. Please refer to `cpp/src/tests` for examples of unit tests on C and C++ using gtest and `python/cuopt/cuopt/tests` for examples of unit tests on Python using pytest.
 7. [Create your pull request](https://github.com/NVIDIA/cuopt/compare). To run continuous integration (CI) tests without requesting review, open a draft pull request.
-8. Verify that CI passes all [status checks](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/collaborating-on-repositories-with-code-quality-features/about-status-checks).
+8. Check if CI is running, if not please request one of the NVIDIA cuOpt developers to trigger it. This might happen in case you have non-verified (non-sign-off) commits or don't have enough permissions to trigger CI.
+9. Verify that CI passes all [status checks](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/collaborating-on-repositories-with-code-quality-features/about-status-checks).
    Fix if needed.
-9. Wait for other developers to review your code and update code as needed.
-10. Once reviewed and approved, a NVIDIAcuOpt developer will merge your pull request.
+10. Github will automatically assign a reviewer to your pull request. Please wait for the reviewer to review your code. If the reviewer has any comments, address them in the pull request.
+11. If your PR is not getting reviewed, please ping the reviewers in the PR.
+12. Once reviewed and approved, a NVIDIA cuOpt developer will merge your pull request.
 
 
 Remember, if you are unsure about anything, don't hesitate to comment on issues
@@ -81,7 +83,7 @@ CUDA/GPU Runtime:
 You can obtain CUDA from
 [https://developer.nvidia.com/cuda-downloads](https://developer.nvidia.com/cuda-downloads).
 
-### Create the build environment
+### Build NVIDIA cuOpt from source
 
 - Clone the repository:
 
@@ -94,12 +96,13 @@ cd $CUOPT_HOME
 #### Building with a conda environment
 
 **Note:** Using a conda environment is the easiest way to satisfy the library's dependencies.
-Instructions for a minimal build environment without conda are included below.
 
 - Create the conda development environment:
 
+Please install conda if you don't have it already. You can install it from [https://docs.conda.io/en/latest/miniconda.html](https://docs.conda.io/en/latest/miniconda.html)
+
 ```bash
-# create the conda environment (assuming in base `cudf` directory)
+# create the conda environment (assuming in base `cuopt` directory)
 # note: cuOpt currently doesn't support `channel_priority: strict`;
 # use `channel_priority: flexible` instead
 conda env create --name cuopt_dev --file conda/environments/all_cuda-128_arch-x86_64.yaml
@@ -111,10 +114,8 @@ conda activate cuopt_dev
   development environment may also need to be updated if dependency versions or
   pinnings are changed.
 
-### Build NVIDIA cuOpt from source
-
 - A `build.sh` script is provided in `$CUOPT_HOME`. Running the script with no additional arguments
-  will install the `libmps_parser`, `libcuopt` and `cuopt` libraries. By default, the libraries are
+  will install the `libmps_parser`, `libcuopt`, `cuopt_mps_parser`, `cuopt`, `cuopt-server`, `cuopt-sh-client` libraries and build the`documentation`. By default, the libraries are
   installed to the `$CONDA_PREFIX` directory. To install into a different location, set the location
   in `$INSTALL_PREFIX`. Finally, note that the script depends on the `nvcc` executable being on your
   path, or defined in `$CUDACXX`.
@@ -126,7 +127,7 @@ cd $CUOPT_HOME
 # you want to build and install the libcuopt C++ library only,
 # or include the libcuopt and/or cuopt Python libraries:
 
-./build.sh  # libmps_parser, libcuopt and cuopt
+./build.sh  # All the libraries
 ./build.sh libmps_parser  # libmps_parser only
 ./build.sh libmps_parser libcuopt  # libmps_parser and libcuopt only
 ```
@@ -139,7 +140,6 @@ cd $CUOPT_HOME
 
 #### Building for development
 
-To build C++ tests, you can also request that build.sh build the `test` target.
 To build all libraries and tests, simply run
 
 ```bash
@@ -152,6 +152,8 @@ To run the C++ tests, run
 
 ```bash
 cd $CUOPT_HOME/datasets && get_test_data.sh
+cd $CUOPT_HOME/datasets/linear_programming && download_pdlp_test_dataset.sh
+cd $CUOPT_HOME/datasets/mip && download_miplib_test_dataset.sh
 export RAPIDS_DATASET_ROOT_DIR=$CUOPT_HOME/datasets/
 ctest --test-dir ${CUOPT_HOME}/cpp/build  # libcuopt
 ```
@@ -162,6 +164,8 @@ To run python tests, run
 ```bash
 
 cd $CUOPT_HOME/datasets && get_test_data.sh
+cd $CUOPT_HOME/datasets/linear_programming && download_pdlp_test_dataset.sh
+cd $CUOPT_HOME/datasets/mip && download_miplib_test_dataset.sh
 export RAPIDS_DATASET_ROOT_DIR=$CUOPT_HOME/datasets/
 cd $CUOPT_HOME/python
 pytest -v ${CUOPT_HOME}/python/cuopt/cuopt/tests
@@ -206,10 +210,10 @@ causes a runtime delay of several minutes when loading the libcuopt.so library.
 
 Therefore, it is recommended to add device debug symbols only to specific files by setting the `-G`
 compile option locally in your `cpp/CMakeLists.txt` for that file. Here is an example of adding the
-`-G` option to the compile command for `src/copying/copy.cu` source file:
+`-G` option to the compile command for `cpp/src/routing/data_model_view.cu` source file:
 
 ```cmake
-set_source_files_properties(src/copying/copy.cu PROPERTIES COMPILE_OPTIONS "-G")
+set_source_files_properties(src/routing/data_model_view.cu PROPERTIES COMPILE_OPTIONS "-G")
 ```
 
 This will add the device debug symbols for this object file in `libcuopt.so`.  You can then use
@@ -237,7 +241,7 @@ pip install pre-commit
 Then run pre-commit hooks before committing code:
 
 ```bash
-pre-commit run
+pre-commit run --all-files --show-diff-on-failure
 ```
 
 By default, pre-commit runs on staged files (only changes and additions that will be committed).
@@ -257,39 +261,6 @@ Now code linters and formatters will be run each time you commit changes.
 
 You can skip these checks with `git commit --no-verify` or with the short version `git commit -n`.
 
-cuOpt uses [pre-commit](https://pre-commit.com/) to execute code linters and formatters such as
-[Black](https://black.readthedocs.io/en/stable/), [isort](https://pycqa.github.io/isort/), and
-[flake8](https://flake8.pycqa.org/en/latest/). These tools ensure a consistent code format
-throughout the project. Using pre-commit ensures that linter versions and options are aligned for
-all developers. Additionally, there is a CI check in place to enforce that committed code follows
-our standards.
-
-To use `pre-commit`, install via `conda` or `pip`:
-
-```bash
-conda install -c conda-forge pre-commit
-```
-
-```bash
-pip install pre-commit
-```
-
-Then run pre-commit hooks before committing code:
-
-```bash
-pre-commit run --all-files --show-diff-on-failure
-```
-
-Optionally, you may set up the pre-commit hooks to run automatically when you make a git commit. This can be done by running:
-
-```bash
-pre-commit install
-```
-
-Now code linters and formatters will be run each time you commit changes.
-
-You can skip these checks with `git commit --no-verify` or with the short version `git commit -n`.
-
 #### Signing Your Work
 
 * We require that all contributors "sign-off" on their commits. This certifies that the contribution is your original work, or you have rights to submit it under the same license, or a compatible license.
@@ -333,3 +304,5 @@ You can skip these checks with `git commit --no-verify` or with the short versio
     (d) I understand and agree that this project and the contribution are public and that a record of the contribution (including all personal information I submit with it, including my sign-off) is maintained indefinitely and may be redistributed consistent with this project or the open source license(s) involved.
   ```
 
+
+

README.md

@@ -46,7 +46,7 @@ pip install --extra-index-url=https://pypi.nvidia.com cuopt-cu12
 
 ### Conda
 
-cuOpt can be installed with conda (via [miniforge](https://github.com/conda-forge/miniforge)) from the `rapidsai` channel:
+cuOpt can be installed with conda (via [miniforge](https://github.com/conda-forge/miniforge)) from the `nvidia` channel:
 
 For CUDA 11.x:
 ```bash
@@ -58,10 +58,9 @@ For CUDA 12.x:
 ```bash
 conda install -c rapidsai -c conda-forge -c nvidia \
     cuopt=25.05 python=3.12 cuda-version=12.8
+```
 
 We also provide [nightly Conda packages](https://anaconda.org/rapidsai-nightly) built from the HEAD
 of our latest development branch.
 
 Note: cuOpt is supported only on Linux, and with Python versions 3.10 and later.
-
-See the [NVIDIA cuOpt installation guide](https://docs.nvidia.com/cuopt/user-guide/latest/introduction.html) for more OS and version info.

benchmarks/linear_programming/cuopt/run_mip.cpp

@@ -140,10 +140,10 @@ int run_single_file(std::string file_path,
     if (out_dir != "") {
       std::string log_file =
         out_dir + "/" + base_filename.substr(0, base_filename.find(".mps")) + ".log";
-      settings.set_log_file(log_file);
+      settings.log_file = log_file;
     } else {
       std::string log_file = base_filename.substr(0, base_filename.find(".mps")) + ".log";
-      settings.set_log_file(log_file);
+      settings.log_file    = log_file;
     }
   }
 
@@ -169,17 +169,17 @@ int run_single_file(std::string file_path,
     settings.set_initial_solution(initial_solution.data(), initial_solution.size());
     test_constraint_sanity(mps_data_model,
                            initial_solution,
-                           settings.get_absolute_tolerance(),
-                           settings.get_relative_tolerance(),
-                           settings.get_integrality_tolerance());
+                           settings.tolerances.absolute_tolerance,
+                           settings.tolerances.relative_tolerance,
+                           settings.tolerances.integrality_tolerance);
   }
 
-  settings.set_time_limit(time_limit);
-  settings.set_heuristics_only(heuristics_only);
-  settings.set_num_cpu_threads(num_cpu_threads);
-  settings.set_log_to_console(log_to_console);
-  auto start_run_solver = std::chrono::high_resolution_clock::now();
-  auto solution         = cuopt::linear_programming::solve_mip(&handle_, mps_data_model, settings);
+  settings.time_limit      = time_limit;
+  settings.heuristics_only = heuristics_only;
+  settings.num_cpu_threads = num_cpu_threads;
+  settings.log_to_console  = log_to_console;
+  auto start_run_solver    = std::chrono::high_resolution_clock::now();
+  auto solution = cuopt::linear_programming::solve_mip(&handle_, mps_data_model, settings);
   // solution.write_to_sol_file(base_filename + ".sol", handle_.get_stream());
   std::chrono::milliseconds duration;
   auto end = std::chrono::high_resolution_clock::now();

benchmarks/linear_programming/cuopt/run_pdlp.cu

@@ -99,14 +99,12 @@ static cuopt::linear_programming::pdlp_solver_settings_t<int, double> create_sol
   cuopt::linear_programming::pdlp_solver_settings_t<int, double> settings =
     cuopt::linear_programming::pdlp_solver_settings_t<int, double>{};
 
-  settings.set_time_limit(program.get<double>("--time-limit"));
-  settings.set_iteration_limit(program.get<int>("--iteration-limit"));
+  settings.time_limit      = program.get<double>("--time-limit");
+  settings.iteration_limit = program.get<int>("--iteration-limit");
   settings.set_optimality_tolerance(program.get<double>("--optimality-tolerance"));
-  settings.set_pdlp_solver_mode(
-    string_to_pdlp_solver_mode(program.get<std::string>("--pdlp-solver-mode")));
-  settings.set_method(
-    static_cast<cuopt::linear_programming::method_t>(program.get<int>("--method")));
-  settings.set_crossover(program.get<int>("--crossover"));
+  settings.solver_mode = string_to_pdlp_solver_mode(program.get<std::string>("--pdlp-solver-mode"));
+  settings.method = static_cast<cuopt::linear_programming::method_t>(program.get<int>("--method"));
+  settings.crossover = program.get<int>("--crossover");
 
   return settings;
 }

build.sh

@@ -27,15 +27,16 @@ REPODIR=$(cd $(dirname $0); pwd)
 LIBCUOPT_BUILD_DIR=${LIBCUOPT_BUILD_DIR:=${REPODIR}/cpp/build}
 LIBMPS_PARSER_BUILD_DIR=${LIBMPS_PARSER_BUILD_DIR:=${REPODIR}/cpp/libmps_parser/build}
 
-VALIDARGS="clean libcuopt libmps_parser cuopt_mps_parser cuopt cuopt_server cuopt_sh_client cpp-mgtests docs -a -b -d -g -v -l= --verbose-pdlp  [--cmake-args=\\\"<args>\\\"] [--cache-tool=<tool>] -n --no-fetch-rapids --skip_l1_tests --allgpuarch --ci-only-arch --show_depr_warn -h --help"
+VALIDARGS="clean libcuopt libmps_parser cuopt_mps_parser cuopt cuopt_server cuopt_sh_client docs -a -b -d -g -v -l= --verbose-pdlp  [--cmake-args=\\\"<args>\\\"] [--cache-tool=<tool>] -n --no-fetch-rapids --skip_l1_tests --allgpuarch --ci-only-arch --show_depr_warn -h --help"
 HELP="$0 [<target> ...] [<flag> ...]
  where <target> is:
    clean            - remove all existing build artifacts and configuration (start over)
    libcuopt         - build the cuopt C++ code
+   libmps_parser    - build the libmps_parser C++ code
+   cuopt_mps_parser - build the cuopt_mps_parser python package
    cuopt            - build the cuopt Python package
    cuopt_server     - build the cuopt_server Python package
-   cuopt_sh_client     - build cuopt self host client
-   cpp-mgtests      - build libcuopt mnmg tests. Builds MPI communicator, adding MPI as a dependency.
+   cuopt_sh_client  - build cuopt self host client
    docs             - build the docs
  and <flag> is:
    -v               - verbose build mode
@@ -63,11 +64,12 @@ HELP="$0 [<target> ...] [<flag> ...]
  Set env var LIBCUOPT_BUILD_DIR to override libcuopt build dir.
 "
 CUOPT_MPS_PARSER_BUILD_DIR=${REPODIR}/python/cuopt/cuopt/linear_programming/build
-PY_LIBCUOPT_BUILD_DIR=${REPODIR}/python/cuopt/build
+PY_LIBCUOPT_BUILD_DIR=${REPODIR}/python/libcuopt/build
 CUOPT_BUILD_DIR=${REPODIR}/python/cuopt/build
 CUOPT_SERVER_BUILD_DIR=${REPODIR}/python/cuopt_server/build
 CUOPT_SH_CLIENT_BUILD_DIR=${REPODIR}/python/cuopt_self_hosted/build
-BUILD_DIRS="${LIBCUOPT_BUILD_DIR} ${LIBMPS_PARSER_BUILD_DIR} ${CUOPT_BUILD_DIR} ${CUOPT_SERVER_BUILD_DIR} ${CUOPT_SERVICE_CLIENT_BUILD_DIR} ${CUOPT_SH_CLIENT_BUILD_DIR} ${CUOPT_MPS_PARSER_BUILD_DIR} ${PY_LIBCUOPT_BUILD_DIR}"
+DOCS_BUILD_DIR=${REPODIR}/docs/cuopt/build
+BUILD_DIRS="${LIBCUOPT_BUILD_DIR} ${LIBMPS_PARSER_BUILD_DIR} ${CUOPT_BUILD_DIR} ${CUOPT_SERVER_BUILD_DIR} ${CUOPT_SERVICE_CLIENT_BUILD_DIR} ${CUOPT_SH_CLIENT_BUILD_DIR} ${CUOPT_MPS_PARSER_BUILD_DIR} ${PY_LIBCUOPT_BUILD_DIR} ${DOCS_BUILD_DIR}"
 
 # Set defaults for vars modified by flags to this script
 VERBOSE_FLAG=""
@@ -76,7 +78,6 @@ DEFINE_ASSERT=False
 DEFINE_PDLP_VERBOSE_MODE=False
 INSTALL_TARGET=install
 BUILD_DISABLE_DEPRECATION_WARNING=ON
-BUILD_CPP_MG_TESTS=OFF
 BUILD_L1_TESTS=ON
 BUILD_ALL_GPU_ARCH=0
 BUILD_CI_ONLY=0
@@ -226,9 +227,6 @@ fi
 if hasArg --show_depr_warn; then
     BUILD_DISABLE_DEPRECATION_WARNING=OFF
 fi
-if hasArg cpp-mgtests; then
-    BUILD_CPP_MG_TESTS=ON
-fi
 
 # Append `-DFIND_CUOPT_CPP=ON` to CMAKE_ARGS unless a user specified the option.
 if [[ "${EXTRA_CMAKE_ARGS}" != *"DFIND_CUOPT_CPP"* ]]; then
@@ -311,7 +309,6 @@ if buildAll || hasArg libcuopt; then
           -DDISABLE_DEPRECATION_WARNING=${BUILD_DISABLE_DEPRECATION_WARNING} \
           -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
           -DBUILD_L1_TESTS=${BUILD_L1_TESTS} \
-          -DBUILD_CUOPT_MG_TESTS=${BUILD_CPP_MG_TESTS} \
           -DFETCH_RAPIDS=${FETCH_RAPIDS} \
           ${EXTRA_CMAKE_ARGS} \
           ${REPODIR}/cpp
@@ -350,3 +347,13 @@ if buildAll || hasArg cuopt_sh_client; then
     cd ${REPODIR}/python/cuopt_self_hosted/
     python ${PYTHON_ARGS_FOR_INSTALL} .
 fi
+
+# Build the docs
+if buildAll || hasArg docs; then
+    cd ${REPODIR}/cpp/doxygen
+    doxygen Doxyfile
+
+    cd ${REPODIR}/docs/cuopt
+    make clean
+    make html
+fi