Decompression for .mps.gz and .mps.bz2 files

ci/build_wheel_cuopt_mps_parser.sh

@@ -33,7 +33,13 @@ fi
 
 ci/build_wheel.sh cuopt_mps_parser ${package_dir}
 
+
+EXCLUDE_ARGS=(
+  --exclude "libzlib.so"
+  --exclude "libbz2.so"
+)
+
 # repair wheels and write to the location that artifact-uploading code expects to find them
-python -m auditwheel repair -w "${RAPIDS_WHEEL_BLD_OUTPUT_DIR}" ${package_dir}/dist/*
+python -m auditwheel repair "${EXCLUDE_ARGS[@]}" -w "${RAPIDS_WHEEL_BLD_OUTPUT_DIR}" ${package_dir}/dist/*
 
 ci/validate_wheel.sh "${package_dir}" "${RAPIDS_WHEEL_BLD_OUTPUT_DIR}"

conda/environments/all_cuda-129_arch-aarch64.yaml

@@ -7,6 +7,7 @@ channels:
 dependencies:
 - boost
 - breathe
+- bzip2
 - c-compiler
 - ccache
 - clang-tools=20.1.4
@@ -79,6 +80,7 @@ dependencies:
 - sphinxcontrib-websupport
 - sysroot_linux-aarch64==2.28
 - uvicorn==0.34.*
+- zlib
 - pip:
   - nvidia_sphinx_theme
   - swagger-plugin-for-sphinx

conda/environments/all_cuda-129_arch-x86_64.yaml

@@ -7,6 +7,7 @@ channels:
 dependencies:
 - boost
 - breathe
+- bzip2
 - c-compiler
 - ccache
 - clang-tools=20.1.4
@@ -79,6 +80,7 @@ dependencies:
 - sphinxcontrib-websupport
 - sysroot_linux-64==2.28
 - uvicorn==0.34.*
+- zlib
 - pip:
   - nvidia_sphinx_theme
   - swagger-plugin-for-sphinx

conda/recipes/libcuopt/recipe.yaml

@@ -57,6 +57,8 @@ cache:
       - cuda-version =${{ cuda_version }}
       - cmake ${{ cmake_version }}
       - ninja
+      - zlib
+      - bzip2
     host:
       - cpp-argparse
       - cuda-version =${{ cuda_version }}
@@ -70,6 +72,8 @@ cache:
       - libcusparse-dev
       - cuda-cudart-dev
       - boost
+      - zlib
+      - bzip2
 
 outputs:
   - package:
@@ -90,6 +94,14 @@ outputs:
       build:
         - cmake ${{ cmake_version }}
         - ${{ stdlib("c") }}
+        - zlib
+        - bzip2
+      host:
+        - zlib
+        - bzip2
+      run:
+        - zlib
+        - bzip2
       ignore_run_exports:
         by_name:
           - cuda-cudart
@@ -99,6 +111,8 @@ outputs:
           - libcurand
           - libcusparse
           - librmm
+          - libzlib
+          - libbz2
     tests:
     - package_contents:
         files:

conda/recipes/mps-parser/recipe.yaml

@@ -32,6 +32,7 @@ requirements:
   build:
     - cmake ${{ cmake_version }}
     - ninja
+    - libmps-parser =${{ version }}
     - ${{ compiler("c") }}
     - ${{ compiler("cxx") }}
     - ${{ stdlib("c") }}

cpp/libmps_parser/CMakeLists.txt

@@ -38,6 +38,8 @@ rapids_cmake_build_type(Release)
 # #############################################################################
 # - User Options  ------------------------------------------------------------
 option(BUILD_TESTS "Configure CMake to build tests" ON)
+option(MPS_PARSER_WITH_BZIP2 "Build with bzip2 decompression" ON)
+option(MPS_PARSER_WITH_ZLIB "Build with zlib decompression" ON)
 
 message(VERBOSE "cuOpt: Build mps-parser unit-tests: ${BUILD_TESTS}")
 
@@ -50,6 +52,16 @@ if(CMAKE_COMPILER_IS_GNUCXX)
   list(APPEND MPS_PARSER_CXX_FLAGS -Werror -Wno-error=deprecated-declarations)
 endif(CMAKE_COMPILER_IS_GNUCXX)
 
+if(MPS_PARSER_WITH_BZIP2)
+  find_package(BZip2 REQUIRED)
+  add_compile_definitions(-DMPS_PARSER_WITH_BZIP2)
+endif(MPS_PARSER_WITH_BZIP2)
+
+if(MPS_PARSER_WITH_ZLIB)
+  find_package(ZLIB REQUIRED)
+  add_compile_definitions(-DMPS_PARSER_WITH_ZLIB)
+endif(MPS_PARSER_WITH_ZLIB)
+
 if(DEFINE_ASSERT)
   add_definitions(-DASSERT_MODE)
 endif(DEFINE_ASSERT)
@@ -107,6 +119,14 @@ target_include_directories(mps_parser
   "$<INSTALL_INTERFACE:include>"
 )
 
+if(MPS_PARSER_WITH_BZIP2)
+    target_include_directories(mps_parser PRIVATE BZip2::BZip2)
+endif(MPS_PARSER_WITH_BZIP2)
+
+if(MPS_PARSER_WITH_ZLIB)
+    target_include_directories(mps_parser PRIVATE ZLIB::ZLIB)
+endif(MPS_PARSER_WITH_ZLIB)
+
 # ##################################################################################################
 # - generate tests --------------------------------------------------------------------------------
 if(BUILD_TESTS)

cpp/libmps_parser/include/mps_parser/parser.hpp

@@ -22,19 +22,24 @@
 namespace cuopt::mps_parser {
 
 /**
- * @brief Reads the equation from the input text file which is MPS or QPS formatted
+ * @brief Reads the equation from an MPS or QPS file.
+ *
+ * The input file can be a plain text file in MPS-/QPS-format or a compressed MPS/QPS
+ * file (.mps.gz or .mps.bz2).
  *
  * Read this link http://lpsolve.sourceforge.net/5.5/mps-format.htm for more
  * details on both free and fixed MPS format.
- *
  * This function supports both standard MPS files (for linear programming) and
  * QPS files (for quadratic programming). QPS files are MPS files with additional
  * sections:
  * - QUADOBJ: Defines quadratic terms in the objective function
  *
- * @param[in] mps_file_path Path to MPS or QPS formatted file.
- * @param[in] fixed_mps_format If MPS/QPS file should be parsed as fixed format, false by default
- * @return mps_data_model_t A fully formed LP/QP problem which represents the given MPS/QPS file
+ * Note: Compressed MPS files .mps.gz, .mps.bz2 can only be read if the compression
+ * libraries zlib or libbzip2 are installed, respectively.
+ *
+ * @param[in] mps_file_path Path to MPS/QPSfile.
+ * @param[in] fixed_mps_format If MPS/QPS file should be parsed as fixed, false by default
+ * @return mps_data_model_t A fully formed LP/QP problem which represents the given file
  */
 template <typename i_t, typename f_t>
 mps_data_model_t<i_t, f_t> parse_mps(const std::string& mps_file_path,

cpp/libmps_parser/src/mps_parser.cpp

@@ -26,9 +26,193 @@
 #include <fstream>
 #include <iostream>
 #include <limits>
+#include <memory>
 #include <sstream>
 #include <string>
 
+#ifdef MPS_PARSER_WITH_BZIP2
+#include <bzlib.h>
+#endif  // MPS_PARSER_WITH_BZIP2
+
+#ifdef MPS_PARSER_WITH_ZLIB
+#include <zlib.h>
+#endif  // MPS_PARSER_WITH_ZLIB
+
+#if defined(MPS_PARSER_WITH_BZIP2) || defined(MPS_PARSER_WITH_ZLIB)
+#include <dlfcn.h>
+#endif  // MPS_PARSER_WITH_BZIP2 || MPS_PARSER_WITH_ZLIB
+
+namespace {
+using cuopt::mps_parser::error_type_t;
+using cuopt::mps_parser::mps_parser_expects;
+using cuopt::mps_parser::mps_parser_expects_fatal;
+
+struct FcloseDeleter {
+  void operator()(FILE* fp)
+  {
+    mps_parser_expects_fatal(
+      fclose(fp) == 0, error_type_t::ValidationError, "Error closing MPS file!");
+  }
+};
+}  // end namespace
+
+#ifdef MPS_PARSER_WITH_BZIP2
+namespace {
+using BZ2_bzReadOpen_t  = decltype(&BZ2_bzReadOpen);
+using BZ2_bzReadClose_t = decltype(&BZ2_bzReadClose);
+using BZ2_bzRead_t      = decltype(&BZ2_bzRead);
+
+std::vector<char> bz2_file_to_string(const std::string& file)
+{
+  struct DlCloseDeleter {
+    void operator()(void* fp)
+    {
+      mps_parser_expects_fatal(
+        dlclose(fp) == 0, error_type_t::ValidationError, "Error closing libbz2.so!");
+    }
+  };
+  struct BzReadCloseDeleter {
+    void operator()(void* f)
+    {
+      int bzerror;
+      if (f != nullptr) fptr(&bzerror, f);
+      mps_parser_expects_fatal(
+        bzerror == BZ_OK, error_type_t::ValidationError, "Error closing bzip2 file!");
+    }
+    BZ2_bzReadClose_t fptr = nullptr;
+  };
+
+  std::unique_ptr<void, DlCloseDeleter> lbz2handle{dlopen("libbz2.so", RTLD_LAZY)};
+  mps_parser_expects(
+    lbz2handle != nullptr,
+    error_type_t::ValidationError,
+    "Could not open .mps.bz2 file since libbz2.so was not found. In order to open .mps.bz2 files "
+    "directly, please ensure libbzip2 is installed. Alternatively, decompress the .mps.bz2 file "
+    "manually and open the uncompressed .mps file. Given path: %s",
+    file.c_str());
+
+  BZ2_bzReadOpen_t BZ2_bzReadOpen =
+    reinterpret_cast<BZ2_bzReadOpen_t>(dlsym(lbz2handle.get(), "BZ2_bzReadOpen"));
+  BZ2_bzReadClose_t BZ2_bzReadClose =
+    reinterpret_cast<BZ2_bzReadClose_t>(dlsym(lbz2handle.get(), "BZ2_bzReadClose"));
+  BZ2_bzRead_t BZ2_bzRead = reinterpret_cast<BZ2_bzRead_t>(dlsym(lbz2handle.get(), "BZ2_bzRead"));
+  mps_parser_expects(
+    BZ2_bzReadOpen != nullptr && BZ2_bzReadClose != nullptr && BZ2_bzRead != nullptr,
+    error_type_t::ValidationError,
+    "Error loading libbzip2! Library version might be incompatible. Please decompress the .mps.bz2 "
+    "file manually and open the uncompressed .mps file. Given path: %s",
+    file.c_str());
+
+  std::unique_ptr<FILE, FcloseDeleter> fp{fopen(file.c_str(), "rb")};
+  mps_parser_expects(fp != nullptr,
+                     error_type_t::ValidationError,
+                     "Error opening MPS file! Given path: %s",
+                     file.c_str());
+  int bzerror = BZ_OK;
+  std::unique_ptr<void, BzReadCloseDeleter> bzfile{
+    BZ2_bzReadOpen(&bzerror, fp.get(), 0, 0, nullptr, 0), {BZ2_bzReadClose}};
+  mps_parser_expects(bzerror == BZ_OK,
+                     error_type_t::ValidationError,
+                     "Could not open bzip2 compressed file! Given path: %s",
+                     file.c_str());
+
+  std::vector<char> buf;
+  const size_t readbufsize = 1ull << 24;  // 16MiB - just a guess.
+  std::vector<char> readbuf(readbufsize);
+  while (bzerror == BZ_OK) {
+    const size_t bytes_read = BZ2_bzRead(&bzerror, bzfile.get(), readbuf.data(), readbuf.size());
+    if (bzerror == BZ_OK || bzerror == BZ_STREAM_END) {
+      buf.insert(buf.end(), begin(readbuf), begin(readbuf) + bytes_read);
+    }
+  }
+  buf.push_back('\0');
+  mps_parser_expects(bzerror == BZ_STREAM_END,
+                     error_type_t::ValidationError,
+                     "Error in bzip2 decompression of MPS file! Given path: %s",
+                     file.c_str());
+  return buf;
+}
+}  // end namespace
+#endif  // MPS_PARSER_WITH_BZIP2
+
+#ifdef MPS_PARSER_WITH_ZLIB
+namespace {
+using gzopen_t    = decltype(&gzopen);
+using gzclose_r_t = decltype(&gzclose_r);
+using gzbuffer_t  = decltype(&gzbuffer);
+using gzread_t    = decltype(&gzread);
+using gzerror_t   = decltype(&gzerror);
+std::vector<char> zlib_file_to_string(const std::string& file)
+{
+  struct DlCloseDeleter {
+    void operator()(void* fp)
+    {
+      mps_parser_expects_fatal(
+        dlclose(fp) == 0, error_type_t::ValidationError, "Error closing libbz2.so!");
+    }
+  };
+  struct GzCloseDeleter {
+    void operator()(gzFile_s* f)
+    {
+      int err = fptr(f);
+      mps_parser_expects_fatal(
+        err == Z_OK, error_type_t::ValidationError, "Error closing gz file!");
+    }
+    gzclose_r_t fptr = nullptr;
+  };
+
+  std::unique_ptr<void, DlCloseDeleter> lzhandle{dlopen("libz.so.1", RTLD_LAZY)};
+  mps_parser_expects(
+    lzhandle != nullptr,
+    error_type_t::ValidationError,
+    "Could not open .mps.gz file since libz.so was not found. In order to open .mps.gz files "
+    "directly, please ensure zlib is installed. Alternatively, decompress the .mps.gz file "
+    "manually and open the uncompressed .mps file. Given path: %s",
+    file.c_str());
+  gzopen_t gzopen       = reinterpret_cast<gzopen_t>(dlsym(lzhandle.get(), "gzopen"));
+  gzclose_r_t gzclose_r = reinterpret_cast<gzclose_r_t>(dlsym(lzhandle.get(), "gzclose_r"));
+  gzbuffer_t gzbuffer   = reinterpret_cast<gzbuffer_t>(dlsym(lzhandle.get(), "gzbuffer"));
+  gzread_t gzread       = reinterpret_cast<gzread_t>(dlsym(lzhandle.get(), "gzread"));
+  gzerror_t gzerror     = reinterpret_cast<gzerror_t>(dlsym(lzhandle.get(), "gzerror"));
+  mps_parser_expects(
+    gzopen != nullptr && gzclose_r != nullptr && gzbuffer != nullptr && gzread != nullptr &&
+      gzerror != nullptr,
+    error_type_t::ValidationError,
+    "Error loading zlib! Library version might be incompatible. Please decompress the .mps.gz file "
+    "manually and open the uncompressed .mps file. Given path: %s",
+    file.c_str());
+  std::unique_ptr<gzFile_s, GzCloseDeleter> gzfp{gzopen(file.c_str(), "rb"), {gzclose_r}};
+  mps_parser_expects(gzfp != nullptr,
+                     error_type_t::ValidationError,
+                     "Error opening compressed MPS file! Given path: %s",
+                     file.c_str());
+  int zlib_status = gzbuffer(gzfp.get(), 1 << 20);  // 1 MiB
+  mps_parser_expects(zlib_status == Z_OK,
+                     error_type_t::ValidationError,
+                     "Could not set zlib internal buffer size for decompression! Given path: %s",
+                     file.c_str());
+  std::vector<char> buf;
+  const size_t readbufsize = 1ull << 24;  // 16MiB
+  std::vector<char> readbuf(readbufsize);
+  int bytes_read = -1;
+  while (bytes_read != 0) {
+    bytes_read = gzread(gzfp.get(), readbuf.data(), readbuf.size());
+    if (bytes_read > 0) { buf.insert(buf.end(), begin(readbuf), begin(readbuf) + bytes_read); }
+    if (bytes_read < 0) {
+      gzerror(gzfp.get(), &zlib_status);
+      break;
+    }
+  }
+  buf.push_back('\0');
+  mps_parser_expects(zlib_status == Z_OK,
+                     error_type_t::ValidationError,
+                     "Error in zlib decompression of MPS file! Given path: %s",
+                     file.c_str());
+  return buf;
+}
+}  // end namespace
+#endif  // MPS_PARSER_WITH_ZLIB
+
 namespace cuopt::mps_parser {
 
 template <typename i_t>
@@ -342,6 +526,18 @@ std::vector<char> mps_parser_t<i_t, f_t>::file_to_string(const std::string& file
 {
   // raft::common::nvtx::range fun_scope("file to string");
 
+#ifdef MPS_PARSER_WITH_BZIP2
+  if (file.size() > 4 && file.substr(file.size() - 4, 4) == ".bz2") {
+    return bz2_file_to_string(file);
+  }
+#endif  // MPS_PARSER_WITH_BZIP2
+
+#ifdef MPS_PARSER_WITH_ZLIB
+  if (file.size() > 3 && file.substr(file.size() - 3, 3) == ".gz") {
+    return zlib_file_to_string(file);
+  }
+#endif  // MPS_PARSER_WITH_ZLIB
+
   // Faster than using C++ I/O
   FILE* fp = fopen(file.c_str(), "r");
   mps_parser_expects(fp != nullptr,

cpp/libmps_parser/src/mps_parser.hpp

@@ -145,10 +145,10 @@ class mps_parser_t {
   std::unordered_set<i_t> bounds_defined_for_var_id{};
   static constexpr f_t unset_range_value = std::numeric_limits<f_t>::infinity();
 
-  /* Reads the equation from the input text file which is MPS formatted
+  /* Reads an MPS input file into a buffer.
    *
-   * Read this link http://lpsolve.sourceforge.net/5.5/mps-format.htm for more
-   * details on this format.
+   * If the file has a .gz or .bz2 suffix and zlib or libbzip2 are installed, respectively,
+   * the function directly reads and decompresses the compressed MPS file.
    */
   std::vector<char> file_to_string(const std::string& file);
   void fill_problem(mps_data_model_t<i_t, f_t>& problem);