[ML] Upgrade to Boost 1.86.0 on Windows 2022 builds

.buildkite/pipelines/build_windows.json.py

@@ -45,7 +45,7 @@ def main(args):
               "provider": "gcp",
               "machineType": "c2-standard-16",
               "minCpuPlatform": "Intel Cascade Lake",
-              "image": "family/ml-cpp-1-windows-2022",
+              "image": "family/ml-cpp-2-windows-2022",
             },
             "commands": [
               f'if ( "{args.action}" -eq "debug" ) {{\$Env:ML_DEBUG="1"}}',

3rd_party/3rd_party.cmake

@@ -142,7 +142,7 @@ else()
   # server is currently set up
   set(BOOST_LOCATION "${LOCAL_DRIVE}/usr/local/lib")
   set(BOOST_COMPILER "vc")
-  set(BOOST_EXTENSION "mt-x64-1_83.dll")
+  set(BOOST_EXTENSION "mt-x64-1_86.dll")
   set(BOOST_LIBRARIES "atomic" "chrono" "date_time" "filesystem" "iostreams" "log" "log_setup" "program_options" "regex" "system" "thread" "unit_test_framework")
   set(XML_LOCATION "${LOCAL_DRIVE}/usr/local/bin")
   set(XML_EXTENSION ".dll")

build-setup/windows.md

@@ -129,38 +129,35 @@ nmake
 nmake install
 ```
 
-### Boost 1.83.0
+### Boost 1.86.0
 
-Download version 1.83.0 of Boost from <https://boostorg.jfrog.io/artifactory/main/release/1.83.0/source/boost_1_83_0.tar.bz2> . You must get this exact version, as the Machine Learning build system requires it.
+Download version 1.86.0 of Boost from <https://boostorg.jfrog.io/artifactory/main/release/1.86.0/source/boost_1_86_0.tar.bz2> . You must get this exact version, as the Machine Learning build system requires it.
 
 Assuming you chose the `.bz2` version, extract it in a Git bash shell using the GNU tar that comes with Git for Windows, e.g.:
 
 ```
 cd /c/tools
-tar jxvf /z/cpp_src/boost_1_83_0.tar.bz2
+tar jxvf /z/cpp_src/boost_1_86_0.tar.bz2
 ```
 
-Edit `boost/unordered/detail/prime_fmod.hpp` and change line 134 from:
+Edit `boost/unordered/detail/prime_fmod.hpp` and change line 37 from:
 
 ```
-    (13ul)(29ul)(53ul)(97ul)(193ul)(389ul)(769ul)(1543ul)(3079ul)(6151ul)(       \
+    constexpr static std::size_t const sizes[] = {13ul, 29ul, 53ul, 97ul,
 ```
 
 to:
 
 ```
-    (3ul)(13ul)(29ul)(53ul)(97ul)(193ul)(389ul)(769ul)(1543ul)(3079ul)(6151ul)(       \
+    constexpr static std::size_t const sizes[] = {3ul, 13ul, 29ul, 53ul, 97ul,
 ```
 
-Also edit `tools/build/src/tools/msvc.jam` and change all occurrences of `14.3` to `14.4`.
-
 Start a command prompt using Start Menu -&gt; Apps -&gt; Visual Studio 2022 -&gt; x64 Native Tools Command Prompt for VS 2022, then in it type:
 
 ```
-cd \tools\boost_1_83_0
+cd \tools\boost_1_86_0
 bootstrap.bat
-b2 -j6 --layout=versioned --disable-icu --toolset=msvc-14.2 cxxflags="-std:c++17" linkflags="-std:c++17" --build-type=complete -sZLIB_INCLUDE="C:\tools\zlib-1.2.13" -sZLIB_LIBPATH="C:\tools\zlib-1.2.13" -sZLIB_NAME=zdll --without-context --without-coroutine --without-graph_parallel --without-mpi --without-python architecture=x86 address-model=64 optimization=speed inlining=full define=BOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS define=BOOST_LOG_WITHOUT_DEBUG_OUTPUT define=BOOST_LOG_WITHOUT_EVENT_LOG define=BOOST_LOG_WITHOUT_SYSLOG define=BOOST_LOG_WITHOUT_IPC define=_WIN32_WINNT=0x0601
-b2 install --prefix=C:\usr\local --layout=versioned --disable-icu --toolset=msvc-14.2 cxxflags="-std:c++17" linkflags="-std:c++17" --build-type=complete -sZLIB_INCLUDE="C:\tools\zlib-1.2.13" -sZLIB_LIBPATH="C:\tools\zlib-1.2.13" -sZLIB_NAME=zdll --without-context --without-coroutine --without-graph_parallel --without-mpi --without-python architecture=x86 address-model=64 optimization=speed inlining=full define=BOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS define=BOOST_LOG_WITHOUT_DEBUG_OUTPUT define=BOOST_LOG_WITHOUT_EVENT_LOG define=BOOST_LOG_WITHOUT_SYSLOG define=BOOST_LOG_WITHOUT_IPC define=_WIN32_WINNT=0x0601
+b2 -j6 install --prefix=C:\usr\local --layout=versioned --disable-icu --toolset=msvc-14.2 cxxflags="-std:c++17" linkflags="-std:c++17" --build-type=complete -sZLIB_INCLUDE="C:\tools\zlib-1.2.13" -sZLIB_LIBPATH="C:\tools\zlib-1.2.13" -sZLIB_NAME=zdll --without-context --without-coroutine --without-graph_parallel --without-mpi --without-python architecture=x86 address-model=64 optimization=speed inlining=full define=BOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS define=BOOST_LOG_WITHOUT_DEBUG_OUTPUT define=BOOST_LOG_WITHOUT_EVENT_LOG define=BOOST_LOG_WITHOUT_SYSLOG define=BOOST_LOG_WITHOUT_IPC define=_WIN32_WINNT=0x0601
 ```
 
 The Boost headers and appropriate libraries should end up in `C:\usr\local\include` and `C:\usr\local\lib` respectively.

cmake/variables.cmake

@@ -243,7 +243,12 @@ set(Boost_USE_STATIC_LIBS OFF)
 set(Boost_USE_DEBUG_RUNTIME OFF)
 set(Boost_COMPILER "${ML_BOOST_COMPILER_VER}")
 
-find_package(Boost 1.83.0 CONFIG EXACT REQUIRED COMPONENTS iostreams filesystem program_options regex date_time log log_setup thread unit_test_framework)
+set(Boost_VERSION 1.83.0)
+if (CMAKE_SYSTEM_NAME STREQUAL "Windows")
+  message(WARNING "Using Boost 1.86.0 on Windows only.")
+  set(Boost_VERSION 1.86.0)
+endif()
+find_package(Boost ${Boost_VERSION} CONFIG EXACT REQUIRED COMPONENTS iostreams filesystem program_options regex date_time log log_setup thread unit_test_framework)
 if(Boost_FOUND)
   list(APPEND ML_SYSTEM_INCLUDE_DIRECTORIES ${Boost_INCLUDE_DIRS})
 endif()

dev-tools/build_windows_third_party_deps.ps1

@@ -58,27 +58,25 @@ $ZipDestination="\tools\$Archive"
 (New-Object Net.WebClient).DownloadFile($ZipSource, $ZipDestination)
 C:\Progra~1\git\bin\bash.exe -c "tar Jxvf libxml2-2.9.14.tar.xz"
 cd \tools\libxml2-2.9.14\win32
-& 'C:\Program Files\Git\usr\bin\sed.exe' -i -e 's|CFLAGS = `$(CFLAGS) /D "NDEBUG" /O2|CFLAGS = `$(CFLAGS) /D "NDEBUG" /O2 /Zi /D_WIN32_WINNT=0x0601|' Makefile.msvc
+(Get-Content .\Makefile.msvc) -replace """NDEBUG"" /O2" , """NDEBUG"" /O2 /Zi /D_WIN32_WINNT=0x0601" | Set-Content .\Makefile.msvc
 cscript configure.js iconv=no prefix=C:\usr\local
 nmake
 nmake install
 Write-Host "--- Done Installing libxml2 2.9.14"
 
-# Build and install Boost 1.83.0
-Write-Host "--- Installing boost 1.83.0"
+# Build and install Boost 1.86.0
+Write-Host "--- Installing boost 1.86.0"
 cd c:\tools
-$Archive="boost_1_83_0.tar.bz2"
-$ZipSource="https://boostorg.jfrog.io/artifactory/main/release/1.83.0/source/$Archive"
+$Archive="boost_1_86_0.tar.bz2"
+$ZipSource="https://boostorg.jfrog.io/artifactory/main/release/1.86.0/source/$Archive"
 $ZipDestination="\tools\$Archive"
 (New-Object Net.WebClient).DownloadFile($ZipSource, $ZipDestination)
-C:\Progra~1\git\bin\bash.exe -c "tar jxvf boost_1_83_0.tar.bz2"
-cd \tools\boost_1_83_0
-& 'C:\Program Files\Git\usr\bin\sed.exe' -i -e 's|(13ul)(29ul)(53ul)(97ul)(193ul)(389ul)(769ul)(1543ul)(3079ul)(6151ul)( |(3ul)(13ul)(29ul)(53ul)(97ul)(193ul)(389ul)(769ul)(1543ul)(3079ul)(6151ul)(       |' boost/unordered/detail/prime_fmod.hpp
-& 'C:\Program Files\Git\usr\bin\sed.exe' -i -e 's/14\.3/14\.4/' tools/build/src/tools/msvc.jam
-c:\tools\boost_1_83_0\bootstrap.bat
-c:\tools\boost_1_83_0\b2 -j6 --layout=versioned --disable-icu --toolset=msvc-14.4 cxxflags="-std:c++17" linkflags="-std:c++17" --build-type=complete -sZLIB_INCLUDE="C:\tools\zlib-1.3.1" -sZLIB_LIBPATH="C:\tools\zlib-1.3.1" -sZLIB_NAME=zdll --without-context --without-coroutine --without-graph_parallel --without-mpi --without-python architecture=x86 address-model=64 optimization=speed inlining=full define=BOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS define=BOOST_LOG_WITHOUT_DEBUG_OUTPUT define=BOOST_LOG_WITHOUT_EVENT_LOG define=BOOST_LOG_WITHOUT_SYSLOG define=BOOST_LOG_WITHOUT_IPC define=_WIN32_WINNT=0x0601
-c:\tools\boost_1_83_0\b2 install --prefix=C:\usr\local --layout=versioned --disable-icu --toolset=msvc-14.4 cxxflags="-std:c++17" linkflags="-std:c++17" --build-type=complete -sZLIB_INCLUDE="C:\tools\zlib-1.3.1" -sZLIB_LIBPATH="C:\tools\zlib-1.3.1" -sZLIB_NAME=zdll --without-context --without-coroutine --without-graph_parallel --without-mpi --without-python architecture=x86 address-model=64 optimization=speed inlining=full define=BOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS define=BOOST_LOG_WITHOUT_DEBUG_OUTPUT define=BOOST_LOG_WITHOUT_EVENT_LOG define=BOOST_LOG_WITHOUT_SYSLOG define=BOOST_LOG_WITHOUT_IPC define=_WIN32_WINNT=0x0601
-Write-Host "--- Done Installing boost 1.83.0"
+C:\Progra~1\git\bin\bash.exe -c "tar jxvf boost_1_86_0.tar.bz2"
+cd \tools\boost_1_86_0
+(Get-Content .\boost\unordered\detail\prime_fmod.hpp) -replace "{13ul" , "{3ul, 13ul" | Set-Content .\boost\unordered\detail\prime_fmod.hpp
+c:\tools\boost_1_86_0\bootstrap.bat
+c:\tools\boost_1_86_0\b2 -j6 install --prefix=C:\usr\local --layout=versioned --disable-icu --toolset=msvc-14.4 cxxflags="-std:c++17" linkflags="-std:c++17" --build-type=complete -sZLIB_INCLUDE="C:\tools\zlib-1.3.1" -sZLIB_LIBPATH="C:\tools\zlib-1.3.1" -sZLIB_NAME=zdll --without-context --without-coroutine --without-graph_parallel --without-mpi --without-python architecture=x86 address-model=64 optimization=speed inlining=full define=BOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS define=BOOST_LOG_WITHOUT_DEBUG_OUTPUT define=BOOST_LOG_WITHOUT_EVENT_LOG define=BOOST_LOG_WITHOUT_SYSLOG define=BOOST_LOG_WITHOUT_IPC define=_WIN32_WINNT=0x0601
+Write-Host "--- Done Installing boost 1.86.0"
 
 
 # Build and install strptime
@@ -111,8 +109,9 @@ pip install numpy ninja pyyaml mkl mkl-include setuptools cmake cffi typing_exte
 Write-Host "--- Done Installing python modules"
 
 # Move the experimental MS libomp libraries out of the way temporarily, as we don't want to inadvertently redistribute them.
-mv "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\14.41.34120\lib\x64\libomp.lib" "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\14.41.34120\lib\x64\libomp.lib.bak"
-mv "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\14.41.34120\lib\x64\libompd.lib" "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\14.41.34120\lib\x64\libompd.lib.bak"
+$f = ls 'C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\' -name | Select-Object -Last 1
+mv "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\$f\lib\x64\libomp.lib" "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\$f\lib\x64\libomp.lib.bak"
+mv "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\$f\lib\x64\libompd.lib" "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\$f\lib\x64\libompd.lib.bak"
 
 
 # Build and install PyTorch (this may take a while)
@@ -122,7 +121,7 @@ git clone --depth=1 --branch=v2.3.1 https://github.com/pytorch/pytorch.git
 cd pytorch
 git submodule sync
 git submodule update --init --recursive
-& 'C:\Program Files\Git\usr\bin\sed.exe' -i -e '/pipe(/{N; s/std::unique_ptr<FILE, decltype(&_pclose)> pipe(\n      _wpopen(cmd.c_str(), L"r"), _pclose);/std::unique_ptr<FILE> pipe;/}' -e '/_wspawnve/{N; s/intptr_t r = _wspawnve(_P_WAIT, comspec, a, e.data());\n  return r/return -1;/}' torch/csrc/jit/codegen/fuser/cpu/fused_kernel.cpp
+(Get-Content ".\torch\csrc\jit\codegen\fuser\cpu\fused_kernel.cpp" -Raw) -replace "(?ms)std::unique_ptr.*?pipe.*?;" , "std::unique_ptr<FILE> pipe;" -replace "(?ms)intptr_t r = _wspawnve.*?return r;" , "return -1;" | Set-Content -Path ".\torch\csrc\jit\codegen\fuser\cpu\fused_kernel.cpp"
 set BUILD_TEST=OFF
 set BUILD_CAFFE2=OFF
 set USE_NUMPY=OFF
@@ -154,8 +153,8 @@ cp torch\lib\asmjit.lib c:\usr\local\lib\
 Write-Host "--- Done Installing PyTorch globally"
 
 # Move the libomp libraries back where we found them
-mv "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\14.41.34120\lib\x64\libomp.lib.bak" "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\14.41.34120\lib\x64\libomp.lib"
-mv "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\14.41.34120\lib\x64\libompd.lib.bak" "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\14.41.34120\lib\x64\libompd.lib"
+mv "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\$f\lib\x64\libomp.lib.bak" "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\$f\lib\x64\libomp.lib"
+mv "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\$f\lib\x64\libompd.lib.bak" "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\$f\lib\x64\libompd.lib"
 
 # Uninstall python, it was only needed for the PyTorch build
 & c:\tools\$PythonArchive /uninstall /quiet

docs/CHANGELOG.asciidoc

@@ -28,6 +28,12 @@
 
 //=== Regressions
 
+== {es} version 8.18.0
+
+=== Enhancements
+
+* Upgrade Boost libraries to version 1.86. (See {ml-pull}2780[#2780].)
+
 == {es} version 8.16.0
 
 === Enhancements

lib/core/unittest/CMemoryUsageTest.cc

@@ -1287,12 +1287,12 @@ BOOST_AUTO_TEST_CASE(testSmallVector) {
     BOOST_REQUIRE_EQUAL(0, extraMem);
     growShrink.push_back(1.7);
     extraMem = core::memory::dynamicSize(growShrink);
-    // Interesting (shocking?) result: once a boost::small_vector has switched
-    // off of internal storage it will NEVER go back to internal storage.
-    // Arguably this is a bug, and this assertion might start failing after a
-    // Boost upgrade.  If that happens and changing it to assert extraMem is 0
-    // fixes it then this means boost::small_vector has been improved.
-    BOOST_TEST_REQUIRE(extraMem > 0);
+    // Interestingly we used to assert extraMem > 0 here as it used to be the case
+    // that once a boost::small_vector had switched
+    // off of internal storage it would NEVER go back to internal storage.
+    // Arguably that was a bug, and this assertion started failing after
+    // upgrading Boost to 1.86.0, meaning that boost::small_vector has been improved.
+    BOOST_TEST_REQUIRE(extraMem >= 0); // Change this to "==" when all platforms have been upgraded to Boost 1.86.0
 }
 
 BOOST_AUTO_TEST_CASE(testAlignedVector) {

lib/model/CBucketGatherer.cc

@@ -24,6 +24,8 @@
 
 #include <model/CDataGatherer.h>
 
+#include <boost/tuple/tuple.hpp>
+
 #include <algorithm>
 #include <map>