[ML] Upgrade to PyTorch 2.5.0 on Windows 2022 builds

.buildkite/pipelines/build_windows.json.py

@@ -45,7 +45,7 @@ def main(args):
               "provider": "gcp",
               "machineType": "c2-standard-16",
               "minCpuPlatform": "Intel Cascade Lake",
-              "image": "family/ml-cpp-1-windows-2022",
+              "image": "family/ml-cpp-3-windows-2022",
             },
             "commands": [
               f'if ( "{args.action}" -eq "debug" ) {{\$Env:ML_DEBUG="1"}}',

3rd_party/3rd_party.cmake

@@ -142,7 +142,7 @@ else()
   # server is currently set up
   set(BOOST_LOCATION "${LOCAL_DRIVE}/usr/local/lib")
   set(BOOST_COMPILER "vc")
-  set(BOOST_EXTENSION "mt-x64-1_83.dll")
+  set(BOOST_EXTENSION "mt-x64-1_86.dll")
   set(BOOST_LIBRARIES "atomic" "chrono" "date_time" "filesystem" "iostreams" "log" "log_setup" "program_options" "regex" "system" "thread" "unit_test_framework")
   set(XML_LOCATION "${LOCAL_DRIVE}/usr/local/bin")
   set(XML_EXTENSION ".dll")

build-setup/windows.md

@@ -205,7 +205,7 @@ On the "Advanced Options" screen, check "Install for all users" and "Add Python
 
 For the time being, do not take advantage of the option on the final installer screen to reconfigure the machine to allow paths longer than 260 characters.  We still support Windows versions that do not have this option.
 
-### PyTorch 2.3.1
+### PyTorch 2.5.0
 
 (This step requires a lot of memory. It failed on a machine with 12GB of RAM. It just about fitted on a 20GB machine. 32GB RAM is recommended.)
 
@@ -219,7 +219,7 @@ Next, in a Git bash shell run:
 
 ```
 cd /c/tools
-git clone --depth=1 --branch=v2.3.1 https://github.com/pytorch/pytorch.git
+git clone --depth=1 --branch=v2.5.0 https://github.com/pytorch/pytorch.git
 cd pytorch
 git submodule sync
 git submodule update --init --recursive
@@ -275,7 +275,7 @@ set USE_QNNPACK=OFF
 set USE_PYTORCH_QNNPACK=OFF
 set USE_XNNPACK=OFF
 set MSVC_Z7_OVERRIDE=OFF
-set PYTORCH_BUILD_VERSION=2.3.1
+set PYTORCH_BUILD_VERSION=2.5.0
 set PYTORCH_BUILD_NUMBER=1
 python setup.py install
 ```

cmake/variables.cmake

@@ -243,7 +243,12 @@ set(Boost_USE_STATIC_LIBS OFF)
 set(Boost_USE_DEBUG_RUNTIME OFF)
 set(Boost_COMPILER "${ML_BOOST_COMPILER_VER}")
 
-find_package(Boost 1.83.0 CONFIG EXACT REQUIRED COMPONENTS iostreams filesystem program_options regex date_time log log_setup thread unit_test_framework)
+set(Boost_VERSION 1.83.0)
+if (CMAKE_SYSTEM_NAME STREQUAL "Windows")
+  message(WARNING "Using Boost 1.86.0 on Windows only.")
+  set(Boost_VERSION 1.86.0)
+endif()
+find_package(Boost ${Boost_VERSION} CONFIG EXACT REQUIRED COMPONENTS iostreams filesystem program_options regex date_time log log_setup thread unit_test_framework)
 if(Boost_FOUND)
   list(APPEND ML_SYSTEM_INCLUDE_DIRECTORIES ${Boost_INCLUDE_DIRS})
 endif()

dev-tools/build_windows_third_party_deps.ps1

@@ -58,27 +58,25 @@ $ZipDestination="\tools\$Archive"
 (New-Object Net.WebClient).DownloadFile($ZipSource, $ZipDestination)
 C:\Progra~1\git\bin\bash.exe -c "tar Jxvf libxml2-2.9.14.tar.xz"
 cd \tools\libxml2-2.9.14\win32
-& 'C:\Program Files\Git\usr\bin\sed.exe' -i -e 's|CFLAGS = `$(CFLAGS) /D "NDEBUG" /O2|CFLAGS = `$(CFLAGS) /D "NDEBUG" /O2 /Zi /D_WIN32_WINNT=0x0601|' Makefile.msvc
+(Get-Content .\Makefile.msvc) -replace """NDEBUG"" /O2" , """NDEBUG"" /O2 /Zi /D_WIN32_WINNT=0x0601" | Set-Content .\Makefile.msvc
 cscript configure.js iconv=no prefix=C:\usr\local
 nmake
 nmake install
 Write-Host "--- Done Installing libxml2 2.9.14"
 
-# Build and install Boost 1.83.0
-Write-Host "--- Installing boost 1.83.0"
+# Build and install Boost 1.86.0
+Write-Host "--- Installing boost 1.86.0"
 cd c:\tools
-$Archive="boost_1_83_0.tar.bz2"
-$ZipSource="https://boostorg.jfrog.io/artifactory/main/release/1.83.0/source/$Archive"
+$Archive="boost_1_86_0.tar.bz2"
+$ZipSource="https://boostorg.jfrog.io/artifactory/main/release/1.86.0/source/$Archive"
 $ZipDestination="\tools\$Archive"
 (New-Object Net.WebClient).DownloadFile($ZipSource, $ZipDestination)
-C:\Progra~1\git\bin\bash.exe -c "tar jxvf boost_1_83_0.tar.bz2"
-cd \tools\boost_1_83_0
-& 'C:\Program Files\Git\usr\bin\sed.exe' -i -e 's|(13ul)(29ul)(53ul)(97ul)(193ul)(389ul)(769ul)(1543ul)(3079ul)(6151ul)( |(3ul)(13ul)(29ul)(53ul)(97ul)(193ul)(389ul)(769ul)(1543ul)(3079ul)(6151ul)(       |' boost/unordered/detail/prime_fmod.hpp
-& 'C:\Program Files\Git\usr\bin\sed.exe' -i -e 's/14\.3/14\.4/' tools/build/src/tools/msvc.jam
-c:\tools\boost_1_83_0\bootstrap.bat
-c:\tools\boost_1_83_0\b2 -j6 --layout=versioned --disable-icu --toolset=msvc-14.4 cxxflags="-std:c++17" linkflags="-std:c++17" --build-type=complete -sZLIB_INCLUDE="C:\tools\zlib-1.3.1" -sZLIB_LIBPATH="C:\tools\zlib-1.3.1" -sZLIB_NAME=zdll --without-context --without-coroutine --without-graph_parallel --without-mpi --without-python architecture=x86 address-model=64 optimization=speed inlining=full define=BOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS define=BOOST_LOG_WITHOUT_DEBUG_OUTPUT define=BOOST_LOG_WITHOUT_EVENT_LOG define=BOOST_LOG_WITHOUT_SYSLOG define=BOOST_LOG_WITHOUT_IPC define=_WIN32_WINNT=0x0601
-c:\tools\boost_1_83_0\b2 install --prefix=C:\usr\local --layout=versioned --disable-icu --toolset=msvc-14.4 cxxflags="-std:c++17" linkflags="-std:c++17" --build-type=complete -sZLIB_INCLUDE="C:\tools\zlib-1.3.1" -sZLIB_LIBPATH="C:\tools\zlib-1.3.1" -sZLIB_NAME=zdll --without-context --without-coroutine --without-graph_parallel --without-mpi --without-python architecture=x86 address-model=64 optimization=speed inlining=full define=BOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS define=BOOST_LOG_WITHOUT_DEBUG_OUTPUT define=BOOST_LOG_WITHOUT_EVENT_LOG define=BOOST_LOG_WITHOUT_SYSLOG define=BOOST_LOG_WITHOUT_IPC define=_WIN32_WINNT=0x0601
-Write-Host "--- Done Installing boost 1.83.0"
+C:\Progra~1\git\bin\bash.exe -c "tar jxvf boost_1_86_0.tar.bz2"
+cd \tools\boost_1_86_0
+(Get-Content .\boost\unordered\detail\prime_fmod.hpp) -replace "{13ul" , "{3ul, 13ul" | Set-Content .\boost\unordered\detail\prime_fmod.hpp
+c:\tools\boost_1_86_0\bootstrap.bat
+c:\tools\boost_1_86_0\b2 -j6 install --prefix=C:\usr\local --layout=versioned --disable-icu --toolset=msvc-14.4 cxxflags="-std:c++17" linkflags="-std:c++17" --build-type=complete -sZLIB_INCLUDE="C:\tools\zlib-1.3.1" -sZLIB_LIBPATH="C:\tools\zlib-1.3.1" -sZLIB_NAME=zdll --without-context --without-coroutine --without-graph_parallel --without-mpi --without-python architecture=x86 address-model=64 optimization=speed inlining=full define=BOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS define=BOOST_LOG_WITHOUT_DEBUG_OUTPUT define=BOOST_LOG_WITHOUT_EVENT_LOG define=BOOST_LOG_WITHOUT_SYSLOG define=BOOST_LOG_WITHOUT_IPC define=_WIN32_WINNT=0x0601
+Write-Host "--- Done Installing boost 1.86.0"
 
 
 # Build and install strptime
@@ -111,18 +109,19 @@ pip install numpy ninja pyyaml mkl mkl-include setuptools cmake cffi typing_exte
 Write-Host "--- Done Installing python modules"
 
 # Move the experimental MS libomp libraries out of the way temporarily, as we don't want to inadvertently redistribute them.
-mv "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\14.41.34120\lib\x64\libomp.lib" "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\14.41.34120\lib\x64\libomp.lib.bak"
-mv "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\14.41.34120\lib\x64\libompd.lib" "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\14.41.34120\lib\x64\libompd.lib.bak"
+$f = ls 'C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\' -name | Select-Object -Last 1
+mv "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\$f\lib\x64\libomp.lib" "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\$f\lib\x64\libomp.lib.bak"
+mv "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\$f\lib\x64\libompd.lib" "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\$f\lib\x64\libompd.lib.bak"
 
 
 # Build and install PyTorch (this may take a while)
 Write-Host "--- Installing PyTorch locally"
 cd c:\tools
-git clone --depth=1 --branch=v2.3.1 https://github.com/pytorch/pytorch.git
+git clone --depth=1 --branch=v2.5.0 https://github.com/pytorch/pytorch.git
 cd pytorch
 git submodule sync
 git submodule update --init --recursive
-& 'C:\Program Files\Git\usr\bin\sed.exe' -i -e '/pipe(/{N; s/std::unique_ptr<FILE, decltype(&_pclose)> pipe(\n      _wpopen(cmd.c_str(), L"r"), _pclose);/std::unique_ptr<FILE> pipe;/}' -e '/_wspawnve/{N; s/intptr_t r = _wspawnve(_P_WAIT, comspec, a, e.data());\n  return r/return -1;/}' torch/csrc/jit/codegen/fuser/cpu/fused_kernel.cpp
+(Get-Content ".\torch\csrc\jit\codegen\fuser\cpu\fused_kernel.cpp" -Raw) -replace "(?ms)std::unique_ptr.*?pipe.*?;" , "std::unique_ptr<FILE> pipe;" -replace "(?ms)intptr_t r = _wspawnve.*?return r;" , "return -1;" | Set-Content -Path ".\torch\csrc\jit\codegen\fuser\cpu\fused_kernel.cpp"
 set BUILD_TEST=OFF
 set BUILD_CAFFE2=OFF
 set USE_NUMPY=OFF
@@ -132,7 +131,7 @@ set USE_QNNPACK=OFF
 set USE_PYTORCH_QNNPACK=OFF
 set USE_XNNPACK=OFF
 set MSVC_Z7_OVERRIDE=OFF
-set PYTORCH_BUILD_VERSION=2.3.1
+set PYTORCH_BUILD_VERSION=2.5.0
 set PYTORCH_BUILD_NUMBER=1
 python setup.py install
 Write-Host "--- Done Installing PyTorch locally"
@@ -154,8 +153,8 @@ cp torch\lib\asmjit.lib c:\usr\local\lib\
 Write-Host "--- Done Installing PyTorch globally"
 
 # Move the libomp libraries back where we found them
-mv "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\14.41.34120\lib\x64\libomp.lib.bak" "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\14.41.34120\lib\x64\libomp.lib"
-mv "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\14.41.34120\lib\x64\libompd.lib.bak" "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\14.41.34120\lib\x64\libompd.lib"
+mv "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\$f\lib\x64\libomp.lib.bak" "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\$f\lib\x64\libomp.lib"
+mv "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\$f\lib\x64\libompd.lib.bak" "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\$f\lib\x64\libompd.lib"
 
 # Uninstall python, it was only needed for the PyTorch build
 & c:\tools\$PythonArchive /uninstall /quiet

docs/CHANGELOG.asciidoc

@@ -28,6 +28,12 @@
 
 //=== Regressions
 
+== {es} version 8.18.0
+
+=== Enhancements
+
+* Update the PyTorch library to version 2.5.0. (See {ml-pull}2783[#2783].)
+
 == {es} version 8.16.0
 
 === Enhancements

lib/core/unittest/CMemoryUsageTest.cc

@@ -1287,12 +1287,12 @@ BOOST_AUTO_TEST_CASE(testSmallVector) {
     BOOST_REQUIRE_EQUAL(0, extraMem);
     growShrink.push_back(1.7);
     extraMem = core::memory::dynamicSize(growShrink);
-    // Interesting (shocking?) result: once a boost::small_vector has switched
-    // off of internal storage it will NEVER go back to internal storage.
-    // Arguably this is a bug, and this assertion might start failing after a
-    // Boost upgrade.  If that happens and changing it to assert extraMem is 0
-    // fixes it then this means boost::small_vector has been improved.
-    BOOST_TEST_REQUIRE(extraMem > 0);
+    // Interestingly we used to assert extraMem > 0 here as it used to be the case
+    // that once a boost::small_vector had switched
+    // off of internal storage it would NEVER go back to internal storage.
+    // Arguably that was a bug, and this assertion started failing after
+    // upgrading Boost to 1.86.0, meaning that boost::small_vector has been improved.
+    BOOST_TEST_REQUIRE(extraMem >= 0); // Change this to "==" when all platforms have been upgraded to Boost 1.86.0
 }
 
 BOOST_AUTO_TEST_CASE(testAlignedVector) {

lib/model/CBucketGatherer.cc

@@ -24,6 +24,8 @@
 
 #include <model/CDataGatherer.h>
 
+#include <boost/tuple/tuple.hpp>
+
 #include <algorithm>
 #include <map>