Skip to content

Commit 2713107

Browse files
droberts195droberts195
authored
[ML] Second attempt at CMake build (elastic#2335)
The first merge of the CMake changes to the main branch were reverted due to subtle problems with the generated artifacts. This PR will be for a second attempt to integrate the CMake changes after further fixes and testing.
1 parent db07434 commit 2713107

File tree

183 files changed

+3499
-6147
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

183 files changed

+3499
-6147
lines changed

Diff for: ‎.dockerignore

+6
Original file line numberDiff line numberDiff line change
@@ -81,3 +81,9 @@ boost_test_results.xml
8181
**/.objs/*.*
8282
!**/.objs/.gitignore
8383

84+
# CMake
85+
cmake_install.cmake
86+
CMakeCache.txt
87+
CMakeFiles/
88+
cmake-build-release/
89+
cmake-build-debug/

Diff for: ‎.gitignore

+7
Original file line numberDiff line numberDiff line change
@@ -76,3 +76,10 @@ cppunit_results.xml
7676
boost_test_results.xml
7777
**/.objs/*.*
7878
!**/.objs/.gitignore
79+
80+
# CMake
81+
cmake_install.cmake
82+
CMakeCache.txt
83+
CMakeFiles/
84+
cmake-build-release/
85+
cmake-build-debug/

Diff for: ‎3rd_party/3rd_party.cmake

+272
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,272 @@
1+
#
2+
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
# or more contributor license agreements. Licensed under the Elastic License
4+
# 2.0 and the following additional limitation. Functionality enabled by the
5+
# files subject to the Elastic License 2.0 may only be used in production when
6+
# invoked by an Elasticsearch process with a license key installed that permits
7+
# use of machine learning features. You may not use this file except in
8+
# compliance with the Elastic License 2.0 and the foregoing additional
9+
# limitation.
10+
#
11+
12+
#
13+
# Various other C programs/libraries need to be installed on the local machine,
14+
# in directories listed in the platform specific variables listed below.
15+
#
16+
# Usage:
17+
# cmake -D INSTALL_DIR=${INSTALL_DIR} -P 3rd_party.cmake
18+
#
19+
# A complication is that Java's zip functionality turns symlinks into actual
20+
# files, so we need to ensure libraries are only picked up by the name that the
21+
# dynamic loader will look for, and no symlinks are used. Otherwise the
22+
# installer becomes very bloated.
23+
#
24+
string(TOLOWER ${CMAKE_HOST_SYSTEM_NAME} HOST_SYSTEM_NAME)
25+
message(STATUS "3rd_party: HOST_SYSTEM_NAME=${HOST_SYSTEM_NAME}")
26+
27+
if ("${HOST_SYSTEM_NAME}" STREQUAL "windows")
28+
# We only support x86_64
29+
set(HOST_SYSTEM_PROCESSOR "x86_64")
30+
else()
31+
execute_process(COMMAND uname -m OUTPUT_VARIABLE HOST_SYSTEM_PROCESSOR OUTPUT_STRIP_TRAILING_WHITESPACE)
32+
string(REPLACE arm aarch HOST_SYSTEM_PROCESSOR ${HOST_SYSTEM_PROCESSOR})
33+
endif()
34+
message(STATUS "3rd_party: HOST_SYSTEM_PROCESSOR=${HOST_SYSTEM_PROCESSOR}")
35+
set(CMAKE_TOOLCHAIN_FILE "cmake/${HOST_SYSTEM_NAME}-${HOST_SYSTEM_PROCESSOR}.cmake")
36+
37+
set(ARCH ${HOST_SYSTEM_PROCESSOR})
38+
39+
if ("${HOST_SYSTEM_NAME}" STREQUAL "darwin")
40+
message(STATUS "3rd_party: Copying macOS 3rd party libraries")
41+
set(BOOST_LOCATION "/usr/local/lib")
42+
set(BOOST_COMPILER "clang")
43+
if( "${ARCH}" STREQUAL "x86_64" )
44+
set(BOOST_ARCH "x64")
45+
else()
46+
set(BOOST_ARCH "a64")
47+
endif()
48+
set(BOOST_EXTENSION "mt-${BOOST_ARCH}-1_77.dylib")
49+
set(BOOST_LIBRARIES "atomic" "chrono" "date_time" "filesystem" "iostreams" "log" "log_setup" "program_options" "regex" "system" "thread" "unit_test_framework")
50+
set(XML_LOCATION)
51+
set(GCC_RT_LOCATION)
52+
set(STL_LOCATION)
53+
set(OMP_LOCATION)
54+
set(ZLIB_LOCATION)
55+
set(TORCH_LIBRARIES "torch_cpu" "c10")
56+
set(TORCH_LOCATION "/usr/local/lib")
57+
set(TORCH_EXTENSION ".dylib")
58+
elseif ("${HOST_SYSTEM_NAME}" STREQUAL "linux")
59+
if(NOT DEFINED ENV{CPP_CROSS_COMPILE})
60+
message(STATUS "3rd_party: NOT cross compiling. Copying Linux 3rd party libraries")
61+
set(BOOST_LOCATION "/usr/local/gcc103/lib")
62+
set(BOOST_COMPILER "gcc")
63+
if( "${ARCH}" STREQUAL "aarch64" )
64+
set(BOOST_ARCH "a64")
65+
else()
66+
set(BOOST_ARCH "x64")
67+
set(MKL_LOCATION "/usr/local/gcc103/lib")
68+
set(MKL_EXTENSION ".so")
69+
set(MKL_PREFIX "libmkl_")
70+
set(MKL_LIBRARIES "avx" "avx2" "avx512" "avx512_mic" "core" "def" "gnu_thread" "intel_lp64" "mc3" "vml_avx" "vml_avx2" "vml_avx512" "vml_avx512_mic" "vml_cmpt" "vml_def" "vml_mc3")
71+
endif()
72+
set(BOOST_EXTENSION mt-${BOOST_ARCH}-1_77.so.1.77.0)
73+
set(BOOST_LIBRARIES "atomic" "chrono" "date_time" "filesystem" "iostreams" "log" "log_setup" "program_options" "regex" "system" "thread" "unit_test_framework")
74+
set(XML_LOCATION "/usr/local/gcc103/lib")
75+
set(XML_EXTENSION ".so.2")
76+
set(GCC_RT_LOCATION "/usr/local/gcc103/lib64")
77+
set(GCC_RT_EXTENSION ".so.1")
78+
set(STL_LOCATION "/usr/local/gcc103/lib64")
79+
set(STL_PATTERN "libstdc++")
80+
set(STL_EXTENSION ".so.6")
81+
set(OMP_LOCATION "/usr/local/gcc103/lib64")
82+
set(OMP_PATTERN "libgomp")
83+
set(OMP_EXTENSION ".so.1")
84+
set(ZLIB_LOCATION)
85+
set(TORCH_LIBRARIES "libtorch_cpu" "libc10")
86+
set(TORCH_LOCATION "/usr/local/gcc103/lib")
87+
set(TORCH_EXTENSION ".so")
88+
elseif("$ENV{CPP_CROSS_COMPILE}" STREQUAL "macosx")
89+
message(STATUS "3rd_party: Cross compile for macosx: Copying macOS 3rd party libraries")
90+
set(SYSROOT "/usr/local/sysroot-x86_64-apple-macosx10.14")
91+
set(BOOST_LOCATION "${SYSROOT}/usr/local/lib")
92+
set(BOOST_COMPILER "clang")
93+
set(BOOST_EXTENSION "mt-x64-1_77.dylib")
94+
set(BOOST_LIBRARIES "atomic" "chrono" "date_time" "filesystem" "iostreams" "log" "log_setup" "program_options" "regex" "system" "thread" "unit_test_framework")
95+
set(XML_LOCATION)
96+
set(GCC_RT_LOCATION)
97+
set(STL_LOCATION)
98+
set(OMP_LOCATION)
99+
set(ZLIB_LOCATION)
100+
set(TORCH_LIBRARIES "libtorch_cpu" "libc10")
101+
set(TORCH_LOCATION "${SYSROOT}/usr/local/lib")
102+
set(TORCH_EXTENSION ".dylib")
103+
else()
104+
message(STATUS "3rd_party: Cross compile for linux-aarch64: Copying Linux 3rd party libraries")
105+
set(SYSROOT "/usr/local/sysroot-$ENV{CPP_CROSS_COMPILE}-linux-gnu")
106+
set(BOOST_LOCATION "${SYSROOT}/usr/local/gcc103/lib")
107+
set(BOOST_COMPILER "gcc")
108+
if("$ENV{CPP_CROSS_COMPILE}" STREQUAL "aarch64")
109+
set(BOOST_ARCH "a64")
110+
else()
111+
message(FATAL_ERROR "Cannot cross compile to $ENV{CPP_CROSS_COMPILE}")
112+
return()
113+
endif()
114+
set(BOOST_EXTENSION "mt-${BOOST_ARCH}-1_77.so.1.77.0")
115+
set(BOOST_LIBRARIES "atomic" "chrono" "date_time" "filesystem" "iostreams" "log" "log_setup" "program_options" "regex" "system" "thread" "unit_test_framework")
116+
set(XML_LOCATION "${SYSROOT}/usr/local/gcc103/lib")
117+
set(XML_EXTENSION ".so.2")
118+
set(GCC_RT_LOCATION "${SYSROOT}/usr/local/gcc103/lib64")
119+
set(GCC_RT_EXTENSION ".so.1")
120+
set(STL_LOCATION "${SYSROOT}/usr/local/gcc103/lib64")
121+
set(STL_PATTERN "libstdc++")
122+
set(STL_EXTENSION ".so.6")
123+
set(OMP_LOCATION "${SYSROOT}/usr/local/gcc103/lib64")
124+
set(OMP_PATTERN "libgomp")
125+
set(OMP_EXTENSION ".so.1")
126+
set(ZLIB_LOCATION)
127+
set(TORCH_LIBRARIES "libtorch_cpu" "libc10")
128+
set(TORCH_LOCATION "${SYSROOT}/usr/local/gcc103/lib")
129+
set(TORCH_EXTENSION ".so")
130+
endif()
131+
else()
132+
message(STATUS "Copying Windows 3rd party libraries")
133+
set(LOCAL_DRIVE "C:")
134+
if(DEFINED ENV{LOCAL_DRIVE})
135+
set(LOCAL_DRIVE $ENV{LOCAL_DRIVE})
136+
endif()
137+
# These directories are correct for the way our Windows 2016 build
138+
# server is currently set up
139+
set(BOOST_LOCATION "${LOCAL_DRIVE}/usr/local/lib")
140+
set(BOOST_COMPILER "vc")
141+
set(BOOST_EXTENSION "mt-x64-1_77.dll")
142+
set(BOOST_LIBRARIES "chrono" "date_time" "filesystem" "iostreams" "log" "log_setup" "program_options" "regex" "system" "thread" "unit_test_framework")
143+
set(XML_LOCATION "${LOCAL_DRIVE}/usr/local/bin")
144+
set(XML_EXTENSION ".dll")
145+
set(GCC_RT_LOCATION)
146+
# Read VCBASE from environment if defined, otherwise default to VS Professional 2019
147+
set(VCBASE "Program Files (x86)/Microsoft Visual Studio/2019/Professional")
148+
if (DEFINED ENV{VCBASE})
149+
set(VCBASE $ENV{VCBASE})
150+
endif()
151+
152+
file(GLOB MSVC_VERS "${LOCAL_DRIVE}/${VCBASE}/VC/Tools/MSVC/*")
153+
list(GET MSVC_VERS -1 MSVC_VER)
154+
if(${MSVC_VER} MATCHES "/([^/]+)$")
155+
set(VCVER ${CMAKE_MATCH_1})
156+
endif()
157+
message(STATUS "VCVER: ${VCVER}")
158+
159+
set(STL_LOCATION "${LOCAL_DRIVE}/${VCBASE}/VC/Redist/MSVC/${VCVER}/x64/Microsoft.VC142.CRT")
160+
set(STL_PATTERN "140")
161+
set(STL_EXTENSION ".dll")
162+
set(OMP_LOCATION "${LOCAL_DRIVE}/${VCBASE}/VC/Redist/MSVC/${VCVER}/x64/Microsoft.VC142.OpenMP")
163+
set(OMP_PATTERN "vcomp")
164+
set(OMP_EXTENSION ".dll")
165+
set(ZLIB_LOCATION "${LOCAL_DRIVE}/usr/local/bin")
166+
set(ZLIB_EXTENSION "1.dll")
167+
set(TORCH_LIBRARIES "asmjit" "c10" "fbgemm" "torch_cpu")
168+
set(TORCH_LOCATION "${LOCAL_DRIVE}/usr/local/bin")
169+
set(TORCH_EXTENSION ".dll")
170+
endif()
171+
172+
function(install_libs _target _source_dir _prefix _postfix)
173+
174+
if(NOT EXISTS ${_source_dir})
175+
return()
176+
endif()
177+
178+
set(LIBRARIES ${ARGN})
179+
180+
file(GLOB _LIBS ${_source_dir}/*${_prefix}*${_postfix})
181+
182+
if(_LIBS)
183+
foreach(LIBRARY ${LIBRARIES})
184+
file(GLOB _INST_LIBS ${_source_dir}/*${_prefix}${LIBRARY}*${_postfix})
185+
file(GLOB _OLD_LIBS ${INSTALL_DIR}/*${_prefix}${LIBRARY}*${_postfix})
186+
if (_INST_LIBS)
187+
foreach(_OLD_LIB ${_OLD_LIBS})
188+
if(_OLD_LIB)
189+
file(REMOVE ${_OLD_LIB})
190+
endif()
191+
endforeach()
192+
foreach(_INST_LIB ${_INST_LIBS})
193+
# Resolve symlinks so that we have a handle
194+
# on a plain file.
195+
file(REAL_PATH ${_INST_LIB} _RESOLVED_PATH)
196+
# Strip off any leading directories
197+
set(_RESOLVED_LIB)
198+
if(${_RESOLVED_PATH} MATCHES "/([^/]+)$")
199+
set(_RESOLVED_LIB ${CMAKE_MATCH_1})
200+
endif()
201+
if(${_RESOLVED_PATH} MATCHES "/([^/]+${_postfix})")
202+
set(_LIB ${CMAKE_MATCH_1})
203+
file(COPY ${_RESOLVED_PATH} DESTINATION ${INSTALL_DIR})
204+
# The file we copied may not be precisely the name that we want as we had to resolve symlinks.
205+
# If so we need to rename it.
206+
if(NOT EXISTS ${INSTALL_DIR}/${_LIB})
207+
file(RENAME ${INSTALL_DIR}/${_RESOLVED_LIB} ${INSTALL_DIR}/${_LIB})
208+
endif()
209+
file(CHMOD ${INSTALL_DIR}/${_LIB} PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
210+
else()
211+
file(COPY ${_RESOLVED_PATH} DESTINATION ${INSTALL_DIR})
212+
file(CHMOD ${INSTALL_DIR}/${_RESOLVED_LIB} PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
213+
endif()
214+
endforeach()
215+
endif()
216+
endforeach()
217+
else()
218+
message(FATAL_ERROR "${_target} not found")
219+
return()
220+
endif()
221+
endfunction()
222+
223+
function(third_party _arg)
224+
if("${_arg}" STREQUAL "check")
225+
set(INSTALL_DIR "")
226+
elseif("${_arg}" STREQUAL "add")
227+
if(${ARGC} LESS 2)
228+
message(FATAL_ERROR "Install directory required for 'add'")
229+
return()
230+
endif()
231+
message(STATUS "Setting install directory to ${ARGV1}")
232+
set(INSTALL_DIR ${ARGV1})
233+
else()
234+
message(FATAL_ERROR "Action required: 'add' or 'check'")
235+
return()
236+
endif()
237+
238+
file(MAKE_DIRECTORY "${INSTALL_DIR}")
239+
240+
install_libs("Boost" ${BOOST_LOCATION} "boost*" "${BOOST_COMPILER}*${BOOST_EXTENSION}" ${BOOST_LIBRARIES})
241+
install_libs("libxml2" ${XML_LOCATION} "" "${XML_EXTENSION}" "libxml2")
242+
install_libs("gcc runtime library" ${GCC_RT_LOCATION} "" "${GCC_RT_EXTENSION}" "libgcc_s")
243+
install_libs("C++ standard library" ${STL_LOCATION} "" "${STL_EXTENSION}" "${STL_PATTERN}")
244+
install_libs("OMP runtime library" ${OMP_LOCATION} "" "${OMP_EXTENSION}" "${OMP_PATTERN}")
245+
install_libs("zlib" ${ZLIB_LOCATION} "" "${ZLIB_EXTENSION}" "zlib")
246+
install_libs("Torch libraries" ${TORCH_LOCATION} "" "${TORCH_EXTENSION}" "${TORCH_LIBRARIES}")
247+
install_libs("Intel MKL libraries" ${MKL_LOCATION} "${MKL_PREFIX}" "${MKL_EXTENSION}" "${MKL_LIBRARIES}")
248+
249+
# Special extra platform-specific processing
250+
if ("${HOST_SYSTEM_NAME}" STREQUAL "linux")
251+
if (INSTALL_DIR AND NOT DEFINED ENV{CPP_CROSS_COMPILE} OR NOT "$ENV{CPP_CROSS_COMPILE}" STREQUAL "macosx")
252+
execute_process(COMMAND find . -type f COMMAND egrep -v "^core|-debug$|libMl" COMMAND xargs COMMAND sed -e "s/ /;/g" OUTPUT_VARIABLE FIND_RES WORKING_DIRECTORY "${INSTALL_DIR}" OUTPUT_STRIP_TRAILING_WHITESPACE)
253+
foreach(RES ${FIND_RES})
254+
# Replace RPATH for 3rd party libraries that already have one
255+
execute_process(COMMAND patchelf --print-rpath ${RES} COMMAND grep lib OUTPUT_VARIABLE RPATH_VAR ERROR_VARIABLE RPATH_ERR WORKING_DIRECTORY "${INSTALL_DIR}" OUTPUT_STRIP_TRAILING_WHITESPACE)
256+
if(RPATH_VAR)
257+
execute_process(COMMAND patchelf --force-rpath --set-rpath "$ORIGIN" ${RES} OUTPUT_VARIABLE SET_RPATH_OUT ERROR_VARIABLE SET_RPATH_ERR WORKING_DIRECTORY "${INSTALL_DIR}" OUTPUT_STRIP_TRAILING_WHITESPACE)
258+
if(NOT SET_RPATH_ERR)
259+
message(STATUS "Set RPATH in ${RES}: ${SET_RPATH_OUT}")
260+
endif()
261+
else()
262+
message(STATUS "Did not set RPATH in ${RES}")
263+
endif()
264+
endforeach()
265+
endif()
266+
endif()
267+
268+
endfunction()
269+
270+
if(INSTALL_DIR)
271+
third_party(add ${INSTALL_DIR})
272+
endif()

0 commit comments

Comments
 (0)