Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Copy HPCX paths for CPU images #113

Closed
wants to merge 14 commits into from
Closed

Copy HPCX paths for CPU images #113

Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
b75db30
Try patchelf to link to add hpcx to path
dyastremsky Jun 6, 2023
2366d5b
Update dependencies after rpath patched
dyastremsky Jun 7, 2023
b323a47
Separate ldconfig, use add rpath instead
dyastremsky Jun 7, 2023
5fb7519
Add message if PyTorch ldconfig fails
dyastremsky Jun 7, 2023
5a33af0
Remove patchelf
dyastremsky Jun 8, 2023
1895b09
Copy needed libs to Torch folder
dyastremsky Jun 8, 2023
e9ce30f
Change build to copy hpcx libs to install dir
dyastremsky Jun 8, 2023
79c0b13
Only copy HPCX libs when CPU-only
dyastremsky Jun 8, 2023
fe49cc7
Uncomment docker pull
dyastremsky Jun 8, 2023
523e4c0
Uncomment docker pull, second time
dyastremsky Jun 8, 2023
ba31604
Remove debugging print
dyastremsky Jun 8, 2023
45d2f9f
Remove debugging statements
dyastremsky Jun 8, 2023
2158177
Remove more debugging statements
dyastremsky Jun 8, 2023
a23c72c
Use different name for hpcx container
dyastremsky Jun 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 30 additions & 5 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,31 @@ set(OPENCV_LIBS
"libjpeg.so"
)

# In CPU-only mode, get the hpcx libraries neededby libtorch.so.
if(NOT TRITON_ENABLE_GPU)
set(HPCX_LIBS
"libucc.so.1"
"libucm.so.0"
"libucp.so.0"
"libucs.so.0"
"libuct.so.0"
)
add_custom_command(
OUTPUT
${HPCX_LIBS}
COMMAND docker pull ${TRITON_PYTORCH_DOCKER_IMAGE}
COMMAND docker rm pytorch_backend_hpcxlib || echo "error ignored..." || true
COMMAND docker create --name pytorch_backend_hpcxlib ${TRITON_PYTORCH_DOCKER_IMAGE}
COMMAND docker cp -L pytorch_backend_hpcxlib:/opt/hpcx/ucc/lib/libucc.so.1 libucc.so.1
COMMAND docker cp -L pytorch_backend_hpcxlib:/opt/hpcx/ucx/lib/libucm.so.0 libucm.so.0
COMMAND docker cp -L pytorch_backend_hpcxlib:/opt/hpcx/ucx/lib/libucp.so.0 libucp.so.0
COMMAND docker cp -L pytorch_backend_hpcxlib:/opt/hpcx/ucx/lib/libucs.so.0 libucs.so.0
COMMAND docker cp -L pytorch_backend_hpcxlib:/opt/hpcx/ucx/lib/libuct.so.0 libuct.so.0
COMMAND docker rm pytorch_backend_hpcxlib
VERBATIM
Comment on lines +193 to +213
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Instead of modifying cmake can we copy over the whole /opt/hpcx/ucc/ and /opt/hpcx/ucx in the build.py?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, I will try this approach.

)
endif() # TRITON_ENABLE_GPU

# The patchelf commands ensure the MKL libraries are loaded correctly during runtime
# Without these, the framework/backend complains of missing libraries / symbols and
# in some cases leads to segmentation faults.
Expand Down Expand Up @@ -249,7 +274,7 @@ if (${TRITON_PYTORCH_DOCKER_BUILD})
COMMENT "Extracting pytorch and torchvision libraries and includes from ${TRITON_PYTORCH_DOCKER_IMAGE}"
VERBATIM
)
add_custom_target(ptlib_target DEPENDS ${PT_LIBS} ${LIBTORCH_LIBS} ${OPENCV_LIBS})
add_custom_target(ptlib_target DEPENDS ${PT_LIBS} ${HPCX_LIBS} ${LIBTORCH_LIBS} ${OPENCV_LIBS})
add_library(ptlib SHARED IMPORTED GLOBAL)
add_dependencies(ptlib ptlib_target)

Expand Down Expand Up @@ -415,7 +440,7 @@ install(

if (${TRITON_PYTORCH_DOCKER_BUILD})
set(PT_LIB_PATHS "")
FOREACH(plib ${PT_LIBS} ${LIBTORCH_LIBS} ${OPENCV_LIBS})
FOREACH(plib ${PT_LIBS} ${HPCX_LIBS} ${LIBTORCH_LIBS} ${OPENCV_LIBS})
set(PT_LIB_PATHS ${PT_LIB_PATHS} "${CMAKE_CURRENT_BINARY_DIR}/${plib}")
ENDFOREACH(plib)

Expand All @@ -434,7 +459,7 @@ if (${TRITON_PYTORCH_DOCKER_BUILD})
)
endif() # TRITON_PYTORCH_ENABLE_TORCHTRT

FOREACH(plib ${PT_LIBS} ${LIBTORCH_LIBS} ${OPENCV_LIBS})
FOREACH(plib ${PT_LIBS} ${HPCX_LIBS} ${LIBTORCH_LIBS} ${OPENCV_LIBS})
install(
CODE
"EXECUTE_PROCESS(
Expand Down Expand Up @@ -469,7 +494,7 @@ if (${TRITON_PYTORCH_DOCKER_BUILD})
endif()"
)
else()
FOREACH(plib ${PT_LIBS})
FOREACH(plib ${PT_LIBS} ${HPCX_LIBS})
set(PT_LIB_PATHS ${PT_LIB_PATHS} "${TRITON_PYTORCH_LIB_PATHS}/${plib}")
ENDFOREACH(plib)

Expand All @@ -479,7 +504,7 @@ else()
DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/pytorch
)

FOREACH(plib ${PT_LIBS})
FOREACH(plib ${PT_LIBS} ${HPCX_LIBS})
install(
CODE
"EXECUTE_PROCESS(
Expand Down