Skip to content

Commit

Permalink
iGPU build refactor (#6684)
Browse files Browse the repository at this point in the history
  • Loading branch information
@kthui
kthui authored Dec 12, 2023
1 parent aa1badb commit 2b607f2
Show file tree
Hide file tree
Showing 5 changed files with 104 additions and 153 deletions.
20 changes: 11 additions & 9 deletions Dockerfile.QA
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ ARG TRITON_CORE_REPO_TAG=main
ARG TRITON_THIRD_PARTY_REPO_TAG=main
ARG TRITON_BACKEND_REPO_TAG=main
ARG TRITONTMP_DIR=/tmp
ARG IGPU_BUILD=0

############################################################################
## Test artifacts built as part of the tritonserver build are
Expand All @@ -44,6 +45,7 @@ ARG TRITON_COMMON_REPO_TAG
ARG TRITON_CORE_REPO_TAG
ARG TRITON_THIRD_PARTY_REPO_TAG
ARG TRITON_BACKEND_REPO_TAG
ARG IGPU_BUILD

# Ensure apt-get won't prompt for selecting options
ENV DEBIAN_FRONTEND=noninteractive
Expand Down Expand Up @@ -237,21 +239,21 @@ RUN if [ -d qa/L0_model_control_stress ]; then \
cp -r qa/L0_model_control_stress/. qa/L0_model_control_stress_valgrind_massif; \
fi

RUN cp backends/repeat/libtriton_repeat.so qa/L0_model_config/.

RUN mkdir -p qa/L0_decoupled/models/repeat_int32/1 && \
cp backends/repeat/libtriton_repeat.so \
qa/L0_decoupled/models/repeat_int32/1/.
RUN mkdir -p qa/L0_decoupled/models/square_int32/1 && \
cp backends/square/libtriton_square.so \
qa/L0_decoupled/models/square_int32/1/.
RUN mkdir -p qa/L0_decoupled/models/identity_int32/1
RUN mkdir -p qa/L0_decoupled/models/simple_repeat/1 && \
mkdir -p qa/L0_decoupled/models/square_int32/1 && \
mkdir -p qa/L0_decoupled/models/identity_int32/1 && \
mkdir -p qa/L0_decoupled/models/simple_repeat/1 && \
mkdir -p qa/L0_decoupled/models/fan_repeat/1 && \
mkdir -p qa/L0_decoupled/models/sequence_repeat/1 && \
mkdir -p qa/L0_decoupled/models/repeat_square/1 && \
mkdir -p qa/L0_decoupled/models/nested_square/1

RUN if [ "$IGPU_BUILD" == "0" ]; then \
cp backends/repeat/libtriton_repeat.so qa/L0_model_config && \
cp backends/repeat/libtriton_repeat.so qa/L0_decoupled/models/repeat_int32/1 && \
cp backends/square/libtriton_square.so qa/L0_decoupled/models/square_int32/1; \
fi

RUN cp -r qa/L0_decoupled/models qa/L0_decoupled/python_models/ && \
cp /workspace/tritonbuild/python/examples/decoupled/repeat_model.py \
qa/L0_decoupled/python_models/repeat_int32/1/. && \
Expand Down
210 changes: 81 additions & 129 deletions build.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -599,9 +599,9 @@ def backend_cmake_args(images, components, be, install_dir, library_paths):
"Warning: Detected docker build is used for Windows, backend utility 'device memory tracker' will be disabled due to missing library in CUDA Windows docker image."
)
cargs.append(cmake_backend_enable(be, "TRITON_ENABLE_MEMORY_TRACKER", False))
elif target_platform() == "jetpack":
elif target_platform() == "igpu":
print(
"Warning: Detected Jetpack build, backend utility 'device memory tracker' will be disabled as Jetpack doesn't contain required version of the library."
"Warning: Detected iGPU build, backend utility 'device memory tracker' will be disabled as iGPU doesn't contain required version of the library."
)
cargs.append(cmake_backend_enable(be, "TRITON_ENABLE_MEMORY_TRACKER", False))
elif FLAGS.enable_gpu:
Expand All @@ -613,44 +613,23 @@ def backend_cmake_args(images, components, be, install_dir, library_paths):


def pytorch_cmake_args(images):
# If platform is jetpack do not use docker based build
if target_platform() == "jetpack":
if "pytorch" not in library_paths:
raise Exception(
"Must specify library path for pytorch using --library-paths=pytorch:<path_to_pytorch>"
)
pt_lib_path = library_paths["pytorch"] + "/lib"
pt_include_paths = ""
for suffix in [
"include/torch",
"include/torch/torch/csrc/api/include",
"include/torchvision",
]:
pt_include_paths += library_paths["pytorch"] + "/" + suffix + ";"
cargs = [
cmake_backend_arg(
"pytorch", "TRITON_PYTORCH_INCLUDE_PATHS", None, pt_include_paths
),
cmake_backend_arg("pytorch", "TRITON_PYTORCH_LIB_PATHS", None, pt_lib_path),
]
if "pytorch" in images:
image = images["pytorch"]
else:
if "pytorch" in images:
image = images["pytorch"]
else:
image = "nvcr.io/nvidia/pytorch:{}-py3".format(
FLAGS.upstream_container_version
)
cargs = [
cmake_backend_arg("pytorch", "TRITON_PYTORCH_DOCKER_IMAGE", None, image),
]
image = "nvcr.io/nvidia/pytorch:{}-py3".format(
FLAGS.upstream_container_version
)
cargs = [
cmake_backend_arg("pytorch", "TRITON_PYTORCH_DOCKER_IMAGE", None, image),
]

if FLAGS.enable_gpu:
cargs.append(
cmake_backend_enable("pytorch", "TRITON_PYTORCH_ENABLE_TORCHTRT", True)
)
if FLAGS.enable_gpu:
cargs.append(
cmake_backend_enable("pytorch", "TRITON_ENABLE_NVTX", FLAGS.enable_nvtx)
cmake_backend_enable("pytorch", "TRITON_PYTORCH_ENABLE_TORCHTRT", True)
)
cargs.append(
cmake_backend_enable("pytorch", "TRITON_ENABLE_NVTX", FLAGS.enable_nvtx)
)
return cargs


Expand All @@ -672,69 +651,57 @@ def onnxruntime_cmake_args(images, library_paths):
)
)

# If platform is jetpack do not use docker based build
if target_platform() == "jetpack":
if "onnxruntime" not in library_paths:
raise Exception(
"Must specify library path for onnxruntime using --library-paths=onnxruntime:<path_to_onnxruntime>"

if target_platform() == "windows":
if "base" in images:
cargs.append(
cmake_backend_arg(
"onnxruntime", "TRITON_BUILD_CONTAINER", None, images["base"]
)
)
ort_lib_path = library_paths["onnxruntime"] + "/lib"
ort_include_path = library_paths["onnxruntime"] + "/include"
cargs += [
cmake_backend_arg(
"onnxruntime",
"TRITON_ONNXRUNTIME_INCLUDE_PATHS",
None,
ort_include_path,
),
cmake_backend_arg(
"onnxruntime", "TRITON_ONNXRUNTIME_LIB_PATHS", None, ort_lib_path
),
cmake_backend_enable(
"onnxruntime", "TRITON_ENABLE_ONNXRUNTIME_OPENVINO", False
),
]
else:
if target_platform() == "windows":
if "base" in images:
cargs.append(
cmake_backend_arg(
"onnxruntime", "TRITON_BUILD_CONTAINER", None, images["base"]
)
if "base" in images:
cargs.append(
cmake_backend_arg(
"onnxruntime", "TRITON_BUILD_CONTAINER", None, images["base"]
)
)
else:
if "base" in images:
cargs.append(
cmake_backend_arg(
"onnxruntime", "TRITON_BUILD_CONTAINER", None, images["base"]
)
)
else:
cargs.append(
cmake_backend_arg(
"onnxruntime",
"TRITON_BUILD_CONTAINER_VERSION",
None,
TRITON_VERSION_MAP[FLAGS.version][1],
)
cargs.append(
cmake_backend_arg(
"onnxruntime",
"TRITON_BUILD_CONTAINER_VERSION",
None,
TRITON_VERSION_MAP[FLAGS.version][1],
)
)

if (target_machine() != "aarch64") and (
TRITON_VERSION_MAP[FLAGS.version][3] is not None
):
cargs.append(
cmake_backend_enable(
"onnxruntime", "TRITON_ENABLE_ONNXRUNTIME_OPENVINO", True
)
if (target_machine() != "aarch64") and (
TRITON_VERSION_MAP[FLAGS.version][3] is not None
):
cargs.append(
cmake_backend_enable(
"onnxruntime", "TRITON_ENABLE_ONNXRUNTIME_OPENVINO", True
)
cargs.append(
cmake_backend_arg(
"onnxruntime",
"TRITON_BUILD_ONNXRUNTIME_OPENVINO_VERSION",
None,
TRITON_VERSION_MAP[FLAGS.version][3],
)
)
cargs.append(
cmake_backend_arg(
"onnxruntime",
"TRITON_BUILD_ONNXRUNTIME_OPENVINO_VERSION",
None,
TRITON_VERSION_MAP[FLAGS.version][3],
)
)

if target_platform() == "igpu":
cargs.append(
cmake_backend_arg(
"onnxruntime",
"TRITON_BUILD_TARGET_PLATFORM",
None,
target_platform(),
)
)

return cargs

Expand Down Expand Up @@ -790,36 +757,20 @@ def tensorrt_cmake_args():

def tensorflow_cmake_args(images, library_paths):
backend_name = "tensorflow"

# If platform is jetpack do not use docker images
extra_args = []
if target_platform() == "jetpack":
if backend_name in library_paths:
extra_args = [
cmake_backend_arg(
backend_name,
"TRITON_TENSORFLOW_LIB_PATHS",
None,
library_paths[backend_name],
)
]
else:
raise Exception(
f"Must specify library path for {backend_name} using --library-paths={backend_name}:<path_to_{backend_name}>"
)

# If a specific TF image is specified use it, otherwise pull from NGC.
if backend_name in images:
image = images[backend_name]
else:
# If a specific TF image is specified use it, otherwise pull from NGC.
if backend_name in images:
image = images[backend_name]
else:
image = "nvcr.io/nvidia/tensorflow:{}-tf2-py3".format(
FLAGS.upstream_container_version
)
extra_args = [
cmake_backend_arg(
backend_name, "TRITON_TENSORFLOW_DOCKER_IMAGE", None, image
)
]
image = "nvcr.io/nvidia/tensorflow:{}-tf2-py3".format(
FLAGS.upstream_container_version
)
extra_args = [
cmake_backend_arg(
backend_name, "TRITON_TENSORFLOW_DOCKER_IMAGE", None, image
)
]
return extra_args


Expand Down Expand Up @@ -1639,7 +1590,7 @@ def create_docker_build_script(script_name, container_install_dir, container_ci_
docker_script.cmd(["docker", "rm", "tritonserver_builder"])
else:
docker_script._file.write(
'if [ "$(docker ps -a | grep tritonserver_builder)" ]; then docker rm tritonserver_builder; fi\n'
'if [ "$(docker ps -a | grep tritonserver_builder)" ]; then docker rm -f tritonserver_builder; fi\n'
)

docker_script.cmd(runargs, check_exitcode=True)
Expand Down Expand Up @@ -2018,14 +1969,15 @@ def cibase_build(
if "onnxruntime" in backends:
ort_install_dir = os.path.join(build_dir, "onnxruntime", "install")
cmake_script.mkdir(os.path.join(ci_dir, "qa", "L0_custom_ops"))
cmake_script.cp(
os.path.join(ort_install_dir, "test", "libcustom_op_library.so"),
os.path.join(ci_dir, "qa", "L0_custom_ops"),
)
cmake_script.cp(
os.path.join(ort_install_dir, "test", "custom_op_test.onnx"),
os.path.join(ci_dir, "qa", "L0_custom_ops"),
)
if target_platform() != "igpu":
cmake_script.cp(
os.path.join(ort_install_dir, "test", "libcustom_op_library.so"),
os.path.join(ci_dir, "qa", "L0_custom_ops"),
)
cmake_script.cp(
os.path.join(ort_install_dir, "test", "custom_op_test.onnx"),
os.path.join(ci_dir, "qa", "L0_custom_ops"),
)
# [WIP] other way than wildcard?
backend_tests = os.path.join(build_dir, "onnxruntime", "test", "*")
cmake_script.cpdir(backend_tests, os.path.join(ci_dir, "qa"))
Expand Down Expand Up @@ -2187,7 +2139,7 @@ def enable_all():
"--target-platform",
required=False,
default=None,
help='Target platform for build, can be "linux", "windows" or "jetpack". If not specified, build targets the current platform.',
help='Target platform for build, can be "linux", "windows" or "igpu". If not specified, build targets the current platform.',
)
parser.add_argument(
"--target-machine",
Expand Down
5 changes: 3 additions & 2 deletions qa/L0_backend_python/examples/test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,11 @@ RET=0
rm -fr *.log python_backend/

# Install torch
# Skip torch and torchvision install on Jetson since it is already installed.
pip3 uninstall -y torch
if [ "$TEST_JETSON" == "0" ]; then
pip3 uninstall -y torch
pip3 install torch==2.0.0+cu117 -f https://download.pytorch.org/whl/torch_stable.html torchvision==0.15.0+cu117
else
pip3 install torch==2.0.0 -f https://download.pytorch.org/whl/torch_stable.html torchvision==0.15.0
fi

# Install `validators` for Model Instance Kind example
Expand Down
2 changes: 1 addition & 1 deletion qa/L0_backend_python/test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,10 +147,10 @@ mkdir -p models/dlpack_identity/1/
cp ../python_models/dlpack_identity/model.py ./models/dlpack_identity/1/
cp ../python_models/dlpack_identity/config.pbtxt ./models/dlpack_identity

# Skip torch install on Jetson since it is already installed.
if [ "$TEST_JETSON" == "0" ]; then
pip3 install torch==1.13.0+cpu -f https://download.pytorch.org/whl/torch_stable.html
else
pip3 install torch==1.13.0 -f https://download.pytorch.org/whl/torch_stable.html
# GPU tensor tests are disabled on jetson
EXPECTED_NUM_TESTS=9
fi
Expand Down
20 changes: 8 additions & 12 deletions qa/L0_infer/test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,12 +143,10 @@ BATCH=${BATCH:="1"}
export BATCH

if [[ $BACKENDS == *"python_dlpack"* ]]; then
if [ "$TEST_JETSON" == "0" ]; then
if [[ "aarch64" != $(uname -m) ]] ; then
pip3 install torch==1.13.0+cpu -f https://download.pytorch.org/whl/torch_stable.html
else
pip3 install torch==1.13.0 -f https://download.pytorch.org/whl/torch_stable.html
fi
if [[ "aarch64" != $(uname -m) ]] ; then
pip3 install torch==1.13.0+cpu -f https://download.pytorch.org/whl/torch_stable.html
else
pip3 install torch==1.13.0 -f https://download.pytorch.org/whl/torch_stable.html
fi
fi

Expand Down Expand Up @@ -352,12 +350,10 @@ done
if [ "$TEST_VALGRIND" -eq 1 ]; then
TESTING_BACKENDS="python python_dlpack onnx"
EXPECTED_NUM_TESTS=42
if [ "$TEST_JETSON" == "0" ]; then
if [[ "aarch64" != $(uname -m) ]] ; then
pip3 install torch==1.13.0+cpu -f https://download.pytorch.org/whl/torch_stable.html
else
pip3 install torch==1.13.0 -f https://download.pytorch.org/whl/torch_stable.html
fi
if [[ "aarch64" != $(uname -m) ]] ; then
pip3 install torch==1.13.0+cpu -f https://download.pytorch.org/whl/torch_stable.html
else
pip3 install torch==1.13.0 -f https://download.pytorch.org/whl/torch_stable.html
fi

for BACKENDS in $TESTING_BACKENDS; do
Expand Down

0 comments on commit 2b607f2

Please sign in to comment.