From f05aaa6a10418ac2dbb5c6bf1f801dcc36e2e6c5 Mon Sep 17 00:00:00 2001 From: Misha Chornyi <99709299+mc-nv@users.noreply.github.com> Date: Mon, 16 Oct 2023 19:07:39 -0700 Subject: [PATCH] Switch Jetson model TensorRT models generation to container (#6378) * Switch Jetson model TensorRT models generation to container * Adding missed file * Fix typo * Fix typos * Remove extra spaces * Fix typo --- qa/common/gen_jetson_trt_models | 262 +++++++++++++++++--------------- 1 file changed, 143 insertions(+), 119 deletions(-) diff --git a/qa/common/gen_jetson_trt_models b/qa/common/gen_jetson_trt_models index da4d6b3dcb..71b3e80594 100755 --- a/qa/common/gen_jetson_trt_models +++ b/qa/common/gen_jetson_trt_models @@ -27,137 +27,161 @@ ############################################################################ ## This script generates the model repository needed for TensorRT testing -## testing on the Jetson Xavier. Generating these models requires having -## TensorRT on the system. -## -## 1. Update Jetpack version to match what is being used by the -## tritonserver release being tested. -## -## 2. Set CUDA_DEVICE to the ID of the CUDA device present on the -## system that you want to target for the generated models. -## -## 3. Run this script to create a '/tmp/_xavier' directory -## containing all the TensorRT models needed for CI testing. -## -## 4. Copy all non-TensorRT models from the dlcluster directory -## '/mnt/dldata/inferenceserver/' to -## '/mnt/dldata/inferenceserver/_xavier'. -## -## 5. Copy the generated models from '/tmp/_xavier' to the -## dlcluster directory '/mnt/dldata/inferenceserver/_xavier'. -## -## 6. Remember to delete the '/tmp/_xavier' directory after. -## +## on the Jetson device. Generating these models requires having TensorRT +## container. ############################################################################ +#!/bin/bash -xe +# Make all generated files accessible outside of container +umask 0000 TRITON_VERSION=${TRITON_VERSION:=23.10} CUDA_DEVICE=${RUNNER_ID:=0} +# Set TensorRT image +TENSORRT_IMAGE=${TENSORRT_IMAGE:=nvcr.io/nvidia/tensorrt:$TRITON_VERSION-py3-igpu} -HOST_BUILD_DIR=${HOST_BUILD_DIR:=/tmp} -TMP_DIR=$HOST_BUILD_DIR/${TRITON_VERSION} +# Set the path to the host working directory +HOST_BUILD_DIR=${HOST_BUILD_DIR:=/tmp/models_build} +# Set the path to the host model output directory +HOST_MODEL_DIR=${HOST_MODEL_DIR:="${HOST_BUILD_DIR}/${TRITON_VERSION}"} +# Set the source directory to store executable source file to generate models +HOST_SOURCE_DIR=$HOST_BUILD_DIR/gen_srcdir -SRCDIR=$HOST_BUILD_DIR/gen_srcdir -DESTDIR=$TMP_DIR/qa_model_repository +# Set CI specific parameters +DOCKER_GPU_ARGS="${DOCKER_GPU_ARGS:="--gpus device=$CUDA_DEVICE"}" +[[ $RUNNER_GPUS =~ ^[0-9] ]] && DOCKER_GPU_ARGS=$(eval $NV_DOCKER_ARGS) -DATADEPENDENTDIR=$TMP_DIR/qa_trt_data_dependent_model_repository -DYNASEQDESTDIR=$TMP_DIR/qa_dyna_sequence_model_repository -DYNASEQIMPLICITDESTDIR=$TMP_DIR/qa_dyna_sequence_implicit_model_repository -FORMATDESTDIR=$TMP_DIR/qa_trt_format_model_repository -IDENTITYBIGDESTDIR=$TMP_DIR/qa_identity_big_model_repository -IDENTITYDESTDIR=$TMP_DIR/qa_identity_model_repository -IMPLICITSEQDESTDIR=$TMP_DIR/qa_sequence_implicit_model_repository -RAGGEDDESTDIR=$TMP_DIR/qa_ragged_model_repository -RESHAPEDESTDIR=$TMP_DIR/qa_reshape_model_repository -SEQDESTDIR=$TMP_DIR/qa_sequence_model_repository -SHAPEDESTDIR=$TMP_DIR/qa_shapetensor_model_repository -VARDESTDIR=$TMP_DIR/qa_variable_model_repository -VARIMPLICITSEQDESTDIR=$TMP_DIR/qa_variable_sequence_implicit_model_repository -VARSEQDESTDIR=$TMP_DIR/qa_variable_sequence_model_repository +# Set model output directories -rm -fr $DESTDIR -rm -fr $DATADEPENDENTDIR -rm -fr $DYNASEQDESTDIR -rm -fr $DYNASEQIMPLICITDESTDIR -rm -fr $FORMATDESTDIR -rm -fr $IDENTITYBIGDESTDIR -rm -fr $IDENTITYDESTDIR -rm -fr $IMPLICITSEQDESTDIR -rm -fr $RAGGEDDESTDIR -rm -fr $RESHAPEDESTDIR -rm -fr $SEQDESTDIR -rm -fr $SHAPEDESTDIR -rm -fr $VARDESTDIR -rm -fr $VARIMPLICITSEQDESTDIR -rm -fr $VARSEQDESTDIR +HOST_DESTDIR=$HOST_MODEL_DIR/qa_model_repository +HOST_DATADEPENDENTDIR=$HOST_MODEL_DIR/qa_trt_data_dependent_model_repository +HOST_DYNASEQDESTDIR=$HOST_MODEL_DIR/qa_dyna_sequence_model_repository +HOST_DYNASEQIMPLICITDESTDIR=$HOST_MODEL_DIR/qa_dyna_sequence_implicit_model_repository +HOST_FORMATDESTDIR=$HOST_MODEL_DIR/qa_trt_format_model_repository +HOST_IDENTITYBIGDESTDIR=$HOST_MODEL_DIR/qa_identity_big_model_repository +HOST_IDENTITYDESTDIR=$HOST_MODEL_DIR/qa_identity_model_repository +HOST_IMPLICITSEQDESTDIR=$HOST_MODEL_DIR/qa_sequence_implicit_model_repository +HOST_RAGGEDDESTDIR=$HOST_MODEL_DIR/qa_ragged_model_repository +HOST_RESHAPEDESTDIR=$HOST_MODEL_DIR/qa_reshape_model_repository +HOST_SEQDESTDIR=$HOST_MODEL_DIR/qa_sequence_model_repository +HOST_SHAPEDESTDIR=$HOST_MODEL_DIR/qa_shapetensor_model_repository +HOST_VARDESTDIR=$HOST_MODEL_DIR/qa_variable_model_repository +HOST_VARIMPLICITSEQDESTDIR=$HOST_MODEL_DIR/qa_variable_sequence_implicit_model_repository +HOST_VARSEQDESTDIR=$HOST_MODEL_DIR/qa_variable_sequence_model_repository -mkdir -p $SRCDIR -mkdir -p $DESTDIR +# Clean up host working directory +rm -fr $HOST_BUILD_DIR -mkdir -p $DATADEPENDENTDIR -mkdir -p $DYNASEQDESTDIR -mkdir -p $DYNASEQIMPLICITDESTDIR -mkdir -p $FORMATDESTDIR -mkdir -p $IDENTITYBIGDESTDIR -mkdir -p $IDENTITYDESTDIR -mkdir -p $IMPLICITSEQDESTDIR -mkdir -p $RAGGEDDESTDIR -mkdir -p $RESHAPEDESTDIR -mkdir -p $SEQDESTDIR -mkdir -p $SHAPEDESTDIR -mkdir -p $VARDESTDIR -mkdir -p $VARIMPLICITSEQDESTDIR -mkdir -p $VARSEQDESTDIR +# Create the model output directories +mkdir -p $HOST_SOURCE_DIR +mkdir -p $HOST_DESTDIR +mkdir -p $HOST_DATADEPENDENTDIR +mkdir -p $HOST_DYNASEQDESTDIR +mkdir -p $HOST_DYNASEQIMPLICITDESTDIR +mkdir -p $HOST_FORMATDESTDIR +mkdir -p $HOST_IDENTITYBIGDESTDIR +mkdir -p $HOST_IDENTITYDESTDIR +mkdir -p $HOST_IMPLICITSEQDESTDIR +mkdir -p $HOST_RAGGEDDESTDIR +mkdir -p $HOST_RESHAPEDESTDIR +mkdir -p $HOST_SEQDESTDIR +mkdir -p $HOST_SHAPEDESTDIR +mkdir -p $HOST_VARDESTDIR +mkdir -p $HOST_VARIMPLICITSEQDESTDIR +mkdir -p $HOST_VARSEQDESTDIR -cp ./gen_ensemble_model_utils.py $SRCDIR/. -cp ./gen_qa_dyna_sequence_implicit_models.py $SRCDIR/. -cp ./gen_qa_dyna_sequence_models.py $SRCDIR/. -cp ./gen_qa_identity_models.py $SRCDIR/. -cp ./gen_qa_implicit_models.py $SRCDIR/. -cp ./gen_qa_models.py $SRCDIR/. -cp ./gen_qa_noshape_models.py $SRCDIR/. -cp ./gen_qa_ragged_models.py $SRCDIR/. -cp ./gen_qa_reshape_models.py $SRCDIR/. -cp ./gen_qa_sequence_models.py $SRCDIR/. -cp ./gen_qa_trt_data_dependent_shape.py $SRCDIR/. -cp ./gen_qa_trt_format_models.py $SRCDIR/. -cp ./gen_qa_trt_plugin_models.py $SRCDIR/. -cp ./test_util.py $SRCDIR/. +# Copy the executable source file to the host generate models source directory +cp ./gen_ensemble_model_utils.py $HOST_SOURCE_DIR/. +cp ./gen_common.py $HOST_SOURCE_DIR/. +cp ./gen_qa_dyna_sequence_implicit_models.py $HOST_SOURCE_DIR/. +cp ./gen_qa_dyna_sequence_models.py $HOST_SOURCE_DIR/. +cp ./gen_qa_identity_models.py $HOST_SOURCE_DIR/. +cp ./gen_qa_implicit_models.py $HOST_SOURCE_DIR/. +cp ./gen_qa_models.py $HOST_SOURCE_DIR/. +cp ./gen_qa_noshape_models.py $HOST_SOURCE_DIR/. +cp ./gen_qa_ragged_models.py $HOST_SOURCE_DIR/. +cp ./gen_qa_reshape_models.py $HOST_SOURCE_DIR/. +cp ./gen_qa_sequence_models.py $HOST_SOURCE_DIR/. +cp ./gen_qa_trt_data_dependent_shape.py $HOST_SOURCE_DIR/. +cp ./gen_qa_trt_format_models.py $HOST_SOURCE_DIR/. +cp ./gen_qa_trt_plugin_models.py $HOST_SOURCE_DIR/. +cp ./test_util.py $HOST_SOURCE_DIR/. -# TensorRT -set -e +# Set TensorRT model generation script name +TRT_MODEL_SCRIPT=trt_gen.cmds + +# Set container working directory +CONTAINER_SOURCE_DIR=/workspace/src +CONTAINER_MODEL_DIR=/tmp/models +CONTAINER_DESTDIR=$CONTAINER_MODEL_DIR/qa_model_repository +CONTAINER_DATADEPENDENTDIR=$CONTAINER_MODEL_DIR/qa_trt_data_dependent_model_repository +CONTAINER_DYNASEQDESTDIR=$CONTAINER_MODEL_DIR/qa_dyna_sequence_model_repository +CONTAINER_DYNASEQIMPLICITDESTDIR=$CONTAINER_MODEL_DIR/qa_dyna_sequence_implicit_model_repository +CONTAINER_FORMATDESTDIR=$CONTAINER_MODEL_DIR/qa_trt_format_model_repository +CONTAINER_IDENTITYBIGDESTDIR=$CONTAINER_MODEL_DIR/qa_identity_big_model_repository +CONTAINER_IDENTITYDESTDIR=$CONTAINER_MODEL_DIR/qa_identity_model_repository +CONTAINER_IMPLICITSEQDESTDIR=$CONTAINER_MODEL_DIR/qa_sequence_implicit_model_repository +CONTAINER_RAGGEDDESTDIR=$CONTAINER_MODEL_DIR/qa_ragged_model_repository +CONTAINER_RESHAPEDESTDIR=$CONTAINER_MODEL_DIR/qa_reshape_model_repository +CONTAINER_SEQDESTDIR=$CONTAINER_MODEL_DIR/qa_sequence_model_repository +CONTAINER_SHAPEDESTDIR=$CONTAINER_MODEL_DIR/qa_shapetensor_model_repository +CONTAINER_VARDESTDIR=$CONTAINER_MODEL_DIR/qa_variable_model_repository +CONTAINER_VARIMPLICITSEQDESTDIR=$CONTAINER_MODEL_DIR/qa_variable_sequence_implicit_model_repository +CONTAINER_VARSEQDESTDIR=$CONTAINER_MODEL_DIR/qa_variable_sequence_model_repository + +# Set script to generate TensorRT models +cat >$HOST_SOURCE_DIR/$TRT_MODEL_SCRIPT <