Add ability to cross-build Python wheels for Jetson (#2313)

- adds a way to build Python bindings when cross-compiling DALI for Xavier platform - adds a way to generate Python wheel for Xavier Signed-off-by: Janusz Lisiecki <jlisiecki@nvidia.com>
NVIDIA · Oct 16, 2020 · 97152cd · 97152cd
1 parent 14001d6
commit 97152cd
Show file tree

Hide file tree

Showing 17 changed files with 254 additions and 130 deletions.
diff --git a/conda/recipe/build.sh b/conda/recipe/build.sh
@@ -64,6 +64,8 @@ cmake -DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda \
       -DBUILD_PYTHON=${BUILD_PYTHON:-ON}                  \
       -DBUILD_LMDB=${BUILD_LMDB:-ON}                      \
       -DBUILD_JPEG_TURBO=${BUILD_JPEG_TURBO:-ON}          \
+      -DBUILD_OPENCV=${BUILD_OPENCV:-ON}                  \
+      -DBUILD_PROTOBUF=${BUILD_PROTOBUF:-ON}              \
       -DBUILD_NVJPEG=${BUILD_NVJPEG:-ON}                  \
       -DBUILD_NVJPEG2K=${BUILD_NVJPEG2K}                  \
       -DBUILD_LIBTIFF=${BUILD_LIBTIFF:-ON}                \

diff --git a/dali/python/bundle-wheel.sh b/dali/python/bundle-wheel.sh
@@ -44,6 +44,7 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 INWHL=$(readlink -e $1)
+DEPS_PATH=${2:-/usr/local}
 OUTDIR=/wheelhouse
 
 OUTWHLNAME=$(basename $INWHL)
@@ -88,37 +89,21 @@ make_wheel_record() {
 }
 
 DEPS_LIST=(
-    "/usr/local/lib64/libjpeg.so.62"
-    "/usr/local/lib/libavformat.so.58"
-    "/usr/local/lib/libavcodec.so.58"
-    "/usr/local/lib/libavfilter.so.7"
-    "/usr/local/lib/libavutil.so.56"
-    "/usr/local/lib/libtiff.so.5"
-    "/usr/local/lib/libsndfile.so.1"
-    "/usr/local/lib/libFLAC.so.8"
-    "/usr/local/lib/libogg.so.0"
-    "/usr/local/lib/libvorbis.so.0"
-    "/usr/local/lib/libvorbisenc.so.2"
-    "/usr/local/lib/libopenjp2.so.7"
-    "/usr/local/lib/libzstd.so.1"
-    "/usr/local/lib/libz.so.1"
-)
-
-DEPS_SONAME=(
-    "libjpeg.so.62"
-    "libavformat.so.58"
-    "libavcodec.so.58"
-    "libavfilter.so.7"
-    "libavutil.so.56"
-    "libtiff.so.5"
-    "libsndfile.so.1"
-    "libFLAC.so.8"
-    "libogg.so.0"
-    "libvorbis.so.0"
-    "libvorbisenc.so.2"
-    "libopenjp2.so.7"
-    "libzstd.so.1"
-    "libz.so.1"
+    "${DEPS_PATH}/lib64/libjpeg.so.62"
+    "${DEPS_PATH}/lib/libjpeg.so.62"
+    "${DEPS_PATH}/lib/libavformat.so.58"
+    "${DEPS_PATH}/lib/libavcodec.so.58"
+    "${DEPS_PATH}/lib/libavfilter.so.7"
+    "${DEPS_PATH}/lib/libavutil.so.56"
+    "${DEPS_PATH}/lib/libtiff.so.5"
+    "${DEPS_PATH}/lib/libsndfile.so.1"
+    "${DEPS_PATH}/lib/libFLAC.so.8"
+    "${DEPS_PATH}/lib/libogg.so.0"
+    "${DEPS_PATH}/lib/libvorbis.so.0"
+    "${DEPS_PATH}/lib/libvorbisenc.so.2"
+    "${DEPS_PATH}/lib/libopenjp2.so.7"
+    "${DEPS_PATH}/lib/libzstd.so.1"
+    "${DEPS_PATH}/lib/libz.so.1"
 )
 
 TMPDIR=$(mktemp -d)
@@ -128,17 +113,20 @@ mkdir -p $PKGNAME_PATH/.libs
 popd
 
 # copy over needed dependent .so files over and tag them with their hash
+original=()
 patched=()
 for filepath in "${DEPS_LIST[@]}"; do
     filename=$(basename $filepath)
-    patchedname=$(fname_with_sha256 $filepath)
-    patchedpath=$PKGNAME_PATH/.libs/$patchedname
-    patched+=("$patchedname")
 
     if [[ ! -f "$filepath" ]]; then
         echo "Didn't find $filename, skipping..."
         continue
     fi
+    patchedname=$(fname_with_sha256 $filepath)
+    patchedpath=$PKGNAME_PATH/.libs/$patchedname
+    original+=("$filename")
+    patched+=("$patchedname")
+
     echo "Copying $filepath to $patchedpath"
     cp $filepath $TMPDIR/$patchedpath
 
@@ -162,8 +150,8 @@ pushd $TMPDIR
 
 echo "patching to fix the so names to the hashed names"
 find $PKGNAME_PATH -name '*.so*' -o -name '*.bin' | while read sofile; do
-    for ((i=0;i<${#DEPS_LIST[@]};++i)); do
-        origname=${DEPS_SONAME[i]}
+    for ((i=0;i<${#original[@]};++i)); do
+        origname=${original[i]}
         patchedname=${patched[i]}
         if [[ "$origname" != "$patchedname" ]]; then
             set +e

diff --git a/dali_tf_plugin/make_dali_tf_sdist.sh b/dali_tf_plugin/make_dali_tf_sdist.sh
@@ -19,5 +19,6 @@ cmake .. \
       -DGIT_SHA=${GIT_SHA}
 make -j install
 python setup.py sdist
+mkdir -p /dali_tf_sdist
 cp dist/*.tar.gz /dali_tf_sdist
 popd
diff --git a/docker/Dockerfile.build.aarch64-linux b/docker/Dockerfile.build.aarch64-linux
@@ -2,6 +2,8 @@ ARG AARCH64_CUDA_TOOL_IMAGE_NAME
 FROM ${AARCH64_CUDA_TOOL_IMAGE_NAME} as aarch64_cuda_tools
 FROM nvidia/cuda:10.2-devel-ubuntu18.04
 
+ENV DEBIAN_FRONTEND=noninteractive
+
 RUN apt-get update && apt-get install -y --no-install-recommends \
     ca-certificates \
     curl \
@@ -16,7 +18,23 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     pkg-config \
     libtool \
     libtool-bin \
-    && rm -rf /var/lib/apt/lists/*
+    python3-distutils \
+    zip \
+    python3.6 python3.6-dev \
+    python3.7 python3.7-dev \
+    python3.8 python3.8-dev \
+    && \
+    if [ $(apt-cache search python3.9 | wc -l) -eq 0 ]; then \
+        apt-get install software-properties-common -y --no-install-recommends && \
+        add-apt-repository ppa:deadsnakes/ppa -y && \
+        apt-get update; \
+    fi && \
+    apt-get install -y python3.9 python3.9-dev && \
+    rm -rf /var/lib/apt/lists/* && \
+    curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3 get-pip.py && rm get-pip.py && \
+    cd /tmp && git clone https://github.com/NixOS/patchelf && cd patchelf && \
+    ./bootstrap.sh && ./configure --prefix=/usr/ && make -j install && cd / && rm -rf /tmp/patchelf && \
+    ln -s /usr/bin/python3 /usr/bin/python
 
 COPY --from=aarch64_cuda_tools *.deb ./
 
@@ -60,6 +78,17 @@ RUN curl -L https://github.com/google/protobuf/releases/download/v${PROTOBUF_VER
     ./configure CXXFLAGS="-fPIC" --prefix=/usr/local --disable-shared 2>&1 > /dev/null && \
     make -j"$(grep ^processor /proc/cpuinfo | wc -l)" install 2>&1 > /dev/null
 
+# LMDB
+COPY docker/Makefile-lmdb.patch /tmp
+RUN LMDB_VERSION=0.9.24 && \
+    git clone -b LMDB_${LMDB_VERSION} --single-branch https://github.com/LMDB/lmdb && \
+    cd /lmdb/libraries/liblmdb && \
+    patch -p3 < /tmp/Makefile-lmdb.patch && \
+    rm -f /tmp/Makefile-lmdb.patch && \
+      CFLAGS="-fPIC" CXXFLAGS="-fPIC" CC=aarch64-linux-gnu-gcc CXX=aarch64-linux-gnu-g++ prefix=/usr/aarch64-linux-gnu \
+    make -j"$(grep ^processor /proc/cpuinfo | wc -l)" install && \
+    rm -rf /lmdb
+
 RUN cd /protobuf-${PROTOBUF_VERSION} && make clean \
     ./autogen.sh && ./configure \
     CXXFLAGS="-fPIC" \
@@ -82,7 +111,7 @@ RUN JPEG_TURBO_VERSION=2.0.5 && \
       CXXFLAGS="-fPIC" \
       CC=aarch64-linux-gnu-gcc \
       CXX=aarch64-linux-gnu-g++ \
-    cmake -G"Unix Makefiles" -DENABLE_SHARED=FALSE -DCMAKE_TOOLCHAIN_FILE=toolchain.cmake -DCMAKE_INSTALL_PREFIX=/usr/aarch64-linux-gnu/ . 2>&1 >/dev/null && \
+    cmake -G"Unix Makefiles" -DCMAKE_TOOLCHAIN_FILE=toolchain.cmake -DENABLE_SHARED=TRUE -DCMAKE_INSTALL_PREFIX=/usr/aarch64-linux-gnu/ . 2>&1 >/dev/null && \
       CFLAGS="-fPIC" \
       CXXFLAGS="-fPIC" \
       CC=aarch64-linux-gnu-gcc \
@@ -235,40 +264,58 @@ RUN LIBSND_VERSION=1.0.28 && cd /tmp
     make -j"$(grep ^processor /proc/cpuinfo | wc -l)" install                                                  && \
     cd /tmp && rm -rf libsndfile-$LIBSND_VERSION
 
+# hack - install cross headers in the default python paths, so host python3-config would point to them
+RUN export PYVERS="3.6.9 3.7.8 3.8.5 3.9.0" && \
+    for PYVER in ${PYVERS}; do \
+        cd /tmp && curl -L https://www.python.org/ftp/python/${PYVER}/Python-${PYVER}.tgz | tar -xzf - && \
+        rm -rf *.tgz && cd Python*                                                                     && \
+        ./configure --disable-ipv6 ac_cv_file__dev_ptmx=no ac_cv_file__dev_ptc=no \
+            --disable-shared CC=aarch64-linux-gnu-gcc CXX=aarch64-linux-gnu-g++ \
+            --build=x86_64-pc-linux-gnu --host=aarch64-linux-gnu --prefix=/usr/                        && \
+        make -j"$(grep ^processor /proc/cpuinfo | wc -l)" inclinstall                                  && \
+        cd / && rm -rf /tmp/Python*;                                                                      \
+    done                                                                                               && \
+    # hack - patch the host pythonX-config to return --extension-suffix for the target
+    find /usr/ -iname x86_64-linux-gnu-python* -exec sed -i "s/\(SO.*\)\(x86_64\)\(.*\)/\1aarch64\3/" {} \;
+
 VOLUME /dali
 
 WORKDIR /dali
 
-
 ENV PATH=/usr/local/cuda-10.2/bin:$PATH
 
 ARG DALI_BUILD_DIR=build_aarch64_linux
 
 WORKDIR /dali/${DALI_BUILD_DIR}
 
-CMD cmake \
-  -DWERROR=ON \
-  -DCMAKE_TOOLCHAIN_FILE:STRING="$PWD/../platforms/aarch64-linux/aarch64-linux.toolchain.cmake" \
-  -DCMAKE_COLOR_MAKEFILE=ON \
-  -DCMAKE_INSTALL_PREFIX=./install \
-  -DARCH=aarch64-linux \
-  -DCMAKE_CUDA_COMPILER=/usr/local/cuda-10.2/bin/nvcc \
-  -DCUDA_HOST=/usr/local/cuda-10.2 \
-  -DCUDA_TARGET=/usr/local/cuda-10.2/targets/aarch64-linux \
-  -DBUILD_TEST=ON \
-  -DBUILD_BENCHMARK=OFF \
-  -DBUILD_NVTX=OFF \
-  -DBUILD_PYTHON=OFF \
-  -DBUILD_LMDB=OFF \
-  -DBUILD_JPEG_TURBO=ON \
-  -DBUILD_LIBTIFF=ON \
-  -DBUILD_LIBSND=ON \
-  -DBUILD_FFTS=ON \
-  -DBUILD_NVJPEG=OFF \
-  -DBUILD_NVJPEG2K=OFF \
-  -DBUILD_NVOF=OFF \
-  -DBUILD_NVDEC=OFF \
-  -DBUILD_NVML=OFF \
-  -DVERBOSE_LOGS=OFF \
-  ..  && \
-  make -j"$(grep ^processor /proc/cpuinfo | wc -l)"
+CMD WERROR=ON           \
+    ARCH=aarch64-linux  \
+    BUILD_TEST=ON       \
+    BUILD_BENCHMARK=OFF \
+    BUILD_NVTX=OFF      \
+    BUILD_LMDB=ON       \
+    BUILD_JPEG_TURBO=ON \
+    BUILD_LIBTIFF=ON    \
+    BUILD_LIBSND=ON     \
+    BUILD_FFTS=ON       \
+    BUILD_NVJPEG=OFF    \
+    BUILD_NVJPEG2K=OFF  \
+    BUILD_NVOF=OFF      \
+    BUILD_FFMPEG=OFF    \
+    BUILD_NVDEC=OFF     \
+    BUILD_NVML=OFF      \
+    VERBOSE_LOGS=OFF    \
+    TEST_BUNDLED_LIBS=NO\
+    WHL_PLATFORM_NAME=manylinux2014_aarch64            \
+    BUNDLE_PATH_PREFIX="/usr/aarch64-linux-gnu"        \
+    EXTRA_CMAKE_OPTIONS="-DCMAKE_TOOLCHAIN_FILE:STRING=$PWD/../platforms/aarch64-linux/aarch64-linux.toolchain.cmake \
+                        -DCMAKE_COLOR_MAKEFILE=ON                                 \
+                        -DCMAKE_CUDA_COMPILER=/usr/local/cuda-10.2/bin/nvcc       \
+                        -DCUDA_HOST=/usr/local/cuda-10.2                          \
+                        -DCUDA_TARGET=/usr/local/cuda-10.2/targets/aarch64-linux" \
+    /dali/docker/build_helper.sh                    && \
+    rm -rf /dali/${DALI_BUILD_DIR}/nvidia*          && \
+    cd /dali/dali_tf_plugin		            && \
+    bash /dali/dali_tf_plugin/make_dali_tf_sdist.sh && \
+    mv /dali_tf_sdist/*.tar.gz /wheelhouse/         && \
+    cp -r /wheelhouse /dali/
diff --git a/docker/Dockerfile.build.aarch64-qnx b/docker/Dockerfile.build.aarch64-qnx
@@ -86,18 +86,29 @@ RUN cd /protobuf-${PROTOBUF_VERSION} && make clean \
       --with-protoc=/usr/local/bin/protoc && make -j$(nproc) install && \
     rm -rf /protobuf-${PROTOBUF_VERSION}
 
+# LMDB
+COPY docker/Makefile-lmdb.patch /tmp
+RUN LMDB_VERSION=0.9.24 && \
+    git clone -b LMDB_${LMDB_VERSION} --single-branch https://github.com/LMDB/lmdb && \
+    cd /lmdb/libraries/liblmdb && \
+    patch -p3 < /tmp/Makefile-lmdb.patch && \
+    rm -f /tmp/Makefile-lmdb.patch && \
+      CFLAGS="-fPIC" CXXFLAGS="-fPIC" CC=aarch64-unknown-nto-qnx7.0.0-gcc CXX=aarch64-unknown-nto-qnx7.0.0-g++ prefix=/usr/aarch64-unknown-nto-qnx/aarch64le \
+    make -j"$(grep ^processor /proc/cpuinfo | wc -l)" install && \
+    rm -rf /lmdb
+
 # libjpeg-turbo
 RUN JPEG_TURBO_VERSION=2.0.5 && \
     curl -L https://github.com/libjpeg-turbo/libjpeg-turbo/archive/${JPEG_TURBO_VERSION}.tar.gz | tar -xzf - && \
     cd libjpeg-turbo-${JPEG_TURBO_VERSION} && \
-    echo "set(CMAKE_SYSTEM_NAME  Linux)" > toolchain.cmake && \
+    echo "set(CMAKE_SYSTEM_NAME Linux)" > toolchain.cmake && \
     echo "set(CMAKE_SYSTEM_PROCESSOR aarch64)" >> toolchain.cmake && \
     echo "set(CMAKE_C_COMPILER aarch64-unknown-nto-qnx7.0.0-gcc)" >> toolchain.cmake && \
       CFLAGS="-fPIC" \
       CXXFLAGS="-fPIC" \
       CC=aarch64-unknown-nto-qnx7.0.0-gcc \
       CXX=aarch64-unknown-nto-qnx7.0.0-g++ \
-    cmake -G"Unix Makefiles" -DENABLE_SHARED=FALSE -DCMAKE_TOOLCHAIN_FILE=toolchain.cmake -DCMAKE_INSTALL_PREFIX=/usr/aarch64-unknown-nto-qnx/aarch64le . 2>&1 >/dev/null && \
+    cmake -G"Unix Makefiles" -DCMAKE_TOOLCHAIN_FILE=toolchain.cmake -DCMAKE_INSTALL_PREFIX=/usr/aarch64-unknown-nto-qnx/aarch64le . 2>&1 >/dev/null && \
       CFLAGS="-fPIC" \
       CXXFLAGS="-fPIC" \
       CC=aarch64-unknown-nto-qnx7.0.0-gcc \
@@ -268,20 +279,20 @@ ARG DALI_BUILD_DIR=build_aarch64_qnx
 
 WORKDIR /dali/${DALI_BUILD_DIR}
 
-CMD cmake \
-  -DWERROR=ON \
+CMD cmake                \
+  -DWERROR=ON            \
   -DCMAKE_TOOLCHAIN_FILE:STRING="$PWD/../platforms/qnx/aarch64-qnx.toolchain.cmake" \
-  -DCMAKE_COLOR_MAKEFILE=ON \
+  -DCMAKE_COLOR_MAKEFILE=ON        \
   -DCMAKE_INSTALL_PREFIX=./install \
-  -DARCH=aarch64-qnx \
-  -DCMAKE_CUDA_COMPILER=/usr/local/cuda-10.0/bin/nvcc \
-  -DCUDA_HOST=/usr/local/cuda-10.0 \
+  -DARCH=aarch64-qnx     \
+  -DCMAKE_CUDA_COMPILER=/usr/local/cuda-10.0/bin/nvcc    \
+  -DCUDA_HOST=/usr/local/cuda-10.0                       \
   -DCUDA_TARGET=/usr/local/cuda-10.0/targets/aarch64-qnx \
-  -DBUILD_TEST=ON \
-  -DBUILD_BENCHMARK=OFF \
-  -DBUILD_NVTX=OFF \
-  -DBUILD_PYTHON=OFF \
-  -DBUILD_LMDB=OFF \
+  -DBUILD_TEST=ON        \
+  -DBUILD_BENCHMARK=OFF  \
+  -DBUILD_NVTX=OFF       \
+  -DBUILD_PYTHON=OFF     \
+  -DBUILD_LMDB=ON        \
   -DBUILD_TENSORFLOW=OFF \
   -DBUILD_JPEG_TURBO=ON \
   -DBUILD_LIBTIFF=ON \

diff --git a/docker/build.sh b/docker/build.sh
@@ -141,6 +141,8 @@ if [ "$BUILD_INHOST" == "YES" ]; then
                                         BUILD_PYTHON=${BUILD_PYTHON}              \
                                         BUILD_LMDB=${BUILD_LMDB}                  \
                                         BUILD_JPEG_TURBO=${BUILD_JPEG_TURBO}      \
+                                        BUILD_OPENCV=${BUILD_OPENCV}              \
+                                        BUILD_PROTOBUF=${BUILD_PROTOBUF}          \
                                         BUILD_NVJPEG=${BUILD_NVJPEG}              \
                                         BUILD_NVJPEG2K=${BUILD_NVJPEG2K}          \
                                         BUILD_LIBTIFF=${BUILD_LIBTIFF}            \
@@ -173,6 +175,8 @@ else
                                    --build-arg "BUILD_PYTHON=${BUILD_PYTHON}"              \
                                    --build-arg "BUILD_LMDB=${BUILD_LMDB}"                  \
                                    --build-arg "BUILD_JPEG_TURBO=${BUILD_JPEG_TURBO}"      \
+                                   --build-arg "BUILD_OPENCV=${BUILD_OPENCV}"              \
+                                   --build-arg "BUILD_PROTOBUF=${BUILD_PROTOBUF}"          \
                                    --build-arg "BUILD_NVJPEG=${BUILD_NVJPEG}"              \
                                    --build-arg "BUILD_NVJPEG2K=${BUILD_NVJPEG2K}"          \
                                    --build-arg "BUILD_LIBTIFF=${BUILD_LIBTIFF}"            \