diff --git a/docker/dockerfile.tf b/docker/dockerfile.tf index 988eb66bb..0dafdff29 100644 --- a/docker/dockerfile.tf +++ b/docker/dockerfile.tf @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1.2 -ARG MERLIN_VERSION=22.12 -ARG TRITON_VERSION=22.11 -ARG TENSORFLOW_VERSION=22.11 +ARG MERLIN_VERSION=23.06 +ARG TRITON_VERSION=23.06 +ARG TENSORFLOW_VERSION=23.06 ARG DLFW_IMAGE=nvcr.io/nvidia/tensorflow:${TENSORFLOW_VERSION}-tf2-py3 ARG FULL_IMAGE=nvcr.io/nvidia/tritonserver:${TRITON_VERSION}-py3 @@ -21,14 +21,14 @@ RUN pip install --no-cache-dir tensorflow protobuf==3.20.3 wrapt==1.14.0 \ && pip uninstall tensorflow keras -y # DLFW Tensorflow packages -COPY --chown=1000:1000 --from=dlfw /usr/local/lib/python3.8/dist-packages/tensorflow /usr/local/lib/python3.8/dist-packages/tensorflow/ -COPY --chown=1000:1000 --from=dlfw /usr/local/lib/python3.8/dist-packages/tensorflow-*.dist-info /usr/local/lib/python3.8/dist-packages/tensorflow.dist-info/ -COPY --chown=1000:1000 --from=dlfw /usr/local/lib/python3.8/dist-packages/keras /usr/local/lib/python3.8/dist-packages/keras/ -COPY --chown=1000:1000 --from=dlfw /usr/local/lib/python3.8/dist-packages/keras-*.dist-info /usr/local/lib/python3.8/dist-packages/keras.dist-info/ +COPY --chown=1000:1000 --from=dlfw /usr/local/lib/python${PYTHON_VERSION}/dist-packages/tensorflow /usr/local/lib/python${PYTHON_VERSION}/dist-packages/tensorflow/ +COPY --chown=1000:1000 --from=dlfw /usr/local/lib/python${PYTHON_VERSION}/dist-packages/tensorflow-*.dist-info /usr/local/lib/python${PYTHON_VERSION}/dist-packages/tensorflow.dist-info/ +COPY --chown=1000:1000 --from=dlfw /usr/local/lib/python${PYTHON_VERSION}/dist-packages/keras /usr/local/lib/python${PYTHON_VERSION}/dist-packages/keras/ +COPY --chown=1000:1000 --from=dlfw /usr/local/lib/python${PYTHON_VERSION}/dist-packages/keras-*.dist-info /usr/local/lib/python${PYTHON_VERSION}/dist-packages/keras.dist-info/ COPY --chown=1000:1000 --from=dlfw /usr/local/bin/saved_model_cli /usr/local/bin/saved_model_cli COPY --chown=1000:1000 --from=dlfw /usr/local/lib/tensorflow/ /usr/local/lib/tensorflow/ -COPY --chown=1000:1000 --from=dlfw /usr/local/lib/python3.8/dist-packages/horovod /usr/local/lib/python3.8/dist-packages/horovod/ -COPY --chown=1000:1000 --from=dlfw /usr/local/lib/python3.8/dist-packages/horovod-*.dist-info /usr/local/lib/python3.8/dist-packages/horovod.dist-info/ +COPY --chown=1000:1000 --from=dlfw /usr/local/lib/python${PYTHON_VERSION}/dist-packages/horovod /usr/local/lib/python${PYTHON_VERSION}/dist-packages/horovod/ +COPY --chown=1000:1000 --from=dlfw /usr/local/lib/python${PYTHON_VERSION}/dist-packages/horovod-*.dist-info /usr/local/lib/python${PYTHON_VERSION}/dist-packages/horovod.dist-info/ COPY --chown=1000:1000 --from=dlfw /usr/local/bin/horovodrun /usr/local/bin/horovodrun # Need to install transformers after tensorflow has been pulled in, so it builds artifacts correctly. @@ -42,7 +42,7 @@ ARG _CI_JOB_TOKEN="" ARG HUGECTR_VER=main ENV CPATH=$CPATH:${HUGECTR_HOME}/include \ - LD_LIBRARY_PATH=${HUGECTR_HOME}/lib:/usr/local/lib/python3.8/dist-packages/tensorflow:$LD_LIBRARY_PATH \ + LD_LIBRARY_PATH=${HUGECTR_HOME}/lib:/usr/local/lib/python${PYTHON_VERSION}/dist-packages/tensorflow:$LD_LIBRARY_PATH \ LIBRARY_PATH=${HUGECTR_HOME}/lib:$LIBRARY_PATH \ SOK_COMPILE_UNIT_TEST=ON @@ -69,7 +69,7 @@ RUN if [ "$HUGECTR_DEV_MODE" == "false" ]; then \ mv /hugectr/ci ~/hugectr-ci && mv /hugectr/sparse_operation_kit ~/hugectr-sparse_operation_kit && \ rm -rf /hugectr && mkdir -p /hugectr && \ mv ~/hugectr-ci /hugectr/ci && mv ~/hugectr-sparse_operation_kit /hugectr/sparse_operation_kit; \ - fi; \ + fi && \ if [ "$INSTALL_DISTRIBUTED_EMBEDDINGS" == "true" ]; then \ git clone --branch ${TFDE_VER} --depth 1 https://github.com/NVIDIA-Merlin/distributed-embeddings.git /distributed_embeddings/ && \ cd /distributed_embeddings && git submodule update --init --recursive && \ diff --git a/docker/dockerfile.torch b/docker/dockerfile.torch index e2f192972..bbc87040f 100644 --- a/docker/dockerfile.torch +++ b/docker/dockerfile.torch @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1.2 -ARG MERLIN_VERSION=22.12 -ARG TRITON_VERSION=22.11 -ARG TORCH_VERSION=22.11 +ARG MERLIN_VERSION=23.06 +ARG TRITON_VERSION=23.06 +ARG TORCH_VERSION=23.06 ARG DLFW_IMAGE=nvcr.io/nvidia/pytorch:${TORCH_VERSION}-py3 ARG FULL_IMAGE=nvcr.io/nvidia/tritonserver:${TRITON_VERSION}-py3 @@ -23,20 +23,20 @@ RUN apt update -y --fix-missing && \ RUN pip install --no-cache-dir --no-deps torch torchmetrics pytorch-lightning lightning-utilities \ && pip install --no-cache-dir --upgrade pip \ && pip install sympy \ - && rm -rf /usr/local/lib/python3.8/dist-packages/torch \ - && rm -rf /usr/local/lib/python3.8/dist-packages/caffe2 + && rm -rf /usr/local/lib/python${PYTHON_VERSION}/dist-packages/torch \ + && rm -rf /usr/local/lib/python${PYTHON_VERSION}/dist-packages/caffe2 # Triton Torch backend COPY --chown=1000:1000 --from=triton /opt/tritonserver/backends/pytorch backends/pytorch # DLFW Python packages -COPY --chown=1000:1000 --from=dlfw /usr/local/lib/python3.8/dist-packages/numba /usr/local/lib/python3.8/dist-packages/numba -COPY --chown=1000:1000 --from=dlfw /usr/local/lib/python3.8/dist-packages/numpy /usr/local/lib/python3.8/dist-packages/numpy -COPY --chown=1000:1000 --from=dlfw /usr/local/lib/python3.8/dist-packages/torch /usr/local/lib/python3.8/dist-packages/torch +COPY --chown=1000:1000 --from=dlfw /usr/local/lib/python${PYTHON_VERSION}/dist-packages/numba /usr/local/lib/python${PYTHON_VERSION}/dist-packages/numba +COPY --chown=1000:1000 --from=dlfw /usr/local/lib/python${PYTHON_VERSION}/dist-packages/numpy /usr/local/lib/python${PYTHON_VERSION}/dist-packages/numpy +COPY --chown=1000:1000 --from=dlfw /usr/local/lib/python${PYTHON_VERSION}/dist-packages/torch /usr/local/lib/python${PYTHON_VERSION}/dist-packages/torch -COPY --chown=1000:1000 --from=dlfw /usr/local/lib/python3.8/dist-packages/numba-*.dist-info /usr/local/lib/python3.8/dist-packages/numba.dist-info/ -COPY --chown=1000:1000 --from=dlfw /usr/local/lib/python3.8/dist-packages/numpy-*.dist-info /usr/local/lib/python3.8/dist-packages/numpy.dist-info/ -COPY --chown=1000:1000 --from=dlfw /usr/local/lib/python3.8/dist-packages/torch-*.egg-info /usr/local/lib/python3.8/dist-packages/torch.egg-info/ +COPY --chown=1000:1000 --from=dlfw /usr/local/lib/python${PYTHON_VERSION}/dist-packages/numba-*.dist-info /usr/local/lib/python${PYTHON_VERSION}/dist-packages/numba.dist-info/ +COPY --chown=1000:1000 --from=dlfw /usr/local/lib/python${PYTHON_VERSION}/dist-packages/numpy-*.dist-info /usr/local/lib/python${PYTHON_VERSION}/dist-packages/numpy.dist-info/ +COPY --chown=1000:1000 --from=dlfw /usr/local/lib/python${PYTHON_VERSION}/dist-packages/torch-*.egg-info /usr/local/lib/python${PYTHON_VERSION}/dist-packages/torch.egg-info/ # Add all torch libraries to /usr/local