diff --git a/xinference/deploy/docker/Dockerfile b/xinference/deploy/docker/Dockerfile
index 6cf82f8ef3..5fa276d118 100644
--- a/xinference/deploy/docker/Dockerfile
+++ b/xinference/deploy/docker/Dockerfile
@@ -16,6 +16,7 @@ RUN apt-get -y update \
   && apt-get -yq clean
 
 ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH
+ENV LD_LIBRARY_PATH /usr/local/cuda/compat:$LD_LIBRARY_PATH
 
 ARG PIP_INDEX=https://pypi.org/simple
 RUN python -m pip install --upgrade -i "$PIP_INDEX" pip && \