New docker image for rocm / torch cpu (#434)

* add new docker

* add rocm pipeline

libs/infinity_emb/Docker.template.yaml

@@ -4,20 +4,32 @@
 nvidia:
   # 2 .command: jinja2 Dockerfile.jinja2 Docker.template.yaml --format=yaml -s nvidia > Dockerfile.nvidia_auto
   base_image: 'nvidia/cuda:12.1.1-base-ubuntu22.04'
+  main_install: "RUN poetry install --no-interaction --no-ansi --no-root --extras \"${EXTRAS}\" --without lint,test && poetry cache clear pypi --all"
 cpu:
   # 2. command: jinja2 Dockerfile.jinja2 Docker.template.yaml --format=yaml -s cpu > Dockerfile.cpu_auto
   base_image: 'ubuntu:22.04' 
-  pyproject_sed: RUN sed -i 's|"pypi"|"pytorch_cpu"|' pyproject.toml && poetry lock
+  # pyproject_sed: |
+  #   RUN sed -i 's|torch = "2.4.1"|torch = "2.5.0"|' pyproject.toml 
+  #   RUN sed -i 's|"pypi"|"pytorch_cpu"|' pyproject.toml
+  #   RUN poetry lock --no-update
+  main_install: |
+    # "RUN poetry install --no-interaction --no-ansi --no-root --extras \"${EXTRAS}\" --without lint,test && poetry cache clear pypi --all"
+    COPY requirements_install_from_poetry.sh requirements_install_from_poetry.sh
+    RUN ./requirements_install_from_poetry.sh --no-root --without lint,test "https://download.pytorch.org/whl/cpu"
 
 amd:
   # 2 . command: jinja2 Dockerfile.jinja2 Docker.template.yaml --format=yaml -s amd > Dockerfile.amd_auto
   base_image: 'rocm/pytorch:rocm6.2.3_ubuntu22.04_py3.10_pytorch_release_2.3.0'
-  pyproject_sed: |
-    RUN sed -i 's|"pypi"|"pytorch_rocm"|' pyproject.toml 
-    RUN sed -i 's|torch = "2.4.1"|torch = "2.4.1"|' pyproject.toml 
-    RUN sed -i 's|torchvision = {version = "\*"|torchvision = {version = "0.19.1"|' pyproject.toml && poetry lock
-  poetry_virtualenvs_create: "false"
-  poetry_virtualenvs_in_project: "false"
+  # pyproject_sed: |
+  #   RUN sed -i 's|"pypi"|"pytorch_rocm"|' pyproject.toml 
+  #   RUN sed -i 's|torch = "2.4.1"|torch = "2.4.1"|' pyproject.toml 
+  #   RUN sed -i 's|torchvision = {version = "\*"|torchvision = {version = "0.19.1"|' pyproject.toml 
+  #   RUN poetry lock --no-update
+  main_install: |
+    # "RUN poetry install --no-interaction --no-ansi --no-root --extras \"${EXTRAS}\" --without lint,test && poetry cache clear pypi --all"
+    COPY requirements_install_from_poetry.sh requirements_install_from_poetry.sh
+    RUN ./requirements_install_from_poetry.sh --no-root --without lint,test "https://download.pytorch.org/whl/rocm6.2"
+
   poetry_extras: "all onnxruntime-gpu"
   python_version: python3.10
 
@@ -28,8 +40,8 @@ trt:
     # Install utils for tensorrt
     RUN apt-get install -y --no-install-recommends openmpi-bin libopenmpi-dev git git-lfs python3-pip
     RUN poetry run $PYTHON -m pip install --no-cache-dir flash-attn --no-build-isolation
-    RUN poetry run $PYTHON -m pip install --no-cache-dir "tensorrt==10.2.0" "tensorrt_lean==10.2.0" "tensorrt_dispatch==10.2.0"
-    ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/$PYTHON/dist-packages/tensorrt/
-    # ENV LD_LIBRARY_PATH /app/.venv/lib/$(PYTHON)/site-packages/tensorrt:/usr/lib/x86_64-linux-gnu:/app/.venv/lib/$(PYTHON)/site-packages/tensorrt_libs:${LD_LIBRARY_PATH}
-    # ENV PATH /app/.venv/lib/$(PYTHON)/site-packages/tensorrt/bin:${PATH}
-  python_version: python3.10
+    RUN poetry run $PYTHON -m pip install --no-cache-dir "tensorrt==10.0.1" "tensorrt_lean==10.0.1" "tensorrt_dispatch==10.0.1"
+    ENV LD_LIBRARY_PATH /app/.venv/lib/${PYTHON}/site-packages/tensorrt:/usr/lib/x86_64-linux-gnu:/app/.venv/lib/${PYTHON}/site-packages/tensorrt_libs:${LD_LIBRARY_PATH}
+    ENV PATH /app/.venv/lib/${PYTHON}/site-packages/tensorrt/bin:${PATH}
+  python_version: python3.10
+  main_install: "RUN poetry install --no-interaction --no-ansi --no-root --extras \"${EXTRAS}\" --without lint,test && poetry cache clear pypi --all"

libs/infinity_emb/Dockerfile.amd_auto

@@ -13,21 +13,20 @@ ENV PYTHONUNBUFFERED=1 \
     \
     # make poetry create the virtual environment in the project's root
     # it gets named `.venv`
-    POETRY_VIRTUALENVS_CREATE="false" \
-    POETRY_VIRTUALENVS_IN_PROJECT="false" \
+    POETRY_VIRTUALENVS_CREATE="true" \
+    POETRY_VIRTUALENVS_IN_PROJECT="true" \
     # do not ask any interactive question
     POETRY_NO_INTERACTION=1 \
     EXTRAS="all onnxruntime-gpu" \
     PYTHON="python3.10"
-RUN apt-get update && apt-get install -y build-essential python3-dev libsndfile1 $PYTHON-venv $PYTHON curl
+RUN apt-get update && apt-get install --no-install-recommends -y build-essential python3-dev libsndfile1 $PYTHON-venv $PYTHON curl
 WORKDIR /app
 
 FROM base as builder
-
 # Set the working directory for the app
 # Define the version of Poetry to install (default is 1.7.1)
 # Define the directory to install Poetry to (default is /opt/poetry)
-ARG POETRY_VERSION=1.7.1
+ARG POETRY_VERSION=1.8.4
 ARG POETRY_HOME=/opt/poetry
 # Create a Python virtual environment for Poetry and install it
 RUN curl -sSL https://install.python-poetry.org | POETRY_HOME=$POETRY_HOME POETRY_VERSION=$POETRY_VERSION $PYTHON -
@@ -37,20 +36,26 @@ RUN echo "Poetry version:" && poetry --version
 # Copy the rest of the app source code (this layer will be invalidated and rebuilt whenever the source code changes)
 COPY poetry.lock poetry.toml pyproject.toml README.md /app/
 # Install dependencies only
-RUN sed -i 's|"pypi"|"pytorch_rocm"|' pyproject.toml 
-RUN sed -i 's|torch = "2.4.1"|torch = "2.4.1"|' pyproject.toml 
-RUN sed -i 's|torchvision = {version = "\*"|torchvision = {version = "0.19.1"|' pyproject.toml && poetry lock
+#
+# "RUN poetry install --no-interaction --no-ansi --no-root --extras \"${EXTRAS}\" --without lint,test && poetry cache clear pypi --all"
+COPY requirements_install_from_poetry.sh requirements_install_from_poetry.sh
+RUN ./requirements_install_from_poetry.sh --no-root --without lint,test "https://download.pytorch.org/whl/rocm6.2"
 
-RUN poetry install --no-interaction --no-ansi --no-root --extras "${EXTRAS}" --without lint,test && poetry cache clear pypi --all
 COPY infinity_emb infinity_emb
 # Install dependency with infinity_emb package
-RUN poetry install --no-interaction --no-ansi  --extras "${EXTRAS}" --without lint,test && poetry cache clear pypi --all
+# "RUN poetry install --no-interaction --no-ansi  --extras \"${EXTRAS}\" --without lint,test && poetry cache clear pypi --all"
+COPY requirements_install_from_poetry.sh requirements_install_from_poetry.sh
+RUN ./requirements_install_from_poetry.sh  --without lint,test "https://download.pytorch.org/whl/rocm6.2"
+
 #
 
 
 FROM builder as testing
 # install lint and test dependencies
-RUN poetry install --no-interaction --no-ansi  --extras "${EXTRAS}" --with lint,test && poetry cache clear pypi --all
+# "RUN poetry install --no-interaction --no-ansi  --extras \"${EXTRAS}\" --with lint,test && poetry cache clear pypi --all"
+COPY requirements_install_from_poetry.sh requirements_install_from_poetry.sh
+RUN ./requirements_install_from_poetry.sh  --with lint,test "https://download.pytorch.org/whl/rocm6.2"
+
 # lint 
 RUN poetry run ruff check .
 RUN poetry run mypy .
@@ -116,7 +121,6 @@ ARG MODEL_NAME
 RUN if [ -z "${MODEL_NAME}" ]; then echo "Error: Build argument MODEL_NAME not set." && exit 1; fi
 ARG ENGINE
 RUN if [ -z "${ENGINE}" ]; then echo "Error: Build argument ENGINE not set." && exit 1; fi
-
 # will exit with 3 if model is downloaded # TODO: better exit code
 RUN infinity_emb v2 --model-id $MODEL_NAME --engine $ENGINE --preload-only || [ $? -eq 3 ]
 ENTRYPOINT ["infinity_emb"]

libs/infinity_emb/Dockerfile.cpu_auto

@@ -19,15 +19,14 @@ ENV PYTHONUNBUFFERED=1 \
     POETRY_NO_INTERACTION=1 \
     EXTRAS="all" \
     PYTHON="python3.11"
-RUN apt-get update && apt-get install -y build-essential python3-dev libsndfile1 $PYTHON-venv $PYTHON curl
+RUN apt-get update && apt-get install --no-install-recommends -y build-essential python3-dev libsndfile1 $PYTHON-venv $PYTHON curl
 WORKDIR /app
 
 FROM base as builder
-
 # Set the working directory for the app
 # Define the version of Poetry to install (default is 1.7.1)
 # Define the directory to install Poetry to (default is /opt/poetry)
-ARG POETRY_VERSION=1.7.1
+ARG POETRY_VERSION=1.8.4
 ARG POETRY_HOME=/opt/poetry
 # Create a Python virtual environment for Poetry and install it
 RUN curl -sSL https://install.python-poetry.org | POETRY_HOME=$POETRY_HOME POETRY_VERSION=$POETRY_VERSION $PYTHON -
@@ -37,17 +36,26 @@ RUN echo "Poetry version:" && poetry --version
 # Copy the rest of the app source code (this layer will be invalidated and rebuilt whenever the source code changes)
 COPY poetry.lock poetry.toml pyproject.toml README.md /app/
 # Install dependencies only
-RUN sed -i 's|"pypi"|"pytorch_cpu"|' pyproject.toml && poetry lock
-RUN poetry install --no-interaction --no-ansi --no-root --extras "${EXTRAS}" --without lint,test && poetry cache clear pypi --all
+#
+# "RUN poetry install --no-interaction --no-ansi --no-root --extras \"${EXTRAS}\" --without lint,test && poetry cache clear pypi --all"
+COPY requirements_install_from_poetry.sh requirements_install_from_poetry.sh
+RUN ./requirements_install_from_poetry.sh --no-root --without lint,test "https://download.pytorch.org/whl/cpu"
+
 COPY infinity_emb infinity_emb
 # Install dependency with infinity_emb package
-RUN poetry install --no-interaction --no-ansi  --extras "${EXTRAS}" --without lint,test && poetry cache clear pypi --all
+# "RUN poetry install --no-interaction --no-ansi  --extras \"${EXTRAS}\" --without lint,test && poetry cache clear pypi --all"
+COPY requirements_install_from_poetry.sh requirements_install_from_poetry.sh
+RUN ./requirements_install_from_poetry.sh  --without lint,test "https://download.pytorch.org/whl/cpu"
+
 #
 
 
 FROM builder as testing
 # install lint and test dependencies
-RUN poetry install --no-interaction --no-ansi  --extras "${EXTRAS}" --with lint,test && poetry cache clear pypi --all
+# "RUN poetry install --no-interaction --no-ansi  --extras \"${EXTRAS}\" --with lint,test && poetry cache clear pypi --all"
+COPY requirements_install_from_poetry.sh requirements_install_from_poetry.sh
+RUN ./requirements_install_from_poetry.sh  --with lint,test "https://download.pytorch.org/whl/cpu"
+
 # lint 
 RUN poetry run ruff check .
 RUN poetry run mypy .
@@ -113,7 +121,6 @@ ARG MODEL_NAME
 RUN if [ -z "${MODEL_NAME}" ]; then echo "Error: Build argument MODEL_NAME not set." && exit 1; fi
 ARG ENGINE
 RUN if [ -z "${ENGINE}" ]; then echo "Error: Build argument ENGINE not set." && exit 1; fi
-
 # will exit with 3 if model is downloaded # TODO: better exit code
 RUN infinity_emb v2 --model-id $MODEL_NAME --engine $ENGINE --preload-only || [ $? -eq 3 ]
 ENTRYPOINT ["infinity_emb"]

libs/infinity_emb/Dockerfile.jinja2

@@ -19,15 +19,14 @@ ENV PYTHONUNBUFFERED=1 \
     POETRY_NO_INTERACTION=1 \
     EXTRAS="{{poetry_extras | default('all')}}" \
     PYTHON="{{python_version | default('python3.11')}}"
-RUN apt-get update && apt-get install -y build-essential python3-dev libsndfile1 $PYTHON-venv $PYTHON curl
+RUN apt-get update && apt-get install --no-install-recommends -y build-essential python3-dev libsndfile1 $PYTHON-venv $PYTHON curl
 WORKDIR /app
 
 FROM base as builder
-{% set main_install2 = "poetry install --no-interaction --no-ansi --no-root --extras \"${EXTRAS}\" --without lint,test" %}
 # Set the working directory for the app
 # Define the version of Poetry to install (default is 1.7.1)
 # Define the directory to install Poetry to (default is /opt/poetry)
-ARG POETRY_VERSION=1.7.1
+ARG POETRY_VERSION=1.8.4
 ARG POETRY_HOME=/opt/poetry
 # Create a Python virtual environment for Poetry and install it
 RUN curl -sSL https://install.python-poetry.org | POETRY_HOME=$POETRY_HOME POETRY_VERSION=$POETRY_VERSION $PYTHON -
@@ -38,16 +37,16 @@ RUN echo "Poetry version:" && poetry --version
 COPY poetry.lock poetry.toml pyproject.toml README.md /app/
 # Install dependencies only
 {{pyproject_sed | default('#')}}
-RUN {{main_install2}} && poetry cache clear pypi --all
+{{main_install}}
 COPY infinity_emb infinity_emb
 # Install dependency with infinity_emb package
-RUN {{main_install2|replace("--no-root","")}} && poetry cache clear pypi --all
+{{main_install|replace("--no-root","")}}
 {{extra_installs_main | default('#')}}
 
 
 FROM builder as testing
 # install lint and test dependencies
-RUN {{main_install2|replace("--without", "--with")|replace("--no-root","")}} && poetry cache clear pypi --all
+{{main_install|replace("--without", "--with")|replace("--no-root","")}}
 # lint 
 RUN poetry run ruff check .
 RUN poetry run mypy .
@@ -113,7 +112,6 @@ ARG MODEL_NAME
 RUN if [ -z "${MODEL_NAME}" ]; then echo "Error: Build argument MODEL_NAME not set." && exit 1; fi
 ARG ENGINE
 RUN if [ -z "${ENGINE}" ]; then echo "Error: Build argument ENGINE not set." && exit 1; fi
-
 # will exit with 3 if model is downloaded # TODO: better exit code
 RUN infinity_emb v2 --model-id $MODEL_NAME --engine $ENGINE --preload-only || [ $? -eq 3 ]
 ENTRYPOINT ["infinity_emb"]

libs/infinity_emb/Dockerfile.nvidia_auto

@@ -19,15 +19,14 @@ ENV PYTHONUNBUFFERED=1 \
     POETRY_NO_INTERACTION=1 \
     EXTRAS="all" \
     PYTHON="python3.11"
-RUN apt-get update && apt-get install -y build-essential python3-dev libsndfile1 $PYTHON-venv $PYTHON curl
+RUN apt-get update && apt-get install --no-install-recommends -y build-essential python3-dev libsndfile1 $PYTHON-venv $PYTHON curl
 WORKDIR /app
 
 FROM base as builder
-
 # Set the working directory for the app
 # Define the version of Poetry to install (default is 1.7.1)
 # Define the directory to install Poetry to (default is /opt/poetry)
-ARG POETRY_VERSION=1.7.1
+ARG POETRY_VERSION=1.8.4
 ARG POETRY_HOME=/opt/poetry
 # Create a Python virtual environment for Poetry and install it
 RUN curl -sSL https://install.python-poetry.org | POETRY_HOME=$POETRY_HOME POETRY_VERSION=$POETRY_VERSION $PYTHON -
@@ -113,7 +112,6 @@ ARG MODEL_NAME
 RUN if [ -z "${MODEL_NAME}" ]; then echo "Error: Build argument MODEL_NAME not set." && exit 1; fi
 ARG ENGINE
 RUN if [ -z "${ENGINE}" ]; then echo "Error: Build argument ENGINE not set." && exit 1; fi
-
 # will exit with 3 if model is downloaded # TODO: better exit code
 RUN infinity_emb v2 --model-id $MODEL_NAME --engine $ENGINE --preload-only || [ $? -eq 3 ]
 ENTRYPOINT ["infinity_emb"]

libs/infinity_emb/Dockerfile.trt_onnx_auto

@@ -19,15 +19,14 @@ ENV PYTHONUNBUFFERED=1 \
     POETRY_NO_INTERACTION=1 \
     EXTRAS="all onnxruntime-gpu" \
     PYTHON="python3.10"
-RUN apt-get update && apt-get install -y build-essential python3-dev libsndfile1 $PYTHON-venv $PYTHON curl
+RUN apt-get update && apt-get install --no-install-recommends -y build-essential python3-dev libsndfile1 $PYTHON-venv $PYTHON curl
 WORKDIR /app
 
 FROM base as builder
-
 # Set the working directory for the app
 # Define the version of Poetry to install (default is 1.7.1)
 # Define the directory to install Poetry to (default is /opt/poetry)
-ARG POETRY_VERSION=1.7.1
+ARG POETRY_VERSION=1.8.4
 ARG POETRY_HOME=/opt/poetry
 # Create a Python virtual environment for Poetry and install it
 RUN curl -sSL https://install.python-poetry.org | POETRY_HOME=$POETRY_HOME POETRY_VERSION=$POETRY_VERSION $PYTHON -
@@ -45,10 +44,9 @@ RUN poetry install --no-interaction --no-ansi  --extras "${EXTRAS}" --without li
 # Install utils for tensorrt
 RUN apt-get install -y --no-install-recommends openmpi-bin libopenmpi-dev git git-lfs python3-pip
 RUN poetry run $PYTHON -m pip install --no-cache-dir flash-attn --no-build-isolation
-RUN poetry run $PYTHON -m pip install --no-cache-dir "tensorrt==10.2.0" "tensorrt_lean==10.2.0" "tensorrt_dispatch==10.2.0"
-ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/$PYTHON/dist-packages/tensorrt/
-# ENV LD_LIBRARY_PATH /app/.venv/lib/$(PYTHON)/site-packages/tensorrt:/usr/lib/x86_64-linux-gnu:/app/.venv/lib/$(PYTHON)/site-packages/tensorrt_libs:${LD_LIBRARY_PATH}
-# ENV PATH /app/.venv/lib/$(PYTHON)/site-packages/tensorrt/bin:${PATH}
+RUN poetry run $PYTHON -m pip install --no-cache-dir "tensorrt==10.0.1" "tensorrt_lean==10.0.1" "tensorrt_dispatch==10.0.1"
+ENV LD_LIBRARY_PATH /app/.venv/lib/${PYTHON}/site-packages/tensorrt:/usr/lib/x86_64-linux-gnu:/app/.venv/lib/${PYTHON}/site-packages/tensorrt_libs:${LD_LIBRARY_PATH}
+ENV PATH /app/.venv/lib/${PYTHON}/site-packages/tensorrt/bin:${PATH}
 
 
 
@@ -120,7 +118,6 @@ ARG MODEL_NAME
 RUN if [ -z "${MODEL_NAME}" ]; then echo "Error: Build argument MODEL_NAME not set." && exit 1; fi
 ARG ENGINE
 RUN if [ -z "${ENGINE}" ]; then echo "Error: Build argument ENGINE not set." && exit 1; fi
-
 # will exit with 3 if model is downloaded # TODO: better exit code
 RUN infinity_emb v2 --model-id $MODEL_NAME --engine $ENGINE --preload-only || [ $? -eq 3 ]
 ENTRYPOINT ["infinity_emb"]

libs/infinity_emb/poetry.toml

@@ -3,5 +3,5 @@ create = true
 in-project = true
 
 [installer]
-modern-installation = false
+modern-installation = true
 max-workers = 10

libs/infinity_emb/pyproject.toml

@@ -23,7 +23,7 @@ fastapi = {version = ">=0.103.2", optional=true}
 orjson = {version = ">=3.9.8,!=3.10.0", optional=true} 
 prometheus-fastapi-instrumentator = {version = ">=6.1.0", optional=true}
 uvicorn = {version = "^0.32.0", optional=true, extras = ["standard"]}
-typer = {version = "^0.12.5", optional=true, extras = ["all"]}
+typer = {version = "^0.12.5", optional=true}
 pydantic = {version = ">=2.4.0,<3", optional=true} 
 posthog = {version = "*", optional=true}
 # backend
@@ -44,8 +44,10 @@ torchvision = {version = "*", source = "pypi", optional=true}
 # cache
 diskcache = {version = "*", optional=true}
 # gpu
-onnxruntime-gpu = {version = "*", optional=true} 
-tensorrt = {version = "^8.6.1", optional=true}
+# https://onnxruntime.ai/docs/execution-providers/TensorRT-ExecutionProvider.html:
+# Most combinations of onnxruntime and tensorrt are not compatible, and some are broken.
+onnxruntime-gpu = {version = "1.19.*", optional=true} 
+tensorrt = {version = "^10.2.0", optional=true}
 soundfile = {version="^0.12.1", optional=true}