docker file and compose

.gitignore

@@ -1,12 +1,8 @@
 # OSX
 .DS_STORE
 
-# Models
-models/
-
 # Local Chroma db
 .chroma/
-db/
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

Dockerfile

@@ -0,0 +1,20 @@
+#FROM python:3.10.11
+#FROM wallies/python-cuda:3.10-cuda11.6-runtime
+
+# Using argument for base image to avoid multiplying Dockerfiles
+ARG BASEIMAGE
+FROM $BASEIMAGE
+
+RUN groupadd -g 10009 -o privategpt && useradd -m -u 10009 -g 10009 -o -s /bin/bash privategpt
+USER privategpt
+WORKDIR /home/privategpt
+
+COPY ./src/requirements.txt src/requirements.txt
+ARG LLAMA_CMAKE
+#RUN CMAKE_ARGS="-DLLAMA_OPENBLAS=on" FORCE_CMAKE=1 pip install $(grep llama-cpp-python src/requirements.txt)
+RUN ( /bin/bash -c "${LLAMA_CMAKE} pip install \$(grep llama-cpp-python src/requirements.txt)" 2>&1 | tee llama-build.log ) && sleep 10
+RUN pip install --no-cache-dir -r src/requirements.txt 2>&1 | tee pip-install.log
+
+COPY ./src src
+
+# ENTRYPOINT ["python", "src/privateGPT.py"]

README.md

@@ -104,5 +104,41 @@ To install a C++ compiler on Windows 10/11, follow these steps:
 3. Download the MinGW installer from the [MinGW website](https://sourceforge.net/projects/mingw/).
 4. Run the installer and select the `gcc` component.
 
+## Docker Version
+
+1. Put your data in models / source_documents in the project root folder (Can be customized changing the corresponding value in the docker-compose.yaml)
+
+2. You can ingest your data using the docker-compose
+
+   1. Without Cuda
+
+    ```sh
+    docker compose run --rm --build privategpt-ingest
+    ```
+
+   2. With Cuda 11.6 or 11.7
+
+    ```sh
+    docker compose run --rm --build privategpt-cuda-11.6-ingest
+
+    docker compose run --rm --build privategpt-cuda-11.7-ingest
+    ```
+
+3. You can run **after** ingesting your data or using an **existing db** with the docker-compose
+
+   1. Without Cuda
+
+    ```sh
+    docker compose run --rm --build privategpt
+    ```
+
+   2. With Cuda 11.6 or 11.7
+
+    ```sh
+    docker compose run --rm --build privategpt-cuda-11.6
+
+    docker compose run --rm --build privategpt-cuda-11.7
+    ```
+
 # Disclaimer
 This is a test project to validate the feasibility of a fully private solution for question answering using LLMs and Vector embeddings. It is not production ready, and it is not meant to be used in production. The models selection is not optimized for performance, but for privacy; but it is possible to use different models and vectorstores to improve performance.

cache/.gitignore

@@ -0,0 +1,3 @@
+*
+!.gitignore
+!.gitkeep

docker-compose.yml

@@ -0,0 +1,129 @@
+---
+version: '3.9'
+services:
+  #
+  # Base service, without CUDA
+  #
+  privategpt:
+    build:
+      context: .
+      dockerfile: ./Dockerfile
+      args: [ --rm, "BASEIMAGE=python:3.10.11", LLAMA_CMAKE= ]
+    image: privategpt:test
+    command: [ python, src/privateGPT.py ]
+    environment:
+      - PERSIST_DIRECTORY=${PERSIST_DIRECTORY:-/home/privategpt/db}
+      - LLAMA_EMBEDDINGS_MODEL=${LLAMA_EMBEDDINGS_MODEL:-/home/privategpt/models/ggml-model-q4_0.bin}
+      - EMBEDDINGS_MODEL_NAME=${EMBEDDINGS_MODEL_NAME:-all-MiniLM-L6-v2}
+      - MODEL_TYPE=${MODEL_TYPE:-GPT4All}
+      - MODEL_PATH=${MODEL_PATH:-/home/privategpt/models/ggml-gpt4all-j-v1.3-groovy.bin}
+      - MODEL_N_CTX=${MODEL_N_CTX:-1000}
+    volumes:
+      - ${CACHE_MOUNT:-./cache}:/home/privategpt/.cache/torch
+      - ${MODEL_MOUNT:-./models}:/home/privategpt/models
+      - ${PERSIST_MOUNT:-./db}:/home/privategpt/db
+
+  #
+  # To run with CUDA 11.6
+  #
+  #  docker compose run --rm --build privategpt-cuda-11.6
+  #
+  privategpt-cuda-11.6:
+    extends: privategpt
+    image: privategpt:cuda-11.6
+    build:
+      args:
+        - BASEIMAGE=wallies/python-cuda:3.10-cuda11.6-runtime
+        - LLAMA_CMAKE=CMAKE_ARGS='-DLLAMA_OPENBLAS=on' FORCE_CMAKE=1
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: 1
+              capabilities: [ gpu ]
+
+  #
+  # To run with CUDA 11.7
+  #
+  #  docker compose run --rm --build privategpt-cuda-11.7
+  #
+  privategpt-cuda-11.7:
+    extends: privategpt-cuda-11.6
+    image: privategpt:cuda-11.7
+    build:
+      args: [ "BASEIMAGE=wallies/python-cuda:3.10-cuda11.7-runtime" ]
+
+  #
+  # For ingest without cuda:
+  #
+  #  docker compose run --rm --build privategpt-ingest
+  #
+  privategpt-ingest:
+    extends: privategpt
+    command: [ python, src/ingest.py ]
+    environment:
+      - SOURCE_DIRECTORY=${SOURCE_DIRECTORY:-/home/privategpt/source_documents}
+    volumes:
+      - ${SOURCE_MOUNT:-./source_documents}:/home/privategpt/source_documents
+
+  #
+  # To ingest using cuda 11.6:
+  #
+  #  docker compose run --rm --build privategpt-cuda-11.6-ingest
+  #
+  privategpt-cuda-11.6-ingest:
+    extends: privategpt-cuda-11.6
+    image: privategpt:cuda-11.6
+    command: [ python, src/ingest.py ]
+    environment:
+      - SOURCE_DIRECTORY=${SOURCE_DIRECTORY:-/home/privategpt/source_documents}
+    volumes:
+      - ${SOURCE_MOUNT:-./source_documents}:/home/privategpt/source_documents
+
+  #
+  # To ingest using cuda 11.7:
+  #
+  #  docker compose run --rm --build privategpt-cuda-11.7-ingest
+  #
+  privategpt-cuda-11.7-ingest:
+    extends: privategpt-cuda-11.7
+    image: privategpt:cuda-11.7
+    command: [ python, src/ingest.py ]
+    environment:
+      - SOURCE_DIRECTORY=${SOURCE_DIRECTORY:-/home/privategpt/source_documents}
+    volumes:
+      - ${SOURCE_MOUNT:-./source_documents}:/home/privategpt/source_documents
+
+  # Check your system's version using
+  #
+  #  docker compose run --rm check-cuda-version
+  #
+  # then build and test the privateGPT container
+  # using
+  #
+  #  docker compose run --rm check-cuda-<CUDAVERSION>
+  #
+  # Where <CUDAVERSION> is the version you found using 'check-cuda-version'.
+  #
+  # Example if CUDAVERSION == 11.6
+  #
+  #  docker compose run --rm --build check-cuda-11.6
+  #
+  #
+  #
+  # You can update your host's CUDA installation by downloading
+  # a recent version from
+  #
+  # https://developer.nvidia.com/cuda-downloads .
+  #
+
+  check-cuda-version:
+    image: ubuntu
+    command: [ nvidia-smi ]
+  check-cuda-11.6:
+    extends: privategpt-cuda-11.6
+    command: [ nvidia-smi ]
+  check-cuda-11.7:
+    extends: privategpt-cuda-11.7
+    command: [ nvidia-smi ]

models/.gitignore

@@ -0,0 +1,3 @@
+*
+!.gitignore
+!.gitkeep

source_documents/.gitignore

@@ -0,0 +1,3 @@
+*
+!.gitignore
+!.gitkeep