requirements.txt

cmake >= 3.21
ninja  # For faster builds.
psutil
sentencepiece  # Required for LLaMA tokenizer.
numpy < 2.0.0
requests
tqdm
py-cpuinfo
transformers >= 4.43.2  # Required for Chameleon and Llama 3.1 hotfox.
tokenizers >= 0.19.1  # Required for Llama 3.
fastapi
aiohttp
openai
uvicorn[standard]
pydantic >= 2.0  # Required for OpenAI server.
pillow  # Required for image processing
prometheus_client >= 0.18.0
prometheus-fastapi-instrumentator >= 7.0.0
tiktoken >= 0.6.0  # Required for DBRX tokenizer
lm-format-enforcer == 0.10.3
outlines >= 0.0.43, < 0.1 # Requires torch >= 2.1.0
typing_extensions
filelock >= 3.10.4 # filelock starts to support `mode` argument from 3.10.4
pyzmq

# Dependencies for NVIDIA GPUs
nvidia-ml-py # for pynvml package
torch == 2.4.0
# These must be updated alongside torch
torchvision == 0.19   # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
xformers == 0.0.27.post2  # Requires PyTorch 2.4.0
vllm-flash-attn == 2.6.1  # Requires PyTorch 2.4.0