fix: WAR DeepGemm JIT compilation errors (#2937)

Signed-off-by: GuanLuo <gluo@nvidia.com>
Co-authored-by: Harrison Saturley-Hall <hsaturleyhal@nvidia.com>
Signed-off-by: Indrajit Bhosale <iamindrajitb@gmail.com>

Author: 2 people

container/Dockerfile.vllm

@@ -199,7 +199,9 @@ RUN apt-get update && \
         ninja-build \
         g++ \
         # prometheus dependencies
-        ca-certificates && \
+        ca-certificates \
+        # DeepGemm uses 'cuobjdump' which does not come with CUDA image
+        cuda-command-line-tools-12-8 && \
     rm -rf /var/lib/apt/lists/*
 
 # Copy CUDA development tools (nvcc, headers, dependencies, etc.) from base devel image
@@ -249,6 +251,10 @@ $NIXL_PLUGIN_DIR:\
 /usr/local/ucx/lib/ucx:\
 $LD_LIBRARY_PATH
 
+# DeepGemm runs nvcc for JIT kernel compilation, however the CUDA include path
+# is not properly set for complilation. Set CPATH to help nvcc find the headers.
+ENV CPATH=/usr/local/cuda/include:$CPATH
+
 ### VIRTUAL ENVIRONMENT SETUP ###
 
 # Copy uv and entire virtual environment from framework container