Merge branch 'main' into ryan/connector-dev

Author: ryanolson

components/backends/sglang/README.md

@@ -94,12 +94,6 @@ cd $DYNAMO_ROOT/components/backends/sglang
 
 ### Aggregated Serving with KV Routing
 
-> [!NOTE]
-> The current implementation of `components/backends/sglang/src/dynamo/sglang/worker/main.py` publishes _placeholder_ engine metrics to keep the Dynamo KV-router happy. Real-time metrics will be surfaced directly from the SGLang engine once the following pull requests are merged:
-> • Dynamo: [ai-dynamo/dynamo #1465](https://github.com/ai-dynamo/dynamo/pull/1465) – _feat: receive kvmetrics from sglang scheduler_.
->
-> After these are in, the TODOs in `main.py` will be resolved and the placeholder logic removed.
-
 ```bash
 cd $DYNAMO_ROOT/components/backends/sglang
 ./launch/agg_router.sh

components/backends/sglang/slurm_jobs/scripts/gb200.sh

@@ -86,7 +86,7 @@ if [ "$mode" = "prefill" ]; then
         SGLANG_USE_MESSAGE_QUEUE_BROADCASTER=0 \
         SGL_DISABLE_TP_MEMORY_INBALANCE_CHECK=1 \
         PYTHONUNBUFFERED=1 \
-        python3 components/worker.py \
+        python3 -m dynamo.sglang.worker \
             --served-model-name deepseek-ai/DeepSeek-R1 \
             --model-path /model/ \
             --skip-tokenizer-init \
@@ -188,7 +188,7 @@ elif [ "$mode" = "decode" ]; then
         SGLANG_USE_MESSAGE_QUEUE_BROADCASTER=0 \
         SGL_DISABLE_TP_MEMORY_INBALANCE_CHECK=1 \
         PYTHONUNBUFFERED=1 \
-        python3 components/decode_worker.py \
+        python3 -m dynamo.sglang.decode_worker \
             --served-model-name deepseek-ai/DeepSeek-R1 \
             --model-path /model/ \
             --skip-tokenizer-init \

components/backends/sglang/slurm_jobs/scripts/h100.sh

@@ -70,7 +70,7 @@ fi
 if [ "$mode" = "prefill" ]; then
     if [ "$cmd" = "dynamo" ]; then
         # H100 dynamo prefill command
-        python3 components/worker.py \
+        python3 -m dynamo.sglang.worker \
             --model-path /model/ \
             --served-model-name deepseek-ai/DeepSeek-R1 \
             --skip-tokenizer-init \
@@ -131,7 +131,7 @@ if [ "$mode" = "prefill" ]; then
 elif [ "$mode" = "decode" ]; then
     if [ "$cmd" = "dynamo" ]; then
         # H100 dynamo decode command
-        python3 components/decode_worker.py \
+        python3 -m dynamo.sglang.decode_worker \
             --model-path /model/ \
             --served-model-name deepseek-ai/DeepSeek-R1 \
             --skip-tokenizer-init \

components/backends/trtllm/kv-cache-tranfer.md

@@ -61,10 +61,4 @@ To enable NIXL for KV cache transfer in disaggregated serving:
 4. **Send the request:**
    See [client](./README.md#client) section to learn how to send the request to deployment.
 
-**Important:** Ensure that ETCD and NATS services are running before starting the service.
-
-The container will automatically configure the appropriate environment variables (`TRTLLM_USE_NIXL_KVCACHE=1`) when built with the NIXL flag. The same container image can be used to use UCX for KV cache transfer.
-```bash
-unset TRTLLM_USE_NIXL_KVCACHE
-export TRTLLM_USE_UCX_KVCACHE=1
-```
+**Important:** Ensure that ETCD and NATS services are running before starting the service.

components/backends/trtllm/multinode/start_trtllm_worker.sh

@@ -22,13 +22,6 @@ if [[ -z ${ENGINE_CONFIG} ]]; then
     exit 1
 fi
 
-# NOTE: When this script is run directly from srun, the environment variables
-# for TRTLLM KV cache are not set. So we need to set them here.
-# Related issue: https://github.com/ai-dynamo/dynamo/issues/1743
-if [[ -z ${TRTLLM_USE_UCX_KVCACHE} ]] && [[ -z ${TRTLLM_USE_NIXL_KVCACHE} ]]; then
-    export TRTLLM_USE_UCX_KVCACHE=1
-fi
-
 EXTRA_ARGS=""
 if [[ -n ${DISAGGREGATION_MODE} ]]; then
   EXTRA_ARGS+="--disaggregation-mode ${DISAGGREGATION_MODE} "

container/Dockerfile.tensorrt_llm

@@ -332,26 +332,6 @@ RUN pip install dist/ai_dynamo_runtime*cp312*.whl  && \
     pip install dist/ai_dynamo*any.whl
 
 ENV DYNAMO_HOME=/workspace
-
-# Use UCX for TRTLLM KV Cache Transfer
-ARG TRTLLM_USE_NIXL_KVCACHE_EXPERIMENTAL
-ENV TRTLLM_USE_UCX_KVCACHE=1
-
-# Create a script that sets the environment variables and source it.
-# If TRTLLM_USE_NIXL_KVCACHE_EXPERIMENTAL is set to 1, unset TRTLLM_USE_UCX_KVCACHE
-# and set TRTLLM_USE_NIXL_KVCACHE to 1.
-RUN echo '#!/bin/bash' > /usr/local/bin/set_trtllm_env.sh && \
-    if [ "$TRTLLM_USE_NIXL_KVCACHE_EXPERIMENTAL" = "1" ]; then \
-        echo 'unset TRTLLM_USE_UCX_KVCACHE' >> /usr/local/bin/set_trtllm_env.sh; \
-        echo 'export TRTLLM_USE_NIXL_KVCACHE=1' >> /usr/local/bin/set_trtllm_env.sh; \
-    else \
-        echo '# TRTLLM_USE_UCX_KVCACHE should already be set to 1' >> /usr/local/bin/set_trtllm_env.sh; \
-    fi && \
-    chmod +x /usr/local/bin/set_trtllm_env.sh
-
-# Source the script in bashrc
-RUN echo 'source /usr/local/bin/set_trtllm_env.sh' >> /root/.bashrc
-
 # Copy launch banner
 RUN --mount=type=bind,source=./container/launch_message.txt,target=/workspace/launch_message.txt \
     sed '/^#\s/d' /workspace/launch_message.txt > ~/.launch_screen && \
@@ -501,11 +481,6 @@ RUN uv pip install --extra-index-url "${TENSORRTLLM_INDEX_URL}" \
     "${TENSORRTLLM_PIP_WHEEL}" && \
     uv pip install ai-dynamo nixl --find-links wheelhouse
 
-# Setup TRTLLM environment variables, same as in dev image
-ENV TRTLLM_USE_UCX_KVCACHE=1
-COPY --from=dev /usr/local/bin/set_trtllm_env.sh /usr/local/bin/set_trtllm_env.sh
-RUN echo 'source /usr/local/bin/set_trtllm_env.sh' >> /root/.bashrc
-
 # Copy benchmarks, backends and tests for CI
 # TODO: Remove this once we have a functional CI image built on top of the runtime image
 COPY tests /workspace/tests