Merge branch 'main' into phendricks/refactor-choice-and-finish-reason

Author: paulhendricks

CODEOWNERS

@@ -22,7 +22,7 @@ Cargo.toml @ryanolson @grahamking @paulhendricks @biswapanda @tmonty12 @guanluo
 /examples/hello_world/ @alec-flowers @biswapanda @grahamking @hhzhang16 @ishandhanani @julienmancuso @kkranen @mohammedabdulwahhab @nnshah1 @tedzhouhk
 /examples/llm/ @alec-flowers @biswapanda @grahamking @guanluo @hhzhang16 @ishandhanani @julienmancuso @kkranen @mohammedabdulwahhab @nnshah1 @piotrm-nvidia @ptarasiewiczNV @rmccorm4 @tanmayv25 @tedzhouhk
 /examples/multimodal/ @indrajit96 @krishung5 @whoisj
-/examples/sglang/ @biswapanda @ishandhanani @tedzhouhk
+/examples/sglang/ @biswapanda @ishandhanani @tedzhouhk @rmccorm4
 /examples/tensorrt_llm/ @guanluo @richardhuo-nv @rmccorm4 @tanmayv25
 /examples/vllm_v0/ @alec-flowers @tedzhouhk
 /examples/vllm_v1/ @biswapanda @ptarasiewiczNV @tedzhouhk

Cargo.lock

@@ -32,14 +32,13 @@
 
 
 class LocalConnector(PlannerConnector):
-    def __init__(self, namespace: str, runtime: DistributedRuntime, backend: str):
+    def __init__(self, namespace: str, runtime: DistributedRuntime):
         """
         Initialize LocalConnector and connect to CircusController.
 
         Args:
             namespace: The Dynamo namespace
             runtime: Optional DistributedRuntime instance
-            backend: The backend to use ("vllm_v0", "vllm_v1")
         """
         self.namespace = namespace
         self.runtime = runtime

components/planner/src/dynamo/planner/local_connector.py

@@ -35,22 +35,62 @@ ARG ARCH_ALT=x86_64
 
 WORKDIR /sgl-workspace
 
+# Install UCX dependencies
+RUN apt-get update -y && \
+    apt-get install -y --no-install-recommends \
+    --reinstall libibverbs-dev rdma-core ibverbs-utils libibumad-dev \
+    libnuma-dev librdmacm-dev ibverbs-providers \
+    autoconf libtool
+
+# Build UCX from source
+ARG NIXL_UCX_REF=v1.19.x
+RUN rm -rf /opt/hpcx/ucx && \
+    rm -rf /usr/local/ucx && \
+    cd /usr/local/src && \
+    git clone https://github.com/openucx/ucx.git && \
+    cd ucx && \
+    git checkout $NIXL_UCX_REF && \
+    ./autogen.sh && ./configure \
+    --prefix=/usr/local/ucx \
+    --enable-shared \
+    --disable-static \
+    --disable-doxygen-doc \
+    --enable-optimizations \
+    --enable-cma \
+    --enable-devel-headers \
+    --with-cuda=/usr/local/cuda \
+    --with-verbs \
+    --with-efa \
+    --with-dm \
+    --with-gdrcopy=/usr/local \
+    --enable-mt && \
+    make -j && \
+    make -j install-strip && \
+    ldconfig
+
+ENV LD_LIBRARY_PATH=/usr/lib:/usr/local/ucx/lib:$LD_LIBRARY_PATH
+
 # Pinning to NIXL 0.2.1 right now
 # TODO: investigate pip install failure with 0.3.0 release
 ARG NIXL_COMMIT="5e4c179ee850d482a83cb2a211e0947e46281060"
-RUN git clone https://github.com/ai-dynamo/nixl.git && cd nixl && git checkout ${NIXL_COMMIT} &&pip install --break-system-packages . --config-settings=setup-args="-Ducx_path=/opt/hpcx/ucx"
+RUN git clone https://github.com/ai-dynamo/nixl.git && cd nixl && git checkout ${NIXL_COMMIT} && pip install --break-system-packages . --config-settings=setup-args="-Ducx_path=/usr/local/ucx"
 
 WORKDIR /sgl-workspace
 
 RUN pip uninstall --break-system-packages -y sglang
 RUN rm -rf sglang
-# 0.4.7
-RUN pip install --break-system-packages "sglang==0.4.7"
+# 0.4.8 has a bug with CUDA graphs and decode worker
+# https://github.com/sgl-project/sglang/issues/7511
+RUN pip install --break-system-packages "sglang==0.4.7.post1"
+
+# Allow forceful shutdown of inflight requests
+ENV SGL_FORCE_SHUTDOWN=1
 
 WORKDIR /sgl-workspace
 # https://github.com/ai-dynamo/dynamo/pull/1510
 ARG DYNAMO_COMMIT="382e3aedc421b3b3abc338062b332b54b5aa8529"
-RUN git clone https://github.com/ai-dynamo/dynamo.git && cd dynamo && git checkout ${DYNAMO_COMMIT}
+ARG DYNAMO_BRANCH="ishan/cmpl-token-id"
+RUN git clone https://github.com/ai-dynamo/dynamo.git && cd dynamo && git checkout ${DYNAMO_BRANCH}
 
 # install dynamo in editable mode
 WORKDIR /sgl-workspace/dynamo

container/Dockerfile.sglang-deepep

@@ -90,9 +90,17 @@ cp $MAIN_DIR/deps/tensorrt_llm/install_nixl.sh docker/common/install_nixl.sh
 sed -i "s/NIXL_COMMIT=\"[^\"]*\"/NIXL_COMMIT=\"${NIXL_COMMIT}\"/" docker/common/install_nixl.sh
 
 
-# Need to build in the Triton Devel Image for NIXL support.
-make -C docker tritondevel_build
-make -C docker wheel_build DEVEL_IMAGE=tritondevel BUILD_WHEEL_OPTS='--extra-cmake-vars NIXL_ROOT=/opt/nvidia/nvda_nixl'
+
+
+if [ "$ARCH" = "amd64" ]; then
+    # Need to build in the Triton Devel Image for NIXL support.
+    make -C docker tritondevel_build
+    make -C docker wheel_build DEVEL_IMAGE=tritondevel BUILD_WHEEL_OPTS='--extra-cmake-vars NIXL_ROOT=/opt/nvidia/nvda_nixl'
+else
+    # NIXL backend is not supported on arm64 for TensorRT-LLM.
+    # See here: https://github.com/NVIDIA/TensorRT-LLM/blob/main/docker/common/install_nixl.sh
+    make -C docker wheel_build
+fi
 
 # Copy the wheel to the host
 mkdir -p $OUTPUT_DIR

container/build_trtllm_wheel.sh

@@ -24,7 +24,6 @@ uv-base:
     COPY +uv-source/uv /bin/uv
     RUN uv venv
     ENV PATH="/app/.venv/bin:$PATH"
-    RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/*
     WORKDIR /app
     COPY uv.lock pyproject.toml README.md /app
     RUN uv sync --frozen --no-install-project --no-dev --no-install-workspace --no-editable

deploy/cloud/api-store/Earthfile

@@ -105,7 +105,9 @@ def add_to_args(args: list[str], key: str, value):
             # Convert to CLI format
             if isinstance(value, bool):
                 if value:
-                    args.append(f"--{arg_key}")
+                    args.extend([f"--{arg_key}", "true"])
+                else:
+                    args.extend([f"--{arg_key}", "false"])
             elif isinstance(value, dict):
                 args.extend([f"--{arg_key}", json.dumps(value)])
             else:

deploy/sdk/src/dynamo/sdk/lib/config.py

@@ -140,3 +140,41 @@ def test_service_config_override_common_configs():
         assert f"--{key}" in vllm_worker_args
 
     assert vllm_worker_args[vllm_worker_args.index("--block-size") + 1] == "128"
+
+
+def test_explicit_boolean_arguments():
+    """Test that boolean arguments are handled with explicit true/false values"""
+    # Reset singleton instance
+    ServiceConfig._instance = None
+
+    # Set environment variable with boolean configs
+    os.environ[
+        "DYNAMO_SERVICE_CONFIG"
+    ] = """
+    {
+        "VllmWorker": {
+            "enable-prefix-caching": true,
+            "disable-sliding-window": false,
+            "enforce-eager": true
+        }
+    }
+    """
+
+    # Get arguments and verify explicit boolean handling
+    service_config = ServiceConfig.get_instance()
+    vllm_worker_args = service_config.as_args("VllmWorker")
+
+    # Check that true values are passed as --arg true
+    assert "--enable-prefix-caching" in vllm_worker_args
+    enable_idx = vllm_worker_args.index("--enable-prefix-caching")
+    assert vllm_worker_args[enable_idx + 1] == "true"
+
+    # Check that false values are passed as --arg false
+    assert "--disable-sliding-window" in vllm_worker_args
+    disable_idx = vllm_worker_args.index("--disable-sliding-window")
+    assert vllm_worker_args[disable_idx + 1] == "false"
+
+    # Check that another true value works
+    assert "--enforce-eager" in vllm_worker_args
+    enforce_idx = vllm_worker_args.index("--enforce-eager")
+    assert vllm_worker_args[enforce_idx + 1] == "true"

deploy/sdk/src/dynamo/sdk/tests/test_config.py

@@ -106,12 +106,12 @@ Dynamo supports SGLang's implementation of wide expert parallelism and large sca
 
 Steps to run:
 
-1. Build the SGLang DeepEP container
+1. Build the SGLang DeepEP container.
 
 ```bash
-git clone https://github.com/sgl-project/sglang.git
+git clone -b v0.4.8 https://github.com/sgl-project/sglang.git
 cd sglang/docker
-docker build -f Dockerfile.deepep -t deepep .
+docker build -f Dockerfile -t deepep .
 ```
 
 You will now have a `deepep:latest` image

examples/sglang/README.md

@@ -45,7 +45,9 @@ def __init__(self):
     @endpoint()
     async def generate(self, req: DisaggPreprocessedRequest):
         g = await self.engine.async_generate(
-            input_ids=req.request.token_ids,
+            input_ids=req.request.token_ids
+            if req.request.batch_token_ids is None
+            else req.request.batch_token_ids,
             sampling_params=req.sampling_params,
             stream=True,
             bootstrap_host=req.bootstrap_host,