Merge branch 'main' into nmailhot/final-error-log

Author: nv-nmailhot

.devcontainer/README.md

@@ -1,24 +1,26 @@
 <!--
 SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 SPDX-License-Identifier: Apache-2.0
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
 -->
 
 # NVIDIA Dynamo Development Environment
 
 > Warning: Dev Containers (aka `devcontainers`) is an evolving feature and we are not testing in CI. Please submit any problem/feedback using the issues on GitHub.
 
+## Known Issues
+
+### Docker Version Compatibility
+
+**Docker 29.x has compatibility issues with Dev Containers (by Anysphere):**
+- It is known that Docker Engine version **29.0.1** (released November 14, 2025) may cause Dev Containers to hang all the time, rendering it unusable
+- The Dev Containers extension (v1.0.26) and devcontainer CLI (v0.75.0) were not tested against Docker 29.x
+- Symptoms: Container builds successfully but connection hangs, requiring a manual reload to connect
+- This may be fixed in a later version of the Anysphere Dev Containers extension and/or Docker Engine patch
+
+**Recommended Docker Version:**
+- Use Docker Engine **28.5.2** or earlier for stable Dev Container operation
+- Docker 27.x and 28.x series are confirmed working with current Dev Containers tooling
+
 ## Framework-Specific Devcontainers
 
 This directory contains framework-specific devcontainer configurations generated from a Jinja2 template:
@@ -79,16 +81,16 @@ graph TB
         HFCACHE["~/.cache/huggingface<br/>(host cache)"]
         GITCONFIG["~/.gitconfig<br/>(host git config)"]
 
-        subgraph CONTAINER["Docker Container<br/>vsc-dynamo-SHA-uid<br/>Running as: ubuntu user"]
-            MOUNT["/home/ubuntu/dynamo<br/>(mounted directory)"]
-            HFMOUNT["/home/ubuntu/.cache/huggingface<br/>(mounted cache)"]
-            GITCONFIGCOPY["/home/ubuntu/.gitconfig<br/>(via Dev Container setting)"]
+        subgraph CONTAINER["Docker Container<br/>vsc-dynamo-SHA-uid<br/>Running as: dynamo user"]
+            MOUNT["/home/dynamo/dynamo<br/>(mounted directory)"]
+            HFMOUNT["/home/dynamo/.cache/huggingface<br/>(mounted cache)"]
+            GITCONFIGCOPY["/home/dynamo/.gitconfig<br/>(via Dev Container setting)"]
             TOOLS["rust-analyzer<br/>cargo<br/>etc."]
         end
 
         IMAGE["Docker Image<br/>dynamo:latest-{framework}-local-dev<br/>(vllm/sglang/trtllm)"]
 
-        IMAGE -->|"docker run<br/>as ubuntu user"| CONTAINER
+        IMAGE -->|"docker run<br/>as dynamo user"| CONTAINER
     end
 
     TERM -->|"SSH Connection"| DIR
@@ -260,9 +262,9 @@ File Structure:
 - Local dynamo repo mounts to `/workspace`
 - Python venv in `/opt/dynamo/venv`
 - Build artifacts in `/workspace/target`
-- Hugging Face cache preserved between sessions (either mounting your host .cache to the container, or your `HF_HOME` to `/home/ubuntu/.cache/huggingface`)
-- Bash memory preserved between sessions at `/home/ubuntu/.commandhistory` using docker volume `dynamo-bashhistory`
-- Precommit preserved between sessions at `/home/ubuntu/.cache/precommit` using docker volume `dynamo-precommit-cache`
+- Hugging Face cache preserved between sessions (either mounting your host .cache to the container, or your `HF_HOME` to `/home/dynamo/.cache/huggingface`)
+- Bash memory preserved between sessions at `/home/dynamo/.commandhistory` using docker volume `dynamo-bashhistory`
+- Precommit preserved between sessions at `/home/dynamo/.cache/precommit` using docker volume `dynamo-precommit-cache`
 
 ## Documentation
 
@@ -397,8 +399,8 @@ docker volume prune -f
 **Note:** This resets bash history and pre-commit cache.
 
 **Volume Mounts in devcontainer.json:**
-- `dynamo-bashhistory` → `/home/ubuntu/.commandhistory` (bash history)
-- `dynamo-precommit-cache` → `/home/ubuntu/.cache/pre-commit` (pre-commit cache)
+- `dynamo-bashhistory` → `/home/dynamo/.commandhistory` (bash history)
+- `dynamo-precommit-cache` → `/home/dynamo/.cache/pre-commit` (pre-commit cache)
 
 ### Permission Issues
 

.devcontainer/post-create.sh

@@ -11,14 +11,14 @@ WORKSPACE_DIR="${WORKSPACE_DIR:-/workspace}"
 # Ensure we're not running as root
 if [ "$(id -u)" -eq 0 ]; then
     echo "❌ ERROR: This script should not be run as root!"
-    echo "The script should run as the 'ubuntu' user, not root."
+    echo "The script should run as the 'dynamo' user, not root."
     echo "Current user: $(whoami) (UID: $(id -u))"
     exit 1
 fi
 
 # Verify we're running as the expected user
-if [ "$(whoami)" != "ubuntu" ]; then
-    echo "⚠️  WARNING: Expected to run as 'ubuntu' user, but running as '$(whoami)'"
+if [ "$(whoami)" != "dynamo" ]; then
+    echo "⚠️  WARNING: Expected to run as 'dynamo' user, but running as '$(whoami)'"
     echo "This might cause permission issues."
 fi
 
@@ -48,7 +48,9 @@ show_and_run() {
     # Run commands with debug output shown
     set -x
     "$@"
+    local exit_code=$?
     { set +x; } 2>/dev/null
+    return $exit_code
 }
 
 set -x
@@ -62,28 +64,7 @@ pre-commit run --all-files || true # don't fail the build if pre-commit hooks fa
 export CARGO_TARGET_DIR=${CARGO_TARGET_DIR:-$WORKSPACE_DIR/target}
 mkdir -p $CARGO_TARGET_DIR
 
-uv pip uninstall --yes ai-dynamo ai-dynamo-runtime 2>/dev/null || true
-
-# Build project, with `dev` profile it will be saved at $CARGO_TARGET_DIR/debug
-cargo build --locked --profile dev --features dynamo-llm/block-manager
-
-# install the python bindings
-
-# Install maturin if not already installed.
-# TODO: Uncomment for SGLANG. Right now, the CI team is making a refactor
-#       to the Dockerfile, so this is a temporary fix.
-# if ! command -v maturin &> /dev/null; then
-#     echo "Installing maturin..."
-#     retry uv pip install maturin[patchelf]
-# else
-#     echo "maturin is already installed"
-# fi
-
-# install ai-dynamo-runtime
-(cd $WORKSPACE_DIR/lib/bindings/python && retry maturin develop)
-
-# install ai-dynamo
-cd $WORKSPACE_DIR && retry env DYNAMO_BIN_PATH=$CARGO_TARGET_DIR/debug uv pip install -e .
+# Note: Build steps moved to after sanity check - see instructions at the end
 
 { set +x; } 2>/dev/null
 
@@ -114,16 +95,26 @@ else
     echo "⚠️ SSH agent forwarding not configured - SSH_AUTH_SOCK is not set"
 fi
 
-show_and_run $WORKSPACE_DIR/deploy/sanity_check.py
+echo -e "\n🔍 Running sanity check..."
+if show_and_run $WORKSPACE_DIR/deploy/sanity_check.py; then
+    SANITY_STATUS="✅ Sanity check passed"
+else
+    SANITY_STATUS="⚠️ Sanity check failed - review output above"
+fi
 
 cat <<EOF
 
-✅ SUCCESS: Built cargo project, installed Python bindings, configured pre-commit hooks
+========================================
+$SANITY_STATUS
+✅ Pre-commit hooks configured
+
+Now build the project:
+  cargo build --locked --profile dev --features dynamo-llm/block-manager
+  cd lib/bindings/python && maturin develop --uv
+  DYNAMO_BIN_PATH=$CARGO_TARGET_DIR/debug uv pip install -e .
+
+Optional: cd lib/bindings/kvbm && maturin develop --uv  # For KVBM support
 
-Example commands:
-  cargo build --locked --profile dev              # Build Rust project in $CARGO_TARGET_DIR
-  cd lib/bindings/python && maturin develop --uv  # Update Python bindings (if you changed them)
-  cargo fmt && cargo clippy                       # Format and lint code before committing
-  cargo doc --no-deps                             # Generate documentation
-  uv pip install -e .                             # Install various Python packages Dynamo depends on
+If cargo build fails with a Cargo.lock error, try to update it with 'cargo update'
+========================================
 EOF

.devcontainer/sglang/devcontainer.json

@@ -8,7 +8,7 @@
         "SPDX-License-Identifier: Apache-2.0"
     ],
     "name": "Dynamo SGLANG Dev Container",
-    "remoteUser": "ubuntu", // Matches our container user
+    "remoteUser": "dynamo", // Matches our container user
     "updateRemoteUserUID": true, // Updates the UID of the remote user to match the host user, avoids permission errors
     "image": "dynamo:latest-sglang-local-dev", // Use the latest SGLANG dev image
     "runArgs": [
@@ -71,11 +71,11 @@
     },
     "mounts": [
         // These are for convenience, so that the history and pre-commit cache are persisted between sessions
-        "source=dynamo-bashhistory,target=/home/ubuntu/.commandhistory,type=volume",
-        "source=dynamo-precommit-cache,target=/home/ubuntu/.cache/pre-commit,type=volume",
+        "source=dynamo-bashhistory,target=/home/dynamo/.commandhistory,type=volume",
+        "source=dynamo-precommit-cache,target=/home/dynamo/.cache/pre-commit,type=volume",
         // Default mounts
         "source=/tmp/,target=/tmp/,type=bind"
         // Uncomment this to reuse your Hugging Face cache
-        //"source=${localEnv:HOME}/.cache/huggingface,target=/home/ubuntu/.cache/huggingface,type=bind"
+        //"source=${localEnv:HOME}/.cache/huggingface,target=/home/dynamo/.cache/huggingface,type=bind"
     ]
 }

.devcontainer/trtllm/devcontainer.json

@@ -8,7 +8,7 @@
         "SPDX-License-Identifier: Apache-2.0"
     ],
     "name": "Dynamo TRTLLM Dev Container",
-    "remoteUser": "ubuntu", // Matches our container user
+    "remoteUser": "dynamo", // Matches our container user
     "updateRemoteUserUID": true, // Updates the UID of the remote user to match the host user, avoids permission errors
     "image": "dynamo:latest-trtllm-local-dev", // Use the latest TRTLLM dev image
     "runArgs": [
@@ -71,11 +71,11 @@
     },
     "mounts": [
         // These are for convenience, so that the history and pre-commit cache are persisted between sessions
-        "source=dynamo-bashhistory,target=/home/ubuntu/.commandhistory,type=volume",
-        "source=dynamo-precommit-cache,target=/home/ubuntu/.cache/pre-commit,type=volume",
+        "source=dynamo-bashhistory,target=/home/dynamo/.commandhistory,type=volume",
+        "source=dynamo-precommit-cache,target=/home/dynamo/.cache/pre-commit,type=volume",
         // Default mounts
         "source=/tmp/,target=/tmp/,type=bind"
         // Uncomment this to reuse your Hugging Face cache
-        //"source=${localEnv:HOME}/.cache/huggingface,target=/home/ubuntu/.cache/huggingface,type=bind"
+        //"source=${localEnv:HOME}/.cache/huggingface,target=/home/dynamo/.cache/huggingface,type=bind"
     ]
 }

.devcontainer/vllm/devcontainer.json

@@ -8,7 +8,7 @@
         "SPDX-License-Identifier: Apache-2.0"
     ],
     "name": "Dynamo VLLM Dev Container",
-    "remoteUser": "ubuntu", // Matches our container user
+    "remoteUser": "dynamo", // Matches our container user
     "updateRemoteUserUID": true, // Updates the UID of the remote user to match the host user, avoids permission errors
     "image": "dynamo:latest-vllm-local-dev", // Use the latest VLLM dev image
     "runArgs": [
@@ -71,11 +71,11 @@
     },
     "mounts": [
         // These are for convenience, so that the history and pre-commit cache are persisted between sessions
-        "source=dynamo-bashhistory,target=/home/ubuntu/.commandhistory,type=volume",
-        "source=dynamo-precommit-cache,target=/home/ubuntu/.cache/pre-commit,type=volume",
+        "source=dynamo-bashhistory,target=/home/dynamo/.commandhistory,type=volume",
+        "source=dynamo-precommit-cache,target=/home/dynamo/.cache/pre-commit,type=volume",
         // Default mounts
         "source=/tmp/,target=/tmp/,type=bind"
         // Uncomment this to reuse your Hugging Face cache
-        //"source=${localEnv:HOME}/.cache/huggingface,target=/home/ubuntu/.cache/huggingface,type=bind"
+        //"source=${localEnv:HOME}/.cache/huggingface,target=/home/dynamo/.cache/huggingface,type=bind"
     ]
 }

.github/actions/docker-build/action.yml

@@ -174,7 +174,7 @@ runs:
 
         # Create job-specific metrics file
         mkdir -p build-metrics
-        METRICS_FILE="build-metrics/metrics-${{ inputs.framework }}-${PLATFORM_ARCH}.json"
+        METRICS_FILE="build-metrics/metrics-${{ inputs.framework }}-${PLATFORM_ARCH}-${{ github.run_id }}-${{ job.check_run_id }}.json"
 
         # Create the job metrics file directly
         cat > "$METRICS_FILE" << EOF
@@ -199,6 +199,6 @@ runs:
     - name: Upload Build Metrics
       uses: actions/upload-artifact@v4
       with:
-        name: build-metrics-${{ inputs.framework }}-${{ env.PLATFORM_ARCH }}
-        path: build-metrics/metrics-${{ inputs.framework }}-${{ env.PLATFORM_ARCH }}.json
+        name: build-metrics-${{ inputs.framework }}-${{ env.PLATFORM_ARCH }}-${{ github.run_id }}-${{ job.check_run_id }}
+        path: build-metrics/metrics-${{ inputs.framework }}-${{ env.PLATFORM_ARCH }}-${{ github.run_id }}-${{ job.check_run_id }}.json
         retention-days: 7

.github/filters.yaml

@@ -35,7 +35,6 @@ vllm: &vllm
 
 sglang: &sglang
   - 'container/Dockerfile.sglang'
-  - 'container/Dockerfile.sglang-wideep'
   - 'examples/backends/sglang/**'
   - 'components/src/dynamo/sglang/**'
   - 'container/build.sh'