From 86068f90733477fd0c3ca997d6369843c4366be9 Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Tue, 4 Mar 2025 18:39:57 -0500 Subject: [PATCH 1/2] [CI/Build] Use spawn multiprocessing mode for V1 test pipeline I've seen multiple failures related to forking new processes after cuda has been initialized in this pipeline. Other pipelines use `spawn`, so this change applies the same to this one. Signed-off-by: Russell Bryant --- .buildkite/test-pipeline.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml index d0f5c94ffd8d..9477bb8d5aad 100644 --- a/.buildkite/test-pipeline.yaml +++ b/.buildkite/test-pipeline.yaml @@ -196,6 +196,7 @@ steps: - tests/v1 commands: # split the test to avoid interference + - export VLLM_WORKER_MULTIPROC_METHOD=spawn - VLLM_USE_V1=1 pytest -v -s v1/core - VLLM_USE_V1=1 pytest -v -s v1/engine - VLLM_USE_V1=1 pytest -v -s v1/sample From 0977b91ab2405e43c59bb48628d6599656462864 Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Tue, 4 Mar 2025 18:43:40 -0500 Subject: [PATCH 2/2] Apply spawn method to 2 more test pipelines I have seen fail Signed-off-by: Russell Bryant --- .buildkite/test-pipeline.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml index 9477bb8d5aad..521faeedd415 100644 --- a/.buildkite/test-pipeline.yaml +++ b/.buildkite/test-pipeline.yaml @@ -78,6 +78,7 @@ steps: - tests/basic_correctness/test_preemption - tests/basic_correctness/test_cumem.py commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s basic_correctness/test_cumem.py - pytest -v -s basic_correctness/test_basic_correctness.py - pytest -v -s basic_correctness/test_cpu_offload.py @@ -112,6 +113,7 @@ steps: - tests/entrypoints/test_chat_utils - tests/entrypoints/offline_mode commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/llm --ignore=entrypoints/llm/test_lazy_outlines.py --ignore=entrypoints/llm/test_generate.py --ignore=entrypoints/llm/test_generate_multiple_loras.py --ignore=entrypoints/llm/test_guided_generate.py --ignore=entrypoints/llm/test_collective_rpc.py - pytest -v -s entrypoints/llm/test_lazy_outlines.py # it needs a clean process - pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process