Skip to content

Commit 1a31242

Browse files
tlrmchlsmthtlrmchlsmth
committed
Merge branch 'main' into cutlass-moe-bf16-weights
Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com>
2 parents c25904d + 4dc52e1 commit 1a31242

File tree

411 files changed

+18495
-5267
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

411 files changed

+18495
-5267
lines changed

‎.buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh‎

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,15 +10,24 @@ set -x
1010
set -o pipefail
1111

1212
check_gpus() {
13-
# check the number of GPUs and GPU type.
14-
declare -g gpu_count=$(nvidia-smi --list-gpus | wc -l)
13+
if command -v nvidia-smi; then
14+
# check the number of GPUs and GPU type.
15+
declare -g gpu_count=$(nvidia-smi --list-gpus | wc -l)
16+
elif command -v amd-smi; then
17+
declare -g gpu_count=$(amd-smi list | grep 'GPU' | wc -l)
18+
fi
19+
1520
if [[ $gpu_count -gt 0 ]]; then
1621
echo "GPU found."
1722
else
1823
echo "Need at least 1 GPU to run benchmarking."
1924
exit 1
2025
fi
21-
declare -g gpu_type=$(nvidia-smi --query-gpu=name --format=csv,noheader | awk '{print $2}')
26+
if command -v nvidia-smi; then
27+
declare -g gpu_type=$(nvidia-smi --query-gpu=name --format=csv,noheader | awk '{print $2}')
28+
elif command -v amd-smi; then
29+
declare -g gpu_type=$(amd-smi static -g 0 -a | grep 'MARKET_NAME' | awk '{print $2}')
30+
fi
2231
echo "GPU type is $gpu_type"
2332
}
2433

@@ -90,9 +99,15 @@ kill_gpu_processes() {
9099

91100

92101
# wait until GPU memory usage smaller than 1GB
93-
while [ "$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -n 1)" -ge 1000 ]; do
94-
sleep 1
95-
done
102+
if command -v nvidia-smi; then
103+
while [ "$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -n 1)" -ge 1000 ]; do
104+
sleep 1
105+
done
106+
elif command -v amd-smi; then
107+
while [ "$(amd-smi metric -g 0 | grep 'USED_VRAM' | awk '{print $2}')" -ge 1000 ]; do
108+
sleep 1
109+
done
110+
fi
96111

97112
# remove vllm config file
98113
rm -rf ~/.config/vllm

‎.buildkite/nightly-benchmarks/tests/serving-tests.json‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -63,10 +63,12 @@
6363
"model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
6464
"disable_log_requests": "",
6565
"tensor_parallel_size": 4,
66-
"swap_space": 16,
67-
"speculative_model": "turboderp/Qwama-0.5B-Instruct",
68-
"num_speculative_tokens": 4,
69-
"speculative_draft_tensor_parallel_size": 1
66+
"swap_space": 16,
67+
"speculative_config": {
68+
"model": "turboderp/Qwama-0.5B-Instruct",
69+
"num_speculative_tokens": 4,
70+
"draft_tensor_parallel_size": 1
71+
}
7072
},
7173
"client_parameters": {
7274
"model": "meta-llama/Meta-Llama-3.1-70B-Instruct",

‎.buildkite/release-pipeline.yaml‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,21 +3,21 @@ steps:
33
agents:
44
queue: cpu_queue_postmerge
55
commands:
6-
- "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.4.0 --tag vllm-ci:build-image --target build --progress plain ."
6+
- "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.4.0 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
77
- "mkdir artifacts"
88
- "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
9-
- "bash .buildkite/upload-wheels.sh"
9+
- "bash .buildkite/scripts/upload-wheels.sh"
1010
env:
1111
DOCKER_BUILDKIT: "1"
1212

1313
- label: "Build wheel - CUDA 12.1"
1414
agents:
1515
queue: cpu_queue_postmerge
1616
commands:
17-
- "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.1.0 --tag vllm-ci:build-image --target build --progress plain ."
17+
- "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.1.0 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
1818
- "mkdir artifacts"
1919
- "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
20-
- "bash .buildkite/upload-wheels.sh"
20+
- "bash .buildkite/scripts/upload-wheels.sh"
2121
env:
2222
DOCKER_BUILDKIT: "1"
2323

@@ -31,10 +31,10 @@ steps:
3131
agents:
3232
queue: cpu_queue_postmerge
3333
commands:
34-
- "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=11.8.0 --tag vllm-ci:build-image --target build --progress plain ."
34+
- "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=11.8.0 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
3535
- "mkdir artifacts"
3636
- "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
37-
- "bash .buildkite/upload-wheels.sh"
37+
- "bash .buildkite/scripts/upload-wheels.sh"
3838
env:
3939
DOCKER_BUILDKIT: "1"
4040

@@ -48,7 +48,7 @@ steps:
4848
queue: cpu_queue_postmerge
4949
commands:
5050
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
51-
- "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.4.0 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT --target vllm-openai --progress plain ."
51+
- "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.4.0 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT --target vllm-openai --progress plain -f docker/Dockerfile ."
5252
- "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT"
5353

5454
- label: "Build and publish TPU release image"
@@ -57,7 +57,7 @@ steps:
5757
agents:
5858
queue: tpu_queue_postmerge
5959
commands:
60-
- "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --tag vllm/vllm-tpu:nightly --tag vllm/vllm-tpu:$BUILDKITE_COMMIT --progress plain -f Dockerfile.tpu ."
60+
- "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --tag vllm/vllm-tpu:nightly --tag vllm/vllm-tpu:$BUILDKITE_COMMIT --progress plain -f docker/Dockerfile.tpu ."
6161
- "docker push vllm/vllm-tpu:nightly"
6262
- "docker push vllm/vllm-tpu:$BUILDKITE_COMMIT"
6363
plugins:
@@ -82,7 +82,7 @@ steps:
8282
queue: cpu_queue_postmerge
8383
commands:
8484
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
85-
- "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --tag public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:$(buildkite-agent meta-data get release-version) --tag public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:latest --progress plain -f Dockerfile.cpu ."
85+
- "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --tag public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:$(buildkite-agent meta-data get release-version) --tag public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:latest --progress plain --target vllm-openai -f docker/Dockerfile.cpu ."
8686
- "docker push public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:$(buildkite-agent meta-data get release-version)"
8787
env:
8888
DOCKER_BUILDKIT: "1"

‎.buildkite/run-amd-test.sh‎ renamed to ‎.buildkite/scripts/hardware_ci/run-amd-test.sh‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -105,19 +105,33 @@ fi
105105
if [[ $commands == *" entrypoints/openai "* ]]; then
106106
commands=${commands//" entrypoints/openai "/" entrypoints/openai \
107107
--ignore=entrypoints/openai/test_audio.py \
108-
--ignore=entrypoints/openai/test_chat.py \
109108
--ignore=entrypoints/openai/test_shutdown.py \
110109
--ignore=entrypoints/openai/test_completion.py \
111110
--ignore=entrypoints/openai/test_sleep.py \
112111
--ignore=entrypoints/openai/test_models.py \
112+
--ignore=entrypoints/openai/test_lora_adapters.py \
113+
--ignore=entrypoints/openai/test_return_tokens_as_ids.py \
114+
--ignore=entrypoints/openai/test_root_path.py \
115+
--ignore=entrypoints/openai/test_tokenization.py \
113116
--ignore=entrypoints/openai/test_prompt_validation.py "}
114117
fi
115118

116119
#ignore certain Entrypoints/llm tests
117-
if [[ $commands == *" && pytest -v -s entrypoints/llm/test_guided_generate.py"* ]]; then
118-
commands=${commands//" && pytest -v -s entrypoints/llm/test_guided_generate.py"/" "}
120+
if [[ $commands == *" entrypoints/llm "* ]]; then
121+
commands=${commands//" entrypoints/llm "/" entrypoints/llm \
122+
--ignore=entrypoints/llm/test_chat.py \
123+
--ignore=entrypoints/llm/test_accuracy.py \
124+
--ignore=entrypoints/llm/test_init.py \
125+
--ignore=entrypoints/llm/test_generate_multiple_loras.py \
126+
--ignore=entrypoints/llm/test_prompt_validation.py "}
119127
fi
120128

129+
#Obsolete currently
130+
##ignore certain Entrypoints/llm tests
131+
#if [[ $commands == *" && pytest -v -s entrypoints/llm/test_guided_generate.py"* ]]; then
132+
# commands=${commands//" && pytest -v -s entrypoints/llm/test_guided_generate.py"/" "}
133+
#fi
134+
121135
# --ignore=entrypoints/openai/test_encoder_decoder.py \
122136
# --ignore=entrypoints/openai/test_embedding.py \
123137
# --ignore=entrypoints/openai/test_oot_registration.py

‎.buildkite/run-cpu-test-ppc64le.sh‎ renamed to ‎.buildkite/scripts/hardware_ci/run-cpu-test-ppc64le.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,5 @@ trap remove_docker_container EXIT
1010
remove_docker_container
1111

1212
# Try building the docker image
13-
docker build -t cpu-test -f Dockerfile.ppc64le .
13+
docker build -t cpu-test -f docker/Dockerfile.ppc64le .
1414

‎.buildkite/run-cpu-test.sh‎ renamed to ‎.buildkite/scripts/hardware_ci/run-cpu-test.sh‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,19 @@ set -ex
88
CORE_RANGE=${CORE_RANGE:-48-95}
99
NUMA_NODE=${NUMA_NODE:-1}
1010

11-
# Try building the docker image
12-
numactl -C "$CORE_RANGE" -N "$NUMA_NODE" docker build -t cpu-test-"$BUILDKITE_BUILD_NUMBER" -f Dockerfile.cpu .
13-
numactl -C "$CORE_RANGE" -N "$NUMA_NODE" docker build --build-arg VLLM_CPU_DISABLE_AVX512="true" -t cpu-test-"$BUILDKITE_BUILD_NUMBER"-avx2 -f Dockerfile.cpu .
14-
1511
# Setup cleanup
16-
remove_docker_container() { set -e; docker rm -f cpu-test-"$BUILDKITE_BUILD_NUMBER"-"$NUMA_NODE" cpu-test-"$BUILDKITE_BUILD_NUMBER"-avx2-"$NUMA_NODE" || true; }
12+
remove_docker_container() {
13+
set -e;
14+
docker rm -f cpu-test-"$BUILDKITE_BUILD_NUMBER"-"$NUMA_NODE" cpu-test-"$BUILDKITE_BUILD_NUMBER"-avx2-"$NUMA_NODE" || true;
15+
docker image rm cpu-test-"$BUILDKITE_BUILD_NUMBER" cpu-test-"$BUILDKITE_BUILD_NUMBER"-avx2 || true;
16+
}
1717
trap remove_docker_container EXIT
1818
remove_docker_container
1919

20+
# Try building the docker image
21+
numactl -C "$CORE_RANGE" -N "$NUMA_NODE" docker build --tag cpu-test-"$BUILDKITE_BUILD_NUMBER" --target vllm-test -f docker/Dockerfile.cpu .
22+
numactl -C "$CORE_RANGE" -N "$NUMA_NODE" docker build --build-arg VLLM_CPU_DISABLE_AVX512="true" --tag cpu-test-"$BUILDKITE_BUILD_NUMBER"-avx2 --target vllm-test -f docker/Dockerfile.cpu .
23+
2024
# Run the image, setting --shm-size=4g for tensor parallel.
2125
docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --cpuset-cpus="$CORE_RANGE" \
2226
--cpuset-mems="$NUMA_NODE" --privileged=true -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test-"$BUILDKITE_BUILD_NUMBER"-"$NUMA_NODE" cpu-test-"$BUILDKITE_BUILD_NUMBER"
@@ -36,8 +40,6 @@ function cpu_tests() {
3640
# Run basic model test
3741
docker exec cpu-test-"$BUILDKITE_BUILD_NUMBER"-"$NUMA_NODE" bash -c "
3842
set -e
39-
pip install -r vllm/requirements/test.txt
40-
pip install -r vllm/requirements/cpu.txt
4143
pytest -v -s tests/kernels/test_cache.py -m cpu_model
4244
pytest -v -s tests/kernels/test_mla_decode_cpu.py -m cpu_model
4345
pytest -v -s tests/models/decoder_only/language -m cpu_model

‎.buildkite/run-gh200-test.sh‎ renamed to ‎.buildkite/scripts/hardware_ci/run-gh200-test.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ python3 use_existing_torch.py
99

1010
# Try building the docker image
1111
DOCKER_BUILDKIT=1 docker build . \
12+
--file docker/Dockerfile \
1213
--target vllm-openai \
1314
--platform "linux/arm64" \
1415
-t gh200-test \

‎.buildkite/run-hpu-test.sh‎ renamed to ‎.buildkite/scripts/hardware_ci/run-hpu-test.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
set -ex
66

77
# Try building the docker image
8-
docker build -t hpu-test-env -f Dockerfile.hpu .
8+
docker build -t hpu-test-env -f docker/Dockerfile.hpu .
99

1010
# Setup cleanup
1111
# certain versions of HPU software stack have a bug that can

‎.buildkite/run-neuron-test.sh‎ renamed to ‎.buildkite/scripts/hardware_ci/run-neuron-test.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ else
3535
date "+%s" > /tmp/neuron-docker-build-timestamp
3636
fi
3737

38-
docker build -t "${image_name}" -f Dockerfile.neuron .
38+
docker build -t "${image_name}" -f docker/Dockerfile.neuron .
3939

4040
# Setup cleanup
4141
remove_docker_container() {

‎.buildkite/run-tpu-v1-test.sh‎ renamed to ‎.buildkite/scripts/hardware_ci/run-tpu-v1-test.sh‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
#!/bin/bash
22

3-
set -e
3+
set -xue
44

55
# Build the docker image.
6-
docker build -f Dockerfile.tpu -t vllm-tpu .
6+
docker build -f docker/Dockerfile.tpu -t vllm-tpu .
77

88
# Set up cleanup.
99
remove_docker_container() { docker rm -f tpu-test || true; }
@@ -21,6 +21,8 @@ docker run --privileged --net host --shm-size=16G -it \
2121
&& python3 -m pip install lm_eval[api]==0.4.4 \
2222
&& export VLLM_USE_V1=1 \
2323
&& export VLLM_XLA_CHECK_RECOMPILATION=1 \
24+
&& echo TEST_0 \
25+
&& pytest -v -s /workspace/vllm/tests/v1/tpu/test_perf.py \
2426
&& echo TEST_1 \
2527
&& pytest -v -s /workspace/vllm/tests/tpu/test_compilation.py \
2628
&& echo TEST_2 \
@@ -32,9 +34,14 @@ docker run --privileged --net host --shm-size=16G -it \
3234
&& echo TEST_5 \
3335
&& python3 /workspace/vllm/examples/offline_inference/tpu.py \
3436
&& echo TEST_6 \
35-
&& pytest -s -v /workspace/vllm/tests/tpu/worker/test_tpu_model_runner.py" \
37+
&& pytest -s -v /workspace/vllm/tests/v1/tpu/worker/test_tpu_model_runner.py \
38+
&& echo TEST_7 \
39+
&& pytest -s -v /workspace/vllm/tests/v1/tpu/test_sampler.py \
40+
&& echo TEST_8 \
41+
&& pytest -s -v /workspace/vllm/tests/v1/tpu/test_topk_topp_sampler.py \
42+
&& echo TEST_9 \
43+
&& pytest -s -v /workspace/vllm/tests/v1/tpu/test_pallas.py" \
3644

3745

3846
# TODO: This test fails because it uses RANDOM_SEED sampling
3947
# && VLLM_USE_V1=1 pytest -v -s /workspace/vllm/tests/tpu/test_custom_dispatcher.py \
40-

0 commit comments

Comments
 (0)