From 019498e8155674014f6f09721e3acea75409151d Mon Sep 17 00:00:00 2001 From: Peter Schuurman Date: Mon, 15 Sep 2025 13:12:46 -0700 Subject: [PATCH 1/3] Update default version for Run:ai model streamer from 0.11.0 -> 0.14.0 Signed-off-by: Peter Schuurman --- requirements/nightly_torch_test.txt | 2 +- requirements/rocm.txt | 4 ++-- requirements/test.in | 2 +- requirements/test.txt | 21 ++++++++++++++++++++- 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/requirements/nightly_torch_test.txt b/requirements/nightly_torch_test.txt index 67c66a0c03e6..33f1bc04ea90 100644 --- a/requirements/nightly_torch_test.txt +++ b/requirements/nightly_torch_test.txt @@ -43,6 +43,6 @@ tritonclient==2.51.0 numba == 0.60.0; python_version == '3.9' # v0.61 doesn't support Python 3.9. Required for N-gram speculative decoding numba == 0.61.2; python_version > '3.9' numpy -runai-model-streamer[s3]==0.14.0 +runai-model-streamer[s3,gcs]==0.14.0 fastsafetensors>=0.1.10 pydantic>=2.10 # 2.9 leads to error on python 3.10 diff --git a/requirements/rocm.txt b/requirements/rocm.txt index c4aabe2a7314..9077085f2621 100644 --- a/requirements/rocm.txt +++ b/requirements/rocm.txt @@ -13,6 +13,6 @@ tensorizer==2.10.1 packaging>=24.2 setuptools>=77.0.3,<80.0.0 setuptools-scm>=8 -runai-model-streamer[s3]==0.14.0 +runai-model-streamer[s3,gcs]==0.14.0 conch-triton-kernels==1.2.1 -timm>=1.0.17 \ No newline at end of file +timm>=1.0.17 diff --git a/requirements/test.in b/requirements/test.in index e0a9311af29d..ef21d6db5b4f 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -51,7 +51,7 @@ tritonclient==2.51.0 numba == 0.60.0; python_version == '3.9' # v0.61 doesn't support Python 3.9. Required for N-gram speculative decoding numba == 0.61.2; python_version > '3.9' numpy -runai-model-streamer[s3]==0.14.0 +runai-model-streamer[s3,gcs]==0.14.0 fastsafetensors>=0.1.10 pydantic>=2.10 # 2.9 leads to error on python 3.10 decord==0.6.0 diff --git a/requirements/test.txt b/requirements/test.txt index 07457e14ccbd..9cab85ce0ef6 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -251,11 +251,27 @@ gitdb==4.0.12 gitpython==3.1.44 # via mlflow-skinny google-api-core==2.24.2 - # via opencensus + # via + # google-cloud-core + # google-cloud-storage + # opencensus google-auth==2.40.2 # via # databricks-sdk # google-api-core + # google-cloud-core + # google-cloud-storage + # runai-model-streamer-gcs +google-cloud-core==2.4.3 + # via google-cloud-storage +google-cloud-storage==3.4.0 + # via runai-model-streamer-gcs +google-crc32c==1.7.1 + # via + # google-cloud-storage + # google-resumable-media +google-resumable-media==2.7.2 + # via google-cloud-storage googleapis-common-protos==1.70.0 # via google-api-core graphene==3.4.3 @@ -890,6 +906,7 @@ requests==2.32.3 # docker # evaluate # google-api-core + # google-cloud-storage # huggingface-hub # lightly # lm-eval @@ -929,6 +946,8 @@ rtree==1.4.0 # via torchgeo runai-model-streamer==0.14.0 # via -r requirements/test.in +runai-model-streamer-gcs==0.14.0 + # via runai-model-streamer runai-model-streamer-s3==0.14.0 # via runai-model-streamer s3transfer==0.10.3 From c3248638bdd3da1e881710359c0cac61e86c2358 Mon Sep 17 00:00:00 2001 From: Peter Schuurman Date: Mon, 15 Sep 2025 13:18:15 -0700 Subject: [PATCH 2/3] Update documentation for Run:ai model streamer for loading from Google Cloud Storage Signed-off-by: Peter Schuurman --- docs/models/extensions/runai_model_streamer.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/models/extensions/runai_model_streamer.md b/docs/models/extensions/runai_model_streamer.md index 992dddf385d0..8a97a49825a4 100644 --- a/docs/models/extensions/runai_model_streamer.md +++ b/docs/models/extensions/runai_model_streamer.md @@ -24,6 +24,13 @@ vllm serve s3://core-llm/Llama-3-8b \ --load-format runai_streamer ``` +To run model from Google Cloud Storage run: + +```bash +vllm serve gs://core-llm/Llama-3-8b \ + --load-format runai_streamer +``` + To run model from a S3 compatible object store run: ```bash From b1cb285cf8798c28bf0eef67f1c81e1406d5d6a7 Mon Sep 17 00:00:00 2001 From: Peter Schuurman Date: Tue, 16 Sep 2025 15:12:59 -0700 Subject: [PATCH 3/3] Add GCS ObjectStorageModel.pull_files test Signed-off-by: Peter Schuurman --- .../runai_model_streamer/test_runai_utils.py | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/tests/model_executor/model_loader/runai_model_streamer/test_runai_utils.py b/tests/model_executor/model_loader/runai_model_streamer/test_runai_utils.py index bde77ff66506..e11e4c7289bc 100644 --- a/tests/model_executor/model_loader/runai_model_streamer/test_runai_utils.py +++ b/tests/model_executor/model_loader/runai_model_streamer/test_runai_utils.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import glob +import hashlib import os import tempfile @@ -9,7 +10,8 @@ from vllm.model_executor.model_loader.weight_utils import ( download_weights_from_hf) -from vllm.transformers_utils.runai_utils import (is_runai_obj_uri, +from vllm.transformers_utils.runai_utils import (ObjectStorageModel, + is_runai_obj_uri, list_safetensors) @@ -34,6 +36,23 @@ def test_runai_list_safetensors_local(): assert len(safetensors) == len(files) -if __name__ == "__main__": - test_is_runai_obj_uri() - test_runai_list_safetensors_local() +def test_runai_pull_files_gcs(monkeypatch): + monkeypatch.setenv("RUNAI_STREAMER_GCS_USE_ANONYMOUS_CREDENTIALS", "true") + # Bypass default project lookup by setting GOOGLE_CLOUD_PROJECT + monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "fake-project") + filename = "LT08_L1GT_074061_20130309_20170505_01_T2_MTL.txt" + gcs_bucket = "gs://gcp-public-data-landsat/LT08/01/074/061/LT08_L1GT_074061_20130309_20170505_01_T2/" + gcs_url = f"{gcs_bucket}/{filename}" + model = ObjectStorageModel(gcs_url) + model.pull_files(gcs_bucket, allow_pattern=[f"*{filename}"]) + # To re-generate / change URLs: + # gsutil ls -L gs:// | grep "Hash (md5)" | tr -d ' ' \ + # | cut -d":" -f2 | base64 -d | xxd -p + expected_checksum = "f60dea775da1392434275b311b31a431" + hasher = hashlib.new("md5") + with open(os.path.join(model.dir, filename), 'rb') as f: + # Read the file in chunks to handle large files efficiently + for chunk in iter(lambda: f.read(4096), b''): + hasher.update(chunk) + actual_checksum = hasher.hexdigest() + assert actual_checksum == expected_checksum