From 45dceff0bb96d037c7e7db2534b406a13bb6d507 Mon Sep 17 00:00:00 2001 From: Scott Lee Date: Sat, 24 Feb 2024 11:01:00 -0800 Subject: [PATCH 1/5] add custom resource config Signed-off-by: Scott Lee --- configs/text-embeddings/aws.yaml | 5 +++++ configs/text-embeddings/gce.yaml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/configs/text-embeddings/aws.yaml b/configs/text-embeddings/aws.yaml index 9509c90be..23bf7f049 100644 --- a/configs/text-embeddings/aws.yaml +++ b/configs/text-embeddings/aws.yaml @@ -23,6 +23,7 @@ worker_node_types: object_store_memory: custom_resources: "accelerator_type:A10G": 1 + custom_llm_gpu: 1 min_workers: 0 max_workers: 16 use_spot: true @@ -38,6 +39,7 @@ worker_node_types: object_store_memory: custom_resources: "accelerator_type:T4": 1 + custom_llm_gpu: 1 min_workers: 0 max_workers: 16 use_spot: true @@ -53,6 +55,7 @@ worker_node_types: object_store_memory: custom_resources: "accelerator_type:V100": 1 + custom_llm_gpu: 1 min_workers: 0 max_workers: 16 use_spot: true @@ -68,6 +71,7 @@ worker_node_types: object_store_memory: custom_resources: "accelerator_type:A100-40G": 1 + custom_llm_gpu: 8 aws_advanced_configurations_json: BlockDeviceMappings: - DeviceName: /dev/sda1 @@ -92,6 +96,7 @@ worker_node_types: object_store_memory: custom_resources: "accelerator_type:A100-80G": 1 + custom_llm_gpu: 8 aws_advanced_configurations_json: BlockDeviceMappings: - DeviceName: /dev/sda1 diff --git a/configs/text-embeddings/gce.yaml b/configs/text-embeddings/gce.yaml index 6ca108b60..121dbc0df 100644 --- a/configs/text-embeddings/gce.yaml +++ b/configs/text-embeddings/gce.yaml @@ -20,6 +20,7 @@ worker_node_types: object_store_memory: custom_resources: "accelerator_type:T4": 1 + custom_llm_gpu: 1 min_workers: 0 max_workers: 16 use_spot: true @@ -35,6 +36,7 @@ worker_node_types: object_store_memory: custom_resources: "accelerator_type:L4": 1 + custom_llm_gpu: 1 min_workers: 0 max_workers: 16 use_spot: true @@ -55,6 +57,7 @@ gcp_advanced_configurations_json: object_store_memory: custom_resources: "accelerator_type:V100": 1 + custom_llm_gpu: 1 min_workers: 0 max_workers: 16 use_spot: true @@ -70,6 +73,7 @@ gcp_advanced_configurations_json: object_store_memory: custom_resources: "accelerator_type:A100-40G": 1 + custom_llm_gpu: 1 min_workers: 0 max_workers: 8 gcp_advanced_configurations_json: @@ -88,6 +92,7 @@ gcp_advanced_configurations_json: object_store_memory: custom_resources: "accelerator_type:A100-80G": 1 + custom_llm_gpu: 1 min_workers: 0 max_workers: 4 gcp_advanced_configurations_json: From e8ef6bfef13908f5d9372f63e410a793257197ed Mon Sep 17 00:00:00 2001 From: Scott Lee Date: Sat, 24 Feb 2024 11:29:20 -0800 Subject: [PATCH 2/5] update custom_llm_gpu Signed-off-by: Scott Lee --- configs/text-embeddings/aws.yaml | 12 +++++------- configs/text-embeddings/gce.yaml | 10 +++++----- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/configs/text-embeddings/aws.yaml b/configs/text-embeddings/aws.yaml index 23bf7f049..1d3f00311 100644 --- a/configs/text-embeddings/aws.yaml +++ b/configs/text-embeddings/aws.yaml @@ -23,7 +23,7 @@ worker_node_types: object_store_memory: custom_resources: "accelerator_type:A10G": 1 - custom_llm_gpu: 1 + "custom_llm_gpu": 1 min_workers: 0 max_workers: 16 use_spot: true @@ -39,7 +39,7 @@ worker_node_types: object_store_memory: custom_resources: "accelerator_type:T4": 1 - custom_llm_gpu: 1 + "custom_llm_gpu": 1 min_workers: 0 max_workers: 16 use_spot: true @@ -55,7 +55,7 @@ worker_node_types: object_store_memory: custom_resources: "accelerator_type:V100": 1 - custom_llm_gpu: 1 + "custom_llm_gpu": 1 min_workers: 0 max_workers: 16 use_spot: true @@ -71,7 +71,7 @@ worker_node_types: object_store_memory: custom_resources: "accelerator_type:A100-40G": 1 - custom_llm_gpu: 8 + "custom_llm_gpu": 8 aws_advanced_configurations_json: BlockDeviceMappings: - DeviceName: /dev/sda1 @@ -96,7 +96,7 @@ worker_node_types: object_store_memory: custom_resources: "accelerator_type:A100-80G": 1 - custom_llm_gpu: 8 + "custom_llm_gpu": 8 aws_advanced_configurations_json: BlockDeviceMappings: - DeviceName: /dev/sda1 @@ -115,8 +115,6 @@ aws: TagSpecifications: - ResourceType: instance Tags: - - Key: as-feature-enable-multi-az-serve - Value: "true" - Key: as-feature-multi-zone Value: "true" - Key: as-feature-cross-group-min-count-custom_llm_gpu diff --git a/configs/text-embeddings/gce.yaml b/configs/text-embeddings/gce.yaml index 121dbc0df..20db5a334 100644 --- a/configs/text-embeddings/gce.yaml +++ b/configs/text-embeddings/gce.yaml @@ -20,7 +20,7 @@ worker_node_types: object_store_memory: custom_resources: "accelerator_type:T4": 1 - custom_llm_gpu: 1 + "custom_llm_gpu": 1 min_workers: 0 max_workers: 16 use_spot: true @@ -36,7 +36,7 @@ worker_node_types: object_store_memory: custom_resources: "accelerator_type:L4": 1 - custom_llm_gpu: 1 + "custom_llm_gpu": 1 min_workers: 0 max_workers: 16 use_spot: true @@ -57,7 +57,7 @@ gcp_advanced_configurations_json: object_store_memory: custom_resources: "accelerator_type:V100": 1 - custom_llm_gpu: 1 + "custom_llm_gpu": 1 min_workers: 0 max_workers: 16 use_spot: true @@ -73,7 +73,7 @@ gcp_advanced_configurations_json: object_store_memory: custom_resources: "accelerator_type:A100-40G": 1 - custom_llm_gpu: 1 + "custom_llm_gpu": 1 min_workers: 0 max_workers: 8 gcp_advanced_configurations_json: @@ -92,7 +92,7 @@ gcp_advanced_configurations_json: object_store_memory: custom_resources: "accelerator_type:A100-80G": 1 - custom_llm_gpu: 1 + "custom_llm_gpu": 1 min_workers: 0 max_workers: 4 gcp_advanced_configurations_json: From 2fd67b30f97a1cacc8cc4ec815c1b440c0c5bef2 Mon Sep 17 00:00:00 2001 From: Scott Lee Date: Sun, 25 Feb 2024 10:53:06 -0800 Subject: [PATCH 3/5] use default gpu instead of custom resource config Signed-off-by: Scott Lee --- configs/text-embeddings/aws.yaml | 19 +++++-------------- configs/text-embeddings/gce.yaml | 20 +++++--------------- 2 files changed, 10 insertions(+), 29 deletions(-) diff --git a/configs/text-embeddings/aws.yaml b/configs/text-embeddings/aws.yaml index 1d3f00311..846f02ecc 100644 --- a/configs/text-embeddings/aws.yaml +++ b/configs/text-embeddings/aws.yaml @@ -18,12 +18,11 @@ worker_node_types: instance_type: g5.x2large resources: cpu: - gpu: + gpu: 1 memory: object_store_memory: custom_resources: "accelerator_type:A10G": 1 - "custom_llm_gpu": 1 min_workers: 0 max_workers: 16 use_spot: true @@ -34,12 +33,11 @@ worker_node_types: instance_type: g4dn.2xlarge resources: cpu: - gpu: + gpu: 1 memory: object_store_memory: custom_resources: "accelerator_type:T4": 1 - "custom_llm_gpu": 1 min_workers: 0 max_workers: 16 use_spot: true @@ -50,12 +48,11 @@ worker_node_types: instance_type: p3.2xlarge resources: cpu: - gpu: + gpu: 1 memory: object_store_memory: custom_resources: "accelerator_type:V100": 1 - "custom_llm_gpu": 1 min_workers: 0 max_workers: 16 use_spot: true @@ -66,12 +63,11 @@ worker_node_types: instance_type: p4d.24xlarge resources: cpu: - gpu: + gpu: 8 memory: object_store_memory: custom_resources: "accelerator_type:A100-40G": 1 - "custom_llm_gpu": 8 aws_advanced_configurations_json: BlockDeviceMappings: - DeviceName: /dev/sda1 @@ -91,12 +87,11 @@ worker_node_types: instance_type: p4de.24xlarge resources: cpu: - gpu: + gpu: 8 memory: object_store_memory: custom_resources: "accelerator_type:A100-80G": 1 - "custom_llm_gpu": 8 aws_advanced_configurations_json: BlockDeviceMappings: - DeviceName: /dev/sda1 @@ -117,7 +112,3 @@ aws: Tags: - Key: as-feature-multi-zone Value: "true" - - Key: as-feature-cross-group-min-count-custom_llm_gpu - Value: "2" - - Key: as-feature-cross-group-max-count-custom_llm_gpu - Value: "16" diff --git a/configs/text-embeddings/gce.yaml b/configs/text-embeddings/gce.yaml index 20db5a334..3556b1603 100644 --- a/configs/text-embeddings/gce.yaml +++ b/configs/text-embeddings/gce.yaml @@ -15,12 +15,11 @@ worker_node_types: instance_type: n1-standard-8-nvidia-t4-16gb-1 resources: cpu: - gpu: + gpu: 1 memory: object_store_memory: custom_resources: "accelerator_type:T4": 1 - "custom_llm_gpu": 1 min_workers: 0 max_workers: 16 use_spot: true @@ -31,12 +30,11 @@ worker_node_types: instance_type: g2-standard-8-nvidia-l4-1 resources: cpu: - gpu: + gpu: 1 memory: object_store_memory: custom_resources: "accelerator_type:L4": 1 - "custom_llm_gpu": 1 min_workers: 0 max_workers: 16 use_spot: true @@ -45,19 +43,17 @@ gcp_advanced_configurations_json: instance_properties: labels: as-feature-multi-zone: 'true' - as-feature-enable-multi-az-serve: 'true' # 8 vCPU, 1 NVIDIA Tesla V100 16GB GPU, 30 GiB memory - name: gpu-worker-v100 instance_type: n1-standard-8-nvidia-v100-16gb-1 resources: cpu: - gpu: + gpu: 1 memory: object_store_memory: custom_resources: "accelerator_type:V100": 1 - "custom_llm_gpu": 1 min_workers: 0 max_workers: 16 use_spot: true @@ -68,37 +64,31 @@ gcp_advanced_configurations_json: instance_type: a2-highgpu-1g-nvidia-a100-40gb-1 resources: cpu: - gpu: + gpu: 1 memory: object_store_memory: custom_resources: "accelerator_type:A100-40G": 1 - "custom_llm_gpu": 1 min_workers: 0 max_workers: 8 gcp_advanced_configurations_json: instance_properties: labels: as-feature-multi-zone: 'true' - as-feature-enable-multi-az-serve: 'true' # 12 vCPU, 1 NVIDIA A100 80GB GPU, 170 GiB memory - name: gpu-worker-a100-80g instance_type: a2-ultragpu-1g-nvidia-a100-80gb-1 resources: cpu: - gpu: + gpu: 1 memory: object_store_memory: custom_resources: "accelerator_type:A100-80G": 1 - "custom_llm_gpu": 1 min_workers: 0 max_workers: 4 gcp_advanced_configurations_json: instance_properties: labels: as-feature-multi-zone: 'true' - as-feature-enable-multi-az-serve: 'true' - as-feature-cross-group-min-count-custom_llm_gpu: '2' - as-feature-cross-group-max-count-custom_llm_gpu: '16' From daca9509af1740686b4c3102f89b4a2cfe05a2fb Mon Sep 17 00:00:00 2001 From: Scott Lee Date: Sun, 25 Feb 2024 11:47:29 -0800 Subject: [PATCH 4/5] fix gcp config Signed-off-by: Scott Lee --- configs/text-embeddings/gce.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/configs/text-embeddings/gce.yaml b/configs/text-embeddings/gce.yaml index 3556b1603..417aeff83 100644 --- a/configs/text-embeddings/gce.yaml +++ b/configs/text-embeddings/gce.yaml @@ -39,10 +39,6 @@ worker_node_types: max_workers: 16 use_spot: true fallback_to_ondemand: true -gcp_advanced_configurations_json: - instance_properties: - labels: - as-feature-multi-zone: 'true' # 8 vCPU, 1 NVIDIA Tesla V100 16GB GPU, 30 GiB memory - name: gpu-worker-v100 @@ -71,10 +67,6 @@ gcp_advanced_configurations_json: "accelerator_type:A100-40G": 1 min_workers: 0 max_workers: 8 -gcp_advanced_configurations_json: - instance_properties: - labels: - as-feature-multi-zone: 'true' # 12 vCPU, 1 NVIDIA A100 80GB GPU, 170 GiB memory - name: gpu-worker-a100-80g From a2a45dc5be14f9083783590946d171fa320761c2 Mon Sep 17 00:00:00 2001 From: Scott Lee Date: Sun, 25 Feb 2024 22:23:22 -0800 Subject: [PATCH 5/5] remove gpu specification Signed-off-by: Scott Lee --- configs/text-embeddings/aws.yaml | 10 +++++----- configs/text-embeddings/gce.yaml | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/configs/text-embeddings/aws.yaml b/configs/text-embeddings/aws.yaml index 846f02ecc..2c32ed645 100644 --- a/configs/text-embeddings/aws.yaml +++ b/configs/text-embeddings/aws.yaml @@ -18,7 +18,7 @@ worker_node_types: instance_type: g5.x2large resources: cpu: - gpu: 1 + gpu: memory: object_store_memory: custom_resources: @@ -33,7 +33,7 @@ worker_node_types: instance_type: g4dn.2xlarge resources: cpu: - gpu: 1 + gpu: memory: object_store_memory: custom_resources: @@ -48,7 +48,7 @@ worker_node_types: instance_type: p3.2xlarge resources: cpu: - gpu: 1 + gpu: memory: object_store_memory: custom_resources: @@ -63,7 +63,7 @@ worker_node_types: instance_type: p4d.24xlarge resources: cpu: - gpu: 8 + gpu: memory: object_store_memory: custom_resources: @@ -87,7 +87,7 @@ worker_node_types: instance_type: p4de.24xlarge resources: cpu: - gpu: 8 + gpu: memory: object_store_memory: custom_resources: diff --git a/configs/text-embeddings/gce.yaml b/configs/text-embeddings/gce.yaml index 417aeff83..0dbe637a5 100644 --- a/configs/text-embeddings/gce.yaml +++ b/configs/text-embeddings/gce.yaml @@ -15,7 +15,7 @@ worker_node_types: instance_type: n1-standard-8-nvidia-t4-16gb-1 resources: cpu: - gpu: 1 + gpu: memory: object_store_memory: custom_resources: @@ -30,7 +30,7 @@ worker_node_types: instance_type: g2-standard-8-nvidia-l4-1 resources: cpu: - gpu: 1 + gpu: memory: object_store_memory: custom_resources: @@ -45,7 +45,7 @@ worker_node_types: instance_type: n1-standard-8-nvidia-v100-16gb-1 resources: cpu: - gpu: 1 + gpu: memory: object_store_memory: custom_resources: @@ -60,7 +60,7 @@ worker_node_types: instance_type: a2-highgpu-1g-nvidia-a100-40gb-1 resources: cpu: - gpu: 1 + gpu: memory: object_store_memory: custom_resources: @@ -73,7 +73,7 @@ worker_node_types: instance_type: a2-ultragpu-1g-nvidia-a100-80gb-1 resources: cpu: - gpu: 1 + gpu: memory: object_store_memory: custom_resources: