[ML][Pipelines] Support queue settings in pipeline job

sdk/ml/azure-ai-ml/azure/ai/ml/entities/_builders/command.py

@@ -554,7 +554,7 @@ def _to_job(self) -> CommandJob:
 
     @classmethod
     def _picked_fields_from_dict_to_rest_object(cls) -> List[str]:
-        return ["resources", "distribution", "limits", "environment_variables"]
+        return ["resources", "distribution", "limits", "environment_variables", "queue_settings"]
 
     def _to_rest_object(self, **kwargs) -> dict:
         rest_obj = super()._to_rest_object(**kwargs)

sdk/ml/azure-ai-ml/tests/pipeline_job/e2etests/test_pipeline_job.py

@@ -1770,6 +1770,14 @@ def test_serverless_compute_in_pipeline(self, client: MLClient, test_path: str)
         pipeline_job = load_job(yaml_path)
         assert_job_cancel(pipeline_job, client)
 
+    def test_pipeline_job_serverless_compute_with_job_tier(self, client: MLClient) -> None:
+        yaml_path = "./tests/test_configs/pipeline_jobs/serverless_compute/pipeline_with_job_tier.yml"
+        pipeline_job = load_job(yaml_path)
+        created_pipeline_job = assert_job_cancel(pipeline_job, client)
+        rest_obj = created_pipeline_job._to_rest_object()
+        assert rest_obj.properties.jobs["spot_job_tier"]["queue_settings"] == {"job_tier": "Spot"}
+        assert rest_obj.properties.jobs["standard_job_tier"]["queue_settings"] == {"job_tier": "Standard"}
+
     @pytest.mark.disable_mock_code_hash
     def test_register_automl_output(self, client: MLClient, randstr: Callable[[str], str]):
         register_pipeline_path = "./tests/test_configs/pipeline_jobs/jobs_with_automl_nodes/automl_regression_with_command_node_register_output.yml"

sdk/ml/azure-ai-ml/tests/pipeline_job/unittests/test_pipeline_job_entity.py

@@ -2034,3 +2034,10 @@ def empty_value_pipeline(integer: int, boolean: bool, number: float, str_param:
         rest_obj = pipeline._to_rest_object()
         expect_resource = {"instance_count": "${{parent.inputs.integer}}", "shm_size": "${{parent.inputs.shm_size}}"}
         assert rest_obj.properties.jobs["component"]["resources"] == expect_resource
+
+    def test_pipeline_job_serverless_compute_with_job_tier(self) -> None:
+        yaml_path = "./tests/test_configs/pipeline_jobs/serverless_compute/pipeline_with_job_tier.yml"
+        pipeline_job = load_job(yaml_path)
+        rest_obj = pipeline_job._to_rest_object()
+        assert rest_obj.properties.jobs["spot_job_tier"]["queue_settings"] == {"job_tier": "Spot"}
+        assert rest_obj.properties.jobs["standard_job_tier"]["queue_settings"] == {"job_tier": "Standard"}

sdk/ml/azure-ai-ml/tests/schedule/e2etests/test_schedule.py

@@ -199,6 +199,7 @@ def test_command_job_schedule(self, client: MLClient, randstr: Callable[[], str]
         rest_schedule_job_dict.pop("status", None)
         rest_schedule_job_dict.pop("services", None)
         rest_schedule_job_dict.pop("resources", None)
+        rest_schedule_job_dict.pop("queue_settings", None)
         schedule_job_dict = schedule._to_dict()["create_job"]
         # pop job name, empty parameters from local dict
         schedule_job_dict.pop("parameters", None)

sdk/ml/azure-ai-ml/tests/test_configs/pipeline_jobs/serverless_compute/all_types/pipeline/pipeline_component.yml

@@ -4,6 +4,4 @@ name: simple_pipeline_component
 jobs:
   node:
     type: command
-    component: ./component.yml
-    resources:
-      instance_type: Standard_D3_v2
+    component: azureml://registries/testFeed/components/simple_command_component/versions/1

sdk/ml/azure-ai-ml/tests/test_configs/pipeline_jobs/serverless_compute/pipeline_with_job_tier.yml

@@ -0,0 +1,21 @@
+$schema: https://azuremlschemas.azureedge.net/latest/pipelineJob.schema.json
+type: pipeline
+display_name: Serverless Compute in Pipeline with Job Tier
+experiment_name: Serverless in Pipeline
+settings:
+  default_compute: azureml:serverless
+jobs:
+  spot_job_tier:
+    command: echo "Hello World"
+    environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu@latest
+    resources:
+      instance_type: Standard_D3_v2
+    queue_settings:
+      job_tier: spot
+  standard_job_tier:
+    command: echo "Hello World"
+    environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu@latest
+    resources:
+      instance_type: Standard_D3_v2
+    queue_settings:
+      job_tier: standard