[Pipeline] Update deprecated image & add missing notebook to check re…

…adme ignore files (#2978) * Update deprecated image Signed-off-by: Brynn Yin <biyi@microsoft.com> * Fix more images Signed-off-by: Brynn Yin <biyi@microsoft.com> * Use latest version of sklearn Signed-off-by: Brynn Yin <biyi@microsoft.com> * Trigger component register Signed-off-by: Brynn Yin <biyi@microsoft.com> * Update image Signed-off-by: Brynn Yin <biyi@microsoft.com> * Use fixed sklearn Signed-off-by: Brynn Yin <biyi@microsoft.com> --------- Signed-off-by: Brynn Yin <biyi@microsoft.com>
Azure · Feb 2, 2024 · 124d806 · 124d806
1 parent 2d22516
commit 124d806
Show file tree

Hide file tree

Showing 42 changed files with 62 additions and 53 deletions.
diff --git a/cli/jobs/basics/hello-pipeline-abc.yml b/cli/jobs/basics/hello-pipeline-abc.yml
@@ -9,17 +9,17 @@ inputs:
 jobs:
   a:
     command: echo hello ${{inputs.hello_string}}
-    environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu@latest
+    environment: azureml:azureml-sklearn-1.0:23
     inputs:
       hello_string: ${{parent.inputs.hello_string_top_level_input}}
   b:
     command: echo "world" >> ${{outputs.world_output}}/world.txt
-    environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu@latest
+    environment: azureml:azureml-sklearn-1.0:23
     outputs:
       world_output:
   c:
     command: echo ${{inputs.world_input}}/world.txt
-    environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu@latest
+    environment: azureml:azureml-sklearn-1.0:23
     inputs:
       world_input: ${{parent.jobs.b.outputs.world_output}}
 
diff --git a/cli/jobs/pipelines-with-components/basics/1a_e2e_local_components/eval.yml b/cli/jobs/pipelines-with-components/basics/1a_e2e_local_components/eval.yml
@@ -9,7 +9,7 @@ inputs:
 outputs:
   eval_output:
     type: uri_folder
-environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu@latest
+environment: azureml:azureml-sklearn-1.0:23
 code: ./eval_src
 command: >-
   python eval.py 

diff --git a/cli/jobs/pipelines-with-components/basics/1a_e2e_local_components/score.yml b/cli/jobs/pipelines-with-components/basics/1a_e2e_local_components/score.yml
@@ -11,7 +11,7 @@ inputs:
 outputs:
   score_output:
     type: uri_folder
-environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu@latest
+environment: azureml:azureml-sklearn-1.0:23
 code: ./score_src
 command: >-
   python score.py 

diff --git a/cli/jobs/pipelines-with-components/basics/1a_e2e_local_components/train.yml b/cli/jobs/pipelines-with-components/basics/1a_e2e_local_components/train.yml
@@ -18,7 +18,7 @@ outputs:
   model_output:
     type: uri_folder
 code: ./train_src
-environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu@latest
+environment: azureml:azureml-sklearn-1.0:23
 command: >-
   python train.py 
   --training_data ${{inputs.training_data}} 

diff --git a/cli/jobs/pipelines-with-components/basics/1b_e2e_registered_components/eval.yml b/cli/jobs/pipelines-with-components/basics/1b_e2e_registered_components/eval.yml
@@ -1,6 +1,6 @@
 $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
 name: my_eval
-# version: 1b
+#version: 1b
 display_name: Eval
 type: command
 inputs:
@@ -9,7 +9,7 @@ inputs:
 outputs:
   eval_output:
     type: uri_folder
-environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu:1
+environment: azureml:azureml-sklearn-1.0:23
 code: ./eval_src
 command: >-
   python eval.py 

diff --git a/cli/jobs/pipelines-with-components/basics/1b_e2e_registered_components/score.yml b/cli/jobs/pipelines-with-components/basics/1b_e2e_registered_components/score.yml
@@ -1,6 +1,6 @@
 $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
 name: my_score
-# version: 1b
+#version: 1b
 display_name: Score
 type: command
 inputs:
@@ -11,7 +11,7 @@ inputs:
 outputs:
   score_output:
     type: uri_folder
-environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu:1
+environment: azureml:azureml-sklearn-1.0:23
 code: ./score_src
 command: >-
   python score.py 

diff --git a/cli/jobs/pipelines-with-components/basics/1b_e2e_registered_components/train.yml b/cli/jobs/pipelines-with-components/basics/1b_e2e_registered_components/train.yml
@@ -1,7 +1,7 @@
 $schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
 name: my_train
 display_name: Train_upper_case
-# version: 1b
+#version: 1b
 type: command
 inputs:
   training_data: 
@@ -25,7 +25,7 @@ outputs:
   model_output:
     type: uri_folder
 code: ./train_src
-environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu:1
+environment: azureml:azureml-sklearn-1.0:23
 command: >-
   python train.py 
   --training_data ${{inputs.training_data}} 

diff --git a/cli/jobs/pipelines-with-components/basics/5b_env_registered/component.yml b/cli/jobs/pipelines-with-components/basics/5b_env_registered/component.yml
@@ -7,7 +7,7 @@ version: 1
 
 code: ./src
 
-environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu@latest
+environment: azureml:azureml-sklearn-1.0:23
 
 command: >-
   python hello.py
diff --git a/cli/jobs/pipelines-with-components/basics/5c_env_conda_file/component.yml b/cli/jobs/pipelines-with-components/basics/5c_env_conda_file/component.yml
@@ -9,7 +9,7 @@ code: ./src
 
 environment: 
   conda_file: ./conda.yaml
-  image: mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu20.04:20220314.v1
+  image: mcr.microsoft.com/azureml/openmpi4.1.0-cuda11.8-cudnn8-ubuntu22.04:latest
 
 command: >-
   python hello.py
diff --git a/...elines-with-components/image_classification_with_densenet/image_cnn_train/entry.spec.yaml b/...elines-with-components/image_classification_with_densenet/image_cnn_train/entry.spec.yaml
@@ -111,7 +111,7 @@ outputs:
 code: ./
 
 environment:
-  image: mcr.microsoft.com/azureml/openmpi3.1.2-cuda10.2-cudnn7-ubuntu18.04
+  image: mcr.microsoft.com/azureml/openmpi4.1.0-cuda11.8-cudnn8-ubuntu22.04:latest
   conda_file: ./conda.yaml
 
 resources:

diff --git a/...h-components/image_classification_with_densenet/init_image_transformation/entry.spec.yaml b/...h-components/image_classification_with_densenet/init_image_transformation/entry.spec.yaml
@@ -124,4 +124,4 @@ command: >-
   
 environment:
   conda_file: ./conda.yaml
-  image: mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu20.04:latest
+  image: mcr.microsoft.com/azureml/openmpi4.1.0-cuda11.8-cudnn8-ubuntu22.04:latest
diff --git a/cli/jobs/pipelines-with-components/nyc_taxi_data_regression/predict.yml b/cli/jobs/pipelines-with-components/nyc_taxi_data_regression/predict.yml
@@ -12,7 +12,7 @@ inputs:
 outputs:
   predictions:
     type: uri_folder
-environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu@latest
+environment: azureml:azureml-sklearn-1.0:23
 code: ./predict_src
 command: >-
   python predict.py 

diff --git a/cli/jobs/pipelines-with-components/nyc_taxi_data_regression/prep.yml b/cli/jobs/pipelines-with-components/nyc_taxi_data_regression/prep.yml
@@ -11,7 +11,7 @@ outputs:
   prep_data:
     type: uri_folder
 code: ./prep_src
-environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu@latest
+environment: azureml:azureml-sklearn-1.0:23
 command: >-
   python prep.py 
   --raw_data ${{inputs.raw_data}} 

diff --git a/cli/jobs/pipelines-with-components/nyc_taxi_data_regression/score.yml b/cli/jobs/pipelines-with-components/nyc_taxi_data_regression/score.yml
@@ -12,7 +12,7 @@ inputs:
 outputs:
   score_report:
     type: uri_folder
-environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu@latest
+environment: azureml:azureml-sklearn-1.0:23
 code: ./score_src
 command: >-
   python score.py 

diff --git a/cli/jobs/pipelines-with-components/nyc_taxi_data_regression/train.yml b/cli/jobs/pipelines-with-components/nyc_taxi_data_regression/train.yml
@@ -18,7 +18,7 @@ outputs:
   test_data:
     type: uri_folder
 code: ./train_src
-environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu@latest
+environment: azureml:azureml-sklearn-1.0:23
 command: >-
   python train.py 
   --training_data ${{inputs.training_data}} 

diff --git a/cli/jobs/pipelines-with-components/nyc_taxi_data_regression/transform.yml b/cli/jobs/pipelines-with-components/nyc_taxi_data_regression/transform.yml
@@ -11,7 +11,7 @@ outputs:
   transformed_data:
     type: uri_folder
 code: ./transform_src
-environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu@latest
+environment: azureml:azureml-sklearn-1.0:23
 command: >-
   python transform.py 
   --clean_data ${{inputs.clean_data}} 

diff --git a/cli/jobs/pipelines-with-components/pipeline_with_hyperparameter_sweep/predict.yml b/cli/jobs/pipelines-with-components/pipeline_with_hyperparameter_sweep/predict.yml
@@ -13,7 +13,7 @@ outputs:
   predict_result:
     type: uri_folder
 code: ./predict-src
-environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu@latest
+environment: azureml:azureml-sklearn-1.0:23
 command: >-
   python predict.py 
   --model ${{inputs.model}} 

diff --git a/cli/jobs/pipelines-with-components/pipeline_with_hyperparameter_sweep/train.yml b/cli/jobs/pipelines-with-components/pipeline_with_hyperparameter_sweep/train.yml
@@ -59,7 +59,7 @@ outputs:
 
 code: ./train-src
 
-environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu@latest
+environment: azureml:azureml-sklearn-1.0:23
 
 command: >-
   python train.py 

diff --git a/cli/jobs/pipelines/cifar-10/pipeline.yml b/cli/jobs/pipelines/cifar-10/pipeline.yml
@@ -13,7 +13,7 @@ jobs:
       tar -xvzf cifar-10-python.tar.gz -C ${{outputs.cifar}};
       rm cifar-10-python.tar.gz;
     compute: azureml:gpu-cluster
-    environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu@latest
+    environment: azureml:azureml-sklearn-1.0:23
     outputs:
       cifar:
         type: uri_folder
@@ -33,7 +33,7 @@ jobs:
       model_dir:
         type: uri_folder
         mode: upload
-    environment: azureml:AzureML-pytorch-1.9-ubuntu18.04-py37-cuda11-gpu:26
+    environment: azureml:acpt-pytorch-1.13-cuda11.7:27
     compute: azureml:gpu-cluster
     distribution:
       type: pytorch
@@ -47,7 +47,7 @@ jobs:
       --data-dir ${{inputs.cifar}}
       --model-dir ${{inputs.model_dir}}/model
     code: src/eval-model
-    environment: azureml:AzureML-pytorch-1.9-ubuntu18.04-py37-cuda11-gpu:26
+    environment: azureml:acpt-pytorch-1.13-cuda11.7:27
     compute: azureml:gpu-cluster
     distribution:
       type: pytorch

diff --git a/cli/jobs/pipelines/mnist-batch-identification-using-parallel/pipeline.yml b/cli/jobs/pipelines/mnist-batch-identification-using-parallel/pipeline.yml
@@ -73,4 +73,4 @@ jobs:
       inference_result: ${{parent.jobs.predict_digits_mnist.outputs.job_output_file}}
     command: >-
       cat ${{inputs.inference_result}}
-    environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu:1
+    environment: azureml:azureml-sklearn-1.0:23
diff --git a/cli/jobs/pipelines/nyc-taxi/pipeline.yml b/cli/jobs/pipelines/nyc-taxi/pipeline.yml
@@ -17,7 +17,7 @@ jobs:
       prep_data: 
         mode: upload
     code: src/prep
-    environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu@latest
+    environment: azureml:azureml-sklearn-1.0:23
     compute: azureml:cpu-cluster
     command: >-
       python prep.py 
@@ -32,7 +32,7 @@ jobs:
       transformed_data:
         mode: upload
     code: src/transform
-    environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu@latest
+    environment: azureml:azureml-sklearn-1.0:23
     compute: azureml:cpu-cluster
     command: >-
       python transform.py 
@@ -48,7 +48,7 @@ jobs:
       test_data: 
         mode: upload
     code: src/train
-    environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu@latest
+    environment: azureml:azureml-sklearn-1.0:23
     compute: azureml:cpu-cluster
     command: >-
       python train.py 
@@ -64,7 +64,7 @@ jobs:
     outputs:
       predictions: 
     code: src/predict
-    environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu@latest
+    environment: azureml:azureml-sklearn-1.0:23
     compute: azureml:cpu-cluster
     command: >-
       python predict.py 
@@ -80,7 +80,7 @@ jobs:
     outputs:
       score_report: 
     code: src/score
-    environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu@latest
+    environment: azureml:azureml-sklearn-1.0:23
     compute: azureml:cpu-cluster
     command: >-      
       python score.py 

diff --git a/infra/bootstrapping/check-readme.py b/infra/bootstrapping/check-readme.py
@@ -47,7 +47,6 @@ def main():
     try:
         with open(f"{sample_path}/README.md", encoding="utf-8") as readme_file:
             readme_content = readme_file.read()
-
             if all(section in readme_content for section in required_sections):
                 print(
                     f"{sample_path} contains a README.md file with all required sections."

diff --git a/infra/bootstrapping/readme_validation_exclusions.txt b/infra/bootstrapping/readme_validation_exclusions.txt
@@ -142,6 +142,16 @@
 /home/runner/work/azureml-examples/azureml-examples/sdk/python/jobs/pipelines/1h_automl_in_pipeline/automl-image-classification-multilabel-in-pipeline
 /home/runner/work/azureml-examples/azureml-examples/sdk/python/jobs/pipelines/1h_automl_in_pipeline/automl-image-instance-segmentation-in-pipeline
 /home/runner/work/azureml-examples/azureml-examples/sdk/python/jobs/pipelines/1h_automl_in_pipeline/automl-image-object-detection-in-pipeline
+/home/runner/work/azureml-examples/azureml-examples/sdk/python/jobs/pipelines/1l_flow_in_pipeline
+/home/runner/work/azureml-examples/azureml-examples/sdk/python/jobs/pipelines/1i_pipeline_with_spark_nodes
+/home/runner/work/azureml-examples/azureml-examples/sdk/python/jobs/pipelines/1j_pipeline_with_pipeline_component/nyc_taxi_data_regression_with_pipeline_component
+/home/runner/work/azureml-examples/azureml-examples/sdk/python/jobs/pipelines/1j_pipeline_with_pipeline_component/pipeline_with_train_eval_pipeline_component
+/home/runner/work/azureml-examples/azureml-examples/sdk/python/jobs/pipelines/2a_train_mnist_with_tensorflow
+/home/runner/work/azureml-examples/azureml-examples/sdk/python/jobs/pipelines/2b_train_cifar_10_with_pytorch
+/home/runner/work/azureml-examples/azureml-examples/sdk/python/jobs/pipelines/2c_nyc_taxi_data_regression
+/home/runner/work/azureml-examples/azureml-examples/sdk/python/jobs/pipelines/2d_image_classification_with_densenet
+/home/runner/work/azureml-examples/azureml-examples/sdk/python/jobs/pipelines/2e_image_classification_keras_minist_convnet
+/home/runner/work/azureml-examples/azureml-examples/sdk/python/schedules
 /home/runner/work/azureml-examples/azureml-examples/cli/jobs/pipelines/automl/cli-automl-forecasting-demand-with-pipeline-components/cli-automl-forecasting-demand-many-models
 /home/runner/work/azureml-examples/azureml-examples/sdk/python/jobs/automl-standalone-jobs/automl-image-classification-multilabel-task-fridge-items
 /home/runner/work/azureml-examples/azureml-examples/sdk/python/assets/data

diff --git a/sdk/python/jobs/pipelines/1a_pipeline_with_components_from_yaml/eval_model.yml b/sdk/python/jobs/pipelines/1a_pipeline_with_components_from_yaml/eval_model.yml
@@ -11,7 +11,7 @@ inputs:
 outputs:
   eval_output:
     type: uri_folder
-environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu:1
+environment: azureml:azureml-sklearn-1.0:23
 code: ./eval_src
 command: >-
   python eval.py 

diff --git a/sdk/python/jobs/pipelines/1a_pipeline_with_components_from_yaml/score_data.yml b/sdk/python/jobs/pipelines/1a_pipeline_with_components_from_yaml/score_data.yml
@@ -14,7 +14,7 @@ inputs:
 outputs:
   score_output:
     type: uri_folder
-environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu:1
+environment: azureml:azureml-sklearn-1.0:23
 code: ./score_src
 command: >-
   python score.py 

diff --git a/sdk/python/jobs/pipelines/1a_pipeline_with_components_from_yaml/train_model.yml b/sdk/python/jobs/pipelines/1a_pipeline_with_components_from_yaml/train_model.yml
@@ -21,7 +21,7 @@ outputs:
   model_output:
     type: uri_folder
 code: ./train_src
-environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu:1
+environment: azureml:azureml-sklearn-1.0:23
 command: >-
   python train.py 
   --training_data ${{inputs.training_data}} 

diff --git a/sdk/python/jobs/pipelines/1c_pipeline_with_hyperparameter_sweep/predict.yml b/sdk/python/jobs/pipelines/1c_pipeline_with_hyperparameter_sweep/predict.yml
@@ -13,7 +13,7 @@ outputs:
   predict_result:
     type: uri_folder
 code: ./predict-src
-environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu@latest
+environment: azureml:azureml-sklearn-1.0:23
 command: >-
   python predict.py 
   --model ${{inputs.model}} 

diff --git a/sdk/python/jobs/pipelines/1c_pipeline_with_hyperparameter_sweep/train.yml b/sdk/python/jobs/pipelines/1c_pipeline_with_hyperparameter_sweep/train.yml
@@ -59,7 +59,7 @@ outputs:
 
 code: ./train-src
 
-environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu@latest
+environment: azureml:azureml-sklearn-1.0:23
 
 command: >-
   python train.py 

diff --git a/...ipelines/1d_pipeline_with_non_python_components/pipeline_with_non_python_components.ipynb b/...ipelines/1d_pipeline_with_non_python_components/pipeline_with_non_python_components.ipynb
@@ -113,7 +113,7 @@
     "download_url = \"https://azuremlexamples.blob.core.windows.net/datasets/iris.csv\"\n",
     "file_name = \"data.csv\"\n",
     "custom_path = \"azureml://datastores/workspaceblobstore/paths/custom_path/${{name}}/\"\n",
-    "environment = \"AzureML-sklearn-1.0-ubuntu20.04-py38-cpu:1\"\n",
+    "environment = \"azureml:azureml-sklearn-1.0@latest\"\n",
     "# 1. Create a command component to download a input data\n",
     "download_data = command(\n",
     "    name=\"download-input\",\n",

diff --git a/sdk/python/jobs/pipelines/1e_pipeline_with_registered_components/score_data.yml b/sdk/python/jobs/pipelines/1e_pipeline_with_registered_components/score_data.yml
@@ -14,7 +14,7 @@ inputs:
 outputs:
   score_output:
     type: uri_folder
-environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu:1
+environment: azureml:azureml-sklearn-1.0:23
 code: ./score_src
 command: >-
   python score.py 

diff --git a/sdk/python/jobs/pipelines/1e_pipeline_with_registered_components/train_model.yml b/sdk/python/jobs/pipelines/1e_pipeline_with_registered_components/train_model.yml
@@ -4,7 +4,7 @@ type: command
 name: train_model
 display_name: Train Model
 description: A dummy training component
-version: 0.0.2
+version: 0.0.3
 inputs:
   training_data: 
     type: uri_folder
@@ -21,7 +21,7 @@ outputs:
   model_output:
     type: uri_folder
 code: ./train_src
-environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu:1
+environment: azureml:azureml-sklearn-1.0:23
 command: >-
   python train.py 
   --training_data ${{inputs.training_data}} 

diff --git a/sdk/python/jobs/pipelines/1g_pipeline_with_parallel_nodes/pipeline_with_parallel_nodes.ipynb b/sdk/python/jobs/pipelines/1g_pipeline_with_parallel_nodes/pipeline_with_parallel_nodes.ipynb
@@ -169,7 +169,7 @@
     "        code=\"./src\",\n",
     "        entry_script=\"file_batch_inference.py\",\n",
     "        program_arguments=\"--job_output_path ${{outputs.job_output_path}}\",\n",
-    "        environment=\"azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu:1\",\n",
+    "        environment=\"azureml:azureml-sklearn-1.0@latest\",\n",
     "    ),\n",
     ")"
    ]