Merge branch 'Azure:main' into main

.github/CODEOWNERS

@@ -8,6 +8,7 @@
 /assets/models/ @azure/aml-training
 /assets/common/ @azure/aml-training
 /assets/model_monitoring/ @Azure/aml-model-monitoring
-/assets/large_language_models/ @azure/aml-training @azure/aml-llm
+/assets/large_language_models/ @azure/aml-training @azure/aml-llm @azure/azureml-rag
 /assets/evaluation_results/ @azure/aml-benchmarking @azure/aml-llm
 /assets/batch_score/ @Azure/aml-batch-inferencing
+/assets/promptflow/ @Azure/promptflow-builtin @azure/aml-assets

.github/ISSUE_TEMPLATE/ACRImageBuildDeprecationFeedback.yml

@@ -0,0 +1,31 @@
+name: ACR Image Build Deprecation Feedback
+description: File a feedback report detailing your experience with the deprecation process.
+labels: ["ACRImageBuildDeprecation"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thank you for providing feedback on the ACR image build deprecation.
+  - type: markdown
+    attributes:
+      value: |
+        # Describe your experience
+  - type: textarea
+    id: feedback
+    attributes:
+      label: Feedback
+      description: What feedback do you have about the deprecation process? Was the documentation clear and accessible?
+    validations:
+      required: true
+  - type: textarea
+    id: suggestions
+    attributes:
+      label: Suggestions
+      description: What part of the process do you wish was different?
+    validations:
+      required: true
+  - type: textarea
+    id: more-info
+    attributes:
+      label: Addition information
+      description: If there is any additional information you'd like to share, please include it below.

.github/workflows/assets-validation.yaml

@@ -27,10 +27,15 @@ defaults:
   run:
     shell: bash
 
+permissions:
+  # Required for OIDC login to Azure
+  id-token: write
+
 jobs:
   validate:
     name: Validate
     runs-on: ubuntu-latest
+    environment: Testing
 
     steps:
       - name: Clone branch
@@ -47,12 +52,21 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: '>=3.8'
-
+
+      - name: Log in to Azure
+        uses: azure/login@v1
+        with:
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+
       - name: Install dependencies
         run: pip install -e $scripts_azureml_assets_dir
-      
+
       - name: Validate assets
-        run: python -u $scripts_assets_dir/validate_assets.py -i "${{ github.event.inputs.asset_dirs || env.default_asset_dirs }}" -a $asset_config_filename -c "${{ steps.changed-files.outputs.all_modified_files }}" -n -I -C -b
+        run: python -u $scripts_assets_dir/validate_assets.py -i "${{ github.event.inputs.asset_dirs || env.default_asset_dirs }}" -a $asset_config_filename -c "${{ steps.changed-files.outputs.all_modified_files }}" -n -I -C -b -t -e
+        env:
+          SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
 
       - name: Validate source tree
         run: python -u $scripts_assets_dir/validate_tree.py -i "${{ github.event.inputs.asset_dirs || env.default_asset_dirs }}"

.github/workflows/batch-score-ci.yaml

@@ -7,7 +7,8 @@ on:
     paths:
       - assets/batch_score/**
       - .github/workflows/batch-score-ci.yml
-
+  workflow_dispatch:
+
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -55,13 +56,20 @@ jobs:
       - name: Install dependencies
         run: pip install -r ${{ env.testsRootPath }}/requirements.txt
 
-      - name: Test asset
+      - name: Run unit tests
         run: python -m pytest --junitxml=${{ env.pytest_report_folder }}/${{ env.pytest_report_file }} ${{ env.testsRootPath }} --strict-markers -v -s -m "unit" -o log_level=DEBUG -n 8
         env:
           SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
           TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
           RESOURCE_GROUP: ${{ env.resource_group }}
           WORKSPACE_NAME: ${{ env.workspace }}
+      - name: Run e2e tests
+        run: python -m pytest --junitxml=${{ env.pytest_report_folder }}/${{ env.pytest_report_file }} ${{ env.testsRootPath }} --strict-markers -v -s -m "smoke" -o log_level=DEBUG -n 8
+        env:
+          SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
+          RESOURCE_GROUP: ${{ env.resource_group }}
+          WORKSPACE_NAME: ${{ env.workspace }}
       - name: Upload test results
         uses: actions/upload-artifact@v3
         if: always()

.github/workflows/promptflow-ci.yml

@@ -0,0 +1,114 @@
+name: Promptflow CI
+on:
+  # Triggers the Promptflow CI on pull request targeting the main branch
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - assets/promptflow/models/**
+      - .github/workflows/promptflow-ci.yml
+      - scripts/promptflow-ci/**
+  pull_request_target:
+    branches:
+      - main
+    paths:
+      - assets/promptflow/models/**
+      - .github/workflows/promptflow-ci.yml
+      - scripts/promptflow-ci/**
+    types:
+      - opened
+      - labeled
+      - synchronize
+      - reopened
+
+env:
+  PROMPTFLOW_DIR: "assets/promptflow/models"
+
+permissions:
+  # Required for OIDC login to Azure
+  id-token: write
+  # Required to clone repo
+  contents: read
+
+jobs:
+  check-execution-context:
+    uses: Azure/azureml-assets/.github/workflows/check-execution-context.yaml@main
+
+  check_spec_yaml:
+    runs-on: ubuntu-latest
+    if: fromJSON(needs.check-execution-context.outputs.continue)
+    needs: check-execution-context
+    name: Check spec.yaml fields
+    timeout-minutes: 45
+    steps:
+    - name: Clone branch
+      uses: Azure/azureml-assets/.github/actions/clone-repo@main
+      with:
+        forked-pr: ${{ needs.check-execution-context.outputs.forked_pr }}
+        fetch-depth: 2
+
+    - name: Set up Python 3.9 environment
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.9"
+
+    - name: Check spec yaml
+      run: |-
+        pip install PyYAML
+        python scripts/promptflow-ci/check_spec_yaml.py --models_dir ${{ env.PROMPTFLOW_DIR }}
+
+  run_promptflow_ci_job:
+    runs-on: ubuntu-latest
+    if: fromJSON(needs.check-execution-context.outputs.continue)
+    needs: check-execution-context
+    name: Flow tests
+    environment: Testing
+    timeout-minutes: 60
+
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - name: Clone branch
+        uses: Azure/azureml-assets/.github/actions/clone-repo@main
+        with:
+          forked-pr: ${{ needs.check-execution-context.outputs.forked_pr }}
+          fetch-depth: 2
+
+      - name: Use Node.js 18.x  
+        uses: actions/setup-node@v3
+        with:
+          node-version: "18.x"
+
+      - name: Install cspell
+        run: npm install -g cspell
+
+      - name: Set up Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - run: |
+          python -m pip install --upgrade pip
+          pip install -r scripts/promptflow-ci/requirement.txt
+
+      - name: Azure login
+        uses: azure/login@v1
+        with:
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_PROMPT_FLOW_SUBSCRIPTION_ID  }}
+
+      - name: Validate prompt flows
+        run: |
+          python scripts/promptflow-ci/promptflow_ci.py
+
+      - name: Run cspell for typo check
+        working-directory: ${{ env.PROMPTFLOW_DIR }}
+        run: |-
+          cspell lint "**" > cspell_check.log || true
+          if [ -s cspell_check.log ]; then
+            echo -e "\e[91mNote: If the detected words are expected, add them into "assets/promptflow/.cspell.json"\e[0m" 
+            echo -e "====================== Summary of cspell errors ======================"
+            echo -e "\e[31m$(cat cspell_check.log)\e[0m" 
+            exit 1
+          fi
+          rm -rf cspell_check.log

assets/aml-benchmark/components/batch-benchmark-inference/asset.yaml

@@ -4,5 +4,5 @@ categories: ["Benchmarking"]
 test:
   pytest:
     enabled: true
-    pip_requirements: ../../requirements.txt
+    conda_environment: ../../dev_conda_env.yaml
     tests_dir: ../../tests

assets/aml-benchmark/components/batch-benchmark-inference/spec.yaml

@@ -4,7 +4,7 @@ type: pipeline
 name: batch_benchmark_inference
 display_name: Batch Benchmark Inference
 description: Components for batch endpoint inference
-version: 0.0.3
+version: 0.0.4
 
 inputs:
   input_dataset:
@@ -69,6 +69,10 @@ inputs:
     type: string
     optional: True
     description: The label column name.
+  additional_columns:
+    type: string
+    optional: True
+    description: The name(s) for additional columns that could be helpful to calculate some metrics, separated by comma (",").
   n_samples:
     type: integer
     description: The number of top samples send to endpoint. When performance test is enabled, this will be the number of repeated samples send to the endpoint.
@@ -145,7 +149,7 @@ jobs:
   # Preparer
   batch_inference_preparer: 
     type: command
-    component: azureml:batch_inference_preparer:0.0.3
+    component: azureml:batch_inference_preparer:0.0.5
     inputs:
       input_dataset: ${{parent.inputs.input_dataset}}
       model_type: ${{parent.inputs.model_type}}
@@ -154,6 +158,7 @@ jobs:
       n_samples: ${{parent.inputs.n_samples}}
       endpoint_url: ${{parent.inputs.endpoint_url}}
       label_column_name: ${{parent.inputs.label_column_name}}
+      additional_columns: ${{parent.inputs.additional_columns}}
     outputs:
       formatted_data:
         type: mltable
@@ -162,7 +167,7 @@ jobs:
   # Inference
   endpoint_batch_score:
     type: parallel
-    component: azureml:batch_benchmark_score:0.0.3
+    component: azureml:batch_benchmark_score:0.0.4
     inputs:
       model_type: ${{parent.inputs.model_type}}
       online_endpoint_url: ${{parent.inputs.endpoint_url}}
@@ -194,11 +199,12 @@ jobs:
   # Reformat
   batch_output_formatter: 
     type: command
-    component: azureml:batch_output_formatter:0.0.3
+    component: azureml:batch_output_formatter:0.0.5
     inputs:
       model_type: ${{parent.inputs.model_type}}
       batch_inference_output: ${{parent.jobs.endpoint_batch_score.outputs.mini_batch_results_out_directory}}
       label_column_name: ${{parent.inputs.label_column_name}}
+      additional_columns: ${{parent.inputs.additional_columns}}
       ground_truth_input: ${{parent.jobs.batch_inference_preparer.outputs.ground_truth_metadata}}
       fallback_value: ${{parent.inputs.fallback_value}}
       handle_response_failure: ${{parent.inputs.handle_response_failure}}

assets/aml-benchmark/components/batch-benchmark-score/asset.yaml

@@ -4,5 +4,5 @@ categories: ["Benchmarking"]
 test:
   pytest:
     enabled: true
-    pip_requirements: ../../requirements.txt
+    conda_environment: ../../dev_conda_env.yaml
     tests_dir: ../../tests

assets/aml-benchmark/components/batch-inference-preparer/asset.yaml

@@ -4,5 +4,5 @@ categories: ["Benchmarking"]
 test:
   pytest:
     enabled: true
-    pip_requirements: ../../requirements.txt
+    conda_environment: ../../dev_conda_env.yaml
     tests_dir: ../../tests

assets/aml-benchmark/components/batch-inference-preparer/spec.yaml

@@ -4,7 +4,7 @@ type: command
 name: batch_inference_preparer
 display_name: Batch Inference Preparer
 description: Prepare the jsonl file and endpoint for batch inference component.
-version: 0.0.4
+version: 0.0.5
 
 inputs:
   input_dataset: 
@@ -39,6 +39,10 @@ inputs:
     type: string
     optional: True
     description: The label column name.
+  additional_columns:
+    type: string
+    optional: True
+    description: Name(s) of additional column(s) that could be useful to compute metrics, separated by comma (",").
   is_performance_test:
     type: boolean
     default: False
@@ -72,3 +76,4 @@ command: >-
   $[[--n_samples ${{inputs.n_samples}}]]
   $[[--endpoint_url ${{inputs.endpoint_url}}]]
   $[[--label_key ${{inputs.label_column_name}}]]
+  $[[--additional_columns ${{inputs.additional_columns}}]]

assets/aml-benchmark/components/batch-output-formatter/asset.yaml

@@ -4,5 +4,5 @@ categories: ["Benchmarking"]
 test:
   pytest:
     enabled: true
-    pip_requirements: ../../requirements.txt
+    conda_environment: ../../dev_conda_env.yaml
     tests_dir: ../../tests

assets/aml-benchmark/components/batch-output-formatter/spec.yaml

@@ -1,5 +1,5 @@
 name: batch_output_formatter
-version: 0.0.4
+version: 0.0.5
 display_name: Batch Output Formatter
 is_deterministic: True
 type: command
@@ -17,6 +17,10 @@ inputs:
     type: string
     optional: True
     description: The label column name.
+  additional_columns:
+    type: string
+    optional: True
+    description: Name(s) of additional column(s) that could be useful to compute metrics, separated by comma (",").
   endpoint_url:
     type: string
     optional: True
@@ -63,6 +67,7 @@ command: >-
   --predict_ground_truth_data ${{outputs.ground_truth}}
   $[[--endpoint_url ${{inputs.endpoint_url}}]]
   $[[--label_key ${{inputs.label_column_name}}]]
+  $[[--additional_columns ${{inputs.additional_columns}}]]
   --handle_response_failure ${{inputs.handle_response_failure}}
   --is_performance_test ${{inputs.is_performance_test}}
   $[[--fallback_value ${{inputs.fallback_value}}]]

assets/aml-benchmark/components/batch-resource-manager/asset.yaml

@@ -4,5 +4,5 @@ categories: ["Benchmarking"]
 test:
   pytest:
     enabled: true
-    pip_requirements: ../../requirements.txt
+    conda_environment: ../../dev_conda_env.yaml
     tests_dir: ../../tests

assets/aml-benchmark/components/batch-resource-manager/spec.yaml

@@ -4,7 +4,7 @@ is_deterministic: False
 name: batch_resource_manager
 display_name: Batch Inference Resource Manager
 description: Resource Manager for batch inference.
-version: 0.0.2
+version: 0.0.3
 
 inputs:
   wait_input:

assets/aml-benchmark/components/batch_benchmark_inference_claude/asset.yaml

@@ -4,5 +4,5 @@ categories: ["Benchmarking"]
 test:
   pytest:
     enabled: true
-    pip_requirements: ../../requirements.txt
+    conda_environment: ../../dev_conda_env.yaml
     tests_dir: ../../tests