sdk-cli-perf-monitor-test.yml

# execute tests in src/promptflow/tests/sdk_cli_azure_test, src/promptflow/tests/sdk_cli_test with mark perf_monitor_test

name: sdk-cli-perf-monitor-test


on:
  pull_request:
    paths:
      - src/promptflow/**
      - scripts/building/**
      - .github/workflows/sdk-cli-perf-monitor-test.yml

  schedule:
    - cron: "0 */6 * * *"  # Run every 6 hours

  workflow_dispatch:


env:
  packageSetupType: promptflow_with_extra
  testWorkingDirectory: ${{ github.workspace }}/src/promptflow
  PYTHONPATH: ${{ github.workspace }}/src/promptflow
  IS_IN_CI_PIPELINE: "true"
  PROMPT_FLOW_TEST_MODE: "replay"
  RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording


jobs:
  sdk_cli_perf_monitor_test:
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
    runs-on: ${{ matrix.os }}

    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Set Github Run Id to an Environment Variable
        run: echo "GITHUB_RUN_ID=${{ github.run_id }}" >> $GITHUB_ENV

      - name: Print Github Run Id
        run: echo "Github Run Id is GITHUB_RUN_ID"

      - name: Display and Set Environment Variables
        run: |
          export pyVersion="3.9";
          env | sort >> $GITHUB_OUTPUT
        id: display_env
        shell: bash -el {0}

      - name: Python Setup - ${{ matrix.os }} - Python Version ${{ steps.display_env.outputs.pyVersion }}
        uses: "./.github/actions/step_create_python_environment"
        with:
          pythonVersion: ${{ steps.display_env.outputs.pyVersion }}

      - name: Build wheel
        uses: "./.github/actions/step_sdk_setup"
        with:
          setupType: ${{ env.packageSetupType }}
          scriptPath: ${{ env.testWorkingDirectory }}

      - name: install recording
        run:
          pip install vcrpy
          pip install -e .
        working-directory: ${{ env.RECORD_DIRECTORY }}

      - name: Install tracing
        shell: pwsh
        run: |
          pip uninstall -y promptflow-tracing
          pip install -e ${{ github.workspace }}/src/promptflow-tracing
          pip freeze

      - name: Generate (mock) connections.json
        shell: pwsh
        working-directory: ${{ env.testWorkingDirectory }}
        run: cp ${{ github.workspace }}/src/promptflow/dev-connections.json.example ${{ github.workspace }}/src/promptflow/connections.json

      - name: Run Test
        shell: pwsh
        working-directory: ${{ env.testWorkingDirectory }}
        run: |
          gci env:* | sort-object name
          python "../../scripts/building/run_coverage_tests.py" `
            -p promptflow `
            -t ${{ github.workspace }}/src/promptflow/tests/sdk_cli_azure_test ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test `
            -l eastus `
            -m "perf_monitor_test"