action.yml

# Copyright 2025 The IREE Authors
#
# Licensed under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

name: IREE Sharktank Models Tests

inputs:
  MATRIX_NAME:
    description: "Identifying name of the matrix variation"
  SKU:
    description: "Type of SKU to test"
  CHIP:
    description: "Type of chip to test"
  VENV_DIR:
    description: "Path to the virtual environment"
  TARGET:
    description: "Type of target to test"
  GPU:
    description: "Type of GPU to test"
  EXTERNAL_TEST_FILE_DIRECTORY:
    description: "The directory of test files and configurations"
  QUALITY_TEST_DIRECTORY:
    description: "The directory of quality tests"
  BENCHMARK_TEST_DIRECTORY:
    description: "The directory of benchmark tests"

runs:
  using: "composite"
  steps:
    - name: Checkout repository 
      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 
      with:
        clean: false # doesn't clean up the venv created before this composite action
    
    - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
      with:
        python-version: "3.11"

    - name: Setting up virtual environment and install Python dependencies
      shell: bash
      run: |
        source ${{ inputs.VENV_DIR }}/bin/activate
        pip install -e ${GITHUB_ACTION_PATH}
        pip install -r ${GITHUB_ACTION_PATH}/requirements-iree.txt
    
    - name: Run sharktank specific model tests
      shell: bash
      if: contains(inputs.MATRIX_NAME, 'model')
      run: |
        source ${{ inputs.VENV_DIR }}/bin/activate
        pytest ${GITHUB_ACTION_PATH} \
            -rA \
            --log-cli-level=info \
            --override-ini=xfail_strict=false \
            -m ${{ inputs.TARGET }} \
            --timeout=120 \
            --durations=0 \
            --ignore=sharktank_models/benchmarks \
            --ignore=sharktank_models/quality_tests
      env:
        HIP_TARGET: ${{ inputs.GPU }}

    - name: Run regression quality tests
      shell: bash
      if: contains(inputs.MATRIX_NAME, 'regression')
      run: |
        source ${{ inputs.VENV_DIR }}/bin/activate
        pytest \
          ${GITHUB_ACTION_PATH}/quality_tests \
          -rpFe \
          --log-cli-level=info \
          --durations=0 \
          --timeout=1200 \
          --capture=no \
          --test-file-directory=${{ inputs.QUALITY_TEST_DIRECTORY }} \
          --external-file-directory=${{ inputs.EXTERNAL_TEST_FILE_DIRECTORY }}
      env:
        ROCM_CHIP: ${{ inputs.CHIP }}
        SKU: ${{ inputs.SKU }}

    # Since the benchmark tests require compiled model files, this step must run after the quality tests
    - name: Run regression benchmark tests
      shell: bash
      if: contains(inputs.MATRIX_NAME, 'regression')
      run: |
        source ${{ inputs.VENV_DIR }}/bin/activate
        pytest \
          ${GITHUB_ACTION_PATH}/benchmarks \
          --log-cli-level=info \
          --retries=7 \
          --timeout=600 \
          --test-file-directory=${{ inputs.BENCHMARK_TEST_DIRECTORY }} \
          --external-file-directory=${{ inputs.EXTERNAL_TEST_FILE_DIRECTORY }}
          
        echo "$(<${GITHUB_ACTION_PATH}/job_summary.md )" >> $GITHUB_STEP_SUMMARY
      env:
        ROCM_CHIP: ${{ inputs.CHIP }}
        SKU: ${{ inputs.SKU }}