Add CI testing and benchmarking

.github/PULL_REQUEST_TEMPLATE.md

@@ -17,5 +17,5 @@ Describe the intent of your PR here.
 ## PR Merge Checklist
 
 1. [ ] The PR is rebased on the latest `devel` commit and pointing to `devel`.
-2. [ ] Your PR reviewed and approved.
+2. [ ] Your PR has been reviewed and approved.
 3. [ ] All checks are passing.

.github/actions/build/action.yaml

@@ -0,0 +1,31 @@
+# SPDX-FileCopyrightText: Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+name: "Build IRON"
+
+inputs:
+  rebuild_flags:
+    required: false
+    default: ""
+  env_name:
+    required: false
+    default: "ci_env"
+  cmake_extra_flags:
+    required: false
+    default: ""
+
+runs:
+  using: "composite"
+  steps:
+    - name: Build
+      env:
+        HOME: /workspace
+      shell: bash
+      run: |
+        source ${{ inputs.env_name }}/bin/activate
+        set -euxo pipefail
+        if [ -n "${{ inputs.cmake_extra_flags }}" ]; then
+          echo "Using extra CMake flags: ${{ inputs.cmake_extra_flags }}"
+        fi
+        cmake -B build -DCMAKE_BUILD_TYPE=Release -DIRONCLAD_AIE_TARGET=aie2p ${{ inputs.cmake_extra_flags }}
+        cmake --build build -- -j1 VERBOSE=1

.github/actions/commit_results/action.yaml

@@ -0,0 +1,118 @@
+# SPDX-FileCopyrightText: Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+name: "Commit Test Results to a Branch"
+
+inputs:
+  pretty_flags:
+    required: false
+    default: "--metric='Latency (mean)' --metric='Bandwidth (mean)' --metric='GFLOP/s (mean)' -o pretty.md"
+  input_csv:
+    required: false
+    default: "tests_latest.csv"
+  branch:
+    required: false
+    default: "ci"
+  dir:
+    required: true
+
+outputs:
+  results_dir:
+    description: "Absolute path to the directory containing generated test results"
+    value: ${{ steps.merge_results.outputs.results_dir }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Format results and receive previous results from branch
+      id: merge_results
+      env:
+        HOME: /workspace
+      shell: bash
+      run: |
+        export DATE=$(printf '%(%Y_%m_%d_%H_%M_%S)T')
+        export COMMIT_SHA=$(git rev-parse --short HEAD)
+
+        ./ci/scripts/pretty.py --date=${DATE} --commit=${COMMIT_SHA} ${{ inputs.pretty_flags }} ${{ inputs.input_csv }}
+
+        git fetch origin ${{ inputs.branch }}:${{ inputs.branch }}
+        git worktree add ../results ${{ inputs.branch }}
+
+        mkdir -p ../results/${{ inputs.dir }}
+        cp ${{ inputs.input_csv }} ../results/${{ inputs.dir }}/latest.csv
+        cp pretty.md ../results/${{ inputs.dir }}/readme.md
+        touch ../results/${{ inputs.dir }}/all.csv
+
+        RESULTS_DIR=$(realpath ../results/${{ inputs.dir }})
+
+        ./ci/scripts/merge_all.py --all ${RESULTS_DIR}/all.csv --latest ${RESULTS_DIR}/latest.csv
+        ./ci/scripts/pretty_trends.py ${RESULTS_DIR}/all.csv -o ${RESULTS_DIR}/trends.md
+
+        echo "commit_sha=${COMMIT_SHA}" >> $GITHUB_OUTPUT
+        echo "date=${DATE}" >> $GITHUB_OUTPUT
+        echo "results_dir=${RESULTS_DIR}" >> $GITHUB_OUTPUT
+
+    - name: Comment results on PR
+      if: ${{ startsWith(github.event_name, 'pull_request') }}
+      continue-on-error: true  # This step will fail for PRs from forks because of permission issues; ignore
+      uses: actions/github-script@v7
+      with:
+        github-token: ${{ github.token }}
+        script: |
+          const fs = require('fs');
+          const commitSha = '${{ steps.merge_results.outputs.commit_sha }}';
+          const date = '${{ steps.merge_results.outputs.date }}';
+          const server = '${{ github.server_url }}';
+          const repo = '${{ github.repository }}';
+
+          let prettyContent = '';
+          try {
+            prettyContent = fs.readFileSync(`${{ steps.merge_results.outputs.results_dir }}/readme.md`, 'utf8');
+          } catch (error) {
+            prettyContent = 'Pretty output not available.';
+          }
+
+          // Read the trends.md content
+          let trendsContent = '';
+          try {
+            trendsContent = fs.readFileSync(`${{ steps.merge_results.outputs.results_dir }}/trends.md`, 'utf8');
+          } catch (error) {
+            trendsContent = 'Trends not available.';
+          }
+
+          const body = `<details>
+          <summary>📊 Test Results for ${process.env.GITHUB_WORKFLOW}</summary>
+
+          [${commitSha}](${server}/${repo}/commit/${commitSha}) (${date})
+
+          ${prettyContent}
+
+          </details>
+
+          <details>
+          <summary>📈 Trends (vs main branch) for ${process.env.GITHUB_WORKFLOW}</summary>
+
+          [${commitSha}](${server}/${repo}/commit/${commitSha}) (${date})
+
+          ${trendsContent}
+
+          </details>`;
+
+          github.rest.issues.createComment({
+            issue_number: context.issue.number,
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            body: body
+          });
+
+    - name: Push new test results to branch
+      if: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/devel') }}
+      continue-on-error: true  # This step will fail for PRs from forks because of permission issues; ignore
+      shell: bash
+      run: |
+        cd ../results
+        git add ${{ inputs.dir }}/all.csv ${{ inputs.dir }}/latest.csv ${{ inputs.dir }}/readme.md ${{ inputs.dir }}/trends.md
+        git config user.name "github-actions[bot]"
+        git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+        git commit -m "Add test results testing commit \`${{ steps.merge_results.outputs.commit_sha }}\` at ${{ steps.merge_results.outputs.date }}" || echo "No changes to commit"
+        git push origin ${{ inputs.branch }}

.github/actions/prereqs/action.yaml

@@ -0,0 +1,24 @@
+# SPDX-FileCopyrightText: Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+name: "Install IRONCLAD Prerequisites"
+
+inputs:
+  env_name:
+    required: false
+    default: "ci_env"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install Prerequisites
+      env:
+        HOME: /workspace
+      shell: bash
+      run: |
+        set -euxo pipefail
+        python3 -m venv ${{ inputs.env_name }}
+        source ${{ inputs.env_name }}/bin/activate
+        pip install --upgrade pip
+        env MLIR_PYTHON_EXTRAS_SET_VERSION="0.0.8.3" HOST_MLIR_PYTHON_PACKAGE_PREFIX="aie" pip install -r requirements.txt
+        echo "Prerequisites installed into ${{ inputs.env_name }}"

.github/actions/test/action.yaml

@@ -0,0 +1,26 @@
+# SPDX-FileCopyrightText: Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+name: "Run test suite"
+
+inputs:
+  test_flags:
+    required: false
+    default: "--log tests.log --csv-latest tests_latest.csv"
+  env_name:
+    required: false
+    default: "ci_env"
+  build_dir:
+    required: false
+    default: "build"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Test
+      env:
+        HOME: /workspace
+      shell: "bash"
+      run: |
+        source ${{ inputs.env_name }}/bin/activate
+        ./scripts/run_tests.py ${{ inputs.test_flags }} ${{ inputs.build_dir }}

.github/workflows/extensive.yml

@@ -0,0 +1,65 @@
+# SPDX-FileCopyrightText: Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+name: Extensive Benchmark/Test Suite
+
+on:
+  workflow_dispatch:
+  schedule:
+   - cron:  '0 9 * * *'  # 2 AM MT
+  push:
+     branches:
+       - main
+       - devel
+
+# VJUNG: Introduce concurrency groups named with branch refs.
+# VJUNG: If one pushes 5 times on its branch only the most recent commit will trigger CI.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  Test:
+    runs-on: [self-hosted, docker]
+    steps:
+      - name: Pull
+        uses: actions/checkout@v4
+        env:
+          HOME: /workspace
+
+      - name: Prerequisites
+        uses: ./.github/actions/prereqs
+        with:
+          env_name: ci_env
+
+      - name: Build
+        uses: ./.github/actions/build
+        with:
+          env_name: ci_env
+          cmake_extra_flags: "-DEXTENSIVE_TESTING=ON"
+
+      - name: Test
+        id: test
+        continue-on-error: true
+        uses: ./.github/actions/test
+        with:
+          env_name: ci_env
+
+      - name: Commit test results
+        id: commit_results
+        env:
+          HOME: /workspace
+        uses: ./.github/actions/commit_results
+        with:
+          dir: extensive
+
+      - name: Upload test results
+        uses: actions/upload-artifact@v4
+        with:
+          name: results-extensive
+          path: ${{ steps.commit_results.outputs.results_dir }}
+          retention-days: 7
+
+      - name: Mark workflow as failed if tests failed
+        if: steps.test.outcome == 'failure'
+        run: exit 1

.github/workflows/small.yml

@@ -0,0 +1,64 @@
+# SPDX-FileCopyrightText: Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+name: Small Benchmark/Test Suite
+
+on:
+  pull_request: 
+  workflow_dispatch:
+  push:
+     branches:
+       - main
+       - devel
+
+# VJUNG: Introduce concurrency groups named with branch refs.
+# VJUNG: If one pushes 5 times on its branch only the most recent commit will trigger CI.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  Test:
+    runs-on: [self-hosted, docker]
+    steps:
+      - name: Pull
+        uses: actions/checkout@v4
+        env:
+          HOME: /workspace
+
+      - name: Prerequisites
+        uses: ./.github/actions/prereqs
+        with:
+          env_name: ci_env
+
+      - name: Build
+        uses: ./.github/actions/build
+        with:
+          env_name: ci_env
+
+      - name: Test
+        id: test
+        continue-on-error: true
+        uses: ./.github/actions/test
+        with:
+          env_name: ci_env
+
+      - name: Commit test results
+        id: commit_results
+        env:
+          HOME: /workspace
+        uses: ./.github/actions/commit_results
+        with:
+          dir: small
+
+      - name: Upload test results
+        uses: actions/upload-artifact@v4
+        with:
+          name: results-small
+          path: ${{ steps.commit_results.outputs.results_dir }}
+          retention-days: 7
+
+      - name: Mark workflow as failed if tests failed
+        if: steps.test.outcome == 'failure'
+        run: exit 1
+

ci/docker-based/Dockerfile

@@ -0,0 +1,45 @@
+# SPDX-FileCopyrightText: Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# IRON dev container with GitHub Actions runner
+# To run with NPU device passthrough, use:
+# docker run -v /dev/accel/accel0:/dev/accel/accel0 --device-cgroup-rule 'c 261:* rmw' --ulimit memlock=-1:-1 <image>
+
+# To build this docker, run from the repo root:
+# docker build -f ci/docker-based/Dockerfile -t iron-dev-github-runner
+
+FROM ubuntu:24.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    # MLIR-AIE build requirements  
+    build-essential ninja-build python3 python3-dev python3.12-venv unzip \
+    libssl-dev libboost-dev libboost-filesystem-dev libboost-program-options-dev uuid-dev \
+    # Required by MHA kernel
+    llvm-18 \
+    # GitHub Actions runner requirements
+    git jq curl tar \
+ && rm -rf /var/lib/apt/lists/*
+
+ARG USER=iron
+RUN useradd -m -s /bin/bash ${USER}
+
+USER ${USER}
+WORKDIR /workspace
+
+COPY ./requirements.txt /workspace/requirements.txt
+
+# Environment setup 
+# ---
+
+ENV HOME=/workspace 
+
+# Ensure XRT is mounted by the host
+# This errors if it does not exist.
+# copy entrypoint into image and make it executable (ensure file is in the build context)
+COPY --chown=${USER}:${USER} ./ci/docker-based/inside_docker /opt/inside_docker/
+RUN chmod +x /opt/inside_docker/entrypoint.sh /opt/inside_docker/actions_runner.sh
+ENTRYPOINT ["/opt/inside_docker/entrypoint.sh"]
+
+CMD ["/opt/inside_docker/actions_runner.sh"]

ci/docker-based/build_docker.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# SPDX-FileCopyrightText: Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+IMAGE_NAME="iron-public-dev-github-runner"
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "${SCRIPT_DIR}/../.."
+docker build -f ci/docker-based/Dockerfile -t ${IMAGE_NAME} .