test-integration.yml

name: Run integration tests

on:
  create:

  pull_request:
    paths:
      - ".github/workflows/test-integration.yml"
      - ".github/workflows/testdata/**"
      - "poetry.lock"
      - "pyproject.toml"
      - "action.yml"
      - "compilesketches/**"

  push:
    paths:
      - ".github/workflows/test-integration.yml"
      - ".github/workflows/testdata/**"
      - "poetry.lock"
      - "pyproject.toml"
      - "action.yml"
      - "compilesketches/**"

  repository_dispatch:

  schedule:
    # Run periodically to catch breakage caused by external changes.
    - cron: "0 12 * * THU"

  workflow_dispatch:

env:
  SKETCHES_REPORTS_PATH: sketches-reports
  TESTDATA_PLATFORMS_PATH: .github/workflows/testdata/platforms
  TESTDATA_SKETCHES_PATH: .github/workflows/testdata/sketches
  TESTDATA_REPORTS_PATH: .github/workflows/testdata/reports

jobs:
  run-determination:
    runs-on: ubuntu-latest
    outputs:
      result: ${{ steps.determination.outputs.result }}
    steps:
      - name: Determine if the rest of the workflow should run
        id: determination
        run: |
          RELEASE_BRANCH_REGEX="^refs/heads/v[0-9]+$"
          # The `create` event trigger doesn't support `branches` filters, so it's necessary to use Bash instead.
          if [[
            "${{ github.event_name }}" != "create" ||
            "${{ github.ref }}" =~ $RELEASE_BRANCH_REGEX
          ]]; then
            # Run the other jobs.
            RESULT="true"
          else
            # There is no need to run the other jobs.
            RESULT="false"
          fi

          echo "result=$RESULT" >> $GITHUB_OUTPUT

  default-inputs:
    needs: run-determination
    if: needs.run-determination.outputs.result == 'true'
    runs-on: ubuntu-latest

    steps:
      - name: Checkout Servo library
        uses: actions/checkout@v4
        with:
          repository: arduino-libraries/servo
          ref: 1.1.7

      - name: Checkout local repo
        uses: actions/checkout@v4
        with:
          # Must be checked out to a subfolder to not interfere with the checked out library under test
          path: extras/compile-sketches

      - name: Run action with default input values
        # Use action from local path
        uses: ./extras/compile-sketches

  all-inputs:
    needs: run-determination
    if: needs.run-determination.outputs.result == 'true'
    runs-on: ubuntu-latest

    strategy:
      fail-fast: false

      matrix:
        board:
          - fqbn: arduino:avr:uno
            platforms: |
              - name: arduino:avr
                version: 1.8.3
            libraries: |
              - name: Servo
                version: 1.1.7
          # Board that requires Boards Manager URL
          - fqbn: esp8266:esp8266:huzzah
            platforms: |
              - name: esp8266:esp8266
                source-url: https://arduino.esp8266.com/stable/package_esp8266com_index.json
                version: 2.7.4
            libraries: |
              # The official Servo library is not compatible with ESP8266, but that platform has a bundled Servo lib
              # so there are no library dependencies
              []

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Run action
        # Use action from local path
        uses: ./
        with:
          cli-version: 0.15.1
          github-token: ${{ secrets.GITHUB_TOKEN }}
          platforms: ${{ matrix.board.platforms }}
          fqbn: ${{ matrix.board.fqbn }}
          libraries: ${{ matrix.board.libraries }}
          sketch-paths: |
            - ${{ env.TESTDATA_SKETCHES_PATH }}/BareMinimum
            - ${{ env.TESTDATA_SKETCHES_PATH }}/ServoLibrary
            - ${{ env.TESTDATA_SKETCHES_PATH }}/TestCompileFlags
          cli-compile-flags: |
            - --build-property
            - compiler.cpp.extra_flags="-DSTRING_MACRO="hello world"" -DINT_MACRO=2 -DFLAG_MACRO
            - --export-binaries
          enable-deltas-report: true
          enable-warnings-report: true
          sketches-report-path: ${{ env.SKETCHES_REPORTS_PATH }}
          verbose: true

      - name: Verify --export-binaries flag was used by compilation command
        run: |
          [ -d ${{ env.TESTDATA_SKETCHES_PATH }}/BareMinimum/build ]

      - name: Set report artifact name
        id: report-artifact-name
        run: |
          echo "report-artifact-name=${{ github.job }}" >> "$GITHUB_OUTPUT"

      - name: Save sketches report as workflow artifact
        uses: actions/upload-artifact@v3
        with:
          path: sketches-reports
          name: ${{ steps.report-artifact-name.outputs.report-artifact-name }}

    outputs:
      report-artifact-name: ${{ steps.report-artifact-name.outputs.report-artifact-name }}

  multiple-steps:
    name: multiple-steps (${{ matrix.board.source-type }})
    needs: run-determination
    if: needs.run-determination.outputs.result == 'true'
    runs-on: ubuntu-latest

    strategy:
      fail-fast: false

      matrix:
        board:
          - source-type: manager
            fqbn: arduino:avr:uno
            platforms: |
              - name: arduino:avr
                version: 1.8.3
            libraries: |
              - name: Servo
                version: 1.1.7
          - source-type: path
            fqbn: arduino:avr:uno
            platforms: |
              - name: arduino:avr
                version: 1.8.3
              - source-path: extras/ArduinoCore-avr
                name: arduino:avr
            libraries: |
              - source-path: ./
                name: Servo
          - source-type: repo
            fqbn: arduino:avr:uno
            platforms: |
              - name: arduino:avr
                version: 1.8.3
              - source-url: https://github.com/arduino/ArduinoCore-avr.git
                name: arduino:avr
                version: 1.8.3
            libraries: |
              - source-url: https://github.com/arduino-libraries/Servo.git
                name: Servo
                version: 1.1.7
          - source-type: archive
            fqbn: arduino:avr:uno
            platforms: |
              - name: arduino:avr
                version: 1.8.3
              - source-url: https://github.com/arduino/ArduinoCore-avr/archive/refs/tags/1.8.3.zip
                name: arduino:avr
            libraries: |
              - source-url: https://github.com/arduino-libraries/Servo/archive/refs/tags/1.1.7.zip
                name: Servo

    steps:
      - name: Checkout library
        uses: actions/checkout@v4
        with:
          repository: arduino-libraries/Servo
          ref: 1.1.7

      - name: Checkout platform
        if: matrix.board.source-type == 'path'
        uses: actions/checkout@v4
        with:
          repository: arduino/ArduinoCore-avr
          ref: 1.8.3
          path: extras/ArduinoCore-avr

      - name: Checkout local repo
        uses: actions/checkout@v4
        with:
          path: extras/compile-sketches

      - name: Run action
        # Use action from local path
        uses: ./extras/compile-sketches
        with:
          platforms: ${{ matrix.board.platforms }}
          fqbn: ${{ matrix.board.fqbn }}
          libraries: ${{ matrix.board.libraries }}
          sketch-paths: |
            - examples/Sweep

      - name: Run action again
        uses: ./extras/compile-sketches
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          platforms: ${{ matrix.board.platforms }}
          fqbn: ${{ matrix.board.fqbn }}
          libraries: ${{ matrix.board.libraries }}
          sketch-paths: |
            - examples/Sweep

  python-package-dependency:
    needs: run-determination
    if: needs.run-determination.outputs.result == 'true'
    runs-on: ubuntu-latest

    steps:
      - name: Checkout local repo
        uses: actions/checkout@v4

      - name: Install Python package dependency
        run: |
          pip install \
            --ignore-installed \
            --user \
            cowsay

      - name: Run action with board that has external Python package dependency
        # Use action from local path
        uses: ./
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          platforms: |
            - name: arduino:avr
            - source-path: ${{ env.TESTDATA_PLATFORMS_PATH }}/PythonPackageDependent
              name: PythonPackageDependent:avr
          fqbn: PythonPackageDependent:avr:package_dependent
          libraries: |
            []
          sketch-paths: |
            - ${{ env.TESTDATA_SKETCHES_PATH }}/BareMinimum

  # Targeted testing for ESP32 boards platform support.
  pyserial-dependency:
    needs: run-determination
    if: needs.run-determination.outputs.result == 'true'
    runs-on: ubuntu-latest

    steps:
      - name: Checkout local repo
        uses: actions/checkout@v4

      - name: Install pyserial
        run: |
          # Use of pip3 and omission of recommended flags done to reproduce established use pattern:
          # https://github.com/arduino-libraries/ArduinoIoTCloud/blob/1.11.0/.github/workflows/compile-examples.yml#L206
          pip3 install pyserial

      - name: Run action with board that has pyserial dependency
        # Use action from local path
        uses: ./
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          platforms: |
            - name: arduino:avr
            - source-path: ${{ env.TESTDATA_PLATFORMS_PATH }}/PyserialDependent
              name: PyserialDependent:avr
          fqbn: PyserialDependent:avr:pyserial_dependent
          libraries: |
            []
          sketch-paths: |
            - ${{ env.TESTDATA_SKETCHES_PATH }}/BareMinimum

  check-sketches-reports:
    needs:
      - run-determination
      - all-inputs
    if: needs.run-determination.outputs.result == 'true'
    runs-on: ubuntu-latest

    steps:
      # Checkout is needed to get the golden reports
      - name: Checkout local repo
        uses: actions/checkout@v4

      - name: Download sketches reports artifact
        uses: actions/download-artifact@v3
        with:
          name: ${{ needs.all-inputs.outputs.report-artifact-name }}
          path: ${{ env.SKETCHES_REPORTS_PATH }}

      - name: Compare generated sketches report to golden report
        run: |
          # The commit hash changes on every push, so it can't be compared against the golden report
          if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then
            COMMIT_HASH="${{ github.event.pull_request.head.sha }}"
          else
            COMMIT_HASH="$(git rev-parse HEAD)"
          fi

          set +o errexit  # Unset errexit so the results of all report comparisons are shown before exiting
          EXIT_STATUS=0
          while read -r reportPath; do
            echo "Sketches report $reportPath matches expected:"

            (
              # Check static report components against golden reports
              jq \
                --null-input \
                --exit-status \
                --slurpfile \
                  generated \
                  "$reportPath" \
                --slurpfile \
                  golden \
                  "${{ env.TESTDATA_REPORTS_PATH }}/${{ needs.all-inputs.outputs.report-artifact-name }}/$(basename "$reportPath")" \
                '($generated|.[0].boards) == ($golden|.[0].boards)'
            ) && (
              # Check the commit_hash value
              jq \
                --null-input \
                --exit-status \
                --slurpfile generated "$reportPath" \
                --arg commit_hash "$COMMIT_HASH" \
                '$generated|.[0].commit_hash == $commit_hash|.'
            ) && (
              # Check the commit_url value
              jq \
                --null-input \
                --exit-status \
                --slurpfile generated "$reportPath" \
                --arg commit_url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/commit/${COMMIT_HASH}" \
                '$generated|.[0].commit_url == $commit_url|.'
            )

            if [[ $? -ne 0 ]]; then
              echo "::error::Sketches report $reportPath mismatch."
              EXIT_STATUS=1
            fi
          done <<<"$(find "${{ env.SKETCHES_REPORTS_PATH }}" -type f -and -name '*.json')"

          exit $EXIT_STATUS

  expected-failed-compilation:
    needs: run-determination
    if: needs.run-determination.outputs.result == 'true'
    runs-on: ubuntu-latest

    steps:
      - name: Checkout local repo
        uses: actions/checkout@v4

      - name: Compile sketch that is expected to error
        id: compile-sketches
        continue-on-error: true
        uses: ./
        with:
          fqbn: arduino:avr:uno
          github-token: ${{ secrets.GITHUB_TOKEN }}
          libraries: |
            []
          sketch-paths: |
            - ${{ env.TESTDATA_SKETCHES_PATH }}/Error

      - name: Fail the job if the action run succeeded
        if: steps.compile-sketches.outcome == 'success'
        run: |
          echo "::error::The action run was expected to fail, but passed!"
          exit 1