flaky.yaml

# Run all tests, including flaky test.
#
# The default CI workflow ignores flaky tests.  This workflow will run
# all tests, including ignored ones.
#
# To use this workflow you can either:
#
# - Label a PR with "flaky-test", the normal CI workflow will not run
#   any jobs but the jobs here will be run.  Note that to merge the PR
#   you'll need to remove the label eventually because the normal CI
#   jobs are required by branch protection.
#
# - Manually trigger the workflow, you may choose a branch for this to
#   run on.
#
# Additionally this jobs runs once a day on a schedule.
#
# Currently doctests are not run by this workflow.

name: Flaky CI

on:
  pull_request:
    types: [ 'labeled', 'unlabeled', 'opened', 'synchronize', 'reopened' ]
  schedule:
    # 06:30 UTC every day
    - cron: '30 6 * * *'
  workflow_dispatch:
    inputs:
      branch:
        description: 'Branch to run on, defaults to main'
        required: true
        default: 'main'
        type: string

concurrency:
  group: flaky-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  IROH_FORCE_STAGING_RELAYS: "1"

jobs:
  tests:
    if: "contains(github.event.pull_request.labels.*.name, 'flaky-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'"
    uses: './.github/workflows/tests.yaml'
    with:
      flaky: true
      git-ref: ${{ inputs.branch }}
  notify:
    needs: tests
    if: ${{ always() }}
    runs-on: ubuntu-latest
    steps:
      - name: Extract test results
        run: |
          printf '${{ toJSON(needs) }}\n'
          result=$(echo '${{ toJSON(needs) }}' | jq -r .tests.result)
          echo TESTS_RESULT=$result
          echo "TESTS_RESULT=$result" >>"$GITHUB_ENV"
      - name: download nextest reports
        uses: actions/download-artifact@v4
        with:
          pattern: libtest_run_${{ github.run_number }}-${{ github.run_attempt }}-*
          merge-multiple: true
          path: nextest-results
      - name: create summary report
        id: make_summary
        run: |
          # prevent the glob expression in the loop to match on itself when the dir is empty
          shopt -s nullglob
          # to deal with multiline outputs it's recommended to use a random EOF, the syntax is based on
          # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings
          EOF=aP51VriWCxNJ1JjvmO9i
          echo "summary<<$EOF" >> $GITHUB_OUTPUT
          echo "Flaky tests failure:" >> $GITHUB_OUTPUT
          echo " " >> $GITHUB_OUTPUT
          for report in nextest-results/*.json; do
            # remove the name prefix and extension, and split the parts
            name=$(echo ${report:16:-5} | tr _ ' ')
            echo $name
            echo "- **$name**" >> $GITHUB_OUTPUT
            # select the failed tests
            # the tests have this format "crate::module$test_name", the sed expressions remove the quotes and replace $ for ::
            failure=$(jq --slurp '.[] | select(.["type"] == "test" and .["event"] == "failed" ) | .["name"]' $report | sed -e 's/^"//g' -e 's/\$/::/' -e 's/"//')
            echo "$failure"
            echo "$failure" >> $GITHUB_OUTPUT
          done
          echo "" >> $GITHUB_OUTPUT
          echo "See https://github.com/n0-computer/iroh/actions/workflows/flaky.yaml" >> $GITHUB_OUTPUT
          echo "$EOF" >> $GITHUB_OUTPUT
      - name: Notify discord on failure
        uses: n0-computer/discord-webhook-notify@v1
        if: ${{ env.TESTS_RESULT == 'failure' || env.TESTS_RESULT == 'success' }}
        with:
          severity: ${{ env.TESTS_RESULT == 'failure' && 'warn' || 'info' }}
          details: ${{ env.TESTS_RESULT == 'failure' && steps.make_summary.outputs.summary || 'No flaky failures!' }}
          webhookUrl: ${{ secrets.DISCORD_N0_GITHUB_CHANNEL_WEBHOOK_URL }}