Skip to content

Example report - Status and Screenshots #6073

Example report - Status and Screenshots

Example report - Status and Screenshots #6073

name: Example report - Status and Screenshots
on:
workflow_dispatch:
schedule:
- cron: '*/15 * * * *'
concurrency:
group: ${{ github.repository }}-example-report
env:
PER_PAGE: 20
jobs:
get-environment:
name: Get Environment
runs-on: ubuntu-latest
outputs:
pages: ${{ steps.env.outputs.pages }}
gitref: ${{ steps.env.outputs.gitref }}
date: ${{ steps.env.outputs.date }}
linux_percy_project: ${{ steps.env.outputs.linux_percy_project }}
windows_percy_project: ${{ steps.env.outputs.windows_percy_project }}
mobile_percy_project: ${{ steps.env.outputs.mobile_percy_project }}
mobile_nonce: ${{ steps.env.outputs.mobile_nonce }}
updated: ${{ steps.version-check.outputs.updated }}
steps:
- name: Checkout Bevy main branch
uses: actions/checkout@v4
with:
repository: 'bevyengine/bevy'
ref: 'main'
- name: Get Environment
id: env
run: |
example_count=`cat Cargo.toml | grep '\[\[example\]\]' | wc -l`
page_count=$((example_count / ${{ env.PER_PAGE }} + 1))
echo "gitref=`git rev-parse HEAD`" >> $GITHUB_OUTPUT
echo "date=`date +%Y%m%d%H%M`" >> $GITHUB_OUTPUT
echo "linux_percy_project=dede4209/Screenshots-Linux-Vulkan" >> $GITHUB_OUTPUT
echo "windows_percy_project=dede4209/Screenshots-Windows-DX12" >> $GITHUB_OUTPUT
echo "mobile_percy_project=dede4209/Bevy-Mobile-Example" >> $GITHUB_OUTPUT
echo "pages=`python -c \"import json; print(json.dumps([i for i in range($page_count)]))\"`" >> $GITHUB_OUTPUT
echo "mobile_nonce=${{ github.run_id }}-$(date +%s)" >> $GITHUB_OUTPUT
- uses: actions/checkout@v4
with:
ref: 'results'
path: 'results'
- name: Check if current Bevy version already ran
id: version-check
run: |
gitref=`git rev-parse HEAD`
updated=`if ls results/*-$gitref 1> /dev/null 2>&1; then echo "false"; else echo "true"; fi`
echo "updated=$updated" >> $GITHUB_OUTPUT
take-screenshots:
name: Take Screenshots
needs: get-environment
if: needs.get-environment.outputs.updated == 'true' || github.event_name == 'workflow_dispatch'
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-latest, windows-latest]
page: ${{ fromJSON(needs.get-environment.outputs.pages) }}
steps:
- name: Checkout Bevy main branch
uses: actions/checkout@v4
with:
repository: 'bevyengine/bevy'
ref: ${{ needs.get-environment.outputs.gitref }}
- name: Setup Rust
uses: dtolnay/rust-toolchain@stable
- name: Install Bevy dependencies
if: runner.os == 'linux'
run: |
sudo apt-get update;
DEBIAN_FRONTEND=noninteractive sudo apt-get install --no-install-recommends -yq \
libasound2-dev libudev-dev;
- name: install xvfb, llvmpipe and lavapipe
if: runner.os == 'linux'
run: |
sudo apt-get update -y -qq
sudo add-apt-repository ppa:kisak/kisak-mesa -y
sudo apt-get update
sudo apt install -y xvfb libegl1-mesa libgl1-mesa-dri libxcb-xfixes0-dev mesa-vulkan-drivers
- uses: actions/cache/restore@v3
id: restore-cache
with:
path: |
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
target/
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}
restore-keys: ${{ runner.os }}-cargo-
- name: Take Screenshots (Linux)
id: screenshots-linux
if: runner.os == 'linux'
continue-on-error: true
run: xvfb-run -s "-screen 0 1280x1024x24" cargo run -p example-showcase -- --page ${{ matrix.page }} --per-page ${{ env.PER_PAGE }} run --screenshot --in-ci --ignore-stress-tests --report-details
- name: Take Screenshots (Windows)
id: screenshots-windows
if: runner.os == 'windows'
continue-on-error: true
shell: pwsh
run: |
Add-Type -AssemblyName System.Windows.Forms
$screen = [System.Windows.Forms.SystemInformation]::VirtualScreen
[Windows.Forms.Cursor]::Position = "$($screen.Width / 2),$($screen.Height / 2)"
cargo run -p example-showcase -- --page ${{ matrix.page }} --per-page ${{ env.PER_PAGE }} run --screenshot --in-ci --ignore-stress-tests --report-details
- name: Log errors
shell: pwsh
run: |
if (Get-Content no_screenshots) {
perl -p -e 's/(.*) - [.0-9]*\n/\1, /g' no_screenshots > cleaned
$no_screenshots = Get-Content .\cleaned -Raw
echo "::warning title=No Screenshots ${{ runner.os }}/${{ matrix.page }}::$no_screenshots"
}
if (Get-Content failures) {
perl -p -e 's/(.*) - [.0-9]*\n/\1, /g' failures > cleaned
$failures = Get-Content .\cleaned -Raw
echo "::error title=Failed To Run ${{ runner.os }}/${{ matrix.page }}::$failures"
}
- name: Outputs run results
id: run-results
shell: pwsh
run: |
echo "has_success=$(![String]::IsNullOrWhiteSpace((Get-content successes)))" | Out-File -FilePath $env:GITHUB_OUTPUT -Append
- name: Upload Screenshots
uses: actions/upload-artifact@v3
with:
name: screenshots-${{ runner.os }}-${{ matrix.page }}
path: screenshots
- name: Upload Status
uses: actions/upload-artifact@v3
with:
name: status-${{ runner.os }}-${{ matrix.page }}
path: |
successes
failures
no_screenshots
- uses: actions/cache/save@v3
if: steps.run-results.outputs.has_success == 'True'
id: save-cache
with:
path: |
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
target/
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}
send-to-percy:
name: Send screenshots to Percy
runs-on: ubuntu-latest
needs: [take-screenshots, get-environment]
strategy:
fail-fast: false
matrix:
include:
- os: Linux
percy_key: PERCY_TOKEN_LINUX_VULKAN
percy_project: ${{ needs.get-environment.outputs.linux_percy_project }}
- os: Windows
percy_key: PERCY_TOKEN_WINDOWS_DX12
percy_project: ${{ needs.get-environment.outputs.windows_percy_project }}
steps:
- name: Download all artifacts
uses: actions/download-artifact@v3
- name: Move examples to the correct folder
id: gather-examples
continue-on-error: true
run: |
mkdir screenshots-${{ matrix.os }}
for screenshotfolder in screenshots-${{ matrix.os }}-*
do
echo $screenshotfolder
rsync --verbose --archive $screenshotfolder/* screenshots-${{ matrix.os }}/
rm -rf $screenshotfolder
done
- name: Remove images that are all black
if: steps.gather-examples.outcome == 'success'
run: |
sudo apt install -y imagemagick
for image in screenshots-${{ matrix.os }}/*/*.png
do
mean=`convert "$image" -format "%[mean]" info:`
if [ "$mean" = 0 ]; then
echo "$image"
rm "$image"
fi
done
- name: Remove example known to be random
if: steps.gather-examples.outcome == 'success'
run: |
rm "screenshots-${{ matrix.os }}/Animation/animated_fox.png" || true
rm "screenshots-${{ matrix.os }}/Animation/morph_targets.png" || true
rm "screenshots-${{ matrix.os }}/Async Tasks/async_compute.png" || true
rm "screenshots-${{ matrix.os }}/Async Tasks/external_source_external_thread.png" || true
rm "screenshots-${{ matrix.os }}/Games/alien_cake_addict.png" || true
rm "screenshots-${{ matrix.os }}/Games/contributors.png" || true
rm "screenshots-${{ matrix.os }}/UI (User Interface)/font_atlas_debug.png" || true
# Windows only
- name: Remove example known to be random (Windows)
if: steps.gather-examples.outcome == 'success' && matrix.os == 'Windows'
run: |
rm "screenshots-${{ matrix.os }}/Shaders/compute_shader_game_of_life.png" || true
rm "screenshots-${{ matrix.os }}/ECS (Entity Component System)/apply_deferred.png" || true
- name: Send to Percy
if: steps.gather-examples.outcome == 'success'
run: |
npm install -g @percy/cli@latest
npx percy upload screenshots-${{ matrix.os }}
env:
PERCY_TOKEN: ${{ secrets[matrix.percy_key] }}
PERCY_COMMIT: ${{ needs.get-environment.outputs.gitref }}
- name: Wait for result
if: steps.gather-examples.outcome == 'success'
run: |
npx percy build:wait --project ${{ matrix.percy_project }} --commit ${{ needs.get-environment.outputs.gitref }}
env:
PERCY_TOKEN: ${{ secrets[matrix.percy_key] }}
update-results:
name: Update Results
runs-on: ubuntu-latest
needs: [send-to-percy, get-environment, mobile-check-result]
if: always() && (needs.get-environment.outputs.updated == 'true' || github.event_name == 'workflow_dispatch')
permissions:
contents: write
steps:
- uses: actions/checkout@v4
with:
ref: 'results'
path: 'results'
- name: Download all artifacts
uses: actions/download-artifact@v3
- name: Concatenate status
run: |
set -x
mkdir results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}
for report in status-Linux-*
do
(cat $report/successes; echo) >> Linux-successes-concat
(cat $report/failures; echo) >> Linux-failures-concat
(cat $report/no_screenshots; echo) >> Linux-no_screenshots-concat
done
# remove empty lines
grep . Linux-successes-concat > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/Linux-successes || true
grep . Linux-failures-concat > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/Linux-failures || true
grep . Linux-no_screenshots-concat > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/Linux-no_screenshots || true
for report in status-Windows-*
do
(cat $report/successes; echo) >> Windows-successes-concat
(cat $report/failures; echo) >> Windows-failures-concat
(cat $report/no_screenshots; echo) >> Windows-no_screenshots-concat
done
# remove empty lines
grep . Windows-successes-concat > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/Windows-successes || true
grep . Windows-failures-concat > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/Windows-failures || true
grep . Windows-no_screenshots-concat > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/Windows-no_screenshots || true
- name: Save Percy results
run: |
curl 'https://percy.io/api/v1/projects/${{ needs.get-environment.outputs.windows_percy_project }}/builds?filter\[sha\]=${{ needs.get-environment.outputs.gitref }}' | jq '.data[0].attributes' > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/Windows-percy
curl 'https://percy.io/api/v1/projects/${{ needs.get-environment.outputs.linux_percy_project }}/builds?filter\[sha\]=${{ needs.get-environment.outputs.gitref }}' | jq '.data[0].attributes' > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/Linux-percy
curl 'https://percy.io/api/v1/projects/${{ needs.get-environment.outputs.mobile_percy_project }}/builds?filter\[sha\]=${{ needs.get-environment.outputs.gitref }}' | jq '.data[0].attributes' > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/mobile-percy
- name: Store results in git
run: |
cd results
git config user.name 'Workflow'
git config user.email '<>'
git add .
git commit -m "Update Results"
git push
- name: Upload Aggregated Status
uses: actions/upload-artifact@v3
with:
name: aggregated-status
path: |
results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}
rerun-failed-examples:
name: Rerun Failed Examples (without screenshot)
needs: [get-environment, update-results]
if: always() && (needs.get-environment.outputs.updated == 'true' || github.event_name == 'workflow_dispatch')
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-latest, windows-latest]
steps:
- name: Checkout Bevy main branch
uses: actions/checkout@v4
with:
repository: 'bevyengine/bevy'
ref: ${{ needs.get-environment.outputs.gitref }}
- name: Setup Rust
uses: dtolnay/rust-toolchain@stable
- name: Download Aggregated Status
uses: actions/download-artifact@v3
with:
name: aggregated-status
path: aggregated-status
- name: Clean Up failures list
run: |
sed 's/.*\/\(.*\) - [.0-9]*/\1/g' aggregated-status/${{ runner.os }}-failures > failure-list
- name: Install Bevy dependencies
if: runner.os == 'linux'
run: |
sudo apt-get update;
DEBIAN_FRONTEND=noninteractive sudo apt-get install --no-install-recommends -yq \
libasound2-dev libudev-dev;
- name: install xvfb, llvmpipe and lavapipe
if: runner.os == 'linux'
run: |
sudo apt-get update -y -qq
sudo add-apt-repository ppa:kisak/kisak-mesa -y
sudo apt-get update
sudo apt install -y xvfb libegl1-mesa libgl1-mesa-dri libxcb-xfixes0-dev mesa-vulkan-drivers
- uses: actions/cache/restore@v3
id: restore-cache
with:
path: |
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
target/
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}
restore-keys: ${{ runner.os }}-cargo-
- name: Take Screenshots (Linux)
if: runner.os == 'linux'
continue-on-error: true
run: xvfb-run -s "-screen 0 1280x1024x24" cargo run -p example-showcase -- run --in-ci --ignore-stress-tests --report-details --example-list failure-list
- name: Take Screenshots (Windows)
if: runner.os == 'windows'
continue-on-error: true
run: cargo run -p example-showcase -- run --in-ci --ignore-stress-tests --report-details --example-list failure-list
- name: Upload Rerun Status
uses: actions/upload-artifact@v3
with:
name: status-rerun-${{ runner.os }}
path: |
successes
failures
no_screenshots
*.log
update-results-with-rerun:
name: Update Results with Rerun
needs: [rerun-failed-examples, get-environment]
if: always() && (needs.get-environment.outputs.updated == 'true' || github.event_name == 'workflow_dispatch')
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- uses: actions/checkout@v4
with:
ref: 'results'
path: 'results'
- name: Download Rerun Status on Linux
uses: actions/download-artifact@v3
with:
name: status-rerun-Linux
path: status-rerun-Linux
- name: Download Rerun Status on Windows
uses: actions/download-artifact@v3
with:
name: status-rerun-Windows
path: status-rerun-Windows
- name: Store results in git
run: |
mv status-rerun-Windows results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/
mv status-rerun-Linux results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/
cd results
git config user.name 'Workflow'
git config user.email '<>'
git add .
git commit -m "Update Results"
git push
update-website:
name: Update Website
needs: [update-results-with-rerun, get-environment]
if: always() && (needs.get-environment.outputs.updated == 'true' || github.event_name == 'workflow_dispatch')
runs-on: ubuntu-latest
permissions:
contents: read
pages: write
id-token: write
environment:
name: github-pages
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v4
with:
ref: "results"
path: "results"
- name: Setup Rust
uses: dtolnay/rust-toolchain@stable
- name: Build website
run: |
cargo run -- results
- name: Store generated html
uses: actions/upload-pages-artifact@v1
with:
path: ./site
- name: Deploy to GitHub Pages
id: deployment
uses: actions/deploy-pages@v1
build-for-iOS:
runs-on: macos-latest
timeout-minutes: 30
needs: get-environment
if: needs.get-environment.outputs.updated == 'true' || github.event_name == 'workflow_dispatch'
steps:
- uses: actions/checkout@v4
with:
repository: 'bevyengine/bevy'
ref: ${{ needs.get-environment.outputs.gitref }}
- uses: dtolnay/rust-toolchain@stable
- name: Add iOS targets
run: rustup target add aarch64-apple-ios x86_64-apple-ios
- name: Build app for iOS
run: |
cd examples/mobile
make xcodebuild-iphone
mkdir Payload
mv build/Build/Products/Debug-iphoneos/bevy_mobile_example.app Payload
zip -r bevy_mobile_example.zip Payload
mv bevy_mobile_example.zip bevy_mobile_example.ipa
- name: Upload to Browser Stack
run: |
curl -u "${{ secrets.BROWSERSTACK_USERNAME }}:${{ secrets.BROWSERSTACK_ACCESS_KEY }}" \
-X POST "https://api-cloud.browserstack.com/app-automate/upload" \
-F "file=@examples/mobile/bevy_mobile_example.ipa" \
-F "custom_id=$GITHUB_RUN_ID"
build-for-Android:
runs-on: ubuntu-latest
timeout-minutes: 30
needs: get-environment
if: needs.get-environment.outputs.updated == 'true' || github.event_name == 'workflow_dispatch'
steps:
- uses: actions/checkout@v4
with:
repository: 'bevyengine/bevy'
ref: ${{ needs.get-environment.outputs.gitref }}
- uses: dtolnay/rust-toolchain@stable
- name: Add Android targets
run: rustup target add aarch64-linux-android armv7-linux-androideabi
- name: Install Cargo APK
run: cargo install --force cargo-apk
- name: Build app for Android
run: ANDROID_NDK_ROOT=$ANDROID_NDK_LATEST_HOME cargo apk build --package bevy_mobile_example
env:
# This will reduce the APK size from 1GB to ~200MB
CARGO_PROFILE_DEV_DEBUG: false
- name: Upload to Browser Stack
run: |
curl -u "${{ secrets.BROWSERSTACK_USERNAME }}:${{ secrets.BROWSERSTACK_ACCESS_KEY }}" \
-X POST "https://api-cloud.browserstack.com/app-automate/upload" \
-F "file=@target/debug/apk/bevyexample.apk" \
-F "custom_id=$GITHUB_RUN_ID"
mobile-run:
runs-on: ubuntu-latest
timeout-minutes: 30
needs: [build-for-iOS, build-for-Android, get-environment]
env:
PERCY_PARALLEL_NONCE: ${{ needs.get-environment.outputs.mobile_nonce }}
PERCY_PARALLEL_TOTAL: ${{ strategy.job-total }}
strategy:
matrix:
include:
- device: "iPhone 13"
os_version: "15"
- device: "iPhone 14"
os_version: "16"
- device: "iPhone 15"
os_version: "17"
- device: "Xiaomi Redmi Note 11"
os_version: "11.0"
- device: "Google Pixel 6"
os_version: "12.0"
- device: "Samsung Galaxy S23"
os_version: "13.0"
- device: "Google Pixel 8"
os_version: "14.0"
steps:
- uses: actions/checkout@v4
with:
repository: 'bevyengine/bevy'
ref: ${{ needs.get-environment.outputs.gitref }}
- name: Run Example
run: |
cd .github/start-mobile-example
npm install
npm install -g @percy/cli@latest
npx percy app:exec --parallel -- npm run mobile
env:
BROWSERSTACK_APP_ID: ${{ github.run_id }}
BROWSERSTACK_USERNAME: ${{ secrets.BROWSERSTACK_USERNAME }}
BROWSERSTACK_ACCESS_KEY: ${{ secrets.BROWSERSTACK_ACCESS_KEY }}
PERCY_TOKEN: ${{ secrets.PERCY_TOKEN_MOBILE }}
DEVICE: ${{ matrix.device }}
OS_VERSION: ${{ matrix.os_version }}
PERCY_COMMIT: ${{ needs.get-environment.outputs.gitref }}
- name: Save screenshots
if: ${{ always() }}
uses: actions/upload-artifact@v3
with:
name: screenshots-${{ matrix.device }}-${{ matrix.os_version }}
path: .github/start-mobile-example/*.png
mobile-check-result:
runs-on: ubuntu-latest
timeout-minutes: 30
needs: [mobile-run, get-environment]
if: always() && (needs.get-environment.outputs.updated == 'true' || github.event_name == 'workflow_dispatch')
steps:
- name: Wait for screenshots comparison
run: |
npm install -g @percy/cli@latest
npx percy build:wait --project ${{ needs.get-environment.outputs.mobile_percy_project }} --commit ${{ needs.get-environment.outputs.gitref }}
env:
PERCY_TOKEN: ${{ secrets.PERCY_TOKEN_MOBILE }}