Example report - Status and Screenshots #6078
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Example report - Status and Screenshots | |
on: | |
workflow_dispatch: | |
schedule: | |
- cron: '*/15 * * * *' | |
concurrency: | |
group: ${{ github.repository }}-example-report | |
env: | |
PER_PAGE: 20 | |
jobs: | |
get-environment: | |
name: Get Environment | |
runs-on: ubuntu-latest | |
outputs: | |
pages: ${{ steps.env.outputs.pages }} | |
gitref: ${{ steps.env.outputs.gitref }} | |
date: ${{ steps.env.outputs.date }} | |
linux_percy_project: ${{ steps.env.outputs.linux_percy_project }} | |
windows_percy_project: ${{ steps.env.outputs.windows_percy_project }} | |
mobile_percy_project: ${{ steps.env.outputs.mobile_percy_project }} | |
mobile_nonce: ${{ steps.env.outputs.mobile_nonce }} | |
updated: ${{ steps.version-check.outputs.updated }} | |
steps: | |
- name: Checkout Bevy main branch | |
uses: actions/checkout@v4 | |
with: | |
repository: 'bevyengine/bevy' | |
ref: 'main' | |
- name: Get Environment | |
id: env | |
run: | | |
example_count=`cat Cargo.toml | grep '\[\[example\]\]' | wc -l` | |
page_count=$((example_count / ${{ env.PER_PAGE }} + 1)) | |
echo "gitref=`git rev-parse HEAD`" >> $GITHUB_OUTPUT | |
echo "date=`date +%Y%m%d%H%M`" >> $GITHUB_OUTPUT | |
echo "linux_percy_project=dede4209/Screenshots-Linux-Vulkan" >> $GITHUB_OUTPUT | |
echo "windows_percy_project=dede4209/Screenshots-Windows-DX12" >> $GITHUB_OUTPUT | |
echo "mobile_percy_project=dede4209/Bevy-Mobile-Example" >> $GITHUB_OUTPUT | |
echo "pages=`python -c \"import json; print(json.dumps([i for i in range($page_count)]))\"`" >> $GITHUB_OUTPUT | |
echo "mobile_nonce=${{ github.run_id }}-$(date +%s)" >> $GITHUB_OUTPUT | |
- uses: actions/checkout@v4 | |
with: | |
ref: 'results' | |
path: 'results' | |
- name: Check if current Bevy version already ran | |
id: version-check | |
run: | | |
gitref=`git rev-parse HEAD` | |
updated=`if ls results/*-$gitref 1> /dev/null 2>&1; then echo "false"; else echo "true"; fi` | |
echo "updated=$updated" >> $GITHUB_OUTPUT | |
take-screenshots: | |
name: Take Screenshots | |
needs: get-environment | |
if: needs.get-environment.outputs.updated == 'true' || github.event_name == 'workflow_dispatch' | |
runs-on: ${{ matrix.os }} | |
strategy: | |
matrix: | |
os: [ubuntu-latest, windows-latest] | |
page: ${{ fromJSON(needs.get-environment.outputs.pages) }} | |
steps: | |
- name: Checkout Bevy main branch | |
uses: actions/checkout@v4 | |
with: | |
repository: 'bevyengine/bevy' | |
ref: ${{ needs.get-environment.outputs.gitref }} | |
- name: Setup Rust | |
uses: dtolnay/rust-toolchain@stable | |
- name: Install Bevy dependencies | |
if: runner.os == 'linux' | |
run: | | |
sudo apt-get update; | |
DEBIAN_FRONTEND=noninteractive sudo apt-get install --no-install-recommends -yq \ | |
libasound2-dev libudev-dev; | |
- name: install xvfb, llvmpipe and lavapipe | |
if: runner.os == 'linux' | |
run: | | |
sudo apt-get update -y -qq | |
sudo add-apt-repository ppa:kisak/kisak-mesa -y | |
sudo apt-get update | |
sudo apt install -y xvfb libegl1-mesa libgl1-mesa-dri libxcb-xfixes0-dev mesa-vulkan-drivers | |
- uses: actions/cache/restore@v3 | |
id: restore-cache | |
with: | |
path: | | |
~/.cargo/bin/ | |
~/.cargo/registry/index/ | |
~/.cargo/registry/cache/ | |
~/.cargo/git/db/ | |
target/ | |
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }} | |
restore-keys: ${{ runner.os }}-cargo- | |
- name: Take Screenshots (Linux) | |
id: screenshots-linux | |
if: runner.os == 'linux' | |
continue-on-error: true | |
run: xvfb-run -s "-screen 0 1280x1024x24" cargo run -p example-showcase -- --page ${{ matrix.page }} --per-page ${{ env.PER_PAGE }} run --screenshot --in-ci --ignore-stress-tests --report-details | |
- name: Take Screenshots (Windows) | |
id: screenshots-windows | |
if: runner.os == 'windows' | |
continue-on-error: true | |
shell: pwsh | |
run: | | |
Add-Type -AssemblyName System.Windows.Forms | |
$screen = [System.Windows.Forms.SystemInformation]::VirtualScreen | |
[Windows.Forms.Cursor]::Position = "$($screen.Width / 2),$($screen.Height / 2)" | |
cargo run -p example-showcase -- --page ${{ matrix.page }} --per-page ${{ env.PER_PAGE }} run --screenshot --in-ci --ignore-stress-tests --report-details | |
- name: Log errors | |
shell: pwsh | |
run: | | |
if (Get-Content no_screenshots) { | |
perl -p -e 's/(.*) - [.0-9]*\n/\1, /g' no_screenshots > cleaned | |
$no_screenshots = Get-Content .\cleaned -Raw | |
echo "::warning title=No Screenshots ${{ runner.os }}/${{ matrix.page }}::$no_screenshots" | |
} | |
if (Get-Content failures) { | |
perl -p -e 's/(.*) - [.0-9]*\n/\1, /g' failures > cleaned | |
$failures = Get-Content .\cleaned -Raw | |
echo "::error title=Failed To Run ${{ runner.os }}/${{ matrix.page }}::$failures" | |
} | |
- name: Outputs run results | |
id: run-results | |
shell: pwsh | |
run: | | |
echo "has_success=$(![String]::IsNullOrWhiteSpace((Get-content successes)))" | Out-File -FilePath $env:GITHUB_OUTPUT -Append | |
- name: Upload Screenshots | |
uses: actions/upload-artifact@v3 | |
with: | |
name: screenshots-${{ runner.os }}-${{ matrix.page }} | |
path: screenshots | |
- name: Upload Status | |
uses: actions/upload-artifact@v3 | |
with: | |
name: status-${{ runner.os }}-${{ matrix.page }} | |
path: | | |
successes | |
failures | |
no_screenshots | |
- uses: actions/cache/save@v3 | |
if: steps.run-results.outputs.has_success == 'True' | |
id: save-cache | |
with: | |
path: | | |
~/.cargo/bin/ | |
~/.cargo/registry/index/ | |
~/.cargo/registry/cache/ | |
~/.cargo/git/db/ | |
target/ | |
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }} | |
send-to-percy: | |
name: Send screenshots to Percy | |
runs-on: ubuntu-latest | |
needs: [take-screenshots, get-environment] | |
strategy: | |
fail-fast: false | |
matrix: | |
include: | |
- os: Linux | |
percy_key: PERCY_TOKEN_LINUX_VULKAN | |
percy_project: ${{ needs.get-environment.outputs.linux_percy_project }} | |
- os: Windows | |
percy_key: PERCY_TOKEN_WINDOWS_DX12 | |
percy_project: ${{ needs.get-environment.outputs.windows_percy_project }} | |
steps: | |
- name: Download all artifacts | |
uses: actions/download-artifact@v3 | |
- name: Move examples to the correct folder | |
id: gather-examples | |
continue-on-error: true | |
run: | | |
mkdir screenshots-${{ matrix.os }} | |
for screenshotfolder in screenshots-${{ matrix.os }}-* | |
do | |
echo $screenshotfolder | |
rsync --verbose --archive $screenshotfolder/* screenshots-${{ matrix.os }}/ | |
rm -rf $screenshotfolder | |
done | |
- name: Remove images that are all black | |
if: steps.gather-examples.outcome == 'success' | |
run: | | |
sudo apt install -y imagemagick | |
for image in screenshots-${{ matrix.os }}/*/*.png | |
do | |
mean=`convert "$image" -format "%[mean]" info:` | |
if [ "$mean" = 0 ]; then | |
echo "$image" | |
rm "$image" | |
fi | |
done | |
- name: Remove example known to be random | |
if: steps.gather-examples.outcome == 'success' | |
run: | | |
rm "screenshots-${{ matrix.os }}/Animation/animated_fox.png" || true | |
rm "screenshots-${{ matrix.os }}/Animation/morph_targets.png" || true | |
rm "screenshots-${{ matrix.os }}/Async Tasks/async_compute.png" || true | |
rm "screenshots-${{ matrix.os }}/Async Tasks/external_source_external_thread.png" || true | |
rm "screenshots-${{ matrix.os }}/Games/alien_cake_addict.png" || true | |
rm "screenshots-${{ matrix.os }}/Games/contributors.png" || true | |
rm "screenshots-${{ matrix.os }}/UI (User Interface)/font_atlas_debug.png" || true | |
# Windows only | |
- name: Remove example known to be random (Windows) | |
if: steps.gather-examples.outcome == 'success' && matrix.os == 'Windows' | |
run: | | |
rm "screenshots-${{ matrix.os }}/Shaders/compute_shader_game_of_life.png" || true | |
rm "screenshots-${{ matrix.os }}/ECS (Entity Component System)/apply_deferred.png" || true | |
- name: Send to Percy | |
if: steps.gather-examples.outcome == 'success' | |
run: | | |
npm install -g @percy/cli@latest | |
npx percy upload screenshots-${{ matrix.os }} | |
env: | |
PERCY_TOKEN: ${{ secrets[matrix.percy_key] }} | |
PERCY_COMMIT: ${{ needs.get-environment.outputs.gitref }} | |
- name: Wait for result | |
if: steps.gather-examples.outcome == 'success' | |
run: | | |
npx percy build:wait --project ${{ matrix.percy_project }} --commit ${{ needs.get-environment.outputs.gitref }} | |
env: | |
PERCY_TOKEN: ${{ secrets[matrix.percy_key] }} | |
update-results: | |
name: Update Results | |
runs-on: ubuntu-latest | |
needs: [send-to-percy, get-environment, mobile-check-result] | |
if: always() && (needs.get-environment.outputs.updated == 'true' || github.event_name == 'workflow_dispatch') | |
permissions: | |
contents: write | |
steps: | |
- uses: actions/checkout@v4 | |
with: | |
ref: 'results' | |
path: 'results' | |
- name: Download all artifacts | |
uses: actions/download-artifact@v3 | |
- name: Concatenate status | |
run: | | |
set -x | |
mkdir results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }} | |
for report in status-Linux-* | |
do | |
(cat $report/successes; echo) >> Linux-successes-concat | |
(cat $report/failures; echo) >> Linux-failures-concat | |
(cat $report/no_screenshots; echo) >> Linux-no_screenshots-concat | |
done | |
# remove empty lines | |
grep . Linux-successes-concat > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/Linux-successes || true | |
grep . Linux-failures-concat > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/Linux-failures || true | |
grep . Linux-no_screenshots-concat > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/Linux-no_screenshots || true | |
for report in status-Windows-* | |
do | |
(cat $report/successes; echo) >> Windows-successes-concat | |
(cat $report/failures; echo) >> Windows-failures-concat | |
(cat $report/no_screenshots; echo) >> Windows-no_screenshots-concat | |
done | |
# remove empty lines | |
grep . Windows-successes-concat > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/Windows-successes || true | |
grep . Windows-failures-concat > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/Windows-failures || true | |
grep . Windows-no_screenshots-concat > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/Windows-no_screenshots || true | |
- name: Save Percy results | |
run: | | |
curl 'https://percy.io/api/v1/projects/${{ needs.get-environment.outputs.windows_percy_project }}/builds?filter\[sha\]=${{ needs.get-environment.outputs.gitref }}' | jq '.data[0].attributes' > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/Windows-percy | |
curl 'https://percy.io/api/v1/projects/${{ needs.get-environment.outputs.linux_percy_project }}/builds?filter\[sha\]=${{ needs.get-environment.outputs.gitref }}' | jq '.data[0].attributes' > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/Linux-percy | |
curl 'https://percy.io/api/v1/projects/${{ needs.get-environment.outputs.mobile_percy_project }}/builds?filter\[sha\]=${{ needs.get-environment.outputs.gitref }}' | jq '.data[0].attributes' > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/mobile-percy | |
- name: Store results in git | |
run: | | |
cd results | |
git config user.name 'Workflow' | |
git config user.email '<>' | |
git add . | |
git commit -m "Update Results" | |
git push | |
- name: Upload Aggregated Status | |
uses: actions/upload-artifact@v3 | |
with: | |
name: aggregated-status | |
path: | | |
results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }} | |
rerun-failed-examples: | |
name: Rerun Failed Examples (without screenshot) | |
needs: [get-environment, update-results] | |
if: always() && (needs.get-environment.outputs.updated == 'true' || github.event_name == 'workflow_dispatch') | |
runs-on: ${{ matrix.os }} | |
strategy: | |
matrix: | |
os: [ubuntu-latest, windows-latest] | |
steps: | |
- name: Checkout Bevy main branch | |
uses: actions/checkout@v4 | |
with: | |
repository: 'bevyengine/bevy' | |
ref: ${{ needs.get-environment.outputs.gitref }} | |
- name: Setup Rust | |
uses: dtolnay/rust-toolchain@stable | |
- name: Download Aggregated Status | |
uses: actions/download-artifact@v3 | |
with: | |
name: aggregated-status | |
path: aggregated-status | |
- name: Clean Up failures list | |
run: | | |
sed 's/.*\/\(.*\) - [.0-9]*/\1/g' aggregated-status/${{ runner.os }}-failures > failure-list | |
- name: Install Bevy dependencies | |
if: runner.os == 'linux' | |
run: | | |
sudo apt-get update; | |
DEBIAN_FRONTEND=noninteractive sudo apt-get install --no-install-recommends -yq \ | |
libasound2-dev libudev-dev; | |
- name: install xvfb, llvmpipe and lavapipe | |
if: runner.os == 'linux' | |
run: | | |
sudo apt-get update -y -qq | |
sudo add-apt-repository ppa:kisak/kisak-mesa -y | |
sudo apt-get update | |
sudo apt install -y xvfb libegl1-mesa libgl1-mesa-dri libxcb-xfixes0-dev mesa-vulkan-drivers | |
- uses: actions/cache/restore@v3 | |
id: restore-cache | |
with: | |
path: | | |
~/.cargo/bin/ | |
~/.cargo/registry/index/ | |
~/.cargo/registry/cache/ | |
~/.cargo/git/db/ | |
target/ | |
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }} | |
restore-keys: ${{ runner.os }}-cargo- | |
- name: Take Screenshots (Linux) | |
if: runner.os == 'linux' | |
continue-on-error: true | |
run: xvfb-run -s "-screen 0 1280x1024x24" cargo run -p example-showcase -- run --in-ci --ignore-stress-tests --report-details --example-list failure-list | |
- name: Take Screenshots (Windows) | |
if: runner.os == 'windows' | |
continue-on-error: true | |
run: cargo run -p example-showcase -- run --in-ci --ignore-stress-tests --report-details --example-list failure-list | |
- name: Upload Rerun Status | |
uses: actions/upload-artifact@v3 | |
with: | |
name: status-rerun-${{ runner.os }} | |
path: | | |
successes | |
failures | |
no_screenshots | |
*.log | |
update-results-with-rerun: | |
name: Update Results with Rerun | |
needs: [rerun-failed-examples, get-environment] | |
if: always() && (needs.get-environment.outputs.updated == 'true' || github.event_name == 'workflow_dispatch') | |
runs-on: ubuntu-latest | |
permissions: | |
contents: write | |
steps: | |
- uses: actions/checkout@v4 | |
with: | |
ref: 'results' | |
path: 'results' | |
- name: Download Rerun Status on Linux | |
uses: actions/download-artifact@v3 | |
with: | |
name: status-rerun-Linux | |
path: status-rerun-Linux | |
- name: Download Rerun Status on Windows | |
uses: actions/download-artifact@v3 | |
with: | |
name: status-rerun-Windows | |
path: status-rerun-Windows | |
- name: Store results in git | |
run: | | |
mv status-rerun-Windows results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/ | |
mv status-rerun-Linux results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/ | |
cd results | |
git config user.name 'Workflow' | |
git config user.email '<>' | |
git add . | |
git commit -m "Update Results" | |
git push | |
update-website: | |
name: Update Website | |
needs: [update-results-with-rerun, get-environment] | |
if: always() && (needs.get-environment.outputs.updated == 'true' || github.event_name == 'workflow_dispatch') | |
runs-on: ubuntu-latest | |
permissions: | |
contents: read | |
pages: write | |
id-token: write | |
environment: | |
name: github-pages | |
steps: | |
- uses: actions/checkout@v4 | |
- uses: actions/checkout@v4 | |
with: | |
ref: "results" | |
path: "results" | |
- name: Setup Rust | |
uses: dtolnay/rust-toolchain@stable | |
- name: Build website | |
run: | | |
cargo run -- results | |
- name: Store generated html | |
uses: actions/upload-pages-artifact@v1 | |
with: | |
path: ./site | |
- name: Deploy to GitHub Pages | |
id: deployment | |
uses: actions/deploy-pages@v1 | |
build-for-iOS: | |
runs-on: macos-latest | |
timeout-minutes: 30 | |
needs: get-environment | |
if: needs.get-environment.outputs.updated == 'true' || github.event_name == 'workflow_dispatch' | |
steps: | |
- uses: actions/checkout@v4 | |
with: | |
repository: 'bevyengine/bevy' | |
ref: ${{ needs.get-environment.outputs.gitref }} | |
- uses: dtolnay/rust-toolchain@stable | |
- name: Add iOS targets | |
run: rustup target add aarch64-apple-ios x86_64-apple-ios | |
- name: Build app for iOS | |
run: | | |
cd examples/mobile | |
make xcodebuild-iphone | |
mkdir Payload | |
mv build/Build/Products/Debug-iphoneos/bevy_mobile_example.app Payload | |
zip -r bevy_mobile_example.zip Payload | |
mv bevy_mobile_example.zip bevy_mobile_example.ipa | |
- name: Upload to Browser Stack | |
run: | | |
curl -u "${{ secrets.BROWSERSTACK_USERNAME }}:${{ secrets.BROWSERSTACK_ACCESS_KEY }}" \ | |
-X POST "https://api-cloud.browserstack.com/app-automate/upload" \ | |
-F "file=@examples/mobile/bevy_mobile_example.ipa" \ | |
-F "custom_id=$GITHUB_RUN_ID" | |
build-for-Android: | |
runs-on: ubuntu-latest | |
timeout-minutes: 30 | |
needs: get-environment | |
if: needs.get-environment.outputs.updated == 'true' || github.event_name == 'workflow_dispatch' | |
steps: | |
- uses: actions/checkout@v4 | |
with: | |
repository: 'bevyengine/bevy' | |
ref: ${{ needs.get-environment.outputs.gitref }} | |
- uses: dtolnay/rust-toolchain@stable | |
- name: Add Android targets | |
run: rustup target add aarch64-linux-android armv7-linux-androideabi | |
- name: Install Cargo APK | |
run: cargo install --force cargo-apk | |
- name: Build app for Android | |
run: ANDROID_NDK_ROOT=$ANDROID_NDK_LATEST_HOME cargo apk build --package bevy_mobile_example | |
env: | |
# This will reduce the APK size from 1GB to ~200MB | |
CARGO_PROFILE_DEV_DEBUG: false | |
- name: Upload to Browser Stack | |
run: | | |
curl -u "${{ secrets.BROWSERSTACK_USERNAME }}:${{ secrets.BROWSERSTACK_ACCESS_KEY }}" \ | |
-X POST "https://api-cloud.browserstack.com/app-automate/upload" \ | |
-F "file=@target/debug/apk/bevyexample.apk" \ | |
-F "custom_id=$GITHUB_RUN_ID" | |
mobile-run: | |
runs-on: ubuntu-latest | |
timeout-minutes: 30 | |
needs: [build-for-iOS, build-for-Android, get-environment] | |
env: | |
PERCY_PARALLEL_NONCE: ${{ needs.get-environment.outputs.mobile_nonce }} | |
PERCY_PARALLEL_TOTAL: ${{ strategy.job-total }} | |
strategy: | |
matrix: | |
include: | |
- device: "iPhone 13" | |
os_version: "15" | |
- device: "iPhone 14" | |
os_version: "16" | |
- device: "iPhone 15" | |
os_version: "17" | |
- device: "Xiaomi Redmi Note 11" | |
os_version: "11.0" | |
- device: "Google Pixel 6" | |
os_version: "12.0" | |
- device: "Samsung Galaxy S23" | |
os_version: "13.0" | |
- device: "Google Pixel 8" | |
os_version: "14.0" | |
steps: | |
- uses: actions/checkout@v4 | |
with: | |
repository: 'bevyengine/bevy' | |
ref: ${{ needs.get-environment.outputs.gitref }} | |
- name: Run Example | |
run: | | |
cd .github/start-mobile-example | |
npm install | |
npm install -g @percy/cli@latest | |
npx percy app:exec --parallel -- npm run mobile | |
env: | |
BROWSERSTACK_APP_ID: ${{ github.run_id }} | |
BROWSERSTACK_USERNAME: ${{ secrets.BROWSERSTACK_USERNAME }} | |
BROWSERSTACK_ACCESS_KEY: ${{ secrets.BROWSERSTACK_ACCESS_KEY }} | |
PERCY_TOKEN: ${{ secrets.PERCY_TOKEN_MOBILE }} | |
DEVICE: ${{ matrix.device }} | |
OS_VERSION: ${{ matrix.os_version }} | |
PERCY_COMMIT: ${{ needs.get-environment.outputs.gitref }} | |
- name: Save screenshots | |
if: ${{ always() }} | |
uses: actions/upload-artifact@v3 | |
with: | |
name: screenshots-${{ matrix.device }}-${{ matrix.os_version }} | |
path: .github/start-mobile-example/*.png | |
mobile-check-result: | |
runs-on: ubuntu-latest | |
timeout-minutes: 30 | |
needs: [mobile-run, get-environment] | |
if: always() && (needs.get-environment.outputs.updated == 'true' || github.event_name == 'workflow_dispatch') | |
steps: | |
- name: Wait for screenshots comparison | |
run: | | |
npm install -g @percy/cli@latest | |
npx percy build:wait --project ${{ needs.get-environment.outputs.mobile_percy_project }} --commit ${{ needs.get-environment.outputs.gitref }} | |
env: | |
PERCY_TOKEN: ${{ secrets.PERCY_TOKEN_MOBILE }} |