Skip to content

Commit

Permalink
workflow improvements (#492)
Browse files Browse the repository at this point in the history
- esp-idf workflow now uses an imported json matrix produced by a php script
  - builds the jobs list using release and tags informations from esp-idf repository
  - uses the three latest stable versions from the official supported list, plus two hardcoded versions
  - hopefully won't need to be updated every time espressif creates a version tag or a release
  - disabled cache support, max size per repo is 10GB and this one was using 10.9GB, the speed difference isn't significan anyway
- arduino workflow has been bumped to latest stable (pre-3.x.x) core versions
- platformio workflow core versions have been bumped, temporarily fixed applied (was failing on 3.x.x-beta with iram size)
  • Loading branch information
tobozo authored Dec 19, 2023
1 parent 197d733 commit 6ef928c
Show file tree
Hide file tree
Showing 5 changed files with 258 additions and 227 deletions.
121 changes: 121 additions & 0 deletions .github/scripts/esp-idf-versions.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
<?php

$max_versions = 3;
$releases = [];
$patch_versions = [];
$idf_fqbns = [];
$idf_versions = [];
$idf_boards = ['esp32', 'esp32s2', 'esp32s3'];

$git_remote = `git ls-remote https://github.com/espressif/esp-idf`;

!empty($git_remote) or php_die("bad github response");

$lines = explode("\n", $git_remote);

// get version numbers from enumerated releases
foreach( $lines as $num => $line )
{
if( !preg_match("/release/", $line ) )
continue; // tag or commit
$line = trim($line);
if( empty($line) )
continue; // EOL or separator
$line_parts = explode("/", trim($line)); // tag name is the last part
if( !empty( $line_parts ) )
$releases[] = end($line_parts);
}

!empty($releases) or php_die("releases not found");

arsort( $releases );

// get version numbers from enumerated tags
foreach( $lines as $num => $line )
{
if( !preg_match("/tags/", $line ) )
continue;
$line = trim($line);
$tag_parts = explode("/", $line );
$tag_name = end( $tag_parts );
if( substr( $tag_name, 0, 1 ) == 'v' // esp-idf official tag names are prefixed with "v"
&& substr( $tag_name, -3 ) != '^{}' // ignore commit pointers returned by git ls-remote
/*&& !preg_match( '/beta|dev|rc|head|merge/i', $tag_name)*/ ) // ignore beta/dev/rc and other non significant tags
{
if(! preg_match("/^v?(0|(?:[1-9]\d*))(?:\.(0|(?:[1-9]\d*))(?:\.(0|(?:[1-9]\d*)))?(?:\-([\w][\w\.\-_]*))?)?$/i", $tag_name, $results ) )
{
php_die("Bad semver with entry $num: $tag_name");
}
unset($results[0]);
$semver = "v".implode('.', $results );
if( $semver != $tag_name )
continue; // pattern matching failed with $semver
//php_die("uh oh pattern matching failed with $semver/$tag_name");
$minor = $results[1].'.'.$results[2];
$patch = !empty($results[3]) ? $results[1].'.'.$results[2].'.'.$results[3] : "";
if( !in_array( 'v'.$minor, $releases ) )
continue; // this tag is not listed in releases
if( !empty($results[3]) && !in_array( $patch, $patch_versions ) )
$patch_versions[] = $patch;
}
}

!empty($patch_versions) or php_die("tags not found");

arsort( $patch_versions );

$max_boards = (count($idf_boards)*$max_versions);

// match release versions with tag versions
foreach( $releases as $minor )
{
$top_version = '';
foreach( $patch_versions as $patch )
{
if( str_starts_with( 'v'.$patch, $minor ) )
{
if( $patch > $top_version ) // SEQ comparator on a string is just cheap semver, what could go wrong ? :)
{
$top_version = $patch;
}
}
}
if( $top_version == '' )
continue;

$idf_versions[] = str_replace('v', '', $top_version );
if( count( $idf_versions ) == $max_versions )
break;
}

!empty($idf_versions) or php_die("latest versions not found");
!empty($idf_boards) or php_die("no board selected");

// finally fill matrix json array with jobs
foreach( $idf_versions as $idx => $idf_version )
{
if( count( $idf_fqbns ) >= $max_boards ) {
break;
}
foreach( $idf_boards as $idf_board ) {
$idf_fqbns[] = $idf_board.'@'.$idf_version;
}
}

// add hardcoded versions
$idf_fqbns[] = 'esp32@4.1.4';
$idf_fqbns[] = 'esp32@4.3.6';
//$idf_fqbns[] = 'esp32@5.2-beta1';
//$idf_fqbns[] = 'esp32@5.2-dev';
//$idf_fqbns[] = 'esp32@5.3-dev';

$json_array = [ "esp-idf-fqbn" => $idf_fqbns ];

echo json_encode( $json_array, JSON_PRETTY_PRINT );


function php_die($msg)
{
echo $msg.PHP_EOL;
exit(1);
}
18 changes: 9 additions & 9 deletions .github/workflows/ArduinoBuild.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ jobs:
matrix:

platform-url:
- https://raw.githubusercontent.com/espressif/arduino-esp32/gh-pages/package_esp32_dev_index.json
- https://espressif.github.io/arduino-esp32/package_esp32_index.json

board:
# ESP32 devices for 3D matrix
Expand All @@ -42,10 +42,10 @@ jobs:
#- 2.0.1
#- 2.0.2
#- 2.0.3
- 2.0.4
- 2.0.5
- 2.0.6
- 2.0.7
#- 2.0.4
- 2.0.11
- 2.0.12
- 2.0.13

include:
# 3D matrix doesn't apply to these:
Expand All @@ -56,9 +56,9 @@ jobs:
- { board: adafruit_hallowing_m4, platform: adafruit, archi: samd, platform-version: 1.7.10, platform-url: 'https://adafruit.github.io/arduino-board-index/package_adafruit_index.json', ... }
- { board: adafruit_pybadge_m4, platform: adafruit, archi: samd, platform-version: 1.7.10, platform-url: 'https://adafruit.github.io/arduino-board-index/package_adafruit_index.json', ... }
- { board: adafruit_pygamer_m4, platform: adafruit, archi: samd, platform-version: 1.7.10, platform-url: 'https://adafruit.github.io/arduino-board-index/package_adafruit_index.json', ... }
- { board: adafruit_feather_esp32s2_tft, platform: esp32, archi: esp32, platform-version: latest, platform-url: 'https://raw.githubusercontent.com/espressif/arduino-esp32/gh-pages/package_esp32_dev_index.json', ... }
- { board: adafruit_feather_esp32s3_tft, platform: esp32, archi: esp32, platform-version: latest, platform-url: 'https://raw.githubusercontent.com/espressif/arduino-esp32/gh-pages/package_esp32_dev_index.json', ... }
- { board: adafruit_funhouse_esp32s2, platform: esp32, archi: esp32, platform-version: latest, platform-url: 'https://raw.githubusercontent.com/espressif/arduino-esp32/gh-pages/package_esp32_dev_index.json', ... }
- { board: adafruit_feather_esp32s2_tft, platform: esp32, archi: esp32, platform-version: latest, platform-url: 'https://espressif.github.io/arduino-esp32/package_esp32_index.json', ... }
- { board: adafruit_feather_esp32s3_tft, platform: esp32, archi: esp32, platform-version: latest, platform-url: 'https://espressif.github.io/arduino-esp32/package_esp32_index.json', ... }
- { board: adafruit_funhouse_esp32s2, platform: esp32, archi: esp32, platform-version: latest, platform-url: 'https://espressif.github.io/arduino-esp32/package_esp32_index.json', ... }
- { board: rpipico, platform: rp2040, archi: rp2040, platform-version: 2.3.3, cli-args: '--build-property compiler.cpp.extra_flags=-DSKIP_I2C_TEST', platform-url: 'https://github.com/earlephilhower/arduino-pico/releases/download/global/package_rp2040_index.json', ... }

# 3D matrix applies to these:
Expand All @@ -75,7 +75,7 @@ jobs:
#- { board: esp32s3box, platform-version: 2.0.0 }
#- { board: esp32s3box, platform-version: 2.0.1 }
#- { board: esp32s3box, platform-version: 2.0.2 }
- { board: esp32s3box, platform-version: 2.0.4 }
#- { board: esp32s3box, platform-version: 2.0.4 }
#- { board: esp32s2, platform-version: 1.0.6 }
#- { board: esp32s2, platform-version: 2.0.0 }

Expand Down
115 changes: 62 additions & 53 deletions .github/workflows/IDFBuild.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,43 +14,48 @@ on:
- '**.c'
- '**IDFBuild.yml'
- 'CMakeLists.txt'
- 'esp-idf-versions.php'
pull_request:
workflow_dispatch:

jobs:
build:
name: idf ${{ matrix.idf-version }}@${{ matrix.idf-board }}>esp-idf_graphicstest


set_matrix:
name: Version planner ⊹
runs-on: ubuntu-latest
env:
max-versions: 3 # maximum core versions to test, starting at latest
outputs:
matrix: ${{steps.set-matrix.outputs.matrix}}
project_dir: ${{steps.set-matrix.outputs.project_dir}}
repo_url: ${{steps.set-matrix.outputs.repo_url}}

strategy:
matrix:
steps:
- name: Checkout
uses: actions/checkout@v3
with:
ref: ${{ github.event.pull_request.head.sha }}

esp-idf-fqbn:
- esp32@v4.1
- esp32@v4.3.1
- esp32@v4.4.6
- esp32s2@v4.4.6
- esp32s3@v4.4.6
- esp32@v5.0.4
- esp32s2@v5.0.4
- esp32s3@v5.0.4
- esp32@v5.1.2
- esp32s2@v5.1.2
- esp32s3@v5.1.2
- name: Setup matrix
id: set-matrix
run: |
matrix=`php .github/scripts/esp-idf-versions.php`
# echo $matrix | jq # debug
matrix="${matrix//'%'/'%25'}" # escape percent entities
matrix="${matrix//$'\n'/''}" # remove lf
matrix="${matrix//$'\r'/''}" # remove cr
echo "matrix=${matrix}" >> $GITHUB_OUTPUT
echo "project_dir=${{env.PROJECT_DIR}}" >> $GITHUB_OUTPUT
echo "repo_url=${{env.REPO_URL}}" >> $GITHUB_OUTPUT
include:
- { esp-idf-fqbn: esp32@v4.1, idf-board: esp32, idf-version: v4.1 }
- { esp-idf-fqbn: esp32@v4.3.1, idf-board: esp32, idf-version: v4.3.1 }
- { esp-idf-fqbn: esp32@v4.4.6, idf-board: esp32, idf-version: v4.4.6 }
- { esp-idf-fqbn: esp32s2@v4.4.6, idf-board: esp32s2, idf-version: v4.4.6 }
- { esp-idf-fqbn: esp32s3@v4.4.6, idf-board: esp32s3, idf-version: v4.4.6 }
- { esp-idf-fqbn: esp32@v5.0.4, idf-board: esp32, idf-version: v5.0.4 }
- { esp-idf-fqbn: esp32s2@v5.0.4, idf-board: esp32s2, idf-version: v5.0.4 }
- { esp-idf-fqbn: esp32s3@v5.0.4, idf-board: esp32s3, idf-version: v5.0.4 }
- { esp-idf-fqbn: esp32@v5.1.2, idf-board: esp32, idf-version: v5.1.2 }
- { esp-idf-fqbn: esp32s2@v5.1.2, idf-board: esp32s2, idf-version: v5.1.2 }
- { esp-idf-fqbn: esp32s3@v5.1.2, idf-board: esp32s3, idf-version: v5.1.2 }
build:
name: idf ${{ matrix.esp-idf-fqbn }}
needs: set_matrix
runs-on: ubuntu-latest

strategy:
matrix: ${{fromJSON(needs.set_matrix.outputs.matrix)}}
fail-fast: false

steps:
Expand All @@ -59,41 +64,45 @@ jobs:
with:
ref: ${{ github.event.pull_request.head.sha }}

- name: Cache pip for ${{ matrix.esp-idf-fqbn }}
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ matrix.idf-version }}-${{ hashFiles('**/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Cache espressif tools for ${{ matrix.esp-idf-fqbn }}
uses: actions/cache@v3
id: espressif
with:
path: |
~/.espressif
key: ${{ runner.os }}-espressif-${{ matrix.idf-version }}-${{ hashFiles('**/lockfiles') }}

- name: Cache esp-idf for ${{ matrix.esp-idf-fqbn }}
id: cache-idf
uses: actions/cache@v3
with:
path: ~/esp/esp-idf
key: ${{ runner.os }}-idf-${{ matrix.idf-version }}-${{ hashFiles('**/lockfiles') }}
# - name: Cache pip for ${{ matrix.esp-idf-fqbn }}
# uses: actions/cache@v3
# with:
# path: ~/.cache/pip
# key: ${{ runner.os }}-pip-${{ matrix.esp-idf-fqbn }}-${{ hashFiles('**/requirements.txt') }}
# restore-keys: |
# ${{ runner.os }}-pip-
#
# - name: Cache espressif tools for ${{ matrix.esp-idf-fqbn }}
# uses: actions/cache@v3
# id: espressif
# with:
# path: |
# ~/.espressif
# key: ${{ runner.os }}-espressif-${{ matrix.esp-idf-fqbn }}-${{ hashFiles('**/lockfiles') }}
#
# - name: Cache esp-idf for ${{ matrix.esp-idf-fqbn }}
# id: cache-idf
# uses: actions/cache@v3
# with:
# path: ~/esp/esp-idf
# key: ${{ runner.os }}-idf-${{ matrix.esp-idf-fqbn }}-${{ hashFiles('**/lockfiles') }}

- name: Get/Check IDF ${{ matrix.esp-idf-fqbn }}
run: |
mkdir -p ~/esp
cd ~/esp
if [ ! -d "./esp-idf/" ]; then git clone -b ${{ matrix.idf-version }} --recursive $REPO_URL esp-idf; fi
idf_fqbn="${{ matrix.esp-idf-fqbn }}"
idf_version=${idf_fqbn#*@}
if [ ! -d "./esp-idf/" ]; then git clone -b v$idf_version --recursive ${{ needs.set_matrix.outputs.repo_url }} esp-idf; fi
cd ~/esp/esp-idf
if [ ! -d "~/.espressif" ]; then ./install.sh; fi
- name: Build example for ${{ matrix.esp-idf-fqbn }}
run: |
source ~/esp/esp-idf/export.sh
cd ${{ env.PROJECT_DIR }}
idf.py set-target ${{ matrix.idf-board }}
idf_fqbn="${{ matrix.esp-idf-fqbn }}"
idf_board=${idf_fqbn%%@*}
cd ${{ needs.set_matrix.outputs.project_dir }}
idf.py set-target $idf_board
idf.py build
10 changes: 4 additions & 6 deletions .github/workflows/PlatformioBuild.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,17 +36,15 @@ jobs:

platform-version:
- 1.0.6
#- 2.0.4
- 2.0.5
- 2.0.6
- 2.0.7
- 2.0.8
- 2.0.11
- 2.0.12
- 2.0.13
- default

exclude:
- { board: esp32-c3, platform-version: 1.0.6 }
- { board: esp32-s3, platform-version: 1.0.6 }
- { board: esp32-s3, platform-version: default } # 2.0.5 => esp32s3/include/newlib/platform_include/assert.h:20:10: fatal error: sdkconfig.h: No such file or directory
- { board: esp32-s3, platform-version: default }
- { board: esp32-s2, platform-version: 1.0.6 }
- { board: m5stack-cores3, platform-version: 1.0.6 }

Expand Down
Loading

0 comments on commit 6ef928c

Please sign in to comment.