From ea50cf6f5f83ed4ea4048ada08728c9827609a49 Mon Sep 17 00:00:00 2001 From: Me No Dev Date: Wed, 2 Oct 2024 22:18:17 +0300 Subject: [PATCH 01/12] fix(spiram): Fix OPI PSRAM init (#10406) --- cores/esp32/esp32-hal-misc.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/cores/esp32/esp32-hal-misc.c b/cores/esp32/esp32-hal-misc.c index dd7a87a4f22..86140144613 100644 --- a/cores/esp32/esp32-hal-misc.c +++ b/cores/esp32/esp32-hal-misc.c @@ -251,13 +251,11 @@ extern bool btInUse(); #endif #if CONFIG_SPIRAM_SUPPORT || CONFIG_SPIRAM -#ifndef CONFIG_SPIRAM_BOOT_INIT ESP_SYSTEM_INIT_FN(init_psram_new, BIT(0), 99) { psramInit(); return ESP_OK; } #endif -#endif void initArduino() { //init proper ref tick value for PLL (uncomment if REF_TICK is different than 1MHz) From 975377dbadbfa9b3933047626a040e0d1a59108f Mon Sep 17 00:00:00 2001 From: Lucas Saavedra Vaz <32426024+lucasssvaz@users.noreply.github.com> Date: Mon, 30 Sep 2024 21:14:09 -0300 Subject: [PATCH 02/12] ci(partitions): Use default partition for compilation in CI --- .github/scripts/sketch_utils.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/scripts/sketch_utils.sh b/.github/scripts/sketch_utils.sh index fa1458de6f2..417ecd47d35 100755 --- a/.github/scripts/sketch_utils.sh +++ b/.github/scripts/sketch_utils.sh @@ -85,12 +85,12 @@ function build_sketch(){ # build_sketch [ex # Default FQBN options if none were passed in the command line. - esp32_opts="PSRAM=enabled,PartitionScheme=huge_app,FlashMode=dio" - esp32s2_opts="PSRAM=enabled,PartitionScheme=huge_app,FlashMode=dio" - esp32s3_opts="PSRAM=opi,USBMode=default,PartitionScheme=huge_app,FlashMode=dio" - esp32c3_opts="PartitionScheme=huge_app,FlashMode=dio" - esp32c6_opts="PartitionScheme=huge_app,FlashMode=dio" - esp32h2_opts="PartitionScheme=huge_app,FlashMode=dio" + esp32_opts="PSRAM=enabled,FlashMode=dio" + esp32s2_opts="PSRAM=enabled,FlashMode=dio" + esp32s3_opts="PSRAM=opi,USBMode=default,FlashMode=dio" + esp32c3_opts="FlashMode=dio" + esp32c6_opts="FlashMode=dio" + esp32h2_opts="FlashMode=dio" # Select the common part of the FQBN based on the target. The rest will be # appended depending on the passed options. From 98e9712d54f2ecc7225c064acb4176634f5a94f2 Mon Sep 17 00:00:00 2001 From: Lucas Saavedra Vaz <32426024+lucasssvaz@users.noreply.github.com> Date: Wed, 2 Oct 2024 13:22:03 -0300 Subject: [PATCH 03/12] fix(ci): Fix paths for sdkconfig --- .github/scripts/install-platformio-esp32.sh | 6 +++--- .github/scripts/sketch_utils.sh | 4 ++-- .github/scripts/tests_run.sh | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/scripts/install-platformio-esp32.sh b/.github/scripts/install-platformio-esp32.sh index 393c9f3a7d6..7df0e999476 100755 --- a/.github/scripts/install-platformio-esp32.sh +++ b/.github/scripts/install-platformio-esp32.sh @@ -6,7 +6,7 @@ PLATFORMIO_ESP32_URL="https://github.com/platformio/platform-espressif32.git" TOOLCHAIN_VERSION="12.2.0+20230208" ESPTOOLPY_VERSION="~1.40501.0" ESPRESSIF_ORGANIZATION_NAME="espressif" -LIBS_DIR="tools/esp32-arduino-libs" +LIBS_DIR="$PLATFORMIO_ESP32_PATH/tools/esp32-arduino-libs" echo "Installing Python Wheel ..." pip install wheel > /dev/null 2>&1 @@ -100,7 +100,7 @@ function count_sketches(){ # count_sketches requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json) if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then for requirement in $requirements; do - found_line=$(grep -E "^$requirement" $LIBS_DIR/esp32/sdkconfig) + found_line=$(grep -E "^$requirement" "$LIBS_DIR/esp32/sdkconfig") if [[ "$found_line" == "" ]]; then continue 2 fi @@ -190,7 +190,7 @@ function build_pio_sketches(){ # build_pio_sketches [extra-options] while [ ! -z "$1" ]; do @@ -154,7 +154,7 @@ function build_sketch(){ # build_sketch [ex requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json) if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then for requirement in $requirements; do - found_line=$(grep -E "^$requirement" $LIBS_DIR/$target/sdkconfig) + found_line=$(grep -E "^$requirement" "$LIBS_DIR/$target/sdkconfig") if [[ "$found_line" == "" ]]; then echo "Target $target does not meet the requirement $requirement for $sketchname. Skipping." exit 0 diff --git a/.github/scripts/tests_run.sh b/.github/scripts/tests_run.sh index 7f990bea45d..4992f6d8ac3 100755 --- a/.github/scripts/tests_run.sh +++ b/.github/scripts/tests_run.sh @@ -25,7 +25,7 @@ function run_test() { requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json) if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then for requirement in $requirements; do - found_line=$(grep -E "^$requirement" $LIBS_DIR/$target/sdkconfig) + found_line=$(grep -E "^$requirement" "$LIBS_DIR/$target/sdkconfig") if [[ "$found_line" == "" ]]; then printf "\033[93mTarget $target does not meet the requirement $requirement for $sketchname. Skipping.\033[0m\n" printf "\n\n\n" @@ -120,7 +120,7 @@ function run_test() { SCRIPTS_DIR="./.github/scripts" COUNT_SKETCHES="${SCRIPTS_DIR}/sketch_utils.sh count" -LIBS_DIR="tools/esp32-arduino-libs" +LIBS_DIR="$ARDUINO_ESP32_PATH/tools/esp32-arduino-libs" platform="hardware" wokwi_timeout=60000 From 2ef9f78df6ab8a23d9a23a427211aa7632ce8cc5 Mon Sep 17 00:00:00 2001 From: me-no-dev Date: Tue, 1 Oct 2024 12:50:04 +0300 Subject: [PATCH 04/12] Fix build of camera web server --- .../ESP32/examples/Camera/CameraWebServer/ci.json | 15 +++++++++++++++ .../Camera/CameraWebServer/partitions.csv | 5 +++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/libraries/ESP32/examples/Camera/CameraWebServer/ci.json b/libraries/ESP32/examples/Camera/CameraWebServer/ci.json index 7e0f3c89986..35c3056dda8 100644 --- a/libraries/ESP32/examples/Camera/CameraWebServer/ci.json +++ b/libraries/ESP32/examples/Camera/CameraWebServer/ci.json @@ -1,4 +1,19 @@ { + "fqbn": { + "esp32": [ + "espressif:esp32:esp32:PSRAM=enabled,PartitionScheme=custom,FlashMode=dio", + "espressif:esp32:esp32:PSRAM=disabled,PartitionScheme=custom,FlashMode=dio" + ], + "esp32s2": [ + "espressif:esp32:esp32s2:PSRAM=enabled,PartitionScheme=custom,FlashMode=dio", + "espressif:esp32:esp32s2:PSRAM=disabled,PartitionScheme=custom,FlashMode=dio" + ], + "esp32s3": [ + "espressif:esp32:esp32s3:PSRAM=opi,USBMode=default,PartitionScheme=custom,FlashMode=qio", + "espressif:esp32:esp32s3:PSRAM=enabled,USBMode=default,PartitionScheme=custom,FlashMode=qio", + "espressif:esp32:esp32s3:PSRAM=disabled,USBMode=default,PartitionScheme=custom,FlashMode=qio" + ] + }, "requires": [ "CONFIG_CAMERA_TASK_STACK_SIZE=[0-9]+" ] diff --git a/libraries/ESP32/examples/Camera/CameraWebServer/partitions.csv b/libraries/ESP32/examples/Camera/CameraWebServer/partitions.csv index 4f76ca6d746..b9f18c465a7 100644 --- a/libraries/ESP32/examples/Camera/CameraWebServer/partitions.csv +++ b/libraries/ESP32/examples/Camera/CameraWebServer/partitions.csv @@ -1,5 +1,6 @@ # Name, Type, SubType, Offset, Size, Flags nvs, data, nvs, 0x9000, 0x5000, otadata, data, ota, 0xe000, 0x2000, -app0, app, ota_0, 0x10000, 0x3d0000, -fr, data, , 0x3e0000, 0x20000, +app0, app, ota_0, 0x10000, 0x3c0000, +fr, data, , 0x3d0000, 0x20000, +coredump, data, coredump,0x3f0000, 0x10000, From a4e32c140dc2084dd3f4aaeeabac0d71c01489b4 Mon Sep 17 00:00:00 2001 From: Lucas Saavedra Vaz <32426024+lucasssvaz@users.noreply.github.com> Date: Wed, 2 Oct 2024 16:44:54 -0300 Subject: [PATCH 05/12] fix(ci): Fix test requirements check --- .github/scripts/install-platformio-esp32.sh | 6 ++-- .github/scripts/sketch_utils.sh | 6 ++-- .github/scripts/tests_run.sh | 35 ++++++++++++++------- .github/workflows/tests_build.yml | 2 ++ platform.txt | 4 +++ 5 files changed, 36 insertions(+), 17 deletions(-) diff --git a/.github/scripts/install-platformio-esp32.sh b/.github/scripts/install-platformio-esp32.sh index 7df0e999476..69e1c86131f 100755 --- a/.github/scripts/install-platformio-esp32.sh +++ b/.github/scripts/install-platformio-esp32.sh @@ -6,7 +6,7 @@ PLATFORMIO_ESP32_URL="https://github.com/platformio/platform-espressif32.git" TOOLCHAIN_VERSION="12.2.0+20230208" ESPTOOLPY_VERSION="~1.40501.0" ESPRESSIF_ORGANIZATION_NAME="espressif" -LIBS_DIR="$PLATFORMIO_ESP32_PATH/tools/esp32-arduino-libs" +SDKCONFIG_DIR="$PLATFORMIO_ESP32_PATH/tools/esp32-arduino-libs" echo "Installing Python Wheel ..." pip install wheel > /dev/null 2>&1 @@ -100,7 +100,7 @@ function count_sketches(){ # count_sketches requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json) if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then for requirement in $requirements; do - found_line=$(grep -E "^$requirement" "$LIBS_DIR/esp32/sdkconfig") + found_line=$(grep -E "^$requirement" "$SDKCONFIG_DIR/esp32/sdkconfig") if [[ "$found_line" == "" ]]; then continue 2 fi @@ -190,7 +190,7 @@ function build_pio_sketches(){ # build_pio_sketches [extra-options] while [ ! -z "$1" ]; do @@ -154,7 +154,7 @@ function build_sketch(){ # build_sketch [ex requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json) if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then for requirement in $requirements; do - found_line=$(grep -E "^$requirement" "$LIBS_DIR/$target/sdkconfig") + found_line=$(grep -E "^$requirement" "$SDKCONFIG_DIR/$target/sdkconfig") if [[ "$found_line" == "" ]]; then echo "Target $target does not meet the requirement $requirement for $sketchname. Skipping." exit 0 @@ -310,7 +310,7 @@ function count_sketches(){ # count_sketches [target] [file] requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json) if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then for requirement in $requirements; do - found_line=$(grep -E "^$requirement" $LIBS_DIR/$target/sdkconfig) + found_line=$(grep -E "^$requirement" $SDKCONFIG_DIR/$target/sdkconfig) if [[ "$found_line" == "" ]]; then continue 2 fi diff --git a/.github/scripts/tests_run.sh b/.github/scripts/tests_run.sh index 4992f6d8ac3..ce348ff6994 100755 --- a/.github/scripts/tests_run.sh +++ b/.github/scripts/tests_run.sh @@ -10,6 +10,15 @@ function run_test() { local result=0 local error=0 + if [ $options -eq 0 ] && [ -f $sketchdir/ci.json ]; then + len=`jq -r --arg target $target '.fqbn[$target] | length' $sketchdir/ci.json` + if [ $len -eq 0 ]; then + len=1 + fi + else + len=1 + fi + if [ -f $sketchdir/ci.json ]; then # If the target or platform is listed as false, skip the sketch. Otherwise, include it. is_target=$(jq -r --arg target $target '.targets[$target]' $sketchdir/ci.json) @@ -21,11 +30,21 @@ function run_test() { return 0 fi + if [ -d $ARDUINO_ESP32_PATH/tools/esp32-arduino-libs ]; then + SDKCONFIG_PATH="$ARDUINO_ESP32_PATH/tools/esp32-arduino-libs/$target/sdkconfig" + else + if [ $len -eq 1 ]; then + SDKCONFIG_PATH="$HOME/.arduino/tests/$sketchname/build.tmp/sdkconfig" + else + SDKCONFIG_PATH="$HOME/.arduino/tests/$sketchname/build0.tmp/sdkconfig" + fi + fi + # Check if the sketch requires any configuration options requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json) if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then for requirement in $requirements; do - found_line=$(grep -E "^$requirement" "$LIBS_DIR/$target/sdkconfig") + found_line=$(grep -E "^$requirement" "$SDKCONFIG_PATH") if [[ "$found_line" == "" ]]; then printf "\033[93mTarget $target does not meet the requirement $requirement for $sketchname. Skipping.\033[0m\n" printf "\n\n\n" @@ -35,15 +54,6 @@ function run_test() { fi fi - if [ $options -eq 0 ] && [ -f $sketchdir/ci.json ]; then - len=`jq -r --arg target $target '.fqbn[$target] | length' $sketchdir/ci.json` - if [ $len -eq 0 ]; then - len=1 - fi - else - len=1 - fi - if [ $len -eq 1 ]; then # build_dir="$sketchdir/build" build_dir="$HOME/.arduino/tests/$sketchname/build.tmp" @@ -120,7 +130,10 @@ function run_test() { SCRIPTS_DIR="./.github/scripts" COUNT_SKETCHES="${SCRIPTS_DIR}/sketch_utils.sh count" -LIBS_DIR="$ARDUINO_ESP32_PATH/tools/esp32-arduino-libs" + +if [ -d $ARDUINO_ESP32_PATH/tools/esp32-arduino-libs ]; then + SDKCONFIG_DIR="$ARDUINO_ESP32_PATH/tools/esp32-arduino-libs" +fi platform="hardware" wokwi_timeout=60000 diff --git a/.github/workflows/tests_build.yml b/.github/workflows/tests_build.yml index f865078b1c3..b02e23ebd70 100644 --- a/.github/workflows/tests_build.yml +++ b/.github/workflows/tests_build.yml @@ -75,6 +75,7 @@ jobs: ~/.arduino/tests/**/build*.tmp/*.bin ~/.arduino/tests/**/build*.tmp/*.elf ~/.arduino/tests/**/build*.tmp/*.json + ~/.arduino/tests/**/build*.tmp/sdkconfig - name: Upload ${{ inputs.chip }} ${{ inputs.type }} binaries as artifacts uses: actions/upload-artifact@v4 @@ -85,3 +86,4 @@ jobs: ~/.arduino/tests/**/build*.tmp/*.bin ~/.arduino/tests/**/build*.tmp/*.elf ~/.arduino/tests/**/build*.tmp/*.json + ~/.arduino/tests/**/build*.tmp/sdkconfig diff --git a/platform.txt b/platform.txt index 873e27b0ecb..a1574e2f776 100644 --- a/platform.txt +++ b/platform.txt @@ -143,6 +143,10 @@ recipe.hooks.prebuild.7.pattern.windows=cmd /c type nul > "{file_opts.path}" recipe.hooks.core.prebuild.1.pattern.windows=cmd /c echo "-DARDUINO_CORE_BUILD" > "{file_opts.path}" recipe.hooks.core.postbuild.1.pattern.windows=cmd /c type nul > "{file_opts.path}" +# Copy sdkconfig to build folder +recipe.hooks.prebuild.8.pattern=/usr/bin/env bash -c "cp -f "{runtime.platform.path}"/tools/esp32-arduino-libs/{build.mcu}/sdkconfig "{build.path}"/sdkconfig" +recipe.hooks.prebuild.8.pattern.windows=cmd /c COPY /y "{runtime.platform.path}\tools\esp32-arduino-libs\{build.mcu}\sdkconfig" "{build.path}\sdkconfig" + ## Compile c files recipe.c.o.pattern="{compiler.path}{compiler.c.cmd}" {compiler.c.extra_flags} {compiler.c.flags} -DF_CPU={build.f_cpu} -DARDUINO={runtime.ide.version} -DARDUINO_{build.board} -DARDUINO_ARCH_{build.arch} -DARDUINO_BOARD="{build.board}" -DARDUINO_VARIANT="{build.variant}" -DARDUINO_PARTITION_{build.partitions} {build.extra_flags} {compiler.cpreprocessor.flags} {includes} "@{build.opt.path}" "@{file_opts.path}" "{source_file}" -o "{object_file}" From c415743a76b1dbaa5fbbbb2f1c97789a6a658ec9 Mon Sep 17 00:00:00 2001 From: Lucas Saavedra Vaz <32426024+lucasssvaz@users.noreply.github.com> Date: Wed, 2 Oct 2024 17:22:02 -0300 Subject: [PATCH 06/12] ci(append): Add option to append to all FQBNs --- .github/scripts/sketch_utils.sh | 19 +++++++++++++------ .github/scripts/tests_run.sh | 4 ---- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/scripts/sketch_utils.sh b/.github/scripts/sketch_utils.sh index 4b6a5cc74f7..cbdcb45b55f 100755 --- a/.github/scripts/sketch_utils.sh +++ b/.github/scripts/sketch_utils.sh @@ -83,14 +83,21 @@ function build_sketch(){ # build_sketch [ex len=1 + if [ -f $sketchdir/ci.json ]; then + fqbn_append=`jq -r '.fqbn_append' $sketchdir/ci.json` + if [ $fqbn_append == "null" ]; then + fqbn_append="" + fi + fi + # Default FQBN options if none were passed in the command line. - esp32_opts="PSRAM=enabled,FlashMode=dio" - esp32s2_opts="PSRAM=enabled,FlashMode=dio" - esp32s3_opts="PSRAM=opi,USBMode=default,FlashMode=dio" - esp32c3_opts="FlashMode=dio" - esp32c6_opts="FlashMode=dio" - esp32h2_opts="FlashMode=dio" + esp32_opts="PSRAM=enabled,FlashMode=dio${fqbn_append:+,$fqbn_append}" + esp32s2_opts="PSRAM=enabled,FlashMode=dio${fqbn_append:+,$fqbn_append}" + esp32s3_opts="PSRAM=opi,USBMode=default,FlashMode=dio${fqbn_append:+,$fqbn_append}" + esp32c3_opts="FlashMode=dio${fqbn_append:+,$fqbn_append}" + esp32c6_opts="FlashMode=dio${fqbn_append:+,$fqbn_append}" + esp32h2_opts="FlashMode=dio${fqbn_append:+,$fqbn_append}" # Select the common part of the FQBN based on the target. The rest will be # appended depending on the passed options. diff --git a/.github/scripts/tests_run.sh b/.github/scripts/tests_run.sh index ce348ff6994..f2d7288728b 100755 --- a/.github/scripts/tests_run.sh +++ b/.github/scripts/tests_run.sh @@ -131,10 +131,6 @@ function run_test() { SCRIPTS_DIR="./.github/scripts" COUNT_SKETCHES="${SCRIPTS_DIR}/sketch_utils.sh count" -if [ -d $ARDUINO_ESP32_PATH/tools/esp32-arduino-libs ]; then - SDKCONFIG_DIR="$ARDUINO_ESP32_PATH/tools/esp32-arduino-libs" -fi - platform="hardware" wokwi_timeout=60000 chunk_run=0 From c12c2389d2053cf30d67043d9a392243882203b7 Mon Sep 17 00:00:00 2001 From: Lucas Saavedra Vaz <32426024+lucasssvaz@users.noreply.github.com> Date: Wed, 2 Oct 2024 17:28:59 -0300 Subject: [PATCH 07/12] fix(json): Fix JSON files to compile examples --- libraries/RainMaker/examples/RMakerCustom/ci.json | 3 ++- libraries/RainMaker/examples/RMakerCustomAirCooler/ci.json | 3 ++- libraries/RainMaker/examples/RMakerSonoffDualR3/ci.json | 3 ++- libraries/RainMaker/examples/RMakerSwitch/ci.json | 3 ++- libraries/WiFiProv/examples/WiFiProv/ci.json | 1 + 5 files changed, 9 insertions(+), 4 deletions(-) diff --git a/libraries/RainMaker/examples/RMakerCustom/ci.json b/libraries/RainMaker/examples/RMakerCustom/ci.json index de4e92436d7..20037b733fe 100644 --- a/libraries/RainMaker/examples/RMakerCustom/ci.json +++ b/libraries/RainMaker/examples/RMakerCustom/ci.json @@ -1,6 +1,7 @@ { + "fqbn_append": "PartitionScheme=huge_app", "requires": [ "CONFIG_SOC_WIFI_SUPPORTED=y", - "CONFIG_ESP_RMAKER_WORK_QUEUE_TASK_STACK" + "CONFIG_ESP_RMAKER_WORK_QUEUE_TASK_STACK=[1-9][0-9]*" ] } diff --git a/libraries/RainMaker/examples/RMakerCustomAirCooler/ci.json b/libraries/RainMaker/examples/RMakerCustomAirCooler/ci.json index de4e92436d7..20037b733fe 100644 --- a/libraries/RainMaker/examples/RMakerCustomAirCooler/ci.json +++ b/libraries/RainMaker/examples/RMakerCustomAirCooler/ci.json @@ -1,6 +1,7 @@ { + "fqbn_append": "PartitionScheme=huge_app", "requires": [ "CONFIG_SOC_WIFI_SUPPORTED=y", - "CONFIG_ESP_RMAKER_WORK_QUEUE_TASK_STACK" + "CONFIG_ESP_RMAKER_WORK_QUEUE_TASK_STACK=[1-9][0-9]*" ] } diff --git a/libraries/RainMaker/examples/RMakerSonoffDualR3/ci.json b/libraries/RainMaker/examples/RMakerSonoffDualR3/ci.json index de4e92436d7..20037b733fe 100644 --- a/libraries/RainMaker/examples/RMakerSonoffDualR3/ci.json +++ b/libraries/RainMaker/examples/RMakerSonoffDualR3/ci.json @@ -1,6 +1,7 @@ { + "fqbn_append": "PartitionScheme=huge_app", "requires": [ "CONFIG_SOC_WIFI_SUPPORTED=y", - "CONFIG_ESP_RMAKER_WORK_QUEUE_TASK_STACK" + "CONFIG_ESP_RMAKER_WORK_QUEUE_TASK_STACK=[1-9][0-9]*" ] } diff --git a/libraries/RainMaker/examples/RMakerSwitch/ci.json b/libraries/RainMaker/examples/RMakerSwitch/ci.json index de4e92436d7..20037b733fe 100644 --- a/libraries/RainMaker/examples/RMakerSwitch/ci.json +++ b/libraries/RainMaker/examples/RMakerSwitch/ci.json @@ -1,6 +1,7 @@ { + "fqbn_append": "PartitionScheme=huge_app", "requires": [ "CONFIG_SOC_WIFI_SUPPORTED=y", - "CONFIG_ESP_RMAKER_WORK_QUEUE_TASK_STACK" + "CONFIG_ESP_RMAKER_WORK_QUEUE_TASK_STACK=[1-9][0-9]*" ] } diff --git a/libraries/WiFiProv/examples/WiFiProv/ci.json b/libraries/WiFiProv/examples/WiFiProv/ci.json index 36babb82730..04eb62b977a 100644 --- a/libraries/WiFiProv/examples/WiFiProv/ci.json +++ b/libraries/WiFiProv/examples/WiFiProv/ci.json @@ -1,4 +1,5 @@ { + "fqbn_append": "PartitionScheme=huge_app", "requires": [ "CONFIG_SOC_WIFI_SUPPORTED=y" ] From 7cdca56cb6f6a81860bb39bb9746d8d16f614a48 Mon Sep 17 00:00:00 2001 From: Lucas Saavedra Vaz <32426024+lucasssvaz@users.noreply.github.com> Date: Wed, 2 Oct 2024 17:46:51 -0300 Subject: [PATCH 08/12] fix(example): Use requires instead of target in ci.json fix(zigbee): Improve JSON files Co-authored-by: Jan Prochazka <90197375+P-R-O-C-H-Y@users.noreply.github.com> --- .../Zigbee_Color_Dimmable_Light/ci.json | 18 ++++-------------- .../Zigbee_Color_Dimmer_Switch/ci.json | 18 ++++-------------- .../examples/Zigbee_On_Off_Light/ci.json | 18 ++++-------------- .../examples/Zigbee_On_Off_Switch/ci.json | 18 ++++-------------- .../examples/Zigbee_Scan_Networks/ci.json | 18 ++++-------------- .../examples/Zigbee_Temperature_Sensor/ci.json | 18 ++++-------------- .../Zigbee/examples/Zigbee_Thermostat/ci.json | 10 ++++------ 7 files changed, 28 insertions(+), 90 deletions(-) diff --git a/libraries/Zigbee/examples/Zigbee_Color_Dimmable_Light/ci.json b/libraries/Zigbee/examples/Zigbee_Color_Dimmable_Light/ci.json index 3aaf44eb376..7b7ccef8ed7 100644 --- a/libraries/Zigbee/examples/Zigbee_Color_Dimmable_Light/ci.json +++ b/libraries/Zigbee/examples/Zigbee_Color_Dimmable_Light/ci.json @@ -1,16 +1,6 @@ { - "fqbn": { - "esp32c6": [ - "espressif:esp32:esp32c6:PartitionScheme=zigbee,ZigbeeMode=ed" - ], - "esp32h2": [ - "espressif:esp32:esp32h2:PartitionScheme=zigbee,ZigbeeMode=ed" - ] - }, - "targets": { - "esp32": false, - "esp32c3": false, - "esp32s2": false, - "esp32s3": false - } + "fqbn_append": "PartitionScheme=zigbee,ZigbeeMode=ed", + "requires": [ + "CONFIG_SOC_IEEE802154_SUPPORTED=y" + ] } diff --git a/libraries/Zigbee/examples/Zigbee_Color_Dimmer_Switch/ci.json b/libraries/Zigbee/examples/Zigbee_Color_Dimmer_Switch/ci.json index c916121b991..e79a477da11 100644 --- a/libraries/Zigbee/examples/Zigbee_Color_Dimmer_Switch/ci.json +++ b/libraries/Zigbee/examples/Zigbee_Color_Dimmer_Switch/ci.json @@ -1,16 +1,6 @@ { - "fqbn": { - "esp32c6": [ - "espressif:esp32:esp32c6:PartitionScheme=zigbee_zczr,ZigbeeMode=zczr" - ], - "esp32h2": [ - "espressif:esp32:esp32h2:PartitionScheme=zigbee_zczr,ZigbeeMode=zczr" - ] - }, - "targets": { - "esp32": false, - "esp32c3": false, - "esp32s2": false, - "esp32s3": false - } + "fqbn_append": "PartitionScheme=zigbee_zczr,ZigbeeMode=zczr", + "requires": [ + "CONFIG_SOC_IEEE802154_SUPPORTED=y" + ] } diff --git a/libraries/Zigbee/examples/Zigbee_On_Off_Light/ci.json b/libraries/Zigbee/examples/Zigbee_On_Off_Light/ci.json index 3aaf44eb376..7b7ccef8ed7 100644 --- a/libraries/Zigbee/examples/Zigbee_On_Off_Light/ci.json +++ b/libraries/Zigbee/examples/Zigbee_On_Off_Light/ci.json @@ -1,16 +1,6 @@ { - "fqbn": { - "esp32c6": [ - "espressif:esp32:esp32c6:PartitionScheme=zigbee,ZigbeeMode=ed" - ], - "esp32h2": [ - "espressif:esp32:esp32h2:PartitionScheme=zigbee,ZigbeeMode=ed" - ] - }, - "targets": { - "esp32": false, - "esp32c3": false, - "esp32s2": false, - "esp32s3": false - } + "fqbn_append": "PartitionScheme=zigbee,ZigbeeMode=ed", + "requires": [ + "CONFIG_SOC_IEEE802154_SUPPORTED=y" + ] } diff --git a/libraries/Zigbee/examples/Zigbee_On_Off_Switch/ci.json b/libraries/Zigbee/examples/Zigbee_On_Off_Switch/ci.json index c916121b991..e79a477da11 100644 --- a/libraries/Zigbee/examples/Zigbee_On_Off_Switch/ci.json +++ b/libraries/Zigbee/examples/Zigbee_On_Off_Switch/ci.json @@ -1,16 +1,6 @@ { - "fqbn": { - "esp32c6": [ - "espressif:esp32:esp32c6:PartitionScheme=zigbee_zczr,ZigbeeMode=zczr" - ], - "esp32h2": [ - "espressif:esp32:esp32h2:PartitionScheme=zigbee_zczr,ZigbeeMode=zczr" - ] - }, - "targets": { - "esp32": false, - "esp32c3": false, - "esp32s2": false, - "esp32s3": false - } + "fqbn_append": "PartitionScheme=zigbee_zczr,ZigbeeMode=zczr", + "requires": [ + "CONFIG_SOC_IEEE802154_SUPPORTED=y" + ] } diff --git a/libraries/Zigbee/examples/Zigbee_Scan_Networks/ci.json b/libraries/Zigbee/examples/Zigbee_Scan_Networks/ci.json index 3aaf44eb376..7b7ccef8ed7 100644 --- a/libraries/Zigbee/examples/Zigbee_Scan_Networks/ci.json +++ b/libraries/Zigbee/examples/Zigbee_Scan_Networks/ci.json @@ -1,16 +1,6 @@ { - "fqbn": { - "esp32c6": [ - "espressif:esp32:esp32c6:PartitionScheme=zigbee,ZigbeeMode=ed" - ], - "esp32h2": [ - "espressif:esp32:esp32h2:PartitionScheme=zigbee,ZigbeeMode=ed" - ] - }, - "targets": { - "esp32": false, - "esp32c3": false, - "esp32s2": false, - "esp32s3": false - } + "fqbn_append": "PartitionScheme=zigbee,ZigbeeMode=ed", + "requires": [ + "CONFIG_SOC_IEEE802154_SUPPORTED=y" + ] } diff --git a/libraries/Zigbee/examples/Zigbee_Temperature_Sensor/ci.json b/libraries/Zigbee/examples/Zigbee_Temperature_Sensor/ci.json index 3aaf44eb376..7b7ccef8ed7 100644 --- a/libraries/Zigbee/examples/Zigbee_Temperature_Sensor/ci.json +++ b/libraries/Zigbee/examples/Zigbee_Temperature_Sensor/ci.json @@ -1,16 +1,6 @@ { - "fqbn": { - "esp32c6": [ - "espressif:esp32:esp32c6:PartitionScheme=zigbee,ZigbeeMode=ed" - ], - "esp32h2": [ - "espressif:esp32:esp32h2:PartitionScheme=zigbee,ZigbeeMode=ed" - ] - }, - "targets": { - "esp32": false, - "esp32c3": false, - "esp32s2": false, - "esp32s3": false - } + "fqbn_append": "PartitionScheme=zigbee,ZigbeeMode=ed", + "requires": [ + "CONFIG_SOC_IEEE802154_SUPPORTED=y" + ] } diff --git a/libraries/Zigbee/examples/Zigbee_Thermostat/ci.json b/libraries/Zigbee/examples/Zigbee_Thermostat/ci.json index c916121b991..db9830cc8af 100644 --- a/libraries/Zigbee/examples/Zigbee_Thermostat/ci.json +++ b/libraries/Zigbee/examples/Zigbee_Thermostat/ci.json @@ -1,4 +1,5 @@ { + "fqbn_append": "PartitionScheme=zigbee_zczr,ZigbeeMode=zczr", "fqbn": { "esp32c6": [ "espressif:esp32:esp32c6:PartitionScheme=zigbee_zczr,ZigbeeMode=zczr" @@ -7,10 +8,7 @@ "espressif:esp32:esp32h2:PartitionScheme=zigbee_zczr,ZigbeeMode=zczr" ] }, - "targets": { - "esp32": false, - "esp32c3": false, - "esp32s2": false, - "esp32s3": false - } + "requires": [ + "CONFIG_SOC_IEEE802154_SUPPORTED=y" + ] } From 7999f0874e389ce37ea2b590701d7e136c8c6495 Mon Sep 17 00:00:00 2001 From: Lucas Saavedra Vaz <32426024+lucasssvaz@users.noreply.github.com> Date: Wed, 2 Oct 2024 19:28:58 -0300 Subject: [PATCH 09/12] fix(regex): Trim argument before grep --- .github/scripts/install-platformio-esp32.sh | 2 ++ .github/scripts/sketch_utils.sh | 2 ++ .github/scripts/tests_run.sh | 1 + 3 files changed, 5 insertions(+) diff --git a/.github/scripts/install-platformio-esp32.sh b/.github/scripts/install-platformio-esp32.sh index 69e1c86131f..a9aab496e19 100755 --- a/.github/scripts/install-platformio-esp32.sh +++ b/.github/scripts/install-platformio-esp32.sh @@ -100,6 +100,7 @@ function count_sketches(){ # count_sketches requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json) if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then for requirement in $requirements; do + requirement=$(echo $requirement | xargs) found_line=$(grep -E "^$requirement" "$SDKCONFIG_DIR/esp32/sdkconfig") if [[ "$found_line" == "" ]]; then continue 2 @@ -190,6 +191,7 @@ function build_pio_sketches(){ # build_pio_sketches [ex requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json) if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then for requirement in $requirements; do + requirement=$(echo $requirement | xargs) found_line=$(grep -E "^$requirement" "$SDKCONFIG_DIR/$target/sdkconfig") if [[ "$found_line" == "" ]]; then echo "Target $target does not meet the requirement $requirement for $sketchname. Skipping." @@ -317,6 +318,7 @@ function count_sketches(){ # count_sketches [target] [file] requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json) if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then for requirement in $requirements; do + requirement=$(echo $requirement | xargs) found_line=$(grep -E "^$requirement" $SDKCONFIG_DIR/$target/sdkconfig) if [[ "$found_line" == "" ]]; then continue 2 diff --git a/.github/scripts/tests_run.sh b/.github/scripts/tests_run.sh index f2d7288728b..be084e90ce2 100755 --- a/.github/scripts/tests_run.sh +++ b/.github/scripts/tests_run.sh @@ -44,6 +44,7 @@ function run_test() { requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json) if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then for requirement in $requirements; do + requirement=$(echo $requirement | xargs) found_line=$(grep -E "^$requirement" "$SDKCONFIG_PATH") if [[ "$found_line" == "" ]]; then printf "\033[93mTarget $target does not meet the requirement $requirement for $sketchname. Skipping.\033[0m\n" From ad3f665c08250194c1566c87d72c0a20bec9eb6f Mon Sep 17 00:00:00 2001 From: Lucas Saavedra Vaz <32426024+lucasssvaz@users.noreply.github.com> Date: Wed, 2 Oct 2024 21:26:53 -0300 Subject: [PATCH 10/12] docs(ci): Add documentation about FQBNs in CI --- docs/en/contributing.rst | 46 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/docs/en/contributing.rst b/docs/en/contributing.rst index 26272f3d1c3..f4ed6c34761 100644 --- a/docs/en/contributing.rst +++ b/docs/en/contributing.rst @@ -166,8 +166,47 @@ And in the ``README.md`` file: Currently, this example requires Wi-Fi and supports the following targets. - | Supported Targets | ESP32 | ESP32-H2 | ESP32-S3 | ESP32-C3 | ESP32-C6 | - | ----------------- | ----- | -------- | -------- | -------- | -------- | + | Supported Targets | ESP32 | ESP32-S3 | ESP32-C3 | ESP32-C6 | + | ----------------- | ----- | -------- | -------- | -------- | + +By default, the CI system will use the FQBNs specified in the ``.github/scripts/sketch_utils.sh`` file to compile the sketches. +Currently, the default FQBNs are: + +* ``espressif:esp32:esp32:PSRAM=enabled,FlashMode=dio`` +* ``espressif:esp32:esp32s2:PSRAM=enabled,FlashMode=dio`` +* ``espressif:esp32:esp32s3:PSRAM=opi,USBMode=default,FlashMode=dio`` +* ``espressif:esp32:esp32c3:FlashMode=dio`` +* ``espressif:esp32:esp32c6:FlashMode=dio`` +* ``espressif:esp32:esp32h2:FlashMode=dio`` + +There are two ways to alter the FQBNs used to compile the sketches: by using the ``fqbn`` or ``fqbn_append`` fields in the ``ci.json`` file. + +If you just want to append a string to the default FQBNs, you can use the ``fqbn_append`` field. For example, to add the ``DebugLevel=debug`` to the FQBNs, you would use: + +.. code-block:: json + + { + "fqbn_append": "DebugLevel=debug" + } + +If you want to override the default FQBNs, you can use the ``fqbn`` field. It is a dictionary where the key is the target name and the value is a list of FQBNs. +The FQBNs in the list will be used in sequence to compile the sketch. For example, to compile a sketch for ESP32-S2 with and without PSRAM enabled, you would use: + +.. code-block:: json + + { + "fqbn": { + "esp32s2": [ + "espressif:esp32:esp32s2:PSRAM=enabled,FlashMode=dio", + "espressif:esp32:esp32s2:PSRAM=disabled,FlashMode=dio" + ] + } + } + +.. note:: + + The FQBNs specified in the ``fqbn`` field will also override the options specified in the ``fqbn_append`` field. + That means that if the ``fqbn`` field is specified, the ``fqbn_append`` field will be ignored and will have no effect. Example Template **************** @@ -376,9 +415,10 @@ The ``ci.json`` file is used to specify how the test suite and sketches will han * ``platforms``: A dictionary that specifies the supported platforms. The key is the platform name and the value is a boolean that specifies if the platform is supported. By default, all platforms are assumed to be supported. * ``extra_tags``: A list of extra tags that the runner will require when running the test suite in hardware. By default, no extra tags are required. +* ``fqbn_append``: A string to be appended to the default FQBNs. By default, no string is appended. This has no effect if ``fqbn`` is specified. * ``fqbn``: A dictionary that specifies the FQBNs that will be used to compile the sketch. The key is the target name and the value is a list of FQBNs. The `default FQBNs `_ - are used if this field is not specified. + are used if this field is not specified. This overrides the default FQBNs and the ``fqbn_append`` field. The ``wifi`` test suite is a good example of how to use the ``ci.json`` file: From 613536a4bc7e7193396d48f31b1df9f6f5ef58d7 Mon Sep 17 00:00:00 2001 From: Lucas Saavedra Vaz <32426024+lucasssvaz@users.noreply.github.com> Date: Wed, 2 Oct 2024 21:28:45 -0300 Subject: [PATCH 11/12] fix(json): Remove redundant FQBNs --- libraries/Zigbee/examples/Zigbee_Thermostat/ci.json | 8 -------- 1 file changed, 8 deletions(-) diff --git a/libraries/Zigbee/examples/Zigbee_Thermostat/ci.json b/libraries/Zigbee/examples/Zigbee_Thermostat/ci.json index db9830cc8af..e79a477da11 100644 --- a/libraries/Zigbee/examples/Zigbee_Thermostat/ci.json +++ b/libraries/Zigbee/examples/Zigbee_Thermostat/ci.json @@ -1,13 +1,5 @@ { "fqbn_append": "PartitionScheme=zigbee_zczr,ZigbeeMode=zczr", - "fqbn": { - "esp32c6": [ - "espressif:esp32:esp32c6:PartitionScheme=zigbee_zczr,ZigbeeMode=zczr" - ], - "esp32h2": [ - "espressif:esp32:esp32h2:PartitionScheme=zigbee_zczr,ZigbeeMode=zczr" - ] - }, "requires": [ "CONFIG_SOC_IEEE802154_SUPPORTED=y" ] From e1d6da4945944aac1c3b34bfb123ff66863c11a5 Mon Sep 17 00:00:00 2001 From: Lucas Saavedra Vaz <32426024+lucasssvaz@users.noreply.github.com> Date: Wed, 2 Oct 2024 23:34:41 -0300 Subject: [PATCH 12/12] fix(json): Skip requirements if libs are not installed --- .github/scripts/sketch_utils.sh | 33 +++++++++++++++++++++------------ .github/scripts/tests_run.sh | 2 +- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/.github/scripts/sketch_utils.sh b/.github/scripts/sketch_utils.sh index 8316c328e9c..b9ce41653f2 100755 --- a/.github/scripts/sketch_utils.sh +++ b/.github/scripts/sketch_utils.sh @@ -1,6 +1,12 @@ #!/bin/bash -SDKCONFIG_DIR="$ARDUINO_ESP32_PATH/tools/esp32-arduino-libs" +if [ -d "$ARDUINO_ESP32_PATH/tools/esp32-arduino-libs" ]; then + SDKCONFIG_DIR="$ARDUINO_ESP32_PATH/tools/esp32-arduino-libs" +elif [ -d "$GITHUB_WORKSPACE/tools/esp32-arduino-libs" ]; then + SDKCONFIG_DIR="$GITHUB_WORKSPACE/tools/esp32-arduino-libs" +else + SDKCONFIG_DIR="tools/esp32-arduino-libs" +fi function build_sketch(){ # build_sketch [extra-options] while [ ! -z "$1" ]; do @@ -278,10 +284,11 @@ function build_sketch(){ # build_sketch [ex unset options } -function count_sketches(){ # count_sketches [target] [file] +function count_sketches(){ # count_sketches [target] [file] [ignore-requirements] local path=$1 local target=$2 local file=$3 + local ignore_requirements=$4 if [ $# -lt 1 ]; then echo "ERROR: Illegal number of parameters" @@ -314,16 +321,18 @@ function count_sketches(){ # count_sketches [target] [file] continue fi - # Check if the sketch requires any configuration options - requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json) - if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then - for requirement in $requirements; do - requirement=$(echo $requirement | xargs) - found_line=$(grep -E "^$requirement" $SDKCONFIG_DIR/$target/sdkconfig) - if [[ "$found_line" == "" ]]; then - continue 2 - fi - done + if [[ "$ignore_requirements" != "1" ]]; then + # Check if the sketch requires any configuration options + requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json) + if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then + for requirement in $requirements; do + requirement=$(echo $requirement | xargs) + found_line=$(grep -E "^$requirement" $SDKCONFIG_DIR/$target/sdkconfig) + if [[ "$found_line" == "" ]]; then + continue 2 + fi + done + fi fi fi echo $sketch >> sketches.txt diff --git a/.github/scripts/tests_run.sh b/.github/scripts/tests_run.sh index be084e90ce2..dfef2e3b28c 100755 --- a/.github/scripts/tests_run.sh +++ b/.github/scripts/tests_run.sh @@ -233,7 +233,7 @@ else fi set +e - ${COUNT_SKETCHES} $test_folder $target + ${COUNT_SKETCHES} $test_folder $target "1" # Ignore requirements as we don't have the libs. The requirements will be checked in the run_test function sketchcount=$? set -e sketches=$(cat sketches.txt)