ci(compilation): Use default partition and add append to FQBN option (#…

…10392)

* ci(partitions): Use default partition for compilation in CI

* fix(ci): Fix paths for sdkconfig

* Fix build of camera web server

* fix(ci): Fix test requirements check

* ci(append): Add option to append to all FQBNs

* fix(json): Fix JSON files to compile examples

* fix(example): Use requires instead of target in ci.json

fix(zigbee): Improve JSON files

Co-authored-by: Jan Prochazka <90197375+P-R-O-C-H-Y@users.noreply.github.com>

* fix(regex): Trim argument before grep

* docs(ci): Add documentation about FQBNs in CI

* fix(json): Remove redundant FQBNs

* fix(json): Skip requirements if libs are not installed

* fix(partitions): Use rainmaker specific partitions

---------

Co-authored-by: me-no-dev <hristo@espressif.com>
Co-authored-by: Jan Prochazka <90197375+P-R-O-C-H-Y@users.noreply.github.com>

.github/scripts/install-platformio-esp32.sh

@@ -6,7 +6,7 @@ PLATFORMIO_ESP32_URL="https://github.com/platformio/platform-espressif32.git"
 TOOLCHAIN_VERSION="12.2.0+20230208"
 ESPTOOLPY_VERSION="~1.40501.0"
 ESPRESSIF_ORGANIZATION_NAME="espressif"
-LIBS_DIR="tools/esp32-arduino-libs"
+SDKCONFIG_DIR="$PLATFORMIO_ESP32_PATH/tools/esp32-arduino-libs"
 
 echo "Installing Python Wheel ..."
 pip install wheel > /dev/null 2>&1
@@ -100,7 +100,8 @@ function count_sketches(){ # count_sketches <examples-path>
  requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json)
  if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then
  for requirement in $requirements; do
- found_line=$(grep -E "^$requirement" $LIBS_DIR/esp32/sdkconfig)
+ requirement=$(echo $requirement | xargs)
+ found_line=$(grep -E "^$requirement" "$SDKCONFIG_DIR/esp32/sdkconfig")
  if [[ "$found_line" == "" ]]; then
  continue 2
  fi
@@ -190,7 +191,8 @@ function build_pio_sketches(){ # build_pio_sketches <board> <options> <examples-
  requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json)
  if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then
  for requirement in $requirements; do
- found_line=$(grep -E "^$requirement" $LIBS_DIR/esp32/sdkconfig)
+ requirement=$(echo $requirement | xargs)
+ found_line=$(grep -E "^$requirement" "$SDKCONFIG_DIR/esp32/sdkconfig")
  if [[ "$found_line" == "" ]]; then
  continue 2
  fi

.github/scripts/sketch_utils.sh

@@ -1,6 +1,12 @@
 #!/bin/bash
 
-LIBS_DIR="tools/esp32-arduino-libs"
+if [ -d "$ARDUINO_ESP32_PATH/tools/esp32-arduino-libs" ]; then
+ SDKCONFIG_DIR="$ARDUINO_ESP32_PATH/tools/esp32-arduino-libs"
+elif [ -d "$GITHUB_WORKSPACE/tools/esp32-arduino-libs" ]; then
+ SDKCONFIG_DIR="$GITHUB_WORKSPACE/tools/esp32-arduino-libs"
+else
+ SDKCONFIG_DIR="tools/esp32-arduino-libs"
+fi
 
 function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [extra-options]
  while [ ! -z "$1" ]; do
@@ -83,14 +89,21 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
 
  len=1
 
+ if [ -f $sketchdir/ci.json ]; then
+ fqbn_append=`jq -r '.fqbn_append' $sketchdir/ci.json`
+ if [ $fqbn_append == "null" ]; then
+ fqbn_append=""
+ fi
+ fi
+
  # Default FQBN options if none were passed in the command line.
 
- esp32_opts="PSRAM=enabled,PartitionScheme=huge_app,FlashMode=dio"
- esp32s2_opts="PSRAM=enabled,PartitionScheme=huge_app,FlashMode=dio"
- esp32s3_opts="PSRAM=opi,USBMode=default,PartitionScheme=huge_app,FlashMode=dio"
- esp32c3_opts="PartitionScheme=huge_app,FlashMode=dio"
- esp32c6_opts="PartitionScheme=huge_app,FlashMode=dio"
- esp32h2_opts="PartitionScheme=huge_app,FlashMode=dio"
+ esp32_opts="PSRAM=enabled,FlashMode=dio${fqbn_append:+,$fqbn_append}"
+ esp32s2_opts="PSRAM=enabled,FlashMode=dio${fqbn_append:+,$fqbn_append}"
+ esp32s3_opts="PSRAM=opi,USBMode=default,FlashMode=dio${fqbn_append:+,$fqbn_append}"
+ esp32c3_opts="FlashMode=dio${fqbn_append:+,$fqbn_append}"
+ esp32c6_opts="FlashMode=dio${fqbn_append:+,$fqbn_append}"
+ esp32h2_opts="FlashMode=dio${fqbn_append:+,$fqbn_append}"
 
  # Select the common part of the FQBN based on the target. The rest will be
  # appended depending on the passed options.
@@ -154,7 +167,8 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
  requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json)
  if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then
  for requirement in $requirements; do
- found_line=$(grep -E "^$requirement" $LIBS_DIR/$target/sdkconfig)
+ requirement=$(echo $requirement | xargs)
+ found_line=$(grep -E "^$requirement" "$SDKCONFIG_DIR/$target/sdkconfig")
  if [[ "$found_line" == "" ]]; then
  echo "Target $target does not meet the requirement $requirement for $sketchname. Skipping."
  exit 0
@@ -270,10 +284,11 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
  unset options
 }
 
-function count_sketches(){ # count_sketches <path> [target] [file]
+function count_sketches(){ # count_sketches <path> [target] [file] [ignore-requirements]
  local path=$1
  local target=$2
- local file=$3
+ local ignore_requirements=$3
+ local file=$4
 
  if [ $# -lt 1 ]; then
  echo "ERROR: Illegal number of parameters"
@@ -286,7 +301,7 @@ function count_sketches(){ # count_sketches <path> [target] [file]
  return 0
  fi
 
- if [ -n "$file" ]; then
+ if [ -f "$file" ]; then
  local sketches=$(cat $file)
  else
  local sketches=$(find $path -name *.ino | sort)
@@ -306,15 +321,18 @@ function count_sketches(){ # count_sketches <path> [target] [file]
  continue
  fi
 
- # Check if the sketch requires any configuration options
- requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json)
- if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then
- for requirement in $requirements; do
- found_line=$(grep -E "^$requirement" $LIBS_DIR/$target/sdkconfig)
- if [[ "$found_line" == "" ]]; then
- continue 2
- fi
- done
+ if [ "$ignore_requirements" != "1" ]; then
+ # Check if the sketch requires any configuration options
+ requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json)
+ if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then
+ for requirement in $requirements; do
+ requirement=$(echo $requirement | xargs)
+ found_line=$(grep -E "^$requirement" $SDKCONFIG_DIR/$target/sdkconfig)
+ if [[ "$found_line" == "" ]]; then
+ continue 2
+ fi
+ done
+ fi
  fi
  fi
  echo $sketch >> sketches.txt
@@ -392,7 +410,7 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
 
  set +e
  if [ -n "$sketches_file" ]; then
- count_sketches "$path" "$target" "$sketches_file"
+ count_sketches "$path" "$target" "0" "$sketches_file"
  local sketchcount=$?
  else
  count_sketches "$path" "$target"

.github/scripts/tests_run.sh

@@ -10,6 +10,21 @@ function run_test() {
  local result=0
  local error=0
 
+ if [ $options -eq 0 ] && [ -f $sketchdir/ci.json ]; then
+ len=`jq -r --arg target $target '.fqbn[$target] | length' $sketchdir/ci.json`
+ if [ $len -eq 0 ]; then
+ len=1
+ fi
+ else
+ len=1
+ fi
+
+ if [ $len -eq 1 ]; then
+ SDKCONFIG_PATH="$HOME/.arduino/tests/$sketchname/build.tmp/sdkconfig"
+ else
+ SDKCONFIG_PATH="$HOME/.arduino/tests/$sketchname/build0.tmp/sdkconfig"
+ fi
+
  if [ -f $sketchdir/ci.json ]; then
  # If the target or platform is listed as false, skip the sketch. Otherwise, include it.
  is_target=$(jq -r --arg target $target '.targets[$target]' $sketchdir/ci.json)
@@ -25,7 +40,8 @@ function run_test() {
  requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json)
  if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then
  for requirement in $requirements; do
- found_line=$(grep -E "^$requirement" $LIBS_DIR/$target/sdkconfig)
+ requirement=$(echo $requirement | xargs)
+ found_line=$(grep -E "^$requirement" "$SDKCONFIG_PATH")
  if [[ "$found_line" == "" ]]; then
  printf "\033[93mTarget $target does not meet the requirement $requirement for $sketchname. Skipping.\033[0m\n"
  printf "\n\n\n"
@@ -35,15 +51,6 @@ function run_test() {
  fi
  fi
 
- if [ $options -eq 0 ] && [ -f $sketchdir/ci.json ]; then
- len=`jq -r --arg target $target '.fqbn[$target] | length' $sketchdir/ci.json`
- if [ $len -eq 0 ]; then
- len=1
- fi
- else
- len=1
- fi
-
  if [ $len -eq 1 ]; then
  # build_dir="$sketchdir/build"
  build_dir="$HOME/.arduino/tests/$sketchname/build.tmp"
@@ -120,7 +127,6 @@ function run_test() {
 
 SCRIPTS_DIR="./.github/scripts"
 COUNT_SKETCHES="${SCRIPTS_DIR}/sketch_utils.sh count"
-LIBS_DIR="tools/esp32-arduino-libs"
 
 platform="hardware"
 wokwi_timeout=60000
@@ -223,7 +229,8 @@ else
  fi
 
  set +e
- ${COUNT_SKETCHES} $test_folder $target
+ # Ignore requirements as we don't have the libs. The requirements will be checked in the run_test function
+ ${COUNT_SKETCHES} "$test_folder" "$target" "1"
  sketchcount=$?
  set -e
  sketches=$(cat sketches.txt)

.github/workflows/tests_build.yml

@@ -29,6 +29,7 @@ jobs:
  ~/.arduino/tests/**/build*.tmp/*.bin
  ~/.arduino/tests/**/build*.tmp/*.elf
  ~/.arduino/tests/**/build*.tmp/*.json
+ ~/.arduino/tests/**/build*.tmp/sdkconfig
 
  - name: Evaluate if tests should be built
  id: check-build
@@ -75,6 +76,7 @@ jobs:
  ~/.arduino/tests/**/build*.tmp/*.bin
  ~/.arduino/tests/**/build*.tmp/*.elf
  ~/.arduino/tests/**/build*.tmp/*.json
+ ~/.arduino/tests/**/build*.tmp/sdkconfig
 
  - name: Upload ${{ inputs.chip }} ${{ inputs.type }} binaries as artifacts
  uses: actions/upload-artifact@v4
@@ -85,3 +87,4 @@ jobs:
  ~/.arduino/tests/**/build*.tmp/*.bin
  ~/.arduino/tests/**/build*.tmp/*.elf
  ~/.arduino/tests/**/build*.tmp/*.json
+ ~/.arduino/tests/**/build*.tmp/sdkconfig

.github/workflows/tests_hw.yml

@@ -59,10 +59,6 @@ jobs:
  sparse-checkout: |
  *
 
- - name: List files
- if: ${{ steps.check-tests.outputs.enabled == 'true' }}
- run: ls -la
-
  # setup-python currently only works on ubuntu images
  # - uses: actions/setup-python@v5
  # if: ${{ steps.check-tests.outputs.enabled == 'true' }}

docs/en/contributing.rst

@@ -166,8 +166,47 @@ And in the ``README.md`` file:
 
  Currently, this example requires Wi-Fi and supports the following targets.
 
- | Supported Targets | ESP32 | ESP32-H2 | ESP32-S3 | ESP32-C3 | ESP32-C6 |
- | ----------------- | ----- | -------- | -------- | -------- | -------- |
+ | Supported Targets | ESP32 | ESP32-S3 | ESP32-C3 | ESP32-C6 |
+ | ----------------- | ----- | -------- | -------- | -------- |
+
+By default, the CI system will use the FQBNs specified in the ``.github/scripts/sketch_utils.sh`` file to compile the sketches.
+Currently, the default FQBNs are:
+
+* ``espressif:esp32:esp32:PSRAM=enabled,FlashMode=dio``
+* ``espressif:esp32:esp32s2:PSRAM=enabled,FlashMode=dio``
+* ``espressif:esp32:esp32s3:PSRAM=opi,USBMode=default,FlashMode=dio``
+* ``espressif:esp32:esp32c3:FlashMode=dio``
+* ``espressif:esp32:esp32c6:FlashMode=dio``
+* ``espressif:esp32:esp32h2:FlashMode=dio``
+
+There are two ways to alter the FQBNs used to compile the sketches: by using the ``fqbn`` or ``fqbn_append`` fields in the ``ci.json`` file.
+
+If you just want to append a string to the default FQBNs, you can use the ``fqbn_append`` field. For example, to add the ``DebugLevel=debug`` to the FQBNs, you would use:
+
+.. code-block:: json
+
+ {
+ "fqbn_append": "DebugLevel=debug"
+ }
+
+If you want to override the default FQBNs, you can use the ``fqbn`` field. It is a dictionary where the key is the target name and the value is a list of FQBNs.
+The FQBNs in the list will be used in sequence to compile the sketch. For example, to compile a sketch for ESP32-S2 with and without PSRAM enabled, you would use:
+
+.. code-block:: json
+
+ {
+ "fqbn": {
+ "esp32s2": [
+ "espressif:esp32:esp32s2:PSRAM=enabled,FlashMode=dio",
+ "espressif:esp32:esp32s2:PSRAM=disabled,FlashMode=dio"
+ ]
+ }
+ }
+
+.. note::
+
+ The FQBNs specified in the ``fqbn`` field will also override the options specified in the ``fqbn_append`` field.
+ That means that if the ``fqbn`` field is specified, the ``fqbn_append`` field will be ignored and will have no effect.
 
 Example Template
 ****************
@@ -376,9 +415,10 @@ The ``ci.json`` file is used to specify how the test suite and sketches will han
 * ``platforms``: A dictionary that specifies the supported platforms. The key is the platform name and the value is a boolean that specifies if
  the platform is supported. By default, all platforms are assumed to be supported.
 * ``extra_tags``: A list of extra tags that the runner will require when running the test suite in hardware. By default, no extra tags are required.
+* ``fqbn_append``: A string to be appended to the default FQBNs. By default, no string is appended. This has no effect if ``fqbn`` is specified.
 * ``fqbn``: A dictionary that specifies the FQBNs that will be used to compile the sketch. The key is the target name and the value is a list
  of FQBNs. The `default FQBNs <https://github.com/espressif/arduino-esp32/blob/a31a5fca1739993173caba995f7785b8eed6b30e/.github/scripts/sketch_utils.sh#L86-L91>`_
- are used if this field is not specified.
+ are used if this field is not specified. This overrides the default FQBNs and the ``fqbn_append`` field.
 
 The ``wifi`` test suite is a good example of how to use the ``ci.json`` file:
 

libraries/ESP32/examples/Camera/CameraWebServer/ci.json

@@ -1,4 +1,19 @@
 {
+ "fqbn": {
+ "esp32": [
+ "espressif:esp32:esp32:PSRAM=enabled,PartitionScheme=custom,FlashMode=dio",
+ "espressif:esp32:esp32:PSRAM=disabled,PartitionScheme=custom,FlashMode=dio"
+ ],
+ "esp32s2": [
+ "espressif:esp32:esp32s2:PSRAM=enabled,PartitionScheme=custom,FlashMode=dio",
+ "espressif:esp32:esp32s2:PSRAM=disabled,PartitionScheme=custom,FlashMode=dio"
+ ],
+ "esp32s3": [
+ "espressif:esp32:esp32s3:PSRAM=opi,USBMode=default,PartitionScheme=custom,FlashMode=qio",
+ "espressif:esp32:esp32s3:PSRAM=enabled,USBMode=default,PartitionScheme=custom,FlashMode=qio",
+ "espressif:esp32:esp32s3:PSRAM=disabled,USBMode=default,PartitionScheme=custom,FlashMode=qio"
+ ]
+ },
  "requires": [
  "CONFIG_CAMERA_TASK_STACK_SIZE=[0-9]+"
  ]

libraries/ESP32/examples/Camera/CameraWebServer/partitions.csv

@@ -1,5 +1,6 @@
 # Name, Type, SubType, Offset, Size, Flags
 nvs, data, nvs, 0x9000, 0x5000,
 otadata, data, ota, 0xe000, 0x2000,
-app0, app, ota_0, 0x10000, 0x3d0000,
-fr, data, , 0x3e0000, 0x20000,
+app0, app, ota_0, 0x10000, 0x3c0000,
+fr, data, , 0x3d0000, 0x20000,
+coredump, data, coredump,0x3f0000, 0x10000,

libraries/RainMaker/examples/RMakerCustom/ci.json

@@ -1,6 +1,7 @@
 {
+ "fqbn_append": "PartitionScheme=rainmaker_4MB",
  "requires": [
  "CONFIG_SOC_WIFI_SUPPORTED=y",
- "CONFIG_ESP_RMAKER_WORK_QUEUE_TASK_STACK"
+ "CONFIG_ESP_RMAKER_WORK_QUEUE_TASK_STACK=[1-9][0-9]*"
  ]
 }

libraries/RainMaker/examples/RMakerCustomAirCooler/ci.json

@@ -1,6 +1,7 @@
 {
+ "fqbn_append": "PartitionScheme=rainmaker_4MB",
  "requires": [
  "CONFIG_SOC_WIFI_SUPPORTED=y",
- "CONFIG_ESP_RMAKER_WORK_QUEUE_TASK_STACK"
+ "CONFIG_ESP_RMAKER_WORK_QUEUE_TASK_STACK=[1-9][0-9]*"
  ]
 }

libraries/RainMaker/examples/RMakerSonoffDualR3/ci.json

@@ -1,6 +1,7 @@
 {
+ "fqbn_append": "PartitionScheme=rainmaker_4MB",
  "requires": [
  "CONFIG_SOC_WIFI_SUPPORTED=y",
- "CONFIG_ESP_RMAKER_WORK_QUEUE_TASK_STACK"
+ "CONFIG_ESP_RMAKER_WORK_QUEUE_TASK_STACK=[1-9][0-9]*"
  ]
 }

libraries/RainMaker/examples/RMakerSwitch/ci.json

@@ -1,6 +1,7 @@
 {
+ "fqbn_append": "PartitionScheme=rainmaker_4MB",
  "requires": [
  "CONFIG_SOC_WIFI_SUPPORTED=y",
- "CONFIG_ESP_RMAKER_WORK_QUEUE_TASK_STACK"
+ "CONFIG_ESP_RMAKER_WORK_QUEUE_TASK_STACK=[1-9][0-9]*"
  ]
 }

libraries/WiFiProv/examples/WiFiProv/ci.json

@@ -1,4 +1,5 @@
 {
+ "fqbn_append": "PartitionScheme=huge_app",
  "requires": [
  "CONFIG_SOC_WIFI_SUPPORTED=y"
  ]

libraries/Zigbee/examples/Zigbee_Color_Dimmable_Light/ci.json

@@ -1,16 +1,6 @@
 {
- "fqbn": {
- "esp32c6": [
- "espressif:esp32:esp32c6:PartitionScheme=zigbee,ZigbeeMode=ed"
- ],
- "esp32h2": [
- "espressif:esp32:esp32h2:PartitionScheme=zigbee,ZigbeeMode=ed"
- ]
- },
- "targets": {
- "esp32": false,
- "esp32c3": false,
- "esp32s2": false,
- "esp32s3": false
- }
+ "fqbn_append": "PartitionScheme=zigbee,ZigbeeMode=ed",
+ "requires": [
+ "CONFIG_SOC_IEEE802154_SUPPORTED=y"
+ ]
 }