From b3d15f9adb77ac84d74d1d08cec90fff93e5213b Mon Sep 17 00:00:00 2001 From: Lucas Saavedra Vaz <32426024+lucasssvaz@users.noreply.github.com> Date: Thu, 5 Dec 2024 12:17:40 -0300 Subject: [PATCH] fix(formatting): Fix bash scripts --- .github/scripts/check-cmakelists.sh | 5 +- .github/scripts/find_all_boards.sh | 25 +- .github/scripts/find_new_boards.sh | 37 ++- .github/scripts/install-arduino-cli.sh | 3 +- .github/scripts/install-arduino-core-esp32.sh | 14 +- .github/scripts/install-arduino-ide.sh | 3 +- .github/scripts/install-platformio-esp32.sh | 54 ++-- .github/scripts/on-pages.sh | 69 +++-- .github/scripts/on-push.sh | 80 +++--- .github/scripts/on-release.sh | 215 ++++++++++------ .github/scripts/set_push_chunks.sh | 48 ++-- .github/scripts/sketch_utils.sh | 217 +++++++++------- .github/scripts/tests_build.sh | 33 ++- .github/scripts/tests_matrix.sh | 20 +- .github/scripts/tests_run.sh | 243 +++++++++--------- .github/scripts/update-version.sh | 24 +- .github/scripts/upload_py_tools.sh | 3 +- docs/utils.sh | 27 +- tools/add_lib.sh | 166 ++++++------ 19 files changed, 702 insertions(+), 584 deletions(-) diff --git a/.github/scripts/check-cmakelists.sh b/.github/scripts/check-cmakelists.sh index 98d9722ad83..7d4f6b4e2d9 100755 --- a/.github/scripts/check-cmakelists.sh +++ b/.github/scripts/check-cmakelists.sh @@ -1,4 +1,5 @@ #!/bin/bash + # # This script is used in the CI workflow. It checks all non-examples source files in libraries/ and cores/ are listed in # CMakeLists.txt for the cmake-based IDF component @@ -12,10 +13,10 @@ set -e git submodule update --init --recursive # find all source files in repo -REPO_SRCS=`find cores/esp32/ libraries/ -name 'examples' -prune -o -name '*.c' -print -o -name '*.cpp' -print | sort` +REPO_SRCS=$(find cores/esp32/ libraries/ -name 'examples' -prune -o -name '*.c' -print -o -name '*.cpp' -print | sort) # find all source files named in CMakeLists.txt COMPONENT_SRCS -CMAKE_SRCS=`cmake --trace-expand -P CMakeLists.txt 2>&1 | grep set\(srcs | cut -d'(' -f3 | sed 's/ )//' | sed 's/srcs //' | tr ' ;' '\n' | sort` +CMAKE_SRCS=$(cmake --trace-expand -P CMakeLists.txt 2>&1 | grep set\(srcs | cut -d'(' -f3 | sed 's/ )//' | sed 's/srcs //' | tr ' ;' '\n' | sort) if ! diff -u0 --label "Repo Files" --label "srcs" <(echo "$REPO_SRCS") <(echo "$CMAKE_SRCS"); then echo "Source files in repo (-) and source files in CMakeLists.txt (+) don't match" diff --git a/.github/scripts/find_all_boards.sh b/.github/scripts/find_all_boards.sh index b474a49bc2e..67b46661ca5 100755 --- a/.github/scripts/find_all_boards.sh +++ b/.github/scripts/find_all_boards.sh @@ -3,7 +3,9 @@ # Get all boards boards_array=() -for line in `grep '.tarch=' boards.txt`; do +boards_list=$(grep '.tarch=' boards.txt) + +while read -r line; do board_name=$(echo "$line" | cut -d '.' -f1 | cut -d '#' -f1) # skip esp32c2 as we dont build libs for it if [ "$board_name" == "esp32c2" ]; then @@ -12,29 +14,26 @@ for line in `grep '.tarch=' boards.txt`; do fi boards_array+=("espressif:esp32:$board_name") echo "Added 'espressif:esp32:$board_name' to array" -done +done <<< "$boards_list" # Create JSON like string with all boards found and pass it to env variable board_count=${#boards_array[@]} echo "Boards found: $board_count" -echo "BOARD-COUNT=$board_count" >> $GITHUB_ENV +echo "BOARD-COUNT=$board_count" >> "$GITHUB_ENV" -if [ $board_count -gt 0 ] -then +if [ "$board_count" -gt 0 ]; then json_matrix='[' - for board in ${boards_array[@]} - do + for board in "${boards_array[@]}"; do json_matrix+='"'$board'"' - if [ $board_count -gt 1 ] - then + if [ "$board_count" -gt 1 ]; then json_matrix+="," fi - board_count=$(($board_count - 1)) + board_count=$((board_count - 1)) done json_matrix+=']' - echo $json_matrix - echo "FQBNS=${json_matrix}" >> $GITHUB_ENV + echo "$json_matrix" + echo "FQBNS=${json_matrix}" >> "$GITHUB_ENV" else - echo "FQBNS=" >> $GITHUB_ENV + echo "FQBNS=" >> "$GITHUB_ENV" fi diff --git a/.github/scripts/find_new_boards.sh b/.github/scripts/find_new_boards.sh index 706676b4a4c..4482aa2b1da 100755 --- a/.github/scripts/find_new_boards.sh +++ b/.github/scripts/find_new_boards.sh @@ -5,14 +5,13 @@ owner_repository=$1 base_ref=$2 # Download the boards.txt file from the base branch -curl -L -o boards_base.txt https://raw.githubusercontent.com/$owner_repository/$base_ref/boards.txt +curl -L -o boards_base.txt https://raw.githubusercontent.com/"$owner_repository"/"$base_ref"/boards.txt # Compare boards.txt file in the repo with the modified file from PR diff=$(diff -u boards_base.txt boards.txt) # Check if the diff is empty -if [ -z "$diff" ] -then +if [ -z "$diff" ]; then echo "No changes in boards.txt file" echo "FQBNS=" exit 0 @@ -21,7 +20,7 @@ fi # Extract added or modified lines (lines starting with '+' or '-') modified_lines=$(echo "$diff" | grep -E '^[+-][^+-]') -# Print the modified lines for debugging +# Print the modified lines for debugging echo "Modified lines:" echo "$modified_lines" @@ -29,15 +28,12 @@ boards_array=() previous_board="" # Extract board names from the modified lines, and add them to the boards_array -while read -r line -do +while read -r line; do board_name=$(echo "$line" | cut -d '.' -f1 | cut -d '#' -f1) # remove + or - from the board name at the beginning - board_name=$(echo "$board_name" | sed 's/^[+-]//') - if [ "$board_name" != "" ] && [ "$board_name" != "+" ] && [ "$board_name" != "-" ] && [ "$board_name" != "esp32_family" ] - then - if [ "$board_name" != "$previous_board" ] - then + board_name=${board_name#[-+]} + if [ "$board_name" != "" ] && [ "$board_name" != "+" ] && [ "$board_name" != "-" ] && [ "$board_name" != "esp32_family" ]; then + if [ "$board_name" != "$previous_board" ]; then boards_array+=("espressif:esp32:$board_name") previous_board="$board_name" echo "Added 'espressif:esp32:$board_name' to array" @@ -48,22 +44,19 @@ done <<< "$modified_lines" # Create JSON like string with all boards found and pass it to env variable board_count=${#boards_array[@]} -if [ $board_count -gt 0 ] -then +if [ "$board_count" -gt 0 ]; then json_matrix='{"fqbn": [' - for board in ${boards_array[@]} - do + for board in "${boards_array[@]}"; do json_matrix+='"'$board'"' - if [ $board_count -gt 1 ] - then + if [ "$board_count" -gt 1 ]; then json_matrix+="," fi - board_count=$(($board_count - 1)) + board_count=$((board_count - 1)) done json_matrix+=']}' - echo $json_matrix - echo "FQBNS=${json_matrix}" >> $GITHUB_ENV + echo "$json_matrix" + echo "FQBNS=${json_matrix}" >> "$GITHUB_ENV" else - echo "FQBNS=" >> $GITHUB_ENV -fi \ No newline at end of file + echo "FQBNS=" >> "$GITHUB_ENV" +fi diff --git a/.github/scripts/install-arduino-cli.sh b/.github/scripts/install-arduino-cli.sh index 533b39c7ddd..bb7f544e752 100755 --- a/.github/scripts/install-arduino-cli.sh +++ b/.github/scripts/install-arduino-cli.sh @@ -1,6 +1,6 @@ #!/bin/bash -OSBITS=`uname -m` +OSBITS=$(uname -m) if [[ "$OSTYPE" == "linux"* ]]; then export OS_IS_LINUX="1" if [[ "$OSBITS" == "i686" ]]; then @@ -49,4 +49,3 @@ if [ ! -d "$ARDUINO_IDE_PATH" ] || [ ! -f "$ARDUINO_IDE_PATH/arduino-cli" ]; the curl -fsSL https://raw.githubusercontent.com/arduino/arduino-cli/master/install.sh | BINDIR="$ARDUINO_IDE_PATH" sh fi fi - diff --git a/.github/scripts/install-arduino-core-esp32.sh b/.github/scripts/install-arduino-core-esp32.sh index fe50c909198..e0071a0eb83 100755 --- a/.github/scripts/install-arduino-core-esp32.sh +++ b/.github/scripts/install-arduino-core-esp32.sh @@ -5,7 +5,7 @@ if [ ! -d "$ARDUINO_ESP32_PATH" ]; then echo "Installing ESP32 Arduino Core ..." script_init_path="$PWD" mkdir -p "$ARDUINO_USR_PATH/hardware/espressif" - cd "$ARDUINO_USR_PATH/hardware/espressif" + cd "$ARDUINO_USR_PATH/hardware/espressif" || exit echo "Installing Python Serial ..." pip install pyserial > /dev/null @@ -15,25 +15,25 @@ if [ ! -d "$ARDUINO_ESP32_PATH" ]; then pip install requests > /dev/null fi - if [ ! -z "$GITHUB_REPOSITORY" ]; then + if [ -n "$GITHUB_REPOSITORY" ]; then echo "Linking Core..." - ln -s $GITHUB_WORKSPACE esp32 + ln -s "$GITHUB_WORKSPACE" esp32 else echo "Cloning Core Repository..." git clone https://github.com/espressif/arduino-esp32.git esp32 > /dev/null 2>&1 fi #echo "Updating Submodules ..." - cd esp32 + cd esp32 || exit #git submodule update --init --recursive > /dev/null 2>&1 echo "Installing Platform Tools ..." if [ "$OS_IS_WINDOWS" == "1" ]; then - cd tools && ./get.exe + cd tools && ./get.exe else - cd tools && python get.py + cd tools && python get.py fi - cd $script_init_path + cd "$script_init_path" || exit echo "ESP32 Arduino has been installed in '$ARDUINO_ESP32_PATH'" echo "" diff --git a/.github/scripts/install-arduino-ide.sh b/.github/scripts/install-arduino-ide.sh index 7fd95797834..5b3bcb1791e 100755 --- a/.github/scripts/install-arduino-ide.sh +++ b/.github/scripts/install-arduino-ide.sh @@ -4,7 +4,7 @@ #OSTYPE: 'msys', ARCH: 'x86_64' => win32 #OSTYPE: 'darwin18', ARCH: 'i386' => macos -OSBITS=`uname -m` +OSBITS=$(uname -m) if [[ "$OSTYPE" == "linux"* ]]; then export OS_IS_LINUX="1" ARCHIVE_FORMAT="tar.xz" @@ -77,4 +77,3 @@ if [ ! -d "$ARDUINO_IDE_PATH" ]; then echo "Arduino IDE Installed in '$ARDUINO_IDE_PATH'" echo "" fi - diff --git a/.github/scripts/install-platformio-esp32.sh b/.github/scripts/install-platformio-esp32.sh index 80c668bdc0e..8cd6552eb80 100755 --- a/.github/scripts/install-platformio-esp32.sh +++ b/.github/scripts/install-platformio-esp32.sh @@ -52,7 +52,7 @@ python -c "$replace_script" if [ "$GITHUB_REPOSITORY" == "espressif/arduino-esp32" ]; then echo "Linking Core..." - ln -s $GITHUB_WORKSPACE "$PLATFORMIO_ESP32_PATH" + ln -s "$GITHUB_WORKSPACE" "$PLATFORMIO_ESP32_PATH" else echo "Cloning Core Repository ..." git clone --recursive https://github.com/espressif/arduino-esp32.git "$PLATFORMIO_ESP32_PATH" > /dev/null 2>&1 @@ -61,7 +61,7 @@ fi echo "PlatformIO for ESP32 has been installed" echo "" -function build_pio_sketch(){ # build_pio_sketch +function build_pio_sketch { # build_pio_sketch if [ "$#" -lt 3 ]; then echo "ERROR: Illegal number of parameters" echo "USAGE: build_pio_sketch " @@ -71,13 +71,15 @@ function build_pio_sketch(){ # build_pio_sketch local board="$1" local options="$2" local sketch="$3" - local sketch_dir=$(dirname "$sketch") + local sketch_dir + + sketch_dir=$(dirname "$sketch") echo "" - echo "Compiling '"$(basename "$sketch")"' ..." + echo "Compiling '$(basename "$sketch")' ..." python -m platformio ci --board "$board" "$sketch_dir" --project-option="$options" } -function build_pio_sketches(){ # build_pio_sketches +function build_pio_sketches { # build_pio_sketches if [ "$#" -lt 3 ]; then echo "ERROR: Illegal number of parameters" echo "USAGE: build_pio_sketches [ ]" @@ -108,27 +110,34 @@ function build_pio_sketches(){ # build_pio_sketches -function git_remove_from_pages(){ +function git_remove_from_pages { local path=$1 - local info=`curl -s -k -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.object+json" -X GET "https://api.github.com/repos/$GITHUB_REPOSITORY/contents/$path?ref=gh-pages"` - local type=`echo "$info" | jq -r '.type'` - if [ ! $type == "file" ]; then - if [ ! $type == "null" ]; then + local info + local type + local sha + local message + + info=$(curl -s -k -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.object+json" -X GET "https://api.github.com/repos/$GITHUB_REPOSITORY/contents/$path?ref=gh-pages") + type=$(echo "$info" | jq -r '.type') + + if [ ! "$type" == "file" ]; then + if [ ! "$type" == "null" ]; then echo "Wrong type '$type'" else echo "File is not on Pages" fi return 0 fi - local sha=`echo "$info" | jq -r '.sha'` - local message="Deleting "$(basename $path) + + sha=$(echo "$info" | jq -r '.sha') + message="Deleting "$(basename "$path") local json="{\"branch\":\"gh-pages\",\"message\":\"$message\",\"sha\":\"$sha\"}" echo "$json" | curl -s -k -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.raw+json" -X DELETE --data @- "https://api.github.com/repos/$GITHUB_REPOSITORY/contents/$path" } -function git_upload_to_pages(){ +function git_upload_to_pages { local path=$1 local src=$2 @@ -42,41 +50,50 @@ function git_upload_to_pages(){ return 1 fi - local info=`curl -s -k -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.object+json" -X GET "https://api.github.com/repos/$GITHUB_REPOSITORY/contents/$path?ref=gh-pages"` - local type=`echo "$info" | jq -r '.type'` - local message=$(basename $path) + local info + local type + local message local sha="" local content="" - if [ $type == "file" ]; then - sha=`echo "$info" | jq -r '.sha'` + info=$(curl -s -k -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.object+json" -X GET "https://api.github.com/repos/$GITHUB_REPOSITORY/contents/$path?ref=gh-pages") + type=$(echo "$info" | jq -r '.type') + message=$(basename "$path") + + if [ "$type" == "file" ]; then + sha=$(echo "$info" | jq -r '.sha') sha=",\"sha\":\"$sha\"" message="Updating $message" - elif [ ! $type == "null" ]; then + elif [ ! "$type" == "null" ]; then >&2 echo "Wrong type '$type'" return 1 else message="Creating $message" fi - content=`base64 -i "$src"` + content=$(base64 -i "$src") data="{\"branch\":\"gh-pages\",\"message\":\"$message\",\"content\":\"$content\"$sha}" echo "$data" | curl -s -k -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.raw+json" -X PUT --data @- "https://api.github.com/repos/$GITHUB_REPOSITORY/contents/$path" } -function git_safe_upload_to_pages(){ +function git_safe_upload_to_pages { local path=$1 local file="$2" - local name=$(basename "$file") - local size=`get_file_size "$file"` - local upload_res=`git_upload_to_pages "$path" "$file"` - if [ $? -ne 0 ]; then + local name + local size + local upload_res + + name=$(basename "$file") + size=$(get_file_size "$file") + + if ! upload_res=$(git_upload_to_pages "$path" "$file"); then >&2 echo "ERROR: Failed to upload '$name' ($?)" return 1 fi - up_size=`echo "$upload_res" | jq -r '.content.size'` - if [ $up_size -ne $size ]; then + + up_size=$(echo "$upload_res" | jq -r '.content.size') + if [ "$up_size" -ne "$size" ]; then >&2 echo "ERROR: Uploaded size does not match! $up_size != $size" #git_delete_asset return 1 diff --git a/.github/scripts/on-push.sh b/.github/scripts/on-push.sh index 08ff505f1c0..7fe17d5e7b6 100755 --- a/.github/scripts/on-push.sh +++ b/.github/scripts/on-push.sh @@ -4,43 +4,44 @@ set -e export ARDUINO_BUILD_DIR="$HOME/.arduino/build.tmp" -function build(){ +function build { local target=$1 local chunk_index=$2 local chunks_cnt=$3 local build_log=$4 local sketches_file=$5 - shift; shift; shift; shift; shift; - local sketches=$* + shift 5 + local sketches=("$@") local BUILD_SKETCH="${SCRIPTS_DIR}/sketch_utils.sh build" local BUILD_SKETCHES="${SCRIPTS_DIR}/sketch_utils.sh chunk_build" - local args="-ai $ARDUINO_IDE_PATH -au $ARDUINO_USR_PATH" - - args+=" -t $target" + local args=("-ai" "$ARDUINO_IDE_PATH" "-au" "$ARDUINO_USR_PATH" "-t" "$target") if [ "$OS_IS_LINUX" == "1" ]; then - args+=" -p $ARDUINO_ESP32_PATH/libraries" - args+=" -i $chunk_index -m $chunks_cnt" + args+=("-p" "$ARDUINO_ESP32_PATH/libraries" "-i" "$chunk_index" "-m" "$chunks_cnt") if [ -n "$sketches_file" ]; then - args+=" -f $sketches_file" + args+=("-f" "$sketches_file") fi - if [ $build_log -eq 1 ]; then - args+=" -l $build_log" + if [ "$build_log" -eq 1 ]; then + args+=("-l" "$build_log") fi - ${BUILD_SKETCHES} ${args} + ${BUILD_SKETCHES} "${args[@]}" else - for sketch in ${sketches}; do - local sargs="$args -s $(dirname $sketch)" + for sketch in "${sketches[@]}"; do + local sargs=("${args[@]}") + local ctags_version + local preprocessor_version + sargs+=("-s" "$(dirname "$sketch")") if [ "$OS_IS_WINDOWS" == "1" ] && [ -d "$ARDUINO_IDE_PATH/tools-builder" ]; then - local ctags_version=`ls "$ARDUINO_IDE_PATH/tools-builder/ctags/"` - local preprocessor_version=`ls "$ARDUINO_IDE_PATH/tools-builder/arduino-preprocessor/"` - win_opts="-prefs=runtime.tools.ctags.path=$ARDUINO_IDE_PATH/tools-builder/ctags/$ctags_version - -prefs=runtime.tools.arduino-preprocessor.path=$ARDUINO_IDE_PATH/tools-builder/arduino-preprocessor/$preprocessor_version" - sargs+=" ${win_opts}" + ctags_version=$(ls "$ARDUINO_IDE_PATH/tools-builder/ctags/") + preprocessor_version=$(ls "$ARDUINO_IDE_PATH/tools-builder/arduino-preprocessor/") + sargs+=( + "-prefs=runtime.tools.ctags.path=$ARDUINO_IDE_PATH/tools-builder/ctags/$ctags_version" + "-prefs=runtime.tools.arduino-preprocessor.path=$ARDUINO_IDE_PATH/tools-builder/arduino-preprocessor/$preprocessor_version" + ) fi - ${BUILD_SKETCH} ${sargs} + ${BUILD_SKETCH} "${sargs[@]}" done fi } @@ -73,39 +74,39 @@ fi SCRIPTS_DIR="./.github/scripts" if [ "$BUILD_PIO" -eq 0 ]; then - source ${SCRIPTS_DIR}/install-arduino-cli.sh - source ${SCRIPTS_DIR}/install-arduino-core-esp32.sh - - SKETCHES_ESP32="\ - $ARDUINO_ESP32_PATH/libraries/NetworkClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino\ - $ARDUINO_ESP32_PATH/libraries/BLE/examples/Server/Server.ino\ - $ARDUINO_ESP32_PATH/libraries/ESP32/examples/Camera/CameraWebServer/CameraWebServer.ino\ - $ARDUINO_ESP32_PATH/libraries/Insights/examples/MinimalDiagnostics/MinimalDiagnostics.ino\ - " + source "${SCRIPTS_DIR}/install-arduino-cli.sh" + source "${SCRIPTS_DIR}/install-arduino-core-esp32.sh" + + SKETCHES_ESP32=( + "$ARDUINO_ESP32_PATH/libraries/NetworkClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino" + "$ARDUINO_ESP32_PATH/libraries/BLE/examples/Server/Server.ino" + "$ARDUINO_ESP32_PATH/libraries/ESP32/examples/Camera/CameraWebServer/CameraWebServer.ino" + "$ARDUINO_ESP32_PATH/libraries/Insights/examples/MinimalDiagnostics/MinimalDiagnostics.ino" + ) #create sizes_file sizes_file="$GITHUB_WORKSPACE/cli_compile_$CHUNK_INDEX.json" if [ "$BUILD_LOG" -eq 1 ]; then #create sizes_file and echo start of JSON array with "boards" key - echo "{\"boards\": [" > $sizes_file + echo "{\"boards\": [" > "$sizes_file" fi #build sketches for different targets - build "esp32s3" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32" - build "esp32s2" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32" - build "esp32c3" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32" - build "esp32c6" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32" - build "esp32h2" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32" - build "esp32" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32" + build "esp32s3" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "${SKETCHES_ESP32[@]}" + build "esp32s2" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "${SKETCHES_ESP32[@]}" + build "esp32c3" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "${SKETCHES_ESP32[@]}" + build "esp32c6" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "${SKETCHES_ESP32[@]}" + build "esp32h2" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "${SKETCHES_ESP32[@]}" + build "esp32" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "${SKETCHES_ESP32[@]}" if [ "$BUILD_LOG" -eq 1 ]; then #remove last comma from the last JSON object sed -i '$ s/,$//' "$sizes_file" #echo end of JSON array - echo "]}" >> $sizes_file + echo "]}" >> "$sizes_file" fi else - source ${SCRIPTS_DIR}/install-platformio-esp32.sh + source "${SCRIPTS_DIR}/install-platformio-esp32.sh" # PlatformIO ESP32 Test BOARD="esp32dev" OPTIONS="board_build.partitions = huge_app.csv" @@ -116,8 +117,7 @@ else build_pio_sketch "$BOARD" "$OPTIONS" "$PLATFORMIO_ESP32_PATH/libraries/ESP32/examples/Camera/CameraWebServer/CameraWebServer.ino" # Basic sanity testing for other series - for board in "esp32-c3-devkitm-1" "esp32-s2-saola-1" "esp32-s3-devkitc-1" - do + for board in "esp32-c3-devkitm-1" "esp32-s2-saola-1" "esp32-s3-devkitc-1"; do python -m platformio ci --board "$board" "$PLATFORMIO_ESP32_PATH/libraries/WiFi/examples/WiFiClient" --project-option="board_build.partitions = huge_app.csv" done diff --git a/.github/scripts/on-release.sh b/.github/scripts/on-release.sh index e0dd7b752c5..747e7666251 100755 --- a/.github/scripts/on-release.sh +++ b/.github/scripts/on-release.sh @@ -1,29 +1,34 @@ #!/bin/bash +# Disable shellcheck warning about using 'cat' to read a file. +# Disable shellcheck warning about using individual redirections for each command. +# Disable shellcheck warning about $? uses. +# shellcheck disable=SC2002,SC2129,SC2181,SC2319 -if [ ! $GITHUB_EVENT_NAME == "release" ]; then +if [ ! "$GITHUB_EVENT_NAME" == "release" ]; then echo "Wrong event '$GITHUB_EVENT_NAME'!" exit 1 fi -EVENT_JSON=`cat $GITHUB_EVENT_PATH` +EVENT_JSON=$(cat "$GITHUB_EVENT_PATH") -action=`echo $EVENT_JSON | jq -r '.action'` -if [ ! $action == "published" ]; then +action=$(echo "$EVENT_JSON" | jq -r '.action') +if [ ! "$action" == "published" ]; then echo "Wrong action '$action'. Exiting now..." exit 0 fi -draft=`echo $EVENT_JSON | jq -r '.release.draft'` -if [ $draft == "true" ]; then +draft=$(echo "$EVENT_JSON" | jq -r '.release.draft') +if [ "$draft" == "true" ]; then echo "It's a draft release. Exiting now..." exit 0 fi -RELEASE_PRE=`echo $EVENT_JSON | jq -r '.release.prerelease'` -RELEASE_TAG=`echo $EVENT_JSON | jq -r '.release.tag_name'` -RELEASE_BRANCH=`echo $EVENT_JSON | jq -r '.release.target_commitish'` -RELEASE_ID=`echo $EVENT_JSON | jq -r '.release.id'` +RELEASE_PRE=$(echo "$EVENT_JSON" | jq -r '.release.prerelease') +RELEASE_TAG=$(echo "$EVENT_JSON" | jq -r '.release.tag_name') +RELEASE_BRANCH=$(echo "$EVENT_JSON" | jq -r '.release.target_commitish') +RELEASE_ID=$(echo "$EVENT_JSON" | jq -r '.release.id') +SCRIPTS_DIR="./.github/scripts" OUTPUT_DIR="$GITHUB_WORKSPACE/build" PACKAGE_NAME="esp32-$RELEASE_TAG" PACKAGE_JSON_MERGE="$GITHUB_WORKSPACE/.github/scripts/merge_packages.py" @@ -36,17 +41,23 @@ echo "Action: $action, Branch: $RELEASE_BRANCH, ID: $RELEASE_ID" echo "Tag: $RELEASE_TAG, Draft: $draft, Pre-Release: $RELEASE_PRE" # Try extracting something like a JSON with a "boards" array/element and "vendor" fields -BOARDS=`echo $RELEASE_BODY | grep -Pzo '(?s){.*}' | jq -r '.boards[]? // .boards? // empty' | xargs echo -n 2>/dev/null` -VENDOR=`echo $RELEASE_BODY | grep -Pzo '(?s){.*}' | jq -r '.vendor? // empty' | xargs echo -n 2>/dev/null` -if ! [ -z "${BOARDS}" ]; then echo "Releasing board(s): $BOARDS" ; fi -if ! [ -z "${VENDOR}" ]; then echo "Setting packager: $VENDOR" ; fi +BOARDS=$(echo "$RELEASE_BODY" | grep -Pzo '(?s){.*}' | jq -r '.boards[]? // .boards? // empty' | xargs echo -n 2>/dev/null) +VENDOR=$(echo "$RELEASE_BODY" | grep -Pzo '(?s){.*}' | jq -r '.vendor? // empty' | xargs echo -n 2>/dev/null) -function get_file_size(){ +if [ -n "${BOARDS}" ]; then + echo "Releasing board(s): $BOARDS" +fi + +if [ -n "${VENDOR}" ]; then + echo "Setting packager: $VENDOR" +fi + +function get_file_size { local file="$1" if [[ "$OSTYPE" == "darwin"* ]]; then - eval `stat -s "$file"` + eval "$(stat -s "$file")" local res="$?" - echo "$st_size" + echo "${st_size:?}" return $res else stat --printf="%s" "$file" @@ -54,23 +65,29 @@ function get_file_size(){ fi } -function git_upload_asset(){ - local name=$(basename "$1") +function git_upload_asset { + local name + name=$(basename "$1") # local mime=$(file -b --mime-type "$1") curl -k -X POST -sH "Authorization: token $GITHUB_TOKEN" -H "Content-Type: application/octet-stream" --data-binary @"$1" "https://uploads.github.com/repos/$GITHUB_REPOSITORY/releases/$RELEASE_ID/assets?name=$name" } -function git_safe_upload_asset(){ +function git_safe_upload_asset { local file="$1" - local name=$(basename "$file") - local size=`get_file_size "$file"` - local upload_res=`git_upload_asset "$file"` - if [ $? -ne 0 ]; then + local name + local size + local upload_res + + name=$(basename "$file") + size=$(get_file_size "$file") + + if ! upload_res=$(git_upload_asset "$file"); then >&2 echo "ERROR: Failed to upload '$name' ($?)" return 1 fi - up_size=`echo "$upload_res" | jq -r '.size'` - if [ $up_size -ne $size ]; then + + up_size=$(echo "$upload_res" | jq -r '.size') + if [ "$up_size" -ne "$size" ]; then >&2 echo "ERROR: Uploaded size does not match! $up_size != $size" #git_delete_asset return 1 @@ -79,7 +96,7 @@ function git_safe_upload_asset(){ return $? } -function git_upload_to_pages(){ +function git_upload_to_pages { local path=$1 local src=$2 @@ -88,41 +105,50 @@ function git_upload_to_pages(){ return 1 fi - local info=`curl -s -k -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.object+json" -X GET "https://api.github.com/repos/$GITHUB_REPOSITORY/contents/$path?ref=gh-pages"` - local type=`echo "$info" | jq -r '.type'` - local message=$(basename $path) + local info + local type + local message local sha="" local content="" - if [ $type == "file" ]; then - sha=`echo "$info" | jq -r '.sha'` + info=$(curl -s -k -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.object+json" -X GET "https://api.github.com/repos/$GITHUB_REPOSITORY/contents/$path?ref=gh-pages") + type=$(echo "$info" | jq -r '.type') + message=$(basename "$path") + + if [ "$type" == "file" ]; then + sha=$(echo "$info" | jq -r '.sha') sha=",\"sha\":\"$sha\"" message="Updating $message" - elif [ ! $type == "null" ]; then + elif [ ! "$type" == "null" ]; then >&2 echo "Wrong type '$type'" return 1 else message="Creating $message" fi - content=`base64 -i "$src"` + content=$(base64 -i "$src") data="{\"branch\":\"gh-pages\",\"message\":\"$message\",\"content\":\"$content\"$sha}" echo "$data" | curl -s -k -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.raw+json" -X PUT --data @- "https://api.github.com/repos/$GITHUB_REPOSITORY/contents/$path" } -function git_safe_upload_to_pages(){ +function git_safe_upload_to_pages { local path=$1 local file="$2" - local name=$(basename "$file") - local size=`get_file_size "$file"` - local upload_res=`git_upload_to_pages "$path" "$file"` - if [ $? -ne 0 ]; then + local name + local size + local upload_res + + name=$(basename "$file") + size=$(get_file_size "$file") + + if ! upload_res=$(git_upload_to_pages "$path" "$file"); then >&2 echo "ERROR: Failed to upload '$name' ($?)" return 1 fi - up_size=`echo "$upload_res" | jq -r '.content.size'` - if [ $up_size -ne $size ]; then + + up_size=$(echo "$upload_res" | jq -r '.content.size') + if [ "$up_size" -ne "$size" ]; then >&2 echo "ERROR: Uploaded size does not match! $up_size != $size" #git_delete_asset return 1 @@ -131,15 +157,20 @@ function git_safe_upload_to_pages(){ return $? } -function merge_package_json(){ +function merge_package_json { local jsonLink=$1 local jsonOut=$2 local old_json=$OUTPUT_DIR/oldJson.json local merged_json=$OUTPUT_DIR/mergedJson.json + local error_code=0 echo "Downloading previous JSON $jsonLink ..." curl -L -o "$old_json" "https://github.com/$GITHUB_REPOSITORY/releases/download/$jsonLink?access_token=$GITHUB_TOKEN" 2>/dev/null - if [ $? -ne 0 ]; then echo "ERROR: Download Failed! $?"; exit 1; fi + error_code=$? + if [ $error_code -ne 0 ]; then + echo "ERROR: Download Failed! $error_code" + exit 1 + fi echo "Creating new JSON ..." set +e @@ -147,7 +178,7 @@ function merge_package_json(){ set -e set -v - if [ ! -s $merged_json ]; then + if [ ! -s "$merged_json" ]; then rm -f "$merged_json" echo "Nothing to merge" else @@ -188,9 +219,10 @@ else done # Copy only relevant variant files mkdir "$PKG_DIR/variants/" - for variant in `cat ${PKG_DIR}/boards.txt | grep "\.variant=" | cut -d= -f2` ; do + board_list=$(cat "${PKG_DIR}"/boards.txt | grep "\.variant=" | cut -d= -f2) + while IFS= read -r variant; do cp -Rf "$GITHUB_WORKSPACE/variants/${variant}" "$PKG_DIR/variants/" - done + done <<< "$board_list" fi cp -f "$GITHUB_WORKSPACE/CMakeLists.txt" "$PKG_DIR/" cp -f "$GITHUB_WORKSPACE/idf_component.yml" "$PKG_DIR/" @@ -239,34 +271,36 @@ sed "s/{runtime\.platform\.path}.tools.xtensa-esp32s3-elf/\\{runtime.tools.$XS3T sed 's/{runtime\.platform\.path}.tools.riscv32-esp-elf-gdb/\{runtime.tools.riscv32-esp-elf-gdb.path\}/g' | \ sed "s/{runtime\.platform\.path}.tools.riscv32-esp-elf/\\{runtime.tools.$RVTC_NEW_NAME.path\\}/g" | \ sed 's/{runtime\.platform\.path}.tools.esptool/\{runtime.tools.esptool_py.path\}/g' | \ -sed 's/{runtime\.platform\.path}.tools.openocd-esp32/\{runtime.tools.openocd-esp32.path\}/g' \ - > "$PKG_DIR/platform.txt" +sed 's/{runtime\.platform\.path}.tools.openocd-esp32/\{runtime.tools.openocd-esp32.path\}/g' > "$PKG_DIR/platform.txt" -if ! [ -z ${VENDOR} ]; then +if [ -n "${VENDOR}" ]; then # Append vendor name to platform.txt to create a separate section sed -i "/^name=.*/s/$/ ($VENDOR)/" "$PKG_DIR/platform.txt" fi # Add header with version information echo "Generating core_version.h ..." -ver_define=`echo $RELEASE_TAG | tr "[:lower:].\055" "[:upper:]_"` -ver_hex=`git -C "$GITHUB_WORKSPACE" rev-parse --short=8 HEAD 2>/dev/null` -echo \#define ARDUINO_ESP32_GIT_VER 0x$ver_hex > "$PKG_DIR/cores/esp32/core_version.h" -echo \#define ARDUINO_ESP32_GIT_DESC `git -C "$GITHUB_WORKSPACE" describe --tags 2>/dev/null` >> "$PKG_DIR/cores/esp32/core_version.h" -echo \#define ARDUINO_ESP32_RELEASE_$ver_define >> "$PKG_DIR/cores/esp32/core_version.h" -echo \#define ARDUINO_ESP32_RELEASE \"$ver_define\" >> "$PKG_DIR/cores/esp32/core_version.h" +ver_define=$(echo "$RELEASE_TAG" | tr "[:lower:].\055" "[:upper:]_") +ver_hex=$(git -C "$GITHUB_WORKSPACE" rev-parse --short=8 HEAD 2>/dev/null) +echo \#define ARDUINO_ESP32_GIT_VER 0x"$ver_hex" > "$PKG_DIR/cores/esp32/core_version.h" +echo \#define ARDUINO_ESP32_GIT_DESC "$(git -C "$GITHUB_WORKSPACE" describe --tags 2>/dev/null)" >> "$PKG_DIR/cores/esp32/core_version.h" +echo \#define ARDUINO_ESP32_RELEASE_"$ver_define" >> "$PKG_DIR/cores/esp32/core_version.h" +echo \#define ARDUINO_ESP32_RELEASE \""$ver_define"\" >> "$PKG_DIR/cores/esp32/core_version.h" # Compress package folder echo "Creating ZIP ..." pushd "$OUTPUT_DIR" >/dev/null zip -qr "$PACKAGE_ZIP" "$PACKAGE_NAME" -if [ $? -ne 0 ]; then echo "ERROR: Failed to create $PACKAGE_ZIP ($?)"; exit 1; fi +if [ $? -ne 0 ]; then + echo "ERROR: Failed to create $PACKAGE_ZIP ($?)" + exit 1 +fi # Calculate SHA-256 echo "Calculating SHA sum ..." PACKAGE_PATH="$OUTPUT_DIR/$PACKAGE_ZIP" -PACKAGE_SHA=`shasum -a 256 "$PACKAGE_ZIP" | cut -f 1 -d ' '` -PACKAGE_SIZE=`get_file_size "$PACKAGE_ZIP"` +PACKAGE_SHA=$(shasum -a 256 "$PACKAGE_ZIP" | cut -f 1 -d ' ') +PACKAGE_SIZE=$(get_file_size "$PACKAGE_ZIP") popd >/dev/null rm -rf "$PKG_DIR" echo "'$PACKAGE_ZIP' Created! Size: $PACKAGE_SIZE, SHA-256: $PACKAGE_SHA" @@ -274,7 +308,7 @@ echo # Upload package to release page echo "Uploading package to release page ..." -PACKAGE_URL=`git_safe_upload_asset "$PACKAGE_PATH"` +PACKAGE_URL=$(git_safe_upload_asset "$PACKAGE_PATH") echo "Package Uploaded" echo "Download URL: $PACKAGE_URL" echo @@ -282,9 +316,9 @@ echo ## ## TEMP WORKAROUND FOR RV32 LONG PATH ON WINDOWS ## -RVTC_VERSION=`cat $PACKAGE_JSON_TEMPLATE | jq -r ".packages[0].platforms[0].toolsDependencies[] | select(.name == \"$RVTC_NAME\") | .version" | cut -d '_' -f 2` +RVTC_VERSION=$(cat "$PACKAGE_JSON_TEMPLATE" | jq -r ".packages[0].platforms[0].toolsDependencies[] | select(.name == \"$RVTC_NAME\") | .version" | cut -d '_' -f 2) # RVTC_VERSION=`date -j -f '%Y%m%d' "$RVTC_VERSION" '+%y%m'` # MacOS -RVTC_VERSION=`date -d "$RVTC_VERSION" '+%y%m'` +RVTC_VERSION=$(date -d "$RVTC_VERSION" '+%y%m') rvtc_jq_arg="\ (.packages[0].platforms[0].toolsDependencies[] | select(.name==\"$RVTC_NAME\")).version = \"$RVTC_VERSION\" |\ (.packages[0].platforms[0].toolsDependencies[] | select(.name==\"$RVTC_NAME\")).name = \"$RVTC_NEW_NAME\" |\ @@ -317,17 +351,20 @@ jq_arg=".packages[0].platforms[0].version = \"$RELEASE_TAG\" | \ .packages[0].platforms[0].checksum = \"SHA-256:$PACKAGE_SHA\"" # Generate package JSONs -echo "Genarating $PACKAGE_JSON_DEV ..." +echo "Generating $PACKAGE_JSON_DEV ..." cat "$PACKAGE_JSON_TEMPLATE" | jq "$jq_arg" > "$OUTPUT_DIR/$PACKAGE_JSON_DEV" if [ "$RELEASE_PRE" == "false" ]; then - echo "Genarating $PACKAGE_JSON_REL ..." + echo "Generating $PACKAGE_JSON_REL ..." cat "$PACKAGE_JSON_TEMPLATE" | jq "$jq_arg" > "$OUTPUT_DIR/$PACKAGE_JSON_REL" fi # Figure out the last release or pre-release echo "Getting previous releases ..." -releasesJson=`curl -sH "Authorization: token $GITHUB_TOKEN" "https://api.github.com/repos/$GITHUB_REPOSITORY/releases" 2>/dev/null` -if [ $? -ne 0 ]; then echo "ERROR: Get Releases Failed! ($?)"; exit 1; fi +releasesJson=$(curl -sH "Authorization: token $GITHUB_TOKEN" "https://api.github.com/repos/$GITHUB_REPOSITORY/releases" 2>/dev/null) +if [ $? -ne 0 ]; then + echo "ERROR: Get Releases Failed! ($?)" + exit 1 +fi set +e prev_release=$(echo "$releasesJson" | jq -e -r ". | map(select(.draft == false and .prerelease == false)) | sort_by(.published_at | - fromdateiso8601) | .[0].tag_name") @@ -347,13 +384,13 @@ echo "Previous (any)release: $prev_any_release" echo # Merge package JSONs with previous releases -if [ ! -z "$prev_any_release" ] && [ "$prev_any_release" != "null" ]; then +if [ -n "$prev_any_release" ] && [ "$prev_any_release" != "null" ]; then echo "Merging with JSON from $prev_any_release ..." merge_package_json "$prev_any_release/$PACKAGE_JSON_DEV" "$OUTPUT_DIR/$PACKAGE_JSON_DEV" fi if [ "$RELEASE_PRE" == "false" ]; then - if [ ! -z "$prev_release" ] && [ "$prev_release" != "null" ]; then + if [ -n "$prev_release" ] && [ "$prev_release" != "null" ]; then echo "Merging with JSON from $prev_release ..." merge_package_json "$prev_release/$PACKAGE_JSON_REL" "$OUTPUT_DIR/$PACKAGE_JSON_REL" fi @@ -363,21 +400,30 @@ fi echo "Installing arduino-cli ..." export PATH="/home/runner/bin:$PATH" -source ./.github/scripts/install-arduino-cli.sh +source "${SCRIPTS_DIR}/install-arduino-cli.sh" echo "Testing $PACKAGE_JSON_DEV install ..." echo "Installing esp32 ..." arduino-cli core install esp32:esp32 --additional-urls "file://$OUTPUT_DIR/$PACKAGE_JSON_DEV" -if [ $? -ne 0 ]; then echo "ERROR: Failed to install esp32 ($?)"; exit 1; fi +if [ $? -ne 0 ]; then + echo "ERROR: Failed to install esp32 ($?)" + exit 1 +fi echo "Compiling example ..." -arduino-cli compile --fqbn esp32:esp32:esp32 $GITHUB_WORKSPACE/libraries/ESP32/examples/CI/CIBoardsTest/CIBoardsTest.ino -if [ $? -ne 0 ]; then echo "ERROR: Failed to compile example ($?)"; exit 1; fi +arduino-cli compile --fqbn esp32:esp32:esp32 "$GITHUB_WORKSPACE"/libraries/ESP32/examples/CI/CIBoardsTest/CIBoardsTest.ino +if [ $? -ne 0 ]; then + echo "ERROR: Failed to compile example ($?)" + exit 1 +fi echo "Uninstalling esp32 ..." arduino-cli core uninstall esp32:esp32 -if [ $? -ne 0 ]; then echo "ERROR: Failed to uninstall esp32 ($?)"; exit 1; fi +if [ $? -ne 0 ]; then + echo "ERROR: Failed to uninstall esp32 ($?)" + exit 1 +fi echo "Test successful!" @@ -386,15 +432,24 @@ if [ "$RELEASE_PRE" == "false" ]; then echo "Installing esp32 ..." arduino-cli core install esp32:esp32 --additional-urls "file://$OUTPUT_DIR/$PACKAGE_JSON_REL" - if [ $? -ne 0 ]; then echo "ERROR: Failed to install esp32 ($?)"; exit 1; fi + if [ $? -ne 0 ]; then + echo "ERROR: Failed to install esp32 ($?)" + exit 1 + fi echo "Compiling example ..." - arduino-cli compile --fqbn esp32:esp32:esp32 $GITHUB_WORKSPACE/libraries/ESP32/examples/CI/CIBoardsTest/CIBoardsTest.ino - if [ $? -ne 0 ]; then echo "ERROR: Failed to compile example ($?)"; exit 1; fi + arduino-cli compile --fqbn esp32:esp32:esp32 "$GITHUB_WORKSPACE"/libraries/ESP32/examples/CI/CIBoardsTest/CIBoardsTest.ino + if [ $? -ne 0 ]; then + echo "ERROR: Failed to compile example ($?)" + exit 1 + fi echo "Uninstalling esp32 ..." arduino-cli core uninstall esp32:esp32 - if [ $? -ne 0 ]; then echo "ERROR: Failed to uninstall esp32 ($?)"; exit 1; fi + if [ $? -ne 0 ]; then + echo "ERROR: Failed to uninstall esp32 ($?)" + exit 1 + fi echo "Test successful!" fi @@ -402,13 +457,13 @@ fi # Upload package JSONs echo "Uploading $PACKAGE_JSON_DEV ..." -echo "Download URL: "`git_safe_upload_asset "$OUTPUT_DIR/$PACKAGE_JSON_DEV"` -echo "Pages URL: "`git_safe_upload_to_pages "$PACKAGE_JSON_DEV" "$OUTPUT_DIR/$PACKAGE_JSON_DEV"` +echo "Download URL: $(git_safe_upload_asset "$OUTPUT_DIR/$PACKAGE_JSON_DEV")" +echo "Pages URL: $(git_safe_upload_to_pages "$PACKAGE_JSON_DEV" "$OUTPUT_DIR/$PACKAGE_JSON_DEV")" echo if [ "$RELEASE_PRE" == "false" ]; then echo "Uploading $PACKAGE_JSON_REL ..." - echo "Download URL: "`git_safe_upload_asset "$OUTPUT_DIR/$PACKAGE_JSON_REL"` - echo "Pages URL: "`git_safe_upload_to_pages "$PACKAGE_JSON_REL" "$OUTPUT_DIR/$PACKAGE_JSON_REL"` + echo "Download URL: $(git_safe_upload_asset "$OUTPUT_DIR/$PACKAGE_JSON_REL")" + echo "Pages URL: $(git_safe_upload_to_pages "$PACKAGE_JSON_REL" "$OUTPUT_DIR/$PACKAGE_JSON_REL")" echo fi diff --git a/.github/scripts/set_push_chunks.sh b/.github/scripts/set_push_chunks.sh index 11a93a7159d..047cae6efd8 100644 --- a/.github/scripts/set_push_chunks.sh +++ b/.github/scripts/set_push_chunks.sh @@ -11,23 +11,23 @@ elif [[ $LIB_CHANGED == 'true' ]]; then echo "Libraries changed. Building only affected sketches." if [[ $NETWORKING_CHANGED == 'true' ]]; then echo "Networking libraries changed. Building networking related sketches." - networking_sketches="$(find libraries/WiFi -name *.ino) " - networking_sketches+="$(find libraries/Ethernet -name *.ino) " - networking_sketches+="$(find libraries/PPP -name *.ino) " - networking_sketches+="$(find libraries/NetworkClientSecure -name *.ino) " - networking_sketches+="$(find libraries/WebServer -name *.ino) " + networking_sketches="$(find libraries/WiFi -name '*.ino') " + networking_sketches+="$(find libraries/Ethernet -name '*.ino') " + networking_sketches+="$(find libraries/PPP -name '*.ino') " + networking_sketches+="$(find libraries/NetworkClientSecure -name '*.ino') " + networking_sketches+="$(find libraries/WebServer -name '*.ino') " fi if [[ $FS_CHANGED == 'true' ]]; then echo "FS libraries changed. Building FS related sketches." - fs_sketches="$(find libraries/SD -name *.ino) " - fs_sketches+="$(find libraries/SD_MMC -name *.ino) " - fs_sketches+="$(find libraries/SPIFFS -name *.ino) " - fs_sketches+="$(find libraries/LittleFS -name *.ino) " - fs_sketches+="$(find libraries/FFat -name *.ino) " + fs_sketches="$(find libraries/SD -name '*.ino') " + fs_sketches+="$(find libraries/SD_MMC -name '*.ino') " + fs_sketches+="$(find libraries/SPIFFS -name '*.ino') " + fs_sketches+="$(find libraries/LittleFS -name '*.ino') " + fs_sketches+="$(find libraries/FFat -name '*.ino') " fi sketches="$networking_sketches $fs_sketches" for file in $LIB_FILES; do - lib=$(echo $file | awk -F "/" '{print $1"/"$2}') + lib=$(echo "$file" | awk -F "/" '{print $1"/"$2}') if [[ "$file" == *.ino ]]; then # If file ends with .ino, add it to the list of sketches echo "Sketch found: $file" @@ -36,14 +36,14 @@ elif [[ $LIB_CHANGED == 'true' ]]; then # If file is inside the src directory, find all sketches in the lib/examples directory echo "Library src file found: $file" if [[ -d $lib/examples ]]; then - lib_sketches=$(find $lib/examples -name *.ino) + lib_sketches=$(find "$lib"/examples -name '*.ino') sketches+="$lib_sketches " echo "Library sketches: $lib_sketches" fi else # If file is in a example folder but it is not a sketch, find all sketches in the current directory echo "File in example folder found: $file" - sketch=$(find $(dirname $file) -name *.ino) + sketch=$(find "$(dirname "$file")" -name '*.ino') sketches+="$sketch " echo "Sketch in example folder: $sketch" fi @@ -53,9 +53,9 @@ fi if [[ -n $sketches ]]; then # Remove duplicates - sketches=$(echo $sketches | tr ' ' '\n' | sort | uniq) + sketches=$(echo "$sketches" | tr ' ' '\n' | sort | uniq) for sketch in $sketches; do - echo $sketch >> sketches_found.txt + echo "$sketch" >> sketches_found.txt chunks_count=$((chunks_count+1)) done echo "Number of sketches found: $chunks_count" @@ -69,15 +69,17 @@ if [[ -n $sketches ]]; then fi chunks='["0"' -for i in $(seq 1 $(( $chunks_count - 1 )) ); do +for i in $(seq 1 $(( chunks_count - 1 )) ); do chunks+=",\"$i\"" done chunks+="]" -echo "build_all=$build_all" >> $GITHUB_OUTPUT -echo "build_libraries=$BUILD_LIBRARIES" >> $GITHUB_OUTPUT -echo "build_static_sketches=$BUILD_STATIC_SKETCHES" >> $GITHUB_OUTPUT -echo "build_idf=$BUILD_IDF" >> $GITHUB_OUTPUT -echo "build_platformio=$BUILD_PLATFORMIO" >> $GITHUB_OUTPUT -echo "chunk_count=$chunks_count" >> $GITHUB_OUTPUT -echo "chunks=$chunks" >> $GITHUB_OUTPUT +{ + echo "build_all=$build_all" + echo "build_libraries=$BUILD_LIBRARIES" + echo "build_static_sketches=$BUILD_STATIC_SKETCHES" + echo "build_idf=$BUILD_IDF" + echo "build_platformio=$BUILD_PLATFORMIO" + echo "chunk_count=$chunks_count" + echo "chunks=$chunks" +} >> "$GITHUB_OUTPUT" diff --git a/.github/scripts/sketch_utils.sh b/.github/scripts/sketch_utils.sh index e2b2ec0b269..d0cdb7589c8 100755 --- a/.github/scripts/sketch_utils.sh +++ b/.github/scripts/sketch_utils.sh @@ -8,10 +8,12 @@ else SDKCONFIG_DIR="tools/esp32-arduino-libs" fi -function check_requirements(){ # check_requirements +function check_requirements { # check_requirements local sketchdir=$1 local sdkconfig_path=$2 local has_requirements=1 + local requirements + local requirements_or if [ ! -f "$sdkconfig_path" ] || [ ! -f "$sketchdir/ci.json" ]; then echo "ERROR: sdkconfig or ci.json not found" 1>&2 @@ -19,10 +21,10 @@ function check_requirements(){ # check_requirements # CI will fail and the user will know that the sketch has a problem. else # Check if the sketch requires any configuration options (AND) - local requirements=$(jq -r '.requires[]? // empty' "$sketchdir/ci.json") + requirements=$(jq -r '.requires[]? // empty' "$sketchdir/ci.json") if [[ "$requirements" != "null" && "$requirements" != "" ]]; then for requirement in $requirements; do - requirement=$(echo $requirement | xargs) + requirement=$(echo "$requirement" | xargs) found_line=$(grep -E "^$requirement" "$sdkconfig_path") if [[ "$found_line" == "" ]]; then has_requirements=0 @@ -31,11 +33,11 @@ function check_requirements(){ # check_requirements fi # Check if the sketch requires any configuration options (OR) - local requirements_or=$(jq -r '.requires_any[]? // empty' "$sketchdir/ci.json") + requirements_or=$(jq -r '.requires_any[]? // empty' "$sketchdir/ci.json") if [[ "$requirements_or" != "null" && "$requirements_or" != "" ]]; then local found=false for requirement in $requirements_or; do - requirement=$(echo $requirement | xargs) + requirement=$(echo "$requirement" | xargs) found_line=$(grep -E "^$requirement" "$sdkconfig_path") if [[ "$found_line" != "" ]]; then found=true @@ -51,8 +53,8 @@ function check_requirements(){ # check_requirements echo $has_requirements } -function build_sketch(){ # build_sketch [extra-options] - while [ ! -z "$1" ]; do +function build_sketch { # build_sketch [extra-options] + while [ -n "$1" ]; do case "$1" in -ai ) shift @@ -97,10 +99,10 @@ function build_sketch(){ # build_sketch [ex shift done - xtra_opts=$* + xtra_opts=("$@") len=0 - if [ -z $sketchdir ]; then + if [ -z "$sketchdir" ]; then echo "ERROR: Sketch directory not provided" echo "$USAGE" exit 1 @@ -108,8 +110,8 @@ function build_sketch(){ # build_sketch [ex # No FQBN was passed, try to get it from other options - if [ -z $fqbn ]; then - if [ -z $target ]; then + if [ -z "$fqbn" ]; then + if [ -z "$target" ]; then echo "ERROR: Unspecified chip" echo "$USAGE" exit 1 @@ -120,25 +122,25 @@ function build_sketch(){ # build_sketch [ex # precedence. Note that the following logic also falls to the default # parameters if no arguments were passed and no file was found. - if [ -z $options ] && [ -f $sketchdir/ci.json ]; then + if [ -z "$options" ] && [ -f "$sketchdir"/ci.json ]; then # The config file could contain multiple FQBNs for one chip. If # that's the case we build one time for every FQBN. - len=`jq -r --arg target $target '.fqbn[$target] | length' $sketchdir/ci.json` - if [ $len -gt 0 ]; then - fqbn=`jq -r --arg target $target '.fqbn[$target] | sort' $sketchdir/ci.json` + len=$(jq -r --arg target "$target" '.fqbn[$target] | length' "$sketchdir"/ci.json) + if [ "$len" -gt 0 ]; then + fqbn=$(jq -r --arg target "$target" '.fqbn[$target] | sort' "$sketchdir"/ci.json) fi fi - if [ ! -z $options ] || [ $len -eq 0 ]; then + if [ -n "$options" ] || [ "$len" -eq 0 ]; then # Since we are passing options, we will end up with only one FQBN to # build. len=1 - if [ -f $sketchdir/ci.json ]; then - fqbn_append=`jq -r '.fqbn_append' $sketchdir/ci.json` - if [ $fqbn_append == "null" ]; then + if [ -f "$sketchdir"/ci.json ]; then + fqbn_append=$(jq -r '.fqbn_append' "$sketchdir"/ci.json) + if [ "$fqbn_append" == "null" ]; then fqbn_append="" fi fi @@ -184,6 +186,10 @@ function build_sketch(){ # build_sketch [ex [ -n "${options:-$esp32h2_opts}" ] && opt=":${options:-$esp32h2_opts}" fqbn="espressif:esp32:esp32h2$opt" ;; + *) + echo "ERROR: Invalid chip: $target" + exit 1 + ;; esac # Make it look like a JSON array. @@ -202,7 +208,7 @@ function build_sketch(){ # build_sketch [ex exit 1 fi - # The directory that will hold all the artifcats (the build directory) is + # The directory that will hold all the artifacts (the build directory) is # provided through: # 1. An env variable called ARDUINO_BUILD_DIR. # 2. Created at the sketch level as "build" in the case of a single @@ -210,17 +216,18 @@ function build_sketch(){ # build_sketch [ex # 3. Created at the sketch level as "buildX" where X is the number # of configuration built in case of a multiconfiguration test. - sketchname=$(basename $sketchdir) + sketchname=$(basename "$sketchdir") + local has_requirements - if [ -f $sketchdir/ci.json ]; then + if [ -f "$sketchdir"/ci.json ]; then # If the target is listed as false, skip the sketch. Otherwise, include it. - is_target=$(jq -r --arg target $target '.targets[$target]' $sketchdir/ci.json) + is_target=$(jq -r --arg target "$target" '.targets[$target]' "$sketchdir"/ci.json) if [[ "$is_target" == "false" ]]; then echo "Skipping $sketchname for target $target" exit 0 fi - local has_requirements=$(check_requirements "$sketchdir" "$SDKCONFIG_DIR/$target/sdkconfig") + has_requirements=$(check_requirements "$sketchdir" "$SDKCONFIG_DIR/$target/sdkconfig") if [ "$has_requirements" == "0" ]; then echo "Target $target does not meet the requirements for $sketchname. Skipping." exit 0 @@ -230,7 +237,7 @@ function build_sketch(){ # build_sketch [ex ARDUINO_CACHE_DIR="$HOME/.arduino/cache.tmp" if [ -n "$ARDUINO_BUILD_DIR" ]; then build_dir="$ARDUINO_BUILD_DIR" - elif [ $len -eq 1 ]; then + elif [ "$len" -eq 1 ]; then # build_dir="$sketchdir/build" build_dir="$HOME/.arduino/tests/$sketchname/build.tmp" fi @@ -238,52 +245,52 @@ function build_sketch(){ # build_sketch [ex output_file="$HOME/.arduino/cli_compile_output.txt" sizes_file="$GITHUB_WORKSPACE/cli_compile_$chunk_index.json" + echo "${xtra_opts[@]}" + mkdir -p "$ARDUINO_CACHE_DIR" - for i in `seq 0 $(($len - 1))` - do - if [ $len -ne 1 ]; then - # build_dir="$sketchdir/build$i" - build_dir="$HOME/.arduino/tests/$sketchname/build$i.tmp" + for i in $(seq 0 $((len - 1))); do + if [ "$len" -ne 1 ]; then + # build_dir="$sketchdir/build$i" + build_dir="$HOME/.arduino/tests/$sketchname/build$i.tmp" fi - rm -rf $build_dir - mkdir -p $build_dir + rm -rf "$build_dir" + mkdir -p "$build_dir" - currfqbn=`echo $fqbn | jq -r --argjson i $i '.[$i]'` + currfqbn=$(echo "$fqbn" | jq -r --argjson i "$i" '.[$i]') if [ -f "$ide_path/arduino-cli" ]; then echo "Building $sketchname with arduino-cli and FQBN=$currfqbn" - curroptions=`echo "$currfqbn" | cut -d':' -f4` - currfqbn=`echo "$currfqbn" | cut -d':' -f1-3` - $ide_path/arduino-cli compile \ + curroptions=$(echo "$currfqbn" | cut -d':' -f4) + currfqbn=$(echo "$currfqbn" | cut -d':' -f1-3) + "$ide_path"/arduino-cli compile \ --fqbn "$currfqbn" \ --board-options "$curroptions" \ --warnings "all" \ --build-property "compiler.warning_flags.all=-Wall -Werror=all -Wextra" \ - --build-cache-path "$ARDUINO_CACHE_DIR" \ --build-path "$build_dir" \ - $xtra_opts "${sketchdir}" \ - 2>&1 | tee $output_file + "${xtra_opts[@]}" "${sketchdir}" \ + 2>&1 | tee "$output_file" exit_status=${PIPESTATUS[0]} - if [ $exit_status -ne 0 ]; then + if [ "$exit_status" -ne 0 ]; then echo "ERROR: Compilation failed with error code $exit_status" - exit $exit_status + exit "$exit_status" fi - if [ $log_compilation ]; then + if [ -n "$log_compilation" ]; then #Extract the program storage space and dynamic memory usage in bytes and percentage in separate variables from the output, just the value without the string - flash_bytes=$(grep -oE 'Sketch uses ([0-9]+) bytes' $output_file | awk '{print $3}') - flash_percentage=$(grep -oE 'Sketch uses ([0-9]+) bytes \(([0-9]+)%\)' $output_file | awk '{print $5}' | tr -d '(%)') - ram_bytes=$(grep -oE 'Global variables use ([0-9]+) bytes' $output_file | awk '{print $4}') - ram_percentage=$(grep -oE 'Global variables use ([0-9]+) bytes \(([0-9]+)%\)' $output_file | awk '{print $6}' | tr -d '(%)') + flash_bytes=$(grep -oE 'Sketch uses ([0-9]+) bytes' "$output_file" | awk '{print $3}') + flash_percentage=$(grep -oE 'Sketch uses ([0-9]+) bytes \(([0-9]+)%\)' "$output_file" | awk '{print $5}' | tr -d '(%)') + ram_bytes=$(grep -oE 'Global variables use ([0-9]+) bytes' "$output_file" | awk '{print $4}') + ram_percentage=$(grep -oE 'Global variables use ([0-9]+) bytes \(([0-9]+)%\)' "$output_file" | awk '{print $6}' | tr -d '(%)') # Extract the directory path excluding the filename directory_path=$(dirname "$sketch") # Define the constant part constant_part="/home/runner/Arduino/hardware/espressif/esp32/libraries/" - # Extract the desired substring using sed - lib_sketch_name=$(echo "$directory_path" | sed "s|$constant_part||") + # Extract the desired substring + lib_sketch_name="${directory_path#"$constant_part"}" #append json file where key is fqbn, sketch name, sizes -> extracted values echo "{\"name\": \"$lib_sketch_name\", \"sizes\": [{ @@ -299,15 +306,15 @@ function build_sketch(){ # build_sketch [ex echo "Building $sketchname with arduino-builder and FQBN=$currfqbn" echo "Build path = $build_dir" - $ide_path/arduino-builder -compile -logger=human -core-api-version=10810 \ - -fqbn=\"$currfqbn\" \ + "$ide_path"/arduino-builder -compile -logger=human -core-api-version=10810 \ + -fqbn=\""$currfqbn"\" \ -warnings="all" \ -tools "$ide_path/tools-builder" \ -hardware "$user_path/hardware" \ -libraries "$user_path/libraries" \ -build-cache "$ARDUINO_CACHE_DIR" \ -build-path "$build_dir" \ - $xtra_opts "${sketchdir}/${sketchname}.ino" + "${xtra_opts[@]}" "${sketchdir}/${sketchname}.ino" exit_status=$? if [ $exit_status -ne 0 ]; then @@ -334,15 +341,16 @@ function build_sketch(){ # build_sketch [ex unset options } -function count_sketches(){ # count_sketches [target] [file] [ignore-requirements] +function count_sketches { # count_sketches [target] [file] [ignore-requirements] local path=$1 local target=$2 local ignore_requirements=$3 local file=$4 + local sketches if [ $# -lt 1 ]; then - echo "ERROR: Illegal number of parameters" - echo "USAGE: ${0} count [target]" + echo "ERROR: Illegal number of parameters" + echo "USAGE: ${0} count [target]" fi rm -rf sketches.txt @@ -352,42 +360,47 @@ function count_sketches(){ # count_sketches [target] [file] [ignore-requi fi if [ -f "$file" ]; then - local sketches=$(cat $file) + sketches=$(cat "$file") else - local sketches=$(find $path -name *.ino | sort) + sketches=$(find "$path" -name '*.ino' | sort) fi local sketchnum=0 for sketch in $sketches; do - local sketchdir=$(dirname $sketch) - local sketchdirname=$(basename $sketchdir) - local sketchname=$(basename $sketch) + local sketchdir + local sketchdirname + local sketchname + local has_requirements + + sketchdir=$(dirname "$sketch") + sketchdirname=$(basename "$sketchdir") + sketchname=$(basename "$sketch") + if [[ "$sketchdirname.ino" != "$sketchname" ]]; then continue elif [[ -n $target ]] && [[ -f $sketchdir/ci.json ]]; then # If the target is listed as false, skip the sketch. Otherwise, include it. - is_target=$(jq -r --arg target $target '.targets[$target]' $sketchdir/ci.json) + is_target=$(jq -r --arg target "$target" '.targets[$target]' "$sketchdir"/ci.json) if [[ "$is_target" == "false" ]]; then continue fi if [ "$ignore_requirements" != "1" ]; then - local has_requirements=$(check_requirements "$sketchdir" "$SDKCONFIG_DIR/$target/sdkconfig") + has_requirements=$(check_requirements "$sketchdir" "$SDKCONFIG_DIR/$target/sdkconfig") if [ "$has_requirements" == "0" ]; then continue fi fi fi - echo $sketch >> sketches.txt - sketchnum=$(($sketchnum + 1)) + echo "$sketch" >> sketches.txt + sketchnum=$((sketchnum + 1)) done return $sketchnum } -function build_sketches(){ # build_sketches [extra-options] - - local args="" - while [ ! -z "$1" ]; do +function build_sketches { # build_sketches [extra-options] + local args=() + while [ -n "$1" ]; do case $1 in -ai ) shift @@ -400,12 +413,12 @@ function build_sketches(){ # build_sketches info/wokwi_types.txt echo "[$targets]" > info/targets.txt -echo "build-types=[$build_types]" >> $GITHUB_OUTPUT -echo "hw-types=[$hw_types]" >> $GITHUB_OUTPUT -echo "wokwi-types=[$wokwi_types]" >> $GITHUB_OUTPUT -echo "qemu-types=[$qemu_types]" >> $GITHUB_OUTPUT -echo "targets=[$targets]" >> $GITHUB_OUTPUT +{ + echo "build-types=[$build_types]" + echo "hw-types=[$hw_types]" + echo "wokwi-types=[$wokwi_types]" + echo "qemu-types=[$qemu_types]" + echo "targets=[$targets]" +} >> "$GITHUB_OUTPUT" diff --git a/.github/scripts/tests_run.sh b/.github/scripts/tests_run.sh index 16b0f2fb500..513fd16b371 100755 --- a/.github/scripts/tests_run.sh +++ b/.github/scripts/tests_run.sh @@ -1,126 +1,130 @@ #!/bin/bash -function run_test() { +function run_test { local target=$1 local sketch=$2 local options=$3 local erase_flash=$4 - local sketchdir=$(dirname $sketch) - local sketchname=$(basename $sketchdir) + local sketchdir + local sketchname local result=0 local error=0 local sdkconfig_path + local extra_args - if [ $options -eq 0 ] && [ -f $sketchdir/ci.json ]; then - len=`jq -r --arg target $target '.fqbn[$target] | length' $sketchdir/ci.json` - if [ $len -eq 0 ]; then + sketchdir=$(dirname "$sketch") + sketchname=$(basename "$sketchdir") + + if [ "$options" -eq 0 ] && [ -f "$sketchdir"/ci.json ]; then + len=$(jq -r --arg target "$target" '.fqbn[$target] | length' "$sketchdir"/ci.json) + if [ "$len" -eq 0 ]; then len=1 fi else len=1 fi - if [ $len -eq 1 ]; then + if [ "$len" -eq 1 ]; then sdkconfig_path="$HOME/.arduino/tests/$sketchname/build.tmp/sdkconfig" else sdkconfig_path="$HOME/.arduino/tests/$sketchname/build0.tmp/sdkconfig" fi - if [ -f $sketchdir/ci.json ]; then + if [ -f "$sketchdir"/ci.json ]; then # If the target or platform is listed as false, skip the sketch. Otherwise, include it. - is_target=$(jq -r --arg target $target '.targets[$target]' $sketchdir/ci.json) - selected_platform=$(jq -r --arg platform $platform '.platforms[$platform]' $sketchdir/ci.json) + is_target=$(jq -r --arg target "$target" '.targets[$target]' "$sketchdir"/ci.json) + selected_platform=$(jq -r --arg platform "$platform" '.platforms[$platform]' "$sketchdir"/ci.json) if [[ $is_target == "false" ]] || [[ $selected_platform == "false" ]]; then - printf "\033[93mSkipping $sketchname test for $target, platform: $platform\033[0m\n" + printf "\033[93mSkipping %s test for %s, platform: %s\033[0m\n" "$sketchname" "$target" "$platform" printf "\n\n\n" return 0 fi fi - if [ ! -f $sdkconfig_path ]; then - printf "\033[93mSketch $sketchname not built\nMight be due to missing target requirements or build failure\033[0m\n" + if [ ! -f "$sdkconfig_path" ]; then + printf "\033[93mSketch %s not built\nMight be due to missing target requirements or build failure\033[0m\n" "$sketchname" printf "\n\n\n" return 0 fi - local right_target=$(grep -E "^CONFIG_IDF_TARGET=\"$target\"$" "$sdkconfig_path") + local right_target + right_target=$(grep -E "^CONFIG_IDF_TARGET=\"$target\"$" "$sdkconfig_path") if [ -z "$right_target" ]; then - printf "\033[91mError: Sketch $sketchname compiled for different target\n\033[0m\n" + printf "\033[91mError: Sketch %s compiled for different target\n\033[0m\n" "$sketchname" printf "\n\n\n" return 1 fi - if [ $len -eq 1 ]; then - # build_dir="$sketchdir/build" - build_dir="$HOME/.arduino/tests/$sketchname/build.tmp" - report_file="$sketchdir/$target/$sketchname.xml" + if [ "$len" -eq 1 ]; then + # build_dir="$sketchdir/build" + build_dir="$HOME/.arduino/tests/$sketchname/build.tmp" + report_file="$sketchdir/$target/$sketchname.xml" fi - for i in `seq 0 $(($len - 1))` - do + for i in $(seq 0 $((len - 1))); do fqbn="Default" - if [ $len -ne 1 ]; then - fqbn=`jq -r --arg target $target --argjson i $i '.fqbn[$target] | sort | .[$i]' $sketchdir/ci.json` - elif [ -f $sketchdir/ci.json ]; then - has_fqbn=`jq -r --arg target $target '.fqbn[$target]' $sketchdir/ci.json` + if [ "$len" -ne 1 ]; then + fqbn=$(jq -r --arg target "$target" --argjson i "$i" '.fqbn[$target] | sort | .[$i]' "$sketchdir"/ci.json) + elif [ -f "$sketchdir"/ci.json ]; then + has_fqbn=$(jq -r --arg target "$target" '.fqbn[$target]' "$sketchdir"/ci.json) if [ "$has_fqbn" != "null" ]; then - fqbn=`jq -r --arg target $target '.fqbn[$target] | .[0]' $sketchdir/ci.json` + fqbn=$(jq -r --arg target "$target" '.fqbn[$target] | .[0]' "$sketchdir"/ci.json) fi fi - printf "\033[95mRunning test: $sketchname -- Config: $fqbn\033[0m\n" - if [ $erase_flash -eq 1 ]; then - esptool.py -c $target erase_flash + printf "\033[95mRunning test: %s -- Config: %s\033[0m\n" "$sketchname" "$fqbn" + if [ "$erase_flash" -eq 1 ]; then + esptool.py -c "$target" erase_flash fi - if [ $len -ne 1 ]; then + if [ "$len" -ne 1 ]; then # build_dir="$sketchdir/build$i" build_dir="$HOME/.arduino/tests/$sketchname/build$i.tmp" report_file="$sketchdir/$target/$sketchname$i.xml" fi if [ $platform == "wokwi" ]; then - extra_args="--target $target --embedded-services arduino,wokwi --wokwi-timeout=$wokwi_timeout" + extra_args=("--target" "$target" "--embedded-services" "arduino,wokwi" "--wokwi-timeout=$wokwi_timeout") if [[ -f "$sketchdir/scenario.yaml" ]]; then - extra_args+=" --wokwi-scenario $sketchdir/scenario.yaml" + extra_args+=("--wokwi-scenario" "$sketchdir/scenario.yaml") fi if [[ -f "$sketchdir/diagram.$target.json" ]]; then - extra_args+=" --wokwi-diagram $sketchdir/diagram.$target.json" + extra_args+=("--wokwi-diagram" "$sketchdir/diagram.$target.json") fi elif [ $platform == "qemu" ]; then PATH=$HOME/qemu/bin:$PATH - extra_args="--embedded-services qemu --qemu-image-path $build_dir/$sketchname.ino.merged.bin" + extra_args=("--embedded-services" "qemu" "--qemu-image-path" "$build_dir/$sketchname.ino.merged.bin") - if [ $target == "esp32" ] || [ $target == "esp32s3" ]; then - extra_args+=" --qemu-prog-path qemu-system-xtensa --qemu-cli-args=\"-machine $target -m 4M -nographic\"" - elif [ $target == "esp32c3" ]; then - extra_args+=" --qemu-prog-path qemu-system-riscv32 --qemu-cli-args=\"-machine $target -icount 3 -nographic\"" + if [ "$target" == "esp32" ] || [ "$target" == "esp32s3" ]; then + extra_args+=("--qemu-prog-path" "qemu-system-xtensa" "--qemu-cli-args=\"-machine $target -m 4M -nographic\"") + elif [ "$target" == "esp32c3" ]; then + extra_args+=("--qemu-prog-path" "qemu-system-riscv32" "--qemu-cli-args=\"-machine $target -icount 3 -nographic\"") else - printf "\033[91mUnsupported QEMU target: $target\033[0m\n" + printf "\033[91mUnsupported QEMU target: %s\033[0m\n" "$target" exit 1 fi else - extra_args="--embedded-services esp,arduino" + extra_args=("--embedded-services" "esp,arduino") fi - rm $sketchdir/diagram.json 2>/dev/null || true + rm "$sketchdir"/diagram.json 2>/dev/null || true result=0 - printf "\033[95mpytest $sketchdir/test_$sketchname.py --build-dir $build_dir --junit-xml=$report_file $extra_args\033[0m\n" - bash -c "set +e; pytest $sketchdir/test_$sketchname.py --build-dir $build_dir --junit-xml=$report_file $extra_args; exit \$?" || result=$? + printf "\033[95mpytest \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "${extra_args[*]@Q}" + bash -c "set +e; pytest \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" ${extra_args[*]@Q}; exit \$?" || result=$? printf "\n" if [ $result -ne 0 ]; then result=0 - printf "\033[95mRetrying test: $sketchname -- Config: $i\033[0m\n" - printf "\033[95mpytest $sketchdir/test_$sketchname.py --build-dir $build_dir --junit-xml=$report_file $extra_args\033[0m\n" - bash -c "set +e; pytest $sketchdir/test_$sketchname.py --build-dir $build_dir --junit-xml=$report_file $extra_args; exit \$?" || result=$? + printf "\033[95mRetrying test: %s -- Config: %s\033[0m\n" "$sketchname" "$i" + printf "\033[95mpytest \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "${extra_args[*]@Q}" + bash -c "set +e; pytest \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" ${extra_args[*]@Q}; exit \$?" || result=$? printf "\n" if [ $result -ne 0 ]; then - printf "\033[91mFailed test: $sketchname -- Config: $i\033[0m\n\n" - error=$result + printf "\033[91mFailed test: %s -- Config: %s\033[0m\n\n" "$sketchname" "$i" + error=$result fi fi done @@ -136,13 +140,13 @@ chunk_run=0 options=0 erase=0 -while [ ! -z "$1" ]; do +while [ -n "$1" ]; do case $1 in -c ) chunk_run=1 ;; -Q ) - if [ ! -d $QEMU_PATH ]; then + if [ ! -d "$QEMU_PATH" ]; then echo "QEMU path $QEMU_PATH does not exist" exit 1 fi @@ -188,98 +192,97 @@ while [ ! -z "$1" ]; do test_type=$1 ;; * ) - break - ;; + break + ;; esac shift done if [ ! $platform == "qemu" ]; then - source ${SCRIPTS_DIR}/install-arduino-ide.sh + source "${SCRIPTS_DIR}/install-arduino-ide.sh" fi # If sketch is provided and test type is not, test type is inferred from the sketch path if [[ $test_type == "all" ]] || [[ -z $test_type ]]; then if [ -n "$sketch" ]; then - tmp_sketch_path=$(find tests -name $sketch.ino) - test_type=$(basename $(dirname $(dirname "$tmp_sketch_path"))) + tmp_sketch_path=$(find tests -name "$sketch".ino) + test_type=$(basename "$(dirname "$(dirname "$tmp_sketch_path")")") echo "Sketch $sketch test type: $test_type" test_folder="$PWD/tests/$test_type" else - test_folder="$PWD/tests" + test_folder="$PWD/tests" fi else test_folder="$PWD/tests/$test_type" fi if [ $chunk_run -eq 0 ]; then - if [ -z $sketch ]; then + if [ -z "$sketch" ]; then echo "ERROR: Sketch name is required for single test run" exit 1 fi - run_test $target $test_folder/$sketch/$sketch.ino $options $erase + run_test "$target" "$test_folder"/"$sketch"/"$sketch".ino $options $erase exit $? else - if [ "$chunk_max" -le 0 ]; then - echo "ERROR: Chunks count must be positive number" - exit 1 - fi - - if [ "$chunk_index" -ge "$chunk_max" ] && [ "$chunk_max" -ge 2 ]; then - echo "ERROR: Chunk index must be less than chunks count" - exit 1 - fi - - set +e - # Ignore requirements as we don't have the libs. The requirements will be checked in the run_test function - ${COUNT_SKETCHES} "$test_folder" "$target" "1" - sketchcount=$? - set -e - sketches=$(cat sketches.txt) - rm -rf sketches.txt - - chunk_size=$(( $sketchcount / $chunk_max )) - all_chunks=$(( $chunk_max * $chunk_size )) - if [ "$all_chunks" -lt "$sketchcount" ]; then - chunk_size=$(( $chunk_size + 1 )) - fi - - start_index=0 - end_index=0 - if [ "$chunk_index" -ge "$chunk_max" ]; then - start_index=$chunk_index - end_index=$sketchcount - else - start_index=$(( $chunk_index * $chunk_size )) - if [ "$sketchcount" -le "$start_index" ]; then - exit 0 - fi - - end_index=$(( $(( $chunk_index + 1 )) * $chunk_size )) - if [ "$end_index" -gt "$sketchcount" ]; then - end_index=$sketchcount - fi - fi - - start_num=$(( $start_index + 1 )) - sketchnum=0 - error=0 - - for sketch in $sketches; do - - sketchnum=$(($sketchnum + 1)) - if [ "$sketchnum" -le "$start_index" ] \ - || [ "$sketchnum" -gt "$end_index" ]; then - continue - fi - - printf "\033[95mSketch Index $(($sketchnum - 1))\033[0m\n" - - exit_code=0 - run_test $target $sketch $options $erase || exit_code=$? - if [ $exit_code -ne 0 ]; then - error=$exit_code - fi - done - exit $error + if [ "$chunk_max" -le 0 ]; then + echo "ERROR: Chunks count must be positive number" + exit 1 + fi + + if [ "$chunk_index" -ge "$chunk_max" ] && [ "$chunk_max" -ge 2 ]; then + echo "ERROR: Chunk index must be less than chunks count" + exit 1 + fi + + set +e + # Ignore requirements as we don't have the libs. The requirements will be checked in the run_test function + ${COUNT_SKETCHES} "$test_folder" "$target" "1" + sketchcount=$? + set -e + sketches=$(cat sketches.txt) + rm -rf sketches.txt + + chunk_size=$(( sketchcount / chunk_max )) + all_chunks=$(( chunk_max * chunk_size )) + if [ "$all_chunks" -lt "$sketchcount" ]; then + chunk_size=$(( chunk_size + 1 )) + fi + + start_index=0 + end_index=0 + if [ "$chunk_index" -ge "$chunk_max" ]; then + start_index=$chunk_index + end_index=$sketchcount + else + start_index=$(( chunk_index * chunk_size )) + if [ "$sketchcount" -le "$start_index" ]; then + exit 0 + fi + + end_index=$(( $(( chunk_index + 1 )) * chunk_size )) + if [ "$end_index" -gt "$sketchcount" ]; then + end_index=$sketchcount + fi + fi + + sketchnum=0 + error=0 + + for sketch in $sketches; do + + sketchnum=$((sketchnum + 1)) + if [ "$sketchnum" -le "$start_index" ] \ + || [ "$sketchnum" -gt "$end_index" ]; then + continue + fi + + printf "\033[95mSketch Index %s\033[0m\n" "$((sketchnum - 1))" + + exit_code=0 + run_test "$target" "$sketch" $options $erase || exit_code=$? + if [ $exit_code -ne 0 ]; then + error=$exit_code + fi + done + exit $error fi diff --git a/.github/scripts/update-version.sh b/.github/scripts/update-version.sh index cbc31378b15..9a38b27a57a 100755 --- a/.github/scripts/update-version.sh +++ b/.github/scripts/update-version.sh @@ -1,20 +1,21 @@ #!/bin/bash +# shellcheck disable=SC2002 # For reference: add tools for all boards by replacing one line in each board # "[board].upload.tool=esptool_py" to "[board].upload.tool=esptool_py\n[board].upload.tool.default=esptool_py\n[board].upload.tool.network=esp_ota" #cat boards.txt | sed "s/\([a-zA-Z0-9_\-]*\)\.upload\.tool\=esptool_py/\1\.upload\.tool\=esptool_py\\n\1\.upload\.tool\.default\=esptool_py\\n\1\.upload\.tool\.network\=esp_ota/" if [ ! $# -eq 3 ]; then - echo "Bad number of arguments: $#" >&2 - echo "usage: $0 " >&2 - exit 1 + echo "Bad number of arguments: $#" >&2 + echo "usage: $0 " >&2 + exit 1 fi re='^[0-9]+$' if [[ ! $1 =~ $re ]] || [[ ! $2 =~ $re ]] || [[ ! $3 =~ $re ]] ; then - echo "error: Not a valid version: $1.$2.$3" >&2 - echo "usage: $0 " >&2 - exit 1 + echo "error: Not a valid version: $1.$2.$3" >&2 + echo "usage: $0 " >&2 + exit 1 fi ESP_ARDUINO_VERSION_MAJOR="$1" @@ -36,11 +37,12 @@ sed "s/#define ESP_ARDUINO_VERSION_MAJOR.*/#define ESP_ARDUINO_VERSION_MAJOR $ES sed "s/#define ESP_ARDUINO_VERSION_MINOR.*/#define ESP_ARDUINO_VERSION_MINOR $ESP_ARDUINO_VERSION_MINOR/g" | \ sed "s/#define ESP_ARDUINO_VERSION_PATCH.*/#define ESP_ARDUINO_VERSION_PATCH $ESP_ARDUINO_VERSION_PATCH/g" > __esp_arduino_version.h && mv __esp_arduino_version.h cores/esp32/esp_arduino_version.h -for lib in `ls libraries`; do - if [ -f "libraries/$lib/library.properties" ]; then - echo "Updating Library $lib..." - cat "libraries/$lib/library.properties" | sed "s/version=.*/version=$ESP_ARDUINO_VERSION/g" > "libraries/$lib/__library.properties" && mv "libraries/$lib/__library.properties" "libraries/$lib/library.properties" - fi +libraries=$(find libraries -maxdepth 1 -mindepth 1 -type d -exec basename {} \;) +for lib in $libraries; do + if [ -f "libraries/$lib/library.properties" ]; then + echo "Updating Library $lib..." + cat "libraries/$lib/library.properties" | sed "s/version=.*/version=$ESP_ARDUINO_VERSION/g" > "libraries/$lib/__library.properties" && mv "libraries/$lib/__library.properties" "libraries/$lib/library.properties" + fi done exit 0 diff --git a/.github/scripts/upload_py_tools.sh b/.github/scripts/upload_py_tools.sh index 0544bccd710..abe18a50c6e 100755 --- a/.github/scripts/upload_py_tools.sh +++ b/.github/scripts/upload_py_tools.sh @@ -1,11 +1,12 @@ #!/bin/bash + CHANGED_FILES=$1 echo "Pushing '$CHANGED_FILES' as github-actions[bot]" git config --global github.user "github-actions[bot]" git config --global user.name "github-actions[bot]" git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" for tool in $CHANGED_FILES; do - git add tools/$tool.exe + git add tools/"$tool".exe done git commit -m "change(tools): Push generated binaries to PR" git push diff --git a/docs/utils.sh b/docs/utils.sh index 84f37489975..3a860ac8a2c 100644 --- a/docs/utils.sh +++ b/docs/utils.sh @@ -1,18 +1,19 @@ +#!/bin/bash # Bash helper functions for adding SSH keys -function add_ssh_keys() { - local key_string="${1}" - mkdir -p ~/.ssh - chmod 700 ~/.ssh - echo -n "${key_string}" >~/.ssh/id_rsa_base64 - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 >~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa +function add_ssh_keys { + local key_string="${1}" + mkdir -p ~/.ssh + chmod 700 ~/.ssh + echo -n "${key_string}" >~/.ssh/id_rsa_base64 + base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 >~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa } -function add_doc_server_ssh_keys() { - local key_string="${1}" - local server_url="${2}" - local server_user="${3}" - add_ssh_keys "${key_string}" - echo -e "Host ${server_url}\n\tStrictHostKeyChecking no\n\tUser ${server_user}\n" >>~/.ssh/config +function add_doc_server_ssh_keys { + local key_string="${1}" + local server_url="${2}" + local server_user="${3}" + add_ssh_keys "${key_string}" + echo -e "Host ${server_url}\n\tStrictHostKeyChecking no\n\tUser ${server_user}\n" >>~/.ssh/config } diff --git a/tools/add_lib.sh b/tools/add_lib.sh index 4ec73c4f7f7..9760f8114c6 100755 --- a/tools/add_lib.sh +++ b/tools/add_lib.sh @@ -1,4 +1,5 @@ #!/bin/bash + HELP="This script help to add library when using arduino-esp32 as an ESP-IDF component The script accepts up to three arguments: -n NEW: URL address to new library on GIThub (cannot be combined with -e) @@ -26,119 +27,126 @@ n_param="" # Parse the command-line arguments using getopts while getopts "he:l:n:" opt; do - case $opt in - h) - echo "$HELP" - exit 0 - ;; - e) - #e_param="$OPTARG" - e_param="${OPTARG/#~/$HOME}" - ;; - l) - #l_param="$OPTARG" - l_param="${OPTARG/#~/$HOME}" - ;; - n) - n_param=$OPTARG - ;; - \?) - echo "Invalid option: -$OPTARG" >&2 - echo $HELP - exit 1 - ;; - :) - echo "Option -$OPTARG requires an argument." >&2 - echo $HELP - exit 1 - ;; - esac + case $opt in + h) + echo "$HELP" + exit 0 + ;; + e) + #e_param="$OPTARG" + e_param="${OPTARG/#~/$HOME}" + ;; + l) + #l_param="$OPTARG" + l_param="${OPTARG/#~/$HOME}" + ;; + n) + n_param=$OPTARG + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + echo "$HELP" + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + echo "$HELP" + exit 1 + ;; + *) + echo "Invalid option: -$OPTARG" >&2 + echo "$HELP" + exit 1 + ;; + esac done # No parameter check if [[ -z "$e_param" ]] && [[ -z "$l_param" ]] && [[ -z "$n_param" ]]; then - echo "Error: No parameters" >&2 - echo "$HELP" - exit 1 + echo "Error: No parameters" >&2 + echo "$HELP" + exit 1 fi # Only local path check (not permitted) -if [[ -z "$e_param" ]] && [[ ! -z "$l_param" ]] && [[ -z "$n_param" ]]; then - echo "Error: -l parameter must be paired with -e or -n" >&2 - echo "$HELP" - exit 1 +if [[ -z "$e_param" ]] && [[ -n "$l_param" ]] && [[ -z "$n_param" ]]; then + echo "Error: -l parameter must be paired with -e or -n" >&2 + echo "$HELP" + exit 1 fi # Invalid combination check -if [[ ! -z $e_param ]] && [[ ! -z $n_param ]]; then - echo "ERROR: Cannot combine -n with -e" >&2 - echo "$HELP" - exit 1 +if [[ -n $e_param ]] && [[ -n $n_param ]]; then + echo "ERROR: Cannot combine -n with -e" >&2 + echo "$HELP" + exit 1 fi # Check existing lib -if [[ ! -z "$e_param" ]]; then - if [[ ! -d "${e_param/#~/$HOME}" ]]; then # this works! - echo "Error: existing library parameter - path does not exist" >&2 - exit 1 - fi +if [[ -n "$e_param" ]]; then + if [[ ! -d "${e_param/#~/$HOME}" ]]; then # this works! + echo "Error: existing library parameter - path does not exist" >&2 + exit 1 + fi fi LIBRARY="" # Only existing library was supplied -if [[ ! -z $e_param ]] && [[ -z $l_param ]] && [[ -z $n_param ]]; then - LIBRARY=$e_param +if [[ -n $e_param ]] && [[ -z $l_param ]] && [[ -z $n_param ]]; then + LIBRARY=$e_param fi # Install new lib -if [ ! -z $n_param ]; then - INSTALL_TARGET="" - if [ -z $l_param ]; then - # If local path for project is not supplied - use as INSTALL_TARGET Arduino libraries path - INSTALL_TARGET=$ARDUINO_LIBS_PATH/$(basename "$n_param") - else - INSTALL_TARGET=$l_param/components/$(basename "$n_param") - if [ ! -d "$l_param/components" ]; then - echo "Folder components does not exist yet: mkdir -p "$l_param/components"" - mkdir -p "$l_param/components" +if [ -n "$n_param" ]; then + INSTALL_TARGET="" + if [ -z "$l_param" ]; then + # If local path for project is not supplied - use as INSTALL_TARGET Arduino libraries path + INSTALL_TARGET=$ARDUINO_LIBS_PATH/$(basename "$n_param") + else + INSTALL_TARGET=$l_param/components/$(basename "$n_param") + if [ ! -d "$l_param/components" ]; then + echo "Folder components does not exist yet: mkdir -p \"$l_param/components\"" + mkdir -p "$l_param/components" + fi fi - fi - # clone the new lib - echo "Cloning: git clone --recursive $n_param $INSTALL_TARGET" - git clone --recursive $n_param $INSTALL_TARGET - LIBRARY=$INSTALL_TARGET + # clone the new lib + echo "Cloning: git clone --recursive $n_param $INSTALL_TARGET" + git clone --recursive "$n_param" "$INSTALL_TARGET" + LIBRARY=$INSTALL_TARGET fi # Copy existing lib to local project -if [[ ! -z $e_param ]] && [[ ! -z $l_param ]]; then - if [ ! -d "$l_param/components" ]; then - echo "Folder components does not exist yet: mkdir -p "$l_param/components"" - mkdir -p "$l_param/components" - fi - echo "Copy from $e_param to $l_param" - echo "cp -r $e_param $l_param/components/$(basename "$e_param")" - cp -r $e_param $l_param/components/$(basename "$e_param") - LIBRARY=$l_param/components/$(basename "$e_param") +if [[ -n $e_param ]] && [[ -n $l_param ]]; then + if [ ! -d "$l_param/components" ]; then + echo "Folder components does not exist yet: mkdir -p \"$l_param/components\"" + mkdir -p "$l_param/components" + fi + echo "Copy from $e_param to $l_param" + echo "cp -r $e_param $l_param/components/\"$(basename "$e_param")\"" + cp -r "$e_param" "$l_param"/components/"$(basename "$e_param")" + LIBRARY=$l_param/components/"$(basename "$e_param")" fi if [ -z "$LIBRARY" ]; then - echo "ERROR: No library path" >&2 - exit 1 + echo "ERROR: No library path" >&2 + exit 1 fi # 1. get the source list: -FILES=$(find $LIBRARY -name '*.c' -o -name '*.cpp' | xargs -I{} basename {}) +FILES=$(find "$LIBRARY" -print0 -name '*.c' -o -name '*.cpp' | xargs -0 -I{} basename {}) # Fresh start -if [ -f $LIBRARY/CMakeLists.txt ]; then - rm $LIBRARY/CMakeLists.txt - touch $LIBRARY/CMakeLists.txt +if [ -f "$LIBRARY"/CMakeLists.txt ]; then + rm "$LIBRARY"/CMakeLists.txt + touch "$LIBRARY"/CMakeLists.txt fi # Generate CMakeLists.txt -echo "idf_component_register(SRCS $(echo $FILES | sed -e 's/ /" "/g' | sed -e 's/^/"/' -e 's/$/"/')" >> $LIBRARY/CMakeLists.txt -echo " INCLUDE_DIRS \".\"" >> $LIBRARY/CMakeLists.txt -echo " REQUIRES \"arduino-esp32\"" >> $LIBRARY/CMakeLists.txt -echo " )" >> $LIBRARY/CMakeLists.txt +{ + echo "idf_component_register(SRCS $(echo "$FILES" | sed -e 's/ /" "/g' | sed -e 's/^/"/' -e 's/$/"/')" + echo " INCLUDE_DIRS \".\"" + echo " REQUIRES \"arduino-esp32\"" + echo " )" +} >> "$LIBRARY"/CMakeLists.txt