v2.0.0 Add support for ESP32S2 and update ESP-IDF to 4.4 (#4996)

This is very much still work in progress and much more will change before the final 2.0.0

Some APIs have changed. New libraries have been added. LittleFS included.

Co-authored-by: Seon Rozenblum <seonr@3sprockets.com>
Co-authored-by: Me No Dev <me-no-dev@users.noreply.github.com>
Co-authored-by: geeksville <kevinh@geeksville.com>
Co-authored-by: Mike Dunston <m_dunston@comcast.net>
Co-authored-by: Unexpected Maker <seon@unexpectedmaker.com>
Co-authored-by: Seon Rozenblum <seonr@3sprockets.com>
Co-authored-by: microDev <70126934+microDev1@users.noreply.github.com>
Co-authored-by: tobozo <tobozo@users.noreply.github.com>
Co-authored-by: bobobo1618 <bobobo1618@users.noreply.github.com>
Co-authored-by: lorol <lorolouis@gmail.com>
Co-authored-by: geeksville <kevinh@geeksville.com>
Co-authored-by: Limor "Ladyada" Fried <limor@ladyada.net>
Co-authored-by: Sweety <switi.mhaiske@espressif.com>
Co-authored-by: Loick MAHIEUX <loick111@gmail.com>
Co-authored-by: Larry Bernstone <lbernstone@gmail.com>
Co-authored-by: Valerii Koval <valeros@users.noreply.github.com>
Co-authored-by: 快乐的我531 <2302004040@qq.com>
Co-authored-by: chegewara <imperiaonline4@gmail.com>
Co-authored-by: Clemens Kirchgatterer <clemens@1541.org>
Co-authored-by: Aron Rubin <aronrubin@gmail.com>
Co-authored-by: Pete Lewis <601236+lewispg228@users.noreply.github.com>

Author: 18 people

.github/scripts/check-cmakelists.sh

@@ -15,9 +15,9 @@ git submodule update --init --recursive
 REPO_SRCS=`find cores/esp32/ libraries/ -name 'examples' -prune -o -name '*.c' -print -o -name '*.cpp' -print | sort`
 
 # find all source files named in CMakeLists.txt COMPONENT_SRCS
-CMAKE_SRCS=`cmake --trace-expand -C CMakeLists.txt 2>&1 | grep COMPONENT_SRCS | sed 's/.\+COMPONENT_SRCS //' | sed 's/ )//' | tr ' ;' '\n' | sort`
+CMAKE_SRCS=`cmake --trace-expand -C CMakeLists.txt 2>&1 | grep set\(srcs | cut -d'(' -f3 | sed 's/ )//' | sed 's/srcs //' | tr ' ;' '\n' | sort`
 
-if ! diff -u0 --label "Repo Files" --label "COMPONENT_SRCS" <(echo "$REPO_SRCS") <(echo "$CMAKE_SRCS"); then
+if ! diff -u0 --label "Repo Files" --label "srcs" <(echo "$REPO_SRCS") <(echo "$CMAKE_SRCS"); then
     echo "Source files in repo (-) and source files in CMakeLists.txt (+) don't match"
     echo "Edit CMakeLists.txt as appropriate to add/remove source files from COMPONENT_SRCS"
     exit 1

.github/scripts/install-arduino-core-esp32.sh

@@ -23,9 +23,9 @@ if [ ! -d "$ARDUINO_ESP32_PATH" ]; then
 		git clone https://github.com/espressif/arduino-esp32.git esp32 > /dev/null 2>&1
 	fi
 
-	echo "Updating Submodules ..."
+	#echo "Updating Submodules ..."
 	cd esp32
-	git submodule update --init --recursive > /dev/null 2>&1
+	#git submodule update --init --recursive > /dev/null 2>&1
 
 	echo "Installing Platform Tools ..."
 	cd tools && python get.py

.github/scripts/install-arduino-ide.sh

@@ -116,9 +116,10 @@ function build_sketch(){ # build_sketch <fqbn> <path-to-ino> [extra-options]
 		$win_opts $xtra_opts "$sketch"
 }
 
-function count_sketches() # count_sketches <examples-path>
+function count_sketches() # count_sketches <examples-path> <target-mcu>
 {
 	local examples="$1"
+	local target="$2"
     rm -rf sketches.txt
 	if [ ! -d "$examples" ]; then
 		touch sketches.txt
@@ -133,7 +134,7 @@ function count_sketches() # count_sketches <examples-path>
         if [[ "${sketchdirname}.ino" != "$sketchname" ]]; then
             continue
         fi;
-        if [[ -f "$sketchdir/.test.skip" ]]; then
+        if [[ -f "$sketchdir/.skip.$target" ]]; then
             continue
         fi
         echo $sketch >> sketches.txt
@@ -142,24 +143,25 @@ function count_sketches() # count_sketches <examples-path>
     return $sketchnum
 }
 
-function build_sketches() # build_sketches <fqbn> <examples-path> <chunk> <total-chunks> [extra-options]
+function build_sketches() # build_sketches <fqbn> <target-mcu> <examples-path> <chunk> <total-chunks> [extra-options]
 {
     local fqbn=$1
-    local examples=$2
-    local chunk_idex=$3
-    local chunks_num=$4
-    local xtra_opts=$5
+	local target="$2"
+    local examples=$3
+    local chunk_idex=$4
+    local chunks_num=$5
+    local xtra_opts=$6
 
-    if [ "$#" -lt 2 ]; then
+    if [ "$#" -lt 3 ]; then
 		echo "ERROR: Illegal number of parameters"
-		echo "USAGE: build_sketches <fqbn> <examples-path> [<chunk> <total-chunks>] [extra-options]"
+		echo "USAGE: build_sketches <fqbn> <target-mcu <examples-path> [<chunk> <total-chunks>] [extra-options]"
 		return 1
 	fi
 
-    if [ "$#" -lt 4 ]; then
+    if [ "$#" -lt 5 ]; then
 		chunk_idex="0"
 		chunks_num="1"
-		xtra_opts=$3
+		xtra_opts=$4
 	fi
 
 	if [ "$chunks_num" -le 0 ]; then
@@ -208,7 +210,7 @@ function build_sketches() # build_sketches <fqbn> <examples-path> <chunk> <total
         local sketchdirname=$(basename $sketchdir)
         local sketchname=$(basename $sketch)
         if [ "${sketchdirname}.ino" != "$sketchname" ] \
-        || [ -f "$sketchdir/.test.skip" ]; then
+        || [ -f "$sketchdir/.skip.$target" ]; then
             continue
         fi
         sketchnum=$(($sketchnum + 1))

.github/scripts/install-platformio-esp32.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 export PLATFORMIO_ESP32_PATH="$HOME/.platformio/packages/framework-arduinoespressif32"
+PLATFORMIO_ESP32_URL="https://github.com/platformio/platform-espressif32.git#feature/idf-v4.0"
 
 echo "Installing Python Wheel ..."
 pip install wheel > /dev/null 2>&1
@@ -9,10 +10,10 @@ echo "Installing PlatformIO ..."
 pip install -U https://github.com/platformio/platformio/archive/develop.zip > /dev/null 2>&1
 
 echo "Installing Platform ESP32 ..."
-python -m platformio platform install https://github.com/platformio/platform-espressif32.git > /dev/null 2>&1
+python -m platformio platform install $PLATFORMIO_ESP32_URL > /dev/null 2>&1
 
 echo "Replacing the framework version ..."
-python -c "import json; import os; fp=open(os.path.expanduser('~/.platformio/platforms/espressif32/platform.json'), 'r+'); data=json.load(fp); data['packages']['framework-arduinoespressif32']['version'] = '*'; del data['packages']['framework-arduinoespressif32']['owner']; fp.seek(0); fp.truncate(); json.dump(data, fp); fp.close()"
+python -c "import json; import os; fp=open(os.path.expanduser('~/.platformio/platforms/espressif32/platform.json'), 'r+'); data=json.load(fp); data['packages']['framework-arduinoespressif32']['version'] = '*'; fp.seek(0); fp.truncate(); json.dump(data, fp); fp.close()"
 
 if [ "$GITHUB_REPOSITORY" == "espressif/arduino-esp32" ];  then
 	echo "Linking Core..."
@@ -25,19 +26,20 @@ fi
 echo "PlatformIO for ESP32 has been installed"
 echo ""
 
-function build_pio_sketch(){ # build_pio_sketch <board> <path-to-ino>
-    if [ "$#" -lt 2 ]; then
+function build_pio_sketch(){ # build_pio_sketch <board> <options> <path-to-ino>
+    if [ "$#" -lt 3 ]; then
         echo "ERROR: Illegal number of parameters"
-        echo "USAGE: build_pio_sketch <board> <path-to-ino>"
+        echo "USAGE: build_pio_sketch <board> <options> <path-to-ino>"
         return 1
     fi
 
     local board="$1"
-    local sketch="$2"
+    local options="$2"
+    local sketch="$3"
     local sketch_dir=$(dirname "$sketch")
     echo ""
     echo "Compiling '"$(basename "$sketch")"' ..."
-    python -m platformio ci --board "$board" "$sketch_dir" --project-option="board_build.partitions = huge_app.csv"
+    python -m platformio ci --board "$board" "$sketch_dir" --project-option="$options"
 }
 
 function count_sketches() # count_sketches <examples-path>
@@ -66,20 +68,21 @@ function count_sketches() # count_sketches <examples-path>
     return $sketchnum
 }
 
-function build_pio_sketches() # build_pio_sketches <board> <examples-path> <chunk> <total-chunks>
+function build_pio_sketches() # build_pio_sketches <board> <options> <examples-path> <chunk> <total-chunks>
 {
-    if [ "$#" -lt 2 ]; then
+    if [ "$#" -lt 3 ]; then
         echo "ERROR: Illegal number of parameters"
-        echo "USAGE: build_pio_sketches <board> <examples-path> [<chunk> <total-chunks>]"
+        echo "USAGE: build_pio_sketches <board> <options> <examples-path> [<chunk> <total-chunks>]"
         return 1
     fi
 
     local board=$1
-    local examples=$2
-    local chunk_idex=$3
-    local chunks_num=$4
+    local options="$2"
+    local examples=$3
+    local chunk_idex=$4
+    local chunks_num=$5
 
-    if [ "$#" -lt 4 ]; then
+    if [ "$#" -lt 5 ]; then
         chunk_idex="0"
         chunks_num="1"
     fi
@@ -138,7 +141,7 @@ function build_pio_sketches() # build_pio_sketches <board> <examples-path> <chun
         || [ "$sketchnum" -gt "$end_index" ]; then
             continue
         fi
-        build_pio_sketch "$board" "$sketch"
+        build_pio_sketch "$board" "$options" "$sketch"
         local result=$?
         if [ $result -ne 0 ]; then
             return $result

.github/scripts/on-push.sh

@@ -30,42 +30,62 @@ elif [ "$CHUNK_INDEX" -eq "$CHUNKS_CNT" ]; then
 	BUILD_PIO=1
 fi
 
-echo "Updating submodules ..."
-git -C "$GITHUB_WORKSPACE" submodule update --init --recursive > /dev/null 2>&1
+#echo "Updating submodules ..."
+#git -C "$GITHUB_WORKSPACE" submodule update --init --recursive > /dev/null 2>&1
 
 if [ "$BUILD_PIO" -eq 0 ]; then
-	# ArduinoIDE Test
+	# ArduinoIDE ESP32 Test
+    TARGET="esp32"
 	FQBN="espressif:esp32:esp32:PSRAM=enabled,PartitionScheme=huge_app"
 	source ./.github/scripts/install-arduino-ide.sh
 	source ./.github/scripts/install-arduino-core-esp32.sh
 	if [ "$OS_IS_WINDOWS" == "1" ]; then
 		build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFiClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino" && \
 		build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/BLE/examples/BLE_server/BLE_server.ino" && \
-		build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/AzureIoT/examples/GetStarted/GetStarted.ino" && \
 		build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/ESP32/examples/Camera/CameraWebServer/CameraWebServer.ino"
 	elif [ "$OS_IS_MACOS" == "1" ]; then
 		build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFi/examples/WiFiClient/WiFiClient.ino" && \
 		build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFiClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino" && \
 		build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/BluetoothSerial/examples/SerialToSerialBT/SerialToSerialBT.ino" && \
 		build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/BLE/examples/BLE_server/BLE_server.ino" && \
-		build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/AzureIoT/examples/GetStarted/GetStarted.ino" && \
 		build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/ESP32/examples/Camera/CameraWebServer/CameraWebServer.ino"
 	else
 		# CMake Test
 		if [ "$CHUNK_INDEX" -eq 0 ]; then
 			bash "$ARDUINO_ESP32_PATH/.github/scripts/check-cmakelists.sh"
 		fi
-		build_sketches "$FQBN" "$ARDUINO_ESP32_PATH/libraries" "$CHUNK_INDEX" "$CHUNKS_CNT"
+		build_sketches "$FQBN" "$TARGET" "$ARDUINO_ESP32_PATH/libraries" "$CHUNK_INDEX" "$CHUNKS_CNT"
 	fi
+
+	# ArduinoIDE ESP32S2 Test
+    TARGET="esp32s2"
+	FQBN="espressif:esp32:esp32s2:PSRAM=enabled,PartitionScheme=huge_app"
+	if [ "$OS_IS_WINDOWS" == "1" ]; then
+		build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFi/examples/WiFiClient/WiFiClient.ino" && \
+		build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFiClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino"
+	elif [ "$OS_IS_MACOS" == "1" ]; then
+		build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFi/examples/WiFiClient/WiFiClient.ino" && \
+		build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFiClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino"
+	else
+		build_sketches "$FQBN" "$TARGET" "$ARDUINO_ESP32_PATH/libraries" "$CHUNK_INDEX" "$CHUNKS_CNT"
+    fi
 else
-	# PlatformIO Test
 	source ./.github/scripts/install-platformio-esp32.sh
+	# PlatformIO ESP32 Test
 	BOARD="esp32dev"
-	build_pio_sketch "$BOARD" "$PLATFORMIO_ESP32_PATH/libraries/WiFi/examples/WiFiClient/WiFiClient.ino" && \
-	build_pio_sketch "$BOARD" "$PLATFORMIO_ESP32_PATH/libraries/WiFiClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino" && \
-	build_pio_sketch "$BOARD" "$PLATFORMIO_ESP32_PATH/libraries/BluetoothSerial/examples/SerialToSerialBT/SerialToSerialBT.ino" && \
-	build_pio_sketch "$BOARD" "$PLATFORMIO_ESP32_PATH/libraries/BLE/examples/BLE_server/BLE_server.ino" && \
-	build_pio_sketch "$BOARD" "$PLATFORMIO_ESP32_PATH/libraries/AzureIoT/examples/GetStarted/GetStarted.ino" && \
-	build_pio_sketch "$BOARD" "$PLATFORMIO_ESP32_PATH/libraries/ESP32/examples/Camera/CameraWebServer/CameraWebServer.ino"
-	#build_pio_sketches esp32dev "$PLATFORMIO_ESP32_PATH/libraries"
+    OPTIONS="board_build.partitions = huge_app.csv"
+	build_pio_sketch "$BOARD" "$OPTIONS" "$PLATFORMIO_ESP32_PATH/libraries/WiFi/examples/WiFiClient/WiFiClient.ino" && \
+	build_pio_sketch "$BOARD" "$OPTIONS" "$PLATFORMIO_ESP32_PATH/libraries/WiFiClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino" && \
+	build_pio_sketch "$BOARD" "$OPTIONS" "$PLATFORMIO_ESP32_PATH/libraries/BluetoothSerial/examples/SerialToSerialBT/SerialToSerialBT.ino" && \
+	build_pio_sketch "$BOARD" "$OPTIONS" "$PLATFORMIO_ESP32_PATH/libraries/BLE/examples/BLE_server/BLE_server.ino" && \
+	build_pio_sketch "$BOARD" "$OPTIONS" "$PLATFORMIO_ESP32_PATH/libraries/ESP32/examples/Camera/CameraWebServer/CameraWebServer.ino"
+	
+	# PlatformIO ESP32 Test
+    # OPTIONS="board_build.mcu = esp32s2"
+    # build_pio_sketch "$BOARD" "$OPTIONS" "$PLATFORMIO_ESP32_PATH/libraries/WiFi/examples/WiFiClient/WiFiClient.ino" && \
+    # build_pio_sketch "$BOARD" "$OPTIONS" "$PLATFORMIO_ESP32_PATH/libraries/WiFiClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino"
+
+    python -m platformio ci --board "$BOARD" "$PLATFORMIO_ESP32_PATH/libraries/WiFi/examples/WiFiClient" --project-option="board_build.mcu = esp32s2" --project-option="board_build.partitions = huge_app.csv"
+    
+	#build_pio_sketches "$BOARD" "$OPTIONS" "$PLATFORMIO_ESP32_PATH/libraries"
 fi

.github/scripts/on-release.sh

@@ -171,19 +171,19 @@ mkdir -p "$PKG_DIR/tools"
 
 # Copy all core files to the package folder
 echo "Copying files for packaging ..."
-cp -f  "$GITHUB_WORKSPACE/boards.txt"                "$PKG_DIR/"
-cp -f  "$GITHUB_WORKSPACE/programmers.txt"           "$PKG_DIR/"
-cp -Rf "$GITHUB_WORKSPACE/cores"                     "$PKG_DIR/"
-cp -Rf "$GITHUB_WORKSPACE/libraries"                 "$PKG_DIR/"
-cp -Rf "$GITHUB_WORKSPACE/variants"                  "$PKG_DIR/"
-cp -f  "$GITHUB_WORKSPACE/tools/espota.exe"          "$PKG_DIR/tools/"
-cp -f  "$GITHUB_WORKSPACE/tools/espota.py"           "$PKG_DIR/tools/"
-cp -f  "$GITHUB_WORKSPACE/tools/esptool.py"          "$PKG_DIR/tools/"
-cp -f  "$GITHUB_WORKSPACE/tools/gen_esp32part.py"    "$PKG_DIR/tools/"
-cp -f  "$GITHUB_WORKSPACE/tools/gen_esp32part.exe"   "$PKG_DIR/tools/"
-cp -Rf "$GITHUB_WORKSPACE/tools/partitions"          "$PKG_DIR/tools/"
-cp -Rf "$GITHUB_WORKSPACE/tools/sdk"                 "$PKG_DIR/tools/"
-cp -f  "$GITHUB_WORKSPACE/tools/platformio-build.py" "$PKG_DIR/tools/"
+cp -f  "$GITHUB_WORKSPACE/boards.txt"              "$PKG_DIR/"
+cp -f  "$GITHUB_WORKSPACE/programmers.txt"         "$PKG_DIR/"
+cp -Rf "$GITHUB_WORKSPACE/cores"                   "$PKG_DIR/"
+cp -Rf "$GITHUB_WORKSPACE/libraries"               "$PKG_DIR/"
+cp -Rf "$GITHUB_WORKSPACE/variants"                "$PKG_DIR/"
+cp -f  "$GITHUB_WORKSPACE/tools/espota.exe"        "$PKG_DIR/tools/"
+cp -f  "$GITHUB_WORKSPACE/tools/espota.py"         "$PKG_DIR/tools/"
+cp -f  "$GITHUB_WORKSPACE/tools/esptool.py"        "$PKG_DIR/tools/"
+cp -f  "$GITHUB_WORKSPACE/tools/gen_esp32part.py"  "$PKG_DIR/tools/"
+cp -f  "$GITHUB_WORKSPACE/tools/gen_esp32part.exe" "$PKG_DIR/tools/"
+cp -Rf "$GITHUB_WORKSPACE/tools/partitions"        "$PKG_DIR/tools/"
+cp -Rf "$GITHUB_WORKSPACE/tools/sdk"               "$PKG_DIR/tools/"
+cp -f  $GITHUB_WORKSPACE/tools/platformio-build*.py "$PKG_DIR/tools/"
 
 # Remove unnecessary files in the package folder
 echo "Cleaning up folders ..."
@@ -195,6 +195,7 @@ echo "Generating platform.txt..."
 cat "$GITHUB_WORKSPACE/platform.txt" | \
 sed "s/version=.*/version=$ver$extent/g" | \
 sed 's/runtime.tools.xtensa-esp32-elf-gcc.path={runtime.platform.path}\/tools\/xtensa-esp32-elf//g' | \
+sed 's/runtime.tools.xtensa-esp32s2-elf-gcc.path={runtime.platform.path}\/tools\/xtensa-esp32s2-elf//g' | \
 sed 's/tools.esptool_py.path={runtime.platform.path}\/tools\/esptool/tools.esptool_py.path=\{runtime.tools.esptool_py.path\}/g' \
  > "$PKG_DIR/platform.txt"
 

.gitignore

@@ -14,3 +14,6 @@ tools/mklittlefs
 __vm/
 *.vcxproj*
 .vscode/
+platform.sloeber.txt
+boards.sloeber.txt
+tools/mklittlefs

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "libraries/AzureIoT"]
-	path = libraries/AzureIoT
-	url = https://github.com/VSChina/ESP32_AzureIoT_Arduino