From cfadf2073ae52ddaabd59ef807a82d1595be59db Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 29 Jul 2023 17:35:33 +0200 Subject: [PATCH 1/6] flash FS with detected flash size --- pio-tools/get_flash_size.py | 46 +++++++++++++++++++++++++++++++++ pio-tools/post_esp32.py | 51 ++++++++++++++++++++++++++++++++++--- platformio_tasmota32.ini | 1 + 3 files changed, 95 insertions(+), 3 deletions(-) create mode 100644 pio-tools/get_flash_size.py diff --git a/pio-tools/get_flash_size.py b/pio-tools/get_flash_size.py new file mode 100644 index 000000000000..04fcfdd5cff6 --- /dev/null +++ b/pio-tools/get_flash_size.py @@ -0,0 +1,46 @@ +from genericpath import exists +import os +from os.path import join +import subprocess +from SCons.Script import COMMAND_LINE_TARGETS + +Import("env") +env = DefaultEnvironment() +platform = env.PioPlatform() + +if "upload" in COMMAND_LINE_TARGETS: + github_actions = os.getenv('GITHUB_ACTIONS') + + def esp32_detect_flashsize(): + if github_actions: + return "4MB",False + esptoolpy = join(platform.get_package_dir("tool-esptoolpy") or "", "esptool.py") + esptoolpy_flags = ["flash_id"] + esptoolpy_cmd = [env["PYTHONEXE"], esptoolpy] + esptoolpy_flags + try: + output = subprocess.run(esptoolpy_cmd, capture_output=True).stdout.splitlines() + for l in output: + if l.decode().startswith("Detected flash size: "): + size = (l.decode().split(": ")[1]) + print("Did get flash size:",size) + return size, True + return "4MB",False + except subprocess.CalledProcessError as exc: + print("Did get chip info failed with " + str(exc)) + return "4MB",False + + size, overridden = esp32_detect_flashsize() + old_flash_size = env.BoardConfig().get("upload.flash_size") + old_maximum_size = env.BoardConfig().get("upload.maximum_size") + new_maximum_size = int(size.split("MB")[0]) * 0x100000 + + extra_flags = env.BoardConfig().get("build.extra_flags").split(" ") + new_flags = "" + for flag in extra_flags: + if flag.startswith("-DESP32"): + flag = f"-DESP32_{size}" + new_flags += flag + " " + env.BoardConfig().update("build.extra_flags",new_flags) + env.BoardConfig().update("upload.flash_size",size) + + env.BoardConfig().update("upload.maximum_size", new_maximum_size) diff --git a/pio-tools/post_esp32.py b/pio-tools/post_esp32.py index 9f8a2f637483..d458537ab10e 100644 --- a/pio-tools/post_esp32.py +++ b/pio-tools/post_esp32.py @@ -29,6 +29,7 @@ import requests import shutil import subprocess +import codecs sys.path.append(join(platform.get_package_dir("tool-esptoolpy"))) import esptool @@ -52,6 +53,39 @@ variants_dir = join(FRAMEWORK_DIR, "variants", "tasmota") +def patch_partitions_bin(size_string): + partition_bin_path = join(env.subst("$BUILD_DIR"),"partitions.bin") + with open(partition_bin_path, 'r+b') as file: + binary_data = file.read(0xb0) + import hashlib + bin_list = list(binary_data) + size = codecs.decode(size_string[2:], 'hex_codec') # 0xc50000 -> [c5,00,00] + bin_list[0x8a] = size[0] + bin_list[0x8b] = size[1] + result = hashlib.md5(bytes(bin_list[0:0xa0])) + partition_data = bytes(bin_list) + result.digest() + file.seek(0) + file.write(partition_data) + print("New partition hash:",result.digest().hex()) + +def esp32_detect_flashsize(): + if github_actions: + return "4MB",False + esptoolpy = join(platform.get_package_dir("tool-esptoolpy") or "", "esptool.py") + esptoolpy_flags = ["flash_id"] + esptoolpy_cmd = [env["PYTHONEXE"], esptoolpy] + esptoolpy_flags + try: + output = subprocess.run(esptoolpy_cmd, capture_output=True).stdout.splitlines() + for l in output: + if l.decode().startswith("Detected flash size: "): + size = (l.decode().split(": ")[1]) + print("Did get flash size:",size) + return size, True + return "4MB",False + except subprocess.CalledProcessError as exc: + print("Did get chip info failed with " + str(exc)) + return "4MB",False + def esp32_create_chip_string(chip): tasmota_platform = env.subst("$BUILD_DIR").split(os.path.sep)[-1] tasmota_platform = tasmota_platform.split('-')[0] @@ -120,6 +154,8 @@ def esp32_create_combined_bin(source, target, env): # factory_offset = -1 # error code value - currently unused app_offset = 0x10000 # default value for "old" scheme fs_offset = -1 # error code value + flash_size_from_esp, flash_size_was_overridden = esp32_detect_flashsize() + with open(env.BoardConfig().get("build.partitions")) as csv_file: print("Read partitions from ",env.BoardConfig().get("build.partitions")) csv_reader = csv.reader(csv_file, delimiter=',') @@ -136,7 +172,13 @@ def esp32_create_combined_bin(source, target, env): # elif(row[0] == 'factory'): # factory_offset = int(row[3],base=16) elif(row[0] == 'spiffs'): - if esp32_build_filesystem(row[4]): + partition_size = row[4] + if flash_size_was_overridden: + print(f"Will override fixed FS partition size from {env.BoardConfig().get('build.partitions')}: {partition_size} ...") + partition_size = hex(int(flash_size_from_esp.split("MB")[0]) * 0x100000 - int(row[3],base=16)) + print(f"... with computed maximum size from connected {env.get('BOARD_MCU')}: {partition_size}") + patch_partitions_bin(partition_size) + if esp32_build_filesystem(partition_size): fs_offset = int(row[3],base=16) @@ -147,16 +189,19 @@ def esp32_create_combined_bin(source, target, env): tasmota_platform = esp32_create_chip_string(chip) if "-DUSE_USB_CDC_CONSOLE" in env.BoardConfig().get("build.extra_flags") and "cdc" not in tasmota_platform: - tasmota_platform += "cdc" + tasmota_platform += "cdc" print("WARNING: board definition uses CDC configuration, but environment name does not -> changing tasmota safeboot binary to:", tasmota_platform + "-safeboot.bin") - + if not os.path.exists(variants_dir): os.makedirs(variants_dir) if("safeboot" in firmware_name): esp32_copy_new_safeboot_bin(tasmota_platform,firmware_name) else: esp32_fetch_safeboot_bin(tasmota_platform) + flash_size = env.BoardConfig().get("upload.flash_size", "4MB") + if flash_size_was_overridden: + flash_size = flash_size_from_esp flash_freq = env.BoardConfig().get("build.f_flash", "40000000L") flash_freq = str(flash_freq).replace("L", "") flash_freq = str(int(int(flash_freq) / 1000000)) + "m" diff --git a/platformio_tasmota32.ini b/platformio_tasmota32.ini index 814af20a0319..30f42a99a6ff 100644 --- a/platformio_tasmota32.ini +++ b/platformio_tasmota32.ini @@ -37,6 +37,7 @@ build_flags = ${esp_defaults.build_flags} -Wl,--wrap=_Z11analogWritehi ; `analogWrite(unsigned char, int)` use the Tasmota version of analogWrite for deeper integration and phase control -Wl,--wrap=ledcReadFreq ; `uint32_t ledcReadFreq(uint8_t chan)` extra_scripts = pre:pio-tools/add_c_flags.py + pre:pio-tools/get_flash_size.py pre:pio-tools/gen-berry-structures.py post:pio-tools/post_esp32.py ${esp_defaults.extra_scripts} From 20dcdeb7ff424b7373f5e3abd54a19db7de2e827 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 29 Jul 2023 17:47:50 +0200 Subject: [PATCH 2/6] do not query flash size when no `upload` --- pio-tools/post_esp32.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pio-tools/post_esp32.py b/pio-tools/post_esp32.py index d458537ab10e..8c5c3291a813 100644 --- a/pio-tools/post_esp32.py +++ b/pio-tools/post_esp32.py @@ -30,6 +30,7 @@ import shutil import subprocess import codecs +from SCons.Script import COMMAND_LINE_TARGETS sys.path.append(join(platform.get_package_dir("tool-esptoolpy"))) import esptool @@ -71,6 +72,8 @@ def patch_partitions_bin(size_string): def esp32_detect_flashsize(): if github_actions: return "4MB",False + if not "upload" in COMMAND_LINE_TARGETS: + return "4MB",False esptoolpy = join(platform.get_package_dir("tool-esptoolpy") or "", "esptool.py") esptoolpy_flags = ["flash_id"] esptoolpy_cmd = [env["PYTHONEXE"], esptoolpy] + esptoolpy_flags From 94c94d0144e31a6655e9f8520de62ddb37cfc58e Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 29 Jul 2023 17:57:19 +0200 Subject: [PATCH 3/6] check for gh not needed --- pio-tools/get_flash_size.py | 3 --- pio-tools/post_esp32.py | 2 -- 2 files changed, 5 deletions(-) diff --git a/pio-tools/get_flash_size.py b/pio-tools/get_flash_size.py index 04fcfdd5cff6..4fc725bf74bb 100644 --- a/pio-tools/get_flash_size.py +++ b/pio-tools/get_flash_size.py @@ -9,11 +9,8 @@ platform = env.PioPlatform() if "upload" in COMMAND_LINE_TARGETS: - github_actions = os.getenv('GITHUB_ACTIONS') def esp32_detect_flashsize(): - if github_actions: - return "4MB",False esptoolpy = join(platform.get_package_dir("tool-esptoolpy") or "", "esptool.py") esptoolpy_flags = ["flash_id"] esptoolpy_cmd = [env["PYTHONEXE"], esptoolpy] + esptoolpy_flags diff --git a/pio-tools/post_esp32.py b/pio-tools/post_esp32.py index 8c5c3291a813..de72fe376f22 100644 --- a/pio-tools/post_esp32.py +++ b/pio-tools/post_esp32.py @@ -70,8 +70,6 @@ def patch_partitions_bin(size_string): print("New partition hash:",result.digest().hex()) def esp32_detect_flashsize(): - if github_actions: - return "4MB",False if not "upload" in COMMAND_LINE_TARGETS: return "4MB",False esptoolpy = join(platform.get_package_dir("tool-esptoolpy") or "", "esptool.py") From a72cd8bcef8d9ae93dd218eaa3facf2f92b53b31 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 29 Jul 2023 18:36:24 +0200 Subject: [PATCH 4/6] Flash size is detected, no size pinning needed --- platformio_tasmota_cenv_sample.ini | 7 ------- 1 file changed, 7 deletions(-) diff --git a/platformio_tasmota_cenv_sample.ini b/platformio_tasmota_cenv_sample.ini index 2f987d809f6b..ed427a6edfb3 100644 --- a/platformio_tasmota_cenv_sample.ini +++ b/platformio_tasmota_cenv_sample.ini @@ -22,13 +22,6 @@ board_build.f_flash = 80000000L build_flags = ${env:tasmota32_base.build_flags} -DFIRMWARE_TASMOTA32 -DOTA_URL='"http://ota.tasmota.com/tasmota32/release/tasmota32s3.bin"' -; !!! Real flash size needed, avoid autoresize since it is formating FS !!! -board_upload.flash_size = 8MB -board_upload.maximum_size = 8388608 -; Without autoresize a partition scheme is needed which does fit to flash size -board_build.partitions = partitions/esp32_partition_app2944k_fs2M.csv -; Dont use safeboot, not used in this partition scheme -> an empty entry needed to overwrite the default setting -board_upload.arduino.flash_extra_images = ; Example for custom file upload in Tasmota Filesystem custom_files_upload = ${env:tasmota32_base.custom_files_upload} tasmota/berry/modules/Partition_Wizard.tapp From 0352092f1c4360a597f8202e40b318545e0dca2f Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 29 Jul 2023 18:50:25 +0200 Subject: [PATCH 5/6] add credits --- pio-tools/post_esp32.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pio-tools/post_esp32.py b/pio-tools/post_esp32.py index de72fe376f22..7b1e5874fe55 100644 --- a/pio-tools/post_esp32.py +++ b/pio-tools/post_esp32.py @@ -1,6 +1,8 @@ # From: https://github.com/letscontrolit/ESPEasy/blob/mega/tools/pio/post_esp32.py # Thanks TD-er :) +# Thanks @staars for safeboot and auto resizing LittleFS code and enhancements + # Combines separate bin files with their respective offsets into a single file # This single file must then be flashed to an ESP32 node with 0 offset. # From 7d9f9ef094777ae9022f7ef1c79509c2d665ba71 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 29 Jul 2023 19:01:40 +0200 Subject: [PATCH 6/6] rm duplicate code, using an env variable --- pio-tools/get_flash_size.py | 4 ++-- pio-tools/post_esp32.py | 16 +++------------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/pio-tools/get_flash_size.py b/pio-tools/get_flash_size.py index 4fc725bf74bb..7d1bf09f57f7 100644 --- a/pio-tools/get_flash_size.py +++ b/pio-tools/get_flash_size.py @@ -1,5 +1,3 @@ -from genericpath import exists -import os from os.path import join import subprocess from SCons.Script import COMMAND_LINE_TARGETS @@ -41,3 +39,5 @@ def esp32_detect_flashsize(): env.BoardConfig().update("upload.flash_size",size) env.BoardConfig().update("upload.maximum_size", new_maximum_size) + + env.Replace(TASMOTA_flash_size=size) \ No newline at end of file diff --git a/pio-tools/post_esp32.py b/pio-tools/post_esp32.py index 7b1e5874fe55..33e9ffa7e4d6 100644 --- a/pio-tools/post_esp32.py +++ b/pio-tools/post_esp32.py @@ -74,19 +74,9 @@ def patch_partitions_bin(size_string): def esp32_detect_flashsize(): if not "upload" in COMMAND_LINE_TARGETS: return "4MB",False - esptoolpy = join(platform.get_package_dir("tool-esptoolpy") or "", "esptool.py") - esptoolpy_flags = ["flash_id"] - esptoolpy_cmd = [env["PYTHONEXE"], esptoolpy] + esptoolpy_flags try: - output = subprocess.run(esptoolpy_cmd, capture_output=True).stdout.splitlines() - for l in output: - if l.decode().startswith("Detected flash size: "): - size = (l.decode().split(": ")[1]) - print("Did get flash size:",size) - return size, True - return "4MB",False - except subprocess.CalledProcessError as exc: - print("Did get chip info failed with " + str(exc)) + return env.get("TASMOTA_flash_size"),True + except: return "4MB",False def esp32_create_chip_string(chip): @@ -273,4 +263,4 @@ def esp32_create_combined_bin(source, target, env): esptool.main(cmd) -env.AddPostAction("$BUILD_DIR/${PROGNAME}.bin", esp32_create_combined_bin) +env.AddPostAction("$BUILD_DIR/${PROGNAME}.bin", esp32_create_combined_bin) \ No newline at end of file