S3 16MB fix

.github/workflows/test_esptool.yml

@@ -48,7 +48,7 @@ jobs:
 
         export PATH=$PATH:$ESP8266_BINDIR:$ESP32_BINDIR:$ESP32S2_BINDIR:$ESP32S3_BINDIR:$ESP32C3_BINDIR
 
-        ./test/ci/setup_ci_build_env.sh
+        ./ci/setup_ci_build_env.sh
         make -C flasher_stub V=1
         cd flasher_stub && python ./compare_stubs.py
 

.gitlab-ci.yml

@@ -6,6 +6,7 @@ stages:
   - report
   - build_docs
   - deploy_docs
+  - deploy_development_package
 
 # cache the pip download directory in all jobs
 variables:
@@ -175,7 +176,7 @@ check_stub_build:
 
     TOOLCHAIN_PATHS: "${ESP8266_BINDIR}:${ESP32_BINDIR}:${ESP32S2_BINDIR}:${ESP32S3_BINDIR}:${ESP32C3_BINDIR}"
   script:
-    - ./test/ci/setup_ci_build_env.sh
+    - ./ci/setup_ci_build_env.sh
     - make -C flasher_stub V=1 PATH="${TOOLCHAIN_PATHS}:${PATH}"
     - cd flasher_stub && coverage run --parallel-mode ./compare_stubs.py
 
@@ -366,3 +367,25 @@ deploy_docs_production:
     DOCS_DEPLOY_SERVER_USER: "$DOCS_PROD_SERVER_USER"
     DOCS_DEPLOY_PATH: "$DOCS_PROD_PATH"
     DOCS_DEPLOY_URL_BASE: "https://docs.espressif.com/projects/esptool"
+
+deploy_dev_package:
+  <<: *test_template
+  rules:
+    - if: $CI_DEV_PUBLISH != null && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+  variables:
+    TWINE_NON_INTERACTIVE: "true"
+    TWINE_USERNAME: __token__
+    TWINE_PASSWORD: ${PYPI_ESPTOOL_TOKEN}
+  stage: deploy_development_package
+  dependencies: []
+  script:
+    - python -m pip install --upgrade pip
+    - python -m pip install twine setuptools
+    - python ci/patch_dev_release.py --dev-no ${CI_DEV_PUBLISH} esptool/__init__.py
+    # check what has been changed with git
+    - git diff
+    - python setup.py sdist
+    # skip release if it has already been released (with failure)
+    - python -m pip download esptool==$(python setup.py -V) && exit 1
+    - tar -ztvf dist/*
+    - python -m twine upload dist/*

ci/patch_dev_release.py

@@ -0,0 +1,43 @@
+# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+#
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+import argparse
+import re
+
+LINE_RE = re.compile(r"^__version__ = ['\"]([^'\"]*)['\"]")
+NEW_LINE = '__version__ = "{}"'
+
+
+def get_new_version(old_version, dev_number):
+    assert old_version.endswith("-dev")
+    return old_version.replace("-dev", ".dev{}".format(dev_number), 1)
+
+
+def patch_file(path, dev_number):
+    with open(path, "r") as fin:
+        lines = fin.readlines()
+
+    for i, line in enumerate(lines, start=0):
+        m = LINE_RE.search(line)
+        if m:
+            old_version = m.group(1)
+            lines[i] = NEW_LINE.format(get_new_version(old_version, dev_number))
+            break
+
+    with open(path, "w") as fout:
+        fout.writelines(lines)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("file", help="Path to script with __version__")
+    parser.add_argument(
+        "--dev-no", type=int, help="Number N to patch the version to '.devN'"
+    )
+    args = parser.parse_args()
+    patch_file(args.file, args.dev_no)
+
+
+if __name__ == "__main__":
+    main()

esptool/cmds.py

@@ -104,13 +104,13 @@ def detect_chip(
             # cmd not supported on ESP8266 and ESP32 + ESP32-S2 doesn't return chip_id
             if chip_id == cls.IMAGE_CHIP_ID:
                 inst = cls(detect_port._port, baud, trace_enabled=trace_enabled)
-                inst._post_connect()
                 try:
                     inst.read_reg(
                         ESPLoader.CHIP_DETECT_MAGIC_REG_ADDR
                     )  # Dummy read to check Secure Download mode
                 except UnsupportedCommandError:
                     inst.secure_download_mode = True
+                inst._post_connect()
     except (UnsupportedCommandError, struct.error, FatalError) as e:
         # UnsupportedCommmanddError: ESP8266/ESP32 ROM
         # struct.error: ESP32-S2

esptool/loader.py

@@ -672,8 +672,17 @@ def connect(
                         )
             except UnsupportedCommandError:
                 self.secure_download_mode = True
+
+            try:
+                self.check_chip_id()
+            except UnsupportedCommandError:
+                # Fix for ROM not responding in SDM, reconnect and try again
+                if self.secure_download_mode:
+                    self._connect_attempt(mode, usb_jtag_serial, extra_delay)
+                    self.check_chip_id()
+                else:
+                    raise
             self._post_connect()
-            self.check_chip_id()
 
     def _post_connect(self):
         """

esptool/targets/stub_flasher/stub_flasher_32s3beta2.json

@@ -1,4 +1,5 @@
 {
+<<<KP7qNN0CkjjdAcpM3QC+PN0DGjjdAL483QI6PN0BakDdAzZA3QHWQN0BFjTdA8I83QEyQN0CxjzdA8JA3QNqPN0DwkDdAko43QPGON0AvjzdAjo83QKqON0CGjTdAsJE3QCuTN0BijDdAUZM3QGKMN0BijDdAYow3QGKMN0BijDdAYow3QGKMN0BijDdASpE3QGKMN0BFkjdAK5M3QA==",
     "entry": 1077381684,
     "text": "FIADYACAA2BIAMo/BIADYDZBAIH7/wxJwCAAmQjGBAAAgfj/wCAAqAiB9/+goHSICOAIACH2/8AgAIgCJ+jhHfAAAAAIAABgHAAAYBAAAGA2QQAh/P/AIAA4AkH7/8AgACgEICCUnOJB6P9GBAAMODCIAcAgAKgIiASgoHTgCAALImYC6Ib0/yHx/8AgADkCHfAAAOwryz9kq8o/hIAAAEBAAACk68o/8CvLPzZBALH5/yCgdBARIOUqAZYaBoH2/5KhAZCZEZqYwCAAuAmR8/+goHSaiMAgAJIYAJCQ9BvJwMD0wCAAwlgAmpvAIACiSQDAIACSGACB6v+QkPSAgPSHmUeB5f+SoQGQmRGamMAgAMgJoeX/seP/h5wXxgEAfOiHGt7GCADAIACJCsAgALkJRgIAwCAAuQrAIACJCZHX/5qIDAnAIACSWAAd8AAAVCAAYFQwAGA2QQCR/f/AIACICYCAJFZI/5H6/8AgAIgJgIAkVkj/HfAAAAAsIABgACAAYAAAAAg2QQAQESCl/P8h+v8MCMAgAIJiAJH6/4H4/8AgAJJoAMAgAJgIVnn/wCAAiAJ88oAiMCAgBB3wAAAAAEA2QQAQESDl+/8Wav+B7P+R+//AIACSaADAIACYCFZ5/x3wAABYAMo/////AAQgAGCcGgVAaBoFQDZBACH6/0hCFoQJEBEg5ff/FvoIDPgMA0eoDJgigJkQDBiQOIMwMHQQESDl+f8QESCl8v+4IkHu/0CLEYe0LpyT+zsMTLA7swAMQDCwsQwager/4AgAHAjGDgAADEwMGoHn/+AIAAwYxgoAAAAADBqQmgHME4CaAUHf/8AgAIkEgcH/wCAAmQjAIABICFZ0/xwIDBQwhIM4QoAzwDlCOCKKMzkiHfAAAIypBEA2QQAgoiCB/f/gCAAd8AAAcOL6PwggAGBY0gRAeNIEQDZhABARIKXo/zH5/70BrQOB+v/gCABNCgwS7OqIAZKiAJCIEIkBEBEgJe3/kfL/oKIBwCAAiAmgiCDAIACJCbgBrQOB7v/gCACgJIMd8AAA/w8AADZBAIGy/wwZkkgAMJxBmSiR+/85GCk4MDC0miIqMzA8QQwCKVg5SBARICX4/y0KjBoioMUd8AAAtJwEQDZBAIH+/+AIAIIKGAwJgsj8DBKAKZMd8DZBAIH4/+AIAIIKGAwJgsj9DBKAKZMd8DZBABARICX+/yKgAcxqEBEg5fv/LQogIAQd8AD4K8s/9CvLPxAADGBgAAxg//P//zZBABARIKX8/xZaBKH4/4gKvNiB9/+YCLxpsfb/fMzAIACIC5CQFMCIEJCIIMAgAIkLiAqx8f8MOmCqEcAgAJgLoIgQoe7/oJkQkIggwCAAiQsd8FDzzj9EAMo/TADKP1SfBEBAnwRAhKAEQDZhAHzIrQKHky0x9//GBQAAqAMMHL0Bgff/4AgAgfH+ogEAiAjgCACoA4Hz/+AIAOYa3cYKAAAAZgMmDAPNAQwrMmEAge7/4AgAmAGB6P83mQ2oCGYaCDHm/8AgAKJDAJkIHfBAAMo/AADKP+CeBEA2QQAh/P+B3P/IAqgIsfr/gfv/4AgADAiJAh3wUJgEQDZBABARIGXr/4y6gfL/iAiMSBARIKX8/xARIKXo/xYqAKKgBIH2/+AIAB3wIJgEQDZBABARIGXo/7wakeb/iAkbqKkJkeX/DAqKmSJJAILIwQwZgKmDoIB0zIqir0CqIiCYk5zZEBEgZff/RgUArQKB7//gCAAQESCl4v+MShARIOX3/x3wAAA2QQCioMAQESDl+f8d8AAANkEAgqDArQKHkhGioNsQESBl+P+ioNxGBAAAAACCoNuHkggQESAl9/+ioN0QESCl9v8d8DZBADoyxgIAAKICABsiEBEgpfv/N5LxHfAAAADYnwRASEgEQOSfBEBUSARANkEAEBEg5dv/rIoME0GU//AzAYyyqASB9v/gCACtA8YJAK0DgfT/4AgAqASB8//gCAAGCQAQESAl1/8MGPCIASwDoIODrQgWkgCB7P/gCACGAQAAgej/4AgAHfAsGAVA4BUFQDaBAAxLDBqB/P/gCAAcBgYMAAAAYFRDDAgMGtCVEQyNOTHtAolhqVGZQYkhiRHZASwPDMwMS4Hx/+AIAFBEwFozWiLmFM0MAh3wAACgdgNAzOMEQMB2A0AYmQRAQHcDQDZBIWKkHeBmERpmWQYMF1KgAGLREFClIEB3EVJmGhARIKXx/0e3AsZIAK0GgfD/4AgAgqIP0IgRC5MaiJkIxjIAkqIP0JkRGpmYCVBzwCqJkcv+QHdjzQe9Aa0Ch7kJEBEg5fP/hgEAAIHh/+AIAKCgdIyqDAiCZhZ9CEYTAAAAEBEgpeT/vQetARARICXo/xARIKXj/80HELEgYKYggdX/4AgAeiJ6VTe1yIKhB8CIEZKkHRqI4JkRiAgamZgJgHXAl7cCBt7/Bur/AAwIgkZsoqQbEKqggcj/4AgAVgr/sqILogZsELuwEBEgpcAA9+oS9kcPkqINEJmwepmiSQAbd4bx/3zpl5rBZkcSkqEHgiYawJkRGpmJCTe4Aoe1hyKiCxAisL0GrQKBtP/gCAAQESCl2f+tAhwLEBEgJd3/EBEgpdj/DBoQESAl3/8d8DYhIaLREIGl/+AIAIYPAAAMFEBEEVGF/kBDYzqCzQS9Aa0Ch7UKEBEgZeL/xgEAAACBm//gCACgoHT8Os0EvQGi0RCBmP/gCABKIkAzwFbj+yKiCxAisK0CstEQgZT/4AgArQIcCxARIGXV/y0DhgAAIqBjHfAAAHQYBUA2gQAMSwwagXH/4AgALAcGEQAMSwwagfn/4AgAcFRDDAgMFtCVEe0CiUGJMZkhORGJASwPDI0cLAxLrQZpYWlRgWT/4AgADEutBoFg/+AIAFozWiJQRMDmFLcd8OziBEA2YQBBUv5YNFAzYxbTC1gUWlNQXEFGAQAQESBllP9oRKYWBGgkZ6XvEBEgJYz/Fmr/aBRxR/46Vle3bAxLDBqBS//gCABgUHSCoQBQeMB3swjNA70CrQZGDgDNB70CrQZS1f8QESCl8/86VVBYQQwJhgUAAADCoQCZARARICXy/5gBctcBG5mQkHR6pnqyVznjcMPAEBEgpfD/DEsMGoEz/+AIAIYFAADNA70CrQaB0//gCACgoHSMOiKgxClUKBQ6IikUKDQwIsAiZAMd8AAAAADKP09IQUn8K8s/rIA3QJggDGCogjdAJIQ3QAgACGCAIQxgEIA3QBCAA2BQgDdADAAAYDhAAGD//wAALIEAYIyAAAAQQAAAACzLPxAsyz9QAMo/VADKP1wsyz8UAABg8P//APwryz9kK8o/cADKP/hNBEA4SARAKKAEQKyfBEBsOgRAAOEEQHDmBEBIMQRA0LYEQCyjBEAsqQRABFwEQPSLBEDk4QRAeOIEQATiBEBolQRAtPgEQFz6BEDQ+ARALFQDQOxbBEA24QAhzv8MCinBgef/4AgAEBEgJZT/FmoEUVf+MVb+wCAAKAVBU/4pAzFU/sAgAGgDaQR8xmAiEAwmYCIgwCAAKQUoBEFP/kAiEEKkAEAiIMAgACkDxgEASQJLIgYCAAAhtv8xt/8MBDcy7BARIKWp/wxLosEwEBEgJa3/IqEBEBEgZaj/QXf9kCIRKiTAIABJAjGs/yFP/TkCEBEg5Yn/LQoW+gUhTP7BTf6oAgwrgU/+4AgAMaT/saX/HBoMDMAgAKkDgbr/4AgADBrwqgGBqf7gCACxnv+oAgwVgbX/4AgAqAKBof7gCACoAoGy/+AIADGY/8AgACgDUCIgwCAAKQOGGAAQESClgf+8GjGS/xwasZL/wCAAomMAIMIggaP/4AgAMY//DEXAIAAoAwwaUCIgwCAAKQPwqgHGCAAAALGJ/80KDFqBmf/gCAAxhv9SoQHAIAAoAywKUCIgwCAAKQOBhP7gCACBlP/gCAAhf//AIAAoAsy6HMMwIhAiwvgMEyCjgwwLgY3/4AgA8Xj/DB0MHLKgAeKhAEDdEQDMEWC7AaKgAIGG/+AIACFz/ypEIYD9YtIrxhcAAAAAUQj+wCAAMgUAMDB0FtMEDBrwqgHAIAAiRQCBZf7gCACionHAqhGBeP/gCACBd//gCABxYf986MAgADgHfPqAMxAQqgHAIAA5B4Fx/+AIAIFx/+AIAK0CgXD/4AgAwCAAKAQWovkMB8AgADgEDBLAIAB5BCJBJCIDAQwoeaEiQSWCURMcN3cSJBxHdxIhZpIhIgMDcgMCgCIRcCIgZkISKCPAIAAoAimhhgEAAAAcIiJRExARIGWJ/7KgCKLBJBARIOWM/7IDAyIDAoC7ESBbICE7/yAg9FeyGqKgwBARIKWH/6Kg7hARICWH/xARIOWF/wba/yIDARxHJzc39iIbBvoAACLCLyAgdLZCAgYlAHEt/3AioCgCoAIAACLC/iAgdBwnJ7cCRvAAcSf/cCKgKAKgAgBywjBwcHS2V8WG6gAsSQwHIqDAlxUChugAeaEMcq0HEBEgZYD/rQcQESDlf/8QESBlfv8QESAlfv8Mi6LBJCLC/xARIGWB/1Yi/YZFAAwSVuU1wsEQvQWtBYEj/+AIAFbqNBxLosEQEBEgJX//hrEADBJWtTOBHf/gCACgJYPGywAmhQQMEsbJAFgjKDMgdSBwcLRW1/4QESClUP9xCv2s2sb3/1CsQVe3Gb0KDEyioAGBCP3gCADGAwAAItLwUtUQxgIAgQr/4AgAFtr+xuz/zDJGmAAAAFCQ9FZp/HH6/AYNAFCg9Ve3Ir0KwqAEoqABgff84AgAxgUAAHz5AJkRmiIMGQCZEZpVxgIAgfn+4AgAFlr+xtr/DBkAmREnOcJ9BSolxgoAgej8cKxBd7gWvQoMTAwagef84AgARgMActcQRgMAAACB6v7gCAAW6v4GzP8nl9BGdwAMByKgwCaFAsaVAAwHLQcGlAAmtfXGaQAMEia1AsaNALgzoiMCcqAAEBEgZZH/oCeDxogADBlmtV6IQyCpEQwHIqDCh7oCRoYAuFOoI5JhERARIKVE/5IhEaCXgwYOAAwZZrUziEMgqREMByKgwoe6AoZ7ACgzuFOoIyB4gpJhERARIKVB/yG5/AwIkiERiWIi0it5IqCYgy0Jhm4AkbP8DAeiCQAioMZ3mgIGbQB4I7LF8CKgwLeXAShZDAeSoO9GAgB6g4IIGBt3gJkwtyfyggMFcgMEgIgRcIggcgMGAHcRgHcgggMHgIgBcIgggJnAgqDBDAeQKJOGWQCBm/wioMaSCAB9CRaJFZg4DAcioMh3GQLGUgAoWJJIAAZOAByJDAcMEpcVAsZNAPhz6GPYU8hDuDOiIwKBj/7gCAAMCH0KoCiDhkYADBImRQLGQQCoIwwLgYf+4AgABiAAAFCQNAwHIqDAdxkChj0AUFRBi8N8+EYPAKg8gmESkmERwmEQgX7+4AgAwiEQgiESKCx4HKgMkiERcHIQJgINwCAA2AogKDDQIhAgdyDAIAB5ChuZwswQVzm+BpT/ZkUChpL/DAcioMCGJgAMEia1AsYhACFb/ohTeCOJAiFa/nkCDAIGHQChVv4MB+gKDBmyxfCNBy0HsCmT4ImDIIgQIqDGd5hgwVD+fQjYDCKgybc9U7DwFCKgwFavBC0IhgIAACqDiGhLIokHjQkqfiD9wLcy7RZ43fkMeQoGdP8ADBJmhRchQP6IAowYgqDIDAd5AiE8/nkCDBKAJ4MMB0YBAAAMByKg/yCgdBARIGVG/3CgdBARIKVF/xARIGVE/1YStSIDARwnJzce9jICxtD+IsL9ICB0DPcntwJGzf5xK/5wIqAoAqACAHKg0ncSYHKg1HeSAkYhAAbG/gAoMzgjEBEgJSP/jQpWirCionHAqhGCYRKBKv7gCABxHf6RHf7AIAB4B4IhEnC0NcB3EZB3EHC7ICC7gq0IMLvCgSn+4AgAoqPogR7+4AgARrH+AADYU8hDuDOoIxARIKVN/4as/gCyAwMiAwKAuxEguyCyy/CiwxgQESClbv+Gpf4AIgMDcgMCgCIRcCIggRf+4AgAcQf8IsLwiDeAImMWUqeIF4qCgIxBRgMAAACCYRIQESAlAf+CIRKSJwSmGQWSJwKXqOcQESCl+P4Wav+oF80CssMYgQb+4AgAjDoyoMQ5VzgXKjM5Fzg3ICPAKTeBAP7gCACGh/4AAHIDAiLDGDIDAwwZgDMRcDMgMsPwBiMAceL9gZP7mAc5sZCIwIlBiCYMGYezAQw5kmEREBEgZfn+kiERgdr9mQHoB6HZ/d0IILIgwsEs8sEQgmESger94AgAuCadCqixgiESoLvAuSagM8C4B6oiqEEMDKq7DBq5B5DKg4C7wMDQdFaMAMLbgMCtkxZqAa0IgmESkmEREBEgJV3/giESkiERgmcAUcT7eDWMo5CPMZCIwNYoAFb39dapADG/+yKgxylTRgAAjDmcBwZU/hbXlFG6+yKgyClVhlD+ADG3+yKgySlThk3+KCNWEpMQESBlGf+ionHAqhGBtP3gCAAQESBlCP+Bv/3gCAAGRP4oMxaykBARIOUW/6Kj6IGr/eAIABARICUG/+ACAAY8/gAQESBlBf8d8AAANkEAnQKCoMAoA4eZD8wyDBKGBwAMAikDfOKGDwAmEgcmIhiGAwAAAIKg24ApI4eZKgwiKQN88kYIAAAAIqDcJ5kKDBIpAy0IBgQAAACCoN188oeZBgwSKQMioNsd8AAA",
     "text_start": 1077379072,

flasher_stub/stub_commands.c

@@ -119,7 +119,7 @@ void handle_flash_read(uint32_t addr, uint32_t len, uint32_t block_size,
       uint32_t n = len - num_sent;
       if (n > block_size) n = block_size;
       #if defined(ESP32S3)
-        if (addr + len-1 > 0x00ffffff)
+        if (addr + n > 0x01000000)
           res = SPIRead4B(1, SPI_FLASH_FASTRD_MODE, addr, buf, n);
         else
           res = SPIRead(addr, (uint32_t *)buf, n);
@@ -158,7 +158,7 @@ int handle_flash_get_md5sum(uint32_t addr, uint32_t len) {
       n = FLASH_SECTOR_SIZE;
     }
     #if defined(ESP32S3)
-      if (addr + len > 0x00ffffff)
+      if (addr + n > 0x01000000)
         res = SPIRead4B(1, SPI_FLASH_FASTRD_MODE, addr, buf, n);
       else
         res = SPIRead(addr, (uint32_t *)buf, n);

flasher_stub/stub_write_flash.c

@@ -320,7 +320,7 @@ void handle_flash_data(void *data_buf, uint32_t length) {
 
   /* do the actual write */
   #if defined(ESP32S3)
-    if (fs.next_write + length > 0x00ffffff)
+    if (fs.next_write + length > 0x01000000)
       res = SPIWrite4B(1, SPI_FLASH_FASTRD_MODE, fs.next_write, data_buf, length);
     else
       res = SPIWrite(fs.next_write, data_buf, length);

setup.py

@@ -121,7 +121,7 @@ def find_version(*file_paths):
         "cryptography>=2.1.4",
         "ecdsa>=0.16.0",
         "pyserial>=3.0",
-        "reedsolo>=1.5.3,<=1.5.4",
+        "reedsolo>=1.5.3,<=1.6.0",
     ],
     packages=find_packages(),
     include_package_data=True,