From 77c4f5e5cfb2286d6862621bd78120f027d10a9f Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Sat, 7 Sep 2019 18:31:17 -0700 Subject: [PATCH 1/4] Add OSX build to CI, fix OSX builds (#6492) Build a single sketch using a Travis-CI OSX instance to validate the toolchain works properly on Macs. Update the installed python3 symlink to point to the proper spot for OSX (Python3 is in /usr/local/bin, not /usr/bin). Fixes #6490 --- .travis.yml | 6 ++++++ .../package_esp8266com_index.template.json | 8 ++++---- tests/common.sh | 20 +++++++++++++++---- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 36bab4ecf1..341db3dbd3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -107,6 +107,12 @@ jobs: script: $TRAVIS_BUILD_DIR/tests/buildm.sh env: CC=gcc-7 CXX=g++-7 + - name: "MacOS can build sketches" + os: osx + stage: build + script: $TRAVIS_BUILD_DIR/tests/build.sh + env: MACOSX=1 BUILD_PARITY=custom mod=500 rem=1 + - name: "Boards" stage: build script: $TRAVIS_BUILD_DIR/tests/ci/build_boards.sh diff --git a/package/package_esp8266com_index.template.json b/package/package_esp8266com_index.template.json index 63bd091e44..3c96a2e1cf 100644 --- a/package/package_esp8266com_index.template.json +++ b/package/package_esp8266com_index.template.json @@ -177,10 +177,10 @@ }, { "host": "x86_64-apple-darwin", - "url": "https://github.com/earlephilhower/esp-quick-toolchain/releases/download/2.5.0-4/python3-placeholder.tar.gz", - "archiveFileName": "python3-placeholder.tar.gz", - "checksum": "SHA-256:d8cf9d9d66423d7b90978ebe285a73a6e8611995cd0d5e6273e929a0cf2c9850", - "size": "191" + "url": "https://github.com/earlephilhower/esp-quick-toolchain/releases/download/2.5.0-4/python3-macosx-placeholder.tar.gz", + "archiveFileName": "python3-macosx-placeholder.tar.gz", + "checksum": "SHA-256:5bfa0d4c2dc3edeeaa913f4eac42ef3ff0bf8c8fe9f11be112a8ca7911de2dae", + "size": "198" }, { "host": "x86_64-pc-linux-gnu", diff --git a/tests/common.sh b/tests/common.sh index ca99bee7ab..2fe861581c 100755 --- a/tests/common.sh +++ b/tests/common.sh @@ -135,7 +135,7 @@ function install_libraries() pushd $HOME/Arduino/libraries # install ArduinoJson library - { test -r ArduinoJson-v6.11.0.zip || wget https://github.com/bblanchon/ArduinoJson/releases/download/v6.11.0/ArduinoJson-v6.11.0.zip; } && unzip ArduinoJson-v6.11.0.zip + { test -r ArduinoJson-v6.11.0.zip || wget https://github.com/bblanchon/ArduinoJson/releases/download/v6.11.0/ArduinoJson-v6.11.0.zip; } && unzip -q ArduinoJson-v6.11.0.zip popd } @@ -145,8 +145,21 @@ function install_ide() local ide_path=$1 local core_path=$2 local debug=$3 - test -r arduino.tar.xz || wget -O arduino.tar.xz https://www.arduino.cc/download.php?f=/arduino-nightly-linux64.tar.xz - tar xf arduino.tar.xz + if [ "$MACOSX" = "1" ]; then + # MACOS only has next-to-obsolete Python2 installed. Install Python 3 from python.org + wget https://www.python.org/ftp/python/3.7.4/python-3.7.4-macosx10.9.pkg + sudo installer -pkg python-3.7.4-macosx10.9.pkg -target / + # Install the Python3 certificates, because SSL connections fail w/o them and of course they aren't installed by default. + ( cd "/Applications/Python 3.7/" && sudo "./Install Certificates.command" ) + # Hack to place arduino-builder in the same spot as sane OSes + test -r arduino.zip || wget -O arduino.zip https://downloads.arduino.cc/arduino-nightly-macosx.zip + unzip -q arduino.zip + mv Arduino.app arduino-nightly + mv arduino-nightly/Contents/Java/* arduino-nightly/. + else + test -r arduino.tar.xz || wget -O arduino.tar.xz https://www.arduino.cc/download.php?f=/arduino-nightly-linux64.tar.xz + tar xf arduino.tar.xz + fi mv arduino-nightly $ide_path cd $ide_path/hardware mkdir esp8266com @@ -174,7 +187,6 @@ function install_arduino() echo -e "travis_fold:start:sketch_test_env_prepare" cd $TRAVIS_BUILD_DIR install_ide $HOME/arduino_ide $TRAVIS_BUILD_DIR $debug - which arduino cd $TRAVIS_BUILD_DIR install_libraries echo -e "travis_fold:end:sketch_test_env_prepare" From 4f74ed8408abb061c2106a4c1b7e1fc755471ca9 Mon Sep 17 00:00:00 2001 From: Martin Berka Date: Tue, 10 Sep 2019 07:44:19 -0700 Subject: [PATCH 2/4] Edited OTA readme, added Stream Interface snippet (#6487) * Edited OTA readme, added Stream Interface snippet Reworded for easier understanding. Changes mostly in first half of page. * OTA docs: corrected typos, misc edits near end * Incorporated suggestions from earlephilhower Extra ^, removed TODO --- doc/ota_updates/readme.rst | 119 ++++++++++++++++++++----------------- 1 file changed, 63 insertions(+), 56 deletions(-) diff --git a/doc/ota_updates/readme.rst b/doc/ota_updates/readme.rst index 45f0079d9b..67f4f91107 100644 --- a/doc/ota_updates/readme.rst +++ b/doc/ota_updates/readme.rst @@ -5,7 +5,7 @@ OTA Updates Introduction ------------ -OTA (Over the Air) update is the process of loading the firmware to ESP module using Wi-Fi connection rather than a serial port. Such functionality became extremely useful in case of limited or no physical access to the module. +OTA (Over the Air) update is the process of uploading firmware to an ESP module using a Wi-Fi connection rather than a serial port. Such functionality becomes extremely useful in case of limited or no physical access to the module. OTA may be done using: @@ -13,23 +13,23 @@ OTA may be done using: - `Web Browser <#web-browser>`__ - `HTTP Server <#http-server>`__ -Arduino IDE option is intended primarily for software development phase. The two other options would be more useful after deployment, to provide module with application updates manually with a web browser, or automatically using a http server. +The Arduino IDE option is intended primarily for the software development phase. The other two options would be more useful after deployment, to provide the module with application updates either manually with a web browser, or automatically using an HTTP server. -In any case, the first firmware upload has to be done over a serial port. If the OTA routines are correctly implemented in a sketch, then all subsequent uploads may be done over the air. +In any case, the first firmware upload has to be done over a serial port. If the OTA routines are correctly implemented in the sketch, then all subsequent uploads may be done over the air. -By default there is no imposed security on OTA process. It is up to developer to ensure that updates are allowed only from legitimate / trusted sources. Once the update is complete, the module restarts, and the new code is executed. The developer should ensure that the application running on the module is shut down and restarted in a safe manner. Chapters below provide additional information regarding security and safety of OTA process. +By default, there is no imposed security for the OTA process. It is up to the developer to ensure that updates are allowed only from legitimate / trusted sources. Once the update is complete, the module restarts, and the new code is executed. The developer should ensure that the application running on the module is shut down and restarted in a safe manner. Chapters below provide additional information regarding security and safety of OTA updates. Security Disclaimer ~~~~~~~~~~~~~~~~~~~ -No guarantees as to the level of security provided for your application by the following methods is implied. Please refer to the GNU LGPL license associated for this project for full disclaimers. If you do find security weaknesses, please don't hesitate to contact the maintainers or supply pull requests with fixes. The MD5 verification and password protection schemes are already known as supplying a very weak level of security. +No guarantees as to the level of security provided for your application by the following methods is implied. Please refer to the GNU LGPL license associated for this project for full disclaimers. If you do find security weaknesses, please don't hesitate to contact the maintainers or supply pull requests with fixes. The MD5 verification and password protection schemes are already known to supply a very weak level of security. Basic Security ~~~~~~~~~~~~~~ -The module has to be exposed wirelessly to get it updated with a new sketch. That poses chances of module being violently hacked and loaded with some other code. To reduce likelihood of being hacked consider protecting your uploads with a password, selecting certain OTA port, etc. +The module has to be exposed wirelessly to get it updated with a new sketch. That poses a risk of the module being violently hacked and programmed with some other code. To reduce the likelihood of being hacked, consider protecting your uploads with a password, selecting certain OTA port, etc. -Check functionality provided with `ArduinoOTA `__ library that may improve security: +Check functionality provided with the `ArduinoOTA `__ library that may improve security: .. code:: cpp @@ -37,38 +37,38 @@ Check functionality provided with `ArduinoOTA `__ and espota.py use `Digest-MD5 `__ to authenticate upload. Integrity of transferred data is verified on ESP side using `MD5 `__ checksum. +Certain basic protection is already built in and does not require any additional coding by the developer. `ArduinoOTA `__ and espota.py use `Digest-MD5 `__ to authenticate uploads. Integrity of transferred data is verified on the ESP side using `MD5 `__ checksum. -Make your own risk analysis and depending on application decide what library functions to implement. If required, consider implementation of other means of protection from being hacked, e.g. exposing module for uploads only according to specific schedule, trigger OTA only be user pressing dedicated “Update” button wired to ESP, etc. +Make your own risk analysis and, depending on the application, decide what library functions to implement. If required, consider implementation of other means of protection from being hacked, like exposing modules for uploads only according to a specific schedule, triggering OTA only when the user presses a dedicated “Update” button wired to the ESP, etc. Advanced Security - Signed Updates ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -While the above password-based security will dissuade casual hacking attempts, it is not highly secure. For applications where a higher level of security is needed, cryptographically signed OTA updates can be required. It uses SHA256 hashing in place of MD5 (which is known to be cryptographically broken) and RSA-2048 bit level encryption to guarantee only the holder of a cryptographic private key can generate code accepted by the OTA update mechanisms. +While the above password-based security will dissuade casual hacking attempts, it is not highly secure. For applications where a higher level of security is needed, cryptographically signed OTA updates can be required. This uses SHA256 hashing in place of MD5 (which is known to be cryptographically broken) and RSA-2048 bit level encryption to guarantee that only the holder of a cryptographic private key can generate code accepted by the OTA update mechanisms. -These are updates whose compiled binary are signed with a private key (held by the developer) and verified with a public key (stored in the application and available for all to see). The signing process computes a hash of the binary code, encrypts the hash with the developer's private key, and appends this encrypted hash to the binary that is uploaded (via OTA, web, or HTTP server). If the code is modified or replaced in any way by anyone by the developer with the key, the hash will not match and the ESP8266 will reject the upload and not accept it. +Signed updates are updates whose compiled binaries are signed with a private key (held by the developer) and verified with a public key (stored in the application and available for all to see). The signing process computes a hash of the binary code, encrypts the hash with the developer's private key, and appends this encrypted hash to the binary that is uploaded (via OTA, web, or HTTP server). If the code is modified or replaced in any way by anyone except the holder of the developer's private key, the hash will not match and the ESP8266 will reject the upload. -Cryptographic signing only protects against tampering of binaries delivered OTA. If someone has physical access they will always be able to flash the device over the serial port. Signing also does not encrypt anything but the hash (so that it can't be modified), so this does not provide protection for code inside the device. Again, if a user has physical access they can read out your program. +Cryptographic signing only protects against tampering with binaries delivered via OTA. If someone has physical access, they will always be able to flash the device over the serial port. Signing also does not encrypt anything but the hash (so that it can't be modified), so this does not protect code inside the device: if a user has physical access they can read out your program. -**Securing your private key is paramount. The same private/public keypair needs to be used to sign binaries as the original upload. Loss of the private key associated with a binary means that you will not be able to OTA to update any of your devices in the field. Alternatively, if the private key is copied, then the copy can be used to sign binaries which will be accepted.** +**Securing your private key is paramount. The same private/public keypair that was used with the original upload must also be used to sign later binaries. Loss of the private key associated with a binary means that you will not be able to OTA-update any of your devices in the field. Alternatively, if someone else copies the private key, then they will be able to use it to sign binaries which will be accepted by the ESP.** Signed Binary Format ^^^^^^^^^^^^^^^^^^^^ The format of a signed binary is compatible with the standard binary format, and can be uploaded to a non-signed ESP8266 via serial or OTA without any conditions. Note, however, that once an unsigned OTA app is overwritten by this signed version, further updates will require signing. -As shown below, the signed hash is appended to the unsigned binary followed by the total length of the signed hash (i.e. if the signed hash was 64 bytes, then this uint32 will contain 64). This format allows for extensibility (such as adding in a CA-based validation scheme allowing multiple signing keys all based off of a trust anchor), and pull requests are always welcome. +As shown below, the signed hash is appended to the unsigned binary, followed by the total length of the signed hash (i.e., if the signed hash was 64 bytes, then this uint32 data segment will contain 64). This format allows for extensibility (such as adding a CA-based validation scheme allowing multiple signing keys all based on a trust anchor). Pull requests are always welcome. .. code:: bash NORMAL-BINARY -Signed Binary Prequisites -^^^^^^^^^^^^^^^^^^^^^^^^^ +Signed Binary Prerequisites +^^^^^^^^^^^^^^^^^^^^^^^^^^^ OpenSSL is required to run the standard signing steps, and should be available on any UNIX-like or Windows system. As usual, the latest stable version of OpenSSL is recommended. -Signing requires the generation of an RSA-2048 key (other bit lengths are supported as well, but 2048 is a good selection today) using any appropriate tool. The following lines will generate a new public/private keypair. Run them in the sketch directory: +Signing requires the generation of an RSA-2048 key (other bit lengths are supported as well, but 2048 is a good selection today) using any appropriate tool. The following shell commands will generate a new public/private keypair. Run them in the sketch directory: .. code:: bash @@ -78,9 +78,9 @@ Signing requires the generation of an RSA-2048 key (other bit lengths are suppor Automatic Signing -- Only available on Linux and Mac ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The simplest way of implementing signing is to use the automatic mode, which is only possible on Linux and Mac presently due to missing tools under Windows. This mode uses the IDE to configure the source code to enable sigining verification with a given public key, and signs binaries as part of the standard build process using a given public key. +The simplest way of implementing signing is to use the automatic mode, which presently is only possible on Linux and Mac due to some of the tools not being available for Windows. This mode uses the IDE to configure the source code to enable sigining verification with a given public key, and signs binaries as part of the standard build process using a given public key. -To enable this mode, just include `private.key` and `public.key` in the sketch `.ino` directory. The IDE will call a helper script (`tools/signing.py`) before the build begins to create a header to enable key validation using the given public key, and after the build process to actually do the signing, generating a `sketch.bin.signed` file. When OTA is enabled (ArduinoOTA, Web, or HTTP) the binary will only accept signed updates automatically. +To enable this mode, just include `private.key` and `public.key` in the sketch `.ino` directory. The IDE will call a helper script (`tools/signing.py`) before the build begins to create a header to enable key validation using the given public key, and to actually do the signing after the build process, generating a `sketch.bin.signed` file. When OTA is enabled (ArduinoOTA, Web, or HTTP), the binary will automatically only accept signed updates. When the signing process starts, the message: @@ -88,7 +88,7 @@ When the signing process starts, the message: Enabling binary signing -Will appear in the IDE window before a compile is launched, and at the completion of the build the signed binary file well be displayed in the IDE build window as: +will appear in the IDE window before a compile is launched. At the completion of the build, the signed binary file well be displayed in the IDE build window as: .. code:: bash @@ -102,10 +102,10 @@ If you receive either of the following messages in the IDE window, the signing w ... or ... Not signing the generated binary -Manual Signing Binaries -^^^^^^^^^^^^^^^^^^^^^^^ +Manual Signing of Binaries +^^^^^^^^^^^^^^^^^^^^^^^^^^ -Users may also manually sign executables and require the OTA process to verify their signature. In the main code, before enabling any update methods, add the call: +Users may also manually sign executables and require the OTA process to verify their signature. In the main code, before enabling any update methods, add the following declarations and function call: .. code:: cpp @@ -117,7 +117,7 @@ Users may also manually sign executables and require the OTA process to verify t Update.installSignature( &hash, &sign ); -The above snipped creates a BearSSL public key, a SHA256 hash verifier, and tells the Update object to use them to validate any updates it receives from any method. +The above snippet creates a BearSSL public key and a SHA256 hash verifier, and tells the Update object to use them to validate any updates it receives from any method. Compile the sketch normally and, once a `.bin` file is available, sign it using the signer script: @@ -140,11 +140,11 @@ To create a legacy signature, call the signing script with --legacy: Safety ~~~~~~ -OTA process takes ESP’s resources and bandwidth during upload. Then module is restarted and a new sketch executed. Analyse and test how it affects functionality of existing and new sketch. +The OTA process consumes some of the ESP’s resources and bandwidth during upload. Then, the module is restarted and a new sketch executed. Analyse and test how this affects the functionality of the existing and new sketches. -If ESP is placed in remote location and controlling some equipment, you should put additional attention what happens if operation of this equipment is suddenly interrupted by update process. Therefore, decide how to put this equipment into safe state before starting the update. For instance, your module may be controlling a garden watering system in a sequence. If this sequence is not properly shut down and a water valve left open, your garden may be flooded. +If the ESP is in a remote location and controlling some equipment, you should devote additional attention to what happens if operation of this equipment is suddenly interrupted by the update process. Therefore, decide how to put this equipment into a safe state before starting the update. For instance, your module may be controlling a garden watering system in a sequence. If this sequence is not properly shut down and a water valve is left open, the garden may be flooded. -The following functions are provided with `ArduinoOTA `__ library and intended to handle functionality of your application during specific stages of OTA, or on an OTA error: +The following functions are provided with the `ArduinoOTA `__ library and intended to handle functionality of your application during specific stages of OTA, or on an OTA error: .. code:: cpp @@ -156,24 +156,30 @@ The following functions are provided with `ArduinoOTA `__. +Keep in mind that the file system and EEPROM, for example, need space too; see `Flash layout <../filesystem.rst#flash-layout>`__. .. code:: cpp ESP.getFreeSketchSpace(); -can be used for checking the free space for the new sketch. +can be used for checking the free space available for the new sketch. -For overview of memory layout, where new sketch is stored and how it is copied during OTA process, see `Update process - memory view <#update-process-memory-view>`__. +For an overview of memory layout, where the new sketch is stored and how it is copied during the OTA process, see `Update process - memory view <#update-process-memory-view>`__. -The following chapters provide more details and specific methods of doing OTA. +The following chapters provide more details and specific methods for OTA updates. Arduino IDE ----------- -Uploading modules wirelessly from Arduino IDE is intended for the following typical scenarios: - during firmware development as a quicker alternative to loading over a serial, - for updating small quantity of modules, - only if modules are available on the same network as the computer with Arduino IDE. +Uploading modules wirelessly from Arduino IDE is intended for the following typical scenarios: + +- during firmware development as a quicker alternative to loading over a serial port, + +- for updating a small number of modules, + +- only if modules are accessible on the same network as the computer with the Arduino IDE. Requirements ~~~~~~~~~~~~ @@ -183,9 +189,9 @@ Requirements Application Example ~~~~~~~~~~~~~~~~~~~ -Instructions below show configuration of OTA on NodeMCU 1.0 (ESP-12E Module) board. You can use any other board assuming that it meets `requirements <#basic-requirements>`__ described above. This instruction is valid for all operating systems supported by Arduino IDE. Screen captures have been made on Windows 7 and you may see small differences (like name of serial port), if you are using Linux and MacOS. +Instructions below show configuration of OTA on a NodeMCU 1.0 (ESP-12E Module) board. You can use any other board that meets the `requirements <#basic-requirements>`__ described above. This instruction is valid for all operating systems supported by the Arduino IDE. Screen captures have been made on Windows 7 and you may see small differences (like name of the serial port), if you are using Linux or MacOS. -1. Before you begin, please make sure that you have the following s/w +1. Before you begin, please make sure that you have the following software installed: - Arduino IDE 1.6.7 or newer - @@ -201,13 +207,12 @@ Instructions below show configuration of OTA on NodeMCU 1.0 (ESP-12E Module) boa .. figure:: a-ota-python-configuration.png :alt: Python installation set up -2. Now prepare the sketch and configuration for the upload over a serial - port. +2. Now prepare the sketch and configuration for upload via a serial port. - - Start Arduino IDE and load sketch BasicOTA.ino available under + - Start Arduino IDE and upload the sketch BasicOTA.ino, available under File > Examples > ArduinoOTA |ota sketch selection| - - Update SSID and password in the sketch, so the module can join + - Update the SSID and password in the sketch, so that the module can join your Wi-Fi network |ota ssid pass entry| - Configure upload parameters as below (you may need to adjust @@ -225,20 +230,20 @@ Instructions below show configuration of OTA on NodeMCU 1.0 (ESP-12E Module) boa .. figure:: a-ota-upload-complete-and-joined-wifi.png :alt: Check if module joined network -**Note:** ESP module should be reset after serial upload. Otherwise subsequent steps will not work. Reset may be done automatically for you after opening serial monitor as visible on the screenshot above. It depends on how you have DTR and RTS wired from USB-Serial converter to the ESP. If reset is not done automatically, then do it by pressing reset button or manually cycling the power. For more details why this should be done please refer to `FAQ <../faq#i-have-observed-a-case-when-esprestart-doesnt-work-what-is-the-reason-for-that>`__ regarding ``ESP.restart()``. +**Note:** The ESP module should be reset after serial upload. Otherwise, subsequent steps will not work. Reset may be done for you automatically after opening serial monitor, as visible on the screenshot above. It depends on how you have DTR and RTS wired from the USB-Serial converter to the ESP. If reset is not done automatically, then trigger it by pressing reset button or manually cycling the power. For more details why this should be done please refer to `FAQ <../faq#i-have-observed-a-case-when-esprestart-doesnt-work-what-is-the-reason-for-that>`__ regarding ``ESP.restart()``. -4. Only if module is connected to network, after a couple of seconds, +4. Only if the module is connected to network, after a couple of seconds, the esp8266-ota port will show up in Arduino IDE. Select port with IP address shown in the Serial Monitor window in previous step: .. figure:: a-ota-ota-port-selection.png :alt: Selection of OTA port - **Note:** If OTA port does not show up, exit Arduino IDE, open it - again and check if port is there. If it does not help, check your - firewall and router settings. OTA port is advertised using mDNS - service. To check if port is visible by your PC, you can use - application like Bonjour Browser. + **Note:** If the OTA port does not show up, exit Arduino IDE, open it + again and check if the port is there. If it is not, check your + firewall and router settings. The OTA port is advertised using mDNS + service. To check if the port is visible by your PC, you can use + an application like Bonjour Browser. 5. Now get ready for your first OTA upload by selecting the OTA port: @@ -498,9 +503,9 @@ Simple updater downloads the file every time the function is called. Advanced updater ^^^^^^^^^^^^^^^^ -Its possible to point update function to a script at the server. If version string argument is given, it will be sent to the server. Server side script can use this to check if update should be performed. +Its possible to point the update function to a script on the server. If a version string argument is given, it will be sent to the server. The server side script can use this string to check whether an update should be performed. -Server side script can respond as follows: - response code 200, and send the firmware image, - or response code 304 to notify ESP that no update is required. +The server-side script can respond as follows: - response code 200, and send the firmware image, - or response code 304 to notify ESP that no update is required. .. code:: cpp @@ -513,7 +518,7 @@ Server side script can respond as follows: - response code 200, and send the fir Serial.println("[update] Update no Update."); break; case HTTP_UPDATE_OK: - Serial.println("[update] Update ok."); // may not called we reboot the ESP + Serial.println("[update] Update ok."); // may not be called since we reboot the ESP break; } @@ -528,7 +533,7 @@ For the simple updater the server only needs to deliver the binary file for upda Advanced updater ^^^^^^^^^^^^^^^^ -For advanced update management a script needs to run at the server side, for example a PHP script. At every update request the ESP sends some information in HTTP headers to the server. +For advanced update management a script (such as a PHP script) needs to run on the server side. On every update request, the ESP sends some information in HTTP headers to the server. Example header data: @@ -544,9 +549,7 @@ Example header data: [HTTP_X_ESP8266_SDK_VERSION] => 1.3.0 [HTTP_X_ESP8266_VERSION] => DOOR-7-g14f53a19 -With this information the script now can check if an update is needed. It is also possible to deliver different binaries based on the MAC address for example. - -Script example: +With this information the script now can check if an update is needed. It is also possible to deliver different binaries based on the MAC address, as in the following example: .. code:: php @@ -619,9 +622,13 @@ Script example: Stream Interface ---------------- -TODO describe Stream Interface +The Stream Interface is the base for all other update modes like OTA, HTTP Server / client. Given a Stream-class variable `streamVar` providing `byteCount` bytes of firmware, it can store the firmware as follows: + +.. code:: cpp -The Stream Interface is the base for all other update modes like OTA, http Server / client. + Update.begin(firmwareLengthInBytes); + Update.writeStream(streamVar); + Update.end(); Updater class ------------- @@ -635,7 +642,7 @@ Update process - memory view - The new sketch will be stored in the space between the old sketch and the spiff. -- on the next reboot the "eboot" bootloader check for commands. +- on the next reboot, the "eboot" bootloader checks for commands. - the new sketch is now copied "over" the old one. - the new sketch is started. From 1f86311d79a81f6a80a520a053c7697d0fa65bce Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Tue, 10 Sep 2019 14:50:55 -0700 Subject: [PATCH 3/4] Add Win32 build to CI system (#6493) Build a single INO under a Windows VM on Travis to ensure that the Win32 toolchain works properly. In build.py, instead of making a string with spaces and then splitting on " ", just make the list itself with individual parameters. This will allow spaces in paths to be supported properly. --- .travis.yml | 8 ++++++- tests/common.sh | 51 ++++++++++++++++++++++++++++++++++--------- tools/build.py | 57 ++++++++++++++++++++++++++++++++----------------- tools/get.py | 18 ++++++++++++---- 4 files changed, 99 insertions(+), 35 deletions(-) diff --git a/.travis.yml b/.travis.yml index 341db3dbd3..e6a2b4012c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -107,12 +107,18 @@ jobs: script: $TRAVIS_BUILD_DIR/tests/buildm.sh env: CC=gcc-7 CXX=g++-7 - - name: "MacOS can build sketches" + - name: "Mac OSX can build sketches" os: osx stage: build script: $TRAVIS_BUILD_DIR/tests/build.sh env: MACOSX=1 BUILD_PARITY=custom mod=500 rem=1 + - name: "Windows can build sketches" + os: windows + stage: build + script: $TRAVIS_BUILD_DIR/tests/build.sh + env: WINDOWS=1 BUILD_PARITY=custom mod=500 rem=1 + - name: "Boards" stage: build script: $TRAVIS_BUILD_DIR/tests/ci/build_boards.sh diff --git a/tests/common.sh b/tests/common.sh index 2fe861581c..b93bd57ba9 100755 --- a/tests/common.sh +++ b/tests/common.sh @@ -44,8 +44,7 @@ function print_size_info() segments[$seg]=$size fi - - done < <(xtensa-lx106-elf-size --format=sysv $elf_file) + done < <(xtensa-lx106-elf-size --format=sysv $elf_file | sed 's/\r//g' ) total_ram=$((${segments[data]} + ${segments[rodata]} + ${segments[bss]})) total_flash=$((${segments[data]} + ${segments[rodata]} + ${segments[text]} + ${segments[irom0text]})) @@ -66,6 +65,10 @@ function build_sketches() local lwip=$6 mkdir -p $build_dir local build_cmd="python3 tools/build.py -b generic -v -w all -s 4M1M -v -k --build_cache $cache_dir -p $PWD/$build_dir -n $lwip $build_arg " + if [ "$WINDOWS" = "1" ]; then + # Paths to the arduino builder need to be / referenced, not our native ones + build_cmd=$(echo $build_cmd --ide_path $arduino | sed 's/ \/c\// \//g' ) # replace '/c/' with '/' + fi local sketches=$(find $srcpath -name *.ino | sort) print_size_info >size.log export ARDUINO_IDE_PATH=$arduino @@ -107,6 +110,14 @@ function build_sketches() fi echo -e "\n ------------ Building $sketch ------------ \n"; # $arduino --verify $sketch; + if [ "$WINDOWS" == "1" ]; then + sketch=$(echo $sketch | sed 's/^\/c//') + # MINGW will try to be helpful and silently convert args that look like paths to point to a spot inside the MinGW dir. This breaks everything. + # http://www.mingw.org/wiki/Posix_path_conversion + # https://stackoverflow.com/questions/7250130/how-to-stop-mingw-and-msys-from-mangling-path-names-given-at-the-command-line#34386471 + export MSYS2_ARG_CONV_EXC="*" + export MSYS_NO_PATHCONV=1 + fi echo "$build_cmd $sketch" time ($build_cmd $sketch >build.log) local result=$? @@ -135,7 +146,7 @@ function install_libraries() pushd $HOME/Arduino/libraries # install ArduinoJson library - { test -r ArduinoJson-v6.11.0.zip || wget https://github.com/bblanchon/ArduinoJson/releases/download/v6.11.0/ArduinoJson-v6.11.0.zip; } && unzip -q ArduinoJson-v6.11.0.zip + { test -r ArduinoJson-v6.11.0.zip || wget -nv https://github.com/bblanchon/ArduinoJson/releases/download/v6.11.0/ArduinoJson-v6.11.0.zip; } && unzip -q ArduinoJson-v6.11.0.zip popd } @@ -145,17 +156,27 @@ function install_ide() local ide_path=$1 local core_path=$2 local debug=$3 - if [ "$MACOSX" = "1" ]; then + if [ "$WINDOWS" = "1" ]; then + # Acquire needed packages from Windows package manager + choco install --no-progress python3 + export PATH="/c/Python37:$PATH" # Ensure it's live from now on... + cp /c/Python37/python.exe /c/Python37/python3.exe + choco install --no-progress unzip + choco install --no-progress sed + #choco install --no-progress golang + test -r arduino-nightly-windows.zip || wget -nv -O arduino-nightly-windows.zip https://www.arduino.cc/download.php?f=/arduino-nightly-windows.zip + unzip -q arduino-nightly-windows.zip + elif [ "$MACOSX" = "1" ]; then # MACOS only has next-to-obsolete Python2 installed. Install Python 3 from python.org wget https://www.python.org/ftp/python/3.7.4/python-3.7.4-macosx10.9.pkg sudo installer -pkg python-3.7.4-macosx10.9.pkg -target / # Install the Python3 certificates, because SSL connections fail w/o them and of course they aren't installed by default. - ( cd "/Applications/Python 3.7/" && sudo "./Install Certificates.command" ) + ( cd "/Applications/Python 3.7/" && sudo "./Install Certificates.command" ) # Hack to place arduino-builder in the same spot as sane OSes test -r arduino.zip || wget -O arduino.zip https://downloads.arduino.cc/arduino-nightly-macosx.zip unzip -q arduino.zip - mv Arduino.app arduino-nightly - mv arduino-nightly/Contents/Java/* arduino-nightly/. + mv Arduino.app arduino-nightly + mv arduino-nightly/Contents/Java/* arduino-nightly/. else test -r arduino.tar.xz || wget -O arduino.tar.xz https://www.arduino.cc/download.php?f=/arduino-nightly-linux64.tar.xz tar xf arduino.tar.xz @@ -164,7 +185,11 @@ function install_ide() cd $ide_path/hardware mkdir esp8266com cd esp8266com - ln -s $core_path esp8266 + if [ "$WINDOWS" = "1" ]; then + cp -a $core_path esp8266 + else + ln -s $core_path esp8266 + fi local debug_flags="" if [ "$debug" = "debug" ]; then debug_flags="-DDEBUG_ESP_PORT=Serial -DDEBUG_ESP_SSL -DDEBUG_ESP_TLS_MEM -DDEBUG_ESP_HTTP_CLIENT -DDEBUG_ESP_HTTP_SERVER -DDEBUG_ESP_CORE -DDEBUG_ESP_WIFI -DDEBUG_ESP_HTTP_UPDATE -DDEBUG_ESP_UPDATER -DDEBUG_ESP_OTA -DDEBUG_ESP_OOM" @@ -176,8 +201,14 @@ function install_ide() cat esp8266/platform.local.txt echo -e "\n----\n" cd esp8266/tools - python3 get.py - export PATH="$ide_path:$core_path/tools/xtensa-lx106-elf/bin:$PATH" + python3 get.py -q + if [ "$WINDOWS" = "1" ]; then + # Because the symlinks don't work well under Win32, we need to add the path to this copy, not the original... + relbin=$(realpath $PWD/xtensa-lx106-elf/bin) + export PATH="$ide_path:$relbin:$PATH" + else + export PATH="$ide_path:$core_path/tools/xtensa-lx106-elf/bin:$PATH" + fi } function install_arduino() diff --git a/tools/build.py b/tools/build.py index 8859bbdfce..efb1409ba7 100755 --- a/tools/build.py +++ b/tools/build.py @@ -24,28 +24,42 @@ import sys import os import argparse +import platform import subprocess import tempfile import shutil + +# Arduino-builder needs forward-slash paths for passed in params or it cannot +# launch the needed toolset. +def windowsize_paths(l): + """Convert forward-slash paths to backslash paths referenced from C:""" + out = [] + for i in l: + if i.startswith('/'): + i = 'C:' + i + out += [i.replace('/', '\\')] + return out + def compile(tmp_dir, sketch, cache, tools_dir, hardware_dir, ide_path, f, args): - cmd = ide_path + '/arduino-builder ' - cmd += '-compile -logger=human ' - cmd += '-build-path "' + tmp_dir + '" ' - cmd += '-tools "' + ide_path + '/tools-builder" ' + cmd = [] + cmd += [ide_path + '/arduino-builder'] + cmd += ['-compile', '-logger=human'] + cmd += ['-build-path', tmp_dir] + cmd += ['-tools', ide_path + '/tools-builder'] if cache != "": - cmd += '-build-cache "' + cache + '" ' + cmd += ['-build-cache', cache ] if args.library_path: for lib_dir in args.library_path: - cmd += '-libraries "' + lib_dir + '" ' - cmd += '-hardware "' + ide_path + '/hardware" ' + cmd += ['-libraries', lib_dir] + cmd += ['-hardware', ide_path + '/hardware'] if args.hardware_dir: for hw_dir in args.hardware_dir: - cmd += '-hardware "' + hw_dir + '" ' + cmd += ['-hardware', hw_dir] else: - cmd += '-hardware "' + hardware_dir + '" ' + cmd += ['-hardware', hardware_dir] # Debug=Serial,DebugLevel=Core____ - cmd += '-fqbn=esp8266com:esp8266:{board_name}:' \ + fqbn = '-fqbn=esp8266com:esp8266:{board_name}:' \ 'xtal={cpu_freq},' \ 'FlashFreq={flash_freq},' \ 'FlashMode={flash_mode},' \ @@ -54,20 +68,22 @@ def compile(tmp_dir, sketch, cache, tools_dir, hardware_dir, ide_path, f, args): 'ip={lwIP},' \ 'ResetMethod=nodemcu'.format(**vars(args)) if args.debug_port and args.debug_level: - cmd += 'dbg={debug_port},lvl={debug_level}'.format(**vars(args)) - cmd += ' ' - cmd += '-built-in-libraries "' + ide_path + '/libraries" ' - cmd += '-ide-version=10607 ' - cmd += '-warnings={warnings} '.format(**vars(args)) + fqbn += 'dbg={debug_port},lvl={debug_level}'.format(**vars(args)) + cmd += [fqbn] + cmd += ['-built-in-libraries', ide_path + '/libraries'] + cmd += ['-ide-version=10607'] + cmd += ['-warnings={warnings}'.format(**vars(args))] if args.verbose: - cmd += '-verbose ' - cmd += sketch + cmd += ['-verbose'] + cmd += [sketch] + + if platform.system() == "Windows": + cmd = windowsize_paths(cmd) if args.verbose: - print('Building: ' + cmd, file=f) + print('Building: ' + " ".join(cmd), file=f) - cmds = cmd.split(' ') - p = subprocess.Popen(cmds, stdout=f, stderr=subprocess.STDOUT) + p = subprocess.Popen(cmd, stdout=f, stderr=subprocess.STDOUT) p.wait() return p.returncode @@ -127,6 +143,7 @@ def main(): hardware_dir = os.path.dirname(os.path.realpath(__file__)) + '/../cores' output_name = tmp_dir + '/' + os.path.basename(sketch_path) + '.bin' + if args.verbose: print("Sketch: ", sketch_path) print("Build dir: ", tmp_dir) diff --git a/tools/get.py b/tools/get.py index 9a85ae84b4..74b6b510d9 100755 --- a/tools/get.py +++ b/tools/get.py @@ -15,6 +15,9 @@ import tarfile import zipfile import re + +verbose = True + if sys.version_info[0] == 3: from urllib.request import urlretrieve else: @@ -38,10 +41,12 @@ def mkdir_p(path): raise def report_progress(count, blockSize, totalSize): - percent = int(count*blockSize*100/totalSize) - percent = min(100, percent) - sys.stdout.write("\r%d%%" % percent) - sys.stdout.flush() + global verbose + if verbose: + percent = int(count*blockSize*100/totalSize) + percent = min(100, percent) + sys.stdout.write("\r%d%%" % percent) + sys.stdout.flush() def unpack(filename, destination): dirname = '' @@ -111,6 +116,11 @@ def identify_platform(): return arduino_platform_names[sys_name][bits] def main(): + global verbose + # Support optional "-q" quiet mode simply + if len(sys.argv) == 2: + if sys.argv[1] == "-q": + verbose = False print('Platform: {0}'.format(identify_platform())) tools_to_download = load_tools_list('../package/package_esp8266com_index.template.json', identify_platform()) mkdir_p(dist_dir) From d8531cb2c4aec39d917fed8d642afe86af59e1c6 Mon Sep 17 00:00:00 2001 From: Mike Nix Date: Wed, 11 Sep 2019 10:59:07 +0800 Subject: [PATCH 4/4] Ets intr lock nest (#6484) * Replace the SDK's use of ets_intr_lock/unlock with nestable versions Testing has shown that there are several paths in the SDK that result in nested calls to ets_intr_lock() / ets_intr_unlock() which may be a problem. These functions also do not preserve the enabled interrupt level and may result in code running with interrupts enabled when that is not intended. This issue has recently been fixed in the Arduino code by using xt_rsil() / xt_wsr_ps() but still exists in the Espressif SDK code. This commit is intended to fix that and should be used in addition to the above. The maximum nesting I have seen is 2 and lock/unlock calls appear to be balanced. A max of 7 levels of nesting leaves plenty of room for that to change. * make ets_intr_lock_stack uint16_t and behave like the original on over/underflow The PS register is 15 bits, we should store the whole thing as xt_wsr_ps() writes the whole thing. Also if there is an underflow, we should make sure interrupts are enabled. Same goes for overflow making sure interrupts are disabled, although this is less important. * Rename ets_intr_(un)lock_nest to ets_intr_(un)lock This saves having to modify libmain.a, libpp.a and libnet80211.a to use the nested versions. Adjusts fix_sdk_libs.sh accordingly. * Remove ets_intr_(un)lock from the rom .ld as we no longer use them * ets_post() wrapper to preserve interrupt state Add a wrapper around the ets_post code in rom to preserve the interrupt enable state. Rather than modifying the SDK libs, rename ets_post in the .ld file and call the wrapper "ets_post" to replace it. As far as I can establish, ets_post is the only rom function in use by our code or the SDK libs we use that causes calls to ets_intr_(un)lock. * Add IRAM_ATTR to ets_intr_(un)lock and ets_post wrappers. * Throw in a few comments and make ets_intr_lock_stack* static. --- cores/esp8266/core_esp8266_main.cpp | 36 +++++++++++++++++++++++++++++ tools/sdk/ld/eagle.rom.addr.v6.ld | 5 ++++ 2 files changed, 41 insertions(+) diff --git a/cores/esp8266/core_esp8266_main.cpp b/cores/esp8266/core_esp8266_main.cpp index 588019f0dd..61c07da909 100644 --- a/cores/esp8266/core_esp8266_main.cpp +++ b/cores/esp8266/core_esp8266_main.cpp @@ -60,6 +60,13 @@ static os_event_t s_loop_queue[LOOP_QUEUE_SIZE]; /* Used to implement optimistic_yield */ static uint32_t s_micros_at_task_start; +/* For ets_intr_lock_nest / ets_intr_unlock_nest + * Max nesting seen by SDK so far is 2. + */ +#define ETS_INTR_LOCK_NEST_MAX 7 +static uint16_t ets_intr_lock_stack[ETS_INTR_LOCK_NEST_MAX]; +static byte ets_intr_lock_stack_ptr=0; + extern "C" { extern const uint32_t __attribute__((section(".ver_number"))) core_version = ARDUINO_ESP8266_GIT_VER; @@ -121,6 +128,35 @@ extern "C" void optimistic_yield(uint32_t interval_us) { } } + +// Replace ets_intr_(un)lock with nestable versions +extern "C" void IRAM_ATTR ets_intr_lock() { + if (ets_intr_lock_stack_ptr < ETS_INTR_LOCK_NEST_MAX) + ets_intr_lock_stack[ets_intr_lock_stack_ptr++] = xt_rsil(3); + else + xt_rsil(3); +} + +extern "C" void IRAM_ATTR ets_intr_unlock() { + if (ets_intr_lock_stack_ptr > 0) + xt_wsr_ps(ets_intr_lock_stack[--ets_intr_lock_stack_ptr]); + else + xt_rsil(0); +} + + +// Save / Restore the PS state across the rom ets_post call as the rom code +// does not implement this correctly. +extern "C" bool ets_post_rom(uint8 prio, ETSSignal sig, ETSParam par); + +extern "C" bool IRAM_ATTR ets_post(uint8 prio, ETSSignal sig, ETSParam par) { + uint32_t saved; + asm volatile ("rsr %0,ps":"=a" (saved)); + bool rc=ets_post_rom(prio, sig, par); + xt_wsr_ps(saved); + return rc; +} + extern "C" void __loop_end (void) { run_scheduled_functions(); diff --git a/tools/sdk/ld/eagle.rom.addr.v6.ld b/tools/sdk/ld/eagle.rom.addr.v6.ld index 0407fb8806..144c72b557 100644 --- a/tools/sdk/ld/eagle.rom.addr.v6.ld +++ b/tools/sdk/ld/eagle.rom.addr.v6.ld @@ -119,8 +119,10 @@ PROVIDE ( ets_install_external_printf = 0x40002450 ); PROVIDE ( ets_install_putc1 = 0x4000242c ); PROVIDE ( ets_install_putc2 = 0x4000248c ); PROVIDE ( ets_install_uart_printf = 0x40002438 ); +/* permanently hide reimplemented ets_intr_*lock(), see #6484 PROVIDE ( ets_intr_lock = 0x40000f74 ); PROVIDE ( ets_intr_unlock = 0x40000f80 ); +*/ PROVIDE ( ets_isr_attach = 0x40000f88 ); PROVIDE ( ets_isr_mask = 0x40000f98 ); PROVIDE ( ets_isr_unmask = 0x40000fa8 ); @@ -128,7 +130,10 @@ PROVIDE ( ets_memcmp = 0x400018d4 ); PROVIDE ( ets_memcpy = 0x400018b4 ); PROVIDE ( ets_memmove = 0x400018c4 ); PROVIDE ( ets_memset = 0x400018a4 ); +/* renamed to ets_post_rom(), see #6484 PROVIDE ( ets_post = 0x40000e24 ); +*/ +PROVIDE ( ets_post_rom = 0x40000e24 ); PROVIDE ( ets_printf = 0x400024cc ); PROVIDE ( ets_putc = 0x40002be8 ); PROVIDE ( ets_rtc_int_register = 0x40002a40 );