Merge pull request esp8266#13 from esp8266/master

ARP fix

.travis.yml

@@ -107,6 +107,18 @@ jobs:
       script: $TRAVIS_BUILD_DIR/tests/buildm.sh
       env: CC=gcc-7 CXX=g++-7
 
+    - name: "Mac OSX can build sketches"
+      os: osx
+      stage: build
+      script: $TRAVIS_BUILD_DIR/tests/build.sh
+      env: MACOSX=1 BUILD_PARITY=custom mod=500 rem=1
+
+    - name: "Windows can build sketches"
+      os: windows
+      stage: build
+      script: $TRAVIS_BUILD_DIR/tests/build.sh
+      env: WINDOWS=1 BUILD_PARITY=custom mod=500 rem=1
+
     - name: "Boards"
       stage: build
       script: $TRAVIS_BUILD_DIR/tests/ci/build_boards.sh

cores/esp8266/core_esp8266_main.cpp

@@ -60,6 +60,13 @@ static os_event_t s_loop_queue[LOOP_QUEUE_SIZE];
 /* Used to implement optimistic_yield */
 static uint32_t s_micros_at_task_start;
 
+/* For ets_intr_lock_nest / ets_intr_unlock_nest
+ * Max nesting seen by SDK so far is 2.
+ */
+#define ETS_INTR_LOCK_NEST_MAX 7
+static uint16_t ets_intr_lock_stack[ETS_INTR_LOCK_NEST_MAX];
+static byte     ets_intr_lock_stack_ptr=0;
+
 
 extern "C" {
 extern const uint32_t __attribute__((section(".ver_number"))) core_version = ARDUINO_ESP8266_GIT_VER;
@@ -121,6 +128,35 @@ extern "C" void optimistic_yield(uint32_t interval_us) {
     }
 }
 
+
+// Replace ets_intr_(un)lock with nestable versions
+extern "C" void IRAM_ATTR ets_intr_lock() {
+  if (ets_intr_lock_stack_ptr < ETS_INTR_LOCK_NEST_MAX)
+     ets_intr_lock_stack[ets_intr_lock_stack_ptr++] = xt_rsil(3);
+  else
+     xt_rsil(3);
+}
+
+extern "C" void IRAM_ATTR ets_intr_unlock() {
+  if (ets_intr_lock_stack_ptr > 0)
+     xt_wsr_ps(ets_intr_lock_stack[--ets_intr_lock_stack_ptr]);
+  else
+     xt_rsil(0);
+}
+
+
+// Save / Restore the PS state across the rom ets_post call as the rom code
+// does not implement this correctly.
+extern "C" bool ets_post_rom(uint8 prio, ETSSignal sig, ETSParam par);
+
+extern "C" bool IRAM_ATTR ets_post(uint8 prio, ETSSignal sig, ETSParam par) {
+  uint32_t saved;
+  asm volatile ("rsr %0,ps":"=a" (saved));
+  bool rc=ets_post_rom(prio, sig, par);
+  xt_wsr_ps(saved);
+  return rc;
+}
+
 extern "C" void __loop_end (void)
 {
     run_scheduled_functions();

package/package_esp8266com_index.template.json

@@ -177,10 +177,10 @@
                   },
                   {
                      "host": "x86_64-apple-darwin",
-                     "url": "https://github.com/earlephilhower/esp-quick-toolchain/releases/download/2.5.0-4/python3-placeholder.tar.gz",
-                     "archiveFileName": "python3-placeholder.tar.gz",
-                     "checksum": "SHA-256:d8cf9d9d66423d7b90978ebe285a73a6e8611995cd0d5e6273e929a0cf2c9850",
-                     "size": "191"
+                     "url": "https://github.com/earlephilhower/esp-quick-toolchain/releases/download/2.5.0-4/python3-macosx-placeholder.tar.gz",
+                     "archiveFileName": "python3-macosx-placeholder.tar.gz",
+                     "checksum": "SHA-256:5bfa0d4c2dc3edeeaa913f4eac42ef3ff0bf8c8fe9f11be112a8ca7911de2dae",
+                     "size": "198"
                   },
                   {
                      "host": "x86_64-pc-linux-gnu",

tests/common.sh

@@ -44,8 +44,7 @@ function print_size_info()
             segments[$seg]=$size
         fi
 
-
-    done < <(xtensa-lx106-elf-size --format=sysv $elf_file)
+    done < <(xtensa-lx106-elf-size --format=sysv $elf_file | sed 's/\r//g' )
 
     total_ram=$((${segments[data]} + ${segments[rodata]} + ${segments[bss]}))
     total_flash=$((${segments[data]} + ${segments[rodata]} + ${segments[text]} + ${segments[irom0text]}))
@@ -66,6 +65,10 @@ function build_sketches()
     local lwip=$6
     mkdir -p $build_dir
     local build_cmd="python3 tools/build.py -b generic -v -w all -s 4M1M -v -k --build_cache $cache_dir -p $PWD/$build_dir -n $lwip $build_arg "
+    if [ "$WINDOWS" = "1" ]; then
+        # Paths to the arduino builder need to be / referenced, not our native ones
+        build_cmd=$(echo $build_cmd --ide_path $arduino | sed 's/ \/c\// \//g' ) # replace '/c/' with '/'
+    fi
     local sketches=$(find $srcpath -name *.ino | sort)
     print_size_info >size.log
     export ARDUINO_IDE_PATH=$arduino
@@ -107,6 +110,14 @@ function build_sketches()
         fi
         echo -e "\n ------------ Building $sketch ------------ \n";
         # $arduino --verify $sketch;
+	if [ "$WINDOWS" == "1" ]; then
+            sketch=$(echo $sketch | sed 's/^\/c//')
+            # MINGW will try to be helpful and silently convert args that look like paths to point to a spot inside the MinGW dir.  This breaks everything.
+            # http://www.mingw.org/wiki/Posix_path_conversion
+            # https://stackoverflow.com/questions/7250130/how-to-stop-mingw-and-msys-from-mangling-path-names-given-at-the-command-line#34386471
+            export MSYS2_ARG_CONV_EXC="*"
+            export MSYS_NO_PATHCONV=1
+        fi
         echo "$build_cmd $sketch"
         time ($build_cmd $sketch >build.log)
         local result=$?
@@ -135,7 +146,7 @@ function install_libraries()
     pushd $HOME/Arduino/libraries
 
     # install ArduinoJson library
-    { test -r ArduinoJson-v6.11.0.zip || wget https://github.com/bblanchon/ArduinoJson/releases/download/v6.11.0/ArduinoJson-v6.11.0.zip; } && unzip ArduinoJson-v6.11.0.zip
+    { test -r ArduinoJson-v6.11.0.zip || wget -nv https://github.com/bblanchon/ArduinoJson/releases/download/v6.11.0/ArduinoJson-v6.11.0.zip; } && unzip -q ArduinoJson-v6.11.0.zip
 
     popd
 }
@@ -145,13 +156,40 @@ function install_ide()
     local ide_path=$1
     local core_path=$2
     local debug=$3
-    test -r arduino.tar.xz || wget -O arduino.tar.xz https://www.arduino.cc/download.php?f=/arduino-nightly-linux64.tar.xz
-    tar xf arduino.tar.xz
+    if [ "$WINDOWS" = "1" ]; then
+        # Acquire needed packages from Windows package manager
+        choco install --no-progress python3
+        export PATH="/c/Python37:$PATH"  # Ensure it's live from now on...
+        cp /c/Python37/python.exe /c/Python37/python3.exe
+        choco install --no-progress unzip
+        choco install --no-progress sed
+        #choco install --no-progress golang
+        test -r arduino-nightly-windows.zip || wget -nv -O arduino-nightly-windows.zip https://www.arduino.cc/download.php?f=/arduino-nightly-windows.zip
+        unzip -q arduino-nightly-windows.zip
+    elif [ "$MACOSX" = "1" ]; then
+        # MACOS only has next-to-obsolete Python2 installed.  Install Python 3 from python.org
+        wget https://www.python.org/ftp/python/3.7.4/python-3.7.4-macosx10.9.pkg
+        sudo installer -pkg python-3.7.4-macosx10.9.pkg -target /
+        # Install the Python3 certificates, because SSL connections fail w/o them and of course they aren't installed by default.
+        ( cd "/Applications/Python 3.7/" && sudo "./Install Certificates.command" )
+        # Hack to place arduino-builder in the same spot as sane OSes
+        test -r arduino.zip || wget -O arduino.zip https://downloads.arduino.cc/arduino-nightly-macosx.zip
+        unzip -q arduino.zip
+        mv Arduino.app arduino-nightly
+        mv arduino-nightly/Contents/Java/* arduino-nightly/.
+    else
+        test -r arduino.tar.xz || wget -O arduino.tar.xz https://www.arduino.cc/download.php?f=/arduino-nightly-linux64.tar.xz
+        tar xf arduino.tar.xz
+    fi
     mv arduino-nightly $ide_path
     cd $ide_path/hardware
     mkdir esp8266com
     cd esp8266com
-    ln -s $core_path esp8266
+    if [ "$WINDOWS" = "1" ]; then
+        cp -a $core_path esp8266
+    else
+        ln -s $core_path esp8266
+    fi
     local debug_flags=""
     if [ "$debug" = "debug" ]; then
         debug_flags="-DDEBUG_ESP_PORT=Serial -DDEBUG_ESP_SSL -DDEBUG_ESP_TLS_MEM -DDEBUG_ESP_HTTP_CLIENT -DDEBUG_ESP_HTTP_SERVER -DDEBUG_ESP_CORE -DDEBUG_ESP_WIFI -DDEBUG_ESP_HTTP_UPDATE -DDEBUG_ESP_UPDATER -DDEBUG_ESP_OTA -DDEBUG_ESP_OOM"
@@ -163,8 +201,14 @@ function install_ide()
     cat esp8266/platform.local.txt
     echo -e "\n----\n"
     cd esp8266/tools
-    python3 get.py
-    export PATH="$ide_path:$core_path/tools/xtensa-lx106-elf/bin:$PATH"
+    python3 get.py -q
+    if [ "$WINDOWS" = "1" ]; then
+        # Because the symlinks don't work well under Win32, we need to add the path to this copy, not the original...
+        relbin=$(realpath $PWD/xtensa-lx106-elf/bin)
+        export PATH="$ide_path:$relbin:$PATH"
+    else
+        export PATH="$ide_path:$core_path/tools/xtensa-lx106-elf/bin:$PATH"
+    fi
 }
 
 function install_arduino()
@@ -174,7 +218,6 @@ function install_arduino()
     echo -e "travis_fold:start:sketch_test_env_prepare"
     cd $TRAVIS_BUILD_DIR
     install_ide $HOME/arduino_ide $TRAVIS_BUILD_DIR $debug
-    which arduino
     cd $TRAVIS_BUILD_DIR
     install_libraries
     echo -e "travis_fold:end:sketch_test_env_prepare"

tools/build.py

@@ -24,28 +24,42 @@
 import sys
 import os
 import argparse
+import platform
 import subprocess
 import tempfile
 import shutil
 
+
+# Arduino-builder needs forward-slash paths for passed in params or it cannot
+# launch the needed toolset.
+def windowsize_paths(l):
+    """Convert forward-slash paths to backslash paths referenced from C:"""
+    out = []
+    for i in l:
+        if i.startswith('/'):
+            i = 'C:' + i
+        out += [i.replace('/', '\\')]
+    return out
+
 def compile(tmp_dir, sketch, cache, tools_dir, hardware_dir, ide_path, f, args):
-    cmd = ide_path + '/arduino-builder '
-    cmd += '-compile -logger=human '
-    cmd += '-build-path "' + tmp_dir + '" '
-    cmd += '-tools "' +  ide_path + '/tools-builder" '
+    cmd = []
+    cmd += [ide_path + '/arduino-builder']
+    cmd += ['-compile', '-logger=human']
+    cmd += ['-build-path', tmp_dir]
+    cmd += ['-tools', ide_path + '/tools-builder']
     if cache != "":
-        cmd += '-build-cache "' + cache + '" '
+        cmd += ['-build-cache', cache ]
     if args.library_path:
         for lib_dir in args.library_path:
-            cmd += '-libraries "' + lib_dir + '" '
-    cmd += '-hardware "' + ide_path + '/hardware" '
+            cmd += ['-libraries', lib_dir]
+    cmd += ['-hardware', ide_path + '/hardware']
     if args.hardware_dir:
         for hw_dir in args.hardware_dir:
-            cmd += '-hardware "' + hw_dir + '" '
+            cmd += ['-hardware', hw_dir]
     else:
-        cmd += '-hardware "' + hardware_dir + '" '
+        cmd += ['-hardware', hardware_dir]
     # Debug=Serial,DebugLevel=Core____
-    cmd += '-fqbn=esp8266com:esp8266:{board_name}:' \
+    fqbn = '-fqbn=esp8266com:esp8266:{board_name}:' \
             'xtal={cpu_freq},' \
             'FlashFreq={flash_freq},' \
             'FlashMode={flash_mode},' \
@@ -54,20 +68,22 @@ def compile(tmp_dir, sketch, cache, tools_dir, hardware_dir, ide_path, f, args):
             'ip={lwIP},' \
             'ResetMethod=nodemcu'.format(**vars(args))
     if args.debug_port and args.debug_level:
-        cmd += 'dbg={debug_port},lvl={debug_level}'.format(**vars(args))
-    cmd += ' '
-    cmd += '-built-in-libraries "' + ide_path + '/libraries" '
-    cmd += '-ide-version=10607 '
-    cmd += '-warnings={warnings} '.format(**vars(args))
+        fqbn += 'dbg={debug_port},lvl={debug_level}'.format(**vars(args))
+    cmd += [fqbn]
+    cmd += ['-built-in-libraries', ide_path + '/libraries']
+    cmd += ['-ide-version=10607']
+    cmd += ['-warnings={warnings}'.format(**vars(args))]
     if args.verbose:
-        cmd += '-verbose '
-    cmd += sketch
+        cmd += ['-verbose']
+    cmd += [sketch]
+
+    if platform.system() == "Windows":
+        cmd = windowsize_paths(cmd)
 
     if args.verbose:
-        print('Building: ' + cmd, file=f)
+        print('Building: ' + " ".join(cmd), file=f)
 
-    cmds = cmd.split(' ')
-    p = subprocess.Popen(cmds, stdout=f, stderr=subprocess.STDOUT)
+    p = subprocess.Popen(cmd, stdout=f, stderr=subprocess.STDOUT)
     p.wait()
     return p.returncode
 
@@ -127,6 +143,7 @@ def main():
     hardware_dir = os.path.dirname(os.path.realpath(__file__)) + '/../cores'
 
     output_name = tmp_dir + '/' + os.path.basename(sketch_path) + '.bin'
+
     if args.verbose:
         print("Sketch: ", sketch_path)
         print("Build dir: ", tmp_dir)

tools/get.py

@@ -15,6 +15,9 @@
 import tarfile
 import zipfile
 import re
+
+verbose = True
+
 if sys.version_info[0] == 3:
     from urllib.request import urlretrieve
 else:
@@ -38,10 +41,12 @@ def mkdir_p(path):
             raise
 
 def report_progress(count, blockSize, totalSize):
-    percent = int(count*blockSize*100/totalSize)
-    percent = min(100, percent)
-    sys.stdout.write("\r%d%%" % percent)
-    sys.stdout.flush()
+    global verbose
+    if verbose:
+        percent = int(count*blockSize*100/totalSize)
+        percent = min(100, percent)
+        sys.stdout.write("\r%d%%" % percent)
+        sys.stdout.flush()
 
 def unpack(filename, destination):
     dirname = ''
@@ -111,6 +116,11 @@ def identify_platform():
     return arduino_platform_names[sys_name][bits]
 
 def main():
+    global verbose
+    # Support optional "-q" quiet mode simply
+    if len(sys.argv) == 2:
+        if sys.argv[1] == "-q":
+            verbose = False
     print('Platform: {0}'.format(identify_platform()))
     tools_to_download = load_tools_list('../package/package_esp8266com_index.template.json', identify_platform())
     mkdir_p(dist_dir)

tools/sdk/ld/eagle.rom.addr.v6.ld

@@ -119,16 +119,21 @@ PROVIDE ( ets_install_external_printf = 0x40002450 );
 PROVIDE ( ets_install_putc1 = 0x4000242c );
 PROVIDE ( ets_install_putc2 = 0x4000248c );
 PROVIDE ( ets_install_uart_printf = 0x40002438 );
+/* permanently hide reimplemented ets_intr_*lock(), see #6484
 PROVIDE ( ets_intr_lock = 0x40000f74 );
 PROVIDE ( ets_intr_unlock = 0x40000f80 );
+*/
 PROVIDE ( ets_isr_attach = 0x40000f88 );
 PROVIDE ( ets_isr_mask = 0x40000f98 );
 PROVIDE ( ets_isr_unmask = 0x40000fa8 );
 PROVIDE ( ets_memcmp = 0x400018d4 );
 PROVIDE ( ets_memcpy = 0x400018b4 );
 PROVIDE ( ets_memmove = 0x400018c4 );
 PROVIDE ( ets_memset = 0x400018a4 );
+/* renamed to ets_post_rom(), see #6484
 PROVIDE ( ets_post = 0x40000e24 );
+*/
+PROVIDE ( ets_post_rom = 0x40000e24 );
 PROVIDE ( ets_printf = 0x400024cc );
 PROVIDE ( ets_putc = 0x40002be8 );
 PROVIDE ( ets_rtc_int_register = 0x40002a40 );