esp8266: Build the SDK bootloader from source.

We had four versions of pre-built bootloaders for the esp8266 with
different settings of logging and color logging. These bootloaders were
manually built from the SDK and shipped with RIOT-OS source code.
However there are more settings that affect the bootloader build that
are relevant to the app or final board that uses this bootloader. In
particular, flash size and flash speed is important for the bootloader
to be able to load an app from a large partition table at the fastest
speed supported by the board layout and flash chip.

Another example is the UART baudrate of the logging output from the
bootloader. The boot ROM will normally start at a baud rate of 74880
(depending on the crystal installed), so it might make sense to keep
the UART output at the same speed so we can debug boot modes and
bootloader with the same terminal.

This patch builds the bootloader.bin file from the ESP8266 SDK source
code. The code is built as a module (esp8266_bootloader) which at the
moment doesn't generate any object code for the application and only
produces a bootloader.bin file set to the BOOTLOADER_BIN make variable
for the esptool.inc.mk to flash.

The code needs to be compiled and linked with custom rules defined in
the module's Makefile since the bootloader.bin is its own separate
application.

The `BOOTLOADER_BIN` variable is changed from a path relative to the
`$(RIOTCPU)/$(CPU)/bin/` directory to be full path. This makes it easier
for applications or board to provide their own bootloader binary if
needed.

As a result of building the bootloader from source we fixed the issue of
having a large partition table. Fixes RIOT-OS#16402.

makefiles/tools/esptool.inc.mk

@@ -1,11 +1,16 @@
+ifneq ($(CPU),esp8266)
+
+# Select the type of bootloader to use.
 ifneq (,$(filter esp_log_colored,$(USEMODULE)))
   BOOTLOADER_COLOR = _colors
 endif
 ifneq (,$(filter esp_log_startup,$(USEMODULE)))
   BOOTLOADER_INFO = _info
 endif
-
-BOOTLOADER_BIN = bootloader$(BOOTLOADER_COLOR)$(BOOTLOADER_INFO).bin
+# Full path to the bootloader binary. In the ESP8266 case this is set by the
+# esp8266_bootloader module.
+BOOTLOADER_BIN ?= $(RIOTCPU)/$(CPU)/bin/bootloader$(BOOTLOADER_COLOR)$(BOOTLOADER_INFO).bin
+endif
 
 ESPTOOL ?= $(RIOTTOOLS)/esptool/esptool.py
 
@@ -20,7 +25,7 @@ else
   FFLAGS += --chip $(FLASH_CHIP) --port $(PROG_DEV) --baud $(PROGRAMMER_SPEED)
   FFLAGS += --before default_reset write_flash -z
   FFLAGS += --flash_mode $(FLASH_MODE) --flash_freq $(FLASH_FREQ)
-  FFLAGS += $(BOOTLOADER_POS) $(RIOTCPU)/$(CPU)/bin/$(BOOTLOADER_BIN)
+  FFLAGS += $(BOOTLOADER_POS) $(BOOTLOADER_BIN)
   FFLAGS += 0x8000 $(BINDIR)/partitions.bin
   FFLAGS += 0x10000 $(FLASHFILE)
 endif
@@ -31,7 +36,7 @@ esp-qemu:
 	$(Q)dd if=/dev/zero bs=1M count=$(FLASH_SIZE) | \
 	  tr "\\000" "\\377" > tmp.bin && cat tmp.bin | \
 		head -c $$(($(BOOTLOADER_POS))) | \
-		cat - $(RIOTCPU)/$(CPU)/bin/$(BOOTLOADER_BIN) tmp.bin | \
+		cat - $(BOOTLOADER_BIN) tmp.bin | \
 		head -c $$((0x8000)) | \
 		cat - $(BINDIR)/partitions.bin tmp.bin | \
 		head -c $$((0x10000)) | \

pkg/esp8266_sdk/Makefile.dep

@@ -1,2 +1,5 @@
 # This package can only be used with the ESP8266 CPU
 FEATURES_REQUIRED += arch_esp8266
+
+# Always include the ESP8266 SDK bootloader.
+USEMODULE += esp8266_bootloader

pkg/esp8266_sdk/Makefile.include

@@ -21,3 +21,13 @@ LINKFLAGS += -L$(ESP8266_SDK_BUILD_DIR)
 
 # esp8266_sdk doesn't generate any .a
 PSEUDOMODULES += esp8266_sdk
+
+# Bootloader module built from the SDK.
+DIRS += $(RIOTBASE)/pkg/esp8266_sdk/bootloader
+PSEUDOMODULES += esp8266_bootloader
+
+ifneq (,$(filter esp8266_bootloader,$(USEMODULE)))
+# Bootloader file used by esptool.inc.mk
+BOOTLOADER_BIN ?= $(BINDIR)/esp8266_bootloader/bootloader.bin
+TEST_EXTRA_FILES += $(BOOTLOADER_BIN)
+endif

pkg/esp8266_sdk/bootloader/Makefile

@@ -0,0 +1,113 @@
+# Bootloader binary blob
+MODULE := esp8266_bootloader
+
+# We are compiling the bootloader from the ESP8266_RTOS_SDK_DIR sources, so we
+# can't use the automatic module SRC discovery rules. Also, the compiled code
+# would not be linked into the application, instead it is linked into its own
+# binary file.
+NO_AUTO_SRC = 1
+
+include $(RIOTBASE)/Makefile.base
+
+# List of bootloader sources.
+include $(CURDIR)/bootloader.inc.mk
+
+# Bootloader baudrate, set to the application defined one if any or the default
+# in serial.inc.mk
+BOOTLOADER_BAUD ?= $(BAUD)
+
+# Bootloader sdkconfig.h defined in CURDIR directory.
+INCLUDES = \
+  -I$(dir $(RIOTBUILD_CONFIG_HEADER_C)) \
+  -I$(ESP8266_RTOS_SDK_DIR)/components/bootloader_support/include \
+  -I$(ESP8266_RTOS_SDK_DIR)/components/bootloader_support/include_priv \
+  -I$(ESP8266_RTOS_SDK_DIR)/components/esp8266/include \
+  -I$(ESP8266_RTOS_SDK_DIR)/components/heap/include \
+  -I$(ESP8266_RTOS_SDK_DIR)/components/heap/port/esp8266/include \
+  -I$(ESP8266_RTOS_SDK_DIR)/components/log/include/ \
+  -I$(ESP8266_RTOS_SDK_DIR)/components/spi_flash/include \
+  -I$(ESP8266_RTOS_SDK_DIR)/components/util/include \
+  -I$(ESP8266_RTOS_SDK_DIR)/build-libs \
+  -I$(CURDIR)
+  #
+
+# BOOTLOADER_BUILD=1 signals to the SDK that's a bootloader build.
+CFLAGS = \
+  -DBOOTLOADER_BUILD=1 \
+  -DESP_PLATFORM \
+  -DRIOT_BOOTLOADER_BAUD=$(BOOTLOADER_BAUD) \
+  -DRIOT_FLASH_SIZE=$(FLASH_SIZE) \
+  -O2 \
+  -ffunction-sections \
+  -fdata-sections \
+  -fstrict-volatile-bitfields \
+  -mlongcalls \
+  #
+
+# Bootloader link flags. We use the SDK source and linking files instead of the
+# RIOT-OS ones to link the bootloader. Note that we also use the unmodified
+# SDK libraries.
+LINKFLAGS = \
+  -nostdlib \
+  -u call_user_start_cpu0 \
+  -Wl,--gc-sections \
+  -Wl,-static \
+  -Wl,-EL \
+  -Wl,--start-group \
+  $(ESP_SDK_BOOTLOADER_OBJS) \
+  -Wl,--end-group \
+  -lgcc \
+  -lstdc++ \
+  -lgcov \
+  -L$(ESP8266_RTOS_SDK_DIR)/components/esp8266/lib \
+  -lcore \
+  -T$(ESP8266_RTOS_SDK_DIR)/components/esp8266/ld/esp8266.rom.ld \
+  -L$(ESP8266_RTOS_SDK_DIR)/components/bootloader/subproject/main \
+  -Tesp8266.bootloader.ld \
+  -Tesp8266.bootloader.rom.ld \
+
+# Build the bootloader on the application directory as it depends on the current
+# app settings from riotbuild.h.
+ESP_SDK_BOOTLOADER_DIR = $(BINDIR)/$(MODULE)
+ESP_SDK_BOOTLOADER_BIN = $(ESP_SDK_BOOTLOADER_DIR)/bootloader.bin
+ESP_SDK_BOOTLOADER_ELF = $(ESP_SDK_BOOTLOADER_DIR)/bootloader.elf
+
+ESP_SDK_BOOTLOADER_OBJS = \
+  $(addprefix $(ESP_SDK_BOOTLOADER_DIR)/,$(ESP_SDK_BOOTLOADER_SRCS:%.c=%.o))
+
+DEPS := $(ESP_SDK_BOOTLOADER_OBJS:%.o=%.d)
+-include $(DEPS)
+
+# Main module dependency. We only build the bootloader.bin from this module.
+$(MODULE).module: $(ESP_SDK_BOOTLOADER_BIN)
+
+OBJ_DEPS += $(CURDIR)/sdkconfig.h
+
+$(ESP_SDK_BOOTLOADER_DIR)/%.o: $(ESP8266_RTOS_SDK_DIR)/%.c $(OBJ_DEPS)
+	$(Q)mkdir -p $(dir $@)
+	$(Q)$(CCACHE) $(CC) \
+	  $(CFLAGS) $(INCLUDES) -MQ '$@' -MD -MP -c -o $@ $(abspath $<)
+
+$(ESP_SDK_BOOTLOADER_DIR):
+	mkdir -p $@
+
+$(ESP_SDK_BOOTLOADER_ELF): $(ESP_SDK_BOOTLOADER_OBJS) \
+    | $(ESP_SDK_BOOTLOADER_DIR)
+	$(Q)$(CC) -o $@ $(LINKFLAGS) -Wl,-Map=$(@:%.elf=%.map)
+
+FLASH_CHIP = esp8266
+ESPTOOL ?= $(RIOTTOOLS)/esptool/esptool.py
+# TODO: These should be exported/configurable from the app side. That would
+# require to export these values.
+FLASH_MODE ?= dout
+FLASH_FREQ ?= 40m
+FLASH_SIZE ?= 4
+
+# We use esptool to extract a version 1 app from the bootloader.elf. This is
+# like the regular objdump binary file but it contains a 16 byte header which
+# specifies the flash size, mode and speed that the ROM bootloader uses to load
+# this second-stage bootloader image.
+$(ESP_SDK_BOOTLOADER_BIN): $(ESP_SDK_BOOTLOADER_ELF)
+	$(Q)$(ESPTOOL) --chip $(FLASH_CHIP) elf2image --flash_mode $(FLASH_MODE) \
+	  --flash_size $(FLASH_SIZE)MB --flash_freq $(FLASH_FREQ) --version 1 -o $@ $<
+	$(Q)mv $(@)0x00000.bin $@

pkg/esp8266_sdk/bootloader/bootloader.inc.mk

@@ -0,0 +1,26 @@
+# Generated by ./update_mk.sh, don't modify directly.
+
+ESP_SDK_BOOTLOADER_SRCS = \
+  components/bootloader/subproject/main/bootloader_start.c \
+  components/bootloader_support/src/bootloader_clock.c \
+  components/bootloader_support/src/bootloader_common.c \
+  components/bootloader_support/src/bootloader_flash.c \
+  components/bootloader_support/src/bootloader_init.c \
+  components/bootloader_support/src/bootloader_random.c \
+  components/bootloader_support/src/bootloader_sha.c \
+  components/bootloader_support/src/bootloader_utility.c \
+  components/bootloader_support/src/efuse.c \
+  components/bootloader_support/src/esp_image_format.c \
+  components/bootloader_support/src/flash_encrypt.c \
+  components/bootloader_support/src/flash_partitions.c \
+  components/bootloader_support/src/flash_qio_mode.c \
+  components/bootloader_support/src/secure_boot.c \
+  components/bootloader_support/src/secure_boot_signatures.c \
+  components/esp8266/source/ets_printf.c \
+  components/log/log.c \
+  components/spi_flash/port/port.c \
+  components/spi_flash/src/spi_flash.c \
+  components/spi_flash/src/spi_flash_raw.c \
+  components/util/src/base64.c \
+  components/util/src/crc.c \
+  #

pkg/esp8266_sdk/bootloader/sdkconfig.h

@@ -0,0 +1,84 @@
+/*
+ * Copyright (C) 2021 iosabi
+ *
+ * This file is subject to the terms and conditions of the GNU Lesser
+ * General Public License v2.1. See the file LICENSE in the top level
+ * directory for more details.
+ */
+
+/**
+ * @ingroup     pkg_esp8266_sdk
+ * @{
+ *
+ * @file
+ * @brief       RIOT-OS modification of the bootloader SDK configuration
+ *
+ * The bootloader build of the ESP8266 SDK needs some settings from the SDK
+ * configuration. These are normally generated by the menuconfig in the vendor
+ * SDK.
+ *
+ * Some of these parameters are configurable by the application. For example,
+ * the UART baudrate used by the console and the verbose level of the
+ * bootloader.
+ *
+ * @author      iosabi <iosabi@protonmail.com>
+ */
+
+#ifndef SDKCONFIG_H
+#define SDKCONFIG_H
+
+#include "riotbuild.h"
+
+#include "esp8266_idf_version.h"
+#include "sdkconfig_default.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#if MODULE_ESP_LOG_COLORED
+#define CONFIG_LOG_COLORS 1
+#endif
+
+#ifndef CONFIG_LOG_BOOTLOADER_LEVEL
+/* SDK Log levels:
+ *
+ *  0 = NONE
+ *  1 = ERROR
+ *  2 = WARN
+ *  3 = INFO
+ *  4 = DEBUG
+ *  5 = VERBOSE
+ */
+#if MODULE_ESP_LOG_STARTUP
+#define CONFIG_LOG_BOOTLOADER_LEVEL 3 /* INFO */
+#else
+#define CONFIG_LOG_BOOTLOADER_LEVEL 0 /* NONE */
+#endif
+#endif
+
+#if FLASH_MODE_QIO
+#define CONFIG_FLASHMODE_QIO 1
+#elif FLASH_MODE_QOUT
+#define CONFIG_FLASHMODE_QOUT 1
+#elif FLASH_MODE_DIO
+#define CONFIG_FLASHMODE_DIO 1
+#elif FLASH_MODE_DOUT
+#define CONFIG_FLASHMODE_DOUT 1
+#else
+#error "Unknown flash mode selected."
+#endif
+
+/* Bootloader output baudrate, defined by the app settings as BAUD or
+ * BOOTLOADER_BAUD. */
+#define CONFIG_CONSOLE_UART_BAUDRATE (RIOT_BOOTLOADER_BAUD)
+
+/* The Makefile FLASH_SIZE value is set in MB, but set as bytes to
+ * CONFIG_SPI_FLASH_SIZE. */
+#define CONFIG_SPI_FLASH_SIZE ((RIOT_FLASH_SIZE) * 1024 * 1024)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  /* SDKCONFIG_H */

pkg/esp8266_sdk/bootloader/sdkconfig_default.h

@@ -0,0 +1,22 @@
+/* Generated by ./update_mk.sh, don't modify directly.
+ * Default CONFIG_ parameters from the SDK package.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define CONFIG_BOOTLOADER_INIT_SPI_FLASH 1
+#define CONFIG_CONSOLE_UART_NUM 0
+#define CONFIG_CRYSTAL_USED_26MHZ 1
+#define CONFIG_LOG_DEFAULT_LEVEL 3
+#define CONFIG_NEWLIB_STDOUT_LINE_ENDING_CRLF 1
+#define CONFIG_PARTITION_TABLE_OFFSET 0x8000
+#define CONFIG_SPI_FLASH_MODE 0x0
+#define CONFIG_SSL_USING_MBEDTLS 1
+#define CONFIG_TARGET_PLATFORM_ESP8266 1
+#define CONFIG_USING_NEW_ETS_VPRINTF 1
+
+#ifdef __cplusplus
+}
+#endif

pkg/esp8266_sdk/bootloader/update_mk.sh

@@ -0,0 +1,89 @@
+#!/bin/bash
+
+# Script to update the bootloader.inc.mk file.
+#
+# bootloader.inc.mk contains the list of source files and config options that
+# the vendor SDK uses while building the bootloader. This is generated and
+# included in the RIOT-OS source repository since it requires to have both a
+# native toolchain and esp8266 toolchain configured and it was in general tricky
+# to get to work from RIOT-OS build system.
+
+SCRIPTDIR=$(dirname $(realpath "$0"))
+
+set -eu
+
+main() {
+  if ! which xtensa-esp8266-elf-gcc >/dev/null; then
+    echo "Assuming xtensa-esp8266-elf-gcc from /opt/esp/xtensa-esp8266-elf/bin"
+    export PATH="/opt/esp/xtensa-esp8266-elf/bin:${PATH}"
+  fi
+
+  local bldr_dir="${SCRIPTDIR}/bldr_build"
+  rm -rf "${bldr_dir}"
+  mkdir -p "${bldr_dir}"
+  cd "${bldr_dir}"
+
+  local sdk_path=$(realpath "${SCRIPTDIR}/../../../build/pkg/esp8266_sdk")
+  if [[ ! -e "${sdk_path}/Kconfig" ]]; then
+    echo "Download the ESP8266 RTOS SDK to ${sdk_path} by building RIOT first"
+    exit 1
+  fi
+
+  # Builds the bootloader.bin with the default config into the bldr_build
+  PROJECT_NAME=bootloader PROJECT_PATH="${bldr_dir}" \
+  make \
+    -f "${sdk_path}/make/project.mk" IDF_PATH="${sdk_path}" \
+    CONFIG_TOOLPREFIX=xtensa-esp8266-elf- \
+    defconfig bootloader -j
+
+  # List of all the sources and headers used by the build except the generated
+  # sdkconfig.h.
+  local bootloader_srcs=(
+    $(find -name '*.d' | xargs cat | tr ' ' '\n' | grep -E '^/[^ ]+\.[ch]$' -o |
+        xargs -I {} realpath {} | grep -v -F /sdkconfig.h | sort | uniq))
+
+  (
+    echo "# Generated by ./update_mk.sh, don't modify directly."
+    echo
+    # List of source files (.c)
+    echo "ESP_SDK_BOOTLOADER_SRCS = \\"
+    local src
+    for src in "${bootloader_srcs[@]}"; do
+      if [[ "${src%.c}" != "${src}" ]]; then
+        echo "  ${src#${sdk_path}/} \\"
+      fi
+    done
+    echo "  #"
+  ) >"${SCRIPTDIR}/bootloader.inc.mk"
+
+  # List of the relevant CONFIG_ settings used by those files.
+  local configs=(
+    $(grep -h -o -E '\bCONFIG_[A-Z0-9_]+\b' "${bootloader_srcs[@]}" |
+      sort | uniq))
+
+  (
+    echo "/* Generated by ./update_mk.sh, don't modify directly."
+    echo " * Default CONFIG_ parameters from the SDK package."
+    echo " */"
+    echo
+    echo "#ifdef __cplusplus"
+    echo "extern \"C\" {"
+    echo "#endif"
+    echo
+    # Only list those configs not in the bootloader sdkconfig.h included in
+    # RIOT-OS.
+    local conf
+    for conf in "${configs[@]}"; do
+      grep -F "#define ${conf} " "${SCRIPTDIR}/sdkconfig.h" >/dev/null ||
+      grep -F "#define ${conf} " "${bldr_dir}/build/include/sdkconfig.h" || true
+    done
+    echo
+    echo "#ifdef __cplusplus"
+    echo "}"
+    echo "#endif"
+  ) >"${SCRIPTDIR}/sdkconfig_default.h"
+
+  echo "Done."
+}
+
+main "$@"