cpu/esp8266: build the SDK bootloader from source

cpu/esp_common/Makefile.include

@@ -160,9 +160,3 @@ $(BINDIR)/partitions.csv: $(FLASHFILE)
 
 $(BINDIR)/partitions.bin: $(PARTITION_TABLE_CSV)
 	$(Q)python3 $(RIOTTOOLS)/esptools/gen_esp32part.py --verify $< $@
-
-# Convert .elf and .csv to .bin files at build time, but make them available for
-# tests at flash time. These can't be added to FLASHDEPS because they depend on
-# on ELFFILE and would trigger a rebuild with "flash-only".
-BUILD_FILES += $(FLASHFILE) $(BINDIR)/partitions.bin
-TEST_EXTRA_FILES += $(FLASHFILE) $(BINDIR)/partitions.bin

makefiles/tools/esptool.inc.mk

@@ -1,19 +1,7 @@
-ifeq ($(CPU),esp8266)
-  ifneq (,$(filter esp_log_colored,$(USEMODULE)))
-    BOOTLOADER_COLOR = _colors
-  endif
-  ifneq (,$(filter esp_log_startup,$(USEMODULE)))
-    BOOTLOADER_INFO = _info
-  endif
-  # Full path to the bootloader binary. In the ESP32 case this is set by the
-  # esp_bootloader module.
-  BOOTLOADER_BIN ?= $(RIOTCPU)/$(CPU)/bin/bootloader$(BOOTLOADER_COLOR)$(BOOTLOADER_INFO).bin
-else
-  # ESP-IDF uses dio as flash mode for esptool.py when qout or qio mode are
-  # configured to always boot in dual SPI mode
-  ifneq (,$(filter qout qio,$(FLASH_MODE)))
-    FLASH_MODE = dio
-  endif
+# ESP-IDF uses dio as flash mode for esptool.py when qout or qio mode are
+# configured to always boot in dual SPI mode
+ifneq (,$(filter qout qio,$(FLASH_MODE)))
+  FLASH_MODE = dio
 endif
 
 ESPTOOL ?= $(RIOTTOOLS)/esptools/esptool.py

pkg/esp8266_sdk/Makefile.dep

@@ -1,2 +1,5 @@
 # This package can only be used with the ESP8266 CPU
 FEATURES_REQUIRED += arch_esp8266
+
+# Always include the ESP8266 SDK bootloader.
+USEMODULE += esp_bootloader

pkg/esp8266_sdk/Makefile.include

@@ -1,3 +1,5 @@
+PSEUDOMODULES += esp8266_sdk
+
 # Directory with the SDK source checkout. Some modules in the cpu/esp8266 use
 # internal parts of the SDK and for that they need access to the
 # ESP8266_RTOS_SDK_DIR path.
@@ -19,5 +21,13 @@ INCLUDES += -I$(ESP8266_SDK_BUILD_DIR)
 # Modified binary libraries are built here in the Makefile.
 LINKFLAGS += -L$(ESP8266_SDK_BUILD_DIR)
 
-# esp8266_sdk doesn't generate any .a
-PSEUDOMODULES += esp8266_sdk
+# Bootloader module built from the SDK.
+DIRS += $(RIOTBASE)/pkg/esp8266_sdk/bootloader
+PSEUDOMODULES += esp_bootloader
+
+ifneq (,$(filter esp_bootloader,$(USEMODULE)))
+  # Bootloader file used by esptool.inc.mk
+  BOOTLOADER_BIN ?= $(BINDIR)/esp_bootloader/bootloader.bin
+endif
+
+$(BOOTLOADER_BIN):

pkg/esp8266_sdk/bootloader/Makefile

@@ -0,0 +1,113 @@
+# Bootloader binary blob
+MODULE := esp_bootloader
+
+# We are compiling the bootloader from the ESP8266_RTOS_SDK_DIR sources, so we
+# can't use the automatic module SRC discovery rules. Also, the compiled code
+# would not be linked into the application, instead it is linked into its own
+# binary file.
+NO_AUTO_SRC = 1
+
+include $(RIOTBASE)/Makefile.base
+
+# List of bootloader sources.
+include $(CURDIR)/bootloader.inc.mk
+
+# Bootloader baudrate, set to the application defined one if any or the default
+# in serial.inc.mk
+BOOTLOADER_BAUD ?= $(BAUD)
+
+# Bootloader sdkconfig.h defined in CURDIR directory.
+INCLUDES = \
+  -I$(dir $(RIOTBUILD_CONFIG_HEADER_C)) \
+  -I$(ESP8266_RTOS_SDK_DIR)/components/bootloader_support/include \
+  -I$(ESP8266_RTOS_SDK_DIR)/components/bootloader_support/include_priv \
+  -I$(ESP8266_RTOS_SDK_DIR)/components/esp8266/include \
+  -I$(ESP8266_RTOS_SDK_DIR)/components/heap/include \
+  -I$(ESP8266_RTOS_SDK_DIR)/components/heap/port/esp8266/include \
+  -I$(ESP8266_RTOS_SDK_DIR)/components/log/include/ \
+  -I$(ESP8266_RTOS_SDK_DIR)/components/spi_flash/include \
+  -I$(ESP8266_RTOS_SDK_DIR)/components/util/include \
+  -I$(ESP8266_RTOS_SDK_DIR)/build-libs \
+  -I$(CURDIR)
+  #
+
+# BOOTLOADER_BUILD=1 signals to the SDK that's a bootloader build.
+CFLAGS = \
+  -DBOOTLOADER_BUILD=1 \
+  -DESP_PLATFORM \
+  -DRIOT_BOOTLOADER_BAUD=$(BOOTLOADER_BAUD) \
+  -DRIOT_FLASH_SIZE=$(FLASH_SIZE) \
+  -O2 \
+  -ffunction-sections \
+  -fdata-sections \
+  -fstrict-volatile-bitfields \
+  -mlongcalls \
+  #
+
+# Bootloader link flags. We use the SDK source and linking files instead of the
+# RIOT-OS ones to link the bootloader. Note that we also use the unmodified
+# SDK libraries.
+LINKFLAGS = \
+  -nostdlib \
+  -u call_user_start_cpu0 \
+  -Wl,--gc-sections \
+  -Wl,-static \
+  -Wl,-EL \
+  -Wl,--start-group \
+  $(ESP_SDK_BOOTLOADER_OBJS) \
+  -Wl,--end-group \
+  -lgcc \
+  -lstdc++ \
+  -lgcov \
+  -L$(ESP8266_RTOS_SDK_DIR)/components/esp8266/lib \
+  -lcore \
+  -T$(ESP8266_RTOS_SDK_DIR)/components/esp8266/ld/esp8266.rom.ld \
+  -L$(ESP8266_RTOS_SDK_DIR)/components/bootloader/subproject/main \
+  -Tesp8266.bootloader.ld \
+  -Tesp8266.bootloader.rom.ld \
+
+# Build the bootloader on the application directory as it depends on the current
+# app settings from riotbuild.h.
+ESP_SDK_BOOTLOADER_DIR = $(BINDIR)/$(MODULE)
+ESP_SDK_BOOTLOADER_BIN = $(ESP_SDK_BOOTLOADER_DIR)/bootloader.bin
+ESP_SDK_BOOTLOADER_ELF = $(ESP_SDK_BOOTLOADER_DIR)/bootloader.elf
+
+ESP_SDK_BOOTLOADER_OBJS = \
+  $(addprefix $(ESP_SDK_BOOTLOADER_DIR)/,$(ESP_SDK_BOOTLOADER_SRCS:%.c=%.o))
+
+DEPS := $(ESP_SDK_BOOTLOADER_OBJS:%.o=%.d)
+-include $(DEPS)
+
+# Main module dependency. We only build the bootloader.bin from this module.
+$(MODULE).module: $(ESP_SDK_BOOTLOADER_BIN)
+
+OBJ_DEPS += $(CURDIR)/sdkconfig.h
+
+$(ESP_SDK_BOOTLOADER_DIR)/%.o: $(ESP8266_RTOS_SDK_DIR)/%.c $(OBJ_DEPS)
+	$(Q)mkdir -p $(dir $@)
+	$(Q)$(CCACHE) $(CC) \
+	  $(CFLAGS) $(INCLUDES) -MQ '$@' -MD -MP -c -o $@ $(abspath $<)
+
+$(ESP_SDK_BOOTLOADER_DIR):
+	mkdir -p $@
+
+$(ESP_SDK_BOOTLOADER_ELF): $(ESP_SDK_BOOTLOADER_OBJS) \
+    | $(ESP_SDK_BOOTLOADER_DIR)
+	$(Q)$(CC) -o $@ $(LINKFLAGS) -Wl,-Map=$(@:%.elf=%.map)
+
+FLASH_CHIP = esp8266
+ESPTOOL ?= $(RIOTTOOLS)/esptools/esptool.py
+# TODO: These should be exported/configurable from the app side. That would
+# require to export these values.
+FLASH_MODE ?= dout
+FLASH_FREQ ?= 40m
+FLASH_SIZE ?= 4
+
+# We use esptool to extract a version 1 app from the bootloader.elf. This is
+# like the regular objdump binary file but it contains a 16 byte header which
+# specifies the flash size, mode and speed that the ROM bootloader uses to load
+# this second-stage bootloader image.
+$(ESP_SDK_BOOTLOADER_BIN): $(ESP_SDK_BOOTLOADER_ELF)
+	$(Q)$(ESPTOOL) --chip $(FLASH_CHIP) elf2image --flash_mode $(FLASH_MODE) \
+	  --flash_size $(FLASH_SIZE)MB --flash_freq $(FLASH_FREQ) --version 1 -o $@ $<
+	$(Q)mv $(@)0x00000.bin $@

pkg/esp8266_sdk/bootloader/bootloader.inc.mk

@@ -0,0 +1,26 @@
+# Generated by ./update_mk.sh, don't modify directly.
+
+ESP_SDK_BOOTLOADER_SRCS = \
+  components/bootloader/subproject/main/bootloader_start.c \
+  components/bootloader_support/src/bootloader_clock.c \
+  components/bootloader_support/src/bootloader_common.c \
+  components/bootloader_support/src/bootloader_flash.c \
+  components/bootloader_support/src/bootloader_init.c \
+  components/bootloader_support/src/bootloader_random.c \
+  components/bootloader_support/src/bootloader_sha.c \
+  components/bootloader_support/src/bootloader_utility.c \
+  components/bootloader_support/src/efuse.c \
+  components/bootloader_support/src/esp_image_format.c \
+  components/bootloader_support/src/flash_encrypt.c \
+  components/bootloader_support/src/flash_partitions.c \
+  components/bootloader_support/src/flash_qio_mode.c \
+  components/bootloader_support/src/secure_boot.c \
+  components/bootloader_support/src/secure_boot_signatures.c \
+  components/esp8266/source/ets_printf.c \
+  components/log/log.c \
+  components/spi_flash/port/port.c \
+  components/spi_flash/src/spi_flash.c \
+  components/spi_flash/src/spi_flash_raw.c \
+  components/util/src/base64.c \
+  components/util/src/crc.c \
+  #

pkg/esp8266_sdk/bootloader/sdkconfig.h

@@ -0,0 +1,85 @@
+/*
+ * Copyright (C) 2021 iosabi
+ *
+ * This file is subject to the terms and conditions of the GNU Lesser
+ * General Public License v2.1. See the file LICENSE in the top level
+ * directory for more details.
+ */
+
+/**
+ * @ingroup     pkg_esp8266_sdk
+ * @{
+ *
+ * @file
+ * @brief       RIOT-OS modification of the bootloader SDK configuration
+ *
+ * The bootloader build of the ESP8266 SDK needs some settings from the SDK
+ * configuration. These are normally generated by the menuconfig in the vendor
+ * SDK.
+ *
+ * Some of these parameters are configurable by the application. For example,
+ * the UART baudrate used by the console and the verbose level of the
+ * bootloader.
+ *
+ * @author      iosabi <iosabi@protonmail.com>
+ */
+
+#ifndef SDKCONFIG_H
+#define SDKCONFIG_H
+
+#if !DOXYGEN
+
+#include "riotbuild.h"
+
+#include "esp8266_idf_version.h"
+#include "sdkconfig_default.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#if MODULE_ESP_LOG_COLORED
+#define CONFIG_LOG_COLORS   1
+#endif
+
+#ifndef CONFIG_LOG_BOOTLOADER_LEVEL
+/*
+ * SDK Log levels:
+ *
+ *  0 = NONE
+ *  1 = ERROR
+ *  2 = WARN
+ *  3 = INFO
+ *  4 = DEBUG
+ *  5 = VERBOSE
+ */
+#if MODULE_ESP_LOG_STARTUP
+#define CONFIG_LOG_BOOTLOADER_LEVEL     3 /* INFO */
+#else
+#define CONFIG_LOG_BOOTLOADER_LEVEL     0 /* NONE */
+#endif
+#endif
+
+/*
+ * Bootloader output baudrate, defined by the app settings as BAUD or
+ * BOOTLOADER_BAUD.
+ */
+#ifndef CONFIG_CONSOLE_UART_BAUDRATE
+#define CONFIG_CONSOLE_UART_BAUDRATE    (RIOT_BOOTLOADER_BAUD)
+#endif
+
+/*
+ * The makefile FLASH_SIZE value is set in MB, but set as bytes to
+ * CONFIG_SPI_FLASH_SIZE.
+ */
+#ifndef CONFIG_SPI_FLASH_SIZE
+#define CONFIG_SPI_FLASH_SIZE           ((RIOT_FLASH_SIZE) * 1024 * 1024)
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* !DOXYGEN */
+#endif /* SDKCONFIG_H */
+/** @} */

pkg/esp8266_sdk/bootloader/sdkconfig_default.h

@@ -0,0 +1,28 @@
+/*
+ * Generated by ./update_mk.sh, don't modify directly.
+ * Default CONFIG_ parameters from the SDK package.
+ */
+
+#ifndef SDKCONFIG_DEFAULT_H
+#define SDKCONFIG_DEFAULT_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define CONFIG_BOOTLOADER_INIT_SPI_FLASH 1
+#define CONFIG_CONSOLE_UART_NUM 0
+#define CONFIG_CRYSTAL_USED_26MHZ 1
+#define CONFIG_LOG_DEFAULT_LEVEL 3
+#define CONFIG_NEWLIB_STDOUT_LINE_ENDING_CRLF 1
+#define CONFIG_PARTITION_TABLE_OFFSET 0x8000
+#define CONFIG_SPI_FLASH_MODE 0x0
+#define CONFIG_SSL_USING_MBEDTLS 1
+#define CONFIG_TARGET_PLATFORM_ESP8266 1
+#define CONFIG_USING_NEW_ETS_VPRINTF 1
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SDKCONFIG_DEFAULT_H */