Replace metadata header approach with ESP-IDF custom description section

Co-authored-by: netmindz <442066+netmindz@users.noreply.github.com>

Author: Copilot

pio-scripts/output_bins.py

@@ -18,37 +18,6 @@ def _create_dirs(dirs=["map", "release", "firmware"]):
     for d in dirs:
         os.makedirs(os.path.join(OUTPUT_DIR, d), exist_ok=True)
 
-def add_metadata_header(binary_file, release_name):
-    """Add WLED release metadata header to binary file"""
-    # Metadata format: "WLED_META:" + release_name + null terminator + padding to 64 bytes + original binary
-    header_prefix = b"WLED_META:"
-    release_bytes = release_name.encode('utf-8')
-    
-    # Ensure total header is exactly 64 bytes for alignment
-    header_size = 64
-    header_data_size = len(header_prefix) + len(release_bytes) + 1  # +1 for null terminator
-    
-    if header_data_size > header_size:
-        print(f"Warning: Release name '{release_name}' too long, truncating")
-        max_release_len = header_size - len(header_prefix) - 1
-        release_bytes = release_bytes[:max_release_len]
-        header_data_size = len(header_prefix) + len(release_bytes) + 1
-    
-    # Create header with padding
-    header = header_prefix + release_bytes + b'\0'
-    header += b'\xFF' * (header_size - header_data_size)  # Pad with 0xFF (erased flash pattern)
-    
-    # Read original binary
-    with open(binary_file, 'rb') as f:
-        original_data = f.read()
-    
-    # Write header + original binary
-    with open(binary_file, 'wb') as f:
-        f.write(header)
-        f.write(original_data)
-    
-    print(f"Added WLED metadata header with release name '{release_name}' to {binary_file}")
-
 def create_release(source):
     release_name_def = _get_cpp_define_value(env, "WLED_RELEASE_NAME")
     if release_name_def:
@@ -58,10 +27,6 @@ def create_release(source):
         release_gz_file = release_file + ".gz"
         print(f"Copying {source} to {release_file}")
         shutil.copy(source, release_file)
-        
-        # Add metadata header to the release binary
-        add_metadata_header(release_file, release_name)
-        
         bin_gzip(release_file, release_gz_file)
     else:
         variant = env["PIOENV"]

wled00/ota_release_check.cpp

@@ -1,39 +1,75 @@
 #include "ota_release_check.h"
 #include "wled.h"
 
-bool extractMetadataHeader(uint8_t* binaryData, size_t dataSize, char* extractedRelease, size_t* actualBinarySize) {
-  if (!binaryData || !extractedRelease || !actualBinarySize || dataSize < WLED_META_HEADER_SIZE) {
-    *actualBinarySize = dataSize;
-    return false;
-  }
+#ifdef ESP32
+#include <esp_app_format.h>
+#include <esp_ota_ops.h>
+#endif
 
-  // Check if the binary starts with our metadata header
-  if (memcmp(binaryData, WLED_META_PREFIX, strlen(WLED_META_PREFIX)) != 0) {
-    // No metadata header found, this is a legacy binary
-    *actualBinarySize = dataSize;
-    DEBUG_PRINTLN(F("No WLED metadata header found - legacy binary"));
-    return false;
-  }
+bool extractReleaseFromCustomDesc(const uint8_t* binaryData, size_t dataSize, char* extractedRelease) {
+    if (!binaryData || !extractedRelease || dataSize < sizeof(esp_image_header_t)) {
+        return false;
+    }
 
-  DEBUG_PRINTLN(F("Found WLED metadata header"));
+#ifdef ESP32
+    // Look for ESP32 image header to find the custom description section
+    const esp_image_header_t* header = (const esp_image_header_t*)binaryData;
+    
+    // Validate ESP32 image header
+    if (header->magic != ESP_IMAGE_HEADER_MAGIC) {
+        DEBUG_PRINTLN(F("Not a valid ESP32 image - missing magic header"));
+        return false;
+    }
 
-  // Extract release name from header
-  const char* releaseStart = (const char*)(binaryData + strlen(WLED_META_PREFIX));
-  size_t maxReleaseLen = WLED_META_HEADER_SIZE - strlen(WLED_META_PREFIX) - 1;
-  
-  // Copy release name (it should be null-terminated within the header)
-  strncpy(extractedRelease, releaseStart, maxReleaseLen);
-  extractedRelease[maxReleaseLen] = '\0'; // Ensure null termination
+    // The custom description section is located at a fixed offset after the image header
+    // ESP-IDF places custom description at offset 0x20 in the binary for ESP32
+    const size_t custom_desc_offset = 0x20;
+    
+    if (dataSize < custom_desc_offset + sizeof(wled_custom_desc_t)) {
+        DEBUG_PRINTLN(F("Binary too small to contain custom description"));
+        return false;
+    }
 
-  // Remove metadata header by shifting binary data
-  size_t firmwareSize = dataSize - WLED_META_HEADER_SIZE;
-  memmove(binaryData, binaryData + WLED_META_HEADER_SIZE, firmwareSize);
-  *actualBinarySize = firmwareSize;
+    const wled_custom_desc_t* custom_desc = (const wled_custom_desc_t*)(binaryData + custom_desc_offset);
+    
+    // Validate magic number and version
+    if (custom_desc->magic != WLED_CUSTOM_DESC_MAGIC) {
+        DEBUG_PRINTLN(F("No WLED custom description found - no magic number"));
+        return false;
+    }
 
-  DEBUG_PRINTF_P(PSTR("Extracted release name from metadata: '%s', firmware size: %zu bytes\n"), 
-                 extractedRelease, firmwareSize);
+    if (custom_desc->version != WLED_CUSTOM_DESC_VERSION) {
+        DEBUG_PRINTF_P(PSTR("Unsupported custom description version: %u\n"), custom_desc->version);
+        return false;
+    }
 
-  return true;
+    // Validate simple hash checksum (using the same simple hash as in wled_custom_desc.cpp)
+    auto simple_hash = [](const char* str) -> uint32_t {
+        uint32_t hash = 5381;
+        for (int i = 0; str[i]; ++i) {
+            hash = ((hash << 5) + hash) + str[i];
+        }
+        return hash;
+    };
+    
+    uint32_t expected_hash = simple_hash(custom_desc->release_name);
+    if (custom_desc->crc32 != expected_hash) {
+        DEBUG_PRINTF_P(PSTR("Custom description hash mismatch: expected 0x%08x, got 0x%08x\n"), 
+                      expected_hash, custom_desc->crc32);
+        return false;
+    }
+
+    // Extract release name (ensure null termination)
+    strncpy(extractedRelease, custom_desc->release_name, WLED_RELEASE_NAME_MAX_LEN - 1);
+    extractedRelease[WLED_RELEASE_NAME_MAX_LEN - 1] = '\0';
+    
+    DEBUG_PRINTF_P(PSTR("Extracted release name from custom description: '%s'\n"), extractedRelease);
+    return true;
+#else
+    // ESP8266 doesn't use ESP-IDF format, so we can't extract custom description
+    DEBUG_PRINTLN(F("ESP8266 binaries do not support custom description extraction"));
+    return false;
+#endif
 }
 
 bool validateReleaseCompatibility(const char* extractedRelease) {
@@ -50,37 +86,32 @@ bool validateReleaseCompatibility(const char* extractedRelease) {
   return match;
 }
 
-bool shouldAllowOTA(uint8_t* binaryData, size_t dataSize, bool ignoreReleaseCheck, char* errorMessage, size_t* actualBinarySize) {
+bool shouldAllowOTA(const uint8_t* binaryData, size_t dataSize, bool ignoreReleaseCheck, char* errorMessage) {
   // Clear error message
   if (errorMessage) {
     errorMessage[0] = '\0';
   }
-  
-  // Initialize actual binary size to full size by default
-  if (actualBinarySize) {
-    *actualBinarySize = dataSize;
-  }
 
   // If user chose to ignore release check, allow OTA
   if (ignoreReleaseCheck) {
     DEBUG_PRINTLN(F("OTA release check bypassed by user"));
-    // Still need to extract metadata header if present to get clean binary
-    char dummyRelease[64];
-    extractMetadataHeader(binaryData, dataSize, dummyRelease, actualBinarySize);
     return true;
   }
 
-  // Try to extract metadata header
-  char extractedRelease[64];
-  bool hasMetadata = extractMetadataHeader(binaryData, dataSize, extractedRelease, actualBinarySize);
+  // Try to extract release name from custom description section
+  char extractedRelease[WLED_RELEASE_NAME_MAX_LEN];
+  bool hasCustomDesc = extractReleaseFromCustomDesc(binaryData, dataSize, extractedRelease);
 
-  if (!hasMetadata) {
-    // No metadata header - this could be a legacy binary or a binary without our metadata
-    // We cannot determine compatibility for such binaries
+  if (!hasCustomDesc) {
+    // No custom description - this could be a legacy binary or ESP8266 binary
     if (errorMessage) {
+#ifdef ESP32
       strcpy(errorMessage, "Binary has no release compatibility metadata. Check 'Ignore release name check' to proceed.");
+#else
+      strcpy(errorMessage, "ESP8266 binaries do not support release checking. Check 'Ignore release name check' to proceed.");
+#endif
     }
-    DEBUG_PRINTLN(F("OTA blocked: No metadata header found"));
+    DEBUG_PRINTLN(F("OTA blocked: No custom description found"));
     return false;
   }
 

wled00/ota_release_check.h

@@ -2,24 +2,41 @@
 #define WLED_OTA_RELEASE_CHECK_H
 
 /*
- * OTA Release Compatibility Checking with Metadata Headers
- * Functions to extract and validate release names from uploaded binary files with metadata headers
+ * OTA Release Compatibility Checking using ESP-IDF Custom Description Section
+ * Functions to extract and validate release names from uploaded binary files using embedded metadata
  */
 
 #include <Arduino.h>
 
-#define WLED_META_HEADER_SIZE 64
-#define WLED_META_PREFIX "WLED_META:"
+#ifdef ESP32
+#include <esp_app_format.h>
+#endif
+
+#define WLED_CUSTOM_DESC_MAGIC 0x57535453  // "WSTS" (WLED System Tag Structure)
+#define WLED_CUSTOM_DESC_VERSION 1
+#define WLED_RELEASE_NAME_MAX_LEN 48
+
+/**
+ * WLED Custom Description Structure
+ * This structure is embedded in the .rodata_custom_desc section at a fixed offset
+ * in ESP32 binaries, allowing extraction without modifying the binary format
+ */
+typedef struct {
+    uint32_t magic;               // Magic number to identify WLED custom description
+    uint32_t version;             // Structure version for future compatibility
+    char release_name[WLED_RELEASE_NAME_MAX_LEN]; // Release name (null-terminated)
+    uint32_t crc32;              // CRC32 of the above fields for integrity check
+    uint8_t reserved[12];        // Reserved for future use, must be zero
+} __attribute__((packed)) wled_custom_desc_t;
 
 /**
- * Extract and remove metadata header from binary data
- * @param binaryData Pointer to binary file data (will be modified)
+ * Extract release name from binary using ESP-IDF custom description section
+ * @param binaryData Pointer to binary file data
  * @param dataSize Size of binary data in bytes
- * @param extractedRelease Buffer to store extracted release name (should be at least 64 bytes)
- * @param actualBinarySize Pointer to store the size of actual firmware binary (without header)
- * @return true if metadata header was found and extracted, false if no metadata header present
+ * @param extractedRelease Buffer to store extracted release name (should be at least WLED_RELEASE_NAME_MAX_LEN bytes)
+ * @return true if release name was found and extracted, false otherwise
  */
-bool extractMetadataHeader(uint8_t* binaryData, size_t dataSize, char* extractedRelease, size_t* actualBinarySize);
+bool extractReleaseFromCustomDesc(const uint8_t* binaryData, size_t dataSize, char* extractedRelease);
 
 /**
  * Validate if extracted release name matches current release
@@ -29,14 +46,13 @@ bool extractMetadataHeader(uint8_t* binaryData, size_t dataSize, char* extracted
 bool validateReleaseCompatibility(const char* extractedRelease);
 
 /**
- * Check if OTA should be allowed based on release compatibility using metadata headers
- * @param binaryData Pointer to binary file data (will be modified if metadata header present)
+ * Check if OTA should be allowed based on release compatibility using custom description
+ * @param binaryData Pointer to binary file data (not modified)
  * @param dataSize Size of binary data in bytes
  * @param ignoreReleaseCheck If true, skip release validation
  * @param errorMessage Buffer to store error message if validation fails (should be at least 128 bytes)
- * @param actualBinarySize Pointer to store the size of actual firmware binary (without header)
  * @return true if OTA should proceed, false if it should be blocked
  */
-bool shouldAllowOTA(uint8_t* binaryData, size_t dataSize, bool ignoreReleaseCheck, char* errorMessage, size_t* actualBinarySize);
+bool shouldAllowOTA(const uint8_t* binaryData, size_t dataSize, bool ignoreReleaseCheck, char* errorMessage);
 
 #endif // WLED_OTA_RELEASE_CHECK_H

wled00/wled_custom_desc.cpp

@@ -0,0 +1,25 @@
+#include "ota_release_check.h"
+#include "wled.h"
+
+#ifdef ESP32
+
+// Simple hash function for validation (compile-time friendly)
+constexpr uint32_t simple_hash(const char* str) {
+    uint32_t hash = 5381;
+    for (int i = 0; str[i]; ++i) {
+        hash = ((hash << 5) + hash) + str[i];
+    }
+    return hash;
+}
+
+// Create the custom description structure with current release name
+// This will be embedded at a fixed offset in the ESP32 binary
+const wled_custom_desc_t __attribute__((section(".rodata_custom_desc"))) wled_custom_description = {
+    .magic = WLED_CUSTOM_DESC_MAGIC,
+    .version = WLED_CUSTOM_DESC_VERSION,
+    .release_name = WLED_RELEASE_NAME,
+    .crc32 = simple_hash(WLED_RELEASE_NAME),  // Use simple hash for validation
+    .reserved = {0}
+};
+
+#endif // ESP32

wled00/wled_server.cpp

@@ -480,8 +480,7 @@ void initServer()
       bool ignoreRelease = request->hasParam("ignoreRelease", true);
 
       char errorMessage[128];
-      size_t actualFirmwareSize;
-      releaseCheckPassed = shouldAllowOTA(firstChunkBuffer, len, ignoreRelease, errorMessage, &actualFirmwareSize);
+      releaseCheckPassed = shouldAllowOTA(firstChunkBuffer, len, ignoreRelease, errorMessage);
 
       if (!releaseCheckPassed) {
         DEBUG_PRINTF_P(PSTR("OTA blocked: %s\n"), errorMessage);
@@ -497,15 +496,8 @@ void initServer()
 
       DEBUG_PRINTLN(F("Release check passed, starting OTA update"));
 
-      // Calculate expected firmware size (if metadata header was present, actualFirmwareSize will be adjusted)
-      // For final size calculation, we need to account for all chunks
+      // Use full content length as firmware size (no binary modification with custom description approach)
       expectedFirmwareSize = request->contentLength();
-      if (actualFirmwareSize < len) {
-        // Metadata header was removed, adjust expected size
-        size_t headerSize = len - actualFirmwareSize;
-        expectedFirmwareSize -= headerSize;
-        DEBUG_PRINTF_P(PSTR("Metadata header removed, firmware size: %zu bytes\n"), expectedFirmwareSize);
-      }
 
       // Start the actual OTA update
       strip.suspend();
@@ -530,12 +522,12 @@ void initServer()
         return;
       }
 
-      // Write the processed first chunk (with metadata header removed if present)
-      if (actualFirmwareSize > 0 && !Update.hasError()) {
-        if (Update.write(firstChunkBuffer, actualFirmwareSize) != actualFirmwareSize) {
+      // Write the first chunk of firmware data
+      if (len > 0 && !Update.hasError()) {
+        if (Update.write(firstChunkBuffer, len) != len) {
           DEBUG_PRINTF_P(PSTR("OTA write failed on first chunk: %s\n"), Update.getErrorString().c_str());
         } else {
-          totalBytesWritten += actualFirmwareSize;
+          totalBytesWritten += len;
         }
       }