Added new functions for allocation and heap checking

- added `allocate_buffer()` function that can be used to allocate large buffers: takes parameters to set preferred ram location, including 32bit accessible RAM on ESP32. Returns null if heap runs low or switches to PSRAM
- getFreeHeapSize() and getContiguousFreeHeap() helper functions for all platforms to correctly report free useable heap
- updated some constants
- updated segment data allocation to free the data if it is large

Author: DedeHai

wled00/FX.h

@@ -98,7 +98,7 @@ extern byte realtimeMode;           // used in getMappedPixelIndex()
   #define MAX_SEGMENT_DATA  (MAX_NUM_SEGMENTS*1024)  // 20k by default (S2 is short on free RAM)
 #else
   #define MAX_NUM_SEGMENTS  32  // warning: going beyond 32 may consume too much RAM for stable operation
-  #define MAX_SEGMENT_DATA  (MAX_NUM_SEGMENTS*2560) // 80k by default
+  #define MAX_SEGMENT_DATA  (MAX_NUM_SEGMENTS*1920) // 60k by default
 #endif
 
 /* How much data bytes each segment should max allocate to leave enough space for other segments,
@@ -599,12 +599,8 @@ class Segment {
     , _t(nullptr)
     {
       DEBUGFX_PRINTF_P(PSTR("-- Creating segment: %p [%d,%d:%d,%d]\n"), this, (int)start, (int)stop, (int)startY, (int)stopY);
-      // allocate render buffer (always entire segment), prefer PSRAM if DRAM is running low. Note: impact on FPS with PSRAM buffer is low (~2% with QSPI PSRAM)
-      #ifdef CONFIG_IDF_TARGET_ESP32
-      pixels = static_cast<uint32_t*>(pixelbuffer_malloc(sizeof(uint32_t) * length()));
-      #else
-      pixels = static_cast<uint32_t*>(p_calloc(length(), sizeof(uint32_t))); // prefer PSRAM. note: error handling is also done in isActive()
-      #endif
+      // allocate render buffer (always entire segment), prefer PSRAM if DRAM is running low. Note: impact on FPS with PSRAM buffer is low (<2% with QSPI PSRAM)
+      pixels = static_cast<uint32_t*>(allocate_buffer(length() * sizeof(uint32_t), BFRALLOC_PREFER_PSRAM | BFRALLOC_NOBYTEACCESS | BFRALLOC_CLEAR));
       if (!pixels) {
         DEBUGFX_PRINTLN(F("!!! Not enough RAM for pixel buffer !!!"));
         extern byte errorFlag;

wled00/FX_fcn.cpp

@@ -69,14 +69,9 @@ Segment::Segment(const Segment &orig) {
   if (orig.name) { name = static_cast<char*>(d_malloc(strlen(orig.name)+1)); if (name) strcpy(name, orig.name); }
   if (orig.data) { if (allocateData(orig._dataLen)) memcpy(data, orig.data, orig._dataLen); }
   if (orig.pixels) {
-    pixels = static_cast<uint32_t*>(d_malloc(sizeof(uint32_t) * orig.length()));
-
-//  pixels = static_cast<uint32_t*>(heap_caps_malloc(orig.length()* sizeof(uint32_t), MALLOC_CAP_32BIT | MALLOC_CAP_INTERNAL)); // use this for ESP32
-//pixels = static_cast<uint32_t*>(heap_caps_malloc(sizeof(uint32_t) * orig.length(), MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT));
-//pixels = static_cast<uint32_t*>(heap_caps_malloc(sizeof(uint32_t) * orig.length(), MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT));
-
-
-    if (pixels) memcpy(pixels, orig.pixels, sizeof(uint32_t) * orig.length());
+    // allocate pixel buffer: prefer PSRAM if DRAM is running low
+    pixels = static_cast<uint32_t*>(allocate_buffer(orig.length() * sizeof(uint32_t), BFRALLOC_PREFER_PSRAM | BFRALLOC_NOBYTEACCESS));
+    if (pixels) memcpy(pixels, orig.pixels, orig.length() * sizeof(uint32_t));
     else {
       DEBUG_PRINTLN(F("!!! Not enough RAM for pixel buffer !!!"));
       errorFlag = ERR_NORAM_PX;
@@ -116,12 +111,9 @@ Segment& Segment::operator= (const Segment &orig) {
     if (orig.name) { name = static_cast<char*>(d_malloc(strlen(orig.name)+1)); if (name) strcpy(name, orig.name); }
     if (orig.data) { if (allocateData(orig._dataLen)) memcpy(data, orig.data, orig._dataLen); }
     if (orig.pixels) {
-      pixels = static_cast<uint32_t*>(d_malloc(sizeof(uint32_t) * orig.length()));
-      //TODO: also need to put this in 32bit memory on ESP32, maybe make that a function...
-      //pixels = static_cast<uint32_t*>(heap_caps_malloc(sizeof(uint32_t) * orig.length(), MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT));
-      //pixels = static_cast<uint32_t*>(heap_caps_malloc(sizeof(uint32_t) * orig.length(), MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT));
-      
-      if (pixels) memcpy(pixels, orig.pixels, sizeof(uint32_t) * orig.length());
+      // allocate pixel buffer: prefer PSRAM if DRAM is running low
+      pixels = static_cast<uint32_t*>(allocate_buffer(orig.length() * sizeof(uint32_t), BFRALLOC_PREFER_PSRAM | BFRALLOC_NOBYTEACCESS));
+      if (pixels) memcpy(pixels, orig.pixels, orig.length() * sizeof(uint32_t));
       else {
         DEBUG_PRINTLN(F("!!! Not enough RAM for pixel buffer !!!"));
         errorFlag = ERR_NORAM_PX;
@@ -156,50 +148,41 @@ bool Segment::allocateData(size_t len) {
   if (len == 0) return false;    // nothing to do
   if (data && _dataLen >= len) { // already allocated enough (reduce fragmentation)
     if (call == 0) {
-      if(checkHeapHealth()) {
+      if(_dataLen < FAIR_DATA_PER_SEG) { // segment data is small
         //DEBUG_PRINTF_P(PSTR("--   Clearing data (%d): %p\n"), len, this);
         memset(data, 0, len);  // erase buffer if called during effect initialisation
         return true; // no need to reallocate
       }
-      else {
-        d_free(data); // free data and try to allocate again
-        data = nullptr;
-        Segment::addUsedSegmentData(-_dataLen); // subtract buffer size
-      }
     }
     else
       return true;
   }
   //DEBUG_PRINTF_P(PSTR("--   Allocating data (%d): %p\n"), len, this);
-  if (Segment::getUsedSegmentData() + len - _dataLen > MAX_SEGMENT_DATA) {
-    // not enough memory
-    DEBUG_PRINTF_P(PSTR("!!! Not enough RAM: %d/%d !!!\n"), len, Segment::getUsedSegmentData());
-    errorFlag = ERR_NORAM;
-    return false;
+  // limit to MAX_SEGMENT_DATA if there is no PSRAM, otherwise prefer functionality over speed
+  #if defined(ARDUINO_ARCH_ESP32)
+  if(!(psramFound() && psramSafe))
+  #endif
+  {
+    if (Segment::getUsedSegmentData() + len - _dataLen > MAX_SEGMENT_DATA) {
+      // not enough memory
+      DEBUG_PRINTF_P(PSTR("!!! Not enough RAM: %d/%d !!!\n"), len, Segment::getUsedSegmentData());
+      errorFlag = ERR_NORAM;
+      return false;
+    }
   }
   // prefer DRAM over PSRAM for speed
   if (data) {
-    data = (byte*)d_realloc_malloc(data, len); // realloc with malloc fallback
-    if (data == nullptr) { // allocation failed
-      Segment::addUsedSegmentData(-_dataLen); // subtract original buffer size
-      _dataLen = 0;   // reset data length
-      return false;
-    }
+    d_free(data); // free data and try to allocate again (segment buffer may be blocking contiguous heap)
+    Segment::addUsedSegmentData(-_dataLen); // subtract buffer size
   }
-  else data = (byte*)d_malloc(len);
+
+  data = static_cast<byte*>(allocate_buffer(len, BFRALLOC_PREFER_DRAM | BFRALLOC_CLEAR));
 
   if (data) {
-    if(!checkHeapHealth()) {
-      d_free(data);
-      data = nullptr;
-    }
-    else {
-      memset(data, 0, len);  // erase buffer
-      Segment::addUsedSegmentData(len);
-      _dataLen = len;
-      //DEBUG_PRINTF_P(PSTR("---  Allocated data (%p): %d/%d -> %p\n"), this, len, Segment::getUsedSegmentData(), data);
-      return true;
-    }
+    Segment::addUsedSegmentData(len);
+    _dataLen = len;
+    //DEBUG_PRINTF_P(PSTR("---  Allocated data (%p): %d/%d -> %p\n"), this, len, Segment::getUsedSegmentData(), data);
+    return true;
   }
   // allocation failed
   DEBUG_PRINTLN(F("!!! Allocation failed. !!!"));
@@ -486,16 +469,7 @@ void Segment::setGeometry(uint16_t i1, uint16_t i2, uint8_t grp, uint8_t spc, ui
   // allocate FX render buffer
   if (length() != oldLength) {
     if (pixels) free(pixels); // note: using realloc can block larger heap segments
-    #ifdef ARDUINO_ARCH_ESP32
-    pixels = static_cast<uint32_t*>(pixelbuffer_malloc(izeof(uint32_t) * length());
-    #else
-    pixels = static_cast<uint32_t*>(p_malloc(sizeof(uint32_t) * length()));
-    #endif
-
-    if(!checkHeapHealth()) {
-      d_free(pixels);
-      pixels = nullptr;
-    }
+    pixels = static_cast<uint32_t*>(allocate_buffer(length() * sizeof(uint32_t), BFRALLOC_PREFER_PSRAM | BFRALLOC_NOBYTEACCESS));
     if (!pixels) {
       DEBUG_PRINTLN(F("!!! Not enough RAM for pixel buffer !!!"));
       errorFlag = ERR_NORAM_PX;
@@ -1250,14 +1224,8 @@ void WS2812FX::finalizeInit() {
 
   // allocate frame buffer after matrix has been set up (gaps!)
   if (_pixels) d_free(_pixels);
-#ifdef ARDUINO_ARCH_ESP32
-  _pixels = static_cast<uint32_t*>(pixelbuffer_malloc(getLengthTotal() * sizeof(uint32_t), true)); // use 32bit RAM (IRAM) or PSRAM on ESP32
-#elif !defined(ESP8266)
-  // use PSRAM on S2 and S3 if available (C3 defaults to DRAM). Note: there is no measurable perfomance impact between PSRAM and DRAM on S2/S3 with QSPI PSRAM
-  _pixels = static_cast<uint32_t*>(heap_caps_malloc_prefer(size, 2, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT, MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT)); // prefer PSRAM if it exists
-#else
-  _pixels = static_cast<uint32_t*>(malloc(getLengthTotal() * sizeof(uint32_t))); // ESP8266 does not support advanced allocation API
-#endif
+  // use PSRAM if available: there is no measurable perfomance impact between PSRAM and DRAM on S2/S3 with QSPI PSRAM for this buffer
+  _pixels = static_cast<uint32_t*>(allocate_buffer(getLengthTotal() * sizeof(uint32_t), BFRALLOC_ENFORCE_PSRAM | BFRALLOC_NOBYTEACCESS | BFRALLOC_CLEAR));
   DEBUG_PRINTF_P(PSTR("strip buffer size: %uB\n"), getLengthTotal() * sizeof(uint32_t));
   DEBUG_PRINTF_P(PSTR("Heap after strip init: %uB\n"), getFreeHeapSize());
 }

wled00/const.h

@@ -547,16 +547,19 @@ static_assert(WLED_MAX_BUSSES <= 32, "WLED_MAX_BUSSES exceeds hard limit");
 #endif
 
 // minimum heap size required to process web requests: try to keep free heap above this value
-#define MIN_HEAP_SIZE (12*1024)
-
+#ifdef ESP8266
+  #define MIN_HEAP_SIZE (8*1024)
+#else
+  #define MIN_HEAP_SIZE (12*1024)
+#endif
 // threshold for PSRAM use: if heap is running low, requests above PSRAM_THRESHOLD will be allocated in PSRAM
 // if heap is plenty, requests below PSRAM_THRESHOLD will be allocated in DRAM for speed
 #if defined(CONFIG_IDF_TARGET_ESP32S3)
-  #define PSRAM_THRESHOLD 8192
+  #define PSRAM_THRESHOLD 5120
 #elif defined(CONFIG_IDF_TARGET_ESP32)
   #define PSRAM_THRESHOLD 4096
 #else
-  #define PSRAM_THRESHOLD 2048 // S2 does not have a lot of RAM, C3 and ESP8266 do not support PSRAM: the value is not used
+  #define PSRAM_THRESHOLD 1024 // S2 does not have a lot of RAM. C3 and ESP8266 do not support PSRAM: the value is not used
 #endif
 
 // Web server limits

wled00/fcn_declare.h

@@ -582,12 +582,20 @@ extern "C" {
 #define d_realloc_malloc realloc_malloc
 #define d_free free
 #endif
-bool checkHeapHealth(unsigned minFreeBlockSize = MIN_HEAP_SIZE); // checks heap fragmentation: returns true if contiguous free memory is larger than minFreeBlockSize
 #ifndef ESP8266
-inline unsigned getFreeHeapSize() { return heap_caps_get_free_size(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); } // returns free heap (ESP.getFreeHeap() can include other memory types)
+inline size_t getFreeHeapSize() { return heap_caps_get_free_size(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); } // returns free heap (ESP.getFreeHeap() can include other memory types)
+inline size_t getContiguousFreeHeap() { return heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); } // returns largest contiguous free block
 #else
-inline unsigned getFreeHeapSize() { return ESP.getFreeHeap(); } // returns free heap
+inline size_t getFreeHeapSize() { return ESP.getFreeHeap(); } // returns free heap
+inline size_t getContiguousFreeHeap() { return ESP.getMaxFreeBlockSize(); } // returns largest contiguous free block
 #endif
+#define  BFRALLOC_NOBYTEACCESS    (1 << 0) // ESP32 has 32bit accessible DRAM (usually ~50kB free) that must not be byte-accessed
+#define  BFRALLOC_PREFER_DRAM     (1 << 1) // prefer DRAM over PSRAM
+#define  BFRALLOC_ENFORCE_DRAM    (1 << 2) // use DRAM only, no PSRAM
+#define  BFRALLOC_PREFER_PSRAM    (1 << 3) // prefer PSRAM over DRAM
+#define  BFRALLOC_ENFORCE_PSRAM   (1 << 4) // use PSRAM if available, otherwise fall back to DRAM
+#define  BFRALLOC_CLEAR           (1 << 5) // clear allocated buffer after allocation
+void *allocate_buffer(size_t size, uint32_t type);
 
 // RAII guard class for the JSON Buffer lock
 // Modeled after std::lock_guard

wled00/json.cpp

@@ -812,7 +812,7 @@ void serializeInfo(JsonObject root)
   root[F("clock")] = ESP.getCpuFreqMHz();
   root[F("flash")] = (ESP.getFlashChipSize()/1024)/1024;
   #ifdef WLED_DEBUG
-  root[F("maxalloc")] = ESP.getMaxAllocHeap();
+  root[F("maxalloc")] = getContiguousFreeHeap();
   root[F("resetReason0")] = (int)rtc_get_reset_reason(0);
   root[F("resetReason1")] = (int)rtc_get_reset_reason(1);
   #endif
@@ -823,7 +823,7 @@ void serializeInfo(JsonObject root)
   root[F("clock")] = ESP.getCpuFreqMHz();
   root[F("flash")] = (ESP.getFlashChipSize()/1024)/1024;
   #ifdef WLED_DEBUG
-  root[F("maxalloc")] = ESP.getMaxFreeBlockSize();
+  root[F("maxalloc")] = getContiguousFreeHeap();
   root[F("resetReason")] = (int)ESP.getResetInfoPtr()->reason;
   #endif
   root[F("lwip")] = LWIP_VERSION_MAJOR;