Branch flash pr with large eeprom (#177)

* save WIP on flash api + header reorder

* reduce diffs to master

* added support for RP2040 (Raspberry Pi Pico)

* support DPT9.009 (airflow) and DPT9.029 (absolute humidity)

* added support for RP2040 (Raspberry Pi Pico) (#145)

* worked on flash implementation

* worked on flash implementation

* added malloc for _EraseBlockBuffer, fixed some bugs

* fixed memoryread and crash while loading KOs (+debugstuff)

* some fixes and debugs

* align to pagesize

* clean up debug stuff, comments ...

* added support for both Eeprom and Flash (NvMemoryType) plattforms.

* changed memoryReadIndicationP to memoryReadIndication
added stdlib and defines

* fixed std::min

* another try for fixing the min problem

* rolled back linux plattform to eeprom

* comments only, hints for plattforms

* bugfix when calculating memorywrites over multiple blocks by mumpf

* added support for EEPROM-Emulation / RAM-buffered Flash
improvements in RP2040 plattform

* fixed typo in KNX_FLASH_OFFSET

* - writebuffersize clarified (PR discussion)
- changed alignment from flashpagesize to 32bit
- added override modifier (PR discussion)
- changed comment regarding flash/eeprom functions for plattforms to override/implement (PR discussion)

* resolved CodeFactor issue

Co-authored-by: Thomas Kunze <thomas.kunze@gmx.com>

src/knx/bau_systemB.cpp

@@ -112,9 +112,14 @@ void BauSystemB::memoryWriteIndication(Priority priority, HopCountType hopType,
     uint16_t memoryAddress, uint8_t * data)
 {
     _memory.writeMemory(memoryAddress, number, data);
-
     if (_deviceObj.verifyMode())
-        memoryReadIndication(priority, hopType, asap, secCtrl, number, memoryAddress);
+        memoryReadIndication(priority, hopType, asap, secCtrl, number, memoryAddress, data);
+}
+
+void BauSystemB::memoryReadIndication(Priority priority, HopCountType hopType, uint16_t asap, const SecurityControl &secCtrl, uint8_t number,
+    uint16_t memoryAddress, uint8_t * data)
+{
+    applicationLayer().memoryReadResponse(AckRequested, priority, hopType, asap, secCtrl, number, memoryAddress, data);
 }
 
 void BauSystemB::memoryReadIndication(Priority priority, HopCountType hopType, uint16_t asap, const SecurityControl &secCtrl, uint8_t number,

src/knx/bau_systemB.h

@@ -48,6 +48,8 @@ class BauSystemB : protected BusAccessUnit
                                uint16_t memoryAddress, uint8_t* data) override;
     void memoryReadIndication(Priority priority, HopCountType hopType, uint16_t asap, const SecurityControl &secCtrl, uint8_t number,
                               uint16_t memoryAddress) override;
+    void memoryReadIndication(Priority priority, HopCountType hopType, uint16_t asap, const SecurityControl &secCtrl, uint8_t number,
+                              uint16_t memoryAddress, uint8_t * data);
     void memoryExtWriteIndication(Priority priority, HopCountType hopType, uint16_t asap, const SecurityControl &secCtrl, uint8_t number,
                                   uint32_t memoryAddress, uint8_t* data) override;
     void memoryExtReadIndication(Priority priority, HopCountType hopType, uint16_t asap, const SecurityControl &secCtrl, uint8_t number,

src/knx/memory.cpp

@@ -1,29 +1,36 @@
 #include "memory.h"
+
 #include <string.h>
+
 #include "bits.h"
 
 Memory::Memory(Platform& platform, DeviceObject& deviceObject)
     : _platform(platform), _deviceObject(deviceObject)
-{
-}
+{}
+
+Memory::~Memory()
+{}
 
 void Memory::readMemory()
 {
     println("readMemory");
-    if (_data != nullptr)
-        return;
 
-    uint16_t flashSize = KNX_FLASH_SIZE;
-    _data = _platform.getEepromBuffer(flashSize);
+    uint8_t* flashStart = _platform.getNonVolatileMemoryStart();
+    size_t flashSize = _platform.getNonVolatileMemorySize();
+    if (flashStart == nullptr)
+    {
+        println("no user flash available;");
+        return;
+    }
 
-    printHex("RESTORED ", _data, _metadataSize);
+    printHex("RESTORED ", flashStart, _metadataSize);
 
     uint16_t metadataBlockSize = alignToPageSize(_metadataSize);
 
-    _freeList = new MemoryBlock(_data + metadataBlockSize, flashSize - metadataBlockSize);
+    _freeList = new MemoryBlock(flashStart + metadataBlockSize, flashSize - metadataBlockSize);
 
     uint16_t manufacturerId = 0;
-    const uint8_t* buffer = popWord(manufacturerId, _data);
+    const uint8_t* buffer = popWord(manufacturerId, flashStart);
 
     uint8_t hardwareType[LEN_HARDWARE_TYPE] = {0};
     buffer = popByteArray(hardwareType, LEN_HARDWARE_TYPE, buffer);
@@ -58,7 +65,7 @@ void Memory::readMemory()
     println(_saveCount);
     for (int i = 0; i < _saveCount; i++)
     {
-        println(_data - buffer);
+        println(flashStart - buffer);
         println(".");
         buffer = _saveRestores[i]->restore(buffer);
     }
@@ -67,7 +74,7 @@ void Memory::readMemory()
     println(_tableObjCount);
     for (int i = 0; i < _tableObjCount; i++)
     {
-        println(_data - buffer);
+        println(flashStart - buffer);
         println(".");
         buffer = _tableObjects[i]->restore(buffer);
         uint16_t memorySize = 0;
@@ -84,29 +91,37 @@ void Memory::readMemory()
 
 void Memory::writeMemory()
 {
-    uint8_t* buffer = _data;
-    buffer = pushWord(_deviceObject.manufacturerId(), buffer);
-    buffer = pushByteArray(_deviceObject.hardwareType(), LEN_HARDWARE_TYPE, buffer);
-    buffer = pushWord(_deviceObject.version(), buffer);
+    // first get the necessary size of the writeBuffer
+    size_t writeBufferSize = _metadataSize;
+    for (int i = 0; i < _saveCount; i++)
+        writeBufferSize = MAX(writeBufferSize, _saveRestores[i]->saveSize());
+
+    for (int i = 0; i < _tableObjCount; i++)
+        writeBufferSize = MAX(writeBufferSize, _tableObjects[i]->saveSize() + 2 /*for memory pos*/);
+
+    uint8_t buffer[writeBufferSize];
+    uint32_t flashPos = 0;
+    uint8_t* bufferPos = buffer;
+
+    bufferPos = pushWord(_deviceObject.manufacturerId(), bufferPos);
+    bufferPos = pushByteArray(_deviceObject.hardwareType(), LEN_HARDWARE_TYPE, bufferPos);
+    bufferPos = pushWord(_deviceObject.version(), bufferPos);
+
+    flashPos = _platform.writeNonVolatileMemory(flashPos, buffer, bufferPos - buffer);
 
     print("save saveRestores ");
     println(_saveCount);
     for (int i = 0; i < _saveCount; i++)
     {
-        println(_data - buffer);
-        println(".");
-        //println((long)_saveRestores[i], HEX);
-        buffer = _saveRestores[i]->save(buffer);
+        bufferPos = _saveRestores[i]->save(buffer);
+        flashPos = _platform.writeNonVolatileMemory(flashPos, buffer, bufferPos - buffer);
     }
 
     print("save tableobjs ");
     println(_tableObjCount);
     for (int i = 0; i < _tableObjCount; i++)
     {
-        println(_data - buffer);
-        println(".");
-        //println((long)_tableObjects[i], HEX);
-        buffer = _tableObjects[i]->save(buffer);
+        bufferPos = _tableObjects[i]->save(buffer);
 
         //save to size of the memoryblock for tableobject too, so that we can rebuild the usedList and freeList
         if (_tableObjects[i]->_data != nullptr)
@@ -115,17 +130,23 @@ void Memory::writeMemory()
             MemoryBlock* block = findBlockInList(_usedList, _tableObjects[i]->_data);
             if (block == nullptr)
             {
-                println("_data of TableObject not in errorlist");
+                println("_data of TableObject not in _usedList");
                 _platform.fatalError();
             }
-            buffer = pushWord(block->size, buffer);
+            bufferPos = pushWord(block->size, bufferPos);
         }
         else
-            buffer = pushWord(0, buffer);
+            bufferPos = pushWord(0, bufferPos);
+
+        flashPos = _platform.writeNonVolatileMemory(flashPos, buffer, bufferPos - buffer);
     }
 
-    _platform.commitToEeprom();
-    printHex("SAVED ", _data, _metadataSize);
+    _platform.commitNonVolatileMemory();
+}
+
+void Memory::saveMemory()
+{
+    _platform.commitNonVolatileMemory();
 }
 
 void Memory::addSaveRestore(SaveRestore* obj)
@@ -151,7 +172,7 @@ void Memory::addSaveRestore(TableObject* obj)
 
 uint8_t* Memory::allocMemory(size_t size)
 {
-    // always allocate aligned to 32 bit
+    // always allocate aligned to pagesize
     size = alignToPageSize(size);
 
     MemoryBlock* freeBlock = _freeList;
@@ -220,19 +241,19 @@ void Memory::freeMemory(uint8_t* ptr)
 
 void Memory::writeMemory(uint32_t relativeAddress, size_t size, uint8_t* data)
 {
-    memcpy(toAbsolute(relativeAddress), data, size);
+    _platform.writeNonVolatileMemory(relativeAddress, data, size);
 }
 
 
 uint8_t* Memory::toAbsolute(uint32_t relativeAddress)
 {
-    return _data + (ptrdiff_t)relativeAddress;
+    return _platform.getNonVolatileMemoryStart() + (ptrdiff_t)relativeAddress;
 }
 
 
 uint32_t Memory::toRelative(uint8_t* absoluteAddress)
 {
-    return absoluteAddress - _data;
+    return absoluteAddress - _platform.getNonVolatileMemoryStart();
 }
 
 MemoryBlock* Memory::removeFromList(MemoryBlock* head, MemoryBlock* item)
@@ -354,8 +375,9 @@ void Memory::addToFreeList(MemoryBlock* block)
 
 uint16_t Memory::alignToPageSize(size_t size)
 {
-    // to 32 bit for now
-    return (size + 3) & ~0x3;
+    size_t pageSize = 4; //_platform.flashPageSize(); // align to 32bit for now, as aligning to flash-page-size causes side effects in programming
+    // pagesize should be a multiply of two
+    return (size + pageSize - 1) & (-1*pageSize);
 }
 
 MemoryBlock* Memory::findBlockInList(MemoryBlock* head, uint8_t* address)

src/knx/memory.h

@@ -28,8 +28,10 @@ class Memory
 {
 public:
     Memory(Platform& platform, DeviceObject& deviceObject);
+    virtual ~Memory();
     void readMemory();
     void writeMemory();
+    void saveMemory();
     void addSaveRestore(SaveRestore* obj);
     void addSaveRestore(TableObject* obj);
 
@@ -49,13 +51,17 @@ class Memory
     MemoryBlock* findBlockInList(MemoryBlock* head, uint8_t* address);
     void addNewUsedBlock(uint8_t* address, size_t size);
 
+    void readEraseBlockToBuffer(uint32_t blockNum);
+    uint8_t* eraseBlockStart(uint32_t blockNum);
+    uint8_t* eraseBlockEnd(uint32_t blockNum);
+    void saveBufferdEraseBlock();
+
     Platform& _platform;
     DeviceObject& _deviceObject;
     SaveRestore* _saveRestores[MAXSAVE] = {0};
     TableObject* _tableObjects[MAXTABLEOBJ] = {0};
     uint8_t _saveCount = 0;
     uint8_t _tableObjCount = 0;
-    uint8_t* _data = nullptr;
     MemoryBlock* _freeList = nullptr;
     MemoryBlock* _usedList = nullptr;
     uint16_t _metadataSize = 4 + LEN_HARDWARE_TYPE; // accounting for 2x pushWord and pushByteArray of length LEN_HARDWARE_TYPE