Add disk storage support (SD cards) (#2584)

This PR extends the Storage system to support block-based devices such as SD cards.

Storage sizes have been changed from `uint32_t` to `storage_size_t`. Default operation is unchanged, but this allows 64-bit offsets to be used to support devices greater than about 4GB.

The existing `SDCard` library has been taken and reworked extensively with added GNU/Linux source code to handle MBR/GPT partitioning. The result is the `DiskStorage` base library, which is then used by the `SdStorage` library to provide specific SD card support, currently using SPI.

Optional buffering for block devices is included to support other filing systems. LittleFS appears to work but Spiffs doesn't - it requires erased sectors to be 0xFF which is fundamentally incompatible.

Finally, the `FatIFS` library has been added, using the current version of Chan's fatfs library. There are some minor modifications in the code to fix time handling and simplify use.

Testing for these libraries has been implemented by verifying generated filesystem images using standard GNU/Linux tools.

.gitmodules

@@ -213,10 +213,6 @@
 	path = Sming/Libraries/CS5460/CS5460
 	url = https://github.com/xxzl0130/CS5460.git
 	ignore = dirty
-[submodule "Libraries.DIAL"]
-	path = Sming/Libraries/DIAL
-	url = https://github.com/slaff/Sming-DIAL.git
-	ignore = dirty
 [submodule "Libraries.DFRobotDFPlayerMini"]
 	path = Sming/Libraries/DFRobotDFPlayerMini
 	url = https://github.com/DFRobot/DFRobotDFPlayerMini.git
@@ -225,6 +221,18 @@
 	path = Sming/Libraries/DHTesp
 	url = https://github.com/beegee-tokyo/DHTesp.git
 	ignore = dirty
+[submodule "Libraries.DIAL"]
+	path = Sming/Libraries/DIAL
+	url = https://github.com/slaff/Sming-DIAL.git
+	ignore = dirty
+[submodule "Libraries.DiskStorage"]
+	path = Sming/Libraries/DiskStorage
+	url = https://github.com/mikee47/DiskStorage
+	ignore = dirty
+[submodule "Libraries.FatIFS"]
+	path = Sming/Libraries/FatIFS
+	url = https://github.com/mikee47/FatIFS
+	ignore = dirty
 [submodule "Libraries.flatbuffers"]
 	path = Sming/Libraries/flatbuffers/src
 	url = https://github.com/google/flatbuffers.git
@@ -313,6 +321,10 @@
 	path = Sming/Libraries/RingTone
 	url = https://github.com/mikee47/RingTone
 	ignore = dirty
+[submodule "Libraries.SdStorage"]
+	path = Sming/Libraries/SdStorage
+	url = https://github.com/mikee47/SdStorage
+	ignore = dirty
 [submodule "Libraries.SignalGenerator"]
 	path = Sming/Libraries/SignalGenerator
 	url = https://github.com/mikee47/SignalGenerator

Sming/Arch/Esp8266/Components/esp8266/startup.cpp

@@ -48,22 +48,39 @@ extern "C" void ICACHE_FLASH_ATTR WEAK_ATTR user_pre_init(void)
 {
 	Storage::initialize();
 
-	auto sysParam = *Storage::findPartition(Storage::Partition::SubType::Data::sysParam);
-	auto rfCal = *Storage::findPartition(Storage::Partition::SubType::Data::rfCal);
-	auto phy = *Storage::findPartition(Storage::Partition::SubType::Data::phy);
+	using PartType = Storage::Partition::SubType::Data;
+	auto sysParam = *Storage::findPartition(PartType::sysParam);
+	auto rfCal = *Storage::findPartition(PartType::rfCal);
+	auto phy = *Storage::findPartition(PartType::phy);
 
-	static const partition_item_t partitions[] = {
-			{SYSTEM_PARTITION_BOOTLOADER,		0,						SPI_FLASH_SEC_SIZE},
-			{SYSTEM_PARTITION_PHY_DATA,			phy.address(),			phy.size()},
-			{SYSTEM_PARTITION_SYSTEM_PARAMETER,	sysParam.address(),		sysParam.size()},
-			{SYSTEM_PARTITION_RF_CAL,			rfCal.address(),		rfCal.size()},
+	static const partition_item_t partitions[]{
+		{
+			SYSTEM_PARTITION_BOOTLOADER,
+			0,
+			SPI_FLASH_SEC_SIZE,
+		},
+		{
+			SYSTEM_PARTITION_PHY_DATA,
+			uint32_t(phy.address()),
+			uint32_t(phy.size()),
+		},
+		{
+			SYSTEM_PARTITION_SYSTEM_PARAMETER,
+			uint32_t(sysParam.address()),
+			uint32_t(sysParam.size()),
+		},
+		{
+			SYSTEM_PARTITION_RF_CAL,
+			uint32_t(rfCal.address()),
+			uint32_t(rfCal.size()),
+		},
 	};
 
 	enum flash_size_map sizeMap = system_get_flash_size_map();
 	if(!system_partition_table_regist(partitions, ARRAY_SIZE(partitions), sizeMap)) {
 		os_printf("system_partition_table_regist: failed\n");
 		os_printf("size_map = %u\n", sizeMap);
-		for (unsigned i = 0; i < ARRAY_SIZE(partitions); ++i) {
+		for(unsigned i = 0; i < ARRAY_SIZE(partitions); ++i) {
 			auto& part = partitions[i];
 			os_printf("partition[%u]: %u, 0x%08x, 0x%08x\n", i, part.type, part.addr, part.size);
 		}

Sming/Arch/Host/Components/spi_flash/component.mk

@@ -1 +1,3 @@
 COMPONENT_INCDIRS += $(ESP8266_COMPONENTS)/spi_flash/include
+
+COMPONENT_DEPENDS := IFS

Sming/Arch/Rp2040/Components/rp2040/component.mk

@@ -91,7 +91,8 @@ LIBDIRS += \
 EXTRA_LIBS += \
 	pico \
 	m \
-	stdc++
+	stdc++ \
+	gcc
 
 RP2040_CMAKE_OPTIONS := \
 	-G Ninja \

Sming/Components/Storage/README.rst

@@ -304,6 +304,20 @@ Configuration
    Set this to adjust the hardware profile using option fragments. See :ref:`hwconfig_options`.
 
 
+.. envvar:: ENABLE_STORAGE_SIZE64
+
+   Build with ``ENABLE_STORAGE_SIZE64=1`` to enable support for storage devices of more than 4GB capacity.
+
+   Device and partition addresses and sizes use the :cpp:type:`storage_size_t` type, which by default is ``uint32_t``.
+   Setting this value changes it to ``uint64_t``.
+
+   When enabling this setting, care must be taken in code especially with ``printf`` style format strings such
+   as in debug statements. The safest way to handle both cases is like this::
+
+      debug_i("Partition size: %llu", uint64_t(part.size()));
+
+
+
 Binary partition table
 ----------------------
 
@@ -379,6 +393,9 @@ you can take advantage of the partition API to manage them as follows:
    in your ``init()`` function (or elsewhere if more appropriate).
 
 
+See :library:`DiskStorage` for how devices such as SD flash cards are managed.
+
+
 API
 ---
 

Sming/Components/Storage/component.mk

@@ -2,6 +2,11 @@ COMPONENT_INCDIRS	:= src/include
 COMPONENT_SRCDIRS	:= src
 COMPONENT_DOXYGEN_INPUT := src/include
 
+COMPONENT_VARS := ENABLE_STORAGE_SIZE64
+ifeq ($(ENABLE_STORAGE_SIZE64),1)
+GLOBAL_CFLAGS += -DENABLE_STORAGE_SIZE64
+endif
+
 COMPONENT_RELINK_VARS := PARTITION_TABLE_OFFSET
 
 CONFIG_VARS			+= HWCONFIG HWCONFIG_OPTS

Sming/Components/Storage/src/Debug.cpp

@@ -16,6 +16,15 @@ void listPartitions(Print& out)
 	out.println();
 }
 
+void listPartitions(Print& out, const Device& device)
+{
+	out << device.getName() << _F(" partitions:") << endl;
+	for(auto part : device.partitions()) {
+		out << "- " << part << endl;
+	}
+	out.println();
+}
+
 void listDevices(Print& out, bool fullPartitionInfo)
 {
 	out.println();

Sming/Components/Storage/src/Device.cpp

@@ -4,7 +4,7 @@
  * http://github.com/SmingHub/Sming
  * All files of the Sming Core are provided under the LGPL v3 license.
  *
- * Device.h - external storage device API
+ * Device.cpp
  *
  ****/
 
@@ -89,8 +89,8 @@ bool Device::loadPartitions(Device& source, uint32_t tableOffset)
 					toString(Device::Type(entry->subtype)).c_str(), toString(getType()).c_str());
 		}
 		if(entry->size != getSize()) {
-			debug_w("[Device] '%s' size mismatch, 0x%08x in partition table but device reports 0x%08x",
-					getName().c_str(), entry->size, getSize());
+			debug_w("[Device] '%s' size mismatch, 0x%08x in partition table but device reports 0x%08llx",
+					getName().c_str(), entry->size, uint64_t(getSize()));
 		}
 
 		// Skip the storage entry, not required

Sming/Components/Storage/src/Partition.cpp

@@ -4,7 +4,7 @@
  * http://github.com/SmingHub/Sming
  * All files of the Sming Core are provided under the LGPL v3 license.
  *
- * Partition.cpp - Partition support for all architectures
+ * Partition.cpp
  *
  ****/
 
@@ -158,15 +158,15 @@ bool Partition::verify(Partition::Type type, uint8_t subtype) const
 	return true;
 }
 
-bool Partition::getDeviceAddress(uint32_t& address, size_t size) const
+bool Partition::getDeviceAddress(storage_size_t& address, storage_size_t size) const
 {
 	if(mDevice == nullptr || mPart == nullptr) {
 		debug_e("[Partition] Invalid");
 		return false;
 	}
 
 	if(address >= mPart->size || (address + size - 1) >= mPart->size) {
-		debug_e("[Partition] Invalid range, address: 0x%08x, size: 0x%08x", address, size);
+		debug_e("[Partition] Invalid range, address: 0x%08llx, size: 0x%08llx", uint64_t(address), uint64_t(size));
 		return false;
 	}
 
@@ -212,66 +212,82 @@ bool Partition::allowWrite()
 	return true;
 }
 
-bool Partition::read(uint32_t offset, void* dst, size_t size)
+bool Partition::read(storage_size_t offset, void* dst, size_t size)
 {
 	if(!allowRead()) {
 		return false;
 	}
 
-	uint32_t addr = offset;
+	auto addr = offset;
 	if(!getDeviceAddress(addr, size)) {
 		return false;
 	}
 
-	return mDevice ? mDevice->read(addr, dst, size) : false;
+	return mDevice->read(addr, dst, size);
 }
 
-bool Partition::write(uint32_t offset, const void* src, size_t size)
+bool Partition::write(storage_size_t offset, const void* src, size_t size)
 {
 	if(!allowWrite()) {
 		return false;
 	}
 
-	uint32_t addr = offset;
+	auto addr = offset;
 	if(!getDeviceAddress(addr, size)) {
 		return false;
 	}
 
-	return mDevice ? mDevice->write(addr, src, size) : false;
+	return mDevice->write(addr, src, size);
 }
 
-bool Partition::erase_range(uint32_t offset, size_t size)
+bool Partition::erase_range(storage_size_t offset, storage_size_t size)
 {
 	if(!allowWrite()) {
 		return false;
 	}
 
-	uint32_t addr = offset;
+	auto addr = offset;
 	if(!getDeviceAddress(addr, size)) {
 		return false;
 	}
 
-	return mDevice ? mDevice->erase_range(addr, size) : false;
+	return mDevice->erase_range(addr, size);
 }
 
-size_t Partition::printTo(Print& p) const
+uint16_t Partition::getSectorSize() const
+{
+	return mDevice ? mDevice->getSectorSize() : Device::defaultSectorSize;
+}
+
+bool Partition::sync()
+{
+	return mDevice ? mDevice->sync() : false;
+}
+
+size_t Partition::Info::printTo(Print& p) const
 {
 	size_t n{0};
+	n += p.print(name.length() == 0 ? _F("(NO NAME)") : name.c_str());
+	n += p.print(", ");
+	n += p.print(fullType());
+	n += p.print(" @ 0x");
+	n += p.print(offset, HEX);
+	n += p.print(_F(", size 0x"));
+	n += p.print(size, HEX);
+	return n;
+}
+
+size_t Partition::printTo(Print& p) const
+{
 	if(*this) {
+		size_t n{0};
 		n += p.print(getDeviceName());
 		n += p.print('/');
-		n += p.print(name());
-		n += p.print(" (");
-		n += p.print(typeString());
-		n += p.print(" @ 0x");
-		n += p.print(address(), HEX);
-		n += p.print(_F(", size 0x"));
-		n += p.print(size(), HEX);
-		n += p.print(')');
-	} else {
-		n += p.print(_F("(none)"));
+		n += p.print(*mPart);
+		return n;
 	}
-	return n;
+
+	return p.print(_F("(none)"));
 }
 
 } // namespace Storage

Sming/Components/Storage/src/ProgMem.cpp

@@ -15,7 +15,7 @@ namespace Storage
 {
 ProgMem progMem;
 
-bool ProgMem::read(uint32_t address, void* dst, size_t size)
+bool ProgMem::read(storage_size_t address, void* dst, size_t size)
 {
 	size_t readCount = flashmem_read(dst, address, size);
 	return readCount == size;

Sming/Components/Storage/src/SpiFlash.cpp

@@ -33,24 +33,24 @@ size_t SpiFlash::getBlockSize() const
 	return SPI_FLASH_SEC_SIZE;
 }
 
-size_t SpiFlash::getSize() const
+storage_size_t SpiFlash::getSize() const
 {
 	return flashmem_get_size_bytes();
 }
 
-bool SpiFlash::read(uint32_t address, void* dst, size_t size)
+bool SpiFlash::read(storage_size_t address, void* dst, size_t size)
 {
 	size_t readCount = flashmem_read(dst, address, size);
 	return readCount == size;
 }
 
-bool SpiFlash::write(uint32_t address, const void* src, size_t size)
+bool SpiFlash::write(storage_size_t address, const void* src, size_t size)
 {
 	size_t writeCount = flashmem_write(src, address, size);
 	return writeCount == size;
 }
 
-bool SpiFlash::erase_range(uint32_t address, size_t size)
+bool SpiFlash::erase_range(storage_size_t address, storage_size_t size)
 {
 	if(address % SPI_FLASH_SEC_SIZE != 0 || size % SPI_FLASH_SEC_SIZE != 0) {
 		debug_e("[Partition] erase address/size misaligned: 0x%08x / 0x%08x", address, size);

Sming/Components/Storage/src/include/Storage/Debug.h

@@ -8,6 +8,7 @@ namespace Storage
 namespace Debug
 {
 void listPartitions(Print& out);
+void listPartitions(Print& out, const Device& device);
 void listDevices(Print& out, bool fullPartitionInfo = true);
 
 } // namespace Debug