pkg/flashdb: enhance FAL config

pkg/flashdb/Kconfig

@@ -41,6 +41,15 @@ config MODULE_FLASHDB_KVDB_AUTO_UPDATE
           database. If the version changes, it will automatically trigger an upgrade action
           and update the new default KV collection to the current database.
 
+config FLASHDB_MIN_SECTOR_SIZE_DEFAULT_KiB
+    int "Minimal virtual sector size in KiB for FlashDB"
+    default 4
+    help
+        By default, KVDB will use 1 times the block size as the sector size, that is, 4096.
+        At this time, the KVDB cannot store a KV longer than 4096. If you want to save, for example,
+        a KV with a length of 10K, you can use the control function to set the sector size to 12K or
+        larger.
+
 config MODULE_FLASHDB_MTD
 
 endif # PACKAGE_FLASHDB

pkg/flashdb/include/fal_cfg.h

@@ -26,11 +26,40 @@
 #define FAL_CFG_H
 
 #include "board.h"
+#include "macros/units.h"
+#include "mtd_default.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+#if !defined(CONFIG_FLASHDB_MIN_SECTOR_SIZE_DEFAULT_KiB) || defined(DOXYGEN)
+/**
+ * @brief   Minimal virtual sector size in KiB for FlashDB
+ *
+ * This is just a reasonable default for an automatic partition configuration, hence "DEFAULT".
+ * The "MIN" stands for a required minimum to guarantee an expected size of key value pairs.
+ * The actually acceptable sector size must be a multiple of the physical sector size though.
+ * Thus, it can be larger than the minimum required size.
+ *
+ * The default is 4 (4096).
+ * FlashDB sector size is different from MTD sector size as it is a
+ * virtual measure of granularity and not a device property.
+ * The virtual sector size must be a multiple of the physical sector size.
+ *
+ * From the documentation of FLashDB:
+ * The internal storage structure of FlashDB is composed of N sectors, and each formatting takes
+ * sector as the smallest unit. A sector is usually N times the size of the Flash block.
+ * For example, the block size of Nor Flash is generally 4096.
+ *
+ * By default, KVDB will use 1 times the block size as the sector size, that is, 4096.
+ * At this time, the KVDB cannot store a KV longer than 4096. If you want to save, for example,
+ * a KV with a length of 10K, you can use the control function to set the sector size to 12K or larger.
+ *
+ */
+#define CONFIG_FLASHDB_MIN_SECTOR_SIZE_DEFAULT_KiB  4
+#endif
+
 /**
  * @brief   Partition table is defined at compile time (not read from flash)
  */
@@ -49,42 +78,105 @@
     &mtd_flash0,                                                     \
 }
 
+#if !defined(FAL_MTD) || defined(DOXYGEN)
+/**
+ * @brief   Default MTD to use for flashdb
+ */
+#define FAL_MTD                                     mtd_default_get_dev(0)
+#endif
+
+#if !defined(FAL_PART0_LABEL) || defined(DOXYGEN)
+/**
+ * @brief   Have at least the label of partition 0 defined
+ */
+#define FAL_PART0_LABEL                             "part0"
+#endif
+
+#if !defined(FAL_PART0_LENGTH) || defined(DOXYGEN)
+/**
+ * @brief   Have at least the length of partition 0 defined, which must be at least two sectors
+ *          and a multiple of the virtual sector size.
+ *
+ * The virtual sector size is however bound to the physical sector size of @ref FAL_MTD.
+ * So make sure that @ref CONFIG_FLASHDB_MIN_SECTOR_SIZE_DEFAULT_KiB times 1024 is a multiple of the MTD sector size.
+ * For example if the MTD sector size is 4KiB, then @ref CONFIG_FLASHDB_MIN_SECTOR_SIZE_DEFAULT_KiB must be a multiple of 4.
+ * If the MTD sector size is 1KiB, then you have all options for @ref CONFIG_FLASHDB_MIN_SECTOR_SIZE_DEFAULT_KiB.
+ */
+#define FAL_PART0_LENGTH                            (2 * (CONFIG_FLASHDB_MIN_SECTOR_SIZE_DEFAULT_KiB * KiB(1)))
+#endif
+
 /**
  * @brief Partition 0
  */
-#ifdef FAL_PART0_LABEL
+#if defined(FAL_PART0_LABEL) || defined(DOXYGEN)
+#if !defined(FAL_PART0_OFFSET) || defined(DOXYGEN)
+/**
+ * @brief Offset of partition 0
+ */
+#define FAL_PART0_OFFSET    0
+#endif
+/**
+ * @brief   Partition 0 compound definition
+ */
 #define FAL_ROW_PART0   { FAL_PART_MAGIC_WORD, FAL_PART0_LABEL, "fal_mtd", \
-                          0, FAL_PART0_LENGTH, 0 },
+                          FAL_PART0_OFFSET, FAL_PART0_LENGTH, 0 },
 #else
 #define FAL_ROW_PART0
 #endif
 
 /**
  * @brief Partition 1
  */
-#ifdef FAL_PART1_LABEL
+#if defined(FAL_PART1_LABEL) || defined(DOXYGEN)
+#if !defined(FAL_PART1_OFFSET)  || defined(DOXYGEN)
+/**
+ * @brief Offset of partition 1
+ */
+#define FAL_PART1_OFFSET    (FAL_PART0_OFFSET + FAL_PART0_LENGTH)
+#endif
+/**
+ * @brief   Partition 1 compound definition
+ */
 #define FAL_ROW_PART1   { FAL_PART_MAGIC_WORD, FAL_PART1_LABEL, "fal_mtd", \
-                          FAL_PART0_LENGTH, FAL_PART1_LENGTH, 0 },
+                          FAL_PART1_OFFSET, FAL_PART1_LENGTH, 0 },
 #else
 #define FAL_ROW_PART1
 #endif
 
 /**
  * @brief Partition 2
  */
-#ifdef FAL_PART2_LABEL
-#define FAL_ROW_PART2   { FAL_PART_MAGIC_WORD, FAL_PART2_LABEL, "fal_mtd",
-                          FAL_PART1_LENGTH, FAL_PART2_LENGTH, 0 },
+#if defined(FAL_PART2_LABEL) || defined(DOXYGEN)
+#if !defined(FAL_PART2_OFFSET) || defined(DOXYGEN)
+/**
+ * @brief Offset of partition 2
+ */
+#define FAL_PART2_OFFSET    (FAL_PART1_OFFSET + FAL_PART1_LENGTH)
+#endif
+/**
+ * @brief   Partition 2 compound definition
+ */
+#define FAL_ROW_PART2   { FAL_PART_MAGIC_WORD, FAL_PART2_LABEL, "fal_mtd", \
+                          FAL_PART2_OFFSET, FAL_PART2_LENGTH, 0 },
 #else
 #define FAL_ROW_PART2
 #endif
 
 /**
  * @brief Partition 3
  */
-#ifdef FAL_PART3_LABEL
-#define FAL_ROW_PART3   { FAL_PART_MAGIC_WORD, FAL_PART2_LABEL, "fal_mtd",
-                          FAL_PART2_LENGTH, FAL_PART3_LENGTH, 0 },
+#if defined(FAL_PART3_LABEL) || defined(DOXYGEN)
+#if !defined(FAL_PART3_OFFSET) || defined(DOXYGEN)
+/**
+ * @brief Offset of partition 3
+ */
+#define FAL_PART3_OFFSET    (FAL_PART2_OFFSET + FAL_PART2_LENGTH)
+#endif
+/**
+ * @brief   Partition 3 compound definition
+ */
+#define FAL_ROW_PART3   { FAL_PART_MAGIC_WORD, FAL_PART3_LABEL, "fal_mtd", \
+                          FAL_PART3_OFFSET, FAL_PART3_LENGTH, 0 },
 #else
 #define FAL_ROW_PART3
 #endif

pkg/flashdb/mtd/fal_mtd_port.c

@@ -75,7 +75,9 @@ struct fal_flash_dev mtd_flash0 = {
 void fdb_mtd_init(mtd_dev_t *mtd)
 {
     unsigned sector_size;
-
+    if (_mtd) {
+        return;
+    }
     mtd_init(mtd);
     _mtd = mtd;
 

tests/pkg/flashdb_fal_cfg/Makefile

@@ -0,0 +1,29 @@
+include ../Makefile.pkg_common
+
+# select the MTD backend
+USEMODULE += flashdb_mtd
+# enable key-value database
+USEMODULE += flashdb_kvdb
+# enable time series database
+USEMODULE += flashdb_tsdb
+# rtc_localtime()
+USEMODULE += rtc_utils
+
+# prefer periph_rtc over periph_rtt
+FEATURES_OPTIONAL += periph_rtc
+FEATURES_REQUIRED_ANY += periph_rtc|periph_rtt
+
+CFLAGS += -DFAL_PART1_LABEL=\"part1\"
+CFLAGS += -DFAL_PART1_LENGTH=FAL_PART0_LENGTH
+CFLAGS += -DFAL_PART2_LABEL=\"part2\"
+CFLAGS += -DFAL_PART2_LENGTH=FAL_PART0_LENGTH
+CFLAGS += -DFAL_PART3_LABEL=\"part3\"
+CFLAGS += -DFAL_PART3_LENGTH=FAL_PART0_LENGTH
+
+include $(RIOTBASE)/Makefile.include
+
+# handle RTC backend after inclusion of $(RIOTBASE)/Makefile.include
+ifeq (,$(filter periph_rtc,$(FEATURES_USED)))
+  USEMODULE += rtt_rtc
+  USEMODULE += ztimer_no_periph_rtt
+endif

tests/pkg/flashdb_fal_cfg/Makefile.ci

@@ -0,0 +1,7 @@
+BOARD_INSUFFICIENT_MEMORY := \
+    nucleo-f031k6 \
+    nucleo-l011k4 \
+    samd10-xmini \
+    stk3200 \
+    stm32f030f4-demo \
+    #

tests/pkg/flashdb_fal_cfg/main.c

@@ -0,0 +1,149 @@
+/*
+ * Copyright (C) 2023 ML!PA Consulting Gmbh
+ *
+ * This file is subject to the terms and conditions of the GNU Lesser General
+ * Public License v2.1. See the file LICENSE in the top level directory for more
+ * details.
+ */
+
+/**
+ * @ingroup   tests
+ * @{
+ *
+ * @file
+ * @brief       Short test for the FlashDB FAL configuration initialization
+ *
+ * @author      Fabian Hüßler <fabian.huessler@ml-pa.com>
+ *
+ * @}
+ */
+
+#include <stdio.h>
+
+#include "container.h"
+#include "fal.h"
+#include "macros/math.h"
+#include "mutex.h"
+#include "periph/rtc.h"
+
+#include <flashdb.h>
+
+/**
+ * @brief   FlashDB Magic Word
+ *
+ *          The value here is not relevant but it must be defined.
+ */
+#define FAL_PART_MAGIC_WORD         0x45503130
+
+/**
+ * @brief   Number of FAL partitions
+ */
+#define FAL_PART_TABLE_NUMOF        ARRAY_SIZE(((const struct fal_partition[])FAL_PART_TABLE))
+
+#if !defined(FAL_TSDB_MAX) || defined(DOXYGEN)
+/**
+ * @brief   Maximum length of a TSDB entry
+ */
+#define FAL_TSDB_MAX                128
+#endif
+
+#if !defined(FAL_PART_TABLE_KVDB) || defined(DOXYGEN)
+/**
+ * @brief   Indices of partitions in @ref FAL_PART_TABLE that can be initialized as a Key-Value-DB
+ */
+#define FAL_PART_TABLE_KVDB     \
+{                               \
+    0,                          \
+    1,                          \
+}
+#endif
+
+#if !defined(FAL_PART_TABLE_TSDB) || defined(DOXYGEN)
+/**
+ * @brief   Indices of partitions in @ref FAL_PART_TABLE that can be initialized as a Time-Series-DB
+ */
+#define FAL_PART_TABLE_TSDB     \
+{                               \
+    2,                          \
+    3,                          \
+}
+#endif
+
+extern void fdb_mtd_init(mtd_dev_t *mtd);
+
+static mutex_t _locker = MUTEX_INIT;
+static const unsigned _fdb_part_kvdb[] = FAL_PART_TABLE_KVDB;
+static const unsigned _fdb_part_tsdb[] = FAL_PART_TABLE_TSDB;
+static struct fdb_kvdb _kvdb[ARRAY_SIZE(_fdb_part_kvdb)];
+static struct fdb_tsdb _tsdb[ARRAY_SIZE(_fdb_part_tsdb)];
+
+/**
+ * @brief Select MTD device to use for FlashDB
+ */
+#if !defined(FDB_MTD)
+#define FDB_MTD FAL_MTD
+#endif
+
+static void _lock(fdb_db_t db)
+{
+    mutex_lock(db->user_data);
+}
+
+static void _unlock(fdb_db_t db)
+{
+    mutex_unlock(db->user_data);
+}
+
+static fdb_time_t _get_time(void)
+{
+    struct tm now;
+    rtc_get_time(&now);
+    return mktime(&now);
+}
+
+int main(void)
+{
+    int init_failed;
+    fdb_mtd_init(FDB_MTD);
+    size_t size = FDB_MTD->pages_per_sector * FDB_MTD->page_size;
+    /* scale hardware sector size to minimum required virtual sector size */
+    size = DIV_ROUND_UP((CONFIG_FLASHDB_MIN_SECTOR_SIZE_DEFAULT_KiB * KiB(1)), size) * size;
+    printf("Informational: Make sure the following requirements are fulfilled for a successful initialization:\n");
+    printf("The virtual sector size is a multiple of the physical sector size: %lu %% %lu == 0\n",
+           (unsigned long)size, (unsigned long)FDB_MTD->pages_per_sector * FDB_MTD->page_size);
+    printf("The maximum partition size is a multiple of the virtual sector size: %lu %% %lu == 0\n",
+           (unsigned long)FAL_PART0_LENGTH, (unsigned long)size);
+
+    for (unsigned i = 0; i < ARRAY_SIZE(_fdb_part_kvdb); i++) {
+        unsigned part = _fdb_part_kvdb[i];
+        if (part >= FAL_PART_TABLE_NUMOF) {
+            continue;
+        }
+        fdb_kvdb_control(&_kvdb[i], FDB_KVDB_CTRL_SET_SEC_SIZE, &size);
+        fdb_kvdb_control(&_kvdb[i], FDB_KVDB_CTRL_SET_LOCK, (void *)(uintptr_t)_lock);
+        fdb_kvdb_control(&_kvdb[i], FDB_KVDB_CTRL_SET_UNLOCK, (void *)(uintptr_t)_unlock);
+        const char *spart = ((const struct fal_partition[])FAL_PART_TABLE)[part].name;
+        printf("Initializing FlashDB KVDB partition %s\n", spart);
+        if ((init_failed = fdb_kvdb_init(&_kvdb[i], "kvdb", spart, NULL, &_locker)) != FDB_NO_ERR) {
+            printf("Failed to initialize FlashDB KVDB partition %s (%d)\n", spart, init_failed);
+            return 1;
+        }
+    }
+    for (unsigned i = 0; i < ARRAY_SIZE(_fdb_part_tsdb); i++) {
+        unsigned part = _fdb_part_tsdb[i];
+        if (part >= FAL_PART_TABLE_NUMOF) {
+            continue;
+        }
+        fdb_tsdb_control(&_tsdb[i], FDB_TSDB_CTRL_SET_LOCK, (void *)(uintptr_t)_lock);
+        fdb_tsdb_control(&_tsdb[i], FDB_TSDB_CTRL_SET_UNLOCK, (void *)(uintptr_t)_unlock);
+        fdb_tsdb_control(&_tsdb[i], FDB_TSDB_CTRL_SET_SEC_SIZE, &size);
+        const char *spart = ((const struct fal_partition[])FAL_PART_TABLE)[part].name;
+        printf("Initializing FlashDB TSDB partition %s\n", spart);
+        if ((init_failed = fdb_tsdb_init(&_tsdb[i], "tsdb", spart, _get_time, FAL_TSDB_MAX, &_locker)) != FDB_NO_ERR) {
+            printf("Failed to initialize FlashDB TSDB partition %s (%d)\n", spart, init_failed);
+            return 1;
+        }
+    }
+    puts("SUCCESS");
+    return 0;
+}

tests/pkg/flashdb_mtd/Makefile.ci

@@ -0,0 +1,7 @@
+BOARD_INSUFFICIENT_MEMORY := \
+    nucleo-f031k6 \
+    nucleo-l011k4 \
+    samd10-xmini \
+    stk3200 \
+    stm32f030f4-demo \
+    #

tests/pkg/flashdb_vfs/main.c

@@ -9,6 +9,7 @@
 #include <sys/types.h>
 #include <flashdb.h>
 
+#include "fal_cfg.h"
 #include "board.h"
 #include "mutex.h"
 #ifdef MODULE_VFS
@@ -20,8 +21,8 @@
 /**
  * @brief Select MTD device to use for FlashDB
  */
-#if !defined(FDB_MTD) && defined(MTD_0)
-#define FDB_MTD MTD_0
+#if !defined(FDB_MTD)
+#define FDB_MTD FAL_MTD
 #endif
 
 #define FDB_LOG_TAG "[main]"
@@ -110,9 +111,9 @@
          *
          *       &kvdb: database object
          *       "env": database name
          * "fdb_kvdb1": The flash partition name base on FAL. Please make sure it's in FAL partition table.
          *              Please change to YOUR partition name.
          * &default_kv: The default KV nodes. It will auto add to KVDB when first initialize successfully.
          *  &kv_locker: The locker object.
          */
         result = fdb_kvdb_init(&kvdb, "env", FDB_DIR "/fdb_kvdb1", &default_kv, &kv_locker);
@@ -150,7 +151,7 @@
          *
          *       &tsdb: database object
          *       "log": database name
          * "fdb_tsdb1": The flash partition name base on FAL. Please make sure it's in FAL partition table.
          *              Please change to YOUR partition name.
          *    get_time: The get current timestamp function.
          *         128: maximum length of each log