diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 00000000000..e560daa3f6c --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1,2 @@ +*.md.html + diff --git a/docs/design-documents/features/storage/Configuration/CONFIGURATION.md b/docs/design-documents/features/storage/Configuration/CONFIGURATION.md new file mode 100644 index 00000000000..b0ffb95c5de --- /dev/null +++ b/docs/design-documents/features/storage/Configuration/CONFIGURATION.md @@ -0,0 +1,388 @@ +# Storage configuration + +This document describes the configurations of the mbed OS storage based on kv_store interface. + +## New Storage design +![New storage design](./NewStorageDesign.jpg) + +This document describes the configuration for the kvstore part (left side of the diagram above) and its underlying components such as filesystem and block devices. +The configuration concept is based on a number of pre-defined topologies (configurations) allowing customers to select one of them or to define also a user-defined topology. + +The last section of this document will explain how to override the configuration option in order to enable storage configuration which is not supported by any of the configurations above. +In order to use the default configurations, the storage_type parameter should be set to one of the configurations options available. +The implementation of the configuration is composed of a set json files and a set of functions instantiating and initializating the required components. + +## Configuration structure +``` +kvstore +│ +└───conf + │ mbed_lib.json + │ + ├───tdb_external + │ mbed_lib.json + │ + ├───tdb_external_no_rbp + │ mbed_lib.json + │ + ├───filesystem + │ mbed_lib.json + │ + ├───filesystem_no_rbp + │ mbed_lib.json + │ + └───tdb_internal + mbed_lib.json +``` + +The kvstore configuration file structure includes six configuration files. The topmost configuration file is used to set up the full configuration of the storage by defining a single parameter (storage_type) to one of the predefined configurations. The configuration files in the subfolders are used to implement the above top level configurations. + +The configuration files can be found under `conf/`. +* conf/tdb_internal - storage type TDB_INTERNAL configuration is intended to be used when all data will be stored in internal memory only. No need for additional security features. A single TDBStore object will be allocated in internal flash. +* conf/tdb_external - storage type TDB_EXTERNAL configuration is providing full security and intended to be used when data is stored in external flash. It allocates: SecureStore, TDBStore in external flash and TDBStore in internal flash (for rollback protection - RBP) +* conf/tdb_external_no_rbp - storage type TDB_EXTERNAL_NO_RBP configuration allows security but without rollback protection. Similar to tdb_external but without the TDBStore in internal memory. +* conf/filesystem - This configuration will allocate: SecureStore, FileSystemStore, filesystem, TDBStore in internal memory and the required block devices. The allocated file system will be selected according to the COMPONENT set in targets.json, (FATFS for SD card and LITTLEFS for SPIF), however, this can be set differently by overiding the respective parameter. Use this configuration if you need the file system with POSIX API in addition to the set/get API. +* conf/filesystem_no_rbp - storage type FILESYSTEM_NO_RBP configuration allows security like FILESYSTEM configuration but without rollback protection. + +A stand alone block device will be allocated for each component in internal and external memory and SD cards as required for the configurations. In any case, the full size of the memory allocated for each block device will be used by the respective component. + +## Configuration parameters +The following is a list of all storage parameters available and their description. + +* storage_type - Used to select one of the pre-defined configurations. + * TDB_INTERNAL + * TDB_EXTERNAL + * TDB_EXTERNAL_NO_RBP + * FILESYSTEM + * FILESYSTEM_NO_RBP +* default_kv - This is a string representing the path for the default kvstore instantiation. Applications can pass an empty path (only the key name) or pass the generated name for this parameter (MBED_CONF_STORAGE_DEFAULT_KV) as the path to use this configuration. +* internal_size - The size in bytes for the internal FlashIAP block device. This should enable together with the internal_base_address to adjust exactly the size and location where the block device resides on memory. If not defined the block device will try to get the maximum size available. +* internal_base_address - The address where the internal FlashIAP blockDevice starts. This helps to prevent collisions with other needs like firmware updates. If not defined the start address will be set to the first sector after the application code ends in TDB_internal while in any external configurations with rollback protection support it will be set to end of flash - rbp_internal_size. +* rbp_number_of_entries - set the number of entries allowed for rollback protection. The default is set to 64. This parameter controls the max number of different keys that can be created with rollback protection flag. +rbp_internal_size sets the size for the rollback protection TDBStore in the internal memory. the base address will be calculated as flash ends address - size. +* filesystem - Options are FAT, LITTLE or default. If not set or set to default the file system type will be selected according to the storage component selected for the board in targets.json file: FAT for "components": ["SD"] and Littlefs for "components": ["SPIF"]. +* blockdevice - Options are default, SPIF, DATAFLASH, QSPIF or SD. If filesystem set to default this parameter is ignored. +* external_size - The size of the external block device in bytes. If not set the maximum available size will be used. +* external_base_address - The start address of the external block device. if not set 0 address will be used. +* mount_point - mount point for the filesystem. This parameter will be ignored if the filesystem is set to default. +* folder_path - Path for the working directory where the FileSyetemStore stores the data + +## Storage configuration + +Below is the main storage configuration mbed_lib json file. + +``` +{ +"name": "storage", + "config": { + "storage_type": { + "help": "Options are TDB_INTERNAL, TDB_EXTERNAL, TDB_EXTERNAL_NO_RBP, FILESYSTEM or FILESYSTEM_NO_RBP.", + "value": "NULL" + }, + "default_kv": { + "help": "A string name for the default kvstore configurtaion", + "value": "kv" + } + } +} +``` + +### TDB_INTERNAL +The internal configuration should be used for targets willing to save all the data in internal flash + +![TDB_Internal](./Internal.jpg) + +In this configuration, all kvstore C API will be mapped to the TDBStore in the internal flash. To use this configuration the storage_type parameter in storage mbed_lib.json should be set to TDB_INTERNAL. + +Below is the TDB_INTERNAL configuration mbed_lib.json +``` +{ + "name": "tdb_internal", + "config": { + "internal_size": { + "help": "Size of the FlashIAP block device", + "value": "NULL" + }, + "internal_base_address": { + "help": "If not defined the default is the first sector after the application code ends.", + "value": "NULL" + }, + "rbp_number_of_entries": { + "help": "If not defined default is 64", + "value": "64" + } + } +} +``` + +For this configuration please define the section of the internal storage that will be used for data, by defining these parameters in your app.config file: internal_base_address and internal_size. If not defined, the storage will start in the first sector immediately after the end of the application. This can reduce the ability to update the application with a bigger one. + +### TDB_External +![External](./TDB_External.jpg) + +TDB_EXTERNAL uses a TDBStore in the internal flash for security rollback protection and a TDBStore on the external flash for the data. This configuration all kvstore C API calls are mapped to work with the SecureStore class and this class will handle the use of the two TDBStores. Unless configured differently the external TDBStore will work on top of the default block device, while the internal TDBStore will work with the FlashIAPBlockdevice. +The external TDBStore block device can be set to any of the following block devices. SPIF, QSPIF, DATAFASH and SD. + +This configuration can be enabled by setting storage_type in storage mbed_lib.json to TDB_EXTERNAL. + +Below is the TDB_EXTERNAL configuration mbed_lib.json +``` +{ + + "name": "tdb_external", + "config": { + "rbp_internal_size": { + "help": "If not defined default size is 4K*#enteries/32", + "value": "NULL" + }, + "rbp_number_of_entries": { + "help": "If not defined default is 64", + "value": "64" + }, + "internal_base_address": { + "help": "If not defined the default is the first sector after the application code ends.", + "value": "NULL" + }, + "blockdevice": { + "help": "Options are default, SPIF, DATAFASH, QSPIF or SD", + "value": "NULL" + }, + "external_size": { + "help": "Size of the external block device", + "value": "NULL" + }, + "external_base_address": { + "help": "If not defined the default is from address 0", + "value": "NULL" + } + } +} +``` + +### TDB_External_no_RBP +![External](./TDB_External_no_rbp.jpg) + +TDB_EXTERNAL_NO_RBF Configuration has no support for rollback protection and therefore less secure. +The TDB_EXTERNAL_NO_RBP uses only one TDBStore on the external flash for all data. In this configuration all kvstore C API calls are mapped to work with the SecureStore class. The external TDBStore will work on top of the default block device however, external TDBStore block device can be set to any of the following block devices SPIF, QSPIF, DATAFASH and SD. + + +This configuration can be enabled by setting storage_type in storage mbed_lib.json to TDB_EXTERNAL_NO_RBP. + +Below is the TDB_EXTERNAL_NO_RBP configuration mbed_lib.json +``` +{ + "name": "tdb_external_no_rbp", + "config": { + "external_size": { + "help": "Size of the external block device", + "value": "NULL" + }, + "external_base_address": { + "help": "If not defined the default is from address 0", + "value": "NULL" + }, + "blockdevice": { + "help": "Options are default, SPIF, DATAFASH, QSPIF or FILESYSTEM", + "value": "NULL" + } + } +} +``` + +### FILESYSTEM +![FILESYSTEM](./FILESYSTEM.jpg) + +The FILESYSTEM configuration resembles the EXTERNAL with the difference that it uses FileSysytemStore on the external flash. By default, the FileSystemStore will use the default filesystem and the default block device. +In this configuration, all kvstore C API paths are mapped to the SecureStore class and this class will handle the use of the internal TDBStore or external FileSystemStore. +This configuration can be enabled by setting storage_type in storage mbed_lib.json to FILESYSTEM. + +Below is the FILESYSTEM configuration mbed_lib.json +``` +{ + "name": "filesystem_store", + "config": { + "rbp_internal_size": { + "help": "If not defined default size is 4K*#enteries/32", + "value": "NULL" + }, + "rbp_number_of_entries": { + "help": "If not defined default is 64", + "value": "64" + }, + "internal_base_address": { + "help": "If not defined the default is the first sector after the application code ends.", + "value": "NULL" + }, + "filesystem": { + "help": "Options are default, FAT or LITTLE. If not specified default filesystem will be used", + "value": "NULL" + }, + "blockdevice": { + "help": "Options are default, SPIF, DATAFASH, QSPIF or FILESYSTEM. If not set the default block device will be used", + "value": "NULL" + }, + "external_size": { + "help": "Size in bytes of the external block device, if not specified the maximum is the default.", + "value": "NULL" + }, + "external_base_address": { + "help": "If not defined the default is from address 0", + "value": "NULL" + }, + "mount_point": { + "help": "Where to mount the filesystem. Ignored if the default file system is applied.", + "value": "/sd" + }, + "folder_path": { + "help": "Path for the working directory where the FileSyetemStore stores the data", + "value": "/kvstore" + } + } +} +``` +If filesystem is not set the default filesystem and block device will be applied and blockdevice, external_size and external_base_address will be ignored. + +### FILESYSTEM_NO_RBP +![FILESYSTEM](./FILESYSTEM_no_rbp.jpg) + +The FILESYSTEM_NO_RBP configuration resembles the EXTERNAL_NO_RBP with the difference that it uses FileSysytemStore on the external flash. By default, the FileSystemStore will use the default filesystem and the default block device. This Configuration has no support for rollback protection and therefore less secure. +in this configuration all kvstore C API calls are mapped to the SecureStore class and this class will handle the use of the external FileSystemStore. +This configuration can be enabled by setting storage_type in storage mbed_lib.json to FILESYSTEM_NO_RBF. + +Below is the FILESYSTEM configuration mbed_lib.json +``` +{ + "name": "filesystem_store_no_rbp", + "config": { + "filesystem": { + "help": "Options are default, FAT or LITTLE. If not specified default filesystem will be used", + "value": "NULL" + }, + "blockdevice": { + "help": "Options are default, SPIF, DATAFASH, QSPIF or FILESYSTEM. If not set the default block device will be used", + "value": "NULL" + }, + "external_size": { + "help": "Size in bytes of the external block device, if not specified the maximum is the default.", + "value": "NULL" + }, + "external_base_address": { + "help": "If not defined the default is from address 0", + "value": "NULL" + }, + "mount_point": { + "help": "Where to mount the filesystem. Ignored if the default file system is applied.", + "value": "/sd" + }, + "folder_path": { + "help": "Path for the working directory where the FileSyetemStore stores the data", + "value": "/kvstore" + } + } +} +``` + +If filesystem is not set the default filesystem and block device will be applied and blockdevice, external_size and external_base_address will be ignored + +### Configuration functions API. +Applications must call the function **storage_configuration()** to instantiate the required configuration. This function is defined as weak to allow the replacement of this function with a completely different implementation of the instantiation of components. +Below is a list of setup functions that will be called by storage_configuration() in each case, and their description. + +``` +#if MBED_CONF_STORAGE_STORAGE == NULL +define MBED_CONF_STORAGE_STORAGE USER_DEFINED +#endif + +#define _STORAGE_CONFIG_concat(dev) _storage_config_##dev() +#define _STORAGE_CONFIG(dev) _STORAGE_CONFIG_concat(dev) + +/** + * @brief This function initializes internal memory secure storage + * This includes a TDBStore instance with a FlashIAPBlockdevice + * as the supported storage. + * The following is a list of configuration parameter + * MBED_CONF_STORAGE_INTERNAL_SIZE - The size of the underlying FlashIAPBlockdevice + * MBED_CONF_STORAGE_INTERNAL_BASE_ADDRESS - The start address of the underlying FlashIAPBlockdevice + * MBED_CONF_STORAGE_INTERNAL_RBP_NUMBER_OF_ENTRIES - If not defined default is 64 + * @returns true on success or false on failure. + */ +bool _storage_config_TDB_INTERNAL(); + +/** + * @brief This function initialize external memory secure storage + * This includes a SecureStore class with TDBStore over FlashIAPBlockdevice + * and an external TDBStore over a default blockdevice unless configured differently. + * The following is a list of configuration parameter: + * MBED_CONF_STORAGE_TDB_EXTERNAL_RBP_INTERNAL_SIZE - Size of the internal FlashIAPBlockDevice and by default is set to 4K*#enteries/32. The start address will be set to end of flash - rbp_internal_size. + * MBED_CONF_STORAGE_TDB_EXTERNAL_RBP_NUMBER_OF_ENTRIES - If not defined default is 64 + * MBED_CONF_STORAGE_TDB_EXTERNAL_EXTERNAL_SIZE - Size of the external blockdevice in bytes or NULL for max possible size. + * MBED_CONF_STORAGE_TDB_EXTERNAL_EXTERNAL_BASE_ADDRESS - The block device start address. + * MBED_CONF_STORAGE_TDB_EXTERNAL_EXTERNAL_BLOCK_DEVICE - Alowed vlaues are: default, SPIF, DATAFASH, QSPIF or SD + * @returns true on success or false on failure. + */ +bool _storage_config_TDB_EXTERNAL(); + +/** + * @brief This function initialize a predefined external memory secure storage + * This includes a SecureStore class with external TDBStore over a blockdevice or, + * if no blockdevice was set the default blockdevice will be used. + * The following is a list of configuration parameter: + * MBED_CONF_STORAGE_TDB_EXTERNAL_NO_RBP_EXTERNAL_SIZE - Size of the external blockdevice in bytes or NULL for max possible size. + * MBED_CONF_STORAGE_TDB_EXTERNAL_NO_RBP_EXTERNAL_BASE_ADDRESS - The block device start address + * MBED_CONF_STORAGE_TDB_EXTERNAL_NO_RBP_EXTERNAL_BLOCK_DEVICE - Alowed vlaues are: default, SPIF, DATAFASH, QSPIF or SD + * @returns true on success or false on failure. + */ +bool _storage_config_TDB_EXTERNAL_NO_RBP(); + +/** + * @brief This function initialize a predefined FILESYSTEM memory secure storage + * This includes a SecureStore class with TDBStore over FlashIAPBlockdevice + * in the internal memory and an external FileSysteStore. If blockdevice and filesystem not set, + * the system will use the default block device and default filesystem + * The following is a list of configuration parameter: + * MBED_CONF_STORAGE_FILESYSTEM_RBP_INTERNAL_SIZE - Size of the internal FlashIAPBlockDevice and by default is set to 4K*#enteries/32. The start address will be set to end of flash - rbp_internal_size. + * MBED_CONF_STORAGE_FILESYSTEM_RBP_NUMBER_OF_ENTRIES - If not defined default is 64 + * MBED_CONF_STORAGE_FILESYSTEM_FILESYSTEM - Allowed values are: default, FAT or LITTLE + * MBED_CONF_STORAGE_FILESYSTEM_BLOCKDEVICE - Allowed values are: default, SPIF, DATAFASH, QSPIF or SD + * MBED_CONF_STORAGE_FILESYSTEM_EXTERNAL_SIZE - External Blockdevice size in bytes or NULL for max possible size. + * MBED_CONF_STORAGE_FILESYSTEM_EXTERNAL_BASE_ADDRESS - The block device start address. + * MBED_CONF_STORAGE_FILESYSTEM_MOUNT_POINT - Where to mount the filesystem + * MBED_CONF_STORAGE_FILESYSTEM_FOLDER_PATH - The working folder paths + * + * @returns true on success or false on failure. + */ +bool _storage_config_FILESYSTEM(); + +/** + * @brief This function initialize a predefined FILESYSTEM_NO_RBP memory secure storage with no + * rollback protection. This includes a SecureStore class an external FileSysteStore over a default + * filesystem with default blockdevice unless differently configured. + * The following is a list of configuration parameter: + * MBED_CONF_STORAGE_FILESYSTEM_NO_RBP_FILESYSTEM - Allowed values are: default, FAT or LITTLE + * MBED_CONF_STORAGE_FILESYSTEM_NO_RBP_BLOCKDEVICE - Allowed values are: default, SPIF, DATAFASH, QSPIF or SD + * MBED_CONF_STORAGE_FILESYSTEM_NO_RBP_EXTERNAL_SIZE - Blockdevice size in bytes. or NULL for max possible size. + * MBED_CONF_STORAGE_FILESYSTEM_NO_RBP_EXTERNAL_BASE_ADDRESS - The block device start address. + * MBED_CONF_STORAGE_FILESYSTEM_NO_RBP_MOUNT_POINT - Where to mount the filesystem + * MBED_CONF_STORAGE_FILESYSTEM_NO_RBP_FOLDER_PATH - The working folder paths + * + * @returns true on success or false on failure. + */ +bool _storage_config_FILESYSTEM_NO_RBP(); + +MBED_WEAK bool storage_configuration() +{ + return _STORAGE_CONFIG(MBED_CONF_STORAGE_STORAGE_TYPE); +} +``` + +### Override user-defined setup +in order to create a much more complex setup including using other block devices such as MBRBlockDevice or SlicingBlockDevice, there is a need to override the storage_configuration function and generate any storage configuration of choice. + + + + + + + + + diff --git a/docs/design-documents/features/storage/Configuration/FILESYSTEM.jpg b/docs/design-documents/features/storage/Configuration/FILESYSTEM.jpg new file mode 100644 index 00000000000..52ea615a8e2 Binary files /dev/null and b/docs/design-documents/features/storage/Configuration/FILESYSTEM.jpg differ diff --git a/docs/design-documents/features/storage/Configuration/FILESYSTEM.xml b/docs/design-documents/features/storage/Configuration/FILESYSTEM.xml new file mode 100644 index 00000000000..c8434cb7323 --- /dev/null +++ b/docs/design-documents/features/storage/Configuration/FILESYSTEM.xml @@ -0,0 +1 @@ +7Vjfb9owEP5reNyU39DHUKBFYxITaFsf3eRIvJoYOQ4l++tnkwtJmrSqJho0NF6wP/vsu+++s5MM7Nvt4U6QXfyVh8AGlhEeBvZkYFmmY3nqTyN5gYwMpwAiQUOcVAEr+hsQNBDNaAhpY6LknEm6a4IBTxIIZAMjQvDn5rQNZ81ddySCFrAKCGujP2goY4zCNSr8HmgUlzubBo48kuApEjxLcL+BZW+Ov2J4S8q1cH4ak5A/1yB7OrBvBeeyaG0Pt8A0tyVthd3sldGT3wIS+S6DG/RD5mXsECoqsMuFjHnEE8KmFTo+xgd6BUP1YrllqmmqJhyo/Knhzy72HsqRRIq8NqS7D7jAL5AyRw2QTHIFVfsuON/hGu3YMNyUZyJA7y1UCxERnLJTYDqwmh0Scgd8C8obNUEAI5LumxogKKXoNK+iUzWQ0W520Zk9YRkuqkyMp30quQDV8pfzFv01Ol8NeA9CwuHNWHDULoWWlwWG/edK1uYIsbgmads4Q/hGK7p/W1ylkhrqsi+lrtKbmry+fD9uhP5ZBiM5iL4lNupTYsNrk9jZ5YSmS04TWWXNcptZs90X2Sg0hFYvEnJy4105ss3/OfqrHLlebzkqQ6qdJSsIMqHycbyqej1BLK/HE8S5tkvK6bikHOtSl5TTvqTWk3EfqjKtV6qnpqrTKXh2VbUf/WaMpPHcX44ZD54msKdB3ww4Zp8M2NdWV05HXQ0vVldOW2CUwSpPJWz7KC/HeUd5eR8lLrcV/QQ2JGPqHtSvWKvlfFa0vlXNib/2Zwt/da/aXOhZys5TJvb4UXW9SJ6Y6Y01Z9gna961leSwoyTdi5Xk8G1Rzvx1qbzFfL1eTPvWWtfx/2FaG7XImCcShMpyK2wVj2xqS0BKf5PH4wStmp1+xD16444H7kQhWjtpISNtQBiNEtVmsNFLaZJoQJiPsNS6Gqc7EtAkWh9F9sk5D8uW6zbPwY4vLF0kW+cg+aZF8vRwjSS7xovLxjA/imTVrT6/Fm9S1Tdue/oH \ No newline at end of file diff --git a/docs/design-documents/features/storage/Configuration/FILESYSTEM_no_rbp.jpg b/docs/design-documents/features/storage/Configuration/FILESYSTEM_no_rbp.jpg new file mode 100644 index 00000000000..04659b75520 Binary files /dev/null and b/docs/design-documents/features/storage/Configuration/FILESYSTEM_no_rbp.jpg differ diff --git a/docs/design-documents/features/storage/Configuration/FILESYSTEM_no_rbp.xml b/docs/design-documents/features/storage/Configuration/FILESYSTEM_no_rbp.xml new file mode 100644 index 00000000000..6efe447e310 --- /dev/null +++ b/docs/design-documents/features/storage/Configuration/FILESYSTEM_no_rbp.xml @@ -0,0 +1 @@ +3ZjbcpswEIafhst2MALbufQxydSdSceeNrmUYQ1qZOQRwjF5+kqwnIKTSacpmcQ3Xv3alXZXn/DBIrP96VLSQ/RdBMAtxw5OFplbjjNwnaF+M0pWKGPbLYRQsgCdamHNHgFFG9WUBZC0HJUQXLFDW/RFHIOvWhqVUjy03XaCt3c90BA6wtqnvKv+YoGKsArPrvUrYGFU7jywcWZL/ftQijTG/SyH7PJXMb2n5Vron0Q0EA8NiSwsMpNCqMLan2bATW/LthVxy2dmq7wlxOpVAReYh8rK2iHQrcChkCoSoYgpX9TqNK8PzAq2HkVqz7U50CacmLo18lcPR3flTKxk1pgywztc4DcolSEDNFVCS/W+KyEOuEa3Niw3Ean0MXsHaaEyhOp0Cs0U1ojDhlyC2IPORjtI4FSxY5sBiiiFlV/dTm1gR893F5M5Up7iojrEvj8mSkjQ1uTmutP+RjufLfgIUsHpxVpwlpSgZeUFw/FDjfVgjFrUQJrYb1C+3anuY8NVktSii7wXXWU2Dby+/cw3wvwcm9MMZN+IjXtEjAw61X1wxM7j1GbOdf8RMQy9ESxW9Ul6w/ZJEu/JCRU5YNSTQ6rSeB27pMPuGvxU6jPKH429EusMeyTWJZ+MWNc9A+jovR6KZTYNsJaMwzpLFOz7gMsZP3OJGnBVwL05XF6n+jnsaMr1fTVfPdY318vC+lGb88lmslxN1lfaFtJ46bihDiHTrR4OQ1V1preuuaM+uzb8bFdydOZKeu92JUcvQ7mcbEryVtebzWrRN2uDPlm76DRjcVIg9Sl3ytb1qDZbEhL2SLe5g6HmYD6K82y8qeXNtWLYSQqMTADlLIy1zWFnljJNYvpn7gRlZbiaJgfqszjc5JB9cf/Tc7BcotHlc012/r7Jelj/Yi6+jNR/S5DFHw== \ No newline at end of file diff --git a/docs/design-documents/features/storage/Configuration/Internal.jpg b/docs/design-documents/features/storage/Configuration/Internal.jpg new file mode 100644 index 00000000000..3aadaa11201 Binary files /dev/null and b/docs/design-documents/features/storage/Configuration/Internal.jpg differ diff --git a/docs/design-documents/features/storage/Configuration/Internal.xml b/docs/design-documents/features/storage/Configuration/Internal.xml new file mode 100644 index 00000000000..d3e14418372 --- /dev/null +++ b/docs/design-documents/features/storage/Configuration/Internal.xml @@ -0,0 +1 @@ +3VVNT8JAEP01vZptFwgeBfEjamKCUTmu7dBWth2yTKH117ulU9qmYjwoB7kw8+b78UodOU3ya6PW0QMGoB1PBLkjLx3PcwfeyH6VSFEhYzGogNDEASc1wDz+AAYFo1kcwKaTSIia4nUX9DFNwacOpozBXTdtibo7da1C6AFzX+k++hIHFPEVQ9HgNxCHUT3ZFRx5U/4qNJilPM/x5HL/qcKJqntx/iZSAe5akJw5cmoQqbKSfAq65Lamraq7OhI97G0gpR8VnPMeVNS3Q2CpYBcNRRhiqvSsQSf7+6DsIKwXUaKt6VoT8pheS/hsyN6ijqRkilaodBfc4B2ICtaAyggt1My9R1xzj2rTcr2jxzK0wcz4nOWxfJQJ4fBzHZi2CgZMwG5jcwxoRfG2216xlMJDXkOnNZjRr9nl2VulM25qS8RquyE0YK2Lx9se/S06uRoMQf79yf1buEAOWGhF/YCxv2tk7Y4Zi1qSluIXzhe96/6ZuNz6z6mtLnkiddXDW/K6e94P4nU8oVUB5tQSG59QYhXZHQ6eLidzMggnPtsb/d3Z1m1eCftY670rZ58= \ No newline at end of file diff --git a/docs/design-documents/features/storage/Configuration/NewStorageDesign.jpg b/docs/design-documents/features/storage/Configuration/NewStorageDesign.jpg new file mode 100644 index 00000000000..601d57524e5 Binary files /dev/null and b/docs/design-documents/features/storage/Configuration/NewStorageDesign.jpg differ diff --git a/docs/design-documents/features/storage/Configuration/TDB_External.jpg b/docs/design-documents/features/storage/Configuration/TDB_External.jpg new file mode 100644 index 00000000000..10994b66d76 Binary files /dev/null and b/docs/design-documents/features/storage/Configuration/TDB_External.jpg differ diff --git a/docs/design-documents/features/storage/Configuration/TDB_External.xml b/docs/design-documents/features/storage/Configuration/TDB_External.xml new file mode 100644 index 00000000000..d2c83a211b1 --- /dev/null +++ b/docs/design-documents/features/storage/Configuration/TDB_External.xml @@ -0,0 +1 @@ +5VhRb5swEP41PHYCG9rmMYSkrdZJnVJt66MLDnh1cGRMmvTXz4YjQE2nTsqI1uUl9mdzvvv47nyJg2fr3ZUkm+yLSCh3kJvsHBw5CHk+OtdfBtnXyGWAayCVLIFNLbBkLxRAF9CSJbTobVRCcMU2fTAWeU5j1cOIlOK5v20leP/UDUmpBSxjwm30O0tU1kThtvg1ZWnWnOy5sPJI4qdUijKH8xyEV9WnXl6TxhbsLzKSiOcOhOcOnkkhVD1a72aUG24b2urnFm+sHvyWNFfvemACfqh9EztNNBUwFVJlIhU54fMWDav4qLHg6lmm1lwPPT2kO6Z+GPhTALOHZiVXct9ZMtMHMPCTKrUHDZBSCQ21594KsQEbdmwQbiFKGYP3CNRCZEoPb6fGTGCd54CQKyrWVHujN0jKiWLbvgYISCk97Gvp1ANgdJhdcGZLeAlG9SPu07ZQQlI9mt7dWPR36Hwz4C2Viu5+Gwus4kZo+ybBYP7cytq7BCzrSBq7RwjftaL7t8XVKKmnLnwqdTXedOT1+Vt1EPiHXE72VI4tscsxJXbx0SSGbYmhkxUw7P0P9J4sg7FvZfB8p6jUgVm862xUfTolLdgLeaw2GKI2guWq8iYInSDSCOEszTUQa0p0HcChyWqm25wpLKxZklTvjJNHysND8zITXMjq3KZ9GeIXv1UrDo0ZeNdrboZqiPvJnwSoV0aCevZuzsH2nSGhNXx20TN6FvQNiNWqMIXy1Ss7OPi+Ooytt7ikcSl1QlTX/KjVF52PWH0x+mDlAQ1c8E0Xd4L20b7g76NwDFV5r1SFA1tVyP9bd7rdNi84KbKb6V3IRfwU0S2Lx2bA90ZkAONx8uq4uTN0tfonyx27KI+UO/7FKXPH7igiuiIl15ec+e25NL84zSCa3k8Xt9PldT39qhcW9VBf/HpfNDJL4+ZXYLF0kx+z79K5VNRp5XXaME5XaqAJUybPwmJDYpan91XSnfnHYRkFr7qDycRi+XyAZPTnJOtp+7dV3UW1/w3i+S8= \ No newline at end of file diff --git a/docs/design-documents/features/storage/Configuration/TDB_External_no_rbp.jpg b/docs/design-documents/features/storage/Configuration/TDB_External_no_rbp.jpg new file mode 100644 index 00000000000..06644f8156f Binary files /dev/null and b/docs/design-documents/features/storage/Configuration/TDB_External_no_rbp.jpg differ diff --git a/docs/design-documents/features/storage/Configuration/TDB_External_no_rbp.xml b/docs/design-documents/features/storage/Configuration/TDB_External_no_rbp.xml new file mode 100644 index 00000000000..414d36d7e77 --- /dev/null +++ b/docs/design-documents/features/storage/Configuration/TDB_External_no_rbp.xml @@ -0,0 +1 @@ +1VfbbuIwEP2aPG4V7KSCRyDQrrYrdUW1u300yZB4a2JkHC79+nWScS4NVJVKU5UXPGfs8VyOx45Dp+vDjWKb5KeMQDjEjQ4ODRxCBh65Nn85ciyRoeuVQKx4hJNqYMGfAUEX0YxHsG1N1FIKzTdtMJRpCqFuYUwpuW9PW0nR3nXDYugAi5CJLvqHRzrBKHy3xm+Bx4ndeeCiZsnCp1jJLMX9HEJXxa9Ur5m1hfO3CYvkvgHRmUOnSkpdjtaHKYg8tzZt5br5GW3lt4JUv2nBCP3QRxs7RCYVKEqlExnLlIlZjU6K+CC34Bop0WthhgMzhAPXf3P4ykfp0WpSrY4NVS4+ooF/oPUROcAyLQ1U73sn5QZtdGPDcLcyUyF6T5AtTMVQVafE8sAa6zAhNyDXYLwxExQIpvmuzQGGVIqreXU6zQAzejq76MyOiQyNmiXu026rpQIzGt9/76S/kc6zAe9AaTi8GgtqqSXa0R4wlPc1rQdDxJIGpal7gfDdTnRfm1yWSS120c9il/WmQa8fv4uN0D/iCnYE1TfFhj1SjA460X1xitEuxcjFKYZL7yVPdV1Jct2uZFVZa6J0FFe9KFLlxtvq5nW4OztoUCZjnYIaHup2nRRs+TNbFhPyCmxyjwof/YnjBwZhgsepAUKTa3MC6CTnMzcX/BgVax5FBRkEW4KYVNf2VAqpin3txX2qcPTcKameJOhd61o/dXrcK2/kk1ba/VJ6ZzG/jfxTVq0BuVpt4b1VtGRtVHEBYabMSSsuuF77TsXeXvoO7afvXPZtdKq3eJ91fZEueR6CSR/MIaMXfc7zO8wh3kc9irqdL4AVy4Q5jPnrcJG/CfNBMH4Yz+/Gi9tS/GUU83JoGpSZF/ScJc/9uCwZsf7mKRtR/WFJZ/8B \ No newline at end of file diff --git a/docs/design-documents/features/storage/Configuration/UserDefined.jpg b/docs/design-documents/features/storage/Configuration/UserDefined.jpg new file mode 100644 index 00000000000..dbcfe8a2ec9 Binary files /dev/null and b/docs/design-documents/features/storage/Configuration/UserDefined.jpg differ diff --git a/docs/design-documents/features/storage/Configuration/UserDefined.xml b/docs/design-documents/features/storage/Configuration/UserDefined.xml new file mode 100644 index 00000000000..08de035c403 --- /dev/null +++ b/docs/design-documents/features/storage/Configuration/UserDefined.xml @@ -0,0 +1 @@ +3VjLctowFP0aL9ux/CCwNAESprSTDjRtlop9Y6sRFiMLAvn6Srb8ikjbmYLdgQVIR1ePe3yOJWG51+v9Dceb5DOLgFqOHe0td2I5DvKcgfxRyKFAhrZXADEnkQ6qgSV5BQ3aGt2SCLJWoGCMCrJpgyFLUwhFC8Ocs5d22BOj7Vk3OAYDWIaYmuh3EolEZ+HbNX4LJE7KmZGtWx5x+Bxztk31fJbjPuWfonmNy7F0fJbgiL00IHdqudecMVGU1vtroIrbkrai3+yd1mrdHFLxNx2qdYhDmTtEkgpdZVwkLGYpptMaHef5gRrBlrVErKksIlmEPRE/FPzR17WHsiUV/NBoUtUHPcBPEOKgNYC3gkmonnfB2EaPYeam083Ylod69a5WC+Yx6KhhAam8Gt00HzfA1iAXIwM4UCzIri0BrJUUV3E1m7KgCT1Orl7LDtOtHvTTfT6PXp1jU3wAbjyBBqPv5rwDLmD/23x0q1s+Y+1FB/lF/aVWdmW8pKFq1/53CtDVhenLM/Xl9qUvz9CX7GI/7zLBOMhScDfvWFuofEN2oa3S2BejLd/UFupNXL4hrlmwstTznsnvxXy1Wkyr6pdvi8WZpeb7bal59pHXmHsuqSEjuxNIDTWE1tBWF1IbHpGa09s+eRZ27f+LXdQXu0PDyEsIt1wSWewShYNXk3Eb6MDSxsnkqsOTych8v1GcJfPgbkxZ+DyBHQk7ZWPY4/vNvbRrQGm3pgVHfTmwXExDa33Y7Y3AXK/LDfTS7gHlbtkUmN+bwJw/C2xGKGSHTMC6c+G9Pbp1qjxk3sK/Lu/ms4qAVmUSrILZIlje1s2TXk+5V0cuVGfbBcwL5TwVwKU9LGdA5UzjRy5LsaiSbRAhMxRtm3LIyCt+zAOUATeMpCJfnz+2/IlElA2zwpGqA6YkTmWZwpMaStFGQkwDDQtl0XG2wSFJ41Xu1w/eaXhHI7/FuzMYGbwPjtDunIJ286o13Ze0XxLJA/T2vGeK+0Qky2r9J2re1vin2p3+Ag== \ No newline at end of file diff --git a/docs/design-documents/features/storage/FileSystemStore/FileSystemStore_class_hierarchy.jpg b/docs/design-documents/features/storage/FileSystemStore/FileSystemStore_class_hierarchy.jpg new file mode 100644 index 00000000000..ca9f62a31e4 Binary files /dev/null and b/docs/design-documents/features/storage/FileSystemStore/FileSystemStore_class_hierarchy.jpg differ diff --git a/docs/design-documents/features/storage/FileSystemStore/FileSystemStore_class_hierarchy.xml b/docs/design-documents/features/storage/FileSystemStore/FileSystemStore_class_hierarchy.xml new file mode 100644 index 00000000000..173fdab83b4 --- /dev/null +++ b/docs/design-documents/features/storage/FileSystemStore/FileSystemStore_class_hierarchy.xml @@ -0,0 +1 @@ +7VfbbtswDP0aP3bwJU7zmjRNCwwDBgS7PSo2bQuVLU9WEmdfP8qm41u8ZsDWbkDzEFuHFEUe8ZJY3l1aPiiWJx9kCMJy7bC0vLXluo7vz/BhkFON3C4IiBUPSakFtvwHEGgTuuchFD1FLaXQPO+DgcwyCHQPY0rJY18tkqJ/as5iGAHbgIkx+oWHOqnRhW+3+CPwOGlOdmyS7FjwFCu5z+g8y/Wi6lOLU9bYIv0iYaE8diDv3vLulJS6fkvLOxCG24a2et9mQnr2W0Gmr9pAfhyY2EPj8lzg3lVu3NMnomT+fW98WqVMxTyzvCVK7bzEbwSryAx+o2Vey2YdmYZS3zDBY9oXoHOgWpv4FtOzOnnXABsuYHsqNKRbLRU0cgxnN9yDWD7EEmVCaJKrOc2ZPjjkh4tBJ3TXxnnXBDZlAL2obDSo2zPnHkBpjlm2rLlYV2ytiJm1gMjskqgViSopIo6X5a0imWmqEcel9YalXJjqegRxAGPV+KlTYZTOZ3cTgXLD+ABlB6LEeACZglYnVCGp61FyUBG7M1of25JwGizplMOCMEZVGJ9Nt5mIL5SME4npjBLzUwEjRiELl6becSVzQBZXiHS5ClmRQEikXMMPhL3WMGanG/2cIlUgmOaHfve4FD6Z+yg5Hntm2rsdMD1ksJB7FQDt6pbzwJBvP2NIY5mCHhmqbuMc43UX5P4fneNXTWOn3trIX28jo5x80TYy+5ez9P3nt7n2+nNt8ZL56F/Ix+mZthMyeBpOMZRvKgLX9njedahChtTpq1F75zfLb8/RWA+IfvG8ykic6hq/OxJn84Eh70+NRFy2P9Rr9fbfkHf/Ew== \ No newline at end of file diff --git a/docs/design-documents/features/storage/FileSystemStore/FileSystemStore_design.md b/docs/design-documents/features/storage/FileSystemStore/FileSystemStore_design.md new file mode 100644 index 00000000000..bb7077d952b --- /dev/null +++ b/docs/design-documents/features/storage/FileSystemStore/FileSystemStore_design.md @@ -0,0 +1,381 @@ +# FileSystemStore in Mbed OS + +- [FileSystemStore in Mbed OS](#filesystemstore-in-mbed-os) + + [Revision history](#revision-history) +- [Introduction](#introduction) + + [Overview and background](#overview-and-background) + + [Requirements and assumptions](#requirements-and-assumptions) +- [System architecture and high-level design](#system-architecture-and-high-level-design) + * [Design basics](#design-basics) +- [Detailed design](#detailed-design) + + [Class header](#class-header) + + [Important data structures](#important-data-structures) + + [Initialization and reset](#initialization-and-reset) + + [Core APIs](#core-apis) + + [Incremental set APIs](#incremental-set-apis) + + [Key iterator APIs](#key-iterator-apis) +- [Usage scenarios and examples](#usage-scenarios-and-examples) + + [Standard usage of the class](#standard-usage-of-the-class) +- [Other information](#other-information) + + [Open issues](#open-issues) + + +### Revision history + +| Revision | Date | Authors | Mbed OS version | Comments | +|---------- |---------------- |-------------------------------------------------------- |----------------- |------------------ | +| 1.0 | 20 September 2018 | David Saada ([@davidsaada](https://github.com/davidsaada/)) | 5.11+ | Initial revision | + +# Introduction + +### Overview and background + +FileSystemStore is a lightweight implementation of the [KVStore](../KVStore/KVStore_design.md) interface over file systems. + +### Requirements and assumptions + +FileSystemStore assumes the underlying file system qualities for resilience and file validation. This means that if the underlying file system has no protection against power failures, then neither would FileSystemStore have. +When initializing this class, it is assumed that the underlying FileSystem is initialized and mounted. + +# System architecture and high-level design + +## Design basics + +FileSystemStore implements the get/set interface using files, where each key is represented by a single file. Key is represented by the file name and value is stored as file data. Therefore, FileSystemStore imitates the get/set actions using simple file operations. set is achieved using open-write-close, get using open-read-close and so on. +All files are concentrated under a single directory, whose name is hard coded. So actions such as "reset" are mapped to the deletion of all files under this directory, while iteration actions use file system APIs to traverse the directory. + +### Data layout + +When storing the data, it is stored with a preceding 16-byte metadata header. Metadata includes flags and other parameters for basic validity checks. + +![FileSystemStore Record](./FileSystemStore_record.jpg) + +Fields are: +- Magic: A constant value, for quick validity checking +- Metadata size: Size of metadata header +- Revision: FileSystemStore revision (currently 1) +- User flags: Flags received from user. Currently only write once is dealt with (others are ignored) + +# Detailed design + +FileSystemStore fully implements the KVStore interface over a file system. As such, it uses the FileSystem class interface for file operations. + +![FileSystemStore Class Hierarchy](./FileSystemStore_class_hierarchy.jpg) + +Functionality, as defined by KVStore, includes the following: +- Initialization & reset +- Core actions: get, set & remove +- Incremental set actions +- Iterator actions + +### Class header + +FileSystemStore has the following header: + +```C++ +class FileSystemStore : KVStore { + +public: + FileSystemStore(FileSystem *fs); + virtual ~FileSystemStore(); + + // Initialization and reset + virtual int init(); + virtual int deinit(); + virtual int reset(); + + // Core API + virtual int set(const char *key, const void *buffer, size_t size, uint32_t create_flags); + virtual int get(const char *key, void *buffer, size_t buffer_size, size_t *actual_size = NULL, size_t offset = 0); + virtual int get_info(const char *key, info_t *info); + virtual int remove(const char *key); + + // Incremental set API + virtual int set_start(set_handle_t *handle, const char *key, size_t final_data_size, uint32_t create_flags); + virtual int set_add_data(set_handle_t handle, const void *value_data, size_t data_size); + virtual int set_finalize(set_handle_t handle); + + // Key iterator + virtual int iterator_open(iterator_t *it, const char *prefix = NULL); + virtual int iterator_next(iterator_t it, char *key, size_t key_size); + virtual int iterator_close(iterator_t it); + +private: + Mutex _mutex; + FileSystem *_fs; + bool _is_initialized; +} +``` + +### Important data structures + +```C++ +// Key metadata +typedef struct { + uint32_t magic; + uint16_t metadata_size; + uint16_t revision; + uint32_t flags; +} key_metadata_t; + +// incremental set handle +typedef struct { + char *key; + uint32_t create_flags; +} inc_set_handle_t; + +// iterator handle +typedef struct { + void *dir_handle; + char *prefix; +} key_iterator_handle_t; +``` + + +### Initialization and reset + +**init function** + +Header: +`virtual int init();` + +Pseudo code: +- if `_is_initialized` return OK +- Create and take `_mutex` +- Create the FileSystemStore directory if it doesn't exist +- Set `_is_initialized` to true +- Release `_mutex` + +**deinit function** + +Header: +`virtual int deinit();` + +Pseudo code: +- if not `_is_initialized` return OK +- Take `_mutex` +- Set `_is_initialized` to false +- Release `_mutex` + +**reset function** + +Header: +`virtual int reset();` + +Pseudo code: +- Take `_mutex` +- Delete all files under the FileSystemStore directory +- Set `_num_keys` to 0 +- Release `_mutex` + +### Core APIs + +**set function** + +Header: +`virtual int set(const char *key, const void *buffer, size_t size, uint32_t create_flags);` + +Pseudo code: +- if not `_is_initialized` return "not initialized" error +- Call `set_start` with all fields and a local `set_handle_t` variable +- Call `set_add_data` with `buffer` and `size` +- Call `set_finalize` +- Return OK + +**get function** + +Header: +`virtual int get(const char *key, void *buffer, size_t buffer_size, size_t *actual_size = NULL, size_t offset = 0);` + +Pseudo code: +- if not `_is_initialized` return "not initialized" error +- Take `_mutex` +- Using the `stat` API, extract file size +- Open file `key` for reading to achieve a file handle +- If failed, release `_mutex` and return "not found" error +- Read from file into a `key_metadata_t` structure +- Using `size` API, achieve file size +- Seek to `offset` + metadata size +- Set `actual_size` as the minimum of buffer size and remainder of data +- Read data from file to `buffer`, size is `actual_size` +- Close file +- Release `_mutex` +- Return OK + +**get_info function** + +Header: +`virtual int get_info(const char *key, info_t *info);` + +Pseudo code: +- if not `_is_initialized` return "not initialized" error +- Find file `key` under the FileSystemStore directory. If not existing, return "not found" error. +- Take `_mutex` +- Open file `key` for reading to achieve a file handle +- If failed, release `_mutex` and return "not found" error +- Using `size` API, achieve file size +- Read from file into a `key_metadata_t` structure +- Fill `info` structure with all relevant fields +- Close file +- Return OK + +**remove function** + +Header: +`virtual int remove(const char *key);` + +Pseudo code: +- if not `_is_initialized` return "not initialized" error +- Take `_mutex` +- Open file `key` for reading and read data into a `key_metadata_t` structure +- If not existing, return "not found error" +- If flag "write once" is preset, return "write once" error +- Delete file `key` +- Release `_mutex` +- Return OK + +### Incremental set APIs + +**set_start function** + +Header: +`virtual int set_start(set_handle_t *handle, const char *key, size_t final_data_size, uint32_t create_flags);` + +Pseudo code: +- Find file `key` under the FileSystemStore directory. If not existing, increase `_num_keys` by 1. +- Take `_mutex` +- Open file for reading and read data into a `key_metadata_t` structure +- If existing and flag "write once" is preset, return "write once" error +- Close file +- Allocate an `inc_set_handle_t` structure into `handle` +- Duplicate `key` in `handle`. +- Update `create_flags` in `handle` +- Fill `key_metadata_t` structure with all relevant values (`create_flags` from handle) +- Open file `key` for writing to achieve a file handle +- Write metadata structure to the file +- Close file + +**set_add_data function** + +Header: +`virtual int set_add_data(set_handle_t handle, const void *value_data, size_t data_size);` + +Pseudo code: +- Open file `key` for appending to achieve a file handle +- Write `value_data` to the file +- Close file + +**set_finalize function** + +Header: +`virtual int set_finalize(set_handle_t handle);` + +Pseudo code: +- Free `key` in `handle` and then `handle` +- Release `_mutex` + +### Key iterator APIs + +**iterator_open function** + +Header: +`virtual int iterator_open(iterator_t *it, const char *prefix = NULL);` + +Pseudo code: +- Take `_mutex` +- Allocate a `key_iterator_handle_t` structure into `it` +- Duplicate `prefix` into same field in iterator +- Using direcory `open` API, open FileSystemStore directory and store dir handle in the handle's `dir_handle` field +- Release `_mutex` + +**iterator_next function** + +Header: +`virtual int iterator_next(iterator_t it, char *key, size_t key_size);` + +Pseudo code: +- Take `_mutex` +- Using direcory `read` API on handle's `dir_handle` field, read next file in directory +- While not reached end of directory + - If name matches prefix + - Copy file name to `key` and return OK + - Using direcory `read` API on handle's `dir_handle` field, read next file in directory +- Return "not found" error +- Release `_mutex` + +**iterator_close function** + +Header: +`virtual int iterator_close(iterator_t it);` + +Pseudo code: +- Using direcory `close` API on `dir_handle` close handle +- Release `prefix` field in iterator and structure allocated at `it` + + +# Usage scenarios and examples + +### Standard usage of the class + +Following example code shows standard usage of the FileSystemStore class + +**Standard usage example** + +```C++ + +// External file system of LittleFS type. Should be initialized. +extern LittleFileSystem fs; + +// Instantiate fsstore with our file system +FileSystemStore fsstore(&fs); + +int res; + +// Initialize fsstore +res = fsstore.init(); + +// Add "Key1" +const char *val1 = "Value of key 1"; +const char *val2 = "Updated value of key 1"; +res = fsstore.set("Key1", val1, sizeof(val1), 0); +// Update value of "Key1" +res = fsstore.set("Key1", val2, sizeof(val2), 0); + +uint_8 value[32]; +size_t actual_size; +// Get value of "Key1". Value should return the updated value. +res = fsstore.get("Key1", value, sizeof(value), &actual_size); + +// Remove "Key1" +res = fsstore.remove("Key1"); + +// Incremental write, if need to generate large data with a small buffer +const int data_size = 1024; +char buf[8]; + +KVSTore::set_handle_t handle; +res = fsstore.set_start(&handle, "Key2", data_size, 0); +for (int i = 0; i < data_size / sizeof(buf); i++) { + memset(buf, i, sizeof(buf)); + res = fsstore.set_add_data(handle, buf, sizeof(buf)); +} +res = fsstore.set_finalize(handle); + +// Iterate over all keys starting with "Key" +res = 0; +KVSTore::iterator_t it; +fsstore.iterator_open(&it, "Key*"); +char key[KVSTore::KV_MAX_KEY_LENGTH]; +while (!res) { + res = fsstore.iterator_next(&it, key, sizeof(key)); +} +res = fsstore.iterator_close(&it); + +// Deinitialize FileSystemStore +res = fsstore.deinit(); +``` +# Other information + +### Open issues + +- Need to figure a way to prevent mutex abuse in incremental set APIs. diff --git a/docs/design-documents/features/storage/FileSystemStore/FileSystemStore_record.jpg b/docs/design-documents/features/storage/FileSystemStore/FileSystemStore_record.jpg new file mode 100644 index 00000000000..2d0d9d7386e Binary files /dev/null and b/docs/design-documents/features/storage/FileSystemStore/FileSystemStore_record.jpg differ diff --git a/docs/design-documents/features/storage/FileSystemStore/FileSystemStore_record.xml b/docs/design-documents/features/storage/FileSystemStore/FileSystemStore_record.xml new file mode 100644 index 00000000000..6114ab019a4 --- /dev/null +++ b/docs/design-documents/features/storage/FileSystemStore/FileSystemStore_record.xml @@ -0,0 +1 @@ +3VdLk6IwEP41VO0eZgvCQz2Ozusyl7W29hxJhNQEwoQgur9+OxJeCo47pTXlejH5upM0X3/dBMtdJNtnibP4VRDKLWSTreU+WAg5vo/gTyO7CplMvQqIJCPGqQWW7A81oG3QghGa9xyVEFyxrA+GIk1pqHoYllKUfbe14P1TMxzRI2AZYn6M/mZExRU69e0Wf6EsiuuTHdtYVjh8i6QoUnOehdz1/leZE1zvZfzzGBNRdiD30XIXUghVjZLtgnLNbU1bte5pxNrELWmqzlkQTKoVG8wLWocccFg7J2wDw0gPa2hVA684YmGNwu6rAU87VzvDZ/Be6Aear0Wq7vJ9tu/BwbGzbWs82kIeIkMHATYQ6KnYv3nfT0Y+dGxvP9R7NrTPNtV82mAuY6boMsOhtpZQHYDFKuEwc5rV3fyYlG2oVHTbgUy+nqlIqJI7cDHWqZGOKS2nllLZCtUzUNzR6MRg2JRG1GzcygMGRiHDapnZn1MLVZhghW9UMPv2dFkJotuWYKO5ExoMrqVBZ1SDDZE/6ebHKYL/69yc0R/8a+XG+1R/+JVTedkCe+I4ym+03dz6+wm5XydAxx5XYMPkw+ir6KuZO6N0Hfdq1AUD1B0wAtfVTA/DQvLdXMJll6qPqWl5rGYKKyZSmN7N7As1vckhc94Rc2iAOGcWXIK56ajodEc5q9X4J1vN2E3q7B5UxTGicyBZ9TOWKyne6EJwIQFJRUp1yIzzAwhzFulEhpA66OHuXKeMwXfUvTEkjBB9zKBE+vVFmIRPuUoYuSh02q5xV0G1rLtFNaQN79+LCqbtp9ve1vk+dh//Ag== \ No newline at end of file diff --git a/docs/design-documents/features/storage/KVStore/KVStore_classes.jpg b/docs/design-documents/features/storage/KVStore/KVStore_classes.jpg new file mode 100644 index 00000000000..72d89facd00 Binary files /dev/null and b/docs/design-documents/features/storage/KVStore/KVStore_classes.jpg differ diff --git a/docs/design-documents/features/storage/KVStore/KVStore_classes.xml b/docs/design-documents/features/storage/KVStore/KVStore_classes.xml new file mode 100644 index 00000000000..4e0a0cef4a4 --- /dev/null +++ b/docs/design-documents/features/storage/KVStore/KVStore_classes.xml @@ -0,0 +1 @@ +7VjbbuIwEP2aPLZKnITSx1J6kVYrVWK1l0eTDIlVJ84aQ8N+/drxODdAi6ou7UN5gPh4PB6fOeOJ8MLbon6QtMq/ihS4R/y09sK5R0gQkYn+McjOIldBYIFMshSNOmDB/gCCPqIblsJ6YKiE4IpVQzARZQmJGmBUSvEyNFsJPty1ohnsAYuE8n30B0tVbtFp7Hf4I7AsdzsHPs4safKcSbEpcT+PhKvmY6cL6nyh/TqnqXjpQeGdF95KIZR9Kupb4IZbR5tdd39kto1bQqlOWoBxbCnfgAt5wvXaWWXCUzukZPJ7Y2KaFVRmrPTCGz3rV7X+1mBzMoNfKFHZuag3p6BWF5SzDNclOjiQnU/9lOFvs/PSAd/ms4USEtyEPsdybKyxaozl0sTuVOW2CY7vmLLtwdPmmGQTNTEnOuZAR9H4cCgZuCNbkIpped1YEuYNTTOkZM5hZVYJbbXijRpWTGcpnK1EqbA4AoLje1owbsrqEfgWjFcTpyq4MWr37isARWFigLoHoSIeQBSg5E6b4GwYoSqweokbv3S1EDgs79XBFDGK5Ze1rjsJ6gdU4RFFRh9ZkV++fwry/QU5Pace4wN6HLEJZXpjuo4eLblInk0C6TqHFCnQ8/cNgXPfjvoc9qjSDMndT2N2Gbvhr3/RqLTOQQ2LB9JBf9snts/cBFmSwKli22ELPEQdunsSTIfSJikKjtwazsVabGQCuKrfk0aO4quRo3DkyJ54z1GTyfaMpyX36iNfNlozsNitFRSfl86rLp29G+ZAAR29dIJ37YLTjyzMBSQbCZ+iPL8o4/gdRdk27jO0woN97CSm/kNrm5Dry/htmlsUDhPYhvj2zc1l5yxvLjVT9sUlinFoXlwu/EufOOAJJNPh6xvGejs1xTrghtx+yz5L2ok/Kjb/lUkPr0dJj6/fKOl62P1ZYM27f2TCu78= \ No newline at end of file diff --git a/docs/design-documents/features/storage/KVStore/KVStore_design.md b/docs/design-documents/features/storage/KVStore/KVStore_design.md new file mode 100644 index 00000000000..91718490a08 --- /dev/null +++ b/docs/design-documents/features/storage/KVStore/KVStore_design.md @@ -0,0 +1,463 @@ +# KVStore in Mbed OS + +- [KVStore in Mbed OS](#kvstore-in-mbed-os) + + [Revision history](#revision-history) +- [Introduction](#introduction) + + [Overview and background](#overview-and-background) + + [Requirements and assumptions](#requirements-and-assumptions) +- [System architecture and high-level design](#system-architecture-and-high-level-design) + * [Design basics](#design-basics) + + [Derived implementations](#derived-implementations) + * [Global Key Value interface](#global-key-value-interface) +- [Detailed design](#detailed-design) + * [KVStore class design](#kvstore-class-design) + + [KVStore Class header](#kvstore-class-header) + * [Global Key Value interface design](#global-key-value-interface-design) + + [Global Key Value APIs](#global-key-value-apis) + * [Mapping APIs](#mapping-apis) + * [Implementation](#implementation) + + [Important data structures](#important-data-structures) + + [Global Key Value API implementation](#global-key-value-api-implementation) + + [Attachment API implementation](#attachment-api-implementation) +- [Usage scenarios and examples](#usage-scenarios-and-examples) + + [Standard usage of the KVStore class](#standard-usage-of-the-kvstore-class) + + [Standard usage of the Global Key Value interface](#standard-usage-of-the-global-key-value-interface) +- [Other information](#other-information) + + [Open issues](#open-issues) + + +### Revision history + +| Revision | Date | Authors | Mbed OS version | Comments | +|---------- |---------------- |-------------------------------------------------------- |----------------- |------------------ | +| 1.0 | 26 September 2018 | David Saada ([@davidsaada](https://github.com/davidsaada/)) | 5.11+ | Initial revision | + +# Introduction + +### Overview and background + +KVStore is an interface class whose purpose is to define APIs for a Key Value Store like storage over a block device. + +### Requirements and assumptions + +# System architecture and high-level design + +## Design basics + +KVStore defines a simple, key value store like API set via this interface class. Classes implementing this interface store pairs of keys and values, where the keys are represented as character strings and the values are represented as binary blobs. Core APIs here are *get* and *set*, providing read and write access by key to the value in a single call. *remove* completes the set of core APIs. This simplifies the interface for the cases we need an actual key value store (like configurations). +APIs also support an "incremental set" mode, allowing the implementing class to aggregate chunks of data for the set operation. This is for the case the caller needs to generate large portions of data, but doesn't wish to allocate large buffers for a single set operation. Note that *get* API doesn't have or require this functionality. Instead, it has an offset parameter (defaulting to 0) allowing the calling layer to extract portions of the data. +Interface also includes iteration APIs, to let the user iterate over all available keys, given a prefix. +As some of the implementations use files as keys, key names must comply to file naming rules, meaning that characters like * , / etc. are not allowed in key names. + +### Derived implementations +![KVStore Classes](./KVStore_classes.jpg) + +KVStore currently has a few derived implementations. +- [TDBStore](../TDBStore/TDBStore_design.md) should be chosen as the default solution, as it gives the best performance, flash wear leveling and lowest overhead for a limited number of keys. +- [FileSystemStore](../FileSystemStore/FileSystemStore_design.md) is the preferred solution, if we already have a file system and don't wish to have an additional one, or if specific POSIX features (like file seeking) are required. It's also preferred if we don't have a limitaion for the number of keys. +- [SecureStore](../SecureStore/SecureStore_design.md) adds security features such as encryption, rollback protection and authentication. It uses one of the other KVStore solutions as the underlying storage type. + +## Global Key Value interface + +A parallel key-value API is provided as global C-style functions (for all functions, except for the incremental set ones). This API performs a limited type of mapping of partition or mount point names present in the keys. For each of the APIs defined in KVStore, the global version will extract a partition prefix from the key name. The prefix must be in the form "/partition/key-name". Then a lookup will be performed to map the partition name to a concrete KVStore instance, and the API call routed to that instance. The routed key name will have the partition prefix stripped, leaving only "key-name". +In the case of iteration APIs, the prefix must include the partition (in the form of "/partition/prefix"). + + +# Detailed design + +## KVStore class design + +As an interface class, KVStore has no implementation, just a class header. + +### KVStore Class header + +```C++ +class KVStore { + + enum create_flags { + WRITE_ONCE_FLAG = (1 << 0), + ENCRYPT_FLAG = (1 << 1), + AUTHENTICATE_FLAG = (1 << 2), + ROLLBACK_PROTECT_FLAG = (1 << 3), + }; + + static const uint32_t MAX_KEY_LENGTH = 128; + + typedef struct _opaque_set_handle *set_handle_t; + + typedef struct _opaque_key_iterator *iterator_t; + + typedef struct info { + size_t size; + uint32_t flags; + } info_t; + + // Initialization and reset + virtual int init(); + virtual int deinit(); + virtual int reset(); + + // Core API + virtual int set(const char *key, const void *buffer, size_t size, uint32_t create_flags); + virtual int get(const char *key, void *buffer, size_t buffer_size, size_t *actual_size = NULL, size_t offset = 0); + virtual int get_info(const char *key, info_t *info); + virtual int remove(const char *key); + + // Incremental set API + virtual int set_start(set_handle_t *handle, const char *key, size_t final_data_size, uint32_t create_flags); + virtual int set_add_data(set_handle_t handle, const void *value_data, size_t data_size); + virtual int set_finalize(set_handle_t handle); + + // Key iterator + virtual int iterator_open(iterator_t *it, const char *prefix = NULL); + virtual int iterator_next(iterator_t it, char *key, size_t key_size); + virtual int iterator_close(iterator_t it); +} +``` + +## Global Key Value interface design + +As mentioned above, each KVStore API has a parallel C-style API, used globally with a partition name preceding the key name. + +### Global Key Value APIs + +```C +enum kv_create_flags { + KV_WRITE_ONCE_FLAG = (1 << 0), + KV_ENCRYPT_FLAG = (1 << 1), + KV_AUTHENTICATE_FLAG = (1 << 2), + KV_ROLLBACK_PROTECT_FLAG = (1 << 3), +}; + +static const uint32_t KV_MAX_KEY_LENGTH = 128; +typedef struct _opaque_set_handle *kv_set_handle_t; +typedef struct _opaque_key_iterator *kv_key_iterator_handle_t; + +typedef struct info { + size_t size; + uint32_t flags; +} kv_info_t; + +// Core API +int kv_set(const char *full_name_key, const void *buffer, size_t size, uint32_t create_flags); +int kv_get(const char *full_name_key, void *buffer, size_t buffer_size, size_t *actual_size); +int kv_get_info(const char *full_name_key, kv_info_t *info); +int kv_remove(const char *full_name_key); + +// Key iterator +int kv_iterator_open(kv_key_iterator_handle_t *it, const char *full_prefix = nullptr); +int kv_iterator_next(kv_key_iterator_handle_t it, char *key, size_t key_size); +int kv_iterator_close(kv_key_iterator_handle_t it); +``` + +## Mapping APIs + +In order to use the global C style APIs, we need APIs to map the partition name to the instance of the implementing KVStore class, typically called once at initialization time. So, for example a "/tdbstore/key1" name means that we wish to access "key1" key in a TDBStore instance. This means that we will need to attach "tdbstore" string to the TDBStore instance at initialization time. +These APIs will be part of a different header file ("kv_map.h"), as they serve the integration code and not the KVStore user code. +APIs are the following: + +```C +// Attach and detach +int kv_init(); +int kv_attach(const char *partition_name, KVStore *kv_instance); +int kv_detach(const char *partition_name); + +// Full name lookup and then break it into KVStore instance and key +int kv_lookup(const char *full_name, KVStore& *kv_instance, char *key); +``` + +## Implementation + +Following is the implementation of the Global Key Value interface and of the attachment APIs. KVStore class has no implemetation as it's an interface class. + +### Important data structures + +```C +// incremental set handle +typedef struct { + KVStore *kvstore_intance; + KVStore::set_handle_t *set_handle; +} kv_inc_set_handle_t; + +// iterator handle +typedef struct { + KVStore *kvstore_intance; + KVStore::iterator_t *iterator_handle; +} kv_key_iterator_handle_t; + +const int MAX_ATTACHED_KVS 16 + +typedef struct { + char *partition_name; + KVStore *kvstore_instance; +} kv_map_entry_t; + +// Attachment table +kv_map_entry_t kv_map_table[MAX_ATTACHED_KVS]; +int kv_num_attached_kvs; +``` + +### Global Key Value API implementation + +**kv_set function** + +Header: +`int kv_set(const char *full_name_key, const void *buffer, size_t size, uint32_t create_flags);` + +Pseudo code: +- Using `kv_lookup`, break `full_name_key` into `key` and `kvs_instance` +- Call `kvs_instance` `set` method with `key` and the rest of the arguments + +**kv_get function** + +Header: +`int kv_get(const char *full_name_key, void *buffer, size_t buffer_size, size_t *actual_size);` + +Pseudo code: +- Using `kv_lookup`, break `full_name_key` into `key` and `kvs_instance` +- Call `kvs_instance` `get` method with `key` and the rest of the arguments + +**kv_get_info function** + +Header: +`int kv_get_info(const char *full_name_key, kv_info_t *info);` + +Pseudo code: +- Using `kv_lookup`, break `full_name_key` into `key` and `kvs_instance` +- Call `kvs_instance` `get_info` method with `key` and the rest of the arguments + +**kv_remove function** + +Header: +`int kv_remove(const char *full_name_key);` + +Pseudo code: +- Using `kv_lookup`, break `full_name_key` into `key` and `kvs_instance` +- Call `kvs_instance` `remove` method with `key` and the rest of the arguments + +**kv_set_start function** + +Header: +`int kv_set_start(kv_set_handle_t *handle, const char *full_name_key, size_t final_data_size);` + +Pseudo code: +- Allocate an `kv_inc_set_handle_t` structure into `handle` +- Using `kv_lookup`, break `full_name_key` into allocated `key` and `kvs_instance` (in `handle`) +- Call `kvs_instance` `set_start` method with `key` and the rest of the arguments + +**kv_set_add_data function** + +Header: +`int kv_set_add_data(kv_set_handle_t handle, const void *value_data, size_t data_size);` + +Pseudo code: +- Extract `kvs_instance` and `set_handle` from `handle` +- Call `kvs_instance` `set_add_data` method with `set_handle` and the rest of the arguments + +**kv_set_finalize function** + +Header: +`int kv_set_finalize(kv_set_handle_t handle);` + +Pseudo code: +- Extract `kvs_instance` and `set_handle` from `handle` +- Call `kvs_instance` `set_finalize` method with `set_handle` +- Free `key` and `handle` + +**kv_iterator_open function** + +Header: +`int kv_iterator_open(kv_key_iterator_handle_t *it, const char *full_prefix = nullptr);` + +Pseudo code: +- Allocate a `kv_key_iterator_handle_t` structure into `it` +- Using `kv_lookup`, break `full_name_key` into allocated `prefix` and `kvs_instance` (in `handle`) +- Call `kvs_instance` `iterator_open` method with `iterator_handle`, `prefix` and the rest of the arguments + +**kv_iterator_next function** + +Header: +`int kv_iterator_next(kv_key_iterator_handle_t it, char *key, size_t key_size);` + +Pseudo code: +- Extract `kvs_instance` and `iterator_handle` from `handle` +- Call `kvs_instance` `iterator_next` method with `iterator_handle` and the rest of the arguments + +**kv_iterator_close function** + +Header: +`int kv_iterator_close(kv_key_iterator_handle_t it);` + +Pseudo code: +- Extract `kvs_instance` and `iterator_handle` from `handle` +- Call `kvs_instance` `set_finalize` method with `iterator_handle` +- Free `prefix` and `handle` + +### Attachment API implementation + +**kv_init function** + +Header: +`int kv_init();` + +Pseudo code: +- Set `kv_num_attached_kvs` to 0 + +**kv_attach function** + +Header: +`int kv_attach(const char *partition_name, KVStore *kv_instance);` + +Pseudo code: +- Duplicate `partition_name` and `kv_instance` to last entry in `kv_map_table` +- Increment `kv_num_attached_kvs` + +**kv_detach function** + +Header: +`int kv_detach(const char *partition_name);` + +Pseudo code: +- Look for entry with `partition_name` in `kv_map_table` +- Deallocate `partition_name` in this entry +- Copy all preceding entries back one position +- Decrement `kv_num_attached_kvs` + +**kv_lookup function** + +Header: +`int kv_lookup(const char *full_name, KVStore& *kv_instance, char *key);` + +Pseudo code: +- Break `full_name` string to `partition_name` and `key` +- Look for entry with `partition_name` in `kv_map_table` +- Extract `kv_instance` from table entry + +# Usage scenarios and examples + +### Standard usage of the KVStore class + +Following example code shows standard usage of the KVStore, using the TDBStore class + +**Standard usage example - with class APIs** + +```C++ +// Underlying block device. Here, SPI Flash is fully used. +// One can use SlicingBlockDevice if we want a partition. +SPIFBlockDevice bd(PTE2, PTE4, PTE1, PTE5); + +// Instantiate tdbstore with our block device and a maximum of 64 keys +kvstore = new TDBStore(64, &bd); + +int res; + +// Initialize storage +res = kvstore->init(); + +const char *val1 = "Value of key 1"; +const char *val2 = "Updated value of key 1"; +// Add "Key1" +res = kvstore->set("Key1", val1, sizeof(val1), 0); +// Update value of "Key1" +res = kvstore->set("Key1", val2, sizeof(val2), 0); + +uint_8 value[32]; +size_t actual_size; +// Get value of "Key1". Value should return the updated value. +res = kvstore->get("Key1", value, sizeof(value), &actual_size); + +// Remove "Key1" +res = kvstore->remove("Key1"); + +// Incremental write, if need to generate large data with a small buffer +const int data_size = 1024; +char buf[8]; + +KVSTore::set_handle_t handle; +res = kvstore->set_start(&handle, "Key2", data_size, 0); +for (int i = 0; i < data_size / sizeof(buf); i++) { + memset(buf, i, sizeof(buf)); + res = kvstore->set_add_data(handle, buf, sizeof(buf)); +} +res = kvstore->set_finalize(handle); + +// Iterate over all keys starting with "Key" +res = 0; +KVSTore::iterator_t it; +kvstore->iterator_open(&it, "Key*"); +char key[KVSTore::KV_MAX_KEY_LENGTH]; +while (!res) { + res = kvstore->iterator_next(&it, key, sizeof(key)); +} + +// Deinitialize TDBStore +res = kvstore->deinit(); +``` + +### Standard usage of the Global Key Value interface + +Following example code shows how to use the previous example with the global key value interface. Here, `tdtbstore` will be mapped to "/tdbstore/". + +**Standard usage example - with global C-style APIs** + +Assuming this code exists somewhere and called during initialization + +```C +// Assume TDBtore is already instantiated and initialized +extern TDBStore tdbstore; + +int res; + +// Attachment code. Should be called at initialization +res = kv_init(); +res = kv_attach("tdbstore", &tdbstore); + +``` + +Here's how to access kvstore via C global APIs + + +```C +const char *val1 = "Value of key 1"; +const char *val2 = "Updated value of key 1"; + +// Add "Key1", now with full name, including "/tdbstore/" prefix. +res = kv_set("/tdbstore/Key1", val1, sizeof(val1), 0); +// Update value of "Key1" +res = kv_set("/tdbstore/Key1", val2, sizeof(val2), 0); + +uint_8 value[32]; +size_t actual_size; +// Get value of "Key1". Value should return the updated value. +res = kv_get("/tdbstore/Key1", value, sizeof(value), &actual_size); + +// Remove "Key1" +res = kv_remove("/tdbstore/Key1"); + +// Incremental write, if need to generate large data with a small buffer +const int data_size = 1024; +char buf[8]; + +kv_set_handle_t handle; +res = kv_set_start(&handle, "/tdbstore/Key2", data_size, 0); +for (int i = 0; i < data_size / sizeof(buf); i++) { + memset(buf, i, sizeof(buf)); + res = kv_set_add_data(handle, buf, sizeof(buf)); +} +res = kv_set_finalize(handle); + +// Iterate over all keys starting with "Key" in tdbstore +res = 0; +kv_key_iterator_handle_t it; +kv_iterator_open(&it, "/tdbstore/Key"); +char key[KV_MAX_KEY_LENGTH]; +while (!res) { + res = kv_iterator_next(&it, key, sizeof(key)); +} +res = kv_iterator_close(&it); +``` + +# Other information + +### Open issues + diff --git a/docs/design-documents/features/storage/SecureStore/SecureStore_class_hierarchy.jpg b/docs/design-documents/features/storage/SecureStore/SecureStore_class_hierarchy.jpg new file mode 100644 index 00000000000..19047d5e665 Binary files /dev/null and b/docs/design-documents/features/storage/SecureStore/SecureStore_class_hierarchy.jpg differ diff --git a/docs/design-documents/features/storage/SecureStore/SecureStore_class_hierarchy.xml b/docs/design-documents/features/storage/SecureStore/SecureStore_class_hierarchy.xml new file mode 100644 index 00000000000..c19c4debc0f --- /dev/null +++ b/docs/design-documents/features/storage/SecureStore/SecureStore_class_hierarchy.xml @@ -0,0 +1 @@ +7VhZb+IwEP41eWwVHKD0kaOHVK1UCe31aJIhserEWWMo7K/fcTK5QVBp6bISPED8zXg8/jyHieNN4+2T5mn0RQUgHeYGW8ebOYz1BgOGPxbZ5cjdqJ8DoRYBKVXAXPwGAl1C1yKAVUPRKCWNSJugr5IEfNPAuNbqvam2VLK5aspD6ABzn8su+l0EJsrR0cCt8GcQYVSs3HNJsuD+W6jVOqH1HOYts08ujnlhi/RXEQ/Uew3yHhxvqpUy+VO8nYK03Ba05fMeD0hLvzUk5qQJ5MeGyzUULg8lzp2k1j2zI0qGv9bWp0nMdSgSxxuj1E23+I1gtjOL3xiV5rJ+TWZga264FCHN89E50JVNfArpN1t5UQBz8Nca5kZpKGS4lUVbH7G0jUXaul8EVrFS7/Cigdjs3XBE52wdZ3ZThwygF5mNAmUNc2wD2giMsHHOwyxjakKszCQs7SyFWkuZBcRS4EF5k6VKDOVHj9H4kcdC2sx6BrkBa9X6aWJplcq160FAcWF9gG0NoqB4AhWD0TtUISnzKDAogVmfxu9VOvQKLKqlwogwThkYlqarKMQHCsQDQdnrBOXXFXQYhSQY21zHkUoBWZwgUucq4KsIAiLlFH4gaJSFLjv13Q9ppxokN2LTrBz7tk/mXpXAZUumvbsW020GV2qtfaBZ9VRuGRq4RwwZTFEwHUPZaZR7PO2A2CVXjZdvRyvGQl9ryNlrSCcgP7WG9P/zEL0G5Nmb2ugz43GwJx4PN7SFVP5bu4Wh/DEjcOZ2m12NKmRI735YtdtBMfx5jMa8OzST55/0w0NV46P9sD9sGfLO1w/vL7nYzGAjfHiB3bUjXlhH9O4/sQIVfznPdauuqk695ri3fVYWoVfQAv3GqM21TitG95dzOS+L04cv56OWob92Ocdh9bogV6/eyXgPfwA= \ No newline at end of file diff --git a/docs/design-documents/features/storage/SecureStore/SecureStore_design.md b/docs/design-documents/features/storage/SecureStore/SecureStore_design.md new file mode 100644 index 00000000000..b7fba2049c5 --- /dev/null +++ b/docs/design-documents/features/storage/SecureStore/SecureStore_design.md @@ -0,0 +1,448 @@ +# SecureStore in Mbed OS + +- [SecureStore in Mbed OS](#securestore-in-mbed-os) + + [Revision history](#revision-history) +- [Introduction](#introduction) + + [Overview and background](#overview-and-background) + + [Requirements and assumptions](#requirements-and-assumptions) +- [System architecture and high-level design](#system-architecture-and-high-level-design) + * [Design basics](#design-basics) + + [Data layout](#data-layout) + + [Basic implementation concepts](#basic-implementation-concepts) +- [Detailed design](#detailed-design) + + [Class header](#class-header) + + [Important data structures](#important-data-structures) + + [Initialization and reset](#initialization-and-reset) + + [Core APIs](#core-apis) + + [Incremental set APIs](#incremental-set-apis) + + [Key iterator APIs](#key-iterator-apis) +- [Usage scenarios and examples](#usage-scenarios-and-examples) + + [Standard usage of the class](#standard-usage-of-the-class) +- [Other information](#other-information) + + [Open issues](#open-issues) + + +### Revision history + +| Revision | Date | Authors | Mbed OS version | Comments | +|---------- |---------------- |-------------------------------------------------------- |----------------- |------------------ | +| 1.0 | 02 October 2018 | David Saada ([@davidsaada](https://github.com/davidsaada/)) | 5.11+ | Initial revision | + +# Introduction + +### Overview and background + +SecureStore is a [KVStore](../KVStore/KVStore_design.md) based storage solution, providing security features on the stored data, such as encryption, authentication, rollback protection and write once, over an underlying KVStore class. It references an additional KVStore class for storing the rollback protection keys. + +### Requirements and assumptions + +SecureStore assumes that the underlying KVStore instances are instantiated and initialized. + +# System architecture and high-level design + +## Design basics + +SecureStore is a storage class, derived from KVStore. It adds security features to the underlying key value store. + +As such, it offers all KVStore APIs, with additional security options (which can be selected using the creation flags at set). These include: +- Encryption: Data is encrypted using the AES-CTR encryption method, with a randomly generated 8-byte IV. Key is derived from [Device Key](../../../../../../mbed-os/features/device_key/README.md), using the NIST SP 800-108 KDF in counter mode spec, where salt is the key trimmed to 32 bytes, with "ENC" as prefix. +- Authentication: A 16-byte CMAC is calculated on all stored data (including metadata) and stored at the end of the record. When reading the record, calculated CMAC is compared with the stored one. In case of encryption, CMAC is calculated on the encrypted data. The key used for generating the CMAC is derived from [Device Key](../../../../../../mbed-os/features/device_key/README.md), where salt is the key trimmed to 32 bytes, with "AUTH" as prefix. +- Rollback protection: (Requires authentication) CMAC is stored in a designated rollback protected storage (also of KVStore type) and compared to when reading the data under the same KVStore key. A missing or different key in the rollback protected storage would result in an error. +- Write once: Key can only be stored once and can't be removed. + + +![SecureStore Layers](./SecureStore_layers.jpg) + + +### Data layout + +When storing the data, it is stored with a preceding metadata header. Metadata includes flags and security related parameters such as IV. The CMAC, calculated for authentication, is stored at the end of the data as it is calculated on the fly, so it can't be stored with the metadata. + +![SecureStore Record](./SecureStore_record.jpg) + +Fields are: +- Metadata size: Size of metadata header +- Revision: SecureStore revision (currently 1) +- Data size: Size of user data +- Flags: User flags +- IV: Random generated IV +- Pad: Pad data to a multiple of 16 bytes (due to encryption) +- CMAC: CMAC calculated on key, metadata & data + +### Basic implementation concepts + +When setting the data, as the code can't construct a single buffer to store all data (including metadata and possibly encrypted data) in one shot, it will be done in chunks, using the incremental set APIs. As for get, it will use the offset argument to extract metadata, data & CMAC separately. +Rollback protection (RBP) keys are stored in the designated rollback protection storage, of KVStore type also. RBP keys are the same as the SecureStore keys. + + +# Detailed design + +![SecureStore Class Hierarchy](./SecureStore_class_hierarchy.jpg) + +Functionality, as defined by KVStore, includes the following: +- Initialization & reset +- Core actions: get, set & remove +- Incremental set actions +- Iterator actions + +### Class header + +SecureStore has the following header: + +```C++ +class SecureStore : KVStore { + +public: + SecureStore(KVStore *underlying_kv, KVStore *rbp_kv); + virtual ~SecureStore(); + + // Initialization and formatting + virtual int init(); + virtual int deinit(); + virtual int reset(); + + // Core API + virtual int set(const char *key, const void *buffer, size_t size, uint32_t create_flags); + virtual int get(const char *key, void *buffer, size_t buffer_size, size_t *actual_size = NULL, size_t offset = 0); + virtual int get_info(const char *key, info_t *info); + virtual int remove(const char *key); + + // Incremental set API + virtual int set_start(set_handle_t *handle, const char *key, size_t final_data_size, uint32_t create_flags); + virtual int set_add_data(set_handle_t handle, const void *value_data, size_t data_size); + virtual int set_finalize(set_handle_t handle); + + // Key iterator + virtual int iterator_open(iterator_t *it, const char *prefix = NULL); + virtual int iterator_next(iterator_t it, char *key, size_t key_size); + virtual int iterator_close(iterator_t it); + +private: + Mutex _mutex; + KVStore *_underlying_kv; + KVStore *_rbp_kv; + void *_entropy; + uint8_t *_scratch_buf; +} +``` + +### Important data structures + +```C++ +// Record header +typedef struct { + uint16_t metadata_size; + uint16_t revision; + uint32_t data_size; + uint32_t create_flags; + uint8_t iv[8]; +} record_metadata_t; + +// incremental set handle +typedef struct { + record_metadata_t metadata; + bd_size_t offset; + char *key; + void *encrypt_handle; + void *auth_handle; + KVStore::set_handle_t underlying_handle; +} inc_set_handle_t; + +// iterator handle +typedef struct { + KVStore::iterator_t underlying_it; +} key_iterator_handle_t; +``` + + +### Initialization and reset + +**init function** + +Header: +`virtual int init();` + +Pseudo code: +- if `_is_initialized` return OK +- Take `_mutex` +- Initialize `_entropy` with TLS entropy APIs +- Using `DeviceKey` APIs, get the device key +- Allocate `_scratch_buf` as a 32 byte array +- Set `_is_initialized` to true +- Release `_mutex` + +**deinit function** + +Header: +`virtual int deinit();` + +Pseudo code: +- if not `_is_initialized` return OK +- Take `_mutex` +- Deinitialize `_entropy` +- Deallocate `_scratch_buf` +- Release `_mutex` + +**reset function** + +Header: +`virtual int reset();` + +Pseudo code: +- Take `_mutex` +- Call `_underlying_kv` `reset` API +- Call `_rbp_kv` `reset` API +- Release `_mutex` + +### Core APIs + +**set function** + +Header: +`virtual int set(const char *key, const void *buffer, size_t size, uint32_t create_flags);` + +Pseudo code: +- Call `set_start` with all fields and a local `set_handle_t` variable +- Call `set_add_data` with `buffer` and `size` +- Call `set_finalize` +- Return OK + +**get function** + +Header: +`virtual int get(const char *key, void *buffer, size_t buffer_size, size_t *actual_size = NULL, size_t offset = 0);` + +Pseudo code: +- if not `_is_initialized` return error +- Take `_mutex` +- Call `_underlying_kv` `get` API with `metadata` size into a `metadata` local structure +- If failure + - If rollback protection flag set + - Call `_rbp_kv` `get` API on a local `rbp_cmac` variable, key is `key`, size 16 + - If no error, return "RBP authentication" error + - Return "Key not found error" +- If authentication flag set + - Derive a key from device key and `key` + - Allocate and initialize `auth_handle` CMAC calculation local handle with derived key + - Using `auth_handle` handle, calculate CMAC on `key` & `metadata` +- If encrypt flag set + - Derive a key from device key and `key` + - Allocate and initialize a local `enc_handle` AES-CTR local handle with derived key and `iv` field +- Set `data_size` local variable to data size in metadata +- Set `actual_size` to the minimum of `buffer_size` and `data_size` +- Set `current_offset` to 0 +- While `data_size` > 0 + - If `current_offset` between `offset` and `actual_size` + - Set `dest_buf` to `buffer` and `chunk_size` to `actual_size` + - Else + - Set `dest_buf` to `_scratch_buf` and `chunk_size` to `actual_size` + - Call `_underlying_kv` `get` API with `dest_buf` and `chunk_size` + - If authentication flag set, calculate CMAC on `dest_buf`, using `_auth_handle` handle + - If encrypt flag set, decrypt `dest_buf` (in place) using `_enc_handle` handle + - Decrement `data_size` by `chunk_size` +- Call `_underlying_kv` `get` API with on a local `read_cmac` variable, size 16 +- Generate CMAC on local `cmac` variable +- Using `mbedtls_ssl_safer_memcmp` function, compare `read_cmac` with `cmac`. Return "data corrupt error" if no match. +- If rollback protection flag set + - Call `_rbp_kv` `get` API on a local `rbp_cmac` variable, key is `key`, size 16 + - If `rbp_cmac` doesn't match `cmac`, clear `buffer` and return "RBP authentication" error +- Deinitialize and free `auth_handle` and `enc_handle` +- Release `_mutex` +- Return OK + +**get_info function** + +Header: +`virtual int get_info(const char *key, info_t *info);` + +Pseudo code: +- if not `_is_initialized` return error +- Call `get` API with `key` and 0 in `buffer_size` parameter +- If failed, return error code +- Call `_underlying_kv` `get` API with `metadata` size and `key` +- Fill fields in `info` according to `metadata` +- Return OK + +**remove function** + +Header: +`virtual int remove(const char *key);` + +Pseudo code: +- if not `_is_initialized` return error +- Take `_mutex` +- Call `_underlying_kv` `get` API with `metadata` size and `key` +- If not found, return "Not found" error +- If write once flag set, return "Already exists" error +- Call `_underlying_kv` `remove` API with `key` +- If rollback protect flag set, Call `_rbp_kv` `remove` API with `key` as key +- Return OK + +### Incremental set APIs + +**set_start function** + +Header: +`virtual int set_start(set_handle_t *handle, const char *key, size_t final_data_size, uint32_t create_flags);` + +Pseudo code: +- Take `_mutex` +- Allocate an `inc_set_handle_t` and assign in handle +- if flags include write once flag + - Call `_underlying_kv` `get_info` API + - If key exists, return "already exists" error + - Call `_rbp_kv` `get` API with `key` as key. If key exists, return "already exists" error +- If encrypt flag set + - Derive a key from device key and `key` as salt (trimmed to 32 bytes with "ENC" as prefix) + - Using TLS entropy function on `_entropy` handle, randomly generate `iv` field + - Allocate and initialize `enc_handle` AES-CTR handle field with derived key and `iv` field +- Fill all available fields in `metadata` +- If authentication flag set + - Derive a key from device key and `key` as salt (trimmed to 32 bytes with "AUTH" as prefix) + - Allocate and initialize `auth_handle` CMAC calculation handle field with derived key + - Using `auth_handle` handle, calculate CMAC on `key` & `metadata` +- Call `_underlying_kv` `set_start` API +- Call `_underlying_kv` `set_add_data` API with `metadata` field +- Return OK + +**set_add_data function** + +Header: +`virtual int set_add_data(set_handle_t handle, const void *value_data, size_t data_size);` + +Pseudo code: + +- If `offset` + `data_size` > data size in handle, return error +- If flags include encryption + - Iterate over `value_data` field in chunks of `_scratch_buf` size + - Using `enc_handle` handle field, encrypt chunk into `_scratch_buf` + - If authentication flag set, using `auth_handle` handle field, update CMAC of `_scratch_buf` + - Call `_underlying_kv` `set_add_data` API with `_scratch_buf` +- Else + - If authentication flag set, using `auth_handle` handle field, update CMAC of `value_data` + - Call `_underlying_kv` `set_add_data` API with `value_data` +- Update `offset` field in handle +- Return OK + +**set_finalize function** + +Header: +`virtual int set_finalize(set_handle_t handle);` + +Pseudo code: + +- Initialize a local `cmac` 16-byte array to 0. +- If authentication flag set, using `auth_handle` handle field, generate `cmac` +- Call `_underlying_kv` `set_add_data` API with `cmac` +- Call `_underlying_kv` `set_finalize` +- If rollback protect flag set, Call `_rbp_kv` `set` API with `key` as key and `cmac` as data +- Deinitialize and free `auth_handle` and `enc_handle` +- Free `handle` +- Release `_mutex` +- Return OK + +### Key iterator APIs + +**iterator_open function** + +Header: +`virtual int iterator_open(iterator_t *it, const char *prefix = NULL);` + +Pseudo code: +- Allocate a `key_iterator_handle_t` structure into `it` +- Take `_mutex` +- Call `_underlying_kv` `iterator_open` with `underlying_it` field +- Release `_mutex` +- Return OK + +**iterator_next function** + +Header: +`virtual int iterator_next(iterator_t it, char *key, size_t key_size);` + +Pseudo code: +- Take `_mutex` +- Call `_underlying_kv` `iterator_next` with `underlying_it` field +- Release `_mutex` +- Return OK + +**iterator_close function** + +Header: +`virtual int iterator_close(iterator_t it);` + +Pseudo code: +- Take `_mutex` +- Call `_underlying_kv` `iterator_close` with `underlying_it` field +- Release `_mutex` +- Deallocate `it` +- Return OK + + +# Usage scenarios and examples + +### Standard usage of the class + +Following example code shows standard usage of the SecureStore class + +**Standard usage example** + +```C++ +// Underlying key value store - here TDBStore (should be instantiated and initialized) +extern TDBStore tdbstore; + +// Rollback protect store - also of TDBStore type (should be instantiated and initialized) +extern TDBStore rbp_tdbstore; + +// Instantiate SecureStore with tdbstore as underlying key value store and rbp_tdbstore as RBP storage +SecureStore secure_store(&tdbstore, &rbp_tdbstore); + +int res; + +// Initialize secure_store +res = secure_store.init(); + +const char *val1 = "Value of key 1"; +const char *val2 = "Updated value of key 1"; +// Add "Key1" with encryption and authentication flags +res = secure_store.set("Key1", val1, sizeof(val1), KVSTore::ENCRYPT_FLAG | KVSTore::AUTHENTICATE_FLAG); +// Update value of "Key1" (flags must be the same per key) +res = secure_store.set("Key1", val2, sizeof(val2), KVSTore::ENCRYPT_FLAG | KVSTore::AUTHENTICATE_FLAG); + +uint_8 value[32]; +size_t actual_size; +// Get value of "Key1". Value should return the updated value. +res = secure_store.get("Key1", value, sizeof(value), &actual_size); + +// Remove "Key1" +res = secure_store.remove("Key1"); + +// Incremental write, if need to generate large data with a small buffer +const int data_size = 1024; +char buf[8]; + +KVSTore::set_handle_t handle; +res = secure_store.set_start(&handle, "Key2", data_size, 0); +for (int i = 0; i < data_size / sizeof(buf); i++) { + memset(buf, i, sizeof(buf)); + res = secure_store.set_add_data(handle, buf, sizeof(buf)); +} +res = secure_store.set_finalize(handle); + +// Iterate over all keys starting with "Key" +res = 0; +KVSTore::iterator_t it; +secure_store.iterator_open(&it, "Key*"); +char key[KVSTore::KV_MAX_KEY_LENGTH]; +while (!res) { + res = secure_store.iterator_next(&it, key, sizeof(key)e); +} +res = secure_store.iterator_close(&it); + +// Deinitialize SecureStore +res = secure_store.deinit(); +``` +# Other information + +### Open issues +- Need to figure a way to prevent mutex abuse in incremental set APIs. diff --git a/docs/design-documents/features/storage/SecureStore/SecureStore_layers.jpg b/docs/design-documents/features/storage/SecureStore/SecureStore_layers.jpg new file mode 100644 index 00000000000..75b2c800665 Binary files /dev/null and b/docs/design-documents/features/storage/SecureStore/SecureStore_layers.jpg differ diff --git a/docs/design-documents/features/storage/SecureStore/SecureStore_layers.xml b/docs/design-documents/features/storage/SecureStore/SecureStore_layers.xml new file mode 100644 index 00000000000..e4789c966be --- /dev/null +++ b/docs/design-documents/features/storage/SecureStore/SecureStore_layers.xml @@ -0,0 +1 @@ +7VlZc9owEP41zLQP6WD5iPMYkh4znc50Ss9HIS+2JsKiQlz99ZWwZLAtBxoMpUdeYq3O3e/bQ6jn301WrwWeZu94AqyH+smq59/3EPLCEKl/WrIuJNdxUAhSQRMzaCsY0h9ghH0jndMEZpWBknMm6bQqJDzPgciKDAvBl9VhY86qu05xCg3BkGDWlH6hicwKaRz2t/I3QNPM7uz1Tc8Ik4dU8Hlu9ushf7z5K7on2K5lxs8ynPDljsh/2fPvBOey+Jqs7oBp21qzFfNetfSW5xaQy0MmRNfFjAVmc7BHjpiaO0joQh9Qro1Rou9zfarBmOfyaraB7FYN8ILpatupvlLzn9nBx68ysoKhlSiNRq2jgMwFDCUXUHaJ+mDXAkpWnLch3tjCSlFFIbRBG7Q9+6p7mVEJwykmunepvEPJMjlhquVptSljd5xxsZmruYEIUfKZFPwBdnqSaBSFUbnfLqIG5AUICasdkUH4NfAJSLFWQ2xvbNi2LulatJdbbgeWwdkOry3fsXGntFx6Syn1YVjVQsl+3Eoxyw7r/xZ/71EqPJVNryiD4XomYVLhxkFM6BDzBEM8dmIekRhG424wD4Ia5tdNzL3gZJjf7MW8o4jw8X7wD6CJbg5A03egWSamo+C08eKCXPh34T2OCbgj9igOg7B/Gu8t64WzeK/ntcJ9iUXBgHHycA8LSi4+3ychxEngYk+MRn7UUb5vsOessd9DR7PH74Q9j65SkuHt51/PH26edKhj4wjHKf3sNl8/36PNwS5zMt84SyZt1MIu33DVwh1lUt/hHDUjQp7c6jusauU8B80pPMs2VvWqFlRmEOuv2tovQtv8ZvpabQVJ7e4rsUjBjkGB23w75gkfiRwCGJZ0Ud3AZTGzw3tON8S2kcuvouOjmtFnfC4ImFm799o9C5Uw24UKpRsLbQAs1T4Q0z/rDv0/Xe4prp3p0lVcd5IuUXtt/Z89fwB76gnFyZ6T/biCXKV6zYazDE/1J5kLth4ITB50tN9nzK3li5ZUcZ3nqnl109UlB4XVGH0TNiyHHIYr/fU4ywWtfnfeOy0Xx9xlla1lFbgq200FsesaRoQZTTWeRCEISj7QyFGC2a3pmNAk0ds4mVJ1zC7Y4NXYEDX9yBWE6xXC08gQPh6E3SH1PPz4wBnTzwo9rekJIu6FqPlehRj9lPPXKKSvlfqZqfOc+Ne4fFn72Grf4fLlk2D3Ph9dSAL4pKyqMjPNUzXkKay5YH40yOCgTCs//NrvWL6jtOqKH6q5fXUtboHbp23/5U8= \ No newline at end of file diff --git a/docs/design-documents/features/storage/SecureStore/SecureStore_record.jpg b/docs/design-documents/features/storage/SecureStore/SecureStore_record.jpg new file mode 100644 index 00000000000..f2de3c7a703 Binary files /dev/null and b/docs/design-documents/features/storage/SecureStore/SecureStore_record.jpg differ diff --git a/docs/design-documents/features/storage/SecureStore/SecureStore_record.xml b/docs/design-documents/features/storage/SecureStore/SecureStore_record.xml new file mode 100644 index 00000000000..df94e4166d2 --- /dev/null +++ b/docs/design-documents/features/storage/SecureStore/SecureStore_record.xml @@ -0,0 +1 @@ +7VhNj5swEP01SNvDStgESI6bZHfbQ6SqkdqzAw5YazA1zld/fe1gAiQQshEoomouMW/8Mbx5nrExrFm0f+coCRfMx9SApr83rLkBIbBtKP8UcsgQdzzKgIATX3cqgCX5gzVoanRDfJxWOgrGqCBJFfRYHGNPVDDEOdtVu60Zra6aoABfAEsP0Uv0F/FFmKFj2yzwr5gEYb4yMLVlhbyPgLNNrNczoLU+/jJzhPK5dP80RD7blSDr1bBmnDGRtaL9DFPFbU5bNu6twXrym+NY3DJgov3YIrrBucsOlWOnPtnKZqCaObTKgQUWyEcC5Qa5wKqms5mKg6bU+b1R7zRds1g8p8eAv8gOwEz2hfFiCn6O1C0ksRpfr7l/FFynMz7BL1e5qHuRynywwhY8SgirIJnSvAuJwMsEecq6k1tOYqGIqHwCp9HloGsdbDEXeF+CtAjeMYuw4AfZRVvHWgd6v4Jcn7tC/a6GwpLwcwzp/RacJi40Jxtadg0SHN0lwb418kZRkA5U30+jYasROu1ytPuSo9MoxxORP/CWpITF10j+l7MFsB+YLty70sW8pVp1X0T+54f+8gNs19+oJ/0BE9wlwG8/hyqW8bDFYoF2sTi9iaX5cHNDano0c247c8Dqjbq6QnzGiLw+JarpbTg9TLm8fGHRTk3BY/YkkFDF3Jo/T8yOEhRwKsy57gVxsIY3OJ50QVxzhVQJ5aZMY1/NNDlwfg28OQVlfjTIXHIsqgFLBWcfeMYo4xKJWYyVy4TSMwhREqg4ejJyWOJTFTEir/Uv2hAR31fL1Cqkur26OCZZoKKCU00q759JnQ462T+Tu+rUd+QPOt3b5uPOpiAvNZ/kfLZ4mQ31dACcYR8P7BuKXEfHA/lYfNk72kqfT63Xvw== \ No newline at end of file diff --git a/docs/design-documents/features/storage/TDBStore/TDBStore_areas.jpg b/docs/design-documents/features/storage/TDBStore/TDBStore_areas.jpg new file mode 100644 index 00000000000..6d30463cb1c Binary files /dev/null and b/docs/design-documents/features/storage/TDBStore/TDBStore_areas.jpg differ diff --git a/docs/design-documents/features/storage/TDBStore/TDBStore_areas.xml b/docs/design-documents/features/storage/TDBStore/TDBStore_areas.xml new file mode 100644 index 00000000000..97ba384bdba --- /dev/null +++ b/docs/design-documents/features/storage/TDBStore/TDBStore_areas.xml @@ -0,0 +1 @@ +7Zhdk5owFIZ/DZedgURQL127u53p7M160esIR8hsIDRExf76JhA+zY52yth1p94Y3pOQk+c9AaOD12n5LEievPAImIPcqHTwVwchz/eR+tLKqVbmi1ktxIJGplMnbOgvMKJr1D2NoBh0lJwzSfOhGPIsg1AONCIEPw677TgbzpqTGM6ETUjYufqDRjKp1YXvdvo3oHHSzOy5JrIl4Vss+D4z8zkI76pPHU5Jcy/Tv0hIxI89CT86eC04l3UrLdfANNsGWz3u6Z1om7eATF4zIPDrEQfC9tCkXCUmTw2MajmgB7gOfjgmVMImJ6GOHpX9SktkytSVp5rnCZicDiAklD3JJPQMPAUpTqpL2dKsh5jiQb5J8thZgRamT9KzYWn6EeN+3N66I6AaBoIdiD+7DOQCAlLkdU3uaKmxTcHEHzGZuWdIsGdB0op/xcRWJAFTUzzsuFpSH07wc8+bwJei2tgr1cHz87ILqlZsvqu7bBthI0kWbfW6VwJIE1b5bcdDlFbP3cgjjxRXObSlkIK/wZozLpSS8Qx0mpSxkUQYjTN1GSq3QOkP2iWqng0rE0hpFOlprFthuFmmcD4YOo8tu8GbW6xHUzg/v5Xzq1DSA/w3vvcY/KfGL941vnXkhRSaE3JfIeQispv2wd4lwZWvEjwFw+Vlht9BJ+vdBbvllez8Cdg1tX+ZXacLy6Pi49L0ZjcsxcC7Fie+D3jzW9Yi+uy1qDjdsBbxZZxPAvRvgRrCNQQ/zVsfuSNrLE/d9rQ59Vs/sB1+Rta8QgHioJY8sufOSt5ygLJWPPrzM6W67A7wVaz3Lwl+/A0= \ No newline at end of file diff --git a/docs/design-documents/features/storage/TDBStore/TDBStore_class_hierarchy.jpg b/docs/design-documents/features/storage/TDBStore/TDBStore_class_hierarchy.jpg new file mode 100644 index 00000000000..e865ee8df75 Binary files /dev/null and b/docs/design-documents/features/storage/TDBStore/TDBStore_class_hierarchy.jpg differ diff --git a/docs/design-documents/features/storage/TDBStore/TDBStore_class_hierarchy.xml b/docs/design-documents/features/storage/TDBStore/TDBStore_class_hierarchy.xml new file mode 100644 index 00000000000..0bcd9d203cd --- /dev/null +++ b/docs/design-documents/features/storage/TDBStore/TDBStore_class_hierarchy.xml @@ -0,0 +1 @@ +7VhLb9swDP41Prbwu9mxbtoVGAYMSPc6KhZtC5UtT1ZSZ79+ki35WbcNsGU5NIfE+kiRFEV+TGJ5N3n9kaMy+8wwUMu1cW15a8t1nSDw5YdCDi1ytdJAygnWSj2wIb9Bg7ZGdwRDNVIUjFFByjEYs6KAWIwwxDl7GqsljI69liiFGbCJEZ2j3wkWWYuuArvH74GkmfHs2FqyRfFjytmu0P4s10uaVyvOkbGl9asMYfY0gLxby7vhjIn2Ka9vgKrcmrS1++4WpF3cHArxpg06jj2iOzAhh1TujUoVnjjolIS/diqmKEc8JYXlXUupXdbyXYLNyRR+IVjZyvyBTEAtLhAlqd4Xy+CA9zblU6o/G89bAzyso41gHIxAnmM7VZZYOcUyrmI3VWXcOMseMdk/e9pMX7KK2lUnWjIgo2hsGNQdmXP3wAWR5XXdJmHdpCnSKVlTSNQuJrUS2lRDQuQteVHCCqGbw3H1+g7lhKq2uge6B2VVxSlyqpQ638oh1ItF4XSlJlsYWA6CH6SK3uB6uip097q+Xj/1veAYLBv0wUpjSLdf2pnuS1A+6CpcqEhnVpFfK5hlFAp8rRpdrlgJMouRRIa5wqjKAOukzPIDeEYAr2ZnePpQn5QDRYLsx7aeO74294UR6aXLtHc1yfQ0gxXb8Rj0rmEfTwwF9iuGhOxPEDNDzW10Z3zbBbnnTBkRZfHjGvYkfmeN07LGrARPyhr+ORflp2/vY+z/j7HVKesxeKYel0fYVrHWdGhJ+V2TwLU9H2+DVMl08MMPpXYZmOXPI6edLO5mQAy76TQTcIk1jp2Afjgx5P27CRieM9lEuyQBDvh9Ep4L8fgnnYRXs+J03vDtecQnI6apZOMIo40JylmBHzJVzq3I6PoG0KTVmJLEs9FuGRcZS1mB6G2PTu9wetFbJgTLlaWaiAHHydVP62X6s4+iv5ZmrNGP4hElhi9e+YV9GX5wvdG1O3+JIYNxNXXro38jOK8YWmDI3pBRZElSwZEsKpf93xqtev/fkXf7Bw== \ No newline at end of file diff --git a/docs/design-documents/features/storage/TDBStore/TDBStore_design.md b/docs/design-documents/features/storage/TDBStore/TDBStore_design.md new file mode 100644 index 00000000000..aaa2c60aef3 --- /dev/null +++ b/docs/design-documents/features/storage/TDBStore/TDBStore_design.md @@ -0,0 +1,492 @@ +# TDBStore in Mbed OS + +- [TDBStore in Mbed OS](#tdbstore-in-mbed-os) + + [Revision history](#revision-history) +- [Introduction](#introduction) + + [Overview and background](#overview-and-background) + + [Requirements and assumptions](#requirements-and-assumptions) +- [System architecture and high-level design](#system-architecture-and-high-level-design) + * [Design basics](#design-basics) + + [Sequential writes](#sequential-writes) + + [Memory layout and areas](#memory-layout-and-areas) + + [Garbage collection](#garbage-collection) + + [RAM Table](#ram-table) +- [Detailed design](#detailed-design) + + [Class header](#class-header) + + [Important data structures](#important-data-structures) + + [Initialization and reset](#initialization-and-reset) + + [Core APIs](#core-apis) + + [Incremental set APIs](#incremental-set-apis) + + [Key iterator APIs](#key-iterator-apis) +- [Usage scenarios and examples](#usage-scenarios-and-examples) + + [Standard usage of the class](#standard-usage-of-the-class) +- [Other information](#other-information) + + [Open issues](#open-issues) + + +### Revision history + +| Revision | Date | Authors | Mbed OS version | Comments | +|---------- |---------------- |-------------------------------------------------------- |----------------- |------------------ | +| 1.0 | 16 September 2018 | David Saada ([@davidsaada](https://github.com/davidsaada/)) | 5.11+ | Initial revision | + +# Introduction + +### Overview and background + +TDBStore (Tiny Database Storage) is a lightweight module aimed for storing data on a flash storage. It is part of of the [KVStore](../KVStore/KVStore_design.md) class family, meaning that it supports the get/set interface. It is designed to optimize performance (speed of access), reduce wearing of the flash and to minimize storage overhead. It is also resilient to power failures. + +### Requirements and assumptions +TDBStore assumes that the underlying block device is fully dedicated for it (starting offset 0). If one wishes that only a part of the device is dedicated to TDBStore, then a sliced block device should be used, typically with `SlicingBlockDevice`. +In addition, this feature requires a flash based block device such as `FlashIAPBlockDevice` or `SpifBlockDevice`. It can work on top of block devices that don't need erasing before writes, such as `HeapBlockDevice` or `SDBlockDevice`, but requires a flash simulator layer for this purpose, like the one offered by `FlashSimBlockDevice`. + +# System architecture and high-level design + +## Design basics + +TDBStore includes the following design basics: +- Sequential writes: All writes are made sequentially on the physical storage as records, superseding the previous ones for the same key. +- Memory layout - areas: The physical storage is divided equally into two areas - active and standby. All writes are made to the end of the active area's free space. When the active area is exhausted, a garbage collection process is invoked, copying only the up to date values of all keys to the standby area, and turning it active. +- RAM table: Indexes all keys in RAM, thus allowing fast access to their records in the physical storage. + +### Sequential writes +All writes are made sequentially on the physical storage as records, superseding the previous ones for the same key. Each data record is written right after the last written one. If a key is updated, a new record with this key is written, overriding the previous value of this key. If a key is deleted, a new record with a "deleted" flag is added. +Writes expect the storage to be erased. However, TDBStore takes the "erase as you go" approcah, meaning that when it crosses a sector boundary, it checks whether the next sector is erased, and if not - it gets erased. This saves a lot of time during initialization and garbage collection (see below). + +### Memory layout and areas +![TDBStore Areas](./TDBStore_areas.jpg) + +Each key is stored in a separate record on the active area. The first record in the area is the master record. Its main purpose is to hold an area version, protecting us against the case we have two valid areas (can happen in the extreme cases of power failures). + + +![TDBStore Record](./TDBStore_record.jpg) + +Record key and data are preceded by a 24-byte header. Fields are: + +- Magic: A constant value, for quick validity checking +- Header size: Size of header +- Revision: TDBStore revision (currently 1) +- User flags: Flags received from user. Currently only write once is dealt with (others are ignored) +- Internal flags: Internal TDBStore flags (currently only includes deleted flag) +- Key size: Size of key +- Data size: Size of data +- CRC: A 32-bit CRC, caluclated on header (except CRC), key and data +- Programming size pad: Padding to the storage programming size + +### Garbage collection +Garbage collection (GC) is the process of compacting the records stored in the active area to the standby one, by copying only the most recent values of all the keys (without the ones marked as deleted). Then, the standby area becomes the active one and the previously active area is erased (not fully, only its first sector). +GC is invoked in the following cases: +1. When the active area is exhausted. +2. During initialization, when a corruption is found while scanning the active area. In this case, GC is performed up to the record preceding the corruption. + +### Reserved space +Active area includes a fixed and small reserved space. This space is used for a quick storage and extraction of a write once data (like device key). Its size is 32 bytes, aligned up to the underlying block device. Once it is written, it can't be modified. It is also copied between the areas during garbage collection process. + +### RAM Table + +All keys are indexed in memory using a RAM table. Key names are represented by a 32-bit hash. The table includes the hash (and sorted by it) and the offset to the key record in the block device. This allows both fast searching in the table as well as a low memory footprint. In order to keep code simple, the same CRC function, used for recored validation, is used for hash calculation (as TLS hash calculation is too heavy). + +![TDBStore RAM Table](./TDBStore_ram_table.jpg) + +Key names may produce duplicate hash values. This is OK, as the hash is only used for fast access to the key, and the key needs to be verified when accessing the storage. If the key doesn't match, we'll move to the next duplicate in the table. + + +# Detailed design + +TDBStore fully implements the KVStore interface over a block device. Due to the fact it may write to the block device in program units that don't have to match the underlying device program units, it should use a `BufferedBlockDevice` for that purpose. + +![TDBStore Class Hierarchy](./TDBStore_class_hierarchy.jpg) + +Functionality, as defined by KVStore, includes the following: +- Initialization & reset +- Core actions: get, set & remove +- Incremental set actions +- Iterator actions + +### Class header + +TDBStore has the following header: + +```C++ +class TDBStore : KVStore { + +public: + TDBSTore(BlockDevice *bd = 0); + virtual ~TDBSTore(); + + // Initialization and reset + virtual int init(); + virtual int deinit(); + virtual int reset(); + + // Core API + virtual int set(const char *key, const void *buffer, size_t size, uint32_t create_flags); + virtual int get(const char *key, void *buffer, size_t buffer_size, size_t *actual_size = NULL, size_t offset = 0); + virtual int get_info(const char *key, info_t *info); + virtual int remove(const char *key); + + // Incremental set API + virtual int set_start(set_handle_t *handle, const char *key, size_t final_data_size, uint32_t create_flags); + virtual int set_add_data(set_handle_t handle, const void *value_data, size_t data_size); + virtual int set_finalize(set_handle_t handle); + + // Key iterator + virtual int iterator_open(iterator_t *it, const char *prefix = NULL); + virtual int iterator_next(iterator_t it, char *key, size_t key_size); + virtual int iterator_close(iterator_t it); + + // Reserved space APIs + virtual int reserved_space_set(void *data); + virtual int reserved_space_get(void *data); + +private: + Mutex _mutex; + void *_ram_table; + size_t *_max_keys; + size_t *_num_keys; + BlockDevice *_bd; + bd_addr_t _free_space_offset; + BufferedBlockDevice *_buff_bd; + bool _is_initialized; + int _active_area; + + // Important internal functions + + // find record offset in flash + int find_record(const char *key, uint32_t *hash, bd_size_t *bd_offset, size_t ram_table_ind); + + // garbage collection + int garbage_collection(const char *key, const void *buffer, size_t size, uint32_t create_flags); +} +``` + +### Important data structures + +```C++ +// RAM table entry +typedef struct { + uint32_t hash; + bd_size_t bd_offset; +} ram_table_entry_t; + +// Record header +typedef struct { + uint32_t magic; + uint16_t header_size; + uint16_t revision; + uint32_t user_flags; + uint16_t int_flags; + uint16_t key_size; + uint32_t data_size; + uint32_t crc; +} record_header_t; + +// incremental set handle +typedef struct { + record_header_t header; + bd_size_t bd_base_offset; + bd_size_t bd_curr_offset; + uint32_t ram_table_ind; + uint32_t hash; + bool new_key; +} inc_set_handle_t; + +// iterator handle +typedef struct { + size_t ram_table_ind; + char *prefix; +} key_iterator_handle_t; +``` + + +### Initialization and reset + +**init function** + +Header: +`virtual int init();` + +Pseudo code: +- if `_is_initialized` return OK +- Take `_mutex` +- Set `_max_keys` to an initial value of 32 +- Allocate `_ram_table` as an array of `_max_keys` +- Allocate `_buff_bd` with `_bd` as the underlying block device and initialize it +- Check validity of master records on both areas +- If one is valid, set its area as `_active_area` +- If both are valid, set the one area whose master record has the higher version as `_active_area`. Erase first sector of the other one. +- If none are valid, set area 0 as `_active_area`, and write master record with version 0. +- Traverse active area until reaching an erased sector + - Read current record and check its validity (calculte CRC) + - If not valid, perform garbage collection and exit loop + - Advance `_free_space_offset` + - Call `find_record` function to calculate hash and find key + - If not found, add new RAM table entry with current hash + - Update position of key in RAM table +- Set `_is_initialized` to true +- Release `_mutex` + +**deinit function** + +Header: +`virtual int deinit();` + +Pseudo code: +- if not `_is_initialized` return OK +- Take `_mutex` +- Deinitialize `_buff_bd` and free it +- Free `_ram_table` +- Set `_is_initialized` to false +- Release `_mutex` + +**reset function** + +Header: +`virtual int reset();` + +Pseudo code: +- Take `_mutex` +- Erase first sector in both areas +- Set `_active_area` to 0 +- Write a master record with version 0 +- Set `_free_space_offset` to end of master record +- Set `_num_keys` to 0 +- Release `_mutex` + +### Core APIs + +**set function** + +Header: +`virtual int set(const char *key, const void *buffer, size_t size, uint32_t create_flags);` + +Pseudo code: +- if not `_is_initialized` return "not initialized" error +- Call `set_start` with all fields and a local `set_handle_t` variable +- Call `set_add_data` with `buffer` and `size` +- Call `set_finalize` +- Return OK + +**get function** + +Header: +`virtual int get(const char *key, void *buffer, size_t buffer_size, size_t *actual_size = NULL, size_t offset = 0);` + +Pseudo code: +- if not `_is_initialized` return "not initialized" error +- Take `_mutex` +- Call `find_record` to find record in storage +- If not found, return "not found" error +- Read header and calculate CRC on it +- Update CRC with key (if offset is 0) +- Read data into user buffer, starting offset. Actual size is minimum of buffer size and remainder of data +- If offset is 0 + - Update CRC with buffer + - Compare calculate CRC with header CRC. Return "data corrupt" error if different. +- Release `_mutex` +- Return OK + +**get_info function** + +Header: +`virtual int get_info(const char *key, info_t *info);` + +Pseudo code: +- if not `_is_initialized` return "not initialized" error +- Take `_mutex` +- Call `find_record` to find record in storage +- If not found, return "not found" error +- Read header +- Copy relevant fields from header into structure +- Release `_mutex` +- Return OK + +**remove function** + +Header: +`virtual int remove(const char *key);` + +Pseudo code: +- Call `set` function with `key`, delete flag set in flags and empty data + +### Incremental set APIs + +**set_start function** + +Header: +`virtual int set_start(set_handle_t *handle, const char *key, size_t final_data_size, uint32_t create_flags);` + +Pseudo code: +- Take `_mutex` +- Check if final size fits in free space, if not call `garbage_collection` +- Call `find_record` to find record in storage and achieve `ram_table_ind` and `hash` +- If found and `flags` field in header includes write once flag, return "write once" error +- Set `new_key` field in handle to true if not found and delete key not set +- Allocate an `inc_set_handle_t` structure into `handle` +- Calculate hash on `key` and update in `handle` +- Update `bd_base_offset` in handle to `_free_space_offset` +- Update a `record_header_t` structure with all relevant values +- Update all header fields in `handle` +- Calculate crc on header +- Update `ram_table_ind` and `hash` in `handle` +- Program key in position after header +- Advance `_free_space_offset` and update in `bd_curr_offset` field in handle +- Set `_free_space_offset` and update in `bd_curr_offset` field in handle +- Call `find_record` to calculate hash and find record in storage (with null key and current hash) + +**set_add_data function** + +Header: +`virtual int set_add_data(set_handle_t handle, const void *value_data, size_t data_size);` + +Pseudo code: +- Calculate crc on `value_data` and update in handle +- Program `value_data` from `bd_curr_offset` +- Advance `bd_curr_offset` + +**set_finalize function** + +Header: +`virtual int set_finalize(set_handle_t handle);` + +Pseudo code: +- Advance `_free_space_offset` to padded offset +- Update a `record_header_t` structure with all relevant values +- Program header at `bd_base_offset` from handle with pads +- Call `sync` on buffered block device +- If delete flag set + - Remove entry in index `ram_table_ind` from ram table +- Else if `new_key` field is true + - If `_num_keys` = `_max_keys` + - Increase _max_keys by 1 + - Duplicate ram table to with new `_max_keys` entries + - Add entry `ram_table_ind` +- Update `bd_offset` and `hash` in `ram_table_ind` position of ram table +- Free `handle` +- Release `_mutex` + +### Key iterator APIs + +**iterator_open function** + +Header: +`virtual int iterator_open(iterator_t *it, const char *prefix = NULL);` + +Pseudo code: +- Take `_mutex` +- Allocate a `key_iterator_handle_t` structure into `it` +- Set `ram_table_ind` field in iterator to 0 +- Duplicate `prefix` into same field in iterator +- Release `_mutex` + +**iterator_next function** + +Header: +`virtual int iterator_next(iterator_t it, char *key, size_t key_size);` + +Pseudo code: +- Take `_mutex` +- While `ram_table_ind` field in iterator smaller than `_num_keys` + - Read key pointed to by ram table in `ram_table_ind` into a local variable + - If name matches prefix + - Advance `ram_table_ind` field in iterator + - Copy name to `key` and return OK + - Advance `ram_table_ind` field in iterator +- Return "not found" error +- Release `_mutex` + +**iterator_close function** + +Header: +`virtual int iterator_close(iterator_t it);` + +Pseudo code: +- Release `prefix` field in iterator and structure allocated at `it` + + +### Reserved space + +**reserved_space_set function** + +Header: +`virtual int reserved_space_set(void *data);` + +Pseudo code: +- Check if reserved space is not empty, if it is, return a "reserved space programmed error" +- Copy `data` contents to reserved space location + +**reserved_space_get function** + +Header: +`virtual int reserved_space_get(void *data);` + +Pseudo code: +- Copy contents from reserved space location `data` + + +# Usage scenarios and examples + +### Standard usage of the class + +Following example code shows standard usage of the TDBStore class + +**Standard usage example** + +```C++ +// Underlying block device. Here, SPI Flash is fully used. +// One can use SlicingBlockDevice if we want a partition. +SPIFBlockDevice bd(PTE2, PTE4, PTE1, PTE5); + +// Instantiate tdbstore with our block device +TDBStore tdbstore(&bd); + +int res; + +// Initialize tdbstore +res = tdbstore.init(); + +// Add "Key1" +const char *val1 = "Value of key 1"; +const char *val2 = "Updated value of key 1"; +res = tdbstore.set("Key1", val1, sizeof(val1), 0); +// Update value of "Key1" +res = tdbstore.set("Key1", val2, sizeof(val2), 0); + +uint_8 value[32]; +size_t actual_size; +// Get value of "Key1". Value should return the updated value. +res = tdbstore.get("Key1", value, sizeof(value), &actual_size); + +// Remove "Key1" +res = tdbstore.remove("Key1"); + +// Incremental write, if need to generate large data with a small buffer +const int data_size = 1024; +char buf[8]; + +KVSTore::set_handle_t handle; +res = tdbstore.set_start(&handle, "Key2", data_size, 0); +for (int i = 0; i < data_size / sizeof(buf); i++) { + memset(buf, i, sizeof(buf)); + res = tdbstore.set_add_data(handle, buf, sizeof(buf)); +} +res = tdbstore.set_finalize(handle); + +// Iterate over all keys starting with "Key" +res = 0; +KVSTore::iterator_t it; +tdbstore.iterator_open(&it, "Key*"); +char key[KVSTore::KV_MAX_KEY_LENGTH]; +while (!res) { + res = tdbstore.iterator_next(&it, key, sizeof(key)); +} +res = tdbstore.iterator_close(&it); + +// Deinitialize TDBStore +res = tdbstore.deinit(); +``` +# Other information + +### Open issues + +- Need to figure a way to prevent mutex abuse in incremental set APIs. diff --git a/docs/design-documents/features/storage/TDBStore/TDBStore_ram_table.jpg b/docs/design-documents/features/storage/TDBStore/TDBStore_ram_table.jpg new file mode 100644 index 00000000000..f370dc22e12 Binary files /dev/null and b/docs/design-documents/features/storage/TDBStore/TDBStore_ram_table.jpg differ diff --git a/docs/design-documents/features/storage/TDBStore/TDBStore_ram_table.xml b/docs/design-documents/features/storage/TDBStore/TDBStore_ram_table.xml new file mode 100644 index 00000000000..5638d79e1df --- /dev/null +++ b/docs/design-documents/features/storage/TDBStore/TDBStore_ram_table.xml @@ -0,0 +1 @@ +3Zlbb5swGIZ/DZedABdCL5P0JE3VpmbSuksHHLDq4Mw4DdmvnwEbAjgN3SAtrVTFvDa2ed7PB4wB5uv0jsFN9EADRAzbDFIDXBu2bTnOpfjJlH2hTDwphAwHslAlLPAfJEVTqlscoKRWkFNKON7URZ/GMfJ5TYOM0V292IqSeqsbGKKWsPAhaas/ccCjQvUcs9LvEQ4j1bJlypwl9J9DRrexbM+wwSr/K7LXUNUlyycRDOjuQAI3BpgzSnmRWqdzRDK2Cltx3+2R3LLfDMW8yw2OW9zxAskWqS7nHeN7BWMXYY4WG+hn1zthuAFmEV8TcWWJJEw2hQUrnCJR7azdB9mtF8Q4Sg8k2ac7RNeIs70oInMdFQgyfhSuXWWG7UktOjCiFKEMgLCsuoIgEpLDESYTDROXiCZmKyoe6RCO+3tLVcZFksfxVBSwwCatMkUqlL95LUslPE4fROEfcClqU3msWVp0d6nRiq4ouWGZwMzrLiWc0Wc0p4QyocQ0RlmvMSENCRIcxuLSF+Yhoc8y07AYGVOZscZBkDUz04VFHvlZEFyb/QSCV48DSxMI1kQXCH3EgXc0DkpDHmCSYbLNR+RTFug9a5hTh3RidL3jYAJ9MLw6zfAryjprjYKd05Gd0wM71+zKzu46e3wwmuWyeY5QdK2uOME44IFzxqL96WPRO2csgtM4bxkSuwKzgNCF4KdZ9F2zsWBpIr2cOvpe9d3ju7/2imXewyQ6X8C3EGpAv0L11C6g3F31He6TyzfMHiNmqtkdDMfUecNyNmKmmml5OKZvGvvfVqsE8ZFQtbyTw3+oxW7S4T2qGv2jpqqZAAaj2vnNCoydqmYKGIqqp1upGjhQHEyzM8Zsw0RgkmC/TkA8Jds/SVr5xa/s4otT0kFB6/ixtV3ikIVI7eCv9LwOgDgaIEpjiECOX+pN6ijJFr5TnB9zSTuaMzJw6jUkdMt8JG+qULfqAbZ+ClL1FE/cqie3rHzobi7q1sZ3d7F4h/ooLrr/6KJlNaNhOBs7HE6ftDHF/OkgXZnYj8EFtvqUXDP9yFR3HtMbnl+aDa8+ouk9nL47nU7fF5yy7DuQbV6I/6mfcxYJhuBnOXv/r2XYUa8YyvQBD9/FZfXRq4ia6ssiuPkL \ No newline at end of file diff --git a/docs/design-documents/features/storage/TDBStore/TDBStore_record.jpg b/docs/design-documents/features/storage/TDBStore/TDBStore_record.jpg new file mode 100644 index 00000000000..acb4a9a39a7 Binary files /dev/null and b/docs/design-documents/features/storage/TDBStore/TDBStore_record.jpg differ diff --git a/docs/design-documents/features/storage/TDBStore/TDBStore_record.xml b/docs/design-documents/features/storage/TDBStore/TDBStore_record.xml new file mode 100644 index 00000000000..fd8b2b4e74f --- /dev/null +++ b/docs/design-documents/features/storage/TDBStore/TDBStore_record.xml @@ -0,0 +1 @@ +7VnbrqIwFP0ak5mHM4GWiz4qnksyOcmJZjLPFSo0VuqUepuvnyJFQAE9RIdhoi+WtXvZrr26uys96Cx3rxytgnfmYdoDmrfrwXEPAN00gfyKkX2C2H0jAXxOPNUpA6bkN1agptA18XBU6CgYo4KsiqDLwhC7ooAhztm22G3OaHHVFfLxGTB1ET1HfxJPBAnaN7UMf8PED9KVdU1ZZshd+JytQ7VeD8D54ZOYlyidS/WPAuSxbQ6Czz3ocMZE0lruHExjblPaknEvFdaj3xyH4poBlp2M2CC6xqnLFpVjRx7ZyKYfN1NolgLvyCduisrZZyU9tUjsFZ/Wr3X8g0ZzFoqn6BDtoeyga6tdZjybgp8iZQtJrMTROt+/GF9rPS9btjAfKPw2cIg2jvnUpHkbEIGnK+TG1q3cHRILxJLKJ/04Oh8fFbIN5gLvcpCK1ytmSyz4XnZR1r6SjtpaeiqlbSZUQ0FBTqO2wpDaGv5x4kwesqEUUq6WgdZILW8YeZh3VC6H5HRbAYJuC/CouBoFmvdSoF6pwCORE7z5Vkfwfx0bu8XYGJdj8yOqTwR1sXmhyI86mkW6fuiANve82ejUmWCptY3kqKOK6XgmAldkIvteirEaKeY73t+vdnjopFwn0Gixnm12+xkjgf5ukfkv56mOn2ywxfuUrlWXs3WcOxPnoZZW1GJcUQfdTy2gkVo+OPNrL0RN85X2MRx3O5pWm9GEl69LlRVJy8SZ2mXidHg35q64aFaf0S1TZ8FWqWt2l3qkkKpw2q2mkLJ7zgkhUYBWcdNdc7ofceQusLjMTEZj8iSQICyUj08D7TbEAQsUiIP6GW+ghDfY129BXHXhH5dJV9VPZm39lAIT7DLuyf7FP8Gvrq8SdyrELqkWxbhFgrMFdhhlXCIhC3HsOaH0BEKU+HE4XRlA6RUcxYEjLqJDZVgSz4uXKRVKcZPdQAy6bRfEcNwy+aQ4KJED+Pw2ko/Z+7WDLfcSEz7/AQ== \ No newline at end of file diff --git a/features/storage/TESTS/kvstore/filesystemstore/fsst_tests/main.cpp b/features/storage/TESTS/kvstore/filesystemstore/fsst_tests/main.cpp new file mode 100644 index 00000000000..02f0729a13d --- /dev/null +++ b/features/storage/TESTS/kvstore/filesystemstore/fsst_tests/main.cpp @@ -0,0 +1,490 @@ +/* mbed Microcontroller Library + * Copyright (c) 2018 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "greentea-client/test_env.h" +#include "unity.h" +#include "utest.h" +#include "BlockDevice.h" +#include "FileSystem.h" +#include "FileSystemStore.h" +#include "mbed_trace.h" +#include "mbed_error.h" +#include "rtos/Thread.h" +#include + +#define FSST_TEST_NUM_OF_THREADS 5 + +using namespace utest::v1; +using namespace mbed; + +BlockDevice *bd = BlockDevice::get_default_instance(); + +typedef struct { + int thread_num; + FileSystemStore *fsst; +} thread_data_t; + +static void test_set_thread_job(void *data) +{ + char kv_value[12] = {"valuevalue"}; + char kv_key[6] = {"key"}; + char thread_str[3] = {0}; + int err = 0; + + thread_data_t *thread_data = (thread_data_t *)data; + int thread_num = thread_data->thread_num; + FileSystemStore *thread_fsst = (FileSystemStore *)(thread_data->fsst); + + utest_printf("\n Thread %d Started\n", thread_num); + + strcat(kv_value, itoa(thread_num, thread_str, 10)); + strcat(kv_key, itoa(thread_num, thread_str, 10)); + err = thread_fsst->set(kv_key, kv_value, strlen(kv_value) + 1, 0x2); + + TEST_ASSERT_EQUAL(0, err); +} + +void test_file_system_store_functionality_unit_test() +{ + utest_printf("Test FileSystemStore Functionality APIs..\n"); + TEST_SKIP_UNLESS(bd != NULL); + char kv_value1[64] = {"value1value1value1value1value1value1"}; + char kv_key1[16] = {"key1"}; + char kv_value2[64] = {"value2value2value2value2value2value2"}; + char kv_key2[16] = {"key2"}; + char kv_value3[64] = {"valui3valui3"}; + char kv_key3[16] = {"kei3"}; + char kv_value5[64] = {"setonce5555"}; + char kv_key5[16] = {"key5"}; + char kv_buf[64] = {0}; + char kv_name[16] = {0}; + int i_ind = 0; + int err = 0; + size_t actual_size = 0; + + err = bd->init(); + TEST_ASSERT_EQUAL(0, err); + + FileSystem *fs = FileSystem::get_default_instance(); + + err = fs->mount(bd); + if (err) { + err = fs->reformat(bd); + TEST_ASSERT_EQUAL(0, err); + } + + FileSystemStore *fsst = new FileSystemStore(fs); + + err = fsst->init(); + TEST_ASSERT_EQUAL(0, err); + + err = fsst->reset(); + TEST_ASSERT_EQUAL(0, err); + + /* Set key1 */ + err = fsst->set(kv_key1, kv_value1, 64, 0x2); + TEST_ASSERT_EQUAL(0, err); + + /* Set key2 */ + err = fsst->set(kv_key2, kv_value2, strlen(kv_value2), 0x4); + TEST_ASSERT_EQUAL(0, err); + + err = fsst->get(kv_key2, kv_buf, 64, &actual_size, 0); + TEST_ASSERT_EQUAL(0, strcmp(kv_buf, kv_value2)); + TEST_ASSERT_EQUAL(strlen(kv_value2), (int)actual_size); + TEST_ASSERT_EQUAL(0, err); + + KVStore::info_t kv_info; + err = fsst->get_info(kv_key1, &kv_info); + TEST_ASSERT_EQUAL(((int)kv_info.flags), 0x2); + TEST_ASSERT_EQUAL(0, err); + + /* Set kei3 */ + err = fsst->set(kv_key3, kv_value3, 12, 0x8); + TEST_ASSERT_EQUAL(0, err); + + /* Set key5 WRITE_ONCE Twice */ + err = fsst->set(kv_key5, kv_value5, 10, 0x1); + TEST_ASSERT_EQUAL(0, err); + + err = fsst->set(kv_key5, kv_value3, 10, 0x8); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_WRITE_PROTECTED, err); + + /* Verify value remains of first set */ + memset(kv_buf, 0, 64); + err = fsst->get(kv_key5, kv_buf, 64, &actual_size, 0); + TEST_ASSERT_EQUAL(0, strncmp(kv_buf, kv_value5, 10)); + TEST_ASSERT_EQUAL(0, err); + + /* Non existing File get fails */ + err = fsst->get("key4", kv_buf, 64, &actual_size, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_ITEM_NOT_FOUND, err); + + KVStore::iterator_t kv_it; + err = fsst->iterator_open(&kv_it, NULL); + TEST_ASSERT_EQUAL(0, err); + i_ind = 0; + while (fsst->iterator_next(kv_it, kv_name, 16) != MBED_ERROR_ITEM_NOT_FOUND) { + i_ind++; + } + TEST_ASSERT_EQUAL(i_ind, 4); /* 4 Files : key1, key2, kei3, key5 */ + fsst->iterator_close(kv_it); + TEST_ASSERT_EQUAL(0, err); + + /* Remove Write Once key5 - should fail */ + err = fsst->remove(kv_key5); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_WRITE_PROTECTED, err); + + /* Verify key5 still remains and has the same value */ + memset(kv_buf, 0, 64); + err = fsst->get(kv_key5, kv_buf, 64, &actual_size, 0); + TEST_ASSERT_EQUAL(0, strncmp(kv_buf, kv_value5, 10)); + TEST_ASSERT_EQUAL(0, err); + + fsst->iterator_open(&kv_it, "key"); + TEST_ASSERT_EQUAL(0, err); + i_ind = 0; + while (fsst->iterator_next(kv_it, kv_name, 16) != MBED_ERROR_ITEM_NOT_FOUND) { + i_ind++; + TEST_ASSERT_EQUAL(0, strncmp(kv_name, "key", strlen("key"))); + } + TEST_ASSERT_EQUAL(i_ind, 3); /* 3 Files with prefix 'key' : key1, key2, key5 */ + fsst->iterator_close(kv_it); + TEST_ASSERT_EQUAL(0, err); + + /* Verify double Remove kei3: first succeed, second fails */ + err = fsst->remove(kv_key3); + TEST_ASSERT_EQUAL(0, err); + err = fsst->remove(kv_key3); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_ITEM_NOT_FOUND, err); + + /* Verify after removing kei3 that get value/info fail */ + err = fsst->get(kv_key3, kv_buf, 64, &actual_size, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_ITEM_NOT_FOUND, err); + err = fsst->get_info(kv_key3, &kv_info); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_ITEM_NOT_FOUND, err); + + /* Reset - Verify key2 exists before reset, and not found after reset */ + err = fsst->get(kv_key2, kv_buf, 64, &actual_size, 0); + TEST_ASSERT_EQUAL(0, err); + err = fsst->reset(); + err = fsst->get(kv_key2, kv_buf, 64, &actual_size, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_ITEM_NOT_FOUND, err); + + /* Verify that even Write-Once key5 is not found after reset */ + err = fsst->get(kv_key5, kv_buf, 64, &actual_size, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_ITEM_NOT_FOUND, err); + + /* Verify that key5 Write-Once can be set again after Reset*/ + err = fsst->set(kv_key5, kv_value5, 10, 0x1); + TEST_ASSERT_EQUAL(0, err); + + err = fsst->deinit(); + TEST_ASSERT_EQUAL(0, err); + err = bd->deinit(); + TEST_ASSERT_EQUAL(0, err); +} + + + +void test_file_system_store_edge_cases() +{ + utest_printf("Test FileSystemStore Edge Cases..\n"); + TEST_SKIP_UNLESS(bd != NULL); + KVStore::info_t kv_info; + KVStore::iterator_t kv_it; + char kv_value1[64] = {"value1value1value1value1value1value1"}; + char kv_key1[16] = {"key1"}; + char kv_value2[64] = {"value2value2value2value2value2value2"}; + char kv_buf[64] = {0}; + char kv_name[16] = {0}; + + + int err = bd->init(); + TEST_ASSERT_EQUAL(0, err); + FileSystem *fs = FileSystem::get_default_instance(); + + err = fs->mount(bd); + if (err) { + err = fs->reformat(bd); + TEST_ASSERT_EQUAL(0, err); + } + + FileSystemStore *fsst = new FileSystemStore(fs); + + err = fsst->init(); + TEST_ASSERT_EQUAL(0, err); + + err = fsst->reset(); + TEST_ASSERT_EQUAL(0, err); + + size_t actual_size = 0; + + /*********************************/ + /*********** Unit Test ***********/ + /*********************************/ + /* Fail Set - key NULL */ + err = fsst->set(NULL, kv_value1, 64, 0x2); + TEST_ASSERT_NOT_EQUAL(0, err); + + /* Fail Set - Key length exceeds max */ + err = fsst->set(NULL, kv_value1, KVStore::MAX_KEY_SIZE + 10, 0x2); + TEST_ASSERT_NOT_EQUAL(0, err); + + /* Fail Set - NULL Buffer and size larger than 0 */ + err = fsst->set(kv_key1, NULL, 64, 0x2); + TEST_ASSERT_NOT_EQUAL(0, err); + + /* OK Set - NULL Buffer and Size is 0 */ + err = fsst->set(kv_key1, kv_value1, 0, 0x2); + TEST_ASSERT_EQUAL(0, err); + + /* OK Set - Set Key1 twice and get returns second value */ + err = fsst->set(kv_key1, kv_value1, 64, 0x2); + err = fsst->set(kv_key1, kv_value2, strlen(kv_value2), 0x2); + memset(kv_buf, 0, 64); + err = fsst->get(kv_key1, kv_buf, 64, &actual_size, 0); + TEST_ASSERT_EQUAL(0, strcmp(kv_buf, kv_value2)); + TEST_ASSERT_EQUAL(0, err); + + /* Fail Get - NULL Key */ + memset(kv_buf, 0, 64); + err = fsst->get(NULL, kv_buf, 64, &actual_size, 0); + TEST_ASSERT_NOT_EQUAL(0, err); + + /* OK Get - NULL buffer , size=0 */ + memset(kv_buf, 0, 64); + err = fsst->get(kv_key1, NULL, 0, &actual_size, 0); + TEST_ASSERT_EQUAL(0, err); + + /* Fail Get - NULL buffer , size>0 */ + memset(kv_buf, 0, 64); + err = fsst->get(kv_key1, NULL, 64, &actual_size, 0); + TEST_ASSERT_NOT_EQUAL(0, err); + + /* OK Get - buffer is smaller than value's actual size */ + memset(kv_buf, 0, 64); + err = fsst->get(kv_key1, kv_buf, 8, &actual_size, 0); + TEST_ASSERT_EQUAL(0, err); + + /* Fail Get - offset larger than file size */ + memset(kv_buf, 0, 64); + err = fsst->get(kv_key1, kv_buf, 8, &actual_size, 128); + TEST_ASSERT_NOT_EQUAL(0, err); + + /* Fail Remove - NULL Key */ + err = fsst->remove(NULL); + TEST_ASSERT_NOT_EQUAL(0, err); + + /* Fail Remove - Key not found */ + err = fsst->remove("key4"); + TEST_ASSERT_NOT_EQUAL(0, err); + + /* Fail Get_Info - NULL Key */ + err = fsst->get_info(NULL, &kv_info); + TEST_ASSERT_NOT_EQUAL(0, err); + + /* OK Get_Info - NULL info pointer - verifies whether a key exists */ + err = fsst->get_info(kv_key1, NULL); + TEST_ASSERT_EQUAL(0, err); + + /* Fail Get_Info - Key not found */ + err = fsst->get_info("key4", &kv_info); + TEST_ASSERT_NOT_EQUAL(0, err); + + /* Fail Iteraor Open - NULL it */ + err = fsst->iterator_open(NULL, NULL); + TEST_ASSERT_NOT_EQUAL(0, err); + + /* Fail Iteraor Next - key size is 0 */ + err = fsst->iterator_open(&kv_it, NULL); + err = fsst->iterator_next(kv_it, kv_name, 0); + TEST_ASSERT_NOT_EQUAL(0, err); + err = fsst->iterator_close(kv_it); + + /* OK Iteraor Next - empty folder, returns not found */ + err = fsst->reset(); + err = fsst->iterator_open(&kv_it, NULL); + err = fsst->iterator_next(kv_it, kv_name, 16); + TEST_ASSERT_NOT_EQUAL(0, err); + err = fsst->iterator_close(kv_it); + + /* OK Iteraor Next - 1 File in folder, first returns ok, second returns not found */ + err = fsst->set(kv_key1, kv_value1, 64, 0x2); + err = fsst->iterator_open(&kv_it, NULL); + err = fsst->iterator_next(kv_it, kv_name, 16); + TEST_ASSERT_EQUAL(0, err); + err = fsst->iterator_next(kv_it, kv_name, 16); + TEST_ASSERT_NOT_EQUAL(0, err); + err = fsst->iterator_close(kv_it); + + /* OK Iteraor Close - close after open*/ + err = fsst->iterator_open(&kv_it, NULL); + err = fsst->iterator_close(kv_it); + TEST_ASSERT_EQUAL(0, err); + + /* Fail Set_Start - NULL handle */ + err = fsst->set_start(NULL, "key1", 64, 0x2); + TEST_ASSERT_NOT_EQUAL(0, err); + + /* Fail Set_Start - NULL key */ + KVStore::set_handle_t handle; + err = fsst->set_start(&handle, NULL, 64, 0x2); + TEST_ASSERT_NOT_EQUAL(0, err); + + /* OK Set_Finalize - finalize after start, size 0 */ + err = fsst->set_start(&handle, "key1", 0, 0x2); + err = fsst->set_finalize(handle); + TEST_ASSERT_EQUAL(0, err); + + /* Fail Set_Add_Data - NULL handle */ + err = fsst->set_add_data(NULL, "setvalue1", 10); + TEST_ASSERT_NOT_EQUAL(0, err); + + /* Fail Set_Add_Data - NULL value */ + err = fsst->set_start(&handle, "key1", 0, 0x2); + err = fsst->set_add_data(handle, NULL, 10); + TEST_ASSERT_NOT_EQUAL(0, err); + err = fsst->set_finalize(handle); + + /* OK Set_Add_Data - value size 0 */ + err = fsst->set_start(&handle, "key1", 10, 0x2); + err = fsst->set_add_data(handle, "abcde12345", 10); + memset(kv_buf, 0, 64); + err = fsst->get(kv_key1, kv_buf, 10, &actual_size, 0); + err = fsst->set_add_data(handle, "abcde12345", 0); + err = fsst->set_finalize(handle); + + /* OK Get after Finalize */ + memset(kv_buf, 0, 64); + err = fsst->get(kv_key1, kv_buf, 12, &actual_size, 0); + TEST_ASSERT_EQUAL(0, strcmp(kv_buf, "abcde12345")); + + /* FAIL SET_Add_Data - exceed final size */ + err = fsst->set_start(&handle, "key1", 10, 0x2); + err = fsst->set_add_data(handle, "abcde12345", 5); + err = fsst->set_add_data(handle, "abcde12345", 10); + TEST_ASSERT_NOT_EQUAL(0, err); + err = fsst->set_add_data(handle, "abcde12345", 5); + TEST_ASSERT_EQUAL(0, err); + err = fsst->set_finalize(handle); + + /* FAIL SET_Add_Data - final size smaller than set at start */ + err = fsst->set_start(&handle, "key1", 10, 0x2); + err = fsst->set_add_data(handle, "abcde12345", 5); + err = fsst->set_add_data(handle, "abcde12345", 3); + err = fsst->set_finalize(handle); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_INVALID_SIZE, err); + + err = fsst->deinit(); + TEST_ASSERT_EQUAL(0, err); + + err = bd->deinit(); + TEST_ASSERT_EQUAL(0, err); +} + +void test_file_system_store_multi_threads() +{ + utest_printf("\nTest Multi Threaded FileSystemStore Set Starts..\n"); + TEST_SKIP_UNLESS(bd != NULL); + char kv_buf[64] = {0}; + int err = bd->init(); + TEST_ASSERT_EQUAL(0, err); + + FileSystem *fs = FileSystem::get_default_instance(); + + err = fs->mount(bd); + + if (err) { + err = fs->reformat(bd); + TEST_ASSERT_EQUAL(0, err); + } + + FileSystemStore *fsst = new FileSystemStore(fs); + + err = fsst->init(); + TEST_ASSERT_EQUAL(0, err); + + err = fsst->reset(); + TEST_ASSERT_EQUAL(0, err); + + thread_data_t thread_data[3]; + + /* Thread Access Test Starts */ + rtos::Thread set_thread[FSST_TEST_NUM_OF_THREADS]; + + osStatus threadStatus; + int i_ind = 0; + + for (i_ind = 0; i_ind < FSST_TEST_NUM_OF_THREADS; i_ind++) { + thread_data[i_ind].fsst = fsst; + thread_data[i_ind].thread_num = i_ind + 1; + threadStatus = set_thread[i_ind].start(test_set_thread_job, (void *) & (thread_data[i_ind])); + } + + for (i_ind = 0; i_ind < FSST_TEST_NUM_OF_THREADS; i_ind++) { + set_thread[i_ind].join(); + } + + + char kv_value[12] = {"valuevalue"}; + char kv_key[6] = {"key"}; + char thread_str[3] = {0}; + + size_t actual_size = 0; + + for (i_ind = 1; i_ind < (FSST_TEST_NUM_OF_THREADS + 1); i_ind++) { + memset(kv_buf, 0, 64); + strcpy(&kv_value[10], itoa(i_ind, thread_str, 10)); + strcpy(&kv_key[3], itoa(i_ind, thread_str, 10)); + err = fsst->get(kv_key, kv_buf, 12, &actual_size, 0); + TEST_ASSERT_EQUAL(0, err); + TEST_ASSERT_EQUAL(0, strcmp(kv_value, kv_buf)); + } + + err = fsst->reset(); + TEST_ASSERT_EQUAL(0, err); + + err = fsst->deinit(); + TEST_ASSERT_EQUAL(0, err); + + err = bd->deinit(); + TEST_ASSERT_EQUAL(0, err); +} + +// Test setup +utest::v1::status_t test_setup(const size_t number_of_cases) +{ + GREENTEA_SETUP(60, "default_auto"); + return verbose_test_setup_handler(number_of_cases); +} + +Case cases[] = { + Case("Testing functionality APIs unit test", test_file_system_store_functionality_unit_test), + Case("Testing Edge Cases", test_file_system_store_edge_cases), + Case("Testing Multi Threads Set", test_file_system_store_multi_threads) +}; + +Specification specification(test_setup, cases); + + +int main() +{ + mbed_trace_init(); + utest_printf("MAIN STARTS\n"); + return !Harness::run(specification); +} + diff --git a/features/storage/TESTS/kvstore/securestore_whitebox/main.cpp b/features/storage/TESTS/kvstore/securestore_whitebox/main.cpp new file mode 100644 index 00000000000..f19d1877368 --- /dev/null +++ b/features/storage/TESTS/kvstore/securestore_whitebox/main.cpp @@ -0,0 +1,520 @@ +/* +* Copyright (c) 2018 ARM Limited. All rights reserved. +* SPDX-License-Identifier: Apache-2.0 +* Licensed under the Apache License, Version 2.0 (the License); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an AS IS BASIS, WITHOUT +* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#include "SecureStore.h" +#include "TDBStore.h" +#ifdef MBED_CONF_RTOS_PRESENT +#include "Thread.h" +#endif +#include "mbed_error.h" +#include "Timer.h" +#include "HeapBlockDevice.h" +#include "FlashSimBlockDevice.h" +#include "SlicingBlockDevice.h" +#include "greentea-client/test_env.h" +#include "unity/unity.h" +#include "utest/utest.h" +#include +#include +#include +#include + +#define LOCAL_TEST + +// MBED_TEST_SIM_BLOCKDEVICE is used here only to filter out inappropriate boards (little RAM etc.) +#if !defined(MBED_TEST_SIM_BLOCKDEVICE) && !defined(LOCAL_TEST) +#error [NOT_SUPPORTED] KVStore test not supported on this platform +#endif + +using namespace mbed; + +const size_t ul_bd_size = 16 * 4096; +const size_t rbp_bd_size = 4 * 4096; + +#undef TEST_SPIF +#undef TEST_FSSTORE_UL +#undef NO_RBP_MODE + +#ifdef TEST_SPIF +#include "SPIFBlockDevice.h" +SPIFBlockDevice flash_bd(MBED_CONF_SPIF_DRIVER_SPI_MOSI, MBED_CONF_SPIF_DRIVER_SPI_MISO, + MBED_CONF_SPIF_DRIVER_SPI_CLK, MBED_CONF_SPIF_DRIVER_SPI_CS); +SlicingBlockDevice ul_bd(&flash_bd, 0, ul_bd_size); +SlicingBlockDevice rbp_bd(&flash_bd, ul_bd_size, ul_bd_size + rbp_bd_size); +#else +HeapBlockDevice bd(ul_bd_size + rbp_bd_size, 1, 1, 4096); +FlashSimBlockDevice flash_bd(&bd); +SlicingBlockDevice ul_bd(&flash_bd, 0, ul_bd_size); +SlicingBlockDevice rbp_bd(&flash_bd, ul_bd_size, ul_bd_size + rbp_bd_size); +#endif + +#ifdef TEST_FSSTORE_UL +#include "LittleFileSystem.h" +#include "FileSystemStore.h" +#endif + +using namespace utest::v1; + +static const char *const key1 = "key1"; +static const char *const key1_val1 = "val1"; +static const char *const key2 = "name_of_key2"; +static const char *const key2_val1 = "val3"; +static const char *const key2_val2 = "val2 of key 2"; +static const char *const key2_val3 = "Val1 value of key 2 "; +static const char *const key3 = "This_is_the_name_of_key3"; +static const char *const key3_val1 = "Data value of key 3 is the following"; +static const char *const key3_val2 = "Unfollow"; +static const char *const key4 = "This_is_the_name_of_key4"; +static const char *const key4_val1 = "Is this the value of key 4?"; +static const char *const key4_val2 = "What the hell is the value of key 4, god damn it!"; +static const char *const key5 = "This_is_the_real_name_of_Key5"; +static const char *const key5_val1 = "Key 5 value that should definitely be written"; +static const char *const key5_val2 = "Key 5 value that should definitely not be written"; +static const char *const key6 = "Key6_name"; +static const char *const key6_val1 = "Value 1 of key6"; +static const char *const key6_val2 = "Value 2 of key6. That's it."; +static const char *const key7 = "Key7"; +static const char *const key7_val1 = "7 is a lucky number"; + +static void white_box_test() +{ + uint8_t get_buf[256]; + size_t actual_data_size; + int result; + mbed::Timer timer; + int elapsed; + KVStore::info_t info; + +#ifdef TEST_FSSTORE_UL + LittleFileSystem *fs = new LittleFileSystem("fs", &ul_bd); + + result = fs->mount(&ul_bd); + + if (result) { + result = fs->reformat(&ul_bd); + TEST_ASSERT_EQUAL(0, result); + } + + FileSystemStore *ul_kv = new FileSystemStore(fs); +#else + TDBStore *ul_kv = new TDBStore(&ul_bd); +#endif + +#ifdef NO_RBP_MODE + TDBStore *rbp_kv = 0; +#else + TDBStore *rbp_kv = new TDBStore(&rbp_bd); +#endif + + SecureStore *sec_kv = new SecureStore(ul_kv, rbp_kv); + + timer.reset(); + result = sec_kv->init(); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + elapsed = timer.read_ms(); + printf("Elapsed time for init %d ms\n", elapsed); + + timer.reset(); + result = sec_kv->reset(); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + elapsed = timer.read_ms(); + printf("Elapsed time for reset is %d ms\n", elapsed); + + result = sec_kv->set(key1, key1_val1, strlen(key1_val1), KVStore::ENCRYPT_FLAG | KVStore::AUTHENTICATE_FLAG); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = sec_kv->set(key2, key2_val1, strlen(key2_val1), KVStore::AUTHENTICATE_FLAG); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = sec_kv->set(key2, key2_val2, strlen(key2_val2), KVStore::ENCRYPT_FLAG | KVStore::AUTHENTICATE_FLAG); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = sec_kv->get(key2, get_buf, sizeof(get_buf), &actual_data_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + TEST_ASSERT_EQUAL(strlen(key2_val2), actual_data_size); + TEST_ASSERT_EQUAL_STRING_LEN(key2_val2, get_buf, strlen(key2_val2)); + + timer.reset(); + result = sec_kv->set(key2, key2_val3, strlen(key2_val3), KVStore::AUTHENTICATE_FLAG | KVStore::ROLLBACK_PROTECT_FLAG); + elapsed = timer.read_ms(); + printf("Elapsed time for set is %d ms\n", elapsed); + + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = sec_kv->set(key3, key3_val1, strlen(key3_val1), + KVStore::AUTHENTICATE_FLAG | KVStore::ENCRYPT_FLAG | KVStore::ROLLBACK_PROTECT_FLAG); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = sec_kv->set(key3, key3_val2, strlen(key3_val2), + KVStore::AUTHENTICATE_FLAG | KVStore::ENCRYPT_FLAG); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_INVALID_ARGUMENT, result); + + result = sec_kv->get(key3, get_buf, sizeof(get_buf), &actual_data_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + TEST_ASSERT_EQUAL(strlen(key3_val1), actual_data_size); + TEST_ASSERT_EQUAL_STRING_LEN(key3_val1, get_buf, strlen(key3_val1)); + + for (int j = 0; j < 2; j++) { + result = sec_kv->set(key4, key4_val1, strlen(key4_val1), + KVStore::AUTHENTICATE_FLAG | KVStore::ROLLBACK_PROTECT_FLAG); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = sec_kv->set(key4, key4_val2, strlen(key4_val2), + KVStore::AUTHENTICATE_FLAG | KVStore::ENCRYPT_FLAG | KVStore::ROLLBACK_PROTECT_FLAG); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + } + + result = sec_kv->get_info(key3, &info); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + TEST_ASSERT_EQUAL(KVStore::AUTHENTICATE_FLAG | KVStore::ENCRYPT_FLAG | KVStore::ROLLBACK_PROTECT_FLAG, info.flags); + TEST_ASSERT_EQUAL(strlen(key3_val1), info.size); + + result = ul_kv->get_info(key3, &info); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + TEST_ASSERT_EQUAL(KVStore::AUTHENTICATE_FLAG | KVStore::ENCRYPT_FLAG | KVStore::ROLLBACK_PROTECT_FLAG, info.flags); + +#ifndef NO_RBP_MODE + result = rbp_kv->get_info(key3, &info); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); +#endif + + result = sec_kv->remove(key3); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = sec_kv->remove(key3); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_ITEM_NOT_FOUND, result); + + result = ul_kv->get_info(key3, &info); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_ITEM_NOT_FOUND, result); + +#ifndef NO_RBP_MODE + result = rbp_kv->get_info(key3, &info); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_ITEM_NOT_FOUND, result); +#endif + + result = sec_kv->get_info(key5, &info); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_ITEM_NOT_FOUND, result); + + result = sec_kv->set(key5, key5_val1, strlen(key5_val1), + KVStore::AUTHENTICATE_FLAG | KVStore::ROLLBACK_PROTECT_FLAG | KVStore::WRITE_ONCE_FLAG); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + +#ifndef NO_RBP_MODE + result = rbp_kv->get_info(key5, &info); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + TEST_ASSERT_EQUAL(KVStore::WRITE_ONCE_FLAG, info.flags); +#endif + + result = sec_kv->set(key5, key5_val2, strlen(key5_val2), + KVStore::AUTHENTICATE_FLAG | KVStore::ROLLBACK_PROTECT_FLAG | KVStore::WRITE_ONCE_FLAG); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_WRITE_PROTECTED, result); + + result = sec_kv->remove(key5); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_WRITE_PROTECTED, result); + + result = sec_kv->get(key5, get_buf, sizeof(get_buf), &actual_data_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + TEST_ASSERT_EQUAL(strlen(key5_val1), actual_data_size); + TEST_ASSERT_EQUAL_STRING_LEN(key5_val1, get_buf, strlen(key5_val1)); + + result = sec_kv->get(key1, get_buf, sizeof(get_buf), &actual_data_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + TEST_ASSERT_EQUAL(strlen(key1_val1), actual_data_size); + TEST_ASSERT_EQUAL_STRING_LEN(key1_val1, get_buf, strlen(key1_val1)); + + timer.reset(); + result = sec_kv->get(key2, get_buf, sizeof(get_buf), &actual_data_size); + elapsed = timer.read_ms(); + printf("Elapsed time for get is %d ms\n", elapsed); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + TEST_ASSERT_EQUAL(strlen(key2_val3), actual_data_size); + TEST_ASSERT_EQUAL_STRING_LEN(key2_val3, get_buf, strlen(key2_val3)); + + result = sec_kv->get(key4, get_buf, sizeof(get_buf), &actual_data_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + TEST_ASSERT_EQUAL(strlen(key4_val2), actual_data_size); + TEST_ASSERT_EQUAL_STRING_LEN(key4_val2, get_buf, strlen(key4_val2)); + + result = sec_kv->get(key4, get_buf, 7, &actual_data_size, 30); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + TEST_ASSERT_EQUAL(7, actual_data_size); + TEST_ASSERT_EQUAL_STRING_LEN(key4_val2 + 30, get_buf, 7); + + result = sec_kv->get(key5, get_buf, sizeof(get_buf), &actual_data_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + TEST_ASSERT_EQUAL(strlen(key5_val1), actual_data_size); + TEST_ASSERT_EQUAL_STRING_LEN(key5_val1, get_buf, strlen(key5_val1)); + + KVStore::iterator_t it; + char *char_get_buf = reinterpret_cast (get_buf); + + result = sec_kv->iterator_open(&it, "This"); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = sec_kv->iterator_next(it, char_get_buf, sizeof(get_buf)); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + bool got_key4 = !strcmp(key4, char_get_buf); + bool got_key5 = !strcmp(key5, char_get_buf); + TEST_ASSERT_EQUAL(true, got_key4 || got_key5); + + result = sec_kv->iterator_next(it, char_get_buf, sizeof(get_buf)); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + if (got_key4) { + TEST_ASSERT_EQUAL_STRING(key5, char_get_buf); + } else { + TEST_ASSERT_EQUAL_STRING(key4, char_get_buf); + } + + result = sec_kv->iterator_next(it, (char *)get_buf, sizeof(get_buf)); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_ITEM_NOT_FOUND, result); + + result = sec_kv->iterator_close(it); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = sec_kv->deinit(); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + timer.reset(); + result = sec_kv->init(); + elapsed = timer.read_ms(); + printf("Elapsed time for init is %d ms\n", elapsed); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = sec_kv->get(key4, get_buf, sizeof(get_buf), &actual_data_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + TEST_ASSERT_EQUAL(strlen(key4_val2), actual_data_size); + TEST_ASSERT_EQUAL_STRING_LEN(key4_val2, get_buf, strlen(key4_val2)); + + result = sec_kv->set(key6, key6_val1, strlen(key6_val1), + KVStore::AUTHENTICATE_FLAG | KVStore::ENCRYPT_FLAG | KVStore::ROLLBACK_PROTECT_FLAG); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + +#ifndef NO_RBP_MODE + // Simulate a rollback attack + char attack_buf[sizeof(get_buf)]; + size_t attack_size; + result = ul_kv->get(key6, attack_buf, sizeof(attack_buf), &attack_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = sec_kv->set(key6, key6_val2, strlen(key6_val2), + KVStore::AUTHENTICATE_FLAG | KVStore::ENCRYPT_FLAG | KVStore::ROLLBACK_PROTECT_FLAG); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = ul_kv->set(key6, attack_buf, attack_size, + KVStore::AUTHENTICATE_FLAG | KVStore::ENCRYPT_FLAG | KVStore::ROLLBACK_PROTECT_FLAG); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = sec_kv->get_info(key6, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_RBP_AUTHENTICATION_FAILED, result); + + // Make sure encrypted data is truly encrypted + result = rbp_kv->get_info(key6, &info); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + int cmac_size = info.size; + uint8_t *cmac = new uint8_t[cmac_size]; + + result = sec_kv->set(key7, key7_val1, strlen(key7_val1), KVStore::AUTHENTICATE_FLAG); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = ul_kv->get(key7, attack_buf, sizeof(attack_buf), &attack_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + int data_offset = attack_size - cmac_size - strlen(key7_val1); + TEST_ASSERT_EQUAL(0, strncmp(key7_val1, attack_buf + data_offset, strlen(key7_val1))); + + result = sec_kv->set(key7, key7_val1, strlen(key7_val1), KVStore::AUTHENTICATE_FLAG | KVStore::ENCRYPT_FLAG); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = ul_kv->get(key7, attack_buf, sizeof(attack_buf), &attack_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + TEST_ASSERT_NOT_EQUAL(0, strncmp(key7_val1, attack_buf + data_offset, strlen(key7_val1))); + + // Simulate a wrong CMAC + result = ul_kv->get(key7, attack_buf, attack_size - cmac_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + result = ul_kv->get(key7, cmac, cmac_size, &actual_data_size, attack_size - cmac_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + cmac[0]++; + + KVStore::set_handle_t handle; + result = ul_kv->set_start(&handle, key7, attack_size, KVStore::AUTHENTICATE_FLAG | KVStore::ENCRYPT_FLAG); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = ul_kv->set_add_data(handle, attack_buf, attack_size - cmac_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + result = ul_kv->set_add_data(handle, cmac, cmac_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + result = ul_kv->set_finalize(handle); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = ul_kv->get(key7, attack_buf, sizeof(attack_buf), &attack_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = sec_kv->get_info(key7, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_AUTHENTICATION_FAILED, result); + + delete[] cmac; +#endif + + result = sec_kv->deinit(); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + delete sec_kv; + delete ul_kv; + delete rbp_kv; + +#ifdef TEST_FSSTORE_UL + delete fs; +#endif +} + +#if 0 +static void multi_set_test() +{ + char *key; + uint8_t *get_buf, *set_buf; + size_t key_size = 32; + size_t data_size = 512; + size_t num_keys = 16; + size_t set_iters = 3; + size_t actual_data_size; + int result; + mbed::Timer timer; + int elapsed; + size_t i; + uint8_t key_ind; + + timer.start(); +#if !defined(TEST_SPIF) && !defined(TEST_SD) + HeapBlockDevice heap_bd(4096 * 64, 1, 1, 4096); + FlashSimBlockDevice flash_bd(&heap_bd); +#endif + + // TODO: Fix + KVStore *kvs = new TDBStore(&flash_bd); + + timer.reset(); + result = kvs->init(); + elapsed = timer.read_ms(); + printf("Elapsed time for initial init is %d ms\n", elapsed); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + key = new char[key_size + 1]; + get_buf = new uint8_t[data_size]; + set_buf = new uint8_t[data_size]; + + srand(1); + for (i = 0; i < key_size; i++) { + // printable characters only + key[i] = rand() % 223 + 32; + } + key[key_size] = '\0'; + + for (i = 0; i < data_size; i++) { + set_buf[i] = rand() % 256; + } + + int max_set_time = 0, total_set_time = 0; + int max_get_time = 0, total_get_time = 0; + + timer.reset(); + result = kvs->reset(); + elapsed = timer.read_ms(); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + for (i = 0; i < set_iters; i++) { + for (key_ind = 0; key_ind < num_keys; key_ind++) { + key[0] = '0' + key_ind; + set_buf[0] = key_ind * (i + 1); + timer.reset(); + result = kvs->set(key, set_buf, data_size, 0); + elapsed = timer.read_ms(); + TEST_ASSERT_EQUAL(0, result); + if (elapsed > max_set_time) { + max_set_time = elapsed; + } + total_set_time += elapsed; + } + } + + for (key_ind = 0; key_ind < num_keys; key_ind++) { + key[0] = '0' + key_ind; + set_buf[0] = key_ind * set_iters; + timer.reset(); + result = kvs->get(key, get_buf, data_size, &actual_data_size); + elapsed = timer.read_ms(); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + TEST_ASSERT_EQUAL(data_size, actual_data_size); + TEST_ASSERT_EQUAL_STRING_LEN(set_buf, get_buf, data_size); + if (elapsed > max_get_time) { + max_get_time = elapsed; + } + total_get_time += elapsed; + } + + printf("set time: Total (%d * %d times) - %d ms, Average - %d ms, Max - %d ms\n", + set_iters, num_keys, total_set_time, + total_set_time / (set_iters * num_keys), max_set_time); + printf("get time: Total (%d times) - %d ms, Average - %d ms, Max - %d ms\n", + num_keys, total_get_time, + total_get_time / num_keys, max_get_time); + + result = kvs->deinit(); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + timer.reset(); + result = kvs->init(); + elapsed = timer.read_ms(); + printf("Elapsed time for init is %d ms\n", elapsed); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = kvs->deinit(); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + delete[] key; + delete[] get_buf; + delete[] set_buf; + + delete kvs; +} +#endif + + + +utest::v1::status_t greentea_failure_handler(const Case *const source, const failure_t reason) +{ + greentea_case_failure_abort_handler(source, reason); + return STATUS_CONTINUE; +} + +Case cases[] = { + Case("SecureStore: White box test", white_box_test, greentea_failure_handler), +}; + +utest::v1::status_t greentea_test_setup(const size_t number_of_cases) +{ + GREENTEA_SETUP(120, "default_auto"); + return greentea_test_setup_handler(number_of_cases); +} + +Specification specification(greentea_test_setup, cases, greentea_test_teardown_handler); + +int main() +{ + return !Harness::run(specification); +} diff --git a/features/storage/TESTS/kvstore/static_tests/main.cpp b/features/storage/TESTS/kvstore/static_tests/main.cpp new file mode 100644 index 00000000000..cb719bb50a3 --- /dev/null +++ b/features/storage/TESTS/kvstore/static_tests/main.cpp @@ -0,0 +1,798 @@ +/* Copyright (c) 2017 ARM Limited +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#include "kvstore_global_api.h" +#include "greentea-client/test_env.h" +#include "unity/unity.h" +#include "utest/utest.h" +#include "Thread.h" +#include "mbed_error.h" + +using namespace utest::v1; +using namespace mbed; + +static const char data[] = "data"; +static const char key[] = "key"; +static char buffer[20] = {}; +static const size_t data_size = 5; +static size_t actual_size = 0; +static const size_t buffer_size = 20; +static const int num_of_threads = 3; +static const char num_of_keys = 3; + +static char *keys[] = {"key1", "key2", "key3"}; + +kv_info_t info; +kv_iterator_t kvstore_it; + +#define TEST_ASSERT_EQUAL_ERROR_CODE(expected, actual) \ +TEST_ASSERT_EQUAL(expected & MBED_ERROR_STATUS_CODE_MASK, actual & MBED_ERROR_STATUS_CODE_MASK) + +/*----------------initialization------------------*/ + +//init the blockdevice +static void kvstore_init() +{ + int res = kv_reset("/kv/"); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); +} + +/*----------------set()------------------*/ + +//bad params : key is null +static void set_key_null() +{ + int res = kv_set(NULL, data, data_size, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_INVALID_ARGUMENT, res); +} + +//bad params : key length over key max size +static void set_key_length_exceeds_max() +{ + char key_max[KV_MAX_KEY_LENGTH + 1] = "/kv/"; + memset(key_max + 3, '*', KV_MAX_KEY_LENGTH - 3); + int res = kv_set(key_max, data, data_size, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_INVALID_ARGUMENT, res); +} + +//bad params : buffer is null, non zero size +static void set_buffer_null_size_not_zero() +{ + int res = kv_set(key, NULL, data_size, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_INVALID_ARGUMENT, res); +} + +//bad params : buffer full, size is 0 +static void set_buffer_size_is_zero() +{ + int res = kv_set(key, data, 0, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_reset("/kv/"); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); +} + +//set same key several times +static void set_same_key_several_time() +{ + int res = kv_set(key, data, data_size, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_set(key, data, data_size, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_set(key, data, data_size, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_reset("/kv/"); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); +} + +static void test_thread_set(char *th_key) +{ + int res = kv_set((char *)th_key, data, data_size, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); +} + +//get several keys multithreaded +static void set_several_keys_multithreaded() +{ + rtos::Thread kvstore_thread[num_of_threads]; + osStatus threadStatus; + + kvstore_thread[0].start(callback(test_thread_set, keys[0])); + kvstore_thread[1].start(callback(test_thread_set, keys[1])); + kvstore_thread[2].start(callback(test_thread_set, keys[2])); + + + for (int i = 0; i < num_of_threads; i++) { + threadStatus = kvstore_thread[i].join(); + if (threadStatus != 0) { + utest_printf("\nthread %d join failed!", i + 1); + } + } + + for (int i = 0; i < num_of_threads; i++) { + int res = kv_get(keys[i], buffer, buffer_size, &actual_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + TEST_ASSERT_EQUAL_STRING_LEN(buffer, data, data_size); + + } + + int res = kv_reset("/kv/"); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); +} + +//set key "write once" and try to set it again +static void set_write_once_flag_try_set_twice() +{ + int res = kv_set(key, data, data_size, KV_WRITE_ONCE_FLAG); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_set(key, data, data_size, KV_WRITE_ONCE_FLAG); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_WRITE_PROTECTED, res); + + res = kv_reset("/kv/"); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); +} + +//set key "write once" and try to remove it +static void set_write_once_flag_try_remove() +{ + int res = kv_set(key, data, data_size, KV_WRITE_ONCE_FLAG); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_remove(key); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_WRITE_PROTECTED, res); + + res = kv_reset("/kv/"); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); +} + +//set key value one byte size +static void set_key_value_one_byte_size() +{ + char data_one = 'a'; + int res = kv_set(key, &data_one, 1, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_get(key, buffer, buffer_size, &actual_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = strncmp(buffer, &data_one, 1); + TEST_ASSERT_EQUAL_ERROR_CODE(0, res); + memset(buffer, 0, buffer_size); + + res = kv_reset("/kv/"); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); +} + +//set key value two byte size +static void set_key_value_two_byte_size() +{ + char data_two[2] = "d"; + int res = kv_set(key, data_two, 2, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_get(key, buffer, buffer_size, &actual_size); + TEST_ASSERT_EQUAL_STRING_LEN(buffer, data_two, 1); + memset(buffer, 0, buffer_size); + + res = kv_reset("/kv/"); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); +} + +//set key value five byte size +static void set_key_value_five_byte_size() +{ + char data_five[5] = "data"; + int res = kv_set(key, data_five, 5, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_get(key, buffer, buffer_size, &actual_size); + TEST_ASSERT_EQUAL_STRING_LEN(buffer, data_five, 4); + memset(buffer, 0, buffer_size); + + res = kv_reset("/kv/"); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); +} + +//set key value fifteen byte size +static void set_key_value_fifteen_byte_size() +{ + char data_fif[15] = "data"; + int res = kv_set(key, data_fif, 15, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_get(key, buffer, buffer_size, &actual_size); + TEST_ASSERT_EQUAL_STRING_LEN(buffer, data_fif, 14); + memset(buffer, 0, buffer_size); + + res = kv_reset("/kv/"); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); +} + +//set key value seventeen byte size +static void set_key_value_seventeen_byte_size() +{ + char data_fif[17] = "data_is_everythi"; + int res = kv_set(key, data_fif, 17, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_get(key, buffer, buffer_size, &actual_size); + TEST_ASSERT_EQUAL_STRING_LEN(buffer, data_fif, 16); + memset(buffer, 0, buffer_size); + + res = kv_reset("/kv/"); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); +} + +//set several different key value byte size +static void set_several_key_value_sizes() +{ + char name[7] = "name_"; + char c[2] = {0}; + int i = 0, res = 0; + + for (i = 0; i < 30; i++) { + c[0] = i + '0'; + name[6] = c[0]; + res = kv_set(name, name, sizeof(name), 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + } + + for (i = 0; i < 30; i++) { + c[0] = i + '0'; + name[6] = c[0]; + res = kv_get(name, buffer, sizeof(buffer), &actual_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + TEST_ASSERT_EQUAL_STRING_LEN(name, buffer, sizeof(name)); + memset(buffer, 0, sizeof(buffer)); + } + + res = kv_reset("/kv/"); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); +} + +/*----------------get()------------------*/ + +//bad params : key is null +static void get_key_null() +{ + int res = kv_get(NULL, buffer, buffer_size, &actual_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_INVALID_ARGUMENT, res); +} + +//bad params : key length over key max size +static void get_key_length_exceeds_max() +{ + char key_max[KV_MAX_KEY_LENGTH + 1] = "/kv/"; + memset(key_max + 3, '*', KV_MAX_KEY_LENGTH - 3); + int res = kv_get(key_max, buffer, buffer_size, &actual_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_INVALID_ARGUMENT, res); +} + +//bad params : buffer is null, non zero size +static void get_buffer_null_size_not_zero() +{ + int res = kv_get(key, NULL, buffer_size, &actual_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_ITEM_NOT_FOUND, res); +} + +//bad params : buffer full, size is 0 +static void get_buffer_size_is_zero() +{ + int res = kv_set(key, NULL, 0, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_get(key, buffer, 0, &actual_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_reset("/kv/"); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); +} + +//buffer_size smaller than data real size +static void get_buffer_size_smaller_than_data_real_size() +{ + char big_data[25] = "data"; + + int res = kv_set(key, big_data, sizeof(big_data), 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_get(key, buffer, buffer_size, &actual_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + TEST_ASSERT_EQUAL_STRING_LEN(buffer, big_data, &actual_size); + memset(buffer, 0, buffer_size); + + res = kv_reset("/kv/"); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); +} + +//buffer_size bigger than data real size +static void get_buffer_size_bigger_than_data_real_size() +{ + int res = kv_set(key, data, data_size, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + char big_buffer[25] = {}; + res = kv_get(key, big_buffer, sizeof(big_buffer), &actual_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + TEST_ASSERT_EQUAL_STRING_LEN(big_buffer, data, &actual_size); + memset(buffer, 0, buffer_size); + + res = kv_reset("/kv/"); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); +} + +//get a non existing key +static void get_non_existing_key() +{ + int res = kv_get(key, buffer, buffer_size, &actual_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_ITEM_NOT_FOUND, res); +} + +//get a removed key +static void get_removed_key() +{ + int res = kv_set(key, data, data_size, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_remove(key); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_get(key, buffer, buffer_size, &actual_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_ITEM_NOT_FOUND, res); + + res = kv_reset("/kv/"); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); +} + +//set the same key twice and get latest data +static void get_key_that_was_set_twice() +{ + int res = kv_set(key, data, data_size, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + char new_data[] = "new_data"; + res = kv_set(key, new_data, sizeof(new_data), 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_get(key, buffer, buffer_size, &actual_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + TEST_ASSERT_EQUAL_STRING_LEN(buffer, new_data, &actual_size); + memset(buffer, 0, buffer_size); + + res = kv_reset("/kv/"); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); +} + +static void test_thread_get(const void *th_key) +{ + char buffer[5] = {0}; + + int res = kv_get((char *)th_key, buffer, sizeof(buffer), &actual_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + TEST_ASSERT_EQUAL_STRING((char *)th_key, buffer); +} + +//get several keys multithreaded +static void get_several_keys_multithreaded() +{ + rtos::Thread kvstore_thread[num_of_threads]; + osStatus threadStatus; + + for (int i = 0; i < num_of_threads; i++) { + int res = kv_set(keys[i], keys[i], strlen(keys[i]) + 1, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + } + + kvstore_thread[0].start(callback(test_thread_get, "key1")); + kvstore_thread[1].start(callback(test_thread_get, "key2")); + kvstore_thread[2].start(callback(test_thread_get, "key3")); + + for (int i = 0; i < num_of_threads; i++) { + threadStatus = kvstore_thread[i].join(); + if (threadStatus != 0) { + utest_printf("\nthread %d join failed!", i + 1); + } + } + + int res = kv_reset("/kv/"); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); +} + +/*----------------remove()------------------*/ + +//bad params : key is null +static void remove_key_null() +{ + int res = kv_remove(NULL); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_INVALID_ARGUMENT, res); +} + +//bad params : key length over key max size +static void remove_key_length_exceeds_max() +{ + char key_max[KV_MAX_KEY_LENGTH + 1] = "/kv/"; + memset(key_max + 3, '*', KV_MAX_KEY_LENGTH - 3); + int res = kv_remove(key_max); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_INVALID_ARGUMENT, res); +} + +//key doesn’t exist +static void remove_non_existing_key() +{ + char new_key[] = "/kv/remove_key"; + int res = kv_remove(new_key); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_ITEM_NOT_FOUND, res); +} + +//key already removed +static void remove_removed_key() +{ + int res = kv_set(key, data, data_size, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_remove(key); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_remove(key); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_ITEM_NOT_FOUND, res); + + res = kv_reset("/kv/"); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); +} + +//key exist - valid flow +static void remove_existed_key() +{ + int res = kv_set(key, data, data_size, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_remove(key); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_reset("/kv/"); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); +} + +/*----------------get_info()------------------*/ + +//bad params : key is null +static void get_info_key_null() +{ + int res = kv_get_info(NULL, &info); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_INVALID_ARGUMENT, res); +} + +//bad params : key length over key max size +static void get_info_key_length_exceeds_max() +{ + char key_max[KV_MAX_KEY_LENGTH + 1] = "/kv/"; + memset(key_max + 3, '*', KV_MAX_KEY_LENGTH - 3); + int res = kv_get_info(key_max, &info); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_INVALID_ARGUMENT, res); +} + +//bad params : &info is null +static void get_info_info_null() +{ + int res = kv_get_info(key, NULL); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_ITEM_NOT_FOUND, res); +} + +//get_info of non existing key +static void get_info_non_existing_key() +{ + char new_key[] = "/kv/get_info_key"; + int res = kv_get_info(new_key, &info); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_ITEM_NOT_FOUND, res); +} + +//get_info of removed key +static void get_info_removed_key() +{ + int res = kv_set(key, data, data_size, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_remove(key); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_get_info(key, &info); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_ITEM_NOT_FOUND, res); + + res = kv_reset("/kv/"); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); +} + +//get_info of existing key - valid flow +static void get_info_existed_key() +{ + int res = kv_set(key, data, data_size, KV_WRITE_ONCE_FLAG); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_get_info(key, &info); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + TEST_ASSERT_EQUAL_ERROR_CODE(info.flags, KV_WRITE_ONCE_FLAG); + + res = kv_reset("/kv/"); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); +} + +//get_info of overwritten key +static void get_info_overwritten_key() +{ + char new_key[] = "/kv/get_info_key"; + int res = kv_set(new_key, data, data_size, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + char new_data[] = "new_data"; + res = kv_set(key, new_data, sizeof(new_data), 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_get_info(key, &info); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + TEST_ASSERT_EQUAL_ERROR_CODE(info.size, sizeof(new_data)); + + res = kv_reset("/kv/"); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); +} + +/*----------------iterator_open()------------------*/ + +//bad params : it is null +static void iterator_open_it_null() +{ + int res = kv_iterator_open(NULL, NULL); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_INVALID_ARGUMENT, res); +} + +/*----------------iterator_next()------------------*/ + +//key valid, key_size 0 +static void iterator_next_key_size_zero() +{ + int res = kv_iterator_open(&kvstore_it, NULL); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + char key[KV_MAX_KEY_LENGTH]; + + res = kv_iterator_next(kvstore_it, key, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_ITEM_NOT_FOUND, res); +} + +//iteartor_next with empty list +static void iterator_next_empty_list() +{ + int res = kv_iterator_open(&kvstore_it, NULL); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + char key[KV_MAX_KEY_LENGTH]; + + res = kv_iterator_next(kvstore_it, key, sizeof(key)); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_ITEM_NOT_FOUND, res); +} + +//iterator_next for one key list +static void iterator_next_one_key_list() +{ + int res = kv_set(key, data, data_size, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_iterator_open(&kvstore_it, NULL); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + char key[KV_MAX_KEY_LENGTH]; + + res = kv_iterator_next(kvstore_it, key, sizeof(key)); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_reset("/kv/"); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); +} + +//iteartor_next with empty list (all keys removed) +static void iterator_next_empty_list_keys_removed() +{ + char new_key_1[] = "it_1"; + int res = kv_set(new_key_1, data, data_size, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + char new_key_2[] = "it_2"; + res = kv_set(new_key_2, data, data_size, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_remove(new_key_1); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_remove(new_key_2); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_iterator_open(&kvstore_it, NULL); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + char key[KV_MAX_KEY_LENGTH]; + + res = kv_iterator_next(kvstore_it, key, sizeof(key)); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_ITEM_NOT_FOUND, res); + + res = kv_reset("/kv/"); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); +} + +//iteartor_next with non matching prefix (empty list) +static void iterator_next_empty_list_non_matching_prefix() +{ + char new_key_1[] = "it_1"; + int res = kv_set(new_key_1, data, data_size, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + char new_key_2[] = "it_2"; + res = kv_set(new_key_2, data, data_size, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_iterator_open(&kvstore_it, "Key*"); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + char key[KV_MAX_KEY_LENGTH]; + + res = kv_iterator_next(kvstore_it, key, sizeof(key)); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_ITEM_NOT_FOUND, res); + + res = kv_reset("/kv/"); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); +} + +//iteartor_next with several overwritten keys +static void iterator_next_several_overwritten_keys() +{ + for (int i = 0; i < num_of_keys; i++) { + int res = kv_set(key, data, data_size, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + } + + int res = kv_iterator_open(&kvstore_it, NULL); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + char key[KV_MAX_KEY_LENGTH]; + + res = kv_iterator_next(kvstore_it, key, sizeof(key)); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_iterator_next(kvstore_it, key, sizeof(key)); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_ITEM_NOT_FOUND, res); + + res = kv_reset("/kv/"); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); +} + +//iterator_next for full list - check key names for validation +static void iterator_next_full_list() +{ + int i = 0; + for (i = 0; i < num_of_keys; i++) { + int res = kv_set(keys[i], data, data_size, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + } + + int res = kv_iterator_open(&kvstore_it, NULL); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + char temp_key[KV_MAX_KEY_LENGTH]; + + for (i = 0; i < num_of_keys; i++) { + res = kv_iterator_next(kvstore_it, temp_key, sizeof(temp_key)); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_get(temp_key, buffer, buffer_size, &actual_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + TEST_ASSERT_EQUAL_STRING(keys[i], temp_key); + } + + res = kv_reset("/kv/"); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); +} + +/*----------------iterator_close()------------------*/ + +//iterator_close right after iterator_open +static void iterator_close_right_after_iterator_open() +{ + int res = kv_iterator_open(&kvstore_it, NULL); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, res); + + res = kv_iterator_close(kvstore_it); +} + +/*----------------setup------------------*/ + +utest::v1::status_t greentea_failure_handler(const Case *const source, const failure_t reason) +{ + greentea_case_failure_abort_handler(source, reason); + return STATUS_CONTINUE; +} + +Case cases[] = { + + Case("kvstore_init", kvstore_init), //must be first + + Case("set_key_null", set_key_null, greentea_failure_handler), + Case("set_key_length_exceeds_max", set_key_length_exceeds_max, greentea_failure_handler), + Case("set_buffer_null_size_not_zero", set_buffer_null_size_not_zero, greentea_failure_handler), + Case("set_buffer_size_is_zero", set_buffer_size_is_zero, greentea_failure_handler), + Case("set_same_key_several_time", set_same_key_several_time, greentea_failure_handler), + Case("set_several_keys_multithreaded", set_several_keys_multithreaded, greentea_failure_handler), + Case("set_write_once_flag_try_set_twice", set_write_once_flag_try_set_twice, greentea_failure_handler), + Case("set_write_once_flag_try_remove", set_write_once_flag_try_remove, greentea_failure_handler), + Case("set_key_value_one_byte_size", set_key_value_one_byte_size, greentea_failure_handler), + Case("set_key_value_two_byte_size", set_key_value_two_byte_size, greentea_failure_handler), + Case("set_key_value_five_byte_size", set_key_value_five_byte_size, greentea_failure_handler), + Case("set_key_value_fifteen_byte_size", set_key_value_fifteen_byte_size, greentea_failure_handler), + Case("set_key_value_seventeen_byte_size", set_key_value_seventeen_byte_size, greentea_failure_handler), + Case("set_several_key_value_sizes", set_several_key_value_sizes, greentea_failure_handler), + + Case("get_key_null", get_key_null, greentea_failure_handler), + Case("get_key_length_exceeds_max", get_key_length_exceeds_max, greentea_failure_handler), + Case("get_buffer_null_size_not_zero", get_buffer_null_size_not_zero, greentea_failure_handler), + Case("get_buffer_size_is_zero", get_buffer_size_is_zero, greentea_failure_handler), + Case("get_buffer_size_smaller_than_data_real_size", get_buffer_size_smaller_than_data_real_size, greentea_failure_handler), + Case("get_buffer_size_bigger_than_data_real_size", get_buffer_size_bigger_than_data_real_size, greentea_failure_handler), + Case("get_non_existing_key", get_non_existing_key, greentea_failure_handler), + Case("get_removed_key", get_removed_key, greentea_failure_handler), + Case("get_key_that_was_set_twice", get_key_that_was_set_twice, greentea_failure_handler), + Case("get_several_keys_multithreaded", get_several_keys_multithreaded, greentea_failure_handler), + + Case("remove_key_null", remove_key_null, greentea_failure_handler), + Case("remove_key_length_exceeds_max", remove_key_length_exceeds_max, greentea_failure_handler), + Case("remove_non_existing_key", remove_non_existing_key, greentea_failure_handler), + Case("remove_removed_key", remove_removed_key, greentea_failure_handler), + Case("remove_existed_key", remove_existed_key, greentea_failure_handler), + + Case("get_info_key_null", get_info_key_null, greentea_failure_handler), + Case("get_info_key_length_exceeds_max", get_info_key_length_exceeds_max, greentea_failure_handler), + Case("get_info_info_null", get_info_info_null, greentea_failure_handler), + Case("get_info_non_existing_key", get_info_non_existing_key, greentea_failure_handler), + Case("get_info_removed_key", get_info_removed_key, greentea_failure_handler), + Case("get_info_existed_key", get_info_existed_key, greentea_failure_handler), + Case("get_info_overwritten_key", get_info_overwritten_key, greentea_failure_handler), + + Case("iterator_open_it_null", iterator_open_it_null, greentea_failure_handler), + + Case("iterator_next_key_size_zero", iterator_next_key_size_zero, greentea_failure_handler), + Case("iterator_next_empty_list", iterator_next_empty_list, greentea_failure_handler), + Case("iterator_next_one_key_list", iterator_next_one_key_list, greentea_failure_handler), + Case("iterator_next_empty_list_keys_removed", iterator_next_empty_list_keys_removed, greentea_failure_handler), + Case("iterator_next_empty_list_non_matching_prefix", iterator_next_empty_list_non_matching_prefix, greentea_failure_handler), + Case("iterator_next_several_overwritten_keys", iterator_next_several_overwritten_keys, greentea_failure_handler), + Case("iterator_next_full_list", iterator_next_full_list, greentea_failure_handler), + + Case("iterator_close_right_after_iterator_open", iterator_close_right_after_iterator_open, greentea_failure_handler), +}; + + +utest::v1::status_t greentea_test_setup(const size_t number_of_cases) +{ + GREENTEA_SETUP(3000, "default_auto"); + return greentea_test_setup_handler(number_of_cases); +} + +Specification specification(greentea_test_setup, cases, greentea_test_teardown_handler); + +int main() +{ + return !Harness::run(specification); +} diff --git a/features/storage/TESTS/kvstore/tdbstore_whitebox/main.cpp b/features/storage/TESTS/kvstore/tdbstore_whitebox/main.cpp new file mode 100644 index 00000000000..0296b5bf675 --- /dev/null +++ b/features/storage/TESTS/kvstore/tdbstore_whitebox/main.cpp @@ -0,0 +1,558 @@ +/* +* Copyright (c) 2018 ARM Limited. All rights reserved. +* SPDX-License-Identifier: Apache-2.0 +* Licensed under the Apache License, Version 2.0 (the License); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an AS IS BASIS, WITHOUT +* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#include "TDBStore.h" +#ifdef MBED_CONF_RTOS_PRESENT +#include "Thread.h" +#endif +#include "mbed_error.h" +#include "Timer.h" +#include "HeapBlockDevice.h" +#include "FlashSimBlockDevice.h" +#include "SlicingBlockDevice.h" +#include "greentea-client/test_env.h" +#include "unity/unity.h" +#include "utest/utest.h" +#include +#include +#include +#include + +// Define this if you want to run the test locally (requires SPIF/SD block device) +#define LOCAL_TEST + +// MBED_TEST_SIM_BLOCKDEVICE is used here only to filter out inappropriate boards (little RAM etc.) +#if !defined(MBED_TEST_SIM_BLOCKDEVICE) && !defined(LOCAL_TEST) +#error [NOT_SUPPORTED] KVStore test not supported on this platform +#endif + +using namespace mbed; + +#undef TEST_SPIF +#undef TEST_SD + +#ifdef TEST_SPIF +#include "SPIFBlockDevice.h" +SPIFBlockDevice bd(MBED_CONF_SPIF_DRIVER_SPI_MOSI, MBED_CONF_SPIF_DRIVER_SPI_MISO, + MBED_CONF_SPIF_DRIVER_SPI_CLK, MBED_CONF_SPIF_DRIVER_SPI_CS); +SlicingBlockDevice flash_bd(&bd, 0, 16 * 4096); + +#elif defined(TEST_SD) +#include "SDBlockDevice.h" +SDBlockDevice bd(MBED_CONF_SD_SPI_MOSI, MBED_CONF_SD_SPI_MISO, + MBED_CONF_SD_SPI_CLK, MBED_CONF_SD_SPI_CS); +SlicingBlockDevice slice_bd(&bd, 0, 512 * 512); +FlashSimBlockDevice flash_bd(&slice_bd); + +#else +HeapBlockDevice bd(64 * 4096, 1, 1, 4096); +FlashSimBlockDevice flash_bd(&bd); +#endif + + +using namespace utest::v1; + +typedef struct { + size_t size; + size_t read_size; + size_t prog_size; + size_t erase_size; +} bd_params_t; + +static const char *const key1 = "key1"; +static const char *const key1_val1 = "val1"; +static const char *const key2 = "name_of_key2"; +static const char *const key2_val1 = "val3"; +static const char *const key2_val2 = "val2 of key 2"; +static const char *const key2_val3 = "Val1 value of key 2 "; +static const char *const key3 = "This_is_the_name_of_key3"; +static const char *const key3_val1 = "Data value of key 3 is the following"; +static const char *const key4 = "This_is_the_name_of_key4"; +static const char *const key4_val1 = "Is this the value of key 4?"; +static const char *const key4_val2 = "What the hell is the value of key 4, god damn it!"; +static const char *const key5 = "This_is_the_real_name_of_Key5"; +static const char *const key5_val1 = "Key 5 value that should definitely be written"; +static const char *const key5_val2 = "Key 5 value that should definitely not be written"; +static const char *const res_val1 = "This should be saved as the reserved data"; +static const char *const res_val2 = "This should surely not be saved as the reserved data"; + +static void white_box_test() +{ + bd_params_t bd_params[] = { + {8192, 1, 16, 4096}, // Standard + {4096 * 4, 1, 1, 4096}, // K82F like + {8192, 64, 128, 2048}, // Awkward read and program sizes, both larger than header size + {2048, 1, 4, 128}, // Small sector and total sizes + {0, 0, 0, 0}, // Place holder for real non volatile device + }; + + int num_bds = sizeof(bd_params) / sizeof(bd_params_t); + uint8_t get_buf[256]; + size_t actual_data_size; + int result; + mbed::Timer timer; + int elapsed; + KVStore::info_t info; + +#if defined(TEST_SPIF) || defined(TEST_SD) + flash_bd.init(); + bd_params[num_bds - 1].size = flash_bd.size(); + bd_params[num_bds - 1].read_size = flash_bd.get_read_size(); + bd_params[num_bds - 1].prog_size = flash_bd.get_program_size(); + bd_params[num_bds - 1].erase_size = flash_bd.get_erase_size(); + flash_bd.deinit(); +#endif + + timer.start(); + for (int bd_num = 0; bd_num < num_bds; bd_num++) { + bd_params_t *bdp = &bd_params[bd_num]; + if (!bdp->size) { + break; + } + printf("\n\nBD #%d: size %d, read %d, prog %d, erase %d\n", + bd_num, bdp->size, bdp->read_size, bdp->prog_size, bdp->erase_size); + HeapBlockDevice heap_bd(bdp->size, bdp->read_size, bdp->prog_size, bdp->erase_size); + FlashSimBlockDevice flash_sim_bd(&heap_bd); + BlockDevice *test_bd; + if (bd_num == num_bds - 1) { + test_bd = &flash_bd; + // Required for deinit + flash_sim_bd.init(); + } else { + test_bd = &flash_sim_bd; + } + + TDBStore *tdbs = new TDBStore(test_bd); + + timer.reset(); + result = tdbs->init(); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + elapsed = timer.read_ms(); + printf("Elapsed time for init %d ms\n", elapsed); + + timer.reset(); + result = tdbs->reset(); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + elapsed = timer.read_ms(); + printf("Elapsed time for reset is %d ms\n", elapsed); + + result = tdbs->reserved_data_set(res_val1, strlen(res_val1)); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = tdbs->reserved_data_set(res_val2, strlen(res_val2)); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_WRITE_FAILED, result); + + result = tdbs->set(key1, key1_val1, strlen(key1_val1), 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = tdbs->set(key2, key2_val1, strlen(key2_val1), 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = tdbs->set(key2, key2_val2, strlen(key2_val2), 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + timer.reset(); + result = tdbs->set(key2, key2_val3, strlen(key2_val3), 0); + elapsed = timer.read_ms(); + printf("Elapsed time for set is %d ms\n", elapsed); + + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = tdbs->set(key3, key3_val1, strlen(key3_val1), 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = tdbs->get(key3, get_buf, sizeof(get_buf), &actual_data_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + TEST_ASSERT_EQUAL(strlen(key3_val1), actual_data_size); + TEST_ASSERT_EQUAL_STRING_LEN(key3_val1, get_buf, strlen(key3_val1)); + + KVStore::set_handle_t handle; + result = tdbs->set_start(&handle, key4, 15, 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = tdbs->set_add_data(handle, key4_val2, 10); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + result = tdbs->set_add_data(handle, key4_val2 + 10, 5); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + result = tdbs->set_finalize(handle); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = tdbs->get(key4, get_buf, sizeof(get_buf), &actual_data_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + TEST_ASSERT_EQUAL(15, actual_data_size); + TEST_ASSERT_EQUAL_STRING_LEN(key4_val2, get_buf, actual_data_size); + + result = tdbs->get(key4, get_buf, 7, &actual_data_size, 4); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + TEST_ASSERT_EQUAL(7, actual_data_size); + TEST_ASSERT_EQUAL_STRING_LEN(key4_val2 + 4, get_buf, actual_data_size); + + for (int j = 0; j < 2; j++) { + result = tdbs->set(key4, key4_val1, strlen(key4_val1), 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = tdbs->set(key4, key4_val2, strlen(key4_val2), 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + } + + result = tdbs->remove(key3); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = tdbs->remove(key3); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_ITEM_NOT_FOUND, result); + + result = tdbs->get_info(key5, &info); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_ITEM_NOT_FOUND, result); + + result = tdbs->set(key5, key5_val1, strlen(key5_val1), KVStore::WRITE_ONCE_FLAG); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = tdbs->set(key5, key5_val2, strlen(key5_val2), 0); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_WRITE_PROTECTED, result); + + result = tdbs->remove(key5); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_WRITE_PROTECTED, result); + + result = tdbs->get_info(key5, &info); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + TEST_ASSERT_EQUAL(strlen(key5_val1), info.size); + TEST_ASSERT_EQUAL(KVStore::WRITE_ONCE_FLAG, info.flags); + + result = tdbs->get(key5, get_buf, sizeof(get_buf), &actual_data_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + TEST_ASSERT_EQUAL(strlen(key5_val1), actual_data_size); + TEST_ASSERT_EQUAL_STRING_LEN(key5_val1, get_buf, strlen(key5_val1)); + + for (int i = 0; i < 2; i++) { + printf("%s deinit/init\n", i ? "After" : "Before"); + result = tdbs->get(key1, get_buf, sizeof(get_buf), &actual_data_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + TEST_ASSERT_EQUAL(strlen(key1_val1), actual_data_size); + TEST_ASSERT_EQUAL_STRING_LEN(key1_val1, get_buf, strlen(key1_val1)); + + timer.reset(); + result = tdbs->get(key2, get_buf, sizeof(get_buf), &actual_data_size); + elapsed = timer.read_ms(); + printf("Elapsed time for get is %d ms\n", elapsed); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + TEST_ASSERT_EQUAL(strlen(key2_val3), actual_data_size); + TEST_ASSERT_EQUAL_STRING_LEN(key2_val3, get_buf, strlen(key2_val3)); + + result = tdbs->get(key3, get_buf, sizeof(get_buf), &actual_data_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_ITEM_NOT_FOUND, result); + + result = tdbs->get(key4, get_buf, sizeof(get_buf), &actual_data_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + TEST_ASSERT_EQUAL(strlen(key4_val2), actual_data_size); + TEST_ASSERT_EQUAL_STRING_LEN(key4_val2, get_buf, strlen(key4_val2)); + + result = tdbs->get(key5, get_buf, sizeof(get_buf), &actual_data_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + TEST_ASSERT_EQUAL(strlen(key5_val1), actual_data_size); + TEST_ASSERT_EQUAL_STRING_LEN(key5_val1, get_buf, strlen(key5_val1)); + + KVStore::iterator_t it; + char *char_get_buf = reinterpret_cast (get_buf); + + result = tdbs->iterator_open(&it, "This"); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = tdbs->iterator_next(it, char_get_buf, sizeof(get_buf)); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + bool got_key4 = !strcmp(key4, char_get_buf); + bool got_key5 = !strcmp(key5, char_get_buf); + TEST_ASSERT_EQUAL(true, got_key4 || got_key5); + + result = tdbs->iterator_next(it, char_get_buf, sizeof(get_buf)); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + if (got_key4) { + TEST_ASSERT_EQUAL_STRING(key5, char_get_buf); + } else { + TEST_ASSERT_EQUAL_STRING(key4, char_get_buf); + } + + result = tdbs->iterator_next(it, (char *)get_buf, sizeof(get_buf)); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_ITEM_NOT_FOUND, result); + + result = tdbs->iterator_close(it); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = tdbs->reserved_data_get(get_buf, strlen(res_val1)); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + TEST_ASSERT_EQUAL_STRING_LEN(res_val1, get_buf, strlen(res_val1)); + + result = tdbs->deinit(); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + timer.reset(); + result = tdbs->init(); + elapsed = timer.read_ms(); + printf("Elapsed time for init is %d ms\n", elapsed); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + } + + result = tdbs->deinit(); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + delete tdbs; + } +} + + +static void multi_set_test() +{ + char *key; + uint8_t *get_buf, *set_buf; + size_t key_size = 32; + size_t data_size = 512; + size_t num_keys = 16; + size_t set_iters = 3; + size_t actual_data_size; + int result; + mbed::Timer timer; + int elapsed; + size_t i; + uint8_t key_ind; + + timer.start(); +#if !defined(TEST_SPIF) && !defined(TEST_SD) + HeapBlockDevice heap_bd(4096 * 64, 1, 1, 4096); + FlashSimBlockDevice flash_bd(&heap_bd); +#endif + + TDBStore *tdbs = new TDBStore(&flash_bd); + + timer.reset(); + result = tdbs->init(); + elapsed = timer.read_ms(); + printf("Elapsed time for initial init is %d ms\n", elapsed); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + key = new char[key_size + 1]; + get_buf = new uint8_t[data_size]; + set_buf = new uint8_t[data_size]; + + srand(1); + for (i = 0; i < key_size; i++) { + // Alphabet characters only + key[i] = 'a' + rand() % ('z' - 'a' + 1); + } + key[key_size] = '\0'; + + for (i = 0; i < data_size; i++) { + set_buf[i] = rand() % 256; + } + + int max_set_time = 0, total_set_time = 0; + int max_get_time = 0, total_get_time = 0; + + timer.reset(); + result = tdbs->reset(); + elapsed = timer.read_ms(); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + for (i = 0; i < set_iters; i++) { + for (key_ind = 0; key_ind < num_keys; key_ind++) { + key[0] = 'A' + key_ind; + set_buf[0] = key_ind * (i + 1); + timer.reset(); + result = tdbs->set(key, set_buf, data_size, 0); + elapsed = timer.read_ms(); + TEST_ASSERT_EQUAL(0, result); + if (elapsed > max_set_time) { + max_set_time = elapsed; + } + total_set_time += elapsed; + } + } + + for (key_ind = 0; key_ind < num_keys; key_ind++) { + key[0] = 'A' + key_ind; + set_buf[0] = key_ind * set_iters; + timer.reset(); + result = tdbs->get(key, get_buf, data_size, &actual_data_size); + elapsed = timer.read_ms(); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + TEST_ASSERT_EQUAL(data_size, actual_data_size); + TEST_ASSERT_EQUAL_STRING_LEN(set_buf, get_buf, data_size); + if (elapsed > max_get_time) { + max_get_time = elapsed; + } + total_get_time += elapsed; + } + + printf("set time: Total (%d * %d times) - %d ms, Average - %d ms, Max - %d ms\n", + set_iters, num_keys, total_set_time, + total_set_time / (set_iters * num_keys), max_set_time); + printf("get time: Total (%d times) - %d ms, Average - %d ms, Max - %d ms\n", + num_keys, total_get_time, + total_get_time / num_keys, max_get_time); + + result = tdbs->deinit(); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + timer.reset(); + result = tdbs->init(); + elapsed = timer.read_ms(); + printf("Elapsed time for init is %d ms\n", elapsed); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = tdbs->deinit(); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + delete[] key; + delete[] get_buf; + delete[] set_buf; + + delete tdbs; +} + +static void error_inject_test() +{ + char *key; + uint8_t *get_buf, *set_buf, *exists; + size_t key_size = 8; + size_t data_size = 16; + size_t num_keys = 'Z' - 'A' + 1; + size_t num_blocks = 4; + size_t block_size = 1024; + size_t actual_data_size; + int result; + uint8_t set_iters = 5; + uint8_t i, key_ind; + + // Don't use a non volatile BD here (won't work in this test) + HeapBlockDevice bd(num_blocks * block_size, 1, 1, block_size); + FlashSimBlockDevice flash_bd(&bd); + + result = flash_bd.init(); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + result = flash_bd.erase(0, num_blocks * block_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + result = flash_bd.deinit(); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + TDBStore *tdbs = new TDBStore(&flash_bd); + + result = tdbs->init(); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = tdbs->reset(); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + key = new char[key_size + 1]; + get_buf = new uint8_t[data_size]; + set_buf = new uint8_t[data_size]; + exists = new uint8_t[num_keys]; + + memset(exists, 0, num_keys); + + for (i = 0; i < set_iters; i++) { + for (key_ind = 0; key_ind < num_keys; key_ind++) { + memset(key, 'A' + key_ind, key_size); + key[key_size + 1] = '\0'; + memset(set_buf, key_ind * i, data_size); + if ((key_ind != (num_keys - 1)) && exists[key_ind] && !(rand() % 3)) { + result = tdbs->remove(key); + exists[key_ind] = 0; + } else { + result = tdbs->set(key, set_buf, data_size, 0); + exists[key_ind] = 1; + } + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + } + } + + for (uint8_t get_iter = 0; get_iter < 2; get_iter++) { + for (key_ind = 0; key_ind < num_keys; key_ind++) { + memset(key, 'A' + key_ind, key_size); + key[key_size + 1] = '\0'; + if (key_ind == (num_keys - 1)) { + // last key will hold the previous version at the second iteration (after being crippled) + memset(set_buf, key_ind * (set_iters - get_iter - 1), data_size); + } else { + memset(set_buf, key_ind * (set_iters - 1), data_size); + } + result = tdbs->get(key, get_buf, data_size, &actual_data_size); + if (exists[key_ind]) { + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + TEST_ASSERT_EQUAL(data_size, actual_data_size); + TEST_ASSERT_EQUAL_STRING_LEN(set_buf, get_buf, data_size); + } else { + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_ITEM_NOT_FOUND, result); + } + } + + if (get_iter) { + break; + } + + // Cripple the last key + uint8_t erase_val = (uint8_t) flash_bd.get_erase_value(); + uint8_t x; + bd_addr_t addr; + for (addr = bd.size() - 1; addr > 0; addr--) { + bd.read(&x, addr, 1); + if (x != erase_val) { + break; + } + } + x++; + bd.program(&x, addr, 1); + + result = tdbs->get(key, get_buf, data_size, &actual_data_size); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_ERROR_INVALID_DATA_DETECTED, result); + + result = tdbs->deinit(); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + + result = tdbs->init(); + TEST_ASSERT_EQUAL_ERROR_CODE(MBED_SUCCESS, result); + } + + delete[] key; + delete[] get_buf; + delete[] set_buf; + delete[] exists; + + delete tdbs; +} + + +utest::v1::status_t greentea_failure_handler(const Case *const source, const failure_t reason) +{ + greentea_case_failure_abort_handler(source, reason); + return STATUS_CONTINUE; +} + +Case cases[] = { + Case("TDBStore: White box test", white_box_test, greentea_failure_handler), + Case("TDBStore: Multiple set test", multi_set_test, greentea_failure_handler), + Case("TDBStore: Error inject test", error_inject_test, greentea_failure_handler), +}; + +utest::v1::status_t greentea_test_setup(const size_t number_of_cases) +{ + GREENTEA_SETUP(120, "default_auto"); + return greentea_test_setup_handler(number_of_cases); +} + +Specification specification(greentea_test_setup, cases, greentea_test_teardown_handler); + +int main() +{ + return !Harness::run(specification); +} diff --git a/features/storage/blockdevice/BufferedBlockDevice.cpp b/features/storage/blockdevice/BufferedBlockDevice.cpp index 9e87a96f8a7..cc43a53ed27 100644 --- a/features/storage/blockdevice/BufferedBlockDevice.cpp +++ b/features/storage/blockdevice/BufferedBlockDevice.cpp @@ -123,7 +123,10 @@ int BufferedBlockDevice::read(void *b, bd_addr_t addr, bd_size_t size) if (aligned_addr != _curr_aligned_addr) { // Need to flush if moved to another program unit - flush(); + int ret = flush(); + if (ret) { + return ret; + } _curr_aligned_addr = aligned_addr; moved_unit = true; } diff --git a/features/storage/kvstore/KVStore.h b/features/storage/kvstore/KVStore.h new file mode 100644 index 00000000000..d97c0e8dfd4 --- /dev/null +++ b/features/storage/kvstore/KVStore.h @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2018 ARM Limited. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MBED_KVSTORE_H +#define MBED_KVSTORE_H + +#include +#include +#include + +namespace mbed { + +/** KVStore class + * + * Interface class for Key Value Storage + */ + +class KVStore { +public: + enum create_flags { + WRITE_ONCE_FLAG = (1 << 0), + ENCRYPT_FLAG = (1 << 1), + AUTHENTICATE_FLAG = (1 << 2), + ROLLBACK_PROTECT_FLAG = (1 << 3), + }; + + static const uint32_t MAX_KEY_SIZE = 128; + + typedef struct _opaque_set_handle *set_handle_t; + + typedef struct _opaque_key_iterator *iterator_t; + + typedef struct info { + size_t size; + uint32_t flags; + } info_t; + + virtual ~KVStore() {}; + + /** + * @brief Initialize KVStore + * + * @returns 0 on success or a negative error code on failure + */ + virtual int init() = 0; + + /** + * @brief Deinitialize KVStore + * + * @returns 0 on success or a negative error code on failure + */ + virtual int deinit() = 0; + + + /** + * @brief Reset KVStore contents (clear all keys) + * + * @returns 0 on success or a negative error code on failure + */ + virtual int reset() = 0; + + /** + * @brief Set one KVStore item, given key and value. + * + * @param[in] key Key. + * @param[in] buffer Value data buffer. + * @param[in] size Value data size. + * @param[in] create_flags Flag mask. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int set(const char *key, const void *buffer, size_t size, uint32_t create_flags) = 0; + + /** + * @brief Get one KVStore item, given key. + * + * @param[in] key Key. + * @param[in] buffer Value data buffer. + * @param[in] buffer_size Value data buffer size. + * @param[out] actual_size Actual read size (NULL to pass nothing). + * @param[in] offset Offset to read from in data. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int get(const char *key, void *buffer, size_t buffer_size, size_t *actual_size = NULL, size_t offset = 0) = 0; + + /** + * @brief Get information of a given key. + * + * @param[in] key Key. + * @param[out] info Returned information structure (NULL to pass nothing). + * + * @returns 0 on success or a negative error code on failure + */ + virtual int get_info(const char *key, info_t *info = NULL) = 0; + + /** + * @brief Remove a KVStore item, given key. + * + * @param[in] key Key. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int remove(const char *key) = 0; + + + /** + * @brief Start an incremental KVStore set sequence. + * + * @param[out] handle Returned incremental set handle. + * @param[in] key Key. + * @param[in] final_data_size Final value data size. + * @param[in] create_flags Flag mask. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int set_start(set_handle_t *handle, const char *key, size_t final_data_size, uint32_t create_flags) = 0; + + /** + * @brief Add data to incremental KVStore set sequence. + * + * @param[in] handle Incremental set handle. + * @param[in] value_data value data to add. + * @param[in] data_size value data size. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int set_add_data(set_handle_t handle, const void *value_data, size_t data_size) = 0; + + /** + * @brief Finalize an incremental KVStore set sequence. + * + * @param[in] handle Incremental set handle. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int set_finalize(set_handle_t handle) = 0; + + /** + * @brief Start an iteration over KVStore keys. + * + * @param[out] it Returned iterator handle. + * @param[in] prefix Key prefix (null for all keys). + * + * @returns 0 on success or a negative error code on failure + */ + virtual int iterator_open(iterator_t *it, const char *prefix = NULL) = 0; + + /** + * @brief Get next key in iteration. + * + * @param[in] it Iterator handle. + * @param[in] key Buffer for returned key. + * @param[in] key_size Key buffer size. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int iterator_next(iterator_t it, char *key, size_t key_size) = 0; + + /** + * @brief Close iteration. + * + * @param[in] it Iterator handle. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int iterator_close(iterator_t it) = 0; + + /** Convenience function for checking key validity + * + * @param[in] key_size Key buffer size. + * + * @returns 0 on success or a negative error code on failure + */ + bool is_valid_key(const char *key) const + { + if (!key || !strlen(key) || (strlen(key) > MAX_KEY_SIZE)) { + return false; + } + + if (strpbrk(key, " */?:;\"|<>\\")) { + return false; + } + return true; + } + +}; +/** @}*/ + +} // namespace mbed + +#endif diff --git a/features/storage/kvstore/conf/filesystem/mbed_lib.json b/features/storage/kvstore/conf/filesystem/mbed_lib.json new file mode 100644 index 00000000000..866ee34de9f --- /dev/null +++ b/features/storage/kvstore/conf/filesystem/mbed_lib.json @@ -0,0 +1,42 @@ +{ + "name": "storage_filesystem", + "config": { + "rbp_internal_size": { + "help": "If default the size will be 4K*#entries/32", + "value": "0" + }, + "rbp_number_of_entries": { + "help": "If not defined default is 64", + "value": "64" + }, + "internal_base_address": { + "help": "If default, base address is the first sector after the application code", + "value": "0" + }, + "filesystem": { + "help": "Options are default, FAT or LITTLE. If default value the filesystem is chosen by the blockdevice type", + "value": "default" + }, + "blockdevice": { + "help": "Options are default, SPIF, DATAFASH, QSPIF or SD. If default, the block device will be chosen according to the component defined in targets.json", + "value": "default" + }, + "external_size": { + "help": "Size in bytes of the external block device, if default value, the maximum size available is used.", + "value": "0" + }, + "external_base_address": { + "help": "The default will set start address to address 0", + "value": "0" + }, + "mount_point": { + "help": "Where to mount the filesystem.", + "value": "kv" + }, + "folder_path": { + "help": "Path for the working directory where the FileSystemStore stores the data", + "value": "kvstore" + } + } +} + diff --git a/features/storage/kvstore/conf/filesystem_no_rbp/mbed_lib.json b/features/storage/kvstore/conf/filesystem_no_rbp/mbed_lib.json new file mode 100644 index 00000000000..5ecf2841a26 --- /dev/null +++ b/features/storage/kvstore/conf/filesystem_no_rbp/mbed_lib.json @@ -0,0 +1,29 @@ +{ + "name": "storage_filesystem_no_rbp", + "config": { + "filesystem": { + "help": "Options are default, FAT or LITTLE. If default value the filesystem is chosen by the blockdevice type", + "value": "default" + }, + "blockdevice": { + "help": "Options are default, SPIF, DATAFLASH, QSPIF or SD. If default the block device will be chosen by the defined component", + "value": "default" + }, + "external_size": { + "help": "Size in bytes of the external block device, if default the maximum size available is used.", + "value": "0" + }, + "external_base_address": { + "help": "The default will set start address to address 0", + "value": "0" + }, + "mount_point": { + "help": "Where to mount the filesystem.", + "value": "kv" + }, + "folder_path": { + "help": "Path for the working directory where the FileSystemStore stores the data", + "value": "kvstore" + } + } +} diff --git a/features/storage/kvstore/conf/kv_config.cpp b/features/storage/kvstore/conf/kv_config.cpp new file mode 100644 index 00000000000..0338ee6c011 --- /dev/null +++ b/features/storage/kvstore/conf/kv_config.cpp @@ -0,0 +1,822 @@ +/* + * Copyright (c) 2018 ARM Limited. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kv_config.h" +#include "KVStore.h" +#include "KVMap.h" +#include "BlockDevice.h" +#include "FileSystem.h" +#include "FileSystemStore.h" +#include "SlicingBlockDevice.h" +#include "FATFileSystem.h" +#include "LittleFileSystem.h" +#include "TDBStore.h" +#include "mbed_error.h" +#include "FlashIAP.h" +#include "FlashSimBlockDevice.h" +#include "mbed_trace.h" +#define TRACE_GROUP "KVCFG" + +#if COMPONENT_FLASHIAP +#include "FlashIAPBlockDevice.h" +#endif + +#if COMPONENT_QSPIF +#include "QSPIFBlockDevice.h" +#endif + +#if COMPONENT_SPIF +#include "SPIFBlockDevice.h" +#endif + +#if COMPONENT_DATAFLASH +#include "DataFlashBlockDevice.h" +#endif + +#if COMPONENT_SD +#include "SDBlockDevice.h" +#endif + +/** + * @brief This function initializes internal memory secure storage + * This includes a TDBStore instance with a FlashIAPBlockdevice + * as the supported storage. + * The following is a list of configuration parameter + * MBED_CONF_STORAGE_TDB_INTERNAL_SIZE - The size of the underlying FlashIAPBlockdevice + * MBED_CONF_STORAGE_TDB_INTERNAL_BASE_ADDRESS - The start address of the underlying FlashIAPBlockdevice + * @returns 0 on success or negative value on failure. + */ +int _storage_config_TDB_INTERNAL(); + +/** + * @brief This function initialize external memory secure storage + * This includes a SecureStore class with TDBStore over FlashIAPBlockdevice + * and an external TDBStore over a default blockdevice unless configured differently. + * The following is a list of configuration parameter: + * MBED_CONF_STORAGE_TDB_EXTERNAL_RBP_INTERNAL_SIZE - Size of the internal FlashIAPBlockDevice and by + * default is set to 4K*#enteries/32. The start address will be set to end of flash - rbp_internal_size. + * MBED_CONF_STORAGE_TDB_EXTERNAL_RBP_NUMBER_OF_ENTRIES - If not defined default is 64 + * MBED_CONF_STORAGE_TDB_EXTERNAL_INTERNAL_BASE_ADDRESS - The satrt address of the internal FlashIAPBlockDevice. + * MBED_CONF_STORAGE_TDB_EXTERNAL_EXTERNAL_SIZE - Size of the external blockdevice in bytes or NULL for + * max possible size. + * MBED_CONF_STORAGE_TDB_EXTERNAL_EXTERNAL_BASE_ADDRESS - The block device start address. + * MBED_CONF_STORAGE_TDB_EXTERNAL_EXTERNAL_BLOCK_DEVICE - Alowed vlaues are: default, SPIF, DATAFASH, QSPIF or SD + * @returns 0 on success or negative value on failure. + */ +int _storage_config_TDB_EXTERNAL(); + +/** + * @brief This function initialize a external memory secure storage + * This includes a SecureStore class with external TDBStore over a blockdevice or, + * if no blockdevice was set the default blockdevice will be used. + * The following is a list of configuration parameter: + * MBED_CONF_STORAGE_TDB_EXTERNAL_NO_RBP_EXTERNAL_SIZE - Size of the external blockdevice in bytes + * or NULL for max possible size. + * MBED_CONF_STORAGE_TDB_EXTERNAL_NO_RBP_EXTERNAL_BASE_ADDRESS - The block device start address + * MBED_CONF_STORAGE_TDB_EXTERNAL_NO_RBP_EXTERNAL_BLOCK_DEVICE - Alowed vlaues are: default, SPIF, DATAFASH, QSPIF or SD + * @returns 0 on success or negative value on failure. + */ +int _storage_config_TDB_EXTERNAL_NO_RBP(); + +/** + * @brief This function initialize a FILESYSTEM memory secure storage + * This includes a SecureStore class with TDBStore over FlashIAPBlockdevice + * in the internal memory and an external FileSysteStore. If blockdevice and filesystem not set, + * the system will use the default block device and default filesystem + * The following is a list of configuration parameter: + * MBED_CONF_STORAGE_FILESYSTEM_RBP_INTERNAL_SIZE - Size of the internal FlashIAPBlockDevice and by default is + * set to 4K*#enteries/32. The start address will be set to + * end of flash - rbp_internal_size. + * MBED_CONF_STORAGE_FILESYSTEM_RBP_NUMBER_OF_ENTRIES - If not defined default is 64 + * MBED_CONF_STORAGE_FILESYSTEM_INTERNAL_BASE_ADDRESS - The satrt address of the internal FlashIAPBlockDevice. + * MBED_CONF_STORAGE_FILESYSTEM_FILESYSTEM - Allowed values are: default, FAT or LITTLE + * MBED_CONF_STORAGE_FILESYSTEM_BLOCKDEVICE - Allowed values are: default, SPIF, DATAFASH, QSPIF or SD + * MBED_CONF_STORAGE_FILESYSTEM_EXTERNAL_SIZE - External Blockdevice size in bytes or NULL for max possible size. + * MBED_CONF_STORAGE_FILESYSTEM_EXTERNAL_BASE_ADDRESS - The block device start address. + * MBED_CONF_STORAGE_FILESYSTEM_MOUNT_POINT - Where to mount the filesystem + * MBED_CONF_STORAGE_FILESYSTEM_FOLDER_PATH - The working folder paths + * + * @returns 0 on success or negative value on failure. + */ +int _storage_config_FILESYSTEM(); + +/** + * @brief This function initialize a FILESYSTEM_NO_RBP memory secure storage with no + * rollback protection. This includes a SecureStore class an external FileSysteStore over a default + * filesystem with default blockdevice unless differently configured. + * The following is a list of configuration parameter: + * MBED_CONF_STORAGE_FILESYSTEM_NO_RBP_FILESYSTEM - Allowed values are: default, FAT or LITTLE + * MBED_CONF_STORAGE_FILESYSTEM_NO_RBP_BLOCKDEVICE - Allowed values are: default, SPIF, DATAFASH, QSPIF or SD + * MBED_CONF_STORAGE_FILESYSTEM_NO_RBP_EXTERNAL_SIZE - Blockdevice size in bytes. or NULL for max possible size. + * MBED_CONF_STORAGE_FILESYSTEM_NO_RBP_EXTERNAL_BASE_ADDRESS - The block device start address. + * MBED_CONF_STORAGE_FILESYSTEM_NO_RBP_MOUNT_POINT - Where to mount the filesystem + * MBED_CONF_STORAGE_FILESYSTEM_NO_RBP_FOLDER_PATH - The working folder paths + * + * @returns 0 on success or negative value on failure. + */ +int _storage_config_FILESYSTEM_NO_RBP(); + + +using namespace mbed; + + +static SingletonPtr mutex; +static bool is_kv_config_initialize = false; +static kvstore_config_t kvstore_config; + +#define INTERNAL_BLOCKDEVICE_NAME FLASHIAP + +#define STR_EXPAND(tok) #tok +#define STR(tok) STR_EXPAND(tok) + +#define _GET_FILESYSTEM_concat(dev, ...) _get_filesystem_##dev(__VA_ARGS__) +#define GET_FILESYSTEM(dev, ...) _GET_FILESYSTEM_concat(dev, __VA_ARGS__) + +#define _GET_BLOCKDEVICE_concat(dev, ...) _get_blockdevice_##dev(__VA_ARGS__) +#define GET_BLOCKDEVICE(dev, ...) _GET_BLOCKDEVICE_concat(dev, __VA_ARGS__) + +static inline uint32_t align_up(uint64_t val, uint64_t size) +{ + return (((val - 1) / size) + 1) * size; +} + +static inline uint32_t align_down(uint64_t val, uint64_t size) +{ + return (((val) / size)) * size; +} + +int _calculate_blocksize_match_TDBStore(BlockDevice *bd) +{ + bd_size_t size = bd->size(); + bd_size_t erase_size = bd->get_erase_size(); + bd_size_t number_of_sector = size / erase_size; + + if (number_of_sector < 2){ + tr_warning("KV Config: there is less then 2 sector TDBStore will not work."); + return -1; + } + + + if (number_of_sector % 2 != 0){ + tr_warning("KV Config: Number of sector is not even number. Consider changing the BlockDevice size"); + } + + return MBED_SUCCESS; +} + +int _get_addresses(BlockDevice *bd, bd_addr_t start_address, bd_size_t size, bd_addr_t *out_start_addr, + bd_addr_t *out_end_addr ) +{ + bd_addr_t aligned_end_address; + bd_addr_t end_address; + bd_addr_t aligned_start_address; + + aligned_start_address = align_down(start_address, bd->get_erase_size(start_address)); + if (aligned_start_address != start_address) { + tr_error("KV Config: Start address is not aligned. Better use %02llx", aligned_start_address); + return -1; + } + + if (size == 0) { + (*out_start_addr) = aligned_start_address; + (*out_end_addr) = bd->size(); + return 0; + } + + end_address = start_address + size; + aligned_end_address = align_up(end_address, bd->get_erase_size(end_address)); + if (aligned_end_address != end_address) { + tr_error("KV Config: End address is not aligned. Consider changing the size parameter."); + return -1; + } + + if (aligned_end_address > bd->size()) { + tr_error("KV Config: End address is out of boundaries"); + return -1; + } + + (*out_start_addr) = aligned_start_address; + (*out_end_addr) = aligned_end_address; + return 0; +} + +FileSystem *_get_filesystem_FAT(BlockDevice *bd, const char *mount) +{ + static FATFileSystem sdcard(mount, bd); + return &sdcard; + +} + +FileSystem *_get_filesystem_LITTLE(BlockDevice *bd, const char *mount) +{ + static LittleFileSystem flash(mount, bd); + return &flash; +} + +FileSystemStore *_get_fileSystemStore(FileSystem *fs) +{ + static FileSystemStore fss(fs); + return &fss; +} + +FileSystem *_get_filesystem_default(BlockDevice *bd, const char *mount) +{ +#if COMPONENT_QSPIF || COMPONENT_SPIF || COMPONENT_DATAFLASH + return _get_filesystem_LITTLE( bd, mount); +#elif COMPONENT_SD + return _get_filesystem_FAT(bd, mount); +#else + return NULL; +#endif +} + +BlockDevice *_get_blockdevice_FLASHIAP(bd_addr_t start_address, bd_size_t size) +{ +#if COMPONENT_FLASHIAP + + bd_size_t bd_final_size; + bd_addr_t flash_end_address; + bd_addr_t flash_start_address; + bd_addr_t flash_first_writable_sector_address; + bd_addr_t aligned_start_address; + bd_addr_t aligned_end_address; + bd_addr_t end_address; + FlashIAP flash; + + int ret = flash.init(); + if (ret != 0) { + return NULL; + } + + //Get flash parameters before starting + flash_first_writable_sector_address = align_up(FLASHIAP_ROM_END, flash.get_sector_size(FLASHIAP_ROM_END)); + flash_start_address = flash.get_flash_start(); + flash_end_address = flash_start_address + flash.get_flash_size();; + + if (start_address != 0) { + + if (start_address < flash_first_writable_sector_address) { + tr_error("KV Config: Internal block device start address overlapped ROM address "); + flash.deinit(); + return NULL; + } + aligned_start_address = align_down(start_address, flash.get_sector_size(start_address)); + if (start_address != aligned_start_address) { + tr_error("KV Config: Internal block device start address is not aligned. Better use %02llx", aligned_start_address); + flash.deinit(); + return NULL; + } + + if (size == 0) { + //will use 2 sector only. + bd_final_size = (flash_end_address - start_address); + + static FlashIAPBlockDevice bd(start_address, bd_final_size); + flash.deinit(); + return &bd; + } + + if (size != 0) { + + end_address = start_address + size; + if ( end_address > flash_end_address) { + tr_error("KV Config: Internal block device end address is out of boundaries"); + flash.deinit(); + return NULL; + } + + aligned_end_address = align_up(end_address, flash.get_sector_size(end_address - 1)); + if (end_address != aligned_end_address) { + tr_error("KV Config: Internal block device start address is not aligned. Consider changing the size parameter"); + flash.deinit(); + return NULL; + } + + static FlashIAPBlockDevice bd(start_address, size); + flash.deinit(); + return &bd; + } + } + + bool request_default = false; + if (start_address == 0 && size == 0) { + request_default = true; + size = 1; + } + + start_address = flash_end_address - size; + aligned_start_address = align_down(start_address, flash.get_sector_size(start_address)); + //Skip this check if default parameters are set (0 for base address and 0 size). + //We will calculate the address and size by ourselves + if (start_address != aligned_start_address && !request_default) { + tr_error("KV Config: Internal block device start address is not aligned. Consider changing the size parameter"); + flash.deinit(); + return NULL; + } + + if (request_default) { + //update start_address to double the size for TDBStore needs + bd_final_size = (flash_end_address - aligned_start_address) * 2; + start_address = (flash_end_address - bd_final_size); + aligned_start_address = align_down(start_address, flash.get_sector_size(start_address)); + } else { + bd_final_size = (flash_end_address - aligned_start_address); + } + + flash.deinit(); + + if (aligned_start_address < flash_first_writable_sector_address) { + tr_error("KV Config: Internal block device start address overlapped ROM address "); + return NULL; + } + static FlashIAPBlockDevice bd(aligned_start_address, bd_final_size); + return &bd; + +#else + return NULL; +#endif +} + +BlockDevice *_get_blockdevice_SPIF(bd_addr_t start_address, bd_size_t size) +{ +#if COMPONENT_SPIF + + bd_addr_t aligned_end_address; + bd_addr_t aligned_start_address; + + static SPIFBlockDevice bd( + MBED_CONF_SPIF_DRIVER_SPI_MOSI, + MBED_CONF_SPIF_DRIVER_SPI_MISO, + MBED_CONF_SPIF_DRIVER_SPI_CLK, + MBED_CONF_SPIF_DRIVER_SPI_CS, + MBED_CONF_SPIF_DRIVER_SPI_FREQ + ); + + if (bd.init() != MBED_SUCCESS) { + tr_error("KV Config: SPIFBlockDevice init fail"); + return NULL; + } + + if (start_address == 0 && size == 0 ) { + return &bd; + } + + if (_get_addresses(&bd, start_address, size, &aligned_start_address, &aligned_end_address) != 0 ) { + tr_error("KV Config: Fail to get addresses for SlicingBlockDevice."); + return NULL; + } + + static SlicingBlockDevice sbd(&bd, aligned_start_address, aligned_end_address); + return &sbd; + +#else + return NULL; +#endif +} + +BlockDevice *_get_blockdevice_QSPIF(bd_addr_t start_address, bd_size_t size) +{ +#if COMPONENT_QSPIF + + bd_addr_t aligned_end_address; + bd_addr_t aligned_start_address; + + static QSPIFBlockDevice bd( + QSPI_FLASH1_IO0, + QSPI_FLASH1_IO1, + QSPI_FLASH1_IO2, + QSPI_FLASH1_IO3, + QSPI_FLASH1_SCK, + QSPI_FLASH1_CSN, + QSPIF_POLARITY_MODE_0, + MBED_CONF_QSPIF_QSPI_FREQ + ); + + if (bd.init() != MBED_SUCCESS) { + tr_error("KV Config: QSPIFBlockDevice init fail"); + return NULL; + } + + if (start_address == 0 && size == 0 ) { + return &bd; + } + + if (_get_addresses(&bd, start_address, size, &aligned_start_address, &aligned_end_address) != 0 ) { + tr_error("KV Config: Fail to get addresses for SlicingBlockDevice."); + return NULL; + } + + static SlicingBlockDevice sbd(&bd, aligned_start_address, aligned_end_address); + return &sbd; + +#else + return NULL; +#endif +} + +BlockDevice *_get_blockdevice_DATAFLASH(bd_addr_t start_address, bd_size_t size) +{ +#if COMPONENT_DATAFLASH + + bd_addr_t aligned_end_address; + bd_addr_t aligned_start_address; + + static DataFlashBlockDevice bd( + MBED_CONF_DATAFLASH_SPI_MOSI, + MBED_CONF_DATAFLASH_SPI_MISO, + MBED_CONF_DATAFLASH_SPI_CLK, + MBED_CONF_DATAFLASH_SPI_CS + ); + + if (bd.init() != MBED_SUCCESS) { + tr_error("KV Config: DataFlashBlockDevice init fail"); + return NULL; + } + + if (start_address == 0 && size == 0 ) { + return &bd; + } + + if (_get_addresses(&bd, start_address, size, &aligned_start_address, &aligned_end_address) != 0 ) { + tr_error("KV Config: Fail to get addresses for SlicingBlockDevice."); + return NULL; + } + + static SlicingBlockDevice sbd(&bd, aligned_start_address, aligned_end_address); + return &sbd; + + +#else + return NULL; +#endif +} + +BlockDevice *_get_blockdevice_SD(bd_addr_t start_address, bd_size_t size) +{ +#if COMPONENT_SD + + bd_addr_t aligned_end_address; + bd_addr_t aligned_start_address; + + static SDBlockDevice bd( + MBED_CONF_SD_SPI_MOSI, + MBED_CONF_SD_SPI_MISO, + MBED_CONF_SD_SPI_CLK, + MBED_CONF_SD_SPI_CS + ); + + if (bd.init() != MBED_SUCCESS) { + tr_error("KV Config: SDBlockDevice init fail"); + return NULL; + } + +#if MBED_CONF_STORAGE_STORAGE_TYPE == TDB_EXTERNAL_NO_RBP +//In TDBStore we have a constraint of 4GByte + if (start_address == 0 && size == 0 && bd.size() < (uint32_t)(-1)) { + return &bd; + } + + size = size != 0 ? size : align_down(bd.size(), bd.get_erase_size(bd.size())); + + if (_get_addresses(&bd, start_address, size, &aligned_start_address, &aligned_end_address) != 0 ) { + tr_error("KV Config: Fail to get addresses for SlicingBlockDevice."); + return NULL; + } + + if (aligned_end_address - aligned_start_address != (uint32_t)(aligned_end_address - aligned_start_address)) { + aligned_end_address = (uint32_t)(-1);//Support up to 4G only + } + +#else + if (start_address == 0 && size == 0) { + return &bd; + } + + if (_get_addresses(&bd, start_address, size, &aligned_start_address, &aligned_end_address) != 0 ) { + tr_error("KV Config: Fail to get addresses for SlicingBlockDevice."); + return NULL; + } + +#endif + + aligned_end_address = align_down(aligned_end_address, bd.get_erase_size(aligned_end_address)); + static SlicingBlockDevice sbd(&bd, aligned_start_address, aligned_end_address); + return &sbd; + +#else + return NULL; +#endif +} + +BlockDevice *_get_blockdevice_default(bd_addr_t start_address, bd_size_t size) +{ +#if COMPONENT_QSPIF + return _get_blockdevice_QSPIF(start_address, size); +#elif COMPONENT_SPIF + return _get_blockdevice_SPIF(start_address, size); +#elif COMPONENT_DATAFLASH + return _get_blockdevice_DATAFLASH(start_address, size); +#elif COMPONENT_SD + return _get_blockdevice_SD(start_address, size); +#else + tr_error("KV Config: No default component define in target.json for this target."); + return NULL; +#endif +} + +int _storage_config_TDB_INTERNAL() +{ + bd_size_t internal_size = MBED_CONF_STORAGE_TDB_INTERNAL_INTERNAL_SIZE; + bd_addr_t internal_start_address = MBED_CONF_STORAGE_TDB_INTERNAL_INTERNAL_BASE_ADDRESS; + + kvstore_config.internal_bd = GET_BLOCKDEVICE(INTERNAL_BLOCKDEVICE_NAME, internal_start_address, internal_size); + if (kvstore_config.internal_bd == NULL) { + tr_error("KV Config: Fail to get internal BlockDevice."); + return MBED_ERROR_FAILED_OPERATION; + } + + int ret = kvstore_config.internal_bd->init(); + if (ret != MBED_SUCCESS) { + tr_error("KV Config: Fail to init internal BlockDevice."); + return MBED_ERROR_FAILED_OPERATION; + } + + if (_calculate_blocksize_match_TDBStore(kvstore_config.internal_bd) != MBED_SUCCESS) { + tr_error("KV Config: Can not create TDBStore with less then 2 sector."); + return MBED_ERROR_INVALID_ARGUMENT; + } + + static TDBStore tdb_internal(kvstore_config.internal_bd); + kvstore_config.internal_store = &tdb_internal; + + ret = kvstore_config.internal_store->init(); + if (ret != MBED_SUCCESS) { + tr_error("KV Config: Fail to init internal TDBStore."); + return ret; + } + kvstore_config.kvstore_main_instance = + kvstore_config.internal_store; + + KVMap& kv_map = KVMap::get_instance(); + ret = kv_map.init(); + if (MBED_SUCCESS != ret) { + tr_error("KV Config: Fail to init KVStore global API."); + return ret; + } + + ret = kv_map.attach(STR(MBED_CONF_STORAGE_DEFAULT_KV), &kvstore_config); + if (MBED_SUCCESS != ret) { + tr_error("KV Config: Fail to attach KVStore main instance to KVStore global API."); + return ret; + } + + return MBED_SUCCESS; +} + +int _storage_config_TDB_EXTERNAL() +{ + bd_size_t internal_rbp_size = MBED_CONF_STORAGE_TDB_EXTERNAL_RBP_INTERNAL_SIZE; + size_t rbp_num_of_enteries = MBED_CONF_STORAGE_TDB_EXTERNAL_RBP_NUMBER_OF_ENTRIES; + bd_addr_t internal_start_address = MBED_CONF_STORAGE_TDB_EXTERNAL_INTERNAL_BASE_ADDRESS; + + if (internal_rbp_size == 0) { + internal_rbp_size = 4 * 1024 * rbp_num_of_enteries / 32; + } + + kvstore_config.internal_bd = GET_BLOCKDEVICE(INTERNAL_BLOCKDEVICE_NAME, internal_start_address, internal_rbp_size); + if (kvstore_config.internal_bd == NULL) { + tr_error("KV Config: Fail to get internal BlockDevice."); + return MBED_ERROR_FAILED_OPERATION ; + } + + int ret = kvstore_config.internal_bd->init(); + if (ret != MBED_SUCCESS) { + tr_error("KV Config: Fail to init internal BlockDevice."); + return MBED_ERROR_FAILED_OPERATION ; + } + + if (_calculate_blocksize_match_TDBStore(kvstore_config.internal_bd) != MBED_SUCCESS) { + tr_error("KV Config: Can not create TDBStore with less then 2 sector."); + return MBED_ERROR_INVALID_ARGUMENT; + } + + static TDBStore tdb_internal(kvstore_config.internal_bd); + kvstore_config.internal_store = &tdb_internal; + + ret = kvstore_config.internal_store->init(); + if (ret != MBED_SUCCESS) { + tr_error("KV Config: Fail to init internal TDBStore."); + return ret; + } + + return _storage_config_TDB_EXTERNAL_NO_RBP(); +} + +int _storage_config_TDB_EXTERNAL_NO_RBP() +{ + bd_size_t size = MBED_CONF_STORAGE_TDB_EXTERNAL_NO_RBP_EXTERNAL_SIZE; + bd_addr_t address = MBED_CONF_STORAGE_TDB_EXTERNAL_NO_RBP_EXTERNAL_BASE_ADDRESS; + + BlockDevice *bd = GET_BLOCKDEVICE(MBED_CONF_STORAGE_TDB_EXTERNAL_NO_RBP_BLOCKDEVICE, address, size); + if (bd == NULL) { + tr_error("KV Config: Fail to get external BlockDevice."); + return MBED_ERROR_FAILED_OPERATION ; + } + +#if defined(COMPONENT_SD) + if ( strcmp(STR(MBED_CONF_STORAGE_TDB_EXTERNAL_NO_RBP_BLOCKDEVICE), "SD") == 0 +#if defined(COMPONENT_SD) && !defined(COMPONENT_SPIF) && !defined(COMPONENT_QSPIF) && !defined(COMPONENT_DATAFLASH) + || strcmp(STR(MBED_CONF_STORAGE_TDB_EXTERNAL_NO_RBP_BLOCKDEVICE), "default") == 0) { +#else + ) { + +#endif + //TDBStore need FlashSimBlockDevice when working with SD block device + if (bd->init() != MBED_SUCCESS) { + tr_error("KV Config: Fail to init external BlockDevice."); + return MBED_ERROR_FAILED_OPERATION ; + } + + static FlashSimBlockDevice flash_bd(bd); + kvstore_config.external_bd = &flash_bd; + } else { + kvstore_config.external_bd = bd; + } +#else + kvstore_config.external_bd = bd; +#endif + + int ret = kvstore_config.external_bd->init(); + if (ret != MBED_SUCCESS) { + tr_error("KV Config: Fail to init external BlockDevice."); + return MBED_ERROR_FAILED_OPERATION ; + } + + if (_calculate_blocksize_match_TDBStore(kvstore_config.external_bd) != MBED_SUCCESS) { + tr_error("KV Config: Can not create TDBStore with less then 2 sector."); + return MBED_ERROR_INVALID_ARGUMENT; + } + + static TDBStore tdb_external(kvstore_config.external_bd); + kvstore_config.external_store = &tdb_external; + + kvstore_config.kvstore_main_instance = + kvstore_config.external_store;//TODO: change this when secure storage come to live + + ret = kvstore_config.external_store->init(); + if (ret != MBED_SUCCESS) { + tr_error("KV Config: Fail to init external TDBStore"); + return ret; + } + + KVMap& kv_map = KVMap::get_instance(); + ret = kv_map.init(); + if (MBED_SUCCESS != ret) { + tr_error("KV Config: Fail to init KVStore global API"); + return ret; + } + + ret = kv_map.attach(STR(MBED_CONF_STORAGE_DEFAULT_KV), &kvstore_config); + if (MBED_SUCCESS != ret) { + tr_error("KV Config: Fail to attach KvStore main instance to KVStore global API"); + return ret; + } + + return MBED_SUCCESS; +} + +int _storage_config_FILESYSTEM() +{ + bd_size_t internal_rbp_size = MBED_CONF_STORAGE_FILESYSTEM_RBP_INTERNAL_SIZE; + size_t rbp_num_of_enteries = MBED_CONF_STORAGE_FILESYSTEM_RBP_NUMBER_OF_ENTRIES; + bd_addr_t internal_start_address = MBED_CONF_STORAGE_FILESYSTEM_INTERNAL_BASE_ADDRESS; + + if (internal_rbp_size == 0) { + internal_rbp_size = 4 * 1024 * rbp_num_of_enteries / 32; + } + + kvstore_config.internal_bd = GET_BLOCKDEVICE(INTERNAL_BLOCKDEVICE_NAME, internal_start_address, internal_rbp_size); + if (kvstore_config.internal_bd == NULL) { + tr_error("KV Config: Fail to get internal BlockDevice "); + return MBED_ERROR_FAILED_OPERATION ; + } + + int ret = kvstore_config.internal_bd->init(); + if (ret != MBED_SUCCESS) { + tr_error("KV Config: Fail to init internal BlockDevice "); + return MBED_ERROR_FAILED_OPERATION ; + } + + static TDBStore tdb_internal(kvstore_config.internal_bd); + kvstore_config.internal_store = &tdb_internal; + + ret = kvstore_config.internal_store->init(); + if (ret != MBED_SUCCESS) { + tr_error("KV Config: Fail to init internal TDBStore"); + return ret; + } + + return _storage_config_FILESYSTEM_NO_RBP(); +} + +int _storage_config_FILESYSTEM_NO_RBP() +{ + bd_size_t size = MBED_CONF_STORAGE_FILESYSTEM_NO_RBP_EXTERNAL_SIZE; + bd_addr_t address = MBED_CONF_STORAGE_FILESYSTEM_NO_RBP_EXTERNAL_BASE_ADDRESS; + const char *mount_point = STR(MBED_CONF_STORAGE_FILESYSTEM_NO_RBP_MOUNT_POINT); + + kvstore_config.external_bd = GET_BLOCKDEVICE(MBED_CONF_STORAGE_FILESYSTEM_NO_RBP_BLOCKDEVICE, address, size); + if (kvstore_config.external_bd == NULL) { + tr_error("KV Config: Fail to get external BlockDevice "); + return MBED_ERROR_FAILED_OPERATION ; + } + + int ret = kvstore_config.external_bd->init(); + if (MBED_SUCCESS != ret) { + tr_error("KV Config: Fail to init external BlockDevice "); + return MBED_ERROR_FAILED_OPERATION ; + } + + kvstore_config.external_fs = GET_FILESYSTEM(MBED_CONF_STORAGE_FILESYSTEM_NO_RBP_FILESYSTEM, kvstore_config.external_bd, + mount_point); + if (kvstore_config.external_fs == NULL) { + tr_error("KV Config: Fail to get FileSystem"); + return MBED_ERROR_FAILED_OPERATION ; + } + + ret = kvstore_config.external_fs->mount(kvstore_config.external_bd); + if (ret != MBED_SUCCESS) { + ret = kvstore_config.external_fs->reformat(kvstore_config.external_bd); + if (ret != MBED_SUCCESS) { + tr_error("KV Config: Fail to mount FileSystem to %s", mount_point); + return MBED_ERROR_FAILED_OPERATION ; + } + } + + kvstore_config.external_store = _get_fileSystemStore(kvstore_config.external_fs); + if (kvstore_config.external_store == NULL) { + tr_error("KV Config: Fail to get FileSystemStore"); + return MBED_ERROR_FAILED_OPERATION ; + } + + ret = kvstore_config.external_store->init(); + if (ret != MBED_SUCCESS) { + tr_error("KV Config: Fail to init FileSystemStore"); + return ret; + } + + kvstore_config.kvstore_main_instance = + kvstore_config.external_store; //TODO: change this when secure storage come to live + + KVMap& kv_map = KVMap::get_instance(); + ret = kv_map.init(); + if (MBED_SUCCESS != ret) { + tr_error("KV Config: Fail to init KVStore global API"); + return ret; + } + + ret = kv_map.attach(STR(MBED_CONF_STORAGE_DEFAULT_KV), &kvstore_config); + if (MBED_SUCCESS != ret) { + tr_error("KV Config: Fail to attach KvStore main instance to KVStore global API"); + return ret; + } + + return MBED_SUCCESS; +} + +MBED_WEAK int storage_configuration() +{ + + int ret = MBED_SUCCESS; + + mutex->lock(); + + if (is_kv_config_initialize) { + goto exit; + } + + memset(&kvstore_config, 0, sizeof(kvstore_config_t)); + + ret = _STORAGE_CONFIG(MBED_CONF_STORAGE_STORAGE_TYPE); + + if (ret == MBED_SUCCESS) { + is_kv_config_initialize = true; + } + +exit: + mutex->unlock(); + return ret; +} diff --git a/features/storage/kvstore/conf/kv_config.h b/features/storage/kvstore/conf/kv_config.h new file mode 100644 index 00000000000..f49182d5305 --- /dev/null +++ b/features/storage/kvstore/conf/kv_config.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2018 ARM Limited. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef _KV_CONFIG +#define _KV_CONFIG + +#ifdef __cplusplus +extern "C" { +#endif + +#if MBED_CONF_STORAGE_STORAGE_TYPE == FILESYSTEM +#define FSST_FOLDER_PATH MBED_CONF_STORAGE_FILESYSTEM_FOLDER_PATH +#elif MBED_CONF_STORAGE_STORAGE_TYPE == FILESYSTEM_NO_RBP +#define FSST_FOLDER_PATH MBED_CONF_STORAGE_FILESYSTEM_NO_RBP_FOLDER_PATH +#endif + +#if MBED_CONF_STORAGE_STORAGE == NULL +#define MBED_CONF_STORAGE_STORAGE USER_DEFINED +#endif + +#define _STORAGE_CONFIG_concat(dev) _storage_config_##dev() +#define _STORAGE_CONFIG(dev) _STORAGE_CONFIG_concat(dev) + +/** + * @brief This function will initialize one of the configuration exists in mbed-os. In order to overwite + * the default configuration please overwrite this function. + * + * @returns 0 on success or negative value on failure. + */ +int storage_configuration(); + +#ifdef __cplusplus +} // closing brace for extern "C" +#endif +#endif diff --git a/features/storage/kvstore/conf/mbed_lib.json b/features/storage/kvstore/conf/mbed_lib.json new file mode 100644 index 00000000000..e22898d82f9 --- /dev/null +++ b/features/storage/kvstore/conf/mbed_lib.json @@ -0,0 +1,13 @@ +{ +"name": "storage", + "config": { + "storage_type": { + "help": "Options are TDB_INTERNAL, TDB_EXTERNAL, TDB_EXTERNAL_NO_RBP, FILESYSTEM or FILESYSTEM_NO_RBP.", + "value": "TDB_EXTERNAL_NO_RBP" + }, + "default_kv": { + "help": "A string name for the default kvstore configuration", + "value": "kv" + } + } +} diff --git a/features/storage/kvstore/conf/tdb_external/mbed_lib.json b/features/storage/kvstore/conf/tdb_external/mbed_lib.json new file mode 100644 index 00000000000..b0ee1ddc303 --- /dev/null +++ b/features/storage/kvstore/conf/tdb_external/mbed_lib.json @@ -0,0 +1,30 @@ +{ + + "name": "storage_tdb_external", + "config": { + "rbp_internal_size": { + "help": "If default the size will be 4K*#entries/32", + "value": "0" + }, + "rbp_number_of_entries": { + "help": "If not defined default is 64", + "value": "64" + }, + "internal_base_address": { + "help": "If default, the base address is set to the first sector after the application code ends.", + "value": "0" + }, + "blockdevice": { + "help": "Options are default, SPIF, DATAFASH, QSPIF or SD. If default the block device will be chosen by the defined component", + "value": "default" + }, + "external_size": { + "help": "Size in bytes of the external block device, if default the maximum size available is used.", + "value": "0" + }, + "external_base_address": { + "help": "The default will set start address to address 0", + "value": "0" + } + } +} \ No newline at end of file diff --git a/features/storage/kvstore/conf/tdb_external_no_rbp/mbed_lib.json b/features/storage/kvstore/conf/tdb_external_no_rbp/mbed_lib.json new file mode 100644 index 00000000000..ca617b84a66 --- /dev/null +++ b/features/storage/kvstore/conf/tdb_external_no_rbp/mbed_lib.json @@ -0,0 +1,17 @@ +{ + "name": "storage_tdb_external_no_rbp", + "config": { + "blockdevice": { + "help": "Options are default, SPIF, DATAFASH, QSPIF or SD. If default the block device will be chosen by the defined component", + "value": "default" + }, + "external_size": { + "help": "Size in bytes of the external block device, if default the maximum size available is used.", + "value": "0" + }, + "external_base_address": { + "help": "The default will set start address to address 0", + "value": "0" + } + } +} \ No newline at end of file diff --git a/features/storage/kvstore/conf/tdb_internal/mbed_lib.json b/features/storage/kvstore/conf/tdb_internal/mbed_lib.json new file mode 100644 index 00000000000..ae9998ea7cf --- /dev/null +++ b/features/storage/kvstore/conf/tdb_internal/mbed_lib.json @@ -0,0 +1,13 @@ +{ + "name": "storage_tdb_internal", + "config": { + "internal_size": { + "help": "Size of the FlashIAP block device", + "value": "0" + }, + "internal_base_address": { + "help": "If default, the base address is set to the first sector after the application code ends.", + "value": "0" + } + } +} \ No newline at end of file diff --git a/features/storage/kvstore/filesystemstore/FileSystemStore.cpp b/features/storage/kvstore/filesystemstore/FileSystemStore.cpp new file mode 100644 index 00000000000..c0c772facf9 --- /dev/null +++ b/features/storage/kvstore/filesystemstore/FileSystemStore.cpp @@ -0,0 +1,621 @@ +/* mbed Microcontroller Library + * Copyright (c) 2018 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "FileSystemStore.h" +#include "Dir.h" +#include "File.h" +#include "BlockDevice.h" +#include "mbed_error.h" +#include +#include +#include + +#include "mbed_trace.h" +#define TRACE_GROUP "FSST" + +#define FSST_REVISION 1 +#define FSST_MAGIC 0x46535354 // "FSST" hex 'magic' signature + +// incremental set handle +typedef struct { + char *key; + uint32_t create_flags; + size_t data_size; +} inc_set_handle_t; + +// iterator handle +typedef struct { + void *dir_handle; + char *prefix; +} key_iterator_handle_t; + +using namespace mbed; + +// Local Functions +static char *string_ndup(const char *src, size_t size); + + +// Class Functions +FileSystemStore::FileSystemStore(FileSystem *fs) : _fs(fs), + _is_initialized(false) +{ + +} + +int FileSystemStore::init() +{ + int status = MBED_SUCCESS; + + _mutex.lock(); + + _cfg_fs_path_size = strlen(FSST_FOLDER_PATH); + _cfg_fs_path = string_ndup(FSST_FOLDER_PATH, _cfg_fs_path_size); + _full_path_key = new char[_cfg_fs_path_size + KVStore::MAX_KEY_SIZE + 1]; + memset(_full_path_key, 0, (_cfg_fs_path_size + KVStore::MAX_KEY_SIZE + 1)); + strncpy(_full_path_key, _cfg_fs_path, _cfg_fs_path_size); + _full_path_key[_cfg_fs_path_size] = '/'; + _cur_inc_data_size = 0; + _cur_inc_set_handle = NULL; + Dir kv_dir; + + if (kv_dir.open(_fs, _cfg_fs_path) != 0) { + tr_info("KV Dir: %s, doesnt exist - creating new.. ", _cfg_fs_path); //TBD verify ERRNO NOEXIST + if (_fs->mkdir(_cfg_fs_path,/* which flags ? */0777) != 0) { + tr_error("KV Dir: %s, mkdir failed.. ", _cfg_fs_path); //TBD verify ERRNO NOEXIST + status = MBED_ERROR_FAILED_OPERATION ; + goto exit_point; + } + } else { + tr_info("KV Dir: %s, exists(verified) - now closing it", _cfg_fs_path); + if (kv_dir.close() != 0) { + tr_error("KV Dir: %s, dir_close failed", _cfg_fs_path); //TBD verify ERRNO NOEXIST + } + } + + _is_initialized = true; +exit_point: + + _mutex.unlock(); + + return status; + +} + +int FileSystemStore::deinit() +{ + _mutex.lock(); + _is_initialized = false; + delete[] _cfg_fs_path; + delete[] _full_path_key; + _mutex.unlock(); + return MBED_SUCCESS; + +} + +int FileSystemStore::reset() +{ + int status = MBED_SUCCESS; + Dir kv_dir; + struct dirent dir_ent; + + _mutex.lock(); + if (false == _is_initialized) { + status = MBED_ERROR_NOT_READY; + goto exit_point; + } + + kv_dir.open(_fs, _cfg_fs_path); + + while (kv_dir.read(&dir_ent) != 0) { + tr_info("Looping FSST folder: %s, File - %s", _cfg_fs_path, dir_ent.d_name); + if (dir_ent.d_type != DT_REG) { + tr_error("KV_Dir should contain only Regular File - %s", dir_ent.d_name); + continue; + } + // Build File's full path name and delete it (even if write-onced) + _build_full_path_key(dir_ent.d_name); + _fs->remove(_full_path_key); + } + + kv_dir.close(); + +exit_point: + _mutex.unlock(); + return status; +} + +int FileSystemStore::set(const char *key, const void *buffer, size_t size, uint32_t create_flags) +{ + int status = MBED_SUCCESS; + set_handle_t handle; + + if (false == _is_initialized) { + status = MBED_ERROR_NOT_READY; + goto exit_point; + } + + if ((!is_valid_key(key)) || ((buffer == NULL) && (size > 0)) ) { + status = MBED_ERROR_INVALID_ARGUMENT; + goto exit_point; + } + + status = set_start(&handle, key, size, create_flags); + if (status != MBED_SUCCESS) { + tr_error("FSST Set set_start Failed: %d", status); + goto exit_point; + } + + status = set_add_data(handle, buffer, size); + if (status != MBED_SUCCESS) { + tr_error("FSST Set set_add_data Failed: %d", status); + set_finalize(handle); + goto exit_point; + } + + status = set_finalize(handle); + if (status != MBED_SUCCESS) { + tr_error("FSST Set set_finalize Failed: %d", status); + goto exit_point; + } + +exit_point: + + return status; +} + +int FileSystemStore::get(const char *key, void *buffer, size_t buffer_size, size_t *actual_size, size_t offset) +{ + int status = MBED_SUCCESS; + + File kv_file; + size_t kv_file_size = 0; + size_t value_actual_size = 0; + + _mutex.lock(); + + if (false == _is_initialized) { + status = MBED_ERROR_NOT_READY; + goto exit_point; + } + + key_metadata_t key_metadata; + + if ( (status = _verify_key_file(key, &key_metadata, &kv_file)) != MBED_SUCCESS ) { + tr_error("File Verification failed, status: %d", status); + goto exit_point; + } + + kv_file_size = kv_file.size() - key_metadata.metadata_size; + // Actual size is the minimum of buffer_size and remainder of data in file (file's data size - offset) + value_actual_size = buffer_size; + if (offset > kv_file_size) { + status = MBED_ERROR_INVALID_SIZE; + goto exit_point; + } else if ((kv_file_size - offset) < buffer_size) { + value_actual_size = kv_file_size - offset; + } + + if ((buffer == NULL) && (value_actual_size > 0)) { + status = MBED_ERROR_INVALID_DATA_DETECTED; + goto exit_point; + } + + if (actual_size != NULL) { + *actual_size = value_actual_size; + } + + kv_file.seek(key_metadata.metadata_size + offset, SEEK_SET); + // Read remainder of data + kv_file.read(buffer, value_actual_size); + +exit_point: + if ( (status == MBED_SUCCESS) || + (status == MBED_ERROR_INVALID_DATA_DETECTED) ) { + kv_file.close(); + } + _mutex.unlock(); + + return status; +} + +int FileSystemStore::get_info(const char *key, info_t *info) +{ + int status = MBED_SUCCESS; + File kv_file; + + _mutex.lock(); + + if (false == _is_initialized) { + status = MBED_ERROR_NOT_READY; + goto exit_point; + } + + key_metadata_t key_metadata; + + if ( (status = _verify_key_file(key, &key_metadata, &kv_file)) != MBED_SUCCESS ) { + tr_error("File Verification failed, status: %d", status); + goto exit_point; + } + + if (info != NULL) { + info->size = kv_file.size() - key_metadata.metadata_size; + info->flags = key_metadata.user_flags; + } + +exit_point: + if ( (status == MBED_SUCCESS) || + (status == MBED_ERROR_INVALID_DATA_DETECTED) ) { + kv_file.close(); + } + _mutex.unlock(); + + return status; +} + +int FileSystemStore::remove(const char *key) +{ + File kv_file; + key_metadata_t key_metadata; + + _mutex.lock(); + + int status = MBED_SUCCESS; + + if (false == _is_initialized) { + status = MBED_ERROR_NOT_READY; + goto exit_point; + } + + /* If File Exists and is Valid, then check its Write Once Flag to verify its disabled before removing */ + /* If File exists and is not valid, or is Valid and not Write-Onced then remove it */ + if ( (status = _verify_key_file(key, &key_metadata, &kv_file)) == MBED_SUCCESS ) { + tr_error("File: %s, Exists Verifying Write Once Disabled before setting new value", _full_path_key); + if (key_metadata.user_flags & KVStore::WRITE_ONCE_FLAG) { + kv_file.close(); + status = MBED_ERROR_WRITE_PROTECTED; + goto exit_point; + } + } else if ((status == MBED_ERROR_ITEM_NOT_FOUND) || + (status == MBED_ERROR_INVALID_ARGUMENT)) { + goto exit_point; + } + kv_file.close(); + + if (0 != _fs->remove(_full_path_key)) { + status = MBED_ERROR_FAILED_OPERATION; + } + +exit_point: + _mutex.unlock(); + return status; +} + +// Incremental set API +int FileSystemStore::set_start(set_handle_t *handle, const char *key, size_t final_data_size, uint32_t create_flags) +{ + int status = MBED_SUCCESS; + inc_set_handle_t *set_handle = NULL; + File kv_file; + key_metadata_t key_metadata; + int key_len = 0; + + // Only a single key file can be incrementaly editted at a time + _mutex.lock(); + + if (handle == NULL) { + status = MBED_ERROR_INVALID_ARGUMENT; + goto exit_point; + } + + /* If File Exists and is Valid, then check its Write Once Flag to verify its disabled before setting */ + /* If File exists and is not valid, or is Valid and not Write-Onced then erase it */ + status = _verify_key_file(key, &key_metadata, &kv_file); + + if (status == MBED_ERROR_INVALID_ARGUMENT) { + tr_error("File Verification failed, status: %d", status); + goto exit_point; + } + + if (status == MBED_SUCCESS ) { + tr_info("File: %s, Exists. Verifying Write Once Disabled before setting new value", _full_path_key); + if (key_metadata.user_flags & KVStore::WRITE_ONCE_FLAG) { + kv_file.close(); + status = MBED_ERROR_WRITE_PROTECTED; + goto exit_point; + } + } + + /* For Success (not write_once) and for corrupted data close file before recreating it as a new file */ + if (status != MBED_ERROR_ITEM_NOT_FOUND) { + kv_file.close(); + } + + if ( (status = kv_file.open(_fs, _full_path_key, O_WRONLY | O_CREAT | O_TRUNC)) != MBED_SUCCESS ) { + tr_info("set_start failed to open: %s, for writing, err: %d", _full_path_key, status); + status = MBED_ERROR_FAILED_OPERATION ; + goto exit_point; + } + _cur_inc_data_size = 0; + + set_handle = new inc_set_handle_t; + set_handle->create_flags = create_flags; + set_handle->data_size = final_data_size; + key_len = strlen(key); + set_handle->key = string_ndup(key, key_len); + *handle = (set_handle_t)set_handle; + _cur_inc_set_handle = *handle; + + key_metadata.magic = FSST_MAGIC; + key_metadata.metadata_size = sizeof(key_metadata_t); + key_metadata.revision = FSST_REVISION; + key_metadata.user_flags = create_flags; + kv_file.write(&key_metadata, sizeof(key_metadata_t)); + kv_file.close(); +exit_point: + if (status != MBED_SUCCESS) { + _mutex.unlock(); + } + return status; +} + +int FileSystemStore::set_add_data(set_handle_t handle, const void *value_data, size_t data_size) +{ + int status = MBED_SUCCESS; + size_t added_data = 0; + inc_set_handle_t *set_handle = (inc_set_handle_t *)handle; + File kv_file; + + if ( ((value_data == NULL) && (data_size > 0)) || (handle == NULL) || (handle != _cur_inc_set_handle)) { + status = MBED_ERROR_INVALID_ARGUMENT; + goto exit_point; + } + + // Single key incrementally edited, can be edited from multiple threads - lock to protect + _inc_data_add_mutex.lock(); + if ( (_cur_inc_data_size + data_size) > set_handle->data_size ) { + tr_warning("Added Data(%d) will exceed set_start final size(%d) - not adding data to file: %s", + _cur_inc_data_size + data_size, set_handle->data_size, _full_path_key); + status = MBED_ERROR_INVALID_SIZE; + goto exit_point; + } + + if (0 != kv_file.open(_fs, _full_path_key, O_WRONLY | O_APPEND)) { + status = MBED_ERROR_ITEM_NOT_FOUND; + goto exit_point; + } + + added_data = kv_file.write(value_data, data_size); + if (added_data != data_size) { + status = MBED_ERROR_FAILED_OPERATION ; + } + _cur_inc_data_size += added_data; + + kv_file.close(); + +exit_point: + if (status != MBED_ERROR_INVALID_ARGUMENT) { + _inc_data_add_mutex.unlock(); + } + + return status; +} + +int FileSystemStore::set_finalize(set_handle_t handle) +{ + int status = MBED_SUCCESS; + inc_set_handle_t *set_handle = NULL; + + if ((handle == NULL) || (handle != _cur_inc_set_handle)) { + status = MBED_ERROR_INVALID_ARGUMENT; + goto exit_point; + } + + set_handle = (inc_set_handle_t *)handle; + + if (set_handle->key == NULL) { + status = MBED_ERROR_INVALID_DATA_DETECTED; + } else { + if (_cur_inc_data_size != set_handle->data_size ) { + tr_error("Accumulated Data (%d) size doesn't match set_start final size (%d) - file: %s", _cur_inc_data_size, + set_handle->data_size, _full_path_key); + status = MBED_ERROR_INVALID_SIZE; + _fs->remove(_full_path_key); + } + + delete[] set_handle->key; + } + + delete set_handle; + _cur_inc_data_size = 0; + _cur_inc_set_handle = NULL; + +exit_point: + if (status != MBED_ERROR_INVALID_ARGUMENT) { + _mutex.unlock(); + } + + return status; +} + +int FileSystemStore::iterator_open(iterator_t *it, const char *prefix) +{ + int status = MBED_SUCCESS; + Dir *kv_dir = NULL; + key_iterator_handle_t *key_it = NULL; + + if (it == NULL) { + return MBED_ERROR_INVALID_ARGUMENT; + } + + _mutex.lock(); + if (false == _is_initialized) { + status = MBED_ERROR_NOT_READY; + goto exit_point; + } + key_it = new key_iterator_handle_t; + key_it->dir_handle = NULL; + key_it->prefix = NULL; + if (prefix != NULL) { + key_it->prefix = string_ndup(prefix, KVStore::MAX_KEY_SIZE); + } + + kv_dir = new Dir; + if (kv_dir->open(_fs, _cfg_fs_path) != 0) { + tr_error("KV Dir: %s, doesnt exist", _cfg_fs_path); //TBD verify ERRNO NOEXIST + delete kv_dir; + if (key_it->prefix != NULL) { + delete[] key_it->prefix; + } + delete key_it; + status = MBED_ERROR_ITEM_NOT_FOUND; + goto exit_point; + } + + key_it->dir_handle = kv_dir; + + *it = (iterator_t)key_it; + +exit_point: + _mutex.unlock(); + + return status; +} + +int FileSystemStore::iterator_next(iterator_t it, char *key, size_t key_size) +{ + Dir *kv_dir; + struct dirent kv_dir_ent; + int status = MBED_ERROR_ITEM_NOT_FOUND; + key_iterator_handle_t *key_it = NULL; + size_t key_name_size = KVStore::MAX_KEY_SIZE; + if (key_size < key_name_size) { + key_name_size = key_size; + } + + _mutex.lock(); + if (false == _is_initialized) { + status = MBED_ERROR_NOT_READY; + goto exit_point; + } + + key_it = (key_iterator_handle_t *)it; + + if (key_name_size < strlen(key_it->prefix)) { + status = MBED_ERROR_INVALID_SIZE; + goto exit_point; + } + + kv_dir = (Dir *)key_it->dir_handle; + + while (kv_dir->read(&kv_dir_ent) != 0 ) { + if (kv_dir_ent.d_type != DT_REG) { + tr_error("KV_Dir should contain only Regular File - %s", kv_dir_ent.d_name); + continue; + } + + if ( (key_it->prefix == NULL) || + (strncmp(kv_dir_ent.d_name, key_it->prefix, strlen(key_it->prefix)) == 0) ) { + if (key_name_size < strlen(kv_dir_ent.d_name)) { + status = MBED_ERROR_INVALID_SIZE; + break; + } + strncpy(key, kv_dir_ent.d_name, key_name_size); + key[key_name_size - 1] = '\0'; + status = MBED_SUCCESS; + break; + } + } + +exit_point: + _mutex.unlock(); + return status; +} + +int FileSystemStore::iterator_close(iterator_t it) +{ + int status = MBED_SUCCESS; + key_iterator_handle_t *key_it = (key_iterator_handle_t *)it; + + _mutex.lock(); + if (key_it == NULL) { + status = MBED_ERROR_INVALID_ARGUMENT; + goto exit_point; + } + + if (key_it->prefix != NULL) { + delete[] key_it->prefix; + } + + ((Dir *)(key_it->dir_handle))->close(); + + if (key_it->dir_handle != NULL) { + delete ((Dir *)(key_it->dir_handle)); + } + delete key_it; + +exit_point: + _mutex.unlock(); + return status; +} + +int FileSystemStore::_verify_key_file(const char *key, key_metadata_t *key_metadata, File *kv_file) +{ + int status = MBED_SUCCESS; + + if (!is_valid_key(key)) { + status = MBED_ERROR_INVALID_ARGUMENT; + goto exit_point; + } + + _build_full_path_key(key); + + if (0 != kv_file->open(_fs, _full_path_key, O_RDONLY) ) { + tr_info("Couldn't read: %s", _full_path_key); + status = MBED_ERROR_ITEM_NOT_FOUND; + goto exit_point; + } + + //Read Metadata + kv_file->read(key_metadata, sizeof(key_metadata_t)); + + if ((key_metadata->magic != FSST_MAGIC) || + (key_metadata->revision > FSST_REVISION) ) { + status = MBED_ERROR_INVALID_DATA_DETECTED; + goto exit_point; + } + +exit_point: + return status; +} + +int FileSystemStore::_build_full_path_key(const char *key_src) +{ + strncpy(&_full_path_key[_cfg_fs_path_size + 1/* for path's \ */], key_src, KVStore::MAX_KEY_SIZE); + _full_path_key[(_cfg_fs_path_size + KVStore::MAX_KEY_SIZE)] = '\0'; + return 0; +} + +// Local Functions +static char *string_ndup(const char *src, size_t size) +{ + char *string_copy = new char[size + 1]; + strncpy(string_copy, src, size); + string_copy[size] = '\0'; + return string_copy; +} + + + diff --git a/features/storage/kvstore/filesystemstore/FileSystemStore.h b/features/storage/kvstore/filesystemstore/FileSystemStore.h new file mode 100644 index 00000000000..e78a0cbb19b --- /dev/null +++ b/features/storage/kvstore/filesystemstore/FileSystemStore.h @@ -0,0 +1,220 @@ +/* mbed Microcontroller Library + * Copyright (c) 2018 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MBED_FILE_SYSTEM_STORE_H +#define MBED_FILE_SYSTEM_STORE_H + +#include "KVStore.h" +#include "FileSystem.h" + +#ifndef FSST_FOLDER_PATH +#define FSST_FOLDER_PATH "$fsst$" //default FileSystemStore folder path on fs +#endif + +namespace mbed { + +/** FileSystemStore for Secure Store + * + * @code + * ... + * @endcode + */ +class FileSystemStore : public KVStore { + +public: + /** Create FileSystemStore - A Key Value API on top of FS + * + * @param fs File system on top which File System Store is adding KV API + */ + FileSystemStore(FileSystem *fs); + + /** Destroy FileSystemStore instance + * + */ + virtual ~FileSystemStore() {} + + /** + * @brief Initialize FileSystemStore + * + * @returns 0 on success or a negative error code on failure + */ + virtual int init(); + + /** + * @brief Deinitialize FileSystemStore + * + * @returns 0 on success or a negative error code on failure + */ + virtual int deinit(); + + /** + * @brief Reset FileSystemStore contents (clear all keys) + * + * @returns 0 on success or a negative error code on failure + */ + virtual int reset(); + + /** + * @brief Set one FileSystemStore item, given key and value. + * + * @param[in] key Key. + * @param[in] buffer Value data buffer. + * @param[in] size Value data size. + * @param[in] create_flags Flag mask. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int set(const char *key, const void *buffer, size_t size, uint32_t create_flags); + + /** + * @brief Get one FileSystemStore item, given key. + * + * @param[in] key Key. + * @param[in] buffer Value data buffer. + * @param[in] buffer_size Value data buffer size. + * @param[out] actual_size Actual read size. + * @param[in] offset Offset to read from in data. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int get(const char *key, void *buffer, size_t buffer_size, size_t *actual_size = NULL, size_t offset = 0); + + /** + * @brief Get information of a given key. + * + * @param[in] key Key. + * @param[out] info Returned information structure. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int get_info(const char *key, info_t *info); + + /** + * @brief Remove a FileSystemStore item, given key. + * + * @param[in] key Key. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int remove(const char *key); + + /** + * @brief Start an incremental FileSystemStore set sequence. + * + * @param[out] handle Returned incremental set handle. + * @param[in] key Key. + * @param[in] final_data_size Final value data size. + * @param[in] create_flags Flag mask. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int set_start(set_handle_t *handle, const char *key, size_t final_data_size, uint32_t create_flags); + + /** + * @brief Add data to incremental FileSystemStore set sequence. + * + * @param[in] handle Incremental set handle. + * @param[in] value_data value data to add. + * @param[in] data_size value data size. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int set_add_data(set_handle_t handle, const void *value_data, size_t data_size); + + /** + * @brief Finalize an incremental FileSystemStore set sequence. + * + * @param[in] handle Incremental set handle. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int set_finalize(set_handle_t handle); + + /** + * @brief Start an iteration over FileSystemStore keys. + * + * @param[out] it Returned iterator handle. + * @param[in] prefix Key prefix (null for all keys). + * + * @returns 0 on success or a negative error code on failure + */ + virtual int iterator_open(iterator_t *it, const char *prefix = NULL); + + /** + * @brief Get next key in iteration. + * + * @param[in] it Iterator handle. + * @param[in] key Buffer for returned key. + * @param[in] key_size Key buffer size. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int iterator_next(iterator_t it, char *key, size_t key_size); + + /** + * @brief Close iteration. + * + * @param[in] it Iterator handle. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int iterator_close(iterator_t it); + +private: + + // Key metadata + typedef struct { + uint32_t magic; + uint16_t metadata_size; + uint16_t revision; + uint32_t user_flags; + } key_metadata_t; + + /** + * @brief Build Full name class member from Key, as a combination of FSST folder and key name + * + * @param[in] key_src key file name + * + * @returns 0 on success or a negative error code on failure + */ + int _build_full_path_key(const char *key_src); + + /** + * @brief Verify Key file metadata validity and open it if valid + * + * @param[in] key in validated key file name. + * @param[in] key_metadata Returned key file metadata. + * @param[in] kv_file opened kv file handle (unless file doesnt exist) + * + * @returns 0 on success or a negative error code on failure + */ + int _verify_key_file(const char *key, key_metadata_t *key_metadata, File *kv_file); + +private: + FileSystem *_fs; + PlatformMutex _mutex; + PlatformMutex _inc_data_add_mutex; + + bool _is_initialized; + char *_cfg_fs_path; /* FileSystemStore path name on FileSystem */ + size_t _cfg_fs_path_size; /* Size of configured FileSystemStore path name on FileSystem */ + char *_full_path_key; /* Full name of Key file currently working on */ + size_t _cur_inc_data_size; /* Amount of data added to Key file so far, during incremental add data */ + set_handle_t _cur_inc_set_handle; /* handle of currently key file under incremental set process */ +}; + + +} //namespace mbed +#endif //MBED_FILE_SYSTEM_STORE_H diff --git a/features/storage/kvstore/global_api/kvstore_global_api.cpp b/features/storage/kvstore/global_api/kvstore_global_api.cpp new file mode 100644 index 00000000000..55044836300 --- /dev/null +++ b/features/storage/kvstore/global_api/kvstore_global_api.cpp @@ -0,0 +1,196 @@ +/* + * Copyright (c) 2018 ARM Limited. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "kvstore_global_api.h" + +#include "kv_config.h" +#include "KVMap.h" +#include "KVStore.h" +#include "mbed_error.h" + +using namespace mbed; + +// iterator handle +struct _opaque_kv_key_iterator { + bool iterator_is_open; + KVStore *kvstore_intance; + KVStore::iterator_t *iterator_handle; +}; + +int kv_set(const char *full_name_key, const void *buffer, size_t size, uint32_t create_flags) +{ + int ret = storage_configuration(); + if (MBED_SUCCESS != ret) { + return ret; + } + + KVMap& kv_map = KVMap::get_instance(); + KVStore *kv_instance = NULL; + size_t key_index = 0; + ret = kv_map.lookup(full_name_key, &kv_instance, &key_index); + if (ret != MBED_SUCCESS) { + return ret; + } + + ret = kv_instance->set(full_name_key + key_index, buffer, size, create_flags); + return ret; +} + +int kv_get(const char *full_name_key, void *buffer, size_t buffer_size, size_t *actual_size) +{ + int ret = storage_configuration(); + if (MBED_SUCCESS != ret) { + return ret; + } + + KVMap& kv_map = KVMap::get_instance(); + KVStore *kv_instance = NULL; + size_t key_index = 0; + ret = kv_map.lookup(full_name_key, &kv_instance, &key_index); + if (ret != MBED_SUCCESS) { + return ret; + } + + return kv_instance->get(full_name_key + key_index, buffer, buffer_size, actual_size); +} + +int kv_get_info(const char *full_name_key, kv_info_t *info) +{ + int ret = storage_configuration(); + if (MBED_SUCCESS != ret) { + return ret; + } + + KVMap& kv_map = KVMap::get_instance(); + KVStore *kv_instance = NULL; + size_t key_index = 0; + ret = kv_map.lookup(full_name_key, &kv_instance, &key_index); + if (ret != MBED_SUCCESS) { + return ret; + } + + KVStore::info_t inner_info; + ret = kv_instance->get_info(full_name_key + key_index, &inner_info); + if (MBED_SUCCESS != ret) { + return ret; + } + info->flags = inner_info.flags; + info->size = inner_info.size; + return ret; +} + +int kv_remove(const char *full_name_key) +{ + int ret = storage_configuration(); + if (MBED_SUCCESS != ret) { + return ret; + } + + KVMap& kv_map = KVMap::get_instance(); + KVStore *kv_instance = NULL; + size_t key_index = 0; + ret = kv_map.lookup(full_name_key, &kv_instance, &key_index); + if (ret != MBED_SUCCESS) { + return ret; + } + + return kv_instance->remove(full_name_key + key_index); +} + +int kv_iterator_open(kv_iterator_t *it, const char *full_prefix) +{ + if (it == NULL || full_prefix == NULL) { + return MBED_ERROR_INVALID_ARGUMENT; + } + + int ret = storage_configuration(); + if (MBED_SUCCESS != ret) { + return ret; + } + + (*it) = new _opaque_kv_key_iterator; + if (*it == NULL) { + return MBED_ERROR_FAILED_OPERATION; + } + (*it)->iterator_is_open = false; + + KVMap& kv_map = KVMap::get_instance(); + KVStore *kv_instance = NULL; + size_t key_index = 0; + ret = kv_map.lookup(full_prefix, &kv_instance, &key_index); + if (ret != MBED_SUCCESS) { + delete (*it); + return ret; + } + + (*it)->kvstore_intance = kv_instance; + KVStore::iterator_t *inner_it = new KVStore::iterator_t; + ret = kv_instance->iterator_open(inner_it, full_prefix + key_index); + if (MBED_SUCCESS != ret) { + delete inner_it; + delete (*it); + return ret; + } + + (*it)->iterator_handle = inner_it; + (*it)->iterator_is_open = true; + return ret; + +} + +int kv_iterator_next(kv_iterator_t it, char *key, size_t key_size) +{ + if (!it->iterator_is_open) { + return MBED_ERROR_INVALID_ARGUMENT; + } + + return it->kvstore_intance->iterator_next(*it->iterator_handle, key, key_size); +} + +int kv_iterator_close(kv_iterator_t it) +{ + if (!it->iterator_is_open) { + return MBED_ERROR_INVALID_ARGUMENT; + } + + int ret = it->kvstore_intance->iterator_close(*it->iterator_handle); + + delete it->iterator_handle; + delete it; + + return ret; +} + +int kv_reset(const char *kvstore_name) +{ + int ret = storage_configuration(); + if (MBED_SUCCESS != ret) { + return ret; + } + + KVMap& kv_map = KVMap::get_instance(); + KVStore *kv_instance = NULL; + size_t key_index = 0; + ret = kv_map.lookup(kvstore_name, &kv_instance, &key_index); + if (ret != MBED_SUCCESS) { + return ret; + } + + ret = kv_instance->reset(); + + return ret; + +} + diff --git a/features/storage/kvstore/global_api/kvstore_global_api.h b/features/storage/kvstore/global_api/kvstore_global_api.h new file mode 100644 index 00000000000..69ae4921958 --- /dev/null +++ b/features/storage/kvstore/global_api/kvstore_global_api.h @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2018 ARM Limited. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef _KVSTORE_STATIC_API +#define _KVSTORE_STATIC_API + +#include "stddef.h" +#include "stdint.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct _opaque_kv_key_iterator *kv_iterator_t; + +#define KV_WRITE_ONCE_FLAG (1 << 0) +#define KV_ENCRYPT_FLAG (1 << 1) +#define KV_AUTHENTICATE_FLAG (1 << 2) +#define KV_ROLLBACK_PROTECT_FLAG (1 << 3) + +static const uint32_t KV_MAX_KEY_LENGTH = 128; + +typedef struct info { + size_t size; + uint32_t flags; +} kv_info_t; + +/** + * @brief Set one KVStore item, given key and value. + * + * @param[in] full_name_key /Partition_path/Key. + * @param[in] buffer Value data buffer. + * @param[in] size Value data size. + * @param[in] create_flags Flag mask. + * + * @returns 0 on success or a negative error code on failure + */ +int kv_set(const char *full_name_key, const void *buffer, size_t size, uint32_t create_flags); + +/** + * @brief Get one KVStore item, given key. + * + * @param[in] full_name_key /Partition_path/Key. + * @param[in] buffer Value data buffer. + * @param[in] buffer_size Value data buffer size. + * @param[out] actual_size Actual read size. + * + * @returns 0 on success or a negative error code on failure + */ +int kv_get(const char *full_name_key, void *buffer, size_t buffer_size, size_t *actual_size); + +/** + * @brief Get information of a given key. + * + * @param[in] full_name_key /Partition_path/Key. + * @param[out] info Returned information structure. + * + * @returns 0 on success or a negative error code on failure + */ +int kv_get_info(const char *full_name_key, kv_info_t *info); + +/** + * @brief Remove a KVStore item, given key. + * + * @param[in] full_name_key Partition_path/Key. + * + * @returns 0 on success or a negative error code on failure + */ +int kv_remove(const char *full_name_key); + +/** + * @brief Start an iteration over KVStore keys to find all the entries + * that fit the full_prefix + * + * @param[out] it Allocating iterator handle. + * Do not forget to call kv_iterator_close + * to deallocate the memory. + * @param[in] full_prefix full_prefix Partition/Key prefix. If + * empty key or NULL pointer, all keys + * will match. + * + * @returns 0 on success or a negative error code on failure + */ +int kv_iterator_open(kv_iterator_t *it, const char *full_prefix); + +/** + * @brief Get next key in iteration that matches the prefix. + * + * @param[in] it Iterator handle. + * @param[in] key Buffer for returned key. + * @param[in] key_size Key buffer size. + * + * @returns 0 on success or a negative error code on failure + */ +int kv_iterator_next(kv_iterator_t it, char *key, size_t key_size); + +/** + * @brief Close iteration and de-allocate the iterator handle. + * + * @param[in] it Iterator handle. + * + * @returns 0 on success or a negative error code on failure + */ +int kv_iterator_close(kv_iterator_t it); + +/** + * @brief Remove all keys and related data + * + * @param[in] kvstore_path /Pratition/ + * + * @returns 0 on success or a negative error code on failure + */ +int kv_reset(const char *kvstore_path); + +#ifdef __cplusplus +} // closing brace for extern "C" +#endif +#endif diff --git a/features/storage/kvstore/kv_map/KVMap.cpp b/features/storage/kvstore/kv_map/KVMap.cpp new file mode 100644 index 00000000000..273adc092ea --- /dev/null +++ b/features/storage/kvstore/kv_map/KVMap.cpp @@ -0,0 +1,352 @@ +/* + * Copyright (c) 2018 ARM Limited. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "KVStore.h" +#include "KVMap.h" +#include "kv_config.h" +#include +#include "string.h" +#include "mbed_error.h" + +namespace mbed { + +KVMap::~KVMap() +{ + deinit(); +} + +int KVMap::init() +{ + int ret = MBED_SUCCESS; + + _mutex->lock(); + + if (_is_initialized) { + goto exit; + } + + _kv_num_attached_kvs = 0; + memset(&_kv_map_table, 0, sizeof(_kv_map_table)); + + _is_initialized = 1; + +exit: + _mutex->unlock(); + return ret; +} + +int KVMap::attach(const char *partition_name, kvstore_config_t *kv_config) +{ + int ret = MBED_SUCCESS; + char * kv_partition_name = NULL; + + _mutex->lock(); + + if (!_is_initialized) { + ret = MBED_ERROR_NOT_READY; + goto exit; + } + + if (_kv_num_attached_kvs >= MAX_ATTACHED_KVS) { + ret = MBED_ERROR_OUT_OF_MEMORY; + goto exit; + } + + kv_partition_name = new char[strlen(partition_name+1)]; + strcpy(kv_partition_name, partition_name); + _kv_map_table[_kv_num_attached_kvs].partition_name = kv_partition_name; + _kv_map_table[_kv_num_attached_kvs].kv_config = kv_config; + _kv_num_attached_kvs++; + +exit: + _mutex->unlock(); + return ret; +} + +void KVMap::deinit_partition(kv_map_entry_t * partition) +{ + free(partition->partition_name); + + if (partition->kv_config == NULL) { + return; + } + + if (partition->kv_config->external_store != NULL) { + partition->kv_config->external_store->deinit(); + } + + if (partition->kv_config->external_fs != NULL) { + partition->kv_config->external_fs->unmount(); + } + + if (partition->kv_config->external_bd != NULL) { + partition->kv_config->external_bd->deinit(); + } + + if (partition->kv_config->internal_store != NULL) { + partition->kv_config->internal_store->deinit(); + } + + if (partition->kv_config->internal_bd != NULL) { + partition->kv_config->internal_bd->deinit(); + } + + if (partition->kv_config->kvstore_main_instance != NULL) { + partition->kv_config->kvstore_main_instance->deinit(); + } + + delete [] partition->partition_name; + partition->partition_name = NULL; + partition->kv_config = NULL; +} + + +int KVMap::detach(const char *partition_name) +{ + int ret = MBED_SUCCESS; + + _mutex->lock(); + + if (!_is_initialized) { + ret = MBED_ERROR_NOT_READY; + goto exit; + } + + ret = MBED_ERROR_ITEM_NOT_FOUND; + for (int i = 0; i < _kv_num_attached_kvs; i++ ) { + + if (strcmp(partition_name, _kv_map_table[i].partition_name) != 0) { + continue; + } + + deinit_partition(&_kv_map_table[i]); + + memcpy(&_kv_map_table[i], &_kv_map_table[i + 1], sizeof(kv_map_entry_t) * (MAX_ATTACHED_KVS - i - 1)); + _kv_map_table[MAX_ATTACHED_KVS - 1].partition_name = NULL; + _kv_map_table[MAX_ATTACHED_KVS - 1].kv_config->kvstore_main_instance = NULL; + _kv_num_attached_kvs--; + ret = MBED_SUCCESS; + break; + } + +exit: + _mutex->unlock(); + return ret; +} + +int KVMap::deinit() +{ + int ret = MBED_SUCCESS; + + _mutex->lock(); + + if (!_is_initialized) { + ret = MBED_ERROR_NOT_READY; + goto exit; + } + + for (int i = 0; i < _kv_num_attached_kvs; i++ ) { + + if (_kv_map_table[i].kv_config->kvstore_main_instance == NULL) { + goto exit; + } + + deinit_partition(&_kv_map_table[i]); + } + +exit: + _kv_num_attached_kvs = 0; + _mutex->unlock(); + return ret; +} + +// Full name lookup and then break it into KVStore instance and key +int KVMap::lookup(const char *full_name, KVStore **kv_instance, size_t *key_index) +{ + _mutex->lock(); + + kvstore_config_t * kv_config; + int ret = config_lookup(full_name, &kv_config, key_index); + if (ret != MBED_SUCCESS) { + goto exit; + } + + *kv_instance = kv_config->kvstore_main_instance; + +exit: + _mutex->unlock(); + return ret; +} + +// Full name lookup and then break it into KVStore configuration struct and key +int KVMap::config_lookup(const char *full_name, kvstore_config_t **kv_config, size_t *key_index) +{ + int ret = MBED_SUCCESS; + int delimiter_index; + int i; + const char * delimiter_position; + + const char *temp_str = full_name; + + if (!_is_initialized) { + ret = MBED_ERROR_NOT_READY; + goto exit; + } + *key_index = 0; + if (*temp_str == '/') { + temp_str++; + (*key_index)++; + } + + delimiter_position = strchr(temp_str, '/'); + if (delimiter_position == NULL) { //delimiter not found + delimiter_index = -1; + *kv_config = _kv_map_table[0].kv_config; + goto exit; + } + + delimiter_index = delimiter_position - temp_str; + for (i = 0; i < _kv_num_attached_kvs; i++ ) { + + if (strncmp(temp_str, _kv_map_table[i].partition_name, delimiter_index) != 0) { + continue; + } + + *kv_config = _kv_map_table[i].kv_config; + break; + } + if (i == _kv_num_attached_kvs) { + ret = MBED_ERROR_ITEM_NOT_FOUND; + goto exit; + } +exit: + if (ret == MBED_SUCCESS) { + //if success extract the key + *key_index = *key_index + delimiter_index + 1; + } + return ret; +} + +KVStore * KVMap::get_internal_kv_instance(const char *name) +{ + + _mutex->lock(); + + kvstore_config_t * kv_config; + size_t key_index = 0; + + int ret = config_lookup(name, &kv_config, &key_index); + if (ret != MBED_SUCCESS) { + goto exit; + } +exit: + _mutex->unlock(); + + return ret != MBED_SUCCESS ? NULL : kv_config->internal_store; +} + +KVStore * KVMap::get_external_kv_instance(const char *name) +{ + + _mutex->lock(); + + kvstore_config_t * kv_config; + size_t key_index = 0; + + int ret = config_lookup(name, &kv_config, &key_index); + if (ret != MBED_SUCCESS) { + goto exit; + } +exit: + _mutex->unlock(); + + return ret != MBED_SUCCESS ? NULL : kv_config->external_store; +} + +KVStore * KVMap::get_main_kv_instance(const char *name) +{ + + _mutex->lock(); + + kvstore_config_t * kv_config; + size_t key_index = 0; + + int ret = config_lookup(name, &kv_config, &key_index); + if (ret != MBED_SUCCESS) { + goto exit; + } +exit: + _mutex->unlock(); + + return ret != MBED_SUCCESS ? NULL : kv_config->kvstore_main_instance; +} + +BlockDevice * KVMap::get_internal_blockdevice_instance(const char *name) +{ + + _mutex->lock(); + + kvstore_config_t * kv_config; + size_t key_index = 0; + + int ret = config_lookup(name, &kv_config, &key_index); + if (ret != MBED_SUCCESS) { + goto exit; + } +exit: + _mutex->unlock(); + + return ret != MBED_SUCCESS ? NULL : kv_config->internal_bd; +} + +BlockDevice * KVMap::get_external_blockdevice_instance(const char *name) +{ + + _mutex->lock(); + + kvstore_config_t * kv_config; + size_t key_index = 0; + + int ret = config_lookup(name, &kv_config, &key_index); + if (ret != MBED_SUCCESS) { + goto exit; + } +exit: + _mutex->unlock(); + + return ret != MBED_SUCCESS ? NULL : kv_config->external_bd; +} + +FileSystem * KVMap::get_external_filesystem_instance(const char *name) +{ + + _mutex->lock(); + + kvstore_config_t * kv_config; + size_t key_index = 0; + + int ret = config_lookup(name, &kv_config, &key_index); + if (ret != MBED_SUCCESS) { + goto exit; + } +exit: + _mutex->unlock(); + + return ret != MBED_SUCCESS ? NULL : kv_config->external_fs; +} + +} + diff --git a/features/storage/kvstore/kv_map/KVMap.h b/features/storage/kvstore/kv_map/KVMap.h new file mode 100644 index 00000000000..686c27fc29f --- /dev/null +++ b/features/storage/kvstore/kv_map/KVMap.h @@ -0,0 +1,187 @@ +/* + * Copyright (c) 2018 ARM Limited. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef _KV_MAP +#define _KV_MAP + +#include "KVStore.h" +#include "platform/PlatformMutex.h" +#include "platform/SingletonPtr.h" +#include "BlockDevice.h" +#include "FileSystem.h" + +namespace mbed { + +#define MAX_ATTACHED_KVS 3 + +typedef struct { + KVStore *kvstore_main_instance; + KVStore *internal_store; + KVStore *external_store; + BlockDevice *internal_bd; + BlockDevice *external_bd; + FileSystem *external_fs; +} kvstore_config_t; + +typedef struct { + char *partition_name; + kvstore_config_t *kv_config; +} kv_map_entry_t; + +class KVMap : private mbed::NonCopyable { +public: + + /** + * @brief As a singleton, return the single instance of the class. + * Reason for this class being a singleton is the following: + * - Ease of use for users of this class not having to coordinate instantiations. + * - Lazy instantiation of internal data (which we can't achieve with simple static classes). + * + * @returns Singleton instance reference. + */ + static KVMap& get_instance() + { + // Use this implementation of singleton (Meyer's) rather than the one that allocates + // the instance on the heap, as it ensures destruction at program end (preventing warnings + // from memory checking tools, such as valgrind). + static KVMap instance; + return instance; + } + + ~KVMap(); + + /** + * @brief Initializes KVMap + * + * @return 0 on success, negative error code on failure + */ + int init(); + + /** + * @brief attach a kv store partition configuration and add it to the KVMap array + * + * @param partition_name string parameter contains the partition name. + * @param kv_config a configuration struct created by the kv_config or by the user. + * @return 0 on success, negative error code on failure + */ + int attach(const char *partition_name, kvstore_config_t *kv_config); + + /** + * @brief detach a kv store partition configuration from the KVMap array + * and deinitialize its components + * + * @param partition_name string parameter contains the partition name. + * @return 0 on success, negative error code on failure + */ + int detach(const char *partition_name); + + /** + * @brief deinitialize the KVMap array and deinitialize all the attached partitions. + * + * @return 0 on success, negative error code on failure + */ + int deinit(); + + /** + * @brief full name lookup and then break it into KVStore instance and key + * + * @param full_name string parameter contains the /partition name/key. + * @param kv_instance the main KVStore instance associated with the required partition name. + * @param key_index an index to the first character of the key. + * @return 0 on success, negative error code on failure + */ + int lookup(const char *full_name, mbed::KVStore **kv_instance, size_t *key_index); + + /** + * @brief full name lookup and then break it into KVStore instance and key + * + * @param name string parameter contains the /partition name/. + * + * @return pointer to the internal kvstore on success, + * NULL on failure or if not exist + */ + KVStore * get_internal_kv_instance(const char *name); + /** + * @brief full name lookup and then break it into KVStore instance and key + * + * @param name string parameter contains the /partition name/. + * + * @return pointer to the external kvstore on success, + * NULL on failure or if not exist + */ + KVStore * get_external_kv_instance(const char *name); + /** + * @brief full name lookup and then break it into KVStore instance and key + * + * @param name string parameter contains the /partition name/. + * + * @return pointer to the main kvstore on success, + * NULL on failure or if not exist + */ + KVStore * get_main_kv_instance(const char *name); + /** + * @brief full name lookup and then break it into KVStore instance and key + * + * @param name string parameter contains the /partition name/. + * + * @return pointer to the internal BlockDevice on success, + * NULL on failure or if not exist + */ + BlockDevice * get_internal_blockdevice_instance(const char *name); + /** + * @brief full name lookup and then break it into KVStore instance and key + * + * @param name string parameter contains the /partition name/. + * + * @return pointer to the external BlockDevice on success, + * NULL on failure or if not exist + */ + BlockDevice * get_external_blockdevice_instance(const char *name); + /** + * @brief full name lookup and then break it into KVStore instance and key or if not exist + * + * @param name string parameter contains the /partition name/. + * + * @return pointer to the external FileSystem on success, + * NULL on failure or if not exist + */ + FileSystem * get_external_filesystem_instance(const char *name); +private: + + /** + * @brief deinitialize all component of a partition configuration struct. + * + * @param partition partition configuration struct. + */ + void deinit_partition(kv_map_entry_t * partition); + + /** + * @brief full name lookup and then break it into KVStore config and key + * + * @param full_name string parameter contains the /partition name/key. + * @param kv_config the configuration struct associated with the partition name + * @param key_index an index to the first character of the key. + * @return 0 on success, negative error code on failure + */ + int config_lookup(const char *full_name, kvstore_config_t **kv_config, size_t *key_index); + + // Attachment table + kv_map_entry_t _kv_map_table[MAX_ATTACHED_KVS]; + int _kv_num_attached_kvs; + int _is_initialized; + SingletonPtr _mutex; +}; +} +#endif diff --git a/features/storage/kvstore/securestore/SecureStore.cpp b/features/storage/kvstore/securestore/SecureStore.cpp new file mode 100644 index 00000000000..18ca1b83587 --- /dev/null +++ b/features/storage/kvstore/securestore/SecureStore.cpp @@ -0,0 +1,867 @@ +/* + * Copyright (c) 2018 ARM Limited. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// ----------------------------------------------------------- Includes ----------------------------------------------------------- + +#include "SecureStore.h" + +#include "aes.h" +#include "cmac.h" +#include "entropy.h" +#include "DeviceKey.h" +#include "mbed_assert.h" +#include "mbed_wait_api.h" +#include "mbed_error.h" +#include +#include +#include + +using namespace mbed; + +// --------------------------------------------------------- Definitions ---------------------------------------------------------- + +static const uint32_t securestore_revision = 1; + +static const uint32_t enc_block_size = 16; +static const uint32_t cmac_size = 16; +static const uint32_t iv_size = 8; +static const uint32_t scratch_buf_size = 256; +static const uint32_t derived_key_size = 16; + +static const char *const enc_prefix = "ENC"; +static const char *const auth_prefix = "AUTH"; + +typedef struct { + uint16_t metadata_size; + uint16_t revision; + uint32_t data_size; + uint32_t create_flags; + uint8_t iv[iv_size]; +} record_metadata_t; + +// incremental set handle +typedef struct { + record_metadata_t metadata; + char *key; + uint32_t offset_in_data; + uint8_t ctr_buf[enc_block_size]; + mbedtls_aes_context enc_ctx; + mbedtls_cipher_context_t auth_ctx; + KVStore::set_handle_t underlying_handle; +} inc_set_handle_t; + +// iterator handle +typedef struct { + KVStore::iterator_t underlying_it; +} key_iterator_handle_t; + + +// -------------------------------------------------- Local Functions Declaration ---------------------------------------------------- + +// -------------------------------------------------- Functions Implementation ---------------------------------------------------- + +int encrypt_decrypt_start(mbedtls_aes_context& enc_aes_ctx, uint8_t *iv, const char *key, + uint8_t *ctr_buf, uint8_t *salt_buf, int salt_buf_size) +{ + DeviceKey& devkey = DeviceKey::get_instance(); + char *salt = reinterpret_cast (salt_buf); + uint8_t encrypt_key[derived_key_size]; + strcpy(salt, enc_prefix); + int pos = strlen(enc_prefix); + strncpy(salt + pos, key, salt_buf_size - pos - 1); + salt_buf[salt_buf_size - 1] = 0; + int os_ret = devkey.generate_derived_key(salt_buf, strlen(salt), encrypt_key, DEVICE_KEY_16BYTE); + if (os_ret) { + return os_ret; + } + + mbedtls_aes_init(&enc_aes_ctx); + mbedtls_aes_setkey_enc(&enc_aes_ctx, encrypt_key, enc_block_size * 8); + + memcpy(ctr_buf, iv, iv_size); + memset(ctr_buf + iv_size, 0, iv_size); + + return 0; +} + +int encrypt_decrypt_data(mbedtls_aes_context& enc_aes_ctx, const uint8_t *in_buf, + uint8_t *out_buf, uint32_t chunk_size, uint8_t *ctr_buf, size_t& aes_offs) +{ + uint8_t stream_block[enc_block_size]; + + return mbedtls_aes_crypt_ctr(&enc_aes_ctx, chunk_size, &aes_offs, ctr_buf, + stream_block, in_buf, out_buf); +} + +int cmac_calc_start(mbedtls_cipher_context_t& auth_ctx, const char *key, uint8_t *salt_buf, int salt_buf_size) +{ + DeviceKey& devkey = DeviceKey::get_instance(); + char *salt = reinterpret_cast (salt_buf); + uint8_t auth_key[derived_key_size]; + strcpy(salt, auth_prefix); + int pos = strlen(auth_prefix); + strncpy(salt + pos, key, salt_buf_size - pos - 1); + salt_buf[salt_buf_size - 1] = 0; + int os_ret = devkey.generate_derived_key(salt_buf, strlen(salt), auth_key, DEVICE_KEY_16BYTE); + if (os_ret) { + return os_ret; + } + + const mbedtls_cipher_info_t *cipher_info = mbedtls_cipher_info_from_type(MBEDTLS_CIPHER_AES_128_ECB); + + mbedtls_cipher_init(&auth_ctx); + + if ((os_ret = mbedtls_cipher_setup(&auth_ctx, cipher_info)) != 0 ) { + return os_ret; + } + + os_ret = mbedtls_cipher_cmac_starts(&auth_ctx, auth_key, cmac_size * 8); + if (os_ret != 0) { + return os_ret; + } + + return 0; +} + +int cmac_calc_data(mbedtls_cipher_context_t& auth_ctx, const void *input, size_t ilen) +{ + int os_ret; + + os_ret = mbedtls_cipher_cmac_update(&auth_ctx, static_cast (input), ilen); + + return os_ret; +} + +int cmac_calc_finish(mbedtls_cipher_context_t& auth_ctx, uint8_t *output) +{ + int os_ret; + + os_ret = mbedtls_cipher_cmac_finish(&auth_ctx, output); + + return os_ret; +} + + + + +// Class member functions + +SecureStore::SecureStore(KVStore *underlying_kv, KVStore *rbp_kv) : + _is_initialized(false), _underlying_kv(underlying_kv), _rbp_kv(rbp_kv), _entropy(0), + _inc_set_handle(0), _scratch_buf(0) +{ +} + +SecureStore::~SecureStore() +{ + deinit(); +} + + +int SecureStore::set_start(set_handle_t *handle, const char *key, size_t final_data_size, + uint32_t create_flags) +{ + int ret, os_ret; + inc_set_handle_t *ih; + info_t info; + bool enc_started = false, auth_started = false; + + if (!_is_initialized) { + return MBED_ERROR_NOT_READY; + } + + if (!is_valid_key(key)) { + return MBED_ERROR_INVALID_ARGUMENT; + } + + // RBP requires authentication + if ((create_flags & ROLLBACK_PROTECT_FLAG) && !(create_flags & AUTHENTICATE_FLAG)) { + return MBED_ERROR_INVALID_ARGUMENT; + } + + *handle = static_cast (_inc_set_handle); + ih = reinterpret_cast (*handle); + + _mutex.lock(); + + ret = _underlying_kv->get(key, &ih->metadata, sizeof(record_metadata_t)); + if (ret == MBED_SUCCESS) { + // Must not remove Rollback protect flag + if (!(create_flags & ROLLBACK_PROTECT_FLAG) && (ih->metadata.create_flags & ROLLBACK_PROTECT_FLAG)) { + ret = MBED_ERROR_INVALID_ARGUMENT; + goto fail; + } + } else if (ret != MBED_ERROR_ITEM_NOT_FOUND) { + ret = MBED_ERROR_READ_FAILED; + goto fail; + } + + if (ih->metadata.create_flags & WRITE_ONCE_FLAG) { + // If write once flag set, check whether key exists in either of the underlying and RBP stores + if (ret != MBED_ERROR_ITEM_NOT_FOUND) { + ret = MBED_ERROR_WRITE_PROTECTED; + goto fail; + } + + if (_rbp_kv) { + ret = _rbp_kv->get_info(key, &info); + if (ret != MBED_ERROR_ITEM_NOT_FOUND) { + if (ret == MBED_SUCCESS) { + ret = MBED_ERROR_WRITE_PROTECTED; + } + goto fail; + } + } + } + + // Fill metadata + ih->metadata.create_flags = create_flags; + ih->metadata.data_size = final_data_size; + ih->metadata.metadata_size = sizeof(record_metadata_t); + ih->metadata.revision = securestore_revision; + + if (create_flags & ENCRYPT_FLAG) { + // generate a new random iv + os_ret = mbedtls_entropy_func(_entropy, ih->metadata.iv, iv_size); + if (os_ret) { + ret = MBED_ERROR_FAILED_OPERATION; + goto fail; + } + os_ret = encrypt_decrypt_start(ih->enc_ctx, ih->metadata.iv, key, ih->ctr_buf, _scratch_buf, + scratch_buf_size); + if (os_ret) { + ret = MBED_ERROR_FAILED_OPERATION; + goto fail; + } + enc_started = true; + } else { + memset(ih->metadata.iv, 0, iv_size); + } + + if (create_flags & AUTHENTICATE_FLAG) { + os_ret = cmac_calc_start(ih->auth_ctx, key, _scratch_buf, scratch_buf_size); + if (os_ret) { + ret = MBED_ERROR_FAILED_OPERATION; + goto fail; + } + auth_started = true; + // Although name is not part of the data, we calculate CMAC on it as well + os_ret = cmac_calc_data(ih->auth_ctx, key, strlen(key)); + if (os_ret) { + ret = MBED_ERROR_FAILED_OPERATION; + goto fail; + } + os_ret = cmac_calc_data(ih->auth_ctx, &ih->metadata, sizeof(record_metadata_t)); + if (os_ret) { + ret = MBED_ERROR_FAILED_OPERATION; + goto fail; + } + } + + ih->offset_in_data = 0; + ih->key = 0; + + ret = _underlying_kv->set_start(&ih->underlying_handle, key, + sizeof(record_metadata_t) + final_data_size + cmac_size, + create_flags); + if (ret) { + goto fail; + } + + ret = _underlying_kv->set_add_data(ih->underlying_handle, &ih->metadata, + sizeof(record_metadata_t)); + if (ret) { + goto fail; + } + + if (create_flags & ROLLBACK_PROTECT_FLAG) { + ih->key = new char[strlen(key) + 1]; + strcpy(ih->key, key); + } + + goto end; + +fail: + if (enc_started) { + mbedtls_aes_free(&ih->enc_ctx); + } + + if (auth_started) { + mbedtls_cipher_free(&ih->auth_ctx); + } + + // mark handle as invalid by clearing metadata size field in header + ih->metadata.metadata_size = 0; + _mutex.unlock(); + +end: + return ret; +} + +int SecureStore::set_add_data(set_handle_t handle, const void *value_data, size_t data_size) +{ + size_t aes_offs = 0; + int os_ret, ret = MBED_SUCCESS; + inc_set_handle_t *ih; + const uint8_t *src_ptr; + + if (handle != _inc_set_handle) { + return MBED_ERROR_INVALID_ARGUMENT; + } + + if (!value_data && data_size) { + return MBED_ERROR_INVALID_ARGUMENT; + } + + ih = reinterpret_cast (handle); + if (!ih->metadata.metadata_size) { + return MBED_ERROR_INVALID_ARGUMENT; + } + + if (ih->offset_in_data + data_size > ih->metadata.data_size) { + ret = MBED_ERROR_INVALID_SIZE; + goto end; + } + + src_ptr = static_cast (value_data); + while (data_size) { + uint32_t chunk_size; + const uint8_t *dst_ptr; + if (ih->metadata.create_flags & ENCRYPT_FLAG) { + // In encrypt mode we don't want to allocate a buffer in the size given by the user - + // Encrypt the data chunk by chunk + chunk_size = std::min((uint32_t) data_size, scratch_buf_size); + dst_ptr = _scratch_buf; + os_ret = encrypt_decrypt_data(ih->enc_ctx, src_ptr, _scratch_buf, + chunk_size, ih->ctr_buf, aes_offs); + if (os_ret) { + ret = MBED_ERROR_FAILED_OPERATION; + goto fail; + } + } else { + chunk_size = data_size; + dst_ptr = static_cast (value_data); + } + + if (ih->metadata.create_flags & AUTHENTICATE_FLAG) { + os_ret = cmac_calc_data(ih->auth_ctx, dst_ptr, chunk_size); + if (os_ret) { + ret = MBED_ERROR_FAILED_OPERATION; + goto fail; + } + } + + ret = _underlying_kv->set_add_data(ih->underlying_handle, dst_ptr, chunk_size); + if (ret) { + goto fail; + } + data_size -= chunk_size; + src_ptr += chunk_size; + ih->offset_in_data += chunk_size; + } + + goto end; + +fail: + if (ih->key) { + delete[] ih->key; + } + if (ih->metadata.create_flags & ENCRYPT_FLAG) { + mbedtls_aes_free(&ih->enc_ctx); + } + + if (ih->metadata.create_flags & AUTHENTICATE_FLAG) { + mbedtls_cipher_free(&ih->auth_ctx); + } + + // mark handle as invalid by clearing metadata size field in header + ih->metadata.metadata_size = 0; + _mutex.unlock(); + +end: + return ret; +} + +int SecureStore::set_finalize(set_handle_t handle) +{ + int os_ret, ret = MBED_SUCCESS; + inc_set_handle_t *ih; + uint8_t cmac[cmac_size] = {0}; + + if (handle != _inc_set_handle) { + return MBED_ERROR_INVALID_ARGUMENT; + } + + ih = reinterpret_cast (handle); + + if (!ih->metadata.metadata_size) { + return MBED_ERROR_INVALID_ARGUMENT; + } + + if (ih->offset_in_data != ih->metadata.data_size) { + ret = MBED_ERROR_INVALID_SIZE; + goto end; + } + + if (ih->metadata.create_flags & AUTHENTICATE_FLAG) { + os_ret = cmac_calc_finish(ih->auth_ctx, cmac); + if (os_ret) { + ret = MBED_ERROR_FAILED_OPERATION; + goto end; + } + } + + ret = _underlying_kv->set_add_data(ih->underlying_handle, cmac, cmac_size); + if (ret) { + goto end; + } + + ret = _underlying_kv->set_finalize(ih->underlying_handle); + if (ret) { + goto end; + } + + if (_rbp_kv && (ih->metadata.create_flags & ROLLBACK_PROTECT_FLAG)) { + // In rollback protect case, we need to store CMAC in RBP store. + // If it's also write once case, set write once flag in the RBP key as well. + ret = _rbp_kv->set(ih->key, cmac, cmac_size, ih->metadata.create_flags & WRITE_ONCE_FLAG); + delete[] ih->key; + if (ret) { + goto end; + } + } + +end: + // mark handle as invalid by clearing metadata size field in header + ih->metadata.metadata_size = 0; + if (ih->metadata.create_flags & ENCRYPT_FLAG) { + mbedtls_aes_free(&ih->enc_ctx); + } + + if (ih->metadata.create_flags & AUTHENTICATE_FLAG) { + mbedtls_cipher_free(&ih->auth_ctx); + } + + _mutex.unlock(); + return ret; +} + +int SecureStore::set(const char *key, const void *buffer, size_t size, uint32_t create_flags) +{ + int ret; + set_handle_t handle; + + // Don't wait till we get to set_add_data to catch this + if (!buffer && size) { + return MBED_ERROR_INVALID_ARGUMENT; + } + + ret = set_start(&handle, key, size, create_flags); + if (ret) { + return ret; + } + + ret = set_add_data(handle, buffer, size); + if (ret) { + return ret; + } + + ret = set_finalize(handle); + return ret; +} + +int SecureStore::remove(const char *key) +{ + info_t info; + _mutex.lock(); + + int ret = do_get(key, 0, 0, 0, 0, &info); + if (ret) { + goto end; + } + + if (info.flags & WRITE_ONCE_FLAG) { + ret = MBED_ERROR_WRITE_PROTECTED; + goto end; + } + + ret = _underlying_kv->remove(key); + if (ret) { + goto end; + } + + if (_rbp_kv && (info.flags & ROLLBACK_PROTECT_FLAG)) { + ret = _rbp_kv->remove(key); + if ((ret != MBED_SUCCESS) && (ret != MBED_ERROR_ITEM_NOT_FOUND)) { + goto end; + } + } + + ret = MBED_SUCCESS; + +end: + _mutex.unlock(); + return ret; +} + +int SecureStore::do_get(const char *key, void *buffer, size_t buffer_size, size_t *actual_size, + size_t offset, info_t *info) +{ + int os_ret, ret; + bool rbp_key_exists = false; + uint8_t rbp_cmac[cmac_size]; + size_t aes_offs = 0; + uint32_t data_size; + uint32_t actual_data_size; + uint32_t current_offset; + uint32_t chunk_size; + uint32_t enc_lead_size; + uint8_t *dest_buf; + bool enc_started = false, auth_started = false; + uint32_t create_flags; + + if (!is_valid_key(key)) { + return MBED_ERROR_INVALID_ARGUMENT; + } + + // Use member variable _inc_set_handle as no set operation is used now, + // and it saves us the need to define all members on stack + inc_set_handle_t *ih = static_cast (_inc_set_handle); + + if (_rbp_kv) { + ret = _rbp_kv->get(key, rbp_cmac, cmac_size, 0); + if (!ret) { + rbp_key_exists = true; + } else if (ret != MBED_ERROR_ITEM_NOT_FOUND) { + goto end; + } + } + + ret = _underlying_kv->get(key, &ih->metadata, sizeof(record_metadata_t)); + if (ret) { + // In case we have the key in the RBP KV, then even if the key wasn't found in + // the underlying KV, we may have been exposed to an attack. Return an RBP authentication error. + if (rbp_key_exists) { + ret = MBED_ERROR_RBP_AUTHENTICATION_FAILED; + } + goto end; + } + + create_flags = ih->metadata.create_flags; + if (!_rbp_kv) { + create_flags &= ~ROLLBACK_PROTECT_FLAG; + } + + // Another potential attack case - key hasn't got the RBP flag set, but exists in the RBP KV + if (rbp_key_exists && !(create_flags & ROLLBACK_PROTECT_FLAG)) { + ret = MBED_ERROR_RBP_AUTHENTICATION_FAILED; + goto end; + } + + if (create_flags & AUTHENTICATE_FLAG) { + os_ret = cmac_calc_start(ih->auth_ctx, key, _scratch_buf, scratch_buf_size); + if (os_ret) { + ret = MBED_ERROR_FAILED_OPERATION; + goto end; + } + auth_started = true; + + // Although name is not part of the data, we calculate CMAC on it as well + os_ret = cmac_calc_data(ih->auth_ctx, key, strlen(key)); + if (os_ret) { + ret = MBED_ERROR_FAILED_OPERATION; + goto end; + } + os_ret = cmac_calc_data(ih->auth_ctx, &ih->metadata, sizeof(record_metadata_t)); + if (os_ret) { + ret = MBED_ERROR_FAILED_OPERATION; + goto end; + } + } + + if (create_flags & ENCRYPT_FLAG) { + os_ret = encrypt_decrypt_start(ih->enc_ctx, ih->metadata.iv, key, ih->ctr_buf, _scratch_buf, + scratch_buf_size); + if (os_ret) { + ret = MBED_ERROR_FAILED_OPERATION; + goto end; + } + enc_started = true; + } + + data_size = ih->metadata.data_size; + actual_data_size = std::min((uint32_t) buffer_size, data_size - offset); + current_offset = 0; + enc_lead_size = 0; + + while (data_size) { + // Make sure we read to the user buffer only between offset and offset + actual_data_size + if ((current_offset >= offset) && (current_offset < offset + actual_data_size)) { + dest_buf = (static_cast (buffer)) + enc_lead_size; + chunk_size = actual_data_size - enc_lead_size; + enc_lead_size = 0; + } else { + dest_buf = _scratch_buf; + if (current_offset < offset) { + chunk_size = std::min(scratch_buf_size, offset - current_offset); + // A special case: encrypted user data starts at a middle of an encryption block. + // In this case, we need to read entire block into our scratch buffer, and copy + // the encrypted lead size to the user buffer start + if ((create_flags & ENCRYPT_FLAG) && + (chunk_size % enc_block_size)) { + enc_lead_size = std::min(enc_block_size - chunk_size % enc_block_size, actual_data_size); + chunk_size += enc_lead_size; + } + } else { + chunk_size = std::min(scratch_buf_size, data_size); + enc_lead_size = 0; + } + } + + ret = _underlying_kv->get(key, dest_buf, chunk_size, 0, + ih->metadata.metadata_size + current_offset); + if (ret != MBED_SUCCESS) { + goto end; + } + + if (create_flags & AUTHENTICATE_FLAG) { + os_ret = cmac_calc_data(ih->auth_ctx, dest_buf, chunk_size); + if (os_ret) { + ret = MBED_ERROR_FAILED_OPERATION; + goto end; + } + } + + if (create_flags & ENCRYPT_FLAG) { + // Decrypt data in place + os_ret = encrypt_decrypt_data(ih->enc_ctx, dest_buf, dest_buf, chunk_size, ih->ctr_buf, + aes_offs); + if (os_ret) { + ret = MBED_ERROR_FAILED_OPERATION; + goto end; + } + + if (enc_lead_size) { + // Now copy decrypted lead size to user buffer start + memcpy(buffer, dest_buf + chunk_size - enc_lead_size, enc_lead_size); + } + } + + current_offset += chunk_size; + data_size -= chunk_size; + } + + if (actual_size) { + *actual_size = actual_data_size; + } + + if (create_flags & AUTHENTICATE_FLAG) { + uint8_t calc_cmac[cmac_size], read_cmac[cmac_size]; + os_ret = cmac_calc_finish(ih->auth_ctx, calc_cmac); + if (os_ret) { + ret = MBED_ERROR_FAILED_OPERATION; + goto end; + } + + // Check with record CMAC + ret = _underlying_kv->get(key, read_cmac, cmac_size, 0, + ih->metadata.metadata_size + ih->metadata.data_size); + if (ret) { + goto end; + } + if (memcmp(calc_cmac, read_cmac, cmac_size) != 0) { + ret = MBED_ERROR_AUTHENTICATION_FAILED; + goto end; + } + + // If rollback protect, check also CMAC stored in RBP store + if (create_flags & ROLLBACK_PROTECT_FLAG) { + if (!rbp_key_exists) { + ret = MBED_ERROR_RBP_AUTHENTICATION_FAILED; + goto end; + } + if (memcmp(calc_cmac, rbp_cmac, cmac_size) != 0) { + ret = MBED_ERROR_RBP_AUTHENTICATION_FAILED; + goto end; + } + } + } + + if (info) { + info->flags = ih->metadata.create_flags; + info->size = ih->metadata.data_size; + } + +end: + ih->metadata.metadata_size = 0; + + if (enc_started) { + mbedtls_aes_free(&ih->enc_ctx); + } + + if (auth_started) { + mbedtls_cipher_free(&ih->auth_ctx); + } + + return ret; +} + +int SecureStore::get(const char *key, void *buffer, size_t buffer_size, size_t *actual_size, + size_t offset) +{ + _mutex.lock(); + int ret = do_get(key, buffer, buffer_size, actual_size, offset); + _mutex.unlock(); + + return ret; +} + +int SecureStore::get_info(const char *key, info_t *info) +{ + _mutex.lock(); + int ret = do_get(key, 0, 0, 0, 0, info); + _mutex.unlock(); + + return ret; +} + + +int SecureStore::init() +{ + int ret = MBED_SUCCESS; + + MBED_ASSERT(!(scratch_buf_size % enc_block_size)); + + _mutex.lock(); + + _entropy = new mbedtls_entropy_context; + mbedtls_entropy_init(static_cast(_entropy)); + + _scratch_buf = new uint8_t[scratch_buf_size]; + _inc_set_handle = new inc_set_handle_t; + + ret = _underlying_kv->init(); + if (ret) { + goto fail; + } + + if (_rbp_kv) { + ret = _rbp_kv->init(); + if (ret) { + goto fail; + } + } + + _is_initialized = true; + +fail: + _mutex.unlock(); + return ret; +} + +int SecureStore::deinit() +{ + _mutex.lock(); + if (_is_initialized) { + mbedtls_entropy_free(static_cast(_entropy)); + delete static_cast(_entropy); + delete static_cast(_inc_set_handle); + delete _scratch_buf; + // TODO: Deinit member KVs? + } + + _is_initialized = false; + _mutex.unlock(); + + return MBED_SUCCESS; +} + + +int SecureStore::reset() +{ + int ret; + + if (!_is_initialized) { + return MBED_ERROR_NOT_READY; + } + + _mutex.lock(); + ret = _underlying_kv->reset(); + if (ret) { + goto end; + } + + if (_rbp_kv) { + ret = _rbp_kv->reset(); + if (ret) { + goto end; + } + } + +end: + _mutex.unlock(); + return ret; +} + +int SecureStore::iterator_open(iterator_t *it, const char *prefix) +{ + key_iterator_handle_t *handle; + + if (!_is_initialized) { + return MBED_ERROR_NOT_READY; + } + + if (!it) { + return MBED_ERROR_INVALID_ARGUMENT; + } + + handle = new key_iterator_handle_t; + *it = reinterpret_cast(handle); + + return _underlying_kv->iterator_open(&handle->underlying_it, prefix); +} + +int SecureStore::iterator_next(iterator_t it, char *key, size_t key_size) +{ + key_iterator_handle_t *handle; + + if (!_is_initialized) { + return MBED_ERROR_NOT_READY; + } + + handle = reinterpret_cast(it); + + return _underlying_kv->iterator_next(handle->underlying_it, key, key_size); +} + +int SecureStore::iterator_close(iterator_t it) +{ + key_iterator_handle_t *handle; + int ret; + + if (!_is_initialized) { + return MBED_ERROR_NOT_READY; + } + + handle = reinterpret_cast(it); + + ret = _underlying_kv->iterator_close(handle->underlying_it); + + delete handle; + + return ret; +} + diff --git a/features/storage/kvstore/securestore/SecureStore.h b/features/storage/kvstore/securestore/SecureStore.h new file mode 100644 index 00000000000..876ccef06e9 --- /dev/null +++ b/features/storage/kvstore/securestore/SecureStore.h @@ -0,0 +1,213 @@ +/* + * Copyright (c) 2018 ARM Limited. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MBED_SECURESTORE_H +#define MBED_SECURESTORE_H + +#include +#include +#include "KVStore.h" +#include "PlatformMutex.h" + +namespace mbed { + +/** TDBStore class + * + * Lightweight Key Value storage over a block device + */ + +class SecureStore : public KVStore { +public: + + /** + * @brief Class constructor + * + * @param[in] underlying_bd Underlying KVSore. + * @param[in] rbp_kv Rollback protect KVStore. + * + * @returns none + */ + SecureStore(KVStore *underlying_kv, KVStore *rbp_kv = 0); + + /** + * @brief Class destructor + * + * @returns none + */ + virtual ~SecureStore(); + + /** + * @brief Initialize KVStore + * + * @returns 0 on success or a negative error code on failure + */ + virtual int init(); + + /** + * @brief Deinitialize KVStore + * + * @returns 0 on success or a negative error code on failure + */ + virtual int deinit(); + + + /** + * @brief Reset KVStore contents (clear all keys) + * Warning: This function is not thread safe. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int reset(); + + /** + * @brief Set one KVStore item, given key and value. + * + * @param[in] key Key. + * @param[in] buffer Value data buffer. + * @param[in] size Value data size. + * @param[in] create_flags Flag mask. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int set(const char *key, const void *buffer, size_t size, uint32_t create_flags); + + /** + * @brief Get one KVStore item, given key. + * + * @param[in] key Key. + * @param[in] buffer Value data buffer. + * @param[in] buffer_size Value data buffer size. + * @param[out] actual_size Actual read size. + * @param[in] offset Offset to read from in data. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int get(const char *key, void *buffer, size_t buffer_size, size_t *actual_size = NULL, + size_t offset = 0); + + /** + * @brief Get information of a given key. + * + * @param[in] key Key. + * @param[out] info Returned information structure. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int get_info(const char *key, info_t *info); + + /** + * @brief Remove a KVStore item, given key. + * + * @param[in] key Key. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int remove(const char *key); + + + /** + * @brief Start an incremental KVStore set sequence. + * + * @param[out] handle Returned incremental set handle. + * @param[in] key Key. + * @param[in] final_data_size Final value data size. + * @param[in] create_flags Flag mask. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int set_start(set_handle_t *handle, const char *key, size_t final_data_size, uint32_t create_flags); + + /** + * @brief Add data to incremental KVStore set sequence. + * + * @param[in] handle Incremental set handle. + * @param[in] value_data value data to add. + * @param[in] data_size value data size. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int set_add_data(set_handle_t handle, const void *value_data, size_t data_size); + + /** + * @brief Finalize an incremental KVStore set sequence. + * + * @param[in] handle Incremental set handle. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int set_finalize(set_handle_t handle); + + /** + * @brief Start an iteration over KVStore keys. + * + * @param[out] it Returned iterator handle. + * @param[in] prefix Key prefix (null for all keys). + * + * @returns 0 on success or a negative error code on failure + */ + virtual int iterator_open(iterator_t *it, const char *prefix = NULL); + + /** + * @brief Get next key in iteration. + * + * @param[in] it Iterator handle. + * @param[in] key Buffer for returned key. + * @param[in] key_size Key buffer size. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int iterator_next(iterator_t it, char *key, size_t key_size); + + /** + * @brief Close iteration. + * + * @param[in] it Iterator handle. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int iterator_close(iterator_t it); + +private: + + PlatformMutex _mutex; + bool _is_initialized; + KVStore *_underlying_kv, *_rbp_kv; + void *_entropy; + void *_inc_set_handle; + uint8_t *_scratch_buf; + + /** + * @brief Actual get function, serving get and get_info APIs. + * + * @param[in] key Key. + * @param[in] buffer Value data buffer. + * @param[in] buffer_size Value data buffer size. + * @param[out] actual_size Actual read size. + * @param[in] offset Offset to read from in data. + * @param[out] info Returned information structure. + * + * @returns 0 on success or a negative error code on failure + */ + int do_get(const char *key, void *buffer, size_t buffer_size, size_t *actual_size = NULL, + size_t offset = 0, info_t *info = 0); + + +}; +/** @}*/ + +} // namespace mbed + +#endif diff --git a/features/storage/kvstore/securestore/mbed_lib.json b/features/storage/kvstore/securestore/mbed_lib.json new file mode 100644 index 00000000000..0718963316f --- /dev/null +++ b/features/storage/kvstore/securestore/mbed_lib.json @@ -0,0 +1,6 @@ +{ + "name": "SecureStore", + "macros": ["MBEDTLS_CIPHER_MODE_CTR", "MBEDTLS_CMAC_C"], + "config": { + } +} diff --git a/features/storage/kvstore/tdbstore/TDBStore.cpp b/features/storage/kvstore/tdbstore/TDBStore.cpp new file mode 100644 index 00000000000..df5ef6afab6 --- /dev/null +++ b/features/storage/kvstore/tdbstore/TDBStore.cpp @@ -0,0 +1,1386 @@ +/* + * Copyright (c) 2018 ARM Limited. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// ----------------------------------------------------------- Includes ----------------------------------------------------------- + +#include "TDBStore.h" + +#include +#include +#include +#include "mbed_error.h" +#include "mbed_assert.h" +#include "mbed_wait_api.h" + +using namespace mbed; + +// --------------------------------------------------------- Definitions ---------------------------------------------------------- + +static const uint32_t delete_flag = (1UL << 31); + +typedef struct { + uint32_t magic; + uint16_t header_size; + uint16_t revision; + uint32_t flags; + uint16_t key_size; + uint16_t reserved; + uint32_t data_size; + uint32_t crc; +} record_header_t; + +typedef struct { + uint32_t hash; + bd_size_t bd_offset; +} ram_table_entry_t; + +static const char *master_rec_key = "TDBS"; +static const uint32_t tdbstore_magic = 0x54686683; // "TDBS" in ASCII +static const uint32_t tdbstore_revision = 1; + +typedef struct { + uint16_t version; + uint16_t tdbstore_revision; + uint32_t reserved; +} master_record_data_t; + +typedef enum { + TDBSTORE_AREA_STATE_NONE = 0, + TDBSTORE_AREA_STATE_EMPTY, + TDBSTORE_AREA_STATE_VALID, +} area_state_e; + +static const uint32_t work_buf_size = 64; +static const uint32_t initial_crc = 0xFFFFFFFF; +static const uint32_t initial_max_keys = 16; + +// incremental set handle +typedef struct { + record_header_t header; + bd_size_t bd_base_offset; + bd_size_t bd_curr_offset; + uint32_t offset_in_data; + uint32_t ram_table_ind; + uint32_t hash; + bool new_key; +} inc_set_handle_t; + +// iterator handle +typedef struct { + size_t ram_table_ind; + char *prefix; +} key_iterator_handle_t; + + +// -------------------------------------------------- Local Functions Declaration ---------------------------------------------------- + +// -------------------------------------------------- Functions Implementation ---------------------------------------------------- + +static inline uint32_t align_up(uint32_t val, uint32_t size) +{ + return (((val - 1) / size) + 1) * size; +} + + +// CRC32 calculation. Supports "rolling" calculation (using the initial value). +static uint32_t crc32(uint32_t init_crc, uint32_t data_size, const void *data_buf) +{ + uint32_t i, j; + uint32_t crc, mask; + const char *data = static_cast (data_buf); + + crc = init_crc; + for (i = 0; i < data_size; i++) { + crc = crc ^ (uint32_t) (data[i]); + for (j = 0; j < 8; j++) { + mask = -(crc & 1); + crc = (crc >> 1) ^ (0xEDB88320 & mask); + } + } + return crc; +} + + +// Class member functions + +TDBStore::TDBStore(BlockDevice *bd) : _ram_table(0), _max_keys(0), + _num_keys(0), _bd(bd), _buff_bd(0), _free_space_offset(0), _master_record_offset(0), + _is_initialized(false), _active_area(0), _active_area_version(0), _size(0), + _prog_size(0), _work_buf(0), _key_buf(0), _variant_bd_erase_unit_size(false), _inc_set_handle(0) +{ +} + +TDBStore::~TDBStore() +{ + deinit(); +} + +int TDBStore::read_area(uint8_t area, uint32_t offset, uint32_t size, void *buf) +{ + int os_ret = _buff_bd->read(buf, _area_params[area].address + offset, size); + + if (os_ret) { + return MBED_ERROR_READ_FAILED; + } + + return MBED_SUCCESS; +} + +int TDBStore::write_area(uint8_t area, uint32_t offset, uint32_t size, const void *buf) +{ + int os_ret = _buff_bd->program(buf, _area_params[area].address + offset, size); + if (os_ret) { + return MBED_ERROR_WRITE_FAILED; + } + + return MBED_SUCCESS; +} + +int TDBStore::erase_erase_unit(uint8_t area, uint32_t offset) +{ + uint32_t bd_offset = _area_params[area].address + offset; + uint32_t eu_size = _buff_bd->get_erase_size(bd_offset); + + int os_ret = _buff_bd->erase(bd_offset, eu_size); + if (os_ret) { + return MBED_ERROR_WRITE_FAILED; + } + return MBED_SUCCESS; +} + +void TDBStore::calc_area_params() +{ + size_t bd_size = _bd->size(); + + memset(_area_params, 0, sizeof(_area_params)); + size_t area_0_size = 0; + bd_size_t prev_erase_unit_size = _bd->get_erase_size(area_0_size); + _variant_bd_erase_unit_size = 0; + + while (area_0_size < bd_size / 2) { + bd_size_t erase_unit_size = _bd->get_erase_size(area_0_size); + _variant_bd_erase_unit_size |= (erase_unit_size != prev_erase_unit_size); + area_0_size += erase_unit_size; + } + + _area_params[0].address = 0; + _area_params[0].size = area_0_size; + _area_params[1].address = area_0_size; + _area_params[1].size = bd_size - area_0_size; +} + + +// This function, reading a record from the BD, is used for multiple purposes: +// - Init (scan all records, no need to return file name and data) +// - Get (return file data) +// - Get first/next file (check whether name matches, return name if so) +int TDBStore::read_record(uint8_t area, uint32_t offset, char *key, + void *data_buf, uint32_t data_buf_size, + uint32_t& actual_data_size, size_t data_offset, bool copy_key, + bool copy_data, bool check_expected_key, bool calc_hash, + uint32_t& hash, uint32_t& flags, uint32_t& next_offset) +{ + int ret; + record_header_t header; + uint32_t total_size, key_size, data_size; + uint32_t curr_data_offset; + char *user_key_ptr; + uint32_t crc = initial_crc; + // Upper layers typically use non zero offsets for reading the records chunk by chunk, + // so only validate entire record at first chunk (otherwise we'll have a serious performance penalty). + bool validate = (data_offset == 0); + + ret = MBED_SUCCESS; + // next offset should only be updated to the end of record if successful + next_offset = offset; + + ret = read_area(area, offset, sizeof(header), &header); + if (ret) { + return ret; + } + + if (header.magic != tdbstore_magic) { + return MBED_ERROR_INVALID_DATA_DETECTED; + } + + offset += align_up(sizeof(header), _prog_size); + + key_size = header.key_size; + data_size = header.data_size; + flags = header.flags; + + if ((!key_size) || (key_size >= MAX_KEY_SIZE)) { + return MBED_ERROR_INVALID_DATA_DETECTED; + } + + total_size = key_size + data_size; + + if (offset + total_size >= _size) { + return MBED_ERROR_INVALID_DATA_DETECTED; + } + + if (data_offset > data_size) { + return MBED_ERROR_INVALID_SIZE; + } + + actual_data_size = std::min(data_buf_size, data_size - data_offset); + + if (copy_data && actual_data_size && !data_buf) { + return MBED_ERROR_INVALID_ARGUMENT; + } + + if (validate) { + // Calculate CRC on header (excluding CRC itself) + crc = crc32(crc, sizeof(record_header_t) - sizeof(crc), &header); + curr_data_offset = 0; + } else { + // Non validation case: No need to read the key, nor the parts before data_offset + // or after the actual part requested by the user. + total_size = actual_data_size; + curr_data_offset = data_offset; + offset += data_offset + key_size; + // Mark code that key handling is finished + key_size = 0; + } + + user_key_ptr = key; + hash = initial_crc; + + while (total_size) { + uint8_t *dest_buf; + uint32_t chunk_size; + if (key_size) { + // This means that we're on the key part + if (copy_key) { + dest_buf = reinterpret_cast (user_key_ptr); + chunk_size = key_size; + user_key_ptr[key_size] = '\0'; + } else { + dest_buf = _work_buf; + chunk_size = std::min(key_size, work_buf_size); + } + } else { + // This means that we're on the data part + // We have four cases that need different handling: + // 1. Before data_offset - read to work buffer + // 2. After data_offset, but before actual part is finished - read to user buffer + // 3. After actual part is finished - read to work buffer + // 4. Copy data flag not set - read to work buffer + if (curr_data_offset < data_offset) { + chunk_size = std::min(work_buf_size, data_offset - curr_data_offset); + dest_buf = _work_buf; + } else if (copy_data && (curr_data_offset < data_offset + actual_data_size)) { + chunk_size = actual_data_size; + dest_buf = static_cast (data_buf); + } else { + chunk_size = std::min(work_buf_size, total_size); + dest_buf = _work_buf; + } + } + ret = read_area(area, offset, chunk_size, dest_buf); + if (ret) { + goto end; + } + + if (validate) { + // calculate CRC on current read chunk + crc = crc32(crc, chunk_size, dest_buf); + } + + if (key_size) { + // We're on key part. May need to calculate hash or check whether key is the expected one + if (check_expected_key) { + if (memcmp(user_key_ptr, dest_buf, chunk_size)) { + ret = MBED_ERROR_ITEM_NOT_FOUND; + } + } + + if (calc_hash) { + hash = crc32(hash, chunk_size, dest_buf); + } + + user_key_ptr += chunk_size; + key_size -= chunk_size; + if (!key_size) { + offset += data_offset; + } + } else { + curr_data_offset += chunk_size; + } + + total_size -= chunk_size; + offset += chunk_size; + } + + if (validate && (crc != header.crc)) { + ret = MBED_ERROR_INVALID_DATA_DETECTED; + goto end; + } + + next_offset = align_up(offset, _prog_size); + +end: + return ret; +} + +int TDBStore::find_record(uint8_t area, const char *key, uint32_t& offset, + uint32_t& ram_table_ind, uint32_t& hash) +{ + ram_table_entry_t *ram_table = (ram_table_entry_t *) _ram_table; + ram_table_entry_t *entry; + int ret = MBED_ERROR_ITEM_NOT_FOUND; + uint32_t actual_data_size; + uint32_t flags, dummy_hash, next_offset; + + + hash = crc32(initial_crc, strlen(key), key); + + for (ram_table_ind = 0; ram_table_ind < _num_keys; ram_table_ind++) { + entry = &ram_table[ram_table_ind]; + offset = entry->bd_offset; + if (hash < entry->hash) { + continue; + } + if (hash > entry->hash) { + return MBED_ERROR_ITEM_NOT_FOUND; + } + ret = read_record(_active_area, offset, const_cast (key), 0, 0, actual_data_size, 0, + false, false, true, false, dummy_hash, flags, next_offset); + // not found return code here means that hash doesn't belong to name. Continue searching. + if (ret != MBED_ERROR_ITEM_NOT_FOUND) { + break; + } + } + + return ret; +} + +uint32_t TDBStore::record_size(const char *key, uint32_t data_size) +{ + return align_up(sizeof(record_header_t), _prog_size) + + align_up(strlen(key) + data_size, _prog_size); +} + + +int TDBStore::set_start(set_handle_t *handle, const char *key, size_t final_data_size, + uint32_t create_flags) +{ + int ret; + uint32_t offset; + uint32_t hash, ram_table_ind; + inc_set_handle_t *ih; + + if (!is_valid_key(key)) { + return MBED_ERROR_INVALID_ARGUMENT; + } + + *handle = reinterpret_cast (_inc_set_handle); + ih = reinterpret_cast (*handle); + + if (!strcmp(key, master_rec_key)) { + // Master record - special case (no need to protect by the mutex, as it is already covered + // in the upper layers). + ih->bd_base_offset = _master_record_offset; + ih->new_key = false; + } else { + + _mutex.lock(); + + // A valid magic in the header means that this function has been called after an aborted + // incremental set process. This means that our media may be in a bad state - call GC. + if (ih->header.magic == tdbstore_magic) { + ret = garbage_collection(); + if (ret) { + goto fail; + } + } + + // If we have no room for the record, perform garbage collection + uint32_t rec_size = record_size(key, final_data_size); + if (_free_space_offset + rec_size > _size) { + ret = garbage_collection(); + if (ret) { + goto fail; + } + } + + // If even after GC we have no room for the record, return error + if (_free_space_offset + rec_size > _size) { + ret = MBED_ERROR_MEDIA_FULL; + goto fail; + } + + ret = find_record(_active_area, key, offset, ram_table_ind, hash); + + if (ret == MBED_SUCCESS) { + ret = read_area(_active_area, offset, sizeof(ih->header), &ih->header); + if (ret) { + goto fail; + } + if (ih->header.flags & WRITE_ONCE_FLAG) { + ret = MBED_ERROR_WRITE_PROTECTED; + goto fail; + } + ih->new_key = false; + } else if (ret == MBED_ERROR_ITEM_NOT_FOUND) { + if (create_flags & delete_flag) { + goto fail; + } + if (_num_keys >= _max_keys) { + increment_max_keys(); + } + ih->new_key = true; + } else { + goto fail; + } + ih->bd_base_offset = _free_space_offset; + + check_erase_before_write(_active_area, ih->bd_base_offset, rec_size); + } + + ret = MBED_SUCCESS; + + // Fill handle and header fields + // Jump to offset after header (header will be written at finalize phase) + ih->bd_curr_offset = ih->bd_base_offset + align_up(sizeof(record_header_t), _prog_size); + ih->offset_in_data = 0; + ih->hash = hash; + ih->ram_table_ind = ram_table_ind; + ih->header.magic = tdbstore_magic; + ih->header.header_size = sizeof(record_header_t); + ih->header.revision = tdbstore_revision; + ih->header.flags = create_flags; + ih->header.key_size = strlen(key); + ih->header.reserved = 0; + ih->header.data_size = final_data_size; + // Calculate CRC on header and key + ih->header.crc = crc32(initial_crc, sizeof(record_header_t) - sizeof(ih->header.crc), &ih->header); + ih->header.crc = crc32(ih->header.crc, ih->header.key_size, key); + + // Write key now + ret = write_area(_active_area, ih->bd_curr_offset, ih->header.key_size, key); + if (ret) { + goto fail; + } + ih->bd_curr_offset += ih->header.key_size; + goto end; + +fail: + // mark handle as invalid by clearing magic field in header + ih->header.magic = 0; + _mutex.unlock(); + +end: + return ret; +} + +int TDBStore::set_add_data(set_handle_t handle, const void *value_data, size_t data_size) +{ + int ret = MBED_SUCCESS; + inc_set_handle_t *ih; + + if (handle != _inc_set_handle) { + return MBED_ERROR_INVALID_ARGUMENT; + } + + if (!value_data && data_size) { + return MBED_ERROR_INVALID_ARGUMENT; + } + + _inc_set_mutex.lock(); + + ih = reinterpret_cast (handle); + + if (!ih->header.magic) { + ret = MBED_ERROR_INVALID_ARGUMENT; + goto end; + } + + if (ih->offset_in_data + data_size > ih->header.data_size) { + ret = MBED_ERROR_INVALID_SIZE; + goto end; + } + + // Update CRC with data chunk + ih->header.crc = crc32(ih->header.crc, data_size, value_data); + + // Write the data chunk + ret = write_area(_active_area, ih->bd_curr_offset, data_size, value_data); + if (ret) { + goto end; + } + ih->bd_curr_offset += data_size; + ih->offset_in_data += data_size; + +end: + _inc_set_mutex.unlock(); + return ret; +} + +int TDBStore::set_finalize(set_handle_t handle) +{ + int os_ret, ret = MBED_SUCCESS; + inc_set_handle_t *ih; + ram_table_entry_t *ram_table = (ram_table_entry_t *) _ram_table; + ram_table_entry_t *entry; + + if (handle != _inc_set_handle) { + return MBED_ERROR_INVALID_ARGUMENT; + } + + ih = reinterpret_cast (handle); + + if (!ih->header.magic) { + return MBED_ERROR_INVALID_ARGUMENT; + } + + _inc_set_mutex.lock(); + + if (ih->offset_in_data != ih->header.data_size) { + ret = MBED_ERROR_INVALID_SIZE; + // Need GC as otherwise our storage is left in a non-usable state + garbage_collection(); + goto end; + } + + // Write header + ret = write_area(_active_area, ih->bd_base_offset, sizeof(record_header_t), &ih->header); + if (ret) { + goto end; + } + + // Need to flush buffered BD as our record is totally written now + os_ret = _buff_bd->sync(); + if (os_ret) { + ret = MBED_ERROR_WRITE_FAILED; + goto end; + } + + // In master record case we don't update RAM table + if (ih->bd_base_offset == _master_record_offset) { + goto end; + } + + // Update RAM table + if (ih->header.flags & delete_flag) { + _num_keys--; + if (ih->ram_table_ind < _num_keys) { + memmove(&ram_table[ih->ram_table_ind], &ram_table[ih->ram_table_ind + 1], + sizeof(ram_table_entry_t) * (_num_keys - ih->ram_table_ind)); + } + } else { + if (ih->new_key) { + if (ih->ram_table_ind < _num_keys) { + memmove(&ram_table[ih->ram_table_ind + 1], &ram_table[ih->ram_table_ind], + sizeof(ram_table_entry_t) * (_num_keys - ih->ram_table_ind)); + } + _num_keys++; + } + entry = &ram_table[ih->ram_table_ind]; + entry->hash = ih->hash; + entry->bd_offset = ih->bd_base_offset; + } + + _free_space_offset = align_up(ih->bd_curr_offset, _prog_size); + +end: + // mark handle as invalid by clearing magic field in header + ih->header.magic = 0; + + _inc_set_mutex.unlock(); + if (ih->bd_base_offset != _master_record_offset) { + _mutex.unlock(); + } + return ret; +} + +int TDBStore::set(const char *key, const void *buffer, size_t size, uint32_t create_flags) +{ + int ret; + set_handle_t handle; + + // Don't wait till we get to set_add_data to catch this + if (!buffer && size) { + return MBED_ERROR_INVALID_ARGUMENT; + } + + ret = set_start(&handle, key, size, create_flags); + if (ret) { + return ret; + } + + ret = set_add_data(handle, buffer, size); + if (ret) { + return ret; + } + + ret = set_finalize(handle); + return ret; +} + +int TDBStore::remove(const char *key) +{ + return set(key, 0, 0, delete_flag); +} + +int TDBStore::get(const char *key, void *buffer, size_t buffer_size, size_t *actual_size, size_t offset) +{ + int ret; + uint32_t actual_data_size; + uint32_t bd_offset, next_bd_offset; + uint32_t flags, hash, ram_table_ind; + + if (!is_valid_key(key)) { + return MBED_ERROR_INVALID_ARGUMENT; + } + + _mutex.lock(); + + ret = find_record(_active_area, key, bd_offset, ram_table_ind, hash); + + if (ret != MBED_SUCCESS) { + goto end; + } + + ret = read_record(_active_area, bd_offset, const_cast(key), buffer, buffer_size, + actual_data_size, offset, false, true, false, false, hash, flags, next_bd_offset); + + if (actual_size) { + *actual_size = actual_data_size; + } + +end: + _mutex.unlock(); + return ret; +} + +int TDBStore::get_info(const char *key, info_t *info) +{ + int ret; + uint32_t bd_offset, next_bd_offset; + uint32_t flags, hash, ram_table_ind; + uint32_t actual_data_size; + + if (!is_valid_key(key)) { + return MBED_ERROR_INVALID_ARGUMENT; + } + + _mutex.lock(); + + ret = find_record(_active_area, key, bd_offset, ram_table_ind, hash); + + if (ret) { + goto end; + } + + // Give a large dummy buffer size in order to achieve actual data size + // (as copy_data flag is not set, data won't be copied anywhere) + ret = read_record(_active_area, bd_offset, const_cast(key), 0, (uint32_t) -1, + actual_data_size, 0, false, false, false, false, hash, flags, + next_bd_offset); + + if (ret) { + goto end; + } + + if (info) { + info->flags = flags; + info->size = actual_data_size; + } + +end: + _mutex.unlock(); + return ret; +} + +int TDBStore::write_master_record(uint8_t area, uint16_t version, uint32_t& next_offset) +{ + master_record_data_t master_rec; + + master_rec.version = version; + master_rec.tdbstore_revision = tdbstore_revision; + master_rec.reserved = 0; + next_offset = _master_record_offset + record_size(master_rec_key, sizeof(master_rec)); + return set(master_rec_key, &master_rec, sizeof(master_rec), 0); +} + +int TDBStore::copy_record(uint8_t from_area, uint32_t from_offset, uint32_t to_offset, + uint32_t& to_next_offset) +{ + int ret; + record_header_t header; + uint32_t total_size; + uint16_t chunk_size; + + ret = read_area(from_area, from_offset, sizeof(header), &header); + if (ret) { + return ret; + } + + total_size = align_up(sizeof(record_header_t), _prog_size) + + align_up(header.key_size + header.data_size, _prog_size);; + + + ret = check_erase_before_write(1 - from_area, to_offset, total_size); + if (ret) { + return ret; + } + + chunk_size = align_up(sizeof(record_header_t), _prog_size); + ret = write_area(1 - from_area, to_offset, chunk_size, &header); + if (ret) { + return ret; + } + + from_offset += chunk_size; + to_offset += chunk_size; + total_size -= chunk_size; + + while (total_size) { + chunk_size = std::min(total_size, work_buf_size); + ret = read_area(from_area, from_offset, chunk_size, _work_buf); + if (ret) { + return ret; + } + + ret = write_area(1 - from_area, to_offset, chunk_size, _work_buf); + if (ret) { + return ret; + } + + from_offset += chunk_size; + to_offset += chunk_size; + total_size -= chunk_size; + } + + to_next_offset = align_up(to_offset, _prog_size); + return MBED_SUCCESS; +} + +int TDBStore::copy_all_records(uint8_t from_area, uint32_t to_offset, + uint32_t& to_next_offset) +{ + return MBED_SUCCESS; +} + +int TDBStore::garbage_collection() +{ + ram_table_entry_t *ram_table = (ram_table_entry_t *) _ram_table; + uint32_t to_offset, to_next_offset; + uint32_t chunk_size, reserved_size; + int ret; + size_t ind; + + ret = check_erase_before_write(1 - _active_area, 0, _master_record_offset); + if (ret) { + return ret; + } + + // Copy reserved data + to_offset = 0; + reserved_size = RESERVED_AREA_SIZE; + + while (reserved_size) { + chunk_size = std::min(work_buf_size, reserved_size); + ret = read_area(_active_area, to_offset, chunk_size, _work_buf + to_offset); + if (ret) { + return ret; + } + ret = write_area(1 - _active_area, to_offset, chunk_size, _work_buf + to_offset); + if (ret) { + return ret; + } + to_offset += chunk_size; + reserved_size -= chunk_size; + } + + + to_offset = _master_record_offset + record_size(master_rec_key, sizeof(master_record_data_t)); + + // Initialize in case table is empty + to_next_offset = to_offset; + + // Go over ram table and copy all entries to opposite area + for (ind = 0; ind < _num_keys; ind++) { + uint32_t from_offset = ram_table[ind].bd_offset; + ret = copy_record(_active_area, from_offset, to_offset, to_next_offset); + if (ret) { + return ret; + } + // Update RAM table + ram_table[ind].bd_offset = to_offset; + to_offset = to_next_offset; + } + + to_offset = to_next_offset; + _free_space_offset = to_next_offset; + + // Now we can switch to the new active area + _active_area = 1 - _active_area; + + // Now write master record, with version incremented by 1. + _active_area_version++; + ret = write_master_record(_active_area, _active_area_version, to_offset); + if (ret) { + return ret; + } + + // Now reset standby area + ret = reset_area(1 - _active_area); + if (ret) { + return ret; + } + + return MBED_SUCCESS; +} + + +int TDBStore::build_ram_table() +{ + ram_table_entry_t *ram_table = (ram_table_entry_t *) _ram_table; + uint32_t offset, next_offset = 0, dummy; + int ret = MBED_SUCCESS; + uint32_t hash; + uint32_t flags; + uint32_t actual_data_size; + uint32_t ram_table_ind; + + _num_keys = 0; + offset = _master_record_offset; + + while (offset < _free_space_offset) { + ret = read_record(_active_area, offset, _key_buf, 0, 0, actual_data_size, 0, + true, false, false, true, hash, flags, next_offset); + + if (ret) { + goto end; + } + + ret = find_record(_active_area, _key_buf, dummy, ram_table_ind, hash); + + if ((ret != MBED_SUCCESS) && (ret != MBED_ERROR_ITEM_NOT_FOUND)) { + goto end; + } + + uint32_t save_offset = offset; + offset = next_offset; + + if (ret == MBED_ERROR_ITEM_NOT_FOUND) { + // Key doesn't exist, need to add it to RAM table + + if (flags & delete_flag) { + continue; + } + if (_num_keys >= _max_keys) { + // In order to avoid numerous reallocations of ram table, + // Add a chunk of entries now + increment_max_keys(reinterpret_cast(&ram_table)); + } + memmove(&ram_table[ram_table_ind + 1], &ram_table[ram_table_ind], + sizeof(ram_table_entry_t) * (_num_keys - ram_table_ind)); + + _num_keys++; + } else if (flags & delete_flag) { + _num_keys--; + memmove(&ram_table[ram_table_ind], &ram_table[ram_table_ind + 1], + sizeof(ram_table_entry_t) * (_num_keys - ram_table_ind)); + + continue; + } + + // update record parameters + ram_table[ram_table_ind].hash = hash; + ram_table[ram_table_ind].bd_offset = save_offset; + } + +end: + _free_space_offset = next_offset; + return ret; +} + +int TDBStore::increment_max_keys(void **ram_table) +{ + // Reallocate ram table with new size + ram_table_entry_t *old_ram_table = (ram_table_entry_t *) _ram_table; + ram_table_entry_t *new_ram_table = new ram_table_entry_t[_max_keys + 1]; + + // Copy old content to new table + memcpy(new_ram_table, old_ram_table, sizeof(ram_table_entry_t) * _max_keys); + _max_keys++; + + _ram_table = new_ram_table; + delete[] old_ram_table; + + if (ram_table) { + *ram_table = _ram_table; + } + return MBED_SUCCESS; +} + + +int TDBStore::init() +{ + ram_table_entry_t *ram_table; + area_state_e area_state[_num_areas]; + uint32_t next_offset; + uint32_t flags, hash; + uint32_t actual_data_size; + int os_ret, ret = MBED_SUCCESS; + uint16_t versions[_num_areas]; + + _mutex.lock(); + + _max_keys = initial_max_keys; + + ram_table = new ram_table_entry_t[_max_keys]; + _ram_table = ram_table; + _num_keys = 0; + + // Underlying BD size must fit into 32 bits + if ((uint32_t)_bd->size() != _bd->size()) { + MBED_ERROR(MBED_ERROR_INVALID_SIZE, "Underlying BD size should not exceed 32 bits"); + } + + // Underlying BD must have flash attributes, i.e. have an erase value + if (_bd->get_erase_value() == -1) { + MBED_ERROR(MBED_ERROR_INVALID_ARGUMENT, "Underlying BD must have flash attributes"); + } + + _size = (size_t) -1; + + _buff_bd = new BufferedBlockDevice(_bd); + _buff_bd->init(); + + _prog_size = _bd->get_program_size(); + _work_buf = new uint8_t[work_buf_size]; + _key_buf = new char[MAX_KEY_SIZE]; + _inc_set_handle = new inc_set_handle_t; + memset(_inc_set_handle, 0, sizeof(inc_set_handle_t)); + + _master_record_offset = align_up(RESERVED_AREA_SIZE, _prog_size); + + calc_area_params(); + + for (uint8_t area = 0; area < _num_areas; area++) { + area_state[area] = TDBSTORE_AREA_STATE_NONE; + versions[area] = 0; + + _size = std::min(_size, _area_params[area].size); + + // Check validity of master record + master_record_data_t master_rec; + ret = read_record(area, _master_record_offset, const_cast (master_rec_key), + &master_rec, sizeof(master_rec), actual_data_size, 0, false, true, true, false, + hash, flags, next_offset); + if ((ret != MBED_SUCCESS) && (ret != MBED_ERROR_INVALID_DATA_DETECTED)) { + MBED_ERROR(ret, "TDBSTORE: Unable to read record at init"); + } + + // Master record may be corrupt, but it can be erased. Now check if its entire erase unit is erased + if (ret == MBED_ERROR_INVALID_DATA_DETECTED) { + bool erased; + uint32_t erase_unit_num = _master_record_offset / _bd->get_erase_size(_master_record_offset); + if (is_erase_unit_erased(area, erase_unit_num, erased)) { + MBED_ERROR(MBED_ERROR_READ_FAILED, "TDBSTORE: Unable to check whether erase unit is erased at init"); + } + if (erased) { + area_state[area] = TDBSTORE_AREA_STATE_EMPTY; + continue; + } + } + + // We have a non valid master record, just reset the area. + if (ret == MBED_ERROR_INVALID_DATA_DETECTED) { + ret = reset_area(area); + if (ret) { + MBED_ERROR(ret, "TDBSTORE: Unable to reset area at init"); + } + area_state[area] = TDBSTORE_AREA_STATE_EMPTY; + continue; + } + versions[area] = master_rec.version; + + area_state[area] = TDBSTORE_AREA_STATE_VALID; + + // Unless both areas are valid (a case handled later), getting here means + // that we found our active area. + _active_area = area; + _active_area_version = versions[area]; + } + + // In case we have two empty areas, arbitrarily use area 0 as the active one. + if ((area_state[0] == TDBSTORE_AREA_STATE_EMPTY) && (area_state[1] == TDBSTORE_AREA_STATE_EMPTY)) { + _active_area = 0; + _active_area_version = 1; + ret = write_master_record(_active_area, _active_area_version, _free_space_offset); + if (ret) { + MBED_ERROR(ret, "TDBSTORE: Unable to write master record at init"); + } + // Nothing more to do here if active area is empty + goto end; + } + + // In case we have two valid areas, choose the one having the higher version (or 0 + // in case of wrap around). Erase the other one. + if ((area_state[0] == TDBSTORE_AREA_STATE_VALID) && (area_state[1] == TDBSTORE_AREA_STATE_VALID)) { + if ((versions[0] > versions[1]) || (!versions[0])) { + _active_area = 0; + } else { + _active_area = 1; + } + _active_area_version = versions[_active_area]; + ret = erase_erase_unit(1 - _active_area, 0); + if (ret) { + MBED_ERROR(ret, "TDBSTORE: Unable to erase media start at init"); + } + } + + // Currently set free space offset pointer to the end of free space. + // Ram table build process needs it, but will update it. + _free_space_offset = _size; + ret = build_ram_table(); + + if ((ret != MBED_SUCCESS) && (ret != MBED_ERROR_INVALID_DATA_DETECTED)) { + MBED_ERROR(ret, "TDBSTORE: Unable to build RAM table at init"); + } + + if ((ret == MBED_ERROR_INVALID_DATA_DETECTED) && (_free_space_offset < _size)) { + // Space after last valid record may be erased, hence "corrupt". Now check if it really is erased. + bool erased; + if (is_erase_unit_erased(_active_area, _free_space_offset, erased)) { + MBED_ERROR(MBED_ERROR_READ_FAILED, "TDBSTORE: Unable to check whether erase unit is erased at init"); + } + if (erased) { + // Erased - all good + ret = MBED_SUCCESS; + } + } + + // If we have a corrupt record somewhere, perform garbage collection to salvage + // all preceding records + if (ret == MBED_ERROR_INVALID_DATA_DETECTED) { + ret = garbage_collection(); + if (ret) { + MBED_ERROR(ret, "TDBSTORE: Unable to perform GC at init"); + } + os_ret = _buff_bd->sync(); + if (os_ret) { + MBED_ERROR(MBED_ERROR_WRITE_FAILED, "TDBSTORE: Unable to sync BD at init"); + } + } + +end: + _is_initialized = true; + _mutex.unlock(); + return ret; +} + +int TDBStore::deinit() +{ + _mutex.lock(); + if (_is_initialized) { + _buff_bd->deinit(); + delete _buff_bd; + + ram_table_entry_t *ram_table = (ram_table_entry_t *) _ram_table; + delete[] ram_table; + delete[] _work_buf; + delete[] _key_buf; + } + + _is_initialized = false; + _mutex.unlock(); + + return MBED_SUCCESS; +} + +int TDBStore::reset_area(uint8_t area) +{ + int ret; + uint32_t bd_offset = 0; + + // Erase reserved area and master record + do { + ret = erase_erase_unit(area, bd_offset); + if (ret) { + return ret; + } + bd_offset += _buff_bd->get_erase_size(bd_offset); + } while (bd_offset <= _master_record_offset); + + return MBED_SUCCESS; +} + +int TDBStore::reset() +{ + uint8_t area; + int ret; + + if (!_is_initialized) { + return MBED_ERROR_NOT_READY; + } + + _mutex.lock(); + + // Reset both areas + for (area = 0; area < _num_areas; area++) { + ret = reset_area(area); + if (ret) { + goto end; + } + } + + _active_area = 0; + _num_keys = 0; + _free_space_offset = _master_record_offset; + _active_area_version = 1; + + // Write an initial master record on active area + ret = write_master_record(_active_area, _active_area_version, _free_space_offset); + +end: + _mutex.unlock(); + return ret; +} + +int TDBStore::iterator_open(iterator_t *it, const char *prefix) +{ + key_iterator_handle_t *handle; + + if (!_is_initialized) { + return MBED_ERROR_NOT_READY; + } + + if (!it) { + return MBED_ERROR_INVALID_ARGUMENT; + } + + _mutex.lock(); + + handle = new key_iterator_handle_t; + *it = reinterpret_cast(handle); + + if (prefix && strcmp(prefix, "")) { + handle->prefix = new char[strlen(prefix) + 1]; + strcpy(handle->prefix, prefix); + } else { + handle->prefix = 0; + } + handle->ram_table_ind = 0; + + _mutex.unlock(); + + return MBED_SUCCESS; +} + +int TDBStore::iterator_next(iterator_t it, char *key, size_t key_size) +{ + ram_table_entry_t *ram_table = (ram_table_entry_t *) _ram_table; + key_iterator_handle_t *handle; + int ret; + uint32_t actual_data_size, hash, flags, next_offset; + + if (!_is_initialized) { + return MBED_ERROR_NOT_READY; + } + + _mutex.lock(); + + handle = reinterpret_cast(it); + + ret = MBED_ERROR_ITEM_NOT_FOUND; + + while (ret && (handle->ram_table_ind < _num_keys)) { + ret = read_record(_active_area, ram_table[handle->ram_table_ind].bd_offset, _key_buf, + 0, 0, actual_data_size, 0, true, false, false, false, hash, flags, next_offset); + if (ret) { + goto end; + } + if (!handle->prefix || (strstr(_key_buf, handle->prefix) == _key_buf)) { + if (strlen(_key_buf) >= key_size) { + ret = MBED_ERROR_INVALID_SIZE; + goto end; + } + strcpy(key, _key_buf); + } else { + ret = MBED_ERROR_ITEM_NOT_FOUND; + } + handle->ram_table_ind++; + } + +end: + _mutex.unlock(); + return ret; +} + +int TDBStore::iterator_close(iterator_t it) +{ + key_iterator_handle_t *handle; + + if (!_is_initialized) { + return MBED_ERROR_NOT_READY; + } + + _mutex.lock(); + + handle = reinterpret_cast(it); + delete[] handle->prefix; + delete handle; + + _mutex.unlock(); + + return MBED_SUCCESS; +} + +int TDBStore::reserved_data_set(const void *reserved_data, size_t reserved_data_buf_size) +{ + uint32_t check_size = RESERVED_AREA_SIZE, chunk_size, offset = 0; + uint8_t blank = _buff_bd->get_erase_value(); + int os_ret, ret = MBED_SUCCESS; + + if (reserved_data_buf_size > RESERVED_AREA_SIZE) { + return MBED_ERROR_INVALID_SIZE; + } + + _mutex.lock(); + + while (check_size) { + chunk_size = std::min(work_buf_size, (uint32_t) check_size); + ret = read_area(_active_area, offset, chunk_size, _work_buf + offset); + if (ret) { + goto end; + } + for (uint32_t i = 0; i < chunk_size; i++) { + if (_work_buf[i] != blank) { + ret = MBED_ERROR_WRITE_FAILED; + goto end; + } + } + offset += chunk_size; + check_size -= chunk_size; + } + + ret = write_area(_active_area, 0, reserved_data_buf_size, reserved_data); + if (ret) { + goto end; + } + + os_ret = _buff_bd->sync(); + if (os_ret) { + ret = MBED_ERROR_WRITE_FAILED; + } + +end: + _mutex.unlock(); + return ret; +} + +int TDBStore::reserved_data_get(void *reserved_data, size_t reserved_data_buf_size) +{ + _mutex.lock(); + if (reserved_data_buf_size > RESERVED_AREA_SIZE) { + reserved_data_buf_size = RESERVED_AREA_SIZE; + } + + int ret = read_area(_active_area, 0, reserved_data_buf_size, reserved_data); + + _mutex.unlock(); + return ret; +} + + +void TDBStore::offset_in_erase_unit(uint8_t area, uint32_t offset, + uint32_t& offset_from_start, uint32_t& dist_to_end) +{ + uint32_t bd_offset = _area_params[area].address + offset; + if (!_variant_bd_erase_unit_size) { + uint32_t eu_size = _bd->get_erase_size(); + offset_from_start = bd_offset % eu_size; + dist_to_end = eu_size - offset_from_start; + return; + } + + uint32_t agg_offset = 0; + while (bd_offset < agg_offset + _bd->get_erase_size(agg_offset)) { + agg_offset += _bd->get_erase_size(agg_offset); + } + offset_from_start = bd_offset - agg_offset; + dist_to_end = _bd->get_erase_size(agg_offset) - offset_from_start; + +} + +int TDBStore::is_erase_unit_erased(uint8_t area, uint32_t offset, bool& erased) +{ + uint32_t offset_from_start, dist; + offset_in_erase_unit(area, offset, offset_from_start, dist); + uint8_t buf[sizeof(record_header_t)], blanks[sizeof(record_header_t)]; + memset(blanks, _bd->get_erase_value(), sizeof(blanks)); + + while (dist) { + uint32_t chunk = std::min(dist, (uint32_t) sizeof(buf)); + int ret = read_area(area, offset, chunk, buf); + if (ret) { + return MBED_ERROR_READ_FAILED; + } + if (memcmp(buf, blanks, chunk)) { + erased = false; + return MBED_SUCCESS; + } + offset += chunk; + dist -= chunk; + } + erased = true; + return MBED_SUCCESS; +} + +int TDBStore::check_erase_before_write(uint8_t area, uint32_t offset, uint32_t size) +{ + // In order to save init time, we don't check that the entire area is erased. + // Instead, whenever reaching an erase unit start, check that it's erased, and if not - + // erase it. This is very not likely to happen (assuming area was initialized + // by TDBStore). This can be achieved as all records (except for the master record + // in offset 0) are written in an ascending order. + + if (!offset) { + // Master record in offset 0 is a special case - don't check it + return MBED_SUCCESS; + } + + while (size) { + uint32_t dist, offset_from_start; + int ret; + offset_in_erase_unit(area, offset, offset_from_start, dist); + uint32_t chunk = std::min(size, dist); + + if (!offset_from_start) { + // We're at the start of an erase unit. Here (and only here), check if it's erased. + bool erased; + ret = is_erase_unit_erased(area, offset, erased); + if (ret) { + return MBED_ERROR_WRITE_FAILED; + } + if (!erased) { + ret = erase_erase_unit(area, offset); + if (ret) { + return MBED_ERROR_WRITE_FAILED; + } + } + } + offset += chunk; + size -= chunk; + } + return MBED_SUCCESS; +} diff --git a/features/storage/kvstore/tdbstore/TDBStore.h b/features/storage/kvstore/tdbstore/TDBStore.h new file mode 100644 index 00000000000..1a5e3cf94ba --- /dev/null +++ b/features/storage/kvstore/tdbstore/TDBStore.h @@ -0,0 +1,461 @@ +/* + * Copyright (c) 2018 ARM Limited. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MBED_TDBSTORE_H +#define MBED_TDBSTORE_H + +#include +#include +#include "KVStore.h" +#include "BlockDevice.h" +#include "BufferedBlockDevice.h" +#include "PlatformMutex.h" + +namespace mbed { + +/** TDBStore class + * + * Lightweight Key Value storage over a block device + */ + +class TDBStore : public KVStore { +public: + + static const uint32_t RESERVED_AREA_SIZE = 64; + + /** + * @brief Class constructor + * + * @param[in] bd Underlying block device. + * @param[in] max_keys Maximum number of keys (0 if unlimited). + * + * @returns none + */ + TDBStore(BlockDevice *bd); + + /** + * @brief Class destructor + * + * @returns none + */ + virtual ~TDBStore(); + + /** + * @brief Initialize KVStore + * + * @returns 0 on success or a negative error code on failure + */ + virtual int init(); + + /** + * @brief Deinitialize KVStore + * + * @returns 0 on success or a negative error code on failure + */ + virtual int deinit(); + + + /** + * @brief Reset KVStore contents (clear all keys) + * Warning: This function is not thread safe. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int reset(); + + /** + * @brief Set one KVStore item, given key and value. + * + * @param[in] key Key. + * @param[in] buffer Value data buffer. + * @param[in] size Value data size. + * @param[in] create_flags Flag mask. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int set(const char *key, const void *buffer, size_t size, uint32_t create_flags); + + /** + * @brief Get one KVStore item, given key. + * + * @param[in] key Key. + * @param[in] buffer Value data buffer. + * @param[in] buffer_size Value data buffer size. + * @param[out] actual_size Actual read size. + * @param[in] offset Offset to read from in data. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int get(const char *key, void *buffer, size_t buffer_size, size_t *actual_size = NULL, + size_t offset = 0); + + /** + * @brief Get information of a given key. + * + * @param[in] key Key. + * @param[out] info Returned information structure. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int get_info(const char *key, info_t *info); + + /** + * @brief Remove a KVStore item, given key. + * + * @param[in] key Key. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int remove(const char *key); + + + /** + * @brief Start an incremental KVStore set sequence. + * + * @param[out] handle Returned incremental set handle. + * @param[in] key Key. + * @param[in] final_data_size Final value data size. + * @param[in] create_flags Flag mask. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int set_start(set_handle_t *handle, const char *key, size_t final_data_size, uint32_t create_flags); + + /** + * @brief Add data to incremental KVStore set sequence. + * + * @param[in] handle Incremental set handle. + * @param[in] value_data value data to add. + * @param[in] data_size value data size. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int set_add_data(set_handle_t handle, const void *value_data, size_t data_size); + + /** + * @brief Finalize an incremental KVStore set sequence. + * + * @param[in] handle Incremental set handle. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int set_finalize(set_handle_t handle); + + /** + * @brief Start an iteration over KVStore keys. + * + * @param[out] it Returned iterator handle. + * @param[in] prefix Key prefix (null for all keys). + * + * @returns 0 on success or a negative error code on failure + */ + virtual int iterator_open(iterator_t *it, const char *prefix = NULL); + + /** + * @brief Get next key in iteration. + * + * @param[in] it Iterator handle. + * @param[in] key Buffer for returned key. + * @param[in] key_size Key buffer size. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int iterator_next(iterator_t it, char *key, size_t key_size); + + /** + * @brief Close iteration. + * + * @param[in] it Iterator handle. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int iterator_close(iterator_t it); + + /** + * @brief Set data in reserved area. + * + * @param[in] reserved_data Reserved data buffer. + * @param[in] reserved_data_buf_size + * Reserved data buffer size. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int reserved_data_set(const void *reserved_data, size_t reserved_data_buf_size); + + /** + * @brief Get data from reserved area. + * + * @param[in] reserved_data Reserved data buffer. + * @param[in] reserved_data_buf_size + * Reserved data buffer size. + * + * @returns 0 on success or a negative error code on failure + */ + virtual int reserved_data_get(void *reserved_data, size_t reserved_data_buf_size); + +private: + + typedef struct { + uint32_t address; + size_t size; + } tdbstore_area_data_t; + + static const int _num_areas = 2; + + PlatformMutex _mutex; + PlatformMutex _inc_set_mutex; + void *_ram_table; + size_t _max_keys; + size_t _num_keys; + BlockDevice *_bd; + BufferedBlockDevice *_buff_bd; + uint32_t _free_space_offset; + uint32_t _master_record_offset; + bool _is_initialized; + int _active_area; + uint16_t _active_area_version; + size_t _size; + tdbstore_area_data_t _area_params[_num_areas]; + uint32_t _prog_size; + uint8_t *_work_buf; + char *_key_buf; + bool _variant_bd_erase_unit_size; + void *_inc_set_handle; + + /** + * @brief Read a block from an area. + * + * @param[in] area Area. + * @param[in] offset Offset in area. + * @param[in] size Number of bytes to read. + * @param[in] buf Output buffer. + * + * @returns 0 for success, nonzero for failure. + */ + int read_area(uint8_t area, uint32_t offset, uint32_t size, void *buf); + + /** + * @brief Write a block to an area. + * + * @param[in] area Area. + * @param[in] offset Offset in area. + * @param[in] size Number of bytes to write. + * @param[in] buf Input buffer. + * + * @returns 0 for success, non-zero for failure. + */ + int write_area(uint8_t area, uint32_t offset, uint32_t size, const void *buf); + + /** + * @brief Reset an area (erase its start). + * + * @param[in] area Area. + * + * @returns 0 for success, nonzero for failure. + */ + int reset_area(uint8_t area); + + /** + * @brief Erase an erase unit. + * + * @param[in] area Area. + * @param[in] offset Offset in area. + * + * @returns 0 for success, non-zero for failure. + */ + int erase_erase_unit(uint8_t area, uint32_t offset); + + /** + * @brief Calculate addresses and sizes of areas. + */ + void calc_area_params(); + + /** + * @brief Read a TDBStore record from a given location. + * + * @param[in] area Area. + * @param[in] offset Offset of record in area. + * @param[in] key Key. + * @param[in] data_buf Data buffer. + * @param[in] data_buf_size Data buffer size. + * @param[out] actual_data_size Actual data size. + * @param[in] data_offset Offset in data. + * @param[in] copy_key Copy key to user buffer. + * @param[in] copy_data Copy data to user buffer. + * @param[in] check_expected_key Check whether key belongs to this record. + * @param[in] calc_hash Calculate hash (on key). + * @param[out] hash Calculated hash. + * @param[out] flags Record flags. + * @param[out] next_offset Offset of next record. + * + * @returns 0 for success, nonzero for failure. + */ + int read_record(uint8_t area, uint32_t offset, char *key, + void *data_buf, uint32_t data_buf_size, + uint32_t& actual_data_size, size_t data_offset, bool copy_key, + bool copy_data, bool check_expected_key, bool calc_hash, + uint32_t& hash, uint32_t& flags, uint32_t& next_offset); + + /** + * @brief Write a master record of a given area. + * + * @param[in] area Area. + * @param[in] version Area version. + * @param[out] next_offset Offset of next record. + * + * @returns 0 for success, nonzero for failure. + */ + int write_master_record(uint8_t area, uint16_t version, uint32_t& next_offset); + + /** + * @brief Copy a record from one area to the opposite one. + * + * @param[in] from_area Area to copy record from. + * @param[in] from_offset Offset in source area. + * @param[in] to_offset Offset in destination area. + * @param[out] to_next_offset Offset of next record in destination area. + * + * @returns 0 for success, nonzero for failure. + */ + int copy_record(uint8_t from_area, uint32_t from_offset, uint32_t to_offset, + uint32_t& to_next_offset); + + /** + * @brief As part of GC process, copy all records in RAM table to the opposite area. + * + * @param[in] from_area Area to copy record from. + * @param[in] to_offset Offset in destination area. + * @param[out] to_next_offset Offset of next record in destination area. + * + * @returns 0 for success, nonzero for failure. + */ + int copy_all_records(uint8_t from_area, uint32_t to_offset, uint32_t& to_next_offset); + + /** + * @brief Garbage collection (compact all records from active area to the standby one). + * + * @returns 0 for success, nonzero for failure. + */ + int garbage_collection(); + + /** + * @brief Return record size given key and data size. + * + * @param[in] key Key. + * @param[in] data_size Data size. + * + * @returns record size. + */ + uint32_t record_size(const char *key, uint32_t data_size); + + /** + * @brief Find a record given key + * + * @param[in] area Area. + * @param[in] key Key. + * @param[out] offset Offset of record. + * @param[out] ram_table_ind Index in ram table (target one if not found). + * @param[out] hash Calculated key hash. + * + * @returns 0 for success, nonzero for failure. + */ + int find_record(uint8_t area, const char *key, uint32_t& offset, + uint32_t& ram_table_ind, uint32_t& hash); + /** + * @brief Actual logics of get API (covers also all other get APIs). + * + * @param[in] key Key. + * @param[in] copy_data Copy data to user buffer. + * @param[in] data_buf Buffer to store data on. + * @param[in] data_buf_size Data buffer size (bytes). + * @param[out] actual_data_size Actual data size (bytes). + * @param[out] flags Flags. + * + * @returns 0 for success, nonzero for failure. + */ + int do_get(const char *key, bool copy_data, + void *data_buf, uint32_t data_buf_size, uint32_t& actual_data_size, + uint32_t& flags); + + /** + * @brief Actual logics of set API (covers also the remove API). + * + * @param[in] key Key. + * @param[in] data_buf Data buffer. + * @param[in] data_buf_size Data buffer size (bytes). + * @param[in] flags Flags. + * + * @returns 0 for success, nonzero for failure. + */ + int do_set(const char *key, const void *data_buf, uint32_t data_buf_size, uint32_t flags); + + /** + * @brief Build RAM table and update _free_space_offset (scanning all the records in the area). + * + * @returns 0 for success, nonzero for failure. + */ + int build_ram_table(); + + /** + * @brief Increment max number of keys and reallocate ram table accordingly. + * + * @param[out] ram_table Updated ram table. + * + * @returns 0 for success, nonzero for failure. + */ + int increment_max_keys(void **ram_table = 0); + + /** + * @brief Calculate offset from start of erase unit. + * + * @param[in] area Area. + * @param[in] offset Offset in area. + * @param[out] offset_from_start Offset from start of erase unit. + * @param[out] dist_to_end Distance to end of erase unit. + * + * @returns offset in erase unit. + */ + void offset_in_erase_unit(uint8_t area, uint32_t offset, uint32_t& offset_from_start, + uint32_t& dist_to_end); + + /** + * @brief Check whether erase unit is erased (from offset until end of unit). + * + * @param[in] area Area. + * @param[in] offset Offset in area. + * @param[out] erased Unit is erased. + * + * @returns 0 for success, nonzero for failure. + */ + int is_erase_unit_erased(uint8_t area, uint32_t offset, bool& erased); + + /** + * @brief Before writing a record, check whether we are crossing an erase unit. + * If we do, check if it's erased, and erase it if not. + * + * @param[in] area Area. + * @param[in] offset Offset in area. + * @param[in] size Write size. + * + * @returns 0 for success, nonzero for failure. + */ + int check_erase_before_write(uint8_t area, uint32_t offset, uint32_t size); +}; +/** @}*/ + +} // namespace mbed + +#endif diff --git a/platform/mbed_error.h b/platform/mbed_error.h index c48d05c27c7..46cea162040 100644 --- a/platform/mbed_error.h +++ b/platform/mbed_error.h @@ -532,6 +532,9 @@ typedef enum _mbed_module_type { BLE_NO_FRAME_INITIALIZED, 321 BLE No frame initialized BLE_BACKEND_CREATION_FAILED 322 BLE Backend creation failed BLE_BACKEND_NOT_INITIALIZED 323 BLE Backend not initialized + ASSERTION_FAILED 324 Assertion Failed + AUTHENTICATION_FAILED 325 Authentication Failed + RBP_AUTHENTICATION_FAILED 326 Rollback Protect Authentication Failed \endverbatim * * @note @@ -783,6 +786,8 @@ typedef enum _mbed_error_code { MBED_DEFINE_SYSTEM_ERROR(BLE_BACKEND_CREATION_FAILED, 66), /* 322 BLE Backend creation failed */ MBED_DEFINE_SYSTEM_ERROR(BLE_BACKEND_NOT_INITIALIZED, 67), /* 323 BLE Backend not initialized */ MBED_DEFINE_SYSTEM_ERROR(ASSERTION_FAILED, 68), /* 324 Assertion Failed */ + MBED_DEFINE_SYSTEM_ERROR(AUTHENTICATION_FAILED, 69), /* 325 Authentication Failed */ + MBED_DEFINE_SYSTEM_ERROR(RBP_AUTHENTICATION_FAILED, 70), /* 326 Rollback Protection Authentication Failed */ //Everytime you add a new system error code, you must update //Error documentation under Handbook to capture the info on diff --git a/targets/targets.json b/targets/targets.json index 3e093e32fb5..f03f2efde54 100644 --- a/targets/targets.json +++ b/targets/targets.json @@ -637,7 +637,7 @@ }, "K64F": { "supported_form_factors": ["ARDUINO"], - "components": ["SD"], + "components": ["SD", "FLASHIAP"], "core": "Cortex-M4F", "supported_toolchains": ["ARM", "GCC_ARM", "IAR"], "extra_labels": ["Freescale", "MCUXpresso_MCUS", "KSDK2_MCUS", "FRDM", "KPSDK_MCUS", "KPSDK_CODE", "MCU_K64F", "Freescale_EMAC"], @@ -755,7 +755,7 @@ }, "K82F": { "supported_form_factors": ["ARDUINO"], - "components": ["SPIF"], + "components": ["SPIF", "FLASHIAP"], "core": "Cortex-M4F", "supported_toolchains": ["ARM", "GCC_ARM", "IAR"], "extra_labels": ["Freescale", "MCUXpresso_MCUS", "KSDK2_MCUS", "FRDM"], @@ -1236,7 +1236,7 @@ }, "USI_WM_BN_BM_22": { "inherits": ["FAMILY_STM32"], - "components": ["SPIF"], + "components": ["SPIF", "FLASHIAP"], "core": "Cortex-M4F", "extra_labels_add": ["STM32F4", "STM32F412xG", "STM32F412ZG", "WICED", "CYW4343X", "CORDIO"], "features": ["BLE", "STORAGE"], @@ -2084,7 +2084,7 @@ } }, "DISCO_L475VG_IOT01A": { - "components": ["QSPIF"], + "components": ["QSPIF", "FLASHIAP"], "inherits": ["FAMILY_STM32"], "core": "Cortex-M4F", "extra_labels_add": ["STM32L4", "STM32L475xG", "STM32L475VG"], @@ -2108,7 +2108,7 @@ "bootloader_supported": true }, "DISCO_L476VG": { - "components": ["QSPIF"], + "components": ["QSPIF", "FLASHIAP"], "inherits": ["FAMILY_STM32"], "core": "Cortex-M4F", "extra_labels_add": ["STM32L4", "STM32L476xG", "STM32L476VG"],