From 6e1e78d12685934d557d542eb7cc684b56b12f62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20H=C3=BC=C3=9Fler?= Date: Sat, 29 Apr 2023 14:41:44 +0200 Subject: [PATCH 1/6] pkg/flashdb: have FAL MTD and defaults for partition 0 defined --- pkg/flashdb/include/fal_cfg.h | 30 +++++++++++++++++++++++++++++- pkg/flashdb/mtd/fal_mtd_port.c | 4 +++- tests/pkg/flashdb_vfs/main.c | 5 +++-- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/pkg/flashdb/include/fal_cfg.h b/pkg/flashdb/include/fal_cfg.h index 557b28658eb0..e3e0250c02ab 100644 --- a/pkg/flashdb/include/fal_cfg.h +++ b/pkg/flashdb/include/fal_cfg.h @@ -49,6 +49,34 @@ extern struct fal_flash_dev mtd_flash0; &mtd_flash0, \ } +#if !defined(FAL_MTD) +/** + * @brief Default MTD to use for flashdb + */ +#define FAL_MTD MTD_0 +#endif + +#if !defined(FAL_PART_LABEL) +/** + * @brief Default FAL partition to use for flashdb + */ +#define FAL_PART_LABEL FAL_PART0_LABEL +#endif + +#if !defined(FAL_PART0_LABEL) +/** + * @brief Have at least the label of partition 0 defined + */ +#define FAL_PART0_LABEL "part0" +#endif + +#if !defined(FAL_PART0_LENGTH) +/** + * @brief Have at least the length of partition 0 defined + */ +#define FAL_PART0_LENGTH (2 * 4096u) +#endif + /** * @brief Partition 0 */ @@ -83,7 +111,7 @@ extern struct fal_flash_dev mtd_flash0; * @brief Partition 3 */ #ifdef FAL_PART3_LABEL -#define FAL_ROW_PART3 { FAL_PART_MAGIC_WORD, FAL_PART2_LABEL, "fal_mtd", +#define FAL_ROW_PART3 { FAL_PART_MAGIC_WORD, FAL_PART3_LABEL, "fal_mtd", FAL_PART2_LENGTH, FAL_PART3_LENGTH, 0 }, #else #define FAL_ROW_PART3 diff --git a/pkg/flashdb/mtd/fal_mtd_port.c b/pkg/flashdb/mtd/fal_mtd_port.c index 0d50448fca77..14e23ae82599 100644 --- a/pkg/flashdb/mtd/fal_mtd_port.c +++ b/pkg/flashdb/mtd/fal_mtd_port.c @@ -75,7 +75,9 @@ struct fal_flash_dev mtd_flash0 = { void fdb_mtd_init(mtd_dev_t *mtd) { unsigned sector_size; - + if (_mtd) { + return; + } mtd_init(mtd); _mtd = mtd; diff --git a/tests/pkg/flashdb_vfs/main.c b/tests/pkg/flashdb_vfs/main.c index 8d2dbd223b18..e8e8c1a92880 100644 --- a/tests/pkg/flashdb_vfs/main.c +++ b/tests/pkg/flashdb_vfs/main.c @@ -9,6 +9,7 @@ #include #include +#include "fal_cfg.h" #include "board.h" #include "mutex.h" #ifdef MODULE_VFS @@ -20,8 +21,8 @@ /** * @brief Select MTD device to use for FlashDB */ -#if !defined(FDB_MTD) && defined(MTD_0) -#define FDB_MTD MTD_0 +#if !defined(FDB_MTD) +#define FDB_MTD FAL_MTD #endif #define FDB_LOG_TAG "[main]" From bd75060bec6ba0538642a19f7304b85c3e177a4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20H=C3=BC=C3=9Fler?= Date: Fri, 12 May 2023 15:38:28 +0200 Subject: [PATCH 2/6] pkg/flashdb: fix FAL partition offsets --- pkg/flashdb/include/fal_cfg.h | 72 +++++++++++++++++++++++++---------- 1 file changed, 51 insertions(+), 21 deletions(-) diff --git a/pkg/flashdb/include/fal_cfg.h b/pkg/flashdb/include/fal_cfg.h index e3e0250c02ab..3ea4930978e2 100644 --- a/pkg/flashdb/include/fal_cfg.h +++ b/pkg/flashdb/include/fal_cfg.h @@ -26,6 +26,7 @@ #define FAL_CFG_H #include "board.h" +#include "mtd_default.h" #ifdef __cplusplus extern "C" { @@ -49,28 +50,21 @@ extern struct fal_flash_dev mtd_flash0; &mtd_flash0, \ } -#if !defined(FAL_MTD) +#if !defined(FAL_MTD) || defined(DOXYGEN) /** * @brief Default MTD to use for flashdb */ -#define FAL_MTD MTD_0 +#define FAL_MTD mtd_default_get_dev(0) #endif -#if !defined(FAL_PART_LABEL) -/** - * @brief Default FAL partition to use for flashdb - */ -#define FAL_PART_LABEL FAL_PART0_LABEL -#endif - -#if !defined(FAL_PART0_LABEL) +#if !defined(FAL_PART0_LABEL) || defined(DOXYGEN) /** * @brief Have at least the label of partition 0 defined */ #define FAL_PART0_LABEL "part0" #endif -#if !defined(FAL_PART0_LENGTH) +#if !defined(FAL_PART0_LENGTH) || defined(DOXYGEN) /** * @brief Have at least the length of partition 0 defined */ @@ -80,9 +74,18 @@ extern struct fal_flash_dev mtd_flash0; /** * @brief Partition 0 */ -#ifdef FAL_PART0_LABEL +#if defined(FAL_PART0_LABEL) || defined(DOXYGEN) +#if !defined(FAL_PART0_OFFSET) || defined(DOXYGEN) +/** + * @brief Offset of partition 0 + */ +#define FAL_PART0_OFFSET 0 +#endif +/** + * @brief Partition 0 compound definition + */ #define FAL_ROW_PART0 { FAL_PART_MAGIC_WORD, FAL_PART0_LABEL, "fal_mtd", \ - 0, FAL_PART0_LENGTH, 0 }, + FAL_PART0_OFFSET, FAL_PART0_LENGTH, 0 }, #else #define FAL_ROW_PART0 #endif @@ -90,9 +93,18 @@ extern struct fal_flash_dev mtd_flash0; /** * @brief Partition 1 */ -#ifdef FAL_PART1_LABEL +#if defined(FAL_PART1_LABEL) || defined(DOXYGEN) +#if !defined(FAL_PART1_OFFSET) || defined(DOXYGEN) +/** + * @brief Offset of partition 1 + */ +#define FAL_PART1_OFFSET (FAL_PART0_OFFSET + FAL_PART0_LENGTH) +#endif +/** + * @brief Partition 1 compound definition + */ #define FAL_ROW_PART1 { FAL_PART_MAGIC_WORD, FAL_PART1_LABEL, "fal_mtd", \ - FAL_PART0_LENGTH, FAL_PART1_LENGTH, 0 }, + FAL_PART1_OFFSET, FAL_PART1_LENGTH, 0 }, #else #define FAL_ROW_PART1 #endif @@ -100,9 +112,18 @@ extern struct fal_flash_dev mtd_flash0; /** * @brief Partition 2 */ -#ifdef FAL_PART2_LABEL -#define FAL_ROW_PART2 { FAL_PART_MAGIC_WORD, FAL_PART2_LABEL, "fal_mtd", - FAL_PART1_LENGTH, FAL_PART2_LENGTH, 0 }, +#if defined(FAL_PART2_LABEL) || defined(DOXYGEN) +#if !defined(FAL_PART2_OFFSET) || defined(DOXYGEN) +/** + * @brief Offset of partition 2 + */ +#define FAL_PART2_OFFSET (FAL_PART1_OFFSET + FAL_PART1_LENGTH) +#endif +/** + * @brief Partition 2 compound definition + */ +#define FAL_ROW_PART2 { FAL_PART_MAGIC_WORD, FAL_PART2_LABEL, "fal_mtd", \ + FAL_PART2_OFFSET, FAL_PART2_LENGTH, 0 }, #else #define FAL_ROW_PART2 #endif @@ -110,9 +131,18 @@ extern struct fal_flash_dev mtd_flash0; /** * @brief Partition 3 */ -#ifdef FAL_PART3_LABEL -#define FAL_ROW_PART3 { FAL_PART_MAGIC_WORD, FAL_PART3_LABEL, "fal_mtd", - FAL_PART2_LENGTH, FAL_PART3_LENGTH, 0 }, +#if defined(FAL_PART3_LABEL) || defined(DOXYGEN) +#if !defined(FAL_PART3_OFFSET) || defined(DOXYGEN) +/** + * @brief Offset of partition 3 + */ +#define FAL_PART3_OFFSET (FAL_PART2_OFFSET + FAL_PART2_LENGTH) +#endif +/** + * @brief Partition 3 compound definition + */ +#define FAL_ROW_PART3 { FAL_PART_MAGIC_WORD, FAL_PART3_LABEL, "fal_mtd", \ + FAL_PART3_OFFSET, FAL_PART3_LENGTH, 0 }, #else #define FAL_ROW_PART3 #endif From 3167d58e0da75ae09b2eaa1ac28e7d90950dcaea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20H=C3=BC=C3=9Fler?= Date: Tue, 2 Jan 2024 23:51:21 +0100 Subject: [PATCH 3/6] pkg/flashdb: add configurable sector size --- pkg/flashdb/Kconfig | 9 +++++++++ pkg/flashdb/include/fal_cfg.h | 38 +++++++++++++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/pkg/flashdb/Kconfig b/pkg/flashdb/Kconfig index 8d66a50be0a5..0d9fdef41cf3 100644 --- a/pkg/flashdb/Kconfig +++ b/pkg/flashdb/Kconfig @@ -41,6 +41,15 @@ config MODULE_FLASHDB_KVDB_AUTO_UPDATE database. If the version changes, it will automatically trigger an upgrade action and update the new default KV collection to the current database. +config FLASHDB_MIN_SECTOR_SIZE_DEFAULT_KiB + int "Minimal virtual sector size in KiB for FlashDB" + default 4 + help + By default, KVDB will use 1 times the block size as the sector size, that is, 4096. + At this time, the KVDB cannot store a KV longer than 4096. If you want to save, for example, + a KV with a length of 10K, you can use the control function to set the sector size to 12K or + larger. + config MODULE_FLASHDB_MTD endif # PACKAGE_FLASHDB diff --git a/pkg/flashdb/include/fal_cfg.h b/pkg/flashdb/include/fal_cfg.h index 3ea4930978e2..8f960e95880f 100644 --- a/pkg/flashdb/include/fal_cfg.h +++ b/pkg/flashdb/include/fal_cfg.h @@ -26,12 +26,40 @@ #define FAL_CFG_H #include "board.h" +#include "macros/units.h" #include "mtd_default.h" #ifdef __cplusplus extern "C" { #endif +#if !defined(CONFIG_FLASHDB_MIN_SECTOR_SIZE_DEFAULT_KiB) || defined(DOXYGEN) +/** + * @brief Minimal virtual sector size in KiB for FlashDB + * + * This is just a reasonable default for an automatic partition configuration, hence "DEFAULT". + * The "MIN" stands for a required minimum to guarantee an expected size of key value pairs. + * The actually acceptable sector size must be a multiple of the physical sector size though. + * Thus, it can be larger than the minimum required size. + * + * The default is 4 (4096). + * FlashDB sector size is different from MTD sector size as it is a + * virtual measure of granularity and not a device property. + * The virtual sector size must be a multiple of the physical sector size. + * + * From the documentation of FLashDB: + * The internal storage structure of FlashDB is composed of N sectors, and each formatting takes + * sector as the smallest unit. A sector is usually N times the size of the Flash block. + * For example, the block size of Nor Flash is generally 4096. + * + * By default, KVDB will use 1 times the block size as the sector size, that is, 4096. + * At this time, the KVDB cannot store a KV longer than 4096. If you want to save, for example, + * a KV with a length of 10K, you can use the control function to set the sector size to 12K or larger. + * + */ +#define CONFIG_FLASHDB_MIN_SECTOR_SIZE_DEFAULT_KiB 4 +#endif + /** * @brief Partition table is defined at compile time (not read from flash) */ @@ -66,9 +94,15 @@ extern struct fal_flash_dev mtd_flash0; #if !defined(FAL_PART0_LENGTH) || defined(DOXYGEN) /** - * @brief Have at least the length of partition 0 defined + * @brief Have at least the length of partition 0 defined, which must be at least two sectors + * and a multiple of the virtual sector size. + * + * The virtual sector size is however bound to the physical sector size of @ref FAL_MTD. + * So make sure that @ref CONFIG_FLASHDB_MIN_SECTOR_SIZE_DEFAULT_KiB times 1024 is a multiple of the MTD sector size. + * For example if the MTD sector size is 4KiB, then @ref CONFIG_FLASHDB_MIN_SECTOR_SIZE_DEFAULT_KiB must be a multiple of 4. + * If the MTD sector size is 1KiB, then you have all options for @ref CONFIG_FLASHDB_MIN_SECTOR_SIZE_DEFAULT_KiB. */ -#define FAL_PART0_LENGTH (2 * 4096u) +#define FAL_PART0_LENGTH (2 * (CONFIG_FLASHDB_MIN_SECTOR_SIZE_DEFAULT_KiB * KiB(1))) #endif /** From 057695bddec3423445539db9ca9736f31c3db0f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20H=C3=BC=C3=9Fler?= Date: Tue, 2 Jan 2024 23:53:26 +0100 Subject: [PATCH 4/6] tests/pkg/flashdb: add test for FAL config initialization --- tests/pkg/flashdb_fal_cfg/Makefile | 29 +++++ tests/pkg/flashdb_fal_cfg/Makefile.ci | 3 + tests/pkg/flashdb_fal_cfg/main.c | 149 ++++++++++++++++++++++++++ 3 files changed, 181 insertions(+) create mode 100644 tests/pkg/flashdb_fal_cfg/Makefile create mode 100644 tests/pkg/flashdb_fal_cfg/Makefile.ci create mode 100644 tests/pkg/flashdb_fal_cfg/main.c diff --git a/tests/pkg/flashdb_fal_cfg/Makefile b/tests/pkg/flashdb_fal_cfg/Makefile new file mode 100644 index 000000000000..b9730327a015 --- /dev/null +++ b/tests/pkg/flashdb_fal_cfg/Makefile @@ -0,0 +1,29 @@ +include ../Makefile.pkg_common + +# select the MTD backend +USEMODULE += flashdb_mtd +# enable key-value database +USEMODULE += flashdb_kvdb +# enable time series database +USEMODULE += flashdb_tsdb +# rtc_localtime() +USEMODULE += rtc_utils + +# prefer periph_rtc over periph_rtt +FEATURES_OPTIONAL += periph_rtc +FEATURES_REQUIRED_ANY += periph_rtc|periph_rtt + +CFLAGS += -DFAL_PART1_LABEL=\"part1\" +CFLAGS += -DFAL_PART1_LENGTH=FAL_PART0_LENGTH +CFLAGS += -DFAL_PART2_LABEL=\"part2\" +CFLAGS += -DFAL_PART2_LENGTH=FAL_PART0_LENGTH +CFLAGS += -DFAL_PART3_LABEL=\"part3\" +CFLAGS += -DFAL_PART3_LENGTH=FAL_PART0_LENGTH + +include $(RIOTBASE)/Makefile.include + +# handle RTC backend after inclusion of $(RIOTBASE)/Makefile.include +ifeq (,$(filter periph_rtc,$(FEATURES_USED))) + USEMODULE += rtt_rtc + USEMODULE += ztimer_no_periph_rtt +endif diff --git a/tests/pkg/flashdb_fal_cfg/Makefile.ci b/tests/pkg/flashdb_fal_cfg/Makefile.ci new file mode 100644 index 000000000000..c615a1426a38 --- /dev/null +++ b/tests/pkg/flashdb_fal_cfg/Makefile.ci @@ -0,0 +1,3 @@ +BOARD_INSUFFICIENT_MEMORY := \ + stk3200 \ + # diff --git a/tests/pkg/flashdb_fal_cfg/main.c b/tests/pkg/flashdb_fal_cfg/main.c new file mode 100644 index 000000000000..1d7e2b333625 --- /dev/null +++ b/tests/pkg/flashdb_fal_cfg/main.c @@ -0,0 +1,149 @@ +/* + * Copyright (C) 2023 ML!PA Consulting Gmbh + * + * This file is subject to the terms and conditions of the GNU Lesser General + * Public License v2.1. See the file LICENSE in the top level directory for more + * details. + */ + +/** + * @ingroup tests + * @{ + * + * @file + * @brief Short test for the FlashDB FAL configuration initialization + * + * @author Fabian Hüßler + * + * @} + */ + +#include + +#include "container.h" +#include "fal.h" +#include "macros/math.h" +#include "mutex.h" +#include "periph/rtc.h" + +#include + +/** + * @brief FlashDB Magic Word + * + * The value here is not relevant but it must be defined. + */ +#define FAL_PART_MAGIC_WORD 0x45503130 + +/** + * @brief Number of FAL partitions + */ +#define FAL_PART_TABLE_NUMOF ARRAY_SIZE(((const struct fal_partition[])FAL_PART_TABLE)) + +#if !defined(FAL_TSDB_MAX) || defined(DOXYGEN) +/** + * @brief Maximum length of a TSDB entry + */ +#define FAL_TSDB_MAX 128 +#endif + +#if !defined(FAL_PART_TABLE_KVDB) || defined(DOXYGEN) +/** + * @brief Indices of partitions in @ref FAL_PART_TABLE that can be initialized as a Key-Value-DB + */ +#define FAL_PART_TABLE_KVDB \ +{ \ + 0, \ + 1, \ +} +#endif + +#if !defined(FAL_PART_TABLE_TSDB) || defined(DOXYGEN) +/** + * @brief Indices of partitions in @ref FAL_PART_TABLE that can be initialized as a Time-Series-DB + */ +#define FAL_PART_TABLE_TSDB \ +{ \ + 2, \ + 3, \ +} +#endif + +extern void fdb_mtd_init(mtd_dev_t *mtd); + +static mutex_t _locker = MUTEX_INIT; +static const unsigned _fdb_part_kvdb[] = FAL_PART_TABLE_KVDB; +static const unsigned _fdb_part_tsdb[] = FAL_PART_TABLE_TSDB; +static struct fdb_kvdb _kvdb[ARRAY_SIZE(_fdb_part_kvdb)]; +static struct fdb_tsdb _tsdb[ARRAY_SIZE(_fdb_part_tsdb)]; + +/** + * @brief Select MTD device to use for FlashDB + */ +#if !defined(FDB_MTD) +#define FDB_MTD FAL_MTD +#endif + +static void _lock(fdb_db_t db) +{ + mutex_lock(db->user_data); +} + +static void _unlock(fdb_db_t db) +{ + mutex_unlock(db->user_data); +} + +static fdb_time_t _get_time(void) +{ + struct tm now; + rtc_get_time(&now); + return mktime(&now); +} + +int main(void) +{ + int init_failed; + fdb_mtd_init(FDB_MTD); + size_t size = FDB_MTD->pages_per_sector * FDB_MTD->page_size; + /* scale hardware sector size to minimum required virtual sector size */ + size = DIV_ROUND_UP((CONFIG_FLASHDB_MIN_SECTOR_SIZE_DEFAULT_KiB * KiB(1)), size) * size; + printf("Informational: Make sure the following requirements are fulfilled for a successful initialization:\n"); + printf("The virtual sector size is a multiple of the physical sector size: %lu %% %lu == 0\n", + (unsigned long)size, (unsigned long)FDB_MTD->pages_per_sector * FDB_MTD->page_size); + printf("The maximum partition size is a multiple of the virtual sector size: %lu %% %lu == 0\n", + (unsigned long)FAL_PART0_LENGTH, (unsigned long)size); + + for (unsigned i = 0; i < ARRAY_SIZE(_fdb_part_kvdb); i++) { + unsigned part = _fdb_part_kvdb[i]; + if (part >= FAL_PART_TABLE_NUMOF) { + continue; + } + fdb_kvdb_control(&_kvdb[i], FDB_KVDB_CTRL_SET_SEC_SIZE, &size); + fdb_kvdb_control(&_kvdb[i], FDB_KVDB_CTRL_SET_LOCK, (void *)(uintptr_t)_lock); + fdb_kvdb_control(&_kvdb[i], FDB_KVDB_CTRL_SET_UNLOCK, (void *)(uintptr_t)_unlock); + const char *spart = ((const struct fal_partition[])FAL_PART_TABLE)[part].name; + printf("Initializing FlashDB KVDB partition %s\n", spart); + if ((init_failed = fdb_kvdb_init(&_kvdb[i], "kvdb", spart, NULL, &_locker)) != FDB_NO_ERR) { + printf("Failed to initialize FlashDB KVDB partition %s (%d)\n", spart, init_failed); + return 1; + } + } + for (unsigned i = 0; i < ARRAY_SIZE(_fdb_part_tsdb); i++) { + unsigned part = _fdb_part_tsdb[i]; + if (part >= FAL_PART_TABLE_NUMOF) { + continue; + } + fdb_tsdb_control(&_tsdb[i], FDB_TSDB_CTRL_SET_LOCK, (void *)(uintptr_t)_lock); + fdb_tsdb_control(&_tsdb[i], FDB_TSDB_CTRL_SET_UNLOCK, (void *)(uintptr_t)_unlock); + fdb_tsdb_control(&_tsdb[i], FDB_TSDB_CTRL_SET_SEC_SIZE, &size); + const char *spart = ((const struct fal_partition[])FAL_PART_TABLE)[part].name; + printf("Initializing FlashDB TSDB partition %s\n", spart); + if ((init_failed = fdb_tsdb_init(&_tsdb[i], "tsdb", spart, _get_time, FAL_TSDB_MAX, &_locker)) != FDB_NO_ERR) { + printf("Failed to initialize FlashDB TSDB partition %s (%d)\n", spart, init_failed); + return 1; + } + } + puts("SUCCESS"); + return 0; +} From b671dd4d989014e9868dcb76ae8dd2e206850e9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20H=C3=BC=C3=9Fler?= Date: Fri, 9 Feb 2024 13:14:57 +0100 Subject: [PATCH 5/6] tests/pkg/flashdb_mtd: add Makefile.ci --- tests/pkg/flashdb_mtd/Makefile.ci | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 tests/pkg/flashdb_mtd/Makefile.ci diff --git a/tests/pkg/flashdb_mtd/Makefile.ci b/tests/pkg/flashdb_mtd/Makefile.ci new file mode 100644 index 000000000000..d55826c7bf3e --- /dev/null +++ b/tests/pkg/flashdb_mtd/Makefile.ci @@ -0,0 +1,7 @@ +BOARD_INSUFFICIENT_MEMORY := \ + nucleo-f031k6 \ + nucleo-l011k4 \ + samd10-xmini \ + stk3200 \ + stm32f030f4-demo \ + # From bf4bfdfbe96c14104304bf9d497ded0ff1caf369 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20H=C3=BC=C3=9Fler?= Date: Fri, 9 Feb 2024 18:49:01 +0100 Subject: [PATCH 6/6] tests/pkg/flashdb_fal_cfg: add Makefile.ci --- tests/pkg/flashdb_fal_cfg/Makefile.ci | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/pkg/flashdb_fal_cfg/Makefile.ci b/tests/pkg/flashdb_fal_cfg/Makefile.ci index c615a1426a38..d55826c7bf3e 100644 --- a/tests/pkg/flashdb_fal_cfg/Makefile.ci +++ b/tests/pkg/flashdb_fal_cfg/Makefile.ci @@ -1,3 +1,7 @@ BOARD_INSUFFICIENT_MEMORY := \ - stk3200 \ - # + nucleo-f031k6 \ + nucleo-l011k4 \ + samd10-xmini \ + stk3200 \ + stm32f030f4-demo \ + #