diff --git a/components/nvs_flash/host_test/nvs_host_test/main/CMakeLists.txt b/components/nvs_flash/host_test/nvs_host_test/main/CMakeLists.txt index 05663cb008d9..508bfa0e17db 100644 --- a/components/nvs_flash/host_test/nvs_host_test/main/CMakeLists.txt +++ b/components/nvs_flash/host_test/nvs_host_test/main/CMakeLists.txt @@ -12,3 +12,7 @@ idf_component_register(SRCS "main.cpp" "../../../../../tools/catch" WHOLE_ARCHIVE REQUIRES nvs_flash) + +if(CMAKE_C_COMPILER_ID MATCHES "Clang") + target_compile_options(${COMPONENT_LIB} PRIVATE -std=gnu++20) +endif() diff --git a/components/nvs_flash/host_test/nvs_host_test/main/test_nvs.cpp b/components/nvs_flash/host_test/nvs_host_test/main/test_nvs.cpp index 7dd85de5b5a6..40d9d655922e 100644 --- a/components/nvs_flash/host_test/nvs_host_test/main/test_nvs.cpp +++ b/components/nvs_flash/host_test/nvs_host_test/main/test_nvs.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD + * SPDX-FileCopyrightText: 2015-2023 Espressif Systems (Shanghai) CO LTD * * SPDX-License-Identifier: Apache-2.0 */ @@ -1475,6 +1475,8 @@ TEST_CASE("nvs page selection takes into account free entries also not just eras TEST_CASE("calculate used and free space", "[nvs]") { + size_t consumed_entries = 0; + PartitionEmulationFixture f(0, 6); nvs_flash_deinit(); TEST_ESP_ERR(nvs_get_stats(NULL, NULL), ESP_ERR_INVALID_ARG); @@ -1497,113 +1499,154 @@ TEST_CASE("calculate used and free space", "[nvs]") TEST_ESP_ERR(nvs_get_used_entry_count(handle, &h_count_entries), ESP_ERR_NVS_INVALID_HANDLE); CHECK(h_count_entries == 0); - nvs::Page p; // after erase. empty partition TEST_ESP_OK(nvs_get_stats(NULL, &stat1)); CHECK(stat1.free_entries != 0); CHECK(stat1.namespace_count == 0); - CHECK(stat1.total_entries == 6 * p.ENTRY_COUNT); + CHECK(stat1.total_entries == 6 * nvs::Page::ENTRY_COUNT); CHECK(stat1.used_entries == 0); - // create namespace test_k1 + // namespace test_k1 nvs_handle_t handle_1; + size_t ns1_expected_entries = 0; + + // create namepace + consumed_entries = 1; // should consume one entry TEST_ESP_OK(nvs_open("test_k1", NVS_READWRITE, &handle_1)); TEST_ESP_OK(nvs_get_stats(NULL, &stat2)); - CHECK(stat2.free_entries + 1 == stat1.free_entries); + CHECK(stat2.free_entries + consumed_entries == stat1.free_entries); CHECK(stat2.namespace_count == 1); CHECK(stat2.total_entries == stat1.total_entries); - CHECK(stat2.used_entries == 1); + CHECK(stat2.used_entries == stat1.used_entries + consumed_entries); + CHECK(stat2.available_entries + consumed_entries == stat1.available_entries); // create pair key-value com + consumed_entries = 1; // should consume one entry TEST_ESP_OK(nvs_set_i32(handle_1, "com", 0x12345678)); TEST_ESP_OK(nvs_get_stats(NULL, &stat1)); - CHECK(stat1.free_entries + 1 == stat2.free_entries); + CHECK(stat1.free_entries + consumed_entries == stat2.free_entries); CHECK(stat1.namespace_count == 1); CHECK(stat1.total_entries == stat2.total_entries); - CHECK(stat1.used_entries == 2); + CHECK(stat1.used_entries == stat2.used_entries + consumed_entries); + CHECK(stat1.available_entries + consumed_entries == stat2.available_entries); + ns1_expected_entries += consumed_entries; // change value in com + consumed_entries = 0; // should not consume any entry TEST_ESP_OK(nvs_set_i32(handle_1, "com", 0x01234567)); TEST_ESP_OK(nvs_get_stats(NULL, &stat2)); CHECK(stat2.free_entries == stat1.free_entries); CHECK(stat2.namespace_count == 1); CHECK(stat2.total_entries != 0); - CHECK(stat2.used_entries == 2); + CHECK(stat2.used_entries == stat1.used_entries + consumed_entries); + CHECK(stat2.available_entries + consumed_entries == stat1.available_entries); + ns1_expected_entries += consumed_entries; // create pair key-value ru + consumed_entries = 1; // should consume one entry TEST_ESP_OK(nvs_set_i32(handle_1, "ru", 0x00FF00FF)); TEST_ESP_OK(nvs_get_stats(NULL, &stat1)); - CHECK(stat1.free_entries + 1 == stat2.free_entries); + CHECK(stat1.free_entries + consumed_entries == stat2.free_entries); CHECK(stat1.namespace_count == 1); CHECK(stat1.total_entries != 0); - CHECK(stat1.used_entries == 3); + CHECK(stat1.used_entries == stat2.used_entries + consumed_entries); + CHECK(stat1.available_entries + consumed_entries == stat2.available_entries); + ns1_expected_entries += consumed_entries; - // amount valid pair in namespace 1 - size_t h1_count_entries; - TEST_ESP_OK(nvs_get_used_entry_count(handle_1, &h1_count_entries)); - CHECK(h1_count_entries == 2); + // amount of valid pairs in namespace 1 + size_t ns1_reported_entries; + TEST_ESP_OK(nvs_get_used_entry_count(handle_1, &ns1_reported_entries)); + CHECK(ns1_reported_entries == ns1_expected_entries); + // namespace test_k2 nvs_handle_t handle_2; - // create namespace test_k2 + size_t ns2_expected_entries = 0; + + // create namespace + consumed_entries = 1; // should consume one entry TEST_ESP_OK(nvs_open("test_k2", NVS_READWRITE, &handle_2)); TEST_ESP_OK(nvs_get_stats(NULL, &stat2)); - CHECK(stat2.free_entries + 1 == stat1.free_entries); + CHECK(stat2.free_entries + consumed_entries == stat1.free_entries); CHECK(stat2.namespace_count == 2); CHECK(stat2.total_entries == stat1.total_entries); - CHECK(stat2.used_entries == 4); + CHECK(stat2.used_entries == stat1.used_entries + consumed_entries); + CHECK(stat2.available_entries + consumed_entries == stat1.available_entries); - // create pair key-value + // create 3 pairs key-value + consumed_entries = 3; // should consume three entries TEST_ESP_OK(nvs_set_i32(handle_2, "su1", 0x00000001)); TEST_ESP_OK(nvs_set_i32(handle_2, "su2", 0x00000002)); TEST_ESP_OK(nvs_set_i32(handle_2, "sus", 0x00000003)); TEST_ESP_OK(nvs_get_stats(NULL, &stat1)); - CHECK(stat1.free_entries + 3 == stat2.free_entries); + CHECK(stat1.free_entries + consumed_entries == stat2.free_entries); CHECK(stat1.namespace_count == 2); CHECK(stat1.total_entries == stat2.total_entries); - CHECK(stat1.used_entries == 7); - - CHECK(stat1.total_entries == (stat1.used_entries + stat1.free_entries)); + CHECK(stat1.used_entries == stat2.used_entries + consumed_entries); + CHECK(stat1.available_entries + consumed_entries == stat2.available_entries); + ns2_expected_entries += consumed_entries; - // amount valid pair in namespace 2 - size_t h2_count_entries; - TEST_ESP_OK(nvs_get_used_entry_count(handle_2, &h2_count_entries)); - CHECK(h2_count_entries == 3); + // amount of valid pairs in namespace 2 + size_t ns2_reported_entries; + TEST_ESP_OK(nvs_get_used_entry_count(handle_2, &ns2_reported_entries)); + CHECK(ns2_reported_entries == ns2_expected_entries); - CHECK(stat1.used_entries == (h1_count_entries + h2_count_entries + stat1.namespace_count)); + CHECK(stat1.used_entries == (ns1_reported_entries + ns2_reported_entries + stat1.namespace_count)); nvs_close(handle_1); nvs_close(handle_2); - size_t temp = h2_count_entries; - TEST_ESP_ERR(nvs_get_used_entry_count(handle_1, &h2_count_entries), ESP_ERR_NVS_INVALID_HANDLE); - CHECK(h2_count_entries == 0); - h2_count_entries = temp; + size_t temp = ns2_reported_entries; + TEST_ESP_ERR(nvs_get_used_entry_count(handle_1, &ns2_reported_entries), ESP_ERR_NVS_INVALID_HANDLE); + CHECK(ns2_reported_entries == 0); + ns2_reported_entries = temp; TEST_ESP_ERR(nvs_get_used_entry_count(handle_1, NULL), ESP_ERR_INVALID_ARG); + // namespace test_k3 nvs_handle_t handle_3; - // create namespace test_k3 + size_t ns3_expected_entries = 0; + + // create namespace + consumed_entries = 1; // should consume one entry TEST_ESP_OK(nvs_open("test_k3", NVS_READWRITE, &handle_3)); TEST_ESP_OK(nvs_get_stats(NULL, &stat2)); - CHECK(stat2.free_entries + 1 == stat1.free_entries); + CHECK(stat2.free_entries + consumed_entries == stat1.free_entries); CHECK(stat2.namespace_count == 3); CHECK(stat2.total_entries == stat1.total_entries); - CHECK(stat2.used_entries == 8); + CHECK(stat2.used_entries == stat1.used_entries + consumed_entries); + CHECK(stat2.available_entries + consumed_entries == stat1.available_entries); - // create pair blobs + // create pair key - blob uint32_t blob[12]; + consumed_entries = 2 + (sizeof(blob) + 31) / 32; // should consume 2 + entry for each started block of 32 bytes TEST_ESP_OK(nvs_set_blob(handle_3, "bl1", &blob, sizeof(blob))); TEST_ESP_OK(nvs_get_stats(NULL, &stat1)); - CHECK(stat1.free_entries + 4 == stat2.free_entries); + CHECK(stat1.free_entries + consumed_entries == stat2.free_entries); CHECK(stat1.namespace_count == 3); CHECK(stat1.total_entries == stat2.total_entries); - CHECK(stat1.used_entries == 12); + CHECK(stat1.used_entries == stat2.used_entries + consumed_entries); + CHECK(stat1.available_entries + consumed_entries == stat2.available_entries); + CHECK(stat1.total_entries == (stat1.used_entries + stat1.available_entries + nvs::Page::ENTRY_COUNT)); + ns3_expected_entries += consumed_entries; + + // create pair key - string + char input_string[] = "abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789"; + consumed_entries = 1 + (strlen(input_string) + 1 + 31) / 32; // should consume 1 + entry for each started block of 32 bytes + TEST_ESP_OK(nvs_set_str(handle_3, "str1", input_string)); + TEST_ESP_OK(nvs_get_stats(NULL, &stat2)); + CHECK(stat2.free_entries + consumed_entries == stat1.free_entries); + CHECK(stat2.namespace_count == 3); + CHECK(stat2.total_entries == stat1.total_entries); + CHECK(stat2.used_entries == stat1.used_entries + consumed_entries); + CHECK(stat2.available_entries + consumed_entries == stat1.available_entries); + ns3_expected_entries += consumed_entries; - // amount valid pair in namespace 2 - size_t h3_count_entries; - TEST_ESP_OK(nvs_get_used_entry_count(handle_3, &h3_count_entries)); - CHECK(h3_count_entries == 4); + // amount of valid pairs in namespace 3 + size_t ns3_reported_entries; + TEST_ESP_OK(nvs_get_used_entry_count(handle_3, &ns3_reported_entries)); + CHECK(ns3_reported_entries == ns3_expected_entries); - CHECK(stat1.used_entries == (h1_count_entries + h2_count_entries + h3_count_entries + stat1.namespace_count)); + // overall check of used entries across all namespaces + CHECK(stat2.used_entries == (ns1_reported_entries + ns2_reported_entries + ns3_reported_entries + stat2.namespace_count)); nvs_close(handle_3); diff --git a/components/nvs_flash/include/nvs.h b/components/nvs_flash/include/nvs.h index b57d98277607..e45a64134355 100644 --- a/components/nvs_flash/include/nvs.h +++ b/components/nvs_flash/include/nvs.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD + * SPDX-FileCopyrightText: 2015-2023 Espressif Systems (Shanghai) CO LTD * * SPDX-License-Identifier: Apache-2.0 */ @@ -175,7 +175,10 @@ esp_err_t nvs_open_from_partition(const char *part_name, const char* namespace_n * @brief set int8_t value for given key * * Set value for the key, given its name. Note that the actual storage will not be updated - * until \c nvs_commit is called. + * until \c nvs_commit is called. Regardless whether key-value pair is created or updated, + * function always requires at least one nvs available entry. See \c nvs_get_stats . + * After create type of operation, the number of available entries is decreased by one. + * After update type of operation, the number of available entries remains the same. * * @param[in] handle Handle obtained from nvs_open function. * Handles that were opened read only cannot be used. @@ -250,8 +253,15 @@ esp_err_t nvs_set_u64 (nvs_handle_t handle, const char* key, uint64_t value); /** * @brief set string for given key * - * Set value for the key, given its name. Note that the actual storage will not be updated - * until \c nvs_commit is called. + * Sets string value for the key. Function requires whole space for new data to be available + * as contiguous entries in same nvs page. Operation consumes 1 overhead entry and 1 entry per + * each 32 characters of new string including zero character to be set. In case of value update + * for existing key, entries occupied by the previous value and overhead entry are returned to + * the pool of available entries. + * Note that storage of long string values can fail due to fragmentation of nvs pages even if + * \c available_entries returned by \c nvs_get_stats suggests enough overall space available. + * Note that the underlying storage will not be updated until \c nvs_commit is called. + * * * @param[in] handle Handle obtained from nvs_open function. * Handles that were opened read only cannot be used. @@ -280,8 +290,11 @@ esp_err_t nvs_set_str (nvs_handle_t handle, const char* key, const char* value); /** * @brief set variable length binary value for given key * - * This family of functions set value for the key, given its name. Note that - * actual storage will not be updated until nvs_commit function is called. + * Sets variable length binary value for the key. Function uses 2 overhead and 1 entry + * per each 32 bytes of new data from the pool of available entries. See \c nvs_get_stats . + * In case of value update for existing key, space occupied by the existing value and 2 overhead entries + * are returned to the pool of available entries. + * Note that the underlying storage will not be updated until \c nvs_commit is called. * * @param[in] handle Handle obtained from nvs_open function. * Handles that were opened read only cannot be used. @@ -537,24 +550,25 @@ void nvs_close(nvs_handle_t handle); * @note Info about storage space NVS. */ typedef struct { - size_t used_entries; /**< Amount of used entries. */ - size_t free_entries; /**< Amount of free entries. */ - size_t total_entries; /**< Amount all available entries. */ - size_t namespace_count; /**< Amount name space. */ + size_t used_entries; /**< Number of used entries. */ + size_t free_entries; /**< Number of free entries. It includes also reserved entries. */ + size_t available_entries; /**< Number of entries available for data storage. */ + size_t total_entries; /**< Number of all entries. */ + size_t namespace_count; /**< Number of namespaces. */ } nvs_stats_t; /** - * @brief Fill structure nvs_stats_t. It provides info about used memory the partition. + * @brief Fill structure nvs_stats_t. It provides info about memory used by NVS. * - * This function calculates to runtime the number of used entries, free entries, total entries, - * and amount namespace in partition. + * This function calculates the number of used entries, free entries, available entries, total entries + * and number of namespaces in partition. * * \code{c} - * // Example of nvs_get_stats() to get the number of used entries and free entries: + * // Example of nvs_get_stats() to get overview of actual statistics of data entries : * nvs_stats_t nvs_stats; * nvs_get_stats(NULL, &nvs_stats); - * printf("Count: UsedEntries = (%d), FreeEntries = (%d), AllEntries = (%d)\n", - nvs_stats.used_entries, nvs_stats.free_entries, nvs_stats.total_entries); + * printf("Count: UsedEntries = (%lu), FreeEntries = (%lu), AvailableEntries = (%lu), AllEntries = (%lu)\n", + nvs_stats.used_entries, nvs_stats.free_entries, nvs_stats.available_entries, nvs_stats.total_entries); * \endcode * * @param[in] part_name Partition name NVS in the partition table. diff --git a/components/nvs_flash/src/nvs_api.cpp b/components/nvs_flash/src/nvs_api.cpp index 701b09c4cd10..beb9e868ea07 100644 --- a/components/nvs_flash/src/nvs_api.cpp +++ b/components/nvs_flash/src/nvs_api.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD + * SPDX-FileCopyrightText: 2015-2023 Espressif Systems (Shanghai) CO LTD * * SPDX-License-Identifier: Apache-2.0 */ @@ -524,10 +524,11 @@ extern "C" esp_err_t nvs_get_stats(const char* part_name, nvs_stats_t* nvs_stats if (nvs_stats == nullptr) { return ESP_ERR_INVALID_ARG; } - nvs_stats->used_entries = 0; - nvs_stats->free_entries = 0; - nvs_stats->total_entries = 0; - nvs_stats->namespace_count = 0; + nvs_stats->used_entries = 0; + nvs_stats->free_entries = 0; + nvs_stats->total_entries = 0; + nvs_stats->available_entries = 0; + nvs_stats->namespace_count = 0; pStorage = lookup_storage_from_name((part_name == nullptr) ? NVS_DEFAULT_PART_NAME : part_name); if (pStorage == nullptr) { diff --git a/components/nvs_flash/src/nvs_pagemanager.cpp b/components/nvs_flash/src/nvs_pagemanager.cpp index 062f62774381..d74bda5e4ed8 100644 --- a/components/nvs_flash/src/nvs_pagemanager.cpp +++ b/components/nvs_flash/src/nvs_pagemanager.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD + * SPDX-FileCopyrightText: 2015-2023 Espressif Systems (Shanghai) CO LTD * * SPDX-License-Identifier: Apache-2.0 */ @@ -218,6 +218,7 @@ esp_err_t PageManager::fillStats(nvs_stats_t& nvsStats) { nvsStats.used_entries = 0; nvsStats.free_entries = 0; + nvsStats.available_entries = 0; nvsStats.total_entries = 0; esp_err_t err = ESP_OK; @@ -229,10 +230,13 @@ esp_err_t PageManager::fillStats(nvs_stats_t& nvsStats) } } - // free pages + // add free pages nvsStats.total_entries += mFreePageList.size() * Page::ENTRY_COUNT; nvsStats.free_entries += mFreePageList.size() * Page::ENTRY_COUNT; + // calculate available entries from free entries by applying reserved page size + nvsStats.available_entries = (nvsStats.free_entries >= Page::ENTRY_COUNT) ? nvsStats.free_entries - Page::ENTRY_COUNT : 0; + return err; } diff --git a/components/nvs_flash/test_apps/main/test_nvs.c b/components/nvs_flash/test_apps/main/test_nvs.c index 45abee6831ef..dec3eaaa59b0 100644 --- a/components/nvs_flash/test_apps/main/test_nvs.c +++ b/components/nvs_flash/test_apps/main/test_nvs.c @@ -314,7 +314,7 @@ TEST_CASE("calculate used and free space", "[nvs]") TEST_ASSERT_TRUE(stat1.total_entries == stat2.total_entries); TEST_ASSERT_TRUE(stat1.used_entries == 7); - TEST_ASSERT_TRUE(stat1.total_entries == (stat1.used_entries + stat1.free_entries)); + TEST_ASSERT_TRUE(stat1.total_entries == (stat1.used_entries + stat1.free_entries + 126)); // one Page::ENTRY_COUNT is reserved // amount valid pair in namespace 2 size_t h2_count_entries;