Skip to content

Commit

Permalink
change(nvs_flash): nvs_get_stats extended, API documentation improved
Browse files Browse the repository at this point in the history
  • Loading branch information
@rrtandler
rrtandler committed Aug 8, 2023
1 parent 31d87a0 commit ac6c03f
Show file tree
Hide file tree
Showing 6 changed files with 133 additions and 67 deletions.
4 changes: 4 additions & 0 deletions components/nvs_flash/host_test/nvs_host_test/main/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,3 +12,7 @@ idf_component_register(SRCS "main.cpp"
"../../../../../tools/catch"
WHOLE_ARCHIVE
REQUIRES nvs_flash)

if(CMAKE_C_COMPILER_ID MATCHES "Clang")
target_compile_options(${COMPONENT_LIB} PRIVATE -std=gnu++20)
endif()
129 changes: 86 additions & 43 deletions components/nvs_flash/host_test/nvs_host_test/main/test_nvs.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD
* SPDX-FileCopyrightText: 2015-2023 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
Expand Down Expand Up @@ -1475,6 +1475,8 @@ TEST_CASE("nvs page selection takes into account free entries also not just eras

TEST_CASE("calculate used and free space", "[nvs]")
{
size_t consumed_entries = 0;

PartitionEmulationFixture f(0, 6);
nvs_flash_deinit();
TEST_ESP_ERR(nvs_get_stats(NULL, NULL), ESP_ERR_INVALID_ARG);
Expand All @@ -1497,113 +1499,154 @@ TEST_CASE("calculate used and free space", "[nvs]")
TEST_ESP_ERR(nvs_get_used_entry_count(handle, &h_count_entries), ESP_ERR_NVS_INVALID_HANDLE);
CHECK(h_count_entries == 0);

nvs::Page p;
// after erase. empty partition
TEST_ESP_OK(nvs_get_stats(NULL, &stat1));
CHECK(stat1.free_entries != 0);
CHECK(stat1.namespace_count == 0);
CHECK(stat1.total_entries == 6 * p.ENTRY_COUNT);
CHECK(stat1.total_entries == 6 * nvs::Page::ENTRY_COUNT);
CHECK(stat1.used_entries == 0);

// create namespace test_k1
// namespace test_k1
nvs_handle_t handle_1;
size_t ns1_expected_entries = 0;

// create namepace
consumed_entries = 1; // should consume one entry
TEST_ESP_OK(nvs_open("test_k1", NVS_READWRITE, &handle_1));
TEST_ESP_OK(nvs_get_stats(NULL, &stat2));
CHECK(stat2.free_entries + 1 == stat1.free_entries);
CHECK(stat2.free_entries + consumed_entries == stat1.free_entries);
CHECK(stat2.namespace_count == 1);
CHECK(stat2.total_entries == stat1.total_entries);
CHECK(stat2.used_entries == 1);
CHECK(stat2.used_entries == stat1.used_entries + consumed_entries);
CHECK(stat2.available_entries + consumed_entries == stat1.available_entries);

// create pair key-value com
consumed_entries = 1; // should consume one entry
TEST_ESP_OK(nvs_set_i32(handle_1, "com", 0x12345678));
TEST_ESP_OK(nvs_get_stats(NULL, &stat1));
CHECK(stat1.free_entries + 1 == stat2.free_entries);
CHECK(stat1.free_entries + consumed_entries == stat2.free_entries);
CHECK(stat1.namespace_count == 1);
CHECK(stat1.total_entries == stat2.total_entries);
CHECK(stat1.used_entries == 2);
CHECK(stat1.used_entries == stat2.used_entries + consumed_entries);
CHECK(stat1.available_entries + consumed_entries == stat2.available_entries);
ns1_expected_entries += consumed_entries;

// change value in com
consumed_entries = 0; // should not consume any entry
TEST_ESP_OK(nvs_set_i32(handle_1, "com", 0x01234567));
TEST_ESP_OK(nvs_get_stats(NULL, &stat2));
CHECK(stat2.free_entries == stat1.free_entries);
CHECK(stat2.namespace_count == 1);
CHECK(stat2.total_entries != 0);
CHECK(stat2.used_entries == 2);
CHECK(stat2.used_entries == stat1.used_entries + consumed_entries);
CHECK(stat2.available_entries + consumed_entries == stat1.available_entries);
ns1_expected_entries += consumed_entries;

// create pair key-value ru
consumed_entries = 1; // should consume one entry
TEST_ESP_OK(nvs_set_i32(handle_1, "ru", 0x00FF00FF));
TEST_ESP_OK(nvs_get_stats(NULL, &stat1));
CHECK(stat1.free_entries + 1 == stat2.free_entries);
CHECK(stat1.free_entries + consumed_entries == stat2.free_entries);
CHECK(stat1.namespace_count == 1);
CHECK(stat1.total_entries != 0);
CHECK(stat1.used_entries == 3);
CHECK(stat1.used_entries == stat2.used_entries + consumed_entries);
CHECK(stat1.available_entries + consumed_entries == stat2.available_entries);
ns1_expected_entries += consumed_entries;

// amount valid pair in namespace 1
size_t h1_count_entries;
TEST_ESP_OK(nvs_get_used_entry_count(handle_1, &h1_count_entries));
CHECK(h1_count_entries == 2);
// amount of valid pairs in namespace 1
size_t ns1_reported_entries;
TEST_ESP_OK(nvs_get_used_entry_count(handle_1, &ns1_reported_entries));
CHECK(ns1_reported_entries == ns1_expected_entries);

// namespace test_k2
nvs_handle_t handle_2;
// create namespace test_k2
size_t ns2_expected_entries = 0;

// create namespace
consumed_entries = 1; // should consume one entry
TEST_ESP_OK(nvs_open("test_k2", NVS_READWRITE, &handle_2));
TEST_ESP_OK(nvs_get_stats(NULL, &stat2));
CHECK(stat2.free_entries + 1 == stat1.free_entries);
CHECK(stat2.free_entries + consumed_entries == stat1.free_entries);
CHECK(stat2.namespace_count == 2);
CHECK(stat2.total_entries == stat1.total_entries);
CHECK(stat2.used_entries == 4);
CHECK(stat2.used_entries == stat1.used_entries + consumed_entries);
CHECK(stat2.available_entries + consumed_entries == stat1.available_entries);

// create pair key-value
// create 3 pairs key-value
consumed_entries = 3; // should consume three entries
TEST_ESP_OK(nvs_set_i32(handle_2, "su1", 0x00000001));
TEST_ESP_OK(nvs_set_i32(handle_2, "su2", 0x00000002));
TEST_ESP_OK(nvs_set_i32(handle_2, "sus", 0x00000003));
TEST_ESP_OK(nvs_get_stats(NULL, &stat1));
CHECK(stat1.free_entries + 3 == stat2.free_entries);
CHECK(stat1.free_entries + consumed_entries == stat2.free_entries);
CHECK(stat1.namespace_count == 2);
CHECK(stat1.total_entries == stat2.total_entries);
CHECK(stat1.used_entries == 7);

CHECK(stat1.total_entries == (stat1.used_entries + stat1.free_entries));
CHECK(stat1.used_entries == stat2.used_entries + consumed_entries);
CHECK(stat1.available_entries + consumed_entries == stat2.available_entries);
ns2_expected_entries += consumed_entries;

// amount valid pair in namespace 2
size_t h2_count_entries;
TEST_ESP_OK(nvs_get_used_entry_count(handle_2, &h2_count_entries));
CHECK(h2_count_entries == 3);
// amount of valid pairs in namespace 2
size_t ns2_reported_entries;
TEST_ESP_OK(nvs_get_used_entry_count(handle_2, &ns2_reported_entries));
CHECK(ns2_reported_entries == ns2_expected_entries);

CHECK(stat1.used_entries == (h1_count_entries + h2_count_entries + stat1.namespace_count));
CHECK(stat1.used_entries == (ns1_reported_entries + ns2_reported_entries + stat1.namespace_count));

nvs_close(handle_1);
nvs_close(handle_2);

size_t temp = h2_count_entries;
TEST_ESP_ERR(nvs_get_used_entry_count(handle_1, &h2_count_entries), ESP_ERR_NVS_INVALID_HANDLE);
CHECK(h2_count_entries == 0);
h2_count_entries = temp;
size_t temp = ns2_reported_entries;
TEST_ESP_ERR(nvs_get_used_entry_count(handle_1, &ns2_reported_entries), ESP_ERR_NVS_INVALID_HANDLE);
CHECK(ns2_reported_entries == 0);
ns2_reported_entries = temp;
TEST_ESP_ERR(nvs_get_used_entry_count(handle_1, NULL), ESP_ERR_INVALID_ARG);

// namespace test_k3
nvs_handle_t handle_3;
// create namespace test_k3
size_t ns3_expected_entries = 0;

// create namespace
consumed_entries = 1; // should consume one entry
TEST_ESP_OK(nvs_open("test_k3", NVS_READWRITE, &handle_3));
TEST_ESP_OK(nvs_get_stats(NULL, &stat2));
CHECK(stat2.free_entries + 1 == stat1.free_entries);
CHECK(stat2.free_entries + consumed_entries == stat1.free_entries);
CHECK(stat2.namespace_count == 3);
CHECK(stat2.total_entries == stat1.total_entries);
CHECK(stat2.used_entries == 8);
CHECK(stat2.used_entries == stat1.used_entries + consumed_entries);
CHECK(stat2.available_entries + consumed_entries == stat1.available_entries);

// create pair blobs
// create pair key - blob
uint32_t blob[12];
consumed_entries = 2 + (sizeof(blob) + 31) / 32; // should consume 2 + entry for each started block of 32 bytes
TEST_ESP_OK(nvs_set_blob(handle_3, "bl1", &blob, sizeof(blob)));
TEST_ESP_OK(nvs_get_stats(NULL, &stat1));
CHECK(stat1.free_entries + 4 == stat2.free_entries);
CHECK(stat1.free_entries + consumed_entries == stat2.free_entries);
CHECK(stat1.namespace_count == 3);
CHECK(stat1.total_entries == stat2.total_entries);
CHECK(stat1.used_entries == 12);
CHECK(stat1.used_entries == stat2.used_entries + consumed_entries);
CHECK(stat1.available_entries + consumed_entries == stat2.available_entries);
CHECK(stat1.total_entries == (stat1.used_entries + stat1.available_entries + nvs::Page::ENTRY_COUNT));
ns3_expected_entries += consumed_entries;

// create pair key - string
char input_string[] = "abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789";
consumed_entries = 1 + (strlen(input_string) + 1 + 31) / 32; // should consume 1 + entry for each started block of 32 bytes
TEST_ESP_OK(nvs_set_str(handle_3, "str1", input_string));
TEST_ESP_OK(nvs_get_stats(NULL, &stat2));
CHECK(stat2.free_entries + consumed_entries == stat1.free_entries);
CHECK(stat2.namespace_count == 3);
CHECK(stat2.total_entries == stat1.total_entries);
CHECK(stat2.used_entries == stat1.used_entries + consumed_entries);
CHECK(stat2.available_entries + consumed_entries == stat1.available_entries);
ns3_expected_entries += consumed_entries;

// amount valid pair in namespace 2
size_t h3_count_entries;
TEST_ESP_OK(nvs_get_used_entry_count(handle_3, &h3_count_entries));
CHECK(h3_count_entries == 4);
// amount of valid pairs in namespace 3
size_t ns3_reported_entries;
TEST_ESP_OK(nvs_get_used_entry_count(handle_3, &ns3_reported_entries));
CHECK(ns3_reported_entries == ns3_expected_entries);

CHECK(stat1.used_entries == (h1_count_entries + h2_count_entries + h3_count_entries + stat1.namespace_count));
// overall check of used entries across all namespaces
CHECK(stat2.used_entries == (ns1_reported_entries + ns2_reported_entries + ns3_reported_entries + stat2.namespace_count));

nvs_close(handle_3);

Expand Down
46 changes: 30 additions & 16 deletions components/nvs_flash/include/nvs.h
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD
* SPDX-FileCopyrightText: 2015-2023 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
Expand Down Expand Up @@ -175,7 +175,10 @@ esp_err_t nvs_open_from_partition(const char *part_name, const char* namespace_n
* @brief set int8_t value for given key
*
* Set value for the key, given its name. Note that the actual storage will not be updated
* until \c nvs_commit is called.
* until \c nvs_commit is called. Regardless whether key-value pair is created or updated,
* function always requires at least one nvs available entry. See \c nvs_get_stats .
* After create type of operation, the number of available entries is decreased by one.
* After update type of operation, the number of available entries remains the same.
*
* @param[in] handle Handle obtained from nvs_open function.
* Handles that were opened read only cannot be used.
Expand Down Expand Up @@ -250,8 +253,15 @@ esp_err_t nvs_set_u64 (nvs_handle_t handle, const char* key, uint64_t value);
/**
* @brief set string for given key
*
* Set value for the key, given its name. Note that the actual storage will not be updated
* until \c nvs_commit is called.
* Sets string value for the key. Function requires whole space for new data to be available
* as contiguous entries in same nvs page. Operation consumes 1 overhead entry and 1 entry per
* each 32 characters of new string including zero character to be set. In case of value update
* for existing key, entries occupied by the previous value and overhead entry are returned to
* the pool of available entries.
* Note that storage of long string values can fail due to fragmentation of nvs pages even if
* \c available_entries returned by \c nvs_get_stats suggests enough overall space available.
* Note that the underlying storage will not be updated until \c nvs_commit is called.
*
*
* @param[in] handle Handle obtained from nvs_open function.
* Handles that were opened read only cannot be used.
Expand Down Expand Up @@ -280,8 +290,11 @@ esp_err_t nvs_set_str (nvs_handle_t handle, const char* key, const char* value);
/**
* @brief set variable length binary value for given key
*
* This family of functions set value for the key, given its name. Note that
* actual storage will not be updated until nvs_commit function is called.
* Sets variable length binary value for the key. Function uses 2 overhead and 1 entry
* per each 32 bytes of new data from the pool of available entries. See \c nvs_get_stats .
* In case of value update for existing key, space occupied by the existing value and 2 overhead entries
* are returned to the pool of available entries.
* Note that the underlying storage will not be updated until \c nvs_commit is called.
*
* @param[in] handle Handle obtained from nvs_open function.
* Handles that were opened read only cannot be used.
Expand Down Expand Up @@ -537,24 +550,25 @@ void nvs_close(nvs_handle_t handle);
* @note Info about storage space NVS.
*/
typedef struct {
size_t used_entries; /**< Amount of used entries. */
size_t free_entries; /**< Amount of free entries. */
size_t total_entries; /**< Amount all available entries. */
size_t namespace_count; /**< Amount name space. */
size_t used_entries; /**< Number of used entries. */
size_t free_entries; /**< Number of free entries. It includes also reserved entries. */
size_t available_entries; /**< Number of entries available for data storage. */
size_t total_entries; /**< Number of all entries. */
size_t namespace_count; /**< Number of namespaces. */
} nvs_stats_t;

/**
* @brief Fill structure nvs_stats_t. It provides info about used memory the partition.
* @brief Fill structure nvs_stats_t. It provides info about memory used by NVS.
*
* This function calculates to runtime the number of used entries, free entries, total entries,
* and amount namespace in partition.
* This function calculates the number of used entries, free entries, available entries, total entries
* and number of namespaces in partition.
*
* \code{c}
* // Example of nvs_get_stats() to get the number of used entries and free entries:
* // Example of nvs_get_stats() to get overview of actual statistics of data entries :
* nvs_stats_t nvs_stats;
* nvs_get_stats(NULL, &nvs_stats);
* printf("Count: UsedEntries = (%d), FreeEntries = (%d), AllEntries = (%d)\n",
nvs_stats.used_entries, nvs_stats.free_entries, nvs_stats.total_entries);
* printf("Count: UsedEntries = (%lu), FreeEntries = (%lu), AvailableEntries = (%lu), AllEntries = (%lu)\n",
nvs_stats.used_entries, nvs_stats.free_entries, nvs_stats.available_entries, nvs_stats.total_entries);
* \endcode
*
* @param[in] part_name Partition name NVS in the partition table.
Expand Down
11 changes: 6 additions & 5 deletions components/nvs_flash/src/nvs_api.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD
* SPDX-FileCopyrightText: 2015-2023 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
Expand Down Expand Up @@ -524,10 +524,11 @@ extern "C" esp_err_t nvs_get_stats(const char* part_name, nvs_stats_t* nvs_stats
if (nvs_stats == nullptr) {
return ESP_ERR_INVALID_ARG;
}
nvs_stats->used_entries = 0;
nvs_stats->free_entries = 0;
nvs_stats->total_entries = 0;
nvs_stats->namespace_count = 0;
nvs_stats->used_entries = 0;
nvs_stats->free_entries = 0;
nvs_stats->total_entries = 0;
nvs_stats->available_entries = 0;
nvs_stats->namespace_count = 0;

pStorage = lookup_storage_from_name((part_name == nullptr) ? NVS_DEFAULT_PART_NAME : part_name);
if (pStorage == nullptr) {
Expand Down
8 changes: 6 additions & 2 deletions components/nvs_flash/src/nvs_pagemanager.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD
* SPDX-FileCopyrightText: 2015-2023 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
Expand Down Expand Up @@ -218,6 +218,7 @@ esp_err_t PageManager::fillStats(nvs_stats_t& nvsStats)
{
nvsStats.used_entries = 0;
nvsStats.free_entries = 0;
nvsStats.available_entries = 0;
nvsStats.total_entries = 0;
esp_err_t err = ESP_OK;

Expand All @@ -229,10 +230,13 @@ esp_err_t PageManager::fillStats(nvs_stats_t& nvsStats)
}
}

// free pages
// add free pages
nvsStats.total_entries += mFreePageList.size() * Page::ENTRY_COUNT;
nvsStats.free_entries += mFreePageList.size() * Page::ENTRY_COUNT;

// calculate available entries from free entries by applying reserved page size
nvsStats.available_entries = (nvsStats.free_entries >= Page::ENTRY_COUNT) ? nvsStats.free_entries - Page::ENTRY_COUNT : 0;

return err;
}

Expand Down
2 changes: 1 addition & 1 deletion components/nvs_flash/test_apps/main/test_nvs.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -314,7 +314,7 @@ TEST_CASE("calculate used and free space", "[nvs]")
TEST_ASSERT_TRUE(stat1.total_entries == stat2.total_entries);
TEST_ASSERT_TRUE(stat1.used_entries == 7);

TEST_ASSERT_TRUE(stat1.total_entries == (stat1.used_entries + stat1.free_entries));
TEST_ASSERT_TRUE(stat1.total_entries == (stat1.used_entries + stat1.free_entries + 126)); // one Page::ENTRY_COUNT is reserved

// amount valid pair in namespace 2
size_t h2_count_entries;
Expand Down

0 comments on commit ac6c03f

Please sign in to comment.