Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ffi: Add support to serialize msgpack array/map into a JSON string. #465

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions components/core/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -460,6 +460,7 @@ set(SOURCE_FILES_unitTest
tests/test-EncodedVariableInterpreter.cpp
tests/test-encoding_methods.cpp
tests/test-ffi_SchemaTree.cpp
tests/test-ffi_utils.cpp
tests/test-Grep.cpp
tests/test-ir_encoding_methods.cpp
tests/test-ir_parsing.cpp
Expand Down
131 changes: 131 additions & 0 deletions components/core/src/clp/ffi/utils.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,93 @@
#include <cstdint>
#include <cstdio>
#include <optional>
#include <span>
#include <string>
#include <string_view>
#include <tuple>

#include <msgpack.hpp>

#include "../utf8_utils.hpp"

using std::string;
using std::string_view;

namespace clp::ffi {
namespace {
/**
* Serializes and appends a msgpack object to the given JSON string.
* NOTE: Event if the serialization failed, `json_str` may be modified.
* @param obj
* @param json_str Outputs the appended JSON string.
* @return true on success.
* @return false if the type of the object is not supported, or the serialization failed.
*/
[[nodiscard]] auto serialize_and_append_msgpack_object_to_json_str(
msgpack::object const& obj,
string& json_str
) -> bool;

/**
* Wrapper of `validate_and_append_escaped_utf8_string`, with both leading and end double quote
* marks added to match JSON string spec.
* NOTE: Event if the serialization failed, `json_str` may be modified.
* @param src
* @param json_str Outputs the appended JSON string.
* @return Same as `validate_and_append_escaped_utf8_string`.
*/
[[nodiscard]] auto
append_escaped_utf8_string_to_json_str(string_view src, string& json_str) -> bool;

// NOLINTNEXTLINE(misc-no-recursion)
auto serialize_and_append_msgpack_object_to_json_str(
msgpack::object const& obj,
std::string& json_str
) -> bool {
bool ret_val{true};
switch (obj.type) {
case msgpack::type::MAP:
ret_val = serialize_and_append_msgpack_map_to_json_str(obj, json_str);
break;
case msgpack::type::ARRAY:
ret_val = serialize_and_append_msgpack_array_to_json_str(obj, json_str);
break;
case msgpack::type::NIL:
json_str += "null";
break;
case msgpack::type::BOOLEAN:
json_str += obj.as<bool>() ? "true" : "false";
break;
case msgpack::type::STR:
ret_val = append_escaped_utf8_string_to_json_str(obj.as<std::string_view>(), json_str);
break;
case msgpack::type::FLOAT32:
case msgpack::type::FLOAT:
json_str += std::to_string(obj.as<double>());
break;
case msgpack::type::POSITIVE_INTEGER:
json_str += std::to_string(obj.as<uint64_t>());
break;
case msgpack::type::NEGATIVE_INTEGER:
json_str += std::to_string(obj.as<int64_t>());
break;
default:
ret_val = false;
break;
}
return ret_val;
}

auto append_escaped_utf8_string_to_json_str(string_view src, string& json_str) -> bool {
json_str.push_back('"');
if (false == validate_and_append_escaped_utf8_string(src, json_str)) {
return false;
}
json_str.push_back('"');
return true;
}
} // namespace

auto validate_and_escape_utf8_string(string_view raw) -> std::optional<string> {
std::optional<std::string> ret_val;
auto& escaped{ret_val.emplace()};
Expand Down Expand Up @@ -86,4 +163,58 @@ auto validate_and_append_escaped_utf8_string(std::string_view src, std::string&

return true;
}

// NOLINTNEXTLINE(misc-no-recursion)
auto serialize_and_append_msgpack_array_to_json_str(
msgpack::object const& array,
std::string& json_str
) -> bool {
if (msgpack::type::ARRAY != array.type) {
return false;
}
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-union-access)
auto const array_data{array.via.array};
bool is_first_element{true};
json_str.push_back('[');
for (auto const& element : std::span{array_data.ptr, static_cast<size_t>(array_data.size)}) {
if (is_first_element) {
is_first_element = false;
} else {
json_str.push_back(',');
}
if (false == serialize_and_append_msgpack_object_to_json_str(element, json_str)) {
return false;
}
}
json_str.push_back(']');
return true;
}

// NOLINTNEXTLINE(misc-no-recursion)
auto serialize_and_append_msgpack_map_to_json_str(msgpack::object const& map, std::string& json_str)
-> bool {
if (msgpack::type::MAP != map.type) {
return false;
}
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-union-access)
auto const& map_data{map.via.map};
bool is_first_element{true};
json_str.push_back('{');
for (auto const& [key, val] : std::span{map_data.ptr, static_cast<size_t>(map_data.size)}) {
if (is_first_element) {
is_first_element = false;
} else {
json_str.push_back(',');
}
if (false == append_escaped_utf8_string_to_json_str(key.as<std::string_view>(), json_str)) {
return false;
}
json_str.push_back(':');
if (false == serialize_and_append_msgpack_object_to_json_str(val, json_str)) {
return false;
}
}
json_str.push_back('}');
return true;
}
} // namespace clp::ffi
26 changes: 26 additions & 0 deletions components/core/src/clp/ffi/utils.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
#include <string>
#include <string_view>

#include <msgpack.hpp>

namespace clp::ffi {
/**
* Validates whether the given string is UTF-8 encoded, and escapes any characters to make the
Expand All @@ -26,6 +28,30 @@ namespace clp::ffi {
*/
[[nodiscard]] auto
validate_and_append_escaped_utf8_string(std::string_view src, std::string& dst) -> bool;

/**
* Serializes and appends a msgpack array to the given JSON string.
* @param array
* @param json_str Outputs the appended JSON string.
* @return Whether the serialized succeeded. NOTE: Event if the serialization failed, `json_str` may
* be modified.
*/
[[nodiscard]] auto serialize_and_append_msgpack_array_to_json_str(
msgpack::object const& array,
std::string& json_str
) -> bool;

/**
* Serializes and appends a msgpack map to the given JSON string.
* @param map
* @param json_str Outputs the appended JSON string.
* @return Whether the serialized succeeded. NOTE: Event if the serialization failed, `json_str` may
* be modified.
*/
[[nodiscard]] auto serialize_and_append_msgpack_map_to_json_str(
msgpack::object const& map,
std::string& json_str
) -> bool;
} // namespace clp::ffi

#endif // CLP_FFI_UTILS_HPP
122 changes: 122 additions & 0 deletions components/core/tests/test-ffi_utils.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
#include <cstddef>
#include <optional>
#include <string>
#include <string_view>
#include <vector>

#include <Catch2/single_include/catch2/catch.hpp>
#include <json/single_include/nlohmann/json.hpp>
#include <msgpack.hpp>

#include "../src/clp/ffi/utils.hpp"
#include "../src/clp/type_utils.hpp"

using nlohmann::json;
using std::optional;
using std::string;
using std::string_view;
using std::vector;

using clp::ffi::serialize_and_append_msgpack_array_to_json_str;
using clp::ffi::serialize_and_append_msgpack_map_to_json_str;

namespace {
/**
* Serializes the given msgpack byte sequence into a JSON string.
* @param msgpack_bytes
* @return Serialized JSON string on success.
* @return std::nullopt on failure.
*/
[[nodiscard]] auto serialize_msgpack_bytes_to_json_str(vector<unsigned char> const& msgpack_bytes
) -> optional<string>;

auto serialize_msgpack_bytes_to_json_str(vector<unsigned char> const& msgpack_bytes
) -> optional<string> {
msgpack::object_handle msgpack_oh;
msgpack::unpack(
msgpack_oh,
clp::size_checked_pointer_cast<char const>(msgpack_bytes.data()),
msgpack_bytes.size()
);

optional<string> ret_val;
auto const& msgpack_obj{msgpack_oh.get()};
if (msgpack::type::ARRAY == msgpack_obj.type) {
if (false == serialize_and_append_msgpack_array_to_json_str(msgpack_obj, ret_val.emplace()))
{
return std::nullopt;
}
} else if (msgpack::type::MAP == msgpack_obj.type) {
if (false == serialize_and_append_msgpack_map_to_json_str(msgpack_obj, ret_val.emplace())) {
return std::nullopt;
}
} else {
return std::nullopt;
}
return ret_val;
}
} // namespace

TEST_CASE("test_msgpack_to_json", "[ffi][utils]") {
optional<string> result;
constexpr string_view cStringWithEscape{"String with\\\"escaped\"\\ characters\n\t"};

// Test array with primitive values only
json const array_with_primitive_values_only
= {1,
-1,
1.01,
-1.01,
true,
false,
"short_string",
"This is a long string",
cStringWithEscape,
nullptr};
result = serialize_msgpack_bytes_to_json_str(json::to_msgpack(array_with_primitive_values_only)
);
REQUIRE((result.has_value() && array_with_primitive_values_only == json::parse(result.value()))
);

// Test map with primitive values only
json const map_with_primitive_values_only
= {{"int_key", 1},
{"int_key_negative", -1},
{"float_key", 0.01},
{"float_key_negative", -0.01},
{"bool_key_true", false},
{"bool_key_false", true},
{"str_key", "Test string"},
{"str_with_\"escaped\"_key", cStringWithEscape},
{"null_key", nullptr}};
result = serialize_msgpack_bytes_to_json_str(json::to_msgpack(map_with_primitive_values_only));
REQUIRE((result.has_value() && map_with_primitive_values_only == json::parse(result.value())));

// Test array with inner map
json array_with_map = array_with_primitive_values_only;
array_with_map.emplace_back(map_with_primitive_values_only);
result = serialize_msgpack_bytes_to_json_str(json::to_msgpack(array_with_map));
REQUIRE((result.has_value() && array_with_map == json::parse(result.value())));

// Test map with inner array
json map_with_array = map_with_primitive_values_only;
map_with_array.emplace("array_key", array_with_primitive_values_only);
result = serialize_msgpack_bytes_to_json_str(json::to_msgpack(map_with_array));
REQUIRE((result.has_value() && map_with_array == json::parse(result.value())));

// Recursively create inner maps and arrays
// Note: the execution time and memory consumption will grow exponentially as we increase the
// recursive depth.
constexpr size_t cRecursiveDepth{6};
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This depth is chosen to not take too long. Depth == 6 takes 3 seconds on my laptop, whereas Depth == 7 takes 25 seconds (which might be too long).

for (size_t i{0}; i < cRecursiveDepth; ++i) {
array_with_map.emplace_back(map_with_array);
array_with_map.emplace_back(array_with_map);
result = serialize_msgpack_bytes_to_json_str(json::to_msgpack(array_with_map));
REQUIRE((result.has_value() && array_with_map == json::parse(result.value())));

map_with_array.emplace("array_key_" + std::to_string(i), array_with_map);
map_with_array.emplace("map_key_" + std::to_string(i), map_with_array);
result = serialize_msgpack_bytes_to_json_str(json::to_msgpack(map_with_array));
REQUIRE((result.has_value() && map_with_array == json::parse(result.value())));
}
}
Loading