From 69c982e0ae6494f5620e43d5ffb9205b44463bfe Mon Sep 17 00:00:00 2001 From: Denis Biryukov Date: Mon, 30 Sep 2024 00:56:47 +0200 Subject: [PATCH 1/8] wip --- include/zenoh/api/base.hxx | 2 +- include/zenoh/api/bytes.hxx | 642 ++++---------------------- include/zenoh/api/ext/serialization.h | 336 ++++++++++++++ 3 files changed, 438 insertions(+), 542 deletions(-) create mode 100644 include/zenoh/api/ext/serialization.h diff --git a/include/zenoh/api/base.hxx b/include/zenoh/api/base.hxx index 57c98569..9a9348c8 100644 --- a/include/zenoh/api/base.hxx +++ b/include/zenoh/api/base.hxx @@ -84,7 +84,7 @@ class Owned { OwnedType _0; explicit Owned(OwnedType* pv) { - if (pv) { + if (pv != nullptr) { _0 = *pv; ::z_internal_null(pv); } else diff --git a/include/zenoh/api/bytes.hxx b/include/zenoh/api/bytes.hxx index 164fa7b4..d8270482 100644 --- a/include/zenoh/api/bytes.hxx +++ b/include/zenoh/api/bytes.hxx @@ -22,29 +22,16 @@ #include "shm/buffer/buffer.hxx" #endif -#include -#include -#include -#include -#include -#include #include #include -#include #include #include -#include -#include -#include #include namespace zenoh { namespace detail::closures { extern "C" { -inline bool _zenoh_encode_iter(z_owned_bytes_t* b, void* context) { - return IClosure::call_from_context(context, b); -} inline void _zenoh_drop_with_context(void* data, void* context) { (void)data; @@ -61,28 +48,55 @@ struct Slice { inline auto make_slice(const uint8_t* data, size_t len) { return Slice{data, len}; } -template -struct OwnedSlice { - uint8_t* data; - size_t len; - Deleter d; -}; - -template -auto make_owned_slice(uint8_t* data, size_t len, Deleter&& d) { - return OwnedSlice>{data, len, std::forward(d)}; -} - -class ZenohCodec; - -/// @brief A Zenoh serialized data representation. +/// @brief A Zenoh data representation. class Bytes : public Owned<::z_owned_bytes_t> { + private: public: /// @name Constructors - /// @brief Serializes data using default Zenoh codec. - template - Bytes(T data) : Bytes(Bytes::serialize(std::forward(data))) {} + /// @brief Construct by copying sequence of bytes. + template + Bytes(const std::vector& v) : Bytes() { + ::z_bytes_copy_from_buf(interop::as_owned_c_ptr(*this), v.data(), v.size()); + } + + /// @brief Construct by moving sequence of bytes. + template + Bytes(std::vector&& v) : Bytes() { + std::vector* ptr = new std::vector(std::move(v)); + auto d = [p = ptr]() mutable { delete p; }; + using D = decltype(d); + using Dval = std::remove_reference_t; + using DroppableType = typename detail::closures::Droppable; + auto drop = DroppableType::into_context(std::forward(d)); + ::z_bytes_from_buf(interop::as_owned_c_ptr(*this), ptr->data(), ptr->size(), + detail::closures::_zenoh_drop_with_context, drop); + } + + /// @brief Construct by copying sequence of bytes. + template + Bytes(const std::vector& v) : Bytes() { + ::z_bytes_copy_from_buf(interop::as_owned_c_ptr(*this), v.data(), v.size()); + } + + /// @brief Construct by copying sequence of charactes. + Bytes(std::string_view v) : Bytes() { + ::z_view_string_t s; + z_view_string_from_substr(&s, v.data(), v.size()); + ::z_bytes_copy_from_string(interop::as_owned_c_ptr(*this), ::z_loan(s)); + } + + /// @brief Construct by moving a string. + Bytes(std::string&& v) : Bytes() { + std::string* ptr = new std::string(std::move(v)); + auto d = [p = ptr]() mutable { delete p; }; + using D = decltype(d); + using Dval = std::remove_reference_t; + using DroppableType = typename detail::closures::Droppable; + auto drop = DroppableType::into_context(std::forward(d)); + ::z_bytes_from_str(interop::as_owned_c_ptr(*this), const_cast(ptr->c_str()), + detail::closures::_zenoh_drop_with_context, drop); + } /// @brief Construct a shallow copy of this data. Bytes clone() const { @@ -94,83 +108,47 @@ class Bytes : public Owned<::z_owned_bytes_t> { /// @brief Construct an empty data. Bytes() : Owned(nullptr) { ::z_bytes_empty(interop::as_owned_c_ptr(*this)); } - /// @name Methods - - /// @brief Get number of bytes in the pyload. - size_t size() const { return ::z_bytes_len(interop::as_loaned_c_ptr(*this)); } - - /// @brief Serialize specified type. - /// - /// @tparam T type to serialize - /// @tparam Codec codec to use. - /// @param data instance of T to serialize. - /// @param codec instance of Codec to use. - /// @return serialized data. - template - static Bytes serialize(T&& data, Codec codec = Codec()) { - return codec.serialize(std::forward(data)); +#if (defined(Z_FEATURE_SHARED_MEMORY) && defined(Z_FEATURE_UNSTABLE_API)) + Bytes(ZShm&& shm, ZResult* err = nullptr) : Bytes() { + __ZENOH_RESULT_CHECK(::z_bytes_from_shm(interop::as_owned_c_ptr(*this), interop::as_moved_c_ptr(shm)), err, + "Failed to convert from ZShm"); } - /// @brief Serializes multiple pieces of data between begin and end iterators. - /// - /// The data can be later read using Bytes::Iterator provided by Bytes::iter() method. - /// @tparam ForwardIt forward input iterator type - /// @tparam Codec codec type. - /// @param begin start of the iterator range. - /// @param end end of the iterator range. - /// @param codec codec instance. - /// @return serialized data. - template - static Bytes serialize_from_iter(ForwardIt begin, ForwardIt end, Codec codec = Codec()) { - Bytes out; - auto f = [current = begin, end, &codec](z_owned_bytes_t* b) mutable { - if (current == end) { - ::z_internal_null(b); - return false; - } - // increment current, in case iterator dereference might invalidate it, which happens - // for map/set elements extraction while serializing std::move'd map/set - auto it = current++; - *b = interop::move_to_c_obj(Bytes::serialize(*it, codec)); - return true; - }; - using F = decltype(f); - - using ClosureType = typename detail::closures::Closure; - auto closure = ClosureType(std::forward(f), closures::none); - - ::z_bytes_from_iter(interop::as_owned_c_ptr(out), detail::closures::_zenoh_encode_iter, closure.as_context()); - return out; + Bytes(ZShmMut&& shm, ZResult* err = nullptr) : Bytes() { + __ZENOH_RESULT_CHECK(::z_bytes_from_shm_mut(interop::as_owned_c_ptr(*this), interop::as_moved_c_ptr(shm)), err, + "Failed to convert from ZShmMut"); } +#endif + + /// @name Methods - /// @brief Deserialize into specified type. - /// - /// @tparam T Type to deserialize into. - /// @tparam Codec codec to use. - /// @param err if not null, the result code will be written to this location, otherwise ZException exception will be - /// thrown in case of error. - /// @param codec codec instance. - /// @return deserialzied data. - template - T deserialize(ZResult* err, Codec codec = Codec()) const { - return codec.template deserialize(*this, err); + /// @brief Conver to vector of bytes. + std::vector as_vector() const { + Reader r = this->reader(); + std::vector v(this->size()); + r.read(v.data(), v.size()); + return v; } - /// @brief Deserialize into specified type. - /// - /// @tparam T type to deserialize into. - /// @tparam Codec codec to use. - /// @return deserialzied data. - template - T deserialize(Codec codec = Codec()) const { - return codec.template deserialize(*this, nullptr); + /// @brief Convert to string. + std::string as_string() const { + Reader r = this->reader(); + std::string s(this->size(), '\0'); + r.read(reinterpret_cast(s.data()), s.size()); + return s; } - class Iterator; +#if (defined(Z_FEATURE_SHARED_MEMORY) && defined(Z_FEATURE_UNSTABLE_API)) + ZShm as_shm(ZResult* err = nullptr) const { + ZShm shm = interop::detail::null(); + __ZENOH_RESULT_CHECK(::z_bytes_to_owned_shm(interop::as_loaned_c_ptr(*this), interop::as_owned_c_ptr(shm)), err, + "Failed to deserialize into ZShm!"); + return shm; + } +#endif - /// @brief Get iterator to multi-element data serialized previously using ``Bytes::serialize_from_iter``. - /// @return iterator over multiple elements of data. - Iterator iter() const; + /// @brief Get number of bytes in the pyload. + size_t size() const { return ::z_bytes_len(interop::as_loaned_c_ptr(*this)); } class SliceIterator; @@ -187,6 +165,10 @@ class Bytes : public Owned<::z_owned_bytes_t> { friend struct interop::detail::Converter; public: + /// @name Constructors + + /// @brief Construct reader for specified data. + Reader(const Bytes& b) : Copyable(::z_bytes_get_reader(interop::as_loaned_c_ptr(b))) {} /// @name Methods /// @brief Read data into specified destination. @@ -225,31 +207,26 @@ class Bytes : public Owned<::z_owned_bytes_t> { void seek_from_end(int64_t offset, ZResult* err = nullptr) { __ZENOH_RESULT_CHECK(::z_bytes_reader_seek(&this->_0, offset, SEEK_END), err, "seek_from_end failed"); } - - /// @brief Read bounded data previously written by ``Bytes::Writer::write_bounded``. - /// - /// @param err if not null, the result code will be written to this location, otherwise ZException exception - /// will be thrown in case of error. - Bytes read_bounded(ZResult* err = nullptr) { - Bytes b; - __ZENOH_RESULT_CHECK(::z_bytes_reader_read_bounded(&this->_0, &b._0), err, - "Failed to append data with boundaries"); - return b; - } }; /// @brief Create data reader. /// @return reader instance. - Reader reader() const { - return interop::into_copyable_cpp_obj(::z_bytes_get_reader(interop::as_loaned_c_ptr(*this))); - } + Reader reader() const { return Reader(*this); } /// @brief A writer for Zenoh-serialized data. - class Writer : public Copyable<::z_bytes_writer_t> { - using Copyable::Copyable; - friend struct interop::detail::Converter; - + class Writer : public Owned<::z_owned_bytes_writer_t> { public: + /// @name Constructors + + /// Constructs an empty writer + Writer() : Owned(nullptr) { ::z_bytes_writer_empty(interop::as_owned_c_ptr(*this)); } + + /// @brief Construct writer initialized with data. + /// @param b Data to initialize writer with. + Writer(Bytes&& b) : Owned(nullptr) { + z_bytes_writer_from_bytes(interop::as_owned_c_ptr(*this), interop::as_moved_c_ptr(b)); + } + /// @name Methods /// @brief Copy data from sepcified source into underlying ``Bytes`` instance. @@ -262,7 +239,7 @@ class Bytes : public Owned<::z_owned_bytes_t> { } /// @brief Appends another `Bytes` instance. - /// This allows to compose a serialized data out of multiple `Bytes` that may point to different memory regions. + /// This allows to compose data out of multiple `Bytes` that may point to different memory regions. /// Said in other terms, it allows to create a linear view on different memory regions without copy. /// /// @param data data to append. @@ -272,51 +249,16 @@ class Bytes : public Owned<::z_owned_bytes_t> { __ZENOH_RESULT_CHECK(::z_bytes_writer_append(&this->_0, z_move(data._0)), err, "Failed to append data"); } - /// @brief Append another `Bytes` instance, with boundaries information. It would allow to read the same piece - /// of data using ``Bytes::reader::read_bounded``. - /// - /// @param data data to append. - /// @param err if not null, the result code will be written to this location, otherwise ZException exception - /// will be thrown in case of error. - void append_bounded(Bytes&& data, ZResult* err = nullptr) { - __ZENOH_RESULT_CHECK(::z_bytes_writer_append_bounded(&this->_0, z_move(data._0)), err, - "Failed to append data with boundaries"); + /// @brief Finalize all writes and return underlying `Bytes` object. + /// @return Underlying `Bytes` object. + Bytes finish() && { + Bytes b; + ::z_bytes_writer_finish(interop::as_moved_c_ptr(*this), interop::as_owned_c_ptr(b)); + return b; } }; - - /// @brief Create data writer. - /// - /// @note Creating another writer, for the same ``Bytes`` instance, while previous one is still in use is undefined - /// behaviour. - /// @return writer instance. - Writer writer() { - return interop::into_copyable_cpp_obj(::z_bytes_get_writer(interop::as_loaned_c_ptr(*this))); - } -}; - -/// @brief An iterator over multi-element serialized data. -class Bytes::Iterator : Copyable<::z_bytes_iterator_t> { - using Copyable::Copyable; - friend struct interop::detail::Converter; - - public: - /// @name Methods - - /// @brief Return next element of serialized data. - /// @return next element of serialized data. If the iterator reached the end, an empty optional will be returned. - std::optional next() { - std::optional b(std::in_place); - if (!::z_bytes_iterator_next(&this->_0, interop::as_owned_c_ptr(b.value()))) { - b.reset(); - } - return b; - } }; -inline Bytes::Iterator Bytes::iter() const { - return interop::into_copyable_cpp_obj(::z_bytes_get_iterator(interop::as_loaned_c_ptr(*this))); -} - /// @brief An iterator over raw bytes slices. class Bytes::SliceIterator : Copyable<::z_bytes_slice_iterator_t> { using Copyable::Copyable; @@ -341,386 +283,4 @@ inline Bytes::SliceIterator Bytes::slice_iter() const { ::z_bytes_get_slice_iterator(interop::as_loaned_c_ptr(*this))); } -namespace detail { - -template -struct ZenohDeserializer {}; - -template <> -struct ZenohDeserializer { - static std::string deserialize(const Bytes& b, ZResult* err = nullptr) { - (void)err; - auto reader = b.reader(); - std::string s(b.size(), '0'); - reader.read(reinterpret_cast(s.data()), s.size()); - return s; - } -}; - -#if (defined(Z_FEATURE_SHARED_MEMORY) && defined(Z_FEATURE_UNSTABLE_API)) -template <> -struct ZenohDeserializer { - static ZShm deserialize(const Bytes& b, ZResult* err = nullptr) { - ZShm shm = interop::detail::null(); - __ZENOH_RESULT_CHECK( - ::z_bytes_deserialize_into_owned_shm(interop::as_loaned_c_ptr(b), interop::as_owned_c_ptr(shm)), err, - "Failed to deserialize into ZShm!"); - return shm; - } -}; -#endif - -template -struct ZenohDeserializer> { - static std::vector deserialize(const Bytes& b, ZResult* err = nullptr) { - (void)err; - auto reader = b.reader(); - std::vector v(b.size()); - reader.read(v.data(), b.size()); - return v; - } -}; - -template -struct ZenohDeserializer> { - static std::pair deserialize(const Bytes& b, ZResult* err = nullptr) { - zenoh::Bytes ba, bb; - __ZENOH_RESULT_CHECK(::z_bytes_deserialize_into_pair(interop::as_loaned_c_ptr(b), interop::as_owned_c_ptr(ba), - interop::as_owned_c_ptr(bb)), - err, "Failed to deserialize into std::pair"); - return {ZenohDeserializer::deserialize(ba, err), ZenohDeserializer::deserialize(bb, err)}; - } -}; - -template -struct ZenohDeserializer> { - static std::vector deserialize(const Bytes& b, ZResult* err = nullptr) { - std::vector v; - auto it = b.iter(); - for (auto bb = it.next(); bb.has_value(); bb = it.next()) { - v.push_back(bb->deserialize(err)); - } - - return v; - } -}; - -template -struct ZenohDeserializer> { - static std::deque deserialize(const Bytes& b, ZResult* err = nullptr) { - std::deque v; - auto it = b.iter(); - for (auto bb = it.next(); bb.has_value(); bb = it.next()) { - v.push_back(bb->deserialize(err)); - } - - return v; - } -}; - -template -struct ZenohDeserializer> { - static std::unordered_set deserialize(const Bytes& b, ZResult* err = nullptr) { - std::unordered_set s; - auto it = b.iter(); - for (auto bb = it.next(); bb.has_value(); bb = it.next()) { - s.insert(bb->deserialize(err)); - } - - return s; - } -}; - -template -struct ZenohDeserializer> { - static std::set deserialize(const Bytes& b, ZResult* err = nullptr) { - std::set s; - auto it = b.iter(); - for (auto bb = it.next(); bb.has_value(); bb = it.next()) { - s.insert(bb->deserialize(err)); - } - - return s; - } -}; - -template -struct ZenohDeserializer> { - static std::unordered_map deserialize(const Bytes& b, ZResult* err = nullptr) { - std::unordered_map m; - auto it = b.iter(); - for (auto bb = it.next(); bb.has_value(); bb = it.next()) { - m.insert(bb->deserialize>(err)); - } - - return m; - } -}; - -template -struct ZenohDeserializer> { - static std::map deserialize(const Bytes& b, ZResult* err = nullptr) { - std::map m; - auto it = b.iter(); - for (auto bb = it.next(); bb.has_value(); bb = it.next()) { - m.insert(bb->deserialize>(err)); - } - - return m; - } -}; - -#define __ZENOH_DESERIALIZE_ARITHMETIC(TYPE, EXT) \ - template <> \ - struct ZenohDeserializer { \ - static TYPE deserialize(const Bytes& b, ZResult* err = nullptr) { \ - TYPE t; \ - __ZENOH_RESULT_CHECK(::z_bytes_deserialize_into_##EXT(interop::as_loaned_c_ptr(b), &t), err, \ - "Failed to deserialize into " #TYPE); \ - return t; \ - } \ - }; - -__ZENOH_DESERIALIZE_ARITHMETIC(uint8_t, uint8) -__ZENOH_DESERIALIZE_ARITHMETIC(uint16_t, uint16) -__ZENOH_DESERIALIZE_ARITHMETIC(uint32_t, uint32) -__ZENOH_DESERIALIZE_ARITHMETIC(uint64_t, uint64) - -__ZENOH_DESERIALIZE_ARITHMETIC(int8_t, int8) -__ZENOH_DESERIALIZE_ARITHMETIC(int16_t, int16) -__ZENOH_DESERIALIZE_ARITHMETIC(int32_t, int32) -__ZENOH_DESERIALIZE_ARITHMETIC(int64_t, int64) - -__ZENOH_DESERIALIZE_ARITHMETIC(float, float) -__ZENOH_DESERIALIZE_ARITHMETIC(double, double) - -#undef __ZENOH_DESERIALIZE_ARITHMETIC -} // namespace detail - -/// @brief Default codec for Zenoh serialization / deserialziation -class ZenohCodec { - std::shared_ptr _alias_guard; - - public: - /// @brief Consturctor - /// @param alias_guard optional alias guard. If null the data passed by const reference will copied, - /// otherwise it will be aliased instead and a copy if alias_guard will be added to all serialized - /// ``Bytes`` instances, to ensure that aliased data outlives them. - ZenohCodec(std::shared_ptr alias_guard = nullptr) : _alias_guard(std::move(alias_guard)) {} - - /// @brief Serialize pointer and length by copying. - Bytes serialize(const Slice& s) const { - Bytes b; - if (this->_alias_guard == nullptr) { - ::z_bytes_serialize_from_buf(interop::as_owned_c_ptr(b), s.data, s.len); - } else { - auto deleter = [ptr = this->_alias_guard](void*) mutable { ptr.reset(); }; - b = serialize(make_owned_slice(const_cast(s.data), s.len, std::move(deleter))); - } - return b; - } - - /// @brief Serialize pointer and length by moving. - template - Bytes serialize(OwnedSlice&& s) const { - Bytes b; - uint8_t* data = s.data; - size_t len = s.len; - auto d = [s_in = std::move(s)]() mutable { s_in.d(s_in.data); }; - using D = decltype(d); - using Dval = std::remove_reference_t; - using DroppableType = typename detail::closures::Droppable; - auto drop = DroppableType::into_context(std::forward(d)); - ::z_bytes_from_buf(interop::as_owned_c_ptr(b), data, len, detail::closures::_zenoh_drop_with_context, drop); - return b; - } - - Bytes serialize(std::string_view s) const { - return serialize(Slice{reinterpret_cast(s.data()), s.size()}); - } - - Bytes serialize(const char* s) const { return serialize(std::string_view(s)); } - - Bytes serialize(const std::string& s) const { return serialize(static_cast(s)); } - Bytes serialize(std::string&& s) const { - std::string* ptr = new std::string(std::move(s)); - auto deleter = [p = ptr](void*) mutable { delete p; }; - return serialize(make_owned_slice(const_cast(reinterpret_cast(ptr->c_str())), - ptr->size(), std::move(deleter))); - } - - Bytes serialize(Bytes&& b) const { return std::move(b); } - - Bytes serialize(const Bytes& b) const { return b.clone(); } - -#if (defined(Z_FEATURE_SHARED_MEMORY) && defined(Z_FEATURE_UNSTABLE_API)) - Bytes serialize(ZShm&& shm, ZResult* err = nullptr) const { - Bytes b; - __ZENOH_RESULT_CHECK(::z_bytes_serialize_from_shm(interop::as_owned_c_ptr(b), interop::as_moved_c_ptr(shm)), - err, "Failed to serialize ZShm"); - return b; - } - - Bytes serialize(ZShmMut&& shm, ZResult* err = nullptr) const { - Bytes b; - __ZENOH_RESULT_CHECK(::z_bytes_serialize_from_shm_mut(interop::as_owned_c_ptr(b), interop::as_moved_c_ptr(shm)), - err, "Failed to serialize ZShmMut"); - return b; - } -#endif - - template - Bytes serialize(const std::vector& s) const { - auto b = ZenohCodec::serialize(make_slice(s.data(), s.size())); - return b; - } - - template - Bytes serialize(std::vector&& s) const { - std::vector* ptr = new std::vector(std::move(s)); - auto deleter = [p = ptr](void*) mutable { delete p; }; - return serialize(make_owned_slice(ptr->data(), ptr->size(), std::move(deleter))); - } - - template - Bytes serialize(const std::vector& s) const { - return Bytes::serialize_from_iter(s.begin(), s.end(), *this); - } - - template - Bytes serialize(std::vector&& s) const { - auto b = - Bytes::serialize_from_iter(std::make_move_iterator(s.begin()), std::make_move_iterator(s.end()), *this); - s.clear(); - return b; - } - - template - Bytes serialize(const std::deque& s) const { - return Bytes::serialize_from_iter(s.begin(), s.end(), *this); - } - - template - Bytes serialize(std::deque&& s) const { - auto b = - Bytes::serialize_from_iter(std::make_move_iterator(s.begin()), std::make_move_iterator(s.end()), *this); - s.clear(); - return b; - } - - template - Bytes serialize(const std::unordered_set& s) { - return Bytes::serialize_from_iter(s.begin(), s.end(), *this); - } - - template - Bytes serialize(std::unordered_set&& s) const { - auto f = [&s](typename std::unordered_set::iterator it) -> K { - return std::move(s.extract(it).value()); - }; - return Bytes::serialize_from_iter(detail::commons::make_transform_iterator(s.begin(), f), - detail::commons::make_transform_iterator(s.end(), f), *this); - } - - template - Bytes serialize(const std::set& s) const { - return Bytes::serialize_from_iter(s.begin(), s.end(), *this); - } - - template - Bytes serialize(std::set&& s) const { - auto f = [&s](typename std::set::iterator it) -> K { - return std::move(s.extract(it).value()); - }; - return Bytes::serialize_from_iter(detail::commons::make_transform_iterator(s.begin(), f), - detail::commons::make_transform_iterator(s.end(), f), *this); - } - - template - Bytes serialize(const std::unordered_map& s) const { - return Bytes::serialize_from_iter(s.begin(), s.end(), *this); - } - - template - Bytes serialize(std::unordered_map&& s) const { - auto f = [&s](typename std::unordered_map::iterator it) -> std::pair { - auto node = s.extract(it); - return std::pair(std::move(node.key()), std::move(node.mapped())); - }; - return Bytes::serialize_from_iter(detail::commons::make_transform_iterator(s.begin(), f), - detail::commons::make_transform_iterator(s.end(), f), *this); - } - - template - Bytes serialize(const std::map& s) const { - return Bytes::serialize_from_iter(s.begin(), s.end(), *this); - } - - template - Bytes serialize(std::map&& s) const { - auto f = [&s](typename std::map::iterator it) -> std::pair { - auto node = s.extract(it); - return std::pair(std::move(node.key()), std::move(node.mapped())); - }; - return Bytes::serialize_from_iter(detail::commons::make_transform_iterator(s.begin(), f), - detail::commons::make_transform_iterator(s.end(), f), *this); - } - - template - Bytes serialize(const std::pair& s) const { - auto ba = serialize(s.first); - auto bb = serialize(s.second); - Bytes b; - ::z_bytes_from_pair(interop::as_owned_c_ptr(b), interop::as_moved_c_ptr(ba), interop::as_moved_c_ptr(bb)); - return b; - } - - template - Bytes serialize(std::pair&& s) const { - auto ba = serialize(std::move(s.first)); - auto bb = serialize(std::move(s.second)); - Bytes b; - ::z_bytes_from_pair(interop::as_owned_c_ptr(b), interop::as_moved_c_ptr(ba), interop::as_moved_c_ptr(bb)); - return b; - } - - template - Bytes serialize(std::shared_ptr s) const { - if (this->_alias_guard == nullptr) { - T* ptr = s.get(); - ZenohCodec c(std::move(s)); - return c.serialize(*ptr); - } else { - // pointer is inside higher level pointer so there is no need to modify the alias guard. - return this->serialize(*s.get()); - } - } - -#define __ZENOH_SERIALIZE_ARITHMETIC(TYPE, EXT) \ - Bytes serialize(TYPE t) const { \ - Bytes b; \ - ::z_bytes_serialize_from_##EXT(interop::as_owned_c_ptr(b), t); \ - return b; \ - } - - __ZENOH_SERIALIZE_ARITHMETIC(uint8_t, uint8); - __ZENOH_SERIALIZE_ARITHMETIC(uint16_t, uint16); - __ZENOH_SERIALIZE_ARITHMETIC(uint32_t, uint32); - __ZENOH_SERIALIZE_ARITHMETIC(uint64_t, uint64); - - __ZENOH_SERIALIZE_ARITHMETIC(int8_t, int8); - __ZENOH_SERIALIZE_ARITHMETIC(int16_t, int16); - __ZENOH_SERIALIZE_ARITHMETIC(int32_t, int32); - __ZENOH_SERIALIZE_ARITHMETIC(int64_t, int64); - - __ZENOH_SERIALIZE_ARITHMETIC(float, float); - __ZENOH_SERIALIZE_ARITHMETIC(double, double); -#undef __ZENOH_SERIALIZE_ARITHMETIC - - template - T deserialize(const Bytes& b, ZResult* err = nullptr) const { - return detail::ZenohDeserializer::deserialize(b, err); - } -}; - } // namespace zenoh diff --git a/include/zenoh/api/ext/serialization.h b/include/zenoh/api/ext/serialization.h new file mode 100644 index 00000000..acdf2f2e --- /dev/null +++ b/include/zenoh/api/ext/serialization.h @@ -0,0 +1,336 @@ +// +// Copyright (c) 2024 ZettaScale Technology +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ZettaScale Zenoh Team, + +#include "../bytes.hxx" +#include "../base.hxx" +#include "../interop.hxx" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace zenoh { +namespace ext { + +#if defined(Z_FEATURE_UNSTABLE_API) + +class Serializer; +class Deserializer; + +namespace detail { +#define __ZENOH_SERIALIZE_ARITHMETIC(TYPE, EXT) \ +void serialize_with_serializer(zenoh::ext::Serializer& serializer, TYPE t) { \ + ::ze_serializer_serialize_##EXT(interop::as_loaned_c_ptr(serializer), t); \ +} + +__ZENOH_SERIALIZE_ARITHMETIC(uint8_t, uint8) +__ZENOH_SERIALIZE_ARITHMETIC(uint16_t, uint16) +__ZENOH_SERIALIZE_ARITHMETIC(uint32_t, uint32) +__ZENOH_SERIALIZE_ARITHMETIC(uint64_t, uint64) +__ZENOH_SERIALIZE_ARITHMETIC(int8_t, int8) +__ZENOH_SERIALIZE_ARITHMETIC(int16_t, int16) +__ZENOH_SERIALIZE_ARITHMETIC(int32_t, int32) +__ZENOH_SERIALIZE_ARITHMETIC(int64_t, int64) +__ZENOH_SERIALIZE_ARITHMETIC(float, float) +__ZENOH_SERIALIZE_ARITHMETIC(double, double) + +#undef __ZENOH_SERIALIZE_ARITHMETIC +void serialize_with_serializer(zenoh::ext::Serializer& serializer, std::string_view value) { + ::z_view_string_t s; + z_view_string_from_substr(&s, value.data(), value.size()); + ::ze_serializer_serialize_string(interop::as_loaned_c_ptr(serializer), ::z_loan(s)); +} + +void serialize_with_serializer(zenoh::ext::Serializer& serializer, const std::string& value) { + return serialize_with_serializer(serializer, std::string_view(value)); +} + +void serialize_with_serializer(zenoh::ext::Serializer& serializer, const char* value) { + return serialize_with_serializer(serializer, std::string_view(value)); +} + +template +void serialize_with_serializer(zenoh::ext::Serializer& serializer, const std::tuple& value) { + std::apply([&serializer](const auto&... v) { (serialize_with_serializer(serializer, v), ...); } ); +} + +template +void serialize_with_serializer(zenoh::ext::Serializer& serializer, const std::pair& value) { + serialize_with_serializer(value.first); + serialize_with_serializer(value.second); +} + +template +void _serialize_sequence_with_serializer(zenoh::ext::Serializer& serializer, It begin, It end, size_t n) { + ::ze_serializer_serialize_sequence_begin(zenoh::interop::as_loaned_c_ptr(serializer) ,n); + for (const auto it = begin; it != end; ++it) { + serialize_with_serializer(serializer, *it); + } + ::ze_serializer_serialize_sequence_end(zenoh::interop::as_loaned_c_ptr(serializer)); +} + +template +void serialize_with_serializer(zenoh::ext::Serializer& serializer, const std::vector& value) { + _serialize_sequence_with_serializer(v.size(), v.begin(), b.end()); +} + +template +void serialize_with_serializer(zenoh::ext::Serializer& serializer, const std::deque& value) { + _serialize_sequence_with_serializer(value.size(), value.begin(), value.end()); +} + +template +void serialize_with_serializer(zenoh::ext::Serializer& serializer, const std::unordered_set& value) { + return _serialize_sequence_with_serializer(serializer, values.size(), value.begin(), value.end()); +} + +template +void serialize_with_serializer(zenoh::ext::Serializer& serializer, const std::set& value) { + return _serialize_sequence_with_serializer(serializer, values.size(), value.begin(), value.end()); +} + +template +void serialize_with_serializer(zenoh::ext::Serializer& serializer, const std::unordered_map& value) { + return _serialize_sequence_with_serializer(serializer, values.size(), value.begin(), value.end()); +} + +template +void serialize_with_serializer(zenoh::ext::Serializer& serializer, const std::map& value) { + return _serialize_sequence_with_serializer(serializer, values.size(), value.begin(), value.end()); +} + + +#define __ZENOH_DESERIALIZE_ARITHMETIC(TYPE, EXT) \ +bool deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, TYPE& t, zenoh::ZResult* err = nullptr) { \ + __ZENOH_RESULT_CHECK( \ + ::ze_deserializer_deserialize_##EXT(interop::as_copyable_c_ptr(deserializer), &t), \ + err, \ + "Deserialization failure" \ + ); \ + return err == nullptr || *err == Z_OK; \ +} +\ + +__ZENOH_DESERIALIZE_ARITHMETIC(uint8_t, uint8) +__ZENOH_DESERIALIZE_ARITHMETIC(uint16_t, uint16) +__ZENOH_DESERIALIZE_ARITHMETIC(uint32_t, uint32) +__ZENOH_DESERIALIZE_ARITHMETIC(uint64_t, uint64) +__ZENOH_DESERIALIZE_ARITHMETIC(int8_t, int8) +__ZENOH_DESERIALIZE_ARITHMETIC(int16_t, int16) +__ZENOH_DESERIALIZE_ARITHMETIC(int32_t, int32) +__ZENOH_DESERIALIZE_ARITHMETIC(int64_t, int64) +__ZENOH_DESERIALIZE_ARITHMETIC(float, float) +__ZENOH_DESERIALIZE_ARITHMETIC(double, double) + +#undef __ZENOH_DESERIALIZE_ARITHMETIC +bool deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, std::string& value, zenoh::ZResult* err = nullptr) { + z_owned_string_t s; + __ZENOH_RESULT_CHECK( + ::ze_deserializer_deserialize_string(interop::as_copyable_c_ptr(deserializer), &s), + err, + "Deserialization failure" + ); + value = std::string(::z_string_data(::z_loan(s)), ::z_string_len(::z_loan(s))); + return err == nullptr || *err == Z_OK; +} + +template +bool deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, std::tuple& t, zenoh::ZResult* err = nullptr) { + return std::apply([&deserializer, err](const auto&... v) { + bool res = true; + res = deserialize_with_deserializer(deserializer, v, err) && ...; + return res; + } + ); +} + +template +bool deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, std::pair& value, zenoh::ZResult* err = nullptr) { + return deserialize_with_deserializer(deserializer, value.first, err) + && deserialize_with_deserializer(deserialzier, value.second, err); +} + +#define _ZENOH_DESERIALIZE_SEQUENCE_BEGIN \ +size_t len; \ +__ZENOH_RESULT_CHECK( \ + ::ze_deserializer_deserialize_sequence_begin(zenoh::interop::as_copyable_c_ptr(deserializer), &len), \ + err, \ + "Deserialization failure:: Failed to read sequence length" \ +); \ +if (err != nullptr && *err != Z_OK) return false; + +#define _ZENOH_DESERIALIZE_SEQUENCE_END \ +__ZENOH_RESULT_CHECK( \ + ::ze_serializer_serialize_sequence_end(), \ + err, \ + "Deserialization failure:: Failed to finalize sequence read" \ +); \ +return (err == nullptr || *err == Z_OK); + +template +bool deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, std::vector& value, zenoh::ZResult* err = nullptr) { + _ZENOH_DESERIALIZE_SEQUENCE_BEGIN + value.reserve(value.size() + len); + for (size_t i = 0; i < len; ++i) { + T v; + if (!deserialize_with_deserializer(deserializer, &v, err)) return false; + value.push_back(std::move(v)); + } + _ZENOH_DESERIALIZE_SEQUENCE_END +} + +template +bool deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, std::deque& value, zenoh::ZResult* err = nullptr) { + _ZENOH_DESERIALIZE_SEQUENCE_BEGIN + for (size_t i = 0; i < len; ++i) { + T v; + if (!deserialize_with_deserializer(deserializer, &v, err)) return false; + value.push_back(std::move(v)); + } + _ZENOH_DESERIALIZE_SEQUENCE_END +} + +template +bool deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, std::unordered_set& value, zenoh::ZResult* err = nullptr) { + _ZENOH_DESERIALIZE_SEQUENCE_BEGIN + for (size_t i = 0; i < len; ++i) { + K v; + if (!deserialize_with_deserializer(deserializer, &v, err)) return false; + value.insert(std::move(v)); + } + _ZENOH_DESERIALIZE_SEQUENCE_END +} + +template +void deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, std::set& value, zenoh::ZResult* err = nullptr) { + _ZENOH_DESERIALIZE_SEQUENCE_BEGIN + for (size_t i = 0; i < len; ++i) { + K v; + if (!deserialize_with_deserializer(deserializer, &v, err)) return false; + value.insert(std::move(v)); + } + _ZENOH_DESERIALIZE_SEQUENCE_END +} + +template +void deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, std::unordered_map& value, zenoh::ZResult* err = nullptr) { + _ZENOH_DESERIALIZE_SEQUENCE_BEGIN + for (size_t i = 0; i < len; ++i) { + std::pair v; + if (!deserialize_with_deserializer(deserializer, &v, err)) return false; + value.insert(std::move(v)); + } + _ZENOH_DESERIALIZE_SEQUENCE_END +} + +template +void deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, std::map& value, zenoh::ZResult* err = nullptr) { + _ZENOH_DESERIALIZE_SEQUENCE_BEGIN + for (size_t i = 0; i < len; ++i) { + std::pair v; + if (!deserialize_with_deserializer(deserializer, &v, err)) return false; + value.insert(std::move(v)); + } + _ZENOH_DESERIALIZE_SEQUENCE_END +} + +#undef _ZENOH_DESERIALIZE_SEQUENCE_BEGIN +#undef _ZENOH_DESERIALIZE_SEQUENCE_END + +} + +/// @brief A Zenoh data serializer used for incremental serialization of several values. +/// I.e. data produced by subsequent calls to `Serializer::serialize` can be read by corresponding calls to +/// `Deserializer::deserialize` in the same order (or alternatively by a single call to `z_deserialize` +/// into tuple of serialized types). +class Serializer : public Owned<::ze_owned_serializer_t> { +public: + /// @name Constructors + + /// Constructs an empty writer. + Serializer() :Owned(nullptr) { + ::ze_serializer_empty(interop::as_owned_c_ptr(*this)); + } + + /// @name Methods + /// @brief Serialize specified value. + template + void serialize(const T& value) { + return detail::serialize_with_serializer(*this, value); + } + + + /// @brief Finalize serialization and return underlying `Bytes` object. + /// @return Underlying `Bytes` object. + Bytes finish() && { + Bytes b; + ::ze_serializer_finish(interop::as_moved_c_ptr(*this), interop::as_owned_c_ptr(b)); + return b; + } +}; + +/// @brief A Zenoh data deserializer used for incremental deserialization of several values. +/// I.e. data produced by subsequent calls to `Serializer::serialize` can be read by corresponding calls to +/// `Deserializer::deserialize` in the same order (or alternatively by a single call to `z_deserialize` +/// into tuple of serialized types). +class Deserializer : public Copyable<::ze_deserializer_t> { +public: + /// @name Constructors + + /// @brief Construct deserializer for the specified data. + /// @param b Data to initialize deserializer with. + Deserializer(const Bytes& b) :Copyable(::ze_deserializer_from_bytes(zenoh::interop::as_loaned_c_ptr(b))) {} + + /// @name Methods + + /// @brief Deserialize into value of specified type. + template + T deserialize(zenoh::ZResult* err = nullptr) { + T t; + detail::deserialize_with_deserializer(*this, &t, err); + return t; + } +}; + +/// @brief Serializes a single value and returns corresponding `Bytes`. +template +zenoh::Bytes serialize(const T& value) { + Serializer s; + s.serialize(value); + return std::move(s).finish(); +} + + + +/// @brief Serializes a single value and returns corresponding `Bytes`. +template +T deserialize(const zenoh::Bytes& bytes, zenoh::ZResult* err = nullptr) { + Deserializer d(bytes); + // TODO return error if not all bytes are used. + return d.deserialize(err); +} + +#endif +} +} \ No newline at end of file From 90b50c2428693bb5fbcb76d987ae52b0b9717337 Mon Sep 17 00:00:00 2001 From: Denis Biryukov Date: Mon, 30 Sep 2024 20:57:41 +0200 Subject: [PATCH 2/8] implemented serialziation and tests --- examples/universal/z_custom_serializer.cxx | 174 ---------- include/zenoh/api.hxx | 3 + include/zenoh/api/bytes.hxx | 37 +- include/zenoh/api/ext/serialization.h | 336 ------------------ include/zenoh/api/ext/serialization.hxx | 382 +++++++++++++++++++++ tests/CMakeLists.txt | 5 + tests/universal/bytes.cxx | 322 ++--------------- tests/universal/network/implicit.cxx | 36 -- tests/universal/network/pub_sub.cxx | 16 +- tests/universal/network/queryable_get.cxx | 40 +-- tests/universal/serialization.cxx | 108 ++++++ 11 files changed, 575 insertions(+), 884 deletions(-) delete mode 100644 examples/universal/z_custom_serializer.cxx delete mode 100644 include/zenoh/api/ext/serialization.h create mode 100644 include/zenoh/api/ext/serialization.hxx delete mode 100644 tests/universal/network/implicit.cxx create mode 100644 tests/universal/serialization.cxx diff --git a/examples/universal/z_custom_serializer.cxx b/examples/universal/z_custom_serializer.cxx deleted file mode 100644 index decb3465..00000000 --- a/examples/universal/z_custom_serializer.cxx +++ /dev/null @@ -1,174 +0,0 @@ -// -// Copyright (c) 2024 ZettaScale Technology -// -// This program and the accompanying materials are made available under the -// terms of the Eclipse Public License 2.0 which is available at -// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 -// which is available at https://www.apache.org/licenses/LICENSE-2.0. -// -// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 -// -// Contributors: -// ZettaScale Zenoh Team, -// -#include -#include -#include - -#include "zenoh.hxx" - -using namespace zenoh; -using namespace std::chrono_literals; - -struct CustomStruct { - uint32_t u = 0; - double d = 0; - std::string s = {}; -}; - -// Example of codec for a custom class / struct -// We need to define corresponding serialize and deserialize methods -struct CustomCodec { - /// @brief Serialize by copying. - static Bytes serialize(const CustomStruct& s) { - Bytes b; - auto writer = b.writer(); - writer.write_all(serialize_arithmetic(s.u).data(), 4); - writer.write_all(serialize_arithmetic(s.d).data(), 8); - writer.write_all(reinterpret_cast(s.s.data()), s.s.size()); - return b; - } - - /// @brief Serialize by consuming (only applies to string field). - static Bytes serialize(CustomStruct&& s) { - Bytes b; - auto writer = b.writer(); - writer.write_all(serialize_arithmetic(s.u).data(), 4); - writer.write_all(serialize_arithmetic(s.d).data(), 8); - writer.append(std::move(s.s)); - return b; - } - - /// @brief Serialize by aliasing (only applies to string field). - static Bytes serialize(std::shared_ptr s) { - Bytes b; - auto writer = b.writer(); - writer.write_all(serialize_arithmetic(s->u).data(), 4); - writer.write_all(serialize_arithmetic(s->d).data(), 8); - writer.append(Bytes::serialize(s->s, ZenohCodec(s))); - return b; - } - - // deserialize should be a template method - template - static T deserialize(const Bytes& b, ZResult* err = nullptr); - - private: - template - using UintType = typename std::conditional< - T_numBytes == 1, std::uint8_t, - typename std::conditional::type>::type>::type; - - template - static std::enable_if_t, std::array> serialize_arithmetic(T t) { - // use simple little endian encoding - std::array out; - uint8_t mask = 0b11111111; - UintType u = reinterpret_cast&>(t); - for (size_t i = 0; i < out.size(); i++) { - out[i] = static_cast(u & mask); - u = u >> 8; - } - return out; - } - - template - static std::enable_if_t, T> deserialize_arithmetic(const uint8_t* buf) { - // use simple little endian encoding - UintType out = 0; - for (size_t i = 0; i < sizeof(T); i++) { - out = out << 8; - out = out | buf[sizeof(T) - i - 1]; - } - return reinterpret_cast(out); - } -}; - -template <> -CustomStruct CustomCodec::deserialize(const Bytes& b, ZResult* err) { - CustomStruct out; - if (b.size() < 12) { // we should have at least 12 bytes in the payload - if (err != nullptr) { - *err = -1; - return out; - } else { - throw std::runtime_error("Insufficient payload size"); - } - } - - std::array buf; - auto reader = b.reader(); - - reader.read(buf.data(), 4); - out.u = deserialize_arithmetic(buf.data()); - reader.read(buf.data(), 8); - out.d = deserialize_arithmetic(buf.data()); - size_t remaining = b.size() - 12; - out.s = std::string(remaining, 0); - reader.read(reinterpret_cast(out.s.data()), remaining); - return out; -} - -class CustomPublisher { - public: - CustomPublisher(Session& session, std::string_view key_expr) : _pub(session.declare_publisher(KeyExpr(key_expr))) { - ; - } - - void put(const CustomStruct& s) { _pub.put(Bytes::serialize(s, CustomCodec())); } - void put(CustomStruct&& s) { _pub.put(Bytes::serialize(std::move(s), CustomCodec())); } - void put(std::shared_ptr s) { _pub.put(Bytes::serialize(std::move(s), CustomCodec())); } - - private: - Publisher _pub; -}; - -class CustomSubscriber { - public: - CustomSubscriber(Session& session, std::string_view key_expr) - : _sub(session.declare_subscriber( - KeyExpr(key_expr), [this](const Sample& s) { this->on_receive(s); }, closures::none)) { - ; - } - - private: - Subscriber _sub; - - void on_receive(const Sample& sample) { - CustomStruct s = sample.get_payload().deserialize(CustomCodec()); - std::cout << "Received: " - << "{" << s.u << ", " << s.d << ", " << s.s << "}\n"; - } -}; - -int main(int, char**) { - try { -#ifdef ZENOHCXX_ZENOHC - init_log_from_env_or("error"); -#endif - Config config = Config::create_default(); - auto session = Session::open(std::move(config)); - - std::string keyexpr = "demo/example/simple"; - CustomPublisher pub(session, keyexpr); - CustomSubscriber sub(session, keyexpr); - - pub.put(CustomStruct{0, 0.5, "abc"}); - std::this_thread::sleep_for(1s); /// wait a bit to receive the message - - } catch (std::exception& e) { - std::cout << "Error: " << e.what() << std::endl; - } -} \ No newline at end of file diff --git a/include/zenoh/api.hxx b/include/zenoh/api.hxx index f949017a..d3a3c53c 100644 --- a/include/zenoh/api.hxx +++ b/include/zenoh/api.hxx @@ -38,3 +38,6 @@ #if defined Z_FEATURE_SHARED_MEMORY && defined Z_FEATURE_UNSTABLE_API #include "api/shm/shm.hxx" #endif +#if defined Z_FEATURE_UNSTABLE_API +#include "api/ext/serialization.hxx" +#endif diff --git a/include/zenoh/api/bytes.hxx b/include/zenoh/api/bytes.hxx index d8270482..01e1c7c1 100644 --- a/include/zenoh/api/bytes.hxx +++ b/include/zenoh/api/bytes.hxx @@ -73,12 +73,6 @@ class Bytes : public Owned<::z_owned_bytes_t> { detail::closures::_zenoh_drop_with_context, drop); } - /// @brief Construct by copying sequence of bytes. - template - Bytes(const std::vector& v) : Bytes() { - ::z_bytes_copy_from_buf(interop::as_owned_c_ptr(*this), v.data(), v.size()); - } - /// @brief Construct by copying sequence of charactes. Bytes(std::string_view v) : Bytes() { ::z_view_string_t s; @@ -86,6 +80,12 @@ class Bytes : public Owned<::z_owned_bytes_t> { ::z_bytes_copy_from_string(interop::as_owned_c_ptr(*this), ::z_loan(s)); } + /// @brief Construct by copying sequence of charactes. + Bytes(const char* v) : Bytes(std::string_view(v)){}; + + /// @brief Construct by copying sequence of charactes. + Bytes(const std::string& v) : Bytes(std::string_view(v)){}; + /// @brief Construct by moving a string. Bytes(std::string&& v) : Bytes() { std::string* ptr = new std::string(std::move(v)); @@ -109,11 +109,15 @@ class Bytes : public Owned<::z_owned_bytes_t> { Bytes() : Owned(nullptr) { ::z_bytes_empty(interop::as_owned_c_ptr(*this)); } #if (defined(Z_FEATURE_SHARED_MEMORY) && defined(Z_FEATURE_UNSTABLE_API)) + /// @warning This API has been marked as unstable: it works as advertised, but it may be changed in a future + /// release. Bytes(ZShm&& shm, ZResult* err = nullptr) : Bytes() { __ZENOH_RESULT_CHECK(::z_bytes_from_shm(interop::as_owned_c_ptr(*this), interop::as_moved_c_ptr(shm)), err, "Failed to convert from ZShm"); } + /// @warning This API has been marked as unstable: it works as advertised, but it may be changed in a future + /// release. Bytes(ZShmMut&& shm, ZResult* err = nullptr) : Bytes() { __ZENOH_RESULT_CHECK(::z_bytes_from_shm_mut(interop::as_owned_c_ptr(*this), interop::as_moved_c_ptr(shm)), err, "Failed to convert from ZShmMut"); @@ -139,6 +143,8 @@ class Bytes : public Owned<::z_owned_bytes_t> { } #if (defined(Z_FEATURE_SHARED_MEMORY) && defined(Z_FEATURE_UNSTABLE_API)) + /// @warning This API has been marked as unstable: it works as advertised, but it may be changed in a future + /// release. ZShm as_shm(ZResult* err = nullptr) const { ZShm shm = interop::detail::null(); __ZENOH_RESULT_CHECK(::z_bytes_to_owned_shm(interop::as_loaned_c_ptr(*this), interop::as_owned_c_ptr(shm)), err, @@ -181,6 +187,9 @@ class Bytes : public Owned<::z_owned_bytes_t> { /// @return read position indicator on success or -1L if failure occurs. int64_t tell() { return ::z_bytes_reader_tell(&this->_0); } + /// @brief Return the number of bytes that can still be read. + size_t remaining() { return ::z_bytes_reader_remaining(&this->_0); } + /// @brief Set the `reader` position indicator to the value pointed to by offset, starting from the current /// position. /// @param offset offset in bytes starting from the current position. @@ -218,15 +227,9 @@ class Bytes : public Owned<::z_owned_bytes_t> { public: /// @name Constructors - /// Constructs an empty writer + /// Construct an empty writer. Writer() : Owned(nullptr) { ::z_bytes_writer_empty(interop::as_owned_c_ptr(*this)); } - /// @brief Construct writer initialized with data. - /// @param b Data to initialize writer with. - Writer(Bytes&& b) : Owned(nullptr) { - z_bytes_writer_from_bytes(interop::as_owned_c_ptr(*this), interop::as_moved_c_ptr(b)); - } - /// @name Methods /// @brief Copy data from sepcified source into underlying ``Bytes`` instance. @@ -235,10 +238,11 @@ class Bytes : public Owned<::z_owned_bytes_t> { /// @param err if not null, the result code will be written to this location, otherwise ZException exception /// will be thrown in case of error. void write_all(const uint8_t* src, size_t len, ZResult* err = nullptr) { - __ZENOH_RESULT_CHECK(::z_bytes_writer_write_all(&this->_0, src, len), err, "Failed to write data"); + __ZENOH_RESULT_CHECK(::z_bytes_writer_write_all(interop::as_loaned_c_ptr(*this), src, len), err, + "Failed to write data"); } - /// @brief Appends another `Bytes` instance. + /// @brief Append another `Bytes` instance. /// This allows to compose data out of multiple `Bytes` that may point to different memory regions. /// Said in other terms, it allows to create a linear view on different memory regions without copy. /// @@ -246,7 +250,8 @@ class Bytes : public Owned<::z_owned_bytes_t> { /// @param err if not null, the result code will be written to this location, otherwise ZException exception /// will be thrown in case of error. void append(Bytes&& data, ZResult* err = nullptr) { - __ZENOH_RESULT_CHECK(::z_bytes_writer_append(&this->_0, z_move(data._0)), err, "Failed to append data"); + __ZENOH_RESULT_CHECK(::z_bytes_writer_append(interop::as_loaned_c_ptr(*this), z_move(data._0)), err, + "Failed to append data"); } /// @brief Finalize all writes and return underlying `Bytes` object. diff --git a/include/zenoh/api/ext/serialization.h b/include/zenoh/api/ext/serialization.h deleted file mode 100644 index acdf2f2e..00000000 --- a/include/zenoh/api/ext/serialization.h +++ /dev/null @@ -1,336 +0,0 @@ -// -// Copyright (c) 2024 ZettaScale Technology -// -// This program and the accompanying materials are made available under the -// terms of the Eclipse Public License 2.0 which is available at -// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 -// which is available at https://www.apache.org/licenses/LICENSE-2.0. -// -// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 -// -// Contributors: -// ZettaScale Zenoh Team, - -#include "../bytes.hxx" -#include "../base.hxx" -#include "../interop.hxx" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace zenoh { -namespace ext { - -#if defined(Z_FEATURE_UNSTABLE_API) - -class Serializer; -class Deserializer; - -namespace detail { -#define __ZENOH_SERIALIZE_ARITHMETIC(TYPE, EXT) \ -void serialize_with_serializer(zenoh::ext::Serializer& serializer, TYPE t) { \ - ::ze_serializer_serialize_##EXT(interop::as_loaned_c_ptr(serializer), t); \ -} - -__ZENOH_SERIALIZE_ARITHMETIC(uint8_t, uint8) -__ZENOH_SERIALIZE_ARITHMETIC(uint16_t, uint16) -__ZENOH_SERIALIZE_ARITHMETIC(uint32_t, uint32) -__ZENOH_SERIALIZE_ARITHMETIC(uint64_t, uint64) -__ZENOH_SERIALIZE_ARITHMETIC(int8_t, int8) -__ZENOH_SERIALIZE_ARITHMETIC(int16_t, int16) -__ZENOH_SERIALIZE_ARITHMETIC(int32_t, int32) -__ZENOH_SERIALIZE_ARITHMETIC(int64_t, int64) -__ZENOH_SERIALIZE_ARITHMETIC(float, float) -__ZENOH_SERIALIZE_ARITHMETIC(double, double) - -#undef __ZENOH_SERIALIZE_ARITHMETIC -void serialize_with_serializer(zenoh::ext::Serializer& serializer, std::string_view value) { - ::z_view_string_t s; - z_view_string_from_substr(&s, value.data(), value.size()); - ::ze_serializer_serialize_string(interop::as_loaned_c_ptr(serializer), ::z_loan(s)); -} - -void serialize_with_serializer(zenoh::ext::Serializer& serializer, const std::string& value) { - return serialize_with_serializer(serializer, std::string_view(value)); -} - -void serialize_with_serializer(zenoh::ext::Serializer& serializer, const char* value) { - return serialize_with_serializer(serializer, std::string_view(value)); -} - -template -void serialize_with_serializer(zenoh::ext::Serializer& serializer, const std::tuple& value) { - std::apply([&serializer](const auto&... v) { (serialize_with_serializer(serializer, v), ...); } ); -} - -template -void serialize_with_serializer(zenoh::ext::Serializer& serializer, const std::pair& value) { - serialize_with_serializer(value.first); - serialize_with_serializer(value.second); -} - -template -void _serialize_sequence_with_serializer(zenoh::ext::Serializer& serializer, It begin, It end, size_t n) { - ::ze_serializer_serialize_sequence_begin(zenoh::interop::as_loaned_c_ptr(serializer) ,n); - for (const auto it = begin; it != end; ++it) { - serialize_with_serializer(serializer, *it); - } - ::ze_serializer_serialize_sequence_end(zenoh::interop::as_loaned_c_ptr(serializer)); -} - -template -void serialize_with_serializer(zenoh::ext::Serializer& serializer, const std::vector& value) { - _serialize_sequence_with_serializer(v.size(), v.begin(), b.end()); -} - -template -void serialize_with_serializer(zenoh::ext::Serializer& serializer, const std::deque& value) { - _serialize_sequence_with_serializer(value.size(), value.begin(), value.end()); -} - -template -void serialize_with_serializer(zenoh::ext::Serializer& serializer, const std::unordered_set& value) { - return _serialize_sequence_with_serializer(serializer, values.size(), value.begin(), value.end()); -} - -template -void serialize_with_serializer(zenoh::ext::Serializer& serializer, const std::set& value) { - return _serialize_sequence_with_serializer(serializer, values.size(), value.begin(), value.end()); -} - -template -void serialize_with_serializer(zenoh::ext::Serializer& serializer, const std::unordered_map& value) { - return _serialize_sequence_with_serializer(serializer, values.size(), value.begin(), value.end()); -} - -template -void serialize_with_serializer(zenoh::ext::Serializer& serializer, const std::map& value) { - return _serialize_sequence_with_serializer(serializer, values.size(), value.begin(), value.end()); -} - - -#define __ZENOH_DESERIALIZE_ARITHMETIC(TYPE, EXT) \ -bool deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, TYPE& t, zenoh::ZResult* err = nullptr) { \ - __ZENOH_RESULT_CHECK( \ - ::ze_deserializer_deserialize_##EXT(interop::as_copyable_c_ptr(deserializer), &t), \ - err, \ - "Deserialization failure" \ - ); \ - return err == nullptr || *err == Z_OK; \ -} -\ - -__ZENOH_DESERIALIZE_ARITHMETIC(uint8_t, uint8) -__ZENOH_DESERIALIZE_ARITHMETIC(uint16_t, uint16) -__ZENOH_DESERIALIZE_ARITHMETIC(uint32_t, uint32) -__ZENOH_DESERIALIZE_ARITHMETIC(uint64_t, uint64) -__ZENOH_DESERIALIZE_ARITHMETIC(int8_t, int8) -__ZENOH_DESERIALIZE_ARITHMETIC(int16_t, int16) -__ZENOH_DESERIALIZE_ARITHMETIC(int32_t, int32) -__ZENOH_DESERIALIZE_ARITHMETIC(int64_t, int64) -__ZENOH_DESERIALIZE_ARITHMETIC(float, float) -__ZENOH_DESERIALIZE_ARITHMETIC(double, double) - -#undef __ZENOH_DESERIALIZE_ARITHMETIC -bool deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, std::string& value, zenoh::ZResult* err = nullptr) { - z_owned_string_t s; - __ZENOH_RESULT_CHECK( - ::ze_deserializer_deserialize_string(interop::as_copyable_c_ptr(deserializer), &s), - err, - "Deserialization failure" - ); - value = std::string(::z_string_data(::z_loan(s)), ::z_string_len(::z_loan(s))); - return err == nullptr || *err == Z_OK; -} - -template -bool deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, std::tuple& t, zenoh::ZResult* err = nullptr) { - return std::apply([&deserializer, err](const auto&... v) { - bool res = true; - res = deserialize_with_deserializer(deserializer, v, err) && ...; - return res; - } - ); -} - -template -bool deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, std::pair& value, zenoh::ZResult* err = nullptr) { - return deserialize_with_deserializer(deserializer, value.first, err) - && deserialize_with_deserializer(deserialzier, value.second, err); -} - -#define _ZENOH_DESERIALIZE_SEQUENCE_BEGIN \ -size_t len; \ -__ZENOH_RESULT_CHECK( \ - ::ze_deserializer_deserialize_sequence_begin(zenoh::interop::as_copyable_c_ptr(deserializer), &len), \ - err, \ - "Deserialization failure:: Failed to read sequence length" \ -); \ -if (err != nullptr && *err != Z_OK) return false; - -#define _ZENOH_DESERIALIZE_SEQUENCE_END \ -__ZENOH_RESULT_CHECK( \ - ::ze_serializer_serialize_sequence_end(), \ - err, \ - "Deserialization failure:: Failed to finalize sequence read" \ -); \ -return (err == nullptr || *err == Z_OK); - -template -bool deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, std::vector& value, zenoh::ZResult* err = nullptr) { - _ZENOH_DESERIALIZE_SEQUENCE_BEGIN - value.reserve(value.size() + len); - for (size_t i = 0; i < len; ++i) { - T v; - if (!deserialize_with_deserializer(deserializer, &v, err)) return false; - value.push_back(std::move(v)); - } - _ZENOH_DESERIALIZE_SEQUENCE_END -} - -template -bool deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, std::deque& value, zenoh::ZResult* err = nullptr) { - _ZENOH_DESERIALIZE_SEQUENCE_BEGIN - for (size_t i = 0; i < len; ++i) { - T v; - if (!deserialize_with_deserializer(deserializer, &v, err)) return false; - value.push_back(std::move(v)); - } - _ZENOH_DESERIALIZE_SEQUENCE_END -} - -template -bool deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, std::unordered_set& value, zenoh::ZResult* err = nullptr) { - _ZENOH_DESERIALIZE_SEQUENCE_BEGIN - for (size_t i = 0; i < len; ++i) { - K v; - if (!deserialize_with_deserializer(deserializer, &v, err)) return false; - value.insert(std::move(v)); - } - _ZENOH_DESERIALIZE_SEQUENCE_END -} - -template -void deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, std::set& value, zenoh::ZResult* err = nullptr) { - _ZENOH_DESERIALIZE_SEQUENCE_BEGIN - for (size_t i = 0; i < len; ++i) { - K v; - if (!deserialize_with_deserializer(deserializer, &v, err)) return false; - value.insert(std::move(v)); - } - _ZENOH_DESERIALIZE_SEQUENCE_END -} - -template -void deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, std::unordered_map& value, zenoh::ZResult* err = nullptr) { - _ZENOH_DESERIALIZE_SEQUENCE_BEGIN - for (size_t i = 0; i < len; ++i) { - std::pair v; - if (!deserialize_with_deserializer(deserializer, &v, err)) return false; - value.insert(std::move(v)); - } - _ZENOH_DESERIALIZE_SEQUENCE_END -} - -template -void deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, std::map& value, zenoh::ZResult* err = nullptr) { - _ZENOH_DESERIALIZE_SEQUENCE_BEGIN - for (size_t i = 0; i < len; ++i) { - std::pair v; - if (!deserialize_with_deserializer(deserializer, &v, err)) return false; - value.insert(std::move(v)); - } - _ZENOH_DESERIALIZE_SEQUENCE_END -} - -#undef _ZENOH_DESERIALIZE_SEQUENCE_BEGIN -#undef _ZENOH_DESERIALIZE_SEQUENCE_END - -} - -/// @brief A Zenoh data serializer used for incremental serialization of several values. -/// I.e. data produced by subsequent calls to `Serializer::serialize` can be read by corresponding calls to -/// `Deserializer::deserialize` in the same order (or alternatively by a single call to `z_deserialize` -/// into tuple of serialized types). -class Serializer : public Owned<::ze_owned_serializer_t> { -public: - /// @name Constructors - - /// Constructs an empty writer. - Serializer() :Owned(nullptr) { - ::ze_serializer_empty(interop::as_owned_c_ptr(*this)); - } - - /// @name Methods - /// @brief Serialize specified value. - template - void serialize(const T& value) { - return detail::serialize_with_serializer(*this, value); - } - - - /// @brief Finalize serialization and return underlying `Bytes` object. - /// @return Underlying `Bytes` object. - Bytes finish() && { - Bytes b; - ::ze_serializer_finish(interop::as_moved_c_ptr(*this), interop::as_owned_c_ptr(b)); - return b; - } -}; - -/// @brief A Zenoh data deserializer used for incremental deserialization of several values. -/// I.e. data produced by subsequent calls to `Serializer::serialize` can be read by corresponding calls to -/// `Deserializer::deserialize` in the same order (or alternatively by a single call to `z_deserialize` -/// into tuple of serialized types). -class Deserializer : public Copyable<::ze_deserializer_t> { -public: - /// @name Constructors - - /// @brief Construct deserializer for the specified data. - /// @param b Data to initialize deserializer with. - Deserializer(const Bytes& b) :Copyable(::ze_deserializer_from_bytes(zenoh::interop::as_loaned_c_ptr(b))) {} - - /// @name Methods - - /// @brief Deserialize into value of specified type. - template - T deserialize(zenoh::ZResult* err = nullptr) { - T t; - detail::deserialize_with_deserializer(*this, &t, err); - return t; - } -}; - -/// @brief Serializes a single value and returns corresponding `Bytes`. -template -zenoh::Bytes serialize(const T& value) { - Serializer s; - s.serialize(value); - return std::move(s).finish(); -} - - - -/// @brief Serializes a single value and returns corresponding `Bytes`. -template -T deserialize(const zenoh::Bytes& bytes, zenoh::ZResult* err = nullptr) { - Deserializer d(bytes); - // TODO return error if not all bytes are used. - return d.deserialize(err); -} - -#endif -} -} \ No newline at end of file diff --git a/include/zenoh/api/ext/serialization.hxx b/include/zenoh/api/ext/serialization.hxx new file mode 100644 index 00000000..ac83d495 --- /dev/null +++ b/include/zenoh/api/ext/serialization.hxx @@ -0,0 +1,382 @@ +// +// Copyright (c) 2024 ZettaScale Technology +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ZettaScale Zenoh Team, + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../base.hxx" +#include "../bytes.hxx" +#include "../interop.hxx" + +namespace zenoh { +namespace ext { + +#if defined(Z_FEATURE_UNSTABLE_API) + +/// @warning This API has been marked as unstable: it works as advertised, but it may be changed in a future release. +/// @brief A Zenoh data serializer used for incremental serialization of several values. +/// I.e. data produced by subsequent calls to `Serializer::serialize` can be read by corresponding calls to +/// `Deserializer::deserialize` in the same order (or alternatively by a single call to `z_deserialize` +/// into tuple of serialized types). +class Serializer : public Owned<::ze_owned_serializer_t> { + public: + /// @name Constructors + + /// Constructs an empty writer. + Serializer() : Owned(nullptr) { ::ze_serializer_empty(interop::as_owned_c_ptr(*this)); } + + /// @name Methods + /// @brief Serialize specified value. + template + void serialize(const T& value); + + /// @brief Finalize serialization and return underlying `Bytes` object. + /// @return Underlying `Bytes` object. + Bytes finish() && { + Bytes b; + ::ze_serializer_finish(interop::as_moved_c_ptr(*this), interop::as_owned_c_ptr(b)); + return b; + } +}; + +/// @warning This API has been marked as unstable: it works as advertised, but it may be changed in a future release. +/// @brief A Zenoh data deserializer used for incremental deserialization of several values. +/// I.e. data produced by subsequent calls to `Serializer::serialize` can be read by corresponding calls to +/// `Deserializer::deserialize` in the same order (or alternatively by a single call to `z_deserialize` +/// into tuple of serialized types). +class Deserializer : public Copyable<::ze_deserializer_t> { + public: + /// @name Constructors + + /// @brief Construct deserializer for the specified data. + /// @param b Data to initialize deserializer with. + Deserializer(const Bytes& b) : Copyable(::ze_deserializer_from_bytes(zenoh::interop::as_loaned_c_ptr(b))) {} + + /// @name Methods + + /// @brief Deserialize into value of specified type. + template + T deserialize(zenoh::ZResult* err = nullptr); + + /// @brief Checks if deserializer has parsed all the data. + /// @return `true` if there is no more data to parse, `false` otherwise. + bool is_done() { return ::ze_deserializer_is_done(&this->_0); } +}; + +/// @warning This API has been marked as unstable: it works as advertised, but it may be changed in a future release. +/// @brief Serialize a single into `Bytes`. +template +zenoh::Bytes serialize(const T& value) { + Serializer s; + s.serialize(value); + return std::move(s).finish(); +} + +/// @warning This API has been marked as unstable: it works as advertised, but it may be changed in a future release. +/// @brief Deserialize `Bytes` corresponding to a single serialized value. +template +T deserialize(const zenoh::Bytes& bytes, zenoh::ZResult* err = nullptr) { + Deserializer d(bytes); + T t = d.deserialize(err); + if (!d.is_done() && (err == nullptr || *err == Z_OK)) { + __ZENOH_RESULT_CHECK(Z_EDESERIALIZE, err, "Payload contains more bytes than required for deserialization"); + } + return t; +} + +namespace detail { +template +void serialize_with_serializer(zenoh::ext::Serializer& serializer, const T& t); +template +bool deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, T& t, ZResult* err = nullptr); + +#define __ZENOH_SERIALIZE_ARITHMETIC(TYPE, EXT) \ + inline void __zenoh_serialize_with_serializer(zenoh::ext::Serializer& serializer, TYPE t) { \ + ::ze_serializer_serialize_##EXT(zenoh::interop::as_loaned_c_ptr(serializer), t); \ + } + +__ZENOH_SERIALIZE_ARITHMETIC(uint8_t, uint8) +__ZENOH_SERIALIZE_ARITHMETIC(uint16_t, uint16) +__ZENOH_SERIALIZE_ARITHMETIC(uint32_t, uint32) +__ZENOH_SERIALIZE_ARITHMETIC(uint64_t, uint64) +__ZENOH_SERIALIZE_ARITHMETIC(int8_t, int8) +__ZENOH_SERIALIZE_ARITHMETIC(int16_t, int16) +__ZENOH_SERIALIZE_ARITHMETIC(int32_t, int32) +__ZENOH_SERIALIZE_ARITHMETIC(int64_t, int64) +__ZENOH_SERIALIZE_ARITHMETIC(float, float) +__ZENOH_SERIALIZE_ARITHMETIC(double, double) + +#undef __ZENOH_SERIALIZE_ARITHMETIC +inline void __zenoh_serialize_with_serializer(zenoh::ext::Serializer& serializer, std::string_view value) { + ::z_view_string_t s; + z_view_string_from_substr(&s, value.data(), value.size()); + ::ze_serializer_serialize_string(interop::as_loaned_c_ptr(serializer), ::z_loan(s)); +} + +inline void __zenoh_serialize_with_serializer(zenoh::ext::Serializer& serializer, const std::string& value) { + return __zenoh_serialize_with_serializer(serializer, std::string_view(value)); +} + +inline void __zenoh_serialize_with_serializer(zenoh::ext::Serializer& serializer, const char* value) { + return __zenoh_serialize_with_serializer(serializer, std::string_view(value)); +} + +template +void serialize_with_serializer(zenoh::ext::Serializer& serializer, const std::tuple& value) { + std::apply([&serializer](const auto&... v) { (serialize_with_serializer(serializer, v), ...); }, value); +} + +template +void __zenoh_serialize_with_serializer(zenoh::ext::Serializer& serializer, const std::pair& value) { + serialize_with_serializer(serializer, value.first); + serialize_with_serializer(serializer, value.second); +} + +template +void __serialize_sequence_with_serializer(zenoh::ext::Serializer& serializer, It begin, It end, size_t n) { + ::ze_serializer_serialize_sequence_begin(zenoh::interop::as_loaned_c_ptr(serializer), n); + for (auto it = begin; it != end; ++it) { + serialize_with_serializer(serializer, *it); + } + ::ze_serializer_serialize_sequence_end(zenoh::interop::as_loaned_c_ptr(serializer)); +} + +template +void __zenoh_serialize_with_serializer(zenoh::ext::Serializer& serializer, const std::vector& value) { + __serialize_sequence_with_serializer(serializer, value.begin(), value.end(), value.size()); +} + +template +void __zenoh_serialize_with_serializer(zenoh::ext::Serializer& serializer, const std::deque& value) { + __serialize_sequence_with_serializer(serializer, value.begin(), value.end(), value.size()); +} + +template +void __zenoh_serialize_with_serializer(zenoh::ext::Serializer& serializer, + const std::unordered_set& value) { + return __serialize_sequence_with_serializer(serializer, value.begin(), value.end(), value.size()); +} + +template +void __zenoh_serialize_with_serializer(zenoh::ext::Serializer& serializer, + const std::set& value) { + return __serialize_sequence_with_serializer(serializer, value.begin(), value.end(), value.size()); +} + +template +void __zenoh_serialize_with_serializer(zenoh::ext::Serializer& serializer, + const std::unordered_map& value) { + return __serialize_sequence_with_serializer(serializer, value.begin(), value.end(), value.size()); +} + +template +void __zenoh_serialize_with_serializer(zenoh::ext::Serializer& serializer, + const std::map& value) { + return __serialize_sequence_with_serializer(serializer, value.begin(), value.end(), value.size()); +} + +template +void __zenoh_serialize_with_serializer(zenoh::ext::Serializer& serializer, const std::array& value) { + return __serialize_sequence_with_serializer(serializer, value.begin(), value.end(), value.size()); +} + +template +void serialize_with_serializer(zenoh::ext::Serializer& serializer, const T& t) { + __zenoh_serialize_with_serializer(serializer, t); +} + +#define __ZENOH_DESERIALIZE_ARITHMETIC(TYPE, EXT) \ + inline bool __zenoh_deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, TYPE& t, \ + zenoh::ZResult* err = nullptr) { \ + __ZENOH_RESULT_CHECK(::ze_deserializer_deserialize_##EXT(interop::as_copyable_c_ptr(deserializer), &t), err, \ + "Deserialization failure"); \ + return err == nullptr || *err == Z_OK; \ + } + +__ZENOH_DESERIALIZE_ARITHMETIC(uint8_t, uint8) +__ZENOH_DESERIALIZE_ARITHMETIC(uint16_t, uint16) +__ZENOH_DESERIALIZE_ARITHMETIC(uint32_t, uint32) +__ZENOH_DESERIALIZE_ARITHMETIC(uint64_t, uint64) +__ZENOH_DESERIALIZE_ARITHMETIC(int8_t, int8) +__ZENOH_DESERIALIZE_ARITHMETIC(int16_t, int16) +__ZENOH_DESERIALIZE_ARITHMETIC(int32_t, int32) +__ZENOH_DESERIALIZE_ARITHMETIC(int64_t, int64) +__ZENOH_DESERIALIZE_ARITHMETIC(float, float) +__ZENOH_DESERIALIZE_ARITHMETIC(double, double) + +#undef __ZENOH_DESERIALIZE_ARITHMETIC +inline bool __zenoh_deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, std::string& value, + zenoh::ZResult* err = nullptr) { + z_owned_string_t s; + __ZENOH_RESULT_CHECK(::ze_deserializer_deserialize_string(interop::as_copyable_c_ptr(deserializer), &s), err, + "Deserialization failure"); + value = std::string(::z_string_data(::z_loan(s)), ::z_string_len(::z_loan(s))); + return err == nullptr || *err == Z_OK; +} + +template +bool __zenoh_deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, std::tuple& t, + zenoh::ZResult* err = nullptr) { + return std::apply( + [&deserializer, err](auto&... v) { + bool res = true; + res = res && (deserialize_with_deserializer(deserializer, v, err) && ...); + return res; + }, + t); +} + +template +bool __zenoh_deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, std::pair& value, + zenoh::ZResult* err = nullptr) { + return deserialize_with_deserializer(deserializer, value.first, err) && + deserialize_with_deserializer(deserializer, value.second, err); +} + +#define _ZENOH_DESERIALIZE_SEQUENCE_BEGIN \ + size_t len; \ + __ZENOH_RESULT_CHECK( \ + ::ze_deserializer_deserialize_sequence_begin(zenoh::interop::as_copyable_c_ptr(deserializer), &len), err, \ + "Deserialization failure:: Failed to read sequence length"); \ + if (err != nullptr && *err != Z_OK) return false; + +#define _ZENOH_DESERIALIZE_SEQUENCE_END \ + __ZENOH_RESULT_CHECK(::ze_deserializer_deserialize_sequence_end(zenoh::interop::as_copyable_c_ptr(deserializer)), \ + err, "Deserialization failure:: Failed to finalize sequence read"); \ + return (err == nullptr || *err == Z_OK); + +template +bool __zenoh_deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, std::vector& value, + zenoh::ZResult* err = nullptr) { + _ZENOH_DESERIALIZE_SEQUENCE_BEGIN + value.reserve(value.size() + len); + for (size_t i = 0; i < len; ++i) { + T v; + if (!deserialize_with_deserializer(deserializer, v, err)) return false; + value.push_back(std::move(v)); + } + _ZENOH_DESERIALIZE_SEQUENCE_END +} + +template +bool __zenoh_deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, std::array& value, + zenoh::ZResult* err = nullptr) { + _ZENOH_DESERIALIZE_SEQUENCE_BEGIN + if (len != N && (err == nullptr || *err == Z_OK)) { + __ZENOH_RESULT_CHECK(Z_EDESERIALIZE, err, "Incorrect sequence size"); + return false; + } + for (size_t i = 0; i < len; ++i) { + if (!deserialize_with_deserializer(deserializer, value[i], err)) return false; + } + _ZENOH_DESERIALIZE_SEQUENCE_END +} + +template +bool __zenoh_deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, std::deque& value, + zenoh::ZResult* err = nullptr) { + _ZENOH_DESERIALIZE_SEQUENCE_BEGIN + for (size_t i = 0; i < len; ++i) { + T v; + if (!deserialize_with_deserializer(deserializer, v, err)) return false; + value.push_back(std::move(v)); + } + _ZENOH_DESERIALIZE_SEQUENCE_END +} + +template +bool __zenoh_deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, + std::unordered_set& value, + zenoh::ZResult* err = nullptr) { + _ZENOH_DESERIALIZE_SEQUENCE_BEGIN + for (size_t i = 0; i < len; ++i) { + K v; + if (!deserialize_with_deserializer(deserializer, v, err)) return false; + value.insert(std::move(v)); + } + _ZENOH_DESERIALIZE_SEQUENCE_END +} + +template +bool __zenoh_deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, + std::set& value, zenoh::ZResult* err = nullptr) { + _ZENOH_DESERIALIZE_SEQUENCE_BEGIN + for (size_t i = 0; i < len; ++i) { + K v; + if (!deserialize_with_deserializer(deserializer, v, err)) return false; + value.insert(std::move(v)); + } + _ZENOH_DESERIALIZE_SEQUENCE_END +} + +template +bool __zenoh_deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, + std::unordered_map& value, + zenoh::ZResult* err = nullptr) { + _ZENOH_DESERIALIZE_SEQUENCE_BEGIN + for (size_t i = 0; i < len; ++i) { + std::pair v; + if (!deserialize_with_deserializer(deserializer, v, err)) return false; + value.insert(std::move(v)); + } + _ZENOH_DESERIALIZE_SEQUENCE_END +} + +template +bool __zenoh_deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, + std::map& value, zenoh::ZResult* err = nullptr) { + _ZENOH_DESERIALIZE_SEQUENCE_BEGIN + for (size_t i = 0; i < len; ++i) { + std::pair v; + if (!deserialize_with_deserializer(deserializer, v, err)) return false; + value.insert(std::move(v)); + } + _ZENOH_DESERIALIZE_SEQUENCE_END +} + +#undef _ZENOH_DESERIALIZE_SEQUENCE_BEGIN +#undef _ZENOH_DESERIALIZE_SEQUENCE_END + +template +bool deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, T& t, ZResult* err) { + return __zenoh_deserialize_with_deserializer(deserializer, t, err); +} + +} // namespace detail + +template +void Serializer::serialize(const T& value) { + return detail::serialize_with_serializer(*this, value); +} + +template +T Deserializer::deserialize(zenoh::ZResult* err) { + T t; + detail::deserialize_with_deserializer(*this, t, err); + return t; +} + +#endif +} // namespace ext +} // namespace zenoh \ No newline at end of file diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 311180e4..b4bd7bc7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -41,6 +41,11 @@ endfunction() file(GLOB files "${CMAKE_CURRENT_SOURCE_DIR}/universal/*.cxx") foreach(file ${files}) + if(NOT(ZENOHC_BUILD_WITH_UNSTABLE_API)) + if(${file} MATCHES ".*serialization.*$") + continue() + endif() + endif() if(ZENOHCXX_ZENOHC) add_test_instance(${file} zenohc zenohcxx::zenohc "") endif() diff --git a/tests/universal/bytes.cxx b/tests/universal/bytes.cxx index 39add62c..71c56369 100644 --- a/tests/universal/bytes.cxx +++ b/tests/universal/bytes.cxx @@ -23,13 +23,11 @@ using namespace zenoh; void reader_writer() { std::cout << "running reader_writer\n"; std::vector data = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - Bytes b; - { - auto writer = b.writer(); - writer.write_all(data.data(), 5); - writer.write_all(data.data() + 5, 5); - } + Bytes::Writer writer; + writer.write_all(data.data(), 5); + writer.write_all(data.data() + 5, 5); + Bytes b = std::move(writer).finish(); auto reader = b.reader(); std::vector out(3); assert(reader.read(out.data(), 3) == 3); @@ -43,12 +41,10 @@ void reader_writer() { void reader_seek_tell() { std::cout << "running reader_seek_tell\n"; std::vector data = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - Bytes b; - { - auto writer = b.writer(); - writer.write_all(data.data(), 5); - writer.write_all(data.data() + 5, 5); - } + Bytes::Writer writer; + writer.write_all(data.data(), 5); + writer.write_all(data.data() + 5, 5); + Bytes b = std::move(writer).finish(); auto reader = b.reader(); assert(reader.tell() == 0); @@ -76,136 +72,21 @@ void reader_seek_tell() { assert(i == 6); } -void serde_basic() { - std::cout << "running serde_basic\n"; - std::vector data = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - Bytes b = Bytes::serialize(data); - assert(b.deserialize>() == data); - assert(b.size() == 10); - - std::vector data2 = data; - b = Bytes::serialize(std::move(data)); - assert(data.empty()); - assert(b.deserialize>() == data2); - - std::string s = "abc"; - b = Bytes::serialize(s); - assert(b.deserialize() == s); - assert(!s.empty()); - - std::string s2 = s; - b = Bytes::serialize(std::move(s)); - assert(s.empty()); - assert(b.deserialize() == s2); - -#define __ZENOH_TEST_ARITHMETIC(TYPE, VALUE) \ - { \ - TYPE t = VALUE; \ - Bytes b = Bytes::serialize(t); \ - assert(b.deserialize() == t); \ - } - - __ZENOH_TEST_ARITHMETIC(uint8_t, 5); - __ZENOH_TEST_ARITHMETIC(uint16_t, 500); - __ZENOH_TEST_ARITHMETIC(uint32_t, 50000); - __ZENOH_TEST_ARITHMETIC(uint64_t, 500000000000); - - __ZENOH_TEST_ARITHMETIC(int8_t, -5); - __ZENOH_TEST_ARITHMETIC(int16_t, 500); - __ZENOH_TEST_ARITHMETIC(int32_t, -50000); - __ZENOH_TEST_ARITHMETIC(int64_t, -500000000000); - - __ZENOH_TEST_ARITHMETIC(float, 0.5f); - __ZENOH_TEST_ARITHMETIC(double, 123.45); - - auto p = std::make_pair(-12, std::string("123")); - b = Bytes::serialize(p); - assert(b.deserialize() == p); -} - -void serde_iter() { - std::cout << "running serde_iter\n"; - std::vector data = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - auto b = Bytes::serialize_from_iter(data.begin(), data.end()); - auto it = b.iter(); - std::vector out; - for (auto bb = it.next(); bb.has_value(); bb = it.next()) { - out.push_back(bb->deserialize()); - } - assert(data == out); -} - -void serde_advanced() { - std::cout << "running serde_advanced\n"; - std::vector v = {0.1f, 0.2f, 0.3f}; - auto b = Bytes::serialize(v); - assert(b.deserialize() == v); - - std::vector v2 = v; - b = Bytes::serialize(std::move(v)); - assert(v.empty()); - assert(b.deserialize() == v2); - - std::unordered_map mu = {{"a", 0.5}, {"b", -123.45}, {"abc", 3.1415926}}; - b = Bytes::serialize(mu); - assert(b.deserialize() == mu); - - std::unordered_map mu2 = mu; - b = Bytes::serialize(std::move(mu)); - assert(mu.empty()); - assert(b.deserialize() == mu2); - - std::set s = {1, 2, 3, 4, 0}; - b = Bytes::serialize(s); - assert(b.deserialize() == s); - - std::set s2 = s; - b = Bytes::serialize(std::move(s)); - assert(s.empty()); - assert(b.deserialize() == s2); - - std::map> mo = { - {"a", {0.5, 0.2}}, {"b", {-123.45, 0.4}}, {"abc", {3.1415926, -1.0}}}; - - b = Bytes::serialize(mo); - assert(b.deserialize() == mo); - - std::map> mo2 = mo; - b = Bytes::serialize(std::move(mo)); - assert(mo.empty()); - assert(b.deserialize() == mo2); -} - -void serde_shared() { - std::cout << "running serde_shared\n"; - std::vector v = {1, 2, 3, 4, 5}; - auto v_ptr = std::make_shared>(std::move(v)); - auto b = Bytes::serialize(v_ptr); - assert(v_ptr.use_count() == 2); - assert(b.deserialize() == *v_ptr); - b = Bytes(); - assert(v_ptr.use_count() == 1); - - std::unordered_map mu = {{"a", 0.5}, {"b", -123.45}, {"abc", 3.1415926}}; - auto mu_ptr = std::make_shared>(std::move(mu)); - b = Bytes::serialize(mu_ptr); - assert(mu_ptr.use_count() == 4); - auto m = b.deserialize(); - assert(b.deserialize() == *mu_ptr); - b = Bytes(); - assert(mu_ptr.use_count() == 1); -} - void reader_writer_append() { std::cout << "running reader_writer_append\n"; std::vector data = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; std::vector data2 = {11, 12, 13, 14}; - Bytes b; + Bytes b, b2; + { + Bytes::Writer writer; + writer.write_all(data2.data(), 4); + b2 = std::move(writer).finish(); + } { - auto writer = b.writer(); - writer.write_all(data.data(), 5); - writer.write_all(data.data() + 5, 5); - writer.append(data2); + Bytes::Writer writer; + writer.write_all(data.data(), 10); + writer.append(std::move(b2)); + b = std::move(writer).finish(); } auto reader = b.reader(); @@ -222,167 +103,22 @@ void reader_writer_append() { assert(reader.read(out.data(), 1) == 0); // reached the end of the payload } -void reader_writer_append_bounded() { - std::cout << "running reader_writer_append_bounded\n"; - std::vector data = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - std::string s = "abcd"; - float f = 0.5f; - Bytes b; - { - auto writer = b.writer(); - writer.write_all(data.data(), 5); - writer.write_all(data.data() + 5, 5); - writer.append_bounded(s); - writer.append_bounded(f); - } - - auto reader = b.reader(); - std::vector out(3); - assert(reader.read(out.data(), 3) == 3); - assert(out == std::vector(data.begin(), data.begin() + 3)); - out = std::vector(7); - assert(reader.read(out.data(), 7) == 7); - assert(out == std::vector(data.begin() + 3, data.end())); - - assert(reader.read_bounded().deserialize() == s); - assert(reader.read_bounded().deserialize() == f); - assert(reader.read(out.data(), 1) == 0); // reached the end of the payload -} - -struct CustomStruct { - uint32_t u = 0; - double d = 0; - std::string s = {}; -}; - -// Example of codec for a custom class / struct -// We need to define corresponding serialize and deserialize methods -struct CustomCodec { - static Bytes serialize(const CustomStruct& s) { - Bytes b; - auto writer = b.writer(); - writer.write_all(serialize_arithmetic(s.u).data(), 4); - writer.write_all(serialize_arithmetic(s.d).data(), 8); - writer.write_all(reinterpret_cast(s.s.data()), s.s.size()); - return b; - } - - static Bytes serialize(CustomStruct&& s) { - Bytes b; - auto writer = b.writer(); - writer.write_all(serialize_arithmetic(s.u).data(), 4); - writer.write_all(serialize_arithmetic(s.d).data(), 8); - writer.append(std::move(s.s)); - return b; - } - - static Bytes serialize(std::shared_ptr s) { - Bytes b; - auto writer = b.writer(); - writer.write_all(serialize_arithmetic(s->u).data(), 4); - writer.write_all(serialize_arithmetic(s->d).data(), 8); - writer.append(Bytes::serialize(s->s, ZenohCodec(s))); - return b; - } - - // deserialize should be a template method - template - static T deserialize(const Bytes& b, ZResult* err = nullptr); - - private: - template - using UintType = typename std::conditional< - T_numBytes == 1, std::uint8_t, - typename std::conditional::type>::type>::type; - - template - static std::enable_if_t, std::array> serialize_arithmetic(T t) { - // use simple little endian encoding - std::array out; - uint8_t mask = 0b11111111u; - UintType u = reinterpret_cast&>(t); - for (size_t i = 0; i < out.size(); i++) { - out[i] = static_cast(u & mask); - u = u >> 8; - } - return out; - } - - template - static std::enable_if_t, T> deserialize_arithmetic(const uint8_t* buf) { - // use simple little endian encoding - UintType out = 0; - for (size_t i = 0; i < sizeof(T); i++) { - out = out << 8; - out = out | buf[sizeof(T) - i - 1]; - } - return reinterpret_cast(out); - } -}; - -template <> -CustomStruct CustomCodec::deserialize(const Bytes& b, ZResult* err) { - CustomStruct out; - if (b.size() < 12) { // we should have at least 12 bytes in the payload - if (err != nullptr) { - *err = -1; - return out; - } else { - throw std::runtime_error("Insufficient payload size"); - } - } - - std::array buf; - auto reader = b.reader(); - - reader.read(buf.data(), 4); - out.u = deserialize_arithmetic(buf.data()); - reader.read(buf.data(), 8); - out.d = deserialize_arithmetic(buf.data()); - size_t remaining = b.size() - 12; - out.s = std::string(remaining, 0); - reader.read(reinterpret_cast(out.s.data()), remaining); - return out; -} - -void serde_custom() { - std::cout << "running serde_custom\n"; - CustomStruct s; - s.d = 0.5; - s.u = 500; - s.s = "abcd"; - auto b = Bytes::serialize(s, CustomCodec()); - CustomStruct out = b.deserialize(CustomCodec()); - assert(s.d == out.d); - assert(s.u == out.u); - assert(s.s == out.s); - - CustomStruct s2 = s; - - b = Bytes::serialize(std::move(s2), CustomCodec()); - out = b.deserialize(CustomCodec()); - assert(s.d == out.d); - assert(s.u == out.u); - assert(s.s == out.s); +void from_into() { + std::vector v = {1, 2, 4, 5, 6, 7, 8, 9, 10}; + std::vector v2 = v; + std::string s = "abcdefg"; + std::string s2 = s; - b = Bytes::serialize(std::make_shared(s), CustomCodec()); - out = b.deserialize(CustomCodec()); - assert(s.d == out.d); - assert(s.u == out.u); - assert(s.s == out.s); + Bytes bv(v), bs(s), bv2(std::move(v2)), bs2(std::move(s2)); + assert(bv.as_vector() == v); + assert(bs.as_string() == s); + assert(bv2.as_vector() == v); + assert(bs2.as_string() == s); } int main(int argc, char** argv) { reader_writer(); reader_seek_tell(); - serde_basic(); - serde_iter(); - serde_advanced(); - serde_shared(); - reader_writer_append(); - reader_writer_append_bounded(); - serde_custom(); + from_into(); } diff --git a/tests/universal/network/implicit.cxx b/tests/universal/network/implicit.cxx deleted file mode 100644 index 98799054..00000000 --- a/tests/universal/network/implicit.cxx +++ /dev/null @@ -1,36 +0,0 @@ -#include - -#include "zenoh.hxx" - -using namespace zenoh; -using namespace std::chrono_literals; - -#undef NDEBUG -#include - -void put_sub() { - auto session1 = Session::open(Config::create_default()); - auto session2 = Session::open(Config::create_default()); - - std::this_thread::sleep_for(1s); - - auto subscriber = session2.declare_subscriber("zenoh/test", channels::RingChannel(1)); - - std::this_thread::sleep_for(1s); - - Session::PutOptions options; - options.attachment = std::vector{-1, 10, 15000}; - session1.put("zenoh/test", "data", std::move(options)); - - std::this_thread::sleep_for(1s); - - auto res = subscriber.handler().recv(); - assert(std::holds_alternative(res)); - const Sample& msg = std::get(res); - assert(msg.get_keyexpr() == "zenoh/test"); - assert(msg.get_payload().deserialize() == "data"); - assert(msg.get_attachment().has_value()); - assert((msg.get_attachment()->get().deserialize>() == std::vector{-1, 10, 15000})); -} - -int main(int argc, char** argv) { put_sub(); } diff --git a/tests/universal/network/pub_sub.cxx b/tests/universal/network/pub_sub.cxx index 07c656b4..5fe74b65 100644 --- a/tests/universal/network/pub_sub.cxx +++ b/tests/universal/network/pub_sub.cxx @@ -23,7 +23,7 @@ using namespace std::chrono_literals; #include struct CommonAllocator { - Bytes alloc_with_data(const char data[]) { return Bytes::serialize(data); } + Bytes alloc_with_data(const char* data) { return Bytes(data); } }; #if defined Z_FEATURE_SHARED_MEMORY && defined Z_FEATURE_UNSTABLE_API @@ -38,7 +38,7 @@ class SHMAllocator { auto alloc_result = provider.alloc_gc_defrag_blocking(len, AllocAlignment({0})); ZShmMut&& buf = std::get(std::move(alloc_result)); memcpy(buf.data(), data, len + 1); - return Bytes::serialize(std::move(buf)); + return Bytes(std::move(buf)); } }; #endif @@ -59,8 +59,7 @@ void pub_sub(Talloc& alloc) { auto subscriber = session2.declare_subscriber( ke, [&received_messages](const Sample& s) { - received_messages.emplace_back(s.get_keyexpr().as_string_view(), - s.get_payload().deserialize()); + received_messages.emplace_back(s.get_keyexpr().as_string_view(), s.get_payload().as_string()); }, [&subscriber_dropped]() { subscriber_dropped = true; }); @@ -98,8 +97,7 @@ void put_sub(Talloc& alloc) { auto subscriber = session2.declare_subscriber( ke, [&received_messages](const Sample& s) { - received_messages.emplace_back(s.get_keyexpr().as_string_view(), - s.get_payload().deserialize()); + received_messages.emplace_back(s.get_keyexpr().as_string_view(), s.get_payload().as_string()); }, closures::none); @@ -137,11 +135,11 @@ void put_sub_fifo_channel(Talloc& alloc) { auto res = subscriber.handler().recv(); assert(std::holds_alternative(res)); assert(std::get(res).get_keyexpr() == "zenoh/test"); - assert(std::get(res).get_payload().deserialize() == "first"); + assert(std::get(res).get_payload().as_string() == "first"); res = subscriber.handler().try_recv(); assert(std::holds_alternative(res)); assert(std::get(res).get_keyexpr() == "zenoh/test"); - assert(std::get(res).get_payload().deserialize() == "second"); + assert(std::get(res).get_payload().as_string() == "second"); res = subscriber.handler().try_recv(); assert(std::holds_alternative(res)); @@ -168,7 +166,7 @@ void put_sub_ring_channel(Talloc& alloc) { auto res = subscriber.handler().recv(); assert(std::holds_alternative(res)); assert(std::get(res).get_keyexpr() == "zenoh/test"); - assert(std::get(res).get_payload().deserialize() == "second"); + assert(std::get(res).get_payload().as_string() == "second"); res = subscriber.handler().try_recv(); assert(std::holds_alternative(res)); diff --git a/tests/universal/network/queryable_get.cxx b/tests/universal/network/queryable_get.cxx index 81caf812..30a7c467 100644 --- a/tests/universal/network/queryable_get.cxx +++ b/tests/universal/network/queryable_get.cxx @@ -25,7 +25,7 @@ using namespace std::chrono_literals; struct QueryData { std::string key; std::string params; - int32_t payload; + std::string payload; bool operator==(const QueryData& other) { return key == other.key && params == other.params && payload == other.payload; } @@ -46,16 +46,16 @@ void queryable_get() { auto queryable = session1.declare_queryable( ke, [&queries](const Query& q) { - auto payload = q.get_payload()->get().deserialize(); + auto payload = q.get_payload()->get().as_string(); QueryData qd; qd.key = std::string(q.get_keyexpr().as_string_view()); qd.params = std::string(q.get_parameters()); qd.payload = payload; queries.push_back(std::move(qd)); if (q.get_parameters() == "ok") { - q.reply(q.get_keyexpr(), Bytes::serialize(std::to_string(payload))); + q.reply(q.get_keyexpr(), Bytes(payload)); } else { - q.reply_err(Bytes::serialize("err")); + q.reply_err(Bytes("err")); } }, [&queryable_dropped]() { queryable_dropped = true; }); @@ -63,35 +63,35 @@ void queryable_get() { auto on_reply = [&replies, &errors](const Reply& r) { if (r.is_ok()) { - replies.push_back(r.get_ok().get_payload().deserialize()); + replies.push_back(r.get_ok().get_payload().as_string()); } else { - errors.push_back(r.get_err().get_payload().deserialize()); + errors.push_back(r.get_err().get_payload().as_string()); } }; auto on_drop = [&queries_processed]() { queries_processed++; }; Session::GetOptions opt1; - opt1.payload = Bytes::serialize(1); + opt1.payload = Bytes("1"); session2.get(selector, "ok", on_reply, on_drop, std::move(opt1)); std::this_thread::sleep_for(1s); Session::GetOptions opt2; - opt2.payload = Bytes::serialize(2); + opt2.payload = Bytes("2"); session2.get(selector, "ok", on_reply, on_drop, std::move(opt2)); std::this_thread::sleep_for(1s); Session::GetOptions opt3; - opt3.payload = Bytes::serialize(3); + opt3.payload = Bytes("3"); session2.get(selector, "err", on_reply, on_drop, std::move(opt3)); std::this_thread::sleep_for(1s); } assert(queries.size() == 3); - QueryData qd = {"zenoh/test/1", "ok", 1}; + QueryData qd = {"zenoh/test/1", "ok", "1"}; assert(queries[0] == qd); - qd = {"zenoh/test/1", "ok", 2}; + qd = {"zenoh/test/1", "ok", "2"}; assert(queries[1] == qd); - qd = {"zenoh/test/1", "err", 3}; + qd = {"zenoh/test/1", "err", "3"}; assert(queries[2] == qd); /// check that drop does not undeclare @@ -121,7 +121,7 @@ void queryable_get_channel() { std::this_thread::sleep_for(1s); Session::GetOptions opt1; - opt1.payload = Bytes::serialize(1); + opt1.payload = "1"; auto replies = session2.get(selector, "ok", channels::FifoChannel(3), std::move(opt1)); { auto res = queryable.handler().recv(); @@ -129,14 +129,14 @@ void queryable_get_channel() { auto& query = std::get(res); assert(query.get_keyexpr() == selector); assert(query.get_parameters() == "ok"); - assert(query.get_payload()->get().deserialize() == 1); - query.reply(query.get_keyexpr(), Bytes::serialize(std::to_string(1))); + assert(query.get_payload()->get().as_string() == "1"); + query.reply(query.get_keyexpr(), Bytes("1")); } auto res = replies.recv(); assert(std::holds_alternative(res)); assert(std::get(res).is_ok()); - assert(std::get(res).get_ok().get_payload().deserialize() == "1"); + assert(std::get(res).get_ok().get_payload().as_string() == "1"); assert(std::get(res).get_ok().get_keyexpr().as_string_view() == "zenoh/test/1"); res = replies.recv(); @@ -144,7 +144,7 @@ void queryable_get_channel() { assert(std::get(res) == channels::RecvError::Z_DISCONNECTED); Session::GetOptions opt3; - opt3.payload = Bytes::serialize(3); + opt3.payload = Bytes("3"); replies = session2.get(selector, "err", channels::FifoChannel(3), std::move(opt3)); { auto res = queryable.handler().recv(); @@ -152,14 +152,14 @@ void queryable_get_channel() { auto& query = std::get(res); assert(query.get_keyexpr() == selector); assert(query.get_parameters() == "err"); - assert(query.get_payload()->get().deserialize() == 3); - query.reply_err(Bytes::serialize("err")); + assert(query.get_payload()->get().as_string() == "3"); + query.reply_err(Bytes("err")); } res = replies.recv(); assert(std::holds_alternative(res)); assert(!std::get(res).is_ok()); - assert(std::get(res).get_err().get_payload().deserialize() == "err"); + assert(std::get(res).get_err().get_payload().as_string() == "err"); res = replies.recv(); assert(std::holds_alternative(res)); assert(std::get(res) == channels::RecvError::Z_DISCONNECTED); diff --git a/tests/universal/serialization.cxx b/tests/universal/serialization.cxx new file mode 100644 index 00000000..b63b0deb --- /dev/null +++ b/tests/universal/serialization.cxx @@ -0,0 +1,108 @@ +// +// Copyright (c) 2024 ZettaScale Technology +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ZettaScale Zenoh Team, +// + +#include + +#include "zenoh.hxx" +#undef NDEBUG +#include + +#if defined(Z_FEATURE_UNSTABLE_API) +using namespace zenoh; + +template +bool zenoh_test_serialization(const T& t) { + Bytes b = ext::serialize(t); + return ext::deserialize(b) == t; +} + +void serialize_primitive() { + assert(zenoh_test_serialization(5)); + assert(zenoh_test_serialization(500)); + assert(zenoh_test_serialization(50000)); + assert(zenoh_test_serialization(500000000000)); + + assert(zenoh_test_serialization(-5)); + assert(zenoh_test_serialization(500)); + assert(zenoh_test_serialization(50000)); + assert(zenoh_test_serialization(500000000000)); + + assert(zenoh_test_serialization(0.5f)); + assert(zenoh_test_serialization(123.45)); +} + +void serialize_tuple() { + std::tuple t = {0.5, "test", 1}; + assert(zenoh_test_serialization(t)); + + std::pair, float> p = {{-1, -10000, 10000, 1}, 3.1415926f}; + assert(zenoh_test_serialization(p)); +} + +void serialize_container() { + assert(zenoh_test_serialization("abcdefg")); + assert(zenoh_test_serialization>({0.1f, 0.2f, -0.5f, 1000.578f})); + assert(zenoh_test_serialization>({1, 2, 3, -5, 10000, -999999999})); + assert(zenoh_test_serialization>({1, 2, 3, -5, 10000, -999999999})); + std::array a = {1, 2, 3, -5, 5, -500}; + assert(zenoh_test_serialization(a)); + std::unordered_map m; + m[100] = "abc"; + m[10000] = "def"; + m[2000000000] = "hij"; + assert(zenoh_test_serialization(m)); + std::map m2; + m2[100] = "abc"; + m2[10000] = "def"; + m2[2000000000] = "hij"; + assert(zenoh_test_serialization(m2)); +} + +struct CustomStruct { + std::vector vd; + int32_t i; + std::string s; +}; + +void __zenoh_serialize_with_serializer(zenoh::ext::Serializer& serializer, const CustomStruct& s) { + serializer.serialize(s.vd); + serializer.serialize(s.i); + serializer.serialize(s.s); +} + +bool __zenoh_deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, CustomStruct& s, + ZResult* err = nullptr) { + return zenoh::ext::detail::deserialize_with_deserializer(deserializer, s.vd, err) && + zenoh::ext::detail::deserialize_with_deserializer(deserializer, s.i, err) && + zenoh::ext::detail::deserialize_with_deserializer(deserializer, s.s, err); +} + +void serialize_custom() { + CustomStruct s = {{0.1, 0.2, -1000.55}, 32, "test"}; + Bytes b = zenoh::ext::serialize(s); + CustomStruct s_out = zenoh::ext::deserialize(b); + assert(s_out.vd == s_out.vd); + assert(s.i == s_out.i); + assert(s.s == s_out.s); +} + +#endif +int main(int argc, char** argv) { +#if defined(Z_FEATURE_UNSTABLE_API) + serialize_primitive(); + serialize_tuple(); + serialize_container(); + serialize_custom(); +#endif +} From 9319f01eb983f1d5ccb905c28238137168786799 Mon Sep 17 00:00:00 2001 From: Denis Biryukov Date: Mon, 30 Sep 2024 22:26:51 +0200 Subject: [PATCH 3/8] fixed examples --- examples/universal/z_bytes.cxx | 124 ++++++++---------- examples/universal/z_get.cxx | 6 +- examples/universal/z_get_attachment.cxx | 20 ++- examples/universal/z_get_channel.cxx | 9 +- .../universal/z_get_channel_non_blocking.cxx | 9 +- examples/universal/z_pub.cxx | 6 +- examples/universal/z_pub_attachment.cxx | 17 +-- examples/universal/z_put.cxx | 14 +- examples/universal/z_queryable.cxx | 6 +- examples/universal/z_queryable_attachment.cxx | 17 +-- examples/universal/z_sub.cxx | 3 +- examples/universal/z_sub_attachment.cxx | 7 +- examples/zenohc/z_get_shm.cxx | 6 +- examples/zenohc/z_ping_shm.cxx | 4 +- examples/zenohc/z_pub_shm.cxx | 4 +- examples/zenohc/z_pub_shm_thr.cxx | 2 +- examples/zenohc/z_queryable_shm.cxx | 8 +- examples/zenohc/z_sub_shm.cxx | 5 +- 18 files changed, 116 insertions(+), 151 deletions(-) diff --git a/examples/universal/z_bytes.cxx b/examples/universal/z_bytes.cxx index d915f064..9f654727 100644 --- a/examples/universal/z_bytes.cxx +++ b/examples/universal/z_bytes.cxx @@ -21,34 +21,21 @@ using namespace zenoh; int _main(int argc, char** argv) { - // Numeric: u8, u16, u32, u128, usize, i8, i16, i32, i128, isize, f32, f64 - { - const uint32_t input = 1234; - const auto payload = Bytes::serialize(input); - const auto output = payload.deserialize(); - assert(input == output); - // Corresponding encoding to be used in operations like `.put()`, `.reply()`, etc. -#if defined(ZENOHCXX_ZENOHC) || (Z_FEATURE_ENCODING_VALUES == 1) - const auto encoding = Encoding::Predefined::zenoh_uint32(); -#else - const auto encoding = Encoding("zenoh/uint32"); -#endif - } - + // using raw data // String { // C-String { const char* input = "test"; - const auto payload = Bytes::serialize(input); - const auto output = payload.deserialize(); + const auto payload = Bytes(input); + const auto output = payload.as_string(); assert(input == output); } // std::string { const std::string input = "test"; - const auto payload = Bytes::serialize(input); - const auto output = payload.deserialize(); + const auto payload = Bytes(input); + const auto output = payload.as_string(); assert(input == output); } // Corresponding encoding to be used in operations like `.put()`, `.reply()`, etc. @@ -59,11 +46,11 @@ int _main(int argc, char** argv) { #endif } - // Vec: The deserialization should be infallible + // Vector of uint8_t { const std::vector input = {1, 2, 3, 4}; - const auto payload = Bytes::serialize(input); - const auto output = payload.deserialize>(); + const auto payload = Bytes(input); + const auto output = payload.as_vector(); assert(input == output); // Corresponding encoding to be used in operations like `.put()`, `.reply()`, etc. #if defined(ZENOHCXX_ZENOHC) || (Z_FEATURE_ENCODING_VALUES == 1) @@ -73,48 +60,10 @@ int _main(int argc, char** argv) { #endif } - // Writer & Reader - { - // serialization - Bytes bytes; - auto writer = bytes.writer(); - - const uint32_t i1 = 1234; - const std::string i2 = "test"; - const std::vector i3 = {1, 2, 3, 4}; - - writer.append_bounded(i1); - writer.append_bounded(i2); - writer.append_bounded(i3); - - // deserialization - auto reader = bytes.reader(); - - const auto o1 = reader.read_bounded().deserialize(); - const auto o2 = reader.read_bounded().deserialize(); - const auto o3 = reader.read_bounded().deserialize>(); - - assert(i1 == o1); - assert(i2 == o2); - assert(i3 == o3); - } - - // Iterator - { - const int32_t input[] = {1, 2, 3, 4}; - const auto payload = Bytes::serialize_from_iter(input, input + 4); - - auto idx = 0; - auto it = payload.iter(); - for (auto elem = it.next(); elem.has_value(); elem = it.next()) { - assert(input[idx++] == elem.value().deserialize()); - } - } - - // Iterator RAW + // Iterator RAW (in case of fragmented data) { const std::vector input = {1, 2, 3, 4}; - const auto payload = Bytes::serialize(input); + const auto payload = Bytes(input); size_t idx = 0; auto it = payload.slice_iter(); @@ -126,14 +75,48 @@ int _main(int argc, char** argv) { } } +#if defined(Z_FEATURE_UNSTABLE_API) + /// Serialization + // Vector + { + const std::vector input = {1, 2, 3, 4}; + const auto payload = ext::serialize(input); + const auto output = ext::deserialize>(payload); + assert(input == output); + } // HashMap { const std::unordered_map input = {{0, "abc"}, {1, "def"}}; - const auto payload = Bytes::serialize(input); - const auto output = payload.deserialize>(); + const auto payload = ext::serialize(input); + const auto output = ext::deserialize>(payload); assert(input == output); } + // Serializer, deserializer (for struct or tuple serialization) + { + // serialization + auto serializer = ext::Serializer(); + const uint32_t i1 = 1234; + const std::string i2 = "test"; + const std::vector i3 = {1, 2, 3, 4}; + serializer.serialize(i1); + serializer.serialize(i2); + serializer.serialize(i3); + Bytes bytes = std::move(serializer).finish(); + + // deserialization + auto deserializer = ext::Deserializer(bytes); + + const auto o1 = deserializer.deserialize(); + const auto o2 = deserializer.deserialize(); + const auto o3 = deserializer.deserialize>(); + + assert(i1 == o1); + assert(i2 == o2); + assert(i3 == o3); + } +#endif + #ifdef ZENOH_CPP_EXMAPLE_WITH_PROTOBUF // Protobuf // This example is conditionally compiled depending on build system being able to find Protobuf installation @@ -148,20 +131,20 @@ int _main(int argc, char** argv) { input.set_name("John Doe"); // Serialize PB message into wire format - const auto input_wire_pb = input.SerializeAsString(); + // Avoid using std::string for raw bytes, since certain bindigs + // might rise a error if the string is not a valid sequence of utf-8 characters. + std::vector input_wire_pb(input.ByteSizeLong()); + input.SerializeToArray(input_wire_pb.data(), input_wire_pb.size()); // Put PB wire format into Bytes - const auto payload = Bytes::serialize(input_wire_pb); + const auto payload = Bytes(std::move(input_wire_pb)); // Extract PB wire format - const auto output_wire_pb = payload.deserialize(); - - // wire PB data is equal - assert(input_wire_pb == output_wire_pb); + const auto output_wire_pb = payload.as_vector(); // deserialize output wire PB into PB message Entity output; - const auto parsed = output.ParseFromString(output_wire_pb); + const auto parsed = output.ParseFromArray(output_wire_pb.data(), output_wire_pb.size()); assert(parsed); // data is equal @@ -178,7 +161,6 @@ int _main(int argc, char** argv) { google::protobuf::ShutdownProtobufLibrary(); } #endif - return 0; } diff --git a/examples/universal/z_get.cxx b/examples/universal/z_get.cxx index e4959b8c..3c2f6dbd 100644 --- a/examples/universal/z_get.cxx +++ b/examples/universal/z_get.cxx @@ -74,9 +74,9 @@ int _main(int argc, char **argv) { if (reply.is_ok()) { const auto &sample = reply.get_ok(); std::cout << "Received ('" << sample.get_keyexpr().as_string_view() << "' : '" - << sample.get_payload().deserialize() << "')\n"; + << sample.get_payload().as_string() << "')\n"; } else { - std::cout << "Received an error :" << reply.get_err().get_payload().deserialize() << "\n"; + std::cout << "Received an error :" << reply.get_err().get_payload().as_string() << "\n"; } }; @@ -91,7 +91,7 @@ int _main(int argc, char **argv) { #else Session::GetOptions options; options.target = Z_QUERY_TARGET_ALL; - options.payload = Bytes::serialize(value); + options.payload = value; session.get(keyexpr, "", on_reply, on_done, std::move(options)); #endif diff --git a/examples/universal/z_get_attachment.cxx b/examples/universal/z_get_attachment.cxx index 64993866..3a052eda 100644 --- a/examples/universal/z_get_attachment.cxx +++ b/examples/universal/z_get_attachment.cxx @@ -56,17 +56,19 @@ int _main(int argc, char **argv) { if (reply.is_ok()) { const Sample &sample = reply.get_ok(); std::cout << "Received ('" << sample.get_keyexpr().as_string_view() << "' : '" - << sample.get_payload().deserialize() << "')\n"; + << sample.get_payload().as_string() << "')\n"; +#if defined(Z_FEATURE_UNSTABLE_API) auto attachment = sample.get_attachment(); if (!attachment.has_value()) return; // we expect attachment in the form of key-value pairs auto attachment_deserialized = - attachment->get().deserialize>(); + ext::deserialize>(attachment->get()); for (auto &&[key, value] : attachment_deserialized) { std::cout << " attachment: " << key << ": '" << value << "'\n"; } +#endif } else { - std::cout << "Received an error :" << reply.get_err().get_payload().deserialize() << "\n"; + std::cout << "Received an error :" << reply.get_err().get_payload().as_string() << "\n"; } }; @@ -78,17 +80,13 @@ int _main(int argc, char **argv) { std::unordered_map attachment = {{"Source", "C++"}}; -#if __cpp_designated_initializers >= 201707L - session.get( - keyexpr, "", on_reply, on_done, - {.target = Z_QUERY_TARGET_ALL, .payload = Bytes::serialize(value), .attachment = Bytes::serialize(attachment)}); -#else Session::GetOptions options; options.target = QueryTarget::Z_QUERY_TARGET_ALL; - options.payload = Bytes::serialize(value); - options.attachment = Bytes::serialize(attachment); - session.get(keyexpr, "", on_reply, on_done, std::move(options)); + options.payload = value; +#if defined(Z_FEATURE_UNSTABLE_API) + options.attachment = ext::serialize(attachment); #endif + session.get(keyexpr, "", on_reply, on_done, std::move(options)); std::unique_lock lock(m); done_signal.wait(lock, [&done] { return done; }); diff --git a/examples/universal/z_get_channel.cxx b/examples/universal/z_get_channel.cxx index 33a50d89..a80ea6f1 100644 --- a/examples/universal/z_get_channel.cxx +++ b/examples/universal/z_get_channel.cxx @@ -62,20 +62,19 @@ int _main(int argc, char **argv) { std::cout << "Sending Query '" << expr << "'...\n"; #if __cpp_designated_initializers >= 201707L - auto replies = - session.get(keyexpr, "", channels::FifoChannel(16), - {.target = QueryTarget::Z_QUERY_TARGET_ALL, .payload = Bytes::serialize("Get from C++")}); + auto replies = session.get(keyexpr, "", channels::FifoChannel(16), + {.target = QueryTarget::Z_QUERY_TARGET_ALL, .payload = Bytes("Get from C++")}); #else Session::GetOptions options; options.target = QueryTarget::Z_QUERY_TARGET_ALL; - options.payload = Bytes::serialize("Get from C++"); + options.payload = "Get from C++"; auto replies = session.get(keyexpr, "", channels::FifoChannel(16), std::move(options)); #endif for (auto res = replies.recv(); std::holds_alternative(res); res = replies.recv()) { const auto &sample = std::get(res).get_ok(); std::cout << "Received ('" << sample.get_keyexpr().as_string_view() << "' : '" - << sample.get_payload().deserialize() << "')\n"; + << sample.get_payload().as_string() << "')\n"; } return 0; diff --git a/examples/universal/z_get_channel_non_blocking.cxx b/examples/universal/z_get_channel_non_blocking.cxx index d34914c2..2fc8bcfe 100644 --- a/examples/universal/z_get_channel_non_blocking.cxx +++ b/examples/universal/z_get_channel_non_blocking.cxx @@ -65,13 +65,12 @@ int _main(int argc, char **argv) { std::cout << "Sending Query '" << expr << "'...\n"; #if __cpp_designated_initializers >= 201707L - auto replies = - session.get(keyexpr, "", channels::FifoChannel(16), - {.target = QueryTarget::Z_QUERY_TARGET_ALL, .payload = Bytes::serialize("Get from C++")}); + auto replies = session.get(keyexpr, "", channels::FifoChannel(16), + {.target = QueryTarget::Z_QUERY_TARGET_ALL, .payload = "Get from C++"}); #else Session::GetOptions options; options.target = QueryTarget::Z_QUERY_TARGET_ALL; - options.payload = Bytes::serialize("Get from C++"); + options.payload = "Get from C++"; auto replies = session.get(keyexpr, "", channels::FifoChannel(16), std::move(options)); #endif @@ -88,7 +87,7 @@ int _main(int argc, char **argv) { } const auto &sample = std::get(res).get_ok(); std::cout << "Received ('" << sample.get_keyexpr().as_string_view() << "' : '" - << sample.get_payload().deserialize() << "')\n"; + << sample.get_payload().as_string() << "')\n"; } std::cout << std::endl; diff --git a/examples/universal/z_pub.cxx b/examples/universal/z_pub.cxx index db911910..fab190a4 100644 --- a/examples/universal/z_pub.cxx +++ b/examples/universal/z_pub.cxx @@ -86,14 +86,14 @@ int _main(int argc, char **argv) { std::this_thread::sleep_for(1s); std::ostringstream ss; ss << "[" << idx << "] " << value; - auto s = ss.str(); // in C++20 use .view() instead + auto s = ss.str(); std::cout << "Putting Data ('" << keyexpr << "': '" << s << "')...\n"; #if __cpp_designated_initializers >= 201707L - pub.put(Bytes::serialize(s), {.encoding = Encoding("text/plain")}); + pub.put(s, {.encoding = Encoding("text/plain")}); #else auto put_options = Publisher::PutOptions{}; put_options.encoding = Encoding("text/plain"); - pub.put(Bytes::serialize(s), std::move(put_options)); + pub.put(s, std::move(put_options)); #endif } return 0; diff --git a/examples/universal/z_pub_attachment.cxx b/examples/universal/z_pub_attachment.cxx index 1fe2f8ef..458c55d7 100644 --- a/examples/universal/z_pub_attachment.cxx +++ b/examples/universal/z_pub_attachment.cxx @@ -63,9 +63,6 @@ int _main(int argc, char **argv) { std::cout << "Publisher on '" << keyexpr << "' declared" << std::endl; - // allocate attachment map - std::unordered_map attachment_map = {{"source", "C++"}}; - std::cout << "Press CTRL-C to quit..." << std::endl; for (int idx = 0; idx < std::numeric_limits::max(); ++idx) { std::this_thread::sleep_for(1s); @@ -73,17 +70,15 @@ int _main(int argc, char **argv) { ss << "[" << idx << "] " << value; auto s = ss.str(); // in C++20 use .view() instead std::cout << "Putting Data ('" << keyexpr << "': '" << s << "')...\n"; - // add some other attachment value - attachment_map["index"] = std::to_string(idx); -#if __cpp_designated_initializers >= 201707L - pub.put(Bytes::serialize(s), - {.encoding = Encoding("text/plain"), .attachment = Bytes::serialize(attachment_map)}); -#else Publisher::PutOptions options; options.encoding = Encoding("text/plain"); - options.attachment = Bytes::serialize(attachment_map); - pub.put(Bytes::serialize(s), std::move(options)); +#if defined(Z_FEATURE_UNSTABLE_API) + // allocate attachment map + std::unordered_map attachment_map = {{"source", "C++"}}; + attachment_map["index"] = std::to_string(idx); + options.attachment = ext::serialize(attachment_map); #endif + pub.put(s, std::move(options)); } return 0; } diff --git a/examples/universal/z_put.cxx b/examples/universal/z_put.cxx index b097cbb7..c6676433 100644 --- a/examples/universal/z_put.cxx +++ b/examples/universal/z_put.cxx @@ -71,18 +71,14 @@ int _main(int argc, char **argv) { std::cout << "Putting Data (" << "'" << keyexpr << "': '" << value << "')...\n"; + Session::PutOptions put_options; + put_options.encoding = Encoding("text/plain"); +#if defined(Z_FEATURE_UNSTABLE_API) std::unordered_map attachment_map = {{"serial_number", "123"}, {"coordinates", "48.7082,2.1498"}}; -#if __cpp_designated_initializers >= 201707L - session.put(KeyExpr(keyexpr), Bytes::serialize(value), - {.encoding = Encoding("text/plain"), .attachment = Bytes::serialize(attachment_map)}); -#else - auto put_options = Session::PutOptions::create_default(); - put_options.encoding = Encoding("text/plain"); - put_options.attachment = Bytes::serialize(attachment_map); - session.put(KeyExpr(keyexpr), Bytes::serialize(value), std::move(put_options)); + put_options.attachment = ext::serialize(attachment_map); #endif - + session.put(KeyExpr(keyexpr), value, std::move(put_options)); return 0; } diff --git a/examples/universal/z_queryable.cxx b/examples/universal/z_queryable.cxx index 6591301a..08ceceae 100644 --- a/examples/universal/z_queryable.cxx +++ b/examples/universal/z_queryable.cxx @@ -80,15 +80,15 @@ int _main(int argc, char **argv) { std::cout << ">> [Queryable ] Received Query '" << keyexpr.as_string_view() << "?" << params; auto payload = query.get_payload(); if (payload.has_value()) { - std::cout << "' value = '" << payload->get().deserialize(); + std::cout << "' value = '" << payload->get().as_string(); } std::cout << "'\n"; #if __cpp_designated_initializers >= 201707L - query.reply(KeyExpr(expr), Bytes::serialize(value), {.encoding = Encoding("text/plain")}); + query.reply(KeyExpr(expr), value, {.encoding = Encoding("text/plain")}); #else Query::ReplyOptions reply_options; reply_options.encoding = Encoding("text/plain"); - query.reply(KeyExpr(expr), Bytes::serialize(value), std::move(reply_options)); + query.reply(KeyExpr(expr), value, std::move(reply_options)); #endif }; diff --git a/examples/universal/z_queryable_attachment.cxx b/examples/universal/z_queryable_attachment.cxx index e5680c2a..147604f3 100644 --- a/examples/universal/z_queryable_attachment.cxx +++ b/examples/universal/z_queryable_attachment.cxx @@ -63,32 +63,29 @@ int _main(int argc, char **argv) { auto payload = query.get_payload(); std::cout << ">> [Queryable ] Received Query '" << keyexpr.as_string_view() << "?" << params; if (payload.has_value()) { - std::cout << "' value = '" << payload->get().deserialize(); + std::cout << "' value = '" << payload->get().as_string(); } std::cout << "'\n"; - +#if defined(Z_FEATURE_UNSTABLE_API) std::unordered_map attachment_map; auto attachment = query.get_attachment(); if (attachment.has_value()) { // read attachment as a key-value map - attachment_map = attachment->get().deserialize>(); + attachment_map = ext::deserialize>(attachment->get()); for (auto &&[key, value] : attachment_map) { std::cout << " attachment: " << key << ": '" << value << "'\n"; } } -#if __cpp_designated_initializers >= 201707L - query.reply(KeyExpr(expr), Bytes::serialize(value), - {.encoding = Encoding("text/palin"), .attachment = Bytes::serialize(attachment_map)}); -#else +#endif Query::ReplyOptions options; options.encoding = Encoding("text/plain"); - options.attachment = Bytes::serialize(attachment_map); - query.reply(KeyExpr(expr), Bytes::serialize(value), std::move(options)); +#if defined(Z_FEATURE_UNSTABLE_API) + options.attachment = ext::serialize(attachment_map); #endif + query.reply(KeyExpr(expr), value, std::move(options)); }; auto on_drop_queryable = []() { std::cout << "Destroying queryable\n"; }; - auto queryable = session.declare_queryable(keyexpr, on_query, on_drop_queryable); printf("Press CTRL-C to quit...\n"); diff --git a/examples/universal/z_sub.cxx b/examples/universal/z_sub.cxx index 82fa0036..9f30744c 100644 --- a/examples/universal/z_sub.cxx +++ b/examples/universal/z_sub.cxx @@ -36,8 +36,7 @@ const char *kind_to_str(SampleKind kind) { void data_handler(const Sample &sample) { std::cout << ">> [Subscriber] Received " << kind_to_str(sample.get_kind()) << " ('" - << sample.get_keyexpr().as_string_view() << "' : '" << sample.get_payload().deserialize() - << "')\n"; + << sample.get_keyexpr().as_string_view() << "' : '" << sample.get_payload().as_string() << "')\n"; } int _main(int argc, char **argv) { diff --git a/examples/universal/z_sub_attachment.cxx b/examples/universal/z_sub_attachment.cxx index 4a6f6cdd..5b897a43 100644 --- a/examples/universal/z_sub_attachment.cxx +++ b/examples/universal/z_sub_attachment.cxx @@ -59,15 +59,16 @@ int _main(int argc, char **argv) { auto data_handler = [](const Sample &sample) { std::cout << ">> [Subscriber] Received " << kind_to_str(sample.get_kind()) << " ('" - << sample.get_keyexpr().as_string_view() << "' : '" << sample.get_payload().deserialize() - << "')\n"; + << sample.get_keyexpr().as_string_view() << "' : '" << sample.get_payload().as_string() << "')\n"; +#if defined(Z_FEATURE_UNSTABLE_API) auto attachment = sample.get_attachment(); if (!attachment.has_value()) return; // we expect attachment in the form of key-value pairs - auto attachment_data = attachment->get().deserialize>(); + auto attachment_data = ext::deserialize>(attachment->get()); for (auto &&[key, value] : attachment_data) { std::cout << " attachment: " << key << ": '" << value << "'\n"; } +#endif }; std::cout << "Declaring Subscriber on '" << keyexpr.as_string_view() << "'..." << std::endl; diff --git a/examples/zenohc/z_get_shm.cxx b/examples/zenohc/z_get_shm.cxx index 4fcb1055..7ce236f3 100644 --- a/examples/zenohc/z_get_shm.cxx +++ b/examples/zenohc/z_get_shm.cxx @@ -61,9 +61,9 @@ int _main(int argc, char **argv) { if (reply.is_ok()) { const auto &sample = reply.get_ok(); std::cout << "Received ('" << sample.get_keyexpr().as_string_view() << "' : '" - << sample.get_payload().deserialize() << "')\n"; + << sample.get_payload().as_string() << "')\n"; } else { - std::cout << "Received an error :" << reply.get_err().get_payload().deserialize() << "\n"; + std::cout << "Received an error :" << reply.get_err().get_payload().as_string() << "\n"; } }; @@ -89,7 +89,7 @@ int _main(int argc, char **argv) { #else Session::GetOptions options; options.target = Z_QUERY_TARGET_ALL; - options.payload = Bytes::serialize(std::move(buf)); + options.payload = std::move(buf); session.get(keyexpr, "", on_reply, on_done, std::move(options)); #endif diff --git a/examples/zenohc/z_ping_shm.cxx b/examples/zenohc/z_ping_shm.cxx index 755f9c6a..b35e965e 100644 --- a/examples/zenohc/z_ping_shm.cxx +++ b/examples/zenohc/z_ping_shm.cxx @@ -73,13 +73,13 @@ int _main(int argc, char** argv) { if (args.warmup_ms) { auto end = std::chrono::steady_clock::now() + (1ms * args.warmup_ms); while (std::chrono::steady_clock::now() < end) { - pub.put(Bytes::serialize(ZShm(buf))); + pub.put(ZShm(buf)); condvar.wait_for(lock, 1s); } } for (unsigned int i = 0; i < args.number_of_pings; i++) { auto start = std::chrono::steady_clock::now(); - pub.put(Bytes::serialize(ZShm(buf))); + pub.put(ZShm(buf)); if (condvar.wait_for(lock, 1s) == std::cv_status::timeout) { std::cout << "TIMEOUT seq=" << i << "\n"; continue; diff --git a/examples/zenohc/z_pub_shm.cxx b/examples/zenohc/z_pub_shm.cxx index dea79919..38bbbf78 100644 --- a/examples/zenohc/z_pub_shm.cxx +++ b/examples/zenohc/z_pub_shm.cxx @@ -81,11 +81,11 @@ int _main(int argc, char **argv) { memcpy(buf.data(), s.data(), len); #if __cpp_designated_initializers >= 201707L - pub.put(Bytes::serialize(std::move(buf)), {.encoding = Encoding("text/plain")}); + pub.put(std::move(buf), {.encoding = Encoding("text/plain")}); #else Publisher::PutOptions options; options.encoding = Encoding("text/plain"); - pub.put(Bytes::serialize(std::move(buf)), std::move(options)); + pub.put(std::move(buf), std::move(options)); #endif } return 0; diff --git a/examples/zenohc/z_pub_shm_thr.cxx b/examples/zenohc/z_pub_shm_thr.cxx index 81e9efeb..5843cb14 100644 --- a/examples/zenohc/z_pub_shm_thr.cxx +++ b/examples/zenohc/z_pub_shm_thr.cxx @@ -69,7 +69,7 @@ int _main(int argc, char **argv) { ZShm buf(std::move(buf_mut)); printf("Press CTRL-C to quit...\n"); - while (1) pub.put(Bytes::serialize(ZShm(buf))); + while (1) pub.put(ZShm(buf)); } int main(int argc, char **argv) { diff --git a/examples/zenohc/z_queryable_shm.cxx b/examples/zenohc/z_queryable_shm.cxx index 8a0d59f6..dc6cf135 100644 --- a/examples/zenohc/z_queryable_shm.cxx +++ b/examples/zenohc/z_queryable_shm.cxx @@ -66,7 +66,7 @@ int _main(int argc, char **argv) { const char *payload_type = ""; if (payload.has_value()) { ZResult result; - payload->get().deserialize(&result); + payload->get().as_shm(&result); if (result == Z_OK) { payload_type = "SHM"; } else { @@ -79,7 +79,7 @@ int _main(int argc, char **argv) { std::cout << ">> [Queryable ] Received Query [" << payload_type << "] '" << keyexpr.as_string_view() << "?" << params; if (payload.has_value()) { - std::cout << "' value = '" << payload->get().deserialize(); + std::cout << "' value = '" << payload->get().as_string(); } std::cout << "'\n"; @@ -89,11 +89,11 @@ int _main(int argc, char **argv) { memcpy(buf.data(), value, len); #if __cpp_designated_initializers >= 201707L - query.reply(KeyExpr(expr), Bytes::serialize(std::move(buf)), {.encoding = Encoding("text/plain")}); + query.reply(KeyExpr(expr), std::move(buf), {.encoding = Encoding("text/plain")}); #else Query::ReplyOptions options; options.encoding = Encoding("text/plain"); - query.reply(KeyExpr(expr), Bytes::serialize(std::move(buf)), std::move(options)); + query.reply(KeyExpr(expr), std::move(buf), std::move(options)); #endif }; diff --git a/examples/zenohc/z_sub_shm.cxx b/examples/zenohc/z_sub_shm.cxx index 65c03c2f..572fc7f1 100644 --- a/examples/zenohc/z_sub_shm.cxx +++ b/examples/zenohc/z_sub_shm.cxx @@ -51,7 +51,7 @@ void data_handler(const Sample &sample) { const char *payload_type = "RAW"; { ZResult result; - sample.get_payload().deserialize(&result); + sample.get_payload().as_shm(&result); if (result == Z_OK) { payload_type = "SHM"; } @@ -59,8 +59,7 @@ void data_handler(const Sample &sample) { #endif std::cout << ">> [Subscriber] Received [" << payload_type << "] " << kind_to_str(sample.get_kind()) << " ('" - << sample.get_keyexpr().as_string_view() << "' : '" << sample.get_payload().deserialize() - << "')\n"; + << sample.get_keyexpr().as_string_view() << "' : '" << sample.get_payload().as_string() << "')\n"; } int _main(int argc, char **argv) { From ddaf4bd391834bb881aefee0e4bcf65a0ffc261a Mon Sep 17 00:00:00 2001 From: Denis Biryukov Date: Mon, 30 Sep 2024 22:39:11 +0200 Subject: [PATCH 4/8] remove serialize/deserialize_sequence_end --- include/zenoh/api/ext/serialization.hxx | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/include/zenoh/api/ext/serialization.hxx b/include/zenoh/api/ext/serialization.hxx index ac83d495..2212bb0e 100644 --- a/include/zenoh/api/ext/serialization.hxx +++ b/include/zenoh/api/ext/serialization.hxx @@ -155,11 +155,10 @@ void __zenoh_serialize_with_serializer(zenoh::ext::Serializer& serializer, const template void __serialize_sequence_with_serializer(zenoh::ext::Serializer& serializer, It begin, It end, size_t n) { - ::ze_serializer_serialize_sequence_begin(zenoh::interop::as_loaned_c_ptr(serializer), n); + ::ze_serializer_serialize_sequence_length(zenoh::interop::as_loaned_c_ptr(serializer), n); for (auto it = begin; it != end; ++it) { serialize_with_serializer(serializer, *it); } - ::ze_serializer_serialize_sequence_end(zenoh::interop::as_loaned_c_ptr(serializer)); } template @@ -257,14 +256,11 @@ bool __zenoh_deserialize_with_deserializer(zenoh::ext::Deserializer& deserialize #define _ZENOH_DESERIALIZE_SEQUENCE_BEGIN \ size_t len; \ __ZENOH_RESULT_CHECK( \ - ::ze_deserializer_deserialize_sequence_begin(zenoh::interop::as_copyable_c_ptr(deserializer), &len), err, \ + ::ze_deserializer_deserialize_sequence_length(zenoh::interop::as_copyable_c_ptr(deserializer), &len), err, \ "Deserialization failure:: Failed to read sequence length"); \ if (err != nullptr && *err != Z_OK) return false; -#define _ZENOH_DESERIALIZE_SEQUENCE_END \ - __ZENOH_RESULT_CHECK(::ze_deserializer_deserialize_sequence_end(zenoh::interop::as_copyable_c_ptr(deserializer)), \ - err, "Deserialization failure:: Failed to finalize sequence read"); \ - return (err == nullptr || *err == Z_OK); +#define _ZENOH_DESERIALIZE_SEQUENCE_END return (err == nullptr || *err == Z_OK); template bool __zenoh_deserialize_with_deserializer(zenoh::ext::Deserializer& deserializer, std::vector& value, From 390eff85e339814beafc3e817243dbe6491ec3dc Mon Sep 17 00:00:00 2001 From: Denis Biryukov Date: Tue, 1 Oct 2024 11:39:17 +0200 Subject: [PATCH 5/8] bump zenoh-c/pico branches --- zenoh-c | 2 +- zenoh-pico | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/zenoh-c b/zenoh-c index f08b631a..dcec5a53 160000 --- a/zenoh-c +++ b/zenoh-c @@ -1 +1 @@ -Subproject commit f08b631ad486204d81b7389f04cd02616158b99e +Subproject commit dcec5a533bec25a2d32f8e32dc5f229591e38b0b diff --git a/zenoh-pico b/zenoh-pico index c1de135d..1c73e0c8 160000 --- a/zenoh-pico +++ b/zenoh-pico @@ -1 +1 @@ -Subproject commit c1de135de6c7766dd1ea5a96ecce6f0bbac158da +Subproject commit 1c73e0c8aa5ba9a78f0309a9f5bb7fe5e9bfcb16 From 699e2e29a7f54d5f5ee6c4f2048629e32646f6dc Mon Sep 17 00:00:00 2001 From: Denis Biryukov Date: Tue, 1 Oct 2024 11:39:56 +0200 Subject: [PATCH 6/8] format --- include/zenoh/api/ext/serialization.hxx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/zenoh/api/ext/serialization.hxx b/include/zenoh/api/ext/serialization.hxx index 2212bb0e..0b4954e7 100644 --- a/include/zenoh/api/ext/serialization.hxx +++ b/include/zenoh/api/ext/serialization.hxx @@ -253,11 +253,11 @@ bool __zenoh_deserialize_with_deserializer(zenoh::ext::Deserializer& deserialize deserialize_with_deserializer(deserializer, value.second, err); } -#define _ZENOH_DESERIALIZE_SEQUENCE_BEGIN \ - size_t len; \ - __ZENOH_RESULT_CHECK( \ +#define _ZENOH_DESERIALIZE_SEQUENCE_BEGIN \ + size_t len; \ + __ZENOH_RESULT_CHECK( \ ::ze_deserializer_deserialize_sequence_length(zenoh::interop::as_copyable_c_ptr(deserializer), &len), err, \ - "Deserialization failure:: Failed to read sequence length"); \ + "Deserialization failure:: Failed to read sequence length"); \ if (err != nullptr && *err != Z_OK) return false; #define _ZENOH_DESERIALIZE_SEQUENCE_END return (err == nullptr || *err == Z_OK); From c981e4e3e729614772a65ed4113db3ac456a2e4b Mon Sep 17 00:00:00 2001 From: Denis Biryukov Date: Tue, 1 Oct 2024 11:56:15 +0200 Subject: [PATCH 7/8] docs update --- docs/pubsub.rst | 2 +- docs/queryable.rst | 4 ++-- docs/serialization_deserialization.rst | 13 ++++++++++++- docs/session_ex.rst | 2 +- include/zenoh/api/ext/serialization.hxx | 2 +- 5 files changed, 17 insertions(+), 6 deletions(-) diff --git a/docs/pubsub.rst b/docs/pubsub.rst index 30a28d73..0aa6a1db 100644 --- a/docs/pubsub.rst +++ b/docs/pubsub.rst @@ -33,7 +33,7 @@ Publisher example: // Publish from a Publisher object auto publisher = session.declare_publisher(KeyExpr("demo/example/simple")); - publisher.put(Bytes::serialize("Simple!")); + publisher.put("Simple!"); } Subscriber example: diff --git a/docs/queryable.rst b/docs/queryable.rst index 5f5e34fa..9914e6de 100644 --- a/docs/queryable.rst +++ b/docs/queryable.rst @@ -102,11 +102,11 @@ Also notice that the callback is processed asynchronously, so the client must no if (reply->is_ok()) { const Sample& sample = reply->get_ok(); std::cout << "Received ('" << sample.get_keyexpr().as_string_view() << "' : '" - << sample.get_payload().deserialize() << "')\n"; + << sample.get_payload().as_string() << "')\n"; } else { const ReplyError& error = reply->get_err(); std::cout << "Received an error :" - << error.get_payload().deserialzie() << "\n"; + << error.get_payload().as_string() << "\n"; } } diff --git a/docs/serialization_deserialization.rst b/docs/serialization_deserialization.rst index a2169c36..feb97f7a 100644 --- a/docs/serialization_deserialization.rst +++ b/docs/serialization_deserialization.rst @@ -16,4 +16,15 @@ Serialization/Deserialziation ============================= .. doxygenclass:: zenoh::Bytes :members: - :membergroups: Constructors Operators Methods \ No newline at end of file + :membergroups: Constructors Operators Methods + +.. doxygenclass:: zenoh::ext::Serializer + :members: + :membergroups: Constructors Operators Methods + +.. doxygenclass:: zenoh::ext::Deserializer + :members: + :membergroups: Constructors Operators Methods + +.. doxygenfunction:: zenoh::ext::serialize +.. doxygenfunction:: zenoh::ext::deserialize diff --git a/docs/session_ex.rst b/docs/session_ex.rst index 5830fae9..e4182ed0 100644 --- a/docs/session_ex.rst +++ b/docs/session_ex.rst @@ -28,7 +28,7 @@ Then a string is published on "demo/example/simple" key expression. try { Config config = Config::create_default(); auto session = Session::open(std::move(config)); - session.put(KeyExpr("demo/example/simple"), Bytes::serialize("Simple!")); + session.put(KeyExpr("demo/example/simple"), "Simple!"); } catch (ZException e) { std::cout << "Received an error :" << e.what() << "\n"; } diff --git a/include/zenoh/api/ext/serialization.hxx b/include/zenoh/api/ext/serialization.hxx index 0b4954e7..4f2e0dc0 100644 --- a/include/zenoh/api/ext/serialization.hxx +++ b/include/zenoh/api/ext/serialization.hxx @@ -85,7 +85,7 @@ class Deserializer : public Copyable<::ze_deserializer_t> { }; /// @warning This API has been marked as unstable: it works as advertised, but it may be changed in a future release. -/// @brief Serialize a single into `Bytes`. +/// @brief Serialize a single value into `Bytes`. template zenoh::Bytes serialize(const T& value) { Serializer s; From 71c17bb7ebfbc0b168f4a09a591e2bd0ebb6fa8e Mon Sep 17 00:00:00 2001 From: Denis Biryukov Date: Tue, 1 Oct 2024 12:12:54 +0200 Subject: [PATCH 8/8] docs update --- include/zenoh/api/bytes.hxx | 3 ++- include/zenoh/api/ext/serialization.hxx | 25 ++++++++++++++++++------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/include/zenoh/api/bytes.hxx b/include/zenoh/api/bytes.hxx index 01e1c7c1..6feeefae 100644 --- a/include/zenoh/api/bytes.hxx +++ b/include/zenoh/api/bytes.hxx @@ -188,7 +188,8 @@ class Bytes : public Owned<::z_owned_bytes_t> { int64_t tell() { return ::z_bytes_reader_tell(&this->_0); } /// @brief Return the number of bytes that can still be read. - size_t remaining() { return ::z_bytes_reader_remaining(&this->_0); } + /// @return Number of bytes that can still be read. + size_t remaining() const { return ::z_bytes_reader_remaining(&this->_0); } /// @brief Set the `reader` position indicator to the value pointed to by offset, starting from the current /// position. diff --git a/include/zenoh/api/ext/serialization.hxx b/include/zenoh/api/ext/serialization.hxx index 4f2e0dc0..a0684a65 100644 --- a/include/zenoh/api/ext/serialization.hxx +++ b/include/zenoh/api/ext/serialization.hxx @@ -43,16 +43,18 @@ class Serializer : public Owned<::ze_owned_serializer_t> { public: /// @name Constructors - /// Constructs an empty writer. + /// Constructs an empty serializer. Serializer() : Owned(nullptr) { ::ze_serializer_empty(interop::as_owned_c_ptr(*this)); } /// @name Methods - /// @brief Serialize specified value. + + /// @brief Serialize specified value and append it to the underlying `Bytes`. + /// @param value value to serialize. template void serialize(const T& value); - /// @brief Finalize serialization and return underlying `Bytes` object. - /// @return Underlying `Bytes` object. + /// @brief Finalize serialization and return the underlying `Bytes` object. + /// @return underlying `Bytes` object. Bytes finish() && { Bytes b; ::ze_serializer_finish(interop::as_moved_c_ptr(*this), interop::as_owned_c_ptr(b)); @@ -70,22 +72,27 @@ class Deserializer : public Copyable<::ze_deserializer_t> { /// @name Constructors /// @brief Construct deserializer for the specified data. - /// @param b Data to initialize deserializer with. + /// @param b data to initialize deserializer with. Deserializer(const Bytes& b) : Copyable(::ze_deserializer_from_bytes(zenoh::interop::as_loaned_c_ptr(b))) {} /// @name Methods - /// @brief Deserialize into value of specified type. + /// @brief Deserialize next portion of data into specified type. + /// @param err if not null, the result code will be written to this location, otherwise ZException exception + /// will be thrown in case of error. + /// @return deserialized value. template T deserialize(zenoh::ZResult* err = nullptr); /// @brief Checks if deserializer has parsed all the data. /// @return `true` if there is no more data to parse, `false` otherwise. - bool is_done() { return ::ze_deserializer_is_done(&this->_0); } + bool is_done() const { return ::ze_deserializer_is_done(&this->_0); } }; /// @warning This API has been marked as unstable: it works as advertised, but it may be changed in a future release. /// @brief Serialize a single value into `Bytes`. +/// @param value value to serialize. +/// @return 'Bytes' containing serialized value. template zenoh::Bytes serialize(const T& value) { Serializer s; @@ -95,6 +102,10 @@ zenoh::Bytes serialize(const T& value) { /// @warning This API has been marked as unstable: it works as advertised, but it may be changed in a future release. /// @brief Deserialize `Bytes` corresponding to a single serialized value. +/// @param bytes data to deserialize. +/// @param err if not null, the result code will be written to this location, otherwise ZException exception +/// will be thrown in case of error. +/// @return deserialized value. template T deserialize(const zenoh::Bytes& bytes, zenoh::ZResult* err = nullptr) { Deserializer d(bytes);