diff --git a/tests/utils_test.cpp b/tests/utils_test.cpp index 3121074..75ab381 100644 --- a/tests/utils_test.cpp +++ b/tests/utils_test.cpp @@ -33,28 +33,26 @@ www.navitia.io #include #include #include +#include #include "utils/flat_enum_map.h" #include "utils/logger.h" #include "utils/init.h" #include "utils/base64_encode.h" #include "utils/functions.h" +#include "utils/threadbuf.h" +#include "utils/init.h" struct logger_initialized { - logger_initialized() { navitia::init_logger(); } + logger_initialized() { navitia::init_logger(); } }; -BOOST_GLOBAL_FIXTURE( logger_initialized ); +BOOST_GLOBAL_FIXTURE(logger_initialized); -enum class Mode { - bike = 0, - walk, - car, - size -}; +enum class Mode { bike = 0, walk, car, size }; /** - * simple test for the enum map - * - **/ + * simple test for the enum map + * + **/ BOOST_AUTO_TEST_CASE(flatEnumMap_simple_test) { navitia::flat_enum_map map; @@ -62,26 +60,20 @@ BOOST_AUTO_TEST_CASE(flatEnumMap_simple_test) { BOOST_CHECK_EQUAL(map[Mode::bike], 2); - //default initialization - //due to a gcc bug http://gcc.gnu.org/bugzilla/show_bug.cgi?id=57086 the container cannont be + // default initialization + // due to a gcc bug http://gcc.gnu.org/bugzilla/show_bug.cgi?id=57086 the container cannont be // default initialized any longer cf comment in flat_enum_map -// BOOST_CHECK_EQUAL(map[Mode::car], 0); + // BOOST_CHECK_EQUAL(map[Mode::car], 0); } -enum class RawEnum { - first = 0, - second, - last -}; +enum class RawEnum { first = 0, second, last }; namespace navitia { template <> struct enum_size_trait { - static constexpr typename get_enum_type::type size() { - return 3; - } + static constexpr typename get_enum_type::type size() { return 3; } }; -} +} // namespace navitia static std::ostream& operator<<(std::ostream& o, RawEnum e) { return o << static_cast(e); @@ -104,16 +96,16 @@ BOOST_AUTO_TEST_CASE(enum_iterator) { BOOST_CHECK_EQUAL(*++it, RawEnum::second); BOOST_CHECK(it != navitia::enum_iterator()); BOOST_CHECK_EQUAL(*++it, RawEnum::last); - BOOST_CHECK(++it == navitia::enum_iterator()); //invalid iterator + BOOST_CHECK(++it == navitia::enum_iterator()); // invalid iterator } BOOST_AUTO_TEST_CASE(enum_range) { std::vector res; - for (auto e: navitia::enum_range()) { + for (auto e : navitia::enum_range()) { res.push_back(e); } - std::vector wanted_res {RawEnum::first, RawEnum::second, RawEnum::last}; + std::vector wanted_res{RawEnum::first, RawEnum::second, RawEnum::last}; BOOST_CHECK_EQUAL_COLLECTIONS(std::begin(res), std::end(res), std::begin(wanted_res), std::end(wanted_res)); } @@ -121,10 +113,10 @@ BOOST_AUTO_TEST_CASE(enum_range) { BOOST_AUTO_TEST_CASE(enum_reverse_range) { std::vector res; - for (auto e: navitia::reverse_enum_range()) { + for (auto e : navitia::reverse_enum_range()) { res.push_back(e); } - std::vector wanted_res {Mode::car, Mode::walk, Mode::bike}; + std::vector wanted_res{Mode::car, Mode::walk, Mode::bike}; BOOST_CHECK_EQUAL_COLLECTIONS(std::begin(res), std::end(res), std::begin(wanted_res), std::end(wanted_res)); } @@ -132,17 +124,17 @@ BOOST_AUTO_TEST_CASE(enum_reverse_range) { BOOST_AUTO_TEST_CASE(enum_reverse_range_from) { std::vector res; - for (auto e: navitia::reverse_enum_range_from(Mode::walk)) { + for (auto e : navitia::reverse_enum_range_from(Mode::walk)) { res.push_back(e); } - std::vector wanted_res {Mode::walk, Mode::bike}; + std::vector wanted_res{Mode::walk, Mode::bike}; BOOST_CHECK_EQUAL_COLLECTIONS(std::begin(res), std::end(res), std::begin(wanted_res), std::end(wanted_res)); } /** - * test with an enum without a size last field - * - **/ + * test with an enum without a size last field + * + **/ BOOST_AUTO_TEST_CASE(flatEnumMap_no_size_test) { navitia::flat_enum_map map; @@ -150,18 +142,19 @@ BOOST_AUTO_TEST_CASE(flatEnumMap_no_size_test) { BOOST_CHECK_EQUAL(map[RawEnum::second].val, 42); - //default initialization - //due to a gcc bug http://gcc.gnu.org/bugzilla/show_bug.cgi?id=57086 the container cannont be + // default initialization + // due to a gcc bug http://gcc.gnu.org/bugzilla/show_bug.cgi?id=57086 the container cannont be // default initialized any longer -// BOOST_CHECK_EQUAL(map[RawEnum::first].val, 0); + // BOOST_CHECK_EQUAL(map[RawEnum::first].val, 0); } /** - * test with an initilizer construction - * The test is more that it should compile :) - **/ + * test with an initilizer construction + * The test is more that it should compile :) + **/ BOOST_AUTO_TEST_CASE(flatEnumMap_initializer) { - const navitia::flat_enum_map map = {{{1, 3, 4}}}; //yes 3 curly braces :) one for the flat_enum_map and 2 for the underlying array + const navitia::flat_enum_map map = { + {{1, 3, 4}}}; // yes 3 curly braces :) one for the flat_enum_map and 2 for the underlying array BOOST_CHECK_EQUAL(map[RawEnum::first], 1); BOOST_CHECK_EQUAL(map[RawEnum::second], 3); @@ -169,9 +162,9 @@ BOOST_AUTO_TEST_CASE(flatEnumMap_initializer) { } /** - * basic test for iterator - * - **/ + * basic test for iterator + * + **/ BOOST_AUTO_TEST_CASE(flatEnumMap_iterator_test) { navitia::flat_enum_map map; @@ -179,8 +172,8 @@ BOOST_AUTO_TEST_CASE(flatEnumMap_iterator_test) { map[RawEnum::second] = 42; map[RawEnum::last] = 420; - std::vector expected {4, 42, 420}; - std::vector expectedEnum {RawEnum::first, RawEnum::second, RawEnum::last}; + std::vector expected{4, 42, 420}; + std::vector expectedEnum{RawEnum::first, RawEnum::second, RawEnum::last}; std::vector val; std::vector enumVal; @@ -198,15 +191,7 @@ BOOST_AUTO_TEST_CASE(encode_uri_test) { } BOOST_AUTO_TEST_CASE(natural_sort_test) { - std::vector list { - "toto", - "tutu", - "tutu10", - "tutu2", - "15", - "25", - "5" - }; + std::vector list{"toto", "tutu", "tutu10", "tutu2", "15", "25", "5"}; std::sort(list.begin(), list.end(), navitia::pseudo_natural_sort()); @@ -221,24 +206,8 @@ BOOST_AUTO_TEST_CASE(natural_sort_test) { } BOOST_AUTO_TEST_CASE(natural_sort_test2) { - std::vector list { - "38", - "21", - "1", - "B2", - "B", - "B7", - "Bis", - "a3", - "3", - "2", - "Tram 1", - "B2A", - "A", - "251", - "B11", - "B215A", - "3B", + std::vector list{ + "38", "21", "1", "B2", "B", "B7", "Bis", "a3", "3", "2", "Tram 1", "B2A", "A", "251", "B11", "B215A", "3B", }; std::sort(list.begin(), list.end(), navitia::pseudo_natural_sort()); @@ -259,11 +228,11 @@ BOOST_AUTO_TEST_CASE(natural_sort_test2) { BOOST_CHECK_EQUAL(list[i++], "B215A"); BOOST_CHECK_EQUAL(list[i++], "Bis"); BOOST_CHECK_EQUAL(list[i++], "Tram 1"); - BOOST_CHECK_EQUAL(list[i++], "a3");// case insensitive would be better + BOOST_CHECK_EQUAL(list[i++], "a3"); // case insensitive would be better } struct MockedContainerWithFind { - struct iterator{}; + struct iterator {}; iterator end() const { return {}; } bool mutable find_is_called{false}; iterator find(int) const { @@ -272,8 +241,7 @@ struct MockedContainerWithFind { } }; -inline bool operator!=(const MockedContainerWithFind::iterator&, - const MockedContainerWithFind::iterator&) { +inline bool operator!=(const MockedContainerWithFind::iterator&, const MockedContainerWithFind::iterator&) { return false; } @@ -297,6 +265,63 @@ BOOST_AUTO_TEST_CASE(contains_test) { } BOOST_AUTO_TEST_CASE(math_mod_test) { - BOOST_CHECK_EQUAL(navitia::math_mod(1, 5), 1); - BOOST_CHECK_EQUAL(navitia::math_mod(-1, 5), 4); + BOOST_CHECK_EQUAL(navitia::math_mod(1, 5), 1); + BOOST_CHECK_EQUAL(navitia::math_mod(-1, 5), 4); +} + +/* + * Test the behavior of clone data using boost archive and pipe buffer + * */ +BOOST_AUTO_TEST_CASE(clone_large_buffer_test) { + // clone 2Gb of data + constexpr size_t sting_size = 2 * 1024 * 1000 * 1000; + constexpr size_t LOOP_COUNT = 10; + + for (size_t i(0); i < LOOP_COUNT; ++i) { + std::string from_str(sting_size, 'B'), to_str(""); + to_str.reserve(sting_size); + + CloneHelper cloner; + cloner(from_str, to_str); + + BOOST_CHECK_EQUAL(from_str, to_str); + } +} + +/* + * Test the behavior of clone data using boost archive and pipe buffer + * */ +BOOST_AUTO_TEST_CASE(clone_medium_buffer_test) { + // clone 2 Mb of data + constexpr size_t sting_size = 2 * 1024 * 1000; + constexpr size_t LOOP_COUNT = 5000; + + for (size_t i(0); i < LOOP_COUNT; ++i) { + std::string from_str(sting_size, 'B'), to_str(""); + to_str.reserve(sting_size); + + CloneHelper cloner; + cloner(from_str, to_str); + + BOOST_CHECK_EQUAL(from_str, to_str); + } +} + +/* + * Test the behavior of clone data using boost archive and pipe buffer + * */ +BOOST_AUTO_TEST_CASE(clone_small_buffer_test) { + // clone 2 Kb of data + constexpr size_t sting_size = 2 * 1024; + constexpr size_t LOOP_COUNT = 30000; + + for (size_t i(0); i < LOOP_COUNT; ++i) { + std::string from_str(sting_size, 'B'), to_str(""); + to_str.reserve(sting_size); + + CloneHelper cloner; + cloner(from_str, to_str); + + BOOST_CHECK_EQUAL(from_str, to_str); + } } diff --git a/threadbuf.cpp b/threadbuf.cpp index a2b49b5..309855d 100644 --- a/threadbuf.cpp +++ b/threadbuf.cpp @@ -24,5 +24,3 @@ // ---------------------------------------------------------------------------- #include "threadbuf.h" - -threadbuf::~threadbuf() = default; diff --git a/threadbuf.h b/threadbuf.h index 2f4ecc7..7d0e8c0 100644 --- a/threadbuf.h +++ b/threadbuf.h @@ -26,269 +26,166 @@ #pragma once +#include #include #include #include #include #include +#include -// ---------------------------------------------------------------------------- - -class threadbuf : public std::streambuf { -private: - using traits_type = std::streambuf::traits_type; - using string_size_t = std::string::size_type; - - std::mutex d_mutex; - std::condition_variable d_condition; - std::string d_out; - std::string d_in; - std::string d_tmp; - char* d_current; - bool d_closed; - -public: - threadbuf(string_size_t size = 64 * 1024) - : d_out(std::max(string_size_t(1), size), ' '), - d_in(std::max(string_size_t(1), size), ' '), - d_tmp(std::max(string_size_t(1), size), ' '), - d_current(&this->d_tmp[0]), - d_closed(false) { - this->setp(&this->d_out[0], &this->d_out[0] + this->d_out.size() - 1); - this->setg(&this->d_in[0], &this->d_in[0], &this->d_in[0]); - } - ~threadbuf() override; - void close() { - { - std::unique_lock lock(this->d_mutex); - this->d_closed = true; - while (this->pbase() != this->pptr()) { - this->internal_sync(lock); - } - } - this->d_condition.notify_all(); - } - -private: - int_type underflow() override { - if (this->gptr() == this->egptr()) { - std::unique_lock lock(this->d_mutex); - while (&this->d_tmp[0] == this->d_current && !this->d_closed) { - this->d_condition.wait(lock); - } - if (&this->d_tmp[0] != this->d_current) { - std::streamsize size(this->d_current - &this->d_tmp[0]); - traits_type::copy(this->eback(), &this->d_tmp[0], this->d_current - &this->d_tmp[0]); - this->setg(this->eback(), this->eback(), this->eback() + size); - this->d_current = &this->d_tmp[0]; - this->d_condition.notify_one(); - } - } - return this->gptr() == this->egptr() ? traits_type::eof() : traits_type::to_int_type(*this->gptr()); - } - int_type overflow(int_type c) override { - std::unique_lock lock(this->d_mutex); - if (!traits_type::eq_int_type(c, traits_type::eof())) { - *this->pptr() = traits_type::to_char_type(c); - this->pbump(1); - } - return this->internal_sync(lock) ? traits_type::eof() : traits_type::not_eof(c); - } - int sync() override { - std::unique_lock lock(this->d_mutex); - return this->internal_sync(lock); - } - int internal_sync(std::unique_lock& lock) { - char* end(&this->d_tmp[0] + this->d_tmp.size()); - while (this->d_current == end && !this->d_closed) { - this->d_condition.wait(lock); - } - if (this->d_current != end) { - std::streamsize size(std::min(end - d_current, this->pptr() - this->pbase())); - traits_type::copy(d_current, this->pbase(), size); - this->d_current += size; - std::streamsize remain((this->pptr() - this->pbase()) - size); - traits_type::move(this->pbase(), this->pptr(), remain); - this->setp(this->pbase(), this->epptr()); - this->pbump(remain); - this->d_condition.notify_one(); - return 0; - } - return traits_type::eof(); - } -}; - +#include +#include // ---------------------------------------------------------------------------- // From https://stackoverflow.com/questions/43588275/exceptions-in-stl-streams?noredirect=1&lq=1 // basic_pipebuf is a different implementation of threadbuf // basic_pipebuf resolves a bug (deadlock) found in threadbuf on debian10 platform -template> -class basic_pipebuf : public std::basic_streambuf<_Elem, _Traits> -{ +template > +class streambuf_mtsafe : public std::basic_streambuf<_Elem, _Traits> { public: using traits_type = std::streambuf::traits_type; using buffer_type = typename std::vector<_Elem>; using buffer_size_type = typename buffer_type::size_type; - std::mutex m_mutex; - std::condition_variable m_condition; - bool m_closed; + std::mutex m_mutex; + std::condition_variable m_condition; + bool m_closed; - buffer_type m_buffer; - _Elem* m_begin; - _Elem* m_end; - buffer_size_type m_chunk_size; + buffer_type m_buffer; + _Elem* m_begin; + _Elem* m_end; + buffer_size_type m_chunk_size; public: - basic_pipebuf(buffer_size_type buffer_size = 16 * 1024, buffer_size_type chunk_size = 16): - m_closed(false), - m_buffer(buffer_size), - m_begin(&m_buffer[0]), - m_end(&m_buffer[0] + m_buffer.size()), - m_chunk_size(chunk_size) - { - this->setp(m_begin + m_chunk_size, m_begin + 2 * m_chunk_size); + streambuf_mtsafe(buffer_size_type buffer_size = 64 * 1024, buffer_size_type chunk_size = 16 * 1024) + : m_closed(false), + m_buffer(buffer_size), + m_begin(m_buffer.data()), + m_end(m_buffer.data() + m_buffer.size()), + m_chunk_size(chunk_size) { this->setg(m_begin, m_begin + m_chunk_size, m_begin + m_chunk_size); + this->setp(m_begin + m_chunk_size, m_begin + 2 * m_chunk_size); } - void close(){ + void close() { std::unique_lock lock(m_mutex); m_closed = true; m_condition.notify_all(); } private: - traits_type::int_type overflow(traits_type::int_type c) override - { - std::unique_lock lock(m_mutex); + traits_type::int_type overflow(traits_type::int_type c) override { + traits_type::int_type ret = traits_type::eof(); - if (m_closed) { - return traits_type::eof(); - } - - if (traits_type::eq_int_type(traits_type::eof(), c)) - return traits_type::not_eof(c); + std::unique_lock lock(m_mutex); + if (m_closed) { + return traits_type::eof(); + } - traits_type::int_type ret = traits_type::eof(); + if (traits_type::eq_int_type(traits_type::eof(), c)) + return traits_type::not_eof(c); - if (this->epptr() < m_end) - { - while (this->epptr() == this->eback() && !m_closed) - m_condition.wait(lock); + if (this->epptr() < m_end) { + while (this->epptr() == this->eback() && !m_closed) { + m_condition.wait(lock); + } - if (this->epptr() != this->eback()) - { - this->setp(this->pbase() + m_chunk_size, this->epptr() + m_chunk_size); - ret = this->sputc(c); - } - } - else - { - while (!(this->eback() > m_begin || m_closed)) - m_condition.wait(lock); + if (this->epptr() != this->eback()) { + this->setp(this->pbase() + m_chunk_size, this->epptr() + m_chunk_size); + ret = this->sputc(c); + } + } else { + while (!(this->eback() > m_begin || m_closed)) + m_condition.wait(lock); - if (this->eback() > m_begin) - { - this->setp(m_begin, m_begin + m_chunk_size); - ret = this->sputc(c); + if (this->eback() > m_begin) { + this->setp(m_begin, m_begin + m_chunk_size); + ret = this->sputc(c); + } } } - - m_condition.notify_one(); + m_condition.notify_all(); return ret; } - traits_type::int_type underflow() override - { + traits_type::int_type underflow() override { std::unique_lock lock(m_mutex); traits_type::int_type ret = traits_type::eof(); - if (this->eback() != this->pbase()) - { - if (this->egptr() < m_end) - { - while (this->egptr() == this->pbase() && !m_closed) + if (this->eback() != this->pbase()) { + if (this->egptr() < m_end) { + while (this->egptr() == this->pbase() && !m_closed) { m_condition.wait(lock); + } - if (this->egptr() != this->pbase()) - { - this->setg(this->eback() + m_chunk_size, - this->eback() + m_chunk_size, + if (this->egptr() != this->pbase()) { + this->setg(this->eback() + m_chunk_size, this->eback() + m_chunk_size, this->egptr() + m_chunk_size); - } - else // if m_closed - { + } else { this->setg(this->pbase(), this->pbase(), this->pptr()); } - - ret = traits_type::to_int_type(*this->gptr()); - } - else - { + } else { while (!(this->pbase() > m_begin || m_closed)) m_condition.wait(lock); - if (this->pbase() > m_begin) - { + if (this->pbase() > m_begin) { this->setg(m_begin, m_begin, m_begin + m_chunk_size); - } - else // if m_closed - { + } else if (m_closed) { this->setg(this->pbase(), this->pbase(), this->pptr()); } - - ret = traits_type::to_int_type(*this->gptr()); } + ret = traits_type::to_int_type(*this->gptr()); } - m_condition.notify_one(); + m_condition.notify_all(); return ret; } }; -template> -class basic_opipestream : - public std::basic_ostream<_Elem, _Traits> -{ - using buffer_type = basic_pipebuf<_Elem, _Traits> ; - buffer_type* mPbuf; +using streambuf_mtsafe_char = streambuf_mtsafe; -public: - basic_opipestream(buffer_type* pBuf) : - std::basic_ostream<_Elem, _Traits>(pBuf), - mPbuf(pBuf) - { - } +struct Pipe { + streambuf_mtsafe_char sbuf; + std::ostream out; + std::istream in; - void close() - { - mPbuf->close(); - } + void close() { sbuf.close(); } + Pipe() : out(&sbuf), in(&sbuf) {} + Pipe(const Pipe&) = delete; + Pipe& operator=(const Pipe&) = delete; + Pipe(const Pipe&&) = delete; + Pipe& operator=(const Pipe&&) = delete; + ~Pipe() { close(); } }; -template> -class basic_ipipestream : - public std::basic_istream<_Elem, _Traits> -{ - using buffer_type = basic_pipebuf<_Elem, _Traits>; - buffer_type* mPbuf; - -public: - basic_ipipestream(buffer_type* pBuf) : - std::basic_istream<_Elem, _Traits>(pBuf), - mPbuf(pBuf) - { - } +/* + * CloneHelper is a utility class that allow clone of data not easily copyable + * It uses serialization to create an archive then deserialization to create a clone + * To avoid holding 3 copy at the same time, we use a pipe buffer. + * This pipe is Thread safe, then we can serialize data in one thread, and + * deserialize data in another one. + * NOTE : We use boost::archive::no_codecvt, to avoid data race in the Pipe + * internal buffer (streambuf) -> std::local is shared for read/write in streambuf + * */ +struct CloneHelper { + Pipe pipe; + + template + void operator()(const T& from, T& to) { + std::thread write([&]() { + boost::archive::binary_oarchive oa(pipe.out, boost::archive::no_codecvt); + oa << from; + pipe.close(); + }); - void close() - { - mPbuf->close(); + { + boost::archive::binary_iarchive ia(pipe.in, boost::archive::no_codecvt); + ia >> to; + } + if (write.joinable()) + write.join(); } }; - -using pipebuf = basic_pipebuf; -using opipestream = basic_opipestream; -using ipipestream = basic_ipipestream;