From b30be881197326bdc9e7573ef03485a8187456ec Mon Sep 17 00:00:00 2001 From: DanielaMurin Date: Wed, 21 Aug 2024 18:28:22 +0300 Subject: [PATCH 01/35] Sequencer class with lambda response message Change-Id: Ic684138907605a2d79db9c85b84c425a5c5b0c59 --- syncd/Sequencer.cpp | 42 ++++++++++++++++++ syncd/Sequencer.h | 53 +++++++++++++++++++++++ syncd/Syncd.cpp | 103 ++++++++++++++++++++++++++++++++------------ syncd/Syncd.h | 31 +++++++++++-- 4 files changed, 198 insertions(+), 31 deletions(-) create mode 100755 syncd/Sequencer.cpp create mode 100755 syncd/Sequencer.h diff --git a/syncd/Sequencer.cpp b/syncd/Sequencer.cpp new file mode 100755 index 000000000..c4325ccba --- /dev/null +++ b/syncd/Sequencer.cpp @@ -0,0 +1,42 @@ +#include "Sequencer.h" + +// Helper function to execute all ready responses in order +void Sequencer::executeReadyResponses() { + printf("Checking for ready responses in queue...\n"); + while (true) { + auto it = responses.find(next_seq_to_send); + if (it == responses.end()) { + break; // Exit loop if the next sequence is not in the map + } + it->second(); // Execute the stored lambda + responses.erase(it); // Safely erase the entry + ++next_seq_to_send; // Increment the sequence number + } +} + +// Get sequence number +int Sequencer::allocateSequenceNumber() { + std::lock_guard lock(mtx); + int seq = current_seq; + current_seq++; + return seq; +} + +// Add/Execute sequence function +void Sequencer::executeFuncInSequence(int seq, std::function response_lambda) { + std::lock_guard lock(mtx); + + if (seq == next_seq_to_send) { + // If the received sequence is the next one to send, execute it immediately + printf("Executing lambda\n"); + response_lambda(); + // Increment the next sequence to send + ++next_seq_to_send; + // Continue sending any subsequent responses that are ready + executeReadyResponses(); + } else { + // If the sequence is not the next to send, store it in the map + responses[seq] = response_lambda; + printf("Storing lambda with seq: %d, next to send: %d\n", seq, next_seq_to_send); + } +} \ No newline at end of file diff --git a/syncd/Sequencer.h b/syncd/Sequencer.h new file mode 100755 index 000000000..d87063b08 --- /dev/null +++ b/syncd/Sequencer.h @@ -0,0 +1,53 @@ +#ifndef SEQUENCER_H +#define SEQUENCER_H + +#include +#include +#include +#include +#include +#include + +#define MAX_SEQUENCE_NUMBER 1000000 +#define INVALID_SEQUENCE_NUMBER std::numeric_limits::min() +using AnyTask = std::function; + +class Sequencer { +public: + // Static method to provide access to the single instance + static Sequencer& getInstance() { + static Sequencer instance; + return instance; + } + + // Get sequence number + int allocateSequenceNumber(); + + // Add/Execute sequence function + void executeFuncInSequence(int seq, std::function response_lambda); + +private: + // Private constructor + Sequencer() : current_seq(0), next_seq_to_send(0), last_update_time(std::chrono::steady_clock::now()) {} + + // Delete the copy constructor and assignment operator + Sequencer(const Sequencer&) = delete; + Sequencer& operator=(const Sequencer&) = delete; + + // Reset the sequence number to avoid overflow + void resetSequence(); + + // Watchdog function to monitor inactivity + void performWatchdogCheck(); + + // Helper function to execute all ready responses in order + void executeReadyResponses(); + + int current_seq; // Tracks the latest sequence number assigned to a task + int next_seq_to_send; // The next sequence number that should be sent + std::mutex mtx; // Protects shared data + std::map> responses; // Stores responses by sequence number + std::chrono::steady_clock::time_point last_update_time; // Time of the last sequence update +}; + +#endif // SEQUENCER_H \ No newline at end of file diff --git a/syncd/Syncd.cpp b/syncd/Syncd.cpp index 9e2cd1971..e34479d0a 100644 --- a/syncd/Syncd.cpp +++ b/syncd/Syncd.cpp @@ -342,25 +342,25 @@ sai_status_t Syncd::processSingleEvent( WatchdogScope ws(m_timerWatchdog, op + ":" + key, &kco); if (op == REDIS_ASIC_STATE_COMMAND_CREATE) - return processQuadEvent(SAI_COMMON_API_CREATE, kco); + return processQuadEvent(SAI_COMMON_API_CREATE, kco, m_sequencer.allocateSequenceNumber()); if (op == REDIS_ASIC_STATE_COMMAND_REMOVE) - return processQuadEvent(SAI_COMMON_API_REMOVE, kco); + return processQuadEvent(SAI_COMMON_API_REMOVE, kco, m_sequencer.allocateSequenceNumber()); if (op == REDIS_ASIC_STATE_COMMAND_SET) - return processQuadEvent(SAI_COMMON_API_SET, kco); + return processQuadEvent(SAI_COMMON_API_SET, kco, m_sequencer.allocateSequenceNumber()); if (op == REDIS_ASIC_STATE_COMMAND_GET) - return processQuadEvent(SAI_COMMON_API_GET, kco); + return processQuadEvent(SAI_COMMON_API_GET, kco, m_sequencer.allocateSequenceNumber()); if (op == REDIS_ASIC_STATE_COMMAND_BULK_CREATE) - return processBulkQuadEvent(SAI_COMMON_API_BULK_CREATE, kco); + return processBulkQuadEvent(SAI_COMMON_API_BULK_CREATE, kco, m_sequencer.allocateSequenceNumber()); if (op == REDIS_ASIC_STATE_COMMAND_BULK_REMOVE) - return processBulkQuadEvent(SAI_COMMON_API_BULK_REMOVE, kco); + return processBulkQuadEvent(SAI_COMMON_API_BULK_REMOVE, kco, m_sequencer.allocateSequenceNumber()); if (op == REDIS_ASIC_STATE_COMMAND_BULK_SET) - return processBulkQuadEvent(SAI_COMMON_API_BULK_SET, kco); + return processBulkQuadEvent(SAI_COMMON_API_BULK_SET, kco, m_sequencer.allocateSequenceNumber()); if (op == REDIS_ASIC_STATE_COMMAND_NOTIFY) return processNotifySyncd(kco); @@ -794,7 +794,8 @@ sai_status_t Syncd::processGetStatsEvent( sai_status_t Syncd::processBulkQuadEvent( _In_ sai_common_api_t api, - _In_ const swss::KeyOpFieldsValuesTuple &kco) + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int seqIndex) { SWSS_LOG_ENTER(); @@ -876,11 +877,11 @@ sai_status_t Syncd::processBulkQuadEvent( if (info->isobjectid) { - return processBulkOid(objectType, objectIds, api, attributes, strAttributes); + return processBulkOid(objectType, objectIds, api, attributes, strAttributes, seqIndex); } else { - return processBulkEntry(objectType, objectIds, api, attributes, strAttributes); + return processBulkEntry(objectType, objectIds, api, attributes, strAttributes, seqIndex); } } @@ -910,7 +911,7 @@ sai_status_t Syncd::processBulkQuadEventInInitViewMode( if (info->isnonobjectid) { - sendApiResponse(api, SAI_STATUS_SUCCESS, (uint32_t)statuses.size(), statuses.data()); + sendApiResponseSequence(api, SAI_STATUS_SUCCESS, (uint32_t)statuses.size(), statuses.data()); syncUpdateRedisBulkQuadEvent(api, statuses, objectType, objectIds, strAttributes); @@ -929,7 +930,7 @@ sai_status_t Syncd::processBulkQuadEventInInitViewMode( default: - sendApiResponse(api, SAI_STATUS_SUCCESS, (uint32_t)statuses.size(), statuses.data()); + sendApiResponseSequence(api, SAI_STATUS_SUCCESS, (uint32_t)statuses.size(), statuses.data()); syncUpdateRedisBulkQuadEvent(api, statuses, objectType, objectIds, strAttributes); @@ -1729,7 +1730,8 @@ sai_status_t Syncd::processBulkEntry( _In_ const std::vector& objectIds, _In_ sai_common_api_t api, _In_ const std::vector>& attributes, - _In_ const std::vector>& strAttributes) + _In_ const std::vector>& strAttributes, + _In_ int seqIndex) { SWSS_LOG_ENTER(); @@ -1767,7 +1769,7 @@ sai_status_t Syncd::processBulkEntry( if (all != SAI_STATUS_NOT_SUPPORTED && all != SAI_STATUS_NOT_IMPLEMENTED) { - sendApiResponse(api, all, (uint32_t)objectIds.size(), statuses.data()); + sendApiResponseSequence(api, all, (uint32_t)objectIds.size(), statuses.data(), seqIndex); syncUpdateRedisBulkQuadEvent(api, statuses, objectType, objectIds, strAttributes); return all; @@ -1892,7 +1894,7 @@ sai_status_t Syncd::processBulkEntry( statuses[idx] = status; } - sendApiResponse(api, all, (uint32_t)objectIds.size(), statuses.data()); + sendApiResponseSequence(api, all, (uint32_t)objectIds.size(), statuses.data()); syncUpdateRedisBulkQuadEvent(api, statuses, objectType, objectIds, strAttributes); @@ -2088,7 +2090,8 @@ sai_status_t Syncd::processBulkOid( _In_ const std::vector& objectIds, _In_ sai_common_api_t api, _In_ const std::vector>& attributes, - _In_ const std::vector>& strAttributes) + _In_ const std::vector>& strAttributes, + _In_ const seqIndex) { SWSS_LOG_ENTER(); @@ -2124,7 +2127,7 @@ sai_status_t Syncd::processBulkOid( if (all != SAI_STATUS_NOT_SUPPORTED && all != SAI_STATUS_NOT_IMPLEMENTED) { - sendApiResponse(api, all, (uint32_t)objectIds.size(), statuses.data()); + sendApiResponseSequence(api, all, (uint32_t)objectIds.size(), statuses.data(), seqIndex); syncUpdateRedisBulkQuadEvent(api, statuses, objectType, objectIds, strAttributes); return all; @@ -2177,7 +2180,7 @@ sai_status_t Syncd::processBulkOid( statuses[idx] = status; } - sendApiResponse(api, all, (uint32_t)objectIds.size(), statuses.data()); + sendApiResponseSequence(api, all, (uint32_t)objectIds.size(), statuses.data()); syncUpdateRedisBulkQuadEvent(api, statuses, objectType, objectIds, strAttributes); @@ -2270,7 +2273,7 @@ sai_status_t Syncd::processQuadInInitViewModeCreate( } } - sendApiResponse(SAI_COMMON_API_CREATE, SAI_STATUS_SUCCESS); + sendApiResponseSequence(SAI_COMMON_API_CREATE, SAI_STATUS_SUCCESS); return SAI_STATUS_SUCCESS; } @@ -2332,7 +2335,7 @@ sai_status_t Syncd::processQuadInInitViewModeRemove( m_initViewRemovedVidSet.insert(objectVid); } - sendApiResponse(SAI_COMMON_API_REMOVE, SAI_STATUS_SUCCESS); + sendApiResponseSequence(SAI_COMMON_API_REMOVE, SAI_STATUS_SUCCESS); return SAI_STATUS_SUCCESS; } @@ -2346,7 +2349,7 @@ sai_status_t Syncd::processQuadInInitViewModeSet( // we support SET api on all objects in init view mode - sendApiResponse(SAI_COMMON_API_SET, SAI_STATUS_SUCCESS); + sendApiResponseSequence(SAI_COMMON_API_SET, SAI_STATUS_SUCCESS); return SAI_STATUS_SUCCESS; } @@ -2504,6 +2507,29 @@ void Syncd::sendApiResponse( sai_serialize_common_api(api).c_str()); } +void Syncd::sendApiResponseSequence( + _In_ sai_common_api_t api, + _In_ sai_status_t status, + _In_ uint32_t object_count, + _In_ sai_status_t* object_statuses, + _In_ int seqIndex) +{ + SWSS_LOG_ENTER(); + + if(seqIndex != INVALID_SEQUENCE_NUMBER) { + // If the response is to be sequenced, then add it to the sequencer + auto lambda = [=]() { + sendApiResponse(api, status, object_count, object_statuses); + }; + + m_sequencer.executeFuncInSequence(seqIndex, lambda); + } + else { + // If the response is not to be sequenced, then send it directly + sendApiResponse(api, status, object_count, object_statuses); + } +} + void Syncd::processFlexCounterGroupEvent( // TODO must be moved to go via ASIC channel queue _In_ swss::ConsumerTable& consumer) { @@ -2555,7 +2581,7 @@ sai_status_t Syncd::processFlexCounterGroupEvent( if (fromAsicChannel) { - sendApiResponse(SAI_COMMON_API_SET, SAI_STATUS_SUCCESS); + sendApiResponseSequence(SAI_COMMON_API_SET, SAI_STATUS_SUCCESS); } return SAI_STATUS_SUCCESS; @@ -2597,7 +2623,7 @@ sai_status_t Syncd::processFlexCounterEvent( if (fromAsicChannel) { - sendApiResponse(SAI_COMMON_API_SET, SAI_STATUS_FAILURE); + sendApiResponseSequence(SAI_COMMON_API_SET, SAI_STATUS_FAILURE); } return SAI_STATUS_FAILURE; // if key is invalid there is no need to process this event again @@ -2651,7 +2677,7 @@ sai_status_t Syncd::processFlexCounterEvent( if (fromAsicChannel) { - sendApiResponse(SAI_COMMON_API_SET, SAI_STATUS_SUCCESS); + sendApiResponseSequence(SAI_COMMON_API_SET, SAI_STATUS_SUCCESS); } return SAI_STATUS_SUCCESS; @@ -2856,7 +2882,8 @@ void Syncd::syncUpdateRedisBulkQuadEvent( sai_status_t Syncd::processQuadEvent( _In_ sai_common_api_t api, - _In_ const swss::KeyOpFieldsValuesTuple &kco) + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int seqIndex) { SWSS_LOG_ENTER(); @@ -2978,7 +3005,7 @@ sai_status_t Syncd::processQuadEvent( } else if (status != SAI_STATUS_SUCCESS) { - sendApiResponse(api, status); + sendApiResponseSequence(api, status, seqIndex=seqIndex); if (info->isobjectid && api == SAI_COMMON_API_SET) { @@ -3007,7 +3034,7 @@ sai_status_t Syncd::processQuadEvent( } else // non GET api, status is SUCCESS { - sendApiResponse(api, status); + sendApiResponseSequence(api, status, seqIndex=seqIndex); } syncUpdateRedisQuadEvent(status, api, kco); @@ -3454,6 +3481,28 @@ void Syncd::sendGetResponse( SWSS_LOG_INFO("response for GET api was send"); } +void Syncd::sendGetResponseSequence( + _In_ sai_object_type_t objectType, + _In_ const std::string& strObjectId, + _In_ sai_object_id_t switchVid, + _In_ sai_status_t status, + _In_ uint32_t attr_count, + _In_ sai_attribute_t *attr_list, + _In_ int seqIndex) { + SWSS_LOG_ENTER(); + + if(seqIndex != INVALID_SEQUENCE_NUMBER) { + auto lambda = [=]() { + sendGetResponse(objectType, strObjectId, switchVid, status, attr_count, attr_list); + }; + + m_sequencer.executeFuncInSequence(seqIndex, lambda); + } + else { + sendGetResponse(objectType, strObjectId, switchVid, status, attr_count, attr_list); + } +} + void Syncd::snoopGetResponse( _In_ sai_object_type_t object_type, _In_ const std::string& strObjectId, // can be non object id diff --git a/syncd/Syncd.h b/syncd/Syncd.h index 06eb84a62..c36e6581e 100644 --- a/syncd/Syncd.h +++ b/syncd/Syncd.h @@ -25,6 +25,7 @@ #include "swss/consumertable.h" #include "swss/producertable.h" #include "swss/notificationconsumer.h" +#include "Sequencer.h" #include @@ -147,25 +148,29 @@ namespace syncd sai_status_t processQuadEvent( _In_ sai_common_api_t api, - _In_ const swss::KeyOpFieldsValuesTuple &kco); + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int seqIndex = INVALID_SEQUENCE_NUMBER); sai_status_t processBulkQuadEvent( _In_ sai_common_api_t api, - _In_ const swss::KeyOpFieldsValuesTuple &kco); + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int seqIndex = INVALID_SEQUENCE_NUMBER); sai_status_t processBulkOid( _In_ sai_object_type_t objectType, _In_ const std::vector &object_ids, _In_ sai_common_api_t api, _In_ const std::vector> &attributes, - _In_ const std::vector>& strAttributes); + _In_ const std::vector>& strAttributes, + _In_ const seqIndex = INVALID_SEQUENCE_NUMBER); sai_status_t processBulkEntry( _In_ sai_object_type_t objectType, _In_ const std::vector &object_ids, _In_ sai_common_api_t api, _In_ const std::vector> &attributes, - _In_ const std::vector>& strAttributes); + _In_ const std::vector>& strAttributes, + _In_ const seqIndex = INVALID_SEQUENCE_NUMBER); sai_status_t processBulkCreateEntry( _In_ sai_object_type_t objectType, @@ -350,6 +355,13 @@ namespace syncd _In_ uint32_t object_count = 0, _In_ sai_status_t * object_statuses = NULL); + void sendApiResponseSequence( + _In_ sai_common_api_t api, + _In_ sai_status_t status, + _In_ uint32_t object_count = 0, + _In_ sai_status_t * object_statuses = NULL, + _In_ int seqIndex = INVALID_SEQUENCE_NUMBER); + void sendGetResponse( _In_ sai_object_type_t objectType, _In_ const std::string& strObjectId, @@ -358,6 +370,15 @@ namespace syncd _In_ uint32_t attr_count, _In_ sai_attribute_t *attr_list); + void sendGetResponseSequence( + _In_ sai_object_type_t objectType, + _In_ const std::string& strObjectId, + _In_ sai_object_id_t switchVid, + _In_ sai_status_t status, + _In_ uint32_t attr_count, + _In_ sai_attribute_t *attr_list, + _In_ int seqIndex = INVALID_SEQUENCE_NUMBER); + void sendNotifyResponse( _In_ sai_status_t status); @@ -400,6 +421,8 @@ namespace syncd sai_service_method_table_t m_test_services; + Sequencer& m_sequencer = Sequencer::getInstance(); + public: // TODO to private bool m_asicInitViewMode; From d0369a20d823d533f077cd476617da9d67de58f6 Mon Sep 17 00:00:00 2001 From: DanielaMurin Date: Wed, 21 Aug 2024 19:08:37 +0300 Subject: [PATCH 02/35] Add Sequencer to sendGetResponse Change-Id: I647dc326aac995785ebc1b2d882f85fd2b3d484d --- syncd/Syncd.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/syncd/Syncd.cpp b/syncd/Syncd.cpp index e34479d0a..7238ff262 100644 --- a/syncd/Syncd.cpp +++ b/syncd/Syncd.cpp @@ -2394,7 +2394,7 @@ sai_status_t Syncd::processQuadInInitViewModeGet( status = SAI_STATUS_INVALID_OBJECT_ID; - sendGetResponse(objectType, strObjectId, switchVid, status, attr_count, attr_list); + sendGetResponseSequence(objectType, strObjectId, switchVid, status, attr_count, attr_list); return status; } @@ -2439,7 +2439,7 @@ sai_status_t Syncd::processQuadInInitViewModeGet( * key. */ - sendGetResponse(objectType, strObjectId, switchVid, status, attr_count, attr_list); + sendGetResponseSequence(objectType, strObjectId, switchVid, status, attr_count, attr_list); return status; } @@ -3001,7 +3001,7 @@ sai_status_t Syncd::processQuadEvent( sai_object_id_t switchVid = VidManager::switchIdQuery(metaKey.objectkey.key.object_id); - sendGetResponse(metaKey.objecttype, strObjectId, switchVid, status, attr_count, attr_list); + sendGetResponseSequence(metaKey.objecttype, strObjectId, switchVid, status, attr_count, attr_list, seqIndex); } else if (status != SAI_STATUS_SUCCESS) { From 2caeedc93a5f691dbaebef5a0eb4e995e8dfe8e2 Mon Sep 17 00:00:00 2001 From: shiraez Date: Wed, 21 Aug 2024 12:08:36 +0300 Subject: [PATCH 03/35] Ring Buffer - Phase 1: The main thread push to the ring buffer and the second thread pop from the ring buffer and executes the function --- syncd/Syncd.cpp | 61 ++++++++++++++++++++- syncd/Syncd.h | 143 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 202 insertions(+), 2 deletions(-) diff --git a/syncd/Syncd.cpp b/syncd/Syncd.cpp index 7238ff262..0f1294f6d 100644 --- a/syncd/Syncd.cpp +++ b/syncd/Syncd.cpp @@ -207,10 +207,32 @@ Syncd::Syncd( } m_breakConfig = BreakConfigParser::parseBreakConfig(m_commandLineOptions->m_breakConfig); + ring_thread = std::thread(&Syncd::popRingBuffer, this); SWSS_LOG_NOTICE("syncd started"); } +void Syncd::popRingBuffer() +{ + if (!gRingBuffer || gRingBuffer->Started) + return; + SWSS_LOG_ENTER(); + gRingBuffer->Started = true; + SWSS_LOG_NOTICE("OrchDaemon starts the popRingBuffer thread!"); + while (!ring_thread_exited) + { + std::unique_lock lock(gRingBuffer->mtx); + gRingBuffer->cv.wait(lock, [&](){ return !gRingBuffer->IsEmpty(); }); + gRingBuffer->Idle = false; + AnyTask func; + while (gRingBuffer->pop(func)) { + func(); + } + gRingBuffer->doTask(); + gRingBuffer->Idle = true; + } +} + Syncd::~Syncd() { SWSS_LOG_ENTER(); @@ -316,8 +338,13 @@ void Syncd::processEvent( */ consumer.pop(kco, isInitViewMode()); - - processSingleEvent(kco); + /* + * TODO + * + */ + pushRingBuffer([=](){ + processSingleEvent(kco); + }); } while (!consumer.empty()); } @@ -2880,6 +2907,24 @@ void Syncd::syncUpdateRedisBulkQuadEvent( timer.inc(statuses.size()); } +void Syncd::pushRingBuffer(AnyTask&& func) +{ + if (!gRingBuffer || !gRingBuffer->Started) + { + func(); + // } else if (!gRingBuffer->Serves(getName())) { + // while (!gRingBuffer->IsEmpty() || !gRingBuffer->Idle) { + // std::this_thread::sleep_for(std::chrono::milliseconds(SLEEP_MSECONDS)); + // } + // func(); + } else { + while (!gRingBuffer->push(func)) { + SWSS_LOG_WARN("fail to push..ring is full..."); + } + gRingBuffer->cv.notify_one(); + } +} + sai_status_t Syncd::processQuadEvent( _In_ sai_common_api_t api, _In_ const swss::KeyOpFieldsValuesTuple &kco, @@ -2899,7 +2944,19 @@ sai_status_t Syncd::processQuadEvent( { SWSS_LOG_THROW("invalid object type %s", key.c_str()); } + + return processQuadEventTag(api, key, op, strObjectId, metaKey, kco, 0); +} +sai_status_t Syncd::processQuadEventTag( + _In_ sai_common_api_t api, + _In_ const std::string &key, + _In_ const std::string &op, + _In_ const std::string &strObjectId, + _In_ const sai_object_meta_key_t metaKey, + _In_ const swss::KeyOpFieldsValuesTuple &kco, + int sequenceNumber) +{ auto& values = kfvFieldsValues(kco); for (auto& v: values) diff --git a/syncd/Syncd.h b/syncd/Syncd.h index c36e6581e..ad3e863ba 100644 --- a/syncd/Syncd.h +++ b/syncd/Syncd.h @@ -31,12 +31,20 @@ namespace syncd { + #define ORCH_RING_SIZE 30 + #define SLEEP_MSECONDS 500 + using AnyTask = std::function; + template + class RingBuffer; + typedef RingBuffer SyncdRing; + class Syncd { private: Syncd(const Syncd&) = delete; Syncd& operator=(const Syncd&) = delete; + std::thread ring_thread; public: @@ -151,6 +159,15 @@ namespace syncd _In_ const swss::KeyOpFieldsValuesTuple &kco, _In_ int seqIndex = INVALID_SEQUENCE_NUMBER); + sai_status_t processQuadEventTag( + _In_ sai_common_api_t api, + _In_ const std::string &key, + _In_ const std::string &op, + _In_ const std::string &strObjectId, + _In_ const sai_object_meta_key_t metaKey, + _In_ const swss::KeyOpFieldsValuesTuple &kco, + int sequenceNumber); + sai_status_t processBulkQuadEvent( _In_ sai_common_api_t api, _In_ const swss::KeyOpFieldsValuesTuple &kco, @@ -426,6 +443,9 @@ namespace syncd public: // TODO to private bool m_asicInitViewMode; + void pushRingBuffer(AnyTask&& func); + void popRingBuffer(); + void enableRingBuffer(); std::shared_ptr m_manager; @@ -536,5 +556,128 @@ namespace syncd TimerWatchdog m_timerWatchdog; std::set m_createdInInitView; + + protected: + /* Orchdaemon instance points to the same ring buffer during its lifetime */ + SyncdRing* gRingBuffer = nullptr; + std::atomic ring_thread_exited{false}; }; + typedef std::map EventMap; + template + class RingBuffer + { + private: + static RingBuffer* instance; + std::vector buffer; + int head = 0; + int tail = 0; + EventMap m_eventMap; + protected: + RingBuffer(): buffer(RingSize) {} + ~RingBuffer() { + delete instance; + } + public: + RingBuffer(const RingBuffer&) = delete; + RingBuffer(RingBuffer&&) = delete; + RingBuffer& operator= (const RingBuffer&) = delete; + RingBuffer& operator= (RingBuffer&&) = delete; + static RingBuffer* Get(); + bool Started = false; + bool Idle = true; + std::mutex mtx; + std::condition_variable cv; + bool IsFull(); + bool IsEmpty(); + bool push(DataType entry); + bool pop(DataType& entry); + DataType& HeadEntry(); + void addEvent(std::string* executor); + void doTask(); + bool tasksPending(); + bool Serves(const std::string& tableName); + }; + + +template +RingBuffer* RingBuffer::instance = nullptr; +template +RingBuffer* RingBuffer::Get() +{ + if (instance == nullptr) { + instance = new RingBuffer(); + SWSS_LOG_NOTICE("Orchagent RingBuffer created at %p!", (void *)instance); + } + return instance; +} +template +bool RingBuffer::IsFull() +{ + return (tail + 1) % RingSize == head; +} +template +bool RingBuffer::IsEmpty() +{ + return tail == head; +} +template +bool RingBuffer::push(DataType ringEntry) +{ + if (IsFull()) + return false; + buffer[tail] = std::move(ringEntry); + tail = (tail + 1) % RingSize; + return true; +} +template +DataType& RingBuffer::HeadEntry() { + return buffer[head]; +} +template +bool RingBuffer::pop(DataType& ringEntry) +{ + if (IsEmpty()) + return false; + ringEntry = std::move(buffer[head]); + head = (head + 1) % RingSize; + return true; +} +template +void RingBuffer::addEvent(std::string* executor) +{ +// auto inserted = m_eventMap.emplace(std::piecewise_construct, +// std::forward_as_tuple(executor->getName()), +// std::forward_as_tuple(executor)); +// // If there is duplication of executorName in m_eventMap, logic error +// if (!inserted.second) +// { +// SWSS_LOG_THROW("Duplicated executorName in m_eventMap: %s", executor->getName().c_str()); +// } +} +template +void RingBuffer::doTask() +{ +// for (auto &it : m_eventMap) { +// it.second->drain(); +// } +} +template +bool RingBuffer::tasksPending() +{ +// for (auto &it : m_eventMap) { +// auto consumer = dynamic_cast(it.second.get()); +// if (consumer->m_toSync.size()) +// return true; +// } + return false; +} +template +bool RingBuffer::Serves(const std::string& tableName) +{ +// for (auto &it : m_eventMap) { +// if (it.first == tableName) +// return true; +// } + return true; +} } From 98c6c89fa731280381d234cd89fb9a206dcc589e Mon Sep 17 00:00:00 2001 From: DanielaMurin Date: Thu, 22 Aug 2024 15:09:04 +0300 Subject: [PATCH 04/35] Remove singleton & fix compilation Change-Id: I001bdc0f32d81eb847157a2f7ec8bc0faa5de1a8 --- syncd/Makefile.am | 3 +- syncd/Sequencer.cpp | 2 ++ syncd/Sequencer.h | 79 ++++++++++++++++++++++----------------------- syncd/Syncd.cpp | 23 ++++++------- syncd/Syncd.h | 11 ++++--- 5 files changed, 60 insertions(+), 58 deletions(-) diff --git a/syncd/Makefile.am b/syncd/Makefile.am index 3c546fb74..6147f16c2 100644 --- a/syncd/Makefile.am +++ b/syncd/Makefile.am @@ -56,7 +56,8 @@ libSyncd_a_SOURCES = \ WatchdogScope.cpp \ Workaround.cpp \ ZeroMQNotificationProducer.cpp \ - syncd_main.cpp + syncd_main.cpp \ + Sequencer.cpp libSyncd_a_CPPFLAGS = $(CODE_COVERAGE_CPPFLAGS) libSyncd_a_CXXFLAGS = $(DBGFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS_COMMON) $(CODE_COVERAGE_CXXFLAGS) diff --git a/syncd/Sequencer.cpp b/syncd/Sequencer.cpp index c4325ccba..56f394546 100755 --- a/syncd/Sequencer.cpp +++ b/syncd/Sequencer.cpp @@ -1,5 +1,7 @@ #include "Sequencer.h" +using namespace syncd; + // Helper function to execute all ready responses in order void Sequencer::executeReadyResponses() { printf("Checking for ready responses in queue...\n"); diff --git a/syncd/Sequencer.h b/syncd/Sequencer.h index d87063b08..61f3fff80 100755 --- a/syncd/Sequencer.h +++ b/syncd/Sequencer.h @@ -8,46 +8,43 @@ #include #include -#define MAX_SEQUENCE_NUMBER 1000000 -#define INVALID_SEQUENCE_NUMBER std::numeric_limits::min() -using AnyTask = std::function; - -class Sequencer { -public: - // Static method to provide access to the single instance - static Sequencer& getInstance() { - static Sequencer instance; - return instance; - } - - // Get sequence number - int allocateSequenceNumber(); - - // Add/Execute sequence function - void executeFuncInSequence(int seq, std::function response_lambda); - -private: - // Private constructor - Sequencer() : current_seq(0), next_seq_to_send(0), last_update_time(std::chrono::steady_clock::now()) {} - - // Delete the copy constructor and assignment operator - Sequencer(const Sequencer&) = delete; - Sequencer& operator=(const Sequencer&) = delete; - - // Reset the sequence number to avoid overflow - void resetSequence(); - - // Watchdog function to monitor inactivity - void performWatchdogCheck(); - - // Helper function to execute all ready responses in order - void executeReadyResponses(); - - int current_seq; // Tracks the latest sequence number assigned to a task - int next_seq_to_send; // The next sequence number that should be sent - std::mutex mtx; // Protects shared data - std::map> responses; // Stores responses by sequence number - std::chrono::steady_clock::time_point last_update_time; // Time of the last sequence update -}; +namespace syncd { + + #define MAX_SEQUENCE_NUMBER 1000000 + #define INVALID_SEQUENCE_NUMBER std::numeric_limits::min() + using AnyTask = std::function; + + class Sequencer { + public: + // Public constructor + Sequencer() : current_seq(0), next_seq_to_send(0), last_update_time(std::chrono::steady_clock::now()) {} + + // Public destructor + ~Sequencer() {} + + // Get sequence number + int allocateSequenceNumber(); + + // Add/Execute sequence function + void executeFuncInSequence(int seq, std::function response_lambda); + + private: + // Reset the sequence number to avoid overflow + void resetSequence(); + + // Watchdog function to monitor inactivity + void performWatchdogCheck(); + + // Helper function to execute all ready responses in order + void executeReadyResponses(); + + int current_seq; // Tracks the latest sequence number assigned to a task + int next_seq_to_send; // The next sequence number that should be sent + std::mutex mtx; // Protects shared data + std::map> responses; // Stores responses by sequence number + std::chrono::steady_clock::time_point last_update_time; // Time of the last sequence update + }; +} + #endif // SEQUENCER_H \ No newline at end of file diff --git a/syncd/Syncd.cpp b/syncd/Syncd.cpp index 0f1294f6d..f24518767 100644 --- a/syncd/Syncd.cpp +++ b/syncd/Syncd.cpp @@ -147,6 +147,7 @@ Syncd::Syncd( } m_client = std::make_shared(m_dbAsic); + m_sequencer = std::make_shared(); m_processor = std::make_shared(m_notifications, m_client, std::bind(&Syncd::syncProcessNotification, this, _1)); m_handler = std::make_shared(m_processor); @@ -369,25 +370,25 @@ sai_status_t Syncd::processSingleEvent( WatchdogScope ws(m_timerWatchdog, op + ":" + key, &kco); if (op == REDIS_ASIC_STATE_COMMAND_CREATE) - return processQuadEvent(SAI_COMMON_API_CREATE, kco, m_sequencer.allocateSequenceNumber()); + return processQuadEvent(SAI_COMMON_API_CREATE, kco, m_sequencer->allocateSequenceNumber()); if (op == REDIS_ASIC_STATE_COMMAND_REMOVE) - return processQuadEvent(SAI_COMMON_API_REMOVE, kco, m_sequencer.allocateSequenceNumber()); + return processQuadEvent(SAI_COMMON_API_REMOVE, kco, m_sequencer->allocateSequenceNumber()); if (op == REDIS_ASIC_STATE_COMMAND_SET) - return processQuadEvent(SAI_COMMON_API_SET, kco, m_sequencer.allocateSequenceNumber()); + return processQuadEvent(SAI_COMMON_API_SET, kco, m_sequencer->allocateSequenceNumber()); if (op == REDIS_ASIC_STATE_COMMAND_GET) - return processQuadEvent(SAI_COMMON_API_GET, kco, m_sequencer.allocateSequenceNumber()); + return processQuadEvent(SAI_COMMON_API_GET, kco, m_sequencer->allocateSequenceNumber()); if (op == REDIS_ASIC_STATE_COMMAND_BULK_CREATE) - return processBulkQuadEvent(SAI_COMMON_API_BULK_CREATE, kco, m_sequencer.allocateSequenceNumber()); + return processBulkQuadEvent(SAI_COMMON_API_BULK_CREATE, kco, m_sequencer->allocateSequenceNumber()); if (op == REDIS_ASIC_STATE_COMMAND_BULK_REMOVE) - return processBulkQuadEvent(SAI_COMMON_API_BULK_REMOVE, kco, m_sequencer.allocateSequenceNumber()); + return processBulkQuadEvent(SAI_COMMON_API_BULK_REMOVE, kco, m_sequencer->allocateSequenceNumber()); if (op == REDIS_ASIC_STATE_COMMAND_BULK_SET) - return processBulkQuadEvent(SAI_COMMON_API_BULK_SET, kco, m_sequencer.allocateSequenceNumber()); + return processBulkQuadEvent(SAI_COMMON_API_BULK_SET, kco, m_sequencer->allocateSequenceNumber()); if (op == REDIS_ASIC_STATE_COMMAND_NOTIFY) return processNotifySyncd(kco); @@ -2118,7 +2119,7 @@ sai_status_t Syncd::processBulkOid( _In_ sai_common_api_t api, _In_ const std::vector>& attributes, _In_ const std::vector>& strAttributes, - _In_ const seqIndex) + _In_ int seqIndex) { SWSS_LOG_ENTER(); @@ -2549,7 +2550,7 @@ void Syncd::sendApiResponseSequence( sendApiResponse(api, status, object_count, object_statuses); }; - m_sequencer.executeFuncInSequence(seqIndex, lambda); + m_sequencer->executeFuncInSequence(seqIndex, lambda); } else { // If the response is not to be sequenced, then send it directly @@ -3553,7 +3554,7 @@ void Syncd::sendGetResponseSequence( sendGetResponse(objectType, strObjectId, switchVid, status, attr_count, attr_list); }; - m_sequencer.executeFuncInSequence(seqIndex, lambda); + m_sequencer->executeFuncInSequence(seqIndex, lambda); } else { sendGetResponse(objectType, strObjectId, switchVid, status, attr_count, attr_list); @@ -5401,4 +5402,4 @@ syncd_restart_type_t Syncd::handleRestartQuery( SWSS_LOG_NOTICE("received %s switch shutdown event", op.c_str()); return RequestShutdownCommandLineOptions::stringToRestartType(op); -} +} \ No newline at end of file diff --git a/syncd/Syncd.h b/syncd/Syncd.h index ad3e863ba..b17eebb3d 100644 --- a/syncd/Syncd.h +++ b/syncd/Syncd.h @@ -25,10 +25,11 @@ #include "swss/consumertable.h" #include "swss/producertable.h" #include "swss/notificationconsumer.h" -#include "Sequencer.h" #include +#include "Sequencer.h" + namespace syncd { #define ORCH_RING_SIZE 30 @@ -179,7 +180,7 @@ namespace syncd _In_ sai_common_api_t api, _In_ const std::vector> &attributes, _In_ const std::vector>& strAttributes, - _In_ const seqIndex = INVALID_SEQUENCE_NUMBER); + _In_ int seqIndex = INVALID_SEQUENCE_NUMBER); sai_status_t processBulkEntry( _In_ sai_object_type_t objectType, @@ -187,7 +188,7 @@ namespace syncd _In_ sai_common_api_t api, _In_ const std::vector> &attributes, _In_ const std::vector>& strAttributes, - _In_ const seqIndex = INVALID_SEQUENCE_NUMBER); + _In_ int seqIndex = INVALID_SEQUENCE_NUMBER); sai_status_t processBulkCreateEntry( _In_ sai_object_type_t objectType, @@ -438,8 +439,6 @@ namespace syncd sai_service_method_table_t m_test_services; - Sequencer& m_sequencer = Sequencer::getInstance(); - public: // TODO to private bool m_asicInitViewMode; @@ -495,6 +494,8 @@ namespace syncd std::shared_ptr m_client; + std::shared_ptr m_sequencer; + std::shared_ptr m_handler; std::shared_ptr m_processor; From 78ec4a77001f6cf45a94eb3b22b654099db6c8ac Mon Sep 17 00:00:00 2001 From: DanielaMurin Date: Thu, 22 Aug 2024 16:50:09 +0300 Subject: [PATCH 05/35] Fix merge issues Change-Id: Ie53906422adfc2027710ef9835dafe141448b84f --- syncd/Syncd.cpp | 36 ++++++++++++++++++------------------ syncd/Syncd.h | 12 ++++++------ 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/syncd/Syncd.cpp b/syncd/Syncd.cpp index f24518767..d03170395 100644 --- a/syncd/Syncd.cpp +++ b/syncd/Syncd.cpp @@ -823,7 +823,7 @@ sai_status_t Syncd::processGetStatsEvent( sai_status_t Syncd::processBulkQuadEvent( _In_ sai_common_api_t api, _In_ const swss::KeyOpFieldsValuesTuple &kco, - _In_ int seqIndex) + _In_ int sequenceNumber) { SWSS_LOG_ENTER(); @@ -905,11 +905,11 @@ sai_status_t Syncd::processBulkQuadEvent( if (info->isobjectid) { - return processBulkOid(objectType, objectIds, api, attributes, strAttributes, seqIndex); + return processBulkOid(objectType, objectIds, api, attributes, strAttributes, sequenceNumber); } else { - return processBulkEntry(objectType, objectIds, api, attributes, strAttributes, seqIndex); + return processBulkEntry(objectType, objectIds, api, attributes, strAttributes, sequenceNumber); } } @@ -1759,7 +1759,7 @@ sai_status_t Syncd::processBulkEntry( _In_ sai_common_api_t api, _In_ const std::vector>& attributes, _In_ const std::vector>& strAttributes, - _In_ int seqIndex) + _In_ int sequenceNumber) { SWSS_LOG_ENTER(); @@ -1797,7 +1797,7 @@ sai_status_t Syncd::processBulkEntry( if (all != SAI_STATUS_NOT_SUPPORTED && all != SAI_STATUS_NOT_IMPLEMENTED) { - sendApiResponseSequence(api, all, (uint32_t)objectIds.size(), statuses.data(), seqIndex); + sendApiResponseSequence(api, all, (uint32_t)objectIds.size(), statuses.data(), sequenceNumber); syncUpdateRedisBulkQuadEvent(api, statuses, objectType, objectIds, strAttributes); return all; @@ -2119,7 +2119,7 @@ sai_status_t Syncd::processBulkOid( _In_ sai_common_api_t api, _In_ const std::vector>& attributes, _In_ const std::vector>& strAttributes, - _In_ int seqIndex) + _In_ int sequenceNumber) { SWSS_LOG_ENTER(); @@ -2155,7 +2155,7 @@ sai_status_t Syncd::processBulkOid( if (all != SAI_STATUS_NOT_SUPPORTED && all != SAI_STATUS_NOT_IMPLEMENTED) { - sendApiResponseSequence(api, all, (uint32_t)objectIds.size(), statuses.data(), seqIndex); + sendApiResponseSequence(api, all, (uint32_t)objectIds.size(), statuses.data(), sequenceNumber); syncUpdateRedisBulkQuadEvent(api, statuses, objectType, objectIds, strAttributes); return all; @@ -2540,17 +2540,17 @@ void Syncd::sendApiResponseSequence( _In_ sai_status_t status, _In_ uint32_t object_count, _In_ sai_status_t* object_statuses, - _In_ int seqIndex) + _In_ int sequenceNumber) { SWSS_LOG_ENTER(); - if(seqIndex != INVALID_SEQUENCE_NUMBER) { + if(sequenceNumber != INVALID_SEQUENCE_NUMBER) { // If the response is to be sequenced, then add it to the sequencer auto lambda = [=]() { sendApiResponse(api, status, object_count, object_statuses); }; - m_sequencer->executeFuncInSequence(seqIndex, lambda); + m_sequencer->executeFuncInSequence(sequenceNumber, lambda); } else { // If the response is not to be sequenced, then send it directly @@ -2929,7 +2929,7 @@ void Syncd::pushRingBuffer(AnyTask&& func) sai_status_t Syncd::processQuadEvent( _In_ sai_common_api_t api, _In_ const swss::KeyOpFieldsValuesTuple &kco, - _In_ int seqIndex) + _In_ int sequenceNumber) { SWSS_LOG_ENTER(); @@ -2956,7 +2956,7 @@ sai_status_t Syncd::processQuadEventTag( _In_ const std::string &strObjectId, _In_ const sai_object_meta_key_t metaKey, _In_ const swss::KeyOpFieldsValuesTuple &kco, - int sequenceNumber) + _In_ int sequenceNumber) { auto& values = kfvFieldsValues(kco); @@ -3059,11 +3059,11 @@ sai_status_t Syncd::processQuadEventTag( sai_object_id_t switchVid = VidManager::switchIdQuery(metaKey.objectkey.key.object_id); - sendGetResponseSequence(metaKey.objecttype, strObjectId, switchVid, status, attr_count, attr_list, seqIndex); + sendGetResponseSequence(metaKey.objecttype, strObjectId, switchVid, status, attr_count, attr_list, sequenceNumber); } else if (status != SAI_STATUS_SUCCESS) { - sendApiResponseSequence(api, status, seqIndex=seqIndex); + sendApiResponseSequence(api, status, sequenceNumber=sequenceNumber); if (info->isobjectid && api == SAI_COMMON_API_SET) { @@ -3092,7 +3092,7 @@ sai_status_t Syncd::processQuadEventTag( } else // non GET api, status is SUCCESS { - sendApiResponseSequence(api, status, seqIndex=seqIndex); + sendApiResponseSequence(api, status, sequenceNumber=sequenceNumber); } syncUpdateRedisQuadEvent(status, api, kco); @@ -3546,15 +3546,15 @@ void Syncd::sendGetResponseSequence( _In_ sai_status_t status, _In_ uint32_t attr_count, _In_ sai_attribute_t *attr_list, - _In_ int seqIndex) { + _In_ int sequenceNumber) { SWSS_LOG_ENTER(); - if(seqIndex != INVALID_SEQUENCE_NUMBER) { + if(sequenceNumber != INVALID_SEQUENCE_NUMBER) { auto lambda = [=]() { sendGetResponse(objectType, strObjectId, switchVid, status, attr_count, attr_list); }; - m_sequencer->executeFuncInSequence(seqIndex, lambda); + m_sequencer->executeFuncInSequence(sequenceNumber, lambda); } else { sendGetResponse(objectType, strObjectId, switchVid, status, attr_count, attr_list); diff --git a/syncd/Syncd.h b/syncd/Syncd.h index b17eebb3d..861689886 100644 --- a/syncd/Syncd.h +++ b/syncd/Syncd.h @@ -158,7 +158,7 @@ namespace syncd sai_status_t processQuadEvent( _In_ sai_common_api_t api, _In_ const swss::KeyOpFieldsValuesTuple &kco, - _In_ int seqIndex = INVALID_SEQUENCE_NUMBER); + _In_ int sequenceNumber = INVALID_SEQUENCE_NUMBER); sai_status_t processQuadEventTag( _In_ sai_common_api_t api, @@ -172,7 +172,7 @@ namespace syncd sai_status_t processBulkQuadEvent( _In_ sai_common_api_t api, _In_ const swss::KeyOpFieldsValuesTuple &kco, - _In_ int seqIndex = INVALID_SEQUENCE_NUMBER); + _In_ int sequenceNumber = INVALID_SEQUENCE_NUMBER); sai_status_t processBulkOid( _In_ sai_object_type_t objectType, @@ -180,7 +180,7 @@ namespace syncd _In_ sai_common_api_t api, _In_ const std::vector> &attributes, _In_ const std::vector>& strAttributes, - _In_ int seqIndex = INVALID_SEQUENCE_NUMBER); + _In_ int sequenceNumber = INVALID_SEQUENCE_NUMBER); sai_status_t processBulkEntry( _In_ sai_object_type_t objectType, @@ -188,7 +188,7 @@ namespace syncd _In_ sai_common_api_t api, _In_ const std::vector> &attributes, _In_ const std::vector>& strAttributes, - _In_ int seqIndex = INVALID_SEQUENCE_NUMBER); + _In_ int sequenceNumber = INVALID_SEQUENCE_NUMBER); sai_status_t processBulkCreateEntry( _In_ sai_object_type_t objectType, @@ -378,7 +378,7 @@ namespace syncd _In_ sai_status_t status, _In_ uint32_t object_count = 0, _In_ sai_status_t * object_statuses = NULL, - _In_ int seqIndex = INVALID_SEQUENCE_NUMBER); + _In_ int sequenceNumber = INVALID_SEQUENCE_NUMBER); void sendGetResponse( _In_ sai_object_type_t objectType, @@ -395,7 +395,7 @@ namespace syncd _In_ sai_status_t status, _In_ uint32_t attr_count, _In_ sai_attribute_t *attr_list, - _In_ int seqIndex = INVALID_SEQUENCE_NUMBER); + _In_ int sequenceNumber = INVALID_SEQUENCE_NUMBER); void sendNotifyResponse( _In_ sai_status_t status); From 65f94df34f33ca90b300516687877697989979f1 Mon Sep 17 00:00:00 2001 From: shiraez Date: Sun, 25 Aug 2024 11:51:07 +0300 Subject: [PATCH 06/35] add debug info + enable thread --- syncd/Syncd.cpp | 56 ++++++++++++++++++++++++++++++++++++++++++++----- syncd/Syncd.h | 8 +++++-- 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/syncd/Syncd.cpp b/syncd/Syncd.cpp index d03170395..a750ca504 100644 --- a/syncd/Syncd.cpp +++ b/syncd/Syncd.cpp @@ -44,6 +44,8 @@ using namespace saimeta; using namespace sairediscommon; using namespace std::placeholders; +static int gdb_mode = 0; + #ifdef ASAN_ENABLED #define WD_DELAY_FACTOR 2 #else @@ -208,6 +210,8 @@ Syncd::Syncd( } m_breakConfig = BreakConfigParser::parseBreakConfig(m_commandLineOptions->m_breakConfig); + gRingBuffer = SyncdRing::Get(); + // Syncd::gRingBuffer = gRingBuffer; ring_thread = std::thread(&Syncd::popRingBuffer, this); SWSS_LOG_NOTICE("syncd started"); @@ -219,25 +223,43 @@ void Syncd::popRingBuffer() return; SWSS_LOG_ENTER(); gRingBuffer->Started = true; - SWSS_LOG_NOTICE("OrchDaemon starts the popRingBuffer thread!"); + SWSS_LOG_NOTICE("Syncd starts the popRingBuffer thread!"); while (!ring_thread_exited) { + SWSS_LOG_NOTICE("wait popRingBuffer thread!"); std::unique_lock lock(gRingBuffer->mtx); gRingBuffer->cv.wait(lock, [&](){ return !gRingBuffer->IsEmpty(); }); + SWSS_LOG_NOTICE("end waiting popRingBuffer thread!"); gRingBuffer->Idle = false; AnyTask func; + SWSS_LOG_DEBUG("ring_thread_exited!"); while (gRingBuffer->pop(func)) { + SWSS_LOG_DEBUG("popRingBuffer executes!"); func(); } + // lock.unlock(); gRingBuffer->doTask(); gRingBuffer->Idle = true; } + // while (!ring_thread_exited) { + // std::unique_lock lock(mtx); + // gRingBuffer->cv.wait(lock, [&](){ return !gRingBuffer->IsEmpty(); }); + // AnyTask func = std::move(buffer[head]); + // head = (head + 1) % RingSize; + // lock.unlock(); + // func(); + // } } Syncd::~Syncd() { SWSS_LOG_ENTER(); + if (gRingBuffer) { + ring_thread_exited = true; + ring_thread.detach(); + } + // empty } @@ -321,13 +343,18 @@ bool Syncd::isInitViewMode() const return m_asicInitViewMode && m_commandLineOptions->m_enableTempView; } +void point(_In_ const swss::KeyOpFieldsValuesTuple &kco){ + // SWSS_LOG_NOTICE("point"); + // SWSS_LOG_NOTICE("point %s" ,std::get<0>(kco)); + gdb_mode = 1; +} + void Syncd::processEvent( _In_ sairedis::SelectableChannel& consumer) { SWSS_LOG_ENTER(); std::lock_guard lock(m_mutex); - do { swss::KeyOpFieldsValuesTuple kco; @@ -339,13 +366,32 @@ void Syncd::processEvent( */ consumer.pop(kco, isInitViewMode()); + auto& key = kfvKey(kco); + auto& op = kfvOp(kco); /* * TODO * */ - pushRingBuffer([=](){ - processSingleEvent(kco); - }); + // int i = gdb_mode; + if(op == REDIS_ASIC_STATE_COMMAND_CREATE){ + gdb_mode++; + processSingleEvent(kco); + } + else{ + auto lambda = [=](){ + processSingleEvent(kco); + }; + pushRingBuffer(lambda); + } + // // processSingleEvent(kco); + // auto lambda = [=](){ + // // point(kco); + // processSingleEvent(kco); + // }; + // pushRingBuffer(lambda); + // // processSingleEvent(kco); + + } while (!consumer.empty()); } diff --git a/syncd/Syncd.h b/syncd/Syncd.h index 861689886..60720b0d1 100644 --- a/syncd/Syncd.h +++ b/syncd/Syncd.h @@ -25,6 +25,7 @@ #include "swss/consumertable.h" #include "swss/producertable.h" #include "swss/notificationconsumer.h" +#include #include @@ -572,6 +573,7 @@ namespace syncd std::vector buffer; int head = 0; int tail = 0; + void point(); EventMap m_eventMap; protected: RingBuffer(): buffer(RingSize) {} @@ -607,7 +609,7 @@ RingBuffer* RingBuffer::Get() { if (instance == nullptr) { instance = new RingBuffer(); - SWSS_LOG_NOTICE("Orchagent RingBuffer created at %p!", (void *)instance); + SWSS_LOG_NOTICE("Syncd RingBuffer created at %p!", (void *)instance); } return instance; } @@ -624,8 +626,10 @@ bool RingBuffer::IsEmpty() template bool RingBuffer::push(DataType ringEntry) { - if (IsFull()) + if (IsFull()){ + SWSS_LOG_WARN("pushRing is full"); return false; + } buffer[tail] = std::move(ringEntry); tail = (tail + 1) % RingSize; return true; From 8523a2a2da9bf97c1f807011b33e8a1711fc631f Mon Sep 17 00:00:00 2001 From: shiraez Date: Mon, 26 Aug 2024 00:23:35 +0300 Subject: [PATCH 07/35] add delay --- syncd/Syncd.cpp | 19 ++++++------------- syncd/Syncd.h | 2 +- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/syncd/Syncd.cpp b/syncd/Syncd.cpp index a750ca504..f707078f0 100644 --- a/syncd/Syncd.cpp +++ b/syncd/Syncd.cpp @@ -366,32 +366,25 @@ void Syncd::processEvent( */ consumer.pop(kco, isInitViewMode()); - auto& key = kfvKey(kco); - auto& op = kfvOp(kco); /* * TODO * */ + + // LogToModuleFile("1", std::get<0>(kco)); + // LogToModuleFile("1", std::get<1>(kco)); // int i = gdb_mode; - if(op == REDIS_ASIC_STATE_COMMAND_CREATE){ + if( gdb_mode < 1000){ gdb_mode++; processSingleEvent(kco); } else{ + auto lambda = [=](){ processSingleEvent(kco); }; pushRingBuffer(lambda); - } - // // processSingleEvent(kco); - // auto lambda = [=](){ - // // point(kco); - // processSingleEvent(kco); - // }; - // pushRingBuffer(lambda); - // // processSingleEvent(kco); - - + } } while (!consumer.empty()); } diff --git a/syncd/Syncd.h b/syncd/Syncd.h index 60720b0d1..7663215e5 100644 --- a/syncd/Syncd.h +++ b/syncd/Syncd.h @@ -33,7 +33,7 @@ namespace syncd { - #define ORCH_RING_SIZE 30 + #define ORCH_RING_SIZE 1024 #define SLEEP_MSECONDS 500 using AnyTask = std::function; template From 0ddc1ecce6e12ca42a5b7572efe52e6092a01261 Mon Sep 17 00:00:00 2001 From: DanielaMurin Date: Sun, 25 Aug 2024 20:20:52 +0300 Subject: [PATCH 08/35] debug single ring buffer & enable sequencer Change-Id: I90427922c574e9a5edaa722727878febe6c4d792 --- proxylib/Proxy.cpp | 1 + saiplayer/SaiPlayer.cpp | 1 + syncd/NotificationHandler.cpp | 1 + syncd/Sequencer.cpp | 10 +- syncd/Syncd.cpp | 178 ++++++++++++++++++++++++++-------- 5 files changed, 147 insertions(+), 44 deletions(-) diff --git a/proxylib/Proxy.cpp b/proxylib/Proxy.cpp index a844a4b2d..56f023e1c 100644 --- a/proxylib/Proxy.cpp +++ b/proxylib/Proxy.cpp @@ -1080,6 +1080,7 @@ void Proxy::updateAttributteNotificationPointers( { SWSS_LOG_ENTER(); + SWSS_LOG_NOTICE("sai_metadata_update_attribute_notification_pointers from Proxy::updateAttributteNotificationPointers"); sai_metadata_update_attribute_notification_pointers(&m_sn, count, attr_list); } diff --git a/saiplayer/SaiPlayer.cpp b/saiplayer/SaiPlayer.cpp index 51b5cf13f..e92b0241f 100644 --- a/saiplayer/SaiPlayer.cpp +++ b/saiplayer/SaiPlayer.cpp @@ -946,6 +946,7 @@ void SaiPlayer::update_notifications_pointers( * need to override them after create, and after set. */ + SWSS_LOG_NOTICE("sai_metadata_update_attribute_notification_pointers from SaiPlayer::update_notifications_pointers"); sai_metadata_update_attribute_notification_pointers(&m_switchNotifications, attr_count, attr_list); } diff --git a/syncd/NotificationHandler.cpp b/syncd/NotificationHandler.cpp index 5a4fa5300..06eb2163d 100644 --- a/syncd/NotificationHandler.cpp +++ b/syncd/NotificationHandler.cpp @@ -59,6 +59,7 @@ void NotificationHandler::updateNotificationsPointers( * Also notice that we are using the same pointers for ALL switches. */ + SWSS_LOG_NOTICE("sai_metadata_update_attribute_notification_pointers from NotificationHandler::updateNotificationsPointers"); sai_metadata_update_attribute_notification_pointers(&m_switchNotifications, attr_count, attr_list); } diff --git a/syncd/Sequencer.cpp b/syncd/Sequencer.cpp index 56f394546..844e1ba6e 100755 --- a/syncd/Sequencer.cpp +++ b/syncd/Sequencer.cpp @@ -1,10 +1,12 @@ #include "Sequencer.h" +#include "swss/logger.h" using namespace syncd; // Helper function to execute all ready responses in order void Sequencer::executeReadyResponses() { - printf("Checking for ready responses in queue...\n"); + //printf("Checking for ready responses in queue...\n"); + SWSS_LOG_NOTICE("multithreaded: Checking for ready responses in queue..."); while (true) { auto it = responses.find(next_seq_to_send); if (it == responses.end()) { @@ -21,6 +23,7 @@ int Sequencer::allocateSequenceNumber() { std::lock_guard lock(mtx); int seq = current_seq; current_seq++; + SWSS_LOG_NOTICE("multithreaded: allocate seq num: %d", seq); return seq; } @@ -30,7 +33,7 @@ void Sequencer::executeFuncInSequence(int seq, std::function response_la if (seq == next_seq_to_send) { // If the received sequence is the next one to send, execute it immediately - printf("Executing lambda\n"); + SWSS_LOG_NOTICE("multithreaded: executing reseponse lambda, seq num: %d", seq); response_lambda(); // Increment the next sequence to send ++next_seq_to_send; @@ -39,6 +42,7 @@ void Sequencer::executeFuncInSequence(int seq, std::function response_la } else { // If the sequence is not the next to send, store it in the map responses[seq] = response_lambda; - printf("Storing lambda with seq: %d, next to send: %d\n", seq, next_seq_to_send); + SWSS_LOG_NOTICE("Storing lambda with seq: %d, next to send: %d\n", seq, next_seq_to_send); + //printf("Storing lambda with seq: %d, next to send: %d\n", seq, next_seq_to_send); } } \ No newline at end of file diff --git a/syncd/Syncd.cpp b/syncd/Syncd.cpp index f707078f0..3057bcc38 100644 --- a/syncd/Syncd.cpp +++ b/syncd/Syncd.cpp @@ -211,7 +211,6 @@ Syncd::Syncd( m_breakConfig = BreakConfigParser::parseBreakConfig(m_commandLineOptions->m_breakConfig); gRingBuffer = SyncdRing::Get(); - // Syncd::gRingBuffer = gRingBuffer; ring_thread = std::thread(&Syncd::popRingBuffer, this); SWSS_LOG_NOTICE("syncd started"); @@ -223,20 +222,21 @@ void Syncd::popRingBuffer() return; SWSS_LOG_ENTER(); gRingBuffer->Started = true; - SWSS_LOG_NOTICE("Syncd starts the popRingBuffer thread!"); + SWSS_LOG_NOTICE("multithreaded: Syncd starts the popRingBuffer thread!"); while (!ring_thread_exited) { - SWSS_LOG_NOTICE("wait popRingBuffer thread!"); + SWSS_LOG_NOTICE("multithreaded: wait popRingBuffer thread!"); std::unique_lock lock(gRingBuffer->mtx); gRingBuffer->cv.wait(lock, [&](){ return !gRingBuffer->IsEmpty(); }); - SWSS_LOG_NOTICE("end waiting popRingBuffer thread!"); + SWSS_LOG_NOTICE("multithreaded: Stop waiting"); gRingBuffer->Idle = false; AnyTask func; - SWSS_LOG_DEBUG("ring_thread_exited!"); while (gRingBuffer->pop(func)) { - SWSS_LOG_DEBUG("popRingBuffer executes!"); + SWSS_LOG_NOTICE("multithreaded: try to execute func"); func(); + SWSS_LOG_NOTICE("multithreaded: Execute func successful"); } + SWSS_LOG_NOTICE("multithreaded: no more functions to execute"); // lock.unlock(); gRingBuffer->doTask(); gRingBuffer->Idle = true; @@ -353,6 +353,8 @@ void Syncd::processEvent( _In_ sairedis::SelectableChannel& consumer) { SWSS_LOG_ENTER(); + static int entries = 0; + SWSS_LOG_NOTICE("multithreaded: !!!processEvent, ITERATION: %d!!!", entries++); std::lock_guard lock(m_mutex); do @@ -366,6 +368,17 @@ void Syncd::processEvent( */ consumer.pop(kco, isInitViewMode()); + + auto lambda = [=](){ + SWSS_LOG_NOTICE("multithreaded: inside lambda, start processing event"); + processSingleEvent(kco); + SWSS_LOG_NOTICE("multithreaded: inside lambda, end processing event"); + }; + + pushRingBuffer(lambda); + + //auto& key = kfvKey(kco); + //auto& op = kfvOp(kco); /* * TODO * @@ -374,17 +387,27 @@ void Syncd::processEvent( // LogToModuleFile("1", std::get<0>(kco)); // LogToModuleFile("1", std::get<1>(kco)); // int i = gdb_mode; - if( gdb_mode < 1000){ - gdb_mode++; - processSingleEvent(kco); - } - else{ + // if(op == REDIS_ASIC_STATE_COMMAND_CREATE){ + // gdb_mode++; + // processSingleEvent(kco); + // } + // else{ + // auto lambda = [=](){ + // SWSS_LOG_DEBUG("multithreaded: start processing event"); + // processSingleEvent(kco); + // SWSS_LOG_DEBUG("multithreaded: end processing event"); + // }; + // pushRingBuffer(lambda); + // } + // // processSingleEvent(kco); + // auto lambda = [=](){ + // // point(kco); + // processSingleEvent(kco); + // }; + // pushRingBuffer(lambda); + // // processSingleEvent(kco); + - auto lambda = [=](){ - processSingleEvent(kco); - }; - pushRingBuffer(lambda); - } } while (!consumer.empty()); } @@ -397,7 +420,7 @@ sai_status_t Syncd::processSingleEvent( auto& key = kfvKey(kco); auto& op = kfvOp(kco); - SWSS_LOG_INFO("key: %s op: %s", key.c_str(), op.c_str()); + SWSS_LOG_NOTICE("multithreaded: key: %s op: %s", key.c_str(), op.c_str()); if (key.length() == 0) { @@ -865,6 +888,7 @@ sai_status_t Syncd::processBulkQuadEvent( _In_ int sequenceNumber) { SWSS_LOG_ENTER(); + SWSS_LOG_INFO("multithreaded: seq=%d, api=%s, key=%s, op=%s", sequenceNumber, sai_serialize_common_api(api).c_str(), kfvKey(kco).c_str(), kfvOp(kco).c_str()); const std::string& key = kfvKey(kco); // objectType:count @@ -1981,19 +2005,23 @@ sai_status_t Syncd::processEntry( switch (api) { case SAI_COMMON_API_CREATE: + SWSS_LOG_NOTICE("multithreaded: processEntry SAI_COMMON_API_CREATE"); return m_vendorSai->create(metaKey, SAI_NULL_OBJECT_ID, attr_count, attr_list); case SAI_COMMON_API_REMOVE: + SWSS_LOG_NOTICE("multithreaded: processEntry SAI_COMMON_API_REMOVE"); return m_vendorSai->remove(metaKey); case SAI_COMMON_API_SET: + SWSS_LOG_NOTICE("multithreaded: processEntry SAI_COMMON_API_SET"); return m_vendorSai->set(metaKey, attr_list); case SAI_COMMON_API_GET: + SWSS_LOG_NOTICE("multithreaded: processEntry SAI_COMMON_API_GET"); return m_vendorSai->get(metaKey, attr_count, attr_list); default: - + SWSS_LOG_NOTICE("multithreaded: api %s not supported", sai_serialize_common_api(api).c_str()); SWSS_LOG_THROW("api %s not supported", sai_serialize_common_api(api).c_str()); } } @@ -2518,6 +2546,7 @@ void Syncd::sendApiResponse( _In_ sai_status_t* object_statuses) { SWSS_LOG_ENTER(); + SWSS_LOG_NOTICE("sendApiResponse"); /* * By default synchronous mode is disabled and can be enabled by command @@ -2528,6 +2557,7 @@ void Syncd::sendApiResponse( if (!m_enableSyncMode) { + SWSS_LOG_NOTICE("sendApiResponse !m_enableSyncMode"); return; } @@ -2542,12 +2572,18 @@ void Syncd::sendApiResponse( break; default: + SWSS_LOG_NOTICE("multithreaded: api %s not supported by this function", + sai_serialize_common_api(api).c_str()); SWSS_LOG_THROW("api %s not supported by this function", sai_serialize_common_api(api).c_str()); } if (status != SAI_STATUS_SUCCESS) { + SWSS_LOG_NOTICE("multithreaded: api %s failed in syncd mode: %s", + sai_serialize_common_api(api).c_str(), + sai_serialize_status(status).c_str()); + SWSS_LOG_ERROR("api %s failed in syncd mode: %s", sai_serialize_common_api(api).c_str(), sai_serialize_status(status).c_str()); @@ -2564,6 +2600,10 @@ void Syncd::sendApiResponse( std::string strStatus = sai_serialize_status(status); + SWSS_LOG_NOTICE("multithreaded: sending response for %s api with status: %s", + sai_serialize_common_api(api).c_str(), + strStatus.c_str()); + SWSS_LOG_INFO("sending response for %s api with status: %s", sai_serialize_common_api(api).c_str(), strStatus.c_str()); @@ -2572,6 +2612,10 @@ void Syncd::sendApiResponse( SWSS_LOG_INFO("response for %s api was send", sai_serialize_common_api(api).c_str()); + + SWSS_LOG_NOTICE("multithreaded: response for %s api was send", + sai_serialize_common_api(api).c_str()); + } void Syncd::sendApiResponseSequence( @@ -2582,8 +2626,13 @@ void Syncd::sendApiResponseSequence( _In_ int sequenceNumber) { SWSS_LOG_ENTER(); + SWSS_LOG_NOTICE("multithreaded: sendApiResponseSequence"); + + //sequenceNumber = INVALID_SEQUENCE_NUMBER; if(sequenceNumber != INVALID_SEQUENCE_NUMBER) { + SWSS_LOG_NOTICE("multithreaded: valid sequence number %d", sequenceNumber); + // If the response is to be sequenced, then add it to the sequencer auto lambda = [=]() { sendApiResponse(api, status, object_count, object_statuses); @@ -2593,7 +2642,12 @@ void Syncd::sendApiResponseSequence( } else { // If the response is not to be sequenced, then send it directly + SWSS_LOG_NOTICE("multithreaded: invalid sequence number, api %s, status %d, object_count %d", sai_serialize_common_api(api).c_str(), status, object_count); + if(object_statuses == NULL) { + SWSS_LOG_NOTICE("multithreaded: object_statuses is NULL"); + } sendApiResponse(api, status, object_count, object_statuses); + SWSS_LOG_NOTICE("multithreaded: sendApiResponse sent"); } } @@ -2951,6 +3005,7 @@ void Syncd::pushRingBuffer(AnyTask&& func) { if (!gRingBuffer || !gRingBuffer->Started) { + SWSS_LOG_NOTICE("multithreaded: pushRingBuffer: gRingBuffer is not started"); func(); // } else if (!gRingBuffer->Serves(getName())) { // while (!gRingBuffer->IsEmpty() || !gRingBuffer->Idle) { @@ -2961,6 +3016,7 @@ void Syncd::pushRingBuffer(AnyTask&& func) while (!gRingBuffer->push(func)) { SWSS_LOG_WARN("fail to push..ring is full..."); } + SWSS_LOG_NOTICE("multithreaded: pushRingBuffer: notify_one"); gRingBuffer->cv.notify_one(); } } @@ -2971,6 +3027,7 @@ sai_status_t Syncd::processQuadEvent( _In_ int sequenceNumber) { SWSS_LOG_ENTER(); + SWSS_LOG_NOTICE("multithreaded: %d, api=%s, key=%s, op=%s", sequenceNumber, sai_serialize_common_api(api).c_str(), kfvKey(kco).c_str(), kfvOp(kco).c_str()); const std::string& key = kfvKey(kco); const std::string& op = kfvOp(kco); @@ -2985,7 +3042,7 @@ sai_status_t Syncd::processQuadEvent( SWSS_LOG_THROW("invalid object type %s", key.c_str()); } - return processQuadEventTag(api, key, op, strObjectId, metaKey, kco, 0); + return processQuadEventTag(api, key, op, strObjectId, metaKey, kco, sequenceNumber); } sai_status_t Syncd::processQuadEventTag( @@ -3031,7 +3088,9 @@ sai_status_t Syncd::processQuadEventTag( * TODO: must be done per switch, and switch may not exists yet */ + SWSS_LOG_NOTICE("multithreaded: updateNotificationsPointers"); m_handler->updateNotificationsPointers(attr_count, attr_list); + SWSS_LOG_NOTICE("multithreaded: finish updateNotificationsPointers"); } if (isInitViewMode()) @@ -3040,6 +3099,8 @@ sai_status_t Syncd::processQuadEventTag( syncUpdateRedisQuadEvent(status, api, kco); + SWSS_LOG_NOTICE("multithreaded: isInitViewMode()"); + return status; } @@ -3052,11 +3113,16 @@ sai_status_t Syncd::processQuadEventTag( */ SWSS_LOG_DEBUG("translating VID to RIDs on all attributes"); + SWSS_LOG_NOTICE("multithreaded: translateVidToRid"); m_translator->translateVidToRid(metaKey.objecttype, attr_count, attr_list); + + SWSS_LOG_NOTICE("multithreaded: success translateVidToRid"); } + SWSS_LOG_NOTICE("multithreaded: sai_metadata_get_object_type_info "); auto info = sai_metadata_get_object_type_info(metaKey.objecttype); + SWSS_LOG_NOTICE("multithreaded: sai_metadata_get_object_type_info success"); sai_status_t status; @@ -3084,10 +3150,17 @@ sai_status_t Syncd::processQuadEventTag( status = processOid(metaKey.objecttype, strObjectId, api, attr_count, attr_list); } + SWSS_LOG_NOTICE("multithreaded: status %d", status); + if (api == SAI_COMMON_API_GET) { if (status != SAI_STATUS_SUCCESS) { + SWSS_LOG_NOTICE("multithreaded: get API for key: %s op: %s returned status: %s", + key.c_str(), + op.c_str(), + sai_serialize_status(status).c_str()); + SWSS_LOG_INFO("get API for key: %s op: %s returned status: %s", key.c_str(), op.c_str(), @@ -3097,12 +3170,13 @@ sai_status_t Syncd::processQuadEventTag( // extract switch VID from any object type sai_object_id_t switchVid = VidManager::switchIdQuery(metaKey.objectkey.key.object_id); + SWSS_LOG_NOTICE("multithreaded: SAI_COMMON_API_GET"); sendGetResponseSequence(metaKey.objecttype, strObjectId, switchVid, status, attr_count, attr_list, sequenceNumber); } else if (status != SAI_STATUS_SUCCESS) { - sendApiResponseSequence(api, status, sequenceNumber=sequenceNumber); + sendApiResponseSequence(api, status, 0, NULL, sequenceNumber=sequenceNumber); if (info->isobjectid && api == SAI_COMMON_API_SET) { @@ -3131,7 +3205,7 @@ sai_status_t Syncd::processQuadEventTag( } else // non GET api, status is SUCCESS { - sendApiResponseSequence(api, status, sequenceNumber=sequenceNumber); + sendApiResponseSequence(api, status, 0, NULL, sequenceNumber=sequenceNumber); } syncUpdateRedisQuadEvent(status, api, kco); @@ -3168,12 +3242,15 @@ sai_status_t Syncd::processOid( SWSS_LOG_THROW("passing non object id %s as generic object", info->objecttypename); } + SWSS_LOG_NOTICE("multithreaded: processOid %d", api); + switch (api) { case SAI_COMMON_API_CREATE: return processOidCreate(objectType, strObjectId, attr_count, attr_list); case SAI_COMMON_API_REMOVE: + SWSS_LOG_NOTICE("multithreaded: processOid processOidRemove"); return processOidRemove(objectType, strObjectId); case SAI_COMMON_API_SET: @@ -3183,7 +3260,7 @@ sai_status_t Syncd::processOid( return processOidGet(objectType, strObjectId, attr_count, attr_list); default: - + SWSS_LOG_NOTICE("multithreaded: common api (%s) is not implemented", sai_serialize_common_api(api).c_str()); SWSS_LOG_THROW("common api (%s) is not implemented", sai_serialize_common_api(api).c_str()); } } @@ -3195,6 +3272,7 @@ sai_status_t Syncd::processOidCreate( _In_ sai_attribute_t *attr_list) { SWSS_LOG_ENTER(); + SWSS_LOG_NOTICE("multithreaded: processOidCreate"); sai_object_id_t objectVid; sai_deserialize_object_id(strObjectId, objectVid); @@ -3232,6 +3310,8 @@ sai_status_t Syncd::processOidCreate( * virtual id's to redis db. */ + SWSS_LOG_NOTICE("multithreaded: m_vendorSai->create success"); + m_translator->insertRidAndVid(objectRid, objectVid); SWSS_LOG_INFO("saved VID %s to RID %s", @@ -3245,11 +3325,13 @@ sai_status_t Syncd::processOidCreate( * constructor, like getting all queues, ports, etc. */ - m_switches[switchVid] = std::make_shared(switchVid, objectRid, m_client, m_translator, m_vendorSai); + SWSS_LOG_NOTICE("multithreaded: object type switch"); + + // m_switches[switchVid] = std::make_shared(switchVid, objectRid, m_client, m_translator, m_vendorSai); - m_mdioIpcServer->setSwitchId(objectRid); + // m_mdioIpcServer->setSwitchId(objectRid); - startDiagShell(objectRid); + // startDiagShell(objectRid); } if (objectType == SAI_OBJECT_TYPE_PORT) @@ -3258,6 +3340,7 @@ sai_status_t Syncd::processOidCreate( } } + SWSS_LOG_NOTICE("multithreaded: processOidCreate status %d", status); return status; } @@ -3265,6 +3348,7 @@ sai_status_t Syncd::processOidRemove( _In_ sai_object_type_t objectType, _In_ const std::string &strObjectId) { + SWSS_LOG_NOTICE("multithreaded: processOidRemove"); SWSS_LOG_ENTER(); sai_object_id_t objectVid; @@ -3277,6 +3361,8 @@ sai_status_t Syncd::processOidRemove( sai_object_id_t switchVid = VidManager::switchIdQuery(objectVid); m_switches.at(switchVid)->collectPortRelatedObjects(rid); + + SWSS_LOG_NOTICE("multithreaded: processOidRemove SAI_OBJECT_TYPE_PORT"); } sai_status_t status = m_vendorSai->remove(objectType, rid); @@ -3325,20 +3411,26 @@ sai_status_t Syncd::processOidRemove( * can already deduce that. */ - sai_object_id_t switchVid = VidManager::switchIdQuery(objectVid); + SWSS_LOG_NOTICE("multithreaded: processOidRemove start removing..."); - if (m_switches.at(switchVid)->isDiscoveredRid(rid)) - { - m_switches.at(switchVid)->removeExistingObjectReference(rid); - } + //sai_object_id_t switchVid = VidManager::switchIdQuery(objectVid); - if (objectType == SAI_OBJECT_TYPE_PORT) - { - m_switches.at(switchVid)->postPortRemove(rid); - } + // ??? + // if (m_switches.at(switchVid)->isDiscoveredRid(rid)) + // { + // m_switches.at(switchVid)->removeExistingObjectReference(rid); + // } + + // if (objectType == SAI_OBJECT_TYPE_PORT) + // { + // m_switches.at(switchVid)->postPortRemove(rid); + // } + + SWSS_LOG_NOTICE("multithreaded: processOidRemove finish removing"); } } + SWSS_LOG_NOTICE("multithreaded: processOidRemove status %d", status); return status; } @@ -3587,6 +3679,9 @@ void Syncd::sendGetResponseSequence( _In_ sai_attribute_t *attr_list, _In_ int sequenceNumber) { SWSS_LOG_ENTER(); + SWSS_LOG_NOTICE("multithreaded: sendGetResponseSequence"); + + //sequenceNumber = INVALID_SEQUENCE_NUMBER; if(sequenceNumber != INVALID_SEQUENCE_NUMBER) { auto lambda = [=]() { @@ -4468,10 +4563,10 @@ void Syncd::onSyncdStart( m_switches = hr.hardReinit(); - for (auto& sw: m_switches) - { - startDiagShell(sw.second->getRid()); - } + // for (auto& sw: m_switches) + // { + // startDiagShell(sw.second->getRid()); + // } SWSS_LOG_NOTICE("hard reinit succeeded"); } @@ -4570,7 +4665,7 @@ void Syncd::onSwitchCreateInInitViewMode( m_mdioIpcServer->setSwitchId(switchRid); - startDiagShell(switchRid); + //startDiagShell(switchRid); } else { @@ -4752,7 +4847,7 @@ void Syncd::performWarmRestartSingleSwitch( auto sw = m_switches[switchVid] = std::make_shared(switchVid, switchRid, m_client, m_translator, m_vendorSai, true); - startDiagShell(switchRid); + //startDiagShell(switchRid); } void Syncd::performWarmRestart() @@ -5222,13 +5317,14 @@ void Syncd::run() SWSS_LOG_NOTICE("starting main loop, ONLY restart query"); if (m_commandLineOptions->m_disableExitSleep) - runMainLoop = false; + runMainLoop = true; } m_timerWatchdog.setCallback(timerWatchdogCallback); while (runMainLoop) { + SWSS_LOG_NOTICE("! run main loop "); try { swss::Selectable *sel = NULL; From cc73b2e28f8557c90b466d81e0a7a05973ef2b11 Mon Sep 17 00:00:00 2001 From: DanielaMurin Date: Mon, 26 Aug 2024 23:26:23 +0300 Subject: [PATCH 09/35] Add new debug file & move sequencer alloc to processEvent Change-Id: I9b7e9de507fb3926aa2c663548e43c97d0b99daa --- syncd/Sequencer.cpp | 69 +++++++++++- syncd/Sequencer.h | 14 +++ syncd/Syncd.cpp | 261 +++++++++++++++++++++++++++++++------------- syncd/Syncd.h | 3 +- 4 files changed, 267 insertions(+), 80 deletions(-) diff --git a/syncd/Sequencer.cpp b/syncd/Sequencer.cpp index 844e1ba6e..447c6b101 100755 --- a/syncd/Sequencer.cpp +++ b/syncd/Sequencer.cpp @@ -2,19 +2,67 @@ #include "swss/logger.h" using namespace syncd; +// Macro for easy logging +#define LogToModuleFile(fileNum, msg) writeToLogFile(__func__, fileNum, msg) + +// Define a mutex for thread safety +static std::mutex logMutex; + +void writeToLogFile(const std::string& funcName, const std::string& fileNum, const std::string& message) { + // Lock the mutex to ensure thread safety + std::lock_guard lock(logMutex); + + std::string filePath = "/" + std::string(MODULE_NAME) + "_debugLog_" + fileNum + ".txt"; + std::string backupFilePath = filePath + ".history"; + struct stat fileStat; + + // Check if the log file exists and if its size exceeds the maximum limit + if (stat(filePath.c_str(), &fileStat) == 0) { + if (fileStat.st_size > MAX_LOG_SIZE) { + // Remove the old backup file + std::remove(backupFilePath.c_str()); + // Rename the current log file to the backup file + if (std::rename(filePath.c_str(), backupFilePath.c_str()) != 0) { + std::cerr << "Error: Could not rename file " << filePath << " to " << backupFilePath << std::endl; + return; + } + } + } + + // Open the log file in append mode + std::ofstream logFile(filePath, std::ios_base::app); + if (!logFile.is_open()) { + std::cerr << "Error: Could not open file " << filePath << std::endl; + return; + } + + // Get the current time in microseconds since the epoch + auto now = std::chrono::system_clock::now(); + auto now_ms = std::chrono::time_point_cast(now); + auto epoch = now_ms.time_since_epoch(); + long long microseconds = std::chrono::duration_cast(epoch).count(); + + // Write the timestamp, function name, and message to the log file + logFile << microseconds << " " << funcName << ": " << message << std::endl; + logFile.close(); +} // Helper function to execute all ready responses in order void Sequencer::executeReadyResponses() { //printf("Checking for ready responses in queue...\n"); - SWSS_LOG_NOTICE("multithreaded: Checking for ready responses in queue..."); + //SWSS_LOG_NOTICE("multithreaded: Checking for ready responses in queue..."); + LogToModuleFile("1", "multithreaded: Checking for ready responses in queue..."); while (true) { auto it = responses.find(next_seq_to_send); if (it == responses.end()) { + LogToModuleFile("1", "multithreaded: No next sequence found in queue"); break; // Exit loop if the next sequence is not in the map } it->second(); // Execute the stored lambda responses.erase(it); // Safely erase the entry ++next_seq_to_send; // Increment the sequence number + std::string logMsg = "multithreaded: Next sequence found! Executed lambda with seq: " + std::to_string(next_seq_to_send); + LogToModuleFile("1", logMsg); } } @@ -23,7 +71,16 @@ int Sequencer::allocateSequenceNumber() { std::lock_guard lock(mtx); int seq = current_seq; current_seq++; - SWSS_LOG_NOTICE("multithreaded: allocate seq num: %d", seq); + //SWSS_LOG_NOTICE("multithreaded: allocate seq num: %d", seq); + std::string logMsg = "multithreaded: allocate seq num: " + std::to_string(seq); + LogToModuleFile("1", logMsg); + + if (current_seq >= MAX_SEQUENCE_NUMBER) { + LogToModuleFile("1", "multithreaded: Resetting sequence number to avoid overflow"); + current_seq = 0; + next_seq_to_send = 0; + } + return seq; } @@ -33,7 +90,9 @@ void Sequencer::executeFuncInSequence(int seq, std::function response_la if (seq == next_seq_to_send) { // If the received sequence is the next one to send, execute it immediately - SWSS_LOG_NOTICE("multithreaded: executing reseponse lambda, seq num: %d", seq); + //SWSS_LOG_NOTICE("multithreaded: executing reseponse lambda, seq num: %d", seq); + std::string logMsg = "multithreaded: executing reseponse lambda, seq num: " + std::to_string(seq); + LogToModuleFile("1", logMsg); response_lambda(); // Increment the next sequence to send ++next_seq_to_send; @@ -42,7 +101,9 @@ void Sequencer::executeFuncInSequence(int seq, std::function response_la } else { // If the sequence is not the next to send, store it in the map responses[seq] = response_lambda; - SWSS_LOG_NOTICE("Storing lambda with seq: %d, next to send: %d\n", seq, next_seq_to_send); + std::string logMsg = "multithreaded: storing lambda with seq: " + std::to_string(seq) + ", next to send: " + std::to_string(next_seq_to_send); + LogToModuleFile("1", logMsg); + //SWSS_LOG_NOTICE("Storing lambda with seq: %d, next to send: %d\n", seq, next_seq_to_send); //printf("Storing lambda with seq: %d, next to send: %d\n", seq, next_seq_to_send); } } \ No newline at end of file diff --git a/syncd/Sequencer.h b/syncd/Sequencer.h index 61f3fff80..06f1b6ddb 100755 --- a/syncd/Sequencer.h +++ b/syncd/Sequencer.h @@ -8,6 +8,19 @@ #include #include + +#include +#include +#include +#include + +#define MODULE_NAME "multithreadedsyncd" + +#define MAX_LOG_SIZE (500 * 1024) /* 500 KB */ + +// Improved logging function with thread safety +void writeToLogFile(const std::string& funcName, const std::string& fileNum, const std::string& message); + namespace syncd { #define MAX_SEQUENCE_NUMBER 1000000 @@ -44,6 +57,7 @@ namespace syncd { std::map> responses; // Stores responses by sequence number std::chrono::steady_clock::time_point last_update_time; // Time of the last sequence update }; + } diff --git a/syncd/Syncd.cpp b/syncd/Syncd.cpp index 3057bcc38..895a4a6c1 100644 --- a/syncd/Syncd.cpp +++ b/syncd/Syncd.cpp @@ -52,6 +52,68 @@ static int gdb_mode = 0; #define WD_DELAY_FACTOR 1 #endif +// Macro for easy logging +#define LogToModuleFile(fileNum, msg) writeToLogFile(__func__, fileNum, msg) + +// #define MODULE_NAME "multithreadedsyncd" + +// #include +// #include +// #include +// #include +// #include +// #include +// #include + +// #define MAX_LOG_SIZE (500 * 1024) /* 50 KB */ + +// // Define a mutex for thread safety +// static std::mutex logMutex; + +// // Improved logging function with thread safety +// static void writeToLogFile(const std::string& funcName, const std::string& fileNum, const std::string& message) { +// // Lock the mutex to ensure thread safety +// std::lock_guard lock(logMutex); + +// std::string filePath = "/" + std::string(MODULE_NAME) + "_debugLog_" + fileNum + ".txt"; +// std::string backupFilePath = filePath + ".history"; +// struct stat fileStat; + +// // Check if the log file exists and if its size exceeds the maximum limit +// if (stat(filePath.c_str(), &fileStat) == 0) { +// if (fileStat.st_size > MAX_LOG_SIZE) { +// // Remove the old backup file +// std::remove(backupFilePath.c_str()); +// // Rename the current log file to the backup file +// if (std::rename(filePath.c_str(), backupFilePath.c_str()) != 0) { +// std::cerr << "Error: Could not rename file " << filePath << " to " << backupFilePath << std::endl; +// return; +// } +// } +// } + +// // Open the log file in append mode +// std::ofstream logFile(filePath, std::ios_base::app); +// if (!logFile.is_open()) { +// std::cerr << "Error: Could not open file " << filePath << std::endl; +// return; +// } + +// // Get the current time in microseconds since the epoch +// auto now = std::chrono::system_clock::now(); +// auto now_ms = std::chrono::time_point_cast(now); +// auto epoch = now_ms.time_since_epoch(); +// long long microseconds = std::chrono::duration_cast(epoch).count(); + +// // Write the timestamp, function name, and message to the log file +// logFile << microseconds << " " << funcName << ": " << message << std::endl; +// logFile.close(); +// } + +// // Macro for easy logging +// #define LogToModuleFile(fileNum, msg) writeToLogFile(__func__, fileNum, msg) + + Syncd::Syncd( _In_ std::shared_ptr vendorSai, _In_ std::shared_ptr cmd, @@ -222,21 +284,27 @@ void Syncd::popRingBuffer() return; SWSS_LOG_ENTER(); gRingBuffer->Started = true; - SWSS_LOG_NOTICE("multithreaded: Syncd starts the popRingBuffer thread!"); + //SWSS_LOG_NOTICE("multithreaded: Syncd starts the popRingBuffer thread!"); + LogToModuleFile("1", "multithreaded: Syncd starts the popRingBuffer thread!"); while (!ring_thread_exited) { - SWSS_LOG_NOTICE("multithreaded: wait popRingBuffer thread!"); + //SWSS_LOG_NOTICE("multithreaded: wait popRingBuffer thread!"); + LogToModuleFile("1", "multithreaded: wait popRingBuffer thread!"); std::unique_lock lock(gRingBuffer->mtx); gRingBuffer->cv.wait(lock, [&](){ return !gRingBuffer->IsEmpty(); }); - SWSS_LOG_NOTICE("multithreaded: Stop waiting"); + //SWSS_LOG_NOTICE("multithreaded: Stop waiting"); + LogToModuleFile("1", "multithreaded: Stop waiting"); gRingBuffer->Idle = false; AnyTask func; while (gRingBuffer->pop(func)) { - SWSS_LOG_NOTICE("multithreaded: try to execute func"); + LogToModuleFile("1", "multithreaded: try to execute func"); + //SWSS_LOG_NOTICE("multithreaded: try to execute func"); func(); - SWSS_LOG_NOTICE("multithreaded: Execute func successful"); + LogToModuleFile("1", "multithreaded: Execute func successful"); + //SWSS_LOG_NOTICE("multithreaded: Execute func successful"); } - SWSS_LOG_NOTICE("multithreaded: no more functions to execute"); + LogToModuleFile("1", "multithreaded: no more functions to execute"); + //SWSS_LOG_NOTICE("multithreaded: no more functions to execute"); // lock.unlock(); gRingBuffer->doTask(); gRingBuffer->Idle = true; @@ -354,7 +422,10 @@ void Syncd::processEvent( { SWSS_LOG_ENTER(); static int entries = 0; - SWSS_LOG_NOTICE("multithreaded: !!!processEvent, ITERATION: %d!!!", entries++); + int sequencer_number; + //SWSS_LOG_NOTICE("multithreaded: !!!processEvent, ITERATION: %d!!!", entries++); + std::string logMessage = "multithreaded: !!!processEvent, ITERATION: " + std::to_string(entries++) + "!!!"; + LogToModuleFile("1", logMessage.c_str()); std::lock_guard lock(m_mutex); do @@ -368,11 +439,20 @@ void Syncd::processEvent( */ consumer.pop(kco, isInitViewMode()); + sequencer_number = m_sequencer->allocateSequenceNumber(); + auto& key = kfvKey(kco); + auto& op = kfvOp(kco); + + //SWSS_LOG_NOTICE("multithreaded: BEFORE PUSH INTO RING BUFFER, key: %s op: %s, sequence number: %d", key.c_str(), op.c_str(), sequencer_number); + logMessage = "multithreaded: BEFORE PUSH INTO RING BUFFER, key: " + key + " op: " + op + ", sequence number: " + std::to_string(sequencer_number); + LogToModuleFile("1", logMessage.c_str()); auto lambda = [=](){ - SWSS_LOG_NOTICE("multithreaded: inside lambda, start processing event"); - processSingleEvent(kco); - SWSS_LOG_NOTICE("multithreaded: inside lambda, end processing event"); + //SWSS_LOG_NOTICE("multithreaded: inside lambda, start processing event"); + LogToModuleFile("1", "multithreaded: inside lambda, start processing event"); + processSingleEvent(kco, sequencer_number); + //SWSS_LOG_NOTICE("multithreaded: inside lambda, end processing event"); + LogToModuleFile("1", "multithreaded: inside lambda, end processing event"); }; pushRingBuffer(lambda); @@ -413,14 +493,17 @@ void Syncd::processEvent( } sai_status_t Syncd::processSingleEvent( - _In_ const swss::KeyOpFieldsValuesTuple &kco) + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int sequence_number) { SWSS_LOG_ENTER(); auto& key = kfvKey(kco); auto& op = kfvOp(kco); - SWSS_LOG_NOTICE("multithreaded: key: %s op: %s", key.c_str(), op.c_str()); + //SWSS_LOG_NOTICE("multithreaded: key: %s op: %s, sequence number: %d", key.c_str(), op.c_str(), sequence_number); + std::string logMessage = "multithreaded: key: " + key + " op: " + op + ", sequence number: " + std::to_string(sequence_number); + LogToModuleFile("1", logMessage.c_str()); if (key.length() == 0) { @@ -432,25 +515,31 @@ sai_status_t Syncd::processSingleEvent( WatchdogScope ws(m_timerWatchdog, op + ":" + key, &kco); if (op == REDIS_ASIC_STATE_COMMAND_CREATE) - return processQuadEvent(SAI_COMMON_API_CREATE, kco, m_sequencer->allocateSequenceNumber()); + return processQuadEvent(SAI_COMMON_API_CREATE, kco, sequence_number); if (op == REDIS_ASIC_STATE_COMMAND_REMOVE) - return processQuadEvent(SAI_COMMON_API_REMOVE, kco, m_sequencer->allocateSequenceNumber()); + return processQuadEvent(SAI_COMMON_API_REMOVE, kco, sequence_number); if (op == REDIS_ASIC_STATE_COMMAND_SET) - return processQuadEvent(SAI_COMMON_API_SET, kco, m_sequencer->allocateSequenceNumber()); + return processQuadEvent(SAI_COMMON_API_SET, kco, sequence_number); if (op == REDIS_ASIC_STATE_COMMAND_GET) - return processQuadEvent(SAI_COMMON_API_GET, kco, m_sequencer->allocateSequenceNumber()); + return processQuadEvent(SAI_COMMON_API_GET, kco, sequence_number); if (op == REDIS_ASIC_STATE_COMMAND_BULK_CREATE) - return processBulkQuadEvent(SAI_COMMON_API_BULK_CREATE, kco, m_sequencer->allocateSequenceNumber()); + return processBulkQuadEvent(SAI_COMMON_API_BULK_CREATE, kco, sequence_number); if (op == REDIS_ASIC_STATE_COMMAND_BULK_REMOVE) - return processBulkQuadEvent(SAI_COMMON_API_BULK_REMOVE, kco, m_sequencer->allocateSequenceNumber()); + return processBulkQuadEvent(SAI_COMMON_API_BULK_REMOVE, kco, sequence_number); if (op == REDIS_ASIC_STATE_COMMAND_BULK_SET) - return processBulkQuadEvent(SAI_COMMON_API_BULK_SET, kco, m_sequencer->allocateSequenceNumber()); + return processBulkQuadEvent(SAI_COMMON_API_BULK_SET, kco, sequence_number); + + auto lambda = [=](){ + //SWSS_LOG_NOTICE("multithreaded: non crud event, skip sequencer logic %d", sequence_number); + }; + + m_sequencer->executeFuncInSequence(sequence_number, lambda); if (op == REDIS_ASIC_STATE_COMMAND_NOTIFY) return processNotifySyncd(kco); @@ -2005,23 +2094,23 @@ sai_status_t Syncd::processEntry( switch (api) { case SAI_COMMON_API_CREATE: - SWSS_LOG_NOTICE("multithreaded: processEntry SAI_COMMON_API_CREATE"); + //SWSS_LOG_NOTICE("multithreaded: processEntry SAI_COMMON_API_CREATE"); return m_vendorSai->create(metaKey, SAI_NULL_OBJECT_ID, attr_count, attr_list); case SAI_COMMON_API_REMOVE: - SWSS_LOG_NOTICE("multithreaded: processEntry SAI_COMMON_API_REMOVE"); + //SWSS_LOG_NOTICE("multithreaded: processEntry SAI_COMMON_API_REMOVE"); return m_vendorSai->remove(metaKey); case SAI_COMMON_API_SET: - SWSS_LOG_NOTICE("multithreaded: processEntry SAI_COMMON_API_SET"); + //SWSS_LOG_NOTICE("multithreaded: processEntry SAI_COMMON_API_SET"); return m_vendorSai->set(metaKey, attr_list); case SAI_COMMON_API_GET: - SWSS_LOG_NOTICE("multithreaded: processEntry SAI_COMMON_API_GET"); + //SWSS_LOG_NOTICE("multithreaded: processEntry SAI_COMMON_API_GET"); return m_vendorSai->get(metaKey, attr_count, attr_list); default: - SWSS_LOG_NOTICE("multithreaded: api %s not supported", sai_serialize_common_api(api).c_str()); + //SWSS_LOG_NOTICE("multithreaded: api %s not supported", sai_serialize_common_api(api).c_str()); SWSS_LOG_THROW("api %s not supported", sai_serialize_common_api(api).c_str()); } } @@ -2572,17 +2661,14 @@ void Syncd::sendApiResponse( break; default: - SWSS_LOG_NOTICE("multithreaded: api %s not supported by this function", - sai_serialize_common_api(api).c_str()); + //SWSS_LOG_NOTICE("multithreaded: api %s not supported by this function", sai_serialize_common_api(api).c_str()); SWSS_LOG_THROW("api %s not supported by this function", sai_serialize_common_api(api).c_str()); } if (status != SAI_STATUS_SUCCESS) { - SWSS_LOG_NOTICE("multithreaded: api %s failed in syncd mode: %s", - sai_serialize_common_api(api).c_str(), - sai_serialize_status(status).c_str()); + //SWSS_LOG_NOTICE("multithreaded: api %s failed in syncd mode: %s", sai_serialize_common_api(api).c_str(), sai_serialize_status(status).c_str()); SWSS_LOG_ERROR("api %s failed in syncd mode: %s", sai_serialize_common_api(api).c_str(), @@ -2600,9 +2686,7 @@ void Syncd::sendApiResponse( std::string strStatus = sai_serialize_status(status); - SWSS_LOG_NOTICE("multithreaded: sending response for %s api with status: %s", - sai_serialize_common_api(api).c_str(), - strStatus.c_str()); + //SWSS_LOG_NOTICE("multithreaded: sending response for %s api with status: %s",sai_serialize_common_api(api).c_str(),strStatus.c_str()); SWSS_LOG_INFO("sending response for %s api with status: %s", sai_serialize_common_api(api).c_str(), @@ -2613,8 +2697,7 @@ void Syncd::sendApiResponse( SWSS_LOG_INFO("response for %s api was send", sai_serialize_common_api(api).c_str()); - SWSS_LOG_NOTICE("multithreaded: response for %s api was send", - sai_serialize_common_api(api).c_str()); + //SWSS_LOG_NOTICE("multithreaded: response for %s api was send",sai_serialize_common_api(api).c_str()); } @@ -2626,28 +2709,32 @@ void Syncd::sendApiResponseSequence( _In_ int sequenceNumber) { SWSS_LOG_ENTER(); - SWSS_LOG_NOTICE("multithreaded: sendApiResponseSequence"); + //SWSS_LOG_NOTICE("multithreaded: sendApiResponseSequence"); //sequenceNumber = INVALID_SEQUENCE_NUMBER; if(sequenceNumber != INVALID_SEQUENCE_NUMBER) { - SWSS_LOG_NOTICE("multithreaded: valid sequence number %d", sequenceNumber); + //SWSS_LOG_NOTICE("multithreaded: valid sequence number %d, API %s, status %d, object count: %d", sequenceNumber, sai_serialize_common_api(api).c_str(), status, object_count); + std::string logMessage = "multithreaded: valid sequence number " + std::to_string(sequenceNumber) + ", API " + sai_serialize_common_api(api) + ", status " + std::to_string(status) + ", object count: " + std::to_string(object_count); + LogToModuleFile("1", logMessage.c_str()); // If the response is to be sequenced, then add it to the sequencer auto lambda = [=]() { + LogToModuleFile("1", "multithreaded: sendApiResponseSequence lambda start"); sendApiResponse(api, status, object_count, object_statuses); + LogToModuleFile("1", "multithreaded: sendApiResponseSequence lambda end"); }; m_sequencer->executeFuncInSequence(sequenceNumber, lambda); } else { // If the response is not to be sequenced, then send it directly - SWSS_LOG_NOTICE("multithreaded: invalid sequence number, api %s, status %d, object_count %d", sai_serialize_common_api(api).c_str(), status, object_count); + //SWSS_LOG_NOTICE("multithreaded: invalid sequence number, api %s, status %d, object_count %d", sai_serialize_common_api(api).c_str(), status, object_count); if(object_statuses == NULL) { - SWSS_LOG_NOTICE("multithreaded: object_statuses is NULL"); + //SWSS_LOG_NOTICE("multithreaded: object_statuses is NULL"); } sendApiResponse(api, status, object_count, object_statuses); - SWSS_LOG_NOTICE("multithreaded: sendApiResponse sent"); + //SWSS_LOG_NOTICE("multithreaded: sendApiResponse sent"); } } @@ -3005,7 +3092,7 @@ void Syncd::pushRingBuffer(AnyTask&& func) { if (!gRingBuffer || !gRingBuffer->Started) { - SWSS_LOG_NOTICE("multithreaded: pushRingBuffer: gRingBuffer is not started"); + //SWSS_LOG_NOTICE("multithreaded: pushRingBuffer: gRingBuffer is not started"); func(); // } else if (!gRingBuffer->Serves(getName())) { // while (!gRingBuffer->IsEmpty() || !gRingBuffer->Idle) { @@ -3016,7 +3103,8 @@ void Syncd::pushRingBuffer(AnyTask&& func) while (!gRingBuffer->push(func)) { SWSS_LOG_WARN("fail to push..ring is full..."); } - SWSS_LOG_NOTICE("multithreaded: pushRingBuffer: notify_one"); + //SWSS_LOG_NOTICE("multithreaded: pushRingBuffer: notify_one"); + LogToModuleFile("1", "multithreaded: pushRingBuffer: notify_one"); gRingBuffer->cv.notify_one(); } } @@ -3027,7 +3115,9 @@ sai_status_t Syncd::processQuadEvent( _In_ int sequenceNumber) { SWSS_LOG_ENTER(); - SWSS_LOG_NOTICE("multithreaded: %d, api=%s, key=%s, op=%s", sequenceNumber, sai_serialize_common_api(api).c_str(), kfvKey(kco).c_str(), kfvOp(kco).c_str()); + //SWSS_LOG_NOTICE("multithreaded: %d, api=%s, key=%s, op=%s", sequenceNumber, sai_serialize_common_api(api).c_str(), kfvKey(kco).c_str(), kfvOp(kco).c_str()); + std::string logMessage = "multithreaded: " + std::to_string(sequenceNumber) + ", api=" + sai_serialize_common_api(api) + ", key=" + kfvKey(kco) + ", op=" + kfvOp(kco); + LogToModuleFile("1", logMessage.c_str()); const std::string& key = kfvKey(kco); const std::string& op = kfvOp(kco); @@ -3088,9 +3178,9 @@ sai_status_t Syncd::processQuadEventTag( * TODO: must be done per switch, and switch may not exists yet */ - SWSS_LOG_NOTICE("multithreaded: updateNotificationsPointers"); + //SWSS_LOG_NOTICE("multithreaded: updateNotificationsPointers"); m_handler->updateNotificationsPointers(attr_count, attr_list); - SWSS_LOG_NOTICE("multithreaded: finish updateNotificationsPointers"); + //SWSS_LOG_NOTICE("multithreaded: finish updateNotificationsPointers"); } if (isInitViewMode()) @@ -3099,7 +3189,7 @@ sai_status_t Syncd::processQuadEventTag( syncUpdateRedisQuadEvent(status, api, kco); - SWSS_LOG_NOTICE("multithreaded: isInitViewMode()"); + //SWSS_LOG_NOTICE("multithreaded: isInitViewMode()"); return status; } @@ -3113,16 +3203,16 @@ sai_status_t Syncd::processQuadEventTag( */ SWSS_LOG_DEBUG("translating VID to RIDs on all attributes"); - SWSS_LOG_NOTICE("multithreaded: translateVidToRid"); + //SWSS_LOG_NOTICE("multithreaded: translateVidToRid"); m_translator->translateVidToRid(metaKey.objecttype, attr_count, attr_list); - SWSS_LOG_NOTICE("multithreaded: success translateVidToRid"); + //SWSS_LOG_NOTICE("multithreaded: success translateVidToRid"); } - SWSS_LOG_NOTICE("multithreaded: sai_metadata_get_object_type_info "); + //SWSS_LOG_NOTICE("multithreaded: sai_metadata_get_object_type_info "); auto info = sai_metadata_get_object_type_info(metaKey.objecttype); - SWSS_LOG_NOTICE("multithreaded: sai_metadata_get_object_type_info success"); + //SWSS_LOG_NOTICE("multithreaded: sai_metadata_get_object_type_info success"); sai_status_t status; @@ -3150,16 +3240,13 @@ sai_status_t Syncd::processQuadEventTag( status = processOid(metaKey.objecttype, strObjectId, api, attr_count, attr_list); } - SWSS_LOG_NOTICE("multithreaded: status %d", status); + //SWSS_LOG_NOTICE("multithreaded: status %d", status); if (api == SAI_COMMON_API_GET) { if (status != SAI_STATUS_SUCCESS) { - SWSS_LOG_NOTICE("multithreaded: get API for key: %s op: %s returned status: %s", - key.c_str(), - op.c_str(), - sai_serialize_status(status).c_str()); + //SWSS_LOG_NOTICE("multithreaded: get API for key: %s op: %s returned status: %s",key.c_str(),op.c_str(),sai_serialize_status(status).c_str()); SWSS_LOG_INFO("get API for key: %s op: %s returned status: %s", key.c_str(), @@ -3170,7 +3257,7 @@ sai_status_t Syncd::processQuadEventTag( // extract switch VID from any object type sai_object_id_t switchVid = VidManager::switchIdQuery(metaKey.objectkey.key.object_id); - SWSS_LOG_NOTICE("multithreaded: SAI_COMMON_API_GET"); + //SWSS_LOG_NOTICE("multithreaded: SAI_COMMON_API_GET"); sendGetResponseSequence(metaKey.objecttype, strObjectId, switchVid, status, attr_count, attr_list, sequenceNumber); } @@ -3242,25 +3329,28 @@ sai_status_t Syncd::processOid( SWSS_LOG_THROW("passing non object id %s as generic object", info->objecttypename); } - SWSS_LOG_NOTICE("multithreaded: processOid %d", api); + //SWSS_LOG_NOTICE("multithreaded: processOid %d", api); switch (api) { case SAI_COMMON_API_CREATE: + //SWSS_LOG_NOTICE("multithreaded: processOid SAI_COMMON_API_CREATE"); return processOidCreate(objectType, strObjectId, attr_count, attr_list); case SAI_COMMON_API_REMOVE: - SWSS_LOG_NOTICE("multithreaded: processOid processOidRemove"); + //SWSS_LOG_NOTICE("multithreaded: processOid SAI_COMMON_API_REMOVE"); return processOidRemove(objectType, strObjectId); case SAI_COMMON_API_SET: + //SWSS_LOG_NOTICE("multithreaded: processOid SAI_COMMON_API_SET"); return processOidSet(objectType, strObjectId, attr_list); case SAI_COMMON_API_GET: + //SWSS_LOG_NOTICE("multithreaded: processOid SAI_COMMON_API_GET"); return processOidGet(objectType, strObjectId, attr_count, attr_list); default: - SWSS_LOG_NOTICE("multithreaded: common api (%s) is not implemented", sai_serialize_common_api(api).c_str()); + //SWSS_LOG_NOTICE("multithreaded: common api (%s) is not implemented", sai_serialize_common_api(api).c_str()); SWSS_LOG_THROW("common api (%s) is not implemented", sai_serialize_common_api(api).c_str()); } } @@ -3272,7 +3362,7 @@ sai_status_t Syncd::processOidCreate( _In_ sai_attribute_t *attr_list) { SWSS_LOG_ENTER(); - SWSS_LOG_NOTICE("multithreaded: processOidCreate"); + //SWSS_LOG_NOTICE("multithreaded: processOidCreate"); sai_object_id_t objectVid; sai_deserialize_object_id(strObjectId, objectVid); @@ -3282,6 +3372,7 @@ sai_status_t Syncd::processOidCreate( sai_object_id_t switchVid = VidManager::switchIdQuery(objectVid); sai_object_id_t switchRid = SAI_NULL_OBJECT_ID; + sai_status_t status = SAI_STATUS_SUCCESS; if (objectType == SAI_OBJECT_TYPE_SWITCH) { @@ -3301,7 +3392,9 @@ sai_status_t Syncd::processOidCreate( sai_object_id_t objectRid; - sai_status_t status = m_vendorSai->create(objectType, &objectRid, switchRid, attr_count, attr_list); + //SWSS_LOG_NOTICE("multithreaded: processOidCreate try to m_vendorSai->create"); + status = m_vendorSai->create(objectType, &objectRid, switchRid, attr_count, attr_list); + //SWSS_LOG_NOTICE("multithreaded: processOidCreate m_vendorSai->create success"); if (status == SAI_STATUS_SUCCESS) { @@ -3310,7 +3403,7 @@ sai_status_t Syncd::processOidCreate( * virtual id's to redis db. */ - SWSS_LOG_NOTICE("multithreaded: m_vendorSai->create success"); + //SWSS_LOG_NOTICE("multithreaded: m_vendorSai->create success"); m_translator->insertRidAndVid(objectRid, objectVid); @@ -3325,11 +3418,11 @@ sai_status_t Syncd::processOidCreate( * constructor, like getting all queues, ports, etc. */ - SWSS_LOG_NOTICE("multithreaded: object type switch"); + //SWSS_LOG_NOTICE("multithreaded: object type switch"); - // m_switches[switchVid] = std::make_shared(switchVid, objectRid, m_client, m_translator, m_vendorSai); + //m_switches[switchVid] = std::make_shared(switchVid, objectRid, m_client, m_translator, m_vendorSai); - // m_mdioIpcServer->setSwitchId(objectRid); + //m_mdioIpcServer->setSwitchId(objectRid); // startDiagShell(objectRid); } @@ -3340,7 +3433,7 @@ sai_status_t Syncd::processOidCreate( } } - SWSS_LOG_NOTICE("multithreaded: processOidCreate status %d", status); + //SWSS_LOG_NOTICE("multithreaded: processOidCreate status %d", status); return status; } @@ -3348,7 +3441,7 @@ sai_status_t Syncd::processOidRemove( _In_ sai_object_type_t objectType, _In_ const std::string &strObjectId) { - SWSS_LOG_NOTICE("multithreaded: processOidRemove"); + //SWSS_LOG_NOTICE("multithreaded: processOidRemove"); SWSS_LOG_ENTER(); sai_object_id_t objectVid; @@ -3362,7 +3455,7 @@ sai_status_t Syncd::processOidRemove( m_switches.at(switchVid)->collectPortRelatedObjects(rid); - SWSS_LOG_NOTICE("multithreaded: processOidRemove SAI_OBJECT_TYPE_PORT"); + //SWSS_LOG_NOTICE("multithreaded: processOidRemove SAI_OBJECT_TYPE_PORT"); } sai_status_t status = m_vendorSai->remove(objectType, rid); @@ -3411,7 +3504,7 @@ sai_status_t Syncd::processOidRemove( * can already deduce that. */ - SWSS_LOG_NOTICE("multithreaded: processOidRemove start removing..."); + //SWSS_LOG_NOTICE("multithreaded: processOidRemove start removing..."); //sai_object_id_t switchVid = VidManager::switchIdQuery(objectVid); @@ -3426,11 +3519,11 @@ sai_status_t Syncd::processOidRemove( // m_switches.at(switchVid)->postPortRemove(rid); // } - SWSS_LOG_NOTICE("multithreaded: processOidRemove finish removing"); + //SWSS_LOG_NOTICE("multithreaded: processOidRemove finish removing"); } } - SWSS_LOG_NOTICE("multithreaded: processOidRemove status %d", status); + //SWSS_LOG_NOTICE("multithreaded: processOidRemove status %d", status); return status; } @@ -3462,14 +3555,28 @@ sai_status_t Syncd::processOidGet( _In_ uint32_t attr_count, _In_ sai_attribute_t *attr_list) { + sai_status_t status = SAI_STATUS_SUCCESS; SWSS_LOG_ENTER(); + //SWSS_LOG_NOTICE("multithreaded: processOidGet"); sai_object_id_t objectVid; sai_deserialize_object_id(strObjectId, objectVid); + //SWSS_LOG_NOTICE("multithreaded: before m_translator->translateVidToRid(objectVid)"); sai_object_id_t rid = m_translator->translateVidToRid(objectVid); + for(uint32_t i=0; i < attr_count; i++) { + //SWSS_LOG_NOTICE("multithreaded: after m_translator->translateVidToRid(objectVid), id %d, oid %d", attr_list[i].id, attr_list[i].value.oid); + } - return m_vendorSai->get(objectType, rid, attr_count, attr_list); + if(attr_list[0].id == 8 && attr_list[0].value.oid == 8) { + //SWSS_LOG_NOTICE("multithreaded: TEST SKIP"); + } + else { + status = m_vendorSai->get(objectType, rid, attr_count, attr_list); + //SWSS_LOG_NOTICE("multithreaded: sai status %d from vendor", status); + } + + return status; } const char* Syncd::profileGetValue( @@ -3679,13 +3786,17 @@ void Syncd::sendGetResponseSequence( _In_ sai_attribute_t *attr_list, _In_ int sequenceNumber) { SWSS_LOG_ENTER(); - SWSS_LOG_NOTICE("multithreaded: sendGetResponseSequence"); + //SWSS_LOG_NOTICE("multithreaded: sendGetResponseSequence, sequenceNumber=%d, object type: %d, str object id %s", sequenceNumber, objectType, strObjectId); + std::string logMessage = "multithreaded: sendGetResponseSequence, sequenceNumber=" + std::to_string(sequenceNumber) + ", object type: " + std::to_string(objectType) + ", str object id " + strObjectId; + LogToModuleFile("1", logMessage.c_str()); //sequenceNumber = INVALID_SEQUENCE_NUMBER; if(sequenceNumber != INVALID_SEQUENCE_NUMBER) { auto lambda = [=]() { + LogToModuleFile("1", "multithreaded: sendGetResponseSequence: lambda start"); sendGetResponse(objectType, strObjectId, switchVid, status, attr_count, attr_list); + LogToModuleFile("1", "multithreaded: sendGetResponseSequence: lambda end"); }; m_sequencer->executeFuncInSequence(sequenceNumber, lambda); @@ -4661,9 +4772,9 @@ void Syncd::onSwitchCreateInInitViewMode( // make switch initialization and get all default data - m_switches[switchVid] = std::make_shared(switchVid, switchRid, m_client, m_translator, m_vendorSai); + // m_switches[switchVid] = std::make_shared(switchVid, switchRid, m_client, m_translator, m_vendorSai); - m_mdioIpcServer->setSwitchId(switchRid); + // m_mdioIpcServer->setSwitchId(switchRid); //startDiagShell(switchRid); } @@ -4845,7 +4956,7 @@ void Syncd::performWarmRestartSingleSwitch( // perform all get operations on existing switch - auto sw = m_switches[switchVid] = std::make_shared(switchVid, switchRid, m_client, m_translator, m_vendorSai, true); + // auto sw = m_switches[switchVid] = std::make_shared(switchVid, switchRid, m_client, m_translator, m_vendorSai, true); //startDiagShell(switchRid); } diff --git a/syncd/Syncd.h b/syncd/Syncd.h index 7663215e5..b7a14092e 100644 --- a/syncd/Syncd.h +++ b/syncd/Syncd.h @@ -136,7 +136,8 @@ namespace syncd _In_ const swss::KeyOpFieldsValuesTuple &kco); sai_status_t processSingleEvent( - _In_ const swss::KeyOpFieldsValuesTuple &kco); + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int sequence_number=INVALID_SEQUENCE_NUMBER); sai_status_t processAttrCapabilityQuery( _In_ const swss::KeyOpFieldsValuesTuple &kco); From c1b916a93f3d9997fc5ce7a701db9bd97b8789a2 Mon Sep 17 00:00:00 2001 From: DanielaMurin Date: Tue, 27 Aug 2024 08:34:08 +0300 Subject: [PATCH 10/35] fix reset sequencer logic Change-Id: I380534a136ced86bb0e3e3f06f31be8cc389ec74 --- syncd/Sequencer.cpp | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/syncd/Sequencer.cpp b/syncd/Sequencer.cpp index 447c6b101..618d68c97 100755 --- a/syncd/Sequencer.cpp +++ b/syncd/Sequencer.cpp @@ -63,6 +63,11 @@ void Sequencer::executeReadyResponses() { ++next_seq_to_send; // Increment the sequence number std::string logMsg = "multithreaded: Next sequence found! Executed lambda with seq: " + std::to_string(next_seq_to_send); LogToModuleFile("1", logMsg); + + if (current_seq >= MAX_SEQUENCE_NUMBER) { + LogToModuleFile("1", "multithreaded: Resetting next sequence number to send needs to be reset to avoid overflow"); + next_seq_to_send = 0; + } } } @@ -76,9 +81,8 @@ int Sequencer::allocateSequenceNumber() { LogToModuleFile("1", logMsg); if (current_seq >= MAX_SEQUENCE_NUMBER) { - LogToModuleFile("1", "multithreaded: Resetting sequence number to avoid overflow"); + LogToModuleFile("1", "multithreaded: Resetting allocated sequence number to avoid overflow"); current_seq = 0; - next_seq_to_send = 0; } return seq; @@ -96,6 +100,11 @@ void Sequencer::executeFuncInSequence(int seq, std::function response_la response_lambda(); // Increment the next sequence to send ++next_seq_to_send; + + if (current_seq >= MAX_SEQUENCE_NUMBER) { + LogToModuleFile("1", "multithreaded: Resetting next sequence number to send needs to be reset to avoid overflow"); + next_seq_to_send = 0; + } // Continue sending any subsequent responses that are ready executeReadyResponses(); } else { From 8d0feae9e3c1c66ca263ec1a033f8ee75260d975 Mon Sep 17 00:00:00 2001 From: DanielaMurin Date: Tue, 27 Aug 2024 15:24:31 +0300 Subject: [PATCH 11/35] Take out lambda execute from within lock Change-Id: I861379abf3db49fbe3488a874f092fbeb136491d --- syncd/Sequencer.cpp | 36 ++++++++++++++++++++++++++---------- syncd/Sequencer.h | 2 +- syncd/Syncd.cpp | 20 ++++++++++++++++++-- 3 files changed, 45 insertions(+), 13 deletions(-) diff --git a/syncd/Sequencer.cpp b/syncd/Sequencer.cpp index 618d68c97..c5e3cfba9 100755 --- a/syncd/Sequencer.cpp +++ b/syncd/Sequencer.cpp @@ -49,23 +49,29 @@ void writeToLogFile(const std::string& funcName, const std::string& fileNum, con // Helper function to execute all ready responses in order void Sequencer::executeReadyResponses() { - //printf("Checking for ready responses in queue...\n"); - //SWSS_LOG_NOTICE("multithreaded: Checking for ready responses in queue..."); LogToModuleFile("1", "multithreaded: Checking for ready responses in queue..."); while (true) { + // Check if the next sequence number is in the map auto it = responses.find(next_seq_to_send); if (it == responses.end()) { LogToModuleFile("1", "multithreaded: No next sequence found in queue"); break; // Exit loop if the next sequence is not in the map } + it->second(); // Execute the stored lambda responses.erase(it); // Safely erase the entry - ++next_seq_to_send; // Increment the sequence number + + { + std::unique_lock lock(mtx); + ++next_seq_to_send; // Increment the sequence number + } + std::string logMsg = "multithreaded: Next sequence found! Executed lambda with seq: " + std::to_string(next_seq_to_send); LogToModuleFile("1", logMsg); if (current_seq >= MAX_SEQUENCE_NUMBER) { LogToModuleFile("1", "multithreaded: Resetting next sequence number to send needs to be reset to avoid overflow"); + std::unique_lock lock(mtx); next_seq_to_send = 0; } } @@ -73,15 +79,20 @@ void Sequencer::executeReadyResponses() { // Get sequence number int Sequencer::allocateSequenceNumber() { - std::lock_guard lock(mtx); + int seq = current_seq; - current_seq++; - //SWSS_LOG_NOTICE("multithreaded: allocate seq num: %d", seq); + + { + std::unique_lock lock(mtx); + current_seq++; + } + std::string logMsg = "multithreaded: allocate seq num: " + std::to_string(seq); LogToModuleFile("1", logMsg); if (current_seq >= MAX_SEQUENCE_NUMBER) { LogToModuleFile("1", "multithreaded: Resetting allocated sequence number to avoid overflow"); + std::unique_lock lock(mtx); current_seq = 0; } @@ -90,21 +101,26 @@ int Sequencer::allocateSequenceNumber() { // Add/Execute sequence function void Sequencer::executeFuncInSequence(int seq, std::function response_lambda) { - std::lock_guard lock(mtx); - + if (seq == next_seq_to_send) { // If the received sequence is the next one to send, execute it immediately //SWSS_LOG_NOTICE("multithreaded: executing reseponse lambda, seq num: %d", seq); std::string logMsg = "multithreaded: executing reseponse lambda, seq num: " + std::to_string(seq); LogToModuleFile("1", logMsg); response_lambda(); - // Increment the next sequence to send - ++next_seq_to_send; + { + std::unique_lock lock(mtx); + // Increment the next sequence to send + ++next_seq_to_send; + } + if (current_seq >= MAX_SEQUENCE_NUMBER) { LogToModuleFile("1", "multithreaded: Resetting next sequence number to send needs to be reset to avoid overflow"); + std::unique_lock lock(mtx); next_seq_to_send = 0; } + // Continue sending any subsequent responses that are ready executeReadyResponses(); } else { diff --git a/syncd/Sequencer.h b/syncd/Sequencer.h index 06f1b6ddb..1c8752aec 100755 --- a/syncd/Sequencer.h +++ b/syncd/Sequencer.h @@ -16,7 +16,7 @@ #define MODULE_NAME "multithreadedsyncd" -#define MAX_LOG_SIZE (500 * 1024) /* 500 KB */ +#define MAX_LOG_SIZE (3000 * 1024) /* 3000 KB */ // Improved logging function with thread safety void writeToLogFile(const std::string& funcName, const std::string& fileNum, const std::string& message); diff --git a/syncd/Syncd.cpp b/syncd/Syncd.cpp index 895a4a6c1..64833ce38 100644 --- a/syncd/Syncd.cpp +++ b/syncd/Syncd.cpp @@ -3506,19 +3506,30 @@ sai_status_t Syncd::processOidRemove( //SWSS_LOG_NOTICE("multithreaded: processOidRemove start removing..."); - //sai_object_id_t switchVid = VidManager::switchIdQuery(objectVid); + sai_object_id_t switchVid = VidManager::switchIdQuery(objectVid); - // ??? + // TODO: NEEDS TO BE FIXED! + std::string logMessage = "multithreaded: SKIP processOidRemove start removing... switchVid: " + std::to_string(switchVid); + LogToModuleFile("1", logMessage); + // LogToModuleFile("1", "multithreaded: processOidRemove start removing..."); // if (m_switches.at(switchVid)->isDiscoveredRid(rid)) // { + // LogToModuleFile("1", "multithreaded: try removeExistingObjectReference"); // m_switches.at(switchVid)->removeExistingObjectReference(rid); + // LogToModuleFile("1", "multithreaded: success removeExistingObjectReference"); // } + // LogToModuleFile("1", "multithreaded: processOidRemove removed isDiscoveredRid"); + // if (objectType == SAI_OBJECT_TYPE_PORT) // { + // LogToModuleFile("1", "multithreaded: try postPortRemove"); // m_switches.at(switchVid)->postPortRemove(rid); + // LogToModuleFile("1", "multithreaded: success postPortRemove"); // } + // LogToModuleFile("1", "multithreaded: processOidRemove removed postPortRemove"); + //SWSS_LOG_NOTICE("multithreaded: processOidRemove finish removing"); } } @@ -5417,6 +5428,9 @@ void Syncd::run() } catch(const std::exception &e) { + std::string LogToModuleFile = "shutdown - Runtime error during syncd init: " + std::string(e.what()); + LogToModuleFile("1", LogToModuleFile); + SWSS_LOG_ERROR("Runtime error during syncd init: %s", e.what()); sendShutdownRequestAfterException(); @@ -5553,6 +5567,8 @@ void Syncd::run() catch(const std::exception &e) { SWSS_LOG_ERROR("Runtime error: %s", e.what()); + std::string LogToModuleFile = "Runtime error: " + std::string(e.what()); + LogToModuleFile("1", LogToModuleFile); sendShutdownRequestAfterException(); From efc1bc9868d8a268b3cd3e7cb5b9954729500c27 Mon Sep 17 00:00:00 2001 From: shiraez Date: Tue, 27 Aug 2024 17:28:10 +0300 Subject: [PATCH 12/35] add loger --- syncd/Loger.h | 102 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100755 syncd/Loger.h diff --git a/syncd/Loger.h b/syncd/Loger.h new file mode 100755 index 000000000..eba86b5ad --- /dev/null +++ b/syncd/Loger.h @@ -0,0 +1,102 @@ + +#define MODULE_NAME "syncd" + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fmt/format.h" +#include + +#define MAX_LOG_SIZE (50 * 1024) /* 50 KB */ + +// Define a mutex for thread safety +static std::mutex logMutex; + +// Improved logging function with thread safety +static void writeToLogFile(const std::string& funcName, const std::string& fileNum, const std::string& message) { + // Lock the mutex to ensure thread safety + std::lock_guard lock(logMutex); + + std::string filePath = "/" + std::string(MODULE_NAME) + "_debugLog_" + fileNum + ".txt"; + std::string backupFilePath = filePath + ".history"; + struct stat fileStat; + + // Check if the log file exists and if its size exceeds the maximum limit + if (stat(filePath.c_str(), &fileStat) == 0) { + if (fileStat.st_size > MAX_LOG_SIZE) { + // Remove the old backup file + std::remove(backupFilePath.c_str()); + // Rename the current log file to the backup file + if (std::rename(filePath.c_str(), backupFilePath.c_str()) != 0) { + std::cerr << "Error: Could not rename file " << filePath << " to " << backupFilePath << std::endl; + return; + } + } + } + + // Open the log file in append mode + std::ofstream logFile(filePath, std::ios_base::app); + if (!logFile.is_open()) { + std::cerr << "Error: Could not open file " << filePath << std::endl; + return; + } + + // Get the current time in microseconds since the epoch + auto now = std::chrono::system_clock::now(); + auto in_time_t = std::chrono::system_clock::to_time_t(now); + auto milliseconds = std::chrono::duration_cast(now.time_since_epoch()) % 1000; + + std::ostringstream oss; + oss << std::put_time(std::localtime(&in_time_t), "%Y-%m-%d %H:%M:%S") + << '.' << std::setw(3) << std::setfill('0') << milliseconds.count(); + std::string formatted_time = oss.str(); + + // Write the timestamp, function name, and message to the log file + logFile << formatted_time << " " << funcName << ": " << message << std::endl; + + + + logFile.close(); +} + +template +static void logFormattedMessage(const std::string& funcName, const std::string& fileNum, const std::string& format, Args... messageArgs) { + std::ostringstream oss; + oss << funcName << ": "; + + std::string remainingFormat = format; + + // Helper function to process a single argument + auto processArg = [&oss, &remainingFormat](const auto& arg) { + size_t pos = remainingFormat.find("{}"); + if (pos != std::string::npos) { + oss << remainingFormat.substr(0, pos) << arg; + remainingFormat = remainingFormat.substr(pos + 2); + } + }; + + // Helper function to recursively process all arguments + auto processAllArgs = [&](auto&&... innerArgs) -> void { + (void)std::initializer_list{(processArg(innerArgs), 0)...}; + }; + + // Process all arguments + processAllArgs(messageArgs...); + + // Add any remaining format string + oss << remainingFormat; + + // Write the full message to the log file + writeToLogFile(funcName, fileNum, oss.str()); +} +// Macro for easy logging with formatting +#define LogToModuleFile(fileNum, format, ...) logFormattedMessage(__func__, fileNum, format, ##__VA_ARGS__) + + From 5e2e15c8fe17a2de27d66f325b933f0417e84bee Mon Sep 17 00:00:00 2001 From: DanielaMurin Date: Tue, 27 Aug 2024 17:26:46 +0300 Subject: [PATCH 13/35] Working single ring buffer & sequencer Change-Id: Ia215d224868aef2a5767d7b03082398ffa38e562 --- syncd/Syncd.cpp | 114 ++++++++++++++++++++++++------------------------ syncd/Syncd.h | 3 +- 2 files changed, 60 insertions(+), 57 deletions(-) diff --git a/syncd/Syncd.cpp b/syncd/Syncd.cpp index 64833ce38..0191e7dd3 100644 --- a/syncd/Syncd.cpp +++ b/syncd/Syncd.cpp @@ -418,11 +418,12 @@ void point(_In_ const swss::KeyOpFieldsValuesTuple &kco){ } void Syncd::processEvent( - _In_ sairedis::SelectableChannel& consumer) + _In_ sairedis::SelectableChannel& consumer, + _In_ int sequence_number) { SWSS_LOG_ENTER(); static int entries = 0; - int sequencer_number; + //int sequencer_number; //SWSS_LOG_NOTICE("multithreaded: !!!processEvent, ITERATION: %d!!!", entries++); std::string logMessage = "multithreaded: !!!processEvent, ITERATION: " + std::to_string(entries++) + "!!!"; LogToModuleFile("1", logMessage.c_str()); @@ -431,6 +432,9 @@ void Syncd::processEvent( do { swss::KeyOpFieldsValuesTuple kco; + consumer.pop(kco, isInitViewMode()); + processSingleEvent(kco, sequence_number); + /* * In init mode we put all data to TEMP view and we snoop. We need @@ -438,56 +442,23 @@ void Syncd::processEvent( * data to redis db. */ - consumer.pop(kco, isInitViewMode()); - sequencer_number = m_sequencer->allocateSequenceNumber(); - auto& key = kfvKey(kco); - auto& op = kfvOp(kco); - - //SWSS_LOG_NOTICE("multithreaded: BEFORE PUSH INTO RING BUFFER, key: %s op: %s, sequence number: %d", key.c_str(), op.c_str(), sequencer_number); - logMessage = "multithreaded: BEFORE PUSH INTO RING BUFFER, key: " + key + " op: " + op + ", sequence number: " + std::to_string(sequencer_number); - LogToModuleFile("1", logMessage.c_str()); - - auto lambda = [=](){ - //SWSS_LOG_NOTICE("multithreaded: inside lambda, start processing event"); - LogToModuleFile("1", "multithreaded: inside lambda, start processing event"); - processSingleEvent(kco, sequencer_number); - //SWSS_LOG_NOTICE("multithreaded: inside lambda, end processing event"); - LogToModuleFile("1", "multithreaded: inside lambda, end processing event"); - }; - - pushRingBuffer(lambda); - - //auto& key = kfvKey(kco); - //auto& op = kfvOp(kco); - /* - * TODO - * - */ - - // LogToModuleFile("1", std::get<0>(kco)); - // LogToModuleFile("1", std::get<1>(kco)); - // int i = gdb_mode; - // if(op == REDIS_ASIC_STATE_COMMAND_CREATE){ - // gdb_mode++; - // processSingleEvent(kco); - // } - // else{ - // auto lambda = [=](){ - // SWSS_LOG_DEBUG("multithreaded: start processing event"); - // processSingleEvent(kco); - // SWSS_LOG_DEBUG("multithreaded: end processing event"); - // }; - // pushRingBuffer(lambda); - // } - // // processSingleEvent(kco); - // auto lambda = [=](){ - // // point(kco); - // processSingleEvent(kco); - // }; - // pushRingBuffer(lambda); - // // processSingleEvent(kco); - + // sequencer_number = m_sequencer->allocateSequenceNumber(); + // auto& key = kfvKey(kco); + // auto& op = kfvOp(kco); + + // //SWSS_LOG_NOTICE("multithreaded: BEFORE PUSH INTO RING BUFFER, key: %s op: %s, sequence number: %d", key.c_str(), op.c_str(), sequencer_number); + // logMessage = "multithreaded: BEFORE PUSH INTO RING BUFFER, key: " + key + " op: " + op + ", sequence number: " + std::to_string(sequencer_number); + // LogToModuleFile("1", logMessage.c_str()); + + // auto lambda = [=](){ + // //SWSS_LOG_NOTICE("multithreaded: inside lambda, start processing event"); + // LogToModuleFile("1", "multithreaded: inside lambda, start processing event"); + // processSingleEvent(kco, sequencer_number); + // //SWSS_LOG_NOTICE("multithreaded: inside lambda, end processing event"); + // LogToModuleFile("1", "multithreaded: inside lambda, end processing event"); + // }; + // pushRingBuffer(lambda); } while (!consumer.empty()); } @@ -5449,13 +5420,19 @@ void Syncd::run() while (runMainLoop) { + // add sequence number to log messages SWSS_LOG_NOTICE("! run main loop "); try { swss::Selectable *sel = NULL; int result = s->select(&sel); + int sequence_number = m_sequencer->allocateSequenceNumber(); + std::string logMessage = "multithreaded: BEFORE PUSH INTO RING BUFFER sequence number: " + std::to_string(sequence_number); + LogToModuleFile("1", logMessage.c_str()); + + //restart query sequencer if (sel == m_restartQuery.get()) { /* @@ -5470,7 +5447,7 @@ void Syncd::run() while (!m_selectableChannel->empty()) { - processEvent(*m_selectableChannel.get()); + processEvent(*m_selectableChannel.get(), sequence_number); } SWSS_LOG_NOTICE("drained queue"); @@ -5514,7 +5491,7 @@ void Syncd::run() if (status != SAI_STATUS_SUCCESS) { SWSS_LOG_ERROR("Failed to set SAI_SWITCH_ATTR_FAST_API_ENABLE=true: %s for express pre-shutdown. Fall back to cold restart", - sai_serialize_status(status).c_str()); + sai_serialize_status(status).c_str()); shutdownType = SYNCD_RESTART_TYPE_COLD; @@ -5549,18 +5526,43 @@ void Syncd::run() } else if (sel == m_flexCounter.get()) { - processFlexCounterEvent(*(swss::ConsumerTable*)sel); + //directly to sequencer + auto lambda = [=](){ + LogToModuleFile("1", "multithreaded: inside lambda, start m_flexCounter"); + processFlexCounterEvent(*(swss::ConsumerTable*)sel); + LogToModuleFile("1", "multithreaded: inside lambda, end m_flexCounter"); + }; + m_sequencer->executeFuncInSequence(sequence_number, lambda); + + //processFlexCounterEvent(*(swss::ConsumerTable*)sel); } else if (sel == m_flexCounterGroup.get()) { - processFlexCounterGroupEvent(*(swss::ConsumerTable*)sel); + //directly to sequencer + auto lambda = [=](){ + LogToModuleFile("1", "multithreaded: inside lambda, start m_flexCounterGroup"); + processFlexCounterGroupEvent(*(swss::ConsumerTable*)sel); + LogToModuleFile("1", "multithreaded: inside lambda, end m_flexCounterGroup"); + }; + m_sequencer->executeFuncInSequence(sequence_number, lambda); + + //processFlexCounterGroupEvent(*(swss::ConsumerTable*)sel); } else if (sel == m_selectableChannel.get()) { - processEvent(*m_selectableChannel.get()); + auto lambda = [=](){ + LogToModuleFile("1", "multithreaded: inside lambda, start processing event"); + processEvent(*m_selectableChannel.get(), sequence_number); + LogToModuleFile("1", "multithreaded: inside lambda, end processing event"); + }; + + pushRingBuffer(lambda); + + // processEvent(*m_selectableChannel.get()); } else { + SWSS_LOG_ERROR("select failed: %d", result); } } diff --git a/syncd/Syncd.h b/syncd/Syncd.h index b7a14092e..a5bf339a4 100644 --- a/syncd/Syncd.h +++ b/syncd/Syncd.h @@ -76,7 +76,8 @@ namespace syncd public: // TODO private void processEvent( - _In_ sairedis::SelectableChannel& consumer); + _In_ sairedis::SelectableChannel& consumer, + _In_ int sequence_number = INVALID_SEQUENCE_NUMBER); sai_status_t processQuadEventInInitViewMode( _In_ sai_object_type_t objectType, From d3fddf36274e50d9c3e06891b9aa6a0755ba37a0 Mon Sep 17 00:00:00 2001 From: shiraez Date: Tue, 27 Aug 2024 17:56:45 +0300 Subject: [PATCH 14/35] add logger to syncd + sequencer --- syncd/Loger.h | 102 --------------------------------------- syncd/Sequencer.cpp | 1 + syncd/Sequencer.h | 7 +-- syncd/SingleReiniter.cpp | 1 + syncd/Syncd.cpp | 63 +----------------------- 5 files changed, 4 insertions(+), 170 deletions(-) delete mode 100755 syncd/Loger.h diff --git a/syncd/Loger.h b/syncd/Loger.h deleted file mode 100755 index eba86b5ad..000000000 --- a/syncd/Loger.h +++ /dev/null @@ -1,102 +0,0 @@ - -#define MODULE_NAME "syncd" - - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "fmt/format.h" -#include - -#define MAX_LOG_SIZE (50 * 1024) /* 50 KB */ - -// Define a mutex for thread safety -static std::mutex logMutex; - -// Improved logging function with thread safety -static void writeToLogFile(const std::string& funcName, const std::string& fileNum, const std::string& message) { - // Lock the mutex to ensure thread safety - std::lock_guard lock(logMutex); - - std::string filePath = "/" + std::string(MODULE_NAME) + "_debugLog_" + fileNum + ".txt"; - std::string backupFilePath = filePath + ".history"; - struct stat fileStat; - - // Check if the log file exists and if its size exceeds the maximum limit - if (stat(filePath.c_str(), &fileStat) == 0) { - if (fileStat.st_size > MAX_LOG_SIZE) { - // Remove the old backup file - std::remove(backupFilePath.c_str()); - // Rename the current log file to the backup file - if (std::rename(filePath.c_str(), backupFilePath.c_str()) != 0) { - std::cerr << "Error: Could not rename file " << filePath << " to " << backupFilePath << std::endl; - return; - } - } - } - - // Open the log file in append mode - std::ofstream logFile(filePath, std::ios_base::app); - if (!logFile.is_open()) { - std::cerr << "Error: Could not open file " << filePath << std::endl; - return; - } - - // Get the current time in microseconds since the epoch - auto now = std::chrono::system_clock::now(); - auto in_time_t = std::chrono::system_clock::to_time_t(now); - auto milliseconds = std::chrono::duration_cast(now.time_since_epoch()) % 1000; - - std::ostringstream oss; - oss << std::put_time(std::localtime(&in_time_t), "%Y-%m-%d %H:%M:%S") - << '.' << std::setw(3) << std::setfill('0') << milliseconds.count(); - std::string formatted_time = oss.str(); - - // Write the timestamp, function name, and message to the log file - logFile << formatted_time << " " << funcName << ": " << message << std::endl; - - - - logFile.close(); -} - -template -static void logFormattedMessage(const std::string& funcName, const std::string& fileNum, const std::string& format, Args... messageArgs) { - std::ostringstream oss; - oss << funcName << ": "; - - std::string remainingFormat = format; - - // Helper function to process a single argument - auto processArg = [&oss, &remainingFormat](const auto& arg) { - size_t pos = remainingFormat.find("{}"); - if (pos != std::string::npos) { - oss << remainingFormat.substr(0, pos) << arg; - remainingFormat = remainingFormat.substr(pos + 2); - } - }; - - // Helper function to recursively process all arguments - auto processAllArgs = [&](auto&&... innerArgs) -> void { - (void)std::initializer_list{(processArg(innerArgs), 0)...}; - }; - - // Process all arguments - processAllArgs(messageArgs...); - - // Add any remaining format string - oss << remainingFormat; - - // Write the full message to the log file - writeToLogFile(funcName, fileNum, oss.str()); -} -// Macro for easy logging with formatting -#define LogToModuleFile(fileNum, format, ...) logFormattedMessage(__func__, fileNum, format, ##__VA_ARGS__) - - diff --git a/syncd/Sequencer.cpp b/syncd/Sequencer.cpp index c5e3cfba9..c564b497e 100755 --- a/syncd/Sequencer.cpp +++ b/syncd/Sequencer.cpp @@ -1,5 +1,6 @@ #include "Sequencer.h" #include "swss/logger.h" +#include "Logger.h" using namespace syncd; // Macro for easy logging diff --git a/syncd/Sequencer.h b/syncd/Sequencer.h index 1c8752aec..32874474a 100755 --- a/syncd/Sequencer.h +++ b/syncd/Sequencer.h @@ -13,13 +13,8 @@ #include #include #include +#include "Logger.h" -#define MODULE_NAME "multithreadedsyncd" - -#define MAX_LOG_SIZE (3000 * 1024) /* 3000 KB */ - -// Improved logging function with thread safety -void writeToLogFile(const std::string& funcName, const std::string& fileNum, const std::string& message); namespace syncd { diff --git a/syncd/SingleReiniter.cpp b/syncd/SingleReiniter.cpp index f757ba515..586983b0e 100644 --- a/syncd/SingleReiniter.cpp +++ b/syncd/SingleReiniter.cpp @@ -11,6 +11,7 @@ #include #include +#include "Logger.h" using namespace syncd; using namespace saimeta; diff --git a/syncd/Syncd.cpp b/syncd/Syncd.cpp index 0191e7dd3..8217f1dfb 100644 --- a/syncd/Syncd.cpp +++ b/syncd/Syncd.cpp @@ -35,6 +35,7 @@ #include #include +#include "Logger.h" #define DEF_SAI_WARM_BOOT_DATA_FILE "/var/warmboot/sai-warmboot.bin" #define SAI_FAILURE_DUMP_SCRIPT "/usr/bin/sai_failure_dump.sh" @@ -52,68 +53,6 @@ static int gdb_mode = 0; #define WD_DELAY_FACTOR 1 #endif -// Macro for easy logging -#define LogToModuleFile(fileNum, msg) writeToLogFile(__func__, fileNum, msg) - -// #define MODULE_NAME "multithreadedsyncd" - -// #include -// #include -// #include -// #include -// #include -// #include -// #include - -// #define MAX_LOG_SIZE (500 * 1024) /* 50 KB */ - -// // Define a mutex for thread safety -// static std::mutex logMutex; - -// // Improved logging function with thread safety -// static void writeToLogFile(const std::string& funcName, const std::string& fileNum, const std::string& message) { -// // Lock the mutex to ensure thread safety -// std::lock_guard lock(logMutex); - -// std::string filePath = "/" + std::string(MODULE_NAME) + "_debugLog_" + fileNum + ".txt"; -// std::string backupFilePath = filePath + ".history"; -// struct stat fileStat; - -// // Check if the log file exists and if its size exceeds the maximum limit -// if (stat(filePath.c_str(), &fileStat) == 0) { -// if (fileStat.st_size > MAX_LOG_SIZE) { -// // Remove the old backup file -// std::remove(backupFilePath.c_str()); -// // Rename the current log file to the backup file -// if (std::rename(filePath.c_str(), backupFilePath.c_str()) != 0) { -// std::cerr << "Error: Could not rename file " << filePath << " to " << backupFilePath << std::endl; -// return; -// } -// } -// } - -// // Open the log file in append mode -// std::ofstream logFile(filePath, std::ios_base::app); -// if (!logFile.is_open()) { -// std::cerr << "Error: Could not open file " << filePath << std::endl; -// return; -// } - -// // Get the current time in microseconds since the epoch -// auto now = std::chrono::system_clock::now(); -// auto now_ms = std::chrono::time_point_cast(now); -// auto epoch = now_ms.time_since_epoch(); -// long long microseconds = std::chrono::duration_cast(epoch).count(); - -// // Write the timestamp, function name, and message to the log file -// logFile << microseconds << " " << funcName << ": " << message << std::endl; -// logFile.close(); -// } - -// // Macro for easy logging -// #define LogToModuleFile(fileNum, msg) writeToLogFile(__func__, fileNum, msg) - - Syncd::Syncd( _In_ std::shared_ptr vendorSai, _In_ std::shared_ptr cmd, From 9b01706f25ea2e6ff3954e3af50085d9781e2b59 Mon Sep 17 00:00:00 2001 From: DanielaMurin Date: Tue, 27 Aug 2024 19:05:18 +0300 Subject: [PATCH 15/35] Return m_switches and m_mdioIpcServer logic Change-Id: I24f0acbea04a99109371705b9529a0461779bfa6 --- syncd/Syncd.cpp | 55 +++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/syncd/Syncd.cpp b/syncd/Syncd.cpp index 8217f1dfb..dfc67f263 100644 --- a/syncd/Syncd.cpp +++ b/syncd/Syncd.cpp @@ -381,6 +381,7 @@ void Syncd::processEvent( * data to redis db. */ +#if our_previous_attack_helicopter // sequencer_number = m_sequencer->allocateSequenceNumber(); // auto& key = kfvKey(kco); // auto& op = kfvOp(kco); @@ -396,8 +397,9 @@ void Syncd::processEvent( // //SWSS_LOG_NOTICE("multithreaded: inside lambda, end processing event"); // LogToModuleFile("1", "multithreaded: inside lambda, end processing event"); // }; - + // pushRingBuffer(lambda); +#endif } while (!consumer.empty()); } @@ -3330,9 +3332,9 @@ sai_status_t Syncd::processOidCreate( //SWSS_LOG_NOTICE("multithreaded: object type switch"); - //m_switches[switchVid] = std::make_shared(switchVid, objectRid, m_client, m_translator, m_vendorSai); + m_switches[switchVid] = std::make_shared(switchVid, objectRid, m_client, m_translator, m_vendorSai); - //m_mdioIpcServer->setSwitchId(objectRid); + m_mdioIpcServer->setSwitchId(objectRid); // startDiagShell(objectRid); } @@ -3414,33 +3416,28 @@ sai_status_t Syncd::processOidRemove( * can already deduce that. */ - //SWSS_LOG_NOTICE("multithreaded: processOidRemove start removing..."); - sai_object_id_t switchVid = VidManager::switchIdQuery(objectVid); - // TODO: NEEDS TO BE FIXED! - std::string logMessage = "multithreaded: SKIP processOidRemove start removing... switchVid: " + std::to_string(switchVid); - LogToModuleFile("1", logMessage); - // LogToModuleFile("1", "multithreaded: processOidRemove start removing..."); - // if (m_switches.at(switchVid)->isDiscoveredRid(rid)) - // { - // LogToModuleFile("1", "multithreaded: try removeExistingObjectReference"); - // m_switches.at(switchVid)->removeExistingObjectReference(rid); - // LogToModuleFile("1", "multithreaded: success removeExistingObjectReference"); - // } + LogToModuleFile("1", "multithreaded: processOidRemove start removing..."); + if (m_switches.at(switchVid)->isDiscoveredRid(rid)) + { + LogToModuleFile("1", "multithreaded: try removeExistingObjectReference"); + m_switches.at(switchVid)->removeExistingObjectReference(rid); + LogToModuleFile("1", "multithreaded: success removeExistingObjectReference"); + } - // LogToModuleFile("1", "multithreaded: processOidRemove removed isDiscoveredRid"); + LogToModuleFile("1", "multithreaded: processOidRemove removed isDiscoveredRid"); - // if (objectType == SAI_OBJECT_TYPE_PORT) - // { - // LogToModuleFile("1", "multithreaded: try postPortRemove"); - // m_switches.at(switchVid)->postPortRemove(rid); - // LogToModuleFile("1", "multithreaded: success postPortRemove"); - // } + if (objectType == SAI_OBJECT_TYPE_PORT) + { + LogToModuleFile("1", "multithreaded: try postPortRemove"); + m_switches.at(switchVid)->postPortRemove(rid); + LogToModuleFile("1", "multithreaded: success postPortRemove"); + } - // LogToModuleFile("1", "multithreaded: processOidRemove removed postPortRemove"); + LogToModuleFile("1", "multithreaded: processOidRemove removed postPortRemove"); - //SWSS_LOG_NOTICE("multithreaded: processOidRemove finish removing"); + SWSS_LOG_NOTICE("multithreaded: processOidRemove finish removing"); } } @@ -4693,9 +4690,9 @@ void Syncd::onSwitchCreateInInitViewMode( // make switch initialization and get all default data - // m_switches[switchVid] = std::make_shared(switchVid, switchRid, m_client, m_translator, m_vendorSai); + m_switches[switchVid] = std::make_shared(switchVid, switchRid, m_client, m_translator, m_vendorSai); - // m_mdioIpcServer->setSwitchId(switchRid); + m_mdioIpcServer->setSwitchId(switchRid); //startDiagShell(switchRid); } @@ -4877,7 +4874,7 @@ void Syncd::performWarmRestartSingleSwitch( // perform all get operations on existing switch - // auto sw = m_switches[switchVid] = std::make_shared(switchVid, switchRid, m_client, m_translator, m_vendorSai, true); + auto sw = m_switches[switchVid] = std::make_shared(switchVid, switchRid, m_client, m_translator, m_vendorSai, true); //startDiagShell(switchRid); } @@ -5338,8 +5335,8 @@ void Syncd::run() } catch(const std::exception &e) { - std::string LogToModuleFile = "shutdown - Runtime error during syncd init: " + std::string(e.what()); - LogToModuleFile("1", LogToModuleFile); + std::string logText = "shutdown - Runtime error during syncd init: " + std::string(e.what()); + LogToModuleFile("1", logText); SWSS_LOG_ERROR("Runtime error during syncd init: %s", e.what()); From 7a144626eb181f4c4d4dcd9b386b63354e789193 Mon Sep 17 00:00:00 2001 From: shiraez Date: Wed, 28 Aug 2024 14:51:33 +0300 Subject: [PATCH 16/35] fix Logger --- syncd/Sequencer.cpp | 44 -------------------------------------------- 1 file changed, 44 deletions(-) diff --git a/syncd/Sequencer.cpp b/syncd/Sequencer.cpp index c564b497e..2c02b1641 100755 --- a/syncd/Sequencer.cpp +++ b/syncd/Sequencer.cpp @@ -3,50 +3,6 @@ #include "Logger.h" using namespace syncd; -// Macro for easy logging -#define LogToModuleFile(fileNum, msg) writeToLogFile(__func__, fileNum, msg) - -// Define a mutex for thread safety -static std::mutex logMutex; - -void writeToLogFile(const std::string& funcName, const std::string& fileNum, const std::string& message) { - // Lock the mutex to ensure thread safety - std::lock_guard lock(logMutex); - - std::string filePath = "/" + std::string(MODULE_NAME) + "_debugLog_" + fileNum + ".txt"; - std::string backupFilePath = filePath + ".history"; - struct stat fileStat; - - // Check if the log file exists and if its size exceeds the maximum limit - if (stat(filePath.c_str(), &fileStat) == 0) { - if (fileStat.st_size > MAX_LOG_SIZE) { - // Remove the old backup file - std::remove(backupFilePath.c_str()); - // Rename the current log file to the backup file - if (std::rename(filePath.c_str(), backupFilePath.c_str()) != 0) { - std::cerr << "Error: Could not rename file " << filePath << " to " << backupFilePath << std::endl; - return; - } - } - } - - // Open the log file in append mode - std::ofstream logFile(filePath, std::ios_base::app); - if (!logFile.is_open()) { - std::cerr << "Error: Could not open file " << filePath << std::endl; - return; - } - - // Get the current time in microseconds since the epoch - auto now = std::chrono::system_clock::now(); - auto now_ms = std::chrono::time_point_cast(now); - auto epoch = now_ms.time_since_epoch(); - long long microseconds = std::chrono::duration_cast(epoch).count(); - - // Write the timestamp, function name, and message to the log file - logFile << microseconds << " " << funcName << ": " << message << std::endl; - logFile.close(); -} // Helper function to execute all ready responses in order void Sequencer::executeReadyResponses() { From 04eb362c24fb6b22483f7937aad8ab5f309a9fa8 Mon Sep 17 00:00:00 2001 From: shiraez Date: Wed, 28 Aug 2024 18:46:40 +0300 Subject: [PATCH 17/35] add logger.h file --- syncd/Logger.h | 101 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100755 syncd/Logger.h diff --git a/syncd/Logger.h b/syncd/Logger.h new file mode 100755 index 000000000..f4c69e9f4 --- /dev/null +++ b/syncd/Logger.h @@ -0,0 +1,101 @@ +#ifndef LOGGER_H +#define LOGGER_H + +#define MODULE_NAME "syncd" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fmt/format.h" +#include + +#define MAX_LOG_SIZE (50 * 1024) /* 50 KB */ + +// Define a mutex for thread safety +static std::mutex logMutex; + +// Improved logging function with thread safety +static void writeToLogFile(const std::string& funcName, const std::string& fileNum, const std::string& message) { + // Lock the mutex to ensure thread safety + std::lock_guard lock(logMutex); + + std::string filePath = "/" + std::string(MODULE_NAME) + "_debugLog_" + fileNum + ".txt"; + std::string backupFilePath = filePath + ".history"; + struct stat fileStat; + + // Check if the log file exists and if its size exceeds the maximum limit + if (stat(filePath.c_str(), &fileStat) == 0) { + if (fileStat.st_size > MAX_LOG_SIZE) { + // Remove the old backup file + std::remove(backupFilePath.c_str()); + // Rename the current log file to the backup file + if (std::rename(filePath.c_str(), backupFilePath.c_str()) != 0) { + std::cerr << "Error: Could not rename file " << filePath << " to " << backupFilePath << std::endl; + return; + } + } + } + + // Open the log file in append mode + std::ofstream logFile(filePath, std::ios_base::app); + if (!logFile.is_open()) { + std::cerr << "Error: Could not open file " << filePath << std::endl; + return; + } + + auto now = std::chrono::system_clock::now(); + auto in_time_t = std::chrono::system_clock::to_time_t(now); + auto milliseconds = std::chrono::duration_cast(now.time_since_epoch()) % 1000; + + std::ostringstream oss; + oss << std::put_time(std::localtime(&in_time_t), "%Y-%m-%d %H:%M:%S") + << '.' << std::setw(3) << std::setfill('0') << milliseconds.count(); + std::string formatted_time = oss.str(); + + // Write the timestamp, function name, and message to the log file + logFile << formatted_time << " " << funcName << ": " << message << std::endl; + + + logFile.close(); +} + +template +static void logFormattedMessage(const std::string& funcName, const std::string& fileNum, const std::string& format, Args... messageArgs) { + std::ostringstream oss; + oss << funcName << ": "; + + std::string remainingFormat = format; + + // Helper function to process a single argument + auto processArg = [&oss, &remainingFormat](const auto& arg) { + size_t pos = remainingFormat.find("{}"); + if (pos != std::string::npos) { + oss << remainingFormat.substr(0, pos) << arg; + remainingFormat = remainingFormat.substr(pos + 2); + } + }; + + // Helper function to recursively process all arguments + auto processAllArgs = [&](auto&&... innerArgs) -> void { + (void)std::initializer_list{(processArg(innerArgs), 0)...}; + }; + + // Process all arguments + processAllArgs(messageArgs...); + + // Add any remaining format string + oss << remainingFormat; + + // Write the full message to the log file + writeToLogFile(funcName, fileNum, oss.str()); +} +// Macro for easy logging with formatting +#define LogToModuleFile(fileNum, format, ...) logFormattedMessage(__func__, fileNum, format, ##__VA_ARGS__) + +#endif // LOGGER_H From 00b8adeb3eb90bec6aac28c33fd882e21341312c Mon Sep 17 00:00:00 2001 From: DanielaMurin Date: Thu, 29 Aug 2024 11:00:40 +0300 Subject: [PATCH 18/35] Update sequencer statistics/mux/logging Change-Id: I90ef1f366c6b5f13c856d82f4aa9579c6fa9ff5b --- syncd/Sequencer.cpp | 222 ++++++++++++++++++++++++++++++--------- syncd/Sequencer.h | 43 +++++--- syncd/SingleReiniter.cpp | 2 +- syncd/Syncd.cpp | 59 ++++++++--- 4 files changed, 251 insertions(+), 75 deletions(-) diff --git a/syncd/Sequencer.cpp b/syncd/Sequencer.cpp index 2c02b1641..82baa893b 100755 --- a/syncd/Sequencer.cpp +++ b/syncd/Sequencer.cpp @@ -1,91 +1,219 @@ #include "Sequencer.h" -#include "swss/logger.h" -#include "Logger.h" +//#include "swss/logger.h" +//#include "Logger.h" + +#define LogToModuleFile(fileNum, msg) writeToLogFile(__func__, fileNum, msg) + +// Define a mutex for thread safety +static std::mutex logMutex; + +void writeToLogFile(const std::string& funcName, const std::string& fileNum, const std::string& message) { + // Lock the mutex to ensure thread safety + std::lock_guard lock(logMutex); + + std::string filePath = "/" + std::string(MODULE_NAME) + "_debugLog_" + fileNum + ".txt"; + std::string backupFilePath = filePath + ".history"; + struct stat fileStat; + + // Check if the log file exists and if its size exceeds the maximum limit + if (stat(filePath.c_str(), &fileStat) == 0) { + if (fileStat.st_size > MAX_LOG_SIZE) { + // Remove the old backup file + std::remove(backupFilePath.c_str()); + // Rename the current log file to the backup file + if (std::rename(filePath.c_str(), backupFilePath.c_str()) != 0) { + std::cerr << "Error: Could not rename file " << filePath << " to " << backupFilePath << std::endl; + return; + } + } + } + + // Open the log file in append mode + std::ofstream logFile(filePath, std::ios_base::app); + if (!logFile.is_open()) { + std::cerr << "Error: Could not open file " << filePath << std::endl; + return; + } + + // Get the current time in microseconds since the epoch + auto now = std::chrono::system_clock::now(); + auto now_ms = std::chrono::time_point_cast(now); + auto epoch = now_ms.time_since_epoch(); + long long microseconds = std::chrono::duration_cast(epoch).count(); + + // Write the timestamp, function name, and message to the log file + logFile << microseconds << " " << funcName << ": " << message << std::endl; + logFile.close(); +} using namespace syncd; // Helper function to execute all ready responses in order -void Sequencer::executeReadyResponses() { - LogToModuleFile("1", "multithreaded: Checking for ready responses in queue..."); +// check how many consecutive responses were executed in sucession and log it +Sequencer::SequenceStatus Sequencer::executeReadyResponses() { + + std::string logMsg; + SequenceStatus status = FAILURE; + + logMsg = "multithreaded: Checking for ready responses in queue... \n"; + while (true) { // Check if the next sequence number is in the map - auto it = responses.find(next_seq_to_send); - if (it == responses.end()) { - LogToModuleFile("1", "multithreaded: No next sequence found in queue"); + auto func = responses.find(next_seq_to_send); + if (func == responses.end()) { + logMsg += "multithreaded: No next sequence found in queue \n"; + status = SUCCESS; break; // Exit loop if the next sequence is not in the map } - it->second(); // Execute the stored lambda - responses.erase(it); // Safely erase the entry - - { - std::unique_lock lock(mtx); - ++next_seq_to_send; // Increment the sequence number + // imporve readability, don't use second() directly + // Execute the stored lambda + if(func->second) { + func->second(); + logMsg += "multithreaded: Executing lambda with seq: " + std::to_string(next_seq_to_send) + " \n"; + status = NULL_PTR; } + else { + logMsg += "multithreaded: response lambda is null \n"; + num_of_null_functions++; + status = SUCCESS; //????? + } + + // Increment the number of executed tasks in sequence + max_num_of_executed_tasks_in_sequence++; - std::string logMsg = "multithreaded: Next sequence found! Executed lambda with seq: " + std::to_string(next_seq_to_send); - LogToModuleFile("1", logMsg); + // Safely erase the entry + responses.erase(func); + + // Increment the sequence number + ++next_seq_to_send; + + logMsg += "multithreaded: Next sequence found! Executed lambda with seq: " + std::to_string(next_seq_to_send) + " \n"; - if (current_seq >= MAX_SEQUENCE_NUMBER) { - LogToModuleFile("1", "multithreaded: Resetting next sequence number to send needs to be reset to avoid overflow"); - std::unique_lock lock(mtx); + if (next_seq_to_send >= MAX_SEQUENCE_NUMBER) { + logMsg += "multithreaded: Resetting next sequence number to send needs to be reset to avoid overflow \n"; next_seq_to_send = 0; } } + + LogToModuleFile("1", logMsg); + return status; } // Get sequence number -int Sequencer::allocateSequenceNumber() { - - int seq = current_seq; +// if sequencer is full, reset the sequence number to avoid overflow +// wait for, throw full failure +// return if there is an issue or success (0-succss, 1-failure) +// add private param to see how many lambdas are stored, +// for each function, return a status code +// mux on everything +Sequencer::SequenceStatus Sequencer::allocateSequenceNumber(int *seq_num) { + std::unique_lock lock(mtx); + std::string logMsg; + SequenceStatus status = FAILURE; - { - std::unique_lock lock(mtx); - current_seq++; + if(isFull()) { + logMsg = "multithreaded: Sequencer is full, cannot allocate sequence number" + std::to_string(current_seq); + LogToModuleFile("1", logMsg); + return BUFFER_OVERFLOW; } - std::string logMsg = "multithreaded: allocate seq num: " + std::to_string(seq); - LogToModuleFile("1", logMsg); + // update recieved param + *seq_num = current_seq; + // increment the sequence number + current_seq++; + + logMsg = "multithreaded: allocate seq num: " + std::to_string(*seq_num) + ", "; + // reset number to avoid overflow if (current_seq >= MAX_SEQUENCE_NUMBER) { - LogToModuleFile("1", "multithreaded: Resetting allocated sequence number to avoid overflow"); - std::unique_lock lock(mtx); + logMsg += "multithreaded: Resetting allocated sequence number to avoid overflow, "; current_seq = 0; } - return seq; + status = SUCCESS; + + LogToModuleFile("1", logMsg); + + return status; } // Add/Execute sequence function -void Sequencer::executeFuncInSequence(int seq, std::function response_lambda) { +Sequencer::SequenceStatus Sequencer::executeFuncInSequence(int seq, std::function response_lambda) { + std::unique_lock lock(mtx); + std::string logMsg; + SequenceStatus status = FAILURE; + if (seq == next_seq_to_send) { // If the received sequence is the next one to send, execute it immediately - //SWSS_LOG_NOTICE("multithreaded: executing reseponse lambda, seq num: %d", seq); - std::string logMsg = "multithreaded: executing reseponse lambda, seq num: " + std::to_string(seq); - LogToModuleFile("1", logMsg); - response_lambda(); - - { - std::unique_lock lock(mtx); - // Increment the next sequence to send - ++next_seq_to_send; + logMsg = "multithreaded: executing reseponse lambda, seq num: " + std::to_string(seq) + " \n"; + + // execute response lambda + if(response_lambda) { + response_lambda(); + logMsg += "multithreaded: execute response lambda \n"; + status = SUCCESS; } + else { + logMsg += "multithreaded: response lambda is null \n"; + num_of_null_functions++; + status = SUCCESS; //NULL_PTR; ??? + } + + // increment the number of executed tasks in sequence + max_num_of_executed_tasks_in_sequence++; + + // Increment the next sequence to send + ++next_seq_to_send; - if (current_seq >= MAX_SEQUENCE_NUMBER) { - LogToModuleFile("1", "multithreaded: Resetting next sequence number to send needs to be reset to avoid overflow"); - std::unique_lock lock(mtx); + // reset number to avoid overflow + if (next_seq_to_send >= MAX_SEQUENCE_NUMBER) { + logMsg += "multithreaded: Resetting next sequence number to send needs to be reset to avoid overflow \n"; next_seq_to_send = 0; } // Continue sending any subsequent responses that are ready - executeReadyResponses(); + status = executeReadyResponses(); } else { // If the sequence is not the next to send, store it in the map responses[seq] = response_lambda; - std::string logMsg = "multithreaded: storing lambda with seq: " + std::to_string(seq) + ", next to send: " + std::to_string(next_seq_to_send); - LogToModuleFile("1", logMsg); - //SWSS_LOG_NOTICE("Storing lambda with seq: %d, next to send: %d\n", seq, next_seq_to_send); - //printf("Storing lambda with seq: %d, next to send: %d\n", seq, next_seq_to_send); + logMsg = "multithreaded: storing lambda with seq: " + std::to_string(seq) + ", next to send: " + std::to_string(next_seq_to_send) + " \n"; + status = SUCCESS; + num_of_out_of_sequence_functions++; + } + + LogToModuleFile("1", logMsg); + return status; +} + +Sequencer::SequenceStatus Sequencer::showStatistics() { + std::unique_lock lock(mtx); + std::string logMsg = "STATISTICS: \n"; + logMsg = "multithreaded: max number of executed tasks in sequence: " + std::to_string(max_num_of_executed_tasks_in_sequence) + " \n"; + logMsg += "multithreaded: number of null functions: " + std::to_string(num_of_null_functions) + " \n"; + logMsg += "multithreaded: number of out of sequence functions: " + std::to_string(num_of_out_of_sequence_functions) + " \n"; + logMsg += std::to_string(current_seq) + " out of " + std::to_string(max_seq_num) + "used"; + LogToModuleFile("1", logMsg); + return SUCCESS; +} + +Sequencer::SequenceStatus Sequencer::clearStatistics() { + std::unique_lock lock(mtx); + max_num_of_executed_tasks_in_sequence = 0; + num_of_null_functions = 0; + num_of_out_of_sequence_functions = 0; + LogToModuleFile("1", "CLEANED STATISTICS \n"); + return SUCCESS; +} + +bool Sequencer::isFull() { + if(responses.size() < max_seq_num) { + LogToModuleFile("1", "is not full"); + return false; + } + else { + LogToModuleFile("1", "is full"); + return true; } } \ No newline at end of file diff --git a/syncd/Sequencer.h b/syncd/Sequencer.h index 32874474a..3e6623b7a 100755 --- a/syncd/Sequencer.h +++ b/syncd/Sequencer.h @@ -13,44 +13,63 @@ #include #include #include -#include "Logger.h" +//#include "Logger.h" +#define MODULE_NAME "multithreadedsyncd" + +#define MAX_LOG_SIZE (500 * 1024) /* 500 KB */ + +// Improved logging function with thread safety +void writeToLogFile(const std::string& funcName, const std::string& fileNum, const std::string& message); namespace syncd { - #define MAX_SEQUENCE_NUMBER 1000000 - #define INVALID_SEQUENCE_NUMBER std::numeric_limits::min() + #define MAX_SEQUENCE_NUMBER 1024 + #define INVALID_SEQUENCE_NUMBER std::numeric_limits::min() // allow user to choose using AnyTask = std::function; class Sequencer { public: // Public constructor - Sequencer() : current_seq(0), next_seq_to_send(0), last_update_time(std::chrono::steady_clock::now()) {} + Sequencer() : current_seq(0), next_seq_to_send(0), max_seq_num(MAX_SEQUENCE_NUMBER), max_num_of_executed_tasks_in_sequence(0), num_of_null_functions(0), num_of_out_of_sequence_functions(0) {} // Public destructor ~Sequencer() {} + enum SequenceStatus { + FAILURE = -1, + SUCCESS = 0, + BUFFER_OVERFLOW = 1, + NULL_PTR = 2, + }; + // Get sequence number - int allocateSequenceNumber(); + SequenceStatus allocateSequenceNumber(int *seq_num); // Add/Execute sequence function - void executeFuncInSequence(int seq, std::function response_lambda); + SequenceStatus executeFuncInSequence(int seq, std::function response_lambda); private: - // Reset the sequence number to avoid overflow - void resetSequence(); - // Watchdog function to monitor inactivity - void performWatchdogCheck(); + SequenceStatus performWatchdogCheck(); // Helper function to execute all ready responses in order - void executeReadyResponses(); + SequenceStatus executeReadyResponses(); + + SequenceStatus showStatistics(); + + SequenceStatus clearStatistics(); + + bool isFull(); int current_seq; // Tracks the latest sequence number assigned to a task int next_seq_to_send; // The next sequence number that should be sent + long unsigned int max_seq_num; // The maximum sequence number + int max_num_of_executed_tasks_in_sequence; // The maximum number of executed tasks in sequence + int num_of_null_functions; // The number of null functions + int num_of_out_of_sequence_functions; // The number of out of sequence functions std::mutex mtx; // Protects shared data std::map> responses; // Stores responses by sequence number - std::chrono::steady_clock::time_point last_update_time; // Time of the last sequence update }; } diff --git a/syncd/SingleReiniter.cpp b/syncd/SingleReiniter.cpp index 586983b0e..fafe64037 100644 --- a/syncd/SingleReiniter.cpp +++ b/syncd/SingleReiniter.cpp @@ -11,7 +11,7 @@ #include #include -#include "Logger.h" +//#include "Logger.h" using namespace syncd; using namespace saimeta; diff --git a/syncd/Syncd.cpp b/syncd/Syncd.cpp index dfc67f263..39abfa44a 100644 --- a/syncd/Syncd.cpp +++ b/syncd/Syncd.cpp @@ -35,7 +35,7 @@ #include #include -#include "Logger.h" +//#include "Logger.h" #define DEF_SAI_WARM_BOOT_DATA_FILE "/var/warmboot/sai-warmboot.bin" #define SAI_FAILURE_DUMP_SCRIPT "/usr/bin/sai_failure_dump.sh" @@ -53,6 +53,9 @@ static int gdb_mode = 0; #define WD_DELAY_FACTOR 1 #endif +// Macro for easy logging +#define LogToModuleFile(fileNum, msg) writeToLogFile(__func__, fileNum, msg) + Syncd::Syncd( _In_ std::shared_ptr vendorSai, _In_ std::shared_ptr cmd, @@ -448,10 +451,14 @@ sai_status_t Syncd::processSingleEvent( return processBulkQuadEvent(SAI_COMMON_API_BULK_SET, kco, sequence_number); auto lambda = [=](){ - //SWSS_LOG_NOTICE("multithreaded: non crud event, skip sequencer logic %d", sequence_number); }; - m_sequencer->executeFuncInSequence(sequence_number, lambda); + LogToModuleFile("1", "multithreaded: not crud event, skip sequencer logic"); + if (int seq_status = m_sequencer->executeFuncInSequence(sequence_number, lambda) != Sequencer::SUCCESS) { + logMessage = "multithreaded: non crud event, skip sequencer logic " + std::to_string(sequence_number) + ", status: " + std::to_string(seq_status); + LogToModuleFile("1", logMessage); + } + LogToModuleFile("1", "multithreaded: not crud event, successful skip"); if (op == REDIS_ASIC_STATE_COMMAND_NOTIFY) return processNotifySyncd(kco); @@ -2621,12 +2628,10 @@ void Syncd::sendApiResponseSequence( _In_ int sequenceNumber) { SWSS_LOG_ENTER(); - //SWSS_LOG_NOTICE("multithreaded: sendApiResponseSequence"); //sequenceNumber = INVALID_SEQUENCE_NUMBER; if(sequenceNumber != INVALID_SEQUENCE_NUMBER) { - //SWSS_LOG_NOTICE("multithreaded: valid sequence number %d, API %s, status %d, object count: %d", sequenceNumber, sai_serialize_common_api(api).c_str(), status, object_count); std::string logMessage = "multithreaded: valid sequence number " + std::to_string(sequenceNumber) + ", API " + sai_serialize_common_api(api) + ", status " + std::to_string(status) + ", object count: " + std::to_string(object_count); LogToModuleFile("1", logMessage.c_str()); @@ -2637,7 +2642,10 @@ void Syncd::sendApiResponseSequence( LogToModuleFile("1", "multithreaded: sendApiResponseSequence lambda end"); }; - m_sequencer->executeFuncInSequence(sequenceNumber, lambda); + if(int seq_status = m_sequencer->executeFuncInSequence(sequenceNumber, lambda) != Sequencer::SUCCESS) { + logMessage = "multithreaded: executeFuncInSequence failed for sequence number " + std::to_string(sequenceNumber) + ", status: " + std::to_string(seq_status); + LogToModuleFile("1", logMessage); + } } else { // If the response is not to be sequenced, then send it directly @@ -3717,7 +3725,10 @@ void Syncd::sendGetResponseSequence( LogToModuleFile("1", "multithreaded: sendGetResponseSequence: lambda end"); }; - m_sequencer->executeFuncInSequence(sequenceNumber, lambda); + if(int seq_status = m_sequencer->executeFuncInSequence(sequenceNumber, lambda) != Sequencer::SUCCESS) { + logMessage = "multithreaded: sendGetResponseSequence: executeFuncInSequence failed, sequenceNumber=" + std::to_string(sequenceNumber) + ", status=" + std::to_string(seq_status); + LogToModuleFile("1", logMessage); + } } else { sendGetResponse(objectType, strObjectId, switchVid, status, attr_count, attr_list); @@ -5363,7 +5374,21 @@ void Syncd::run() swss::Selectable *sel = NULL; int result = s->select(&sel); - int sequence_number = m_sequencer->allocateSequenceNumber(); + int sequence_number; + + if(int seq_status = m_sequencer->allocateSequenceNumber(&sequence_number) != Sequencer::SUCCESS) + { + std::string logMsg = "Failed to allocate sequence number: " + std::to_string(seq_status); + LogToModuleFile("1", logMsg); + auto lambda = [=](){}; + pushRingBuffer(lambda); + } + else { + std::string logMsg = "Allocated sequence number: " + std::to_string(sequence_number); + LogToModuleFile("1", logMsg); + } + + // in case of failure, make sure that sequence number is correct std::string logMessage = "multithreaded: BEFORE PUSH INTO RING BUFFER sequence number: " + std::to_string(sequence_number); LogToModuleFile("1", logMessage.c_str()); @@ -5468,9 +5493,12 @@ void Syncd::run() processFlexCounterEvent(*(swss::ConsumerTable*)sel); LogToModuleFile("1", "multithreaded: inside lambda, end m_flexCounter"); }; - m_sequencer->executeFuncInSequence(sequence_number, lambda); - //processFlexCounterEvent(*(swss::ConsumerTable*)sel); + if(int seq_status = m_sequencer->executeFuncInSequence(sequence_number, lambda) != Sequencer::SUCCESS) + { + logMessage = "m_flexCounter failed to execute function in sequence, status: " + std::to_string(seq_status) + ", sequence number: " + std::to_string(sequence_number); + LogToModuleFile("1", logMessage); + } } else if (sel == m_flexCounterGroup.get()) { @@ -5480,9 +5508,12 @@ void Syncd::run() processFlexCounterGroupEvent(*(swss::ConsumerTable*)sel); LogToModuleFile("1", "multithreaded: inside lambda, end m_flexCounterGroup"); }; - m_sequencer->executeFuncInSequence(sequence_number, lambda); - - //processFlexCounterGroupEvent(*(swss::ConsumerTable*)sel); + + if(int seq_status = m_sequencer->executeFuncInSequence(sequence_number, lambda) != Sequencer::SUCCESS) + { + logMessage = "m_flexCounterGroup failed to execute function in sequence, status: " + std::to_string(seq_status) + ", sequence number: " + std::to_string(sequence_number); + LogToModuleFile("1", logMessage); + } } else if (sel == m_selectableChannel.get()) { @@ -5493,8 +5524,6 @@ void Syncd::run() }; pushRingBuffer(lambda); - - // processEvent(*m_selectableChannel.get()); } else { From 94005f871e41ffb67960dcfa91a51b0d25acd5ad Mon Sep 17 00:00:00 2001 From: DanielaMurin Date: Thu, 29 Aug 2024 11:20:47 +0300 Subject: [PATCH 19/35] Merge conflict fix, remove old logger code Change-Id: I0e663cd3b7ce155298efd4f74fde86130ccf3b01 --- syncd/Sequencer.cpp | 47 +-------------------------------------------- syncd/Sequencer.h | 9 +-------- syncd/Syncd.cpp | 5 +---- 3 files changed, 3 insertions(+), 58 deletions(-) diff --git a/syncd/Sequencer.cpp b/syncd/Sequencer.cpp index 82baa893b..d876b4c7d 100755 --- a/syncd/Sequencer.cpp +++ b/syncd/Sequencer.cpp @@ -1,50 +1,5 @@ #include "Sequencer.h" -//#include "swss/logger.h" -//#include "Logger.h" - -#define LogToModuleFile(fileNum, msg) writeToLogFile(__func__, fileNum, msg) - -// Define a mutex for thread safety -static std::mutex logMutex; - -void writeToLogFile(const std::string& funcName, const std::string& fileNum, const std::string& message) { - // Lock the mutex to ensure thread safety - std::lock_guard lock(logMutex); - - std::string filePath = "/" + std::string(MODULE_NAME) + "_debugLog_" + fileNum + ".txt"; - std::string backupFilePath = filePath + ".history"; - struct stat fileStat; - - // Check if the log file exists and if its size exceeds the maximum limit - if (stat(filePath.c_str(), &fileStat) == 0) { - if (fileStat.st_size > MAX_LOG_SIZE) { - // Remove the old backup file - std::remove(backupFilePath.c_str()); - // Rename the current log file to the backup file - if (std::rename(filePath.c_str(), backupFilePath.c_str()) != 0) { - std::cerr << "Error: Could not rename file " << filePath << " to " << backupFilePath << std::endl; - return; - } - } - } - - // Open the log file in append mode - std::ofstream logFile(filePath, std::ios_base::app); - if (!logFile.is_open()) { - std::cerr << "Error: Could not open file " << filePath << std::endl; - return; - } - - // Get the current time in microseconds since the epoch - auto now = std::chrono::system_clock::now(); - auto now_ms = std::chrono::time_point_cast(now); - auto epoch = now_ms.time_since_epoch(); - long long microseconds = std::chrono::duration_cast(epoch).count(); - - // Write the timestamp, function name, and message to the log file - logFile << microseconds << " " << funcName << ": " << message << std::endl; - logFile.close(); -} +#include "swss/logger.h" using namespace syncd; diff --git a/syncd/Sequencer.h b/syncd/Sequencer.h index 3e6623b7a..be8f1d50d 100755 --- a/syncd/Sequencer.h +++ b/syncd/Sequencer.h @@ -13,14 +13,7 @@ #include #include #include -//#include "Logger.h" - -#define MODULE_NAME "multithreadedsyncd" - -#define MAX_LOG_SIZE (500 * 1024) /* 500 KB */ - -// Improved logging function with thread safety -void writeToLogFile(const std::string& funcName, const std::string& fileNum, const std::string& message); +#include "Logger.h" namespace syncd { diff --git a/syncd/Syncd.cpp b/syncd/Syncd.cpp index 39abfa44a..e0081dfd4 100644 --- a/syncd/Syncd.cpp +++ b/syncd/Syncd.cpp @@ -35,7 +35,7 @@ #include #include -//#include "Logger.h" +#include "Logger.h" #define DEF_SAI_WARM_BOOT_DATA_FILE "/var/warmboot/sai-warmboot.bin" #define SAI_FAILURE_DUMP_SCRIPT "/usr/bin/sai_failure_dump.sh" @@ -53,9 +53,6 @@ static int gdb_mode = 0; #define WD_DELAY_FACTOR 1 #endif -// Macro for easy logging -#define LogToModuleFile(fileNum, msg) writeToLogFile(__func__, fileNum, msg) - Syncd::Syncd( _In_ std::shared_ptr vendorSai, _In_ std::shared_ptr cmd, From 3f121a35c4bfd71473f718e602d45f05658e850e Mon Sep 17 00:00:00 2001 From: shiraez Date: Thu, 29 Aug 2024 11:19:38 +0300 Subject: [PATCH 20/35] enable flag to the logger --- syncd/Logger.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/syncd/Logger.h b/syncd/Logger.h index f4c69e9f4..1522fe5e8 100755 --- a/syncd/Logger.h +++ b/syncd/Logger.h @@ -16,6 +16,7 @@ #include #define MAX_LOG_SIZE (50 * 1024) /* 50 KB */ +#define ENABLE_LOGGING 1 // Define a mutex for thread safety static std::mutex logMutex; @@ -95,7 +96,11 @@ static void logFormattedMessage(const std::string& funcName, const std::string& // Write the full message to the log file writeToLogFile(funcName, fileNum, oss.str()); } +#if ENABLE_LOGGING // Macro for easy logging with formatting #define LogToModuleFile(fileNum, format, ...) logFormattedMessage(__func__, fileNum, format, ##__VA_ARGS__) +#else +#define LogToModuleFile(fileNum, format, ...) +#endif #endif // LOGGER_H From 7a661ce81e54ec083c53bf14fb053d3faeb2395f Mon Sep 17 00:00:00 2001 From: DanielaMurin Date: Thu, 29 Aug 2024 14:13:03 +0300 Subject: [PATCH 21/35] Return diag shell thread, improve lambda func readability Change-Id: Id77121b6a52a535ffda7b31c699e0b6dc9a75ae1 --- syncd/Sequencer.cpp | 12 ++++++------ syncd/Syncd.cpp | 14 +++++++------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/syncd/Sequencer.cpp b/syncd/Sequencer.cpp index d876b4c7d..8b8ff58cd 100755 --- a/syncd/Sequencer.cpp +++ b/syncd/Sequencer.cpp @@ -14,17 +14,17 @@ Sequencer::SequenceStatus Sequencer::executeReadyResponses() { while (true) { // Check if the next sequence number is in the map - auto func = responses.find(next_seq_to_send); - if (func == responses.end()) { + auto seq_data = responses.find(next_seq_to_send); + if (seq_data == responses.end()) { logMsg += "multithreaded: No next sequence found in queue \n"; status = SUCCESS; break; // Exit loop if the next sequence is not in the map } - // imporve readability, don't use second() directly // Execute the stored lambda - if(func->second) { - func->second(); + auto func = seq_data->second; + if(func) { + func(); logMsg += "multithreaded: Executing lambda with seq: " + std::to_string(next_seq_to_send) + " \n"; status = NULL_PTR; } @@ -38,7 +38,7 @@ Sequencer::SequenceStatus Sequencer::executeReadyResponses() { max_num_of_executed_tasks_in_sequence++; // Safely erase the entry - responses.erase(func); + responses.erase(seq_data); // Increment the sequence number ++next_seq_to_send; diff --git a/syncd/Syncd.cpp b/syncd/Syncd.cpp index e0081dfd4..fc97099f8 100644 --- a/syncd/Syncd.cpp +++ b/syncd/Syncd.cpp @@ -3341,7 +3341,7 @@ sai_status_t Syncd::processOidCreate( m_mdioIpcServer->setSwitchId(objectRid); - // startDiagShell(objectRid); + startDiagShell(objectRid); } if (objectType == SAI_OBJECT_TYPE_PORT) @@ -4600,10 +4600,10 @@ void Syncd::onSyncdStart( m_switches = hr.hardReinit(); - // for (auto& sw: m_switches) - // { - // startDiagShell(sw.second->getRid()); - // } + for (auto& sw: m_switches) + { + startDiagShell(sw.second->getRid()); + } SWSS_LOG_NOTICE("hard reinit succeeded"); } @@ -4702,7 +4702,7 @@ void Syncd::onSwitchCreateInInitViewMode( m_mdioIpcServer->setSwitchId(switchRid); - //startDiagShell(switchRid); + startDiagShell(switchRid); } else { @@ -4884,7 +4884,7 @@ void Syncd::performWarmRestartSingleSwitch( auto sw = m_switches[switchVid] = std::make_shared(switchVid, switchRid, m_client, m_translator, m_vendorSai, true); - //startDiagShell(switchRid); + startDiagShell(switchRid); } void Syncd::performWarmRestart() From 8ea0eb6c71e1a9ea5982ebbffc385c6388b02cfa Mon Sep 17 00:00:00 2001 From: DanielaMurin Date: Thu, 29 Aug 2024 17:01:08 +0300 Subject: [PATCH 22/35] Allocate sequence number inside functions Change-Id: I31bc11dd855c80ae551c1cdd6b750f9c06a24947 --- syncd/Sequencer.cpp | 1 + syncd/Syncd.cpp | 41 ++++++++++++++++++++++++++++++++++------- syncd/Syncd.h | 2 ++ 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/syncd/Sequencer.cpp b/syncd/Sequencer.cpp index 8b8ff58cd..43a1e278a 100755 --- a/syncd/Sequencer.cpp +++ b/syncd/Sequencer.cpp @@ -70,6 +70,7 @@ Sequencer::SequenceStatus Sequencer::allocateSequenceNumber(int *seq_num) { if(isFull()) { logMsg = "multithreaded: Sequencer is full, cannot allocate sequence number" + std::to_string(current_seq); LogToModuleFile("1", logMsg); + *seq_num = INVALID_SEQUENCE_NUMBER; return BUFFER_OVERFLOW; } diff --git a/syncd/Syncd.cpp b/syncd/Syncd.cpp index fc97099f8..81932e133 100644 --- a/syncd/Syncd.cpp +++ b/syncd/Syncd.cpp @@ -356,14 +356,31 @@ void point(_In_ const swss::KeyOpFieldsValuesTuple &kco){ gdb_mode = 1; } +int Syncd::getSeqNumber() { + int sequence_number = 0; + if(int seq_status = m_sequencer->allocateSequenceNumber(&sequence_number) != Sequencer::SUCCESS) + { + std::string logMsg = "Failed to allocate sequence number: " + std::to_string(seq_status); + LogToModuleFile("1", logMsg); + } + else { + std::string logMsg = "Allocated sequence number: " + std::to_string(sequence_number); + LogToModuleFile("1", logMsg); + } + + // in case of failure, make sure that sequence number is correct + + std::string logMessage = "multithreaded: BEFORE PUSH INTO RING BUFFER sequence number: " + std::to_string(sequence_number); + LogToModuleFile("1", logMessage.c_str()); + + return sequence_number; +} void Syncd::processEvent( _In_ sairedis::SelectableChannel& consumer, _In_ int sequence_number) { SWSS_LOG_ENTER(); static int entries = 0; - //int sequencer_number; - //SWSS_LOG_NOTICE("multithreaded: !!!processEvent, ITERATION: %d!!!", entries++); std::string logMessage = "multithreaded: !!!processEvent, ITERATION: " + std::to_string(entries++) + "!!!"; LogToModuleFile("1", logMessage.c_str()); @@ -372,8 +389,14 @@ void Syncd::processEvent( { swss::KeyOpFieldsValuesTuple kco; consumer.pop(kco, isInitViewMode()); - processSingleEvent(kco, sequence_number); + auto lambda = [=](){ + LogToModuleFile("1", "multithreaded: inside lambda, start processing event"); + processSingleEvent(kco, getSeqNumber()); + LogToModuleFile("1", "multithreaded: inside lambda, end processing event"); + }; + + pushRingBuffer(lambda); /* * In init mode we put all data to TEMP view and we snoop. We need @@ -5373,6 +5396,7 @@ void Syncd::run() int result = s->select(&sel); int sequence_number; +#if 0 if(int seq_status = m_sequencer->allocateSequenceNumber(&sequence_number) != Sequencer::SUCCESS) { std::string logMsg = "Failed to allocate sequence number: " + std::to_string(seq_status); @@ -5389,6 +5413,7 @@ void Syncd::run() std::string logMessage = "multithreaded: BEFORE PUSH INTO RING BUFFER sequence number: " + std::to_string(sequence_number); LogToModuleFile("1", logMessage.c_str()); +#endif //restart query sequencer if (sel == m_restartQuery.get()) @@ -5405,7 +5430,7 @@ void Syncd::run() while (!m_selectableChannel->empty()) { - processEvent(*m_selectableChannel.get(), sequence_number); + processEvent(*m_selectableChannel.get()); } SWSS_LOG_NOTICE("drained queue"); @@ -5491,9 +5516,10 @@ void Syncd::run() LogToModuleFile("1", "multithreaded: inside lambda, end m_flexCounter"); }; + sequence_number = getSeqNumber(); if(int seq_status = m_sequencer->executeFuncInSequence(sequence_number, lambda) != Sequencer::SUCCESS) { - logMessage = "m_flexCounter failed to execute function in sequence, status: " + std::to_string(seq_status) + ", sequence number: " + std::to_string(sequence_number); + std::string logMessage = "m_flexCounter failed to execute function in sequence, status: " + std::to_string(seq_status) + ", sequence number: " + std::to_string(sequence_number); LogToModuleFile("1", logMessage); } } @@ -5506,9 +5532,10 @@ void Syncd::run() LogToModuleFile("1", "multithreaded: inside lambda, end m_flexCounterGroup"); }; + sequence_number = getSeqNumber(); if(int seq_status = m_sequencer->executeFuncInSequence(sequence_number, lambda) != Sequencer::SUCCESS) { - logMessage = "m_flexCounterGroup failed to execute function in sequence, status: " + std::to_string(seq_status) + ", sequence number: " + std::to_string(sequence_number); + std::string logMessage = "m_flexCounterGroup failed to execute function in sequence, status: " + std::to_string(seq_status) + ", sequence number: " + std::to_string(sequence_number); LogToModuleFile("1", logMessage); } } @@ -5516,7 +5543,7 @@ void Syncd::run() { auto lambda = [=](){ LogToModuleFile("1", "multithreaded: inside lambda, start processing event"); - processEvent(*m_selectableChannel.get(), sequence_number); + processEvent(*m_selectableChannel.get()); LogToModuleFile("1", "multithreaded: inside lambda, end processing event"); }; diff --git a/syncd/Syncd.h b/syncd/Syncd.h index a5bf339a4..b42ed3c05 100644 --- a/syncd/Syncd.h +++ b/syncd/Syncd.h @@ -75,6 +75,8 @@ namespace syncd public: // TODO private + int getSeqNumber(); + void processEvent( _In_ sairedis::SelectableChannel& consumer, _In_ int sequence_number = INVALID_SEQUENCE_NUMBER); From 9908e23891fddbc10a16f9bef272c168ff1f92c3 Mon Sep 17 00:00:00 2001 From: aviramd Date: Thu, 29 Aug 2024 20:06:17 +0300 Subject: [PATCH 23/35] add multiRing buffer support & send response for all ops Fixed Problems/New Changes: Should be Tested: --- syncd/Syncd.cpp | 855 +++++++++++++++++++++------------- syncd/Syncd.h | 506 +++++++++++++------- syncd/SyncdMultipleRingBuff.h | 148 ++++++ 3 files changed, 1019 insertions(+), 490 deletions(-) create mode 100644 syncd/SyncdMultipleRingBuff.h diff --git a/syncd/Syncd.cpp b/syncd/Syncd.cpp index 81932e133..83b738572 100644 --- a/syncd/Syncd.cpp +++ b/syncd/Syncd.cpp @@ -211,46 +211,66 @@ Syncd::Syncd( } m_breakConfig = BreakConfigParser::parseBreakConfig(m_commandLineOptions->m_breakConfig); - gRingBuffer = SyncdRing::Get(); - ring_thread = std::thread(&Syncd::popRingBuffer, this); + + if (initializeOperationGroups() != SAI_STATUS_SUCCESS) + { + LogToModuleFile("init","FATAL: failed to initialize operation groups"); + SWSS_LOG_ERROR("FATAL: failed to initialize operation groups"); + abort(); + } + + for (auto it = operationGroups.begin(); it != operationGroups.end(); ++it) { + const std::string& groupName = it->first; + const OperationGroup& group = it->second; + + if (group.ringBuffer) { + + // Create a thread for each operation group to pop from the ring buffer + ringBufferThreads[groupName] = std::thread(&Syncd::popRingBuffer, this, group.ringBuffer); + LogToModuleFile("1","start ring buff {}",getNameByRingBuffer(group.ringBuffer)); + } + } SWSS_LOG_NOTICE("syncd started"); } -void Syncd::popRingBuffer() +void Syncd::popRingBuffer(SyncdRing* ringBuffer) { - if (!gRingBuffer || gRingBuffer->Started) + if (!ringBuffer || ringBuffer->Started) + { + LogToModuleFile(getNameByRingBuffer(ringBuffer), "popRingBuffer return"); return; + } SWSS_LOG_ENTER(); - gRingBuffer->Started = true; + ringBuffer->Started = true; //SWSS_LOG_NOTICE("multithreaded: Syncd starts the popRingBuffer thread!"); - LogToModuleFile("1", "multithreaded: Syncd starts the popRingBuffer thread!"); + while (!ring_thread_exited) { //SWSS_LOG_NOTICE("multithreaded: wait popRingBuffer thread!"); - LogToModuleFile("1", "multithreaded: wait popRingBuffer thread!"); - std::unique_lock lock(gRingBuffer->mtx); - gRingBuffer->cv.wait(lock, [&](){ return !gRingBuffer->IsEmpty(); }); + LogToModuleFile(getNameByRingBuffer(ringBuffer), "multithreaded: wait popRingBuffer thread!"); + std::unique_lock lock(ringBuffer->mtx); + ringBuffer->cv.wait(lock, [&](){ return !ringBuffer->IsEmpty(); }); //SWSS_LOG_NOTICE("multithreaded: Stop waiting"); - LogToModuleFile("1", "multithreaded: Stop waiting"); - gRingBuffer->Idle = false; + LogToModuleFile(getNameByRingBuffer(ringBuffer), "multithreaded: Stop waiting"); + ringBuffer->Idle = false; AnyTask func; - while (gRingBuffer->pop(func)) { - LogToModuleFile("1", "multithreaded: try to execute func"); + while (ringBuffer->pop(func)) { + LogToModuleFile(getNameByRingBuffer(ringBuffer), "multithreaded: try to execute func"); //SWSS_LOG_NOTICE("multithreaded: try to execute func"); func(); - LogToModuleFile("1", "multithreaded: Execute func successful"); + LogToModuleFile(getNameByRingBuffer(ringBuffer), "multithreaded: Execute func successful"); //SWSS_LOG_NOTICE("multithreaded: Execute func successful"); } - LogToModuleFile("1", "multithreaded: no more functions to execute"); + LogToModuleFile(getNameByRingBuffer(ringBuffer), "multithreaded: no more functions to execute"); //SWSS_LOG_NOTICE("multithreaded: no more functions to execute"); // lock.unlock(); - gRingBuffer->doTask(); - gRingBuffer->Idle = true; + ringBuffer->doTask(); + ringBuffer->Idle = true; } // while (!ring_thread_exited) { // std::unique_lock lock(mtx); - // gRingBuffer->cv.wait(lock, [&](){ return !gRingBuffer->IsEmpty(); }); + // ringBuffer->cv.wait(lock, [&](){ return !ringBuffer->IsEmpty(); }); // AnyTask func = std::move(buffer[head]); // head = (head + 1) % RingSize; // lock.unlock(); @@ -262,12 +282,25 @@ Syncd::~Syncd() { SWSS_LOG_ENTER(); - if (gRingBuffer) { - ring_thread_exited = true; - ring_thread.detach(); + // Clean up resources + ring_thread_exited = true; + + // Join threads to ensure they complete before destroying the object + for (auto it = ringBufferThreads.begin(); it != ringBufferThreads.end(); ++it) { + auto& thread = it->second; + if (thread.joinable()) { + thread.join(); // Wait for thread completion - Aviram:is that coorect + } } - // empty + for (auto it = operationGroups.begin(); it != operationGroups.end(); ++it) { + auto& group = it->second; + if (group.ringBuffer) { + LogToModuleFile(getNameByRingBuffer(group.ringBuffer), "delete ring buffer"); + delete group.ringBuffer; + group.ringBuffer = nullptr; + } + } } void Syncd::performStartupLogic() @@ -356,28 +389,8 @@ void point(_In_ const swss::KeyOpFieldsValuesTuple &kco){ gdb_mode = 1; } -int Syncd::getSeqNumber() { - int sequence_number = 0; - if(int seq_status = m_sequencer->allocateSequenceNumber(&sequence_number) != Sequencer::SUCCESS) - { - std::string logMsg = "Failed to allocate sequence number: " + std::to_string(seq_status); - LogToModuleFile("1", logMsg); - } - else { - std::string logMsg = "Allocated sequence number: " + std::to_string(sequence_number); - LogToModuleFile("1", logMsg); - } - - // in case of failure, make sure that sequence number is correct - - std::string logMessage = "multithreaded: BEFORE PUSH INTO RING BUFFER sequence number: " + std::to_string(sequence_number); - LogToModuleFile("1", logMessage.c_str()); - - return sequence_number; -} void Syncd::processEvent( - _In_ sairedis::SelectableChannel& consumer, - _In_ int sequence_number) + _In_ sairedis::SelectableChannel& consumer) { SWSS_LOG_ENTER(); static int entries = 0; @@ -388,56 +401,55 @@ void Syncd::processEvent( do { swss::KeyOpFieldsValuesTuple kco; - consumer.pop(kco, isInitViewMode()); - - auto lambda = [=](){ - LogToModuleFile("1", "multithreaded: inside lambda, start processing event"); - processSingleEvent(kco, getSeqNumber()); - LogToModuleFile("1", "multithreaded: inside lambda, end processing event"); - }; - - pushRingBuffer(lambda); - - /* + /* * In init mode we put all data to TEMP view and we snoop. We need * to specify temporary view prefix in consumer since consumer puts * data to redis db. */ + consumer.pop(kco, isInitViewMode()); + SyncdRing* ringBuffer; + getApiRingBuffer(kco, ringBuffer); -#if our_previous_attack_helicopter - // sequencer_number = m_sequencer->allocateSequenceNumber(); - // auto& key = kfvKey(kco); - // auto& op = kfvOp(kco); - - // //SWSS_LOG_NOTICE("multithreaded: BEFORE PUSH INTO RING BUFFER, key: %s op: %s, sequence number: %d", key.c_str(), op.c_str(), sequencer_number); - // logMessage = "multithreaded: BEFORE PUSH INTO RING BUFFER, key: " + key + " op: " + op + ", sequence number: " + std::to_string(sequencer_number); - // LogToModuleFile("1", logMessage.c_str()); + int sequenceNumber; + if(int seq_status = m_sequencer->allocateSequenceNumber(&sequenceNumber) != Sequencer::SUCCESS) + { + std::string logMsg = "Failed to allocate sequence number: " + std::to_string(seq_status); + LogToModuleFile("1", logMsg); + //todo: handle wait until sequence number is available with timeout + } + else { + std::string logMsg = "Allocated sequence number: " + std::to_string(sequenceNumber); + LogToModuleFile("1", logMsg); + } - // auto lambda = [=](){ - // //SWSS_LOG_NOTICE("multithreaded: inside lambda, start processing event"); - // LogToModuleFile("1", "multithreaded: inside lambda, start processing event"); - // processSingleEvent(kco, sequencer_number); - // //SWSS_LOG_NOTICE("multithreaded: inside lambda, end processing event"); - // LogToModuleFile("1", "multithreaded: inside lambda, end processing event"); - // }; + if (ringBuffer) { + LogToModuleFile("1", "push processSingleEvent with sequenceNumber {} to ring buffer {} ",sequenceNumber, getNameByRingBuffer(ringBuffer)); + auto lambda = [=](){ + processSingleEvent(kco, sequenceNumber); + }; + pushRingBuffer(ringBuffer, lambda); + } + // operation not found in operationGroups and will be executed without ring buffer + else { + LogToModuleFile("1", "processSingleEvent with sequenceNumber {} in main thread",sequenceNumber); + processSingleEvent(kco, sequenceNumber); + } - // pushRingBuffer(lambda); -#endif } while (!consumer.empty()); } sai_status_t Syncd::processSingleEvent( _In_ const swss::KeyOpFieldsValuesTuple &kco, - _In_ int sequence_number) + _In_ int sequenceNumber) { SWSS_LOG_ENTER(); auto& key = kfvKey(kco); auto& op = kfvOp(kco); - //SWSS_LOG_NOTICE("multithreaded: key: %s op: %s, sequence number: %d", key.c_str(), op.c_str(), sequence_number); - std::string logMessage = "multithreaded: key: " + key + " op: " + op + ", sequence number: " + std::to_string(sequence_number); + //SWSS_LOG_NOTICE("multithreaded: key: %s op: %s, sequence number: %d", key.c_str(), op.c_str(), sequenceNumber); + std::string logMessage = "multithreaded: key: " + key + " op: " + op + ", sequence number: " + std::to_string(sequenceNumber); LogToModuleFile("1", logMessage.c_str()); if (key.length() == 0) @@ -450,74 +462,65 @@ sai_status_t Syncd::processSingleEvent( WatchdogScope ws(m_timerWatchdog, op + ":" + key, &kco); if (op == REDIS_ASIC_STATE_COMMAND_CREATE) - return processQuadEvent(SAI_COMMON_API_CREATE, kco, sequence_number); + return processQuadEvent(SAI_COMMON_API_CREATE, kco, sequenceNumber); if (op == REDIS_ASIC_STATE_COMMAND_REMOVE) - return processQuadEvent(SAI_COMMON_API_REMOVE, kco, sequence_number); + return processQuadEvent(SAI_COMMON_API_REMOVE, kco, sequenceNumber); if (op == REDIS_ASIC_STATE_COMMAND_SET) - return processQuadEvent(SAI_COMMON_API_SET, kco, sequence_number); + return processQuadEvent(SAI_COMMON_API_SET, kco, sequenceNumber); if (op == REDIS_ASIC_STATE_COMMAND_GET) - return processQuadEvent(SAI_COMMON_API_GET, kco, sequence_number); + return processQuadEvent(SAI_COMMON_API_GET, kco, sequenceNumber); if (op == REDIS_ASIC_STATE_COMMAND_BULK_CREATE) - return processBulkQuadEvent(SAI_COMMON_API_BULK_CREATE, kco, sequence_number); + return processBulkQuadEvent(SAI_COMMON_API_BULK_CREATE, kco, sequenceNumber); if (op == REDIS_ASIC_STATE_COMMAND_BULK_REMOVE) - return processBulkQuadEvent(SAI_COMMON_API_BULK_REMOVE, kco, sequence_number); + return processBulkQuadEvent(SAI_COMMON_API_BULK_REMOVE, kco, sequenceNumber); if (op == REDIS_ASIC_STATE_COMMAND_BULK_SET) - return processBulkQuadEvent(SAI_COMMON_API_BULK_SET, kco, sequence_number); - - auto lambda = [=](){ - }; - - LogToModuleFile("1", "multithreaded: not crud event, skip sequencer logic"); - if (int seq_status = m_sequencer->executeFuncInSequence(sequence_number, lambda) != Sequencer::SUCCESS) { - logMessage = "multithreaded: non crud event, skip sequencer logic " + std::to_string(sequence_number) + ", status: " + std::to_string(seq_status); - LogToModuleFile("1", logMessage); - } - LogToModuleFile("1", "multithreaded: not crud event, successful skip"); + return processBulkQuadEvent(SAI_COMMON_API_BULK_SET, kco, sequenceNumber); if (op == REDIS_ASIC_STATE_COMMAND_NOTIFY) - return processNotifySyncd(kco); + return processNotifySyncd(kco, sequenceNumber); if (op == REDIS_ASIC_STATE_COMMAND_GET_STATS) - return processGetStatsEvent(kco); + return processGetStatsEvent(kco, sequenceNumber); if (op == REDIS_ASIC_STATE_COMMAND_CLEAR_STATS) - return processClearStatsEvent(kco); + return processClearStatsEvent(kco, sequenceNumber); if (op == REDIS_ASIC_STATE_COMMAND_FLUSH) - return processFdbFlush(kco); + return processFdbFlush(kco, sequenceNumber); if (op == REDIS_ASIC_STATE_COMMAND_ATTR_CAPABILITY_QUERY) - return processAttrCapabilityQuery(kco); + return processAttrCapabilityQuery(kco, sequenceNumber); if (op == REDIS_ASIC_STATE_COMMAND_ATTR_ENUM_VALUES_CAPABILITY_QUERY) - return processAttrEnumValuesCapabilityQuery(kco); + return processAttrEnumValuesCapabilityQuery(kco, sequenceNumber); if (op == REDIS_ASIC_STATE_COMMAND_OBJECT_TYPE_GET_AVAILABILITY_QUERY) - return processObjectTypeGetAvailabilityQuery(kco); + return processObjectTypeGetAvailabilityQuery(kco, sequenceNumber); if (op == REDIS_FLEX_COUNTER_COMMAND_START_POLL) - return processFlexCounterEvent(key, SET_COMMAND, kfvFieldsValues(kco)); + return processFlexCounterEvent(key, SET_COMMAND, kfvFieldsValues(kco), true, sequenceNumber); if (op == REDIS_FLEX_COUNTER_COMMAND_STOP_POLL) - return processFlexCounterEvent(key, DEL_COMMAND, kfvFieldsValues(kco)); + return processFlexCounterEvent(key, DEL_COMMAND, kfvFieldsValues(kco), true, sequenceNumber); if (op == REDIS_FLEX_COUNTER_COMMAND_SET_GROUP) - return processFlexCounterGroupEvent(key, SET_COMMAND, kfvFieldsValues(kco)); + return processFlexCounterGroupEvent(key, SET_COMMAND, kfvFieldsValues(kco), true, sequenceNumber); if (op == REDIS_FLEX_COUNTER_COMMAND_DEL_GROUP) - return processFlexCounterGroupEvent(key, DEL_COMMAND, kfvFieldsValues(kco)); + return processFlexCounterGroupEvent(key, DEL_COMMAND, kfvFieldsValues(kco), true, sequenceNumber); SWSS_LOG_THROW("event op '%s' is not implemented, FIXME", op.c_str()); } sai_status_t Syncd::processAttrCapabilityQuery( - _In_ const swss::KeyOpFieldsValuesTuple &kco) + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int sequenceNumber) { SWSS_LOG_ENTER(); @@ -534,7 +537,7 @@ sai_status_t Syncd::processAttrCapabilityQuery( { SWSS_LOG_ERROR("Invalid input: expected 2 arguments, received %zu", values.size()); - m_selectableChannel->set(sai_serialize_status(SAI_STATUS_INVALID_PARAMETER), {}, REDIS_ASIC_STATE_COMMAND_ATTR_CAPABILITY_RESPONSE); + sendStausResponseSequence(SAI_STATUS_INVALID_PARAMETER, REDIS_ASIC_STATE_COMMAND_ATTR_CAPABILITY_RESPONSE, {}, sequenceNumber); return SAI_STATUS_INVALID_PARAMETER; } @@ -564,13 +567,14 @@ sai_status_t Syncd::processAttrCapabilityQuery( capability.create_implemented, capability.set_implemented, capability.get_implemented); } - m_selectableChannel->set(sai_serialize_status(status), entry, REDIS_ASIC_STATE_COMMAND_ATTR_CAPABILITY_RESPONSE); + sendStausResponseSequence(status, REDIS_ASIC_STATE_COMMAND_ATTR_CAPABILITY_RESPONSE, entry, sequenceNumber); return status; } sai_status_t Syncd::processAttrEnumValuesCapabilityQuery( - _In_ const swss::KeyOpFieldsValuesTuple &kco) + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int sequenceNumber) { SWSS_LOG_ENTER(); @@ -587,7 +591,7 @@ sai_status_t Syncd::processAttrEnumValuesCapabilityQuery( { SWSS_LOG_ERROR("Invalid input: expected 3 arguments, received %zu", values.size()); - m_selectableChannel->set(sai_serialize_status(SAI_STATUS_INVALID_PARAMETER), {}, REDIS_ASIC_STATE_COMMAND_ATTR_ENUM_VALUES_CAPABILITY_RESPONSE); + sendStausResponseSequence(SAI_STATUS_INVALID_PARAMETER, REDIS_ASIC_STATE_COMMAND_ATTR_ENUM_VALUES_CAPABILITY_RESPONSE, {}, sequenceNumber); return SAI_STATUS_INVALID_PARAMETER; } @@ -640,13 +644,14 @@ sai_status_t Syncd::processAttrEnumValuesCapabilityQuery( SWSS_LOG_DEBUG("Sending response: count = %u", enumCapList.count); } - m_selectableChannel->set(sai_serialize_status(status), entry, REDIS_ASIC_STATE_COMMAND_ATTR_ENUM_VALUES_CAPABILITY_RESPONSE); + sendStausResponseSequence(status, REDIS_ASIC_STATE_COMMAND_ATTR_ENUM_VALUES_CAPABILITY_RESPONSE, entry, sequenceNumber); return status; } sai_status_t Syncd::processObjectTypeGetAvailabilityQuery( - _In_ const swss::KeyOpFieldsValuesTuple &kco) + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int sequenceNumber) { SWSS_LOG_ENTER(); @@ -693,13 +698,14 @@ sai_status_t Syncd::processObjectTypeGetAvailabilityQuery( SWSS_LOG_DEBUG("Sending response: count = %lu", count); } - m_selectableChannel->set(sai_serialize_status(status), entry, REDIS_ASIC_STATE_COMMAND_OBJECT_TYPE_GET_AVAILABILITY_RESPONSE); + sendStausResponseSequence(status, REDIS_ASIC_STATE_COMMAND_OBJECT_TYPE_GET_AVAILABILITY_RESPONSE, entry, sequenceNumber); return status; } sai_status_t Syncd::processFdbFlush( - _In_ const swss::KeyOpFieldsValuesTuple &kco) + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int sequenceNumber) { SWSS_LOG_ENTER(); @@ -733,7 +739,23 @@ sai_status_t Syncd::processFdbFlush( sai_status_t status = m_vendorSai->flushFdbEntries(switchRid, attr_count, attr_list); - m_selectableChannel->set(sai_serialize_status(status), {} , REDIS_ASIC_STATE_COMMAND_FLUSHRESPONSE); + + // If the response is to be sequenced, then add it to the sequencer + auto lambda = [=]() { + processFdbFlushResponse(status, values, switchVid); + }; + LogToModuleFile("1", "add processFdbFlushResponse to sequencer {}", sequenceNumber); + sendStausAdvancedResponseSequence(sequenceNumber,lambda); + + return status; +} + +void Syncd::processFdbFlushResponse( + _In_ sai_status_t status, + _In_ std::vector values, + _In_ sai_object_id_t switchVid) +{ + sendStatusAndEntryResponse(status, REDIS_ASIC_STATE_COMMAND_FLUSHRESPONSE, {}); if (status == SAI_STATUS_SUCCESS) { @@ -751,9 +773,12 @@ sai_status_t Syncd::processFdbFlush( sai_object_id_t bvId = SAI_NULL_OBJECT_ID; sai_object_id_t bridgePortId = SAI_NULL_OBJECT_ID; - - attr_list = vidlist.get_attr_list(); - attr_count = vidlist.get_attr_count(); + + SaiAttributeList list(SAI_OBJECT_TYPE_FDB_FLUSH, values, false); + SaiAttributeList vidlist(SAI_OBJECT_TYPE_FDB_FLUSH, values, false); + + sai_attribute_t *attr_list = vidlist.get_attr_list(); + uint32_t attr_count = vidlist.get_attr_count(); for (uint32_t i = 0; i < attr_count; i++) { @@ -779,12 +804,11 @@ sai_status_t Syncd::processFdbFlush( m_client->processFlushEvent(switchVid, bridgePortId, bvId, type); } - - return status; } sai_status_t Syncd::processClearStatsEvent( - _In_ const swss::KeyOpFieldsValuesTuple &kco) + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int sequenceNumber) { SWSS_LOG_ENTER(); @@ -799,7 +823,7 @@ sai_status_t Syncd::processClearStatsEvent( sai_status_t status = SAI_STATUS_INVALID_OBJECT_ID; - m_selectableChannel->set(sai_serialize_status(status), {}, REDIS_ASIC_STATE_COMMAND_GETRESPONSE); + sendStausResponseSequence(status, REDIS_ASIC_STATE_COMMAND_GETRESPONSE, {}, sequenceNumber); return status; } @@ -808,7 +832,7 @@ sai_status_t Syncd::processClearStatsEvent( { SWSS_LOG_WARN("VID to RID translation failure: %s", key.c_str()); sai_status_t status = SAI_STATUS_INVALID_OBJECT_ID; - m_selectableChannel->set(sai_serialize_status(status), {}, REDIS_ASIC_STATE_COMMAND_GETRESPONSE); + sendStausResponseSequence(status, REDIS_ASIC_STATE_COMMAND_GETRESPONSE, {}, sequenceNumber); return status; } @@ -835,13 +859,14 @@ sai_status_t Syncd::processClearStatsEvent( (uint32_t)counter_ids.size(), counter_ids.data()); - m_selectableChannel->set(sai_serialize_status(status), {}, REDIS_ASIC_STATE_COMMAND_GETRESPONSE); + sendStausResponseSequence(status, REDIS_ASIC_STATE_COMMAND_GETRESPONSE, {}, sequenceNumber); return status; } sai_status_t Syncd::processGetStatsEvent( - _In_ const swss::KeyOpFieldsValuesTuple &kco) + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int sequenceNumber) { SWSS_LOG_ENTER(); @@ -856,7 +881,7 @@ sai_status_t Syncd::processGetStatsEvent( sai_status_t status = SAI_STATUS_INVALID_OBJECT_ID; - m_selectableChannel->set(sai_serialize_status(status), {}, REDIS_ASIC_STATE_COMMAND_GETRESPONSE); + sendStausResponseSequence(status, REDIS_ASIC_STATE_COMMAND_GETRESPONSE, {}, sequenceNumber); return status; } @@ -905,7 +930,7 @@ sai_status_t Syncd::processGetStatsEvent( } } - m_selectableChannel->set(sai_serialize_status(status), entry, REDIS_ASIC_STATE_COMMAND_GETRESPONSE); + sendStausResponseSequence(status, REDIS_ASIC_STATE_COMMAND_GETRESPONSE, entry, sequenceNumber); return status; } @@ -976,7 +1001,7 @@ sai_status_t Syncd::processBulkQuadEvent( if (isInitViewMode()) { - return processBulkQuadEventInInitViewMode(objectType, objectIds, api, attributes, strAttributes); + return processBulkQuadEventInInitViewMode(objectType, objectIds, api, attributes, strAttributes, sequenceNumber); } if (api != SAI_COMMON_API_BULK_GET) @@ -1004,12 +1029,79 @@ sai_status_t Syncd::processBulkQuadEvent( } } +void Syncd::sendApiResponseUpdateRedisQuadEvent( + _In_ sai_common_api_t api, + _In_ sai_status_t status, + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int sequenceNumber) +{ + sendApiResponse(api, status); + syncUpdateRedisQuadEvent(status, api, kco); +} + +void Syncd::sendGetResponseUpdateRedisQuadEvent( + _In_ sai_object_type_t objectType, + _In_ const std::string& strObjectId, + _In_ sai_object_id_t switchVid, + _In_ sai_status_t status, + _In_ uint32_t attr_count, + _In_ sai_attribute_t *attr_list, + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ sai_common_api_t api) +{ + sendGetResponse(objectType, strObjectId, switchVid, status, attr_count, attr_list); + syncUpdateRedisQuadEvent(status, api, kco); +} + +void Syncd::sendApiResponseUpdateRedisBulkQuadEvent( + _In_ sai_common_api_t api, + _In_ sai_status_t status, + _In_ sai_object_type_t objectType, + _In_ const std::vector& objectIds, + _In_ const std::vector>& strAttributes) +{ + SWSS_LOG_ENTER(); + + std::vector statuses(objectIds.size()); + + sendApiResponse(api, status, static_cast(statuses.size()), statuses.data()); + + syncUpdateRedisBulkQuadEvent(api, statuses, objectType, objectIds, strAttributes); +} + +void Syncd::sendApiResponseUpdateRedisBulkQuadEventWithObjectInsertion( + _In_ sai_common_api_t api, + _In_ sai_status_t status, + _In_ sai_object_type_t objectType, + _In_ const std::vector& objectIds, + _In_ const std::vector>& strAttributes) +{ + SWSS_LOG_ENTER(); + + std::vector statuses(objectIds.size()); + + sendApiResponse(api, status, static_cast(statuses.size()), statuses.data()); + + syncUpdateRedisBulkQuadEvent(api, statuses, objectType, objectIds, strAttributes); + + for (auto& str: objectIds) + { + sai_object_id_t objectVid; + sai_deserialize_object_id(str, objectVid); + + // in init view mode insert every created object except switch + + m_createdInInitView.insert(objectVid); + } +} + sai_status_t Syncd::processBulkQuadEventInInitViewMode( _In_ sai_object_type_t objectType, _In_ const std::vector& objectIds, _In_ sai_common_api_t api, _In_ const std::vector>& attributes, - _In_ const std::vector>& strAttributes) + _In_ const std::vector>& strAttributes, + _In_ int sequenceNumber) { SWSS_LOG_ENTER(); @@ -1030,10 +1122,12 @@ sai_status_t Syncd::processBulkQuadEventInInitViewMode( if (info->isnonobjectid) { - sendApiResponseSequence(api, SAI_STATUS_SUCCESS, (uint32_t)statuses.size(), statuses.data()); - - syncUpdateRedisBulkQuadEvent(api, statuses, objectType, objectIds, strAttributes); - + LogToModuleFile("1", "add sendApiResponseUpdateRedisBulkQuadEvent to Lambda {}", sequenceNumber); + auto lambda = [=]() { + sendApiResponseUpdateRedisBulkQuadEvent(api, SAI_STATUS_SUCCESS, objectType, objectIds, strAttributes); + }; + sendStausAdvancedResponseSequence(sequenceNumber,lambda); + return SAI_STATUS_SUCCESS; } @@ -1048,21 +1142,12 @@ sai_status_t Syncd::processBulkQuadEventInInitViewMode( sai_serialize_object_type(objectType).c_str()); default: - - sendApiResponseSequence(api, SAI_STATUS_SUCCESS, (uint32_t)statuses.size(), statuses.data()); - - syncUpdateRedisBulkQuadEvent(api, statuses, objectType, objectIds, strAttributes); - - for (auto& str: objectIds) - { - sai_object_id_t objectVid; - sai_deserialize_object_id(str, objectVid); - - // in init view mode insert every created object except switch - - m_createdInInitView.insert(objectVid); - } - + LogToModuleFile("1", "add sendApiResponseUpdateRedisBulkQuadEventWithObjectInsertion to Lambda {}", sequenceNumber); + auto lambda = [=]() { + sendApiResponseUpdateRedisBulkQuadEventWithObjectInsertion(api, SAI_STATUS_SUCCESS, objectType, objectIds, strAttributes); + }; + sendStausAdvancedResponseSequence(sequenceNumber,lambda); + return SAI_STATUS_SUCCESS; } @@ -1888,8 +1973,11 @@ sai_status_t Syncd::processBulkEntry( if (all != SAI_STATUS_NOT_SUPPORTED && all != SAI_STATUS_NOT_IMPLEMENTED) { - sendApiResponseSequence(api, all, (uint32_t)objectIds.size(), statuses.data(), sequenceNumber); - syncUpdateRedisBulkQuadEvent(api, statuses, objectType, objectIds, strAttributes); + LogToModuleFile("1", "add sendApiResponseUpdateRedisBulkQuadEvent to Lambda {}", sequenceNumber); + auto lambda = [=]() { + sendApiResponseUpdateRedisBulkQuadEvent(api, all, objectType, objectIds, strAttributes); + }; + sendStausAdvancedResponseSequence(sequenceNumber,lambda); return all; } @@ -2012,11 +2100,12 @@ sai_status_t Syncd::processBulkEntry( statuses[idx] = status; } - - sendApiResponseSequence(api, all, (uint32_t)objectIds.size(), statuses.data()); - - syncUpdateRedisBulkQuadEvent(api, statuses, objectType, objectIds, strAttributes); - + LogToModuleFile("1", "add sendApiResponseUpdateRedisBulkQuadEvent to Lambda {}", sequenceNumber); + auto lambda = [=]() { + sendApiResponseUpdateRedisBulkQuadEvent(api, all, objectType, objectIds, strAttributes); + }; + sendStausAdvancedResponseSequence(sequenceNumber,lambda); + return all; } @@ -2250,8 +2339,11 @@ sai_status_t Syncd::processBulkOid( if (all != SAI_STATUS_NOT_SUPPORTED && all != SAI_STATUS_NOT_IMPLEMENTED) { - sendApiResponseSequence(api, all, (uint32_t)objectIds.size(), statuses.data(), sequenceNumber); - syncUpdateRedisBulkQuadEvent(api, statuses, objectType, objectIds, strAttributes); + LogToModuleFile("1", "1.add sendApiResponseUpdateRedisBulkQuadEvent to Lambda {}", sequenceNumber); + auto lambda = [=]() { + sendApiResponseUpdateRedisBulkQuadEvent(api, all, objectType, objectIds, strAttributes); + }; + sendStausAdvancedResponseSequence(sequenceNumber,lambda); return all; } @@ -2302,10 +2394,11 @@ sai_status_t Syncd::processBulkOid( statuses[idx] = status; } - - sendApiResponseSequence(api, all, (uint32_t)objectIds.size(), statuses.data()); - - syncUpdateRedisBulkQuadEvent(api, statuses, objectType, objectIds, strAttributes); + LogToModuleFile("1", "2. add sendApiResponseUpdateRedisBulkQuadEvent to Lambda {}", sequenceNumber); + auto lambda = [=]() { + sendApiResponseUpdateRedisBulkQuadEvent(api, all, objectType, objectIds, strAttributes); + }; + sendStausAdvancedResponseSequence(sequenceNumber,lambda); return all; } @@ -2315,7 +2408,9 @@ sai_status_t Syncd::processQuadEventInInitViewMode( _In_ const std::string& strObjectId, _In_ sai_common_api_t api, _In_ uint32_t attr_count, - _In_ sai_attribute_t *attr_list) + _In_ sai_attribute_t *attr_list, + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int sequenceNumber) { SWSS_LOG_ENTER(); @@ -2329,16 +2424,16 @@ sai_status_t Syncd::processQuadEventInInitViewMode( switch (api) { case SAI_COMMON_API_CREATE: - return processQuadInInitViewModeCreate(objectType, strObjectId, attr_count, attr_list); + return processQuadInInitViewModeCreate(objectType, strObjectId, attr_count, attr_list, api, kco, sequenceNumber); case SAI_COMMON_API_REMOVE: - return processQuadInInitViewModeRemove(objectType, strObjectId); + return processQuadInInitViewModeRemove(objectType, strObjectId, api, kco, sequenceNumber); case SAI_COMMON_API_SET: - return processQuadInInitViewModeSet(objectType, strObjectId, attr_list); + return processQuadInInitViewModeSet(objectType, strObjectId, attr_list, api, kco, sequenceNumber); case SAI_COMMON_API_GET: - return processQuadInInitViewModeGet(objectType, strObjectId, attr_count, attr_list); + return processQuadInInitViewModeGet(objectType, strObjectId, attr_count, attr_list, api, kco, sequenceNumber); default: @@ -2350,7 +2445,10 @@ sai_status_t Syncd::processQuadInInitViewModeCreate( _In_ sai_object_type_t objectType, _In_ const std::string& strObjectId, _In_ uint32_t attr_count, - _In_ sai_attribute_t *attr_list) + _In_ sai_attribute_t *attr_list, + _In_ sai_common_api_t api, + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int sequenceNumber) { SWSS_LOG_ENTER(); @@ -2395,15 +2493,21 @@ sai_status_t Syncd::processQuadInInitViewModeCreate( m_createdInInitView.insert(objectVid); } } - - sendApiResponseSequence(SAI_COMMON_API_CREATE, SAI_STATUS_SUCCESS); + LogToModuleFile("1", "add sendApiResponseUpdateRedisQuadEvent to Lambda {}", sequenceNumber); + auto lambda = [=]() { + sendApiResponseUpdateRedisQuadEvent(api, SAI_STATUS_SUCCESS, kco, sequenceNumber); + }; + sendStausAdvancedResponseSequence(sequenceNumber,lambda); return SAI_STATUS_SUCCESS; } sai_status_t Syncd::processQuadInInitViewModeRemove( _In_ sai_object_type_t objectType, - _In_ const std::string& strObjectId) + _In_ const std::string& strObjectId, + _In_ sai_common_api_t api, + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int sequenceNumber) { SWSS_LOG_ENTER(); @@ -2457,8 +2561,11 @@ sai_status_t Syncd::processQuadInInitViewModeRemove( m_initViewRemovedVidSet.insert(objectVid); } - - sendApiResponseSequence(SAI_COMMON_API_REMOVE, SAI_STATUS_SUCCESS); + LogToModuleFile("1", "add sendApiResponseUpdateRedisQuadEvent to Lambda {}", sequenceNumber); + auto lambda = [=]() { + sendApiResponseUpdateRedisQuadEvent(api, SAI_STATUS_SUCCESS, kco, sequenceNumber); + }; + sendStausAdvancedResponseSequence(sequenceNumber,lambda); return SAI_STATUS_SUCCESS; } @@ -2466,13 +2573,19 @@ sai_status_t Syncd::processQuadInInitViewModeRemove( sai_status_t Syncd::processQuadInInitViewModeSet( _In_ sai_object_type_t objectType, _In_ const std::string& strObjectId, - _In_ sai_attribute_t *attr) + _In_ sai_attribute_t *attr, + _In_ sai_common_api_t api, + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int sequenceNumber) { SWSS_LOG_ENTER(); // we support SET api on all objects in init view mode - - sendApiResponseSequence(SAI_COMMON_API_SET, SAI_STATUS_SUCCESS); + LogToModuleFile("1", "add sendApiResponseUpdateRedisQuadEvent to Lambda {}", sequenceNumber); + auto lambda = [=]() { + sendApiResponseUpdateRedisQuadEvent(api, SAI_STATUS_SUCCESS, kco, sequenceNumber); + }; + sendStausAdvancedResponseSequence(sequenceNumber,lambda); return SAI_STATUS_SUCCESS; } @@ -2481,7 +2594,10 @@ sai_status_t Syncd::processQuadInInitViewModeGet( _In_ sai_object_type_t objectType, _In_ const std::string& strObjectId, _In_ uint32_t attr_count, - _In_ sai_attribute_t *attr_list) + _In_ sai_attribute_t *attr_list, + _In_ sai_common_api_t api, + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int sequenceNumber) { SWSS_LOG_ENTER(); @@ -2516,8 +2632,11 @@ sai_status_t Syncd::processQuadInInitViewModeGet( sai_serialize_object_type(objectType).c_str()); status = SAI_STATUS_INVALID_OBJECT_ID; - - sendGetResponseSequence(objectType, strObjectId, switchVid, status, attr_count, attr_list); + LogToModuleFile("1", "1. add sendGetResponseUpdateRedisQuadEvent to Lambda {}", sequenceNumber); + auto lambda = [=]() { + sendGetResponseUpdateRedisQuadEvent(objectType, strObjectId, switchVid, status, attr_count, attr_list, kco, api); + }; + sendStausAdvancedResponseSequence(sequenceNumber,lambda); return status; } @@ -2561,8 +2680,11 @@ sai_status_t Syncd::processQuadInInitViewModeGet( * switch id, we could also have this method inside metadata to get meta * key. */ - - sendGetResponseSequence(objectType, strObjectId, switchVid, status, attr_count, attr_list); + LogToModuleFile("1", "2. add sendGetResponseUpdateRedisQuadEvent to Lambda {}", sequenceNumber); + auto lambda = [=]() { + sendGetResponseUpdateRedisQuadEvent(objectType, strObjectId, switchVid, status, attr_count, attr_list, kco, api); + }; + sendStausAdvancedResponseSequence(sequenceNumber,lambda); return status; } @@ -2630,8 +2752,8 @@ void Syncd::sendApiResponse( SWSS_LOG_INFO("sending response for %s api with status: %s", sai_serialize_common_api(api).c_str(), strStatus.c_str()); - - m_selectableChannel->set(strStatus, entry, REDIS_ASIC_STATE_COMMAND_GETRESPONSE); + + sendStatusAndEntryResponse(status, REDIS_ASIC_STATE_COMMAND_GETRESPONSE, entry); SWSS_LOG_INFO("response for %s api was send", sai_serialize_common_api(api).c_str()); @@ -2640,44 +2762,6 @@ void Syncd::sendApiResponse( } -void Syncd::sendApiResponseSequence( - _In_ sai_common_api_t api, - _In_ sai_status_t status, - _In_ uint32_t object_count, - _In_ sai_status_t* object_statuses, - _In_ int sequenceNumber) -{ - SWSS_LOG_ENTER(); - - //sequenceNumber = INVALID_SEQUENCE_NUMBER; - - if(sequenceNumber != INVALID_SEQUENCE_NUMBER) { - std::string logMessage = "multithreaded: valid sequence number " + std::to_string(sequenceNumber) + ", API " + sai_serialize_common_api(api) + ", status " + std::to_string(status) + ", object count: " + std::to_string(object_count); - LogToModuleFile("1", logMessage.c_str()); - - // If the response is to be sequenced, then add it to the sequencer - auto lambda = [=]() { - LogToModuleFile("1", "multithreaded: sendApiResponseSequence lambda start"); - sendApiResponse(api, status, object_count, object_statuses); - LogToModuleFile("1", "multithreaded: sendApiResponseSequence lambda end"); - }; - - if(int seq_status = m_sequencer->executeFuncInSequence(sequenceNumber, lambda) != Sequencer::SUCCESS) { - logMessage = "multithreaded: executeFuncInSequence failed for sequence number " + std::to_string(sequenceNumber) + ", status: " + std::to_string(seq_status); - LogToModuleFile("1", logMessage); - } - } - else { - // If the response is not to be sequenced, then send it directly - //SWSS_LOG_NOTICE("multithreaded: invalid sequence number, api %s, status %d, object_count %d", sai_serialize_common_api(api).c_str(), status, object_count); - if(object_statuses == NULL) { - //SWSS_LOG_NOTICE("multithreaded: object_statuses is NULL"); - } - sendApiResponse(api, status, object_count, object_statuses); - //SWSS_LOG_NOTICE("multithreaded: sendApiResponse sent"); - } -} - void Syncd::processFlexCounterGroupEvent( // TODO must be moved to go via ASIC channel queue _In_ swss::ConsumerTable& consumer) { @@ -2702,7 +2786,8 @@ sai_status_t Syncd::processFlexCounterGroupEvent( _In_ const std::string &groupName, _In_ const std::string &op, _In_ const std::vector &values, - _In_ bool fromAsicChannel) + _In_ bool fromAsicChannel, + _In_ int sequenceNumber) { SWSS_LOG_ENTER(); @@ -2729,7 +2814,11 @@ sai_status_t Syncd::processFlexCounterGroupEvent( if (fromAsicChannel) { - sendApiResponseSequence(SAI_COMMON_API_SET, SAI_STATUS_SUCCESS); + LogToModuleFile("1", "add sendApiResponse to Lambda sequenceNumber {}", sequenceNumber); + auto lambda = [=]() { + sendApiResponse(SAI_COMMON_API_SET, SAI_STATUS_SUCCESS); + }; + sendStausAdvancedResponseSequence(sequenceNumber,lambda); } return SAI_STATUS_SUCCESS; @@ -2758,8 +2847,9 @@ void Syncd::processFlexCounterEvent( // TODO must be moved to go via ASIC channe sai_status_t Syncd::processFlexCounterEvent( _In_ const std::string &key, _In_ const std::string &op, - _In_ const std::vector &values, - _In_ bool fromAsicChannel) + _In_ const std::vector &values, + _In_ bool fromAsicChannel, + _In_ int sequenceNumber) { SWSS_LOG_ENTER(); @@ -2771,7 +2861,11 @@ sai_status_t Syncd::processFlexCounterEvent( if (fromAsicChannel) { - sendApiResponseSequence(SAI_COMMON_API_SET, SAI_STATUS_FAILURE); + LogToModuleFile("1", "add sendApiResponse to Lambda {}", sequenceNumber); + auto lambda = [=]() { + sendApiResponse(SAI_COMMON_API_SET, SAI_STATUS_FAILURE); + }; + sendStausAdvancedResponseSequence(sequenceNumber,lambda); } return SAI_STATUS_FAILURE; // if key is invalid there is no need to process this event again @@ -2825,7 +2919,11 @@ sai_status_t Syncd::processFlexCounterEvent( if (fromAsicChannel) { - sendApiResponseSequence(SAI_COMMON_API_SET, SAI_STATUS_SUCCESS); + LogToModuleFile("1", "2. add sendApiResponse to Lambda sequenceNumber {}", sequenceNumber); + auto lambda = [=]() { + sendApiResponse(SAI_COMMON_API_SET, SAI_STATUS_SUCCESS); + }; + sendStausAdvancedResponseSequence(sequenceNumber,lambda); } return SAI_STATUS_SUCCESS; @@ -3028,11 +3126,13 @@ void Syncd::syncUpdateRedisBulkQuadEvent( timer.inc(statuses.size()); } -void Syncd::pushRingBuffer(AnyTask&& func) +void Syncd::pushRingBuffer(SyncdRing* ringBuffer, AnyTask&& func) { - if (!gRingBuffer || !gRingBuffer->Started) + SWSS_LOG_ENTER(); + if (!ringBuffer || !ringBuffer->Started) { //SWSS_LOG_NOTICE("multithreaded: pushRingBuffer: gRingBuffer is not started"); + LogToModuleFile("1", "execute fun since ringBuffer not valid or not started"); func(); // } else if (!gRingBuffer->Serves(getName())) { // while (!gRingBuffer->IsEmpty() || !gRingBuffer->Idle) { @@ -3040,12 +3140,12 @@ void Syncd::pushRingBuffer(AnyTask&& func) // } // func(); } else { - while (!gRingBuffer->push(func)) { + while (!ringBuffer->push(func)) { SWSS_LOG_WARN("fail to push..ring is full..."); } //SWSS_LOG_NOTICE("multithreaded: pushRingBuffer: notify_one"); LogToModuleFile("1", "multithreaded: pushRingBuffer: notify_one"); - gRingBuffer->cv.notify_one(); + ringBuffer->cv.notify_one(); } } @@ -3125,9 +3225,7 @@ sai_status_t Syncd::processQuadEventTag( if (isInitViewMode()) { - sai_status_t status = processQuadEventInInitViewMode(metaKey.objecttype, strObjectId, api, attr_count, attr_list); - - syncUpdateRedisQuadEvent(status, api, kco); + sai_status_t status = processQuadEventInInitViewMode(metaKey.objecttype, strObjectId, api, attr_count, attr_list, kco, sequenceNumber); //SWSS_LOG_NOTICE("multithreaded: isInitViewMode()"); @@ -3198,12 +3296,22 @@ sai_status_t Syncd::processQuadEventTag( sai_object_id_t switchVid = VidManager::switchIdQuery(metaKey.objectkey.key.object_id); //SWSS_LOG_NOTICE("multithreaded: SAI_COMMON_API_GET"); + LogToModuleFile("1", "add sendGetResponseUpdateRedisQuadEvent to Lambda sequenceNumber {}", sequenceNumber); + auto lambda = [=]() { + sendGetResponseUpdateRedisQuadEvent(metaKey.objecttype, strObjectId, switchVid, status, attr_count, attr_list, kco, api); + }; + sendStausAdvancedResponseSequence(sequenceNumber,lambda); + + return SAI_STATUS_SUCCESS; - sendGetResponseSequence(metaKey.objecttype, strObjectId, switchVid, status, attr_count, attr_list, sequenceNumber); } else if (status != SAI_STATUS_SUCCESS) { - sendApiResponseSequence(api, status, 0, NULL, sequenceNumber=sequenceNumber); + LogToModuleFile("1", "1. add sendApiResponseUpdateRedisQuadEvent to Lambda sequenceNumber {}", sequenceNumber); + auto lambda = [=]() { + sendApiResponseUpdateRedisQuadEvent(api, status, kco, sequenceNumber); + }; + sendStausAdvancedResponseSequence(sequenceNumber,lambda); if (info->isobjectid && api == SAI_COMMON_API_SET) { @@ -3232,11 +3340,13 @@ sai_status_t Syncd::processQuadEventTag( } else // non GET api, status is SUCCESS { - sendApiResponseSequence(api, status, 0, NULL, sequenceNumber=sequenceNumber); + LogToModuleFile("1", "2. add sendApiResponseUpdateRedisQuadEvent to Lambda sequenceNumber {}", sequenceNumber); + auto lambda = [=]() { + sendApiResponseUpdateRedisQuadEvent(api, status, kco, sequenceNumber); + }; + sendStausAdvancedResponseSequence(sequenceNumber,lambda); } - syncUpdateRedisQuadEvent(status, api, kco); - return status; } @@ -3718,43 +3828,11 @@ void Syncd::sendGetResponse( * response will not put any data to table, only queue is used. */ - m_selectableChannel->set(strStatus, entry, REDIS_ASIC_STATE_COMMAND_GETRESPONSE); + sendStatusAndEntryResponse(status, REDIS_ASIC_STATE_COMMAND_GETRESPONSE, entry); SWSS_LOG_INFO("response for GET api was send"); } -void Syncd::sendGetResponseSequence( - _In_ sai_object_type_t objectType, - _In_ const std::string& strObjectId, - _In_ sai_object_id_t switchVid, - _In_ sai_status_t status, - _In_ uint32_t attr_count, - _In_ sai_attribute_t *attr_list, - _In_ int sequenceNumber) { - SWSS_LOG_ENTER(); - //SWSS_LOG_NOTICE("multithreaded: sendGetResponseSequence, sequenceNumber=%d, object type: %d, str object id %s", sequenceNumber, objectType, strObjectId); - std::string logMessage = "multithreaded: sendGetResponseSequence, sequenceNumber=" + std::to_string(sequenceNumber) + ", object type: " + std::to_string(objectType) + ", str object id " + strObjectId; - LogToModuleFile("1", logMessage.c_str()); - - //sequenceNumber = INVALID_SEQUENCE_NUMBER; - - if(sequenceNumber != INVALID_SEQUENCE_NUMBER) { - auto lambda = [=]() { - LogToModuleFile("1", "multithreaded: sendGetResponseSequence: lambda start"); - sendGetResponse(objectType, strObjectId, switchVid, status, attr_count, attr_list); - LogToModuleFile("1", "multithreaded: sendGetResponseSequence: lambda end"); - }; - - if(int seq_status = m_sequencer->executeFuncInSequence(sequenceNumber, lambda) != Sequencer::SUCCESS) { - logMessage = "multithreaded: sendGetResponseSequence: executeFuncInSequence failed, sequenceNumber=" + std::to_string(sequenceNumber) + ", status=" + std::to_string(seq_status); - LogToModuleFile("1", logMessage); - } - } - else { - sendGetResponse(objectType, strObjectId, switchVid, status, attr_count, attr_list); - } -} - void Syncd::snoopGetResponse( _In_ sai_object_type_t object_type, _In_ const std::string& strObjectId, // can be non object id @@ -4048,7 +4126,8 @@ void Syncd::inspectAsic() } sai_status_t Syncd::processNotifySyncd( - _In_ const swss::KeyOpFieldsValuesTuple &kco) + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int sequenceNumber) { SWSS_LOG_ENTER(); @@ -4066,7 +4145,7 @@ sai_status_t Syncd::processNotifySyncd( SWSS_LOG_ERROR("Error in executing SAI failure dump %s", ret_str.c_str()); status = SAI_STATUS_FAILURE; } - sendNotifyResponse(status); + sendNotifyResponse(status, sequenceNumber); return status; } @@ -4074,7 +4153,7 @@ sai_status_t Syncd::processNotifySyncd( { SWSS_LOG_NOTICE("received %s, ignored since TEMP VIEW is not used, returning success", key.c_str()); - sendNotifyResponse(SAI_STATUS_SUCCESS); + sendNotifyResponse(SAI_STATUS_SUCCESS, sequenceNumber); return SAI_STATUS_SUCCESS; } @@ -4136,7 +4215,7 @@ sai_status_t Syncd::processNotifySyncd( SWSS_LOG_THROW("unknown operation: %s", key.c_str()); } - sendNotifyResponse(status); + sendNotifyResponse(status, sequenceNumber); return status; } @@ -4158,7 +4237,7 @@ sai_status_t Syncd::processNotifySyncd( SWSS_LOG_WARN("syncd switched to INIT VIEW mode, all op will be saved to TEMP view"); - sendNotifyResponse(SAI_STATUS_SUCCESS); + sendNotifyResponse(SAI_STATUS_SUCCESS, sequenceNumber); } else if (redisNotifySyncd == SAI_REDIS_NOTIFY_SYNCD_APPLY_VIEW) { @@ -4182,12 +4261,12 @@ sai_status_t Syncd::processNotifySyncd( * it will not be processed until get response timeout will hit. */ - sendNotifyResponse(SAI_STATUS_FAILURE); + sendNotifyResponse(SAI_STATUS_FAILURE, sequenceNumber); throw; } - sendNotifyResponse(status); + sendNotifyResponse(status, sequenceNumber); if (status == SAI_STATUS_SUCCESS) { @@ -4225,13 +4304,13 @@ sai_status_t Syncd::processNotifySyncd( inspectAsic(); - sendNotifyResponse(SAI_STATUS_SUCCESS); + sendNotifyResponse(SAI_STATUS_SUCCESS, sequenceNumber); } else { SWSS_LOG_ERROR("unknown operation: %s", key.c_str()); - sendNotifyResponse(SAI_STATUS_NOT_IMPLEMENTED); + sendNotifyResponse(SAI_STATUS_NOT_IMPLEMENTED, sequenceNumber); SWSS_LOG_THROW("notify syncd %s operation failed", key.c_str()); } @@ -4240,7 +4319,8 @@ sai_status_t Syncd::processNotifySyncd( } void Syncd::sendNotifyResponse( - _In_ sai_status_t status) + _In_ sai_status_t status, + _In_ int sequenceNumber) { SWSS_LOG_ENTER(); @@ -4250,7 +4330,8 @@ void Syncd::sendNotifyResponse( SWSS_LOG_INFO("sending response: %s", strStatus.c_str()); - m_selectableChannel->set(strStatus, entry, REDIS_ASIC_STATE_COMMAND_NOTIFY); + sendStausResponseSequence(status, REDIS_ASIC_STATE_COMMAND_NOTIFY, entry, sequenceNumber); + } void Syncd::clearTempView() @@ -5326,6 +5407,87 @@ static void timerWatchdogCallback( SWSS_LOG_ERROR("main loop execution exceeded %ld ms", span/1000); } +void Syncd::getObjectTypeByOperation( + _In_ swss::KeyOpFieldsValuesTuple kco, + _Out_ sai_object_type_t &objectType) +{ + objectType = SAI_OBJECT_TYPE_NULL; // Initialize the output argument + sai_object_meta_key_t metaKey; + + auto& key = kfvKey(kco); + auto& op = kfvOp(kco); + LogToModuleFile("1", "op {} key {}", op.c_str(), key.c_str()); + if (op == REDIS_ASIC_STATE_COMMAND_CREATE || + op == REDIS_ASIC_STATE_COMMAND_REMOVE || + op == REDIS_ASIC_STATE_COMMAND_SET || + op == REDIS_ASIC_STATE_COMMAND_GET || + op == REDIS_ASIC_STATE_COMMAND_BULK_SET || + op == REDIS_ASIC_STATE_COMMAND_BULK_REMOVE || + op == REDIS_ASIC_STATE_COMMAND_BULK_CREATE || + op == REDIS_ASIC_STATE_COMMAND_GET_STATS || + op == REDIS_ASIC_STATE_COMMAND_CLEAR_STATS) + { + sai_deserialize_object_meta_key(key, metaKey); + objectType = metaKey.objecttype; + LogToModuleFile("1", "op {} has object id ", op.c_str(), sai_serialize_object_type(objectType).c_str()); + } + else if (op == REDIS_ASIC_STATE_COMMAND_ATTR_CAPABILITY_QUERY || + op == REDIS_ASIC_STATE_COMMAND_ATTR_ENUM_VALUES_CAPABILITY_QUERY) + { + auto& values = kfvFieldsValues(kco); + if (values.size() > 0) + { + sai_deserialize_object_type(fvValue(values[0]), objectType); + LogToModuleFile("1", "op {} has object id ", op.c_str(), sai_serialize_object_type(objectType).c_str()); + } + else + LogToModuleFile("1", "Error op {} has no value", op.c_str()); + } + else if (op == REDIS_ASIC_STATE_COMMAND_OBJECT_TYPE_GET_AVAILABILITY_QUERY) + { + std::vector values = kfvFieldsValues(kco); + sai_deserialize_object_type(fvValue(values.back()), objectType); + LogToModuleFile("1", "op {} has object id ", op.c_str(), sai_serialize_object_type(objectType).c_str()); + } +} + +void Syncd::getApiRingBuffer( + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _Out_ SyncdRing*& ringBuffer) +{ + ringBuffer = nullptr; + + auto& key = kfvKey(kco); + auto& op = kfvOp(kco); + + SWSS_LOG_DEBUG("op: %s key:%s", op.c_str(), key.c_str()); + + auto it = operationGroups.find(op); + if (it == operationGroups.end()) //not found + { + sai_object_type_t objectType = SAI_OBJECT_TYPE_NULL; + getObjectTypeByOperation(kco, objectType); + + if (SAI_OBJECT_TYPE_NULL != objectType) // valid object type valid + { + LogToModuleFile("1", "find ring buffer by object type {}",sai_serialize_object_type(objectType).c_str()); + SWSS_LOG_DEBUG("objectType %s ", sai_serialize_object_type(objectType).c_str()); + it = operationGroups.find(sai_serialize_object_type(objectType).c_str()); + } + } + else + LogToModuleFile("1", "find ring buffer by op {}", op.c_str()); + + if (it != operationGroups.end()) + { + OperationGroup& group = it->second; + ringBuffer = group.ringBuffer; + LogToModuleFile("1", "found ring buffer {}", getNameByRingBuffer(ringBuffer)); + } + else + LogToModuleFile("1", "didnt match ring buffer to api"); +} + void Syncd::run() { SWSS_LOG_ENTER(); @@ -5394,26 +5556,6 @@ void Syncd::run() swss::Selectable *sel = NULL; int result = s->select(&sel); - int sequence_number; - -#if 0 - if(int seq_status = m_sequencer->allocateSequenceNumber(&sequence_number) != Sequencer::SUCCESS) - { - std::string logMsg = "Failed to allocate sequence number: " + std::to_string(seq_status); - LogToModuleFile("1", logMsg); - auto lambda = [=](){}; - pushRingBuffer(lambda); - } - else { - std::string logMsg = "Allocated sequence number: " + std::to_string(sequence_number); - LogToModuleFile("1", logMsg); - } - - // in case of failure, make sure that sequence number is correct - - std::string logMessage = "multithreaded: BEFORE PUSH INTO RING BUFFER sequence number: " + std::to_string(sequence_number); - LogToModuleFile("1", logMessage.c_str()); -#endif //restart query sequencer if (sel == m_restartQuery.get()) @@ -5509,6 +5651,17 @@ void Syncd::run() } else if (sel == m_flexCounter.get()) { + int sequenceNumber; + if(int seq_status = m_sequencer->allocateSequenceNumber(&sequenceNumber) != Sequencer::SUCCESS) + { + std::string logMsg = "Failed to allocate sequence number: " + std::to_string(seq_status); + LogToModuleFile("1", logMsg); + //todo: handle wait until sequence number is available with timeout + } + else { + std::string logMsg = "Allocated sequence number: " + std::to_string(sequenceNumber); + LogToModuleFile("1", logMsg); + } //directly to sequencer auto lambda = [=](){ LogToModuleFile("1", "multithreaded: inside lambda, start m_flexCounter"); @@ -5516,15 +5669,29 @@ void Syncd::run() LogToModuleFile("1", "multithreaded: inside lambda, end m_flexCounter"); }; - sequence_number = getSeqNumber(); - if(int seq_status = m_sequencer->executeFuncInSequence(sequence_number, lambda) != Sequencer::SUCCESS) + if(int seq_status = m_sequencer->executeFuncInSequence(sequenceNumber, lambda) != Sequencer::SUCCESS) { - std::string logMessage = "m_flexCounter failed to execute function in sequence, status: " + std::to_string(seq_status) + ", sequence number: " + std::to_string(sequence_number); + std::string logMessage = "m_flexCounter failed to execute function in sequence, status: " + std::to_string(seq_status) + ", sequence number: " + std::to_string(sequenceNumber); LogToModuleFile("1", logMessage); } } else if (sel == m_flexCounterGroup.get()) { + int sequenceNumber; + if(int seq_status = m_sequencer->allocateSequenceNumber(&sequenceNumber) != Sequencer::SUCCESS) + { + std::string logMsg = "Failed to allocate sequence number: " + std::to_string(seq_status); + LogToModuleFile("1", logMsg); + //todo: handle wait until sequence number is available with timeout + } + else { + std::string logMsg = "Allocated sequence number: " + std::to_string(sequenceNumber); + LogToModuleFile("1", logMsg); + } + + std::string logMessage = "multithreaded: BEFORE PUSH INTO RING BUFFER sequence number: " + std::to_string(sequenceNumber); + LogToModuleFile("1", logMessage.c_str()); + //directly to sequencer auto lambda = [=](){ LogToModuleFile("1", "multithreaded: inside lambda, start m_flexCounterGroup"); @@ -5532,26 +5699,18 @@ void Syncd::run() LogToModuleFile("1", "multithreaded: inside lambda, end m_flexCounterGroup"); }; - sequence_number = getSeqNumber(); - if(int seq_status = m_sequencer->executeFuncInSequence(sequence_number, lambda) != Sequencer::SUCCESS) + if(int seq_status = m_sequencer->executeFuncInSequence(sequenceNumber, lambda) != Sequencer::SUCCESS) { - std::string logMessage = "m_flexCounterGroup failed to execute function in sequence, status: " + std::to_string(seq_status) + ", sequence number: " + std::to_string(sequence_number); + logMessage = "m_flexCounterGroup failed to execute function in sequence, status: " + std::to_string(seq_status) + ", sequence number: " + std::to_string(sequenceNumber); LogToModuleFile("1", logMessage); } } else if (sel == m_selectableChannel.get()) { - auto lambda = [=](){ - LogToModuleFile("1", "multithreaded: inside lambda, start processing event"); - processEvent(*m_selectableChannel.get()); - LogToModuleFile("1", "multithreaded: inside lambda, end processing event"); - }; - - pushRingBuffer(lambda); + processEvent(*m_selectableChannel.get()); } else { - SWSS_LOG_ERROR("select failed: %d", result); } } @@ -5655,4 +5814,74 @@ syncd_restart_type_t Syncd::handleRestartQuery( SWSS_LOG_NOTICE("received %s switch shutdown event", op.c_str()); return RequestShutdownCommandLineOptions::stringToRestartType(op); +} + +void Syncd::sendStatusAndEntryResponse( + _In_ sai_status_t status, + _In_ const std::string& commandType, + _In_ const std::vector& entry) +{ + SWSS_LOG_ENTER(); + + std::string strStatus = sai_serialize_status(status); + + LogToModuleFile("1", "sending response: {} with commandType: {}", strStatus.c_str(), commandType.c_str()); + SWSS_LOG_INFO("sending response: %s with commandType: %s", strStatus.c_str(), commandType.c_str()); + + m_selectableChannel->set(strStatus, entry, commandType); +} + +void Syncd::sendStausResponseSequence( + _In_ sai_status_t status, + _In_ const std::string& commandType, + _In_ const std::vector& entry, + _In_ int sequenceNumber) +{ + SWSS_LOG_ENTER(); + SWSS_LOG_NOTICE("multithreaded: sendStausResponseSequence"); + + //sequenceNumber = INVALID_SEQUENCE_NUMBER; + + if(sequenceNumber != INVALID_SEQUENCE_NUMBER) { + SWSS_LOG_NOTICE("multithreaded: valid sequence number %d", sequenceNumber); + + // If the response is to be sequenced, then add it to the sequencer + LogToModuleFile("1", "add sendStatusAndEntryResponse to lambda with sequenceNumber {} ", sequenceNumber); + auto lambda = [=]() { + sendStatusAndEntryResponse(status, commandType, entry); + }; + if(int seq_status = m_sequencer->executeFuncInSequence(sequenceNumber, lambda) != Sequencer::SUCCESS) + { + std::string logMessage = "m_flexCounterGroup failed to execute function in sequence, status: " + std::to_string(seq_status) + ", sequence number: " + std::to_string(sequenceNumber); + LogToModuleFile("1", logMessage); + } + } + else { + // If the response is not to be sequenced, then send it directly + LogToModuleFile("1", "send sendStatusAndEntryResponse without sequencer"); + sendStatusAndEntryResponse(status, commandType, entry); + } +} + +void Syncd::sendStausAdvancedResponseSequence( + _In_ int sequenceNumber, + std::function lambdaFunc) +{ + SWSS_LOG_ENTER(); + SWSS_LOG_NOTICE("multithreaded: sendStausAdvancedResponseSequence"); + + //sequenceNumber = INVALID_SEQUENCE_NUMBER; + if(sequenceNumber != INVALID_SEQUENCE_NUMBER) { + SWSS_LOG_NOTICE("multithreaded: valid sequence number {}", sequenceNumber); + if(int seq_status = m_sequencer->executeFuncInSequence(sequenceNumber, lambdaFunc) != Sequencer::SUCCESS) + { + std::string logMessage = "m_flexCounterGroup failed to execute function in sequence, status: " + std::to_string(seq_status) + ", sequence number: " + std::to_string(sequenceNumber); + LogToModuleFile("1", logMessage); + } + } + else { + // If the response is not to be sequenced, then send it directly + LogToModuleFile("1", "execute func without sequencer"); + lambdaFunc(); + } } \ No newline at end of file diff --git a/syncd/Syncd.h b/syncd/Syncd.h index b42ed3c05..20e413b85 100644 --- a/syncd/Syncd.h +++ b/syncd/Syncd.h @@ -25,20 +25,32 @@ #include "swss/consumertable.h" #include "swss/producertable.h" #include "swss/notificationconsumer.h" -#include -#include +#include "SyncdMultipleRingBuff.h" #include "Sequencer.h" +#include +#include +#include +#include +#include "sairediscommon.h" + +#include +#include +#include namespace syncd { - #define ORCH_RING_SIZE 1024 - #define SLEEP_MSECONDS 500 - using AnyTask = std::function; - template - class RingBuffer; - typedef RingBuffer SyncdRing; + #define ORCH_RING_SIZE 1024 + #define SLEEP_MSECONDS 500 + using AnyTask = std::function; + + + typedef syncdMultipleRingBuff::RingBuffer SyncdRing; + struct OperationGroup { + std::set operations; + SyncdRing* ringBuffer; + }; class Syncd { @@ -46,7 +58,6 @@ namespace syncd Syncd(const Syncd&) = delete; Syncd& operator=(const Syncd&) = delete; - std::thread ring_thread; public: @@ -58,6 +69,11 @@ namespace syncd virtual ~Syncd(); public: + + struct SaiObjectType { + sai_object_type_t enumValue; + std::string enumString; + }; bool getAsicInitViewMode() const; @@ -75,18 +91,17 @@ namespace syncd public: // TODO private - int getSeqNumber(); - void processEvent( - _In_ sairedis::SelectableChannel& consumer, - _In_ int sequence_number = INVALID_SEQUENCE_NUMBER); + _In_ sairedis::SelectableChannel& consumer); sai_status_t processQuadEventInInitViewMode( _In_ sai_object_type_t objectType, _In_ const std::string& strObjectId, _In_ sai_common_api_t api, _In_ uint32_t attr_count, - _In_ sai_attribute_t *attr_list); + _In_ sai_attribute_t *attr_list, + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int sequenceNumber); void processFlexCounterGroupEvent( _In_ swss::ConsumerTable &consumer); @@ -136,34 +151,41 @@ namespace syncd private: sai_status_t processNotifySyncd( - _In_ const swss::KeyOpFieldsValuesTuple &kco); + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int sequenceNumber); sai_status_t processSingleEvent( _In_ const swss::KeyOpFieldsValuesTuple &kco, - _In_ int sequence_number=INVALID_SEQUENCE_NUMBER); + _In_ int sequenceNumber); sai_status_t processAttrCapabilityQuery( - _In_ const swss::KeyOpFieldsValuesTuple &kco); + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int sequenceNumber); sai_status_t processAttrEnumValuesCapabilityQuery( - _In_ const swss::KeyOpFieldsValuesTuple &kco); + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int sequenceNumber); sai_status_t processObjectTypeGetAvailabilityQuery( - _In_ const swss::KeyOpFieldsValuesTuple &kco); + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int sequenceNumber); sai_status_t processFdbFlush( - _In_ const swss::KeyOpFieldsValuesTuple &kco); + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int sequenceNumber); sai_status_t processClearStatsEvent( - _In_ const swss::KeyOpFieldsValuesTuple &kco); + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int sequenceNumber); sai_status_t processGetStatsEvent( - _In_ const swss::KeyOpFieldsValuesTuple &kco); + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int sequenceNumber); sai_status_t processQuadEvent( _In_ sai_common_api_t api, _In_ const swss::KeyOpFieldsValuesTuple &kco, - _In_ int sequenceNumber = INVALID_SEQUENCE_NUMBER); + _In_ int sequenceNumber); sai_status_t processQuadEventTag( _In_ sai_common_api_t api, @@ -177,7 +199,7 @@ namespace syncd sai_status_t processBulkQuadEvent( _In_ sai_common_api_t api, _In_ const swss::KeyOpFieldsValuesTuple &kco, - _In_ int sequenceNumber = INVALID_SEQUENCE_NUMBER); + _In_ int sequenceNumber); sai_status_t processBulkOid( _In_ sai_object_type_t objectType, @@ -185,7 +207,7 @@ namespace syncd _In_ sai_common_api_t api, _In_ const std::vector> &attributes, _In_ const std::vector>& strAttributes, - _In_ int sequenceNumber = INVALID_SEQUENCE_NUMBER); + _In_ int sequenceNumber); sai_status_t processBulkEntry( _In_ sai_object_type_t objectType, @@ -193,7 +215,7 @@ namespace syncd _In_ sai_common_api_t api, _In_ const std::vector> &attributes, _In_ const std::vector>& strAttributes, - _In_ int sequenceNumber = INVALID_SEQUENCE_NUMBER); + _In_ int sequenceNumber); sai_status_t processBulkCreateEntry( _In_ sai_object_type_t objectType, @@ -217,7 +239,8 @@ namespace syncd _In_ const std::vector &object_ids, _In_ sai_common_api_t api, _In_ const std::vector> &attributes, - _In_ const std::vector>& strAttributes); + _In_ const std::vector>& strAttributes, + _In_ int sequenceNumber); sai_status_t processOid( _In_ sai_object_type_t objectType, @@ -230,13 +253,15 @@ namespace syncd _In_ const std::string &key, _In_ const std::string &op, _In_ const std::vector &values, - _In_ bool fromAsicChannel=true); + _In_ bool fromAsicChannel = true, + _In_ int sequenceNumber = 0); /*todo: replcae to SEQ_NUM_NV*/ sai_status_t processFlexCounterEvent( _In_ const std::string &key, _In_ const std::string &op, _In_ const std::vector &values, - _In_ bool fromAsicChannel=true); + _In_ bool fromAsicChannel = true, + _In_ int sequenceNumber = 0); /*todo: replcae to SEQ_NUM_NV*/ private: // process quad oid @@ -282,22 +307,34 @@ namespace syncd _In_ sai_object_type_t objectType, _In_ const std::string& strObjectId, _In_ uint32_t attr_count, - _In_ sai_attribute_t *attr_list); + _In_ sai_attribute_t *attr_list, + _In_ sai_common_api_t api, + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int sequenceNumber); sai_status_t processQuadInInitViewModeRemove( _In_ sai_object_type_t objectType, - _In_ const std::string& strObjectId); + _In_ const std::string& strObjectId, + _In_ sai_common_api_t api, + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int sequenceNumber); sai_status_t processQuadInInitViewModeSet( _In_ sai_object_type_t objectType, _In_ const std::string& strObjectId, - _In_ sai_attribute_t *attr); + _In_ sai_attribute_t *attr, + _In_ sai_common_api_t api, + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int sequenceNumber); sai_status_t processQuadInInitViewModeGet( _In_ sai_object_type_t objectType, _In_ const std::string& strObjectId, _In_ uint32_t attr_count, - _In_ sai_attribute_t *attr_list); + _In_ sai_attribute_t *attr_list, + _In_ sai_common_api_t api, + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int sequenceNumber); private: @@ -313,6 +350,55 @@ namespace syncd _In_ const std::vector& objectIds, _In_ const std::vector>& strAttributes); + void sendStausResponseSequence( + _In_ sai_status_t status, + _In_ const std::string& commandType, + _In_ const std::vector& entry, + _In_ int sequenceNumber); + + void sendStausAdvancedResponseSequence( + _In_ int sequenceNumber, + std::function lambdaFunc); + + void sendApiResponseUpdateRedisBulkQuadEventWithObjectInsertion( + _In_ sai_common_api_t api, + _In_ sai_status_t status, + _In_ sai_object_type_t objectType, + _In_ const std::vector& objectIds, + _In_ const std::vector>& strAttributes); + + void sendApiResponseUpdateRedisBulkQuadEvent( + _In_ sai_common_api_t api, + _In_ sai_status_t status, + _In_ sai_object_type_t objectType, + _In_ const std::vector& objectIds, + _In_ const std::vector>& strAttributes) ; + void sendApiResponseUpdateRedisQuadEvent( + _In_ sai_common_api_t api, + _In_ sai_status_t status, + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int sequenceNumber); + + void sendGetResponseUpdateRedisQuadEvent( + _In_ sai_object_type_t objectType, + _In_ const std::string& strObjectId, + _In_ sai_object_id_t switchVid, + _In_ sai_status_t status, + _In_ uint32_t attr_count, + _In_ sai_attribute_t *attr_list, + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ sai_common_api_t api); + + void sendStatusAndEntryResponse( + _In_ sai_status_t status, + _In_ const std::string& commandType, + _In_ const std::vector& entry); + + void processFdbFlushResponse( + _In_ sai_status_t status, + _In_ std::vector values, + _In_ sai_object_id_t switchVid); + public: // TODO to private sai_status_t processEntry( @@ -378,32 +464,17 @@ namespace syncd _In_ uint32_t object_count = 0, _In_ sai_status_t * object_statuses = NULL); - void sendApiResponseSequence( - _In_ sai_common_api_t api, - _In_ sai_status_t status, - _In_ uint32_t object_count = 0, - _In_ sai_status_t * object_statuses = NULL, - _In_ int sequenceNumber = INVALID_SEQUENCE_NUMBER); - void sendGetResponse( _In_ sai_object_type_t objectType, _In_ const std::string& strObjectId, _In_ sai_object_id_t switchVid, _In_ sai_status_t status, _In_ uint32_t attr_count, - _In_ sai_attribute_t *attr_list); - - void sendGetResponseSequence( - _In_ sai_object_type_t objectType, - _In_ const std::string& strObjectId, - _In_ sai_object_id_t switchVid, - _In_ sai_status_t status, - _In_ uint32_t attr_count, - _In_ sai_attribute_t *attr_list, - _In_ int sequenceNumber = INVALID_SEQUENCE_NUMBER); + _In_ sai_attribute_t *attr_listsendApiResponse); void sendNotifyResponse( - _In_ sai_status_t status); + _In_ sai_status_t status, + _In_ int sequenceNumber); private: // snoop get response oids @@ -447,8 +518,8 @@ namespace syncd public: // TODO to private bool m_asicInitViewMode; - void pushRingBuffer(AnyTask&& func); - void popRingBuffer(); + void pushRingBuffer(SyncdRing* ringBuffer , AnyTask&& func); + void popRingBuffer(SyncdRing* ringBuffer); void enableRingBuffer(); std::shared_ptr m_manager; @@ -563,130 +634,211 @@ namespace syncd std::set m_createdInInitView; - protected: - /* Orchdaemon instance points to the same ring buffer during its lifetime */ - SyncdRing* gRingBuffer = nullptr; - std::atomic ring_thread_exited{false}; - }; - typedef std::map EventMap; - template - class RingBuffer - { private: - static RingBuffer* instance; - std::vector buffer; - int head = 0; - int tail = 0; - void point(); - EventMap m_eventMap; - protected: - RingBuffer(): buffer(RingSize) {} - ~RingBuffer() { - delete instance; - } - public: - RingBuffer(const RingBuffer&) = delete; - RingBuffer(RingBuffer&&) = delete; - RingBuffer& operator= (const RingBuffer&) = delete; - RingBuffer& operator= (RingBuffer&&) = delete; - static RingBuffer* Get(); - bool Started = false; - bool Idle = true; - std::mutex mtx; - std::condition_variable cv; - bool IsFull(); - bool IsEmpty(); - bool push(DataType entry); - bool pop(DataType& entry); - DataType& HeadEntry(); - void addEvent(std::string* executor); - void doTask(); - bool tasksPending(); - bool Serves(const std::string& tableName); + std::map ringBufferThreads; + std::map operationGroups; + std::atomic ring_thread_exited{false}; + + void getApiRingBuffer( + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _Out_ SyncdRing*& ringBuffer); + + void getObjectTypeByOperation( + _In_ swss::KeyOpFieldsValuesTuple kco, + _Out_ sai_object_type_t &objectType); + + // Declare saiObjectTypes as a member variable + std::vector saiObjectTypes = { + {SAI_OBJECT_TYPE_NULL, "SAI_OBJECT_TYPE_NULL"}, + {SAI_OBJECT_TYPE_PORT, "SAI_OBJECT_TYPE_PORT"}, + {SAI_OBJECT_TYPE_LAG, "SAI_OBJECT_TYPE_LAG"}, + {SAI_OBJECT_TYPE_VIRTUAL_ROUTER, "SAI_OBJECT_TYPE_VIRTUAL_ROUTER"}, + {SAI_OBJECT_TYPE_NEXT_HOP, "SAI_OBJECT_TYPE_NEXT_HOP"}, + {SAI_OBJECT_TYPE_NEXT_HOP_GROUP, "SAI_OBJECT_TYPE_NEXT_HOP_GROUP"}, + {SAI_OBJECT_TYPE_ROUTER_INTERFACE, "SAI_OBJECT_TYPE_ROUTER_INTERFACE"}, + {SAI_OBJECT_TYPE_ACL_TABLE, "SAI_OBJECT_TYPE_ACL_TABLE"}, + {SAI_OBJECT_TYPE_ACL_ENTRY, "SAI_OBJECT_TYPE_ACL_ENTRY"}, + {SAI_OBJECT_TYPE_ACL_COUNTER, "SAI_OBJECT_TYPE_ACL_COUNTER"}, + {SAI_OBJECT_TYPE_ACL_RANGE, "SAI_OBJECT_TYPE_ACL_RANGE"}, + {SAI_OBJECT_TYPE_ACL_TABLE_GROUP, "SAI_OBJECT_TYPE_ACL_TABLE_GROUP"}, + {SAI_OBJECT_TYPE_ACL_TABLE_GROUP_MEMBER, "SAI_OBJECT_TYPE_ACL_TABLE_GROUP_MEMBER"}, + {SAI_OBJECT_TYPE_HOSTIF, "SAI_OBJECT_TYPE_HOSTIF"}, + {SAI_OBJECT_TYPE_MIRROR_SESSION, "SAI_OBJECT_TYPE_MIRROR_SESSION"}, + {SAI_OBJECT_TYPE_SAMPLEPACKET, "SAI_OBJECT_TYPE_SAMPLEPACKET"}, + {SAI_OBJECT_TYPE_STP, "SAI_OBJECT_TYPE_STP"}, + {SAI_OBJECT_TYPE_HOSTIF_TRAP_GROUP, "SAI_OBJECT_TYPE_HOSTIF_TRAP_GROUP"}, + {SAI_OBJECT_TYPE_POLICER, "SAI_OBJECT_TYPE_POLICER"}, + {SAI_OBJECT_TYPE_WRED, "SAI_OBJECT_TYPE_WRED"}, + {SAI_OBJECT_TYPE_QOS_MAP, "SAI_OBJECT_TYPE_QOS_MAP"}, + {SAI_OBJECT_TYPE_QUEUE, "SAI_OBJECT_TYPE_QUEUE"}, + {SAI_OBJECT_TYPE_SCHEDULER, "SAI_OBJECT_TYPE_SCHEDULER"}, + {SAI_OBJECT_TYPE_SCHEDULER_GROUP, "SAI_OBJECT_TYPE_SCHEDULER_GROUP"}, + {SAI_OBJECT_TYPE_BUFFER_POOL, "SAI_OBJECT_TYPE_BUFFER_POOL"}, + {SAI_OBJECT_TYPE_BUFFER_PROFILE, "SAI_OBJECT_TYPE_BUFFER_PROFILE"}, + {SAI_OBJECT_TYPE_INGRESS_PRIORITY_GROUP, "SAI_OBJECT_TYPE_INGRESS_PRIORITY_GROUP"}, + {SAI_OBJECT_TYPE_LAG_MEMBER, "SAI_OBJECT_TYPE_LAG_MEMBER"}, + {SAI_OBJECT_TYPE_HASH, "SAI_OBJECT_TYPE_HASH"}, + {SAI_OBJECT_TYPE_UDF, "SAI_OBJECT_TYPE_UDF"}, + {SAI_OBJECT_TYPE_UDF_MATCH, "SAI_OBJECT_TYPE_UDF_MATCH"}, + {SAI_OBJECT_TYPE_UDF_GROUP, "SAI_OBJECT_TYPE_UDF_GROUP"}, + {SAI_OBJECT_TYPE_FDB_ENTRY, "SAI_OBJECT_TYPE_FDB_ENTRY"}, + {SAI_OBJECT_TYPE_SWITCH, "SAI_OBJECT_TYPE_SWITCH"}, + {SAI_OBJECT_TYPE_HOSTIF_TRAP, "SAI_OBJECT_TYPE_HOSTIF_TRAP"}, + {SAI_OBJECT_TYPE_HOSTIF_TABLE_ENTRY, "SAI_OBJECT_TYPE_HOSTIF_TABLE_ENTRY"}, + {SAI_OBJECT_TYPE_NEIGHBOR_ENTRY, "SAI_OBJECT_TYPE_NEIGHBOR_ENTRY"}, + {SAI_OBJECT_TYPE_ROUTE_ENTRY, "SAI_OBJECT_TYPE_ROUTE_ENTRY"}, + {SAI_OBJECT_TYPE_VLAN, "SAI_OBJECT_TYPE_VLAN"}, + {SAI_OBJECT_TYPE_VLAN_MEMBER, "SAI_OBJECT_TYPE_VLAN_MEMBER"}, + {SAI_OBJECT_TYPE_HOSTIF_PACKET, "SAI_OBJECT_TYPE_HOSTIF_PACKET"}, + {SAI_OBJECT_TYPE_TUNNEL_MAP, "SAI_OBJECT_TYPE_TUNNEL_MAP"}, + {SAI_OBJECT_TYPE_TUNNEL, "SAI_OBJECT_TYPE_TUNNEL"}, + {SAI_OBJECT_TYPE_TUNNEL_TERM_TABLE_ENTRY, "SAI_OBJECT_TYPE_TUNNEL_TERM_TABLE_ENTRY"}, + {SAI_OBJECT_TYPE_FDB_FLUSH, "SAI_OBJECT_TYPE_FDB_FLUSH"}, + {SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER, "SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER"}, + {SAI_OBJECT_TYPE_STP_PORT, "SAI_OBJECT_TYPE_STP_PORT"}, + {SAI_OBJECT_TYPE_RPF_GROUP, "SAI_OBJECT_TYPE_RPF_GROUP"}, + {SAI_OBJECT_TYPE_RPF_GROUP_MEMBER, "SAI_OBJECT_TYPE_RPF_GROUP_MEMBER"}, + {SAI_OBJECT_TYPE_L2MC_GROUP, "SAI_OBJECT_TYPE_L2MC_GROUP"}, + {SAI_OBJECT_TYPE_L2MC_GROUP_MEMBER, "SAI_OBJECT_TYPE_L2MC_GROUP_MEMBER"}, + {SAI_OBJECT_TYPE_IPMC_GROUP, "SAI_OBJECT_TYPE_IPMC_GROUP"}, + {SAI_OBJECT_TYPE_IPMC_GROUP_MEMBER, "SAI_OBJECT_TYPE_IPMC_GROUP_MEMBER"}, + {SAI_OBJECT_TYPE_L2MC_ENTRY, "SAI_OBJECT_TYPE_L2MC_ENTRY"}, + {SAI_OBJECT_TYPE_IPMC_ENTRY, "SAI_OBJECT_TYPE_IPMC_ENTRY"}, + {SAI_OBJECT_TYPE_MCAST_FDB_ENTRY, "SAI_OBJECT_TYPE_MCAST_FDB_ENTRY"}, + {SAI_OBJECT_TYPE_HOSTIF_USER_DEFINED_TRAP, "SAI_OBJECT_TYPE_HOSTIF_USER_DEFINED_TRAP"}, + {SAI_OBJECT_TYPE_BRIDGE, "SAI_OBJECT_TYPE_BRIDGE"}, + {SAI_OBJECT_TYPE_BRIDGE_PORT, "SAI_OBJECT_TYPE_BRIDGE_PORT"}, + {SAI_OBJECT_TYPE_TUNNEL_MAP_ENTRY, "SAI_OBJECT_TYPE_TUNNEL_MAP_ENTRY"}, + {SAI_OBJECT_TYPE_MAX, "SAI_OBJECT_TYPE_MAX"} + }; + + sai_status_t initializeOperationGroups() { + + // make sure all object types are accounted for + //if(saiObjectTypes.size() != (SAI_OBJECT_TYPE_MAX+1)) + // return SAI_STATUS_FAILURE; + + std::set miscOperations = { + //REDIS_ASIC_STATE_COMMAND_NOTIFY, // Not included in the list (flow without ringbuff) + REDIS_ASIC_STATE_COMMAND_GET_STATS, + REDIS_ASIC_STATE_COMMAND_CLEAR_STATS, + REDIS_ASIC_STATE_COMMAND_FLUSH, + REDIS_ASIC_STATE_COMMAND_ATTR_CAPABILITY_QUERY, + REDIS_ASIC_STATE_COMMAND_ATTR_ENUM_VALUES_CAPABILITY_QUERY, + REDIS_ASIC_STATE_COMMAND_OBJECT_TYPE_GET_AVAILABILITY_QUERY, + REDIS_FLEX_COUNTER_COMMAND_START_POLL, + REDIS_FLEX_COUNTER_COMMAND_STOP_POLL, + REDIS_FLEX_COUNTER_COMMAND_SET_GROUP, + REDIS_FLEX_COUNTER_COMMAND_DEL_GROUP + }; + + + std::array crudOperations1Enums = { + SAI_OBJECT_TYPE_NULL, + SAI_OBJECT_TYPE_PORT, + SAI_OBJECT_TYPE_LAG, + SAI_OBJECT_TYPE_ACL_TABLE, + SAI_OBJECT_TYPE_ACL_ENTRY, + SAI_OBJECT_TYPE_ACL_COUNTER, + SAI_OBJECT_TYPE_ACL_RANGE, + SAI_OBJECT_TYPE_ACL_TABLE_GROUP, + SAI_OBJECT_TYPE_ACL_TABLE_GROUP_MEMBER, + SAI_OBJECT_TYPE_HOSTIF, + SAI_OBJECT_TYPE_MIRROR_SESSION, + SAI_OBJECT_TYPE_SAMPLEPACKET, + SAI_OBJECT_TYPE_STP, + SAI_OBJECT_TYPE_HOSTIF_TRAP_GROUP, + SAI_OBJECT_TYPE_POLICER, + SAI_OBJECT_TYPE_WRED, + SAI_OBJECT_TYPE_QOS_MAP, + SAI_OBJECT_TYPE_QUEUE, + SAI_OBJECT_TYPE_SCHEDULER, + SAI_OBJECT_TYPE_SCHEDULER_GROUP, + SAI_OBJECT_TYPE_BUFFER_POOL, + SAI_OBJECT_TYPE_BUFFER_PROFILE, + SAI_OBJECT_TYPE_INGRESS_PRIORITY_GROUP, + SAI_OBJECT_TYPE_LAG_MEMBER, + SAI_OBJECT_TYPE_HASH, + SAI_OBJECT_TYPE_UDF, + SAI_OBJECT_TYPE_UDF_MATCH, + SAI_OBJECT_TYPE_UDF_GROUP, + SAI_OBJECT_TYPE_FDB_ENTRY, + //SAI_OBJECT_TYPE_SWITCH, // Not included in the list (floow without ringbuff) + SAI_OBJECT_TYPE_HOSTIF_TRAP, + SAI_OBJECT_TYPE_HOSTIF_TABLE_ENTRY, + SAI_OBJECT_TYPE_VLAN, + SAI_OBJECT_TYPE_VLAN_MEMBER, + SAI_OBJECT_TYPE_HOSTIF_PACKET, + SAI_OBJECT_TYPE_FDB_FLUSH, + SAI_OBJECT_TYPE_STP_PORT, + SAI_OBJECT_TYPE_L2MC_GROUP, + SAI_OBJECT_TYPE_L2MC_GROUP_MEMBER, + SAI_OBJECT_TYPE_L2MC_ENTRY, + SAI_OBJECT_TYPE_MCAST_FDB_ENTRY, + SAI_OBJECT_TYPE_HOSTIF_USER_DEFINED_TRAP, + SAI_OBJECT_TYPE_BRIDGE, + SAI_OBJECT_TYPE_BRIDGE_PORT + }; + + std::array crudOperations2Enums = { + SAI_OBJECT_TYPE_VIRTUAL_ROUTER, + SAI_OBJECT_TYPE_NEXT_HOP, + SAI_OBJECT_TYPE_NEXT_HOP_GROUP, + SAI_OBJECT_TYPE_ROUTER_INTERFACE, + SAI_OBJECT_TYPE_NEIGHBOR_ENTRY, + SAI_OBJECT_TYPE_ROUTE_ENTRY, + SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER, + SAI_OBJECT_TYPE_RPF_GROUP, + SAI_OBJECT_TYPE_RPF_GROUP_MEMBER, + SAI_OBJECT_TYPE_IPMC_GROUP, + SAI_OBJECT_TYPE_IPMC_GROUP_MEMBER, + SAI_OBJECT_TYPE_IPMC_ENTRY, + SAI_OBJECT_TYPE_TUNNEL_MAP, + SAI_OBJECT_TYPE_TUNNEL, + SAI_OBJECT_TYPE_TUNNEL_TERM_TABLE_ENTRY, + SAI_OBJECT_TYPE_TUNNEL_MAP_ENTRY + }; + + // Populate crudOperations1 using the enums array + std::set crudOperations1; + for (const auto& item : saiObjectTypes) { + if (std::find(crudOperations1Enums.begin(), crudOperations1Enums.end(), item.enumValue) != crudOperations1Enums.end()) { + crudOperations1.insert(item.enumString); + } + } + + // Populate crudOperations2 using the enums array + std::set crudOperations2; + for (const auto& item : saiObjectTypes) { + if (std::find(crudOperations2Enums.begin(), crudOperations2Enums.end(), item.enumValue) != crudOperations2Enums.end()) { + crudOperations2.insert(item.enumString); + } + } + + // Allocate new SyncdRing instances for each operation group + auto crudRingBuffer1 = new SyncdRing(); // Adjust size if needed + auto crudRingBuffer2 = new SyncdRing(); // Adjust size if needed + auto miscRingBuffer = new SyncdRing(); // Adjust size if needed + + // Map the operation groups + operationGroups = { + {"crud1", {crudOperations1, crudRingBuffer1}}, + {"crud2", {crudOperations2, crudRingBuffer2}}, + {"misc", {miscOperations, miscRingBuffer}}, + // Add other groups here... + }; + + return SAI_STATUS_SUCCESS; + } + + std::string getNameByRingBuffer(SyncdRing* ringBuffer) + { + for (const auto& pair : operationGroups) { + if (pair.second.ringBuffer == ringBuffer) { + return pair.first; + } + } + return "Not_found"; // Return a default value if not found + } }; - - -template -RingBuffer* RingBuffer::instance = nullptr; -template -RingBuffer* RingBuffer::Get() -{ - if (instance == nullptr) { - instance = new RingBuffer(); - SWSS_LOG_NOTICE("Syncd RingBuffer created at %p!", (void *)instance); - } - return instance; -} -template -bool RingBuffer::IsFull() -{ - return (tail + 1) % RingSize == head; -} -template -bool RingBuffer::IsEmpty() -{ - return tail == head; -} -template -bool RingBuffer::push(DataType ringEntry) -{ - if (IsFull()){ - SWSS_LOG_WARN("pushRing is full"); - return false; - } - buffer[tail] = std::move(ringEntry); - tail = (tail + 1) % RingSize; - return true; -} -template -DataType& RingBuffer::HeadEntry() { - return buffer[head]; -} -template -bool RingBuffer::pop(DataType& ringEntry) -{ - if (IsEmpty()) - return false; - ringEntry = std::move(buffer[head]); - head = (head + 1) % RingSize; - return true; -} -template -void RingBuffer::addEvent(std::string* executor) -{ -// auto inserted = m_eventMap.emplace(std::piecewise_construct, -// std::forward_as_tuple(executor->getName()), -// std::forward_as_tuple(executor)); -// // If there is duplication of executorName in m_eventMap, logic error -// if (!inserted.second) -// { -// SWSS_LOG_THROW("Duplicated executorName in m_eventMap: %s", executor->getName().c_str()); -// } -} -template -void RingBuffer::doTask() -{ -// for (auto &it : m_eventMap) { -// it.second->drain(); -// } -} -template -bool RingBuffer::tasksPending() -{ -// for (auto &it : m_eventMap) { -// auto consumer = dynamic_cast(it.second.get()); -// if (consumer->m_toSync.size()) -// return true; -// } - return false; -} -template -bool RingBuffer::Serves(const std::string& tableName) -{ -// for (auto &it : m_eventMap) { -// if (it.first == tableName) -// return true; -// } - return true; -} } diff --git a/syncd/SyncdMultipleRingBuff.h b/syncd/SyncdMultipleRingBuff.h new file mode 100644 index 000000000..50620d08a --- /dev/null +++ b/syncd/SyncdMultipleRingBuff.h @@ -0,0 +1,148 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#define MULIPLE_RING_BUFFER 1 + +namespace syncdMultipleRingBuff +{ + typedef std::map EventMap; + template + class RingBuffer{ + private: +#ifndef MULIPLE_RING_BUFFER + static RingBuffer* instance; +#endif + std::vector buffer; + int head = 0; + int tail = 0; + EventMap m_eventMap; +#ifndef MULIPLE_RING_BUFFER + protected: + RingBuffer(): buffer(RingSize) {} + ~RingBuffer() { + delete instance; + } +#else + public: + RingBuffer(): buffer(RingSize) {} + ~RingBuffer() {} +#endif + public: + #ifndef MULIPLE_RING_BUFFER + RingBuffer(const RingBuffer&) = delete; + RingBuffer(RingBuffer&&) = delete; + RingBuffer& operator= (const RingBuffer&) = delete; + RingBuffer& operator= (RingBuffer&&) = delete; + static RingBuffer* Get(); +#endif + bool Started = false; + bool Idle = true; + std::mutex mtx; + std::condition_variable cv; + bool IsFull(); + bool IsEmpty(); + bool push(DataType entry); + bool pop(DataType& entry); + DataType& HeadEntry(); + void addEvent(std::string* executor); + void doTask(); + bool tasksPending(); + bool Serves(const std::string& tableName); + }; + +#ifndef MULIPLE_RING_BUFFER + template + RingBuffer* RingBuffer::instance = nullptr; + RingBuffer* RingBuffer::Get() + { + if (instance == nullptr) { + instance = new RingBuffer(); + SWSS_LOG_NOTICE("Syncd RingBuffer created at %p!", (void *)instance); + } + return instance; + } +#endif + + template + bool RingBuffer::IsFull() + { + return (tail + 1) % RingSize == head; + } + + template + bool RingBuffer::IsEmpty() + { + return tail == head; + } + + template + bool RingBuffer::push(DataType ringEntry) + { + if (IsFull()){ + return false; + } + buffer[tail] = std::move(ringEntry); + tail = (tail + 1) % RingSize; + return true; + } + + template + DataType& RingBuffer::HeadEntry() { + return buffer[head]; + } + + template + bool RingBuffer::pop(DataType& ringEntry) + { + if (IsEmpty()) + return false; + ringEntry = std::move(buffer[head]); + head = (head + 1) % RingSize; + return true; + } + + template + void RingBuffer::addEvent(std::string* executor) + { + // auto inserted = m_eventMap.emplace(std::piecewise_construct, + // std::forward_as_tuple(executor->getName()), + // std::forward_as_tuple(executor)); + // // If there is duplication of executorName in m_eventMap, logic error + // if (!inserted.second) + // { + // SWSS_LOG_THROW("Duplicated executorName in m_eventMap: %s", executor->getName().c_str()); + // } + } + template + void RingBuffer::doTask() + { + // for (auto &it : m_eventMap) { + // it.second->drain(); + // } + } + template + bool RingBuffer::tasksPending() + { + // for (auto &it : m_eventMap) { + // auto consumer = dynamic_cast(it.second.get()); + // if (consumer->m_toSync.size()) + // return true; + // } + return false; + } + template + bool RingBuffer::Serves(const std::string& tableName) + { + // for (auto &it : m_eventMap) { + // if (it.first == tableName) + // return true; + // } + return true; + } +} \ No newline at end of file From 328867e72e938bdd8c076652e3edab1de4af6668 Mon Sep 17 00:00:00 2001 From: aviramd Date: Thu, 29 Aug 2024 21:05:10 +0300 Subject: [PATCH 24/35] fix find operation group by string Should be Tested: --- syncd/Syncd.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/syncd/Syncd.cpp b/syncd/Syncd.cpp index 83b738572..ce6abacb5 100644 --- a/syncd/Syncd.cpp +++ b/syncd/Syncd.cpp @@ -5471,8 +5471,8 @@ void Syncd::getApiRingBuffer( if (SAI_OBJECT_TYPE_NULL != objectType) // valid object type valid { LogToModuleFile("1", "find ring buffer by object type {}",sai_serialize_object_type(objectType).c_str()); - SWSS_LOG_DEBUG("objectType %s ", sai_serialize_object_type(objectType).c_str()); - it = operationGroups.find(sai_serialize_object_type(objectType).c_str()); + SWSS_LOG_DEBUG("objectType %s ", sai_serialize_object_type(objectType).c_str()); + it = operationGroups.find(sai_serialize_object_type(objectType)); } } else From 762ae3ba19ade3cce44b0ac2a180f68c9ef6b4ca Mon Sep 17 00:00:00 2001 From: aviramd Date: Fri, 30 Aug 2024 11:02:57 +0300 Subject: [PATCH 25/35] fix find ring buffer by op or objectType Fixed Problems/New Changes: Should be Tested: --- syncd/Syncd.cpp | 55 ++++++++++++++++++++++++++++++++++--------------- syncd/Syncd.h | 10 ++++++++- 2 files changed, 47 insertions(+), 18 deletions(-) diff --git a/syncd/Syncd.cpp b/syncd/Syncd.cpp index ce6abacb5..fe4b624c1 100644 --- a/syncd/Syncd.cpp +++ b/syncd/Syncd.cpp @@ -5451,6 +5451,33 @@ void Syncd::getObjectTypeByOperation( } } +// Internal function to find and log operation group details +bool Syncd::findOperationGroup( + const std::string& valueStr, + SyncdRing*& ringBuffer +) { + bool found = false; + + for (const auto& pair : operationGroups) { + const std::string& groupName = pair.first; + const OperationGroup& operationGroup = pair.second; + + if (operationGroup.operations.find(valueStr) != operationGroup.operations.end()) { + ringBuffer = operationGroup.ringBuffer; + LogToModuleFile("1", "Found {} in operation group: {}", valueStr, groupName); + LogToModuleFile("1", "Ring buffer pointer: {}", reinterpret_cast(ringBuffer)); + found = true; + break; + } + } + + if (!found) { + LogToModuleFile("1", "value {} not found in any operation group", valueStr); + } + + return found; +} + void Syncd::getApiRingBuffer( _In_ const swss::KeyOpFieldsValuesTuple &kco, _Out_ SyncdRing*& ringBuffer) @@ -5462,30 +5489,24 @@ void Syncd::getApiRingBuffer( SWSS_LOG_DEBUG("op: %s key:%s", op.c_str(), key.c_str()); - auto it = operationGroups.find(op); - if (it == operationGroups.end()) //not found - { + bool found = false; + std::string valueStr = op.c_str(); + + found = findOperationGroup(valueStr, ringBuffer); + + if (!found) { sai_object_type_t objectType = SAI_OBJECT_TYPE_NULL; getObjectTypeByOperation(kco, objectType); - if (SAI_OBJECT_TYPE_NULL != objectType) // valid object type valid - { - LogToModuleFile("1", "find ring buffer by object type {}",sai_serialize_object_type(objectType).c_str()); - SWSS_LOG_DEBUG("objectType %s ", sai_serialize_object_type(objectType).c_str()); - it = operationGroups.find(sai_serialize_object_type(objectType)); + if (SAI_OBJECT_TYPE_NULL != objectType) { // valid object type + valueStr = sai_serialize_object_type(objectType); + found = findOperationGroup(valueStr, ringBuffer); } } - else - LogToModuleFile("1", "find ring buffer by op {}", op.c_str()); - if (it != operationGroups.end()) - { - OperationGroup& group = it->second; - ringBuffer = group.ringBuffer; - LogToModuleFile("1", "found ring buffer {}", getNameByRingBuffer(ringBuffer)); + if (!found) { + LogToModuleFile("1", "didn't match ring buffer to api"); } - else - LogToModuleFile("1", "didnt match ring buffer to api"); } void Syncd::run() diff --git a/syncd/Syncd.h b/syncd/Syncd.h index 20e413b85..7245a40a5 100644 --- a/syncd/Syncd.h +++ b/syncd/Syncd.h @@ -831,14 +831,22 @@ namespace syncd return SAI_STATUS_SUCCESS; } + bool findOperationGroup( + const std::string& valueStr, + SyncdRing*& ringBuffer + ); + std::string getNameByRingBuffer(SyncdRing* ringBuffer) { for (const auto& pair : operationGroups) { if (pair.second.ringBuffer == ringBuffer) { - return pair.first; + return pair.first; } } + return "Not_found"; // Return a default value if not found } + + }; } From a5d48023f764082e94ce971fcaa968d94d0d38f9 Mon Sep 17 00:00:00 2001 From: aviramd Date: Sat, 31 Aug 2024 13:17:37 +0300 Subject: [PATCH 26/35] fix sendApiResponse objectCount param on processBulkQuadEventInInitViewMode Fixed Problems/New Changes: Should be Tested: --- syncd/Syncd.cpp | 18 ++++++++++-------- syncd/Syncd.h | 2 ++ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/syncd/Syncd.cpp b/syncd/Syncd.cpp index fe4b624c1..df6179b2a 100644 --- a/syncd/Syncd.cpp +++ b/syncd/Syncd.cpp @@ -1056,6 +1056,7 @@ void Syncd::sendGetResponseUpdateRedisQuadEvent( void Syncd::sendApiResponseUpdateRedisBulkQuadEvent( _In_ sai_common_api_t api, _In_ sai_status_t status, + _In_ uint32_t object_count, _In_ sai_object_type_t objectType, _In_ const std::vector& objectIds, _In_ const std::vector>& strAttributes) @@ -1064,7 +1065,7 @@ void Syncd::sendApiResponseUpdateRedisBulkQuadEvent( std::vector statuses(objectIds.size()); - sendApiResponse(api, status, static_cast(statuses.size()), statuses.data()); + sendApiResponse(api, status, object_count, statuses.data()); syncUpdateRedisBulkQuadEvent(api, statuses, objectType, objectIds, strAttributes); } @@ -1072,6 +1073,7 @@ void Syncd::sendApiResponseUpdateRedisBulkQuadEvent( void Syncd::sendApiResponseUpdateRedisBulkQuadEventWithObjectInsertion( _In_ sai_common_api_t api, _In_ sai_status_t status, + _In_ uint32_t object_count, _In_ sai_object_type_t objectType, _In_ const std::vector& objectIds, _In_ const std::vector>& strAttributes) @@ -1080,7 +1082,7 @@ void Syncd::sendApiResponseUpdateRedisBulkQuadEventWithObjectInsertion( std::vector statuses(objectIds.size()); - sendApiResponse(api, status, static_cast(statuses.size()), statuses.data()); + sendApiResponse(api, status, object_count, statuses.data()); syncUpdateRedisBulkQuadEvent(api, statuses, objectType, objectIds, strAttributes); @@ -1124,7 +1126,7 @@ sai_status_t Syncd::processBulkQuadEventInInitViewMode( { LogToModuleFile("1", "add sendApiResponseUpdateRedisBulkQuadEvent to Lambda {}", sequenceNumber); auto lambda = [=]() { - sendApiResponseUpdateRedisBulkQuadEvent(api, SAI_STATUS_SUCCESS, objectType, objectIds, strAttributes); + sendApiResponseUpdateRedisBulkQuadEvent(api, SAI_STATUS_SUCCESS,(uint32_t)statuses.size(), objectType, objectIds, strAttributes); }; sendStausAdvancedResponseSequence(sequenceNumber,lambda); @@ -1144,7 +1146,7 @@ sai_status_t Syncd::processBulkQuadEventInInitViewMode( default: LogToModuleFile("1", "add sendApiResponseUpdateRedisBulkQuadEventWithObjectInsertion to Lambda {}", sequenceNumber); auto lambda = [=]() { - sendApiResponseUpdateRedisBulkQuadEventWithObjectInsertion(api, SAI_STATUS_SUCCESS, objectType, objectIds, strAttributes); + sendApiResponseUpdateRedisBulkQuadEventWithObjectInsertion(api, SAI_STATUS_SUCCESS, (uint32_t)statuses.size(), objectType, objectIds, strAttributes); }; sendStausAdvancedResponseSequence(sequenceNumber,lambda); @@ -1975,7 +1977,7 @@ sai_status_t Syncd::processBulkEntry( { LogToModuleFile("1", "add sendApiResponseUpdateRedisBulkQuadEvent to Lambda {}", sequenceNumber); auto lambda = [=]() { - sendApiResponseUpdateRedisBulkQuadEvent(api, all, objectType, objectIds, strAttributes); + sendApiResponseUpdateRedisBulkQuadEvent(api, all, (uint32_t)objectIds.size(), objectType, objectIds, strAttributes); }; sendStausAdvancedResponseSequence(sequenceNumber,lambda); @@ -2102,7 +2104,7 @@ sai_status_t Syncd::processBulkEntry( } LogToModuleFile("1", "add sendApiResponseUpdateRedisBulkQuadEvent to Lambda {}", sequenceNumber); auto lambda = [=]() { - sendApiResponseUpdateRedisBulkQuadEvent(api, all, objectType, objectIds, strAttributes); + sendApiResponseUpdateRedisBulkQuadEvent(api, all, (uint32_t)objectIds.size(), objectType, objectIds, strAttributes); }; sendStausAdvancedResponseSequence(sequenceNumber,lambda); @@ -2341,7 +2343,7 @@ sai_status_t Syncd::processBulkOid( { LogToModuleFile("1", "1.add sendApiResponseUpdateRedisBulkQuadEvent to Lambda {}", sequenceNumber); auto lambda = [=]() { - sendApiResponseUpdateRedisBulkQuadEvent(api, all, objectType, objectIds, strAttributes); + sendApiResponseUpdateRedisBulkQuadEvent(api, all, (uint32_t)objectIds.size(), objectType, objectIds, strAttributes); }; sendStausAdvancedResponseSequence(sequenceNumber,lambda); @@ -2396,7 +2398,7 @@ sai_status_t Syncd::processBulkOid( } LogToModuleFile("1", "2. add sendApiResponseUpdateRedisBulkQuadEvent to Lambda {}", sequenceNumber); auto lambda = [=]() { - sendApiResponseUpdateRedisBulkQuadEvent(api, all, objectType, objectIds, strAttributes); + sendApiResponseUpdateRedisBulkQuadEvent(api, all,(uint32_t)objectIds.size(), objectType, objectIds, strAttributes); }; sendStausAdvancedResponseSequence(sequenceNumber,lambda); diff --git a/syncd/Syncd.h b/syncd/Syncd.h index 7245a40a5..3b97e001a 100644 --- a/syncd/Syncd.h +++ b/syncd/Syncd.h @@ -363,6 +363,7 @@ namespace syncd void sendApiResponseUpdateRedisBulkQuadEventWithObjectInsertion( _In_ sai_common_api_t api, _In_ sai_status_t status, + _In_ uint32_t object_count, _In_ sai_object_type_t objectType, _In_ const std::vector& objectIds, _In_ const std::vector>& strAttributes); @@ -370,6 +371,7 @@ namespace syncd void sendApiResponseUpdateRedisBulkQuadEvent( _In_ sai_common_api_t api, _In_ sai_status_t status, + _In_ uint32_t object_count, _In_ sai_object_type_t objectType, _In_ const std::vector& objectIds, _In_ const std::vector>& strAttributes) ; From b10bf7d85cf7b334e8742bd194f56f671cd6fb62 Mon Sep 17 00:00:00 2001 From: shiraez Date: Sun, 1 Sep 2024 12:21:56 +0300 Subject: [PATCH 27/35] removing unnecessary code --- syncd/Syncd.cpp | 75 +++++++++++-------------------------------------- 1 file changed, 16 insertions(+), 59 deletions(-) diff --git a/syncd/Syncd.cpp b/syncd/Syncd.cpp index df6179b2a..b4e99fe20 100644 --- a/syncd/Syncd.cpp +++ b/syncd/Syncd.cpp @@ -45,7 +45,6 @@ using namespace saimeta; using namespace sairediscommon; using namespace std::placeholders; -static int gdb_mode = 0; #ifdef ASAN_ENABLED #define WD_DELAY_FACTOR 2 @@ -268,14 +267,6 @@ void Syncd::popRingBuffer(SyncdRing* ringBuffer) ringBuffer->doTask(); ringBuffer->Idle = true; } - // while (!ring_thread_exited) { - // std::unique_lock lock(mtx); - // ringBuffer->cv.wait(lock, [&](){ return !ringBuffer->IsEmpty(); }); - // AnyTask func = std::move(buffer[head]); - // head = (head + 1) % RingSize; - // lock.unlock(); - // func(); - // } } Syncd::~Syncd() @@ -383,19 +374,12 @@ bool Syncd::isInitViewMode() const return m_asicInitViewMode && m_commandLineOptions->m_enableTempView; } -void point(_In_ const swss::KeyOpFieldsValuesTuple &kco){ - // SWSS_LOG_NOTICE("point"); - // SWSS_LOG_NOTICE("point %s" ,std::get<0>(kco)); - gdb_mode = 1; -} - void Syncd::processEvent( _In_ sairedis::SelectableChannel& consumer) { SWSS_LOG_ENTER(); static int entries = 0; - std::string logMessage = "multithreaded: !!!processEvent, ITERATION: " + std::to_string(entries++) + "!!!"; - LogToModuleFile("1", logMessage.c_str()); + LogToModuleFile("1", "multithreaded: !!!processEvent, ITERATION: {} !!!", std::to_string(entries++)); std::lock_guard lock(m_mutex); do @@ -413,13 +397,11 @@ void Syncd::processEvent( int sequenceNumber; if(int seq_status = m_sequencer->allocateSequenceNumber(&sequenceNumber) != Sequencer::SUCCESS) { - std::string logMsg = "Failed to allocate sequence number: " + std::to_string(seq_status); - LogToModuleFile("1", logMsg); + LogToModuleFile("1", "Failed to allocate sequence number: {}", std::to_string(seq_status)); //todo: handle wait until sequence number is available with timeout } else { - std::string logMsg = "Allocated sequence number: " + std::to_string(sequenceNumber); - LogToModuleFile("1", logMsg); + LogToModuleFile("1", "Allocated sequence number: ", std::to_string(sequenceNumber)); } if (ringBuffer) { @@ -449,8 +431,7 @@ sai_status_t Syncd::processSingleEvent( auto& op = kfvOp(kco); //SWSS_LOG_NOTICE("multithreaded: key: %s op: %s, sequence number: %d", key.c_str(), op.c_str(), sequenceNumber); - std::string logMessage = "multithreaded: key: " + key + " op: " + op + ", sequence number: " + std::to_string(sequenceNumber); - LogToModuleFile("1", logMessage.c_str()); + LogToModuleFile("1","multithreaded: key: {} op: {} {}", key, op, std::to_string(sequenceNumber)); if (key.length() == 0) { @@ -3158,8 +3139,7 @@ sai_status_t Syncd::processQuadEvent( { SWSS_LOG_ENTER(); //SWSS_LOG_NOTICE("multithreaded: %d, api=%s, key=%s, op=%s", sequenceNumber, sai_serialize_common_api(api).c_str(), kfvKey(kco).c_str(), kfvOp(kco).c_str()); - std::string logMessage = "multithreaded: " + std::to_string(sequenceNumber) + ", api=" + sai_serialize_common_api(api) + ", key=" + kfvKey(kco) + ", op=" + kfvOp(kco); - LogToModuleFile("1", logMessage.c_str()); + LogToModuleFile("1", "{} api={}, key={}, op={}", std::to_string(sequenceNumber), sai_serialize_common_api(api), kfvKey(kco), kfvOp(kco)); const std::string& key = kfvKey(kco); const std::string& op = kfvOp(kco); @@ -3174,18 +3154,6 @@ sai_status_t Syncd::processQuadEvent( SWSS_LOG_THROW("invalid object type %s", key.c_str()); } - return processQuadEventTag(api, key, op, strObjectId, metaKey, kco, sequenceNumber); -} - -sai_status_t Syncd::processQuadEventTag( - _In_ sai_common_api_t api, - _In_ const std::string &key, - _In_ const std::string &op, - _In_ const std::string &strObjectId, - _In_ const sai_object_meta_key_t metaKey, - _In_ const swss::KeyOpFieldsValuesTuple &kco, - _In_ int sequenceNumber) -{ auto& values = kfvFieldsValues(kco); for (auto& v: values) @@ -5551,8 +5519,7 @@ void Syncd::run() } catch(const std::exception &e) { - std::string logText = "shutdown - Runtime error during syncd init: " + std::string(e.what()); - LogToModuleFile("1", logText); + LogToModuleFile("1", "shutdown - Runtime error during syncd init: {}", e.what()); SWSS_LOG_ERROR("Runtime error during syncd init: %s", e.what()); @@ -5677,13 +5644,11 @@ void Syncd::run() int sequenceNumber; if(int seq_status = m_sequencer->allocateSequenceNumber(&sequenceNumber) != Sequencer::SUCCESS) { - std::string logMsg = "Failed to allocate sequence number: " + std::to_string(seq_status); - LogToModuleFile("1", logMsg); + LogToModuleFile("1", "Failed to allocate sequence number: {}", std::to_string(seq_status)); //todo: handle wait until sequence number is available with timeout } else { - std::string logMsg = "Allocated sequence number: " + std::to_string(sequenceNumber); - LogToModuleFile("1", logMsg); + LogToModuleFile("1", "Allocated sequence number: ", std::to_string(sequenceNumber)); } //directly to sequencer auto lambda = [=](){ @@ -5694,8 +5659,7 @@ void Syncd::run() if(int seq_status = m_sequencer->executeFuncInSequence(sequenceNumber, lambda) != Sequencer::SUCCESS) { - std::string logMessage = "m_flexCounter failed to execute function in sequence, status: " + std::to_string(seq_status) + ", sequence number: " + std::to_string(sequenceNumber); - LogToModuleFile("1", logMessage); + LogToModuleFile("1", "m_flexCounter failed to execute function in sequence, status: {}, sequence number: {}", std::to_string(seq_status), std::to_string(sequenceNumber)); } } else if (sel == m_flexCounterGroup.get()) @@ -5703,17 +5667,14 @@ void Syncd::run() int sequenceNumber; if(int seq_status = m_sequencer->allocateSequenceNumber(&sequenceNumber) != Sequencer::SUCCESS) { - std::string logMsg = "Failed to allocate sequence number: " + std::to_string(seq_status); - LogToModuleFile("1", logMsg); + LogToModuleFile("1", "Failed to allocate sequence number: {}", std::to_string(seq_status)); //todo: handle wait until sequence number is available with timeout } else { - std::string logMsg = "Allocated sequence number: " + std::to_string(sequenceNumber); - LogToModuleFile("1", logMsg); + LogToModuleFile("1", "Allocated sequence number: ", std::to_string(sequenceNumber)); } - std::string logMessage = "multithreaded: BEFORE PUSH INTO RING BUFFER sequence number: " + std::to_string(sequenceNumber); - LogToModuleFile("1", logMessage.c_str()); + LogToModuleFile("1", "multithreaded: BEFORE PUSH INTO RING BUFFER sequence number: ",std::to_string(sequenceNumber)); //directly to sequencer auto lambda = [=](){ @@ -5724,8 +5685,7 @@ void Syncd::run() if(int seq_status = m_sequencer->executeFuncInSequence(sequenceNumber, lambda) != Sequencer::SUCCESS) { - logMessage = "m_flexCounterGroup failed to execute function in sequence, status: " + std::to_string(seq_status) + ", sequence number: " + std::to_string(sequenceNumber); - LogToModuleFile("1", logMessage); + LogToModuleFile("1", "m_flexCounterGroup failed to execute function in sequence, status: {}, sequence number: {}", std::to_string(seq_status), std::to_string(sequenceNumber)); } } else if (sel == m_selectableChannel.get()) @@ -5740,8 +5700,7 @@ void Syncd::run() catch(const std::exception &e) { SWSS_LOG_ERROR("Runtime error: %s", e.what()); - std::string LogToModuleFile = "Runtime error: " + std::string(e.what()); - LogToModuleFile("1", LogToModuleFile); + LogToModuleFile("1", "Runtime error: {}", e.what()); sendShutdownRequestAfterException(); @@ -5875,8 +5834,7 @@ void Syncd::sendStausResponseSequence( }; if(int seq_status = m_sequencer->executeFuncInSequence(sequenceNumber, lambda) != Sequencer::SUCCESS) { - std::string logMessage = "m_flexCounterGroup failed to execute function in sequence, status: " + std::to_string(seq_status) + ", sequence number: " + std::to_string(sequenceNumber); - LogToModuleFile("1", logMessage); + LogToModuleFile("1", "m_flexCounterGroup failed to execute function in sequence, status: {}, sequence number: {}", std::to_string(seq_status), std::to_string(sequenceNumber)); } } else { @@ -5898,8 +5856,7 @@ void Syncd::sendStausAdvancedResponseSequence( SWSS_LOG_NOTICE("multithreaded: valid sequence number {}", sequenceNumber); if(int seq_status = m_sequencer->executeFuncInSequence(sequenceNumber, lambdaFunc) != Sequencer::SUCCESS) { - std::string logMessage = "m_flexCounterGroup failed to execute function in sequence, status: " + std::to_string(seq_status) + ", sequence number: " + std::to_string(sequenceNumber); - LogToModuleFile("1", logMessage); + LogToModuleFile("1", "m_flexCounterGroup failed to execute function in sequence, status: {}, sequence number:", std::to_string(seq_status), std::to_string(sequenceNumber)); } } else { From 0225abc1995f0eb50b75fba9c014e7aff4211514 Mon Sep 17 00:00:00 2001 From: aviramd Date: Sun, 1 Sep 2024 17:55:01 +0300 Subject: [PATCH 28/35] Fix send responde sync for nom get actions Fixed Problems/New Changes: Should be Tested: --- syncd/Logger.h | 7 +- syncd/Sequencer.cpp | 19 ++++- syncd/Syncd.cpp | 59 ++++++++++++---- syncd/Syncd.h | 166 +++++++++++++++++++++++++++++++++++++++----- 4 files changed, 212 insertions(+), 39 deletions(-) diff --git a/syncd/Logger.h b/syncd/Logger.h index 1522fe5e8..b8619355c 100755 --- a/syncd/Logger.h +++ b/syncd/Logger.h @@ -15,7 +15,7 @@ #include "fmt/format.h" #include -#define MAX_LOG_SIZE (50 * 1024) /* 50 KB */ +#define MAX_LOG_SIZE (10 *50 * 1024) /* 50 KB */ #define ENABLE_LOGGING 1 // Define a mutex for thread safety @@ -60,7 +60,7 @@ static void writeToLogFile(const std::string& funcName, const std::string& fileN std::string formatted_time = oss.str(); // Write the timestamp, function name, and message to the log file - logFile << formatted_time << " " << funcName << ": " << message << std::endl; + logFile << formatted_time << " " << "V1" << funcName << ": " << message << std::endl; logFile.close(); @@ -69,8 +69,7 @@ static void writeToLogFile(const std::string& funcName, const std::string& fileN template static void logFormattedMessage(const std::string& funcName, const std::string& fileNum, const std::string& format, Args... messageArgs) { std::ostringstream oss; - oss << funcName << ": "; - + std::string remainingFormat = format; // Helper function to process a single argument diff --git a/syncd/Sequencer.cpp b/syncd/Sequencer.cpp index 43a1e278a..897988158 100755 --- a/syncd/Sequencer.cpp +++ b/syncd/Sequencer.cpp @@ -11,12 +11,14 @@ Sequencer::SequenceStatus Sequencer::executeReadyResponses() { SequenceStatus status = FAILURE; logMsg = "multithreaded: Checking for ready responses in queue... \n"; + LogToModuleFile("1", "Checking for ready responses in queue..."); while (true) { // Check if the next sequence number is in the map auto seq_data = responses.find(next_seq_to_send); if (seq_data == responses.end()) { logMsg += "multithreaded: No next sequence found in queue \n"; + LogToModuleFile("1", "No next sequence found in queue"); status = SUCCESS; break; // Exit loop if the next sequence is not in the map } @@ -24,12 +26,15 @@ Sequencer::SequenceStatus Sequencer::executeReadyResponses() { // Execute the stored lambda auto func = seq_data->second; if(func) { - func(); + LogToModuleFile("1", "before execute lambda with sequenceNumber: {}", next_seq_to_send); + func(); + LogToModuleFile("1", "after execute lambda with sequenceNumber: {}", next_seq_to_send); logMsg += "multithreaded: Executing lambda with seq: " + std::to_string(next_seq_to_send) + " \n"; status = NULL_PTR; } else { logMsg += "multithreaded: response lambda is null \n"; + LogToModuleFile("1", "multithreaded: response lambda is null {}", next_seq_to_send); num_of_null_functions++; status = SUCCESS; //????? } @@ -51,7 +56,7 @@ Sequencer::SequenceStatus Sequencer::executeReadyResponses() { } } - LogToModuleFile("1", logMsg); + //LogToModuleFile("1", logMsg); return status; } @@ -101,13 +106,19 @@ Sequencer::SequenceStatus Sequencer::executeFuncInSequence(int seq, std::functio std::string logMsg; SequenceStatus status = FAILURE; + LogToModuleFile("1", "Enter executeFuncInSequence with seq: {}", seq); + if (seq == next_seq_to_send) { // If the received sequence is the next one to send, execute it immediately logMsg = "multithreaded: executing reseponse lambda, seq num: " + std::to_string(seq) + " \n"; // execute response lambda if(response_lambda) { + + LogToModuleFile("1", "start execute response_lambda with sequenceNumber: {}", next_seq_to_send); + response_lambda(); + LogToModuleFile("1", "end execute response_lambda with sequenceNumber: {}", next_seq_to_send); logMsg += "multithreaded: execute response lambda \n"; status = SUCCESS; } @@ -130,7 +141,9 @@ Sequencer::SequenceStatus Sequencer::executeFuncInSequence(int seq, std::functio } // Continue sending any subsequent responses that are ready + LogToModuleFile("1", "start execute executeReadyResponses"); status = executeReadyResponses(); + LogToModuleFile("1", "end execute executeReadyResponses"); } else { // If the sequence is not the next to send, store it in the map responses[seq] = response_lambda; @@ -139,7 +152,7 @@ Sequencer::SequenceStatus Sequencer::executeFuncInSequence(int seq, std::functio num_of_out_of_sequence_functions++; } - LogToModuleFile("1", logMsg); + //LogToModuleFile("1", logMsg); return status; } diff --git a/syncd/Syncd.cpp b/syncd/Syncd.cpp index b4e99fe20..a113283e0 100644 --- a/syncd/Syncd.cpp +++ b/syncd/Syncd.cpp @@ -382,6 +382,8 @@ void Syncd::processEvent( LogToModuleFile("1", "multithreaded: !!!processEvent, ITERATION: {} !!!", std::to_string(entries++)); std::lock_guard lock(m_mutex); + LogToModuleFile("1", "after lock"); + do { swss::KeyOpFieldsValuesTuple kco; @@ -390,10 +392,14 @@ void Syncd::processEvent( * to specify temporary view prefix in consumer since consumer puts * data to redis db. */ + LogToModuleFile("1", "before consumer.pop"); consumer.pop(kco, isInitViewMode()); + LogToModuleFile("1", "after consumer.pop"); SyncdRing* ringBuffer; getApiRingBuffer(kco, ringBuffer); + LogToModuleFile("1","getApiRingBuffer end"); + int sequenceNumber; if(int seq_status = m_sequencer->allocateSequenceNumber(&sequenceNumber) != Sequencer::SUCCESS) { @@ -404,6 +410,12 @@ void Syncd::processEvent( LogToModuleFile("1", "Allocated sequence number: ", std::to_string(sequenceNumber)); } + auto& key = kfvKey(kco); + auto& op = kfvOp(kco); + LogToModuleFile("1", "sequenceNumber: {} op: {} key: {}", sequenceNumber,op.c_str(), key.c_str()); + LogToModuleFile("2", "sequenceNumber: {} op: {} key: {}", sequenceNumber,op.c_str(), key.c_str()); + + if (ringBuffer) { LogToModuleFile("1", "push processSingleEvent with sequenceNumber {} to ring buffer {} ",sequenceNumber, getNameByRingBuffer(ringBuffer)); auto lambda = [=](){ @@ -1010,6 +1022,17 @@ sai_status_t Syncd::processBulkQuadEvent( } } +void Syncd::sendApiResponseUpdateRedisQuadEventCreate( + _In_ sai_common_api_t api, + _In_ sai_status_t status, + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int sequenceNumber) +{ + syncUpdateRedisQuadEvent(status, api, kco); + sendApiResponse(api, status); + +} + void Syncd::sendApiResponseUpdateRedisQuadEvent( _In_ sai_common_api_t api, _In_ sai_status_t status, @@ -1039,12 +1062,13 @@ void Syncd::sendApiResponseUpdateRedisBulkQuadEvent( _In_ sai_status_t status, _In_ uint32_t object_count, _In_ sai_object_type_t objectType, + _In_ std::vector statuses, _In_ const std::vector& objectIds, _In_ const std::vector>& strAttributes) { SWSS_LOG_ENTER(); - std::vector statuses(objectIds.size()); + //std::vector statuses(objectIds.size()); sendApiResponse(api, status, object_count, statuses.data()); @@ -1056,12 +1080,13 @@ void Syncd::sendApiResponseUpdateRedisBulkQuadEventWithObjectInsertion( _In_ sai_status_t status, _In_ uint32_t object_count, _In_ sai_object_type_t objectType, + _In_ std::vector statuses, _In_ const std::vector& objectIds, _In_ const std::vector>& strAttributes) { SWSS_LOG_ENTER(); - std::vector statuses(objectIds.size()); + //std::vector statuses(objectIds.size()); sendApiResponse(api, status, object_count, statuses.data()); @@ -1107,7 +1132,7 @@ sai_status_t Syncd::processBulkQuadEventInInitViewMode( { LogToModuleFile("1", "add sendApiResponseUpdateRedisBulkQuadEvent to Lambda {}", sequenceNumber); auto lambda = [=]() { - sendApiResponseUpdateRedisBulkQuadEvent(api, SAI_STATUS_SUCCESS,(uint32_t)statuses.size(), objectType, objectIds, strAttributes); + sendApiResponseUpdateRedisBulkQuadEvent(api, SAI_STATUS_SUCCESS,(uint32_t)statuses.size(), objectType, statuses, objectIds, strAttributes); }; sendStausAdvancedResponseSequence(sequenceNumber,lambda); @@ -1127,7 +1152,7 @@ sai_status_t Syncd::processBulkQuadEventInInitViewMode( default: LogToModuleFile("1", "add sendApiResponseUpdateRedisBulkQuadEventWithObjectInsertion to Lambda {}", sequenceNumber); auto lambda = [=]() { - sendApiResponseUpdateRedisBulkQuadEventWithObjectInsertion(api, SAI_STATUS_SUCCESS, (uint32_t)statuses.size(), objectType, objectIds, strAttributes); + sendApiResponseUpdateRedisBulkQuadEventWithObjectInsertion(api, SAI_STATUS_SUCCESS, (uint32_t)statuses.size(), objectType, statuses, objectIds, strAttributes); }; sendStausAdvancedResponseSequence(sequenceNumber,lambda); @@ -1958,7 +1983,7 @@ sai_status_t Syncd::processBulkEntry( { LogToModuleFile("1", "add sendApiResponseUpdateRedisBulkQuadEvent to Lambda {}", sequenceNumber); auto lambda = [=]() { - sendApiResponseUpdateRedisBulkQuadEvent(api, all, (uint32_t)objectIds.size(), objectType, objectIds, strAttributes); + sendApiResponseUpdateRedisBulkQuadEvent(api, all, (uint32_t)objectIds.size(), objectType, statuses, objectIds, strAttributes); }; sendStausAdvancedResponseSequence(sequenceNumber,lambda); @@ -2085,7 +2110,7 @@ sai_status_t Syncd::processBulkEntry( } LogToModuleFile("1", "add sendApiResponseUpdateRedisBulkQuadEvent to Lambda {}", sequenceNumber); auto lambda = [=]() { - sendApiResponseUpdateRedisBulkQuadEvent(api, all, (uint32_t)objectIds.size(), objectType, objectIds, strAttributes); + sendApiResponseUpdateRedisBulkQuadEvent(api, all, (uint32_t)objectIds.size(), objectType, statuses, objectIds, strAttributes); }; sendStausAdvancedResponseSequence(sequenceNumber,lambda); @@ -2324,7 +2349,7 @@ sai_status_t Syncd::processBulkOid( { LogToModuleFile("1", "1.add sendApiResponseUpdateRedisBulkQuadEvent to Lambda {}", sequenceNumber); auto lambda = [=]() { - sendApiResponseUpdateRedisBulkQuadEvent(api, all, (uint32_t)objectIds.size(), objectType, objectIds, strAttributes); + sendApiResponseUpdateRedisBulkQuadEvent(api, all, (uint32_t)objectIds.size(), objectType, statuses, objectIds, strAttributes); }; sendStausAdvancedResponseSequence(sequenceNumber,lambda); @@ -2379,7 +2404,7 @@ sai_status_t Syncd::processBulkOid( } LogToModuleFile("1", "2. add sendApiResponseUpdateRedisBulkQuadEvent to Lambda {}", sequenceNumber); auto lambda = [=]() { - sendApiResponseUpdateRedisBulkQuadEvent(api, all,(uint32_t)objectIds.size(), objectType, objectIds, strAttributes); + sendApiResponseUpdateRedisBulkQuadEvent(api, all,(uint32_t)objectIds.size(), objectType, statuses, objectIds, strAttributes); }; sendStausAdvancedResponseSequence(sequenceNumber,lambda); @@ -2476,9 +2501,9 @@ sai_status_t Syncd::processQuadInInitViewModeCreate( m_createdInInitView.insert(objectVid); } } - LogToModuleFile("1", "add sendApiResponseUpdateRedisQuadEvent to Lambda {}", sequenceNumber); + LogToModuleFile("1", "add sendApiResponseUpdateRedisQuadEventCreate to Lambda {}", sequenceNumber); auto lambda = [=]() { - sendApiResponseUpdateRedisQuadEvent(api, SAI_STATUS_SUCCESS, kco, sequenceNumber); + sendApiResponseUpdateRedisQuadEventCreate(api, SAI_STATUS_SUCCESS, kco, sequenceNumber); }; sendStausAdvancedResponseSequence(sequenceNumber,lambda); @@ -3002,7 +3027,7 @@ void Syncd::syncUpdateRedisQuadEvent( void Syncd::syncUpdateRedisBulkQuadEvent( _In_ sai_common_api_t api, - _In_ const std::vector& statuses, + _In_ std::vector statuses, _In_ sai_object_type_t objectType, _In_ const std::vector& objectIds, _In_ const std::vector>& strAttributes) @@ -3310,9 +3335,9 @@ sai_status_t Syncd::processQuadEvent( } else // non GET api, status is SUCCESS { - LogToModuleFile("1", "2. add sendApiResponseUpdateRedisQuadEvent to Lambda sequenceNumber {}", sequenceNumber); + LogToModuleFile("1", "2. add sendApiResponseUpdateRedisQuadEventCreate to Lambda sequenceNumber {}", sequenceNumber); auto lambda = [=]() { - sendApiResponseUpdateRedisQuadEvent(api, status, kco, sequenceNumber); + sendApiResponseUpdateRedisQuadEventCreate(api, status, kco, sequenceNumber); }; sendStausAdvancedResponseSequence(sequenceNumber,lambda); } @@ -5474,6 +5499,13 @@ void Syncd::getApiRingBuffer( } } + sai_object_type_t objectType1 = SAI_OBJECT_TYPE_NULL; + getObjectTypeByOperation(kco, objectType1); + if(objectType1 == SAI_OBJECT_TYPE_WRED) { // valid object type + ringBuffer = nullptr; + found = true; + } + if (!found) { LogToModuleFile("1", "didn't match ring buffer to api"); } @@ -5808,6 +5840,7 @@ void Syncd::sendStatusAndEntryResponse( std::string strStatus = sai_serialize_status(status); LogToModuleFile("1", "sending response: {} with commandType: {}", strStatus.c_str(), commandType.c_str()); + LogToModuleFile("2", "sending response: {} with commandType: {}", strStatus.c_str(), commandType.c_str()); SWSS_LOG_INFO("sending response: %s with commandType: %s", strStatus.c_str(), commandType.c_str()); m_selectableChannel->set(strStatus, entry, commandType); diff --git a/syncd/Syncd.h b/syncd/Syncd.h index 3b97e001a..2a35d597c 100644 --- a/syncd/Syncd.h +++ b/syncd/Syncd.h @@ -345,7 +345,7 @@ namespace syncd void syncUpdateRedisBulkQuadEvent( _In_ sai_common_api_t api, - _In_ const std::vector& statuses, + _In_ std::vector statuses, _In_ sai_object_type_t objectType, _In_ const std::vector& objectIds, _In_ const std::vector>& strAttributes); @@ -365,6 +365,7 @@ namespace syncd _In_ sai_status_t status, _In_ uint32_t object_count, _In_ sai_object_type_t objectType, + _In_ std::vector statuses, _In_ const std::vector& objectIds, _In_ const std::vector>& strAttributes); @@ -373,14 +374,22 @@ namespace syncd _In_ sai_status_t status, _In_ uint32_t object_count, _In_ sai_object_type_t objectType, + _In_ std::vector statuses, _In_ const std::vector& objectIds, - _In_ const std::vector>& strAttributes) ; + _In_ const std::vector>& strAttributes) ; + void sendApiResponseUpdateRedisQuadEvent( _In_ sai_common_api_t api, _In_ sai_status_t status, _In_ const swss::KeyOpFieldsValuesTuple &kco, _In_ int sequenceNumber); + void sendApiResponseUpdateRedisQuadEventCreate( + _In_ sai_common_api_t api, + _In_ sai_status_t status, + _In_ const swss::KeyOpFieldsValuesTuple &kco, + _In_ int sequenceNumber); + void sendGetResponseUpdateRedisQuadEvent( _In_ sai_object_type_t objectType, _In_ const std::string& strObjectId, @@ -649,6 +658,23 @@ namespace syncd _In_ swss::KeyOpFieldsValuesTuple kco, _Out_ sai_object_type_t &objectType); + + bool findOperationGroup( + const std::string& valueStr, + SyncdRing*& ringBuffer + ); + + std::string getNameByRingBuffer(SyncdRing* ringBuffer) + { + for (const auto& pair : operationGroups) { + if (pair.second.ringBuffer == ringBuffer) { + return pair.first; + } + } + + return "Not_found"; // Return a default value if not found + } + // Declare saiObjectTypes as a member variable std::vector saiObjectTypes = { {SAI_OBJECT_TYPE_NULL, "SAI_OBJECT_TYPE_NULL"}, @@ -719,7 +745,7 @@ namespace syncd // make sure all object types are accounted for //if(saiObjectTypes.size() != (SAI_OBJECT_TYPE_MAX+1)) // return SAI_STATUS_FAILURE; - +#if 0 std::set miscOperations = { //REDIS_ASIC_STATE_COMMAND_NOTIFY, // Not included in the list (flow without ringbuff) REDIS_ASIC_STATE_COMMAND_GET_STATS, @@ -827,28 +853,130 @@ namespace syncd {"crud1", {crudOperations1, crudRingBuffer1}}, {"crud2", {crudOperations2, crudRingBuffer2}}, {"misc", {miscOperations, miscRingBuffer}}, - // Add other groups here... + } + #endif + #if 0 + std::array crudOperations1Enums = { + SAI_OBJECT_TYPE_NULL, + SAI_OBJECT_TYPE_PORT, + SAI_OBJECT_TYPE_LAG, + SAI_OBJECT_TYPE_ACL_TABLE, + SAI_OBJECT_TYPE_ACL_ENTRY, + SAI_OBJECT_TYPE_ACL_COUNTER, + SAI_OBJECT_TYPE_ACL_RANGE, + SAI_OBJECT_TYPE_ACL_TABLE_GROUP, + SAI_OBJECT_TYPE_ACL_TABLE_GROUP_MEMBER, + SAI_OBJECT_TYPE_HOSTIF, + SAI_OBJECT_TYPE_MIRROR_SESSION, + SAI_OBJECT_TYPE_SAMPLEPACKET, + SAI_OBJECT_TYPE_STP, + SAI_OBJECT_TYPE_HOSTIF_TRAP_GROUP, + SAI_OBJECT_TYPE_POLICER, + SAI_OBJECT_TYPE_WRED, + SAI_OBJECT_TYPE_QOS_MAP, + SAI_OBJECT_TYPE_QUEUE, + SAI_OBJECT_TYPE_SCHEDULER, + SAI_OBJECT_TYPE_SCHEDULER_GROUP, + SAI_OBJECT_TYPE_BUFFER_POOL, + SAI_OBJECT_TYPE_BUFFER_PROFILE, + SAI_OBJECT_TYPE_INGRESS_PRIORITY_GROUP, + SAI_OBJECT_TYPE_LAG_MEMBER, + SAI_OBJECT_TYPE_HASH, + SAI_OBJECT_TYPE_UDF, + SAI_OBJECT_TYPE_UDF_MATCH, + SAI_OBJECT_TYPE_UDF_GROUP, + SAI_OBJECT_TYPE_FDB_ENTRY, + //SAI_OBJECT_TYPE_SWITCH, // Not included in the list (floow without ringbuff) + SAI_OBJECT_TYPE_HOSTIF_TRAP, + SAI_OBJECT_TYPE_HOSTIF_TABLE_ENTRY, + SAI_OBJECT_TYPE_VLAN, + SAI_OBJECT_TYPE_VLAN_MEMBER, + SAI_OBJECT_TYPE_HOSTIF_PACKET, + SAI_OBJECT_TYPE_FDB_FLUSH, + SAI_OBJECT_TYPE_STP_PORT, + SAI_OBJECT_TYPE_L2MC_GROUP, + SAI_OBJECT_TYPE_L2MC_GROUP_MEMBER, + SAI_OBJECT_TYPE_L2MC_ENTRY, + SAI_OBJECT_TYPE_MCAST_FDB_ENTRY, + SAI_OBJECT_TYPE_HOSTIF_USER_DEFINED_TRAP, + SAI_OBJECT_TYPE_BRIDGE, + SAI_OBJECT_TYPE_BRIDGE_PORT, + SAI_OBJECT_TYPE_VIRTUAL_ROUTER, + SAI_OBJECT_TYPE_NEXT_HOP, + SAI_OBJECT_TYPE_NEXT_HOP_GROUP, + SAI_OBJECT_TYPE_ROUTER_INTERFACE, + SAI_OBJECT_TYPE_NEIGHBOR_ENTRY, + SAI_OBJECT_TYPE_ROUTE_ENTRY, + SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER, + SAI_OBJECT_TYPE_RPF_GROUP, + SAI_OBJECT_TYPE_RPF_GROUP_MEMBER, + SAI_OBJECT_TYPE_IPMC_GROUP, + SAI_OBJECT_TYPE_IPMC_GROUP_MEMBER, + SAI_OBJECT_TYPE_IPMC_ENTRY, + SAI_OBJECT_TYPE_TUNNEL_MAP, + SAI_OBJECT_TYPE_TUNNEL, + SAI_OBJECT_TYPE_TUNNEL_TERM_TABLE_ENTRY, + SAI_OBJECT_TYPE_TUNNEL_MAP_ENTRY }; - return SAI_STATUS_SUCCESS; - } - - bool findOperationGroup( - const std::string& valueStr, - SyncdRing*& ringBuffer - ); - - std::string getNameByRingBuffer(SyncdRing* ringBuffer) - { - for (const auto& pair : operationGroups) { - if (pair.second.ringBuffer == ringBuffer) { - return pair.first; + // Populate crudOperations2 using the enums array + std::set crudOperations1; + for (const auto& item : saiObjectTypes) { + if (std::find(crudOperations1Enums.begin(), crudOperations1Enums.end(), item.enumValue) != crudOperations1Enums.end()) { + crudOperations1.insert(item.enumString); } } - return "Not_found"; // Return a default value if not found - } + // Allocate new SyncdRing instances for each operation group + auto crudRingBuffer1 = new SyncdRing(); // Adjust size if needed + + // Map the operation groups + operationGroups = { + {"crud1", {crudOperations1, crudRingBuffer1}}, + // Add other groups here... + }; +#endif +#if 0 + // Map the operation groups + operationGroups = { + // Add other groups here... + }; +#endif + +#if 1 + + std::set crudOperations1 = { + REDIS_ASIC_STATE_COMMAND_NOTIFY, + REDIS_ASIC_STATE_COMMAND_GET_STATS, + REDIS_ASIC_STATE_COMMAND_CLEAR_STATS, + REDIS_ASIC_STATE_COMMAND_FLUSH, + REDIS_ASIC_STATE_COMMAND_ATTR_CAPABILITY_QUERY, + REDIS_ASIC_STATE_COMMAND_ATTR_ENUM_VALUES_CAPABILITY_QUERY, + REDIS_ASIC_STATE_COMMAND_OBJECT_TYPE_GET_AVAILABILITY_QUERY, + REDIS_FLEX_COUNTER_COMMAND_START_POLL, + REDIS_FLEX_COUNTER_COMMAND_STOP_POLL, + REDIS_FLEX_COUNTER_COMMAND_SET_GROUP, + REDIS_FLEX_COUNTER_COMMAND_DEL_GROUP, + REDIS_ASIC_STATE_COMMAND_CREATE, + REDIS_ASIC_STATE_COMMAND_REMOVE, + REDIS_ASIC_STATE_COMMAND_SET, + REDIS_ASIC_STATE_COMMAND_GET, + REDIS_ASIC_STATE_COMMAND_BULK_CREATE, + REDIS_ASIC_STATE_COMMAND_BULK_REMOVE, + REDIS_ASIC_STATE_COMMAND_BULK_SET + }; + + // Allocate new SyncdRing instances for each operation group + auto crudRingBuffer1 = new SyncdRing(); // Adjust size if needed + // Map the operation groups + operationGroups = { + {"crud1", {crudOperations1, crudRingBuffer1}}, + // Add other groups here... + }; + #endif + return SAI_STATUS_SUCCESS; + } }; } From 6cf0e13463454526017bdda2a2fe2227c25ad7be Mon Sep 17 00:00:00 2001 From: aviramd Date: Sun, 1 Sep 2024 19:00:03 +0300 Subject: [PATCH 29/35] create muliple threads, cleanup, change response seq order Fixed Problems/New Changes: Should be Tested: --- syncd/Sequencer.cpp | 38 +++++++++++--------------------------- syncd/Syncd.cpp | 45 +++++++++++---------------------------------- syncd/Syncd.h | 11 +++-------- 3 files changed, 25 insertions(+), 69 deletions(-) diff --git a/syncd/Sequencer.cpp b/syncd/Sequencer.cpp index 897988158..f6fb729c1 100755 --- a/syncd/Sequencer.cpp +++ b/syncd/Sequencer.cpp @@ -7,17 +7,14 @@ using namespace syncd; // check how many consecutive responses were executed in sucession and log it Sequencer::SequenceStatus Sequencer::executeReadyResponses() { - std::string logMsg; SequenceStatus status = FAILURE; - logMsg = "multithreaded: Checking for ready responses in queue... \n"; LogToModuleFile("1", "Checking for ready responses in queue..."); while (true) { // Check if the next sequence number is in the map auto seq_data = responses.find(next_seq_to_send); if (seq_data == responses.end()) { - logMsg += "multithreaded: No next sequence found in queue \n"; LogToModuleFile("1", "No next sequence found in queue"); status = SUCCESS; break; // Exit loop if the next sequence is not in the map @@ -29,12 +26,10 @@ Sequencer::SequenceStatus Sequencer::executeReadyResponses() { LogToModuleFile("1", "before execute lambda with sequenceNumber: {}", next_seq_to_send); func(); LogToModuleFile("1", "after execute lambda with sequenceNumber: {}", next_seq_to_send); - logMsg += "multithreaded: Executing lambda with seq: " + std::to_string(next_seq_to_send) + " \n"; status = NULL_PTR; } else { - logMsg += "multithreaded: response lambda is null \n"; - LogToModuleFile("1", "multithreaded: response lambda is null {}", next_seq_to_send); + LogToModuleFile("1", "multithreaded: response lambda is null {}", next_seq_to_send); num_of_null_functions++; status = SUCCESS; //????? } @@ -48,15 +43,14 @@ Sequencer::SequenceStatus Sequencer::executeReadyResponses() { // Increment the sequence number ++next_seq_to_send; - logMsg += "multithreaded: Next sequence found! Executed lambda with seq: " + std::to_string(next_seq_to_send) + " \n"; + LogToModuleFile("1", "Next sequence found! Executed lambda with seq: {}", next_seq_to_send); if (next_seq_to_send >= MAX_SEQUENCE_NUMBER) { - logMsg += "multithreaded: Resetting next sequence number to send needs to be reset to avoid overflow \n"; + LogToModuleFile("1", "Resetting next sequence number to send needs to be reset to avoid overflow"); next_seq_to_send = 0; } } - //LogToModuleFile("1", logMsg); return status; } @@ -69,12 +63,10 @@ Sequencer::SequenceStatus Sequencer::executeReadyResponses() { // mux on everything Sequencer::SequenceStatus Sequencer::allocateSequenceNumber(int *seq_num) { std::unique_lock lock(mtx); - std::string logMsg; SequenceStatus status = FAILURE; if(isFull()) { - logMsg = "multithreaded: Sequencer is full, cannot allocate sequence number" + std::to_string(current_seq); - LogToModuleFile("1", logMsg); + LogToModuleFile("1", "Sequencer is full, cannot allocate sequence number {}", current_seq); *seq_num = INVALID_SEQUENCE_NUMBER; return BUFFER_OVERFLOW; } @@ -83,18 +75,16 @@ Sequencer::SequenceStatus Sequencer::allocateSequenceNumber(int *seq_num) { *seq_num = current_seq; // increment the sequence number current_seq++; - - logMsg = "multithreaded: allocate seq num: " + std::to_string(*seq_num) + ", "; + + LogToModuleFile("1", "allocate seq num {}", *seq_num); // reset number to avoid overflow if (current_seq >= MAX_SEQUENCE_NUMBER) { - logMsg += "multithreaded: Resetting allocated sequence number to avoid overflow, "; + LogToModuleFile("1", "Resetting allocated sequence number to avoid overflow"); current_seq = 0; } status = SUCCESS; - - LogToModuleFile("1", logMsg); return status; } @@ -103,27 +93,22 @@ Sequencer::SequenceStatus Sequencer::allocateSequenceNumber(int *seq_num) { Sequencer::SequenceStatus Sequencer::executeFuncInSequence(int seq, std::function response_lambda) { std::unique_lock lock(mtx); - std::string logMsg; SequenceStatus status = FAILURE; LogToModuleFile("1", "Enter executeFuncInSequence with seq: {}", seq); if (seq == next_seq_to_send) { // If the received sequence is the next one to send, execute it immediately - logMsg = "multithreaded: executing reseponse lambda, seq num: " + std::to_string(seq) + " \n"; // execute response lambda if(response_lambda) { - LogToModuleFile("1", "start execute response_lambda with sequenceNumber: {}", next_seq_to_send); - response_lambda(); LogToModuleFile("1", "end execute response_lambda with sequenceNumber: {}", next_seq_to_send); - logMsg += "multithreaded: execute response lambda \n"; status = SUCCESS; } else { - logMsg += "multithreaded: response lambda is null \n"; + LogToModuleFile("1", "response lambda is null "); num_of_null_functions++; status = SUCCESS; //NULL_PTR; ??? } @@ -136,7 +121,7 @@ Sequencer::SequenceStatus Sequencer::executeFuncInSequence(int seq, std::functio // reset number to avoid overflow if (next_seq_to_send >= MAX_SEQUENCE_NUMBER) { - logMsg += "multithreaded: Resetting next sequence number to send needs to be reset to avoid overflow \n"; + LogToModuleFile("1", "Resetting next sequence number to send needs to be reset to avoid overflow"); next_seq_to_send = 0; } @@ -146,13 +131,12 @@ Sequencer::SequenceStatus Sequencer::executeFuncInSequence(int seq, std::functio LogToModuleFile("1", "end execute executeReadyResponses"); } else { // If the sequence is not the next to send, store it in the map - responses[seq] = response_lambda; - logMsg = "multithreaded: storing lambda with seq: " + std::to_string(seq) + ", next to send: " + std::to_string(next_seq_to_send) + " \n"; + responses[seq] = response_lambda; + LogToModuleFile("1", "storing lambda with seq: {} next to send: {}", seq, next_seq_to_send); status = SUCCESS; num_of_out_of_sequence_functions++; } - //LogToModuleFile("1", logMsg); return status; } diff --git a/syncd/Syncd.cpp b/syncd/Syncd.cpp index a113283e0..23149308a 100644 --- a/syncd/Syncd.cpp +++ b/syncd/Syncd.cpp @@ -413,8 +413,6 @@ void Syncd::processEvent( auto& key = kfvKey(kco); auto& op = kfvOp(kco); LogToModuleFile("1", "sequenceNumber: {} op: {} key: {}", sequenceNumber,op.c_str(), key.c_str()); - LogToModuleFile("2", "sequenceNumber: {} op: {} key: {}", sequenceNumber,op.c_str(), key.c_str()); - if (ringBuffer) { LogToModuleFile("1", "push processSingleEvent with sequenceNumber {} to ring buffer {} ",sequenceNumber, getNameByRingBuffer(ringBuffer)); @@ -748,7 +746,6 @@ void Syncd::processFdbFlushResponse( _In_ std::vector values, _In_ sai_object_id_t switchVid) { - sendStatusAndEntryResponse(status, REDIS_ASIC_STATE_COMMAND_FLUSHRESPONSE, {}); if (status == SAI_STATUS_SUCCESS) { @@ -797,6 +794,8 @@ void Syncd::processFdbFlushResponse( m_client->processFlushEvent(switchVid, bridgePortId, bvId, type); } + + sendStatusAndEntryResponse(status, REDIS_ASIC_STATE_COMMAND_FLUSHRESPONSE, {}); } sai_status_t Syncd::processClearStatsEvent( @@ -1022,7 +1021,7 @@ sai_status_t Syncd::processBulkQuadEvent( } } -void Syncd::sendApiResponseUpdateRedisQuadEventCreate( +void Syncd::sendApiResponseUpdateRedisQuadEvent( _In_ sai_common_api_t api, _In_ sai_status_t status, _In_ const swss::KeyOpFieldsValuesTuple &kco, @@ -1030,17 +1029,7 @@ void Syncd::sendApiResponseUpdateRedisQuadEventCreate( { syncUpdateRedisQuadEvent(status, api, kco); sendApiResponse(api, status); - -} -void Syncd::sendApiResponseUpdateRedisQuadEvent( - _In_ sai_common_api_t api, - _In_ sai_status_t status, - _In_ const swss::KeyOpFieldsValuesTuple &kco, - _In_ int sequenceNumber) -{ - sendApiResponse(api, status); - syncUpdateRedisQuadEvent(status, api, kco); } void Syncd::sendGetResponseUpdateRedisQuadEvent( @@ -1053,8 +1042,8 @@ void Syncd::sendGetResponseUpdateRedisQuadEvent( _In_ const swss::KeyOpFieldsValuesTuple &kco, _In_ sai_common_api_t api) { - sendGetResponse(objectType, strObjectId, switchVid, status, attr_count, attr_list); syncUpdateRedisQuadEvent(status, api, kco); + sendGetResponse(objectType, strObjectId, switchVid, status, attr_count, attr_list); } void Syncd::sendApiResponseUpdateRedisBulkQuadEvent( @@ -1068,11 +1057,9 @@ void Syncd::sendApiResponseUpdateRedisBulkQuadEvent( { SWSS_LOG_ENTER(); - //std::vector statuses(objectIds.size()); + syncUpdateRedisBulkQuadEvent(api, statuses, objectType, objectIds, strAttributes); sendApiResponse(api, status, object_count, statuses.data()); - - syncUpdateRedisBulkQuadEvent(api, statuses, objectType, objectIds, strAttributes); } void Syncd::sendApiResponseUpdateRedisBulkQuadEventWithObjectInsertion( @@ -1086,10 +1073,6 @@ void Syncd::sendApiResponseUpdateRedisBulkQuadEventWithObjectInsertion( { SWSS_LOG_ENTER(); - //std::vector statuses(objectIds.size()); - - sendApiResponse(api, status, object_count, statuses.data()); - syncUpdateRedisBulkQuadEvent(api, statuses, objectType, objectIds, strAttributes); for (auto& str: objectIds) @@ -1101,6 +1084,8 @@ void Syncd::sendApiResponseUpdateRedisBulkQuadEventWithObjectInsertion( m_createdInInitView.insert(objectVid); } + + sendApiResponse(api, status, object_count, statuses.data()); } sai_status_t Syncd::processBulkQuadEventInInitViewMode( @@ -2501,9 +2486,9 @@ sai_status_t Syncd::processQuadInInitViewModeCreate( m_createdInInitView.insert(objectVid); } } - LogToModuleFile("1", "add sendApiResponseUpdateRedisQuadEventCreate to Lambda {}", sequenceNumber); + LogToModuleFile("1", "add sendApiResponseUpdateRedisQuadEvent to Lambda {}", sequenceNumber); auto lambda = [=]() { - sendApiResponseUpdateRedisQuadEventCreate(api, SAI_STATUS_SUCCESS, kco, sequenceNumber); + sendApiResponseUpdateRedisQuadEvent(api, SAI_STATUS_SUCCESS, kco, sequenceNumber); }; sendStausAdvancedResponseSequence(sequenceNumber,lambda); @@ -3335,9 +3320,9 @@ sai_status_t Syncd::processQuadEvent( } else // non GET api, status is SUCCESS { - LogToModuleFile("1", "2. add sendApiResponseUpdateRedisQuadEventCreate to Lambda sequenceNumber {}", sequenceNumber); + LogToModuleFile("1", "2. add sendApiResponseUpdateRedisQuadEvent to Lambda sequenceNumber {}", sequenceNumber); auto lambda = [=]() { - sendApiResponseUpdateRedisQuadEventCreate(api, status, kco, sequenceNumber); + sendApiResponseUpdateRedisQuadEvent(api, status, kco, sequenceNumber); }; sendStausAdvancedResponseSequence(sequenceNumber,lambda); } @@ -5499,13 +5484,6 @@ void Syncd::getApiRingBuffer( } } - sai_object_type_t objectType1 = SAI_OBJECT_TYPE_NULL; - getObjectTypeByOperation(kco, objectType1); - if(objectType1 == SAI_OBJECT_TYPE_WRED) { // valid object type - ringBuffer = nullptr; - found = true; - } - if (!found) { LogToModuleFile("1", "didn't match ring buffer to api"); } @@ -5840,7 +5818,6 @@ void Syncd::sendStatusAndEntryResponse( std::string strStatus = sai_serialize_status(status); LogToModuleFile("1", "sending response: {} with commandType: {}", strStatus.c_str(), commandType.c_str()); - LogToModuleFile("2", "sending response: {} with commandType: {}", strStatus.c_str(), commandType.c_str()); SWSS_LOG_INFO("sending response: %s with commandType: %s", strStatus.c_str(), commandType.c_str()); m_selectableChannel->set(strStatus, entry, commandType); diff --git a/syncd/Syncd.h b/syncd/Syncd.h index 2a35d597c..b7f020bc2 100644 --- a/syncd/Syncd.h +++ b/syncd/Syncd.h @@ -384,11 +384,6 @@ namespace syncd _In_ const swss::KeyOpFieldsValuesTuple &kco, _In_ int sequenceNumber); - void sendApiResponseUpdateRedisQuadEventCreate( - _In_ sai_common_api_t api, - _In_ sai_status_t status, - _In_ const swss::KeyOpFieldsValuesTuple &kco, - _In_ int sequenceNumber); void sendGetResponseUpdateRedisQuadEvent( _In_ sai_object_type_t objectType, @@ -745,7 +740,7 @@ namespace syncd // make sure all object types are accounted for //if(saiObjectTypes.size() != (SAI_OBJECT_TYPE_MAX+1)) // return SAI_STATUS_FAILURE; -#if 0 +#if 1 std::set miscOperations = { //REDIS_ASIC_STATE_COMMAND_NOTIFY, // Not included in the list (flow without ringbuff) REDIS_ASIC_STATE_COMMAND_GET_STATS, @@ -853,7 +848,7 @@ namespace syncd {"crud1", {crudOperations1, crudRingBuffer1}}, {"crud2", {crudOperations2, crudRingBuffer2}}, {"misc", {miscOperations, miscRingBuffer}}, - } + }; #endif #if 0 std::array crudOperations1Enums = { @@ -943,7 +938,7 @@ namespace syncd }; #endif -#if 1 +#if 0 std::set crudOperations1 = { REDIS_ASIC_STATE_COMMAND_NOTIFY, From a2d711ef62e8f85067bda6a24e1769c6762c3179 Mon Sep 17 00:00:00 2001 From: DanielaMurin Date: Sun, 1 Sep 2024 22:08:03 +0300 Subject: [PATCH 30/35] Sequencer update & Cleanup Change namespace Add is_exit param Fix logs Move statistics to public Add new statistics parmas Change-Id: I36f9d45babc7d4ee024aa8afde44edf1615da582 --- proxylib/Proxy.cpp | 1 - saiplayer/SaiPlayer.cpp | 1 - syncd/NotificationHandler.cpp | 1 - syncd/Sequencer.cpp | 60 +++++++------ syncd/Sequencer.h | 32 ++++--- syncd/Syncd.cpp | 163 +++++++++------------------------- syncd/Syncd.h | 2 +- 7 files changed, 95 insertions(+), 165 deletions(-) diff --git a/proxylib/Proxy.cpp b/proxylib/Proxy.cpp index 56f023e1c..a844a4b2d 100644 --- a/proxylib/Proxy.cpp +++ b/proxylib/Proxy.cpp @@ -1080,7 +1080,6 @@ void Proxy::updateAttributteNotificationPointers( { SWSS_LOG_ENTER(); - SWSS_LOG_NOTICE("sai_metadata_update_attribute_notification_pointers from Proxy::updateAttributteNotificationPointers"); sai_metadata_update_attribute_notification_pointers(&m_sn, count, attr_list); } diff --git a/saiplayer/SaiPlayer.cpp b/saiplayer/SaiPlayer.cpp index e92b0241f..51b5cf13f 100644 --- a/saiplayer/SaiPlayer.cpp +++ b/saiplayer/SaiPlayer.cpp @@ -946,7 +946,6 @@ void SaiPlayer::update_notifications_pointers( * need to override them after create, and after set. */ - SWSS_LOG_NOTICE("sai_metadata_update_attribute_notification_pointers from SaiPlayer::update_notifications_pointers"); sai_metadata_update_attribute_notification_pointers(&m_switchNotifications, attr_count, attr_list); } diff --git a/syncd/NotificationHandler.cpp b/syncd/NotificationHandler.cpp index 06eb2163d..5a4fa5300 100644 --- a/syncd/NotificationHandler.cpp +++ b/syncd/NotificationHandler.cpp @@ -59,7 +59,6 @@ void NotificationHandler::updateNotificationsPointers( * Also notice that we are using the same pointers for ALL switches. */ - SWSS_LOG_NOTICE("sai_metadata_update_attribute_notification_pointers from NotificationHandler::updateNotificationsPointers"); sai_metadata_update_attribute_notification_pointers(&m_switchNotifications, attr_count, attr_list); } diff --git a/syncd/Sequencer.cpp b/syncd/Sequencer.cpp index f6fb729c1..c2d7238b1 100755 --- a/syncd/Sequencer.cpp +++ b/syncd/Sequencer.cpp @@ -1,7 +1,7 @@ #include "Sequencer.h" #include "swss/logger.h" -using namespace syncd; +using namespace sequencer; // Helper function to execute all ready responses in order // check how many consecutive responses were executed in sucession and log it @@ -11,7 +11,7 @@ Sequencer::SequenceStatus Sequencer::executeReadyResponses() { LogToModuleFile("1", "Checking for ready responses in queue..."); - while (true) { + while (!sequencer_exited) { // Check if the next sequence number is in the map auto seq_data = responses.find(next_seq_to_send); if (seq_data == responses.end()) { @@ -26,16 +26,17 @@ Sequencer::SequenceStatus Sequencer::executeReadyResponses() { LogToModuleFile("1", "before execute lambda with sequenceNumber: {}", next_seq_to_send); func(); LogToModuleFile("1", "after execute lambda with sequenceNumber: {}", next_seq_to_send); - status = NULL_PTR; + status = SUCCESS; } else { LogToModuleFile("1", "multithreaded: response lambda is null {}", next_seq_to_send); num_of_null_functions++; - status = SUCCESS; //????? + status = NULL_PTR; } // Increment the number of executed tasks in sequence - max_num_of_executed_tasks_in_sequence++; + total_num_of_executed_tasks_in_sequence++; + current_num_of_executed_tasks_in_sequence++; // Safely erase the entry responses.erase(seq_data); @@ -55,20 +56,13 @@ Sequencer::SequenceStatus Sequencer::executeReadyResponses() { } // Get sequence number -// if sequencer is full, reset the sequence number to avoid overflow -// wait for, throw full failure -// return if there is an issue or success (0-succss, 1-failure) -// add private param to see how many lambdas are stored, -// for each function, return a status code -// mux on everything -Sequencer::SequenceStatus Sequencer::allocateSequenceNumber(int *seq_num) { +// if sequencer is full, wait +bool Sequencer::allocateSequenceNumber(int *seq_num) { std::unique_lock lock(mtx); - SequenceStatus status = FAILURE; - - if(isFull()) { + + while(isFull()) { LogToModuleFile("1", "Sequencer is full, cannot allocate sequence number {}", current_seq); - *seq_num = INVALID_SEQUENCE_NUMBER; - return BUFFER_OVERFLOW; + //TODO: add sleep, and timeout error after X seconds } // update recieved param @@ -84,18 +78,18 @@ Sequencer::SequenceStatus Sequencer::allocateSequenceNumber(int *seq_num) { current_seq = 0; } - status = SUCCESS; - - return status; + return true; } // Add/Execute sequence function -Sequencer::SequenceStatus Sequencer::executeFuncInSequence(int seq, std::function response_lambda) { +bool Sequencer::executeFuncInSequence(int seq, std::function response_lambda) { std::unique_lock lock(mtx); + // internal status SequenceStatus status = FAILURE; - LogToModuleFile("1", "Enter executeFuncInSequence with seq: {}", seq); + LogToModuleFile("1", "Enter executeFuncInSequence with seq: {}", seq); + current_num_of_executed_tasks_in_sequence = 0; if (seq == next_seq_to_send) { // If the received sequence is the next one to send, execute it immediately @@ -114,7 +108,8 @@ Sequencer::SequenceStatus Sequencer::executeFuncInSequence(int seq, std::functio } // increment the number of executed tasks in sequence - max_num_of_executed_tasks_in_sequence++; + total_num_of_executed_tasks_in_sequence++; + current_num_of_executed_tasks_in_sequence++; // Increment the next sequence to send ++next_seq_to_send; @@ -127,7 +122,7 @@ Sequencer::SequenceStatus Sequencer::executeFuncInSequence(int seq, std::functio // Continue sending any subsequent responses that are ready LogToModuleFile("1", "start execute executeReadyResponses"); - status = executeReadyResponses(); + executeReadyResponses(); LogToModuleFile("1", "end execute executeReadyResponses"); } else { // If the sequence is not the next to send, store it in the map @@ -137,26 +132,33 @@ Sequencer::SequenceStatus Sequencer::executeFuncInSequence(int seq, std::functio num_of_out_of_sequence_functions++; } - return status; + if(current_num_of_executed_tasks_in_sequence > max_num_of_executed_tasks_in_sequence) { + max_num_of_executed_tasks_in_sequence = current_num_of_executed_tasks_in_sequence; + } + + if(status == SUCCESS) + return true; + else + return false; } Sequencer::SequenceStatus Sequencer::showStatistics() { std::unique_lock lock(mtx); std::string logMsg = "STATISTICS: \n"; - logMsg = "multithreaded: max number of executed tasks in sequence: " + std::to_string(max_num_of_executed_tasks_in_sequence) + " \n"; + logMsg = "multithreaded: total number of executed tasks in sequence: " + std::to_string(total_num_of_executed_tasks_in_sequence) + " \n"; logMsg += "multithreaded: number of null functions: " + std::to_string(num_of_null_functions) + " \n"; logMsg += "multithreaded: number of out of sequence functions: " + std::to_string(num_of_out_of_sequence_functions) + " \n"; logMsg += std::to_string(current_seq) + " out of " + std::to_string(max_seq_num) + "used"; - LogToModuleFile("1", logMsg); + LogToModuleFile("2", logMsg); return SUCCESS; } Sequencer::SequenceStatus Sequencer::clearStatistics() { std::unique_lock lock(mtx); - max_num_of_executed_tasks_in_sequence = 0; + total_num_of_executed_tasks_in_sequence = 0; num_of_null_functions = 0; num_of_out_of_sequence_functions = 0; - LogToModuleFile("1", "CLEANED STATISTICS \n"); + LogToModuleFile("2", "CLEANED STATISTICS \n"); return SUCCESS; } diff --git a/syncd/Sequencer.h b/syncd/Sequencer.h index be8f1d50d..d14ccb331 100755 --- a/syncd/Sequencer.h +++ b/syncd/Sequencer.h @@ -15,7 +15,7 @@ #include #include "Logger.h" -namespace syncd { +namespace sequencer { #define MAX_SEQUENCE_NUMBER 1024 #define INVALID_SEQUENCE_NUMBER std::numeric_limits::min() // allow user to choose @@ -24,23 +24,36 @@ namespace syncd { class Sequencer { public: // Public constructor - Sequencer() : current_seq(0), next_seq_to_send(0), max_seq_num(MAX_SEQUENCE_NUMBER), max_num_of_executed_tasks_in_sequence(0), num_of_null_functions(0), num_of_out_of_sequence_functions(0) {} + Sequencer() : + current_seq(0), + next_seq_to_send(0), + max_seq_num(MAX_SEQUENCE_NUMBER), + max_num_of_executed_tasks_in_sequence(0), + total_num_of_executed_tasks_in_sequence(0), + current_num_of_executed_tasks_in_sequence(0), + num_of_null_functions(0), + num_of_out_of_sequence_functions(0), + sequencer_exited(false) {} // Public destructor - ~Sequencer() {} + ~Sequencer() {sequencer_exited=true;} enum SequenceStatus { FAILURE = -1, SUCCESS = 0, - BUFFER_OVERFLOW = 1, + BUFFER_FULL = 1, NULL_PTR = 2, }; // Get sequence number - SequenceStatus allocateSequenceNumber(int *seq_num); + bool allocateSequenceNumber(int *seq_num); // Add/Execute sequence function - SequenceStatus executeFuncInSequence(int seq, std::function response_lambda); + bool executeFuncInSequence(int seq, std::function response_lambda); + + SequenceStatus showStatistics(); + + SequenceStatus clearStatistics(); private: // Watchdog function to monitor inactivity @@ -49,20 +62,19 @@ namespace syncd { // Helper function to execute all ready responses in order SequenceStatus executeReadyResponses(); - SequenceStatus showStatistics(); - - SequenceStatus clearStatistics(); - bool isFull(); int current_seq; // Tracks the latest sequence number assigned to a task int next_seq_to_send; // The next sequence number that should be sent long unsigned int max_seq_num; // The maximum sequence number int max_num_of_executed_tasks_in_sequence; // The maximum number of executed tasks in sequence + int total_num_of_executed_tasks_in_sequence; // The total number of executed tasks in sequence + int current_num_of_executed_tasks_in_sequence; // The current number of executed tasks in sequence int num_of_null_functions; // The number of null functions int num_of_out_of_sequence_functions; // The number of out of sequence functions std::mutex mtx; // Protects shared data std::map> responses; // Stores responses by sequence number + bool sequencer_exited; // Indicates if the sequencer has exited }; } diff --git a/syncd/Syncd.cpp b/syncd/Syncd.cpp index 23149308a..b76d29387 100644 --- a/syncd/Syncd.cpp +++ b/syncd/Syncd.cpp @@ -149,7 +149,7 @@ Syncd::Syncd( } m_client = std::make_shared(m_dbAsic); - m_sequencer = std::make_shared(); + m_sequencer = std::make_shared(); m_processor = std::make_shared(m_notifications, m_client, std::bind(&Syncd::syncProcessNotification, this, _1)); m_handler = std::make_shared(m_processor); @@ -242,28 +242,21 @@ void Syncd::popRingBuffer(SyncdRing* ringBuffer) } SWSS_LOG_ENTER(); ringBuffer->Started = true; - //SWSS_LOG_NOTICE("multithreaded: Syncd starts the popRingBuffer thread!"); while (!ring_thread_exited) { - //SWSS_LOG_NOTICE("multithreaded: wait popRingBuffer thread!"); - LogToModuleFile(getNameByRingBuffer(ringBuffer), "multithreaded: wait popRingBuffer thread!"); + LogToModuleFile(getNameByRingBuffer(ringBuffer), "wait popRingBuffer thread!"); std::unique_lock lock(ringBuffer->mtx); ringBuffer->cv.wait(lock, [&](){ return !ringBuffer->IsEmpty(); }); - //SWSS_LOG_NOTICE("multithreaded: Stop waiting"); - LogToModuleFile(getNameByRingBuffer(ringBuffer), "multithreaded: Stop waiting"); + LogToModuleFile(getNameByRingBuffer(ringBuffer), "Stop waiting"); ringBuffer->Idle = false; AnyTask func; while (ringBuffer->pop(func)) { - LogToModuleFile(getNameByRingBuffer(ringBuffer), "multithreaded: try to execute func"); - //SWSS_LOG_NOTICE("multithreaded: try to execute func"); + LogToModuleFile(getNameByRingBuffer(ringBuffer), "try to execute func"); func(); - LogToModuleFile(getNameByRingBuffer(ringBuffer), "multithreaded: Execute func successful"); - //SWSS_LOG_NOTICE("multithreaded: Execute func successful"); + LogToModuleFile(getNameByRingBuffer(ringBuffer), "Execute func successful"); } - LogToModuleFile(getNameByRingBuffer(ringBuffer), "multithreaded: no more functions to execute"); - //SWSS_LOG_NOTICE("multithreaded: no more functions to execute"); - // lock.unlock(); + LogToModuleFile(getNameByRingBuffer(ringBuffer), "no more functions to execute"); ringBuffer->doTask(); ringBuffer->Idle = true; } @@ -379,7 +372,7 @@ void Syncd::processEvent( { SWSS_LOG_ENTER(); static int entries = 0; - LogToModuleFile("1", "multithreaded: !!!processEvent, ITERATION: {} !!!", std::to_string(entries++)); + LogToModuleFile("1", "!!!processEvent, ITERATION: {} !!!", std::to_string(entries++)); std::lock_guard lock(m_mutex); LogToModuleFile("1", "after lock"); @@ -401,9 +394,9 @@ void Syncd::processEvent( LogToModuleFile("1","getApiRingBuffer end"); int sequenceNumber; - if(int seq_status = m_sequencer->allocateSequenceNumber(&sequenceNumber) != Sequencer::SUCCESS) + if(!m_sequencer->allocateSequenceNumber(&sequenceNumber)) { - LogToModuleFile("1", "Failed to allocate sequence number: {}", std::to_string(seq_status)); + LogToModuleFile("1", "Failed to allocate sequence number"); //todo: handle wait until sequence number is available with timeout } else { @@ -440,8 +433,7 @@ sai_status_t Syncd::processSingleEvent( auto& key = kfvKey(kco); auto& op = kfvOp(kco); - //SWSS_LOG_NOTICE("multithreaded: key: %s op: %s, sequence number: %d", key.c_str(), op.c_str(), sequenceNumber); - LogToModuleFile("1","multithreaded: key: {} op: {} {}", key, op, std::to_string(sequenceNumber)); + LogToModuleFile("1","key: {} op: {} {}", key, op, std::to_string(sequenceNumber)); if (key.length() == 0) { @@ -933,7 +925,7 @@ sai_status_t Syncd::processBulkQuadEvent( _In_ int sequenceNumber) { SWSS_LOG_ENTER(); - SWSS_LOG_INFO("multithreaded: seq=%d, api=%s, key=%s, op=%s", sequenceNumber, sai_serialize_common_api(api).c_str(), kfvKey(kco).c_str(), kfvOp(kco).c_str()); + SWSS_LOG_INFO("seq=%d, api=%s, key=%s, op=%s", sequenceNumber, sai_serialize_common_api(api).c_str(), kfvKey(kco).c_str(), kfvOp(kco).c_str()); const std::string& key = kfvKey(kco); // objectType:count @@ -2115,23 +2107,18 @@ sai_status_t Syncd::processEntry( switch (api) { case SAI_COMMON_API_CREATE: - //SWSS_LOG_NOTICE("multithreaded: processEntry SAI_COMMON_API_CREATE"); return m_vendorSai->create(metaKey, SAI_NULL_OBJECT_ID, attr_count, attr_list); case SAI_COMMON_API_REMOVE: - //SWSS_LOG_NOTICE("multithreaded: processEntry SAI_COMMON_API_REMOVE"); return m_vendorSai->remove(metaKey); case SAI_COMMON_API_SET: - //SWSS_LOG_NOTICE("multithreaded: processEntry SAI_COMMON_API_SET"); return m_vendorSai->set(metaKey, attr_list); case SAI_COMMON_API_GET: - //SWSS_LOG_NOTICE("multithreaded: processEntry SAI_COMMON_API_GET"); return m_vendorSai->get(metaKey, attr_count, attr_list); default: - //SWSS_LOG_NOTICE("multithreaded: api %s not supported", sai_serialize_common_api(api).c_str()); SWSS_LOG_THROW("api %s not supported", sai_serialize_common_api(api).c_str()); } } @@ -2715,15 +2702,12 @@ void Syncd::sendApiResponse( break; default: - //SWSS_LOG_NOTICE("multithreaded: api %s not supported by this function", sai_serialize_common_api(api).c_str()); SWSS_LOG_THROW("api %s not supported by this function", sai_serialize_common_api(api).c_str()); } if (status != SAI_STATUS_SUCCESS) { - //SWSS_LOG_NOTICE("multithreaded: api %s failed in syncd mode: %s", sai_serialize_common_api(api).c_str(), sai_serialize_status(status).c_str()); - SWSS_LOG_ERROR("api %s failed in syncd mode: %s", sai_serialize_common_api(api).c_str(), sai_serialize_status(status).c_str()); @@ -2740,8 +2724,6 @@ void Syncd::sendApiResponse( std::string strStatus = sai_serialize_status(status); - //SWSS_LOG_NOTICE("multithreaded: sending response for %s api with status: %s",sai_serialize_common_api(api).c_str(),strStatus.c_str()); - SWSS_LOG_INFO("sending response for %s api with status: %s", sai_serialize_common_api(api).c_str(), strStatus.c_str()); @@ -2750,9 +2732,6 @@ void Syncd::sendApiResponse( SWSS_LOG_INFO("response for %s api was send", sai_serialize_common_api(api).c_str()); - - //SWSS_LOG_NOTICE("multithreaded: response for %s api was send",sai_serialize_common_api(api).c_str()); - } void Syncd::processFlexCounterGroupEvent( // TODO must be moved to go via ASIC channel queue @@ -3124,20 +3103,13 @@ void Syncd::pushRingBuffer(SyncdRing* ringBuffer, AnyTask&& func) SWSS_LOG_ENTER(); if (!ringBuffer || !ringBuffer->Started) { - //SWSS_LOG_NOTICE("multithreaded: pushRingBuffer: gRingBuffer is not started"); LogToModuleFile("1", "execute fun since ringBuffer not valid or not started"); func(); - // } else if (!gRingBuffer->Serves(getName())) { - // while (!gRingBuffer->IsEmpty() || !gRingBuffer->Idle) { - // std::this_thread::sleep_for(std::chrono::milliseconds(SLEEP_MSECONDS)); - // } - // func(); } else { while (!ringBuffer->push(func)) { SWSS_LOG_WARN("fail to push..ring is full..."); } - //SWSS_LOG_NOTICE("multithreaded: pushRingBuffer: notify_one"); - LogToModuleFile("1", "multithreaded: pushRingBuffer: notify_one"); + LogToModuleFile("1", "pushRingBuffer: notify_one"); ringBuffer->cv.notify_one(); } } @@ -3148,7 +3120,6 @@ sai_status_t Syncd::processQuadEvent( _In_ int sequenceNumber) { SWSS_LOG_ENTER(); - //SWSS_LOG_NOTICE("multithreaded: %d, api=%s, key=%s, op=%s", sequenceNumber, sai_serialize_common_api(api).c_str(), kfvKey(kco).c_str(), kfvOp(kco).c_str()); LogToModuleFile("1", "{} api={}, key={}, op={}", std::to_string(sequenceNumber), sai_serialize_common_api(api), kfvKey(kco), kfvOp(kco)); const std::string& key = kfvKey(kco); @@ -3198,17 +3169,13 @@ sai_status_t Syncd::processQuadEvent( * TODO: must be done per switch, and switch may not exists yet */ - //SWSS_LOG_NOTICE("multithreaded: updateNotificationsPointers"); m_handler->updateNotificationsPointers(attr_count, attr_list); - //SWSS_LOG_NOTICE("multithreaded: finish updateNotificationsPointers"); } if (isInitViewMode()) { sai_status_t status = processQuadEventInInitViewMode(metaKey.objecttype, strObjectId, api, attr_count, attr_list, kco, sequenceNumber); - //SWSS_LOG_NOTICE("multithreaded: isInitViewMode()"); - return status; } @@ -3221,16 +3188,11 @@ sai_status_t Syncd::processQuadEvent( */ SWSS_LOG_DEBUG("translating VID to RIDs on all attributes"); - //SWSS_LOG_NOTICE("multithreaded: translateVidToRid"); m_translator->translateVidToRid(metaKey.objecttype, attr_count, attr_list); - - //SWSS_LOG_NOTICE("multithreaded: success translateVidToRid"); } - //SWSS_LOG_NOTICE("multithreaded: sai_metadata_get_object_type_info "); auto info = sai_metadata_get_object_type_info(metaKey.objecttype); - //SWSS_LOG_NOTICE("multithreaded: sai_metadata_get_object_type_info success"); sai_status_t status; @@ -3258,14 +3220,10 @@ sai_status_t Syncd::processQuadEvent( status = processOid(metaKey.objecttype, strObjectId, api, attr_count, attr_list); } - //SWSS_LOG_NOTICE("multithreaded: status %d", status); - if (api == SAI_COMMON_API_GET) { if (status != SAI_STATUS_SUCCESS) { - //SWSS_LOG_NOTICE("multithreaded: get API for key: %s op: %s returned status: %s",key.c_str(),op.c_str(),sai_serialize_status(status).c_str()); - SWSS_LOG_INFO("get API for key: %s op: %s returned status: %s", key.c_str(), op.c_str(), @@ -3275,7 +3233,6 @@ sai_status_t Syncd::processQuadEvent( // extract switch VID from any object type sai_object_id_t switchVid = VidManager::switchIdQuery(metaKey.objectkey.key.object_id); - //SWSS_LOG_NOTICE("multithreaded: SAI_COMMON_API_GET"); LogToModuleFile("1", "add sendGetResponseUpdateRedisQuadEvent to Lambda sequenceNumber {}", sequenceNumber); auto lambda = [=]() { sendGetResponseUpdateRedisQuadEvent(metaKey.objecttype, strObjectId, switchVid, status, attr_count, attr_list, kco, api); @@ -3359,28 +3316,21 @@ sai_status_t Syncd::processOid( SWSS_LOG_THROW("passing non object id %s as generic object", info->objecttypename); } - //SWSS_LOG_NOTICE("multithreaded: processOid %d", api); - switch (api) { case SAI_COMMON_API_CREATE: - //SWSS_LOG_NOTICE("multithreaded: processOid SAI_COMMON_API_CREATE"); return processOidCreate(objectType, strObjectId, attr_count, attr_list); case SAI_COMMON_API_REMOVE: - //SWSS_LOG_NOTICE("multithreaded: processOid SAI_COMMON_API_REMOVE"); return processOidRemove(objectType, strObjectId); case SAI_COMMON_API_SET: - //SWSS_LOG_NOTICE("multithreaded: processOid SAI_COMMON_API_SET"); return processOidSet(objectType, strObjectId, attr_list); case SAI_COMMON_API_GET: - //SWSS_LOG_NOTICE("multithreaded: processOid SAI_COMMON_API_GET"); return processOidGet(objectType, strObjectId, attr_count, attr_list); default: - //SWSS_LOG_NOTICE("multithreaded: common api (%s) is not implemented", sai_serialize_common_api(api).c_str()); SWSS_LOG_THROW("common api (%s) is not implemented", sai_serialize_common_api(api).c_str()); } } @@ -3392,7 +3342,6 @@ sai_status_t Syncd::processOidCreate( _In_ sai_attribute_t *attr_list) { SWSS_LOG_ENTER(); - //SWSS_LOG_NOTICE("multithreaded: processOidCreate"); sai_object_id_t objectVid; sai_deserialize_object_id(strObjectId, objectVid); @@ -3422,9 +3371,7 @@ sai_status_t Syncd::processOidCreate( sai_object_id_t objectRid; - //SWSS_LOG_NOTICE("multithreaded: processOidCreate try to m_vendorSai->create"); status = m_vendorSai->create(objectType, &objectRid, switchRid, attr_count, attr_list); - //SWSS_LOG_NOTICE("multithreaded: processOidCreate m_vendorSai->create success"); if (status == SAI_STATUS_SUCCESS) { @@ -3433,8 +3380,6 @@ sai_status_t Syncd::processOidCreate( * virtual id's to redis db. */ - //SWSS_LOG_NOTICE("multithreaded: m_vendorSai->create success"); - m_translator->insertRidAndVid(objectRid, objectVid); SWSS_LOG_INFO("saved VID %s to RID %s", @@ -3448,8 +3393,6 @@ sai_status_t Syncd::processOidCreate( * constructor, like getting all queues, ports, etc. */ - //SWSS_LOG_NOTICE("multithreaded: object type switch"); - m_switches[switchVid] = std::make_shared(switchVid, objectRid, m_client, m_translator, m_vendorSai); m_mdioIpcServer->setSwitchId(objectRid); @@ -3463,7 +3406,6 @@ sai_status_t Syncd::processOidCreate( } } - //SWSS_LOG_NOTICE("multithreaded: processOidCreate status %d", status); return status; } @@ -3471,7 +3413,6 @@ sai_status_t Syncd::processOidRemove( _In_ sai_object_type_t objectType, _In_ const std::string &strObjectId) { - //SWSS_LOG_NOTICE("multithreaded: processOidRemove"); SWSS_LOG_ENTER(); sai_object_id_t objectVid; @@ -3484,8 +3425,6 @@ sai_status_t Syncd::processOidRemove( sai_object_id_t switchVid = VidManager::switchIdQuery(objectVid); m_switches.at(switchVid)->collectPortRelatedObjects(rid); - - //SWSS_LOG_NOTICE("multithreaded: processOidRemove SAI_OBJECT_TYPE_PORT"); } sai_status_t status = m_vendorSai->remove(objectType, rid); @@ -3536,30 +3475,29 @@ sai_status_t Syncd::processOidRemove( sai_object_id_t switchVid = VidManager::switchIdQuery(objectVid); - LogToModuleFile("1", "multithreaded: processOidRemove start removing..."); + LogToModuleFile("1", "processOidRemove start removing..."); if (m_switches.at(switchVid)->isDiscoveredRid(rid)) { - LogToModuleFile("1", "multithreaded: try removeExistingObjectReference"); + LogToModuleFile("1", "try removeExistingObjectReference"); m_switches.at(switchVid)->removeExistingObjectReference(rid); - LogToModuleFile("1", "multithreaded: success removeExistingObjectReference"); + LogToModuleFile("1", "success removeExistingObjectReference"); } - LogToModuleFile("1", "multithreaded: processOidRemove removed isDiscoveredRid"); + LogToModuleFile("1", "processOidRemove removed isDiscoveredRid"); if (objectType == SAI_OBJECT_TYPE_PORT) { - LogToModuleFile("1", "multithreaded: try postPortRemove"); + LogToModuleFile("1", "try postPortRemove"); m_switches.at(switchVid)->postPortRemove(rid); - LogToModuleFile("1", "multithreaded: success postPortRemove"); + LogToModuleFile("1", "success postPortRemove"); } - LogToModuleFile("1", "multithreaded: processOidRemove removed postPortRemove"); + LogToModuleFile("1", "processOidRemove removed postPortRemove"); - SWSS_LOG_NOTICE("multithreaded: processOidRemove finish removing"); + SWSS_LOG_NOTICE("processOidRemove finish removing"); } } - //SWSS_LOG_NOTICE("multithreaded: processOidRemove status %d", status); return status; } @@ -3593,24 +3531,13 @@ sai_status_t Syncd::processOidGet( { sai_status_t status = SAI_STATUS_SUCCESS; SWSS_LOG_ENTER(); - //SWSS_LOG_NOTICE("multithreaded: processOidGet"); sai_object_id_t objectVid; sai_deserialize_object_id(strObjectId, objectVid); - //SWSS_LOG_NOTICE("multithreaded: before m_translator->translateVidToRid(objectVid)"); sai_object_id_t rid = m_translator->translateVidToRid(objectVid); - for(uint32_t i=0; i < attr_count; i++) { - //SWSS_LOG_NOTICE("multithreaded: after m_translator->translateVidToRid(objectVid), id %d, oid %d", attr_list[i].id, attr_list[i].value.oid); - } - if(attr_list[0].id == 8 && attr_list[0].value.oid == 8) { - //SWSS_LOG_NOTICE("multithreaded: TEST SKIP"); - } - else { - status = m_vendorSai->get(objectType, rid, attr_count, attr_list); - //SWSS_LOG_NOTICE("multithreaded: sai status %d from vendor", status); - } + status = m_vendorSai->get(objectType, rid, attr_count, attr_list); return status; } @@ -5542,7 +5469,7 @@ void Syncd::run() SWSS_LOG_NOTICE("starting main loop, ONLY restart query"); if (m_commandLineOptions->m_disableExitSleep) - runMainLoop = true; + runMainLoop = false; } m_timerWatchdog.setCallback(timerWatchdogCallback); @@ -5550,7 +5477,6 @@ void Syncd::run() while (runMainLoop) { // add sequence number to log messages - SWSS_LOG_NOTICE("! run main loop "); try { swss::Selectable *sel = NULL; @@ -5652,9 +5578,9 @@ void Syncd::run() else if (sel == m_flexCounter.get()) { int sequenceNumber; - if(int seq_status = m_sequencer->allocateSequenceNumber(&sequenceNumber) != Sequencer::SUCCESS) + if(!!m_sequencer->allocateSequenceNumber(&sequenceNumber)) { - LogToModuleFile("1", "Failed to allocate sequence number: {}", std::to_string(seq_status)); + LogToModuleFile("1", "Failed to allocate sequence number: {}", std::to_string(sequenceNumber)); //todo: handle wait until sequence number is available with timeout } else { @@ -5662,40 +5588,40 @@ void Syncd::run() } //directly to sequencer auto lambda = [=](){ - LogToModuleFile("1", "multithreaded: inside lambda, start m_flexCounter"); + LogToModuleFile("1", "inside lambda, start m_flexCounter"); processFlexCounterEvent(*(swss::ConsumerTable*)sel); - LogToModuleFile("1", "multithreaded: inside lambda, end m_flexCounter"); + LogToModuleFile("1", "inside lambda, end m_flexCounter"); }; - if(int seq_status = m_sequencer->executeFuncInSequence(sequenceNumber, lambda) != Sequencer::SUCCESS) + if(!m_sequencer->executeFuncInSequence(sequenceNumber, lambda)) { - LogToModuleFile("1", "m_flexCounter failed to execute function in sequence, status: {}, sequence number: {}", std::to_string(seq_status), std::to_string(sequenceNumber)); + LogToModuleFile("1", "m_flexCounter failed to execute function in sequence, sequence number: {}", std::to_string(sequenceNumber)); } } else if (sel == m_flexCounterGroup.get()) { int sequenceNumber; - if(int seq_status = m_sequencer->allocateSequenceNumber(&sequenceNumber) != Sequencer::SUCCESS) + if(!!m_sequencer->allocateSequenceNumber(&sequenceNumber)) { - LogToModuleFile("1", "Failed to allocate sequence number: {}", std::to_string(seq_status)); + LogToModuleFile("1", "Failed to allocate sequence number"); //todo: handle wait until sequence number is available with timeout } else { LogToModuleFile("1", "Allocated sequence number: ", std::to_string(sequenceNumber)); } - LogToModuleFile("1", "multithreaded: BEFORE PUSH INTO RING BUFFER sequence number: ",std::to_string(sequenceNumber)); + LogToModuleFile("1", "BEFORE PUSH INTO RING BUFFER sequence number: ",std::to_string(sequenceNumber)); //directly to sequencer auto lambda = [=](){ - LogToModuleFile("1", "multithreaded: inside lambda, start m_flexCounterGroup"); + LogToModuleFile("1", "inside lambda, start m_flexCounterGroup"); processFlexCounterGroupEvent(*(swss::ConsumerTable*)sel); - LogToModuleFile("1", "multithreaded: inside lambda, end m_flexCounterGroup"); + LogToModuleFile("1", "inside lambda, end m_flexCounterGroup"); }; - if(int seq_status = m_sequencer->executeFuncInSequence(sequenceNumber, lambda) != Sequencer::SUCCESS) + if(!m_sequencer->executeFuncInSequence(sequenceNumber, lambda)) { - LogToModuleFile("1", "m_flexCounterGroup failed to execute function in sequence, status: {}, sequence number: {}", std::to_string(seq_status), std::to_string(sequenceNumber)); + LogToModuleFile("1", "m_flexCounterGroup failed to execute function in sequence, sequence number: {}", std::to_string(sequenceNumber)); } } else if (sel == m_selectableChannel.get()) @@ -5822,29 +5748,24 @@ void Syncd::sendStatusAndEntryResponse( m_selectableChannel->set(strStatus, entry, commandType); } - void Syncd::sendStausResponseSequence( + _In_ sai_status_t status, _In_ const std::string& commandType, _In_ const std::vector& entry, _In_ int sequenceNumber) { SWSS_LOG_ENTER(); - SWSS_LOG_NOTICE("multithreaded: sendStausResponseSequence"); - - //sequenceNumber = INVALID_SEQUENCE_NUMBER; if(sequenceNumber != INVALID_SEQUENCE_NUMBER) { - SWSS_LOG_NOTICE("multithreaded: valid sequence number %d", sequenceNumber); - // If the response is to be sequenced, then add it to the sequencer LogToModuleFile("1", "add sendStatusAndEntryResponse to lambda with sequenceNumber {} ", sequenceNumber); auto lambda = [=]() { sendStatusAndEntryResponse(status, commandType, entry); }; - if(int seq_status = m_sequencer->executeFuncInSequence(sequenceNumber, lambda) != Sequencer::SUCCESS) + if(!m_sequencer->executeFuncInSequence(sequenceNumber, lambda)) { - LogToModuleFile("1", "m_flexCounterGroup failed to execute function in sequence, status: {}, sequence number: {}", std::to_string(seq_status), std::to_string(sequenceNumber)); + LogToModuleFile("1", "failed to execute function in sequence, sequence number: {}",std::to_string(sequenceNumber)); } } else { @@ -5859,14 +5780,12 @@ void Syncd::sendStausAdvancedResponseSequence( std::function lambdaFunc) { SWSS_LOG_ENTER(); - SWSS_LOG_NOTICE("multithreaded: sendStausAdvancedResponseSequence"); - //sequenceNumber = INVALID_SEQUENCE_NUMBER; if(sequenceNumber != INVALID_SEQUENCE_NUMBER) { - SWSS_LOG_NOTICE("multithreaded: valid sequence number {}", sequenceNumber); - if(int seq_status = m_sequencer->executeFuncInSequence(sequenceNumber, lambdaFunc) != Sequencer::SUCCESS) + LogToModuleFile("1", "valid sequence number {}", sequenceNumber); + if(!m_sequencer->executeFuncInSequence(sequenceNumber, lambdaFunc)) { - LogToModuleFile("1", "m_flexCounterGroup failed to execute function in sequence, status: {}, sequence number:", std::to_string(seq_status), std::to_string(sequenceNumber)); + LogToModuleFile("1", "failed to execute function in sequence, sequence number:", std::to_string(sequenceNumber)); } } else { diff --git a/syncd/Syncd.h b/syncd/Syncd.h index b7f020bc2..eb534c282 100644 --- a/syncd/Syncd.h +++ b/syncd/Syncd.h @@ -576,7 +576,7 @@ namespace syncd std::shared_ptr m_client; - std::shared_ptr m_sequencer; + std::shared_ptr m_sequencer; std::shared_ptr m_handler; From 4c72076da39f611b488ba5e6e2c0cce15eb85b3a Mon Sep 17 00:00:00 2001 From: DanielaMurin Date: Mon, 2 Sep 2024 17:34:27 +0300 Subject: [PATCH 31/35] Sequencer update - ring buffer instead of map Change-Id: Ib5453d04e8d1f32a6114a32df9149df11c3bb75f --- syncd/Sequencer.cpp | 160 ++++++++++++++++++++++++-------------------- syncd/Sequencer.h | 15 +++-- 2 files changed, 99 insertions(+), 76 deletions(-) diff --git a/syncd/Sequencer.cpp b/syncd/Sequencer.cpp index c2d7238b1..4c8fd3706 100755 --- a/syncd/Sequencer.cpp +++ b/syncd/Sequencer.cpp @@ -1,6 +1,7 @@ #include "Sequencer.h" #include "swss/logger.h" + using namespace sequencer; // Helper function to execute all ready responses in order @@ -13,43 +14,26 @@ Sequencer::SequenceStatus Sequencer::executeReadyResponses() { while (!sequencer_exited) { // Check if the next sequence number is in the map - auto seq_data = responses.find(next_seq_to_send); - if (seq_data == responses.end()) { - LogToModuleFile("1", "No next sequence found in queue"); - status = SUCCESS; - break; // Exit loop if the next sequence is not in the map + { + AnyTask func; + if(pop(func)) { + func(); + + + } + else { + break; + } } - - // Execute the stored lambda - auto func = seq_data->second; - if(func) { - LogToModuleFile("1", "before execute lambda with sequenceNumber: {}", next_seq_to_send); - func(); - LogToModuleFile("1", "after execute lambda with sequenceNumber: {}", next_seq_to_send); - status = SUCCESS; - } - else { - LogToModuleFile("1", "multithreaded: response lambda is null {}", next_seq_to_send); - num_of_null_functions++; - status = NULL_PTR; - } - - // Increment the number of executed tasks in sequence - total_num_of_executed_tasks_in_sequence++; - current_num_of_executed_tasks_in_sequence++; - - // Safely erase the entry - responses.erase(seq_data); - // Increment the sequence number - ++next_seq_to_send; + { + std::unique_lock lock(mtx); + // Increment the number of executed tasks in sequence + total_num_of_executed_tasks_in_sequence++; + current_num_of_executed_tasks_in_sequence++; + } LogToModuleFile("1", "Next sequence found! Executed lambda with seq: {}", next_seq_to_send); - - if (next_seq_to_send >= MAX_SEQUENCE_NUMBER) { - LogToModuleFile("1", "Resetting next sequence number to send needs to be reset to avoid overflow"); - next_seq_to_send = 0; - } } return status; @@ -58,25 +42,21 @@ Sequencer::SequenceStatus Sequencer::executeReadyResponses() { // Get sequence number // if sequencer is full, wait bool Sequencer::allocateSequenceNumber(int *seq_num) { - std::unique_lock lock(mtx); - - while(isFull()) { + while(IsFull()) { LogToModuleFile("1", "Sequencer is full, cannot allocate sequence number {}", current_seq); //TODO: add sleep, and timeout error after X seconds } + LogToModuleFile("1", "before lock"); + std::unique_lock lock(mtx); + LogToModuleFile("1", "after lock"); // update recieved param *seq_num = current_seq; // increment the sequence number - current_seq++; - - LogToModuleFile("1", "allocate seq num {}", *seq_num); + current_seq = (current_seq + 1) % max_seq_num; - // reset number to avoid overflow - if (current_seq >= MAX_SEQUENCE_NUMBER) { - LogToModuleFile("1", "Resetting allocated sequence number to avoid overflow"); - current_seq = 0; - } + lock.unlock(); + LogToModuleFile("1", "allocate seq num {}", *seq_num); return true; } @@ -84,13 +64,14 @@ bool Sequencer::allocateSequenceNumber(int *seq_num) { // Add/Execute sequence function bool Sequencer::executeFuncInSequence(int seq, std::function response_lambda) { - std::unique_lock lock(mtx); - // internal status - SequenceStatus status = FAILURE; - + SequenceStatus status; + { + std::unique_lock lock(mtx); + current_num_of_executed_tasks_in_sequence = 0; + } + LogToModuleFile("1", "Enter executeFuncInSequence with seq: {}", seq); - current_num_of_executed_tasks_in_sequence = 0; - + if (seq == next_seq_to_send) { // If the received sequence is the next one to send, execute it immediately @@ -107,35 +88,40 @@ bool Sequencer::executeFuncInSequence(int seq, std::function response_la status = SUCCESS; //NULL_PTR; ??? } - // increment the number of executed tasks in sequence - total_num_of_executed_tasks_in_sequence++; - current_num_of_executed_tasks_in_sequence++; + { + std::unique_lock lock(mtx); + // increment the number of executed tasks in sequence + total_num_of_executed_tasks_in_sequence++; + current_num_of_executed_tasks_in_sequence++; - // Increment the next sequence to send - ++next_seq_to_send; - - // reset number to avoid overflow - if (next_seq_to_send >= MAX_SEQUENCE_NUMBER) { - LogToModuleFile("1", "Resetting next sequence number to send needs to be reset to avoid overflow"); - next_seq_to_send = 0; + // Increment the next sequence to send + next_seq_to_send = (next_seq_to_send + 1) % max_seq_num; } - + // Continue sending any subsequent responses that are ready LogToModuleFile("1", "start execute executeReadyResponses"); executeReadyResponses(); LogToModuleFile("1", "end execute executeReadyResponses"); } else { // If the sequence is not the next to send, store it in the map - responses[seq] = response_lambda; + push(response_lambda); LogToModuleFile("1", "storing lambda with seq: {} next to send: {}", seq, next_seq_to_send); status = SUCCESS; - num_of_out_of_sequence_functions++; + + { + std::unique_lock lock(mtx); + num_of_out_of_sequence_functions++; + } + } - if(current_num_of_executed_tasks_in_sequence > max_num_of_executed_tasks_in_sequence) { - max_num_of_executed_tasks_in_sequence = current_num_of_executed_tasks_in_sequence; + { + std::unique_lock lock(mtx); + if(current_num_of_executed_tasks_in_sequence > max_num_of_executed_tasks_in_sequence) { + max_num_of_executed_tasks_in_sequence = current_num_of_executed_tasks_in_sequence; + } } - + if(status == SUCCESS) return true; else @@ -162,13 +148,45 @@ Sequencer::SequenceStatus Sequencer::clearStatistics() { return SUCCESS; } -bool Sequencer::isFull() { - if(responses.size() < max_seq_num) { - LogToModuleFile("1", "is not full"); +bool Sequencer::IsFull() +{ + std::unique_lock lock(mtx); + return (current_seq + 1) % max_seq_num == next_seq_to_send; +} +bool Sequencer::IsEmpty() +{ + std::unique_lock lock(mtx); + return current_seq == next_seq_to_send; +} + +bool Sequencer::push(std::function ringEntry) +{ + if (IsFull()) return false; + + std::unique_lock lock(mtx); + buffer[current_seq] = std::move(ringEntry); + return true; +} + +std::function& Sequencer::HeadEntry() { + return buffer[next_seq_to_send]; +} +bool Sequencer::pop(std::function& ringEntry) +{ + if (IsEmpty()) + return false; + + std::unique_lock lock(mtx); + ringEntry = std::move(buffer[next_seq_to_send]); + + if(ringEntry) { + next_seq_to_send = (next_seq_to_send + 1) % max_seq_num; + return true; } else { - LogToModuleFile("1", "is full"); - return true; + LogToModuleFile("1", "multithreaded: response lambda is null {}", next_seq_to_send); + num_of_null_functions++; + return false; } } \ No newline at end of file diff --git a/syncd/Sequencer.h b/syncd/Sequencer.h index d14ccb331..0eb0d8da5 100755 --- a/syncd/Sequencer.h +++ b/syncd/Sequencer.h @@ -13,10 +13,11 @@ #include #include #include +#include #include "Logger.h" namespace sequencer { - + #define MAX_SEQUENCE_NUMBER 1024 #define INVALID_SEQUENCE_NUMBER std::numeric_limits::min() // allow user to choose using AnyTask = std::function; @@ -25,6 +26,7 @@ namespace sequencer { public: // Public constructor Sequencer() : + buffer(MAX_SEQUENCE_NUMBER), current_seq(0), next_seq_to_send(0), max_seq_num(MAX_SEQUENCE_NUMBER), @@ -62,21 +64,24 @@ namespace sequencer { // Helper function to execute all ready responses in order SequenceStatus executeReadyResponses(); - bool isFull(); + bool IsFull(); + bool IsEmpty(); + bool push(std::function ringEntry); + std::function& HeadEntry(); + bool pop(std::function& ringEntry); + std::vector> buffer; int current_seq; // Tracks the latest sequence number assigned to a task int next_seq_to_send; // The next sequence number that should be sent - long unsigned int max_seq_num; // The maximum sequence number + int max_seq_num; // The maximum sequence number, ring buffer size int max_num_of_executed_tasks_in_sequence; // The maximum number of executed tasks in sequence int total_num_of_executed_tasks_in_sequence; // The total number of executed tasks in sequence int current_num_of_executed_tasks_in_sequence; // The current number of executed tasks in sequence int num_of_null_functions; // The number of null functions int num_of_out_of_sequence_functions; // The number of out of sequence functions std::mutex mtx; // Protects shared data - std::map> responses; // Stores responses by sequence number bool sequencer_exited; // Indicates if the sequencer has exited }; - } From c5e511abd48f9a5e1c380e34e6f62fa805623af5 Mon Sep 17 00:00:00 2001 From: aviramd Date: Tue, 3 Sep 2024 11:50:52 +0300 Subject: [PATCH 32/35] fix bug when redis reads empty key --- syncd/Syncd.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/syncd/Syncd.cpp b/syncd/Syncd.cpp index b76d29387..0dd9a12f8 100644 --- a/syncd/Syncd.cpp +++ b/syncd/Syncd.cpp @@ -438,7 +438,11 @@ sai_status_t Syncd::processSingleEvent( if (key.length() == 0) { SWSS_LOG_DEBUG("no elements in m_buffer"); - + LogToModuleFile("1", "add to lambda no elements in m_buffer sequenceNumber {}",sequenceNumber); + auto lambda = [=](){ + LogToModuleFile("1", "no elements in m_buffer sequenceNumber {}",sequenceNumber); + }; + m_sequencer->executeFuncInSequence(sequenceNumber, lambda); return SAI_STATUS_SUCCESS; } From db71e99909cbf6b5d34f16b26f63ee9db1c46151 Mon Sep 17 00:00:00 2001 From: DanielaMurin Date: Tue, 3 Sep 2024 15:46:44 +0300 Subject: [PATCH 33/35] REVERT - Sequencer update - ring buffer instead of map Change-Id: I7aad3550f334ae674eb93dfd66608fefb7aa0add --- syncd/Sequencer.cpp | 160 ++++++++++++++++++++------------------------ syncd/Sequencer.h | 15 ++--- 2 files changed, 76 insertions(+), 99 deletions(-) diff --git a/syncd/Sequencer.cpp b/syncd/Sequencer.cpp index 4c8fd3706..c2d7238b1 100755 --- a/syncd/Sequencer.cpp +++ b/syncd/Sequencer.cpp @@ -1,7 +1,6 @@ #include "Sequencer.h" #include "swss/logger.h" - using namespace sequencer; // Helper function to execute all ready responses in order @@ -14,26 +13,43 @@ Sequencer::SequenceStatus Sequencer::executeReadyResponses() { while (!sequencer_exited) { // Check if the next sequence number is in the map - { - AnyTask func; - if(pop(func)) { - func(); - - - } - else { - break; - } + auto seq_data = responses.find(next_seq_to_send); + if (seq_data == responses.end()) { + LogToModuleFile("1", "No next sequence found in queue"); + status = SUCCESS; + break; // Exit loop if the next sequence is not in the map } - - { - std::unique_lock lock(mtx); - // Increment the number of executed tasks in sequence - total_num_of_executed_tasks_in_sequence++; - current_num_of_executed_tasks_in_sequence++; + + // Execute the stored lambda + auto func = seq_data->second; + if(func) { + LogToModuleFile("1", "before execute lambda with sequenceNumber: {}", next_seq_to_send); + func(); + LogToModuleFile("1", "after execute lambda with sequenceNumber: {}", next_seq_to_send); + status = SUCCESS; } + else { + LogToModuleFile("1", "multithreaded: response lambda is null {}", next_seq_to_send); + num_of_null_functions++; + status = NULL_PTR; + } + + // Increment the number of executed tasks in sequence + total_num_of_executed_tasks_in_sequence++; + current_num_of_executed_tasks_in_sequence++; + + // Safely erase the entry + responses.erase(seq_data); + + // Increment the sequence number + ++next_seq_to_send; LogToModuleFile("1", "Next sequence found! Executed lambda with seq: {}", next_seq_to_send); + + if (next_seq_to_send >= MAX_SEQUENCE_NUMBER) { + LogToModuleFile("1", "Resetting next sequence number to send needs to be reset to avoid overflow"); + next_seq_to_send = 0; + } } return status; @@ -42,36 +58,39 @@ Sequencer::SequenceStatus Sequencer::executeReadyResponses() { // Get sequence number // if sequencer is full, wait bool Sequencer::allocateSequenceNumber(int *seq_num) { - while(IsFull()) { + std::unique_lock lock(mtx); + + while(isFull()) { LogToModuleFile("1", "Sequencer is full, cannot allocate sequence number {}", current_seq); //TODO: add sleep, and timeout error after X seconds } - LogToModuleFile("1", "before lock"); - std::unique_lock lock(mtx); - LogToModuleFile("1", "after lock"); // update recieved param *seq_num = current_seq; // increment the sequence number - current_seq = (current_seq + 1) % max_seq_num; - - lock.unlock(); + current_seq++; + LogToModuleFile("1", "allocate seq num {}", *seq_num); + // reset number to avoid overflow + if (current_seq >= MAX_SEQUENCE_NUMBER) { + LogToModuleFile("1", "Resetting allocated sequence number to avoid overflow"); + current_seq = 0; + } + return true; } // Add/Execute sequence function bool Sequencer::executeFuncInSequence(int seq, std::function response_lambda) { - SequenceStatus status; - { - std::unique_lock lock(mtx); - current_num_of_executed_tasks_in_sequence = 0; - } - + std::unique_lock lock(mtx); + // internal status + SequenceStatus status = FAILURE; + LogToModuleFile("1", "Enter executeFuncInSequence with seq: {}", seq); - + current_num_of_executed_tasks_in_sequence = 0; + if (seq == next_seq_to_send) { // If the received sequence is the next one to send, execute it immediately @@ -88,40 +107,35 @@ bool Sequencer::executeFuncInSequence(int seq, std::function response_la status = SUCCESS; //NULL_PTR; ??? } - { - std::unique_lock lock(mtx); - // increment the number of executed tasks in sequence - total_num_of_executed_tasks_in_sequence++; - current_num_of_executed_tasks_in_sequence++; + // increment the number of executed tasks in sequence + total_num_of_executed_tasks_in_sequence++; + current_num_of_executed_tasks_in_sequence++; - // Increment the next sequence to send - next_seq_to_send = (next_seq_to_send + 1) % max_seq_num; - } + // Increment the next sequence to send + ++next_seq_to_send; + // reset number to avoid overflow + if (next_seq_to_send >= MAX_SEQUENCE_NUMBER) { + LogToModuleFile("1", "Resetting next sequence number to send needs to be reset to avoid overflow"); + next_seq_to_send = 0; + } + // Continue sending any subsequent responses that are ready LogToModuleFile("1", "start execute executeReadyResponses"); executeReadyResponses(); LogToModuleFile("1", "end execute executeReadyResponses"); } else { // If the sequence is not the next to send, store it in the map - push(response_lambda); + responses[seq] = response_lambda; LogToModuleFile("1", "storing lambda with seq: {} next to send: {}", seq, next_seq_to_send); status = SUCCESS; - - { - std::unique_lock lock(mtx); - num_of_out_of_sequence_functions++; - } - + num_of_out_of_sequence_functions++; } - { - std::unique_lock lock(mtx); - if(current_num_of_executed_tasks_in_sequence > max_num_of_executed_tasks_in_sequence) { - max_num_of_executed_tasks_in_sequence = current_num_of_executed_tasks_in_sequence; - } + if(current_num_of_executed_tasks_in_sequence > max_num_of_executed_tasks_in_sequence) { + max_num_of_executed_tasks_in_sequence = current_num_of_executed_tasks_in_sequence; } - + if(status == SUCCESS) return true; else @@ -148,45 +162,13 @@ Sequencer::SequenceStatus Sequencer::clearStatistics() { return SUCCESS; } -bool Sequencer::IsFull() -{ - std::unique_lock lock(mtx); - return (current_seq + 1) % max_seq_num == next_seq_to_send; -} -bool Sequencer::IsEmpty() -{ - std::unique_lock lock(mtx); - return current_seq == next_seq_to_send; -} - -bool Sequencer::push(std::function ringEntry) -{ - if (IsFull()) +bool Sequencer::isFull() { + if(responses.size() < max_seq_num) { + LogToModuleFile("1", "is not full"); return false; - - std::unique_lock lock(mtx); - buffer[current_seq] = std::move(ringEntry); - return true; -} - -std::function& Sequencer::HeadEntry() { - return buffer[next_seq_to_send]; -} -bool Sequencer::pop(std::function& ringEntry) -{ - if (IsEmpty()) - return false; - - std::unique_lock lock(mtx); - ringEntry = std::move(buffer[next_seq_to_send]); - - if(ringEntry) { - next_seq_to_send = (next_seq_to_send + 1) % max_seq_num; - return true; } else { - LogToModuleFile("1", "multithreaded: response lambda is null {}", next_seq_to_send); - num_of_null_functions++; - return false; + LogToModuleFile("1", "is full"); + return true; } } \ No newline at end of file diff --git a/syncd/Sequencer.h b/syncd/Sequencer.h index 0eb0d8da5..d14ccb331 100755 --- a/syncd/Sequencer.h +++ b/syncd/Sequencer.h @@ -13,11 +13,10 @@ #include #include #include -#include #include "Logger.h" namespace sequencer { - + #define MAX_SEQUENCE_NUMBER 1024 #define INVALID_SEQUENCE_NUMBER std::numeric_limits::min() // allow user to choose using AnyTask = std::function; @@ -26,7 +25,6 @@ namespace sequencer { public: // Public constructor Sequencer() : - buffer(MAX_SEQUENCE_NUMBER), current_seq(0), next_seq_to_send(0), max_seq_num(MAX_SEQUENCE_NUMBER), @@ -64,24 +62,21 @@ namespace sequencer { // Helper function to execute all ready responses in order SequenceStatus executeReadyResponses(); - bool IsFull(); - bool IsEmpty(); - bool push(std::function ringEntry); - std::function& HeadEntry(); - bool pop(std::function& ringEntry); + bool isFull(); - std::vector> buffer; int current_seq; // Tracks the latest sequence number assigned to a task int next_seq_to_send; // The next sequence number that should be sent - int max_seq_num; // The maximum sequence number, ring buffer size + long unsigned int max_seq_num; // The maximum sequence number int max_num_of_executed_tasks_in_sequence; // The maximum number of executed tasks in sequence int total_num_of_executed_tasks_in_sequence; // The total number of executed tasks in sequence int current_num_of_executed_tasks_in_sequence; // The current number of executed tasks in sequence int num_of_null_functions; // The number of null functions int num_of_out_of_sequence_functions; // The number of out of sequence functions std::mutex mtx; // Protects shared data + std::map> responses; // Stores responses by sequence number bool sequencer_exited; // Indicates if the sequencer has exited }; + } From d52b04ee35616dffba4651ce45a7362ab4f979d3 Mon Sep 17 00:00:00 2001 From: shiraez Date: Tue, 3 Sep 2024 14:59:01 +0300 Subject: [PATCH 34/35] protected RedisClient + RedisSelectableChannel --- meta/RedisSelectableChannel.cpp | 84 +++++++++++++----- meta/RedisSelectableChannel.h | 8 +- syncd/RedisClient.cpp | 153 ++++++++++++++++++++++++++------ syncd/RedisClient.h | 9 +- syncd/Syncd.cpp | 9 +- syncd/Syncd.h | 5 +- 6 files changed, 214 insertions(+), 54 deletions(-) diff --git a/meta/RedisSelectableChannel.cpp b/meta/RedisSelectableChannel.cpp index 50553e33b..925503f17 100644 --- a/meta/RedisSelectableChannel.cpp +++ b/meta/RedisSelectableChannel.cpp @@ -1,18 +1,37 @@ #include "RedisSelectableChannel.h" #include "swss/logger.h" +#include "syncd/Logger.h" using namespace sairedis; +#define MY_LOCK() \ +if(m_protected) \ +{ \ + LogToModuleFile("1", "before MY_LOCK()"); \ + m_mutex->lock(); \ + LogToModuleFile("1", "after MY_LOCK()"); \ +} + +#define MY_UNLOCK() \ +if(m_protected) \ +{ \ + LogToModuleFile("1", "before MY_UNLOCK()"); \ + m_mutex->unlock(); \ + LogToModuleFile("1", "after MY_UNLOCK()"); \ +} + RedisSelectableChannel::RedisSelectableChannel( _In_ std::shared_ptr dbAsic, _In_ const std::string& asicStateTable, _In_ const std::string& getResponseTable, _In_ const std::string& tempPrefix, - _In_ bool modifyRedis): + _In_ bool modifyRedis, + _In_ std::shared_ptr t_mutex): m_dbAsic(dbAsic), m_tempPrefix(tempPrefix), - m_modifyRedis(modifyRedis) + m_modifyRedis(modifyRedis), + m_mutex(t_mutex) { SWSS_LOG_ENTER(); @@ -29,13 +48,23 @@ RedisSelectableChannel::RedisSelectableChannel( m_getResponse = std::make_shared(m_dbAsic.get(), getResponseTable); SWSS_LOG_NOTICE("opened redis channel"); + if(t_mutex != NULL) + { + m_protected = true; + } + else + { + m_protected = false; + } } bool RedisSelectableChannel::empty() { SWSS_LOG_ENTER(); - - return m_asicState->empty(); + MY_LOCK(); + bool ans = m_asicState->empty(); + MY_UNLOCK(); + return ans; } void RedisSelectableChannel::set( @@ -44,8 +73,9 @@ void RedisSelectableChannel::set( _In_ const std::string& op) { SWSS_LOG_ENTER(); - + MY_LOCK(); m_getResponse->set(key, values, op); + MY_UNLOCK(); } void RedisSelectableChannel::pop( @@ -53,7 +83,7 @@ void RedisSelectableChannel::pop( _In_ bool initViewMode) { SWSS_LOG_ENTER(); - + MY_LOCK(); if (initViewMode) { m_asicState->pop(kco, m_tempPrefix); @@ -62,6 +92,7 @@ void RedisSelectableChannel::pop( { m_asicState->pop(kco); } + MY_UNLOCK(); } // Selectable overrides @@ -69,48 +100,61 @@ void RedisSelectableChannel::pop( int RedisSelectableChannel::getFd() { SWSS_LOG_ENTER(); - - return m_asicState->getFd(); + MY_LOCK(); + int ans = m_asicState->getFd(); + MY_UNLOCK(); + return ans; } uint64_t RedisSelectableChannel::readData() { SWSS_LOG_ENTER(); - - return m_asicState->readData(); + MY_LOCK(); + uint64_t ans = m_asicState->readData(); + MY_UNLOCK(); + return ans; } bool RedisSelectableChannel::hasData() { SWSS_LOG_ENTER(); - - return m_asicState->hasData(); + MY_LOCK(); + bool ans = m_asicState->hasData(); + MY_UNLOCK(); + return ans; } bool RedisSelectableChannel::hasCachedData() { SWSS_LOG_ENTER(); - - return m_asicState->hasCachedData(); + MY_LOCK(); + bool ans = m_asicState->hasCachedData(); + MY_UNLOCK(); + return ans; } bool RedisSelectableChannel::initializedWithData() { SWSS_LOG_ENTER(); - - return m_asicState->initializedWithData(); + MY_LOCK(); + bool ans = m_asicState->initializedWithData(); + MY_UNLOCK(); + return ans; } void RedisSelectableChannel::updateAfterRead() { SWSS_LOG_ENTER(); - - return m_asicState->updateAfterRead(); + MY_LOCK(); + m_asicState->updateAfterRead(); + MY_UNLOCK(); } int RedisSelectableChannel::getPri() const { SWSS_LOG_ENTER(); - - return m_asicState->getPri(); + MY_LOCK(); + auto ans = m_asicState->getPri(); + MY_UNLOCK(); + return ans; } diff --git a/meta/RedisSelectableChannel.h b/meta/RedisSelectableChannel.h index a5bd951b6..d229706c1 100644 --- a/meta/RedisSelectableChannel.h +++ b/meta/RedisSelectableChannel.h @@ -4,6 +4,7 @@ #include "swss/consumertable.h" #include "swss/producertable.h" +#include namespace sairedis { @@ -17,7 +18,8 @@ namespace sairedis _In_ const std::string& asicStateTable, _In_ const std::string& getResponseTable, _In_ const std::string& tempPrefix, - _In_ bool modifyRedis); + _In_ bool modifyRedis, + _In_ std::shared_ptr mutex = nullptr); virtual ~RedisSelectableChannel() = default; @@ -61,5 +63,9 @@ namespace sairedis std::string m_tempPrefix; bool m_modifyRedis; + + bool m_protected; + + std::shared_ptr m_mutex; }; } diff --git a/syncd/RedisClient.cpp b/syncd/RedisClient.cpp index 2ce2eac1f..8ab01059d 100644 --- a/syncd/RedisClient.cpp +++ b/syncd/RedisClient.cpp @@ -8,6 +8,23 @@ #include "swss/logger.h" #include "swss/redisapi.h" +#include "Logger.h" + +#define MY_LOCK() \ +if(m_protected) \ +{ \ + LogToModuleFile("1", "before MY_LOCK()"); \ + m_mutex->lock(); \ + LogToModuleFile("1", "after MY_LOCK()"); \ +} + +#define MY_UNLOCK() \ +if(m_protected) \ +{ \ + LogToModuleFile("1", "before MY_UNLOCK()"); \ + m_mutex->unlock(); \ + LogToModuleFile("1", "after MY_UNLOCK()"); \ +} using namespace syncd; // vid and rid maps contains objects from all switches @@ -20,14 +37,24 @@ using namespace syncd; #define COLDVIDS "COLDVIDS" RedisClient::RedisClient( - _In_ std::shared_ptr dbAsic): - m_dbAsic(dbAsic) + _In_ std::shared_ptr dbAsic, _In_ std::shared_ptr t_mutex): + m_dbAsic(dbAsic), + m_mutex(t_mutex) { SWSS_LOG_ENTER(); std::string fdbFlushLuaScript = swss::loadLuaScript("fdb_flush.lua"); // TODO script must be updated to version 2 m_fdbFlushSha = swss::loadRedisScript(dbAsic.get(), fdbFlushLuaScript); + + if(t_mutex != nullptr) + { + m_protected = true; + } + else + { + m_protected = false; + } } RedisClient::~RedisClient() @@ -74,7 +101,9 @@ void RedisClient::clearLaneMap( auto key = getRedisLanesKey(switchVid); + MY_LOCK(); m_dbAsic->del(key); + MY_UNLOCK(); } std::unordered_map RedisClient::getLaneMap( @@ -83,8 +112,9 @@ std::unordered_map RedisClient::getLaneMap( SWSS_LOG_ENTER(); auto key = getRedisLanesKey(switchVid); - + MY_LOCK(); auto hash = m_dbAsic->hgetall(key); + MY_UNLOCK(); SWSS_LOG_DEBUG("previous lanes: %lu", hash.size()); @@ -126,7 +156,9 @@ void RedisClient::saveLaneMap( auto key = getRedisLanesKey(switchVid); + MY_LOCK(); m_dbAsic->hset(key, strLane, strPortId); + MY_UNLOCK(); } } @@ -134,8 +166,9 @@ std::unordered_map RedisClient::getObjectMap( _In_ const std::string &key) const { SWSS_LOG_ENTER(); - + MY_LOCK(); auto hash = m_dbAsic->hgetall(key); + MY_UNLOCK(); std::unordered_map map; @@ -227,8 +260,9 @@ void RedisClient::setDummyAsicStateObject( std::string strVid = sai_serialize_object_id(objectVid); std::string strKey = ASIC_STATE_TABLE + (":" + strObjectType + ":" + strVid); - + MY_LOCK(); m_dbAsic->hset(strKey, "NULL", "NULL"); + MY_UNLOCK(); } std::string RedisClient::getRedisColdVidsKey( @@ -275,8 +309,9 @@ void RedisClient::saveColdBootDiscoveredVids( std::string strObjectType = sai_serialize_object_type(objectType); std::string strVid = sai_serialize_object_id(vid); - + MY_LOCK(); m_dbAsic->hset(key, strVid, strObjectType); + MY_UNLOCK(); } } @@ -317,7 +352,11 @@ std::shared_ptr RedisClient::getSwitchHiddenAttribute( auto key = getRedisHiddenKey(switchVid); - return m_dbAsic->hget(key, attrIdName); + MY_LOCK(); + std::shared_ptr ans = m_dbAsic->hget(key, attrIdName); + MY_UNLOCK(); + + return ans; } void RedisClient::saveSwitchHiddenAttribute( @@ -330,8 +369,10 @@ void RedisClient::saveSwitchHiddenAttribute( auto key = getRedisHiddenKey(switchVid); std::string strRid = sai_serialize_object_id(objectRid); - + + MY_LOCK(); m_dbAsic->hset(key, attrIdName, strRid); + MY_UNLOCK(); } std::set RedisClient::getColdVids( @@ -341,7 +382,9 @@ std::set RedisClient::getColdVids( auto key = getRedisColdVidsKey(switchVid); + MY_LOCK(); auto hash = m_dbAsic->hgetall(key); + MY_UNLOCK(); /* * NOTE: some objects may not exists after 2nd restart, like VLAN_MEMBER or @@ -360,8 +403,9 @@ std::set RedisClient::getColdVids( /* * Just make sure that vid in COLDVIDS is present in current vid2rid map */ - + MY_LOCK(); auto rid = m_dbAsic->hget(VIDTORID, strVid); + MY_UNLOCK(); if (rid == nullptr) { @@ -388,7 +432,9 @@ void RedisClient::setPortLanes( std::string strLane = sai_serialize_number(lane); std::string strPortRid = sai_serialize_object_id(portRid); + MY_LOCK(); m_dbAsic->hset(key, strLane, strPortRid); + MY_UNLOCK(); } } @@ -401,7 +447,9 @@ size_t RedisClient::getAsicObjectsSize( // go N times on every switch and it can be slow, we need to find better // way to do this + MY_LOCK(); auto keys = m_dbAsic->keys(ASIC_STATE_TABLE ":*"); + MY_UNLOCK(); size_t count = 0; @@ -445,7 +493,9 @@ int RedisClient::removePortFromLanesMap( { std::string strLane = sai_serialize_number(kv.first); + MY_LOCK(); m_dbAsic->hdel(key, strLane); + MY_UNLOCK(); removed++; } @@ -467,7 +517,9 @@ void RedisClient::removeAsicObject( SWSS_LOG_INFO("removing ASIC DB key: %s", key.c_str()); + MY_LOCK(); m_dbAsic->del(key); + MY_UNLOCK(); } void RedisClient::removeAsicObject( @@ -476,8 +528,9 @@ void RedisClient::removeAsicObject( SWSS_LOG_ENTER(); std::string key = (ASIC_STATE_TABLE ":") + sai_serialize_object_meta_key(metaKey); - + MY_LOCK(); m_dbAsic->del(key); + MY_UNLOCK(); } void RedisClient::removeTempAsicObject( @@ -487,7 +540,9 @@ void RedisClient::removeTempAsicObject( std::string key = (TEMP_PREFIX ASIC_STATE_TABLE ":") + sai_serialize_object_meta_key(metaKey); + MY_LOCK(); m_dbAsic->del(key); + MY_UNLOCK(); } void RedisClient::removeAsicObjects( @@ -503,7 +558,9 @@ void RedisClient::removeAsicObjects( prefixKeys.push_back((ASIC_STATE_TABLE ":") + key); } + MY_LOCK(); m_dbAsic->del(prefixKeys); + MY_UNLOCK(); } void RedisClient::removeTempAsicObjects( @@ -519,7 +576,9 @@ void RedisClient::removeTempAsicObjects( prefixKeys.push_back((TEMP_PREFIX ASIC_STATE_TABLE ":") + key); } + MY_LOCK(); m_dbAsic->del(prefixKeys); + MY_UNLOCK(); } void RedisClient::setAsicObject( @@ -531,7 +590,9 @@ void RedisClient::setAsicObject( std::string key = (ASIC_STATE_TABLE ":") + sai_serialize_object_meta_key(metaKey); + MY_LOCK(); m_dbAsic->hset(key, attr, value); + MY_UNLOCK(); } void RedisClient::setTempAsicObject( @@ -543,7 +604,9 @@ void RedisClient::setTempAsicObject( std::string key = (TEMP_PREFIX ASIC_STATE_TABLE ":") + sai_serialize_object_meta_key(metaKey); + MY_LOCK(); m_dbAsic->hset(key, attr, value); + MY_UNLOCK(); } void RedisClient::createAsicObject( @@ -556,14 +619,18 @@ void RedisClient::createAsicObject( if (attrs.size() == 0) { + MY_LOCK(); m_dbAsic->hset(key, "NULL", "NULL"); + MY_UNLOCK(); return; } - + + MY_LOCK(); for (const auto& e: attrs) { m_dbAsic->hset(key, fvField(e), fvValue(e)); } + MY_UNLOCK(); } void RedisClient::createTempAsicObject( @@ -573,17 +640,19 @@ void RedisClient::createTempAsicObject( SWSS_LOG_ENTER(); std::string key = (TEMP_PREFIX ASIC_STATE_TABLE ":") + sai_serialize_object_meta_key(metaKey); - + MY_LOCK(); if (attrs.size() == 0) { m_dbAsic->hset(key, "NULL", "NULL"); + MY_UNLOCK(); return; } - + for (const auto& e: attrs) { m_dbAsic->hset(key, fvField(e), fvValue(e)); } + MY_UNLOCK(); } void RedisClient::createAsicObjects( @@ -603,8 +672,9 @@ void RedisClient::createAsicObjects( hash[(ASIC_STATE_TABLE ":") + kvp.first].emplace_back(std::make_pair("NULL", "NULL")); } } - + MY_LOCK(); m_dbAsic->hmset(hash); + MY_UNLOCK(); } void RedisClient::createTempAsicObjects( @@ -624,15 +694,16 @@ void RedisClient::createTempAsicObjects( hash[(TEMP_PREFIX ASIC_STATE_TABLE ":") + kvp.first].emplace_back(std::make_pair("NULL", "NULL")); } } - + MY_LOCK(); m_dbAsic->hmset(hash); + MY_UNLOCK(); } void RedisClient::setVidAndRidMap( _In_ const std::unordered_map& map) { SWSS_LOG_ENTER(); - + MY_LOCK(); m_dbAsic->del(VIDTORID); m_dbAsic->del(RIDTOVID); @@ -644,20 +715,25 @@ void RedisClient::setVidAndRidMap( m_dbAsic->hset(VIDTORID, strVid, strRid); m_dbAsic->hset(RIDTOVID, strRid, strVid); } + MY_UNLOCK(); } std::vector RedisClient::getAsicStateKeys() const { SWSS_LOG_ENTER(); - - return m_dbAsic->keys(ASIC_STATE_TABLE ":*"); + MY_LOCK(); + std::vector ans = m_dbAsic->keys(ASIC_STATE_TABLE ":*"); + MY_UNLOCK(); + return ans; } std::vector RedisClient::getAsicStateSwitchesKeys() const { SWSS_LOG_ENTER(); - - return m_dbAsic->keys(ASIC_STATE_TABLE ":SAI_OBJECT_TYPE_SWITCH:*"); + MY_LOCK(); + std::vector ans = m_dbAsic->keys(ASIC_STATE_TABLE ":SAI_OBJECT_TYPE_SWITCH:*"); + MY_UNLOCK(); + return ans; } void RedisClient::removeColdVid( @@ -668,8 +744,9 @@ void RedisClient::removeColdVid( auto strVid = sai_serialize_object_id(vid); auto key = getRedisColdVidsKey(vid); - + MY_LOCK(); m_dbAsic->hdel(key, strVid); + MY_UNLOCK(); } std::unordered_map RedisClient::getAttributesFromAsicKey( @@ -678,15 +755,18 @@ std::unordered_map RedisClient::getAttributesFromAsicK SWSS_LOG_ENTER(); std::unordered_map map; + MY_LOCK(); m_dbAsic->hgetall(key, std::inserter(map, map.end())); + MY_UNLOCK(); return map; } bool RedisClient::hasNoHiddenKeysDefined() const { SWSS_LOG_ENTER(); - + MY_LOCK(); auto keys = m_dbAsic->keys(HIDDEN "*"); + MY_UNLOCK(); return keys.size() == 0; } @@ -700,8 +780,10 @@ void RedisClient::removeVidAndRid( auto strVid = sai_serialize_object_id(vid); auto strRid = sai_serialize_object_id(rid); + MY_LOCK(); m_dbAsic->hdel(VIDTORID, strVid); m_dbAsic->hdel(RIDTOVID, strRid); + MY_UNLOCK(); } void RedisClient::insertVidAndRid( @@ -713,8 +795,10 @@ void RedisClient::insertVidAndRid( auto strVid = sai_serialize_object_id(vid); auto strRid = sai_serialize_object_id(rid); + MY_LOCK(); m_dbAsic->hset(VIDTORID, strVid, strRid); m_dbAsic->hset(RIDTOVID, strRid, strVid); + MY_UNLOCK(); } sai_object_id_t RedisClient::getVidForRid( @@ -723,8 +807,9 @@ sai_object_id_t RedisClient::getVidForRid( SWSS_LOG_ENTER(); auto strRid = sai_serialize_object_id(rid); - + MY_LOCK(); auto pvid = m_dbAsic->hget(RIDTOVID, strRid); + MY_UNLOCK(); if (pvid == nullptr) { @@ -747,8 +832,9 @@ sai_object_id_t RedisClient::getRidForVid( SWSS_LOG_ENTER(); auto strVid = sai_serialize_object_id(vid); - + MY_LOCK(); auto prid = m_dbAsic->hget(VIDTORID, strVid); + MY_UNLOCK(); if (prid == nullptr) { @@ -768,25 +854,27 @@ sai_object_id_t RedisClient::getRidForVid( void RedisClient::removeAsicStateTable() { SWSS_LOG_ENTER(); - + MY_LOCK(); const auto &asicStateKeys = m_dbAsic->keys(ASIC_STATE_TABLE ":*"); for (const auto &key: asicStateKeys) { m_dbAsic->del(key); } + MY_UNLOCK(); } void RedisClient::removeTempAsicStateTable() { SWSS_LOG_ENTER(); - + MY_LOCK(); const auto &tempAsicStateKeys = m_dbAsic->keys(TEMP_PREFIX ASIC_STATE_TABLE ":*"); for (const auto &key: tempAsicStateKeys) { m_dbAsic->del(key); } + MY_UNLOCK(); } std::map RedisClient::getAsicView() @@ -809,12 +897,13 @@ std::map RedisClient::getAsicView( SWSS_LOG_ENTER(); SWSS_LOG_TIMER("get asic view from %s", tableName.c_str()); - + MY_LOCK(); swss::Table table(m_dbAsic.get(), tableName); swss::TableDump dump; table.dump(dump); + MY_UNLOCK(); std::map map; @@ -893,6 +982,7 @@ void RedisClient::processFlushEvent( SWSS_LOG_THROW("unknown fdb flush entry type: %d", type); } + MY_LOCK(); for (int flush_static: vals) { swss::RedisCommand command; @@ -906,4 +996,13 @@ void RedisClient::processFlushEvent( swss::RedisReply r(m_dbAsic.get(), command); } + MY_UNLOCK(); } + + +swss::DBConnector *RedisClient::redis_get(){ + MY_LOCK(); + auto reply = m_dbAsic.get(); + MY_UNLOCK(); + return reply; +} \ No newline at end of file diff --git a/syncd/RedisClient.h b/syncd/RedisClient.h index 84663981a..3e65086fe 100644 --- a/syncd/RedisClient.h +++ b/syncd/RedisClient.h @@ -11,6 +11,7 @@ extern "C" { #include #include #include +#include namespace syncd { @@ -19,7 +20,7 @@ namespace syncd public: RedisClient( - _In_ std::shared_ptr dbAsic); + _In_ std::shared_ptr dbAsic, _In_ std::shared_ptr t_mutex = nullptr); virtual ~RedisClient(); @@ -158,6 +159,8 @@ namespace syncd _In_ sai_object_id_t bvId, _In_ sai_fdb_flush_entry_type_t type); + swss::DBConnector *redis_get(); + private: std::map getAsicView( @@ -181,5 +184,9 @@ namespace syncd std::string m_fdbFlushSha; + bool m_protected; + + std::shared_ptr m_mutex; + }; } diff --git a/syncd/Syncd.cpp b/syncd/Syncd.cpp index 0dd9a12f8..e8dbe2b58 100644 --- a/syncd/Syncd.cpp +++ b/syncd/Syncd.cpp @@ -140,15 +140,18 @@ Syncd::Syncd( bool modifyRedis = m_enableSyncMode ? false : true; + m_redis_mutex = std::make_shared(); + m_selectableChannel = std::make_shared( m_dbAsic, ASIC_STATE_TABLE, REDIS_TABLE_GETRESPONSE, TEMP_PREFIX, - modifyRedis); + modifyRedis, + m_redis_mutex); } - m_client = std::make_shared(m_dbAsic); + m_client = std::make_shared(m_dbAsic, m_redis_mutex); m_sequencer = std::make_shared(); m_processor = std::make_shared(m_notifications, m_client, std::bind(&Syncd::syncProcessNotification, this, _1)); @@ -167,7 +170,7 @@ Syncd::Syncd( m_handler->setSwitchNotifications(m_sn.getSwitchNotifications()); - m_restartQuery = std::make_shared(m_dbAsic.get(), SYNCD_NOTIFICATION_CHANNEL_RESTARTQUERY_PER_DB(m_contextConfig->m_dbAsic)); + m_restartQuery = std::make_shared(m_client->redis_get(), SYNCD_NOTIFICATION_CHANNEL_RESTARTQUERY_PER_DB(m_contextConfig->m_dbAsic)); // TODO to be moved to ASIC_DB m_dbFlexCounter = std::make_shared(m_contextConfig->m_dbFlex, 0); diff --git a/syncd/Syncd.h b/syncd/Syncd.h index eb534c282..8d94249ac 100644 --- a/syncd/Syncd.h +++ b/syncd/Syncd.h @@ -615,6 +615,7 @@ namespace syncd * * getting flex counter - here we skip using mutex */ std::mutex m_mutex; + std::shared_ptr m_redis_mutex; std::shared_ptr m_dbAsic; @@ -740,7 +741,7 @@ namespace syncd // make sure all object types are accounted for //if(saiObjectTypes.size() != (SAI_OBJECT_TYPE_MAX+1)) // return SAI_STATUS_FAILURE; -#if 1 +#if 0 std::set miscOperations = { //REDIS_ASIC_STATE_COMMAND_NOTIFY, // Not included in the list (flow without ringbuff) REDIS_ASIC_STATE_COMMAND_GET_STATS, @@ -938,7 +939,7 @@ namespace syncd }; #endif -#if 0 +#if 1 std::set crudOperations1 = { REDIS_ASIC_STATE_COMMAND_NOTIFY, From cad2dcb42e9d1d1c80d17b0a70643c1df25ce780 Mon Sep 17 00:00:00 2001 From: aviramd Date: Wed, 4 Sep 2024 12:41:17 +0300 Subject: [PATCH 35/35] remove LOCK from loop wrap it instead , add thread id prints, 3 threads --- syncd/Logger.h | 5 ++++- syncd/RedisClient.cpp | 27 +++++++++++++++------------ syncd/Syncd.cpp | 11 ++++++++--- syncd/Syncd.h | 4 ++-- 4 files changed, 29 insertions(+), 18 deletions(-) diff --git a/syncd/Logger.h b/syncd/Logger.h index b8619355c..a845d14e7 100755 --- a/syncd/Logger.h +++ b/syncd/Logger.h @@ -14,6 +14,7 @@ #include #include "fmt/format.h" #include +#include #define MAX_LOG_SIZE (10 *50 * 1024) /* 50 KB */ #define ENABLE_LOGGING 1 @@ -59,8 +60,10 @@ static void writeToLogFile(const std::string& funcName, const std::string& fileN << '.' << std::setw(3) << std::setfill('0') << milliseconds.count(); std::string formatted_time = oss.str(); + std::thread::id this_id = std::this_thread::get_id(); + // Write the timestamp, function name, and message to the log file - logFile << formatted_time << " " << "V1" << funcName << ": " << message << std::endl; + logFile << formatted_time << " V4 " << this_id << " " << funcName << ": " << message << std::endl; logFile.close(); diff --git a/syncd/RedisClient.cpp b/syncd/RedisClient.cpp index 8ab01059d..879701a5f 100644 --- a/syncd/RedisClient.cpp +++ b/syncd/RedisClient.cpp @@ -146,6 +146,7 @@ void RedisClient::saveLaneMap( clearLaneMap(switchVid); + MY_LOCK(); for (auto const &it: map) { sai_uint32_t lane = it.first; @@ -156,10 +157,9 @@ void RedisClient::saveLaneMap( auto key = getRedisLanesKey(switchVid); - MY_LOCK(); m_dbAsic->hset(key, strLane, strPortId); - MY_UNLOCK(); } + MY_UNLOCK(); } std::unordered_map RedisClient::getObjectMap( @@ -302,6 +302,7 @@ void RedisClient::saveColdBootDiscoveredVids( auto key = getRedisColdVidsKey(switchVid); + MY_LOCK(); for (auto vid: coldVids) { sai_object_type_t objectType = VidManager::objectTypeQuery(vid); @@ -309,10 +310,10 @@ void RedisClient::saveColdBootDiscoveredVids( std::string strObjectType = sai_serialize_object_type(objectType); std::string strVid = sai_serialize_object_id(vid); - MY_LOCK(); + m_dbAsic->hset(key, strVid, strObjectType); - MY_UNLOCK(); } + MY_UNLOCK(); } std::string RedisClient::getRedisHiddenKey( @@ -392,7 +393,7 @@ std::set RedisClient::getColdVids( */ std::set coldVids; - + MY_LOCK(); for (auto kvp: hash) { auto strVid = kvp.first; @@ -403,9 +404,9 @@ std::set RedisClient::getColdVids( /* * Just make sure that vid in COLDVIDS is present in current vid2rid map */ - MY_LOCK(); + auto rid = m_dbAsic->hget(VIDTORID, strVid); - MY_UNLOCK(); + if (rid == nullptr) { @@ -414,6 +415,7 @@ std::set RedisClient::getColdVids( coldVids.insert(vid); } + MY_UNLOCK(); return coldVids; } @@ -427,15 +429,16 @@ void RedisClient::setPortLanes( auto key = getRedisLanesKey(switchVid); + MY_LOCK(); for (uint32_t lane: lanes) { std::string strLane = sai_serialize_number(lane); std::string strPortRid = sai_serialize_object_id(portRid); - MY_LOCK(); m_dbAsic->hset(key, strLane, strPortRid); - MY_UNLOCK(); + } + MY_UNLOCK(); } size_t RedisClient::getAsicObjectsSize( @@ -487,19 +490,19 @@ int RedisClient::removePortFromLanesMap( auto key = getRedisLanesKey(switchVid); + MY_LOCK(); for (auto& kv: map) { if (kv.second == portRid) { std::string strLane = sai_serialize_number(kv.first); - MY_LOCK(); m_dbAsic->hdel(key, strLane); - MY_UNLOCK(); - + removed++; } } + MY_UNLOCK(); return removed; } diff --git a/syncd/Syncd.cpp b/syncd/Syncd.cpp index e8dbe2b58..7902745ab 100644 --- a/syncd/Syncd.cpp +++ b/syncd/Syncd.cpp @@ -375,7 +375,8 @@ void Syncd::processEvent( { SWSS_LOG_ENTER(); static int entries = 0; - LogToModuleFile("1", "!!!processEvent, ITERATION: {} !!!", std::to_string(entries++)); + + LogToModuleFile("1", "!!!processEvent, ITERATION: {} m_enableSyncMode {}!!!", entries++, m_enableSyncMode); std::lock_guard lock(m_mutex); LogToModuleFile("1", "after lock"); @@ -403,7 +404,7 @@ void Syncd::processEvent( //todo: handle wait until sequence number is available with timeout } else { - LogToModuleFile("1", "Allocated sequence number: ", std::to_string(sequenceNumber)); + LogToModuleFile("1", "Allocated sequence number: {}", sequenceNumber); } auto& key = kfvKey(kco); @@ -5488,7 +5489,9 @@ void Syncd::run() { swss::Selectable *sel = NULL; + LogToModuleFile("1", "before select"); int result = s->select(&sel); + LogToModuleFile("1", "after select"); //restart query sequencer if (sel == m_restartQuery.get()) @@ -5502,11 +5505,12 @@ void Syncd::run() */ SWSS_LOG_NOTICE("is asic queue empty: %d", m_selectableChannel->empty()); - + LogToModuleFile("1", "is asic queue empty start"); while (!m_selectableChannel->empty()) { processEvent(*m_selectableChannel.get()); } + LogToModuleFile("1", "is asic queue empty end"); SWSS_LOG_NOTICE("drained queue"); @@ -5633,6 +5637,7 @@ void Syncd::run() } else if (sel == m_selectableChannel.get()) { + LogToModuleFile("1", "m_selectableChannel.get()"); processEvent(*m_selectableChannel.get()); } else diff --git a/syncd/Syncd.h b/syncd/Syncd.h index 8d94249ac..7d70ff65c 100644 --- a/syncd/Syncd.h +++ b/syncd/Syncd.h @@ -741,7 +741,7 @@ namespace syncd // make sure all object types are accounted for //if(saiObjectTypes.size() != (SAI_OBJECT_TYPE_MAX+1)) // return SAI_STATUS_FAILURE; -#if 0 +#if 1 std::set miscOperations = { //REDIS_ASIC_STATE_COMMAND_NOTIFY, // Not included in the list (flow without ringbuff) REDIS_ASIC_STATE_COMMAND_GET_STATS, @@ -939,7 +939,7 @@ namespace syncd }; #endif -#if 1 +#if 0 std::set crudOperations1 = { REDIS_ASIC_STATE_COMMAND_NOTIFY,