[active-standby] Add oscillation logic when there is no heartbeat on …

…both sides (#221) Description of PR Summary: Fixes # (issue) This PR is to cover an extreme edge case when none of the dualtors can receive ICMP heartbeat. The goal is to oscillate mux direction between two sides so if one side recovers, mux direction can have a chance to park on this side. Note that toggles in an unhealthy scenario like this will cause longer disruption than in a healthy scenario. sign-off: Jing Zhang [email protected]
sonic-net · Nov 10, 2023 · 489f6ce · 489f6ce
1 parent 10b7c0b
commit 489f6ce
Show file tree

Hide file tree

Showing 6 changed files with 133 additions and 2 deletions.
diff --git a/src/DbInterface.cpp b/src/DbInterface.cpp
@@ -47,7 +47,7 @@ std::vector<std::string> DbInterface::mMuxState = {"active", "standby", "unknown
 std::vector<std::string> DbInterface::mMuxLinkmgrState = {"uninitialized", "unhealthy", "healthy"};
 std::vector<std::string> DbInterface::mMuxMetrics = {"start", "end"};
 std::vector<std::string> DbInterface::mLinkProbeMetrics = {"link_prober_unknown_start", "link_prober_unknown_end", "link_prober_wait_start", "link_prober_active_start", "link_prober_standby_start"};
-std::vector<std::string> DbInterface::mActiveStandbySwitchCause = {"Peer_Heartbeat_Missing" , "Peer_Link_Down" , "Tlv_Switch_Active_Command" , "Link_Down" , "Transceiver_Daemon_Timeout" , "Matching_Hardware_State" , "Config_Mux_Mode", "Hardware_State_Unknown"};
+std::vector<std::string> DbInterface::mActiveStandbySwitchCause = {"Peer_Heartbeat_Missing" , "Peer_Link_Down" , "Tlv_Switch_Active_Command" , "Link_Down" , "Transceiver_Daemon_Timeout" , "Matching_Hardware_State" , "Config_Mux_Mode", "Hardware_State_Unknown", "Timed_Oscillation"};
 
 //
 // ---> DbInterface(mux::MuxManager *muxManager);

diff --git a/src/common/MuxConfig.h b/src/common/MuxConfig.h
@@ -274,6 +274,33 @@ class MuxConfig
     */
     inline uint32_t getSuspendTimeout_msec() const {return (mNegativeStateChangeRetryCount + 1) * mTimeoutIpv4_msec;};
 
+    /**
+    *@method getOscillationInterval_sec
+    *
+    *@brief getter for mux state oscillation interval
+    *
+    *@return oscillation interval
+    */
+    inline uint32_t getOscillationInterval_sec() const {return mOscillationTimeout_sec;};
+
+    /**
+    *@method setOscillationInterval_sec
+    *
+    *@brief setter for mux state oscillation interval
+    *
+    *@param oscillationInterval_sec (in) oscillation interval
+    *@param force (in) force set the oscillation interval
+    *
+    *@return none
+    */
+    inline void setOscillationInterval_sec(uint32_t oscillationInterval_sec, bool force=false) {
+        if (force || oscillationInterval_sec > 300) {
+            mOscillationTimeout_sec = oscillationInterval_sec;
+        } else {
+            mOscillationTimeout_sec = 300;
+        }
+    };
+
     /**
     *@method getMuxStateChangeRetryCount
     *
@@ -406,6 +433,7 @@ class MuxConfig
 
 private:
     uint8_t mNumberOfThreads = 5;
+    uint32_t mOscillationTimeout_sec = 300;
     uint32_t mTimeoutIpv4_msec = 100;
     uint32_t mTimeoutIpv6_msec = 1000;
     uint32_t mPositiveStateChangeRetryCount = 1;

diff --git a/src/common/MuxPortConfig.h b/src/common/MuxPortConfig.h
@@ -192,6 +192,15 @@ class MuxPortConfig
     */
     inline uint32_t getLinkWaitTimeout_msec() const {return mMuxConfig.getSuspendTimeout_msec();};
 
+    /**
+    *@method getOscillationInterval_sec
+    *
+    *@brief getter for mux state oscillation interval
+    *
+    *@return oscillation interval
+    */
+    inline uint32_t getOscillationInterval_sec() const {return mMuxConfig.getOscillationInterval_sec();};
+
     /**
     *@method getMuxStateChangeRetryCount
     *

diff --git a/src/link_manager/LinkManagerStateMachineActiveStandby.cpp b/src/link_manager/LinkManagerStateMachineActiveStandby.cpp
@@ -54,12 +54,15 @@ ActiveStandbyStateMachine::ActiveStandbyStateMachine(
         {link_prober::LinkProberState::Label::Unknown, mux_state::MuxState::Label::Wait, link_state::LinkState::Label::Down}
     ),
     mDeadlineTimer(strand.context()),
-    mWaitTimer(strand.context())
+    mWaitTimer(strand.context()),
+    mOscillationTimer(strand.context())
 {
     assert(muxPortPtr != nullptr);
     mMuxStateMachine.setWaitStateCause(mux_state::WaitState::WaitStateCause::SwssUpdate);
     mMuxPortPtr->setMuxLinkmgrState(mLabel);
     initializeTransitionFunctionTable();
+
+    mOscillationTimer.expires_from_now(boost::posix_time::seconds(1)); 
 }
 
 //
@@ -1037,6 +1040,41 @@ void ActiveStandbyStateMachine::handleMuxWaitTimeout(boost::system::error_code e
     }
 }
 
+//
+// ---> startOscillationTimer();
+//
+// when there is no icmp heartbeat, start a timer to oscillate between active and standby
+//
+void ActiveStandbyStateMachine::startOscillationTimer()
+{
+    // Note: This timer is started when Mux state is active and link prober is in wait state.
+    mOscillationTimer.expires_from_now(boost::posix_time::seconds(
+        mMuxPortConfig.getOscillationInterval_sec()
+    ));
+    mOscillationTimer.async_wait(boost::asio::bind_executor(getStrand(), boost::bind(
+            &ActiveStandbyStateMachine::handleOscillationTimeout,
+            this,
+            boost::asio::placeholders::error
+    )));
+}
+
+//
+// ---> handleOscillationTimeout(boost::system::error_code errorCode);
+//
+// handle when oscillation timer expires
+//
+void ActiveStandbyStateMachine::handleOscillationTimeout(boost::system::error_code errorCode)
+{
+    MUXLOGDEBUG(mMuxPortConfig.getPortName());
+
+    if (errorCode == boost::system::errc::success &&
+        ps(mCompositeState) == link_prober::LinkProberState::Label::Wait &&
+        ms(mCompositeState) == mux_state::MuxState::Label::Active &&
+        ls(mCompositeState) == link_state::LinkState::Label::Up) {
+            switchMuxState(link_manager::ActiveStandbyStateMachine::SwitchCause::TimedOscillation, mCompositeState, mux_state::MuxState::Label::Standby);
+    }
+}
+
 //
 // ---> initLinkProberState(CompositeState &compositeState, bool forceReset);
 //
@@ -1262,6 +1300,10 @@ void ActiveStandbyStateMachine::LinkProberWaitMuxActiveLinkUpTransitionFunction(
     if (mWaitActiveUpCount++ & 0x1) {
         mSuspendTxFnPtr(mMuxPortConfig.getLinkWaitTimeout_msec());
     }
+
+    if (mOscillationTimer.expires_at() < boost::posix_time::microsec_clock::local_time()) {
+        startOscillationTimer();
+    }
 }
 
 //

diff --git a/src/link_manager/LinkManagerStateMachineActiveStandby.h b/src/link_manager/LinkManagerStateMachineActiveStandby.h
@@ -73,6 +73,7 @@ class ActiveStandbyStateMachine: public LinkManagerStateMachineBase,
         MatchingHardwareState,
         ConfigMuxMode,
         HarewareStateUnknown,
+        TimedOscillation,
 
         Count
     };
@@ -454,6 +455,26 @@ class ActiveStandbyStateMachine: public LinkManagerStateMachineBase,
     */
     void handleMuxWaitTimeout(boost::system::error_code errorCode);
 
+    /**
+    *@method startOscillationTimer
+    *
+    *@brief when there is no icmp heartbeat, start a timer to oscillate between active and standby
+    *
+    *@return none
+    */
+    void startOscillationTimer();
+
+    /**
+    *@method handleOscillationTimeout
+    *
+    *@brief handle when oscillation timer expires
+    *
+    *@param errorCode (in)          timer error code
+    *
+    *@return none
+    */
+    void handleOscillationTimeout(boost::system::error_code errorCode);
+
     /**
     *@method initLinkProberState
     *
@@ -826,6 +847,7 @@ class ActiveStandbyStateMachine: public LinkManagerStateMachineBase,
 
     boost::asio::deadline_timer mDeadlineTimer;
     boost::asio::deadline_timer mWaitTimer;
+    boost::asio::deadline_timer mOscillationTimer;
 
     boost::function<void ()> mInitializeProberFnPtr;
     boost::function<void ()> mStartProbingFnPtr;

diff --git a/test/LinkManagerStateMachineTest.cpp b/test/LinkManagerStateMachineTest.cpp
@@ -49,6 +49,7 @@ LinkManagerStateMachineTest::LinkManagerStateMachineTest() :
     mMuxConfig.setPositiveStateChangeRetryCount(mPositiveUpdateCount);
     mMuxConfig.setMuxStateChangeRetryCount(mPositiveUpdateCount);
     mMuxConfig.setLinkStateChangeRetryCount(mPositiveUpdateCount);
+    mMuxConfig.setOscillationInterval_sec(1,true);
 }
 
 void LinkManagerStateMachineTest::runIoService(uint32_t count)
@@ -1435,5 +1436,34 @@ TEST_F(LinkManagerStateMachineTest, MuxStandbyLinkProberStandbyLinkDownLinkUpRes
     VALIDATE_STATE(Standby, Standby, Up);
 }
 
+TEST_F(LinkManagerStateMachineTest, TimedOscillation)
+{
+    setMuxStandby();
+
+    // set icmp timeout to be 500ms otherwise it will probe mux state endlessly and get no chance to do timed oscillation
+    mMuxConfig.setTimeoutIpv4_msec(500);
+
+    postLinkProberEvent(link_prober::LinkProberState::Unknown, 2);
+    VALIDATE_STATE(Wait, Wait, Up);
+    EXPECT_EQ(mDbInterfacePtr->mSetMuxStateInvokeCount, 1);
+    EXPECT_EQ(mDbInterfacePtr->mLastPostedSwitchCause, link_manager::ActiveStandbyStateMachine::SwitchCause::PeerHeartbeatMissing);
+
+    // swss notification
+    handleMuxState("active", 3);
+    VALIDATE_STATE(Wait, Active, Up);
+
+    runIoService(2);
+    VALIDATE_STATE(Wait, Wait, Up);
+
+    handleProbeMuxState("active", 3);
+    VALIDATE_STATE(Wait, Active, Up);
+
+    runIoService(2);
+    VALIDATE_STATE(Wait, Wait, Up);
+    EXPECT_EQ(mDbInterfacePtr->mLastPostedSwitchCause, link_manager::ActiveStandbyStateMachine::SwitchCause::TimedOscillation);
+
+    mMuxConfig.setTimeoutIpv4_msec(10);
+}
+
 
 } /* namespace test */