Adding minor misc fixes for pfcwd scripts. (sonic-net#15286)

Description of PR
This PR addresses the following issues:

The dut-counter check on pfcwd scripts is using a wrong logic, it attempts to read the incoming drops and incoming packets and compares. Unfortunately, the packets are not being dropped in ingress, but they are dropped in egress. So this PR changes the logic to read the outgoing port's drops, and incoming ports' total packets.
The flow-completion-check in the common code is for 20 seconds. But some of the traffic streams is configured for 26 seconds, so we need to check for atleast 26 seconds. So I have increased the time to 30 seconds.
Along with the above, the pfcwd_basic script is updated to use the common code as implemented in the PR:15099.
Summary:
Pls see above.

Approach
What is the motivation for this PR?
Fixing some of the issues that we came across during the testing.
move the repeated code to the common code.
How did you do it?
Updated the counter reading mechanism.
Updated the max_attempts for flow-completion to 30 seconds.
Updated the pfcwd_basic to use the new lib code from Move the common code from testcases to the fixtures. sonic-net#15099.
How did you verify/test it?
Ran it on my TB:
Any platform specific information?
Changes were tested on cisco-8000 only. The one fail below is not related to this change.

My run result:
=========================================================================================================================== PASSES =========================================================================================================================== ______________________________________________________________________________________________ test_pfcwd_basic_single_lossless_prio[multidut_port_info0-True] _______________________________________________________________________________________________ ______________________________________________________________________________________________ test_pfcwd_basic_single_lossless_prio[multidut_port_info1-True] _______________________________________________________________________________________________ ______________________________________________________________________________________________ test_pfcwd_basic_single_lossless_prio[multidut_port_info1-False] ______________________________________________________________________________________________ ----------------------------------------------------------------------------- generated xml file: /run_logs/ixia/fixturize-reboot/2024-10-30-21-43-12/tr_2024-10-30-21-43-12.xml ----------------------------------------------------------------------------- INFO:root:Can not get Allure report URL. Please check logs ------------------------------------------------------------------------------------------------------------------- live log sessionfinish ------------------------------------------------------------------------------------------------------------------- 21:54:15 __init__.pytest_terminal_summary         L0067 INFO   | Can not get Allure report URL. Please check logs ================================================================================================================== short test summary info =================================================================================================================== PASSED snappi_tests/multidut/pfcwd/test_multidut_pfcwd_basic_with_snappi.py::test_pfcwd_basic_single_lossless_prio[multidut_port_info0-True] PASSED snappi_tests/multidut/pfcwd/test_multidut_pfcwd_basic_with_snappi.py::test_pfcwd_basic_single_lossless_prio[multidut_port_info1-True] PASSED snappi_tests/multidut/pfcwd/test_multidut_pfcwd_basic_with_snappi.py::test_pfcwd_basic_single_lossless_prio[multidut_port_info1-False] FAILED snappi_tests/multidut/pfcwd/test_multidut_pfcwd_basic_with_snappi.py::test_pfcwd_basic_single_lossless_prio[multidut_port_info0-False] - Failed: Loss rate of Data Flow 1 (0.17994320225890004) should be in [0, 0] ==================================================================================================== 1 failed, 3 passed, 7 warnings in 660.78s (0:11:00) ===================================================================================================== sonic@ixia-sonic-mgmt-whitebox:/data/tests$  

co-authrized by: [email protected]

tests/common/snappi_tests/common_helpers.py

@@ -1134,9 +1134,9 @@ def get_interface_stats(duthost, port):
     """
     # Initializing nested dictionary i_stats
     i_stats = defaultdict(dict)
-    i_stats[duthost.hostname][port] = {}
 
     n_out = parse_portstat(duthost.command('portstat -i {}'.format(port))['stdout_lines'])[port]
+    i_stats[duthost.hostname][port] = n_out
     # rx_err, rx_ovr and rx_drp are counted in single counter rx_fail
     # tx_err, tx_ovr and tx_drp are counted in single counter tx_fail
     rx_err = ['rx_err', 'rx_ovr', 'rx_drp']

tests/snappi_tests/multidut/pfc/files/lossless_response_to_external_pause_storms_helper.py

@@ -137,15 +137,13 @@ def run_lossless_response_to_external_pause_storms_test(api,
                                                    exp_dur_sec=DATA_FLOW_DURATION_SEC + DATA_FLOW_DELAY_SEC,
                                                    snappi_extra_params=snappi_extra_params)
 
-    tx_port1 = tx_port[0]['peer_port']
-    tx_port2 = tx_port[1]['peer_port']
+    dut_tx_port = rx_port['peer_port']
+    dut_rx_port1 = tx_port[0]['peer_port']
+    dut_rx_port2 = tx_port[1]['peer_port']
     # Fetch relevant statistics
-    pkt_drop1 = get_interface_stats(ingress_duthost, tx_port1)[ingress_duthost.hostname][tx_port1]['rx_fail']
-    pkt_drop2 = get_interface_stats(ingress_duthost, tx_port2)[ingress_duthost.hostname][tx_port2]['rx_fail']
-    rx_pkts_1 = get_interface_stats(ingress_duthost, tx_port1)[ingress_duthost.hostname][tx_port1]['rx_pkts']
-    rx_pkts_2 = get_interface_stats(ingress_duthost, tx_port2)[ingress_duthost.hostname][tx_port2]['rx_pkts']
-    # Calculate the total packet drop
-    pkt_drop = pkt_drop1 + pkt_drop2
+    pkt_drop = get_interface_stats(egress_duthost, dut_tx_port)[ingress_duthost.hostname][dut_tx_port]['tx_drp']
+    rx_pkts_1 = get_interface_stats(ingress_duthost, dut_rx_port1)[ingress_duthost.hostname][dut_rx_port1]['rx_ok']
+    rx_pkts_2 = get_interface_stats(ingress_duthost, dut_rx_port2)[ingress_duthost.hostname][dut_rx_port2]['rx_ok']
     # Calculate the total received packets
     total_rx_pkts = rx_pkts_1 + rx_pkts_2
     # Calculate the drop percentage

tests/snappi_tests/multidut/pfc/files/lossless_response_to_throttling_pause_storms_helper.py

@@ -144,15 +144,13 @@ def run_lossless_response_to_throttling_pause_storms_test(api,
                                                    exp_dur_sec=DATA_FLOW_DURATION_SEC + DATA_FLOW_DELAY_SEC,
                                                    snappi_extra_params=snappi_extra_params)
 
-    tx_port1 = tx_port[0]['peer_port']
-    tx_port2 = tx_port[1]['peer_port']
+    dut_tx_port = rx_port['peer_port']
+    dut_rx_port1 = tx_port[0]['peer_port']
+    dut_rx_port2 = tx_port[1]['peer_port']
     # Fetch relevant statistics
-    pkt_drop1 = get_interface_stats(ingress_duthost, tx_port1)[ingress_duthost.hostname][tx_port1]['rx_fail']
-    pkt_drop2 = get_interface_stats(ingress_duthost, tx_port2)[ingress_duthost.hostname][tx_port2]['rx_fail']
-    rx_pkts_1 = get_interface_stats(ingress_duthost, tx_port1)[ingress_duthost.hostname][tx_port1]['rx_pkts']
-    rx_pkts_2 = get_interface_stats(ingress_duthost, tx_port2)[ingress_duthost.hostname][tx_port2]['rx_pkts']
-    # Calculate the total packet drop
-    pkt_drop = pkt_drop1 + pkt_drop2
+    pkt_drop = get_interface_stats(egress_duthost, dut_tx_port)[ingress_duthost.hostname][dut_tx_port]['tx_drp']
+    rx_pkts_1 = get_interface_stats(ingress_duthost, dut_rx_port1)[ingress_duthost.hostname][dut_rx_port1]['rx_ok']
+    rx_pkts_2 = get_interface_stats(ingress_duthost, dut_rx_port2)[ingress_duthost.hostname][dut_rx_port2]['rx_ok']
     # Calculate the total received packets
     total_rx_pkts = rx_pkts_1 + rx_pkts_2
     # Calculate the drop percentage

tests/snappi_tests/multidut/pfc/files/m2o_fluctuating_lossless_helper.py

@@ -1,5 +1,6 @@
 import logging                                                                          # noqa: F401
 import random
+from math import ceil
 from tests.common.helpers.assertions import pytest_assert, pytest_require               # noqa: F401
 from tests.common.fixtures.conn_graph_facts import conn_graph_facts, fanout_graph_facts  # noqa: F401
 from tests.common.snappi_tests.snappi_helpers import get_dut_port_id                     # noqa: F401
@@ -10,7 +11,6 @@
 from tests.common.snappi_tests.traffic_generation import run_traffic, \
      setup_base_traffic_config          # noqa: F401
 from tests.snappi_tests.variables import pfcQueueGroupSize, pfcQueueValueDict
-from math import ceil
 logger = logging.getLogger(__name__)
 
 PAUSE_FLOW_NAME = 'Pause Storm'
@@ -126,15 +126,13 @@ def run_m2o_fluctuating_lossless_test(api,
                                                    exp_dur_sec=DATA_FLOW_DURATION_SEC + DATA_FLOW_DELAY_SEC,
                                                    snappi_extra_params=snappi_extra_params)
 
-    tx_port1 = tx_port[0]['peer_port']
-    tx_port2 = tx_port[1]['peer_port']
+    dut_tx_port = rx_port['peer_port']
+    dut_rx_port1 = tx_port[0]['peer_port']
+    dut_rx_port2 = tx_port[1]['peer_port']
     # Fetch relevant statistics
-    pkt_drop1 = get_interface_stats(ingress_duthost, tx_port1)[ingress_duthost.hostname][tx_port1]['rx_fail']
-    pkt_drop2 = get_interface_stats(ingress_duthost, tx_port2)[ingress_duthost.hostname][tx_port2]['rx_fail']
-    rx_pkts_1 = get_interface_stats(ingress_duthost, tx_port1)[ingress_duthost.hostname][tx_port1]['rx_pkts']
-    rx_pkts_2 = get_interface_stats(ingress_duthost, tx_port2)[ingress_duthost.hostname][tx_port2]['rx_pkts']
-    # Calculate the total packet drop
-    pkt_drop = pkt_drop1 + pkt_drop2
+    pkt_drop = get_interface_stats(egress_duthost, dut_tx_port)[ingress_duthost.hostname][dut_tx_port]['tx_drp']
+    rx_pkts_1 = get_interface_stats(ingress_duthost, dut_rx_port1)[ingress_duthost.hostname][dut_rx_port1]['rx_ok']
+    rx_pkts_2 = get_interface_stats(ingress_duthost, dut_rx_port2)[ingress_duthost.hostname][dut_rx_port2]['rx_ok']
     # Calculate the total received packets
     total_rx_pkts = rx_pkts_1 + rx_pkts_2
     # Calculate the drop percentage

tests/snappi_tests/multidut/pfc/files/m2o_oversubscribe_lossless_helper.py

@@ -129,15 +129,13 @@ def run_m2o_oversubscribe_lossless_test(api,
                                                    exp_dur_sec=DATA_FLOW_DURATION_SEC + DATA_FLOW_DELAY_SEC,
                                                    snappi_extra_params=snappi_extra_params)
 
-    tx_port1 = tx_port[0]['peer_port']
-    tx_port2 = tx_port[1]['peer_port']
+    dut_tx_port = rx_port['peer_port']
+    dut_rx_port1 = tx_port[0]['peer_port']
+    dut_rx_port2 = tx_port[1]['peer_port']
     # Fetch relevant statistics
-    pkt_drop1 = get_interface_stats(ingress_duthost, tx_port1)[ingress_duthost.hostname][tx_port1]['rx_fail']
-    pkt_drop2 = get_interface_stats(ingress_duthost, tx_port2)[ingress_duthost.hostname][tx_port2]['rx_fail']
-    rx_pkts_1 = get_interface_stats(ingress_duthost, tx_port1)[ingress_duthost.hostname][tx_port1]['rx_pkts']
-    rx_pkts_2 = get_interface_stats(ingress_duthost, tx_port2)[ingress_duthost.hostname][tx_port2]['rx_pkts']
-    # Calculate the total packet drop
-    pkt_drop = pkt_drop1 + pkt_drop2
+    pkt_drop = get_interface_stats(egress_duthost, dut_tx_port)[ingress_duthost.hostname][dut_tx_port]['tx_drp']
+    rx_pkts_1 = get_interface_stats(ingress_duthost, dut_rx_port1)[ingress_duthost.hostname][dut_rx_port1]['rx_ok']
+    rx_pkts_2 = get_interface_stats(ingress_duthost, dut_rx_port2)[ingress_duthost.hostname][dut_rx_port2]['rx_ok']
     # Calculate the total received packets
     total_rx_pkts = rx_pkts_1 + rx_pkts_2
     # Calculate the drop percentage

tests/snappi_tests/multidut/pfc/files/m2o_oversubscribe_lossless_lossy_helper.py

@@ -10,12 +10,13 @@
 from tests.common.fixtures.conn_graph_facts import conn_graph_facts, fanout_graph_facts  # noqa: F401
 from tests.common.snappi_tests.snappi_helpers import get_dut_port_id                     # noqa: F401
 from tests.common.snappi_tests.common_helpers import pfc_class_enable_vector, \
-    stop_pfcwd, disable_packet_aging, sec_to_nanosec, get_interface_stats               # noqa: F401
+    stop_pfcwd, disable_packet_aging, sec_to_nanosec, get_interface_stats       # noqa: F401
 from tests.common.snappi_tests.port import select_ports                                 # noqa: F401
 from tests.common.snappi_tests.snappi_test_params import SnappiTestParams
 from tests.common.snappi_tests.traffic_generation import run_traffic, \
      setup_base_traffic_config                      # noqa: F401
 from tests.snappi_tests.variables import pfcQueueGroupSize, pfcQueueValueDict
+from tests.common.portstat_utilities import parse_portstat                              # noqa: F401
 
 logger = logging.getLogger(__name__)
 
@@ -133,15 +134,13 @@ def run_pfc_m2o_oversubscribe_lossless_lossy_test(api,
                                                    exp_dur_sec=DATA_FLOW_DURATION_SEC + DATA_FLOW_DELAY_SEC,
                                                    snappi_extra_params=snappi_extra_params)
 
-    tx_port1 = tx_port[0]['peer_port']
-    tx_port2 = tx_port[1]['peer_port']
+    dut_rx_port1 = tx_port[0]['peer_port']
+    dut_rx_port2 = tx_port[1]['peer_port']
+    dut_tx_port = rx_port['peer_port']
     # Fetch relevant statistics
-    pkt_drop1 = get_interface_stats(ingress_duthost, tx_port1)[ingress_duthost.hostname][tx_port1]['rx_fail']
-    pkt_drop2 = get_interface_stats(ingress_duthost, tx_port2)[ingress_duthost.hostname][tx_port2]['rx_fail']
-    rx_pkts_1 = get_interface_stats(ingress_duthost, tx_port1)[ingress_duthost.hostname][tx_port1]['rx_pkts']
-    rx_pkts_2 = get_interface_stats(ingress_duthost, tx_port2)[ingress_duthost.hostname][tx_port2]['rx_pkts']
-    # Calculate the total packet drop
-    pkt_drop = pkt_drop1 + pkt_drop2
+    pkt_drop = get_interface_stats(egress_duthost, dut_tx_port)[ingress_duthost.hostname][dut_tx_port]['tx_drp']
+    rx_pkts_1 = get_interface_stats(ingress_duthost, dut_rx_port1)[ingress_duthost.hostname][dut_rx_port1]['rx_ok']
+    rx_pkts_2 = get_interface_stats(ingress_duthost, dut_rx_port2)[ingress_duthost.hostname][dut_rx_port2]['rx_ok']
     # Calculate the total received packets
     total_rx_pkts = rx_pkts_1 + rx_pkts_2
     # Calculate the drop percentage

tests/snappi_tests/multidut/pfc/files/m2o_oversubscribe_lossy_helper.py

@@ -135,14 +135,13 @@ def run_pfc_m2o_oversubscribe_lossy_test(api,
                                                    exp_dur_sec=DATA_FLOW_DURATION_SEC + DATA_FLOW_DELAY_SEC,
                                                    snappi_extra_params=snappi_extra_params)
 
-    tx_port1 = tx_port[0]['peer_port']
-    tx_port2 = tx_port[1]['peer_port']
-    pkt_drop1 = get_interface_stats(ingress_duthost, tx_port1)[ingress_duthost.hostname][tx_port1]['rx_fail']
-    pkt_drop2 = get_interface_stats(ingress_duthost, tx_port2)[ingress_duthost.hostname][tx_port2]['rx_fail']
-    rx_pkts_1 = get_interface_stats(ingress_duthost, tx_port1)[ingress_duthost.hostname][tx_port1]['rx_pkts']
-    rx_pkts_2 = get_interface_stats(ingress_duthost, tx_port2)[ingress_duthost.hostname][tx_port2]['rx_pkts']
-    # Calculate the total packet drop
-    pkt_drop = pkt_drop1 + pkt_drop2
+    dut_tx_port = rx_port['peer_port']
+    dut_rx_port1 = tx_port[0]['peer_port']
+    dut_rx_port2 = tx_port[1]['peer_port']
+
+    pkt_drop = get_interface_stats(egress_duthost, dut_tx_port)[ingress_duthost.hostname][dut_tx_port]['tx_drp']
+    rx_pkts_1 = get_interface_stats(ingress_duthost, dut_rx_port1)[ingress_duthost.hostname][dut_rx_port1]['rx_ok']
+    rx_pkts_2 = get_interface_stats(ingress_duthost, dut_rx_port2)[ingress_duthost.hostname][dut_rx_port2]['rx_ok']
     # Calculate the total received packets
     total_rx_pkts = rx_pkts_1 + rx_pkts_2
     # Calculate the drop percentage

tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_burst_storm_helper.py

@@ -285,7 +285,7 @@ def __run_traffic(api, config, all_flow_names, exp_dur_sec):
     time.sleep(exp_dur_sec)
 
     attempts = 0
-    max_attempts = 20
+    max_attempts = 30
 
     while attempts < max_attempts:
         request = api.metrics_request()

tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_basic_with_snappi.py

@@ -11,13 +11,12 @@
     get_snappi_ports_multi_dut, is_snappi_multidut, \
     snappi_api, snappi_dut_base_config, get_snappi_ports, get_snappi_ports_for_rdma, cleanup_config      # noqa: F401
 from tests.common.snappi_tests.qos_fixtures import prio_dscp_map, lossless_prio_list      # noqa F401
-from tests.snappi_tests.variables import MULTIDUT_PORT_INFO, MULTIDUT_TESTBED    # noqa: F401
 from tests.common.reboot import reboot                              # noqa: F401
 from tests.common.utilities import wait_until                       # noqa: F401
 from tests.snappi_tests.multidut.pfcwd.files.pfcwd_multidut_basic_helper import run_pfcwd_basic_test
 from tests.common.snappi_tests.snappi_test_params import SnappiTestParams
 from tests.snappi_tests.files.helper import skip_pfcwd_test, reboot_duts, \
-    setup_ports_and_dut, multidut_port_info  # noqa: F401
+    setup_ports_and_dut   # noqa: F401
 logger = logging.getLogger(__name__)
 pytestmark = [pytest.mark.topology('multidut-tgen', 'tgen')]
 
@@ -124,13 +123,12 @@ def test_pfcwd_basic_single_lossless_prio_reboot(snappi_api,                # no
                                                  fanout_graph_facts_multidut,        # noqa F811
                                                  localhost,
                                                  duthosts,
-                                                 lossless_prio_list,   # noqa: F811
-                                                 tbinfo,      # noqa: F811
-                                                 prio_dscp_map,             # noqa: F811
-                                                 setup_ports_and_dut,       # noqa: F811
+                                                 enum_dut_lossless_prio_with_completeness_level,   # noqa: F811
+                                                 get_snappi_ports,   # noqa: F811
+                                                 prio_dscp_map,             # noqa F811
+                                                 setup_ports_and_dut,    # noqa: F811
                                                  reboot_duts,               # noqa: F811
-                                                 trigger_pfcwd              # noqa: F811
-                                                 ):
+                                                 trigger_pfcwd):
     """
     Verify PFC watchdog basic test works on a single lossless priority after various types of reboot
 
@@ -141,7 +139,6 @@ def test_pfcwd_basic_single_lossless_prio_reboot(snappi_api,                # no
         localhost (pytest fixture): localhost handle
         duthosts (pytest fixture): list of DUTs
         prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority)
-        reboot_type (str): reboot type to be issued on the DUT
         trigger_pfcwd (bool): if PFC watchdog is expected to be triggered
 
     Returns:
@@ -150,8 +147,8 @@ def test_pfcwd_basic_single_lossless_prio_reboot(snappi_api,                # no
 
     testbed_config, port_config_list, snappi_ports = setup_ports_and_dut
 
-    lossless_prio = random.sample(lossless_prio_list, 1)
-    lossless_prio = int(lossless_prio[0])
+    _, lossless_prio = enum_dut_lossless_prio_with_completeness_level.split('|')
+    lossless_prio = int(lossless_prio)
     snappi_extra_params = SnappiTestParams()
     snappi_extra_params.multi_dut_params.multi_dut_ports = snappi_ports
 
@@ -174,11 +171,10 @@ def test_pfcwd_basic_multi_lossless_prio_reboot(snappi_api,                 # no
                                                 fanout_graph_facts_multidut,         # noqa F811
                                                 localhost,
                                                 duthosts,
-                                                lossless_prio_list,    # noqa: F811
-                                                tbinfo,                # noqa: F811
-                                                prio_dscp_map,         # noqa F811
-                                                setup_ports_and_dut,   # noqa: F811
-                                                reboot_duts,           # noqa: F811
+                                                enum_dut_lossless_prio_with_completeness_level,   # noqa: F811
+                                                tbinfo,      # noqa: F811
+                                                prio_dscp_map,              # noqa F811
+                                                reboot_duts,                # noqa: F811
                                                 trigger_pfcwd):
     """
     Verify PFC watchdog basic test works on multiple lossless priorities after various kinds of reboots
@@ -191,7 +187,6 @@ def test_pfcwd_basic_multi_lossless_prio_reboot(snappi_api,                 # no
         duthosts (pytest fixture): list of DUTs
         lossless_prio_list (pytest fixture): list of all the lossless priorities
         prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority)
-        reboot_type (str): reboot type to be issued on the DUT
         trigger_pfcwd (bool): if PFC watchdog is expected to be triggered
 
     Returns: