[Snappi] Change multidut reboot cases to use enum with completeness_l…

…evel (#15057)

Description of PR
Summary:
Fixes #14639

After the change:

module	running time(T0/T1)
before fix (mins)	running time(T0/T1)
after fix (mins)
test_multidut_pfcwd_basic_with_snappi.py	167.13	112.48
test_multidut_pfc_pause_lossy_with_snappi.py	295.25	86.53
test_multidut_pfc_pause_lossless_with_snappi.py	140.87	65.70

Approach
What is the motivation for this PR?
Current snappi multidut test cases took too long to run.

How did you do it?
Use rand_lossless_prio/rand_lossy_prio instead of enum_dut_lossless_prio/enum_dut_lossy_prio in the reboot test cases. This is the same method used in the existing T0/T1 snappi test cases.
Add skip warm reboot check in test_multidut_pfcwd_basic_with_snappi.py
Use set() to avoid rebooting same DUT several times.
How did you verify/test it?
Verified on physical testbed.

co-authorized by: [email protected]

tests/conftest.py

@@ -1472,7 +1472,7 @@ def generate_dut_backend_asics(request, duts_selected):
     return dut_asic_list
 
 
-def generate_priority_lists(request, prio_scope):
+def generate_priority_lists(request, prio_scope, with_completeness_level=False):
     empty = []
 
     tbname = request.config.getoption("--testbed")
@@ -1498,6 +1498,22 @@ def generate_priority_lists(request, prio_scope):
         for p in priorities:
             ret.append('{}|{}'.format(dut, p))
 
+    if with_completeness_level:
+        completeness_level = get_completeness_level_metadata(request)
+        # if completeness_level in ["debug", "basic", "confident"],
+        # select a small subnet to save test time
+        # if completeness_level in ["debug"], only select one item
+        # if completeness_level in ["basic", "confident"], select 1 priority per DUT
+
+        if completeness_level in ["debug"]:
+            ret = random.sample(ret, 1)
+        elif completeness_level in ["basic", "confident"]:
+            ret = []
+            for dut, priorities in list(dut_prio.items()):
+                if priorities:
+                    p = random.choice(priorities)
+                    ret.append('{}|{}'.format(dut, p))
+
     return ret if ret else empty
 
 
@@ -1697,8 +1713,14 @@ def format_portautoneg_test_id(param):
         metafunc.parametrize("enum_dut_all_prio", generate_priority_lists(metafunc, 'all'))
     if 'enum_dut_lossless_prio' in metafunc.fixturenames:
         metafunc.parametrize("enum_dut_lossless_prio", generate_priority_lists(metafunc, 'lossless'))
+    if 'enum_dut_lossless_prio_with_completeness_level' in metafunc.fixturenames:
+        metafunc.parametrize("enum_dut_lossless_prio_with_completeness_level",
+                             generate_priority_lists(metafunc, 'lossless', with_completeness_level=True))
     if 'enum_dut_lossy_prio' in metafunc.fixturenames:
         metafunc.parametrize("enum_dut_lossy_prio", generate_priority_lists(metafunc, 'lossy'))
+    if 'enum_dut_lossy_prio_with_completeness_level' in metafunc.fixturenames:
+        metafunc.parametrize("enum_dut_lossy_prio_with_completeness_level",
+                             generate_priority_lists(metafunc, 'lossy', with_completeness_level=True))
     if 'enum_pfc_pause_delay_test_params' in metafunc.fixturenames:
         metafunc.parametrize("enum_pfc_pause_delay_test_params", pfc_pause_delay_test_params(metafunc))
 

tests/snappi_tests/multidut/pfc/test_multidut_pfc_pause_lossless_with_snappi.py

@@ -191,7 +191,7 @@ def test_pfc_pause_single_lossless_prio_reboot(snappi_api,                  # no
                                                fanout_graph_facts_multidut,          # noqa: F811
                                                duthosts,
                                                localhost,
-                                               enum_dut_lossless_prio,    # noqa: F811
+                                               enum_dut_lossless_prio_with_completeness_level,    # noqa: F811
                                                prio_dscp_map,            # noqa: F811
                                                lossless_prio_list,         # noqa: F811
                                                all_prio_list,        # noqa: F811
@@ -210,7 +210,7 @@ def test_pfc_pause_single_lossless_prio_reboot(snappi_api,                  # no
         localhost (pytest fixture): localhost handle
         all_prio_list (pytest fixture): list of all the priorities
         prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority).
-        lossless_prio_list (pytest fixture): list of all the lossless priorities
+        enum_dut_lossless_prio_with_completeness_level (str): lossless priority to test, e.g., 's6100-1|3'
         reboot_type (str): reboot type to be issued on the DUT
         tbinfo (pytest fixture): fixture provides information about testbed
         get_snappi_ports (pytest fixture): gets snappi ports and connected DUT port info and returns as a list
@@ -246,7 +246,7 @@ def test_pfc_pause_single_lossless_prio_reboot(snappi_api,                  # no
     skip_warm_reboot(snappi_ports[0]['duthost'], reboot_type)
     skip_warm_reboot(snappi_ports[1]['duthost'], reboot_type)
 
-    _, lossless_prio = enum_dut_lossless_prio.split('|')
+    _, lossless_prio = enum_dut_lossless_prio_with_completeness_level.split('|')
     lossless_prio = int(lossless_prio)
     pause_prio_list = [lossless_prio]
     test_prio_list = [lossless_prio]
@@ -257,7 +257,7 @@ def test_pfc_pause_single_lossless_prio_reboot(snappi_api,                  # no
     snappi_extra_params = SnappiTestParams()
     snappi_extra_params.multi_dut_params.multi_dut_ports = snappi_ports
 
-    for duthost in [snappi_ports[0]['duthost'], snappi_ports[1]['duthost']]:
+    for duthost in set([snappi_ports[0]['duthost'], snappi_ports[1]['duthost']]):
         logger.info("Issuing a {} reboot on the dut {}".format(reboot_type, duthost.hostname))
         reboot(duthost, localhost, reboot_type=reboot_type, safe_reboot=True)
         logger.info("Wait until the system is stable")
@@ -347,7 +347,7 @@ def test_pfc_pause_multi_lossless_prio_reboot(snappi_api,                  # noq
     snappi_extra_params = SnappiTestParams()
     snappi_extra_params.multi_dut_params.multi_dut_ports = snappi_ports
 
-    for duthost in [snappi_ports[0]['duthost'], snappi_ports[1]['duthost']]:
+    for duthost in set([snappi_ports[0]['duthost'], snappi_ports[1]['duthost']]):
         logger.info("Issuing a {} reboot on the dut {}".format(reboot_type, duthost.hostname))
         reboot(duthost, localhost, reboot_type=reboot_type, safe_reboot=True)
         logger.info("Wait until the system is stable")

tests/snappi_tests/multidut/pfc/test_multidut_pfc_pause_lossy_with_snappi.py

@@ -185,7 +185,7 @@ def test_pfc_pause_single_lossy_prio_reboot(snappi_api,             # noqa: F811
                                             fanout_graph_facts_multidut,     # noqa: F811
                                             duthosts,
                                             localhost,
-                                            enum_dut_lossy_prio,
+                                            enum_dut_lossy_prio_with_completeness_level,
                                             prio_dscp_map,                   # noqa: F811
                                             lossy_prio_list,              # noqa: F811
                                             all_prio_list,                   # noqa: F811
@@ -202,7 +202,7 @@ def test_pfc_pause_single_lossy_prio_reboot(snappi_api,             # noqa: F811
         fanout_graph_facts (pytest fixture): fanout graph
         duthosts (pytest fixture): list of DUTs
         localhost (pytest fixture): localhost handle
-        enum_dut_lossy_prio (str): name of lossy priority to test, e.g., 's6100-1|2'
+        enum_dut_lossy_prio_with_completeness_level (str): lossy priority to test, e.g., 's6100-1|2'
         prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority).
         lossy_prio_list (pytest fixture): list of all the lossy priorities
         all_prio_list (pytest fixture): list of all the priorities
@@ -241,14 +241,14 @@ def test_pfc_pause_single_lossy_prio_reboot(snappi_api,             # noqa: F811
     skip_warm_reboot(snappi_ports[0]['duthost'], reboot_type)
     skip_warm_reboot(snappi_ports[1]['duthost'], reboot_type)
 
-    _, lossy_prio = enum_dut_lossy_prio.split('|')
+    _, lossy_prio = enum_dut_lossy_prio_with_completeness_level.split('|')
     lossy_prio = int(lossy_prio)
     pause_prio_list = [lossy_prio]
     test_prio_list = [lossy_prio]
     bg_prio_list = [p for p in all_prio_list]
     bg_prio_list.remove(lossy_prio)
 
-    for duthost in [snappi_ports[0]['duthost'], snappi_ports[1]['duthost']]:
+    for duthost in set([snappi_ports[0]['duthost'], snappi_ports[1]['duthost']]):
         logger.info("Issuing a {} reboot on the dut {}".format(reboot_type, duthost.hostname))
         reboot(duthost, localhost, reboot_type=reboot_type, safe_reboot=True)
         logger.info("Wait until the system is stable")
@@ -338,7 +338,7 @@ def test_pfc_pause_multi_lossy_prio_reboot(snappi_api,          # noqa: F811
     test_prio_list = lossy_prio_list
     bg_prio_list = lossless_prio_list
 
-    for duthost in [snappi_ports[0]['duthost'], snappi_ports[1]['duthost']]:
+    for duthost in set([snappi_ports[0]['duthost'], snappi_ports[1]['duthost']]):
         logger.info("Issuing a {} reboot on the dut {}".format(reboot_type, duthost.hostname))
         reboot(duthost, localhost, reboot_type=reboot_type, safe_reboot=True)
         logger.info("Wait until the system is stable")

tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_basic_with_snappi.py

@@ -174,7 +174,7 @@ def test_pfcwd_basic_single_lossless_prio_reboot(snappi_api,                # no
                                                  fanout_graph_facts_multidut,        # noqa F811
                                                  localhost,
                                                  duthosts,
-                                                 lossless_prio_list,   # noqa: F811
+                                                 enum_dut_lossless_prio_with_completeness_level,   # noqa: F811
                                                  get_snappi_ports,   # noqa: F811
                                                  tbinfo,      # noqa: F811
                                                  multidut_port_info,
@@ -190,6 +190,7 @@ def test_pfcwd_basic_single_lossless_prio_reboot(snappi_api,                # no
         fanout_graph_facts_multidut (pytest fixture): fanout graph
         localhost (pytest fixture): localhost handle
         duthosts (pytest fixture): list of DUTs
+        enum_dut_lossless_prio_with_completeness_level (str): lossless priority to test, e.g., 's6100-1|3'
         prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority)
         reboot_type (str): reboot type to be issued on the DUT
         trigger_pfcwd (bool): if PFC watchdog is expected to be triggered
@@ -223,13 +224,15 @@ def test_pfcwd_basic_single_lossless_prio_reboot(snappi_api,                # no
         testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(duthosts,
                                                                                 snappi_ports,
                                                                                 snappi_api)
+    skip_warm_reboot(snappi_ports[0]['duthost'], reboot_type)
+    skip_warm_reboot(snappi_ports[1]['duthost'], reboot_type)
 
-    lossless_prio = random.sample(lossless_prio_list, 1)
-    lossless_prio = int(lossless_prio[0])
+    _, lossless_prio = enum_dut_lossless_prio_with_completeness_level.split('|')
+    lossless_prio = int(lossless_prio)
     snappi_extra_params = SnappiTestParams()
     snappi_extra_params.multi_dut_params.multi_dut_ports = snappi_ports
 
-    for duthost in [snappi_ports[0]['duthost'], snappi_ports[1]['duthost']]:
+    for duthost in set([snappi_ports[0]['duthost'], snappi_ports[1]['duthost']]):
         logger.info("Issuing a {} reboot on the dut {}".format(reboot_type, duthost.hostname))
         reboot(duthost, localhost, reboot_type=reboot_type, safe_reboot=True)
         logger.info("Wait until the system is stable")
@@ -309,7 +312,10 @@ def test_pfcwd_basic_multi_lossless_prio_reboot(snappi_api,                 # no
                                                                                 snappi_ports,
                                                                                 snappi_api)
 
-    for duthost in [snappi_ports[0]['duthost'], snappi_ports[1]['duthost']]:
+    skip_warm_reboot(snappi_ports[0]['duthost'], reboot_type)
+    skip_warm_reboot(snappi_ports[1]['duthost'], reboot_type)
+
+    for duthost in set([snappi_ports[0]['duthost'], snappi_ports[1]['duthost']]):
         logger.info("Issuing a {} reboot on the dut {}".format(reboot_type, duthost.hostname))
         reboot(duthost, localhost, reboot_type=reboot_type, safe_reboot=True)
         logger.info("Wait until the system is stable")