Set default --completeness_level to debug (#14375) (#15334)

What is the motivation for this PR? The completeness_level is used as the test level for stress tests, with "confident" indicating a high number of stress test iterations. In Elastictest, the completeness_level has been set to "confident," which results in longer test times. Always run stress tests at the "confident" level is unnecessary. Moving forward, we can configure the test plan to run high level stress tests weekly at a scheduled time. How did you do it? Remove --completeness_level=confident in Elastictest, allowing stress tests to run at their default iteration count. Set default --completeness_level=debug in testcases Fix typo conpleteness to completeness Co-authored-by: xwjiang-ms <[email protected]> Co-authored-by: xwjiang2021 <[email protected]>
sonic-net · Nov 6, 2024 · 335db7b · 335db7b
1 parent 24cb878
commit 335db7b
Show file tree

Hide file tree

Showing 18 changed files with 29 additions and 29 deletions.
diff --git a/.azure-pipelines/test_plan.py b/.azure-pipelines/test_plan.py
@@ -221,7 +221,7 @@ def create(self, topology, test_plan_name="my_test_plan", deploy_mg_extra_params
         print("Test scripts to be covered in this test plan:")
         print(json.dumps(scripts, indent=4))
 
-        common_extra_params = common_extra_params + " --completeness_level=confident --allow_recover"
+        common_extra_params = common_extra_params + " --allow_recover"
 
         # Add topo and device type args for PR test
         if test_plan_type == "PR":

diff --git a/tests/acl/conftest.py b/tests/acl/conftest.py
@@ -2,5 +2,5 @@
 
 
 @pytest.fixture(scope='module')
-def get_function_conpleteness_level(pytestconfig):
+def get_function_completeness_level(pytestconfig):
     return pytestconfig.getoption("--completeness_level")
diff --git a/tests/acl/test_stress_acl.py b/tests/acl/test_stress_acl.py
@@ -165,7 +165,7 @@ def acl_rule_loaded(rand_selected_dut, acl_rule_list):
 
 
 def test_acl_add_del_stress(rand_selected_dut, tbinfo, ptfadapter, prepare_test_file,
-                            prepare_test_port, get_function_conpleteness_level,
+                            prepare_test_port, get_function_completeness_level,
                             toggle_all_simulator_ports_to_rand_selected_tor):   # noqa F811
 
     ptf_src_port, ptf_dst_ports, dut_port = prepare_test_port
@@ -175,9 +175,9 @@ def test_acl_add_del_stress(rand_selected_dut, tbinfo, ptfadapter, prepare_test_
     cmd_add_rules = "sonic-cfggen -j {} -w".format(STRESS_ACL_RULE_JSON_FILE)
     cmd_rm_all_rules = "acl-loader delete STRESS_ACL"
 
-    normalized_level = get_function_conpleteness_level
+    normalized_level = get_function_completeness_level
     if normalized_level is None:
-        normalized_level = 'basic'
+        normalized_level = 'debug'
     loop_times = LOOP_TIMES_LEVEL_MAP[normalized_level]
     wait_timeout = 15
 

diff --git a/tests/arp/conftest.py b/tests/arp/conftest.py
@@ -51,7 +51,7 @@ def pytest_addoption(parser):
 
 
 @pytest.fixture(scope='module')
-def get_function_conpleteness_level(pytestconfig):
+def get_function_completeness_level(pytestconfig):
     return pytestconfig.getoption("--completeness_level")
 
 

diff --git a/tests/arp/test_stress_arp.py b/tests/arp/test_stress_arp.py
@@ -85,13 +85,13 @@ def genrate_ipv4_ip():
 
 
 def test_ipv4_arp(duthost, garp_enabled, ip_and_intf_info, intfs_for_test,
-                  ptfadapter, get_function_conpleteness_level):
+                  ptfadapter, get_function_completeness_level):
     """
     Send gratuitous ARP (GARP) packet sfrom the PTF to the DUT
 
     The DUT should learn the (previously unseen) ARP info from the packet
     """
-    normalized_level = get_function_conpleteness_level
+    normalized_level = get_function_completeness_level
     if normalized_level is None:
         normalized_level = "debug"
 
@@ -174,13 +174,13 @@ def add_nd(ptfadapter, ip_and_intf_info, ptf_intf_index, nd_avaliable):
 
 
 def test_ipv6_nd(duthost, ptfhost, config_facts, tbinfo, ip_and_intf_info,
-                 ptfadapter, get_function_conpleteness_level, proxy_arp_enabled):
+                 ptfadapter, get_function_completeness_level, proxy_arp_enabled):
     _, _, ptf_intf_ipv6_addr, _, ptf_intf_index = ip_and_intf_info
     ptf_intf_ipv6_addr = increment_ipv6_addr(ptf_intf_ipv6_addr)
     pytest_require(proxy_arp_enabled, 'Proxy ARP not enabled for all VLANs')
     pytest_require(ptf_intf_ipv6_addr is not None, 'No IPv6 VLAN address configured on device')
 
-    normalized_level = get_function_conpleteness_level
+    normalized_level = get_function_completeness_level
     if normalized_level is None:
         normalized_level = "debug"
 

diff --git a/tests/bfd/test_bfd_static_route.py b/tests/bfd/test_bfd_static_route.py
@@ -350,7 +350,7 @@ def test_bfd_flap(
 
         completeness_level = get_function_completeness_level
         if completeness_level is None:
-            completeness_level = "thorough"
+            completeness_level = "debug"
 
         total_iterations = self.COMPLETENESS_TO_ITERATIONS[completeness_level]
         successful_iterations = 0  # Counter for successful iterations

diff --git a/tests/common/dualtor/dual_tor_utils.py b/tests/common/dualtor/dual_tor_utils.py
@@ -1271,7 +1271,7 @@ def _get_iface_ip(mg_facts, ifacename):
     res['target_server_ipv6'] = servers[random_server_iface]['server_ipv6'].split('/')[0]
     res['target_server_port'] = standby_tor_mg_facts['minigraph_ptf_indices'][random_server_iface]
 
-    normalize_level = get_function_completeness_level if get_function_completeness_level else 'thorough'
+    normalize_level = get_function_completeness_level if get_function_completeness_level else 'debug'
     res['completeness_level'] = normalize_level
 
     logger.debug("dualtor info is generated {}".format(res))

diff --git a/tests/conftest.py b/tests/conftest.py
@@ -1306,7 +1306,7 @@ def get_completeness_level_metadata(request):
     # if completeness_level is not set or an unknown completeness_level is set
     # return "thorough" to run all test set
     if not completeness_level or completeness_level not in ["debug", "basic", "confident", "thorough"]:
-        return "thorough"
+        return "debug"
     return completeness_level
 
 

diff --git a/tests/ecmp/inner_hashing/test_inner_hashing.py b/tests/ecmp/inner_hashing/test_inner_hashing.py
@@ -47,7 +47,7 @@ def test_inner_hashing(self, request, hash_keys, ptfhost, outer_ipver, inner_ipv
             outer_src_ip_range, outer_dst_ip_range = get_src_dst_ip_range(outer_ipver)
             inner_src_ip_range, inner_dst_ip_range = get_src_dst_ip_range(inner_ipver)
 
-            normalize_level = get_function_completeness_level if get_function_completeness_level else 'thorough'
+            normalize_level = get_function_completeness_level if get_function_completeness_level else 'debug'
 
             if normalize_level == 'thorough':
                 balancing_test_times = 120

diff --git a/tests/ecmp/inner_hashing/test_inner_hashing_lag.py b/tests/ecmp/inner_hashing/test_inner_hashing_lag.py
@@ -45,7 +45,7 @@ def test_inner_hashing(self, hash_keys, ptfhost, outer_ipver, inner_ipver, route
             outer_src_ip_range, outer_dst_ip_range = get_src_dst_ip_range(outer_ipver)
             inner_src_ip_range, inner_dst_ip_range = get_src_dst_ip_range(inner_ipver)
 
-            normalize_level = get_function_completeness_level if get_function_completeness_level else 'thorough'
+            normalize_level = get_function_completeness_level if get_function_completeness_level else 'debug'
 
             if normalize_level == 'thorough':
                 balancing_test_times = 120

diff --git a/tests/ecmp/inner_hashing/test_wr_inner_hashing.py b/tests/ecmp/inner_hashing/test_wr_inner_hashing.py
@@ -45,7 +45,7 @@ def test_inner_hashing(self, duthost, hash_keys, ptfhost, outer_ipver, inner_ipv
             outer_src_ip_range, outer_dst_ip_range = get_src_dst_ip_range(outer_ipver)
             inner_src_ip_range, inner_dst_ip_range = get_src_dst_ip_range(inner_ipver)
 
-            normalize_level = get_function_completeness_level if get_function_completeness_level else 'thorough'
+            normalize_level = get_function_completeness_level if get_function_completeness_level else 'debug'
 
             if normalize_level == 'thorough':
                 balancing_test_times = 200

diff --git a/tests/ecmp/inner_hashing/test_wr_inner_hashing_lag.py b/tests/ecmp/inner_hashing/test_wr_inner_hashing_lag.py
@@ -46,7 +46,7 @@ def test_inner_hashing(self, duthost, hash_keys, ptfhost, outer_ipver, inner_ipv
         outer_src_ip_range, outer_dst_ip_range = get_src_dst_ip_range(outer_ipver)
         inner_src_ip_range, inner_dst_ip_range = get_src_dst_ip_range(inner_ipver)
 
-        normalize_level = get_function_completeness_level if get_function_completeness_level else 'thorough'
+        normalize_level = get_function_completeness_level if get_function_completeness_level else 'debug'
 
         if normalize_level == 'thorough':
             balancing_test_times = 200

diff --git a/tests/fdb/conftest.py b/tests/fdb/conftest.py
@@ -36,7 +36,7 @@ def set_polling_interval(duthost):
 
 
 @pytest.fixture(scope='module')
-def get_function_conpleteness_level(pytestconfig, duthost):
+def get_function_completeness_level(pytestconfig, duthost):
     asic_name = duthost.get_asic_name()
     if asic_name in ['td2']:
         return None

diff --git a/tests/fdb/test_fdb_mac_move.py b/tests/fdb/test_fdb_mac_move.py
@@ -100,14 +100,14 @@ def get_fdb_dict(ptfadapter, vlan_table, dummay_mac_count):
     return fdb
 
 
-def test_fdb_mac_move(ptfadapter, duthosts, rand_one_dut_hostname, ptfhost, get_function_conpleteness_level,
+def test_fdb_mac_move(ptfadapter, duthosts, rand_one_dut_hostname, ptfhost, get_function_completeness_level,
                       fixture_rsyslog_conf_setup_teardown):
     # Perform FDB clean up before each test
     fdb_cleanup(duthosts, rand_one_dut_hostname)
 
-    normalized_level = get_function_conpleteness_level
+    normalized_level = get_function_completeness_level
     if normalized_level is None:
-        normalized_level = "basic"
+        normalized_level = "debug"
     loop_times = LOOP_TIMES_LEVEL_MAP[normalized_level]
 
     duthost = duthosts[rand_one_dut_hostname]

diff --git a/tests/route/conftest.py b/tests/route/conftest.py
@@ -16,7 +16,7 @@ def pytest_addoption(parser):
 
 
 @pytest.fixture(scope='module')
-def get_function_conpleteness_level(pytestconfig):
+def get_function_completeness_level(pytestconfig):
     return pytestconfig.getoption("--completeness_level")
 
 

diff --git a/tests/route/test_route_flap.py b/tests/route/test_route_flap.py
@@ -375,7 +375,7 @@ def get_dev_port_and_route(duthost, asichost, dst_prefix_set):
 
 
 def test_route_flap(duthosts, tbinfo, ptfhost, ptfadapter,
-                    get_function_conpleteness_level, announce_default_routes,
+                    get_function_completeness_level, announce_default_routes,
                     enum_rand_one_per_hwsku_frontend_hostname, enum_rand_one_frontend_asic_index,
                     setup_standby_ports_on_non_enum_rand_one_per_hwsku_frontend_host_m,                     # noqa F811
                     toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_frontend_host_m, loganalyzer):    # noqa F811
@@ -448,9 +448,9 @@ def test_route_flap(duthosts, tbinfo, ptfhost, ptfadapter,
     logger.info("exabgp_port = %d" % exabgp_port)
     ping_ip = route_to_ping.strip('/{}'.format(route_prefix_len))
 
-    normalized_level = get_function_conpleteness_level
+    normalized_level = get_function_completeness_level
     if normalized_level is None:
-        normalized_level = 'basic'
+        normalized_level = 'debug'
 
     loop_times = LOOP_TIMES_LEVEL_MAP[normalized_level]
 

diff --git a/tests/stress/conftest.py b/tests/stress/conftest.py
@@ -15,7 +15,7 @@
 
 
 @pytest.fixture(scope='module')
-def get_function_conpleteness_level(pytestconfig):
+def get_function_completeness_level(pytestconfig):
     return pytestconfig.getoption("--completeness_level")
 
 

diff --git a/tests/stress/test_stress_routes.py b/tests/stress/test_stress_routes.py
@@ -40,7 +40,7 @@ def announce_withdraw_routes(duthost, namespace, localhost, ptf_ip, topo_name):
     logger.info("ipv6 route used {}".format(get_crm_resource_status(duthost, "ipv6_route", "used", namespace)))
 
 
-def test_announce_withdraw_route(duthosts, localhost, tbinfo, get_function_conpleteness_level,
+def test_announce_withdraw_route(duthosts, localhost, tbinfo, get_function_completeness_level,
                                  withdraw_and_announce_existing_routes, loganalyzer,
                                  enum_rand_one_per_hwsku_frontend_hostname, enum_rand_one_frontend_asic_index):
     ptf_ip = tbinfo["ptf_ip"]
@@ -67,9 +67,9 @@ def test_announce_withdraw_route(duthosts, localhost, tbinfo, get_function_conpl
         for dut in duthosts:
             loganalyzer[dut.hostname].ignore_regex.extend(ignoreRegex)
 
-    normalized_level = get_function_conpleteness_level
+    normalized_level = get_function_completeness_level
     if normalized_level is None:
-        normalized_level = "basic"
+        normalized_level = "debug"
 
     ipv4_route_used_before, ipv6_route_used_before = withdraw_and_announce_existing_routes