From 335db7b8d98bd0f057751a41edbf9475474d7dba Mon Sep 17 00:00:00 2001 From: Chenyang Wang <49756587+cyw233@users.noreply.github.com> Date: Wed, 6 Nov 2024 14:20:46 +1100 Subject: [PATCH] Set default --completeness_level to debug (#14375) (#15334) What is the motivation for this PR? The completeness_level is used as the test level for stress tests, with "confident" indicating a high number of stress test iterations. In Elastictest, the completeness_level has been set to "confident," which results in longer test times. Always run stress tests at the "confident" level is unnecessary. Moving forward, we can configure the test plan to run high level stress tests weekly at a scheduled time. How did you do it? Remove --completeness_level=confident in Elastictest, allowing stress tests to run at their default iteration count. Set default --completeness_level=debug in testcases Fix typo conpleteness to completeness Co-authored-by: xwjiang-ms <96218837+xwjiang-ms@users.noreply.github.com> Co-authored-by: xwjiang2021 <96218837+xwjiang2021@users.noreply.github.com> --- .azure-pipelines/test_plan.py | 2 +- tests/acl/conftest.py | 2 +- tests/acl/test_stress_acl.py | 6 +++--- tests/arp/conftest.py | 2 +- tests/arp/test_stress_arp.py | 8 ++++---- tests/bfd/test_bfd_static_route.py | 2 +- tests/common/dualtor/dual_tor_utils.py | 2 +- tests/conftest.py | 2 +- tests/ecmp/inner_hashing/test_inner_hashing.py | 2 +- tests/ecmp/inner_hashing/test_inner_hashing_lag.py | 2 +- tests/ecmp/inner_hashing/test_wr_inner_hashing.py | 2 +- tests/ecmp/inner_hashing/test_wr_inner_hashing_lag.py | 2 +- tests/fdb/conftest.py | 2 +- tests/fdb/test_fdb_mac_move.py | 6 +++--- tests/route/conftest.py | 2 +- tests/route/test_route_flap.py | 6 +++--- tests/stress/conftest.py | 2 +- tests/stress/test_stress_routes.py | 6 +++--- 18 files changed, 29 insertions(+), 29 deletions(-) diff --git a/.azure-pipelines/test_plan.py b/.azure-pipelines/test_plan.py index c238ddd5504..260cdcceb07 100644 --- a/.azure-pipelines/test_plan.py +++ b/.azure-pipelines/test_plan.py @@ -221,7 +221,7 @@ def create(self, topology, test_plan_name="my_test_plan", deploy_mg_extra_params print("Test scripts to be covered in this test plan:") print(json.dumps(scripts, indent=4)) - common_extra_params = common_extra_params + " --completeness_level=confident --allow_recover" + common_extra_params = common_extra_params + " --allow_recover" # Add topo and device type args for PR test if test_plan_type == "PR": diff --git a/tests/acl/conftest.py b/tests/acl/conftest.py index 4321f78f937..49fa6cf508c 100644 --- a/tests/acl/conftest.py +++ b/tests/acl/conftest.py @@ -2,5 +2,5 @@ @pytest.fixture(scope='module') -def get_function_conpleteness_level(pytestconfig): +def get_function_completeness_level(pytestconfig): return pytestconfig.getoption("--completeness_level") diff --git a/tests/acl/test_stress_acl.py b/tests/acl/test_stress_acl.py index c1ca832cd4a..1514b28dcaa 100644 --- a/tests/acl/test_stress_acl.py +++ b/tests/acl/test_stress_acl.py @@ -165,7 +165,7 @@ def acl_rule_loaded(rand_selected_dut, acl_rule_list): def test_acl_add_del_stress(rand_selected_dut, tbinfo, ptfadapter, prepare_test_file, - prepare_test_port, get_function_conpleteness_level, + prepare_test_port, get_function_completeness_level, toggle_all_simulator_ports_to_rand_selected_tor): # noqa F811 ptf_src_port, ptf_dst_ports, dut_port = prepare_test_port @@ -175,9 +175,9 @@ def test_acl_add_del_stress(rand_selected_dut, tbinfo, ptfadapter, prepare_test_ cmd_add_rules = "sonic-cfggen -j {} -w".format(STRESS_ACL_RULE_JSON_FILE) cmd_rm_all_rules = "acl-loader delete STRESS_ACL" - normalized_level = get_function_conpleteness_level + normalized_level = get_function_completeness_level if normalized_level is None: - normalized_level = 'basic' + normalized_level = 'debug' loop_times = LOOP_TIMES_LEVEL_MAP[normalized_level] wait_timeout = 15 diff --git a/tests/arp/conftest.py b/tests/arp/conftest.py index be8354a596c..d382fddd059 100644 --- a/tests/arp/conftest.py +++ b/tests/arp/conftest.py @@ -51,7 +51,7 @@ def pytest_addoption(parser): @pytest.fixture(scope='module') -def get_function_conpleteness_level(pytestconfig): +def get_function_completeness_level(pytestconfig): return pytestconfig.getoption("--completeness_level") diff --git a/tests/arp/test_stress_arp.py b/tests/arp/test_stress_arp.py index 94f797d518f..ffd6b9dc66a 100644 --- a/tests/arp/test_stress_arp.py +++ b/tests/arp/test_stress_arp.py @@ -85,13 +85,13 @@ def genrate_ipv4_ip(): def test_ipv4_arp(duthost, garp_enabled, ip_and_intf_info, intfs_for_test, - ptfadapter, get_function_conpleteness_level): + ptfadapter, get_function_completeness_level): """ Send gratuitous ARP (GARP) packet sfrom the PTF to the DUT The DUT should learn the (previously unseen) ARP info from the packet """ - normalized_level = get_function_conpleteness_level + normalized_level = get_function_completeness_level if normalized_level is None: normalized_level = "debug" @@ -174,13 +174,13 @@ def add_nd(ptfadapter, ip_and_intf_info, ptf_intf_index, nd_avaliable): def test_ipv6_nd(duthost, ptfhost, config_facts, tbinfo, ip_and_intf_info, - ptfadapter, get_function_conpleteness_level, proxy_arp_enabled): + ptfadapter, get_function_completeness_level, proxy_arp_enabled): _, _, ptf_intf_ipv6_addr, _, ptf_intf_index = ip_and_intf_info ptf_intf_ipv6_addr = increment_ipv6_addr(ptf_intf_ipv6_addr) pytest_require(proxy_arp_enabled, 'Proxy ARP not enabled for all VLANs') pytest_require(ptf_intf_ipv6_addr is not None, 'No IPv6 VLAN address configured on device') - normalized_level = get_function_conpleteness_level + normalized_level = get_function_completeness_level if normalized_level is None: normalized_level = "debug" diff --git a/tests/bfd/test_bfd_static_route.py b/tests/bfd/test_bfd_static_route.py index 98fda6e06c5..9c48bc19e77 100644 --- a/tests/bfd/test_bfd_static_route.py +++ b/tests/bfd/test_bfd_static_route.py @@ -350,7 +350,7 @@ def test_bfd_flap( completeness_level = get_function_completeness_level if completeness_level is None: - completeness_level = "thorough" + completeness_level = "debug" total_iterations = self.COMPLETENESS_TO_ITERATIONS[completeness_level] successful_iterations = 0 # Counter for successful iterations diff --git a/tests/common/dualtor/dual_tor_utils.py b/tests/common/dualtor/dual_tor_utils.py index 3d08fcddb87..87f0cb0217d 100644 --- a/tests/common/dualtor/dual_tor_utils.py +++ b/tests/common/dualtor/dual_tor_utils.py @@ -1271,7 +1271,7 @@ def _get_iface_ip(mg_facts, ifacename): res['target_server_ipv6'] = servers[random_server_iface]['server_ipv6'].split('/')[0] res['target_server_port'] = standby_tor_mg_facts['minigraph_ptf_indices'][random_server_iface] - normalize_level = get_function_completeness_level if get_function_completeness_level else 'thorough' + normalize_level = get_function_completeness_level if get_function_completeness_level else 'debug' res['completeness_level'] = normalize_level logger.debug("dualtor info is generated {}".format(res)) diff --git a/tests/conftest.py b/tests/conftest.py index 4dae762e2b4..d92f458c3a5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1306,7 +1306,7 @@ def get_completeness_level_metadata(request): # if completeness_level is not set or an unknown completeness_level is set # return "thorough" to run all test set if not completeness_level or completeness_level not in ["debug", "basic", "confident", "thorough"]: - return "thorough" + return "debug" return completeness_level diff --git a/tests/ecmp/inner_hashing/test_inner_hashing.py b/tests/ecmp/inner_hashing/test_inner_hashing.py index 3b2f0aa1812..6f906075dd1 100644 --- a/tests/ecmp/inner_hashing/test_inner_hashing.py +++ b/tests/ecmp/inner_hashing/test_inner_hashing.py @@ -47,7 +47,7 @@ def test_inner_hashing(self, request, hash_keys, ptfhost, outer_ipver, inner_ipv outer_src_ip_range, outer_dst_ip_range = get_src_dst_ip_range(outer_ipver) inner_src_ip_range, inner_dst_ip_range = get_src_dst_ip_range(inner_ipver) - normalize_level = get_function_completeness_level if get_function_completeness_level else 'thorough' + normalize_level = get_function_completeness_level if get_function_completeness_level else 'debug' if normalize_level == 'thorough': balancing_test_times = 120 diff --git a/tests/ecmp/inner_hashing/test_inner_hashing_lag.py b/tests/ecmp/inner_hashing/test_inner_hashing_lag.py index 39df82b3983..a16ab2d7ec9 100644 --- a/tests/ecmp/inner_hashing/test_inner_hashing_lag.py +++ b/tests/ecmp/inner_hashing/test_inner_hashing_lag.py @@ -45,7 +45,7 @@ def test_inner_hashing(self, hash_keys, ptfhost, outer_ipver, inner_ipver, route outer_src_ip_range, outer_dst_ip_range = get_src_dst_ip_range(outer_ipver) inner_src_ip_range, inner_dst_ip_range = get_src_dst_ip_range(inner_ipver) - normalize_level = get_function_completeness_level if get_function_completeness_level else 'thorough' + normalize_level = get_function_completeness_level if get_function_completeness_level else 'debug' if normalize_level == 'thorough': balancing_test_times = 120 diff --git a/tests/ecmp/inner_hashing/test_wr_inner_hashing.py b/tests/ecmp/inner_hashing/test_wr_inner_hashing.py index 3e090c3f75e..635f0f4e1b9 100644 --- a/tests/ecmp/inner_hashing/test_wr_inner_hashing.py +++ b/tests/ecmp/inner_hashing/test_wr_inner_hashing.py @@ -45,7 +45,7 @@ def test_inner_hashing(self, duthost, hash_keys, ptfhost, outer_ipver, inner_ipv outer_src_ip_range, outer_dst_ip_range = get_src_dst_ip_range(outer_ipver) inner_src_ip_range, inner_dst_ip_range = get_src_dst_ip_range(inner_ipver) - normalize_level = get_function_completeness_level if get_function_completeness_level else 'thorough' + normalize_level = get_function_completeness_level if get_function_completeness_level else 'debug' if normalize_level == 'thorough': balancing_test_times = 200 diff --git a/tests/ecmp/inner_hashing/test_wr_inner_hashing_lag.py b/tests/ecmp/inner_hashing/test_wr_inner_hashing_lag.py index d34ddf6f5c2..3df177133cb 100644 --- a/tests/ecmp/inner_hashing/test_wr_inner_hashing_lag.py +++ b/tests/ecmp/inner_hashing/test_wr_inner_hashing_lag.py @@ -46,7 +46,7 @@ def test_inner_hashing(self, duthost, hash_keys, ptfhost, outer_ipver, inner_ipv outer_src_ip_range, outer_dst_ip_range = get_src_dst_ip_range(outer_ipver) inner_src_ip_range, inner_dst_ip_range = get_src_dst_ip_range(inner_ipver) - normalize_level = get_function_completeness_level if get_function_completeness_level else 'thorough' + normalize_level = get_function_completeness_level if get_function_completeness_level else 'debug' if normalize_level == 'thorough': balancing_test_times = 200 diff --git a/tests/fdb/conftest.py b/tests/fdb/conftest.py index 54eb2a4cefb..af8d5cdb3fb 100644 --- a/tests/fdb/conftest.py +++ b/tests/fdb/conftest.py @@ -36,7 +36,7 @@ def set_polling_interval(duthost): @pytest.fixture(scope='module') -def get_function_conpleteness_level(pytestconfig, duthost): +def get_function_completeness_level(pytestconfig, duthost): asic_name = duthost.get_asic_name() if asic_name in ['td2']: return None diff --git a/tests/fdb/test_fdb_mac_move.py b/tests/fdb/test_fdb_mac_move.py index dddb90fded8..2f38ccb5b97 100644 --- a/tests/fdb/test_fdb_mac_move.py +++ b/tests/fdb/test_fdb_mac_move.py @@ -100,14 +100,14 @@ def get_fdb_dict(ptfadapter, vlan_table, dummay_mac_count): return fdb -def test_fdb_mac_move(ptfadapter, duthosts, rand_one_dut_hostname, ptfhost, get_function_conpleteness_level, +def test_fdb_mac_move(ptfadapter, duthosts, rand_one_dut_hostname, ptfhost, get_function_completeness_level, fixture_rsyslog_conf_setup_teardown): # Perform FDB clean up before each test fdb_cleanup(duthosts, rand_one_dut_hostname) - normalized_level = get_function_conpleteness_level + normalized_level = get_function_completeness_level if normalized_level is None: - normalized_level = "basic" + normalized_level = "debug" loop_times = LOOP_TIMES_LEVEL_MAP[normalized_level] duthost = duthosts[rand_one_dut_hostname] diff --git a/tests/route/conftest.py b/tests/route/conftest.py index 27682c86744..4a8cd62eb28 100644 --- a/tests/route/conftest.py +++ b/tests/route/conftest.py @@ -16,7 +16,7 @@ def pytest_addoption(parser): @pytest.fixture(scope='module') -def get_function_conpleteness_level(pytestconfig): +def get_function_completeness_level(pytestconfig): return pytestconfig.getoption("--completeness_level") diff --git a/tests/route/test_route_flap.py b/tests/route/test_route_flap.py index df0693502b8..d418e85cb7c 100644 --- a/tests/route/test_route_flap.py +++ b/tests/route/test_route_flap.py @@ -375,7 +375,7 @@ def get_dev_port_and_route(duthost, asichost, dst_prefix_set): def test_route_flap(duthosts, tbinfo, ptfhost, ptfadapter, - get_function_conpleteness_level, announce_default_routes, + get_function_completeness_level, announce_default_routes, enum_rand_one_per_hwsku_frontend_hostname, enum_rand_one_frontend_asic_index, setup_standby_ports_on_non_enum_rand_one_per_hwsku_frontend_host_m, # noqa F811 toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_frontend_host_m, loganalyzer): # noqa F811 @@ -448,9 +448,9 @@ def test_route_flap(duthosts, tbinfo, ptfhost, ptfadapter, logger.info("exabgp_port = %d" % exabgp_port) ping_ip = route_to_ping.strip('/{}'.format(route_prefix_len)) - normalized_level = get_function_conpleteness_level + normalized_level = get_function_completeness_level if normalized_level is None: - normalized_level = 'basic' + normalized_level = 'debug' loop_times = LOOP_TIMES_LEVEL_MAP[normalized_level] diff --git a/tests/stress/conftest.py b/tests/stress/conftest.py index 5aea844939e..e430d452cb1 100644 --- a/tests/stress/conftest.py +++ b/tests/stress/conftest.py @@ -15,7 +15,7 @@ @pytest.fixture(scope='module') -def get_function_conpleteness_level(pytestconfig): +def get_function_completeness_level(pytestconfig): return pytestconfig.getoption("--completeness_level") diff --git a/tests/stress/test_stress_routes.py b/tests/stress/test_stress_routes.py index 0dfd5a27b4b..e5b9f9a16df 100644 --- a/tests/stress/test_stress_routes.py +++ b/tests/stress/test_stress_routes.py @@ -40,7 +40,7 @@ def announce_withdraw_routes(duthost, namespace, localhost, ptf_ip, topo_name): logger.info("ipv6 route used {}".format(get_crm_resource_status(duthost, "ipv6_route", "used", namespace))) -def test_announce_withdraw_route(duthosts, localhost, tbinfo, get_function_conpleteness_level, +def test_announce_withdraw_route(duthosts, localhost, tbinfo, get_function_completeness_level, withdraw_and_announce_existing_routes, loganalyzer, enum_rand_one_per_hwsku_frontend_hostname, enum_rand_one_frontend_asic_index): ptf_ip = tbinfo["ptf_ip"] @@ -67,9 +67,9 @@ def test_announce_withdraw_route(duthosts, localhost, tbinfo, get_function_conpl for dut in duthosts: loganalyzer[dut.hostname].ignore_regex.extend(ignoreRegex) - normalized_level = get_function_conpleteness_level + normalized_level = get_function_completeness_level if normalized_level is None: - normalized_level = "basic" + normalized_level = "debug" ipv4_route_used_before, ipv6_route_used_before = withdraw_and_announce_existing_routes