diff --git a/tests/common/helpers/dut_utils.py b/tests/common/helpers/dut_utils.py index e7307c7c4dc..093900418c5 100644 --- a/tests/common/helpers/dut_utils.py +++ b/tests/common/helpers/dut_utils.py @@ -295,18 +295,23 @@ def verify_features_state(duthost): return True -def verify_orchagent_running_or_assert(duthost): +def verify_orchagent_running_or_assert(duthost, asic_id=None): """ Verifies that orchagent is running, asserts otherwise Args: duthost: Device Under Test (DUT) + asic_id: Asic ID to verify. If None verifies for all asics that duthost contains. """ def _orchagent_running(): - cmds = 'docker exec swss supervisorctl status orchagent' - output = duthost.shell(cmds, module_ignore_errors=True) - pytest_assert(not output['rc'], "Unable to check orchagent status output") + asic_ids = duthost.get_asic_ids() if asic_id is None else [asic_id] + for asic in asic_ids: + cmd = 'docker exec swss supervisorctl status orchagent' + if asic is not None: + cmd = 'docker exec swss{} supervisorctl status orchagent'.format(asic) + output = duthost.shell(cmd, module_ignore_errors=True) + pytest_assert(not output['rc'], "Unable to check orchagent status output for asic_id {}".format(asic)) return 'RUNNING' in output['stdout'] pytest_assert( diff --git a/tests/generic_config_updater/test_pfcwd_interval.py b/tests/generic_config_updater/test_pfcwd_interval.py index 0a7e095aaef..932b22991bd 100644 --- a/tests/generic_config_updater/test_pfcwd_interval.py +++ b/tests/generic_config_updater/test_pfcwd_interval.py @@ -47,22 +47,35 @@ def enable_default_pfcwd_configuration(duthost): pfc_status = meta_data["DEVICE_METADATA|localhost"]["value"].get("default_pfcwd_status", "") if pfc_status == 'disable': duthost.shell('redis-cli -n 4 hset \"DEVICE_METADATA|localhost\" default_pfcwd_status enable') + # apply cofig to all namespaces also for multi-asic platforms + for asic_id in duthost.get_asic_ids(): + if asic_id: + duthost.asic_instance(asic_id).command( + 'redis-cli -n 4 hset \"DEVICE_METADATA|localhost\" default_pfcwd_status enable' + ) # Enable default pfcwd configuration start_pfcwd = duthost.shell('config pfcwd start_default') pytest_assert(not start_pfcwd['rc'], "Failed to start default pfcwd config") + # apply cofig to all namespaces also for multi-asic platforms + for asic_id in duthost.get_asic_ids(): + if asic_id: + start_pfcwd = duthost.asic_instance(asic_id).command('config pfcwd start_default') + pytest_assert(not start_pfcwd['rc'], "Failed to start default pfcwd config for asic") -def ensure_application_of_updated_config(duthost, value): +def ensure_application_of_updated_config(duthost, value, namespace=None): """ Ensures application of the JSON patch config update by verifying field value presence in FLEX COUNTER DB Args: duthost: DUT host object value: expected value of POLL_INTERVAL + namespace: Namespace to run the command in. Ex. asic0, asic1, None """ def _confirm_value_in_flex_counter_db(): - poll_interval = duthost.shell( - 'sonic-db-cli PFC_WD_DB hget FLEX_COUNTER_GROUP_TABLE:PFC_WD POLL_INTERVAL')["stdout"] + ns_flag_prefix = '' if namespace is None else '-n ' + namespace + cmd = 'sonic-db-cli {} PFC_WD_DB hget FLEX_COUNTER_GROUP_TABLE:PFC_WD POLL_INTERVAL'.format(ns_flag_prefix) + poll_interval = duthost.shell(cmd)["stdout"] return value == poll_interval pytest_assert( @@ -71,7 +84,7 @@ def _confirm_value_in_flex_counter_db(): ) -def prepare_pfcwd_interval_config(duthost, value): +def prepare_pfcwd_interval_config(duthost, value, namespace=None): """ Prepares config db by setting pfcwd poll interval to specified value. If value is empty string, delete the current entry. @@ -79,41 +92,48 @@ def prepare_pfcwd_interval_config(duthost, value): Args: duthost: DUT host object value: poll interval value to be set + namespace: Namespace to run the command in. Ex. asic0, asic1, None """ - + netns_exec_prefix = '' if namespace is None else 'sudo ip netns exec {}'.format(namespace) + ns_flag_prefix = '' if namespace is None else '-n ' + namespace logger.info("Setting configdb entry pfcwd poll interval to value: {}".format(value)) if value: - cmd = "pfcwd interval {}".format(value) + cmd = "{} pfcwd interval {}".format(netns_exec_prefix, value) else: - cmd = r"sonic-db-cli CONFIG_DB del \PFC_WD\GLOBAL\POLL_INTERVAL" + cmd = r"sonic-db-cli {} CONFIG_DB del \PFC_WD\GLOBAL\POLL_INTERVAL".format(ns_flag_prefix) duthost.shell(cmd) -def get_detection_restoration_times(duthost): +def get_detection_restoration_times(duthost, namespace=None): """ Returns detection_time, restoration_time for an interface. Poll_interval must be greater than both in order to be valid Args: duthost: DUT host object + namespace: Namespace to run the command in. Ex. asic0, asic1, None """ - duthost.shell('config pfcwd start --action drop all 400 --restoration-time 400', module_ignore_errors=True) + ns_flag_prefix = '' if namespace is None else '-n ' + namespace + netns_exec_prefix = '' if namespace is None else 'sudo ip netns exec {}'.format(namespace) + cmd = '{} config pfcwd start --action drop all 400 --restoration-time 400'.format(netns_exec_prefix) + duthost.shell(cmd, module_ignore_errors=True) pfcwd_config = duthost.shell("show pfcwd config") pytest_assert(not pfcwd_config['rc'], "Unable to read pfcwd config") for line in pfcwd_config['stdout_lines']: if line.startswith('Ethernet'): interface = line.split()[0] # Since line starts with Ethernet, we can safely use 0 index - - cmd = "sonic-db-cli CONFIG_DB hget \"PFC_WD|{}\" \"detection_time\" ".format(interface) + cmd = "sonic-db-cli {} CONFIG_DB hget \"PFC_WD|{}\" \"detection_time\" ".format( + ns_flag_prefix, interface) output = duthost.shell(cmd, module_ignore_errors=True) pytest_assert(not output['rc'], "Unable to read detection time") detection_time = output["stdout"] - cmd = "sonic-db-cli CONFIG_DB hget \"PFC_WD|{}\" \"restoration_time\" ".format(interface) + cmd = "sonic-db-cli {} CONFIG_DB hget \"PFC_WD|{}\" \"restoration_time\" ".format( + ns_flag_prefix, interface) output = duthost.shell(cmd, module_ignore_errors=True) pytest_assert(not output['rc'], "Unable to read restoration time") restoration_time = output["stdout"] @@ -123,7 +143,7 @@ def get_detection_restoration_times(duthost): pytest_assert(True, "Failed to read detection_time and/or restoration time") -def get_new_interval(duthost, is_valid): +def get_new_interval(duthost, is_valid, namespace=None): """ Returns new interval value for pfcwd poll interval, based on the operation being performed @@ -131,9 +151,10 @@ def get_new_interval(duthost, is_valid): duthost: DUT host object is_valid: if is_valid is true, return a valid new interval. Config update should succeed. If is_valid is false, return an invalid new interval. Config update should fail. + namespace: Namespace to run the command in. Ex. asic0, asic1, None """ - detection_time, restoration_time = get_detection_restoration_times(duthost) + detection_time, restoration_time = get_detection_restoration_times(duthost, namespace) if is_valid: return max(detection_time, restoration_time) - 10 else: @@ -144,25 +165,27 @@ def get_new_interval(duthost, is_valid): @pytest.mark.parametrize("field_pre_status", ["existing", "nonexistent"]) @pytest.mark.parametrize("is_valid_config_update", [True, False]) def test_pfcwd_interval_config_updates(duthost, ensure_dut_readiness, oper, - field_pre_status, is_valid_config_update): - new_interval = get_new_interval(duthost, is_valid_config_update) + field_pre_status, is_valid_config_update, rand_asic_namespace): + asic_namespace, _asic_id = rand_asic_namespace + new_interval = get_new_interval(duthost, is_valid_config_update, asic_namespace) operation_to_new_value_map = {"add": "{}".format(new_interval), "replace": "{}".format(new_interval)} - detection_time, restoration_time = get_detection_restoration_times(duthost) + detection_time, restoration_time = get_detection_restoration_times(duthost, asic_namespace) pre_status = max(detection_time, restoration_time) field_pre_status_to_value_map = {"existing": "{}".format(pre_status), "nonexistent": ""} - prepare_pfcwd_interval_config(duthost, field_pre_status_to_value_map[field_pre_status]) + prepare_pfcwd_interval_config(duthost, field_pre_status_to_value_map[field_pre_status], asic_namespace) tmpfile = generate_tmpfile(duthost) logger.info("tmpfile {} created for json patch of pfcwd poll interval and operation: {}".format(tmpfile, oper)) value = operation_to_new_value_map[oper] logger.info("value to be added to json patch: {}".format(value)) + json_namespace = '' if asic_namespace is None else '/' + asic_namespace json_patch = [ { "op": "{}".format(oper), - "path": "/PFC_WD/GLOBAL/POLL_INTERVAL", + "path": "{}/PFC_WD/GLOBAL/POLL_INTERVAL".format(json_namespace), "value": "{}".format(value) }] json_patch = format_json_patch_for_multiasic(duthost=duthost, json_data=json_patch) @@ -172,7 +195,7 @@ def test_pfcwd_interval_config_updates(duthost, ensure_dut_readiness, oper, if is_valid_config_update and is_valid_platform_and_version(duthost, "PFC_WD", "PFCWD enable/disable", oper): expect_op_success(duthost, output) - ensure_application_of_updated_config(duthost, value) + ensure_application_of_updated_config(duthost, value, asic_namespace) else: expect_op_failure(output) finally: diff --git a/tests/generic_config_updater/test_pfcwd_status.py b/tests/generic_config_updater/test_pfcwd_status.py index c522c800ef4..1cac8b6beab 100644 --- a/tests/generic_config_updater/test_pfcwd_status.py +++ b/tests/generic_config_updater/test_pfcwd_status.py @@ -60,14 +60,24 @@ def set_default_pfcwd_config(duthost): meta_data = json.loads(res["stdout"]) pfc_status = meta_data["DEVICE_METADATA|localhost"]["value"].get("default_pfcwd_status", "") if pfc_status == 'disable': - duthost.shell('sonic-db-cli CONFIG_DB hset \"DEVICE_METADATA|localhost\" default_pfcwd_status enable') + cmd = 'sonic-db-cli CONFIG_DB hset \"DEVICE_METADATA|localhost\" default_pfcwd_status enable' + for asic_id in duthost.get_asic_ids(): + if asic_id: + duthost.asic_instance(asic_id).command(cmd) + else: + duthost.shell(cmd) yield # Restore default config duthost.shell('config pfcwd stop') if pfc_status == 'disable': - duthost.shell('sonic-db-cli CONFIG_DB hset \"DEVICE_METADATA|localhost\" default_pfcwd_status disable') + cmd = 'sonic-db-cli CONFIG_DB hset \"DEVICE_METADATA|localhost\" default_pfcwd_status disable' + for asic_id in duthost.get_asic_ids(): + if asic_id: + duthost.asic_instance(asic_id).command(cmd) + else: + duthost.shell(cmd) else: start_pfcwd = duthost.shell('config pfcwd start_default') pytest_assert(not start_pfcwd['rc'], "Failed to start default pfcwd config") @@ -106,7 +116,12 @@ def stop_pfcwd(duthost): Args: duthost: DUT host object """ - duthost.shell('config pfcwd stop') + cmd = 'config pfcwd stop' + for asic_id in duthost.get_asic_ids(): + if asic_id: + duthost.asic_instance(asic_id).command(cmd) + else: + duthost.shell(cmd) yield @@ -118,7 +133,12 @@ def start_pfcwd(duthost): Args: duthost: DUT host object """ - duthost.shell('config pfcwd start_default') + cmd = 'config pfcwd start_default' + for asic_id in duthost.get_asic_ids(): + if asic_id: + duthost.asic_instance(asic_id).command(cmd) + else: + duthost.shell(cmd) yield @@ -148,46 +168,53 @@ def extract_pfcwd_config(duthost, start_pfcwd): yield pfcwd_config -def get_flex_db_count(duthost): +def get_flex_db_count(duthost, namespace=None): """ Get the count of the number of pfcwd entries seen in flex db For every port, there will be 3 entries - 1 for the port, 1 for queue 3 and 1 for queue 4 Args: duthost: DUT host object + namespace: namespace to be used for the command Returns: Number of PFCWD related flex db entries """ - db_entries = duthost.shell('sonic-db-cli FLEX_COUNTER_DB keys *FLEX_COUNTER_TABLE:PFC_WD*')["stdout"] + ns_flag_prefix = '' if namespace is None else '-n ' + namespace + cmd = 'sonic-db-cli {} FLEX_COUNTER_DB keys *FLEX_COUNTER_TABLE:PFC_WD*'.format(ns_flag_prefix) + db_entries = duthost.shell(cmd)["stdout"] if db_entries == '': return 0 else: return len(db_entries.split('\n')) -def check_config_update(duthost, expected_count): +def check_config_update(duthost, expected_count, namespace=None): """ Ensures application of the JSON patch config update Args: duthost: DUT host object expected_count: number of pfcwd entries expected in the updated config + namespace: namespace to be used for the command """ - def _confirm_value_in_flex_db(duthost, expected_count): - pfcwd_entries_count = get_flex_db_count(duthost) + def _confirm_value_in_flex_db(): + pfcwd_entries_count = get_flex_db_count(duthost, namespace) logger.info("Actual number of entries: {}".format(pfcwd_entries_count)) return pfcwd_entries_count == expected_count logger.info("Validating in FLEX COUNTER DB...") pytest_assert( - wait_until(READ_FLEXDB_TIMEOUT, READ_FLEXDB_INTERVAL, 0, _confirm_value_in_flex_db, duthost, expected_count), - "FLEX DB does not properly reflect Pfcwd status: Expected number of entries {}" - .format(expected_count) - ) + wait_until( + READ_FLEXDB_TIMEOUT, + READ_FLEXDB_INTERVAL, + 0, + _confirm_value_in_flex_db + ), "FLEX DB does not properly reflect Pfcwd status: Expected number of entries {}".format(expected_count)) @pytest.mark.parametrize('port', ['single', 'all']) -def test_stop_pfcwd(duthost, extract_pfcwd_config, ensure_dut_readiness, port): +def test_stop_pfcwd(duthost, rand_asic_namespace, + extract_pfcwd_config, ensure_dut_readiness, port): """ Tests GCU config for pfcwd stop scenario 1. Covers the case for stopping pfcwd on single port and all ports @@ -195,6 +222,7 @@ def test_stop_pfcwd(duthost, extract_pfcwd_config, ensure_dut_readiness, port): 3. Validates the number of PFC_WD related entries in FLEX DB is as expected 4. Validates that orchagent is running fine pre and post test """ + asic_namespace, _asic_id = rand_asic_namespace pfcwd_config = extract_pfcwd_config initial_count = len(pfcwd_config) * FLEXDB_COUNTERS_PER_PORT @@ -205,10 +233,16 @@ def test_stop_pfcwd(duthost, extract_pfcwd_config, ensure_dut_readiness, port): json_patch = list() exp_str = 'Ethernet' for interface in pfcwd_config: + asic_index = None + json_namespace = '' + if duthost.is_multi_asic: + asic_index = duthost.get_port_asic_instance(interface).asic_index + ns = duthost.get_namespace_from_asic_id(asic_index) + json_namespace = '/' + ns json_patch.extend([ { 'op': 'remove', - 'path': '/PFC_WD/{}'.format(interface) + 'path': '{}/PFC_WD/{}'.format(json_namespace, interface) }]) if port == 'single': exp_str = interface @@ -223,13 +257,14 @@ def test_stop_pfcwd(duthost, extract_pfcwd_config, ensure_dut_readiness, port): pytest_assert(not pfcwd_updated_config['rc'], "Unable to read updated pfcwd config") pytest_assert(exp_str not in pfcwd_updated_config['stdout'].split(), "pfcwd unexpectedly still running") - check_config_update(duthost, expected_count) + check_config_update(duthost, expected_count, asic_namespace) finally: delete_tmpfile(duthost, tmpfile) @pytest.mark.parametrize('port', ['single', 'all']) -def test_start_pfcwd(duthost, extract_pfcwd_config, ensure_dut_readiness, stop_pfcwd, port): +def test_start_pfcwd(duthost, rand_asic_namespace, + extract_pfcwd_config, ensure_dut_readiness, stop_pfcwd, port): """ Tests GCU config for pfcwd start scenario 1. Covers the case for starting pfcwd on single port and all ports @@ -237,6 +272,7 @@ def test_start_pfcwd(duthost, extract_pfcwd_config, ensure_dut_readiness, stop_p 3. Validates the number of PFC_WD related entries in FLEX DB is as expected 4. Validates that orchagent is running fine pre and post test """ + asic_namespace, _asic_id = rand_asic_namespace pfcwd_config = extract_pfcwd_config if port == 'single': @@ -247,10 +283,16 @@ def test_start_pfcwd(duthost, extract_pfcwd_config, ensure_dut_readiness, stop_p exp_str = 'Ethernet' op = 'add' for interface, value in pfcwd_config.items(): + asic_index = None + json_namespace = '' + if duthost.is_multi_asic: + asic_index = duthost.get_port_asic_instance(interface).asic_index + ns = duthost.get_namespace_from_asic_id(asic_index) + json_namespace = '/' + ns json_patch.extend([ { 'op': op, - 'path': '/PFC_WD/{}'.format(interface), + 'path': '{}/PFC_WD/{}'.format(json_namespace, interface), 'value': {'action': value['action'], 'detection_time': value['detect_time'], 'restoration_time': value['restore_time']}}]) @@ -268,7 +310,7 @@ def test_start_pfcwd(duthost, extract_pfcwd_config, ensure_dut_readiness, stop_p pytest_assert(not pfcwd_updated_config['rc'], "Unable to read updated pfcwd config") pytest_assert(exp_str in pfcwd_updated_config['stdout'], "pfcwd not started - unexpected") - check_config_update(duthost, expected_count) + check_config_update(duthost, expected_count, asic_namespace) else: expect_op_failure(output) finally: