Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[GCU] [MA] Adding support in existing tests - PFCWD #15183

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

[GCU] [MA] Adding support in existing tests - PFCWD #15183

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a215f52
GCU Adding multi-asic support in existing test suites pfcwd
okaravasi Oct 25, 2024
fc17a52
Fixing pre-commit errors
okaravasi Oct 25, 2024
fa289ec
Merge remote-tracking branch 'origin/master' into master-gcu_t2_suppo…
okaravasi Nov 29, 2024
893ccaa
Update ASIC namespace fixture following changes in dependent PR #15182
okaravasi Nov 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions tests/common/helpers/dut_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -295,18 +295,23 @@ def verify_features_state(duthost):
return True


def verify_orchagent_running_or_assert(duthost):
def verify_orchagent_running_or_assert(duthost, asic_id=None):
"""
Verifies that orchagent is running, asserts otherwise

Args:
duthost: Device Under Test (DUT)
asic_id: Asic ID to verify. If None verifies for all asics that duthost contains.
"""

def _orchagent_running():
cmds = 'docker exec swss supervisorctl status orchagent'
output = duthost.shell(cmds, module_ignore_errors=True)
pytest_assert(not output['rc'], "Unable to check orchagent status output")
asic_ids = duthost.get_asic_ids() if asic_id is None else [asic_id]
for asic in asic_ids:
cmd = 'docker exec swss supervisorctl status orchagent'
if asic is not None:
cmd = 'docker exec swss{} supervisorctl status orchagent'.format(asic)
output = duthost.shell(cmd, module_ignore_errors=True)
pytest_assert(not output['rc'], "Unable to check orchagent status output for asic_id {}".format(asic))
return 'RUNNING' in output['stdout']

pytest_assert(
Expand Down
63 changes: 43 additions & 20 deletions tests/generic_config_updater/test_pfcwd_interval.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,22 +47,35 @@ def enable_default_pfcwd_configuration(duthost):
pfc_status = meta_data["DEVICE_METADATA|localhost"]["value"].get("default_pfcwd_status", "")
if pfc_status == 'disable':
duthost.shell('redis-cli -n 4 hset \"DEVICE_METADATA|localhost\" default_pfcwd_status enable')
# apply cofig to all namespaces also for multi-asic platforms
for asic_id in duthost.get_asic_ids():
if asic_id:
duthost.asic_instance(asic_id).command(
'redis-cli -n 4 hset \"DEVICE_METADATA|localhost\" default_pfcwd_status enable'
)
# Enable default pfcwd configuration
start_pfcwd = duthost.shell('config pfcwd start_default')
pytest_assert(not start_pfcwd['rc'], "Failed to start default pfcwd config")
# apply cofig to all namespaces also for multi-asic platforms
for asic_id in duthost.get_asic_ids():
if asic_id:
start_pfcwd = duthost.asic_instance(asic_id).command('config pfcwd start_default')
pytest_assert(not start_pfcwd['rc'], "Failed to start default pfcwd config for asic")


def ensure_application_of_updated_config(duthost, value):
def ensure_application_of_updated_config(duthost, value, namespace=None):
"""
Ensures application of the JSON patch config update by verifying field value presence in FLEX COUNTER DB

Args:
duthost: DUT host object
value: expected value of POLL_INTERVAL
namespace: Namespace to run the command in. Ex. asic0, asic1, None
"""
def _confirm_value_in_flex_counter_db():
poll_interval = duthost.shell(
'sonic-db-cli PFC_WD_DB hget FLEX_COUNTER_GROUP_TABLE:PFC_WD POLL_INTERVAL')["stdout"]
ns_flag_prefix = '' if namespace is None else '-n ' + namespace
cmd = 'sonic-db-cli {} PFC_WD_DB hget FLEX_COUNTER_GROUP_TABLE:PFC_WD POLL_INTERVAL'.format(ns_flag_prefix)
poll_interval = duthost.shell(cmd)["stdout"]
return value == poll_interval

pytest_assert(
Expand All @@ -71,49 +84,56 @@ def _confirm_value_in_flex_counter_db():
)


def prepare_pfcwd_interval_config(duthost, value):
def prepare_pfcwd_interval_config(duthost, value, namespace=None):
"""
Prepares config db by setting pfcwd poll interval to specified value.
If value is empty string, delete the current entry.

Args:
duthost: DUT host object
value: poll interval value to be set
namespace: Namespace to run the command in. Ex. asic0, asic1, None
"""

netns_exec_prefix = '' if namespace is None else 'sudo ip netns exec {}'.format(namespace)
ns_flag_prefix = '' if namespace is None else '-n ' + namespace
logger.info("Setting configdb entry pfcwd poll interval to value: {}".format(value))

if value:
cmd = "pfcwd interval {}".format(value)
cmd = "{} pfcwd interval {}".format(netns_exec_prefix, value)
else:
cmd = r"sonic-db-cli CONFIG_DB del \PFC_WD\GLOBAL\POLL_INTERVAL"
cmd = r"sonic-db-cli {} CONFIG_DB del \PFC_WD\GLOBAL\POLL_INTERVAL".format(ns_flag_prefix)

duthost.shell(cmd)


def get_detection_restoration_times(duthost):
def get_detection_restoration_times(duthost, namespace=None):
"""
Returns detection_time, restoration_time for an interface.
Poll_interval must be greater than both in order to be valid

Args:
duthost: DUT host object
namespace: Namespace to run the command in. Ex. asic0, asic1, None
"""

duthost.shell('config pfcwd start --action drop all 400 --restoration-time 400', module_ignore_errors=True)
ns_flag_prefix = '' if namespace is None else '-n ' + namespace
netns_exec_prefix = '' if namespace is None else 'sudo ip netns exec {}'.format(namespace)
cmd = '{} config pfcwd start --action drop all 400 --restoration-time 400'.format(netns_exec_prefix)
duthost.shell(cmd, module_ignore_errors=True)
pfcwd_config = duthost.shell("show pfcwd config")
pytest_assert(not pfcwd_config['rc'], "Unable to read pfcwd config")

for line in pfcwd_config['stdout_lines']:
if line.startswith('Ethernet'):
interface = line.split()[0] # Since line starts with Ethernet, we can safely use 0 index

cmd = "sonic-db-cli CONFIG_DB hget \"PFC_WD|{}\" \"detection_time\" ".format(interface)
cmd = "sonic-db-cli {} CONFIG_DB hget \"PFC_WD|{}\" \"detection_time\" ".format(
ns_flag_prefix, interface)
output = duthost.shell(cmd, module_ignore_errors=True)
pytest_assert(not output['rc'], "Unable to read detection time")
detection_time = output["stdout"]

cmd = "sonic-db-cli CONFIG_DB hget \"PFC_WD|{}\" \"restoration_time\" ".format(interface)
cmd = "sonic-db-cli {} CONFIG_DB hget \"PFC_WD|{}\" \"restoration_time\" ".format(
ns_flag_prefix, interface)
output = duthost.shell(cmd, module_ignore_errors=True)
pytest_assert(not output['rc'], "Unable to read restoration time")
restoration_time = output["stdout"]
Expand All @@ -123,17 +143,18 @@ def get_detection_restoration_times(duthost):
pytest_assert(True, "Failed to read detection_time and/or restoration time")


def get_new_interval(duthost, is_valid):
def get_new_interval(duthost, is_valid, namespace=None):
"""
Returns new interval value for pfcwd poll interval, based on the operation being performed

Args:
duthost: DUT host object
is_valid: if is_valid is true, return a valid new interval. Config update should succeed.
If is_valid is false, return an invalid new interval. Config update should fail.
namespace: Namespace to run the command in. Ex. asic0, asic1, None
"""

detection_time, restoration_time = get_detection_restoration_times(duthost)
detection_time, restoration_time = get_detection_restoration_times(duthost, namespace)
if is_valid:
return max(detection_time, restoration_time) - 10
else:
Expand All @@ -144,25 +165,27 @@ def get_new_interval(duthost, is_valid):
@pytest.mark.parametrize("field_pre_status", ["existing", "nonexistent"])
@pytest.mark.parametrize("is_valid_config_update", [True, False])
def test_pfcwd_interval_config_updates(duthost, ensure_dut_readiness, oper,
field_pre_status, is_valid_config_update):
new_interval = get_new_interval(duthost, is_valid_config_update)
field_pre_status, is_valid_config_update, rand_asic_namespace):
asic_namespace, _asic_id = rand_asic_namespace
new_interval = get_new_interval(duthost, is_valid_config_update, asic_namespace)

operation_to_new_value_map = {"add": "{}".format(new_interval), "replace": "{}".format(new_interval)}
detection_time, restoration_time = get_detection_restoration_times(duthost)
detection_time, restoration_time = get_detection_restoration_times(duthost, asic_namespace)
pre_status = max(detection_time, restoration_time)
field_pre_status_to_value_map = {"existing": "{}".format(pre_status), "nonexistent": ""}

prepare_pfcwd_interval_config(duthost, field_pre_status_to_value_map[field_pre_status])
prepare_pfcwd_interval_config(duthost, field_pre_status_to_value_map[field_pre_status], asic_namespace)

tmpfile = generate_tmpfile(duthost)
logger.info("tmpfile {} created for json patch of pfcwd poll interval and operation: {}".format(tmpfile, oper))
value = operation_to_new_value_map[oper]
logger.info("value to be added to json patch: {}".format(value))

json_namespace = '' if asic_namespace is None else '/' + asic_namespace
json_patch = [
{
"op": "{}".format(oper),
"path": "/PFC_WD/GLOBAL/POLL_INTERVAL",
"path": "{}/PFC_WD/GLOBAL/POLL_INTERVAL".format(json_namespace),
"value": "{}".format(value)
}]
json_patch = format_json_patch_for_multiasic(duthost=duthost, json_data=json_patch)
Expand All @@ -172,7 +195,7 @@ def test_pfcwd_interval_config_updates(duthost, ensure_dut_readiness, oper,

if is_valid_config_update and is_valid_platform_and_version(duthost, "PFC_WD", "PFCWD enable/disable", oper):
expect_op_success(duthost, output)
ensure_application_of_updated_config(duthost, value)
ensure_application_of_updated_config(duthost, value, asic_namespace)
else:
expect_op_failure(output)
finally:
Expand Down
80 changes: 61 additions & 19 deletions tests/generic_config_updater/test_pfcwd_status.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,14 +60,24 @@ def set_default_pfcwd_config(duthost):
meta_data = json.loads(res["stdout"])
pfc_status = meta_data["DEVICE_METADATA|localhost"]["value"].get("default_pfcwd_status", "")
if pfc_status == 'disable':
duthost.shell('sonic-db-cli CONFIG_DB hset \"DEVICE_METADATA|localhost\" default_pfcwd_status enable')
cmd = 'sonic-db-cli CONFIG_DB hset \"DEVICE_METADATA|localhost\" default_pfcwd_status enable'
for asic_id in duthost.get_asic_ids():
if asic_id:
duthost.asic_instance(asic_id).command(cmd)
else:
duthost.shell(cmd)

yield

# Restore default config
duthost.shell('config pfcwd stop')
if pfc_status == 'disable':
duthost.shell('sonic-db-cli CONFIG_DB hset \"DEVICE_METADATA|localhost\" default_pfcwd_status disable')
cmd = 'sonic-db-cli CONFIG_DB hset \"DEVICE_METADATA|localhost\" default_pfcwd_status disable'
for asic_id in duthost.get_asic_ids():
if asic_id:
duthost.asic_instance(asic_id).command(cmd)
else:
duthost.shell(cmd)
else:
start_pfcwd = duthost.shell('config pfcwd start_default')
pytest_assert(not start_pfcwd['rc'], "Failed to start default pfcwd config")
Expand Down Expand Up @@ -106,7 +116,12 @@ def stop_pfcwd(duthost):
Args:
duthost: DUT host object
"""
duthost.shell('config pfcwd stop')
cmd = 'config pfcwd stop'
for asic_id in duthost.get_asic_ids():
if asic_id:
duthost.asic_instance(asic_id).command(cmd)
else:
duthost.shell(cmd)
yield


Expand All @@ -118,7 +133,12 @@ def start_pfcwd(duthost):
Args:
duthost: DUT host object
"""
duthost.shell('config pfcwd start_default')
cmd = 'config pfcwd start_default'
for asic_id in duthost.get_asic_ids():
if asic_id:
duthost.asic_instance(asic_id).command(cmd)
else:
duthost.shell(cmd)
yield


Expand Down Expand Up @@ -148,53 +168,61 @@ def extract_pfcwd_config(duthost, start_pfcwd):
yield pfcwd_config


def get_flex_db_count(duthost):
def get_flex_db_count(duthost, namespace=None):
"""
Get the count of the number of pfcwd entries seen in flex db
For every port, there will be 3 entries - 1 for the port, 1 for queue 3 and 1 for queue 4
Args:
duthost: DUT host object
namespace: namespace to be used for the command

Returns:
Number of PFCWD related flex db entries
"""
db_entries = duthost.shell('sonic-db-cli FLEX_COUNTER_DB keys *FLEX_COUNTER_TABLE:PFC_WD*')["stdout"]
ns_flag_prefix = '' if namespace is None else '-n ' + namespace
cmd = 'sonic-db-cli {} FLEX_COUNTER_DB keys *FLEX_COUNTER_TABLE:PFC_WD*'.format(ns_flag_prefix)
db_entries = duthost.shell(cmd)["stdout"]
if db_entries == '':
return 0
else:
return len(db_entries.split('\n'))


def check_config_update(duthost, expected_count):
def check_config_update(duthost, expected_count, namespace=None):
"""
Ensures application of the JSON patch config update

Args:
duthost: DUT host object
expected_count: number of pfcwd entries expected in the updated config
namespace: namespace to be used for the command
"""
def _confirm_value_in_flex_db(duthost, expected_count):
pfcwd_entries_count = get_flex_db_count(duthost)
def _confirm_value_in_flex_db():
pfcwd_entries_count = get_flex_db_count(duthost, namespace)
logger.info("Actual number of entries: {}".format(pfcwd_entries_count))
return pfcwd_entries_count == expected_count

logger.info("Validating in FLEX COUNTER DB...")
pytest_assert(
wait_until(READ_FLEXDB_TIMEOUT, READ_FLEXDB_INTERVAL, 0, _confirm_value_in_flex_db, duthost, expected_count),
"FLEX DB does not properly reflect Pfcwd status: Expected number of entries {}"
.format(expected_count)
)
wait_until(
READ_FLEXDB_TIMEOUT,
READ_FLEXDB_INTERVAL,
0,
_confirm_value_in_flex_db
), "FLEX DB does not properly reflect Pfcwd status: Expected number of entries {}".format(expected_count))


@pytest.mark.parametrize('port', ['single', 'all'])
def test_stop_pfcwd(duthost, extract_pfcwd_config, ensure_dut_readiness, port):
def test_stop_pfcwd(duthost, rand_asic_namespace,
extract_pfcwd_config, ensure_dut_readiness, port):
"""
Tests GCU config for pfcwd stop scenario
1. Covers the case for stopping pfcwd on single port and all ports
2. Verifies that the config is as expected in CONFIG DB
3. Validates the number of PFC_WD related entries in FLEX DB is as expected
4. Validates that orchagent is running fine pre and post test
"""
asic_namespace, _asic_id = rand_asic_namespace
pfcwd_config = extract_pfcwd_config
initial_count = len(pfcwd_config) * FLEXDB_COUNTERS_PER_PORT

Expand All @@ -205,10 +233,16 @@ def test_stop_pfcwd(duthost, extract_pfcwd_config, ensure_dut_readiness, port):
json_patch = list()
exp_str = 'Ethernet'
for interface in pfcwd_config:
asic_index = None
json_namespace = ''
if duthost.is_multi_asic:
asic_index = duthost.get_port_asic_instance(interface).asic_index
ns = duthost.get_namespace_from_asic_id(asic_index)
json_namespace = '/' + ns
json_patch.extend([
{
'op': 'remove',
'path': '/PFC_WD/{}'.format(interface)
'path': '{}/PFC_WD/{}'.format(json_namespace, interface)
}])
if port == 'single':
exp_str = interface
Expand All @@ -223,20 +257,22 @@ def test_stop_pfcwd(duthost, extract_pfcwd_config, ensure_dut_readiness, port):
pytest_assert(not pfcwd_updated_config['rc'], "Unable to read updated pfcwd config")
pytest_assert(exp_str not in pfcwd_updated_config['stdout'].split(),
"pfcwd unexpectedly still running")
check_config_update(duthost, expected_count)
check_config_update(duthost, expected_count, asic_namespace)
finally:
delete_tmpfile(duthost, tmpfile)


@pytest.mark.parametrize('port', ['single', 'all'])
def test_start_pfcwd(duthost, extract_pfcwd_config, ensure_dut_readiness, stop_pfcwd, port):
def test_start_pfcwd(duthost, rand_asic_namespace,
extract_pfcwd_config, ensure_dut_readiness, stop_pfcwd, port):
"""
Tests GCU config for pfcwd start scenario
1. Covers the case for starting pfcwd on single port and all ports
2. Verifies that the config is as expected in CONFIG DB
3. Validates the number of PFC_WD related entries in FLEX DB is as expected
4. Validates that orchagent is running fine pre and post test
"""
asic_namespace, _asic_id = rand_asic_namespace
pfcwd_config = extract_pfcwd_config

if port == 'single':
Expand All @@ -247,10 +283,16 @@ def test_start_pfcwd(duthost, extract_pfcwd_config, ensure_dut_readiness, stop_p
exp_str = 'Ethernet'
op = 'add'
for interface, value in pfcwd_config.items():
asic_index = None
json_namespace = ''
if duthost.is_multi_asic:
asic_index = duthost.get_port_asic_instance(interface).asic_index
ns = duthost.get_namespace_from_asic_id(asic_index)
json_namespace = '/' + ns
json_patch.extend([
{
'op': op,
'path': '/PFC_WD/{}'.format(interface),
'path': '{}/PFC_WD/{}'.format(json_namespace, interface),
'value': {'action': value['action'],
'detection_time': value['detect_time'],
'restoration_time': value['restore_time']}}])
Expand All @@ -268,7 +310,7 @@ def test_start_pfcwd(duthost, extract_pfcwd_config, ensure_dut_readiness, stop_p
pytest_assert(not pfcwd_updated_config['rc'], "Unable to read updated pfcwd config")
pytest_assert(exp_str in pfcwd_updated_config['stdout'],
"pfcwd not started - unexpected")
check_config_update(duthost, expected_count)
check_config_update(duthost, expected_count, asic_namespace)
else:
expect_op_failure(output)
finally:
Expand Down
Loading