CLI support for SmartSwitch PMON

config/chassis_modules.py

@@ -30,8 +30,10 @@ def shutdown_chassis_module(db, chassis_module_name):
 
     if not chassis_module_name.startswith("SUPERVISOR") and \
        not chassis_module_name.startswith("LINE-CARD") and \
-       not chassis_module_name.startswith("FABRIC-CARD"):
-        ctx.fail("'module_name' has to begin with 'SUPERVISOR', 'LINE-CARD' or 'FABRIC-CARD'")
+       not chassis_module_name.startswith("FABRIC-CARD") and \
+       not chassis_module_name.startswith("DPU") and \
+       not chassis_module_name.startswith("SWITCH"):
+        ctx.fail("'module_name' has to begin with 'SUPERVISOR', 'LINE-CARD', 'FABRIC-CARD', 'DPU' or 'SWITCH'")
 
     fvs = {'admin_status': 'down'}
     config_db.set_entry('CHASSIS_MODULE', chassis_module_name, fvs)

show/reboot_cause.py

@@ -4,6 +4,7 @@
 
 import click
 from tabulate import tabulate
+import textwrap
 from swsscommon.swsscommon import SonicV2Connector
 import utilities_common.cli as clicommon
 
@@ -23,6 +24,100 @@ def read_reboot_cause_file():
 
     return reboot_cause_dict
 
+# Function to fetch reboot cause data from database
+def fetch_reboot_cause_from_db(module_name):
+    table = []
+    r = []
+    wrapper = textwrap.TextWrapper(width=30)
+
+    # Read the previous reboot cause
+    if module_name == "all" or module_name == "SWITCH":
+        reboot_cause_dict = read_reboot_cause_file()
+        reboot_cause = reboot_cause_dict.get("cause", "Unknown")
+        reboot_user = reboot_cause_dict.get("user", "N/A")
+        reboot_time = reboot_cause_dict.get("time", "N/A")
+
+        r.append("SWITCH")
+        r.append(reboot_cause if reboot_cause else "")
+        r.append(reboot_time if reboot_time else "")
+        r.append(reboot_user if reboot_user else "")
+        table.append(r)
+
+        if module_name == "SWITCH":
+            return table
+
+    REBOOT_CAUSE_TABLE_NAME = "REBOOT_CAUSE"
+    TABLE_NAME_SEPARATOR = '|'
+    db = SonicV2Connector(host='127.0.0.1')
+    db.connect(db.STATE_DB, False)   # Make one attempt only
+    prefix = REBOOT_CAUSE_TABLE_NAME + TABLE_NAME_SEPARATOR
+    _hash = '{}{}'.format(prefix, '*')
+    table_keys = db.keys(db.STATE_DB, _hash)
+    if table_keys is not None:
+        table_keys.sort(reverse=True)
+
+    d = []
+    append = False
+    for tk in table_keys:
+        r = []
+        entry = db.get_all(db.STATE_DB, tk)
+        if 'device' in entry:
+            if module_name != entry['device'] and module_name != "all":
+                continue
+            if entry['device'] in d:
+                append = False
+                continue
+            else:
+                append = True
+                d.append(entry['device'])
+            if not module_name is None:
+                r.append(entry['device'] if 'device' in entry else "")
+            if 'cause' in entry:
+                wrp_cause = wrapper.fill(entry['cause'])
+            r.append(wrp_cause if 'cause' in entry else "")
+            r.append(entry['time'] if 'time' in entry else "")
+            r.append(entry['user'] if 'user' in entry else "")
+            if append == True:
+                table.append(r)
+
+    return table
+
+# Function to fetch reboot cause history data from database
+def fetch_reboot_cause_history_from_db(module_name):
+    REBOOT_CAUSE_TABLE_NAME = "REBOOT_CAUSE"
+    TABLE_NAME_SEPARATOR = '|'
+    db = SonicV2Connector(host='127.0.0.1')
+    db.connect(db.STATE_DB, False)   # Make one attempt only
+    prefix = REBOOT_CAUSE_TABLE_NAME + TABLE_NAME_SEPARATOR
+    _hash = '{}{}'.format(prefix, '*')
+    table_keys = db.keys(db.STATE_DB, _hash)
+    wrapper = textwrap.TextWrapper(width=30)
+
+    if table_keys is not None:
+        table_keys.sort(reverse=True)
+
+        table = []
+        device_present = False
+        for tk in table_keys:
+            entry = db.get_all(db.STATE_DB, tk)
+            if 'device' in entry:
+                device_present = True
+            r = []
+            if not module_name is None and device_present: 
+                r.append(entry['device'] if 'device' in entry else "SWITCH")
+            r.append(tk.replace(prefix, ""))
+            if 'cause' in entry:
+                wrp_cause = wrapper.fill(entry['cause'])
+            r.append(wrp_cause if 'cause' in entry else "")
+            r.append(entry['time'] if 'time' in entry else "")
+            r.append(entry['user'] if 'user' in entry else "")
+            if 'comment' in entry:
+                wrp_comment = wrapper.fill(entry['comment'])
+            r.append(wrp_comment if 'comment' in entry else "")
+            if module_name == 'all' or module_name == entry['device']:
+                table.append(r)
+
+    return table
 
 #
 # 'reboot-cause' group ("show reboot-cause")
@@ -61,34 +156,30 @@ def reboot_cause(ctx):
 
         click.echo(reboot_cause_str)
 
+# 'all' command within 'reboot-cause'
+@reboot_cause.command()
+def all():
+    """Show cause of most recent reboot"""
+    reboot_cause_data = fetch_reboot_cause_from_db("all")
+    if not reboot_cause_data:
+        click.echo("Reboot-cause history is not yet available in StateDB")
+    else:
+        header = ['Device', 'Name', 'Cause', 'Time', 'User']
+        click.echo(tabulate(reboot_cause_data, header, numalign="left"))
 
-# 'history' subcommand ("show reboot-cause history")
+# 'history' command within 'reboot-cause'
 @reboot_cause.command()
-def history():
+@click.argument('module_name', required=False)
+def history(module_name):
     """Show history of reboot-cause"""
-    REBOOT_CAUSE_TABLE_NAME = "REBOOT_CAUSE"
-    TABLE_NAME_SEPARATOR = '|'
-    db = SonicV2Connector(host='127.0.0.1')
-    db.connect(db.STATE_DB, False)   # Make one attempt only
-    prefix = REBOOT_CAUSE_TABLE_NAME + TABLE_NAME_SEPARATOR
-    _hash = '{}{}'.format(prefix, '*')
-    table_keys = db.keys(db.STATE_DB, _hash)
-    if table_keys is not None:
-        table_keys.sort(reverse=True)
-
-        table = []
-        for tk in table_keys:
-            entry = db.get_all(db.STATE_DB, tk)
-            r = []
-            r.append(tk.replace(prefix, ""))
-            r.append(entry['cause'] if 'cause' in entry else "")
-            r.append(entry['time'] if 'time' in entry else "")
-            r.append(entry['user'] if 'user' in entry else "")
-            r.append(entry['comment'] if 'comment' in entry else "")
-            table.append(r)
-
-        header = ['Name', 'Cause', 'Time', 'User', 'Comment']
-        click.echo(tabulate(table, header, numalign="left"))
-    else:
+    reboot_cause_history = fetch_reboot_cause_history_from_db(module_name)
+    if not reboot_cause_history:
         click.echo("Reboot-cause history is not yet available in StateDB")
-        sys.exit(1)
+    else:
+        if not module_name is None :
+            header = ['Device', 'Name', 'Cause', 'Time', 'User', 'Comment']
+            click.echo(tabulate(reboot_cause_history, header, numalign="left"))
+        else:
+            header = ['Name', 'Cause', 'Time', 'User', 'Comment']
+            click.echo(tabulate(reboot_cause_history, header, numalign="left"))
+

show/system_health.py

@@ -4,7 +4,11 @@
 import click
 from tabulate import tabulate
 import utilities_common.cli as clicommon
+from swsscommon.swsscommon import SonicV2Connector
 
+DPU_STATE = 'DPU_STATE'
+CHASSIS_SERVER='redis_chassis.server'
+CHASSIS_SERVER_PORT=6380
 
 def get_system_health_status():
     if os.environ.get("UTILITIES_UNIT_TESTING") == "1":
@@ -33,6 +37,90 @@ def get_system_health_status():
 
     return manager, chassis, stat
 
+def get_module_health(module_name):
+    try:
+        _, chassis, _ = get_system_health_status()
+        moduleindex = chassis.get_module_index(module_name)
+        if moduleindex:
+            module = chassis.get_module(moduleindex)
+            health_results = module.get_health_info()
+            if health_results:
+                return health_results.summary, health_results.monitorlist
+    except Exception as e:
+        click.echo("Error retrieving module health list:", e)
+    exit(1)
+
+def show_module_health_all(mode):
+    _, chassis, _ = get_system_health_status()
+    for index, mod in enumerate(chassis._module_list):
+        module_name = mod.get_name()
+        if "DPU" in module_name:
+            health_summary, health_monitorlist = get_module_health(module_name)
+            if mode == "monitorlist":
+                display_monitor_list(health_monitorlist)
+            elif mode == "summary":
+                display_monitor_list(health_summary)
+            else:
+                display_monitor_list(health_summary)
+                display_monitor_list(health_monitorlist)
+
+def show_module_state(module_name):
+    chassis_state_db = SonicV2Connector(host=CHASSIS_SERVER, port=CHASSIS_SERVER_PORT)
+    chassis_state_db.connect(chassis_state_db.CHASSIS_STATE_DB)
+
+    key_pattern = '*' if not module_name else '|' + module_name
+
+    keys = chassis_state_db.keys(chassis_state_db.CHASSIS_STATE_DB, DPU_STATE + key_pattern)
+    if not keys:
+        print('Key {} not found in {} table'.format(key_pattern, DPU_STATE))
+        return
+
+    table = []
+    for dbkey in natsorted(keys):
+        key_list = dbkey.split('|')
+        if len(key_list) != 2:  # error data in DB, log it and ignore
+            print('Warn: Invalid Key {} in {} table'.format(dbkey, DPU_STATE))
+            continue
+
+        state_info = chassis_state_db.get_all(chassis_state_db.CHASSIS_STATE_DB, dbkey)
+
+        # Determine operational status
+        dpu_states = [value for key, value in state_info.items() if key.startswith('dpu')]
+        if all(state == "up" for state in dpu_states):
+            oper_status = "Online"
+        elif any(state == "up" for state in dpu_states):
+            oper_status = "Partial Online"
+        else:
+            oper_status = "Offline"
+
+        row = [module_name, state_info.get('id', ''), oper_status, "", "", "", ""]
+        for key, value in state_info.items():
+            if key.startswith('dpu'):
+                if key.endswith('_time'):
+                    row[5] = value
+                elif key.endswith('_reason'):
+                    row[6] = value
+                if not key.endswith('_state'):
+                    row[0] = ""
+                    row[1] = ""
+                    row[2] = ""
+                    table.append(row)
+            else:
+                state_detail = key
+                row[3] = state_detail
+                row[4] = value
+        table.append(row)
+
+    headers = ["Name", "ID", "Oper-Status", "State-Detail", "State-Value", "Time", "Reason"]
+    click.echo(tabulate(table, headers=headers))
+
+def show_module_state_all():
+    _, chassis, _ = get_system_health_status()
+    for index, mod in enumerate(chassis._module_list):
+        module_name = mod.get_name()
+        if "DPU" in module_name:
+            show_module_state(module_name)
+
 def display_system_health_summary(stat, led):
     click.echo("System status summary\n\n  System status LED  " + led)
     services_list = []
@@ -108,27 +196,63 @@ def system_health():
     return
 
 @system_health.command()
-def summary():
+@click.argument('module_name', required=False)
+def summary(module_name):
     """Show system-health summary information"""
-    _, chassis, stat = get_system_health_status()
-    display_system_health_summary(stat, chassis.get_status_led())
-
+    if not module_name or module_name == "all":
+        _, chassis, stat = get_system_health_status()
+        display_system_health_summary(stat, chassis.get_status_led())
+    elif module_name.startswith("DPU"):
+        health_summary, _ = get_module_health(module_name)
+        display_monitor_list(health_summary)
+    elif module_name == "all":
+        show_module_health_all("summary")
+    else:
+        click.echo("Valid module-names are DPU0, DPU1, ...")
 
 @system_health.command()
-def detail():
+@click.argument('module_name', required=False)
+def detail(module_name):
     """Show system-health detail information"""
     manager, chassis, stat = get_system_health_status()
-    display_system_health_summary(stat, chassis.get_status_led())
-    display_monitor_list(stat)
-    display_ignore_list(manager)
-
+    if not module_name or module_name == "all":
+        display_system_health_summary(stat, chassis.get_status_led())
+        display_monitor_list(stat)
+        display_ignore_list(manager)
+    elif module_name.startswith("DPU"):
+        health_summary, health_monitorlist = get_module_health(module_name)
+        display_monitor_list(health_summary)
+        display_monitor_list(health_monitorlist)
+    elif module_name.startswith("all"):
+        show_module_health_all("detail")
+    else:
+        click.echo("Valid module-names are DPU0, DPU1, ...")
 
 @system_health.command()
-def monitor_list():
+@click.argument('module_name', required=False)
+def monitor_list(module_name):
     """Show system-health monitored services and devices name list"""
     _, _, stat = get_system_health_status()
-    display_monitor_list(stat)
+    if not module_name or module_name == "all":
+        display_monitor_list(stat)
+    elif module_name.startswith("DPU"):
+        _, health_monitorlist = get_module_health(module_name)
+        display_monitor_list(health_monitorlist)
+    elif module_name == "all":
+        show_module_health_all("monitorlist")
+    else:
+        click.echo("Valid module-names are DPU0, DPU1, ...")
 
+@system_health.command()
+@click.argument('module_name', required=False)
+def dpu(module_name):
+    """Show system-health dpu information"""
+    if module_name.startswith("DPU"):
+        show_module_state(module_name)
+    elif module_name == "all":
+        show_module_state_all()
+    else:
+        click.echo("Valid module-names are DPU0, DPU1, ...")
 
 @system_health.group('sysready-status',invoke_without_command=True)
 @click.pass_context