Capture logs before reboot (cherry-pick 202311) (#15059)

* Capture before reboot logs

This aids in diagnosing issues in the event of failures.

* Addressed PR feedback

tests/common/fixtures/advanced_reboot.py

@@ -435,6 +435,11 @@ def __fetchTestLogs(self, rebootOper=None):
             os.makedirs(log_dir)
         log_dir = log_dir + "/"
 
+        # Create a sub-directory to store the logs before the reboot happened
+        log_dir_before_reboot = os.path.join(log_dir, "before_reboot/")
+        if not os.path.exists(log_dir_before_reboot):
+            os.makedirs(log_dir_before_reboot)
+
         if "warm" in self.rebootType:
             # normalize "warm-reboot -f", "warm-reboot -c" to "warm-reboot" for report collection
             reboot_file_prefix = "warm-reboot"
@@ -481,6 +486,12 @@ def __fetchTestLogs(self, rebootOper=None):
                 {'src': syslogFile, 'dest': log_dir, 'flat': True},
                 {'src': sairedisRec, 'dest': log_dir, 'flat': True},
                 {'src': swssRec, 'dest': log_dir, 'flat': True},
+                # Logs from before reboot
+                {'src': '/host/syslog.99', 'dest': log_dir_before_reboot, 'flat': True, 'fail_on_missing': False},
+                {'src': '/host/sairedis.rec.99', 'dest': log_dir_before_reboot, 'flat': True,
+                 'fail_on_missing': False},
+                {'src': '/host/swss.rec.99', 'dest': log_dir_before_reboot, 'flat': True, 'fail_on_missing': False},
+                {'src': '/host/bgpd.log.99', 'dest': log_dir_before_reboot, 'flat': True, 'fail_on_missing': False},
             ],
         }
         for host, logs in list(logFiles.items()):

tests/platform_tests/conftest.py

@@ -16,6 +16,7 @@
 from .args.counterpoll_cpu_usage_args import add_counterpoll_cpu_usage_args
 from .mellanox.mellanox_thermal_control_test_helper import suspend_hw_tc_service, resume_hw_tc_service
 
+logger = logging.getLogger(__name__)
 
 TEMPLATES_DIR = os.path.join(os.path.dirname(
     os.path.realpath(__file__)), "templates")
@@ -525,6 +526,7 @@ def pre_reboot_analysis():
         base_os_version.append(get_current_sonic_version(duthost))
         bgpd_log = bgpd_log_handler(preboot=True)
         if platform in LOGS_ON_TMPFS_PLATFORMS or (len(logs_in_tmpfs) > 0 and logs_in_tmpfs[0] is True):
+            logger.info("Inserting step to back up logs to /host/ before reboot")
             # For small disk devices, /var/log in mounted in tmpfs.
             # Hence, after reboot the preboot logs are lost.
             # For log_analyzer to work, it needs logs from the shutdown path
@@ -547,6 +549,7 @@ def pre_reboot_analysis():
     def post_reboot_analysis(marker, event_counters=None, reboot_oper=None, log_dir=None):
         bgpd_log_handler()
         if platform in LOGS_ON_TMPFS_PLATFORMS or (len(logs_in_tmpfs) > 0 and logs_in_tmpfs[0] is True):
+            logger.info("Restoring log backup from /host/ after reboot")
             restore_backup = "mv /host/syslog.99 /var/log/; " +\
                 "mv /host/sairedis.rec.99 /var/log/swss/; " +\
                 "mv /host/swss.rec.99 /var/log/swss/; " +\