Merge pull request #129 from EC-DIGIT-CSIRC/quick-fixes

Quick fixes on parsers and logging

src/sysdiagnose/parsers/accessibility_tcc.py

@@ -6,7 +6,7 @@
 
 import glob
 import os
-from sysdiagnose.utils.base import BaseParserInterface
+from sysdiagnose.utils.base import BaseParserInterface, logger
 from sysdiagnose.utils.apollo import Apollo
 
 
@@ -31,7 +31,7 @@ def execute(self) -> list | dict:
         # only one file to parse
         try:
             result = []
-            apollo = Apollo(os_version='yolo')  # FIXME get right OS version, but also update the Apollo modules to be aware of the latest OS versions
+            apollo = Apollo(logger=logger, os_version='yolo')  # FIXME get right OS version, but also update the Apollo modules to be aware of the latest OS versions
             for logfile in self.get_log_files():
                 result.extend(apollo.parse_db(db_fname=logfile, db_type='TCC.db'))
             return result

src/sysdiagnose/parsers/powerlogs.py

@@ -6,7 +6,7 @@
 
 import glob
 import os
-from sysdiagnose.utils.base import BaseParserInterface
+from sysdiagnose.utils.base import BaseParserInterface, logger
 from sysdiagnose.utils.apollo import Apollo
 
 
@@ -34,7 +34,7 @@ def get_log_files(self) -> list:
 
     def execute(self) -> list:
         result = []
-        apollo = Apollo(os_version='yolo')  # FIXME get right OS version, but also update the Apollo modules to be aware of the latest OS versions
+        apollo = Apollo(logger=logger, os_version='yolo')  # FIXME get right OS version, but also update the Apollo modules to be aware of the latest OS versions
         for logfile in self.get_log_files():
             result.extend(apollo.parse_db(db_fname=logfile, db_type='CurrentPowerlog.PLSQL'))
 

src/sysdiagnose/parsers/spindumpnosymbols.py

@@ -45,7 +45,11 @@ def parse_file(path: str) -> list:
 
                 # stripping
                 for line in f_in:
-                    if line.strip() == "" or line.strip() == "Heavy format: stacks are sorted by count" or line.strip() == "Use -i and -timeline to re-report with chronological sorting":
+                    if line.strip() == "No samples":
+                        status = 'empty'
+                        # Since the rest is just 'binary format', we ignore the rest of the file.
+                        break
+                    elif line.strip() == "" or line.strip() == "Heavy format: stacks are sorted by count" or line.strip() == "Use -i and -timeline to re-report with chronological sorting":
                         continue
                     elif line.strip() == "------------------------------------------------------------":
                         status = 'processes_raw'
@@ -62,9 +66,12 @@ def parse_file(path: str) -> list:
 
                 # call parsing function per section
                 events = []
-                basic = SpindumpNoSymbolsParser.parse_basic(headers)
-                events.append(basic)
-                events.extend(SpindumpNoSymbolsParser.parse_processes(processes_raw, start_timestamp=basic['timestamp']))
+                if status != 'empty':
+                    basic = SpindumpNoSymbolsParser.parse_basic(headers)
+                    events.append(basic)
+                    events.extend(SpindumpNoSymbolsParser.parse_processes(processes_raw, start_timestamp=basic['timestamp']))
+                # Logging
+                logger.debug(f"{len(events)} events retrieved", extra={'num_events': len(events)})
 
                 return events
 

src/sysdiagnose/utils/apollo.py

@@ -64,19 +64,21 @@
 import re
 from datetime import datetime, timezone
 import glob
+import logging
 
 default_mod_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'apollo_modules')
 
 
 class Apollo():
-    def __init__(self, mod_dir: str = default_mod_dir, os_version: str = 'yolo'):
+    def __init__(self, logger: logging.Logger, mod_dir: str = default_mod_dir, os_version: str = 'yolo'):
         """
         Initialize the Apollo class for parsing databases
 
         Args:
             mod_dir (str): The directory where the module definitions are stored
             os_version (str): The version of the OS for which to parse the modules. 'yolo' means all versions.
         """
+        self.logger = logger
         self.os_version = os_version
         self.mod_dir = mod_dir
 
@@ -125,7 +127,7 @@ def parse_db(self, db_fname: str, db_type: str = None) -> list:
         try:
             module_queries = self.modules[db_type]
         except KeyError:
-            print(f"No modules with queries for {db_type}.")
+            self.logger.exception(f"No modules with queries for {db_type}.")
             return results
 
         # establish db connection
@@ -140,11 +142,14 @@ def parse_db(self, db_fname: str, db_type: str = None) -> list:
                 cur.execute(module_query['sql'])
                 rows = cur.fetchall()
             except Exception:
-                print(f"WARNING: Cannot fetch query contents for query with name: {module_query['name']}.")
+                self.logger.warning(
+                    f"WARNING: Cannot fetch query contents for query with name: {module_query['name']}.",
+                    extra={"apollo_module": module_query['name']}, exc_info=True)
                 continue
 
             if not rows:
-                print(f"No Records Found for {module_query['name']}.")
+                self.logger.info(f"No Records Found for {module_query['name']}.",
+                                 extra={"apollo_module": module_query['name']})
                 continue
 
             headers = []
@@ -164,7 +169,9 @@ def parse_db(self, db_fname: str, db_type: str = None) -> list:
                     results.append(item)
                 except TypeError:
                     # problem with timestamp parsing
-                    print(f"WARNING: Problem with timestamp parsing for table {db_fname}, row {list(row)}")
+                    self.logger.warning(f"WARNING: Problem with timestamp parsing for table {db_fname}, row {list(row)}",
+                                        extra={"apollo_module": module_query['name'], "table": db_fname, "row": list(row)},
+                                        exc_info=True)
 
-        print("Executing module on: " + db_fname)
+        self.logger.info("Executing module on: " + db_fname, extra={"apollo_module": module_query['name'], "table": db_fname})
         return results

src/sysdiagnose/utils/logger.py

@@ -3,8 +3,8 @@
 from datetime import datetime
 
 logger = logging.getLogger('sysdiagnose')
-# By default, we want to have the possibility to log almost everything.
-logger.setLevel(logging.INFO)
+# By default, we want to have the possibility to log everything.
+logger.setLevel(logging.DEBUG)
 
 
 class SysdiagnoseJsonFormatter(jsonlogger.JsonFormatter):
@@ -31,13 +31,13 @@ def get_console_handler(level: str) -> logging.StreamHandler:
     return ch
 
 
-def get_json_handler(filename: str, level: int = logging.INFO) -> logging.FileHandler:
+def get_json_handler(filename: str, level: int = logging.DEBUG) -> logging.FileHandler:
     '''
     Creates a logging JSON format file handler.
 
     Args:
         filename: Filename where to log.
-        level: Logging level. By default to INFO. https://docs.python.org/3/library/logging.html#logging-levels
+        level: Logging level. By default to DEBUG. https://docs.python.org/3/library/logging.html#logging-levels
     '''
     fmt_json = SysdiagnoseJsonFormatter(
         fmt='%(created)f %(asctime)s %(levelname)s %(module)s %(message)s',