Consolidate cmor setups and logging

e3sm_to_cmip/__main__.py

@@ -4,6 +4,7 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import argparse
+import logging
 import os
 import signal
 import sys
@@ -21,25 +22,28 @@
 from tqdm import tqdm
 
 from e3sm_to_cmip import ROOT_HANDLERS_DIR, __version__, resources
-from e3sm_to_cmip._logger import _setup_logger, _setup_root_logger
+from e3sm_to_cmip._logger import _logger
+from e3sm_to_cmip.cmor_handlers.handler import _instantiate_handler_logger
 from e3sm_to_cmip.cmor_handlers.utils import (
     MPAS_REALMS,
     REALMS,
     Frequency,
     MPASRealm,
     Realm,
     _get_mpas_handlers,
+    _instantiate_h_utils_logger,
     derive_handlers,
     load_all_handlers,
 )
 from e3sm_to_cmip.util import (
     FREQUENCIES,
     _get_table_info,
-    get_handler_info_msg,
+    _instantiate_util_logger,
     add_metadata,
     copy_user_metadata,
     find_atm_files,
     find_mpas_files,
+    get_handler_info_msg,
     precheck,
     print_debug,
     print_message,
@@ -50,11 +54,6 @@
 warnings.filterwarnings("ignore")
 
 
-# Setup the root logger and this module's logger.
-log_filename = _setup_root_logger()
-logger = _setup_logger(__name__, propagate=True)
-
-
 @dataclass
 class CLIArguments:
     """A data class storing the command line arguments for e3sm_to_cmip.
@@ -97,9 +96,19 @@ class CLIArguments:
 
 class E3SMtoCMIP:
     def __init__(self, args: Optional[List[str]] = None):
+        # logger assignment is moved into __init__ AFTER the call to _parse_args
+        # to prevent the default logfile directory being created whenever a call
+        # to "--help" or "--version" is invoked.  Doing so, however, makes the
+        # logger unavailable to the functions in this class unless made global.
+        global logger
+
         # A dictionary of command line arguments.
         parsed_args = self._parse_args(args)
 
+        # Setup this module's logger AFTER args are parsed in __init__, so that
+        # default log file is NOT created for "--help" or "--version" calls.
+        logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True)
+
         # NOTE: The order of these attributes align with class CLIArguments.
         # ======================================================================
         # Run Mode settings.
@@ -115,8 +124,6 @@ def __init__(self, args: Optional[List[str]] = None):
         self.debug: bool = parsed_args.debug
         self.timeout: int = parsed_args.timeout
 
-        # ======================================================================
-        # CMOR settings.
         # ======================================================================
         self.var_list: List[str] = self._get_var_list(parsed_args.var_list)
         self.realm: Union[Realm, MPASRealm] = parsed_args.realm
@@ -140,6 +147,9 @@ def __init__(self, args: Optional[List[str]] = None):
         if self.precheck_path is not None:
             self._run_precheck()
 
+        self.handlers = self._get_handlers()
+
+    def print_config(self):
         logger.info("--------------------------------------")
         logger.info("| E3SM to CMIP Configuration")
         logger.info("--------------------------------------")
@@ -149,9 +159,6 @@ def __init__(self, args: Optional[List[str]] = None):
         logger.info(f"    * precheck_path='{self.precheck_path}'")
         logger.info(f"    * freq='{self.freq}'")
         logger.info(f"    * realm='{self.realm}'")
-        logger.info(f"    * Writing log output file to: {log_filename}")
-
-        self.handlers = self._get_handlers()
 
     def run(self):
         # Setup logger information and print out e3sm_to_cmip CLI arguments.
@@ -161,17 +168,16 @@ def run(self):
                 self.output_path, "user_metadata.json"
             )
 
-        # Setup directories using the CLI argument paths (e.g., output dir).
-        # ======================================================================
-        if not self.info_mode:
-            self._setup_dirs_with_paths()
-
         # Run e3sm_to_cmip with info mode.
         # ======================================================================
         if self.info_mode:
             self._run_info_mode()
             sys.exit(0)
 
+        # Setup directories using the CLI argument paths (e.g., output dir).
+        # ======================================================================
+        self._setup_dirs_with_paths()
+
         # Run e3sm_to_cmip to CMORize serially or in parallel.
         # ======================================================================
         timer = None
@@ -306,7 +312,9 @@ def _parse_args(self, args: Optional[List[str]]) -> CLIArguments:
             sys.exit(1)
 
         # Parse the arguments and perform validation.
+        # NOTE: exits here if args == "-h" or "--help" or "--version", else validate
         parsed_args = argparser.parse_args(args_to_parse)
+
         self._validate_parsed_args(parsed_args)
 
         # Convert to this data class for type checking to work.
@@ -410,7 +418,9 @@ def _setup_argparser(self) -> argparse.ArgumentParser:
             nargs="+",
             required=True,
             metavar="",
-            help=("Space separated list of CMIP variables to convert from E3SM to CMIP."),
+            help=(
+                "Space separated list of CMIP variables to convert from E3SM to CMIP."
+            ),
         )
         optional_run.add_argument(
             "--realm",
@@ -531,7 +541,6 @@ def _setup_argparser(self) -> argparse.ArgumentParser:
             "-h",
             "--help",
             action="help",
-            default=argparse.SUPPRESS,
             help="show this help message and exit",
         )
 
@@ -658,23 +667,24 @@ def _run_info_mode(self):  # noqa: C901
 
         # if the user asked if the variable is included in the table
         # but didnt ask about the files in the inpath
-        elif self.freq and self.tables_path and not self.input_path:    # info mode 2
+        elif self.freq and self.tables_path and not self.input_path:  # info mode 2
             for handler in self.handlers:
                 table_info = _get_table_info(self.tables_path, handler["table"])
                 if handler["name"] not in table_info["variable_entry"]:
                     msg = f"Variable {handler['name']} is not included in the table {handler['table']}"  # type: ignore
                     print_message(msg, status="error")
                     continue
                 else:
-                    if self.freq == "mon" and handler['table'] == "CMIP6_day.json":
+                    if self.freq == "mon" and handler["table"] == "CMIP6_day.json":
                         continue
-                    if ( self.freq == "day" or self.freq == "3hr" ) and handler['table'] == "CMIP6_Amon.json":
+                    if (self.freq == "day" or self.freq == "3hr") and handler[
+                        "table"
+                    ] == "CMIP6_Amon.json":
                         continue
                     hand_msg = get_handler_info_msg(handler)
                     messages.append(hand_msg)
 
-        elif self.freq and self.tables_path and self.input_path:        # info mode 3
-
+        elif self.freq and self.tables_path and self.input_path:  # info mode 3
             file_path = next(Path(self.input_path).glob("*.nc"))
 
             with xr.open_dataset(file_path) as ds:
@@ -714,9 +724,11 @@ def _run_info_mode(self):  # noqa: C901
                     #    but we only want the latter in the "hand_msg" output.
                     #    The vars "hass" and "rlut" have multiple freqs.
 
-                    if self.freq == "mon" and handler['table'] == "CMIP6_day.json":
+                    if self.freq == "mon" and handler["table"] == "CMIP6_day.json":
                         continue
-                    if ( self.freq == "day" or self.freq == "3hr" ) and handler['table'] == "CMIP6_Amon.json":
+                    if (self.freq == "day" or self.freq == "3hr") and handler[
+                        "table"
+                    ] == "CMIP6_Amon.json":
                         continue
 
                     hand_msg = None
@@ -778,12 +790,17 @@ def _run_serial(self) -> int:  # noqa: C901
         int
             1 if an error occurs, else 0
         """
+
+        # TODO: Make this a command-line flag.
+        use_pbar = False
+
         try:
             num_handlers = len(self.handlers)
             num_success = 0
+            num_failure = 0
             name = None
 
-            if self.realm != "atm":
+            if self.realm != "atm" and use_pbar:
                 pbar = tqdm(total=len(self.handlers))
 
             for _, handler in enumerate(self.handlers):
@@ -838,26 +855,31 @@ def _run_serial(self) -> int:  # noqa: C901
                 except Exception as e:
                     print_debug(e)
 
-                if name is not None:
+                if name is not None and name != "":
                     num_success += 1
-                    msg = f"Finished {name}, {num_success}/{num_handlers} jobs complete"
+                    msg = f"Finished {name}, {num_success}/{num_handlers} jobs complete (via run_serial)"
                     logger.info(msg)
                 else:
+                    num_failure += 1
                     msg = f"Error running handler {handler['name']}"
                     logger.info(msg)
 
-                if self.realm != "atm":
+                if self.realm != "atm" and use_pbar:
                     pbar.update(1)
-            if self.realm != "atm":
-                pbar.close()
 
+            if self.realm != "atm" and use_pbar:
+                pbar.close()
         except Exception as error:
             print_debug(error)
+
             return 1
         else:
             msg = f"{num_success} of {num_handlers} handlers complete"
             logger.info(msg)
 
+            if num_failure > 0:
+                return 1
+
             return 0
 
     def _run_parallel(self) -> int:  # noqa: C901
@@ -872,6 +894,9 @@ def _run_parallel(self) -> int:  # noqa: C901
         pool_res = list()
         will_run = []
 
+        # TODO: Make this a command-line flag.
+        do_pbar = False
+
         for idx, handler in enumerate(self.handlers):
             handler_method = handler["method"]
             handler_variables = handler["raw_variables"]
@@ -926,8 +951,10 @@ def _run_parallel(self) -> int:  # noqa: C901
             pool_res.append(res)
 
         # wait for each result to complete
-        pbar = tqdm(total=len(pool_res))
+        if do_pbar:
+            pbar = tqdm(total=len(pool_res))
         num_success = 0
+        num_failure = 0
         num_handlers = len(self.handlers)
         finished_success = []
         for idx, res in enumerate(pool_res):
@@ -936,26 +963,32 @@ def _run_parallel(self) -> int:  # noqa: C901
                 finished_success.append(out)
                 if out:
                     num_success += 1
-                    msg = f"Finished {out}, {idx + 1}/{num_handlers} jobs complete"
+                    msg = f"Finished {out}, {idx + 1}/{num_handlers} jobs complete (via run_parallel)"
                 else:
+                    num_failure += 1
                     msg = f'Error running handler {self.handlers[idx]["name"]}'
                     logger.error(msg)
 
                 logger.info(msg)
             except Exception as e:
                 print_debug(e)
-            pbar.update(1)
 
-        pbar.close()
+            if do_pbar:
+                pbar.update(1)
+
+        if do_pbar:
+            pbar.close()
+
         pool.shutdown()
 
         msg = f"{num_success} of {num_handlers} handlers complete"
         logger.info(msg)
 
         failed = set(will_run) - set(finished_success)
-        if failed:
+        if failed or num_failure > 0:
             logger.error(f"{', '.join(list(failed))} failed to complete")
             logger.error(msg)
+            return 1
 
         return 0
 
@@ -966,7 +999,15 @@ def _timeout_exit(self):
 
 def main(args: Optional[List[str]] = None):
     app = E3SMtoCMIP(args)
-    app.run()
+
+    # These calls allow module loggers that create default logfiles to avoid being
+    # instantiated by arguments "--help" or "--version" upon import.
+    _instantiate_util_logger()
+    _instantiate_h_utils_logger()
+    _instantiate_handler_logger()
+
+    app.print_config()
+    return app.run()
 
 
 if __name__ == "__main__":