diff --git a/e3sm_to_cmip/__main__.py b/e3sm_to_cmip/__main__.py index c939abb9..17e1a4ee 100755 --- a/e3sm_to_cmip/__main__.py +++ b/e3sm_to_cmip/__main__.py @@ -4,6 +4,7 @@ from __future__ import absolute_import, division, print_function, unicode_literals import argparse +import logging import os import signal import sys @@ -21,7 +22,8 @@ from tqdm import tqdm from e3sm_to_cmip import ROOT_HANDLERS_DIR, __version__, resources -from e3sm_to_cmip._logger import _setup_logger, _setup_root_logger +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip.cmor_handlers.handler import _instantiate_handler_logger from e3sm_to_cmip.cmor_handlers.utils import ( MPAS_REALMS, REALMS, @@ -29,17 +31,19 @@ MPASRealm, Realm, _get_mpas_handlers, + _instantiate_h_utils_logger, derive_handlers, load_all_handlers, ) from e3sm_to_cmip.util import ( FREQUENCIES, _get_table_info, - get_handler_info_msg, + _instantiate_util_logger, add_metadata, copy_user_metadata, find_atm_files, find_mpas_files, + get_handler_info_msg, precheck, print_debug, print_message, @@ -50,11 +54,6 @@ warnings.filterwarnings("ignore") -# Setup the root logger and this module's logger. -log_filename = _setup_root_logger() -logger = _setup_logger(__name__, propagate=True) - - @dataclass class CLIArguments: """A data class storing the command line arguments for e3sm_to_cmip. @@ -97,9 +96,19 @@ class CLIArguments: class E3SMtoCMIP: def __init__(self, args: Optional[List[str]] = None): + # logger assignment is moved into __init__ AFTER the call to _parse_args + # to prevent the default logfile directory being created whenever a call + # to "--help" or "--version" is invoked. Doing so, however, makes the + # logger unavailable to the functions in this class unless made global. + global logger + # A dictionary of command line arguments. parsed_args = self._parse_args(args) + # Setup this module's logger AFTER args are parsed in __init__, so that + # default log file is NOT created for "--help" or "--version" calls. + logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True) + # NOTE: The order of these attributes align with class CLIArguments. # ====================================================================== # Run Mode settings. @@ -115,8 +124,6 @@ def __init__(self, args: Optional[List[str]] = None): self.debug: bool = parsed_args.debug self.timeout: int = parsed_args.timeout - # ====================================================================== - # CMOR settings. # ====================================================================== self.var_list: List[str] = self._get_var_list(parsed_args.var_list) self.realm: Union[Realm, MPASRealm] = parsed_args.realm @@ -140,6 +147,9 @@ def __init__(self, args: Optional[List[str]] = None): if self.precheck_path is not None: self._run_precheck() + self.handlers = self._get_handlers() + + def print_config(self): logger.info("--------------------------------------") logger.info("| E3SM to CMIP Configuration") logger.info("--------------------------------------") @@ -149,9 +159,6 @@ def __init__(self, args: Optional[List[str]] = None): logger.info(f" * precheck_path='{self.precheck_path}'") logger.info(f" * freq='{self.freq}'") logger.info(f" * realm='{self.realm}'") - logger.info(f" * Writing log output file to: {log_filename}") - - self.handlers = self._get_handlers() def run(self): # Setup logger information and print out e3sm_to_cmip CLI arguments. @@ -161,17 +168,16 @@ def run(self): self.output_path, "user_metadata.json" ) - # Setup directories using the CLI argument paths (e.g., output dir). - # ====================================================================== - if not self.info_mode: - self._setup_dirs_with_paths() - # Run e3sm_to_cmip with info mode. # ====================================================================== if self.info_mode: self._run_info_mode() sys.exit(0) + # Setup directories using the CLI argument paths (e.g., output dir). + # ====================================================================== + self._setup_dirs_with_paths() + # Run e3sm_to_cmip to CMORize serially or in parallel. # ====================================================================== timer = None @@ -306,7 +312,9 @@ def _parse_args(self, args: Optional[List[str]]) -> CLIArguments: sys.exit(1) # Parse the arguments and perform validation. + # NOTE: exits here if args == "-h" or "--help" or "--version", else validate parsed_args = argparser.parse_args(args_to_parse) + self._validate_parsed_args(parsed_args) # Convert to this data class for type checking to work. @@ -410,7 +418,9 @@ def _setup_argparser(self) -> argparse.ArgumentParser: nargs="+", required=True, metavar="", - help=("Space separated list of CMIP variables to convert from E3SM to CMIP."), + help=( + "Space separated list of CMIP variables to convert from E3SM to CMIP." + ), ) optional_run.add_argument( "--realm", @@ -531,7 +541,6 @@ def _setup_argparser(self) -> argparse.ArgumentParser: "-h", "--help", action="help", - default=argparse.SUPPRESS, help="show this help message and exit", ) @@ -658,7 +667,7 @@ def _run_info_mode(self): # noqa: C901 # if the user asked if the variable is included in the table # but didnt ask about the files in the inpath - elif self.freq and self.tables_path and not self.input_path: # info mode 2 + elif self.freq and self.tables_path and not self.input_path: # info mode 2 for handler in self.handlers: table_info = _get_table_info(self.tables_path, handler["table"]) if handler["name"] not in table_info["variable_entry"]: @@ -666,15 +675,16 @@ def _run_info_mode(self): # noqa: C901 print_message(msg, status="error") continue else: - if self.freq == "mon" and handler['table'] == "CMIP6_day.json": + if self.freq == "mon" and handler["table"] == "CMIP6_day.json": continue - if ( self.freq == "day" or self.freq == "3hr" ) and handler['table'] == "CMIP6_Amon.json": + if (self.freq == "day" or self.freq == "3hr") and handler[ + "table" + ] == "CMIP6_Amon.json": continue hand_msg = get_handler_info_msg(handler) messages.append(hand_msg) - elif self.freq and self.tables_path and self.input_path: # info mode 3 - + elif self.freq and self.tables_path and self.input_path: # info mode 3 file_path = next(Path(self.input_path).glob("*.nc")) with xr.open_dataset(file_path) as ds: @@ -714,9 +724,11 @@ def _run_info_mode(self): # noqa: C901 # but we only want the latter in the "hand_msg" output. # The vars "hass" and "rlut" have multiple freqs. - if self.freq == "mon" and handler['table'] == "CMIP6_day.json": + if self.freq == "mon" and handler["table"] == "CMIP6_day.json": continue - if ( self.freq == "day" or self.freq == "3hr" ) and handler['table'] == "CMIP6_Amon.json": + if (self.freq == "day" or self.freq == "3hr") and handler[ + "table" + ] == "CMIP6_Amon.json": continue hand_msg = None @@ -778,12 +790,17 @@ def _run_serial(self) -> int: # noqa: C901 int 1 if an error occurs, else 0 """ + + # TODO: Make this a command-line flag. + use_pbar = False + try: num_handlers = len(self.handlers) num_success = 0 + num_failure = 0 name = None - if self.realm != "atm": + if self.realm != "atm" and use_pbar: pbar = tqdm(total=len(self.handlers)) for _, handler in enumerate(self.handlers): @@ -838,26 +855,31 @@ def _run_serial(self) -> int: # noqa: C901 except Exception as e: print_debug(e) - if name is not None: + if name is not None and name != "": num_success += 1 - msg = f"Finished {name}, {num_success}/{num_handlers} jobs complete" + msg = f"Finished {name}, {num_success}/{num_handlers} jobs complete (via run_serial)" logger.info(msg) else: + num_failure += 1 msg = f"Error running handler {handler['name']}" logger.info(msg) - if self.realm != "atm": + if self.realm != "atm" and use_pbar: pbar.update(1) - if self.realm != "atm": - pbar.close() + if self.realm != "atm" and use_pbar: + pbar.close() except Exception as error: print_debug(error) + return 1 else: msg = f"{num_success} of {num_handlers} handlers complete" logger.info(msg) + if num_failure > 0: + return 1 + return 0 def _run_parallel(self) -> int: # noqa: C901 @@ -872,6 +894,9 @@ def _run_parallel(self) -> int: # noqa: C901 pool_res = list() will_run = [] + # TODO: Make this a command-line flag. + do_pbar = False + for idx, handler in enumerate(self.handlers): handler_method = handler["method"] handler_variables = handler["raw_variables"] @@ -926,8 +951,10 @@ def _run_parallel(self) -> int: # noqa: C901 pool_res.append(res) # wait for each result to complete - pbar = tqdm(total=len(pool_res)) + if do_pbar: + pbar = tqdm(total=len(pool_res)) num_success = 0 + num_failure = 0 num_handlers = len(self.handlers) finished_success = [] for idx, res in enumerate(pool_res): @@ -936,26 +963,32 @@ def _run_parallel(self) -> int: # noqa: C901 finished_success.append(out) if out: num_success += 1 - msg = f"Finished {out}, {idx + 1}/{num_handlers} jobs complete" + msg = f"Finished {out}, {idx + 1}/{num_handlers} jobs complete (via run_parallel)" else: + num_failure += 1 msg = f'Error running handler {self.handlers[idx]["name"]}' logger.error(msg) logger.info(msg) except Exception as e: print_debug(e) - pbar.update(1) - pbar.close() + if do_pbar: + pbar.update(1) + + if do_pbar: + pbar.close() + pool.shutdown() msg = f"{num_success} of {num_handlers} handlers complete" logger.info(msg) failed = set(will_run) - set(finished_success) - if failed: + if failed or num_failure > 0: logger.error(f"{', '.join(list(failed))} failed to complete") logger.error(msg) + return 1 return 0 @@ -966,7 +999,15 @@ def _timeout_exit(self): def main(args: Optional[List[str]] = None): app = E3SMtoCMIP(args) - app.run() + + # These calls allow module loggers that create default logfiles to avoid being + # instantiated by arguments "--help" or "--version" upon import. + _instantiate_util_logger() + _instantiate_h_utils_logger() + _instantiate_handler_logger() + + app.print_config() + return app.run() if __name__ == "__main__": diff --git a/e3sm_to_cmip/_logger.py b/e3sm_to_cmip/_logger.py index 2e14df7c..06d5d43d 100644 --- a/e3sm_to_cmip/_logger.py +++ b/e3sm_to_cmip/_logger.py @@ -1,68 +1,67 @@ import logging import os -import time -from datetime import datetime +from datetime import datetime, timezone -from pytz import UTC +DEFAULT_LOG_LEVEL = logging.DEBUG +DEFAULT_LOG_DIR = "e2c_logs" +DEFAULT_LOG = f"{DEFAULT_LOG_DIR}/e2c_root_log-{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S_%f')}.log" -def _setup_root_logger() -> str: # pragma: no cover - """Sets up the root logger. +def _logger( + name: str | None = None, + logfilename: str = DEFAULT_LOG, + log_level: int = DEFAULT_LOG_LEVEL, + to_console: bool = False, + to_logfile: bool = False, + propagate: bool = False, +): + """Return a root or named logger with variable configuration. - The logger module will write to a log file and stream the console - simultaneously. - - The log files are saved in a `/logs` directory relative to where - `e3sm_to_cmip` is executed. - - Returns - ------- - str - The name of the logfile. + Parameters + ---------- + name : str + The name displayed for the logger in messages. + If name == None or name == "__main__", the root logger is returned + logfilename : str + If logfile handling is requested, any logfile may be specified, or else + the default (e2c_logs/dflt_log-{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S_%f')}.log) is used. + log_level : int + Logging.DEBUG (default), logging.INFO, logging.WARNING, logging.ERROR, + or logging.CRITICAL, by default logging.DEBUG. + to_console : bool + If True, a logging.StreamHandler is supplied, by default False. + to_logfile : bool + If True, a logging.FileHandler is supplied, by default False. + propagate : boolean + If True, messages logged are propagated to the root logger, by default + False. """ - os.makedirs("logs", exist_ok=True) - filename = f'logs/{UTC.localize(datetime.utcnow()).strftime("%Y%m%d_%H%M%S_%f")}' - log_format = "%(asctime)s_%(msecs)03d:%(levelname)s:%(funcName)s:%(message)s" - - # Setup the logging module. - logging.basicConfig( - filename=filename, - format=log_format, - datefmt="%Y%m%d_%H%M%S", - level=logging.DEBUG, - ) - logging.captureWarnings(True) - logging.Formatter.converter = time.gmtime - - # Configure and add a console stream handler. - console_handler = logging.StreamHandler() - console_handler.setLevel(logging.INFO) - log_formatter = logging.Formatter(log_format) - console_handler.setFormatter(log_formatter) - logging.getLogger().addHandler(console_handler) + if to_logfile: + dn = os.path.dirname(logfilename) + if len(dn) and not os.path.exists(dn): + os.makedirs(dn) - return filename + if name is None or name == "__main__": + logger = logger.root # noqa: F821 + else: + logger = logging.getLogger(name) + logger.propagate = propagate + logger.setLevel(log_level) -def _setup_logger(name, propagate=True) -> logging.Logger: - """Sets up a logger object. + logger.handlers = [] - This function is intended to be used at the top-level of a module. + msgfmt = "%(asctime)s_%(msecs)03d:%(levelname)s:%(name)s:%(funcName)s:%(message)s" + datefmt = "%Y%m%d_%H%M%S" - Parameters - ---------- - name : str - Name of the file where this function is called. - propagate : bool, optional - Propogate this logger module's messages to the root logger or not, by - default True. + if to_console: + logStreamHandler = logging.StreamHandler() + logStreamHandler.setFormatter(logging.Formatter(msgfmt, datefmt=datefmt)) + logger.addHandler(logStreamHandler) - Returns - ------- - logging.Logger - The logger. - """ - logger = logging.getLogger(name) - logger.propagate = propagate + if to_logfile: + logFileHandler = logging.FileHandler(logfilename) + logFileHandler.setFormatter(logging.Formatter(msgfmt, datefmt=datefmt)) + logger.addHandler(logFileHandler) return logger diff --git a/e3sm_to_cmip/cmor_handlers/handler.py b/e3sm_to_cmip/cmor_handlers/handler.py index 7766504e..ab965826 100644 --- a/e3sm_to_cmip/cmor_handlers/handler.py +++ b/e3sm_to_cmip/cmor_handlers/handler.py @@ -12,11 +12,20 @@ import xcdat as xc import yaml -from e3sm_to_cmip._logger import _setup_logger +from e3sm_to_cmip._logger import _logger from e3sm_to_cmip.cmor_handlers import FILL_VALUE, _formulas from e3sm_to_cmip.util import _get_table_for_non_monthly_freq -logger = _setup_logger(__name__) +logger = None + + +def _instantiate_handler_logger(): + global logger + + logger = _logger( + name=__name__, log_level=logging.INFO, to_logfile=True, propagate=True + ) + # The names for valid hybrid sigma levels. HYBRID_SIGMA_LEVEL_NAMES = [ @@ -267,9 +276,7 @@ def cmorize( # the variables. Otherwise, the IDs of cmor objects gets wiped after # every loop. cmor.close() - logger.debug( - f"{self.name}: CMORized and file write complete, closing CMOR I/O." - ) + logger.info(f"{self.name}: CMORized and file write complete, closing CMOR I/O.") return self.name @@ -290,7 +297,7 @@ def _all_vars_have_filepaths( """ for var, filepaths in vars_to_filespaths.items(): if len(filepaths) == 0: - logging.error(f"{var}: Unable to find input files for {var}") + logger.error(f"{var}: Unable to find input files for {var}") return False return True @@ -332,7 +339,7 @@ def _setup_cmor_module( cmor.dataset_json(metadata_path) cmor.load_table(self.table) - logging.info(f"{var_name}: CMOR setup complete") + logger.info(f"{var_name}: CMOR setup complete") def _get_var_time_dim(self, table_path: str) -> str | None: """Get the CMIP variable's time dimension, if it exists. diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/areacello.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/areacello.py index 867c8d95..07200c44 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/areacello.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/areacello.py @@ -4,8 +4,9 @@ import xarray import logging -from e3sm_to_cmip import mpas -from e3sm_to_cmip.util import print_message, setup_cmor +from e3sm_to_cmip import mpas, util +from e3sm_to_cmip.util import print_message +from e3sm_to_cmip._logger import _logger # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPAS_mesh', 'MPAS_map'] @@ -15,6 +16,7 @@ VAR_UNITS = 'm2' TABLE = 'CMIP6_Ofx.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -41,7 +43,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -65,8 +67,7 @@ def handle(infiles, tables, user_input_path, **kwargs): # area_b is in square radians, so need to multiply by the earth_radius**2 ds[VAR_NAME] = earth_radius**2*area_b*ds[VAR_NAME] - setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, - user_input_path=user_input_path) + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'latitude', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/fsitherm.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/fsitherm.py index 351efc6a..eb930a98 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/fsitherm.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/fsitherm.py @@ -6,8 +6,8 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -18,6 +18,7 @@ VAR_UNITS = 'kg m-2 s-1' TABLE = 'CMIP6_Omon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -43,7 +44,7 @@ def handle(infiles, tables, user_input_path, **kwargs): print_message(f'Simple CMOR output not supported for {VAR_NAME}', 'error') return None - logging.info(f'Starting {VAR_NAME}') + logger.info(f'Starting {VAR_NAME}') mappingFileName = infiles['MPAS_map'] timeSeriesFiles = infiles['MPASO'] @@ -60,7 +61,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/hfds.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/hfds.py index a7c0d281..d0cae9fe 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/hfds.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/hfds.py @@ -6,8 +6,8 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -18,6 +18,7 @@ VAR_UNITS = 'W m-2' TABLE = 'CMIP6_Omon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -46,7 +47,7 @@ def handle(infiles, tables, user_input_path, **kwargs): print_message(f'Simple CMOR output not supported for {VAR_NAME}', 'error') return None - logging.info(f'Starting {VAR_NAME}') + logger.info(f'Starting {VAR_NAME}') mappingFileName = infiles['MPAS_map'] timeSeriesFiles = infiles['MPASO'] @@ -73,7 +74,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/hfsifrazil.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/hfsifrazil.py index baf5c2e1..bea6a040 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/hfsifrazil.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/hfsifrazil.py @@ -6,8 +6,8 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -18,6 +18,7 @@ VAR_UNITS = 'W m-2' TABLE = 'CMIP6_Omon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -44,7 +45,7 @@ def handle(infiles, tables, user_input_path, **kwargs): print_message(f'Simple CMOR output not supported for {VAR_NAME}', 'error') return None - logging.info(f'Starting {VAR_NAME}') + logger.info(f'Starting {VAR_NAME}') timeSeriesFiles = infiles['MPASO'] mappingFileName = infiles['MPAS_map'] @@ -76,7 +77,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/masscello.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/masscello.py index 6030ccd1..74795aa9 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/masscello.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/masscello.py @@ -7,8 +7,8 @@ import xarray import logging import netCDF4 - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPASO_namelist', 'MPAS_mesh', 'MPAS_map'] @@ -18,6 +18,7 @@ VAR_UNITS = 'kg m-2' TABLE = 'CMIP6_Omon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -45,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) namelistFileName = infiles['MPASO_namelist'] meshFileName = infiles['MPAS_mesh'] @@ -77,7 +78,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds[VAR_NAME] = ds[VAR_NAME].where( ds[VAR_NAME] != netCDF4.default_fillvals['f4'], 0.) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/masso.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/masso.py index 8155fcea..dc810d5e 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/masso.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/masso.py @@ -7,8 +7,8 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPASO_namelist', 'MPAS_mesh'] @@ -18,6 +18,7 @@ VAR_UNITS = 'kg' TABLE = 'CMIP6_Omon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -45,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) namelistFileName = infiles['MPASO_namelist'] meshFileName = infiles['MPAS_mesh'] @@ -69,7 +70,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.add_time(ds, dsIn) ds.compute() - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/mlotst.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/mlotst.py index 43309b1e..9a774b9f 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/mlotst.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/mlotst.py @@ -6,8 +6,8 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh', 'MPAS_map'] @@ -17,6 +17,7 @@ VAR_UNITS = 'm' TABLE = 'CMIP6_Omon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -44,7 +45,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -66,7 +67,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.add_mask(ds, cellMask2D) ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/msftmz.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/msftmz.py index 1e255108..770b26e3 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/msftmz.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/msftmz.py @@ -6,8 +6,8 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh', 'MPASO_MOC_regions', 'MPASO_namelist'] @@ -17,6 +17,7 @@ VAR_UNITS = 'kg s-1' TABLE = 'CMIP6_Omon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -47,7 +48,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] timeSeriesFiles = infiles['MPASO'] @@ -76,7 +77,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = ds.rename({'moc': VAR_NAME}) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) region = ['global_ocean', 'atlantic_arctic_ocean'] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/pbo.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/pbo.py index 792fa1b8..2827082a 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/pbo.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/pbo.py @@ -6,8 +6,8 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPASO_namelist', 'MPAS_mesh', 'MPAS_map', 'PSL'] @@ -17,6 +17,7 @@ VAR_UNITS = 'Pa' TABLE = 'CMIP6_Omon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -46,7 +47,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) namelistFileName = infiles['MPASO_namelist'] meshFileName = infiles['MPAS_mesh'] @@ -84,7 +85,7 @@ def handle(infiles, tables, user_input_path, **kwargs): with xarray.open_mfdataset(pslFileNames, concat_dim='time') as dsIn: ds[VAR_NAME] = ds[VAR_NAME] + dsIn.PSL.values - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/pso.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/pso.py index 304ac8c8..def4bdce 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/pso.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/pso.py @@ -6,8 +6,8 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPASO_namelist', 'MPAS_mesh', 'MPAS_map', 'PSL'] @@ -17,6 +17,7 @@ VAR_UNITS = 'Pa' TABLE = 'CMIP6_Omon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -45,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) namelistFileName = infiles['MPASO_namelist'] meshFileName = infiles['MPAS_mesh'] @@ -80,7 +81,7 @@ def handle(infiles, tables, user_input_path, **kwargs): with xarray.open_mfdataset(pslFileNames, concat_dim='time') as dsIn: ds[VAR_NAME] = ds[VAR_NAME] + dsIn.PSL.values - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/sfdsi.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/sfdsi.py index af9e28fa..56126110 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/sfdsi.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/sfdsi.py @@ -6,8 +6,8 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_map'] @@ -17,6 +17,7 @@ VAR_UNITS = 'kg m-2 s-1' TABLE = 'CMIP6_Omon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -44,7 +45,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) mappingFileName = infiles['MPAS_map'] timeSeriesFiles = infiles['MPASO'] @@ -61,7 +62,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/siconc.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/siconc.py index f9a38943..914e617b 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/siconc.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/siconc.py @@ -7,8 +7,8 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASSI', 'MPAS_mesh', 'MPAS_map'] @@ -18,6 +18,7 @@ VAR_UNITS = '%' TABLE = 'CMIP6_SImon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -45,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -63,7 +64,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasseaice', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='seaice') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/simass.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/simass.py index ec514576..030b937c 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/simass.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/simass.py @@ -7,8 +7,8 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASSI', 'MPAS_mesh', 'MPAS_map'] @@ -18,6 +18,7 @@ VAR_UNITS = 'kg m-2' TABLE = 'CMIP6_SImon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -41,7 +42,7 @@ def handle(infiles, tables, user_input_path, **kwargs): the name of the processed variable after processing is complete """ msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -62,7 +63,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasseaice', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='seaice') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/sisnmass.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/sisnmass.py index 1542d421..6706c7a6 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/sisnmass.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/sisnmass.py @@ -7,8 +7,8 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASSI', 'MPAS_mesh', 'MPAS_map'] @@ -18,6 +18,7 @@ VAR_UNITS = 'kg m-2' TABLE = 'CMIP6_SImon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -46,7 +47,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -67,7 +68,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasseaice', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='seaice') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/sisnthick.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/sisnthick.py index 13de8a81..498617ad 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/sisnthick.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/sisnthick.py @@ -7,8 +7,8 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASSI', 'MPAS_mesh', 'MPAS_map'] @@ -18,6 +18,7 @@ VAR_UNITS = 'm' TABLE = 'CMIP6_SImon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -46,7 +47,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -67,7 +68,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasseaice', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='seaice') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/sitemptop.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/sitemptop.py index 706ea8e1..b912e553 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/sitemptop.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/sitemptop.py @@ -5,11 +5,10 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray - -from e3sm_to_cmip import mpas +import logging +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -20,6 +19,7 @@ VAR_UNITS = 'K' TABLE = 'CMIP6_SImon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -48,7 +48,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -67,7 +67,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasseaice', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='seaice') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/sithick.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/sithick.py index e9b66c89..704e5ca2 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/sithick.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/sithick.py @@ -7,8 +7,8 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASSI', 'MPAS_mesh', 'MPAS_map'] @@ -18,6 +18,7 @@ VAR_UNITS = 'm' TABLE = 'CMIP6_SImon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -46,7 +47,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -66,7 +67,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasseaice', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='seaice') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/sitimefrac.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/sitimefrac.py index 591ff362..4689a0bd 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/sitimefrac.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/sitimefrac.py @@ -7,8 +7,8 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASSI', 'MPAS_mesh', 'MPAS_map'] @@ -18,6 +18,7 @@ VAR_UNITS = '1' TABLE = 'CMIP6_SImon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -45,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -65,7 +66,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasseaice', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='seaice') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/siu.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/siu.py index b3105fbf..623e6792 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/siu.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/siu.py @@ -7,8 +7,8 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASSI', 'MPAS_mesh', 'MPAS_map'] @@ -18,6 +18,7 @@ VAR_UNITS = 'm s-1' TABLE = 'CMIP6_SImon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -45,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -69,7 +70,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasseaice', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='seaice') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/siv.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/siv.py index 4fd8b4f4..159d6ba1 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/siv.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/siv.py @@ -7,9 +7,9 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message +from e3sm_to_cmip._logger import _logger # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASSI', 'MPAS_mesh', 'MPAS_map'] @@ -18,6 +18,7 @@ VAR_UNITS = 'm s-1' TABLE = 'CMIP6_SImon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_console=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -45,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -70,7 +71,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasseaice', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='seaice') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/so.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/so.py index d622383e..18090935 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/so.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/so.py @@ -6,8 +6,8 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh', 'MPAS_map'] @@ -17,6 +17,7 @@ VAR_UNITS = '0.001' TABLE = 'CMIP6_Omon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -44,7 +45,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -67,7 +68,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/sob.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/sob.py index 81626534..15b501be 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/sob.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/sob.py @@ -6,8 +6,8 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh', 'MPAS_map'] @@ -17,6 +17,7 @@ VAR_UNITS = '0.001' TABLE = 'CMIP6_Omon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -44,7 +45,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -67,7 +68,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/soga.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/soga.py index 7f3f0d5c..1674635d 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/soga.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/soga.py @@ -7,8 +7,8 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh'] @@ -18,6 +18,7 @@ VAR_UNITS = '0.001' TABLE = 'CMIP6_Omon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -46,7 +47,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] timeSeriesFiles = infiles['MPASO'] @@ -70,7 +71,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.add_time(ds, dsIn) ds.compute() - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/sos.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/sos.py index 0d4b731e..dfb70769 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/sos.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/sos.py @@ -6,8 +6,8 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh', 'MPAS_map'] @@ -17,6 +17,7 @@ VAR_UNITS = '0.001' TABLE = 'CMIP6_Omon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -44,7 +45,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -67,7 +68,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/sosga.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/sosga.py index be153e90..a89aaa67 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/sosga.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/sosga.py @@ -6,8 +6,8 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh'] @@ -17,6 +17,7 @@ VAR_UNITS = '0.001' TABLE = 'CMIP6_Omon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -44,7 +45,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] timeSeriesFiles = infiles['MPASO'] @@ -66,7 +67,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds.compute() ds.compute() - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/tauuo.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/tauuo.py index dea84253..5926de7d 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/tauuo.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/tauuo.py @@ -6,8 +6,8 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_map'] @@ -17,6 +17,7 @@ VAR_UNITS = 'N m-2' TABLE = 'CMIP6_Omon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -44,7 +45,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) mappingFileName = infiles['MPAS_map'] timeSeriesFiles = infiles['MPASO'] @@ -61,7 +62,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/tauvo.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/tauvo.py index 032a2436..db08ad7a 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/tauvo.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/tauvo.py @@ -6,8 +6,8 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_map'] @@ -17,6 +17,7 @@ VAR_UNITS = 'N m-2' TABLE = 'CMIP6_Omon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -44,7 +45,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) mappingFileName = infiles['MPAS_map'] timeSeriesFiles = infiles['MPASO'] @@ -61,7 +62,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/thetao.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/thetao.py index f35af82c..fd19d7aa 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/thetao.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/thetao.py @@ -6,8 +6,8 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh', 'MPAS_map'] @@ -17,6 +17,7 @@ VAR_UNITS = 'degC' TABLE = 'CMIP6_Omon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -43,8 +44,8 @@ def handle(infiles, tables, user_input_path, **kwargs): print_message(msg) return - msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + msg = f'Starting {__name__}' + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -67,7 +68,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/thetaoga.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/thetaoga.py index 632c0a21..6bbff77c 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/thetaoga.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/thetaoga.py @@ -7,8 +7,8 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh'] @@ -18,6 +18,7 @@ VAR_UNITS = 'degC' TABLE = 'CMIP6_Omon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -46,7 +47,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] timeSeriesFiles = infiles['MPASO'] @@ -70,7 +71,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.add_time(ds, dsIn) ds.compute() - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/thkcello.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/thkcello.py index 01e6420e..969fd314 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/thkcello.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/thkcello.py @@ -5,9 +5,9 @@ import xarray import logging import netCDF4 - -from e3sm_to_cmip import mpas -from e3sm_to_cmip.util import print_message, setup_cmor +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util +from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh', 'MPAS_map'] @@ -16,6 +16,7 @@ VAR_UNITS = 'm' TABLE = 'CMIP6_Omon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -42,19 +43,25 @@ def handle(infiles, tables, user_input_path, **kwargs): print_message(msg) return - msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + msg = f'Starting {__name__}' + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] timeSeriesFiles = infiles['MPASO'] + msg = f' Calling xarray.open_dataset()' + logger.info(msg) + dsMesh = xarray.open_dataset(meshFileName, mask_and_scale=False) _, cellMask3D = mpas.get_mpaso_cell_masks(dsMesh) variableList = ['timeMonthly_avg_layerThickness', 'xtime_startMonthly', 'xtime_endMonthly'] + msg = f' Calling mpas.open_mfdataset()' + logger.info(msg) + ds = xarray.Dataset() with mpas.open_mfdataset(timeSeriesFiles, variableList) as dsIn: ds[VAR_NAME] = \ @@ -71,8 +78,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds[VAR_NAME] = ds[VAR_NAME].where( ds[VAR_NAME] != netCDF4.default_fillvals['f4'], 0.) - setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, - user_input_path=user_input_path) + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/tob.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/tob.py index 8c08df06..027aefba 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/tob.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/tob.py @@ -6,8 +6,8 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh', 'MPAS_map'] @@ -17,6 +17,7 @@ VAR_UNITS = 'degC' TABLE = 'CMIP6_Omon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -44,7 +45,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -67,7 +68,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/tos.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/tos.py index 4bc4d36d..e0607021 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/tos.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/tos.py @@ -6,8 +6,8 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh', 'MPAS_map'] @@ -17,6 +17,7 @@ VAR_UNITS = 'degC' TABLE = 'CMIP6_Omon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -44,7 +45,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -67,7 +68,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/tosga.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/tosga.py index 34bb9e96..91898ca9 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/tosga.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/tosga.py @@ -6,8 +6,8 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh'] @@ -17,6 +17,7 @@ VAR_UNITS = 'degC' TABLE = 'CMIP6_Omon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -44,7 +45,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] timeSeriesFiles = infiles['MPASO'] @@ -66,7 +67,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds.compute() ds.compute() - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/uo.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/uo.py index da0562e4..b7498054 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/uo.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/uo.py @@ -6,8 +6,8 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh', 'MPAS_map'] @@ -17,6 +17,7 @@ VAR_UNITS = 'm s-1' TABLE = 'CMIP6_Omon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -44,7 +45,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -67,7 +68,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/vo.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/vo.py index 0085a97a..976c9722 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/vo.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/vo.py @@ -6,8 +6,8 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh', 'MPAS_map'] @@ -17,6 +17,7 @@ VAR_UNITS = 'm s-1' TABLE = 'CMIP6_Omon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -44,7 +45,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -67,7 +68,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/volcello.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/volcello.py index 9a228bea..96f2c534 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/volcello.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/volcello.py @@ -5,9 +5,9 @@ import xarray import logging import netCDF4 - -from e3sm_to_cmip import mpas -from e3sm_to_cmip.util import print_message, setup_cmor +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util +from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh', 'MPAS_map'] @@ -16,6 +16,7 @@ VAR_UNITS = 'm3' TABLE = 'CMIP6_Omon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -43,7 +44,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -85,8 +86,7 @@ def handle(infiles, tables, user_input_path, **kwargs): # multiply variables in this order so they don't get transposed ds[VAR_NAME] = ds[VAR_NAME]*earth_radius**2*area_b - setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, - user_input_path=user_input_path) + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/volo.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/volo.py index b2e838a7..0249dc44 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/volo.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/volo.py @@ -7,8 +7,8 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh'] @@ -18,6 +18,7 @@ VAR_UNITS = 'm3' TABLE = 'CMIP6_Omon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -45,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] timeSeriesFiles = infiles['MPASO'] @@ -67,7 +68,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds.compute() - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/wfo.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/wfo.py index 2a4ab15d..8f3bb5ce 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/wfo.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/wfo.py @@ -6,8 +6,8 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_map'] @@ -17,6 +17,7 @@ VAR_UNITS = 'kg m-2 s-1' TABLE = 'CMIP6_Omon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -46,7 +47,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) mappingFileName = infiles['MPAS_map'] timeSeriesFiles = infiles['MPASO'] @@ -71,7 +72,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/wo.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/wo.py index 59995dca..59d50de0 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/wo.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/wo.py @@ -7,8 +7,8 @@ import xarray import logging import numpy - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh', 'MPAS_map'] @@ -18,6 +18,7 @@ VAR_UNITS = 'm s-1' TABLE = 'CMIP6_Omon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -45,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -70,7 +71,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/zhalfo.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/zhalfo.py index 42fe931c..28189541 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/zhalfo.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/zhalfo.py @@ -7,8 +7,8 @@ import xarray import logging import numpy - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh', 'MPAS_map'] @@ -18,6 +18,7 @@ VAR_UNITS = 'm' TABLE = 'CMIP6_Omon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -45,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -76,7 +77,7 @@ def handle(infiles, tables, user_input_path, **kwargs): zLayerBot = (zLayerBot - layerThickness.isel(nVertLevels=zIndex)).where(mask) zLayerBot.compute() - # print('done zLayerBot {}/{}'.format(zIndex+1, nVertLevels)) + logger.info(f"done zLayerBot {zIndex+1}/{nVertLevels}") slices.append(zLayerBot) maskSlices.append(mask) ds[VAR_NAME] = xarray.concat(slices, dim='olevhalf') @@ -85,12 +86,13 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = ds.transpose('Time', 'olevhalf', 'nCells') ds = mpas.add_time(ds, dsIn) ds.compute() + logger.info(f"returned from ds.compute()") ds = mpas.remap(ds, 'mpasocean', mappingFileName) depth_coord_half = numpy.zeros(nVertLevels+1) depth_coord_half[1:] = dsMesh.refBottomDepth.values - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/zos.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/zos.py index 7903d818..e79f429a 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/zos.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/zos.py @@ -6,8 +6,8 @@ import xarray import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip._logger import _logger +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -18,6 +18,7 @@ VAR_UNITS = 'm' TABLE = 'CMIP6_Omon.json' +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle(infiles, tables, user_input_path, **kwargs): """ @@ -45,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -69,7 +70,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/utils.py b/e3sm_to_cmip/cmor_handlers/utils.py index 8f946e8e..993fad2c 100644 --- a/e3sm_to_cmip/cmor_handlers/utils.py +++ b/e3sm_to_cmip/cmor_handlers/utils.py @@ -11,11 +11,18 @@ LEGACY_HANDLER_DIR_PATH, MPAS_HANDLER_DIR_PATH, ) -from e3sm_to_cmip._logger import _setup_logger +from e3sm_to_cmip._logger import _logger from e3sm_to_cmip.cmor_handlers.handler import VarHandler from e3sm_to_cmip.util import _get_table_for_non_monthly_freq -logger = _setup_logger(__name__) + +def _instantiate_h_utils_logger(): + global logger + + logger = _logger( + name=__name__, set_log_level="INFO", to_logfile=True, propagate=False + ) + # Type aliases Frequency = Literal["mon", "day", "6hrLev", "6hrPlev", "6hrPlevPt", "3hr", "1hr"] @@ -55,6 +62,7 @@ def load_all_handlers( KeyError If no handlers are defined for a CMIP6 variable in `handlers.yaml`. """ + handlers_by_var: Dict[str, List[Dict[str, Any]]] = _get_handlers_by_var() missing_handlers: List[str] = [] @@ -172,6 +180,7 @@ def derive_handlers( If a handler could not be derived for a CMIP6 variable using the existing E3SM variables. """ + # TODO: Refactor the function parameters. handlers_by_var: Dict[str, List[Dict[str, Any]]] = _get_handlers_by_var() derived_handlers: List[Dict[str, Any]] = [] @@ -360,15 +369,15 @@ def _get_handlers_from_modules(path: str) -> Dict[str, List[Dict[str, Any]]]: "handler.py", "utils.py", ]: - var = file.split(".")[0] - filepath = os.path.join(root, file) - module = _get_handler_module(var, filepath) + module_name = file.split(".")[0] + module_path = os.path.join(root, file) + module = SourceFileLoader(module_name, module_path).load_module() # NOTE: The value is set to a list with a single dict entry # so that it is compatible with the data structure for storing # all var handlers (which can have multiple handlers per # variable). - handlers[var] = [ + handlers[module_name] = [ { "name": module.VAR_NAME, "units": module.VAR_UNITS, @@ -383,24 +392,3 @@ def _get_handlers_from_modules(path: str) -> Dict[str, List[Dict[str, Any]]]: ] return handlers - - -def _get_handler_module(module_name: str, module_path: str): - """Get the variable handler Python module. - - Parameters - ---------- - module_name : str - The name of the module, which should be the key of the variable (e.g., - "orog"). - module_path : str - The absolute path to the variable handler Python module. - - Returns - ------- - module - The module. - """ - module = SourceFileLoader(module_name, module_path).load_module() - - return module diff --git a/e3sm_to_cmip/cmor_handlers/vars/areacella.py b/e3sm_to_cmip/cmor_handlers/vars/areacella.py index dcd4d0e7..57fcdd1e 100644 --- a/e3sm_to_cmip/cmor_handlers/vars/areacella.py +++ b/e3sm_to_cmip/cmor_handlers/vars/areacella.py @@ -4,7 +4,6 @@ from __future__ import absolute_import, division, print_function, unicode_literals import json -import logging import os import numpy as np @@ -12,12 +11,10 @@ import cmor from e3sm_to_cmip import resources -from e3sm_to_cmip._logger import _setup_logger from e3sm_to_cmip.mpas import write_netcdf -from e3sm_to_cmip.util import print_message - -logger = _setup_logger(__name__) - +from e3sm_to_cmip.util import setup_cmor, print_message +import logging +from e3sm_to_cmip._logger import _logger # list of raw variable names needed RAW_VARIABLES = [str("area")] VAR_NAME = str("areacella") @@ -25,6 +22,7 @@ TABLE = str("CMIP6_fx.json") RADIUS = 6.37122e6 +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle_simple(infiles): resource_path, _ = os.path.split(os.path.abspath(resources.__file__)) @@ -56,46 +54,30 @@ def handle_simple(infiles): def handle(infiles, tables, user_input_path, table, logdir): - msg = "{}: Starting".format(VAR_NAME) - logger.info(msg) + logger.info(f"{VAR_NAME}: Starting") # check that we have some input files for every variable zerofiles = False for variable in RAW_VARIABLES: if len(infiles[variable]) == 0: - msg = "{}: Unable to find input files for {}".format(VAR_NAME, variable) + msg = f"{VAR_NAME}: Unable to find input files for {variable}" print_message(msg) - logging.error(msg) + logger.error(msg) zerofiles = True if zerofiles: return None - # Create the logging directory and setup cmor - if logdir: - logpath = logdir - else: - outpath, _ = os.path.split(logger.__dict__["handlers"][0].baseFilename) - logpath = os.path.join(outpath, "cmor_logs") - os.makedirs(logpath, exist_ok=True) - - logfile = os.path.join(logpath, VAR_NAME + ".log") - - cmor.setup(inpath=tables, netcdf_file_action=cmor.CMOR_REPLACE, logfile=logfile) - - cmor.dataset_json(str(user_input_path)) - cmor.load_table(str(TABLE)) + setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) - msg = "{}: CMOR setup complete".format(VAR_NAME) - logging.info(msg) + logger.info(f"{VAR_NAME}: CMOR setup complete") # extract data from the input file - msg = "areacella: loading area" - logger.info(msg) + logger.info("areacella: loading area") filename = infiles["area"][0] if not os.path.exists(filename): - raise IOError("File not found: {}".format(filename)) + raise IOError(f"File not found: {filename}") ds = xr.open_dataset(filename, decode_times=False) @@ -108,8 +90,7 @@ def handle(infiles, tables, user_input_path, table, logdir): "area": ds["area"], } - msg = "{name}: loading axes".format(name=VAR_NAME) - logger.info(msg) + logger.info(f"{VAR_NAME}: loading axes") axes = [ { @@ -126,8 +107,7 @@ def handle(infiles, tables, user_input_path, table, logdir): }, ] - msg = "areacella: running CMOR" - logging.info(msg) + logger.info(f"{VAR_NAME}: running CMOR") axis_ids = list() for axis in axes: @@ -139,12 +119,10 @@ def handle(infiles, tables, user_input_path, table, logdir): outdata = data["area"].values * pow(RADIUS, 2) cmor.write(varid, outdata) - msg = "{}: write complete, closing".format(VAR_NAME) - logger.debug(msg) + logger.debug(f"{VAR_NAME}: write complete, closing") cmor.close() - msg = "{}: file close complete".format(VAR_NAME) - logger.debug(msg) + logger.debug(f"{VAR_NAME}: file close complete") - return "areacella" + return VAR_NAME diff --git a/e3sm_to_cmip/cmor_handlers/vars/clisccp.py b/e3sm_to_cmip/cmor_handlers/vars/clisccp.py index 05368eb8..8ace54c6 100644 --- a/e3sm_to_cmip/cmor_handlers/vars/clisccp.py +++ b/e3sm_to_cmip/cmor_handlers/vars/clisccp.py @@ -5,7 +5,6 @@ """ from __future__ import absolute_import, annotations, division, unicode_literals -import logging import os from typing import Dict, List, Union @@ -13,10 +12,9 @@ import xarray as xr import cmor -from e3sm_to_cmip._logger import _setup_logger -from e3sm_to_cmip.util import print_message - -logger = _setup_logger(__name__) +from e3sm_to_cmip.util import setup_cmor, print_message +import logging +from e3sm_to_cmip._logger import _logger # list of raw variable names needed RAW_VARIABLES = [str("FISCCP1_COSP")] @@ -24,6 +22,7 @@ VAR_UNITS = str("%") TABLE = str("CMIP6_CFmon.json") +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle( # noqa: C901 vars_to_filepaths: Dict[str, List[str]], @@ -59,35 +58,23 @@ def handle( # noqa: C901 If CMORizing was successful, return the output CMIP variable name to indicate success. If failed, return None .""" - logging.info(f"Starting {VAR_NAME}") + logger.info(f"Starting {VAR_NAME}") nonzero = False for variable in RAW_VARIABLES: if len(vars_to_filepaths[variable]) == 0: msg = f"{variable}: Unable to find input files for {RAW_VARIABLES}" print_message(msg) - logging.error(msg) + logger.error(msg) nonzero = True if nonzero: return None - msg = f"{VAR_NAME}: running with input files: {vars_to_filepaths}" - logger.debug(msg) - - if logdir is not None: - logfile = logfile = os.path.join(logdir, VAR_NAME + ".log") - else: - logfile = os.path.join(os.getcwd(), "logs") - if not os.path.exists(logfile): - os.makedirs(logfile) - logfile = os.path.join(logfile, VAR_NAME + ".log") + logger.debug(f"{VAR_NAME}: running with input files: {vars_to_filepaths}") - cmor.setup(inpath=tables, netcdf_file_action=cmor.CMOR_REPLACE, logfile=logfile) - cmor.dataset_json(metadata_path) - cmor.load_table(TABLE) + setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=metadata_path) - msg = f"{VAR_NAME}: CMOR setup complete" - logger.info(msg) + logger.info(f"{VAR_NAME}: CMOR setup complete") data: Dict[str, Union[np.ndarray, xr.DataArray]] = {} @@ -164,8 +151,7 @@ def handle( # noqa: C901 varid = cmor.variable(VAR_NAME, VAR_UNITS, axis_ids) # write out the data - msg = f"{VAR_NAME}: time {data['time_bnds'][0][0]:1.1f} - {data['time_bnds'][-1][-1]:1.1f}" - logger.info(msg) + logger.info(f"{VAR_NAME}: time {data['time_bnds'][0][0]:1.1f} - {data['time_bnds'][-1][-1]:1.1f}") cmor.write( varid, @@ -174,11 +160,9 @@ def handle( # noqa: C901 time_bnds=data["time_bnds"], ) - msg = f"{VAR_NAME}: write complete, closing" - logger.info(msg) + logger.info(f"{VAR_NAME}: write complete, closing") cmor.close() - msg = f"{VAR_NAME}: file close complete" - logger.info(msg) + logger.info(f"{VAR_NAME}: file close complete") return VAR_NAME diff --git a/e3sm_to_cmip/cmor_handlers/vars/orog.py b/e3sm_to_cmip/cmor_handlers/vars/orog.py index 6767e71b..f86b3084 100644 --- a/e3sm_to_cmip/cmor_handlers/vars/orog.py +++ b/e3sm_to_cmip/cmor_handlers/vars/orog.py @@ -4,7 +4,6 @@ from __future__ import absolute_import, division, print_function, unicode_literals import json -import logging import os import numpy as np @@ -12,11 +11,10 @@ import cmor from e3sm_to_cmip import resources -from e3sm_to_cmip._logger import _setup_logger from e3sm_to_cmip.mpas import write_netcdf -from e3sm_to_cmip.util import print_message - -logger = _setup_logger(__name__) +from e3sm_to_cmip.util import setup_cmor, print_message +import logging +from e3sm_to_cmip._logger import _logger # list of raw variable names needed RAW_VARIABLES = [str("PHIS")] @@ -25,6 +23,7 @@ TABLE = str("CMIP6_fx.json") GRAV = 9.80616 +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle_simple(infiles): resource_path, _ = os.path.split(os.path.abspath(resources.__file__)) @@ -56,8 +55,7 @@ def handle_simple(infiles): def handle(infiles, tables, user_input_path, table, logdir): - msg = f"{VAR_NAME}: Starting" - logger.info(msg) + logger.info(f"{VAR_NAME}: Starting") # check that we have some input files for every variable zerofiles = False @@ -65,32 +63,17 @@ def handle(infiles, tables, user_input_path, table, logdir): if len(infiles[variable]) == 0: msg = f"{VAR_NAME}: Unable to find input files for {variable}" print_message(msg) - logging.error(msg) + logger.error(msg) zerofiles = True if zerofiles: return None - # Create the logging directory and setup cmor - if logdir: - logpath = logdir - else: - outpath, _ = os.path.split(logger.__dict__["handlers"][0].baseFilename) - logpath = os.path.join(outpath, "cmor_logs") - os.makedirs(logpath, exist_ok=True) - - logfile = os.path.join(logpath, VAR_NAME + ".log") - - cmor.setup(inpath=tables, netcdf_file_action=cmor.CMOR_REPLACE, logfile=logfile) - - cmor.dataset_json(str(user_input_path)) - cmor.load_table(str(TABLE)) + setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) - msg = "{}: CMOR setup complete".format(VAR_NAME) - logging.info(msg) + logger.info(f"{VAR_NAME}: CMOR setup complete") # extract data from the input file - msg = "orog: loading PHIS" - logger.info(msg) + logger.info("orog: loading PHIS") filename = infiles["PHIS"][0] @@ -108,8 +91,7 @@ def handle(infiles, tables, user_input_path, table, logdir): "PHIS": ds["PHIS"], } - msg = f"{VAR_NAME}: loading axes" - logger.info(msg) + logger.info(f"{VAR_NAME}: loading axes") axes = [ { @@ -126,8 +108,7 @@ def handle(infiles, tables, user_input_path, table, logdir): }, ] - msg = "orog: running CMOR" - logging.info(msg) + logger.info("orog: running CMOR") axis_ids = list() for axis in axes: @@ -139,12 +120,10 @@ def handle(infiles, tables, user_input_path, table, logdir): outdata = data["PHIS"].values / GRAV cmor.write(varid, outdata) - msg = "{}: write complete, closing".format(VAR_NAME) - logger.debug(msg) + logger.debug(f"{VAR_NAME}: write complete, closing") cmor.close() - msg = "{}: file close complete".format(VAR_NAME) - logger.debug(msg) + logger.debug(f"{VAR_NAME}: file close complete") - return "orog" + return VARNAME diff --git a/e3sm_to_cmip/cmor_handlers/vars/sftlf.py b/e3sm_to_cmip/cmor_handlers/vars/sftlf.py index 0a5f4da2..835b7b6e 100644 --- a/e3sm_to_cmip/cmor_handlers/vars/sftlf.py +++ b/e3sm_to_cmip/cmor_handlers/vars/sftlf.py @@ -4,7 +4,6 @@ from __future__ import absolute_import, division, print_function, unicode_literals import json -import logging import os import numpy as np @@ -12,11 +11,10 @@ import cmor from e3sm_to_cmip import resources -from e3sm_to_cmip._logger import _setup_logger from e3sm_to_cmip.mpas import write_netcdf -from e3sm_to_cmip.util import print_message - -logger = _setup_logger(__name__) +from e3sm_to_cmip.util import setup_cmor, print_message +import logging +from e3sm_to_cmip._logger import _logger # list of raw variable names needed RAW_VARIABLES = [str("LANDFRAC")] @@ -24,6 +22,7 @@ VAR_UNITS = str("%") TABLE = str("CMIP6_fx.json") +logger = _logger(name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False) def handle_simple(infiles): resource_path, _ = os.path.split(os.path.abspath(resources.__file__)) @@ -55,8 +54,7 @@ def handle_simple(infiles): def handle(infiles, tables, user_input_path, table, logdir): - msg = f"{VAR_NAME}: Starting" - logger.info(msg) + logger.info(f"{VAR_NAME}: Starting") # check that we have some input files for every variable zerofiles = False @@ -64,32 +62,17 @@ def handle(infiles, tables, user_input_path, table, logdir): if len(infiles[variable]) == 0: msg = f"{VAR_NAME}: Unable to find input files for {variable}" print_message(msg) - logging.error(msg) + logger.error(msg) zerofiles = True if zerofiles: return None - # Create the logging directory and setup cmor - if logdir: - logpath = logdir - else: - outpath, _ = os.path.split(logger.__dict__["handlers"][0].baseFilename) - logpath = os.path.join(outpath, "cmor_logs") - os.makedirs(logpath, exist_ok=True) - - logfile = os.path.join(logpath, VAR_NAME + ".log") - - cmor.setup(inpath=tables, netcdf_file_action=cmor.CMOR_REPLACE, logfile=logfile) - - cmor.dataset_json(str(user_input_path)) - cmor.load_table(str(TABLE)) + setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, user_input_path=user_input_path) - msg = "{}: CMOR setup complete".format(VAR_NAME) - logging.info(msg) + logger.info(f"{VAR_NAME}: CMOR setup complete") # extract data from the input file - msg = "sftlf: loading LANDFRAC" - logger.info(msg) + logger.info("sftlf: loading LANDFRAC") filename = infiles["LANDFRAC"][0] @@ -107,8 +90,7 @@ def handle(infiles, tables, user_input_path, table, logdir): "LANDFRAC": ds["LANDFRAC"], } - msg = f"{VAR_NAME}: loading axes" - logger.info(msg) + logger.info(f"{VAR_NAME}: loading axes") axes = [ { @@ -125,8 +107,7 @@ def handle(infiles, tables, user_input_path, table, logdir): }, ] - msg = "sftlf: running CMOR" - logging.info(msg) + logger.info(f"{VAR_NAME}: running CMOR") axis_ids = list() for axis in axes: @@ -138,12 +119,10 @@ def handle(infiles, tables, user_input_path, table, logdir): outdata = data["LANDFRAC"].values * 100.0 cmor.write(varid, outdata) - msg = "{}: write complete, closing".format(VAR_NAME) - logger.debug(msg) + logger.debug(f"{VAR_NAME}: write complete, closing") cmor.close() - msg = "{}: file close complete".format(VAR_NAME) - logger.debug(msg) + logger.debug(f"{VAR_NAME}: file close complete") - return "sftlf" + return VAR_NAME diff --git a/e3sm_to_cmip/mpas.py b/e3sm_to_cmip/mpas.py index 755db9e1..c682cb02 100644 --- a/e3sm_to_cmip/mpas.py +++ b/e3sm_to_cmip/mpas.py @@ -23,21 +23,25 @@ import xarray from dask.diagnostics import ProgressBar +from e3sm_to_cmip._logger import _logger + +logger = _logger( + name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False +) -def run_ncremap_cmd(args, env): - logtext = f"mpas.py: remap: ncremap args = {args}" - logging.info(logtext) +def run_ncremap_cmd(args, env): proc = subprocess.Popen( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env ) - (out, err) = proc.communicate() - logging.info(out) + (_, err) = proc.communicate() + if proc.returncode: - print("Error running ncremap command: {}".format(" ".join(args))) + arglist = " ".join(args) + logger.error(f"Error running ncremap command: {arglist}") print(err.decode("utf-8")) raise subprocess.CalledProcessError( # type: ignore - "ncremap returned {}".format(proc.returncode) # type: ignore + f"ncremap returned {proc.returncode}" # type: ignore ) @@ -46,6 +50,11 @@ def remap_seaice_sgs(inFileName, outFileName, mappingFileName, renorm_threshold= ds_in = xarray.open_dataset(inFileName, decode_times=False) outFilePath = f"{outFileName}sub" os.makedirs(outFilePath) + + logger.info( + f"Calling run_ncremap_cmd for each ds_slice in {range(ds_in.sizes['time'])}" + ) + for t_index in range(ds_in.sizes["time"]): ds_slice = ds_in.isel(time=slice(t_index, t_index + 1)) ds_slice.to_netcdf(f"{outFilePath}/temp_in{t_index}.nc") @@ -77,13 +86,18 @@ def remap_seaice_sgs(inFileName, outFileName, mappingFileName, renorm_threshold= def remap(ds, pcode, mappingFileName, threshold=0.0): """Use ncreamp to remap the xarray Dataset to a new target grid""" + # TODO: Make this a global commandline switch + delete_tempfiles = True + # write the dataset to a temp file inFileName = _get_temp_path() outFileName = _get_temp_path() if "depth" in ds.dims: + logger.info("Calling ds.transpose") ds = ds.transpose("time", "depth", "nCells", "nbnd") + logger.info(f"Calling write_netcdf() (inFileName={inFileName})") write_netcdf(ds, inFileName, unlimited="time") # set an environment variable to make sure we're not using czender's @@ -111,6 +125,7 @@ def remap(ds, pcode, mappingFileName, threshold=0.0): inFileName, outFileName, ] + logger.info(f"Calling run_ncremap_cmd with args {args}") run_ncremap_cmd(args, env) elif pcode == "mpasseaice": # MPAS-Seaice is a special case because the of the time-varying SGS field @@ -127,9 +142,9 @@ def remap(ds, pcode, mappingFileName, threshold=0.0): ds.load() - # remove the temporary files - os.remove(inFileName) - os.remove(outFileName) + if delete_tempfiles: + os.remove(inFileName) + os.remove(outFileName) return ds @@ -321,7 +336,6 @@ def open_mfdataset( concat_dim="Time", mask_and_scale=False, chunks=chunks, - lock=False, ) if variableList is not None: @@ -351,6 +365,9 @@ def write_netcdf(ds, fileName, fillValues=netCDF4.default_fillvals, unlimited=No encodingDict = {} variableNames = list(ds.data_vars.keys()) + list(ds.coords.keys()) for variableName in variableNames: + if "_FillValue" in ds[variableName].attrs: + # There's already a fill value attribute so don't add a new one + continue isNumeric = np.issubdtype(ds[variableName].dtype, np.number) if isNumeric: dtype = ds[variableName].dtype @@ -395,27 +412,6 @@ def convert_namelist_to_dict(fileName): return nml -def setup_cmor(varname, tables, user_input_path, component="ocean", table=None): - """Set up CMOR for MPAS-Ocean or MPAS-Seaice""" - logfile = os.path.join(os.getcwd(), "cmor_logs") - if not os.path.exists(logfile): - os.makedirs(logfile) - logfile = os.path.join(logfile, varname + ".log") - cmor.setup(inpath=tables, netcdf_file_action=cmor.CMOR_REPLACE, logfile=logfile) - cmor.dataset_json(str(user_input_path)) - if table is None: - if component == "ocean": - table = "CMIP6_Omon.json" - elif component == "seaice": - table = "CMIP6_SImon.json" - else: - raise ValueError("Unexpected component {}".format(component)) - try: - cmor.load_table(table) - except Exception: - raise ValueError("Unable to load table from {}".format(varname)) - - def write_cmor(axes, ds, varname, varunits, d2f=True, **kwargs): """Write a time series of a variable in the format expected by CMOR""" axis_ids = list() @@ -437,6 +433,11 @@ def write_cmor(axes, ds, varname, varunits, d2f=True, **kwargs): str(varname), str(varunits), axis_ids, missing_value=fillValue, **kwargs ) + # adding "shuffle" reduced size on disk by 14%, need to test for performance impact. + # TODO: Pass a commandline arg "--shuffle" down to this point. + # if do_shuffle and varname not in [ "lat", "lev", "lon" ]: + # cmor.set_deflate(varid, True, True, 1) + # write out the data try: if "time" not in ds.dims: @@ -449,7 +450,7 @@ def write_cmor(axes, ds, varname, varunits, d2f=True, **kwargs): time_bnds=ds.time_bnds.values, ) except Exception as error: - logging.exception(f"Error in cmor.write for {varname}") + logger.exception(f"Error in cmor.write for {varname}") raise Exception(error) finally: cmor.close(varid) diff --git a/e3sm_to_cmip/util.py b/e3sm_to_cmip/util.py index 8c57c995..e97d6659 100644 --- a/e3sm_to_cmip/util.py +++ b/e3sm_to_cmip/util.py @@ -1,6 +1,7 @@ from __future__ import absolute_import, division, print_function, unicode_literals import json +import logging import os import re import sys @@ -14,9 +15,15 @@ import yaml from tqdm import tqdm -from e3sm_to_cmip._logger import _setup_logger +from e3sm_to_cmip._logger import _logger -logger = _setup_logger(__name__) + +def _instantiate_util_logger(): + global logger + + logger = _logger( + name=__name__, log_level=logging.INFO, to_logfile=True, propagate=False + ) ATMOS_TABLES = [ @@ -107,20 +114,19 @@ def setup_cmor(var_name, table_path, table_name, user_input_path): """ Sets up cmor and logging for a single handler """ - var_name = str(var_name) - table_path = str(table_path) - table_name = str(table_name) - user_input_path = str(user_input_path) - - logfile = os.path.join(os.getcwd(), "logs") + logfile = os.path.join(os.getcwd(), "cmor_logs") if not os.path.exists(logfile): os.makedirs(logfile) - logfile = os.path.join(logfile, var_name + ".log") + cmor.setup(inpath=table_path, netcdf_file_action=cmor.CMOR_REPLACE, logfile=logfile) - cmor.dataset_json(user_input_path) - cmor.load_table(table_name) + cmor.dataset_json(str(user_input_path)) + + try: + cmor.load_table(table_name) + except Exception: + raise ValueError(f"Unable to load table {table_name} for {var_name}") # ------------------------------------------------------------------ @@ -315,6 +321,7 @@ def _get_table_info(tables, table): with open(table, "r") as instream: return json.load(instream) + def get_handler_info_msg(handler): msg = { "CMIP6 Name": handler["name"], @@ -561,7 +568,7 @@ def find_mpas_files(component, path, map_path=None): # noqa: C901 pattern_v2 = "mocBasinsAndTransects" for infile in contents: if pattern_v1 in infile or pattern_v2 in infile: - logger.info(f"component mpas0_moc_regions found: {infile}") + logger.info(f"component mpaso_moc_regions found: {infile}") return os.path.abspath(os.path.join(path, infile)) raise IOError("Unable to find mpaso_moc_regions in the input directory")