From f1d61ae59079ab8d2869bf0083e6a4a0ce1670e1 Mon Sep 17 00:00:00 2001 From: Janosh Riebesell Date: Sat, 16 Nov 2024 13:50:47 -0500 Subject: [PATCH 1/5] JobflowSettings add LOG_FORMAT --- src/jobflow/settings.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/jobflow/settings.py b/src/jobflow/settings.py index 5f59ab94..1181e834 100644 --- a/src/jobflow/settings.py +++ b/src/jobflow/settings.py @@ -11,6 +11,8 @@ from jobflow import JobStore DEFAULT_CONFIG_FILE_PATH = Path("~/.jobflow.yaml").expanduser().as_posix() +DEFAULT_LOG_FORMAT = "%(asctime)s %(levelname)s %(message)s" +DEFAULT_DIRECTORY_FORMAT = "%Y-%m-%d-%H-%M-%S-%f" def _default_additional_store(): @@ -28,7 +30,7 @@ class JobflowSettings(BaseSettings): """ Settings for jobflow. - The default way to modify these is to modify ~/.jobflow.yaml. Alternatively, + The default way to modify these is to create a ~/.jobflow.yaml. Alternatively, the environment variable ``JOBFLOW_CONFIG_FILE`` can be set to point to a yaml file with jobflow settings. @@ -114,9 +116,18 @@ class JobflowSettings(BaseSettings): "accepted formats.", ) DIRECTORY_FORMAT: str = Field( - "%Y-%m-%d-%H-%M-%S-%f", + DEFAULT_DIRECTORY_FORMAT, description="Date stamp format used to create directories", ) + LOG_FORMAT: str = Field( + DEFAULT_LOG_FORMAT, + description="""Logging format string. Common format codes: + - %(message)s - The logged message + - %(asctime)s - Human-readable time + - %(levelname)s - DEBUG, INFO, WARNING, ERROR, or CRITICAL + - %(name)s - Logger name + See Python logging documentation for more format codes.""", + ) UID_TYPE: str = Field( "uuid4", description="Type of unique identifier to use to track jobs. " From 4b0536796caa9067e9c7fc3010742a478edb8fca Mon Sep 17 00:00:00 2001 From: Janosh Riebesell Date: Sat, 16 Nov 2024 13:54:22 -0500 Subject: [PATCH 2/5] initialize_logger add keyword fmt: str = "" --- src/jobflow/utils/log.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/jobflow/utils/log.py b/src/jobflow/utils/log.py index 408c08f9..eab81afe 100644 --- a/src/jobflow/utils/log.py +++ b/src/jobflow/utils/log.py @@ -1,15 +1,24 @@ """Tools for logging.""" +from __future__ import annotations + import logging -def initialize_logger(level: int = logging.INFO) -> logging.Logger: +def initialize_logger(level: int = logging.INFO, fmt: str = "") -> logging.Logger: """Initialize the default logger. Parameters ---------- level The log level. + fmt + Custom logging format string. Defaults to JobflowSettings.LOG_FORMAT. + Common format codes: + - %(message)s - The logged message + - %(asctime)s - Human-readable time + - %(levelname)s - DEBUG, INFO, WARNING, ERROR, or CRITICAL + See Python logging documentation for more format codes. Returns ------- @@ -18,12 +27,15 @@ def initialize_logger(level: int = logging.INFO) -> logging.Logger: """ import sys + from jobflow import SETTINGS + log = logging.getLogger("jobflow") log.setLevel(level) log.handlers = [] # reset logging handlers if they already exist - fmt = logging.Formatter("%(asctime)s %(levelname)s %(message)s") + formatter = logging.Formatter(fmt or SETTINGS.LOG_FORMAT) + screen_handler = logging.StreamHandler(stream=sys.stdout) - screen_handler.setFormatter(fmt) + screen_handler.setFormatter(formatter) log.addHandler(screen_handler) return log From 28ed2941395cfc855edc2ada816b0358722ae08e Mon Sep 17 00:00:00 2001 From: Janosh Riebesell Date: Sat, 16 Nov 2024 13:55:51 -0500 Subject: [PATCH 3/5] run_locally allow log=str for custom log format --- src/jobflow/managers/local.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/jobflow/managers/local.py b/src/jobflow/managers/local.py index 9f70a809..821f2a80 100644 --- a/src/jobflow/managers/local.py +++ b/src/jobflow/managers/local.py @@ -15,7 +15,7 @@ def run_locally( flow: jobflow.Flow | jobflow.Job | list[jobflow.Job], - log: bool = True, + log: bool | str = True, store: jobflow.JobStore | None = None, create_folders: bool = False, root_dir: str | Path | None = None, @@ -30,8 +30,11 @@ def run_locally( ---------- flow : Flow | Job | list[Job] A job or flow. - log : bool - Whether to print log messages. + log : bool | str + Controls logging. Defaults to True. Can be: + - False: disable logging + - True: use default logging format (read from ~/.jobflow.yaml) + - str: custom logging format string (e.g. "%(message)s" for more concise output) store : JobStore A job store. If a job store is not specified then :obj:`JobflowSettings.JOB_STORE` will be used. By default this is a maggma @@ -77,7 +80,7 @@ def run_locally( store.connect() if log: - initialize_logger() + initialize_logger(fmt=log if isinstance(log, str) else "") flow = get_flow(flow, allow_external_references=allow_external_references) From 983de1072bb05ce7ac9dbac2b48339e38d8df21b Mon Sep 17 00:00:00 2001 From: Janosh Riebesell Date: Sat, 16 Nov 2024 13:57:41 -0500 Subject: [PATCH 4/5] run_locally test custom log format --- tests/managers/test_local.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/managers/test_local.py b/tests/managers/test_local.py index b26dce69..e7c9e3c9 100644 --- a/tests/managers/test_local.py +++ b/tests/managers/test_local.py @@ -44,7 +44,15 @@ def test_simple_flow(memory_jobstore, clean_dir, simple_flow, capsys): assert "INFO Started executing jobs locally" not in captured.out assert "INFO Finished executing jobs locally" not in captured.out - # run with log + # run with custom log format + custom_fmt = "%(name)s: %(levelname)s - %(message)s" + run_locally(flow, store=memory_jobstore, log=custom_fmt) + stdout, stderr = capsys.readouterr() + assert "jobflow.managers.local: INFO - Started executing jobs locally" in stdout + assert "jobflow.managers.local: INFO - Finished executing jobs locally" in stdout + assert stderr == "" + + # run with log=True responses = run_locally(flow, store=memory_jobstore) # check responses has been filled From ddb53ddc8599ee7b237c6e4cf65bb70de537dd80 Mon Sep 17 00:00:00 2001 From: Janosh Riebesell Date: Sat, 16 Nov 2024 13:57:56 -0500 Subject: [PATCH 5/5] initialize_logger test custom log format --- tests/utils/test_log.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/tests/utils/test_log.py b/tests/utils/test_log.py index bb8309b8..30d75103 100644 --- a/tests/utils/test_log.py +++ b/tests/utils/test_log.py @@ -10,15 +10,27 @@ def test_initialize_logger(capsys): logger.info("123") logger.debug("ABC") - captured = capsys.readouterr() - assert "INFO 123" in captured.out - assert "DEBUG" not in captured.out + stdout, stderr = capsys.readouterr() + assert stdout.endswith("INFO 123\n") + assert stdout.count("DEBUG") == 0 + assert stderr == "" # initialize logger with debug level initialize_logger(level=logging.DEBUG) logger.info("123") + stdout, stderr = capsys.readouterr() + assert stdout.endswith("INFO 123\n") + logger.debug("ABC") + stdout, stderr = capsys.readouterr() + assert stdout.endswith("DEBUG ABC\n") + assert stderr == "" + + # test with custom format string + custom_fmt = "%(levelname)s - %(message)s" + initialize_logger(fmt=custom_fmt) + logger.info("custom format") - captured = capsys.readouterr() - assert "INFO 123" in captured.out - assert "DEBUG ABC" in captured.out + stdout, stderr = capsys.readouterr() + assert stdout == "INFO - custom format\n" + assert stderr == ""