From 3750d1685587e157403b7a37ba3b7999b9b84c2e Mon Sep 17 00:00:00 2001 From: Ric Evans <19216225+ric-evans@users.noreply.github.com> Date: Fri, 22 Dec 2023 12:06:27 -0600 Subject: [PATCH] Use Local Loggers (#256) * Use Local Loggers * remove hardcoded loggers * set first party logger by `__name__` * add missing logger * remove unneeded `if-main` blocks * restore `if-main` block for testing * test formatting * pt-2 * pt-3 * pt-4 * remove `hostname` * format time * set handler after * move back, remove line numbers * remove `coloredlogs` dep * update dependencies*.log files(s) --------- Co-authored-by: wipacdevbot --- dependencies-from-Dockerfile.log | 4 --- dependencies-from-Dockerfile_no_cvmfs.log | 4 --- dependencies-from-Dockerfile_pulsar.log | 4 --- skymap_scanner/client/client.py | 7 ++-- skymap_scanner/client/reco_icetray.py | 4 ++- skymap_scanner/config.py | 13 ++++++-- skymap_scanner/server/__init__.py | 6 ---- skymap_scanner/server/collector.py | 5 ++- skymap_scanner/server/pixels.py | 9 ++++-- skymap_scanner/server/reporter.py | 6 ++-- skymap_scanner/server/start_scan.py | 11 +++---- skymap_scanner/server/utils.py | 3 +- skymap_scanner/utils/__init__.py | 6 ---- skymap_scanner/utils/data_handling.py | 13 +++++--- skymap_scanner/utils/extract_json_message.py | 7 ++-- skymap_scanner/utils/load_scan_state.py | 34 +++----------------- skymap_scanner/utils/prepare_frames.py | 27 ++++++++-------- skymap_scanner/utils/to_skyscan_result.py | 5 ++- skymap_scanner/utils/utils.py | 16 ++++++--- 19 files changed, 84 insertions(+), 100 deletions(-) diff --git a/dependencies-from-Dockerfile.log b/dependencies-from-Dockerfile.log index cad939a7f..8b0e7a772 100644 --- a/dependencies-from-Dockerfile.log +++ b/dependencies-from-Dockerfile.log @@ -23,7 +23,6 @@ cffi==1.15.0 chardet==4.0.0 click==8.0.3 colorama==0.4.4 -coloredlogs==15.0.1 configobj==5.0.6 cov-core==1.15.0 coverage==6.2 @@ -42,7 +41,6 @@ healpy==1.15.0 htchirp==3.0 htcondor==23.2.0 html5lib==1.1 -humanfriendly==10.0 hypothesis==6.36.0 icecube-skyreader==1.2.8 idna==3.3 @@ -174,8 +172,6 @@ cffi==1.15.0 chardet==4.0.0 click==8.0.3 colorama==0.4.4 -coloredlogs==15.0.1 -└── humanfriendly [required: >=9.1, installed: 10.0] configobj==5.0.6 cov-core==1.15.0 coverage==6.2 diff --git a/dependencies-from-Dockerfile_no_cvmfs.log b/dependencies-from-Dockerfile_no_cvmfs.log index 91a23fa89..733855553 100644 --- a/dependencies-from-Dockerfile_no_cvmfs.log +++ b/dependencies-from-Dockerfile_no_cvmfs.log @@ -23,7 +23,6 @@ cffi==1.15.0 chardet==4.0.0 click==8.0.3 colorama==0.4.4 -coloredlogs==15.0.1 configobj==5.0.6 cov-core==1.15.0 coverage==6.2 @@ -42,7 +41,6 @@ healpy==1.15.0 htchirp==3.0 htcondor==23.2.0 html5lib==1.1 -humanfriendly==10.0 hypothesis==6.36.0 icecube-skyreader==1.2.8 idna==3.3 @@ -174,8 +172,6 @@ cffi==1.15.0 chardet==4.0.0 click==8.0.3 colorama==0.4.4 -coloredlogs==15.0.1 -└── humanfriendly [required: >=9.1, installed: 10.0] configobj==5.0.6 cov-core==1.15.0 coverage==6.2 diff --git a/dependencies-from-Dockerfile_pulsar.log b/dependencies-from-Dockerfile_pulsar.log index c88b2ba84..33f944a79 100644 --- a/dependencies-from-Dockerfile_pulsar.log +++ b/dependencies-from-Dockerfile_pulsar.log @@ -23,7 +23,6 @@ cffi==1.15.0 chardet==4.0.0 click==8.0.3 colorama==0.4.4 -coloredlogs==15.0.1 configobj==5.0.6 cov-core==1.15.0 coverage==6.2 @@ -42,7 +41,6 @@ healpy==1.15.0 htchirp==3.0 htcondor==23.2.0 html5lib==1.1 -humanfriendly==10.0 hypothesis==6.36.0 icecube-skyreader==1.2.8 idna==3.3 @@ -174,8 +172,6 @@ cffi==1.15.0 chardet==4.0.0 click==8.0.3 colorama==0.4.4 -coloredlogs==15.0.1 -└── humanfriendly [required: >=9.1, installed: 10.0] configobj==5.0.6 cov-core==1.15.0 coverage==6.2 diff --git a/skymap_scanner/client/client.py b/skymap_scanner/client/client.py index 1fcbf0a27..5abd53731 100644 --- a/skymap_scanner/client/client.py +++ b/skymap_scanner/client/client.py @@ -1,5 +1,6 @@ """The Client service.""" + import argparse import asyncio import json @@ -11,7 +12,7 @@ from .. import config as cfg -LOGGER = logging.getLogger("skyscan.client") +LOGGER = logging.getLogger(__name__) def main() -> None: @@ -92,7 +93,3 @@ def main() -> None: ) ) LOGGER.info("Done.") - - -if __name__ == "__main__": - main() diff --git a/skymap_scanner/client/reco_icetray.py b/skymap_scanner/client/reco_icetray.py index 40603cdc9..c43ddadef 100644 --- a/skymap_scanner/client/reco_icetray.py +++ b/skymap_scanner/client/reco_icetray.py @@ -1,5 +1,6 @@ """Reco a single pixel.""" + # pylint: skip-file import argparse @@ -30,7 +31,7 @@ from ..utils.pixel_classes import RecoPixelVariation, pframe_tuple from ..utils.utils import save_GCD_frame_packet_to_file -LOGGER = logging.getLogger("skyscan.client.reco") +LOGGER = logging.getLogger(__name__) def frame_for_logging(frame: icetray.I3Frame) -> str: @@ -294,5 +295,6 @@ def main() -> None: LOGGER.info("Done reco'ing pixel.") +# This entrypoint is only used in CI testing if __name__ == "__main__": main() diff --git a/skymap_scanner/config.py b/skymap_scanner/config.py index c13d28d64..b46b2e01c 100644 --- a/skymap_scanner/config.py +++ b/skymap_scanner/config.py @@ -1,6 +1,8 @@ """Configuration constants.""" + import dataclasses as dc +import logging from pathlib import Path from typing import Final, List @@ -147,11 +149,18 @@ def __post_init__(self) -> None: def configure_loggers() -> None: """Set up loggers with common configurations.""" + hand = logging.StreamHandler() + hand.setFormatter( + logging.Formatter( + "%(asctime)s.%(msecs)03d [%(levelname)8s] %(name)s[%(process)d] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + ) + logging.getLogger().addHandler(hand) logging_tools.set_level( ENV.SKYSCAN_LOG, # type: ignore[arg-type] - first_party_loggers="skyscan", + first_party_loggers=__name__.split(".", maxsplit=1)[0], third_party_level=ENV.SKYSCAN_LOG_THIRD_PARTY, # type: ignore[arg-type] - use_coloredlogs=True, future_third_parties=["google", "pika"], specialty_loggers={ ewms_pilot.pilot.LOGGER: ENV.SKYSCAN_EWMS_PILOT_LOG, # type: ignore[attr-defined, dict-item] diff --git a/skymap_scanner/server/__init__.py b/skymap_scanner/server/__init__.py index 312759d72..e69de29bb 100644 --- a/skymap_scanner/server/__init__.py +++ b/skymap_scanner/server/__init__.py @@ -1,6 +0,0 @@ -"""Init.""" - - -import logging - -LOGGER = logging.getLogger("skyscan.server") diff --git a/skymap_scanner/server/collector.py b/skymap_scanner/server/collector.py index 0084d962b..c0960da22 100644 --- a/skymap_scanner/server/collector.py +++ b/skymap_scanner/server/collector.py @@ -2,6 +2,7 @@ import itertools +import logging import time from bisect import bisect from typing import Any, Dict, List, Optional, Set, Tuple @@ -16,9 +17,11 @@ RecoPixelVariation, SentPixelVariation, ) -from . import LOGGER from .reporter import Reporter +LOGGER = logging.getLogger(__name__) + + StrDict = Dict[str, Any] diff --git a/skymap_scanner/server/pixels.py b/skymap_scanner/server/pixels.py index b3ac9917a..fcd698b51 100644 --- a/skymap_scanner/server/pixels.py +++ b/skymap_scanner/server/pixels.py @@ -1,18 +1,21 @@ """Tools for picking pixels to scan.""" + # fmt: off # pylint: skip-file +import logging from typing import List, Optional, Set, Tuple -import healpy # type: ignore[import] +import healpy # type: ignore[import-untyped] import numpy -from icecube import icetray # type: ignore[import] +from icecube import icetray # type: ignore[import-not-found] from ..utils.pixel_classes import NSidesDict -from . import LOGGER from .utils import NSideProgression +LOGGER = logging.getLogger(__name__) + def __healpix_pixel_upgrade(nside, pix) -> list: pix_nested = healpy.ring2nest(nside, pix) diff --git a/skymap_scanner/server/reporter.py b/skymap_scanner/server/reporter.py index 3635a3335..f50239775 100644 --- a/skymap_scanner/server/reporter.py +++ b/skymap_scanner/server/reporter.py @@ -1,13 +1,12 @@ """Contains a class for reporting progress and result for a scan.""" -# pylint: disable=import-error - import bisect import dataclasses as dc import datetime as dt import itertools import json +import logging import math import statistics import time @@ -21,9 +20,10 @@ from ..utils import to_skyscan_result from ..utils.pixel_classes import NSidesDict from ..utils.utils import pyobj_to_string_repr -from . import LOGGER from .utils import NSideProgression, connect_to_skydriver, nonurgent_request +LOGGER = logging.getLogger(__name__) + StrDict = Dict[str, Any] diff --git a/skymap_scanner/server/start_scan.py b/skymap_scanner/server/start_scan.py index 9f6ded7e3..35917d2ad 100644 --- a/skymap_scanner/server/start_scan.py +++ b/skymap_scanner/server/start_scan.py @@ -1,5 +1,6 @@ """The Skymap Scanner Server.""" + # pylint: disable=invalid-name,import-error # fmt:quotes-ok @@ -36,7 +37,6 @@ SentPixelVariation, pframe_tuple, ) -from . import LOGGER from .collector import Collector, ExtraRecoPixelVariationException from .pixels import choose_pixels_to_reconstruct from .reporter import Reporter @@ -47,6 +47,9 @@ kill_switch_check_from_skydriver, ) +LOGGER = logging.getLogger(__name__) + + StrDict = Dict[str, Any] @@ -472,7 +475,7 @@ async def _serve_and_collect( msg["reco_pixel_variation"], msg["runtime"] ) except ExtraRecoPixelVariationException as e: - logging.error(e) + LOGGER.error(e) # if we've got enough pixfins, let's get a jump on the next round if max_nside_thresholded := collector.get_max_nside_thresholded(): @@ -747,7 +750,3 @@ def _nside_and_pixelextension(val: str) -> Tuple[int, int]: ) ) LOGGER.info("Done.") - - -if __name__ == "__main__": - main() diff --git a/skymap_scanner/server/utils.py b/skymap_scanner/server/utils.py index 954953d18..5f8c8c944 100644 --- a/skymap_scanner/server/utils.py +++ b/skymap_scanner/server/utils.py @@ -15,7 +15,8 @@ from rest_tools.client import CalcRetryFromWaittimeMax, RestClient from .. import config as cfg -from . import LOGGER + +LOGGER = logging.getLogger(__name__) def connect_to_skydriver(urgent: bool) -> RestClient: diff --git a/skymap_scanner/utils/__init__.py b/skymap_scanner/utils/__init__.py index 19fc4bb0a..e69de29bb 100644 --- a/skymap_scanner/utils/__init__.py +++ b/skymap_scanner/utils/__init__.py @@ -1,6 +0,0 @@ -"""Init.""" - - -import logging - -LOGGER = logging.getLogger("skyscan.utils") diff --git a/skymap_scanner/utils/data_handling.py b/skymap_scanner/utils/data_handling.py index f44f0e972..75732ef10 100644 --- a/skymap_scanner/utils/data_handling.py +++ b/skymap_scanner/utils/data_handling.py @@ -1,9 +1,14 @@ -from .. import config as cfg # type: ignore[import] -from pathlib import Path +"""data_handling.py.""" + + +import logging import subprocess -from typing import Dict, List, Union +from pathlib import Path +from typing import List + +from .. import config as cfg -from . import LOGGER +LOGGER = logging.getLogger(__name__) class DataStager: diff --git a/skymap_scanner/utils/extract_json_message.py b/skymap_scanner/utils/extract_json_message.py index b15ad03a1..92e0f71a6 100644 --- a/skymap_scanner/utils/extract_json_message.py +++ b/skymap_scanner/utils/extract_json_message.py @@ -1,17 +1,18 @@ """Tools for extracting json messages.""" + # fmt: off # pylint: skip-file import json +import logging import os from typing import Tuple -from icecube import full_event_followup, icetray # type: ignore[import] +from icecube import full_event_followup, icetray # type: ignore[import-not-found] from skyreader import EventMetadata from .. import config as cfg -from . import LOGGER from .load_scan_state import load_scan_state from .prepare_frames import prepare_frames from .utils import ( @@ -22,6 +23,8 @@ save_GCD_frame_packet_to_file, ) +LOGGER = logging.getLogger(__name__) + def extract_GCD_diff_base_filename(frame_packet): # check the base filename (it should be the same for all "Diff" objects) diff --git a/skymap_scanner/utils/load_scan_state.py b/skymap_scanner/utils/load_scan_state.py index f873ea7de..943ef497c 100644 --- a/skymap_scanner/utils/load_scan_state.py +++ b/skymap_scanner/utils/load_scan_state.py @@ -1,20 +1,22 @@ """Tools for loading the scan state.""" + # fmt: off -# mypy: ignore-errors # pylint: skip-file +import logging import os from typing import List -from icecube import icetray +from icecube import icetray # type: ignore[import-not-found] from skyreader import EventMetadata from .. import config as cfg -from . import LOGGER from .pixel_classes import RecoPixelFinal, RecoPixelVariation from .utils import hash_frame_packet, load_framepacket_from_file +LOGGER = logging.getLogger(__name__) + def load_cache_state( event_metadata: EventMetadata, @@ -179,29 +181,3 @@ def load_GCDQp_state(event_metadata: EventMetadata, cache_dir="./cache/") -> dic cfg.STATEDICT_GCDQP_PACKET: frame_packet, cfg.STATEDICT_BASELINE_GCD_FILE: source_baseline_GCD, } - - -if __name__ == "__main__": - from optparse import OptionParser - - parser = OptionParser() - usage = """%prog [options]""" - parser.set_usage(usage) - parser.add_option("-c", "--cache-dir", action="store", type="string", - default="./cache/", dest="CACHEDIR", help="The cache directory to use") - - # get parsed args - (options,args) = parser.parse_args() - - if len(args) != 1: - raise RuntimeError("You need to specify exatcly one event ID") - eventID = args[0] - - # do the work - packets = load_cache_state( - eventID, - args.reco_algo, # TODO: add --reco-algo (see start_scan.py) - cache_dir=options.CACHEDIR - ) - - print(("got:", packets)) diff --git a/skymap_scanner/utils/prepare_frames.py b/skymap_scanner/utils/prepare_frames.py index f7155652a..02b02ddc9 100644 --- a/skymap_scanner/utils/prepare_frames.py +++ b/skymap_scanner/utils/prepare_frames.py @@ -1,25 +1,26 @@ -""" -prepare the GCDQp packet by adding frame objects that might be missing -""" +"""prepare the GCDQp packet by adding frame objects that might be missing.""" + # fmt: off # pylint: skip-file import copy +import logging import os -from typing import Union, List - -from icecube.icetray import I3Tray # type: ignore[import] -from icecube import icetray # type: ignore[import] -from icecube.frame_object_diff.segments import uncompress # type: ignore[import] +from typing import List, Union -from icecube import ( - DomTools, # for I3LCPulseCleaning -) # type: ignore[import] +from icecube import icetray # type: ignore[import-not-found] +from icecube import ( # type: ignore[import-not-found] # for I3LCPulseCleaning # noqa: F401 + DomTools, +) +from icecube.frame_object_diff.segments import ( # type: ignore[import-not-found] + uncompress, +) +from icecube.icetray import I3Tray # type: ignore[import-not-found] -from .. import config as cfg from .. import recos -from . import LOGGER + +LOGGER = logging.getLogger(__name__) class FrameArraySource(icetray.I3Module): diff --git a/skymap_scanner/utils/to_skyscan_result.py b/skymap_scanner/utils/to_skyscan_result.py index 0e4614f02..967b2fa0c 100644 --- a/skymap_scanner/utils/to_skyscan_result.py +++ b/skymap_scanner/utils/to_skyscan_result.py @@ -2,14 +2,17 @@ import dataclasses as dc +import logging from typing import Optional, Tuple import numpy as np from skyreader import EventMetadata, SkyScanResult -from . import LOGGER from .pixel_classes import NSidesDict, RecoPixelFinal +LOGGER = logging.getLogger(__name__) + + PixelTuple = Tuple[int, float, float, float] diff --git a/skymap_scanner/utils/utils.py b/skymap_scanner/utils/utils.py index 99d8c8b58..49f56531b 100644 --- a/skymap_scanner/utils/utils.py +++ b/skymap_scanner/utils/utils.py @@ -1,16 +1,23 @@ """Generic utility functions.""" + # fmt: off # pylint: skip-file import hashlib import json +import logging from pprint import pformat from typing import Any, List, Optional, Tuple -from icecube import astro, dataclasses, dataio, icetray # type: ignore[import] +from icecube import ( # type: ignore[import-not-found] + astro, + dataclasses, + dataio, + icetray, +) -from . import LOGGER +LOGGER = logging.getLogger(__name__) def pyobj_to_string_repr(obj: Any) -> str: @@ -34,9 +41,8 @@ def get_event_mjd(frame_packet: List[icetray.I3Frame]) -> float: def load_framepacket_from_file(filename : str) -> List[icetray.I3Frame]: - """ - Loads an I3 file provided a filename and returns a list of I3Frame objects (frame packet) - """ + """Loads an I3 file provided a filename and returns a list of I3Frame + objects (frame packet)""" # Legacy code used to loop over GCD_BASE_DIRS. # Now it is assumed that filename points to a valid GCD file. frame_packet: icetray.I3Frame = []