Use Local Loggers (#256)

* Use Local Loggers * remove hardcoded loggers * set first party logger by `__name__` * add missing logger * remove unneeded `if-main` blocks * restore `if-main` block for testing * test formatting * pt-2 * pt-3 * pt-4 * remove `hostname` * format time * set handler after * move back, remove line numbers * remove `coloredlogs` dep * <bot> update dependencies*.log files(s) --------- Co-authored-by: wipacdevbot <[email protected]>
icecube · Dec 22, 2023 · 3750d16 · 3750d16
1 parent fd1a0b7
commit 3750d16
Show file tree

Hide file tree

Showing 19 changed files with 84 additions and 100 deletions.
diff --git a/dependencies-from-Dockerfile.log b/dependencies-from-Dockerfile.log
@@ -23,7 +23,6 @@ cffi==1.15.0
 chardet==4.0.0
 click==8.0.3
 colorama==0.4.4
-coloredlogs==15.0.1
 configobj==5.0.6
 cov-core==1.15.0
 coverage==6.2
@@ -42,7 +41,6 @@ healpy==1.15.0
 htchirp==3.0
 htcondor==23.2.0
 html5lib==1.1
-humanfriendly==10.0
 hypothesis==6.36.0
 icecube-skyreader==1.2.8
 idna==3.3
@@ -174,8 +172,6 @@ cffi==1.15.0
 chardet==4.0.0
 click==8.0.3
 colorama==0.4.4
-coloredlogs==15.0.1
-└── humanfriendly [required: >=9.1, installed: 10.0]
 configobj==5.0.6
 cov-core==1.15.0
 coverage==6.2

diff --git a/dependencies-from-Dockerfile_no_cvmfs.log b/dependencies-from-Dockerfile_no_cvmfs.log
@@ -23,7 +23,6 @@ cffi==1.15.0
 chardet==4.0.0
 click==8.0.3
 colorama==0.4.4
-coloredlogs==15.0.1
 configobj==5.0.6
 cov-core==1.15.0
 coverage==6.2
@@ -42,7 +41,6 @@ healpy==1.15.0
 htchirp==3.0
 htcondor==23.2.0
 html5lib==1.1
-humanfriendly==10.0
 hypothesis==6.36.0
 icecube-skyreader==1.2.8
 idna==3.3
@@ -174,8 +172,6 @@ cffi==1.15.0
 chardet==4.0.0
 click==8.0.3
 colorama==0.4.4
-coloredlogs==15.0.1
-└── humanfriendly [required: >=9.1, installed: 10.0]
 configobj==5.0.6
 cov-core==1.15.0
 coverage==6.2

diff --git a/dependencies-from-Dockerfile_pulsar.log b/dependencies-from-Dockerfile_pulsar.log
@@ -23,7 +23,6 @@ cffi==1.15.0
 chardet==4.0.0
 click==8.0.3
 colorama==0.4.4
-coloredlogs==15.0.1
 configobj==5.0.6
 cov-core==1.15.0
 coverage==6.2
@@ -42,7 +41,6 @@ healpy==1.15.0
 htchirp==3.0
 htcondor==23.2.0
 html5lib==1.1
-humanfriendly==10.0
 hypothesis==6.36.0
 icecube-skyreader==1.2.8
 idna==3.3
@@ -174,8 +172,6 @@ cffi==1.15.0
 chardet==4.0.0
 click==8.0.3
 colorama==0.4.4
-coloredlogs==15.0.1
-└── humanfriendly [required: >=9.1, installed: 10.0]
 configobj==5.0.6
 cov-core==1.15.0
 coverage==6.2

diff --git a/skymap_scanner/client/client.py b/skymap_scanner/client/client.py
@@ -1,5 +1,6 @@
 """The Client service."""
 
+
 import argparse
 import asyncio
 import json
@@ -11,7 +12,7 @@
 
 from .. import config as cfg
 
-LOGGER = logging.getLogger("skyscan.client")
+LOGGER = logging.getLogger(__name__)
 
 
 def main() -> None:
@@ -92,7 +93,3 @@ def main() -> None:
         )
     )
     LOGGER.info("Done.")
-
-
-if __name__ == "__main__":
-    main()
diff --git a/skymap_scanner/client/reco_icetray.py b/skymap_scanner/client/reco_icetray.py
@@ -1,5 +1,6 @@
 """Reco a single pixel."""
 
+
 # pylint: skip-file
 
 import argparse
@@ -30,7 +31,7 @@
 from ..utils.pixel_classes import RecoPixelVariation, pframe_tuple
 from ..utils.utils import save_GCD_frame_packet_to_file
 
-LOGGER = logging.getLogger("skyscan.client.reco")
+LOGGER = logging.getLogger(__name__)
 
 
 def frame_for_logging(frame: icetray.I3Frame) -> str:
@@ -294,5 +295,6 @@ def main() -> None:
     LOGGER.info("Done reco'ing pixel.")
 
 
+# This entrypoint is only used in CI testing
 if __name__ == "__main__":
     main()
diff --git a/skymap_scanner/config.py b/skymap_scanner/config.py
@@ -1,6 +1,8 @@
 """Configuration constants."""
 
+
 import dataclasses as dc
+import logging
 from pathlib import Path
 from typing import Final, List
 
@@ -147,11 +149,18 @@ def __post_init__(self) -> None:
 
 def configure_loggers() -> None:
     """Set up loggers with common configurations."""
+    hand = logging.StreamHandler()
+    hand.setFormatter(
+        logging.Formatter(
+            "%(asctime)s.%(msecs)03d [%(levelname)8s] %(name)s[%(process)d] %(message)s",
+            datefmt="%Y-%m-%d %H:%M:%S",
+        )
+    )
+    logging.getLogger().addHandler(hand)
     logging_tools.set_level(
         ENV.SKYSCAN_LOG,  # type: ignore[arg-type]
-        first_party_loggers="skyscan",
+        first_party_loggers=__name__.split(".", maxsplit=1)[0],
         third_party_level=ENV.SKYSCAN_LOG_THIRD_PARTY,  # type: ignore[arg-type]
-        use_coloredlogs=True,
         future_third_parties=["google", "pika"],
         specialty_loggers={
             ewms_pilot.pilot.LOGGER: ENV.SKYSCAN_EWMS_PILOT_LOG,  # type: ignore[attr-defined, dict-item]

diff --git a/skymap_scanner/server/__init__.py b/skymap_scanner/server/__init__.py
@@ -1,6 +0,0 @@
-"""Init."""
-
-
-import logging
-
-LOGGER = logging.getLogger("skyscan.server")

diff --git a/skymap_scanner/server/collector.py b/skymap_scanner/server/collector.py
@@ -2,6 +2,7 @@
 
 
 import itertools
+import logging
 import time
 from bisect import bisect
 from typing import Any, Dict, List, Optional, Set, Tuple
@@ -16,9 +17,11 @@
     RecoPixelVariation,
     SentPixelVariation,
 )
-from . import LOGGER
 from .reporter import Reporter
 
+LOGGER = logging.getLogger(__name__)
+
+
 StrDict = Dict[str, Any]
 
 

diff --git a/skymap_scanner/server/pixels.py b/skymap_scanner/server/pixels.py
@@ -1,18 +1,21 @@
 """Tools for picking pixels to scan."""
 
+
 # fmt: off
 # pylint: skip-file
 
+import logging
 from typing import List, Optional, Set, Tuple
 
-import healpy  # type: ignore[import]
+import healpy  # type: ignore[import-untyped]
 import numpy
-from icecube import icetray  # type: ignore[import]
+from icecube import icetray  # type: ignore[import-not-found]
 
 from ..utils.pixel_classes import NSidesDict
-from . import LOGGER
 from .utils import NSideProgression
 
+LOGGER = logging.getLogger(__name__)
+
 
 def __healpix_pixel_upgrade(nside, pix) -> list:
     pix_nested = healpy.ring2nest(nside, pix)

diff --git a/skymap_scanner/server/reporter.py b/skymap_scanner/server/reporter.py
@@ -1,13 +1,12 @@
 """Contains a class for reporting progress and result for a scan."""
 
-# pylint: disable=import-error
-
 
 import bisect
 import dataclasses as dc
 import datetime as dt
 import itertools
 import json
+import logging
 import math
 import statistics
 import time
@@ -21,9 +20,10 @@
 from ..utils import to_skyscan_result
 from ..utils.pixel_classes import NSidesDict
 from ..utils.utils import pyobj_to_string_repr
-from . import LOGGER
 from .utils import NSideProgression, connect_to_skydriver, nonurgent_request
 
+LOGGER = logging.getLogger(__name__)
+
 StrDict = Dict[str, Any]
 
 

diff --git a/skymap_scanner/server/start_scan.py b/skymap_scanner/server/start_scan.py
@@ -1,5 +1,6 @@
 """The Skymap Scanner Server."""
 
+
 # pylint: disable=invalid-name,import-error
 # fmt:quotes-ok
 
@@ -36,7 +37,6 @@
     SentPixelVariation,
     pframe_tuple,
 )
-from . import LOGGER
 from .collector import Collector, ExtraRecoPixelVariationException
 from .pixels import choose_pixels_to_reconstruct
 from .reporter import Reporter
@@ -47,6 +47,9 @@
     kill_switch_check_from_skydriver,
 )
 
+LOGGER = logging.getLogger(__name__)
+
+
 StrDict = Dict[str, Any]
 
 
@@ -472,7 +475,7 @@ async def _serve_and_collect(
                             msg["reco_pixel_variation"], msg["runtime"]
                         )
                     except ExtraRecoPixelVariationException as e:
-                        logging.error(e)
+                        LOGGER.error(e)
 
                     # if we've got enough pixfins, let's get a jump on the next round
                     if max_nside_thresholded := collector.get_max_nside_thresholded():
@@ -747,7 +750,3 @@ def _nside_and_pixelextension(val: str) -> Tuple[int, int]:
         )
     )
     LOGGER.info("Done.")
-
-
-if __name__ == "__main__":
-    main()
diff --git a/skymap_scanner/server/utils.py b/skymap_scanner/server/utils.py
@@ -15,7 +15,8 @@
 from rest_tools.client import CalcRetryFromWaittimeMax, RestClient
 
 from .. import config as cfg
-from . import LOGGER
+
+LOGGER = logging.getLogger(__name__)
 
 
 def connect_to_skydriver(urgent: bool) -> RestClient:

diff --git a/skymap_scanner/utils/__init__.py b/skymap_scanner/utils/__init__.py
@@ -1,6 +0,0 @@
-"""Init."""
-
-
-import logging
-
-LOGGER = logging.getLogger("skyscan.utils")

diff --git a/skymap_scanner/utils/data_handling.py b/skymap_scanner/utils/data_handling.py
@@ -1,9 +1,14 @@
-from .. import config as cfg  # type: ignore[import]
-from pathlib import Path
+"""data_handling.py."""
+
+
+import logging
 import subprocess
-from typing import Dict, List, Union
+from pathlib import Path
+from typing import List
+
+from .. import config as cfg
 
-from . import LOGGER
+LOGGER = logging.getLogger(__name__)
 
 
 class DataStager:

diff --git a/skymap_scanner/utils/extract_json_message.py b/skymap_scanner/utils/extract_json_message.py
@@ -1,17 +1,18 @@
 """Tools for extracting json messages."""
 
+
 # fmt: off
 # pylint: skip-file
 
 import json
+import logging
 import os
 from typing import Tuple
 
-from icecube import full_event_followup, icetray  # type: ignore[import]
+from icecube import full_event_followup, icetray  # type: ignore[import-not-found]
 from skyreader import EventMetadata
 
 from .. import config as cfg
-from . import LOGGER
 from .load_scan_state import load_scan_state
 from .prepare_frames import prepare_frames
 from .utils import (
@@ -22,6 +23,8 @@
     save_GCD_frame_packet_to_file,
 )
 
+LOGGER = logging.getLogger(__name__)
+
 
 def extract_GCD_diff_base_filename(frame_packet):
     # check the base filename (it should be the same for all "Diff" objects)

diff --git a/skymap_scanner/utils/load_scan_state.py b/skymap_scanner/utils/load_scan_state.py
@@ -1,20 +1,22 @@
 """Tools for loading the scan state."""
 
+
 # fmt: off
-# mypy: ignore-errors
 # pylint: skip-file
 
+import logging
 import os
 from typing import List
 
-from icecube import icetray
+from icecube import icetray  # type: ignore[import-not-found]
 from skyreader import EventMetadata
 
 from .. import config as cfg
-from . import LOGGER
 from .pixel_classes import RecoPixelFinal, RecoPixelVariation
 from .utils import hash_frame_packet, load_framepacket_from_file
 
+LOGGER = logging.getLogger(__name__)
+
 
 def load_cache_state(
     event_metadata: EventMetadata,
@@ -179,29 +181,3 @@ def load_GCDQp_state(event_metadata: EventMetadata, cache_dir="./cache/") -> dic
         cfg.STATEDICT_GCDQP_PACKET: frame_packet,
         cfg.STATEDICT_BASELINE_GCD_FILE: source_baseline_GCD,
     }
-
-
-if __name__ == "__main__":
-    from optparse import OptionParser
-
-    parser = OptionParser()
-    usage = """%prog [options]"""
-    parser.set_usage(usage)
-    parser.add_option("-c", "--cache-dir", action="store", type="string",
-        default="./cache/", dest="CACHEDIR", help="The cache directory to use")
-
-    # get parsed args
-    (options,args) = parser.parse_args()
-
-    if len(args) != 1:
-        raise RuntimeError("You need to specify exatcly one event ID")
-    eventID = args[0]
-
-    # do the work
-    packets = load_cache_state(
-        eventID,
-        args.reco_algo,  # TODO: add --reco-algo (see start_scan.py)
-        cache_dir=options.CACHEDIR
-    )
-
-    print(("got:", packets))