give client/ its own ENV

.github/workflows/tests.yml

@@ -367,11 +367,11 @@ jobs:
           export _RECO_ALGO=crash_dummy
           export _EVENTS_FILE=$(realpath $REALTIME_EVENTS_DIR/hese_event_01.json)
           export _NSIDES="1:0 2:12 4:12"
-          export SKYSCAN_CRASH_DUMMY_PROBABILITY=0.75
+          export _SKYSCAN_CI_CRASH_DUMMY_PROBABILITY=0.75
           export EWMS_PILOT_TASK_TIMEOUT=15
 
           cd ./resources/launch_scripts
-          # since SKYSCAN_CRASH_DUMMY_PROBABILITY<1, this step will go forever
+          # since _SKYSCAN_CI_CRASH_DUMMY_PROBABILITY<1, this step will go forever
           #   so, stop it after some time and chek that it has stderrfiles
           timeout 240 ./local-scan.sh $N_WORKERS $CI_TEST_RUN_STDOUT_STDERR_DIR  || true
       - name: look at stderrfiles
@@ -550,7 +550,7 @@ jobs:
           source tests/env-vars.sh
           
           docker run --rm -i \
-            $(env | grep '^SKYSCAN_' | cut -d'=' -f1 | sed 's/^/--env /') \
+            $(env | grep -E '^(SKYSCAN_|_SKYSCAN_)' | cut -d'=' -f1 | sed 's/^/--env /') \
             $CI_DOCKER_IMAGE_TAG \
             python tests/file_staging.py
 
@@ -598,9 +598,9 @@ jobs:
             --shm-size=6gb \
             --mount type=bind,source=$(readlink -f tests/data/reco_pixel_single/${{ matrix.reco_algo }}/${{ matrix.dir }}),target=/local/test-data \
             --env PY_COLORS=1 \
-            $(env | grep '^SKYSCAN_' | cut -d'=' -f1 | sed 's/^/--env /') \
+            $(env | grep -E '^(SKYSCAN_|_SKYSCAN_)' | cut -d'=' -f1 | sed 's/^/--env /') \
             $CI_DOCKER_IMAGE_TAG \
-            python -m skymap_scanner.client.reco_icetray \
+            python -m skymap_scanner.client \
             --infile /local/test-data/in.json \
             --client-startup-json /local/test-data/startup.json \
             --outfile /local/test-data/out-actual.json
@@ -617,7 +617,7 @@ jobs:
             --shm-size=6gb \
             --mount type=bind,source=$(readlink -f tests/data/reco_pixel_single/${{ matrix.reco_algo }}/${{ matrix.dir }}),target=/local/test-data \
             --env PY_COLORS=1 \
-            $(env | grep '^SKYSCAN_' | cut -d'=' -f1 | sed 's/^/--env /') \
+            $(env | grep -E '^(SKYSCAN_|_SKYSCAN_)' | cut -d'=' -f1 | sed 's/^/--env /') \
             $CI_DOCKER_IMAGE_TAG \
             python tests/compare_reco_pixel_single.py \
             --actual /local/test-data/out-actual.json \

resources/launch_scripts/local-scan.sh

@@ -80,7 +80,7 @@ else
         --mount type=bind,source="$CI_SKYSCAN_OUTPUT_DIR",target=/local/output \
         --mount type=bind,source="$(dirname "$CI_SKYSCAN_STARTUP_JSON")",target=/local/startup \
         --env PY_COLORS=1 \
-        $(env | grep '^SKYSCAN_' | cut -d'=' -f1 | sed 's/^/--env /') \
+        $(env | grep -E '^(SKYSCAN_|_SKYSCAN_)' | cut -d'=' -f1 | sed 's/^/--env /') \
         $(env | grep '^EWMS_' | cut -d'=' -f1 | sed 's/^/--env /') \
         "$CI_DOCKER_IMAGE_TAG" \
         python -m skymap_scanner.server \

skymap_scanner/client/__init__.py

@@ -0,0 +1,31 @@
+"""The Skymap Scanner Client."""
+
+import dataclasses as dc
+
+from wipac_dev_tools import from_environment_as_dataclass
+
+from .. import config as cfg
+
+
+#
+# Env var constants: set as constants & typecast
+#
+
+
+@dc.dataclass(frozen=True)
+class EnvConfig:
+    """For storing environment variables, typed."""
+
+    # LOGGING VARS
+    SKYSCAN_LOG: str = cfg.LOG_LEVEL_DEFAULT
+    SKYSCAN_LOG_THIRD_PARTY: str = cfg.LOG_THIRD_PARTY_LEVEL_DEFAULT
+    SKYSCAN_EWMS_PILOT_LOG: str = cfg.LOG_LEVEL_DEFAULT
+    SKYSCAN_MQ_CLIENT_LOG: str = cfg.LOG_LEVEL_DEFAULT
+
+    # TESTING/DEBUG VARS
+    # NOTE - these are accessed via `os.getenv` -- ctrl+F for usages
+    # _SKYSCAN_CI_MINI_TEST: bool = False  # run minimal variations for testing (mini-scale)
+    # _SKYSCAN_CI_CRASH_DUMMY_PROBABILITY: float = 0.5  # for reco algo: crash-dummy
+
+
+ENV = from_environment_as_dataclass(EnvConfig)

skymap_scanner/client/reco_icetray.py

@@ -23,6 +23,7 @@
 from wipac_dev_tools import argparse_tools, logging_tools
 
 from .. import config as cfg, recos
+from . import ENV
 from ..utils import messages
 from ..utils.data_handling import get_gcd_datastager
 from ..utils.load_scan_state import get_baseline_gcd_frames
@@ -260,9 +261,9 @@ def main() -> None:
 
     args = parser.parse_args()
     logging_tools.set_level(
-        os.getenv("SKYSCAN_LOG", cfg.LOG_LEVEL_DEFAULT),  # type: ignore[arg-type]
+        ENV.SKYSCAN_LOG,  # type: ignore[arg-type]
         first_party_loggers=__name__.split(".", maxsplit=1)[0],
-        third_party_level=os.getenv("SKYSCAN_LOG", cfg.LOG_THIRD_PARTY_LEVEL_DEFAULT),  # type: ignore[arg-type]
+        third_party_level=ENV.SKYSCAN_LOG_THIRD_PARTY,  # type: ignore[arg-type]
         formatter=logging_tools.WIPACDevToolsFormatter(),
     )
     logging_tools.log_argparse_args(args, logger=LOGGER, level="WARNING")
@@ -303,8 +304,3 @@ def main() -> None:
         args.outfile,
     )
     LOGGER.info("Done reco'ing pixel.")
-
-
-# This entrypoint is only used in CI testing
-if __name__ == "__main__":
-    main()

skymap_scanner/recos/crash_dummy.py

@@ -21,11 +21,10 @@ def traysegment(tray, name, logger, **kwargs):
 
         def crash(frame):
             rand = random.random()
-            logger.debug(
-                f"crash probability: {os.getenv('SKYSCAN_CRASH_DUMMY_PROBABILITY')}"
-            )
+            prob = float(os.getenv("_SKYSCAN_CI_CRASH_DUMMY_PROBABILITY", 0.5))
+            logger.debug(f"crash probability: {prob}")
 
-            if rand < float(os.getenv("SKYSCAN_CRASH_DUMMY_PROBABILITY")):
+            if rand < prob:
                 logger.debug(f"crash! {rand=}")
 
                 # now, pick what to fail with

skymap_scanner/recos/millipede_original.py

@@ -43,7 +43,7 @@ def get_vertex_variations() -> List[dataclasses.I3Position]:
         """
         variation_distance = 20.*I3Units.m
 
-        if os.getenv('SKYSCAN_MINI_TEST'):
+        if os.getenv('_SKYSCAN_CI_MINI_TEST'):
             return VertexGenerator.mini_test(variation_distance=variation_distance)
         else:    
             return VertexGenerator.octahedron(radius=variation_distance)

tests/env-vars.sh

@@ -42,7 +42,7 @@ export EWMS_PILOT_OKAY_ERRORS=${EWMS_PILOT_OKAY_ERRORS:-"TimeoutError"} # this i
 export SKYSCAN_MQ_TIMEOUT_FROM_CLIENTS=${SKYSCAN_MQ_TIMEOUT_FROM_CLIENTS:-$((60 * 10))} # just need a big value -- only used to detect MIA workers (it isn't important in a successful scan)
 # other/misc
 # -> worker/client/pilot
-export SKYSCAN_MINI_TEST=${SKYSCAN_MINI_TEST:-'yes'}
+export _SKYSCAN_CI_MINI_TEST=${_SKYSCAN_CI_MINI_TEST:-'yes'}
 export SKYSCAN_LOG=${SKYSCAN_LOG:-"DEBUG"}
 export SKYSCAN_LOG_THIRD_PARTY=${SKYSCAN_LOG_THIRD_PARTY:-"INFO"}
 # -> worker/client/pilot