From c5a2d7e4a52be328cea5d1a2552e1abd390c91bb Mon Sep 17 00:00:00 2001 From: ric-evans Date: Mon, 28 Oct 2024 17:28:31 -0500 Subject: [PATCH] give `client/` its own `ENV` --- .github/workflows/tests.yml | 12 ++++----- resources/launch_scripts/local-scan.sh | 2 +- skymap_scanner/client/__init__.py | 31 ++++++++++++++++++++++ skymap_scanner/client/reco_icetray.py | 10 +++---- skymap_scanner/recos/crash_dummy.py | 7 +++-- skymap_scanner/recos/millipede_original.py | 2 +- tests/env-vars.sh | 2 +- 7 files changed, 46 insertions(+), 20 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5db415f6c..477045faf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -367,11 +367,11 @@ jobs: export _RECO_ALGO=crash_dummy export _EVENTS_FILE=$(realpath $REALTIME_EVENTS_DIR/hese_event_01.json) export _NSIDES="1:0 2:12 4:12" - export SKYSCAN_CRASH_DUMMY_PROBABILITY=0.75 + export _SKYSCAN_CI_CRASH_DUMMY_PROBABILITY=0.75 export EWMS_PILOT_TASK_TIMEOUT=15 cd ./resources/launch_scripts - # since SKYSCAN_CRASH_DUMMY_PROBABILITY<1, this step will go forever + # since _SKYSCAN_CI_CRASH_DUMMY_PROBABILITY<1, this step will go forever # so, stop it after some time and chek that it has stderrfiles timeout 240 ./local-scan.sh $N_WORKERS $CI_TEST_RUN_STDOUT_STDERR_DIR || true - name: look at stderrfiles @@ -550,7 +550,7 @@ jobs: source tests/env-vars.sh docker run --rm -i \ - $(env | grep '^SKYSCAN_' | cut -d'=' -f1 | sed 's/^/--env /') \ + $(env | grep -E '^(SKYSCAN_|_SKYSCAN_)' | cut -d'=' -f1 | sed 's/^/--env /') \ $CI_DOCKER_IMAGE_TAG \ python tests/file_staging.py @@ -598,9 +598,9 @@ jobs: --shm-size=6gb \ --mount type=bind,source=$(readlink -f tests/data/reco_pixel_single/${{ matrix.reco_algo }}/${{ matrix.dir }}),target=/local/test-data \ --env PY_COLORS=1 \ - $(env | grep '^SKYSCAN_' | cut -d'=' -f1 | sed 's/^/--env /') \ + $(env | grep -E '^(SKYSCAN_|_SKYSCAN_)' | cut -d'=' -f1 | sed 's/^/--env /') \ $CI_DOCKER_IMAGE_TAG \ - python -m skymap_scanner.client.reco_icetray \ + python -m skymap_scanner.client \ --infile /local/test-data/in.json \ --client-startup-json /local/test-data/startup.json \ --outfile /local/test-data/out-actual.json @@ -617,7 +617,7 @@ jobs: --shm-size=6gb \ --mount type=bind,source=$(readlink -f tests/data/reco_pixel_single/${{ matrix.reco_algo }}/${{ matrix.dir }}),target=/local/test-data \ --env PY_COLORS=1 \ - $(env | grep '^SKYSCAN_' | cut -d'=' -f1 | sed 's/^/--env /') \ + $(env | grep -E '^(SKYSCAN_|_SKYSCAN_)' | cut -d'=' -f1 | sed 's/^/--env /') \ $CI_DOCKER_IMAGE_TAG \ python tests/compare_reco_pixel_single.py \ --actual /local/test-data/out-actual.json \ diff --git a/resources/launch_scripts/local-scan.sh b/resources/launch_scripts/local-scan.sh index ace5e48a0..a20143eca 100755 --- a/resources/launch_scripts/local-scan.sh +++ b/resources/launch_scripts/local-scan.sh @@ -80,7 +80,7 @@ else --mount type=bind,source="$CI_SKYSCAN_OUTPUT_DIR",target=/local/output \ --mount type=bind,source="$(dirname "$CI_SKYSCAN_STARTUP_JSON")",target=/local/startup \ --env PY_COLORS=1 \ - $(env | grep '^SKYSCAN_' | cut -d'=' -f1 | sed 's/^/--env /') \ + $(env | grep -E '^(SKYSCAN_|_SKYSCAN_)' | cut -d'=' -f1 | sed 's/^/--env /') \ $(env | grep '^EWMS_' | cut -d'=' -f1 | sed 's/^/--env /') \ "$CI_DOCKER_IMAGE_TAG" \ python -m skymap_scanner.server \ diff --git a/skymap_scanner/client/__init__.py b/skymap_scanner/client/__init__.py index e69de29bb..8408a5da0 100644 --- a/skymap_scanner/client/__init__.py +++ b/skymap_scanner/client/__init__.py @@ -0,0 +1,31 @@ +"""The Skymap Scanner Client.""" + +import dataclasses as dc + +from wipac_dev_tools import from_environment_as_dataclass + +from .. import config as cfg + + +# +# Env var constants: set as constants & typecast +# + + +@dc.dataclass(frozen=True) +class EnvConfig: + """For storing environment variables, typed.""" + + # LOGGING VARS + SKYSCAN_LOG: str = cfg.LOG_LEVEL_DEFAULT + SKYSCAN_LOG_THIRD_PARTY: str = cfg.LOG_THIRD_PARTY_LEVEL_DEFAULT + SKYSCAN_EWMS_PILOT_LOG: str = cfg.LOG_LEVEL_DEFAULT + SKYSCAN_MQ_CLIENT_LOG: str = cfg.LOG_LEVEL_DEFAULT + + # TESTING/DEBUG VARS + # NOTE - these are accessed via `os.getenv` -- ctrl+F for usages + # _SKYSCAN_CI_MINI_TEST: bool = False # run minimal variations for testing (mini-scale) + # _SKYSCAN_CI_CRASH_DUMMY_PROBABILITY: float = 0.5 # for reco algo: crash-dummy + + +ENV = from_environment_as_dataclass(EnvConfig) diff --git a/skymap_scanner/client/reco_icetray.py b/skymap_scanner/client/reco_icetray.py index 3fd8094dd..813c0b7fb 100644 --- a/skymap_scanner/client/reco_icetray.py +++ b/skymap_scanner/client/reco_icetray.py @@ -23,6 +23,7 @@ from wipac_dev_tools import argparse_tools, logging_tools from .. import config as cfg, recos +from . import ENV from ..utils import messages from ..utils.data_handling import get_gcd_datastager from ..utils.load_scan_state import get_baseline_gcd_frames @@ -260,9 +261,9 @@ def main() -> None: args = parser.parse_args() logging_tools.set_level( - os.getenv("SKYSCAN_LOG", cfg.LOG_LEVEL_DEFAULT), # type: ignore[arg-type] + ENV.SKYSCAN_LOG, # type: ignore[arg-type] first_party_loggers=__name__.split(".", maxsplit=1)[0], - third_party_level=os.getenv("SKYSCAN_LOG", cfg.LOG_THIRD_PARTY_LEVEL_DEFAULT), # type: ignore[arg-type] + third_party_level=ENV.SKYSCAN_LOG_THIRD_PARTY, # type: ignore[arg-type] formatter=logging_tools.WIPACDevToolsFormatter(), ) logging_tools.log_argparse_args(args, logger=LOGGER, level="WARNING") @@ -303,8 +304,3 @@ def main() -> None: args.outfile, ) LOGGER.info("Done reco'ing pixel.") - - -# This entrypoint is only used in CI testing -if __name__ == "__main__": - main() diff --git a/skymap_scanner/recos/crash_dummy.py b/skymap_scanner/recos/crash_dummy.py index 725e91302..dce870e89 100644 --- a/skymap_scanner/recos/crash_dummy.py +++ b/skymap_scanner/recos/crash_dummy.py @@ -21,11 +21,10 @@ def traysegment(tray, name, logger, **kwargs): def crash(frame): rand = random.random() - logger.debug( - f"crash probability: {os.getenv('SKYSCAN_CRASH_DUMMY_PROBABILITY')}" - ) + prob = float(os.getenv("_SKYSCAN_CI_CRASH_DUMMY_PROBABILITY", 0.5)) + logger.debug(f"crash probability: {prob}") - if rand < float(os.getenv("SKYSCAN_CRASH_DUMMY_PROBABILITY")): + if rand < prob: logger.debug(f"crash! {rand=}") # now, pick what to fail with diff --git a/skymap_scanner/recos/millipede_original.py b/skymap_scanner/recos/millipede_original.py index 0827f485b..a3fe63b6e 100644 --- a/skymap_scanner/recos/millipede_original.py +++ b/skymap_scanner/recos/millipede_original.py @@ -43,7 +43,7 @@ def get_vertex_variations() -> List[dataclasses.I3Position]: """ variation_distance = 20.*I3Units.m - if os.getenv('SKYSCAN_MINI_TEST'): + if os.getenv('_SKYSCAN_CI_MINI_TEST'): return VertexGenerator.mini_test(variation_distance=variation_distance) else: return VertexGenerator.octahedron(radius=variation_distance) diff --git a/tests/env-vars.sh b/tests/env-vars.sh index db235f775..66c3d374d 100755 --- a/tests/env-vars.sh +++ b/tests/env-vars.sh @@ -42,7 +42,7 @@ export EWMS_PILOT_OKAY_ERRORS=${EWMS_PILOT_OKAY_ERRORS:-"TimeoutError"} # this i export SKYSCAN_MQ_TIMEOUT_FROM_CLIENTS=${SKYSCAN_MQ_TIMEOUT_FROM_CLIENTS:-$((60 * 10))} # just need a big value -- only used to detect MIA workers (it isn't important in a successful scan) # other/misc # -> worker/client/pilot -export SKYSCAN_MINI_TEST=${SKYSCAN_MINI_TEST:-'yes'} +export _SKYSCAN_CI_MINI_TEST=${_SKYSCAN_CI_MINI_TEST:-'yes'} export SKYSCAN_LOG=${SKYSCAN_LOG:-"DEBUG"} export SKYSCAN_LOG_THIRD_PARTY=${SKYSCAN_LOG_THIRD_PARTY:-"INFO"} # -> worker/client/pilot