From 5d25a3647552660c9d25b099ef9c03d7b8147017 Mon Sep 17 00:00:00 2001 From: Ric Evans Date: Thu, 28 Sep 2023 10:12:30 -0500 Subject: [PATCH 1/3] unpin mq deps --- dependencies-from-Dockerfile.log | 4 ++-- dependencies-from-Dockerfile_pulsar.log | 4 ++-- setup.cfg | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dependencies-from-Dockerfile.log b/dependencies-from-Dockerfile.log index 644f3ef98..aece7d2c7 100644 --- a/dependencies-from-Dockerfile.log +++ b/dependencies-from-Dockerfile.log @@ -26,7 +26,7 @@ decorator==4.4.2 defusedxml==0.7.1 docutils==0.20.1 et-xmlfile==1.0.1 -ewms-pilot==0.12.2 +ewms-pilot==0.14.0 fonttools==4.29.1 fs==2.4.12 gast==0.5.2 @@ -64,7 +64,7 @@ numexpr==2.8.1 numpy==1.21.5 odfpy==1.4.2 olefile==0.46 -oms-mqclient==2.4.4 +oms-mqclient==2.4.6 openpyxl==3.0.9 packaging==21.3 pandas==1.3.5 diff --git a/dependencies-from-Dockerfile_pulsar.log b/dependencies-from-Dockerfile_pulsar.log index 52fdf8ad0..bd3a86e5f 100644 --- a/dependencies-from-Dockerfile_pulsar.log +++ b/dependencies-from-Dockerfile_pulsar.log @@ -26,7 +26,7 @@ decorator==4.4.2 defusedxml==0.7.1 docutils==0.20.1 et-xmlfile==1.0.1 -ewms-pilot==0.12.2 +ewms-pilot==0.14.0 fonttools==4.29.1 fs==2.4.12 gast==0.5.2 @@ -63,7 +63,7 @@ numexpr==2.8.1 numpy==1.21.5 odfpy==1.4.2 olefile==0.46 -oms-mqclient==2.4.4 +oms-mqclient==2.4.6 openpyxl==3.0.9 packaging==21.3 pandas==1.3.5 diff --git a/setup.cfg b/setup.cfg index ca115999b..18fc5ca9f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,12 +18,12 @@ branch = main [options] # generated by wipac:cicd_setup_builder: python_requires, packages install_requires = - ewms-pilot==0.12.2 # the extras here are the same as 'oms-mqclient', so just define one place + ewms-pilot # the extras here are the same as 'oms-mqclient', so just define one place healpy icecube-skyreader iminuit numpy - oms-mqclient==2.4.4 + oms-mqclient wipac-dev-tools[coloredlogs] wipac-rest-tools python_requires = >=3.9, <3.12 From 733b3e7cabf9375083f0f65ff0bf0cc5662b74c9 Mon Sep 17 00:00:00 2001 From: Ric Evans Date: Thu, 28 Sep 2023 10:14:19 -0500 Subject: [PATCH 2/3] add `reco-icetray logs` step --- .github/workflows/tests.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9a1307ab3..a59792e2c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -289,6 +289,14 @@ jobs: nsides = "${{ matrix.nsides }}" assert len(pydict) == nsides.count(":") ' + + - name: reco-icetray logs + if: always() + run: | + sudo apt install tree + tree $SKYSCAN_DEBUG_DIR + find $SKYSCAN_DEBUG_DIR -type f -not -name "*.i3" -not -name "*.pkl" -exec "more" {} + | cat # recursively cats with filenames (delimited by :::::::) + - name: broker docker logs if: always() run: | @@ -379,6 +387,14 @@ jobs: --expected tests/data/results_json/${{ matrix.reco_algo }}/$(basename $outfile) \ --assert \ || (cat $(ls *.diff.json) && false) + + - name: reco-icetray logs + if: always() + run: | + sudo apt install tree + tree $SKYSCAN_DEBUG_DIR + find $SKYSCAN_DEBUG_DIR -type f -not -name "*.i3" -not -name "*.pkl" -exec "more" {} + | cat # recursively cats with filenames (delimited by :::::::) + - name: broker docker logs if: always() run: | From c4a40a1cefe8644e033abdeb36e4ceb720466e14 Mon Sep 17 00:00:00 2001 From: Ric Evans Date: Thu, 28 Sep 2023 10:25:13 -0500 Subject: [PATCH 3/3] update env vars --- .github/workflows/tests.yml | 1 + tests/env-vars.sh | 29 +++++++++++++++++------------ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a59792e2c..13045afc1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -23,6 +23,7 @@ env: SKYSCAN_CACHE_DIR: /home/runner/work/skymap_scanner/skymap_scanner/cache SKYSCAN_OUTPUT_DIR: /home/runner/work/skymap_scanner/skymap_scanner/output SKYSCAN_DEBUG_DIR: /home/runner/work/skymap_scanner/skymap_scanner/debug + EWMS_PILOT_DUMP_SUBPROC_OUTPUT: False # get logs in "reco-icetray logs" step instead # see source tests/env-vars.sh diff --git a/tests/env-vars.sh b/tests/env-vars.sh index 1fcc9188a..6150b0c99 100755 --- a/tests/env-vars.sh +++ b/tests/env-vars.sh @@ -3,18 +3,23 @@ set -ex # file is sourced so turn off at end # export SKYSCAN_CACHE_DIR=$PWD/cache-dir -- rely on user value # export SKYSCAN_OUTPUT_DIR=$PWD/output-dir -- rely on user value -export SKYSCAN_BROKER_CLIENT=rabbitmq +export SKYSCAN_BROKER_CLIENT=${SKYSCAN_BROKER_CLIENT:-"rabbitmq"} # note=auth env vars are in job(s) -export EWMS_PILOT_TASK_TIMEOUT=1800 + +export EWMS_PILOT_TASK_TIMEOUT=${EWMS_PILOT_TASK_TIMEOUT:-1800} +export EWMS_PILOT_DUMP_SUBPROC_OUTPUT=${EWMS_PILOT_DUMP_SUBPROC_OUTPUT:-"True"} + # export SKYSCAN_DEBUG_DIR=debug-pkl-dir -- rely on user value -export SKYSCAN_MQ_TIMEOUT_TO_CLIENTS=5 -export SKYSCAN_MQ_TIMEOUT_FROM_CLIENTS=1800 -# export SKYSCAN_MQ_CLIENT_TIMEOUT_WAIT_FOR_FIRST_MESSAGE=120 -export SKYSCAN_DOCKER_PULL_ALWAYS=0 -export SKYSCAN_DOCKER_IMAGE_TAG=local -export SKYSCAN_MINI_TEST='yes' -export SKYSCAN_LOG=DEBUG -export SKYSCAN_LOG_THIRD_PARTY=INFO -export CLIENT_STARTER_WAIT_FOR_STARTUP_JSON=120 +export SKYSCAN_MQ_TIMEOUT_TO_CLIENTS=${SKYSCAN_MQ_TIMEOUT_TO_CLIENTS:-5} +export SKYSCAN_MQ_TIMEOUT_FROM_CLIENTS=${SKYSCAN_MQ_TIMEOUT_FROM_CLIENTS:-1800} +# export SKYSCAN_MQ_CLIENT_TIMEOUT_WAIT_FOR_FIRST_MESSAGE=0 + +export SKYSCAN_DOCKER_PULL_ALWAYS=${SKYSCAN_DOCKER_PULL_ALWAYS:-0} +export SKYSCAN_DOCKER_IMAGE_TAG=${SKYSCAN_DOCKER_IMAGE_TAG:-"local"} +export SKYSCAN_MINI_TEST=${SKYSCAN_MINI_TEST:-'yes'} +export SKYSCAN_LOG=${SKYSCAN_LOG:-"DEBUG"} +export SKYSCAN_LOG_THIRD_PARTY=${SKYSCAN_LOG_THIRD_PARTY:-"INFO"} + +export CLIENT_STARTER_WAIT_FOR_STARTUP_JSON=${CLIENT_STARTER_WAIT_FOR_STARTUP_JSON:-12} -set +ex +set +ex # file is sourced so turn off \ No newline at end of file