finally fix singularity test

icecube · Oct 3, 2024 · ecae72b · ecae72b
1 parent 0e3473c
commit ecae72b
Show file tree

Hide file tree

Showing 4 changed files with 55 additions and 63 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -166,45 +166,16 @@ jobs:
           ls -lh skymap_scanner.sif
       - name: run singularity container
         run: |
+          set -x
           source tests/env-vars.sh
+          export _RUN_THIS_SINGULARITY_IMAGE="$(realpath skymap_scanner.sif)"
+          export _RECO_ALGO=dummy
+          export _EVENTS_FILE=$(realpath $REALTIME_EVENTS_DIR/hese_event_01.json)
+          export _NSIDES="1:0"
+          
+          cd ./resources/launch_scripts
+          ./local-scan.sh $N_WORKERS $CI_TEST_RUN_STDOUT_STDERR_DIR
 
-          mkdir $SKYSCAN_CACHE_DIR
-          mkdir $SKYSCAN_OUTPUT_DIR
-
-          # Launch Server
-          singularity run skymap_scanner.sif \
-            python -m skymap_scanner.server \
-            --reco-algo dummy \
-            --event-file $REALTIME_EVENTS_DIR/hese_event_01.json \
-            --cache-dir $SKYSCAN_CACHE_DIR \
-            --output-dir $SKYSCAN_OUTPUT_DIR \
-            --client-startup-json ./startup.json \
-            --nsides 1:0 \
-            --simulated-event \
-            2>&1 | tee "$CI_TEST_RUN_STDOUT_STDERR_DIR"/server.out \
-            &
-
-          ./resources/launch_scripts/wait_for_file.sh ./startup.json $WAIT_FOR_STARTUP_JSON
-
-          # Launch Clients
-          nworkers=2
-          echo "Launching $nworkers workers"
-          mkdir $SKYSCAN_DEBUG_DIR
-          export EWMS_PILOT_TASK_TIMEOUT=1800  # 30 mins
-          for i in $( seq 1 $nworkers ); do
-            singularity run skymap_scanner.sif \
-              python -m skymap_scanner.client \
-              --client-startup-json ./startup.json \
-              --debug-directory $SKYSCAN_DEBUG_DIR \
-              2>&1 | tee "$CI_TEST_RUN_STDOUT_STDERR_DIR"/worker-$i/worker-$i.out \
-              &
-            echo -e "\tworker #$i launched"
-          done
-
-          wait -n  # for server
-          for i in $( seq 1 $nworkers ); do
-            wait -n  # for worker
-          done
       - name: look at results file (.npz)
         run: |
           ls .

diff --git a/Dockerfile b/Dockerfile
@@ -52,7 +52,7 @@ RUN pip freeze
 ENV OPENBLAS_CORETYPE="Haswell"
 ENV NPY_DISABLE_CPU_FEATURES="AVX512F,AVX512_KNL,AVX512_KNM,AVX512_CLX,AVX512_CNL,AVX512_ICL,AVX512CD,AVX512_SKX"
 
-# set the entry point so that module is called with any parameters given to the `docker run` command
+# set the entry point so that module is called with any parameters given to the `"docker run` command
 ENTRYPOINT ["/bin/bash", "/usr/local/icetray/env-shell.sh"]
 
 CMD []
diff --git a/...es/launch_scripts/docker/launch_worker.sh → resources/launch_scripts/launch_worker.sh b/...es/launch_scripts/docker/launch_worker.sh → resources/launch_scripts/launch_worker.sh
@@ -21,14 +21,19 @@ python -c 'import os; assert os.listdir(os.path.dirname(os.environ["CI_SKYSCAN_S
 export EWMS_PILOT_EXTERNAL_DIRECTORIES="$(dirname "$CI_SKYSCAN_STARTUP_JSON")"
 
 # task image, args, env
-export EWMS_PILOT_TASK_IMAGE="$DOCKER_IMAGE_TAG"
+if [ -n "$_RUN_THIS_SINGULARITY_IMAGE" ]; then
+    export EWMS_PILOT_TASK_IMAGE="$_RUN_THIS_SINGULARITY_IMAGE"
+    export _EWMS_PILOT_CONTAINER_PLATFORM="apptainer"
+else
+    export EWMS_PILOT_TASK_IMAGE="$DOCKER_IMAGE_TAG"
+    export _EWMS_PILOT_CONTAINER_PLATFORM="docker" # NOTE: technically not needed b/c this is the default value
+    export _EWMS_PILOT_DOCKER_SHM_SIZE="6gb"       # this only needed in ci--the infra would set this in prod
+fi
 export EWMS_PILOT_TASK_ARGS="python -m skymap_scanner.client.reco_icetray --infile {{INFILE}} --outfile {{OUTFILE}} --client-startup-json $CI_SKYSCAN_STARTUP_JSON"
 json_var=$(env | grep '^SKYSCAN_' | awk -F= '{printf "\"%s\":\"%s\",", $1, $2}' | sed 's/,$//') # must remove last comma
 json_var="{$json_var}"
 export EWMS_PILOT_TASK_ENV_JSON="$json_var"
 
-export _EWMS_PILOT_DOCKER_SHM_SIZE="6gb" # this only needed in ci--the infra would set this in prod
-
 # file types -- controls intermittent serialization
 export EWMS_PILOT_INFILE_EXT="JSON"
 export EWMS_PILOT_OUTFILE_EXT="JSON"

diff --git a/resources/launch_scripts/local-scan.sh b/resources/launch_scripts/local-scan.sh
@@ -59,27 +59,43 @@ mkdir "$(dirname "$CI_SKYSCAN_STARTUP_JSON")"
 ########################################################################
 # Launch Server
 
-docker run --network="host" --rm \
-    --mount type=bind,source="$(dirname "$_EVENTS_FILE")",target=/local/event,readonly \
-    --mount type=bind,source="$SKYSCAN_CACHE_DIR",target=/local/cache \
-    --mount type=bind,source="$SKYSCAN_OUTPUT_DIR",target=/local/output \
-    --mount type=bind,source="$(dirname "$CI_SKYSCAN_STARTUP_JSON")",target=/local/startup \
-    --env PY_COLORS=1 \
-    $(env | grep '^SKYSCAN_' | cut -d'=' -f1 | sed 's/^/--env /') \
-    $(env | grep '^EWMS_' | cut -d'=' -f1 | sed 's/^/--env /') \
-    --env "EWMS_PILOT_TASK_TIMEOUT=${EWMS_PILOT_TASK_TIMEOUT:-900}" \
-    icecube/skymap_scanner:"${SKYSCAN_DOCKER_IMAGE_TAG:-"latest"}" \
-    python -m skymap_scanner.server \
-    --reco-algo $_RECO_ALGO \
-    --event-file "/local/event/$(basename "$_EVENTS_FILE")" \
-    --cache-dir /local/cache \
-    --output-dir /local/output \
-    --client-startup-json "/local/startup/$(basename $CI_SKYSCAN_STARTUP_JSON)" \
-    --nsides $_NSIDES \
-    $arg_predictive_scanning_threshold \
-    --real-event \
-    2>&1 | tee "$outdir"/server.out \
-    &
+if [ -n "$_RUN_THIS_SINGULARITY_IMAGE" ]; then
+    # SINGULARITY
+    singularity run "$_RUN_THIS_SINGULARITY_IMAGE" \
+        python -m skymap_scanner.server \
+        --reco-algo $_RECO_ALGO \
+        --event-file $_EVENTS_FILE \
+        --cache-dir $SKYSCAN_CACHE_DIR \
+        --output-dir $SKYSCAN_OUTPUT_DIR \
+        --client-startup-json $CI_SKYSCAN_STARTUP_JSON \
+        --nsides $_NSIDES \
+        --simulated-event \
+        2>&1 | tee "$outdir"/server.out \
+        &
+else
+    # DOCKER
+    docker run --network="host" --rm \
+        --mount type=bind,source="$(dirname "$_EVENTS_FILE")",target=/local/event,readonly \
+        --mount type=bind,source="$SKYSCAN_CACHE_DIR",target=/local/cache \
+        --mount type=bind,source="$SKYSCAN_OUTPUT_DIR",target=/local/output \
+        --mount type=bind,source="$(dirname "$CI_SKYSCAN_STARTUP_JSON")",target=/local/startup \
+        --env PY_COLORS=1 \
+        $(env | grep '^SKYSCAN_' | cut -d'=' -f1 | sed 's/^/--env /') \
+        $(env | grep '^EWMS_' | cut -d'=' -f1 | sed 's/^/--env /') \
+        --env "EWMS_PILOT_TASK_TIMEOUT=${EWMS_PILOT_TASK_TIMEOUT:-900}" \
+        icecube/skymap_scanner:"${SKYSCAN_DOCKER_IMAGE_TAG:-"latest"}" \
+        python -m skymap_scanner.server \
+        --reco-algo $_RECO_ALGO \
+        --event-file "/local/event/$(basename "$_EVENTS_FILE")" \
+        --cache-dir /local/cache \
+        --output-dir /local/output \
+        --client-startup-json "/local/startup/$(basename $CI_SKYSCAN_STARTUP_JSON)" \
+        --nsides $_NSIDES \
+        $arg_predictive_scanning_threshold \
+        --real-event \
+        2>&1 | tee "$outdir"/server.out \
+        &
+fi
 pidmap["$!"]="central server"
 
 ########################################################################
@@ -90,7 +106,7 @@ pidmap["$!"]="central server"
 ########################################################################
 # Launch Workers that each run a Pilot which each run Skyscan Clients
 
-launch_scripts_dir=$(realpath "./docker/")
+launch_scripts_dir=$(pwd)
 echo "Launching $nworkers workers"
 export EWMS_PILOT_TASK_TIMEOUT=${EWMS_PILOT_TASK_TIMEOUT:-"1800"} # 30 mins
 for i in $(seq 1 $nworkers); do