Client Upgrades: startup.json + ewms-pilot (#67)

Co-authored-by: github-actions <[email protected]>

.github/workflows/tests.yml

@@ -11,11 +11,9 @@ on:
 env:
   PY_COLORS: "1"
   REALTIME_EVENTS_DIR: tests/data/realtime_events
-  PULSAR_CONTAINER: pulsar_local
   SKYSCAN_CACHE_DIR: cache-dir
   SKYSCAN_OUTPUT_DIR: output-dir
-  PULSAR_ADDRESS: localhost:6650
-  PULSAR_AUTH: ""
+  BROKER_ADDRESS: localhost
   SKYSCAN_DEBUG_DIR: debug-pkl-dir
   SKYSCAN_REPORT_INTERVAL_SEC: 60
   CI_TESTING_USE_LOCAL_DOCKER: 1
@@ -44,20 +42,21 @@ jobs:
       CLIENTS_PER_CPU: 1  # there isn't any improvement when >1
     steps:
       - uses: actions/checkout@v3
-      - name: Run Pulsar Server Docker Container (starts up during skymap scanner image build)
+      - name: Checkout MQClient repo (for broker startup scripts)
+        uses: actions/checkout@v3
+        with:
+          repository: WIPACrepo/MQClient
+          path: MQClient
+      - name: Start Broker
         run: |
-          ./scripts/launch_scripts/pulsar_run.sh $PULSAR_CONTAINER
+          set -x
+          ./MQClient/resources/docker-pulsar.sh
       - name: Build Skymap Scanner Docker Image
         run: |
           docker build --no-cache -t icecube/skymap_scanner .
-      - name: Configure Pulsar Server
-        run: |
-          export DOCKERIZE_VERSION=v0.3.0
-          wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz
-          dockerize -wait tcp://localhost:8080 -timeout 1m
-          dockerize -wait tcp://localhost:6650 -timeout 1m
       - name: Run
         run: |
+          set -x
 
           mkdir $SKYSCAN_CACHE_DIR
           mkdir $SKYSCAN_OUTPUT_DIR
@@ -66,27 +65,25 @@ jobs:
             --event-file $REALTIME_EVENTS_DIR/${{ matrix.eventfile }} \
             --cache-dir $SKYSCAN_CACHE_DIR \
             --output-dir $SKYSCAN_OUTPUT_DIR \
-            --startup-files-dir . \
-            --broker $PULSAR_ADDRESS \
+            --startup-json-dir . \
+            --broker $BROKER_ADDRESS \
             --log DEBUG \
             --log-third-party INFO \
             --mini-test-variations \
             --min-nside 1 \
             --max-nside 1 \
             &
 
-          ./scripts/launch_scripts/wait_for_startup_files.sh .
+          ./scripts/launch_scripts/wait_for_startup_json.sh .
 
           nclients=$(( $CLIENTS_PER_CPU * $(nproc) ))
           echo "Launching $nclients clients"
           mkdir $SKYSCAN_DEBUG_DIR
           export PULSAR_UNACKED_MESSAGES_TIMEOUT_SEC=1800  # 30 mins
           for i in $( seq 1 $nclients ); do
             ./scripts/launch_scripts/launch_client.sh \
-              --mq-basename $(cat mq-basename.txt) \
-              --baseline-gcd-file $(cat baseline_GCD_file.txt) \
-              --gcdqp-packet-pkl GCDQp_packet.pkl \
-              --broker $PULSAR_ADDRESS \
+              --startup-json-dir . \
+              --broker $BROKER_ADDRESS \
               --log DEBUG \
               --log-third-party INFO \
               --debug-directory $SKYSCAN_DEBUG_DIR \
@@ -129,6 +126,8 @@ jobs:
           docker build --no-cache -t icecube/skymap_scanner .
       - name: Run
         run: |
+          jq '.GCDQp_packet' tests/data/reco_pixel_pkls/${{ matrix.dir }}/startup.json > tests/data/reco_pixel_pkls/${{ matrix.dir }}//GCDQp_packet.json
+
           docker run --network="host" --rm -i \
             --shm-size=6gb \
             --mount type=bind,source=$(readlink -f tests/data/reco_pixel_pkls/${{ matrix.dir }}),target=/local/pkls \
@@ -137,8 +136,8 @@ jobs:
             icecube/skymap_scanner:latest \
             python -m skymap_scanner.client.reco_pixel_pkl \
             --in-pkl /local/pkls/$(basename $(ls tests/data/reco_pixel_pkls/${{ matrix.dir }}/*.in.pkl)) \
-            --gcdqp-packet-pkl /local/pkls/GCDQp_packet.pkl \
-            --baseline-gcd-file $(cat tests/data/reco_pixel_pkls/${{ matrix.dir }}/baseline_GCD_file.txt) \
+            --gcdqp-packet-json /local/pkls/GCDQp_packet.json \
+            --baseline-gcd-file $(jq -r '.baseline_GCD_file' tests/data/reco_pixel_pkls/${{ matrix.dir }}/startup.json) \
             --out-pkl /local/pkls/out.pkl \
             --log DEBUG
 

.github/workflows/wipac-cicd.yml

@@ -18,7 +18,7 @@ jobs:
   #     - uses: actions/setup-python@v3
   #       with:
   #         python-version: '3.9'
-  #     - uses: WIPACrepo/wipac-dev-mypy-action@v1
+  #     - uses: WIPACrepo/wipac-dev-mypy-action@v1.1
 
   py-setup:
     runs-on: ubuntu-latest
@@ -28,7 +28,7 @@ jobs:
         uses: actions/checkout@v3
         with:
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-      - uses: WIPACrepo/wipac-dev-py-setup-action@v1.6
+      - uses: WIPACrepo/wipac-dev-py-setup-action@v1.8
 
   py-versions:
     needs: [py-setup]
@@ -38,20 +38,21 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - id: versions
-        uses: WIPACrepo/wipac-dev-py-versions-action@v1
+        uses: WIPACrepo/wipac-dev-py-versions-action@v2
 
   pip-install:
     # FUTURE DEV: this can be expanded to run actual tests instead of just pip installing
     needs: [py-versions]
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
-      matrix: ${{ fromJSON(needs.py-versions.outputs.matrix) }}
+      matrix:
+        py3: ${{ fromJSON(needs.py-versions.outputs.matrix) }}
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v3
         with:
-          python-version: ${{ matrix.py3_versions }}
+          python-version: ${{ matrix.py3 }}
       - run: |
           pip install --upgrade pip wheel setuptools
           pip install .

README.md

@@ -10,7 +10,7 @@ Distributed likelihood scan of event directions for real-time alerts using inter
 ## How to Run
 ***There are two\* important helper scripts that will make this easy***: `scripts/launch_scripts/launch_server.sh` and `scripts/launch_scripts/launch_client.sh`. Pass in arguments like you would for the desired python sub-module. These will launch docker containers, auto-manage file transfer/binding, and copy over `SKYSCAN_*` [environment variables](#environment-variables) for you!
 
-\* _Another useful script is `scripts/launch_scripts/wait_for_startup_files.sh`. Use this before launching each client to eliminate file-writing race conditions._
+\* _Another useful script is `scripts/launch_scripts/wait_for_startup_json.sh`. Use this before launching each client to eliminate file-writing race conditions._
 
 ### Example Startup
 #### 1. Launch the Server
@@ -20,7 +20,7 @@ Distributed likelihood scan of event directions for real-time alerts using inter
     --event-file `pwd`/run00136662-evt000035405932-BRONZE.pkl \  # could also be a .json file
     --cache-dir `pwd`/server_cache \
     --output-dir `pwd` \
-    --startup-files-dir <STARTUP_FILES_DIR> \
+    --startup-json-dir <STARTUP_JSON_DIR> \
     --broker <BROKER_ADDRESS> \
     --auth-token `cat ~/skyscan-broker.token` \
     --log DEBUG \
@@ -31,12 +31,12 @@ _NOTE: The `--*dir` arguments can all be the same if you'd like. Relative paths
 #### 2. Launch Each Client
 Each on a different CPU:
 ```
-./scripts/launch_scripts/wait_for_startup_files.sh <STARTUP_FILES_DIR>
+./scripts/launch_scripts/wait_for_startup_json.sh <STARTUP_JSON_DIR>
 
 ./scripts/launch_scripts/launch_client.sh \
-    --mq-basename $(cat <STARTUP_FILES_DIR>/mq-basename.txt) \
-    --baseline-gcd-file $(cat <STARTUP_FILES_DIR>/baseline_GCD_file.txt) \
-    --gcdqp-packet-pkl <STARTUP_FILES_DIR>/GCDQp_packet.pkl \
+    --mq-basename $(cat <STARTUP_JSON_DIR>/mq-basename.txt) \
+    --baseline-gcd-file $(cat <STARTUP_JSON_DIR>/baseline_GCD_file.txt) \
+    --gcdqp-packet-json <STARTUP_JSON_DIR>/GCDQp_packet.json \
     --broker <BROKER_ADDRESS> \
     --auth-token `cat ~/skyscan-broker.token` \
     --log DEBUG \

requirements.txt

@@ -1,15 +1,15 @@
 #
-# This file is autogenerated by pip-compile with python 3.8
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile
 #
-apache-bookkeeper-client==4.15.1
-    # via skymap-scanner (setup.py)
 astropy==5.1
     # via healpy
 asyncstdlib==3.10.5
-    # via skymap-scanner (setup.py)
+    # via
+    #   ewms-pilot
+    #   skymap-scanner (setup.py)
 certifi==2022.9.24
     # via
     #   pulsar-client
@@ -22,10 +22,10 @@ contourpy==1.0.5
     # via matplotlib
 cycler==0.11.0
     # via matplotlib
+ewms-pilot==0.1.3
+    # via skymap-scanner (setup.py)
 fonttools==4.37.4
     # via matplotlib
-grpcio==1.27.2
-    # via apache-bookkeeper-client
 healpy==1.16.1
     # via skymap-scanner (setup.py)
 humanfriendly==10.0
@@ -34,7 +34,7 @@ idna==3.4
     # via requests
 kiwisolver==1.4.4
     # via matplotlib
-matplotlib==3.6.0
+matplotlib==3.6.1
     # via healpy
 numpy==1.23.3
     # via
@@ -50,32 +50,24 @@ packaging==21.3
     #   matplotlib
 pillow==9.2.0
     # via matplotlib
-protobuf==4.21.7
-    # via apache-bookkeeper-client
 pulsar-client==2.10.1
     # via wipac-mqclient
 pyerfa==2.0.0.1
     # via astropy
-pymmh3==0.0.5
-    # via apache-bookkeeper-client
 pyparsing==3.0.9
     # via
     #   matplotlib
     #   packaging
 python-dateutil==2.8.2
     # via matplotlib
-pytz==2022.4
-    # via apache-bookkeeper-client
 pyyaml==6.0
     # via astropy
 requests==2.28.1
     # via wipac-dev-tools
-scipy==1.9.1
+scipy==1.9.2
     # via healpy
 six==1.16.0
     # via
-    #   apache-bookkeeper-client
-    #   grpcio
     #   pulsar-client
     #   python-dateutil
 typing-extensions==4.4.0
@@ -87,7 +79,6 @@ wipac-dev-tools[coloredlogs]==1.5.6
     #   skymap-scanner (setup.py)
     #   wipac-mqclient
 wipac-mqclient[pulsar]==0.6.3
-    # via skymap-scanner (setup.py)
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools
+    # via
+    #   ewms-pilot
+    #   skymap-scanner (setup.py)

scripts/launch_scripts/launch_client.sh

@@ -32,7 +32,7 @@ def extract_opt_path(py_args, opt):
 
 py_args, debug_dir = extract_opt_path(py_args, "--debug-directory")
 py_args, gcd = extract_opt_path(py_args, "--gcd-dir")
-py_args, gcdqp = extract_opt_path(py_args, "--gcdqp-packet-pkl")
+py_args, startup = extract_opt_path(py_args, "--startup-json-dir")
 
 dockermount_args = ""
 py_args += " "
@@ -43,9 +43,9 @@ if debug_dir:
 if gcd:
     dockermount_args += f"--mount type=bind,source={gcd},target=/local/gcd,readonly "
     py_args += f"--gcd-dir /local/gcd "
-if gcdqp:
-    dockermount_args += f"--mount type=bind,source={os.path.dirname(gcdqp)},target=/local/gcdqp,readonly "
-    py_args += f"--gcdqp-packet-pkl /local/gcdqp/{os.path.basename(gcdqp)} "
+if startup:
+    dockermount_args += f"--mount type=bind,source={startup},target=/local/startup-json-dir "
+    py_args += f"--startup-json-dir /local/startup-json-dir "
 
 print(f"{dockermount_args}#{py_args}")
 ')

scripts/launch_scripts/launch_server.sh

@@ -34,7 +34,7 @@ py_args, event = extract_opt_path(py_args, "--event-file")
 py_args, cache = extract_opt_path(py_args, "--cache-dir")
 py_args, output = extract_opt_path(py_args, "--output-dir")
 py_args, gcd = extract_opt_path(py_args, "--gcd-dir")
-py_args, startup = extract_opt_path(py_args, "--startup-files-dir")
+py_args, startup = extract_opt_path(py_args, "--startup-json-dir")
 
 dockermount_args = ""
 py_args += " "
@@ -52,8 +52,8 @@ if gcd:
     dockermount_args += f"--mount type=bind,source={gcd},target=/local/gcd,readonly "
     py_args += f"--gcd-dir /local/gcd "
 if startup:
-    dockermount_args += f"--mount type=bind,source={startup},target=/local/startup-files "
-    py_args += f"--startup-files-dir /local/startup-files "
+    dockermount_args += f"--mount type=bind,source={startup},target=/local/startup-json-dir "
+    py_args += f"--startup-json-dir /local/startup-json-dir "
 
 print(f"{dockermount_args}#{py_args}")
 ')

scripts/launch_scripts/wait_for_startup_files.sh → scripts/launch_scripts/wait_for_startup_json.sh

@@ -2,36 +2,36 @@
 
 ########################################################################
 #
-# Wait for all startup files after launching a Skymap Scanner server
+# Wait for startup.json after launching a Skymap Scanner server
 # and before launching any clients
 #
-# Pass in one argument, the directory that will contain the startup files
+# Pass in one argument, the directory that will contain the startup.json
 #
 ########################################################################
 
 if [ -z "$1" ]; then
-    echo "Usage: wait_for_startup_files.sh STARTUP_DIRECTORY"
+    echo "Usage: wait_for_startup_json.sh STARTUP_DIRECTORY"
     exit 1
 fi
 if [ ! -d "$1" ]; then
     echo "Directory Not Found: $1"
     exit 2
 fi
 
-# wait until the mq-basename.txt file exists (with a timeout)
+# wait until the startup.json file exists (with a timeout)
 waitsec="5"
 timeout="10 minutes"
 endtime=$(date -ud "$timeout" +%s)  # wait this long for server startup
 while [[ $(date -u +%s) -le $endtime ]]; do
-    if [[ -e "$1/mq-basename.txt" && -e "$1/baseline_GCD_file.txt" && -e "$1/GCDQp_packet.pkl" ]]; then
-        echo "Success! All startup files found:"
+    if [[ -e "$1/startup.json" ]]; then
+        echo "Success! 'startup.json' file found:"
         ls $1
         exit 0  # Done!
     fi
-    echo "waiting for startup files ($waitsec second intervals)..."
+    echo "waiting for 'startup.json' ($waitsec second intervals)..."
     sleep $waitsec
 done
 
-echo "Failed. Not all startup files found within time limit ($timeout):"
+echo "Failed. 'startup.json' not found within time limit ($timeout):"
 ls $1
 exit 62  # Timer expired

setup.cfg

@@ -1,5 +1,5 @@
 [wipac:cicd_setup_builder]
-python_min = 3.6
+python_min = 3.7
 python_max = 3.9
 package_dirs = skymap_scanner
 
@@ -17,12 +17,12 @@ branch = master
 
 [options]  # generated by wipac:cicd_setup_builder: python_requires, packages
 install_requires =
-	apache-bookkeeper-client
 	asyncstdlib
+	ewms-pilot # the extras here are the same as 'wipac-mqclient', so just define one place
 	healpy
 	wipac-dev-tools[coloredlogs]
 	wipac-mqclient[pulsar]
-python_requires = >=3.6, <3.10
+python_requires = >=3.7, <3.10
 packages = find:
 
 [options.package_data]  # generated by wipac:cicd_setup_builder: '*'
@@ -32,6 +32,15 @@ packages = find:
 include =
 	skymap_scanner
 	skymap_scanner.*
+exclude =
+	test
+	tests
+	doc
+	docs
+	resource
+	resources
+	example
+	examples
 
 [flake8]
 ignore = E501