tests

add sleep for non parallel case - 2 #2573

Workflow file for this run

	name: tests


	on:
	push:
	branches:
	- '**'
	tags-ignore:
	- '**'


	# concurrency:
	# group: ${{ github.workflow }}-${{ github.ref }}
	# # don't cancel on main/master/default
	# cancel-in-progress: ${{ format('refs/heads/{0}', github.event.repository.default_branch) != github.ref }}


	env:
	PY_COLORS: "1"
	BOT_NAME: wipacdevbot
	BOT_EMAIL: [email protected]
	REALTIME_EVENTS_DIR: /home/runner/work/skymap_scanner/skymap_scanner/tests/data/realtime_events
	SKYSCAN_CACHE_DIR: /home/runner/work/skymap_scanner/skymap_scanner/cache
	SKYSCAN_OUTPUT_DIR: /home/runner/work/skymap_scanner/skymap_scanner/output
	SKYSCAN_DEBUG_DIR: /home/runner/work/skymap_scanner/skymap_scanner/debug
	EWMS_PILOT_DUMP_SUBPROC_OUTPUT: False # get logs in "client/pilot subprocesses stderr & stdout" step instead
	# see source tests/env-vars.sh


	jobs:


	#############################################################################
	# PACKAGING & LINTERS
	#############################################################################


	py-versions:
	runs-on: ubuntu-latest
	outputs:
	matrix: ${{ steps.versions.outputs.matrix }}
	steps:
	- uses: actions/checkout@v3
	- id: versions
	uses: WIPACrepo/[email protected]

	# flake8:
	# runs-on: ubuntu-latest
	# steps:
	# - uses: actions/checkout@v3
	# - uses: actions/setup-python@v3
	# - uses: WIPACrepo/[email protected]

	mypy:
	needs: [py-versions]
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	py3: ${{ fromJSON(needs.py-versions.outputs.matrix) }}
	steps:
	- uses: actions/checkout@v3
	- uses: actions/setup-python@v3
	with:
	python-version: ${{ matrix.py3 }}
	- uses: WIPACrepo/[email protected]

	py-setup:
	# needs: [wait-for-tests] # NOTE: this would be needed if enabling "all tests to pass" branch protection
	runs-on: ubuntu-latest
	steps:
	- uses: jlumbroso/free-disk-space@main
	with:
	tool-cache: 'true'
	large-packages: false # TODO: remove once https://github.com/jlumbroso/free-disk-space/issues/4 is fixed
	# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
	- name: checkout
	uses: actions/checkout@v3
	with:
	token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
	- uses: WIPACrepo/[email protected]
	if: github.actor != ${{ env.BOT_NAME }} # no auto-updates for bots # should match all 'git_committer_name' uses
	with:
	git_committer_name: ${{ env.BOT_NAME }}
	git_committer_email: ${{ env.BOT_EMAIL }}


	#############################################################################
	# TESTS
	#############################################################################


	test-build-docker:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v3
	- uses: docker/setup-buildx-action@v2
	- uses: docker/build-push-action@v3
	with:
	context: .
	cache-from: type=gha
	cache-to: type=gha,mode=min
	file: Dockerfile
	tags: icecube/skymap_scanner:local

	test-build-docker-pulsar:
	needs: test-build-docker
	runs-on: ubuntu-latest
	steps:
	- uses: jlumbroso/free-disk-space@main
	with:
	tool-cache: 'true'
	large-packages: false # TODO: remove once https://github.com/jlumbroso/free-disk-space/issues/4 is fixed
	- uses: actions/checkout@v3
	- uses: docker/setup-buildx-action@v2
	- uses: docker/build-push-action@v3
	with:
	context: .
	cache-from: type=gha
	file: Dockerfile_pulsar
	tags: icecube/skymap_scanner:local-pulsar

	test-run-singularity-dummy-reco:
	needs: test-build-docker
	runs-on: ubuntu-latest
	env:
	SKYSCAN_BROKER_ADDRESS: user1:password@localhost/test
	# SKYSCAN_BROKER_AUTH: user1 # using this would override password in address
	services:
	rabbitmq:
	image: bitnami/rabbitmq:latest
	env:
	RABBITMQ_USERNAME: user1
	RABBITMQ_PASSWORD: password
	RABBITMQ_VHOST: test
	BITNAMI_DEBUG: true
	# Note: `--network` option is not supported.
	options: >-
	--name rabbitmq
	--health-cmd "rabbitmqctl node_health_check"
	--health-interval 5s
	--health-timeout 5s
	--health-retries 10
	ports:
	- 5672:5672
	- 15672:15672
	steps:
	- uses: jlumbroso/free-disk-space@main
	with:
	tool-cache: 'true'
	large-packages: false # TODO: remove once https://github.com/jlumbroso/free-disk-space/issues/4 is fixed
	- uses: actions/checkout@v3
	- uses: docker/setup-buildx-action@v2
	- uses: docker/build-push-action@v3
	with:
	context: .
	cache-from: type=gha
	# cache-to: type=gha,mode=min
	file: Dockerfile
	tags: icecube/skymap_scanner:local
	load: true
	- uses: eWaterCycle/setup-apptainer@v2
	with:
	apptainer-version: 1.1.2
	- name: build singularity image
	run: \|
	sudo singularity build skymap_scanner.sif docker-daemon://icecube/skymap_scanner:local
	ls -lh skymap_scanner.sif
	- name: run singularity container
	run: \|
	source tests/env-vars.sh

	mkdir $SKYSCAN_CACHE_DIR
	mkdir $SKYSCAN_OUTPUT_DIR

	# Launch Server
	singularity run skymap_scanner.sif \
	python -m skymap_scanner.server \
	--reco-algo dummy \
	--event-file $REALTIME_EVENTS_DIR/hese_event_01.json \
	--cache-dir $SKYSCAN_CACHE_DIR \
	--output-dir $SKYSCAN_OUTPUT_DIR \
	--client-startup-json ./startup.json \
	--nsides 1:0 \
	--simulated-event \
	&

	./resources/launch_scripts/wait_for_file.sh ./startup.json $CLIENT_STARTER_WAIT_FOR_STARTUP_JSON

	# Launch Clients
	nclients=$(( $_NCLIENTS * $(nproc) ))
	echo "Launching $nclients clients"
	mkdir $SKYSCAN_DEBUG_DIR
	export EWMS_PILOT_TASK_TIMEOUT=1800 # 30 mins
	for i in $( seq 1 $nclients ); do
	singularity run skymap_scanner.sif \
	python -m skymap_scanner.client \
	--client-startup-json ./startup.json \
	--debug-directory $SKYSCAN_DEBUG_DIR \
	&
	echo -e "\tclient #$i launched"
	done

	wait -n # for server
	for i in $( seq 1 $nclients ); do
	wait -n # for client
	done
	- name: look at results file (.npz)
	run: \|
	ls .
	ls $SKYSCAN_OUTPUT_DIR
	outfile=$(ls -d $SKYSCAN_OUTPUT_DIR/*.npz)
	echo $outfile
	- name: broker docker logs
	if: always()
	run: \|
	docker logs rabbitmq

	test-run-nsides-thresholds-dummy:
	needs: test-build-docker
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	nsides: [
	"1:0",
	"1:0 2:12",
	"1:0 2:12 4:12"
	]
	predictive_scanning_threshold: [
	1.0,
	0.65,
	]
	ewms_pilot_concurrent_tasks: [
	0,
	2,
	]
	env:
	SKYSCAN_BROKER_ADDRESS: user1@localhost/test
	SKYSCAN_BROKER_AUTH: password # using this would override password in address
	services:
	rabbitmq:
	image: bitnami/rabbitmq:latest
	env:
	RABBITMQ_USERNAME: user1
	RABBITMQ_PASSWORD: password
	RABBITMQ_VHOST: test
	BITNAMI_DEBUG: true
	# Note: `--network` option is not supported.
	options: >-
	--name rabbitmq
	--health-cmd "rabbitmqctl node_health_check"
	--health-interval 5s
	--health-timeout 5s
	--health-retries 10
	ports:
	- 5672:5672
	- 15672:15672
	steps:
	- uses: actions/checkout@v3
	- uses: docker/setup-buildx-action@v2
	- uses: docker/build-push-action@v3
	with:
	context: .
	cache-from: type=gha
	# cache-to: type=gha,mode=min
	file: Dockerfile
	tags: icecube/skymap_scanner:local
	load: true
	- name: run
	run: \|
	set -x
	source tests/env-vars.sh
	export _RECO_ALGO=dummy
	export _EVENTS_FILE=$(realpath $REALTIME_EVENTS_DIR/hese_event_01.json)
	export _NSIDES="${{ matrix.nsides }}"
	export _PREDICTIVE_SCANNING_THRESHOLD=${{ matrix.predictive_scanning_threshold }}
	export _NCLIENTS=$(nproc)

	export EWMS_PILOT_CONCURRENT_TASKS=${{ matrix.ewms_pilot_concurrent_tasks }}
	free # dump memory stats

	cd ./resources/launch_scripts
	./local-scan.sh

	free # dump memory stats

	- name: check no nsides skipped
	run: \|
	ls $SKYSCAN_OUTPUT_DIR
	# get newest run*.json
	export outfile=$(find $SKYSCAN_OUTPUT_DIR -type f -name "run*.json" -exec stat -c '%y %n' {} + \| sort \| tail -1 \| awk '{print $4}')
	echo $outfile
	python3 -c '
	import json
	import os
	with open(os.getenv("outfile")) as f:
	pydict = json.load(f)
	nsides = "${{ matrix.nsides }}"
	assert len(pydict) == nsides.count(":")
	'

	- name: client/pilot subprocesses stderr & stdout
	if: always()
	run: \|
	sudo apt install tree
	tree $SKYSCAN_DEBUG_DIR
	find $SKYSCAN_DEBUG_DIR -type f -not -name ".i3" -not -name ".pkl" -exec "more" {} + \| cat # recursively cats with filenames (delimited by :::::::)

	- name: broker docker logs
	if: always()
	run: \|
	docker logs rabbitmq

	test-run-realistic:
	needs: test-build-docker
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	reco_algo: [
	millipede_original,
	millipede_wilks,
	splinempe
	]
	eventfile: [
	hese_event_01.json,
	run00136766-evt000007637140-GOLD.pkl,
	run00136662-evt000035405932-BRONZE.pkl
	]
	ewms_pilot_concurrent_tasks: [
	0,
	2,
	]
	exclude:
	# splinempe should not run on HESE
	- reco_algo: splinempe
	eventfile: hese_event_01.json
	env:
	SKYSCAN_BROKER_ADDRESS: user1@localhost/test
	SKYSCAN_BROKER_AUTH: password # using this would override password in address
	services:
	rabbitmq:
	image: bitnami/rabbitmq:latest
	env:
	RABBITMQ_USERNAME: user1
	RABBITMQ_PASSWORD: password
	RABBITMQ_VHOST: test
	BITNAMI_DEBUG: true
	# Note: `--network` option is not supported.
	options: >-
	--name rabbitmq
	--health-cmd "rabbitmqctl node_health_check"
	--health-interval 5s
	--health-timeout 5s
	--health-retries 10
	ports:
	- 5672:5672
	- 15672:15672
	steps:
	- uses: jlumbroso/free-disk-space@main
	with:
	tool-cache: 'true'
	large-packages: false # TODO: remove once https://github.com/jlumbroso/free-disk-space/issues/4 is fixed
	- uses: actionhippie/swap-space@v1
	with:
	size: 10G
	- uses: actions/checkout@v3
	- uses: docker/setup-buildx-action@v2
	- uses: docker/build-push-action@v3
	with:
	context: .
	cache-from: type=gha
	# cache-to: type=gha,mode=min
	file: Dockerfile
	tags: icecube/skymap_scanner:local
	load: true
	- name: run
	run: \|
	set -x
	source tests/env-vars.sh
	export _RECO_ALGO=${{ matrix.reco_algo }}
	export _EVENTS_FILE=$(realpath $REALTIME_EVENTS_DIR/${{ matrix.eventfile }})
	export _NSIDES="1:0"

	export EWMS_PILOT_CONCURRENT_TASKS=${{ matrix.ewms_pilot_concurrent_tasks }}
	if [[ $EWMS_PILOT_CONCURRENT_TASKS == "0" ]]; then
	# if not using concurrent tasks, then use multiple clients
	export _NCLIENTS=$(nproc)
	else
	export _NCLIENTS=1
	fi

	ls /dev/shm
	free # dump memory stats

	cd ./resources/launch_scripts
	./local-scan.sh

	free # dump memory stats
	ls /dev/shm

	- name: test output against known result (.json)
	run: \|
	ls $SKYSCAN_OUTPUT_DIR
	# get newest run*.json
	outfile=$(find $SKYSCAN_OUTPUT_DIR -type f -name "run*.json" -exec stat -c '%y %n' {} + \| sort \| tail -1 \| awk '{print $4}')
	echo $outfile
	cat $outfile
	pip install . # don't need icecube, so no docker container needed
	python tests/compare_scan_results.py \
	--actual $outfile \
	--expected tests/data/results_json/${{ matrix.reco_algo }}/$(basename $outfile) \
	--assert \
	\|\| (cat $(ls *.diff.json) && false)

	- name: client/pilot subprocesses stderr & stdout
	if: always()
	run: \|
	sudo apt install tree
	tree $SKYSCAN_DEBUG_DIR
	find $SKYSCAN_DEBUG_DIR -type f -not -name ".i3" -not -name ".pkl" -exec "more" {} + \| cat # recursively cats with filenames (delimited by :::::::)

	- name: broker docker logs
	if: always()
	run: \|
	docker logs rabbitmq

	test-file-staging:
	needs: test-build-docker
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v3
	- uses: docker/setup-buildx-action@v2
	- uses: docker/build-push-action@v3
	with:
	context: .
	cache-from: type=gha
	# cache-to: type=gha,mode=min
	file: Dockerfile
	tags: icecube/skymap_scanner:local
	load: true
	- name: run
	run: \|
	docker run --rm -i \
	icecube/skymap_scanner:local \
	python tests/file_staging.py

	test-run-single-pixel:
	needs: test-build-docker
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	dir: [
	"BRONZE",
	"GOLD",
	"JSON",
	]
	reco_algo:
	[
	millipede_original,
	millipede_wilks,
	splinempe
	]
	exclude:
	# splinempe should not run on HESE
	- reco_algo: splinempe
	dir: "JSON"
	steps:
	- uses: actions/checkout@v3
	- uses: docker/setup-buildx-action@v2
	- uses: docker/build-push-action@v3
	with:
	context: .
	cache-from: type=gha
	# cache-to: type=gha,mode=min
	file: Dockerfile
	tags: icecube/skymap_scanner:local
	load: true
	- name: run
	run: \|
	source tests/env-vars.sh

	# grab the GCDQp_packet key and throw into a file
	jq '.GCDQp_packet' \
	tests/data/reco_pixel_pkls/${{ matrix.reco_algo }}/${{ matrix.dir }}/startup.json > \
	tests/data/reco_pixel_pkls/${{ matrix.reco_algo }}/${{ matrix.dir }}//GCDQp_packet.json

	# move python file to subdir to make bindings simpler
	cp tests/data/reco_pixel_pkls/get_toclient_msg_pkl.py \
	tests/data/reco_pixel_pkls/${{ matrix.reco_algo }}/${{ matrix.dir }}

	# create the in.pkl
	docker run --network="host" --rm -i \
	--shm-size=6gb \
	--mount type=bind,source=$(readlink -f tests/data/reco_pixel_pkls/${{ matrix.reco_algo }}/${{ matrix.dir }}),target=/local/pkls \
	--env PY_COLORS=1 \
	$(env \| grep '^SKYSCAN_' \| awk '$0="--env "$0') \
	icecube/skymap_scanner:local \
	python /local/pkls/get_toclient_msg_pkl.py \
	--pframe-pkl /local/pkls/pframe.pkl \
	--reco-algo ${{ matrix.reco_algo }}

	hexdump -C tests/data/reco_pixel_pkls/${{ matrix.reco_algo }}/${{ matrix.dir }}/in.pkl

	# run
	docker run --network="host" --rm -i \
	--shm-size=6gb \
	--mount type=bind,source=$(readlink -f tests/data/reco_pixel_pkls/${{ matrix.reco_algo }}/${{ matrix.dir }}),target=/local/pkls \
	--env PY_COLORS=1 \
	$(env \| grep '^SKYSCAN_' \| awk '$0="--env "$0') \
	icecube/skymap_scanner:local \
	python -m skymap_scanner.client.reco_icetray \
	--in-pkl /local/pkls/in.pkl \
	--reco-algo ${{ matrix.reco_algo }} \
	--gcdqp-packet-json /local/pkls/GCDQp_packet.json \
	--baseline-gcd-file $(jq -r '.baseline_GCD_file' tests/data/reco_pixel_pkls/${{ matrix.reco_algo }}/${{ matrix.dir }}/startup.json) \
	--out-pkl /local/pkls/out.pkl

	- name: test output against known result (.pkl)
	run: \|
	ls tests/data/reco_pixel_pkls/${{ matrix.reco_algo }}/${{ matrix.dir }}

	# need icecube for depickling, so docker container needed
	docker run --network="host" --rm -i \
	--shm-size=6gb \
	--mount type=bind,source=$(readlink -f tests/data/reco_pixel_pkls/${{ matrix.reco_algo }}/${{ matrix.dir }}),target=/local/pkls \
	--env PY_COLORS=1 \
	$(env \| grep '^SKYSCAN_' \| awk '$0="--env "$0') \
	icecube/skymap_scanner:local \
	python tests/compare_reco_pixel_pkl.py \
	--actual /local/pkls/out.pkl \
	--expected /local/pkls/$(basename $(ls tests/data/reco_pixel_pkls/${{ matrix.reco_algo }}/${{ matrix.dir }}/*.out.pkl)) \
	--diff-out-dir /local/pkls/ \
	--assert \
	\|\| (cat $(ls tests/data/reco_pixel_pkls/${{ matrix.reco_algo }}/${{ matrix.dir }}/*.diff.json) && false)


	#############################################################################
	# GITHUB RELEASE
	#############################################################################


	release:
	# only run on main/master/default
	if: format('refs/heads/{0}', github.event.repository.default_branch) == github.ref
	# have to wait for tests so python-semantic-release can push (branch protection on main don't work w/ the bot)
	needs: [
	mypy,
	py-setup,
	test-build-docker,
	test-build-docker-pulsar,
	test-run-singularity-dummy-reco,
	test-file-staging,
	test-run-nsides-thresholds-dummy,
	test-run-realistic,
	test-run-single-pixel
	]
	runs-on: ubuntu-latest
	concurrency: release
	steps:
	- uses: actions/checkout@v3
	with:
	fetch-depth: 0
	token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
	- name: Python Semantic Release
	uses: python-semantic-release/[email protected]
	with:
	git_committer_name: ${{ env.BOT_NAME }}
	git_committer_email: ${{ env.BOT_EMAIL }}
	github_token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
	# repository_username: __token__
	# repository_password: ${{ secrets.PYPI_TOKEN }}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

add sleep for non parallel case - 2 #2573

Workflow file

add sleep for non parallel case - 2 #2573

Jobs

Run details

Workflow file for this run