From c32ff3c3dd09a3d07149660507382a99c7369412 Mon Sep 17 00:00:00 2001
From: Alex Austregesilo <aaustreg@jlab.org>
Date: Thu, 2 May 2024 09:13:38 -0400
Subject: [PATCH] initial version of launch script with singularity support

---
 launch_scripts/launch/gluexrunscript.sh   |  50 +++
 launch_scripts/launch/script_container.sh | 418 ++++++++++++++++++++++
 2 files changed, 468 insertions(+)
 create mode 100755 launch_scripts/launch/gluexrunscript.sh
 create mode 100755 launch_scripts/launch/script_container.sh

diff --git a/launch_scripts/launch/gluexrunscript.sh b/launch_scripts/launch/gluexrunscript.sh
new file mode 100755
index 0000000..aad7f6e
--- /dev/null
+++ b/launch_scripts/launch/gluexrunscript.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+if [ -d /group ]; then
+  echo "I am running in singularity."
+else
+  echo "Something went wrong with setting up singularity! Abort."
+  exit 9
+fi
+
+VERSION=version.xml
+
+CMDLINE_ARGS=""
+# prepare command line arguments for evaluation
+#for arg in "$@"; do
+while [[ $# -gt 0 ]]; do
+    key="$1"
+    case $key in
+	-v|--version)
+	    VERSION="$2"
+	    shift # past argument
+	    shift # past value
+	    ;;
+	-s|--sqlite)
+	    SQLITE=true
+	    shift # past argument
+	    ;;
+	*)
+	    CMDLINE_ARGS="${CMDLINE_ARGS} $1"
+	    shift # past argument
+	    ;;
+    esac
+done
+
+if [ -e /group/halld/www/halldweb/html/halld_versions/"$VERSION" ] ; then
+    echo "source /group/halld/Software/build_scripts/gluex_env_jlab.sh /group/halld/www/halldweb/html/halld_versions/"$VERSION
+    source /group/halld/Software/build_scripts/gluex_env_jlab.sh /group/halld/www/halldweb/html/halld_versions/$VERSION
+else
+    echo $VERSION "does not exist! Abort."
+    exit 9
+fi
+
+if [ "$SQLITE" = true ] ; then
+    export JANA_CALIB_URL=sqlite:///$PWD/ccdb.sqlite
+    export CCDB_CONNECTION=sqlite:///$PWD/ccdb.sqlite
+fi
+
+printenv
+
+echo $CMDLINE_ARGS
+$CMDLINE_ARGS
diff --git a/launch_scripts/launch/script_container.sh b/launch_scripts/launch/script_container.sh
new file mode 100755
index 0000000..7cea85c
--- /dev/null
+++ b/launch_scripts/launch/script_container.sh
@@ -0,0 +1,418 @@
+#!/bin/bash
+
+############################################################# SETUP #############################################################
+
+Setup_Script()
+{
+	# PWD, STATUS OF MOUNTED DISKS
+	echo "pwd =" $PWD
+	if [[ $PWD != /scratch/slurm/${SLURM_JOB_ID} ]] ; then
+	    echo "LOCAL DIRECTORY " $PWD " NOT SUPPORTED"
+	    exit 1
+	fi
+	echo "df -h:"
+	df -h
+
+	# ENVIRONMENT (shell script or xml?)
+	ENV_EXT="${ENVIRONMENT##*.}"
+	ENV_DIR=`dirname $ENVIRONMENT`
+	if [[ $ENV_EXT == "sh" ]] ; then
+	    source $ENVIRONMENT
+	elif [[ $ENV_EXT == "xml" ]] ; then
+	    if [[ $ENV_DIR == "." ]] ; then
+		source /group/halld/Software/build_scripts/gluex_env_jlab.sh /group/halld/www/halldweb/html/halld_versions/$ENVIRONMENT
+	    else
+		source /group/halld/Software/build_scripts/gluex_env_jlab.sh $ENVIRONMENT
+		local XML=1
+	    fi
+	else
+	    echo "ENVIRONMENT " $ENVIRONMENT " not supported"
+	    exit 1
+	fi
+	printenv
+
+	# COPY CCDB SQLITE FILE TO LOCAL DISK
+	if [[ $CCDB_CONNECTION  == *"sqlite"* ]] ; then
+	    local SCRATCH=/scratch/slurm/${SLURM_JOB_ID}
+	    ls -l $SCRATCH
+	    local NEW_SQLITE=${SCRATCH}/ccdb.sqlite
+            cp -v ${CCDB_CONNECTION:10} $NEW_SQLITE
+            export CCDB_CONNECTION=sqlite:///$NEW_SQLITE
+            export JANA_CALIB_URL=sqlite:///$NEW_SQLITE
+            echo "JANA_CALIB_URL: " $JANA_CALIB_URL
+	fi
+
+	if [[ $INPUTFILE  == *"tar" ]] ; then
+	    # extract and flatten directories
+	    tar xvf $INPUTFILE --transform='s/.*\///'
+	    rm $INPUTFILE
+	    # update input files
+	    export INPUTFILE=${INPUTFILE/.hddm.tar/_*.hddm}
+	    echo INPUTFILE = $INPUTFILE
+	fi
+
+	# LIST WORKING DIRECTORY
+	echo "LOCAL FILES"
+	ls -l
+}
+
+####################################################### UTILITY FUNCTIONS #######################################################
+
+Extract_SkimName()
+{
+	# to extract the skim name, first extract the locations of the last two periods in the file name
+	local LAST_INDEX=0
+	local SECOND_TO_LAST_INDEX=0
+	local INPUT_FILE=$1
+	for INDEX in `echo $INPUT_FILE | grep -bo '\.' | awk 'BEGIN {FS=":"}{print $1}'`; do
+		SECOND_TO_LAST_INDEX=$LAST_INDEX
+		LAST_INDEX=$INDEX
+	done
+
+	# extract the skim name: awk & grep use different location #'ing, so must convert
+	local LENGTH=$[$LAST_INDEX - $SECOND_TO_LAST_INDEX - 1]
+	local START=$[$SECOND_TO_LAST_INDEX + 2]
+	local LOCAL_SKIM_NAME=`echo $INPUT_FILE | awk -v size="$LENGTH" -v start="$START" '{print substr($0,start,size)}'`
+	echo "SKIM_NAME:" $LOCAL_SKIM_NAME
+	
+	#return the result "by reference"
+	local __result=$2
+	eval $__result="'$LOCAL_SKIM_NAME'"
+}
+
+Extract_BaseName()
+{
+	# base name is everything before the last period
+	local INPUT_FILE=$1
+	local LENGTH=`echo $INPUT_FILE | awk '{print index($0,".")}'`
+	let LENGTH-=1
+	local LOCAL_BASE_NAME=`echo $INPUT_FILE | awk -v size="$LENGTH" '{print substr($0,1,size)}'`
+	echo "BASE_NAME: " $LOCAL_BASE_NAME
+
+	#return the result "by reference"
+	local __result=$2
+	eval $__result="'$LOCAL_BASE_NAME'"
+}
+
+####################################################### SAVE OUTPUT FILES #######################################################
+
+Save_OutputFiles()
+{
+	# SEE WHAT FILES ARE PRESENT
+	echo "FILES PRESENT PRIOR TO SAVE:"
+	ls -l
+
+	# REMOVE INPUT FILE: so that it's easier to determine which remaining files are skims
+	rm -f $INPUTFILE
+
+	# BUILD TAPEDIR, IF $OUTDIR_LARGE STARTS WITH "/cache/"
+	# AND CACHE_PIN_DAYS WAS GIVEN AND GREATER THAN 0
+	# If so, output files are pinned & jcache put.  If not, then they aren't. 
+	local TAPEDIR=""
+	local OUTDIR_LARGE_BASE=`echo $OUTDIR_LARGE | awk '{print substr($0,1,7)}'`
+	# first strip /cache/, then insert /mss/
+	if [ "$OUTDIR_LARGE_BASE" == "/cache/" ] && [ $CACHE_PIN_DAYS -gt 0 ] ; then
+		local OUTPATH=`echo $OUTDIR_LARGE | awk '{print substr($0,8)}'`
+		TAPEDIR=/mss/${OUTPATH}/
+	fi
+
+	# CALL SAVE FUNCTIONS
+	Save_Histograms
+	Save_Histograms
+	Save_REST
+	Save_REST
+	Save_JANADot
+	Save_EVIOSkims
+	Save_EVIOSkims
+	Save_HDDMSkims
+	Save_HDDMSkims
+	Save_ROOTFiles
+	Save_ROOTFiles
+	Save_IDXA
+
+	# SEE WHAT FILES ARE LEFT
+	echo "FILES REMAINING AFTER SAVING:"
+	ls -l
+}
+
+Save_Histograms()
+{
+	# SAVE ROOT HISTOGRAMS
+	if [ -e hd_root.root ]; then
+		echo "Saving histogram file"
+
+		# setup output dirs
+		local OUTDIR_THIS=${OUTDIR_LARGE}/hists/${RUN_NUMBER}/
+		mkdir -p -m 755 ${OUTDIR_THIS}
+
+		# save it
+		local OUTPUT_FILE=${OUTDIR_THIS}/hd_root_${RUN_NUMBER}_${FILE_NUMBER}.root
+		mv -v hd_root.root $OUTPUT_FILE
+		chmod 644 $OUTPUT_FILE
+		
+		# check if the target file is empty, if yes try again
+		if [ ! -s $OUTPUT_FILE ] ; then 
+		    mv -v hd_root.root $OUTPUT_FILE
+		    chmod 644 $OUTPUT_FILE
+		fi
+
+		# if the target file is still empty, fail job
+		if [ ! -s $OUTPUT_FILE ] ; then
+		    exit 13
+		fi
+
+		# force save to tape & pin
+		if [ "$TAPEDIR" != "" ]; then
+			echo jcache pin $OUTPUT_FILE -D $CACHE_PIN_DAYS
+			jcache pin $OUTPUT_FILE -D $CACHE_PIN_DAYS
+			echo jcache put $OUTPUT_FILE
+			jcache put $OUTPUT_FILE
+		fi
+	fi
+}
+
+Save_REST()
+{
+	# SAVE REST FILE
+	if [ -e dana_rest.hddm ]; then
+		echo "Saving REST file"
+
+		# setup output dirs
+		local OUTDIR_THIS=${OUTDIR_LARGE}/REST/${RUN_NUMBER}/
+		mkdir -p -m 755 $OUTDIR_THIS
+
+		# save it
+		local OUTPUT_FILE=${OUTDIR_THIS}/dana_rest_${RUN_NUMBER}_${FILE_NUMBER}.hddm
+		mv -v dana_rest.hddm $OUTPUT_FILE
+		chmod 644 $OUTPUT_FILE
+
+		# check if the target file is empty, if yes try again
+		if [ ! -s $OUTPUT_FILE ] ; then 
+		    mv -v dana_rest $OUTPUT_FILE
+		    chmod 644 $OUTPUT_FILE
+		fi
+
+		# if the target file is still empty, fail job
+		if [ ! -s $OUTPUT_FILE ] ; then
+		    exit 13
+		fi
+
+		# force save to tape & pin
+		if [ "$TAPEDIR" != "" ]; then
+		        echo jcache pin $OUTPUT_FILE -D $CACHE_PIN_DAYS
+			jcache pin $OUTPUT_FILE -D $CACHE_PIN_DAYS
+			echo jcache put $OUTPUT_FILE
+			jcache put $OUTPUT_FILE
+		fi
+	fi
+}
+
+Save_JANADot()
+{
+	# SAVE JANADOT FILE
+	if [ -e jana.dot ]; then
+		echo "Saving JANADOT file"
+		dot -Tps2 jana.dot -o jana.ps
+		ps2pdf jana.ps
+
+		# setup output dir
+		local OUTDIR_THIS=${OUTDIR_SMALL}/log/${RUN_NUMBER}/
+		mkdir -p -m 755 $OUTDIR_THIS
+
+		# save it
+		local OUTPUT_FILE=${OUTDIR_THIS}/janadot_${RUN_NUMBER}_${FILE_NUMBER}.pdf
+		mv -v jana.pdf $OUTPUT_FILE
+		chmod 644 $OUTPUT_FILE
+	fi
+}
+
+Save_EVIOSkims()
+{
+	# SAVE EVIO SKIMS
+        local NUM_FILES=`ls *.evio 2>/dev/null | wc -l`
+        if [ $NUM_FILES -eq 0 ] ; then
+                echo "No EVIO skim files produced"
+                return
+        fi
+
+	echo "Saving EVIO skim files"
+	for EVIO_FILE in `ls *.evio`; do
+		Extract_SkimName $EVIO_FILE SKIM_NAME
+
+		# setup output dir
+		local OUTDIR_THIS=${OUTDIR_LARGE}/${SKIM_NAME}/${RUN_NUMBER}/
+		mkdir -p -m 755 $OUTDIR_THIS
+
+		# save it
+		local OUTPUT_FILE=${OUTDIR_THIS}/${SKIM_NAME}_${RUN_NUMBER}_${FILE_NUMBER}.evio
+		mv -v $EVIO_FILE $OUTPUT_FILE
+		chmod 644 $OUTPUT_FILE
+		
+		# check if the target file is empty, if yes try again
+		if [ ! -s $OUTPUT_FILE ] ; then 
+		    mv -v $EVIO_FILE $OUTPUT_FILE
+		    chmod 644 $OUTPUT_FILE
+		fi
+
+		# force save to tape & pin
+		if [ "$TAPEDIR" != "" ]; then
+			jcache pin $OUTPUT_FILE -D $CACHE_PIN_DAYS
+			jcache put $OUTPUT_FILE
+		fi
+	done
+}
+
+Save_HDDMSkims()
+{
+	local NUM_FILES=`ls *.hddm 2>/dev/null | wc -l`
+	if [ $NUM_FILES -eq 0 ] ; then
+		echo "No HDDM skim files produced"
+		return
+	fi
+
+	# SAVE HDDM SKIMS #assumes REST file already backed up and removed!
+	echo "Saving HDDM skim files"
+	for HDDM_FILE in `ls *.hddm`; do
+		Extract_BaseName $HDDM_FILE SKIM_NAME
+
+		# setup output dir
+		local OUTDIR_THIS=${OUTDIR_LARGE}/${SKIM_NAME}/${RUN_NUMBER}/
+		mkdir -p -m 755 $OUTDIR_THIS
+
+		# save it
+		local OUTPUT_FILE=${OUTDIR_THIS}/${SKIM_NAME}_${RUN_NUMBER}_${FILE_NUMBER}.hddm
+		mv -v $HDDM_FILE $OUTPUT_FILE
+		chmod 644 $OUTPUT_FILE
+
+		# check if the target file is empty, if yes try again
+		if [ ! -s $OUTPUT_FILE ] ; then 
+		    mv -v $HDDM_FILE $OUTPUT_FILE
+		    chmod 644 $OUTPUT_FILE
+		fi
+		# force save to tape & pin
+		if [ "$TAPEDIR" != "" ]; then
+			jcache pin $OUTPUT_FILE -D $CACHE_PIN_DAYS
+			jcache put $OUTPUT_FILE
+		fi
+	done
+}
+
+Save_ROOTFiles()
+{
+	# SAVE OTHER ROOT FILES
+        local NUM_FILES=`ls *.root 2>/dev/null | wc -l`
+        if [ $NUM_FILES -eq 0 ] ; then
+                echo "No additional ROOT files produced"
+                return
+        fi
+
+	echo "Saving other ROOT files"
+	for ROOT_FILE in `ls *.root`; do
+		Extract_BaseName $ROOT_FILE BASE_NAME
+
+		# setup output dir
+		local OUTDIR_THIS=${OUTDIR_LARGE}/${BASE_NAME}/${RUN_NUMBER}/
+		mkdir -p -m 755 $OUTDIR_THIS
+
+		# save it
+		local OUTPUT_FILE=${OUTDIR_THIS}/${BASE_NAME}_${RUN_NUMBER}_${FILE_NUMBER}.root
+		mv -v $ROOT_FILE $OUTPUT_FILE
+		chmod 644 $OUTPUT_FILE
+
+		# check if the target file is empty, if yes try again
+		if [ ! -s $OUTPUT_FILE ] ; then 
+		    mv -v $ROOT_FILE $OUTPUT_FILE
+		    chmod 644 $OUTPUT_FILE
+		fi
+
+		# if the target file is still empty, fail job
+		if [ ! -s $OUTPUT_FILE ] ; then
+		    exit 13
+		fi
+
+		# force save to tape & pin
+		if [ "$TAPEDIR" != "" ]; then
+			jcache pin $OUTPUT_FILE -D $CACHE_PIN_DAYS
+			jcache put $OUTPUT_FILE
+		fi
+	done
+}
+
+Save_IDXA()
+{
+	# SAVE IDXA FILES
+        local NUM_FILES=`ls *.idxa 2>/dev/null | wc -l`
+        if [ $NUM_FILES -eq 0 ] ; then
+                echo "No IDXA files produced"
+                return
+        fi
+
+	echo "Saving IDXA files"
+	for IDXA_FILE in `ls *.idxa`; do
+		Extract_BaseName $IDXA_FILE BASE_NAME
+
+		# setup output dir
+		local OUTDIR_THIS=${OUTDIR_SMALL}/IDXA/${RUN_NUMBER}/
+		mkdir -p -m 755 $OUTDIR_THIS
+
+		# save it
+		local OUTPUT_FILE=${OUTDIR_THIS}/${BASE_NAME}_${RUN_NUMBER}_${FILE_NUMBER}.idxa
+		mv -v $IDXA_FILE $OUTPUT_FILE
+		chmod 644 $OUTPUT_FILE
+	done
+}
+
+########################################################## MAIN FUNCTION ########################################################
+
+Run_Script()
+{
+	Setup_Script
+
+	# RUN JANA IN SINGULARITY
+	cp $CONFIG_FILE .
+	local CONFIG_FILE=`basename $CONFIG_FILE`
+	cp /group/halld/Software/hd_utilities/launch_scripts/launch/gluexrunscript.sh .
+	ls -l
+	if [[ $CCDB_CONNECTION  == *"sqlite"* ]] ; then
+	    singularity run -e -H $PWD:/home --bind /cvmfs/oasis.opensciencegrid.org/gluex/group/halld:/group/halld /cvmfs/singularity.opensciencegrid.org/jeffersonlab/gluex_devel:latest ./gluexrunscript.sh -v $ENVIRONMENT -s hd_root $INPUTFILE --config=$CONFIG_FILE
+	else
+	    singularity run -e -H $PWD:/home --bind /cvmfs/oasis.opensciencegrid.org/gluex/group/halld:/group/halld /cvmfs/singularity.opensciencegrid.org/jeffersonlab/gluex_devel:latest ./gluexrunscript.sh -v $ENVIRONMENT hd_root $INPUTFILE --config=$CONFIG_FILE
+	fi
+	
+	# RETURN CODE
+	RETURN_CODE=$?
+	echo "Return Code = " $RETURN_CODE
+	if [ $RETURN_CODE -ne 0 ]; then
+		exit $RETURN_CODE
+	fi
+
+	# SAVE OUTPUTS
+	Save_OutputFiles
+}
+
+######################################################### EXECUTE SCRIPT ########################################################
+
+# SET INPUTS
+ENVIRONMENT=$1
+INPUTFILE=$2
+CONFIG_FILE=$3
+OUTDIR_LARGE=$4
+OUTDIR_SMALL=$5
+RUN_NUMBER=$6
+FILE_NUMBER=$7
+CACHE_PIN_DAYS=$8
+
+# PRINT INPUTS
+echo "HOSTNAME          = $HOSTNAME"
+echo "ENVIRONMENT       = $ENVIRONMENT"
+echo "INPUTFILE         = $INPUTFILE"
+echo "CONFIG_FILE       = $CONFIG_FILE"
+echo "OUTDIR_LARGE      = $OUTDIR_LARGE"
+echo "OUTDIR_SMALL      = $OUTDIR_SMALL"
+echo "RUN_NUMBER        = $RUN_NUMBER"
+echo "FILE_NUMBER       = $FILE_NUMBER"
+echo "CACHE_PIN_DAYS    = $CACHE_PIN_DAYS"
+
+# RUN
+Run_Script
+