diff --git a/Campaigns/CRVWB/reco.sh b/Campaigns/CRVWB/reco.sh index c8e260c..d9e5022 100755 --- a/Campaigns/CRVWB/reco.sh +++ b/Campaigns/CRVWB/reco.sh @@ -19,16 +19,14 @@ tee_date "args are: $@" RCT=0 -setup mu2e -setup CRVTeststand $MOO_CRVTESTSTAND -muse setup Offline v10_27_00 -setup -B mu2etools -setup -B mu2efiletools -setup -B sam_web_client -setup -B ifdhc +source /cvmfs/mu2e.opensciencegrid.org/setupmu2e-art.sh +printenv +muse setup ops +printenv +spack load $MOO_CRVTESTSTAND # modify CRV exe control to use local directory -cat $CRVTESTSTAND_FQ_DIR/config.txt | \ +cat $CRVTESTSTAND_DIR/config.txt | \ awk '{ if(index($1,"crv")==1) { print $1" ./" @@ -77,6 +75,7 @@ do RNFN=rec.mu2e.${DES}-noadc.${MOO_CONFIG}.${SEQ}.root RPFN=etc.mu2e.${DES}_reco.${MOO_CONFIG}.${SEQ}.pdf RTFN=etc.mu2e.${DES}_reco.${MOO_CONFIG}.${SEQ}.txt + DQMN=ntd.mu2e.${DES}-DQM.${MOO_CONFIG}-file.${SEQ}.root [ $RCT -ne 0 ] && break @@ -89,6 +88,7 @@ do date > rec2.mu2e.${DES}.${CFG}.${SEQ}.root date > rec.mu2e.${DES}.${CFG}.${SEQ}.pdf date > rec.mu2e.${DES}.${CFG}.${SEQ}.txt + date > dqm.mu2e.${DES}.${CFG}.$SEQ.root else tee_date "Running parseCrv $SEQ" @@ -123,6 +123,7 @@ do mv rec2.mu2e.${DES}.${CFG}.${SEQ}.root $RNFN mv rec.mu2e.${DES}.${CFG}.${SEQ}.pdf $RPFN mv rec.mu2e.${DES}.${CFG}.${SEQ}.txt $RTFN + mv dqm.mu2e.${DES}.${CFG}.$SEQ.root $DQMN echo $BNAME > parents_${BNAME} @@ -134,6 +135,7 @@ do echo "$LOCROOT $RNFN parents_${BNAME}" >> output.txt echo "$LOCTXT $RPFN parents_${BNAME}" >> output.txt echo "$LOCTXT $RTFN parents_${BNAME}" >> output.txt + echo "$LOCTXT $DQMN parents_${BNAME}" >> output.txt if [ $RCT -eq 0 ]; then release_SAM_file consumed @@ -156,6 +158,9 @@ echo "$LOCTXT $LGFN none" >> output.txt tee_date "Final ls" ls -l +tee_date "decode token" +httokendecode -H +save_environment final_env_check if [ "$MOO_LOCAL" ]; then RCP=0 diff --git a/Util/functions.sh b/Util/functions.sh index 85c5c74..4300b6a 100644 --- a/Util/functions.sh +++ b/Util/functions.sh @@ -47,17 +47,21 @@ node_summary() { echo "cvmfs: ERROR on simple ls" fi - if [[ $RC -ne 0 || $VERBOSE -gt 0 ]]; then - if command -v cvmfs_config ; then - cvmfs_config stat -v fermilab.opensciencegrid.org - cvmfs_config stat -v mu2e.opensciencegrid.org + if [ $VERBOSE -gt 1 ]; then + if [[ $RC -ne 0 || $VERBOSE -gt 0 ]]; then + if command -v cvmfs_config ; then + cvmfs_config stat -v fermilab.opensciencegrid.org + cvmfs_config stat -v mu2e.opensciencegrid.org + fi fi fi - if [ "$(rpm -q -a | grep zstd)" ]; then - echo "rpm_check: has art 3.6 rpms" - else - echo "rpm_check: ERROR - does not have art 3.6 rpms" + if [ $VERBOSE -gt 1 ]; then + if [ "$(rpm -q -a | grep zstd)" ]; then + echo "rpm_check: has art 3.6 rpms" + else + echo "rpm_check: ERROR - does not have art 3.6 rpms" + fi fi echo "GRID_USER: $GRID_USER" @@ -69,7 +73,11 @@ node_summary() { if [ $RC -eq 0 ] ; then echo "kerberos: OK" else - echo "kerberos: NOT OK" + if [ "$GRID_USER" ]; then + echo "kerberos: NOT OK (as expected for grid jobs)" + else + echo "kerberos: NOT OK" + fi fi # kerberos not expected on a grid node if [[ ( $RC -ne 0 && ! "$GRID_USER" ) || VERBOSE -gt 0 ]]; then @@ -105,7 +113,7 @@ node_summary() { echo "token: NOT OK" fi if [[ $RC -ne 0 || VERBOSE -gt 0 ]]; then - httokendecode + httokendecode -H fi else echo "token: could not find httokendecode" @@ -184,6 +192,14 @@ save_environment() { echo "********* ups active" ups active fi + if command -v spack 2>&1 > /dev/null ; then + echo "********* spack variables" + echo "SPACK_ROOT=$SPACK_ROOT" + echo "SPACK_ENV=$SPACK_ENV" + echo "SPACK_ENV_VIEW=$SPACK_ENV_VIEW" + echo "spack loaded:" + spack find -lv --loaded + fi echo "********* ls" /bin/ls -al echo "********* ls of \*" @@ -358,7 +374,6 @@ release_SAM_file() { # # $1 = optional time limit in s # outdir assumed: /pnfs/mu2e/scratch/users/$GRID_USER/watchdog -# ifdh assumed setup watchdog() { @@ -374,10 +389,17 @@ watchdog() { DD=/pnfs/mu2e/scratch/users/$ULOC/watchdog - [ -z "$MU2E" ] && source /cvmfs/mu2e.opensciencegrid.org/setupmu2e-art.sh - [ -z "$SETUP_IFDHC" ] && setup ifdhc + source /cvmfs/mu2e.opensciencegrid.org/setupmu2e-art.sh + if ! command -v ifdh >& /dev/null ; then + if [ "$MU2E_SPACK" ]; then + muse setup ops + else + setup ifdhc + fi + fi - ifdh mkdir_p $DD + #ifdh mkdir_p $DD + gfal-mkdir -p $DD local T0=$( date +%s ) local DT=0 diff --git a/Util/pushOutput.py b/Util/pushOutput.py index 977c703..a51d92d 100644 --- a/Util/pushOutput.py +++ b/Util/pushOutput.py @@ -2,7 +2,6 @@ #! /usr/bin/python #from __future__ import absolute_import #from __future__ import print_function -import gfal2 import argparse import sys import os @@ -290,7 +289,6 @@ def fillDataFile(line): #/pnfs/mu2e/persistent/datasets/phy-etc/etc/mu2e/test/000_000/txt/16/14 #/pnfs/mu2e/scratch/datasets/phy-etc/etc/mu2e/test/000_000/txt/16/14 - # # return success, deleted or not found, (0), or fail (2) # @@ -302,21 +300,24 @@ def rmFile(dfile): dfile.dcacheinfo = {} token = getToken() - ctx.set_opt_string("BEARER", "TOKEN", token) + env = {"BEARER_TOKEN" : token} + cmd = f"gfal-rm -t 300 {dfile.url}" for itry, tsleep in enumerate(retries) : time.sleep(tsleep) - try: - stat = ctx.unlink(dfile.url) - teeDate(1,"Removed {}/{}".format(dfile.path,dfile.fn)) + result = subprocess.run(cmd, shell=True, timeout=320, + encoding="utf-8",capture_output=True, env=env) + teeDate(1,"Removed {}/{}".format(dfile.path,dfile.fn)) + + if result.returncode == 0 : return 0 - except Exception as e: - message = str(e) - if "File not found" in message : - return 0 + elif result.returncode == 2 and "MISSING" in result.stdout : + return 0 + else : print("ERROR - rm failed for try {} for {}"\ .format(itry,dfile.url)) - print("message: "+message) + print(result.stdout) + print(result.stderr) return 2 @@ -489,36 +490,31 @@ def copyFile(dfile): if failRate > 0. and random.uniform(0.0,4.0) < failRate : return 2 - # Set gfal transfer parameters - params = ctx.transfer_parameters() - params.overwrite = False - params.set_checksum = False - params.timeout = 300 - dfile.donecopy = False dfile.copytime = -1 localurl = "file://"+os.path.realpath(dfile.localfs) token = getToken() - ctx.set_opt_string("BEARER", "TOKEN", token) + + env = {"BEARER_TOKEN" : token} + cmd = "gfal-copy --parent --timeout 1000" + cmd = cmd + " " + localurl + " " + dfile.url rc = 999 for itry, tsleep in enumerate(retries) : time.sleep(tsleep) - try: - rc = ctx.filecopy(params, localurl, dfile.url) + result = subprocess.run(cmd, shell=True, timeout=1100, + encoding="utf-8", capture_output=True, env=env) + rc = result.returncode + if rc == 0 : break - except Exception as e: - rc = 1 - # gfal only raises generic errors, so have to parse the text - message = str(e) - # if the output file already exists, then this function is done - if "file exists" in message : - teeDate(1,"WARNING - output file exists for {}/{}"\ - .format(dfile.path,dfile.fn)) - return 1 + elif "exists" in result.stderr : + teeDate(1,"WARNING - output file exists for {}/{}"\ + .format(dfile.path,dfile.fn)) + return 1 + else : teeDate(0,"ERROR - copy failed for try {} for {}".\ format(itry,dfile.url)) print("message: "+message) @@ -865,12 +861,7 @@ def writeLog(dfile): runTime = int( time.time() ) recoverDelay = 7200 - # gfal2 and samweb functions are methods of global objects - - ctx = gfal2.creat_context() - token = getToken() - ctx.set_opt_string("BEARER", "TOKEN", token) - + # samweb functions are methods of global objects samweb = samweb_client.SAMWebClient() # default, normally take app name and verison from MOO_CONFIG diff --git a/Util/wrapper.sh b/Util/wrapper.sh index 780ccc6..59e75c4 100755 --- a/Util/wrapper.sh +++ b/Util/wrapper.sh @@ -20,6 +20,14 @@ printenv | grep LC_ printenv LANG unset LC_CTYPE +tee_date Check BEARER_TOKEN +if [ "$BEARER_TOKEN" ]; then + echo "found BEARER_TOKEN set, unsetting it" + unset BEARER_TOKEN +else + echo "found BEARER_TOKEN is not set" +fi + # always need to find setup source /cvmfs/mu2e.opensciencegrid.org/setupmu2e-art.sh @@ -42,9 +50,13 @@ save_environment wrapper_start # if [[ "$MOO_SOURCE" =~ "/" ]]; then tee_date fetching Offline Ops from $MOO_SOURCE - setup ifdhc LFN=$(basename $MOO_SOURCE) - ifdh cp --cp_maxretries=1 $MOO_SOURCE $LFN + ( + if ! command -v ifdh >& /dev/null ; then + muse setup ops + fi + ifdh cp --cp_maxretries=1 $MOO_SOURCE $LFN + ) tee_date untar OfflineOps source $LFN tar -xf $LFN rm -f $LFN @@ -71,6 +83,21 @@ if [ "$PRODUCTS" ]; then #unset PRODUCTS fi +# +# undo whatever spack exists +# +if [ "$SPACK_ROOT" ]; then + spack unload -a + unset SPACK_ROOT + unset SPACK_ENV + unset SPACK_ENV_VIEW +fi + + +# allow mu2einit to run again +unset MU2E + + tee_date "mv CONDOR_DIR_INPUT to cwd" if [ "$CONDOR_DIR_INPUT" ]; then /bin/ls -al $CONDOR_DIR_INPUT diff --git a/bin/pushOutput b/bin/pushOutput index 2512a0d..e072ffa 100755 --- a/bin/pushOutput +++ b/bin/pushOutput @@ -20,7 +20,7 @@ source /cvmfs/mu2e.opensciencegrid.org/setupmu2e-art.sh setup sam_web_client #setup root v6_26_06 -q e20:p3913:prof # for gfal2 -export PYTHONPATH=$PYTHONPATH:/usr/lib64/python3.6/site-packages:/usr/lib/python3.6/site-packages +#export PYTHONPATH=$PYTHONPATH:/usr/lib64/python3.6/site-packages:/usr/lib/python3.6/site-packages export SAMWEB_HTTP_HANDLER=urllib #which python unset X509_USER_PROXY