adjustments for al9/spack and crv v21, add DQM

Campaigns/CRVWB/reco.sh

@@ -19,16 +19,14 @@ tee_date "args are: $@"
 
 RCT=0
 
-setup mu2e
-setup CRVTeststand $MOO_CRVTESTSTAND
-muse setup Offline v10_27_00
-setup -B mu2etools
-setup -B mu2efiletools
-setup -B sam_web_client
-setup -B ifdhc
+source /cvmfs/mu2e.opensciencegrid.org/setupmu2e-art.sh
+printenv
+muse setup ops
+printenv
+spack load $MOO_CRVTESTSTAND
 
 # modify CRV exe control to use local directory
-cat $CRVTESTSTAND_FQ_DIR/config.txt | \
+cat $CRVTESTSTAND_DIR/config.txt | \
 awk '{
   if(index($1,"crv")==1) {
     print $1" ./"
@@ -77,6 +75,7 @@ do
     RNFN=rec.mu2e.${DES}-noadc.${MOO_CONFIG}.${SEQ}.root
     RPFN=etc.mu2e.${DES}_reco.${MOO_CONFIG}.${SEQ}.pdf
     RTFN=etc.mu2e.${DES}_reco.${MOO_CONFIG}.${SEQ}.txt
+    DQMN=ntd.mu2e.${DES}-DQM.${MOO_CONFIG}-file.${SEQ}.root
 
     [ $RCT -ne 0 ] && break
 
@@ -89,6 +88,7 @@ do
         date > rec2.mu2e.${DES}.${CFG}.${SEQ}.root
         date > rec.mu2e.${DES}.${CFG}.${SEQ}.pdf
         date > rec.mu2e.${DES}.${CFG}.${SEQ}.txt
+        date > dqm.mu2e.${DES}.${CFG}.$SEQ.root
     else
 
         tee_date "Running parseCrv $SEQ"
@@ -123,6 +123,7 @@ do
     mv rec2.mu2e.${DES}.${CFG}.${SEQ}.root $RNFN
     mv rec.mu2e.${DES}.${CFG}.${SEQ}.pdf  $RPFN
     mv rec.mu2e.${DES}.${CFG}.${SEQ}.txt  $RTFN
+    mv dqm.mu2e.${DES}.${CFG}.$SEQ.root $DQMN
 
     echo $BNAME > parents_${BNAME}
 
@@ -134,6 +135,7 @@ do
     echo "$LOCROOT $RNFN  parents_${BNAME}" >> output.txt
     echo "$LOCTXT  $RPFN  parents_${BNAME}" >> output.txt
     echo "$LOCTXT  $RTFN  parents_${BNAME}" >> output.txt
+    echo "$LOCTXT  $DQMN  parents_${BNAME}" >> output.txt
 
     if [ $RCT -eq 0 ]; then
         release_SAM_file consumed
@@ -156,6 +158,9 @@ echo "$LOCTXT $LGFN none" >> output.txt
 tee_date "Final ls"
 ls -l
 
+tee_date "decode token"
+httokendecode -H
+save_environment final_env_check
 
 if [ "$MOO_LOCAL" ]; then
     RCP=0

Util/functions.sh

@@ -47,17 +47,21 @@ node_summary() {
         echo "cvmfs: ERROR on simple ls"
     fi
 
-    if [[ $RC -ne 0 || $VERBOSE -gt 0 ]]; then
-        if command -v  cvmfs_config ; then
-            cvmfs_config stat -v fermilab.opensciencegrid.org
-            cvmfs_config stat -v mu2e.opensciencegrid.org
+    if [ $VERBOSE -gt 1 ]; then
+        if [[ $RC -ne 0 || $VERBOSE -gt 0 ]]; then
+            if command -v  cvmfs_config ; then
+                cvmfs_config stat -v fermilab.opensciencegrid.org
+                cvmfs_config stat -v mu2e.opensciencegrid.org
+            fi
         fi
     fi
 
-    if [ "$(rpm -q -a | grep zstd)" ]; then
-        echo "rpm_check: has art 3.6 rpms"
-    else
-        echo "rpm_check: ERROR - does not have art 3.6 rpms"
+    if [ $VERBOSE -gt 1 ]; then
+        if [ "$(rpm -q -a | grep zstd)" ]; then
+            echo "rpm_check: has art 3.6 rpms"
+        else
+            echo "rpm_check: ERROR - does not have art 3.6 rpms"
+        fi
     fi
 
     echo "GRID_USER: $GRID_USER"
@@ -69,7 +73,11 @@ node_summary() {
         if [ $RC -eq 0 ] ; then
             echo "kerberos: OK"
         else
-            echo "kerberos: NOT OK"
+            if [ "$GRID_USER" ]; then
+                echo "kerberos: NOT OK (as expected for grid jobs)"
+            else
+                echo "kerberos: NOT OK"
+            fi
         fi
         # kerberos not expected on a grid node
         if [[ ( $RC -ne 0 && ! "$GRID_USER" ) || VERBOSE -gt 0 ]]; then
@@ -105,7 +113,7 @@ node_summary() {
             echo "token: NOT OK"
         fi
         if [[ $RC -ne 0 || VERBOSE -gt 0 ]]; then
-            httokendecode
+            httokendecode -H
         fi
     else
         echo "token: could not find httokendecode"
@@ -184,6 +192,14 @@ save_environment() {
             echo "********* ups active"
             ups active
         fi
+        if command -v spack 2>&1 > /dev/null ; then
+            echo "********* spack variables"
+            echo "SPACK_ROOT=$SPACK_ROOT"
+            echo "SPACK_ENV=$SPACK_ENV"
+            echo "SPACK_ENV_VIEW=$SPACK_ENV_VIEW"
+            echo "spack loaded:"
+            spack find -lv --loaded
+        fi
         echo "********* ls"
         /bin/ls -al
         echo "********* ls of \*"
@@ -358,7 +374,6 @@ release_SAM_file() {
 #
 # $1 = optional time limit in s
 # outdir assumed: /pnfs/mu2e/scratch/users/$GRID_USER/watchdog
-# ifdh assumed setup
 
 watchdog() {
 
@@ -374,10 +389,17 @@ watchdog() {
 
     DD=/pnfs/mu2e/scratch/users/$ULOC/watchdog
 
-    [ -z "$MU2E" ] && source /cvmfs/mu2e.opensciencegrid.org/setupmu2e-art.sh
-    [ -z "$SETUP_IFDHC" ] && setup ifdhc
+    source /cvmfs/mu2e.opensciencegrid.org/setupmu2e-art.sh
+    if ! command -v ifdh >& /dev/null ; then
+        if [ "$MU2E_SPACK" ]; then
+            muse setup ops
+        else
+            setup ifdhc
+        fi
+    fi
 
-    ifdh mkdir_p $DD
+    #ifdh mkdir_p $DD
+    gfal-mkdir -p $DD
 
     local T0=$( date +%s )
     local DT=0

Util/pushOutput.py

@@ -2,7 +2,6 @@
 #! /usr/bin/python
 #from __future__ import absolute_import
 #from __future__ import print_function
-import gfal2
 import argparse
 import sys
 import os
@@ -290,7 +289,6 @@ def fillDataFile(line):
 #/pnfs/mu2e/persistent/datasets/phy-etc/etc/mu2e/test/000_000/txt/16/14
 #/pnfs/mu2e/scratch/datasets/phy-etc/etc/mu2e/test/000_000/txt/16/14
 
-
 #
 # return success, deleted or not found, (0), or fail (2)
 #
@@ -302,21 +300,24 @@ def rmFile(dfile):
     dfile.dcacheinfo = {}
 
     token = getToken()
-    ctx.set_opt_string("BEARER", "TOKEN", token)
+    env = {"BEARER_TOKEN" : token}
+    cmd = f"gfal-rm -t 300 {dfile.url}"
 
     for itry, tsleep in enumerate(retries) :
         time.sleep(tsleep)
-        try:
-            stat = ctx.unlink(dfile.url)
-            teeDate(1,"Removed {}/{}".format(dfile.path,dfile.fn))
+        result = subprocess.run(cmd, shell=True, timeout=320,
+                    encoding="utf-8",capture_output=True, env=env)
+        teeDate(1,"Removed {}/{}".format(dfile.path,dfile.fn))
+
+        if result.returncode == 0 :
             return 0
-        except Exception as e:
-            message = str(e)
-            if "File not found" in message :
-                return 0
+        elif result.returncode == 2 and "MISSING" in result.stdout :
+            return 0
+        else :
             print("ERROR - rm failed for try {} for {}"\
                   .format(itry,dfile.url))
-            print("message: "+message)
+            print(result.stdout)
+            print(result.stderr)
 
     return 2
 
@@ -489,36 +490,31 @@ def copyFile(dfile):
     if failRate > 0. and random.uniform(0.0,4.0) < failRate :
         return 2
 
-    # Set gfal transfer parameters
-    params = ctx.transfer_parameters()
-    params.overwrite = False
-    params.set_checksum = False
-    params.timeout = 300
-
     dfile.donecopy = False
     dfile.copytime = -1
 
     localurl = "file://"+os.path.realpath(dfile.localfs)
 
     token = getToken()
-    ctx.set_opt_string("BEARER", "TOKEN", token)
+
+    env = {"BEARER_TOKEN" : token}
+    cmd = "gfal-copy --parent --timeout 1000"
+    cmd = cmd + " " + localurl + " " + dfile.url
 
     rc = 999
     for itry, tsleep in enumerate(retries) :
         time.sleep(tsleep)
 
-        try:
-            rc = ctx.filecopy(params, localurl, dfile.url)
+        result = subprocess.run(cmd, shell=True, timeout=1100,
+                    encoding="utf-8", capture_output=True, env=env)
+        rc = result.returncode
+        if rc == 0 :
             break
-        except Exception as e:
-            rc = 1
-            # gfal only raises generic errors, so have to parse the text
-            message = str(e)
-            # if the output file already exists, then this function is done
-            if "file exists" in message :
-                teeDate(1,"WARNING - output file exists for {}/{}"\
-                          .format(dfile.path,dfile.fn))
-                return 1
+        elif "exists" in result.stderr :
+            teeDate(1,"WARNING - output file exists for {}/{}"\
+                    .format(dfile.path,dfile.fn))
+            return 1
+        else :
             teeDate(0,"ERROR - copy failed for try {} for {}".\
                   format(itry,dfile.url))
             print("message: "+message)
@@ -865,12 +861,7 @@ def writeLog(dfile):
     runTime = int( time.time() )
     recoverDelay = 7200
 
-    # gfal2 and samweb functions are methods of global objects
-
-    ctx = gfal2.creat_context()
-    token = getToken()
-    ctx.set_opt_string("BEARER", "TOKEN", token)
-
+    # samweb functions are methods of global objects
     samweb = samweb_client.SAMWebClient()
 
     # default, normally take app name and verison from MOO_CONFIG

Util/wrapper.sh

@@ -20,6 +20,14 @@ printenv | grep LC_
 printenv LANG
 unset LC_CTYPE
 
+tee_date Check BEARER_TOKEN
+if [ "$BEARER_TOKEN" ]; then
+    echo "found BEARER_TOKEN set, unsetting it"
+    unset BEARER_TOKEN
+else
+    echo "found BEARER_TOKEN is not set"
+fi
+
 # always need to find setup
 source /cvmfs/mu2e.opensciencegrid.org/setupmu2e-art.sh
 
@@ -42,9 +50,13 @@ save_environment wrapper_start
 #
 if [[ "$MOO_SOURCE" =~ "/" ]]; then
     tee_date fetching Offline Ops from $MOO_SOURCE
-    setup ifdhc
     LFN=$(basename $MOO_SOURCE)
-    ifdh cp --cp_maxretries=1 $MOO_SOURCE $LFN
+    (
+        if ! command -v ifdh >& /dev/null ; then
+            muse setup ops
+        fi
+        ifdh cp --cp_maxretries=1 $MOO_SOURCE $LFN
+    )
     tee_date untar OfflineOps source $LFN
     tar -xf $LFN
     rm -f $LFN
@@ -71,6 +83,21 @@ if [ "$PRODUCTS" ]; then
     #unset PRODUCTS
 fi
 
+#
+# undo whatever spack exists
+#
+if [ "$SPACK_ROOT" ]; then
+    spack unload -a
+    unset SPACK_ROOT
+    unset SPACK_ENV
+    unset SPACK_ENV_VIEW
+fi
+
+
+# allow mu2einit to run again
+unset MU2E
+
+
 tee_date "mv CONDOR_DIR_INPUT to cwd"
 if [ "$CONDOR_DIR_INPUT" ]; then
     /bin/ls -al $CONDOR_DIR_INPUT

bin/pushOutput

@@ -20,7 +20,7 @@ source /cvmfs/mu2e.opensciencegrid.org/setupmu2e-art.sh
 setup sam_web_client
 #setup root v6_26_06 -q e20:p3913:prof
 # for gfal2
-export PYTHONPATH=$PYTHONPATH:/usr/lib64/python3.6/site-packages:/usr/lib/python3.6/site-packages
+#export PYTHONPATH=$PYTHONPATH:/usr/lib64/python3.6/site-packages:/usr/lib/python3.6/site-packages
 export SAMWEB_HTTP_HANDLER=urllib
 #which python
 unset X509_USER_PROXY