Merge pull request #2 from rlcee/update_230330

Update 230330

Campaigns/CRVWB/reco.sh

@@ -17,15 +17,14 @@ source /cvmfs/mu2e.opensciencegrid.org/bin/OfflineOps/functions.sh
 tee_date "Starting CRVWB reco.sh"
 tee_date "args are: $@"
 
-source /cvmfs/fermilab.opensciencegrid.org/products/common/etc/setups
 setup mu2e
 setup CRVTeststand $MOO_CRVTESTSTAND
+muse setup Offline v10_20_00
+setup -B mu2etools
 setup -B mu2efiletools
 setup -B sam_web_client
 setup -B ifdhc
 
-printenv | grep SAM
-
 # modify CRV exe control to use local directory
 cat $CRVTESTSTAND_FQ_DIR/config.txt | \
 awk '{
@@ -61,8 +60,6 @@ do
     ifdh cp $MOO_INPUT $BNAME
     RCT=$((RCT+$?))
 
-    [ $RCT -ne 0 ] && break
-
     DES=$(echo $BNAME | awk -F. '{print $3}' )
     CFG=$(echo $BNAME | awk -F. '{print $4}' )
     SEQ=$(echo $BNAME | awk -F. '{print $5}' )
@@ -77,6 +74,8 @@ do
     RPFN=etc.mu2e.${DES}_reco.${MOO_CONFIG}.${SEQ}.pdf
     RTFN=etc.mu2e.${DES}_reco.${MOO_CONFIG}.${SEQ}.txt
 
+    [ $RCT -ne 0 ] && break
+
     if [ "$MOO_FAKE" ]; then
         date > ntd.mu2e.${DES}.${CFG}.${SEQ}.root
         date > cal.mu2e.${DES}.${CFG}.${SEQ}.pdf
@@ -85,13 +84,20 @@ do
         date > rec.mu2e.${DES}.${CFG}.${SEQ}.pdf
         date > rec.mu2e.${DES}.${CFG}.${SEQ}.txt
     else
+
+        tee_date "Running parseCrv $SEQ"
         parserCrv $SEQ
         RCT=$((RCT+$?))
 
+        tee_date "Running calibCrv $SEQ"
         calibCrv $SEQ
         RCT=$((RCT+$?))
 
-        recoCrv $SEQ -p
+        FLAG=""
+        [[ "$BNAME" =~ "crvled" ]] && FLAG="-p"
+
+        tee_date "Running recoCrv $SEQ $FLAG"
+        recoCrv $SEQ $FLAG
         RCT=$((RCT+$?))
     fi
 
@@ -120,18 +126,17 @@ do
 done
 
 tee_date "Done file loop, RCT=$RCT"
+tee_date "Processed files: $MOO_INPUT_LIST"
 
 if [ $RCT -ne 0 ]; then
-    tee_date "removing data files from output list"
+    tee_date "processing failed, removing data files from output list"
     rm output.txt
 fi
 echo "$LOCTXT $LGFN none" >> output.txt
 
+tee_date "Final ls"
+ls -l
 
-tee_date "cat output.txt"
-cat ouput.txt
-
-control_summary final
 
 if [ "$MOO_LOCAL" ]; then
     RCP=0

Campaigns/Demo/reco.sh

@@ -102,6 +102,9 @@ if [ $RCT -ne 0 ]; then
 fi
 echo "$LOCTXT $LGFN none" >> output.txt
 
+tee_date "Final ls"
+ls -l
+
 [[ "$MOO_FAIL" && $((RANDOM%400)) -lt $MOO_FAIL ]] && exit 1
 
 control_summary final

Util/functions.sh

@@ -223,6 +223,22 @@ create_config() {
 # MOO_INPUT and add the file to a list at MOO_INPUTS
 #
 get_next_SAM_file() {
+
+    if [ -n "$MOO_LOCAL_INPUT" ]; then
+        export MOO_INPUT=""
+
+        for FN in $(echo $MOO_LOCAL_INPUT | tr "," " " )
+        do
+            if [[ ! "$MOO_INPUT_LIST" =~ "$FN" ]]; then
+                export MOO_INPUT="$FN"
+                export MOO_INPUT_LIST=${MOO_INPUT_LIST:+$MOO_INPUT_LIST,}$MOO_INPUT
+                break
+            fi
+        done
+
+        return 0
+    fi
+
     if ! command -v samweb > /dev/null 2>&1 ; then
         echo "ERROR - get_next_SAM_file called without samweb available"
         return 1
@@ -234,11 +250,14 @@ get_next_SAM_file() {
     local TEMP=$(samweb get-next-file $SAM_PROJECT $SAM_CONSUMER_ID 2>&1 )
     local TT=$?
 
-    if [ $TT -eq 0 ]; then
+    if [[ $TT -eq 0 ]]; then
         export MOO_INPUT="$TEMP"
-        export MOO_INPUT_LIST=${MOO_INPUT_LIST:+$MOO_INPUT_LIST,}$MOO_INPUT
+        if [ -n "$TEMP" ]; then
+            export MOO_INPUT_LIST=${MOO_INPUT_LIST:+$MOO_INPUT_LIST,}$MOO_INPUT
+        fi
     else
-        [ $MOO_VERBOSE -ge 1 ] && echo $TEMP
+        export MOO_INPUT=""
+        [ $MOO_VERBOSE -ge 1 ] && echo "get-next-file output: $TEMP"
     fi
 
     return $TT
@@ -252,6 +271,12 @@ get_next_SAM_file() {
 # if no argument, assume consumed
 #
 release_SAM_file() {
+
+    if [ -n "$MOO_LOCAL_INPUT" ]; then
+        unset MOO_INPUT
+        return 0
+    fi
+
     if ! command -v samweb > /dev/null 2>&1 ; then
         echo "ERROR - release_SAM_file called without samweb available"
         return 1
@@ -302,6 +327,8 @@ watchdog() {
     [ -z "$MU2E" ] && source /cvmfs/mu2e.opensciencegrid.org/setupmu2e-art.sh
     [ -z "$SETUP_IFDH" ] && setup ifdhc
 
+    ifdh mkdir_p $DD
+
     local T0=$( date +%s )
     local DT=0
     local FN="watchdog"

Util/pushOutput.py

@@ -136,7 +136,7 @@ def fillDataFile(line):
         print("processing: " +  line, end="")
 
     dfile = DataFile()
- 
+
     aa = line.split()
     aa = [ x.strip() for x in aa ]
     if len(aa) <2 :
@@ -311,7 +311,7 @@ def getSamMetadata(dfile):
         dfile.samtime = -1
         return 2
 
-        
+
     dd = json.loads(rec)
     dfile.locations = dd.pop("locations",[])
     dfile.samInfo = dd
@@ -530,7 +530,7 @@ def createMetadata(dfile):
         return 2
 
     ext = dfile.fn.split()[-1]
-    
+
     cmd = ""
     if dfile.parents == "" or dfile.parents == "none" :
         cmd = cmd+"printJson --no-parents "+dfile.localfs+" 2> err.txt"
@@ -540,7 +540,7 @@ def createMetadata(dfile):
 
     try:
         jsontext = subprocess.check_output(cmd,shell=True,\
-            stderr=subprocess.STDOUT,timeout=300,text=True)
+            stderr=subprocess.STDOUT,timeout=600,text=True)
     except Exception as e:
         teeDate(0,"ERROR - printJson failed for "+dfile.fn
                 +"\nmessage: "+str(e))
@@ -619,8 +619,8 @@ def declareSam(dfile):
 
 
 #
-# returns success, all output was either written by this process or 
-# there is no output, or the previous output was old, ready to recover (0), 
+# returns success, all output was either written by this process or
+# there is no output, or the previous output was old, ready to recover (0),
 # there is a recent output from a previous run (1), or a failure (2)
 #
 
@@ -661,7 +661,7 @@ def checkTimes(dfile):
 
         # if rc==1 then, no output file, continue to check SAM
 
-    # if flow comes here, the file was written by this process, 
+    # if flow comes here, the file was written by this process,
     # was not there, or was old, i.e. OK to recover
 
     if dfile.donesam :
@@ -807,7 +807,7 @@ def writeLog(dfile):
     verbose = 1
     retries = [0,10,30]
     runTime = int( time.time() )
-    recoverDelay = 30
+    recoverDelay = 7200
 
     # gfal2 and samweb functions are methods of global objects
     ctx = gfal2.creat_context()
@@ -820,7 +820,7 @@ def writeLog(dfile):
     appVersion=None
 
     # intentionaly fail at a rate given by MOO_FAIL/100
-    failrate = 0.0
+    failRate = 0.0
     if 'MOO_FAIL' in os.environ :
         failRate = float(os.environ["MOO_FAIL"])/100.0
 
@@ -830,10 +830,10 @@ def writeLog(dfile):
 
     parser.add_argument("filelist", metavar="file_of_files",
                         type=str, help="text file with list of files to transfer")
-    parser.add_argument("-v","--verbose", action="store_const", 
+    parser.add_argument("-v","--verbose", action="store_const",
                         dest="verbose", default=1,const=None,
                         help="int, 0,1,2 for verbosity (default=1)")
-    parser.add_argument("-a","--app_default", action="store_const", 
+    parser.add_argument("-a","--app_default", action="store_const",
                         dest="app_default", default=True, const=None,
                         help="True/False take app name, version from $MOO_CONFIG")
 
@@ -911,7 +911,7 @@ def writeLog(dfile):
             rcCheck = checkTimes(dfile)
             if rcCheck != 0 :
                 break
-        
+
         rcRecover = 0
         if rcCheck == 1 :
             # recent files from another job exist
@@ -949,4 +949,3 @@ def writeLog(dfile):
             writeLog(dfile)
 
     sys.exit(rcJob)
-

Util/upsTarball.sh

@@ -2,6 +2,7 @@
 #
 # script to make a ups tarball
 # args: $1=version
+# the content of the tarball will be whatever is in this script's directory
 # tarball will be created in the cwd
 #
 VERSION="$1"
@@ -11,7 +12,9 @@ if [ ! "$VERSION" ]; then
 fi
 
 OWD=$PWD
-SDIR=$(dirname $(readlink -f $BASH_SOURCE)  | sed 's|/build||' )
+
+echo ${BASH_SOURCE}
+SDIR=$(readlink -f $(dirname ${BASH_SOURCE} )/.. )
 
 PDIR=$(mktemp -d)
 cd $PDIR
@@ -22,11 +25,11 @@ mkdir -p $VERSION
 cd $VERSION
 
 rsync --exclude "*~" --exclude "*__*"  \
-    -r $SDIR/bin $SDIR/Campaigns $SIDR/Util  .
+    -r $SDIR/bin $SDIR/Campaigns $SDIR/Util  .
 mkdir -p ups
 cd ups
 
-cat > dhtools.table <<EOL
+cat > OfflineOps.table <<EOL
 File    = table
 Product = OfflineOps
 
@@ -36,6 +39,8 @@ Product = OfflineOps
 Group:
 
 Flavor     = ANY
+  Action = flavorSetup
+
 
 Common:
   Action = setup
@@ -44,44 +49,38 @@ Common:
     prodDir()
     setupEnv()
     envSet(\${UPS_PROD_NAME_UC}_VERSION, $VERSION)
-    sourceRequired(\${UPS_PROD_DIR}/bin/dh_functions.sh,UPS_ENV)
     pathPrepend(PATH,\${UPS_PROD_DIR}/bin)
-    pathPrepend(PYTHONPATH,\${UPS_PROD_DIR}/python)
+    #pathPrepend(PYTHONPATH,\${UPS_PROD_DIR}/python)
     exeActionRequired(flavorSetup)
 
 End:
 # End Group definition
 #*************************************************
 EOL
 
-# up to dhtools dir
+# up to OfflineOps dir
 cd ../..
 
 mkdir -p ${VERSION}.version
 cd ${VERSION}.version
 
 cat > NULL <<EOL
 FILE = version
-PRODUCT = dhtools
+PRODUCT = OfflineOps
 VERSION = $VERSION
 
 FLAVOR = NULL
 QUALIFIERS = ""
-  PROD_DIR = dhtools/$VERSION
+  PROD_DIR = OfflineOps/$VERSION
   UPS_DIR = ups
-  TABLE_FILE = dhtools.table
+  TABLE_FILE = OfflineOps.table
 
-FLAVOR = NULL
-QUALIFIERS = "grid"
-  PROD_DIR = dhtools/$VERSION
-  UPS_DIR = ups
-  TABLE_FILE = dhtools.table
 EOL
 
 cd ../..
 
-tar -cjf $OWD/dhtools-${VERSION}.bz2 dhtools/${VERSION} dhtools/${VERSION}.version
-rm -rf $PDIR/dhtools
+tar -cjf $OWD/OfflineOps-${VERSION}.bz2 OfflineOps/${VERSION} OfflineOps/${VERSION}.version
+rm -rf $PDIR/OfflineOps
 rmdir $PDIR
 
 cd $OWD