Merge pull request #65 from cernbox/cboxsls_merge

merged changes from cboxsls

bin/smash

@@ -242,7 +242,8 @@ def main():
          logger.critical("Wrong testset specification: %d index out of range for %s",args.testset,barename(t))
          sys.exit(1)
 
-      for j in range(1,args.loop+1):
+      j = 1
+      while j <= args.loop or args.loop == 0:
          log_quiet ("Running iteration %d" % j)
 
          if not smashbox.no_engine.testsets:
@@ -254,6 +255,7 @@ def main():
             else:
                i = args.testset # this may be None if no testset indicated
                run_test(t,j,i)
+         j=j+1
 
    if args.dry_run:
       log_quiet('*** DRY RUN ***')

corruption_test/run_nplusone_loop

@@ -10,9 +10,10 @@ conf_file = os.environ.get('SMASHBOX_CONF',os.path.join(thisdir,"smashbox.conf")
 
 dirs = {'thisdir':thisdir,'smashdir':smashdir, 'conf_file':conf_file}
 
-os.environ['OWNCLOUD_MAX_PARALLEL'] = '10'
+os.environ['OWNCLOUD_MAX_PARALLEL'] = '3'
 
-os.environ['OWNCLOUD_USE_LEGACY_JOBS'] = '1'
+# this disables the checksumming!
+#os.environ['OWNCLOUD_USE_LEGACY_JOBS'] = '1'
 
 import datetime
 now = datetime.datetime.now().strftime('%Y-%m-%d-%H%M%S')
@@ -36,6 +37,9 @@ i = 1
 dirs['options']="-o nplusone_nfiles=20 -o nplusone_filesize='(5.0,1.37)'" # --keep-state"
 #dirs['options']="-o nplusone_nfiles=20 -o nplusone_filesize=30000000"
 
+# infinite loop and ignore any casual errors (stop on fatal errors only)
+dirs['options'] += " --loop=0 --keep-going "
+
 cmd = '%(smashdir)s/bin/smash -c %(conf_file)s %(options)s %(smashdir)s/lib/test_nplusone.py' % dirs
 
 #print cmd

lib/test_nplusone.py

@@ -8,10 +8,14 @@
 
 from smashbox.utilities import *
 from smashbox.utilities.hash_files import *
+from smashbox.utilities.monitoring import push_to_monitoring
 
 nfiles = int(config.get('nplusone_nfiles',10))
 filesize = config.get('nplusone_filesize',1000)
 
+# optional fs check before files are uploaded by worker0
+fscheck = config.get('nplusone_fscheck',False)
+
 if type(filesize) is type(''):
     filesize = eval(filesize)
 
@@ -54,8 +58,29 @@ def worker0(step):
 
     step(2,'Add %s files and check if we still have k1+nfiles after resync'%nfiles)
 
+    total_size=0
+    sizes=[]
+
+    # compute the file sizes in the set
     for i in range(nfiles):
-        create_hashfile(d,size=filesize)
+        size=size2nbytes(filesize)
+        sizes.append(size)
+        total_size+=size
+
+    time0=time.time()
+
+    logger.log(35,"Timestamp %f Files %d TotalSize %d",time.time(),nfiles,total_size)
+
+    # create the test files
+    for size in sizes:
+        create_hashfile(d,size=size)
+
+    if fscheck:
+        # drop the caches (must be running as root on Linux)
+        runcmd('echo 3 > /proc/sys/vm/drop_caches')
+
+        ncorrupt = analyse_hashfiles(d)[2]
+        fatal_check(ncorrupt==0, 'Corrupted files ON THE FILESYSTEM (%s) found'%ncorrupt)
 
     run_ocsync(d)
 
@@ -68,6 +93,17 @@ def worker0(step):
     fatal_check(ncorrupt==0, 'Corrupted files (%s) found'%ncorrupt)
 
     logger.info('SUCCESS: %d files found',k1)
+
+    step(4,"Final report")
+
+    time1 = time.time()
+    push_to_monitoring("cernbox.cboxsls.nplusone.nfiles",nfiles)
+    push_to_monitoring("cernbox.cboxsls.nplusone.total_size",total_size)
+    push_to_monitoring("cernbox.cboxsls.nplusone.elapsed",time1-time0)
+    push_to_monitoring("cernbox.cboxsls.nplusone.total_size",total_size)
+    push_to_monitoring("cernbox.cboxsls.nplusone.transfer_rate",total_size/(time1-time0))
+    push_to_monitoring("cernbox.cboxsls.nplusone.worker0.synced_files",k1-k0)
+
 
 @add_worker
 def worker1(step):
@@ -83,9 +119,13 @@ def worker1(step):
     ncorrupt = analyse_hashfiles(d)[2]
     k1 = count_files(d)
 
+    push_to_monitoring("cernbox.cboxsls.nplusone.worker1.synced_files",k1-k0)
+    push_to_monitoring("cernbox.cboxsls.nplusone.worker1.cor",ncorrupt)
+
     error_check(k1-k0==nfiles,'Expecting to have %d files more: see k1=%d k0=%d'%(nfiles,k1,k0))
 
-    fatal_check(ncorrupt==0, 'Corrupted files (%s) found'%ncorrupt)
+    fatal_check(ncorrupt==0, 'Corrupted files (%d) found'%ncorrupt) #Massimo 12-APR
+
 
 
 

python/smashbox/utilities/monitoring.py

@@ -0,0 +1,16 @@
+from smashbox.utilities import *
+
+# simple monitoring to grafana (disabled if not set in config)
+
+def push_to_monitoring(metric,value,timestamp=None):
+
+    monitoring_host=config.get('monitoring_host',None)
+    monitoring_port=config.get('monitoring_port',2003)
+
+    if not monitoring_host:
+        return
+
+    if not timestamp:
+        timestamp = time.time()
+
+    os.system("echo '%s %s %s' | nc %s %s"%(metric,value,timestamp,monitoring_host,monitoring_port))