Merge branch 'main' of github.com:neu-se/CONFETTI-artifact into main

.gitignore

@@ -8,3 +8,8 @@ README.pdf
 *_over_time.pdf
 tool_output
 vendor
+debug
+debug-preinst
+prebake*
+downloads
+local_eval_output

experiments/ant-confetti-no-global-hint/run.sh → experiments/ant-knarr-z3-no-global-hint/run.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-RES=$RESULTS_DIR/ant-confetti-no-global-hint
+RES=$RESULTS_DIR/ant-knarr-z3-no-global-hint
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
 export NO_GLOBAL_DICT=TRUE

experiments/ant-confetti/run.sh → experiments/ant-knarr-z3/run.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-RES=$RESULTS_DIR/ant-confetti
+RES=$RESULTS_DIR/ant-knarr-z3
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
 export EXP_CP="`$ANT_DIR/classpath.sh`"

experiments/bcelgen-confetti-no-global-hint/run.sh → experiments/bcelgen-knarr-z3-no-global-hint/run.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-RES=$RESULTS_DIR/bcelgen-confetti-no-global-hint
+RES=$RESULTS_DIR/bcelgen-knarr-z3-no-global-hint
 export NO_GLOBAL_DICT=TRUE
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 

experiments/bcelgen-confetti/run.sh → experiments/bcelgen-knarr-z3/run.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-RES=$RESULTS_DIR/bcelgen-confetti
+RES=$RESULTS_DIR/bcelgen-knarr-z3
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
 export EXP_CP="`$BCEL_DIR/classpath.sh`"

experiments/closure-confetti-no-global-hint/run.sh → experiments/closure-knarr-z3-no-global-hint/run.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-RES=$RESULTS_DIR/closure-confetti-no-global-hint
+RES=$RESULTS_DIR/closure-knarr-z3-no-global-hint
 export NO_GLOBAL_DICT=TRUE
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 

experiments/closure-confetti/run.sh → experiments/closure-knarr-z3/run.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-RES=$RESULTS_DIR/closure-confetti
+RES=$RESULTS_DIR/closure-knarr-z3
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
 export EXP_CP="`$CLOSURE_DIR/classpath.sh`"

experiments/fix.php

@@ -0,0 +1,7 @@
+<?php
+foreach(glob("*-confetti*") as $confettiDir ){
+    `sed -i 's/-confetti/-knarr-z3/' $confettiDir/run.sh`;
+    $newDir = str_replace("-confetti","-knarr-z3", $confettiDir);
+    `mv $confettiDir $newDir`;
+}
+?>

experiments/maven-confetti-no-global-hint/run.sh → experiments/maven-knarr-z3-no-global-hint/run.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-RES=$RESULTS_DIR/maven-confetti-no-global-hint
+RES=$RESULTS_DIR/maven-knarr-z3-no-global-hint
 export NO_GLOBAL_DICT=TRUE
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"

experiments/maven-confetti/run.sh → experiments/maven-knarr-z3/run.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-RES=$RESULTS_DIR/maven-confetti
+RES=$RESULTS_DIR/maven-knarr-z3
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
 export EXP_CP="`$MAVEN_DIR/classpath.sh`"

experiments/rhino-confetti-no-global-hint/run.sh → experiments/rhino-knarr-z3-no-global-hint/run.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-RES=$RESULTS_DIR/rhino-confetti-no-global-hint
+RES=$RESULTS_DIR/rhino-knarr-z3-no-global-hint
 export NO_GLOBAL_DICT=TRUE
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"

experiments/rhino-confetti/run.sh → experiments/rhino-knarr-z3/run.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-RES=$RESULTS_DIR/rhino-confetti
+RES=$RESULTS_DIR/rhino-knarr-z3
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
 export EXP_CP="`$RHINO_DIR/classpath.sh`"

experiments/run-jqf.sh

@@ -19,12 +19,14 @@ source $ROOT/experiments/jvm.sh
 
 # How long to run the experiment (seconds)
 #DURATION=$((60*2 ))
-DURATION=$((60*60*24 ))
+if [ -z "$DURATION" ]; then
+    DURATION=$((60*60*24))
+fi
 
 JQF_SENSITIVITY=3
 #JQF_DEBUG="-Xdebug -Xrunjdwp:server=y,transport=dt_socket,address=5555,suspend=n"
-JQF_OPTS="$JQF_JVM_OPTS -Dtime=$(($DURATION + 60))" #-Djanala.verbose=true"
-JQF_CMD="JVM_OPTS=\"-Djqf.ei.UNIQUE_SENSITIVITY=$JQF_SENSITIVITY $JQF_OPTS $JQF_DEBUG\" timeout $(($DURATION + 60)) $JQF_VANILLA_DIR/bin/jqf-ei -c $JQF_VANILLA_DIR/examples/target/classes:$JQF_VANILLA_DIR/examples/target/test-classes:`$JQF_VANILLA_DIR/scripts/classpath.sh`:$EXP_CP $EXP_CMD"
+JQF_OPTS="$JQF_JVM_OPTS -Dtime=$(($DURATION))" #-Djanala.verbose=true"
+JQF_CMD="JVM_OPTS=\"-Djqf.ei.UNIQUE_SENSITIVITY=$JQF_SENSITIVITY $JQF_OPTS $JQF_DEBUG\" timeout $(($DURATION)) $JQF_VANILLA_DIR/bin/jqf-ei -c $JQF_VANILLA_DIR/examples/target/classes:$JQF_VANILLA_DIR/examples/target/test-classes:`$JQF_VANILLA_DIR/scripts/classpath.sh`:$EXP_CP $EXP_CMD"
 
 
 # Launch program driver

experiments/run-knarr.sh

@@ -31,7 +31,9 @@ fi
 
 # How long to run the experiment (seconds)
 #DURATION=$((60*60*24))
-DURATION=$((60*60*24))
+if [ -z "$DURATION" ]; then
+    DURATION=$((60*60*24))
+fi
 #DURATION=$((60*2))
 echo $DURATION
 

scripts/collectExtendedHintInfo.php

@@ -1,4 +1,13 @@
 <?php
+if($argc != 3)
+    die("Usage: php collectExtendedHintInfo.php fuzzOutputDir forensicsOutputDir");
+
+$resultDir = $argv[1];
+$outputDir = $argv[2];
+if(!is_dir($outputDir)){
+    mkdir($outputDir);
+}
+
     $configVanilla = "jqf";
     $configKnarr = "knarr-z3";
     $benchmarks = [
@@ -15,10 +24,9 @@
             "org/mozilla/javascript/CodeGenerator*"]]
     ];
 
-    $resultDir = getenv("FUZZ_OUTPUT");
     $jqfDir = getenv("JQF_DIR");
     $jqfVanillaDir = getenv("JQF_VANILLA_DIR");
-    $cmds = "";
+    // $cmds = "";
     foreach($benchmarks as $bm => $config){
         //print "=======$bm=======\n";
         $knarrDirs = glob($resultDir."/*".$bm."-".$configKnarr."-*.tgz");
@@ -28,11 +36,11 @@
             $exp = basename($experiment);
             $thisExp = [];
             $batches = [];
-			$outputFile = $experiment.".forensics-1k.csv";
-		//	$cmds.= "EXP_NAME=$exp APP_NAME=$bm TRIALS=100 bash $jqfDir/scripts/evaluate_extended_dict.sh ".$config["class"]. " ". $config['method']. " " .$outputFile." $experiment/corpus/ \n";
-			$cmds.= "EXP_NAME=$exp APP_NAME=$bm TRIALS=1000 bash /experiment/confetti/evaluate_extended_dict_wrapper.sh ".$config["class"]. " ". $config['method']. " " .$outputFile." $experiment \n";
+			$outputFile = $outputDir."/".str_replace($resultDir, "", $experiment).".forensics-1k.csv";
+			$cmd = "EXP_NAME=$exp APP_NAME=$bm TRIALS=1000 bash /home/icse22ae/confetti-artifact/scripts/evaluate_extended_dict_wrapper.sh ".$config["class"]. " ". $config['method']. " " .$outputFile." $experiment \n";
+            echo $cmd;
+            echo `$cmd`;
 
         }
     }
-	print $cmds;
 ?>

scripts/env.sh

@@ -9,7 +9,7 @@ export WGET="$(which wget) -nc"
 export ROOT=/home/icse22ae/confetti-artifact
 
 # Location of fuzz_output
-export FUZZ_OUTPUT=/home/icse22ae/icse_22_fuzz_output
+export FUZZ_OUTPUT=/home/icse22ae/confetti-artifact/icse_22_fuzz_output
 
 
 if [ -z "$ROOT" ]; then

scripts/evaluate_extended_dict_wrapper.sh

@@ -1,20 +1,16 @@
 #!/bin/bash
-cd /home/ubuntu
-if [ ! -d "/home/ubuntu/jqf-artifact" ]
-then
-	cp -r /experiment/confetti/jqf-artifact.tgz .
-	tar xzf jqf-artifact.tgz
-fi
+cd /home/icse22ae/confetti-artifact
 
-source jqf-artifact/scripts/env.sh
+
+source scripts/env.sh
 LOCALOUT=forensics_results.csv
 ZIP=$4
 FILENAME=$(basename $ZIP)
 TEMPDIR="${FILENAME%.*}"
 
 
 tar xzf $ZIP
-bash jqf-artifact/software/jqf//scripts/evaluate_extended_dict.sh $1 $2 $LOCALOUT $TEMPDIR/corpus
+bash software/jqf/scripts/evaluate_extended_dict.sh $1 $2 $LOCALOUT $TEMPDIR/corpus
 
 mv $LOCALOUT $3
 rm -rf $TEMPDIR

scripts/extract-coverage.php

@@ -1,4 +1,9 @@
 <?php
+if($argc != 3)
+	die("Usage: php extract-coverage.php primaryDataInputDir intermediateDataOutputDir");
+
+$inputDir = $argv[1];
+$outputDir = $argv[2];
     $configVanilla = "jqf";
     $configKnarr = "knarr-z3";
     $configKnarrNoDict = "knarr-z3-no-global-hint";
@@ -81,9 +86,8 @@ function createTruncatedFile($from,$to){
     }
 
 	$root = getenv("ROOT");
-    $resultDir = getenv("FUZZ_OUTPUT");
-    $localDir = "$root/generated/coverage-tmp";
-    $outputDir = "$root/generated/coverage";
+    $resultDir = $inputDir;
+    $localDir = "$root/coverage-tmp";
     if(!file_exists($outputDir)){
 	mkdir($outputDir);
     }

scripts/extract-last-line-of-fuzz-stats.php

@@ -29,9 +29,12 @@ function expandIfNeeded($tgzs, $to){
 	$lastCov = 0;
 
     $resultDir = getenv("FUZZ_OUTPUT");
-    if(!file_exists($outputDir)){
-		mkdir($outputDir);
+    if($resultDir == ""){
+        die("Please be sure to source scripts/env.sh in this artifact before running this command\n");
     }
+    // if(!file_exists($outputDir)){
+		// mkdir($outputDir);
+    // }
 	$res = "bm,experiment,unix_time, cycles_done, cur_path, paths_total, pending_total, pending_favs, map_size, unique_crashes, unique_hangs, max_depth, execs_per_sec, total_inputs, mutated_bytes, valid_inputs, invalid_inputs, all_cov, z3, inputsSavedBy_StrHint, inputsCreatedBy_StrHint, inputsSavedBy_MultipleStrHint, inputsCreatedBy_MultipleStrHint, inputsSavedBy_CharHint, inputsCreatedBy_CharHint, inputsSavedBy_Z3, inputsCreatedBy_Z3, inputsSavedBy_Random, inputsCreatedBy_Random, inputsSavedWith_Hints, inputsSavedWith_Z3Origin, inputsSavedWithoutHintsOrZ3,countOfSavedInputsWithExtendedDictionaryHints,countOfCreatedInputsWithExtendedDictionaryHints,extendedDictionarySize\n";
     foreach($benchmarks as $bm => $config){
         $kd = glob($resultDir."/*".$bm."-".$configKnarr."-*.tgz");
@@ -46,5 +49,8 @@ function expandIfNeeded($tgzs, $to){
 			$res .= "$bm,$n,$lastLine\n";
 		}
     }
-	file_put_contents("fuzz_stats.csv",$res);
+    if(!is_dir("/home/icse22ae/confetti-artifact/generated")){
+        mkdir("/home/icse22ae/confetti-artifact/generated");
+    }
+	file_put_contents("/home/icse22ae/confetti-artifact/generated/fuzz_stats.csv",$res);
 ?>

scripts/graphCoverage-fig2.R

@@ -9,9 +9,15 @@ library(viridis)
 library(gridExtra)
 library(forcats)
 
+args = commandArgs(trailingOnly=TRUE)
+if (length(args)!=1) {
+  stop("Error. Exactly one argument required: the input directory with coverage data")
+}
+coverageDir <- args[1]
+
 bms <- c('ant', 'bcelgen', 'maven', 'closure', 'rhino')
 targets <- c('knarr-z3-no-global-hint','knarr-z3', 'jqf')
-files<-basename(Sys.glob("/home/icse22ae/confetti-artifact/generated/coverage/*-coverage.csv"))
+files<-basename(Sys.glob(paste0(coverageDir,"/*-coverage.csv")))
 getBm <- function(filename){
   for(bm in bms){
     if(grepl(bm, filename))
@@ -39,7 +45,8 @@ read_coverage <- function(filename){
 }
 
 
-cov <- ldply(Sys.glob("/home/icse22ae/confetti-artifact/generated/coverage/*-coverage.csv"), read_coverage)
+cov <- ldply(Sys.glob(paste0(coverageDir,"/*-coverage.csv")), read_coverage)
+# cov <- ldply(Sys.glob("/home/icse22ae/confetti-artifact/generated/coverage/*-coverage.csv"), read_coverage)
 cov$experiment <- as.factor(cov$experiment)
 cov$bm <- as.factor(cov$bm)
 cov$config <- as.factor(cov$config)

scripts/runOneExperiment.php

@@ -32,7 +32,7 @@ function execAndLog($note,$cmd){
 	die('Usage: php runOneExperiment.php <evalScript> <experimentName>
 	
 	Where evalScript is one of: (benchmark)-(fuzzer), for valid combinations of:
-		benchmark (ant, bcelgen, closure, maven, rhino), fuzzer (jqf,confetti,confetti-no-global-hint)
+		benchmark (ant, bcelgen, closure, maven, rhino), fuzzer (jqf,knarr-z3,knarr-z3-no-global-hint)
 
 	experimentName is a string that you can choose, and will be included in the output file names.
 

scripts/runOneSmokeTest.php

@@ -0,0 +1,71 @@
+<?php
+
+$BASE_DIR = "/home/icse22ae/confetti-artifact";
+
+function execAndLog($note,$cmd){
+    print "------------------------------------------------------------------------\n";
+    print "$note\n";
+    print "------------------------------------------------------------------------\n";
+
+    print "Running: $cmd\n";
+    $err = 0;
+    $time = time();
+    passthru($cmd." 2>&1",$err);
+    $duration = time()- $time;
+    print "Return code: $err. Duration: $duration seconds \n";
+    return $err;
+}
+
+if($argc != 2){
+	die('Usage: php runOneSmokeTest.php <evalScript> 
+	
+	Where evalScript is one of: (benchmark)-(fuzzer), for valid combinations of:
+		benchmark (ant, bcelgen, closure, maven, rhino), fuzzer (jqf,knarr-z3,knarr-z3-no-global-hint)
+');
+}
+$script = $argv[1];
+$expName = "$script-1";
+
+$resultsDir = "$BASE_DIR/results/$script";
+
+
+$startTime = time();
+print "------------------------------------------------------------------------\n";
+print "confetti eval runner on: $script\n";
+print "Start time: ".date("r")."\n";
+print "------------------------------------------------------------------------\n";
+
+if(file_exists("$BASE_DIR/experiment-constraints")){
+	execAndLog("Cleaning up old constraints" ,"rm -rf $BASE_DIR/experiment-constraints");
+}
+
+chdir("$BASE_DIR");
+execAndLog("Confirming jqf-artifact revision", "git -C $BASE_DIR/ rev-parse HEAD");
+execAndLog("Confirming jqf-confetti revision", "git -C $BASE_DIR/software/jqf/ rev-parse HEAD");
+execAndLog("Confirming jqf-vanilla revision", "git -C $BASE_DIR/software/jqf-vanilla/ rev-parse HEAD");
+
+
+$screenName = "exp-$expName-$startTime";
+$extraArgs = "DURATION=600";
+execAndLog("Running the experiment in screen. If you want to 
+snoop on the run, in another shell run `screen -r $screenName` (do not terminate it though!)",
+	"$extraArgs screen -S '$screenName' -d -m bash -c \"$BASE_DIR/experiments/$script/run.sh\"");
+print "Waiting for screen to terminate\n";
+$running = true;
+while($running){
+	sleep(30);
+	$status = trim(exec("screen -list | grep $screenName"));
+	$running = $status != "";
+}
+
+execAndLog("Moving results to follow same format as full experimental runs...", "mv results/$script results/$expName");
+$archiveName = "$expName.tgz";
+execAndLog("Creating result archive","tar czf local_eval_output/$expName.tgz -C results/ $expName");
+execAndLog("Removing the files that are now archived", "rm -rf results/*");
+
+print "------------------------------------------------------------------------\n";
+print "confetti eval runner $expName job done\n";
+print "End time: ".date("r"). " (".(time()-$startTime)." seconds)\n";
+print "------------------------------------------------------------------------\n";
+
+?>

scripts/runSmokeTest.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+export FUZZ_OUTPUT=/home/icse22ae/confetti-artifact/local_eval_output
+
+php scripts/runOneSmokeTest.php ant-jqf
+php scripts/runOneSmokeTest.php bcelgen-jqf
+php scripts/runOneSmokeTest.php closure-jqf
+php scripts/runOneSmokeTest.php maven-jqf
+php scripts/runOneSmokeTest.php rhino-jqf
+
+php scripts/runOneSmokeTest.php ant-knarr-z3
+php scripts/runOneSmokeTest.php bcelgen-knarr-z3
+php scripts/runOneSmokeTest.php closure-knarr-z3
+php scripts/runOneSmokeTest.php maven-knarr-z3
+php scripts/runOneSmokeTest.php rhino-knarr-z3
+
+php scripts/runOneSmokeTest.php ant-knarr-z3-no-global-hint
+php scripts/runOneSmokeTest.php bcelgen-knarr-z3-no-global-hint
+php scripts/runOneSmokeTest.php closure-knarr-z3-no-global-hint
+php scripts/runOneSmokeTest.php maven-knarr-z3-no-global-hint
+php scripts/runOneSmokeTest.php rhino-knarr-z3-no-global-hint