Using host.json as project JSON to specificy docker container and whe…

…ther docker, local, or remote

scripts/bug_mining.py

@@ -117,8 +117,6 @@ def progress(msg):
     # global curtail
     curtail = int(sys.argv[4])
 
-panda_os_string = project.get('panda_os_string', 'linux-32-debian:3.2.0-4-686-pae')
-
 lavadir = dirname(dirname(abspath(sys.argv[0])))
 
 progress("Entering {}".format(project['output_dir']))
@@ -190,7 +188,7 @@ def progress(msg):
                     'g_debugpath': installdir,
                     'h_debugpath': installdir
                 })
-
+# pri_taint is almost same as Zhenghao's hypercall
 # Chaffx64 branch says these are needed?
 # if panda.arch != 'i386':
 #    panda.load_plugin('hypercall')
@@ -209,16 +207,19 @@ def progress(msg):
                       'pos': True,
                       'cache_process_details_on_basic_block': True,
                       'first_instr' : 1,
-                      'use_stdin' : proc_name
+                      'use_stdin' : proc_name,
+                      'verbose' : True
                   })
 else:
     panda.load_plugin("file_taint", 
                   args={
                       'filename' : input_file_guest,
                       'pos': True,
                       'cache_process_details_on_basic_block': True,
-                      'enable_taint_on_open': True
-                  })
+                      'enable_taint_on_open': True,
+                      'verbose' : True  
+                })
+panda.load_plugin("pri_taint")
 
 # Default name is 'recording'
 # https://github.com/panda-re/panda/blob/dev/panda/python/core/pandare/panda.py#L2595

scripts/competition.sh

@@ -4,7 +4,6 @@
 # Json file required params
 #
 # lava:        directory of lava repository
-# pandahost:   what remote host to run panda on
 
 trap '' PIPE
 set -e # Exit on error

scripts/docker-shell.sh

@@ -1,26 +1,28 @@
 #!/bin/bash
 
-# Single argument of project name will get container name
+# Single argument of project name will get Docker name
 # from project config. Then 2nd optional argument is command to run
 # With no arguments, just give us a shell
 
 lava="$(dirname $(dirname $(readlink -f $0)))"
+hostjson="$lava/host.json"
+dockername="$(jq -r '.docker // "lava32"' $hostjson)"
 
-if [ "$#" -eq 0 ]; then
-    container="lava32"
-else
+echo "You are connecting to the Docker container: ${dockername}"
+
+if [ "$#" -ne 0 ]; then
     project_name=$1
     cmd="${@:2}"
-#Container name (lava32 or lava32debug) comes from config
+    # Docker name (lava32 or lava32debug) comes from config
     . `dirname $0`/vars.sh
 
     docker_map_args="-v $tarfiledir:$tarfiledir"
     if [[ "$directory" = "$tarfiledir"* ]]; then true; else
       docker_map_args="$docker_map_args -v $directory:$directory"
     fi
 
-    if ! ( docker images ${container} | grep -q ${container} ); then
-        docker build -t ${container} "$(dirname $(dirname $(readlink -f $0)))/docker/debug"
+    if ! ( docker images ${dockername} | grep -q ${dockername} ); then
+        docker build -t ${dockername} "$(dirname $(dirname $(readlink -f $0)))/docker/debug"
     fi
 
     [ "$extradockerargs" = "null" ] && extradockerargs="";
@@ -57,4 +59,4 @@ docker run --rm -it \
     --cap-add=SYS_PTRACE \
     $docker_map_args \
     $extradockerargs \
-    ${container} sh -c "trap '' PIPE; $cmd"
+    ${dockername} sh -c "trap '' PIPE; $cmd"

scripts/inject.sh

@@ -4,7 +4,7 @@
 # Json file required params
 #
 # lava:        directory of lava repository
-# pandahost:   what remote host to run panda on
+
 
 trap '' PIPE
 set -e # Exit on error

scripts/lava.py

@@ -576,7 +576,7 @@ def __init__(self, project):
             tar_files = subprocess.check_output(['tar', 'tf',
                                                  project['tarfile']],
                                                 stderr=sys.stderr)
-            self.source_root = tar_files.splitlines()[0].split(os.path.sep)[0]
+            self.source_root = tar_files.decode().splitlines()[0].split(os.path.sep)[0]
         self.queries_build = join(self.top_dir, self.source_root)
         self.bugs_top_dir = join(self.top_dir, 'bugs')
 

scripts/lava.sh

@@ -27,7 +27,6 @@
 # name:        a name for this project (used to create directories)
 # inputs:      a list of inputs that will be used to find potential bugs (think coverage)
 # buildhost:   what remote host to build source on
-# pandahost:   what remote host to run panda and postgres on
 # testinghost: what host to test injected bugs on
 # fixupscript: script to run after add_query to fix up src before make
 #
@@ -132,9 +131,9 @@ RESET_DB() {
     lf="$logs/dbwipe.log"
     truncate "$lf"
     progress "everything" 1  "Resetting lava db -- logging to $lf"
-    run_remote "$buildhost" "dropdb -U postgres -h $dbhost $db || true" "$lf"
-    run_remote "$buildhost" "createdb -U postgres -h $dbhost $db || true" "$lf"
-    run_remote "$buildhost" "psql -d $db -h $dbhost -f $lava/tools/lavaODB/generated/lava.sql -U postgres" "$lf"
+    run_remote "$buildhost" "dropdb -U $pguser -h $dbhost $db || true" "$lf"
+    run_remote "$buildhost" "createdb -U $pguser -h $dbhost $db || true" "$lf"
+    run_remote "$buildhost" "psql -d $db -h $dbhost -f $lava/tools/lavaODB/generated/lava.sql -U $pguser" "$lf"
     run_remote "$buildhost" "echo dbwipe complete" "$lf"
 }
 
@@ -146,6 +145,7 @@ if [ $reset -eq 1 ]; then
     deldir "$directory/$name/"'*rr-*'
     # remove all plog files in the directory
     deldir "$directory/$name/*.plog"
+    deldir "$directory/$name/*.json"
     progress "everything" 0 "Truncating logs..."
     for i in $(ls "$logs" | grep '.log$'); do
         truncate "$logs/$i"
@@ -212,7 +212,7 @@ if [ $taint -eq 1 ]; then
         # If we didn't just reset the DB, we need clear out any existing taint labels before running FBI
         progress "everything" 1 "Clearing taint data from DB"
         lf="$logs/dbwipe_taint.log"
-        run_remote "$buildhost" "psql -U postgres -h $dbhost -c \"delete from dua_viable_bytes; delete from labelset;\" $db" "$lf"
+        run_remote "$buildhost" "psql -U $pguser -h $dbhost -c \"delete from dua_viable_bytes; delete from labelset;\" $db" "$lf"
     fi
     progress "everything" 1 "Taint step -- running panda and fbi"
     for input in $inputs
@@ -223,14 +223,14 @@ if [ $taint -eq 1 ]; then
         progress "everything" 1 "PANDA taint analysis prospective bug mining -- input $input -- logging to $lf"
         run_remote "$buildhost" "$python $scripts/bug_mining.py $hostjson $project_name $input $curtail" "$lf"
         echo -n "Num Bugs in db: "
-        bug_count=$(run_remote "$buildhost" "psql -At $db -U postgres -h $dbhost -c 'select count(*) from bug'")
+        bug_count=$(run_remote "$buildhost" "psql -At $db -U $pguser -h $dbhost -c 'select count(*) from bug'")
         if [ "$bug_count" = "0" ]; then
             echo "FATAL ERROR: no bugs found"
             exit 1
         fi
         echo "Found $bug_count bugs"
         echo
-        run_remote "$buildhost" "psql $db -U postgres -h $dbhost -c 'select count(*), type from bug group by type order by type'"
+        run_remote "$buildhost" "psql $db -U $pguser -h $dbhost -c 'select count(*), type from bug group by type order by type'"
     done
     tock
     echo "bug_mining complete $time_diff seconds"

scripts/setup_postgres.sh

@@ -8,12 +8,10 @@ if [ $EUID -ne 0 ]; then
 fi
 
 PGPASS="${HOME}/.pgpass"
+PG_VERSION=$(psql --version | awk '{print $3}' | cut -d '.' -f 1)
 
 if [ ! -f "${PGPASS}" ]; then
-    postgres_depends=$(dpkg-query -W -f='${depends}' 'postgresql')
-    postgres_pkg=$(echo "${postgres_depends}" | grep -oP 'postgresql-[0-9]+.?[0-9]+')
-    postgres_version=${postgres_pkg/postgresql-/}
-    pg_hba="/etc/postgresql/${postgres_version}/main/pg_hba.conf"
+    pg_hba="/etc/postgresql/${PG_VERSION}/main/pg_hba.conf"
     postgres_password='postgrespostgres'
 
     $SUDO sed -i.bak -E 's/^(local\s+all\s+postgres\s+)md5$/\1peer/' "${pg_hba}"
@@ -30,7 +28,7 @@ if [ ! -f "${PGPASS}" ]; then
 fi
 
 # Define the PostgreSQL version
-PG_VERSION=$(psql --version | awk '{print $3}' | cut -d '.' -f 1)
+
 
 # Define the configuration file paths
 PG_CONF="/etc/postgresql/${PG_VERSION}/main/postgresql.conf"

scripts/vars.sh

@@ -25,6 +25,16 @@ output_dir="$(jq -r '.output_dir // ""' $hostjson)"
 config_dir="$(jq -r '.config_dir // ""' $hostjson)/$project_name"
 tar_dir="$(jq -r '.tar_dir // ""' $hostjson)"
 db_suffix="$(jq -r '.db_suffix // ""' $hostjson)"
+buildhost="$(jq -r '.buildhost // "localhost"' $hostjson)"
+testinghost="$(jq -r '.testinghost // "localhost"' $hostjson)"
+dockername="$(jq -r '.docker // "lava32"' $hostjson)"
+pguser="$(jq -r '.pguser // "postgres"' $hostjson)"
+pgpass="$(jq -r '.pgpass // "postgrespostgres"' $hostjson)"
+dbhost="$(jq -r '.host // "database"' $hostjson)"
+
+export PGUSER=$pguser
+export PGPASS=$pgpass
+
 json="${config_dir}/$project_name.json"
 
 if [ ! -f $json ]; then
@@ -66,10 +76,6 @@ if [ "$(jq -r .injfixupsscript $json)" != "null" ]; then
     injfixupsscript="${injfixupsscript/\{bug_build\}/$bug_build}"
 fi
 
-buildhost="$(jq -r '.buildhost // "docker"' $json)"
-# buildhost="localhost"
-pandahost="$(jq -r '.pandahost // "docker"' $json)"
-testinghost="$(jq -r '.testinghost // "docker"' $json)"
 logs="$output_dir/$name/logs"
 
 makecmd="$(jq -r .make $json)"
@@ -78,15 +84,8 @@ install="${install/\{config_dir\}/$config_dir}" # Format string replacement for
 post_install="$(jq -r .post_install $json)"
 install_simple=$(jq -r .install_simple $json)
 configure_cmd=$(jq -r '.configure // "/bin/true"' $json)
-container="$(jq -r '.docker // "lava32"' $json)"
 
 # Constants
 scripts="$lava/scripts"
-python="/usr/bin/python3"
-pdb="/usr/bin/python3 -m pdb "
-dockername="lava32"
-
-dbhost="database"
-if [ "$buildhost" = "localhost" ]; then
-    dbhost="localhost"
-fi
+python="python3"
+pdb="python3 -m pdb "

tools/fbi/src/find_bug_inj.cpp

@@ -1084,9 +1084,9 @@ int main (int argc, char **argv) {
             update_liveness(ple);
         } else if (ple.isMember("attackPoint")) {
             attack_point_lval_usage(ple);
-        } else if (ple.isMember("dwarfCall")) {
+        } else if (ple.isMember("dwarf2Call")) {
             record_call(ple);
-        } else if (ple.isMember("dwarfRet")) {
+        } else if (ple.isMember("dwarf2Ret")) {
             record_ret(ple);
         }
         // pandalog_free_entry(ple);