FBI now uses LLVM-11 and now uses JSON instead of Panda Logs.

bug_mining.py is also updated

.gitignore

@@ -1,4 +1,5 @@
 # ignore IDE and any panda wheel/debian packages
+.vscode
 .idea
 *.deb
 *.whl

Dockerfile

@@ -1,62 +1,45 @@
-FROM ubuntu:20.04 as builder
+ARG BASE_IMAGE="ubuntu:20.04"
+
+### BASE IMAGE
+FROM $BASE_IMAGE as base
+ARG BASE_IMAGE
 
 ENV DEBIAN_FRONTEND=noninteractive
+ENV LLVM_DIR=/usr/lib/llvm-11
+
+# Copy dependencies lists into container. We copy them all and then do a mv because
+# we need to transform base_image into a windows compatible filename which we can't
+# do in a COPY command.
+COPY ./dependencies/* /tmp
+RUN mv /tmp/$(echo "$BASE_IMAGE" | sed 's/:/_/g')_build.txt /tmp/build_dep.txt && \
+    mv /tmp/$(echo "$BASE_IMAGE" | sed 's/:/_/g')_base.txt /tmp/base_dep.txt
 
-RUN apt-get update && apt-get install -qq -y \
-    bc \
-    build-essential \
-    clang-tools-11 \
-    cmake \
-    git \
-    inotify-tools \
-    jq \
-    libclang-11-dev \
-    libfdt-dev \
-    libjsoncpp-dev \
-    libjsoncpp1 \
-    libpq-dev \
-    llvm-11-dev \
-    postgresql \
-    python3-psycopg2 \
-    python3-sqlalchemy \
-    socat \
-    wget
+# Base image just needs runtime dependencies
+RUN [ -e /tmp/base_dep.txt ] && \
+    apt-get -qq update && \
+    apt-get -qq install -y --no-install-recommends curl $(cat /tmp/base_dep.txt | grep -o '^[^#]*') && \
+    apt-get clean
 
-# Step 1: Install panda debian package, you need a version that has Dwarf2 Plugin
+# Finally: Install panda debian package, you need a version that has the Dwarf2 Plugin
 RUN wget https://github.com/panda-re/panda/releases/download/v1.8.23/pandare_20.04.deb
 RUN command apt install -qq -y ./pandare_20.04.deb
 RUN pip install pandare
 
-# Libodb
-RUN cd /tmp && \
-    wget http://codesynthesis.com/download/odb/2.4/odb_2.4.0-1_amd64.deb && \
-    wget http://codesynthesis.com/download/odb/2.4/libodb-2.4.0.tar.gz && \
-    wget http://codesynthesis.com/download/odb/2.4/libodb-pgsql-2.4.0.tar.gz && \
-    dpkg -i odb_2.4.0-1_amd64.deb && \
-    tar xf libodb-pgsql-2.4.0.tar.gz && \
-    tar xf libodb-2.4.0.tar.gz && \
-    cd /tmp/libodb-2.4.0 && \
-    CXXFLAGS='-D_GLIBCXX_USE_CXX11_ABI=0' ./configure --enable-shared && \
-    make -j $(nproc) && \
-    make install && \
-    cd /tmp/libodb-pgsql-2.4.0 && \
-    CXXFLAGS='-D_GLIBCXX_USE_CXX11_ABI=0' ./configure --enable-shared && \
-    make -j $(nproc) && \
-    make install
-# TODO in main container
-#RUN echo "/usr/local/lib" > /etc/ld.so.conf.d/usr-local-lib.conf
-#RUN ldconfig
+### BUILD IMAGE - STAGE 2
+FROM base AS builder
+ARG BASE_IMAGE
 
-# Build btrace
-COPY tools/btrace /tools/btrace
-RUN cd /tools/btrace && \
-    bash compile.sh
+RUN [ -e /tmp/build_dep.txt ] && \
+    apt-get -qq update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends $(cat /tmp/build_dep.txt | grep -o '^[^#]*') && \
+    apt-get clean
 
-# Build lavaTool. Depends on headers in lavaODB and tools/lavaDB
-#COPY tools/lavaODB/ tools/lavaDB/ tools/lavaTool/ /tools/
-COPY tools/ /tools
+#### Develop setup: panda built + pypanda installed (in develop mode) - Stage 3
+FROM builder as developer
+
+COPY ./tools/ /tools
 COPY setup_container.py /
-ENV LLVM_DIR=/usr/lib/llvm-11
+
 RUN python3 setup_container.py
 
 # RUN cd /tools && \

dependencies/ubuntu_20.04_base.txt

@@ -0,0 +1 @@
+# lava dependencies, needed to run LAVA

dependencies/ubuntu_20.04_build.txt

@@ -0,0 +1,24 @@
+# lava dependencies, to compile LAVA
+
+# Based on original setup.py after panda is installed step
+# https://installati.one/install-odb-ubuntu-20-04/?expand_article=1
+odb
+
+# https://installati.one/install-libodbc2-ubuntu-22-04/?expand_article=1
+libodbc2
+
+# https://installati.one/install-libodb-pgsql-2.4-ubuntu-20-04/
+libodb-pgsql-dev
+
+# Brendan noticed these libraries were needed to compile FBI
+libodb-pgsql-2.4
+
+
+# https://pypi.org/project/pyzmq/
+libzmq3-dev
+
+# libc6 needed for compiling btrace?
+
+# libjsoncpp needed for fbi json parsing?
+
+# I may need g++-10?

docker/requirements.txt

@@ -1,10 +1,13 @@
-SQLAlchemy==2.0.25
-pyzmq==23.2.1
-psycopg2-binary==2.9.9
-tabulate==0.9.0
-pexpect>=4.8.0
-psutil==5.9.8
-ipython==7.16.3
-colorama==0.4.6
-numpy==1.22.0
-argparse==1.4.0
+SQLAlchemy==1.0.14
+zmq==0.0.0
+pyzmq==15.2.0
+psycopg2==2.6.1
+lockfile==0.10.2
+tabulate==0.7.5
+pexpect==3.2
+psutil==5.6.6
+ipython==2.3.0
+colorama==0.3.2
+numpy==1.8.2
+argparse==1.2.1
+pandare

install.sh

@@ -14,7 +14,7 @@ progress() {
 }
 
 # Step 1: Install panda debian package
-wget https://github.com/panda-re/panda/releases/download/v1.6/pandare_22.04.deb
+wget https://github.com/panda-re/panda/releases/download/v1.8.23/pandare_22.04.deb
 $SUDO apt install ./pandare_22.04.deb
 
 # Remove the comment to update all debian stuff from sources.list
@@ -34,7 +34,7 @@ progress "Updates complete"
 # Dependencies are for a major version, but the filenames include minor versions
 # So take our major version, find the first match in dependencies directory and run with it.
 # This will give us "./panda/dependencies/ubuntu:20.04" where ubuntu:20.04_build.txt or 20.04_base.txt exists
-version=$(lsb_release -r| awk '{print $2}' | awk -F'.' '{print $1}')
+version=$(lsb_release -r | awk '{print $2}' | awk -F'.' '{print $1}')
 # shellcheck disable=SC2086
 dep_base=$(find ./dependencies/ubuntu_${version}.* -print -quit | sed  -e "s/_build\.txt\|_base\.txt//")
 
@@ -59,16 +59,6 @@ $SUDO pip3 install --upgrade pip
 $SUDO pip3 install -r docker/requirements.txt
 progress "Installed Python requirements"
 
-# Everything here is everything in setup_container.py
-rm -rf tools/build
-mkdir -p tools/build
-mkdir -p tools/install
-cmake -Btools/build -Htools -DCMAKE_INSTALL_PREFIX=tools/install
-
-# Compile lavaTool
-make --no-print-directory -j4 install -Ctools/build/lavaTool
-
-# Compile FBI
-make --no-print-directory -j4 -C fbi install -Ctools/build
+$SUDO python3 setup_container.py
 
 progress "Installed LAVA"

scripts/bug_mining.py

@@ -95,7 +95,7 @@ def progress(msg):
 print("bug_mining.py %s %s" % (project_name, input_file))
 
 if len(sys.argv) > 4:
-    #global curtail
+    # global curtail
     curtail = int(sys.argv[4])
 
 qemu_path = project['qemu']
@@ -166,8 +166,8 @@ def progress(msg):
 else:
     proc_name = basename(command_args[0])
 
-pandalog = "{}/queries-{}.plog".format(project['output_dir'],
-                                       os.path.basename(isoname))
+pandalog = "{}/queries-{}.plog".format(project['output_dir'], os.path.basename(isoname))
+pandalog_json = "{}/queries-{}.json".format(project['output_dir'], os.path.basename(isoname))
 
 print("pandalog = [%s] " % pandalog)
 
@@ -201,10 +201,10 @@ def progress(msg):
     '-pandalog', pandalog, '-os', panda_os_string
 ]
 
-for plugin, plugin_args in panda_args.iteritems():
+for plugin, plugin_args in panda_args.items():
     qemu_args.append('-panda')
     arg_string = ",".join(["{}={}".format(arg, val)
-                           for arg, val in plugin_args.iteritems()])
+                           for arg, val in plugin_args.items()])
     qemu_args.append('{}{}{}'.format(plugin, ':'
     if arg_string else '', arg_string))
 
@@ -247,14 +247,24 @@ def progress(msg):
 
 print()
 progress("Calling the FBI on queries.plog...")
+
+convert_json_args = ['python', '-m', 'pandare.plog_reader', pandalog]
+print("panda log JSON invocation: [%s]" % (subprocess.list2cmdline(convert_json_args)))
+try:
+    with open(pandalog_json, 'wb') as fd:
+        subprocess.check_call(convert_json_args, stdout=fd, stderr=sys.stderr)
+except subprocess.CalledProcessError as e:
+    print("The script to convert panda log into JSON has failed")
+    raise e
+
 # fbi_args = [join(lavadir, 'fbi', 'fbi'),
 # project_file, pandalog, input_file_base]
 fbi_args = [join(lavadir, 'tools', 'install', 'bin', 'fbi'), host_json,
-            project_name, pandalog, input_file_base]
+            project_name, pandalog_json, input_file_base]
 
 # Command line curtial argument takes priority, otherwise use project specific one
-#global curtail
-if curtail !=0 :
+# global curtail
+if curtail != 0:
     fbi_args.append(str(curtail))
 elif "curtail" in project:
     fbi_args.append(str(project.get("curtail", 0)))

tools/fbi/src/CMakeLists.txt

@@ -3,15 +3,14 @@ project (FBI LANGUAGES CXX)
 
 # fbi target
 add_executable(fbi find_bug_inj.cpp)
-set_property(TARGET fbi PROPERTY CXX_STANDARD 14)
+set_property(TARGET fbi PROPERTY CXX_STANDARD 17)
 
-target_compile_options(fbi PRIVATE -D_GLIBCXX_USE_CXX11_ABI=0)
-if (${DEBUG})
-    target_compile_options(fbi PRIVATE -fno-omit-frame-pointer -g -O0)
-else()
-    target_compile_options(fbi PRIVATE -flto -O3)
-    set_target_properties(fbi PROPERTIES LINK_FLAGS "-flto -fuse-ld=gold")
-endif()
+#if (${DEBUG})
+#    target_compile_options(fbi PRIVATE -fno-omit-frame-pointer -g -O0)
+#else()
+#    target_compile_options(fbi PRIVATE -flto -O3)
+#    set_target_properties(fbi PROPERTIES LINK_FLAGS "-flto -fuse-ld=gold")
+#endif()
 
 target_include_directories(fbi BEFORE
         PUBLIC
@@ -21,7 +20,9 @@ target_include_directories(fbi BEFORE
         ${CMAKE_CURRENT_SOURCE_DIR}/../../include
         /usr/lib/odb/x86_64-linux-gnu/include
         /usr/local/include 
-    )
+        /usr/include/jsoncpp
+)
+
 add_dependencies(fbi lava-odb_x64)
 set_target_properties(fbi PROPERTIES LINK_FLAGS "-L/usr/local/lib")
 target_link_libraries(fbi
@@ -31,10 +32,10 @@ target_link_libraries(fbi
     z
     odb
     odb-pgsql
-    jsoncpp
-    pq
     protobuf
+    jsoncpp
 )
+
 install (TARGETS fbi
          RUNTIME DESTINATION bin
          LIBRARY DESTINATION lib