difficult to get internal method address that we want to ignore....

.gitignore

@@ -1,3 +1,5 @@
 *.o
 jit-test
 .deps
+build/
+release/

make

@@ -13,29 +13,19 @@ TARGET = 'jit-test'
 UNIT_TARGET = 'build/unit_tests'
 
 CXX_FLAGS = (
+    '-fPIC '
     '-std=c++14 '
     '-I ./deps/ '
     '-ggdb '
     '-O0 '
-    '-I ./deps/udis86'
-    # '-I /home/matthew/Downloads/bochs-code/bochs/cpu/ '
-    # '-I /home/matthew/Downloads/bochs-code/bochs/ '
-    # '-I /home/matthew/Downloads/bochs-code/bochs/instrument/stubs '
-
+    '-I ./deps/udis86 '
 )
 CXX_FLAGS_UNIT = (
     '-I ./deps/catch/ '
     '-I ./src/ '
 )
 LIBS = (
     '-pthread '
-    'deps/udis86/libudis86/.libs/libudis86.a '
-    # '/home/matthew/Downloads/bochs-code/bochs/cpu/libcpu.a '
-    # '/home/matthew/Downloads/bochs-code/bochs/logio.o '
-    # '/home/matthew/Downloads/bochs-code/bochs/cpu/fpu/libfpu.a '
-    # '/home/matthew/Downloads/bochs-code/bochs/cpu/cpudb/libcpudb.a '
-    # '/home/matthew/Downloads/bochs-code/bochs/gui/libgui.a '
-
     # '-ljemalloc'
 )
 LD_FLAGS = ''
@@ -56,13 +46,19 @@ def mic():
 
 def release():
     global CXX_FLAGS
-    CXX_FLAGS = CXX_FLAGS.replace('-O0', '-O3')
+    CXX_FLAGS = CXX_FLAGS.replace('-O0', '-O2')
     CXX_FLAGS = CXX_FLAGS.replace('-ggdb', '')
+    CXX_FLAGS += ' -DNDEBUG'
     build()
+    Run('mkdir -p release')
+    Run('cp build/libredmagic.so.1.0.0 release/')
+    Run('cp src/redmagic.h release/')
+    Run('strip --strip-unneeded -w -K redmagic_* release/libredmagic.so.1.0.0')
+
 
 def clean():
-    Shell('cd deps/udis86 && make clean', shell=True)
     autoclean()
+    Shell('cd deps/udis86 && make clean', shell=True)
 
 def run():
     build()
@@ -75,8 +71,15 @@ def debug():
     ))
 
 def link():
-    objs = ' '.join(filter(lambda x: 'unit_' not in x, glob.glob('build/*.o')))
-    Run('{CXX} {LD_FLAGS} -o {TARGET} {objs} {LIBS}'.format(
+    objs = ' '.join(filter(lambda x: 'unit_' not in x and 'main.o' not in x, glob.glob('build/*.o')))
+    # Run('ar rcs build/redmagic.a {objs} {LIBRARY_LIBS}'.format(
+    #     **dict(globals(), **locals())
+    # ))
+    udis_libs = ' '.join(glob.glob('deps/udis86/libudis86/.libs/*.o'))
+    Run('{LD} -shared -fPIC -Wl,-soname,libredmagic.so.1.0.0 -o build/libredmagic.so.1.0.0 {objs} {udis_libs} {LIBS}'.format(
+        **dict(globals(), **locals())
+    ))
+    Run('{LD} {LD_FLAGS} -o {TARGET} build/main.o build/libredmagic.so.1.0.0 -Wl,-rpath=$ORIGIN/build/'.format(
         **dict(globals(), **locals())
     ))
     after()
@@ -119,8 +122,8 @@ def unit():
 
 def deps():
     # udis86 version 1.7.2
-    if not os.path.isfile('deps/udis86/libudis86/.libs/libudis86.a'):
-        Shell('cd deps/udis86 && ./autogen.sh && PYTHON=`which python2` ./configure --enable-static && make', shell=True)
+    if not os.path.isfile('deps/udis86/libudis86/.libs/libudis86.so') or not os.path.isfile('deps/udis86/libudis86/itab.h'):
+        Shell('cd deps/udis86 && ./autogen.sh && PYTHON=`which python2` ./configure && make', shell=True)
     after()
 
 

src/asm.s

@@ -19,3 +19,11 @@ red_asm_end_trace:
 red_asm_begin_trace:
   int3
   ret
+
+
+  .global red_asm_return_after_method_call
+red_asm_return_after_method_call:
+  # at this point we will replace the instruction pointer with where we should have returned to
+  int3
+  # this next line will never run
+  jmp red_asm_return_after_method_call

src/child_manager.cc

@@ -19,19 +19,24 @@ extern "C" void redmagic_backwards_branch(void *id) {
   child_manager->backwards_branch(id);
 }
 
-extern "C" void redmagic_force_begin_trace() {
+extern "C" void redmagic_force_begin_trace(void *id) {
   child_manager->begin_trace();
 }
 
-extern "C" void redmagic_force_end_trace() {
+extern "C" void redmagic_force_end_trace(void *id) {
   child_manager->end_trace();
 }
 
+extern "C" void redmagic_force_jump_to_trace(void *id) {
+
+}
+
 extern "C" {
-  void red_asm_temp_disable_trace ();
-  void red_asm_temp_enable_trace ();
-  void red_asm_end_trace ();
-  void red_asm_begin_trace ();
+  void red_asm_temp_disable_trace();
+  void red_asm_temp_enable_trace();
+  void red_asm_end_trace();
+  void red_asm_begin_trace();
+  void red_asm_return_after_method_call();
 }
 
 namespace redmagic {
@@ -40,6 +45,7 @@ namespace redmagic {
     { red_asm_temp_enable_trace, TEMP_ENABLE_ACT },
     { red_asm_end_trace, END_TRACE_ACT },
     { red_asm_begin_trace, BEGIN_TRACE_ACT },
+    { red_asm_return_after_method_call, RETURN_FROM_METHOD_ACT },
     { NULL, MAX_ACT }
   };
 }
@@ -100,7 +106,7 @@ void ChildManager::end_trace() {
   if(!is_traced) {
     perror("ending trace when was not started\n");
   }
-  asm("act_end_trace: int3");
+  red_asm_end_trace();
   is_traced = false;
 
 }

src/jit_internal.h

@@ -21,6 +21,8 @@
 #include <thread>
 #include <atomic>
 #include <map>
+#include <vector>
+#include <set>
 
 #include <errno.h>
 
@@ -84,11 +86,13 @@ namespace redmagic {
 
     void set_program_pval(mem_loc_t where, uint8_t what);
     int get_program_pval(mem_loc_t where);
+    bool is_ignored_method(mem_loc_t where);
   private:
     int send_pipe, recv_pipe;
     pid_t child_pid;
     std::map<pid_t, Tracer*> tracers;
     std::map<mem_loc_t, uint8_t> program_map;
+    std::set<mem_loc_t> ignored_methods;
   };
 
   class Tracer {
@@ -120,16 +124,13 @@ namespace redmagic {
     mem_loc_t read_cache_loc = -1;
 
     unsigned int num_ins = 0;
+
+    std::vector<JumpTrace> traces;
   };
 
   extern ChildManager *child_manager;
   extern ParentManager *parent_manager;
 
-  enum CommOp {
-    START_TRACE,
-    END_TRACE,
-  };
-
   struct Check_struct {
     // which register to check
     // -1 if there is no need for a check
@@ -147,7 +148,18 @@ namespace redmagic {
     };
   };
 
+  enum TraceOp {
+    // represents that this is a typical instruction
+    INST_TRACE_OP, // a standard jump instruction
+    INST_LOOP_TRACE_OP, // a jump that is backwards and will execute multiple times in a row, eg only wait to exit is for this branch to fall through
+    BEGIN_TRACE_OP, // pushed at the start of the tracing processes
+    END_TRACE_OP,
+    TEMP_BREAK_TRACE_OP, // temp_disable/enable method
+    IGNORED_CALL_TRACE_OP, // when there is a call like malloc or some other registered call, work around it instead of tracing through it
+  };
+
   struct JumpTrace {
+    TraceOp op;
     register_t ins_pc;    // pc of where the instruction is located
     register_t target_pc; // pc after the instruction executed
     ud_mnemonic_code instruction;
@@ -157,6 +169,16 @@ namespace redmagic {
     // int instruction_len;
   };
 
+  enum CommOp {
+    // client ops
+    START_TRACE,
+    END_TRACE,
+
+    // parent ops
+    SEND_TRACE,
+
+  };
+
   struct Communication_struct {
     CommOp op;
     pid_t thread_pid;
@@ -170,6 +192,7 @@ namespace redmagic {
     BEGIN_TRACE_ACT,
     TEMP_DISABLE_ACT,
     TEMP_ENABLE_ACT,
+    RETURN_FROM_METHOD_ACT,
 
     NO_ACT,