it is working with ipython now

src/manager.cc

@@ -617,12 +617,35 @@ void Manager::disable_branch(void *id) {
 
 void* Manager::ensure_not_traced() {
   auto head = get_tracer_head();
-  assert(!head->is_traced);
-  assert(!head->is_temp_disabled);
-  if(head->trace_id) {
+  if(head->is_traced) {
+    if(head->is_compiled) {
+      // must be in the context of a not traced function
+      // which doesn't have any external viewable info (except maybe ret addr doesn't point to some code buffer.....)
+      return NULL;
+    }
     auto info = &branches[head->trace_id];
-    info->disabled = true;
+    Tracer *l = head->tracer;
+    assert(info->tracer == head->tracer);
+    void *ret = l->EndTraceEnsure();
+    if(ret == NULL) {
+      // this must be a not inlined function which is fine I guess?
+      return NULL;
+    }
+    info->tracer = head->tracer = nullptr;
+
+    Tracer *expected = nullptr;
+    if(!free_tracer_list.compare_exchange_strong(expected, l)) {
+      // failled to save the tracer to the free list head
+      delete l;
+    }
+
+    return ret;
   }
+  assert(!head->is_temp_disabled);
+  // if(head->trace_id) {
+  //   auto info = &branches[head->trace_id];
+  //   info->disabled = true;
+  // }
   return NULL;
 }
 

src/tracer.cc

@@ -202,6 +202,18 @@ extern "C" void* red_end_trace(mem_loc_t normal_end_address) {
   return ret;
 }
 
+extern "C" void red_end_trace_ensure() {
+  auto head = manager->get_tracer_head();
+  assert(head->is_traced);
+  head->is_traced = false;
+  head->is_compiled = false;
+  assert(!head->resume_addr);
+  assert(!head->tracer);
+#ifdef CONF_VERBOSE
+  red_printf("exiting trace ensure %#016lx\n", head->trace_id);
+#endif
+}
+
 extern "C" void* red_branch_to_sub_trace(void *resume_addr, void *sub_trace_id, void* target_rip) {
   auto head = manager->get_tracer_head();
   assert(head->is_traced);
@@ -609,6 +621,52 @@ void* Tracer::EndTraceEndBranchable() {
   return EndTraceFallthrough(false);
 }
 
+void* Tracer::EndTraceEnsure() {
+  if(current_not_traced_call_addr != (mem_loc_t)&redmagic_ensure_not_traced) {
+    // then this is probably inside some not trace method block
+    // which is fine I guess
+    return NULL;
+  }
+  current_not_traced_call_addr = 0;
+  assert(icount - last_call_instruction < 2);
+
+#ifdef CONF_VERBOSE
+  red_printf("tracer end ensure\n");
+#endif
+
+
+  auto head = manager->get_tracer_head();
+  auto info = &manager->branches[head->trace_id];
+  info->traced_instruction_count += icount;
+  info->finish_traces++;
+  if(info->longest_trace_instruction_count < icount)
+    info->longest_trace_instruction_count = icount;
+
+  buffer->setOffset(last_call_generated_op);
+  SimpleCompiler compiler(buffer);
+  compiler.call(asmjit::imm_ptr(&red_end_trace_ensure));
+  compiler.jmp(asmjit::imm_ptr(last_call_ret_addr));
+  auto w = compiler.finalize();
+
+  merge_resume = 0;
+  while(merge_block_stack.size() > 0) {
+    mem_loc_t write_addr = merge_block_stack.back();
+    merge_block_stack.pop_back();
+    while(write_addr != 0) {
+      mem_loc_t next_addr = *(mem_loc_t*)write_addr;
+      *(mem_loc_t*)write_addr = 0;
+      write_addr = next_addr;
+    }
+  }
+  merge_block_stack.push_back(0);
+  method_address_stack.clear();
+
+  finish_patch();
+  tracing_from = 0;
+
+  return (void*)w.getRawBuffer();
+}
+
 
 
 void* Tracer::TempDisableTrace() {

src/tracer.h

@@ -40,6 +40,10 @@ namespace redmagic {
     // need to have their proper fallthrough methods called
     void* EndTraceEndBranchable();
 
+    // drop all the way back to normal code, used with ensure not traced
+    // trace may end up getting resumed on backwards branch or fellthrough branch operation
+    void* EndTraceEnsure();
+
     // if there is another backwards branch inside of this backwards branch
     // the there is a nested loop that we should trace
     void JumpToNestedLoop(void *nested_trace_id);