diff --git a/compiler-rt/cmake/config-ix.cmake b/compiler-rt/cmake/config-ix.cmake index 8d3dc8d208b2..95598712e17a 100644 --- a/compiler-rt/cmake/config-ix.cmake +++ b/compiler-rt/cmake/config-ix.cmake @@ -191,6 +191,7 @@ if (ANDROID AND COMPILER_RT_HAS_LIBDL) endif() check_library_exists(c++ __cxa_throw "" COMPILER_RT_HAS_LIBCXX) check_library_exists(stdc++ __cxa_throw "" COMPILER_RT_HAS_LIBSTDCXX) +check_library_exists(snappy snappy_compress "" COMPILER_RT_HAS_SNAPPY) # Linker flags. llvm_check_compiler_linker_flag(C "-Wl,-z,text" COMPILER_RT_HAS_Z_TEXT) @@ -640,6 +641,15 @@ if(APPLE) list_intersect(ORC_SUPPORTED_ARCH ALL_ORC_SUPPORTED_ARCH SANITIZER_COMMON_SUPPORTED_ARCH) + list_intersect(CSI_SUPPORTED_ARCH + ALL_CSI_SUPPORTED_ARCH + SANITIZER_COMMON_SUPPORTED_ARCH) + list_intersect(CILKSAN_SUPPORTED_ARCH + ALL_CILKSAN_SUPPORTED_ARCH + SANITIZER_COMMON_SUPPORTED_ARCH) + list_intersect(CILKSCALE_SUPPORTED_ARCH + ALL_CILKSCALE_SUPPORTED_ARCH + SANITIZER_COMMON_SUPPORTED_ARCH) else() # Architectures supported by compiler-rt libraries. @@ -670,6 +680,9 @@ else() ${ALL_SHADOWCALLSTACK_SUPPORTED_ARCH}) filter_available_targets(GWP_ASAN_SUPPORTED_ARCH ${ALL_GWP_ASAN_SUPPORTED_ARCH}) filter_available_targets(ORC_SUPPORTED_ARCH ${ALL_ORC_SUPPORTED_ARCH}) + filter_available_targets(CSI_SUPPORTED_ARCH ${ALL_CSI_SUPPORTED_ARCH}) + filter_available_targets(CILKSAN_SUPPORTED_ARCH ${ALL_CILKSAN_SUPPORTED_ARCH}) + filter_available_targets(CILKSCALE_SUPPORTED_ARCH ${ALL_CILKSCALE_SUPPORTED_ARCH}) endif() if (MSVC) @@ -882,3 +895,24 @@ else() set(COMPILER_RT_HAS_GWP_ASAN FALSE) endif() pythonize_bool(COMPILER_RT_HAS_GWP_ASAN) + +if (COMPILER_RT_HAS_SANITIZER_COMMON AND CSI_SUPPORTED_ARCH AND + OS_NAME MATCHES "Linux") + set(COMPILER_RT_HAS_CSI TRUE) +else() + set(COMPILER_RT_HAS_CSI FALSE) +endif() + +if (COMPILER_RT_HAS_SANITIZER_COMMON AND CILKSAN_SUPPORTED_ARCH AND + OS_NAME MATCHES "Linux") + set(COMPILER_RT_HAS_CILKSAN TRUE) +else() + set(COMPILER_RT_HAS_CILKSAN FALSE) +endif() + +if (COMPILER_RT_HAS_SANITIZER_COMMON AND CILKSCALE_SUPPORTED_ARCH AND + OS_NAME MATCHES "Linux") + set(COMPILER_RT_HAS_CILKSCALE TRUE) +else() + set(COMPILER_RT_HAS_CILKSCALE FALSE) +endif() diff --git a/compiler-rt/lib/CMakeLists.txt b/compiler-rt/lib/CMakeLists.txt index 43ba9a102c84..952210153686 100644 --- a/compiler-rt/lib/CMakeLists.txt +++ b/compiler-rt/lib/CMakeLists.txt @@ -67,6 +67,19 @@ if(COMPILER_RT_BUILD_ORC) compiler_rt_build_runtime(orc) endif() +if(COMPILER_RT_BUILD_CSI) + compiler_rt_build_runtime(csi) +endif() + +if(COMPILER_RT_BUILD_CILKTOOLS) + compiler_rt_build_runtime(cilksan) + if(NOT COMPILER_RT_HAS_SNAPPY) + mesage(FATAL_ERROR + "The snappy compression library is required to build Cilksan.") + endif() + compiler_rt_build_runtime(cilkscale) +endif() + # It doesn't normally make sense to build runtimes when a sanitizer is enabled, # so we don't add_subdirectory the runtimes in that case. However, the opposite # is true for fuzzers that exercise parts of the runtime. So we add the fuzzer diff --git a/compiler-rt/lib/cilksan/CMakeLists.txt b/compiler-rt/lib/cilksan/CMakeLists.txt new file mode 100644 index 000000000000..7f85681ca2c3 --- /dev/null +++ b/compiler-rt/lib/cilksan/CMakeLists.txt @@ -0,0 +1,65 @@ +# Build for the Cilksan runtime support library. + +set(CILKSAN_SOURCES + cilksan.cpp + debug_util.cpp + mem_access.cpp + print_addr.cpp + shadow_mem.cpp + compresseddict_shadow_mem.cpp + static_dictionary.cpp + drivercsan.cpp + csanrt.cpp) + +include_directories(..) + +set(CILKSAN_CFLAGS ${SANITIZER_COMMON_CFLAGS} -std=c++11) + +append_rtti_flag(OFF CILKSAN_CFLAGS) + +set(CILKSAN_DYNAMIC_LINK_FLAGS) + +set(CILKSAN_DYNAMIC_DEFINITIONS + ${CILKSAN_COMMON_DEFINITIONS}) + +set(CILKSAN_DYNAMIC_CFLAGS ${CILKSAN_CFLAGS}) + +#append_list_if(COMPILER_RT_HAS_LIBC c CILKSAN_DYNAMIC_LIBS) +append_list_if(COMPILER_RT_HAS_LIBDL dl CILKSAN_DYNAMIC_LIBS) +append_list_if(COMPILER_RT_HAS_SNAPPY snappy CILKSAN_DYNAMIC_LIBS) +#append_list_if(COMPILER_RT_HAS_LIBRT rt CILKSAN_DYNAMIC_LIBS) +#append_list_if(COMPILER_RT_HAS_LIBM m CILKSAN_DYNAMIC_LIBS) +#append_list_if(COMPILER_RT_HAS_LIBPTHREAD pthread CILKSAN_DYNAMIC_LIBS) +#append_list_if(COMPILER_RT_HAS_LIBSTDCXX stdc++ CILKSAN_DYNAMIC_LIBS) +#append_list_if(COMPILER_RT_HAS_LIBLOG log CILKSAN_DYNAMIC_LIBS) + +# Compile CilkSan sources into an object library + +add_compiler_rt_object_libraries(RTCilksan_dynamic + OS ${SANITIZER_COMMON_SUPPORTED_OS} + ARCHS ${CILKSAN_SUPPORTED_ARCH} + SOURCES ${CILKSAN_SOURCES} + CFLAGS ${CILKSAN_DYNAMIC_CFLAGS} + DEFS ${CILKSAN_DYNAMIC_DEFINITIONS}) + +# Build Cilksan runtimes shipped with Clang. +add_compiler_rt_component(cilksan) + +foreach (arch ${CILKSAN_SUPPORTED_ARCH}) + add_compiler_rt_runtime(clang_rt.cilksan + SHARED + ARCHS ${arch} + OBJECT_LIBS RTCilksan_dynamic + CFLAGS ${CILKSAN_DYNAMIC_CFLAGS} + LINK_FLAGS ${CILKSAN_DYNAMIC_LINK_FLAGS} + LINK_LIBS ${CILKSAN_DYNAMIC_LIBS} + DEFS ${CILKSAN_DYNAMIC_DEFINITIONS} + PARENT_TARGET cilksan) + + # add_dependencies(cilksan + # clang_rt.cilksan-${arch}) +endforeach() + +if (COMPILER_RT_INCLUDE_TESTS) + # TODO(bruening): add tests via add_subdirectory(tests) +endif() diff --git a/compiler-rt/lib/cilksan/cilksan.cpp b/compiler-rt/lib/cilksan/cilksan.cpp new file mode 100644 index 000000000000..624176f9ee9c --- /dev/null +++ b/compiler-rt/lib/cilksan/cilksan.cpp @@ -0,0 +1,736 @@ +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "cilksan_internal.h" +#include "debug_util.h" +#include "disjointset.h" +#include "frame_data.h" +#include "mem_access.h" +#include "race_detect_update.h" +#include "shadow_mem.h" +#include "spbag.h" +#include "stack.h" + +#if CILKSAN_DEBUG +enum EventType_t last_event = NONE; +#endif + +static bool CILKSAN_INITIALIZED = false; + +// declared in driver.cpp +extern FILE *err_io; +// declared in print_addr.cpp +extern uintptr_t *load_pc; +extern uintptr_t *store_pc; + +// --------------------- stuff from racedetector --------------------------- + +// ------------------------------------------------------------------------- +// Analysis data structures and fields +// ------------------------------------------------------------------------- + +// List used for the disjoint set data structure's find_set operation. +List_t disjoint_set_list; + +#if CILKSAN_DEBUG +template<> +long DisjointSet_t::debug_count = 0; + +long SBag_t::debug_count = 0; +long PBag_t::debug_count = 0; +#endif + +void free_bag(DisjointSet_t *ptr) { + // TODO(ddoucet): ...... + // delete ptr->get_node(); + // TODO(denizokt): temporary fix, but introduces memory leak + delete ptr->get_my_set_node(); +} + +template<> +void (*DisjointSet_t::dtor_callback)(DisjointSet_t *) = + &free_bag; + +// Free list for disjoint-set nodes +template<> +DisjointSet_t * +DisjointSet_t::free_list = nullptr; + +// Free lists for SBags and PBags +SBag_t::FreeNode_t *SBag_t::free_list = nullptr; +PBag_t::FreeNode_t *PBag_t::free_list = nullptr; + +// Code to handle references to the stack. +// range of stack used by the process +uint64_t stack_low_addr = 0; +uint64_t stack_high_addr = 0; + +// small helper functions +static inline bool is_on_stack(uintptr_t addr) { + cilksan_assert(stack_high_addr != stack_low_addr); + return (addr <= stack_high_addr && addr >= stack_low_addr); +} + +// ANGE: Each function that causes a Disjoint set to be created has a +// unique ID (i.e., Cilk function and spawned C function). +// If a spawned function is also a Cilk function, a Disjoint Set is created +// for code between the point where detach finishes and the point the Cilk +// function calls enter_frame, which may be unnecessary in some case. +// (But potentially necessary in the case where the base case is executed.) +static uint64_t frame_id = 0; + +// Data associated with the stack of Cilk frames or spawned C frames. +// head contains the SP bags for the function we are currently processing +static Stack_t frame_stack; +call_stack_t call_stack; +Stack_t sp_stack; + +// Free list for call-stack nodes +call_stack_node_t *call_stack_node_t::free_list = nullptr; + +// Shadow memory, or the unordered hashmap that maps a memory address to its +// last reader and writer +Shadow_Memory shadow_memory; + +// extern functions, defined in print_addr.cpp +extern void print_race_report(); +extern int get_num_races_found(); + + +//////////////////////////////////////////////////////////////////////// +// Events functions +//////////////////////////////////////////////////////////////////////// + +/// Helper function for merging returning child's bag into parent's +static inline void merge_bag_from_returning_child(bool returning_from_detach) { + FrameData_t *parent = frame_stack.ancestor(1); + FrameData_t *child = frame_stack.head(); + cilksan_assert(parent->Sbag); + cilksan_assert(child->Sbag); + // cilksan_assert(!child->Pbag); + + if (returning_from_detach) { + // We are returning from a detach. Merge the child S- and P-bags + // into the parent P-bag. + DBG_TRACE(DEBUG_BAGS, + "Merge S-bag from detached child %ld to P-bag from parent %ld.\n", + child->Sbag->get_set_node()->get_func_id(), + parent->Sbag->get_set_node()->get_func_id()); + + // Get the parent P-bag. + DisjointSet_t *parent_pbag = parent->Pbag; + if (!parent_pbag) { // lazily create PBag when needed + DBG_TRACE(DEBUG_BAGS, + "frame %ld creates a PBag ", + parent->Sbag->get_set_node()->get_func_id()); + parent_pbag = + new DisjointSet_t(new PBag_t(parent->Sbag->get_node())); + parent->set_pbag(parent_pbag); + DBG_TRACE(DEBUG_BAGS, "%p\n", parent_pbag); + } + + cilksan_assert(parent_pbag && parent_pbag->get_set_node()->is_PBag()); + // Combine child S-bag into parent P-bag. + cilksan_assert(child->Sbag->get_set_node()->is_SBag()); + parent_pbag->combine(child->Sbag); + // Combine might destroy child->Sbag + cilksan_assert(child->Sbag->get_set_node()->is_PBag()); + + // Combine child P-bag into parent P-bag. + if (child->Pbag) { + DBG_TRACE(DEBUG_BAGS, + "Merge P-bag from spawned child %ld to P-bag from parent %ld.\n", + child->Sbag->get_set_node()->get_func_id(), + parent->Sbag->get_set_node()->get_func_id()); + cilksan_assert(child->Pbag->get_set_node()->is_PBag()); + parent_pbag->combine(child->Pbag); + cilksan_assert(child->Pbag->get_set_node()->is_PBag()); + } + + } else { + // We are returning from a call. Merge the child S-bag into the + // parent S-bag, and merge the child P-bag into the parent P-bag. + DBG_TRACE(DEBUG_BAGS, "Merge S-bag from called child %ld to S-bag from parent %ld.\n", + child->Sbag->get_set_node()->get_func_id(), + parent->Sbag->get_set_node()->get_func_id()); + // fprintf(stderr, "parent->Sbag = %p, child->Sbag = %p\n", + // parent->Sbag, child->Sbag); + cilksan_assert(parent->Sbag->get_set_node()->is_SBag()); + parent->Sbag->combine(child->Sbag); + + // fprintf(stderr, "parent->Pbag = %p, child->Pbag = %p\n", + // parent->Pbag, child->Pbag); + // Combine child P-bag into parent P-bag. + if (child->Pbag) { + // Get the parent P-bag. + DisjointSet_t *parent_pbag = parent->Pbag; + if (!parent_pbag) { // lazily create PBag when needed + DBG_TRACE(DEBUG_BAGS, + "frame %ld creates a PBag ", + parent->Sbag->get_set_node()->get_func_id()); + parent_pbag = + new DisjointSet_t(new PBag_t(parent->Sbag->get_node())); + parent->set_pbag(parent_pbag); + DBG_TRACE(DEBUG_BAGS, "%p\n", parent_pbag); + } + + DBG_TRACE(DEBUG_BAGS, "Merge P-bag from called child %ld to P-bag from parent %ld.\n", + child->frame_data.frame_id, + parent->Sbag->get_set_node()->get_func_id()); + cilksan_assert(parent_pbag && parent_pbag->get_set_node()->is_PBag()); + // Combine child P-bag into parent P-bag. + cilksan_assert(child->Pbag->get_set_node()->is_PBag()); + parent_pbag->combine(child->Pbag); + cilksan_assert(child->Pbag->get_set_node()->is_PBag()); + } + } + DBG_TRACE(DEBUG_BAGS, "After merge, parent set node func id: %ld.\n", + parent->Sbag->get_set_node()->get_func_id()); + cilksan_assert(parent->Sbag->get_node()->get_func_id() == + parent->Sbag->get_set_node()->get_func_id()); + + child->set_sbag(NULL); + child->set_pbag(NULL); +} + +/// Helper function for handling the start of a new function. This +/// function can be a spawned or called Cilk function or a spawned C +/// function. A called C function is treated as inlined. +static inline void start_new_function() { + frame_id++; + frame_stack.push(); + + DBG_TRACE(DEBUG_CALLBACK, "Enter frame %ld, ", frame_id); + + // Get the parent pointer after we push, because once pused, the + // pointer may no longer be valid due to resize. + FrameData_t *parent = frame_stack.ancestor(1); + DBG_TRACE(DEBUG_CALLBACK, "parent frame %ld.\n", parent->frame_data.frame_id); + DisjointSet_t *child_sbag, *parent_sbag = parent->Sbag; + + FrameData_t *child = frame_stack.head(); + cilksan_assert(child->Sbag == NULL); + cilksan_assert(child->Pbag == NULL); + + child_sbag = + new DisjointSet_t(new SBag_t(frame_id, + parent_sbag->get_node())); + + child->init_new_function(child_sbag); + + // We do the assertion after the init so that ref_count is 1. + cilksan_assert(child_sbag->get_set_node()->is_SBag()); + + WHEN_CILKSAN_DEBUG(frame_stack.head()->frame_data.frame_id = frame_id); + + DBG_TRACE(DEBUG_CALLBACK, "Enter function id %ld\n", frame_id); +} + +/// Helper function for exiting a function; counterpart of +/// start_new_function. +static inline void exit_function() { + // Popping doesn't actually destruct the object so we need to + // manually dec the ref counts here. + frame_stack.head()->reset(); + frame_stack.pop(); +} + +/// Action performed on entering a Cilk function (excluding spawn +/// helper). +static inline void enter_cilk_function() { + DBG_TRACE(DEBUG_CALLBACK, "entering a Cilk function, push frame_stack\n"); + start_new_function(); +} + +/// Action performed on leaving a Cilk function (excluding spawn +/// helper). +static inline void leave_cilk_function() { + DBG_TRACE(DEBUG_CALLBACK, + "leaving a Cilk function (spawner or helper), pop frame_stack\n"); + + /* param: not returning from a spawn */ + merge_bag_from_returning_child(0); + exit_function(); +} + +/// Action performed on entering a spawned child. +/// (That is, right after detach.) +static inline void enter_detach_child() { + DBG_TRACE(DEBUG_CALLBACK, "done detach, push frame_stack\n"); + start_new_function(); + // Copy the rsp from the parent. + FrameData_t *detached = frame_stack.head(); + FrameData_t *parent = frame_stack.ancestor(1); + detached->Sbag->get_node()->set_rsp(parent->Sbag->get_node()->get_rsp()); + // Set the frame data. + frame_stack.head()->frame_data.entry_type = DETACHER; + frame_stack.head()->frame_data.frame_type = SHADOW_FRAME; + DBG_TRACE(DEBUG_CALLBACK, "new detach frame started\n"); +} + +/// Action performed when returning from a spawned child. +/// (That is, returning from a spawn helper.) +static inline void return_from_detach() { + + DBG_TRACE(DEBUG_CALLBACK, "return from detach, pop frame_stack\n"); + cilksan_assert(DETACHER == frame_stack.head()->frame_data.entry_type); + /* param: we are returning from a spawn */ + merge_bag_from_returning_child(1); + exit_function(); + // Detacher frames do not have separate leave calls from the helpers + // containing them, so we manually call leave_cilk_function again. + leave_cilk_function(); +} + +/// Action performed immediately after passing a sync. +static void complete_sync() { + FrameData_t *f = frame_stack.head(); + DBG_TRACE(DEBUG_CALLBACK, "frame %d done sync\n", + f->Sbag->get_node()->get_func_id()); + + cilksan_assert(f->Sbag->get_set_node()->is_SBag()); + // Pbag could be NULL if we encounter a sync without any spawn (i.e., any Cilk + // function that executes the base case) + if (f->Pbag) { + cilksan_assert(f->Pbag->get_set_node()->is_PBag()); + f->Sbag->combine(f->Pbag); + cilksan_assert(f->Pbag->get_set_node()->is_SBag()); + cilksan_assert(f->Sbag->get_node()->get_func_id() == + f->Sbag->get_set_node()->get_func_id()); + f->set_pbag(NULL); + } +} + +//--------------------------------------------------------------- +// Callback functions +//--------------------------------------------------------------- +void cilksan_do_enter_begin() { + cilksan_assert(CILKSAN_INITIALIZED); + cilksan_assert(last_event == NONE); + WHEN_CILKSAN_DEBUG(last_event = ENTER_FRAME); + DBG_TRACE(DEBUG_CALLBACK, "frame %ld cilk_enter_frame_begin, stack depth %d\n", + frame_id+1, frame_stack.size()); + +/* + if (entry_stack.size() == 1) { + // we are entering the top-level Cilk function; everything we did + // before can be cleared, since we can't possibly be racing with + // anything old at this point + shadow_mem.clear(); + } +*/ + // entry_stack.push(); + // entry_stack.head()->entry_type = SPAWNER; + // entry_stack.head()->frame_type = SHADOW_FRAME; + // entry_stack always gets pushed slightly before frame_id gets incremented + // WHEN_CILKSAN_DEBUG(entry_stack.head()->frame_id = frame_id+1); + enter_cilk_function(); + frame_stack.head()->frame_data.entry_type = SPAWNER; + frame_stack.head()->frame_data.frame_type = SHADOW_FRAME; +} + +void cilksan_do_enter_helper_begin() { + cilksan_assert(CILKSAN_INITIALIZED); + DBG_TRACE(DEBUG_CALLBACK, "frame %ld cilk_enter_helper_begin\n", frame_id+1); + cilksan_assert(last_event == NONE); + WHEN_CILKSAN_DEBUG(last_event = ENTER_HELPER;); + + // entry_stack.push(); + // entry_stack.head()->entry_type = HELPER; + // entry_stack.head()->frame_type = SHADOW_FRAME; + // entry_stack always gets pushed slightly before frame_id gets incremented + // WHEN_CILKSAN_DEBUG(entry_stack.head()->frame_id = frame_id+1;); + // WHEN_CILKSAN_DEBUG(update_deque_for_entering_helper();); + enter_cilk_function(); + frame_stack.head()->frame_data.entry_type = HELPER; + frame_stack.head()->frame_data.frame_type = SHADOW_FRAME; +} + +void cilksan_do_enter_end(uintptr_t stack_ptr) { + cilksan_assert(CILKSAN_INITIALIZED); + FrameData_t *cilk_func = frame_stack.head(); + cilk_func->Sbag->get_node()->set_rsp(stack_ptr); + cilksan_assert(last_event == ENTER_FRAME || last_event == ENTER_HELPER); + WHEN_CILKSAN_DEBUG(last_event = NONE); + DBG_TRACE(DEBUG_CALLBACK, "cilk_enter_end, frame stack ptr: %p\n", stack_ptr); +} + +void cilksan_do_detach_begin() { + cilksan_assert(CILKSAN_INITIALIZED); + cilksan_assert(last_event == NONE); + WHEN_CILKSAN_DEBUG(last_event = DETACH); +} + +void cilksan_do_detach_end() { + cilksan_assert(CILKSAN_INITIALIZED); + DBG_TRACE(DEBUG_CALLBACK, "cilk_detach\n"); + + // cilksan_assert(frame_stack.head()->frame_data.entry_type == HELPER); + cilksan_assert(last_event == DETACH); + WHEN_CILKSAN_DEBUG(last_event = NONE); + + // At this point, the frame_stack.head is still the parent (spawning) frame + FrameData_t *parent = frame_stack.head(); + + DBG_TRACE(DEBUG_CALLBACK, + "frame %ld about to spawn.\n", + parent->Sbag->get_node()->get_func_id()); + + // if (!parent->Pbag) { // lazily create PBag when needed + // DBG_TRACE(DEBUG_BAGS, + // "frame %ld creates a PBag.\n", + // parent->Sbag->get_set_node()->get_func_id()); + // DisjointSet_t *parent_pbag = + // new DisjointSet_t(new PBag_t(parent->Sbag->get_node())); + // parent->set_pbag(parent_pbag); + // } + enter_detach_child(); +} + +void cilksan_do_sync_begin() { + cilksan_assert(CILKSAN_INITIALIZED); + DBG_TRACE(DEBUG_CALLBACK, "frame %ld cilk_sync_begin\n", + frame_stack.head()->Sbag->get_node()->get_func_id()); + cilksan_assert(last_event == NONE); + WHEN_CILKSAN_DEBUG(last_event = CILK_SYNC); +} + +void cilksan_do_sync_end() { + cilksan_assert(CILKSAN_INITIALIZED); + DBG_TRACE(DEBUG_CALLBACK, "cilk_sync_end\n"); + cilksan_assert(last_event == CILK_SYNC); + WHEN_CILKSAN_DEBUG(last_event = NONE); + complete_sync(); +} + +void cilksan_do_leave_begin() { + cilksan_assert(CILKSAN_INITIALIZED); + cilksan_assert(last_event == NONE); + WHEN_CILKSAN_DEBUG(last_event = LEAVE_FRAME_OR_HELPER); + DBG_TRACE(DEBUG_CALLBACK, "frame %ld cilk_leave_begin\n", + frame_stack.head()->frame_data.frame_id); + cilksan_assert(frame_stack.size() > 1); + + switch(frame_stack.head()->frame_data.entry_type) { + case SPAWNER: + DBG_TRACE(DEBUG_CALLBACK, "cilk_leave_frame_begin\n"); + break; + case HELPER: + DBG_TRACE(DEBUG_CALLBACK, "cilk_leave_helper_begin\n"); + break; + case DETACHER: + DBG_TRACE(DEBUG_CALLBACK, "cilk_leave_begin from detach\n"); + break; + } + + if (DETACHER == frame_stack.head()->frame_data.entry_type) + return_from_detach(); + else + leave_cilk_function(); +} + +void cilksan_do_leave_end() { + cilksan_assert(CILKSAN_INITIALIZED); + // DBG_TRACE(DEBUG_CALLBACK, "frame %ld cilk_leave_end\n", + // frame_stack.head()->frame_data.frame_id); + DBG_TRACE(DEBUG_CALLBACK, "cilk_leave_end\n"); + cilksan_assert(last_event == LEAVE_FRAME_OR_HELPER); + WHEN_CILKSAN_DEBUG(last_event = NONE); + // cilksan_assert(frame_stack.size() > 1); +} + +// called by record_memory_read/write, with the access broken down into 64-byte +// aligned memory accesses +static void record_mem_helper(bool is_read, const csi_id_t acc_id, + uintptr_t addr, + size_t mem_size, bool on_stack) { + FrameData_t *f = frame_stack.head(); + bool is_in_shadow_memory = + shadow_memory.does_access_exists(is_read, addr, mem_size); + + if (is_in_shadow_memory == false) + shadow_memory.insert_access(is_read, acc_id, addr, mem_size, f, + call_stack); + else + check_races_and_update(is_read, acc_id, addr, mem_size, on_stack, + f, call_stack, shadow_memory); +} + +void cilksan_do_read(const csi_id_t load_id, + uintptr_t addr, size_t mem_size) { + cilksan_assert(CILKSAN_INITIALIZED); + DBG_TRACE(DEBUG_MEMORY, "record read %lu: %lu bytes at addr %p and rip %p.\n", + load_id, mem_size, addr, load_pc[load_id]); + + // for now we assume the stack doesn't change + bool on_stack = is_on_stack(addr); + if (on_stack) + if (addr < *sp_stack.head()) + *sp_stack.head() = addr; + // handle the prefix + uintptr_t next_addr = ALIGN_BY_NEXT_MAX_GRAIN_SIZE(addr); + size_t prefix_size = next_addr - addr; + cilksan_assert(prefix_size >= 0 && prefix_size < MAX_GRAIN_SIZE); + + if (prefix_size >= mem_size) { // access falls within a max grain sized block + record_mem_helper(true, load_id, addr, mem_size, on_stack); + } else { + cilksan_assert( prefix_size <= mem_size ); + if (prefix_size) { // do the prefix first + record_mem_helper(true, load_id, addr, prefix_size, on_stack); + mem_size -= prefix_size; + } + addr = next_addr; + // then do the rest of the max-grain size aligned blocks + uint32_t i = 0; + for (i = 0; (i + MAX_GRAIN_SIZE) < mem_size; i += MAX_GRAIN_SIZE) { + record_mem_helper(true, load_id, addr + i, MAX_GRAIN_SIZE, + on_stack); + } + // trailing bytes + record_mem_helper(true, load_id, addr+i, mem_size-i, on_stack); + } +} + +void cilksan_do_write(const csi_id_t store_id, + uintptr_t addr, size_t mem_size) { + cilksan_assert(CILKSAN_INITIALIZED); + DBG_TRACE(DEBUG_MEMORY, "record write %ld: %lu bytes at addr %p and rip %p.\n", + store_id, mem_size, addr, store_pc[store_id]); + + bool on_stack = is_on_stack(addr); + if (on_stack) + if (addr < *sp_stack.head()) + *sp_stack.head() = addr; + // handle the prefix + uintptr_t next_addr = ALIGN_BY_NEXT_MAX_GRAIN_SIZE(addr); + size_t prefix_size = next_addr - addr; + cilksan_assert(prefix_size >= 0 && prefix_size < MAX_GRAIN_SIZE); + + if (prefix_size >= mem_size) { // access falls within a max grain sized block + record_mem_helper(false, store_id, addr, mem_size, on_stack); + } else { + cilksan_assert(prefix_size <= mem_size); + if (prefix_size) { // do the prefix first + record_mem_helper(false, store_id, addr, prefix_size, + on_stack); + mem_size -= prefix_size; + } + addr = next_addr; + // then do the rest of the max-grain size aligned blocks + uint32_t i = 0; + for(i = 0; (i + MAX_GRAIN_SIZE) < mem_size; i += MAX_GRAIN_SIZE) { + record_mem_helper(false, store_id, addr + i, MAX_GRAIN_SIZE, + on_stack); + } + // trailing bytes + record_mem_helper(false, store_id, addr+i, mem_size-i, on_stack); + } +} + +// clear the memory block at [start,start+size) (end is exclusive). +void cilksan_clear_shadow_memory(size_t start, size_t size) { + DBG_TRACE(DEBUG_MEMORY, "cilksan_clear_shadow_memory(%p, %ld)\n", + start, size); + shadow_memory.clear(start,size); +} + +static void print_cilksan_stat() { + // std::cout << "max sync block size seen: " + // << accounted_max_sync_block_size + // << " (from user input: " << max_sync_block_size << ", average: " + // << ( (float)accum_sync_block_size / num_of_sync_blocks ) << ")" + // << std::endl; + // std::cout << "max continuation depth seen: " + // << accounted_max_cont_depth << std::endl; +} + +void cilksan_deinit() { + static bool deinit = false; + // XXX: kind of a hack, but somehow this gets called twice. + if (!deinit) deinit = true; + else return; /* deinit-ed already */ + + print_race_report(); + print_cilksan_stat(); + + cilksan_assert(frame_stack.size() == 1); + // cilksan_assert(entry_stack.size() == 1); + + shadow_memory.destruct(); + + // Remove references to the disjoint set nodes so they can be freed. + frame_stack.head()->reset(); + frame_stack.pop(); + + WHEN_CILKSAN_DEBUG({ + if (DisjointSet_t::debug_count != 0) + fprintf(stderr, "DisjointSet_t::debug_count = %ld\n", + DisjointSet_t::debug_count); + if (SBag_t::debug_count != 0) + fprintf(stderr, "SBag_t::debug_count = %ld\n", + SBag_t::debug_count); + if (PBag_t::debug_count != 0) + fprintf(stderr, "PBag_t::debug_count = %ld\n", + PBag_t::debug_count); + // cilksan_assert(DisjointSet_t::debug_count == 0); + // cilksan_assert(SBag_t::debug_count == 0); + // cilksan_assert(PBag_t::debug_count == 0); + }); + + // Free the call-stack nodes in the free list. + call_stack_node_t::cleanup_freelist(); + + // Free the disjoint-set nodes in the free list. + DisjointSet_t::cleanup_freelist(); + + // Free the free lists for SBags and PBags. + SBag_t::cleanup_freelist(); + PBag_t::cleanup_freelist(); + + disjoint_set_list.free_list(); + + // if(first_error != 0) exit(first_error); +} + +void cilksan_init() { + DBG_TRACE(DEBUG_CALLBACK, "cilksan_init()\n"); + std::cout<< "cilksan_init() version 19\n"; + + cilksan_assert(stack_high_addr != 0 && stack_low_addr != 0); + + // these are true upon creation of the stack + cilksan_assert(frame_stack.size() == 1); + // cilksan_assert(entry_stack.size() == 1); + // // actually only used for debugging of reducer race detection + // WHEN_CILKSAN_DEBUG(rts_deque_begin = rts_deque_end = 1); + + shadow_memory.init(); + + // for the main function before we enter the first Cilk context + DisjointSet_t *sbag; + sbag = new DisjointSet_t(new SBag_t(frame_id, NULL)); + cilksan_assert(sbag->get_set_node()->is_SBag()); + frame_stack.head()->set_sbag(sbag); + WHEN_CILKSAN_DEBUG(frame_stack.head()->frame_data.frame_type = FULL_FRAME); + +#if CILKSAN_DEBUG + CILKSAN_INITIALIZED = true; +#endif +} + +extern "C" int __cilksan_error_count() { + return get_num_races_found(); +} + +// This funciton parse the input supplied to the user program and get the params +// meant for cilksan (everything after "--"). It return the index in which it +// found "--" so the user program knows when to stop parsing inputs. +extern "C" int __cilksan_parse_input(int argc, char *argv[]) { + int i = 0; + // uint32_t seed = 0; + int stop = 0; + + while(i < argc) { + if(!strncmp(argv[i], "--", strlen("--")+1)) { + stop = i++; + break; + } + i++; + } + + while(i < argc) { + char *arg = argv[i]; + // if(!strncmp(arg, "-cr", strlen("-cr")+1)) { + // i++; + // check_reduce = true; + // continue; + + // } else if(!strncmp(arg, "-update", strlen("-update")+1)) { + // i++; + // cont_depth_to_check = (uint64_t) atol(argv[i++]); + // continue; + + // } else if(!strncmp(arg, "-sb_size", strlen("-sb_size")+1)) { + // i++; + // max_sync_block_size = (uint32_t) atoi(argv[i++]); + // continue; + + // } else if(!strncmp(arg, "-s", strlen("-s")+1)) { + // i++; + // seed = (uint32_t) atoi(argv[i++]); + // continue; + + // } else if(!strncmp(arg, "-steal", strlen("-steal")+1)) { + // i++; + // cilksan_assert(steal_point1 < steal_point2 + // && steal_point2 < steal_point3); + // check_reduce = true; + // continue; + + // } else { + i++; + std::cout << "Unrecognized input " << arg << ", ignore and continue." + << std::endl; + // } + } + + // std::cout << "===============================================================" + // << std::endl; + // if(cont_depth_to_check != 0) { + // check_reduce = false; + // std::cout << "This run will check updates for races with " << std::endl + // << "steals at continuation depth " << cont_depth_to_check; + // } else if(check_reduce) { + // std::cout << "This run will check reduce functions for races " << std::endl + // << "with simulated steals "; + // if(max_sync_block_size > 1) { + // std::cout << "at randomly chosen continuation points \n" + // << "(assume block size " + // << max_sync_block_size << ")"; + // if(seed) { + // std::cout << ", chosen using seed " << seed; + // srand(seed); + // } else { + // // srand(time(NULL)); + // } + // } else { + // if(steal_point1 != steal_point2 && steal_point2 != steal_point3) { + // std::cout << "at steal points: " << steal_point1 << ", " + // << steal_point2 << ", " << steal_point3 << "."; + // } else { + // simulate_all_steals = true; + // check_reduce = false; + // std::cout << "at every continuation point."; + // } + // } + // } else { + // // cont_depth_to_check == 0 and check_reduce = false + // std::cout << "This run will check for races without simulated steals."; + // } + // std::cout << std::endl; + // std::cout << "===============================================================" + // << std::endl; + + // cilksan_assert(!check_reduce || cont_depth_to_check == 0); + // cilksan_assert(!check_reduce || max_sync_block_size > 1 || steal_point1 != steal_point2); + + return (stop == 0 ? argc : stop); +} + +// XXX: Should really be in print_addr.cpp, but this will do for now +void print_current_function_info() { + FrameData_t *f = frame_stack.head(); + // std::cout << "steal points: " << f->steal_points[0] << ", " + // << f->steal_points[1] << ", " << f->steal_points[2] << std::endl; + // std::cout << "curr sync block size: " << f->current_sync_block_size << std::endl; + std::cout << "frame id: " << f->Sbag->get_node()->get_func_id() << std::endl; +} diff --git a/compiler-rt/lib/cilksan/cilksan.h b/compiler-rt/lib/cilksan/cilksan.h new file mode 100644 index 000000000000..0c9501245b3e --- /dev/null +++ b/compiler-rt/lib/cilksan/cilksan.h @@ -0,0 +1,25 @@ +#ifndef __CILKSAN_H__ +#define __CILKSAN_H__ + +#if defined (__cplusplus) +extern "C" { +#endif + +int __cilksan_error_count(void); +void __cilksan_enable_checking(void); +void __cilksan_disable_checking(void); +void __cilksan_begin_reduce_strand(void); +void __cilksan_end_reduce_strand(void); +void __cilksan_begin_update_strand(void); +void __cilksan_end_update_strand(void); + +// This funciton parse the input supplied to the user program and get the params +// meant for cilksan (everything after "--"). It return the index in which it +// found "--" so the user program knows when to stop parsing inputs. +int __cilksan_parse_input(int argc, char *argv[]); + +#if defined (__cplusplus) +} +#endif + +#endif // __CILKSAN_H__ diff --git a/compiler-rt/lib/cilksan/cilksan_internal.h b/compiler-rt/lib/cilksan/cilksan_internal.h new file mode 100644 index 000000000000..ad9158a89b90 --- /dev/null +++ b/compiler-rt/lib/cilksan/cilksan_internal.h @@ -0,0 +1,467 @@ +// -*- C++ -*- +#ifndef __CILKSAN_INTERNAL_H__ +#define __CILKSAN_INTERNAL_H__ + +#include +#include +#include +#include + +#include "csan.h" + +#define UNINIT_STACK_PTR ((uintptr_t)0LL) +#define UNINIT_VIEW_ID ((uint64_t)0LL) + +#include "cilksan.h" +#include "stack.h" + +#define BT_OFFSET 1 +#define BT_DEPTH 2 + +// The context in which the access is made; user = user code, update = update +// methods for a reducer object; reduce = reduce method for a reducer object +enum AccContextType_t { USER = 1, UPDATE = 2, REDUCE = 3 }; +// W stands for write, R stands for read +enum RaceType_t { RW_RACE = 1, WW_RACE = 2, WR_RACE = 3 }; + +enum CallType_t { CALL, SPAWN }; +// typedef std::pair CallID_t; +// typedef Stack_t call_stack_t; +struct CallID_t { + csi_id_t id; + CallType_t type; + CallID_t() : id(UNKNOWN_CSI_ID), type(CALL) {} + + CallID_t(CallType_t type, csi_id_t id) + : id(id), type(type) + { + // if (SPAWN == type) + // // Assumes UNKNOWN_CSI_ID == -1 + // id = -id - 3; + } + + CallID_t(const CallID_t ©) : id(copy.id), type(copy.type) {} + + CallID_t &operator=(const CallID_t ©) { + id = copy.id; + type = copy.type; + return *this; + } + + CallType_t getType() const { + // if (id < -1) + // return SPAWN; + // return CALL; + return type; + } + + csi_id_t getID() const { + // if (id < -1) + // // Assumes UNKNOWN_CSI_ID == -1 + // return -id - 3; + return id; + } + + bool operator==(const CallID_t &that) { + return (id == that.id) && (type == that.type); + } + + bool operator!=(const CallID_t &that) { + return !(*this == that); + } + + inline friend + std::ostream& operator<<(std::ostream &os, const CallID_t &id) { + switch (id.type) { + case CALL: + os << "CALL " << id.id; + break; + case SPAWN: + os << "SPAWN " << id.id; + break; + } + return os; + } +}; + +struct call_stack_node_t { + CallID_t id; + // std::shared_ptr prev; + call_stack_node_t *prev; + int64_t ref_count; + call_stack_node_t(CallID_t id, + call_stack_node_t *prev = nullptr) + // std::shared_ptr prev = nullptr) + // Set ref_count to 1, under the assumption that only call_stack_t + // constructs nodes and will immediately set the tail pointer to point to + // this node. + : id(id), prev(prev), ref_count(1) { + if (prev) + prev->ref_count++; + } + + ~call_stack_node_t() { + // std::cerr << "call_stack_node_t destructing"; + // if (prev) + // std::cerr << "(PREV " << prev->id << " REF_COUNT " << prev->ref_count << ")"; + // std::cerr << std::endl; + assert(!ref_count); + // prev.reset(); + if (prev) { + call_stack_node_t *old_prev = prev; + prev = nullptr; + assert(old_prev->ref_count > 0); + old_prev->ref_count--; + if (!old_prev->ref_count) + delete old_prev; + } + } + + // Simple free-list allocator to conserve space and time in managing + // call_stack_node_t objects. + static call_stack_node_t *free_list; + + void *operator new(size_t size) { + if (free_list) { + call_stack_node_t *new_node = free_list; + free_list = free_list->prev; + return new_node; + } + return ::operator new(size); + } + + void operator delete(void *ptr) { + call_stack_node_t *del_node = reinterpret_cast(ptr); + del_node->prev = free_list; + free_list = del_node; + } + + static void cleanup_freelist() { + call_stack_node_t *node = free_list; + call_stack_node_t *next = nullptr; + while (node) { + next = node->prev; + ::operator delete(node); + node = next; + } + } +}; + +struct call_stack_t { + // int size; + // std::shared_ptr tail; + call_stack_node_t *tail; + call_stack_t() : // size(0), + tail(nullptr) {} + + call_stack_t(const call_stack_t ©) + : // size(copy.size), + tail(copy.tail) + { + if (tail) + tail->ref_count++; + } + + ~call_stack_t() { + // std::cerr << "call_stack_t destructing"; + // if (tail) + // std::cerr << " (TAIL " << tail->id << " REF_COUNT " << tail->ref_count << ")"; + // std::cerr << std::endl; + // tail.reset(); + if (tail) { + call_stack_node_t *old_tail = tail; + tail = nullptr; + assert(old_tail->ref_count > 0); + old_tail->ref_count--; + if (!old_tail->ref_count) + delete old_tail; + } + } + + call_stack_t &operator=(const call_stack_t ©) { + // size = copy.size; + call_stack_node_t *old_tail = tail; + tail = copy.tail; + if (tail) + tail->ref_count++; + if (old_tail) { + old_tail->ref_count--; + if (!old_tail->ref_count) + delete old_tail; + } + return *this; + } + + call_stack_t &operator=(const call_stack_t &&move) { + // size = copy.size; + if (tail) { + tail->ref_count--; + if (!tail->ref_count) + delete tail; + } + tail = move.tail; + return *this; + } + + void overwrite(const call_stack_t ©) { + tail = copy.tail; + } + + void push(CallID_t id) { + // size++; + // tail = std::make_shared(id, tail); + call_stack_node_t *new_node = new call_stack_node_t(id, tail); + // new_node has ref_count 1 and, if tail was not null, has incremented + // tail's ref count. + tail = new_node; + if (tail->prev) { + assert(tail->prev->ref_count > 1); + tail->prev->ref_count--; + } + // now the ref_count's should reflect the pointer structure. + } + + void pop() { + assert(tail); + // size--; + call_stack_node_t *old_node = tail; + tail = tail->prev; + if (tail) + tail->ref_count++; + assert(old_node->ref_count > 0); + old_node->ref_count--; + if (!old_node->ref_count) + // Deleting the old node will decrement tail's ref count. + delete old_node; + } + + int size() const { + call_stack_node_t *node = tail; + int size = 0; + while (node) { + ++size; + node = node->prev; + } + return size; + } +}; + +class AccessLoc_t { +public: + csi_id_t acc_loc; + // std::vector call_stack; + // const std::shared_ptr call_stack; + call_stack_t call_stack; + // int64_t ref_count; + + AccessLoc_t() : acc_loc(), call_stack()// , ref_count(1) + {} + + AccessLoc_t(csi_id_t _acc_loc, const call_stack_t &_call_stack) + : acc_loc(_acc_loc), call_stack(_call_stack)// , ref_count(1) + // call_stack(_call_stack.size()-1) + { + // std::cerr << "AccessLoc_t constructing"; + // if (call_stack.tail) + // std::cerr << " (TAIL " << call_stack.tail->id << " REF_COUNT " << call_stack.tail->ref_count << ")"; + // std::cerr << std::endl; + // for (int i = 0; i < _call_stack.size()-1; ++i) + // call_stack[i] = *_call_stack.at(i); + } + + AccessLoc_t(const AccessLoc_t ©) + : acc_loc(copy.acc_loc), call_stack(copy.call_stack)// , ref_count(1) + { + // std::cerr << "AccessLoc_t copy-constructing"; + // if (call_stack.tail) + // std::cerr << " (TAIL " << call_stack.tail->id << " REF_COUNT " << call_stack.tail->ref_count << ")"; + // std::cerr << std::endl; + } + + AccessLoc_t(const AccessLoc_t &&move) + : acc_loc(move.acc_loc)// , ref_count(1) + { + call_stack.overwrite(move.call_stack); + // std::cerr << "AccessLoc_t copy-constructing"; + // if (call_stack.tail) + // std::cerr << " (TAIL " << call_stack.tail->id << " REF_COUNT " << call_stack.tail->ref_count << ")"; + // std::cerr << std::endl; + } + + ~AccessLoc_t() { + // assert(ref_count <= 1); + // std::cerr << "AccessLoc_t destructing"; + // if (call_stack.tail) + // std::cerr << " (TAIL " << call_stack.tail->id << " REF_COUNT " << call_stack.tail->ref_count << ")"; + // std::cerr << std::endl; + } + + csi_id_t getID() const { + return acc_loc; + } + + AccessLoc_t& operator=(const AccessLoc_t ©) { + acc_loc = copy.acc_loc; + call_stack = copy.call_stack; + return *this; + } + + AccessLoc_t& operator=(const AccessLoc_t &&move) { + acc_loc = move.acc_loc; + call_stack = std::move(move.call_stack); + return *this; + } + + int64_t inc_ref_count(int64_t count = 1) { + // ref_count += count; + // return ref_count; + if (!call_stack.tail) + return 0; + call_stack.tail->ref_count += count; + return call_stack.tail->ref_count; + } + + int64_t dec_ref_count(int64_t count = 1) { + // assert(ref_count >= count); + // ref_count -= count; + // if (!ref_count) { + // delete this; + // return 0; + // } + // return ref_count; + if (!call_stack.tail) + return 0; + assert(call_stack.tail->ref_count >= count); + call_stack.tail->ref_count -= count; + if (!call_stack.tail->ref_count) { + delete call_stack.tail; + call_stack.tail = nullptr; + return 0; + } + return call_stack.tail->ref_count; + } + + bool operator==(const AccessLoc_t &that) const { + if (acc_loc != that.acc_loc) + return false; + + call_stack_node_t *this_node = call_stack.tail; + call_stack_node_t *that_node = that.call_stack.tail; + while (this_node && that_node) { + if (this_node->id != that_node->id) + return false; + this_node = this_node->prev; + that_node = that_node->prev; + } + if (this_node || that_node) + return false; + return true; + } + + // Unsafe method! Only use this if you know what you're doing. + void overwrite(const AccessLoc_t ©) { + acc_loc = copy.acc_loc; + call_stack.overwrite(copy.call_stack); + } + + // Unsafe method! Only use this if you know what you're doing. + void clear() { + call_stack.tail = nullptr; + } + + // bool hasValidLoc() const { + // return nullptr != call_stack.tail; + // } + + bool operator!=(const AccessLoc_t &that) const { + return !(*this == that); + } + + bool operator<(const AccessLoc_t &that) const { + return acc_loc < that.acc_loc; + } + + inline friend + std::ostream& operator<<(std::ostream &os, const AccessLoc_t &loc) { + os << loc.acc_loc; + // std::shared_ptr node(loc.call_stack.tail); + call_stack_node_t *node = loc.call_stack.tail; + while (node) { + switch (node->id.getType()) { + case CALL: + os << " CALL"; + break; + case SPAWN: + os << " SPAWN"; + break; + } + os << " " << std::dec << node->id.getID(); + node = node->prev; + } + return os; + } +}; + +typedef struct RaceInfo_t { + const AccessLoc_t first_inst; // instruction addr of the first access + const AccessLoc_t second_inst; // instruction addr of the second access + uintptr_t addr; // addr of memory location that got raced on + enum RaceType_t type; // type of race + + RaceInfo_t(const AccessLoc_t &_first, AccessLoc_t &&_second, + uintptr_t _addr, enum RaceType_t _type) + : first_inst(_first), second_inst(_second), addr(_addr), + type(_type) + { + // std::cerr << "RaceInfo_t constructing (" << first_inst << ", " << second_inst << ")\n"; + } + + ~RaceInfo_t() { + // std::cerr << "RaceInfo_t destructing (" << first_inst << ", " << second_inst << ")\n"; + } + + bool is_equivalent_race(const struct RaceInfo_t& other) const { + /* + if( (type == other.type && + first_inst == other.first_inst && second_inst == other.second_inst) || + (first_inst == other.second_inst && second_inst == other.first_inst && + ((type == RW_RACE && other.type == WR_RACE) || + (type == WR_RACE && other.type == RW_RACE))) ) { + return true; + } */ + // Angelina: It turns out that, Cilkscreen does not care about the race + // types. As long as the access instructions are the same, it's considered + // as a duplicate. + if ((first_inst == other.first_inst && second_inst == other.second_inst) || + (first_inst == other.second_inst && second_inst == other.first_inst)) { + return true; + } + return false; + } +} RaceInfo_t; + +// defined in print_addr.cpp +void report_race(const AccessLoc_t &first_inst, AccessLoc_t &&second_inst, + uintptr_t addr, enum RaceType_t race_type); + +// public functions +void cilksan_init(); +void cilksan_deinit(); +void cilksan_do_enter_begin(); +void cilksan_do_enter_helper_begin(); +void cilksan_do_enter_end(uintptr_t stack_ptr); +void cilksan_do_detach_begin(); +void cilksan_do_detach_end(); +void cilksan_do_sync_begin(); +void cilksan_do_sync_end(); +void cilksan_do_return(); +void cilksan_do_leave_begin(); +void cilksan_do_leave_end(); +void cilksan_do_leave_stolen_callback(); + +void cilksan_do_read(const csi_id_t load_id, uintptr_t addr, size_t len); +void cilksan_do_write(const csi_id_t store_id, uintptr_t addr, size_t len); +void cilksan_clear_shadow_memory(size_t start, size_t end); +// void cilksan_do_function_entry(uint64_t an_address); +// void cilksan_do_function_exit(); +#endif // __CILKSAN_INTERNAL_H__ diff --git a/compiler-rt/lib/cilksan/compresseddict_shadow_mem.cpp b/compiler-rt/lib/cilksan/compresseddict_shadow_mem.cpp new file mode 100644 index 000000000000..b911c0fd9caf --- /dev/null +++ b/compiler-rt/lib/cilksan/compresseddict_shadow_mem.cpp @@ -0,0 +1,322 @@ +#include + +#include "compresseddict_shadow_mem.h" + +size_t MemoryAccess_t::FreeNode_t::FreeNode_ObjSize = 0; +MemoryAccess_t::FreeNode_t *MemoryAccess_t::free_list = nullptr; + +CompressedDictShadowMem::CompressedDictShadowMem() { + int dict_type = 2; + // element_count = 0; + switch (dict_type) { + // case 0: + // my_read_dict = new Run_Dictionary(); + // my_write_dict = new Run_Dictionary(); + // break; + // case 1: + // my_read_dict = new LZ_Dictionary(); + // my_write_dict = new LZ_Dictionary(); + // break; + case 2: + my_read_dict = new Static_Dictionary(); + my_write_dict = new Static_Dictionary(); + break; + } +} + +void CompressedDictShadowMem::insert_access(bool is_read, const csi_id_t acc_id, + uintptr_t addr, + size_t mem_size, FrameData_t *f, + const call_stack_t &call_stack) { + // element_count += mem_size; + uint64_t key = addr; + // std::cerr << __PRETTY_FUNCTION__ << " key = 0x" << std::hex << key << "\n"; + if (is_read) { + // my_read_dict->erase(key, mem_size); + // my_read_dict->insert(key, mem_size, + // MemoryAccess_t(f->Sbag, // inst_addr, + // acc_id, call_stack)); + my_read_dict->set(key, mem_size, + MemoryAccess_t(f->Sbag, acc_id, call_stack)); + } else { + // my_write_dict->erase(key, mem_size); + // my_write_dict->insert(key, mem_size, + // MemoryAccess_t(f->Sbag, // inst_addr, + // acc_id, call_stack)); + my_write_dict->set(key, mem_size, + MemoryAccess_t(f->Sbag, acc_id, call_stack)); + } +} + +void CompressedDictShadowMem::insert_access_into_group(bool is_read, const csi_id_t acc_id, + uintptr_t addr, + size_t mem_size, FrameData_t *f, + const call_stack_t &call_stack, + value_type00 *dst) { + // element_count += mem_size; + uint64_t key = addr; + // std::cerr << __PRETTY_FUNCTION__ << " key = 0x" << std::hex << key << "\n"; + if (is_read) { + // my_read_dict->erase(key, mem_size); + my_read_dict->insert_into_found_group(key, mem_size, dst, + MemoryAccess_t(f->Sbag, acc_id, + call_stack)); + } else { + // my_write_dict->erase(key, mem_size); + my_write_dict->insert_into_found_group(key, mem_size, dst, + MemoryAccess_t(f->Sbag, acc_id, + call_stack)); + } +} + +// given an address in the (hash) shadow memory, return the mem-access object +// for that address (or null if the address is not in the shadow memory) +value_type00 *CompressedDictShadowMem::find(bool is_read, uintptr_t addr) { + if (is_read) + return my_read_dict->find(addr); + else + return my_write_dict->find(addr); +} + +value_type00 *CompressedDictShadowMem::find_group(bool is_read, uintptr_t addr, + size_t max_size, + size_t &num_elems) { + if (is_read) + return my_read_dict->find_group(addr, max_size, num_elems); + else + return my_write_dict->find_group(addr, max_size, num_elems); +} + +value_type00 *CompressedDictShadowMem::find_exact_group(bool is_read, + uintptr_t addr, + size_t max_size, + size_t &num_elems) { + if (is_read) + return my_read_dict->find_exact_group(addr, max_size, num_elems); + else + return my_write_dict->find_exact_group(addr, max_size, num_elems); +} + +bool CompressedDictShadowMem::does_access_exists(bool is_read, uintptr_t addr, + size_t mem_size) { + if (is_read) + return my_read_dict->includes(addr, mem_size); + else + return my_write_dict->includes(addr, mem_size); +} + +void CompressedDictShadowMem::clear(size_t start, size_t size) { + // my_read_dict->erase(start, end - start); + // my_write_dict->erase(start, end - start); + my_read_dict->erase(start, size); + my_write_dict->erase(start, size); + //while(start != end) { + // uint64_t key = start; + // my_read_dict->erase(key); + // my_write_dict->erase(key); + // + // start++; + //} +} + +// prev_read: are we checking with previous reads or writes? +// is_read: is the current access read or write? +void CompressedDictShadowMem::check_race(bool prev_read, bool is_read, + const csi_id_t acc_id, + uintptr_t addr, + size_t mem_size, bool on_stack, + FrameData_t *f, + const call_stack_t &call_stack) { + // std::cerr << __PRETTY_FUNCTION__ << " prev_read " << prev_read << " is_read " << is_read << "\n"; + size_t index = 0; + while (index < mem_size) { + uintptr_t shifted_addr = addr + index; + size_t current_size = 0; + value_type00 *access = find_group(prev_read, shifted_addr, mem_size - index, + current_size); + assert(current_size > 0); + assert(shifted_addr + current_size <= addr + mem_size); + if (access && access->isValid()) { + auto func = access->getFunc(); + bool has_race = false; + cilksan_assert(func); + // cilksan_assert(curr_vid != UNINIT_VIEW_ID); + + SPBagInterface *lca = func->get_set_node(); + // SPBagInterface *cur_node = func->get_node(); + if (lca->is_PBag()) { + // If memory is allocated on stack, the accesses race with each other + // only if the mem location is allocated in shared ancestor's stack + + // if stack_check = false, there is no race. + // if stack_check = true, it's a race only if all other conditions apply. + //bool stack_check = (!on_stack || addr >= cur_node->get_rsp()); + bool stack_check = (!on_stack || shifted_addr >= lca->get_rsp()); + has_race = stack_check; + } + if (has_race) { + if (prev_read) // checking the current access with previous reads + report_race(access->getLoc(), + AccessLoc_t(acc_id, call_stack), + shifted_addr, RW_RACE); + else { // check the current access with previous writes + if (is_read) // the current access is a read + report_race(access->getLoc(), + AccessLoc_t(acc_id, call_stack), + shifted_addr, WR_RACE); + else + report_race(access->getLoc(), + AccessLoc_t(acc_id, call_stack), + shifted_addr, WW_RACE); + } + } + } + index += current_size; + } +} + +void CompressedDictShadowMem::check_race_with_prev_read(const csi_id_t acc_id, + uintptr_t addr, + size_t mem_size, + bool on_stack, + FrameData_t *f, + const call_stack_t &call_stack) { + // the second argument does not matter here + check_race(true, false, acc_id, addr, mem_size, on_stack, f, call_stack); +} + +void CompressedDictShadowMem::check_race_with_prev_write(bool is_read, + const csi_id_t acc_id, + uintptr_t addr, + size_t mem_size, + bool on_stack, + FrameData_t *f, + const call_stack_t &call_stack) { + check_race(false, is_read, acc_id, addr, mem_size, on_stack, f, call_stack); +} + +void CompressedDictShadowMem::update(bool with_read, const csi_id_t acc_id, + uintptr_t addr, + size_t mem_size, bool on_stack, + FrameData_t *f, + const call_stack_t &call_stack) { + size_t index = 0; + while (index < mem_size) { + uintptr_t shifted_addr = addr + index; + size_t current_size = 0; + value_type00 *access = find_exact_group(with_read, shifted_addr, + mem_size - index, current_size); + assert(current_size > 0); + assert(shifted_addr + current_size <= addr + mem_size); + if (!access || !access->isValid()) + insert_access_into_group(with_read, acc_id, shifted_addr, + current_size, f, call_stack, access); + else { + auto func = access->getFunc(); + SPBagInterface *last_rset = func->get_set_node(); + // replace it only if it is in series with this access, i.e., if it's + // one of the following: + // a) in a SBag + // b) in a PBag but should have been replaced because the access is + // actually on the newly allocated stack frame (i.e., cactus stack abstraction) + // // c) access is made by a REDUCE strand and previous access is in the + // // top-most PBag. + if (last_rset->is_SBag() || + (on_stack && last_rset->get_rsp() >= shifted_addr)) { + // func->dec_ref_count(current_size); + // access->dec_AccessLoc_ref_count(current_size); + access->dec_ref_counts(current_size); + // note that ref count is decremented regardless + insert_access_into_group(with_read, acc_id, shifted_addr, + current_size, f, call_stack, access); + } + } + index += current_size; + } +} + +void CompressedDictShadowMem::update_with_write(const csi_id_t acc_id, + uintptr_t addr, size_t mem_size, + bool on_stack, FrameData_t *f, + const call_stack_t &call_stack) { + update(false, acc_id, addr, mem_size, on_stack, f, call_stack); +} + +void CompressedDictShadowMem::update_with_read(const csi_id_t acc_id, + uintptr_t addr, size_t mem_size, + bool on_stack, FrameData_t *f, + const call_stack_t &call_stack) { + update(true, acc_id, addr, mem_size, on_stack, f, call_stack); +} + +void CompressedDictShadowMem::check_and_update_write( + const csi_id_t acc_id, uintptr_t addr, size_t mem_size, + bool on_stack, FrameData_t *f, const call_stack_t &call_stack) { + size_t index = 0; + while (index < mem_size) { + uintptr_t shifted_addr = addr + index; + size_t current_size = 0; + value_type00 *access = find_exact_group(false, shifted_addr, + mem_size - index, current_size); + assert(current_size > 0); + assert(shifted_addr + current_size <= addr + mem_size); + if (!access || !access->isValid()) + insert_access_into_group(false, acc_id, shifted_addr, + current_size, f, call_stack, access); + else { + auto func = access->getFunc(); + bool has_race = false; + cilksan_assert(func); + // cilksan_assert(curr_vid != UNINIT_VIEW_ID); + SPBagInterface *lca = func->get_set_node(); + + // Check for races + + // SPBagInterface *cur_node = func->get_node(); + if (lca->is_PBag()) { + // If memory is allocated on stack, the accesses race with each other + // only if the mem location is allocated in shared ancestor's stack + + // if stack_check = false, there is no race. + // if stack_check = true, it's a race only if all other conditions apply. + //bool stack_check = (!on_stack || addr >= cur_node->get_rsp()); + bool stack_check = (!on_stack || shifted_addr >= lca->get_rsp()); + has_race = stack_check; + } + if (has_race) { + // check the current access with previous writes + report_race(access->getLoc(), + AccessLoc_t(acc_id, call_stack), + shifted_addr, WW_RACE); + } + + // Update the table + + // replace it only if it is in series with this access, i.e., if it's + // one of the following: + // a) in a SBag + // b) in a PBag but should have been replaced because the access is + // actually on the newly allocated stack frame (i.e., cactus stack abstraction) + // // c) access is made by a REDUCE strand and previous access is in the + // // top-most PBag. + if (lca->is_SBag() || + (on_stack && lca->get_rsp() >= shifted_addr)) { + // func->dec_ref_count(current_size); + // access->dec_AccessLoc_ref_count(current_size); + access->dec_ref_counts(current_size); + // note that ref count is decremented regardless + insert_access_into_group(false, acc_id, shifted_addr, + current_size, f, call_stack, access); + } + } + index += current_size; + } +} + +void CompressedDictShadowMem::destruct() { + //my_read_dict->destruct(); + //my_write_dict->destruct(); + delete my_read_dict; + delete my_write_dict; + MemoryAccess_t::cleanup_freelist(); +} diff --git a/compiler-rt/lib/cilksan/compresseddict_shadow_mem.h b/compiler-rt/lib/cilksan/compresseddict_shadow_mem.h new file mode 100644 index 000000000000..9c0d6e1f5a9c --- /dev/null +++ b/compiler-rt/lib/cilksan/compresseddict_shadow_mem.h @@ -0,0 +1,86 @@ +// -*- C++ -*- +#ifndef __COMPRESSEDDICT_SHADOW_MEM__ +#define __COMPRESSEDDICT_SHADOW_MEM__ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "cilksan_internal.h" +#include "debug_util.h" +#include "disjointset.h" +#include "frame_data.h" +#include "mem_access.h" +#include "shadow_mem.h" +#include "spbag.h" +#include "stack.h" +// #include "run_dictionary.h" +// #include "lz_dictionary.h" +#include "static_dictionary.h" + +class CompressedDictShadowMem : public ShadowMemoryType { + +private: + // address, function + typedef std::map::iterator ShadowMemIter_t; + + Dictionary *my_read_dict; + Dictionary *my_write_dict; + + value_type00 *find(bool is_read, uintptr_t addr); + value_type00 *find_group(bool is_read, uintptr_t addr, size_t max_size, + size_t &num_elems); + value_type00 *find_exact_group(bool is_read, uintptr_t addr, + size_t max_size, size_t &num_elems); + // int element_count; + void update(bool with_read, const csi_id_t acc_id, + uintptr_t addr, size_t mem_size, bool on_stack, FrameData_t *f, + const call_stack_t &call_stack); + void check_race(bool prev_read, bool is_read, const csi_id_t acc_id, + uintptr_t addr, size_t mem_size, + bool on_stack, FrameData_t *f, + const call_stack_t &call_stack); + +public: + CompressedDictShadowMem(); + ~CompressedDictShadowMem() {destruct();} + void insert_access(bool is_read, const csi_id_t acc_id, + uintptr_t addr, size_t mem_size, FrameData_t *f, + const call_stack_t &call_stack); + void insert_access_into_group(bool is_read, const csi_id_t acc_id, + uintptr_t addr, + size_t mem_size, FrameData_t *f, + const call_stack_t &call_stack, + value_type00 *dst); + bool does_access_exists(bool is_read, uintptr_t addr, size_t mem_size); + void clear(size_t start, size_t size); + void check_race_with_prev_read(const csi_id_t acc_id, + uintptr_t addr, size_t mem_size, bool on_stack, + FrameData_t *f, + const call_stack_t &call_stack); + void check_race_with_prev_write(bool is_read, const csi_id_t acc_id, + uintptr_t addr, + size_t mem_size, bool on_stack, + FrameData_t *f, + const call_stack_t &call_stack); + void update_with_write(const csi_id_t acc_id, + uintptr_t addr, size_t mem_size, bool on_stack, + FrameData_t *f, const call_stack_t &call_stack); + void update_with_read(const csi_id_t acc_id, + uintptr_t addr, size_t mem_size, bool on_stack, + FrameData_t *f, const call_stack_t &call_stack); + void check_and_update_write(const csi_id_t acc_id, + uintptr_t addr, size_t mem_size, bool on_stack, + FrameData_t *f, const call_stack_t &call_stack); + void destruct(); + +}; + +#endif // __COMPRESSEDDICT_SHADOW_MEM__ diff --git a/compiler-rt/lib/cilksan/csan.h b/compiler-rt/lib/cilksan/csan.h new file mode 100644 index 000000000000..34ca17e02e14 --- /dev/null +++ b/compiler-rt/lib/cilksan/csan.h @@ -0,0 +1,55 @@ +// -*- C++ -*- +#ifndef __CSAN_H__ +#define __CSAN_H__ + +#include + +#include + +typedef struct { + csi_id_t num_func; + csi_id_t num_func_exit; + csi_id_t num_bb; + csi_id_t num_call; + csi_id_t num_load; + csi_id_t num_store; + csi_id_t num_detach; + csi_id_t num_task; + csi_id_t num_task_exit; + csi_id_t num_detach_continue; + csi_id_t num_sync; +} csan_instrumentation_counts_t; + +struct csan_source_loc_t { + char *name; + int32_t line_number; + int32_t column_number; + char *filename; +}; + +struct obj_source_loc_t { + char *name; + int32_t line_number; + char *filename; +}; + +EXTERN_C + +const csan_source_loc_t *__csan_get_func_source_loc(const csi_id_t func_id); +const csan_source_loc_t *__csan_get_func_exit_source_loc(const csi_id_t func_exit_id); +const csan_source_loc_t *__csan_get_bb_source_loc(const csi_id_t bb_id); +const csan_source_loc_t *__csan_get_call_source_loc(const csi_id_t call_id); +const csan_source_loc_t *__csan_get_load_source_loc(const csi_id_t load_id); +const csan_source_loc_t *__csan_get_store_source_loc(const csi_id_t store_id); +const csan_source_loc_t *__csan_get_detach_source_loc(const csi_id_t detach_id); +const csan_source_loc_t *__csan_get_task_source_loc(const csi_id_t task_id); +const csan_source_loc_t *__csan_get_task_exit_source_loc(const csi_id_t task_exit_id); +const csan_source_loc_t *__csan_get_detach_continue_source_loc(const csi_id_t detach_continue_id); +const csan_source_loc_t *__csan_get_sync_source_loc(const csi_id_t sync_id); + +const obj_source_loc_t *__csan_get_load_obj_source_loc(const csi_id_t load_id); +const obj_source_loc_t *__csan_get_store_obj_source_loc(const csi_id_t store_id); + +EXTERN_C_END + +#endif // __CSAN_H__ diff --git a/compiler-rt/lib/cilksan/csanrt.cpp b/compiler-rt/lib/cilksan/csanrt.cpp new file mode 100644 index 000000000000..4dbeca310bf1 --- /dev/null +++ b/compiler-rt/lib/cilksan/csanrt.cpp @@ -0,0 +1,359 @@ +#include +#include +#include "csan.h" + +#define CSIRT_API __attribute__((visibility("default"))) + +// ------------------------------------------------------------------------ +// Front end data (FED) table structures. +// ------------------------------------------------------------------------ + +// A FED table is a flat list of FED entries, indexed by a CSI +// ID. Each FED table has its own private ID space. +typedef struct { + int64_t num_entries; + csan_source_loc_t *entries; +} fed_table_t; + +// Types of FED tables that we maintain across all units. +typedef enum { + FED_TYPE_FUNCTIONS, + FED_TYPE_FUNCTION_EXIT, + FED_TYPE_BASICBLOCK, + FED_TYPE_CALLSITE, + FED_TYPE_LOAD, + FED_TYPE_STORE, + FED_TYPE_DETACH, + FED_TYPE_TASK, + FED_TYPE_TASK_EXIT, + FED_TYPE_DETACH_CONTINUE, + FED_TYPE_SYNC, + NUM_FED_TYPES // Must be last +} fed_type_t; + +// An object table is a flat list of object source-location entries indexed by +// CSI ID. +typedef struct { + int64_t num_entries; + obj_source_loc_t *entries; +} obj_table_t; + +// Types of object tables that we maintain across all units. +typedef enum { + OBJ_TYPE_LOAD, + OBJ_TYPE_STORE, + NUM_OBJ_TYPES // Must be last +} obj_type_t; + +// ------------------------------------------------------------------------ +// Globals +// ------------------------------------------------------------------------ + +// The list of FED tables. This is indexed by a value of +// 'fed_type_t'. +static fed_table_t *fed_tables = NULL; + +// Initially false, set to true once the first unit is initialized, +// which results in the FED list being initialized. +static bool fed_tables_initialized = false; + +// The list of object tables. This is indexed by a value of 'obj_type_t'. +static obj_table_t *obj_tables = NULL; + +// Initially false, set to true once the first unit is initialized, +// which results in the FED list being initialized. +static bool obj_tables_initialized = false; + +// Initially false, set to true once the first unit is initialized, +// which results in the __csi_init() function being called. +static bool csi_init_called = false; + +// ------------------------------------------------------------------------ +// Private function definitions +// ------------------------------------------------------------------------ + +// Initialize the FED tables list, indexed by a value of type +// fed_type_t. This is called once, by the first unit to load. +static void initialize_fed_tables() { + fed_tables = new fed_table_t[NUM_FED_TYPES]; + assert(fed_tables != NULL); + for (int i = 0; i < (int)NUM_FED_TYPES; ++i) { + fed_table_t table; + table.num_entries = 0; + table.entries = NULL; + fed_tables[i] = table; + } + fed_tables_initialized = true; +} + +// Ensure that the FED table of the given type has enough memory +// allocated to add a new unit's entries. +static void ensure_fed_table_capacity(fed_type_t fed_type, + int64_t num_new_entries) { + if (!fed_tables_initialized) + initialize_fed_tables(); + + assert(num_new_entries >= 0); + + fed_table_t *table = &fed_tables[fed_type]; + int64_t total_num_entries = table->num_entries + num_new_entries; + if (num_new_entries > 0) { + if (!table->entries) + table->entries = new csan_source_loc_t[total_num_entries]; + else { + csan_source_loc_t *old_entries = table->entries; + table->entries = new csan_source_loc_t[total_num_entries]; + for (int i = 0; i < table->num_entries; ++i) + table->entries[i] = old_entries[i]; + delete[] old_entries; + } + table->num_entries = total_num_entries; + assert(table->entries != NULL); + } +} + +// Add a new FED table of the given type. +static inline void add_fed_table(fed_type_t fed_type, int64_t num_entries, + const csan_source_loc_t *fed_entries) { + ensure_fed_table_capacity(fed_type, num_entries); + fed_table_t *table = &fed_tables[fed_type]; + csi_id_t base = table->num_entries - num_entries; + for (csi_id_t i = 0; i < num_entries; ++i) + table->entries[base + i] = fed_entries[i]; +} + +// The unit-local counter pointed to by 'fed_id_base' keeps track of +// that unit's "base" ID value of the given type (recall that there is +// a private ID space per FED type). The "base" ID value is the global +// ID that corresponds to the unit's local ID 0. This function stores +// the correct value into a unit's base ID. +static inline void update_ids(fed_type_t fed_type, int64_t num_entries, + csi_id_t *fed_id_base) { + fed_table_t *table = &fed_tables[fed_type]; + // The base ID is the current number of FED entries before adding + // the new FED table. + *fed_id_base = table->num_entries - num_entries; +} + +// Return the FED entry of the given type, corresponding to the given +// CSI ID. +static inline const csan_source_loc_t *get_fed_entry(fed_type_t fed_type, + const csi_id_t csi_id) { + // TODO(ddoucet): threadsafety + fed_table_t *table = &fed_tables[fed_type]; + if (csi_id < table->num_entries) { + assert(table->entries != NULL); + return &table->entries[csi_id]; + } + return NULL; +} + +// Initialize the object tables list, indexed by a value of type +// obj_type_t. This is called once, by the first unit to load. +static void initialize_obj_tables() { + obj_tables = new obj_table_t[NUM_OBJ_TYPES]; + assert(obj_tables != NULL); + for (int i = 0; i < (int)NUM_OBJ_TYPES; ++i) { + obj_table_t table; + table.num_entries = 0; + table.entries = NULL; + obj_tables[i] = table; + } + obj_tables_initialized = true; +} + +// Ensure that the object table of the given type has enough memory allocated to +// add a new unit's entries. +static void ensure_obj_table_capacity(obj_type_t obj_type, + int64_t num_new_entries) { + if (!obj_tables_initialized) + initialize_obj_tables(); + + assert(num_new_entries >= 0); + + obj_table_t *table = &obj_tables[obj_type]; + int64_t total_num_entries = table->num_entries + num_new_entries; + if (num_new_entries > 0) { + if (!table->entries) + table->entries = new obj_source_loc_t[total_num_entries]; + else { + obj_source_loc_t *old_entries = table->entries; + table->entries = new obj_source_loc_t[total_num_entries]; + for (int i = 0; i < table->num_entries; ++i) + table->entries[i] = old_entries[i]; + delete[] old_entries; + } + table->num_entries = total_num_entries; + assert(table->entries != NULL); + } +} + +// Add a new object table of the given type. +static inline void add_obj_table(obj_type_t obj_type, int64_t num_entries, + const obj_source_loc_t *obj_entries) { + ensure_obj_table_capacity(obj_type, num_entries); + obj_table_t *table = &obj_tables[obj_type]; + csi_id_t base = table->num_entries - num_entries; + for (csi_id_t i = 0; i < num_entries; ++i) + table->entries[base + i] = obj_entries[i]; +} + +// Return the object entry of the given type, corresponding to the given +// CSI ID. +static inline const obj_source_loc_t *get_obj_entry(obj_type_t obj_type, + const csi_id_t csi_id) { + // TODO(ddoucet): threadsafety + obj_table_t *table = &obj_tables[obj_type]; + if (csi_id < table->num_entries) { + assert(table->entries != NULL); + return &table->entries[csi_id]; + } + return NULL; +} + +// ------------------------------------------------------------------------ +// External function definitions, including CSIRT API functions. +// ------------------------------------------------------------------------ + +EXTERN_C + +void __csan_unit_init(const char * const file_name, + const csan_instrumentation_counts_t counts); + +// Not used at the moment +// __thread bool __csi_disable_instrumentation; + +typedef struct { + int64_t num_entries; + csi_id_t *id_base; + const csan_source_loc_t *entries; +} unit_fed_table_t; + +typedef struct { + int64_t num_entries; + const obj_source_loc_t *entries; +} unit_obj_table_t; + +// Function signature for the function (generated by the CSI compiler +// pass) that updates the callsite to function ID mappings. +typedef void (*__csi_init_callsite_to_functions)(); + +static inline void compute_inst_counts(csan_instrumentation_counts_t *counts, + unit_fed_table_t *unit_fed_tables) { + int64_t *base = (int64_t *)counts; + for (int i = 0; i < NUM_FED_TYPES; i++) + *(base + i) = unit_fed_tables[i].num_entries; +} + +// A call to this is inserted by the CSI compiler pass, and occurs +// before main(). +CSIRT_API +void __csirt_unit_init(const char * const name, + unit_fed_table_t *unit_fed_tables, + unit_obj_table_t *unit_obj_tables, + __csi_init_callsite_to_functions callsite_to_func_init) { + // Make sure we don't instrument things in __csi_init or __csi_unit init. + // __csi_disable_instrumentation = true; + + // TODO(ddoucet): threadsafety + if (!csi_init_called) { + __csi_init(); + csi_init_called = true; + } + + // Add all FED tables from the new unit + for (int i = 0; i < NUM_FED_TYPES; ++i) { + add_fed_table((fed_type_t)i, unit_fed_tables[i].num_entries, + unit_fed_tables[i].entries); + update_ids((fed_type_t)i, unit_fed_tables[i].num_entries, + unit_fed_tables[i].id_base); + } + + // Add all object tables from the new unit + for (int i = 0; i < NUM_OBJ_TYPES; ++i) { + add_obj_table((obj_type_t)i, unit_obj_tables[i].num_entries, + unit_obj_tables[i].entries); + } + + // Initialize the callsite -> function mappings. This must happen + // after the base IDs have been updated. + callsite_to_func_init(); + + // Call into the tool implementation. + csan_instrumentation_counts_t counts; + compute_inst_counts(&counts, unit_fed_tables); + __csan_unit_init(name, counts); + + // Reset disable flag. + // __csi_disable_instrumentation = false; +} + +CSIRT_API +const csan_source_loc_t *__csan_get_func_source_loc(const csi_id_t func_id) { + return get_fed_entry(FED_TYPE_FUNCTIONS, func_id); +} + +CSIRT_API +const csan_source_loc_t *__csan_get_func_exit_source_loc( + const csi_id_t func_exit_id) { + return get_fed_entry(FED_TYPE_FUNCTION_EXIT, func_exit_id); +} + +CSIRT_API +const csan_source_loc_t *__csan_get_bb_source_loc(const csi_id_t bb_id) { + return get_fed_entry(FED_TYPE_BASICBLOCK, bb_id); +} + +CSIRT_API +const csan_source_loc_t *__csan_get_call_source_loc(const csi_id_t call_id) { + return get_fed_entry(FED_TYPE_CALLSITE, call_id); +} + +CSIRT_API +const csan_source_loc_t *__csan_get_load_source_loc(const csi_id_t load_id) { + return get_fed_entry(FED_TYPE_LOAD, load_id); +} + +CSIRT_API +const csan_source_loc_t *__csan_get_store_source_loc(const csi_id_t store_id) { + return get_fed_entry(FED_TYPE_STORE, store_id); +} + +CSIRT_API +const csan_source_loc_t *__csan_get_detach_source_loc(const csi_id_t detach_id) { + return get_fed_entry(FED_TYPE_DETACH, detach_id); +} + +CSIRT_API +const csan_source_loc_t *__csan_get_task_source_loc(const csi_id_t task_id) { + return get_fed_entry(FED_TYPE_TASK, task_id); +} + +CSIRT_API +const csan_source_loc_t *__csan_get_task_exit_source_loc( + const csi_id_t task_exit_id) { + return get_fed_entry(FED_TYPE_TASK_EXIT, task_exit_id); +} + +CSIRT_API +const csan_source_loc_t *__csan_get_detach_continue_source_loc( + const csi_id_t detach_continue_id) { + return get_fed_entry(FED_TYPE_DETACH_CONTINUE, detach_continue_id); +} + +CSIRT_API +const csan_source_loc_t *__csan_get_sync_source_loc(const csi_id_t sync_id) { + return get_fed_entry(FED_TYPE_SYNC, sync_id); +} + +CSIRT_API +const obj_source_loc_t *__csan_get_load_obj_source_loc(const csi_id_t load_id) { + return get_obj_entry(OBJ_TYPE_LOAD, load_id); +} + +CSIRT_API +const obj_source_loc_t *__csan_get_store_obj_source_loc(const csi_id_t store_id) { + return get_obj_entry(OBJ_TYPE_STORE, store_id); +} + +EXTERN_C_END diff --git a/compiler-rt/lib/cilksan/debug_util.cpp b/compiler-rt/lib/cilksan/debug_util.cpp new file mode 100644 index 000000000000..76e54906b259 --- /dev/null +++ b/compiler-rt/lib/cilksan/debug_util.cpp @@ -0,0 +1,46 @@ +#include +#include +#include +#include +#include "debug_util.h" + +/* + static void print_bt(FILE *f) { + const int N=10; + void *buf[N]; + int n = backtrace(buf, N); + if (1) { + for (int i = 0; i < n; i++) { + print_addr(f, buf[i]); + } + } else { + assert(n>=2); + print_addr(f, buf[2]); + } + } +*/ + +void debug_printf(int level, const char *fmt, ...) { + if(debug_level & level) { + std::va_list l; + va_start(l, fmt); + std::vfprintf(stderr, fmt, l); + va_end(l); + } +} + +// Print out the error message and exit +__attribute__((noreturn)) +void die(const char *fmt, ...) { + std::va_list l; + std::fprintf(stderr, "=================================================\n"); + std::fprintf(stderr, "racedetector: fatal error\n"); + + va_start(l, fmt); + std::vfprintf(stderr, fmt, l); + std::fprintf(stderr, "=================================================\n"); + fflush(stderr); + va_end(l); + std::exit(1); +} + diff --git a/compiler-rt/lib/cilksan/debug_util.h b/compiler-rt/lib/cilksan/debug_util.h new file mode 100644 index 000000000000..4df7d2cd0000 --- /dev/null +++ b/compiler-rt/lib/cilksan/debug_util.h @@ -0,0 +1,61 @@ +#ifndef __DEBUG_UTIL_H__ +#define __DEBUG_UTIL_H__ + +#include + +#define CILKSAN_DEBUG 1 + +// if NULL, err_io is set to stderr +#define ERROR_FILE NULL + +// debug_level is a bitmap +// 1 is basic debugging (old level 1) +// 2 is debug the backtrace +enum debug_levels { + DEBUG_BASIC = 1, + DEBUG_BACKTRACE = 2, + DEBUG_BAGS = 4, + DEBUG_CALLBACK = 8, + DEBUG_MEMORY = 16, + DEBUG_DEQUE = 32, + DEBUG_REDUCER = 64 +}; + +#if CILKSAN_DEBUG +//static int debug_level = DEBUG_BAGS | DEBUG_CALLBACK | DEBUG_MEMORY | DEBUG_DEQUE | DEBUG_REDUCER; +//static int debug_level = DEBUG_BAGS | DEBUG_CALLBACK; +static int debug_level = 0; // DEBUG_CALLBACK | DEBUG_MEMORY; +#else +//static int debug_level = DEBUG_BAGS | DEBUG_CALLBACK | DEBUG_MEMORY | DEBUG_DEQUE | DEBUG_REDUCER; +static int debug_level = 0; +#endif + +#if CILKSAN_DEBUG +#define WHEN_CILKSAN_DEBUG(stmt) do { stmt; } while(0) +#define cilksan_assert(c) \ + do { if (!(c)) { die("%s:%d assertion failure: %s\n", \ + __FILE__, __LINE__, #c);} } while (0) +#else +#define WHEN_CILKSAN_DEBUG(stmt) +#define cilksan_assert(c) +#endif + +#if CILKSAN_DEBUG +// debugging assert to check that the tool is catching all the runtime events +// that are supposed to match up (i.e., has event begin and event end) +enum EventType_t { ENTER_FRAME = 1, ENTER_HELPER = 2, SPAWN_PREPARE = 3, + DETACH = 4, CILK_SYNC = 5, LEAVE_FRAME_OR_HELPER = 6, + RUNTIME_LOOP = 7, NONE = 8 }; +#endif + +__attribute__((noreturn)) +void die(const char *fmt, ...); +void debug_printf(int level, const char *fmt, ...); + +#if CILKSAN_DEBUG +#define DBG_TRACE(level,...) debug_printf(level, __VA_ARGS__) +#else +#define DBG_TRACE(level,...) +#endif + +#endif diff --git a/compiler-rt/lib/cilksan/dictionary.h b/compiler-rt/lib/cilksan/dictionary.h new file mode 100644 index 000000000000..0fd0ce95a795 --- /dev/null +++ b/compiler-rt/lib/cilksan/dictionary.h @@ -0,0 +1,315 @@ +// -*- C++ -*- +#ifndef __DICTIONARY__ +#define __DICTIONARY__ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "cilksan_internal.h" +#include "debug_util.h" +#include "disjointset.h" +#include "frame_data.h" +#include "mem_access.h" +#include "spbag.h" +#include "stack.h" + +class MemoryAccess_t { +public: + DisjointSet_t *func; + AccessLoc_t loc; + + MemoryAccess_t() + : func(nullptr), loc() { } + + MemoryAccess_t(DisjointSet_t *func, + csi_id_t acc_id, const call_stack_t &call_stack) + : func(func), loc(AccessLoc_t(acc_id, call_stack)) { + if (func) + func->inc_ref_count(); + } + + MemoryAccess_t(const MemoryAccess_t ©) + : func(copy.func), loc(copy.loc) { + if (func) + func->inc_ref_count(); + // if (loc) + // loc->inc_ref_count(); + } + + MemoryAccess_t(const MemoryAccess_t &&move) + : func(move.func), loc(std::move(move.loc)) {} + + ~MemoryAccess_t() { + if (func) + func->dec_ref_count(); + // if (loc) + // loc->dec_ref_count(); + } + + bool isValid() const { + if (nullptr == func) + assert(nullptr == loc.call_stack.tail); + return (nullptr != func); + } + + void invalidate() { + if (func) + func->dec_ref_count(); + func = nullptr; + loc.dec_ref_count(); + // if (loc) + // loc->dec_ref_count(); + // loc = nullptr; + } + + DisjointSet_t *getFunc() const { + return func; + } + + const AccessLoc_t &getLoc() const { + return loc; + } + + MemoryAccess_t &operator=(const MemoryAccess_t ©) { + if (func != copy.func) { + if (copy.func) + copy.func->inc_ref_count(); + if (func) + func->dec_ref_count(); + func = copy.func; + } + loc = copy.loc; + // if (loc != copy.loc) { + // if (copy.loc) + // copy.loc->inc_ref_count(); + // if (loc) + // loc->dec_ref_count(); + // loc = copy.loc; + // } + return *this; + } + + MemoryAccess_t &operator=(const MemoryAccess_t &&move) { + if (func) + func->dec_ref_count(); + // if (loc) + // loc->dec_ref_count(); + func = move.func; + loc = std::move(move.loc); + return *this; + } + + void inc_ref_counts(int64_t count) { + assert(func); + // assert(loc); + func->inc_ref_count(count); + loc.inc_ref_count(count); + // loc->inc_ref_count(count); + } + + void dec_ref_counts(int64_t count) { + if (!func->dec_ref_count(count)) + func = nullptr; + loc.dec_ref_count(count); + // if (!loc->dec_ref_count(count)) + // loc = nullptr; + } + + void inherit(const MemoryAccess_t ©) { + if (func) + func->dec_ref_count(); + func = copy.func; + loc.dec_ref_count(); + // if (loc) + // loc->dec_ref_count(); + loc = copy.loc; + } + + // Unsafe method! Only use this if you know what you're doing. + void overwrite(const MemoryAccess_t ©) { + func = copy.func; + loc.overwrite(copy.loc); + } + + // Unsafe method! Only use this if you know what you're doing. + void clear() { + func = nullptr; + loc.clear(); + } + + bool sameAccessLocPtr(const MemoryAccess_t &that) const { + return loc.call_stack.tail == that.loc.call_stack.tail; + } + + // TODO: Get rid of PC from these comparisons + bool operator==(const MemoryAccess_t &that) const { + return (func == that.func); // && (loc == that.loc); + } + + bool operator!=(const MemoryAccess_t &that) const { + // return (func != that.func) || (loc != that.loc); + return !(*this == that); + } + + inline friend + std::ostream& operator<<(std::ostream &os, const MemoryAccess_t &acc) { + os << "function " << acc.func->get_node()->get_func_id() << + ", " << acc.loc; + return os; + } + + // Simple free-list allocator to conserve space and time in managing + // arrays of PAGE_SIZE MemoryAccess_t objects. + struct FreeNode_t { + static size_t FreeNode_ObjSize; + FreeNode_t *next; + }; + static FreeNode_t *free_list; + + void *operator new[](size_t size) { + if (!FreeNode_t::FreeNode_ObjSize) + FreeNode_t::FreeNode_ObjSize = size; + if (free_list) { + assert(size == FreeNode_t::FreeNode_ObjSize); + FreeNode_t *new_node = free_list; + free_list = free_list->next; + return new_node; + } + // std::cerr << "MemoryAccess_t::new[] called, size " << size << "\n"; + return ::operator new[](size); + } + + void operator delete[](void *ptr) { + FreeNode_t *del_node = reinterpret_cast(ptr); + del_node->next = free_list; + free_list = del_node; + } + + static void cleanup_freelist() { + FreeNode_t *node = free_list; + FreeNode_t *next = nullptr; + while (node) { + next = node->next; + ::operator delete[](node); + node = next; + } + } + +}; + +// typedef DisjointSet_t * value_type00; +typedef MemoryAccess_t value_type00; +// struct value_type00 { +// std::shared_ptr val; + +// value_type00() : val(nullptr) {} + +// value_type00(MemoryAccess_t acc) +// : val(std::make_shared(acc)) +// {} + +// value_type00(const value_type00 ©) +// : val(copy.val) +// {} + +// value_type00(const value_type00 &&move) +// : val(std::move(move.val)) +// {} + +// value_type00 &operator=(const value_type00 ©) { +// val = copy.val; +// return *this; +// } + +// value_type00 &operator=(const value_type00 &&move) { +// val = std::move(move.val); +// return *this; +// } + +// bool isValid() const { +// return (bool)val && val->isValid(); +// } + +// void invalidate() { +// return val.reset(); +// } + +// bool operator==(const value_type00 &that) const { +// if (val == that.val) +// return true; +// if (((bool)val && !(bool)that.val) || +// (!(bool)val && (bool)that.val)) +// return false; +// return *val == *that.val; +// } + +// bool operator!=(const value_type00 &that) const { +// return !(val == that.val); +// } +// }; + +class Dictionary { +public: + static const value_type00 null_val; + + virtual value_type00 *find(uint64_t key) { + return nullptr; + } + + virtual value_type00 *find_group(uint64_t key, size_t max_size, + size_t &num_elems) { + num_elems = 1; + return nullptr; + } + + virtual value_type00 *find_exact_group(uint64_t key, size_t max_size, + size_t &num_elems) { + num_elems = 1; + return nullptr; + } + + virtual const value_type00 &operator[] (uint64_t key) { + return null_val; + } + + virtual void erase(uint64_t key) {} + + virtual void erase(uint64_t key, size_t size) {} + + virtual bool includes(uint64_t key) { + return false; + } + + virtual bool includes(uint64_t key, size_t size) { + return false; + } + + virtual void insert(uint64_t key, const value_type00 &f) {} + + virtual void insert(uint64_t key, size_t size, const value_type00 &f) {} + virtual void set(uint64_t key, size_t size, value_type00 &&f) { + insert(key, size, std::move(f)); + } + virtual void insert_into_found_group(uint64_t key, size_t size, + value_type00 *dst, + value_type00 &&f) { + insert(key, size, std::move(f)); + } + + virtual ~Dictionary() {}; + + //uint32_t run_length(uint64_t key) {return 0;} + + //virtual void destruct() {} +}; + +#endif // __DICTIONARY__ diff --git a/compiler-rt/lib/cilksan/disjointset.h b/compiler-rt/lib/cilksan/disjointset.h new file mode 100644 index 000000000000..c95bdaa15e2d --- /dev/null +++ b/compiler-rt/lib/cilksan/disjointset.h @@ -0,0 +1,274 @@ +/* -*- Mode: C++ -*- */ + +#ifndef _DISJOINTSET_H +#define _DISJOINTSET_H + +#include +#include +#include +#include +#include "list.h" +#include "debug_util.h" +#include "spbag.h" + +extern List_t disjoint_set_list; + +static int64_t DS_ID = 0; + +template +class DisjointSet_t { +private: + // the node that initialized this set; const field that does not change + DISJOINTSET_DATA_T const _node; + // the oldest node representing the set that the _node belongs to + DISJOINTSET_DATA_T _set_node; + DisjointSet_t *_set_parent; + uint64_t _rank; // roughly as the height of this node + + int64_t _ref_count; + + // HACK: The destructor calls a callback to free the set node, but in order + // for that callback to get the set node, it needs to call find_set which has + // assertions for ref counts. Thus, we don't dec our ref count if we're + // destructing. + bool _destructing; + + void assert_not_freed() { + cilksan_assert(_destructing || _ref_count >= 0); + } + + /* + * The only reason we need this function is to ensure that the _set_node + * returned for representing this set is the oldest node in the set. + */ + __attribute__((always_inline)) + void swap_set_node_with(DisjointSet_t *that) { + assert_not_freed(); + that->assert_not_freed(); + DISJOINTSET_DATA_T tmp; + tmp = this->_set_node; + this->_set_node = that->_set_node; + that->_set_node = tmp; + } + + // Frees the old parent if it has no more references. + void set_parent(DisjointSet_t *that) { + assert_not_freed(); + + DisjointSet_t *old_parent = this->_set_parent; + + this->_set_parent = that; + that->inc_ref_count(); + + // dec_ref_count checks whether a node is its only reference (through + // parent). If we called dec_ref_count (removing the parent relationship) + // before setting this's parent and we had another reference besides the + // parent relationship, dec_ref_count would incorrectly believe that this's + // only reference is in having itself as a parent. + assert(old_parent != NULL); + + old_parent->dec_ref_count(); + } + + /* + * Links this disjoint set to that disjoint set. + * Don't need to be public. + * + * @param that that disjoint set. + */ + __attribute__((always_inline)) + void link(DisjointSet_t *that) { + assert_not_freed(); + cilksan_assert(that != NULL); + + // link the node with smaller height into the node with larger height + if (this->_rank > that->_rank) { + that->set_parent(this); + } else { + this->set_parent(that); + if (this->_rank == that->_rank) + ++that->_rank; + // because we are linking this into the forest rooted at that, let's + // swap the nodes in this object and that object to keep the metadata + // hold in the node consistent. + this->swap_set_node_with(that); + } + } + + /* + * Finds the set containing this disjoint set element. + * + * Note: Performs path compression along the way. + * The _set_parent field will be updated after the call. + */ + __attribute__((always_inline)) + DisjointSet_t* find_set() { + assert_not_freed(); + + cilksan_assert(!_destructing); + disjoint_set_list.lock(); + DisjointSet_t *node = this; + + int64_t tmp_ref_count = _ref_count; + + node->assert_not_freed(); + + while (node->_set_parent != node) { + cilksan_assert(node->_set_parent); + + if (__builtin_expect(!_destructing || node != this, 1)) { + disjoint_set_list.push(node); + } + node = node->_set_parent; + } + + cilksan_assert(tmp_ref_count == _ref_count); + + // node is now the root. Perform path compression by updating the parents + // of each of the nodes we saw. + // We process backwards so that in case a node ought to be freed (i.e. its + // child was the last referencing it), we don't process it after freeing. + for (int i = disjoint_set_list.length() - 2; i >= 0; i--) { + DisjointSet_t *p = (DisjointSet_t *)disjoint_set_list.list()[i]; + // We don't need to check that p != p->_set_parent because the root of + // the set wasn't pushed to the list (see the while loop above). + p->set_parent(node); + } + + disjoint_set_list.unlock(); + return node; + } + +public: + int64_t _ID; + + DisjointSet_t(DISJOINTSET_DATA_T node) : + _node(node), _set_node(node), _set_parent(NULL), _rank(0), _ref_count(0), + _destructing(false), _ID(DS_ID++) { + this->_set_parent = this; + this->inc_ref_count(); + WHEN_CILKSAN_DEBUG(debug_count++); + } + +#if CILKSAN_DEBUG + static long debug_count; + static uint64_t nodes_created; +#endif + + static void (*dtor_callback)(DisjointSet_t *); + + ~DisjointSet_t() { + _destructing = true; + dtor_callback(this); + if (this->_set_parent != this) { + // Otherwise, we run the risk of double freeing. + _set_parent->dec_ref_count(); + } + +#if CILKSAN_DEBUG + _destructing = false; + _set_parent = NULL; + _ref_count = -1; + + debug_count--; +#endif + } + + // Decrements the ref count. Returns true if the node was deleted + // as a result. + inline int64_t dec_ref_count(int64_t count = 1) { + assert_not_freed(); + assert(_ref_count >= count); + _ref_count -= count; + if (_ref_count == 0 || (_ref_count == 1 && this->_set_parent == this)) { + delete this; + return 0; + } + return _ref_count; + } + + inline void inc_ref_count(int64_t count = 1) { + assert_not_freed(); + + _ref_count += count; + } + + __attribute__((always_inline)) + DISJOINTSET_DATA_T get_node() { + assert_not_freed(); + + return _node; + } + + __attribute__((always_inline)) + DISJOINTSET_DATA_T get_set_node() { + assert_not_freed(); + + return find_set()->_set_node; + } + + __attribute__((always_inline)) + DISJOINTSET_DATA_T get_my_set_node() { + assert_not_freed(); + + return _set_node; + } + + /* + * Unions this disjoint set and that disjoint set. + * + * NOTE: Implicitly, in order to maintain the oldest _set_node, one should + * always combine younger set into this set (defined by creation time). Since + * we union by rank, we may end up linking this set to the younger set. To + * make sure that we always return the oldest _node to represent the set, we + * use an additional _set_node field to keep track of the oldest node and use + * that to represent the set. + * + * @param that that (younger) disjoint set. + */ + // Called "combine," because "union" is a reserved keyword in C + void combine(DisjointSet_t *that) { + assert_not_freed(); + + cilksan_assert(that); + cilksan_assert(this->find_set() != that->find_set()); + this->find_set()->link(that->find_set()); + cilksan_assert(this->find_set() == that->find_set()); + } + + // Simple free-list allocator to conserve space and time in managing + // DisjointSet_t objects. + static DisjointSet_t *free_list; + + void *operator new(size_t size) { + if (free_list) { + DisjointSet_t *new_node = free_list; + free_list = free_list->_set_parent; + return new_node; + } + return ::operator new(size); + } + + void operator delete(void *ptr) { + DisjointSet_t *del_node = reinterpret_cast(ptr); + del_node->_set_parent = free_list; + free_list = del_node; + } + + static void cleanup_freelist() { + DisjointSet_t *node = free_list; + DisjointSet_t *next = nullptr; + while (node) { + next = node->_set_parent; + ::operator delete(node); + node = next; + } + } +}; + +#if CILKSAN_DEBUG +template<> +long DisjointSet_t::debug_count; +#endif + +#endif // #ifndef _DISJOINTSET_H diff --git a/compiler-rt/lib/cilksan/drivercsan.cpp b/compiler-rt/lib/cilksan/drivercsan.cpp new file mode 100644 index 000000000000..cdcb321522ff --- /dev/null +++ b/compiler-rt/lib/cilksan/drivercsan.cpp @@ -0,0 +1,573 @@ +// #include +#include +#include +#include +#include +#include +// #include +#include +#include +#include + +#include "cilksan_internal.h" +#include "debug_util.h" +#include "mem_access.h" +#include "stack.h" + +#define CALLERPC ((uintptr_t)__builtin_return_address(0)) + +#define CILKSAN_API extern "C" __attribute__((visibility("default"))) + +// global var: FILE io used to print error messages +FILE *err_io; + +// Defined in print_addr.cpp +extern void read_proc_maps(); +extern void delete_proc_maps(); +extern void print_addr(FILE *f, void *a); +// declared in cilksan; for debugging only +#if CILKSAN_DEBUG +extern enum EventType_t last_event; +#endif + +// Defined in cilksan.cpp +extern call_stack_t call_stack; +extern Stack_t sp_stack; +// Defined in print_addr.cpp +extern uintptr_t *call_pc; +extern uintptr_t *spawn_pc; +extern uintptr_t *load_pc; +extern uintptr_t *store_pc; +static csi_id_t total_call = 0; +static csi_id_t total_spawn = 0; +static csi_id_t total_load = 0; +static csi_id_t total_store = 0; + +static bool TOOL_INITIALIZED = false; + +// When either is set to false, no errors are output +static bool instrumentation = false; +// needs to be reentrant due to reducer operations; 0 means checking +static int checking_disabled = 0; + +static inline void enable_instrumentation() { + DBG_TRACE(DEBUG_BASIC, "Enable instrumentation.\n"); + instrumentation = true; +} + +static inline void disable_instrumentation() { + DBG_TRACE(DEBUG_BASIC, "Disable instrumentation.\n"); + instrumentation = false; +} + +static inline void enable_checking() { + checking_disabled--; + DBG_TRACE(DEBUG_BASIC, "%d: Enable checking.\n", checking_disabled); + cilksan_assert(checking_disabled >= 0); +} + +static inline void disable_checking() { + cilksan_assert(checking_disabled >= 0); + checking_disabled++; + DBG_TRACE(DEBUG_BASIC, "%d: Disable checking.\n", checking_disabled); +} + +// outside world (including runtime). +// Non-inlined version for user code to use +CILKSAN_API void __cilksan_enable_checking() { + checking_disabled--; + cilksan_assert(checking_disabled >= 0); + DBG_TRACE(DEBUG_BASIC, "External enable checking (%d).\n", checking_disabled); +} + +// Non-inlined version for user code to use +CILKSAN_API void __cilksan_disable_checking() { + cilksan_assert(checking_disabled >= 0); + checking_disabled++; + DBG_TRACE(DEBUG_BASIC, "External disable checking (%d).\n", checking_disabled); +} + +// Non-inlined callback for user code to check if checking is enabled. +CILKSAN_API bool __cilksan_is_checking_enabled() { + return (checking_disabled == 0); +} + +static inline bool should_check() { + return (instrumentation && checking_disabled == 0); +} + +// called upon process exit +static void csan_destroy(void) { + // fprintf(err_io, "csan_destroy called.\n"); + disable_instrumentation(); + disable_checking(); + cilksan_deinit(); + // std::cerr << "call_stack.size " << call_stack.size() << std::endl; + fflush(stdout); + delete_proc_maps(); +} + +static void init_internal() { + read_proc_maps(); + if (ERROR_FILE) { + FILE *tmp = fopen(ERROR_FILE, "w+"); + if (tmp) err_io = tmp; + } + if (err_io == NULL) err_io = stderr; + + char *e = getenv("CILK_NWORKERS"); + if (!e || 0 != strcmp(e, "1")) { + // fprintf(err_io, "Setting CILK_NWORKERS to be 1\n"); + if( setenv("CILK_NWORKERS", "1", 1) ) { + fprintf(err_io, "Error setting CILK_NWORKERS to be 1\n"); + exit(1); + } + } + // Force reductions. + // XXX: Does not work with SP+ algorithm, but works with ordinary + // SP bags. + e = getenv("CILK_FORCE_REDUCE"); + if (!e || 0 != strcmp(e, "1")) { + // fprintf(err_io, "Setting CILK_FORCE_REDUCE to be 1\n"); + if( setenv("CILK_FORCE_REDUCE", "1", 1) ) { + fprintf(err_io, "Error setting CILK_FORCE_REDUCE to be 1\n"); + exit(1); + } + } +} + +CILKSAN_API void __csi_init() { + // This method should only be called once. + if (TOOL_INITIALIZED) assert(0); + + atexit(csan_destroy); + init_internal(); + // moved this later when we enter the first Cilk frame + // cilksan_init(); + // enable_instrumentation(); + TOOL_INITIALIZED = true; + // fprintf(err_io, "tsan_init called.\n"); +} + +static void grow_pc_table(uintptr_t *&table, csi_id_t &table_cap, + csi_id_t extra_cap) { + csi_id_t new_cap = table_cap + extra_cap; + table = (uintptr_t *)realloc(table, new_cap * sizeof(uintptr_t)); + for (csi_id_t i = table_cap; i < new_cap; ++i) + table[i] = (uintptr_t)nullptr; + table_cap = new_cap; +} + +CILKSAN_API +void __csan_unit_init(const char * const file_name, + const csan_instrumentation_counts_t counts) +{ + disable_checking(); + // Grow the tables mapping CSI ID's to PC values. + if (counts.num_call) + grow_pc_table(call_pc, total_call, counts.num_call); + if (counts.num_detach) + grow_pc_table(spawn_pc, total_spawn, counts.num_detach); + if (counts.num_load) + grow_pc_table(load_pc, total_load, counts.num_load); + if (counts.num_store) + grow_pc_table(store_pc, total_store, counts.num_store); + enable_checking(); +} + +// invoked whenever a function enters; no need for this +CILKSAN_API void __csan_func_entry(const csi_id_t func_id, + void *sp, + const func_prop_t prop) { + const csan_source_loc_t *srcloc = __csan_get_func_source_loc(func_id); + DBG_TRACE(DEBUG_CALLBACK, "__csan_func_entry(%d) at %s (%s:%d)\n", + func_id, + srcloc->name, srcloc->filename, + srcloc->line_number); + cilksan_assert(TOOL_INITIALIZED); + if (!prop.may_spawn) + // Ignore entry calls into non-Cilk functions. + return; + + disable_checking(); + static bool first_call = true; + if (first_call) { + cilksan_init(); + first_call = false; + } + // Record high location of the stack for this frame. + sp_stack.push(); + *sp_stack.head() = (uintptr_t)sp; + // Record low location of the stack for this frame. This value will be + // updated by reads and writes to the stack. + sp_stack.push(); + *sp_stack.head() = (uintptr_t)sp; + + // Update the tool for entering a Cilk function. + cilksan_do_enter_begin(); + cilksan_do_enter_end((uintptr_t)sp); + enable_instrumentation(); + enable_checking(); +} + +CILKSAN_API void __csan_func_exit(const csi_id_t func_exit_id, + const csi_id_t func_id, + const func_exit_prop_t prop) { + cilksan_assert(TOOL_INITIALIZED); + if (prop.may_spawn) { + disable_checking(); + // Update the tool for leaving a Cilk function. + cilksan_do_leave_begin(); + cilksan_do_leave_end(); + + // Pop stack pointers. + uintptr_t low_stack = *sp_stack.head(); + sp_stack.pop(); + uintptr_t high_stack = *sp_stack.head(); + sp_stack.pop(); + assert(low_stack <= high_stack); + // Clear shadow memory of stack locations. + if (low_stack != high_stack) + cilksan_clear_shadow_memory(low_stack, high_stack - low_stack + 1); + enable_checking(); + } + // XXX Let's focus on Cilk function for now; maybe put it back later + // cilksan_do_function_exit(); +} + +CILKSAN_API void __csi_before_call(const csi_id_t call_id, + const csi_id_t func_id, + const call_prop_t prop) { + DBG_TRACE(DEBUG_CALLBACK, "__csi_before_call(%ld, %ld)\n", + call_id, func_id); + + disable_checking(); + // Record the address of this call site. + if (!call_pc[call_id]) + call_pc[call_id] = CALLERPC; + // Push the call onto the call stack. + call_stack.push(CallID_t(CALL, call_id)); + enable_checking(); +} + +CILKSAN_API void __csi_after_call(const csi_id_t call_id, + const csi_id_t func_id, + const call_prop_t prop) { + DBG_TRACE(DEBUG_CALLBACK, "__csi_after_call(%ld, %ld)\n", + call_id, func_id); + // Pop the call off of the call stack. + disable_checking(); + assert(call_stack.tail->id == CallID_t(CALL, call_id) && + "ERROR: after_call encountered without corresponding before_call"); + call_stack.pop(); + enable_checking(); +} + +CILKSAN_API void __csan_detach(const csi_id_t detach_id) { + DBG_TRACE(DEBUG_CALLBACK, "__csan_detach(%ld)\n", + detach_id); + disable_checking(); + cilksan_assert(last_event == NONE); + WHEN_CILKSAN_DEBUG(last_event = SPAWN_PREPARE); + WHEN_CILKSAN_DEBUG(last_event = NONE); + // Record the address of this detach. + if (!spawn_pc[detach_id]) + spawn_pc[detach_id] = CALLERPC; + // Push the detach onto the call stack. + call_stack.push(CallID_t(SPAWN, detach_id)); + enable_checking(); +} + +CILKSAN_API void __csan_task(const csi_id_t task_id, const csi_id_t detach_id, + void *sp) { + WHEN_CILKSAN_DEBUG(last_event = NONE); + disable_checking(); + // Record high location of the stack for this frame. + sp_stack.push(); + *sp_stack.head() = (uintptr_t)sp; + // Record low location of the stack for this frame. This value will be + // updated by reads and writes to the stack. + sp_stack.push(); + *sp_stack.head() = (uintptr_t)sp; + + // Update tool for entering detach-helper function and performing detach. + cilksan_do_enter_helper_begin(); + cilksan_do_enter_end((uintptr_t)sp); + cilksan_do_detach_begin(); + cilksan_do_detach_end(); + enable_checking(); +} + +CILKSAN_API void __csan_task_exit(const csi_id_t task_exit_id, + const csi_id_t task_id, + const csi_id_t detach_id) { + disable_checking(); + // Update tool for leaving a detach-helper function. + cilksan_do_leave_begin(); + cilksan_do_leave_end(); + + // Pop stack pointers. + uintptr_t low_stack = *sp_stack.head(); + sp_stack.pop(); + uintptr_t high_stack = *sp_stack.head(); + sp_stack.pop(); + assert(low_stack <= high_stack); + // Clear shadow memory of stack locations. + if (low_stack != high_stack) + cilksan_clear_shadow_memory(low_stack, high_stack - low_stack + 1); + enable_checking(); +} + +CILKSAN_API void __csan_detach_continue(const csi_id_t detach_continue_id, + const csi_id_t detach_id) { + DBG_TRACE(DEBUG_CALLBACK, "__csan_detach_continue(%ld)\n", + detach_id); + disable_checking(); + assert(call_stack.tail->id == CallID_t(SPAWN, detach_id) && + "ERROR: detach_continue encountered without corresponding detach"); + call_stack.pop(); + + if (last_event == LEAVE_FRAME_OR_HELPER) + cilksan_do_leave_end(); + WHEN_CILKSAN_DEBUG(last_event = NONE); + enable_checking(); +} + +CILKSAN_API void __csan_sync(csi_id_t sync_id) { + disable_checking(); + cilksan_assert(TOOL_INITIALIZED); + // Because this is a serial tool, we can safely perform all operations related + // to a sync. + cilksan_do_sync_begin(); + cilksan_do_sync_end(); + enable_checking(); +} + +// Assuming __csan_load/store is inlined, the stack should look like this: +// +// ------------------------------------------- +// | user func that is about to do a memop | +// ------------------------------------------- +// | __csan_load/store | +// ------------------------------------------- +// | backtrace (assume __csan_load/store and | +// | get_user_code_rip is inlined)| +// ------------------------------------------- +// +// In the user program, __csan_load/store are inlined +// right before the corresponding read / write in the user code. +// the return addr of __csan_load/store is the rip for the read / write +CILKSAN_API +void __csan_load(csi_id_t load_id, void *addr, int32_t size, load_prop_t prop) { + // TODO: Use alignment information. + cilksan_assert(TOOL_INITIALIZED); + if (should_check()) { + disable_checking(); + // Record the address of this load. + if (!load_pc[load_id]) + load_pc[load_id] = CALLERPC; + + DBG_TRACE(DEBUG_MEMORY, "%s read %p\n", __FUNCTION__, addr); + // Record this read. + cilksan_do_read(load_id, (uintptr_t)addr, size); + enable_checking(); + } else { + DBG_TRACE(DEBUG_MEMORY, "SKIP %s read %p\n", __FUNCTION__, addr); + } +} + +CILKSAN_API +void __csan_large_load(csi_id_t load_id, void *addr, size_t size, + load_prop_t prop) { + // TODO: Use alignment information. + cilksan_assert(TOOL_INITIALIZED); + if (should_check()) { + disable_checking(); + // Record the address of this load. + if (!load_pc[load_id]) + load_pc[load_id] = CALLERPC; + + DBG_TRACE(DEBUG_MEMORY, "%s read %p\n", __FUNCTION__, addr); + // Record this read. + cilksan_do_read(load_id, (uintptr_t)addr, size); + enable_checking(); + } else { + DBG_TRACE(DEBUG_MEMORY, "SKIP %s read %p\n", __FUNCTION__, addr); + } +} + +CILKSAN_API +void __csan_store(csi_id_t store_id, void *addr, int32_t size, store_prop_t prop) { + // TODO: Use alignment information. + cilksan_assert(TOOL_INITIALIZED); + if (should_check()) { + disable_checking(); + // Record the address of this store. + if (!store_pc[store_id]) + store_pc[store_id] = CALLERPC; + + DBG_TRACE(DEBUG_MEMORY, "%s wrote %p\n", __FUNCTION__, addr); + // Record this write. + cilksan_do_write(store_id, (uintptr_t)addr, size); + enable_checking(); + } else { + DBG_TRACE(DEBUG_MEMORY, "SKIP %s wrote %p\n", __FUNCTION__, addr); + } +} + +CILKSAN_API +void __csan_large_store(csi_id_t store_id, void *addr, size_t size, + store_prop_t prop) { + // TODO: Use alignment information. + cilksan_assert(TOOL_INITIALIZED); + if (should_check()) { + disable_checking(); + // Record the address of this store. + if (!store_pc[store_id]) + store_pc[store_id] = CALLERPC; + + DBG_TRACE(DEBUG_MEMORY, "%s wrote %p\n", __FUNCTION__, addr); + // Record this write. + cilksan_do_write(store_id, (uintptr_t)addr, size); + enable_checking(); + } else { + DBG_TRACE(DEBUG_MEMORY, "SKIP %s wrote %p\n", __FUNCTION__, addr); + } +} + +static std::unordered_map malloc_sizes; + +typedef void*(*malloc_t)(size_t); +static malloc_t real_malloc = NULL; + +CILKSAN_API void* malloc(size_t s) { + + // align the allocation to simplify erasing from shadow mem + // uint64_t new_size = ALIGN_BY_NEXT_MAX_GRAIN_SIZE(s); + size_t new_size = ALIGN_FOR_MALLOC(s); + assert(s == new_size); + disable_checking(); + + // Don't try to init, since that needs malloc. + if (real_malloc == NULL) { + real_malloc = (malloc_t)dlsym(RTLD_NEXT, "malloc"); + char *error = dlerror(); + if (error != NULL) { + fputs(error, err_io); + fflush(err_io); + abort(); + } + } + void *r = real_malloc(new_size); + enable_checking(); + //printf("%d\n", should_check()); + if (TOOL_INITIALIZED && should_check()) { + disable_checking(); + malloc_sizes.insert({(uintptr_t)r, new_size}); + // cilksan_clear_shadow_memory((size_t)r, (size_t)r+malloc_usable_size(r)-1); + cilksan_clear_shadow_memory((size_t)r, new_size); + enable_checking(); + } + + return r; +} + +typedef void(*free_t)(void*); +static free_t real_free = NULL; + +CILKSAN_API void free(void *ptr) { + + disable_checking(); + if (real_free == NULL) { + real_free = (free_t)dlsym(RTLD_NEXT, "free"); + char *error = dlerror(); + if (error != NULL) { + fputs(error, err_io); + fflush(err_io); + abort(); + } + } + real_free(ptr); + enable_checking(); + + //printf("%d\n", should_check()); + if (TOOL_INITIALIZED && should_check()) { + disable_checking(); + auto iter = malloc_sizes.find((uintptr_t)ptr); + if (iter != malloc_sizes.end()) { + // cilksan_clear_shadow_memory((size_t)ptr, iter->second); + + // Treat a free as a write to all freed addresses. This way, the tool + // will report a race if an operation tries to access a location that was + // freed in parallel. + cilksan_do_write(UNKNOWN_CSI_ID, (uintptr_t)ptr, iter->second); + malloc_sizes.erase(iter); + } + enable_checking(); + } +} + +// typedef void(*__cilkrts_hyper_create_t)(__cilkrts_hyperobject_base *); +// static __cilkrts_hyper_create_t real___cilkrts_hyper_create = NULL; +// CILKSAN_API +// void __cilkrts_hyper_create(__cilkrts_hyperobject_base *hb) { +// disable_checking(); +// if (real___cilkrts_hyper_create == NULL) { +// real___cilkrts_hyper_create = +// (__cilkrts_hyper_create_t)dlsym(RTLD_NEXT, "__cilkrts_hyper_create"); +// char *error = dlerror(); +// if (error != NULL) { +// fputs(error, err_io); +// fflush(err_io); +// abort(); +// } +// } +// fprintf(stderr, "my cilkrts_hyper_create\n"); +// real___cilkrts_hyper_create(hb); +// enable_checking(); +// } + +// typedef void *(*__cilkrts_hyper_lookup_t)(__cilkrts_hyperobject_base); +// static __cilkrts_hyper_lookup_t real___cilkrts_hyper_lookup = NULL; +// CILKSAN_API +// void *__cilkrts_hyper_lookup(__cilkrts_hyperobject_base *hb) { +// disable_checking(); +// if (real___cilkrts_hyper_lookup == NULL) { +// real___cilkrts_hyper_lookup = +// (__cilkrts_hyper_lookup_t)dlsym(RTLD_NEXT, "__cilkrts_hyper_lookup"); +// char *error = dlerror(); +// if (error != NULL) { +// fputs(error, err_io); +// fflush(err_io); +// abort(); +// } +// } +// void *r = __real___cilkrts_hyper_lookup(hb); +// enable_checking(); +// return r; +// } + +// typedef cilkred_map *(*merge_reducer_maps_t)(__cilkrts_worker **, +// cilkred_map *, +// cilkred_map *); +// static merge_reducer_maps_t real_merge_reducer_maps = NULL; +// CILKSAN_API +// cilkred_map *merge_reducer_maps(__cilkrts_worker **w_ptr, +// cilkred_map *left_map, +// cilkred_map *right_map) { +// disable_checking(); +// if (real_merge_reducer_maps == NULL) { +// real_merge_reducer_maps = +// (merge_reducer_maps_t)dlsym(RTLD_NEXT, "merge_reducer_maps"); +// char *error = dlerror(); +// if (error != NULL) { +// fputs(error, err_io); +// fflush(err_io); +// abort(); +// } +// } +// std::cerr << "my merge_reducer_maps\n"; +// cilkred_map *r = real_merge_reducer_maps(w_ptr, left_map, right_map); +// enable_checking(); +// return r; +// } diff --git a/compiler-rt/lib/cilksan/frame_data.h b/compiler-rt/lib/cilksan/frame_data.h new file mode 100644 index 000000000000..1ac7da4f5f7d --- /dev/null +++ b/compiler-rt/lib/cilksan/frame_data.h @@ -0,0 +1,84 @@ +// -*- C++ -*- +#ifndef __FRAME_DATA_T__ +#define __FRAME_DATA_T__ + +#include "csan.h" +#include "disjointset.h" + +enum EntryType_t { SPAWNER = 1, HELPER = 2, DETACHER = 3 }; +enum FrameType_t { SHADOW_FRAME = 1, FULL_FRAME = 2 }; + +typedef struct Entry_t { + enum EntryType_t entry_type; + enum FrameType_t frame_type; + // const csi_id_t func_id; + + // fields that are for debugging purpose only +#if CILKSAN_DEBUG + uint64_t frame_id; + // uint32_t prev_helper; // index of the HELPER frame right above this entry + // // initialized only for a HELPER frame +#endif +} Entry_t; + +// Struct for keeping track of shadow frame +typedef struct FrameData_t { + Entry_t frame_data; + DisjointSet_t *Sbag; + DisjointSet_t *Pbag; + + void set_sbag(DisjointSet_t *that) { + if (Sbag) + Sbag->dec_ref_count(); + + Sbag = that; + if (Sbag) + Sbag->inc_ref_count(); + } + + void set_pbag(DisjointSet_t *that) { + if (Pbag) + Pbag->dec_ref_count(); + + Pbag = that; + if (Pbag) + Pbag->inc_ref_count(); + } + + void reset() { + set_sbag(NULL); + set_pbag(NULL); + } + + FrameData_t() : + Sbag(NULL), Pbag(NULL) { } + + ~FrameData_t() { + // update ref counts + reset(); + } + + // Copy constructor and assignment operator ensure that reference + // counts are properly maintained during resizing. + FrameData_t(const FrameData_t ©) : + frame_data(copy.frame_data), + Sbag(NULL), Pbag(NULL) { + set_sbag(copy.Sbag); + set_pbag(copy.Pbag); + } + + FrameData_t& operator=(const FrameData_t ©) { + frame_data = copy.frame_data; + set_sbag(copy.Sbag); + set_pbag(copy.Pbag); + return *this; + } + + // remember to update this whenever new fields are added + inline void init_new_function(DisjointSet_t *_sbag) { + cilksan_assert(Pbag == NULL); + set_sbag(_sbag); + } + +} FrameData_t; +#endif // __FRAME_DATA_T__ diff --git a/compiler-rt/lib/cilksan/list.h b/compiler-rt/lib/cilksan/list.h new file mode 100644 index 000000000000..89138fbaa127 --- /dev/null +++ b/compiler-rt/lib/cilksan/list.h @@ -0,0 +1,112 @@ +/* -*- Mode: C++ -*- */ +/* + * A list of pointers. It uses table-doubling to support larger capacities, but + * does not release memory until explicitly told to do so (free()). It has a + * crude form of locking in which a second lock before an unlock will crash the + * program. + * + * Usage: + * - First lock the list via list.lock() + * + * - Next, call list.push(obj) as many times as desired + * + * - You can iterate over the list, e.g. + * + * // Note that it's important to use the getter functions for the list + * // and length if you plan on pushing while iterating. Otherwise, it's + * // fine to cache the values. + * for (int i = 0; i < list.length(); i++) { + * void *obj = list.list()[i]; + * // do something with obj + * } + * + * - Finally, the list should eventually be freed using list.free_list() + */ + +#ifndef _LIST_H +#define _LIST_H + +#include +#include "debug_util.h" + +class List_t { +private: + const int _DEFAULT_CAPACITY = 128; + + int _length = 0; + void **_list = NULL; +#if CILKSAN_DEBUG + bool _locked = false; +#endif + int _capacity = 0; + +public: + List_t() : + _length(0), +#if CILKSAN_DEBUG + _locked(false), +#endif + _capacity(_DEFAULT_CAPACITY) { + + _list = (void**)malloc(_capacity * sizeof(void*)); + } + + // Returns a pointer to the first element in the list. Elements are stored + // contiguously. + // + // This must be called again after a push() in case the list has changed. + // + // The ordering of the elements will not be changed, even if the result + // changes. + __attribute__((always_inline)) + void **list() { return _list; } + + // The length of the list. Automically reset to 0 on unlock(). + __attribute__((always_inline)) + int length() { return _length; } + + // Crashes the program if lock() is called a second time before unlock(). + __attribute__((always_inline)) + void lock() { + cilksan_assert(!_locked); + +#if CILKSAN_DEBUG + _locked = true; +#endif + } + + __attribute__((always_inline)) + void unlock() { + cilksan_assert(_locked); + +#if CILKSAN_DEBUG + _locked = false; +#endif + _length = 0; + } + + // Reclaims any memory used by the list. Should be called at the end of the + // program. + __attribute__((always_inline)) + void free_list() { + cilksan_assert(!_locked); + + if (_list != NULL) + free(_list); + } + + // Adds an element to the end of the list. + __attribute__((always_inline)) + void push(void *obj) { + cilksan_assert(_locked); + + if (__builtin_expect(_length == _capacity, 0)) { + _capacity *= 2; + _list = (void**)realloc(_list, _capacity * sizeof(void*)); + } + + _list[_length++] = obj; + } +}; + +#endif // _LIST_H diff --git a/compiler-rt/lib/cilksan/mem_access.cpp b/compiler-rt/lib/cilksan/mem_access.cpp new file mode 100644 index 000000000000..c38425aee531 --- /dev/null +++ b/compiler-rt/lib/cilksan/mem_access.cpp @@ -0,0 +1,125 @@ +#include + +#include "cilksan_internal.h" +#include "mem_access.h" + +extern void report_race(uintptr_t first_inst, uintptr_t second_inst, + uintptr_t addr, enum RaceType_t race_type); + +// get the start and end indices and gtype to use for accesing the readers / +// writers lists; the gtype is the largest granularity that this memory access +// is aligned with +enum GrainType_t +MemAccessList_t::get_mem_index(uintptr_t addr, size_t size, + int& start, int& end) { + + start = (int) (addr & (uintptr_t)(MAX_GRAIN_SIZE-1)); + end = (int) ((addr + size) & (uintptr_t)(MAX_GRAIN_SIZE-1)); + if (end == 0) end = MAX_GRAIN_SIZE; + + enum GrainType_t gtype = mem_size_to_gtype(size); + if (IS_ALIGNED_WITH_GTYPE(addr, gtype) == false) { gtype = ONE; } + + return gtype; +} + +void MemAccessList_t::break_list_into_smaller_gtype(bool for_read, + enum GrainType_t new_gtype) { + + + MemAccess_t **l = writers; + enum GrainType_t gtype = writer_gtype; + if (for_read) { + l = readers; + gtype = reader_gtype; + } + const int stride = gtype_to_mem_size[new_gtype]; + MemAccess_t *acc = l[0]; + + for (int i = stride; i < MAX_GRAIN_SIZE; i += stride) { + if (IS_ALIGNED_WITH_GTYPE(i, gtype)) { + acc = l[i]; + } else if(acc) { + acc->inc_ref_count(); + l[i] = acc; + } + } + + if(for_read) { + reader_gtype = new_gtype; + } else { + writer_gtype = new_gtype; + } +} + + +MemAccessList_t::MemAccessList_t(uintptr_t addr, bool is_read, + MemAccess_t *acc, size_t mem_size) + : start_addr(ALIGN_BY_PREV_MAX_GRAIN_SIZE(addr)), + reader_gtype(UNINIT), writer_gtype(UNINIT) { + + for (int i = 0; i < MAX_GRAIN_SIZE; i++) { + readers[i] = writers[i] = NULL; + } + + int start, end; + const enum GrainType_t gtype = get_mem_index(addr, mem_size, start, end); + + MemAccess_t **l; + if (is_read) { + reader_gtype = gtype; + l = readers; + } else { + writer_gtype = gtype; + l = writers; + } + for (int i = start; i < end; i += gtype_to_mem_size[gtype]) { + acc->inc_ref_count(); + l[i] = acc; + } +} + +MemAccessList_t::~MemAccessList_t() { + MemAccess_t *acc; + if (reader_gtype != UNINIT) { + for (int i = 0; i < MAX_GRAIN_SIZE; i+=gtype_to_mem_size[reader_gtype]) { + acc = readers[i]; + if(acc && acc->dec_ref_count() == 0) { + delete acc; + readers[i] = 0; + } + } + } + + if(writer_gtype != UNINIT) { + for(int i=0; i < MAX_GRAIN_SIZE; i+=gtype_to_mem_size[writer_gtype]) { + acc = writers[i]; + if(acc && acc->dec_ref_count() == 0) { + delete acc; + writers[i] = 0; + } + } + } +} + +/* +#if CILKSAN_DEBUG +void MemAccessList_t::check_invariants(uint64_t current_func_id) { + SPBagInterface *lca; + for (int i = 0; i < MAX_GRAIN_SIZE; i++) { + if (readers[i]) { + lca = readers[i]->func->get_set_node(); + cilksan_assert(current_func_id >= lca->get_func_id()); + // if LCA is a P-node (Cilk function), its rsp must have been initialized + cilksan_assert(lca->is_SBag() || lca->get_rsp() != UNINIT_STACK_PTR); + } + if (writers[i]) { // same checks for the writers + lca = writers[i]->func->get_set_node(); + cilksan_assert(current_func_id >= lca->get_func_id()); + cilksan_assert(lca->is_SBag() || lca->get_rsp() != UNINIT_STACK_PTR); + } + } +} +#endif // CILKSAN_DEBUG +*/ + diff --git a/compiler-rt/lib/cilksan/mem_access.h b/compiler-rt/lib/cilksan/mem_access.h new file mode 100644 index 000000000000..c2cd4cd67357 --- /dev/null +++ b/compiler-rt/lib/cilksan/mem_access.h @@ -0,0 +1,225 @@ +/* -*- Mode: C++ -*- */ + +#ifndef _MEM_ACCESS_H +#define _MEM_ACCESS_H + +#include +#include + +#include "cilksan_internal.h" +#include "debug_util.h" +#include "disjointset.h" +#include "spbag.h" + +// macro for address manipulation for shadow mem +// used in all shadow memory implementations +#define ADDR_TO_KEY(addr) ((uint64_t) ((uint64_t)addr >> 3)) + +#define MAX_GRAIN_SIZE 64 +// #define MAX_GRAIN_SIZE 8 +// a mask that keeps all the bits set except for the least significant bits +// that represent the max grain size +#define MAX_GRAIN_MASK (~(uintptr_t)(MAX_GRAIN_SIZE-1)) + +#define MALLOC_ALIGN_SIZE 1 +#define MALLOC_ALIGN_MASK (~(size_t)(MALLOC_ALIGN_SIZE-1)) +#define ALIGN_FOR_MALLOC(size) \ + ((size_t) ((size + (MALLOC_ALIGN_SIZE-1)) & MALLOC_ALIGN_MASK)) + +// If the value is already divisible by MAX_GRAIN_SIZE, return the value; +// otherwise return the previous / next value divisible by MAX_GRAIN_SIZE. +#define ALIGN_BY_PREV_MAX_GRAIN_SIZE(addr) ((uintptr_t) (addr & MAX_GRAIN_MASK)) +#define ALIGN_BY_NEXT_MAX_GRAIN_SIZE(addr) \ + ((uintptr_t) ((addr+(MAX_GRAIN_SIZE-1)) & MAX_GRAIN_MASK)) + +enum GrainType_t { UNINIT = -1, ONE = 0, TWO = 1, FOUR = 2, EIGHT = 3 }; +static const int gtype_to_mem_size[4] = { 1, 2, 4, 8 }; +#define MAX_GTYPE EIGHT // the max value that the enum GrainType_t can take + +// check if addr is aligned with the granularity represented by gtype +#define IS_ALIGNED_WITH_GTYPE(addr, gtype) \ + ((addr & ((uint64_t)gtype_to_mem_size[gtype]-1)) == 0) + +// Instantiation declarations for functions and static members declared in +// cilksan.cpp. +template<> +void (*DisjointSet_t::dtor_callback)(DisjointSet_t *); +template<> +DisjointSet_t *DisjointSet_t::free_list; + +// Struct to hold a pair of disjoint sets corresponding to the last reader and writer +typedef struct MemAccess_t { + + // the containing function of this access + DisjointSet_t *func; + uint64_t rip; // the instruction address of this access + int16_t ref_count; // number of pointers aliasing to this object + // ref_count == 0 if only a single unique pointer to this object exists + + MemAccess_t(DisjointSet_t *_func, uint64_t _rip) + : func(_func), rip(_rip), ref_count(0) + { + func->inc_ref_count(); + } + + ~MemAccess_t() { + func->dec_ref_count(); + } + + // NOTE: curr_pbag may be NULL because we create it lazily. + inline bool races_with(uintptr_t addr, bool on_stack, + DisjointSet_t *curr_pbag) { + bool has_race = false; + // cilksan_assert(curr_vid != UNINIT_VIEW_ID); + + SPBagInterface *lca = func->get_set_node(); + // SPBagInterface *cur_node = func->get_node(); + // we are done if LCA is an S-node. + if (lca->is_PBag()) { + // if memory is allocated on stack, the accesses race with each other + // only if the mem location is allocated in shared ancestor's stack + // if stack_check = false, there is no race. + // if stack_check = true, it's a race only if all other conditions apply. + //bool stack_check = (!on_stack || addr >= cur_node->get_rsp()); + bool stack_check = (!on_stack || addr >= lca->get_rsp()); + has_race = stack_check; + // if (cnt == USER) { + // has_race = stack_check; + // } else { + // cilksan_assert(lca->get_view_id() != UNINIT_VIEW_ID); + // if(cnt == REDUCE) { + // // use the top_pbag's view id as the view id of the REDUCE strand + // curr_vid = curr_top_pbag->get_set_node()->get_view_id(); + // } + // has_race = (lca->get_view_id() != curr_vid) && stack_check; + // } + } + return has_race; + } + + inline int16_t inc_ref_count() { ref_count++; return ref_count; } + inline int16_t dec_ref_count() { ref_count--; return ref_count; } + + // for debugging use + inline friend + std::ostream& operator<<(std::ostream & ostr, MemAccess_t *acc) { + ostr << "function: " << acc->func->get_node()->get_func_id(); + ostr << ", rip " << std::hex << "0x" << acc->rip; + return ostr; + } + +} MemAccess_t; + + +class MemAccessList_t { + +private: + // the smallest addr of memory locations that this MemAccessList represents + + static inline enum GrainType_t mem_size_to_gtype(size_t size) { + switch(size) { + case 8: + return EIGHT; + case 4: + return FOUR; + case 2: + return TWO; + default: // everything else gets byte-granularity + return ONE; + } + } + + // get the start and end indices and gtype to use for accesing the readers / + // writers lists; the gtype is the largest granularity that this memory access + // is aligned with + static enum GrainType_t + get_mem_index(uintptr_t addr, size_t size, int& start, int& end); + + // helper function: break one of the MemAcess list into a smaller granularity; + // called by break_readers/writers_into_smaller_gtype. + // + // for_read: if true, break the readers list, otherwise break the writers + // new_gtype: the desired granularity + void break_list_into_smaller_gtype(bool for_read, enum GrainType_t new_gtype); + + inline void break_readers_into_smaller_gtype(enum GrainType_t new_gtype) { + break_list_into_smaller_gtype(true, new_gtype); + } + + inline void break_writers_into_smaller_gtype(enum GrainType_t new_gtype) { + break_list_into_smaller_gtype(false, new_gtype); + } + + // Check races on memory represented by this mem list with this read access + // Once done checking, update the mem list with this new read access + void check_races_and_update_with_read(uintptr_t inst_addr, uintptr_t addr, + size_t mem_size, bool on_stack, + DisjointSet_t *curr_sbag, + DisjointSet_t *curr_pbag); + + // Check races on memory represented by this mem list with this write access + // Also, update the writers list. Very similar to + // check_races_and_update_with_read function above. + void check_races_and_update_with_write(uintptr_t inst_addr, uintptr_t addr, + size_t mem_size, bool on_stack, + DisjointSet_t *curr_sbag, + DisjointSet_t *curr_pbag); + +public: + + static inline int get_prev_aligned_index(int index, enum GrainType_t gtype) { + + if( IS_ALIGNED_WITH_GTYPE(index, gtype) ) { + return index; + } + return ((index >> gtype) << gtype); + } //sk made public + + uint64_t start_addr; + enum GrainType_t reader_gtype; + MemAccess_t *readers[MAX_GRAIN_SIZE]; + enum GrainType_t writer_gtype; + MemAccess_t *writers[MAX_GRAIN_SIZE]; //sk made public + + // Constructor. + // + // addr: the memory address of the access + // is_read: whether the initializing memory access is a read + // acc: the memory access that causes this MemAccessList_t to be created + // mem_size: the size of the access + MemAccessList_t(uintptr_t addr, bool is_read, + MemAccess_t *acc, size_t mem_size); + + ~MemAccessList_t(); + + // Check races on memory represented by this mem list with this mem access + // Once done checking, update the mem list with the new mem access + // + // is_read: whether this access is a read or not + // in_user_context: whether this access is made by user strand or runtime + // strand (e.g., update / reduce) + // inst_addr: the instruction that performs the read + // addr: the actual memory location accessed + // mem_size: the size of this memory access + // on_stack: whether this access is accessing a memory allocated on stack + // curr_sbag: the SBag of the current function context + // curr_top_pbag: the top-most PBag of the current function context + // curr_view_id: the view id of the currently executing strand, which is the + // same as the view_id stored in the curr_top_pbag, but since we + // create PBag lazily, the curr_top_pbag may be NULL. + inline void + check_races_and_update(bool is_read, uint64_t inst_addr, uint64_t addr, + size_t mem_size, bool on_stack, + DisjointSet_t *curr_sbag, + DisjointSet_t *curr_pbag) { + if (is_read) + check_races_and_update_with_read(inst_addr, addr, mem_size, on_stack, + curr_sbag, curr_pbag); + else + check_races_and_update_with_write(inst_addr, addr, mem_size, on_stack, + curr_sbag, curr_pbag); + } + +}; // end of class MemAccessList_t def + +#endif // _MEM_ACCESS_H diff --git a/compiler-rt/lib/cilksan/print_addr.cpp b/compiler-rt/lib/cilksan/print_addr.cpp new file mode 100644 index 000000000000..167894a211f2 --- /dev/null +++ b/compiler-rt/lib/cilksan/print_addr.cpp @@ -0,0 +1,544 @@ +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "csan.h" +#include "cilksan_internal.h" +#include "debug_util.h" + +// A map keeping track of races found, keyed by the larger instruction address +// involved in the race. Races that have same instructions that made the same +// types of accesses are considered as the the same race (even for races where +// one is read followed by write and the other is write followed by read, they +// are still considered as the same race). Races that have the same instruction +// addresses but different address for memory location is considered as a +// duplicate. The value of the map stores the number duplicates for the given +// race. +typedef std::unordered_multimap RaceMap_t; +static RaceMap_t races_found; +// The number of duplicated races found +static uint32_t duplicated_races = 0; + +// Mappings from CSI ID to associated program counter. +uintptr_t *call_pc = nullptr; +uintptr_t *spawn_pc = nullptr; +uintptr_t *load_pc = nullptr; +uintptr_t *store_pc = nullptr; + +class ProcMapping_t { +public: + unsigned long low,high; + std::string path; + ProcMapping_t(unsigned long low, unsigned long high, std::string path) : + low(low), high(high), path(path) { } +}; + +static std::vector *proc_maps = NULL; + +// declared in cilksan.cpp +extern uint64_t stack_low_addr; +extern uint64_t stack_high_addr; + +void read_proc_maps(void) { + if (proc_maps) return; + + proc_maps = new std::vector; + pid_t pid = getpid(); + char path[100]; + snprintf(path, sizeof(path), "/proc/%d/maps", pid); + DBG_TRACE(DEBUG_BACKTRACE, "path = %s\n", path); + FILE *f = fopen(path, "r"); + DBG_TRACE(DEBUG_BACKTRACE, "file = %p\n", f); + assert(f); + + char *lineptr = NULL; + size_t n; + while (1) { + ssize_t s = getline(&lineptr, &n, f); + if (s==-1) break; + DBG_TRACE(DEBUG_BACKTRACE, "Got %ld line = %s size = %ld\n", + s, lineptr, n); + unsigned long start, end; + char c0, c1, c2, c3; + int off, major, minor, inode; + char *pathname; + sscanf(lineptr, "%lx-%lx %c%c%c%c %x %x:%x %x %ms", + &start, &end, &c0, &c1, &c2, &c3, &off, &major, &minor, + &inode, &pathname); + DBG_TRACE(DEBUG_BACKTRACE, " start = %lx end = %lx path = %s\n", + start, end, pathname); + std::string paths(pathname ? pathname : ""); + ProcMapping_t m(start, end, paths); + //if (paths.compare("[stack]") == 0) { + if (paths.find("[stack") != std::string::npos) { + assert(stack_low_addr == 0); + stack_low_addr = start; + stack_high_addr = end; + DBG_TRACE(DEBUG_BACKTRACE, " stack = %lx--%lx\n", start, end); + } + free(pathname); + proc_maps->push_back(m); + } + DBG_TRACE(DEBUG_BACKTRACE, "proc_maps size = %lu\n", proc_maps->size()); + fclose(f); + if (lineptr) free(lineptr); +} + +void delete_proc_maps() { + if (proc_maps) { + delete proc_maps; + proc_maps = NULL; + } +} + +static void get_info_on_inst_addr(uint64_t addr, int *line_no, std::string *file) { + + for (unsigned int i=0; i < proc_maps->size(); i++) { + if ((*proc_maps)[i].low <= addr && addr < (*proc_maps)[i].high) { + unsigned long off = addr - (*proc_maps)[i].low; + const char *path = (*proc_maps)[i].path.c_str(); + bool is_so = strcmp(".so", path+strlen(path)-3) == 0; + char *command; + if (is_so) { + asprintf(&command, "echo %lx | addr2line -e %s", off, path); + } else { + asprintf(&command, "echo %lx | addr2line -e %s", addr, path); + } + FILE *afile = popen(command, "r"); + if (afile) { + size_t linelen = -1; + char *line = NULL; + if (getline(&line, &linelen, afile)>=0) { + const char *path = strtok(line, ":"); + const char *lno = strtok(NULL, ":"); + *file = std::string(path); + *line_no = atoi(lno); + } + if (line) free(line); + pclose(afile); + } + free(command); + return; + } + } + fprintf(stderr, "%lu is not in range\n", addr); +} + +static std::string +get_info_on_mem_access(uint64_t inst_addr + /*, DisjointSet_t *d*/) { + std::string file; + int32_t line_no; + std::ostringstream convert; + // SPBagInterface *bag = d->get_node(); + // racedetector_assert(bag); + + get_info_on_inst_addr(inst_addr, &line_no, &file); + convert << std::hex << "0x" << inst_addr << std::dec + << ": (" << file << ":" << std::dec << line_no << ")"; + // XXX: Let's not do this for now; maybe come back later + // // convert << "\t called at " << bag->get_call_context(); + + return convert.str(); +} + +typedef enum { + LOAD_ACC, + STORE_ACC, +} ACC_TYPE; + +static std::string +get_info_on_mem_access(const csi_id_t acc_id, ACC_TYPE type) { + std::ostringstream convert; + // assert(UNKNOWN_CSI_ID != acc_id); + + switch (type) { + case LOAD_ACC: + convert << " Read access to "; + break; + case STORE_ACC: + convert << " Write access to "; + break; + } + + // Get object information + const obj_source_loc_t *obj_src_loc = nullptr; + if (UNKNOWN_CSI_ID != acc_id) { + switch (type) { + case LOAD_ACC: + obj_src_loc = __csan_get_load_obj_source_loc(acc_id); + break; + case STORE_ACC: + obj_src_loc = __csan_get_store_obj_source_loc(acc_id); + break; + } + } + if (obj_src_loc && + obj_src_loc->filename && + obj_src_loc->name) { + std::string variable(obj_src_loc->name); + std::string filename(obj_src_loc->filename); + int32_t line_no = obj_src_loc->line_number; + + convert << variable + << " (declared at " << filename + << ":" << std::dec << line_no << ")"; + } else { + convert << ""; + } + + convert << std::endl << " from "; + + // Get PC for this access. + if (UNKNOWN_CSI_ID != acc_id) { + switch (type) { + case LOAD_ACC: + convert << "0x" << std::hex << load_pc[acc_id]; + break; + case STORE_ACC: + convert << "0x" << std::hex << store_pc[acc_id]; + break; + } + } + + // Get source information. + const csan_source_loc_t *src_loc = nullptr; + if (UNKNOWN_CSI_ID != acc_id) { + switch (type) { + case LOAD_ACC: + src_loc = __csan_get_load_source_loc(acc_id); + // std::cerr << "Load src loc for " << acc_id; + break; + case STORE_ACC: + src_loc = __csan_get_store_source_loc(acc_id); + // std::cerr << "Store src loc for " << acc_id; + break; + } + } + // if (!src_loc) + // std::cerr << " is null\n"; + // else if (!src_loc->filename) + // std::cerr << " has null filename\n"; + // else if (!src_loc->name) + // std::cerr << " has null function name\n"; + // else + // std::cerr << " is valid\n"; + + if (src_loc && + src_loc->filename && + src_loc->name) { + std::string file(src_loc->filename); + std::string funcname(src_loc->name); + int32_t line_no = src_loc->line_number; + int32_t col_no = src_loc->column_number; + + // switch (type) { + // case LOAD_ACC: + // convert << "LOAD_ID " << std::dec << acc_id; + // break; + // case STORE_ACC: + // convert << "STORE_ID " << std::dec << acc_id; + // break; + // } + convert << " " << funcname; + convert << " " << file + << ":" << std::dec << line_no + << ":" << std::dec << col_no; + } else + convert << " "; + + return convert.str(); +} + +static std::string +get_info_on_call(const CallID_t &call) { + std::ostringstream convert; + switch (call.getType()) { + case CALL: + convert << " Called from "; + break; + case SPAWN: + convert << "Spawned from "; + break; + } + + if (UNKNOWN_CSI_ID == call.getID()) { + convert << ""; + return convert.str(); + } + + uintptr_t pc = (uintptr_t)nullptr; + switch (call.getType()) { + case CALL: + pc = call_pc[call.getID()]; + break; + case SPAWN: + pc = spawn_pc[call.getID()]; + break; + } + convert << "0x" << std::hex << pc; + + const csan_source_loc_t *src_loc = nullptr; + switch (call.getType()) { + case CALL: + src_loc = __csan_get_call_source_loc(call.getID()); + break; + case SPAWN: + src_loc = __csan_get_detach_source_loc(call.getID()); + break; + } + if (src_loc && + src_loc->filename && + src_loc->name) { + std::string file(src_loc->filename); + std::string funcname(src_loc->name); + int32_t line_no = src_loc->line_number; + int32_t col_no = src_loc->column_number; + convert << " " << funcname; + convert << " " << file + << ":" << std::dec << line_no + << ":" << std::dec << col_no; + } else { + convert << " "; + } + + return convert.str(); +} + +int get_call_stack_divergence_pt( + const std::unique_ptr &first_call_stack, + int first_call_stack_size, + const std::unique_ptr &second_call_stack, + int second_call_stack_size) { + int i; + int end = + (first_call_stack_size < second_call_stack_size) ? + first_call_stack_size : second_call_stack_size; + for (i = 0; i < end; ++i) + if (first_call_stack[i] != second_call_stack[i]) + break; + return i; +} + +extern void print_current_function_info(); + +static void print_race_info(const RaceInfo_t& race) { + std::cerr << "Race detected at address " + // << (is_on_stack(race.addr) ? "stack address " : "address ") + << std::hex << "0x" << race.addr << std::dec << std::endl; + + // std::string first_acc_info = get_info_on_mem_access(race.first_inst); + // std::string second_acc_info = get_info_on_mem_access(race.second_inst); + + std::string first_acc_info, second_acc_info; + switch(race.type) { + case RW_RACE: + first_acc_info = + get_info_on_mem_access(race.first_inst.getID(), LOAD_ACC); + second_acc_info = + get_info_on_mem_access(race.second_inst.getID(), STORE_ACC); + break; + case WW_RACE: + first_acc_info = + get_info_on_mem_access(race.first_inst.getID(), STORE_ACC); + second_acc_info = + get_info_on_mem_access(race.second_inst.getID(), STORE_ACC); + break; + case WR_RACE: + first_acc_info = + get_info_on_mem_access(race.first_inst.getID(), STORE_ACC); + second_acc_info = + get_info_on_mem_access(race.second_inst.getID(), LOAD_ACC); + break; + } + + // Extract the two call stacks + int first_call_stack_size = race.first_inst.call_stack.size(); + std::unique_ptr first_call_stack( + new CallID_t[first_call_stack_size]); + { + call_stack_node_t *call_stack_node = race.first_inst.call_stack.tail; + for (int i = first_call_stack_size - 1; + i >= 0; + --i, call_stack_node = call_stack_node->prev) { + first_call_stack[i] = call_stack_node->id; + } + } + int second_call_stack_size = race.second_inst.call_stack.size(); + std::unique_ptr second_call_stack( + new CallID_t[second_call_stack_size]); + { + call_stack_node_t *call_stack_node = race.second_inst.call_stack.tail; + for (int i = second_call_stack_size - 1; + i >= 0; + --i, call_stack_node = call_stack_node->prev) { + second_call_stack[i] = call_stack_node->id; + } + } + + // Determine where the two call stacks diverge + int divergence = get_call_stack_divergence_pt( + first_call_stack, + first_call_stack_size, + second_call_stack, + second_call_stack_size); + + // Print the two accesses involved in the race + switch(race.type) { + case RW_RACE: + std::cerr << first_acc_info << std::endl; + for (int i = first_call_stack_size - 1; + i >= divergence; --i) + std::cerr << " " << get_info_on_call(first_call_stack[i]) + << std::endl; + std::cerr << second_acc_info << std::endl; + for (int i = second_call_stack_size - 1; + i >= divergence; --i) + std::cerr << " " << get_info_on_call(second_call_stack[i]) + << std::endl; + break; + + case WW_RACE: + std::cerr << first_acc_info << std::endl; + for (int i = first_call_stack_size - 1; + i >= divergence; --i) + std::cerr << " " << get_info_on_call(first_call_stack[i]) + << std::endl; + std::cerr << second_acc_info << std::endl; + for (int i = second_call_stack_size - 1; + i >= divergence; --i) + std::cerr << " " << get_info_on_call(second_call_stack[i]) + << std::endl; + break; + + case WR_RACE: + std::cerr << first_acc_info << std::endl; + for (int i = first_call_stack_size - 1; + i >= divergence; --i) + std::cerr << " " << get_info_on_call(first_call_stack[i]) + << std::endl; + std::cerr << second_acc_info << std::endl; + for (int i = second_call_stack_size - 1; + i >= divergence; --i) + std::cerr << " " << get_info_on_call(second_call_stack[i]) + << std::endl; + break; + } + + if (divergence > 0) { + std::cerr << " Common calling context" << std::endl; + for (int i = divergence - 1; i >= 0; --i) + std::cerr << " " << get_info_on_call(first_call_stack[i]) + << std::endl; + } + + std::cerr << std::endl; + + // print_current_function_info(); +} + +// Log the race detected +void report_race(const AccessLoc_t &first_inst, AccessLoc_t &&second_inst, + uint64_t addr, enum RaceType_t race_type) { + bool found = false; + uint64_t key = + first_inst < second_inst ? + first_inst.getID() : second_inst.getID(); + RaceInfo_t race(first_inst, std::move(second_inst), addr, race_type); + + std::pair range; + range = races_found.equal_range(key); + while (range.first != range.second) { + const RaceInfo_t& in_map = range.first->second; + if (race.is_equivalent_race(in_map)) { + found = true; + break; + } + range.first++; + } + if (found) { // increment the dup count + // std::cerr << "REDUNDANT "; + // print_race_info(race); + duplicated_races++; + } else { + // have to get the info before user program exits + print_race_info(race); + races_found.insert(std::make_pair(key, race)); + } +} + +// Report viewread race +void report_viewread_race(uint64_t first_inst, uint64_t second_inst, + uint64_t addr) { + // For now, just print the viewread race + std::cerr << "Race detected at address " + // << (is_on_stack(race.addr) ? "stack address " : "address ") + << std::hex << "0x" << addr << std::dec << std::endl; + std::string first_acc_info = get_info_on_mem_access(first_inst); + std::string second_acc_info = get_info_on_mem_access(second_inst); + std::cerr << " read access at " << first_acc_info << std::endl; + std::cerr << " write access at " << second_acc_info << std::endl; + std::cerr << std::endl; +} + +int get_num_races_found() { + return races_found.size(); +} + +void print_race_report() { + std::cerr << std::endl; + std::cerr << "Race detector detected total of " << races_found.size() + << " races." << std::endl; + std::cerr << "Race detector suppressed " << duplicated_races + << " duplicate error messages " << std::endl; + std::cerr << std::endl; + +} + +void print_addr(FILE *f, void *a) { + read_proc_maps(); + unsigned long ai = (long)a; + DBG_TRACE(DEBUG_BACKTRACE, "print addr = %p.\n", a); + + for (unsigned int i=0; i < proc_maps->size(); i++) { + DBG_TRACE(DEBUG_BACKTRACE, "Comparing %lu to %lu:%lu.\n", + ai, (*proc_maps)[i].low, (*proc_maps)[i].high); + if ((*proc_maps)[i].low <= ai && ai < (*proc_maps)[i].high) { + unsigned long off = ai-(*proc_maps)[i].low; + const char *path = (*proc_maps)[i].path.c_str(); + DBG_TRACE(DEBUG_BACKTRACE, + "%p is offset 0x%lx in %s\n", a, off, path); + bool is_so = strcmp(".so", path+strlen(path)-3) == 0; + char *command; + if (is_so) { + asprintf(&command, "echo %lx | addr2line -e %s", off, path); + } else { + asprintf(&command, "echo %lx | addr2line -e %s", ai, path); + } + DBG_TRACE(DEBUG_BACKTRACE, "Doing system(\"%s\");\n", command); + FILE *afile = popen(command, "r"); + if (afile) { + size_t linelen = -1; + char *line = NULL; + while (getline(&line, &linelen, afile)>=0) { + fputs(line, f); + } + if (line) free(line); + pclose(afile); + } + free(command); + return; + } + } + fprintf(stderr, "%p is not in range\n", a); +} + diff --git a/compiler-rt/lib/cilksan/race_detect_update.h b/compiler-rt/lib/cilksan/race_detect_update.h new file mode 100644 index 000000000000..dd09146fb4c0 --- /dev/null +++ b/compiler-rt/lib/cilksan/race_detect_update.h @@ -0,0 +1,70 @@ +// -*- C++ -*- +#ifndef __RACE_DETECT_UPDATE__ +#define __RACE_DETECT_UPDATE__ + +#include "csan.h" +#include "shadow_mem.h" + +// Check races on memory represented by this mem list with this read access. +// Once done checking, update the mem list with this new read access. +void check_races_and_update_with_read(const csi_id_t acc_id, + uintptr_t addr, + size_t mem_size, bool on_stack, + FrameData_t *f, + const call_stack_t &call_stack, + Shadow_Memory &shadow_memory) { + shadow_memory.update_with_read(acc_id, addr, mem_size, on_stack, + f, call_stack); + shadow_memory.check_race_with_prev_write(true, acc_id, addr, + mem_size, on_stack, f, call_stack); +} + +// Check races on memory represented by this mem list with this write access. +// Also, update the writers list. Very similar to +// check_races_and_update_with_read function above. +void check_races_and_update_with_write(const csi_id_t acc_id, + uintptr_t addr, + size_t mem_size, bool on_stack, + FrameData_t *f, + const call_stack_t &call_stack, + Shadow_Memory& shadow_memory) { + // shadow_memory.check_race_with_prev_write(false, acc_id, inst_addr, addr, + // mem_size, on_stack, f, + // call_stack); + // shadow_memory.update_with_write(acc_id, inst_addr, addr, mem_size, on_stack, + // f, call_stack); + shadow_memory.check_and_update_write(acc_id, addr, mem_size, + on_stack, f, call_stack); + shadow_memory.check_race_with_prev_read(acc_id, addr, mem_size, + on_stack, f, call_stack); +} + +// Check races on memory represented by this mem list with this mem access. +// Once done checking, update the mem list with the new mem access. +// +// is_read: whether this access is a read or not +// in_user_context: whether this access is made by user strand or runtime +// strand (e.g., update / reduce) +// inst_addr: the instruction that performs the read +// addr: the actual memory location accessed +// mem_size: the size of this memory access +// on_stack: whether this access is accessing a memory allocated on stack +// curr_sbag: the SBag of the current function context +// curr_top_pbag: the top-most PBag of the current function context +// curr_view_id: the view id of the currently executing strand, which is the +// same as the view_id stored in the curr_top_pbag, but since we +// create PBag lazily, the curr_top_pbag may be NULL. +inline void +check_races_and_update(bool is_read, const csi_id_t acc_id, + uintptr_t addr, size_t mem_size, bool on_stack, + FrameData_t *f, const call_stack_t &call_stack, + Shadow_Memory& shadow_memory) { + if (is_read) + check_races_and_update_with_read(acc_id, addr, mem_size, + on_stack, f, call_stack, shadow_memory); + else + check_races_and_update_with_write(acc_id, addr, mem_size, + on_stack, f, call_stack, shadow_memory); +} + +#endif // __RACE_DETECT_UPDATE__ diff --git a/compiler-rt/lib/cilksan/shadow_mem.cpp b/compiler-rt/lib/cilksan/shadow_mem.cpp new file mode 100644 index 000000000000..6289c5310907 --- /dev/null +++ b/compiler-rt/lib/cilksan/shadow_mem.cpp @@ -0,0 +1,17 @@ +#include "shadow_mem.h" +// #include "simple_shadow_mem.h" +// #include "hash_shadow_mem.h" +#include "compresseddict_shadow_mem.h" +// #include "noop_shadow_mem.h" + +void Shadow_Memory::init() +{ + type = 2; + std::cout << "shadow memory type is: " << type << std::endl; + switch(type) { + // case 0: shadow_mem = new Simple_Shadow_Memory(); break; + // case 1: shadow_mem = new Hash_Shadow_Memory(); break; + case 2: shadow_mem = new CompressedDictShadowMem(); break; + // case 3: shadow_mem = new Noop_Shadow_Memory(); break; + } +} diff --git a/compiler-rt/lib/cilksan/shadow_mem.h b/compiler-rt/lib/cilksan/shadow_mem.h new file mode 100644 index 000000000000..c10c79a37fef --- /dev/null +++ b/compiler-rt/lib/cilksan/shadow_mem.h @@ -0,0 +1,135 @@ +// -*- C++ -*- +#ifndef __SHADOW_MEM__ +#define __SHADOW_MEM__ + +#include "cilksan_internal.h" +#include "csan.h" +#include "debug_util.h" +#include "disjointset.h" +#include "frame_data.h" +#include "spbag.h" +#include "stack.h" + +class ShadowMemoryType { +public: + virtual void insert_access(bool is_read, const csi_id_t acc_id, + uintptr_t addr, + size_t mem_size, FrameData_t *f, + const call_stack_t &call_stack) = 0; + + virtual bool does_access_exists(bool is_read, uintptr_t addr, + size_t mem_size) = 0; + + virtual void clear(size_t start, size_t end) = 0; + + virtual void check_race_with_prev_read(const csi_id_t acc_id, + uintptr_t addr, + size_t mem_size, bool on_stack, + FrameData_t *f, + const call_stack_t &call_stack) = 0; + + virtual void check_race_with_prev_write(bool is_read, const csi_id_t acc_id, + uintptr_t addr, + size_t mem_size, bool on_stack, + FrameData_t *f, + const call_stack_t &call_stack) = 0; + + virtual void update_with_write(const csi_id_t acc_id, + uintptr_t addr, size_t mem_size, + bool on_stack, FrameData_t *f, + const call_stack_t &call_stack) = 0; + + virtual void update_with_read(const csi_id_t acc_id, + uintptr_t addr, size_t mem_size, + bool on_stack, FrameData_t *f, + const call_stack_t &call_stack) = 0; + + virtual void check_and_update_write(const csi_id_t acc_id, + uintptr_t addr, size_t mem_size, + bool on_stack, FrameData_t *f, + const call_stack_t &call_stack) = 0; + + virtual void destruct() = 0; + + virtual ~ShadowMemoryType() = 0; +}; + +inline ShadowMemoryType::~ShadowMemoryType() {} + +// to be performance-engineered later +class Shadow_Memory { + short type; // hash + ShadowMemoryType* shadow_mem; + +public: + void test(){} + + void init(); + + // Inserts access, and replaces any that are already in the shadow memory. + void insert_access(bool is_read, const csi_id_t acc_id, + uintptr_t addr, size_t mem_size, FrameData_t *f, + const call_stack_t &call_stack) { + shadow_mem->insert_access(is_read, acc_id, addr, mem_size, f, + call_stack); + } + + // Returns true if ANY bytes between addr and addr+mem_size are in the shadow memory. + bool does_access_exists (bool is_read, uintptr_t addr, size_t mem_size){ + return shadow_mem->does_access_exists(is_read, addr, mem_size); + } + + void clear(size_t start, size_t size) { + shadow_mem->clear(start, size); + } + + void check_race_with_prev_read(const csi_id_t acc_id, + uintptr_t addr, size_t mem_size, bool on_stack, + FrameData_t *f, + const call_stack_t &call_stack) { + shadow_mem->check_race_with_prev_read(acc_id, addr, mem_size, + on_stack, f, call_stack); + } + + void check_race_with_prev_write(bool is_read, const csi_id_t acc_id, + uintptr_t addr, + size_t mem_size, bool on_stack, + FrameData_t *f, + const call_stack_t &call_stack) { + shadow_mem->check_race_with_prev_write(is_read, acc_id, addr, + mem_size, on_stack, f, + call_stack); + } + + void update_with_write(const csi_id_t acc_id, + uintptr_t addr, size_t mem_size, bool on_stack, + FrameData_t *f, const call_stack_t &call_stack) { + shadow_mem->update_with_write(acc_id, addr, mem_size, on_stack, + f, call_stack); + } + + void update_with_read(const csi_id_t acc_id, + uintptr_t addr, size_t mem_size, bool on_stack, + FrameData_t *f, const call_stack_t &call_stack) { + shadow_mem->update_with_read(acc_id, addr, mem_size, on_stack, + f, call_stack); + } + + void check_and_update_write(const csi_id_t acc_id, + uintptr_t addr, size_t mem_size, bool on_stack, + FrameData_t *f, const call_stack_t &call_stack) { + shadow_mem->check_and_update_write(acc_id, addr, mem_size, on_stack, + f, call_stack); + } + + void destruct() { + if (shadow_mem) { + delete shadow_mem; + shadow_mem = nullptr; + } + } + + ~Shadow_Memory() {destruct();} +}; + +#endif // __SHADOW_MEM__ diff --git a/compiler-rt/lib/cilksan/spbag.h b/compiler-rt/lib/cilksan/spbag.h new file mode 100644 index 000000000000..5ed679c588af --- /dev/null +++ b/compiler-rt/lib/cilksan/spbag.h @@ -0,0 +1,195 @@ +/* -*- Mode: C++ -*- */ + +#ifndef _SPBAG_H +#define _SPBAG_H + +#include +#include + +#include +#include +#include +#include + +#include "debug_util.h" +#include "disjointset.h" +#include "cilksan_internal.h" + + +class SPBagInterface { +public: + // Note to self: base class must declare a virtual destructor; it does not + // have to be pure and must provide a definition. + // http://stackoverflow.com/questions/461203/when-to-use-virtual-destructors + // /3336499/virtual-desctructor-on-pure-abstract-base-class + virtual ~SPBagInterface() { } + virtual bool is_SBag() = 0; + virtual bool is_PBag() = 0; + virtual uint64_t get_func_id() = 0; + virtual uint64_t get_rsp() = 0; + virtual void set_rsp(uint64_t stack_ptr) = 0; + // virtual std::string get_call_context() = 0; +}; + + +class SBag_t : public SPBagInterface { +private: + uint64_t _func_id; + // std::string _func_name; + // SBag of the parent function (whether this function is called or spawned) + // SPBagInterface *_parent; + uintptr_t _stack_ptr; + + SBag_t() {} // disable default constructor + +public: + SBag_t(uint64_t id, SPBagInterface *parent) : + _func_id(id), + // _func_name(name), + // _parent(parent), + _stack_ptr(UNINIT_STACK_PTR) { + WHEN_CILKSAN_DEBUG(debug_count++); + } + +#if CILKSAN_DEBUG + static long debug_count; + ~SBag_t() { + debug_count--; + } +#endif + + bool is_SBag() { return true; } + bool is_PBag() { return false; } + + uint64_t get_func_id() { return _func_id; } + + uint64_t get_rsp() { + cilksan_assert(_stack_ptr != UNINIT_STACK_PTR); + return _stack_ptr; + } + void set_rsp(uintptr_t stack_ptr) { _stack_ptr = stack_ptr; } + + // Note to self: Apparently the compiler will generate a default inline + // destructor, and it's better to let the compiler to that than define your + // own empty destructor. This is true even when the parent class has a + // virtual destructor. + // http://stackoverflow.com/questions/827196/virtual-default-destructors-in-c + // ~SBag_t() { fprintf(stderr, "Called SBag destructor.\n"); } + + /* + std::string get_call_context() { + std::string res; + if(_parent) { + res = _func_name + " called in \n" + _parent->get_call_context(); + } else { + res = _func_name + "\n"; + } + return res; + }*/ + + // Simple free-list allocator to conserve space and time in managing + // SBag_t objects. + + // The structure of a node in the SBag free list. + struct FreeNode_t { + FreeNode_t *next; + }; + static FreeNode_t *free_list; + + void *operator new(size_t size) { + if (free_list) { + FreeNode_t *new_node = free_list; + free_list = free_list->next; + return new_node; + } + return ::operator new(size); + } + + void operator delete(void *ptr) { + FreeNode_t *del_node = reinterpret_cast(ptr); + del_node->next = free_list; + free_list = del_node; + } + + static void cleanup_freelist() { + FreeNode_t *node = free_list; + FreeNode_t *next = nullptr; + while (node) { + next = node->next; + ::operator delete(node); + node = next; + } + } +}; + +static_assert(sizeof(SBag_t) >= sizeof(SBag_t::FreeNode_t), + "Node structure in SBag free list must be as large as SBag."); + +class PBag_t : public SPBagInterface { +private: + // the SBag that corresponds to the function instance that holds this PBag + SPBagInterface *_sib_sbag; + + PBag_t() {} // disable default constructor + +public: + PBag_t(SPBagInterface *sib) : + _sib_sbag(sib) { + WHEN_CILKSAN_DEBUG( debug_count++; ); + } + +#if CILKSAN_DEBUG + static long debug_count; + ~PBag_t() { + debug_count--; + } +#endif + + bool is_SBag() { return false; } + bool is_PBag() { return true; } + uint64_t get_func_id() { return _sib_sbag->get_func_id(); } + uint64_t get_rsp() { return _sib_sbag->get_rsp(); } + void set_rsp(uint64_t stack_ptr) { cilksan_assert(0); /* Should never happen; */ } + + /* + std::string get_call_context() { + return _sib_sbag->get_call_context(); + } */ + + // Simple free-list allocator to conserve space and time in managing + // PBag_t objects. + struct FreeNode_t { + FreeNode_t *next; + }; + static FreeNode_t *free_list; + + void *operator new(size_t size) { + if (free_list) { + FreeNode_t *new_node = free_list; + free_list = free_list->next; + return new_node; + } + return ::operator new(size); + } + + void operator delete(void *ptr) { + FreeNode_t *del_node = reinterpret_cast(ptr); + del_node->next = free_list; + free_list = del_node; + } + + static void cleanup_freelist() { + FreeNode_t *node = free_list; + FreeNode_t *next = nullptr; + while (node) { + next = node->next; + ::operator delete(node); + node = next; + } + } +}; + +static_assert(sizeof(PBag_t) >= sizeof(PBag_t::FreeNode_t), + "Node structure in PBag free list must be as large as PBag."); + +#endif // #ifndef _SPBAG_H diff --git a/compiler-rt/lib/cilksan/stack.h b/compiler-rt/lib/cilksan/stack.h new file mode 100644 index 000000000000..a0219fac2be8 --- /dev/null +++ b/compiler-rt/lib/cilksan/stack.h @@ -0,0 +1,148 @@ +/* -*- Mode: C++ -*- */ + +#ifndef _STACK_H +#define _STACK_H + +#include +#include +#include +#include + +#include "debug_util.h" + + +// TB 20130123: I'm using my own custom stack type to let me +// performance engineer this later. +/* + * Stack data structure for storing and maintaining data + * associated with the call stack. + */ +template +class Stack_t { +private: + /* Default capacity for call stack. Tunable to minimize + * resizing. */ + static const uint32_t DEFAULT_CAPACITY = 128; + + /* call stack, implemented as an array of STACK_DATA_T's */ + STACK_DATA_T *_stack; + /* current capacity of call stack */ + uint32_t _capacity; + /* current head of call stack */ + uint32_t _head; + + /* General method to resize the call stack. + * Called by _double_cap() and _halve_cap(). + * + * @param new_capacity New capacity of the call stack. + */ + void _resize(uint32_t new_capacity) { + // Save a pointer to the call stack + STACK_DATA_T *old_stack = _stack; + // Allocate new call stack array + _stack = new STACK_DATA_T[new_capacity]; + // Determine amount to copy over + uint32_t copy_end = _capacity > new_capacity ? new_capacity : _capacity; + + // Copy contents of old call stack + for (uint32_t i = 0; i < copy_end; ++i) { + _stack[i] = old_stack[i]; + } + _capacity = new_capacity; + + // Delete old call_stack + delete[] old_stack; + } + + /* + * Doubles the capacity of the call stack. + */ + void _double_cap() { _resize(_capacity * 2); } + + /* + * Halves the capacity of the call stack. + */ + void _halve_cap() { _resize(_capacity / 2); } + + +public: + /* + * Default constructor. + */ + Stack_t() : + _capacity(DEFAULT_CAPACITY), + _head(0) + { _stack = new STACK_DATA_T[_capacity]; } + + /* + * Destructor. + */ + ~Stack_t() { delete[] _stack; } + + /* + * Simulate entering a function. Effectively pushes a new + * STACK_DATA_T onto the head of the call stack. + */ + void push() { + ++_head; + + if (_head == _capacity) { + _double_cap(); + } + } + + /* + * Simulate exiting a function. Effectively pops the head + * STACK_DATA_T off of the stack. + */ + void pop() { + --_head; + if (_capacity > DEFAULT_CAPACITY && _head < _capacity / 2) { + _halve_cap(); + } + } + + /* + * Retrieves an arbitrary ancestor's STACK_DATA_T, specifically a + * pointer to that data on the call stack. + * + * @param i the ancestor for the call at the head of the stack, + * where i = 0 indicates the head of the call stack. + */ + STACK_DATA_T* ancestor(uint32_t i) const { + assert(i <= _head); + cilksan_assert(_head < _capacity); + return &(_stack[_head - i]); + } + + /* + * Retrieves a STACK_DATA_T at index i, specifically a + * pointer to that data on the call stack. + * + * @param i the index of the stack element, + * where element at index 0 is the oldest element. + */ + STACK_DATA_T* at(uint32_t i) const { + assert(i >= 0 && i <= _head); + cilksan_assert(_head < _capacity); + return &(_stack[i]); + } + + /* + * Retrieves the STACK_DATA_T at the head of the call stack. + */ + STACK_DATA_T* head() const { + return ancestor(0); + } + + /* + * Returns the current size of the stack, i.e. the number of entries + * on the stack. + */ + uint32_t size() const { + return _head + 1; + } + +}; + +#endif // #define _STACK_H diff --git a/compiler-rt/lib/cilksan/static_dictionary.cpp b/compiler-rt/lib/cilksan/static_dictionary.cpp new file mode 100644 index 000000000000..a7774089e0ce --- /dev/null +++ b/compiler-rt/lib/cilksan/static_dictionary.cpp @@ -0,0 +1,576 @@ +#include "static_dictionary.h" +#include + +// Definitions of static members +const value_type00 Dictionary::null_val = value_type00(); + +int n_threads(void) +{ + struct stat task_stat; + if (stat("/proc/self/task", &task_stat)) + return -1; + + return task_stat.st_nlink - 2; +} + +LRU_List::LRU_List() { + head = NULL; + tail = NULL; + cache_size = 0; + + for (int i = 0; i < MAX_CACHE_SIZE; i++) { + free_list[i].page = NULL; + free_list[i].next = NULL; + free_list[i].previous = NULL; + } +} + +// Debugging +void LRU_List::print_lru() { + std::cout << "Forward:" << std::endl; + LRU_Node *current = head; + std::cout << head << std::endl; + while (current != tail) { + std::cout << current->page->page_id << " -> "; + current = current->next; + } + std::cout << current->page->page_id << std::endl; + std::cout << "Backward:" << std::endl; + current = tail; + while (current != head) { + std::cout << current->page->page_id << " -> "; + current = current->previous; + } + std::cout << current->page->page_id << std::endl; +} + +void LRU_List::check_invariants(int label) { + LRU_Node *current = head; + if (cache_size == 0) { + assert(head == NULL && tail == NULL); + return; + } + assert(head->previous == NULL); + assert(tail->next == NULL); + + int counter = 1; + while (current != tail) { + assert(current->next); + assert(current->next->previous == current); + current = current->next; + counter++; + assert(counter <= MAX_CACHE_SIZE); + } + assert(lru_page_ids.size() == counter); + assert(counter == cache_size && counter == lru_page_ids.size()); +} + +void LRU_List::access(Page_t *page) { + // std::cerr << __PRETTY_FUNCTION__; + // Check if in LRU. If it is, move to front and return. + // If not then decompress, add to LRU, if max size then compress and remove last element. + assert(cache_size <= MAX_CACHE_SIZE); + uint64_t page_id = page->page_id; + if (head && head->page->page_id == page_id) { + // std::cerr << " accessed head\n"; + return; + } + if (lru_page_ids.count(page_id) == 0) { + // std::cerr << " decompressing a page\n"; + // Decompress + page->decompress(); + if (cache_size == MAX_CACHE_SIZE) { + // std::cerr << " Must compress LRU page\n"; + uint64_t page_to_remove = tail->page->page_id; + tail->page->compress(); + //tail->page->fut = std::async(&Page_t::compress, tail->page); + tail->page = NULL; + tail = tail->previous; + tail->next = NULL; + + LRU_Node *recycled = lru_page_ids[page_to_remove]; + lru_page_ids.erase(page_to_remove); + // int num_removed = lru_page_ids.erase(page_to_remove); + lru_page_ids.insert({page_id, recycled}); + + recycled->page = page; + recycled->next = head; + recycled->previous = NULL; + head = recycled; + assert(head->next); + head->next->previous = head; + } else { + // std::cerr << " Free space in LRU list\n"; + LRU_Node *new_node = &(free_list[cache_size]); + lru_page_ids[page_id] = new_node; + new_node->page = page; + new_node->next = head; + new_node->previous = NULL; + if (head == NULL) { + tail = new_node; + } + head = new_node; + if (head->next) { + head->next->previous = head; + } + cache_size++; + } + } else { + // std::cerr << " found the page in LRU\n"; + // Page found. + LRU_Node *node = lru_page_ids[page_id]; + if (node->previous != NULL) { + node->previous->next = node->next; + } else { + return; // Already first element. + } + if (tail == node) { + tail = node->previous; + } + + if (node->next != NULL) { + node->next->previous = node->previous; + } + + node->next = head; + node->previous = NULL; + node->next->previous = node; + head = node; + } +} + +Page_t *LRU_List::find_after_head(uint64_t page_id) { + // Scan the list for the page + if (!head) + return nullptr; + int count = 0; + LRU_Node *node = head->next; + while (__builtin_expect((node && count < MAX_CACHE_SCAN_LENGTH && + node->page->page_id != page_id), 0)) { + node = node->next; + count++; + } + if (node && node->page->page_id == page_id) { + assert(node->previous != nullptr); + // Move the node to the front of the list. + node->previous->next = node->next; + if (tail == node) + tail = node->previous; + if (node->next != NULL) + node->next->previous = node->previous; + node->next = head; + node->previous = NULL; + node->next->previous = node; + head = node; + + return node->page; + } + return nullptr; +} + +Static_Dictionary::Static_Dictionary() { +} + +// const value_type00 * +// Static_Dictionary::find_group(uint64_t key, size_t max_size, size_t &num_elems) { +// uint64_t page_id = GET_PAGE_ID(key); +// size_t my_num_elems = 1; +// if (__builtin_expect((lru_list.head && lru_list.head->page->page_id == page_id), 1)) { +// const value_type00 *acc = &lru_list.head->page->buffer[GET_PAGE_OFFSET(key)]; +// #pragma unroll(4) +// for (int i = 1; i < max_size; i++) { +// const value_type00 *next = &acc[i]; +// if (__builtin_expect(next->getFunc() != acc->getFunc(), 0)) { +// num_elems = my_num_elems; +// return acc; +// } else { +// my_num_elems++; +// } +// } +// num_elems = my_num_elems; +// return acc; +// } +// auto found_page = page_table.find(page_id); +// if (found_page != page_table.end()) { +// lru_list.access(found_page->second); +// const value_type00 *acc = &found_page->second->buffer[GET_PAGE_OFFSET(key)]; +// #pragma unroll(4) +// for (int i = 1; i < max_size; i++) { +// const value_type00 *next = &acc[i]; +// if (__builtin_expect(next->getFunc() != acc->getFunc(), 0)) { +// num_elems = my_num_elems; +// return acc; +// } else { +// my_num_elems++; +// } +// } +// num_elems = my_num_elems; +// return acc; +// } else { +// num_elems = max_size; +// return &null_val; +// } +// } + +value_type00 * +Static_Dictionary::find_group(uint64_t key, size_t max_size, + size_t &num_elems) { + uint64_t page_id = GET_PAGE_ID(key); + size_t my_num_elems = 1; + value_type00 *acc = nullptr; + if (__builtin_expect((lru_list.head && + lru_list.head->page->page_id == page_id), 1)) { + acc = &lru_list.head->page->buffer[GET_PAGE_OFFSET(key)]; + } else if (auto *page = lru_list.find_after_head(page_id)) { + acc = &page->buffer[GET_PAGE_OFFSET(key)]; + } else { + auto found_page = page_table.find(page_id); + if (found_page != page_table.end()) { + lru_list.access(found_page->second); + acc = &found_page->second->buffer[GET_PAGE_OFFSET(key)]; + } else { + num_elems = max_size; + return nullptr; + } + } + // #pragma unroll(8) + for (int i = 1; i < max_size; i++) { + const value_type00 *next = &acc[i]; + if (__builtin_expect(next->getFunc() == acc->getFunc(), 1)) { + my_num_elems++; + } else { + break; + } + } + num_elems = my_num_elems; + return acc; +} + +value_type00 * +Static_Dictionary::find_exact_group(uint64_t key, size_t max_size, + size_t &num_elems) { + uint64_t page_id = GET_PAGE_ID(key); + size_t my_num_elems = 1; + value_type00 *acc = nullptr; + if (__builtin_expect((lru_list.head && + lru_list.head->page->page_id == page_id), 1)) { + acc = &lru_list.head->page->buffer[GET_PAGE_OFFSET(key)]; + } else if (auto *page = lru_list.find_after_head(page_id)) { + acc = &page->buffer[GET_PAGE_OFFSET(key)]; + } else { + auto found_page = page_table.find(page_id); + if (found_page != page_table.end()) { + lru_list.access(found_page->second); + acc = &found_page->second->buffer[GET_PAGE_OFFSET(key)]; + } else { + num_elems = max_size; + return nullptr; + } + } + // #pragma unroll(8) + for (int i = 1; i < max_size; i++) { + const value_type00 *next = &acc[i]; + if (__builtin_expect((next->getFunc() == acc->getFunc()) && + next->sameAccessLocPtr(*acc), 1)) { + my_num_elems++; + } else { + break; + } + } + num_elems = my_num_elems; + return acc; +} + +value_type00 *Static_Dictionary::find(uint64_t key) { + uint64_t page_id = GET_PAGE_ID(key); + if (lru_list.head && lru_list.head->page->page_id == page_id) { + return &lru_list.head->page->buffer[GET_PAGE_OFFSET(key)]; + } + if (auto *page = lru_list.find_after_head(page_id)) { + return &page->buffer[GET_PAGE_OFFSET(key)]; + } + auto found_page = page_table.find(page_id); + if (found_page != page_table.end()) { + lru_list.access(found_page->second); + return &found_page->second->buffer[GET_PAGE_OFFSET(key)]; + } else { + return nullptr; + } +} + +const value_type00 &Static_Dictionary::operator[] (uint64_t key) { + value_type00 *found = find(key); + if (!found) + return null_val; + return *found; +} + +void Static_Dictionary::erase(uint64_t key, size_t size) { + while (size > 0) { + size_t eff_size = size; + if (GET_PAGE_OFFSET(key) + size > PAGE_SIZE) { + eff_size = (PAGE_SIZE - GET_PAGE_OFFSET(key)); + } + assert(eff_size <= PAGE_SIZE); + uint64_t page_id = GET_PAGE_ID(key); + value_type00 *acc = nullptr; + if (__builtin_expect((lru_list.head && + lru_list.head->page->page_id == page_id), 1)) { + acc = &(lru_list.head->page->buffer[GET_PAGE_OFFSET(key)]); + } else if (auto *page = lru_list.find_after_head(page_id)) { + acc = &(page->buffer[GET_PAGE_OFFSET(key)]); + } else { + auto found_page = page_table.find(page_id); + if (found_page != page_table.end()) { + // Decompress the page if need be. + lru_list.access(found_page->second); + acc = &(found_page->second->buffer[GET_PAGE_OFFSET(key)]); + } else { + acc = nullptr; + } + } + if (acc) { + size_t group_start = 0; + size_t group_size = 1; + while (group_start + group_size < eff_size) { + while (group_start + group_size < eff_size && + (acc[group_start + group_size].getFunc() == + acc[group_start].getFunc()) && + acc[group_start + group_size].sameAccessLocPtr(acc[group_start])) + group_size++; + if (acc[group_start].isValid()) { + acc[group_start].dec_ref_counts(group_size); + for (size_t i = 0; i < group_size; ++i) + acc[group_start + i].clear(); + } + group_start += group_size; + group_size = 1; + } + // for (size_t i = 0; i < eff_size; ++i) + // if (acc[i].isValid()) + // acc[i].invalidate(); + } + // for (int i = 0; i < eff_size; i++) { + // // if (lru_list.head->page->buffer[GET_PAGE_OFFSET(key) + i].isValid()) { + // // lru_list.head->page->buffer[GET_PAGE_OFFSET(key) + i].invalidate(); + // // } + // } + // return; + // } + // auto found_page = page_table.find(page_id); + // if (found_page != page_table.end()) { + // lru_list.access(found_page->second); + // for (int i = 0; i < eff_size; i++) { + // if (found_page->second->buffer[GET_PAGE_OFFSET(key) + i].isValid()) { + // found_page->second->buffer[GET_PAGE_OFFSET(key) + i].invalidate(); + // } + // } + // } + size -= eff_size; + key += eff_size; + } +} + +void Static_Dictionary::erase(uint64_t key) { + assert(false); +} + +bool Static_Dictionary::includes(uint64_t key, size_t size) { + uint64_t page_id = GET_PAGE_ID(key); + if (__builtin_expect((lru_list.head && + lru_list.head->page->page_id == page_id), 1)) { + const value_type00 *acc = &(lru_list.head->page->buffer[GET_PAGE_OFFSET(key)]); + for (int i = 0; i < size; i++) + if (acc[i].isValid()) + return true; + return false; + } + if (auto *page = lru_list.find_after_head(page_id)) { + const value_type00 *acc = &(page->buffer[GET_PAGE_OFFSET(key)]); + for (int i = 0; i < size; i++) + if (acc[i].isValid()) + return true; + return false; + } + auto found_page = page_table.find(page_id); + if (found_page != page_table.end()) { + lru_list.access(found_page->second); + const value_type00 *acc = &(found_page->second->buffer[GET_PAGE_OFFSET(key)]); + for (int i = 0; i < size; i++) + if (acc[i].isValid()) + return true; + return false; + } else { + return false; + } +} + +bool Static_Dictionary::includes(uint64_t key) { + uint64_t page_id = GET_PAGE_ID(key); + if (lru_list.head && lru_list.head->page->page_id == page_id) { + return lru_list.head->page->buffer[GET_PAGE_OFFSET(key)].isValid(); + } + if (auto *page = lru_list.find_after_head(page_id)) { + return page->buffer[GET_PAGE_OFFSET(key)].isValid(); + } + auto found_page = page_table.find(page_id); + if (found_page != page_table.end()) { + lru_list.access(found_page->second); + return found_page->second->buffer[GET_PAGE_OFFSET(key)].isValid(); + } else { + return false; + } +} + +void Static_Dictionary::insert(uint64_t key, size_t size, const value_type00 &f) { + value_type00 tmp(f); + tmp.inc_ref_counts(size); + uint64_t page_id = GET_PAGE_ID(key); + if (lru_list.head && lru_list.head->page->page_id == page_id) { + for (int i = 0; i < size; i++) { + // lru_list.head->page->buffer[GET_PAGE_OFFSET(key) + i] = f; + lru_list.head->page->buffer[GET_PAGE_OFFSET(key) + i].inherit(tmp); + } + return; + } + if (auto *page = lru_list.find_after_head(page_id)) { + for (int i = 0; i < size; i++) { + page->buffer[GET_PAGE_OFFSET(key) + i].inherit(tmp); + } + return; + } + auto found_page = page_table.find(page_id); + if (found_page != page_table.end()) { + lru_list.access(found_page->second); + for (int i = 0; i < size; i++) { + // found_page->second->buffer[GET_PAGE_OFFSET(key) + i] = f; + found_page->second->buffer[GET_PAGE_OFFSET(key) + i].inherit(tmp); + } + } else { + Page_t *page = new Page_t(page_id); + page_table[page_id] = page; + lru_list.access(page); + for (int i = 0; i < size; i++) { + // page->buffer[GET_PAGE_OFFSET(key) + i] = f; + page->buffer[GET_PAGE_OFFSET(key) + i].inherit(tmp); + } + } +} + +void Static_Dictionary::insert(uint64_t key, const value_type00 &f) { + uint64_t page_id = GET_PAGE_ID(key); + if (lru_list.head && lru_list.head->page->page_id == page_id) { + lru_list.head->page->buffer[GET_PAGE_OFFSET(key)] = f; + return; + } + if (auto *page = lru_list.find_after_head(page_id)) { + page->buffer[GET_PAGE_OFFSET(key)] = f; + return; + } + auto found_page = page_table.find(page_id); + if (found_page != page_table.end()) { + lru_list.access(found_page->second); + found_page->second->buffer[GET_PAGE_OFFSET(key)] = f; + } else { + Page_t *page = new Page_t(page_id); + page_table[page_id] = page; + lru_list.access(page); + page->buffer[GET_PAGE_OFFSET(key)] = f; + } +} + +void Static_Dictionary::set(uint64_t key, size_t size, value_type00 &&f) { + f.inc_ref_counts(size); + while (size > 0) { + size_t eff_size = size; + if (GET_PAGE_OFFSET(key) + size > PAGE_SIZE) { + eff_size = PAGE_SIZE - GET_PAGE_OFFSET(key); + } + uint64_t page_id = GET_PAGE_ID(key); + value_type00 *acc = nullptr; + if (__builtin_expect((lru_list.head && + lru_list.head->page->page_id == page_id), 1)) { + acc = &(lru_list.head->page->buffer[GET_PAGE_OFFSET(key)]); + } else if (auto *page = lru_list.find_after_head(page_id)) { + acc = &(page->buffer[GET_PAGE_OFFSET(key)]); + } else { + auto found_page = page_table.find(page_id); + if (__builtin_expect(found_page != page_table.end(), 1)) { + lru_list.access(found_page->second); + acc = &(found_page->second->buffer[GET_PAGE_OFFSET(key)]); + } else { + Page_t *page = new Page_t(page_id); + page_table[page_id] = page; + lru_list.access(page); + acc = &(page->buffer[GET_PAGE_OFFSET(key)]); + for (int i = 0; i < eff_size; ++i) + acc[i].overwrite(f); + acc = nullptr; + } + } + if (acc) { + size_t group_start = 0; + size_t group_size = 1; + while (group_start + group_size < eff_size) { + while (group_start + group_size < eff_size && + (acc[group_start + group_size].getFunc() == + acc[group_start].getFunc()) && + acc[group_start + group_size].sameAccessLocPtr(acc[group_start])) + group_size++; + if (acc[group_start].isValid()) + acc[group_start].dec_ref_counts(group_size); + + // for (size_t i = 0; i < group_size; ++i) + // acc[group_start + i].overwrite(f); + group_start += group_size; + group_size = 1; + } + for (size_t i = 0; i < eff_size; ++i) + acc[i].overwrite(f); + } + + // for (int i = 0; i < eff_size; i++) { + // // if (lru_list.head->page->buffer[GET_PAGE_OFFSET(key) + i].isValid()) { + // // lru_list.head->page->buffer[GET_PAGE_OFFSET(key) + i].invalidate(); + // // } + // } + // return; + // } + // auto found_page = page_table.find(page_id); + // if (found_page != page_table.end()) { + // lru_list.access(found_page->second); + // for (int i = 0; i < eff_size; i++) { + // if (found_page->second->buffer[GET_PAGE_OFFSET(key) + i].isValid()) { + // found_page->second->buffer[GET_PAGE_OFFSET(key) + i].invalidate(); + // } + // } + // } + size -= eff_size; + key += eff_size; + } +} + +void Static_Dictionary::insert_into_found_group(uint64_t key, size_t size, + value_type00 *dst, + value_type00 &&f) { + f.inc_ref_counts(size); + uint64_t page_id = GET_PAGE_ID(key); + assert(lru_list.head && lru_list.head->page->page_id == page_id); + assert(dst == &lru_list.head->page->buffer[GET_PAGE_OFFSET(key)]); + // value_type00 *my_dst = &lru_list.head->page->buffer[GET_PAGE_OFFSET(key)]; + value_type00 *my_dst = dst; + // Overwrite the table entries. + for (int i = 0; i < size; i++) { + my_dst[i].overwrite(f); + } +} + +Static_Dictionary::~Static_Dictionary() { + for (auto iter = page_table.begin(); iter != page_table.end(); iter++) { + delete iter->second; + } + page_table.clear(); + lru_list.lru_page_ids.clear(); + lru_list.head = NULL; + lru_list.tail = NULL; +} diff --git a/compiler-rt/lib/cilksan/static_dictionary.h b/compiler-rt/lib/cilksan/static_dictionary.h new file mode 100644 index 000000000000..79437943e776 --- /dev/null +++ b/compiler-rt/lib/cilksan/static_dictionary.h @@ -0,0 +1,216 @@ +// -*- C++ -*- +#ifndef __STATIC_DICTIONARY__ +#define __STATIC_DICTIONARY__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +//#include "mem_access.h" +//#include "cilksan_internal.h" +//#include "debug_util.h" +#include "dictionary.h" +#include "disjointset.h" +#include "frame_data.h" +#include "spbag.h" +#include "stack.h" + +#include + +/* Macros for memory pages in our dictionary. */ + +#define PAGE_SIZE_BITS 10 +#define PAGE_SIZE ((uint64_t)(1ULL << PAGE_SIZE_BITS)) +#define GET_PAGE_ID(key) (((uint64_t)(key)) >> PAGE_SIZE_BITS) +#define GET_PAGE_OFFSET(key) (((uint64_t)(key)) & (PAGE_SIZE-1)) +// MAX_CACHE_SIZE amortizes the cost of compressing/decompressing pages. +// #define MAX_CACHE_SIZE 100 +#define MAX_CACHE_SIZE 512 +// MAX_CACHE_SCAN_LENGTH exploits locality to amortize the cost of looking up a +// page from the main structure. +#define MAX_CACHE_SCAN_LENGTH 4 + +class Static_Dictionary; + +struct Page_t { + bool is_compressed; + uint64_t page_id; + + value_type00 *buffer; + + size_t compressed_size; + unsigned char *compressed_buffer; + std::future fut; + + Page_t(uint64_t pg_id) { + page_id = pg_id; + is_compressed = false; + buffer = new value_type00[PAGE_SIZE]; + // for (int i = 0; i < PAGE_SIZE; i++) { + // buffer[i] = (value_type00) 0ul; + // } + compressed_buffer = NULL; + } + + ~Page_t() { + decompress(); + + for (int i = 0; i < PAGE_SIZE; i++) { + // if (buffer[i].isValid()) + // buffer[i].invalidate(); + // if (buffer[i] != 0) { + // buffer[i]->dec_ref_count(); + // buffer[i] = 0; + // } + } + + delete[] buffer; + } + + // TODO. Compress here. Should delete uncompressed_buffer and fill up compressed buffer + // with the new compressed version. The LRU updates should be taken care of in a separate + // function. + void compress() { + if (is_compressed) + return; + + size_t in_len = PAGE_SIZE * sizeof(value_type00); + char *out_buf = + new char[snappy::MaxCompressedLength(in_len)]; + snappy::RawCompress((const char *)buffer, in_len, + out_buf, &compressed_size); + + /* check for an incompressible block */ + //printf("Compressed %d to %d bytes.\n", in_len, out_len); + if (compressed_size >= in_len) + printf("This block contains incompressible data.\n"); + + compressed_buffer = new unsigned char[compressed_size]; + memcpy((void *)compressed_buffer, (void *)out_buf, compressed_size); + + delete[] out_buf; + + // We want to avoid invoking destructors on the elements in the buffer, + // because these destructors affect reference counts. (Conceputally, + // the same references still exist, but in a compressed form.) Hence, + // we zero out the buffer before deleting it. + memset((void *)buffer, 0, in_len); + delete[] buffer; + buffer = NULL; + is_compressed = true; + //std::cout << "done compressing" << std::endl; + } + + // TODO: Same but decompress. + void decompress() { + // Get future state to wait for compression to finish. + if (fut.valid()) + fut.get(); + + if (!is_compressed) + return; + + bool result; + size_t uncompressed_length = sizeof(value_type00) * PAGE_SIZE; + if (!snappy::GetUncompressedLength((const char *)compressed_buffer, + compressed_size, &uncompressed_length)) + std::cerr << "Problem computing uncompressed length.\n"; + assert(uncompressed_length == sizeof(value_type00) * PAGE_SIZE && + "Uncompressed length does not match buffer size."); + char *out_buf = new char[uncompressed_length]; + result = snappy::RawUncompress((const char *)compressed_buffer, + compressed_size, out_buf); + assert(result && "decompression failed"); + + buffer = new value_type00[PAGE_SIZE]; + memcpy((void *)buffer, (void *)out_buf, uncompressed_length); + + delete[] out_buf; + + delete[] compressed_buffer; + compressed_buffer = NULL; + compressed_size = -1; + is_compressed = false; + //std::cout << "done decompressing" << std::endl; + } +}; + +class LRU_Node { +public: + Page_t *page; + LRU_Node *next; + LRU_Node *previous; + + LRU_Node() { + next = NULL; + page = NULL; + previous = NULL; + } + + LRU_Node(Page_t *p) { + page = p; + next = NULL; + previous = NULL; + } +}; + +class LRU_List { +public: + LRU_Node free_list[MAX_CACHE_SIZE]; + LRU_Node *tail; + LRU_Node *head; + size_t cache_size; + + std::unordered_map lru_page_ids; + + LRU_List(); + + // Debugging + void print_lru(); + void check_invariants(int label); + + void access(Page_t *page); + Page_t *find_after_head(uint64_t page_id); +}; + +class Static_Dictionary : public Dictionary { +private: + std::unordered_map page_table; + LRU_List lru_list; + +public: + Static_Dictionary(); + ~Static_Dictionary(); + + void print_lru() { + lru_list.print_lru(); + } + + value_type00 *find(uint64_t key); + value_type00 *find_group(uint64_t key, size_t max_size, size_t &num_elems); + value_type00 *find_exact_group(uint64_t key, size_t max_size, + size_t &num_elems); + + const value_type00 &operator[] (uint64_t key); + + void erase(uint64_t key); + void erase(uint64_t key, size_t size); + bool includes(uint64_t key); + bool includes(uint64_t key, size_t size); + void insert(uint64_t key, const value_type00 &f); + void insert(uint64_t key, size_t size, const value_type00 &f); + void set(uint64_t key, size_t size, value_type00 &&f); + void insert_into_found_group(uint64_t key, size_t size, + value_type00 *dst, + value_type00 &&f); + void destruct(); +}; + +#endif // __STATIC_DICTIONARY__ diff --git a/compiler-rt/lib/cilkscale/CMakeLists.txt b/compiler-rt/lib/cilkscale/CMakeLists.txt new file mode 100644 index 000000000000..97f65a7bd613 --- /dev/null +++ b/compiler-rt/lib/cilkscale/CMakeLists.txt @@ -0,0 +1,27 @@ +# Build for the Cilkscale runtime support library. + +set(CILKSCALE_SOURCES + cilkscale.c + csanrt.c) + +include_directories(..) + +set(CILKSCALE_RTL_CFLAGS ${SANITIZER_COMMON_CFLAGS} -std=c++11) + +append_rtti_flag(OFF CILKSCALE_CFLAGS) + +# Build Cilkscale runtimes shipped with Clang. +add_compiler_rt_component(cilkscale) + +foreach (arch ${CILKSCALE_SUPPORTED_ARCH}) + add_compiler_rt_runtime(clang_rt.cilkscale + STATIC + ARCHS ${arch} + SOURCES ${CILKSCALE_SOURCES} + CFLAGS ${CILKSCALE_RTL_CFLAGS} + PARENT_TARGET cilkscale) +endforeach() + +if (COMPILER_RT_INCLUDE_TESTS) + # TODO(bruening): add tests via add_subdirectory(tests) +endif() diff --git a/compiler-rt/lib/cilkscale/cilkscale.c b/compiler-rt/lib/cilkscale/cilkscale.c new file mode 100644 index 000000000000..11f413d4db21 --- /dev/null +++ b/compiler-rt/lib/cilkscale/cilkscale.c @@ -0,0 +1,462 @@ +#include +#include +#include +/* #define _POSIX_C_SOURCE 200112L */ +#include +#include +#include + +/* #include */ +/* #include */ + +#include +#include "context_stack.h" + +#define CILKTOOL_API __attribute__((visibility("default"))) + +#ifndef SERIAL_TOOL +#define SERIAL_TOOL 1 +#endif + +#if !SERIAL_TOOL +#include +#include "context_stack_reducer.h" +#endif + +#ifndef TRACE_CALLS +#define TRACE_CALLS 0 +#endif + +#ifndef INSTCOUNT +#define INSTCOUNT 1 +#endif + +/*************************************************************************/ +/** + * Data structures for tracking work and span. + */ +#if SERIAL_TOOL +context_stack_t ctx_stack; +#else +CILK_C_DECLARE_REDUCER(context_stack_t) ctx_stack = + CILK_C_INIT_REDUCER(context_stack_t, + reduce_context_stack, + identity_context_stack, + destroy_context_stack, + {NULL}); +#endif + +bool TOOL_INITIALIZED = false; + +/*************************************************************************/ +/** + * Data structures and helper methods for time of user strands. + */ + +static inline uint64_t elapsed_nsec(const struct timespec *stop, + const struct timespec *start) { + return (uint64_t)(stop->tv_sec - start->tv_sec) * 1000000000ll + + (stop->tv_nsec - start->tv_nsec); +} + +static inline void gettime(struct timespec *timer) { +#if INSTCOUNT +#else + // TB 2014-08-01: This is the "clock_gettime" variant I could get + // working with -std=c11. I want to use TIME_MONOTONIC instead, but + // it does not appear to be supported on my system. + /* timespec_get(timer, TIME_UTC); */ + clock_gettime(CLOCK_MONOTONIC, timer); +#endif +} + +#if SERIAL_TOOL +// Ensure that this tool is run serially +static inline void ensure_serial_tool(void) { + // assert(1 == __cilkrts_get_nworkers()); + fprintf(stderr, "Forcing CILK_NWORKERS=1.\n"); + char *e = getenv("CILK_NWORKERS"); + if (!e || 0!=strcmp(e, "1")) { + // fprintf(err_io, "Setting CILK_NWORKERS to be 1\n"); + if( setenv("CILK_NWORKERS", "1", 1) ) { + fprintf(stderr, "Error setting CILK_NWORKERS to be 1\n"); + exit(1); + } + } +} +#endif + +void print_analysis(void) { + + assert(TOOL_INITIALIZED); +#if SERIAL_TOOL + assert(NULL != ctx_stack.bot); + + uint64_t span = ctx_stack.bot->prefix_spn + ctx_stack.bot->contin_spn; + uint64_t work = ctx_stack.running_wrk; +#else + assert(MAIN == REDUCER_VIEW(ctx_stack).bot->func_type); + assert(NULL != REDUCER_VIEW(ctx_stack).bot); + + uint64_t span = REDUCER_VIEW(ctx_stack).bot->prefix_spn + REDUCER_VIEW(ctx_stack).bot->contin_spn; + uint64_t work = REDUCER_VIEW(ctx_stack).running_wrk; +#endif + +#if INSTCOUNT + fprintf(stderr, "work %f MInstructions, span %f MInstructions, parallelism %f\n", + work / (1000000.0), + span / (1000000.0), + work / (double)span); +#else + fprintf(stderr, "work %fs, span %fs, parallelism %f\n", + work / (1000000000.0), + span / (1000000000.0), + work / (double)span); +#endif +} + +void cilkscale_destroy(void) { +#if SERIAL_TOOL + gettime(&(ctx_stack.stop)); +#else + gettime(&(REDUCER_VIEW(ctx_stack).stop)); +#endif +#if TRACE_CALLS + fprintf(stderr, "cilkscale_destroy()\n"); +#endif + + print_analysis(); + +#if SERIAL_TOOL +#else + CILK_C_UNREGISTER_REDUCER(ctx_stack); +#endif + TOOL_INITIALIZED = false; +} + +CILKTOOL_API void __csi_init() { +#if TRACE_CALLS + fprintf(stderr, "__csi_init()\n"); +#endif + + atexit(cilkscale_destroy); + + TOOL_INITIALIZED = true; + +#if SERIAL_TOOL + ensure_serial_tool(); + + context_stack_init(&ctx_stack, MAIN); + + ctx_stack.in_user_code = true; + + gettime(&(ctx_stack.start)); +#else + context_stack_init(&(REDUCER_VIEW(ctx_stack)), MAIN); + + CILK_C_REGISTER_REDUCER(ctx_stack); + + REDUCER_VIEW(ctx_stack).in_user_code = true; + + gettime(&(REDUCER_VIEW(ctx_stack).start)); +#endif +} + +CILKTOOL_API void __csi_unit_init(const char *const file_name, + const instrumentation_counts_t counts) { + return; +} + +/*************************************************************************/ +/** + * Hooks into runtime system. + */ + +CILKTOOL_API +void __csi_func_entry(const csi_id_t func_id, const func_prop_t prop) { + return; +} + +CILKTOOL_API +void __csi_func_exit(const csi_id_t func_exit_id, const csi_id_t func_id, + const func_exit_prop_t prop) { + return; +} + +CILKTOOL_API +void __csi_bb_entry(const csi_id_t bb_id, const bb_prop_t prop) { +#if INSTCOUNT + context_stack_t *stack; + +#if SERIAL_TOOL + stack = &(ctx_stack); +#else + stack = &(REDUCER_VIEW(ctx_stack)); +#endif // SERIAL_TOOL + uint64_t inst_count = __csi_get_bb_sizeinfo(bb_id)->non_empty_size; + stack->running_wrk += inst_count; + stack->bot->contin_spn += inst_count; + +#endif // INSTCOUNT + return; +} + +CILKTOOL_API +void __csi_bb_exit(const csi_id_t bb_id, const bb_prop_t prop) { + return; +} + +CILKTOOL_API +void __csi_before_call(const csi_id_t call_id, + const csi_id_t func_id, + const call_prop_t prop) { + return; +} + +CILKTOOL_API +void __csi_after_call(const csi_id_t call_id, + const csi_id_t func_id, + const call_prop_t prop) { + return; +} + +CILKTOOL_API +void __csi_before_load(const csi_id_t load_id, const void *addr, int32_t size, + load_prop_t prop) { + return; +} + +CILKTOOL_API +void __csi_after_load(const csi_id_t load_id, const void *addr, int32_t size, + load_prop_t prop) { + return; +} + +CILKTOOL_API +void __csi_before_store(const csi_id_t store_id, const void *addr, int32_t size, + store_prop_t prop) { + return; +} + +CILKTOOL_API +void __csi_after_store(const csi_id_t store_id, const void *addr, int32_t size, + store_prop_t prop) { + return; +} + +/* void __csan_func_entry(const csi_id_t func_id, void *sp, */ +/* const func_prop_t prop) */ +/* { */ +/* context_stack_t *stack; */ +/* gettime(&(stack->stop)); */ + +/* #if TRACE_CALLS */ +/* fprintf(stderr, "cilk_enter_begin(%p, %p, %p)\n", sf, this_fn, rip); */ +/* #endif */ + +/* #if SERIAL_TOOL */ +/* stack = &(ctx_stack); */ +/* #else */ +/* stack = &(REDUCER_VIEW(ctx_stack)); */ +/* #endif */ + +/* assert(NULL != stack->bot); */ + +/* if (stack->bot->func_type != HELPER) { */ +/* #if TRACE_CALLS */ +/* if (MAIN == stack->bot->func_type) { */ +/* printf("parent is MAIN\n"); */ +/* } else { */ +/* printf("parent is SPAWN\n"); */ +/* } */ +/* #endif */ +/* // TB 2014-12-18: This assert won't necessarily pass, if shrink-wrapping has */ +/* // taken place. */ +/* /\* assert(stack->in_user_code); *\/ */ + +/* uint64_t strand_time = elapsed_nsec(&(stack->stop), &(stack->start)); */ +/* stack->running_wrk += strand_time; */ +/* stack->bot->contin_spn += strand_time; */ + +/* stack->in_user_code = false; */ +/* } else { */ +/* assert(!(stack->in_user_code)); */ +/* } */ + +/* /\* Push new frame onto the stack *\/ */ +/* context_stack_push(stack, SPAWN); */ +/* stack->in_user_code = true; */ +/* gettime(&(stack->start)); */ +/* } */ + +/* void __csan_func_exit(const csi_id_t func_exit_id, */ +/* const csi_id_t func_id, */ +/* const func_exit_prop_t prop) { */ +/* context_stack_t *stack; */ +/* #if SERIAL_TOOL */ +/* stack = &(ctx_stack); */ +/* #else */ +/* stack = &(REDUCER_VIEW(ctx_stack)); */ +/* #endif */ + +/* context_stack_frame_t *old_bottom; */ + +/* gettime(&(stack->stop)); */ + +/* assert(stack->in_user_code); */ +/* stack->in_user_code = false; */ + +/* if (SPAWN == stack->bot->func_type) { */ +/* #if TRACE_CALLS */ +/* fprintf(stderr, "cilk_leave_begin(%p) from SPAWN\n", sf); */ +/* #endif */ +/* uint64_t strand_time = elapsed_nsec(&(stack->stop), &(stack->start)); */ +/* stack->running_wrk += strand_time; */ +/* stack->bot->contin_spn += strand_time; */ +/* assert(NULL != stack->bot->parent); */ + +/* assert(0 == stack->bot->lchild_spn); */ +/* stack->bot->prefix_spn += stack->bot->contin_spn; */ + +/* /\* Pop the stack *\/ */ +/* old_bottom = context_stack_pop(stack); */ +/* stack->bot->contin_spn += old_bottom->prefix_spn; */ + +/* } else { */ +/* #if TRACE_CALLS */ +/* fprintf(stderr, "cilk_leave_begin(%p) from HELPER\n", sf); */ +/* #endif */ + +/* assert(HELPER != stack->bot->parent->func_type); */ + +/* assert(0 == stack->bot->lchild_spn); */ +/* stack->bot->prefix_spn += stack->bot->contin_spn; */ + +/* /\* Pop the stack *\/ */ +/* old_bottom = context_stack_pop(stack); */ +/* if (stack->bot->contin_spn + old_bottom->prefix_spn > stack->bot->lchild_spn) { */ +/* // fprintf(stderr, "updating longest child\n"); */ +/* stack->bot->prefix_spn += stack->bot->contin_spn; */ +/* stack->bot->lchild_spn = old_bottom->prefix_spn; */ +/* stack->bot->contin_spn = 0; */ +/* } */ + +/* } */ + +/* free(old_bottom); */ +/* gettime(&(stack->start)); */ +/* } */ + +CILKTOOL_API +void __csi_detach(const csi_id_t detach_id) { + context_stack_t *stack; + +#if SERIAL_TOOL + stack = &(ctx_stack); +#else + stack = &(REDUCER_VIEW(ctx_stack)); +#endif + gettime(&(stack->stop)); + +#if TRACE_CALLS + fprintf(stderr, "detach(%ld)\n", detach_id); +#endif + + uint64_t strand_time = elapsed_nsec(&(stack->stop), &(stack->start)); + stack->running_wrk += strand_time; + stack->bot->contin_spn += strand_time; +} + +CILKTOOL_API +void __csi_task(const csi_id_t task_id, const csi_id_t detach_id, void *sp) { + context_stack_t *stack; +#if SERIAL_TOOL + stack = &(ctx_stack); +#else + stack = &(REDUCER_VIEW(ctx_stack)); +#endif + + assert(NULL != stack->bot); + + /* Push new frame onto the stack */ + context_stack_push(stack, HELPER); + gettime(&(stack->start)); +} + +CILKTOOL_API +void __csi_task_exit(const csi_id_t task_exit_id, + const csi_id_t task_id, + const csi_id_t detach_id) { + context_stack_t *stack; +#if SERIAL_TOOL + stack = &(ctx_stack); +#else + stack = &(REDUCER_VIEW(ctx_stack)); +#endif + gettime(&(stack->stop)); + + context_stack_frame_t *old_bottom; + +#if TRACE_CALLS + fprintf(stderr, "task_exit(%ld, %ld, %ld)\n", + task_exit_id, task_id, detach_id); +#endif + + assert(0 == stack->bot->lchild_spn); + stack->bot->prefix_spn += stack->bot->contin_spn; + + /* Pop the stack */ + old_bottom = context_stack_pop(stack); + if (stack->bot->contin_spn + old_bottom->prefix_spn > stack->bot->lchild_spn) { + // fprintf(stderr, "updating longest child\n"); + stack->bot->prefix_spn += stack->bot->contin_spn; + stack->bot->lchild_spn = old_bottom->prefix_spn; + stack->bot->contin_spn = 0; + } + + free(old_bottom); +} + +CILKTOOL_API +void __csi_detach_continue(const csi_id_t detach_continue_id, + const csi_id_t detach_id) { + // In the continuation +#if TRACE_CALLS + fprintf(stderr, "detach_continue(%ld, %ld)\n", + detach_continue_id, detach_id); +#endif + + context_stack_t *stack; +#if SERIAL_TOOL + stack = &(ctx_stack); +#else + stack = &(REDUCER_VIEW(ctx_stack)); +#endif + + gettime(&(stack->start)); +} + +CILKTOOL_API +void __csi_sync(const csi_id_t sync_id) { + context_stack_t *stack; +#if SERIAL_TOOL + stack = &(ctx_stack); +#else + stack = &(REDUCER_VIEW(ctx_stack)); +#endif + gettime(&(stack->stop)); + + uint64_t strand_time = elapsed_nsec(&(stack->stop), &(stack->start)); + stack->running_wrk += strand_time; + stack->bot->contin_spn += strand_time; + + if (stack->bot->lchild_spn > stack->bot->contin_spn) { + stack->bot->prefix_spn += stack->bot->lchild_spn; + } else { + stack->bot->prefix_spn += stack->bot->contin_spn; + } + stack->bot->lchild_spn = 0; + stack->bot->contin_spn = 0; + + gettime(&(stack->start)); +} diff --git a/compiler-rt/lib/cilkscale/context_stack.h b/compiler-rt/lib/cilkscale/context_stack.h new file mode 100644 index 000000000000..81bdd3643f01 --- /dev/null +++ b/compiler-rt/lib/cilkscale/context_stack.h @@ -0,0 +1,123 @@ +#ifndef INCLUDED_CONTEXT_STACK_H +#define INCLUDED_CONTEXT_STACK_H + +#include +#include +/* #define _POSIX_C_SOURCE 200112L */ + +/* Enum for types of functions */ +typedef enum { + MAIN, + SPAWN, + HELPER, +} cilk_function_type; + +/* Type for a context stack frame */ +typedef struct context_stack_frame_t { + /* Function type */ + cilk_function_type func_type; + + /* Height of the function */ + int32_t height; + + /* Return address of this function */ + void* rip; + + /* Pointer to the frame's parent */ + struct context_stack_frame_t *parent; + + /* Span of the prefix of this function */ + uint64_t prefix_spn; + /* Data associated with the function's prefix */ + void* prefix_data; + + /* Span of the longest spawned child of this function observed so + far */ + uint64_t lchild_spn; + /* Data associated with the function's longest child */ + void* lchild_data; + + /* Span of the continuation of the function since the spawn of its + longest child */ + uint64_t contin_spn; + /* Data associated with the function's continuation */ + void* contin_data; + +} context_stack_frame_t; + +/* Initializes the context stack frame *frame */ +void context_stack_frame_init(context_stack_frame_t *frame, cilk_function_type func_type) +{ + frame->parent = NULL; + frame->func_type = func_type; + frame->rip = __builtin_extract_return_addr(__builtin_return_address(0)); + frame->height = 0; + + frame->prefix_spn = 0; + frame->prefix_data = NULL; + frame->lchild_spn = 0; + frame->lchild_data = NULL; + frame->contin_spn = 0; + frame->contin_data = NULL; +} + +/* Type for a context stack */ +typedef struct { + /* Flag to indicate whether user code is being executed. This flag + is mostly used for debugging. */ + bool in_user_code; + + /* Start and stop timers for measuring the execution time of a + strand. */ + struct timespec start; + struct timespec stop; + + /* Pointer to bottom of the stack, onto which frames are pushed. */ + context_stack_frame_t *bot; + + /* Running total of work. */ + uint64_t running_wrk; + + /* Data associated with the running work */ + void* running_wrk_data; + +} context_stack_t; + +/* Initializes the context stack */ +void context_stack_init(context_stack_t *stack, cilk_function_type func_type) +{ + context_stack_frame_t *new_frame = + (context_stack_frame_t *)malloc(sizeof(context_stack_frame_t)); + context_stack_frame_init(new_frame, func_type); + stack->bot = new_frame; + stack->running_wrk = 0; + stack->running_wrk_data = NULL; + stack->in_user_code = false; +} + +/* Push new frame of function type func_type onto the stack *stack */ +context_stack_frame_t* context_stack_push(context_stack_t *stack, cilk_function_type func_type) +{ + context_stack_frame_t *new_frame + = (context_stack_frame_t *)malloc(sizeof(context_stack_frame_t)); + context_stack_frame_init(new_frame, func_type); + new_frame->parent = stack->bot; + stack->bot = new_frame; + + return new_frame; +} + +/* Pops the bottommost frame off of the stack *stack, and returns a + pointer to it. */ +context_stack_frame_t* context_stack_pop(context_stack_t *stack) +{ + context_stack_frame_t *old_bottom = stack->bot; + stack->bot = stack->bot->parent; + if (stack->bot->height < old_bottom->height + 1) { + stack->bot->height = old_bottom->height + 1; + } + + return old_bottom; +} + +#endif diff --git a/compiler-rt/lib/cilkscale/context_stack_reducer.h b/compiler-rt/lib/cilkscale/context_stack_reducer.h new file mode 100644 index 000000000000..758ef3cf9199 --- /dev/null +++ b/compiler-rt/lib/cilkscale/context_stack_reducer.h @@ -0,0 +1,57 @@ +#ifndef INCLUDED_CONTEXT_STACK_REDUCER_H +#define INCLUDED_CONTEXT_STACK_REDUCER_H + +#include +#include + +#include +#include +#include + +#include "context_stack.h" + +/* Identity method for context stack reducer */ +void identity_context_stack(void *reducer, void *view) +{ + context_stack_init((context_stack_t*)view, SPAWN); +} + +/* Reduce method for context stack reducer */ +void reduce_context_stack(void *reducer, void *l, void *r) +{ + context_stack_t *left = (context_stack_t*)l; + context_stack_t *right = (context_stack_t*)r; + + assert(NULL == right->bot->parent); + assert(SPAWN == right->bot->func_type); + assert(right->bot->func_type == left->bot->func_type); + + assert(!(left->in_user_code)); + assert(!(right->in_user_code)); + + /* height is maintained as a max reducer */ + if (right->bot->height > left->bot->height) { + left->bot->height = right->bot->height; + } + /* running_wrk is maintained as a sum reducer */ + left->running_wrk += right->running_wrk; + + /* assert(0 == left->bot->contin_spn); */ + + if (left->bot->contin_spn + right->bot->prefix_spn + right->bot->lchild_spn + > left->bot->lchild_spn) { + left->bot->prefix_spn += left->bot->contin_spn + right->bot->prefix_spn; + left->bot->lchild_spn = right->bot->lchild_spn; + left->bot->contin_spn = right->bot->contin_spn; + } else { + left->bot->contin_spn += right->bot->prefix_spn + right->bot->contin_spn; + } +} + +/* Destructor for context stack reducer */ +void destroy_context_stack(void *reducer, void *view) +{ + free(((context_stack_t*)view)->bot); +} + +#endif diff --git a/compiler-rt/lib/cilkscale/csanrt.c b/compiler-rt/lib/cilkscale/csanrt.c new file mode 100644 index 000000000000..e450310f00a6 --- /dev/null +++ b/compiler-rt/lib/cilkscale/csanrt.c @@ -0,0 +1,358 @@ +#include +#include +//#include "csan.h" +#include + +#define CSIRT_API __attribute__((visibility("default"))) + +// ------------------------------------------------------------------------ +// Front end data (FED) table structures. +// ------------------------------------------------------------------------ + +// A FED table is a flat list of FED entries, indexed by a CSI +// ID. Each FED table has its own private ID space. +typedef struct { + int64_t num_entries; + source_loc_t *entries; +} fed_table_t; + +// Types of FED tables that we maintain across all units. +typedef enum { + FED_TYPE_FUNCTIONS, + FED_TYPE_FUNCTION_EXIT, + FED_TYPE_BASICBLOCK, + FED_TYPE_CALLSITE, + FED_TYPE_LOAD, + FED_TYPE_STORE, + FED_TYPE_DETACH, + FED_TYPE_TASK, + FED_TYPE_TASK_EXIT, + FED_TYPE_DETACH_CONTINUE, + FED_TYPE_SYNC, + NUM_FED_TYPES // Must be last +} fed_type_t; + +// A SizeInfo table is a flat list of SizeInfo entries, indexed by a CSI ID. +typedef struct { + int64_t num_entries; + sizeinfo_t *entries; +} sizeinfo_table_t; + +// Types of sizeinfo tables that we maintain across all units. +typedef enum { + SIZEINFO_TYPE_BASICBLOCK, + NUM_SIZEINFO_TYPES // Must be last +} sizeinfo_type_t; + +// ------------------------------------------------------------------------ +// Globals +// ------------------------------------------------------------------------ + +// The list of FED tables. This is indexed by a value of +// 'fed_type_t'. +static fed_table_t *fed_tables = NULL; + +// Initially false, set to true once the first unit is initialized, +// which results in the FED list being initialized. +static bool fed_tables_initialized = false; + +// The list of SizeInfo tables. This is indexed by a value of +// 'sizeinfo_type_t'. +static sizeinfo_table_t *sizeinfo_tables = NULL; + +// Initially false, set to true once the first unit is initialized, +// which results in the SizeInfo list being initialized. +static bool sizeinfo_tables_initialized = false; + +// Initially false, set to true once the first unit is initialized, +// which results in the __csi_init() function being called. +static bool csi_init_called = false; + +// ------------------------------------------------------------------------ +// Private function definitions +// ------------------------------------------------------------------------ + +// Initialize the FED tables list, indexed by a value of type +// fed_type_t. This is called once, by the first unit to load. +static void initialize_fed_tables() { + fed_tables = (fed_table_t *)malloc(sizeof(fed_table_t) * NUM_FED_TYPES); + assert(fed_tables != NULL); + for (int i = 0; i < (int)NUM_FED_TYPES; ++i) { + fed_table_t table; + table.num_entries = 0; + table.entries = NULL; + fed_tables[i] = table; + } + fed_tables_initialized = true; +} + +// Ensure that the FED table of the given type has enough memory +// allocated to add a new unit's entries. +static void ensure_fed_table_capacity(fed_type_t fed_type, + int64_t num_new_entries) { + if (!fed_tables_initialized) + initialize_fed_tables(); + + assert(num_new_entries >= 0); + + fed_table_t *table = &fed_tables[fed_type]; + int64_t total_num_entries = table->num_entries + num_new_entries; + if (num_new_entries > 0) { + if (!table->entries) + table->entries = (source_loc_t *)malloc(sizeof(source_loc_t) * total_num_entries); + else { + source_loc_t *old_entries = table->entries; + table->entries = (source_loc_t *)malloc(sizeof(source_loc_t) * total_num_entries); + for (int i = 0; i < table->num_entries; ++i) + table->entries[i] = old_entries[i]; + free(old_entries); + } + table->num_entries = total_num_entries; + assert(table->entries != NULL); + } +} + +// Add a new FED table of the given type. +static inline void add_fed_table(fed_type_t fed_type, int64_t num_entries, + const source_loc_t *fed_entries) { + ensure_fed_table_capacity(fed_type, num_entries); + fed_table_t *table = &fed_tables[fed_type]; + csi_id_t base = table->num_entries - num_entries; + for (csi_id_t i = 0; i < num_entries; ++i) + table->entries[base + i] = fed_entries[i]; +} + +// The unit-local counter pointed to by 'fed_id_base' keeps track of +// that unit's "base" ID value of the given type (recall that there is +// a private ID space per FED type). The "base" ID value is the global +// ID that corresponds to the unit's local ID 0. This function stores +// the correct value into a unit's base ID. +static inline void update_ids(fed_type_t fed_type, int64_t num_entries, + csi_id_t *fed_id_base) { + fed_table_t *table = &fed_tables[fed_type]; + // The base ID is the current number of FED entries before adding + // the new FED table. + *fed_id_base = table->num_entries - num_entries; +} + +// Return the FED entry of the given type, corresponding to the given +// CSI ID. +static inline const source_loc_t *get_fed_entry(fed_type_t fed_type, + const csi_id_t csi_id) { + // TODO(ddoucet): threadsafety + fed_table_t *table = &fed_tables[fed_type]; + if (csi_id < table->num_entries) { + assert(table->entries != NULL); + return &table->entries[csi_id]; + } + return NULL; +} + +// Initialize the object tables list, indexed by a value of type +// sizeinfo_type_t. This is called once, by the first unit to load. +static void initialize_sizeinfo_tables() { + sizeinfo_tables = + (sizeinfo_table_t *)malloc(sizeof(sizeinfo_table_t) * NUM_SIZEINFO_TYPES); + assert(sizeinfo_tables != NULL); + for (int i = 0; i < (int)NUM_SIZEINFO_TYPES; ++i) { + sizeinfo_table_t table; + table.num_entries = 0; + table.entries = NULL; + sizeinfo_tables[i] = table; + } + sizeinfo_tables_initialized = true; +} + +// Ensure that the sizeinfo table of the given type has enough memory allocated +// to add a new unit's entries. +static void ensure_sizeinfo_table_capacity(sizeinfo_type_t sizeinfo_type, + int64_t num_new_entries) { + if (!sizeinfo_tables_initialized) + initialize_sizeinfo_tables(); + + assert(num_new_entries >= 0); + + sizeinfo_table_t *table = &sizeinfo_tables[sizeinfo_type]; + int64_t total_num_entries = table->num_entries + num_new_entries; + if (num_new_entries > 0) { + if (!table->entries) + table->entries = + (sizeinfo_t *)malloc(sizeof(sizeinfo_t) * total_num_entries); + else { + sizeinfo_t *old_entries = table->entries; + table->entries = + (sizeinfo_t *)malloc(sizeof(sizeinfo_t) * total_num_entries); + for (int i = 0; i < table->num_entries; ++i) + table->entries[i] = old_entries[i]; + free(old_entries); + } + table->num_entries = total_num_entries; + assert(table->entries != NULL); + } +} + +// Add a new object table of the given type. +static inline void add_sizeinfo_table(sizeinfo_type_t sizeinfo_type, + int64_t num_entries, + const sizeinfo_t *sizeinfo_entries) { + ensure_sizeinfo_table_capacity(sizeinfo_type, num_entries); + sizeinfo_table_t *table = &sizeinfo_tables[sizeinfo_type]; + csi_id_t base = table->num_entries - num_entries; + for (csi_id_t i = 0; i < num_entries; ++i) + table->entries[base + i] = sizeinfo_entries[i]; +} + +// Return the SizeInfo entry of the given type, corresponding to the given CSI +// ID. +static inline const sizeinfo_t *get_sizeinfo_entry(sizeinfo_type_t sizeinfo_type, + const csi_id_t csi_id) { + // TODO(ddoucet): threadsafety + sizeinfo_table_t *table = &sizeinfo_tables[sizeinfo_type]; + if (csi_id < table->num_entries) { + assert(table->entries != NULL); + return &table->entries[csi_id]; + } + return NULL; +} + +// ------------------------------------------------------------------------ +// External function definitions, including CSIRT API functions. +// ------------------------------------------------------------------------ + +EXTERN_C + +void __csi_unit_init(const char * const file_name, + const instrumentation_counts_t counts); + +// Not used at the moment +// __thread bool __csi_disable_instrumentation; + +typedef struct { + int64_t num_entries; + csi_id_t *id_base; + const source_loc_t *entries; +} unit_fed_table_t; + +typedef struct { + int64_t num_entries; + const sizeinfo_t *entries; +} unit_sizeinfo_table_t; + +// Function signature for the function (generated by the CSI compiler +// pass) that updates the callsite to function ID mappings. +typedef void (*__csi_init_callsite_to_functions)(); + +static inline void compute_inst_counts(instrumentation_counts_t *counts, + unit_fed_table_t *unit_fed_tables) { + int64_t *base = (int64_t *)counts; + for (int i = 0; i < NUM_FED_TYPES; i++) + *(base + i) = unit_fed_tables[i].num_entries; +} + +// A call to this is inserted by the CSI compiler pass, and occurs +// before main(). +CSIRT_API +void __csirt_unit_init(const char * const name, + unit_fed_table_t *unit_fed_tables, + unit_sizeinfo_table_t *unit_sizeinfo_tables, + __csi_init_callsite_to_functions callsite_to_func_init) { + // Make sure we don't instrument things in __csi_init or __csi_unit init. + // __csi_disable_instrumentation = true; + + // TODO(ddoucet): threadsafety + if (!csi_init_called) { + __csi_init(); + csi_init_called = true; + } + + // Add all FED tables from the new unit + for (int i = 0; i < NUM_FED_TYPES; ++i) { + add_fed_table((fed_type_t)i, unit_fed_tables[i].num_entries, + unit_fed_tables[i].entries); + update_ids((fed_type_t)i, unit_fed_tables[i].num_entries, + unit_fed_tables[i].id_base); + } + + // Add all object tables from the new unit + for (int i = 0; i < NUM_SIZEINFO_TYPES; ++i) { + add_sizeinfo_table((sizeinfo_type_t)i, unit_sizeinfo_tables[i].num_entries, + unit_sizeinfo_tables[i].entries); + } + + // Initialize the callsite -> function mappings. This must happen + // after the base IDs have been updated. + callsite_to_func_init(); + + // Call into the tool implementation. + instrumentation_counts_t counts; + compute_inst_counts(&counts, unit_fed_tables); + __csi_unit_init(name, counts); + + // Reset disable flag. + // __csi_disable_instrumentation = false; +} + +CSIRT_API +const source_loc_t *__csi_get_func_source_loc(const csi_id_t func_id) { + return get_fed_entry(FED_TYPE_FUNCTIONS, func_id); +} + +CSIRT_API +const source_loc_t *__csi_get_func_exit_source_loc( + const csi_id_t func_exit_id) { + return get_fed_entry(FED_TYPE_FUNCTION_EXIT, func_exit_id); +} + +CSIRT_API +const source_loc_t *__csi_get_bb_source_loc(const csi_id_t bb_id) { + return get_fed_entry(FED_TYPE_BASICBLOCK, bb_id); +} + +CSIRT_API +const source_loc_t *__csi_get_call_source_loc(const csi_id_t call_id) { + return get_fed_entry(FED_TYPE_CALLSITE, call_id); +} + +CSIRT_API +const source_loc_t *__csi_get_load_source_loc(const csi_id_t load_id) { + return get_fed_entry(FED_TYPE_LOAD, load_id); +} + +CSIRT_API +const source_loc_t *__csi_get_store_source_loc(const csi_id_t store_id) { + return get_fed_entry(FED_TYPE_STORE, store_id); +} + +CSIRT_API +const source_loc_t *__csi_get_detach_source_loc(const csi_id_t detach_id) { + return get_fed_entry(FED_TYPE_DETACH, detach_id); +} + +CSIRT_API +const source_loc_t *__csi_get_task_source_loc(const csi_id_t task_id) { + return get_fed_entry(FED_TYPE_TASK, task_id); +} + +CSIRT_API +const source_loc_t *__csi_get_task_exit_source_loc( + const csi_id_t task_exit_id) { + return get_fed_entry(FED_TYPE_TASK_EXIT, task_exit_id); +} + +CSIRT_API +const source_loc_t *__csi_get_detach_continue_source_loc( + const csi_id_t detach_continue_id) { + return get_fed_entry(FED_TYPE_DETACH_CONTINUE, detach_continue_id); +} + +CSIRT_API +const source_loc_t *__csi_get_sync_source_loc(const csi_id_t sync_id) { + return get_fed_entry(FED_TYPE_SYNC, sync_id); +} + +CSIRT_API +const sizeinfo_t *__csi_get_bb_sizeinfo(const csi_id_t bb_id) { + return get_sizeinfo_entry(SIZEINFO_TYPE_BASICBLOCK, bb_id); +} + +EXTERN_C_END diff --git a/compiler-rt/lib/csi/CMakeLists.txt b/compiler-rt/lib/csi/CMakeLists.txt new file mode 100644 index 000000000000..5b13fb5a1e0f --- /dev/null +++ b/compiler-rt/lib/csi/CMakeLists.txt @@ -0,0 +1,26 @@ +# Build for the ComprehensiveStaticInstrumentation runtime support library. + +add_custom_target(csi) + +set(CSI_RTL_CFLAGS ${SANITIZER_COMMON_CFLAGS} -std=c11) +append_rtti_flag(OFF CSI_RTL_CFLAGS) + +include_directories(..) + +set(CSI_SOURCES csirt.c) + +foreach (arch ${CSI_SUPPORTED_ARCH}) + add_compiler_rt_runtime(clang_rt.csi + STATIC + ARCHS ${arch} + SOURCES ${CSI_SOURCES} + CFLAGS ${CSI_RTL_CFLAGS}) + add_dependencies(csi + clang_rt.csi-${arch}) +endforeach() + +add_dependencies(compiler-rt csi) + +if (COMPILER_RT_INCLUDE_TESTS) + # TODO(bruening): add tests via add_subdirectory(tests) +endif() diff --git a/compiler-rt/lib/csi/csi.h b/compiler-rt/lib/csi/csi.h new file mode 100644 index 000000000000..0d53a0720104 --- /dev/null +++ b/compiler-rt/lib/csi/csi.h @@ -0,0 +1,197 @@ +#ifndef __CSI_H__ +#define __CSI_H__ + +#include + +#ifdef __cplusplus +#define EXTERN_C extern "C" { +#define EXTERN_C_END } +#else +#define EXTERN_C +#define EXTERN_C_END +#include // for C99 bool type +#endif + +#define WEAK __attribute__((weak)) + +// API function signatures +EXTERN_C + +/** + * Unless a type requires bitwise operations (e.g., property lists), we use + * signed integers. We don't need the extra bit of data, and using unsigned + * integers can lead to subtle bugs. See + * http://www.soundsoftware.ac.uk/c-pitfall-unsigned + */ + +typedef int64_t csi_id_t; + +#define UNKNOWN_CSI_ID ((csi_id_t)-1) + +typedef struct { + csi_id_t num_func; + csi_id_t num_func_exit; + csi_id_t num_bb; + csi_id_t num_callsite; + csi_id_t num_load; + csi_id_t num_store; + csi_id_t num_detach; + csi_id_t num_task; + csi_id_t num_task_exit; + csi_id_t num_detach_continue; + csi_id_t num_sync; +} instrumentation_counts_t; + +// Property bitfields. + +typedef struct { + // The function might spawn. + unsigned may_spawn : 1; + // Pad struct to 64 total bits. + uint64_t _padding : 63; +} func_prop_t; + +typedef struct { + // The function might have spawned. + unsigned may_spawn : 1; + // Pad struct to 64 total bits. + uint64_t _padding : 63; +} func_exit_prop_t; + +typedef struct { + // The basic block is a landingpad. + unsigned is_landingpad : 1; + // The basic block is an exception-handling pad. + unsigned is_ehpad : 1; + // Pad struct to 64 total bits. + uint64_t _padding : 62; +} bb_prop_t; + +typedef struct { + // The call is indirect. + unsigned is_indirect : 1; + // Pad struct to 64 total bits. + uint64_t _padding : 63; +} call_prop_t; + +typedef struct { + // The alignment of the load. + unsigned alignment : 8; + // The loaded address is in a vtable. + unsigned is_vtable_access : 1; + // The loaded address points to constant data. + unsigned is_constant : 1; + // The loaded address is on the stack. + unsigned is_on_stack : 1; + // The loaded address cannot be captured. + unsigned may_be_captured : 1; + // The loaded address is read before it is written in the same basic block. + unsigned is_read_before_write_in_bb : 1; + // Pad struct to 64 total bits. + uint64_t _padding : 51; +} load_prop_t; + +typedef struct { + // The alignment of the store. + unsigned alignment : 8; + // The stored address is in a vtable. + unsigned is_vtable_access : 1; + // The stored address points to constant data. + unsigned is_constant : 1; + // The stored address is on the stack. + unsigned is_on_stack : 1; + // The stored address cannot be captured. + unsigned may_be_captured : 1; + // Pad struct to 64 total bits. + uint64_t _padding : 52; +} store_prop_t; + +WEAK void __csi_init(); + +WEAK void __csi_unit_init(const char * const file_name, + const instrumentation_counts_t counts); + +WEAK void __csi_func_entry(const csi_id_t func_id, const func_prop_t prop); + +WEAK void __csi_func_exit(const csi_id_t func_exit_id, + const csi_id_t func_id, const func_exit_prop_t prop); + +WEAK void __csi_bb_entry(const csi_id_t bb_id, const bb_prop_t prop); + +WEAK void __csi_bb_exit(const csi_id_t bb_id, const bb_prop_t prop); + +WEAK void __csi_before_call(const csi_id_t call_id, const csi_id_t func_id, + const call_prop_t prop); + +WEAK void __csi_after_call(const csi_id_t call_id, const csi_id_t func_id, + const call_prop_t prop); + +WEAK void __csi_before_load(const csi_id_t load_id, + const void *addr, + const int32_t num_bytes, + const load_prop_t prop); + +WEAK void __csi_after_load(const csi_id_t load_id, + const void *addr, + const int32_t num_bytes, + const load_prop_t prop); + +WEAK void __csi_before_store(const csi_id_t store_id, + const void *addr, + const int32_t num_bytes, + const store_prop_t prop); + +WEAK void __csi_after_store(const csi_id_t store_id, + const void *addr, + const int32_t num_bytes, + const store_prop_t prop); + +WEAK void __csi_detach(const csi_id_t detach_id); + +WEAK void __csi_task(const csi_id_t task_id, const csi_id_t detach_id, + void *sp); + +WEAK void __csi_task_exit(const csi_id_t task_exit_id, + const csi_id_t task_id, + const csi_id_t detach_id); + +WEAK void __csi_detach_continue(const csi_id_t detach_continue_id, + const csi_id_t detach_id); + +WEAK void __csi_sync(const csi_id_t sync_id); + +// This struct is mirrored in ComprehensiveStaticInstrumentation.cpp, +// FrontEndDataTable::getSourceLocStructType. +typedef struct { + char *name; + // TODO(ddoucet): Why is this 32 bits? + int32_t line_number; + int32_t column_number; + char *filename; +} source_loc_t; + +typedef struct sizeinfo_t { + int32_t full_ir_size; + int32_t non_empty_size; +} sizeinfo_t; + +// Front-end data (FED) table accessors. +const source_loc_t * __csi_get_func_source_loc(const csi_id_t func_id); +const source_loc_t * __csi_get_func_exit_source_loc(const csi_id_t func_exit_id); +const source_loc_t * __csi_get_bb_source_loc(const csi_id_t bb_id); +const source_loc_t * __csi_get_callsite_source_loc(const csi_id_t call_id); +const source_loc_t * __csi_get_load_source_loc(const csi_id_t load_id); +const source_loc_t * __csi_get_store_source_loc(const csi_id_t store_id); +const source_loc_t * __csi_get_detach_source_loc(const csi_id_t detach_id); +const source_loc_t * __csi_get_task_source_loc(const csi_id_t task_id); +const source_loc_t * __csi_get_task_exit_source_loc(const csi_id_t task_exit_id); +const source_loc_t * __csi_get_detach_continue_source_loc(const csi_id_t detach_continue_id); +const source_loc_t * __csi_get_sync_source_loc(const csi_id_t sync_id); +const sizeinfo_t *__csi_get_bb_sizeinfo(const csi_id_t bb_id); + +// Load property: +//#define CSI_PROP_LOAD_READ_BEFORE_WRITE_IN_BB 0x1 + +EXTERN_C_END + +#endif diff --git a/compiler-rt/lib/csi/csirt.c b/compiler-rt/lib/csi/csirt.c new file mode 100644 index 000000000000..5e1aaa73f96f --- /dev/null +++ b/compiler-rt/lib/csi/csirt.c @@ -0,0 +1,345 @@ +#include +#include +#include + +#include "csi.h" + +// Compile-time assert the property structs are 64 bits. +static_assert(sizeof(func_prop_t) == 8, "Size of func_prop_t is not 64 bits."); +static_assert(sizeof(func_exit_prop_t) == 8, "Size of func_exit_prop_t is not 64 bits."); +static_assert(sizeof(bb_prop_t) == 8, "Size of bb_prop_t is not 64 bits."); +static_assert(sizeof(call_prop_t) == 8, "Size of call_prop_t is not 64 bits."); +static_assert(sizeof(load_prop_t) == 8, "Size of load_prop_t is not 64 bits."); +static_assert(sizeof(store_prop_t) == 8, "Size of store_prop_t is not 64 bits."); + +#define CSIRT_API __attribute__((visibility("default"))) + +// ------------------------------------------------------------------------ +// Front end data (FED) table structures. +// ------------------------------------------------------------------------ + +// A FED table is a flat list of FED entries, indexed by a CSI +// ID. Each FED table has its own private ID space. +typedef struct { + int64_t num_entries; + source_loc_t *entries; +} fed_table_t; + +// Types of FED tables that we maintain across all units. +typedef enum { + FED_TYPE_FUNCTIONS, + FED_TYPE_FUNCTION_EXIT, + FED_TYPE_BASICBLOCK, + FED_TYPE_CALLSITE, + FED_TYPE_LOAD, + FED_TYPE_STORE, + FED_TYPE_DETACH, + FED_TYPE_TASK, + FED_TYPE_TASK_EXIT, + FED_TYPE_DETACH_CONTINUE, + FED_TYPE_SYNC, + NUM_FED_TYPES // Must be last +} fed_type_t; +// A SizeInfo table is a flat list of SizeInfo entries, indexed by a CSI ID. +typedef struct { + int64_t num_entries; + sizeinfo_t *entries; +} sizeinfo_table_t; + +// Types of sizeinfo tables that we maintain across all units. +typedef enum { + SIZEINFO_TYPE_BASICBLOCK, + NUM_SIZEINFO_TYPES // Must be last +} sizeinfo_type_t; + +// ------------------------------------------------------------------------ +// Globals +// ------------------------------------------------------------------------ + +// The list of FED tables. This is indexed by a value of +// 'fed_type_t'. +static fed_table_t *fed_tables = NULL; + +// Initially false, set to true once the first unit is initialized, +// which results in the FED list being initialized. +static bool fed_tables_initialized = false; + +// The list of SizeInfo tables. This is indexed by a value of +// 'sizeinfo_type_t'. +static sizeinfo_table_t *sizeinfo_tables = NULL; + +// Initially false, set to true once the first unit is initialized, +// which results in the SizeInfo list being initialized. +static bool sizeinfo_tables_initialized = false; + +// Initially false, set to true once the first unit is initialized, +// which results in the __csi_init() function being called. +static bool csi_init_called = false; + +// ------------------------------------------------------------------------ +// Private function definitions +// ------------------------------------------------------------------------ + +// Initialize the FED tables list, indexed by a value of type +// fed_type_t. This is called once, by the first unit to load. +static void initialize_fed_tables() { + fed_tables = (fed_table_t *)malloc(NUM_FED_TYPES * sizeof(fed_table_t)); + assert(fed_tables != NULL); + for (unsigned i = 0; i < NUM_FED_TYPES; i++) { + fed_table_t table; + table.num_entries = 0; + table.entries = NULL; + fed_tables[i] = table; + } + fed_tables_initialized = true; +} + +// Ensure that the FED table of the given type has enough memory +// allocated to add a new unit's entries. +static void ensure_fed_table_capacity(fed_type_t fed_type, + int64_t num_new_entries) { + if (!fed_tables_initialized) { + initialize_fed_tables(); + } + fed_table_t *table = &fed_tables[fed_type]; + int64_t total_num_entries = table->num_entries + num_new_entries; + if (total_num_entries > 0) { + table->entries = + (source_loc_t *)realloc(table->entries, + total_num_entries * sizeof(source_loc_t)); + table->num_entries = total_num_entries; + assert(table->entries != NULL); + } +} + +// Add a new FED table of the given type. +static inline void add_fed_table(fed_type_t fed_type, int64_t num_entries, + const source_loc_t *fed_entries) { + ensure_fed_table_capacity(fed_type, num_entries); + fed_table_t *table = &fed_tables[fed_type]; + csi_id_t base = table->num_entries - num_entries; + for (csi_id_t i = 0; i < num_entries; i++) { + table->entries[base + i] = fed_entries[i]; + } +} + +// The unit-local counter pointed to by 'fed_id_base' keeps track of +// that unit's "base" ID value of the given type (recall that there is +// a private ID space per FED type). The "base" ID value is the global +// ID that corresponds to the unit's local ID 0. This function stores +// the correct value into a unit's base ID. +static inline void update_ids(fed_type_t fed_type, int64_t num_entries, + csi_id_t *fed_id_base) { + fed_table_t *table = &fed_tables[fed_type]; + // The base ID is the current number of FED entries before adding + // the new FED table. + *fed_id_base = table->num_entries - num_entries; +} + +// Return the FED entry of the given type, corresponding to the given +// CSI ID. +static inline const source_loc_t *get_fed_entry(fed_type_t fed_type, + const csi_id_t csi_id) { + // TODO(ddoucet): threadsafety + fed_table_t *table = &fed_tables[fed_type]; + if (csi_id < table->num_entries) { + assert(table->entries != NULL); + return &table->entries[csi_id]; + } else { + return NULL; + } +} + +// Initialize the SizeInfo tables list, indexed by a value of type +// sizeinfo_type_t. This is called once, by the first unit to load. +static void initialize_sizeinfo_tables() { + sizeinfo_tables = + (sizeinfo_table_t *)malloc(NUM_SIZEINFO_TYPES * sizeof(sizeinfo_table_t)); + assert(sizeinfo_tables != NULL); + for (unsigned i = 0; i < NUM_SIZEINFO_TYPES; i++) { + sizeinfo_table_t table; + table.num_entries = 0; + table.entries = NULL; + sizeinfo_tables[i] = table; + } + sizeinfo_tables_initialized = true; +} + +// Ensure that the SizeInfo table of the given type has enough memory +// allocated to add a new unit's entries. +static void ensure_sizeinfo_table_capacity(sizeinfo_type_t sizeinfo_type, + int64_t num_new_entries) { + if (!sizeinfo_tables_initialized) { + initialize_sizeinfo_tables(); + } + sizeinfo_table_t *table = &sizeinfo_tables[sizeinfo_type]; + int64_t total_num_entries = table->num_entries + num_new_entries; + if (total_num_entries > 0) { + table->entries = (sizeinfo_t *)realloc(table->entries, + total_num_entries * sizeof(sizeinfo_t)); + table->num_entries = total_num_entries; + assert(table->entries != NULL); + } +} + +// Add a new SizeInfo table of the given type. +static inline void add_sizeinfo_table(sizeinfo_type_t sizeinfo_type, + int64_t num_entries, + const sizeinfo_t *sizeinfo_entries) { + ensure_sizeinfo_table_capacity(sizeinfo_type, num_entries); + sizeinfo_table_t *table = &sizeinfo_tables[sizeinfo_type]; + csi_id_t base = table->num_entries - num_entries; + for (csi_id_t i = 0; i < num_entries; i++) { + table->entries[base + i] = sizeinfo_entries[i]; + } +} + +// Return the SIZEINFO entry of the given type, corresponding to the given +// CSI ID. +static inline +const sizeinfo_t *get_sizeinfo_entry(sizeinfo_type_t sizeinfo_type, + const csi_id_t csi_id) { + // TODO(ddoucet): threadsafety + sizeinfo_table_t *table = &sizeinfo_tables[sizeinfo_type]; + if (csi_id < table->num_entries) { + assert(table->entries != NULL); + return &table->entries[csi_id]; + } else { + return NULL; + } +} + +// ------------------------------------------------------------------------ +// External function definitions, including CSIRT API functions. +// ------------------------------------------------------------------------ + +EXTERN_C + +// Not used at the moment +// __thread bool __csi_disable_instrumentation; + +typedef struct { + int64_t num_entries; + csi_id_t *id_base; + const source_loc_t *entries; +} unit_fed_table_t; + +typedef struct { + int64_t num_entries; + const sizeinfo_t *entries; +} unit_sizeinfo_table_t; + +// Function signature for the function (generated by the CSI compiler +// pass) that updates the callsite to function ID mappings. +typedef void (*__csi_init_callsite_to_functions)(); + +static inline instrumentation_counts_t compute_inst_counts(unit_fed_table_t *unit_fed_tables) { + instrumentation_counts_t counts; + int64_t *base = (int64_t *)&counts; + for (unsigned i = 0; i < NUM_FED_TYPES; i++) + *(base + i) = unit_fed_tables[i].num_entries; + return counts; +} + +// A call to this is inserted by the CSI compiler pass, and occurs +// before main(). +CSIRT_API void __csirt_unit_init( + const char * const name, + unit_fed_table_t *unit_fed_tables, + unit_sizeinfo_table_t *unit_sizeinfo_tables, + __csi_init_callsite_to_functions callsite_to_func_init) { + // Make sure we don't instrument things in __csi_init or __csi_unit init. + // __csi_disable_instrumentation = true; + + // TODO(ddoucet): threadsafety + if (!csi_init_called) { + __csi_init(); + csi_init_called = true; + } + + // Add all FED tables from the new unit + for (unsigned i = 0; i < NUM_FED_TYPES; i++) { + add_fed_table(i, unit_fed_tables[i].num_entries, unit_fed_tables[i].entries); + update_ids(i, unit_fed_tables[i].num_entries, unit_fed_tables[i].id_base); + } + + // Add all SizeInfo tables from the new unit + for (int i = 0; i < NUM_SIZEINFO_TYPES; ++i) { + add_sizeinfo_table((sizeinfo_type_t)i, unit_sizeinfo_tables[i].num_entries, + unit_sizeinfo_tables[i].entries); + } + + // Initialize the callsite -> function mappings. This must happen + // after the base IDs have been updated. + callsite_to_func_init(); + + // Call into the tool implementation. + __csi_unit_init(name, compute_inst_counts(unit_fed_tables)); + + // Reset disable flag. + // __csi_disable_instrumentation = false; +} + +CSIRT_API +const source_loc_t *__csi_get_func_source_loc(const csi_id_t func_id) { + return get_fed_entry(FED_TYPE_FUNCTIONS, func_id); +} + +CSIRT_API +const source_loc_t *__csi_get_func_exit_source_loc(const csi_id_t func_exit_id) { + return get_fed_entry(FED_TYPE_FUNCTION_EXIT, func_exit_id); +} + +CSIRT_API +const source_loc_t *__csi_get_bb_source_loc(const csi_id_t bb_id) { + return get_fed_entry(FED_TYPE_BASICBLOCK, bb_id); +} + +CSIRT_API +const source_loc_t *__csi_get_callsite_source_loc(const csi_id_t callsite_id) { + return get_fed_entry(FED_TYPE_CALLSITE, callsite_id); +} + +CSIRT_API +const source_loc_t *__csi_get_load_source_loc(const csi_id_t load_id) { + return get_fed_entry(FED_TYPE_LOAD, load_id); +} + +CSIRT_API +const source_loc_t *__csi_get_store_source_loc(const csi_id_t store_id) { + return get_fed_entry(FED_TYPE_STORE, store_id); +} + +CSIRT_API +const source_loc_t *__csi_get_detach_source_loc(const csi_id_t detach_id) { + return get_fed_entry(FED_TYPE_DETACH, detach_id); +} + +CSIRT_API +const source_loc_t *__csi_get_task_source_loc(const csi_id_t task_id) { + return get_fed_entry(FED_TYPE_TASK, task_id); +} + +CSIRT_API +const source_loc_t *__csi_get_task_exit_source_loc( + const csi_id_t task_exit_id) { + return get_fed_entry(FED_TYPE_TASK_EXIT, task_exit_id); +} + +CSIRT_API +const source_loc_t *__csi_get_detach_continue_source_loc( + const csi_id_t detach_continue_id) { + return get_fed_entry(FED_TYPE_DETACH_CONTINUE, detach_continue_id); +} + +CSIRT_API +const source_loc_t *__csi_get_sync_source_loc(const csi_id_t sync_id) { + return get_fed_entry(FED_TYPE_SYNC, sync_id); +} + +CSIRT_API +const sizeinfo_t *__csi_get_bb_sizeinfo(const csi_id_t bb_id) { + return get_sizeinfo_entry(SIZEINFO_TYPE_BASICBLOCK, bb_id); +} + +EXTERN_C_END diff --git a/compiler-rt/test/CMakeLists.txt b/compiler-rt/test/CMakeLists.txt index bc37c85a140a..7a54daf6c4b5 100644 --- a/compiler-rt/test/CMakeLists.txt +++ b/compiler-rt/test/CMakeLists.txt @@ -103,6 +103,9 @@ if(COMPILER_RT_CAN_EXECUTE_TESTS) if(COMPILER_RT_BUILD_ORC) compiler_rt_Test_runtime(orc) endif() + if(COMPILER_RT_HAS_CSI) + add_subdirectory(csi) + endif() # ShadowCallStack does not yet provide a runtime with compiler-rt, the tests # include their own minimal runtime add_subdirectory(shadowcallstack) diff --git a/compiler-rt/test/csi/.gitignore b/compiler-rt/test/csi/.gitignore new file mode 100644 index 000000000000..5d7b356e6faa --- /dev/null +++ b/compiler-rt/test/csi/.gitignore @@ -0,0 +1,5 @@ +*.o +# Logs from llvm crash dumps +*.o-* +*.S +*.ll diff --git a/compiler-rt/test/csi/CMakeLists.txt b/compiler-rt/test/csi/CMakeLists.txt new file mode 100644 index 000000000000..67f697af46cd --- /dev/null +++ b/compiler-rt/test/csi/CMakeLists.txt @@ -0,0 +1,36 @@ +set(CSI_TEST_DEPS ${SANITIZER_COMMON_LIT_TEST_DEPS}) + +if(NOT COMPILER_RT_STANDALONE_BUILD) + list(APPEND CSI_TEST_DEPS csi) +endif() + +# Add a dependency on the LTO plugin. +if(LLVM_BINUTILS_INCDIR) + list(APPEND CSI_TEST_DEPS LLVMgold) +endif() + +set(CSI_TESTSUITES) + +set(CSI_TEST_ARCH ${CSI_SUPPORTED_ARCH}) + +set(CSI_LIT_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + +foreach(arch ${CSI_TEST_ARCH}) + set(CSI_TEST_TARGET_ARCH ${arch}) + string(TOLOWER "-${arch}" CSI_TEST_CONFIG_SUFFIX) + get_target_flags_for_arch(${arch} CSI_TEST_TARGET_CFLAGS) + string(REPLACE ";" " " CSI_TEST_TARGET_CFLAGS "${CSI_TEST_TARGET_CFLAGS}") + + string(TOUPPER ${arch} ARCH_UPPER_CASE) + set(CONFIG_NAME ${ARCH_UPPER_CASE}Config) + + configure_lit_site_cfg( + ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.in + ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg) + list(APPEND CSI_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}) +endforeach() + +add_lit_testsuite(check-csi "Running ComprehensiveStaticInstrumentation tests" + ${CSI_TESTSUITES} + DEPENDS ${CSI_TEST_DEPS}) +set_target_properties(check-csi PROPERTIES FOLDER "Csi tests") diff --git a/compiler-rt/test/csi/fed-test.c b/compiler-rt/test/csi/fed-test.c new file mode 100644 index 000000000000..47379e5349f4 --- /dev/null +++ b/compiler-rt/test/csi/fed-test.c @@ -0,0 +1,20 @@ +// RUN: %clang_csi_toolc %tooldir/null-tool.c -o %t-null-tool.o +// RUN: %clang_csi_toolc %tooldir/fed-test-tool.c -o %t-tool.o +// RUN: %link_csi %t-tool.o %t-null-tool.o -o %t-tool.o +// RUN: %clang_csi_c %s -o %t.o +// RUN: %clang_csi %t.o %t-tool.o %csirtlib -o %t +// RUN: %run %t | FileCheck %s + +#include + +static void foo() { + printf("In foo.\n"); +} + +int main(int argc, char **argv) { + printf("In main.\n"); + foo(); + // CHECK: Enter function 0 [{{.*}}fed-test.c:14] + // CHECK: Enter function 1 [{{.*}}fed-test.c:10] + return 0; +} diff --git a/compiler-rt/test/csi/function-call-count.c b/compiler-rt/test/csi/function-call-count.c new file mode 100644 index 000000000000..1f2545818f8e --- /dev/null +++ b/compiler-rt/test/csi/function-call-count.c @@ -0,0 +1,15 @@ +// RUN: %clang_csi_toolc %tooldir/null-tool.c -o %t-null-tool.o +// RUN: %clang_csi_toolc %tooldir/function-call-count-tool.c -o %t-tool.o +// RUN: %link_csi %t-tool.o %t-null-tool.o -o %t-tool.o +// RUN: %clang_csi_c %s -o %t.o +// RUN: %clang_csi %t.o %t-tool.o %csirtlib -o %t +// RUN: %run %t | FileCheck %s + +#include + +int main(int argc, char **argv) { + printf("One call.\n"); + printf("Two calls.\n"); + // CHECK: num_function_calls = 2 + return 0; +} diff --git a/compiler-rt/test/csi/lit.cfg b/compiler-rt/test/csi/lit.cfg new file mode 100644 index 000000000000..7e205434c26a --- /dev/null +++ b/compiler-rt/test/csi/lit.cfg @@ -0,0 +1,71 @@ +# -*- Python -*- + +import glob +import os + +# Setup config name. +config.name = 'ComprehensiveStaticInstrumentation' + config.name_suffix + +# Setup source root. +config.test_source_root = os.path.dirname(__file__) + +# Setup default compiler flags used with -fcsi option. +base_cflags = ([config.target_cflags] + config.debug_info_flags) +base_cxxflags = config.cxx_mode_flags + base_cflags + +llvm_link = os.path.join(config.llvm_tools_dir, "llvm-link") +clang_cpp = config.clang + "++" + +csi_libdir = os.path.join(config.test_source_root, "..", "..", "lib", "csi") +csi_rt_lib = os.path.join(config.compiler_rt_libdir, "libclang_rt.csi-%s.a" % config.target_arch) +csi_testtoolsdir = os.path.join(config.test_source_root, "tools") +csi_testsupportdir = os.path.join(config.test_source_root, "support") + +csi_tool_cflags = (["-g", "-O0", "-c", "-emit-llvm", "-I" + csi_libdir] + base_cflags) +csi_tool_cppflags = (csi_tool_cflags + ["-fno-exceptions"]) +csi_compile_cflags = (["-g", "-O0", "-c", "-fcsi", "-emit-llvm", "-I" + csi_libdir] + base_cflags) +csi_compile_cppflags = (csi_compile_cflags + ["-fno-exceptions"]) +csi_cflags = (["-g", "-O0", "-flto", "-fuse-ld=gold"] + base_cflags) +csi_cppflags = (csi_cflags + ["-fno-exceptions"]) + +def build_invocation(compile_flags, clang=config.clang): + return " " + " ".join([clang] + compile_flags) + " " + +# clang -fcsi ... +config.substitutions.append(("%clang_csi ", + build_invocation(csi_cflags))) +# clang++ -fcsi ... +config.substitutions.append(("%clang_cpp_csi ", + build_invocation(csi_cppflags, clang=clang_cpp))) +# clang -fcsi -c ... +config.substitutions.append(("%clang_csi_c ", + build_invocation(csi_compile_cflags))) +# clang++ -fcsi -c ... +config.substitutions.append(("%clang_cpp_csi_c ", + build_invocation(csi_compile_cppflags, clang=clang_cpp))) +# clang -I -c ... +config.substitutions.append(("%clang_csi_toolc ", + build_invocation(csi_tool_cflags))) +# clang++ -I -c ... +config.substitutions.append(("%clang_cpp_csi_toolc ", + build_invocation(csi_tool_cppflags, clang=clang_cpp))) +# llvm-link ... +config.substitutions.append(("%link_csi ", + llvm_link + " ")) +config.substitutions.append(("%tooldir", csi_testtoolsdir)) +config.substitutions.append(("%supportdir", csi_testsupportdir)) +config.substitutions.append(("%csirtlib", csi_rt_lib)) + +# Default test suffixes. +config.suffixes = ['.c', '.cpp'] + +# Ignore 'tools- and 'support' +config.excludes.update(map(os.path.basename, glob.glob(os.path.join(csi_testtoolsdir, "*")))) +config.excludes.update(map(os.path.basename, glob.glob(os.path.join(csi_testsupportdir, "*")))) + +# Disable STL test for now. +config.excludes.update(["tix-and-tool-use-stl.cpp"]) + +# CSI tests are currently supported on Linux x86-64 only. +if config.host_os not in ['Linux'] or config.target_arch != 'x86_64': + config.unsupported = True diff --git a/compiler-rt/test/csi/lit.site.cfg.in b/compiler-rt/test/csi/lit.site.cfg.in new file mode 100644 index 000000000000..2457a154dec8 --- /dev/null +++ b/compiler-rt/test/csi/lit.site.cfg.in @@ -0,0 +1,14 @@ +## Autogenerated by LLVM/Clang configuration. +# Do not edit! + +# Tool-specific config options. +config.name_suffix = "@CSI_TEST_CONFIG_SUFFIX@" +config.csi_lit_source_dir = "@CSI_LIT_SOURCE_DIR@" +config.target_cflags = "@CSI_TEST_TARGET_CFLAGS@" +config.target_arch = "@CSI_TEST_TARGET_ARCH@" + +# Load common config for all compiler-rt lit tests. +lit_config.load_config(config, "@COMPILER_RT_BINARY_DIR@/test/lit.common.configured") + +# Load tool-specific config that would do the real work. +lit_config.load_config(config, "@CSI_LIT_SOURCE_DIR@/lit.cfg") diff --git a/compiler-rt/test/csi/load-property-test.c b/compiler-rt/test/csi/load-property-test.c new file mode 100644 index 000000000000..f0e5723e0b3a --- /dev/null +++ b/compiler-rt/test/csi/load-property-test.c @@ -0,0 +1,22 @@ +// RUN: %clang_csi_toolc %tooldir/null-tool.c -o %t-null-tool.o +// RUN: %clang_csi_toolc %tooldir/load-property-test-tool.c -o %t-tool.o +// RUN: %link_csi %t-tool.o %t-null-tool.o -o %t-tool.o +// RUN: %clang_csi_c %s -o %t.o +// RUN: %clang_csi %t.o %t-tool.o %csirtlib -o %t +// RUN: %run %t | FileCheck %s + +#include + +static int global = 0; + +int main(int argc, char **argv) { + int x = global + 1; // Read of global + x++; // Read-before-write of x + printf("x is %d\n", x); // Function calls partition read-before-write analysis + x += global++; // Read-before-write of global and x */ + printf("x is %d\n", x); // Read on x + printf("global is %d\n", global); // Read on global + // CHECK: num_loads = 7 + // CHECK: num_read_before_writes = 3 + return 0; +} diff --git a/compiler-rt/test/csi/multiple-units-function-call-count.c b/compiler-rt/test/csi/multiple-units-function-call-count.c new file mode 100644 index 000000000000..9fd9732ad659 --- /dev/null +++ b/compiler-rt/test/csi/multiple-units-function-call-count.c @@ -0,0 +1,21 @@ +// RUN: %clang_csi_toolc %tooldir/null-tool.c -o %t-null-tool.o +// RUN: %clang_csi_toolc %tooldir/function-call-count-tool.c -o %t-tool.o +// RUN: %link_csi %t-tool.o %t-null-tool.o -o %t-tool.o +// RUN: %clang_csi_c %s -o %t.o +// RUN: %clang_csi_c %supportdir/a.c -o %t.a.o +// RUN: %clang_csi_c %supportdir/b.c -o %t.b.o +// RUN: %clang_csi %t.o %t.a.o %t.b.o %t-tool.o %csirtlib -o %t +// RUN: %run %t | FileCheck %s + +#include + +#include "support/a.h" + +int main(int argc, char **argv) { + printf("One call.\n"); + printf("Two calls.\n"); + a(); + // Calls are: main + a + b + one printf each. + // CHECK: num_function_calls = 6 + return 0; +} diff --git a/compiler-rt/test/csi/shobj-function-call-count.c b/compiler-rt/test/csi/shobj-function-call-count.c new file mode 100644 index 000000000000..1e853b21cf84 --- /dev/null +++ b/compiler-rt/test/csi/shobj-function-call-count.c @@ -0,0 +1,19 @@ +// RUN: %clang_csi_toolc %tooldir/null-tool.c -o %t-null-tool.o +// RUN: %clang_csi_toolc %tooldir/function-call-count-tool.c -o %t-tool.o +// RUN: %link_csi %t-tool.o %t-null-tool.o -o %t-tool.o +// RUN: %clang_csi_c -fPIC %supportdir/libtest.c -o %t-libtest.o +// RUN: %clang_csi -Wl,-soname,libtest.so -shared %t-libtest.o %t-tool.o -o %T/libtest.so +// RUN: %clang_csi_c %s -o %t.o +// RUN: %clang_csi -Wl,-rpath,%T -L %T %t.o %t-tool.o -ltest %csirtlib -o %t +// RUN: LD_LIBRARY_PATH=%T:$LD_LIBRARY_PATH %run %t | FileCheck %s + +#include +#include "support/libtest.h" + +int main(int argc, char **argv) { + printf("One call.\n"); + printf("Two calls.\n"); + libtest(); + // CHECK: num_function_calls = 4 + return 0; +} diff --git a/compiler-rt/test/csi/support/a.c b/compiler-rt/test/csi/support/a.c new file mode 100644 index 000000000000..430870cc05b2 --- /dev/null +++ b/compiler-rt/test/csi/support/a.c @@ -0,0 +1,7 @@ +#include +#include "b.h" + +void a() { + printf("In a.\n"); + b(); +} diff --git a/compiler-rt/test/csi/support/a.h b/compiler-rt/test/csi/support/a.h new file mode 100644 index 000000000000..d25b22de6dea --- /dev/null +++ b/compiler-rt/test/csi/support/a.h @@ -0,0 +1 @@ +void a(); diff --git a/compiler-rt/test/csi/support/b.c b/compiler-rt/test/csi/support/b.c new file mode 100644 index 000000000000..e8a1edd682f2 --- /dev/null +++ b/compiler-rt/test/csi/support/b.c @@ -0,0 +1,5 @@ +#include + +void b() { + printf("In b.\n"); +} diff --git a/compiler-rt/test/csi/support/b.h b/compiler-rt/test/csi/support/b.h new file mode 100644 index 000000000000..8bc68e0915f1 --- /dev/null +++ b/compiler-rt/test/csi/support/b.h @@ -0,0 +1 @@ +void b(); diff --git a/compiler-rt/test/csi/support/libtest.c b/compiler-rt/test/csi/support/libtest.c new file mode 100644 index 000000000000..b25e098ab99c --- /dev/null +++ b/compiler-rt/test/csi/support/libtest.c @@ -0,0 +1,5 @@ +#include + +void libtest() { + printf("In libtest.\n"); +} diff --git a/compiler-rt/test/csi/support/libtest.h b/compiler-rt/test/csi/support/libtest.h new file mode 100644 index 000000000000..7bb2d72ad000 --- /dev/null +++ b/compiler-rt/test/csi/support/libtest.h @@ -0,0 +1 @@ +void libtest(); diff --git a/compiler-rt/test/csi/tix-and-tool-use-stl.cpp b/compiler-rt/test/csi/tix-and-tool-use-stl.cpp new file mode 100644 index 000000000000..9d52340b0af3 --- /dev/null +++ b/compiler-rt/test/csi/tix-and-tool-use-stl.cpp @@ -0,0 +1,24 @@ +// RUN: %clang_cpp_csi_toolc %tooldir/null-tool.c -o %t-null-tool.o +// RUN: %clang_cpp_csi_toolc %tooldir/function-call-count-uses-stl-tool.cpp -o %t-tool.o +// RUN: %link_csi %t-tool.o %t-null-tool.o -o %t-tool.o +// RUN: %clang_cpp_csi_c %s -o %t.o +// RUN: %clang_cpp_csi %t.o %t-tool.o %csirtlib -o %t +// RUN: %run %t | FileCheck %s + +// In this test, both the TIX and the tool use the STL. The resulting +// function call count should be the same as the test whose tool does +// not use the STL, because we want the tool's usage of STL not to be +// instrumented. + +#include +#include + +int main(int argc, char **argv) { + printf("One call.\n"); + printf("Two calls.\n"); + std::vector stackVec; + for (int i = 0; i < 10; i++) + stackVec.push_back(i); + // CHECK: num_function_calls = 412 + return 0; +} diff --git a/compiler-rt/test/csi/tix-uses-stl.cpp b/compiler-rt/test/csi/tix-uses-stl.cpp new file mode 100644 index 000000000000..a40b94c1a367 --- /dev/null +++ b/compiler-rt/test/csi/tix-uses-stl.cpp @@ -0,0 +1,22 @@ +// RUN: %clang_cpp_csi_toolc %tooldir/null-tool.c -o %t-null-tool.o +// RUN: %clang_cpp_csi_toolc %tooldir/function-call-count-tool.c -o %t-tool.o +// RUN: %link_csi %t-tool.o %t-null-tool.o -o %t-tool.o +// RUN: %clang_cpp_csi_c %s -o %t.o +// RUN: %clang_cpp_csi %t.o %t-tool.o %csirtlib -o %t +// RUN: %run %t | FileCheck %s + +// In this test, the TIX uses the STL, but the tool does not. + +#include +#include + +int main(int argc, char **argv) { + printf("One call.\n"); + printf("Two calls.\n"); + std::vector stackVec; + for (int i = 0; i < 10; i++) + stackVec.push_back(i); + // CHECK: num_function_calls = + // CHECK-NOT: {{0|1}} + return 0; +} diff --git a/compiler-rt/test/csi/tools/fed-test-tool.c b/compiler-rt/test/csi/tools/fed-test-tool.c new file mode 100644 index 000000000000..a1848bf6cc7f --- /dev/null +++ b/compiler-rt/test/csi/tools/fed-test-tool.c @@ -0,0 +1,9 @@ +#include +#include +#include "csi.h" + +void __csi_func_entry(const csi_id_t func_id, const func_prop_t prop) { + printf("Enter function %ld [%s:%d]\n", func_id, + __csi_get_func_source_loc(func_id)->filename, + __csi_get_func_source_loc(func_id)->line_number); +} diff --git a/compiler-rt/test/csi/tools/function-call-count-tool.c b/compiler-rt/test/csi/tools/function-call-count-tool.c new file mode 100644 index 000000000000..b345a54b16f1 --- /dev/null +++ b/compiler-rt/test/csi/tools/function-call-count-tool.c @@ -0,0 +1,19 @@ +#include +#include +#include "csi.h" + +static int num_function_calls = 0; + +void report() { + printf("num_function_calls = %d\n", num_function_calls); +} + +void __csi_init() { + num_function_calls = 0; + atexit(report); +} + +void __csi_before_call(const csi_id_t call_id, const csi_id_t func_id, + const call_prop_t prop) { + num_function_calls++; +} diff --git a/compiler-rt/test/csi/tools/function-call-count-uses-stl-tool.cpp b/compiler-rt/test/csi/tools/function-call-count-uses-stl-tool.cpp new file mode 100644 index 000000000000..4f248c3fe722 --- /dev/null +++ b/compiler-rt/test/csi/tools/function-call-count-uses-stl-tool.cpp @@ -0,0 +1,28 @@ +#include +#include +#include +#include "csi.h" + +static int num_function_calls = 0; +static std::vector *global_vector = NULL; + +void report() { + printf("num_function_calls = %d\n", num_function_calls); + if (global_vector) delete global_vector; +} + +extern "C" { + +void __csi_init() { + num_function_calls = 0; + global_vector = new std::vector(); + atexit(report); +} + +void __csi_before_call(const csi_id_t call_id, const csi_id_t func_id, + const csi_prop_t prop) { + global_vector->push_back(call_id); + num_function_calls++; +} + +} diff --git a/compiler-rt/test/csi/tools/load-property-test-tool.c b/compiler-rt/test/csi/tools/load-property-test-tool.c new file mode 100644 index 000000000000..bc4af8d599f9 --- /dev/null +++ b/compiler-rt/test/csi/tools/load-property-test-tool.c @@ -0,0 +1,21 @@ +#include +#include +#include "csi.h" + +static int num_loads = 0, num_read_before_writes = 0; + +void report() { + printf("num_loads = %d\n", num_loads); + printf("num_read_before_writes = %d\n", num_read_before_writes); +} + +void __csi_init() { + num_loads = num_read_before_writes = 0; + atexit(report); +} + +void __csi_before_load(const csi_id_t load_id, const void *addr, + const int32_t num_bytes, const load_prop_t prop) { + num_loads++; + if (prop.is_read_before_write_in_bb) num_read_before_writes++; +} diff --git a/compiler-rt/test/csi/tools/null-tool.c b/compiler-rt/test/csi/tools/null-tool.c new file mode 100644 index 000000000000..bdf76b25bd05 --- /dev/null +++ b/compiler-rt/test/csi/tools/null-tool.c @@ -0,0 +1,71 @@ +#include "csi.h" + +WEAK void __csi_init() {} + +__attribute__((always_inline)) +WEAK void __csi_unit_init(const char * const file_name, + const instrumentation_counts_t counts) {} + +__attribute__((always_inline)) +WEAK void __csi_before_load(const csi_id_t load_id, + const void *addr, + const int32_t num_bytes, + const load_prop_t prop) {} + +__attribute__((always_inline)) +WEAK void __csi_after_load(const csi_id_t load_id, + const void *addr, + const int32_t num_bytes, + const load_prop_t prop) {} + +__attribute__((always_inline)) +WEAK void __csi_before_store(const csi_id_t store_id, + const void *addr, + const int32_t num_bytes, + const store_prop_t prop) {} + +__attribute__((always_inline)) +WEAK void __csi_after_store(const csi_id_t store_id, + const void *addr, + const int32_t num_bytes, + const store_prop_t prop) {} + +__attribute__((always_inline)) +WEAK void __csi_func_entry(const csi_id_t func_id, const func_prop_t prop) {} + +__attribute__((always_inline)) +WEAK void __csi_func_exit(const csi_id_t func_exit_id, + const csi_id_t func_id, const func_exit_prop_t prop) {} + +__attribute__((always_inline)) +WEAK void __csi_bb_entry(const csi_id_t bb_id, const bb_prop_t prop) {} + +__attribute__((always_inline)) +WEAK void __csi_bb_exit(const csi_id_t bb_id, const bb_prop_t prop) {} + +__attribute__((always_inline)) +WEAK void __csi_before_call(csi_id_t callsite_id, csi_id_t func_id, + const call_prop_t prop) {} + +__attribute__((always_inline)) +WEAK void __csi_after_call(csi_id_t callsite_id, csi_id_t func_id, + const call_prop_t prop) {} + +__attribute__((always_inline)) +WEAK void __csi_detach(const csi_id_t detach_id) {} + +__attribute__((always_inline)) +WEAK void __csi_task(const csi_id_t task_id, const csi_id_t detach_id, + void *sp) {} + +__attribute__((always_inline)) +WEAK void __csi_task_exit(const csi_id_t task_exit_id, + const csi_id_t task_id, + const csi_id_t detach_id) {} + +__attribute__((always_inline)) +WEAK void __csi_detach_continue(const csi_id_t detach_continue_id, + const csi_id_t detach_id) {} + +__attribute__((always_inline)) +WEAK void __csi_sync(const csi_id_t sync_id) {} diff --git a/compiler-rt/test/csi/tools/unknown-function-call-count-tool.c b/compiler-rt/test/csi/tools/unknown-function-call-count-tool.c new file mode 100644 index 000000000000..06833a02a3f6 --- /dev/null +++ b/compiler-rt/test/csi/tools/unknown-function-call-count-tool.c @@ -0,0 +1,21 @@ +#include +#include +#include "csi.h" + +static int num_function_calls = 0, num_unknown_function_calls = 0; + +void report() { + printf("num_function_calls = %d\n", num_function_calls); + printf("num_unknown_function_calls = %d\n", num_unknown_function_calls); +} + +void __csi_init() { + num_function_calls = num_unknown_function_calls = 0; + atexit(report); +} + +void __csi_before_call(const csi_id_t call_id, const csi_id_t func_id, + const call_prop_t prop) { + num_function_calls++; + if (func_id == UNKNOWN_CSI_ID) num_unknown_function_calls++; +} diff --git a/compiler-rt/test/csi/unknown-function-call-count.c b/compiler-rt/test/csi/unknown-function-call-count.c new file mode 100644 index 000000000000..82d6c0fbdc55 --- /dev/null +++ b/compiler-rt/test/csi/unknown-function-call-count.c @@ -0,0 +1,21 @@ +// RUN: %clang_csi_toolc %tooldir/null-tool.c -o %t-null-tool.o +// RUN: %clang_csi_toolc %tooldir/unknown-function-call-count-tool.c -o %t-tool.o +// RUN: %link_csi %t-tool.o %t-null-tool.o -o %t-tool.o +// RUN: %clang_csi_c %s -o %t.o +// RUN: %clang_csi %t.o %t-tool.o %csirtlib -o %t +// RUN: %run %t | FileCheck %s + +#include + +static void foo() { + printf("In foo.\n"); +} + +int main(int argc, char **argv) { + printf("One call.\n"); + printf("Two calls.\n"); + foo(); + // CHECK: num_function_calls = 4 + // CHECK: num_unknown_function_calls = 3 + return 0; +}