Enable memtracing for stb and zstd (#227)

- **ADDED** Longtail_SetReAllocAndFree
- **ADDED** Longtail_ReAlloc
- **ADDED** Longtail_MemTracer_ReAlloc
- **ADDED** memtracer now tracks allocations in stb_ds
- **ADDED** memtracer now tracks allocations in zstd

CHANGELOG.md

@@ -1,4 +1,11 @@
 ##
+- **ADDED** Longtail_SetReAllocAndFree
+- **ADDED** Longtail_ReAlloc
+- **ADDED** Longtail_MemTracer_ReAlloc
+- **ADDED** memtracer now tracks allocations in stb_ds
+- **ADDED** memtracer now tracks allocations in zstd
+
+## 0.4.1
 - **NEW API** `Longtail_ChangeVersion2` added
 
   Implements a new strategy for decompressing/writing version assets which is significantly faster for files that spans multiple blocks while retaining the same speed for assets smaller than a block. It removes redundant decompression of blocks (so the LRU block store is no longer needed) at the expense of doing random access when writing files.

default_build_options.bat

@@ -1,2 +1,2 @@
 set CXXFLAGS=%CXXFLAGS% /wd4244 /wd4316 /wd4996 /DLONGTAIL_LOG_LEVEL=5 /D__SSE2__
-set CXXFLAGS_DEBUG=%CXXFLAGS_DEBUG% /DBIKESHED_ASSERTS /DLONGTAIL_ASSERTS /D_DEBUG /DLONGTAIL_LOG_LEVEL=3 /D__SSE2__ /DLONGTAIL_EXPORT_SYMBOLS /DZSTDLIB_VISIBILITY="" /DLZ4LIB_VISIBILITY=""
+set CXXFLAGS_DEBUG=%CXXFLAGS_DEBUG% /DBIKESHED_ASSERTS /DLONGTAIL_ASSERTS /D_DEBUG /DLONGTAIL_LOG_LEVEL=3 /D__SSE2__ /DLONGTAIL_EXPORT_SYMBOLS /DZSTDLIB_VISIBILITY="" /DLZ4LIB_VISIBILITY="" /DSTBDS_REALLOC=Longtail_STBRealloc /DSTBDS_FREE=Longtail_STBFree

default_build_options.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
 export CXXFLAGS="$CXXFLAGS -pthread -U_WIN32 -DLONGTAIL_LOG_LEVEL=5"
-export CXXFLAGS_DEBUG="$CXXFLAGS_DEBUG -DBIKESHED_ASSERTS -DLONGTAIL_LOG_LEVEL=3 -DLONGTAIL_ASSERTS"
+export CXXFLAGS_DEBUG="$CXXFLAGS_DEBUG -DBIKESHED_ASSERTS -DLONGTAIL_LOG_LEVEL=3 -DLONGTAIL_ASSERTS -DSTBDS_REALLOC=Longtail_STBRealloc -DSTBDS_FREE=Longtail_STBFree"

lib/blockstorestorage/longtail_blockstorestorage.c

@@ -8,6 +8,7 @@
 
 #include <errno.h>
 #include <inttypes.h>
+#include <stdlib.h>
 #include <string.h>
 
 #if !defined(alloca)

lib/memtracer/longtail_memtracer.c

@@ -279,18 +279,33 @@ void Longtail_MemTracer_Dispose() {
     gMemTracer_Context = 0;
 }
 
-void* Longtail_MemTracer_Alloc(const char* context, size_t s)
+void* Longtail_MemTracer_ReAlloc(const char* context, void* old, size_t s)
 {
 #if defined(LONGTAIL_ASSERTS)
     const char* context_safe = context ? context : "";
     MAKE_LOG_CONTEXT_FIELDS(ctx)
         LONGTAIL_LOGFIELD(context_safe, "%s"),
-        LONGTAIL_LOGFIELD(s, "%" PRIu64)
+        LONGTAIL_LOGFIELD(s, "%" PRIu64),
+        LONGTAIL_LOGFIELD(old, "%p")
     MAKE_LOG_CONTEXT_WITH_FIELDS(ctx, 0, LONGTAIL_LOG_LEVEL_OFF)
 #else
     struct Longtail_LogContextFmt_Private* ctx = 0;
 #endif // defined(LONGTAIL_ASSERTS)
-    uint32_t context_id = context ? MemTracer_ContextIdHash(context) : 0;
+    void* realloc_ptr = 0;
+    size_t old_size = 0;
+    uint32_t context_id = 0;
+    if (old)
+    {
+        struct MemTracer_Header* header_ptr = (struct MemTracer_Header*)old;
+        --header_ptr;
+        context_id = header_ptr->id;
+        realloc_ptr = header_ptr;
+        old_size = header_ptr->size;
+    }
+    else
+    {
+        context_id = context ? MemTracer_ContextIdHash(context) : 0;
+    }
 
     struct MemTracer_ContextStats* contextStats = 0;
     Longtail_LockSpinLock(gMemTracer_Context->m_Spinlock);
@@ -306,7 +321,13 @@ void* Longtail_MemTracer_Alloc(const char* context, size_t s)
     {
         contextStats = &gMemTracer_Context->m_ContextStats[*context_index_ptr];
     }
-
+    if (old)
+    {
+        gMemTracer_Context->m_AllocationCurrentMem -= old_size;
+        gMemTracer_Context->m_AllocationCurrentCount--;
+        contextStats->current_mem -= old_size;
+        contextStats->current_count--;
+    }
     contextStats->total_count++;
     contextStats->current_count++;
     contextStats->total_mem += s;
@@ -319,7 +340,7 @@ void* Longtail_MemTracer_Alloc(const char* context, size_t s)
     {
         contextStats->peak_count = contextStats->current_count;
     }
-    
+
     gMemTracer_Context->m_AllocationTotalCount++;
     gMemTracer_Context->m_AllocationCurrentCount++;
     gMemTracer_Context->m_AllocationTotalMem += s;
@@ -346,13 +367,27 @@ void* Longtail_MemTracer_Alloc(const char* context, size_t s)
     Longtail_UnlockSpinLock(gMemTracer_Context->m_Spinlock);
 
     size_t padded_size = sizeof(struct MemTracer_Header) + s;
-    void* mem = malloc(padded_size);
+    void* mem = realloc(realloc_ptr, padded_size);
     struct MemTracer_Header* header_ptr = (struct MemTracer_Header*)mem;
     header_ptr->id = context_id;
     header_ptr->size = s;
     return &header_ptr[1];
 }
 
+void* Longtail_MemTracer_Alloc(const char* context, size_t s)
+{
+#if defined(LONGTAIL_ASSERTS)
+    const char* context_safe = context ? context : "";
+    MAKE_LOG_CONTEXT_FIELDS(ctx)
+        LONGTAIL_LOGFIELD(context_safe, "%s"),
+        LONGTAIL_LOGFIELD(s, "%" PRIu64),
+    MAKE_LOG_CONTEXT_WITH_FIELDS(ctx, 0, LONGTAIL_LOG_LEVEL_OFF)
+#else
+    struct Longtail_LogContextFmt_Private* ctx = 0;
+#endif // defined(LONGTAIL_ASSERTS)
+    return Longtail_MemTracer_ReAlloc(context, 0, s);
+}
+
 void Longtail_MemTracer_Free(void* p)
 {
 #if defined(LONGTAIL_ASSERTS)

lib/memtracer/longtail_memtracer.h

@@ -13,6 +13,7 @@ LONGTAIL_EXPORT void Longtail_MemTracer_Init();
 LONGTAIL_EXPORT char* Longtail_MemTracer_GetStats(uint32_t log_level);
 LONGTAIL_EXPORT void Longtail_MemTracer_Dispose();
 LONGTAIL_EXPORT void* Longtail_MemTracer_Alloc(const char* context, size_t s);
+LONGTAIL_EXPORT void* Longtail_MemTracer_ReAlloc(const char* context, void* old, size_t s);
 LONGTAIL_EXPORT void Longtail_MemTracer_Free(void* p);
 
 LONGTAIL_EXPORT int Longtail_MemTracer_DumpStats(const char* name);

lib/zstd/longtail_zstd.c

@@ -74,6 +74,36 @@ static size_t ZStdCompressionAPI_GetMaxCompressedSize(struct Longtail_Compressio
     return ZSTD_COMPRESSBOUND(size);
 }
 
+static void* alloc(void* opaque, size_t size)
+{
+    return Longtail_Alloc("ZStdCompressionAPI", size);
+}
+
+static void  free(void* opaque, void* address)
+{
+    Longtail_Free(address);
+}
+
+ZSTD_CCtx* ZSTD_CreateCompressContext()
+{
+    ZSTD_customMem customMem;
+    customMem.customAlloc = alloc;
+    customMem.customFree = free;
+    customMem.opaque = 0;
+    ZSTD_CCtx* ctx = ZSTD_createCCtx_advanced(customMem);
+    return ctx;
+}
+
+ZSTD_DCtx* ZSTD_CreateDecompressContext()
+{
+    ZSTD_customMem customMem;
+    customMem.customAlloc = alloc;
+    customMem.customFree = free;
+    customMem.opaque = 0;
+    ZSTD_DCtx* ctx = ZSTD_createDCtx_advanced(customMem);
+    return ctx;
+}
+
 int ZStdCompressionAPI_Compress(struct Longtail_CompressionAPI* compression_api, uint32_t settings_id, const char* uncompressed, char* compressed, size_t uncompressed_size, size_t max_compressed_size, size_t* out_compressed_size)
 {
 #if defined(LONGTAIL_ASSERTS)
@@ -89,16 +119,24 @@ int ZStdCompressionAPI_Compress(struct Longtail_CompressionAPI* compression_api,
 #else
     struct Longtail_LogContextFmt_Private* ctx = 0;
 #endif // defined(LONGTAIL_ASSERTS)
-
+    ZSTD_CCtx* zstd_ctx = ZSTD_CreateCompressContext();
+    if (!zstd_ctx)
+    {
+        int err = ENOMEM;
+        LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "ZSTD_CreateContext() failed with %d", err);
+        return err;
+    }
     int compression_setting = SettingsIDToCompressionSetting(settings_id);
-    size_t size = ZSTD_compress(compressed, max_compressed_size, uncompressed, uncompressed_size, compression_setting);
+    size_t size = ZSTD_compressCCtx(zstd_ctx, compressed, max_compressed_size, uncompressed, uncompressed_size, compression_setting);
     if (ZSTD_isError(size))
     {
         int err = ZSTD_getErrorCode(size);
         LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "ZSTD_compress() failed with %d", err);
+        ZSTD_freeCCtx(zstd_ctx);
         return EINVAL;
     }
     *out_compressed_size = size;
+    ZSTD_freeCCtx(zstd_ctx);
     return 0;
 }
 
@@ -117,15 +155,24 @@ int ZStdCompressionAPI_Decompress(struct Longtail_CompressionAPI* compression_ap
 #else
     struct Longtail_LogContextFmt_Private* ctx = 0;
 #endif // defined(LONGTAIL_ASSERTS)
+    ZSTD_DCtx* zstd_ctx = ZSTD_CreateDecompressContext();
+    if (!zstd_ctx)
+    {
+        int err = ENOMEM;
+        LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "ZSTD_CreateContext() failed with %d", err);
+        return err;
+    }
 
-    size_t size = ZSTD_decompress(uncompressed, max_uncompressed_size, compressed, compressed_size);
+    size_t size = ZSTD_decompressDCtx(zstd_ctx, uncompressed, max_uncompressed_size, compressed, compressed_size);
     if (ZSTD_isError(size))
     {
         int err = ZSTD_getErrorCode(size);
         LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "ZSTD_decompress() failed with %d", err);
+        ZSTD_freeDCtx(zstd_ctx);
         return EINVAL;
     }
     *out_uncompressed_size = size;
+    ZSTD_freeDCtx(zstd_ctx);
     return 0;
 }
 

premake5.lua

@@ -18,7 +18,7 @@ workspace "longtail"
 
    filter "configurations:Debug"
       runtime "Debug"
-      defines { "DEBUG", "BIKESHED_ASSERTS",  "LONGTAIL_ASSERTS", "LONGTAIL_LOG_LEVEL=3", "__SSE2__",  "LONGTAIL_EXPORT_SYMBOLS"}
+      defines { "DEBUG", "BIKESHED_ASSERTS",  "LONGTAIL_ASSERTS", "LONGTAIL_LOG_LEVEL=3", "__SSE2__",  "LONGTAIL_EXPORT_SYMBOLS", "STBDS_REALLOC=Longtail_STBRealloc", "STBDS_FREE=Longtail_STBFree"}
 
    filter "configurations:Release"
       runtime "Release"

src/ext/stb_ds.c

@@ -1,64 +1,22 @@
 #define STB_DS_IMPLEMENTATION
 
-//#define CUSTOM_STB_ALLOC
-
 #ifdef STBDS_REALLOC
 
-#define CUSTOM_STB_ALLOC 1
-extern void* Longtail_STBRealloc(void* old_ptr, size_t size);
-extern void Longtail_STBFree(void* ptr);
-
-
-// TODO These needs to be defined at a global level!
-//#define STBDS_REALLOC(context,ptr,size) Longtail_STBRealloc((ptr), (size))
-//#define STBDS_FREE(context,ptr)         Longtail_STBFree((ptr))
-
-#endif // STBDS_REALLOC
-
+#include "../longtail.h"
 #include "stb_ds.h"
 
-#if CUSTOM_STB_ALLOC
-
-#include <stddef.h>
-#include <stdint.h>
-
-extern void* Longtail_Alloc(const char* context, size_t s);
-extern void Longtail_Free(void* p);
-
-struct Longtail_STBAllocHeader
-{
-	uint64_t Size;
-	uint64_t _;
-};
-
-inline void* Longtail_STBRealloc(void* old_ptr, size_t size)
+void* Longtail_STBRealloc(void* context, void* old_ptr, size_t size)
 {
-	if (size == 0)
-	{
-		return 0;
-	}
-	struct Longtail_STBAllocHeader* header = (struct Longtail_STBAllocHeader*)Longtail_Alloc("stb", sizeof(struct Longtail_STBAllocHeader) + size);
-	header->Size = size;
-	header->_ = 0;
-	void* new_ptr = &header[1];
-	if (old_ptr != 0)
-	{
-		struct Longtail_STBAllocHeader* old_header = &((struct Longtail_STBAllocHeader*)old_ptr)[-1];
-		size_t old_size = old_header->Size;
-		memcpy(new_ptr, old_ptr, old_size);
-		Longtail_Free((void*)old_header);
-	}
-	return new_ptr;
+	return Longtail_ReAlloc("stb", old_ptr, size);
 }
 
-inline void Longtail_STBFree(void* ptr)
+void Longtail_STBFree(void* context, void* ptr)
 {
-	if (!ptr)
-	{
-		return;
-	}
-	struct Longtail_STBAllocHeader* header = &((struct Longtail_STBAllocHeader*)ptr)[-1];
-	Longtail_Free((void*)header);
+	Longtail_Free(ptr);
 }
 
-#endif // CUSTOM_STB_ALLOC
+#else
+
+#include "stb_ds.h"
+
+#endif // STBDS_REALLOC

src/longtail.c

@@ -7,40 +7,11 @@
 #include "ext/stb_ds.h"
 
 #include <ctype.h>
-#include <stdio.h>
+#include <errno.h>
 #include <inttypes.h>
 #include <stdarg.h>
-#include <errno.h>
-
-#if !defined(alloca)
-    #if defined(__GLIBC__) || defined(__sun) || defined(__CYGWIN__)
-        #include <alloca.h>     // alloca
-    #elif defined(_WIN32)
-        #include <malloc.h>     // alloca
-        #if !defined(alloca)
-            #define alloca _alloca  // for clang with MS Codegen
-        #endif
-        #define CompareIgnoreCase _stricmp
-    #else
-        #include <stdlib.h>     // alloca
-    #endif
-#endif
-
-/*
-#if defined(LONGTAIL_ASSERTS)
-void* Longtail_NukeMalloc(size_t s);
-void Longtail_NukeFree(void* p);
-#    define Longtail_Alloc(s) \
-        Longtail_NukeMalloc(s)
-#    define Longtail_Free(p) \
-        Longtail_NukeFree(p)
-#else
-#    define Longtail_Alloc(s) \
-        malloc(s)
-#    define Longtail_Free(p) \
-        Longtail_Free(p)
-#endif // defined(LONGTAIL_ASSERTS)
-*/
+#include <stdio.h>
+#include <stdlib.h>
 
 #define LONGTAIL_VERSION(major, minor, patch)  ((((uint32_t)major) << 24) | ((uint32_t)minor << 16) | ((uint32_t)patch))
 #define LONGTAIL_VERSION_INDEX_VERSION_0_0_2  LONGTAIL_VERSION(0,0,2)
@@ -784,6 +755,7 @@ void Longtail_DisposeAPI(struct Longtail_API* api)
 }
 
 static Longtail_Alloc_Func Longtail_Alloc_private = 0;
+static Longtail_ReAlloc_Func Longtail_ReAlloc_private = 0;
 static Longtail_Free_Func Free_private = 0;
 
 void Longtail_SetAllocAndFree(Longtail_Alloc_Func alloc, Longtail_Free_Func Longtail_Free)
@@ -792,6 +764,18 @@ void Longtail_SetAllocAndFree(Longtail_Alloc_Func alloc, Longtail_Free_Func Long
     Free_private = Longtail_Free;
 }
 
+static void* Longtail_SetAllocWrapper_private(const char* context, size_t s)
+{
+    return Longtail_ReAlloc_private(context, 0, s);
+}
+
+void Longtail_SetReAllocAndFree(Longtail_ReAlloc_Func alloc, Longtail_Free_Func Longtail_Free)
+{
+    Longtail_ReAlloc_private = alloc;
+    Longtail_Alloc_private = Longtail_SetAllocWrapper_private;
+    Free_private = Longtail_Free;
+}
+
 void* Longtail_Alloc(const char* context, size_t s)
 {
 #if defined(LONGTAIL_ASSERTS)
@@ -810,6 +794,25 @@ void* Longtail_Alloc(const char* context, size_t s)
     return mem;
 }
 
+void* Longtail_ReAlloc(const char* context, void* old, size_t s)
+{
+#if defined(LONGTAIL_ASSERTS)
+    MAKE_LOG_CONTEXT_FIELDS(ctx)
+        LONGTAIL_LOGFIELD(s, "%" PRIu64),
+        LONGTAIL_LOGFIELD(old, "%p")
+    MAKE_LOG_CONTEXT_WITH_FIELDS(ctx, 0, LONGTAIL_LOG_LEVEL_OFF)
+#else
+    struct Longtail_LogContextFmt_Private* ctx = 0;
+#endif // defined(LONGTAIL_ASSERTS)
+    void* mem = Longtail_ReAlloc_private ? Longtail_ReAlloc_private(context, old, s) : realloc(old, s);
+    if (!mem)
+    {
+        LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "%s failed with %d", Longtail_ReAlloc_private ? "Longtail_ReAlloc_private" : "realloc()", ENOMEM);
+        return 0;
+    }
+    return mem;
+}
+
 void Longtail_Free(void* p)
 {
     Free_private ? Free_private(p) : free(p);