feat(concat): use arena allocations

video/concat/arena.h

@@ -0,0 +1,293 @@
+// Copyright 2022 Alexey Kutepov <[email protected]>
+
+// Permission is hereby granted, free of charge, to any person obtaining
+// a copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to
+// permit persons to whom the Software is furnished to do so, subject to
+// the following conditions:
+
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+#ifndef ARENA_H_
+#define ARENA_H_
+
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+
+#ifndef ARENA_NOSTDIO
+#include <stdarg.h>
+#include <stdio.h>
+#endif // ARENA_NOSTDIO
+
+#ifndef ARENA_ASSERT
+#include <assert.h>
+#define ARENA_ASSERT assert
+#endif
+
+#define ARENA_BACKEND_LIBC_MALLOC 0
+#define ARENA_BACKEND_LINUX_MMAP 1
+#define ARENA_BACKEND_WIN32_VIRTUALALLOC 2
+#define ARENA_BACKEND_WASM_HEAPBASE 3
+
+#ifndef ARENA_BACKEND
+#define ARENA_BACKEND ARENA_BACKEND_LIBC_MALLOC
+#endif // ARENA_BACKEND
+
+typedef struct Region Region;
+
+struct Region {
+  Region *next;
+  size_t count;
+  size_t capacity;
+  uintptr_t data[];
+};
+
+typedef struct {
+  Region *begin, *end;
+} Arena;
+
+#define REGION_DEFAULT_CAPACITY (8 * 1024)
+
+Region *new_region(size_t capacity);
+void free_region(Region *r);
+
+// TODO: snapshot/rewind capability for the arena
+// - Snapshot should be combination of a->end and a->end->count.
+// - Rewinding should be restoring a->end and a->end->count from the snapshot
+// and setting count-s of all the Region-s after the remembered a->end to 0.
+void *arena_alloc(Arena *a, size_t size_bytes);
+void *arena_realloc(Arena *a, void *oldptr, size_t oldsz, size_t newsz);
+char *arena_strdup(Arena *a, const char *cstr);
+void *arena_memdup(Arena *a, void *data, size_t size);
+#ifndef ARENA_NOSTDIO
+char *arena_sprintf(Arena *a, const char *format, ...);
+#endif // ARENA_NOSTDIO
+
+void arena_reset(Arena *a);
+void arena_free(Arena *a);
+
+#define ARENA_DA_INIT_CAP 256
+
+#ifdef __cplusplus
+#define cast_ptr(ptr) (decltype(ptr))
+#else
+#define cast_ptr(...)
+#endif
+
+#define arena_da_append(a, da, item)                                           \
+  do {                                                                         \
+    if ((da)->count >= (da)->capacity) {                                       \
+      size_t new_capacity =                                                    \
+          (da)->capacity == 0 ? ARENA_DA_INIT_CAP : (da)->capacity * 2;        \
+      (da)->items = cast_ptr((da)->items) arena_realloc(                       \
+          (a), (da)->items, (da)->capacity * sizeof(*(da)->items),             \
+          new_capacity * sizeof(*(da)->items));                                \
+      (da)->capacity = new_capacity;                                           \
+    }                                                                          \
+                                                                               \
+    (da)->items[(da)->count++] = (item);                                       \
+  } while (0)
+
+#endif // ARENA_H_
+
+#ifdef ARENA_IMPLEMENTATION
+
+#if ARENA_BACKEND == ARENA_BACKEND_LIBC_MALLOC
+#include <stdlib.h>
+
+// TODO: instead of accepting specific capacity new_region() should accept the
+// size of the object we want to fit into the region It should be up to
+// new_region() to decide the actual capacity to allocate
+Region *new_region(size_t capacity) {
+  size_t size_bytes = sizeof(Region) + sizeof(uintptr_t) * capacity;
+  // TODO: it would be nice if we could guarantee that the regions are allocated
+  // by ARENA_BACKEND_LIBC_MALLOC are page aligned
+  Region *r = (Region *)malloc(size_bytes);
+  ARENA_ASSERT(r);
+  r->next = NULL;
+  r->count = 0;
+  r->capacity = capacity;
+  return r;
+}
+
+void free_region(Region *r) { free(r); }
+#elif ARENA_BACKEND == ARENA_BACKEND_LINUX_MMAP
+#include <sys/mman.h>
+#include <unistd.h>
+
+Region *new_region(size_t capacity) {
+  size_t size_bytes = sizeof(Region) + sizeof(uintptr_t) * capacity;
+  Region *r = mmap(NULL, size_bytes, PROT_READ | PROT_WRITE,
+                   MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+  ARENA_ASSERT(r != MAP_FAILED);
+  r->next = NULL;
+  r->count = 0;
+  r->capacity = capacity;
+  return r;
+}
+
+void free_region(Region *r) {
+  size_t size_bytes = sizeof(Region) + sizeof(uintptr_t) * r->capacity;
+  int ret = munmap(r, size_bytes);
+  ARENA_ASSERT(ret == 0);
+}
+
+#elif ARENA_BACKEND == ARENA_BACKEND_WIN32_VIRTUALALLOC
+
+#if !defined(_WIN32)
+#error "Current platform is not Windows"
+#endif
+
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+
+#define INV_HANDLE(x) (((x) == NULL) || ((x) == INVALID_HANDLE_VALUE))
+
+Region *new_region(size_t capacity) {
+  SIZE_T size_bytes = sizeof(Region) + sizeof(uintptr_t) * capacity;
+  Region *r = VirtualAllocEx(
+      GetCurrentProcess(),      /* Allocate in current process address space */
+      NULL,                     /* Unknown position */
+      size_bytes,               /* Bytes to allocate */
+      MEM_COMMIT | MEM_RESERVE, /* Reserve and commit allocated page */
+      PAGE_READWRITE            /* Permissions ( Read/Write )*/
+  );
+  if (INV_HANDLE(r))
+    ARENA_ASSERT(0 && "VirtualAllocEx() failed.");
+
+  r->next = NULL;
+  r->count = 0;
+  r->capacity = capacity;
+  return r;
+}
+
+void free_region(Region *r) {
+  if (INV_HANDLE(r))
+    return;
+
+  BOOL free_result = VirtualFreeEx(
+      GetCurrentProcess(), /* Deallocate from current process address space */
+      (LPVOID)r,           /* Address to deallocate */
+      0,          /* Bytes to deallocate ( Unknown, deallocate entire page ) */
+      MEM_RELEASE /* Release the page ( And implicitly decommit it ) */
+  );
+
+  if (FALSE == free_result)
+    ARENA_ASSERT(0 && "VirtualFreeEx() failed.");
+}
+
+#elif ARENA_BACKEND == ARENA_BACKEND_WASM_HEAPBASE
+#error "TODO: WASM __heap_base backend is not implemented yet"
+#else
+#error "Unknown Arena backend"
+#endif
+
+// TODO: add debug statistic collection mode for arena
+// Should collect things like:
+// - How many times new_region was called
+// - How many times existing region was skipped
+// - How many times allocation exceeded REGION_DEFAULT_CAPACITY
+
+void *arena_alloc(Arena *a, size_t size_bytes) {
+  size_t size = (size_bytes + sizeof(uintptr_t) - 1) / sizeof(uintptr_t);
+
+  if (a->end == NULL) {
+    ARENA_ASSERT(a->begin == NULL);
+    size_t capacity = REGION_DEFAULT_CAPACITY;
+    if (capacity < size)
+      capacity = size;
+    a->end = new_region(capacity);
+    a->begin = a->end;
+  }
+
+  while (a->end->count + size > a->end->capacity && a->end->next != NULL) {
+    a->end = a->end->next;
+  }
+
+  if (a->end->count + size > a->end->capacity) {
+    ARENA_ASSERT(a->end->next == NULL);
+    size_t capacity = REGION_DEFAULT_CAPACITY;
+    if (capacity < size)
+      capacity = size;
+    a->end->next = new_region(capacity);
+    a->end = a->end->next;
+  }
+
+  void *result = &a->end->data[a->end->count];
+  a->end->count += size;
+  return result;
+}
+
+void *arena_realloc(Arena *a, void *oldptr, size_t oldsz, size_t newsz) {
+  if (newsz <= oldsz)
+    return oldptr;
+  void *newptr = arena_alloc(a, newsz);
+  char *newptr_char = (char *)newptr;
+  char *oldptr_char = (char *)oldptr;
+  for (size_t i = 0; i < oldsz; ++i) {
+    newptr_char[i] = oldptr_char[i];
+  }
+  return newptr;
+}
+
+char *arena_strdup(Arena *a, const char *cstr) {
+  size_t n = strlen(cstr);
+  char *dup = (char *)arena_alloc(a, n + 1);
+  memcpy(dup, cstr, n);
+  dup[n] = '\0';
+  return dup;
+}
+
+void *arena_memdup(Arena *a, void *data, size_t size) {
+  return memcpy(arena_alloc(a, size), data, size);
+}
+
+#ifndef ARENA_NOSTDIO
+char *arena_sprintf(Arena *a, const char *format, ...) {
+  va_list args;
+  va_start(args, format);
+  int n = vsnprintf(NULL, 0, format, args);
+  va_end(args);
+
+  ARENA_ASSERT(n >= 0);
+  char *result = (char *)arena_alloc(a, n + 1);
+  va_start(args, format);
+  vsnprintf(result, n + 1, format, args);
+  va_end(args);
+
+  return result;
+}
+#endif // ARENA_NOSTDIO
+
+void arena_reset(Arena *a) {
+  for (Region *r = a->begin; r != NULL; r = r->next) {
+    r->count = 0;
+  }
+
+  a->end = a->begin;
+}
+
+void arena_free(Arena *a) {
+  Region *r = a->begin;
+  while (r) {
+    Region *r0 = r;
+    r = r->next;
+    free_region(r0);
+  }
+  a->begin = NULL;
+  a->end = NULL;
+}
+
+#endif // ARENA_IMPLEMENTATION

video/concat/concat.c

@@ -1,5 +1,7 @@
+#define ARENA_IMPLEMENTATION
 #include "concat.h"
 
+#include "arena.h"
 #include <inttypes.h>
 #include <libavformat/avformat.h>
 #include <libavutil/avutil.h>
@@ -83,6 +85,8 @@ int concat(void *ctx, const char *output_file, size_t input_files_count,
     return 0;
   }
 
+  Arena arena = {0};
+
   go_span span = NULL;
   AVFormatContext *ifmt_ctx = NULL, *ofmt_ctx = NULL;
   AVPacket *pkt = NULL;
@@ -102,23 +106,25 @@ int concat(void *ctx, const char *output_file, size_t input_files_count,
   int ret;
 
   // Alloc arrays
-  stream_mapping = av_calloc(input_files_count, sizeof(*stream_mapping));
+  stream_mapping =
+      arena_alloc(&arena, input_files_count * sizeof(*stream_mapping));
   if (!stream_mapping) {
     ret = AVERROR(ENOMEM);
     goto end;
   }
   stream_mapping_size =
-      av_calloc(input_files_count, sizeof(*stream_mapping_size));
+      arena_alloc(&arena, input_files_count * sizeof(*stream_mapping_size));
   if (!stream_mapping_size) {
     ret = AVERROR(ENOMEM);
     goto end;
   }
-  prev_dts = av_calloc(input_files_count, sizeof(*prev_dts));
+  prev_dts = arena_alloc(&arena, input_files_count * sizeof(*prev_dts));
   if (!prev_dts) {
     ret = AVERROR(ENOMEM);
     goto end;
   }
-  prev_duration = av_calloc(input_files_count, sizeof(*prev_duration));
+  prev_duration =
+      arena_alloc(&arena, input_files_count * sizeof(*prev_duration));
   if (!prev_duration) {
     ret = AVERROR(ENOMEM);
     goto end;
@@ -163,24 +169,27 @@ int concat(void *ctx, const char *output_file, size_t input_files_count,
     // Alloc array of streams
     stream_mapping_size[input_idx] = ifmt_ctx->nb_streams;
     stream_mapping[input_idx] =
-        av_calloc(stream_mapping_size[input_idx], sizeof(*stream_mapping));
+        arena_alloc(&arena, stream_mapping_size[input_idx] *
+                                sizeof(*stream_mapping[input_idx]));
     if (!stream_mapping) {
       ret = AVERROR(ENOMEM);
       goto end;
     }
-    dts_offset = av_calloc(stream_mapping_size[input_idx], sizeof(*dts_offset));
+    dts_offset = arena_alloc(&arena, stream_mapping_size[input_idx] *
+                                         sizeof(*dts_offset));
     if (!dts_offset) {
       ret = AVERROR(ENOMEM);
       goto end;
     }
     prev_dts[input_idx] =
-        av_calloc(stream_mapping_size[input_idx], sizeof(**prev_dts));
+        arena_alloc(&arena, stream_mapping_size[input_idx] * sizeof(*prev_dts));
     if (!prev_dts[input_idx]) {
       ret = AVERROR(ENOMEM);
       goto end;
     }
     prev_duration[input_idx] =
-        av_calloc(stream_mapping_size[input_idx], sizeof(**prev_duration));
+        arena_alloc(&arena, stream_mapping_size[input_idx] *
+                                sizeof(*prev_duration[input_idx]));
     if (!prev_duration[input_idx]) {
       ret = AVERROR(ENOMEM);
       goto end;
@@ -326,20 +335,7 @@ int concat(void *ctx, const char *output_file, size_t input_files_count,
   if (ofmt_ctx)
     avformat_free_context(ofmt_ctx);
 
-  av_freep(&dts_offset);
-  for (size_t i = 0; i < input_files_count; i++) {
-    av_freep(&prev_dts[i]);
-  }
-  av_freep(&prev_dts);
-  for (size_t i = 0; i < input_files_count; i++) {
-    av_freep(&prev_duration[i]);
-  }
-  av_freep(&prev_duration);
-  for (size_t i = 0; i < input_files_count; i++) {
-    av_freep(&stream_mapping[i]);
-  }
-  av_freep(&stream_mapping);
-  av_freep(&stream_mapping_size);
+  arena_free(&arena);
 
   if (opts)
     av_dict_free(&opts);