Jtv/strcontainer (#41)

* Add a really simple data type base for strings, buffers and utf-32 intended for the con4m runtime (non-memory managed, c string compatable, but tracks length)

* Add some basic stuff here to support con4m

* Add libffi for x86 linux

* Some visibility changes

* linux-arm fully built

* Helps if I add in the actual build .a files

* Have the 'add' function call the right API

---------

Co-authored-by: John Viega <[email protected]>
Co-authored-by: John Viega <[email protected]>
Co-authored-by: John Viega <[email protected]>

bin/buildlibs.sh

@@ -264,18 +264,33 @@ EOL
 function ensure_hatrack {
     if ! copy_from_package libhatrack.a ; then
         get_src hatrack https://github.com/viega/hatrack
+        autoreconf -i
         aclocal
         autoheader
         autoconf
         automake
-        ./configure
+        ./configure --enable-quark=yes
         make libhatrack.a
+        mv libhatrack.a ${MY_LIBS}
         if [[ -f ${MY_LIBS}/libhatrack.a ]] ; then
             echo $(color GREEN Installed libhatrack to:) ${MY_LIBS}/libhatrack.a
         fi
     fi
 }
 
+function ensure_ffi {
+    if ! copy_from_package libffi.a ; then
+        get_src libffi https://github.com/libffi/libffi.git
+	sh ./autogen.sh
+        ./configure
+        make
+        mv */.libs/libffi.a ${MY_LIBS}
+        if [[ -f ${MY_LIBS}/libffi.a ]] ; then
+            echo $(color GREEN Installed libhffi to:) ${MY_LIBS}/libffi.a
+        fi
+    fi
+}
+
 function remove_src {
   # Don't nuke the src if CON4M_DEV is on.
   if [[ -d ${SRC_DIR} ]] ; then
@@ -293,6 +308,7 @@ ensure_openssl
 ensure_pcre
 ensure_gumbo
 ensure_hatrack
+ensure_ffi
 
 colorln GREEN All dependencies satisfied.
 remove_src

nimutils.nim

@@ -16,13 +16,13 @@ import nimutils/[box, random, unicodeid, pubsub, sinks, auth, misc, texttable],
        nimutils/[sha, aes, prp, hexdump, markdown, htmlparse, net, colortable],
        nimutils/[rope_base, rope_styles, rope_construct, rope_prerender],
        nimutils/[rope_ansirender, rope_htmlrender, rope_textrender],
-       nimutils/[switchboard, subproc, int128_t, dict, list]
+       nimutils/[switchboard, subproc, int128_t, dict, list, c4str]
 export box, random, unicodeid, pubsub, sinks, auth, misc, random, texttable,
        file, filetable, encodings, advisory_lock, progress, sha,
        aes, prp, hexdump, markdown, htmlparse, net, colortable, rope_base,
        rope_styles, rope_construct, rope_prerender, rope_ansirender,
        rope_htmlrender, rope_textrender, switchboard, subproc, int128_t,
-       dict, list
+       dict, list, c4str
 
 when defined(macosx):
   import nimutils/macproc

nimutils/c/strcontainer.c

@@ -0,0 +1,42 @@
+#include <stdint.h>
+#include <string.h>
+#include <stdlib.h>
+
+typedef struct {
+    int64_t  len;
+    char     data[];
+} real_str_t;
+
+typedef char str_t;
+
+str_t *
+c4string_new(int64_t *len)
+{
+    real_str_t *real_obj = calloc(len + sizeof(int64_t) + 1, 1);
+
+    real_obj->len = len;
+    return real_obj->data;
+}
+
+str_t *
+c4string_from_cstr(char *s) {
+    int64_t  l      = (int64_t)strlen(s);
+    str_t   *result = c4string_new(l);
+
+    memcpy(result, s, (size_t)(l + 1));
+
+    return result;
+}
+
+int64_t
+c4string_len(str_t *s)
+{
+    real_str_t *p = (real_str_t *)(s - sizeof(int64_t));
+    return p->len;
+}
+
+void
+c4string_free(str_t *s)
+{
+    free((real_str_t *)(s - sizeof(int64_t)));
+}

nimutils/c4str.nim

@@ -0,0 +1,49 @@
+import os
+
+static:
+  {.compile: joinPath(splitPath(currentSourcePath()).head, "c/strcontainer.c").}
+
+type C4Str* = pointer
+
+proc newC4Str*(l: int64): C4Str {.importc: "c4string_new", cdecl.}
+proc newC4Str*(s: cstring): C4Str {.importc: "c4string_from_cstr", cdecl.}
+proc c4str_len*(s: C4Str): int64 {.importc: "c4string_len", cdecl.}
+proc free*(s: C4Str) {.importc: "c4string_free", cdecl.}
+
+template len*(s: C4Str): int = int(s.c4str_len())
+
+proc newC4Str*(s: string): C4Str {.cdecl, exportc.} =
+  let l = s.len()
+  result = newC4Str(l)
+  if l != 0:
+    copyMem(cast[pointer](result), addr s[0], l)
+
+proc c4str_eq*(s1, s2: C4Str): bool {.cdecl, exportc.} =
+  if s1.len() != s2.len():
+    return false
+  return cast[cstring](s1) == cast[cstring](s2)
+
+proc c4str_lt*(s1, s2: C4Str): bool {.cdecl, exportc.} =
+  return cast[cstring](s1) < cast[cstring](s2)
+
+proc c4str_gt*(s1, s2: C4Str): bool {.cdecl, exportc.} =
+  return cast[cstring](s1) > cast[cstring](s2)
+
+proc c4str_add*(s1, s2: C4Str): C4Str {.cdecl, exportc.} =
+  let
+    l1   = s1.len()
+    l2   = s2.len()
+
+  result = newC4Str(l1 + l2)
+
+  # Where to start writing the 2nd string.
+  let p = cast[pointer](cast[uint](cast[pointer](result)) + uint(l1))
+
+  copyMem(cast[pointer](result), cast[pointer](s1), l1)
+  copyMem(p, cast[pointer](s2), l2)
+
+proc c4str_copy*(s1: C4Str): C4Str {.cdecl, exportc.} =
+  let l = s1.len()
+
+  result = newC4Str(l)
+  copyMem(cast[pointer](result), cast[pointer](s1), l)

nimutils/crownhash.nim

@@ -64,7 +64,7 @@ import sugar, os, macros, options
 type
   Dict*[T, V] {. importc: "hatrack_dict_t", header: "crownhash.h", nodecl.} = object
   DictRef*[T, V] = ref Dict[T, V]
-  DictKeyType = enum
+  DictKeyType* = enum
     KTInt, KTFloat, KtCStr, KtPtr, KtObjInt, KtObjReal, KtObjCstr,
     KtObjPtr, KtObjCustom, KtForce32Bits = 0x0fffffff
   StackBox[T] = ref object
@@ -112,34 +112,34 @@ proc ejectStackBox[T](s: StackBox[T]) =
   if s.ownedByNim:
     GC_unref(s)
 
-proc hatrack_dict_init(ctx: var Dict, key_type: DictKeyType) {.hatc.}
-proc hatrack_dict_cleanup(ctx: ptr Dict) {.hatc.}
-proc hatrack_dict_set_consistent_views(ctx: var Dict, yes: bool) {.hatc.}
-proc hatrack_dict_get_consistent_views(ctx: var Dict): bool {.hatc.}
-proc hatrack_dict_set_hash_offset(ctx: var Dict, offset: cint) {.hatc.}
-proc hatrack_dict_get(ctx: var Dict, key: pointer, found: var bool):
+proc hatrack_dict_new*(a: DictKeyType): pointer {.hatc.};
+proc hatrack_dict_init*(ctx: var Dict, key_type: DictKeyType) {.hatc.}
+proc hatrack_dict_cleanup*(ctx: ptr Dict) {.hatc.}
+proc hatrack_dict_set_consistent_views*(ctx: var Dict, yes: bool) {.hatc.}
+proc hatrack_dict_get_consistent_views*(ctx: var Dict): bool {.hatc.}
+proc hatrack_dict_set_hash_offset*(ctx: var Dict, offset: cint) {.hatc.}
+proc hatrack_dict_get*(ctx: var Dict, key: pointer, found: var bool):
                      pointer {.hatc.}
-proc hatrack_dict_put(ctx: var Dict, key: pointer,
-                            value: pointer) {.
-  hatc.}
-proc hatrack_dict_replace(ctx: var Dict, key: pointer, value: pointer):
+proc hatrack_dict_put*(ctx: var Dict, key: pointer,
+                       value: pointer) {.hatc.}
+proc hatrack_dict_replace*(ctx: var Dict, key: pointer, value: pointer):
                      bool {.hatc.}
 proc hatrack_dict_add*(ctx: var Dict, key: pointer, value: pointer):
                      bool {.hatc.}
-proc hatrack_dict_remove(ctx: var Dict, key: pointer): bool {.hatc.}
-proc hatrack_dict_keys_sort(ctx: var Dict, n: ptr uint64):
+proc hatrack_dict_remove*(ctx: var Dict, key: pointer): bool {.hatc.}
+proc hatrack_dict_keys_sort*(ctx: var Dict, n: ptr uint64):
                            pointer {.hatc.}
-proc hatrack_dict_values_sort(ctx: var Dict, n: ptr uint64):
+proc hatrack_dict_values_sort*(ctx: var Dict, n: ptr uint64):
                              pointer {.hatc.}
-proc hatrack_dict_items_sort(ctx: var Dict, n: ptr uint64):
+proc hatrack_dict_items_sort*(ctx: var Dict, n: ptr uint64):
                             pointer {.hatc.}
-proc hatrack_dict_keys_nosort(ctx: var Dict, n: ptr uint64):
+proc hatrack_dict_keys_nosort*(ctx: var Dict, n: ptr uint64):
                              pointer {.hatc.}
-proc hatrack_dict_values_nosort(ctx: var Dict, n: ptr uint64):
+proc hatrack_dict_values_nosort*(ctx: var Dict, n: ptr uint64):
                                pointer {.hatc.}
-proc hatrack_dict_items_nosort(ctx: var Dict, n: ptr uint64):
+proc hatrack_dict_items_nosort*(ctx: var Dict, n: ptr uint64):
                               pointer {.hatc.}
-proc hatrack_dict_set_free_handler[T, V](ctx: var Dict[T, V],
+proc hatrack_dict_set_free_handler*[T, V](ctx: var Dict[T, V],
                        cb: (var Dict[T, V], ptr RawItem) -> void) {.hatc.}
 proc register_thread() {.cdecl, importc: "mmm_register_thread" .}
 
@@ -257,18 +257,18 @@ proc add*[T, V](dict: var Dict[T, V], key: T, value: sink V): bool =
     p = cast[pointer](key)
 
   when V is SomeOrdinal:
-    return dict.hatrack_dict_replace(p, cast[pointer](int64(value)))
+    return dict.hatrack_dict_add(p, cast[pointer](int64(value)))
   elif V is SomeFloat:
-    return dict.hatrack_dict_replace(p, cast[pointer](float(value)))
+    return dict.hatrack_dict_add(p, cast[pointer](float(value)))
   elif V is SomeString:
-    return dict.hatrack_dict_replace(p, cast[pointer](value.toStrBox()))
+    return dict.hatrack_dict_add(p, cast[pointer](value.toStrBox()))
   elif V is ref:
     GC_ref(value)
-    return dict.hatrack_dict_replace(p, cast[pointer](value))
+    return dict.hatrack_dict_add(p, cast[pointer](value))
   elif V is pointer:
-    return dict.hatrack_dict_replace(p, cast[pointer](value))
+    return dict.hatrack_dict_add(p, cast[pointer](value))
   else:
-    return dict.hatrack_dict_replace(p, cast[pointer](value.toStackBox()))
+    return dict.hatrack_dict_add(p, cast[pointer](value.toStackBox()))
 
 proc add*[T, V](dict: DictRef[T, V], key: T, value: sink V): bool =
   return add(dict[], key, value)

nimutils/lfarray.nim

@@ -3,7 +3,7 @@ import options
 {.pragma: hatc, cdecl, importc.}
 
 type
-  FlexArrayObj = pointer
+  FlexArrayObj* = pointer
 
   FlexArray*[T] = ref object
     arr*:      ptr FlexArrayObj

nimutils/rope_ansirender.nim

@@ -113,7 +113,6 @@ proc preRenderBoxToAnsiString*(b: TextPlane, noColor = false): string =
           shouldTitle = true
       else:
         if ch == uint32('\e'):
-          continue
           raise newException(ValueError, "ANSI escape codes are not allowed " &
             "in text in this API")
         case styleInfo.casing