faster RTABX conversion

src/asammdf/blocks/cutils.c

@@ -1,5 +1,5 @@
 #define NPY_NO_DEPRECATED_API NPY_1_22_API_VERSION
-#define Py_LIMITED_API 0x030900f0
+//#define Py_LIMITED_API 0x030900f0
 
 #define PY_SSIZE_T_CLEAN 1
 #include <Python.h>
@@ -1500,6 +1500,39 @@ static PyObject *reverse_transposition(PyObject *self, PyObject *args)
     }
 }
 
+static PyObject *bytes_dtype_size(PyObject *self, PyObject *args)
+{
+    Py_ssize_t i = 0, j = 0;
+    Py_ssize_t count, size = 0, current_size=0;
+    PyObject *data, *values, **pointer;
+    char *read, *write_original, *write;
+    bool all_bytes = true;
+
+    if (!PyArg_ParseTuple(args, "O", &data))
+    {
+        return NULL;
+    }
+    else
+    {
+        count = (Py_ssize_t) *PyArray_SHAPE(data);
+        pointer = (PyObject **)PyArray_GETPTR1((PyArrayObject *)data, 0);
+        for (i=0; i<count; i++, pointer++) {
+            if (!PyBytes_Check(*pointer)) {
+                all_bytes = false;
+                size = -1;
+                break;
+            }
+
+            current_size = PyBytes_GET_SIZE(*pointer);
+
+            if (current_size > size) size = current_size;
+        }
+
+        return PyLong_FromSsize_t(size);
+    }
+}
+
+
 // Our Module's Function Definition struct
 // We require this `NULL` to signal the end of our method
 // definition
@@ -1514,6 +1547,7 @@ static PyMethodDef myMethods[] = {
     {"data_block_from_arrays", data_block_from_arrays, METH_VARARGS, "data_block_from_arrays"},
     {"get_idx_with_edges", get_idx_with_edges, METH_VARARGS, "get_idx_with_edges"},
     {"reverse_transposition", reverse_transposition, METH_VARARGS, "reverse_transposition"},
+    {"bytes_dtype_size", bytes_dtype_size, METH_VARARGS, "bytes_dtype_size"},
 
     {NULL, NULL, 0, NULL}};
 

src/asammdf/blocks/v4_blocks.py

@@ -42,6 +42,7 @@
 
 from .. import tool
 from . import v4_constants as v4c
+from .cutils import bytes_dtype_size
 from .utils import (
     block_fields,
     escape_xml_string,
@@ -3359,13 +3360,10 @@ def convert(self, values, as_object=False, as_bytes=False, ignore_value2text_con
                                 vals[idx[idx_]], ignore_value2text_conversions=ignore_value2text_conversions
                             )
 
-                all_bytes = True
-                for v in ret.tolist():
-                    if not isinstance(v, bytes):
-                        all_bytes = False
-                        break
-
-                if not all_bytes:
+                size = bytes_dtype_size(ret)
+                if size >= 0:
+                    ret = ret.astype(f"S{size}")
+                else:
                     try:
                         ret = ret.astype("f8")
                     except:
@@ -3374,9 +3372,6 @@ def convert(self, values, as_object=False, as_bytes=False, ignore_value2text_con
                         elif not as_object:
                             ret = np.array([np.nan if isinstance(v, bytes) else v for v in ret.tolist()])
 
-                else:
-                    ret = ret.astype(bytes)
-
                 ret = ret.reshape(shape)
                 values = np.rec.fromarrays(
                     [ret] + [values[name] for name in names[1:]],
@@ -3439,22 +3434,17 @@ def convert(self, values, as_object=False, as_bytes=False, ignore_value2text_con
                                     values[idx_], ignore_value2text_conversions=ignore_value2text_conversions
                                 )
 
-                all_bytes = True
-                for v in ret.tolist():
-                    if not isinstance(v, bytes):
-                        all_bytes = False
-                        break
-
-                if not all_bytes:
+                size = bytes_dtype_size(ret)
+                if size >= 0:
+                    ret = ret.astype(f"S{size}")
+                else:
                     try:
                         ret = ret.astype("f8")
                     except:
                         if as_bytes:
                             ret = ret.astype(bytes)
                         elif not as_object:
                             ret = np.array([np.nan if isinstance(v, bytes) else v for v in ret.tolist()])
-                else:
-                    ret = ret.astype(bytes)
 
                 values = ret
 
@@ -3690,22 +3680,17 @@ def convert(self, values, as_object=False, as_bytes=False, ignore_value2text_con
                         except:
                             raise
 
-            all_bytes = True
-            for v in ret.tolist():
-                if not isinstance(v, bytes):
-                    all_bytes = False
-                    break
-
-            if not all_bytes:
+            size = bytes_dtype_size(ret)
+            if size >= 0:
+                ret = ret.astype(f"S{size}")
+            else:
                 try:
                     ret = ret.astype("f8")
                 except:
                     if as_bytes:
                         ret = ret.astype(bytes)
                     elif not as_object:
                         ret = np.array([np.nan if isinstance(v, bytes) else v for v in ret.tolist()])
-            else:
-                ret = ret.astype(bytes)
 
             values = ret
 

src/asammdf/version.py

@@ -1,3 +1,3 @@
 """ asammdf version module """
 
-__version__ = "8.0.1"
+__version__ = "8.0.2.dev1"