Add support for encoded refs in external terms

Also add partial support for external refs.
Also fix list representation of local refs as a ref with two words,
matching external term encoding.
Also fix order of pids to match OTP.

Signed-off-by: Paul Guyot <[email protected]>

src/libAtomVM/context.c

@@ -231,7 +231,7 @@ void context_process_flush_monitor_signal(Context *ctx, uint64_t ref_ticks, bool
         if (term_is_tuple(msg)
             && term_get_tuple_arity(msg) == 5
             && term_get_tuple_element(msg, 0) == DOWN_ATOM
-            && term_is_reference(term_get_tuple_element(msg, 1))
+            && term_is_local_reference(term_get_tuple_element(msg, 1))
             && term_to_ref_ticks(term_get_tuple_element(msg, 1)) == ref_ticks) {
             mailbox_remove_message(&ctx->mailbox, &ctx->heap);
             // If option info is combined with option flush, false is returned if a flush was needed, otherwise true.

src/libAtomVM/ets_hashtable.c

@@ -266,7 +266,7 @@ static uint32_t hash_external_pid(term t, int32_t h, GlobalContext *global)
     return h * LARGE_PRIME_PID;
 }
 
-static uint32_t hash_reference(term t, int32_t h, GlobalContext *global)
+static uint32_t hash_local_reference(term t, int32_t h, GlobalContext *global)
 {
     UNUSED(global);
     uint64_t n = term_to_ref_ticks(t);
@@ -277,6 +277,21 @@ static uint32_t hash_reference(term t, int32_t h, GlobalContext *global)
     return h * LARGE_PRIME_REF;
 }
 
+static uint32_t hash_external_reference(term t, int32_t h, GlobalContext *global)
+{
+    UNUSED(global);
+    uint32_t l = term_get_external_reference_len(t);
+    const uint32_t *words = term_get_external_reference_words(t);
+    for (uint32_t i = 0; i < l; i++) {
+        uint32_t n = words[i];
+        while (n) {
+            h = h * LARGE_PRIME_REF + (n & 0xFF);
+            n >>= 8;
+        }
+    }
+    return h * LARGE_PRIME_REF;
+}
+
 static uint32_t hash_binary(term t, int32_t h, GlobalContext *global)
 {
     UNUSED(global);
@@ -300,8 +315,10 @@ static uint32_t hash_term_incr(term t, int32_t h, GlobalContext *global)
         return hash_local_pid(t, h, global);
     } else if (term_is_external_pid(t)) {
         return hash_external_pid(t, h, global);
-    } else if (term_is_reference(t)) {
-        return hash_reference(t, h, global);
+    } else if (term_is_local_reference(t)) {
+        return hash_local_reference(t, h, global);
+    } else if (term_is_external_reference(t)) {
+        return hash_external_reference(t, h, global);
     } else if (term_is_binary(t)) {
         return hash_binary(t, h, global);
     } else if (term_is_tuple(t)) {

src/libAtomVM/externalterm.c

@@ -36,6 +36,7 @@
 
 #define NEW_FLOAT_EXT 70
 #define NEW_PID_EXT 88
+#define NEWER_REFERENCE_EXT 90
 #define SMALL_INTEGER_EXT 97
 #define INTEGER_EXT 98
 #define ATOM_EXT 100
@@ -421,6 +422,47 @@ static int serialize_term(uint8_t *buf, term t, GlobalContext *glb)
             WRITE_32_UNALIGNED(buf + k + 8, term_get_external_node_creation(t));
         }
         return k + 12;
+    } else if (term_is_local_reference(t)) {
+        if (!IS_NULL_PTR(buf)) {
+            buf[0] = NEWER_REFERENCE_EXT;
+        }
+        size_t k = 1;
+        if (!IS_NULL_PTR(buf)) {
+            WRITE_16_UNALIGNED(buf + k, 2); // len
+        }
+        k += 2;
+        term node_name = glb->node_name;
+        uint32_t creation = node_name == NONODE_AT_NOHOST_ATOM ? 0 : glb->creation;
+        k += serialize_term(IS_NULL_PTR(buf) ? NULL : buf + k, node_name, glb);
+        if (!IS_NULL_PTR(buf)) {
+            uint64_t ticks = term_to_ref_ticks(t);
+            WRITE_32_UNALIGNED(buf + k, creation);
+            WRITE_64_UNALIGNED(buf + k + 4, ticks);
+        }
+        return k + 12;
+    } else if (term_is_external_reference(t)) {
+        if (!IS_NULL_PTR(buf)) {
+            buf[0] = NEWER_REFERENCE_EXT;
+        }
+        size_t k = 1;
+        if (!IS_NULL_PTR(buf)) {
+            WRITE_16_UNALIGNED(buf + k, 2); // len
+        }
+        k += 2;
+        term node = term_get_external_node(t);
+        k += serialize_term(IS_NULL_PTR(buf) ? NULL : buf + k, node, glb);
+        uint32_t len = term_get_external_reference_len(t);
+        if (!IS_NULL_PTR(buf)) {
+            WRITE_32_UNALIGNED(buf + k, term_get_external_node_creation(t));
+        }
+        k += 4;
+        if (!IS_NULL_PTR(buf)) {
+            const uint32_t *data = term_get_external_reference_words(t);
+            for (uint32_t i = 0; i < len; i++) {
+                WRITE_32_UNALIGNED(buf + k + (i * 4), data[i]);
+            }
+        }
+        return k + (4 * len);
     } else {
         fprintf(stderr, "Unknown external term type: %" TERM_U_FMT "\n", t);
         AVM_ABORT();
@@ -716,6 +758,37 @@ static term parse_external_terms(const uint8_t *external_term_buf, size_t *eterm
             }
         }
 
+        case NEWER_REFERENCE_EXT: {
+            uint16_t len = READ_16_UNALIGNED(external_term_buf + 1);
+            if (UNLIKELY(len > 5)) {
+                return term_invalid_term();
+            }
+            size_t node_size;
+            term node = parse_external_terms(external_term_buf + 3, &node_size, copy, heap, glb);
+            if (UNLIKELY(!term_is_atom(node))) {
+                return term_invalid_term();
+            }
+            uint32_t creation = READ_32_UNALIGNED(external_term_buf + node_size + 3);
+            uint32_t data[5];
+            for (uint16_t i = 0; i < len; i++) {
+                data[i] = READ_32_UNALIGNED(external_term_buf + node_size + 7 + (i * 4));
+            }
+            *eterm_size = node_size + 7 + (len * 4);
+            if (node != NONODE_AT_NOHOST_ATOM || len != 2 || creation != 0) {
+                term this_node = glb->node_name;
+                uint32_t this_creation = this_node == NONODE_AT_NOHOST_ATOM ? 0 : glb->creation;
+                if (len == 2 && node == this_node && creation == this_creation) {
+                    uint64_t ticks = ((uint64_t) data[0]) << 32 | data[1];
+                    return term_from_ref_ticks(ticks, heap);
+                } else {
+                    return term_make_external_reference(node, len, data, creation, heap);
+                }
+            } else {
+                uint64_t ticks = ((uint64_t) data[0]) << 32 | data[1];
+                return term_from_ref_ticks(ticks, heap);
+            }
+        }
+
         default:
             return term_invalid_term();
     }
@@ -1028,10 +1101,45 @@ static int calculate_heap_usage(const uint8_t *external_term_buf, size_t remaini
                 return INVALID_TERM_SIZE;
             }
             buf_pos += node_size;
+            remaining -= node_size;
+            if (UNLIKELY(remaining < 3 * 4)) {
+                return INVALID_TERM_SIZE;
+            }
             *eterm_size = buf_pos + 12;
             return heap_size + u;
         }
 
+        case NEWER_REFERENCE_EXT: {
+            if (UNLIKELY(remaining < 3)) {
+                return INVALID_TERM_SIZE;
+            }
+            remaining -= 3;
+            int buf_pos = 3;
+            uint16_t len = READ_16_UNALIGNED(external_term_buf + 1);
+            size_t heap_size = EXTERNAL_REF_SIZE(len);
+            size_t node_size = 0;
+            int u = calculate_heap_usage(external_term_buf + buf_pos, remaining, &node_size, copy);
+            if (UNLIKELY(u == INVALID_TERM_SIZE)) {
+                return INVALID_TERM_SIZE;
+            }
+            if (external_term_buf[3] == SMALL_ATOM_UTF8_EXT) {
+                // Check if it's non-distributed node, in which case it's always a local ref
+                if (len == 2 && external_term_buf[4] == strlen("nonode@nohost") && memcmp(external_term_buf + 5, "nonode@nohost", strlen("nonode@nohost")) == 0) {
+                    heap_size = REF_SIZE;
+                }
+                // See above for pids
+            } else if (UNLIKELY(external_term_buf[3] != ATOM_EXT)) {
+                return INVALID_TERM_SIZE;
+            }
+            buf_pos += node_size;
+            remaining -= node_size;
+            if (UNLIKELY(remaining < (size_t) ((len + 1) * 4))) {
+                return INVALID_TERM_SIZE;
+            }
+            *eterm_size = buf_pos + 4 + (len * 4);
+            return heap_size + u;
+        }
+
         default:
             return INVALID_TERM_SIZE;
     }

src/libAtomVM/memory.c

@@ -624,6 +624,10 @@ static void memory_scan_and_copy(HeapFragment *old_fragment, term *mem_start, co
                     TRACE("- Found external pid.\n");
                     break;
 
+                case TERM_BOXED_EXTERNAL_REF:
+                    TRACE("- Found external ref.\n");
+                    break;
+
                 case TERM_BOXED_FUN: {
                     TRACE("- Found fun, size: %i.\n", (int) arity);
 
@@ -747,6 +751,10 @@ static void memory_scan_and_rewrite(size_t count, term *terms, const term *old_s
                     ptr += term_get_size_from_boxed_header(t);
                     break;
 
+                case TERM_BOXED_EXTERNAL_REF:
+                    ptr += term_get_size_from_boxed_header(t);
+                    break;
+
                 case TERM_BOXED_FUN:
                     // Skip header and module and process next terms
                     ptr++;

src/libAtomVM/nifs.c

@@ -3343,7 +3343,7 @@ static term nif_ets_new(Context *ctx, int argc, term argv[])
 
 static inline bool is_ets_table_id(term t)
 {
-    return term_is_reference(t) || term_is_atom(t);
+    return term_is_local_reference(t) || term_is_atom(t);
 }
 
 static term nif_ets_insert(Context *ctx, int argc, term argv[])
@@ -3477,10 +3477,9 @@ static term nif_erlang_pid_to_list(Context *ctx, int argc, term argv[])
 
     term t = argv[0];
     VALIDATE_VALUE(t, term_is_pid);
-    size_t max_len = term_is_external(t) ? EXTERNAL_PID_AS_CSTRING_LEN : LOCAL_PID_AS_CSTRING_LEN;
 
-    char buf[max_len];
-    int str_len = term_snprint(buf, max_len, t, ctx->global);
+    char buf[PID_AS_CSTRING_LEN];
+    int str_len = term_snprint(buf, PID_AS_CSTRING_LEN, t, ctx->global);
     if (UNLIKELY(str_len < 0)) {
         RAISE_ERROR(OUT_OF_MEMORY_ATOM);
     }
@@ -3498,7 +3497,6 @@ static term nif_erlang_ref_to_list(Context *ctx, int argc, term argv[])
     char buf[REF_AS_CSTRING_LEN];
     int str_len = term_snprint(buf, REF_AS_CSTRING_LEN, t, ctx->global);
     if (UNLIKELY(str_len < 0)) {
-        // TODO: change to internal error or something like that
         RAISE_ERROR(OUT_OF_MEMORY_ATOM);
     }
 
@@ -3798,7 +3796,7 @@ static term nif_erlang_demonitor(Context *ctx, int argc, term argv[])
         info = interop_proplist_get_value_default(options, INFO_ATOM, FALSE_ATOM) == TRUE_ATOM;
     }
 
-    VALIDATE_VALUE(ref, term_is_reference);
+    VALIDATE_VALUE(ref, term_is_local_reference);
     uint64_t ref_ticks = term_to_ref_ticks(ref);
 
     bool result = globalcontext_demonitor(ctx->global, ref_ticks);

src/libAtomVM/otp_socket.c

@@ -593,7 +593,7 @@ bool term_is_otp_socket(term socket_term)
     bool ret = term_is_tuple(socket_term)
         && term_get_tuple_arity(socket_term) == 2
         && term_is_binary(term_get_tuple_element(socket_term, 0))
-        && term_is_reference(term_get_tuple_element(socket_term, 1));
+        && term_is_local_reference(term_get_tuple_element(socket_term, 1));
 
     TRACE("term is a socket: %i\n", ret);
 
@@ -898,7 +898,7 @@ static term nif_socket_select_read(Context *ctx, int argc, term argv[])
 
     term select_ref_term = argv[1];
     if (select_ref_term != UNDEFINED_ATOM) {
-        VALIDATE_VALUE(select_ref_term, term_is_reference);
+        VALIDATE_VALUE(select_ref_term, term_is_local_reference);
     }
     struct SocketResource *rsrc_obj;
     if (UNLIKELY(!term_to_otp_socket(argv[0], &rsrc_obj, ctx))) {

src/libAtomVM/posix_nifs.c

@@ -431,7 +431,7 @@ static term nif_atomvm_posix_select(Context *ctx, term argv[], enum ErlNifSelect
     int32_t process_pid = term_to_local_process_id(process_pid_term);
     term select_ref_term = argv[2];
     if (select_ref_term != UNDEFINED_ATOM) {
-        VALIDATE_VALUE(select_ref_term, term_is_reference);
+        VALIDATE_VALUE(select_ref_term, term_is_local_reference);
     }
     void *fd_obj_ptr;
     if (UNLIKELY(!enif_get_resource(erl_nif_env_from_context(ctx), argv[0], ctx->global->posix_fd_resource_type, &fd_obj_ptr))) {

src/libAtomVM/resources.c

@@ -121,7 +121,7 @@ int enif_select(ErlNifEnv *env, ErlNifEvent event, enum ErlNifSelectFlags mode,
     if (!(mode & (ERL_NIF_SELECT_STOP | ERL_NIF_SELECT_READ | ERL_NIF_SELECT_WRITE))) {
         return ERL_NIF_SELECT_BADARG;
     }
-    if (UNLIKELY(mode & (ERL_NIF_SELECT_READ | ERL_NIF_SELECT_WRITE) && !term_is_reference(ref) && ref != UNDEFINED_ATOM)) {
+    if (UNLIKELY(mode & (ERL_NIF_SELECT_READ | ERL_NIF_SELECT_WRITE) && !term_is_local_reference(ref) && ref != UNDEFINED_ATOM)) {
         return ERL_NIF_SELECT_BADARG;
     }