Improve dlopen() on Apple Silicon

- Introduce MAP_JIT which is zero on other platforms
- Invent __jit_begin() and __jit_end() which wrap Apple's APIs
- Runtime dispatch to sys_icache_invalidate() in __clear_cache()

ape/ape-m1.c

@@ -36,7 +36,7 @@
 
 #define pagesz         16384
 #define SYSLIB_MAGIC   ('s' | 'l' << 8 | 'i' << 16 | 'b' << 24)
-#define SYSLIB_VERSION 7
+#define SYSLIB_VERSION 8
 
 struct Syslib {
   int magic;
@@ -891,6 +891,7 @@ int main(int argc, char **argv, char **envp) {
   M->lib.pthread_jit_write_protect_supported_np =
       pthread_jit_write_protect_supported_np;
   M->lib.pthread_jit_write_protect_np = pthread_jit_write_protect_np_workaround;
+  M->lib.sys_icache_invalidate = sys_icache_invalidate;
   M->lib.pthread_create = pthread_create;
   M->lib.pthread_exit = pthread_exit;
   M->lib.pthread_kill = pthread_kill;

libc/calls/munmap-sysv.c

@@ -21,6 +21,7 @@
 #include "libc/intrin/describeflags.internal.h"
 #include "libc/intrin/directmap.internal.h"
 #include "libc/intrin/strace.internal.h"
+#include "libc/runtime/syslib.internal.h"
 
 /**
  * Unmaps memory directly with system.
@@ -32,10 +33,12 @@
  */
 int sys_munmap(void *p, size_t n) {
   int rc;
-  if (!IsMetal()) {
-    rc = __sys_munmap(p, n);
-  } else {
+  if (IsXnuSilicon()) {
+    rc = _sysret(__syslib->__munmap(p, n));
+  } else if (IsMetal()) {
     rc = sys_munmap_metal(p, n);
+  } else {
+    rc = __sys_munmap(p, n);
   }
   KERNTRACE("sys_munmap(%p /* %s */, %'zu) → %d", p,
             DescribeFrame((intptr_t)p >> 16), n, rc);

libc/calls/syscall-sysv.internal.h

@@ -20,6 +20,7 @@ i32 __sys_fcntl_cp(i32, i32, ...);
 i32 __sys_fstat(i32, void *);
 i32 __sys_fstatat(i32, const char *, void *, i32);
 i32 __sys_gettid(i64 *);
+i32 __sys_mprotect(void *, u64, i32);
 i32 __sys_munmap(void *, u64);
 i32 __sys_openat(i32, const char *, i32, u32);
 i32 __sys_openat_nc(i32, const char *, i32, u32);

libc/dlopen/dlopen.c

@@ -424,12 +424,12 @@ static char *dlerror_set(const char *str) {
   return dlerror_buf;
 }
 
-static char *foreign_alloc_block(void) {
+static dontinline char *foreign_alloc_block(void) {
   char *p = 0;
   size_t sz = 65536;
   if (!IsWindows()) {
     p = __sys_mmap(0, sz, PROT_READ | PROT_WRITE | PROT_EXEC,
-                   MAP_PRIVATE | MAP_ANONYMOUS, -1, 0, 0);
+                   MAP_PRIVATE | MAP_ANONYMOUS | MAP_JIT, -1, 0, 0);
     if (p == MAP_FAILED) {
       p = 0;
     }
@@ -448,7 +448,7 @@ static char *foreign_alloc_block(void) {
   return p;
 }
 
-static void *foreign_alloc(size_t n) {
+static dontinline void *foreign_alloc(size_t n) {
   void *res;
   static char *block;
   static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
@@ -520,11 +520,14 @@ static void *foreign_thunk_sysv(void *func) {
   *p++ = 0xff;
   *p++ = 0xe2;
 #elif defined(__aarch64__)
-  if (!(p = code = foreign_alloc(36))) return 0;  // 16 + 16 + 4 = 36
-  p = movimm(p, 5, (uintptr_t)func);
-  p = movimm(p, 10, (uintptr_t)foreign_tramp);
-  *(uint32_t *)p = 0xd61f0140;  // br x10
-  __clear_cache(code, p + 4);
+  __jit_begin();
+  if ((p = code = foreign_alloc(36))) {
+    p = movimm(p, 5, (uintptr_t)func);
+    p = movimm(p, 10, (uintptr_t)foreign_tramp);
+    *(uint32_t *)p = 0xd61f0140;  // br x10
+    __clear_cache(code, p + 4);
+  }
+  __jit_end();
 #else
 #error "unsupported architecture"
 #endif

libc/dlopen/dlopen.mk

@@ -33,8 +33,7 @@ LIBC_DLOPEN_A_DIRECTDEPS =						\
 	LIBC_RUNTIME							\
 	LIBC_SYSV							\
 	LIBC_SYSV_CALLS							\
-	LIBC_STR							\
-	THIRD_PARTY_COMPILER_RT
+	LIBC_STR
 
 LIBC_DLOPEN_A_DEPS :=							\
 	$(call uniq,$(foreach x,$(LIBC_DLOPEN_A_DIRECTDEPS),$($(x))))

libc/intrin/directmap.c

@@ -17,12 +17,14 @@
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/calls/syscall-sysv.internal.h"
+#include "libc/dce.h"
 #include "libc/errno.h"
 #include "libc/intrin/describeflags.internal.h"
 #include "libc/intrin/directmap.internal.h"
 #include "libc/intrin/strace.internal.h"
 #include "libc/nt/runtime.h"
 #include "libc/runtime/memtrack.internal.h"
+#include "libc/runtime/syslib.internal.h"
 
 /**
  * Obtains memory mapping directly from system.
@@ -37,7 +39,11 @@
 struct DirectMap sys_mmap(void *addr, size_t size, int prot, int flags, int fd,
                           int64_t off) {
   struct DirectMap d;
-  if (!IsWindows() && !IsMetal()) {
+  if (IsXnuSilicon()) {
+    long p = _sysret(__syslib->__mmap(addr, size, prot, flags, fd, off));
+    d.maphandle = kNtInvalidHandleValue;
+    d.addr = (void *)p;
+  } else if (!IsWindows() && !IsMetal()) {
     d.addr = __sys_mmap(addr, size, prot, flags, fd, off, off);
     d.maphandle = kNtInvalidHandleValue;
   } else if (IsMetal()) {

libc/intrin/mprotect-sysv.c

@@ -0,0 +1,29 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8                                :vi│
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2023 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/calls/syscall-sysv.internal.h"
+#include "libc/dce.h"
+#include "libc/runtime/syslib.internal.h"
+
+int sys_mprotect(void *data, size_t size, int prot) {
+  if (IsXnuSilicon()) {
+    return _sysret(__syslib->__mprotect(data, size, prot));
+  } else {
+    return __sys_mprotect(data, size, prot);
+  }
+}

libc/runtime/jit.c

@@ -0,0 +1,37 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8                                :vi│
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2023 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/dce.h"
+#include "libc/runtime/runtime.h"
+#include "libc/runtime/syslib.internal.h"
+
+void __jit_begin(void) {
+  if (IsXnuSilicon()) {
+    if (__syslib->__pthread_jit_write_protect_supported_np()) {
+      __syslib->__pthread_jit_write_protect_np(false);
+    }
+  }
+}
+
+void __jit_end(void) {
+  if (IsXnuSilicon()) {
+    if (__syslib->__pthread_jit_write_protect_supported_np()) {
+      __syslib->__pthread_jit_write_protect_np(true);
+    }
+  }
+}

libc/runtime/runtime.h

@@ -115,6 +115,8 @@ bool32 _isheap(void *);
 /* code morphing */
 void __morph_begin(void);
 void __morph_end(void);
+void __jit_begin(void);
+void __jit_end(void);
 void __clear_cache(void *, void *);
 /* portability */
 int NtGetVersion(void) pureconst;

libc/runtime/runtime.mk

@@ -45,6 +45,7 @@ LIBC_RUNTIME_A_DIRECTDEPS =				\
 	LIBC_STR					\
 	LIBC_SYSV					\
 	LIBC_SYSV_CALLS					\
+	THIRD_PARTY_COMPILER_RT				\
 	THIRD_PARTY_NSYNC				\
 	THIRD_PARTY_PUFF				\
 	THIRD_PARTY_XED

libc/runtime/syslib.internal.h

@@ -12,7 +12,7 @@ COSMOPOLITAN_C_START_
  */
 
 #define SYSLIB_MAGIC   ('s' | 'l' << 8 | 'i' << 16 | 'b' << 24)
-#define SYSLIB_VERSION 6
+#define SYSLIB_VERSION 8
 
 typedef uint64_t dispatch_time_t;
 typedef uint64_t dispatch_semaphore_t;

libc/sysv/calls/__sys_mprotect.S

@@ -0,0 +1,2 @@
+#include "libc/sysv/macros.internal.h"
+.scall __sys_mprotect,0x04a04a04a204a00a,226,74,globl,hidden

libc/sysv/consts.sh

@@ -234,6 +234,7 @@ syscon	mmap	MAP_INHERIT				-1			-1			-1			-1			-1			-1			0x00000080		-1			# make
 syscon	mmap	MAP_HASSEMAPHORE			0			0			0x00000200		0x00000200		0x00000200		0			0x00000200		0			# does it matter on x86?
 syscon	mmap	MAP_NOSYNC				0			0			0			0			0x00000800		0			0			0			# flush to physical media only when necessary rather than gratuitously; be sure to use write() rather than ftruncate() with this!
 syscon	mmap	MAP_CONCEAL				0			0			0			0			0x00020000		0x00008000		0x00008000		0			# omit from core dumps; MAP_NOCORE on FreeBSD
+syscon	mmap	MAP_JIT					0			0			0			0x00000800		0			0			0			0			# omit from core dumps; MAP_NOCORE on FreeBSD
 syscon	compat	MAP_NOCORE				0			0			0			0			0x00020000		0x00008000		0x00008000		0			# use MAP_CONCEAL
 syscon	compat	MAP_ANON				0x00000020		0x00000020		0x00001000		0x00001000		0x00001000		0x00001000		0x00001000		0x00000020		# bsd consensus; faked nt
 syscon	compat	MAP_EXECUTABLE				0x00001000		0x00001000		0			0			0			0			0			0			# ignored

libc/sysv/consts/MAP_JIT.S

@@ -0,0 +1,2 @@
+#include "libc/sysv/consts/syscon.internal.h"
+.syscon mmap,MAP_JIT,0,0,0,0x00000800,0,0,0,0

libc/sysv/consts/RLIMIT_AS.S

@@ -1,2 +1,2 @@
 #include "libc/sysv/consts/syscon.internal.h"
-.syscon rlimit,RLIMIT_AS,9,9,5,5,10,2,10,0
+.syscon rlimit,RLIMIT_AS,9	,9,5,5,10,2,10,0

libc/sysv/consts/map.h

@@ -14,6 +14,7 @@ extern const int MAP_FIXED;
 extern const int MAP_FIXED_NOREPLACE;
 extern const int MAP_HASSEMAPHORE;
 extern const int MAP_INHERIT;
+extern const int MAP_JIT;
 extern const int MAP_LOCKED;
 extern const int MAP_NONBLOCK;
 extern const int MAP_NORESERVE;
@@ -43,5 +44,4 @@ COSMOPOLITAN_C_END_
 #define MAP_ANON   MAP_ANONYMOUS
 #define MAP_NOCORE MAP_CONCEAL
 
-
 #endif /* COSMOPOLITAN_LIBC_SYSV_CONSTS_MAP_H_ */

libc/sysv/syscalls.sh

@@ -45,7 +45,7 @@ scall	sys_ppoll		0xfff86da21ffff90f	0x849	globl hidden # consider INTON/INTOFF t
 scall	sys_lseek		0x0c70a61de20c7008	0x03e	globl hidden # netbsd:evilpad, OpenBSD 7.3+
 scall	__sys_mmap		0x0c50311dd20c5009	0x0de	globl hidden # netbsd:pad, OpenBSD 7.3+
 scall	sys_msync		0x915900841284181a	0x8e3	globl hidden
-scall	sys_mprotect		0x04a04a04a204a00a	0x0e2	globl hidden
+scall	__sys_mprotect		0x04a04a04a204a00a	0x0e2	globl hidden
 scall	__sys_munmap		0x049049049204900b	0x0d7	globl hidden
 scall	sys_sigaction		0x15402e1a0202e00d	0x086	globl hidden # rt_sigaction on Lunix; __sigaction_sigtramp() on NetBSD
 scall	__sys_sigprocmask	0x125030154214900e	0x087	globl hidden # a.k.a. rt_sigprocmask, openbsd:byvalue, a.k.a. pthread_sigmask