diff --git a/libc/calls/getloadavg-nt.c b/libc/calls/getloadavg-nt.c index 08d733536c9..77e0a83ed4a 100644 --- a/libc/calls/getloadavg-nt.c +++ b/libc/calls/getloadavg-nt.c @@ -21,24 +21,23 @@ #include "libc/calls/syscall_support-nt.internal.h" #include "libc/dce.h" #include "libc/fmt/conv.h" +#include "libc/intrin/cxaatexit.h" #include "libc/macros.h" #include "libc/nt/accounting.h" #include "libc/runtime/runtime.h" -#include "libc/thread/thread.h" +#define CTOR __attribute__((__constructor__(99))) #define FT(x) (x.dwLowDateTime | (uint64_t)x.dwHighDateTime << 32) static int cpus; static double load; -static pthread_spinlock_t lock; static struct NtFileTime idle1, kern1, user1; textwindows int sys_getloadavg_nt(double *a, int n) { int i, rc; uint64_t elapsed, used; struct NtFileTime idle, kern, user; - BLOCK_SIGNALS; - pthread_spin_lock(&lock); + __cxa_lock(); if (GetSystemTimes(&idle, &kern, &user)) { elapsed = (FT(kern) - FT(kern1)) + (FT(user) - FT(user1)); if (elapsed) { @@ -54,12 +53,11 @@ textwindows int sys_getloadavg_nt(double *a, int n) { } else { rc = __winerr(); } - pthread_spin_unlock(&lock); - ALLOW_SIGNALS; + __cxa_unlock(); return rc; } -__attribute__((__constructor__(40))) static textstartup void ntinitload(void) { +CTOR static textstartup void sys_getloadavg_nt_init(void) { if (IsWindows()) { load = 1; cpus = __get_cpu_count() / 2; diff --git a/libc/calls/getprogramexecutablename.greg.c b/libc/calls/getprogramexecutablename.greg.c index 8e6e9e1c75f..8589bb099fa 100644 --- a/libc/calls/getprogramexecutablename.greg.c +++ b/libc/calls/getprogramexecutablename.greg.c @@ -96,9 +96,8 @@ static int OldApeLoader(char *s) { static int CopyWithCwd(const char *q, char *p, char *e) { char c; if (*q != '/') { - if (q[0] == '.' && q[1] == '/') { + if (q[0] == '.' && q[1] == '/') q += 2; - } int got = __getcwd(p, e - p - 1 /* '/' */); if (got != -1) { p += got - 1; @@ -118,9 +117,10 @@ static int CopyWithCwd(const char *q, char *p, char *e) { // if q exists then turn it into an absolute path. static int TryPath(const char *q) { - if (!CopyWithCwd(q, g_prog.u.buf, g_prog.u.buf + sizeof(g_prog.u.buf))) { + if (!q) + return 0; + if (!CopyWithCwd(q, g_prog.u.buf, g_prog.u.buf + sizeof(g_prog.u.buf))) return 0; - } return !sys_faccessat(AT_FDCWD, g_prog.u.buf, F_OK, 0); } @@ -129,9 +129,8 @@ static int TryPath(const char *q) { void __init_program_executable_name(void) { if (__program_executable_name && *__program_executable_name != '/' && CopyWithCwd(__program_executable_name, g_prog.u.buf, - g_prog.u.buf + sizeof(g_prog.u.buf))) { + g_prog.u.buf + sizeof(g_prog.u.buf))) __program_executable_name = g_prog.u.buf; - } } static inline void InitProgramExecutableNameImpl(void) { @@ -212,14 +211,12 @@ static inline void InitProgramExecutableNameImpl(void) { } // don't trust argv or envp if set-id. - if (issetugid()) { + if (issetugid()) goto UseEmpty; - } // try argv[0], then then $_. - if (TryPath(__argv[0]) || TryPath(__getenv(__envp, "_").s)) { + if (TryPath(__argv[0]) || TryPath(__getenv(__envp, "_").s)) goto UseBuf; - } // give up and just copy argv[0] into it if ((q = __argv[0])) { diff --git a/libc/calls/state.internal.h b/libc/calls/state.internal.h index 00326586742..3d4d2a2d923 100644 --- a/libc/calls/state.internal.h +++ b/libc/calls/state.internal.h @@ -13,7 +13,6 @@ extern unsigned __sighandflags[NSIG + 1]; extern uint64_t __sighandmask[NSIG + 1]; extern const struct NtSecurityAttributes kNtIsInheritable; -void __fds_wipe(void); void __fds_lock(void); void __fds_unlock(void); diff --git a/libc/calls/struct/sigset.internal.h b/libc/calls/struct/sigset.internal.h index b31093dbbc8..77af35704ae 100644 --- a/libc/calls/struct/sigset.internal.h +++ b/libc/calls/struct/sigset.internal.h @@ -5,27 +5,15 @@ #include "libc/sysv/consts/sig.h" COSMOPOLITAN_C_START_ -#ifndef MODE_DBG -/* block sigs because theoretical edge cases */ #define BLOCK_SIGNALS \ do { \ sigset_t _SigMask; \ _SigMask = __sig_block() + #define ALLOW_SIGNALS \ __sig_unblock(_SigMask); \ } \ while (0) -#else -/* doesn't block signals so we can get a crash - report, when a core runtime library crashes */ -#define BLOCK_SIGNALS \ - do { \ - sigset_t _SigMask; \ - sigprocmask(SIG_SETMASK, 0, &_SigMask) -#define ALLOW_SIGNALS \ - } \ - while (0) -#endif sigset_t __sig_block(void); void __sig_unblock(sigset_t); diff --git a/libc/cosmo.h b/libc/cosmo.h index 21b1de175b7..d027d6dfc7b 100644 --- a/libc/cosmo.h +++ b/libc/cosmo.h @@ -22,5 +22,11 @@ int cosmo_futex_wake(_COSMO_ATOMIC(int) *, int, char); int cosmo_futex_wait(_COSMO_ATOMIC(int) *, int, char, int, const struct timespec *); +int __deadlock_check(void *, int) libcesque; +int __deadlock_tracked(void *) libcesque; +void __deadlock_record(void *, int) libcesque; +void __deadlock_track(void *, int) libcesque; +void __deadlock_untrack(void *) libcesque; + COSMOPOLITAN_C_END_ #endif /* COSMOPOLITAN_LIBC_COSMO_H_ */ diff --git a/libc/errno.h b/libc/errno.h index 8a3a04f307f..f8963ed98e6 100644 --- a/libc/errno.h +++ b/libc/errno.h @@ -26,11 +26,11 @@ COSMOPOLITAN_C_START_ /* this header is included by 700+ files; therefore we */ /* hand-roll &__get_tls()->tib_errno to avoid #include */ /* cosmopolitan uses x28 as the tls register b/c apple */ -#define errno \ - (*__extension__({ \ - errno_t *__ep; \ - __asm__("sub\t%0,x28,#512-0x3c" : "=r"(__ep)); \ - __ep; \ +#define errno \ + (*__extension__({ \ + errno_t *__ep; \ + __asm__("sub\t%0,x28,#1024-0x3c" : "=r"(__ep)); \ + __ep; \ })) #else #define errno (*__errno_location()) diff --git a/libc/integral/c.inc b/libc/integral/c.inc index 04aeb22294d..7a00cd8da58 100644 --- a/libc/integral/c.inc +++ b/libc/integral/c.inc @@ -135,7 +135,7 @@ typedef struct { #define strftimeesque(n) __attribute__((__format__(__strftime__, n, 0))) #ifndef privileged -#define privileged _Section(".privileged") dontinline dontinstrument dontubsan +#define privileged _Section(".privileged") dontinstrument dontubsan #endif #ifndef wontreturn diff --git a/libc/intrin/__getenv.c b/libc/intrin/__getenv.c index b387b458d8b..6d40aa91d14 100644 --- a/libc/intrin/__getenv.c +++ b/libc/intrin/__getenv.c @@ -20,7 +20,7 @@ #include "libc/intrin/getenv.h" #include "libc/intrin/kprintf.h" -privileged struct Env __getenv(char **p, const char *k) { +privileged optimizesize struct Env __getenv(char **p, const char *k) { char *t; int i, j; for (i = 0; (t = p[i]); ++i) { diff --git a/libc/intrin/bzero.c b/libc/intrin/bzero.c index 8f5087109c8..2d51a93149e 100644 --- a/libc/intrin/bzero.c +++ b/libc/intrin/bzero.c @@ -16,155 +16,18 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/assert.h" -#include "libc/dce.h" -#include "libc/nexgen32e/nexgen32e.h" -#include "libc/nexgen32e/x86feature.h" #include "libc/str/str.h" -typedef char xmm_t __attribute__((__vector_size__(16), __aligned__(1))); -typedef long long xmm_a __attribute__((__vector_size__(16), __aligned__(16))); - -static void bzero128(char *p, size_t n) { - xmm_t v = {0}; - if (n <= 32) { - *(xmm_t *)(p + n - 16) = v; - *(xmm_t *)p = v; - } else { - do { - n -= 32; - *(xmm_t *)(p + n) = v; - *(xmm_t *)(p + n + 16) = v; - } while (n > 32); - *(xmm_t *)(p + 16) = v; - *(xmm_t *)p = v; - } -} - -#if defined(__x86_64__) && !defined(__chibicc__) -_Microarchitecture("avx") static void bzero_avx(char *p, size_t n) { - xmm_t v = {0}; - if (n <= 32) { - *(xmm_t *)(p + n - 16) = v; - *(xmm_t *)p = v; - } else if (n >= 1024 && X86_HAVE(ERMS)) { - asm("rep stosb" : "+D"(p), "+c"(n), "=m"(*(char(*)[n])p) : "a"(0)); - } else { - if (n < kHalfCache3 || !kHalfCache3) { - do { - n -= 32; - *(xmm_t *)(p + n) = v; - *(xmm_t *)(p + n + 16) = v; - } while (n > 32); - } else { - while ((uintptr_t)(p + n) & 15) { - p[--n] = 0; - } - do { - n -= 32; - __builtin_ia32_movntdq((xmm_a *)(p + n), (xmm_a)v); - __builtin_ia32_movntdq((xmm_a *)(p + n + 16), (xmm_a)v); - } while (n > 32); - asm("sfence"); - } - *(xmm_t *)(p + 16) = v; - *(xmm_t *)p = v; - } -} -#endif - /** * Sets memory to zero. * - * bzero n=0 661 picoseconds - * bzero n=1 661 ps/byte 1,476 mb/s - * bzero n=2 330 ps/byte 2,952 mb/s - * bzero n=3 220 ps/byte 4,428 mb/s - * bzero n=4 165 ps/byte 5,904 mb/s - * bzero n=7 94 ps/byte 10,333 mb/s - * bzero n=8 41 ps/byte 23,618 mb/s - * bzero n=15 44 ps/byte 22,142 mb/s - * bzero n=16 20 ps/byte 47,236 mb/s - * bzero n=31 21 ps/byte 45,760 mb/s - * bzero n=32 20 ps/byte 47,236 mb/s - * bzero n=63 10 ps/byte 92,997 mb/s - * bzero n=64 15 ps/byte 62,982 mb/s - * bzero n=127 15 ps/byte 62,490 mb/s - * bzero n=128 10 ps/byte 94,473 mb/s - * bzero n=255 14 ps/byte 68,439 mb/s - * bzero n=256 9 ps/byte 105 gb/s - * bzero n=511 15 ps/byte 62,859 mb/s - * bzero n=512 11 ps/byte 83,976 mb/s - * bzero n=1023 15 ps/byte 61,636 mb/s - * bzero n=1024 10 ps/byte 88,916 mb/s - * bzero n=2047 9 ps/byte 105 gb/s - * bzero n=2048 8 ps/byte 109 gb/s - * bzero n=4095 8 ps/byte 115 gb/s - * bzero n=4096 8 ps/byte 118 gb/s - * bzero n=8191 7 ps/byte 129 gb/s - * bzero n=8192 7 ps/byte 130 gb/s - * bzero n=16383 6 ps/byte 136 gb/s - * bzero n=16384 6 ps/byte 137 gb/s - * bzero n=32767 6 ps/byte 140 gb/s - * bzero n=32768 6 ps/byte 141 gb/s - * bzero n=65535 15 ps/byte 64,257 mb/s - * bzero n=65536 15 ps/byte 64,279 mb/s - * bzero n=131071 15 ps/byte 63,166 mb/s - * bzero n=131072 15 ps/byte 63,115 mb/s - * bzero n=262143 15 ps/byte 62,052 mb/s - * bzero n=262144 15 ps/byte 62,097 mb/s - * bzero n=524287 15 ps/byte 61,699 mb/s - * bzero n=524288 15 ps/byte 61,674 mb/s - * bzero n=1048575 16 ps/byte 60,179 mb/s - * bzero n=1048576 15 ps/byte 61,330 mb/s - * bzero n=2097151 15 ps/byte 61,071 mb/s - * bzero n=2097152 15 ps/byte 61,065 mb/s - * bzero n=4194303 16 ps/byte 60,942 mb/s - * bzero n=4194304 16 ps/byte 60,947 mb/s - * bzero n=8388607 16 ps/byte 60,872 mb/s - * bzero n=8388608 16 ps/byte 60,879 mb/s - * * @param p is memory address * @param n is byte length * @return p * @asyncsignalsafe */ void bzero(void *p, size_t n) { - char *b; - uint64_t x; - b = p; -#ifdef __x86_64__ - asm("xorl\t%k0,%k0" : "=r"(x)); -#else - if (1) { - memset(p, 0, n); - return; - } - x = 0; -#endif - if (n <= 16) { - if (n >= 8) { - __builtin_memcpy(b, &x, 8); - __builtin_memcpy(b + n - 8, &x, 8); - } else if (n >= 4) { - __builtin_memcpy(b, &x, 4); - __builtin_memcpy(b + n - 4, &x, 4); - } else if (n) { - do { - asm volatile("" ::: "memory"); - b[--n] = x; - } while (n); - } -#if defined(__x86_64__) && !defined(__chibicc__) - } else if (IsTiny()) { - asm("rep stosb" : "+D"(b), "+c"(n), "=m"(*(char(*)[n])b) : "a"(0)); - return; - } else if (X86_HAVE(AVX)) { - bzero_avx(b, n); -#endif - } else { - bzero128(b, n); - } + memset(p, 0, n); } __weak_reference(bzero, explicit_bzero); diff --git a/libc/intrin/cxalock.c b/libc/intrin/cxalock.c index e0d43f53408..f7211d7d317 100644 --- a/libc/intrin/cxalock.c +++ b/libc/intrin/cxalock.c @@ -19,11 +19,7 @@ #include "libc/intrin/cxaatexit.h" #include "libc/thread/thread.h" -static pthread_mutex_t __cxa_lock_obj; - -void __cxa_wipe(void) { - pthread_mutex_init(&__cxa_lock_obj, 0); -} +pthread_mutex_t __cxa_lock_obj = PTHREAD_MUTEX_INITIALIZER; void __cxa_lock(void) { pthread_mutex_lock(&__cxa_lock_obj); @@ -32,7 +28,3 @@ void __cxa_lock(void) { void __cxa_unlock(void) { pthread_mutex_unlock(&__cxa_lock_obj); } - -__attribute__((__constructor__(60))) static textstartup void __cxa_init() { - pthread_atfork(__cxa_lock, __cxa_unlock, __cxa_wipe); -} diff --git a/libc/intrin/deadlock.c b/libc/intrin/deadlock.c new file mode 100644 index 00000000000..57da577a48f --- /dev/null +++ b/libc/intrin/deadlock.c @@ -0,0 +1,277 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2024 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "ape/sections.internal.h" +#include "libc/assert.h" +#include "libc/atomic.h" +#include "libc/cosmo.h" +#include "libc/dce.h" +#include "libc/errno.h" +#include "libc/intrin/atomic.h" +#include "libc/intrin/kprintf.h" +#include "libc/intrin/maps.h" +#include "libc/macros.h" +#include "libc/str/str.h" +#include "libc/thread/lock.h" +#include "libc/thread/thread.h" +#include "libc/thread/tls.h" + +/** + * @fileoverview deadlock detector for statically allocated locks + * + * This module helps you spot multi-threading bugs in your program. + * High-level abstractions like mutexes are much easier to use than + * atomics, but they still carry their own non-obvious dangers. For + * example, nesting locks need to be nested in a consistent way and + * normal mutexes can't be required recursively. Normally this will + * cause your program to deadlock, i.e. hang indefinitely, but this + * module can detect such conditions and return errors instead, and + * better yet print helpful information when using `cosmocc -mdbg`. + */ + +#define ABI privileged optimizesize + +// building our visitor function using this optimizesize keyword shrinks +// the stack memory requirement from 7168 to 2048 bytes. totally amazing +// although please note this maximum isn't a hard limit. for normal mode +// builds your posix mandated mutex error checking will be less accurate +// but still helpful and reliable, although your cosmocc -mdbg will trap +// and report that you've run into the limit, so you can talk to justine +#define MAX_LOCKS 64 + +// cosmo's tib reserves space for 64 nested locks before things degrade. +// the cosmopolitan c runtime defines 16 locks, which are all registered +// with pthread_atfork(). it means you get to have 48 mutexes right now, +// and if you register all of them, then calling fork() will cause there +// to be 2080 edges in your lock graph. talk to justine if you need more +// because we're obviously going to need to find a way to make this grow +#define LOCK_EDGES_MAX 2080 + +// supported lock objects must define `void *_edges` +#define LOCK_EDGES_OFFSET 0 +static_assert(offsetof(struct MapLock, edges) == LOCK_EDGES_OFFSET); +static_assert(offsetof(pthread_mutex_t, _edges) == LOCK_EDGES_OFFSET); + +struct LockEdge { + struct LockEdge *next; + void *dest; +}; + +struct VisitedLock { + struct VisitedLock *next; + void *lock; +}; + +typedef _Atomic(struct LockEdge *) LockEdges; + +static struct DeadlockDetector { + atomic_size_t edges_allocated; + struct LockEdge edges_memory[LOCK_EDGES_MAX]; +} __deadlock; + +forceinline struct CosmoTib *__deadlock_tls(void) { + return __get_tls_privileged(); +} + +forceinline LockEdges *get_lock_edges(void *lock) { + return (LockEdges *)((char *)lock + LOCK_EDGES_OFFSET); +} + +forceinline struct LockEdge *load_lock_edges(LockEdges *edges) { + return atomic_load_explicit(edges, memory_order_relaxed); +} + +ABI static int is_static_memory(void *lock) { + return _etext <= (unsigned char *)lock && (unsigned char *)lock < _end; +} + +ABI static struct LockEdge *__deadlock_alloc(void) { + size_t edges_allocated = + atomic_load_explicit(&__deadlock.edges_allocated, memory_order_relaxed); + for (;;) { + if (edges_allocated == LOCK_EDGES_MAX) { + if (IsModeDbg()) { + kprintf("error: cosmo LOCK_EDGES_MAX needs to be increased\n"); + DebugBreak(); + } + return 0; + } + if (atomic_compare_exchange_weak_explicit( + &__deadlock.edges_allocated, &edges_allocated, edges_allocated + 1, + memory_order_relaxed, memory_order_relaxed)) + return &__deadlock.edges_memory[edges_allocated]; + } +} + +ABI static void __deadlock_add_edge(void *from, void *dest) { + LockEdges *edges = get_lock_edges(from); + for (struct LockEdge *e = load_lock_edges(edges); e; e = e->next) + if (e->dest == dest) + return; + struct LockEdge *edge; + if ((edge = __deadlock_alloc())) { + edge->next = load_lock_edges(edges); + edge->dest = dest; + // we tolerate duplicate elements in the interest of performance. + // once an element is inserted, it's never removed. that's why we + // don't need need to worry about the aba problem. the cas itself + // is very important since it ensures inserted edges aren't lost. + for (;;) + if (atomic_compare_exchange_weak_explicit(edges, &edge->next, edge, + memory_order_relaxed, + memory_order_relaxed)) + break; + } +} + +ABI static bool __deadlock_visit(void *lock, struct VisitedLock *visited, + int notrap, int depth) { + if (++depth == MAX_LOCKS) { + if (IsModeDbg()) { + kprintf("error: too much recursion in deadlock detector\n"); + DebugBreak(); + } + return false; + } + for (struct VisitedLock *v = visited; v; v = v->next) { + if (v->lock == lock) { + if (IsModeDbg() && !notrap) { + // lock hierarchy violated! + // + // when you lock mutexes in a nested way, your locks must be + // nested in the same order globally. otherwise deadlocks might + // occur. for example, if you say in your first thread + // + // pthread_mutex_lock(&x); + // pthread_mutex_lock(&y); + // pthread_mutex_unlock(&y); + // pthread_mutex_unlock(&x); + // + // then in your second thread you say + // + // pthread_mutex_lock(&y); + // pthread_mutex_lock(&x); + // pthread_mutex_unlock(&x); + // pthread_mutex_unlock(&y); + // + // then a deadlock might happen, because {x→y, y→x} is cyclic! + // they don't happen often, but this is the kind of thing that + // matters if you want to build carrier grade production stuff + kprintf("error: cycle detected in directed graph of nested locks\n"); + for (struct VisitedLock *v = visited; v; v = v->next) + kprintf("\t- %t\n", v->lock); // strongly connected component + DebugBreak(); + } + return true; + } + } + LockEdges *edges = get_lock_edges(lock); + struct VisitedLock visit = {visited, lock}; + for (struct LockEdge *e = load_lock_edges(edges); e; e = e->next) + if (__deadlock_visit(e->dest, &visit, notrap, depth)) + return true; + return false; +} + +/** + * Returns true if lock is already locked by calling thread. + * + * This function may return false negatives if we run out of TLS memory. + * That suboptimal condition will be reported in debug mode. + * + * @return 1 if lock is certainly owned by calling thread, 0 if lock is + * certainly not owned by calling thread, and -1 if we're uncertain + */ +ABI int __deadlock_tracked(void *lock) { + int full = 1; + int owned = 0; + struct CosmoTib *tib = __deadlock_tls(); + for (int i = 0; i < ARRAYLEN(tib->tib_locks); ++i) { + full &= tib->tib_locks[i] != NULL; + owned |= tib->tib_locks[i] == lock; + } + if (full) + return -1; + if (!owned && !is_static_memory(lock)) + return -1; + return owned; +} + +/** + * Records that lock is held by thread. + * @param notrap can prevent error printing and debug breaking + * @asyncsignalsafe + */ +ABI void __deadlock_track(void *lock, int notrap) { + if (!notrap && !is_static_memory(lock)) + return; + struct CosmoTib *tib = __deadlock_tls(); + for (int i = 0; i < ARRAYLEN(tib->tib_locks); ++i) { + if (!tib->tib_locks[i]) { + tib->tib_locks[i] = lock; + return; + } + } + if (IsModeDbg()) { + kprintf("error: cosmo tls max lock depth needs to be increased!\n"); + DebugBreak(); + } +} + +/** + * Records relationship for all held locks to `lock`. + * @param notrap can prevent error printing and debug breaking + * @asyncsignalsafe + */ +ABI void __deadlock_record(void *lock, int notrap) { + if (!notrap && !is_static_memory(lock)) + return; + struct CosmoTib *tib = __deadlock_tls(); + for (int i = 0; i < ARRAYLEN(tib->tib_locks); ++i) + if (tib->tib_locks[i] && tib->tib_locks[i] != lock) + __deadlock_add_edge(tib->tib_locks[i], lock); +} + +/** + * Returns EDEADLK if locking `lock` could cause a deadlock. + * @param notrap can prevent error printing and debug breaking + * @asyncsignalsafe + */ +ABI int __deadlock_check(void *lock, int notrap) { + struct CosmoTib *tib = __deadlock_tls(); + for (int i = 0; i < ARRAYLEN(tib->tib_locks); ++i) { + if (tib->tib_locks[i] == lock) + return 0; + if (tib->tib_locks[i]) { + struct VisitedLock visit = {0, tib->tib_locks[i]}; + if (__deadlock_visit(lock, &visit, notrap, 0)) + return EDEADLK; + } + } + return 0; +} + +/** + * Records that lock isn't held by thread. + * @asyncsignalsafe + */ +ABI void __deadlock_untrack(void *lock) { + struct CosmoTib *tib = __deadlock_tls(); + for (int i = 0; i < ARRAYLEN(tib->tib_locks); ++i) + tib->tib_locks[i] = tib->tib_locks[i] != lock ? tib->tib_locks[i] : 0; +} diff --git a/libc/intrin/demangle.c b/libc/intrin/demangle.c index 85c1d418a78..c44803f1220 100644 --- a/libc/intrin/demangle.c +++ b/libc/intrin/demangle.c @@ -91,6 +91,8 @@ Copyright (c) 2024 Justine Tunney "); * */ +#define ABI privileged optimizesize + #define DEMANGLE_NO_FLOATING_POINT #define ASSERT(x) (void)0 @@ -222,16 +224,18 @@ static int demangle_read_sname(struct demangle_data *); static int demangle_read_subst(struct demangle_data *); static int demangle_read_type(struct demangle_data *, struct type_delimit *); -static privileged size_t +ABI static size_t demangle_strlen(const char *s) { size_t n = 0; - while (*s++) + while (*s++) { + asm volatile("" ::: "memory"); ++n; + } return n; } -static privileged char * +ABI static char * demangle_stpcpy(char *d, const char *s) { size_t i = 0; @@ -242,7 +246,7 @@ demangle_stpcpy(char *d, const char *s) } } -static privileged void * +ABI static void * demangle_mempcpy(void *a, const void *b, size_t n) { char *d = a; @@ -252,14 +256,14 @@ demangle_mempcpy(void *a, const void *b, size_t n) return d; } -static privileged void * +ABI static void * demangle_memcpy(void *a, const void *b, size_t n) { demangle_mempcpy(a, b, n); return a; } -static privileged int +ABI static int demangle_strncmp(const char *a, const char *b, size_t n) { size_t i = 0; @@ -270,7 +274,7 @@ demangle_strncmp(const char *a, const char *b, size_t n) return (a[i] & 0xff) - (b[i] & 0xff); } -static privileged int +ABI static int demangle_memcmp(const void *a, const void *b, size_t n) { int c; @@ -285,7 +289,7 @@ demangle_memcmp(const void *a, const void *b, size_t n) return 0; } -static privileged void +ABI static void demangle_strlcpy(char *dst, const char *src, size_t dsize) { size_t remain; @@ -297,7 +301,7 @@ demangle_strlcpy(char *dst, const char *src, size_t dsize) *dst = 0; } -static privileged long +ABI static long demangle_strtol(const char *s, int base) { static const uint8_t demangle_base36[80] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, @@ -314,7 +318,7 @@ demangle_strtol(const char *s, int base) return x; } -static privileged char * +ABI static char * demangle_strstr(const char *haystack, const char *needle) { size_t i; @@ -335,7 +339,7 @@ demangle_strstr(const char *haystack, const char *needle) return 0; } -static privileged char * +ABI static char * demangle_utoa(char *p, unsigned long long x) { char t; @@ -356,7 +360,7 @@ demangle_utoa(char *p, unsigned long long x) return p + i; } -static privileged char * +ABI static char * demangle_itoa(char *p, long long x) { if (x < 0) @@ -364,7 +368,7 @@ demangle_itoa(char *p, long long x) return demangle_utoa(p, x); } -static privileged void +ABI static void demangle_free(struct demangle_data *h, void *ptr) { index_t base; @@ -381,7 +385,7 @@ demangle_free(struct demangle_data *h, void *ptr) } } -static privileged returnspointerwithnoaliases returnsnonnull void * +ABI static returnspointerwithnoaliases returnsnonnull void * demangle_malloc(struct demangle_data *h, long a, long n) { long rem; @@ -438,7 +442,7 @@ demangle_malloc(struct demangle_data *h, long a, long n) } } -static privileged returnspointerwithnoaliases char * +ABI static returnspointerwithnoaliases char * demangle_strdup(struct demangle_data *h, const char *s) { char *d = 0; @@ -450,7 +454,7 @@ demangle_strdup(struct demangle_data *h, const char *s) return d; } -static privileged void +ABI static void demangle_vector_str_dest(struct demangle_data *h, struct vector_str *v) { int i; @@ -459,7 +463,7 @@ demangle_vector_str_dest(struct demangle_data *h, struct vector_str *v) demangle_free(h, v->container); } -static privileged void +ABI static void demangle_vector_type_qualifier_dest(struct demangle_data *d, struct vector_type_qualifier *v) { @@ -467,7 +471,7 @@ demangle_vector_type_qualifier_dest(struct demangle_data *d, demangle_vector_str_dest(d, &v->ext_name); } -static privileged void +ABI static void demangle_stack_str_init(struct stack_str *ss) { ss->str = ss->buf; @@ -476,7 +480,7 @@ demangle_stack_str_init(struct stack_str *ss) ss->cap = sizeof(ss->buf); } -static privileged void +ABI static void demangle_stack_str_append(struct demangle_data *h, struct stack_str *ss, const char *str, size_t len) { @@ -499,7 +503,7 @@ demangle_stack_str_append(struct demangle_data *h, struct stack_str *ss, #define demangle_stack_str_append_str(h, ss, s) \ demangle_stack_str_append(h, ss, s, demangle_strlen(s)) -static privileged size_t +ABI static size_t demangle_get_strlen_sum(struct demangle_data *h, const struct vector_str *v) { size_t i, len = 0; @@ -509,7 +513,7 @@ demangle_get_strlen_sum(struct demangle_data *h, const struct vector_str *v) return len; } -static privileged int +ABI static int demangle_demangle_strncmp(const char *a, const char *b, size_t n) { size_t i = 0; @@ -527,7 +531,7 @@ demangle_demangle_strncmp(const char *a, const char *b, size_t n) * @param l Length of the string. * @return -1 at failed, 0 at not found, 1 at found. */ -static privileged int +ABI static int demangle_vector_str_find(struct demangle_data *h, const struct vector_str *v, const char *o, size_t l) { @@ -551,7 +555,7 @@ demangle_vector_str_find(struct demangle_data *h, const struct vector_str *v, * @param l Length of the string. * @return NULL at failed or NUL terminated new allocated string. */ -static privileged char * +ABI static char * demangle_vector_str_get_flat(struct demangle_data *ddata, const struct vector_str *v, size_t *l) { @@ -577,7 +581,7 @@ demangle_vector_str_get_flat(struct demangle_data *ddata, return rtn; } -static privileged void +ABI static void demangle_vector_str_grow(struct demangle_data *ddata, struct vector_str *v) { size_t i, tmp_cap; @@ -605,7 +609,7 @@ demangle_vector_str_grow(struct demangle_data *ddata, struct vector_str *v) * @brief Initialize vector_str. * @return false at failed, true at success. */ -static privileged void +ABI static void demangle_vector_str_init(struct demangle_data *ddata, struct vector_str *v) { v->size = 0; @@ -621,7 +625,7 @@ demangle_vector_str_init(struct demangle_data *ddata, struct vector_str *v) * @brief Remove last element in vector_str. * @return false at failed, true at success. */ -static privileged bool +ABI static bool demangle_vector_str_pop(struct vector_str *v) { if (!v) @@ -641,7 +645,7 @@ demangle_vector_str_pop(struct vector_str *v) * @brief Push back string to vector. * @return false at failed, true at success. */ -static privileged bool +ABI static bool demangle_vector_str_push(struct demangle_data *ddata, struct vector_str *v, const char *str, size_t len) { @@ -665,7 +669,7 @@ demangle_vector_str_push(struct demangle_data *ddata, struct vector_str *v, * @brief Push front org vector to det vector. * @return false at failed, true at success. */ -static privileged bool +ABI static bool demangle_vector_str_push_vector_head(struct demangle_data *ddata, struct vector_str *dst, struct vector_str *org) { @@ -698,7 +702,7 @@ demangle_vector_str_push_vector_head(struct demangle_data *ddata, * @brief Push org vector to the tail of det vector. * @return false at failed, true at success. */ -static privileged bool +ABI static bool demangle_vector_str_push_vector(struct demangle_data *ddata, struct vector_str *dst, struct vector_str *org) { @@ -736,7 +740,7 @@ demangle_vector_str_push_vector(struct demangle_data *ddata, * If r_len is not NULL, string length will be returned. * @return NULL at failed or NUL terminated new allocated string. */ -static privileged returnspointerwithnoaliases char * +ABI static returnspointerwithnoaliases char * demangle_vector_str_substr(struct demangle_data *ddata, const struct vector_str *v, size_t begin, size_t end, size_t *r_len) { @@ -762,7 +766,7 @@ demangle_vector_str_substr(struct demangle_data *ddata, return rtn; } -static privileged int +ABI static int demangle_vector_read_cmd_pop(struct vector_read_cmd *v) { if (!v->size) @@ -775,7 +779,7 @@ demangle_vector_read_cmd_pop(struct vector_read_cmd *v) return 1; } -static privileged void +ABI static void demangle_vector_read_cmd_init(struct demangle_data *ddata, struct vector_read_cmd *v) { @@ -786,7 +790,7 @@ demangle_vector_read_cmd_init(struct demangle_data *ddata, alignof(*v->r_container), sizeof(*v->r_container) * v->capacity); } -static privileged void +ABI static void demangle_data_init(struct demangle_data *d, const char *cur) { demangle_vector_str_init(d, &d->output); @@ -816,7 +820,7 @@ demangle_data_init(struct demangle_data *d, const char *cur) d->last_sname = NULL; } -static privileged int +ABI static int demangle_push_str(struct demangle_data *ddata, const char *str, size_t len) { if (!str || !len) @@ -833,7 +837,7 @@ demangle_push_str(struct demangle_data *ddata, const char *str, size_t len) } #ifndef DEMANGLE_NO_FLOATING_POINT -static privileged int +ABI static int demangle_push_fp(struct demangle_data *ddata, char *decoder(struct demangle_data *, const char *, size_t)) { @@ -862,13 +866,13 @@ demangle_push_fp(struct demangle_data *ddata, } #endif // DEMANGLE_NO_FLOATING_POINT -static privileged int +ABI static int demangle_pop_str(struct demangle_data *ddata) { return demangle_vector_str_pop(ddata->cur_output); } -static privileged int +ABI static int demangle_push_subst(struct demangle_data *ddata, const char *str, size_t len) { if (!str || !len) @@ -880,7 +884,7 @@ demangle_push_subst(struct demangle_data *ddata, const char *str, size_t len) return 1; } -static privileged int +ABI static int demangle_push_subst_v(struct demangle_data *ddata, struct vector_str *v) { int rtn; @@ -900,7 +904,7 @@ demangle_push_subst_v(struct demangle_data *ddata, struct vector_str *v) return rtn; } -static privileged int +ABI static int demangle_push_type_qualifier(struct demangle_data *ddata, struct vector_type_qualifier *v, const char *type_str) { @@ -1133,7 +1137,7 @@ demangle_push_type_qualifier(struct demangle_data *ddata, return 1; } -static privileged int +ABI static int demangle_get_subst(struct demangle_data *ddata, size_t idx) { size_t len; @@ -1151,7 +1155,7 @@ demangle_get_subst(struct demangle_data *ddata, size_t idx) return 1; } -static privileged int +ABI static int demangle_get_tmpl_param(struct demangle_data *ddata, size_t idx) { size_t len; @@ -1168,7 +1172,7 @@ demangle_get_tmpl_param(struct demangle_data *ddata, size_t idx) return 1; } -static privileged int +ABI static int demangle_read_array(struct demangle_data *ddata) { size_t i, num_len, exp_len, p_idx, idx; @@ -1240,7 +1244,7 @@ demangle_read_array(struct demangle_data *ddata) #ifndef DEMANGLE_NO_FLOATING_POINT /* Simple hex to integer function used by decode_to_* function. */ -static privileged int +ABI static int hex_to_dec(char c) { switch (c) { @@ -1288,7 +1292,7 @@ hex_to_dec(char c) * Todo * Replace these functions to macro. */ -static privileged returnspointerwithnoaliases char * +ABI static returnspointerwithnoaliases char * decode_fp_to_double(struct demangle_data *ddata, const char *p, size_t len) { double f; @@ -1332,7 +1336,7 @@ decode_fp_to_double(struct demangle_data *ddata, const char *p, size_t len) return rtn; } -static privileged returnspointerwithnoaliases char * +ABI static returnspointerwithnoaliases char * decode_fp_to_float(struct demangle_data *ddata, const char *p, size_t len) { size_t i, rtn_len, limit; @@ -1374,7 +1378,7 @@ decode_fp_to_float(struct demangle_data *ddata, const char *p, size_t len) return rtn; } -static privileged returnspointerwithnoaliases char * +ABI static returnspointerwithnoaliases char * decode_fp_to_long_double(struct demangle_data *ddata, const char *p, size_t len) { long double f; @@ -1418,7 +1422,7 @@ decode_fp_to_long_double(struct demangle_data *ddata, const char *p, size_t len) return rtn; } -static privileged returnspointerwithnoaliases char * +ABI static returnspointerwithnoaliases char * decode_fp_to_float128(struct demangle_data *ddata, const char *p, size_t len) { long double f; @@ -1475,7 +1479,7 @@ decode_fp_to_float128(struct demangle_data *ddata, const char *p, size_t len) } } -static privileged returnspointerwithnoaliases char * +ABI static returnspointerwithnoaliases char * decode_fp_to_float80(struct demangle_data *ddata, const char *p, size_t len) { long double f; @@ -1538,7 +1542,7 @@ decode_fp_to_float80(struct demangle_data *ddata, const char *p, size_t len) #endif // DEMANGLE_NO_FLOATING_POINT -static privileged int +ABI static int demangle_read_expr_primary(struct demangle_data *ddata) { const char *num; @@ -1630,7 +1634,7 @@ demangle_read_expr_primary(struct demangle_data *ddata) * http://gcc.gnu.org/bugzilla/show_bug.cgi?id=31775 * http://gcc.gnu.org/viewcvs?view=rev&revision=124467 */ -static privileged int +ABI static int demangle_local_source_name(struct demangle_data *ddata) { /* L */ @@ -1656,7 +1660,7 @@ demangle_local_source_name(struct demangle_data *ddata) * read unqualified-name, unqualified name are operator-name, ctor-dtor-name, * source-name */ -static privileged int +ABI static int demangle_read_uqname(struct demangle_data *ddata) { size_t len; @@ -2085,7 +2089,7 @@ demangle_read_uqname(struct demangle_data *ddata) * Read template parameter that forms in 'T[number]_'. * This function much like to read_subst but only for types. */ -static privileged int +ABI static int demangle_read_tmpl_param(struct demangle_data *ddata) { long nth; @@ -2116,7 +2120,7 @@ demangle_read_tmpl_param(struct demangle_data *ddata) return 0; } -static privileged int +ABI static int demangle_vector_read_cmd_push(struct demangle_data *ddata, struct vector_read_cmd *v, enum read_cmd cmd, void *data) { @@ -2145,7 +2149,7 @@ demangle_vector_read_cmd_push(struct demangle_data *ddata, return 1; } -static privileged int +ABI static int demangle_read_tmpl_arg(struct demangle_data *ddata) { if (*ddata->cur == '\0') @@ -2164,7 +2168,7 @@ demangle_read_tmpl_arg(struct demangle_data *ddata) return demangle_read_type(ddata, NULL); } -static privileged int +ABI static int demangle_read_tmpl_args(struct demangle_data *ddata) { struct vector_str *v; @@ -2217,7 +2221,7 @@ demangle_read_tmpl_args(struct demangle_data *ddata) return demangle_vector_read_cmd_pop(&ddata->cmd); } -static privileged int +ABI static int demangle_read_expression_trinary(struct demangle_data *ddata, const char *name1, size_t len1, const char *name2, size_t len2) { @@ -2236,7 +2240,7 @@ demangle_read_expression_trinary(struct demangle_data *ddata, const char *name1, return demangle_read_expression(ddata); } -static privileged int +ABI static int demangle_read_expression_unary(struct demangle_data *ddata, const char *name, size_t len) { @@ -2248,7 +2252,7 @@ demangle_read_expression_unary(struct demangle_data *ddata, const char *name, return demangle_push_str(ddata, name, len); } -static privileged int +ABI static int demangle_read_expression_binary(struct demangle_data *ddata, const char *name, size_t len) { @@ -2262,7 +2266,7 @@ demangle_read_expression_binary(struct demangle_data *ddata, const char *name, return demangle_read_expression(ddata); } -static privileged int +ABI static int demangle_read_expression_impl(struct demangle_data *ddata) { if (*ddata->cur == '\0') @@ -2544,7 +2548,7 @@ demangle_read_expression_impl(struct demangle_data *ddata) return 0; } -static privileged int +ABI static int demangle_read_expression(struct demangle_data *ddata) { if (ddata->depth == MAX_DEPTH) @@ -2555,7 +2559,7 @@ demangle_read_expression(struct demangle_data *ddata) return res; } -static privileged int +ABI static int demangle_read_expression_flat(struct demangle_data *ddata, char **str) { struct vector_str *output; @@ -2584,7 +2588,7 @@ demangle_read_expression_flat(struct demangle_data *ddata, char **str) } /* size, capacity, ext_name */ -static privileged void +ABI static void demangle_vector_type_qualifier_init(struct demangle_data *ddata, struct vector_type_qualifier *v) { @@ -2600,7 +2604,7 @@ demangle_vector_type_qualifier_init(struct demangle_data *ddata, demangle_vector_str_init(ddata, &v->ext_name); } -static privileged struct read_cmd_item * +ABI static struct read_cmd_item * demangle_vector_read_cmd_find(struct vector_read_cmd *v, enum read_cmd dst) { int i; @@ -2615,7 +2619,7 @@ demangle_vector_read_cmd_find(struct vector_read_cmd *v, enum read_cmd dst) return 0; } -static privileged int +ABI static int demangle_read_function(struct demangle_data *ddata, int *ext_c, struct vector_type_qualifier *v) { @@ -2751,7 +2755,7 @@ demangle_read_function(struct demangle_data *ddata, int *ext_c, return 1; } -static privileged int +ABI static int demangle_read_offset_number(struct demangle_data *ddata) { bool negative; @@ -2787,7 +2791,7 @@ demangle_read_offset_number(struct demangle_data *ddata) return 1; } -static privileged int +ABI static int demangle_read_nv_offset(struct demangle_data *ddata) { if (!DEM_PUSH_STR(ddata, "offset : ")) @@ -2796,7 +2800,7 @@ demangle_read_nv_offset(struct demangle_data *ddata) return demangle_read_offset_number(ddata); } -static privileged int +ABI static int demangle_read_v_offset(struct demangle_data *ddata) { if (!DEM_PUSH_STR(ddata, "offset : ")) @@ -2812,7 +2816,7 @@ demangle_read_v_offset(struct demangle_data *ddata) } /* read offset, offset are nv-offset, v-offset */ -static privileged int +ABI static int demangle_read_offset(struct demangle_data *ddata) { if (*ddata->cur == 'h') { @@ -2826,7 +2830,7 @@ demangle_read_offset(struct demangle_data *ddata) return 0; } -static privileged int +ABI static int demangle_read_type_flat(struct demangle_data *ddata, char **str) { struct vector_str *output; @@ -2858,7 +2862,7 @@ demangle_read_type_flat(struct demangle_data *ddata, char **str) * read number * number ::= [n] */ -static privileged int +ABI static int demangle_read_number(struct demangle_data *ddata, long *rtn) { long len, negative_factor; @@ -2887,7 +2891,7 @@ demangle_read_number(struct demangle_data *ddata, long *rtn) return 1; } -static privileged int +ABI static int demangle_read_number_as_string(struct demangle_data *ddata, char **str) { long n; @@ -2904,7 +2908,7 @@ demangle_read_number_as_string(struct demangle_data *ddata, char **str) return 1; } -static privileged int +ABI static int demangle_read_encoding_impl(struct demangle_data *ddata) { char *name, *type, *num_str; @@ -3113,7 +3117,7 @@ demangle_read_encoding_impl(struct demangle_data *ddata) } /* read encoding, encoding are function name, data name, special-name */ -static privileged int +ABI static int demangle_read_encoding(struct demangle_data *ddata) { if (ddata->depth == MAX_DEPTH) @@ -3124,7 +3128,7 @@ demangle_read_encoding(struct demangle_data *ddata) return res; } -static privileged int +ABI static int demangle_read_local_name(struct demangle_data *ddata) { struct vector_str local_name; @@ -3205,7 +3209,7 @@ demangle_read_local_name(struct demangle_data *ddata) return 1; } -static privileged int +ABI static int demangle_read_nested_name(struct demangle_data *ddata) { struct stack_str v; @@ -3293,7 +3297,7 @@ demangle_read_nested_name(struct demangle_data *ddata) return 1; } -static privileged int +ABI static int demangle_read_name_impl(struct demangle_data *ddata) { struct stack_str v; @@ -3355,7 +3359,7 @@ demangle_read_name_impl(struct demangle_data *ddata) return rtn; } -static privileged int +ABI static int demangle_read_name(struct demangle_data *ddata) { if (ddata->depth == MAX_DEPTH) @@ -3366,7 +3370,7 @@ demangle_read_name(struct demangle_data *ddata) return res; } -static privileged int +ABI static int demangle_read_name_flat(struct demangle_data *ddata, char **str) { struct vector_str *output; @@ -3394,7 +3398,7 @@ demangle_read_name_flat(struct demangle_data *ddata, char **str) return 1; } -static privileged int +ABI static int demangle_read_pointer_to_member(struct demangle_data *ddata, struct vector_type_qualifier *v) { @@ -3454,7 +3458,7 @@ demangle_read_pointer_to_member(struct demangle_data *ddata, } /* read source-name, source-name is */ -static privileged int +ABI static int demangle_read_sname(struct demangle_data *ddata) { size_t lim; @@ -3485,7 +3489,7 @@ demangle_read_sname(struct demangle_data *ddata) return 1; } -static privileged int +ABI static int demangle_read_subst_stdtmpl(struct demangle_data *ddata, const char *str) { struct vector_str *output; @@ -3523,7 +3527,7 @@ demangle_read_subst_stdtmpl(struct demangle_data *ddata, const char *str) return 1; } -static privileged int +ABI static int demangle_read_subst_std(struct demangle_data *ddata) { struct vector_str *output, v; @@ -3574,7 +3578,7 @@ demangle_read_subst_std(struct demangle_data *ddata) return 1; } -static privileged int +ABI static int demangle_read_subst(struct demangle_data *ddata) { long nth; @@ -3702,7 +3706,7 @@ demangle_read_subst(struct demangle_data *ddata) return 0; } -static privileged int +ABI static int demangle_vector_type_qualifier_push(struct demangle_data *ddata, struct vector_type_qualifier *v, enum type_qualifier t) { @@ -3731,7 +3735,7 @@ demangle_vector_type_qualifier_push(struct demangle_data *ddata, return 1; } -static privileged int +ABI static int demangle_read_type_impl(struct demangle_data *ddata, struct type_delimit *td) { struct vector_type_qualifier v; @@ -4254,7 +4258,7 @@ demangle_read_type_impl(struct demangle_data *ddata, struct type_delimit *td) return 0; } -static privileged int +ABI static int demangle_read_type(struct demangle_data *ddata, struct type_delimit *td) { if (ddata->depth == MAX_DEPTH) @@ -4265,7 +4269,7 @@ demangle_read_type(struct demangle_data *ddata, struct type_delimit *td) return res; } -static privileged int +ABI static int demangle_copy_output(struct demangle_data *ddata, char *buf, const struct vector_str *v, size_t buflen) { @@ -4288,14 +4292,14 @@ demangle_copy_output(struct demangle_data *ddata, char *buf, return -1; } -static privileged int +ABI static int demangle_failure(char *buf, const char *org, size_t buflen) { demangle_strlcpy(buf, org, buflen); return -1; } -static privileged int +ABI static int demangle(struct demangle_data *ddata, char *buf, const char *org, size_t buflen) { struct vector_str ret_type; @@ -4447,7 +4451,7 @@ demangle(struct demangle_data *ddata, char *buf, const char *org, size_t buflen) * @return bytes of output name or -1 upon error or truncation * @asyncsignalsafe */ -privileged int +ABI int __demangle(char *buf, const char *org, size_t buflen) { struct demangle_data ddata[1]; @@ -4461,7 +4465,7 @@ __demangle(char *buf, const char *org, size_t buflen) * * This means it starts with either "_Z" or "_GLOBAL__I_". */ -privileged int +ABI int __is_mangled(const char *org) { if (!org) diff --git a/libc/intrin/describebacktrace.c b/libc/intrin/describebacktrace.c index 8c92e93ebee..7d61f5bc988 100644 --- a/libc/intrin/describebacktrace.c +++ b/libc/intrin/describebacktrace.c @@ -24,13 +24,15 @@ #define N 160 -privileged static bool IsDangerous(const void *ptr) { +#define ABI privileged optimizesize + +ABI static bool IsDangerous(const void *ptr) { if (_weaken(kisdangerous)) return _weaken(kisdangerous)(ptr); return false; } -privileged static char *FormatHex(char *p, unsigned long x) { +ABI static char *FormatHex(char *p, unsigned long x) { int k = x ? (__builtin_clzl(x) ^ 63) + 1 : 1; k = (k + 3) & -4; while (k > 0) @@ -39,8 +41,7 @@ privileged static char *FormatHex(char *p, unsigned long x) { return p; } -privileged dontinstrument const char *_DescribeBacktrace( - char buf[N], const struct StackFrame *fr) { +ABI const char *_DescribeBacktrace(char buf[N], const struct StackFrame *fr) { char *p = buf; char *pe = p + N; bool gotsome = false; diff --git a/libc/intrin/fds.c b/libc/intrin/fds.c index 67e610bfc9b..02c5ebbf7d8 100644 --- a/libc/intrin/fds.c +++ b/libc/intrin/fds.c @@ -44,6 +44,7 @@ #include "libc/sysv/consts/o.h" #include "libc/sysv/consts/prot.h" #include "libc/thread/thread.h" +#include "libc/thread/tls.h" #define OPEN_MAX 16 @@ -86,6 +87,7 @@ static textwindows void SetupWinStd(struct Fds *fds, int i, uint32_t x) { } textstartup void __init_fds(int argc, char **argv, char **envp) { + struct Fds *fds; fds = &g_fds; fds->n = 4; diff --git a/libc/intrin/getsafesize.greg.c b/libc/intrin/getsafesize.greg.c index 83a772e8ada..c7735cd1f3d 100644 --- a/libc/intrin/getsafesize.greg.c +++ b/libc/intrin/getsafesize.greg.c @@ -22,7 +22,6 @@ #include "libc/runtime/stack.h" #include "libc/thread/posixthread.internal.h" #include "libc/thread/tls.h" -#include "libc/thread/tls2.internal.h" /** * Computes safer buffer size for alloca(). @@ -32,7 +31,7 @@ * @return number of bytes to use for your buffer, or negative if the * allocation would likely cause a stack overflow */ -privileged long __get_safe_size(long want, long extraspace) { +privileged optimizesize long __get_safe_size(long want, long extraspace) { if (!__tls_enabled) return want; struct PosixThread *pt; diff --git a/libc/intrin/kprintf.greg.c b/libc/intrin/kprintf.greg.c index 1654038fea2..e8444fde020 100644 --- a/libc/intrin/kprintf.greg.c +++ b/libc/intrin/kprintf.greg.c @@ -65,10 +65,11 @@ #include "libc/sysv/consts/o.h" #include "libc/sysv/consts/prot.h" #include "libc/thread/tls.h" -#include "libc/thread/tls2.internal.h" #include "libc/vga/vga.internal.h" #include "libc/wctype.h" +#define ABI privileged optimizesize + #define STACK_ERROR "kprintf error: stack is about to overflow\n" #define KGETINT(x, va, t, s) \ @@ -159,7 +160,7 @@ __funline bool kischarmisaligned(const char *p, signed char t) { return false; } -privileged bool32 kisdangerous(const void *addr) { +ABI bool32 kisdangerous(const void *addr) { bool32 res = true; __maps_lock(); if (__maps.maps) { @@ -175,7 +176,7 @@ privileged bool32 kisdangerous(const void *addr) { return res; } -privileged static void klogclose(long fd) { +ABI static void klogclose(long fd) { #ifdef __x86_64__ long ax = __NR_close; asm volatile("syscall" @@ -192,7 +193,7 @@ privileged static void klogclose(long fd) { #endif } -privileged static long klogfcntl(long fd, long cmd, long arg) { +ABI static long klogfcntl(long fd, long cmd, long arg) { #ifdef __x86_64__ char cf; long ax = __NR_fcntl; @@ -224,7 +225,7 @@ privileged static long klogfcntl(long fd, long cmd, long arg) { #endif } -privileged static long klogopen(const char *path) { +ABI static long klogopen(const char *path) { long dirfd = AT_FDCWD; long flags = O_WRONLY | O_CREAT | O_APPEND; long mode = 0600; @@ -263,7 +264,7 @@ privileged static long klogopen(const char *path) { } // returns log handle or -1 if logging shouldn't happen -privileged long kloghandle(void) { +ABI long kloghandle(void) { // kprintf() needs to own a file descriptor in case apps closes stderr // our close() and dup() implementations will trigger this initializer // to minimize a chance that the user accidentally closes their logger @@ -342,7 +343,7 @@ privileged long kloghandle(void) { } #ifdef __x86_64__ -privileged void _klog_serial(const char *b, size_t n) { +ABI void _klog_serial(const char *b, size_t n) { size_t i; uint16_t dx; unsigned char al; @@ -362,7 +363,7 @@ privileged void _klog_serial(const char *b, size_t n) { } #endif /* __x86_64__ */ -privileged void klog(const char *b, size_t n) { +ABI void klog(const char *b, size_t n) { #ifdef __x86_64__ long h; uint32_t wrote; @@ -420,8 +421,7 @@ privileged void klog(const char *b, size_t n) { #endif } -privileged static size_t kformat(char *b, size_t n, const char *fmt, - va_list va) { +ABI static size_t kformat(char *b, size_t n, const char *fmt, va_list va) { int si; wint_t t, u; const char *abet; @@ -1033,7 +1033,7 @@ privileged static size_t kformat(char *b, size_t n, const char *fmt, * @asyncsignalsafe * @vforksafe */ -privileged size_t ksnprintf(char *b, size_t n, const char *fmt, ...) { +ABI size_t ksnprintf(char *b, size_t n, const char *fmt, ...) { size_t m; va_list v; va_start(v, fmt); @@ -1052,7 +1052,7 @@ privileged size_t ksnprintf(char *b, size_t n, const char *fmt, ...) { * @asyncsignalsafe * @vforksafe */ -privileged size_t kvsnprintf(char *b, size_t n, const char *fmt, va_list v) { +ABI size_t kvsnprintf(char *b, size_t n, const char *fmt, va_list v) { return kformat(b, n, fmt, v); } @@ -1063,7 +1063,7 @@ privileged size_t kvsnprintf(char *b, size_t n, const char *fmt, va_list v) { * @asyncsignalsafe * @vforksafe */ -privileged void kvprintf(const char *fmt, va_list v) { +ABI void kvprintf(const char *fmt, va_list v) { #pragma GCC push_options #pragma GCC diagnostic ignored "-Walloca-larger-than=" long size = __get_safe_size(8000, 8000); @@ -1149,7 +1149,7 @@ privileged void kvprintf(const char *fmt, va_list v) { * @asyncsignalsafe * @vforksafe */ -privileged void kprintf(const char *fmt, ...) { +ABI void kprintf(const char *fmt, ...) { // system call support runtime depends on this function // function tracing runtime depends on this function // asan runtime depends on this function diff --git a/libc/stdio/srand.c b/libc/intrin/localtime_lock.c similarity index 84% rename from libc/stdio/srand.c rename to libc/intrin/localtime_lock.c index 8b072163ee2..b8d2868607f 100644 --- a/libc/stdio/srand.c +++ b/libc/intrin/localtime_lock.c @@ -1,7 +1,7 @@ /*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ │ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │ ╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2020 Justine Alexandra Roberts Tunney │ +│ Copyright 2024 Justine Alexandra Roberts Tunney │ │ │ │ Permission to use, copy, modify, and/or distribute this software for │ │ any purpose with or without fee is hereby granted, provided that the │ @@ -16,13 +16,14 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/stdio/rand.h" +#include "third_party/tz/lock.h" -extern uint64_t g_rando; +pthread_mutex_t __localtime_lock_obj = PTHREAD_MUTEX_INITIALIZER; -/** - * Seeds random number generator that's used by rand(). - */ -void srand(unsigned seed) { - g_rando = seed; +void __localtime_lock(void) { + pthread_mutex_lock(&__localtime_lock_obj); +} + +void __localtime_unlock(void) { + pthread_mutex_unlock(&__localtime_lock_obj); } diff --git a/libc/intrin/maps.c b/libc/intrin/maps.c index ede90e395b5..8379f8cf157 100644 --- a/libc/intrin/maps.c +++ b/libc/intrin/maps.c @@ -19,14 +19,15 @@ #include "libc/intrin/maps.h" #include "ape/sections.internal.h" #include "libc/calls/state.internal.h" +#include "libc/cosmo.h" #include "libc/dce.h" #include "libc/intrin/describebacktrace.h" #include "libc/intrin/dll.h" #include "libc/intrin/kprintf.h" #include "libc/intrin/maps.h" +#include "libc/nexgen32e/rdtsc.h" #include "libc/runtime/runtime.h" #include "libc/runtime/stack.h" -#include "libc/sysv/consts/auxv.h" #include "libc/sysv/consts/prot.h" #include "libc/thread/lock.h" @@ -34,6 +35,12 @@ __static_yoink("_init_maps"); #endif +#define ABI privileged optimizespeed + +// take great care if you enable this +// especially if you're using --ftrace too +#define DEBUG_MAPS_LOCK 0 + struct Maps __maps; void __maps_add(struct Map *map) { @@ -65,6 +72,10 @@ void __maps_stack(char *stackaddr, int pagesz, int guardsize, size_t stacksize, void __maps_init(void) { int pagesz = __pagesize; + // initialize lemur64 rng + __maps.rand = 2131259787901769494; + __maps.rand ^= rdtsc(); + // record _start() stack mapping if (!IsWindows()) { struct AddrSize stack; @@ -88,7 +99,16 @@ void __maps_init(void) { __maps_adder(&text, pagesz); } -privileged bool __maps_lock(void) { +#if DEBUG_MAPS_LOCK +privileged static void __maps_panic(const char *msg) { + // it's only safe to pass a format string. if we use directives such + // as %s, %t etc. then kprintf() will recursively call __maps_lock() + kprintf(msg); + DebugBreak(); +} +#endif + +ABI bool __maps_lock(void) { int me; uint64_t word, lock; struct CosmoTib *tib; @@ -101,24 +121,35 @@ privileged bool __maps_lock(void) { me = atomic_load_explicit(&tib->tib_tid, memory_order_acquire); if (me <= 0) return false; - word = atomic_load_explicit(&__maps.lock, memory_order_relaxed); + word = atomic_load_explicit(&__maps.lock.word, memory_order_relaxed); for (;;) { if (MUTEX_OWNER(word) == me) { if (atomic_compare_exchange_weak_explicit( - &__maps.lock, &word, MUTEX_INC_DEPTH(word), memory_order_relaxed, - memory_order_relaxed)) + &__maps.lock.word, &word, MUTEX_INC_DEPTH(word), + memory_order_relaxed, memory_order_relaxed)) return true; continue; } +#if DEBUG_MAPS_LOCK + if (__deadlock_tracked(&__maps.lock) == 1) + __maps_panic("error: maps lock already held\n"); + if (__deadlock_check(&__maps.lock, 1)) + __maps_panic("error: maps lock is cyclic\n"); +#endif word = 0; lock = MUTEX_LOCK(word); lock = MUTEX_SET_OWNER(lock, me); - if (atomic_compare_exchange_weak_explicit(&__maps.lock, &word, lock, + if (atomic_compare_exchange_weak_explicit(&__maps.lock.word, &word, lock, memory_order_acquire, - memory_order_relaxed)) + memory_order_relaxed)) { +#if DEBUG_MAPS_LOCK + __deadlock_track(&__maps.lock, 0); + __deadlock_record(&__maps.lock, 0); +#endif return false; + } for (;;) { - word = atomic_load_explicit(&__maps.lock, memory_order_relaxed); + word = atomic_load_explicit(&__maps.lock.word, memory_order_relaxed); if (MUTEX_OWNER(word) == me) break; if (!word) @@ -127,7 +158,7 @@ privileged bool __maps_lock(void) { } } -privileged void __maps_unlock(void) { +ABI void __maps_unlock(void) { int me; uint64_t word; struct CosmoTib *tib; @@ -140,16 +171,25 @@ privileged void __maps_unlock(void) { me = atomic_load_explicit(&tib->tib_tid, memory_order_acquire); if (me <= 0) return; - word = atomic_load_explicit(&__maps.lock, memory_order_relaxed); + word = atomic_load_explicit(&__maps.lock.word, memory_order_relaxed); +#if DEBUG_MAPS_LOCK + if (__deadlock_tracked(&__maps.lock) == 0) + __maps_panic("error: maps lock not owned by caller\n"); +#endif for (;;) { if (MUTEX_DEPTH(word)) { if (atomic_compare_exchange_weak_explicit( - &__maps.lock, &word, MUTEX_DEC_DEPTH(word), memory_order_relaxed, - memory_order_relaxed)) + &__maps.lock.word, &word, MUTEX_DEC_DEPTH(word), + memory_order_relaxed, memory_order_relaxed)) break; } - if (atomic_compare_exchange_weak_explicit( - &__maps.lock, &word, 0, memory_order_release, memory_order_relaxed)) + if (atomic_compare_exchange_weak_explicit(&__maps.lock.word, &word, 0, + memory_order_release, + memory_order_relaxed)) { +#if DEBUG_MAPS_LOCK + __deadlock_untrack(&__maps.lock); +#endif break; + } } } diff --git a/libc/intrin/maps.h b/libc/intrin/maps.h index 8546a6c5ed2..ad439448d2e 100644 --- a/libc/intrin/maps.h +++ b/libc/intrin/maps.h @@ -3,7 +3,6 @@ #include "libc/intrin/atomic.h" #include "libc/intrin/tree.h" #include "libc/runtime/runtime.h" -#include "libc/thread/tls2.internal.h" COSMOPOLITAN_C_START_ #define MAPS_RETRY ((void *)-1) @@ -26,9 +25,15 @@ struct Map { }; }; +struct MapLock { + void *edges; + _Atomic(uint64_t) word; +}; + struct Maps { + uint128_t rand; struct Tree *maps; - _Atomic(uint64_t) lock; + struct MapLock lock; _Atomic(uintptr_t) freed; size_t count; size_t pages; diff --git a/libc/intrin/mmap.c b/libc/intrin/mmap.c index cb39dd2bd44..c35e83466e4 100644 --- a/libc/intrin/mmap.c +++ b/libc/intrin/mmap.c @@ -34,7 +34,6 @@ #include "libc/nt/runtime.h" #include "libc/runtime/runtime.h" #include "libc/runtime/zipos.internal.h" -#include "libc/stdio/rand.h" #include "libc/stdio/sysparam.h" #include "libc/sysv/consts/map.h" #include "libc/sysv/consts/mremap.h" @@ -42,7 +41,7 @@ #include "libc/sysv/consts/prot.h" #include "libc/sysv/errfuns.h" -#define MMDEBUG IsModeDbg() +#define MMDEBUG 0 #define MAX_SIZE 0x0ff800000000ul #define MAX_TRIES 50 @@ -404,7 +403,9 @@ static int __munmap(char *addr, size_t size) { void *__maps_randaddr(void) { uintptr_t addr; - addr = _rand64(); + __maps_lock(); + addr = (__maps.rand *= 15750249268501108917ull) >> 64; + __maps_unlock(); addr &= 0x3fffffffffff; addr |= 0x004000000000; addr &= -__gransize; diff --git a/libc/intrin/pthread_atfork.c b/libc/intrin/pthread_atfork.c deleted file mode 100644 index 5093ed5940a..00000000000 --- a/libc/intrin/pthread_atfork.c +++ /dev/null @@ -1,77 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ -│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2022 Justine Alexandra Roberts Tunney │ -│ │ -│ Permission to use, copy, modify, and/or distribute this software for │ -│ any purpose with or without fee is hereby granted, provided that the │ -│ above copyright notice and this permission notice appear in all copies. │ -│ │ -│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ -│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ -│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ -│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ -│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ -│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ -│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ -│ PERFORMANCE OF THIS SOFTWARE. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/intrin/weaken.h" -#include "libc/thread/posixthread.internal.h" -#include "libc/thread/thread.h" - -/** - * Registers fork() handlers. - * - * Parent and child functions are called in the same order they're - * registered. Prepare functions are called in reverse order. - * - * Here's an example of how pthread_atfork() can be used: - * - * static struct { - * pthread_once_t once; - * pthread_mutex_t lock; - * // data structures... - * } g_lib; - * - * static void lib_wipe(void) { - * pthread_mutex_init(&g_lib.lock, 0); - * } - * - * static void lib_lock(void) { - * pthread_mutex_lock(&g_lib.lock); - * } - * - * static void lib_unlock(void) { - * pthread_mutex_unlock(&g_lib.lock); - * } - * - * static void lib_setup(void) { - * lib_wipe(); - * pthread_atfork(lib_lock, lib_unlock, lib_wipe); - * } - * - * static void lib_init(void) { - * pthread_once(&g_lib.once, lib_setup); - * } - * - * void lib(void) { - * lib_init(); - * lib_lock(); - * // do stuff... - * lib_unlock(); - * } - * - * @param prepare is run by fork() before forking happens - * @param parent is run by fork() after forking happens in parent process - * @param child is run by fork() after forking happens in childe process - * @return 0 on success, or errno on error - * @raise ENOMEM if we require more vespene gas - */ -int pthread_atfork(atfork_f prepare, atfork_f parent, atfork_f child) { - if (_weaken(_pthread_atfork)) { - return _weaken(_pthread_atfork)(prepare, parent, child); - } else { - return 0; - } -} diff --git a/libc/intrin/pthread_atfork_actual.c b/libc/intrin/pthread_atfork_actual.c deleted file mode 100644 index 505cbdc9672..00000000000 --- a/libc/intrin/pthread_atfork_actual.c +++ /dev/null @@ -1,101 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ -│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2022 Justine Alexandra Roberts Tunney │ -│ │ -│ Permission to use, copy, modify, and/or distribute this software for │ -│ any purpose with or without fee is hereby granted, provided that the │ -│ above copyright notice and this permission notice appear in all copies. │ -│ │ -│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ -│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ -│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ -│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ -│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ -│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ -│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ -│ PERFORMANCE OF THIS SOFTWARE. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/atomic.h" -#include "libc/calls/state.internal.h" -#include "libc/cosmo.h" -#include "libc/dce.h" -#include "libc/errno.h" -#include "libc/intrin/atomic.h" -#include "libc/intrin/dll.h" -#include "libc/intrin/strace.h" -#include "libc/macros.h" -#include "libc/proc/proc.internal.h" -#include "libc/runtime/runtime.h" -#include "libc/str/str.h" -#include "libc/thread/posixthread.internal.h" -#include "libc/thread/thread.h" -#include "libc/thread/tls.h" - -struct AtFork { - struct AtFork *p[2]; - atfork_f f[3]; -}; - -static struct AtForks { - pthread_spinlock_t lock; - struct AtFork *list; - struct AtFork pool[64]; - atomic_int allocated; -} _atforks; - -static void _pthread_onfork(int i, const char *op) { - struct AtFork *a; - if (!i) - pthread_spin_lock(&_atforks.lock); - for (a = _atforks.list; a; a = a->p[!i]) { - if (a->f[i]) { - STRACE("pthread_atfork(%s, %t)", op, a->f[i]); - a->f[i](); - } - _atforks.list = a; - } - if (i) - pthread_spin_unlock(&_atforks.lock); -} - -void _pthread_onfork_prepare(void) { - _pthread_onfork(0, "prepare"); -} - -void _pthread_onfork_parent(void) { - _pthread_onfork(1, "parent"); -} - -void _pthread_onfork_child(void) { - _pthread_onfork(2, "child"); -} - -static struct AtFork *_pthread_atfork_alloc(void) { - int i, n = ARRAYLEN(_atforks.pool); - if (atomic_load_explicit(&_atforks.allocated, memory_order_relaxed) < n && - (i = atomic_fetch_add(&_atforks.allocated, 1)) < n) { - return _atforks.pool + i; - } else { - return 0; - } -} - -int _pthread_atfork(atfork_f prepare, atfork_f parent, atfork_f child) { - int rc; - struct AtFork *a; - if (!(a = _pthread_atfork_alloc())) - return ENOMEM; - a->f[0] = prepare; - a->f[1] = parent; - a->f[2] = child; - pthread_spin_lock(&_atforks.lock); - a->p[0] = 0; - a->p[1] = _atforks.list; - if (_atforks.list) - _atforks.list->p[0] = a; - _atforks.list = a; - pthread_spin_unlock(&_atforks.lock); - rc = 0; - return rc; -} diff --git a/libc/stdio/fflushimpl.c b/libc/intrin/pthread_mutex_consistent.c similarity index 64% rename from libc/stdio/fflushimpl.c rename to libc/intrin/pthread_mutex_consistent.c index 41e047f01ac..44a5fd5f680 100644 --- a/libc/stdio/fflushimpl.c +++ b/libc/intrin/pthread_mutex_consistent.c @@ -1,7 +1,7 @@ /*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ │ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │ ╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2021 Justine Alexandra Roberts Tunney │ +│ Copyright 2024 Justine Alexandra Roberts Tunney │ │ │ │ Permission to use, copy, modify, and/or distribute this software for │ │ any purpose with or without fee is hereby granted, provided that the │ @@ -16,41 +16,26 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/calls/calls.h" -#include "libc/errno.h" -#include "libc/intrin/weaken.h" -#include "libc/mem/mem.h" -#include "libc/runtime/runtime.h" -#include "libc/stdio/internal.h" -#include "libc/stdio/stdio.h" -#include "libc/sysv/consts/o.h" +#include "libc/cosmo.h" +#include "libc/dce.h" +#include "libc/intrin/atomic.h" +#include "libc/thread/lock.h" +#include "libc/thread/thread.h" + +/** + * Recovers mutex whose owner died. + * + * @return 0 on success, or errno on error + */ +int pthread_mutex_consistent(pthread_mutex_t *mutex) { + + // The POSIX concept of robust mutexes is a bit cray. So let's change + // things up a bit. Rather than implementing all those goofy behaviors + // we shall simply use this function to weasel around the ownership + // check in pthread_mutex_unlock(). + uint64_t word = atomic_load_explicit(&mutex->_word, memory_order_relaxed); + if (IsModeDbg() || MUTEX_TYPE(word) == PTHREAD_MUTEX_ERRORCHECK) + __deadlock_track(mutex, 0); -int __fflush_impl(FILE *f) { - size_t i; - ssize_t rc; - if (f->getln) { - if (_weaken(free)) { - _weaken(free)(f->getln); - } - f->getln = 0; - } - if (f->fd != -1) { - if (f->beg && !f->end && (f->iomode & O_ACCMODE) != O_RDONLY) { - for (i = 0; i < f->beg; i += rc) { - if ((rc = write(f->fd, f->buf + i, f->beg - i)) == -1) { - f->state = errno; - return -1; - } - } - f->beg = 0; - } - if (f->beg < f->end && (f->iomode & O_ACCMODE) != O_WRONLY) { - if (lseek(f->fd, -(int)(f->end - f->beg), SEEK_CUR) == -1) { - f->state = errno; - return -1; - } - f->end = f->beg; - } - } return 0; } diff --git a/libc/intrin/pthread_mutex_init.c b/libc/intrin/pthread_mutex_init.c index 8801f23720c..1ce34716bfd 100644 --- a/libc/intrin/pthread_mutex_init.c +++ b/libc/intrin/pthread_mutex_init.c @@ -24,7 +24,7 @@ * pthread_mutex_t lock; * pthread_mutexattr_t attr; * pthread_mutexattr_init(&attr); - * pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_NORMAL); + * pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_DEFAULT); * pthread_mutex_init(&lock, &attr); * pthread_mutexattr_destroy(&attr); * // ... diff --git a/libc/intrin/pthread_mutex_lock.c b/libc/intrin/pthread_mutex_lock.c index ea2c7d09caf..9947bbc5ec1 100644 --- a/libc/intrin/pthread_mutex_lock.c +++ b/libc/intrin/pthread_mutex_lock.c @@ -24,62 +24,82 @@ #include "libc/errno.h" #include "libc/intrin/atomic.h" #include "libc/intrin/describeflags.h" +#include "libc/intrin/kprintf.h" #include "libc/intrin/strace.h" #include "libc/intrin/weaken.h" +#include "libc/macros.h" #include "libc/runtime/internal.h" #include "libc/thread/lock.h" #include "libc/thread/thread.h" +#include "libc/thread/tls.h" #include "third_party/nsync/mu.h" +static errno_t pthread_mutex_lock_normal_success(pthread_mutex_t *mutex, + uint64_t word) { + if (IsModeDbg() || MUTEX_TYPE(word) == PTHREAD_MUTEX_ERRORCHECK) { + __deadlock_track(mutex, MUTEX_TYPE(word) == PTHREAD_MUTEX_ERRORCHECK); + __deadlock_record(mutex, MUTEX_TYPE(word) == PTHREAD_MUTEX_ERRORCHECK); + } + return 0; +} + // see "take 3" algorithm in "futexes are tricky" by ulrich drepper // slightly improved to attempt acquiring multiple times b4 syscall -static void pthread_mutex_lock_drepper(atomic_int *futex, char pshare) { - int word = 0; +static int pthread_mutex_lock_drepper(pthread_mutex_t *mutex, uint64_t word, + bool is_trylock) { + int val = 0; if (atomic_compare_exchange_strong_explicit( - futex, &word, 1, memory_order_acquire, memory_order_acquire)) - return; - LOCKTRACE("acquiring pthread_mutex_lock_drepper(%t)...", futex); - if (word == 1) - word = atomic_exchange_explicit(futex, 2, memory_order_acquire); + &mutex->_futex, &val, 1, memory_order_acquire, memory_order_acquire)) + return pthread_mutex_lock_normal_success(mutex, word); + if (is_trylock) + return EBUSY; + LOCKTRACE("acquiring pthread_mutex_lock_drepper(%t)...", mutex); + if (val == 1) + val = atomic_exchange_explicit(&mutex->_futex, 2, memory_order_acquire); BLOCK_CANCELATION; - while (word > 0) { - cosmo_futex_wait(futex, 2, pshare, 0, 0); - word = atomic_exchange_explicit(futex, 2, memory_order_acquire); + while (val > 0) { + cosmo_futex_wait(&mutex->_futex, 2, MUTEX_PSHARED(word), 0, 0); + val = atomic_exchange_explicit(&mutex->_futex, 2, memory_order_acquire); } ALLOW_CANCELATION; + return pthread_mutex_lock_normal_success(mutex, word); } static errno_t pthread_mutex_lock_recursive(pthread_mutex_t *mutex, - uint64_t word) { + uint64_t word, bool is_trylock) { uint64_t lock; int backoff = 0; int me = gettid(); bool once = false; for (;;) { if (MUTEX_OWNER(word) == me) { - if (MUTEX_TYPE(word) != PTHREAD_MUTEX_ERRORCHECK) { - if (MUTEX_DEPTH(word) < MUTEX_DEPTH_MAX) { - if (atomic_compare_exchange_weak_explicit( - &mutex->_word, &word, MUTEX_INC_DEPTH(word), - memory_order_relaxed, memory_order_relaxed)) - return 0; - continue; - } else { - return EAGAIN; - } + if (MUTEX_DEPTH(word) < MUTEX_DEPTH_MAX) { + if (atomic_compare_exchange_weak_explicit( + &mutex->_word, &word, MUTEX_INC_DEPTH(word), + memory_order_relaxed, memory_order_relaxed)) + return 0; + continue; } else { - return EDEADLK; + return EAGAIN; } } + if (IsModeDbg()) + __deadlock_check(mutex, 0); word = MUTEX_UNLOCK(word); lock = MUTEX_LOCK(word); lock = MUTEX_SET_OWNER(lock, me); if (atomic_compare_exchange_weak_explicit(&mutex->_word, &word, lock, memory_order_acquire, memory_order_relaxed)) { + if (IsModeDbg()) { + __deadlock_track(mutex, 0); + __deadlock_record(mutex, 0); + } mutex->_pid = __pid; return 0; } + if (is_trylock) + return EBUSY; if (!once) { LOCKTRACE("acquiring pthread_mutex_lock_recursive(%t)...", mutex); once = true; @@ -97,25 +117,33 @@ static errno_t pthread_mutex_lock_recursive(pthread_mutex_t *mutex, #if PTHREAD_USE_NSYNC static errno_t pthread_mutex_lock_recursive_nsync(pthread_mutex_t *mutex, - uint64_t word) { + uint64_t word, + bool is_trylock) { int me = gettid(); for (;;) { if (MUTEX_OWNER(word) == me) { - if (MUTEX_TYPE(word) != PTHREAD_MUTEX_ERRORCHECK) { - if (MUTEX_DEPTH(word) < MUTEX_DEPTH_MAX) { - if (atomic_compare_exchange_weak_explicit( - &mutex->_word, &word, MUTEX_INC_DEPTH(word), - memory_order_relaxed, memory_order_relaxed)) - return 0; - continue; - } else { - return EAGAIN; - } + if (MUTEX_DEPTH(word) < MUTEX_DEPTH_MAX) { + if (atomic_compare_exchange_weak_explicit( + &mutex->_word, &word, MUTEX_INC_DEPTH(word), + memory_order_relaxed, memory_order_relaxed)) + return 0; + continue; } else { - return EDEADLK; + return EAGAIN; } } - _weaken(nsync_mu_lock)((nsync_mu *)mutex->_nsyncx); + if (IsModeDbg()) + __deadlock_check(mutex, 0); + if (!is_trylock) { + _weaken(nsync_mu_lock)((nsync_mu *)mutex->_nsync); + } else { + if (!_weaken(nsync_mu_trylock)((nsync_mu *)mutex->_nsync)) + return EBUSY; + } + if (IsModeDbg()) { + __deadlock_track(mutex, 0); + __deadlock_record(mutex, 0); + } word = MUTEX_UNLOCK(word); word = MUTEX_LOCK(word); word = MUTEX_SET_OWNER(word, me); @@ -126,69 +154,82 @@ static errno_t pthread_mutex_lock_recursive_nsync(pthread_mutex_t *mutex, } #endif -static errno_t pthread_mutex_lock_impl(pthread_mutex_t *mutex) { - uint64_t word; +static errno_t pthread_mutex_lock_impl(pthread_mutex_t *mutex, + bool is_trylock) { + uint64_t word = atomic_load_explicit(&mutex->_word, memory_order_relaxed); + + // handle recursive mutexes + if (MUTEX_TYPE(word) == PTHREAD_MUTEX_RECURSIVE) { +#if PTHREAD_USE_NSYNC + if (_weaken(nsync_mu_lock) && + MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE) { + return pthread_mutex_lock_recursive_nsync(mutex, word, is_trylock); + } else { + return pthread_mutex_lock_recursive(mutex, word, is_trylock); + } +#else + return pthread_mutex_lock_recursive(mutex, word, is_trylock); +#endif + } + + // check if normal mutex is already owned by calling thread + if (!is_trylock && + (MUTEX_TYPE(word) == PTHREAD_MUTEX_ERRORCHECK || + (IsModeDbg() && MUTEX_TYPE(word) == PTHREAD_MUTEX_DEFAULT))) { + if (__deadlock_tracked(mutex) == 1) { + if (IsModeDbg() && MUTEX_TYPE(word) != PTHREAD_MUTEX_ERRORCHECK) { + kprintf("error: attempted to lock non-recursive mutex that's already " + "held by the calling thread: %t\n", + mutex); + DebugBreak(); + } + return EDEADLK; + } + } - // get current state of lock - word = atomic_load_explicit(&mutex->_word, memory_order_relaxed); + // check if locking will create cycle in lock graph + if (IsModeDbg() || MUTEX_TYPE(word) == PTHREAD_MUTEX_ERRORCHECK) + if (__deadlock_check(mutex, MUTEX_TYPE(word) == PTHREAD_MUTEX_ERRORCHECK)) + return EDEADLK; #if PTHREAD_USE_NSYNC // use superior mutexes if possible - if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL && // - MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE && // + if (MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE && _weaken(nsync_mu_lock)) { // on apple silicon we should just put our faith in ulock // otherwise *nsync gets struck down by the eye of sauron if (!IsXnuSilicon()) { - _weaken(nsync_mu_lock)((nsync_mu *)mutex); - return 0; + if (!is_trylock) { + _weaken(nsync_mu_lock)((nsync_mu *)mutex->_nsync); + return pthread_mutex_lock_normal_success(mutex, word); + } else { + if (_weaken(nsync_mu_trylock)((nsync_mu *)mutex->_nsync)) + return pthread_mutex_lock_normal_success(mutex, word); + return EBUSY; + } } } #endif - // handle normal mutexes - if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL) { - pthread_mutex_lock_drepper(&mutex->_futex, MUTEX_PSHARED(word)); - return 0; - } - -// handle recursive and error checking mutexes -#if PTHREAD_USE_NSYNC - if (_weaken(nsync_mu_lock) && - MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE) { - return pthread_mutex_lock_recursive_nsync(mutex, word); - } else { - return pthread_mutex_lock_recursive(mutex, word); - } -#else - return pthread_mutex_lock_recursive(mutex, word); -#endif + // isc licensed non-recursive mutex implementation + return pthread_mutex_lock_drepper(mutex, word, is_trylock); } /** - * Locks mutex. - * - * Here's an example of using a normal mutex: + * Locks mutex, e.g. * * pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; * pthread_mutex_lock(&lock); * // do work... * pthread_mutex_unlock(&lock); - * pthread_mutex_destroy(&lock); - * - * Cosmopolitan permits succinct notation for normal mutexes: * - * pthread_mutex_t lock = {0}; - * pthread_mutex_lock(&lock); - * // do work... - * pthread_mutex_unlock(&lock); - * - * Here's an example of the proper way to do recursive mutexes: + * The long way to do that is: * * pthread_mutex_t lock; * pthread_mutexattr_t attr; * pthread_mutexattr_init(&attr); - * pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); + * pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_DEFAULT); + * pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_PRIVATE); * pthread_mutex_init(&lock, &attr); * pthread_mutexattr_destroy(&attr); * pthread_mutex_lock(&lock); @@ -196,28 +237,99 @@ static errno_t pthread_mutex_lock_impl(pthread_mutex_t *mutex) { * pthread_mutex_unlock(&lock); * pthread_mutex_destroy(&lock); * - * This function does nothing in vfork() children. + * The following non-POSIX initializers are also provided by cosmo libc: + * + * - `PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP` + * - `PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP` + * - `PTHREAD_SIGNAL_SAFE_MUTEX_INITIALIZER_NP` + * - `PTHREAD_NORMAL_MUTEX_INITIALIZER_NP` + * + * Locking a mutex that's already locked by the calling thread will make + * the thread hang indefinitely, i.e. it's a deadlock condition. You can + * use `PTHREAD_MUTEX_RECURSIVE` to allow recursive locking, which could + * result in somewhat less performance. An alternative solution is using + * the `PTHREAD_MUTEX_ERRORCHECK` mode, which raises `EDEADLK` for that. + * + * If a thread locks a mutex while other mutexes are already locked then + * you need to observe a consistent global ordering, otherwise deadlocks + * might occur. The Cosmopolitan runtime can detect these cycles quickly + * so you can fix your code before it becomes an issue. With error check + * mode, an EPERM will be returned. If your app is using `cosmocc -mdbg` + * then an error message will be printed including the demangled symbols + * of the mutexes in the strongly connected component that was detected. + * Please note that, even for debug builds mutexes set to explicitly use + * the `PTHREAD_MUTEX_ERRORCHECK` mode will return an error code instead + * which means the cosmo debug mode only influences undefined behaviors. + * + * Cosmopolitan only supports error checking on mutexes stored in static + * memory, i.e. your `mutex` pointer must point inside the .data or .bss + * sections of your executable. When compiling your programs using -mdbg + * all your locks will gain error checking automatically. When deadlocks + * are detected an error message will be printed and a SIGTRAP signal is + * raised, which may be ignored to force EDEADLK and EPERM to be raised. + * + * Using `cosmocc -mdbg` also enhances `--strace` with information about + * mutexes. First, locks and unlocks will be logged. Since the lock line + * only appears after the lock is acquired, that might mean you'll never + * get an indication about a lock that takes a very long time to acquire + * so, whenever a lock can't immediately be acquired, a second line gets + * printed *before* the lock is acquired to let you know that the thread + * is waiting for a particular lock. If your mutex object resides within + * static memory, then its demangled symbol name will be printed. If you + * call ShowCrashReports() at the beginning of your main() function then + * you'll also see a backtrace when a locking violation occurs. When the + * symbols in the violation error messages show up as numbers, and it is + * desirable to see demangled symbols without enabling full crash report + * functionality the GetSymbolTable() function may be called for effect. * - * You can debug locks the acquisition of locks by building your program - * with `cosmocc -mdbg` and passing the `--strace` flag to your program. - * This will cause a line to be logged each time a mutex or spin lock is - * locked or unlocked. When locking, this is printed after the lock gets - * acquired. The entry to the lock operation will be logged too but only - * if the lock couldn't be immediately acquired. Lock logging works best - * when `mutex` refers to a static variable, in which case its name will - * be printed in the log. + * If you use `PTHREAD_MUTEX_NORMAL`, instead of `PTHREAD_MUTEX_DEFAULT` + * then deadlocking is actually defined behavior according to POSIX.1 so + * the helpfulness of `cosmocc -mdbg` will be somewhat weakened. + * + * If your `mutex` object resides in `MAP_SHARED` memory, then undefined + * behavior will happen unless you use `PTHREAD_PROCESS_SHARED` mode, if + * the lock is used by multiple processes. + * + * This function does nothing when the process is in vfork() mode. * * @return 0 on success, or error number on failure + * @raise EDEADLK if mutex is recursive and locked by another thread + * @raise EDEADLK if mutex is non-recursive and locked by current thread + * @raise EDEADLK if cycle is detected in global nested lock graph + * @raise EAGAIN if maximum recursive locks is exceeded * @see pthread_spin_lock() * @vforksafe */ errno_t pthread_mutex_lock(pthread_mutex_t *mutex) { - if (!__vforked) { - errno_t err = pthread_mutex_lock_impl(mutex); + if (__tls_enabled && !__vforked) { + errno_t err = pthread_mutex_lock_impl(mutex, false); LOCKTRACE("pthread_mutex_lock(%t) → %s", mutex, DescribeErrno(err)); return err; } else { - LOCKTRACE("skipping pthread_mutex_lock(%t) due to vfork", mutex); + LOCKTRACE("skipping pthread_mutex_lock(%t) due to runtime state", mutex); + return 0; + } +} + +/** + * Attempts acquiring lock. + * + * Unlike pthread_mutex_lock() this function won't block and instead + * returns an error immediately if the lock couldn't be acquired. + * + * @return 0 if lock was acquired, otherwise an errno + * @raise EBUSY if lock is currently held by another thread + * @raise EAGAIN if maximum number of recursive locks is held + * @raise EDEADLK if `mutex` is `PTHREAD_MUTEX_ERRORCHECK` and the + * current thread already holds this mutex + */ +errno_t pthread_mutex_trylock(pthread_mutex_t *mutex) { + if (__tls_enabled && !__vforked) { + errno_t err = pthread_mutex_lock_impl(mutex, true); + LOCKTRACE("pthread_mutex_trylock(%t) → %s", mutex, DescribeErrno(err)); + return err; + } else { + LOCKTRACE("skipping pthread_mutex_trylock(%t) due to runtime state", mutex); return 0; } } diff --git a/libc/intrin/pthread_mutex_trylock.c b/libc/intrin/pthread_mutex_trylock.c deleted file mode 100644 index 8391ebfe7cc..00000000000 --- a/libc/intrin/pthread_mutex_trylock.c +++ /dev/null @@ -1,152 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ -│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2023 Justine Alexandra Roberts Tunney │ -│ │ -│ Permission to use, copy, modify, and/or distribute this software for │ -│ any purpose with or without fee is hereby granted, provided that the │ -│ above copyright notice and this permission notice appear in all copies. │ -│ │ -│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ -│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ -│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ -│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ -│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ -│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ -│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ -│ PERFORMANCE OF THIS SOFTWARE. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/calls/calls.h" -#include "libc/dce.h" -#include "libc/errno.h" -#include "libc/intrin/atomic.h" -#include "libc/intrin/weaken.h" -#include "libc/runtime/internal.h" -#include "libc/thread/lock.h" -#include "libc/thread/thread.h" -#include "third_party/nsync/mu.h" - -static errno_t pthread_mutex_trylock_drepper(atomic_int *futex) { - int word = 0; - if (atomic_compare_exchange_strong_explicit( - futex, &word, 1, memory_order_acquire, memory_order_acquire)) - return 0; - return EBUSY; -} - -static errno_t pthread_mutex_trylock_recursive(pthread_mutex_t *mutex, - uint64_t word) { - uint64_t lock; - int me = gettid(); - for (;;) { - if (MUTEX_OWNER(word) == me) { - if (MUTEX_TYPE(word) != PTHREAD_MUTEX_ERRORCHECK) { - if (MUTEX_DEPTH(word) < MUTEX_DEPTH_MAX) { - if (atomic_compare_exchange_weak_explicit( - &mutex->_word, &word, MUTEX_INC_DEPTH(word), - memory_order_relaxed, memory_order_relaxed)) - return 0; - continue; - } else { - return EAGAIN; - } - } else { - return EDEADLK; - } - } - word = MUTEX_UNLOCK(word); - lock = MUTEX_LOCK(word); - lock = MUTEX_SET_OWNER(lock, me); - if (atomic_compare_exchange_weak_explicit(&mutex->_word, &word, lock, - memory_order_acquire, - memory_order_relaxed)) { - mutex->_pid = __pid; - return 0; - } - return EBUSY; - } -} - -static errno_t pthread_mutex_trylock_recursive_nsync(pthread_mutex_t *mutex, - uint64_t word) { - int me = gettid(); - for (;;) { - if (MUTEX_OWNER(word) == me) { - if (MUTEX_TYPE(word) != PTHREAD_MUTEX_ERRORCHECK) { - if (MUTEX_DEPTH(word) < MUTEX_DEPTH_MAX) { - if (atomic_compare_exchange_weak_explicit( - &mutex->_word, &word, MUTEX_INC_DEPTH(word), - memory_order_relaxed, memory_order_relaxed)) - return 0; - continue; - } else { - return EAGAIN; - } - } else { - return EDEADLK; - } - } - if (_weaken(nsync_mu_trylock)((nsync_mu *)mutex->_nsyncx)) { - word = MUTEX_UNLOCK(word); - word = MUTEX_LOCK(word); - word = MUTEX_SET_OWNER(word, me); - mutex->_word = word; - mutex->_pid = __pid; - return 0; - } else { - return EBUSY; - } - } -} - -/** - * Attempts acquiring lock. - * - * Unlike pthread_mutex_lock() this function won't block and instead - * returns an error immediately if the lock couldn't be acquired. - * - * @return 0 if lock was acquired, otherwise an errno - * @raise EAGAIN if maximum number of recursive locks is held - * @raise EBUSY if lock is currently held in read or write mode - * @raise EINVAL if `mutex` doesn't refer to an initialized lock - * @raise EDEADLK if `mutex` is `PTHREAD_MUTEX_ERRORCHECK` and the - * current thread already holds this mutex - */ -errno_t pthread_mutex_trylock(pthread_mutex_t *mutex) { - - // get current state of lock - uint64_t word = atomic_load_explicit(&mutex->_word, memory_order_relaxed); - -#if PTHREAD_USE_NSYNC - // use superior mutexes if possible - if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL && - MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE && // - _weaken(nsync_mu_trylock)) { - // on apple silicon we should just put our faith in ulock - // otherwise *nsync gets struck down by the eye of sauron - if (!IsXnuSilicon()) { - if (_weaken(nsync_mu_trylock)((nsync_mu *)mutex)) { - return 0; - } else { - return EBUSY; - } - } - } -#endif - - // handle normal mutexes - if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL) - return pthread_mutex_trylock_drepper(&mutex->_futex); - - // handle recursive and error checking mutexes -#if PTHREAD_USE_NSYNC - if (_weaken(nsync_mu_trylock) && - MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE) { - return pthread_mutex_trylock_recursive_nsync(mutex, word); - } else { - return pthread_mutex_trylock_recursive(mutex, word); - } -#else - return pthread_mutex_trylock_recursive(mutex, word); -#endif -} diff --git a/libc/intrin/pthread_mutex_unlock.c b/libc/intrin/pthread_mutex_unlock.c index a1a224a9c30..2a088beba5a 100644 --- a/libc/intrin/pthread_mutex_unlock.c +++ b/libc/intrin/pthread_mutex_unlock.c @@ -22,6 +22,8 @@ #include "libc/dce.h" #include "libc/errno.h" #include "libc/intrin/atomic.h" +#include "libc/intrin/describeflags.h" +#include "libc/intrin/kprintf.h" #include "libc/intrin/strace.h" #include "libc/intrin/weaken.h" #include "libc/runtime/internal.h" @@ -61,8 +63,11 @@ static errno_t pthread_mutex_unlock_recursive(pthread_mutex_t *mutex, // actually unlock the mutex if (atomic_compare_exchange_weak_explicit( &mutex->_word, &word, MUTEX_UNLOCK(word), memory_order_release, - memory_order_relaxed)) + memory_order_relaxed)) { + if (IsModeDbg()) + __deadlock_untrack(mutex); return 0; + } } } @@ -89,63 +94,85 @@ static errno_t pthread_mutex_unlock_recursive_nsync(pthread_mutex_t *mutex, // actually unlock the mutex mutex->_word = MUTEX_UNLOCK(word); - _weaken(nsync_mu_unlock)((nsync_mu *)mutex->_nsyncx); + _weaken(nsync_mu_unlock)((nsync_mu *)mutex->_nsync); + if (IsModeDbg()) + __deadlock_untrack(mutex); return 0; } } #endif -/** - * Releases mutex. - * - * This function does nothing in vfork() children. - * - * @return 0 on success or error number on failure - * @raises EPERM if in error check mode and not owned by caller - * @vforksafe - */ -errno_t pthread_mutex_unlock(pthread_mutex_t *mutex) { - uint64_t word; +static errno_t pthread_mutex_unlock_impl(pthread_mutex_t *mutex) { + uint64_t word = atomic_load_explicit(&mutex->_word, memory_order_relaxed); - if (__vforked) { - LOCKTRACE("skipping pthread_mutex_lock(%t) due to vfork", mutex); - return 0; + // check if mutex isn't held by calling thread + if (MUTEX_TYPE(word) == PTHREAD_MUTEX_ERRORCHECK || IsModeDbg()) { + if (__deadlock_tracked(mutex) == 0) { + if (IsModeDbg() && MUTEX_TYPE(word) != PTHREAD_MUTEX_ERRORCHECK) { + kprintf("error: unlock mutex not owned by calling thread: %t\n", mutex); + DebugBreak(); + } + return EPERM; + } } - LOCKTRACE("pthread_mutex_unlock(%t)", mutex); - - // get current state of lock - word = atomic_load_explicit(&mutex->_word, memory_order_relaxed); + // handle recursive mutexes + if (MUTEX_TYPE(word) == PTHREAD_MUTEX_RECURSIVE) { +#if PTHREAD_USE_NSYNC + if (_weaken(nsync_mu_unlock) && + MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE) { + return pthread_mutex_unlock_recursive_nsync(mutex, word); + } else { + return pthread_mutex_unlock_recursive(mutex, word); + } +#else + return pthread_mutex_unlock_recursive(mutex, word); +#endif + } #if PTHREAD_USE_NSYNC // use superior mutexes if possible - if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL && // - MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE && // + if (MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE && // _weaken(nsync_mu_unlock)) { // on apple silicon we should just put our faith in ulock // otherwise *nsync gets struck down by the eye of sauron if (!IsXnuSilicon()) { - _weaken(nsync_mu_unlock)((nsync_mu *)mutex); + _weaken(nsync_mu_unlock)((nsync_mu *)mutex->_nsync); + if (MUTEX_TYPE(word) == PTHREAD_MUTEX_ERRORCHECK || IsModeDbg()) + __deadlock_untrack(mutex); return 0; } } #endif // implement barebones normal mutexes - if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL) { - pthread_mutex_unlock_drepper(&mutex->_futex, MUTEX_PSHARED(word)); - return 0; - } + pthread_mutex_unlock_drepper(&mutex->_futex, MUTEX_PSHARED(word)); + if (MUTEX_TYPE(word) == PTHREAD_MUTEX_ERRORCHECK || IsModeDbg()) + __deadlock_untrack(mutex); + return 0; +} - // handle recursive and error checking mutexes -#if PTHREAD_USE_NSYNC - if (_weaken(nsync_mu_unlock) && - MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE) { - return pthread_mutex_unlock_recursive_nsync(mutex, word); +/** + * Releases mutex. + * + * POSIX.1 says it's undefined behavior to unlock a mutex that wasn't + * locked by the calling thread. Therefore, if `mutex` isn't locked, or + * it is locked and the thing that locked it was a different thread or + * process, then you should expect your program to deadlock or crash. + * + * This function does nothing in vfork() children. + * + * @return 0 on success or error number on failure + * @raises EPERM if mutex ownership isn't acceptable + * @vforksafe + */ +errno_t pthread_mutex_unlock(pthread_mutex_t *mutex) { + if (__tls_enabled && !__vforked) { + errno_t err = pthread_mutex_unlock_impl(mutex); + LOCKTRACE("pthread_mutex_unlock(%t) → %s", mutex, DescribeErrno(err)); + return err; } else { - return pthread_mutex_unlock_recursive(mutex, word); + LOCKTRACE("skipping pthread_mutex_lock(%t) due to runtime state", mutex); + return 0; } -#else - return pthread_mutex_unlock_recursive(mutex, word); -#endif } diff --git a/libc/intrin/pthread_mutex_wipe_np.c b/libc/intrin/pthread_mutex_wipe_np.c new file mode 100644 index 00000000000..0f0b5cb2608 --- /dev/null +++ b/libc/intrin/pthread_mutex_wipe_np.c @@ -0,0 +1,33 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2024 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/str/str.h" +#include "libc/thread/lock.h" +#include "libc/thread/thread.h" + +/** + * Unlocks mutex from child process after fork. + */ +int pthread_mutex_wipe_np(pthread_mutex_t *mutex) { + void *edges = mutex->_edges; + uint64_t word = mutex->_word; + bzero(mutex, sizeof(*mutex)); + mutex->_word = MUTEX_UNLOCK(word); + mutex->_edges = edges; + return 0; +} diff --git a/libc/intrin/pthread_mutexattr_gettype.c b/libc/intrin/pthread_mutexattr_gettype.c index 9b85dca0d5d..6e4caa149bd 100644 --- a/libc/intrin/pthread_mutexattr_gettype.c +++ b/libc/intrin/pthread_mutexattr_gettype.c @@ -23,6 +23,7 @@ * Gets mutex type. * * @param type will be set to one of these on success + * - `PTHREAD_MUTEX_DEFAULT` * - `PTHREAD_MUTEX_NORMAL` * - `PTHREAD_MUTEX_RECURSIVE` * - `PTHREAD_MUTEX_ERRORCHECK` diff --git a/libc/intrin/pthread_mutexattr_settype.c b/libc/intrin/pthread_mutexattr_settype.c index 70a421abe28..aefe262f438 100644 --- a/libc/intrin/pthread_mutexattr_settype.c +++ b/libc/intrin/pthread_mutexattr_settype.c @@ -24,6 +24,7 @@ * Sets mutex type. * * @param type can be one of + * - `PTHREAD_MUTEX_DEFAULT` * - `PTHREAD_MUTEX_NORMAL` * - `PTHREAD_MUTEX_RECURSIVE` * - `PTHREAD_MUTEX_ERRORCHECK` @@ -32,6 +33,7 @@ */ errno_t pthread_mutexattr_settype(pthread_mutexattr_t *attr, int type) { switch (type) { + case PTHREAD_MUTEX_DEFAULT: case PTHREAD_MUTEX_NORMAL: case PTHREAD_MUTEX_RECURSIVE: case PTHREAD_MUTEX_ERRORCHECK: diff --git a/libc/intrin/pthreadlock.c b/libc/intrin/pthreadlock.c index c7ef23ae498..92f784548c0 100644 --- a/libc/intrin/pthreadlock.c +++ b/libc/intrin/pthreadlock.c @@ -18,12 +18,12 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/thread/posixthread.internal.h" -pthread_mutex_t _pthread_lock_obj = PTHREAD_MUTEX_INITIALIZER; +pthread_mutex_t __pthread_lock_obj = PTHREAD_MUTEX_INITIALIZER; void _pthread_lock(void) { - pthread_mutex_lock(&_pthread_lock_obj); + pthread_mutex_lock(&__pthread_lock_obj); } void _pthread_unlock(void) { - pthread_mutex_unlock(&_pthread_lock_obj); + pthread_mutex_unlock(&__pthread_lock_obj); } diff --git a/libc/intrin/rand64.c b/libc/intrin/rand64.c index e6aa1fd2057..97b687a2d32 100644 --- a/libc/intrin/rand64.c +++ b/libc/intrin/rand64.c @@ -27,7 +27,7 @@ static int _rand64_pid; static unsigned __int128 _rand64_pool; -pthread_mutex_t _rand64_lock_obj = PTHREAD_SIGNAL_SAFE_MUTEX_INITIALIZER_NP; +pthread_mutex_t __rand64_lock_obj = PTHREAD_MUTEX_INITIALIZER; /** * Returns nondeterministic random data. @@ -38,12 +38,11 @@ pthread_mutex_t _rand64_lock_obj = PTHREAD_SIGNAL_SAFE_MUTEX_INITIALIZER_NP; * * @see rdseed(), rdrand(), rand(), random(), rngset() * @note this function passes bigcrush and practrand - * @asyncsignalsafe */ uint64_t _rand64(void) { void *p; uint128_t s; - pthread_mutex_lock(&_rand64_lock_obj); + pthread_mutex_lock(&__rand64_lock_obj); if (__pid == _rand64_pid) { s = _rand64_pool; // normal path } else { @@ -64,6 +63,6 @@ uint64_t _rand64(void) { _rand64_pid = __pid; } _rand64_pool = (s *= 15750249268501108917ull); // lemur64 - pthread_mutex_unlock(&_rand64_lock_obj); + pthread_mutex_unlock(&__rand64_lock_obj); return s >> 64; } diff --git a/libc/intrin/sig.c b/libc/intrin/sig.c index 4dd2f75e4b2..aecd085c9ac 100644 --- a/libc/intrin/sig.c +++ b/libc/intrin/sig.c @@ -73,6 +73,8 @@ struct SignalFrame { ucontext_t ctx; }; +extern pthread_mutex_t __sig_worker_lock; + static textwindows bool __sig_ignored_by_default(int sig) { return sig == SIGURG || // sig == SIGCONT || // @@ -667,9 +669,6 @@ textwindows int __sig_check(void) { return res; } -// this mutex is needed so execve() can shut down the signal worker -pthread_mutex_t __sig_worker_lock; - // background thread for delivering inter-process signals asynchronously // this checks for undelivered process-wide signals, once per scheduling // quantum, which on windows should be every ~15ms or so, unless somehow diff --git a/libc/intrin/sigblock.c b/libc/intrin/sigblock.c index 4fdd97914eb..b0fb34a429c 100644 --- a/libc/intrin/sigblock.c +++ b/libc/intrin/sigblock.c @@ -16,14 +16,20 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/sysv/consts/sig.h" #include "libc/calls/sig.internal.h" #include "libc/calls/struct/sigset.internal.h" #include "libc/dce.h" #include "libc/intrin/atomic.h" #include "libc/intrin/weaken.h" +#include "libc/sysv/consts/sig.h" #include "libc/thread/tls.h" +// since there's so many c library interfaces and system call wrappers +// that always need to block signals we avoid the distraction of their +// ftrace and strace output being muddied with sigprocmask lines. it's +// usually better that sigprocmask only strace the user is calling it. +// plus, since we have a very specific use case, this code goes faster + struct Signals __sig; sigset_t __sig_block(void) { diff --git a/libc/intrin/flushers.c b/libc/intrin/siglock.c similarity index 88% rename from libc/intrin/flushers.c rename to libc/intrin/siglock.c index 9ef0e057681..ab6045f4bd7 100644 --- a/libc/intrin/flushers.c +++ b/libc/intrin/siglock.c @@ -1,7 +1,7 @@ /*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ │ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │ ╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2021 Justine Alexandra Roberts Tunney │ +│ Copyright 2024 Justine Alexandra Roberts Tunney │ │ │ │ Permission to use, copy, modify, and/or distribute this software for │ │ any purpose with or without fee is hereby granted, provided that the │ @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/stdio/fflush.internal.h" +#include "libc/thread/thread.h" -pthread_mutex_t __fflush_lock_obj; -struct StdioFlush __fflush; +// this mutex is needed so execve() can shut down the signal worker +pthread_mutex_t __sig_worker_lock = PTHREAD_MUTEX_INITIALIZER; diff --git a/libc/intrin/sigprocmask-nt.c b/libc/intrin/sigprocmask-nt.c index 72ee8d79b4b..38246b43056 100644 --- a/libc/intrin/sigprocmask-nt.c +++ b/libc/intrin/sigprocmask-nt.c @@ -16,8 +16,6 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/assert.h" -#include "libc/atomic.h" #include "libc/calls/sig.internal.h" #include "libc/calls/struct/sigset.h" #include "libc/intrin/atomic.h" @@ -28,37 +26,25 @@ #ifdef __x86_64__ textwindows int __sig_mask(int how, const sigset_t *neu, sigset_t *old) { - - // validate api usage - if (how != SIG_BLOCK && how != SIG_UNBLOCK && how != SIG_SETMASK) { + if (how != SIG_BLOCK && how != SIG_UNBLOCK && how != SIG_SETMASK) return einval(); - } - - // get address of sigset to modify - atomic_ulong *mask = &__get_tls()->tib_sigmask; - - // handle read-only case sigset_t oldmask; + atomic_ulong *mask = &__get_tls()->tib_sigmask; if (neu) { if (how == SIG_BLOCK) { - oldmask = atomic_fetch_or_explicit(mask, *neu, memory_order_acq_rel); + oldmask = atomic_fetch_or(mask, *neu); } else if (how == SIG_UNBLOCK) { - oldmask = atomic_fetch_and_explicit(mask, ~*neu, memory_order_acq_rel); - } else { // SIG_SETMASK - oldmask = atomic_exchange_explicit(mask, *neu, memory_order_acq_rel); + oldmask = atomic_fetch_and(mask, ~*neu); + } else { + oldmask = atomic_exchange(mask, *neu); } - if (_weaken(__sig_check)) { + if (_weaken(__sig_check)) _weaken(__sig_check)(); - } } else { - oldmask = atomic_load_explicit(mask, memory_order_acquire); + oldmask = atomic_load(mask); } - - // return old signal mask to caller - if (old) { + if (old) *old = oldmask; - } - return 0; } diff --git a/libc/intrin/sigprocmask.c b/libc/intrin/sigprocmask.c index aa76966ab3b..bb1406624ff 100644 --- a/libc/intrin/sigprocmask.c +++ b/libc/intrin/sigprocmask.c @@ -16,18 +16,12 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/calls/calls.h" -#include "libc/calls/internal.h" #include "libc/calls/sig.internal.h" #include "libc/calls/struct/sigset.h" #include "libc/calls/struct/sigset.internal.h" #include "libc/dce.h" -#include "libc/fmt/itoa.h" #include "libc/intrin/describeflags.h" #include "libc/intrin/strace.h" -#include "libc/str/str.h" -#include "libc/sysv/consts/sig.h" -#include "libc/sysv/errfuns.h" /** * Changes signal blocking state of calling thread, e.g.: @@ -55,9 +49,8 @@ int sigprocmask(int how, const sigset_t *opt_set, sigset_t *opt_out_oldset) { } else { rc = sys_sigprocmask(how, opt_set, opt_out_oldset ? &old : 0); } - if (rc != -1 && opt_out_oldset) { + if (rc != -1 && opt_out_oldset) *opt_out_oldset = old; - } STRACE("sigprocmask(%s, %s, [%s]) → %d% m", DescribeHow(how), DescribeSigset(0, opt_set), DescribeSigset(rc, opt_out_oldset), rc); return rc; diff --git a/libc/stdio/g_rando.c b/libc/intrin/sigvar.c similarity index 93% rename from libc/stdio/g_rando.c rename to libc/intrin/sigvar.c index c702b2fd8d8..21c1d29454b 100644 --- a/libc/stdio/g_rando.c +++ b/libc/intrin/sigvar.c @@ -1,7 +1,7 @@ /*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ │ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │ ╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2023 Justine Alexandra Roberts Tunney │ +│ Copyright 2024 Justine Alexandra Roberts Tunney │ │ │ │ Permission to use, copy, modify, and/or distribute this software for │ │ any purpose with or without fee is hereby granted, provided that the │ @@ -16,6 +16,6 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/stdio/stdio.h" +#include "libc/calls/sig.internal.h" -uint64_t g_rando = 1; +struct Signals __sig; diff --git a/libc/intrin/stdio.c b/libc/intrin/stdio.c new file mode 100644 index 00000000000..9a6b75f2c06 --- /dev/null +++ b/libc/intrin/stdio.c @@ -0,0 +1,95 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2021 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/assert.h" +#include "libc/intrin/atomic.h" +#include "libc/intrin/kprintf.h" +#include "libc/intrin/weaken.h" +#include "libc/mem/mem.h" +#include "libc/stdio/internal.h" + +#define STDIO_FILE_USE_AFTER_FREE 1 +#define CORRUPT_STDIO_FILE_OBJECT 1 + +struct Stdio __stdio = { + .lock = PTHREAD_MUTEX_INITIALIZER, +}; + +void __stdio_lock(void) { + pthread_mutex_lock(&__stdio.lock); +} + +void __stdio_unlock(void) { + pthread_mutex_unlock(&__stdio.lock); +} + +static int refchk(int refs) { + unassert(refs != STDIO_FILE_USE_AFTER_FREE); + unassert(refs < CORRUPT_STDIO_FILE_OBJECT); + return refs; +} + +void __stdio_ref(FILE *f) { + refchk(atomic_fetch_sub_explicit(&f->refs, 1, memory_order_relaxed)); +} + +static void __stdio_unref_impl(FILE *f, bool should_lock) { + int refs = atomic_load_explicit(&f->refs, memory_order_relaxed); + for (;;) { + refchk(refs); + if (refs) { + if (atomic_compare_exchange_strong_explicit(&f->refs, &refs, refs + 1, + memory_order_acq_rel, + memory_order_relaxed)) + return; + continue; + } + if (should_lock) { + __stdio_lock(); + if ((refs = atomic_load_explicit(&f->refs, memory_order_relaxed))) { + __stdio_unlock(); + continue; + } + } + if (!atomic_compare_exchange_strong_explicit( + &f->refs, &refs, 1, memory_order_acq_rel, memory_order_relaxed)) { + if (should_lock) + __stdio_unlock(); + continue; + } + dll_remove(&__stdio.files, &f->elem); + if (should_lock) + __stdio_unlock(); + break; + } + if (_weaken(free)) { + _weaken(free)(f->getln); + if (f->freebuf) + _weaken(free)(f->buf); + if (f->freethis) + _weaken(free)(f); + } +} + +void __stdio_unref(FILE *f) { + __stdio_unref_impl(f, true); +} + +void __stdio_unref_unlocked(FILE *f) { + __stdio_unref_impl(f, false); +} diff --git a/libc/intrin/sys_gettid.greg.c b/libc/intrin/sys_gettid.greg.c index 408025bc03a..fbc4dadd0ee 100644 --- a/libc/intrin/sys_gettid.greg.c +++ b/libc/intrin/sys_gettid.greg.c @@ -25,7 +25,10 @@ __msabi extern typeof(GetCurrentThreadId) *const __imp_GetCurrentThreadId; -int sys_gettid(void) { +// it's important that this be noinstrument because the child process +// created by fork() needs to update this value quickly, since ftrace +// will deadlock __maps_lock() if the wrong tid is accidentally used. +dontinstrument int sys_gettid(void) { int64_t wut; #ifdef __x86_64__ int tid; diff --git a/libc/intrin/tls.c b/libc/intrin/tls.c new file mode 100644 index 00000000000..3a6d82db2d9 --- /dev/null +++ b/libc/intrin/tls.c @@ -0,0 +1,54 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2024 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/thread/tls.h" +#include "libc/dce.h" + +/** + * Returns location of thread information block. + * + * This should be favored over __get_tls() for .privileged code that + * can't be self-modified by __enable_tls(). + */ +privileged optimizespeed struct CosmoTib *__get_tls_privileged(void) { +#if defined(__x86_64__) + char *tib, *lin = (char *)0x30; + if (IsNetbsd() || IsOpenbsd()) { + asm("mov\t%%fs:(%1),%0" : "=a"(tib) : "r"(lin) : "memory"); + } else { + asm("mov\t%%gs:(%1),%0" : "=a"(tib) : "r"(lin) : "memory"); + if (IsWindows()) + tib = *(char **)(tib + 0x1480 + __tls_index * 8); + } + return (struct CosmoTib *)tib; +#elif defined(__aarch64__) + return __get_tls(); +#endif +} + +#if defined(__x86_64__) +privileged optimizespeed struct CosmoTib *__get_tls_win32(void) { + char *tib, *lin = (char *)0x30; + asm("mov\t%%gs:(%1),%0" : "=a"(tib) : "r"(lin) : "memory"); + tib = *(char **)(tib + 0x1480 + __tls_index * 8); + return (struct CosmoTib *)tib; +} +privileged void __set_tls_win32(void *tls) { + asm("mov\t%1,%%gs:%0" : "=m"(*((long *)0x1480 + __tls_index)) : "r"(tls)); +} +#endif diff --git a/libc/intrin/winerr.greg.c b/libc/intrin/winerr.greg.c index b960296a178..68abab78e50 100644 --- a/libc/intrin/winerr.greg.c +++ b/libc/intrin/winerr.greg.c @@ -24,7 +24,7 @@ #include "libc/nt/runtime.h" #include "libc/sock/internal.h" #include "libc/sysv/errfuns.h" -#include "libc/thread/tls2.internal.h" +#include "libc/thread/tls.h" /** * Return path for failed Win32 API calls. @@ -32,7 +32,7 @@ * @return -1 w/ few exceptions * @note this is a code-size saving device */ -privileged int64_t __winerr(void) { +privileged optimizesize int64_t __winerr(void) { errno_t e; if (IsWindows()) { e = __dos2errno(__imp_GetLastError()); diff --git a/libc/intrin/wintlsinit.c b/libc/intrin/wintlsinit.c index 599bffb1337..a678a0d2de8 100644 --- a/libc/intrin/wintlsinit.c +++ b/libc/intrin/wintlsinit.c @@ -21,7 +21,7 @@ #include "libc/nt/thunk/msabi.h" #include "libc/runtime/runtime.h" #include "libc/thread/tls.h" -#include "libc/thread/tls2.internal.h" +#ifdef __x86_64__ __msabi extern typeof(GetCurrentThreadId) *const __imp_GetCurrentThreadId; @@ -41,3 +41,5 @@ textwindows dontinstrument void __bootstrap_tls(struct CosmoTib *tib, tib->tib_tid = __imp_GetCurrentThreadId(); __set_tls_win32(tib); } + +#endif /* __x86_64__ */ diff --git a/libc/mem/leaks.c b/libc/mem/leaks.c index 3fa67773a4e..ba0da6edc20 100644 --- a/libc/mem/leaks.c +++ b/libc/mem/leaks.c @@ -16,16 +16,19 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/mem/leaks.h" #include "libc/cxxabi.h" #include "libc/intrin/cxaatexit.h" #include "libc/intrin/dll.h" #include "libc/intrin/kprintf.h" #include "libc/intrin/weaken.h" +#include "libc/macros.h" #include "libc/mem/mem.h" #include "libc/nt/typedef/imagetlscallback.h" #include "libc/runtime/runtime.h" #include "libc/thread/posixthread.internal.h" #include "libc/thread/thread.h" +#include "libc/thread/tls.h" #define LEAK_CONTAINER(e) DLL_CONTAINER(struct Leak, elem, e) @@ -87,8 +90,29 @@ void CheckForMemoryLeaks(void) { // check for leaks malloc_inspect_all(visitor, 0); if (leak_count) { - kprintf("you forgot to call free %'d time%s\n", leak_count, + kprintf(" you forgot to call free %'d time%s\n", leak_count, leak_count == 1 ? "" : "s"); _exit(73); } } + +static bool IsHoldingLocks(struct CosmoTib *tib) { + for (int i = 0; i < ARRAYLEN(tib->tib_locks); ++i) + if (tib->tib_locks[i]) + return true; + return false; +} + +/** + * Aborts if any locks are held by calling thread. + */ +void AssertNoLocksAreHeld(void) { + struct CosmoTib *tib = __get_tls(); + if (IsHoldingLocks(tib)) { + kprintf("error: the following locks are held by this thread:\n"); + for (int i = 0; i < ARRAYLEN(tib->tib_locks); ++i) + if (tib->tib_locks[i]) + kprintf("\t- %t\n", tib->tib_locks[i]); + _Exit(74); + } +} diff --git a/libc/mem/leaks.h b/libc/mem/leaks.h index dcf2ad464d5..f77c609d2a4 100644 --- a/libc/mem/leaks.h +++ b/libc/mem/leaks.h @@ -4,6 +4,7 @@ COSMOPOLITAN_C_START_ void CheckForMemoryLeaks(void) libcesque; +void AssertNoLocksAreHeld(void) libcesque; /** * Declares that allocation needn't be freed. diff --git a/libc/nexgen32e/gc.S b/libc/nexgen32e/gc.S index 4fb1ebff5b3..8dd47a41d0d 100644 --- a/libc/nexgen32e/gc.S +++ b/libc/nexgen32e/gc.S @@ -66,7 +66,7 @@ __gc: .ftrace2 // if this code fails // check if CosmoTib's size changed - sub x8,x28,#512 // __get_tls() + sub x8,x28,#1024 // __get_tls() ldr x9,[x8,0x18] // tib::garbages ldr x10,[x9] // g->i ldr x8,[x9,8] // g->p diff --git a/libc/proc/BUILD.mk b/libc/proc/BUILD.mk index 1ddefad2bf1..3e0e0c8941e 100644 --- a/libc/proc/BUILD.mk +++ b/libc/proc/BUILD.mk @@ -35,6 +35,8 @@ LIBC_PROC_A_DIRECTDEPS = \ LIBC_STR \ LIBC_SYSV \ LIBC_SYSV_CALLS \ + THIRD_PARTY_DLMALLOC \ + THIRD_PARTY_GDTOA \ THIRD_PARTY_NSYNC \ LIBC_PROC_A_DEPS := \ diff --git a/libc/proc/fork-nt.c b/libc/proc/fork-nt.c index ce5907a8add..20ca74f7ea4 100644 --- a/libc/proc/fork-nt.c +++ b/libc/proc/fork-nt.c @@ -65,7 +65,6 @@ #include "libc/thread/itimer.internal.h" #include "libc/thread/posixthread.internal.h" #include "libc/thread/tls.h" -#include "libc/thread/tls2.internal.h" #ifdef __x86_64__ extern long __klog_handle; diff --git a/libc/proc/fork.c b/libc/proc/fork.c index bd020151754..031ecef3163 100644 --- a/libc/proc/fork.c +++ b/libc/proc/fork.c @@ -16,90 +16,160 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/assert.h" -#include "libc/atomic.h" #include "libc/calls/calls.h" #include "libc/calls/state.internal.h" -#include "libc/calls/struct/sigset.h" #include "libc/calls/struct/sigset.internal.h" #include "libc/calls/struct/timespec.h" #include "libc/calls/syscall-nt.internal.h" #include "libc/calls/syscall-sysv.internal.h" #include "libc/dce.h" #include "libc/intrin/atomic.h" +#include "libc/intrin/cxaatexit.h" #include "libc/intrin/dll.h" -#include "libc/intrin/kprintf.h" #include "libc/intrin/maps.h" #include "libc/intrin/strace.h" #include "libc/intrin/weaken.h" #include "libc/nt/files.h" #include "libc/nt/process.h" #include "libc/nt/runtime.h" -#include "libc/nt/synchronization.h" #include "libc/nt/thread.h" +#include "libc/nt/thunk/msabi.h" #include "libc/proc/proc.internal.h" #include "libc/runtime/internal.h" -#include "libc/runtime/memtrack.internal.h" -#include "libc/runtime/runtime.h" #include "libc/runtime/syslib.internal.h" -#include "libc/sysv/consts/sig.h" +#include "libc/stdio/internal.h" +#include "libc/str/str.h" #include "libc/thread/posixthread.internal.h" -#include "libc/thread/tls.h" +#include "libc/thread/thread.h" +#include "third_party/dlmalloc/dlmalloc.h" +#include "third_party/gdtoa/lock.h" +#include "third_party/tz/lock.h" -__static_yoink("_pthread_atfork"); +__msabi extern typeof(GetCurrentProcessId) *const __imp_GetCurrentProcessId; -extern pthread_mutex_t _rand64_lock_obj; -extern pthread_mutex_t _pthread_lock_obj; +extern pthread_mutex_t __rand64_lock_obj; +extern pthread_mutex_t __pthread_lock_obj; +extern pthread_mutex_t __cxa_lock_obj; +extern pthread_mutex_t __sig_worker_lock; -// fork needs to lock every lock, which makes it very single-threaded in -// nature. the outermost lock matters the most because it serializes all -// threads attempting to spawn processes. the outer lock can't be a spin -// lock that a pthread_atfork() caller slipped in. to ensure it's a fair -// lock, we add an additional one of our own, which protects other locks -static pthread_mutex_t _fork_gil = PTHREAD_MUTEX_INITIALIZER; +void nsync_mu_semaphore_sem_fork_child(void); -static void _onfork_prepare(void) { +// first and last and always +// it is the lord of all locks +// subordinate to no other lock +static pthread_mutex_t supreme_lock = PTHREAD_MUTEX_INITIALIZER; + +static void fork_prepare_stdio(void) { + struct Dll *e; + // we acquire the following locks, in order + // + // 1. FILE objects created by the user + // 2. stdin, stdout, and stderr + // 3. __stdio.lock + // +StartOver: + __stdio_lock(); + for (e = dll_last(__stdio.files); e; e = dll_prev(__stdio.files, e)) { + FILE *f = FILE_CONTAINER(e); + if (f->forking) + continue; + f->forking = 1; + __stdio_ref(f); + __stdio_unlock(); + pthread_mutex_lock(&f->lock); + __stdio_unref(f); + goto StartOver; + } +} + +static void fork_parent_stdio(void) { + struct Dll *e; + for (e = dll_first(__stdio.files); e; e = dll_next(__stdio.files, e)) { + FILE_CONTAINER(e)->forking = 0; + pthread_mutex_unlock(&FILE_CONTAINER(e)->lock); + } + __stdio_unlock(); +} + +static void fork_child_stdio(void) { + struct Dll *e; + for (e = dll_first(__stdio.files); e; e = dll_next(__stdio.files, e)) { + pthread_mutex_wipe_np(&FILE_CONTAINER(e)->lock); + FILE_CONTAINER(e)->forking = 0; + } + pthread_mutex_wipe_np(&__stdio.lock); +} + +static void fork_prepare(void) { + pthread_mutex_lock(&supreme_lock); if (_weaken(_pthread_onfork_prepare)) _weaken(_pthread_onfork_prepare)(); - if (IsWindows()) + if (IsWindows()) { + pthread_mutex_lock(&__sig_worker_lock); __proc_lock(); + } + fork_prepare_stdio(); + __localtime_lock(); + __cxa_lock(); + __gdtoa_lock1(); + __gdtoa_lock(); _pthread_lock(); - __maps_lock(); + dlmalloc_pre_fork(); __fds_lock(); - pthread_mutex_lock(&_rand64_lock_obj); - LOCKTRACE("READY TO ROCK AND ROLL"); + pthread_mutex_lock(&__rand64_lock_obj); + __maps_lock(); + LOCKTRACE("READY TO LOCK AND ROLL"); } -static void _onfork_parent(void) { - pthread_mutex_unlock(&_rand64_lock_obj); - __fds_unlock(); +static void fork_parent(void) { __maps_unlock(); + pthread_mutex_unlock(&__rand64_lock_obj); + __fds_unlock(); + dlmalloc_post_fork_parent(); _pthread_unlock(); - if (IsWindows()) + __gdtoa_unlock(); + __gdtoa_unlock1(); + __cxa_unlock(); + __localtime_unlock(); + fork_parent_stdio(); + if (IsWindows()) { __proc_unlock(); + pthread_mutex_unlock(&__sig_worker_lock); + } if (_weaken(_pthread_onfork_parent)) _weaken(_pthread_onfork_parent)(); + pthread_mutex_unlock(&supreme_lock); } -static void _onfork_child(void) { - if (IsWindows()) +static void fork_child(void) { + nsync_mu_semaphore_sem_fork_child(); + pthread_mutex_wipe_np(&__rand64_lock_obj); + pthread_mutex_wipe_np(&__fds_lock_obj); + dlmalloc_post_fork_child(); + pthread_mutex_wipe_np(&__gdtoa_lock_obj); + pthread_mutex_wipe_np(&__gdtoa_lock1_obj); + fork_child_stdio(); + pthread_mutex_wipe_np(&__pthread_lock_obj); + pthread_mutex_wipe_np(&__cxa_lock_obj); + pthread_mutex_wipe_np(&__localtime_lock_obj); + if (IsWindows()) { __proc_wipe(); - __fds_lock_obj = (pthread_mutex_t)PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP; - _rand64_lock_obj = (pthread_mutex_t)PTHREAD_SIGNAL_SAFE_MUTEX_INITIALIZER_NP; - _pthread_lock_obj = (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER; - atomic_store_explicit(&__maps.lock, 0, memory_order_relaxed); + pthread_mutex_wipe_np(&__sig_worker_lock); + } if (_weaken(_pthread_onfork_child)) _weaken(_pthread_onfork_child)(); + pthread_mutex_wipe_np(&supreme_lock); } -static int _forker(uint32_t dwCreationFlags) { +int _fork(uint32_t dwCreationFlags) { long micros; struct Dll *e; struct timespec started; int ax, dx, tid, parent; parent = __pid; started = timespec_mono(); - _onfork_prepare(); + BLOCK_SIGNALS; + fork_prepare(); if (!IsWindows()) { ax = sys_fork(); } else { @@ -112,15 +182,27 @@ static int _forker(uint32_t dwCreationFlags) { if (!IsWindows()) { dx = sys_getpid().ax; } else { - dx = GetCurrentProcessId(); + dx = __imp_GetCurrentProcessId(); } __pid = dx; + // get new thread id + struct CosmoTib *tib = __get_tls(); + struct PosixThread *pt = (struct PosixThread *)tib->tib_pthread; + tid = IsLinux() || IsXnuSilicon() ? dx : sys_gettid(); + atomic_store_explicit(&tib->tib_tid, tid, memory_order_relaxed); + atomic_store_explicit(&pt->ptid, tid, memory_order_relaxed); + + // tracing and kisdangerous need this lock wiped a little earlier + atomic_store_explicit(&__maps.lock.word, 0, memory_order_relaxed); + + /* + * it's now safe to call normal functions again + */ + // turn other threads into zombies // we can't free() them since we're monopolizing all locks // we assume the operating system already reclaimed system handles - struct CosmoTib *tib = __get_tls(); - struct PosixThread *pt = (struct PosixThread *)tib->tib_pthread; dll_remove(&_pthread_list, &pt->list); for (e = dll_first(_pthread_list); e; e = dll_next(_pthread_list, e)) { atomic_store_explicit(&POSIXTHREAD_CONTAINER(e)->pt_status, @@ -130,11 +212,6 @@ static int _forker(uint32_t dwCreationFlags) { } dll_make_first(&_pthread_list, &pt->list); - // get new main thread id - tid = IsLinux() || IsXnuSilicon() ? dx : sys_gettid(); - atomic_store_explicit(&tib->tib_tid, tid, memory_order_relaxed); - atomic_store_explicit(&pt->ptid, tid, memory_order_relaxed); - // get new system thread handle intptr_t syshand = 0; if (IsXnuSilicon()) { @@ -149,29 +226,19 @@ static int _forker(uint32_t dwCreationFlags) { // we can't be canceled if the canceler no longer exists atomic_store_explicit(&pt->pt_canceled, false, memory_order_relaxed); + // forget locks + memset(tib->tib_locks, 0, sizeof(tib->tib_locks)); + // run user fork callbacks - _onfork_child(); + fork_child(); STRACE("fork() → 0 (child of %d; took %ld us)", parent, micros); } else { // this is the parent process - _onfork_parent(); + fork_parent(); STRACE("fork() → %d% m (took %ld us)", ax, micros); } - return ax; -} - -int _fork(uint32_t dwCreationFlags) { - int rc; - BLOCK_SIGNALS; - pthread_mutex_lock(&_fork_gil); - rc = _forker(dwCreationFlags); - if (!rc) { - pthread_mutex_init(&_fork_gil, 0); - } else { - pthread_mutex_unlock(&_fork_gil); - } ALLOW_SIGNALS; - return rc; + return ax; } /** diff --git a/libc/proc/posix_spawn.c b/libc/proc/posix_spawn.c index 9392ee54b30..3e653ff2292 100644 --- a/libc/proc/posix_spawn.c +++ b/libc/proc/posix_spawn.c @@ -612,7 +612,7 @@ errno_t posix_spawn(int *pid, const char *path, struct sigaction dfl = {0}; if (use_pipe) close(pfds[0]); - for (int sig = 1; sig < _NSIG; sig++) + for (int sig = 1; sig <= NSIG; sig++) if (__sighandrvas[sig] != (long)SIG_DFL && (__sighandrvas[sig] != (long)SIG_IGN || ((flags & POSIX_SPAWN_SETSIGDEF) && diff --git a/libc/proc/proc.c b/libc/proc/proc.c index df9fee0c1c3..97ba83c6929 100644 --- a/libc/proc/proc.c +++ b/libc/proc/proc.c @@ -54,6 +54,7 @@ #include "libc/sysv/consts/sicode.h" #include "libc/sysv/consts/sig.h" #include "libc/sysv/errfuns.h" +#include "libc/thread/thread.h" #include "libc/thread/tls.h" #include "third_party/nsync/mu.h" #ifdef __x86_64__ @@ -64,7 +65,9 @@ #define STACK_SIZE 65536 -struct Procs __proc; +struct Procs __proc = { + .lock = PTHREAD_MUTEX_INITIALIZER, +}; static textwindows void __proc_stats(int64_t h, struct rusage *ru) { bzero(ru, sizeof(*ru)); @@ -252,21 +255,24 @@ static textwindows void __proc_setup(void) { */ textwindows void __proc_lock(void) { cosmo_once(&__proc.once, __proc_setup); - nsync_mu_lock(&__proc.lock); + pthread_mutex_lock(&__proc.lock); } /** * Unlocks process tracker. */ textwindows void __proc_unlock(void) { - nsync_mu_unlock(&__proc.lock); + pthread_mutex_unlock(&__proc.lock); } /** * Resets process tracker from forked child. */ textwindows void __proc_wipe(void) { + pthread_mutex_t lock = __proc.lock; bzero(&__proc, sizeof(__proc)); + __proc.lock = lock; + pthread_mutex_wipe_np(&__proc.lock); } /** diff --git a/libc/proc/proc.internal.h b/libc/proc/proc.internal.h index fd59bc5f150..46ef01e8558 100644 --- a/libc/proc/proc.internal.h +++ b/libc/proc/proc.internal.h @@ -5,7 +5,6 @@ #include "libc/intrin/atomic.h" #include "libc/intrin/dll.h" #include "libc/thread/thread.h" -#include "third_party/nsync/mu.h" COSMOPOLITAN_C_START_ #define PROC_ALIVE 0 @@ -28,7 +27,7 @@ struct Proc { struct Procs { int waiters; atomic_uint once; - nsync_mu lock; + pthread_mutex_t lock; intptr_t thread; intptr_t onbirth; intptr_t haszombies; diff --git a/libc/proc/vfork.S b/libc/proc/vfork.S index e9b7911278f..3f87d74e1bb 100644 --- a/libc/proc/vfork.S +++ b/libc/proc/vfork.S @@ -121,7 +121,7 @@ vfork: // } else { // __get_tls()->tib_flags &= ~TIB_FLAG_VFORKED; // } - sub x1,x28,#512 // sizeof(CosmoTib) + sub x1,x28,#1024 // sizeof(CosmoTib) ldr x2,[x1,64] cbnz x0,2f orr x2,x2,#TIB_FLAG_VFORKED diff --git a/libc/runtime/at_quick_exit.c b/libc/runtime/at_quick_exit.c index 26c1c86afd9..c9aa389b938 100644 --- a/libc/runtime/at_quick_exit.c +++ b/libc/runtime/at_quick_exit.c @@ -16,35 +16,32 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/atomic.h" +#include "libc/intrin/cxaatexit.h" #include "libc/macros.h" -#include "libc/runtime/runtime.h" -#include "libc/thread/thread.h" +#include "libc/stdlib.h" static void (*funcs[32])(void); static int count; -static pthread_spinlock_t lock; -pthread_spinlock_t *const __at_quick_exit_lockptr = &lock; void __funcs_on_quick_exit(void) { void (*func)(void); - pthread_spin_lock(&lock); + __cxa_lock(); while (count) { func = funcs[--count]; - pthread_spin_unlock(&lock); + __cxa_unlock(); func(); - pthread_spin_lock(&lock); + __cxa_lock(); } } int at_quick_exit(void func(void)) { int res = 0; - pthread_spin_lock(&lock); + __cxa_lock(); if (count == ARRAYLEN(funcs)) { res = -1; } else { funcs[count++] = func; } - pthread_spin_unlock(&lock); + __cxa_unlock(); return res; } diff --git a/libc/runtime/clone.c b/libc/runtime/clone.c index d7cc911c3c0..e24782a3e48 100644 --- a/libc/runtime/clone.c +++ b/libc/runtime/clone.c @@ -58,7 +58,6 @@ #include "libc/thread/openbsd.internal.h" #include "libc/thread/thread.h" #include "libc/thread/tls.h" -#include "libc/thread/tls2.internal.h" #include "libc/thread/xnu.internal.h" #define kMaxThreadIds 32768 diff --git a/libc/runtime/ftraceinit.greg.c b/libc/runtime/ftraceinit.greg.c index f0f4a1e48cf..0f18fcf68e7 100644 --- a/libc/runtime/ftraceinit.greg.c +++ b/libc/runtime/ftraceinit.greg.c @@ -16,6 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/dce.h" #include "libc/runtime/internal.h" #include "libc/runtime/runtime.h" #include "libc/runtime/symbols.internal.h" @@ -37,7 +38,7 @@ __static_yoink("zipos"); * @see libc/runtime/_init.S for documentation */ textstartup int ftrace_init(void) { - if (strace_enabled(0) > 0) { + if (IsModeDbg() || strace_enabled(0) > 0) { GetSymbolTable(); } if (__intercept_flag(&__argc, __argv, "--ftrace")) { diff --git a/libc/runtime/ftracer.c b/libc/runtime/ftracer.c index ca09b1d5ac1..6317b0cf0d8 100644 --- a/libc/runtime/ftracer.c +++ b/libc/runtime/ftracer.c @@ -29,7 +29,6 @@ #include "libc/thread/posixthread.internal.h" #include "libc/thread/thread.h" #include "libc/thread/tls.h" -#include "libc/thread/tls2.internal.h" /** * @fileoverview Plain-text function call logging. diff --git a/libc/runtime/set_tls.c b/libc/runtime/set_tls.c index 0ed3609d022..c8385bacc8d 100644 --- a/libc/runtime/set_tls.c +++ b/libc/runtime/set_tls.c @@ -24,7 +24,6 @@ #include "libc/nt/thread.h" #include "libc/sysv/consts/arch.h" #include "libc/thread/tls.h" -#include "libc/thread/tls2.internal.h" #define AMD64_SET_FSBASE 129 #define AMD64_SET_GSBASE 131 diff --git a/libc/stdio/BUILD.mk b/libc/stdio/BUILD.mk index 069e5cf089d..c4d60fc7fdf 100644 --- a/libc/stdio/BUILD.mk +++ b/libc/stdio/BUILD.mk @@ -32,12 +32,13 @@ LIBC_STDIO_A_DIRECTDEPS = \ LIBC_NEXGEN32E \ LIBC_NT_ADVAPI32 \ LIBC_NT_KERNEL32 \ - LIBC_RUNTIME \ LIBC_PROC \ + LIBC_RUNTIME \ LIBC_STR \ LIBC_SYSV \ LIBC_SYSV_CALLS \ - THIRD_PARTY_GDTOA + THIRD_PARTY_DLMALLOC \ + THIRD_PARTY_GDTOA \ LIBC_STDIO_A_DEPS := \ $(call uniq,$(foreach x,$(LIBC_STDIO_A_DIRECTDEPS),$($(x)))) diff --git a/libc/stdio/alloc.c b/libc/stdio/alloc.c index ce348098d15..ace00fa2f94 100644 --- a/libc/stdio/alloc.c +++ b/libc/stdio/alloc.c @@ -22,20 +22,14 @@ FILE *__stdio_alloc(void) { FILE *f; + __stdio_lock(); if ((f = calloc(1, sizeof(FILE)))) { - pthread_mutexattr_t attr; - pthread_mutexattr_init(&attr); - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); - pthread_mutex_init(&f->lock, &attr); - pthread_mutexattr_destroy(&attr); - f->dynamic = 1; + f->freethis = 1; + f->fd = -1; + f->lock = (pthread_mutex_t)PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP; + dll_init(&f->elem); + dll_make_last(&__stdio.files, &f->elem); } + __stdio_unlock(); return f; } - -void __stdio_free(FILE *f) { - pthread_mutex_destroy(&f->lock); - if (f->dynamic) { - free(f); - } -} diff --git a/libc/stdio/fclose.c b/libc/stdio/fclose.c index b02c8bf20f1..2fcf0f7903d 100644 --- a/libc/stdio/fclose.c +++ b/libc/stdio/fclose.c @@ -16,47 +16,26 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/assert.h" #include "libc/calls/calls.h" #include "libc/errno.h" -#include "libc/intrin/weaken.h" -#include "libc/mem/mem.h" -#include "libc/runtime/runtime.h" #include "libc/stdio/internal.h" -#include "libc/stdio/stdio.h" /** * Closes standard i/o stream and its underlying thing. - * - * @param f is the file object - * @return 0 on success or -1 on error, which can be a trick for - * differentiating between EOF and real errors during previous - * i/o calls, without needing to call ferror() + * @return 0 on success, or EOF w/ errno */ int fclose(FILE *f) { - int rc; - if (!f) - return 0; - __fflush_unregister(f); - fflush(f); - if (_weaken(free)) { - _weaken(free)(f->getln); - if (!f->nofree && f->buf != f->mem) { - _weaken(free)(f->buf); - } - } - f->state = EOF; - if (f->noclose) { + int rc = 0; + if (f) { + flockfile(f); + rc |= fflush(f); + int fd = f->fd; f->fd = -1; - } else if (f->fd != -1 && close(f->fd) == -1) { - f->state = errno; - } - if (f->state == EOF) { - rc = 0; - } else { - errno = f->state; - rc = EOF; + f->state = EOF; + if (fd != -1) + rc |= close(fd); + funlockfile(f); + __stdio_unref(f); } - __stdio_free(f); return rc; } diff --git a/libc/stdio/fdopen.c b/libc/stdio/fdopen.c index eb4437a0af4..5f7191d07ad 100644 --- a/libc/stdio/fdopen.c +++ b/libc/stdio/fdopen.c @@ -16,14 +16,12 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/calls/calls.h" #include "libc/calls/struct/stat.h" +#include "libc/mem/mem.h" #include "libc/stdio/internal.h" -#include "libc/stdio/stdio.h" -#include "libc/sysv/consts/o.h" #include "libc/sysv/consts/s.h" -#include "libc/sysv/errfuns.h" -#include "libc/thread/thread.h" + +__static_yoink("fflush"); /** * Allocates stream object for already-opened file descriptor. @@ -38,16 +36,16 @@ FILE *fdopen(int fd, const char *mode) { struct stat st; if (fstat(fd, &st)) return 0; - if ((f = __stdio_alloc())) { - f->fd = fd; - f->bufmode = S_ISREG(st.st_mode) ? _IOFBF : _IONBF; - f->iomode = fopenflags(mode); - f->buf = f->mem; - f->size = BUFSIZ; - if ((f->iomode & O_ACCMODE) != O_RDONLY) { - __fflush_register(f); - } - return f; + if (!(f = __stdio_alloc())) + return 0; + f->bufmode = S_ISCHR(st.st_mode) ? _IONBF : _IOFBF; + f->oflags = fopenflags(mode); + f->size = BUFSIZ; + if (!(f->buf = malloc(f->size))) { + __stdio_unref(f); + return 0; } - return NULL; + f->freebuf = 1; + f->fd = fd; + return f; } diff --git a/libc/stdio/fflush.c b/libc/stdio/fflush.c index 4a9ef6c8edf..4a408d31342 100644 --- a/libc/stdio/fflush.c +++ b/libc/stdio/fflush.c @@ -16,20 +16,38 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/stdio/stdio.h" +#include "libc/cxxabi.h" +#include "libc/stdio/internal.h" /** * Blocks until data from stream buffer is written out. * * @param f is the stream handle, or 0 for all streams - * @return is 0 on success or -1 on error + * @return is 0 on success or EOF on error */ int fflush(FILE *f) { int rc; - if (f) + if (f) { flockfile(f); - rc = fflush_unlocked(f); - if (f) + rc = fflush_unlocked(f); funlockfile(f); + } else { + __stdio_lock(); + struct Dll *e, *e2; + for (rc = 0, e = dll_last(__stdio.files); e; e = e2) { + f = FILE_CONTAINER(e); + __stdio_ref(f); + __stdio_unlock(); + rc |= fflush(FILE_CONTAINER(e)); + __stdio_lock(); + e2 = dll_prev(__stdio.files, e); + __stdio_unref_unlocked(f); + } + __stdio_unlock(); + } return rc; } + +__attribute__((__constructor__(60))) static textstartup void fflush_init(void) { + __cxa_atexit((void *)fflush, 0, 0); +} diff --git a/libc/stdio/fflush.internal.h b/libc/stdio/fflush.internal.h deleted file mode 100644 index 75e3f3fc25c..00000000000 --- a/libc/stdio/fflush.internal.h +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef COSMOPOLITAN_LIBC_STDIO_FFLUSH_H_ -#define COSMOPOLITAN_LIBC_STDIO_FFLUSH_H_ -#include "libc/stdio/stdio.h" -#include "libc/thread/thread.h" -#include "libc/thread/tls.h" -COSMOPOLITAN_C_START_ - -struct StdioFlushHandles { - size_t i, n; - FILE **p; -}; - -struct StdioFlush { - struct StdioFlushHandles handles; - FILE *handles_initmem[8]; -}; - -extern struct StdioFlush __fflush; -extern pthread_mutex_t __fflush_lock_obj; - -void __fflush_lock(void); -void __fflush_unlock(void); - -COSMOPOLITAN_C_END_ -#endif /* COSMOPOLITAN_LIBC_STDIO_FFLUSH_H_ */ diff --git a/libc/stdio/fflush_unlocked.c b/libc/stdio/fflush_unlocked.c index 49099d7e72f..9532bf9d550 100644 --- a/libc/stdio/fflush_unlocked.c +++ b/libc/stdio/fflush_unlocked.c @@ -16,75 +16,46 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/cxxabi.h" -#include "libc/intrin/pushpop.h" -#include "libc/mem/arraylist.internal.h" -#include "libc/stdio/fflush.internal.h" +#include "libc/calls/calls.h" +#include "libc/errno.h" +#include "libc/intrin/weaken.h" +#include "libc/mem/mem.h" #include "libc/stdio/internal.h" +#include "libc/sysv/consts/o.h" /** * Blocks until data from stream buffer is written out. * - * @param f is the stream handle, or 0 for all streams - * @return is 0 on success or -1 on error + * @param f is the stream handle, which must not be null + * @return is 0 on success or EOF on error */ int fflush_unlocked(FILE *f) { - int rc = 0; size_t i; - if (!f) { - __fflush_lock(); - for (i = __fflush.handles.i; i; --i) { - if ((f = __fflush.handles.p[i - 1])) { - if (fflush(f) == -1) { - rc = -1; + if (f->getln) { + if (_weaken(free)) + _weaken(free)(f->getln); + f->getln = 0; + } + if (f->fd != -1) { + if (f->beg && !f->end && (f->oflags & O_ACCMODE) != O_RDONLY) { + ssize_t rc; + for (i = 0; i < f->beg; i += rc) { + if ((rc = write(f->fd, f->buf + i, f->beg - i)) == -1) { + f->state = errno; + return EOF; } } + f->beg = 0; } - __fflush_unlock(); - } else if (f->fd != -1) { - if (__fflush_impl(f) == -1) { - rc = -1; - } - } else if (f->beg && f->beg < f->size) { - f->buf[f->beg] = 0; - } - return rc; -} - -textstartup int __fflush_register(FILE *f) { - int rc; - size_t i; - struct StdioFlush *sf; - __fflush_lock(); - sf = &__fflush; - if (!sf->handles.p) { - sf->handles.p = sf->handles_initmem; - pushmov(&sf->handles.n, ARRAYLEN(sf->handles_initmem)); - __cxa_atexit((void *)fflush_unlocked, 0, 0); - } - for (i = sf->handles.i; i; --i) { - if (!sf->handles.p[i - 1]) { - sf->handles.p[i - 1] = f; - __fflush_unlock(); - return 0; - } - } - rc = append(&sf->handles, &f); - __fflush_unlock(); - return rc; -} - -void __fflush_unregister(FILE *f) { - size_t i; - struct StdioFlush *sf; - __fflush_lock(); - sf = &__fflush; - sf = pushpop(sf); - for (i = sf->handles.i; i; --i) { - if (sf->handles.p[i - 1] == f) { - pushmov(&sf->handles.p[i - 1], 0); - break; + if (f->beg < f->end && (f->oflags & O_ACCMODE) != O_WRONLY) { + if (lseek(f->fd, -(int)(f->end - f->beg), SEEK_CUR) == -1) { + f->state = errno; + return EOF; + } + f->end = f->beg; } } - __fflush_unlock(); + if (f->buf && f->beg && f->beg < f->size) + f->buf[f->beg] = 0; + return 0; } diff --git a/libc/stdio/flockfile.c b/libc/stdio/flockfile.c index 06bfe235945..61bac167bda 100644 --- a/libc/stdio/flockfile.c +++ b/libc/stdio/flockfile.c @@ -17,10 +17,7 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/assert.h" -#include "libc/stdio/fflush.internal.h" #include "libc/stdio/internal.h" -#include "libc/stdio/stdio.h" -#include "libc/str/str.h" #include "libc/thread/thread.h" /** @@ -30,39 +27,3 @@ void flockfile(FILE *f) { unassert(f != NULL); pthread_mutex_lock(&f->lock); } - -void(__fflush_lock)(void) { - pthread_mutex_lock(&__fflush_lock_obj); -} - -void(__fflush_unlock)(void) { - pthread_mutex_unlock(&__fflush_lock_obj); -} - -static void __stdio_fork_prepare(void) { - FILE *f; - __fflush_lock(); - for (int i = 0; i < __fflush.handles.i; ++i) - if ((f = __fflush.handles.p[i])) - pthread_mutex_lock(&f->lock); -} - -static void __stdio_fork_parent(void) { - FILE *f; - for (int i = __fflush.handles.i; i--;) - if ((f = __fflush.handles.p[i])) - pthread_mutex_unlock(&f->lock); - __fflush_unlock(); -} - -static void __stdio_fork_child(void) { - FILE *f; - for (int i = __fflush.handles.i; i--;) - if ((f = __fflush.handles.p[i])) - f->lock = (pthread_mutex_t)PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP; - pthread_mutex_init(&__fflush_lock_obj, 0); -} - -__attribute__((__constructor__(60))) static textstartup void stdioinit(void) { - pthread_atfork(__stdio_fork_prepare, __stdio_fork_parent, __stdio_fork_child); -} diff --git a/libc/stdio/flushlbf.c b/libc/stdio/flushlbf.c index 53a7d1a808e..860e093b170 100644 --- a/libc/stdio/flushlbf.c +++ b/libc/stdio/flushlbf.c @@ -17,7 +17,6 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/calls.h" -#include "libc/stdio/fflush.internal.h" #include "libc/stdio/internal.h" #include "libc/stdio/stdio.h" #include "libc/stdio/stdio_ext.h" @@ -26,17 +25,18 @@ * Flushes all line-buffered streams. */ void _flushlbf(void) { - int i; - FILE *f; - __fflush_lock(); - for (i = 0; i < __fflush.handles.i; ++i) { - if ((f = __fflush.handles.p[i])) { - flockfile(f); - if (f->bufmode == _IOLBF) { - fflush_unlocked(f); - } - funlockfile(f); + __stdio_lock(); + struct Dll *e, *e2; + for (e = dll_last(__stdio.files); e; e = e2) { + FILE *f = FILE_CONTAINER(e); + if (f->bufmode == _IOLBF) { + __stdio_ref(f); + __stdio_unlock(); + fflush(FILE_CONTAINER(e)); + __stdio_lock(); + e2 = dll_prev(__stdio.files, e); + __stdio_unref_unlocked(f); } } - __fflush_unlock(); + __stdio_unlock(); } diff --git a/libc/stdio/fmemopen.c b/libc/stdio/fmemopen.c index 21945de76cd..3834a7d1e44 100644 --- a/libc/stdio/fmemopen.c +++ b/libc/stdio/fmemopen.c @@ -37,36 +37,31 @@ FILE *fmemopen(void *buf, size_t size, const char *mode) { FILE *f; char *p; - int iomode; - iomode = fopenflags(mode); + int oflags; + oflags = fopenflags(mode); if ((size && size > 0x7ffff000) || // - (!buf && (iomode & O_ACCMODE) != O_RDWR)) { + (!buf && (oflags & O_ACCMODE) != O_RDWR)) { einval(); return NULL; } - if (!(f = __stdio_alloc())) { + if (!(f = __stdio_alloc())) return NULL; - } - if (buf) { - f->nofree = true; - } else { + if (!buf) { if (!size) size = BUFSIZ; - // TODO(jart): Why do we need calloc()? - if (!_weaken(calloc) || !(buf = _weaken(calloc)(1, size))) { - __stdio_free(f); + if (!(buf = malloc(size))) { + __stdio_unref(f); enomem(); return NULL; } + f->freebuf = 1; } - f->fd = -1; f->buf = buf; - if (!(iomode & O_TRUNC)) { + if (!(oflags & O_TRUNC)) f->end = size; - } f->size = size; - f->iomode = iomode; - if (iomode & O_APPEND) { + f->oflags = oflags; + if (oflags & O_APPEND) { if ((p = memchr(buf, '\0', size))) { f->beg = p - (char *)buf; } else { diff --git a/libc/stdio/fopen.c b/libc/stdio/fopen.c index 0294c5c2d54..d077f3c0135 100644 --- a/libc/stdio/fopen.c +++ b/libc/stdio/fopen.c @@ -17,36 +17,9 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/calls.h" -#include "libc/mem/mem.h" -#include "libc/stdio/internal.h" #include "libc/stdio/stdio.h" -#include "libc/str/str.h" -#include "libc/sysv/consts/o.h" -#include "libc/sysv/errfuns.h" -static const char *fixpathname(const char *pathname, int flags) { - if ((flags & O_ACCMODE) == O_RDONLY && strcmp(pathname, "-") == 0) { - return "/dev/stdin"; - } else if ((flags & O_ACCMODE) == O_WRONLY && strcmp(pathname, "-") == 0) { - return "/dev/stdout"; - } else { - return pathname; - } -} - -static int openpathname(const char *pathname, int flags, bool *out_noclose) { - if ((flags & O_ACCMODE) == O_RDONLY && strcmp(pathname, "/dev/stdin") == 0) { - *out_noclose = true; - return fileno(stdin); - } else if ((flags & O_ACCMODE) == O_WRONLY && - strcmp(pathname, "/dev/stdout") == 0) { - *out_noclose = true; - return fileno(stdout); - } else { - *out_noclose = false; - return open(pathname, flags, 0666); - } -} +__static_yoink("fflush"); /** * Opens file as stream object. @@ -57,21 +30,13 @@ static int openpathname(const char *pathname, int flags, bool *out_noclose) { * @note microsoft unilaterally deprecated this function lool */ FILE *fopen(const char *pathname, const char *mode) { - FILE *f = 0; - bool noclose; - int fd, flags; - if (!pathname) { - efault(); + int fd; + if ((fd = open(pathname, fopenflags(mode), 0666)) == -1) + return 0; + FILE *f; + if (!(f = fdopen(fd, mode))) { + close(fd); return 0; - } - flags = fopenflags(mode); - pathname = fixpathname(pathname, flags); - if ((fd = openpathname(pathname, flags, &noclose)) != -1) { - if ((f = fdopen(fd, mode)) != NULL) { - f->noclose = noclose; - } else if (!noclose) { - close(fd); - } } return f; } diff --git a/libc/stdio/fread_unlocked.c b/libc/stdio/fread_unlocked.c index ef341d8ecfd..98179bf528a 100644 --- a/libc/stdio/fread_unlocked.c +++ b/libc/stdio/fread_unlocked.c @@ -86,7 +86,7 @@ size_t fread_unlocked(void *buf, size_t stride, size_t count, FILE *f) { size_t n, m, got, need; // check state and parameters - if ((f->iomode & O_ACCMODE) == O_WRONLY) { + if ((f->oflags & O_ACCMODE) == O_WRONLY) { f->state = errno = EBADF; return 0; } diff --git a/libc/stdio/freadable.c b/libc/stdio/freadable.c index ff78a7a8429..8a623623acf 100644 --- a/libc/stdio/freadable.c +++ b/libc/stdio/freadable.c @@ -24,6 +24,6 @@ * Returns nonzero if stream allows reading. */ int __freadable(FILE *f) { - return (f->iomode & O_ACCMODE) == O_RDONLY || - (f->iomode & O_ACCMODE) == O_RDWR; + return (f->oflags & O_ACCMODE) == O_RDONLY || + (f->oflags & O_ACCMODE) == O_RDWR; } diff --git a/libc/stdio/freading.c b/libc/stdio/freading.c index 2e3782b3f77..0f447bf5b33 100644 --- a/libc/stdio/freading.c +++ b/libc/stdio/freading.c @@ -24,5 +24,5 @@ * Returns nonzero if stream is read only. */ int __freading(FILE *f) { - return (f->iomode & O_ACCMODE) == O_RDONLY; + return (f->oflags & O_ACCMODE) == O_RDONLY; } diff --git a/libc/stdio/freopen.c b/libc/stdio/freopen.c index 6c29021639b..c2db89e6084 100644 --- a/libc/stdio/freopen.c +++ b/libc/stdio/freopen.c @@ -51,7 +51,7 @@ FILE *freopen(const char *pathname, const char *mode, FILE *stream) { close(fd); if (fd2 != -1) { stream->fd = fd2; - stream->iomode = flags; + stream->oflags = flags; stream->beg = 0; stream->end = 0; res = stream; diff --git a/libc/stdio/fseek_unlocked.c b/libc/stdio/fseek_unlocked.c index f2acddc267c..b3fd2bbbecd 100644 --- a/libc/stdio/fseek_unlocked.c +++ b/libc/stdio/fseek_unlocked.c @@ -34,13 +34,13 @@ * @param f is a non-null stream handle * @param offset is the byte delta * @param whence can be SEET_SET, SEEK_CUR, or SEEK_END - * @returns 0 on success or -1 on error + * @returns 0 on success or -1 w/ errno */ int fseek_unlocked(FILE *f, int64_t offset, int whence) { int res; int64_t pos; if (f->fd != -1) { - if (__fflush_impl(f) == -1) + if (fflush_unlocked(f) == EOF) return -1; if (whence == SEEK_CUR && f->beg < f->end) { offset -= f->end - f->beg; diff --git a/libc/stdio/ftell.c b/libc/stdio/ftell.c index 7330e35d6ef..103a7d217d4 100644 --- a/libc/stdio/ftell.c +++ b/libc/stdio/ftell.c @@ -26,7 +26,7 @@ static inline int64_t ftell_unlocked(FILE *f) { int64_t pos; if (f->fd != -1) { - if (__fflush_impl(f) == -1) + if (fflush_unlocked(f) == EOF) return -1; if ((pos = lseek(f->fd, 0, SEEK_CUR)) != -1) { if (f->beg < f->end) diff --git a/libc/stdio/fwritable.c b/libc/stdio/fwritable.c index df10a0aeafd..ef1205bb8ce 100644 --- a/libc/stdio/fwritable.c +++ b/libc/stdio/fwritable.c @@ -24,6 +24,6 @@ * Returns nonzero if stream allows reading. */ int __fwritable(FILE *f) { - return (f->iomode & O_ACCMODE) == O_WRONLY || - (f->iomode & O_ACCMODE) == O_RDWR; + return (f->oflags & O_ACCMODE) == O_WRONLY || + (f->oflags & O_ACCMODE) == O_RDWR; } diff --git a/libc/stdio/fwrite_unlocked.c b/libc/stdio/fwrite_unlocked.c index 927f05c8c24..ed1cc7c3963 100644 --- a/libc/stdio/fwrite_unlocked.c +++ b/libc/stdio/fwrite_unlocked.c @@ -76,7 +76,7 @@ size_t fwrite_unlocked(const void *data, size_t stride, size_t count, FILE *f) { struct iovec iov[2]; if (!stride || !count) return 0; - if ((f->iomode & O_ACCMODE) == O_RDONLY) { + if ((f->oflags & O_ACCMODE) == O_RDONLY) { f->state = errno = EBADF; return 0; } diff --git a/libc/stdio/fwriting.c b/libc/stdio/fwriting.c index 8a4f012a102..8a755bcb2ab 100644 --- a/libc/stdio/fwriting.c +++ b/libc/stdio/fwriting.c @@ -24,5 +24,5 @@ * Returns nonzero if stream is write only. */ int __fwriting(FILE *f) { - return (f->iomode & O_ACCMODE) == O_WRONLY; + return (f->oflags & O_ACCMODE) == O_WRONLY; } diff --git a/libc/stdio/getdelim_unlocked.c b/libc/stdio/getdelim_unlocked.c index 03601709763..44a1f156bbb 100644 --- a/libc/stdio/getdelim_unlocked.c +++ b/libc/stdio/getdelim_unlocked.c @@ -32,7 +32,7 @@ ssize_t getdelim_unlocked(char **s, size_t *n, int delim, FILE *f) { ssize_t rc; char *p, *s2; size_t i, m, n2; - if ((f->iomode & O_ACCMODE) == O_WRONLY) { + if ((f->oflags & O_ACCMODE) == O_WRONLY) { f->state = errno = EBADF; return -1; } diff --git a/libc/stdio/internal.h b/libc/stdio/internal.h index e5f848f8052..2f4857a718f 100644 --- a/libc/stdio/internal.h +++ b/libc/stdio/internal.h @@ -1,39 +1,49 @@ #ifndef COSMOPOLITAN_LIBC_STDIO_INTERNAL_H_ #define COSMOPOLITAN_LIBC_STDIO_INTERNAL_H_ +#include "libc/atomic.h" +#include "libc/intrin/dll.h" #include "libc/stdio/stdio.h" #include "libc/thread/thread.h" #define PUSHBACK 12 +#define FILE_CONTAINER(e) DLL_CONTAINER(struct FILE, elem, e) + COSMOPOLITAN_C_START_ struct FILE { - uint8_t bufmode; /* _IOFBF, etc. (ignored if fd=-1) */ - char noclose; /* for fake dup() todo delete! */ - char dynamic; /* did malloc() create this object? */ - uint32_t iomode; /* O_RDONLY, etc. (ignored if fd=-1) */ - int32_t state; /* 0=OK, -1=EOF, >0=errno */ - int fd; /* ≥0=fd, -1=closed|buffer */ - uint32_t beg; - uint32_t end; - char *buf; - uint32_t size; - uint32_t nofree; + char bufmode; /* _IOFBF, _IOLBF, or _IONBF */ + char freethis; /* fclose() should free(this) */ + char freebuf; /* fclose() should free(this->buf) */ + char forking; /* used by fork() implementation */ + int oflags; /* O_RDONLY, etc. */ + int state; /* 0=OK, -1=EOF, >0=errno */ + int fd; /* ≥0=fd, -1=closed|buffer */ int pid; - char *getln; + atomic_int refs; + unsigned size; + unsigned beg; + unsigned end; + char *buf; pthread_mutex_t lock; - struct FILE *next; - char mem[BUFSIZ]; + struct Dll elem; + char *getln; +}; + +struct Stdio { + pthread_mutex_t lock; /* Subordinate to FILE::lock */ + struct Dll *files; }; -extern uint64_t g_rando; +extern struct Stdio __stdio; -int __fflush_impl(FILE *); -int __fflush_register(FILE *); -void __fflush_unregister(FILE *); +void __stdio_lock(void); +void __stdio_unlock(void); +void __stdio_ref(FILE *); +void __stdio_unref(FILE *); +void __stdio_unref_unlocked(FILE *); bool __stdio_isok(FILE *); FILE *__stdio_alloc(void); -void __stdio_free(FILE *); COSMOPOLITAN_C_END_ #endif /* COSMOPOLITAN_LIBC_STDIO_INTERNAL_H_ */ diff --git a/libc/stdio/rand.c b/libc/stdio/rand.c index 1802d99b270..1a5aad654c3 100644 --- a/libc/stdio/rand.c +++ b/libc/stdio/rand.c @@ -17,9 +17,17 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/stdio/rand.h" -#include "libc/stdio/internal.h" #include "libc/stdio/lcg.internal.h" +static uint64_t rando; + +/** + * Seeds random number generator that's used by rand(). + */ +void srand(unsigned seed) { + rando = seed; +} + /** * Returns 31-bit linear congruential pseudorandom number, e.g. * @@ -39,5 +47,5 @@ * @threadunsafe */ int rand(void) { - return KnuthLinearCongruentialGenerator(&g_rando) >> 33; + return KnuthLinearCongruentialGenerator(&rando) >> 33; } diff --git a/libc/stdio/setvbuf.c b/libc/stdio/setvbuf.c index 6be7ca74bd3..e3ef1d1fb88 100644 --- a/libc/stdio/setvbuf.c +++ b/libc/stdio/setvbuf.c @@ -38,15 +38,13 @@ int setvbuf(FILE *f, char *buf, int mode, size_t size) { if (buf) { if (!size) size = BUFSIZ; - if (!f->nofree && // - f->buf != buf && // - f->buf != f->mem && // - _weaken(free)) { - _weaken(free)(f->buf); - } + if (f->freebuf) + if (f->buf != buf) + if (_weaken(free)) + _weaken(free)(f->buf); f->buf = buf; f->size = size; - f->nofree = true; + f->freebuf = 0; } f->bufmode = mode; funlockfile(f); diff --git a/libc/stdio/stderr.c b/libc/stdio/stderr.c index de694ed6228..72af4c8284b 100644 --- a/libc/stdio/stderr.c +++ b/libc/stdio/stderr.c @@ -1,5 +1,5 @@ /*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ -│ vi: set et ft=c ts=8 sts=2 sw=2 fenc=utf-8 :vi │ +│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │ ╞══════════════════════════════════════════════════════════════════════════════╡ │ Copyright 2020 Justine Alexandra Roberts Tunney │ │ │ @@ -16,18 +16,17 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/intrin/dll.h" #include "libc/stdio/internal.h" #include "libc/sysv/consts/fileno.h" #include "libc/sysv/consts/o.h" -#include "libc/thread/thread.h" static FILE __stderr = { .fd = STDERR_FILENO, .bufmode = _IONBF, - .iomode = O_WRONLY, - .buf = __stderr.mem, - .size = sizeof(stderr->mem), + .oflags = O_WRONLY, .lock = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP, + .elem = {&__stderr.elem, &__stderr.elem}, }; /** @@ -35,6 +34,6 @@ static FILE __stderr = { */ FILE *stderr = &__stderr; -__attribute__((__constructor__(60))) static textstartup void errinit(void) { - __fflush_register(stderr); +__attribute__((__constructor__(60))) static textstartup void stderr_init(void) { + dll_make_last(&__stdio.files, &__stderr.elem); } diff --git a/libc/stdio/stdin.c b/libc/stdio/stdin.c index c5c3f6c2e09..8b1b44b9d6d 100644 --- a/libc/stdio/stdin.c +++ b/libc/stdio/stdin.c @@ -1,5 +1,5 @@ /*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ -│ vi: set et ft=c ts=8 sts=2 sw=2 fenc=utf-8 :vi │ +│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │ ╞══════════════════════════════════════════════════════════════════════════════╡ │ Copyright 2020 Justine Alexandra Roberts Tunney │ │ │ @@ -17,19 +17,25 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/struct/stat.h" +#include "libc/intrin/dll.h" #include "libc/stdio/internal.h" #include "libc/sysv/consts/fileno.h" #include "libc/sysv/consts/o.h" #include "libc/sysv/consts/s.h" #include "libc/thread/thread.h" +__static_yoink("fflush"); + +static char __stdin_buf[BUFSIZ]; + static FILE __stdin = { .fd = STDIN_FILENO, - .iomode = O_RDONLY, + .oflags = O_RDONLY, .bufmode = _IOFBF, - .buf = __stdin.mem, - .size = sizeof(stdin->mem), + .buf = __stdin_buf, + .size = sizeof(__stdin_buf), .lock = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP, + .elem = {&__stdin.elem, &__stdin.elem}, }; /** @@ -37,9 +43,9 @@ static FILE __stdin = { */ FILE *stdin = &__stdin; -__attribute__((__constructor__(60))) static textstartup void initin(void) { +__attribute__((__constructor__(60))) static textstartup void stdin_init(void) { struct stat st; - if (fstat(STDIN_FILENO, &st) || !S_ISREG(st.st_mode)) + if (fstat(STDIN_FILENO, &st) || S_ISCHR(st.st_mode)) stdin->bufmode = _IONBF; - __fflush_register(stdin); + dll_make_last(&__stdio.files, &__stdin.elem); } diff --git a/libc/stdio/stdout.c b/libc/stdio/stdout.c index 4c6b9b2d68b..86a34f9f3e6 100644 --- a/libc/stdio/stdout.c +++ b/libc/stdio/stdout.c @@ -1,5 +1,5 @@ /*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ -│ vi: set et ft=c ts=8 sts=2 sw=2 fenc=utf-8 :vi │ +│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │ ╞══════════════════════════════════════════════════════════════════════════════╡ │ Copyright 2020 Justine Alexandra Roberts Tunney │ │ │ @@ -16,17 +16,22 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/intrin/dll.h" #include "libc/stdio/internal.h" #include "libc/sysv/consts/fileno.h" #include "libc/sysv/consts/o.h" -#include "libc/thread/thread.h" + +__static_yoink("fflush"); + +static char __stdout_buf[BUFSIZ]; static FILE __stdout = { .fd = STDOUT_FILENO, - .iomode = O_WRONLY, - .buf = __stdout.mem, - .size = sizeof(stdout->mem), + .oflags = O_WRONLY, + .buf = __stdout_buf, + .size = sizeof(__stdout_buf), .lock = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP, + .elem = {&__stdout.elem, &__stdout.elem}, // Unlike other C libraries we don't bother calling fstat() to check // if stdio is a character device and we instead choose to always @@ -42,6 +47,6 @@ static FILE __stdout = { */ FILE *stdout = &__stdout; -__attribute__((__constructor__(60))) static textstartup void outinit(void) { - __fflush_register(stdout); +__attribute__((__constructor__(60))) static textstartup void stdout_init(void) { + dll_make_last(&__stdio.files, &__stdout.elem); } diff --git a/libc/stdio/pclose.c b/libc/system/pclose.c similarity index 100% rename from libc/stdio/pclose.c rename to libc/system/pclose.c diff --git a/libc/system/popen.c b/libc/system/popen.c index a7489d26126..b15b8adcafb 100644 --- a/libc/system/popen.c +++ b/libc/system/popen.c @@ -22,7 +22,6 @@ #include "libc/intrin/weaken.h" #include "libc/paths.h" #include "libc/runtime/runtime.h" -#include "libc/stdio/fflush.internal.h" #include "libc/stdio/internal.h" #include "libc/stdio/stdio.h" #include "libc/sysv/consts/f.h" @@ -54,7 +53,7 @@ * @cancelationpoint */ FILE *popen(const char *cmdline, const char *mode) { - FILE *f, *f2; + FILE *f; int e, rc, pid, dir, flags, pipefds[2]; flags = fopenflags(mode); if ((flags & O_ACCMODE) == O_RDONLY) { @@ -84,14 +83,21 @@ FILE *popen(const char *cmdline, const char *mode) { unassert(!close(pipefds[0])); if (pipefds[1] != !dir) unassert(!close(pipefds[1])); + // "The popen() function shall ensure that any streams from // previous popen() calls that remain open in the parent // process are closed in the new child process." -POSIX - for (int i = 0; i < __fflush.handles.i; ++i) { - if ((f2 = __fflush.handles.p[i]) && f2->pid) { + __stdio_lock(); + for (struct Dll *e = dll_first(__stdio.files); e; + e = dll_next(__stdio.files, e)) { + FILE *f2 = FILE_CONTAINER(e); + if (f != f2 && f2->pid && f2->fd != -1) { close(f2->fd); + f2->fd = -1; } } + __stdio_unlock(); + _Exit(_cocmd(3, (char *[]){ "popen", diff --git a/libc/sysv/errno.c b/libc/sysv/errno.c index 438ee9508b3..570f29d5b63 100644 --- a/libc/sysv/errno.c +++ b/libc/sysv/errno.c @@ -17,7 +17,7 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/errno.h" -#include "libc/thread/tls2.internal.h" +#include "libc/thread/tls.h" /** * Global variable for last error. diff --git a/libc/testlib/testmain.c b/libc/testlib/testmain.c index 55d34f80ff2..1b56570f1f6 100644 --- a/libc/testlib/testmain.c +++ b/libc/testlib/testmain.c @@ -163,6 +163,7 @@ int main(int argc, char *argv[]) { } // check for memory leaks + AssertNoLocksAreHeld(); if (!g_testlib_failed) CheckForMemoryLeaks(); diff --git a/libc/thread/itimer.c b/libc/thread/itimer.c index 15db1893d34..6df84c7e4d7 100644 --- a/libc/thread/itimer.c +++ b/libc/thread/itimer.c @@ -34,13 +34,16 @@ #include "libc/sysv/consts/sig.h" #include "libc/sysv/errfuns.h" #include "libc/thread/itimer.internal.h" +#include "libc/thread/thread2.h" #include "libc/thread/tls.h" -#include "third_party/nsync/mu.h" #ifdef __x86_64__ #define STACK_SIZE 65536 -struct IntervalTimer __itimer; +struct IntervalTimer __itimer = { + .lock = PTHREAD_MUTEX_INITIALIZER, + .cond = PTHREAD_COND_INITIALIZER, +}; static textwindows dontinstrument uint32_t __itimer_worker(void *arg) { struct CosmoTib tls; @@ -52,7 +55,7 @@ static textwindows dontinstrument uint32_t __itimer_worker(void *arg) { for (;;) { bool dosignal = false; struct timeval now, waituntil; - nsync_mu_lock(&__itimer.lock); + pthread_mutex_lock(&__itimer.lock); now = timeval_real(); if (timeval_iszero(__itimer.it.it_value)) { waituntil = timeval_max; @@ -73,13 +76,13 @@ static textwindows dontinstrument uint32_t __itimer_worker(void *arg) { dosignal = true; } } - nsync_mu_unlock(&__itimer.lock); + pthread_mutex_unlock(&__itimer.lock); if (dosignal) __sig_generate(SIGALRM, SI_TIMER); - nsync_mu_lock(&__itimer.lock); - nsync_cv_wait_with_deadline(&__itimer.cond, &__itimer.lock, CLOCK_REALTIME, - timeval_totimespec(waituntil), 0); - nsync_mu_unlock(&__itimer.lock); + pthread_mutex_lock(&__itimer.lock); + struct timespec deadline = timeval_totimespec(waituntil); + pthread_cond_timedwait(&__itimer.cond, &__itimer.lock, &deadline); + pthread_mutex_unlock(&__itimer.lock); } return 0; } @@ -109,7 +112,7 @@ textwindows int sys_setitimer_nt(int which, const struct itimerval *neu, config = *neu; } BLOCK_SIGNALS; - nsync_mu_lock(&__itimer.lock); + pthread_mutex_lock(&__itimer.lock); if (old) { old->it_interval = __itimer.it.it_interval; old->it_value = timeval_subz(__itimer.it.it_value, timeval_real()); @@ -119,9 +122,9 @@ textwindows int sys_setitimer_nt(int which, const struct itimerval *neu, config.it_value = timeval_add(config.it_value, timeval_real()); } __itimer.it = config; - nsync_cv_signal(&__itimer.cond); + pthread_cond_signal(&__itimer.cond); } - nsync_mu_unlock(&__itimer.lock); + pthread_mutex_unlock(&__itimer.lock); ALLOW_SIGNALS; return 0; } diff --git a/libc/thread/itimer.internal.h b/libc/thread/itimer.internal.h index 41d72121645..204c3bf8d81 100644 --- a/libc/thread/itimer.internal.h +++ b/libc/thread/itimer.internal.h @@ -2,15 +2,14 @@ #define COSMOPOLITAN_LIBC_ITIMER_H_ #include "libc/atomic.h" #include "libc/calls/struct/itimerval.h" -#include "third_party/nsync/cv.h" -#include "third_party/nsync/mu.h" +#include "libc/thread/thread.h" COSMOPOLITAN_C_START_ struct IntervalTimer { atomic_uint once; intptr_t thread; - nsync_mu lock; - nsync_cv cond; + pthread_mutex_t lock; + pthread_cond_t cond; struct itimerval it; }; diff --git a/libc/thread/lock.h b/libc/thread/lock.h index 2947da75fb0..75b0177a2de 100644 --- a/libc/thread/lock.h +++ b/libc/thread/lock.h @@ -3,18 +3,25 @@ COSMOPOLITAN_C_START_ // -// ┌depth -// │ -// COSMOPOLITAN MUTEXES │ ┌waited -// │ │ -// │ │┌locked -// │ ││ -// │ ││┌pshared -// owner │ │││ -// tid │ │││┌type -// │ │ ││││ -// ┌──────────────┴───────────────┐ ┌─┴──┐│││├┐ +// ┌undead +// │ +// │┌dead +// ││ +// ││┌robust +// │││ +// │││ ┌depth +// │││ │ +// COSMOPOLITAN MUTEXES │││ │ ┌waited +// │││ │ │ +// │││ │ │┌locked +// │││ │ ││ +// │││ │ ││┌pshared +// owner │││ │ │││ +// tid │││ │ │││┌type +// │ │││ │ ││││ +// ┌──────────────┴───────────────┐ │││┌─┴──┐│││├┐ // 0b0000000000000000000000000000000000000000000000000000000000000000 +// #define MUTEX_DEPTH_MIN 0x00000020ull #define MUTEX_DEPTH_MAX 0x000007e0ull diff --git a/libc/thread/posixthread.internal.h b/libc/thread/posixthread.internal.h index 829217fa29b..41d268ed138 100644 --- a/libc/thread/posixthread.internal.h +++ b/libc/thread/posixthread.internal.h @@ -98,7 +98,6 @@ extern struct Dll *_pthread_list; extern struct PosixThread _pthread_static; extern _Atomic(pthread_key_dtor) _pthread_key_dtor[PTHREAD_KEYS_MAX]; -int _pthread_atfork(atfork_f, atfork_f, atfork_f) libcesque; int _pthread_reschedule(struct PosixThread *) libcesque; int _pthread_setschedparam_freebsd(int, int, const struct sched_param *); int _pthread_tid(struct PosixThread *) libcesque; diff --git a/libc/thread/pthread_atfork.c b/libc/thread/pthread_atfork.c new file mode 100644 index 00000000000..668e221f347 --- /dev/null +++ b/libc/thread/pthread_atfork.c @@ -0,0 +1,179 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/errno.h" +#include "libc/intrin/strace.h" +#include "libc/mem/mem.h" +#include "libc/thread/posixthread.internal.h" +#include "libc/thread/thread.h" + +struct AtFork { + struct AtFork *p[2]; + atfork_f f[3]; +}; + +struct AtForks { + pthread_once_t once; + pthread_mutex_t lock; + struct AtFork *list; +}; + +static struct AtForks _atforks = { + .once = PTHREAD_ONCE_INIT, + .lock = PTHREAD_MUTEX_INITIALIZER, +}; + +static void pthread_atfork_clear(void) { + struct AtFork *a, *b; + for (a = _atforks.list; a; a = b) { + b = a->p[0]; + free(a); + } +} + +static void pthread_atfork_init(void) { + atexit(pthread_atfork_clear); +} + +static void _pthread_onfork(int i, const char *op) { + struct AtFork *a; + for (a = _atforks.list; a; a = a->p[!i]) { + if (a->f[i]) { + STRACE("pthread_atfork(%s, %t)", op, a->f[i]); + a->f[i](); + } + _atforks.list = a; + } +} + +void _pthread_onfork_prepare(void) { + _pthread_onfork(0, "prepare"); +} + +void _pthread_onfork_parent(void) { + _pthread_onfork(1, "parent"); +} + +void _pthread_onfork_child(void) { + pthread_mutex_wipe_np(&_atforks.lock); + _pthread_onfork(2, "child"); +} + +/** + * Registers fork callbacks. + * + * When fork happens, your prepare functions will be called in the + * reverse order they were registered. Then, in the parent and child + * processes, their callbacks will be called in the same order they were + * registered. + * + * One big caveat with fork() is that it hard kills all threads except + * the calling thread. So let's say one of those threads was printing to + * stdout while it was killed. In that case, the stdout lock will still + * be held when your child process comes alive, which means that the + * child will deadlock next time it tries to print. + * + * The solution for that is simple. Every lock in your process should be + * registered with this interface. However there's one highly important + * thing you need to know. Locks must follow a consistent hierarchy. So + * the order in which you register locks matters. If nested locks aren't + * acquired in the same order globally, then rarely occurring deadlocks + * will happen. So what we recommend is that you hunt down all the locks + * that exist in your app and its dependencies, and register them all at + * once from your main() function at startup. This ensures a clear order + * and if you aren't sure what that order should be, cosmo libc has got + * you covered. Simply link your program with the `cosmocc -mdbg` flag + * and cosmo will detect locking violations with your `pthread_mutex_t` + * objects and report them by printing the strongly connected component. + * This will include the demangled symbol name of each mutex, assuming + * the `pthread_mutex_t` objects are stored in static memory. cosmo.h + * also exposes a deadlock API that lets you incorporate your own lock + * object types into this error checking system, which we also use to + * verify the entire libc runtime itself. See libc/intrin/deadlock.c. + * + * Special care should be taken when using this interface in libraries. + * While it may seem tempting to use something like a `__constructor__` + * attribute to register your mutexes in a clean and abstracted way, it + * is only appropriate if your mutex is guarding pure memory operations + * and poses zero risk of nesting with locks outside your library. For + * example, calling open() or printf() while holding your lock will do + * just that, since the C runtime functions you may consider pure will + * actually use mutexes under the hood, which are also validated under + * `cosmocc -mdbg` builds. So if your locks can't be made unnestable + * pure memory operations, then you should consider revealing their + * existence to users of your library. + * + * Here's an example of how pthread_atfork() can be used: + * + * static struct { + * pthread_once_t once; + * pthread_mutex_t lock; + * // data structures... + * } g_lib; + * + * static void lib_lock(void) { + * pthread_mutex_lock(&g_lib.lock); + * } + * + * static void lib_unlock(void) { + * pthread_mutex_unlock(&g_lib.lock); + * } + * + * static void lib_wipe(void) { + * pthread_mutex_wipe_np(&g_lib.lock); + * } + * + * static void lib_setup(void) { + * pthread_mutex_init(&g_lib.lock, 0); + * pthread_atfork(lib_lock, lib_unlock, lib_wipe); + * } + * + * static void lib_init(void) { + * pthread_once(&g_lib.once, lib_setup); + * } + * + * void lib(void) { + * lib_init(); + * lib_lock(); + * // do stuff... + * lib_unlock(); + * } + * + * @param prepare is run by fork() before forking happens + * @param parent is run by fork() after forking happens in parent process + * @param child is run by fork() after forking happens in childe process + * @return 0 on success, or errno on error + * @raise ENOMEM if we require more vespene gas + */ +int pthread_atfork(atfork_f prepare, atfork_f parent, atfork_f child) { + pthread_once(&_atforks.once, pthread_atfork_init); + struct AtFork *a; + if (!(a = calloc(1, sizeof(struct AtFork)))) + return ENOMEM; + a->f[0] = prepare; + a->f[1] = parent; + a->f[2] = child; + pthread_mutex_lock(&_atforks.lock); + a->p[0] = 0; + a->p[1] = _atforks.list; + if (_atforks.list) + _atforks.list->p[0] = a; + _atforks.list = a; + pthread_mutex_unlock(&_atforks.lock); + return 0; +} diff --git a/libc/thread/pthread_cond_broadcast.c b/libc/thread/pthread_cond_broadcast.c index f50d5b3ea3c..894a76fb411 100644 --- a/libc/thread/pthread_cond_broadcast.c +++ b/libc/thread/pthread_cond_broadcast.c @@ -55,7 +55,7 @@ errno_t pthread_cond_broadcast(pthread_cond_t *cond) { // favor *NSYNC if this is a process private condition variable // if using Mike Burrows' code isn't possible, use a naive impl if (!cond->_footek) { - nsync_cv_broadcast((nsync_cv *)cond); + nsync_cv_broadcast((nsync_cv *)cond->_nsync); return 0; } #endif diff --git a/libc/thread/pthread_cond_destroy.c b/libc/thread/pthread_cond_destroy.c index c5a180e4a3d..bb0783671b8 100644 --- a/libc/thread/pthread_cond_destroy.c +++ b/libc/thread/pthread_cond_destroy.c @@ -33,7 +33,7 @@ errno_t pthread_cond_destroy(pthread_cond_t *cond) { // check if there's active waiters #if PTHREAD_USE_NSYNC if (!cond->_pshared) { - if (((nsync_cv *)cond)->waiters) + if (((nsync_cv *)cond->_nsync)->waiters) return EINVAL; } else { if (atomic_load_explicit(&cond->_waiters, memory_order_relaxed)) diff --git a/libc/thread/pthread_cond_signal.c b/libc/thread/pthread_cond_signal.c index b85522ad4a9..df0de5bb425 100644 --- a/libc/thread/pthread_cond_signal.c +++ b/libc/thread/pthread_cond_signal.c @@ -54,7 +54,7 @@ errno_t pthread_cond_signal(pthread_cond_t *cond) { // favor *NSYNC if this is a process private condition variable // if using Mike Burrows' code isn't possible, use a naive impl if (!cond->_footek) { - nsync_cv_signal((nsync_cv *)cond); + nsync_cv_signal((nsync_cv *)cond->_nsync); return 0; } #endif diff --git a/libc/thread/pthread_cond_timedwait.c b/libc/thread/pthread_cond_timedwait.c index 55ab6038c7a..cc39e5f3fbf 100644 --- a/libc/thread/pthread_cond_timedwait.c +++ b/libc/thread/pthread_cond_timedwait.c @@ -43,7 +43,7 @@ struct PthreadWait { static bool can_use_nsync(uint64_t muword) { return !IsXnuSilicon() && // - MUTEX_TYPE(muword) == PTHREAD_MUTEX_NORMAL && + MUTEX_TYPE(muword) != PTHREAD_MUTEX_RECURSIVE && MUTEX_PSHARED(muword) == PTHREAD_PROCESS_PRIVATE; } @@ -124,9 +124,9 @@ errno_t pthread_cond_timedwait(pthread_cond_t *cond, pthread_mutex_t *mutex, uint64_t muword = atomic_load_explicit(&mutex->_word, memory_order_relaxed); // check that mutex is held by caller - if (MUTEX_TYPE(muword) == PTHREAD_MUTEX_ERRORCHECK && - MUTEX_OWNER(muword) != gettid()) - return EPERM; + if (IsModeDbg() || MUTEX_TYPE(muword) == PTHREAD_MUTEX_ERRORCHECK) + if (__deadlock_tracked(mutex) == 0) + return EPERM; // if the cond is process shared then the mutex needs to be too if ((cond->_pshared == PTHREAD_PROCESS_SHARED) ^ @@ -154,7 +154,7 @@ errno_t pthread_cond_timedwait(pthread_cond_t *cond, pthread_mutex_t *mutex, // if using Mike Burrows' code isn't possible, use a naive impl if (!cond->_footek) { err = nsync_cv_wait_with_deadline( - (nsync_cv *)cond, (nsync_mu *)mutex, cond->_clock, + (nsync_cv *)cond->_nsync, (nsync_mu *)mutex->_nsync, cond->_clock, abstime ? *abstime : nsync_time_no_deadline, 0); } else { err = pthread_cond_timedwait_impl(cond, mutex, abstime); diff --git a/libc/thread/pthread_create.c b/libc/thread/pthread_create.c index ec19ee9a758..02289027600 100644 --- a/libc/thread/pthread_create.c +++ b/libc/thread/pthread_create.c @@ -61,7 +61,6 @@ __static_yoink("nsync_mu_unlock"); __static_yoink("nsync_mu_trylock"); __static_yoink("nsync_mu_rlock"); __static_yoink("nsync_mu_runlock"); -__static_yoink("_pthread_atfork"); __static_yoink("_pthread_onfork_prepare"); __static_yoink("_pthread_onfork_parent"); __static_yoink("_pthread_onfork_child"); diff --git a/libc/thread/thread.h b/libc/thread/thread.h index 3ff51f6c629..cda6ae38b51 100644 --- a/libc/thread/thread.h +++ b/libc/thread/thread.h @@ -8,11 +8,13 @@ #define PTHREAD_BARRIER_SERIAL_THREAD 31337 -#define PTHREAD_MUTEX_NORMAL 0 -#define PTHREAD_MUTEX_RECURSIVE 1 -#define PTHREAD_MUTEX_ERRORCHECK 2 -#define PTHREAD_MUTEX_STALLED 0 -#define PTHREAD_MUTEX_ROBUST 1 +#define PTHREAD_MUTEX_DEFAULT 0 +#define PTHREAD_MUTEX_NORMAL 1 +#define PTHREAD_MUTEX_RECURSIVE 2 +#define PTHREAD_MUTEX_ERRORCHECK 3 + +#define PTHREAD_MUTEX_STALLED 0 +#define PTHREAD_MUTEX_ROBUST 2048 #define PTHREAD_PROCESS_PRIVATE 0 #define PTHREAD_PROCESS_SHARED 4 @@ -43,12 +45,14 @@ COSMOPOLITAN_C_START_ #define PTHREAD_ONCE_INIT {0} #define PTHREAD_COND_INITIALIZER {0} #define PTHREAD_RWLOCK_INITIALIZER {0} -#define PTHREAD_MUTEX_INITIALIZER {0} -#define PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP {0, {}, PTHREAD_MUTEX_RECURSIVE} +#define PTHREAD_MUTEX_INITIALIZER {0, PTHREAD_MUTEX_DEFAULT} +#define PTHREAD_NORMAL_MUTEX_INITIALIZER_NP {0, PTHREAD_MUTEX_NORMAL} +#define PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP {0, PTHREAD_MUTEX_RECURSIVE} +#define PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP {0, PTHREAD_MUTEX_ERRORCHECK} #define PTHREAD_SIGNAL_SAFE_MUTEX_INITIALIZER_NP \ - {0, {}, PTHREAD_MUTEX_RECURSIVE | PTHREAD_PROCESS_SHARED} + {0, PTHREAD_MUTEX_RECURSIVE | PTHREAD_PROCESS_SHARED} #ifndef __cplusplus #define _PTHREAD_ATOMIC(x) _Atomic(x) @@ -72,14 +76,11 @@ typedef struct pthread_spinlock_s { } pthread_spinlock_t; typedef struct pthread_mutex_s { - uint32_t _nsync; - union { - int32_t _pid; - _PTHREAD_ATOMIC(int32_t) _futex; - }; - /* this cleverly overlaps with NSYNC struct Dll *waiters; */ + void *_edges; _PTHREAD_ATOMIC(uint64_t) _word; - long _nsyncx[2]; + _PTHREAD_ATOMIC(int) _futex; + int _pid; + void *_nsync[2]; } pthread_mutex_t; typedef struct pthread_mutexattr_s { @@ -92,18 +93,13 @@ typedef struct pthread_condattr_s { } pthread_condattr_t; typedef struct pthread_cond_s { - union { - void *_align; - struct { - uint32_t _nsync; - char _pshared; - char _clock; - char _footek; - _PTHREAD_ATOMIC(char) _waited; - }; - }; + char _pshared; + char _clock; + char _footek; + _PTHREAD_ATOMIC(char) _waited; _PTHREAD_ATOMIC(uint32_t) _sequence; _PTHREAD_ATOMIC(uint32_t) _waiters; + void *_nsync[2]; } pthread_cond_t; typedef struct pthread_rwlock_s { @@ -156,20 +152,20 @@ int pthread_attr_getguardsize(const pthread_attr_t *, size_t *) libcesque params int pthread_attr_getinheritsched(const pthread_attr_t *, int *) libcesque paramsnonnull(); int pthread_attr_getschedpolicy(const pthread_attr_t *, int *) libcesque paramsnonnull(); int pthread_attr_getscope(const pthread_attr_t *, int *) libcesque paramsnonnull(); -int pthread_attr_getstack(const pthread_attr_t *, void **, size_t *) libcesque paramsnonnull(); -int pthread_attr_getstacksize(const pthread_attr_t *, size_t *) libcesque paramsnonnull(); int pthread_attr_getsigaltstack_np(const pthread_attr_t *, void **, size_t *) libcesque paramsnonnull(); int pthread_attr_getsigaltstacksize_np(const pthread_attr_t *, size_t *) libcesque paramsnonnull(); +int pthread_attr_getstack(const pthread_attr_t *, void **, size_t *) libcesque paramsnonnull(); +int pthread_attr_getstacksize(const pthread_attr_t *, size_t *) libcesque paramsnonnull(); int pthread_attr_init(pthread_attr_t *) libcesque paramsnonnull(); int pthread_attr_setdetachstate(pthread_attr_t *, int) libcesque paramsnonnull(); int pthread_attr_setguardsize(pthread_attr_t *, size_t) libcesque paramsnonnull(); int pthread_attr_setinheritsched(pthread_attr_t *, int) libcesque paramsnonnull(); int pthread_attr_setschedpolicy(pthread_attr_t *, int) libcesque paramsnonnull(); int pthread_attr_setscope(pthread_attr_t *, int) libcesque paramsnonnull(); -int pthread_attr_setstack(pthread_attr_t *, void *, size_t) libcesque paramsnonnull((1)); -int pthread_attr_setstacksize(pthread_attr_t *, size_t) libcesque paramsnonnull(); int pthread_attr_setsigaltstack_np(pthread_attr_t *, void *, size_t) libcesque paramsnonnull((1)); int pthread_attr_setsigaltstacksize_np(pthread_attr_t *, size_t); +int pthread_attr_setstack(pthread_attr_t *, void *, size_t) libcesque paramsnonnull((1)); +int pthread_attr_setstacksize(pthread_attr_t *, size_t) libcesque paramsnonnull(); int pthread_barrier_destroy(pthread_barrier_t *) libcesque paramsnonnull(); int pthread_barrier_init(pthread_barrier_t *, const pthread_barrierattr_t *, unsigned) libcesque paramsnonnull((1)); int pthread_barrier_wait(pthread_barrier_t *) libcesque paramsnonnull(); @@ -183,13 +179,15 @@ int pthread_cond_destroy(pthread_cond_t *) libcesque paramsnonnull(); int pthread_cond_init(pthread_cond_t *, const pthread_condattr_t *) libcesque paramsnonnull((1)); int pthread_cond_signal(pthread_cond_t *) libcesque paramsnonnull(); int pthread_cond_wait(pthread_cond_t *, pthread_mutex_t *) libcesque paramsnonnull(); -int pthread_condattr_init(pthread_condattr_t *) libcesque paramsnonnull(); int pthread_condattr_destroy(pthread_condattr_t *) libcesque paramsnonnull(); -int pthread_condattr_setpshared(pthread_condattr_t *, int) libcesque paramsnonnull(); +int pthread_condattr_getclock(const pthread_condattr_t *, int *) libcesque paramsnonnull(); int pthread_condattr_getpshared(const pthread_condattr_t *, int *) libcesque paramsnonnull(); +int pthread_condattr_init(pthread_condattr_t *) libcesque paramsnonnull(); int pthread_condattr_setclock(pthread_condattr_t *, int) libcesque paramsnonnull(); -int pthread_condattr_getclock(const pthread_condattr_t *, int *) libcesque paramsnonnull(); +int pthread_condattr_setpshared(pthread_condattr_t *, int) libcesque paramsnonnull(); int pthread_create(pthread_t *, const pthread_attr_t *, void *(*)(void *), void *) dontthrow paramsnonnull((1)); +int pthread_decimate_np(void) libcesque; +int pthread_delay_np(const void *, int) libcesque; int pthread_detach(pthread_t) libcesque; int pthread_equal(pthread_t, pthread_t) libcesque; int pthread_getattr_np(pthread_t, pthread_attr_t *) libcesque paramsnonnull(); @@ -205,15 +203,17 @@ int pthread_mutex_init(pthread_mutex_t *, const pthread_mutexattr_t *) libcesque int pthread_mutex_lock(pthread_mutex_t *) libcesque paramsnonnull(); int pthread_mutex_trylock(pthread_mutex_t *) libcesque paramsnonnull(); int pthread_mutex_unlock(pthread_mutex_t *) libcesque paramsnonnull(); +int pthread_mutex_wipe_np(pthread_mutex_t *) libcesque paramsnonnull(); int pthread_mutexattr_destroy(pthread_mutexattr_t *) libcesque paramsnonnull(); int pthread_mutexattr_getpshared(const pthread_mutexattr_t *, int *) libcesque paramsnonnull(); +int pthread_mutexattr_getrobust(const pthread_mutexattr_t *, int *) libcesque paramsnonnull(); int pthread_mutexattr_gettype(const pthread_mutexattr_t *, int *) libcesque paramsnonnull(); int pthread_mutexattr_init(pthread_mutexattr_t *) libcesque paramsnonnull(); int pthread_mutexattr_setpshared(pthread_mutexattr_t *, int) libcesque paramsnonnull(); +int pthread_mutexattr_setrobust(const pthread_mutexattr_t *, int) libcesque paramsnonnull(); int pthread_mutexattr_settype(pthread_mutexattr_t *, int) libcesque paramsnonnull(); int pthread_once(pthread_once_t *, void (*)(void)) paramsnonnull(); int pthread_orphan_np(void) libcesque; -int pthread_decimate_np(void) libcesque; int pthread_rwlock_destroy(pthread_rwlock_t *) libcesque paramsnonnull(); int pthread_rwlock_init(pthread_rwlock_t *, const pthread_rwlockattr_t *) libcesque paramsnonnull((1)); int pthread_rwlock_rdlock(pthread_rwlock_t *) libcesque paramsnonnull(); @@ -237,17 +237,16 @@ int pthread_spin_trylock(pthread_spinlock_t *) libcesque paramsnonnull(); int pthread_spin_unlock(pthread_spinlock_t *) libcesque paramsnonnull(); int pthread_testcancel_np(void) libcesque; int pthread_tryjoin_np(pthread_t, void **) libcesque; -int pthread_delay_np(const void *, int) libcesque; -int pthread_yield_np(void) libcesque; int pthread_yield(void) libcesque; +int pthread_yield_np(void) libcesque; pthread_id_np_t pthread_getthreadid_np(void) libcesque; pthread_t pthread_self(void) libcesque pureconst; void *pthread_getspecific(pthread_key_t) libcesque; void pthread_cleanup_pop(struct _pthread_cleanup_buffer *, int) libcesque paramsnonnull(); void pthread_cleanup_push(struct _pthread_cleanup_buffer *, void (*)(void *), void *) libcesque paramsnonnull((1)); void pthread_exit(void *) libcesque wontreturn; -void pthread_testcancel(void) libcesque; void pthread_pause_np(void) libcesque; +void pthread_testcancel(void) libcesque; /* clang-format on */ diff --git a/libc/thread/tls.h b/libc/thread/tls.h index 6c8be574703..daf661835a8 100644 --- a/libc/thread/tls.h +++ b/libc/thread/tls.h @@ -15,7 +15,7 @@ struct CosmoFtrace { /* 16 */ int64_t ft_lastaddr; /* 8 */ }; -/* cosmopolitan thread information block (512 bytes) */ +/* cosmopolitan thread information block (1024 bytes) */ /* NOTE: update aarch64 libc/errno.h if sizeof changes */ /* NOTE: update aarch64 libc/proc/vfork.S if sizeof changes */ /* NOTE: update aarch64 libc/nexgen32e/gc.S if sizeof changes */ @@ -40,6 +40,7 @@ struct CosmoTib { void *tib_nsync; void *tib_atexit; _Atomic(void *) tib_keys[46]; + void *tib_locks[64]; } __attribute__((__aligned__(64))); extern char __tls_morphed; @@ -78,6 +79,10 @@ forceinline pureconst struct CosmoTib *__get_tls(void) { #endif } +struct CosmoTib *__get_tls_privileged(void) dontthrow pureconst; +struct CosmoTib *__get_tls_win32(void) dontthrow; +void __set_tls_win32(void *) libcesque; + #ifdef __x86_64__ #define __adj_tls(tib) (tib) #elif defined(__aarch64__) diff --git a/libc/thread/tls2.internal.h b/libc/thread/tls2.internal.h deleted file mode 100644 index be2e1c02a22..00000000000 --- a/libc/thread/tls2.internal.h +++ /dev/null @@ -1,43 +0,0 @@ -#ifndef COSMOPOLITAN_LIBC_THREAD_TLS2_H_ -#define COSMOPOLITAN_LIBC_THREAD_TLS2_H_ -#include "libc/dce.h" -#include "libc/thread/tls.h" -COSMOPOLITAN_C_START_ -#if defined(__GNUC__) && defined(__x86_64__) - -/** - * Returns location of thread information block. - * - * This should be favored over __get_tls() for .privileged code that - * can't be self-modified by __enable_tls(). - */ -forceinline struct CosmoTib *__get_tls_privileged(void) { - char *tib, *lin = (char *)0x30; - if (IsNetbsd() || IsOpenbsd()) { - __asm__("mov\t%%fs:(%1),%0" : "=a"(tib) : "r"(lin) : "memory"); - } else { - __asm__("mov\t%%gs:(%1),%0" : "=a"(tib) : "r"(lin) : "memory"); - if (IsWindows()) - tib = *(char **)(tib + 0x1480 + __tls_index * 8); - } - return (struct CosmoTib *)tib; -} - -forceinline struct CosmoTib *__get_tls_win32(void) { - char *tib, *lin = (char *)0x30; - __asm__("mov\t%%gs:(%1),%0" : "=a"(tib) : "r"(lin) : "memory"); - tib = *(char **)(tib + 0x1480 + __tls_index * 8); - return (struct CosmoTib *)tib; -} - -forceinline void __set_tls_win32(void *tls) { - __asm__("mov\t%1,%%gs:%0" : "=m"(*((long *)0x1480 + __tls_index)) : "r"(tls)); -} - -#elif defined(__aarch64__) -#define __get_tls_privileged() __get_tls() -#define __get_tls_win32() ((struct CosmoTib *)0) -#define __set_tls_win32(tls) (void)0 -#endif /* GNU x86-64 */ -COSMOPOLITAN_C_END_ -#endif /* COSMOPOLITAN_LIBC_THREAD_TLS2_H_ */ diff --git a/test/libc/calls/pledge_test.c b/test/libc/calls/pledge_test.c index 71d600834bc..089d965ef6b 100644 --- a/test/libc/calls/pledge_test.c +++ b/test/libc/calls/pledge_test.c @@ -64,6 +64,14 @@ void SetUpOnce(void) { testlib_enable_tmp_setup_teardown(); + if (pledge(0, 0) == -1) { + fprintf(stderr, "warning: pledge() not supported on this system %m\n"); + exit(0); + } +} + +void SetUp(void) { + __pledge_mode = PLEDGE_PENALTY_RETURN_EPERM; } void OnSig(int sig) { @@ -72,16 +80,6 @@ void OnSig(int sig) { int sys_memfd_secret(unsigned int); // our ENOSYS threshold -void SetUp(void) { - if (pledge(0, 0) == -1) { - fprintf(stderr, "warning: pledge() not supported on this system %m\n"); - exit(0); - } - testlib_extract("/zip/life.elf", "life.elf", 0755); - testlib_extract("/zip/sock.elf", "sock.elf", 0755); - __pledge_mode = PLEDGE_PENALTY_RETURN_EPERM; -} - TEST(pledge, default_allowsExit) { int *job; int ws, pid; @@ -107,6 +105,7 @@ TEST(pledge, execpromises_notok) { if (IsOpenbsd()) return; // b/c testing linux bpf int ws, pid; + testlib_extract("/zip/sock.elf", "sock.elf", 0755); ASSERT_NE(-1, (pid = fork())); if (!pid) { putenv("COMDBG=REDACTED"); @@ -532,6 +531,7 @@ TEST(pledge, open_cpath) { TEST(pledge, execpromises_ok) { if (IsOpenbsd()) return; // b/c testing linux bpf + testlib_extract("/zip/life.elf", "life.elf", 0755); int ws, pid; ASSERT_NE(-1, (pid = fork())); if (!pid) { @@ -549,6 +549,7 @@ TEST(pledge, execpromises_notok1) { if (IsOpenbsd()) return; // b/c testing linux bpf int ws, pid; + testlib_extract("/zip/sock.elf", "sock.elf", 0755); ASSERT_NE(-1, (pid = fork())); if (!pid) { putenv("COMDBG=REDACTED"); @@ -565,6 +566,7 @@ TEST(pledge, execpromises_reducesAtExecOnLinux) { if (IsOpenbsd()) return; // b/c testing linux bpf int ws, pid; + testlib_extract("/zip/sock.elf", "sock.elf", 0755); ASSERT_NE(-1, (pid = fork())); if (!pid) { putenv("COMDBG=REDACTED"); @@ -583,6 +585,7 @@ TEST(pledge_openbsd, execpromisesIsNull_letsItDoAnything) { if (!IsOpenbsd()) return; int ws, pid; + testlib_extract("/zip/sock.elf", "sock.elf", 0755); ASSERT_NE(-1, (pid = fork())); if (!pid) { ASSERT_SYS(0, 0, pledge("stdio exec", 0)); @@ -602,6 +605,7 @@ TEST(pledge_openbsd, execpromisesIsSuperset_letsItDoAnything) { if (!IsOpenbsd()) return; int ws, pid; + testlib_extract("/zip/sock.elf", "sock.elf", 0755); ASSERT_NE(-1, (pid = fork())); if (!pid) { ASSERT_SYS(0, 0, pledge("stdio rpath exec", "stdio rpath tty inet")); @@ -623,6 +627,7 @@ TEST(pledge_openbsd, execpromises_notok) { if (IsOpenbsd()) return; // mimmutable() ugh int ws, pid; + testlib_extract("/zip/sock.elf", "sock.elf", 0755); ASSERT_NE(-1, (pid = fork())); if (!pid) { putenv("COMDBG=REDACTED"); diff --git a/test/libc/calls/raise_test.c b/test/libc/calls/raise_test.c index 5ebb8189afa..ee891715af3 100644 --- a/test/libc/calls/raise_test.c +++ b/test/libc/calls/raise_test.c @@ -20,6 +20,7 @@ #include "libc/calls/struct/sigaction.h" #include "libc/calls/struct/siginfo.h" #include "libc/dce.h" +#include "libc/mem/leaks.h" #include "libc/runtime/runtime.h" #include "libc/sysv/consts/sa.h" #include "libc/sysv/consts/sicode.h" @@ -30,6 +31,7 @@ #include "libc/thread/thread.h" TEST(raise, trap) { + AssertNoLocksAreHeld(); signal(SIGTRAP, SIG_DFL); SPAWN(fork); raise(SIGTRAP); @@ -44,6 +46,7 @@ TEST(raise, fpe) { } TEST(raise, usr1) { + AssertNoLocksAreHeld(); SPAWN(fork); raise(SIGUSR1); TERMS(SIGUSR1); @@ -69,6 +72,7 @@ void *Worker(void *arg) { TEST(raise, threaded) { SPAWN(fork); + AssertNoLocksAreHeld(); signal(SIGILL, SIG_DFL); pthread_t worker; ASSERT_EQ(0, pthread_create(&worker, 0, Worker, 0)); diff --git a/test/libc/intrin/lock_test.c b/test/libc/intrin/lock_test.c index 06782deed12..f52eb07a5eb 100644 --- a/test/libc/intrin/lock_test.c +++ b/test/libc/intrin/lock_test.c @@ -18,6 +18,7 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/atomic.h" #include "libc/calls/calls.h" +#include "libc/calls/struct/sigaction.h" #include "libc/calls/struct/timespec.h" #include "libc/errno.h" #include "libc/fmt/itoa.h" @@ -28,8 +29,10 @@ #include "libc/runtime/internal.h" #include "libc/runtime/runtime.h" #include "libc/runtime/stack.h" +#include "libc/runtime/symbols.internal.h" #include "libc/str/str.h" #include "libc/sysv/consts/clone.h" +#include "libc/sysv/consts/sig.h" #include "libc/thread/thread.h" #include "libc/thread/tls.h" #include "third_party/nsync/mu.h" @@ -62,6 +65,9 @@ pthread_mutex_t mu; __assert_eq_fail(__FILE__, __LINE__, #WANT, #GOT, _want, _got); \ } while (0) +void ignore_signal(int sig) { +} + void __assert_eq_fail(const char *file, int line, const char *wantstr, const char *gotstr, long want, long got) { kprintf("%s:%d: %s vs. %s was %ld vs. %ld (%s)\n", file, line, wantstr, @@ -177,6 +183,12 @@ void TestUncontendedLock(const char *name, int kind) { int main(int argc, char *argv[]) { pthread_mutexattr_t attr; +#ifdef MODE_DBG + GetSymbolTable(); + signal(SIGTRAP, ignore_signal); + kprintf("running %s\n", argv[0]); +#endif + #ifdef __aarch64__ // our usage of raw clone() is probably broken in aarch64 // we should just get rid of clone() @@ -190,7 +202,7 @@ int main(int argc, char *argv[]) { } ASSERT_EQ(0, pthread_mutexattr_init(&attr)); - ASSERT_EQ(0, pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_NORMAL)); + ASSERT_EQ(0, pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_DEFAULT)); ASSERT_EQ(0, pthread_mutex_init(&mu, &attr)); ASSERT_EQ(0, pthread_mutexattr_destroy(&attr)); ASSERT_EQ(0, pthread_mutex_lock(&mu)); @@ -216,28 +228,12 @@ int main(int argc, char *argv[]) { ASSERT_EQ(0, pthread_mutex_unlock(&mu)); ASSERT_EQ(0, pthread_mutex_destroy(&mu)); - ASSERT_EQ(1, __tls_enabled); - - TestUncontendedLock("PTHREAD_MUTEX_NORMAL RAW TLS", PTHREAD_MUTEX_NORMAL); + TestUncontendedLock("PTHREAD_MUTEX_DEFAULT RAW TLS", PTHREAD_MUTEX_DEFAULT); TestUncontendedLock("PTHREAD_MUTEX_RECURSIVE RAW TLS", PTHREAD_MUTEX_RECURSIVE); - TestUncontendedLock("PTHREAD_MUTEX_ERRORCHECK RAW TLS", - PTHREAD_MUTEX_ERRORCHECK); - TestContendedLock("PTHREAD_MUTEX_NORMAL RAW TLS", PTHREAD_MUTEX_NORMAL); + TestContendedLock("PTHREAD_MUTEX_DEFAULT RAW TLS", PTHREAD_MUTEX_DEFAULT); TestContendedLock("PTHREAD_MUTEX_RECURSIVE RAW TLS", PTHREAD_MUTEX_RECURSIVE); - TestContendedLock("PTHREAD_MUTEX_ERRORCHECK RAW TLS", - PTHREAD_MUTEX_ERRORCHECK); - - __tls_enabled_set(false); - - TestUncontendedLock("PTHREAD_MUTEX_NORMAL RAW", PTHREAD_MUTEX_NORMAL); - TestUncontendedLock("PTHREAD_MUTEX_RECURSIVE RAW", PTHREAD_MUTEX_RECURSIVE); - TestUncontendedLock("PTHREAD_MUTEX_ERRORCHECK RAW", PTHREAD_MUTEX_ERRORCHECK); - - TestContendedLock("PTHREAD_MUTEX_NORMAL RAW", PTHREAD_MUTEX_NORMAL); - TestContendedLock("PTHREAD_MUTEX_RECURSIVE RAW", PTHREAD_MUTEX_RECURSIVE); - TestContendedLock("PTHREAD_MUTEX_ERRORCHECK RAW", PTHREAD_MUTEX_ERRORCHECK); // } diff --git a/test/libc/intrin/lockipc_test.c b/test/libc/intrin/lockipc_test.c index 0f3467bc2f6..30878c6999d 100644 --- a/test/libc/intrin/lockipc_test.c +++ b/test/libc/intrin/lockipc_test.c @@ -52,7 +52,7 @@ TEST(lockipc, mutex) { // create shared mutex pthread_mutexattr_t mattr; pthread_mutexattr_init(&mattr); - pthread_mutexattr_settype(&mattr, PTHREAD_MUTEX_NORMAL); + pthread_mutexattr_settype(&mattr, PTHREAD_MUTEX_DEFAULT); pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED); pthread_mutex_init(&shm->mutex, &mattr); pthread_mutexattr_destroy(&mattr); diff --git a/test/libc/intrin/memset_test.c b/test/libc/intrin/memset_test.c index f935e8bfaff..cd05645e9ee 100644 --- a/test/libc/intrin/memset_test.c +++ b/test/libc/intrin/memset_test.c @@ -66,9 +66,11 @@ TEST(bzero, hug) { #define N (256 * 1024 * 1024) -BENCH(strlen, bench) { +BENCH(memset, bench) { + void *memset_(void *, int, size_t) asm("memset"); + printf("\n"); static char A[N]; memset(A, 2, N); for (int n = 1; n <= N; n *= 2) - BENCHMARK(100, n, X(memset(V(A), 1, n))); + BENCHMARK(100, n, X(memset_(V(A), 0, n))); } diff --git a/test/libc/intrin/pthread_mutex_lock2_test.c b/test/libc/intrin/pthread_mutex_lock2_test.c index 93224da8443..b530ac04b19 100644 --- a/test/libc/intrin/pthread_mutex_lock2_test.c +++ b/test/libc/intrin/pthread_mutex_lock2_test.c @@ -40,7 +40,7 @@ pthread_mutexattr_t attr; FIXTURE(pthread_mutex_lock, normal) { ASSERT_EQ(0, pthread_mutexattr_init(&attr)); - ASSERT_EQ(0, pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_NORMAL)); + ASSERT_EQ(0, pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_DEFAULT)); ASSERT_EQ(0, pthread_mutex_init(&lock, &attr)); ASSERT_EQ(0, pthread_mutexattr_destroy(&attr)); } @@ -79,7 +79,7 @@ TEST(pthread_mutex_lock, contention) { int i; pthread_t *th = gc(malloc(sizeof(pthread_t) * THREADS)); pthread_mutexattr_init(&attr); - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_NORMAL); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_DEFAULT); pthread_mutex_init(&lock, &attr); pthread_mutexattr_destroy(&attr); count = 0; @@ -128,7 +128,7 @@ BENCH(pthread_mutex_lock, bench_uncontended) { pthread_mutex_t m; pthread_mutexattr_t attr; pthread_mutexattr_init(&attr); - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_NORMAL); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_DEFAULT); pthread_mutex_init(&m, &attr); EZBENCH2("normal 1x", donothing, BenchLockUnlock(&m)); } @@ -226,7 +226,7 @@ BENCH(pthread_mutex_lock, bench_contended) { pthread_mutex_t m; pthread_mutexattr_t attr; pthread_mutexattr_init(&attr); - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_NORMAL); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_DEFAULT); pthread_mutex_init(&m, &attr); struct MutexContentionArgs a = {&m}; pthread_create(&t, 0, MutexContentionWorker, &a); diff --git a/test/libc/intrin/pthread_mutex_lock_test.c b/test/libc/intrin/pthread_mutex_lock_test.c index 4881733f5c5..ffecfe9f7b4 100644 --- a/test/libc/intrin/pthread_mutex_lock_test.c +++ b/test/libc/intrin/pthread_mutex_lock_test.c @@ -20,12 +20,16 @@ #include "libc/atomic.h" #include "libc/calls/calls.h" #include "libc/calls/state.internal.h" +#include "libc/calls/struct/sigaction.h" +#include "libc/cosmo.h" #include "libc/errno.h" +#include "libc/intrin/kprintf.h" #include "libc/intrin/strace.h" #include "libc/log/check.h" #include "libc/macros.h" #include "libc/math.h" #include "libc/mem/gc.h" +#include "libc/mem/leaks.h" #include "libc/mem/mem.h" #include "libc/runtime/internal.h" #include "libc/runtime/runtime.h" @@ -34,6 +38,7 @@ #include "libc/sysv/consts/map.h" #include "libc/sysv/consts/prot.h" #include "libc/sysv/consts/rlimit.h" +#include "libc/sysv/consts/sig.h" #include "libc/testlib/ezbench.h" #include "libc/testlib/testlib.h" #include "libc/thread/thread.h" @@ -48,16 +53,38 @@ int count; atomic_int started; atomic_int finished; +pthread_mutex_t lock; pthread_mutex_t mylock; pthread_spinlock_t slock; pthread_t th[THREADS]; +void ignore_signal(int sig) { +} + void SetUpOnce(void) { ASSERT_SYS(0, 0, pledge("stdio rpath", 0)); + kprintf("running %s\n", program_invocation_name); + signal(SIGTRAP, ignore_signal); +} + +TEST(pthread_mutex_lock, default) { + pthread_mutexattr_t attr; + ASSERT_EQ(0, pthread_mutexattr_init(&attr)); + ASSERT_EQ(0, pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_DEFAULT)); + ASSERT_EQ(0, pthread_mutex_init(&lock, &attr)); + ASSERT_EQ(0, pthread_mutexattr_destroy(&attr)); + ASSERT_EQ(0, pthread_mutex_init(&lock, 0)); + ASSERT_EQ(0, pthread_mutex_lock(&lock)); + ASSERT_EQ(EBUSY, pthread_mutex_trylock(&lock)); + ASSERT_EQ(0, pthread_mutex_unlock(&lock)); + ASSERT_EQ(0, pthread_mutex_trylock(&lock)); + ASSERT_EQ(0, pthread_mutex_unlock(&lock)); + ASSERT_EQ(0, pthread_mutex_lock(&lock)); + ASSERT_EQ(0, pthread_mutex_unlock(&lock)); + ASSERT_EQ(0, pthread_mutex_destroy(&lock)); } TEST(pthread_mutex_lock, normal) { - pthread_mutex_t lock; pthread_mutexattr_t attr; ASSERT_EQ(0, pthread_mutexattr_init(&attr)); ASSERT_EQ(0, pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_NORMAL)); @@ -75,7 +102,6 @@ TEST(pthread_mutex_lock, normal) { } TEST(pthread_mutex_lock, recursive) { - pthread_mutex_t lock; pthread_mutexattr_t attr; ASSERT_EQ(0, pthread_mutexattr_init(&attr)); ASSERT_EQ(0, pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE)); @@ -99,15 +125,15 @@ TEST(pthread_mutex_lock, recursive) { } TEST(pthread_mutex_lock, errorcheck) { - pthread_mutex_t lock; pthread_mutexattr_t attr; ASSERT_EQ(0, pthread_mutexattr_init(&attr)); ASSERT_EQ(0, pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK)); ASSERT_EQ(0, pthread_mutex_init(&lock, &attr)); ASSERT_EQ(0, pthread_mutexattr_destroy(&attr)); ASSERT_EQ(0, pthread_mutex_lock(&lock)); + ASSERT_EQ(1, __deadlock_tracked(&lock)); ASSERT_EQ(EDEADLK, pthread_mutex_lock(&lock)); - ASSERT_EQ(EDEADLK, pthread_mutex_trylock(&lock)); + ASSERT_EQ(EBUSY, pthread_mutex_trylock(&lock)); ASSERT_EQ(0, pthread_mutex_unlock(&lock)); ASSERT_EQ(0, pthread_mutex_destroy(&lock)); } @@ -130,7 +156,7 @@ TEST(pthread_mutex_lock, contention) { int i; pthread_mutexattr_t attr; pthread_mutexattr_init(&attr); - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_NORMAL); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_DEFAULT); pthread_mutex_init(&mylock, &attr); pthread_mutexattr_destroy(&attr); count = 0; @@ -171,27 +197,28 @@ TEST(pthread_mutex_lock, rcontention) { EXPECT_EQ(0, pthread_mutex_destroy(&mylock)); } -TEST(pthread_mutex_lock, econtention) { - int i; - pthread_mutexattr_t attr; - pthread_mutexattr_init(&attr); - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK); - pthread_mutex_init(&mylock, &attr); - pthread_mutexattr_destroy(&attr); - count = 0; - started = 0; - finished = 0; - for (i = 0; i < THREADS; ++i) { - ASSERT_NE(-1, pthread_create(th + i, 0, MutexWorker, (void *)(intptr_t)i)); - } - for (i = 0; i < THREADS; ++i) { - pthread_join(th[i], 0); - } - EXPECT_EQ(THREADS, started); - EXPECT_EQ(THREADS, finished); - EXPECT_EQ(THREADS * ITERATIONS, count); - EXPECT_EQ(0, pthread_mutex_destroy(&mylock)); -} +/* TEST(pthread_mutex_lock, econtention) { */ +/* int i; */ +/* pthread_mutexattr_t attr; */ +/* pthread_mutexattr_init(&attr); */ +/* pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK); */ +/* pthread_mutex_init(&mylock, &attr); */ +/* pthread_mutexattr_destroy(&attr); */ +/* count = 0; */ +/* started = 0; */ +/* finished = 0; */ +/* for (i = 0; i < THREADS; ++i) { */ +/* ASSERT_NE(-1, pthread_create(th + i, 0, MutexWorker, (void + * *)(intptr_t)i)); */ +/* } */ +/* for (i = 0; i < THREADS; ++i) { */ +/* pthread_join(th[i], 0); */ +/* } */ +/* EXPECT_EQ(THREADS, started); */ +/* EXPECT_EQ(THREADS, finished); */ +/* EXPECT_EQ(THREADS * ITERATIONS, count); */ +/* EXPECT_EQ(0, pthread_mutex_destroy(&mylock)); */ +/* } */ void *SpinlockWorker(void *p) { int i; diff --git a/test/libc/mem/malloc_torture_test.c b/test/libc/mem/malloc_torture_test.c index 40ae5493426..f20c1dc2088 100644 --- a/test/libc/mem/malloc_torture_test.c +++ b/test/libc/mem/malloc_torture_test.c @@ -19,6 +19,7 @@ #include "libc/calls/struct/timespec.h" #include "libc/intrin/safemacros.h" #include "libc/mem/gc.h" +#include "libc/mem/leaks.h" #include "libc/mem/mem.h" #include "libc/stdio/rand.h" #include "libc/stdio/stdio.h" @@ -33,8 +34,8 @@ void *Worker(void *arg) { for (int i = 0; i < ITERATIONS; ++i) { char *p; - ASSERT_NE(NULL, (p = malloc(lemur64() % SIZE))); - ASSERT_NE(NULL, (p = realloc(p, max(lemur64() % SIZE, 1)))); + ASSERT_NE(NULL, (p = malloc(rand() % SIZE))); + ASSERT_NE(NULL, (p = realloc(p, rand() % SIZE))); free(p); } return 0; @@ -48,6 +49,7 @@ TEST(malloc, torture) { printf("\nmalloc torture test w/ %d threads and %d iterations\n", n, ITERATIONS); SPAWN(fork); + AssertNoLocksAreHeld(); struct timespec t1 = timespec_real(); for (i = 0; i < n; ++i) ASSERT_EQ(0, pthread_create(t + i, 0, Worker, 0)); diff --git a/test/libc/stdio/fgetwc_test.c b/test/libc/stdio/fgetwc_test.c index e7a55ceff8c..1f072928260 100644 --- a/test/libc/stdio/fgetwc_test.c +++ b/test/libc/stdio/fgetwc_test.c @@ -16,7 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/stdio/stdio.h" +#include "libc/stdio/internal.h" #include "libc/testlib/testlib.h" TEST(fgetwc, testAscii_oneChar) { diff --git a/test/libc/system/popen_test.c b/test/libc/system/popen_test.c index 10e53cd8754..ef37402b9a0 100644 --- a/test/libc/system/popen_test.c +++ b/test/libc/system/popen_test.c @@ -34,7 +34,6 @@ #include "libc/sysv/consts/sig.h" #include "libc/testlib/testlib.h" #include "libc/thread/thread.h" -#ifdef __x86_64__ FILE *f; char buf[32]; @@ -160,7 +159,7 @@ void *Worker(void *arg) { } TEST(popen, torture) { - int i, n = 4; + int i, n = 40; pthread_t *t = gc(malloc(sizeof(pthread_t) * n)); testlib_extract("/zip/echo", "echo", 0755); for (i = 0; i < n; ++i) @@ -169,5 +168,3 @@ TEST(popen, torture) { ASSERT_EQ(0, pthread_join(t[i], 0)); CheckForFdLeaks(); } - -#endif /* __x86_64__ */ diff --git a/test/libc/thread/footek_test.c b/test/libc/thread/footek_test.c index a07ea6a3872..b08846ae39f 100644 --- a/test/libc/thread/footek_test.c +++ b/test/libc/thread/footek_test.c @@ -349,7 +349,7 @@ int main() { #if USE == POSIX_RECURSIVE pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); #else - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_NORMAL); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_DEFAULT); #endif pthread_mutex_init(&g_locker, &attr); pthread_mutexattr_destroy(&attr); diff --git a/test/libc/thread/pthread_atfork_test.c b/test/libc/thread/pthread_atfork_test.c index ba3c9b056d0..8a6d5d4d0c0 100644 --- a/test/libc/thread/pthread_atfork_test.c +++ b/test/libc/thread/pthread_atfork_test.c @@ -22,6 +22,7 @@ #include "libc/intrin/atomic.h" #include "libc/intrin/kprintf.h" #include "libc/mem/gc.h" +#include "libc/mem/leaks.h" #include "libc/mem/mem.h" #include "libc/runtime/internal.h" #include "libc/runtime/runtime.h" @@ -51,7 +52,6 @@ TEST(pthread_atfork, test) { SPAWN(fork); ASSERT_EQ(0, pthread_atfork(prepare1, parent1, child1)); ASSERT_EQ(0, pthread_atfork(prepare2, parent2, child2)); - flockfile(stdout); SPAWN(fork); flockfile(stdout); ASSERT_STREQ("prepare2", A[0]); @@ -60,7 +60,6 @@ TEST(pthread_atfork, test) { ASSERT_STREQ("child2", A[3]); funlockfile(stdout); EXITS(0); - funlockfile(stdout); ASSERT_STREQ("prepare2", A[0]); ASSERT_STREQ("prepare1", A[1]); ASSERT_STREQ("parent1", A[2]); @@ -79,7 +78,7 @@ void mu_unlock(void) { } void mu_wipe(void) { - pthread_mutex_init(&mu, 0); + pthread_mutex_wipe_np(&mu); } void *Worker(void *arg) { diff --git a/test/libc/thread/pthread_cancel_deferred_cond_test.c b/test/libc/thread/pthread_cancel_deferred_cond_test.c index 76d9eb928fa..90d00b02964 100644 --- a/test/libc/thread/pthread_cancel_deferred_cond_test.c +++ b/test/libc/thread/pthread_cancel_deferred_cond_test.c @@ -1,8 +1,24 @@ +// Copyright 2024 Justine Alexandra Roberts Tunney +// +// Permission to use, copy, modify, and/or distribute this software for +// any purpose with or without fee is hereby granted, provided that the +// above copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL +// WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE +// AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL +// DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR +// PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +// TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +#include #include #include +#include #include #include -#include "libc/stdio/stdio.h" int got_cleanup; pthread_cond_t cv; @@ -22,11 +38,12 @@ void* worker(void* arg) { } int main(int argc, char* argv[]) { + ShowCrashReports(); void* rc; pthread_t th; pthread_mutexattr_t at; pthread_mutexattr_init(&at); - pthread_mutexattr_settype(&at, PTHREAD_MUTEX_NORMAL); + pthread_mutexattr_settype(&at, PTHREAD_MUTEX_DEFAULT); pthread_mutex_init(&mu, &at); pthread_mutexattr_destroy(&at); pthread_cond_init(&cv, 0); @@ -42,8 +59,6 @@ int main(int argc, char* argv[]) { return 6; if (pthread_mutex_trylock(&mu) != EBUSY) return 7; - if (pthread_mutex_unlock(&mu)) - return 8; pthread_mutex_destroy(&mu); pthread_cond_destroy(&cv); } diff --git a/test/libc/thread/pthread_cancel_test.c b/test/libc/thread/pthread_cancel_test.c index 56f9fa5d550..7c7b4739b7e 100644 --- a/test/libc/thread/pthread_cancel_test.c +++ b/test/libc/thread/pthread_cancel_test.c @@ -40,11 +40,7 @@ atomic_int gotcleanup; void SetUpOnce(void) { testlib_enable_tmp_setup_teardown(); - pthread_mutexattr_t at; - pthread_mutexattr_init(&at); - pthread_mutexattr_settype(&at, PTHREAD_MUTEX_NORMAL); - pthread_mutex_init(&mu, &at); - pthread_mutexattr_destroy(&at); + pthread_mutex_init(&mu, 0); pthread_cond_init(&cv, 0); } @@ -194,6 +190,7 @@ TEST(pthread_cancel, condDeferredWait_reacquiresMutex) { ASSERT_EQ(0, pthread_join(th, &rc)); ASSERT_EQ(PTHREAD_CANCELED, rc); ASSERT_EQ(EBUSY, pthread_mutex_trylock(&mu)); + ASSERT_EQ(0, pthread_mutex_consistent(&mu)); ASSERT_EQ(0, pthread_mutex_unlock(&mu)); } @@ -206,6 +203,7 @@ TEST(pthread_cancel, condDeferredWaitDelayed) { ASSERT_EQ(0, pthread_join(th, &rc)); ASSERT_EQ(PTHREAD_CANCELED, rc); ASSERT_EQ(EBUSY, pthread_mutex_trylock(&mu)); + ASSERT_EQ(0, pthread_mutex_consistent(&mu)); ASSERT_EQ(0, pthread_mutex_unlock(&mu)); } diff --git a/test/libc/thread/pthread_rwlock_rdlock_test.c b/test/libc/thread/pthread_rwlock_rdlock_test.c index f804efe49d6..4fba1f5034b 100644 --- a/test/libc/thread/pthread_rwlock_rdlock_test.c +++ b/test/libc/thread/pthread_rwlock_rdlock_test.c @@ -76,11 +76,11 @@ void *Writer(void *arg) { ASSERT_EQ(0, pthread_rwlock_wrlock(&lock)); // cosmo_trace_begin("writer"); ++foo; - delay(100); + delay(10); ++bar; // cosmo_trace_end("writer"); ASSERT_EQ(0, pthread_rwlock_unlock(&lock)); - delay(100); + delay(10); } done = true; return 0; diff --git a/test/libc/thread/setitimer_test.c b/test/libc/thread/setitimer_test.c index d63c65e5fcf..061faf459d6 100644 --- a/test/libc/thread/setitimer_test.c +++ b/test/libc/thread/setitimer_test.c @@ -16,6 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/sysv/consts/itimer.h" #include "libc/atomic.h" #include "libc/calls/calls.h" #include "libc/calls/struct/itimerval.h" @@ -28,7 +29,6 @@ #include "libc/errno.h" #include "libc/limits.h" #include "libc/runtime/runtime.h" -#include "libc/sysv/consts/itimer.h" #include "libc/sysv/consts/sa.h" #include "libc/sysv/consts/sicode.h" #include "libc/sysv/consts/sig.h" diff --git a/test/posix/cyclic_mutex_test.c b/test/posix/cyclic_mutex_test.c new file mode 100644 index 00000000000..28c733751fe --- /dev/null +++ b/test/posix/cyclic_mutex_test.c @@ -0,0 +1,71 @@ +// Copyright 2024 Justine Alexandra Roberts Tunney +// +// Permission to use, copy, modify, and/or distribute this software for +// any purpose with or without fee is hereby granted, provided that the +// above copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL +// WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE +// AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL +// DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR +// PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +// TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +#include +#include +#include +#include + +pthread_mutex_t x; +pthread_mutex_t y; + +void ignore_signal(int sig) { +} + +int main(int argc, char *argv[]) { + +#ifdef MODE_DBG + GetSymbolTable(); + signal(SIGTRAP, ignore_signal); + kprintf("running %s\n", argv[0]); +#endif + + pthread_mutexattr_t attr; + if (pthread_mutexattr_init(&attr)) + return 1; + if (pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK)) + return 2; + if (pthread_mutex_init(&x, &attr)) + return 3; + if (pthread_mutex_init(&y, &attr)) + return 4; + if (pthread_mutexattr_destroy(&attr)) + return 5; + + if (pthread_mutex_lock(&x)) + return 6; + if (pthread_mutex_lock(&y)) + return 7; + if (pthread_mutex_unlock(&y)) + return 8; + if (pthread_mutex_unlock(&x)) + return 9; + + if (pthread_mutex_lock(&y)) + return 10; + if (pthread_mutex_lock(&y) != EDEADLK) + return 11; + if (pthread_mutex_lock(&x) != EDEADLK) + return 12; + if (pthread_mutex_unlock(&x) != EPERM) + return 13; + if (pthread_mutex_unlock(&y)) + return 14; + + if (pthread_mutex_destroy(&y)) + return 15; + if (pthread_mutex_destroy(&x)) + return 16; +} diff --git a/test/posix/file_offset_exec_test.c b/test/posix/file_offset_exec_test.c index aafc9061ae7..adbe1f6173a 100644 --- a/test/posix/file_offset_exec_test.c +++ b/test/posix/file_offset_exec_test.c @@ -13,6 +13,7 @@ // TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR // PERFORMANCE OF THIS SOFTWARE. +#include #include #include #include @@ -36,6 +37,8 @@ void on_unexpected_death(int sig) { } int main() { + GetSymbolTable(); + ShowCrashReports(); signal(SIGCHLD, on_unexpected_death); // extract test program diff --git a/test/posix/interprocess_signaling_test.c b/test/posix/interprocess_signaling_test.c index d6372492e4a..08176da90df 100644 --- a/test/posix/interprocess_signaling_test.c +++ b/test/posix/interprocess_signaling_test.c @@ -13,6 +13,7 @@ // TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR // PERFORMANCE OF THIS SOFTWARE. +#include #include #include #include @@ -26,6 +27,8 @@ void onsig(int sig) { int main(int argc, char *argv[]) { + ShowCrashReports(); + // create process shared memory got = mmap(0, 4, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); if (got == MAP_FAILED) diff --git a/test/posix/mutex_async_signal_safety_test.c b/test/posix/mutex_async_signal_safety_test.c index 08cc268e8f0..d861ba42aa8 100644 --- a/test/posix/mutex_async_signal_safety_test.c +++ b/test/posix/mutex_async_signal_safety_test.c @@ -1,3 +1,19 @@ +// Copyright 2024 Justine Alexandra Roberts Tunney +// +// Permission to use, copy, modify, and/or distribute this software for +// any purpose with or without fee is hereby granted, provided that the +// above copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL +// WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE +// AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL +// DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR +// PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +// TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +#include #include #include #include @@ -35,6 +51,11 @@ void* work(void* arg) { int main() { + if (IsModeDbg()) { + kprintf("mutex_async_signal_safety_test not feasible in debug mode\n"); + return 0; + } + struct sigaction sa; sa.sa_handler = hand; sa.sa_flags = SA_NODEFER; diff --git a/test/posix/pending_signal_execve_test.c b/test/posix/pending_signal_execve_test.c index 326f3b84154..0b97b794b45 100644 --- a/test/posix/pending_signal_execve_test.c +++ b/test/posix/pending_signal_execve_test.c @@ -43,7 +43,7 @@ int main(int argc, char* argv[]) { execlp(argv[0], argv[0], "childe", NULL); _Exit(127); } - if (IsNetbsd()) { + if (IsNetbsd() || IsOpenbsd()) { // NetBSD has a bug where pending signals don't inherit across // execve, even though POSIX.1 literally says you must do this sleep(1); diff --git a/test/posix/signal_latency_test.c b/test/posix/signal_latency_test.c index 9f599f43809..c9ee5c2692d 100644 --- a/test/posix/signal_latency_test.c +++ b/test/posix/signal_latency_test.c @@ -14,6 +14,7 @@ // PERFORMANCE OF THIS SOFTWARE. #include +#include #include #include #include @@ -25,13 +26,14 @@ #define ITERATIONS 10000 +atomic_bool got_sigusr2; pthread_t sender_thread; pthread_t receiver_thread; struct timespec send_time; double latencies[ITERATIONS]; void sender_signal_handler(int signo) { - // Empty handler to unblock sigsuspend() + got_sigusr2 = true; } void receiver_signal_handler(int signo) { @@ -77,14 +79,16 @@ void *sender_func(void *arg) { exit(5); // Send SIGUSR1 to receiver_thread + got_sigusr2 = false; if (pthread_kill(receiver_thread, SIGUSR1)) exit(6); // Unblock SIGUSR2 and wait for it sigset_t wait_set; sigemptyset(&wait_set); - if (sigsuspend(&wait_set) && errno != EINTR) - exit(7); + while (!got_sigusr2) + if (sigsuspend(&wait_set) && errno != EINTR) + exit(7); } return 0; @@ -125,6 +129,10 @@ int compare(const void *a, const void *b) { int main() { + // TODO(jart): Why is this test flaky on Windows? + if (IsWindows()) + return 0; + // Block SIGUSR1 and SIGUSR2 in main thread sigset_t block_set; sigemptyset(&block_set); diff --git a/third_party/dlmalloc/README.cosmo b/third_party/dlmalloc/README.cosmo index 0db6ea93718..097b9342af4 100644 --- a/third_party/dlmalloc/README.cosmo +++ b/third_party/dlmalloc/README.cosmo @@ -9,6 +9,7 @@ LICENSE LOCAL CHANGES + - Fix MT-safety bugs in DEBUG mode - Fix bug in dlmalloc_inspect_all() - Define dlmalloc_requires_more_vespene_gas() - Make dlmalloc scalable using sched_getcpu() diff --git a/third_party/dlmalloc/dlmalloc.c b/third_party/dlmalloc/dlmalloc.c index 389fff109c7..5f990db59aa 100644 --- a/third_party/dlmalloc/dlmalloc.c +++ b/third_party/dlmalloc/dlmalloc.c @@ -31,13 +31,14 @@ #define FOOTERS 1 #define MSPACES 1 #define ONLY_MSPACES 1 // enables scalable multi-threaded malloc -#define USE_SPIN_LOCKS 0 // only profitable using sched_getcpu() +#define USE_SPIN_LOCKS 0 // set to 0 to use scalable nsync locks #else #define INSECURE 1 #define PROCEED_ON_ERROR 1 #define FOOTERS 0 #define MSPACES 0 #define ONLY_MSPACES 0 +#define USE_SPIN_LOCKS 1 #endif #define HAVE_MMAP 1 @@ -1263,12 +1264,15 @@ void* dlrealloc_single(void* oldmem, size_t bytes) { #endif /* FOOTERS */ if (!PREACTION(m)) { mchunkptr newp = try_realloc_chunk(m, oldp, nb, MREMAP_MAYMOVE); - POSTACTION(m); if (newp != 0) { + /* [jart] fix realloc MT bug in DEBUG mode + https://github.com/intel/linux-sgx/issues/534 */ check_inuse_chunk(m, newp); + POSTACTION(m); mem = chunk2mem(newp); } else { + POSTACTION(m); mem = internal_malloc(m, bytes); if (mem != 0) { size_t oc = chunksize(oldp) - overhead_for(oldp); @@ -1301,11 +1305,13 @@ void* dlrealloc_in_place(void* oldmem, size_t bytes) { #endif /* FOOTERS */ if (!PREACTION(m)) { mchunkptr newp = try_realloc_chunk(m, oldp, nb, 0); - POSTACTION(m); if (newp == oldp) { + /* [jart] fix realloc MT bug in DEBUG mode + https://github.com/intel/linux-sgx/issues/534 */ check_inuse_chunk(m, newp); mem = oldmem; } + POSTACTION(m); } } } @@ -1319,13 +1325,6 @@ void* dlmemalign_single(size_t alignment, size_t bytes) { return internal_memalign(gm, alignment, bytes); } -#if USE_LOCKS -void dlmalloc_atfork(void) { - bzero(&gm->mutex, sizeof(gm->mutex)); - bzero(&malloc_global_mutex, sizeof(malloc_global_mutex)); -} -#endif - void** dlindependent_calloc(size_t n_elements, size_t elem_size, void* chunks[]) { size_t sz = elem_size; /* serves as 1-element array */ diff --git a/third_party/dlmalloc/dlmalloc.h b/third_party/dlmalloc/dlmalloc.h index edb86f27a10..5bbb9a179c7 100644 --- a/third_party/dlmalloc/dlmalloc.h +++ b/third_party/dlmalloc/dlmalloc.h @@ -9,7 +9,6 @@ #define dlmallinfo __dlmallinfo #define dlmalloc __dlmalloc #define dlmalloc_abort __dlmalloc_abort -#define dlmalloc_atfork __dlmalloc_atfork #define dlmalloc_footprint __dlmalloc_footprint #define dlmalloc_footprint_limit __dlmalloc_footprint_limit #define dlmalloc_inspect_all __dlmalloc_inspect_all @@ -527,7 +526,10 @@ void mspace_inspect_all(mspace msp, void (*handler)(void*, void*, size_t, void*), void* arg); -void dlmalloc_atfork(void); +void dlmalloc_pre_fork(void) libcesque; +void dlmalloc_post_fork_parent(void) libcesque; +void dlmalloc_post_fork_child(void) libcesque; + void dlmalloc_abort(void) relegated wontreturn; COSMOPOLITAN_C_END_ diff --git a/third_party/dlmalloc/init.inc b/third_party/dlmalloc/init.inc index 682b5040891..0c2e1e80220 100644 --- a/third_party/dlmalloc/init.inc +++ b/third_party/dlmalloc/init.inc @@ -7,31 +7,34 @@ #if LOCK_AT_FORK #if ONLY_MSPACES -static void dlmalloc_pre_fork(void) { +void dlmalloc_pre_fork(void) { mstate h; - for (unsigned i = 0; i < ARRAYLEN(g_heaps); ++i) + ACQUIRE_MALLOC_GLOBAL_LOCK(); + for (unsigned i = ARRAYLEN(g_heaps); i--;) if ((h = atomic_load_explicit(&g_heaps[i], memory_order_acquire))) ACQUIRE_LOCK(&h->mutex); } -static void dlmalloc_post_fork_parent(void) { +void dlmalloc_post_fork_parent(void) { mstate h; for (unsigned i = 0; i < ARRAYLEN(g_heaps); ++i) if ((h = atomic_load_explicit(&g_heaps[i], memory_order_acquire))) RELEASE_LOCK(&h->mutex); + RELEASE_MALLOC_GLOBAL_LOCK(); } -static void dlmalloc_post_fork_child(void) { +void dlmalloc_post_fork_child(void) { mstate h; for (unsigned i = 0; i < ARRAYLEN(g_heaps); ++i) if ((h = atomic_load_explicit(&g_heaps[i], memory_order_acquire))) - (void)INITIAL_LOCK(&h->mutex); + (void)REFRESH_LOCK(&h->mutex); + (void)REFRESH_MALLOC_GLOBAL_LOCK(); } #else -static void dlmalloc_pre_fork(void) { ACQUIRE_LOCK(&(gm)->mutex); } -static void dlmalloc_post_fork_parent(void) { RELEASE_LOCK(&(gm)->mutex); } -static void dlmalloc_post_fork_child(void) { (void)INITIAL_LOCK(&(gm)->mutex); } +void dlmalloc_pre_fork(void) { ACQUIRE_LOCK(&(gm)->mutex); } +void dlmalloc_post_fork_parent(void) { RELEASE_LOCK(&(gm)->mutex); } +void dlmalloc_post_fork_child(void) { (void)REFRESH_LOCK(&(gm)->mutex); } #endif /* ONLY_MSPACES */ #endif /* LOCK_AT_FORK */ @@ -95,12 +98,6 @@ __attribute__((__constructor__(49))) int init_mparams(void) { (void)INITIAL_LOCK(&gm->mutex); #endif -#if LOCK_AT_FORK - pthread_atfork(&dlmalloc_pre_fork, - &dlmalloc_post_fork_parent, - &dlmalloc_post_fork_child); -#endif - { #if USE_DEV_RANDOM int fd; diff --git a/third_party/dlmalloc/locks.inc b/third_party/dlmalloc/locks.inc index 4e6c0198a9f..3079b8dcdef 100644 --- a/third_party/dlmalloc/locks.inc +++ b/third_party/dlmalloc/locks.inc @@ -1,3 +1,7 @@ +#include "libc/cosmo.h" +#include "libc/intrin/kprintf.h" +#include "libc/intrin/maps.h" +#include "libc/thread/thread.h" /* --------------------------- Lock preliminaries ------------------------ */ @@ -33,11 +37,20 @@ #define MLOCK_T atomic_uint +static int malloc_inlk(MLOCK_T *lk) { + atomic_store_explicit(lk, 0, memory_order_relaxed); + return 0; +} + static int malloc_wipe(MLOCK_T *lk) { atomic_store_explicit(lk, 0, memory_order_relaxed); return 0; } +static int malloc_kilk(MLOCK_T *lk) { + return 0; +} + static int malloc_lock(MLOCK_T *lk) { for (;;) { if (!atomic_exchange_explicit(lk, 1, memory_order_acquire)) @@ -49,36 +62,71 @@ static int malloc_lock(MLOCK_T *lk) { return 0; } -static int malloc_unlock(MLOCK_T *lk) { +static int malloc_unlk(MLOCK_T *lk) { atomic_store_explicit(lk, 0, memory_order_release); return 0; } #else -#define MLOCK_T nsync_mu +#define MLOCK_T struct MallocLock -static int malloc_wipe(MLOCK_T *lk) { +struct MallocLock { +#if DEBUG + void *edges; +#endif + nsync_mu mu; +}; + +static int malloc_inlk(MLOCK_T *lk) { bzero(lk, sizeof(*lk)); return 0; } +static int malloc_wipe(MLOCK_T *lk) { + bzero(&lk->mu, sizeof(lk->mu)); + return 0; +} + +static int malloc_kilk(MLOCK_T *lk) { + return 0; +} + static int malloc_lock(MLOCK_T *lk) { - nsync_mu_lock(lk); +#if DEBUG + __deadlock_check(lk, 0); +#endif + nsync_mu_lock(&lk->mu); +#if DEBUG + __deadlock_record(lk, 0); + __deadlock_track(lk, 0); +#endif return 0; } -static int malloc_unlock(MLOCK_T *lk) { - nsync_mu_unlock(lk); +static int malloc_unlk(MLOCK_T *lk) { +#if DEBUG + if (__deadlock_tracked(lk) == 0) { + kprintf("error: unlock malloc mutex not owned by caller: %t\n", lk); + DebugBreak(); + } +#endif + nsync_mu_unlock(&lk->mu); +#if DEBUG + __deadlock_untrack(lk); +#endif return 0; } #endif #define ACQUIRE_LOCK(lk) malloc_lock(lk) -#define RELEASE_LOCK(lk) malloc_unlock(lk) -#define INITIAL_LOCK(lk) malloc_wipe(lk) -#define DESTROY_LOCK(lk) malloc_wipe(lk) +#define RELEASE_LOCK(lk) malloc_unlk(lk) +#define INITIAL_LOCK(lk) malloc_inlk(lk) +#define REFRESH_LOCK(lk) malloc_wipe(lk) +#define DESTROY_LOCK(lk) malloc_kilk(lk) +#define INITIAL_MALLOC_GLOBAL_LOCK() INITIAL_LOCK(&malloc_global_mutex); +#define REFRESH_MALLOC_GLOBAL_LOCK() REFRESH_LOCK(&malloc_global_mutex); #define ACQUIRE_MALLOC_GLOBAL_LOCK() ACQUIRE_LOCK(&malloc_global_mutex); #define RELEASE_MALLOC_GLOBAL_LOCK() RELEASE_LOCK(&malloc_global_mutex); diff --git a/third_party/dlmalloc/mspaces.inc b/third_party/dlmalloc/mspaces.inc index 1f048d0eb18..d17d96549c8 100644 --- a/third_party/dlmalloc/mspaces.inc +++ b/third_party/dlmalloc/mspaces.inc @@ -368,12 +368,15 @@ void* mspace_realloc(mspace msp, void* oldmem, size_t bytes) { #endif /* FOOTERS */ if (!PREACTION(m)) { mchunkptr newp = try_realloc_chunk(m, oldp, nb, 1); - POSTACTION(m); if (newp != 0) { + /* [jart] fix realloc MT bug in DEBUG mode + https://github.com/intel/linux-sgx/issues/534 */ check_inuse_chunk(m, newp); + POSTACTION(m); mem = chunk2mem(newp); } else { + POSTACTION(m); mem = mspace_malloc(m, bytes); if (mem != 0) { size_t oc = chunksize(oldp) - overhead_for(oldp); @@ -407,11 +410,13 @@ void* mspace_realloc_in_place(mspace msp, void* oldmem, size_t bytes) { #endif /* FOOTERS */ if (!PREACTION(m)) { mchunkptr newp = try_realloc_chunk(m, oldp, nb, 0); - POSTACTION(m); if (newp == oldp) { + /* [jart] fix realloc_in_place MT bug in DEBUG mode + https://github.com/intel/linux-sgx/issues/534 */ check_inuse_chunk(m, newp); mem = oldmem; } + POSTACTION(m); } } } diff --git a/third_party/gdtoa/lock.c b/third_party/gdtoa/lock.c new file mode 100644 index 00000000000..85b0f5c8a66 --- /dev/null +++ b/third_party/gdtoa/lock.c @@ -0,0 +1,59 @@ +/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ +│ vi: set noet ft=c ts=8 sw=8 fenc=utf-8 :vi │ +╚──────────────────────────────────────────────────────────────────────────────╝ +│ │ +│ The author of this software is David M. Gay. │ +│ Please send bug reports to David M. Gay │ +│ or Justine Tunney │ +│ │ +│ Copyright (C) 1998, 1999 by Lucent Technologies │ +│ All Rights Reserved │ +│ │ +│ Permission to use, copy, modify, and distribute this software and │ +│ its documentation for any purpose and without fee is hereby │ +│ granted, provided that the above copyright notice appear in all │ +│ copies and that both that the copyright notice and this │ +│ permission notice and warranty disclaimer appear in supporting │ +│ documentation, and that the name of Lucent or any of its entities │ +│ not be used in advertising or publicity pertaining to │ +│ distribution of the software without specific, written prior │ +│ permission. │ +│ │ +│ LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, │ +│ INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. │ +│ IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY │ +│ SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES │ +│ WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER │ +│ IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, │ +│ ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF │ +│ THIS SOFTWARE. │ +│ │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "third_party/gdtoa/lock.h" + +pthread_mutex_t __gdtoa_lock_obj = PTHREAD_MUTEX_INITIALIZER; +pthread_mutex_t __gdtoa_lock1_obj = PTHREAD_MUTEX_INITIALIZER; + +void +__gdtoa_lock(void) +{ + pthread_mutex_lock(&__gdtoa_lock_obj); +} + +void +__gdtoa_unlock(void) +{ + pthread_mutex_unlock(&__gdtoa_lock_obj); +} + +void +__gdtoa_lock1(void) +{ + pthread_mutex_lock(&__gdtoa_lock1_obj); +} + +void +__gdtoa_unlock1(void) +{ + pthread_mutex_unlock(&__gdtoa_lock1_obj); +} diff --git a/third_party/gdtoa/lock.h b/third_party/gdtoa/lock.h new file mode 100644 index 00000000000..e630e31e1f3 --- /dev/null +++ b/third_party/gdtoa/lock.h @@ -0,0 +1,15 @@ +#ifndef COSMOPOLITAN_THIRD_PARTY_GDTOA_LOCK_H_ +#define COSMOPOLITAN_THIRD_PARTY_GDTOA_LOCK_H_ +#include "libc/thread/thread.h" +COSMOPOLITAN_C_START_ + +extern pthread_mutex_t __gdtoa_lock_obj; +extern pthread_mutex_t __gdtoa_lock1_obj; + +void __gdtoa_lock(void); +void __gdtoa_unlock(void); +void __gdtoa_lock1(void); +void __gdtoa_unlock1(void); + +COSMOPOLITAN_C_END_ +#endif /* COSMOPOLITAN_THIRD_PARTY_GDTOA_LOCK_H_ */ diff --git a/third_party/gdtoa/misc.c b/third_party/gdtoa/misc.c index 0ff9afa1242..2d3809a9c10 100644 --- a/third_party/gdtoa/misc.c +++ b/third_party/gdtoa/misc.c @@ -35,46 +35,9 @@ #include "libc/thread/thread.h" #include "libc/thread/tls.h" #include "third_party/gdtoa/gdtoa.internal.h" +#include "third_party/gdtoa/lock.h" static ThInfo TI0; -static pthread_mutex_t __gdtoa_lock_obj; -static pthread_mutex_t __gdtoa_lock1_obj; - -static void -__gdtoa_lock(void) -{ - pthread_mutex_lock(&__gdtoa_lock_obj); -} - -static void -__gdtoa_unlock(void) -{ - pthread_mutex_unlock(&__gdtoa_lock_obj); -} - -static void -__gdtoa_initlock(void) -{ - pthread_mutex_init(&__gdtoa_lock_obj, 0); -} - -static void -__gdtoa_lock1(void) -{ - pthread_mutex_lock(&__gdtoa_lock1_obj); -} - -static void -__gdtoa_unlock1(void) -{ - pthread_mutex_unlock(&__gdtoa_lock1_obj); -} - -static void -__gdtoa_initlock1(void) -{ - pthread_mutex_init(&__gdtoa_lock1_obj, 0); -} static void __gdtoa_Brelease(Bigint *rv) @@ -88,24 +51,20 @@ static void __gdtoa_Bclear(void) { int i; - __gdtoa_lock(); + __gdtoa_lock1(); for (i = 0; i < ARRAYLEN(TI0.Freelist); ++i) __gdtoa_Brelease(TI0.Freelist[i]); - __gdtoa_lock1(); + __gdtoa_lock(); __gdtoa_Brelease(TI0.P5s); - __gdtoa_unlock1(); - bzero(&TI0, sizeof(TI0)); __gdtoa_unlock(); + bzero(&TI0, sizeof(TI0)); + __gdtoa_unlock1(); } __attribute__((__constructor__(60))) static void __gdtoa_Binit(void) { - __gdtoa_initlock(); - __gdtoa_initlock1(); atexit(__gdtoa_Bclear); - pthread_atfork(__gdtoa_lock1, __gdtoa_unlock1, __gdtoa_initlock1); - pthread_atfork(__gdtoa_lock, __gdtoa_unlock, __gdtoa_initlock); } static ThInfo * diff --git a/third_party/lua/llock.c b/third_party/lua/llock.c index 9a0f0bfbb18..359140f5502 100644 --- a/third_party/lua/llock.c +++ b/third_party/lua/llock.c @@ -19,12 +19,16 @@ #include "libc/thread/thread.h" #include "third_party/lua/lrepl.h" -static pthread_mutex_t lua_repl_lock_obj; +static pthread_mutex_t lua_repl_lock_obj = PTHREAD_MUTEX_INITIALIZER; -void(lua_repl_lock)(void) { +void lua_repl_wock(void) { + lua_repl_lock_obj = (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER; +} + +void lua_repl_lock(void) { pthread_mutex_lock(&lua_repl_lock_obj); } -void(lua_repl_unlock)(void) { +void lua_repl_unlock(void) { pthread_mutex_unlock(&lua_repl_lock_obj); } diff --git a/third_party/lua/lrepl.h b/third_party/lua/lrepl.h index a2294c5ca95..7d08b07309c 100644 --- a/third_party/lua/lrepl.h +++ b/third_party/lua/lrepl.h @@ -11,6 +11,7 @@ extern struct linenoiseState *lua_repl_linenoise; extern linenoiseCompletionCallback *lua_repl_completions_callback; void lua_freerepl(void); +void lua_repl_wock(void); void lua_repl_lock(void); void lua_repl_unlock(void); int lua_loadline(lua_State *); diff --git a/third_party/lua/lunix.c b/third_party/lua/lunix.c index b8e4219c70f..f5007e414a7 100644 --- a/third_party/lua/lunix.c +++ b/third_party/lua/lunix.c @@ -2959,7 +2959,7 @@ static int LuaUnixMapshared(lua_State *L) { m->mapsize = c; m->lock = (pthread_mutex_t *)p; pthread_mutexattr_init(&mattr); - pthread_mutexattr_settype(&mattr, PTHREAD_MUTEX_NORMAL); + pthread_mutexattr_settype(&mattr, PTHREAD_MUTEX_DEFAULT); pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED); pthread_mutex_init(m->lock, &mattr); pthread_mutexattr_destroy(&mattr); diff --git a/third_party/nsync/common.c b/third_party/nsync/common.c index a7fd4a068c6..c3d2c764d03 100644 --- a/third_party/nsync/common.c +++ b/third_party/nsync/common.c @@ -39,6 +39,7 @@ #include "third_party/nsync/common.internal.h" #include "third_party/nsync/mu_semaphore.h" #include "third_party/nsync/mu_semaphore.internal.h" +#include "libc/intrin/kprintf.h" #include "third_party/nsync/wait_s.internal.h" __static_yoink("nsync_notice"); diff --git a/third_party/nsync/mu_semaphore_sem.c b/third_party/nsync/mu_semaphore_sem.c index 41d92acfa0c..4ae67cb84bd 100644 --- a/third_party/nsync/mu_semaphore_sem.c +++ b/third_party/nsync/mu_semaphore_sem.c @@ -78,7 +78,7 @@ static bool nsync_mu_semaphore_sem_create (struct sem *f) { return true; } -static void nsync_mu_semaphore_sem_fork_child (void) { +void nsync_mu_semaphore_sem_fork_child (void) { struct sem *f; for (f = atomic_load_explicit (&g_sems, memory_order_relaxed); f; f = f->next) { int rc = sys_close (f->id); @@ -87,17 +87,11 @@ static void nsync_mu_semaphore_sem_fork_child (void) { } } -static void nsync_mu_semaphore_sem_init (void) { - pthread_atfork (0, 0, nsync_mu_semaphore_sem_fork_child); -} - /* Initialize *s; the initial value is 0. */ bool nsync_mu_semaphore_init_sem (nsync_semaphore *s) { - static atomic_uint once; struct sem *f = (struct sem *) s; if (!nsync_mu_semaphore_sem_create (f)) return false; - cosmo_once (&once, nsync_mu_semaphore_sem_init); sems_push(f); return true; } diff --git a/third_party/nsync/panic.c b/third_party/nsync/panic.c index 2ffd4086ab0..7c6cebf3818 100644 --- a/third_party/nsync/panic.c +++ b/third_party/nsync/panic.c @@ -28,5 +28,5 @@ void nsync_panic_ (const char *s) { "cosmoaddr2line ", program_invocation_name, " ", DescribeBacktrace (__builtin_frame_address (0)), "\n", NULL); - _Exit (44); + __builtin_trap (); } diff --git a/third_party/tz/localtime.c b/third_party/tz/localtime.c index 06139e49f9e..34f7cf64853 100644 --- a/third_party/tz/localtime.c +++ b/third_party/tz/localtime.c @@ -2,6 +2,10 @@ │ vi: set noet ft=c ts=8 sw=8 fenc=utf-8 :vi │ ╚─────────────────────────────────────────────────────────────────────────────*/ #define LOCALTIME_IMPLEMENTATION +#include "lock.h" +#include "tzdir.h" +#include "tzfile.h" +#include "private.h" #include "libc/calls/blockcancel.internal.h" #include "libc/calls/calls.h" #include "libc/cxxabi.h" @@ -10,20 +14,15 @@ #include "libc/serialize.h" #include "libc/str/str.h" #include "libc/sysv/consts/o.h" -#include "libc/thread/thread.h" -#include "libc/thread/tls.h" #include "libc/time.h" #include "libc/inttypes.h" #include "libc/sysv/consts/ok.h" #include "libc/runtime/runtime.h" #include "libc/stdckdint.h" #include "libc/time.h" -#include "tzdir.h" -#include "tzfile.h" #include "libc/nt/struct/timezoneinformation.h" #include "libc/nt/time.h" #include "libc/dce.h" -#include "private.h" /* Convert timestamp from time_t to struct tm. */ @@ -624,34 +623,10 @@ localtime_windows_init(void) setenv("TZ", buf, true); } -static pthread_mutex_t locallock = PTHREAD_MUTEX_INITIALIZER; - -static dontinline void -localtime_wipe(void) -{ - pthread_mutex_init(&locallock, 0); -} - -static dontinline void -localtime_lock(void) -{ - pthread_mutex_lock(&locallock); -} - -static dontinline void -localtime_unlock(void) -{ - pthread_mutex_unlock(&locallock); -} - __attribute__((__constructor__(80))) textstartup static void localtime_init(void) { - localtime_wipe(); - pthread_atfork(localtime_lock, - localtime_unlock, - localtime_wipe); if (IsWindows()) localtime_windows_init(); } @@ -2052,9 +2027,9 @@ localtime_tzset_unlocked(void) void tzset(void) { - localtime_lock(); + __localtime_lock(); localtime_tzset_unlocked(); - localtime_unlock(); + __localtime_unlock(); } static void @@ -2067,7 +2042,7 @@ static void localtime_gmtcheck(void) { static bool gmt_is_set; - localtime_lock(); + __localtime_lock(); if (! gmt_is_set) { #ifdef ALL_STATE gmtptr = malloc(sizeof *gmtptr); @@ -2077,7 +2052,7 @@ localtime_gmtcheck(void) localtime_gmtload(gmtptr); gmt_is_set = true; } - localtime_unlock(); + __localtime_unlock(); } /* @@ -2193,11 +2168,11 @@ localsub(struct state const *sp, time_t const *timep, int_fast32_t setname, static struct tm * localtime_tzset(time_t const *timep, struct tm *tmp, bool setname) { - localtime_lock(); + __localtime_lock(); if (setname || !lcl_is_set) localtime_tzset_unlocked(); tmp = localsub(lclptr, timep, setname, tmp); - localtime_unlock(); + __localtime_unlock(); return tmp; } @@ -2834,10 +2809,10 @@ time_t mktime(struct tm *tmp) { time_t t; - localtime_lock(); + __localtime_lock(); localtime_tzset_unlocked(); t = mktime_tzname(lclptr, tmp, true); - localtime_unlock(); + __localtime_unlock(); return t; } diff --git a/third_party/tz/lock.h b/third_party/tz/lock.h new file mode 100644 index 00000000000..60070aad112 --- /dev/null +++ b/third_party/tz/lock.h @@ -0,0 +1,12 @@ +#ifndef COSMOPOLITAN_THIRD_PARTY_TZ_LOCK_H_ +#define COSMOPOLITAN_THIRD_PARTY_TZ_LOCK_H_ +#include "libc/thread/thread.h" +COSMOPOLITAN_C_START_ + +extern pthread_mutex_t __localtime_lock_obj; + +void __localtime_lock(void); +void __localtime_unlock(void); + +COSMOPOLITAN_C_END_ +#endif /* COSMOPOLITAN_THIRD_PARTY_TZ_LOCK_H_ */ diff --git a/tool/cosmocc/README.md b/tool/cosmocc/README.md index 362b2168153..84802987e44 100644 --- a/tool/cosmocc/README.md +++ b/tool/cosmocc/README.md @@ -191,15 +191,22 @@ The following supplemental flags are defined by cosmocc: - `-mdbg` may be passed when linking programs. It has the same effect as `export MODE=dbg` in that it will cause an alternative build of the Cosmopolitan Libc runtime to be linked that was built with `-O0 -g`. - Under the normal build mode, `--ftrace` output is oftentimes missing - important pieces of the puzzle due to inlining. This mode makes it - more comprehensible. It's also the only way to make using GDB to - troubleshoot issues inside Cosmo Libc work reliably. Please be warned - that this flag may enable some heavyweight runtime checks. For - example, mmap() will become O(n) rather than O(logn) in an effort to - spot data structure corruption. Lastly, the linked Cosmo runtime was - compiled with `-fsanitize=undefined` (UBSAN) although you still need - to pass that flag too if you want it for your own code. + Under the normal build mode, `--ftrace` output generated by your libc + is oftentimes missing important details due to inlining. If your build + your code with `cosmocc -O0 -mdbg` then `--ftrace` will make much more + sense. It's also the only way to make using GDB to troubleshoot issues + inside Cosmo Libc work reliably. Please be warned, this flag enables + some heavy-hitting runtime checks, such such lock graph validation. + The debug Cosmopolitan runtime is able to detect lock cycles globally + automatically via your normal usage of `pthread_mutex_t` and then + report strongly connected components with C++ symbol demangling. This + runtime will absolutely crash your entire process, if it helps you + spot a bug. For example, debug cosmo is build with UBSAN so even an + undiscovered yet innocent bit shift of a negative number could take + you down. So you wouldn't want to use this in prod very often. Please + note that passing `-mdbg` doesn't imply `-g -O0 -fsanitize=undefined` + which must be passed separately if you want your code to be compiled + with the same stuff as libc. - `-mtiny` may be passed when linking programs. It has the same effect as `export MODE=tiny` in that it will cause an alternative build of diff --git a/tool/net/redbean.c b/tool/net/redbean.c index e3b9ec65ae6..93816d1aa43 100644 --- a/tool/net/redbean.c +++ b/tool/net/redbean.c @@ -6799,6 +6799,8 @@ static int HandleConnection(size_t i) { } else { switch ((pid = fork())) { case 0: + lua_repl_wock(); + lua_repl_lock(); meltdown = false; __isworker = true; connectionclose = false;