restore lock-skipping for processes that return to single-threaded state

the design used here relies on the barrier provided by the first lock
operation after the process returns to single-threaded state to
synchronize with actions by the last thread that exited. by storing
the intent to change modes in the same object used to detect whether
locking is needed, it's possible to avoid an extra (possibly costly)
memory load after the lock is taken.

diff --git a/src/internal/libc.h b/src/internal/libc.h
index d47f58e01df958988347a2fcc9fd63ff3703ba19..619bba8613e8d20cc00c69eec83dd054348b5005 100644 (file)
--- a/src/internal/libc.h
+++ b/src/internal/libc.h
@@ -21,6 +21,7 @@ struct __libc {
        char can_do_threads;
        char threaded;
        char secure;
+       volatile signed char need_locks;
        int threads_minus_1;
        size_t *auxv;
        struct tls_module *tls_head;

diff --git a/src/malloc/malloc.c b/src/malloc/malloc.c
index 2553a62e0d09d0b6e0fec141c439f57a4028535e..a803d4c938bd2cebc2c82acf3eaaf969f2702917 100644 (file)
--- a/src/malloc/malloc.c
+++ b/src/malloc/malloc.c
@@ -26,8 +26,11 @@ int __malloc_replaced;
 
 static inline void lock(volatile int *lk)
 {
-       if (libc.threaded)
+       int need_locks = libc.need_locks;
+       if (need_locks) {
                while(a_swap(lk, 1)) __wait(lk, lk+1, 1, 1);
+               if (need_locks < 0) libc.need_locks = 0;
+       }
 }
 
 static inline void unlock(volatile int *lk)

diff --git a/src/thread/__lock.c b/src/thread/__lock.c
index 5b9b144e9c83c14f6cb54f80af889b060ec62660..60eece49a28888dc3dd1a4498c0c6d9dd85effd4 100644 (file)
--- a/src/thread/__lock.c
+++ b/src/thread/__lock.c
@@ -18,9 +18,11 @@
 
 void __lock(volatile int *l)
 {
-       if (!libc.threaded) return;
+       int need_locks = libc.need_locks;
+       if (!need_locks) return;
        /* fast path: INT_MIN for the lock, +1 for the congestion */
        int current = a_cas(l, 0, INT_MIN + 1);
+       if (need_locks < 0) libc.need_locks = 0;
        if (!current) return;
        /* A first spin loop, for medium congestion. */
        for (unsigned i = 0; i < 10; ++i) {

diff --git a/src/thread/pthread_create.c b/src/thread/pthread_create.c
index 6a3b0c21659a4e7bb1a822f9fea070bdc9de2f91..6bdfb44f9cde29f656dcca6763be610f05204cd4 100644 (file)
--- a/src/thread/pthread_create.c
+++ b/src/thread/pthread_create.c
@@ -118,8 +118,8 @@ _Noreturn void __pthread_exit(void *result)
         * until the lock is released, which only happens after SYS_exit
         * has been called, via the exit futex address pointing at the lock.
         * This needs to happen after any possible calls to LOCK() that might
-        * skip locking if libc.threads_minus_1 is zero. */
-       libc.threads_minus_1--;
+        * skip locking if process appears single-threaded. */
+       if (!--libc.threads_minus_1) libc.need_locks = -1;
        self->next->prev = self->prev;
        self->prev->next = self->next;
        self->prev = self->next = self;
@@ -339,7 +339,7 @@ int __pthread_create(pthread_t *restrict res, const pthread_attr_t *restrict att
                ~(1UL<<((SIGCANCEL-1)%(8*sizeof(long))));
 
        __tl_lock();
-       libc.threads_minus_1++;
+       if (!libc.threads_minus_1++) libc.need_locks = 1;
        ret = __clone((c11 ? start_c11 : start), stack, flags, args, &new->tid, TP_ADJ(new), &__thread_list_lock);
 
        /* All clone failures translate to EAGAIN. If explicit scheduling
@@ -363,7 +363,7 @@ int __pthread_create(pthread_t *restrict res, const pthread_attr_t *restrict att
                new->next->prev = new;
                new->prev->next = new;
        } else {
-               libc.threads_minus_1--;
+               if (!--libc.threads_minus_1) libc.need_locks = 0;
        }
        __tl_unlock();
        __restore_sigs(&set);