support for TLS in dynamic-loaded (dlopen) modules

unlike other implementations, this one reserves memory for new TLS in
all pre-existing threads at dlopen-time, and dlopen will fail with no
resources consumed and no new libraries loaded if memory is not
available. memory is not immediately distributed to running threads;
that would be too complex and too costly. instead, assurances are made
that threads needing the new TLS can obtain it in an async-signal-safe
way from a buffer belonging to the dynamic linker/new module (via
atomic fetch-and-add based allocator).

I've re-appropriated the lock that was previously used for __synccall
(synchronizing set*id() syscalls between threads) as a general
pthread_create lock. it's a "backwards" rwlock where the "read"
operation is safe atomic modification of the live thread count, which
multiple threads can perform at the same time, and the "write"
operation is making sure the count does not increase during an
operation that depends on it remaining bounded (__synccall or dlopen).
in static-linked programs that don't use __synccall, this lock is a
no-op and has no cost.

diff --git a/src/env/__init_tls.c b/src/env/__init_tls.c
index aff388bda36cd6862cf4a487df44e5bd3c5574d1..b19bdb6493a3c00e6e646c9c9d13d9e0ea46de56 100644 (file)
--- a/src/env/__init_tls.c
+++ b/src/env/__init_tls.c
@@ -9,7 +9,7 @@
 static void *image;
 static size_t len, size, align;
 
-void *__copy_tls(unsigned char *mem, size_t cnt)
+void *__copy_tls(unsigned char *mem)
 {
        mem += -size & (4*sizeof(size_t)-1);
        mem += ((uintptr_t)image - (uintptr_t)mem) & (align-1);
@@ -64,7 +64,7 @@ void __init_tls(size_t *auxv)
                MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
        if (mem == MAP_FAILED) a_crash();
 
-       if (!__install_initial_tls(__copy_tls(mem, 0))) a_crash();
+       if (!__install_initial_tls(__copy_tls(mem))) a_crash();
 }
 #else
 void __init_tls(size_t *auxv) { }

diff --git a/src/internal/libc.h b/src/internal/libc.h
index ca06f3199340d04e92b8c5f69fa0b74a864236f5..f88d846d006930c19a35d3b887ba8051061d5716 100644 (file)
--- a/src/internal/libc.h
+++ b/src/internal/libc.h
@@ -16,7 +16,7 @@ struct __libc {
        int canceldisable;
        FILE *ofl_head;
        int ofl_lock[2];
-       volatile size_t tls_size, tls_cnt;
+       size_t tls_size;
 };
 
 extern size_t __hwcap;

diff --git a/src/internal/pthread_impl.h b/src/internal/pthread_impl.h
index f7facba35c0c4a8209d41ba9c21e95d398212db4..0f10cc48b116ff026fb59bf4dfd13deb0189deb3 100644 (file)
--- a/src/internal/pthread_impl.h
+++ b/src/internal/pthread_impl.h
@@ -111,8 +111,9 @@ int __timedwait(volatile int *, int, clockid_t, const struct timespec *, void (*
 void __wait(volatile int *, volatile int *, int, int);
 void __wake(volatile int *, int, int);
 
-void __synccall_lock();
-void __synccall_unlock();
+void __acquire_ptc();
+void __release_ptc();
+void __inhibit_ptc();
 
 #define DEFAULT_STACK_SIZE 81920
 #define DEFAULT_GUARD_SIZE PAGE_SIZE

diff --git a/src/ldso/dynlink.c b/src/ldso/dynlink.c
index 2bbd492a3ff7fd2aaaaa331b159bc1c504230607..f2bcd0f3d401e8dca13178d3b1a8dba64866dd0d 100644 (file)
--- a/src/ldso/dynlink.c
+++ b/src/ldso/dynlink.c
@@ -69,6 +69,9 @@ struct dso {
        struct dso **deps;
        void *tls_image;
        size_t tls_len, tls_size, tls_align, tls_id, tls_offset;
+       void **new_dtv;
+       unsigned char *new_tls;
+       int new_dtv_idx, new_tls_idx;
        char *shortname;
        char buf[];
 };
@@ -420,6 +423,8 @@ static struct dso *load_library(const char *name)
        struct dso *p, temp_dso = {0};
        int fd;
        struct stat st;
+       size_t alloc_size;
+       int n_th = 0;
 
        /* Catch and block attempts to reload the implementation itself */
        if (name[0]=='l' && name[1]=='i' && name[2]=='b') {
@@ -487,18 +492,27 @@ static struct dso *load_library(const char *name)
        map = map_library(fd, &temp_dso);
        close(fd);
        if (!map) return 0;
-       p = malloc(sizeof *p + strlen(pathname) + 1);
+
+       /* Allocate storage for the new DSO. When there is TLS, this
+        * storage must include a reservation for all pre-existing
+        * threads to obtain copies of both the new TLS, and an
+        * extended DTV capable of storing an additional slot for
+        * the newly-loaded DSO. */
+       alloc_size = sizeof *p + strlen(pathname) + 1;
+       if (runtime && temp_dso.tls_image) {
+               size_t per_th = temp_dso.tls_size + temp_dso.tls_align
+                       + sizeof(void *) * (tls_cnt+3);
+               n_th = __libc.threads_minus_1 + 1;
+               if (n_th > SSIZE_MAX / per_th) alloc_size = SIZE_MAX;
+               else alloc_size += n_th * per_th;
+       }
+       p = calloc(1, alloc_size);
        if (!p) {
                munmap(map, map_len);
                return 0;
        }
        memcpy(p, &temp_dso, sizeof temp_dso);
        decode_dyn(p);
-       if (p->tls_image) {
-               p->tls_id = ++tls_cnt;
-               tls_size += p->tls_size + p->tls_align + 8*sizeof(size_t) - 1
-                       & -4*sizeof(size_t);
-       }
        p->dev = st.st_dev;
        p->ino = st.st_ino;
        p->refcnt = 1;
@@ -506,6 +520,14 @@ static struct dso *load_library(const char *name)
        strcpy(p->name, pathname);
        /* Add a shortname only if name arg was not an explicit pathname. */
        if (pathname != name) p->shortname = strrchr(p->name, '/')+1;
+       if (p->tls_image) {
+               p->tls_id = ++tls_cnt;
+               tls_size += p->tls_size + p->tls_align + 8*sizeof(size_t) - 1
+                       & -4*sizeof(size_t);
+               p->new_dtv = (void *)(-sizeof(size_t) &
+                       (uintptr_t)(p->name+strlen(p->name)+sizeof(size_t)));
+               p->new_tls = (void *)(p->new_dtv + n_th*(tls_cnt+1));
+       }
 
        tail->next = p;
        p->prev = tail;
@@ -637,14 +659,14 @@ void _dl_debug_state(void)
 {
 }
 
-void *__copy_tls(unsigned char *mem, size_t cnt)
+void *__copy_tls(unsigned char *mem)
 {
        struct dso *p;
        void **dtv = (void *)mem;
-       dtv[0] = (void *)cnt;
-       mem = (void *)(dtv + cnt + 1);
+       dtv[0] = (void *)tls_cnt;
+       mem = (void *)(dtv + tls_cnt + 1);
        for (p=tail; p; p=p->prev) {
-               if (p->tls_id-1 >= cnt) continue;
+               if (!p->tls_id) continue;
                mem += -p->tls_len & (4*sizeof(size_t)-1);
                mem += ((uintptr_t)p->tls_image - (uintptr_t)mem)
                        & (p->tls_align-1);
@@ -656,14 +678,46 @@ void *__copy_tls(unsigned char *mem, size_t cnt)
        return mem;
 }
 
-void *__tls_get_addr(size_t *p)
+void *__tls_get_addr(size_t *v)
 {
        pthread_t self = __pthread_self();
-       if ((size_t)self->dtv[0] < p[0]) {
-               // FIXME: obtain new DTV and TLS from the DSO
-               a_crash();
+       if (self->dtv && v[0]<=(size_t)self->dtv[0] && self->dtv[v[0]])
+               return (char *)self->dtv[v[0]]+v[1];
+
+       /* Block signals to make accessing new TLS async-signal-safe */
+       sigset_t set;
+       sigfillset(&set);
+       pthread_sigmask(SIG_BLOCK, &set, &set);
+       if (self->dtv && v[0]<=(size_t)self->dtv[0] && self->dtv[v[0]]) {
+               pthread_sigmask(SIG_SETMASK, &set, 0);
+               return (char *)self->dtv[v[0]]+v[1];
+       }
+
+       /* This is safe without any locks held because, if the caller
+        * is able to request the Nth entry of the DTV, the DSO list
+        * must be valid at least that far out and it was synchronized
+        * at program startup or by an already-completed call to dlopen. */
+       struct dso *p;
+       for (p=head; p->tls_id != v[0]; p=p->next);
+
+       /* Get new DTV space from new DSO if needed */
+       if (!self->dtv || v[0] > (size_t)self->dtv[0]) {
+               void **newdtv = p->new_dtv +
+                       (v[0]+1)*sizeof(void *)*a_fetch_add(&p->new_dtv_idx,1);
+               if (self->dtv) memcpy(newdtv, self->dtv,
+                       ((size_t)self->dtv[0]+1) * sizeof(void *));
+               newdtv[0] = (void *)v[0];
+               self->dtv = newdtv;
        }
-       return (char *)self->dtv[p[0]] + p[1];
+
+       /* Get new TLS memory from new DSO */
+       unsigned char *mem = p->new_tls +
+               (p->tls_size + p->tls_align) * a_fetch_add(&p->new_tls_idx,1);
+       mem += ((uintptr_t)p->tls_image - (uintptr_t)mem) & (p->tls_align-1);
+       self->dtv[v[0]] = mem;
+       memcpy(mem, p->tls_image, p->tls_size);
+       pthread_sigmask(SIG_SETMASK, &set, 0);
+       return mem + v[1];
 }
 
 void *__dynlink(int argc, char **argv)
@@ -830,13 +884,12 @@ void *__dynlink(int argc, char **argv)
         * to copy the TLS images again in case they had relocs. */
        tls_size += sizeof(struct pthread) + 4*sizeof(size_t);
        __libc.tls_size = tls_size;
-       __libc.tls_cnt = tls_cnt;
        if (tls_cnt) {
                struct dso *p;
                void *mem = mmap(0, __libc.tls_size, PROT_READ|PROT_WRITE,
                        MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
                if (mem==MAP_FAILED ||
-                   !__install_initial_tls(__copy_tls(mem, tls_cnt))) {
+                   !__install_initial_tls(__copy_tls(mem))) {
                        dprintf(2, "%s: Error getting %zu bytes thread-local storage: %m\n",
                                argv[0], tls_size);
                        _exit(127);
@@ -853,7 +906,7 @@ void *__dynlink(int argc, char **argv)
 
        /* The initial DTV is located at the base of the memory
         * allocated for TLS. Repeat copying TLS to pick up relocs. */
-       if (tls_cnt) __copy_tls((void *)__pthread_self()->dtv, tls_cnt);
+       if (tls_cnt) __copy_tls((void *)__pthread_self()->dtv);
 
        if (ldso_fail) _exit(127);
        if (ldd_mode) _exit(0);
@@ -887,6 +940,7 @@ void *__dynlink(int argc, char **argv)
 void *dlopen(const char *file, int mode)
 {
        struct dso *volatile p, *orig_tail, *next;
+       size_t orig_tls_cnt;
        size_t i;
        int cs;
 
@@ -894,12 +948,15 @@ void *dlopen(const char *file, int mode)
 
        pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cs);
        pthread_rwlock_wrlock(&lock);
+       __inhibit_ptc();
 
+       p = 0;
+       orig_tls_cnt = tls_cnt;
        orig_tail = tail;
 
        if (setjmp(rtld_fail)) {
                /* Clean up anything new that was (partially) loaded */
-               if (p->deps) for (i=0; p->deps[i]; i++)
+               if (p && p->deps) for (i=0; p->deps[i]; i++)
                        if (p->deps[i]->global < 0)
                                p->deps[i]->global = 0;
                for (p=orig_tail->next; p; p=next) {
@@ -908,6 +965,8 @@ void *dlopen(const char *file, int mode)
                        free(p->deps);
                        free(p);
                }
+               tls_cnt = orig_tls_cnt;
+               tls_size = __libc.tls_size;
                tail = orig_tail;
                tail->next = 0;
                p = 0;
@@ -942,12 +1001,15 @@ void *dlopen(const char *file, int mode)
                p->global = 1;
        }
 
+       __libc.tls_size = tls_size;
+
        if (ssp_used) __init_ssp(auxv);
 
        _dl_debug_state();
 
        do_init_fini(tail);
 end:
+       __release_ptc();
        pthread_rwlock_unlock(&lock);
        pthread_setcancelstate(cs, 0);
        return p;

diff --git a/src/thread/lock_ptc.c b/src/thread/lock_ptc.c
new file mode 100644 (file)
index 0000000..7adedab
--- /dev/null
+++ b/src/thread/lock_ptc.c
@@ -0,0 +1,18 @@
+#include <pthread.h>
+
+static pthread_rwlock_t lock = PTHREAD_RWLOCK_INITIALIZER;
+
+void __inhibit_ptc()
+{
+       pthread_rwlock_wrlock(&lock);
+}
+
+void __acquire_ptc()
+{
+       pthread_rwlock_rdlock(&lock);
+}
+
+void __release_ptc()
+{
+       pthread_rwlock_unlock(&lock);
+}

diff --git a/src/thread/pthread_create.c b/src/thread/pthread_create.c
index f53fc1baf0a5e5faf35f51f025e9a50fad4c4f24..92ce9ffb5d6e7852445de7eb1bb0d03571132b6b 100644 (file)
--- a/src/thread/pthread_create.c
+++ b/src/thread/pthread_create.c
@@ -4,8 +4,8 @@
 static void dummy_0()
 {
 }
-weak_alias(dummy_0, __synccall_lock);
-weak_alias(dummy_0, __synccall_unlock);
+weak_alias(dummy_0, __acquire_ptc);
+weak_alias(dummy_0, __release_ptc);
 weak_alias(dummy_0, __pthread_tsd_run_dtors);
 
 _Noreturn void pthread_exit(void *result)
@@ -84,7 +84,7 @@ static void init_file_lock(FILE *f)
        if (f && f->lock<0) f->lock = 0;
 }
 
-void *__copy_tls(unsigned char *, size_t);
+void *__copy_tls(unsigned char *);
 
 int pthread_create(pthread_t *restrict res, const pthread_attr_t *restrict attr, void *(*entry)(void *), void *restrict arg)
 {
@@ -94,8 +94,6 @@ int pthread_create(pthread_t *restrict res, const pthread_attr_t *restrict attr,
        struct pthread *self = pthread_self(), *new;
        unsigned char *map, *stack, *tsd;
        unsigned flags = 0x7d8f00;
-       size_t tls_cnt = libc.tls_cnt;
-       size_t tls_size = libc.tls_size;
 
        if (!self) return ENOSYS;
        if (!libc.threaded) {
@@ -107,6 +105,8 @@ int pthread_create(pthread_t *restrict res, const pthread_attr_t *restrict attr,
                libc.threaded = 1;
        }
 
+       __acquire_ptc();
+
        if (attr && attr->_a_stackaddr) {
                map = 0;
                tsd = (void *)(attr->_a_stackaddr-__pthread_tsd_size & -16);
@@ -114,7 +114,7 @@ int pthread_create(pthread_t *restrict res, const pthread_attr_t *restrict attr,
                if (attr) {
                        guard = ROUND(attr->_a_guardsize + DEFAULT_GUARD_SIZE);
                        size = guard + ROUND(attr->_a_stacksize
-                               + DEFAULT_STACK_SIZE + tls_size);
+                               + DEFAULT_STACK_SIZE + libc.tls_size);
                }
                size += __pthread_tsd_size;
                if (guard) {
@@ -130,7 +130,7 @@ int pthread_create(pthread_t *restrict res, const pthread_attr_t *restrict attr,
                }
                tsd = map + size - __pthread_tsd_size;
        }
-       new = __copy_tls(tsd - tls_size, tls_cnt);
+       new = __copy_tls(tsd - libc.tls_size);
        new->map_base = map;
        new->map_size = size;
        new->pid = self->pid;
@@ -147,12 +147,10 @@ int pthread_create(pthread_t *restrict res, const pthread_attr_t *restrict attr,
        new->canary = self->canary ^ (uintptr_t)&new;
        stack = (void *)new;
 
-       __synccall_lock();
-
        a_inc(&libc.threads_minus_1);
        ret = __clone(start, stack, flags, new, &new->tid, new, &new->tid);
 
-       __synccall_unlock();
+       __release_ptc();
 
        if (ret < 0) {
                a_dec(&libc.threads_minus_1);

diff --git a/src/thread/synccall.c b/src/thread/synccall.c
index fd377cb360b0bdd9b8f14fc0306148c436ee28e3..2b7eac259ce2f3332a09cf3398014d8b99b5e426 100644 (file)
--- a/src/thread/synccall.c
+++ b/src/thread/synccall.c
@@ -9,7 +9,6 @@ static struct chain {
 static void (*callback)(void *), *context;
 static int chainlen;
 static sem_t chainlock, chaindone;
-static pthread_rwlock_t lock = PTHREAD_RWLOCK_INITIALIZER;
 
 static void handler(int sig, siginfo_t *si, void *ctx)
 {
@@ -59,7 +58,7 @@ void __synccall(void (*func)(void *), void *ctx)
                return;
        }
 
-       pthread_rwlock_wrlock(&lock);
+       __inhibit_ptc();
 
        __syscall(SYS_rt_sigprocmask, SIG_BLOCK, SIGALL_SET,
                &oldmask, __SYSCALL_SSLEN);
@@ -97,15 +96,5 @@ void __synccall(void (*func)(void *), void *ctx)
        __syscall(SYS_rt_sigprocmask, SIG_SETMASK,
                &oldmask, 0, __SYSCALL_SSLEN);
 
-       pthread_rwlock_unlock(&lock);
-}
-
-void __synccall_lock()
-{
-       pthread_rwlock_rdlock(&lock);
-}
-
-void __synccall_unlock()
-{
-       pthread_rwlock_unlock(&lock);
+       __release_ptc();
 }