combine arch ABI's DTP_OFFSET into DTV pointers

as explained in commit 6ba5517a460c6c438f64d69464fdfc3269a4c91a, some
archs use an offset (typicaly -0x8000) with their DTPOFF relocations,
which __tls_get_addr needs to invert. on affected archs, which lack
direct support for large immediates, this can cost multiple extra
instructions in the hot path. instead, incorporate the DTP_OFFSET into
the DTV entries. this means they are no longer valid pointers, so
store them as an array of uintptr_t rather than void *; this also
makes it easier to access slot 0 as a valid slot count.

commit e75b16cf93ebbc1ce758d3ea6b2923e8b2457c68 left behind cruft in
two places, __reset_tls and __tls_get_new, from back when it was
possible to have uninitialized gap slots indicated by a null pointer
in the DTV. since the concept of null pointer is no longer meaningful
with an offset applied, remove this cruft.

presently there are no archs with both TLSDESC and nonzero DTP_OFFSET,
but the dynamic TLSDESC relocation code is also updated to apply an
inverted offset to its offset field, so that the offset DTV would not
impose a runtime cost in TLSDESC resolver functions.

diff --git a/ldso/dynlink.c b/ldso/dynlink.c
index c2892b9032d7fbc2e1002cc4b08d0a73f9750598..a3ca3cdf03f0f2a6988463dd50e9e57e39a75338 100644 (file)
--- a/ldso/dynlink.c
+++ b/ldso/dynlink.c
@@ -72,7 +72,7 @@ struct dso {
        struct tls_module tls;
        size_t tls_id;
        size_t relro_start, relro_end;
-       void **new_dtv;
+       uintptr_t *new_dtv;
        unsigned char *new_tls;
        volatile int new_dtv_idx, new_tls_idx;
        struct td_index *td_index;
@@ -445,7 +445,7 @@ static void do_relocs(struct dso *dso, size_t *rel, size_t rel_size, size_t stri
                                new->next = dso->td_index;
                                dso->td_index = new;
                                new->args[0] = def.dso->tls_id;
-                               new->args[1] = tls_val + addend;
+                               new->args[1] = tls_val + addend - DTP_OFFSET;
                                reloc_addr[0] = (size_t)__tlsdesc_dynamic;
                                reloc_addr[1] = (size_t)new;
                        } else {
@@ -1345,9 +1345,9 @@ hidden void *__tls_get_new(tls_mod_off_t *v)
        /* Block signals to make accessing new TLS async-signal-safe */
        sigset_t set;
        __block_all_sigs(&set);
-       if (v[0]<=(size_t)self->dtv[0]) {
+       if (v[0] <= self->dtv[0]) {
                __restore_sigs(&set);
-               return (char *)self->dtv[v[0]]+v[1]+DTP_OFFSET;
+               return (void *)(self->dtv[v[0]] + v[1]);
        }
 
        /* This is safe without any locks held because, if the caller
@@ -1357,15 +1357,12 @@ hidden void *__tls_get_new(tls_mod_off_t *v)
        struct dso *p;
        for (p=head; p->tls_id != v[0]; p=p->next);
 
-       /* Get new DTV space from new DSO if needed */
-       if (v[0] > (size_t)self->dtv[0]) {
-               void **newdtv = p->new_dtv +
-                       (v[0]+1)*a_fetch_add(&p->new_dtv_idx,1);
-               memcpy(newdtv, self->dtv,
-                       ((size_t)self->dtv[0]+1) * sizeof(void *));
-               newdtv[0] = (void *)v[0];
-               self->dtv = self->dtv_copy = newdtv;
-       }
+       /* Get new DTV space from new DSO */
+       uintptr_t *newdtv = p->new_dtv +
+               (v[0]+1)*a_fetch_add(&p->new_dtv_idx,1);
+       memcpy(newdtv, self->dtv, (self->dtv[0]+1) * sizeof(uintptr_t));
+       newdtv[0] = v[0];
+       self->dtv = self->dtv_copy = newdtv;
 
        /* Get new TLS memory from all new DSOs up to the requested one */
        unsigned char *mem;
@@ -1375,7 +1372,7 @@ hidden void *__tls_get_new(tls_mod_off_t *v)
                        * a_fetch_add(&p->new_tls_idx,1);
                mem += ((uintptr_t)p->tls.image - (uintptr_t)mem)
                        & (p->tls.align-1);
-               self->dtv[p->tls_id] = mem;
+               self->dtv[p->tls_id] = (uintptr_t)mem + DTP_OFFSET;
                memcpy(mem, p->tls.image, p->tls.len);
                if (p->tls_id == v[0]) break;
        }

diff --git a/src/env/__init_tls.c b/src/env/__init_tls.c
index 96d0e2843da00caf57d101a61291fece78369153..842886f6c4661bf67932286a81899a222294bd57 100644 (file)
--- a/src/env/__init_tls.c
+++ b/src/env/__init_tls.c
@@ -36,32 +36,32 @@ void *__copy_tls(unsigned char *mem)
        pthread_t td;
        struct tls_module *p;
        size_t i;
-       void **dtv;
+       uintptr_t *dtv;
 
 #ifdef TLS_ABOVE_TP
-       dtv = (void **)(mem + libc.tls_size) - (libc.tls_cnt + 1);
+       dtv = (uintptr_t*)(mem + libc.tls_size) - (libc.tls_cnt + 1);
 
        mem += -((uintptr_t)mem + sizeof(struct pthread)) & (libc.tls_align-1);
        td = (pthread_t)mem;
        mem += sizeof(struct pthread);
 
        for (i=1, p=libc.tls_head; p; i++, p=p->next) {
-               dtv[i] = mem + p->offset;
-               memcpy(dtv[i], p->image, p->len);
+               dtv[i] = (uintptr_t)(mem + p->offset) + DTP_OFFSET;
+               memcpy(mem + p->offset, p->image, p->len);
        }
 #else
-       dtv = (void **)mem;
+       dtv = (uintptr_t *)mem;
 
        mem += libc.tls_size - sizeof(struct pthread);
        mem -= (uintptr_t)mem & (libc.tls_align-1);
        td = (pthread_t)mem;
 
        for (i=1, p=libc.tls_head; p; i++, p=p->next) {
-               dtv[i] = mem - p->offset;
-               memcpy(dtv[i], p->image, p->len);
+               dtv[i] = (uintptr_t)(mem - p->offset) + DTP_OFFSET;
+               memcpy(mem - p->offset, p->image, p->len);
        }
 #endif
-       dtv[0] = (void *)libc.tls_cnt;
+       dtv[0] = libc.tls_cnt;
        td->dtv = td->dtv_copy = dtv;
        return td;
 }

diff --git a/src/env/__reset_tls.c b/src/env/__reset_tls.c
index 677e57f58b45c6db27c9d1f0ceda3f1b72b08c95..15685bc666988e62524b5e3106e3919aa803238c 100644 (file)
--- a/src/env/__reset_tls.c
+++ b/src/env/__reset_tls.c
@@ -6,11 +6,10 @@ void __reset_tls()
 {
        pthread_t self = __pthread_self();
        struct tls_module *p;
-       size_t i, n = (size_t)self->dtv[0];
+       size_t i, n = self->dtv[0];
        if (n) for (p=libc.tls_head, i=1; i<=n; i++, p=p->next) {
-               if (!self->dtv[i]) continue;
-               memcpy(self->dtv[i], p->image, p->len);
-               memset((char *)self->dtv[i]+p->len, 0,
-                       p->size - p->len);
+               char *mem = (char *)(self->dtv[i] - DTP_OFFSET);
+               memcpy(mem, p->image, p->len);
+               memset(mem+p->len, 0, p->size - p->len);
        }
 }

diff --git a/src/internal/pthread_impl.h b/src/internal/pthread_impl.h
index d491f9753a6d97ecb988910b662dc1bb73312134..7a25b88ebd6ca27acae2fbba2f675287e946b262 100644 (file)
--- a/src/internal/pthread_impl.h
+++ b/src/internal/pthread_impl.h
@@ -17,7 +17,8 @@ struct pthread {
        /* Part 1 -- these fields may be external or
         * internal (accessed via asm) ABI. Do not change. */
        struct pthread *self;
-       void **dtv, *unused1, *unused2;
+       uintptr_t *dtv;
+       void *unused1, *unused2;
        uintptr_t sysinfo;
        uintptr_t canary, canary2;
 
@@ -54,7 +55,7 @@ struct pthread {
        /* Part 3 -- the positions of these fields relative to
         * the end of the structure is external and internal ABI. */
        uintptr_t canary_at_end;
-       void **dtv_copy;
+       uintptr_t *dtv_copy;
 };
 
 struct start_sched_args {

diff --git a/src/thread/__tls_get_addr.c b/src/thread/__tls_get_addr.c
index 34fbc46c3a3a6c6120cc37a3891b9ea7141678da..d7afdabdad8ffa3d5f373f4c2173e8e5e3e56835 100644 (file)
--- a/src/thread/__tls_get_addr.c
+++ b/src/thread/__tls_get_addr.c
@@ -4,8 +4,8 @@
 void *__tls_get_addr(tls_mod_off_t *v)
 {
        pthread_t self = __pthread_self();
-       if (v[0]<=(size_t)self->dtv[0])
-               return (char *)self->dtv[v[0]]+v[1]+DTP_OFFSET;
+       if (v[0] <= self->dtv[0])
+               return (void *)(self->dtv[v[0]] + v[1]);
        return __tls_get_new(v);
 }