From: Rich Felker Date: Fri, 12 Oct 2018 04:30:34 +0000 (-0400) Subject: combine arch ABI's DTP_OFFSET into DTV pointers X-Git-Tag: v1.1.21~62 X-Git-Url: https://git.librecmc.org/?a=commitdiff_plain;h=b6d701a47504e5ef9c6a19b2f6a703c72cb9e8ac;p=oweals%2Fmusl.git combine arch ABI's DTP_OFFSET into DTV pointers as explained in commit 6ba5517a460c6c438f64d69464fdfc3269a4c91a, some archs use an offset (typicaly -0x8000) with their DTPOFF relocations, which __tls_get_addr needs to invert. on affected archs, which lack direct support for large immediates, this can cost multiple extra instructions in the hot path. instead, incorporate the DTP_OFFSET into the DTV entries. this means they are no longer valid pointers, so store them as an array of uintptr_t rather than void *; this also makes it easier to access slot 0 as a valid slot count. commit e75b16cf93ebbc1ce758d3ea6b2923e8b2457c68 left behind cruft in two places, __reset_tls and __tls_get_new, from back when it was possible to have uninitialized gap slots indicated by a null pointer in the DTV. since the concept of null pointer is no longer meaningful with an offset applied, remove this cruft. presently there are no archs with both TLSDESC and nonzero DTP_OFFSET, but the dynamic TLSDESC relocation code is also updated to apply an inverted offset to its offset field, so that the offset DTV would not impose a runtime cost in TLSDESC resolver functions. --- diff --git a/ldso/dynlink.c b/ldso/dynlink.c index c2892b90..a3ca3cdf 100644 --- a/ldso/dynlink.c +++ b/ldso/dynlink.c @@ -72,7 +72,7 @@ struct dso { struct tls_module tls; size_t tls_id; size_t relro_start, relro_end; - void **new_dtv; + uintptr_t *new_dtv; unsigned char *new_tls; volatile int new_dtv_idx, new_tls_idx; struct td_index *td_index; @@ -445,7 +445,7 @@ static void do_relocs(struct dso *dso, size_t *rel, size_t rel_size, size_t stri new->next = dso->td_index; dso->td_index = new; new->args[0] = def.dso->tls_id; - new->args[1] = tls_val + addend; + new->args[1] = tls_val + addend - DTP_OFFSET; reloc_addr[0] = (size_t)__tlsdesc_dynamic; reloc_addr[1] = (size_t)new; } else { @@ -1345,9 +1345,9 @@ hidden void *__tls_get_new(tls_mod_off_t *v) /* Block signals to make accessing new TLS async-signal-safe */ sigset_t set; __block_all_sigs(&set); - if (v[0]<=(size_t)self->dtv[0]) { + if (v[0] <= self->dtv[0]) { __restore_sigs(&set); - return (char *)self->dtv[v[0]]+v[1]+DTP_OFFSET; + return (void *)(self->dtv[v[0]] + v[1]); } /* This is safe without any locks held because, if the caller @@ -1357,15 +1357,12 @@ hidden void *__tls_get_new(tls_mod_off_t *v) struct dso *p; for (p=head; p->tls_id != v[0]; p=p->next); - /* Get new DTV space from new DSO if needed */ - if (v[0] > (size_t)self->dtv[0]) { - void **newdtv = p->new_dtv + - (v[0]+1)*a_fetch_add(&p->new_dtv_idx,1); - memcpy(newdtv, self->dtv, - ((size_t)self->dtv[0]+1) * sizeof(void *)); - newdtv[0] = (void *)v[0]; - self->dtv = self->dtv_copy = newdtv; - } + /* Get new DTV space from new DSO */ + uintptr_t *newdtv = p->new_dtv + + (v[0]+1)*a_fetch_add(&p->new_dtv_idx,1); + memcpy(newdtv, self->dtv, (self->dtv[0]+1) * sizeof(uintptr_t)); + newdtv[0] = v[0]; + self->dtv = self->dtv_copy = newdtv; /* Get new TLS memory from all new DSOs up to the requested one */ unsigned char *mem; @@ -1375,7 +1372,7 @@ hidden void *__tls_get_new(tls_mod_off_t *v) * a_fetch_add(&p->new_tls_idx,1); mem += ((uintptr_t)p->tls.image - (uintptr_t)mem) & (p->tls.align-1); - self->dtv[p->tls_id] = mem; + self->dtv[p->tls_id] = (uintptr_t)mem + DTP_OFFSET; memcpy(mem, p->tls.image, p->tls.len); if (p->tls_id == v[0]) break; } diff --git a/src/env/__init_tls.c b/src/env/__init_tls.c index 96d0e284..842886f6 100644 --- a/src/env/__init_tls.c +++ b/src/env/__init_tls.c @@ -36,32 +36,32 @@ void *__copy_tls(unsigned char *mem) pthread_t td; struct tls_module *p; size_t i; - void **dtv; + uintptr_t *dtv; #ifdef TLS_ABOVE_TP - dtv = (void **)(mem + libc.tls_size) - (libc.tls_cnt + 1); + dtv = (uintptr_t*)(mem + libc.tls_size) - (libc.tls_cnt + 1); mem += -((uintptr_t)mem + sizeof(struct pthread)) & (libc.tls_align-1); td = (pthread_t)mem; mem += sizeof(struct pthread); for (i=1, p=libc.tls_head; p; i++, p=p->next) { - dtv[i] = mem + p->offset; - memcpy(dtv[i], p->image, p->len); + dtv[i] = (uintptr_t)(mem + p->offset) + DTP_OFFSET; + memcpy(mem + p->offset, p->image, p->len); } #else - dtv = (void **)mem; + dtv = (uintptr_t *)mem; mem += libc.tls_size - sizeof(struct pthread); mem -= (uintptr_t)mem & (libc.tls_align-1); td = (pthread_t)mem; for (i=1, p=libc.tls_head; p; i++, p=p->next) { - dtv[i] = mem - p->offset; - memcpy(dtv[i], p->image, p->len); + dtv[i] = (uintptr_t)(mem - p->offset) + DTP_OFFSET; + memcpy(mem - p->offset, p->image, p->len); } #endif - dtv[0] = (void *)libc.tls_cnt; + dtv[0] = libc.tls_cnt; td->dtv = td->dtv_copy = dtv; return td; } diff --git a/src/env/__reset_tls.c b/src/env/__reset_tls.c index 677e57f5..15685bc6 100644 --- a/src/env/__reset_tls.c +++ b/src/env/__reset_tls.c @@ -6,11 +6,10 @@ void __reset_tls() { pthread_t self = __pthread_self(); struct tls_module *p; - size_t i, n = (size_t)self->dtv[0]; + size_t i, n = self->dtv[0]; if (n) for (p=libc.tls_head, i=1; i<=n; i++, p=p->next) { - if (!self->dtv[i]) continue; - memcpy(self->dtv[i], p->image, p->len); - memset((char *)self->dtv[i]+p->len, 0, - p->size - p->len); + char *mem = (char *)(self->dtv[i] - DTP_OFFSET); + memcpy(mem, p->image, p->len); + memset(mem+p->len, 0, p->size - p->len); } } diff --git a/src/internal/pthread_impl.h b/src/internal/pthread_impl.h index d491f975..7a25b88e 100644 --- a/src/internal/pthread_impl.h +++ b/src/internal/pthread_impl.h @@ -17,7 +17,8 @@ struct pthread { /* Part 1 -- these fields may be external or * internal (accessed via asm) ABI. Do not change. */ struct pthread *self; - void **dtv, *unused1, *unused2; + uintptr_t *dtv; + void *unused1, *unused2; uintptr_t sysinfo; uintptr_t canary, canary2; @@ -54,7 +55,7 @@ struct pthread { /* Part 3 -- the positions of these fields relative to * the end of the structure is external and internal ABI. */ uintptr_t canary_at_end; - void **dtv_copy; + uintptr_t *dtv_copy; }; struct start_sched_args { diff --git a/src/thread/__tls_get_addr.c b/src/thread/__tls_get_addr.c index 34fbc46c..d7afdabd 100644 --- a/src/thread/__tls_get_addr.c +++ b/src/thread/__tls_get_addr.c @@ -4,8 +4,8 @@ void *__tls_get_addr(tls_mod_off_t *v) { pthread_t self = __pthread_self(); - if (v[0]<=(size_t)self->dtv[0]) - return (char *)self->dtv[v[0]]+v[1]+DTP_OFFSET; + if (v[0] <= self->dtv[0]) + return (void *)(self->dtv[v[0]] + v[1]); return __tls_get_new(v); }