src/ldso/dynlink.c

   1 #define _GNU_SOURCE
   2 #include <stdio.h>
   3 #include <stdlib.h>
   4 #include <stdarg.h>
   5 #include <stddef.h>
   6 #include <string.h>
   7 #include <unistd.h>
   8 #include <stdint.h>
   9 #include <elf.h>
  10 #include <sys/mman.h>
  11 #include <limits.h>
  12 #include <fcntl.h>
  13 #include <sys/stat.h>
  14 #include <errno.h>
  15 #include <link.h>
  16 #include <setjmp.h>
  17 #include <pthread.h>
  18 #include <ctype.h>
  19 #include <dlfcn.h>
  20 #include "pthread_impl.h"
  21 #include "libc.h"
  22 #include "dynlink.h"
  23
  24 static void error(const char *, ...);
  25
  26 #ifdef SHARED
  27
  28 #define MAXP2(a,b) (-(-(a)&-(b)))
  29 #define ALIGN(x,y) ((x)+(y)-1 & -(y))
  30
  31 struct debug {
  32         int ver;
  33         void *head;
  34         void (*bp)(void);
  35         int state;
  36         void *base;
  37 };
  38
  39 struct td_index {
  40         size_t args[2];
  41         struct td_index *next;
  42 };
  43
  44 struct dso {
  45 #if DL_FDPIC
  46         struct fdpic_loadmap *loadmap;
  47 #else
  48         unsigned char *base;
  49 #endif
  50         char *name;
  51         size_t *dynv;
  52         struct dso *next, *prev;
  53
  54         Phdr *phdr;
  55         int phnum;
  56         size_t phentsize;
  57         int refcnt;
  58         Sym *syms;
  59         uint32_t *hashtab;
  60         uint32_t *ghashtab;
  61         int16_t *versym;
  62         char *strings;
  63         unsigned char *map;
  64         size_t map_len;
  65         dev_t dev;
  66         ino_t ino;
  67         signed char global;
  68         char relocated;
  69         char constructed;
  70         char kernel_mapped;
  71         struct dso **deps, *needed_by;
  72         char *rpath_orig, *rpath;
  73         struct tls_module tls;
  74         size_t tls_id;
  75         size_t relro_start, relro_end;
  76         void **new_dtv;
  77         unsigned char *new_tls;
  78         volatile int new_dtv_idx, new_tls_idx;
  79         struct td_index *td_index;
  80         struct dso *fini_next;
  81         char *shortname;
  82 #if DL_FDPIC
  83         unsigned char *base;
  84 #else
  85         struct fdpic_loadmap *loadmap;
  86 #endif
  87         struct funcdesc {
  88                 void *addr;
  89                 size_t *got;
  90         } *funcdescs;
  91         size_t *got;
  92         char buf[];
  93 };
  94
  95 struct symdef {
  96         Sym *sym;
  97         struct dso *dso;
  98 };
  99
 100 int __init_tp(void *);
 101 void __init_libc(char **, char *);
 102 void *__copy_tls(unsigned char *);
 103
 104 __attribute__((__visibility__("hidden")))
 105 const char *__libc_get_version(void);
 106
 107 static struct builtin_tls {
 108         char c;
 109         struct pthread pt;
 110         void *space[16];
 111 } builtin_tls[1];
 112 #define MIN_TLS_ALIGN offsetof(struct builtin_tls, pt)
 113
 114 #define ADDEND_LIMIT 4096
 115 static size_t *saved_addends, *apply_addends_to;
 116
 117 static struct dso ldso;
 118 static struct dso *head, *tail, *fini_head;
 119 static char *env_path, *sys_path;
 120 static unsigned long long gencnt;
 121 static int runtime;
 122 static int ldd_mode;
 123 static int ldso_fail;
 124 static int noload;
 125 static jmp_buf *rtld_fail;
 126 static pthread_rwlock_t lock;
 127 static struct debug debug;
 128 static struct tls_module *tls_tail;
 129 static size_t tls_cnt, tls_offset, tls_align = MIN_TLS_ALIGN;
 130 static size_t static_tls_cnt;
 131 static pthread_mutex_t init_fini_lock = { ._m_type = PTHREAD_MUTEX_RECURSIVE };
 132 static struct fdpic_loadmap *app_loadmap;
 133 static struct fdpic_dummy_loadmap app_dummy_loadmap;
 134
 135 struct debug *_dl_debug_addr = &debug;
 136
 137 static int dl_strcmp(const char *l, const char *r)
 138 {
 139         for (; *l==*r && *l; l++, r++);
 140         return *(unsigned char *)l - *(unsigned char *)r;
 141 }
 142 #define strcmp(l,r) dl_strcmp(l,r)
 143
 144 /* Compute load address for a virtual address in a given dso. */
 145 #if DL_FDPIC
 146 static void *laddr(const struct dso *p, size_t v)
 147 {
 148         size_t j=0;
 149         if (!p->loadmap) return p->base + v;
 150         for (j=0; v-p->loadmap->segs[j].p_vaddr >= p->loadmap->segs[j].p_memsz; j++);
 151         return (void *)(v - p->loadmap->segs[j].p_vaddr + p->loadmap->segs[j].addr);
 152 }
 153 #define fpaddr(p, v) ((void (*)())&(struct funcdesc){ \
 154         laddr(p, v), (p)->got })
 155 #else
 156 #define laddr(p, v) (void *)((p)->base + (v))
 157 #define fpaddr(p, v) ((void (*)())laddr(p, v))
 158 #endif
 159
 160 static void decode_vec(size_t *v, size_t *a, size_t cnt)
 161 {
 162         size_t i;
 163         for (i=0; i<cnt; i++) a[i] = 0;
 164         for (; v[0]; v+=2) if (v[0]-1<cnt-1) {
 165                 a[0] |= 1UL<<v[0];
 166                 a[v[0]] = v[1];
 167         }
 168 }
 169
 170 static int search_vec(size_t *v, size_t *r, size_t key)
 171 {
 172         for (; v[0]!=key; v+=2)
 173                 if (!v[0]) return 0;
 174         *r = v[1];
 175         return 1;
 176 }
 177
 178 static uint32_t sysv_hash(const char *s0)
 179 {
 180         const unsigned char *s = (void *)s0;
 181         uint_fast32_t h = 0;
 182         while (*s) {
 183                 h = 16*h + *s++;
 184                 h ^= h>>24 & 0xf0;
 185         }
 186         return h & 0xfffffff;
 187 }
 188
 189 static uint32_t gnu_hash(const char *s0)
 190 {
 191         const unsigned char *s = (void *)s0;
 192         uint_fast32_t h = 5381;
 193         for (; *s; s++)
 194                 h += h*32 + *s;
 195         return h;
 196 }
 197
 198 static Sym *sysv_lookup(const char *s, uint32_t h, struct dso *dso)
 199 {
 200         size_t i;
 201         Sym *syms = dso->syms;
 202         uint32_t *hashtab = dso->hashtab;
 203         char *strings = dso->strings;
 204         for (i=hashtab[2+h%hashtab[0]]; i; i=hashtab[2+hashtab[0]+i]) {
 205                 if ((!dso->versym || dso->versym[i] >= 0)
 206                     && (!strcmp(s, strings+syms[i].st_name)))
 207                         return syms+i;
 208         }
 209         return 0;
 210 }
 211
 212 static Sym *gnu_lookup(uint32_t h1, uint32_t *hashtab, struct dso *dso, const char *s)
 213 {
 214         uint32_t nbuckets = hashtab[0];
 215         uint32_t *buckets = hashtab + 4 + hashtab[2]*(sizeof(size_t)/4);
 216         uint32_t i = buckets[h1 % nbuckets];
 217
 218         if (!i) return 0;
 219
 220         uint32_t *hashval = buckets + nbuckets + (i - hashtab[1]);
 221
 222         for (h1 |= 1; ; i++) {
 223                 uint32_t h2 = *hashval++;
 224                 if ((h1 == (h2|1)) && (!dso->versym || dso->versym[i] >= 0)
 225                     && !strcmp(s, dso->strings + dso->syms[i].st_name))
 226                         return dso->syms+i;
 227                 if (h2 & 1) break;
 228         }
 229
 230         return 0;
 231 }
 232
 233 static Sym *gnu_lookup_filtered(uint32_t h1, uint32_t *hashtab, struct dso *dso, const char *s, uint32_t fofs, size_t fmask)
 234 {
 235         const size_t *bloomwords = (const void *)(hashtab+4);
 236         size_t f = bloomwords[fofs & (hashtab[2]-1)];
 237         if (!(f & fmask)) return 0;
 238
 239         f >>= (h1 >> hashtab[3]) % (8 * sizeof f);
 240         if (!(f & 1)) return 0;
 241
 242         return gnu_lookup(h1, hashtab, dso, s);
 243 }
 244
 245 #define OK_TYPES (1<<STT_NOTYPE | 1<<STT_OBJECT | 1<<STT_FUNC | 1<<STT_COMMON | 1<<STT_TLS)
 246 #define OK_BINDS (1<<STB_GLOBAL | 1<<STB_WEAK | 1<<STB_GNU_UNIQUE)
 247
 248 #ifndef ARCH_SYM_REJECT_UND
 249 #define ARCH_SYM_REJECT_UND(s) 0
 250 #endif
 251
 252 static struct symdef find_sym(struct dso *dso, const char *s, int need_def)
 253 {
 254         uint32_t h = 0, gh, gho, *ght;
 255         size_t ghm = 0;
 256         struct symdef def = {0};
 257         for (; dso; dso=dso->next) {
 258                 Sym *sym;
 259                 if (!dso->global) continue;
 260                 if ((ght = dso->ghashtab)) {
 261                         if (!ghm) {
 262                                 gh = gnu_hash(s);
 263                                 int maskbits = 8 * sizeof ghm;
 264                                 gho = gh / maskbits;
 265                                 ghm = 1ul << gh % maskbits;
 266                         }
 267                         sym = gnu_lookup_filtered(gh, ght, dso, s, gho, ghm);
 268                 } else {
 269                         if (!h) h = sysv_hash(s);
 270                         sym = sysv_lookup(s, h, dso);
 271                 }
 272                 if (!sym) continue;
 273                 if (!sym->st_shndx)
 274                         if (need_def || (sym->st_info&0xf) == STT_TLS
 275                             || ARCH_SYM_REJECT_UND(sym))
 276                                 continue;
 277                 if (!sym->st_value)
 278                         if ((sym->st_info&0xf) != STT_TLS)
 279                                 continue;
 280                 if (!(1<<(sym->st_info&0xf) & OK_TYPES)) continue;
 281                 if (!(1<<(sym->st_info>>4) & OK_BINDS)) continue;
 282
 283                 if (def.sym && sym->st_info>>4 == STB_WEAK) continue;
 284                 def.sym = sym;
 285                 def.dso = dso;
 286                 if (sym->st_info>>4 == STB_GLOBAL) break;
 287         }
 288         return def;
 289 }
 290
 291 __attribute__((__visibility__("hidden")))
 292 ptrdiff_t __tlsdesc_static(), __tlsdesc_dynamic();
 293
 294 static void do_relocs(struct dso *dso, size_t *rel, size_t rel_size, size_t stride)
 295 {
 296         unsigned char *base = dso->base;
 297         Sym *syms = dso->syms;
 298         char *strings = dso->strings;
 299         Sym *sym;
 300         const char *name;
 301         void *ctx;
 302         int type;
 303         int sym_index;
 304         struct symdef def;
 305         size_t *reloc_addr;
 306         size_t sym_val;
 307         size_t tls_val;
 308         size_t addend;
 309         int skip_relative = 0, reuse_addends = 0, save_slot = 0;
 310
 311         if (dso == &ldso) {
 312                 /* Only ldso's REL table needs addend saving/reuse. */
 313                 if (rel == apply_addends_to)
 314                         reuse_addends = 1;
 315                 skip_relative = 1;
 316         }
 317
 318         for (; rel_size; rel+=stride, rel_size-=stride*sizeof(size_t)) {
 319                 if (skip_relative && IS_RELATIVE(rel[1], dso->syms)) continue;
 320                 type = R_TYPE(rel[1]);
 321                 if (type == REL_NONE) continue;
 322                 sym_index = R_SYM(rel[1]);
 323                 reloc_addr = laddr(dso, rel[0]);
 324                 if (sym_index) {
 325                         sym = syms + sym_index;
 326                         name = strings + sym->st_name;
 327                         ctx = type==REL_COPY ? head->next : head;
 328                         def = (sym->st_info&0xf) == STT_SECTION
 329                                 ? (struct symdef){ .dso = dso, .sym = sym }
 330                                 : find_sym(ctx, name, type==REL_PLT);
 331                         if (!def.sym && (sym->st_shndx != SHN_UNDEF
 332                             || sym->st_info>>4 != STB_WEAK)) {
 333                                 error("Error relocating %s: %s: symbol not found",
 334                                         dso->name, name);
 335                                 if (runtime) longjmp(*rtld_fail, 1);
 336                                 continue;
 337                         }
 338                 } else {
 339                         sym = 0;
 340                         def.sym = 0;
 341                         def.dso = dso;
 342                 }
 343
 344                 if (stride > 2) {
 345                         addend = rel[2];
 346                 } else if (type==REL_GOT || type==REL_PLT|| type==REL_COPY) {
 347                         addend = 0;
 348                 } else if (reuse_addends) {
 349                         /* Save original addend in stage 2 where the dso
 350                          * chain consists of just ldso; otherwise read back
 351                          * saved addend since the inline one was clobbered. */
 352                         if (head==&ldso)
 353                                 saved_addends[save_slot] = *reloc_addr;
 354                         addend = saved_addends[save_slot++];
 355                 } else {
 356                         addend = *reloc_addr;
 357                 }
 358
 359                 sym_val = def.sym ? (size_t)laddr(def.dso, def.sym->st_value) : 0;
 360                 tls_val = def.sym ? def.sym->st_value : 0;
 361
 362                 switch(type) {
 363                 case REL_NONE:
 364                         break;
 365                 case REL_OFFSET:
 366                         addend -= (size_t)reloc_addr;
 367                 case REL_SYMBOLIC:
 368                 case REL_GOT:
 369                 case REL_PLT:
 370                         *reloc_addr = sym_val + addend;
 371                         break;
 372                 case REL_RELATIVE:
 373                         *reloc_addr = (size_t)base + addend;
 374                         break;
 375                 case REL_SYM_OR_REL:
 376                         if (sym) *reloc_addr = sym_val + addend;
 377                         else *reloc_addr = (size_t)base + addend;
 378                         break;
 379                 case REL_COPY:
 380                         memcpy(reloc_addr, (void *)sym_val, sym->st_size);
 381                         break;
 382                 case REL_OFFSET32:
 383                         *(uint32_t *)reloc_addr = sym_val + addend
 384                                 - (size_t)reloc_addr;
 385                         break;
 386                 case REL_FUNCDESC:
 387                         *reloc_addr = def.sym ? (size_t)(def.dso->funcdescs
 388                                 + (def.sym - def.dso->syms)) : 0;
 389                         break;
 390                 case REL_FUNCDESC_VAL:
 391                         if ((sym->st_info&0xf) == STT_SECTION) *reloc_addr += sym_val;
 392                         else *reloc_addr = sym_val;
 393                         reloc_addr[1] = def.sym ? (size_t)def.dso->got : 0;
 394                         break;
 395                 case REL_DTPMOD:
 396                         *reloc_addr = def.dso->tls_id;
 397                         break;
 398                 case REL_DTPOFF:
 399                         *reloc_addr = tls_val + addend - DTP_OFFSET;
 400                         break;
 401 #ifdef TLS_ABOVE_TP
 402                 case REL_TPOFF:
 403                         *reloc_addr = tls_val + def.dso->tls.offset + TPOFF_K + addend;
 404                         break;
 405 #else
 406                 case REL_TPOFF:
 407                         *reloc_addr = tls_val - def.dso->tls.offset + addend;
 408                         break;
 409                 case REL_TPOFF_NEG:
 410                         *reloc_addr = def.dso->tls.offset - tls_val + addend;
 411                         break;
 412 #endif
 413                 case REL_TLSDESC:
 414                         if (stride<3) addend = reloc_addr[1];
 415                         if (runtime && def.dso->tls_id >= static_tls_cnt) {
 416                                 struct td_index *new = malloc(sizeof *new);
 417                                 if (!new) {
 418                                         error(
 419                                         "Error relocating %s: cannot allocate TLSDESC for %s",
 420                                         dso->name, sym ? name : "(local)" );
 421                                         longjmp(*rtld_fail, 1);
 422                                 }
 423                                 new->next = dso->td_index;
 424                                 dso->td_index = new;
 425                                 new->args[0] = def.dso->tls_id;
 426                                 new->args[1] = tls_val + addend;
 427                                 reloc_addr[0] = (size_t)__tlsdesc_dynamic;
 428                                 reloc_addr[1] = (size_t)new;
 429                         } else {
 430                                 reloc_addr[0] = (size_t)__tlsdesc_static;
 431 #ifdef TLS_ABOVE_TP
 432                                 reloc_addr[1] = tls_val + def.dso->tls.offset
 433                                         + TPOFF_K + addend;
 434 #else
 435                                 reloc_addr[1] = tls_val - def.dso->tls.offset
 436                                         + addend;
 437 #endif
 438                         }
 439                         break;
 440                 default:
 441                         error("Error relocating %s: unsupported relocation type %d",
 442                                 dso->name, type);
 443                         if (runtime) longjmp(*rtld_fail, 1);
 444                         continue;
 445                 }
 446         }
 447 }
 448
 449 /* A huge hack: to make up for the wastefulness of shared libraries
 450  * needing at least a page of dirty memory even if they have no global
 451  * data, we reclaim the gaps at the beginning and end of writable maps
 452  * and "donate" them to the heap by setting up minimal malloc
 453  * structures and then freeing them. */
 454
 455 static void reclaim(struct dso *dso, size_t start, size_t end)
 456 {
 457         size_t *a, *z;
 458         if (start >= dso->relro_start && start < dso->relro_end) start = dso->relro_end;
 459         if (end   >= dso->relro_start && end   < dso->relro_end) end = dso->relro_start;
 460         start = start + 6*sizeof(size_t)-1 & -4*sizeof(size_t);
 461         end = (end & -4*sizeof(size_t)) - 2*sizeof(size_t);
 462         if (start>end || end-start < 4*sizeof(size_t)) return;
 463         a = laddr(dso, start);
 464         z = laddr(dso, end);
 465         a[-2] = 1;
 466         a[-1] = z[0] = end-start + 2*sizeof(size_t) | 1;
 467         z[1] = 1;
 468         free(a);
 469 }
 470
 471 static void reclaim_gaps(struct dso *dso)
 472 {
 473         Phdr *ph = dso->phdr;
 474         size_t phcnt = dso->phnum;
 475
 476         if (DL_FDPIC) return; // FIXME
 477         for (; phcnt--; ph=(void *)((char *)ph+dso->phentsize)) {
 478                 if (ph->p_type!=PT_LOAD) continue;
 479                 if ((ph->p_flags&(PF_R|PF_W))!=(PF_R|PF_W)) continue;
 480                 reclaim(dso, ph->p_vaddr & -PAGE_SIZE, ph->p_vaddr);
 481                 reclaim(dso, ph->p_vaddr+ph->p_memsz,
 482                         ph->p_vaddr+ph->p_memsz+PAGE_SIZE-1 & -PAGE_SIZE);
 483         }
 484 }
 485
 486 static void *mmap_fixed(void *p, size_t n, int prot, int flags, int fd, off_t off)
 487 {
 488         static int no_map_fixed;
 489         char *q;
 490         if (!no_map_fixed) {
 491                 q = mmap(p, n, prot, flags|MAP_FIXED, fd, off);
 492                 if (!DL_NOMMU_SUPPORT || q != MAP_FAILED || errno != EINVAL)
 493                         return q;
 494                 no_map_fixed = 1;
 495         }
 496         /* Fallbacks for MAP_FIXED failure on NOMMU kernels. */
 497         if (flags & MAP_ANONYMOUS) {
 498                 memset(p, 0, n);
 499                 return p;
 500         }
 501         ssize_t r;
 502         if (lseek(fd, off, SEEK_SET) < 0) return MAP_FAILED;
 503         for (q=p; n; q+=r, off+=r, n-=r) {
 504                 r = read(fd, q, n);
 505                 if (r < 0 && errno != EINTR) return MAP_FAILED;
 506                 if (!r) {
 507                         memset(q, 0, n);
 508                         break;
 509                 }
 510         }
 511         return p;
 512 }
 513
 514 static void unmap_library(struct dso *dso)
 515 {
 516         if (dso->loadmap) {
 517                 size_t i;
 518                 for (i=0; i<dso->loadmap->nsegs; i++) {
 519                         if (!dso->loadmap->segs[i].p_memsz)
 520                                 continue;
 521                         munmap((void *)dso->loadmap->segs[i].addr,
 522                                 dso->loadmap->segs[i].p_memsz);
 523                 }
 524                 free(dso->loadmap);
 525         } else if (dso->map && dso->map_len) {
 526                 munmap(dso->map, dso->map_len);
 527         }
 528 }
 529
 530 static void *map_library(int fd, struct dso *dso)
 531 {
 532         Ehdr buf[(896+sizeof(Ehdr))/sizeof(Ehdr)];
 533         void *allocated_buf=0;
 534         size_t phsize;
 535         size_t addr_min=SIZE_MAX, addr_max=0, map_len;
 536         size_t this_min, this_max;
 537         size_t nsegs = 0;
 538         off_t off_start;
 539         Ehdr *eh;
 540         Phdr *ph, *ph0;
 541         unsigned prot;
 542         unsigned char *map=MAP_FAILED, *base;
 543         size_t dyn=0;
 544         size_t tls_image=0;
 545         size_t i;
 546
 547         ssize_t l = read(fd, buf, sizeof buf);
 548         eh = buf;
 549         if (l<0) return 0;
 550         if (l<sizeof *eh || (eh->e_type != ET_DYN && eh->e_type != ET_EXEC))
 551                 goto noexec;
 552         phsize = eh->e_phentsize * eh->e_phnum;
 553         if (phsize > sizeof buf - sizeof *eh) {
 554                 allocated_buf = malloc(phsize);
 555                 if (!allocated_buf) return 0;
 556                 l = pread(fd, allocated_buf, phsize, eh->e_phoff);
 557                 if (l < 0) goto error;
 558                 if (l != phsize) goto noexec;
 559                 ph = ph0 = allocated_buf;
 560         } else if (eh->e_phoff + phsize > l) {
 561                 l = pread(fd, buf+1, phsize, eh->e_phoff);
 562                 if (l < 0) goto error;
 563                 if (l != phsize) goto noexec;
 564                 ph = ph0 = (void *)(buf + 1);
 565         } else {
 566                 ph = ph0 = (void *)((char *)buf + eh->e_phoff);
 567         }
 568         for (i=eh->e_phnum; i; i--, ph=(void *)((char *)ph+eh->e_phentsize)) {
 569                 if (ph->p_type == PT_DYNAMIC) {
 570                         dyn = ph->p_vaddr;
 571                 } else if (ph->p_type == PT_TLS) {
 572                         tls_image = ph->p_vaddr;
 573                         dso->tls.align = ph->p_align;
 574                         dso->tls.len = ph->p_filesz;
 575                         dso->tls.size = ph->p_memsz;
 576                 } else if (ph->p_type == PT_GNU_RELRO) {
 577                         dso->relro_start = ph->p_vaddr & -PAGE_SIZE;
 578                         dso->relro_end = (ph->p_vaddr + ph->p_memsz) & -PAGE_SIZE;
 579                 }
 580                 if (ph->p_type != PT_LOAD) continue;
 581                 nsegs++;
 582                 if (ph->p_vaddr < addr_min) {
 583                         addr_min = ph->p_vaddr;
 584                         off_start = ph->p_offset;
 585                         prot = (((ph->p_flags&PF_R) ? PROT_READ : 0) |
 586                                 ((ph->p_flags&PF_W) ? PROT_WRITE: 0) |
 587                                 ((ph->p_flags&PF_X) ? PROT_EXEC : 0));
 588                 }
 589                 if (ph->p_vaddr+ph->p_memsz > addr_max) {
 590                         addr_max = ph->p_vaddr+ph->p_memsz;
 591                 }
 592         }
 593         if (!dyn) goto noexec;
 594         if (DL_FDPIC && !(eh->e_flags & FDPIC_CONSTDISP_FLAG)) {
 595                 dso->loadmap = calloc(1, sizeof *dso->loadmap
 596                         + nsegs * sizeof *dso->loadmap->segs);
 597                 if (!dso->loadmap) goto error;
 598                 dso->loadmap->nsegs = nsegs;
 599                 for (ph=ph0, i=0; i<nsegs; ph=(void *)((char *)ph+eh->e_phentsize)) {
 600                         if (ph->p_type != PT_LOAD) continue;
 601                         prot = (((ph->p_flags&PF_R) ? PROT_READ : 0) |
 602                                 ((ph->p_flags&PF_W) ? PROT_WRITE: 0) |
 603                                 ((ph->p_flags&PF_X) ? PROT_EXEC : 0));
 604                         map = mmap(0, ph->p_memsz + (ph->p_vaddr & PAGE_SIZE-1),
 605                                 prot, MAP_PRIVATE,
 606                                 fd, ph->p_offset & -PAGE_SIZE);
 607                         if (map == MAP_FAILED) {
 608                                 unmap_library(dso);
 609                                 goto error;
 610                         }
 611                         dso->loadmap->segs[i].addr = (size_t)map +
 612                                 (ph->p_vaddr & PAGE_SIZE-1);
 613                         dso->loadmap->segs[i].p_vaddr = ph->p_vaddr;
 614                         dso->loadmap->segs[i].p_memsz = ph->p_memsz;
 615                         i++;
 616                         if (prot & PROT_WRITE) {
 617                                 size_t brk = (ph->p_vaddr & PAGE_SIZE-1)
 618                                         + ph->p_filesz;
 619                                 size_t pgbrk = brk + PAGE_SIZE-1 & -PAGE_SIZE;
 620                                 size_t pgend = brk + ph->p_memsz - ph->p_filesz
 621                                         + PAGE_SIZE-1 & -PAGE_SIZE;
 622                                 if (pgend > pgbrk && mmap_fixed(map+pgbrk,
 623                                         pgend-pgbrk, prot,
 624                                         MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS,
 625                                         -1, off_start) == MAP_FAILED)
 626                                         goto error;
 627                                 memset(map + brk, 0, pgbrk-brk);
 628                         }
 629                 }
 630                 map = (void *)dso->loadmap->segs[0].addr;
 631                 map_len = 0;
 632                 goto done_mapping;
 633         }
 634         addr_max += PAGE_SIZE-1;
 635         addr_max &= -PAGE_SIZE;
 636         addr_min &= -PAGE_SIZE;
 637         off_start &= -PAGE_SIZE;
 638         map_len = addr_max - addr_min + off_start;
 639         /* The first time, we map too much, possibly even more than
 640          * the length of the file. This is okay because we will not
 641          * use the invalid part; we just need to reserve the right
 642          * amount of virtual address space to map over later. */
 643         map = DL_NOMMU_SUPPORT
 644                 ? mmap((void *)addr_min, map_len, PROT_READ|PROT_WRITE|PROT_EXEC,
 645                         MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
 646                 : mmap((void *)addr_min, map_len, prot,
 647                         MAP_PRIVATE, fd, off_start);
 648         if (map==MAP_FAILED) goto error;
 649         dso->map = map;
 650         dso->map_len = map_len;
 651         /* If the loaded file is not relocatable and the requested address is
 652          * not available, then the load operation must fail. */
 653         if (eh->e_type != ET_DYN && addr_min && map!=(void *)addr_min) {
 654                 errno = EBUSY;
 655                 goto error;
 656         }
 657         base = map - addr_min;
 658         dso->phdr = 0;
 659         dso->phnum = 0;
 660         for (ph=ph0, i=eh->e_phnum; i; i--, ph=(void *)((char *)ph+eh->e_phentsize)) {
 661                 if (ph->p_type != PT_LOAD) continue;
 662                 /* Check if the programs headers are in this load segment, and
 663                  * if so, record the address for use by dl_iterate_phdr. */
 664                 if (!dso->phdr && eh->e_phoff >= ph->p_offset
 665                     && eh->e_phoff+phsize <= ph->p_offset+ph->p_filesz) {
 666                         dso->phdr = (void *)(base + ph->p_vaddr
 667                                 + (eh->e_phoff-ph->p_offset));
 668                         dso->phnum = eh->e_phnum;
 669                         dso->phentsize = eh->e_phentsize;
 670                 }
 671                 /* Reuse the existing mapping for the lowest-address LOAD */
 672                 if ((ph->p_vaddr & -PAGE_SIZE) == addr_min && !DL_NOMMU_SUPPORT)
 673                         continue;
 674                 this_min = ph->p_vaddr & -PAGE_SIZE;
 675                 this_max = ph->p_vaddr+ph->p_memsz+PAGE_SIZE-1 & -PAGE_SIZE;
 676                 off_start = ph->p_offset & -PAGE_SIZE;
 677                 prot = (((ph->p_flags&PF_R) ? PROT_READ : 0) |
 678                         ((ph->p_flags&PF_W) ? PROT_WRITE: 0) |
 679                         ((ph->p_flags&PF_X) ? PROT_EXEC : 0));
 680                 if (mmap_fixed(base+this_min, this_max-this_min, prot, MAP_PRIVATE|MAP_FIXED, fd, off_start) == MAP_FAILED)
 681                         goto error;
 682                 if (ph->p_memsz > ph->p_filesz) {
 683                         size_t brk = (size_t)base+ph->p_vaddr+ph->p_filesz;
 684                         size_t pgbrk = brk+PAGE_SIZE-1 & -PAGE_SIZE;
 685                         memset((void *)brk, 0, pgbrk-brk & PAGE_SIZE-1);
 686                         if (pgbrk-(size_t)base < this_max && mmap_fixed((void *)pgbrk, (size_t)base+this_max-pgbrk, prot, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) == MAP_FAILED)
 687                                 goto error;
 688                 }
 689         }
 690         for (i=0; ((size_t *)(base+dyn))[i]; i+=2)
 691                 if (((size_t *)(base+dyn))[i]==DT_TEXTREL) {
 692                         if (mprotect(map, map_len, PROT_READ|PROT_WRITE|PROT_EXEC)
 693                             && errno != ENOSYS)
 694                                 goto error;
 695                         break;
 696                 }
 697 done_mapping:
 698         dso->base = base;
 699         dso->dynv = laddr(dso, dyn);
 700         if (dso->tls.size) dso->tls.image = laddr(dso, tls_image);
 701         if (!runtime) reclaim_gaps(dso);
 702         free(allocated_buf);
 703         return map;
 704 noexec:
 705         errno = ENOEXEC;
 706 error:
 707         if (map!=MAP_FAILED) unmap_library(dso);
 708         free(allocated_buf);
 709         return 0;
 710 }
 711
 712 static int path_open(const char *name, const char *s, char *buf, size_t buf_size)
 713 {
 714         size_t l;
 715         int fd;
 716         for (;;) {
 717                 s += strspn(s, ":\n");
 718                 l = strcspn(s, ":\n");
 719                 if (l-1 >= INT_MAX) return -1;
 720                 if (snprintf(buf, buf_size, "%.*s/%s", (int)l, s, name) < buf_size) {
 721                         if ((fd = open(buf, O_RDONLY|O_CLOEXEC))>=0) return fd;
 722                         switch (errno) {
 723                         case ENOENT:
 724                         case ENOTDIR:
 725                         case EACCES:
 726                         case ENAMETOOLONG:
 727                                 break;
 728                         default:
 729                                 /* Any negative value but -1 will inhibit
 730                                  * futher path search. */
 731                                 return -2;
 732                         }
 733                 }
 734                 s += l;
 735         }
 736 }
 737
 738 static int fixup_rpath(struct dso *p, char *buf, size_t buf_size)
 739 {
 740         size_t n, l;
 741         const char *s, *t, *origin;
 742         char *d;
 743         if (p->rpath || !p->rpath_orig) return 0;
 744         if (!strchr(p->rpath_orig, '$')) {
 745                 p->rpath = p->rpath_orig;
 746                 return 0;
 747         }
 748         n = 0;
 749         s = p->rpath_orig;
 750         while ((t=strchr(s, '$'))) {
 751                 if (strncmp(t, "$ORIGIN", 7) && strncmp(t, "${ORIGIN}", 9))
 752                         return 0;
 753                 s = t+1;
 754                 n++;
 755         }
 756         if (n > SSIZE_MAX/PATH_MAX) return 0;
 757
 758         if (p->kernel_mapped) {
 759                 /* $ORIGIN searches cannot be performed for the main program
 760                  * when it is suid/sgid/AT_SECURE. This is because the
 761                  * pathname is under the control of the caller of execve.
 762                  * For libraries, however, $ORIGIN can be processed safely
 763                  * since the library's pathname came from a trusted source
 764                  * (either system paths or a call to dlopen). */
 765                 if (libc.secure)
 766                         return 0;
 767                 l = readlink("/proc/self/exe", buf, buf_size);
 768                 if (l == -1) switch (errno) {
 769                 case ENOENT:
 770                 case ENOTDIR:
 771                 case EACCES:
 772                         break;
 773                 default:
 774                         return -1;
 775                 }
 776                 if (l >= buf_size)
 777                         return 0;
 778                 buf[l] = 0;
 779                 origin = buf;
 780         } else {
 781                 origin = p->name;
 782         }
 783         t = strrchr(origin, '/');
 784         l = t ? t-origin : 0;
 785         p->rpath = malloc(strlen(p->rpath_orig) + n*l + 1);
 786         if (!p->rpath) return -1;
 787
 788         d = p->rpath;
 789         s = p->rpath_orig;
 790         while ((t=strchr(s, '$'))) {
 791                 memcpy(d, s, t-s);
 792                 d += t-s;
 793                 memcpy(d, origin, l);
 794                 d += l;
 795                 /* It was determined previously that the '$' is followed
 796                  * either by "ORIGIN" or "{ORIGIN}". */
 797                 s = t + 7 + 2*(t[1]=='{');
 798         }
 799         strcpy(d, s);
 800         return 0;
 801 }
 802
 803 static void decode_dyn(struct dso *p)
 804 {
 805         size_t dyn[DYN_CNT];
 806         decode_vec(p->dynv, dyn, DYN_CNT);
 807         p->syms = laddr(p, dyn[DT_SYMTAB]);
 808         p->strings = laddr(p, dyn[DT_STRTAB]);
 809         if (dyn[0]&(1<<DT_HASH))
 810                 p->hashtab = laddr(p, dyn[DT_HASH]);
 811         if (dyn[0]&(1<<DT_RPATH))
 812                 p->rpath_orig = p->strings + dyn[DT_RPATH];
 813         if (dyn[0]&(1<<DT_RUNPATH))
 814                 p->rpath_orig = p->strings + dyn[DT_RUNPATH];
 815         if (dyn[0]&(1<<DT_PLTGOT))
 816                 p->got = laddr(p, dyn[DT_PLTGOT]);
 817         if (search_vec(p->dynv, dyn, DT_GNU_HASH))
 818                 p->ghashtab = laddr(p, *dyn);
 819         if (search_vec(p->dynv, dyn, DT_VERSYM))
 820                 p->versym = laddr(p, *dyn);
 821 }
 822
 823 static size_t count_syms(struct dso *p)
 824 {
 825         if (p->hashtab) return p->hashtab[1];
 826
 827         size_t nsym, i;
 828         uint32_t *buckets = p->ghashtab + 4 + (p->ghashtab[2]*sizeof(size_t)/4);
 829         uint32_t *hashval;
 830         for (i = nsym = 0; i < p->ghashtab[0]; i++) {
 831                 if (buckets[i] > nsym)
 832                         nsym = buckets[i];
 833         }
 834         if (nsym) {
 835                 hashval = buckets + p->ghashtab[0] + (nsym - p->ghashtab[1]);
 836                 do nsym++;
 837                 while (!(*hashval++ & 1));
 838         }
 839         return nsym;
 840 }
 841
 842 static void *dl_mmap(size_t n)
 843 {
 844         void *p;
 845         int prot = PROT_READ|PROT_WRITE, flags = MAP_ANONYMOUS|MAP_PRIVATE;
 846 #ifdef SYS_mmap2
 847         p = (void *)__syscall(SYS_mmap2, 0, n, prot, flags, -1, 0);
 848 #else
 849         p = (void *)__syscall(SYS_mmap, 0, n, prot, flags, -1, 0);
 850 #endif
 851         return p == MAP_FAILED ? 0 : p;
 852 }
 853
 854 static void makefuncdescs(struct dso *p)
 855 {
 856         static int self_done;
 857         size_t nsym = count_syms(p);
 858         size_t i, size = nsym * sizeof(*p->funcdescs);
 859
 860         if (!self_done) {
 861                 p->funcdescs = dl_mmap(size);
 862                 self_done = 1;
 863         } else {
 864                 p->funcdescs = malloc(size);
 865         }
 866         if (!p->funcdescs) {
 867                 if (!runtime) a_crash();
 868                 error("Error allocating function descriptors for %s", p->name);
 869                 longjmp(*rtld_fail, 1);
 870         }
 871         for (i=0; i<nsym; i++) {
 872                 if ((p->syms[i].st_info&0xf)==STT_FUNC && p->syms[i].st_shndx) {
 873                         p->funcdescs[i].addr = laddr(p, p->syms[i].st_value);
 874                         p->funcdescs[i].got = p->got;
 875                 } else {
 876                         p->funcdescs[i].addr = 0;
 877                         p->funcdescs[i].got = 0;
 878                 }
 879         }
 880 }
 881
 882 static struct dso *load_library(const char *name, struct dso *needed_by)
 883 {
 884         char buf[2*NAME_MAX+2];
 885         const char *pathname;
 886         unsigned char *map;
 887         struct dso *p, temp_dso = {0};
 888         int fd;
 889         struct stat st;
 890         size_t alloc_size;
 891         int n_th = 0;
 892         int is_self = 0;
 893
 894         if (!*name) {
 895                 errno = EINVAL;
 896                 return 0;
 897         }
 898
 899         /* Catch and block attempts to reload the implementation itself */
 900         if (name[0]=='l' && name[1]=='i' && name[2]=='b') {
 901                 static const char *rp, reserved[] =
 902                         "c\0pthread\0rt\0m\0dl\0util\0xnet\0";
 903                 char *z = strchr(name, '.');
 904                 if (z) {
 905                         size_t l = z-name;
 906                         for (rp=reserved; *rp && strncmp(name+3, rp, l-3); rp+=strlen(rp)+1);
 907                         if (*rp) {
 908                                 if (ldd_mode) {
 909                                         /* Track which names have been resolved
 910                                          * and only report each one once. */
 911                                         static unsigned reported;
 912                                         unsigned mask = 1U<<(rp-reserved);
 913                                         if (!(reported & mask)) {
 914                                                 reported |= mask;
 915                                                 dprintf(1, "\t%s => %s (%p)\n",
 916                                                         name, ldso.name,
 917                                                         ldso.base);
 918                                         }
 919                                 }
 920                                 is_self = 1;
 921                         }
 922                 }
 923         }
 924         if (!strcmp(name, ldso.name)) is_self = 1;
 925         if (is_self) {
 926                 if (!ldso.prev) {
 927                         tail->next = &ldso;
 928                         ldso.prev = tail;
 929                         tail = ldso.next ? ldso.next : &ldso;
 930                 }
 931                 return &ldso;
 932         }
 933         if (strchr(name, '/')) {
 934                 pathname = name;
 935                 fd = open(name, O_RDONLY|O_CLOEXEC);
 936         } else {
 937                 /* Search for the name to see if it's already loaded */
 938                 for (p=head->next; p; p=p->next) {
 939                         if (p->shortname && !strcmp(p->shortname, name)) {
 940                                 p->refcnt++;
 941                                 return p;
 942                         }
 943                 }
 944                 if (strlen(name) > NAME_MAX) return 0;
 945                 fd = -1;
 946                 if (env_path) fd = path_open(name, env_path, buf, sizeof buf);
 947                 for (p=needed_by; fd == -1 && p; p=p->needed_by) {
 948                         if (fixup_rpath(p, buf, sizeof buf) < 0)
 949                                 fd = -2; /* Inhibit further search. */
 950                         if (p->rpath)
 951                                 fd = path_open(name, p->rpath, buf, sizeof buf);
 952                 }
 953                 if (fd == -1) {
 954                         if (!sys_path) {
 955                                 char *prefix = 0;
 956                                 size_t prefix_len;
 957                                 if (ldso.name[0]=='/') {
 958                                         char *s, *t, *z;
 959                                         for (s=t=z=ldso.name; *s; s++)
 960                                                 if (*s=='/') z=t, t=s;
 961                                         prefix_len = z-ldso.name;
 962                                         if (prefix_len < PATH_MAX)
 963                                                 prefix = ldso.name;
 964                                 }
 965                                 if (!prefix) {
 966                                         prefix = "";
 967                                         prefix_len = 0;
 968                                 }
 969                                 char etc_ldso_path[prefix_len + 1
 970                                         + sizeof "/etc/ld-musl-" LDSO_ARCH ".path"];
 971                                 snprintf(etc_ldso_path, sizeof etc_ldso_path,
 972                                         "%.*s/etc/ld-musl-" LDSO_ARCH ".path",
 973                                         (int)prefix_len, prefix);
 974                                 FILE *f = fopen(etc_ldso_path, "rbe");
 975                                 if (f) {
 976                                         if (getdelim(&sys_path, (size_t[1]){0}, 0, f) <= 0) {
 977                                                 free(sys_path);
 978                                                 sys_path = "";
 979                                         }
 980                                         fclose(f);
 981                                 } else if (errno != ENOENT) {
 982                                         sys_path = "";
 983                                 }
 984                         }
 985                         if (!sys_path) sys_path = "/lib:/usr/local/lib:/usr/lib";
 986                         fd = path_open(name, sys_path, buf, sizeof buf);
 987                 }
 988                 pathname = buf;
 989         }
 990         if (fd < 0) return 0;
 991         if (fstat(fd, &st) < 0) {
 992                 close(fd);
 993                 return 0;
 994         }
 995         for (p=head->next; p; p=p->next) {
 996                 if (p->dev == st.st_dev && p->ino == st.st_ino) {
 997                         /* If this library was previously loaded with a
 998                          * pathname but a search found the same inode,
 999                          * setup its shortname so it can be found by name. */
1000                         if (!p->shortname && pathname != name)
1001                                 p->shortname = strrchr(p->name, '/')+1;
1002                         close(fd);
1003                         p->refcnt++;
1004                         return p;
1005                 }
1006         }
1007         map = noload ? 0 : map_library(fd, &temp_dso);
1008         close(fd);
1009         if (!map) return 0;
1010
1011         /* Allocate storage for the new DSO. When there is TLS, this
1012          * storage must include a reservation for all pre-existing
1013          * threads to obtain copies of both the new TLS, and an
1014          * extended DTV capable of storing an additional slot for
1015          * the newly-loaded DSO. */
1016         alloc_size = sizeof *p + strlen(pathname) + 1;
1017         if (runtime && temp_dso.tls.image) {
1018                 size_t per_th = temp_dso.tls.size + temp_dso.tls.align
1019                         + sizeof(void *) * (tls_cnt+3);
1020                 n_th = libc.threads_minus_1 + 1;
1021                 if (n_th > SSIZE_MAX / per_th) alloc_size = SIZE_MAX;
1022                 else alloc_size += n_th * per_th;
1023         }
1024         p = calloc(1, alloc_size);
1025         if (!p) {
1026                 unmap_library(&temp_dso);
1027                 return 0;
1028         }
1029         memcpy(p, &temp_dso, sizeof temp_dso);
1030         decode_dyn(p);
1031         p->dev = st.st_dev;
1032         p->ino = st.st_ino;
1033         p->refcnt = 1;
1034         p->needed_by = needed_by;
1035         p->name = p->buf;
1036         strcpy(p->name, pathname);
1037         /* Add a shortname only if name arg was not an explicit pathname. */
1038         if (pathname != name) p->shortname = strrchr(p->name, '/')+1;
1039         if (p->tls.image) {
1040                 p->tls_id = ++tls_cnt;
1041                 tls_align = MAXP2(tls_align, p->tls.align);
1042 #ifdef TLS_ABOVE_TP
1043                 p->tls.offset = tls_offset + ( (tls_align-1) &
1044                         -(tls_offset + (uintptr_t)p->tls.image) );
1045                 tls_offset += p->tls.size;
1046 #else
1047                 tls_offset += p->tls.size + p->tls.align - 1;
1048                 tls_offset -= (tls_offset + (uintptr_t)p->tls.image)
1049                         & (p->tls.align-1);
1050                 p->tls.offset = tls_offset;
1051 #endif
1052                 p->new_dtv = (void *)(-sizeof(size_t) &
1053                         (uintptr_t)(p->name+strlen(p->name)+sizeof(size_t)));
1054                 p->new_tls = (void *)(p->new_dtv + n_th*(tls_cnt+1));
1055                 if (tls_tail) tls_tail->next = &p->tls;
1056                 else libc.tls_head = &p->tls;
1057                 tls_tail = &p->tls;
1058         }
1059
1060         tail->next = p;
1061         p->prev = tail;
1062         tail = p;
1063
1064         if (DL_FDPIC) makefuncdescs(p);
1065
1066         if (ldd_mode) dprintf(1, "\t%s => %s (%p)\n", name, pathname, p->base);
1067
1068         return p;
1069 }
1070
1071 static void load_deps(struct dso *p)
1072 {
1073         size_t i, ndeps=0;
1074         struct dso ***deps = &p->deps, **tmp, *dep;
1075         for (; p; p=p->next) {
1076                 for (i=0; p->dynv[i]; i+=2) {
1077                         if (p->dynv[i] != DT_NEEDED) continue;
1078                         dep = load_library(p->strings + p->dynv[i+1], p);
1079                         if (!dep) {
1080                                 error("Error loading shared library %s: %m (needed by %s)",
1081                                         p->strings + p->dynv[i+1], p->name);
1082                                 if (runtime) longjmp(*rtld_fail, 1);
1083                                 continue;
1084                         }
1085                         if (runtime) {
1086                                 tmp = realloc(*deps, sizeof(*tmp)*(ndeps+2));
1087                                 if (!tmp) longjmp(*rtld_fail, 1);
1088                                 tmp[ndeps++] = dep;
1089                                 tmp[ndeps] = 0;
1090                                 *deps = tmp;
1091                         }
1092                 }
1093         }
1094 }
1095
1096 static void load_preload(char *s)
1097 {
1098         int tmp;
1099         char *z;
1100         for (z=s; *z; s=z) {
1101                 for (   ; *s && (isspace(*s) || *s==':'); s++);
1102                 for (z=s; *z && !isspace(*z) && *z!=':'; z++);
1103                 tmp = *z;
1104                 *z = 0;
1105                 load_library(s, 0);
1106                 *z = tmp;
1107         }
1108 }
1109
1110 static void make_global(struct dso *p)
1111 {
1112         for (; p; p=p->next) p->global = 1;
1113 }
1114
1115 static void do_mips_relocs(struct dso *p, size_t *got)
1116 {
1117         size_t i, j, rel[2];
1118         unsigned char *base = p->base;
1119         i=0; search_vec(p->dynv, &i, DT_MIPS_LOCAL_GOTNO);
1120         if (p==&ldso) {
1121                 got += i;
1122         } else {
1123                 while (i--) *got++ += (size_t)base;
1124         }
1125         j=0; search_vec(p->dynv, &j, DT_MIPS_GOTSYM);
1126         i=0; search_vec(p->dynv, &i, DT_MIPS_SYMTABNO);
1127         Sym *sym = p->syms + j;
1128         rel[0] = (unsigned char *)got - base;
1129         for (i-=j; i; i--, sym++, rel[0]+=sizeof(size_t)) {
1130                 rel[1] = sym-p->syms << 8 | R_MIPS_JUMP_SLOT;
1131                 do_relocs(p, rel, sizeof rel, 2);
1132         }
1133 }
1134
1135 static void reloc_all(struct dso *p)
1136 {
1137         size_t dyn[DYN_CNT];
1138         for (; p; p=p->next) {
1139                 if (p->relocated) continue;
1140                 decode_vec(p->dynv, dyn, DYN_CNT);
1141                 if (NEED_MIPS_GOT_RELOCS)
1142                         do_mips_relocs(p, laddr(p, dyn[DT_PLTGOT]));
1143                 do_relocs(p, laddr(p, dyn[DT_JMPREL]), dyn[DT_PLTRELSZ],
1144                         2+(dyn[DT_PLTREL]==DT_RELA));
1145                 do_relocs(p, laddr(p, dyn[DT_REL]), dyn[DT_RELSZ], 2);
1146                 do_relocs(p, laddr(p, dyn[DT_RELA]), dyn[DT_RELASZ], 3);
1147
1148                 if (head != &ldso && p->relro_start != p->relro_end &&
1149                     mprotect(laddr(p, p->relro_start), p->relro_end-p->relro_start, PROT_READ)
1150                     && errno != ENOSYS) {
1151                         error("Error relocating %s: RELRO protection failed: %m",
1152                                 p->name);
1153                         if (runtime) longjmp(*rtld_fail, 1);
1154                 }
1155
1156                 p->relocated = 1;
1157         }
1158 }
1159
1160 static void kernel_mapped_dso(struct dso *p)
1161 {
1162         size_t min_addr = -1, max_addr = 0, cnt;
1163         Phdr *ph = p->phdr;
1164         for (cnt = p->phnum; cnt--; ph = (void *)((char *)ph + p->phentsize)) {
1165                 if (ph->p_type == PT_DYNAMIC) {
1166                         p->dynv = laddr(p, ph->p_vaddr);
1167                 } else if (ph->p_type == PT_GNU_RELRO) {
1168                         p->relro_start = ph->p_vaddr & -PAGE_SIZE;
1169                         p->relro_end = (ph->p_vaddr + ph->p_memsz) & -PAGE_SIZE;
1170                 }
1171                 if (ph->p_type != PT_LOAD) continue;
1172                 if (ph->p_vaddr < min_addr)
1173                         min_addr = ph->p_vaddr;
1174                 if (ph->p_vaddr+ph->p_memsz > max_addr)
1175                         max_addr = ph->p_vaddr+ph->p_memsz;
1176         }
1177         min_addr &= -PAGE_SIZE;
1178         max_addr = (max_addr + PAGE_SIZE-1) & -PAGE_SIZE;
1179         p->map = p->base + min_addr;
1180         p->map_len = max_addr - min_addr;
1181         p->kernel_mapped = 1;
1182 }
1183
1184 void __libc_exit_fini()
1185 {
1186         struct dso *p;
1187         size_t dyn[DYN_CNT];
1188         for (p=fini_head; p; p=p->fini_next) {
1189                 if (!p->constructed) continue;
1190                 decode_vec(p->dynv, dyn, DYN_CNT);
1191                 if (dyn[0] & (1<<DT_FINI_ARRAY)) {
1192                         size_t n = dyn[DT_FINI_ARRAYSZ]/sizeof(size_t);
1193                         size_t *fn = (size_t *)laddr(p, dyn[DT_FINI_ARRAY])+n;
1194                         while (n--) ((void (*)(void))*--fn)();
1195                 }
1196 #ifndef NO_LEGACY_INITFINI
1197                 if ((dyn[0] & (1<<DT_FINI)) && dyn[DT_FINI])
1198                         fpaddr(p, dyn[DT_FINI])();
1199 #endif
1200         }
1201 }
1202
1203 static void do_init_fini(struct dso *p)
1204 {
1205         size_t dyn[DYN_CNT];
1206         int need_locking = libc.threads_minus_1;
1207         /* Allow recursive calls that arise when a library calls
1208          * dlopen from one of its constructors, but block any
1209          * other threads until all ctors have finished. */
1210         if (need_locking) pthread_mutex_lock(&init_fini_lock);
1211         for (; p; p=p->prev) {
1212                 if (p->constructed) continue;
1213                 p->constructed = 1;
1214                 decode_vec(p->dynv, dyn, DYN_CNT);
1215                 if (dyn[0] & ((1<<DT_FINI) | (1<<DT_FINI_ARRAY))) {
1216                         p->fini_next = fini_head;
1217                         fini_head = p;
1218                 }
1219 #ifndef NO_LEGACY_INITFINI
1220                 if ((dyn[0] & (1<<DT_INIT)) && dyn[DT_INIT])
1221                         fpaddr(p, dyn[DT_INIT])();
1222 #endif
1223                 if (dyn[0] & (1<<DT_INIT_ARRAY)) {
1224                         size_t n = dyn[DT_INIT_ARRAYSZ]/sizeof(size_t);
1225                         size_t *fn = laddr(p, dyn[DT_INIT_ARRAY]);
1226                         while (n--) ((void (*)(void))*fn++)();
1227                 }
1228                 if (!need_locking && libc.threads_minus_1) {
1229                         need_locking = 1;
1230                         pthread_mutex_lock(&init_fini_lock);
1231                 }
1232         }
1233         if (need_locking) pthread_mutex_unlock(&init_fini_lock);
1234 }
1235
1236 void __libc_start_init(void)
1237 {
1238         do_init_fini(tail);
1239 }
1240
1241 static void dl_debug_state(void)
1242 {
1243 }
1244
1245 weak_alias(dl_debug_state, _dl_debug_state);
1246
1247 void __init_tls(size_t *auxv)
1248 {
1249 }
1250
1251 __attribute__((__visibility__("hidden")))
1252 void *__tls_get_new(size_t *v)
1253 {
1254         pthread_t self = __pthread_self();
1255
1256         /* Block signals to make accessing new TLS async-signal-safe */
1257         sigset_t set;
1258         __block_all_sigs(&set);
1259         if (v[0]<=(size_t)self->dtv[0]) {
1260                 __restore_sigs(&set);
1261                 return (char *)self->dtv[v[0]]+v[1]+DTP_OFFSET;
1262         }
1263
1264         /* This is safe without any locks held because, if the caller
1265          * is able to request the Nth entry of the DTV, the DSO list
1266          * must be valid at least that far out and it was synchronized
1267          * at program startup or by an already-completed call to dlopen. */
1268         struct dso *p;
1269         for (p=head; p->tls_id != v[0]; p=p->next);
1270
1271         /* Get new DTV space from new DSO if needed */
1272         if (v[0] > (size_t)self->dtv[0]) {
1273                 void **newdtv = p->new_dtv +
1274                         (v[0]+1)*sizeof(void *)*a_fetch_add(&p->new_dtv_idx,1);
1275                 memcpy(newdtv, self->dtv,
1276                         ((size_t)self->dtv[0]+1) * sizeof(void *));
1277                 newdtv[0] = (void *)v[0];
1278                 self->dtv = self->dtv_copy = newdtv;
1279         }
1280
1281         /* Get new TLS memory from all new DSOs up to the requested one */
1282         unsigned char *mem;
1283         for (p=head; ; p=p->next) {
1284                 if (!p->tls_id || self->dtv[p->tls_id]) continue;
1285                 mem = p->new_tls + (p->tls.size + p->tls.align)
1286                         * a_fetch_add(&p->new_tls_idx,1);
1287                 mem += ((uintptr_t)p->tls.image - (uintptr_t)mem)
1288                         & (p->tls.align-1);
1289                 self->dtv[p->tls_id] = mem;
1290                 memcpy(mem, p->tls.image, p->tls.len);
1291                 if (p->tls_id == v[0]) break;
1292         }
1293         __restore_sigs(&set);
1294         return mem + v[1] + DTP_OFFSET;
1295 }
1296
1297 static void update_tls_size()
1298 {
1299         libc.tls_cnt = tls_cnt;
1300         libc.tls_align = tls_align;
1301         libc.tls_size = ALIGN(
1302                 (1+tls_cnt) * sizeof(void *) +
1303                 tls_offset +
1304                 sizeof(struct pthread) +
1305                 tls_align * 2,
1306         tls_align);
1307 }
1308
1309 /* Stage 1 of the dynamic linker is defined in dlstart.c. It calls the
1310  * following stage 2 and stage 3 functions via primitive symbolic lookup
1311  * since it does not have access to their addresses to begin with. */
1312
1313 /* Stage 2 of the dynamic linker is called after relative relocations
1314  * have been processed. It can make function calls to static functions
1315  * and access string literals and static data, but cannot use extern
1316  * symbols. Its job is to perform symbolic relocations on the dynamic
1317  * linker itself, but some of the relocations performed may need to be
1318  * replaced later due to copy relocations in the main program. */
1319
1320 __attribute__((__visibility__("hidden")))
1321 void __dls2(unsigned char *base, size_t *sp)
1322 {
1323         if (DL_FDPIC) {
1324                 void *p1 = (void *)sp[-2];
1325                 void *p2 = (void *)sp[-1];
1326                 if (!p1) {
1327                         size_t *auxv, aux[AUX_CNT];
1328                         for (auxv=sp+1+*sp+1; *auxv; auxv++); auxv++;
1329                         decode_vec(auxv, aux, AUX_CNT);
1330                         if (aux[AT_BASE]) ldso.base = (void *)aux[AT_BASE];
1331                         else ldso.base = (void *)(aux[AT_PHDR] & -4096);
1332                 }
1333                 app_loadmap = p2 ? p1 : 0;
1334                 ldso.loadmap = p2 ? p2 : p1;
1335                 ldso.base = laddr(&ldso, 0);
1336         } else {
1337                 ldso.base = base;
1338         }
1339         Ehdr *ehdr = (void *)ldso.base;
1340         ldso.name = ldso.shortname = "libc.so";
1341         ldso.global = 1;
1342         ldso.phnum = ehdr->e_phnum;
1343         ldso.phdr = laddr(&ldso, ehdr->e_phoff);
1344         ldso.phentsize = ehdr->e_phentsize;
1345         kernel_mapped_dso(&ldso);
1346         decode_dyn(&ldso);
1347
1348         if (DL_FDPIC) makefuncdescs(&ldso);
1349
1350         /* Prepare storage for to save clobbered REL addends so they
1351          * can be reused in stage 3. There should be very few. If
1352          * something goes wrong and there are a huge number, abort
1353          * instead of risking stack overflow. */
1354         size_t dyn[DYN_CNT];
1355         decode_vec(ldso.dynv, dyn, DYN_CNT);
1356         size_t *rel = laddr(&ldso, dyn[DT_REL]);
1357         size_t rel_size = dyn[DT_RELSZ];
1358         size_t symbolic_rel_cnt = 0;
1359         apply_addends_to = rel;
1360         for (; rel_size; rel+=2, rel_size-=2*sizeof(size_t))
1361                 if (!IS_RELATIVE(rel[1], ldso.syms)) symbolic_rel_cnt++;
1362         if (symbolic_rel_cnt >= ADDEND_LIMIT) a_crash();
1363         size_t addends[symbolic_rel_cnt+1];
1364         saved_addends = addends;
1365
1366         head = &ldso;
1367         reloc_all(&ldso);
1368
1369         ldso.relocated = 0;
1370
1371         /* Call dynamic linker stage-3, __dls3, looking it up
1372          * symbolically as a barrier against moving the address
1373          * load across the above relocation processing. */
1374         struct symdef dls3_def = find_sym(&ldso, "__dls3", 0);
1375         if (DL_FDPIC) ((stage3_func)&ldso.funcdescs[dls3_def.sym-ldso.syms])(sp);
1376         else ((stage3_func)laddr(&ldso, dls3_def.sym->st_value))(sp);
1377 }
1378
1379 /* Stage 3 of the dynamic linker is called with the dynamic linker/libc
1380  * fully functional. Its job is to load (if not already loaded) and
1381  * process dependencies and relocations for the main application and
1382  * transfer control to its entry point. */
1383
1384 _Noreturn void __dls3(size_t *sp)
1385 {
1386         static struct dso app, vdso;
1387         size_t aux[AUX_CNT], *auxv;
1388         size_t i;
1389         char *env_preload=0;
1390         size_t vdso_base;
1391         int argc = *sp;
1392         char **argv = (void *)(sp+1);
1393         char **argv_orig = argv;
1394         char **envp = argv+argc+1;
1395
1396         /* Find aux vector just past environ[] and use it to initialize
1397          * global data that may be needed before we can make syscalls. */
1398         __environ = envp;
1399         for (i=argc+1; argv[i]; i++);
1400         libc.auxv = auxv = (void *)(argv+i+1);
1401         decode_vec(auxv, aux, AUX_CNT);
1402         __hwcap = aux[AT_HWCAP];
1403         libc.page_size = aux[AT_PAGESZ];
1404         libc.secure = ((aux[0]&0x7800)!=0x7800 || aux[AT_UID]!=aux[AT_EUID]
1405                 || aux[AT_GID]!=aux[AT_EGID] || aux[AT_SECURE]);
1406
1407         /* Setup early thread pointer in builtin_tls for ldso/libc itself to
1408          * use during dynamic linking. If possible it will also serve as the
1409          * thread pointer at runtime. */
1410         libc.tls_size = sizeof builtin_tls;
1411         libc.tls_align = tls_align;
1412         if (__init_tp(__copy_tls((void *)builtin_tls)) < 0) {
1413                 a_crash();
1414         }
1415
1416         /* Only trust user/env if kernel says we're not suid/sgid */
1417         if (!libc.secure) {
1418                 env_path = getenv("LD_LIBRARY_PATH");
1419                 env_preload = getenv("LD_PRELOAD");
1420         }
1421
1422         /* If the main program was already loaded by the kernel,
1423          * AT_PHDR will point to some location other than the dynamic
1424          * linker's program headers. */
1425         if (aux[AT_PHDR] != (size_t)ldso.phdr) {
1426                 size_t interp_off = 0;
1427                 size_t tls_image = 0;
1428                 /* Find load address of the main program, via AT_PHDR vs PT_PHDR. */
1429                 Phdr *phdr = app.phdr = (void *)aux[AT_PHDR];
1430                 app.phnum = aux[AT_PHNUM];
1431                 app.phentsize = aux[AT_PHENT];
1432                 for (i=aux[AT_PHNUM]; i; i--, phdr=(void *)((char *)phdr + aux[AT_PHENT])) {
1433                         if (phdr->p_type == PT_PHDR)
1434                                 app.base = (void *)(aux[AT_PHDR] - phdr->p_vaddr);
1435                         else if (phdr->p_type == PT_INTERP)
1436                                 interp_off = (size_t)phdr->p_vaddr;
1437                         else if (phdr->p_type == PT_TLS) {
1438                                 tls_image = phdr->p_vaddr;
1439                                 app.tls.len = phdr->p_filesz;
1440                                 app.tls.size = phdr->p_memsz;
1441                                 app.tls.align = phdr->p_align;
1442                         }
1443                 }
1444                 if (DL_FDPIC) app.loadmap = app_loadmap;
1445                 if (app.tls.size) app.tls.image = laddr(&app, tls_image);
1446                 if (interp_off) ldso.name = laddr(&app, interp_off);
1447                 if ((aux[0] & (1UL<<AT_EXECFN))
1448                     && strncmp((char *)aux[AT_EXECFN], "/proc/", 6))
1449                         app.name = (char *)aux[AT_EXECFN];
1450                 else
1451                         app.name = argv[0];
1452                 kernel_mapped_dso(&app);
1453         } else {
1454                 int fd;
1455                 char *ldname = argv[0];
1456                 size_t l = strlen(ldname);
1457                 if (l >= 3 && !strcmp(ldname+l-3, "ldd")) ldd_mode = 1;
1458                 argv++;
1459                 while (argv[0] && argv[0][0]=='-' && argv[0][1]=='-') {
1460                         char *opt = argv[0]+2;
1461                         *argv++ = (void *)-1;
1462                         if (!*opt) {
1463                                 break;
1464                         } else if (!memcmp(opt, "list", 5)) {
1465                                 ldd_mode = 1;
1466                         } else if (!memcmp(opt, "library-path", 12)) {
1467                                 if (opt[12]=='=') env_path = opt+13;
1468                                 else if (opt[12]) *argv = 0;
1469                                 else if (*argv) env_path = *argv++;
1470                         } else if (!memcmp(opt, "preload", 7)) {
1471                                 if (opt[7]=='=') env_preload = opt+8;
1472                                 else if (opt[7]) *argv = 0;
1473                                 else if (*argv) env_preload = *argv++;
1474                         } else {
1475                                 argv[0] = 0;
1476                         }
1477                 }
1478                 argv[-1] = (void *)(argc - (argv-argv_orig));
1479                 if (!argv[0]) {
1480                         dprintf(2, "musl libc\n"
1481                                 "Version %s\n"
1482                                 "Dynamic Program Loader\n"
1483                                 "Usage: %s [options] [--] pathname%s\n",
1484                                 __libc_get_version(), ldname,
1485                                 ldd_mode ? "" : " [args]");
1486                         _exit(1);
1487                 }
1488                 fd = open(argv[0], O_RDONLY);
1489                 if (fd < 0) {
1490                         dprintf(2, "%s: cannot load %s: %s\n", ldname, argv[0], strerror(errno));
1491                         _exit(1);
1492                 }
1493                 runtime = 1;
1494                 Ehdr *ehdr = (void *)map_library(fd, &app);
1495                 if (!ehdr) {
1496                         dprintf(2, "%s: %s: Not a valid dynamic program\n", ldname, argv[0]);
1497                         _exit(1);
1498                 }
1499                 runtime = 0;
1500                 close(fd);
1501                 ldso.name = ldname;
1502                 app.name = argv[0];
1503                 aux[AT_ENTRY] = (size_t)laddr(&app, ehdr->e_entry);
1504                 /* Find the name that would have been used for the dynamic
1505                  * linker had ldd not taken its place. */
1506                 if (ldd_mode) {
1507                         for (i=0; i<app.phnum; i++) {
1508                                 if (app.phdr[i].p_type == PT_INTERP)
1509                                         ldso.name = laddr(&app, app.phdr[i].p_vaddr);
1510                         }
1511                         dprintf(1, "\t%s (%p)\n", ldso.name, ldso.base);
1512                 }
1513         }
1514         if (app.tls.size) {
1515                 libc.tls_head = &app.tls;
1516                 app.tls_id = tls_cnt = 1;
1517 #ifdef TLS_ABOVE_TP
1518                 app.tls.offset = 0;
1519                 tls_offset = app.tls.size
1520                         + ( -((uintptr_t)app.tls.image + app.tls.size)
1521                         & (app.tls.align-1) );
1522 #else
1523                 tls_offset = app.tls.offset = app.tls.size
1524                         + ( -((uintptr_t)app.tls.image + app.tls.size)
1525                         & (app.tls.align-1) );
1526 #endif
1527                 tls_align = MAXP2(tls_align, app.tls.align);
1528         }
1529         app.global = 1;
1530         decode_dyn(&app);
1531         if (DL_FDPIC) {
1532                 makefuncdescs(&app);
1533                 if (!app.loadmap) {
1534                         app.loadmap = (void *)&app_dummy_loadmap;
1535                         app.loadmap->nsegs = 1;
1536                         app.loadmap->segs[0].addr = (size_t)app.map;
1537                         app.loadmap->segs[0].p_vaddr = (size_t)app.map
1538                                 - (size_t)app.base;
1539                         app.loadmap->segs[0].p_memsz = app.map_len;
1540                 }
1541                 argv[-3] = (void *)app.loadmap;
1542         }
1543
1544         /* Attach to vdso, if provided by the kernel */
1545         if (search_vec(auxv, &vdso_base, AT_SYSINFO_EHDR)) {
1546                 Ehdr *ehdr = (void *)vdso_base;
1547                 Phdr *phdr = vdso.phdr = (void *)(vdso_base + ehdr->e_phoff);
1548                 vdso.phnum = ehdr->e_phnum;
1549                 vdso.phentsize = ehdr->e_phentsize;
1550                 for (i=ehdr->e_phnum; i; i--, phdr=(void *)((char *)phdr + ehdr->e_phentsize)) {
1551                         if (phdr->p_type == PT_DYNAMIC)
1552                                 vdso.dynv = (void *)(vdso_base + phdr->p_offset);
1553                         if (phdr->p_type == PT_LOAD)
1554                                 vdso.base = (void *)(vdso_base - phdr->p_vaddr + phdr->p_offset);
1555                 }
1556                 vdso.name = "";
1557                 vdso.shortname = "linux-gate.so.1";
1558                 vdso.global = 1;
1559                 vdso.relocated = 1;
1560                 decode_dyn(&vdso);
1561                 vdso.prev = &ldso;
1562                 ldso.next = &vdso;
1563         }
1564
1565         /* Initial dso chain consists only of the app. */
1566         head = tail = &app;
1567
1568         /* Donate unused parts of app and library mapping to malloc */
1569         reclaim_gaps(&app);
1570         reclaim_gaps(&ldso);
1571
1572         /* Load preload/needed libraries, add their symbols to the global
1573          * namespace, and perform all remaining relocations. */
1574         if (env_preload) load_preload(env_preload);
1575         load_deps(&app);
1576         make_global(&app);
1577
1578 #ifndef DYNAMIC_IS_RO
1579         for (i=0; app.dynv[i]; i+=2)
1580                 if (app.dynv[i]==DT_DEBUG)
1581                         app.dynv[i+1] = (size_t)&debug;
1582 #endif
1583
1584         /* The main program must be relocated LAST since it may contin
1585          * copy relocations which depend on libraries' relocations. */
1586         reloc_all(app.next);
1587         reloc_all(&app);
1588
1589         update_tls_size();
1590         if (libc.tls_size > sizeof builtin_tls || tls_align > MIN_TLS_ALIGN) {
1591                 void *initial_tls = calloc(libc.tls_size, 1);
1592                 if (!initial_tls) {
1593                         dprintf(2, "%s: Error getting %zu bytes thread-local storage: %m\n",
1594                                 argv[0], libc.tls_size);
1595                         _exit(127);
1596                 }
1597                 if (__init_tp(__copy_tls(initial_tls)) < 0) {
1598                         a_crash();
1599                 }
1600         } else {
1601                 size_t tmp_tls_size = libc.tls_size;
1602                 pthread_t self = __pthread_self();
1603                 /* Temporarily set the tls size to the full size of
1604                  * builtin_tls so that __copy_tls will use the same layout
1605                  * as it did for before. Then check, just to be safe. */
1606                 libc.tls_size = sizeof builtin_tls;
1607                 if (__copy_tls((void*)builtin_tls) != self) a_crash();
1608                 libc.tls_size = tmp_tls_size;
1609         }
1610         static_tls_cnt = tls_cnt;
1611
1612         if (ldso_fail) _exit(127);
1613         if (ldd_mode) _exit(0);
1614
1615         /* Switch to runtime mode: any further failures in the dynamic
1616          * linker are a reportable failure rather than a fatal startup
1617          * error. */
1618         runtime = 1;
1619
1620         debug.ver = 1;
1621         debug.bp = dl_debug_state;
1622         debug.head = head;
1623         debug.base = ldso.base;
1624         debug.state = 0;
1625         _dl_debug_state();
1626
1627         errno = 0;
1628
1629         CRTJMP((void *)aux[AT_ENTRY], argv-1);
1630         for(;;);
1631 }
1632
1633 void *dlopen(const char *file, int mode)
1634 {
1635         struct dso *volatile p, *orig_tail, *next;
1636         struct tls_module *orig_tls_tail;
1637         size_t orig_tls_cnt, orig_tls_offset, orig_tls_align;
1638         size_t i;
1639         int cs;
1640         jmp_buf jb;
1641
1642         if (!file) return head;
1643
1644         pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cs);
1645         pthread_rwlock_wrlock(&lock);
1646         __inhibit_ptc();
1647
1648         p = 0;
1649         orig_tls_tail = tls_tail;
1650         orig_tls_cnt = tls_cnt;
1651         orig_tls_offset = tls_offset;
1652         orig_tls_align = tls_align;
1653         orig_tail = tail;
1654         noload = mode & RTLD_NOLOAD;
1655
1656         rtld_fail = &jb;
1657         if (setjmp(*rtld_fail)) {
1658                 /* Clean up anything new that was (partially) loaded */
1659                 if (p && p->deps) for (i=0; p->deps[i]; i++)
1660                         if (p->deps[i]->global < 0)
1661                                 p->deps[i]->global = 0;
1662                 for (p=orig_tail->next; p; p=next) {
1663                         next = p->next;
1664                         while (p->td_index) {
1665                                 void *tmp = p->td_index->next;
1666                                 free(p->td_index);
1667                                 p->td_index = tmp;
1668                         }
1669                         free(p->funcdescs);
1670                         if (p->rpath != p->rpath_orig)
1671                                 free(p->rpath);
1672                         free(p->deps);
1673                         unmap_library(p);
1674                         free(p);
1675                 }
1676                 if (!orig_tls_tail) libc.tls_head = 0;
1677                 tls_tail = orig_tls_tail;
1678                 tls_cnt = orig_tls_cnt;
1679                 tls_offset = orig_tls_offset;
1680                 tls_align = orig_tls_align;
1681                 tail = orig_tail;
1682                 tail->next = 0;
1683                 p = 0;
1684                 goto end;
1685         } else p = load_library(file, head);
1686
1687         if (!p) {
1688                 error(noload ?
1689                         "Library %s is not already loaded" :
1690                         "Error loading shared library %s: %m",
1691                         file);
1692                 goto end;
1693         }
1694
1695         /* First load handling */
1696         if (!p->deps) {
1697                 load_deps(p);
1698                 if (p->deps) for (i=0; p->deps[i]; i++)
1699                         if (!p->deps[i]->global)
1700                                 p->deps[i]->global = -1;
1701                 if (!p->global) p->global = -1;
1702                 reloc_all(p);
1703                 if (p->deps) for (i=0; p->deps[i]; i++)
1704                         if (p->deps[i]->global < 0)
1705                                 p->deps[i]->global = 0;
1706                 if (p->global < 0) p->global = 0;
1707         }
1708
1709         if (mode & RTLD_GLOBAL) {
1710                 if (p->deps) for (i=0; p->deps[i]; i++)
1711                         p->deps[i]->global = 1;
1712                 p->global = 1;
1713         }
1714
1715         update_tls_size();
1716         _dl_debug_state();
1717         orig_tail = tail;
1718 end:
1719         __release_ptc();
1720         if (p) gencnt++;
1721         pthread_rwlock_unlock(&lock);
1722         if (p) do_init_fini(orig_tail);
1723         pthread_setcancelstate(cs, 0);
1724         return p;
1725 }
1726
1727 static int invalid_dso_handle(void *h)
1728 {
1729         struct dso *p;
1730         for (p=head; p; p=p->next) if (h==p) return 0;
1731         error("Invalid library handle %p", (void *)h);
1732         return 1;
1733 }
1734
1735 static void *addr2dso(size_t a)
1736 {
1737         struct dso *p;
1738         size_t i;
1739         if (DL_FDPIC) for (p=head; p; p=p->next) {
1740                 i = count_syms(p);
1741                 if (a-(size_t)p->funcdescs < i*sizeof(*p->funcdescs))
1742                         return p;
1743         }
1744         for (p=head; p; p=p->next) {
1745                 if (DL_FDPIC && p->loadmap) {
1746                         for (i=0; i<p->loadmap->nsegs; i++) {
1747                                 if (a-p->loadmap->segs[i].p_vaddr
1748                                     < p->loadmap->segs[i].p_memsz)
1749                                         return p;
1750                         }
1751                 } else {
1752                         if (a-(size_t)p->map < p->map_len)
1753                                 return p;
1754                 }
1755         }
1756         return 0;
1757 }
1758
1759 void *__tls_get_addr(size_t *);
1760
1761 static void *do_dlsym(struct dso *p, const char *s, void *ra)
1762 {
1763         size_t i;
1764         uint32_t h = 0, gh = 0, *ght;
1765         Sym *sym;
1766         if (p == head || p == RTLD_DEFAULT || p == RTLD_NEXT) {
1767                 if (p == RTLD_DEFAULT) {
1768                         p = head;
1769                 } else if (p == RTLD_NEXT) {
1770                         p = addr2dso((size_t)ra);
1771                         if (!p) p=head;
1772                         p = p->next;
1773                 }
1774                 struct symdef def = find_sym(p, s, 0);
1775                 if (!def.sym) goto failed;
1776                 if ((def.sym->st_info&0xf) == STT_TLS)
1777                         return __tls_get_addr((size_t []){def.dso->tls_id, def.sym->st_value});
1778                 if (DL_FDPIC && (def.sym->st_info&0xf) == STT_FUNC)
1779                         return def.dso->funcdescs + (def.sym - def.dso->syms);
1780                 return laddr(def.dso, def.sym->st_value);
1781         }
1782         if (invalid_dso_handle(p))
1783                 return 0;
1784         if ((ght = p->ghashtab)) {
1785                 gh = gnu_hash(s);
1786                 sym = gnu_lookup(gh, ght, p, s);
1787         } else {
1788                 h = sysv_hash(s);
1789                 sym = sysv_lookup(s, h, p);
1790         }
1791         if (sym && (sym->st_info&0xf) == STT_TLS)
1792                 return __tls_get_addr((size_t []){p->tls_id, sym->st_value});
1793         if (DL_FDPIC && sym && sym->st_shndx && (sym->st_info&0xf) == STT_FUNC)
1794                 return p->funcdescs + (sym - p->syms);
1795         if (sym && sym->st_value && (1<<(sym->st_info&0xf) & OK_TYPES))
1796                 return laddr(p, sym->st_value);
1797         if (p->deps) for (i=0; p->deps[i]; i++) {
1798                 if ((ght = p->deps[i]->ghashtab)) {
1799                         if (!gh) gh = gnu_hash(s);
1800                         sym = gnu_lookup(gh, ght, p->deps[i], s);
1801                 } else {
1802                         if (!h) h = sysv_hash(s);
1803                         sym = sysv_lookup(s, h, p->deps[i]);
1804                 }
1805                 if (sym && (sym->st_info&0xf) == STT_TLS)
1806                         return __tls_get_addr((size_t []){p->deps[i]->tls_id, sym->st_value});
1807                 if (DL_FDPIC && sym && sym->st_shndx && (sym->st_info&0xf) == STT_FUNC)
1808                         return p->deps[i]->funcdescs + (sym - p->deps[i]->syms);
1809                 if (sym && sym->st_value && (1<<(sym->st_info&0xf) & OK_TYPES))
1810                         return laddr(p->deps[i], sym->st_value);
1811         }
1812 failed:
1813         error("Symbol not found: %s", s);
1814         return 0;
1815 }
1816
1817 int __dladdr(const void *addr, Dl_info *info)
1818 {
1819         struct dso *p;
1820         Sym *sym, *bestsym;
1821         uint32_t nsym;
1822         char *strings;
1823         void *best = 0;
1824
1825         pthread_rwlock_rdlock(&lock);
1826         p = addr2dso((size_t)addr);
1827         pthread_rwlock_unlock(&lock);
1828
1829         if (!p) return 0;
1830
1831         sym = p->syms;
1832         strings = p->strings;
1833         nsym = count_syms(p);
1834
1835         if (DL_FDPIC) {
1836                 size_t idx = ((size_t)addr-(size_t)p->funcdescs)
1837                         / sizeof(*p->funcdescs);
1838                 if (idx < nsym && (sym[idx].st_info&0xf) == STT_FUNC) {
1839                         best = p->funcdescs + idx;
1840                         bestsym = sym + idx;
1841                 }
1842         }
1843
1844         if (!best) for (; nsym; nsym--, sym++) {
1845                 if (sym->st_value
1846                  && (1<<(sym->st_info&0xf) & OK_TYPES)
1847                  && (1<<(sym->st_info>>4) & OK_BINDS)) {
1848                         void *symaddr = laddr(p, sym->st_value);
1849                         if (symaddr > addr || symaddr < best)
1850                                 continue;
1851                         best = symaddr;
1852                         bestsym = sym;
1853                         if (addr == symaddr)
1854                                 break;
1855                 }
1856         }
1857
1858         if (!best) return 0;
1859
1860         if (DL_FDPIC && (bestsym->st_info&0xf) == STT_FUNC)
1861                 best = p->funcdescs + (bestsym - p->syms);
1862
1863         info->dli_fname = p->name;
1864         info->dli_fbase = p->base;
1865         info->dli_sname = strings + bestsym->st_name;
1866         info->dli_saddr = best;
1867
1868         return 1;
1869 }
1870
1871 __attribute__((__visibility__("hidden")))
1872 void *__dlsym(void *restrict p, const char *restrict s, void *restrict ra)
1873 {
1874         void *res;
1875         pthread_rwlock_rdlock(&lock);
1876         res = do_dlsym(p, s, ra);
1877         pthread_rwlock_unlock(&lock);
1878         return res;
1879 }
1880
1881 int dl_iterate_phdr(int(*callback)(struct dl_phdr_info *info, size_t size, void *data), void *data)
1882 {
1883         struct dso *current;
1884         struct dl_phdr_info info;
1885         int ret = 0;
1886         for(current = head; current;) {
1887                 info.dlpi_addr      = (uintptr_t)current->base;
1888                 info.dlpi_name      = current->name;
1889                 info.dlpi_phdr      = current->phdr;
1890                 info.dlpi_phnum     = current->phnum;
1891                 info.dlpi_adds      = gencnt;
1892                 info.dlpi_subs      = 0;
1893                 info.dlpi_tls_modid = current->tls_id;
1894                 info.dlpi_tls_data  = current->tls.image;
1895
1896                 ret = (callback)(&info, sizeof (info), data);
1897
1898                 if (ret != 0) break;
1899
1900                 pthread_rwlock_rdlock(&lock);
1901                 current = current->next;
1902                 pthread_rwlock_unlock(&lock);
1903         }
1904         return ret;
1905 }
1906 #else
1907 static int invalid_dso_handle(void *h)
1908 {
1909         error("Invalid library handle %p", (void *)h);
1910         return 1;
1911 }
1912 void *dlopen(const char *file, int mode)
1913 {
1914         error("Dynamic loading not supported");
1915         return 0;
1916 }
1917 void *__dlsym(void *restrict p, const char *restrict s, void *restrict ra)
1918 {
1919         error("Symbol not found: %s", s);
1920         return 0;
1921 }
1922 int __dladdr (const void *addr, Dl_info *info)
1923 {
1924         return 0;
1925 }
1926 #endif
1927
1928 int __dlinfo(void *dso, int req, void *res)
1929 {
1930         if (invalid_dso_handle(dso)) return -1;
1931         if (req != RTLD_DI_LINKMAP) {
1932                 error("Unsupported request %d", req);
1933                 return -1;
1934         }
1935         *(struct link_map **)res = dso;
1936         return 0;
1937 }
1938
1939 char *dlerror()
1940 {
1941         pthread_t self = __pthread_self();
1942         if (!self->dlerror_flag) return 0;
1943         self->dlerror_flag = 0;
1944         char *s = self->dlerror_buf;
1945         if (s == (void *)-1)
1946                 return "Dynamic linker failed to allocate memory for error message";
1947         else
1948                 return s;
1949 }
1950
1951 int dlclose(void *p)
1952 {
1953         return invalid_dso_handle(p);
1954 }
1955
1956 void __dl_thread_cleanup(void)
1957 {
1958         pthread_t self = __pthread_self();
1959         if (self->dlerror_buf != (void *)-1)
1960                 free(self->dlerror_buf);
1961 }
1962
1963 static void error(const char *fmt, ...)
1964 {
1965         va_list ap;
1966         va_start(ap, fmt);
1967 #ifdef SHARED
1968         if (!runtime) {
1969                 vdprintf(2, fmt, ap);
1970                 dprintf(2, "\n");
1971                 ldso_fail = 1;
1972                 va_end(ap);
1973                 return;
1974         }
1975 #endif
1976         pthread_t self = __pthread_self();
1977         if (self->dlerror_buf != (void *)-1)
1978                 free(self->dlerror_buf);
1979         size_t len = vsnprintf(0, 0, fmt, ap);
1980         va_end(ap);
1981         char *buf = malloc(len+1);
1982         if (buf) {
1983                 va_start(ap, fmt);
1984                 vsnprintf(buf, len+1, fmt, ap);
1985                 va_end(ap);
1986         } else {
1987                 buf = (void *)-1;
1988         }
1989         self->dlerror_buf = buf;
1990         self->dlerror_flag = 1;
1991 }