work around incorrect EPERM from mmap syscall

under some conditions, the mmap syscall wrongly fails with EPERM
instead of ENOMEM when memory is exhausted; this is probably the
result of the kernel trying to fit the allocation somewhere that
crosses into the kernel range or below mmap_min_addr. in any case it's
a conformance bug, so work around it. for now, only handle the case of
anonymous mappings with no requested address; in other cases EPERM may
be a legitimate error.

this indirectly fixes the possibility of malloc failing with the wrong
errno value.

diff --git a/src/mman/mmap.c b/src/mman/mmap.c
index 19caadbdb6ef75da3df74f1edb9f06bf8aeaa541..15924033f3e3c501fc48ba7a3be581abd336687e 100644 (file)
--- a/src/mman/mmap.c
+++ b/src/mman/mmap.c
@@ -14,6 +14,7 @@ weak_alias(dummy, __vm_wait);
 
 void *__mmap(void *start, size_t len, int prot, int flags, int fd, off_t off)
 {
+       long ret;
        if (off & OFF_MASK) {
                errno = EINVAL;
                return MAP_FAILED;
@@ -26,10 +27,14 @@ void *__mmap(void *start, size_t len, int prot, int flags, int fd, off_t off)
                __vm_wait();
        }
 #ifdef SYS_mmap2
-       return (void *)syscall(SYS_mmap2, start, len, prot, flags, fd, off/UNIT);
+       ret = __syscall(SYS_mmap2, start, len, prot, flags, fd, off/UNIT);
 #else
-       return (void *)syscall(SYS_mmap, start, len, prot, flags, fd, off);
+       ret = __syscall(SYS_mmap, start, len, prot, flags, fd, off);
 #endif
+       /* Fixup incorrect EPERM from kernel. */
+       if (ret == -EPERM && !start && (flags&MAP_ANON) && !(flags&MAP_FIXED))
+               ret = -ENOMEM;
+       return (void *)__syscall_ret(ret);
 }
 
 weak_alias(__mmap, mmap);