disallow creation of objects larger than PTRDIFF_MAX via mmap

internally, other parts of the library assume sizes don't overflow
ssize_t and/or ptrdiff_t, and the way this assumption is made valid is
by preventing creating of such large objects. malloc already does so,
but the check was missing from mmap.

this is also a quality of implementation issue: even if the
implementation internally could handle such objects, applications
could inadvertently invoke undefined behavior by subtracting pointers
within an object. it is very difficult to guard against this in
applications, so a good implementation should simply ensure that it
does not happen.

diff --git a/src/mman/mmap.c b/src/mman/mmap.c
index e99271f7e5dd7092cf5b84201906527425151ed9..b56cff86b6da4b9265b3a1350fb4ea54c0f46015 100644 (file)
--- a/src/mman/mmap.c
+++ b/src/mman/mmap.c
@@ -1,6 +1,7 @@
 #include <unistd.h>
 #include <sys/mman.h>
 #include <errno.h>
+#include <stdint.h>
 #include <limits.h>
 #include "syscall.h"
 #include "libc.h"
@@ -20,6 +21,10 @@ void *__mmap(void *start, size_t len, int prot, int flags, int fd, off_t off)
                errno = EINVAL;
                return MAP_FAILED;
        }
+       if (len >= PTRDIFF_MAX) {
+               errno = ENOMEM;
+               return MAP_FAILED;
+       }
        if (flags & MAP_FIXED) __vm_lock(-1);
 #ifdef SYS_mmap2
        ret = (void *)syscall(SYS_mmap2, start, len, prot, flags, fd, off>>12);