check for fd exhaustion in forkpty

we cannot report failure after forking, so the idea is to ensure prior
to fork that fd 0,1,2 exist. this will prevent dup2 from possibly
hitting a resource limit and failing in the child process. fcntl
rather than dup2 is used prior to forking to avoid race conditions.

diff --git a/src/misc/forkpty.c b/src/misc/forkpty.c
index 0bbf2de2746bba58d38b258428700320415706f8..07f8d01eed66741e31ec3c83a07ab5d4306d2fd0 100644 (file)
--- a/src/misc/forkpty.c
+++ b/src/misc/forkpty.c
@@ -1,14 +1,25 @@
 #include <pty.h>
 #include <unistd.h>
 #include <sys/ioctl.h>
+#include <fcntl.h>
 
 int forkpty(int *m, char *name, const struct termios *tio, const struct winsize *ws)
 {
-       int s;
+       int s, t, i, istmp[3]={0};
        pid_t pid;
 
        if (openpty(m, &s, name, tio, ws) < 0) return -1;
-       pid = fork();
+
+       /* Ensure before forking that we don't exceed fd limit */
+       for (i=0; i<3; i++) {
+               if (fcntl(i, F_GETFL) < 0) {
+                       t = fcntl(s, F_DUPFD, i);
+                       if (t<0) break;
+                       else if (t!=i) close(t);
+                       else istmp[i] = 1;
+               }
+       }
+       pid = i==3 ? fork() : -1;
        if (!pid) {
                close(*m);
                setsid();
@@ -19,6 +30,8 @@ int forkpty(int *m, char *name, const struct termios *tio, const struct winsize
                if (s>2) close(s);
                return 0;
        }
+       for (i=0; i<3; i++)
+               if (istmp[i]) close(i);
        close(s);
        if (pid < 0) close(*m);
        return pid;