Fix Bugzilla 24715 - std/process: Default to libc closefrom in spaw…

…nProcessPosix

The current implementation of spawnProcessPosix is broken on systems
with a large `ulimit -n` because it always OOMs making it impossible
to spawn processes. Using the libc implementation, when available, for
doing file descriptor operations en-mass solves this problem.

Signed-off-by: Andrei Horodniceanu <[email protected]>

std/process.d

@@ -880,6 +880,7 @@ version (Posix) private enum InternalError : ubyte
     doubleFork,
     malloc,
     preExec,
+    closefds_dup2,
 }
 
 /*
@@ -1008,7 +1009,7 @@ private Pid spawnProcessPosix(scope const(char[])[] args,
         if (config.flags & Config.Flags.detached)
             close(pidPipe[0]);
         close(forkPipe[0]);
-        immutable forkPipeOut = forkPipe[1];
+        auto forkPipeOut = forkPipe[1];
         immutable pidPipeOut = pidPipe[1];
 
         // Set the working directory.
@@ -1042,56 +1043,67 @@ private Pid spawnProcessPosix(scope const(char[])[] args,
 
             if (!(config.flags & Config.Flags.inheritFDs))
             {
-                // NOTE: malloc() and getrlimit() are not on the POSIX async
-                // signal safe functions list, but practically this should
-                // not be a problem. Java VM and CPython also use malloc()
-                // in its own implementation via opendir().
-                import core.stdc.stdlib : malloc;
-                import core.sys.posix.poll : pollfd, poll, POLLNVAL;
-                import core.sys.posix.sys.resource : rlimit, getrlimit, RLIMIT_NOFILE;
-
-                // Get the maximum number of file descriptors that could be open.
-                rlimit r;
-                if (getrlimit(RLIMIT_NOFILE, &r) != 0)
-                {
-                    abortOnError(forkPipeOut, InternalError.getrlimit, .errno);
-                }
-                immutable maxDescriptors = cast(int) r.rlim_cur;
-
-                // The above, less stdin, stdout, and stderr
-                immutable maxToClose = maxDescriptors - 3;
-
-                // Call poll() to see which ones are actually open:
-                auto pfds = cast(pollfd*) malloc(pollfd.sizeof * maxToClose);
-                if (pfds is null)
-                {
-                    abortOnError(forkPipeOut, InternalError.malloc, .errno);
-                }
-                foreach (i; 0 .. maxToClose)
-                {
-                    pfds[i].fd = i + 3;
-                    pfds[i].events = 0;
-                    pfds[i].revents = 0;
-                }
-                if (poll(pfds, maxToClose, 0) >= 0)
-                {
-                    foreach (i; 0 .. maxToClose)
-                    {
-                        // don't close pipe write end
-                        if (pfds[i].fd == forkPipeOut) continue;
-                        // POLLNVAL will be set if the file descriptor is invalid.
-                        if (!(pfds[i].revents & POLLNVAL)) close(pfds[i].fd);
-                    }
-                }
+                version (CRuntime_Glibc)
+                    import core.sys.linux.unistd : closefrom;
+                else version (FreeBSD)
+                    import core.sys.freebsd.unistd : closefrom;
+                else version (OpenBSD)
+                    import core.sys.openbsd.unistd : closefrom;
                 else
-                {
-                    // Fall back to closing everything.
-                    foreach (i; 3 .. maxDescriptors)
-                    {
-                        if (i == forkPipeOut) continue;
-                        close(i);
+                    void closefrom (int lowfd) {
+                        // NOTE: malloc() and getrlimit() are not on the POSIX async
+                        // signal safe functions list, but practically this should
+                        // not be a problem. Java VM and CPython also use malloc()
+                        // in its own implementation via opendir().
+                        import core.stdc.stdlib : malloc;
+                        import core.sys.posix.poll : pollfd, poll, POLLNVAL;
+                        import core.sys.posix.sys.resource : rlimit, getrlimit, RLIMIT_NOFILE;
+
+                        // Get the maximum number of file descriptors that could be open.
+                        rlimit r;
+                        if (getrlimit(RLIMIT_NOFILE, &r) != 0)
+                        {
+                            abortOnError(forkPipeOut, InternalError.getrlimit, .errno);
+                        }
+                        immutable maxDescriptors = cast(int) r.rlim_cur;
+
+                        immutable maxToClose = maxDescriptors - lowfd;
+
+                        // Call poll() to see which ones are actually open:
+                        auto pfds = cast(pollfd*) malloc(pollfd.sizeof * maxToClose);
+                        if (pfds is null)
+                        {
+                            abortOnError(forkPipeOut, InternalError.malloc, .errno);
+                        }
+                        foreach (i; 0 .. maxToClose)
+                        {
+                            pfds[i].fd = i + lowfd;
+                            pfds[i].events = 0;
+                            pfds[i].revents = 0;
+                        }
+                        if (poll(pfds, maxToClose, 0) >= 0)
+                        {
+                            foreach (i; 0 .. maxToClose)
+                            {
+                                // POLLNVAL will be set if the file descriptor is invalid.
+                                if (!(pfds[i].revents & POLLNVAL)) close(pfds[i].fd);
+                            }
+                        }
+                        else
+                        {
+                            // Fall back to closing everything.
+                            foreach (i; lowfd .. maxDescriptors)
+                            {
+                                close(i);
+                            }
+                        }
                     }
-                }
+
+                if (dup2(forkPipeOut, 3) == -1)
+                    abortOnError(forkPipeOut, InternalError.closefds_dup2, .errno);
+                forkPipeOut = 3;
+                setCLOEXEC(forkPipeOut, true);
+                closefrom(forkPipeOut + 1);
             }
             else // This is already done if we don't inherit descriptors.
             {
@@ -1205,6 +1217,10 @@ private Pid spawnProcessPosix(scope const(char[])[] args,
                 case InternalError.preExec:
                     errorMsg = "Failed to execute preExecFunction or preExecDelegate";
                     break;
+                case InternalError.closefds_dup2:
+                    assert(!(config.flags & Config.Flags.inheritFDs));
+                    errorMsg = "Failed to close inherited file descriptors";
+                    break;
                 case InternalError.noerror:
                     assert(false);
             }