ntdll: Try to raise the file limit above OPEN_MAX on MacOS.

OPEN_MAX is 10240, which is usually lower than the kern.maxfilesperproc sysctl value. Said value from sysctl works on Mojave to Monterey (and most likely earlier Mac OS versions, but I can't test). Since Big Sur we can successfully set the reported rlim_max value.

ntdll: Try to raise the file limit above OPEN_MAX on MacOS.
de7604b6 · Stefan Dösinger · Alexandre Julliard · 57943168 · de7604b6
Commit de7604b6 authored Aug 03, 2023 by Stefan Dösinger Committed by Alexandre Julliard Aug 14, 2023
Hide whitespace changes
Inline Side-by-side

Showing with 31 additions and 7 deletions

loader.c dlls/ntdll/unix/loader.c +31 -7

No files found.
--- a/dlls/ntdll/unix/loader.c
+++ b/dlls/ntdll/unix/loader.c
@@ -446,18 +446,42 @@ static void set_max_limit( int limit )
    if (!getrlimit( limit, &rlimit ))
    {
        rlimit.rlim_cur = rlimit.rlim_max;
-        if (setrlimit( limit, &rlimit ) != 0)
-        {
+
+        if (!setrlimit( limit, &rlimit ))
+            return;
+
 #if defined(__APPLE__) && defined(RLIMIT_NOFILE) && defined(OPEN_MAX)
+        if (limit == RLIMIT_NOFILE)
+        {
+            unsigned int nlimit = 0;
+            size_t size;
+
            /* On Leopard, setrlimit(RLIMIT_NOFILE, ...) fails on attempts to set
-             * rlim_cur above OPEN_MAX (even if rlim_max > OPEN_MAX). */
-            if (limit == RLIMIT_NOFILE && rlimit.rlim_cur > OPEN_MAX)
-            {
+             * rlim_cur above OPEN_MAX (even if rlim_max > OPEN_MAX).
+             *
+             * In later versions it can be set to kern.maxfilesperproc (from
+             * sysctl). In Big Sur and later it can be set to rlim_max. */
+            size = sizeof(nlimit);
+            if (sysctlbyname("kern.maxfilesperproc", &nlimit, &size, NULL, 0) != 0 || nlimit < OPEN_MAX)
                rlimit.rlim_cur = OPEN_MAX;
-                setrlimit( limit, &rlimit );
+            else
+                rlimit.rlim_cur = nlimit;
+
+            if (!setrlimit( limit, &rlimit ))
+            {
+                TRACE("Fallback 1: RLIMIT_NOFILE to kern.maxfilesperproc\n");
+                return;
+            }
+
+            rlimit.rlim_cur = OPEN_MAX;
+            if (!setrlimit( limit, &rlimit ))
+            {
+                TRACE("Fallback 2: RLIMIT_NOFILE to OPEN_MAX(%d)\n", OPEN_MAX);
+                return;
            }
-#endif
        }
+#endif
+        WARN("Failed to raise limit %d\n", limit);
    }
 }