Emulate the behaviour of Linux mmap() on Solaris. Based on ideas and

library/port.c

@@ -396,6 +396,87 @@ int getrlimit (int resource, struct rlimit *rlim)
 }
 #endif /* HAVE_GETRLIMIT */
 
+
+#ifdef __svr4__
+/***********************************************************************
+ *             solaris_try_mmap
+ *
+ * The purpose of this routine is to emulate the behaviour of
+ * the Linux mmap() routine if a non-NULL address is passed,
+ * but the MAP_FIXED flag is not set.  Linux in this case tries
+ * to place the mapping at the specified address, *unless* the
+ * range is already in use.  Solaris, however, completely ignores
+ * the address argument in this case.
+ *
+ * As Wine code occasionally relies on the Linux behaviour, e.g. to
+ * be able to map non-relocateable PE executables to their proper
+ * start addresses, or to map the DOS memory to 0, this routine
+ * emulates the Linux behaviour by checking whether the desired
+ * address range is still available, and placing the mapping there
+ * using MAP_FIXED if so.
+ */
+static int solaris_try_mmap (void *addr, size_t len, int prot, int flags,
+                            int fildes, off_t off)
+{
+    char * volatile result = NULL;
+    int pagesize = getpagesize();
+    pid_t pid;
+
+    /* We only try to map to a fixed address if
+       addr is non-NULL and properly aligned,
+       and MAP_FIXED isn't already specified. */
+
+    if ( !addr )
+        return FALSE;
+    if ( (uintptr_t)addr & (pagesize-1) )
+        return FALSE;
+    if ( flags & MAP_FIXED )
+        return FALSE;
+
+    /* We use vfork() to freeze all threads of the
+       current process.  This allows us to check without
+       race condition whether the desired memory range is
+       already in use.  Note that because vfork() shares
+       the address spaces between parent and child, we
+       can actually perform the mapping in the child. */
+
+    if ( (pid = vfork()) == -1 )
+    {
+        perror("solaris_try_mmap: vfork");
+        exit(1);
+    }
+    if ( pid == 0 )
+    {
+        int i;
+        char vec;
+
+        /* We call mincore() for every page in the desired range.
+           If any of these calls succeeds, the page is already
+           mapped and we must fail. */
+        for ( i = 0; i < len; i += pagesize )
+            if ( mincore( (caddr_t)addr + i, pagesize, &vec ) != -1 )
+               _exit(1);
+
+        /* Perform the mapping with MAP_FIXED set.  This is safe
+           now, as none of the pages is currently in use. */
+        result = mmap( addr, len, prot, flags | MAP_FIXED, fildes, off );
+        if ( result == addr )
+            _exit(0);
+
+        if ( result != (void *) -1 ) /* This should never happen ... */
+            munmap( result, len );
+
+       _exit(1);
+    }
+
+    /* vfork() lets the parent continue only after the child
+       has exited.  Furthermore, Wine sets SIGCHLD to SIG_IGN,
+       so we don't need to wait for the child. */
+
+    return result == addr;
+}
+#endif
+
 /***********************************************************************
  *		wine_anon_mmap
  *
@@ -427,6 +508,11 @@ void *wine_anon_mmap( void *start, size_t size, int prot, int flags )
     flags |= MAP_PRIVATE;
 #endif
 
+#ifdef __svr4__
+    if ( solaris_try_mmap( start, size, prot, flags, fdzero, 0 ) )
+        return start;
+#endif
+
     return mmap( start, size, prot, flags, fdzero, 0 );
 }