server: Limit the number of allocated handles before running out of memory.

server/handle.c

@@ -64,6 +64,7 @@ static struct handle_table *global_table;
 #define RESERVED_ALL           (RESERVED_INHERIT | RESERVED_CLOSE_PROTECT)
 
 #define MIN_HANDLE_ENTRIES  32
+#define MAX_HANDLE_ENTRIES  0x00ffffff
 
 
 /* handle to table index conversion */
@@ -191,13 +192,12 @@ struct handle_table *alloc_handle_table( struct process *process, int count )
 static int grow_handle_table( struct handle_table *table )
 {
     struct handle_entry *new_entries;
-    int count = table->count;
+    int count = min( table->count * 2, MAX_HANDLE_ENTRIES );
 
-    if (count >= INT_MAX / 2) return 0;
-    count *= 2;
-    if (!(new_entries = realloc( table->entries, count * sizeof(struct handle_entry) )))
+    if (count == table->count ||
+        !(new_entries = realloc( table->entries, count * sizeof(struct handle_entry) )))
     {
-        set_error( STATUS_NO_MEMORY );
+        set_error( STATUS_INSUFFICIENT_RESOURCES );
         return 0;
     }
     table->entries = new_entries;

server/trace.c

@@ -4582,6 +4582,7 @@ static const struct
     { "HANDLE_NOT_CLOSABLE",         STATUS_HANDLE_NOT_CLOSABLE },
     { "ILLEGAL_FUNCTION",            STATUS_ILLEGAL_FUNCTION },
     { "INSTANCE_NOT_AVAILABLE",      STATUS_INSTANCE_NOT_AVAILABLE },
+    { "INSUFFICIENT_RESOURCES",      STATUS_INSUFFICIENT_RESOURCES },
     { "INVALID_CID",                 STATUS_INVALID_CID },
     { "INVALID_DEVICE_REQUEST",      STATUS_INVALID_DEVICE_REQUEST },
     { "INVALID_FILE_FOR_SECTION",    STATUS_INVALID_FILE_FOR_SECTION },