ntdll: Reimplement FLS data management on top of up to date structures.

dlls/kernelbase/thread.c

@@ -762,7 +762,7 @@ struct fiber_data
     CONTEXT               context;           /* 14/30 fiber context */
     DWORD                 flags;             /*       fiber flags */
     LPFIBER_START_ROUTINE start;             /*       start routine */
-    void                **fls_slots;         /*       fiber storage slots */
+    void                 *fls_slots;         /*       fiber storage slots */
 };
 
 extern void WINAPI switch_fiber( CONTEXT *old, CONTEXT *new );

dlls/ntdll/loader.c

@@ -168,7 +168,6 @@ static RTL_CRITICAL_SECTION peb_lock = { &peb_critsect_debug, -1, 0, 0, 0, 0 };
 static PEB_LDR_DATA ldr = { sizeof(ldr), TRUE };
 static RTL_BITMAP tls_bitmap;
 static RTL_BITMAP tls_expansion_bitmap;
-static RTL_BITMAP fls_bitmap;
 
 static WINE_MODREF *cached_modref;
 static WINE_MODREF *current_modref;
@@ -4005,7 +4004,6 @@ static NTSTATUS process_init(void)
     peb->FastPebLock        = &peb_lock;
     peb->TlsBitmap          = &tls_bitmap;
     peb->TlsExpansionBitmap = &tls_expansion_bitmap;
-    peb->FlsBitmap          = &fls_bitmap;
     peb->LoaderLock         = &loader_section;
     peb->OSMajorVersion     = 5;
     peb->OSMinorVersion     = 1;
@@ -4014,13 +4012,11 @@ static NTSTATUS process_init(void)
     peb->SessionId          = 1;
     peb->ProcessHeap        = RtlCreateHeap( HEAP_GROWABLE, NULL, 0, 0, NULL, NULL );
 
-    InitializeListHead( &peb->FlsListHead );
     RtlInitializeBitMap( &tls_bitmap, peb->TlsBitmapBits, sizeof(peb->TlsBitmapBits) * 8 );
     RtlInitializeBitMap( &tls_expansion_bitmap, peb->TlsExpansionBitmapBits,
                          sizeof(peb->TlsExpansionBitmapBits) * 8 );
-    RtlInitializeBitMap( &fls_bitmap, peb->FlsBitmapBits, sizeof(peb->FlsBitmapBits) * 8 );
     RtlSetBits( peb->TlsBitmap, 0, 1 ); /* TLS index 0 is reserved and should be initialized to NULL. */
-    RtlSetBits( peb->FlsBitmap, 0, 1 );
+    init_global_fls_data();
 
     InitializeListHead( &ldr.InLoadOrderModuleList );
     InitializeListHead( &ldr.InMemoryOrderModuleList );

dlls/ntdll/ntdll_misc.h

@@ -127,4 +127,7 @@ static inline void ascii_to_unicode( WCHAR *dst, const char *src, size_t len )
     while (len--) *dst++ = (unsigned char)*src++;
 }
 
+/* FLS data */
+extern void init_global_fls_data(void) DECLSPEC_HIDDEN;
+
 #endif

dlls/ntdll/thread.c

@@ -34,6 +34,7 @@
 #include "wine/exception.h"
 
 WINE_DECLARE_DEBUG_CHANNEL(relay);
+WINE_DECLARE_DEBUG_CHANNEL(thread);
 
 struct _KUSER_SHARED_DATA *user_shared_data = (void *)0x7ffe0000;
 
@@ -253,41 +254,125 @@ TEB_ACTIVE_FRAME * WINAPI RtlGetFrame(void)
  ***********************************************************************/
 
 
+static GLOBAL_FLS_DATA fls_data;
+
+#define MAX_FLS_DATA_COUNT 0xff0
+
+void init_global_fls_data(void)
+{
+    InitializeListHead( &fls_data.fls_list_head );
+}
+
+static void lock_fls_data(void)
+{
+    RtlAcquirePebLock();
+}
+
+static void unlock_fls_data(void)
+{
+    RtlReleasePebLock();
+}
+
+static unsigned int fls_chunk_size( unsigned int chunk_index )
+{
+    return 0x10 << chunk_index;
+}
+
+static unsigned int fls_index_from_chunk_index( unsigned int chunk_index, unsigned int index )
+{
+    return 0x10 * ((1 << chunk_index) - 1) + index;
+}
+
+static unsigned int fls_chunk_index_from_index( unsigned int index, unsigned int *index_in_chunk )
+{
+    unsigned int chunk_index = 0;
+
+    while (index >= fls_chunk_size( chunk_index ))
+        index -= fls_chunk_size( chunk_index++ );
+
+    *index_in_chunk = index;
+    return chunk_index;
+}
+
+static TEB_FLS_DATA * fls_alloc_data(void)
+{
+    TEB_FLS_DATA *fls;
+
+    if (!(fls = RtlAllocateHeap( GetProcessHeap(), HEAP_ZERO_MEMORY, sizeof(*fls) )))
+        return NULL;
+
+    lock_fls_data();
+    InsertTailList( &fls_data.fls_list_head, &fls->fls_list_entry );
+    unlock_fls_data();
+
+    return fls;
+}
+
+
 /***********************************************************************
  *              RtlFlsAlloc  (NTDLL.@)
  */
 NTSTATUS WINAPI DECLSPEC_HOTPATCH RtlFlsAlloc( PFLS_CALLBACK_FUNCTION callback, ULONG *ret_index )
 {
-    PEB * const peb = NtCurrentTeb()->Peb;
-    NTSTATUS status = STATUS_NO_MEMORY;
-    DWORD index;
+    unsigned int chunk_index, index, i;
+    FLS_INFO_CHUNK *chunk;
+    TEB_FLS_DATA *fls;
 
-    RtlAcquirePebLock();
-    if (peb->FlsCallback ||
-        (peb->FlsCallback = RtlAllocateHeap( GetProcessHeap(), HEAP_ZERO_MEMORY,
-                                        8 * sizeof(peb->FlsBitmapBits) * sizeof(void*) )))
+    if (!(fls = NtCurrentTeb()->FlsSlots)
+            && !(NtCurrentTeb()->FlsSlots = fls = fls_alloc_data()))
+        return STATUS_NO_MEMORY;
+
+    lock_fls_data();
+    for (i = 0; i < ARRAY_SIZE(fls_data.fls_callback_chunks); ++i)
     {
-        index = RtlFindClearBitsAndSet( peb->FlsBitmap, 1, 1 );
-        if (index != ~0U)
+        if (!fls_data.fls_callback_chunks[i] || fls_data.fls_callback_chunks[i]->count < fls_chunk_size( i ))
+            break;
+    }
+
+    if ((chunk_index = i) == ARRAY_SIZE(fls_data.fls_callback_chunks))
     {
-            if (!NtCurrentTeb()->FlsSlots &&
-                !(NtCurrentTeb()->FlsSlots = RtlAllocateHeap( GetProcessHeap(), HEAP_ZERO_MEMORY,
-                                                        8 * sizeof(peb->FlsBitmapBits) * sizeof(void*) )))
+        unlock_fls_data();
+        return STATUS_NO_MEMORY;
+    }
+
+    if ((chunk = fls_data.fls_callback_chunks[chunk_index]))
     {
-                RtlClearBits( peb->FlsBitmap, index, 1 );
+        for (index = 0; index < fls_chunk_size( chunk_index ); ++index)
+            if (!chunk->callbacks[index].callback)
+                break;
+        assert( index < fls_chunk_size( chunk_index ));
     }
     else
     {
-                NtCurrentTeb()->FlsSlots[index] = 0; /* clear the value */
-                peb->FlsCallback[index] = callback;
-                status = STATUS_SUCCESS;
+        fls_data.fls_callback_chunks[chunk_index] = chunk = RtlAllocateHeap( GetProcessHeap(), HEAP_ZERO_MEMORY,
+                offsetof(FLS_INFO_CHUNK, callbacks) + sizeof(*chunk->callbacks) * fls_chunk_size( chunk_index ));
+        if (!chunk)
+        {
+            unlock_fls_data();
+            return STATUS_NO_MEMORY;
+        }
+
+        if (chunk_index)
+        {
+            index = 0;
         }
+        else
+        {
+            chunk->count = 1; /* FLS index 0 is prohibited. */
+            chunk->callbacks[0].callback = (void *)~(ULONG_PTR)0;
+            index = 1;
         }
     }
-    RtlReleasePebLock();
-    if (!status)
-        *ret_index = index;
-    return status;
+
+    ++chunk->count;
+    chunk->callbacks[index].callback = callback ? callback : (void *)~(ULONG_PTR)0;
+
+    if ((*ret_index = fls_index_from_chunk_index( chunk_index, index )) > fls_data.fls_high_index)
+        fls_data.fls_high_index = *ret_index;
+
+    unlock_fls_data();
+
+    return STATUS_SUCCESS;
 }
 
 
@@ -296,20 +381,37 @@ NTSTATUS WINAPI DECLSPEC_HOTPATCH RtlFlsAlloc( PFLS_CALLBACK_FUNCTION callback,
  */
 NTSTATUS WINAPI DECLSPEC_HOTPATCH RtlFlsFree( ULONG index )
 {
-    NTSTATUS status;
+    unsigned int chunk_index, idx;
+    FLS_INFO_CHUNK *chunk;
+    TEB_FLS_DATA *fls;
 
-    RtlAcquirePebLock();
-    if (RtlAreBitsSet( NtCurrentTeb()->Peb->FlsBitmap, index, 1 ))
+    lock_fls_data();
+
+    if (!index || index > fls_data.fls_high_index)
+    {
+        unlock_fls_data();
+        return STATUS_INVALID_PARAMETER;
+    }
+
+    chunk_index = fls_chunk_index_from_index( index, &idx );
+    if (!(chunk = fls_data.fls_callback_chunks[chunk_index])
+            || !chunk->callbacks[idx].callback)
+    {
+        unlock_fls_data();
+        return STATUS_INVALID_PARAMETER;
+    }
+
+    if ((fls = NtCurrentTeb()->FlsSlots) && fls->fls_data_chunks[chunk_index])
     {
-        RtlClearBits( NtCurrentTeb()->Peb->FlsBitmap, index, 1 );
         /* FIXME: call Fls callback */
-        /* FIXME: add equivalent of ThreadZeroTlsCell here */
-        if (NtCurrentTeb()->FlsSlots) NtCurrentTeb()->FlsSlots[index] = 0;
-        status = STATUS_SUCCESS;
+        fls->fls_data_chunks[chunk_index][idx + 1] = NULL;
     }
-    else status = STATUS_INVALID_PARAMETER;
-    RtlReleasePebLock();
-    return status;
+
+    --chunk->count;
+    chunk->callbacks[idx].callback = NULL;
+
+    unlock_fls_data();
+    return STATUS_SUCCESS;
 }
 
 
@@ -318,15 +420,25 @@ NTSTATUS WINAPI DECLSPEC_HOTPATCH RtlFlsFree( ULONG index )
  */
 NTSTATUS WINAPI DECLSPEC_HOTPATCH RtlFlsSetValue( ULONG index, void *data )
 {
-    if (!index || index >= 8 * sizeof(NtCurrentTeb()->Peb->FlsBitmapBits))
+    unsigned int chunk_index, idx;
+    TEB_FLS_DATA *fls;
+
+    if (!index || index >= MAX_FLS_DATA_COUNT)
         return STATUS_INVALID_PARAMETER;
 
-    if (!NtCurrentTeb()->FlsSlots &&
-        !(NtCurrentTeb()->FlsSlots = RtlAllocateHeap( GetProcessHeap(), HEAP_ZERO_MEMORY,
-                                        8 * sizeof(NtCurrentTeb()->Peb->FlsBitmapBits) * sizeof(void*) )))
+    if (!(fls = NtCurrentTeb()->FlsSlots)
+            && !(NtCurrentTeb()->FlsSlots = fls = fls_alloc_data()))
         return STATUS_NO_MEMORY;
 
-    NtCurrentTeb()->FlsSlots[index] = data;
+    chunk_index = fls_chunk_index_from_index( index, &idx );
+
+    if (!fls->fls_data_chunks[chunk_index] &&
+            !(fls->fls_data_chunks[chunk_index] = RtlAllocateHeap( GetProcessHeap(), HEAP_ZERO_MEMORY,
+            (fls_chunk_size( chunk_index ) + 1) * sizeof(*fls->fls_data_chunks[chunk_index]) )))
+        return STATUS_NO_MEMORY;
+
+    fls->fls_data_chunks[chunk_index][idx + 1] = data;
+
     return STATUS_SUCCESS;
 }
 
@@ -336,11 +448,15 @@ NTSTATUS WINAPI DECLSPEC_HOTPATCH RtlFlsSetValue( ULONG index, void *data )
  */
 NTSTATUS WINAPI DECLSPEC_HOTPATCH RtlFlsGetValue( ULONG index, void **data )
 {
-    if (!index || index >= 8 * sizeof(NtCurrentTeb()->Peb->FlsBitmapBits) || !NtCurrentTeb()->FlsSlots)
+    unsigned int chunk_index, idx;
+    TEB_FLS_DATA *fls;
+
+    if (!index || index >= MAX_FLS_DATA_COUNT || !(fls = NtCurrentTeb()->FlsSlots))
         return STATUS_INVALID_PARAMETER;
 
-    *data = NtCurrentTeb()->FlsSlots[index];
+    chunk_index = fls_chunk_index_from_index( index, &idx );
 
+    *data = fls->fls_data_chunks[chunk_index] ? fls->fls_data_chunks[chunk_index][idx + 1] : NULL;
     return STATUS_SUCCESS;
 }
 
@@ -350,6 +466,31 @@ NTSTATUS WINAPI DECLSPEC_HOTPATCH RtlFlsGetValue( ULONG index, void **data )
  */
 void WINAPI DECLSPEC_HOTPATCH RtlProcessFlsData( void *teb_fls_data, ULONG flags )
 {
+    TEB_FLS_DATA *fls = teb_fls_data;
+    unsigned int i;
+
+    TRACE_(thread)( "teb_fls_data %p, flags %#x.\n", teb_fls_data, flags );
+
+    if (flags & ~3)
+        FIXME_(thread)( "Unknown flags %#x.\n", flags );
+
+    if (!fls)
+        return;
+
+    if (flags & 1)
+    {
+        lock_fls_data();
+        /* Not using RemoveEntryList() as Windows does not zero list entry here. */
+        fls->fls_list_entry.Flink->Blink = fls->fls_list_entry.Blink;
+        fls->fls_list_entry.Blink->Flink = fls->fls_list_entry.Flink;
+        unlock_fls_data();
+    }
+
     if (flags & 2)
-        RtlFreeHeap( GetProcessHeap(), 0, teb_fls_data );
+    {
+        for (i = 0; i < ARRAY_SIZE(fls->fls_data_chunks); ++i)
+            RtlFreeHeap( GetProcessHeap(), 0, fls->fls_data_chunks[i] );
+
+        RtlFreeHeap( GetProcessHeap(), 0, fls );
+    }
 }

include/winternl.h

@@ -265,6 +265,32 @@ typedef struct _TEB_ACTIVE_FRAME_EX
     void            *ExtensionIdentifier;
 } TEB_ACTIVE_FRAME_EX, *PTEB_ACTIVE_FRAME_EX;
 
+typedef struct _FLS_CALLBACK
+{
+    void                  *unknown;
+    PFLS_CALLBACK_FUNCTION callback; /* ~0 if NULL callback is set, NULL if FLS index is free. */
+} FLS_CALLBACK, *PFLS_CALLBACK;
+
+typedef struct _FLS_INFO_CHUNK
+{
+    ULONG           count;         /* number of allocated FLS indexes in the chunk. */
+    FLS_CALLBACK    callbacks[1];  /* the size is 0x10 for chunk 0 and is twice as
+                                    * the previous chunk size for the rest. */
+} FLS_INFO_CHUNK, *PFLS_INFO_CHUNK;
+
+typedef struct _GLOBAL_FLS_DATA
+{
+    FLS_INFO_CHUNK *fls_callback_chunks[8];
+    LIST_ENTRY      fls_list_head;
+    ULONG           fls_high_index;
+} GLOBAL_FLS_DATA, *PGLOBAL_FLS_DATA;
+
+typedef struct _TEB_FLS_DATA
+{
+    LIST_ENTRY      fls_list_entry;
+    void          **fls_data_chunks[8];
+} TEB_FLS_DATA, *PTEB_FLS_DATA;
+
 #define TEB_ACTIVE_FRAME_CONTEXT_FLAG_EXTENDED 0x00000001
 #define TEB_ACTIVE_FRAME_FLAG_EXTENDED         0x00000001
 
@@ -446,7 +472,7 @@ typedef struct _TEB
     ULONG                        HeapVirtualAffinity;               /* fa8/17b0 */
     PVOID                        CurrentTransactionHandle;          /* fac/17b8 */
     TEB_ACTIVE_FRAME            *ActiveFrame;                       /* fb0/17c0 */
-    PVOID                       *FlsSlots;                          /* fb4/17c8 */
+    TEB_FLS_DATA                *FlsSlots;                          /* fb4/17c8 */
     PVOID                        PreferredLanguages;                /* fb8/17d0 */
     PVOID                        UserPrefLanguages;                 /* fbc/17d8 */
     PVOID                        MergedPrefLanguages;               /* fc0/17e0 */