user32: Handle invalid dialog handles in IsDialogMessage().

dlls/user32/dialog.c

@@ -1152,6 +1152,9 @@ BOOL WINAPI IsDialogMessageW( HWND hwndDlg, LPMSG msg )
 {
     INT dlgCode;
 
+    if (!IsWindow( hwndDlg ))
+        return FALSE;
+
     if (CallMsgFilterW( msg, MSGF_DIALOGBOX )) return TRUE;
 
     hwndDlg = WIN_GetFullHandle( hwndDlg );

dlls/user32/tests/dialog.c

@@ -710,8 +710,23 @@ static void test_WM_NEXTDLGCTL(void)
     DestroyWindow(g_hwndTestDlg);
 }
 
+static LRESULT CALLBACK hook_proc(INT code, WPARAM wParam, LPARAM lParam)
+{
+    ok(0, "unexpected hook called, code %d\n", code);
+    return CallNextHookEx(NULL, code, wParam, lParam);
+}
+
+static BOOL g_MSGF_DIALOGBOX;
+static LRESULT CALLBACK hook_proc2(INT code, WPARAM wParam, LPARAM lParam)
+{
+    ok(code == MSGF_DIALOGBOX, "unexpected hook called, code %d\n", code);
+    g_MSGF_DIALOGBOX = code == MSGF_DIALOGBOX;
+    return CallNextHookEx(NULL, code, wParam, lParam);
+}
+
 static void test_IsDialogMessage(void)
 {
+    HHOOK hook;
     MSG msg;
 
     g_hwndMain = CreateWindowA("IsDialogMessageWindowClass", "IsDialogMessageWindowClass",
@@ -723,11 +738,34 @@ static void test_IsDialogMessage(void)
     assert (g_hwndButton1);
     assert (g_hwndButtonCancel);
 
+    if (0)
+    {
+        /* crashes on Windows */
+        IsDialogMessageA(NULL, NULL);
+        IsDialogMessageA(g_hwndMain, NULL);
+    }
+
     /* The focus should initially be nowhere.  The first TAB should take it
      * to the first button.  The second TAB should take it to the Cancel
      * button.
      */
+
+    /* valid window, invalid message window */
+    hook = SetWindowsHookExA(WH_MSGFILTER, hook_proc2, NULL, GetCurrentThreadId());
+    FormTabMsg (&msg, (HWND)0xbeefbeef);
+    ok (!IsDialogMessageA(g_hwndMain, &msg), "expected failure\n");
+    ok(g_MSGF_DIALOGBOX, "hook wasn't called\n");
+    g_MSGF_DIALOGBOX = FALSE;
+    UnhookWindowsHookEx(hook);
+
+    hook = SetWindowsHookExA(WH_MSGFILTER, hook_proc, NULL, GetCurrentThreadId());
     FormTabMsg (&msg, g_hwndMain);
+
+    ok (!IsDialogMessageA(NULL, &msg), "expected failure\n");
+    ok (!IsDialogMessageA((HWND)0xbeefbeef, &msg), "expected failure\n");
+
+    UnhookWindowsHookEx(hook);
+
     ok (IsDialogMessageA(g_hwndMain, &msg), "Did not handle first TAB\n");
     ok ((GetFocus() == g_hwndButton1), "Focus did not move to first button\n");
     FormTabMsg (&msg, g_hwndButton1);
@@ -737,6 +775,18 @@ static void test_IsDialogMessage(void)
     FormEnterMsg (&msg, g_hwndButtonCancel);
     ok (IsDialogMessageA(g_hwndMain, &msg), "Did not handle the ENTER\n");
     ok (g_terminated, "ENTER did not terminate\n");
+
+    /* matching but invalid window handles, NULL */
+    hook = SetWindowsHookExA(WH_MSGFILTER, hook_proc, NULL, GetCurrentThreadId());
+
+    FormTabMsg (&msg, NULL);
+    ok (!IsDialogMessageA(msg.hwnd, &msg), "expected failure\n");
+
+    /* matching but invalid window handles, not NULL */
+    FormTabMsg (&msg, (HWND)0xbeefbeef);
+    ok (!IsDialogMessageA(msg.hwnd, &msg), "expected failure\n");
+
+    UnhookWindowsHookEx(hook);
 }