richedit: Return correct values when EM_SETTEXTMODE fails.

dlls/riched20/editor.c

@@ -4316,36 +4316,26 @@ LRESULT ME_HandleMessage(ME_TextEditor *editor, UINT msg, WPARAM wParam,
     return editor->mode;
   case EM_SETTEXTMODE:
   {
-    LRESULT ret;
     int mask = 0;
     int changes = 0;
-    GETTEXTLENGTHEX how;
 
-    /* CR/LF conversion required in 2.0 mode, verbatim in 1.0 mode */
-    how.flags = GTL_CLOSE | (editor->bEmulateVersion10 ? 0 : GTL_USECRLF) | GTL_NUMCHARS;
-    how.codepage = unicode ? 1200 : CP_ACP;
-    ret = ME_GetTextLengthEx(editor, &how);
-    if (!ret)
-    {
-      /*Check for valid wParam*/
-      if ((((wParam & TM_RICHTEXT) && ((wParam & TM_PLAINTEXT) << 1))) ||
-	  (((wParam & TM_MULTILEVELUNDO) && ((wParam & TM_SINGLELEVELUNDO) << 1))) ||
-	  (((wParam & TM_MULTICODEPAGE) && ((wParam & TM_SINGLECODEPAGE) << 1))))
-	return 1;
-      else
-      {
+    if (ME_GetTextLength(editor))
+      return E_UNEXPECTED;
+
+    /* Check for mutually exclusive flags in adjacent bits of wParam */
+    if ((wParam & (TM_RICHTEXT | TM_MULTILEVELUNDO | TM_MULTICODEPAGE)) &
+        (wParam & (TM_PLAINTEXT | TM_SINGLELEVELUNDO | TM_SINGLECODEPAGE)) << 1)
+      return E_INVALIDARG;
+
     if (wParam & (TM_RICHTEXT | TM_PLAINTEXT))
     {
-	  mask |= (TM_RICHTEXT | TM_PLAINTEXT);
-	  changes |= (wParam & (TM_RICHTEXT | TM_PLAINTEXT));
+      mask |= TM_RICHTEXT | TM_PLAINTEXT;
+      changes |= wParam & (TM_RICHTEXT | TM_PLAINTEXT);
     }
-	/*FIXME: Currently no support for undo level and code page options*/ 
-	editor->mode = (editor->mode & (~mask)) | changes;
+    /* FIXME: Currently no support for undo level and code page options */
+    editor->mode = (editor->mode & ~mask) | changes;
     return 0;
   }
-    }
-    return ret;
-  }
   case EM_SETPASSWORDCHAR:
   {
     editor->cPasswordMask = wParam;

dlls/riched20/tests/editor.c

@@ -1053,6 +1053,11 @@ static void test_EM_SETTEXTMODE(void)
   CHARRANGE cr;
   int rc = 0;
 
+  /*Attempt to use mutually exclusive modes*/
+  rc = SendMessage(hwndRichEdit, EM_SETTEXTMODE, (WPARAM) TM_PLAINTEXT|TM_RICHTEXT, 0);
+  ok(rc == E_INVALIDARG,
+     "EM_SETTEXTMODE: using mutually exclusive mode flags - returned: %x\n", rc);
+
   /*Test that EM_SETTEXTMODE fails if text exists within the control*/
   /*Insert text into the control*/
 
@@ -1060,7 +1065,8 @@ static void test_EM_SETTEXTMODE(void)
 
   /*Attempt to change the control to plain text mode*/
   rc = SendMessage(hwndRichEdit, EM_SETTEXTMODE, (WPARAM) TM_PLAINTEXT, 0);
-  ok(rc != 0, "EM_SETTEXTMODE: changed text mode in control containing text - returned: %d\n", rc);
+  ok(rc == E_UNEXPECTED,
+     "EM_SETTEXTMODE: changed text mode in control containing text - returned: %x\n", rc);
 
   /*Test that EM_SETTEXTMODE does not allow rich edit text to be pasted.
   If rich text is pasted, it should have the same formatting as the rest