oleaut32: Fix some corner cases in VarBstrCmp.

dlls/oleaut32/tests/vartype.c

@@ -4983,6 +4983,8 @@ static void test_VarBstrCmp(void)
     static const WCHAR s2[] = { 'a',0,'b' };
     static const char sb1[] = {1,0,1};
     static const char sb2[] = {1,0,2};
+    static const char sbchr0[] = {0,0};
+    static const char sbchr00[] = {0,0,0};
     BSTR bstr, bstrempty, bstr2;
 
     CHECKPTR(VarBstrCmp);
@@ -5021,6 +5023,15 @@ static void test_VarBstrCmp(void)
 
     SysFreeString(bstr);
 
+    bstr = SysAllocStringByteLen(sbchr0, sizeof(sbchr0));
+    bstr2 = SysAllocStringByteLen(sbchr0, sizeof(sbchr00));
+    VARBSTRCMP(bstr,bstrempty,0,VARCMP_GT);
+    VARBSTRCMP(bstrempty,bstr,0,VARCMP_LT);
+    VARBSTRCMP(bstr2,bstrempty,0,VARCMP_GT);
+    VARBSTRCMP(bstr2,bstr,0,VARCMP_EQ);
+    SysFreeString(bstr2);
+    SysFreeString(bstr);
+
     /* When (LCID == 0) it should be a binary comparison
      * so these two strings could not match.
      */
@@ -5030,6 +5041,15 @@ static void test_VarBstrCmp(void)
     VARBSTRCMP(bstr,bstr2,0,VARCMP_LT);
     SysFreeString(bstr2);
     SysFreeString(bstr);
+
+    bstr = SysAllocStringByteLen(sbchr0, sizeof(sbchr0));
+    bstr2 = SysAllocStringByteLen(sbchr0, sizeof(sbchr00));
+    VARBSTRCMP(bstr,bstrempty,0,VARCMP_GT);
+    VARBSTRCMP(bstrempty,bstr,0,VARCMP_LT);
+    VARBSTRCMP(bstr2,bstrempty,0,VARCMP_GT);
+    VARBSTRCMP(bstr2,bstr,0,VARCMP_GT);
+    SysFreeString(bstr2);
+    SysFreeString(bstr);
 }
 
 /* Get the internal representation of a BSTR */

dlls/oleaut32/vartype.c

@@ -6957,9 +6957,8 @@ HRESULT WINAPI VarBstrCmp(BSTR pbstrLeft, BSTR pbstrRight, LCID lcid, DWORD dwFl
 
     if (!pbstrLeft || !*pbstrLeft)
     {
-      if (!pbstrRight || !*pbstrRight)
-        return VARCMP_EQ;
-      return VARCMP_LT;
+      if (pbstrRight && *pbstrRight)
+        return VARCMP_LT;
     }
     else if (!pbstrRight || !*pbstrRight)
         return VARCMP_GT;
@@ -6981,8 +6980,17 @@ HRESULT WINAPI VarBstrCmp(BSTR pbstrLeft, BSTR pbstrRight, LCID lcid, DWORD dwFl
     }
     else
     {
-      hres = CompareStringW(lcid, dwFlags, pbstrLeft, SysStringLen(pbstrLeft),
-              pbstrRight, SysStringLen(pbstrRight)) - 1;
+      unsigned int lenLeft = SysStringLen(pbstrLeft);
+      unsigned int lenRight = SysStringLen(pbstrRight);
+
+      if (lenLeft == 0 || lenRight == 0)
+      {
+          if (lenLeft == 0 && lenRight == 0) return VARCMP_EQ;
+          return lenLeft < lenRight ? VARCMP_LT : VARCMP_GT;
+      }
+
+      hres = CompareStringW(lcid, dwFlags, pbstrLeft, lenLeft,
+              pbstrRight, lenRight) - 1;
       TRACE("%d\n", hres);
       return hres;
     }