rsaenh: Fail on unsupported flag values only in CryptHashData().

dlls/rsaenh/rsaenh.c

@@ -4114,7 +4114,7 @@ BOOL WINAPI RSAENH_CPHashData(HCRYPTPROV hProv, HCRYPTHASH hHash, const BYTE *pb
     TRACE("(hProv=%08lx, hHash=%08lx, pbData=%p, dwDataLen=%d, dwFlags=%08x)\n",
           hProv, hHash, pbData, dwDataLen, dwFlags);
 
-    if (dwFlags)
+    if (dwFlags & ~CRYPT_USERDATA)
     {
         SetLastError(NTE_BAD_FLAGS);
         return FALSE;

dlls/rsaenh/tests/rsaenh.c

@@ -446,7 +446,10 @@ static void test_hashes(void)
     result = CryptCreateHash(hProv, CALG_MD4, 0, 0, &hHash);
     ok(result, "%08x\n", GetLastError());
 
-    result = CryptHashData(hHash, pbData, sizeof(pbData), 0);
+    result = CryptHashData(hHash, pbData, sizeof(pbData), ~0);
+    ok(!result && GetLastError() == NTE_BAD_FLAGS, "%08x\n", GetLastError());
+
+    result = CryptHashData(hHash, pbData, sizeof(pbData), CRYPT_USERDATA);
     ok(result, "%08x\n", GetLastError());
 
     len = sizeof(DWORD);
@@ -470,7 +473,10 @@ static void test_hashes(void)
     result = CryptGetHashParam(hHash, HP_HASHSIZE, (BYTE*)&hashlen, &len, 0);
     ok(result && (hashlen == 16), "%08x, hashlen: %d\n", GetLastError(), hashlen);
 
-    result = CryptHashData(hHash, pbData, sizeof(pbData), 0);
+    result = CryptHashData(hHash, pbData, sizeof(pbData), ~0);
+    ok(!result && GetLastError() == NTE_BAD_FLAGS, "%08x\n", GetLastError());
+
+    result = CryptHashData(hHash, pbData, sizeof(pbData), CRYPT_USERDATA);
     ok(result, "%08x\n", GetLastError());
 
     len = 16;
@@ -519,7 +525,7 @@ static void test_hashes(void)
     result = CryptCreateHash(hProv, CALG_SHA, 0, 0, &hHash);
     ok(result, "%08x\n", GetLastError());
 
-    result = CryptHashData(hHash, pbData, 5, 0);
+    result = CryptHashData(hHash, pbData, 5, CRYPT_USERDATA);
     ok(result, "%08x\n", GetLastError());
 
     if(pCryptDuplicateHash) {

include/wincrypt.h

@@ -3890,6 +3890,8 @@ typedef BOOL (WINAPI *PFN_CMSG_IMPORT_KEY_TRANS)(
 #define EXPORT_PRIVATE_KEYS                   0x00000004
 #define PKCS12_EXPORT_RESERVED_MASK           0xffff0000
 
+#define CRYPT_USERDATA    0x00000001
+
 /* function declarations */
 /* advapi32.dll */
 WINADVAPI BOOL WINAPI CryptAcquireContextA(HCRYPTPROV *, LPCSTR, LPCSTR, DWORD, DWORD);