rsaenh: Implement RSA OAEP.

dlls/rsaenh/tests/rsaenh.c

@@ -2282,6 +2282,7 @@ static void test_rsa_encrypt(void)
     BYTE abData[2048] = "Wine rocks!";
     BOOL result;
     DWORD dwVal, dwLen;
+    DWORD err;
 
     /* It is allowed to use the key exchange key for encryption/decryption */
     result = CryptGetUserKey(hProv, AT_KEYEXCHANGE, &hRSAKey);
@@ -2297,6 +2298,7 @@ static void test_rsa_encrypt(void)
     }
     ok(result, "CryptEncrypt failed: %08x\n", GetLastError());
     ok(dwLen == 128, "Unexpected length %d\n", dwLen);
+    /* PKCS1 V1.5 */
     dwLen = 12;
     result = CryptEncrypt(hRSAKey, 0, TRUE, 0, abData, &dwLen, (DWORD)sizeof(abData));
     ok (result, "%08x\n", GetLastError());
@@ -2305,6 +2307,51 @@ static void test_rsa_encrypt(void)
     result = CryptDecrypt(hRSAKey, 0, TRUE, 0, abData, &dwLen);
     ok (result && dwLen == 12 && !memcmp(abData, "Wine rocks!", 12), "%08x\n", GetLastError());
 
+    /* OAEP, RFC 8017 PKCS #1 V2.2 */
+    /* Test minimal buffer length requirement */
+    dwLen = 1;
+    SetLastError(0xdeadbeef);
+    result = CryptEncrypt(hRSAKey, 0, TRUE, CRYPT_OAEP, abData, &dwLen, 20 * 2 + 2);
+    err = GetLastError();
+    ok(!result && err == ERROR_MORE_DATA, "%08x\n", err);
+
+    /* Test data length limit */
+    dwLen = sizeof(abData) - (20 * 2 + 2) + 1;
+    result = CryptEncrypt(hRSAKey, 0, TRUE, CRYPT_OAEP, abData, &dwLen, (DWORD)sizeof(abData));
+    err = GetLastError();
+    ok(!result && err == NTE_BAD_LEN, "%08x\n", err);
+
+    /* Test malformed data */
+    dwLen = 12;
+    SetLastError(0xdeadbeef);
+    memcpy(abData, "Wine rocks!", dwLen);
+    result = CryptDecrypt(hRSAKey, 0, TRUE, CRYPT_OAEP, abData, &dwLen);
+    err = GetLastError();
+    /* NTE_DOUBLE_ENCRYPT on xp or 2003 */
+    ok(!result && (err == NTE_BAD_DATA || broken(err == NTE_DOUBLE_ENCRYPT)), "%08x\n", err);
+
+    /* Test decrypt with insufficient buffer */
+    dwLen = 12;
+    SetLastError(0xdeadbeef);
+    memcpy(abData, "Wine rocks!", 12);
+    result = CryptEncrypt(hRSAKey, 0, TRUE, CRYPT_OAEP, abData, &dwLen, (DWORD)sizeof(abData));
+    ok(result, "%08x\n", GetLastError());
+    dwLen = 11;
+    SetLastError(0xdeadbeef);
+    result = CryptDecrypt(hRSAKey, 0, TRUE, CRYPT_OAEP, abData, &dwLen);
+    err = GetLastError();
+    /* broken on xp or 2003 */
+    ok((!result && dwLen == 11 && err == NTE_BAD_DATA) || broken(result == TRUE && dwLen == 12 && err == ERROR_NO_TOKEN),
+       "%08x %d %08x\n", result, dwLen, err);
+
+    /* Test normal encryption and decryption */
+    dwLen = 12;
+    memcpy(abData, "Wine rocks!", dwLen);
+    result = CryptEncrypt(hRSAKey, 0, TRUE, CRYPT_OAEP, abData, &dwLen, (DWORD)sizeof(abData));
+    ok(result, "%08x\n", GetLastError());
+    result = CryptDecrypt(hRSAKey, 0, TRUE, CRYPT_OAEP, abData, &dwLen);
+    ok(result && dwLen == 12 && !memcmp(abData, "Wine rocks!", 12), "%08x\n", GetLastError());
+
     dwVal = 0xdeadbeef;
     dwLen = sizeof(DWORD);
     result = CryptGetKeyParam(hRSAKey, KP_PERMISSIONS, (BYTE*)&dwVal, &dwLen, 0);