rsaenh: Handle the failure to set Base provider effective key length properly.

dlls/rsaenh/rsaenh.c

@@ -3496,7 +3496,8 @@ BOOL WINAPI RSAENH_CPSetKeyParam(HCRYPTPROV hProv, HCRYPTKEY hKey, DWORD dwParam
             switch (pCryptKey->aiAlgid) {
                 case CALG_RC2:
                 {
-                    DWORD keylen;
+                    DWORD keylen, deflen;
+                    BOOL ret = TRUE;
                     KEYCONTAINER *pKeyContainer = get_key_container(pCryptKey->hProv);
 
                     if (!pbData)
@@ -3505,18 +3506,28 @@ BOOL WINAPI RSAENH_CPSetKeyParam(HCRYPTPROV hProv, HCRYPTKEY hKey, DWORD dwParam
                         return FALSE;
                     }
                     keylen = *(DWORD *)pbData;
-                    if (!keylen || keylen > 1024 || (keylen != 40 &&
-                        pKeyContainer->dwPersonality == RSAENH_PERSONALITY_BASE))
+                    if (!keylen || keylen > 1024)
                     {
                         SetLastError(NTE_BAD_DATA);
                         return FALSE;
                     }
-                    else
+
+                    /*
+                     * The Base provider will force the key length to default
+                     * and set an error state if a key length different from
+                     * the default is tried.
+                     */
+                    deflen = aProvEnumAlgsEx[pKeyContainer->dwPersonality]->dwDefaultLen;
+                    if (pKeyContainer->dwPersonality == RSAENH_PERSONALITY_BASE
+                        && keylen != deflen)
                     {
-                        pCryptKey->dwEffectiveKeyLen = keylen;
-                        setup_key(pCryptKey);
+                        keylen = deflen;
+                        SetLastError(NTE_BAD_DATA);
+                        ret = FALSE;
                     }
-                    break;
+                    pCryptKey->dwEffectiveKeyLen = keylen;
+                    setup_key(pCryptKey);
+                    return ret;
                 }
                 default:
                     SetLastError(NTE_BAD_TYPE);

dlls/rsaenh/tests/rsaenh.c

@@ -1534,11 +1534,20 @@ static void test_rc2(void)
         SetLastError(0xdeadbeef);
         result = CryptSetKeyParam(hKey, KP_EFFECTIVE_KEYLEN, (LPBYTE)&dwKeyLen, 0);
         if (!BASE_PROV)
+        {
+            dwKeyLen = 12345;
             ok(result, "expected success, got error 0x%08X\n", GetLastError());
+            result = CryptGetKeyParam(hKey, KP_EFFECTIVE_KEYLEN, (BYTE *)&dwKeyLen, &dwLen, 0);
+            ok(result, "%08x", GetLastError());
+            ok(dwKeyLen == 128, "Expected 128, got %d\n", dwKeyLen);
+        }
         else
         {
             ok(!result, "expected error\n");
             ok(GetLastError() == NTE_BAD_DATA, "Expected 0x80009005, got 0x%08X\n", GetLastError());
+            result = CryptGetKeyParam(hKey, KP_EFFECTIVE_KEYLEN, (BYTE *)&dwKeyLen, &dwLen, 0);
+            ok(result, "%08x", GetLastError());
+            ok(dwKeyLen == 40, "Expected 40, got %d\n", dwKeyLen);
         }
 
         dwLen = sizeof(dwKeyLen);
@@ -1547,18 +1556,8 @@ static void test_rc2(void)
         ok(dwKeyLen == 56, "%d (%08x)\n", dwKeyLen, GetLastError());
         result = CryptGetKeyParam(hKey, KP_EFFECTIVE_KEYLEN, (BYTE *)&dwKeyLen, &dwLen, 0);
         ok(result, "%08x", GetLastError());
-        /* Remove IF when fixed */
-        if(BASE_PROV)
-        {
-        todo_wine
-        ok((!BASE_PROV && dwKeyLen == 128) || (BASE_PROV && dwKeyLen == 40),
-           "%d (%08x)\n", dwKeyLen, GetLastError());
-        }
-        else
-        {
         ok((!BASE_PROV && dwKeyLen == 128) || (BASE_PROV && dwKeyLen == 40),
            "%d (%08x)\n", dwKeyLen, GetLastError());
-        }
 
         result = CryptDestroyHash(hHash);
         ok(result, "%08x\n", GetLastError());
@@ -1566,19 +1565,8 @@ static void test_rc2(void)
         dwDataLen = 13;
         result = CryptEncrypt(hKey, 0, TRUE, 0, pbData, &dwDataLen, 24);
         ok(result, "%08x\n", GetLastError());
-
-        /* Remove IF when fixed */
-        if(BASE_PROV)
-        {
-        todo_wine
         ok(!memcmp(pbData, !BASE_PROV ? rc2_128_encrypted : rc2_40def_encrypted,
            sizeof(rc2_128_encrypted)), "RC2 encryption failed!\n");
-        }
-        else
-        {
-        ok(!memcmp(pbData, !BASE_PROV ? rc2_128_encrypted : rc2_40def_encrypted,
-           sizeof(rc2_128_encrypted)), "RC2 encryption failed!\n");
-        }
 
         /* Oddly enough this succeeds, though it should have no effect */
         dwKeyLen = 40;