rsaenh: Support setting the effective key length of RC2 keys.

dlls/rsaenh/implglue.c

@@ -166,8 +166,8 @@ BOOL free_key_impl(ALG_ID aiAlgid, KEY_CONTEXT *pKeyContext)
     return TRUE;
 }
 
-BOOL setup_key_impl(ALG_ID aiAlgid, KEY_CONTEXT *pKeyContext, DWORD dwKeyLen, DWORD dwSaltLen, 
-                    BYTE *abKeyValue) 
+BOOL setup_key_impl(ALG_ID aiAlgid, KEY_CONTEXT *pKeyContext, DWORD dwKeyLen,
+                    DWORD dwEffectiveKeyLen, DWORD dwSaltLen, BYTE *abKeyValue)
 {
     switch (aiAlgid) 
     {
@@ -178,7 +178,8 @@ BOOL setup_key_impl(ALG_ID aiAlgid, KEY_CONTEXT *pKeyContext, DWORD dwKeyLen, DW
             break;
         
         case CALG_RC2:
-            rc2_setup(abKeyValue, dwKeyLen + dwSaltLen, dwKeyLen << 3, 0, &pKeyContext->rc2);
+            rc2_setup(abKeyValue, dwKeyLen + dwSaltLen, dwEffectiveKeyLen ?
+                      dwEffectiveKeyLen : dwKeyLen << 3, 0, &pKeyContext->rc2);
             break;
         
         case CALG_3DES:

dlls/rsaenh/implglue.h

@@ -76,8 +76,8 @@ BOOL duplicate_hash_impl(ALG_ID aiAlgid, CONST HASH_CONTEXT *pSrcHashContext,
 
 BOOL new_key_impl(ALG_ID aiAlgid, KEY_CONTEXT *pKeyContext, DWORD dwKeyLen);
 BOOL free_key_impl(ALG_ID aiAlgid, KEY_CONTEXT *pKeyContext);
-BOOL setup_key_impl(ALG_ID aiAlgid, KEY_CONTEXT *pKeyContext, DWORD dwKeyLen, DWORD dwSaltLen, 
-                    BYTE *abKeyValue);
+BOOL setup_key_impl(ALG_ID aiAlgid, KEY_CONTEXT *pKeyContext, DWORD dwKeyLen,
+                    DWORD dwEffectiveKeyLen, DWORD dwSaltLen, BYTE *abKeyValue);
 BOOL duplicate_key_impl(ALG_ID aiAlgid, CONST KEY_CONTEXT *pSrcKeyContext, 
                         KEY_CONTEXT *pDestKeyContext);
 

dlls/rsaenh/rsaenh.c

@@ -93,6 +93,7 @@ typedef struct tagCRYPTKEY
     DWORD       dwModeBits;
     DWORD       dwPermissions;
     DWORD       dwKeyLen;
+    DWORD       dwEffectiveKeyLen;
     DWORD       dwSaltLen;
     DWORD       dwBlockLen;
     DWORD       dwState;
@@ -718,7 +719,8 @@ static inline void setup_key(CRYPTKEY *pCryptKey) {
     pCryptKey->dwState = RSAENH_KEYSTATE_IDLE;
     memcpy(pCryptKey->abChainVector, pCryptKey->abInitVector, sizeof(pCryptKey->abChainVector));
     setup_key_impl(pCryptKey->aiAlgid, &pCryptKey->context, pCryptKey->dwKeyLen, 
-                   pCryptKey->dwSaltLen, pCryptKey->abKeyValue);
+                   pCryptKey->dwEffectiveKeyLen, pCryptKey->dwSaltLen,
+                   pCryptKey->abKeyValue);
 }
 
 /******************************************************************************
@@ -813,6 +815,7 @@ static HCRYPTKEY new_key(HCRYPTPROV hProv, ALG_ID aiAlgid, DWORD dwFlags, CRYPTK
         pCryptKey->dwPermissions = CRYPT_ENCRYPT | CRYPT_DECRYPT | CRYPT_READ | CRYPT_WRITE | 
                                    CRYPT_MAC;
         pCryptKey->dwKeyLen = dwKeyLen >> 3;
+        pCryptKey->dwEffectiveKeyLen = 0;
         if ((dwFlags & CRYPT_CREATE_SALT) || (dwKeyLen == 40 && !(dwFlags & CRYPT_NO_SALT))) 
             pCryptKey->dwSaltLen = 16 /*FIXME*/ - pCryptKey->dwKeyLen;
         else
@@ -2710,6 +2713,31 @@ BOOL WINAPI RSAENH_CPSetKeyParam(HCRYPTPROV hProv, HCRYPTKEY hKey, DWORD dwParam
             setup_key(pCryptKey);
             return TRUE;
 
+        case KP_EFFECTIVE_KEYLEN:
+            switch (pCryptKey->aiAlgid) {
+                case CALG_RC2:
+                    if (!pbData)
+                    {
+                        SetLastError(ERROR_INVALID_PARAMETER);
+                        return FALSE;
+                    }
+                    else if (!*(DWORD *)pbData || *(DWORD *)pbData > 1024)
+                    {
+                        SetLastError(NTE_BAD_DATA);
+                        return FALSE;
+                    }
+                    else
+                    {
+                        pCryptKey->dwEffectiveKeyLen = *(DWORD *)pbData;
+                        setup_key(pCryptKey);
+                    }
+                    break;
+                default:
+                    SetLastError(NTE_BAD_TYPE);
+                    return FALSE;
+            }
+            return TRUE;
+
         case KP_SCHANNEL_ALG:
             switch (((PSCHANNEL_ALG)pbData)->dwUse) {
                 case SCHANNEL_ENC_KEY:
@@ -2807,6 +2835,13 @@ BOOL WINAPI RSAENH_CPGetKeyParam(HCRYPTPROV hProv, HCRYPTKEY hKey, DWORD dwParam
             dwBitLen = pCryptKey->dwKeyLen << 3;
             return copy_param(pbData, pdwDataLen, (CONST BYTE*)&dwBitLen, sizeof(DWORD));
         
+        case KP_EFFECTIVE_KEYLEN:
+            if (pCryptKey->dwEffectiveKeyLen)
+                dwBitLen = pCryptKey->dwEffectiveKeyLen;
+            else
+                dwBitLen = pCryptKey->dwKeyLen << 3;
+            return copy_param(pbData, pdwDataLen, (CONST BYTE*)&dwBitLen, sizeof(DWORD));
+
         case KP_BLOCKLEN:
             dwBitLen = pCryptKey->dwBlockLen << 3;
             return copy_param(pbData, pdwDataLen, (CONST BYTE*)&dwBitLen, sizeof(DWORD));

dlls/rsaenh/tests/rsaenh.c

@@ -498,6 +498,9 @@ static void test_rc2(void)
     static const BYTE rc2encrypted[16] = { 
         0x02, 0x34, 0x7d, 0xf6, 0x1d, 0xc5, 0x9b, 0x8b, 
         0x2e, 0x0d, 0x63, 0x80, 0x72, 0xc1, 0xc2, 0xb1 };
+    static const BYTE rc2_128_encrypted[] = {
+        0x82,0x81,0xf7,0xff,0xdd,0xd7,0x88,0x8c,0x2a,0x2a,0xc0,0xce,0x4c,0x89,
+        0xb6,0x66 };
     HCRYPTHASH hHash;
     HCRYPTKEY hKey;
     BOOL result;
@@ -579,6 +582,69 @@ static void test_rc2(void)
         result = CryptDestroyKey(hKey);
         ok(result, "%08x\n", GetLastError());
     }
+
+    /* Again, but test setting the effective key len */
+    for (i=0; i<2000; i++) pbData[i] = (unsigned char)i;
+
+    result = CryptCreateHash(hProv, CALG_MD2, 0, 0, &hHash);
+    if (!result) {
+        ok(GetLastError()==NTE_BAD_ALGID, "%08x\n", GetLastError());
+    } else {
+        result = CryptHashData(hHash, (BYTE*)pbData, sizeof(pbData), 0);
+        ok(result, "%08x\n", GetLastError());
+
+        dwLen = 16;
+        result = CryptGetHashParam(hHash, HP_HASHVAL, pbHashValue, &dwLen, 0);
+        ok(result, "%08x\n", GetLastError());
+
+        result = CryptDeriveKey(hProv, CALG_RC2, hHash, 56 << 16, &hKey);
+        ok(result, "%08x\n", GetLastError());
+
+        SetLastError(0xdeadbeef);
+        result = CryptSetKeyParam(hKey, KP_EFFECTIVE_KEYLEN, NULL, 0);
+        ok(!result && GetLastError()==ERROR_INVALID_PARAMETER, "%08x\n", GetLastError());
+        dwKeyLen = 0;
+        SetLastError(0xdeadbeef);
+        result = CryptSetKeyParam(hKey, KP_EFFECTIVE_KEYLEN, (LPBYTE)&dwKeyLen, 0);
+        ok(!result && GetLastError()==NTE_BAD_DATA, "%08x\n", GetLastError());
+        dwKeyLen = 1025;
+        SetLastError(0xdeadbeef);
+        result = CryptSetKeyParam(hKey, KP_EFFECTIVE_KEYLEN, (LPBYTE)&dwKeyLen, 0);
+
+        dwLen = sizeof(dwKeyLen);
+        CryptGetKeyParam(hKey, KP_KEYLEN, (BYTE *)&dwKeyLen, &dwLen, 0);
+        ok(dwKeyLen == 56, "%d (%08x)\n", dwKeyLen, GetLastError());
+        CryptGetKeyParam(hKey, KP_EFFECTIVE_KEYLEN, (BYTE *)&dwKeyLen, &dwLen, 0);
+        ok(dwKeyLen == 56, "%d (%08x)\n", dwKeyLen, GetLastError());
+
+        dwKeyLen = 128;
+        result = CryptSetKeyParam(hKey, KP_EFFECTIVE_KEYLEN, (LPBYTE)&dwKeyLen, 0);
+        ok(result, "%d\n", GetLastError());
+
+        dwLen = sizeof(dwKeyLen);
+        CryptGetKeyParam(hKey, KP_KEYLEN, (BYTE *)&dwKeyLen, &dwLen, 0);
+        ok(dwKeyLen == 56, "%d (%08x)\n", dwKeyLen, GetLastError());
+        CryptGetKeyParam(hKey, KP_EFFECTIVE_KEYLEN, (BYTE *)&dwKeyLen, &dwLen, 0);
+        ok(dwKeyLen == 128, "%d (%08x)\n", dwKeyLen, GetLastError());
+
+        result = CryptDestroyHash(hHash);
+        ok(result, "%08x\n", GetLastError());
+
+        dwDataLen = 13;
+        result = CryptEncrypt(hKey, (HCRYPTHASH)NULL, TRUE, 0, pbData, &dwDataLen, 24);
+        ok(result, "%08x\n", GetLastError());
+
+        ok(!memcmp(pbData, rc2_128_encrypted, sizeof(rc2_128_encrypted)),
+                "RC2 encryption failed!\n");
+
+        /* Oddly enough this succeeds, though it should have no effect */
+        dwKeyLen = 40;
+        result = CryptSetKeyParam(hKey, KP_EFFECTIVE_KEYLEN, (LPBYTE)&dwKeyLen, 0);
+        ok(result, "%d\n", GetLastError());
+
+        result = CryptDestroyKey(hKey);
+        ok(result, "%08x\n", GetLastError());
+    }
 }
 
 static void test_rc4(void)