Small bugfixes backported from the LibTomCrypt v1.0rc1 release.

dlls/rsaenh/mpi.c

@@ -3257,7 +3257,7 @@ int mp_prime_random_ex(mp_int *a, int t, int size, int flags, ltm_prime_callback
    }
 
    /* calc the byte size */
-   bsize = (size>>3)+(size&7?1:0);
+   bsize = (size>>3)+((size&7)?1:0);
 
    /* we need a buffer of bsize bytes */
    tmp = malloc(bsize);
@@ -3266,11 +3266,11 @@ int mp_prime_random_ex(mp_int *a, int t, int size, int flags, ltm_prime_callback
    }
 
    /* calc the maskAND value for the MSbyte*/
-   maskAND = 0xFF >> (8 - (size & 7));
+   maskAND = ((size&7) == 0) ? 0xFF : (0xFF >> (8 - (size & 7))); 
 
    /* calc the maskOR_msb */
    maskOR_msb        = 0;
-   maskOR_msb_offset = (size - 2) >> 3;
+   maskOR_msb_offset = ((size & 7) == 1) ? 1 : 0;
    if (flags & LTM_PRIME_2MSB_ON) {
       maskOR_msb     |= 1 << ((size - 2) & 7);
    } else if (flags & LTM_PRIME_2MSB_OFF) {

dlls/rsaenh/rsa.c

@@ -73,7 +73,9 @@ int rand_prime(mp_int *N, long len)
       type = LTM_PRIME_BBS;
       len = -len;
    } else {
-      type = 0;
+      /* This seems to be what MS CSP's do: */
+      type = LTM_PRIME_2MSB_ON;
+      /* Original LibTomCrypt: type = 0; */
    }
 
    /* New prime generation makes the code even more cryptoish-insane.  Do you know what this means!!!