# HG changeset patch
# Parent  4900740557bc1073f8b3d9e2ad5f8cdad0ecabfc
Make RSA key generation compliant with FIPS 186-4, section B.3.1
(and following).

bsc#917319

diff --git a/lib/freebl/mpi/mpprime.c b/lib/freebl/mpi/mpprime.c
--- a/lib/freebl/mpi/mpprime.c
+++ b/lib/freebl/mpi/mpprime.c
@@ -9,16 +9,18 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "mpi-priv.h"
 #include "mpprime.h"
 #include "mplogic.h"
 #include <stdlib.h>
 #include <string.h>
 
+#include "../fips.h"
+
 #define SMALL_TABLE 0 /* determines size of hard-wired prime table */
 
 #define RANDOM() rand()
 
 #include "primes.c"  /* pull in the prime digit table */
 
 /* 
    Test if any of a given vector of digits divides a.  If not, MP_NO
@@ -433,16 +435,35 @@ mp_err mpp_make_prime(mp_int *start, mp_
     num_tests = 15;
   } else if (nBits >= 150) {
     num_tests = 18;
   } else if (nBits >= 100) {
     num_tests = 27;
   } else
     num_tests = 50;
 
+  /* FIPS 186-4 mandates more M-R tests for probable primes generation - make
+   * sure the minimums are observed (see Appendix C, tables C.1 and C.2).
+   * For DSA this is handled in pqg_ParamGen() through the use of
+   * prime_testcount_p() and prime_testcount_q() respectively.
+   * For RSA this unfortunately seems to be the right place to prevent larger
+   * code changes. On the other hand, it seems to generally speed things up,
+   * since there are measurably less errors while calculating inverse modulo in
+   * rsa_build_from_primes().
+   */
+  if (FIPS_mode()) {
+    if (nBits >= 1536)
+      i = 4;
+    else
+      i = 5;
+    if (i > num_tests)
+	num_tests = i;
+    i = 0;
+  }
+
   if (strong) 
     --nBits;
   MP_CHECKOK( mpl_set_bit(start, nBits - 1, 1) );
   MP_CHECKOK( mpl_set_bit(start,         0, 1) );
   for (i = mpl_significant_bits(start) - 1; i >= nBits; --i) {
     MP_CHECKOK( mpl_set_bit(start, i, 0) );
   }
   /* start sieveing with prime value of 3. */
diff --git a/lib/freebl/rsa.c b/lib/freebl/rsa.c
--- a/lib/freebl/rsa.c
+++ b/lib/freebl/rsa.c
@@ -11,32 +11,35 @@
 
 #include "secerr.h"
 
 #include "prclist.h"
 #include "nssilock.h"
 #include "prinit.h"
 #include "blapi.h"
 #include "mpi.h"
+#include "mpi-priv.h"
 #include "mpprime.h"
 #include "mplogic.h"
 #include "secmpi.h"
 #include "secitem.h"
 #include "blapii.h"
+#include "fips.h"
 
 /*
 ** Number of times to attempt to generate a prime (p or q) from a random
 ** seed (the seed changes for each iteration).
 */
 #define MAX_PRIME_GEN_ATTEMPTS 10
 /*
 ** Number of times to attempt to generate a key.  The primes p and q change
 ** for each attempt.
 */
 #define MAX_KEY_GEN_ATTEMPTS 10
+#define MAX_KEY_GEN_ATTEMPTS_FIPS (5 * MAX_KEY_GEN_ATTEMPTS)
 
 /* Blinding Parameters max cache size  */
 #define RSA_BLINDING_PARAMS_MAX_CACHE_SIZE 20
 
 /* exponent should not be greater than modulus */
 #define BAD_RSA_KEY_SIZE(modLen, expLen) \
     ((expLen) > (modLen) || (modLen) > RSA_MAX_MODULUS_BITS/8 || \
     (expLen) > RSA_MAX_EXPONENT_BITS/8)
@@ -133,28 +136,41 @@ rsa_build_from_primes(const mp_int *p, c
 
     /* at least one exponent must be given */
     PORT_Assert(!(needPublicExponent && needPrivateExponent));
 
     /* 2.  Compute phi = (p-1)*(q-1) */
     CHECK_MPI_OK( mp_sub_d(p, 1, &psub1) );
     CHECK_MPI_OK( mp_sub_d(q, 1, &qsub1) );
     if (needPublicExponent || needPrivateExponent) {
-	CHECK_MPI_OK( mp_mul(&psub1, &qsub1, &phi) );
+	CHECK_MPI_OK( mp_lcm(&psub1, &qsub1, &phi) );
 	/* 3.  Compute d = e**-1 mod(phi) */
 	/*     or      e = d**-1 mod(phi) as necessary */
 	if (needPublicExponent) {
 	    err = mp_invmod(d, &phi, e);
 	} else {
 	    err = mp_invmod(e, &phi, d);
+	    /* FIPS 186-4 (B.3.1.3.a) places additional requirements on the
+	     * private exponent d:
+	     *   2^(n/2) < d < lcm(p-1, q-1) = phi
+	     */
+	    if (FIPS_mode() && (MP_OKAY == err)) {
+		CHECK_MPI_OK( mp_2expt(&tmp, keySizeInBits / 2) );
+		if ((mp_cmp(d, &tmp) <= 0) || (mp_cmp(d, &phi) >= 0)) {
+		    /* new set of p, q is needed for another calculation of d */
+		    err = MP_UNDEF;
+		}
+	    }
 	}
     } else {
 	err = MP_OKAY;
     }
-    /*     Verify that phi(n) and e have no common divisors */
+    /*     Verify that phi(n) and e have no common divisors
+     *     This is also the coprimality constraint from FIPS 186-4 (B.3.1.2.a)
+     */
     if (err != MP_OKAY) {
 	if (err == MP_UNDEF) {
 	    PORT_SetError(SEC_ERROR_NEED_RANDOM);
 	    err = MP_OKAY; /* to keep PORT_SetError from being called again */
 	    rv = SECFailure;
 	}
 	goto cleanup;
     }
@@ -234,29 +250,36 @@ cleanup:
 **	"publicExponent" when not NULL is a pointer to some data that
 **	   represents the public exponent to use. The data is a byte
 **	   encoded integer, in "big endian" order.
 */
 RSAPrivateKey *
 RSA_NewKey(int keySizeInBits, SECItem *publicExponent)
 {
     unsigned int primeLen;
+    unsigned int retrials;
     mp_int p, q, e, d;
+    mp_int u, v;
     int kiter;
     mp_err   err = MP_OKAY;
-    SECStatus rv = SECSuccess;
+    SECStatus rv = SECFailure;
     int prerr = 0;
     RSAPrivateKey *key = NULL;
     PLArenaPool *arena = NULL;
     /* Require key size to be a multiple of 16 bits. */
     if (!publicExponent || keySizeInBits % 16 != 0 ||
 	    BAD_RSA_KEY_SIZE((unsigned int)keySizeInBits/8, publicExponent->len)) {
 	PORT_SetError(SEC_ERROR_INVALID_ARGS);
 	return NULL;
     }
+    /* FIPS 186-4 mandates keys to be either 2048 or 3072 bits long */
+    if (FIPS_mode() && (keySizeInBits != 2048) && (keySizeInBits != 3072)) {
+	PORT_SetError(SEC_ERROR_INVALID_ARGS);
+	return NULL;
+    }
     /* 1. Allocate arena & key */
     arena = PORT_NewArena(NSS_FREEBL_DEFAULT_CHUNKSIZE);
     if (!arena) {
 	PORT_SetError(SEC_ERROR_NO_MEMORY);
 	return NULL;
     }
     key = PORT_ArenaZNew(arena, RSAPrivateKey);
     if (!key) {
@@ -266,56 +289,132 @@ RSA_NewKey(int keySizeInBits, SECItem *p
     }
     key->arena = arena;
     /* length of primes p and q (in bytes) */
     primeLen = keySizeInBits / (2 * PR_BITS_PER_BYTE);
     MP_DIGITS(&p) = 0;
     MP_DIGITS(&q) = 0;
     MP_DIGITS(&e) = 0;
     MP_DIGITS(&d) = 0;
+    MP_DIGITS(&u) = 0;
+    MP_DIGITS(&v) = 0;
     CHECK_MPI_OK( mp_init(&p) );
     CHECK_MPI_OK( mp_init(&q) );
     CHECK_MPI_OK( mp_init(&e) );
     CHECK_MPI_OK( mp_init(&d) );
+    CHECK_MPI_OK( mp_init(&u) );
+    CHECK_MPI_OK( mp_init(&v) );
     /* 2.  Set the version number (PKCS1 v1.5 says it should be zero) */
     SECITEM_AllocItem(arena, &key->version, 1);
     key->version.data[0] = 0;
     /* 3.  Set the public exponent */
     SECITEM_TO_MPINT(*publicExponent, &e);
+
+    /* FIPS 186-4 requires 2^16 < e < 2^256 (B.3.1.1.b) */
+    if (FIPS_mode()) {
+	CHECK_MPI_OK( mp_2expt(&u, 16) );
+	CHECK_MPI_OK( mp_2expt(&v, 256) );
+	if (!((mp_cmp(&u, &e) < 0) && (mp_cmp(&e, &v) < 0 ))) {
+	    err = MP_BADARG;
+	    goto cleanup;
+	}
+    }
+
     kiter = 0;
+    /* allow more retrials in FIPS mode, since there are more chances for a
+     * respin due to additional checks */
+    retrials = FIPS_mode() ? MAX_KEY_GEN_ATTEMPTS_FIPS : MAX_KEY_GEN_ATTEMPTS;
     do {
-	prerr = 0;
-	PORT_SetError(0);
 	CHECK_SEC_OK( generate_prime(&p, primeLen) );
 	CHECK_SEC_OK( generate_prime(&q, primeLen) );
-	/* Assure p > q */
+	/* Assure p >= q */
 	/* NOTE: PKCS #1 does not require p > q, and NSS doesn't use any
 	 * implementation optimization that requires p > q. We can remove
 	 * this code in the future.
 	 */
 	if (mp_cmp(&p, &q) < 0)
 	    mp_exch(&p, &q);
+
+	/* FIPS 186-4 puts additional requirements on the primes (B.3.1.2.a-d)
+	 * (n = key bit length):
+	 * 1) both (p-1) and (q-1) are coprime to e (B.3.1.2.a), i.e.:
+	 *    gcd(p-1,e) = 1, gcd(q-1,e) = 1
+	 *    this is ensured in rsa_build_from_primes(), where
+	 *    phi = lcm(p-1)(q-1) is tested for coprimality to e
+	 * 2) magnitude constraint (B.3.1.2.b and B.3.1.2.c):
+	 *    both p and q are from open the (closed) interval
+	 *    I = < sqrt(2) * 2^(n/2 - 1) , 2^(n/2) - 1) >
+	 * 3) minimum distance (B.3.1.2.d): abs(p-q) > 2 ^ (n/2 - 100)
+	 */
+	if (FIPS_mode()) {
+	    /* set default error which will cause the loop to re-iterate if we
+	     * bail out because of unfulfilled FIPS conditions; the value is
+	     * reset to 0 when the FIPS checks pass
+	     */
+	    prerr = SEC_ERROR_NEED_RANDOM;
+	    /* 2 */
+	    /* in order not to constrain the selection too much,
+	     * expand the inequality:
+	     *   x >= 2^(1/2) * 2^(n/2 - 1)
+	     *         = 2^(1/2 + k) * 2^(n/2 - k - 1)
+	     *         =      y(k)   *     r(k)
+	     * for z(k) >= y(k) it clearly holds:
+	     *   x >= z(k) * r(k)
+	     * one suitable z(k) such that z(k)/y(k) - 1 = o(1) is
+	     * ceil(y(k)) for big-enough k
+	     * ceil(y(30))/y(30) - 1 < 10^-10, so lets use that
+	     * 2^30.5 = 1518500249.98802484622388101120...
+	     * the magic constant is thus z(30) = 1518500250 < 2^31
+	     *
+	     * Additionally, since p >= q is required above, the
+	     * condtitions can be shortened to:
+	     *   sqrt(2) * 2^(n/2 - 1) < 1518500250 * 2^(n/2 - 31) = v <= q
+	     * and:
+	     *   p <= 2^(n/2) - 1 < 2^(n/2) = u
+	     */
+	    CHECK_MPI_OK( mp_2expt(&u, keySizeInBits / 2 - 31) );
+	    CHECK_MPI_OK( mp_mul_d(&u, 1518500250, &v) );
+	    CHECK_MPI_OK( mp_2expt(&u, keySizeInBits / 2) );
+	    if ((mp_cmp(&q, &v) < 0) || (mp_cmp(&p, &u) >= 0)) {
+		kiter++;
+		continue;
+	    }
+	    /* 3 */
+	    CHECK_MPI_OK( mp_sub(&p, &q, &u) );
+	    CHECK_MPI_OK( mp_abs(&u, &u) );
+	    CHECK_MPI_OK( mp_2expt(&v, keySizeInBits / 2 - 100) );
+	    if (mp_cmp(&u, &v) < 0) {
+		kiter++;
+		continue;
+	    }
+	}
+	
+	prerr = 0;
+	PORT_SetError(0);
+
 	/* Attempt to use these primes to generate a key */
 	rv = rsa_build_from_primes(&p, &q, 
 			&e, PR_FALSE,  /* needPublicExponent=false */
 			&d, PR_TRUE,   /* needPrivateExponent=true */
 			key, keySizeInBits);
 	if (rv == SECSuccess)
 	    break; /* generated two good primes */
 	prerr = PORT_GetError();
 	kiter++;
 	/* loop until have primes */
-    } while (prerr == SEC_ERROR_NEED_RANDOM && kiter < MAX_KEY_GEN_ATTEMPTS);
+    } while (prerr == SEC_ERROR_NEED_RANDOM && kiter < retrials);
     if (prerr)
 	goto cleanup;
 cleanup:
     mp_clear(&p);
     mp_clear(&q);
     mp_clear(&e);
     mp_clear(&d);
+    mp_clear(&u);
+    mp_clear(&v);
     if (err) {
 	MP_TO_SEC_ERROR(err);
 	rv = SECFailure;
     }
     if (rv && arena) {
 	PORT_FreeArena(arena, PR_TRUE);
 	key = NULL;
     }
