commit 009141efaa9f051caee9ccb4c9c57e393e6a36b2
Author: Hans Petter Jansson <hpj@cl.no>
Date:   Sun Oct 15 19:45:28 2017 +0200

    Patch 16: nss-fips-rsa-keygen-strictness.patch

diff --git a/nss/lib/freebl/mpi/mpprime.c b/nss/lib/freebl/mpi/mpprime.c
index 5828719..90c5521 100644
--- a/nss/lib/freebl/mpi/mpprime.c
+++ b/nss/lib/freebl/mpi/mpprime.c
@@ -14,6 +14,8 @@
 #include <stdlib.h>
 #include <string.h>
 
+#include "../fips.h"
+
 #define SMALL_TABLE 0 /* determines size of hard-wired prime table */
 
 #define RANDOM() rand()
@@ -452,6 +454,25 @@ mpp_make_prime(mp_int *start, mp_size nBits, mp_size strong,
     } else
         num_tests = 50;
 
+    /* FIPS 186-4 mandates more M-R tests for probable primes generation - make
+     * sure the minimums are observed (see Appendix C, tables C.1 and C.2).
+     * For DSA this is handled in pqg_ParamGen() through the use of
+     * prime_testcount_p() and prime_testcount_q() respectively.
+     * For RSA this unfortunately seems to be the right place to prevent larger
+     * code changes. On the other hand, it seems to generally speed things up,
+     * since there are measurably less errors while calculating inverse modulo in
+     * rsa_build_from_primes().
+     */
+    if (FIPS_mode()) {
+        if (nBits >= 1536)
+            i = 4;
+        else
+            i = 5;
+        if (i > num_tests)
+            num_tests = i;
+        i = 0;
+    }
+
     if (strong)
         --nBits;
     MP_CHECKOK(mpl_set_bit(start, nBits - 1, 1));
diff --git a/nss/lib/freebl/rsa.c b/nss/lib/freebl/rsa.c
index 3617afa..8918f2b 100644
--- a/nss/lib/freebl/rsa.c
+++ b/nss/lib/freebl/rsa.c
@@ -16,11 +16,13 @@
 #include "prinit.h"
 #include "blapi.h"
 #include "mpi.h"
+#include "mpi-priv.h"
 #include "mpprime.h"
 #include "mplogic.h"
 #include "secmpi.h"
 #include "secitem.h"
 #include "blapii.h"
+#include "fips.h"
 
 /*
 ** Number of times to attempt to generate a prime (p or q) from a random
@@ -143,11 +145,24 @@ rsa_build_from_primes(const mp_int *p, const mp_int *q,
             err = mp_invmod(d, &phi, e);
         } else {
             err = mp_invmod(e, &phi, d);
-        }
+            /* FIPS 186-4 (B.3.1.3.a) places additional requirements on the
+             * private exponent d:
+             *   2^(n/2) < d < lcm(p-1, q-1) = phi
+             */
+            if (FIPS_mode() && (MP_OKAY == err)) {
+                CHECK_MPI_OK( mp_2expt(&tmp, keySizeInBits / 2) );
+                if ((mp_cmp(d, &tmp) <= 0) || (mp_cmp(d, &phi) >= 0)) {
+                    /* new set of p, q is needed for another calculation of d */
+                    err = MP_UNDEF;
+                }
+            }
+       }
     } else {
         err = MP_OKAY;
     }
-    /*     Verify that phi(n) and e have no common divisors */
+    /*     Verify that phi(n) and e have no common divisors
+     *     This is also the coprimality constraint from FIPS 186-4 (B.3.1.2.a)
+     */
     if (err != MP_OKAY) {
         if (err == MP_UNDEF) {
             PORT_SetError(SEC_ERROR_NEED_RANDOM);
@@ -277,10 +292,11 @@ RSA_NewKey(int keySizeInBits, SECItem *publicExponent)
 {
     unsigned int primeLen;
     mp_int p, q, e, d;
+    mp_int u, v;
     int kiter;
     int max_attempts;
     mp_err err = MP_OKAY;
-    SECStatus rv = SECSuccess;
+    SECStatus rv = SECFailure;
     int prerr = 0;
     RSAPrivateKey *key = NULL;
     PLArenaPool *arena = NULL;
@@ -290,6 +306,14 @@ RSA_NewKey(int keySizeInBits, SECItem *publicExponent)
         PORT_SetError(SEC_ERROR_INVALID_ARGS);
         return NULL;
     }
+    /* FIPS 186-4 mandates keys to be either 2048, 3072 or 4096 bits long.
+     * We also allow a key length of 4096, since this is needed in order to
+     * pass the CAVS RSA SigGen test. */
+    if (FIPS_mode() && (keySizeInBits != 2048) && (keySizeInBits != 3072)
+        && (keySizeInBits != 4096)) {
+        PORT_SetError(SEC_ERROR_INVALID_ARGS);
+        return NULL;
+    }
     /* 1. Allocate arena & key */
     arena = PORT_NewArena(NSS_FREEBL_DEFAULT_CHUNKSIZE);
     if (!arena) {
@@ -309,15 +333,30 @@ RSA_NewKey(int keySizeInBits, SECItem *publicExponent)
     MP_DIGITS(&q) = 0;
     MP_DIGITS(&e) = 0;
     MP_DIGITS(&d) = 0;
+    MP_DIGITS(&u) = 0;
+    MP_DIGITS(&v) = 0;
     CHECK_MPI_OK(mp_init(&p));
     CHECK_MPI_OK(mp_init(&q));
     CHECK_MPI_OK(mp_init(&e));
     CHECK_MPI_OK(mp_init(&d));
-    /* 2.  Set the version number (PKCS1 v1.5 says it should be zero) */
+    CHECK_MPI_OK(mp_init(&u));
+    CHECK_MPI_OK(mp_init(&v));
+   /* 2.  Set the version number (PKCS1 v1.5 says it should be zero) */
     SECITEM_AllocItem(arena, &key->version, 1);
     key->version.data[0] = 0;
     /* 3.  Set the public exponent */
     SECITEM_TO_MPINT(*publicExponent, &e);
+
+    /* FIPS 186-4 requires 2^16 < e < 2^256 (B.3.1.1.b) */
+    if (FIPS_mode()) {
+        CHECK_MPI_OK( mp_2expt(&u, 16) );
+        CHECK_MPI_OK( mp_2expt(&v, 256) );
+        if (!((mp_cmp(&u, &e) < 0) && (mp_cmp(&e, &v) < 0 ))) {
+            err = MP_BADARG;
+            goto cleanup;
+        }
+    }
+
     kiter = 0;
     max_attempts = 5 * (keySizeInBits / 2); /* FIPS 186-4 B.3.3 steps 4.7 and 5.8 */
     do {
@@ -325,13 +364,64 @@ RSA_NewKey(int keySizeInBits, SECItem *publicExponent)
         PORT_SetError(0);
         CHECK_SEC_OK(generate_prime(&p, primeLen));
         CHECK_SEC_OK(generate_prime(&q, primeLen));
-        /* Assure p > q */
+        /* Assure p >= q */
         /* NOTE: PKCS #1 does not require p > q, and NSS doesn't use any
          * implementation optimization that requires p > q. We can remove
          * this code in the future.
          */
         if (mp_cmp(&p, &q) < 0)
             mp_exch(&p, &q);
+
+        /* FIPS 186-4 puts additional requirements on the primes (B.3.1.2.a-d)
+         * (n = key bit length):
+         * 1) both (p-1) and (q-1) are coprime to e (B.3.1.2.a), i.e.:
+         *    gcd(p-1,e) = 1, gcd(q-1,e) = 1
+         *    this is ensured in rsa_build_from_primes(), where
+         *    phi = lcm(p-1)(q-1) is tested for coprimality to e
+         * 2) magnitude constraint (B.3.1.2.b and B.3.1.2.c):
+         *    both p and q are from open the interval
+         *    I = ( sqrt(2) * 2^(n/2 - 1) ,  2^(n/2 - 1) )
+         * 3) minimum distance (B.3.1.2.d): abs(p-q) > 2 ^ (n/2 - 100)
+         */
+        if (FIPS_mode()) {
+            /* 2 */
+            /* in order not to constrain the selection too much,
+             * expand the inequality:
+             *   x > 2^(1/2) * 2^(n/2 - 1)
+             *         = 2^(1/2 + k) * 2^(n/2 - k - 1)
+             *         =      y(k)   *     r(k)
+             * for z(k) >= y(k) it clearly holds:
+             *   x > z(k) * r(k)
+             * one suitable z(k) such that z(k)/y(k) - 1 = o(1) is
+             * ceil(y(k)) for big-enough k
+             * ceil(y(30))/y(30) - 1 < 10^-10, so lets use that
+             * 2^30.5 = 1518500249.98802484622388101120...
+             * the magic constant is thus z(30) = 1518500250 < 2^31
+             *
+             * Additionally, since p >= q is required above, the
+             * condtitions can be shortened to:
+             *   1518500250 * 2^(n/2 - 31) = v < q 
+             *                               p < u = 2^(n/2 - 1)
+             */
+            CHECK_MPI_OK( mp_2expt(&u, keySizeInBits / 2 - 31) );
+            CHECK_MPI_OK( mp_mul_d(&u, 1518500250, &v) );
+            CHECK_MPI_OK( mp_2expt(&u, keySizeInBits / 2) );
+            if ((mp_cmp(&q, &v) <= 0) || (mp_cmp(&p, &u) >= 0)) {
+                prerr = SEC_ERROR_NEED_RANDOM; /* retry with different values */
+                kiter++;
+                continue;
+            }
+            /* 3 */
+            CHECK_MPI_OK( mp_sub(&p, &q, &u) );
+            CHECK_MPI_OK( mp_abs(&u, &u) );
+            CHECK_MPI_OK( mp_2expt(&v, keySizeInBits / 2 - 100) );
+            if (mp_cmp(&u, &v) < 0) {
+                prerr = SEC_ERROR_NEED_RANDOM; /* retry with different values */
+                kiter++;
+                continue;
+            }
+        }
+
         /* Attempt to use these primes to generate a key */
         rv = rsa_build_from_primes(&p, &q,
                                    &e, PR_FALSE, /* needPublicExponent=false */
@@ -355,6 +445,8 @@ cleanup:
     mp_clear(&q);
     mp_clear(&e);
     mp_clear(&d);
+    mp_clear(&u);
+    mp_clear(&v);
     if (err) {
         MP_TO_SEC_ERROR(err);
         rv = SECFailure;
