OSSL_FN: Wrap BIGNUM 'add' and 'sub' functions around OSSL_FN functions

This involves the following functions:

- 'BN_add'
- 'BN_sub'
- 'BN_uadd'
- 'BN_usub'

Care is taken to use the previous (now legacy) code if any of the argument
BIGNUMs isn't backed by an OSSL_FN (i.e. if its 'data' field is NULL).

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/29028)

crypto/bn/bn_add.c

@@ -10,6 +10,21 @@
 #include "internal/cryptlib.h"
 #include "bn_local.h"
 
+static size_t calculate_max_limbs(const BIGNUM *a, const BIGNUM *b)
+{
+    OSSL_FN *af = a->data;
+    OSSL_FN *bf = b->data;
+
+    return (af->dsize > bf->dsize) ? af->dsize : bf->dsize;
+}
+
+static bool is_highest_bit_set(const BIGNUM *a)
+{
+    OSSL_FN *af = a->data;
+
+    return (af->d[af->dsize - 1] & OSSL_FN_HIGH_BIT_MASK) != 0;
+}
+
 /* signed add of b to a. */
 int BN_add(BIGNUM *r, const BIGNUM *a, const BIGNUM *b)
 {
@@ -72,8 +87,9 @@ int BN_sub(BIGNUM *r, const BIGNUM *a, const BIGNUM *b)
     return ret;
 }
 
-/* unsigned add of b to a, r can be equal to a or b. */
-int BN_uadd(BIGNUM *r, const BIGNUM *a, const BIGNUM *b)
+/* TODO(FIXNUM): TO BE REMOVED */
+/* pure BIGNUM unsigned add of b to a, r can be equal to a or b. */
+static int bn_uadd_legacy(BIGNUM *r, const BIGNUM *a, const BIGNUM *b)
 {
     int max, min, dif;
     const BN_ULONG *ap, *bp;
@@ -121,8 +137,35 @@ int BN_uadd(BIGNUM *r, const BIGNUM *a, const BIGNUM *b)
     return 1;
 }
 
-/* unsigned subtraction of b from a, a must be larger than b. */
-int BN_usub(BIGNUM *r, const BIGNUM *a, const BIGNUM *b)
+/* unsigned add of b to a, r can be equal to a or b. */
+int BN_uadd(BIGNUM *r, const BIGNUM *a, const BIGNUM *b)
+{
+    /* TODO(FIXNUM): TO BE REMOVED */
+    if (r->data == NULL || a->data == NULL || b->data == NULL)
+        return bn_uadd_legacy(r, a, b);
+
+    bn_check_top(a);
+    bn_check_top(b);
+
+    size_t max = calculate_max_limbs(a, b);
+
+    /*
+     * If either operands have the highest bit set the result may become
+     * one limb larger.
+     */
+    if (is_highest_bit_set(a) || is_highest_bit_set(b))
+        max++;
+
+    OSSL_FN *rf = bn_acquire_ossl_fn(r, (int)max);
+    int ret = OSSL_FN_add(rf, a->data, b->data);
+    bn_release(r, (int)max);
+
+    return ret;
+}
+
+/* TODO(FIXNUM): TO BE REMOVED */
+/* pure BIGNUM unsigned subtraction of b from a, a must be larger than b. */
+static int bn_usub_legacy(BIGNUM *r, const BIGNUM *a, const BIGNUM *b)
 {
     int max, min, dif;
     BN_ULONG t1, t2, borrow, *rp;
@@ -167,3 +210,27 @@ int BN_usub(BIGNUM *r, const BIGNUM *a, const BIGNUM *b)
 
     return 1;
 }
+
+/* unsigned subtraction of b from a, a must be larger than b. */
+int BN_usub(BIGNUM *r, const BIGNUM *a, const BIGNUM *b)
+{
+    /* TODO(FIXNUM): TO BE REMOVED */
+    if (r->data == NULL || a->data == NULL || b->data == NULL)
+        return bn_usub_legacy(r, a, b);
+
+    bn_check_top(a);
+    bn_check_top(b);
+
+    if (a->top < b->top) { /* hmm... should not be happening */
+        ERR_raise(ERR_LIB_BN, BN_R_ARG2_LT_ARG3);
+        return 0;
+    }
+
+    size_t max = calculate_max_limbs(a, b);
+
+    OSSL_FN *rf = bn_acquire_ossl_fn(r, (int)max);
+    int ret = OSSL_FN_sub(rf, a->data, b->data);
+    bn_release(r, (int)max);
+
+    return ret;
+}

crypto/bn/bn_lib.c

@@ -279,7 +279,7 @@ static OSSL_FN *bn_expand_internal(const BIGNUM *b, int words)
 {
     OSSL_FN *a = NULL;
 
-    if (ossl_unlikely(words > (INT_MAX / (4 * BN_BITS2)))) {
+    if (ossl_unlikely(words > BN_MAX_WORDS)) {
         ERR_raise(ERR_LIB_BN, BN_R_BIGNUM_TOO_LONG);
         return NULL;
     }

crypto/bn/bn_local.h

@@ -227,6 +227,7 @@ static ossl_inline void bn_check_top(const BIGNUM *bn)
         } else if ((bn->flags & BN_FLG_FIXED_TOP) == 0) {
             assert(bn->d[bn->top - 1] != 0);
         }
+        assert(bn->dmax >= 0 && bn->dmax <= BN_MAX_WORDS);
     }
 }