diff --git a/crypto/fipsmodule/bn/asm/x86_64-mont.pl b/crypto/fipsmodule/bn/asm/x86_64-mont.pl index 0a5a662da9..2b3a1e470b 100755 --- a/crypto/fipsmodule/bn/asm/x86_64-mont.pl +++ b/crypto/fipsmodule/bn/asm/x86_64-mont.pl @@ -89,8 +89,6 @@ $code=<<___; .text -.extern OPENSSL_ia32cap_P - .globl bn_mul_mont_nohw .type bn_mul_mont_nohw,\@function,6 .align 16 @@ -789,7 +787,7 @@ # int bn_sqr8x_mont( my $rptr="%rdi"; # const BN_ULONG *rptr, my $aptr="%rsi"; # const BN_ULONG *aptr, -my $bptr="%rdx"; # not used +my $mulx_adx_capable="%rdx"; # Different than upstream! my $nptr="%rcx"; # const BN_ULONG *nptr, my $n0 ="%r8"; # const BN_ULONG *n0); my $num ="%r9"; # int num, has to be divisible by 8 @@ -893,11 +891,8 @@ ___ $code.=<<___ if ($addx); #ifndef MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX - leaq OPENSSL_ia32cap_P(%rip),%rax - mov 8(%rax),%eax - and \$0x80100,%eax - cmp \$0x80100,%eax - jne .Lsqr8x_nox + test $mulx_adx_capable,$mulx_adx_capable + jz .Lsqr8x_nox call bn_sqrx8x_internal # see x86_64-mont5 module # %rax top-most carry diff --git a/crypto/fipsmodule/bn/bn_test.cc b/crypto/fipsmodule/bn/bn_test.cc index 5e13d782f8..1751349ff1 100644 --- a/crypto/fipsmodule/bn/bn_test.cc +++ b/crypto/fipsmodule/bn/bn_test.cc @@ -2864,8 +2864,8 @@ TEST_F(BNTest, BNMulMontABI) { mont->n0, words); #if !defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX) if (bn_sqr8x_mont_capable(words)) { - CHECK_ABI(bn_sqr8x_mont, r.data(), a.data(), a.data(), mont->N.d, - mont->n0, words); + CHECK_ABI(bn_sqr8x_mont, r.data(), a.data(), bn_mulx_adx_capable(), + mont->N.d, mont->n0, words); } #endif // !defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX) #else diff --git a/crypto/fipsmodule/bn/internal.h b/crypto/fipsmodule/bn/internal.h index b8674b1db1..343854339c 100644 --- a/crypto/fipsmodule/bn/internal.h +++ b/crypto/fipsmodule/bn/internal.h @@ -407,6 +407,10 @@ int bn_mul_mont(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp, const BN_ULONG *np, const BN_ULONG *n0, size_t num); #if defined(OPENSSL_X86_64) +OPENSSL_INLINE int bn_mulx_adx_capable(void) { + // MULX is in BMI2. + return CRYPTO_is_BMI2_capable() && CRYPTO_is_ADX_capable(); +} int bn_mul_mont_nohw(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp, const BN_ULONG *np, const BN_ULONG *n0, size_t num); OPENSSL_INLINE int bn_mul4x_mont_capable(size_t num) { @@ -416,16 +420,14 @@ int bn_mul4x_mont(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp, const BN_ULONG *np, const BN_ULONG *n0, size_t num); #if !defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX) OPENSSL_INLINE int bn_mulx4x_mont_capable(size_t num) { - // MULX is in BMI2. - return bn_mul4x_mont_capable(num) && CRYPTO_is_BMI2_capable() && - CRYPTO_is_ADX_capable(); + return bn_mul4x_mont_capable(num) && bn_mulx_adx_capable(); } int bn_mulx4x_mont(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp, const BN_ULONG *np, const BN_ULONG *n0, size_t num); OPENSSL_INLINE int bn_sqr8x_mont_capable(size_t num) { return (num >= 8) && ((num & 7) == 0); } -int bn_sqr8x_mont(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *unused_bp, +int bn_sqr8x_mont(BN_ULONG *rp, const BN_ULONG *ap, BN_ULONG mulx_adx_capable, const BN_ULONG *np, const BN_ULONG *n0, size_t num); #endif // !defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX) #endif // defined(OPENSSL_X86_64) diff --git a/crypto/fipsmodule/bn/montgomery.c b/crypto/fipsmodule/bn/montgomery.c index 9028a77acc..628f265eaa 100644 --- a/crypto/fipsmodule/bn/montgomery.c +++ b/crypto/fipsmodule/bn/montgomery.c @@ -639,7 +639,7 @@ int bn_mul_mont(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp, { #if !defined(MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX) if (ap == bp && bn_sqr8x_mont_capable(num)) { - return bn_sqr8x_mont(rp, ap, bp, np, n0, num); + return bn_sqr8x_mont(rp, ap, bn_mulx_adx_capable(), np, n0, num); } if (bn_mulx4x_mont_capable(num)) { return bn_mulx4x_mont(rp, ap, bp, np, n0, num);