[PATCH] aarch64: Improve SVE exponential special cases
Adhemerval Zanella Netto
adhemerval.zanella@linaro.org
Wed Nov 4 15:39:55 GMT 2026
On 04/11/26 12:06, Dylan Fleming wrote:
> Replaces the current exponential special case for SVE
> with an error free scaling, drastically reducing the
> worst-case errors for large magnitude inputs while having
> ~10% higher throughput
I think it shoud split one patch for symbol and add some test input to keep
track of the expected precision, as we for float and double.
> ---
> SHARED-FILES | 12 +--
> sysdeps/aarch64/fpu/sve/exp.c | 134 ++++++++++++++++++-------------
> sysdeps/aarch64/fpu/sve/exp10.c | 114 +++++++++++++++++---------
> sysdeps/aarch64/fpu/sve/exp10f.c | 68 +++++++++++-----
> sysdeps/aarch64/fpu/sve/exp2.c | 100 ++++++++++++++---------
> sysdeps/aarch64/fpu/sve/exp2f.c | 65 ++++++++++-----
> sysdeps/aarch64/fpu/sve/expf.c | 86 ++++++++++----------
> 7 files changed, 357 insertions(+), 222 deletions(-)
>
> diff --git a/SHARED-FILES b/SHARED-FILES
> index 8f89400177..ae894dbecb 100644
> --- a/SHARED-FILES
> +++ b/SHARED-FILES
> @@ -495,25 +495,25 @@ arm-optimized-routines:
> sysdeps/aarch64/fpu/sve/erfcf.c
> # math/aarch64/sve/erff.c, revision 1aad38d
> sysdeps/aarch64/fpu/sve/erff.c
> - # math/aarch64/sve/exp.c, revision d664b69
> + # math/aarch64/sve/exp.c, revision 22bc70e0
> sysdeps/aarch64/fpu/sve/exp.c
> - # math/aarch64/sve/exp10.c, revision d664b69
> + # math/aarch64/sve/exp10.c, revision 22bc70e0
> sysdeps/aarch64/fpu/sve/exp10.c
> - # math/aarch64/sve/exp10f.c, revision d664b69
> + # math/aarch64/sve/exp10f.c, revision ef813ddd
> sysdeps/aarch64/fpu/sve/exp10f.c
> # math/aarch64/sve/exp10m1.c, revision d664b69
> sysdeps/aarch64/fpu/sve/exp10m1.c
> # math/aarch64/sve/exp10m1f.c, revision d664b69
> sysdeps/aarch64/fpu/sve/exp10m1f.c
> - # math/aarch64/sve/exp2.c, revision d664b69
> + # math/aarch64/sve/exp2.c, revision 22bc70e0
> sysdeps/aarch64/fpu/sve/exp2.c
> - # math/aarch64/sve/exp2f.c, revision d664b69
> + # math/aarch64/sve/exp2f.c, revision 5e51e52e
> sysdeps/aarch64/fpu/sve/exp2f.c
> # math/aarch64/sve/exp2m1.c, revision d664b69
> sysdeps/aarch64/fpu/sve/exp2m1.c
> # math/aarch64/sve/exp2m1f.c, revision d664b69
> sysdeps/aarch64/fpu/sve/exp2m1f.c
> - # math/aarch64/sve/expf.c, revision 47044a5
> + # math/aarch64/sve/expf.c, revision 22bc70e0
> sysdeps/aarch64/fpu/sve/expf.c
> # math/aarch64/sve/expm1.c, revision d664b69
> sysdeps/aarch64/fpu/sve/expm1.c
> diff --git a/sysdeps/aarch64/fpu/sve/exp.c b/sysdeps/aarch64/fpu/sve/exp.c
> index 7c8c5e0559..be9e233509 100644
> --- a/sysdeps/aarch64/fpu/sve/exp.c
> +++ b/sysdeps/aarch64/fpu/sve/exp.c
> @@ -18,21 +18,21 @@
> <https://www.gnu.org/licenses/>. */
>
> #include "sv_math.h"
> -#include "sv_exp_special_inline.h"
>
> /* Value of |x| above which scale overflows without special treatment.
> - ln(2^(1022 + 1/128)) ~ 708.40. */
> -#define SpecialBound 0x1.62336f49c49a2p+9
> + ln(2^1022) ~ 708.3964. */
> +#define SpecialBound 0x1.6232bdd7abcd2p+9
>
> +/* Values of x which exp overflows or underflows. */
> +#define InfBound 0x1.63p+9 /* 710.0. */
> +#define ZeroBound -0x1.74910d52d3053p+9 /* ~ -745.13. */
> static const struct data
> {
> - struct sv_exp_special_data special_data;
> double c0, c2;
> double c1, c3;
> double ln2_hi, ln2_lo;
> - double inv_ln2, shift, special_bound;
> + double inv_ln2, shift, special_bound, inf_bound, zero_bound;
> } data = {
> - .special_data = SV_EXP_SPECIAL_DATA,
> .c0 = 0x1.fffffffffdbcdp-2,
> .c1 = 0x1.555555555444cp-3,
> .c2 = 0x1.555573c6a9f7dp-5,
> @@ -44,75 +44,101 @@ static const struct data
> /* 1.5*2^46+1023. This value is further explained below. */
> .shift = 0x1.800000000ffc0p+46,
> .special_bound = SpecialBound,
> + .inf_bound = InfBound,
> + .zero_bound = ZeroBound,
> };
>
> -static svfloat64_t NOINLINE
> -special_exp (svfloat64_t poly, svfloat64_t scale, svfloat64_t n, svuint64_t u,
> - const struct sv_exp_special_data *ds)
> +static inline svfloat64_t
> +exp_inline (svfloat64_t x, const svbool_t pg, const struct data *d)
> {
> - /* FEXPA zeroes the sign bit, however the sign is meaningful to the
> - special case function so needs to be copied.
> - e = sign bit of u << 46. */
> - svuint64_t e = svand_x (svptrue_b64 (), svlsl_x (svptrue_b64 (), u, 46),
> - 0x8000000000000000);
> - /* Copy sign to scale. */
> - scale = svreinterpret_f64 (
> - svadd_x (svptrue_b64 (), e, svreinterpret_u64 (scale)));
> - return special_case (scale, poly, n, ds);
> + /* Use a modifed version of the shift used for flooring, such that x/ln2 is
> + rounded to a multiple of 2^-6=1/64, shift = 1.5 * 2^52 * 2^-6 = 1.5 *
> + 2^46.
> +
> + n is not an integer but can be written as n = m + i/64, with i and m
> + integer, 0 <= i < 64 and m <= n.
> +
> + Bits 5:0 of z will be null every time x/ln2 reaches a new integer value
> + (n=m, i=0), and is incremented every time z (or n) is incremented by 1/64.
> + FEXPA expects i in bits 5:0 of the input so it can be used as index into
> + FEXPA hardwired table T[i] = 2^(i/64) for i = 0:63, that will in turn
> + populate the mantissa of the output. Therefore, we use u=asuint(z) as
> + input to FEXPA.
> +
> + We add 1023 to the modified shift value in order to set bits 16:6 of u to
> + 1, such that once these bits are moved to the exponent of the output of
> + FEXPA, we get the exponent of 2^n right, i.e. we get 2^m. */
> + svfloat64_t z = svmla_x (pg, sv_f64 (d->shift), x, d->inv_ln2);
> + svfloat64_t n = svsub_x (pg, z, d->shift);
> +
> + svfloat64_t ln2_hi_lo = svld1rq (svptrue_b64 (), &d->ln2_hi);
> + svfloat64_t r = x;
> + r = svmls_lane (r, n, ln2_hi_lo, 0);
> + r = svmls_lane (r, n, ln2_hi_lo, 1);
> +
> + svfloat64_t c13 = svld1rq (svptrue_b64 (), &d->c1);
> + svfloat64_t r2 = svmul_x (svptrue_b64 (), r, r);
> + svfloat64_t p01 = svmla_lane (sv_f64 (d->c0), r, c13, 0);
> + svfloat64_t p23 = svmla_lane (sv_f64 (d->c2), r, c13, 1);
> + svfloat64_t p04 = svmla_x (pg, p01, p23, r2);
> + svfloat64_t poly = svmla_x (pg, r, p04, r2);
> +
> + svfloat64_t scale = svexpa (svreinterpret_u64 (z));
> +
> + return svmla_x (pg, scale, scale, poly);
> }
>
> -/* SVE exp algorithm. Maximum measured error is 1.01ulps:
> - SV_NAME_D1 (exp)(0x1.4619d7b04da41p+6) got 0x1.885d9acc41da7p+117
> - want 0x1.885d9acc41da6p+117. */
> -svfloat64_t SV_NAME_D1 (exp) (svfloat64_t x, const svbool_t pg)
> +static svfloat64_t NOINLINE
> +special_case (svfloat64_t x, const svbool_t pg, const struct data *d)
> {
> - const struct data *d = ptr_barrier (&data);
> + /* Computes the offset and scale factor based on sign of the input. */
> + svbool_t is_negative = svcmplt (pg, x, 0.0);
> + svfloat64_t offset = svneg_m (sv_f64 (53.0), is_negative, sv_f64 (53.0));
> + svint64_t scale_adjust = svneg_m (sv_s64 (53), is_negative, sv_s64 (53));
> +
> + /* Bounds x between zero_bound and inf_bound, where exp(x) would return
> + 0 or inf. By clamping x to these bounds, the behaviour of large values
> + is more predictable. */
> + x = svmin_x (pg, svmax_x (pg, x, d->zero_bound), d->inf_bound);
>
> - /* Use a modifed version of the shift used for flooring, such that x/ln2 is
> - rounded to a multiple of 2^-6=1/64, shift = 1.5 * 2^52 * 2^-6 = 1.5 *
> - 2^46.
> -
> - n is not an integer but can be written as n = m + i/64, with i and m
> - integer, 0 <= i < 64 and m <= n.
> -
> - Bits 5:0 of z will be null every time x/ln2 reaches a new integer value
> - (n=m, i=0), and is incremented every time z (or n) is incremented by 1/64.
> - FEXPA expects i in bits 5:0 of the input so it can be used as index into
> - FEXPA hardwired table T[i] = 2^(i/64) for i = 0:63, that will in turn
> - populate the mantissa of the output. Therefore, we use u=asuint(z) as
> - input to FEXPA.
> -
> - We add 1023 to the modified shift value in order to set bits 16:6 of u to
> - 1, such that once these bits are moved to the exponent of the output of
> - FEXPA, we get the exponent of 2^n right, i.e. we get 2^m. */
> svfloat64_t z = svmla_x (pg, sv_f64 (d->shift), x, d->inv_ln2);
> - svuint64_t u = svreinterpret_u64 (z);
> svfloat64_t n = svsub_x (pg, z, d->shift);
> - svfloat64_t c13 = svld1rq (svptrue_b64 (), &d->c1);
> - /* r = x - n * ln2, r is in [-ln2/(2N), ln2/(2N)]. */
> +
> svfloat64_t ln2_hi_lo = svld1rq (svptrue_b64 (), &d->ln2_hi);
> svfloat64_t r = x;
> r = svmls_lane (r, n, ln2_hi_lo, 0);
> r = svmls_lane (r, n, ln2_hi_lo, 1);
>
> - /* poly = exp(r) - 1 ~= r + C0 r^2 + C1 r^3 + C2 r^4 + C3 r^5. */
> + svfloat64_t c13 = svld1rq (svptrue_b64 (), &d->c1);
> svfloat64_t r2 = svmul_x (svptrue_b64 (), r, r);
> svfloat64_t p01 = svmla_lane (sv_f64 (d->c0), r, c13, 0);
> svfloat64_t p23 = svmla_lane (sv_f64 (d->c2), r, c13, 1);
> svfloat64_t p04 = svmla_x (pg, p01, p23, r2);
> svfloat64_t poly = svmla_x (pg, r, p04, r2);
>
> - /* scale = 2^n, computed using FEXPA. FEXPA does not propagate NaNs, so for
> - consistent NaN handling we have to manually propagate them. This comes at
> - significant performance cost. */
> - svfloat64_t scale = svexpa (u);
> + /* Computes scale with an offset, which returns:
> + scale = 2^(n - 53). */
> + z = svsub_x (pg, z, offset);
> + svfloat64_t scale = svexpa (svreinterpret_u64 (z));
>
> + /* Reconstruct y as 2^(n - 53) * (1 + poly(r)). */
> + svfloat64_t y = svmla_x (pg, scale, scale, poly);
> +
> + /* Scale the result by 2^53:
> + 2^n * (1 + poly(r)) = 2^53 * 2^(n - 53) * (1 + poly(r)). */
> + return svscale_x (pg, y, scale_adjust);
> +}
> +
> +/* Vector version of exp
> + The maximum observed error is 0.51 + 0.5 ULP.
> + _ZGVsMxv_exp (0x1.4619d7b04da41p+6)
> + got 0x1.885d9acc41da7p+117
> + want 0x1.885d9acc41da6p+117. */
> +svfloat64_t SV_NAME_D1 (exp) (svfloat64_t x, const svbool_t pg)
> +{
> + const struct data *d = ptr_barrier (&data);
> svbool_t special = svacge (svptrue_b64 (), x, d->special_bound);
> - /* Assemble result as exp(x) = 2^n * exp(r). If |x| > Thresh the
> - multiplication may overflow, so use special case routine. */
> if (__glibc_unlikely (svptest_any (special, special)))
> - return special_exp (poly, scale, n, u, &d->special_data);
> -
> - /* No special case. */
> - return svmla_x (pg, scale, scale, poly);
> + return special_case (x, pg, d);
> + return exp_inline (x, pg, d);
> }
> diff --git a/sysdeps/aarch64/fpu/sve/exp10.c b/sysdeps/aarch64/fpu/sve/exp10.c
> index fd04bab44a..b8183aeda0 100644
> --- a/sysdeps/aarch64/fpu/sve/exp10.c
> +++ b/sysdeps/aarch64/fpu/sve/exp10.c
> @@ -18,21 +18,22 @@
> <https://www.gnu.org/licenses/>. */
>
> #include "sv_math.h"
> -#include "sv_exp_special_inline.h"
>
> /* Value of |x| above which scale overflows without special treatment.
> - log10(2^1022 + 1/128) ~ 307.65. */
> -#define SpecialBound 0x1.33a7ae900b507p+8
> + log10(2^1022) ~ 307.65. */
> +#define SpecialBound 0x1.33a7146f72a41p+8
> +
> +/* Values of x over which exp10 overflows or underflows. */
> +#define InfBound 0x1.35p+8 /* 309.0. */
> +#define ZeroBound -0x1.439b746e36b53p+8 /* ~ -323.61. */
>
> static const struct data
> {
> - struct sv_exp_special_data special_data;
> double log2_10_hi, log2_10_lo;
> double log10_2, c0;
> double c1, c3, c2, c4;
> - double shift, special_bound;
> + double shift, special_bound, inf_bound, zero_bound;
> } data = {
> - .special_data = SV_EXP_SPECIAL_DATA,
> /* Coefficients generated using Remez algorithm.
> rel error: 0x1.9fcb9b3p-60
> abs error: 0x1.a20d9598p-60 in [ -log10(2)/128, log10(2)/128 ]
> @@ -48,49 +49,74 @@ static const struct data
> .log2_10_hi = 0x1.34413509f79ffp-2, /* log2(10). */
> .log2_10_lo = -0x1.9dc1da994fd21p-59,
> .special_bound = SpecialBound,
> + .inf_bound = InfBound,
> + .zero_bound = ZeroBound,
> };
>
> -static svfloat64_t NOINLINE
> -special_exp (svfloat64_t scale, svfloat64_t poly, svfloat64_t n, svuint64_t u,
> - const struct sv_exp_special_data *ds)
> +static inline svfloat64_t
> +exp10_inline (svfloat64_t x, const svbool_t pg, const struct data *d)
> {
> - /* FEXPA zeroes the sign bit, however the sign is meaningful to the
> - special case function so needs to be copied.
> - e = sign bit of u << 46. */
> - svuint64_t e = svand_x (svptrue_b64 (), svlsl_x (svptrue_b64 (), u, 46),
> - 0x8000000000000000);
> - /* Copy sign to scale. */
> - scale = svreinterpret_f64 (
> - svadd_x (svptrue_b64 (), e, svreinterpret_u64 (scale)));
> - return special_case (scale, poly, n, ds);
> + /* n is x/log10(2) rounded to the nearest multiple of 1/64. */
> + svfloat64_t shift = sv_f64 (d->shift);
> + svfloat64_t log10_2_c0 = svld1rq (svptrue_b64 (), &d->log10_2);
> + svfloat64_t z = svmla_lane (shift, x, log10_2_c0, 0);
> + svfloat64_t n = svsub_x (pg, z, shift);
> +
> + /* r = x - n*log10(2). */
> + svfloat64_t log2_10 = svld1rq (svptrue_b64 (), &d->log2_10_hi);
> + svfloat64_t r = x;
> + r = svmls_lane (r, n, log2_10, 0);
> + r = svmls_lane (r, n, log2_10, 1);
> +
> + svfloat64_t scale = svexpa (svreinterpret_u64 (z));
> + svfloat64_t c24 = svld1rq (svptrue_b64 (), &d->c2);
> +
> + /* Approximate exp10(r) using polynomial. */
> + svfloat64_t r2 = svmul_x (svptrue_b64 (), r, r);
> + svfloat64_t p12 = svmla_lane (sv_f64 (d->c1), r, c24, 0);
> + svfloat64_t p34 = svmla_lane (sv_f64 (d->c3), r, c24, 1);
> + svfloat64_t p14 = svmla_x (pg, p12, p34, r2);
> +
> + svfloat64_t poly = svmla_x (pg, svmul_lane (r, log10_2_c0, 1), r2, p14);
> +
> + return svmla_x (pg, scale, scale, poly);
> }
>
> -/* Fast vector implementation of exp10 using FEXPA instruction.
> - Maximum measured error is 1.02 ulp.
> - SV_NAME_D1 (exp10)(-0x1.2862fec805e58p+2) got 0x1.885a89551d782p-16
> - want 0x1.885a89551d781p-16. */
> -svfloat64_t SV_NAME_D1 (exp10) (svfloat64_t x, svbool_t pg)
> +static svfloat64_t NOINLINE
> +special_case (svfloat64_t x, const svbool_t pg, const struct data *d)
> {
> - const struct data *d = ptr_barrier (&data);
> + /* Computes the offset and scale factor based on sign of the input. */
> + svbool_t is_negative = svcmplt (pg, x, 0.0);
> + svfloat64_t offset = svneg_m (sv_f64 (53.0), is_negative, sv_f64 (53.0));
> + svint64_t scale_adjust = svneg_m (sv_s64 (53), is_negative, sv_s64 (53));
> +
> + /* Bounds x between zero_bound and inf_bound, where exp10(x) would return
> + 0 or inf. By clamping x to these bounds, the behaviour of large values
> + is more predictable. */
> + x = svmin_x (pg, svmax_x (pg, x, d->zero_bound), d->inf_bound);
>
> - /* n = round(x/(log10(2)/N)). */
> + /* exp10(x) = 2^(n/N) * 10^r = 2^n * (1 + poly(r)),
> + with 1 + poly(r) in [1/sqrt(2), sqrt(2)] and
> + x = r + n * log10(2) / N, with r in [-log10(2)/2N, log10(2)/2N]. */
> svfloat64_t shift = sv_f64 (d->shift);
> svfloat64_t log10_2_c0 = svld1rq (svptrue_b64 (), &d->log10_2);
> +
> + /* n is x/log10(2) rounded to the nearest multiple of 1/64. */
> svfloat64_t z = svmla_lane (shift, x, log10_2_c0, 0);
> svfloat64_t n = svsub_x (pg, z, shift);
>
> - /* r = x - n*log10(2)/N. */
> + /* r = x - n*log10(2). */
> svfloat64_t log2_10 = svld1rq (svptrue_b64 (), &d->log2_10_hi);
> svfloat64_t r = x;
> r = svmls_lane (r, n, log2_10, 0);
> r = svmls_lane (r, n, log2_10, 1);
>
> - /* scale = 2^(n/N), computed using FEXPA. FEXPA does not propagate NaNs, so
> - for consistent NaN handling we have to manually propagate them. This
> - comes at significant performance cost. */
> - svuint64_t u = svreinterpret_u64 (z);
> - svfloat64_t scale = svexpa (u);
> + /* Computes scale with an offset, which returns scale = 2^(n - 53). */
> + z = svsub_x (pg, z, offset);
> + svfloat64_t scale = svexpa (svreinterpret_u64 (z));
> +
> svfloat64_t c24 = svld1rq (svptrue_b64 (), &d->c2);
> +
> /* Approximate exp10(r) using polynomial. */
> svfloat64_t r2 = svmul_x (svptrue_b64 (), r, r);
> svfloat64_t p12 = svmla_lane (sv_f64 (d->c1), r, c24, 0);
> @@ -99,12 +125,24 @@ svfloat64_t SV_NAME_D1 (exp10) (svfloat64_t x, svbool_t pg)
>
> svfloat64_t poly = svmla_x (pg, svmul_lane (r, log10_2_c0, 1), r2, p14);
>
> - svbool_t special = svacge (svptrue_b64 (), x, d->special_bound);
> - /* Assemble result as exp(x) = 2^n * exp(r). If |x| > Thresh the
> - multiplication may overflow, so use special case routine. */
> - if (__glibc_unlikely (svptest_any (special, special)))
> - return special_exp (scale, poly, n, u, &d->special_data);
> + /* Reconstruct y as 2^(n - 53) * (1 + poly(r)). */
> + svfloat64_t y = svmla_x (pg, scale, scale, poly);
>
> - /* No special case. */
> - return svmla_x (pg, scale, scale, poly);
> + /* Scale the result by 2^53:
> + 2^n * (1 + poly(r)) = 2^53 * 2^(n - 53) * (1 + poly(r)). */
> + return svscale_x (pg, y, scale_adjust);
> +}
> +
> +/* Vector version of exp10
> + The maximum observed error is 0.52 + 0.5 ULP.
> + _ZGVsMxv_exp10(-0x1.2862fec805e58p+2)
> + got 0x1.885a89551d782p-16
> + want 0x1.885a89551d781p-16. */
> +svfloat64_t SV_NAME_D1 (exp10) (svfloat64_t x, svbool_t pg)
> +{
> + const struct data *d = ptr_barrier (&data);
> + svbool_t special = svacgt (pg, x, d->special_bound);
> + if (__glibc_unlikely (svptest_any (special, special)))
> + return special_case (x, pg, d);
> + return exp10_inline (x, pg, d);
> }
> diff --git a/sysdeps/aarch64/fpu/sve/exp10f.c b/sysdeps/aarch64/fpu/sve/exp10f.c
> index 0c426797ff..a900176b39 100644
> --- a/sysdeps/aarch64/fpu/sve/exp10f.c
> +++ b/sysdeps/aarch64/fpu/sve/exp10f.c
> @@ -23,7 +23,7 @@
> correctly by FEXPA. */
> #define SpecialBound 0x1.2f702p+5 /* log10(2^126) ~ 37.93. */
>
> -/* Values of x over which exp overflows or underflows. */
> +/* Values of x over which exp10f overflows or underflows. */
> #define InfBound 0x1.34ccccccccccdp+5 /* 38.6. */
> #define ZeroBound -0x1.68ccccccccccdp+5 /* -45.1. */
>
> @@ -73,36 +73,64 @@ sv_exp10f_inline (svfloat32_t x, const svbool_t pg, const struct data *d)
> return svmla_x (pg, scale, scale, poly);
> }
>
> +/* Special case to handle large magnitudes of x, where fexpa returns nan.
> + By applying a fixed offset to the value passed to fexpa,
> + and then reapplying a scaling factor afterwards, this is avoided. */
> static svfloat32_t NOINLINE
> -special_case (svfloat32_t x, svbool_t pg, svbool_t special,
> - const struct data *d)
> +special_case (svfloat32_t x, const svbool_t pg, const struct data *d)
> {
> - /* This deals with overflow and underflow in exponential for special case
> - lanes. */
> - svbool_t is_inf = svcmpgt (pg, x, d->inf_bound);
> - svbool_t is_zero = svcmplt (pg, x, d->zero_bound);
> + /* Computes the offset and scale factor based on sign of the input. */
> + svbool_t is_negative = svcmplt (pg, x, 0.0f);
> + svfloat32_t offset = svneg_m (sv_f32 (23.0f), is_negative, sv_f32 (23.0f));
> + svint32_t scale_adjust = svneg_m (sv_s32 (23), is_negative, sv_s32 (23));
>
> - /* The input `x` is further reduced (to `x/2`) to allow for accurate
> - approximation on the interval `x > SpecialBound ~ 0x1.2f702p+5`. */
> - x = svmul_m (special, x, 0.5);
> + /* Bounds x between zero_bound and inf_bound, where exp10f(x) would return
> + 0 or inf. By clamping x to these bounds, the behaviour of large values
> + is more predictable. */
> + x = svmin_x (pg, svmax_x (pg, x, d->zero_bound), d->inf_bound);
>
> - /* Computes exp(x/2), and set lanes with underflow/overflow. */
> - svfloat32_t half_exp = sv_exp10f_inline (x, svptrue_b32 (), d);
> - half_exp = svmul_m (special, half_exp, half_exp);
> - half_exp = svsel (is_inf, sv_f32 (INFINITY), half_exp);
> + /* exp10(x) = 2^(n/N) * 10^r = 2^n * (1 + poly(r)),
> + with poly(r) in [1/sqrt(2), sqrt(2)] and
> + x = r + n * log10(2) / N, with r in [-log10(2)/2N, log10(2)/2N]. */
> + svfloat32_t lane_consts = svld1rq (svptrue_b32 (), &d->log10_2);
> +
> + /* n = round(x/(log10(2)/N)). */
> + svfloat32_t shift = sv_f32 (d->shift);
> + svfloat32_t z = svmla_lane (shift, x, lane_consts, 0);
> + svfloat32_t n = svsub_x (pg, z, shift);
> +
> + /* r = x - n*log10(2)/N. */
> + svfloat32_t r = x;
> + r = svmls_lane (r, n, lane_consts, 1);
> + r = svmls_lane (r, n, lane_consts, 2);
> +
> + /* Computes scale with an offset, which returns:
> + scale = 2^(n/N - 23). */
> + z = svsub_x (pg, z, offset);
> + svfloat32_t scale = svexpa (svreinterpret_u32 (z));
> +
> + /* poly(r) = exp(r) - 1 ~= r + 0.5 r^2. */
> + svfloat32_t poly = svmla_lane (sv_f32 (d->c0), r, lane_consts, 3);
> + poly = svmul_x (pg, poly, r);
> +
> + /* Reconstruct y as 2^(n/N - 23) * (1 + poly(r)). */
> + svfloat32_t y = svmla_x (pg, scale, scale, poly);
>
> - return svsel (is_zero, sv_f32 (0), half_exp);
> + /* Scale the result by 2^23:
> + 2^(n/N) * (1 + poly(r)) = 2^23 * 2^(n/N - 23) * (1 + poly(r)). */
> + return svscale_x (pg, y, scale_adjust);
> }
>
> -/* Single-precision SVE exp10f routine. Based on the FEXPA instruction.
> - Worst case error is 2.86 ULP +0.50 ULP.
> - _ZGVsMxv_exp10f (0x1.31b778p+5) got 0x1.ed399p+126
> - want 0x1.ed398ap+126. */
> +/* Vector version of exp10f.
> + The maximum observed error is 0.59 + 0.5 ULP.
> + _ZGVsMxv_exp10f (-0x1.041162p-4)
> + got 0x1.ba5c6cp-1
> + want 0x1.ba5c6ap-1. */
> svfloat32_t SV_NAME_F1 (exp10) (svfloat32_t x, const svbool_t pg)
> {
> const struct data *d = ptr_barrier (&data);
> svbool_t special = svacgt (pg, x, d->special_bound);
> if (__glibc_unlikely (svptest_any (special, special)))
> - return special_case (x, pg, special, d);
> + return special_case (x, svptrue_b32 (), d);
> return sv_exp10f_inline (x, svptrue_b32 (), d);
> }
> diff --git a/sysdeps/aarch64/fpu/sve/exp2.c b/sysdeps/aarch64/fpu/sve/exp2.c
> index a6f002f9b0..acc5726181 100644
> --- a/sysdeps/aarch64/fpu/sve/exp2.c
> +++ b/sysdeps/aarch64/fpu/sve/exp2.c
> @@ -18,62 +18,73 @@
> <https://www.gnu.org/licenses/>. */
>
> #include "sv_math.h"
> -#include "sv_exp_special_inline.h"
>
> /* Value of |x| above which scale overflows without special treatment.
> - log2(2^(1022 + 1/128)) ~ 1022.00. */
> -#define SpecialBound 0x1.ff01p+9
> + log2(2^1022) = 1022.00. */
> +#define SpecialBound 0x1.ffp+9
>
> -/* Value of n above which scale overflows even with special treatment. */
> -#define ScaleBound 1280
> +/* Values of x which exp2 overflows or underflows. */
> +#define InfBound 0x1p10 /* 1024. */
> +#define ZeroBound -0x1.0d008p+10 /* -1076.00781253. */
>
> static const struct data
> {
> double c2, c4;
> double c0, c1, c3;
> double shift, special_bound;
> - struct sv_exp_special_data special_data;
> + double inf_bound, zero_bound;
> } data = {
> - .special_data = SV_EXP_SPECIAL_DATA,
> /* Coefficients are computed using Remez algorithm with
> minimisation of the absolute error. */
> - .c0 = 0x1.62e42fefa39efp-1,
> - .c1 = 0x1.ebfbdff82a31bp-3,
> - .c2 = 0x1.c6b08d706c8a5p-5,
> - .c3 = 0x1.3b2ad2ff7d2f3p-7,
> - .c4 = 0x1.5d8761184beb3p-10,
> - .shift = 0x1.800000000ffc0p+46,
> - .special_bound = SpecialBound,
> + .c0 = 0x1.62e42fefa39efp-1, .c1 = 0x1.ebfbdff82a31bp-3,
> + .c2 = 0x1.c6b08d706c8a5p-5, .c3 = 0x1.3b2ad2ff7d2f3p-7,
> + .c4 = 0x1.5d8761184beb3p-10, .shift = 0x1.800000000ffc0p+46,
> + .special_bound = SpecialBound, .inf_bound = InfBound,
> + .zero_bound = ZeroBound,
> };
>
> -static svfloat64_t NOINLINE
> -special_exp (svfloat64_t poly, svfloat64_t scale, svfloat64_t n, svfloat64_t z,
> - const struct sv_exp_special_data *ds)
> +static svfloat64_t inline exp2_inline (svfloat64_t x, svbool_t pg,
> + const struct data *d)
> {
> - /* FEXPA zeroes the sign bit, however the sign is meaningful to the
> - special case function so needs to be copied.
> - e = sign bit of u << 46. */
> - svuint64_t u = svreinterpret_u64 (z);
> - svuint64_t e = svand_x (svptrue_b64 (), svlsl_x (svptrue_b64 (), u, 46),
> - 0x8000000000000000);
> - /* Copy sign to scale. */
> - scale = svreinterpret_f64 (
> - svadd_x (svptrue_b64 (), e, svreinterpret_u64 (scale)));
> - return special_case (scale, poly, n, ds);
> + svfloat64_t z = svadd_x (svptrue_b64 (), x, d->shift);
> + svfloat64_t n = svsub_x (pg, z, d->shift);
> + svfloat64_t r = svsub_x (pg, x, n);
> +
> + svfloat64_t scale = svexpa (svreinterpret_u64 (z));
> +
> + svfloat64_t r2 = svmul_x (svptrue_b64 (), r, r);
> + svfloat64_t c24 = svld1rq (svptrue_b64 (), &d->c2);
> +
> + /* Approximate exp2(r) using polynomial. */
> + /* y = exp2(r) - 1 ~= r * (C0 + C1 r + C2 r^2 + C3 r^3 + C4 r^4). */
> + svfloat64_t p12 = svmla_lane (sv_f64 (d->c1), r, c24, 0);
> + svfloat64_t p34 = svmla_lane (sv_f64 (d->c3), r, c24, 1);
> + svfloat64_t p = svmla_x (pg, p12, p34, r2);
> + p = svmad_x (pg, p, r, d->c0);
> + svfloat64_t poly = svmul_x (svptrue_b64 (), r, p);
> +
> + return svmla_x (pg, scale, scale, poly);
> }
>
> -/* Fast vector implementation of exp2.
> - Maximum measured error is 0.52 + 0.5 ulp.
> - _ZGVsMxv_exp2 (0x1.3b72ad5b701bfp-1) got 0x1.8861641b49e08p+0
> - want 0x1.8861641b49e07p+0. */
> -svfloat64_t SV_NAME_D1 (exp2) (svfloat64_t x, svbool_t pg)
> +static svfloat64_t NOINLINE
> +special_case (svfloat64_t x, svbool_t pg, const struct data *d)
> {
> - const struct data *d = ptr_barrier (&data);
> + /* Computes the offset and scale factor based on sign of the input. */
> + svbool_t is_negative = svcmplt (pg, x, 0.0);
> + svfloat64_t offset = svneg_m (sv_f64 (53.0), is_negative, sv_f64 (53.0));
> + svint64_t scale_adjust = svneg_m (sv_s64 (53), is_negative, sv_s64 (53));
> +
> + /* Bounds x between zero_bound and inf_bound, where exp2(x) would return
> + 0 or inf. By clamping x to these bounds, the behaviour of large values
> + is more predictable. */
> + x = svmin_x (pg, svmax_x (pg, x, d->zero_bound), d->inf_bound);
>
> svfloat64_t z = svadd_x (svptrue_b64 (), x, d->shift);
> svfloat64_t n = svsub_x (pg, z, d->shift);
> svfloat64_t r = svsub_x (pg, x, n);
>
> + /* Computes scale with an offset, which returns scale = 2^(n - 53). */
> + z = svsub_x (pg, z, offset);
> svfloat64_t scale = svexpa (svreinterpret_u64 (z));
>
> svfloat64_t r2 = svmul_x (svptrue_b64 (), r, r);
> @@ -87,11 +98,24 @@ svfloat64_t SV_NAME_D1 (exp2) (svfloat64_t x, svbool_t pg)
> p = svmad_x (pg, p, r, d->c0);
> svfloat64_t poly = svmul_x (svptrue_b64 (), r, p);
>
> + /* Reconstruct y as 2^(n - 53) * (1 + poly(r)). */
> + svfloat64_t y = svmla_x (pg, scale, scale, poly);
> +
> + /* Scale the result by 2^53:
> + 2^n * (1 + poly(r)) = 2^53 * 2^(n - 53) * (1 + poly(r)). */
> + return svscale_x (pg, y, scale_adjust);
> +}
> +
> +/* Vector version of exp2
> + The maximum observed error is 0.52 + 0.5 ULP.
> + _ZGVsMxv_exp2 (0x1.3b72ad5b701bfp-1)
> + got 0x1.8861641b49e08p+0
> + want 0x1.8861641b49e07p+0. */
> +svfloat64_t SV_NAME_D1 (exp2) (svfloat64_t x, svbool_t pg)
> +{
> + const struct data *d = ptr_barrier (&data);
> svbool_t special = svacge (svptrue_b64 (), x, d->special_bound);
> - /* Assemble result as exp(x) = 2^n * exp(r). If |x| > Thresh the
> - multiplication may overflow, so use special case routine. */
> if (__glibc_unlikely (svptest_any (special, special)))
> - return special_exp (poly, scale, n, z, &d->special_data);
> -
> - return svmla_x (pg, scale, scale, poly);
> + return special_case (x, pg, d);
> + return exp2_inline (x, pg, d);
> }
> diff --git a/sysdeps/aarch64/fpu/sve/exp2f.c b/sysdeps/aarch64/fpu/sve/exp2f.c
> index b71f472d44..b513e0775f 100644
> --- a/sysdeps/aarch64/fpu/sve/exp2f.c
> +++ b/sysdeps/aarch64/fpu/sve/exp2f.c
> @@ -25,7 +25,7 @@
>
> /* Values of x over which exp overflows or underflows. */
> #define InfBound 0x1.0p7f /* 128.0f. */
> -#define ZeroBound -0x1.2a8p7f /* -149.0f. */
> +#define ZeroBound -0x1.2cp7f /* -150.0f. */
>
> static const struct data
> {
> @@ -46,7 +46,8 @@ static const struct data
> static inline svfloat32_t
> sv_exp2f_inline (svfloat32_t x, const svbool_t pg, const struct data *d)
> {
> - /* exp2(x) = 2^n (1 + poly(r)), with 1 + poly(r) in [1/sqrt(2),sqrt(2)]
> + /* exp2(x) =
> + 2^n (1 + r * poly(r)), with 1 + r * poly(r) in [1/sqrt(2),sqrt(2)]
> x = n + r, with r in [-1/2, 1/2]. */
> svfloat32_t z = svadd_x (svptrue_b32 (), x, d->shift);
> svfloat32_t n = svsub_x (svptrue_b32 (), z, d->shift);
> @@ -60,36 +61,58 @@ sv_exp2f_inline (svfloat32_t x, const svbool_t pg, const struct data *d)
> return svmla_x (pg, scale, scale, poly);
> }
>
> +/* Special case to handle large magnitudes of x, where fexpa returns nan.
> + By applying a fixed offset to the value passed to fexpa,
> + and then reapplying a scaling factor afterwards, this is avoided. */
> static svfloat32_t NOINLINE
> -special_case (svfloat32_t x, svbool_t pg, svbool_t special,
> - const struct data *d)
> +special_case (svfloat32_t x, const svbool_t pg, const struct data *d)
> {
> - /* This deals with overflow and underflow in exponential for special case
> - lanes. */
> - svbool_t is_inf = svcmpgt (pg, x, d->inf_bound);
> - svbool_t is_zero = svcmplt (pg, x, d->zero_bound);
> + /* Computes the offset and scale factor based on sign of the input. */
> + svbool_t is_negative = svcmplt (pg, x, 0.0f);
> + svfloat32_t offset = svneg_m (sv_f32 (23.0f), is_negative, sv_f32 (23.0f));
> + svint32_t scale_adjust = svneg_m (sv_s32 (23), is_negative, sv_s32 (23));
> +
> + /* Bounds x between zero_bound and inf_bound, where exp2(x) would return
> + 0 or inf. By clamping x to these bounds, the behaviour of large values
> + is more predictable. */
> + x = svmin_x (pg, svmax_x (pg, x, d->zero_bound), d->inf_bound);
> +
> + /* exp2(x) =
> + 2^n (1 + r * poly(r)), with 1 + r * poly(r) in [1/sqrt(2),sqrt(2)]
> + x = n + r, with r in [-1/2, 1/2]. */
> + svfloat32_t z = svadd_x (svptrue_b32 (), x, d->shift);
> + svfloat32_t n = svsub_x (svptrue_b32 (), z, d->shift);
> +
> + svfloat32_t r = svsub_x (svptrue_b32 (), x, n);
>
> - /* The input `x` is further reduced (to `x/2`) to allow for accurate
> - approximation on the interval `x > SpecialBound = 126.0`. */
> - x = svmul_x (special, x, 0.5);
> + /* Computes scale with an offset, which returns:
> + scale = 2^(n/N - 23). */
> + z = svsub_x (pg, z, offset);
> + svfloat32_t scale = svexpa (svreinterpret_u32 (z));
> +
> + /* poly(r) = (exp2(r) - 1) / r = log(2) + r*log(2)^2/2. */
> + svfloat32_t poly = svmla_x (pg, sv_f32 (d->c0), r, sv_f32 (d->c1));
> + poly = svmul_x (svptrue_b32 (), poly, r);
>
> - /* Computes exp(x/2), and set lanes with underflow/overflow. */
> - svfloat32_t half_exp = sv_exp2f_inline (x, svptrue_b32 (), d);
> - half_exp = svmul_m (special, half_exp, half_exp);
> - half_exp = svsel (is_inf, sv_f32 (INFINITY), half_exp);
> + /* Reconstruct y as 2^(n - 23) * (1 + r * poly(r)). */
> + svfloat32_t y = svmla_x (pg, scale, scale, poly);
>
> - return svsel (is_zero, sv_f32 (0), half_exp);
> + /* Scale the result by 2^23:
> + 2^n * (1 + r * poly(r))
> + = 2^23 * 2^(n - 23) * (1 + r * poly(r)). */
> + return svscale_x (pg, y, scale_adjust);
> }
>
> -/* Single-precision SVE exp2f routine, based on the FEXPA instruction.
> - Worst case error is 2.87 +0.50 ULP.
> - _ZGVsMxv_exp2f (0x1.fbcb78p+6) got 0x1.ee1d32p+126
> - want 0x1.ee1d2cp+126. */
> +/* Vector version of exp2f.
> + The maximum observed error is 0.59 + 0.5 ULP.
> + _ZGVsMxv_exp2f (-0x1.97fa44p-3)
> + got 0x1.bdf77cp-1
> + want 0x1.bdf77ap-1. */
> svfloat32_t SV_NAME_F1 (exp2) (svfloat32_t x, const svbool_t pg)
> {
> const struct data *d = ptr_barrier (&data);
> svbool_t special = svacgt (pg, x, d->special_bound);
> if (__glibc_unlikely (svptest_any (special, special)))
> - return special_case (x, pg, special, d);
> + return special_case (x, svptrue_b32 (), d);
> return sv_exp2f_inline (x, pg, d);
> }
> diff --git a/sysdeps/aarch64/fpu/sve/expf.c b/sysdeps/aarch64/fpu/sve/expf.c
> index 9c552958ed..d5d6de9b68 100644
> --- a/sysdeps/aarch64/fpu/sve/expf.c
> +++ b/sysdeps/aarch64/fpu/sve/expf.c
> @@ -21,7 +21,7 @@
>
> /* For x < -SpecialBound, the result is subnormal and not handled
> correctly by FEXPA. */
> -#define SpecialBound 0x1.5d5e2ap+6f /* ln(2^126) ~ 87.34. */
> +#define SpecialBound 0x1.5d589ep+6f /* ln(2^126) ~ 87.34. */
>
> /* Values of x which exp overflows or underflows. */
> #define InfBound 0x1.62e42fp6f /* ~ 88.72. */
> @@ -29,8 +29,8 @@
>
> static const struct data
> {
> - float ln2_hi, ln2_lo, c1, c3;
> - float shift, inv_ln2, c2;
> + float ln2_hi, ln2_lo, c1;
> + float shift, inv_ln2;
> float32_t special_bound, inf_bound, zero_bound;
> } data = {
> /* shift = 1.5*2^17 + 127. */
> @@ -41,8 +41,6 @@ static const struct data
> /* Coefficients generated using Remez algorithm with minimisation of relative
> error. */
> .c1 = 0.5f,
> - .c2 = 0x1.55559p-3,
> - .c3 = 0x1.55555cp-5,
> .special_bound = SpecialBound,
> .inf_bound = InfBound,
> .zero_bound = ZeroBound,
> @@ -74,64 +72,62 @@ expf_inline (svfloat32_t x, const svbool_t pg, const struct data *d)
> return svmla_x (pg, scale, scale, poly);
> }
>
> -/* This is the version used if there is any special lane in the input vector.
> - The approximation needs to match that of the fast path.
> - To achieve this we assemble the same polynomial, ie `r + 0.5 * r^2`,
> - then we conditionally add an extra `c2 * r^3` term. */
> -static inline svfloat32_t
> -expf_slow_inline (svfloat32_t x, const svbool_t special, const struct data *d)
> +/* Special case to handle large magnitudes of x, where fexpa returns nan.
> + By applying a fixed offset to the value passed to fexpa,
> + and then reapplying a scaling factor afterwards, this is avoided. */
> +static svfloat32_t NOINLINE
> +special_case (svfloat32_t x, const svbool_t pg, const struct data *d)
> {
> + /* Computes the offset and scale factor based on sign of the input. */
> + svbool_t is_negative = svcmplt (pg, x, 0.0f);
> + svfloat32_t offset = svneg_m (sv_f32 (23.0f), is_negative, sv_f32 (23.0f));
> + svint32_t scale_adjust = svneg_m (sv_s32 (23), is_negative, sv_s32 (23));
> +
> + /* Bounds x between zero_bound and inf_bound, where exp(x) would return
> + 0 or inf. By clamping x to these bounds, the behaviour of large values
> + is more predictable. */
> + x = svmin_x (pg, svmax_x (pg, x, d->zero_bound), d->inf_bound);
> +
> + /* exp(x) = 2^n (1 + poly(r)), with 1 + poly(r) in [1/sqrt(2),sqrt(2)]
> + x = ln2*n + r, with r in [-ln2/2, ln2/2]. */
> svfloat32_t lane_constants = svld1rq (svptrue_b32 (), &d->ln2_hi);
>
> - svfloat32_t z = svmad_x (svptrue_b32 (), sv_f32 (d->inv_ln2), x, d->shift);
> - svfloat32_t n = svsub_x (svptrue_b32 (), z, d->shift);
> + /* n = round(x/(ln2/N)). */
> + svfloat32_t z = svmad_x (pg, sv_f32 (d->inv_ln2), x, d->shift);
> + svfloat32_t n = svsub_x (pg, z, d->shift);
>
> + /* r = x - n*ln2/N. */
> svfloat32_t r = x;
> r = svmls_lane (r, n, lane_constants, 0);
> r = svmls_lane (r, n, lane_constants, 1);
>
> - /* poly(r) = exp(r) - 1 ~= r + 0.5 r^2 + c2 r^3. */
> - svfloat32_t r2 = svmul_x (svptrue_b32 (), r, r);
> - svfloat32_t r3 = svmul_x (svptrue_b32 (), r, r2);
> - svfloat32_t poly_slow = svmla_lane (sv_f32 (d->c2), r, lane_constants, 3);
> - svfloat32_t poly_fast = svmla_lane (r, r2, lane_constants, 2);
> - svfloat32_t poly = svmla_m (special, poly_fast, poly_slow, r3);
> -
> + /* Computes scale with an offset, which returns:
> + scale = 2^(n/N - 23). */
> + z = svsub_x (pg, z, offset);
> svfloat32_t scale = svexpa (svreinterpret_u32 (z));
>
> - return svmla_x (svptrue_b32 (), scale, scale, poly);
> -}
> -
> -static svfloat32_t NOINLINE
> -special_case (svfloat32_t x, svbool_t pg, svbool_t special,
> - const struct data *d)
> -{
> - /* This deals with overflow and underflow in exponential for special case
> - lanes. */
> - svbool_t is_inf = svcmpgt (pg, x, d->inf_bound);
> - svbool_t is_zero = svcmplt (pg, x, d->zero_bound);
> -
> - /* The input `x` is further reduced (to `x/2`) to allow for accurate
> - approximation on the interval `x > SpecialBound ~ 87.3`. */
> - x = svmul_m (special, x, 0.5);
> + /* poly(r) = exp(r) - 1 ~= r + 0.5 r^2. */
> + svfloat32_t r2 = svmul_x (svptrue_b32 (), r, r);
> + svfloat32_t poly = svmla_lane (r, r2, lane_constants, 2);
>
> - /* Computes exp(x/2), and set lanes with underflow/overflow. */
> - svfloat32_t half_exp = expf_slow_inline (x, special, d);
> - half_exp = svmul_m (special, half_exp, half_exp);
> - half_exp = svsel (is_inf, sv_f32 (INFINITY), half_exp);
> + /* Reconstruct y as 2^(n/N - 23) * (1 + poly(r)). */
> + svfloat32_t y = svmla_x (pg, scale, scale, poly);
>
> - return svsel (is_zero, sv_f32 (0), half_exp);
> + /* Scale the result by 2^23:
> + 2^(n/N) * (1 + poly(r)) = 2^23 * 2^(n/N - 23) * (1 + poly(r)). */
> + return svscale_x (pg, y, scale_adjust);
> }
>
> -/* Optimised single-precision SVE exp function.
> - Worst-case error is 2.70 +0.50 ULP:
> - _ZGVsMxv_expf(0x1.5fec38p+6) got 0x1.e7831ep+126
> - want 0x1.e78318p+126. */
> +/* Vector version of expf.
> + The maximum observed error is 0.88 + 0.5 ULP.
> + _ZGVsMxv_expf (-0x1.bba276p-6)
> + got 0x1.f25288p-1
> + want 0x1.f2528ap-1. */
> svfloat32_t SV_NAME_F1 (exp) (svfloat32_t x, const svbool_t pg)
> {
> const struct data *d = ptr_barrier (&data);
> svbool_t special = svacgt (pg, x, d->special_bound);
> if (__glibc_unlikely (svptest_any (special, special)))
> - return special_case (x, pg, special, d);
> + return special_case (x, svptrue_b32 (), d);
> return expf_inline (x, svptrue_b32 (), d);
> }
More information about the Libc-alpha
mailing list