[PATCH] aarch64: Improve SVE exponential special cases
Dylan Fleming
Dylan.Fleming@arm.com
Wed Nov 4 15:06:41 GMT 2026
Replaces the current exponential special case for SVE
with an error free scaling, drastically reducing the
worst-case errors for large magnitude inputs while having
~10% higher throughput
---
SHARED-FILES | 12 +--
sysdeps/aarch64/fpu/sve/exp.c | 134 ++++++++++++++++++-------------
sysdeps/aarch64/fpu/sve/exp10.c | 114 +++++++++++++++++---------
sysdeps/aarch64/fpu/sve/exp10f.c | 68 +++++++++++-----
sysdeps/aarch64/fpu/sve/exp2.c | 100 ++++++++++++++---------
sysdeps/aarch64/fpu/sve/exp2f.c | 65 ++++++++++-----
sysdeps/aarch64/fpu/sve/expf.c | 86 ++++++++++----------
7 files changed, 357 insertions(+), 222 deletions(-)
diff --git a/SHARED-FILES b/SHARED-FILES
index 8f89400177..ae894dbecb 100644
--- a/SHARED-FILES
+++ b/SHARED-FILES
@@ -495,25 +495,25 @@ arm-optimized-routines:
sysdeps/aarch64/fpu/sve/erfcf.c
# math/aarch64/sve/erff.c, revision 1aad38d
sysdeps/aarch64/fpu/sve/erff.c
- # math/aarch64/sve/exp.c, revision d664b69
+ # math/aarch64/sve/exp.c, revision 22bc70e0
sysdeps/aarch64/fpu/sve/exp.c
- # math/aarch64/sve/exp10.c, revision d664b69
+ # math/aarch64/sve/exp10.c, revision 22bc70e0
sysdeps/aarch64/fpu/sve/exp10.c
- # math/aarch64/sve/exp10f.c, revision d664b69
+ # math/aarch64/sve/exp10f.c, revision ef813ddd
sysdeps/aarch64/fpu/sve/exp10f.c
# math/aarch64/sve/exp10m1.c, revision d664b69
sysdeps/aarch64/fpu/sve/exp10m1.c
# math/aarch64/sve/exp10m1f.c, revision d664b69
sysdeps/aarch64/fpu/sve/exp10m1f.c
- # math/aarch64/sve/exp2.c, revision d664b69
+ # math/aarch64/sve/exp2.c, revision 22bc70e0
sysdeps/aarch64/fpu/sve/exp2.c
- # math/aarch64/sve/exp2f.c, revision d664b69
+ # math/aarch64/sve/exp2f.c, revision 5e51e52e
sysdeps/aarch64/fpu/sve/exp2f.c
# math/aarch64/sve/exp2m1.c, revision d664b69
sysdeps/aarch64/fpu/sve/exp2m1.c
# math/aarch64/sve/exp2m1f.c, revision d664b69
sysdeps/aarch64/fpu/sve/exp2m1f.c
- # math/aarch64/sve/expf.c, revision 47044a5
+ # math/aarch64/sve/expf.c, revision 22bc70e0
sysdeps/aarch64/fpu/sve/expf.c
# math/aarch64/sve/expm1.c, revision d664b69
sysdeps/aarch64/fpu/sve/expm1.c
diff --git a/sysdeps/aarch64/fpu/sve/exp.c b/sysdeps/aarch64/fpu/sve/exp.c
index 7c8c5e0559..be9e233509 100644
--- a/sysdeps/aarch64/fpu/sve/exp.c
+++ b/sysdeps/aarch64/fpu/sve/exp.c
@@ -18,21 +18,21 @@
<https://www.gnu.org/licenses/>. */
#include "sv_math.h"
-#include "sv_exp_special_inline.h"
/* Value of |x| above which scale overflows without special treatment.
- ln(2^(1022 + 1/128)) ~ 708.40. */
-#define SpecialBound 0x1.62336f49c49a2p+9
+ ln(2^1022) ~ 708.3964. */
+#define SpecialBound 0x1.6232bdd7abcd2p+9
+/* Values of x which exp overflows or underflows. */
+#define InfBound 0x1.63p+9 /* 710.0. */
+#define ZeroBound -0x1.74910d52d3053p+9 /* ~ -745.13. */
static const struct data
{
- struct sv_exp_special_data special_data;
double c0, c2;
double c1, c3;
double ln2_hi, ln2_lo;
- double inv_ln2, shift, special_bound;
+ double inv_ln2, shift, special_bound, inf_bound, zero_bound;
} data = {
- .special_data = SV_EXP_SPECIAL_DATA,
.c0 = 0x1.fffffffffdbcdp-2,
.c1 = 0x1.555555555444cp-3,
.c2 = 0x1.555573c6a9f7dp-5,
@@ -44,75 +44,101 @@ static const struct data
/* 1.5*2^46+1023. This value is further explained below. */
.shift = 0x1.800000000ffc0p+46,
.special_bound = SpecialBound,
+ .inf_bound = InfBound,
+ .zero_bound = ZeroBound,
};
-static svfloat64_t NOINLINE
-special_exp (svfloat64_t poly, svfloat64_t scale, svfloat64_t n, svuint64_t u,
- const struct sv_exp_special_data *ds)
+static inline svfloat64_t
+exp_inline (svfloat64_t x, const svbool_t pg, const struct data *d)
{
- /* FEXPA zeroes the sign bit, however the sign is meaningful to the
- special case function so needs to be copied.
- e = sign bit of u << 46. */
- svuint64_t e = svand_x (svptrue_b64 (), svlsl_x (svptrue_b64 (), u, 46),
- 0x8000000000000000);
- /* Copy sign to scale. */
- scale = svreinterpret_f64 (
- svadd_x (svptrue_b64 (), e, svreinterpret_u64 (scale)));
- return special_case (scale, poly, n, ds);
+ /* Use a modifed version of the shift used for flooring, such that x/ln2 is
+ rounded to a multiple of 2^-6=1/64, shift = 1.5 * 2^52 * 2^-6 = 1.5 *
+ 2^46.
+
+ n is not an integer but can be written as n = m + i/64, with i and m
+ integer, 0 <= i < 64 and m <= n.
+
+ Bits 5:0 of z will be null every time x/ln2 reaches a new integer value
+ (n=m, i=0), and is incremented every time z (or n) is incremented by 1/64.
+ FEXPA expects i in bits 5:0 of the input so it can be used as index into
+ FEXPA hardwired table T[i] = 2^(i/64) for i = 0:63, that will in turn
+ populate the mantissa of the output. Therefore, we use u=asuint(z) as
+ input to FEXPA.
+
+ We add 1023 to the modified shift value in order to set bits 16:6 of u to
+ 1, such that once these bits are moved to the exponent of the output of
+ FEXPA, we get the exponent of 2^n right, i.e. we get 2^m. */
+ svfloat64_t z = svmla_x (pg, sv_f64 (d->shift), x, d->inv_ln2);
+ svfloat64_t n = svsub_x (pg, z, d->shift);
+
+ svfloat64_t ln2_hi_lo = svld1rq (svptrue_b64 (), &d->ln2_hi);
+ svfloat64_t r = x;
+ r = svmls_lane (r, n, ln2_hi_lo, 0);
+ r = svmls_lane (r, n, ln2_hi_lo, 1);
+
+ svfloat64_t c13 = svld1rq (svptrue_b64 (), &d->c1);
+ svfloat64_t r2 = svmul_x (svptrue_b64 (), r, r);
+ svfloat64_t p01 = svmla_lane (sv_f64 (d->c0), r, c13, 0);
+ svfloat64_t p23 = svmla_lane (sv_f64 (d->c2), r, c13, 1);
+ svfloat64_t p04 = svmla_x (pg, p01, p23, r2);
+ svfloat64_t poly = svmla_x (pg, r, p04, r2);
+
+ svfloat64_t scale = svexpa (svreinterpret_u64 (z));
+
+ return svmla_x (pg, scale, scale, poly);
}
-/* SVE exp algorithm. Maximum measured error is 1.01ulps:
- SV_NAME_D1 (exp)(0x1.4619d7b04da41p+6) got 0x1.885d9acc41da7p+117
- want 0x1.885d9acc41da6p+117. */
-svfloat64_t SV_NAME_D1 (exp) (svfloat64_t x, const svbool_t pg)
+static svfloat64_t NOINLINE
+special_case (svfloat64_t x, const svbool_t pg, const struct data *d)
{
- const struct data *d = ptr_barrier (&data);
+ /* Computes the offset and scale factor based on sign of the input. */
+ svbool_t is_negative = svcmplt (pg, x, 0.0);
+ svfloat64_t offset = svneg_m (sv_f64 (53.0), is_negative, sv_f64 (53.0));
+ svint64_t scale_adjust = svneg_m (sv_s64 (53), is_negative, sv_s64 (53));
+
+ /* Bounds x between zero_bound and inf_bound, where exp(x) would return
+ 0 or inf. By clamping x to these bounds, the behaviour of large values
+ is more predictable. */
+ x = svmin_x (pg, svmax_x (pg, x, d->zero_bound), d->inf_bound);
- /* Use a modifed version of the shift used for flooring, such that x/ln2 is
- rounded to a multiple of 2^-6=1/64, shift = 1.5 * 2^52 * 2^-6 = 1.5 *
- 2^46.
-
- n is not an integer but can be written as n = m + i/64, with i and m
- integer, 0 <= i < 64 and m <= n.
-
- Bits 5:0 of z will be null every time x/ln2 reaches a new integer value
- (n=m, i=0), and is incremented every time z (or n) is incremented by 1/64.
- FEXPA expects i in bits 5:0 of the input so it can be used as index into
- FEXPA hardwired table T[i] = 2^(i/64) for i = 0:63, that will in turn
- populate the mantissa of the output. Therefore, we use u=asuint(z) as
- input to FEXPA.
-
- We add 1023 to the modified shift value in order to set bits 16:6 of u to
- 1, such that once these bits are moved to the exponent of the output of
- FEXPA, we get the exponent of 2^n right, i.e. we get 2^m. */
svfloat64_t z = svmla_x (pg, sv_f64 (d->shift), x, d->inv_ln2);
- svuint64_t u = svreinterpret_u64 (z);
svfloat64_t n = svsub_x (pg, z, d->shift);
- svfloat64_t c13 = svld1rq (svptrue_b64 (), &d->c1);
- /* r = x - n * ln2, r is in [-ln2/(2N), ln2/(2N)]. */
+
svfloat64_t ln2_hi_lo = svld1rq (svptrue_b64 (), &d->ln2_hi);
svfloat64_t r = x;
r = svmls_lane (r, n, ln2_hi_lo, 0);
r = svmls_lane (r, n, ln2_hi_lo, 1);
- /* poly = exp(r) - 1 ~= r + C0 r^2 + C1 r^3 + C2 r^4 + C3 r^5. */
+ svfloat64_t c13 = svld1rq (svptrue_b64 (), &d->c1);
svfloat64_t r2 = svmul_x (svptrue_b64 (), r, r);
svfloat64_t p01 = svmla_lane (sv_f64 (d->c0), r, c13, 0);
svfloat64_t p23 = svmla_lane (sv_f64 (d->c2), r, c13, 1);
svfloat64_t p04 = svmla_x (pg, p01, p23, r2);
svfloat64_t poly = svmla_x (pg, r, p04, r2);
- /* scale = 2^n, computed using FEXPA. FEXPA does not propagate NaNs, so for
- consistent NaN handling we have to manually propagate them. This comes at
- significant performance cost. */
- svfloat64_t scale = svexpa (u);
+ /* Computes scale with an offset, which returns:
+ scale = 2^(n - 53). */
+ z = svsub_x (pg, z, offset);
+ svfloat64_t scale = svexpa (svreinterpret_u64 (z));
+ /* Reconstruct y as 2^(n - 53) * (1 + poly(r)). */
+ svfloat64_t y = svmla_x (pg, scale, scale, poly);
+
+ /* Scale the result by 2^53:
+ 2^n * (1 + poly(r)) = 2^53 * 2^(n - 53) * (1 + poly(r)). */
+ return svscale_x (pg, y, scale_adjust);
+}
+
+/* Vector version of exp
+ The maximum observed error is 0.51 + 0.5 ULP.
+ _ZGVsMxv_exp (0x1.4619d7b04da41p+6)
+ got 0x1.885d9acc41da7p+117
+ want 0x1.885d9acc41da6p+117. */
+svfloat64_t SV_NAME_D1 (exp) (svfloat64_t x, const svbool_t pg)
+{
+ const struct data *d = ptr_barrier (&data);
svbool_t special = svacge (svptrue_b64 (), x, d->special_bound);
- /* Assemble result as exp(x) = 2^n * exp(r). If |x| > Thresh the
- multiplication may overflow, so use special case routine. */
if (__glibc_unlikely (svptest_any (special, special)))
- return special_exp (poly, scale, n, u, &d->special_data);
-
- /* No special case. */
- return svmla_x (pg, scale, scale, poly);
+ return special_case (x, pg, d);
+ return exp_inline (x, pg, d);
}
diff --git a/sysdeps/aarch64/fpu/sve/exp10.c b/sysdeps/aarch64/fpu/sve/exp10.c
index fd04bab44a..b8183aeda0 100644
--- a/sysdeps/aarch64/fpu/sve/exp10.c
+++ b/sysdeps/aarch64/fpu/sve/exp10.c
@@ -18,21 +18,22 @@
<https://www.gnu.org/licenses/>. */
#include "sv_math.h"
-#include "sv_exp_special_inline.h"
/* Value of |x| above which scale overflows without special treatment.
- log10(2^1022 + 1/128) ~ 307.65. */
-#define SpecialBound 0x1.33a7ae900b507p+8
+ log10(2^1022) ~ 307.65. */
+#define SpecialBound 0x1.33a7146f72a41p+8
+
+/* Values of x over which exp10 overflows or underflows. */
+#define InfBound 0x1.35p+8 /* 309.0. */
+#define ZeroBound -0x1.439b746e36b53p+8 /* ~ -323.61. */
static const struct data
{
- struct sv_exp_special_data special_data;
double log2_10_hi, log2_10_lo;
double log10_2, c0;
double c1, c3, c2, c4;
- double shift, special_bound;
+ double shift, special_bound, inf_bound, zero_bound;
} data = {
- .special_data = SV_EXP_SPECIAL_DATA,
/* Coefficients generated using Remez algorithm.
rel error: 0x1.9fcb9b3p-60
abs error: 0x1.a20d9598p-60 in [ -log10(2)/128, log10(2)/128 ]
@@ -48,49 +49,74 @@ static const struct data
.log2_10_hi = 0x1.34413509f79ffp-2, /* log2(10). */
.log2_10_lo = -0x1.9dc1da994fd21p-59,
.special_bound = SpecialBound,
+ .inf_bound = InfBound,
+ .zero_bound = ZeroBound,
};
-static svfloat64_t NOINLINE
-special_exp (svfloat64_t scale, svfloat64_t poly, svfloat64_t n, svuint64_t u,
- const struct sv_exp_special_data *ds)
+static inline svfloat64_t
+exp10_inline (svfloat64_t x, const svbool_t pg, const struct data *d)
{
- /* FEXPA zeroes the sign bit, however the sign is meaningful to the
- special case function so needs to be copied.
- e = sign bit of u << 46. */
- svuint64_t e = svand_x (svptrue_b64 (), svlsl_x (svptrue_b64 (), u, 46),
- 0x8000000000000000);
- /* Copy sign to scale. */
- scale = svreinterpret_f64 (
- svadd_x (svptrue_b64 (), e, svreinterpret_u64 (scale)));
- return special_case (scale, poly, n, ds);
+ /* n is x/log10(2) rounded to the nearest multiple of 1/64. */
+ svfloat64_t shift = sv_f64 (d->shift);
+ svfloat64_t log10_2_c0 = svld1rq (svptrue_b64 (), &d->log10_2);
+ svfloat64_t z = svmla_lane (shift, x, log10_2_c0, 0);
+ svfloat64_t n = svsub_x (pg, z, shift);
+
+ /* r = x - n*log10(2). */
+ svfloat64_t log2_10 = svld1rq (svptrue_b64 (), &d->log2_10_hi);
+ svfloat64_t r = x;
+ r = svmls_lane (r, n, log2_10, 0);
+ r = svmls_lane (r, n, log2_10, 1);
+
+ svfloat64_t scale = svexpa (svreinterpret_u64 (z));
+ svfloat64_t c24 = svld1rq (svptrue_b64 (), &d->c2);
+
+ /* Approximate exp10(r) using polynomial. */
+ svfloat64_t r2 = svmul_x (svptrue_b64 (), r, r);
+ svfloat64_t p12 = svmla_lane (sv_f64 (d->c1), r, c24, 0);
+ svfloat64_t p34 = svmla_lane (sv_f64 (d->c3), r, c24, 1);
+ svfloat64_t p14 = svmla_x (pg, p12, p34, r2);
+
+ svfloat64_t poly = svmla_x (pg, svmul_lane (r, log10_2_c0, 1), r2, p14);
+
+ return svmla_x (pg, scale, scale, poly);
}
-/* Fast vector implementation of exp10 using FEXPA instruction.
- Maximum measured error is 1.02 ulp.
- SV_NAME_D1 (exp10)(-0x1.2862fec805e58p+2) got 0x1.885a89551d782p-16
- want 0x1.885a89551d781p-16. */
-svfloat64_t SV_NAME_D1 (exp10) (svfloat64_t x, svbool_t pg)
+static svfloat64_t NOINLINE
+special_case (svfloat64_t x, const svbool_t pg, const struct data *d)
{
- const struct data *d = ptr_barrier (&data);
+ /* Computes the offset and scale factor based on sign of the input. */
+ svbool_t is_negative = svcmplt (pg, x, 0.0);
+ svfloat64_t offset = svneg_m (sv_f64 (53.0), is_negative, sv_f64 (53.0));
+ svint64_t scale_adjust = svneg_m (sv_s64 (53), is_negative, sv_s64 (53));
+
+ /* Bounds x between zero_bound and inf_bound, where exp10(x) would return
+ 0 or inf. By clamping x to these bounds, the behaviour of large values
+ is more predictable. */
+ x = svmin_x (pg, svmax_x (pg, x, d->zero_bound), d->inf_bound);
- /* n = round(x/(log10(2)/N)). */
+ /* exp10(x) = 2^(n/N) * 10^r = 2^n * (1 + poly(r)),
+ with 1 + poly(r) in [1/sqrt(2), sqrt(2)] and
+ x = r + n * log10(2) / N, with r in [-log10(2)/2N, log10(2)/2N]. */
svfloat64_t shift = sv_f64 (d->shift);
svfloat64_t log10_2_c0 = svld1rq (svptrue_b64 (), &d->log10_2);
+
+ /* n is x/log10(2) rounded to the nearest multiple of 1/64. */
svfloat64_t z = svmla_lane (shift, x, log10_2_c0, 0);
svfloat64_t n = svsub_x (pg, z, shift);
- /* r = x - n*log10(2)/N. */
+ /* r = x - n*log10(2). */
svfloat64_t log2_10 = svld1rq (svptrue_b64 (), &d->log2_10_hi);
svfloat64_t r = x;
r = svmls_lane (r, n, log2_10, 0);
r = svmls_lane (r, n, log2_10, 1);
- /* scale = 2^(n/N), computed using FEXPA. FEXPA does not propagate NaNs, so
- for consistent NaN handling we have to manually propagate them. This
- comes at significant performance cost. */
- svuint64_t u = svreinterpret_u64 (z);
- svfloat64_t scale = svexpa (u);
+ /* Computes scale with an offset, which returns scale = 2^(n - 53). */
+ z = svsub_x (pg, z, offset);
+ svfloat64_t scale = svexpa (svreinterpret_u64 (z));
+
svfloat64_t c24 = svld1rq (svptrue_b64 (), &d->c2);
+
/* Approximate exp10(r) using polynomial. */
svfloat64_t r2 = svmul_x (svptrue_b64 (), r, r);
svfloat64_t p12 = svmla_lane (sv_f64 (d->c1), r, c24, 0);
@@ -99,12 +125,24 @@ svfloat64_t SV_NAME_D1 (exp10) (svfloat64_t x, svbool_t pg)
svfloat64_t poly = svmla_x (pg, svmul_lane (r, log10_2_c0, 1), r2, p14);
- svbool_t special = svacge (svptrue_b64 (), x, d->special_bound);
- /* Assemble result as exp(x) = 2^n * exp(r). If |x| > Thresh the
- multiplication may overflow, so use special case routine. */
- if (__glibc_unlikely (svptest_any (special, special)))
- return special_exp (scale, poly, n, u, &d->special_data);
+ /* Reconstruct y as 2^(n - 53) * (1 + poly(r)). */
+ svfloat64_t y = svmla_x (pg, scale, scale, poly);
- /* No special case. */
- return svmla_x (pg, scale, scale, poly);
+ /* Scale the result by 2^53:
+ 2^n * (1 + poly(r)) = 2^53 * 2^(n - 53) * (1 + poly(r)). */
+ return svscale_x (pg, y, scale_adjust);
+}
+
+/* Vector version of exp10
+ The maximum observed error is 0.52 + 0.5 ULP.
+ _ZGVsMxv_exp10(-0x1.2862fec805e58p+2)
+ got 0x1.885a89551d782p-16
+ want 0x1.885a89551d781p-16. */
+svfloat64_t SV_NAME_D1 (exp10) (svfloat64_t x, svbool_t pg)
+{
+ const struct data *d = ptr_barrier (&data);
+ svbool_t special = svacgt (pg, x, d->special_bound);
+ if (__glibc_unlikely (svptest_any (special, special)))
+ return special_case (x, pg, d);
+ return exp10_inline (x, pg, d);
}
diff --git a/sysdeps/aarch64/fpu/sve/exp10f.c b/sysdeps/aarch64/fpu/sve/exp10f.c
index 0c426797ff..a900176b39 100644
--- a/sysdeps/aarch64/fpu/sve/exp10f.c
+++ b/sysdeps/aarch64/fpu/sve/exp10f.c
@@ -23,7 +23,7 @@
correctly by FEXPA. */
#define SpecialBound 0x1.2f702p+5 /* log10(2^126) ~ 37.93. */
-/* Values of x over which exp overflows or underflows. */
+/* Values of x over which exp10f overflows or underflows. */
#define InfBound 0x1.34ccccccccccdp+5 /* 38.6. */
#define ZeroBound -0x1.68ccccccccccdp+5 /* -45.1. */
@@ -73,36 +73,64 @@ sv_exp10f_inline (svfloat32_t x, const svbool_t pg, const struct data *d)
return svmla_x (pg, scale, scale, poly);
}
+/* Special case to handle large magnitudes of x, where fexpa returns nan.
+ By applying a fixed offset to the value passed to fexpa,
+ and then reapplying a scaling factor afterwards, this is avoided. */
static svfloat32_t NOINLINE
-special_case (svfloat32_t x, svbool_t pg, svbool_t special,
- const struct data *d)
+special_case (svfloat32_t x, const svbool_t pg, const struct data *d)
{
- /* This deals with overflow and underflow in exponential for special case
- lanes. */
- svbool_t is_inf = svcmpgt (pg, x, d->inf_bound);
- svbool_t is_zero = svcmplt (pg, x, d->zero_bound);
+ /* Computes the offset and scale factor based on sign of the input. */
+ svbool_t is_negative = svcmplt (pg, x, 0.0f);
+ svfloat32_t offset = svneg_m (sv_f32 (23.0f), is_negative, sv_f32 (23.0f));
+ svint32_t scale_adjust = svneg_m (sv_s32 (23), is_negative, sv_s32 (23));
- /* The input `x` is further reduced (to `x/2`) to allow for accurate
- approximation on the interval `x > SpecialBound ~ 0x1.2f702p+5`. */
- x = svmul_m (special, x, 0.5);
+ /* Bounds x between zero_bound and inf_bound, where exp10f(x) would return
+ 0 or inf. By clamping x to these bounds, the behaviour of large values
+ is more predictable. */
+ x = svmin_x (pg, svmax_x (pg, x, d->zero_bound), d->inf_bound);
- /* Computes exp(x/2), and set lanes with underflow/overflow. */
- svfloat32_t half_exp = sv_exp10f_inline (x, svptrue_b32 (), d);
- half_exp = svmul_m (special, half_exp, half_exp);
- half_exp = svsel (is_inf, sv_f32 (INFINITY), half_exp);
+ /* exp10(x) = 2^(n/N) * 10^r = 2^n * (1 + poly(r)),
+ with poly(r) in [1/sqrt(2), sqrt(2)] and
+ x = r + n * log10(2) / N, with r in [-log10(2)/2N, log10(2)/2N]. */
+ svfloat32_t lane_consts = svld1rq (svptrue_b32 (), &d->log10_2);
+
+ /* n = round(x/(log10(2)/N)). */
+ svfloat32_t shift = sv_f32 (d->shift);
+ svfloat32_t z = svmla_lane (shift, x, lane_consts, 0);
+ svfloat32_t n = svsub_x (pg, z, shift);
+
+ /* r = x - n*log10(2)/N. */
+ svfloat32_t r = x;
+ r = svmls_lane (r, n, lane_consts, 1);
+ r = svmls_lane (r, n, lane_consts, 2);
+
+ /* Computes scale with an offset, which returns:
+ scale = 2^(n/N - 23). */
+ z = svsub_x (pg, z, offset);
+ svfloat32_t scale = svexpa (svreinterpret_u32 (z));
+
+ /* poly(r) = exp(r) - 1 ~= r + 0.5 r^2. */
+ svfloat32_t poly = svmla_lane (sv_f32 (d->c0), r, lane_consts, 3);
+ poly = svmul_x (pg, poly, r);
+
+ /* Reconstruct y as 2^(n/N - 23) * (1 + poly(r)). */
+ svfloat32_t y = svmla_x (pg, scale, scale, poly);
- return svsel (is_zero, sv_f32 (0), half_exp);
+ /* Scale the result by 2^23:
+ 2^(n/N) * (1 + poly(r)) = 2^23 * 2^(n/N - 23) * (1 + poly(r)). */
+ return svscale_x (pg, y, scale_adjust);
}
-/* Single-precision SVE exp10f routine. Based on the FEXPA instruction.
- Worst case error is 2.86 ULP +0.50 ULP.
- _ZGVsMxv_exp10f (0x1.31b778p+5) got 0x1.ed399p+126
- want 0x1.ed398ap+126. */
+/* Vector version of exp10f.
+ The maximum observed error is 0.59 + 0.5 ULP.
+ _ZGVsMxv_exp10f (-0x1.041162p-4)
+ got 0x1.ba5c6cp-1
+ want 0x1.ba5c6ap-1. */
svfloat32_t SV_NAME_F1 (exp10) (svfloat32_t x, const svbool_t pg)
{
const struct data *d = ptr_barrier (&data);
svbool_t special = svacgt (pg, x, d->special_bound);
if (__glibc_unlikely (svptest_any (special, special)))
- return special_case (x, pg, special, d);
+ return special_case (x, svptrue_b32 (), d);
return sv_exp10f_inline (x, svptrue_b32 (), d);
}
diff --git a/sysdeps/aarch64/fpu/sve/exp2.c b/sysdeps/aarch64/fpu/sve/exp2.c
index a6f002f9b0..acc5726181 100644
--- a/sysdeps/aarch64/fpu/sve/exp2.c
+++ b/sysdeps/aarch64/fpu/sve/exp2.c
@@ -18,62 +18,73 @@
<https://www.gnu.org/licenses/>. */
#include "sv_math.h"
-#include "sv_exp_special_inline.h"
/* Value of |x| above which scale overflows without special treatment.
- log2(2^(1022 + 1/128)) ~ 1022.00. */
-#define SpecialBound 0x1.ff01p+9
+ log2(2^1022) = 1022.00. */
+#define SpecialBound 0x1.ffp+9
-/* Value of n above which scale overflows even with special treatment. */
-#define ScaleBound 1280
+/* Values of x which exp2 overflows or underflows. */
+#define InfBound 0x1p10 /* 1024. */
+#define ZeroBound -0x1.0d008p+10 /* -1076.00781253. */
static const struct data
{
double c2, c4;
double c0, c1, c3;
double shift, special_bound;
- struct sv_exp_special_data special_data;
+ double inf_bound, zero_bound;
} data = {
- .special_data = SV_EXP_SPECIAL_DATA,
/* Coefficients are computed using Remez algorithm with
minimisation of the absolute error. */
- .c0 = 0x1.62e42fefa39efp-1,
- .c1 = 0x1.ebfbdff82a31bp-3,
- .c2 = 0x1.c6b08d706c8a5p-5,
- .c3 = 0x1.3b2ad2ff7d2f3p-7,
- .c4 = 0x1.5d8761184beb3p-10,
- .shift = 0x1.800000000ffc0p+46,
- .special_bound = SpecialBound,
+ .c0 = 0x1.62e42fefa39efp-1, .c1 = 0x1.ebfbdff82a31bp-3,
+ .c2 = 0x1.c6b08d706c8a5p-5, .c3 = 0x1.3b2ad2ff7d2f3p-7,
+ .c4 = 0x1.5d8761184beb3p-10, .shift = 0x1.800000000ffc0p+46,
+ .special_bound = SpecialBound, .inf_bound = InfBound,
+ .zero_bound = ZeroBound,
};
-static svfloat64_t NOINLINE
-special_exp (svfloat64_t poly, svfloat64_t scale, svfloat64_t n, svfloat64_t z,
- const struct sv_exp_special_data *ds)
+static svfloat64_t inline exp2_inline (svfloat64_t x, svbool_t pg,
+ const struct data *d)
{
- /* FEXPA zeroes the sign bit, however the sign is meaningful to the
- special case function so needs to be copied.
- e = sign bit of u << 46. */
- svuint64_t u = svreinterpret_u64 (z);
- svuint64_t e = svand_x (svptrue_b64 (), svlsl_x (svptrue_b64 (), u, 46),
- 0x8000000000000000);
- /* Copy sign to scale. */
- scale = svreinterpret_f64 (
- svadd_x (svptrue_b64 (), e, svreinterpret_u64 (scale)));
- return special_case (scale, poly, n, ds);
+ svfloat64_t z = svadd_x (svptrue_b64 (), x, d->shift);
+ svfloat64_t n = svsub_x (pg, z, d->shift);
+ svfloat64_t r = svsub_x (pg, x, n);
+
+ svfloat64_t scale = svexpa (svreinterpret_u64 (z));
+
+ svfloat64_t r2 = svmul_x (svptrue_b64 (), r, r);
+ svfloat64_t c24 = svld1rq (svptrue_b64 (), &d->c2);
+
+ /* Approximate exp2(r) using polynomial. */
+ /* y = exp2(r) - 1 ~= r * (C0 + C1 r + C2 r^2 + C3 r^3 + C4 r^4). */
+ svfloat64_t p12 = svmla_lane (sv_f64 (d->c1), r, c24, 0);
+ svfloat64_t p34 = svmla_lane (sv_f64 (d->c3), r, c24, 1);
+ svfloat64_t p = svmla_x (pg, p12, p34, r2);
+ p = svmad_x (pg, p, r, d->c0);
+ svfloat64_t poly = svmul_x (svptrue_b64 (), r, p);
+
+ return svmla_x (pg, scale, scale, poly);
}
-/* Fast vector implementation of exp2.
- Maximum measured error is 0.52 + 0.5 ulp.
- _ZGVsMxv_exp2 (0x1.3b72ad5b701bfp-1) got 0x1.8861641b49e08p+0
- want 0x1.8861641b49e07p+0. */
-svfloat64_t SV_NAME_D1 (exp2) (svfloat64_t x, svbool_t pg)
+static svfloat64_t NOINLINE
+special_case (svfloat64_t x, svbool_t pg, const struct data *d)
{
- const struct data *d = ptr_barrier (&data);
+ /* Computes the offset and scale factor based on sign of the input. */
+ svbool_t is_negative = svcmplt (pg, x, 0.0);
+ svfloat64_t offset = svneg_m (sv_f64 (53.0), is_negative, sv_f64 (53.0));
+ svint64_t scale_adjust = svneg_m (sv_s64 (53), is_negative, sv_s64 (53));
+
+ /* Bounds x between zero_bound and inf_bound, where exp2(x) would return
+ 0 or inf. By clamping x to these bounds, the behaviour of large values
+ is more predictable. */
+ x = svmin_x (pg, svmax_x (pg, x, d->zero_bound), d->inf_bound);
svfloat64_t z = svadd_x (svptrue_b64 (), x, d->shift);
svfloat64_t n = svsub_x (pg, z, d->shift);
svfloat64_t r = svsub_x (pg, x, n);
+ /* Computes scale with an offset, which returns scale = 2^(n - 53). */
+ z = svsub_x (pg, z, offset);
svfloat64_t scale = svexpa (svreinterpret_u64 (z));
svfloat64_t r2 = svmul_x (svptrue_b64 (), r, r);
@@ -87,11 +98,24 @@ svfloat64_t SV_NAME_D1 (exp2) (svfloat64_t x, svbool_t pg)
p = svmad_x (pg, p, r, d->c0);
svfloat64_t poly = svmul_x (svptrue_b64 (), r, p);
+ /* Reconstruct y as 2^(n - 53) * (1 + poly(r)). */
+ svfloat64_t y = svmla_x (pg, scale, scale, poly);
+
+ /* Scale the result by 2^53:
+ 2^n * (1 + poly(r)) = 2^53 * 2^(n - 53) * (1 + poly(r)). */
+ return svscale_x (pg, y, scale_adjust);
+}
+
+/* Vector version of exp2
+ The maximum observed error is 0.52 + 0.5 ULP.
+ _ZGVsMxv_exp2 (0x1.3b72ad5b701bfp-1)
+ got 0x1.8861641b49e08p+0
+ want 0x1.8861641b49e07p+0. */
+svfloat64_t SV_NAME_D1 (exp2) (svfloat64_t x, svbool_t pg)
+{
+ const struct data *d = ptr_barrier (&data);
svbool_t special = svacge (svptrue_b64 (), x, d->special_bound);
- /* Assemble result as exp(x) = 2^n * exp(r). If |x| > Thresh the
- multiplication may overflow, so use special case routine. */
if (__glibc_unlikely (svptest_any (special, special)))
- return special_exp (poly, scale, n, z, &d->special_data);
-
- return svmla_x (pg, scale, scale, poly);
+ return special_case (x, pg, d);
+ return exp2_inline (x, pg, d);
}
diff --git a/sysdeps/aarch64/fpu/sve/exp2f.c b/sysdeps/aarch64/fpu/sve/exp2f.c
index b71f472d44..b513e0775f 100644
--- a/sysdeps/aarch64/fpu/sve/exp2f.c
+++ b/sysdeps/aarch64/fpu/sve/exp2f.c
@@ -25,7 +25,7 @@
/* Values of x over which exp overflows or underflows. */
#define InfBound 0x1.0p7f /* 128.0f. */
-#define ZeroBound -0x1.2a8p7f /* -149.0f. */
+#define ZeroBound -0x1.2cp7f /* -150.0f. */
static const struct data
{
@@ -46,7 +46,8 @@ static const struct data
static inline svfloat32_t
sv_exp2f_inline (svfloat32_t x, const svbool_t pg, const struct data *d)
{
- /* exp2(x) = 2^n (1 + poly(r)), with 1 + poly(r) in [1/sqrt(2),sqrt(2)]
+ /* exp2(x) =
+ 2^n (1 + r * poly(r)), with 1 + r * poly(r) in [1/sqrt(2),sqrt(2)]
x = n + r, with r in [-1/2, 1/2]. */
svfloat32_t z = svadd_x (svptrue_b32 (), x, d->shift);
svfloat32_t n = svsub_x (svptrue_b32 (), z, d->shift);
@@ -60,36 +61,58 @@ sv_exp2f_inline (svfloat32_t x, const svbool_t pg, const struct data *d)
return svmla_x (pg, scale, scale, poly);
}
+/* Special case to handle large magnitudes of x, where fexpa returns nan.
+ By applying a fixed offset to the value passed to fexpa,
+ and then reapplying a scaling factor afterwards, this is avoided. */
static svfloat32_t NOINLINE
-special_case (svfloat32_t x, svbool_t pg, svbool_t special,
- const struct data *d)
+special_case (svfloat32_t x, const svbool_t pg, const struct data *d)
{
- /* This deals with overflow and underflow in exponential for special case
- lanes. */
- svbool_t is_inf = svcmpgt (pg, x, d->inf_bound);
- svbool_t is_zero = svcmplt (pg, x, d->zero_bound);
+ /* Computes the offset and scale factor based on sign of the input. */
+ svbool_t is_negative = svcmplt (pg, x, 0.0f);
+ svfloat32_t offset = svneg_m (sv_f32 (23.0f), is_negative, sv_f32 (23.0f));
+ svint32_t scale_adjust = svneg_m (sv_s32 (23), is_negative, sv_s32 (23));
+
+ /* Bounds x between zero_bound and inf_bound, where exp2(x) would return
+ 0 or inf. By clamping x to these bounds, the behaviour of large values
+ is more predictable. */
+ x = svmin_x (pg, svmax_x (pg, x, d->zero_bound), d->inf_bound);
+
+ /* exp2(x) =
+ 2^n (1 + r * poly(r)), with 1 + r * poly(r) in [1/sqrt(2),sqrt(2)]
+ x = n + r, with r in [-1/2, 1/2]. */
+ svfloat32_t z = svadd_x (svptrue_b32 (), x, d->shift);
+ svfloat32_t n = svsub_x (svptrue_b32 (), z, d->shift);
+
+ svfloat32_t r = svsub_x (svptrue_b32 (), x, n);
- /* The input `x` is further reduced (to `x/2`) to allow for accurate
- approximation on the interval `x > SpecialBound = 126.0`. */
- x = svmul_x (special, x, 0.5);
+ /* Computes scale with an offset, which returns:
+ scale = 2^(n/N - 23). */
+ z = svsub_x (pg, z, offset);
+ svfloat32_t scale = svexpa (svreinterpret_u32 (z));
+
+ /* poly(r) = (exp2(r) - 1) / r = log(2) + r*log(2)^2/2. */
+ svfloat32_t poly = svmla_x (pg, sv_f32 (d->c0), r, sv_f32 (d->c1));
+ poly = svmul_x (svptrue_b32 (), poly, r);
- /* Computes exp(x/2), and set lanes with underflow/overflow. */
- svfloat32_t half_exp = sv_exp2f_inline (x, svptrue_b32 (), d);
- half_exp = svmul_m (special, half_exp, half_exp);
- half_exp = svsel (is_inf, sv_f32 (INFINITY), half_exp);
+ /* Reconstruct y as 2^(n - 23) * (1 + r * poly(r)). */
+ svfloat32_t y = svmla_x (pg, scale, scale, poly);
- return svsel (is_zero, sv_f32 (0), half_exp);
+ /* Scale the result by 2^23:
+ 2^n * (1 + r * poly(r))
+ = 2^23 * 2^(n - 23) * (1 + r * poly(r)). */
+ return svscale_x (pg, y, scale_adjust);
}
-/* Single-precision SVE exp2f routine, based on the FEXPA instruction.
- Worst case error is 2.87 +0.50 ULP.
- _ZGVsMxv_exp2f (0x1.fbcb78p+6) got 0x1.ee1d32p+126
- want 0x1.ee1d2cp+126. */
+/* Vector version of exp2f.
+ The maximum observed error is 0.59 + 0.5 ULP.
+ _ZGVsMxv_exp2f (-0x1.97fa44p-3)
+ got 0x1.bdf77cp-1
+ want 0x1.bdf77ap-1. */
svfloat32_t SV_NAME_F1 (exp2) (svfloat32_t x, const svbool_t pg)
{
const struct data *d = ptr_barrier (&data);
svbool_t special = svacgt (pg, x, d->special_bound);
if (__glibc_unlikely (svptest_any (special, special)))
- return special_case (x, pg, special, d);
+ return special_case (x, svptrue_b32 (), d);
return sv_exp2f_inline (x, pg, d);
}
diff --git a/sysdeps/aarch64/fpu/sve/expf.c b/sysdeps/aarch64/fpu/sve/expf.c
index 9c552958ed..d5d6de9b68 100644
--- a/sysdeps/aarch64/fpu/sve/expf.c
+++ b/sysdeps/aarch64/fpu/sve/expf.c
@@ -21,7 +21,7 @@
/* For x < -SpecialBound, the result is subnormal and not handled
correctly by FEXPA. */
-#define SpecialBound 0x1.5d5e2ap+6f /* ln(2^126) ~ 87.34. */
+#define SpecialBound 0x1.5d589ep+6f /* ln(2^126) ~ 87.34. */
/* Values of x which exp overflows or underflows. */
#define InfBound 0x1.62e42fp6f /* ~ 88.72. */
@@ -29,8 +29,8 @@
static const struct data
{
- float ln2_hi, ln2_lo, c1, c3;
- float shift, inv_ln2, c2;
+ float ln2_hi, ln2_lo, c1;
+ float shift, inv_ln2;
float32_t special_bound, inf_bound, zero_bound;
} data = {
/* shift = 1.5*2^17 + 127. */
@@ -41,8 +41,6 @@ static const struct data
/* Coefficients generated using Remez algorithm with minimisation of relative
error. */
.c1 = 0.5f,
- .c2 = 0x1.55559p-3,
- .c3 = 0x1.55555cp-5,
.special_bound = SpecialBound,
.inf_bound = InfBound,
.zero_bound = ZeroBound,
@@ -74,64 +72,62 @@ expf_inline (svfloat32_t x, const svbool_t pg, const struct data *d)
return svmla_x (pg, scale, scale, poly);
}
-/* This is the version used if there is any special lane in the input vector.
- The approximation needs to match that of the fast path.
- To achieve this we assemble the same polynomial, ie `r + 0.5 * r^2`,
- then we conditionally add an extra `c2 * r^3` term. */
-static inline svfloat32_t
-expf_slow_inline (svfloat32_t x, const svbool_t special, const struct data *d)
+/* Special case to handle large magnitudes of x, where fexpa returns nan.
+ By applying a fixed offset to the value passed to fexpa,
+ and then reapplying a scaling factor afterwards, this is avoided. */
+static svfloat32_t NOINLINE
+special_case (svfloat32_t x, const svbool_t pg, const struct data *d)
{
+ /* Computes the offset and scale factor based on sign of the input. */
+ svbool_t is_negative = svcmplt (pg, x, 0.0f);
+ svfloat32_t offset = svneg_m (sv_f32 (23.0f), is_negative, sv_f32 (23.0f));
+ svint32_t scale_adjust = svneg_m (sv_s32 (23), is_negative, sv_s32 (23));
+
+ /* Bounds x between zero_bound and inf_bound, where exp(x) would return
+ 0 or inf. By clamping x to these bounds, the behaviour of large values
+ is more predictable. */
+ x = svmin_x (pg, svmax_x (pg, x, d->zero_bound), d->inf_bound);
+
+ /* exp(x) = 2^n (1 + poly(r)), with 1 + poly(r) in [1/sqrt(2),sqrt(2)]
+ x = ln2*n + r, with r in [-ln2/2, ln2/2]. */
svfloat32_t lane_constants = svld1rq (svptrue_b32 (), &d->ln2_hi);
- svfloat32_t z = svmad_x (svptrue_b32 (), sv_f32 (d->inv_ln2), x, d->shift);
- svfloat32_t n = svsub_x (svptrue_b32 (), z, d->shift);
+ /* n = round(x/(ln2/N)). */
+ svfloat32_t z = svmad_x (pg, sv_f32 (d->inv_ln2), x, d->shift);
+ svfloat32_t n = svsub_x (pg, z, d->shift);
+ /* r = x - n*ln2/N. */
svfloat32_t r = x;
r = svmls_lane (r, n, lane_constants, 0);
r = svmls_lane (r, n, lane_constants, 1);
- /* poly(r) = exp(r) - 1 ~= r + 0.5 r^2 + c2 r^3. */
- svfloat32_t r2 = svmul_x (svptrue_b32 (), r, r);
- svfloat32_t r3 = svmul_x (svptrue_b32 (), r, r2);
- svfloat32_t poly_slow = svmla_lane (sv_f32 (d->c2), r, lane_constants, 3);
- svfloat32_t poly_fast = svmla_lane (r, r2, lane_constants, 2);
- svfloat32_t poly = svmla_m (special, poly_fast, poly_slow, r3);
-
+ /* Computes scale with an offset, which returns:
+ scale = 2^(n/N - 23). */
+ z = svsub_x (pg, z, offset);
svfloat32_t scale = svexpa (svreinterpret_u32 (z));
- return svmla_x (svptrue_b32 (), scale, scale, poly);
-}
-
-static svfloat32_t NOINLINE
-special_case (svfloat32_t x, svbool_t pg, svbool_t special,
- const struct data *d)
-{
- /* This deals with overflow and underflow in exponential for special case
- lanes. */
- svbool_t is_inf = svcmpgt (pg, x, d->inf_bound);
- svbool_t is_zero = svcmplt (pg, x, d->zero_bound);
-
- /* The input `x` is further reduced (to `x/2`) to allow for accurate
- approximation on the interval `x > SpecialBound ~ 87.3`. */
- x = svmul_m (special, x, 0.5);
+ /* poly(r) = exp(r) - 1 ~= r + 0.5 r^2. */
+ svfloat32_t r2 = svmul_x (svptrue_b32 (), r, r);
+ svfloat32_t poly = svmla_lane (r, r2, lane_constants, 2);
- /* Computes exp(x/2), and set lanes with underflow/overflow. */
- svfloat32_t half_exp = expf_slow_inline (x, special, d);
- half_exp = svmul_m (special, half_exp, half_exp);
- half_exp = svsel (is_inf, sv_f32 (INFINITY), half_exp);
+ /* Reconstruct y as 2^(n/N - 23) * (1 + poly(r)). */
+ svfloat32_t y = svmla_x (pg, scale, scale, poly);
- return svsel (is_zero, sv_f32 (0), half_exp);
+ /* Scale the result by 2^23:
+ 2^(n/N) * (1 + poly(r)) = 2^23 * 2^(n/N - 23) * (1 + poly(r)). */
+ return svscale_x (pg, y, scale_adjust);
}
-/* Optimised single-precision SVE exp function.
- Worst-case error is 2.70 +0.50 ULP:
- _ZGVsMxv_expf(0x1.5fec38p+6) got 0x1.e7831ep+126
- want 0x1.e78318p+126. */
+/* Vector version of expf.
+ The maximum observed error is 0.88 + 0.5 ULP.
+ _ZGVsMxv_expf (-0x1.bba276p-6)
+ got 0x1.f25288p-1
+ want 0x1.f2528ap-1. */
svfloat32_t SV_NAME_F1 (exp) (svfloat32_t x, const svbool_t pg)
{
const struct data *d = ptr_barrier (&data);
svbool_t special = svacgt (pg, x, d->special_bound);
if (__glibc_unlikely (svptest_any (special, special)))
- return special_case (x, pg, special, d);
+ return special_case (x, svptrue_b32 (), d);
return expf_inline (x, svptrue_b32 (), d);
}
--
2.43.0
More information about the Libc-alpha
mailing list