[PATCH] aarch64: Improve SVE exponential special cases

Dylan Fleming Dylan.Fleming@arm.com
Wed Nov 4 15:06:41 GMT 2026


Replaces the current exponential special case for SVE
with an error free scaling, drastically reducing the
worst-case errors for large magnitude inputs while having
~10% higher throughput
---
 SHARED-FILES                     |  12 +--
 sysdeps/aarch64/fpu/sve/exp.c    | 134 ++++++++++++++++++-------------
 sysdeps/aarch64/fpu/sve/exp10.c  | 114 +++++++++++++++++---------
 sysdeps/aarch64/fpu/sve/exp10f.c |  68 +++++++++++-----
 sysdeps/aarch64/fpu/sve/exp2.c   | 100 ++++++++++++++---------
 sysdeps/aarch64/fpu/sve/exp2f.c  |  65 ++++++++++-----
 sysdeps/aarch64/fpu/sve/expf.c   |  86 ++++++++++----------
 7 files changed, 357 insertions(+), 222 deletions(-)

diff --git a/SHARED-FILES b/SHARED-FILES
index 8f89400177..ae894dbecb 100644
--- a/SHARED-FILES
+++ b/SHARED-FILES
@@ -495,25 +495,25 @@ arm-optimized-routines:
   sysdeps/aarch64/fpu/sve/erfcf.c
   # math/aarch64/sve/erff.c, revision 1aad38d
   sysdeps/aarch64/fpu/sve/erff.c
-  # math/aarch64/sve/exp.c, revision d664b69
+  # math/aarch64/sve/exp.c, revision 22bc70e0
   sysdeps/aarch64/fpu/sve/exp.c
-  # math/aarch64/sve/exp10.c, revision d664b69
+  # math/aarch64/sve/exp10.c, revision 22bc70e0
   sysdeps/aarch64/fpu/sve/exp10.c
-  # math/aarch64/sve/exp10f.c, revision d664b69
+  # math/aarch64/sve/exp10f.c, revision ef813ddd
   sysdeps/aarch64/fpu/sve/exp10f.c
   # math/aarch64/sve/exp10m1.c, revision d664b69
   sysdeps/aarch64/fpu/sve/exp10m1.c
   # math/aarch64/sve/exp10m1f.c, revision d664b69
   sysdeps/aarch64/fpu/sve/exp10m1f.c
-  # math/aarch64/sve/exp2.c, revision d664b69
+  # math/aarch64/sve/exp2.c, revision 22bc70e0
   sysdeps/aarch64/fpu/sve/exp2.c
-  # math/aarch64/sve/exp2f.c, revision d664b69
+  # math/aarch64/sve/exp2f.c, revision 5e51e52e
   sysdeps/aarch64/fpu/sve/exp2f.c
   # math/aarch64/sve/exp2m1.c, revision d664b69
   sysdeps/aarch64/fpu/sve/exp2m1.c
   # math/aarch64/sve/exp2m1f.c, revision d664b69
   sysdeps/aarch64/fpu/sve/exp2m1f.c
-  # math/aarch64/sve/expf.c, revision 47044a5
+  # math/aarch64/sve/expf.c, revision 22bc70e0
   sysdeps/aarch64/fpu/sve/expf.c
   # math/aarch64/sve/expm1.c, revision d664b69
   sysdeps/aarch64/fpu/sve/expm1.c
diff --git a/sysdeps/aarch64/fpu/sve/exp.c b/sysdeps/aarch64/fpu/sve/exp.c
index 7c8c5e0559..be9e233509 100644
--- a/sysdeps/aarch64/fpu/sve/exp.c
+++ b/sysdeps/aarch64/fpu/sve/exp.c
@@ -18,21 +18,21 @@
    <https://www.gnu.org/licenses/>.  */
 
 #include "sv_math.h"
-#include "sv_exp_special_inline.h"
 
 /* Value of |x| above which scale overflows without special treatment.
-   ln(2^(1022 + 1/128)) ~ 708.40.  */
-#define SpecialBound 0x1.62336f49c49a2p+9
+   ln(2^1022) ~ 708.3964.  */
+#define SpecialBound 0x1.6232bdd7abcd2p+9
 
+/* Values of x which exp overflows or underflows.  */
+#define InfBound 0x1.63p+9		/* 710.0.  */
+#define ZeroBound -0x1.74910d52d3053p+9 /* ~ -745.13.  */
 static const struct data
 {
-  struct sv_exp_special_data special_data;
   double c0, c2;
   double c1, c3;
   double ln2_hi, ln2_lo;
-  double inv_ln2, shift, special_bound;
+  double inv_ln2, shift, special_bound, inf_bound, zero_bound;
 } data = {
-  .special_data = SV_EXP_SPECIAL_DATA,
   .c0 = 0x1.fffffffffdbcdp-2,
   .c1 = 0x1.555555555444cp-3,
   .c2 = 0x1.555573c6a9f7dp-5,
@@ -44,75 +44,101 @@ static const struct data
   /* 1.5*2^46+1023. This value is further explained below.  */
   .shift = 0x1.800000000ffc0p+46,
   .special_bound = SpecialBound,
+  .inf_bound = InfBound,
+  .zero_bound = ZeroBound,
 };
 
-static svfloat64_t NOINLINE
-special_exp (svfloat64_t poly, svfloat64_t scale, svfloat64_t n, svuint64_t u,
-	     const struct sv_exp_special_data *ds)
+static inline svfloat64_t
+exp_inline (svfloat64_t x, const svbool_t pg, const struct data *d)
 {
-  /* FEXPA zeroes the sign bit, however the sign is meaningful to the
-  special case function so needs to be copied.
-  e = sign bit of u << 46.  */
-  svuint64_t e = svand_x (svptrue_b64 (), svlsl_x (svptrue_b64 (), u, 46),
-			  0x8000000000000000);
-  /* Copy sign to scale.  */
-  scale = svreinterpret_f64 (
-      svadd_x (svptrue_b64 (), e, svreinterpret_u64 (scale)));
-  return special_case (scale, poly, n, ds);
+  /* Use a modifed version of the shift used for flooring, such that x/ln2 is
+    rounded to a multiple of 2^-6=1/64, shift = 1.5 * 2^52 * 2^-6 = 1.5 *
+    2^46.
+
+    n is not an integer but can be written as n = m + i/64, with i and m
+    integer, 0 <= i < 64 and m <= n.
+
+    Bits 5:0 of z will be null every time x/ln2 reaches a new integer value
+    (n=m, i=0), and is incremented every time z (or n) is incremented by 1/64.
+    FEXPA expects i in bits 5:0 of the input so it can be used as index into
+    FEXPA hardwired table T[i] = 2^(i/64) for i = 0:63, that will in turn
+    populate the mantissa of the output. Therefore, we use u=asuint(z) as
+    input to FEXPA.
+
+    We add 1023 to the modified shift value in order to set bits 16:6 of u to
+    1, such that once these bits are moved to the exponent of the output of
+    FEXPA, we get the exponent of 2^n right, i.e. we get 2^m.  */
+  svfloat64_t z = svmla_x (pg, sv_f64 (d->shift), x, d->inv_ln2);
+  svfloat64_t n = svsub_x (pg, z, d->shift);
+
+  svfloat64_t ln2_hi_lo = svld1rq (svptrue_b64 (), &d->ln2_hi);
+  svfloat64_t r = x;
+  r = svmls_lane (r, n, ln2_hi_lo, 0);
+  r = svmls_lane (r, n, ln2_hi_lo, 1);
+
+  svfloat64_t c13 = svld1rq (svptrue_b64 (), &d->c1);
+  svfloat64_t r2 = svmul_x (svptrue_b64 (), r, r);
+  svfloat64_t p01 = svmla_lane (sv_f64 (d->c0), r, c13, 0);
+  svfloat64_t p23 = svmla_lane (sv_f64 (d->c2), r, c13, 1);
+  svfloat64_t p04 = svmla_x (pg, p01, p23, r2);
+  svfloat64_t poly = svmla_x (pg, r, p04, r2);
+
+  svfloat64_t scale = svexpa (svreinterpret_u64 (z));
+
+  return svmla_x (pg, scale, scale, poly);
 }
 
-/* SVE exp algorithm. Maximum measured error is 1.01ulps:
-   SV_NAME_D1 (exp)(0x1.4619d7b04da41p+6) got 0x1.885d9acc41da7p+117
-					 want 0x1.885d9acc41da6p+117.  */
-svfloat64_t SV_NAME_D1 (exp) (svfloat64_t x, const svbool_t pg)
+static svfloat64_t NOINLINE
+special_case (svfloat64_t x, const svbool_t pg, const struct data *d)
 {
-  const struct data *d = ptr_barrier (&data);
+  /* Computes the offset and scale factor based on sign of the input.  */
+  svbool_t is_negative = svcmplt (pg, x, 0.0);
+  svfloat64_t offset = svneg_m (sv_f64 (53.0), is_negative, sv_f64 (53.0));
+  svint64_t scale_adjust = svneg_m (sv_s64 (53), is_negative, sv_s64 (53));
+
+  /* Bounds x between zero_bound and inf_bound, where exp(x) would return
+    0 or inf. By clamping x to these bounds, the behaviour of large values
+    is more predictable.  */
+  x = svmin_x (pg, svmax_x (pg, x, d->zero_bound), d->inf_bound);
 
-  /* Use a modifed version of the shift used for flooring, such that x/ln2 is
-     rounded to a multiple of 2^-6=1/64, shift = 1.5 * 2^52 * 2^-6 = 1.5 *
-     2^46.
-
-     n is not an integer but can be written as n = m + i/64, with i and m
-     integer, 0 <= i < 64 and m <= n.
-
-     Bits 5:0 of z will be null every time x/ln2 reaches a new integer value
-     (n=m, i=0), and is incremented every time z (or n) is incremented by 1/64.
-     FEXPA expects i in bits 5:0 of the input so it can be used as index into
-     FEXPA hardwired table T[i] = 2^(i/64) for i = 0:63, that will in turn
-     populate the mantissa of the output. Therefore, we use u=asuint(z) as
-     input to FEXPA.
-
-     We add 1023 to the modified shift value in order to set bits 16:6 of u to
-     1, such that once these bits are moved to the exponent of the output of
-     FEXPA, we get the exponent of 2^n right, i.e. we get 2^m.  */
   svfloat64_t z = svmla_x (pg, sv_f64 (d->shift), x, d->inv_ln2);
-  svuint64_t u = svreinterpret_u64 (z);
   svfloat64_t n = svsub_x (pg, z, d->shift);
-  svfloat64_t c13 = svld1rq (svptrue_b64 (), &d->c1);
-  /* r = x - n * ln2, r is in [-ln2/(2N), ln2/(2N)].  */
+
   svfloat64_t ln2_hi_lo = svld1rq (svptrue_b64 (), &d->ln2_hi);
   svfloat64_t r = x;
   r = svmls_lane (r, n, ln2_hi_lo, 0);
   r = svmls_lane (r, n, ln2_hi_lo, 1);
 
-  /* poly = exp(r) - 1 ~= r + C0 r^2 + C1 r^3 + C2 r^4 + C3 r^5.  */
+  svfloat64_t c13 = svld1rq (svptrue_b64 (), &d->c1);
   svfloat64_t r2 = svmul_x (svptrue_b64 (), r, r);
   svfloat64_t p01 = svmla_lane (sv_f64 (d->c0), r, c13, 0);
   svfloat64_t p23 = svmla_lane (sv_f64 (d->c2), r, c13, 1);
   svfloat64_t p04 = svmla_x (pg, p01, p23, r2);
   svfloat64_t poly = svmla_x (pg, r, p04, r2);
 
-  /* scale = 2^n, computed using FEXPA. FEXPA does not propagate NaNs, so for
-     consistent NaN handling we have to manually propagate them. This comes at
-     significant performance cost.  */
-  svfloat64_t scale = svexpa (u);
+  /* Computes scale with an offset, which returns:
+    scale = 2^(n - 53).  */
+  z = svsub_x (pg, z, offset);
+  svfloat64_t scale = svexpa (svreinterpret_u64 (z));
 
+  /* Reconstruct y as 2^(n - 53) * (1 + poly(r)).  */
+  svfloat64_t y = svmla_x (pg, scale, scale, poly);
+
+  /* Scale the result by 2^53:
+    2^n * (1 + poly(r)) = 2^53 * 2^(n - 53) * (1 + poly(r)).  */
+  return svscale_x (pg, y, scale_adjust);
+}
+
+/* Vector version of exp
+   The maximum observed error is 0.51 + 0.5 ULP.
+   _ZGVsMxv_exp (0x1.4619d7b04da41p+6)
+    got 0x1.885d9acc41da7p+117
+   want 0x1.885d9acc41da6p+117.  */
+svfloat64_t SV_NAME_D1 (exp) (svfloat64_t x, const svbool_t pg)
+{
+  const struct data *d = ptr_barrier (&data);
   svbool_t special = svacge (svptrue_b64 (), x, d->special_bound);
-  /* Assemble result as exp(x) = 2^n * exp(r).  If |x| > Thresh the
-     multiplication may overflow, so use special case routine.  */
   if (__glibc_unlikely (svptest_any (special, special)))
-    return special_exp (poly, scale, n, u, &d->special_data);
-
-  /* No special case.  */
-  return svmla_x (pg, scale, scale, poly);
+    return special_case (x, pg, d);
+  return exp_inline (x, pg, d);
 }
diff --git a/sysdeps/aarch64/fpu/sve/exp10.c b/sysdeps/aarch64/fpu/sve/exp10.c
index fd04bab44a..b8183aeda0 100644
--- a/sysdeps/aarch64/fpu/sve/exp10.c
+++ b/sysdeps/aarch64/fpu/sve/exp10.c
@@ -18,21 +18,22 @@
    <https://www.gnu.org/licenses/>.  */
 
 #include "sv_math.h"
-#include "sv_exp_special_inline.h"
 
 /* Value of |x| above which scale overflows without special treatment.
-   log10(2^1022 + 1/128) ~ 307.65.  */
-#define SpecialBound 0x1.33a7ae900b507p+8
+   log10(2^1022) ~ 307.65.  */
+#define SpecialBound 0x1.33a7146f72a41p+8
+
+/* Values of x over which exp10 overflows or underflows.  */
+#define InfBound 0x1.35p+8		/* 309.0.  */
+#define ZeroBound -0x1.439b746e36b53p+8 /* ~ -323.61.  */
 
 static const struct data
 {
-  struct sv_exp_special_data special_data;
   double log2_10_hi, log2_10_lo;
   double log10_2, c0;
   double c1, c3, c2, c4;
-  double shift, special_bound;
+  double shift, special_bound, inf_bound, zero_bound;
 } data = {
-  .special_data = SV_EXP_SPECIAL_DATA,
   /* Coefficients generated using Remez algorithm.
      rel error: 0x1.9fcb9b3p-60
      abs error: 0x1.a20d9598p-60 in [ -log10(2)/128, log10(2)/128 ]
@@ -48,49 +49,74 @@ static const struct data
   .log2_10_hi = 0x1.34413509f79ffp-2, /* log2(10).  */
   .log2_10_lo = -0x1.9dc1da994fd21p-59,
   .special_bound = SpecialBound,
+  .inf_bound = InfBound,
+  .zero_bound = ZeroBound,
 };
 
-static svfloat64_t NOINLINE
-special_exp (svfloat64_t scale, svfloat64_t poly, svfloat64_t n, svuint64_t u,
-	     const struct sv_exp_special_data *ds)
+static inline svfloat64_t
+exp10_inline (svfloat64_t x, const svbool_t pg, const struct data *d)
 {
-  /* FEXPA zeroes the sign bit, however the sign is meaningful to the
-  special case function so needs to be copied.
-  e = sign bit of u << 46.  */
-  svuint64_t e = svand_x (svptrue_b64 (), svlsl_x (svptrue_b64 (), u, 46),
-			  0x8000000000000000);
-  /* Copy sign to scale.  */
-  scale = svreinterpret_f64 (
-      svadd_x (svptrue_b64 (), e, svreinterpret_u64 (scale)));
-  return special_case (scale, poly, n, ds);
+  /* n is x/log10(2) rounded to the nearest multiple of 1/64.  */
+  svfloat64_t shift = sv_f64 (d->shift);
+  svfloat64_t log10_2_c0 = svld1rq (svptrue_b64 (), &d->log10_2);
+  svfloat64_t z = svmla_lane (shift, x, log10_2_c0, 0);
+  svfloat64_t n = svsub_x (pg, z, shift);
+
+  /* r = x - n*log10(2).  */
+  svfloat64_t log2_10 = svld1rq (svptrue_b64 (), &d->log2_10_hi);
+  svfloat64_t r = x;
+  r = svmls_lane (r, n, log2_10, 0);
+  r = svmls_lane (r, n, log2_10, 1);
+
+  svfloat64_t scale = svexpa (svreinterpret_u64 (z));
+  svfloat64_t c24 = svld1rq (svptrue_b64 (), &d->c2);
+
+  /* Approximate exp10(r) using polynomial.  */
+  svfloat64_t r2 = svmul_x (svptrue_b64 (), r, r);
+  svfloat64_t p12 = svmla_lane (sv_f64 (d->c1), r, c24, 0);
+  svfloat64_t p34 = svmla_lane (sv_f64 (d->c3), r, c24, 1);
+  svfloat64_t p14 = svmla_x (pg, p12, p34, r2);
+
+  svfloat64_t poly = svmla_x (pg, svmul_lane (r, log10_2_c0, 1), r2, p14);
+
+  return svmla_x (pg, scale, scale, poly);
 }
 
-/* Fast vector implementation of exp10 using FEXPA instruction.
-   Maximum measured error is 1.02 ulp.
-   SV_NAME_D1 (exp10)(-0x1.2862fec805e58p+2) got 0x1.885a89551d782p-16
-					    want 0x1.885a89551d781p-16.  */
-svfloat64_t SV_NAME_D1 (exp10) (svfloat64_t x, svbool_t pg)
+static svfloat64_t NOINLINE
+special_case (svfloat64_t x, const svbool_t pg, const struct data *d)
 {
-  const struct data *d = ptr_barrier (&data);
+  /* Computes the offset and scale factor based on sign of the input.  */
+  svbool_t is_negative = svcmplt (pg, x, 0.0);
+  svfloat64_t offset = svneg_m (sv_f64 (53.0), is_negative, sv_f64 (53.0));
+  svint64_t scale_adjust = svneg_m (sv_s64 (53), is_negative, sv_s64 (53));
+
+  /* Bounds x between zero_bound and inf_bound, where exp10(x) would return
+     0 or inf. By clamping x to these bounds, the behaviour of large values
+     is more predictable.  */
+  x = svmin_x (pg, svmax_x (pg, x, d->zero_bound), d->inf_bound);
 
-  /* n = round(x/(log10(2)/N)).  */
+  /* exp10(x) = 2^(n/N) * 10^r = 2^n * (1 + poly(r)),
+     with 1 + poly(r) in [1/sqrt(2), sqrt(2)] and
+     x = r + n * log10(2) / N, with r in [-log10(2)/2N, log10(2)/2N].  */
   svfloat64_t shift = sv_f64 (d->shift);
   svfloat64_t log10_2_c0 = svld1rq (svptrue_b64 (), &d->log10_2);
+
+  /* n is x/log10(2) rounded to the nearest multiple of 1/64.  */
   svfloat64_t z = svmla_lane (shift, x, log10_2_c0, 0);
   svfloat64_t n = svsub_x (pg, z, shift);
 
-  /* r = x - n*log10(2)/N.  */
+  /* r = x - n*log10(2).  */
   svfloat64_t log2_10 = svld1rq (svptrue_b64 (), &d->log2_10_hi);
   svfloat64_t r = x;
   r = svmls_lane (r, n, log2_10, 0);
   r = svmls_lane (r, n, log2_10, 1);
 
-  /* scale = 2^(n/N), computed using FEXPA. FEXPA does not propagate NaNs, so
-     for consistent NaN handling we have to manually propagate them. This
-     comes at significant performance cost.  */
-  svuint64_t u = svreinterpret_u64 (z);
-  svfloat64_t scale = svexpa (u);
+  /* Computes scale with an offset, which returns scale = 2^(n - 53).  */
+  z = svsub_x (pg, z, offset);
+  svfloat64_t scale = svexpa (svreinterpret_u64 (z));
+
   svfloat64_t c24 = svld1rq (svptrue_b64 (), &d->c2);
+
   /* Approximate exp10(r) using polynomial.  */
   svfloat64_t r2 = svmul_x (svptrue_b64 (), r, r);
   svfloat64_t p12 = svmla_lane (sv_f64 (d->c1), r, c24, 0);
@@ -99,12 +125,24 @@ svfloat64_t SV_NAME_D1 (exp10) (svfloat64_t x, svbool_t pg)
 
   svfloat64_t poly = svmla_x (pg, svmul_lane (r, log10_2_c0, 1), r2, p14);
 
-  svbool_t special = svacge (svptrue_b64 (), x, d->special_bound);
-  /* Assemble result as exp(x) = 2^n * exp(r).  If |x| > Thresh the
-     multiplication may overflow, so use special case routine.  */
-  if (__glibc_unlikely (svptest_any (special, special)))
-    return special_exp (scale, poly, n, u, &d->special_data);
+  /* Reconstruct y as 2^(n - 53) * (1 + poly(r)).  */
+  svfloat64_t y = svmla_x (pg, scale, scale, poly);
 
-  /* No special case.  */
-  return svmla_x (pg, scale, scale, poly);
+  /* Scale the result by 2^53:
+     2^n * (1 + poly(r)) = 2^53 * 2^(n - 53) * (1 + poly(r)).  */
+  return svscale_x (pg, y, scale_adjust);
+}
+
+/* Vector version of exp10
+   The maximum observed error is 0.52 + 0.5 ULP.
+   _ZGVsMxv_exp10(-0x1.2862fec805e58p+2)
+    got 0x1.885a89551d782p-16
+   want 0x1.885a89551d781p-16.  */
+svfloat64_t SV_NAME_D1 (exp10) (svfloat64_t x, svbool_t pg)
+{
+  const struct data *d = ptr_barrier (&data);
+  svbool_t special = svacgt (pg, x, d->special_bound);
+  if (__glibc_unlikely (svptest_any (special, special)))
+    return special_case (x, pg, d);
+  return exp10_inline (x, pg, d);
 }
diff --git a/sysdeps/aarch64/fpu/sve/exp10f.c b/sysdeps/aarch64/fpu/sve/exp10f.c
index 0c426797ff..a900176b39 100644
--- a/sysdeps/aarch64/fpu/sve/exp10f.c
+++ b/sysdeps/aarch64/fpu/sve/exp10f.c
@@ -23,7 +23,7 @@
    correctly by FEXPA.  */
 #define SpecialBound 0x1.2f702p+5 /* log10(2^126) ~ 37.93.  */
 
-/* Values of x over which exp overflows or underflows.  */
+/* Values of x over which exp10f overflows or underflows.  */
 #define InfBound 0x1.34ccccccccccdp+5 /* 38.6.  */
 #define ZeroBound -0x1.68ccccccccccdp+5 /* -45.1.  */
 
@@ -73,36 +73,64 @@ sv_exp10f_inline (svfloat32_t x, const svbool_t pg, const struct data *d)
   return svmla_x (pg, scale, scale, poly);
 }
 
+/* Special case to handle large magnitudes of x, where fexpa returns nan.
+   By applying a fixed offset to the value passed to fexpa,
+   and then reapplying a scaling factor afterwards, this is avoided.  */
 static svfloat32_t NOINLINE
-special_case (svfloat32_t x, svbool_t pg, svbool_t special,
-	      const struct data *d)
+special_case (svfloat32_t x, const svbool_t pg, const struct data *d)
 {
-  /* This deals with overflow and underflow in exponential for special case
-     lanes.  */
-  svbool_t is_inf = svcmpgt (pg, x, d->inf_bound);
-  svbool_t is_zero = svcmplt (pg, x, d->zero_bound);
+  /* Computes the offset and scale factor based on sign of the input.  */
+  svbool_t is_negative = svcmplt (pg, x, 0.0f);
+  svfloat32_t offset = svneg_m (sv_f32 (23.0f), is_negative, sv_f32 (23.0f));
+  svint32_t scale_adjust = svneg_m (sv_s32 (23), is_negative, sv_s32 (23));
 
-  /* The input `x` is further reduced (to `x/2`) to allow for accurate
-     approximation on the interval `x > SpecialBound ~ 0x1.2f702p+5`.  */
-  x = svmul_m (special, x, 0.5);
+  /* Bounds x between zero_bound and inf_bound, where exp10f(x) would return
+     0 or inf. By clamping x to these bounds, the behaviour of large values
+     is more predictable.  */
+  x = svmin_x (pg, svmax_x (pg, x, d->zero_bound), d->inf_bound);
 
-  /* Computes exp(x/2), and set lanes with underflow/overflow.  */
-  svfloat32_t half_exp = sv_exp10f_inline (x, svptrue_b32 (), d);
-  half_exp = svmul_m (special, half_exp, half_exp);
-  half_exp = svsel (is_inf, sv_f32 (INFINITY), half_exp);
+  /* exp10(x) = 2^(n/N) * 10^r = 2^n * (1 + poly(r)),
+     with poly(r) in [1/sqrt(2), sqrt(2)] and
+     x = r + n * log10(2) / N, with r in [-log10(2)/2N, log10(2)/2N].  */
+  svfloat32_t lane_consts = svld1rq (svptrue_b32 (), &d->log10_2);
+
+  /* n = round(x/(log10(2)/N)).  */
+  svfloat32_t shift = sv_f32 (d->shift);
+  svfloat32_t z = svmla_lane (shift, x, lane_consts, 0);
+  svfloat32_t n = svsub_x (pg, z, shift);
+
+  /* r = x - n*log10(2)/N.  */
+  svfloat32_t r = x;
+  r = svmls_lane (r, n, lane_consts, 1);
+  r = svmls_lane (r, n, lane_consts, 2);
+
+  /* Computes scale with an offset, which returns:
+     scale = 2^(n/N - 23).  */
+  z = svsub_x (pg, z, offset);
+  svfloat32_t scale = svexpa (svreinterpret_u32 (z));
+
+  /* poly(r) = exp(r) - 1 ~= r + 0.5 r^2.  */
+  svfloat32_t poly = svmla_lane (sv_f32 (d->c0), r, lane_consts, 3);
+  poly = svmul_x (pg, poly, r);
+
+  /* Reconstruct y as 2^(n/N - 23) * (1 + poly(r)).  */
+  svfloat32_t y = svmla_x (pg, scale, scale, poly);
 
-  return svsel (is_zero, sv_f32 (0), half_exp);
+  /* Scale the result by 2^23:
+     2^(n/N) * (1 + poly(r)) = 2^23 * 2^(n/N - 23) * (1 + poly(r)).  */
+  return svscale_x (pg, y, scale_adjust);
 }
 
-/* Single-precision SVE exp10f routine. Based on the FEXPA instruction.
-   Worst case error is 2.86 ULP +0.50 ULP.
-   _ZGVsMxv_exp10f (0x1.31b778p+5) got 0x1.ed399p+126
-				  want 0x1.ed398ap+126.  */
+/* Vector version of exp10f.
+   The maximum observed error is 0.59 + 0.5 ULP.
+   _ZGVsMxv_exp10f (-0x1.041162p-4)
+    got 0x1.ba5c6cp-1
+   want 0x1.ba5c6ap-1.  */
 svfloat32_t SV_NAME_F1 (exp10) (svfloat32_t x, const svbool_t pg)
 {
   const struct data *d = ptr_barrier (&data);
   svbool_t special = svacgt (pg, x, d->special_bound);
   if (__glibc_unlikely (svptest_any (special, special)))
-    return special_case (x, pg, special, d);
+    return special_case (x, svptrue_b32 (), d);
   return sv_exp10f_inline (x, svptrue_b32 (), d);
 }
diff --git a/sysdeps/aarch64/fpu/sve/exp2.c b/sysdeps/aarch64/fpu/sve/exp2.c
index a6f002f9b0..acc5726181 100644
--- a/sysdeps/aarch64/fpu/sve/exp2.c
+++ b/sysdeps/aarch64/fpu/sve/exp2.c
@@ -18,62 +18,73 @@
    <https://www.gnu.org/licenses/>.  */
 
 #include "sv_math.h"
-#include "sv_exp_special_inline.h"
 
 /* Value of |x| above which scale overflows without special treatment.
-   log2(2^(1022 + 1/128)) ~ 1022.00.  */
-#define SpecialBound 0x1.ff01p+9
+   log2(2^1022) = 1022.00.  */
+#define SpecialBound 0x1.ffp+9
 
-/* Value of n above which scale overflows even with special treatment.  */
-#define ScaleBound 1280
+/* Values of x which exp2 overflows or underflows.  */
+#define InfBound 0x1p10		 /* 1024.  */
+#define ZeroBound -0x1.0d008p+10 /* -1076.00781253.  */
 
 static const struct data
 {
   double c2, c4;
   double c0, c1, c3;
   double shift, special_bound;
-  struct sv_exp_special_data special_data;
+  double inf_bound, zero_bound;
 } data = {
-  .special_data = SV_EXP_SPECIAL_DATA,
   /* Coefficients are computed using Remez algorithm with
      minimisation of the absolute error.  */
-  .c0 = 0x1.62e42fefa39efp-1,
-  .c1 = 0x1.ebfbdff82a31bp-3,
-  .c2 = 0x1.c6b08d706c8a5p-5,
-  .c3 = 0x1.3b2ad2ff7d2f3p-7,
-  .c4 = 0x1.5d8761184beb3p-10,
-  .shift = 0x1.800000000ffc0p+46,
-  .special_bound = SpecialBound,
+  .c0 = 0x1.62e42fefa39efp-1,	 .c1 = 0x1.ebfbdff82a31bp-3,
+  .c2 = 0x1.c6b08d706c8a5p-5,	 .c3 = 0x1.3b2ad2ff7d2f3p-7,
+  .c4 = 0x1.5d8761184beb3p-10,	 .shift = 0x1.800000000ffc0p+46,
+  .special_bound = SpecialBound, .inf_bound = InfBound,
+  .zero_bound = ZeroBound,
 };
 
-static svfloat64_t NOINLINE
-special_exp (svfloat64_t poly, svfloat64_t scale, svfloat64_t n, svfloat64_t z,
-	     const struct sv_exp_special_data *ds)
+static svfloat64_t inline exp2_inline (svfloat64_t x, svbool_t pg,
+				       const struct data *d)
 {
-  /* FEXPA zeroes the sign bit, however the sign is meaningful to the
-  special case function so needs to be copied.
-  e = sign bit of u << 46.  */
-  svuint64_t u = svreinterpret_u64 (z);
-  svuint64_t e = svand_x (svptrue_b64 (), svlsl_x (svptrue_b64 (), u, 46),
-			  0x8000000000000000);
-  /* Copy sign to scale.  */
-  scale = svreinterpret_f64 (
-      svadd_x (svptrue_b64 (), e, svreinterpret_u64 (scale)));
-  return special_case (scale, poly, n, ds);
+  svfloat64_t z = svadd_x (svptrue_b64 (), x, d->shift);
+  svfloat64_t n = svsub_x (pg, z, d->shift);
+  svfloat64_t r = svsub_x (pg, x, n);
+
+  svfloat64_t scale = svexpa (svreinterpret_u64 (z));
+
+  svfloat64_t r2 = svmul_x (svptrue_b64 (), r, r);
+  svfloat64_t c24 = svld1rq (svptrue_b64 (), &d->c2);
+
+  /* Approximate exp2(r) using polynomial.  */
+  /* y = exp2(r) - 1 ~= r * (C0 + C1 r + C2 r^2 + C3 r^3 + C4 r^4).  */
+  svfloat64_t p12 = svmla_lane (sv_f64 (d->c1), r, c24, 0);
+  svfloat64_t p34 = svmla_lane (sv_f64 (d->c3), r, c24, 1);
+  svfloat64_t p = svmla_x (pg, p12, p34, r2);
+  p = svmad_x (pg, p, r, d->c0);
+  svfloat64_t poly = svmul_x (svptrue_b64 (), r, p);
+
+  return svmla_x (pg, scale, scale, poly);
 }
 
-/* Fast vector implementation of exp2.
-   Maximum measured error is 0.52 + 0.5 ulp.
-   _ZGVsMxv_exp2 (0x1.3b72ad5b701bfp-1) got 0x1.8861641b49e08p+0
-				       want 0x1.8861641b49e07p+0.  */
-svfloat64_t SV_NAME_D1 (exp2) (svfloat64_t x, svbool_t pg)
+static svfloat64_t NOINLINE
+special_case (svfloat64_t x, svbool_t pg, const struct data *d)
 {
-  const struct data *d = ptr_barrier (&data);
+  /* Computes the offset and scale factor based on sign of the input.  */
+  svbool_t is_negative = svcmplt (pg, x, 0.0);
+  svfloat64_t offset = svneg_m (sv_f64 (53.0), is_negative, sv_f64 (53.0));
+  svint64_t scale_adjust = svneg_m (sv_s64 (53), is_negative, sv_s64 (53));
+
+  /* Bounds x between zero_bound and inf_bound, where exp2(x) would return
+     0 or inf. By clamping x to these bounds, the behaviour of large values
+     is more predictable.  */
+  x = svmin_x (pg, svmax_x (pg, x, d->zero_bound), d->inf_bound);
 
   svfloat64_t z = svadd_x (svptrue_b64 (), x, d->shift);
   svfloat64_t n = svsub_x (pg, z, d->shift);
   svfloat64_t r = svsub_x (pg, x, n);
 
+  /* Computes scale with an offset, which returns scale = 2^(n - 53).  */
+  z = svsub_x (pg, z, offset);
   svfloat64_t scale = svexpa (svreinterpret_u64 (z));
 
   svfloat64_t r2 = svmul_x (svptrue_b64 (), r, r);
@@ -87,11 +98,24 @@ svfloat64_t SV_NAME_D1 (exp2) (svfloat64_t x, svbool_t pg)
   p = svmad_x (pg, p, r, d->c0);
   svfloat64_t poly = svmul_x (svptrue_b64 (), r, p);
 
+  /* Reconstruct y as 2^(n - 53) * (1 + poly(r)).  */
+  svfloat64_t y = svmla_x (pg, scale, scale, poly);
+
+  /* Scale the result by 2^53:
+    2^n * (1 + poly(r)) = 2^53 * 2^(n - 53) * (1 + poly(r)).  */
+  return svscale_x (pg, y, scale_adjust);
+}
+
+/* Vector version of exp2
+   The maximum observed error is 0.52 + 0.5 ULP.
+   _ZGVsMxv_exp2 (0x1.3b72ad5b701bfp-1)
+    got 0x1.8861641b49e08p+0
+   want 0x1.8861641b49e07p+0.  */
+svfloat64_t SV_NAME_D1 (exp2) (svfloat64_t x, svbool_t pg)
+{
+  const struct data *d = ptr_barrier (&data);
   svbool_t special = svacge (svptrue_b64 (), x, d->special_bound);
-  /* Assemble result as exp(x) = 2^n * exp(r).  If |x| > Thresh the
-     multiplication may overflow, so use special case routine.  */
   if (__glibc_unlikely (svptest_any (special, special)))
-    return special_exp (poly, scale, n, z, &d->special_data);
-
-  return svmla_x (pg, scale, scale, poly);
+    return special_case (x, pg, d);
+  return exp2_inline (x, pg, d);
 }
diff --git a/sysdeps/aarch64/fpu/sve/exp2f.c b/sysdeps/aarch64/fpu/sve/exp2f.c
index b71f472d44..b513e0775f 100644
--- a/sysdeps/aarch64/fpu/sve/exp2f.c
+++ b/sysdeps/aarch64/fpu/sve/exp2f.c
@@ -25,7 +25,7 @@
 
 /* Values of x over which exp overflows or underflows.  */
 #define InfBound 0x1.0p7f /* 128.0f.  */
-#define ZeroBound -0x1.2a8p7f /* -149.0f.  */
+#define ZeroBound -0x1.2cp7f /* -150.0f.  */
 
 static const struct data
 {
@@ -46,7 +46,8 @@ static const struct data
 static inline svfloat32_t
 sv_exp2f_inline (svfloat32_t x, const svbool_t pg, const struct data *d)
 {
-  /* exp2(x) = 2^n (1 + poly(r)), with 1 + poly(r) in [1/sqrt(2),sqrt(2)]
+  /* exp2(x) =
+      2^n (1 + r * poly(r)), with 1 + r * poly(r) in [1/sqrt(2),sqrt(2)]
      x = n + r, with r in [-1/2, 1/2].  */
   svfloat32_t z = svadd_x (svptrue_b32 (), x, d->shift);
   svfloat32_t n = svsub_x (svptrue_b32 (), z, d->shift);
@@ -60,36 +61,58 @@ sv_exp2f_inline (svfloat32_t x, const svbool_t pg, const struct data *d)
   return svmla_x (pg, scale, scale, poly);
 }
 
+/* Special case to handle large magnitudes of x, where fexpa returns nan.
+   By applying a fixed offset to the value passed to fexpa,
+   and then reapplying a scaling factor afterwards, this is avoided.  */
 static svfloat32_t NOINLINE
-special_case (svfloat32_t x, svbool_t pg, svbool_t special,
-	      const struct data *d)
+special_case (svfloat32_t x, const svbool_t pg, const struct data *d)
 {
-  /* This deals with overflow and underflow in exponential for special case
-     lanes.  */
-  svbool_t is_inf = svcmpgt (pg, x, d->inf_bound);
-  svbool_t is_zero = svcmplt (pg, x, d->zero_bound);
+  /* Computes the offset and scale factor based on sign of the input.  */
+  svbool_t is_negative = svcmplt (pg, x, 0.0f);
+  svfloat32_t offset = svneg_m (sv_f32 (23.0f), is_negative, sv_f32 (23.0f));
+  svint32_t scale_adjust = svneg_m (sv_s32 (23), is_negative, sv_s32 (23));
+
+  /* Bounds x between zero_bound and inf_bound, where exp2(x) would return
+     0 or inf. By clamping x to these bounds, the behaviour of large values
+     is more predictable.  */
+  x = svmin_x (pg, svmax_x (pg, x, d->zero_bound), d->inf_bound);
+
+  /* exp2(x) =
+      2^n (1 + r * poly(r)), with 1 + r * poly(r) in [1/sqrt(2),sqrt(2)]
+     x = n + r, with r in [-1/2, 1/2].  */
+  svfloat32_t z = svadd_x (svptrue_b32 (), x, d->shift);
+  svfloat32_t n = svsub_x (svptrue_b32 (), z, d->shift);
+
+  svfloat32_t r = svsub_x (svptrue_b32 (), x, n);
 
-  /* The input `x` is further reduced (to `x/2`) to allow for accurate
-     approximation on the interval `x > SpecialBound = 126.0`.  */
-  x = svmul_x (special, x, 0.5);
+  /* Computes scale with an offset, which returns:
+     scale = 2^(n/N - 23).  */
+  z = svsub_x (pg, z, offset);
+  svfloat32_t scale = svexpa (svreinterpret_u32 (z));
+
+  /* poly(r) = (exp2(r) - 1) / r = log(2) + r*log(2)^2/2.  */
+  svfloat32_t poly = svmla_x (pg, sv_f32 (d->c0), r, sv_f32 (d->c1));
+  poly = svmul_x (svptrue_b32 (), poly, r);
 
-  /* Computes exp(x/2), and set lanes with underflow/overflow.  */
-  svfloat32_t half_exp = sv_exp2f_inline (x, svptrue_b32 (), d);
-  half_exp = svmul_m (special, half_exp, half_exp);
-  half_exp = svsel (is_inf, sv_f32 (INFINITY), half_exp);
+  /* Reconstruct y as 2^(n - 23) * (1 + r * poly(r)).  */
+  svfloat32_t y = svmla_x (pg, scale, scale, poly);
 
-  return svsel (is_zero, sv_f32 (0), half_exp);
+  /* Scale the result by 2^23:
+     2^n * (1 + r * poly(r))
+     = 2^23 * 2^(n - 23) * (1 + r * poly(r)).  */
+  return svscale_x (pg, y, scale_adjust);
 }
 
-/* Single-precision SVE exp2f routine, based on the FEXPA instruction.
-   Worst case error is 2.87 +0.50 ULP.
-   _ZGVsMxv_exp2f (0x1.fbcb78p+6) got 0x1.ee1d32p+126
-				 want 0x1.ee1d2cp+126.  */
+/* Vector version of exp2f.
+   The maximum observed error is 0.59 + 0.5 ULP.
+   _ZGVsMxv_exp2f (-0x1.97fa44p-3)
+    got 0x1.bdf77cp-1
+   want 0x1.bdf77ap-1.  */
 svfloat32_t SV_NAME_F1 (exp2) (svfloat32_t x, const svbool_t pg)
 {
   const struct data *d = ptr_barrier (&data);
   svbool_t special = svacgt (pg, x, d->special_bound);
   if (__glibc_unlikely (svptest_any (special, special)))
-    return special_case (x, pg, special, d);
+    return special_case (x, svptrue_b32 (), d);
   return sv_exp2f_inline (x, pg, d);
 }
diff --git a/sysdeps/aarch64/fpu/sve/expf.c b/sysdeps/aarch64/fpu/sve/expf.c
index 9c552958ed..d5d6de9b68 100644
--- a/sysdeps/aarch64/fpu/sve/expf.c
+++ b/sysdeps/aarch64/fpu/sve/expf.c
@@ -21,7 +21,7 @@
 
 /* For x < -SpecialBound, the result is subnormal and not handled
    correctly by FEXPA.  */
-#define SpecialBound 0x1.5d5e2ap+6f /* ln(2^126) ~ 87.34.  */
+#define SpecialBound 0x1.5d589ep+6f /* ln(2^126) ~ 87.34.  */
 
 /* Values of x which exp overflows or underflows.  */
 #define InfBound 0x1.62e42fp6f /* ~ 88.72.  */
@@ -29,8 +29,8 @@
 
 static const struct data
 {
-  float ln2_hi, ln2_lo, c1, c3;
-  float shift, inv_ln2, c2;
+  float ln2_hi, ln2_lo, c1;
+  float shift, inv_ln2;
   float32_t special_bound, inf_bound, zero_bound;
 } data = {
   /* shift = 1.5*2^17 + 127.  */
@@ -41,8 +41,6 @@ static const struct data
   /* Coefficients generated using Remez algorithm with minimisation of relative
      error.  */
   .c1 = 0.5f,
-  .c2 = 0x1.55559p-3,
-  .c3 = 0x1.55555cp-5,
   .special_bound = SpecialBound,
   .inf_bound = InfBound,
   .zero_bound = ZeroBound,
@@ -74,64 +72,62 @@ expf_inline (svfloat32_t x, const svbool_t pg, const struct data *d)
   return svmla_x (pg, scale, scale, poly);
 }
 
-/* This is the version used if there is any special lane in the input vector.
-   The approximation needs to match that of the fast path.
-   To achieve this we assemble the same polynomial, ie `r + 0.5 * r^2`,
-   then we conditionally add an extra `c2 * r^3` term.  */
-static inline svfloat32_t
-expf_slow_inline (svfloat32_t x, const svbool_t special, const struct data *d)
+/* Special case to handle large magnitudes of x, where fexpa returns nan.
+   By applying a fixed offset to the value passed to fexpa,
+   and then reapplying a scaling factor afterwards, this is avoided.  */
+static svfloat32_t NOINLINE
+special_case (svfloat32_t x, const svbool_t pg, const struct data *d)
 {
+  /* Computes the offset and scale factor based on sign of the input.  */
+  svbool_t is_negative = svcmplt (pg, x, 0.0f);
+  svfloat32_t offset = svneg_m (sv_f32 (23.0f), is_negative, sv_f32 (23.0f));
+  svint32_t scale_adjust = svneg_m (sv_s32 (23), is_negative, sv_s32 (23));
+
+  /* Bounds x between zero_bound and inf_bound, where exp(x) would return
+     0 or inf. By clamping x to these bounds, the behaviour of large values
+     is more predictable.  */
+  x = svmin_x (pg, svmax_x (pg, x, d->zero_bound), d->inf_bound);
+
+  /* exp(x) = 2^n (1 + poly(r)), with 1 + poly(r) in [1/sqrt(2),sqrt(2)]
+    x = ln2*n + r, with r in [-ln2/2, ln2/2].  */
   svfloat32_t lane_constants = svld1rq (svptrue_b32 (), &d->ln2_hi);
 
-  svfloat32_t z = svmad_x (svptrue_b32 (), sv_f32 (d->inv_ln2), x, d->shift);
-  svfloat32_t n = svsub_x (svptrue_b32 (), z, d->shift);
+  /* n = round(x/(ln2/N)).  */
+  svfloat32_t z = svmad_x (pg, sv_f32 (d->inv_ln2), x, d->shift);
+  svfloat32_t n = svsub_x (pg, z, d->shift);
 
+  /* r = x - n*ln2/N.  */
   svfloat32_t r = x;
   r = svmls_lane (r, n, lane_constants, 0);
   r = svmls_lane (r, n, lane_constants, 1);
 
-  /* poly(r) = exp(r) - 1 ~= r + 0.5 r^2 + c2 r^3.  */
-  svfloat32_t r2 = svmul_x (svptrue_b32 (), r, r);
-  svfloat32_t r3 = svmul_x (svptrue_b32 (), r, r2);
-  svfloat32_t poly_slow = svmla_lane (sv_f32 (d->c2), r, lane_constants, 3);
-  svfloat32_t poly_fast = svmla_lane (r, r2, lane_constants, 2);
-  svfloat32_t poly = svmla_m (special, poly_fast, poly_slow, r3);
-
+  /* Computes scale with an offset, which returns:
+     scale = 2^(n/N - 23).  */
+  z = svsub_x (pg, z, offset);
   svfloat32_t scale = svexpa (svreinterpret_u32 (z));
 
-  return svmla_x (svptrue_b32 (), scale, scale, poly);
-}
-
-static svfloat32_t NOINLINE
-special_case (svfloat32_t x, svbool_t pg, svbool_t special,
-	      const struct data *d)
-{
-  /* This deals with overflow and underflow in exponential for special case
-     lanes.  */
-  svbool_t is_inf = svcmpgt (pg, x, d->inf_bound);
-  svbool_t is_zero = svcmplt (pg, x, d->zero_bound);
-
-  /* The input `x` is further reduced (to `x/2`) to allow for accurate
-     approximation on the interval `x > SpecialBound ~ 87.3`.  */
-  x = svmul_m (special, x, 0.5);
+  /* poly(r) = exp(r) - 1 ~= r + 0.5 r^2.  */
+  svfloat32_t r2 = svmul_x (svptrue_b32 (), r, r);
+  svfloat32_t poly = svmla_lane (r, r2, lane_constants, 2);
 
-  /* Computes exp(x/2), and set lanes with underflow/overflow.  */
-  svfloat32_t half_exp = expf_slow_inline (x, special, d);
-  half_exp = svmul_m (special, half_exp, half_exp);
-  half_exp = svsel (is_inf, sv_f32 (INFINITY), half_exp);
+  /* Reconstruct y as 2^(n/N - 23) * (1 + poly(r)).  */
+  svfloat32_t y = svmla_x (pg, scale, scale, poly);
 
-  return svsel (is_zero, sv_f32 (0), half_exp);
+  /* Scale the result by 2^23:
+     2^(n/N) * (1 + poly(r)) = 2^23 * 2^(n/N - 23) * (1 + poly(r)).  */
+  return svscale_x (pg, y, scale_adjust);
 }
 
-/* Optimised single-precision SVE exp function.
-   Worst-case error is 2.70 +0.50 ULP:
-   _ZGVsMxv_expf(0x1.5fec38p+6) got 0x1.e7831ep+126
-			       want 0x1.e78318p+126.  */
+/* Vector version of expf.
+   The maximum observed error is 0.88 + 0.5 ULP.
+   _ZGVsMxv_expf (-0x1.bba276p-6)
+    got 0x1.f25288p-1
+   want 0x1.f2528ap-1.  */
 svfloat32_t SV_NAME_F1 (exp) (svfloat32_t x, const svbool_t pg)
 {
   const struct data *d = ptr_barrier (&data);
   svbool_t special = svacgt (pg, x, d->special_bound);
   if (__glibc_unlikely (svptest_any (special, special)))
-    return special_case (x, pg, special, d);
+    return special_case (x, svptrue_b32 (), d);
   return expf_inline (x, svptrue_b32 (), d);
 }
-- 
2.43.0



More information about the Libc-alpha mailing list