[PATCH v3 11/12] math: Shorten the sinf/cosf/sincosf fast path
Adhemerval Zanella
adhemerval.zanella@linaro.org
Wed Sep 23 21:50:52 GMT 2026
Converting the rounded value with (int) id gives the same low bits and
lets gcc on aarch64 fold the rounding and the conversion into a
single fcvtns of the scaled argument (and it s cvttsd2si on x86_64).
The polynomial coefficients and the sin/cos tables are also grouped i
one hidden object, so a function materializes one base address instead of
one per table.
On aarch64 with gcc 16 the fast path goes from 59 to 54 instructions
for sinf, 55 to 50 for cosf and 63 to 58 for sincosf, with 4 instead
of 8 register moves and 3 instead of 5 adrp.
Measured against the previous commit (gcc 16.2.1) with the benchtests
inputs that preceded the workload rewrite of this series (random
[-2pi, 2pi], large [-64, 64], huge [-1024, 1024]), geomean of the
reciprocal-throughput and latency, negative is faster:
sinf cosf sincosf
x86_64 +1.3% -0.4% +0.3%
x86_64-fma +0.9% -2.6% -1.1%
aarch64 -0.1% +0.4% -6.0%
power10 +2.3% +2.9% +0.2%
loongarch -0.6% -1.4% -0.5%
The aarch64 sincosf gain comes from the large and huge inputs (-8% and
-9% reciprocal-throughput).
Checked on x86_64-linux-gnu, aarch64-linux-gnu, powerpc64le-linux-gnu
and loongarch64-linux-gnuf64.
---
sysdeps/ieee754/flt-32/s_sincosf_common.h | 4 +-
sysdeps/ieee754/flt-32/s_sincosf_data.c | 76 ++++++++++++-----------
sysdeps/ieee754/flt-32/s_sincosf_data.h | 21 ++++---
3 files changed, 53 insertions(+), 48 deletions(-)
diff --git a/sysdeps/ieee754/flt-32/s_sincosf_common.h b/sysdeps/ieee754/flt-32/s_sincosf_common.h
index a15a0afe806..c7f385d07eb 100644
--- a/sysdeps/ieee754/flt-32/s_sincosf_common.h
+++ b/sysdeps/ieee754/flt-32/s_sincosf_common.h
@@ -34,7 +34,7 @@ static inline double
rltl0 (double x, int *q)
{
double idh = 0x1.45f306dc9c883p+2 * x, id = roundeven_finite (idh);
- *q = asuint64 (0x1.8p52 + id);
+ *q = (int) id;
return idh - id;
}
@@ -44,7 +44,7 @@ rltl (float z, int *q)
double x = z;
double idl = -0x1.b1bbead603d8bp-29 * x, idh = 0x1.45f306ep+2 * x,
id = roundeven_finite (idh);
- *q = asuint64 (0x1.8p52 + id);
+ *q = (int) id;
return (idh - id) + idl;
}
diff --git a/sysdeps/ieee754/flt-32/s_sincosf_data.c b/sysdeps/ieee754/flt-32/s_sincosf_data.c
index a86640c4048..d248db8c08f 100644
--- a/sysdeps/ieee754/flt-32/s_sincosf_data.c
+++ b/sysdeps/ieee754/flt-32/s_sincosf_data.c
@@ -33,44 +33,46 @@ const uint64_t __sinf_ipi[] =
0xa2f9836e4e441529
};
-const double __sinf_b[] =
+const sincosf_tables_t __sincosf_tables =
{
- 0x1.3bd3cc9be45dcp-6, -0x1.03c1f081b0833p-14, 0x1.55d3c6fc9ac1fp-24,
- -0x1.e1d3ff281b40dp-35
-};
-const double __sinf_a[] =
-{
- 0x1.921fb54442d17p-3, -0x1.4abbce6256a39p-10, 0x1.466bc5a518c16p-19,
- -0x1.32bdc61074ff6p-29
-};
-const double __sinf_tb[] =
-{
- 0x0p+0, 0x1.8f8b83c69a60bp-3, 0x1.87de2a6aea963p-2,
- 0x1.1c73b39ae68c8p-1, 0x1.6a09e667f3bcdp-1, 0x1.a9b66290ea1a3p-1,
- 0x1.d906bcf328d46p-1, 0x1.f6297cff75cbp-1, 0x1p+0,
- 0x1.f6297cff75cbp-1, 0x1.d906bcf328d46p-1, 0x1.a9b66290ea1a3p-1,
- 0x1.6a09e667f3bcdp-1, 0x1.1c73b39ae68c8p-1, 0x1.87de2a6aea963p-2,
- 0x1.8f8b83c69a60bp-3, 0x0p+0, -0x1.8f8b83c69a60bp-3,
- -0x1.87de2a6aea963p-2, -0x1.1c73b39ae68c8p-1, -0x1.6a09e667f3bcdp-1,
- -0x1.a9b66290ea1a3p-1, -0x1.d906bcf328d46p-1, -0x1.f6297cff75cbp-1,
- -0x1p+0, -0x1.f6297cff75cbp-1, -0x1.d906bcf328d46p-1,
- -0x1.a9b66290ea1a3p-1, -0x1.6a09e667f3bcdp-1, -0x1.1c73b39ae68c8p-1,
- -0x1.87de2a6aea963p-2, -0x1.8f8b83c69a60bp-3
-};
-
-const double __cosf_tb[] =
-{
- 0x1p+0, 0x1.f6297cff75cbp-1, 0x1.d906bcf328d46p-1,
- 0x1.a9b66290ea1a3p-1, 0x1.6a09e667f3bcdp-1, 0x1.1c73b39ae68c8p-1,
- 0x1.87de2a6aea963p-2, 0x1.8f8b83c69a60bp-3, 0x0p+0,
- -0x1.8f8b83c69a60bp-3, -0x1.87de2a6aea963p-2, -0x1.1c73b39ae68c8p-1,
- -0x1.6a09e667f3bcdp-1, -0x1.a9b66290ea1a3p-1, -0x1.d906bcf328d46p-1,
- -0x1.f6297cff75cbp-1, -0x1p+0, -0x1.f6297cff75cbp-1,
- -0x1.d906bcf328d46p-1, -0x1.a9b66290ea1a3p-1, -0x1.6a09e667f3bcdp-1,
- -0x1.1c73b39ae68c8p-1, -0x1.87de2a6aea963p-2, -0x1.8f8b83c69a60bp-3,
- 0x0p+0, 0x1.8f8b83c69a60bp-3, 0x1.87de2a6aea963p-2,
- 0x1.1c73b39ae68c8p-1, 0x1.6a09e667f3bcdp-1, 0x1.a9b66290ea1a3p-1,
- 0x1.d906bcf328d46p-1, 0x1.f6297cff75cbp-1
+ .b =
+ {
+ 0x1.3bd3cc9be45dcp-6, -0x1.03c1f081b0833p-14, 0x1.55d3c6fc9ac1fp-24,
+ -0x1.e1d3ff281b40dp-35
+ },
+ .a =
+ {
+ 0x1.921fb54442d17p-3, -0x1.4abbce6256a39p-10, 0x1.466bc5a518c16p-19,
+ -0x1.32bdc61074ff6p-29
+ },
+ .tb =
+ {
+ 0x0p+0, 0x1.8f8b83c69a60bp-3, 0x1.87de2a6aea963p-2,
+ 0x1.1c73b39ae68c8p-1, 0x1.6a09e667f3bcdp-1, 0x1.a9b66290ea1a3p-1,
+ 0x1.d906bcf328d46p-1, 0x1.f6297cff75cbp-1, 0x1p+0,
+ 0x1.f6297cff75cbp-1, 0x1.d906bcf328d46p-1, 0x1.a9b66290ea1a3p-1,
+ 0x1.6a09e667f3bcdp-1, 0x1.1c73b39ae68c8p-1, 0x1.87de2a6aea963p-2,
+ 0x1.8f8b83c69a60bp-3, 0x0p+0, -0x1.8f8b83c69a60bp-3,
+ -0x1.87de2a6aea963p-2, -0x1.1c73b39ae68c8p-1, -0x1.6a09e667f3bcdp-1,
+ -0x1.a9b66290ea1a3p-1, -0x1.d906bcf328d46p-1, -0x1.f6297cff75cbp-1,
+ -0x1p+0, -0x1.f6297cff75cbp-1, -0x1.d906bcf328d46p-1,
+ -0x1.a9b66290ea1a3p-1, -0x1.6a09e667f3bcdp-1, -0x1.1c73b39ae68c8p-1,
+ -0x1.87de2a6aea963p-2, -0x1.8f8b83c69a60bp-3
+ },
+ .tb_cosf =
+ {
+ 0x1p+0, 0x1.f6297cff75cbp-1, 0x1.d906bcf328d46p-1,
+ 0x1.a9b66290ea1a3p-1, 0x1.6a09e667f3bcdp-1, 0x1.1c73b39ae68c8p-1,
+ 0x1.87de2a6aea963p-2, 0x1.8f8b83c69a60bp-3, 0x0p+0,
+ -0x1.8f8b83c69a60bp-3, -0x1.87de2a6aea963p-2, -0x1.1c73b39ae68c8p-1,
+ -0x1.6a09e667f3bcdp-1, -0x1.a9b66290ea1a3p-1, -0x1.d906bcf328d46p-1,
+ -0x1.f6297cff75cbp-1, -0x1p+0, -0x1.f6297cff75cbp-1,
+ -0x1.d906bcf328d46p-1, -0x1.a9b66290ea1a3p-1, -0x1.6a09e667f3bcdp-1,
+ -0x1.1c73b39ae68c8p-1, -0x1.87de2a6aea963p-2, -0x1.8f8b83c69a60bp-3,
+ 0x0p+0, 0x1.8f8b83c69a60bp-3, 0x1.87de2a6aea963p-2,
+ 0x1.1c73b39ae68c8p-1, 0x1.6a09e667f3bcdp-1, 0x1.a9b66290ea1a3p-1,
+ 0x1.d906bcf328d46p-1, 0x1.f6297cff75cbp-1
+ },
};
const sincosf_database_t __sinf_st[] =
diff --git a/sysdeps/ieee754/flt-32/s_sincosf_data.h b/sysdeps/ieee754/flt-32/s_sincosf_data.h
index ca7164de52b..7d9a7028f39 100644
--- a/sysdeps/ieee754/flt-32/s_sincosf_data.h
+++ b/sysdeps/ieee754/flt-32/s_sincosf_data.h
@@ -24,15 +24,18 @@
extern const uint64_t __sinf_ipi[] attribute_hidden;
#define IPI __sinf_ipi
-extern const double __sinf_b[] attribute_hidden;
-#define B __sinf_b
-extern const double __sinf_a[] attribute_hidden;
-#define A __sinf_a
-extern const double __sinf_tb[] attribute_hidden;
-#define TB __sinf_tb
-
-extern const double __cosf_tb[] attribute_hidden;
-#define TB_COSF __cosf_tb
+typedef struct
+{
+ double b[4];
+ double a[4];
+ double tb[32];
+ double tb_cosf[32];
+} sincosf_tables_t;
+extern const sincosf_tables_t __sincosf_tables attribute_hidden;
+#define B __sincosf_tables.b
+#define A __sincosf_tables.a
+#define TB __sincosf_tables.tb
+#define TB_COSF __sincosf_tables.tb_cosf
typedef struct
{
--
2.53.0
More information about the Libc-alpha
mailing list