[PATCH v5 4/4] x86/string: Make page-unrolled memmove default for Intel processors
Noah Goldstein
goldstein.w.n@gmail.com
Thu Nov 27 08:43:16 GMT 2025
Continue using page-unrolled large memmove implementation for all
Intel processors to stay consistent with how it has been since glibc
2.34.
So far the page-unrolled implementation has been benchmarked on
HSW-SPR, with it yielding 5-15% performance improvements on
HSW-ICX. On SPR it appears to be roughly equal, and it is not known
how it compares on newer hardware (although Sunil has some benchmarks
that suggest it may still yield performance improvements).
Since as of now, we only have positive or roughly neutral benchmark
results on Intel hardware, it seems fair to continue using this
implementation as the default. It would be prudent, however, to
continue benchmarking the page-unrolled vs standard implementation for
future hardware to ensure that is remains performant.
---
sysdeps/x86/cpu-features.c | 11 +++
sysdeps/x86_64/multiarch/Makefile | 2 +
sysdeps/x86_64/multiarch/ifunc-impl-list.c | 72 +++++++++++++++++--
sysdeps/x86_64/multiarch/ifunc-memmove.h | 38 +++++++---
...move-avx512-unaligned-erms-page-unrolled.S | 24 +++++++
...emmove-sse2-unaligned-erms-page-unrolled.S | 24 +++++++
.../multiarch/memmove-sse2-unaligned-erms.S | 2 +-
7 files changed, 158 insertions(+), 15 deletions(-)
create mode 100644 sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms-page-unrolled.S
create mode 100644 sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms-page-unrolled.S
diff --git a/sysdeps/x86/cpu-features.c b/sysdeps/x86/cpu-features.c
index 36803aa53f..13029c7834 100644
--- a/sysdeps/x86/cpu-features.c
+++ b/sysdeps/x86/cpu-features.c
@@ -896,6 +896,17 @@ disable_tsx:
/* Bigcore/Default Tuning. */
default:
default_tuning:
+ /* Enable page unrolled large implementation to remain consistent
+ with glibc 2.34 and earlier. Thus far all benchmarks on Intel
+ hardware suggest the large implementation is equal to or more
+ performant than the standard large memmove implementation.
+ However, since the page-unrolled implementation is less
+ standard/obvious, it may not receive as much optimization
+ attention from hardware implementors, so we should continue to
+ benchmark new hardware to ensure this is the right decision. */
+ cpu_features->preferred[index_arch_Prefer_Page_Unrolled_Large_Copy]
+ |= bit_arch_Prefer_Page_Unrolled_Large_Copy;
+
/* Unknown Intel processors. Assuming this is one of Core
i3/i5/i7 processors if AVX is available. */
if (!CPU_FEATURES_CPU_P (cpu_features, AVX))
diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile
index 381eaef455..c4573f27df 100644
--- a/sysdeps/x86_64/multiarch/Makefile
+++ b/sysdeps/x86_64/multiarch/Makefile
@@ -21,10 +21,12 @@ sysdep_routines += \
memmove-avx-unaligned-erms-rtm \
memmove-avx512-no-vzeroupper \
memmove-avx512-unaligned-erms \
+ memmove-avx512-unaligned-erms-page-unrolled \
memmove-erms \
memmove-evex-unaligned-erms \
memmove-evex-unaligned-erms-page-unrolled \
memmove-sse2-unaligned-erms \
+ memmove-sse2-unaligned-erms-page-unrolled \
memmove-ssse3 \
memrchr-avx2 \
memrchr-avx2-rtm \
diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
index f9add65d24..26278da8d4 100644
--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c
+++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
@@ -127,9 +127,15 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
X86_IFUNC_IMPL_ADD_V4 (array, i, __memmove_chk,
CPU_FEATURE_USABLE (AVX512VL),
__memmove_chk_avx512_unaligned)
+ X86_IFUNC_IMPL_ADD_V4 (array, i, __memmove_chk,
+ CPU_FEATURE_USABLE (AVX512VL),
+ __memmove_chk_avx512_unaligned_page_unrolled)
X86_IFUNC_IMPL_ADD_V4 (array, i, __memmove_chk,
CPU_FEATURE_USABLE (AVX512VL),
__memmove_chk_avx512_unaligned_erms)
+ X86_IFUNC_IMPL_ADD_V4 (array, i, __memmove_chk,
+ CPU_FEATURE_USABLE (AVX512VL),
+ __memmove_chk_avx512_unaligned_erms_page_unrolled)
X86_IFUNC_IMPL_ADD_V4 (array, i, __memmove_chk,
CPU_FEATURE_USABLE (AVX512VL),
__memmove_chk_evex_unaligned)
@@ -181,7 +187,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
X86_IFUNC_IMPL_ADD_V2 (array, i, __memmove_chk, 1,
__memmove_chk_sse2_unaligned)
X86_IFUNC_IMPL_ADD_V2 (array, i, __memmove_chk, 1,
- __memmove_chk_sse2_unaligned_erms))
+ __memmove_chk_sse2_unaligned_page_unrolled)
+ X86_IFUNC_IMPL_ADD_V2 (array, i, __memmove_chk, 1,
+ __memmove_chk_sse2_unaligned_erms)
+ X86_IFUNC_IMPL_ADD_V2 (array, i, __memmove_chk, 1,
+ __memmove_chk_sse2_unaligned_erms_page_unrolled))
#endif
/* Support sysdeps/x86_64/multiarch/memmove.c. */
@@ -194,9 +204,15 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
X86_IFUNC_IMPL_ADD_V4 (array, i, memmove,
CPU_FEATURE_USABLE (AVX512VL),
__memmove_avx512_unaligned)
+ X86_IFUNC_IMPL_ADD_V4 (array, i, memmove,
+ CPU_FEATURE_USABLE (AVX512VL),
+ __memmove_avx512_unaligned_page_unrolled)
X86_IFUNC_IMPL_ADD_V4 (array, i, memmove,
CPU_FEATURE_USABLE (AVX512VL),
__memmove_avx512_unaligned_erms)
+ X86_IFUNC_IMPL_ADD_V4 (array, i, memmove,
+ CPU_FEATURE_USABLE (AVX512VL),
+ __memmove_avx512_unaligned_erms_page_unrolled)
X86_IFUNC_IMPL_ADD_V4 (array, i, memmove,
CPU_FEATURE_USABLE (AVX512VL),
__memmove_evex_unaligned)
@@ -248,7 +264,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
X86_IFUNC_IMPL_ADD_V2 (array, i, memmove, 1,
__memmove_sse2_unaligned)
X86_IFUNC_IMPL_ADD_V2 (array, i, memmove, 1,
- __memmove_sse2_unaligned_erms))
+ __memmove_sse2_unaligned_page_unrolled)
+ X86_IFUNC_IMPL_ADD_V2 (array, i, memmove, 1,
+ __memmove_sse2_unaligned_erms)
+ X86_IFUNC_IMPL_ADD_V2 (array, i, memmove, 1,
+ __memmove_sse2_unaligned_erms_page_unrolled))
/* Support sysdeps/x86_64/multiarch/memrchr.c. */
IFUNC_IMPL (i, name, memrchr,
@@ -1174,9 +1194,15 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
X86_IFUNC_IMPL_ADD_V4 (array, i, __memcpy_chk,
CPU_FEATURE_USABLE (AVX512VL),
__memcpy_chk_avx512_unaligned)
+ X86_IFUNC_IMPL_ADD_V4 (array, i, __memcpy_chk,
+ CPU_FEATURE_USABLE (AVX512VL),
+ __memcpy_chk_avx512_unaligned_page_unrolled)
X86_IFUNC_IMPL_ADD_V4 (array, i, __memcpy_chk,
CPU_FEATURE_USABLE (AVX512VL),
__memcpy_chk_avx512_unaligned_erms)
+ X86_IFUNC_IMPL_ADD_V4 (array, i, __memcpy_chk,
+ CPU_FEATURE_USABLE (AVX512VL),
+ __memcpy_chk_avx512_unaligned_erms_page_unrolled)
X86_IFUNC_IMPL_ADD_V4 (array, i, __memcpy_chk,
CPU_FEATURE_USABLE (AVX512VL),
__memcpy_chk_evex_unaligned)
@@ -1228,7 +1254,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
X86_IFUNC_IMPL_ADD_V2 (array, i, __memcpy_chk, 1,
__memcpy_chk_sse2_unaligned)
X86_IFUNC_IMPL_ADD_V2 (array, i, __memcpy_chk, 1,
- __memcpy_chk_sse2_unaligned_erms))
+ __memcpy_chk_sse2_unaligned_page_unrolled)
+ X86_IFUNC_IMPL_ADD_V2 (array, i, __memcpy_chk, 1,
+ __memcpy_chk_sse2_unaligned_erms)
+ X86_IFUNC_IMPL_ADD_V2 (array, i, __memcpy_chk, 1,
+ __memcpy_chk_sse2_unaligned_erms_page_unrolled))
#endif
/* Support sysdeps/x86_64/multiarch/memcpy.c. */
@@ -1241,9 +1271,15 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
X86_IFUNC_IMPL_ADD_V4 (array, i, memcpy,
CPU_FEATURE_USABLE (AVX512VL),
__memcpy_avx512_unaligned)
+ X86_IFUNC_IMPL_ADD_V4 (array, i, memcpy,
+ CPU_FEATURE_USABLE (AVX512VL),
+ __memcpy_avx512_unaligned_page_unrolled)
X86_IFUNC_IMPL_ADD_V4 (array, i, memcpy,
CPU_FEATURE_USABLE (AVX512VL),
__memcpy_avx512_unaligned_erms)
+ X86_IFUNC_IMPL_ADD_V4 (array, i, memcpy,
+ CPU_FEATURE_USABLE (AVX512VL),
+ __memcpy_avx512_unaligned_erms_page_unrolled)
X86_IFUNC_IMPL_ADD_V4 (array, i, memcpy,
CPU_FEATURE_USABLE (AVX512VL),
__memcpy_evex_unaligned)
@@ -1295,7 +1331,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
X86_IFUNC_IMPL_ADD_V2 (array, i, memcpy, 1,
__memcpy_sse2_unaligned)
X86_IFUNC_IMPL_ADD_V2 (array, i, memcpy, 1,
- __memcpy_sse2_unaligned_erms))
+ __memcpy_sse2_unaligned_page_unrolled)
+ X86_IFUNC_IMPL_ADD_V2 (array, i, memcpy, 1,
+ __memcpy_sse2_unaligned_erms)
+ X86_IFUNC_IMPL_ADD_V2 (array, i, memcpy, 1,
+ __memcpy_sse2_unaligned_erms_page_unrolled))
#ifdef SHARED
/* Support sysdeps/x86_64/multiarch/mempcpy_chk.c. */
@@ -1308,9 +1348,15 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
X86_IFUNC_IMPL_ADD_V4 (array, i, __mempcpy_chk,
CPU_FEATURE_USABLE (AVX512VL),
__mempcpy_chk_avx512_unaligned)
+ X86_IFUNC_IMPL_ADD_V4 (array, i, __mempcpy_chk,
+ CPU_FEATURE_USABLE (AVX512VL),
+ __mempcpy_chk_avx512_unaligned_page_unrolled)
X86_IFUNC_IMPL_ADD_V4 (array, i, __mempcpy_chk,
CPU_FEATURE_USABLE (AVX512VL),
__mempcpy_chk_avx512_unaligned_erms)
+ X86_IFUNC_IMPL_ADD_V4 (array, i, __mempcpy_chk,
+ CPU_FEATURE_USABLE (AVX512VL),
+ __mempcpy_chk_avx512_unaligned_erms_page_unrolled)
X86_IFUNC_IMPL_ADD_V4 (array, i, __mempcpy_chk,
CPU_FEATURE_USABLE (AVX512VL),
__mempcpy_chk_evex_unaligned)
@@ -1362,7 +1408,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
X86_IFUNC_IMPL_ADD_V2 (array, i, __mempcpy_chk, 1,
__mempcpy_chk_sse2_unaligned)
X86_IFUNC_IMPL_ADD_V2 (array, i, __mempcpy_chk, 1,
- __mempcpy_chk_sse2_unaligned_erms))
+ __mempcpy_chk_sse2_unaligned_page_unrolled)
+ X86_IFUNC_IMPL_ADD_V2 (array, i, __mempcpy_chk, 1,
+ __mempcpy_chk_sse2_unaligned_erms)
+ X86_IFUNC_IMPL_ADD_V2 (array, i, __mempcpy_chk, 1,
+ __mempcpy_chk_sse2_unaligned_erms_page_unrolled))
#endif
/* Support sysdeps/x86_64/multiarch/mempcpy.c. */
@@ -1375,9 +1425,15 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
X86_IFUNC_IMPL_ADD_V4 (array, i, mempcpy,
CPU_FEATURE_USABLE (AVX512VL),
__mempcpy_avx512_unaligned)
+ X86_IFUNC_IMPL_ADD_V4 (array, i, mempcpy,
+ CPU_FEATURE_USABLE (AVX512VL),
+ __mempcpy_avx512_unaligned_page_unrolled)
X86_IFUNC_IMPL_ADD_V4 (array, i, mempcpy,
CPU_FEATURE_USABLE (AVX512VL),
__mempcpy_avx512_unaligned_erms)
+ X86_IFUNC_IMPL_ADD_V4 (array, i, mempcpy,
+ CPU_FEATURE_USABLE (AVX512VL),
+ __mempcpy_avx512_unaligned_erms_page_unrolled)
X86_IFUNC_IMPL_ADD_V4 (array, i, mempcpy,
CPU_FEATURE_USABLE (AVX512VL),
__mempcpy_evex_unaligned)
@@ -1429,7 +1485,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
X86_IFUNC_IMPL_ADD_V2 (array, i, mempcpy, 1,
__mempcpy_sse2_unaligned)
X86_IFUNC_IMPL_ADD_V2 (array, i, mempcpy, 1,
- __mempcpy_sse2_unaligned_erms))
+ __mempcpy_sse2_unaligned_page_unrolled)
+ X86_IFUNC_IMPL_ADD_V2 (array, i, mempcpy, 1,
+ __mempcpy_sse2_unaligned_erms)
+ X86_IFUNC_IMPL_ADD_V2 (array, i, mempcpy, 1,
+ __mempcpy_sse2_unaligned_erms_page_unrolled))
/* Support sysdeps/x86_64/multiarch/strncmp.c. */
IFUNC_IMPL (i, name, strncmp,
diff --git a/sysdeps/x86_64/multiarch/ifunc-memmove.h b/sysdeps/x86_64/multiarch/ifunc-memmove.h
index 6d5df8a9eb..dc03269d8f 100644
--- a/sysdeps/x86_64/multiarch/ifunc-memmove.h
+++ b/sysdeps/x86_64/multiarch/ifunc-memmove.h
@@ -23,10 +23,14 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (erms) attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_unaligned)
attribute_hidden;
-extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_unaligned_erms)
- attribute_hidden;
-extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_no_vzeroupper)
- attribute_hidden;
+extern __typeof (REDIRECT_NAME)
+ OPTIMIZE (avx512_unaligned_page_unrolled) attribute_hidden;
+extern __typeof (REDIRECT_NAME)
+ OPTIMIZE (avx512_unaligned_erms) attribute_hidden;
+extern __typeof (REDIRECT_NAME)
+ OPTIMIZE (avx512_unaligned_erms_page_unrolled) attribute_hidden;
+extern __typeof (REDIRECT_NAME)
+ OPTIMIZE (avx512_no_vzeroupper) attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_unaligned) attribute_hidden;
extern __typeof (REDIRECT_NAME)
@@ -54,8 +58,12 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned)
attribute_hidden;
-extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned_erms)
- attribute_hidden;
+extern __typeof (REDIRECT_NAME)
+ OPTIMIZE (sse2_unaligned_page_unrolled) attribute_hidden;
+extern __typeof (REDIRECT_NAME)
+ OPTIMIZE (sse2_unaligned_erms) attribute_hidden;
+extern __typeof (REDIRECT_NAME)
+ OPTIMIZE (sse2_unaligned_erms_page_unrolled) attribute_hidden;
static inline void *
IFUNC_SELECTOR (void)
@@ -72,8 +80,16 @@ IFUNC_SELECTOR (void)
if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512VL))
{
if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
- return OPTIMIZE (avx512_unaligned_erms);
+ {
+ if (CPU_FEATURES_ARCH_P (cpu_features,
+ Prefer_Page_Unrolled_Large_Copy))
+ return OPTIMIZE (avx512_unaligned_erms_page_unrolled);
+ return OPTIMIZE (avx512_unaligned_erms);
+ }
+ if (CPU_FEATURES_ARCH_P (cpu_features,
+ Prefer_Page_Unrolled_Large_Copy))
+ return OPTIMIZE (avx512_unaligned_page_unrolled);
return OPTIMIZE (avx512_unaligned);
}
@@ -140,7 +156,13 @@ IFUNC_SELECTOR (void)
}
if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
- return OPTIMIZE (sse2_unaligned_erms);
+ {
+ if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_Page_Unrolled_Large_Copy))
+ return OPTIMIZE (sse2_unaligned_erms_page_unrolled);
+ return OPTIMIZE (sse2_unaligned_erms);
+ }
+ if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_Page_Unrolled_Large_Copy))
+ return OPTIMIZE (sse2_unaligned_page_unrolled);
return OPTIMIZE (sse2_unaligned);
}
diff --git a/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms-page-unrolled.S b/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms-page-unrolled.S
new file mode 100644
index 0000000000..a7dee420b4
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms-page-unrolled.S
@@ -0,0 +1,24 @@
+/* Memmove w/ AVX512 and Page Unrolled Large Copy
+ Copyright (C) 2025 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+
+#ifndef MEMMOVE_SYMBOL
+# define MEMMOVE_SYMBOL(p,s) p##_avx512_##s##_page_unrolled
+#endif
+#define MEMMOVE_VEC_LARGE_IMPL "memmove-vec-large-page-unrolled.S"
+#include "memmove-avx512-unaligned-erms.S"
diff --git a/sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms-page-unrolled.S b/sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms-page-unrolled.S
new file mode 100644
index 0000000000..9ecd223e4e
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms-page-unrolled.S
@@ -0,0 +1,24 @@
+/* Memmove w/ SSE2 and Page Unrolled Large Copy
+ Copyright (C) 2025 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#ifndef MEMMOVE_SYMBOL
+# define MEMMOVE_SYMBOL(p,s) p##_sse2_##s##_page_unrolled
+#endif
+#define MEMMOVE_VEC_LARGE_IMPL "memmove-vec-large-page-unrolled.S"
+#define PAGE_UNROLLED 1
+#include "memmove-sse2-unaligned-erms.S"
diff --git a/sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms.S
index aeaa3bd2f0..c941d62279 100644
--- a/sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms.S
+++ b/sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms.S
@@ -32,7 +32,7 @@
# include "multiarch/memmove-vec-unaligned-erms.S"
-# if MINIMUM_X86_ISA_LEVEL <= 2
+# if MINIMUM_X86_ISA_LEVEL <= 2 && !(defined PAGE_UNROLLED)
# include "memmove-shlib-compat.h"
# endif
#endif
--
2.43.0
More information about the Libc-alpha
mailing list