[PATCH 1/2] x86: Enable ERMS memcpy/memmove for AMD Zen5/Zen6 servers
Sajan Karumanchi
sajan.karumanchi@gmail.com
Fri Nov 13 12:43:35 GMT 2026
Microcode optimizations on AMD Zen5+ CPUs allow REP MOVSB to outperform
traditional vectorized copy routines across medium-to-large buffer sizes.
Introduce a new CPU feature flag, Prefer_Server_ERMS, to identify Zen5+
server-class models and enable the ERMS code path for memcpy and memmove
with thresholds tuned from real-world benchmarks.
Performance improvements:
- Range: [16KB, NT threshold]
- Improvement: 11% to 25%
Forward-overlapping memmove cases that regress with REP MOVSB on AMD
Zen5+ are avoided via the newly introduced Avoid_Overlap_REP_MOVSB flag.
Co-authored-by: Sundari Swathy Meena H <sundari.swathymeenah@amd.com>
Reviewed-by: Ganesh Gopalasubramanian <Ganesh.Gopalasubramanian@amd.com>
---
sysdeps/x86/cacheinfo.h | 7 +++++
sysdeps/x86/cpu-features.c | 26 +++++++++++++----
sysdeps/x86/dl-cacheinfo.h | 29 +++++++++++++++----
...cpu-features-preferred_feature_index_1.def | 2 ++
sysdeps/x86/sysdep.h | 2 ++
.../multiarch/memmove-vec-unaligned-erms.S | 13 +++++++++
6 files changed, 68 insertions(+), 11 deletions(-)
diff --git a/sysdeps/x86/cacheinfo.h b/sysdeps/x86/cacheinfo.h
index 5c1b1a88c7..9e2c1e782a 100644
--- a/sysdeps/x86/cacheinfo.h
+++ b/sysdeps/x86/cacheinfo.h
@@ -90,5 +90,12 @@ init_cacheinfo (void)
if (CPU_FEATURES_ARCH_P (cpu_features, Avoid_Short_Distance_REP_MOVSB))
__x86_string_control
|= X86_STRING_CONTROL_AVOID_SHORT_DISTANCE_REP_MOVSB;
+
+ /* Avoid REP MOVSB for forward-overlapping memmove, where it is
+ much slower than the vector copy (about 2x regression on
+ bench-memmove-large). */
+ if (CPU_FEATURES_ARCH_P (cpu_features, Avoid_Overlap_REP_MOVSB))
+ __x86_string_control
+ |= X86_STRING_CONTROL_AVOID_OVERLAP_REP_MOVSB;
}
#endif
diff --git a/sysdeps/x86/cpu-features.c b/sysdeps/x86/cpu-features.c
index 3ca8ae866c..bdd8ddb9ab 100644
--- a/sysdeps/x86/cpu-features.c
+++ b/sysdeps/x86/cpu-features.c
@@ -1051,11 +1051,27 @@ disable_tsx:
cpu_features->preferred[index_arch_Avoid_Non_Temporal_Memset]
&= ~bit_arch_Avoid_Non_Temporal_Memset;
- /* Prefer EVEX512 string/memory variants on AMD Zen5 (Family 0x1A)
- when AVX-512 is usable. */
- if (family == 0x1A && CPU_FEATURE_USABLE_P (cpu_features, AVX512F))
- cpu_features->preferred[index_arch_Prefer_EVEX512]
- |= bit_arch_Prefer_EVEX512;
+ if (family == 0x1A)
+ {
+ /* Prefer EVEX512 string/memory variants on AMD Zen5 (Family 0x1A)
+ when AVX-512 is usable. */
+ if (CPU_FEATURE_USABLE_P (cpu_features, AVX512F))
+ cpu_features->preferred[index_arch_Prefer_EVEX512]
+ |= bit_arch_Prefer_EVEX512;
+
+ cpu_features->preferred[index_arch_Avoid_Overlap_REP_MOVSB]
+ |= bit_arch_Avoid_Overlap_REP_MOVSB;
+
+ /* Zen5+ server-class models. Newer models will be retuned as
+ needed after their official release. */
+ if (model <= 0x1F
+ || (model >= 0x50 && model <= 0x5F)
+ || (model >= 0x90 && model <= 0x9B)
+ || (model >= 0xA0 && model <= 0xAB)
+ || (model >= 0xC0 && model <= 0xCF))
+ cpu_features->preferred[index_arch_Prefer_Server_ERMS]
+ |= bit_arch_Prefer_Server_ERMS;
+ }
if (CPU_FEATURE_USABLE_P (cpu_features, AVX))
{
diff --git a/sysdeps/x86/dl-cacheinfo.h b/sysdeps/x86/dl-cacheinfo.h
index 2d6b979215..3fd7c77bc1 100644
--- a/sysdeps/x86/dl-cacheinfo.h
+++ b/sysdeps/x86/dl-cacheinfo.h
@@ -1244,12 +1244,6 @@ dl_init_cacheinfo (struct cpu_features *cpu_features)
if (CPU_FEATURE_USABLE_P (cpu_features, FSRM))
rep_movsb_threshold = 2112;
- /* For AMD CPUs that support ERMS (Zen3+), REP MOVSB is in a lot of
- cases slower than the vectorized path (and for some alignments,
- it is really slow, check BZ #30994). */
- if (cpu_features->basic.kind == arch_kind_amd)
- rep_movsb_threshold = non_temporal_threshold;
-
/* The default threshold to use Enhanced REP STOSB. */
unsigned long int rep_stosb_threshold = 2048;
@@ -1297,6 +1291,29 @@ dl_init_cacheinfo (struct cpu_features *cpu_features)
&& tunable_size <= maximum_non_temporal_threshold)
memset_non_temporal_threshold = tunable_size;
+ if (cpu_features->basic.kind == arch_kind_amd)
+ {
+ if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_Server_ERMS))
+ {
+ /* Setting erms:rep-movsb lower bound threshold to 16KB for
+ Zen5 & Zen6 server models. */
+ rep_movsb_threshold = 16 * 1024;
+ }
+ else
+ {
+ /* For AMD CPUs that support ERMS (Zen3 and Zen4), REP MOVSB is in
+ many cases slower than the vectorized path, and for some
+ alignments it is considerably slower (see BZ #30994). Assign
+ the non-temporal threshold to rep_movsb_threshold *after* the
+ x86_non_temporal_threshold Tunable check, so that the ERMS code
+ path is not accidentally enabled when the tuned value is higher
+ than the default x86_non_temporal_threshold. */
+ rep_movsb_threshold = non_temporal_threshold;
+ }
+ }
+
+ /* Now apply user tunables, which override the architecture-specific
+ defaults. */
tunable_size = TUNABLE_GET (x86_rep_movsb_threshold, long int, NULL);
if (tunable_size > minimum_rep_movsb_threshold)
rep_movsb_threshold = tunable_size;
diff --git a/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def b/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def
index 915f2d4e8c..8ed6389347 100644
--- a/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def
+++ b/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def
@@ -37,3 +37,5 @@ BIT (Avoid_Non_Temporal_Memset)
BIT (Avoid_STOSB)
BIT (Prefer_EVEX512)
BIT (Prefer_Page_Unrolled_Large_Copy)
+BIT (Prefer_Server_ERMS)
+BIT (Avoid_Overlap_REP_MOVSB)
diff --git a/sysdeps/x86/sysdep.h b/sysdeps/x86/sysdep.h
index 41b040d51b..da2754df51 100644
--- a/sysdeps/x86/sysdep.h
+++ b/sysdeps/x86/sysdep.h
@@ -144,6 +144,8 @@
/* Avoid short distance REP MOVSB. */
#define X86_STRING_CONTROL_AVOID_SHORT_DISTANCE_REP_MOVSB (1 << 0)
+/* Avoid REP MOVSB for forward-overlapping memmove. */
+#define X86_STRING_CONTROL_AVOID_OVERLAP_REP_MOVSB (1 << 1)
#ifdef __ASSEMBLER__
diff --git a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
index fe700fa189..ae60d0cb58 100644
--- a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
+++ b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
@@ -551,9 +551,22 @@ L(movsb):
/* Go to backwards temporal copy if overlap no matter what as
backward REP MOVSB is slow and we don't want to use NT stores if
there is overlap. */
+ leaq -1(%rcx, %rdx), %r8
cmpq %rdx, %rcx
/* L(more_8x_vec_backward_check_nop) checks for src == dst. */
jb L(more_8x_vec_backward_check_nop)
+ /* Detect forward overlap using XOR sign-change test.
+ r8 = (dst - src) + len - 1 was computed in parallel with cmpq.
+ If (dst - src) and r8 have different signs, there is forward
+ overlap (src > dst with gap < len). */
+ xorq %rcx, %r8
+ jns L(movsb_erms)
+ /* Forward overlap detected. If the avoid-overlap control flag
+ is set, avoid rep movsb for overlapping memmove. Otherwise,
+ fall through to the short-distance check. */
+ testb $X86_STRING_CONTROL_AVOID_OVERLAP_REP_MOVSB, __x86_string_control(%rip)
+ jnz L(more_8x_vec_forward)
+L(movsb_erms):
# if ALIGN_MOVSB
/* Save dest for storing aligning VECs later. */
movq %rdi, %r8
--
2.34.1
More information about the Libc-alpha
mailing list