[PATCH 1/2] x86: Enable ERMS memcpy/memmove for AMD Zen5/Zen6 servers

Sajan Karumanchi sajan.karumanchi@gmail.com
Fri Nov 13 12:43:35 GMT 2026


Microcode optimizations on AMD Zen5+ CPUs allow REP MOVSB to outperform
traditional vectorized copy routines across medium-to-large buffer sizes.

Introduce a new CPU feature flag, Prefer_Server_ERMS, to identify Zen5+
server-class models and enable the ERMS code path for memcpy and memmove
with thresholds tuned from real-world benchmarks.

Performance improvements:
    - Range: [16KB, NT threshold]
    - Improvement: 11% to 25%

Forward-overlapping memmove cases that regress with REP MOVSB on AMD
Zen5+ are avoided via the newly introduced Avoid_Overlap_REP_MOVSB flag.

Co-authored-by: Sundari Swathy Meena H <sundari.swathymeenah@amd.com>
Reviewed-by: Ganesh Gopalasubramanian <Ganesh.Gopalasubramanian@amd.com>
---
 sysdeps/x86/cacheinfo.h                       |  7 +++++
 sysdeps/x86/cpu-features.c                    | 26 +++++++++++++----
 sysdeps/x86/dl-cacheinfo.h                    | 29 +++++++++++++++----
 ...cpu-features-preferred_feature_index_1.def |  2 ++
 sysdeps/x86/sysdep.h                          |  2 ++
 .../multiarch/memmove-vec-unaligned-erms.S    | 13 +++++++++
 6 files changed, 68 insertions(+), 11 deletions(-)

diff --git a/sysdeps/x86/cacheinfo.h b/sysdeps/x86/cacheinfo.h
index 5c1b1a88c7..9e2c1e782a 100644
--- a/sysdeps/x86/cacheinfo.h
+++ b/sysdeps/x86/cacheinfo.h
@@ -90,5 +90,12 @@ init_cacheinfo (void)
   if (CPU_FEATURES_ARCH_P (cpu_features, Avoid_Short_Distance_REP_MOVSB))
     __x86_string_control
       |= X86_STRING_CONTROL_AVOID_SHORT_DISTANCE_REP_MOVSB;
+
+  /* Avoid REP MOVSB for forward-overlapping memmove, where it is
+     much slower than the vector copy (about 2x regression on
+     bench-memmove-large).  */
+  if (CPU_FEATURES_ARCH_P (cpu_features, Avoid_Overlap_REP_MOVSB))
+    __x86_string_control
+      |= X86_STRING_CONTROL_AVOID_OVERLAP_REP_MOVSB;
 }
 #endif
diff --git a/sysdeps/x86/cpu-features.c b/sysdeps/x86/cpu-features.c
index 3ca8ae866c..bdd8ddb9ab 100644
--- a/sysdeps/x86/cpu-features.c
+++ b/sysdeps/x86/cpu-features.c
@@ -1051,11 +1051,27 @@ disable_tsx:
       cpu_features->preferred[index_arch_Avoid_Non_Temporal_Memset]
 	  &= ~bit_arch_Avoid_Non_Temporal_Memset;
 
-      /* Prefer EVEX512 string/memory variants on AMD Zen5 (Family 0x1A)
-         when AVX-512 is usable. */
-      if (family == 0x1A && CPU_FEATURE_USABLE_P (cpu_features, AVX512F))
-          cpu_features->preferred[index_arch_Prefer_EVEX512]
-            |= bit_arch_Prefer_EVEX512;
+      if (family == 0x1A)
+	{
+	  /* Prefer EVEX512 string/memory variants on AMD Zen5 (Family 0x1A)
+	     when AVX-512 is usable.  */
+	  if (CPU_FEATURE_USABLE_P (cpu_features, AVX512F))
+	    cpu_features->preferred[index_arch_Prefer_EVEX512]
+	      |= bit_arch_Prefer_EVEX512;
+
+	  cpu_features->preferred[index_arch_Avoid_Overlap_REP_MOVSB]
+	    |= bit_arch_Avoid_Overlap_REP_MOVSB;
+
+	  /* Zen5+ server-class models.  Newer models will be retuned as
+	     needed after their official release.  */
+	  if (model <= 0x1F
+	      || (model >= 0x50 && model <= 0x5F)
+	      || (model >= 0x90 && model <= 0x9B)
+	      || (model >= 0xA0 && model <= 0xAB)
+	      || (model >= 0xC0 && model <= 0xCF))
+	    cpu_features->preferred[index_arch_Prefer_Server_ERMS]
+	      |= bit_arch_Prefer_Server_ERMS;
+	}
 
       if (CPU_FEATURE_USABLE_P (cpu_features, AVX))
 	{
diff --git a/sysdeps/x86/dl-cacheinfo.h b/sysdeps/x86/dl-cacheinfo.h
index 2d6b979215..3fd7c77bc1 100644
--- a/sysdeps/x86/dl-cacheinfo.h
+++ b/sysdeps/x86/dl-cacheinfo.h
@@ -1244,12 +1244,6 @@ dl_init_cacheinfo (struct cpu_features *cpu_features)
   if (CPU_FEATURE_USABLE_P (cpu_features, FSRM))
     rep_movsb_threshold = 2112;
 
-  /* For AMD CPUs that support ERMS (Zen3+), REP MOVSB is in a lot of
-     cases slower than the vectorized path (and for some alignments,
-     it is really slow, check BZ #30994).  */
-  if (cpu_features->basic.kind == arch_kind_amd)
-    rep_movsb_threshold = non_temporal_threshold;
-
   /* The default threshold to use Enhanced REP STOSB.  */
   unsigned long int rep_stosb_threshold = 2048;
 
@@ -1297,6 +1291,29 @@ dl_init_cacheinfo (struct cpu_features *cpu_features)
       && tunable_size <= maximum_non_temporal_threshold)
     memset_non_temporal_threshold = tunable_size;
 
+  if (cpu_features->basic.kind == arch_kind_amd)
+    {
+      if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_Server_ERMS))
+        {
+          /* Setting erms:rep-movsb lower bound threshold to 16KB for
+             Zen5 & Zen6 server models.  */
+          rep_movsb_threshold = 16 * 1024;
+        }
+      else
+        {
+          /* For AMD CPUs that support ERMS (Zen3 and Zen4), REP MOVSB is in
+             many cases slower than the vectorized path, and for some
+             alignments it is considerably slower (see BZ #30994).  Assign
+             the non-temporal threshold to rep_movsb_threshold *after* the
+             x86_non_temporal_threshold Tunable check, so that the ERMS code
+             path is not accidentally enabled when the tuned value is higher
+             than the default x86_non_temporal_threshold.  */
+          rep_movsb_threshold = non_temporal_threshold;
+        }
+    }
+
+  /* Now apply user tunables, which override the architecture-specific
+     defaults.  */
   tunable_size = TUNABLE_GET (x86_rep_movsb_threshold, long int, NULL);
   if (tunable_size > minimum_rep_movsb_threshold)
     rep_movsb_threshold = tunable_size;
diff --git a/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def b/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def
index 915f2d4e8c..8ed6389347 100644
--- a/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def
+++ b/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def
@@ -37,3 +37,5 @@ BIT (Avoid_Non_Temporal_Memset)
 BIT (Avoid_STOSB)
 BIT (Prefer_EVEX512)
 BIT (Prefer_Page_Unrolled_Large_Copy)
+BIT (Prefer_Server_ERMS)
+BIT (Avoid_Overlap_REP_MOVSB)
diff --git a/sysdeps/x86/sysdep.h b/sysdeps/x86/sysdep.h
index 41b040d51b..da2754df51 100644
--- a/sysdeps/x86/sysdep.h
+++ b/sysdeps/x86/sysdep.h
@@ -144,6 +144,8 @@
 
 /* Avoid short distance REP MOVSB.  */
 #define X86_STRING_CONTROL_AVOID_SHORT_DISTANCE_REP_MOVSB	(1 << 0)
+/* Avoid REP MOVSB for forward-overlapping memmove.  */
+#define X86_STRING_CONTROL_AVOID_OVERLAP_REP_MOVSB		(1 << 1)
 
 #ifdef	__ASSEMBLER__
 
diff --git a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
index fe700fa189..ae60d0cb58 100644
--- a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
+++ b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
@@ -551,9 +551,22 @@ L(movsb):
 	/* Go to backwards temporal copy if overlap no matter what as
 	   backward REP MOVSB is slow and we don't want to use NT stores if
 	   there is overlap.  */
+	leaq	-1(%rcx, %rdx), %r8
 	cmpq	%rdx, %rcx
 	/* L(more_8x_vec_backward_check_nop) checks for src == dst.  */
 	jb	L(more_8x_vec_backward_check_nop)
+	/* Detect forward overlap using XOR sign-change test.
+	   r8 = (dst - src) + len - 1 was computed in parallel with cmpq.
+	   If (dst - src) and r8 have different signs, there is forward
+	   overlap (src > dst with gap < len).  */
+	xorq	%rcx, %r8
+	jns	L(movsb_erms)
+	/* Forward overlap detected.  If the avoid-overlap control flag
+	   is set, avoid rep movsb for overlapping memmove.  Otherwise,
+	   fall through to the short-distance check.  */
+	testb	$X86_STRING_CONTROL_AVOID_OVERLAP_REP_MOVSB, __x86_string_control(%rip)
+	jnz	L(more_8x_vec_forward)
+L(movsb_erms):
 # if ALIGN_MOVSB
 	/* Save dest for storing aligning VECs later.  */
 	movq	%rdi, %r8
-- 
2.34.1



More information about the Libc-alpha mailing list