[PATCH] x86_64: Prefer EVEX512 code-path on AMD Zen5 CPUs
Sajan Karumanchi
sajan.karumanchi@gmail.com
Wed Feb 18 18:09:07 GMT 2026
Introduced a synthetic architecture preference flag (Prefer_EVEX512)
and enabled it for AMD Zen5 (CPUID Family 0x1A) when AVX-512F, AVX-512VL,
and AVX-512BW are supported.
This flag ensures that IFUNC dispatch selects the 512-bit EVEX variants
on Zen5, leveraging its native 512-bit AVX-512 execution for higher
throughput. EVEX (256-bit) implementations are chosen only when this
flag is not set, while EVEX512 entries remain intact.
Affected functions:
- strchr
- strlen
- strnlen
- strrchr
- strchrnul
- memchr
Benchmarks conducted on AMD Zen5 hardware demonstrate significant
performance improvements across all affected functions:
Function Baseline Patched Avg Avg Avg Max
Variant Variant Baseline Patched Change Improve
(ns) (ns) % %
------------+----------+----------+-----------+----------+--------+--------
STRCHR evex evex512 16.408 12.293 25.08% 37.69%
STRLEN evex evex512 16.862 11.436 32.18% 56.74%
STRNLEN evex evex512 18.493 11.762 36.40% 64.40%
STRRCHR evex evex512 15.154 10.874 28.24% 44.38%
STRCHRNUL evex evex512 16.464 12.605 23.44% 45.56%
MEMCHR evex evex512 9.984 8.268 17.19% 39.99%
diff --git a/sysdeps/x86/cpu-features.c b/sysdeps/x86/cpu-features.c
index ccfef55652..882e28743a 100644
--- a/sysdeps/x86/cpu-features.c
+++ b/sysdeps/x86/cpu-features.c
@@ -1009,6 +1009,17 @@ disable_tsx:
cpu_features->preferred[index_arch_Avoid_Non_Temporal_Memset]
&= ~bit_arch_Avoid_Non_Temporal_Memset;
+ /* Prefer EVEX512 string/memory variants on AMD Zen5 (Family 0x1A)
+ when AVX-512F/VL/BW are usable. */
+ if (family == 0x1A
+ && CPU_FEATURE_USABLE_P (cpu_features, AVX512F)
+ && CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
+ && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW))
+ {
+ cpu_features->preferred[index_arch_Prefer_EVEX512]
+ |= bit_arch_Prefer_EVEX512;
+ }
+
if (CPU_FEATURE_USABLE_P (cpu_features, AVX))
{
/* Since the FMA4 bit is in CPUID_INDEX_80000001 and
diff --git a/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def b/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def
index 8cab2ae248..74acb3fde1 100644
--- a/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def
+++ b/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def
@@ -35,3 +35,4 @@ BIT (Prefer_FSRM)
BIT (Avoid_Short_Distance_REP_MOVSB)
BIT (Avoid_Non_Temporal_Memset)
BIT (Avoid_STOSB)
+BIT (Prefer_EVEX512)
diff --git a/sysdeps/x86_64/multiarch/ifunc-evex512.h b/sysdeps/x86_64/multiarch/ifunc-evex512.h
new file mode 100644
index 0000000000..54ea6f9ae3
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/ifunc-evex512.h
@@ -0,0 +1,60 @@
+/* Common definition for ifunc selection optimized with EVEX512.
+ All versions must be listed in ifunc-impl-list.c.
+ Copyright (C) 2026 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <init-arch.h>
+
+extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (evex512) attribute_hidden;
+
+extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden;
+
+extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden;
+
+static inline void *
+IFUNC_SELECTOR (void)
+{
+ const struct cpu_features *cpu_features = __get_cpu_features ();
+
+ if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX2)
+ && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, BMI1)
+ && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, BMI2)
+ && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, LZCNT)
+ && X86_ISA_CPU_FEATURES_ARCH_P (cpu_features,
+ AVX_Fast_Unaligned_Load, ))
+ {
+ if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
+ && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512BW))
+ {
+ if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_EVEX512))
+ return OPTIMIZE (evex512);
+
+ return OPTIMIZE (evex);
+ }
+
+ if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
+ return OPTIMIZE (avx2_rtm);
+
+ if (X86_ISA_CPU_FEATURES_ARCH_P (cpu_features,
+ Prefer_No_VZEROUPPER, !))
+ return OPTIMIZE (avx2);
+ }
+
+ return OPTIMIZE (sse2);
+}
diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
index 40d5a8819a..a721ecce60 100644
--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c
+++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
@@ -58,8 +58,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
/* Support sysdeps/x86_64/multiarch/memchr.c. */
IFUNC_IMPL (i, name, memchr,
+ /* EVEX only when Prefer_EVEX512 is NOT set. */
X86_IFUNC_IMPL_ADD_V4 (array, i, memchr,
- (CPU_FEATURE_USABLE (AVX512VL)
+ (!CPU_FEATURES_ARCH_P (__get_cpu_features (),
+ Prefer_EVEX512)
+ && CPU_FEATURE_USABLE (AVX512VL)
&& CPU_FEATURE_USABLE (AVX512BW)
&& CPU_FEATURE_USABLE (BMI2)),
__memchr_evex)
@@ -369,8 +372,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
/* Support sysdeps/x86_64/multiarch/strlen.c. */
IFUNC_IMPL (i, name, strlen,
+ /* EVEX only when Prefer_EVEX512 is NOT set. */
X86_IFUNC_IMPL_ADD_V4 (array, i, strlen,
- (CPU_FEATURE_USABLE (AVX512VL)
+ (!CPU_FEATURES_ARCH_P (__get_cpu_features (),
+ Prefer_EVEX512)
+ && CPU_FEATURE_USABLE (AVX512VL)
&& CPU_FEATURE_USABLE (AVX512BW)
&& CPU_FEATURE_USABLE (BMI2)),
__strlen_evex)
@@ -396,8 +402,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
/* Support sysdeps/x86_64/multiarch/strnlen.c. */
IFUNC_IMPL (i, name, strnlen,
+ /* EVEX only when Prefer_EVEX512 is NOT set. */
X86_IFUNC_IMPL_ADD_V4 (array, i, strnlen,
- (CPU_FEATURE_USABLE (AVX512VL)
+ (!CPU_FEATURES_ARCH_P (__get_cpu_features (),
+ Prefer_EVEX512)
+ && CPU_FEATURE_USABLE (AVX512VL)
&& CPU_FEATURE_USABLE (AVX512BW)
&& CPU_FEATURE_USABLE (BMI2)),
__strnlen_evex)
@@ -539,8 +548,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
/* Support sysdeps/x86_64/multiarch/strchr.c. */
IFUNC_IMPL (i, name, strchr,
+ /* EVEX only when Prefer_EVEX512 is NOT set. */
X86_IFUNC_IMPL_ADD_V4 (array, i, strchr,
- (CPU_FEATURE_USABLE (AVX512VL)
+ (!CPU_FEATURES_ARCH_P (__get_cpu_features (),
+ Prefer_EVEX512)
+ && CPU_FEATURE_USABLE (AVX512VL)
&& CPU_FEATURE_USABLE (AVX512BW)
&& CPU_FEATURE_USABLE (BMI2)),
__strchr_evex)
@@ -568,8 +580,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
/* Support sysdeps/x86_64/multiarch/strchrnul.c. */
IFUNC_IMPL (i, name, strchrnul,
+ /* EVEX only when Prefer_EVEX512 is NOT set. */
X86_IFUNC_IMPL_ADD_V4 (array, i, strchrnul,
- (CPU_FEATURE_USABLE (AVX512VL)
+ (!CPU_FEATURES_ARCH_P (__get_cpu_features (),
+ Prefer_EVEX512)
+ && CPU_FEATURE_USABLE (AVX512VL)
&& CPU_FEATURE_USABLE (AVX512BW)
&& CPU_FEATURE_USABLE (BMI2)),
__strchrnul_evex)
@@ -594,8 +609,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
/* Support sysdeps/x86_64/multiarch/strrchr.c. */
IFUNC_IMPL (i, name, strrchr,
+ /* EVEX only when Prefer_EVEX512 is NOT set. */
X86_IFUNC_IMPL_ADD_V4 (array, i, strrchr,
- (CPU_FEATURE_USABLE (AVX512VL)
+ (!CPU_FEATURES_ARCH_P (__get_cpu_features (),
+ Prefer_EVEX512)
+ && CPU_FEATURE_USABLE (AVX512VL)
&& CPU_FEATURE_USABLE (AVX512BW)
&& CPU_FEATURE_USABLE (BMI1)
&& CPU_FEATURE_USABLE (BMI2)),
diff --git a/sysdeps/x86_64/multiarch/memchr.c b/sysdeps/x86_64/multiarch/memchr.c
index cea0518787..2b4f0f0689 100644
--- a/sysdeps/x86_64/multiarch/memchr.c
+++ b/sysdeps/x86_64/multiarch/memchr.c
@@ -24,7 +24,53 @@
# undef memchr
# define SYMBOL_NAME memchr
-# include "ifunc-evex.h"
+# include <init-arch.h>
+
+extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (evex512) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_rtm) attribute_hidden;
+
+extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden;
+
+extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden;
+
+static inline void *
+IFUNC_SELECTOR (void)
+{
+ const struct cpu_features *cpu_features = __get_cpu_features ();
+
+ /* NB: The X86_ISA_* feature check macros are evaluated at
+ compile time. */
+ if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX2)
+ && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, BMI2)
+ && X86_ISA_CPU_FEATURES_ARCH_P (cpu_features,
+ AVX_Fast_Unaligned_Load, ))
+ {
+ if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
+ && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512BW))
+ {
+ if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_EVEX512))
+ return OPTIMIZE (evex512);
+
+ if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
+ return OPTIMIZE (evex_rtm);
+
+ return OPTIMIZE (evex);
+ }
+
+ if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
+ return OPTIMIZE (avx2_rtm);
+
+ if (X86_ISA_CPU_FEATURES_ARCH_P (cpu_features,
+ Prefer_No_VZEROUPPER, !))
+ return OPTIMIZE (avx2);
+ }
+
+ /* This is unreachable (compile time checked) if ISA level >= 3
+ so no need for a robust fallback here. */
+ return OPTIMIZE (sse2);
+}
libc_ifunc_redirected (__redirect_memchr, memchr, IFUNC_SELECTOR ());
strong_alias (memchr, __memchr)
diff --git a/sysdeps/x86_64/multiarch/strchr.c b/sysdeps/x86_64/multiarch/strchr.c
index 1064bc8e5b..f6bd36ba73 100644
--- a/sysdeps/x86_64/multiarch/strchr.c
+++ b/sysdeps/x86_64/multiarch/strchr.c
@@ -27,6 +27,7 @@
# include <init-arch.h>
extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (evex512) attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden;
@@ -46,7 +47,12 @@ IFUNC_SELECTOR (void)
{
if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
&& X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512BW))
- return OPTIMIZE (evex);
+ {
+ if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_EVEX512))
+ return OPTIMIZE (evex512);
+
+ return OPTIMIZE (evex);
+ }
if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
return OPTIMIZE (avx2_rtm);
diff --git a/sysdeps/x86_64/multiarch/strchrnul.c b/sysdeps/x86_64/multiarch/strchrnul.c
index f4bfc28c7b..81ea03b91b 100644
--- a/sysdeps/x86_64/multiarch/strchrnul.c
+++ b/sysdeps/x86_64/multiarch/strchrnul.c
@@ -26,10 +26,10 @@
# undef strchrnul
# define SYMBOL_NAME strchrnul
-# include "ifunc-avx2.h"
+# include "ifunc-evex512.h"
libc_ifunc_redirected (__redirect_strchrnul, __strchrnul,
- IFUNC_SELECTOR ());
+ IFUNC_SELECTOR ());
weak_alias (__strchrnul, strchrnul)
# ifdef SHARED
__hidden_ver1 (__strchrnul, __GI___strchrnul, __redirect_strchrnul)
diff --git a/sysdeps/x86_64/multiarch/strlen.c b/sysdeps/x86_64/multiarch/strlen.c
index 95aa48e5d3..76f51b077a 100644
--- a/sysdeps/x86_64/multiarch/strlen.c
+++ b/sysdeps/x86_64/multiarch/strlen.c
@@ -24,7 +24,7 @@
# undef strlen
# define SYMBOL_NAME strlen
-# include "ifunc-avx2.h"
+# include "ifunc-evex512.h"
libc_ifunc_redirected (__redirect_strlen, strlen, IFUNC_SELECTOR ());
# ifdef SHARED
diff --git a/sysdeps/x86_64/multiarch/strnlen.c b/sysdeps/x86_64/multiarch/strnlen.c
index bfe4d7abf0..47ab8a5932 100644
--- a/sysdeps/x86_64/multiarch/strnlen.c
+++ b/sysdeps/x86_64/multiarch/strnlen.c
@@ -26,7 +26,7 @@
# undef strnlen
# define SYMBOL_NAME strnlen
-# include "ifunc-avx2.h"
+# include "ifunc-evex512.h"
libc_ifunc_redirected (__redirect_strnlen, __strnlen, IFUNC_SELECTOR ());
weak_alias (__strnlen, strnlen);
diff --git a/sysdeps/x86_64/multiarch/strrchr.c b/sysdeps/x86_64/multiarch/strrchr.c
index 66d3f5b1d1..15c6443902 100644
--- a/sysdeps/x86_64/multiarch/strrchr.c
+++ b/sysdeps/x86_64/multiarch/strrchr.c
@@ -23,7 +23,7 @@
# undef strrchr
# define SYMBOL_NAME strrchr
-# include "ifunc-avx2.h"
+# include "ifunc-evex512.h"
libc_ifunc_redirected (__redirect_strrchr, strrchr, IFUNC_SELECTOR ());
weak_alias (strrchr, rindex);
--
2.34.1
More information about the Libc-alpha
mailing list