[PATCH v1 4/4] x86: Add optimized functions for the wide-character strcpy family
Noah Goldstein
goldstein.w.n@gmail.com
Thu Nov 3 08:53:17 GMT 2022
Implemented:
wcscat-avx2{+rtm}
wcscpy-avx2{+rtm}
wcpcpy-avx2{+rtm}
wcsncpy-avx2{+rtm}
wcpncpy-avx2{+rtm}
wcsncat-avx2{+rtm}
wcscat-evex
wcscpy-evex
wcpcpy-evex
wcsncpy-evex
wcpncpy-evex
wcsncat-evex
Performance Changes:
Times are from N = 10 runs of the benchmark suite and are reported
as geometric mean of all ratios of New Implementation / Best Old
Implementation. Best Old Implementation was determined with the
highest ISA implementation.
wcscat-avx2 -> 0.975
wcscpy-avx2 -> 0.591
wcpcpy-avx2 -> 0.698
wcsncpy-avx2 -> 0.730
wcpncpy-avx2 -> 0.711
wcsncat-avx2 -> 0.938
wcscat-evex -> 0.991
wcscpy-evex -> 0.587
wcpcpy-evex -> 0.695
wcsncpy-evex -> 0.719
wcpncpy-evex -> 0.694
wcsncat-evex -> 0.959
Code Size Changes:
This change (compared with the last two commits without it)
increase the size of libc.so by 19392 bytes. For reference this
entire patchset increases libc.so by 2624 bytes (so without the
wide-character functions libc.so would decrease by 16768 bytes).
Full check passes on x86-64 and build succeeds for all ISA levels w/
and w/o multiarch.
---
Results attached to the evex/avx2 patches.
sysdeps/x86_64/Makefile | 5 +
sysdeps/x86_64/multiarch/Makefile | 26 +++-
sysdeps/x86_64/multiarch/ifunc-impl-list.c | 135 +++++++++++++++++++-
sysdeps/x86_64/multiarch/ifunc-wcs.h | 60 +++++++++
sysdeps/x86_64/multiarch/wcpcpy-avx2-rtm.S | 3 +
sysdeps/x86_64/multiarch/wcpcpy-avx2.S | 8 ++
sysdeps/x86_64/multiarch/wcpcpy-evex.S | 8 ++
sysdeps/x86_64/multiarch/wcpcpy-generic.c | 27 ++++
sysdeps/x86_64/multiarch/wcpcpy.c | 37 ++++++
sysdeps/x86_64/multiarch/wcpncpy-avx2-rtm.S | 3 +
sysdeps/x86_64/multiarch/wcpncpy-avx2.S | 8 ++
sysdeps/x86_64/multiarch/wcpncpy-evex.S | 8 ++
sysdeps/x86_64/multiarch/wcpncpy-generic.c | 27 ++++
sysdeps/x86_64/multiarch/wcpncpy.c | 37 ++++++
sysdeps/x86_64/multiarch/wcscat-avx2-rtm.S | 3 +
sysdeps/x86_64/multiarch/wcscat-avx2.S | 10 ++
sysdeps/x86_64/multiarch/wcscat-evex.S | 9 ++
sysdeps/x86_64/multiarch/wcscat-generic.c | 27 ++++
sysdeps/x86_64/multiarch/wcscat.c | 37 ++++++
sysdeps/x86_64/multiarch/wcscpy-avx2-rtm.S | 3 +
sysdeps/x86_64/multiarch/wcscpy-avx2.S | 7 +
sysdeps/x86_64/multiarch/wcscpy-evex.S | 7 +
sysdeps/x86_64/multiarch/wcscpy-generic.c | 3 +-
sysdeps/x86_64/multiarch/wcscpy.c | 21 +++
sysdeps/x86_64/multiarch/wcsncat-avx2-rtm.S | 3 +
sysdeps/x86_64/multiarch/wcsncat-avx2.S | 9 ++
sysdeps/x86_64/multiarch/wcsncat-evex.S | 9 ++
sysdeps/x86_64/multiarch/wcsncat-generic.c | 27 ++++
sysdeps/x86_64/multiarch/wcsncat.c | 34 +++++
sysdeps/x86_64/multiarch/wcsncpy-avx2-rtm.S | 3 +
sysdeps/x86_64/multiarch/wcsncpy-avx2.S | 7 +
sysdeps/x86_64/multiarch/wcsncpy-evex.S | 7 +
sysdeps/x86_64/multiarch/wcsncpy-generic.c | 27 ++++
sysdeps/x86_64/multiarch/wcsncpy.c | 37 ++++++
sysdeps/x86_64/wcpcpy-generic.c | 31 +++++
sysdeps/x86_64/wcpcpy.S | 41 ++++++
sysdeps/x86_64/wcpncpy-generic.c | 31 +++++
sysdeps/x86_64/wcpncpy.S | 41 ++++++
sysdeps/x86_64/wcscat-generic.c | 31 +++++
sysdeps/x86_64/wcscat.S | 41 ++++++
sysdeps/x86_64/wcscpy.S | 2 +
sysdeps/x86_64/wcsncat-generic.c | 31 +++++
sysdeps/x86_64/wcsncat.S | 39 ++++++
sysdeps/x86_64/wcsncpy-generic.c | 31 +++++
sysdeps/x86_64/wcsncpy.S | 41 ++++++
45 files changed, 1036 insertions(+), 6 deletions(-)
create mode 100644 sysdeps/x86_64/multiarch/ifunc-wcs.h
create mode 100644 sysdeps/x86_64/multiarch/wcpcpy-avx2-rtm.S
create mode 100644 sysdeps/x86_64/multiarch/wcpcpy-avx2.S
create mode 100644 sysdeps/x86_64/multiarch/wcpcpy-evex.S
create mode 100644 sysdeps/x86_64/multiarch/wcpcpy-generic.c
create mode 100644 sysdeps/x86_64/multiarch/wcpcpy.c
create mode 100644 sysdeps/x86_64/multiarch/wcpncpy-avx2-rtm.S
create mode 100644 sysdeps/x86_64/multiarch/wcpncpy-avx2.S
create mode 100644 sysdeps/x86_64/multiarch/wcpncpy-evex.S
create mode 100644 sysdeps/x86_64/multiarch/wcpncpy-generic.c
create mode 100644 sysdeps/x86_64/multiarch/wcpncpy.c
create mode 100644 sysdeps/x86_64/multiarch/wcscat-avx2-rtm.S
create mode 100644 sysdeps/x86_64/multiarch/wcscat-avx2.S
create mode 100644 sysdeps/x86_64/multiarch/wcscat-evex.S
create mode 100644 sysdeps/x86_64/multiarch/wcscat-generic.c
create mode 100644 sysdeps/x86_64/multiarch/wcscat.c
create mode 100644 sysdeps/x86_64/multiarch/wcscpy-avx2-rtm.S
create mode 100644 sysdeps/x86_64/multiarch/wcscpy-avx2.S
create mode 100644 sysdeps/x86_64/multiarch/wcscpy-evex.S
create mode 100644 sysdeps/x86_64/multiarch/wcsncat-avx2-rtm.S
create mode 100644 sysdeps/x86_64/multiarch/wcsncat-avx2.S
create mode 100644 sysdeps/x86_64/multiarch/wcsncat-evex.S
create mode 100644 sysdeps/x86_64/multiarch/wcsncat-generic.c
create mode 100644 sysdeps/x86_64/multiarch/wcsncat.c
create mode 100644 sysdeps/x86_64/multiarch/wcsncpy-avx2-rtm.S
create mode 100644 sysdeps/x86_64/multiarch/wcsncpy-avx2.S
create mode 100644 sysdeps/x86_64/multiarch/wcsncpy-evex.S
create mode 100644 sysdeps/x86_64/multiarch/wcsncpy-generic.c
create mode 100644 sysdeps/x86_64/multiarch/wcsncpy.c
create mode 100644 sysdeps/x86_64/wcpcpy-generic.c
create mode 100644 sysdeps/x86_64/wcpcpy.S
create mode 100644 sysdeps/x86_64/wcpncpy-generic.c
create mode 100644 sysdeps/x86_64/wcpncpy.S
create mode 100644 sysdeps/x86_64/wcscat-generic.c
create mode 100644 sysdeps/x86_64/wcscat.S
create mode 100644 sysdeps/x86_64/wcsncat-generic.c
create mode 100644 sysdeps/x86_64/wcsncat.S
create mode 100644 sysdeps/x86_64/wcsncpy-generic.c
create mode 100644 sysdeps/x86_64/wcsncpy.S
diff --git a/sysdeps/x86_64/Makefile b/sysdeps/x86_64/Makefile
index 3627c5659f..688eb2d7c4 100644
--- a/sysdeps/x86_64/Makefile
+++ b/sysdeps/x86_64/Makefile
@@ -188,8 +188,13 @@ endif
ifeq ($(subdir),wcsmbs)
sysdep_routines += \
+ wcpcpy-generic \
+ wcpncpy-generic \
+ wcscat-generic \
wcscpy-generic \
+ wcsncat-generic \
wcsncmp-generic \
+ wcsncpy-generic \
wcsnlen-generic \
# sysdep_routines
diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile
index 597ac9d5e9..8b5a3a4ee5 100644
--- a/sysdeps/x86_64/multiarch/Makefile
+++ b/sysdeps/x86_64/multiarch/Makefile
@@ -130,6 +130,18 @@ endif
ifeq ($(subdir),wcsmbs)
sysdep_routines += \
+ wcpcpy-avx2 \
+ wcpcpy-avx2-rtm \
+ wcpcpy-evex \
+ wcpcpy-generic \
+ wcpncpy-avx2 \
+ wcpncpy-avx2-rtm \
+ wcpncpy-evex \
+ wcpncpy-generic \
+ wcscat-avx2 \
+ wcscat-avx2-rtm \
+ wcscat-evex \
+ wcscat-generic \
wcschr-avx2 \
wcschr-avx2-rtm \
wcschr-evex \
@@ -139,6 +151,10 @@ sysdep_routines += \
wcscmp-avx2-rtm \
wcscmp-evex \
wcscmp-sse2 \
+ wcscpy-avx2 \
+ wcscpy-avx2-rtm \
+ wcscpy-evex \
+ wcscpy-generic \
wcscpy-ssse3 \
wcslen-avx2 \
wcslen-avx2-rtm \
@@ -146,9 +162,17 @@ sysdep_routines += \
wcslen-evex512 \
wcslen-sse2 \
wcslen-sse4_1 \
+ wcsncat-avx2 \
+ wcsncat-avx2-rtm \
+ wcsncat-evex \
+ wcsncat-generic \
wcsncmp-avx2 \
wcsncmp-avx2-rtm \
wcsncmp-evex \
+ wcsncpy-avx2 \
+ wcsncpy-avx2-rtm \
+ wcsncpy-evex \
+ wcsncpy-generic \
wcsnlen-avx2 \
wcsnlen-avx2-rtm \
wcsnlen-evex \
@@ -161,8 +185,8 @@ sysdep_routines += \
wmemchr-avx2 \
wmemchr-avx2-rtm \
wmemchr-evex \
- wmemchr-evex512 \
wmemchr-evex-rtm \
+ wmemchr-evex512 \
wmemchr-sse2 \
wmemcmp-avx2-movbe \
wmemcmp-avx2-movbe-rtm \
diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
index c3d75a09f4..f31fdf4d05 100644
--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c
+++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
@@ -891,16 +891,145 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
/* Support sysdeps/x86_64/multiarch/wcscpy.c. */
IFUNC_IMPL (i, name, wcscpy,
- /* ISA V4 wrapper for SSSE3 implementation because
- the SSSE3 implementation is also used at ISA
- level 3/4. */
X86_IFUNC_IMPL_ADD_V4 (array, i, wcscpy,
+ (CPU_FEATURE_USABLE (AVX512VL)
+ && CPU_FEATURE_USABLE (AVX512BW)
+ && CPU_FEATURE_USABLE (BMI1)
+ && CPU_FEATURE_USABLE (BMI2)),
+ __wcscpy_evex)
+ X86_IFUNC_IMPL_ADD_V3 (array, i, wcscpy,
+ (CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI1)
+ && CPU_FEATURE_USABLE (BMI2)),
+ __wcscpy_avx2)
+ X86_IFUNC_IMPL_ADD_V3 (array, i, wcscpy,
+ (CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI1)
+ && CPU_FEATURE_USABLE (BMI2)
+ && CPU_FEATURE_USABLE (RTM)),
+ __wcscpy_avx2_rtm)
+ X86_IFUNC_IMPL_ADD_V2 (array, i, wcscpy,
CPU_FEATURE_USABLE (SSSE3),
__wcscpy_ssse3)
X86_IFUNC_IMPL_ADD_V1 (array, i, wcscpy,
1,
__wcscpy_generic))
+ /* Support sysdeps/x86_64/multiarch/wcsncpy.c. */
+ IFUNC_IMPL (i, name, wcsncpy,
+ X86_IFUNC_IMPL_ADD_V4 (array, i, wcsncpy,
+ (CPU_FEATURE_USABLE (AVX512VL)
+ && CPU_FEATURE_USABLE (AVX512BW)
+ && CPU_FEATURE_USABLE (BMI1)
+ && CPU_FEATURE_USABLE (BMI2)),
+ __wcsncpy_evex)
+ X86_IFUNC_IMPL_ADD_V3 (array, i, wcsncpy,
+ (CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI1)
+ && CPU_FEATURE_USABLE (BMI2)),
+ __wcsncpy_avx2)
+ X86_IFUNC_IMPL_ADD_V3 (array, i, wcsncpy,
+ (CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI1)
+ && CPU_FEATURE_USABLE (BMI2)
+ && CPU_FEATURE_USABLE (RTM)),
+ __wcsncpy_avx2_rtm)
+ X86_IFUNC_IMPL_ADD_V1 (array, i, wcpncpy,
+ 1,
+ __wcsncpy_generic))
+
+ /* Support sysdeps/x86_64/multiarch/wcpcpy.c. */
+ IFUNC_IMPL (i, name, wcpcpy,
+ X86_IFUNC_IMPL_ADD_V4 (array, i, wcpcpy,
+ (CPU_FEATURE_USABLE (AVX512VL)
+ && CPU_FEATURE_USABLE (AVX512BW)
+ && CPU_FEATURE_USABLE (BMI1)
+ && CPU_FEATURE_USABLE (BMI2)),
+ __wcpcpy_evex)
+ X86_IFUNC_IMPL_ADD_V3 (array, i, wcpcpy,
+ (CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI1)
+ && CPU_FEATURE_USABLE (BMI2)),
+ __wcpcpy_avx2)
+ X86_IFUNC_IMPL_ADD_V3 (array, i, wcpcpy,
+ (CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI1)
+ && CPU_FEATURE_USABLE (BMI2)
+ && CPU_FEATURE_USABLE (RTM)),
+ __wcpcpy_avx2_rtm)
+ X86_IFUNC_IMPL_ADD_V1 (array, i, wcpcpy,
+ 1,
+ __wcpcpy_generic))
+
+ /* Support sysdeps/x86_64/multiarch/wcpncpy.c. */
+ IFUNC_IMPL (i, name, wcpncpy,
+ X86_IFUNC_IMPL_ADD_V4 (array, i, wcpncpy,
+ (CPU_FEATURE_USABLE (AVX512VL)
+ && CPU_FEATURE_USABLE (AVX512BW)
+ && CPU_FEATURE_USABLE (BMI1)
+ && CPU_FEATURE_USABLE (BMI2)),
+ __wcpncpy_evex)
+ X86_IFUNC_IMPL_ADD_V3 (array, i, wcpncpy,
+ (CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI1)
+ && CPU_FEATURE_USABLE (BMI2)),
+ __wcpncpy_avx2)
+ X86_IFUNC_IMPL_ADD_V3 (array, i, wcpncpy,
+ (CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI1)
+ && CPU_FEATURE_USABLE (BMI2)
+ && CPU_FEATURE_USABLE (RTM)),
+ __wcpncpy_avx2_rtm)
+ X86_IFUNC_IMPL_ADD_V1 (array, i, wcsncpy,
+ 1,
+ __wcpncpy_generic))
+
+ /* Support sysdeps/x86_64/multiarch/wcscat.c. */
+ IFUNC_IMPL (i, name, wcscat,
+ X86_IFUNC_IMPL_ADD_V4 (array, i, wcscat,
+ (CPU_FEATURE_USABLE (AVX512VL)
+ && CPU_FEATURE_USABLE (AVX512BW)
+ && CPU_FEATURE_USABLE (BMI1)
+ && CPU_FEATURE_USABLE (BMI2)),
+ __wcscat_evex)
+ X86_IFUNC_IMPL_ADD_V3 (array, i, wcscat,
+ (CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI1)
+ && CPU_FEATURE_USABLE (BMI2)),
+ __wcscat_avx2)
+ X86_IFUNC_IMPL_ADD_V3 (array, i, wcscat,
+ (CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI1)
+ && CPU_FEATURE_USABLE (BMI2)
+ && CPU_FEATURE_USABLE (RTM)),
+ __wcscat_avx2_rtm)
+ X86_IFUNC_IMPL_ADD_V1 (array, i, wcscat,
+ 1,
+ __wcscat_generic))
+
+ /* Support sysdeps/x86_64/multiarch/wcsncat.c. */
+ IFUNC_IMPL (i, name, wcsncat,
+ X86_IFUNC_IMPL_ADD_V4 (array, i, wcsncat,
+ (CPU_FEATURE_USABLE (AVX512VL)
+ && CPU_FEATURE_USABLE (AVX512BW)
+ && CPU_FEATURE_USABLE (BMI1)
+ && CPU_FEATURE_USABLE (BMI2)),
+ __wcsncat_evex)
+ X86_IFUNC_IMPL_ADD_V3 (array, i, wcsncat,
+ (CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI1)
+ && CPU_FEATURE_USABLE (BMI2)),
+ __wcsncat_avx2)
+ X86_IFUNC_IMPL_ADD_V3 (array, i, wcsncat,
+ (CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI1)
+ && CPU_FEATURE_USABLE (BMI2)
+ && CPU_FEATURE_USABLE (RTM)),
+ __wcsncat_avx2_rtm)
+ X86_IFUNC_IMPL_ADD_V1 (array, i, wcsncat,
+ 1,
+ __wcsncat_generic))
+
/* Support sysdeps/x86_64/multiarch/wcslen.c. */
IFUNC_IMPL (i, name, wcslen,
X86_IFUNC_IMPL_ADD_V4 (array, i, wcslen,
diff --git a/sysdeps/x86_64/multiarch/ifunc-wcs.h b/sysdeps/x86_64/multiarch/ifunc-wcs.h
new file mode 100644
index 0000000000..cda633d8fb
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/ifunc-wcs.h
@@ -0,0 +1,60 @@
+/* Common definition for ifunc selections optimized wide-character
+ string copy functions.
+
+ All versions must be listed in ifunc-impl-list.c.
+ Copyright (C) 2022 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <init-arch.h>
+
+#ifndef GENERIC
+# define GENERIC generic
+#endif
+
+extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden;
+
+extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden;
+
+extern __typeof (REDIRECT_NAME) OPTIMIZE (GENERIC) attribute_hidden;
+
+static inline void *
+IFUNC_SELECTOR (void)
+{
+ const struct cpu_features *cpu_features = __get_cpu_features ();
+
+ if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX2)
+ && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, BMI1)
+ && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, BMI2)
+ && X86_ISA_CPU_FEATURES_ARCH_P (cpu_features,
+ AVX_Fast_Unaligned_Load, ))
+ {
+ if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
+ && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512BW))
+ return OPTIMIZE (evex);
+
+ if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
+ return OPTIMIZE (avx2_rtm);
+
+ if (X86_ISA_CPU_FEATURES_ARCH_P (cpu_features,
+ Prefer_No_VZEROUPPER, !))
+ return OPTIMIZE (avx2);
+
+ }
+
+ return OPTIMIZE (GENERIC);
+}
diff --git a/sysdeps/x86_64/multiarch/wcpcpy-avx2-rtm.S b/sysdeps/x86_64/multiarch/wcpcpy-avx2-rtm.S
new file mode 100644
index 0000000000..756280a3ab
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcpcpy-avx2-rtm.S
@@ -0,0 +1,3 @@
+#define WCPCPY __wcpcpy_avx2_rtm
+#include "x86-avx2-rtm-vecs.h"
+#include "wcpcpy-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/wcpcpy-avx2.S b/sysdeps/x86_64/multiarch/wcpcpy-avx2.S
new file mode 100644
index 0000000000..0fffd912d3
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcpcpy-avx2.S
@@ -0,0 +1,8 @@
+#ifndef WCPCPY
+# define WCPCPY __wcpcpy_avx2
+#endif
+
+#define USE_AS_STPCPY
+#define USE_AS_WCSCPY
+#define STRCPY WCPCPY
+#include "strcpy-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/wcpcpy-evex.S b/sysdeps/x86_64/multiarch/wcpcpy-evex.S
new file mode 100644
index 0000000000..ac6429cc07
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcpcpy-evex.S
@@ -0,0 +1,8 @@
+#ifndef WCPCPY
+# define WCPCPY __wcpcpy_evex
+#endif
+
+#define USE_AS_STPCPY
+#define USE_AS_WCSCPY
+#define STRCPY WCPCPY
+#include "strcpy-evex.S"
diff --git a/sysdeps/x86_64/multiarch/wcpcpy-generic.c b/sysdeps/x86_64/multiarch/wcpcpy-generic.c
new file mode 100644
index 0000000000..0ba29b081f
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcpcpy-generic.c
@@ -0,0 +1,27 @@
+/* wcpcpy.
+ Copyright (C) 2022 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+/* We always need to build this implementation as strspn-sse4 needs to
+ be able to fallback to it. */
+#include <isa-level.h>
+#if ISA_SHOULD_BUILD (2)
+
+# define WCPCPY __wcpcpy_generic
+# include <wcsmbs/wcpcpy.c>
+
+#endif
diff --git a/sysdeps/x86_64/multiarch/wcpcpy.c b/sysdeps/x86_64/multiarch/wcpcpy.c
new file mode 100644
index 0000000000..8f96ddbc99
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcpcpy.c
@@ -0,0 +1,37 @@
+/* Multiple versions of wcpcpy.
+ All versions must be listed in ifunc-impl-list.c.
+ Copyright (C) 2022 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+/* Define multiple versions only for the definition in libc. */
+#if IS_IN (libc)
+# define __wcpcpy __redirect_wcpcpy
+# include <wchar.h>
+# undef __wcpcpy
+
+# define SYMBOL_NAME wcpcpy
+# include <init-arch.h>
+
+# include "ifunc-wcs.h"
+
+libc_ifunc_redirected (__redirect_wcpcpy, __wcpcpy, IFUNC_SELECTOR ());
+weak_alias (__wcpcpy, wcpcpy)
+# ifdef SHARED
+__hidden_ver1 (__wcpcpy, __GI___wcpcpy, __redirect_wcpcpy)
+ __attribute__((visibility ("hidden"))) __attribute_copy__ (wcpcpy);
+# endif
+#endif
diff --git a/sysdeps/x86_64/multiarch/wcpncpy-avx2-rtm.S b/sysdeps/x86_64/multiarch/wcpncpy-avx2-rtm.S
new file mode 100644
index 0000000000..80600d6b01
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcpncpy-avx2-rtm.S
@@ -0,0 +1,3 @@
+#define WCPNCPY __wcpncpy_avx2_rtm
+#include "x86-avx2-rtm-vecs.h"
+#include "wcpncpy-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/wcpncpy-avx2.S b/sysdeps/x86_64/multiarch/wcpncpy-avx2.S
new file mode 100644
index 0000000000..b7e594f7b7
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcpncpy-avx2.S
@@ -0,0 +1,8 @@
+#ifndef WCPNCPY
+# define WCPNCPY __wcpncpy_avx2
+#endif
+
+#define USE_AS_WCSCPY
+#define USE_AS_STPCPY
+#define STRNCPY WCPNCPY
+#include "strncpy-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/wcpncpy-evex.S b/sysdeps/x86_64/multiarch/wcpncpy-evex.S
new file mode 100644
index 0000000000..62ddb694fe
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcpncpy-evex.S
@@ -0,0 +1,8 @@
+#ifndef WCPNCPY
+# define WCPNCPY __wcpncpy_evex
+#endif
+
+#define USE_AS_WCSCPY
+#define USE_AS_STPCPY
+#define STRNCPY WCPNCPY
+#include "strncpy-evex.S"
diff --git a/sysdeps/x86_64/multiarch/wcpncpy-generic.c b/sysdeps/x86_64/multiarch/wcpncpy-generic.c
new file mode 100644
index 0000000000..4aab4ecdd2
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcpncpy-generic.c
@@ -0,0 +1,27 @@
+/* wcpncpy.
+ Copyright (C) 2022 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+/* We always need to build this implementation as strspn-sse4 needs to
+ be able to fallback to it. */
+#include <isa-level.h>
+#if ISA_SHOULD_BUILD (2)
+
+# define WCPNCPY __wcpncpy_generic
+# include <wcsmbs/wcpncpy.c>
+
+#endif
diff --git a/sysdeps/x86_64/multiarch/wcpncpy.c b/sysdeps/x86_64/multiarch/wcpncpy.c
new file mode 100644
index 0000000000..ed8f307e07
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcpncpy.c
@@ -0,0 +1,37 @@
+/* Multiple versions of wcpncpy.
+ All versions must be listed in ifunc-impl-list.c.
+ Copyright (C) 2022 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+/* Define multiple versions only for the definition in libc. */
+#if IS_IN (libc)
+# define __wcpncpy __redirect_wcpncpy
+# include <wchar.h>
+# undef __wcpncpy
+
+# define SYMBOL_NAME wcpncpy
+# include <init-arch.h>
+
+# include "ifunc-wcs.h"
+
+libc_ifunc_redirected (__redirect_wcpncpy, __wcpncpy, IFUNC_SELECTOR ());
+weak_alias (__wcpncpy, wcpncpy)
+# ifdef SHARED
+__hidden_ver1 (__wcpncpy, __GI___wcpncpy, __redirect_wcpncpy)
+ __attribute__((visibility ("hidden"))) __attribute_copy__ (wcpncpy);
+# endif
+#endif
diff --git a/sysdeps/x86_64/multiarch/wcscat-avx2-rtm.S b/sysdeps/x86_64/multiarch/wcscat-avx2-rtm.S
new file mode 100644
index 0000000000..e99449a2dc
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcscat-avx2-rtm.S
@@ -0,0 +1,3 @@
+#define WCSCAT __wcscat_avx2_rtm
+#include "x86-avx2-rtm-vecs.h"
+#include "wcscat-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/wcscat-avx2.S b/sysdeps/x86_64/multiarch/wcscat-avx2.S
new file mode 100644
index 0000000000..a20f23c09d
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcscat-avx2.S
@@ -0,0 +1,10 @@
+#ifndef WCSCAT
+# define WCSCAT __wcscat_avx2
+#endif
+
+#define USE_AS_WCSCPY
+#define USE_AS_STRCAT
+
+#define STRCPY WCSCAT
+
+#include "strcpy-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/wcscat-evex.S b/sysdeps/x86_64/multiarch/wcscat-evex.S
new file mode 100644
index 0000000000..1d017e4899
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcscat-evex.S
@@ -0,0 +1,9 @@
+#ifndef WCSCAT
+# define WCSCAT __wcscat_evex
+#endif
+
+#define USE_AS_WCSCPY
+#define USE_AS_STRCAT
+
+#define STRCPY WCSCAT
+#include "strcpy-evex.S"
diff --git a/sysdeps/x86_64/multiarch/wcscat-generic.c b/sysdeps/x86_64/multiarch/wcscat-generic.c
new file mode 100644
index 0000000000..6476f85bbb
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcscat-generic.c
@@ -0,0 +1,27 @@
+/* wcscat.
+ Copyright (C) 2022 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+/* We always need to build this implementation as strspn-sse4 needs to
+ be able to fallback to it. */
+#include <isa-level.h>
+#if ISA_SHOULD_BUILD (2)
+
+# define WCSCAT __wcscat_generic
+# include <wcsmbs/wcscat.c>
+
+#endif
diff --git a/sysdeps/x86_64/multiarch/wcscat.c b/sysdeps/x86_64/multiarch/wcscat.c
new file mode 100644
index 0000000000..3277c44561
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcscat.c
@@ -0,0 +1,37 @@
+/* Multiple versions of wcscat.
+ All versions must be listed in ifunc-impl-list.c.
+ Copyright (C) 2022 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+/* Define multiple versions only for the definition in libc. */
+#if IS_IN (libc)
+# define __wcscat __redirect_wcscat
+# include <wchar.h>
+# undef __wcscat
+
+# define SYMBOL_NAME wcscat
+# include <init-arch.h>
+
+# include "ifunc-wcs.h"
+
+libc_ifunc_redirected (__redirect_wcscat, __wcscat, IFUNC_SELECTOR ());
+weak_alias (__wcscat, wcscat)
+# ifdef SHARED
+__hidden_ver1 (__wcscat, __GI___wcscat, __redirect_wcscat)
+ __attribute__((visibility ("hidden"))) __attribute_copy__ (wcscat);
+# endif
+#endif
diff --git a/sysdeps/x86_64/multiarch/wcscpy-avx2-rtm.S b/sysdeps/x86_64/multiarch/wcscpy-avx2-rtm.S
new file mode 100644
index 0000000000..2f800c8d3e
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcscpy-avx2-rtm.S
@@ -0,0 +1,3 @@
+#define WCSCPY __wcscpy_avx2_rtm
+#include "x86-avx2-rtm-vecs.h"
+#include "wcscpy-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/wcscpy-avx2.S b/sysdeps/x86_64/multiarch/wcscpy-avx2.S
new file mode 100644
index 0000000000..6bc509da07
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcscpy-avx2.S
@@ -0,0 +1,7 @@
+#ifndef WCSCPY
+# define WCSCPY __wcscpy_avx2
+#endif
+
+#define USE_AS_WCSCPY
+#define STRCPY WCSCPY
+#include "strcpy-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/wcscpy-evex.S b/sysdeps/x86_64/multiarch/wcscpy-evex.S
new file mode 100644
index 0000000000..1069a8e224
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcscpy-evex.S
@@ -0,0 +1,7 @@
+#ifndef WCSCPY
+# define WCSCPY __wcscpy_evex
+#endif
+
+#define USE_AS_WCSCPY
+#define STRCPY WCSCPY
+#include "strcpy-evex.S"
diff --git a/sysdeps/x86_64/multiarch/wcscpy-generic.c b/sysdeps/x86_64/multiarch/wcscpy-generic.c
index 93d314aaad..600d606c45 100644
--- a/sysdeps/x86_64/multiarch/wcscpy-generic.c
+++ b/sysdeps/x86_64/multiarch/wcscpy-generic.c
@@ -18,8 +18,7 @@
#include <isa-level.h>
-
-#if ISA_SHOULD_BUILD (1)
+#if ISA_SHOULD_BUILD (2)
# define WCSCPY __wcscpy_generic
# include <wcsmbs/wcscpy.c>
diff --git a/sysdeps/x86_64/multiarch/wcscpy.c b/sysdeps/x86_64/multiarch/wcscpy.c
index 92c917b6b4..7f6387817b 100644
--- a/sysdeps/x86_64/multiarch/wcscpy.c
+++ b/sysdeps/x86_64/multiarch/wcscpy.c
@@ -26,6 +26,11 @@
# define SYMBOL_NAME wcscpy
# include <init-arch.h>
+extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden;
+
+extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden;
+
extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE (generic) attribute_hidden;
@@ -35,6 +40,22 @@ IFUNC_SELECTOR (void)
{
const struct cpu_features* cpu_features = __get_cpu_features ();
+ if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX2)
+ && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, BMI1)
+ && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, BMI2)
+ && X86_ISA_CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load, ))
+ {
+ if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
+ && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512BW))
+ return OPTIMIZE (evex);
+
+ if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
+ return OPTIMIZE (avx2_rtm);
+
+ if (X86_ISA_CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER, !))
+ return OPTIMIZE (avx2);
+ }
+
if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, SSSE3))
return OPTIMIZE (ssse3);
diff --git a/sysdeps/x86_64/multiarch/wcsncat-avx2-rtm.S b/sysdeps/x86_64/multiarch/wcsncat-avx2-rtm.S
new file mode 100644
index 0000000000..609d6e69c0
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcsncat-avx2-rtm.S
@@ -0,0 +1,3 @@
+#define WCSNCAT __wcsncat_avx2_rtm
+#include "x86-avx2-rtm-vecs.h"
+#include "wcsncat-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/wcsncat-avx2.S b/sysdeps/x86_64/multiarch/wcsncat-avx2.S
new file mode 100644
index 0000000000..a72105b7e9
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcsncat-avx2.S
@@ -0,0 +1,9 @@
+#ifndef WCSNCAT
+# define WCSNCAT __wcsncat_avx2
+#endif
+
+#define USE_AS_WCSCPY
+#define USE_AS_STRCAT
+
+#define STRNCAT WCSNCAT
+#include "strncat-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/wcsncat-evex.S b/sysdeps/x86_64/multiarch/wcsncat-evex.S
new file mode 100644
index 0000000000..392215950a
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcsncat-evex.S
@@ -0,0 +1,9 @@
+#ifndef WCSCAT
+# define WCSCAT __wcsncat_evex
+#endif
+
+#define USE_AS_WCSCPY
+#define USE_AS_STRCAT
+
+#define STRNCAT WCSCAT
+#include "strncat-evex.S"
diff --git a/sysdeps/x86_64/multiarch/wcsncat-generic.c b/sysdeps/x86_64/multiarch/wcsncat-generic.c
new file mode 100644
index 0000000000..9ced02b35e
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcsncat-generic.c
@@ -0,0 +1,27 @@
+/* wcsncat.
+ Copyright (C) 2022 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+/* We always need to build this implementation as strspn-sse4 needs to
+ be able to fallback to it. */
+#include <isa-level.h>
+#if ISA_SHOULD_BUILD (2)
+
+# define WCSNCAT __wcsncat_generic
+# include <wcsmbs/wcsncat.c>
+
+#endif
diff --git a/sysdeps/x86_64/multiarch/wcsncat.c b/sysdeps/x86_64/multiarch/wcsncat.c
new file mode 100644
index 0000000000..49c46aef08
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcsncat.c
@@ -0,0 +1,34 @@
+/* Multiple versions of wcsncat.
+ All versions must be listed in ifunc-impl-list.c.
+ Copyright (C) 2022 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+/* Define multiple versions only for the definition in libc. */
+#if IS_IN (libc)
+# define wcsncat __redirect_wcsncat
+# include <wchar.h>
+# undef wcsncat
+
+# define SYMBOL_NAME wcsncat
+# include "ifunc-wcs.h"
+
+libc_ifunc_redirected (__redirect_wcsncat, wcsncat, IFUNC_SELECTOR ());
+# ifdef SHARED
+__hidden_ver1 (wcsncat, __GI_wcsncat, __redirect_wcsncat)
+ __attribute__((visibility ("hidden"))) __attribute_copy__ (wcsncat);
+# endif
+#endif
diff --git a/sysdeps/x86_64/multiarch/wcsncpy-avx2-rtm.S b/sysdeps/x86_64/multiarch/wcsncpy-avx2-rtm.S
new file mode 100644
index 0000000000..cab5a6b820
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcsncpy-avx2-rtm.S
@@ -0,0 +1,3 @@
+#define WCSNCPY __wcsncpy_avx2_rtm
+#include "x86-avx2-rtm-vecs.h"
+#include "wcsncpy-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/wcsncpy-avx2.S b/sysdeps/x86_64/multiarch/wcsncpy-avx2.S
new file mode 100644
index 0000000000..3a1a8a372c
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcsncpy-avx2.S
@@ -0,0 +1,7 @@
+#ifndef WCSNCPY
+# define WCSNCPY __wcsncpy_avx2
+#endif
+
+#define USE_AS_WCSCPY
+#define STRNCPY WCSNCPY
+#include "strncpy-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/wcsncpy-evex.S b/sysdeps/x86_64/multiarch/wcsncpy-evex.S
new file mode 100644
index 0000000000..2debb8fd6b
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcsncpy-evex.S
@@ -0,0 +1,7 @@
+#ifndef WCSNCPY
+# define WCSNCPY __wcsncpy_evex
+#endif
+
+#define USE_AS_WCSCPY
+#define STRNCPY WCSNCPY
+#include "strncpy-evex.S"
diff --git a/sysdeps/x86_64/multiarch/wcsncpy-generic.c b/sysdeps/x86_64/multiarch/wcsncpy-generic.c
new file mode 100644
index 0000000000..693521713b
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcsncpy-generic.c
@@ -0,0 +1,27 @@
+/* wcsncpy.
+ Copyright (C) 2022 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+/* We always need to build this implementation as strspn-sse4 needs to
+ be able to fallback to it. */
+#include <isa-level.h>
+#if ISA_SHOULD_BUILD (2)
+
+# define WCSNCPY __wcsncpy_generic
+# include <wcsmbs/wcsncpy.c>
+
+#endif
diff --git a/sysdeps/x86_64/multiarch/wcsncpy.c b/sysdeps/x86_64/multiarch/wcsncpy.c
new file mode 100644
index 0000000000..5b89dd4d27
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcsncpy.c
@@ -0,0 +1,37 @@
+/* Multiple versions of wcsncpy.
+ All versions must be listed in ifunc-impl-list.c.
+ Copyright (C) 2022 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+/* Define multiple versions only for the definition in libc. */
+#if IS_IN (libc)
+# define __wcsncpy __redirect_wcsncpy
+# include <wchar.h>
+# undef __wcsncpy
+
+# define SYMBOL_NAME wcsncpy
+# include <init-arch.h>
+
+# include "ifunc-wcs.h"
+
+libc_ifunc_redirected (__redirect_wcsncpy, __wcsncpy, IFUNC_SELECTOR ());
+weak_alias (__wcsncpy, wcsncpy)
+# ifdef SHARED
+__hidden_ver1 (__wcsncpy, __GI___wcsncpy, __redirect_wcsncpy)
+ __attribute__((visibility ("hidden"))) __attribute_copy__ (wcsncpy);
+# endif
+#endif
diff --git a/sysdeps/x86_64/wcpcpy-generic.c b/sysdeps/x86_64/wcpcpy-generic.c
new file mode 100644
index 0000000000..d52525f288
--- /dev/null
+++ b/sysdeps/x86_64/wcpcpy-generic.c
@@ -0,0 +1,31 @@
+/* wcpcpy dispatch for RTLD and non-multiarch .c ISA level 1 build.
+ Copyright (C) 2022 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+/* wcpcpy non-multiarch build is split into two files,
+ wcpcpy-generic.c and wcpcpy.S. The wcpcpy-generic.c build is
+ for ISA level <= 1 and just uses multiarch/wcpcpy-generic.c.
+ This must be split into two files because we cannot include C
+ code from assembly or vice versa. */
+
+#include <isa-level.h>
+
+#if MINIMUM_X86_ISA_LEVEL <= 3
+
+# include <wcsmbs/wcpcpy.c>
+
+#endif
diff --git a/sysdeps/x86_64/wcpcpy.S b/sysdeps/x86_64/wcpcpy.S
new file mode 100644
index 0000000000..ec32dc070a
--- /dev/null
+++ b/sysdeps/x86_64/wcpcpy.S
@@ -0,0 +1,41 @@
+/* wcpcpy dispatch for RTLD and non-multiarch .c ISA level 1 build.
+ Copyright (C) 2022 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+/* wcpcpy non-multiarch build is split into two files,
+ wcpcpy-generic.c and wcpcpy.S. The wcpcpy-generic.c build is
+ for ISA level <= 1 and just uses multiarch/wcpcpy-generic.c.
+ This must be split into two files because we cannot include C
+ code from assembly or vice versa. */
+
+#include <isa-level.h>
+
+#if MINIMUM_X86_ISA_LEVEL >= 4
+
+# define WCPCPY __wcpcpy
+
+# define DEFAULT_IMPL_V4 "multiarch/wcpcpy-evex.S"
+# define DEFAULT_IMPL_V3 "multiarch/wcpcpy-avx2.S"
+/* isa-default-impl.h expects DEFAULT_IMPL_V1 to be defined but it
+ should never be used from here. */
+# define DEFAULT_IMPL_V1 "ERROR -- Invalid ISA IMPL"
+
+# include "isa-default-impl.h"
+
+weak_alias (__wcpcpy, wcpcpy)
+libc_hidden_def (__wcpcpy)
+#endif
diff --git a/sysdeps/x86_64/wcpncpy-generic.c b/sysdeps/x86_64/wcpncpy-generic.c
new file mode 100644
index 0000000000..871219a445
--- /dev/null
+++ b/sysdeps/x86_64/wcpncpy-generic.c
@@ -0,0 +1,31 @@
+/* wcpncpy dispatch for RTLD and non-multiarch .c ISA level 1 build.
+ Copyright (C) 2022 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+/* wcpncpy non-multiarch build is split into two files,
+ wcpncpy-generic.c and wcpncpy.S. The wcpncpy-generic.c build is
+ for ISA level <= 1 and just uses multiarch/wcpncpy-generic.c.
+ This must be split into two files because we cannot include C
+ code from assembly or vice versa. */
+
+#include <isa-level.h>
+
+#if MINIMUM_X86_ISA_LEVEL <= 3
+
+# include <wcsmbs/wcpncpy.c>
+
+#endif
diff --git a/sysdeps/x86_64/wcpncpy.S b/sysdeps/x86_64/wcpncpy.S
new file mode 100644
index 0000000000..68e6ff1836
--- /dev/null
+++ b/sysdeps/x86_64/wcpncpy.S
@@ -0,0 +1,41 @@
+/* wcpncpy dispatch for RTLD and non-multiarch .c ISA level 1 build.
+ Copyright (C) 2022 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+/* wcpncpy non-multiarch build is split into two files,
+ wcpncpy-generic.c and wcpncpy.S. The wcpncpy-generic.c build is
+ for ISA level <= 1 and just uses multiarch/wcpncpy-generic.c.
+ This must be split into two files because we cannot include C
+ code from assembly or vice versa. */
+
+#include <isa-level.h>
+
+#if MINIMUM_X86_ISA_LEVEL >= 4
+
+# define WCPNCPY __wcpncpy
+
+# define DEFAULT_IMPL_V4 "multiarch/wcpncpy-evex.S"
+# define DEFAULT_IMPL_V3 "multiarch/wcpncpy-avx2.S"
+/* isa-default-impl.h expects DEFAULT_IMPL_V1 to be defined but it
+ should never be used from here. */
+# define DEFAULT_IMPL_V1 "ERROR -- Invalid ISA IMPL"
+
+# include "isa-default-impl.h"
+
+weak_alias (__wcpncpy, wcpncpy)
+libc_hidden_def (__wcpncpy)
+#endif
diff --git a/sysdeps/x86_64/wcscat-generic.c b/sysdeps/x86_64/wcscat-generic.c
new file mode 100644
index 0000000000..85f981a81f
--- /dev/null
+++ b/sysdeps/x86_64/wcscat-generic.c
@@ -0,0 +1,31 @@
+/* wcscat dispatch for RTLD and non-multiarch .c ISA level 1 build.
+ Copyright (C) 2022 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+/* wcscat non-multiarch build is split into two files,
+ wcscat-generic.c and wcscat.S. The wcscat-generic.c build is
+ for ISA level <= 1 and just uses multiarch/wcscat-generic.c.
+ This must be split into two files because we cannot include C
+ code from assembly or vice versa. */
+
+#include <isa-level.h>
+
+#if MINIMUM_X86_ISA_LEVEL <= 3
+
+# include <wcsmbs/wcscat.c>
+
+#endif
diff --git a/sysdeps/x86_64/wcscat.S b/sysdeps/x86_64/wcscat.S
new file mode 100644
index 0000000000..007de3c40c
--- /dev/null
+++ b/sysdeps/x86_64/wcscat.S
@@ -0,0 +1,41 @@
+/* wcscat dispatch for RTLD and non-multiarch .c ISA level 1 build.
+ Copyright (C) 2022 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+/* wcscat non-multiarch build is split into two files,
+ wcscat-generic.c and wcscat.S. The wcscat-generic.c build is
+ for ISA level <= 1 and just uses multiarch/wcscat-generic.c.
+ This must be split into two files because we cannot include C
+ code from assembly or vice versa. */
+
+#include <isa-level.h>
+
+#if MINIMUM_X86_ISA_LEVEL >= 4
+
+# define WCSCAT __wcscat
+
+# define DEFAULT_IMPL_V4 "multiarch/wcscat-evex.S"
+# define DEFAULT_IMPL_V3 "multiarch/wcscat-avx2.S"
+/* isa-default-impl.h expects DEFAULT_IMPL_V1 to be defined but it
+ should never be used from here. */
+# define DEFAULT_IMPL_V1 "ERROR -- Invalid ISA IMPL"
+
+# include "isa-default-impl.h"
+
+weak_alias (__wcscat, wcscat)
+libc_hidden_def (__wcscat)
+#endif
diff --git a/sysdeps/x86_64/wcscpy.S b/sysdeps/x86_64/wcscpy.S
index 11d0bb4bab..ab9288ed74 100644
--- a/sysdeps/x86_64/wcscpy.S
+++ b/sysdeps/x86_64/wcscpy.S
@@ -28,6 +28,8 @@
# define WCSCPY __wcscpy
+# define DEFAULT_IMPL_V4 "multiarch/wcscpy-evex.S"
+# define DEFAULT_IMPL_V3 "multiarch/wcscpy-avx2.S"
# define DEFAULT_IMPL_V2 "multiarch/wcscpy-ssse3.S"
/* isa-default-impl.h expects DEFAULT_IMPL_V1 to be defined but it
should never be used from here. */
diff --git a/sysdeps/x86_64/wcsncat-generic.c b/sysdeps/x86_64/wcsncat-generic.c
new file mode 100644
index 0000000000..2cc0f7b11a
--- /dev/null
+++ b/sysdeps/x86_64/wcsncat-generic.c
@@ -0,0 +1,31 @@
+/* wcsncat dispatch for RTLD and non-multiarch .c ISA level 1 build.
+ Copyright (C) 2022 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+/* wcsncat non-multiarch build is split into two files,
+ wcsncat-generic.c and wcsncat.S. The wcsncat-generic.c build is
+ for ISA level <= 1 and just uses multiarch/wcsncat-generic.c.
+ This must be split into two files because we cannot include C
+ code from assembly or vice versa. */
+
+#include <isa-level.h>
+
+#if MINIMUM_X86_ISA_LEVEL <= 3
+
+# include <wcsmbs/wcsncat.c>
+
+#endif
diff --git a/sysdeps/x86_64/wcsncat.S b/sysdeps/x86_64/wcsncat.S
new file mode 100644
index 0000000000..3f4c7948db
--- /dev/null
+++ b/sysdeps/x86_64/wcsncat.S
@@ -0,0 +1,39 @@
+/* wcsncat dispatch for RTLD and non-multiarch .c ISA level 1 build.
+ Copyright (C) 2022 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+/* wcsncat non-multiarch build is split into two files,
+ wcsncat-generic.c and wcsncat.S. The wcsncat-generic.c build is
+ for ISA level <= 1 and just uses multiarch/wcsncat-generic.c.
+ This must be split into two files because we cannot include C
+ code from assembly or vice versa. */
+
+#include <isa-level.h>
+
+#if MINIMUM_X86_ISA_LEVEL >= 4
+
+# define WCSNCAT wcsncat
+
+# define DEFAULT_IMPL_V4 "multiarch/wcsncat-evex.S"
+# define DEFAULT_IMPL_V3 "multiarch/wcsncat-avx2.S"
+/* isa-default-impl.h expects DEFAULT_IMPL_V1 to be defined but it
+ should never be used from here. */
+# define DEFAULT_IMPL_V1 "ERROR -- Invalid ISA IMPL"
+
+# include "isa-default-impl.h"
+
+#endif
diff --git a/sysdeps/x86_64/wcsncpy-generic.c b/sysdeps/x86_64/wcsncpy-generic.c
new file mode 100644
index 0000000000..49d06b8ae8
--- /dev/null
+++ b/sysdeps/x86_64/wcsncpy-generic.c
@@ -0,0 +1,31 @@
+/* wcsncpy dispatch for RTLD and non-multiarch .c ISA level 1 build.
+ Copyright (C) 2022 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+/* wcsncpy non-multiarch build is split into two files,
+ wcsncpy-generic.c and wcsncpy.S. The wcsncpy-generic.c build is
+ for ISA level <= 1 and just uses multiarch/wcsncpy-generic.c.
+ This must be split into two files because we cannot include C
+ code from assembly or vice versa. */
+
+#include <isa-level.h>
+
+#if MINIMUM_X86_ISA_LEVEL <= 3
+
+# include <wcsmbs/wcsncpy.c>
+
+#endif
diff --git a/sysdeps/x86_64/wcsncpy.S b/sysdeps/x86_64/wcsncpy.S
new file mode 100644
index 0000000000..e1428fd4c1
--- /dev/null
+++ b/sysdeps/x86_64/wcsncpy.S
@@ -0,0 +1,41 @@
+/* wcsncpy dispatch for RTLD and non-multiarch .c ISA level 1 build.
+ Copyright (C) 2022 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+/* wcsncpy non-multiarch build is split into two files,
+ wcsncpy-generic.c and wcsncpy.S. The wcsncpy-generic.c build is
+ for ISA level <= 1 and just uses multiarch/wcsncpy-generic.c.
+ This must be split into two files because we cannot include C
+ code from assembly or vice versa. */
+
+#include <isa-level.h>
+
+#if MINIMUM_X86_ISA_LEVEL >= 4
+
+# define WCSNCPY __wcsncpy
+
+# define DEFAULT_IMPL_V4 "multiarch/wcsncpy-evex.S"
+# define DEFAULT_IMPL_V3 "multiarch/wcsncpy-avx2.S"
+/* isa-default-impl.h expects DEFAULT_IMPL_V1 to be defined but it
+ should never be used from here. */
+# define DEFAULT_IMPL_V1 "ERROR -- Invalid ISA IMPL"
+
+# include "isa-default-impl.h"
+
+weak_alias (__wcsncpy, wcsncpy)
+libc_hidden_def (__wcsncpy)
+#endif
--
2.34.1
More information about the Libc-alpha
mailing list