[PATCH v3 3/3] x86/string: Add version of memmove with page unrolled large impl
Noah Goldstein
goldstein.w.n@gmail.com
Wed Nov 26 01:56:00 GMT 2025
On Tue, Nov 25, 2025 at 2:12 PM DJ Delorie <dj@redhat.com> wrote:
>
>
> Note: I'm not checking for performance here; the other thread discussed
> it already.
>
> One minor change needed (missing newline), otherwise OK.
> Reviewed-by: DJ Delorie <dj@redhat.com>
>
> Noah Goldstein <goldstein.w.n@gmail.com> writes:
> > diff --git a/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def b/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def
> > index 0f14aaf071..5943fc1423 100644
> > --- a/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def
> > +++ b/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def
> > @@ -35,3 +35,4 @@ BIT (Prefer_FSRM)
> > BIT (Avoid_Short_Distance_REP_MOVSB)
> > BIT (Avoid_Non_Temporal_Memset)
> > BIT (Avoid_STOSB)
> > +BIT (Prefer_Page_Unrolled_Large_Copy)
> > \ No newline at end of file
>
> Newline needed.
Fixed + added your reviewed-by tag to the series.
Also added copyrights to all files.
>
> > diff --git a/sysdeps/x86/tst-hwcap-tunables.c b/sysdeps/x86/tst-hwcap-tunables.c
> > index 3e06048dcc..985153fb38 100644
> > --- a/sysdeps/x86/tst-hwcap-tunables.c
> > +++ b/sysdeps/x86/tst-hwcap-tunables.c
> > @@ -61,7 +61,7 @@ static const struct test_t
> > "-Prefer_ERMS,-Prefer_FSRM,-AVX,-AVX2,-AVX512F,-AVX512VL,"
> > "-SSE4_1,-SSE4_2,-SSSE3,-Fast_Unaligned_Load,-ERMS,"
> > "-AVX_Fast_Unaligned_Load,-Avoid_Non_Temporal_Memset,"
> > - "-Avoid_STOSB",
> > + "-Avoid_STOSB,-Prefer_Page_Unrolled_Large_Copy",
> > test_1,
> > array_length (test_1)
> > },
> > @@ -70,7 +70,7 @@ static const struct test_t
> > ",-,-Prefer_ERMS,-Prefer_FSRM,-AVX,-AVX2,-AVX512F,-AVX512VL,"
> > "-SSE4_1,-SSE4_2,-SSSE3,-Fast_Unaligned_Load,,-,"
> > "-ERMS,-AVX_Fast_Unaligned_Load,-Avoid_Non_Temporal_Memset,"
> > - "-Avoid_STOSB,-,",
> > + "-Avoid_STOSB,-Prefer_Page_Unrolled_Large_Copy,-,",
> > test_1,
> > array_length (test_1)
> > }
>
> Ok.
>
> > diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile
> > index 696cb66991..381eaef455 100644
> > --- a/sysdeps/x86_64/multiarch/Makefile
> > +++ b/sysdeps/x86_64/multiarch/Makefile
> > @@ -16,11 +16,14 @@ sysdep_routines += \
> > memcmpeq-evex \
> > memcmpeq-sse2 \
> > memmove-avx-unaligned-erms \
> > + memmove-avx-unaligned-erms-page-unrolled \
> > + memmove-avx-unaligned-erms-page-unrolled-rtm \
> > memmove-avx-unaligned-erms-rtm \
> > memmove-avx512-no-vzeroupper \
> > memmove-avx512-unaligned-erms \
> > memmove-erms \
> > memmove-evex-unaligned-erms \
> > + memmove-evex-unaligned-erms-page-unrolled \
> > memmove-sse2-unaligned-erms \
> > memmove-ssse3 \
> > memrchr-avx2 \
>
> avx-erms-unrolled
> avx-erms-unrolled-rtm
> evex-erms-unrolled
>
> each *.o has both erms and non-erms functions
>
> > diff --git a/sysdeps/x86_64/multiarch/ifunc-memmove.h b/sysdeps/x86_64/multiarch/ifunc-memmove.h
> > index de0ac73a2a..6d5df8a9eb 100644
> > --- a/sysdeps/x86_64/multiarch/ifunc-memmove.h
> > +++ b/sysdeps/x86_64/multiarch/ifunc-memmove.h
> > @@ -28,18 +28,27 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_unaligned_erms)
> > extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_no_vzeroupper)
> > attribute_hidden;
> >
> > -extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_unaligned)
> > - attribute_hidden;
> > -extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_unaligned_erms)
> > - attribute_hidden;
> > +extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_unaligned) attribute_hidden;
> > +extern __typeof (REDIRECT_NAME)
> > + OPTIMIZE (evex_unaligned_page_unrolled) attribute_hidden;
> > +extern __typeof (REDIRECT_NAME)
> > + OPTIMIZE (evex_unaligned_erms) attribute_hidden;
> > +extern __typeof (REDIRECT_NAME)
> > + OPTIMIZE (evex_unaligned_erms_page_unrolled) attribute_hidden;
>
> evex-unrolled NA
> evex-erms-unrolled
>
> > extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned) attribute_hidden;
> > -extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned_erms)
> > - attribute_hidden;
> > -extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned_rtm)
> > - attribute_hidden;
> > -extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned_erms_rtm)
> > - attribute_hidden;
> > +extern __typeof (REDIRECT_NAME)
> > + OPTIMIZE (avx_unaligned_page_unrolled) attribute_hidden;
> > +extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned_erms) attribute_hidden;
> > +extern __typeof (REDIRECT_NAME)
> > + OPTIMIZE (avx_unaligned_erms_page_unrolled) attribute_hidden;
> > +extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned_rtm) attribute_hidden;
> > +extern __typeof (REDIRECT_NAME)
> > + OPTIMIZE (avx_unaligned_page_unrolled_rtm) attribute_hidden;
> > +extern __typeof (REDIRECT_NAME)
> > + OPTIMIZE (avx_unaligned_erms_rtm) attribute_hidden;
> > +extern __typeof (REDIRECT_NAME)
> > + OPTIMIZE (avx_unaligned_erms_page_unrolled_rtm) attribute_hidden;
>
> avx-unrolled
> avx-erms-unrolled OK
> avx-unrolled-rtm NA
> avx-erms-unrolled-rtm
>
> Ok, everything matches.
>
> > extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden;
> >
> > @@ -71,40 +80,60 @@ IFUNC_SELECTOR (void)
> > return OPTIMIZE (avx512_no_vzeroupper);
> > }
> >
> > - if (X86_ISA_CPU_FEATURES_ARCH_P (cpu_features,
> > - AVX_Fast_Unaligned_Load, ))
> > + if (X86_ISA_CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load, ))
> > {
> > if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512VL))
> > {
> > if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
> > - return OPTIMIZE (evex_unaligned_erms);
> > -
> > + {
> > + if (CPU_FEATURES_ARCH_P (cpu_features,
> > + Prefer_Page_Unrolled_Large_Copy))
> > + return OPTIMIZE (evex_unaligned_erms_page_unrolled);
> > + return OPTIMIZE (evex_unaligned_erms);
> > + }
> > +
> > + if (CPU_FEATURES_ARCH_P (cpu_features,
> > + Prefer_Page_Unrolled_Large_Copy))
> > + return OPTIMIZE (evex_unaligned_page_unrolled);
> > return OPTIMIZE (evex_unaligned);
> > }
> >
> > if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
> > {
> > if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
> > - return OPTIMIZE (avx_unaligned_erms_rtm);
> > -
> > + {
> > + if (CPU_FEATURES_ARCH_P (cpu_features,
> > + Prefer_Page_Unrolled_Large_Copy))
> > + return OPTIMIZE (avx_unaligned_erms_page_unrolled_rtm);
> > + return OPTIMIZE (avx_unaligned_erms_rtm);
> > + }
> > + if (CPU_FEATURES_ARCH_P (cpu_features,
> > + Prefer_Page_Unrolled_Large_Copy))
> > + return OPTIMIZE (avx_unaligned_page_unrolled_rtm);
> > return OPTIMIZE (avx_unaligned_rtm);
> > }
> >
> > - if (X86_ISA_CPU_FEATURES_ARCH_P (cpu_features,
> > - Prefer_No_VZEROUPPER, !))
> > + if (X86_ISA_CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER, !))
> > {
> > if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
> > - return OPTIMIZE (avx_unaligned_erms);
> > -
> > + {
> > + if (CPU_FEATURES_ARCH_P (cpu_features,
> > + Prefer_Page_Unrolled_Large_Copy))
> > + return OPTIMIZE (avx_unaligned_erms_page_unrolled);
> > + return OPTIMIZE (avx_unaligned_erms);
> > + }
> > + if (CPU_FEATURES_ARCH_P (cpu_features,
> > + Prefer_Page_Unrolled_Large_Copy))
> > + return OPTIMIZE (avx_unaligned_page_unrolled);
> > return OPTIMIZE (avx_unaligned);
> > }
> > }
>
> Ok.
>
> > if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, SSSE3)
> > /* Leave this as runtime check. The SSSE3 is optimized almost
> > - exclusively for avoiding unaligned memory access during the
> > - copy and by and large is not better than the sse2
> > - implementation as a general purpose memmove. */
> > + exclusively for avoiding unaligned memory access during the
> > + copy and by and large is not better than the sse2
> > + implementation as a general purpose memmove. */
> > && !CPU_FEATURES_ARCH_P (cpu_features, Fast_Unaligned_Copy))
>
> Unneeded formatting change, but ok.
>
> > diff --git a/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-page-unrolled-rtm.S b/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-page-unrolled-rtm.S
> > new file mode 100644
> > index 0000000000..683d903243
> > --- /dev/null
> > +++ b/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-page-unrolled-rtm.S
> > @@ -0,0 +1,5 @@
> > +#ifndef MEMMOVE_SYMBOL
> > +# define MEMMOVE_SYMBOL(p,s) p##_avx_##s##_page_unrolled_rtm
> > +#endif
> > +#define MEMMOVE_VEC_LARGE_IMPL "memmove-vec-large-page-unrolled.S"
> > +#include "memmove-avx-unaligned-erms-rtm.S"
>
> Ok.
>
> > diff --git a/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-page-unrolled.S b/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-page-unrolled.S
> > new file mode 100644
> > index 0000000000..57b518e16f
> > --- /dev/null
> > +++ b/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-page-unrolled.S
> > @@ -0,0 +1,5 @@
> > +#ifndef MEMMOVE_SYMBOL
> > +# define MEMMOVE_SYMBOL(p,s) p##_avx_##s##_page_unrolled
> > +#endif
> > +#define MEMMOVE_VEC_LARGE_IMPL "memmove-vec-large-page-unrolled.S"
> > +#include "memmove-avx-unaligned-erms.S"
>
> Ok.
>
> > diff --git a/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-rtm.S b/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-rtm.S
> > index 20746e6713..36e864e935 100644
> > --- a/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-rtm.S
> > +++ b/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-rtm.S
> > @@ -2,7 +2,9 @@
> >
> > # include "x86-avx-rtm-vecs.h"
> >
> > +#ifndef MEMMOVE_SYMBOL
> > # define MEMMOVE_SYMBOL(p,s) p##_avx_##s##_rtm
> > +#endif
> >
> > # include "memmove-vec-unaligned-erms.S"
> > #endif
>
> Ok. Inconsistent blank lines compared to the others, but meh.
>
> > diff --git a/sysdeps/x86_64/multiarch/memmove-evex-unaligned-erms-page-unrolled.S b/sysdeps/x86_64/multiarch/memmove-evex-unaligned-erms-page-unrolled.S
> > new file mode 100644
> > index 0000000000..371b454819
> > --- /dev/null
> > +++ b/sysdeps/x86_64/multiarch/memmove-evex-unaligned-erms-page-unrolled.S
> > @@ -0,0 +1,5 @@
> > +#ifndef MEMMOVE_SYMBOL
> > +# define MEMMOVE_SYMBOL(p,s) p##_evex_##s##_page_unrolled
> > +#endif
> > +#define MEMMOVE_VEC_LARGE_IMPL "memmove-vec-large-page-unrolled.S"
> > +#include "memmove-evex-unaligned-erms.S"
>
> Ok.
>
More information about the Libc-alpha
mailing list