[PATCH v3 3/3] x86/string: Add version of memmove with page unrolled large impl
DJ Delorie
dj@redhat.com
Tue Nov 25 19:12:12 GMT 2025
Note: I'm not checking for performance here; the other thread discussed
it already.
One minor change needed (missing newline), otherwise OK.
Reviewed-by: DJ Delorie <dj@redhat.com>
Noah Goldstein <goldstein.w.n@gmail.com> writes:
> diff --git a/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def b/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def
> index 0f14aaf071..5943fc1423 100644
> --- a/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def
> +++ b/sysdeps/x86/include/cpu-features-preferred_feature_index_1.def
> @@ -35,3 +35,4 @@ BIT (Prefer_FSRM)
> BIT (Avoid_Short_Distance_REP_MOVSB)
> BIT (Avoid_Non_Temporal_Memset)
> BIT (Avoid_STOSB)
> +BIT (Prefer_Page_Unrolled_Large_Copy)
> \ No newline at end of file
Newline needed.
> diff --git a/sysdeps/x86/tst-hwcap-tunables.c b/sysdeps/x86/tst-hwcap-tunables.c
> index 3e06048dcc..985153fb38 100644
> --- a/sysdeps/x86/tst-hwcap-tunables.c
> +++ b/sysdeps/x86/tst-hwcap-tunables.c
> @@ -61,7 +61,7 @@ static const struct test_t
> "-Prefer_ERMS,-Prefer_FSRM,-AVX,-AVX2,-AVX512F,-AVX512VL,"
> "-SSE4_1,-SSE4_2,-SSSE3,-Fast_Unaligned_Load,-ERMS,"
> "-AVX_Fast_Unaligned_Load,-Avoid_Non_Temporal_Memset,"
> - "-Avoid_STOSB",
> + "-Avoid_STOSB,-Prefer_Page_Unrolled_Large_Copy",
> test_1,
> array_length (test_1)
> },
> @@ -70,7 +70,7 @@ static const struct test_t
> ",-,-Prefer_ERMS,-Prefer_FSRM,-AVX,-AVX2,-AVX512F,-AVX512VL,"
> "-SSE4_1,-SSE4_2,-SSSE3,-Fast_Unaligned_Load,,-,"
> "-ERMS,-AVX_Fast_Unaligned_Load,-Avoid_Non_Temporal_Memset,"
> - "-Avoid_STOSB,-,",
> + "-Avoid_STOSB,-Prefer_Page_Unrolled_Large_Copy,-,",
> test_1,
> array_length (test_1)
> }
Ok.
> diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile
> index 696cb66991..381eaef455 100644
> --- a/sysdeps/x86_64/multiarch/Makefile
> +++ b/sysdeps/x86_64/multiarch/Makefile
> @@ -16,11 +16,14 @@ sysdep_routines += \
> memcmpeq-evex \
> memcmpeq-sse2 \
> memmove-avx-unaligned-erms \
> + memmove-avx-unaligned-erms-page-unrolled \
> + memmove-avx-unaligned-erms-page-unrolled-rtm \
> memmove-avx-unaligned-erms-rtm \
> memmove-avx512-no-vzeroupper \
> memmove-avx512-unaligned-erms \
> memmove-erms \
> memmove-evex-unaligned-erms \
> + memmove-evex-unaligned-erms-page-unrolled \
> memmove-sse2-unaligned-erms \
> memmove-ssse3 \
> memrchr-avx2 \
avx-erms-unrolled
avx-erms-unrolled-rtm
evex-erms-unrolled
each *.o has both erms and non-erms functions
> diff --git a/sysdeps/x86_64/multiarch/ifunc-memmove.h b/sysdeps/x86_64/multiarch/ifunc-memmove.h
> index de0ac73a2a..6d5df8a9eb 100644
> --- a/sysdeps/x86_64/multiarch/ifunc-memmove.h
> +++ b/sysdeps/x86_64/multiarch/ifunc-memmove.h
> @@ -28,18 +28,27 @@ extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_unaligned_erms)
> extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_no_vzeroupper)
> attribute_hidden;
>
> -extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_unaligned)
> - attribute_hidden;
> -extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_unaligned_erms)
> - attribute_hidden;
> +extern __typeof (REDIRECT_NAME) OPTIMIZE (evex_unaligned) attribute_hidden;
> +extern __typeof (REDIRECT_NAME)
> + OPTIMIZE (evex_unaligned_page_unrolled) attribute_hidden;
> +extern __typeof (REDIRECT_NAME)
> + OPTIMIZE (evex_unaligned_erms) attribute_hidden;
> +extern __typeof (REDIRECT_NAME)
> + OPTIMIZE (evex_unaligned_erms_page_unrolled) attribute_hidden;
evex-unrolled NA
evex-erms-unrolled
> extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned) attribute_hidden;
> -extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned_erms)
> - attribute_hidden;
> -extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned_rtm)
> - attribute_hidden;
> -extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned_erms_rtm)
> - attribute_hidden;
> +extern __typeof (REDIRECT_NAME)
> + OPTIMIZE (avx_unaligned_page_unrolled) attribute_hidden;
> +extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned_erms) attribute_hidden;
> +extern __typeof (REDIRECT_NAME)
> + OPTIMIZE (avx_unaligned_erms_page_unrolled) attribute_hidden;
> +extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned_rtm) attribute_hidden;
> +extern __typeof (REDIRECT_NAME)
> + OPTIMIZE (avx_unaligned_page_unrolled_rtm) attribute_hidden;
> +extern __typeof (REDIRECT_NAME)
> + OPTIMIZE (avx_unaligned_erms_rtm) attribute_hidden;
> +extern __typeof (REDIRECT_NAME)
> + OPTIMIZE (avx_unaligned_erms_page_unrolled_rtm) attribute_hidden;
avx-unrolled
avx-erms-unrolled OK
avx-unrolled-rtm NA
avx-erms-unrolled-rtm
Ok, everything matches.
> extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden;
>
> @@ -71,40 +80,60 @@ IFUNC_SELECTOR (void)
> return OPTIMIZE (avx512_no_vzeroupper);
> }
>
> - if (X86_ISA_CPU_FEATURES_ARCH_P (cpu_features,
> - AVX_Fast_Unaligned_Load, ))
> + if (X86_ISA_CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load, ))
> {
> if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512VL))
> {
> if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
> - return OPTIMIZE (evex_unaligned_erms);
> -
> + {
> + if (CPU_FEATURES_ARCH_P (cpu_features,
> + Prefer_Page_Unrolled_Large_Copy))
> + return OPTIMIZE (evex_unaligned_erms_page_unrolled);
> + return OPTIMIZE (evex_unaligned_erms);
> + }
> +
> + if (CPU_FEATURES_ARCH_P (cpu_features,
> + Prefer_Page_Unrolled_Large_Copy))
> + return OPTIMIZE (evex_unaligned_page_unrolled);
> return OPTIMIZE (evex_unaligned);
> }
>
> if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
> {
> if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
> - return OPTIMIZE (avx_unaligned_erms_rtm);
> -
> + {
> + if (CPU_FEATURES_ARCH_P (cpu_features,
> + Prefer_Page_Unrolled_Large_Copy))
> + return OPTIMIZE (avx_unaligned_erms_page_unrolled_rtm);
> + return OPTIMIZE (avx_unaligned_erms_rtm);
> + }
> + if (CPU_FEATURES_ARCH_P (cpu_features,
> + Prefer_Page_Unrolled_Large_Copy))
> + return OPTIMIZE (avx_unaligned_page_unrolled_rtm);
> return OPTIMIZE (avx_unaligned_rtm);
> }
>
> - if (X86_ISA_CPU_FEATURES_ARCH_P (cpu_features,
> - Prefer_No_VZEROUPPER, !))
> + if (X86_ISA_CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER, !))
> {
> if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
> - return OPTIMIZE (avx_unaligned_erms);
> -
> + {
> + if (CPU_FEATURES_ARCH_P (cpu_features,
> + Prefer_Page_Unrolled_Large_Copy))
> + return OPTIMIZE (avx_unaligned_erms_page_unrolled);
> + return OPTIMIZE (avx_unaligned_erms);
> + }
> + if (CPU_FEATURES_ARCH_P (cpu_features,
> + Prefer_Page_Unrolled_Large_Copy))
> + return OPTIMIZE (avx_unaligned_page_unrolled);
> return OPTIMIZE (avx_unaligned);
> }
> }
Ok.
> if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, SSSE3)
> /* Leave this as runtime check. The SSSE3 is optimized almost
> - exclusively for avoiding unaligned memory access during the
> - copy and by and large is not better than the sse2
> - implementation as a general purpose memmove. */
> + exclusively for avoiding unaligned memory access during the
> + copy and by and large is not better than the sse2
> + implementation as a general purpose memmove. */
> && !CPU_FEATURES_ARCH_P (cpu_features, Fast_Unaligned_Copy))
Unneeded formatting change, but ok.
> diff --git a/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-page-unrolled-rtm.S b/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-page-unrolled-rtm.S
> new file mode 100644
> index 0000000000..683d903243
> --- /dev/null
> +++ b/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-page-unrolled-rtm.S
> @@ -0,0 +1,5 @@
> +#ifndef MEMMOVE_SYMBOL
> +# define MEMMOVE_SYMBOL(p,s) p##_avx_##s##_page_unrolled_rtm
> +#endif
> +#define MEMMOVE_VEC_LARGE_IMPL "memmove-vec-large-page-unrolled.S"
> +#include "memmove-avx-unaligned-erms-rtm.S"
Ok.
> diff --git a/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-page-unrolled.S b/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-page-unrolled.S
> new file mode 100644
> index 0000000000..57b518e16f
> --- /dev/null
> +++ b/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-page-unrolled.S
> @@ -0,0 +1,5 @@
> +#ifndef MEMMOVE_SYMBOL
> +# define MEMMOVE_SYMBOL(p,s) p##_avx_##s##_page_unrolled
> +#endif
> +#define MEMMOVE_VEC_LARGE_IMPL "memmove-vec-large-page-unrolled.S"
> +#include "memmove-avx-unaligned-erms.S"
Ok.
> diff --git a/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-rtm.S b/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-rtm.S
> index 20746e6713..36e864e935 100644
> --- a/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-rtm.S
> +++ b/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-rtm.S
> @@ -2,7 +2,9 @@
>
> # include "x86-avx-rtm-vecs.h"
>
> +#ifndef MEMMOVE_SYMBOL
> # define MEMMOVE_SYMBOL(p,s) p##_avx_##s##_rtm
> +#endif
>
> # include "memmove-vec-unaligned-erms.S"
> #endif
Ok. Inconsistent blank lines compared to the others, but meh.
> diff --git a/sysdeps/x86_64/multiarch/memmove-evex-unaligned-erms-page-unrolled.S b/sysdeps/x86_64/multiarch/memmove-evex-unaligned-erms-page-unrolled.S
> new file mode 100644
> index 0000000000..371b454819
> --- /dev/null
> +++ b/sysdeps/x86_64/multiarch/memmove-evex-unaligned-erms-page-unrolled.S
> @@ -0,0 +1,5 @@
> +#ifndef MEMMOVE_SYMBOL
> +# define MEMMOVE_SYMBOL(p,s) p##_evex_##s##_page_unrolled
> +#endif
> +#define MEMMOVE_VEC_LARGE_IMPL "memmove-vec-large-page-unrolled.S"
> +#include "memmove-evex-unaligned-erms.S"
Ok.
More information about the Libc-alpha
mailing list