[PATCH v3 1/3] x86/string: Factor out large memmove implemention to seperate file
Noah Goldstein
goldstein.w.n@gmail.com
Sat Nov 15 09:42:47 GMT 2025
On Sat, Nov 15, 2025 at 4:41 AM Noah Goldstein <goldstein.w.n@gmail.com> wrote:
>
> On Sat, Nov 15, 2025 at 4:33 AM Noah Goldstein <goldstein.w.n@gmail.com> wrote:
> >
> > This is to enable us to support multiple large (size greater than
> > non-temporal threshold) implementations.
> >
> > This patch has no affect on the resulting libc.so library.
> > ---
> > .../memmove-vec-large-page-unrolled.S | 272 ++++++++++++++++++
> > .../multiarch/memmove-vec-unaligned-erms.S | 272 +-----------------
> > 2 files changed, 279 insertions(+), 265 deletions(-)
> > create mode 100644 sysdeps/x86_64/multiarch/memmove-vec-large-page-unrolled.S
> >
> > diff --git a/sysdeps/x86_64/multiarch/memmove-vec-large-page-unrolled.S b/sysdeps/x86_64/multiarch/memmove-vec-large-page-unrolled.S
> > new file mode 100644
> > index 0000000000..b5b4593279
> > --- /dev/null
> > +++ b/sysdeps/x86_64/multiarch/memmove-vec-large-page-unrolled.S
> > @@ -0,0 +1,272 @@
> > +#ifdef MEMMOVE_LARGE_IMPL
> > +# error "Multiple large memmove impls included!"
> > +#endif
> > +#define MEMMOVE_LARGE_IMPL 1
> > +
> > +/* Copies large regions by copying multiple pages at once. This is
> > + beneficial on some older Intel hardware (Broadwell, Skylake, and
> > + Icelake).
> > + 1. If size < 16 * __x86_shared_non_temporal_threshold and
> > + source and destination do not page alias, copy from 2 pages
> > + at once using non-temporal stores. Page aliasing in this case is
> > + considered true if destination's page alignment - sources' page
> > + alignment is less than 8 * VEC_SIZE.
> > + 2. If size >= 16 * __x86_shared_non_temporal_threshold or source
> > + and destination do page alias copy from 4 pages at once using
> > + non-temporal stores. */
> > +
> > +#ifndef LOG_PAGE_SIZE
> > +# define LOG_PAGE_SIZE 12
> > +#endif
> > +
> > +#if PAGE_SIZE != (1 << LOG_PAGE_SIZE)
> > +# error Invalid LOG_PAGE_SIZE
> > +#endif
> > +
> > +/* Byte per page for large_memcpy inner loop. */
> > +#if VEC_SIZE == 64
> > +# define LARGE_LOAD_SIZE (VEC_SIZE * 2)
> > +#else
> > +# define LARGE_LOAD_SIZE (VEC_SIZE * 4)
> > +#endif
> > +
> > +/* Amount to shift __x86_shared_non_temporal_threshold by for
> > + bound for memcpy_large_4x. This is essentially use to to
> > + indicate that the copy is far beyond the scope of L3
> > + (assuming no user config x86_non_temporal_threshold) and to
> > + use a more aggressively unrolled loop. NB: before
> > + increasing the value also update initialization of
> > + x86_non_temporal_threshold. */
> > +#ifndef LOG_4X_MEMCPY_THRESH
> > +# define LOG_4X_MEMCPY_THRESH 4
> > +#endif
> > +
> > +#if LARGE_LOAD_SIZE == (VEC_SIZE * 2)
> > +# define LOAD_ONE_SET(base, offset, vec0, vec1, ...) \
> > + VMOVU (offset)base, vec0; \
> > + VMOVU ((offset) + VEC_SIZE)base, vec1;
> > +# define STORE_ONE_SET(base, offset, vec0, vec1, ...) \
> > + VMOVNT vec0, (offset)base; \
> > + VMOVNT vec1, ((offset) + VEC_SIZE)base;
> > +#elif LARGE_LOAD_SIZE == (VEC_SIZE * 4)
> > +# define LOAD_ONE_SET(base, offset, vec0, vec1, vec2, vec3) \
> > + VMOVU (offset)base, vec0; \
> > + VMOVU ((offset) + VEC_SIZE)base, vec1; \
> > + VMOVU ((offset) + VEC_SIZE * 2)base, vec2; \
> > + VMOVU ((offset) + VEC_SIZE * 3)base, vec3;
> > +# define STORE_ONE_SET(base, offset, vec0, vec1, vec2, vec3) \
> > + VMOVNT vec0, (offset)base; \
> > + VMOVNT vec1, ((offset) + VEC_SIZE)base; \
> > + VMOVNT vec2, ((offset) + VEC_SIZE * 2)base; \
> > + VMOVNT vec3, ((offset) + VEC_SIZE * 3)base;
> > +#else
> > +# error Invalid LARGE_LOAD_SIZE
> > +#endif
> > +
> > + .p2align 4,, 10
> > +#if (defined USE_MULTIARCH || VEC_SIZE == 16) && IS_IN (libc)
> > +L(large_memcpy_check):
> > + /* Entry from L(large_memcpy_2x) has a redundant load of
> > + __x86_shared_non_temporal_threshold(%rip). L(large_memcpy_2x)
> > + is only use for the non-erms memmove which is generally less
> > + common. */
> > +L(large_memcpy):
> > + mov __x86_shared_non_temporal_threshold(%rip), %R11_LP
> > + cmp %R11_LP, %RDX_LP
> > + jb L(more_8x_vec_check)
> > + /* To reach this point it is impossible for dst > src and
> > + overlap. Remaining to check is src > dst and overlap. rcx
> > + already contains dst - src. Negate rcx to get src - dst. If
> > + length > rcx then there is overlap and forward copy is best. */
> > + negq %rcx
> > + cmpq %rcx, %rdx
> > + ja L(more_8x_vec_forward)
> > +
> > + /* Cache align destination. First store the first 64 bytes then
> > + adjust alignments. */
> > +
> > + /* First vec was also loaded into VEC(0). */
> > +# if VEC_SIZE < 64
> > + VMOVU VEC_SIZE(%rsi), %VMM(1)
> > +# if VEC_SIZE < 32
> > + VMOVU (VEC_SIZE * 2)(%rsi), %VMM(2)
> > + VMOVU (VEC_SIZE * 3)(%rsi), %VMM(3)
> > +# endif
> > +# endif
> > + VMOVU %VMM(0), (%rdi)
> > +# if VEC_SIZE < 64
> > + VMOVU %VMM(1), VEC_SIZE(%rdi)
> > +# if VEC_SIZE < 32
> > + VMOVU %VMM(2), (VEC_SIZE * 2)(%rdi)
> > + VMOVU %VMM(3), (VEC_SIZE * 3)(%rdi)
> > +# endif
> > +# endif
> > +
> > + /* Adjust source, destination, and size. */
> > + movq %rdi, %r8
> > + andq $63, %r8
> > + /* Get the negative of offset for alignment. */
> > + subq $64, %r8
> > + /* Adjust source. */
> > + subq %r8, %rsi
> > + /* Adjust destination which should be aligned now. */
> > + subq %r8, %rdi
> > + /* Adjust length. */
> > + addq %r8, %rdx
> > +
> > + /* Test if source and destination addresses will alias. If they
> > + do the larger pipeline in large_memcpy_4x alleviated the
> > + performance drop. */
> > +
> > + /* ecx contains -(dst - src). not ecx will return dst - src - 1
> > + which works for testing aliasing. */
> > + notl %ecx
> > + movq %rdx, %r10
> > + testl $(PAGE_SIZE - VEC_SIZE * 8), %ecx
> > + jz L(large_memcpy_4x)
> > +
> > + /* r11 has __x86_shared_non_temporal_threshold. Shift it left
> > + by LOG_4X_MEMCPY_THRESH to get L(large_memcpy_4x) threshold. */
> > + shlq $LOG_4X_MEMCPY_THRESH, %r11
> > + cmp %r11, %rdx
> > + jae L(large_memcpy_4x)
> > +
> > + /* edx will store remainder size for copying tail. */
> > + andl $(PAGE_SIZE * 2 - 1), %edx
> > + /* r10 stores outer loop counter. */
> > + shrq $(LOG_PAGE_SIZE + 1), %r10
> > + /* Copy 4x VEC at a time from 2 pages. */
> > + .p2align 4
> > +L(loop_large_memcpy_2x_outer):
> > + /* ecx stores inner loop counter. */
> > + movl $(PAGE_SIZE / LARGE_LOAD_SIZE), %ecx
> > +L(loop_large_memcpy_2x_inner):
> > + PREFETCH_ONE_SET (1, (%rsi), PREFETCHED_LOAD_SIZE)
> > + PREFETCH_ONE_SET (1, (%rsi), PREFETCHED_LOAD_SIZE * 2)
> > + PREFETCH_ONE_SET (1, (%rsi), PAGE_SIZE + PREFETCHED_LOAD_SIZE)
> > + PREFETCH_ONE_SET (1, (%rsi), PAGE_SIZE + PREFETCHED_LOAD_SIZE * 2)
> > + /* Load vectors from rsi. */
> > + LOAD_ONE_SET ((%rsi), 0, %VMM(0), %VMM(1), %VMM(2), %VMM(3))
> > + LOAD_ONE_SET ((%rsi), PAGE_SIZE, %VMM(4), %VMM(5), %VMM(6), %VMM(7))
> > + subq $-LARGE_LOAD_SIZE, %rsi
> > + /* Non-temporal store vectors to rdi. */
> > + STORE_ONE_SET ((%rdi), 0, %VMM(0), %VMM(1), %VMM(2), %VMM(3))
> > + STORE_ONE_SET ((%rdi), PAGE_SIZE, %VMM(4), %VMM(5), %VMM(6), %VMM(7))
> > + subq $-LARGE_LOAD_SIZE, %rdi
> > + decl %ecx
> > + jnz L(loop_large_memcpy_2x_inner)
> > + addq $PAGE_SIZE, %rdi
> > + addq $PAGE_SIZE, %rsi
> > + decq %r10
> > + jne L(loop_large_memcpy_2x_outer)
> > + sfence
> > +
> > + /* Check if only last 4 loads are needed. */
> > + cmpl $(VEC_SIZE * 4), %edx
> > + jbe L(large_memcpy_2x_end)
> > +
> > + /* Handle the last 2 * PAGE_SIZE bytes. */
> > +L(loop_large_memcpy_2x_tail):
> > + /* Copy 4 * VEC a time forward with non-temporal stores. */
> > + PREFETCH_ONE_SET (1, (%rsi), PREFETCHED_LOAD_SIZE)
> > + PREFETCH_ONE_SET (1, (%rdi), PREFETCHED_LOAD_SIZE)
> > + VMOVU (%rsi), %VMM(0)
> > + VMOVU VEC_SIZE(%rsi), %VMM(1)
> > + VMOVU (VEC_SIZE * 2)(%rsi), %VMM(2)
> > + VMOVU (VEC_SIZE * 3)(%rsi), %VMM(3)
> > + subq $-(VEC_SIZE * 4), %rsi
> > + addl $-(VEC_SIZE * 4), %edx
> > + VMOVA %VMM(0), (%rdi)
> > + VMOVA %VMM(1), VEC_SIZE(%rdi)
> > + VMOVA %VMM(2), (VEC_SIZE * 2)(%rdi)
> > + VMOVA %VMM(3), (VEC_SIZE * 3)(%rdi)
> > + subq $-(VEC_SIZE * 4), %rdi
> > + cmpl $(VEC_SIZE * 4), %edx
> > + ja L(loop_large_memcpy_2x_tail)
> > +
> > +L(large_memcpy_2x_end):
> > + /* Store the last 4 * VEC. */
> > + VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VMM(0)
> > + VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VMM(1)
> > + VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VMM(2)
> > + VMOVU -VEC_SIZE(%rsi, %rdx), %VMM(3)
> > +
> > + VMOVU %VMM(0), -(VEC_SIZE * 4)(%rdi, %rdx)
> > + VMOVU %VMM(1), -(VEC_SIZE * 3)(%rdi, %rdx)
> > + VMOVU %VMM(2), -(VEC_SIZE * 2)(%rdi, %rdx)
> > + VMOVU %VMM(3), -VEC_SIZE(%rdi, %rdx)
> > + VZEROUPPER_RETURN
> > +
> > + .p2align 4
> > +L(large_memcpy_4x):
> > + /* edx will store remainder size for copying tail. */
> > + andl $(PAGE_SIZE * 4 - 1), %edx
> > + /* r10 stores outer loop counter. */
> > + shrq $(LOG_PAGE_SIZE + 2), %r10
> > + /* Copy 4x VEC at a time from 4 pages. */
> > + .p2align 4
> > +L(loop_large_memcpy_4x_outer):
> > + /* ecx stores inner loop counter. */
> > + movl $(PAGE_SIZE / LARGE_LOAD_SIZE), %ecx
> > +L(loop_large_memcpy_4x_inner):
> > + /* Only one prefetch set per page as doing 4 pages give more
> > + time for prefetcher to keep up. */
> > + PREFETCH_ONE_SET (1, (%rsi), PREFETCHED_LOAD_SIZE)
> > + PREFETCH_ONE_SET (1, (%rsi), PAGE_SIZE + PREFETCHED_LOAD_SIZE)
> > + PREFETCH_ONE_SET (1, (%rsi), PAGE_SIZE * 2 + PREFETCHED_LOAD_SIZE)
> > + PREFETCH_ONE_SET (1, (%rsi), PAGE_SIZE * 3 + PREFETCHED_LOAD_SIZE)
> > + /* Load vectors from rsi. */
> > + LOAD_ONE_SET ((%rsi), 0, %VMM(0), %VMM(1), %VMM(2), %VMM(3))
> > + LOAD_ONE_SET ((%rsi), PAGE_SIZE, %VMM(4), %VMM(5), %VMM(6), %VMM(7))
> > + LOAD_ONE_SET ((%rsi), PAGE_SIZE * 2, %VMM(8), %VMM(9), %VMM(10), %VMM(11))
> > + LOAD_ONE_SET ((%rsi), PAGE_SIZE * 3, %VMM(12), %VMM(13), %VMM(14), %VMM(15))
> > + subq $-LARGE_LOAD_SIZE, %rsi
> > + /* Non-temporal store vectors to rdi. */
> > + STORE_ONE_SET ((%rdi), 0, %VMM(0), %VMM(1), %VMM(2), %VMM(3))
> > + STORE_ONE_SET ((%rdi), PAGE_SIZE, %VMM(4), %VMM(5), %VMM(6), %VMM(7))
> > + STORE_ONE_SET ((%rdi), PAGE_SIZE * 2, %VMM(8), %VMM(9), %VMM(10), %VMM(11))
> > + STORE_ONE_SET ((%rdi), PAGE_SIZE * 3, %VMM(12), %VMM(13), %VMM(14), %VMM(15))
> > + subq $-LARGE_LOAD_SIZE, %rdi
> > + decl %ecx
> > + jnz L(loop_large_memcpy_4x_inner)
> > + addq $(PAGE_SIZE * 3), %rdi
> > + addq $(PAGE_SIZE * 3), %rsi
> > + decq %r10
> > + jne L(loop_large_memcpy_4x_outer)
> > + sfence
> > + /* Check if only last 4 loads are needed. */
> > + cmpl $(VEC_SIZE * 4), %edx
> > + jbe L(large_memcpy_4x_end)
> > +
> > + /* Handle the last 4 * PAGE_SIZE bytes. */
> > +L(loop_large_memcpy_4x_tail):
> > + /* Copy 4 * VEC a time forward with non-temporal stores. */
> > + PREFETCH_ONE_SET (1, (%rsi), PREFETCHED_LOAD_SIZE)
> > + PREFETCH_ONE_SET (1, (%rdi), PREFETCHED_LOAD_SIZE)
> > + VMOVU (%rsi), %VMM(0)
> > + VMOVU VEC_SIZE(%rsi), %VMM(1)
> > + VMOVU (VEC_SIZE * 2)(%rsi), %VMM(2)
> > + VMOVU (VEC_SIZE * 3)(%rsi), %VMM(3)
> > + subq $-(VEC_SIZE * 4), %rsi
> > + addl $-(VEC_SIZE * 4), %edx
> > + VMOVA %VMM(0), (%rdi)
> > + VMOVA %VMM(1), VEC_SIZE(%rdi)
> > + VMOVA %VMM(2), (VEC_SIZE * 2)(%rdi)
> > + VMOVA %VMM(3), (VEC_SIZE * 3)(%rdi)
> > + subq $-(VEC_SIZE * 4), %rdi
> > + cmpl $(VEC_SIZE * 4), %edx
> > + ja L(loop_large_memcpy_4x_tail)
> > +
> > +L(large_memcpy_4x_end):
> > + /* Store the last 4 * VEC. */
> > + VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VMM(0)
> > + VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VMM(1)
> > + VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VMM(2)
> > + VMOVU -VEC_SIZE(%rsi, %rdx), %VMM(3)
> > +
> > + VMOVU %VMM(0), -(VEC_SIZE * 4)(%rdi, %rdx)
> > + VMOVU %VMM(1), -(VEC_SIZE * 3)(%rdi, %rdx)
> > + VMOVU %VMM(2), -(VEC_SIZE * 2)(%rdi, %rdx)
> > + VMOVU %VMM(3), -VEC_SIZE(%rdi, %rdx)
> > + VZEROUPPER_RETURN
> > +#endif
> > diff --git a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
> > index 5cd8a6286e..70d303687c 100644
> > --- a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
> > +++ b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
> > @@ -34,17 +34,8 @@
> > __x86_rep_movsb_threshold and less than
> > __x86_rep_movsb_stop_threshold, then REP MOVSB will be used.
> > 7. If size >= __x86_shared_non_temporal_threshold and there is no
> > - overlap between destination and source, use non-temporal store
> > - instead of aligned store copying from either 2 or 4 pages at
> > - once.
> > - 8. For point 7) if size < 16 * __x86_shared_non_temporal_threshold
> > - and source and destination do not page alias, copy from 2 pages
> > - at once using non-temporal stores. Page aliasing in this case is
> > - considered true if destination's page alignment - sources' page
> > - alignment is less than 8 * VEC_SIZE.
> > - 9. If size >= 16 * __x86_shared_non_temporal_threshold or source
> > - and destination do page alias copy from 4 pages at once using
> > - non-temporal stores. */
> > + overlap between destination and source, the exact method varies
> > + and is set with MEMMOVE_VEC_LARGE_IMPL". */
> >
> > #include <sysdep.h>
> >
> > @@ -95,31 +86,6 @@
> > # error Unsupported PAGE_SIZE
> > #endif
> >
> > -#ifndef LOG_PAGE_SIZE
> > -# define LOG_PAGE_SIZE 12
> > -#endif
> > -
> > -#if PAGE_SIZE != (1 << LOG_PAGE_SIZE)
> > -# error Invalid LOG_PAGE_SIZE
> > -#endif
> > -
> > -/* Byte per page for large_memcpy inner loop. */
> > -#if VEC_SIZE == 64
> > -# define LARGE_LOAD_SIZE (VEC_SIZE * 2)
> > -#else
> > -# define LARGE_LOAD_SIZE (VEC_SIZE * 4)
> > -#endif
> > -
> > -/* Amount to shift __x86_shared_non_temporal_threshold by for
> > - bound for memcpy_large_4x. This is essentially use to to
> > - indicate that the copy is far beyond the scope of L3
> > - (assuming no user config x86_non_temporal_threshold) and to
> > - use a more aggressively unrolled loop. NB: before
> > - increasing the value also update initialization of
> > - x86_non_temporal_threshold. */
> > -#ifndef LOG_4X_MEMCPY_THRESH
> > -# define LOG_4X_MEMCPY_THRESH 4
> > -#endif
> >
> > /* Avoid short distance rep movsb only with non-SSE vector. */
> > #ifndef AVOID_SHORT_DISTANCE_REP_MOVSB
> > @@ -160,26 +126,8 @@
> > # error Unsupported PREFETCH_SIZE!
> > #endif
> >
> > -#if LARGE_LOAD_SIZE == (VEC_SIZE * 2)
> > -# define LOAD_ONE_SET(base, offset, vec0, vec1, ...) \
> > - VMOVU (offset)base, vec0; \
> > - VMOVU ((offset) + VEC_SIZE)base, vec1;
> > -# define STORE_ONE_SET(base, offset, vec0, vec1, ...) \
> > - VMOVNT vec0, (offset)base; \
> > - VMOVNT vec1, ((offset) + VEC_SIZE)base;
> > -#elif LARGE_LOAD_SIZE == (VEC_SIZE * 4)
> > -# define LOAD_ONE_SET(base, offset, vec0, vec1, vec2, vec3) \
> > - VMOVU (offset)base, vec0; \
> > - VMOVU ((offset) + VEC_SIZE)base, vec1; \
> > - VMOVU ((offset) + VEC_SIZE * 2)base, vec2; \
> > - VMOVU ((offset) + VEC_SIZE * 3)base, vec3;
> > -# define STORE_ONE_SET(base, offset, vec0, vec1, vec2, vec3) \
> > - VMOVNT vec0, (offset)base; \
> > - VMOVNT vec1, ((offset) + VEC_SIZE)base; \
> > - VMOVNT vec2, ((offset) + VEC_SIZE * 2)base; \
> > - VMOVNT vec3, ((offset) + VEC_SIZE * 3)base;
> > -#else
> > -# error Invalid LARGE_LOAD_SIZE
> > +#ifndef MEMMOVE_VEC_LARGE_IMPL
> > +# define MEMMOVE_VEC_LARGE_IMPL "memmove-vec-large-page-unrolled.S"
> > #endif
> >
> > #ifndef SECTION
> > @@ -426,7 +374,7 @@ L(more_8x_vec):
> > #if (defined USE_MULTIARCH || VEC_SIZE == 16) && IS_IN (libc)
> > /* Check non-temporal store threshold. */
> > cmp __x86_shared_non_temporal_threshold(%rip), %RDX_LP
> > - ja L(large_memcpy_2x)
> > + ja L(large_memcpy)
> > #endif
> > /* To reach this point there cannot be overlap and dst > src. So
> > check for overlap and src > dst in which case correctness
> > @@ -613,7 +561,7 @@ L(movsb):
> > /* If above __x86_rep_movsb_stop_threshold most likely is
> > candidate for NT moves as well. */
> > cmp __x86_rep_movsb_stop_threshold(%rip), %RDX_LP
> > - jae L(large_memcpy_2x_check)
> > + jae L(large_memcpy_check)
> > # if AVOID_SHORT_DISTANCE_REP_MOVSB || ALIGN_MOVSB
> > /* Only avoid short movsb if CPU has FSRM. */
> > # if X86_STRING_CONTROL_AVOID_SHORT_DISTANCE_REP_MOVSB < 256
> > @@ -673,214 +621,8 @@ L(skip_short_movsb_check):
> > # endif
> > #endif
> >
> > - .p2align 4,, 10
> > -#if (defined USE_MULTIARCH || VEC_SIZE == 16) && IS_IN (libc)
> > -L(large_memcpy_2x_check):
> > - /* Entry from L(large_memcpy_2x) has a redundant load of
> > - __x86_shared_non_temporal_threshold(%rip). L(large_memcpy_2x)
> > - is only use for the non-erms memmove which is generally less
> > - common. */
> > -L(large_memcpy_2x):
> > - mov __x86_shared_non_temporal_threshold(%rip), %R11_LP
> > - cmp %R11_LP, %RDX_LP
> > - jb L(more_8x_vec_check)
> > - /* To reach this point it is impossible for dst > src and
> > - overlap. Remaining to check is src > dst and overlap. rcx
> > - already contains dst - src. Negate rcx to get src - dst. If
> > - length > rcx then there is overlap and forward copy is best. */
> > - negq %rcx
> > - cmpq %rcx, %rdx
> > - ja L(more_8x_vec_forward)
> > -
> > - /* Cache align destination. First store the first 64 bytes then
> > - adjust alignments. */
> > -
> > - /* First vec was also loaded into VEC(0). */
> > -# if VEC_SIZE < 64
> > - VMOVU VEC_SIZE(%rsi), %VMM(1)
> > -# if VEC_SIZE < 32
> > - VMOVU (VEC_SIZE * 2)(%rsi), %VMM(2)
> > - VMOVU (VEC_SIZE * 3)(%rsi), %VMM(3)
> > -# endif
> > -# endif
> > - VMOVU %VMM(0), (%rdi)
> > -# if VEC_SIZE < 64
> > - VMOVU %VMM(1), VEC_SIZE(%rdi)
> > -# if VEC_SIZE < 32
> > - VMOVU %VMM(2), (VEC_SIZE * 2)(%rdi)
> > - VMOVU %VMM(3), (VEC_SIZE * 3)(%rdi)
> > -# endif
> > -# endif
> > +#include MEMMOVE_VEC_LARGE_IMPL
> >
> > - /* Adjust source, destination, and size. */
> > - movq %rdi, %r8
> > - andq $63, %r8
> > - /* Get the negative of offset for alignment. */
> > - subq $64, %r8
> > - /* Adjust source. */
> > - subq %r8, %rsi
> > - /* Adjust destination which should be aligned now. */
> > - subq %r8, %rdi
> > - /* Adjust length. */
> > - addq %r8, %rdx
> > -
> > - /* Test if source and destination addresses will alias. If they
> > - do the larger pipeline in large_memcpy_4x alleviated the
> > - performance drop. */
> > -
> > - /* ecx contains -(dst - src). not ecx will return dst - src - 1
> > - which works for testing aliasing. */
> > - notl %ecx
> > - movq %rdx, %r10
> > - testl $(PAGE_SIZE - VEC_SIZE * 8), %ecx
> > - jz L(large_memcpy_4x)
> > -
> > - /* r11 has __x86_shared_non_temporal_threshold. Shift it left
> > - by LOG_4X_MEMCPY_THRESH to get L(large_memcpy_4x) threshold.
> > - */
> > - shlq $LOG_4X_MEMCPY_THRESH, %r11
> > - cmp %r11, %rdx
> > - jae L(large_memcpy_4x)
> > -
> > - /* edx will store remainder size for copying tail. */
> > - andl $(PAGE_SIZE * 2 - 1), %edx
> > - /* r10 stores outer loop counter. */
> > - shrq $(LOG_PAGE_SIZE + 1), %r10
> > - /* Copy 4x VEC at a time from 2 pages. */
> > - .p2align 4
> > -L(loop_large_memcpy_2x_outer):
> > - /* ecx stores inner loop counter. */
> > - movl $(PAGE_SIZE / LARGE_LOAD_SIZE), %ecx
> > -L(loop_large_memcpy_2x_inner):
> > - PREFETCH_ONE_SET(1, (%rsi), PREFETCHED_LOAD_SIZE)
> > - PREFETCH_ONE_SET(1, (%rsi), PREFETCHED_LOAD_SIZE * 2)
> > - PREFETCH_ONE_SET(1, (%rsi), PAGE_SIZE + PREFETCHED_LOAD_SIZE)
> > - PREFETCH_ONE_SET(1, (%rsi), PAGE_SIZE + PREFETCHED_LOAD_SIZE * 2)
> > - /* Load vectors from rsi. */
> > - LOAD_ONE_SET((%rsi), 0, %VMM(0), %VMM(1), %VMM(2), %VMM(3))
> > - LOAD_ONE_SET((%rsi), PAGE_SIZE, %VMM(4), %VMM(5), %VMM(6), %VMM(7))
> > - subq $-LARGE_LOAD_SIZE, %rsi
> > - /* Non-temporal store vectors to rdi. */
> > - STORE_ONE_SET((%rdi), 0, %VMM(0), %VMM(1), %VMM(2), %VMM(3))
> > - STORE_ONE_SET((%rdi), PAGE_SIZE, %VMM(4), %VMM(5), %VMM(6), %VMM(7))
> > - subq $-LARGE_LOAD_SIZE, %rdi
> > - decl %ecx
> > - jnz L(loop_large_memcpy_2x_inner)
> > - addq $PAGE_SIZE, %rdi
> > - addq $PAGE_SIZE, %rsi
> > - decq %r10
> > - jne L(loop_large_memcpy_2x_outer)
> > - sfence
> > -
> > - /* Check if only last 4 loads are needed. */
> > - cmpl $(VEC_SIZE * 4), %edx
> > - jbe L(large_memcpy_2x_end)
> > -
> > - /* Handle the last 2 * PAGE_SIZE bytes. */
> > -L(loop_large_memcpy_2x_tail):
> > - /* Copy 4 * VEC a time forward with non-temporal stores. */
> > - PREFETCH_ONE_SET (1, (%rsi), PREFETCHED_LOAD_SIZE)
> > - PREFETCH_ONE_SET (1, (%rdi), PREFETCHED_LOAD_SIZE)
> > - VMOVU (%rsi), %VMM(0)
> > - VMOVU VEC_SIZE(%rsi), %VMM(1)
> > - VMOVU (VEC_SIZE * 2)(%rsi), %VMM(2)
> > - VMOVU (VEC_SIZE * 3)(%rsi), %VMM(3)
> > - subq $-(VEC_SIZE * 4), %rsi
> > - addl $-(VEC_SIZE * 4), %edx
> > - VMOVA %VMM(0), (%rdi)
> > - VMOVA %VMM(1), VEC_SIZE(%rdi)
> > - VMOVA %VMM(2), (VEC_SIZE * 2)(%rdi)
> > - VMOVA %VMM(3), (VEC_SIZE * 3)(%rdi)
> > - subq $-(VEC_SIZE * 4), %rdi
> > - cmpl $(VEC_SIZE * 4), %edx
> > - ja L(loop_large_memcpy_2x_tail)
> > -
> > -L(large_memcpy_2x_end):
> > - /* Store the last 4 * VEC. */
> > - VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VMM(0)
> > - VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VMM(1)
> > - VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VMM(2)
> > - VMOVU -VEC_SIZE(%rsi, %rdx), %VMM(3)
> > -
> > - VMOVU %VMM(0), -(VEC_SIZE * 4)(%rdi, %rdx)
> > - VMOVU %VMM(1), -(VEC_SIZE * 3)(%rdi, %rdx)
> > - VMOVU %VMM(2), -(VEC_SIZE * 2)(%rdi, %rdx)
> > - VMOVU %VMM(3), -VEC_SIZE(%rdi, %rdx)
> > - VZEROUPPER_RETURN
> > -
> > - .p2align 4
> > -L(large_memcpy_4x):
> > - /* edx will store remainder size for copying tail. */
> > - andl $(PAGE_SIZE * 4 - 1), %edx
> > - /* r10 stores outer loop counter. */
> > - shrq $(LOG_PAGE_SIZE + 2), %r10
> > - /* Copy 4x VEC at a time from 4 pages. */
> > - .p2align 4
> > -L(loop_large_memcpy_4x_outer):
> > - /* ecx stores inner loop counter. */
> > - movl $(PAGE_SIZE / LARGE_LOAD_SIZE), %ecx
> > -L(loop_large_memcpy_4x_inner):
> > - /* Only one prefetch set per page as doing 4 pages give more
> > - time for prefetcher to keep up. */
> > - PREFETCH_ONE_SET(1, (%rsi), PREFETCHED_LOAD_SIZE)
> > - PREFETCH_ONE_SET(1, (%rsi), PAGE_SIZE + PREFETCHED_LOAD_SIZE)
> > - PREFETCH_ONE_SET(1, (%rsi), PAGE_SIZE * 2 + PREFETCHED_LOAD_SIZE)
> > - PREFETCH_ONE_SET(1, (%rsi), PAGE_SIZE * 3 + PREFETCHED_LOAD_SIZE)
> > - /* Load vectors from rsi. */
> > - LOAD_ONE_SET((%rsi), 0, %VMM(0), %VMM(1), %VMM(2), %VMM(3))
> > - LOAD_ONE_SET((%rsi), PAGE_SIZE, %VMM(4), %VMM(5), %VMM(6), %VMM(7))
> > - LOAD_ONE_SET((%rsi), PAGE_SIZE * 2, %VMM(8), %VMM(9), %VMM(10), %VMM(11))
> > - LOAD_ONE_SET((%rsi), PAGE_SIZE * 3, %VMM(12), %VMM(13), %VMM(14), %VMM(15))
> > - subq $-LARGE_LOAD_SIZE, %rsi
> > - /* Non-temporal store vectors to rdi. */
> > - STORE_ONE_SET((%rdi), 0, %VMM(0), %VMM(1), %VMM(2), %VMM(3))
> > - STORE_ONE_SET((%rdi), PAGE_SIZE, %VMM(4), %VMM(5), %VMM(6), %VMM(7))
> > - STORE_ONE_SET((%rdi), PAGE_SIZE * 2, %VMM(8), %VMM(9), %VMM(10), %VMM(11))
> > - STORE_ONE_SET((%rdi), PAGE_SIZE * 3, %VMM(12), %VMM(13), %VMM(14), %VMM(15))
> > - subq $-LARGE_LOAD_SIZE, %rdi
> > - decl %ecx
> > - jnz L(loop_large_memcpy_4x_inner)
> > - addq $(PAGE_SIZE * 3), %rdi
> > - addq $(PAGE_SIZE * 3), %rsi
> > - decq %r10
> > - jne L(loop_large_memcpy_4x_outer)
> > - sfence
> > - /* Check if only last 4 loads are needed. */
> > - cmpl $(VEC_SIZE * 4), %edx
> > - jbe L(large_memcpy_4x_end)
> > -
> > - /* Handle the last 4 * PAGE_SIZE bytes. */
> > -L(loop_large_memcpy_4x_tail):
> > - /* Copy 4 * VEC a time forward with non-temporal stores. */
> > - PREFETCH_ONE_SET (1, (%rsi), PREFETCHED_LOAD_SIZE)
> > - PREFETCH_ONE_SET (1, (%rdi), PREFETCHED_LOAD_SIZE)
> > - VMOVU (%rsi), %VMM(0)
> > - VMOVU VEC_SIZE(%rsi), %VMM(1)
> > - VMOVU (VEC_SIZE * 2)(%rsi), %VMM(2)
> > - VMOVU (VEC_SIZE * 3)(%rsi), %VMM(3)
> > - subq $-(VEC_SIZE * 4), %rsi
> > - addl $-(VEC_SIZE * 4), %edx
> > - VMOVA %VMM(0), (%rdi)
> > - VMOVA %VMM(1), VEC_SIZE(%rdi)
> > - VMOVA %VMM(2), (VEC_SIZE * 2)(%rdi)
> > - VMOVA %VMM(3), (VEC_SIZE * 3)(%rdi)
> > - subq $-(VEC_SIZE * 4), %rdi
> > - cmpl $(VEC_SIZE * 4), %edx
> > - ja L(loop_large_memcpy_4x_tail)
> > -
> > -L(large_memcpy_4x_end):
> > - /* Store the last 4 * VEC. */
> > - VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VMM(0)
> > - VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VMM(1)
> > - VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VMM(2)
> > - VMOVU -VEC_SIZE(%rsi, %rdx), %VMM(3)
> > -
> > - VMOVU %VMM(0), -(VEC_SIZE * 4)(%rdi, %rdx)
> > - VMOVU %VMM(1), -(VEC_SIZE * 3)(%rdi, %rdx)
> > - VMOVU %VMM(2), -(VEC_SIZE * 2)(%rdi, %rdx)
> > - VMOVU %VMM(3), -VEC_SIZE(%rdi, %rdx)
> > - VZEROUPPER_RETURN
> > -#endif
> > END (MEMMOVE_SYMBOL (__memmove, unaligned_erms))
> >
> > #if IS_IN (libc)
> > --
> > 2.43.0
> >
>
> For the entire series:
> I tested this natively on x86_64 when I wrote this, but I no longer
> have any x86 machines.
> I re-tested w/ QEMU before re-posting, but I think it would be worthwhile if
... if someone could re-check natively before commit (after review has
been completed).
More information about the Libc-alpha
mailing list