[PATCH v6 6/8] x86: Optimize memrchr-avx2.S
H.J. Lu
hjl.tools@gmail.com
Tue Jun 7 18:17:26 GMT 2022
On Mon, Jun 6, 2022 at 9:11 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote:
>
> The new code:
> 1. prioritizes smaller user-arg lengths more.
> 2. optimizes target placement more carefully
> 3. reuses logic more
> 4. fixes up various inefficiencies in the logic. The biggest
> case here is the `lzcnt` logic for checking returns which
> saves either a branch or multiple instructions.
>
> The total code size saving is: 306 bytes
> Geometric Mean of all benchmarks New / Old: 0.760
>
> Regressions:
> There are some regressions. Particularly where the length (user arg
> length) is large but the position of the match char is near the
> beginning of the string (in first VEC). This case has roughly a
> 10-20% regression.
>
> This is because the new logic gives the hot path for immediate matches
> to shorter lengths (the more common input). This case has roughly
> a 15-45% speedup.
>
> Full xcheck passes on x86_64.
> ---
> sysdeps/x86_64/multiarch/memrchr-avx2-rtm.S | 1 +
> sysdeps/x86_64/multiarch/memrchr-avx2.S | 534 ++++++++++----------
> 2 files changed, 257 insertions(+), 278 deletions(-)
>
> diff --git a/sysdeps/x86_64/multiarch/memrchr-avx2-rtm.S b/sysdeps/x86_64/multiarch/memrchr-avx2-rtm.S
> index cea2d2a72d..5e9beeeef2 100644
> --- a/sysdeps/x86_64/multiarch/memrchr-avx2-rtm.S
> +++ b/sysdeps/x86_64/multiarch/memrchr-avx2-rtm.S
> @@ -2,6 +2,7 @@
> # define MEMRCHR __memrchr_avx2_rtm
> #endif
>
> +#define COND_VZEROUPPER COND_VZEROUPPER_XTEST
> #define ZERO_UPPER_VEC_REGISTERS_RETURN \
> ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST
>
> diff --git a/sysdeps/x86_64/multiarch/memrchr-avx2.S b/sysdeps/x86_64/multiarch/memrchr-avx2.S
> index ba2ce7cb03..bea4528068 100644
> --- a/sysdeps/x86_64/multiarch/memrchr-avx2.S
> +++ b/sysdeps/x86_64/multiarch/memrchr-avx2.S
> @@ -21,340 +21,318 @@
> # include <sysdep.h>
>
> # ifndef MEMRCHR
> -# define MEMRCHR __memrchr_avx2
> +# define MEMRCHR __memrchr_avx2
> # endif
>
> # ifndef VZEROUPPER
> -# define VZEROUPPER vzeroupper
> +# define VZEROUPPER vzeroupper
> # endif
>
> # ifndef SECTION
> # define SECTION(p) p##.avx
> # endif
>
> -# define VEC_SIZE 32
> +# define VEC_SIZE 32
> +# define PAGE_SIZE 4096
> + .section SECTION(.text), "ax", @progbits
> +ENTRY(MEMRCHR)
> +# ifdef __ILP32__
> + /* Clear upper bits. */
> + and %RDX_LP, %RDX_LP
> +# else
> + test %RDX_LP, %RDX_LP
> +# endif
> + jz L(zero_0)
>
> - .section SECTION(.text),"ax",@progbits
> -ENTRY (MEMRCHR)
> - /* Broadcast CHAR to YMM0. */
> vmovd %esi, %xmm0
> - vpbroadcastb %xmm0, %ymm0
> -
> - sub $VEC_SIZE, %RDX_LP
> - jbe L(last_vec_or_less)
> -
> - add %RDX_LP, %RDI_LP
> -
> - /* Check the last VEC_SIZE bytes. */
> - vpcmpeqb (%rdi), %ymm0, %ymm1
> - vpmovmskb %ymm1, %eax
> - testl %eax, %eax
> - jnz L(last_vec_x0)
> + /* Get end pointer. Minus one for two reasons. 1) It is necessary for a
> + correct page cross check and 2) it correctly sets up end ptr to be
> + subtract by lzcnt aligned. */
> + leaq -1(%rdx, %rdi), %rax
>
> - subq $(VEC_SIZE * 4), %rdi
> - movl %edi, %ecx
> - andl $(VEC_SIZE - 1), %ecx
> - jz L(aligned_more)
> + vpbroadcastb %xmm0, %ymm0
>
> - /* Align data for aligned loads in the loop. */
> - addq $VEC_SIZE, %rdi
> - addq $VEC_SIZE, %rdx
> - andq $-VEC_SIZE, %rdi
> - subq %rcx, %rdx
> + /* Check if we can load 1x VEC without cross a page. */
> + testl $(PAGE_SIZE - VEC_SIZE), %eax
> + jz L(page_cross)
> +
> + vpcmpeqb -(VEC_SIZE - 1)(%rax), %ymm0, %ymm1
> + vpmovmskb %ymm1, %ecx
> + cmpq $VEC_SIZE, %rdx
> + ja L(more_1x_vec)
> +
> +L(ret_vec_x0_test):
> + /* If ecx is zero (no matches) lzcnt will set it 32 (VEC_SIZE) which
> + will gurantee edx (len) is less than it. */
> + lzcntl %ecx, %ecx
> +
> + /* Hoist vzeroupper (not great for RTM) to save code size. This allows
> + all logic for edx (len) <= VEC_SIZE to fit in first cache line. */
> + COND_VZEROUPPER
> + cmpl %ecx, %edx
> + jle L(zero_0)
> + subq %rcx, %rax
> + ret
>
> - .p2align 4
> -L(aligned_more):
> - subq $(VEC_SIZE * 4), %rdx
> - jbe L(last_4x_vec_or_less)
> -
> - /* Check the last 4 * VEC_SIZE. Only one VEC_SIZE at a time
> - since data is only aligned to VEC_SIZE. */
> - vpcmpeqb (VEC_SIZE * 3)(%rdi), %ymm0, %ymm1
> - vpmovmskb %ymm1, %eax
> - testl %eax, %eax
> - jnz L(last_vec_x3)
> -
> - vpcmpeqb (VEC_SIZE * 2)(%rdi), %ymm0, %ymm2
> - vpmovmskb %ymm2, %eax
> - testl %eax, %eax
> - jnz L(last_vec_x2)
> -
> - vpcmpeqb VEC_SIZE(%rdi), %ymm0, %ymm3
> - vpmovmskb %ymm3, %eax
> - testl %eax, %eax
> - jnz L(last_vec_x1)
> -
> - vpcmpeqb (%rdi), %ymm0, %ymm4
> - vpmovmskb %ymm4, %eax
> - testl %eax, %eax
> - jnz L(last_vec_x0)
> -
> - /* Align data to 4 * VEC_SIZE for loop with fewer branches.
> - There are some overlaps with above if data isn't aligned
> - to 4 * VEC_SIZE. */
> - movl %edi, %ecx
> - andl $(VEC_SIZE * 4 - 1), %ecx
> - jz L(loop_4x_vec)
> -
> - addq $(VEC_SIZE * 4), %rdi
> - addq $(VEC_SIZE * 4), %rdx
> - andq $-(VEC_SIZE * 4), %rdi
> - subq %rcx, %rdx
> + /* Fits in aligning bytes of first cache line. */
> +L(zero_0):
> + xorl %eax, %eax
> + ret
>
> - .p2align 4
> -L(loop_4x_vec):
> - /* Compare 4 * VEC at a time forward. */
> - subq $(VEC_SIZE * 4), %rdi
> - subq $(VEC_SIZE * 4), %rdx
> - jbe L(last_4x_vec_or_less)
> -
> - vmovdqa (%rdi), %ymm1
> - vmovdqa VEC_SIZE(%rdi), %ymm2
> - vmovdqa (VEC_SIZE * 2)(%rdi), %ymm3
> - vmovdqa (VEC_SIZE * 3)(%rdi), %ymm4
> -
> - vpcmpeqb %ymm1, %ymm0, %ymm1
> - vpcmpeqb %ymm2, %ymm0, %ymm2
> - vpcmpeqb %ymm3, %ymm0, %ymm3
> - vpcmpeqb %ymm4, %ymm0, %ymm4
> -
> - vpor %ymm1, %ymm2, %ymm5
> - vpor %ymm3, %ymm4, %ymm6
> - vpor %ymm5, %ymm6, %ymm5
> -
> - vpmovmskb %ymm5, %eax
> - testl %eax, %eax
> - jz L(loop_4x_vec)
> -
> - /* There is a match. */
> - vpmovmskb %ymm4, %eax
> - testl %eax, %eax
> - jnz L(last_vec_x3)
> -
> - vpmovmskb %ymm3, %eax
> - testl %eax, %eax
> - jnz L(last_vec_x2)
> -
> - vpmovmskb %ymm2, %eax
> - testl %eax, %eax
> - jnz L(last_vec_x1)
> -
> - vpmovmskb %ymm1, %eax
> - bsrl %eax, %eax
> - addq %rdi, %rax
> + .p2align 4,, 9
> +L(ret_vec_x0):
> + lzcntl %ecx, %ecx
> + subq %rcx, %rax
> L(return_vzeroupper):
> ZERO_UPPER_VEC_REGISTERS_RETURN
>
> - .p2align 4
> -L(last_4x_vec_or_less):
> - addl $(VEC_SIZE * 4), %edx
> - cmpl $(VEC_SIZE * 2), %edx
> - jbe L(last_2x_vec)
> -
> - vpcmpeqb (VEC_SIZE * 3)(%rdi), %ymm0, %ymm1
> - vpmovmskb %ymm1, %eax
> - testl %eax, %eax
> - jnz L(last_vec_x3)
> -
> - vpcmpeqb (VEC_SIZE * 2)(%rdi), %ymm0, %ymm2
> - vpmovmskb %ymm2, %eax
> - testl %eax, %eax
> - jnz L(last_vec_x2)
> -
> - vpcmpeqb VEC_SIZE(%rdi), %ymm0, %ymm3
> - vpmovmskb %ymm3, %eax
> - testl %eax, %eax
> - jnz L(last_vec_x1_check)
> - cmpl $(VEC_SIZE * 3), %edx
> - jbe L(zero)
> -
> - vpcmpeqb (%rdi), %ymm0, %ymm4
> - vpmovmskb %ymm4, %eax
> - testl %eax, %eax
> - jz L(zero)
> - bsrl %eax, %eax
> - subq $(VEC_SIZE * 4), %rdx
> - addq %rax, %rdx
> - jl L(zero)
> - addq %rdi, %rax
> - VZEROUPPER_RETURN
> -
> - .p2align 4
> + .p2align 4,, 10
> +L(more_1x_vec):
> + testl %ecx, %ecx
> + jnz L(ret_vec_x0)
> +
> + /* Align rax (string pointer). */
> + andq $-VEC_SIZE, %rax
> +
> + /* Recompute remaining length after aligning. */
> + movq %rax, %rdx
> + /* Need this comparison next no matter what. */
> + vpcmpeqb -(VEC_SIZE)(%rax), %ymm0, %ymm1
> + subq %rdi, %rdx
> + decq %rax
> + vpmovmskb %ymm1, %ecx
> + /* Fall through for short (hotter than length). */
> + cmpq $(VEC_SIZE * 2), %rdx
> + ja L(more_2x_vec)
> L(last_2x_vec):
> - vpcmpeqb (VEC_SIZE * 3)(%rdi), %ymm0, %ymm1
> - vpmovmskb %ymm1, %eax
> - testl %eax, %eax
> - jnz L(last_vec_x3_check)
> cmpl $VEC_SIZE, %edx
> - jbe L(zero)
> -
> - vpcmpeqb (VEC_SIZE * 2)(%rdi), %ymm0, %ymm1
> - vpmovmskb %ymm1, %eax
> - testl %eax, %eax
> - jz L(zero)
> - bsrl %eax, %eax
> - subq $(VEC_SIZE * 2), %rdx
> - addq %rax, %rdx
> - jl L(zero)
> - addl $(VEC_SIZE * 2), %eax
> - addq %rdi, %rax
> - VZEROUPPER_RETURN
> -
> - .p2align 4
> -L(last_vec_x0):
> - bsrl %eax, %eax
> - addq %rdi, %rax
> - VZEROUPPER_RETURN
> + jbe L(ret_vec_x0_test)
> +
> + testl %ecx, %ecx
> + jnz L(ret_vec_x0)
> +
> + vpcmpeqb -(VEC_SIZE * 2 - 1)(%rax), %ymm0, %ymm1
> + vpmovmskb %ymm1, %ecx
> + /* 64-bit lzcnt. This will naturally add 32 to position. */
> + lzcntq %rcx, %rcx
> + COND_VZEROUPPER
> + cmpl %ecx, %edx
> + jle L(zero_0)
> + subq %rcx, %rax
> + ret
>
> - .p2align 4
> -L(last_vec_x1):
> - bsrl %eax, %eax
> - addl $VEC_SIZE, %eax
> - addq %rdi, %rax
> - VZEROUPPER_RETURN
>
> - .p2align 4
> -L(last_vec_x2):
> - bsrl %eax, %eax
> - addl $(VEC_SIZE * 2), %eax
> - addq %rdi, %rax
> + /* Inexpensive place to put this regarding code size / target alignments
> + / ICache NLP. Necessary for 2-byte encoding of jump to page cross
> + case which in turn in necessary for hot path (len <= VEC_SIZE) to fit
is necessary?
> + in first cache line. */
> +L(page_cross):
> + movq %rax, %rsi
> + andq $-VEC_SIZE, %rsi
> + vpcmpeqb (%rsi), %ymm0, %ymm1
> + vpmovmskb %ymm1, %ecx
> + /* Shift out negative alignment (because we are starting from endptr and
> + working backwards). */
> + movl %eax, %r8d
> + /* notl because eax already has endptr - 1. (-x = ~(x - 1)). */
> + notl %r8d
> + shlxl %r8d, %ecx, %ecx
> + cmpq %rdi, %rsi
> + ja L(more_1x_vec)
> + lzcntl %ecx, %ecx
> + COND_VZEROUPPER
> + cmpl %ecx, %edx
> + jle L(zero_0)
> + subq %rcx, %rax
> + ret
> + .p2align 4,, 11
> +L(ret_vec_x1):
> + /* This will naturally add 32 to position. */
> + lzcntq %rcx, %rcx
> + subq %rcx, %rax
> VZEROUPPER_RETURN
> + .p2align 4,, 10
> +L(more_2x_vec):
> + testl %ecx, %ecx
> + jnz L(ret_vec_x0)
>
> - .p2align 4
> -L(last_vec_x3):
> - bsrl %eax, %eax
> - addl $(VEC_SIZE * 3), %eax
> - addq %rdi, %rax
> - ret
> + vpcmpeqb -(VEC_SIZE * 2 - 1)(%rax), %ymm0, %ymm1
> + vpmovmskb %ymm1, %ecx
> + testl %ecx, %ecx
> + jnz L(ret_vec_x1)
>
> - .p2align 4
> -L(last_vec_x1_check):
> - bsrl %eax, %eax
> - subq $(VEC_SIZE * 3), %rdx
> - addq %rax, %rdx
> - jl L(zero)
> - addl $VEC_SIZE, %eax
> - addq %rdi, %rax
> - VZEROUPPER_RETURN
>
> - .p2align 4
> -L(last_vec_x3_check):
> - bsrl %eax, %eax
> - subq $VEC_SIZE, %rdx
> - addq %rax, %rdx
> - jl L(zero)
> - addl $(VEC_SIZE * 3), %eax
> - addq %rdi, %rax
> - VZEROUPPER_RETURN
> + /* Needed no matter what. */
> + vpcmpeqb -(VEC_SIZE * 3 - 1)(%rax), %ymm0, %ymm1
> + vpmovmskb %ymm1, %ecx
>
> - .p2align 4
> -L(zero):
> - xorl %eax, %eax
> - VZEROUPPER_RETURN
> + subq $(VEC_SIZE * 4), %rdx
> + ja L(more_4x_vec)
> +
> + cmpl $(VEC_SIZE * -1), %edx
> + jle L(ret_vec_x2_test)
> +
> +L(last_vec):
> + testl %ecx, %ecx
> + jnz L(ret_vec_x2)
> +
> + /* Needed no matter what. */
> + vpcmpeqb -(VEC_SIZE * 4 - 1)(%rax), %ymm0, %ymm1
> + vpmovmskb %ymm1, %ecx
> + lzcntl %ecx, %ecx
> + subq $(VEC_SIZE * 3), %rax
> + COND_VZEROUPPER
> + subq %rcx, %rax
> + cmpq %rax, %rdi
> + ja L(zero_2)
> + ret
>
> - .p2align 4
> -L(null):
> + /* First in aligning bytes. */
> +L(zero_2):
> xorl %eax, %eax
> ret
>
> - .p2align 4
> -L(last_vec_or_less_aligned):
> - movl %edx, %ecx
> + .p2align 4,, 4
> +L(ret_vec_x2_test):
> + lzcntl %ecx, %ecx
> + subq $(VEC_SIZE * 2), %rax
> + COND_VZEROUPPER
> + subq %rcx, %rax
> + cmpq %rax, %rdi
> + ja L(zero_2)
> + ret
>
> - vpcmpeqb (%rdi), %ymm0, %ymm1
>
> - movl $1, %edx
> - /* Support rdx << 32. */
> - salq %cl, %rdx
> - subq $1, %rdx
> + .p2align 4,, 11
> +L(ret_vec_x2):
> + /* ecx must be non-zero. */
> + bsrl %ecx, %ecx
> + leaq (VEC_SIZE * -3 + 1)(%rcx, %rax), %rax
> + VZEROUPPER_RETURN
>
> - vpmovmskb %ymm1, %eax
> + .p2align 4,, 14
> +L(ret_vec_x3):
> + /* ecx must be non-zero. */
> + bsrl %ecx, %ecx
> + leaq (VEC_SIZE * -4 + 1)(%rcx, %rax), %rax
> + VZEROUPPER_RETURN
>
> - /* Remove the trailing bytes. */
> - andl %edx, %eax
> - testl %eax, %eax
> - jz L(zero)
>
> - bsrl %eax, %eax
> - addq %rdi, %rax
> - VZEROUPPER_RETURN
>
> .p2align 4
> -L(last_vec_or_less):
> - addl $VEC_SIZE, %edx
> +L(more_4x_vec):
> + testl %ecx, %ecx
> + jnz L(ret_vec_x2)
>
> - /* Check for zero length. */
> - testl %edx, %edx
> - jz L(null)
> + vpcmpeqb -(VEC_SIZE * 4 - 1)(%rax), %ymm0, %ymm1
> + vpmovmskb %ymm1, %ecx
>
> - movl %edi, %ecx
> - andl $(VEC_SIZE - 1), %ecx
> - jz L(last_vec_or_less_aligned)
> + testl %ecx, %ecx
> + jnz L(ret_vec_x3)
>
> - movl %ecx, %esi
> - movl %ecx, %r8d
> - addl %edx, %esi
> - andq $-VEC_SIZE, %rdi
> + /* Check if near end before re-aligning (otherwise might do an
> + unnecissary loop iteration). */
> + addq $-(VEC_SIZE * 4), %rax
> + cmpq $(VEC_SIZE * 4), %rdx
> + jbe L(last_4x_vec)
>
> - subl $VEC_SIZE, %esi
> - ja L(last_vec_2x_aligned)
> + /* Align rax to (VEC_SIZE - 1). */
> + orq $(VEC_SIZE * 4 - 1), %rax
> + movq %rdi, %rdx
> + /* Get endptr for loop in rdx. NB: Can't just do while rax > rdi because
> + lengths that overflow can be valid and break the comparison. */
> + orq $(VEC_SIZE * 4 - 1), %rdx
>
> - /* Check the last VEC. */
> - vpcmpeqb (%rdi), %ymm0, %ymm1
> - vpmovmskb %ymm1, %eax
> -
> - /* Remove the leading and trailing bytes. */
> - sarl %cl, %eax
> - movl %edx, %ecx
> + .p2align 4
> +L(loop_4x_vec):
> + /* Need this comparison next no matter what. */
> + vpcmpeqb -(VEC_SIZE * 1 - 1)(%rax), %ymm0, %ymm1
> + vpcmpeqb -(VEC_SIZE * 2 - 1)(%rax), %ymm0, %ymm2
> + vpcmpeqb -(VEC_SIZE * 3 - 1)(%rax), %ymm0, %ymm3
> + vpcmpeqb -(VEC_SIZE * 4 - 1)(%rax), %ymm0, %ymm4
>
> - movl $1, %edx
> - sall %cl, %edx
> - subl $1, %edx
> + vpor %ymm1, %ymm2, %ymm2
> + vpor %ymm3, %ymm4, %ymm4
> + vpor %ymm2, %ymm4, %ymm4
> + vpmovmskb %ymm4, %esi
>
> - andl %edx, %eax
> - testl %eax, %eax
> - jz L(zero)
> + testl %esi, %esi
> + jnz L(loop_end)
>
> - bsrl %eax, %eax
> - addq %rdi, %rax
> - addq %r8, %rax
> - VZEROUPPER_RETURN
> + addq $(VEC_SIZE * -4), %rax
> + cmpq %rdx, %rax
> + jne L(loop_4x_vec)
>
> - .p2align 4
> -L(last_vec_2x_aligned):
> - movl %esi, %ecx
> + subl %edi, %edx
> + incl %edx
>
> - /* Check the last VEC. */
> - vpcmpeqb VEC_SIZE(%rdi), %ymm0, %ymm1
> +L(last_4x_vec):
> + /* Used no matter what. */
> + vpcmpeqb -(VEC_SIZE * 1 - 1)(%rax), %ymm0, %ymm1
> + vpmovmskb %ymm1, %ecx
>
> - movl $1, %edx
> - sall %cl, %edx
> - subl $1, %edx
> + cmpl $(VEC_SIZE * 2), %edx
> + jbe L(last_2x_vec)
>
> - vpmovmskb %ymm1, %eax
> + testl %ecx, %ecx
> + jnz L(ret_vec_x0_end)
>
> - /* Remove the trailing bytes. */
> - andl %edx, %eax
> + vpcmpeqb -(VEC_SIZE * 2 - 1)(%rax), %ymm0, %ymm1
> + vpmovmskb %ymm1, %ecx
> + testl %ecx, %ecx
> + jnz L(ret_vec_x1_end)
>
> - testl %eax, %eax
> - jnz L(last_vec_x1)
> + /* Used no matter what. */
> + vpcmpeqb -(VEC_SIZE * 3 - 1)(%rax), %ymm0, %ymm1
> + vpmovmskb %ymm1, %ecx
>
> - /* Check the second last VEC. */
> - vpcmpeqb (%rdi), %ymm0, %ymm1
> + cmpl $(VEC_SIZE * 3), %edx
> + ja L(last_vec)
> +
> + lzcntl %ecx, %ecx
> + subq $(VEC_SIZE * 2), %rax
> + COND_VZEROUPPER
> + subq %rcx, %rax
> + cmpq %rax, %rdi
> + jbe L(ret0)
> + xorl %eax, %eax
> +L(ret0):
> + ret
>
> - movl %r8d, %ecx
>
> - vpmovmskb %ymm1, %eax
> + .p2align 4
> +L(loop_end):
> + vpmovmskb %ymm1, %ecx
> + testl %ecx, %ecx
> + jnz L(ret_vec_x0_end)
> +
> + vpmovmskb %ymm2, %ecx
> + testl %ecx, %ecx
> + jnz L(ret_vec_x1_end)
> +
> + vpmovmskb %ymm3, %ecx
> + /* Combine last 2 VEC matches. If ecx (VEC3) is zero (no CHAR in VEC3)
> + then it won't affect the result in esi (VEC4). If ecx is non-zero
> + then CHAR in VEC3 and bsrq will use that position. */
> + salq $32, %rcx
> + orq %rsi, %rcx
> + bsrq %rcx, %rcx
> + leaq (VEC_SIZE * -4 + 1)(%rcx, %rax), %rax
> + VZEROUPPER_RETURN
>
> - /* Remove the leading bytes. Must use unsigned right shift for
> - bsrl below. */
> - shrl %cl, %eax
> - testl %eax, %eax
> - jz L(zero)
> + .p2align 4,, 4
> +L(ret_vec_x1_end):
> + /* 64-bit version will automatically add 32 (VEC_SIZE). */
> + lzcntq %rcx, %rcx
> + subq %rcx, %rax
> + VZEROUPPER_RETURN
>
> - bsrl %eax, %eax
> - addq %rdi, %rax
> - addq %r8, %rax
> + .p2align 4,, 4
> +L(ret_vec_x0_end):
> + lzcntl %ecx, %ecx
> + subq %rcx, %rax
> VZEROUPPER_RETURN
> -END (MEMRCHR)
> +
> + /* 2 bytes until next cache line. */
> +END(MEMRCHR)
> #endif
> --
> 2.34.1
>
OK with the updated comments.
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
Thanks.
--
H.J.
More information about the Libc-alpha
mailing list