[RFC PATCH 16/19] riscv: Add accelerated strcmp routines
Xi Ruoyao
xry111@xry111.site
Tue Feb 7 11:57:10 GMT 2023
Is it possible to make the optimized generic string routine [1] ifunc-
aware in some way? Or can we "templatize" it so we can make vectorized
string routines more easily?
[1]: https://sourceware.org/pipermail/libc-alpha/2023-February/145211.html
On Tue, 2023-02-07 at 01:16 +0100, Christoph Muellner wrote:
> From: Christoph Müllner <christoph.muellner@vrull.eu>
>
> The implementation of strcmp() can be accelerated using Zbb's orc.b
> instruction and fast unaligned accesses. Howver, strcmp can use
> unaligned accesses only if such an address does not change the
> exception behaviour (compared to a single-byte compare loop).
> Let's add an implementation that does all that.
> Additionally, let's add the strcmp implementation from the
> Bitmanip specification, which does not do any unaligned accesses.
>
> Signed-off-by: Christoph Müllner <christoph.muellner@vrull.eu>
> ---
> sysdeps/riscv/multiarch/Makefile | 4 +-
> sysdeps/riscv/multiarch/ifunc-impl-list.c | 4 +-
> sysdeps/riscv/multiarch/strcmp.c | 11 +-
> sysdeps/riscv/multiarch/strcmp_zbb.S | 104 +++++++++
> .../riscv/multiarch/strcmp_zbb_unaligned.S | 213
> ++++++++++++++++++
> 5 files changed, 332 insertions(+), 4 deletions(-)
> create mode 100644 sysdeps/riscv/multiarch/strcmp_zbb.S
> create mode 100644 sysdeps/riscv/multiarch/strcmp_zbb_unaligned.S
>
> diff --git a/sysdeps/riscv/multiarch/Makefile
> b/sysdeps/riscv/multiarch/Makefile
> index 3017bde75a..73a62be85d 100644
> --- a/sysdeps/riscv/multiarch/Makefile
> +++ b/sysdeps/riscv/multiarch/Makefile
> @@ -11,5 +11,7 @@ sysdep_routines += \
> strlen_generic \
> strlen_zbb \
> \
> - strcmp_generic
> + strcmp_generic \
> + strcmp_zbb \
> + strcmp_zbb_unaligned
> endif
> diff --git a/sysdeps/riscv/multiarch/ifunc-impl-list.c
> b/sysdeps/riscv/multiarch/ifunc-impl-list.c
> index 64331a4c7f..d354aa1178 100644
> --- a/sysdeps/riscv/multiarch/ifunc-impl-list.c
> +++ b/sysdeps/riscv/multiarch/ifunc-impl-list.c
> @@ -59,7 +59,9 @@ __libc_ifunc_impl_list (const char *name, struct
> libc_ifunc_impl *array,
> IFUNC_IMPL_ADD (array, i, strlen, 1, __strlen_generic))
>
> IFUNC_IMPL (i, name, strcmp,
> - IFUNC_IMPL_ADD (array, i, strcpy, 1, __strcmp_generic))
> + IFUNC_IMPL_ADD (array, i, strcmp, 1,
> __strcmp_zbb_unaligned)
> + IFUNC_IMPL_ADD (array, i, strcmp, 1, __strcmp_zbb)
> + IFUNC_IMPL_ADD (array, i, strcmp, 1, __strcmp_generic))
>
> return i;
> }
> diff --git a/sysdeps/riscv/multiarch/strcmp.c
> b/sysdeps/riscv/multiarch/strcmp.c
> index 8c21a90afd..d3f2fe19ae 100644
> --- a/sysdeps/riscv/multiarch/strcmp.c
> +++ b/sysdeps/riscv/multiarch/strcmp.c
> @@ -30,8 +30,15 @@
>
> extern __typeof (__redirect_strcmp) __libc_strcmp;
> extern __typeof (__redirect_strcmp) __strcmp_generic
> attribute_hidden;
> -
> -libc_ifunc (__libc_strcmp, __strcmp_generic);
> +extern __typeof (__redirect_strcmp) __strcmp_zbb attribute_hidden;
> +extern __typeof (__redirect_strcmp) __strcmp_zbb_unaligned
> attribute_hidden;
> +
> +libc_ifunc (__libc_strcmp,
> + HAVE_RV(zbb) && HAVE_FAST_UNALIGNED()
> + ? __strcmp_zbb_unaligned
> + : HAVE_RV(zbb)
> + ? __strcmp_zbb
> + : __strcmp_generic);
>
> # undef strcmp
> strong_alias (__libc_strcmp, strcmp);
> diff --git a/sysdeps/riscv/multiarch/strcmp_zbb.S
> b/sysdeps/riscv/multiarch/strcmp_zbb.S
> new file mode 100644
> index 0000000000..1c265d6107
> --- /dev/null
> +++ b/sysdeps/riscv/multiarch/strcmp_zbb.S
> @@ -0,0 +1,104 @@
> +/* Copyright (C) 2022 Free Software Foundation, Inc.
> +
> + This file is part of the GNU C Library.
> +
> + The GNU C Library is free software; you can redistribute it and/or
> + modify it under the terms of the GNU Lesser General Public
> + License as published by the Free Software Foundation; either
> + version 2.1 of the License, or (at your option) any later version.
> +
> + The GNU C Library is distributed in the hope that it will be
> useful,
> + but WITHOUT ANY WARRANTY; without even the implied warranty of
> + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + Lesser General Public License for more details.
> +
> + You should have received a copy of the GNU Lesser General Public
> + License along with the GNU C Library. If not, see
> + <https://www.gnu.org/licenses/>. */
> +
> +#include <sysdep.h>
> +#include <sys/asm.h>
> +
> +/* Assumptions: rvi_zbb. */
> +/* Implementation from the Bitmanip specification. */
> +
> +#define src1 a0
> +#define result a0
> +#define src2 a1
> +#define data1 a2
> +#define data2 a3
> +#define align a4
> +#define data1_orcb t0
> +#define m1 t2
> +
> +#if __riscv_xlen == 64
> +# define REG_L ld
> +# define SZREG 8
> +#else
> +# define REG_L lw
> +# define SZREG 4
> +#endif
> +
> +#ifndef STRCMP
> +# define STRCMP __strcmp_zbb
> +#endif
> +
> +.option push
> +.option arch,+zbb
> +
> +ENTRY_ALIGN (STRCMP, 6)
> + or align, src1, src2
> + and align, align, SZREG-1
> + bnez align, L(simpleloop)
> + li m1, -1
> +
> + /* Main loop for aligned strings. */
> + .p2align 2
> +L(loop):
> + REG_L data1, 0(src1)
> + REG_L data2, 0(src2)
> + orc.b data1_orcb, data1
> + bne data1_orcb, m1, L(foundnull)
> + addi src1, src1, SZREG
> + addi src2, src2, SZREG
> + beq data1, data2, L(loop)
> +
> + /* Words don't match, and no null byte in the first word.
> + * Get bytes in big-endian order and compare. */
> +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
> + rev8 data1, data1
> + rev8 data2, data2
> +#endif
> + /* Synthesize (data1 >= data2) ? 1 : -1 in a branchless
> sequence. */
> + sltu result, data1, data2
> + neg result, result
> + ori result, result, 1
> + ret
> +
> +L(foundnull):
> + /* Found a null byte.
> + * If words don't match, fall back to simple loop. */
> + bne data1, data2, L(simpleloop)
> +
> + /* Otherwise, strings are equal. */
> + li result, 0
> + ret
> +
> + /* Simple loop for misaligned strings. */
> + .p2align 3
> +L(simpleloop):
> + lbu data1, 0(src1)
> + lbu data2, 0(src2)
> + addi src1, src1, 1
> + addi src2, src2, 1
> + bne data1, data2, L(sub)
> + bnez data1, L(simpleloop)
> +
> +L(sub):
> + sub result, data1, data2
> + ret
> +
> +.option pop
> +
> +END (STRCMP)
> +libc_hidden_builtin_def (STRCMP)
> diff --git a/sysdeps/riscv/multiarch/strcmp_zbb_unaligned.S
> b/sysdeps/riscv/multiarch/strcmp_zbb_unaligned.S
> new file mode 100644
> index 0000000000..ec21982b65
> --- /dev/null
> +++ b/sysdeps/riscv/multiarch/strcmp_zbb_unaligned.S
> @@ -0,0 +1,213 @@
> +/* Copyright (C) 2022 Free Software Foundation, Inc.
> +
> + This file is part of the GNU C Library.
> +
> + The GNU C Library is free software; you can redistribute it and/or
> + modify it under the terms of the GNU Lesser General Public
> + License as published by the Free Software Foundation; either
> + version 2.1 of the License, or (at your option) any later version.
> +
> + The GNU C Library is distributed in the hope that it will be
> useful,
> + but WITHOUT ANY WARRANTY; without even the implied warranty of
> + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + Lesser General Public License for more details.
> +
> + You should have received a copy of the GNU Lesser General Public
> + License along with the GNU C Library. If not, see
> + <https://www.gnu.org/licenses/>. */
> +
> +#include <sysdep.h>
> +#include <sys/asm.h>
> +
> +/* Assumptions: rvi_zbb with fast unaligned access. */
> +/* Implementation inspired by aarch64/strcmp.S. */
> +
> +#define src1 a0
> +#define result a0
> +#define src2 a1
> +#define off a3
> +#define m1 a4
> +#define align1 a5
> +#define src3 a6
> +#define tmp a7
> +
> +#define data1 t0
> +#define data2 t1
> +#define b1 t0
> +#define b2 t1
> +#define data3 t2
> +#define data1_orcb t3
> +#define data3_orcb t4
> +#define shift t5
> +
> +#if __riscv_xlen == 64
> +# define REG_L ld
> +# define SZREG 8
> +# define PTRLOG 3
> +#else
> +# define REG_L lw
> +# define SZREG 4
> +# define PTRLOG 2
> +#endif
> +
> +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
> +# error big endian is untested!
> +# define CZ ctz
> +# define SHIFT srl
> +# define SHIFT2 sll
> +#else
> +# define CZ ctz
> +# define SHIFT sll
> +# define SHIFT2 srl
> +#endif
> +
> +#ifndef STRCMP
> +# define STRCMP __strcmp_zbb_unaligned
> +#endif
> +
> +.option push
> +.option arch,+zbb
> +
> +ENTRY_ALIGN (STRCMP, 6)
> + /* off...delta from src1 to src2. */
> + sub off, src2, src1
> + li m1, -1
> + andi tmp, off, SZREG-1
> + andi align1, src1, SZREG-1
> + bnez tmp, L(misaligned8)
> + bnez align1, L(mutual_align)
> +
> + .p2align 4
> +L(loop_aligned):
> + REG_L data1, 0(src1)
> + add tmp, src1, off
> + addi src1, src1, SZREG
> + REG_L data2, 0(tmp)
> +
> +L(start_realigned):
> + orc.b data1_orcb, data1
> + bne data1_orcb, m1, L(end)
> + beq data1, data2, L(loop_aligned)
> +
> +L(fast_end):
> + /* Words don't match, and no NUL byte in one word.
> + Get bytes in big-endian order and compare as words. */
> +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
> + rev8 data1, data1
> + rev8 data2, data2
> +#endif
> + /* Synthesize (data1 >= data2) ? 1 : -1 in a branchless
> sequence. */
> + sltu result, data1, data2
> + neg result, result
> + ori result, result, 1
> + ret
> +
> +L(end_orc):
> + orc.b data1_orcb, data1
> +L(end):
> + /* Words don't match or NUL byte in at least one word.
> + data1_orcb holds orc.b value of data1. */
> + xor tmp, data1, data2
> + orc.b tmp, tmp
> +
> + orn tmp, tmp, data1_orcb
> + CZ shift, tmp
> +
> +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
> + rev8 data1, data1
> + rev8 data2, data2
> +#endif
> + sll data1, data1, shift
> + sll data2, data2, shift
> + srl b1, data1, SZREG*8-8
> + srl b2, data2, SZREG*8-8
> +
> +L(end_singlebyte):
> + sub result, b1, b2
> + ret
> +
> + .p2align 4
> +L(mutual_align):
> + /* Sources are mutually aligned, but are not currently at an
> + alignment boundary. Round down the addresses and then mask
> off
> + the bytes that precede the start point. */
> + andi src1, src1, -SZREG
> + add tmp, src1, off
> + REG_L data1, 0(src1)
> + addi src1, src1, SZREG
> + REG_L data2, 0(tmp)
> + /* Get number of bits to mask. */
> + sll shift, src2, 3
> + /* Bits to mask are now 0, others are 1. */
> + SHIFT tmp, m1, shift
> + /* Or with inverted value -> masked bits become 1. */
> + orn data1, data1, tmp
> + orn data2, data2, tmp
> + j L(start_realigned)
> +
> +L(misaligned8):
> + /* Skip slow loop if SRC1 is aligned. */
> + beqz align1, L(src1_aligned)
> +L(do_misaligned):
> + /* Align SRC1 to 8 bytes. */
> + lbu b1, 0(src1)
> + lbu b2, 0(src2)
> + beqz b1, L(end_singlebyte)
> + bne b1, b2, L(end_singlebyte)
> + addi src1, src1, 1
> + addi src2, src2, 1
> + andi align1, src1, SZREG-1
> + bnez align1, L(do_misaligned)
> +
> +L(src1_aligned):
> + /* SRC1 is aligned. Align SRC2 down and check for NUL there.
> + * If there is no NUL, we may read the next word from SRC2.
> + * If there is a NUL, we must not read a complete word from
> SRC2
> + * because we might cross a page boundary. */
> + /* Get number of bits to mask (upper bits are ignored by
> shifts). */
> + sll shift, src2, 3
> + /* src3 := align_down (src2) */
> + andi src3, src2, -SZREG
> + REG_L data3, 0(src3)
> + addi src3, src3, SZREG
> +
> + /* Bits to mask are now 0, others are 1. */
> + SHIFT tmp, m1, shift
> + /* Or with inverted value -> masked bits become 1. */
> + orn data3_orcb, data3, tmp
> + /* Check for NUL in next aligned word. */
> + orc.b data3_orcb, data3_orcb
> + bne data3_orcb, m1, L(unaligned_nul)
> +
> + .p2align 4
> +L(loop_unaligned):
> + /* Read the (aligned) data1 and the unaligned data2. */
> + REG_L data1, 0(src1)
> + addi src1, src1, SZREG
> + REG_L data2, 0(src2)
> + addi src2, src2, SZREG
> + orc.b data1_orcb, data1
> + bne data1_orcb, m1, L(end)
> + bne data1, data2, L(end)
> +
> + /* Read the next aligned-down word. */
> + REG_L data3, 0(src3)
> + addi src3, src3, SZREG
> + orc.b data3_orcb, data3
> + beq data3_orcb, m1, L(loop_unaligned)
> +
> +L(unaligned_nul):
> + /* src1 points to unread word (only first bytes relevant).
> + * data3 holds next aligned-down word with NUL.
> + * Compare the first bytes of data1 with the last bytes of
> data3. */
> + REG_L data1, 0(src1)
> + /* Shift NUL bytes into data3 to become data2. */
> + SHIFT2 data2, data3, shift
> + bne data1, data2, L(end_orc)
> + li result, 0
> + ret
> +
> +.option pop
> +
> +END (STRCMP)
> +libc_hidden_builtin_def (STRCMP)
--
Xi Ruoyao <xry111@xry111.site>
School of Aerospace Science and Technology, Xidian University
More information about the Libc-alpha
mailing list