powerpc: Optimized rawmemchr for POWER9

This version uses vector instructions and is up to 60% faster on medium
matches and up to 90% faster on long matches, compared to the POWER7
version. A few examples:

                            __rawmemchr_power9  __rawmemchr_power7
Length   32, alignment  0:   2.27566             3.77765
Length   64, alignment  2:   2.46231             3.51064
Length 1024, alignment  0:  17.3059             32.6678

diff --git a/sysdeps/powerpc/powerpc64/le/power9/rawmemchr.S b/sysdeps/powerpc/powerpc64/le/power9/rawmemchr.S
new file mode 100644 (file)
index 0000000..9d0276c
--- /dev/null
+++ b/sysdeps/powerpc/powerpc64/le/power9/rawmemchr.S
@@ -0,0 +1,107 @@
+/* Optimized rawmemchr implementation for PowerPC64/POWER9.
+   Copyright (C) 2020 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#include <sysdep.h>
+
+#ifndef RAWMEMCHR
+# define RAWMEMCHR __rawmemchr
+#endif
+
+/* Implements the function
+
+   int [r3] rawmemchr (void *s [r3], int c [r4])
+
+   The implementation can load bytes past a matching byte, but only
+   up to the next 16B boundary, so it never crosses a page.  */
+
+.machine power9
+ENTRY_TOCLESS (RAWMEMCHR, 4)
+       CALL_MCOUNT 2
+
+       xori    r5,r4,0xff
+
+       mtvsrd  v18+32,r4       /* matching char in v18  */
+       mtvsrd  v19+32,r5       /* non matching char in v19  */
+
+       vspltb  v18,v18,7       /* replicate  */
+       vspltb  v19,v19,7       /* replicate  */
+
+       neg     r5,r3
+       rldicl  r9,r5,0,60      /* How many bytes to get source 16B aligned?  */
+
+       /* Align data and fill bytes not loaded with non matching char  */
+       lvx     v0,0,r3
+       lvsr    v1,0,r3
+       vperm   v0,v19,v0,v1
+
+       vcmpequb. v6,v0,v18     /* 0xff if byte matches, 0x00 otherwise  */
+       beq     cr6,L(aligned)
+
+       vctzlsbb r0,v6
+       add     r3,r3,r0
+       blr
+
+L(aligned):
+       add     r3,r3,r9
+
+L(loop):
+       lxv     v0+32,0(r3)
+       vcmpequb. v6,v0,v18     /* 0xff if byte matches, 0x00 otherwise  */
+       bne     cr6,L(tail1)
+
+       lxv     v0+32,16(r3)
+       vcmpequb. v6,v0,v18     /* 0xff if byte matches, 0x00 otherwise  */
+       bne     cr6,L(tail2)
+
+       lxv     v0+32,32(r3)
+       vcmpequb. v6,v0,v18     /* 0xff if byte matches, 0x00 otherwise  */
+       bne     cr6,L(tail3)
+
+       lxv     v0+32,48(r3)
+       vcmpequb. v6,v0,v18     /* 0xff if byte matches, 0x00 otherwise  */
+       bne     cr6,L(tail4)
+
+       addi    r3,r3,64
+       b       L(loop)
+
+L(tail1):
+       vctzlsbb r0,v6
+       add     r3,r3,r0
+       blr
+
+L(tail2):
+       vctzlsbb r0,v6
+       add     r3,r3,r0
+       addi    r3,r3,16
+       blr
+
+L(tail3):
+       vctzlsbb r0,v6
+       add     r3,r3,r0
+       addi    r3,r3,32
+       blr
+
+L(tail4):
+       vctzlsbb r0,v6
+       add     r3,r3,r0
+       addi    r3,r3,48
+       blr
+
+END (RAWMEMCHR)
+weak_alias (__rawmemchr,rawmemchr)
+libc_hidden_builtin_def (__rawmemchr)

diff --git a/sysdeps/powerpc/powerpc64/multiarch/Makefile b/sysdeps/powerpc/powerpc64/multiarch/Makefile
index 61a6901f7da63d30f43bf8287a66a35e850b7f71..fc2268f6b5c85a81e2cc29be1de34edf83c62f41 100644 (file)
--- a/sysdeps/powerpc/powerpc64/multiarch/Makefile
+++ b/sysdeps/powerpc/powerpc64/multiarch/Makefile
@@ -32,7 +32,8 @@ sysdep_routines += memcpy-power8-cached memcpy-power7 memcpy-a2 memcpy-power6 \
                   strncase-power8
 
 ifneq (,$(filter %le,$(config-machine)))
-sysdep_routines += strcmp-power9 strncmp-power9 strcpy-power9 stpcpy-power9
+sysdep_routines += strcmp-power9 strncmp-power9 strcpy-power9 stpcpy-power9 \
+                  rawmemchr-power9
 endif
 CFLAGS-strncase-power7.c += -mcpu=power7 -funroll-loops
 CFLAGS-strncase_l-power7.c += -mcpu=power7 -funroll-loops

diff --git a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c
index 8021d8d8fa0282c904016946cb7fec0894bba902..59a227ee222821b0fc8af98b85cc27264920727d 100644 (file)
--- a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c
+++ b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c
@@ -216,6 +216,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
 
   /* Support sysdeps/powerpc/powerpc64/multiarch/rawmemchr.c.  */
   IFUNC_IMPL (i, name, rawmemchr,
+#ifdef __LITTLE_ENDIAN__
+             IFUNC_IMPL_ADD (array, i, rawmemchr,
+                             hwcap2 & PPC_FEATURE2_ARCH_3_00,
+                             __rawmemchr_power9)
+#endif
              IFUNC_IMPL_ADD (array, i, rawmemchr,
                              hwcap & PPC_FEATURE_HAS_VSX,
                              __rawmemchr_power7)

diff --git a/sysdeps/powerpc/powerpc64/multiarch/rawmemchr-power9.S b/sysdeps/powerpc/powerpc64/multiarch/rawmemchr-power9.S
new file mode 100644 (file)
index 0000000..bac0a90
--- /dev/null
+++ b/sysdeps/powerpc/powerpc64/multiarch/rawmemchr-power9.S
@@ -0,0 +1,21 @@
+/* Optimized rawmemchr implementation for PowerPC64/POWER9.
+   Copyright (C) 2020 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#define RAWMEMCHR __rawmemchr_power9
+
+#include <sysdeps/powerpc/powerpc64/le/power9/rawmemchr.S>

diff --git a/sysdeps/powerpc/powerpc64/multiarch/rawmemchr.c b/sysdeps/powerpc/powerpc64/multiarch/rawmemchr.c
index 847157e5f0a9ad9799f0f6956159f8206210344f..c49c0c51ff76d120ed45958ba5c1e999ea700801 100644 (file)
--- a/sysdeps/powerpc/powerpc64/multiarch/rawmemchr.c
+++ b/sysdeps/powerpc/powerpc64/multiarch/rawmemchr.c
@@ -24,13 +24,21 @@
 
 extern __typeof (__rawmemchr) __rawmemchr_ppc attribute_hidden;
 extern __typeof (__rawmemchr) __rawmemchr_power7 attribute_hidden;
+# ifdef __LITTLE_ENDIAN__
+extern __typeof (__rawmemchr) __rawmemchr_power9 attribute_hidden;
+# endif
+
 # undef __rawmemchr
 
 /* Avoid DWARF definition DIE on ifunc symbol so that GDB can handle
    ifunc symbol properly.  */
 libc_ifunc_redirected (__redirect___rawmemchr, __rawmemchr,
-                      (hwcap & PPC_FEATURE_HAS_VSX)
-                      ? __rawmemchr_power7
+# ifdef __LITTLE_ENDIAN__
+                      (hwcap2 & PPC_FEATURE2_ARCH_3_00)
+                      ? __rawmemchr_power9 :
+# endif
+                        (hwcap & PPC_FEATURE_HAS_VSX)
+                        ? __rawmemchr_power7
                       : __rawmemchr_ppc);
 
 weak_alias (__rawmemchr, rawmemchr)