This patch introduces a z13 specific ifunc variant for memmove.
As the common code implementation, it checks if we can copy from
the beginning to the end - with z196 memcpy implementation - or
if we have to copy from the end to the beginning.
The latter case is done by using vector load/store instructions.
If vector instructions are not available, the common-code is
used as fallback. Therefore it is implemented in memmove-c with
a different name.
Furthermore the ifunc logic decides if we need the common-code
implementation at all. If vector instructions are supported
due to the minimum architecture level set we can skip the
common-code ifunc variant.
ChangeLog:
* sysdeps/s390/Makefile (sysdep_routines): Add memmove-c.
* sysdeps/s390/ifunc-memcpy.h (HAVE_MEMMOVE_IFUNC,
HAVE_MEMMOVE_IFUNC_AND_VX_SUPPORT, MEMMOVE_DEFAULT,
HAVE_MEMMOVE_C, MEMMOVE_C, HAVE_MEMMOVE_Z13, MEMMOVE_Z13):
New defines.
* sysdeps/s390/memcpy-z900.S: Add z13 memmove implementation.
* sysdeps/s390/memmove-c.c: New file.
* sysdeps/s390/memmove.c: Likewise.
* sysdeps/s390/multiarch/ifunc-impl-list.c
(__libc_ifunc_impl_list): Add ifunc variants for memmove.
+2018-12-18 Stefan Liebler <stli@linux.ibm.com>
+
+ * sysdeps/s390/Makefile (sysdep_routines): Add memmove-c.
+ * sysdeps/s390/ifunc-memcpy.h (HAVE_MEMMOVE_IFUNC,
+ HAVE_MEMMOVE_IFUNC_AND_VX_SUPPORT, MEMMOVE_DEFAULT,
+ HAVE_MEMMOVE_C, MEMMOVE_C, HAVE_MEMMOVE_Z13, MEMMOVE_Z13):
+ New defines.
+ * sysdeps/s390/memcpy-z900.S: Add z13 memmove implementation.
+ * sysdeps/s390/memmove-c.c: New file.
+ * sysdeps/s390/memmove.c: Likewise.
+ * sysdeps/s390/multiarch/ifunc-impl-list.c
+ (__libc_ifunc_impl_list): Add ifunc variants for memmove.
+
2018-12-18 Stefan Liebler <stli@linux.ibm.com>
* config.h.in (HAVE_S390_MIN_Z13_ZARCH_ASM_SUPPORT): New undefine.
ifeq ($(subdir),string)
sysdep_routines += bzero memset memset-z900 \
memcmp memcmp-z900 \
- mempcpy memcpy memcpy-z900
+ mempcpy memcpy memcpy-z900 \
+ memmove memmove-c
endif
# define HAVE_MEMCPY_Z196 HAVE_MEMCPY_IFUNC
#endif
+#if defined SHARED && defined USE_MULTIARCH && IS_IN (libc) \
+ && ! defined HAVE_S390_MIN_Z13_ZARCH_ASM_SUPPORT
+# define HAVE_MEMMOVE_IFUNC 1
+#else
+# define HAVE_MEMMOVE_IFUNC 0
+#endif
+
+#ifdef HAVE_S390_VX_ASM_SUPPORT
+# define HAVE_MEMMOVE_IFUNC_AND_VX_SUPPORT HAVE_MEMMOVE_IFUNC
+#else
+# define HAVE_MEMMOVE_IFUNC_AND_VX_SUPPORT 0
+#endif
+
+#if defined HAVE_S390_MIN_Z13_ZARCH_ASM_SUPPORT
+# define MEMMOVE_DEFAULT MEMMOVE_Z13
+# define HAVE_MEMMOVE_C 0
+# define HAVE_MEMMOVE_Z13 1
+#else
+# define MEMMOVE_DEFAULT MEMMOVE_C
+# define HAVE_MEMMOVE_C 1
+# define HAVE_MEMMOVE_Z13 HAVE_MEMMOVE_IFUNC_AND_VX_SUPPORT
+#endif
+
#if HAVE_MEMCPY_Z900_G5
# define MEMCPY_Z900_G5 __memcpy_default
# define MEMPCPY_Z900_G5 __mempcpy_default
# define MEMCPY_Z196 NULL
# define MEMPCPY_Z196 NULL
#endif
+
+#if HAVE_MEMMOVE_C
+# define MEMMOVE_C __memmove_c
+#else
+# define MEMMOVE_C NULL
+#endif
+
+#if HAVE_MEMMOVE_Z13
+# define MEMMOVE_Z13 __memmove_z13
+#else
+# define MEMMOVE_Z13 NULL
+#endif
# endif /* !defined __s390x__ */
ltgr %r4,%r4
je .L_Z196_4
+.L_Z196_start2:
aghi %r4,-1
srlg %r5,%r4,8
ltgr %r5,%r5
END(MEMCPY_Z196)
#endif /* HAVE_MEMCPY_Z196 */
+#if HAVE_MEMMOVE_Z13
+ENTRY(MEMMOVE_Z13)
+ .machine "z13"
+ .machinemode "zarch_nohighgprs"
+# if !defined __s390x__
+ /* Note: The 31bit dst and src pointers are prefixed with zeroes. */
+ llgfr %r4,%r4
+ llgfr %r3,%r3
+ llgfr %r2,%r2
+# endif /* !defined __s390x__ */
+ sgrk %r0,%r2,%r3
+ clgijh %r4,16,.L_MEMMOVE_Z13_LARGE
+ aghik %r5,%r4,-1
+.L_MEMMOVE_Z13_SMALL:
+ jl .L_MEMMOVE_Z13_END /* Jump away if len was zero. */
+ /* Store up to 16 bytes with vll/vstl which needs the index
+ instead of lengths. */
+ vll %v16,%r5,0(%r3)
+ vstl %v16,%r5,0(%r2)
+.L_MEMMOVE_Z13_END:
+ br %r14
+.L_MEMMOVE_Z13_LARGE:
+ lgr %r1,%r2 /* For memcpy: r1: Use as dest ;
+ r2: Return dest */
+ /* The unsigned comparison (dst - src >= len) determines if we can
+ execute the forward case with memcpy. */
+#if ! HAVE_MEMCPY_Z196
+# error The z13 variant of memmove needs the z196 variant of memcpy!
+#endif
+ clgrjhe %r0,%r4,.L_Z196_start2
+ risbgn %r5,%r4,4,128+63,60 /* r5 = r4 / 16 */
+ aghi %r4,-16
+ clgijhe %r5,8,.L_MEMMOVE_Z13_LARGE_64B
+.L_MEMMOVE_Z13_LARGE_16B_LOOP:
+ /* Store at least 16 bytes with vl/vst. The number of 16byte blocks
+ is stored in r5. */
+ vl %v16,0(%r4,%r3)
+ vst %v16,0(%r4,%r2)
+ aghi %r4,-16
+ brctg %r5,.L_MEMMOVE_Z13_LARGE_16B_LOOP
+ aghik %r5,%r4,15
+ j .L_MEMMOVE_Z13_SMALL
+.L_MEMMOVE_Z13_LARGE_64B:
+ /* Store at least 128 bytes with 4x vl/vst. The number of 64byte blocks
+ will be stored in r0. */
+ aghi %r4,-48
+ srlg %r0,%r5,2 /* r5 = %r0 / 4
+ => Number of 64byte blocks. */
+.L_MEMMOVE_Z13_LARGE_64B_LOOP:
+ vl %v20,48(%r4,%r3)
+ vl %v19,32(%r4,%r3)
+ vl %v18,16(%r4,%r3)
+ vl %v17,0(%r4,%r3)
+ vst %v20,48(%r4,%r2)
+ vst %v19,32(%r4,%r2)
+ vst %v18,16(%r4,%r2)
+ vst %v17,0(%r4,%r2)
+ aghi %r4,-64
+ brctg %r0,.L_MEMMOVE_Z13_LARGE_64B_LOOP
+ aghi %r4,48
+ /* Recalculate the number of 16byte blocks. */
+ risbg %r5,%r5,62,128+63,0 /* r5 = r5 & 3
+ => Remaining 16byte blocks. */
+ jne .L_MEMMOVE_Z13_LARGE_16B_LOOP
+ aghik %r5,%r4,15
+ j .L_MEMMOVE_Z13_SMALL
+END(MEMMOVE_Z13)
+#endif /* HAVE_MEMMOVE_Z13 */
+
#if ! HAVE_MEMCPY_IFUNC
/* If we don't use ifunc, define an alias for mem[p]cpy here.
Otherwise see sysdeps/s390/mem[p]cpy.c. */
weak_alias (__mempcpy, mempcpy)
#endif
+#if ! HAVE_MEMMOVE_IFUNC
+/* If we don't use ifunc, define an alias for memmove here.
+ Otherwise see sysdeps/s390/memmove.c. */
+# if ! HAVE_MEMMOVE_C
+/* If the c variant is needed, then sysdeps/s390/memmove-c.c
+ defines memmove.
+ Otherwise MEMMOVE_DEFAULT is implemented here and we have to define it. */
+strong_alias (MEMMOVE_DEFAULT, memmove)
+# endif
+#endif
+
#if defined SHARED && IS_IN (libc)
/* Defines the internal symbols.
Compare to libc_hidden_[builtin_]def (mem[p]cpy) in string/mem[p]cpy.c. */
strong_alias (MEMCPY_DEFAULT, __GI_memcpy)
strong_alias (MEMPCPY_DEFAULT, __GI_mempcpy)
strong_alias (MEMPCPY_DEFAULT, __GI___mempcpy)
+# if ! HAVE_MEMMOVE_C
+/* If the c variant is needed, then sysdeps/s390/memmove-c.c
+ defines the internal symbol.
+ Otherwise MEMMOVE_DEFAULT is implemented here and we have to define it. */
+strong_alias (MEMMOVE_DEFAULT, __GI_memmove)
+# endif
#endif
--- /dev/null
+/* Fallback C version of memmove.
+ Copyright (C) 2018 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#include <ifunc-memcpy.h>
+
+#if HAVE_MEMMOVE_C
+# if HAVE_MEMMOVE_IFUNC
+/* If we use ifunc, then the memmove symbol is defined
+ in sysdeps/s390/memmove.c and we use a different name here.
+ Otherwise, we have to define memmove here or in
+ sysdeps/s390/memcpy.S depending on the used default implementation. */
+# define MEMMOVE MEMMOVE_C
+# if defined SHARED && IS_IN (libc)
+/* Define the internal symbol. */
+# undef libc_hidden_builtin_def
+# define libc_hidden_builtin_def(name) \
+ __hidden_ver1 (__memmove_c, __GI_memmove, __memmove_c);
+# endif
+# endif
+
+# include <string/memmove.c>
+#endif
--- /dev/null
+/* Multiple versions of memmove.
+ Copyright (C) 2016-2018 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#include <ifunc-memcpy.h>
+
+#if HAVE_MEMMOVE_IFUNC
+/* If we don't use ifunc, an alias is defined for memmove
+ in sysdeps/s390/memmove-c.c or sysdeps/s390/memcpy.S
+ depending on the used default implementation. */
+# undef memmove
+# define memmove __redirect_memmove
+# include <string.h>
+# include <ifunc-resolve.h>
+# undef memmove
+
+# if HAVE_MEMMOVE_C
+extern __typeof (__redirect_memmove) MEMMOVE_C attribute_hidden;
+# endif
+
+# if HAVE_MEMMOVE_Z13
+extern __typeof (__redirect_memmove) MEMMOVE_Z13 attribute_hidden;
+# endif
+
+s390_libc_ifunc_expr (__redirect_memmove, memmove,
+ (HAVE_MEMMOVE_Z13 && (hwcap & HWCAP_S390_VX))
+ ? MEMMOVE_Z13
+ : MEMMOVE_DEFAULT
+ )
+#endif
)
#endif /* HAVE_MEMCPY_IFUNC */
+#if HAVE_MEMMOVE_IFUNC
+ IFUNC_IMPL (i, name, memmove,
+# if HAVE_MEMMOVE_Z13
+ IFUNC_IMPL_ADD (array, i, memmove,
+ dl_hwcap & HWCAP_S390_VX, MEMMOVE_Z13)
+# endif
+# if HAVE_MEMMOVE_C
+ IFUNC_IMPL_ADD (array, i, memmove, 1, MEMMOVE_C)
+# endif
+ )
+#endif /* HAVE_MEMMOVE_IFUNC */
+
#ifdef HAVE_S390_VX_ASM_SUPPORT
# define IFUNC_VX_IMPL(FUNC) \
git://sourceware.org / glibc.git / commitdiff