2017-11-20 Siddhesh Poyarekar <siddhesh@sourceware.org>
+ * sysdeps/aarch64/memset-reg.h: New file.
+ * sysdeps/aarch64/memset.S: Use it.
+ (__memset): Rename to MEMSET macro.
+ [ZVA_MACRO]: Use zva_macro.
+ * sysdeps/aarch64/multiarch/Makefile (sysdep_routines):
+ Add memset_generic and memset_falkor.
+ * sysdeps/aarch64/multiarch/ifunc-impl-list.c
+ (__libc_ifunc_impl_list): Add memset ifuncs.
+ * sysdeps/aarch64/multiarch/init-arch.h (INIT_ARCH): New
+ local variable zva_size.
+ * sysdeps/aarch64/multiarch/memset.c: New file.
+ * sysdeps/aarch64/multiarch/memset_generic.S: New file.
+ * sysdeps/aarch64/multiarch/memset_falkor.S: New file.
+ * sysdeps/aarch64/multiarch/rtld-memset.S: New file.
+ * sysdeps/unix/sysv/linux/aarch64/cpu-features.c
+ (DCZID_DZP_MASK): New macro.
+ (DCZID_BS_MASK): Likewise.
+ (init_cpu_features): Read and set zva_size.
+ * sysdeps/unix/sysv/linux/aarch64/cpu-features.h
+ (struct cpu_features): New member zva_size.
+
* benchtests/bench-memcpy-walk.c (START_SIZE): Set to 128.
* benchtests/bench-memmove-walk.c (START_SIZE): Likewise.
* benchtests/bench-memset-walk.c (START_SIZE): Likewise.
--- /dev/null
+/* Register aliases for memset to be used across implementations.
+ Copyright (C) 2017 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#define dstin x0
+#define val x1
+#define valw w1
+#define count x2
+#define dst x3
+#define dstend x4
+#define tmp1 x5
+#define tmp1w w5
+#define tmp2 x6
+#define tmp2w w6
+#define zva_len x7
+#define zva_lenw w7
<http://www.gnu.org/licenses/>. */
#include <sysdep.h>
+#include "memset-reg.h"
/* Assumptions:
*
*
*/
-#define dstin x0
-#define val x1
-#define valw w1
-#define count x2
-#define dst x3
-#define dstend x4
-#define tmp1 x5
-#define tmp1w w5
-#define tmp2 x6
-#define tmp2w w6
-#define zva_len x7
-#define zva_lenw w7
-
-ENTRY_ALIGN (__memset, 6)
+ENTRY_ALIGN (MEMSET, 6)
DELOUSE (0)
DELOUSE (2)
stp q0, q0, [dstend, -32]
ret
- .p2align 3
L(try_zva):
+#ifdef ZVA_MACRO
+ zva_macro
+#else
+ .p2align 3
mrs tmp1, dczid_el0
tbnz tmp1w, 4, L(no_zva)
and tmp1w, tmp1w, 15
b.hs 3b
4: add count, count, zva_len
b L(tail64)
+#endif
-END (__memset)
-weak_alias (__memset, memset)
-libc_hidden_builtin_def (memset)
+END (MEMSET)
+libc_hidden_builtin_def (MEMSET)
ifeq ($(subdir),string)
sysdep_routines += memcpy_generic memcpy_thunderx memcpy_falkor \
- memmove_falkor
+ memmove_falkor memset_generic memset_falkor
endif
IFUNC_IMPL_ADD (array, i, memmove, 1, __memmove_thunderx)
IFUNC_IMPL_ADD (array, i, memmove, 1, __memmove_falkor)
IFUNC_IMPL_ADD (array, i, memmove, 1, __memmove_generic))
+ IFUNC_IMPL (i, name, memset,
+ /* Enable this on non-falkor processors too so that other cores
+ can do a comparative analysis with __memset_generic. */
+ IFUNC_IMPL_ADD (array, i, memset, (zva_size == 64), __memset_falkor)
+ IFUNC_IMPL_ADD (array, i, memset, 1, __memset_generic))
return i;
}
#include <ldsodefs.h>
-#define INIT_ARCH() \
- uint64_t __attribute__((unused)) midr = \
- GLRO(dl_aarch64_cpu_features).midr_el1;
+#define INIT_ARCH() \
+ uint64_t __attribute__((unused)) midr = \
+ GLRO(dl_aarch64_cpu_features).midr_el1; \
+ unsigned __attribute__((unused)) zva_size = \
+ GLRO(dl_aarch64_cpu_features).zva_size;
--- /dev/null
+/* Multiple versions of memset. AARCH64 version.
+ Copyright (C) 2017 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+/* Define multiple versions only for the definition in libc. */
+
+#if IS_IN (libc)
+/* Redefine memset so that the compiler won't complain about the type
+ mismatch with the IFUNC selector in strong_alias, below. */
+# undef memset
+# define memset __redirect_memset
+# include <string.h>
+# include <init-arch.h>
+
+extern __typeof (__redirect_memset) __libc_memset;
+
+extern __typeof (__redirect_memset) __memset_falkor attribute_hidden;
+extern __typeof (__redirect_memset) __memset_generic attribute_hidden;
+
+libc_ifunc (__libc_memset, (IS_FALKOR (midr) && zva_size == 64
+ ? __memset_falkor
+ : __memset_generic));
+
+# undef memset
+strong_alias (__libc_memset, memset);
+#endif
--- /dev/null
+/* Memset for falkor.
+ Copyright (C) 2017 Free Software Foundation, Inc.
+
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library. If not, see
+ <http://www.gnu.org/licenses/>. */
+
+#include <memset-reg.h>
+
+/* Reading dczid_el0 is expensive on falkor so move it into the ifunc
+ resolver and assume ZVA size of 64 bytes. The IFUNC resolver takes care to
+ use this function only when ZVA is enabled. */
+
+#if IS_IN (libc)
+.macro zva_macro
+ .p2align 4
+ /* Write the first and last 64 byte aligned block using stp rather
+ than using DC ZVA. This is faster on some cores. */
+ str q0, [dst, 16]
+ stp q0, q0, [dst, 32]
+ bic dst, dst, 63
+ stp q0, q0, [dst, 64]
+ stp q0, q0, [dst, 96]
+ sub count, dstend, dst /* Count is now 128 too large. */
+ sub count, count, 128+64+64 /* Adjust count and bias for loop. */
+ add dst, dst, 128
+1: dc zva, dst
+ add dst, dst, 64
+ subs count, count, 64
+ b.hi 1b
+ stp q0, q0, [dst, 0]
+ stp q0, q0, [dst, 32]
+ stp q0, q0, [dstend, -64]
+ stp q0, q0, [dstend, -32]
+ ret
+.endm
+
+# define ZVA_MACRO zva_macro
+# define MEMSET __memset_falkor
+# include <sysdeps/aarch64/memset.S>
+#endif
--- /dev/null
+/* Memset for aarch64, default version for internal use.
+ Copyright (C) 2017 Free Software Foundation, Inc.
+
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library. If not, see
+ <http://www.gnu.org/licenses/>. */
+
+#if IS_IN (libc)
+# define MEMSET __memset_generic
+/* Add a hidden definition for use within libc.so. */
+# ifdef SHARED
+ .globl __GI_memset; __GI_memset = __memset_generic
+# endif
+# include <sysdeps/aarch64/memset.S>
+#endif
--- /dev/null
+/* Memset for aarch64, for the dynamic linker.
+ Copyright (C) 2017 Free Software Foundation, Inc.
+
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library. If not, see
+ <http://www.gnu.org/licenses/>. */
+
+#if IS_IN (rtld)
+# define MEMSET memset
+# include <sysdeps/aarch64/memset.S>
+#endif
#include <sys/auxv.h>
#include <elf/dl-hwcaps.h>
+#define DCZID_DZP_MASK (1 << 4)
+#define DCZID_BS_MASK (0xf)
+
#if HAVE_TUNABLES
struct cpu_list
{
}
cpu_features->midr_el1 = midr;
+
+ /* Check if ZVA is enabled. */
+ unsigned dczid;
+ asm volatile ("mrs %0, dczid_el0" : "=r"(dczid));
+
+ if ((dczid & DCZID_DZP_MASK) == 0)
+ cpu_features->zva_size = 4 << (dczid & DCZID_BS_MASK);
}
struct cpu_features
{
uint64_t midr_el1;
+ unsigned zva_size;
};
#endif /* _CPU_FEATURES_AARCH64_H */
git://sourceware.org / glibc.git / commitdiff