This is the mail archive of the libc-alpha@sourceware.org mailing list for the glibc project.
Index Nav:	[Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav:	[Date Prev] [Date Next]	[Thread Prev] [Thread Next]
Other format:	[Raw text]
Re: [PATCH RFC] Imporve 64bit memset performance for Haswell CPU with AVX2 instruction

From: Ling Ma <ling dot ma dot program at gmail dot com>
To: neleai at seznam dot cz
Cc: libc-alpha at sourceware dot org, liubov dot dmitrieva at gmail dot com, hjl dot tools at gmail dot com, yumkam at gmail dot com, Ling Ma <ling dot ml at alibaba-inc dot com>
Date: Sun, 27 Apr 2014 19:45:34 +0800
Subject: Re: [PATCH RFC] Imporve 64bit memset performance for Haswell CPU with AVX2 instruction
Authentication-results: sourceware.org; auth=none
References: <1398055921-4455-1-git-send-email-ling dot ma at alipay dot com>
Any comments ?

2014-04-21 12:52 GMT+08:00, ling.ma.program@gmail.com
<ling.ma.program@gmail.com>:
> From: Ling Ma <ling.ml@alibaba-inc.com>
>
> In this patch we take advantage of HSW memory bandwidth, manage to
> reduce miss branch prediction by avoiding using branch instructions and
> force destination to be aligned with avx & avx2 instruction.
>
> The CPU2006 403.gcc benchmark indicates this patch improves performance
> from 23.7% to 57%.
>
> This version is based on latest ChangeLog, Appreciate your comments.
> ---
>  ChangeLog                              |   9 ++
>  sysdeps/x86_64/multiarch/Makefile      |   4 +-
>  sysdeps/x86_64/multiarch/memset-avx2.S | 198
> +++++++++++++++++++++++++++++++++
>  sysdeps/x86_64/multiarch/memset.S      |  58 ++++++++++
>  sysdeps/x86_64/multiarch/memset_chk.S  |  43 +++++++
>  5 files changed, 311 insertions(+), 1 deletion(-)
>  create mode 100644 sysdeps/x86_64/multiarch/memset-avx2.S
>  create mode 100644 sysdeps/x86_64/multiarch/memset.S
>  create mode 100644 sysdeps/x86_64/multiarch/memset_chk.S
>
> diff --git a/ChangeLog b/ChangeLog
> index e09fdc6..9bb48ab 100644
> --- a/ChangeLog
> +++ b/ChangeLog
> @@ -1,3 +1,12 @@
> +2014-04-21  Ling Ma  <ling.ml@alibaba-inc.com>
> +
> +	* sysdeps/x86_64/multiarch/Makefile: Add memset-avx2
> +	* sysdeps/x86_64/multiarch/memset-avx2.S: New file for AVX2 memset
> +	* sysdeps/x86_64/multiarch/memset.S: New file for multiple memset
> +	versions
> +	* sysdeps/x86_64/multiarch/memset_chk.S: New file for multiple memset_chk
> +	versions
> +
>  2014-04-17  Carlos O'Donell  <carlos@redhat.com>
>
>  	* manual/intro.texi (Roadmap to the manual): Sort chapter listing to
> diff --git a/sysdeps/x86_64/multiarch/Makefile
> b/sysdeps/x86_64/multiarch/Makefile
> index 57a3c13..42df96f 100644
> --- a/sysdeps/x86_64/multiarch/Makefile
> +++ b/sysdeps/x86_64/multiarch/Makefile
> @@ -17,7 +17,9 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c
> strcmp-ssse3 \
>  		   strcpy-sse2-unaligned strncpy-sse2-unaligned \
>  		   stpcpy-sse2-unaligned stpncpy-sse2-unaligned \
>  		   strcat-sse2-unaligned strncat-sse2-unaligned \
> -		   strchr-sse2-no-bsf memcmp-ssse3 strstr-sse2-unaligned
> +		   strchr-sse2-no-bsf memcmp-ssse3 strstr-sse2-unaligned \
> +		   memset-avx2
> +
>  ifeq (yes,$(config-cflags-sse4))
>  sysdep_routines += strcspn-c strpbrk-c strspn-c varshift
>  CFLAGS-varshift.c += -msse4
> diff --git a/sysdeps/x86_64/multiarch/memset-avx2.S
> b/sysdeps/x86_64/multiarch/memset-avx2.S
> new file mode 100644
> index 0000000..524898a
> --- /dev/null
> +++ b/sysdeps/x86_64/multiarch/memset-avx2.S
> @@ -0,0 +1,198 @@
> +/* memset with AVX2
> +   Copyright (C) 2014 Free Software Foundation, Inc.
> +   This file is part of the GNU C Library.
> +
> +   The GNU C Library is free software; you can redistribute it and/or
> +   modify it under the terms of the GNU Lesser General Public
> +   License as published by the Free Software Foundation; either
> +   version 2.1 of the License, or (at your option) any later version.
> +
> +   The GNU C Library is distributed in the hope that it will be useful,
> +   but WITHOUT ANY WARRANTY; without even the implied warranty of
> +   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> +   Lesser General Public License for more details.
> +
> +   You should have received a copy of the GNU Lesser General Public
> +   License along with the GNU C Library; if not, see
> +   <http://www.gnu.org/licenses/>.  */
> +
> +#include <sysdep.h>
> +
> +#if !defined NOT_IN_libc
> +
> +#include "asm-syntax.h"
> +#ifndef MEMSET
> +# define MEMSET	__memset_avx2
> +# define MEMSET_CHK	__memset_chk_avx2
> +#endif
> +
> +	.section .text.avx2,"ax",@progbits
> +#if defined PIC
> +ENTRY (MEMSET_CHK)
> +	cmpq	%rdx, %rcx
> +	jb	HIDDEN_JUMPTARGET (__chk_fail)
> +END (MEMSET_CHK)
> +#endif
> +
> +ENTRY (MEMSET)
> +	vpxor	%xmm0, %xmm0, %xmm0
> +	vmovd %esi, %xmm1
> +	mov	%rdi, %rsi
> +	mov	%rdi, %rax
> +	vpshufb	%xmm0, %xmm1, %xmm0
> +	cmp	$256, %rdx
> +	ja	L(256bytesormore)
> +	add	%rdx, %rsi
> +	vmovd %xmm0, %rcx
> +	cmp	$128, %edx
> +	jb	L(less_128bytes)
> +	vmovdqu %xmm0, (%rdi)
> +	vmovdqu %xmm0, 0x10(%rdi)
> +	vmovdqu %xmm0, 0x20(%rdi)
> +	vmovdqu %xmm0, 0x30(%rdi)
> +	vmovdqu %xmm0, 0x40(%rdi)
> +	vmovdqu %xmm0, 0x50(%rdi)
> +	vmovdqu %xmm0, 0x60(%rdi)
> +	vmovdqu %xmm0, 0x70(%rdi)
> +	vmovdqu %xmm0, -0x80(%rsi)
> +	vmovdqu %xmm0, -0x70(%rsi)
> +	vmovdqu %xmm0, -0x60(%rsi)
> +	vmovdqu %xmm0, -0x50(%rsi)
> +	vmovdqu %xmm0, -0x40(%rsi)
> +	vmovdqu %xmm0, -0x30(%rsi)
> +	vmovdqu %xmm0, -0x20(%rsi)
> +	vmovdqu %xmm0, -0x10(%rsi)
> +	ret
> +
> +	.p2align 4
> +L(less_128bytes):
> +	cmp	$64, %dl
> +	jb	L(less_64bytes)
> +	vmovdqu %xmm0, (%rdi)
> +	vmovdqu %xmm0, 0x10(%rdi)
> +	vmovdqu %xmm0, 0x20(%rdi)
> +	vmovdqu %xmm0, 0x30(%rdi)
> +	vmovdqu %xmm0, -0x40(%rsi)
> +	vmovdqu %xmm0, -0x30(%rsi)
> +	vmovdqu %xmm0, -0x20(%rsi)
> +	vmovdqu %xmm0, -0x10(%rsi)
> +	ret
> +
> +	.p2align 4
> +L(less_64bytes):
> +	cmp	$32, %dl
> +	jb	L(less_32bytes)
> +	vmovdqu %xmm0, (%rdi)
> +	vmovdqu %xmm0, 0x10(%rdi)
> +	vmovdqu %xmm0, -0x20(%rsi)
> +	vmovdqu %xmm0, -0x10(%rsi)
> +	ret
> +
> +	.p2align 4
> +L(less_32bytes):
> +	cmp	$16, %dl
> +	jb	L(less_16bytes)
> +	vmovdqu %xmm0, (%rdi)
> +	vmovdqu %xmm0, -0x10(%rsi)
> +	ret
> +
> +	.p2align 4
> +L(less_16bytes):
> +	cmp	$8, %dl
> +	jb	L(less_8bytes)
> +	mov %rcx, (%rdi)
> +	mov %rcx, -0x08(%rsi)
> +	ret
> +
> +	.p2align 4
> +L(less_8bytes):
> +	cmp	$4, %dl
> +	jb	L(less_4bytes)
> +	mov %ecx, (%rdi)
> +	mov %ecx, -0x04(%rsi)
> +	ret
> +
> +	.p2align 4
> +L(less_4bytes):
> +	cmp	$2, %dl
> +	jb	L(less_2bytes)
> +	mov	%cx, (%rdi)
> +	mov	%cx, -0x02(%rsi)
> +	ret
> +
> +	.p2align 4
> +L(less_2bytes):
> +	cmp	$1, %dl
> +	jb	L(less_1bytes)
> +	mov	%cl, (%rdi)
> +L(less_1bytes):
> +	ret
> +
> +	.p2align 4
> +L(256bytesormore):
> +	vinserti128 $1, %xmm0, %ymm0, %ymm0
> +	mov	$0x80, %rcx
> +	add	%rdx, %rsi
> +	mov	%rdi, %r9
> +	vmovdqu	%ymm0, (%rdi)
> +	and	$-0x20, %rdi
> +	add	$32, %rdi
> +	sub	%rdi, %r9
> +	add	%r9, %rdx
> +	cmp	$4096, %rdx
> +	ja	L(gobble_data)
> +	sub	%ecx, %edx
> +L(gobble_128_loop):
> +	vmovdqa	%ymm0, (%rdi)
> +	vmovdqa	%ymm0, 0x20(%rdi)
> +	vmovdqa	%ymm0, 0x40(%rdi)
> +	vmovdqa	%ymm0, 0x60(%rdi)
> +	add	%rcx, %rdi
> +	sub	%ecx, %edx
> +	jae	L(gobble_128_loop)
> +	vmovdqu	%ymm0, -0x80(%rsi)
> +	vmovdqu	%ymm0, -0x60(%rsi)
> +	vmovdqu	%ymm0, -0x40(%rsi)
> +	vmovdqu	%ymm0, -0x20(%rsi)
> +	vzeroupper
> +	ret
> +
> +	.p2align 4
> +L(gobble_data):
> +#ifdef SHARED_CACHE_SIZE_HALF
> +	mov	$SHARED_CACHE_SIZE_HALF, %r9
> +#else
> +	mov	__x86_shared_cache_size_half(%rip), %r9
> +#endif
> +	shl	$4, %r9
> +	cmp	%r9, %rdx
> +	ja	L(gobble_big_data)
> +	mov	%rax, %rsi
> +	vmovd %xmm0, %eax
> +	mov	%rdx, %rcx
> +	rep	stosb
> +	mov	%rsi, %rax
> +	vzeroupper
> +	ret
> +
> +	.p2align 4
> +L(gobble_big_data):
> +	sub	%rcx, %rdx
> +L(gobble_big_data_loop):
> +	vmovntdq	%ymm0, (%rdi)
> +	vmovntdq	%ymm0, 0x20(%rdi)
> +	vmovntdq	%ymm0, 0x40(%rdi)
> +	vmovntdq	%ymm0, 0x60(%rdi)
> +	add	%rcx, %rdi
> +	sub	%rcx, %rdx
> +	jae	L(gobble_big_data_loop)
> +	vmovdqu	%ymm0, -0x80(%rsi)
> +	vmovdqu	%ymm0, -0x60(%rsi)
> +	vmovdqu	%ymm0, -0x40(%rsi)
> +	vmovdqu	%ymm0, -0x20(%rsi)
> +	vzeroupper
> +	sfence
> +	ret
> +
> +END (MEMSET)
> +#endif
> diff --git a/sysdeps/x86_64/multiarch/memset.S
> b/sysdeps/x86_64/multiarch/memset.S
> new file mode 100644
> index 0000000..03132db
> --- /dev/null
> +++ b/sysdeps/x86_64/multiarch/memset.S
> @@ -0,0 +1,58 @@
> +/* Multiple versions of memset
> +   Copyright (C) 2014 Free Software Foundation, Inc.
> +   This file is part of the GNU C Library.
> +
> +   The GNU C Library is free software; you can redistribute it and/or
> +   modify it under the terms of the GNU Lesser General Public
> +   License as published by the Free Software Foundation; either
> +   version 2.1 of the License, or (at your option) any later version.
> +
> +   The GNU C Library is distributed in the hope that it will be useful,
> +   but WITHOUT ANY WARRANTY; without even the implied warranty of
> +   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> +   Lesser General Public License for more details.
> +
> +   You should have received a copy of the GNU Lesser General Public
> +   License along with the GNU C Library; if not, see
> +   <http://www.gnu.org/licenses/>.  */
> +
> +#include <sysdep.h>
> +#include <shlib-compat.h>
> +#include <init-arch.h>
> +
> +/* Define multiple versions only for the definition in lib.  */
> +#ifndef NOT_IN_libc
> +ENTRY(memset)
> +	.type	memset, @gnu_indirect_function
> +	cmpl	$0, __cpu_features+KIND_OFFSET(%rip)
> +	jne	1f
> +	call	__init_cpu_features
> +1:	leaq	__memset_sse2(%rip), %rax
> +	testl	$bit_AVX2_Usable,
> __cpu_features+FEATURE_OFFSET+index_AVX2_Usable(%rip)
> +	jz	2f
> +	leaq	__memset_avx2(%rip), %rax
> +2:	ret
> +END(memset)
> +#endif
> +
> +#if !defined NOT_IN_libc
> +# undef memset
> +# define memset __memset_sse2
> +
> +# undef __memset_chk
> +# define __memset_chk __memset_chk_sse2
> +
> +# ifdef SHARED
> +#  undef libc_hidden_builtin_def
> +/* It doesn't make sense to send libc-internal memset calls through a PLT.
> +   The speedup we get from using GPR instruction is likely eaten away
> +   by the indirect call in the PLT.  */
> +#  define libc_hidden_builtin_def(name) \
> +	.globl __GI_memset; __GI_memset = __memset_sse2
> +# endif
> +
> +# undef strong_alias
> +# define strong_alias(original, alias)
> +#endif
> +
> +#include "../memset.S"
> diff --git a/sysdeps/x86_64/multiarch/memset_chk.S
> b/sysdeps/x86_64/multiarch/memset_chk.S
> new file mode 100644
> index 0000000..aeae56f
> --- /dev/null
> +++ b/sysdeps/x86_64/multiarch/memset_chk.S
> @@ -0,0 +1,43 @@
> +/* Multiple versions of memset_chk
> +   Copyright (C) 2014 Free Software Foundation, Inc.
> +   This file is part of the GNU C Library.
> +
> +   The GNU C Library is free software; you can redistribute it and/or
> +   modify it under the terms of the GNU Lesser General Public
> +   License as published by the Free Software Foundation; either
> +   version 2.1 of the License, or (at your option) any later version.
> +
> +   The GNU C Library is distributed in the hope that it will be useful,
> +   but WITHOUT ANY WARRANTY; without even the implied warranty of
> +   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> +   Lesser General Public License for more details.
> +
> +   You should have received a copy of the GNU Lesser General Public
> +   License along with the GNU C Library; if not, see
> +   <http://www.gnu.org/licenses/>.  */
> +
> +#include <sysdep.h>
> +#include <init-arch.h>
> +
> +/* Define multiple versions only for the definition in lib.  */
> +#ifndef NOT_IN_libc
> +# ifdef SHARED
> +ENTRY(__memset_chk)
> +	.type	__memset_chk, @gnu_indirect_function
> +	cmpl	$0, __cpu_features+KIND_OFFSET(%rip)
> +	jne	1f
> +	call	__init_cpu_features
> +1:	leaq	__memset_chk_sse2(%rip), %rax
> +	testl	$bit_AVX2_Usable,
> __cpu_features+FEATURE_OFFSET+index_AVX2_Usable(%rip)
> +	jz	2f
> +	leaq	__memset_chk_avx2(%rip), %rax
> +2:	ret
> +END(__memset_chk)
> +
> +strong_alias (__memset_chk, __memset_zero_constant_len_parameter)
> +	.section .gnu.warning.__memset_zero_constant_len_parameter
> +	.string "memset used with constant zero length parameter; this could be
> due to transposed parameters"
> +# else
> +#  include "../memset_chk.S"
> +# endif
> +#endif
> --
> 1.8.1.4
>
>
Index Nav:	[Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav:	[Date Prev] [Date Next]	[Thread Prev] [Thread Next]