[PATCH 2/2] Improve memset for newer processors.

[OndÅej BÃlka <neleai at seznam dot cz>]

Hi, this optimizes memset for rest of processors. I simplified a control
flow to get 5% speedup on gcc workloads for most architectures.
Exception is xeon where rep implementation is better.

I used a 32-byte loop here as it has nearly identical performance as a
64 byte one and is shorter. Originally I wanted to use a 16 byte loop
but it was slow on nehalem, performance characteristics there are
chaotic, for example I could get a 20% speedup by filling a nop before 
a loop start.

I header there could be probably optimized a bit more but my attempts
were hard to distinguish from noise.

Benchmarks are here.

http://kam.mff.cuni.cz/~ondra/benchmark_string/memset_profile_loop.html
http://kam.mff.cuni.cz/~ondra/benchmark_string/memset_profile_loop300913.tar.bz2

Comments?

	* sysdeps/x86_64/multiarch/memset-ssse3.S: Improve implementation.

---
 sysdeps/x86_64/multiarch/memset-ssse3.S | 111 +++++++++++++++-----------------
 1 file changed, 52 insertions(+), 59 deletions(-)

diff --git a/sysdeps/x86_64/multiarch/memset-ssse3.S b/sysdeps/x86_64/multiarch/memset-ssse3.S
index 68c9490..7fe6866 100644
--- a/sysdeps/x86_64/multiarch/memset-ssse3.S
+++ b/sysdeps/x86_64/multiarch/memset-ssse3.S
@@ -25,76 +25,69 @@
 
 	.text
 ENTRY (__memset_ssse3)
-	movd	%esi, %xmm8
+	movd	%esi, %xmm0
+	pxor	%xmm1, %xmm1
+	pshufb	%xmm1, %xmm0
+	lea	(%rdi, %rdx), %rsi
+	cmp	$48, %rdx
+	jbe	L(less_48_bytes)
+	leaq	16(%rdi), %rcx
+	andq	$-16, %rcx
+	movdqu	%xmm0, -16(%rsi)
+	movdqu	%xmm0, -32(%rsi)
 	movq	%rdi, %rax
-        punpcklbw       %xmm8, %xmm8
-        punpcklwd       %xmm8, %xmm8
-        pshufd  $0, %xmm8, %xmm8
-	cmpq	$64, %rdx
-	ja	L(loop_start)
-	cmpq	$16, %rdx
-	jbe	L(less_16_bytes)
-	cmpq	$32, %rdx
-	movdqu	%xmm8, (%rdi)
-	movdqu	%xmm8, -16(%rdi,%rdx)
-	ja	L(between_32_64_bytes)
-L(return):
-	rep
-	ret
-	ALIGN (4)
-L(between_32_64_bytes):
-	movdqu	%xmm8, 16(%rdi)
-	movdqu	%xmm8, -32(%rdi,%rdx)
-	ret
-	ALIGN (4)
-L(loop_start):
-	leaq	64(%rdi), %rcx
-	movdqu	%xmm8, (%rdi)
-	andq	$-64, %rcx
-	movdqu	%xmm8, -16(%rdi,%rdx)
-	movdqu	%xmm8, 16(%rdi)
-	movdqu	%xmm8, -32(%rdi,%rdx)
-	movdqu	%xmm8, 32(%rdi)
-	movdqu	%xmm8, -48(%rdi,%rdx)
-	movdqu	%xmm8, 48(%rdi)
-	movdqu	%xmm8, -64(%rdi,%rdx)
-	addq	%rdi, %rdx
-	andq	$-64, %rdx
-	cmpq	%rdx, %rcx
-	je	L(return)
-	ALIGN (4)
+	movdqu	%xmm0, (%rdi)
+	subq	%rcx, %rsi
+	shrq	$5, %rsi
 L(loop):
-	movdqa	%xmm8, (%rcx)
-	movdqa	%xmm8, 16(%rcx)
-	movdqa	%xmm8, 32(%rcx)
-	movdqa	%xmm8, 48(%rcx)
-	addq	$64, %rcx
-	cmpq	%rcx, %rdx
+	movdqa	%xmm0, (%rcx)
+	movdqa	%xmm0, 16(%rcx)
+	addq	$32, %rcx
+	subq	$1, %rsi
 	jne	L(loop)
-	rep
 	ret
+
+L(between_0_1_bytes):
+	jb	L(zero_byte)
+	movb	%cl, (%rdi)
+L(zero_byte):
+       	ret
+
+	.p2align 4
+L(less_48_bytes):
+	movq	%rdi, %rax
+	cmpl	$16, %edx
+	jb	L(less_16_bytes)
+	subq	$16, %rdx
+	shrq	$1, %rdx
+	movdqu	%xmm0, (%rdi)
+	movdqu	%xmm0, -16(%rsi)
+	movdqu	%xmm0, (%rdi, %rdx)
+	ret
+
+	.p2align 4
 L(less_16_bytes):
-	movq	%xmm8, %rcx
-	testb	$24, %dl
-	jne	L(between8_16bytes)
+	movq	%xmm0, %rcx
+	testb	$8, %dl
+	jne	L(between_8_15_bytes)
 	testb	$4, %dl
-	jne	L(between4_7bytes)
-	testb	$1, %dl
-	je	L(odd_byte)
+	jne	L(between_4_7_bytes)
+	cmp	$1, %dl
+	jbe	L(between_0_1_bytes)
+	movw	%cx, -2(%rsi)
 	movb	%cl, (%rdi)
-L(odd_byte):
-	testb	$2, %dl
-	je	L(return)
-	movw	%cx, -2(%rax,%rdx)
 	ret
-L(between4_7bytes):
+
+	.p2align 3
+L(between_4_7_bytes):
 	movl	%ecx, (%rdi)
-	movl	%ecx, -4(%rdi,%rdx)
+	movl	%ecx, -4(%rsi)
 	ret
-L(between8_16bytes):
+
+	.p2align 3
+L(between_8_15_bytes):
 	movq	%rcx, (%rdi)
-	movq	%rcx, -8(%rdi,%rdx)
+	movq	%rcx, -8(%rsi)
 	ret
-
 END (__memset_ssse3)
 
-- 
1.8.4.rc3