This is the mail archive of the libc-hacker@sources.redhat.com mailing list for the glibc project.
Note that libc-hacker is a closed list. You may look at the archives of this list, but subscription and posting are not open.
| Index Nav: | [Date Index] [Subject Index] [Author Index] [Thread Index] | |
|---|---|---|
| Message Nav: | [Date Prev] [Date Next] | [Thread Prev] [Thread Next] |
| Other format: | [Raw text] | |
It appears that the Itanium 2 optimized versions of these routines
never were checked in (unless I'm missing something). I cleaned them
up some more and if they look OK, I'd appreciate it if someone could
check them in. I did run "make check" successfully and the routines
have been in use as part of the Debian glibc package for some time.
There is also an optimized memcmp() routine but it needs some more
debugging/testing before it's ready for prime-time.
--david
2002-10-02 David Mosberger <davidm@hpl.hp.com>
* sysdeps/ia64/bzero.S: Rewritten by Sverre Jarp to tune for
Itanium 2 (and Itanium).
Fix unwind directives and make it fit in 80 columns.
* sysdeps/ia64/memset.S: Ditto.
* sysdeps/ia64/memcpy.S: Ditto.
Move jump table to .rodata section.
Index: sysdeps/ia64/bzero.S
===================================================================
RCS file: /cvs/glibc/libc/sysdeps/ia64/bzero.S,v
retrieving revision 1.3
diff -u -r1.3 bzero.S
--- sysdeps/ia64/sysdeps/ia64/bzero.S 25 Aug 2002 05:25:12 -0000 1.3
+++ sysdeps/ia64/sysdeps/ia64/bzero.S 3 Oct 2002 06:00:19 -0000
@@ -1,7 +1,8 @@
/* Optimized version of the standard bzero() function.
This file is part of the GNU C Library.
Copyright (C) 2000, 2001, 2002 Free Software Foundation, Inc.
- Contributed by Dan Pop <Dan.Pop@cern.ch>.
+ Contributed by Dan Pop for Itanium <Dan.Pop@cern.ch>.
+ Rewritten for McKinley by Sverre Jarp, HP Labs/CERN <Sverre.Jarp@cern.ch>
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
@@ -25,8 +26,11 @@
in1: count
The algorithm is fairly straightforward: set byte by byte until we
- we get to a word aligned address, then set word by word as much as
- possible; the remaining few bytes are set one by one. */
+ we get to a 16B-aligned address, then loop on 128 B chunks using an
+ early store as prefetching, then loop on 32B chucks, then clear remaining
+ words, finally clear remaining bytes.
+ Since a stf.spill f0 can store 16B in one go, we use this instruction
+ to get peak speed. */
#include <sysdep.h>
#undef ret
@@ -34,62 +38,278 @@
#define dest in0
#define cnt in1
-#define save_pfs loc0
-#define ptr1 loc1
-#define ptr2 loc2
-#define tmp loc3
-#define loopcnt loc4
-#define save_lc loc5
+#define tmp r31
+#define save_lc r30
+#define ptr0 r29
+#define ptr1 r28
+#define ptr2 r27
+#define ptr3 r26
+#define ptr9 r24
+#define loopcnt r23
+#define linecnt r22
+#define bytecnt r21
-ENTRY(__bzero)
+// This routine uses only scratch predicate registers (p6 - p15)
+#define p_scr p6 // default register for same-cycle branches
+#define p_unalgn p9
+#define p_y p11
+#define p_n p12
+#define p_yy p13
+#define p_nn p14
+
+#define movi0 mov
+
+#define MIN1 15
+#define MIN1P1HALF 8
+#define LINE_SIZE 128
+#define LSIZE_SH 7 // shift amount
+#define PREF_AHEAD 8
+
+#define USE_FLP
+#if defined(USE_INT)
+#define store st8
+#define myval r0
+#elif defined(USE_FLP)
+#define store stf8
+#define myval f0
+#endif
+
+.align 64
+ENTRY(bzero)
+{ .mmi
.prologue
- alloc save_pfs = ar.pfs, 2, 6, 0, 0
- .save ar.lc, save_lc
- mov save_lc = ar.lc
+ alloc tmp = ar.pfs, 2, 0, 0, 0
+ lfetch.nt1 [dest]
+ .save ar.lc, save_lc
+ movi0 save_lc = ar.lc
+} { .mmi
.body
- mov ret0 = dest
- and tmp = 7, dest
- cmp.eq p6, p0 = cnt, r0
-(p6) br.cond.spnt .restore_and_exit ;;
+ mov ret0 = dest // return value
+ nop.m 0
+ cmp.eq p_scr, p0 = cnt, r0
+;; }
+{ .mmi
+ and ptr2 = -(MIN1+1), dest // aligned address
+ and tmp = MIN1, dest // prepare to check for alignment
+ tbit.nz p_y, p_n = dest, 0 // Do we have an odd address? (M_B_U)
+} { .mib
mov ptr1 = dest
- sub loopcnt = 8, tmp
- cmp.gt p6, p0 = 16, cnt
-(p6) br.cond.spnt .set_few;;
- cmp.eq p6, p0 = tmp, r0
-(p6) br.cond.sptk .dest_aligned
- sub cnt = cnt, loopcnt
- adds loopcnt = -1, loopcnt;;
- mov ar.lc = loopcnt;;
-.l1:
- st1 [ptr1] = r0, 1
- br.cloop.dptk .l1 ;;
-.dest_aligned:
- adds ptr2 = 8, ptr1
- shr.u loopcnt = cnt, 4 ;; // loopcnt = cnt / 16
- cmp.eq p6, p0 = loopcnt, r0
-(p6) br.cond.spnt .one_more
- and cnt = 0xf, cnt // compute the remaining cnt
- adds loopcnt = -1, loopcnt;;
- mov ar.lc = loopcnt;;
-.l2:
- st8 [ptr1] = r0, 16
- st8 [ptr2] = r0, 16
- br.cloop.dptk .l2
- cmp.le p6, p0 = 8, cnt ;;
-.one_more:
-(p6) st8 [ptr1] = r0, 8
-(p6) adds cnt = -8, cnt ;;
- cmp.eq p6, p0 = cnt, r0
-(p6) br.cond.spnt .restore_and_exit
-.set_few:
- adds loopcnt = -1, cnt;;
- mov ar.lc = loopcnt;;
-.l3:
- st1 [ptr1] = r0, 1
- br.cloop.dptk .l3 ;;
+ nop.i 0
+(p_scr) br.ret.dpnt.many rp // return immediately if count = 0
+;; }
+{ .mib
+ cmp.ne p_unalgn, p0 = tmp, r0
+} { .mib // NB: # of bytes to move is 1
+ sub bytecnt = (MIN1+1), tmp // higher than loopcnt
+ cmp.gt p_scr, p0 = 16, cnt // is it a minimalistic task?
+(p_scr) br.cond.dptk.many .move_bytes_unaligned // go move just a few (M_B_U)
+;; }
+{ .mmi
+(p_unalgn) add ptr1 = (MIN1+1), ptr2 // after alignment
+(p_unalgn) add ptr2 = MIN1P1HALF, ptr2 // after alignment
+(p_unalgn) tbit.nz.unc p_y, p_n = bytecnt, 3 // should we do a st8 ?
+;; }
+{ .mib
+(p_y) add cnt = -8, cnt
+(p_unalgn) tbit.nz.unc p_yy, p_nn = bytecnt, 2 // should we do a st4 ?
+} { .mib
+(p_y) st8 [ptr2] = r0,-4
+(p_n) add ptr2 = 4, ptr2
+;; }
+{ .mib
+(p_yy) add cnt = -4, cnt
+(p_unalgn) tbit.nz.unc p_y, p_n = bytecnt, 1 // should we do a st2 ?
+} { .mib
+(p_yy) st4 [ptr2] = r0,-2
+(p_nn) add ptr2 = 2, ptr2
+;; }
+{ .mmi
+ mov tmp = LINE_SIZE+1 // for compare
+(p_y) add cnt = -2, cnt
+(p_unalgn) tbit.nz.unc p_yy, p_nn = bytecnt, 0 // should we do a st1 ?
+} { .mmi
+ nop.m 0
+(p_y) st2 [ptr2] = r0,-1
+(p_n) add ptr2 = 1, ptr2
+;; }
+
+{ .mmi
+(p_yy) st1 [ptr2] = r0
+ cmp.gt p_scr, p0 = tmp, cnt // is it a minimalistic task?
+} { .mbb
+(p_yy) add cnt = -1, cnt
+(p_scr) br.cond.dpnt.many .fraction_of_line // go move just a few
+;; }
+{ .mib
+ nop.m 0
+ shr.u linecnt = cnt, LSIZE_SH
+ nop.b 0
+;; }
+
+ .align 32
+.l1b: // ------------------// L1B: store ahead into cache lines; fill later
+{ .mmi
+ and tmp = -(LINE_SIZE), cnt // compute end of range
+ mov ptr9 = ptr1 // used for prefetching
+ and cnt = (LINE_SIZE-1), cnt // remainder
+} { .mmi
+ mov loopcnt = PREF_AHEAD-1 // default prefetch loop
+ cmp.gt p_scr, p0 = PREF_AHEAD, linecnt // check against actual value
+;; }
+{ .mmi
+(p_scr) add loopcnt = -1, linecnt
+ add ptr2 = 16, ptr1 // start of stores (beyond prefetch stores)
+ add ptr1 = tmp, ptr1 // first address beyond total range
+;; }
+{ .mmi
+ add tmp = -1, linecnt // next loop count
+ movi0 ar.lc = loopcnt
+;; }
+.pref_l1b:
+{ .mib
+ stf.spill [ptr9] = f0, 128 // Do stores one cache line apart
+ nop.i 0
+ br.cloop.dptk.few .pref_l1b
+;; }
+{ .mmi
+ add ptr0 = 16, ptr2 // Two stores in parallel
+ movi0 ar.lc = tmp
+;; }
+.l1bx:
+ { .mmi
+ stf.spill [ptr2] = f0, 32
+ stf.spill [ptr0] = f0, 32
+ ;; }
+ { .mmi
+ stf.spill [ptr2] = f0, 32
+ stf.spill [ptr0] = f0, 32
+ ;; }
+ { .mmi
+ stf.spill [ptr2] = f0, 32
+ stf.spill [ptr0] = f0, 64
+ cmp.lt p_scr, p0 = ptr9, ptr1 // do we need more prefetching?
+ ;; }
+{ .mmb
+ stf.spill [ptr2] = f0, 32
+(p_scr) stf.spill [ptr9] = f0, 128
+ br.cloop.dptk.few .l1bx
+;; }
+{ .mib
+ cmp.gt p_scr, p0 = 8, cnt // just a few bytes left ?
+(p_scr) br.cond.dpnt.many .move_bytes_from_alignment
+;; }
+
+.fraction_of_line:
+{ .mib
+ add ptr2 = 16, ptr1
+ shr.u loopcnt = cnt, 5 // loopcnt = cnt / 32
+;; }
+{ .mib
+ cmp.eq p_scr, p0 = loopcnt, r0
+ add loopcnt = -1, loopcnt
+(p_scr) br.cond.dpnt.many .store_words
+;; }
+{ .mib
+ and cnt = 0x1f, cnt // compute the remaining cnt
+ movi0 ar.lc = loopcnt
+;; }
+ .align 32
+.l2: // -----------------------------// L2A: store 32B in 2 cycles
+{ .mmb
+ store [ptr1] = myval, 8
+ store [ptr2] = myval, 8
+;; } { .mmb
+ store [ptr1] = myval, 24
+ store [ptr2] = myval, 24
+ br.cloop.dptk.many .l2
+;; }
+.store_words:
+{ .mib
+ cmp.gt p_scr, p0 = 8, cnt // just a few bytes left ?
+(p_scr) br.cond.dpnt.many .move_bytes_from_alignment // Branch
+;; }
+
+{ .mmi
+ store [ptr1] = myval, 8 // store
+ cmp.le p_y, p_n = 16, cnt //
+ add cnt = -8, cnt // subtract
+;; }
+{ .mmi
+(p_y) store [ptr1] = myval, 8 // store
+(p_y) cmp.le.unc p_yy, p_nn = 16, cnt
+(p_y) add cnt = -8, cnt // subtract
+;; }
+{ .mmi // store
+(p_yy) store [ptr1] = myval, 8
+(p_yy) add cnt = -8, cnt // subtract
+;; }
+
+.move_bytes_from_alignment:
+{ .mib
+ cmp.eq p_scr, p0 = cnt, r0
+ tbit.nz.unc p_y, p0 = cnt, 2 // should we terminate with a st4 ?
+(p_scr) br.cond.dpnt.few .restore_and_exit
+;; }
+{ .mib
+(p_y) st4 [ptr1] = r0,4
+ tbit.nz.unc p_yy, p0 = cnt, 1 // should we terminate with a st2 ?
+;; }
+{ .mib
+(p_yy) st2 [ptr1] = r0,2
+ tbit.nz.unc p_y, p0 = cnt, 0 // should we terminate with a st1 ?
+;; }
+
+{ .mib
+(p_y) st1 [ptr1] = r0
+;; }
.restore_and_exit:
- mov ar.lc = save_lc
- mov ar.pfs = save_pfs
- br.ret.sptk.many b0
-END(__bzero)
-weak_alias (__bzero, bzero)
+{ .mib
+ nop.m 0
+ movi0 ar.lc = save_lc
+ br.ret.sptk.many rp
+;; }
+
+.move_bytes_unaligned:
+{ .mmi
+ .pred.rel "mutex",p_y, p_n
+ .pred.rel "mutex",p_yy, p_nn
+(p_n) cmp.le p_yy, p_nn = 4, cnt
+(p_y) cmp.le p_yy, p_nn = 5, cnt
+(p_n) add ptr2 = 2, ptr1
+} { .mmi
+(p_y) add ptr2 = 3, ptr1
+(p_y) st1 [ptr1] = r0, 1 // fill 1 (odd-aligned) byte
+(p_y) add cnt = -1, cnt // [15, 14 (or less) left]
+;; }
+{ .mmi
+(p_yy) cmp.le.unc p_y, p0 = 8, cnt
+ add ptr3 = ptr1, cnt // prepare last store
+ movi0 ar.lc = save_lc
+} { .mmi
+(p_yy) st2 [ptr1] = r0, 4 // fill 2 (aligned) bytes
+(p_yy) st2 [ptr2] = r0, 4 // fill 2 (aligned) bytes
+(p_yy) add cnt = -4, cnt // [11, 10 (o less) left]
+;; }
+{ .mmi
+(p_y) cmp.le.unc p_yy, p0 = 8, cnt
+ add ptr3 = -1, ptr3 // last store
+ tbit.nz p_scr, p0 = cnt, 1 // will there be a st2 at the end ?
+} { .mmi
+(p_y) st2 [ptr1] = r0, 4 // fill 2 (aligned) bytes
+(p_y) st2 [ptr2] = r0, 4 // fill 2 (aligned) bytes
+(p_y) add cnt = -4, cnt // [7, 6 (or less) left]
+;; }
+{ .mmi
+(p_yy) st2 [ptr1] = r0, 4 // fill 2 (aligned) bytes
+(p_yy) st2 [ptr2] = r0, 4 // fill 2 (aligned) bytes
+ // [3, 2 (or less) left]
+ tbit.nz p_y, p0 = cnt, 0 // will there be a st1 at the end ?
+} { .mmi
+(p_yy) add cnt = -4, cnt
+;; }
+{ .mmb
+(p_scr) st2 [ptr1] = r0 // fill 2 (aligned) bytes
+(p_y) st1 [ptr3] = r0 // fill last byte (using ptr3)
+ br.ret.sptk.many rp
+;; }
+END(bzero)
Index: sysdeps/ia64/memcpy.S
===================================================================
RCS file: /cvs/glibc/libc/sysdeps/ia64/memcpy.S,v
retrieving revision 1.8
diff -u -r1.8 memcpy.S
--- sysdeps/ia64/sysdeps/ia64/memcpy.S 6 Jul 2001 04:55:54 -0000 1.8
+++ sysdeps/ia64/sysdeps/ia64/memcpy.S 3 Oct 2002 06:00:19 -0000
@@ -1,7 +1,8 @@
/* Optimized version of the standard memcpy() function.
This file is part of the GNU C Library.
Copyright (C) 2000, 2001 Free Software Foundation, Inc.
- Contributed by Dan Pop <Dan.Pop@cern.ch>.
+ Contributed by Dan Pop for Itanium <Dan.Pop@cern.ch>.
+ Rewritten for McKinley by Sverre Jarp, HP Labs/CERN <Sverre.Jarp@cern.ch>
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
@@ -26,27 +27,39 @@
in2: byte count
An assembly implementation of the algorithm used by the generic C
- version from glibc. The case when all three arguments are multiples
- of 8 is treated separatedly, for extra performance.
+ version from glibc. The case when source and sest are aligned is
+ treated separately, for extra performance.
- In this form, it assumes little endian mode. For big endian mode,
+ In this form, memcpy assumes little endian mode. For big endian mode,
sh1 must be computed using an extra instruction: sub sh1 = 64, sh1
and the order of r[MEMLAT] and r[MEMLAT+1] must be reverted in the
shrp instruction. */
+#define USE_LFETCH
+#define USE_FLP
#include <sysdep.h>
#undef ret
+#define LFETCH_DIST 500
+
+#define ALIGN_UNROLL_no 4 // no. of elements
+#define ALIGN_UNROLL_sh 2 // (shift amount)
+
+#define MEMLAT 8
+#define Nrot ((4*(MEMLAT+2) + 7) & ~7)
+
#define OP_T_THRES 16
#define OPSIZ 8
-#define adest r15
-#define saved_pr r17
-#define saved_lc r18
+#define loopcnt r14
+#define elemcnt r15
+#define saved_pr r16
+#define saved_lc r17
+#define adest r18
#define dest r19
-#define src r20
-#define len r21
-#define asrc r22
+#define asrc r20
+#define src r21
+#define len r22
#define tmp2 r23
#define tmp3 r24
#define tmp4 r25
@@ -54,113 +67,339 @@
#define ploop56 r27
#define loopaddr r28
#define sh1 r29
-#define loopcnt r30
-#define value r31
+#define ptr1 r30
+#define ptr2 r31
-#define LOOP(shift) \
- .align 32 ; \
-.loop##shift##: \
-(p[0]) ld8 r[0] = [asrc], 8 ; /* w1 */ \
-(p[MEMLAT+1]) st8 [dest] = value, 8 ; \
-(p[MEMLAT]) shrp value = r[MEMLAT], r[MEMLAT+1], shift ; \
- nop.b 0 ; \
- nop.b 0 ; \
- br.ctop.sptk .loop##shift ; \
- br.cond.sptk .cpyfew ; /* deal with the remaining bytes */
+#define movi0 mov
+
+#define p_scr p6
+#define p_xtr p7
+#define p_nxtr p8
+#define p_few p9
+
+#if defined(USE_FLP)
+#define load ldf8
+#define store stf8
+#define tempreg f6
+#define the_r fr
+#define the_s fs
+#define the_t ft
+#define the_q fq
+#define the_w fw
+#define the_x fx
+#define the_y fy
+#define the_z fz
+#elif defined(USE_INT)
+#define load ld8
+#define store st8
+#define tempreg tmp2
+#define the_r r
+#define the_s s
+#define the_t t
+#define the_q q
+#define the_w w
+#define the_x x
+#define the_y y
+#define the_z z
+#endif
+
+
+#if defined(USE_LFETCH)
+#define LOOP(shift) \
+ .align 32 ; \
+.loop##shift##: \
+{ .mmb \
+(p[0]) ld8.nt1 r[0] = [asrc], 8 ; \
+(p[0]) lfetch.nt1 [ptr1], 16 ; \
+ nop.b 0 ; \
+} { .mib \
+(p[MEMLAT+1]) st8 [dest] = tmp3, 8 ; \
+(p[MEMLAT]) shrp tmp3 = r[MEMLAT], s[MEMLAT+1], shift ; \
+ nop.b 0 ;; \
+ } { .mmb \
+(p[0]) ld8.nt1 s[0] = [asrc], 8 ; \
+(p[0]) lfetch.nt1 [ptr2], 16 ; \
+ nop.b 0 ; \
+} { .mib \
+(p[MEMLAT+1]) st8 [dest] = tmp4, 8 ; \
+(p[MEMLAT]) shrp tmp4 = s[MEMLAT], r[MEMLAT], shift ; \
+ br.ctop.sptk.many .loop##shift \
+;; } \
+{ .mib \
+ br.cond.sptk.many .copy_bytes ; /* deal with the remaining bytes */ \
+}
+#else
+#define LOOP(shift) \
+ .align 32 ; \
+.loop##shift##: \
+{ .mmb \
+(p[0]) ld8.nt1 r[0] = [asrc], 8 ; \
+ nop.b 0 ; \
+} { .mib \
+(p[MEMLAT+1]) st8 [dest] = tmp3, 8 ; \
+(p[MEMLAT]) shrp tmp3 = r[MEMLAT], s[MEMLAT+1], shift ; \
+ nop.b 0 ;; \
+ } { .mmb \
+(p[0]) ld8.nt1 s[0] = [asrc], 8 ; \
+ nop.b 0 ; \
+} { .mib \
+(p[MEMLAT+1]) st8 [dest] = tmp4, 8 ; \
+(p[MEMLAT]) shrp tmp4 = s[MEMLAT], r[MEMLAT], shift ; \
+ br.ctop.sptk.many .loop##shift \
+;; } \
+{ .mib \
+ br.cond.sptk.many .copy_bytes ; /* deal with the remaining bytes */ \
+}
+#endif
-#define MEMLAT 21
-#define Nrot (((2*MEMLAT+3) + 7) & ~7)
ENTRY(memcpy)
+{ .mmi
.prologue
alloc r2 = ar.pfs, 3, Nrot - 3, 0, Nrot
- .rotr r[MEMLAT + 2], q[MEMLAT + 1]
- .rotp p[MEMLAT + 2]
- mov ret0 = in0 // return value = dest
- .save pr, saved_pr
- mov saved_pr = pr // save the predicate registers
- .save ar.lc, saved_lc
- mov saved_lc = ar.lc // save the loop counter
- .body
- or tmp3 = in0, in1 ;; // tmp3 = dest | src
- or tmp3 = tmp3, in2 // tmp3 = dest | src | len
+ .rotr r[MEMLAT+1], s[MEMLAT+2], q[MEMLAT+1], t[MEMLAT+1]
+ .rotp p[MEMLAT+2]
+ .rotf fr[MEMLAT+1], fq[MEMLAT+1], fs[MEMLAT+1], ft[MEMLAT+1]
+ mov ret0 = in0 // return tmp2 = dest
+ .save pr, saved_pr
+ movi0 saved_pr = pr // save the predicate registers
+} { .mmi
+ and tmp4 = 7, in0 // check if destination is aligned
mov dest = in0 // dest
mov src = in1 // src
+;; }
+{ .mii
+ cmp.eq p_scr, p0 = in2, r0 // if (len == 0)
+ .save ar.lc, saved_lc
+ movi0 saved_lc = ar.lc // save the loop counter
+ .body
+ cmp.ge p_few, p0 = OP_T_THRES, in2 // is len <= OP_T_THRESH
+} { .mbb
mov len = in2 // len
- sub tmp2 = r0, in0 // tmp2 = -dest
- cmp.eq p6, p0 = in2, r0 // if (len == 0)
-(p6) br.cond.spnt .restore_and_exit;;// return dest;
- and tmp4 = 7, tmp3 // tmp4 = (dest | src | len) & 7
- shr.u loopcnt = len, 4 ;; // loopcnt = len / 16
- cmp.ne p6, p0 = tmp4, r0 // if ((dest | src | len) & 7 != 0)
-(p6) br.cond.sptk .next // goto next;
-
-// The optimal case, when dest, src and len are all multiples of 8
-
- and tmp3 = 0xf, len // tmp3 = len % 16
- mov pr.rot = 1 << 16 // set rotating predicates
- mov ar.ec = MEMLAT + 1 ;; // set the epilog counter
- cmp.ne p6, p0 = tmp3, r0 // do we have to copy an extra word?
- adds loopcnt = -1, loopcnt;; // --loopcnt
-(p6) ld8 value = [src], 8;;
-(p6) st8 [dest] = value, 8 // copy the "extra" word
- mov ar.lc = loopcnt // set the loop counter
- cmp.eq p6, p0 = 8, len
-(p6) br.cond.spnt .restore_and_exit;;// there was only one word to copy
- adds adest = 8, dest
- adds asrc = 8, src ;;
+(p_scr) br.cond.dpnt.few .restore_and_exit // Branch no. 1: return dest
+(p_few) br.cond.dpnt.many .copy_bytes // Branch no. 2: copy byte by byte
+;; }
+{ .mmi
+#if defined(USE_LFETCH)
+ lfetch.nt1 [dest] //
+ lfetch.nt1 [src] //
+#endif
+ shr.u elemcnt = len, 3 // elemcnt = len / 8
+} { .mib
+ cmp.eq p_scr, p0 = tmp4, r0 // is destination aligned?
+ sub loopcnt = 7, tmp4 //
+(p_scr) br.cond.dptk.many .dest_aligned
+;; }
+{ .mmi
+ ld1 tmp2 = [src], 1 //
+ sub len = len, loopcnt, 1 // reduce len
+ movi0 ar.lc = loopcnt //
+} { .mib
+ cmp.ne p_scr, p0 = 0, loopcnt // avoid loading beyond end-point
+;; }
+
+.l0: // ---------------------------- // L0: Align src on 8-byte boundary
+{ .mmi
+ st1 [dest] = tmp2, 1 //
+(p_scr) ld1 tmp2 = [src], 1 //
+} { .mib
+ cmp.lt p_scr, p0 = 1, loopcnt // avoid load beyond end-point
+ add loopcnt = -1, loopcnt
+ br.cloop.dptk.few .l0 //
+;; }
+
+.dest_aligned:
+{ .mmi
+ and tmp4 = 7, src // ready for alignment check
+ shr.u elemcnt = len, 3 // elemcnt = len / 8
+;; }
+{ .mib
+ cmp.ne p_scr, p0 = tmp4, r0 // is source also aligned
+ tbit.nz p_xtr, p_nxtr = src, 3 // prepare a separate move if src
+} { .mib // is not 16B aligned
+ add ptr2 = LFETCH_DIST, dest // prefetch address
+ add ptr1 = LFETCH_DIST, src
+(p_scr) br.cond.dptk.many .src_not_aligned
+;; }
+
+// The optimal case, when dest, and src are aligned
+
+.both_aligned:
+{ .mmi
+ .pred.rel "mutex",p_xtr,p_nxtr
+(p_xtr) cmp.gt p_scr, p0 = ALIGN_UNROLL_no+1, elemcnt // Need N + 1 to qualify
+(p_nxtr) cmp.gt p_scr, p0 = ALIGN_UNROLL_no, elemcnt // Need only N to qualify
+ movi0 pr.rot = 1 << 16 // set rotating predicates
+} { .mib
+(p_scr) br.cond.dpnt.many .copy_full_words
+;; }
+
+{ .mmi
+(p_xtr) load tempreg = [src], 8
+(p_xtr) add elemcnt = -1, elemcnt
+ movi0 ar.ec = MEMLAT + 1 // set the epilog counter
+;; }
+{ .mmi
+(p_xtr) add len = -8, len //
+ add asrc = 16, src // one bank apart (for USE_INT)
+ shr.u loopcnt = elemcnt, ALIGN_UNROLL_sh // cater for unrolling
+;;}
+{ .mmi
+ add loopcnt = -1, loopcnt
+(p_xtr) store [dest] = tempreg, 8 // copy the "extra" word
+ nop.i 0
+;; }
+{ .mib
+ add adest = 16, dest
+ movi0 ar.lc = loopcnt // set the loop counter
+;; }
+
.align 32
-.l0:
-(p[0]) ld8 r[0] = [src], 16
-(p[0]) ld8 q[0] = [asrc], 16
-(p[MEMLAT]) st8 [dest] = r[MEMLAT], 16
-(p[MEMLAT]) st8 [adest] = q[MEMLAT], 16
- br.ctop.dptk .l0 ;;
+#if defined(USE_FLP)
+.l1: // ------------------------------- // L1: Everything a multiple of 8
+{ .mmi
+#if defined(USE_LFETCH)
+(p[0]) lfetch.nt1 [ptr2],32
+#endif
+(p[0]) ldfp8 the_r[0],the_q[0] = [src], 16
+(p[0]) add len = -32, len
+} {.mmb
+(p[MEMLAT]) store [dest] = the_r[MEMLAT], 8
+(p[MEMLAT]) store [adest] = the_s[MEMLAT], 8
+;; }
+{ .mmi
+#if defined(USE_LFETCH)
+(p[0]) lfetch.nt1 [ptr1],32
+#endif
+(p[0]) ldfp8 the_s[0], the_t[0] = [src], 16
+} {.mmb
+(p[MEMLAT]) store [dest] = the_q[MEMLAT], 24
+(p[MEMLAT]) store [adest] = the_t[MEMLAT], 24
+ br.ctop.dptk.many .l1
+;; }
+#elif defined(USE_INT)
+.l1: // ------------------------------- // L1: Everything a multiple of 8
+{ .mmi
+(p[0]) load the_r[0] = [src], 8
+(p[0]) load the_q[0] = [asrc], 8
+(p[0]) add len = -32, len
+} {.mmb
+(p[MEMLAT]) store [dest] = the_r[MEMLAT], 8
+(p[MEMLAT]) store [adest] = the_q[MEMLAT], 8
+;; }
+{ .mmi
+(p[0]) load the_s[0] = [src], 24
+(p[0]) load the_t[0] = [asrc], 24
+} {.mmb
+(p[MEMLAT]) store [dest] = the_s[MEMLAT], 24
+(p[MEMLAT]) store [adest] = the_t[MEMLAT], 24
+#if defined(USE_LFETCH)
+;; }
+{ .mmb
+(p[0]) lfetch.nt1 [ptr2],32
+(p[0]) lfetch.nt1 [ptr1],32
+#endif
+ br.ctop.dptk.many .l1
+;; }
+#endif
+
+.copy_full_words:
+{ .mib
+ cmp.gt p_scr, p0 = 8, len //
+ shr.u elemcnt = len, 3 //
+(p_scr) br.cond.dpnt.many .copy_bytes
+;; }
+{ .mii
+ load tempreg = [src], 8
+ add loopcnt = -1, elemcnt //
+;; }
+{ .mii
+ cmp.ne p_scr, p0 = 0, loopcnt //
+ mov ar.lc = loopcnt //
+;; }
+
+.l2: // ------------------------------- // L2: Max 4 words copied separately
+{ .mmi
+ store [dest] = tempreg, 8
+(p_scr) load tempreg = [src], 8 //
+ add len = -8, len
+} { .mib
+ cmp.lt p_scr, p0 = 1, loopcnt // avoid load beyond end-point
+ add loopcnt = -1, loopcnt
+ br.cloop.dptk.few .l2
+;; }
+
+.copy_bytes:
+{ .mib
+ cmp.eq p_scr, p0 = len, r0 // is len == 0 ?
+ add loopcnt = -1, len // len--;
+(p_scr) br.cond.spnt .restore_and_exit
+;; }
+{ .mii
+ ld1 tmp2 = [src], 1
+ movi0 ar.lc = loopcnt
+ cmp.ne p_scr, p0 = 0, loopcnt // avoid load beyond end-point
+;; }
+
+.l3: // ------------------------------- // L3: Final byte move
+{ .mmi
+ st1 [dest] = tmp2, 1
+(p_scr) ld1 tmp2 = [src], 1
+} { .mib
+ cmp.lt p_scr, p0 = 1, loopcnt // avoid load beyond end-point
+ add loopcnt = -1, loopcnt
+ br.cloop.dptk.few .l3
+;; }
- mov pr = saved_pr, -1 // restore the predicate registers
- mov ar.lc = saved_lc // restore the loop counter
+.restore_and_exit:
+{ .mmi
+ movi0 pr = saved_pr, -1 // restore the predicate registers
+;; }
+{ .mib
+ movi0 ar.lc = saved_lc // restore the loop counter
br.ret.sptk.many b0
-.next:
- cmp.ge p6, p0 = OP_T_THRES, len // is len <= OP_T_THRES
- and loopcnt = 7, tmp2 // loopcnt = -dest % 8
-(p6) br.cond.spnt .cpyfew // copy byte by byte
- ;;
- cmp.eq p6, p0 = loopcnt, r0
-(p6) br.cond.sptk .dest_aligned
- sub len = len, loopcnt // len -= -dest % 8
- adds loopcnt = -1, loopcnt // --loopcnt
- ;;
- mov ar.lc = loopcnt
-.l1: // copy -dest % 8 bytes
- ld1 value = [src], 1 // value = *src++
- ;;
- st1 [dest] = value, 1 // *dest++ = value
- br.cloop.dptk .l1 ;;
-.dest_aligned:
+;; }
+
+
+.src_not_aligned:
+{ .mmi
+ cmp.gt p_scr, p0 = 16, len
and sh1 = 7, src // sh1 = src % 8
- and tmp2 = -8, len // tmp2 = len & -OPSIZ
- and asrc = -8, src // asrc = src & -OPSIZ -- align src
- shr.u loopcnt = len, 3 // loopcnt = len / 8
- and len = 7, len;; // len = len % 8
- adds loopcnt = -1, loopcnt // --loopcnt
- addl tmp4 = @ltoff(.table), gp
- addl tmp3 = @ltoff(.loop56), gp
- mov ar.ec = MEMLAT + 1 // set EC
- mov pr.rot = 1 << 16;; // set rotating predicates
- mov ar.lc = loopcnt // set LC
- cmp.eq p6, p0 = sh1, r0 // is the src aligned?
-(p6) br.cond.sptk .src_aligned
- add src = src, tmp2 // src += len & -OPSIZ
+ shr.u loopcnt = len, 4 // element-cnt = len / 16
+} { .mib
+ add tmp4 = @ltoff(.table), gp
+ add tmp3 = @ltoff(.loop56), gp
+(p_scr) br.cond.dpnt.many .copy_bytes // do byte by byte if too few
+;; }
+{ .mmi
+ and asrc = -8, src // asrc = (-8) -- align src for loop
+ add loopcnt = -1, loopcnt // loopcnt--
shl sh1 = sh1, 3 // sh1 = 8 * (src % 8)
+} { .mmi
+ ld8 ptable = [tmp4] // ptable = &table
ld8 ploop56 = [tmp3] // ploop56 = &loop56
- ld8 ptable = [tmp4];; // ptable = &table
- add tmp3 = ptable, sh1;; // tmp3 = &table + sh1
- mov ar.ec = MEMLAT + 1 + 1 // one more pass needed
- ld8 tmp4 = [tmp3];; // tmp4 = loop offset
+ and tmp2 = -16, len // tmp2 = len & -OPSIZ
+;; }
+{ .mmi
+ add tmp3 = ptable, sh1 // tmp3 = &table + sh1
+ add src = src, tmp2 // src += len & (-16)
+ movi0 ar.lc = loopcnt // set LC
+;; }
+{ .mmi
+ ld8 tmp4 = [tmp3] // tmp4 = loop offset
+ sub len = len, tmp2 // len -= len & (-16)
+ movi0 ar.ec = MEMLAT + 2 // one more pass needed
+;; }
+{ .mmi
+ ld8 s[1] = [asrc], 8 // preload
sub loopaddr = ploop56,tmp4 // loopadd = &loop56 - loop offset
- ld8 r[1] = [asrc], 8;; // w0
- mov b6 = loopaddr;;
+ movi0 pr.rot = 1 << 16 // set rotating predicates
+;; }
+{ .mib
+ nop.m 0
+ movi0 b6 = loopaddr
br b6 // jump to the appropriate loop
+;; }
LOOP(8)
LOOP(16)
@@ -169,26 +408,9 @@
LOOP(40)
LOOP(48)
LOOP(56)
-
-.src_aligned:
-.l3:
-(p[0]) ld8 r[0] = [src], 8
-(p[MEMLAT]) st8 [dest] = r[MEMLAT], 8
- br.ctop.dptk .l3 ;;
-.cpyfew:
- cmp.eq p6, p0 = len, r0 // is len == 0 ?
- adds len = -1, len // --len;
-(p6) br.cond.spnt .restore_and_exit ;;
- mov ar.lc = len
-.l4:
- ld1 value = [src], 1
- ;;
- st1 [dest] = value, 1
- br.cloop.dptk .l4 ;;
-.restore_and_exit:
- mov pr = saved_pr, -1 // restore the predicate registers
- mov ar.lc = saved_lc // restore the loop counter
- br.ret.sptk.many b0
+END(memcpy)
+
+ .rodata
.align 8
.table:
data8 0 // dummy entry
@@ -199,5 +421,3 @@
data8 .loop56 - .loop40
data8 .loop56 - .loop48
data8 .loop56 - .loop56
-
-END(memcpy)
Index: sysdeps/ia64/memset.S
===================================================================
RCS file: /cvs/glibc/libc/sysdeps/ia64/memset.S,v
retrieving revision 1.4
diff -u -r1.4 memset.S
--- sysdeps/ia64/sysdeps/ia64/memset.S 6 Jul 2001 04:55:54 -0000 1.4
+++ sysdeps/ia64/sysdeps/ia64/memset.S 3 Oct 2002 06:00:19 -0000
@@ -1,7 +1,8 @@
/* Optimized version of the standard memset() function.
This file is part of the GNU C Library.
- Copyright (C) 2000, 2001 Free Software Foundation, Inc.
- Contributed by Dan Pop <Dan.Pop@cern.ch>.
+ Copyright (C) 2000, 2001, 2002 Free Software Foundation, Inc.
+ Contributed by Dan Pop for Itanium <Dan.Pop@cern.ch>.
+ Rewritten for McKinley by Sverre Jarp, HP Labs/CERN <Sverre.Jarp@cern.ch>
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
@@ -19,80 +20,373 @@
02111-1307 USA. */
/* Return: dest
-
+
Inputs:
in0: dest
in1: value
in2: count
The algorithm is fairly straightforward: set byte by byte until we
- we get to a word aligned address, then set word by word as much as
- possible; the remaining few bytes are set one by one. */
+ we get to a 16B-aligned address, then loop on 128 B chunks using an
+ early store as prefetching, then loop on 32B chucks, then clear remaining
+ words, finally clear remaining bytes.
+ Since a stf.spill f0 can store 16B in one go, we use this instruction
+ to get peak speed when value = 0. */
#include <sysdep.h>
#undef ret
#define dest in0
-#define byteval in1
+#define value in1
#define cnt in2
-#define save_pfs loc0
-#define ptr1 loc1
-#define ptr2 loc2
-#define tmp loc3
-#define loopcnt loc4
-#define save_lc loc5
-#define wordval loc6
+#define tmp r31
+#define save_lc r30
+#define ptr0 r29
+#define ptr1 r28
+#define ptr2 r27
+#define ptr3 r26
+#define ptr9 r24
+#define loopcnt r23
+#define linecnt r22
+#define bytecnt r21
+
+#define fvalue f6
+
+// This routine uses only scratch predicate registers (p6 - p15)
+#define p_scr p6 // default register for same-cycle branches
+#define p_nz p7
+#define p_zr p8
+#define p_unalgn p9
+#define p_y p11
+#define p_n p12
+#define p_yy p13
+#define p_nn p14
+
+#define movi0 mov
+#define MIN1 15
+#define MIN1P1HALF 8
+#define LINE_SIZE 128
+#define LSIZE_SH 7 // shift amount
+#define PREF_AHEAD 8
+
+#define USE_FLP
+#if defined(USE_INT)
+#define store st8
+#define myval value
+#elif defined(USE_FLP)
+#define store stf8
+#define myval fvalue
+#endif
+
+.align 64
ENTRY(memset)
+{ .mmi
.prologue
- alloc save_pfs = ar.pfs, 3, 7, 0, 0
- .save ar.lc, save_lc
- mov save_lc = ar.lc
+ alloc tmp = ar.pfs, 3, 0, 0, 0
+ lfetch.nt1 [dest]
+ .save ar.lc, save_lc
+ movi0 save_lc = ar.lc
+} { .mmi
.body
- mov ret0 = dest
- and tmp = 7, dest
- cmp.eq p6, p0 = cnt, r0
-(p6) br.cond.spnt .restore_and_exit ;;
+ mov ret0 = dest // return value
+ cmp.ne p_nz, p_zr = value, r0 // use stf.spill if value is zero
+ cmp.eq p_scr, p0 = cnt, r0
+;; }
+{ .mmi
+ and ptr2 = -(MIN1+1), dest // aligned address
+ and tmp = MIN1, dest // prepare to check for alignment
+ tbit.nz p_y, p_n = dest, 0 // Do we have an odd address? (M_B_U)
+} { .mib
mov ptr1 = dest
- sub loopcnt = 8, tmp
- cmp.gt p6, p0 = 16, cnt
-(p6) br.cond.spnt .set_few;;
- cmp.eq p6, p0 = tmp, r0
-(p6) br.cond.sptk .dest_aligned
- sub cnt = cnt, loopcnt
- adds loopcnt = -1, loopcnt;;
- mov ar.lc = loopcnt;;
-.l1:
- st1 [ptr1] = byteval, 1
- br.cloop.dptk .l1 ;;
-.dest_aligned:
- adds ptr2 = 8, ptr1
- mux1 wordval = byteval, @brcst
- shr.u loopcnt = cnt, 4 ;; // loopcnt = cnt / 16
- cmp.eq p6, p0 = loopcnt, r0
-(p6) br.cond.spnt .one_more
- and cnt = 0xf, cnt // compute the remaining cnt
- adds loopcnt = -1, loopcnt;;
- mov ar.lc = loopcnt;;
-.l2:
- st8 [ptr1] = wordval, 16
- st8 [ptr2] = wordval, 16
- br.cloop.dptk .l2
- cmp.le p6, p0 = 8, cnt ;;
-.one_more:
-(p6) st8 [ptr1] = wordval, 8
-(p6) adds cnt = -8, cnt ;;
- cmp.eq p6, p0 = cnt, r0
-(p6) br.cond.spnt .restore_and_exit
-.set_few:
- adds loopcnt = -1, cnt;;
- mov ar.lc = loopcnt;;
-.l3:
- st1 [ptr1] = byteval, 1
- br.cloop.dptk .l3 ;;
+ mux1 value = value, @brcst // create 8 identical bytes in word
+(p_scr) br.ret.dpnt.many rp // return immediately if count = 0
+;; }
+{ .mib
+ cmp.ne p_unalgn, p0 = tmp, r0
+} { .mib // NB: # of bytes to move is 1 higher
+ sub bytecnt = (MIN1+1), tmp // than loopcnt
+ cmp.gt p_scr, p0 = 16, cnt // is it a minimalistic task?
+(p_scr) br.cond.dptk.many .move_bytes_unaligned // go move just a few (M_B_U)
+;; }
+{ .mmi
+(p_unalgn) add ptr1 = (MIN1+1), ptr2 // after alignment
+(p_unalgn) add ptr2 = MIN1P1HALF, ptr2 // after alignment
+(p_unalgn) tbit.nz.unc p_y, p_n = bytecnt, 3 // should we do a st8 ?
+;; }
+{ .mib
+(p_y) add cnt = -8, cnt
+(p_unalgn) tbit.nz.unc p_yy, p_nn = bytecnt, 2 // should we do a st4 ?
+} { .mib
+(p_y) st8 [ptr2] = value, -4
+(p_n) add ptr2 = 4, ptr2
+;; }
+{ .mib
+(p_yy) add cnt = -4, cnt
+(p_unalgn) tbit.nz.unc p_y, p_n = bytecnt, 1 // should we do a st2 ?
+} { .mib
+(p_yy) st4 [ptr2] = value, -2
+(p_nn) add ptr2 = 2, ptr2
+;; }
+{ .mmi
+ mov tmp = LINE_SIZE+1 // for compare
+(p_y) add cnt = -2, cnt
+(p_unalgn) tbit.nz.unc p_yy, p_nn = bytecnt, 0 // should we do a st1 ?
+} { .mmi
+ setf.sig fvalue=value // transfer value to FLP side
+(p_y) st2 [ptr2] = value, -1
+(p_n) add ptr2 = 1, ptr2
+;; }
+
+{ .mmi
+(p_yy) st1 [ptr2] = value
+ cmp.gt p_scr, p0 = tmp, cnt // is it a minimalistic task?
+} { .mbb
+(p_yy) add cnt = -1, cnt
+(p_scr) br.cond.dpnt.many .fraction_of_line // go move just a few
+;; }
+
+{ .mib
+ nop.m 0
+ shr.u linecnt = cnt, LSIZE_SH
+(p_zr) br.cond.dptk.many .l1b // Jump to use stf.spill
+;; }
+
+ .align 32 // -------- // L1A: store ahead into cache lines; fill later
+{ .mmi
+ and tmp = -(LINE_SIZE), cnt // compute end of range
+ mov ptr9 = ptr1 // used for prefetching
+ and cnt = (LINE_SIZE-1), cnt // remainder
+} { .mmi
+ mov loopcnt = PREF_AHEAD-1 // default prefetch loop
+ cmp.gt p_scr, p0 = PREF_AHEAD, linecnt // check against actual value
+;; }
+{ .mmi
+(p_scr) add loopcnt = -1, linecnt // start of stores
+ add ptr2 = 8, ptr1 // (beyond prefetch stores)
+ add ptr1 = tmp, ptr1 // first address beyond total
+;; } // range
+{ .mmi
+ add tmp = -1, linecnt // next loop count
+ movi0 ar.lc = loopcnt
+;; }
+.pref_l1a:
+{ .mib
+ store [ptr9] = myval, 128 // Do stores one cache line apart
+ nop.i 0
+ br.cloop.dptk.few .pref_l1a
+;; }
+{ .mmi
+ add ptr0 = 16, ptr2 // Two stores in parallel
+ movi0 ar.lc = tmp
+;; }
+.l1ax:
+ { .mmi
+ store [ptr2] = myval, 8
+ store [ptr0] = myval, 8
+ ;; }
+ { .mmi
+ store [ptr2] = myval, 24
+ store [ptr0] = myval, 24
+ ;; }
+ { .mmi
+ store [ptr2] = myval, 8
+ store [ptr0] = myval, 8
+ ;; }
+ { .mmi
+ store [ptr2] = myval, 24
+ store [ptr0] = myval, 24
+ ;; }
+ { .mmi
+ store [ptr2] = myval, 8
+ store [ptr0] = myval, 8
+ ;; }
+ { .mmi
+ store [ptr2] = myval, 24
+ store [ptr0] = myval, 24
+ ;; }
+ { .mmi
+ store [ptr2] = myval, 8
+ store [ptr0] = myval, 32
+ cmp.lt p_scr, p0 = ptr9, ptr1 // do we need more prefetching?
+ ;; }
+{ .mmb
+ store [ptr2] = myval, 24
+(p_scr) store [ptr9] = myval, 128
+ br.cloop.dptk.few .l1ax
+;; }
+{ .mbb
+ cmp.le p_scr, p0 = 8, cnt // just a few bytes left ?
+(p_scr) br.cond.dpnt.many .fraction_of_line // Branch no. 2
+ br.cond.dpnt.many .move_bytes_from_alignment // Branch no. 3
+;; }
+
+ .align 32
+.l1b: // ------------------ // L1B: store ahead into cache lines; fill later
+{ .mmi
+ and tmp = -(LINE_SIZE), cnt // compute end of range
+ mov ptr9 = ptr1 // used for prefetching
+ and cnt = (LINE_SIZE-1), cnt // remainder
+} { .mmi
+ mov loopcnt = PREF_AHEAD-1 // default prefetch loop
+ cmp.gt p_scr, p0 = PREF_AHEAD, linecnt // check against actual value
+;; }
+{ .mmi
+(p_scr) add loopcnt = -1, linecnt
+ add ptr2 = 16, ptr1 // start of stores (beyond prefetch stores)
+ add ptr1 = tmp, ptr1 // first address beyond total range
+;; }
+{ .mmi
+ add tmp = -1, linecnt // next loop count
+ movi0 ar.lc = loopcnt
+;; }
+.pref_l1b:
+{ .mib
+ stf.spill [ptr9] = f0, 128 // Do stores one cache line apart
+ nop.i 0
+ br.cloop.dptk.few .pref_l1b
+;; }
+{ .mmi
+ add ptr0 = 16, ptr2 // Two stores in parallel
+ movi0 ar.lc = tmp
+;; }
+.l1bx:
+ { .mmi
+ stf.spill [ptr2] = f0, 32
+ stf.spill [ptr0] = f0, 32
+ ;; }
+ { .mmi
+ stf.spill [ptr2] = f0, 32
+ stf.spill [ptr0] = f0, 32
+ ;; }
+ { .mmi
+ stf.spill [ptr2] = f0, 32
+ stf.spill [ptr0] = f0, 64
+ cmp.lt p_scr, p0 = ptr9, ptr1 // do we need more prefetching?
+ ;; }
+{ .mmb
+ stf.spill [ptr2] = f0, 32
+(p_scr) stf.spill [ptr9] = f0, 128
+ br.cloop.dptk.few .l1bx
+;; }
+{ .mib
+ cmp.gt p_scr, p0 = 8, cnt // just a few bytes left ?
+(p_scr) br.cond.dpnt.many .move_bytes_from_alignment
+;; }
+
+.fraction_of_line:
+{ .mib
+ add ptr2 = 16, ptr1
+ shr.u loopcnt = cnt, 5 // loopcnt = cnt / 32
+;; }
+{ .mib
+ cmp.eq p_scr, p0 = loopcnt, r0
+ add loopcnt = -1, loopcnt
+(p_scr) br.cond.dpnt.many .store_words
+;; }
+{ .mib
+ and cnt = 0x1f, cnt // compute the remaining cnt
+ movi0 ar.lc = loopcnt
+;; }
+ .align 32
+.l2: // ---------------------------- // L2A: store 32B in 2 cycles
+{ .mmb
+ store [ptr1] = myval, 8
+ store [ptr2] = myval, 8
+;; } { .mmb
+ store [ptr1] = myval, 24
+ store [ptr2] = myval, 24
+ br.cloop.dptk.many .l2
+;; }
+.store_words:
+{ .mib
+ cmp.gt p_scr, p0 = 8, cnt // just a few bytes left ?
+(p_scr) br.cond.dpnt.many .move_bytes_from_alignment // Branch
+;; }
+
+{ .mmi
+ store [ptr1] = myval, 8 // store
+ cmp.le p_y, p_n = 16, cnt //
+ add cnt = -8, cnt // subtract
+;; }
+{ .mmi
+(p_y) store [ptr1] = myval, 8 // store
+(p_y) cmp.le.unc p_yy, p_nn = 16, cnt //
+(p_y) add cnt = -8, cnt // subtract
+;; }
+{ .mmi // store
+(p_yy) store [ptr1] = myval, 8 //
+(p_yy) add cnt = -8, cnt // subtract
+;; }
+
+.move_bytes_from_alignment:
+{ .mib
+ cmp.eq p_scr, p0 = cnt, r0
+ tbit.nz.unc p_y, p0 = cnt, 2 // should we terminate with a st4 ?
+(p_scr) br.cond.dpnt.few .restore_and_exit
+;; }
+{ .mib
+(p_y) st4 [ptr1] = value, 4
+ tbit.nz.unc p_yy, p0 = cnt, 1 // should we terminate with a st2 ?
+;; }
+{ .mib
+(p_yy) st2 [ptr1] = value, 2
+ tbit.nz.unc p_y, p0 = cnt, 0
+;; }
+
+{ .mib
+(p_y) st1 [ptr1] = value
+;; }
.restore_and_exit:
- mov ar.lc = save_lc
- mov ar.pfs = save_pfs
- br.ret.sptk.many b0
+{ .mib
+ nop.m 0
+ movi0 ar.lc = save_lc
+ br.ret.sptk.many rp
+;; }
+
+.move_bytes_unaligned:
+{ .mmi
+ .pred.rel "mutex",p_y, p_n
+ .pred.rel "mutex",p_yy, p_nn
+(p_n) cmp.le p_yy, p_nn = 4, cnt
+(p_y) cmp.le p_yy, p_nn = 5, cnt
+(p_n) add ptr2 = 2, ptr1
+} { .mmi
+(p_y) add ptr2 = 3, ptr1
+(p_y) st1 [ptr1] = value, 1 // fill 1 (odd-aligned) byte
+(p_y) add cnt = -1, cnt // [15, 14 (or less) left]
+;; }
+{ .mmi
+(p_yy) cmp.le.unc p_y, p0 = 8, cnt
+ add ptr3 = ptr1, cnt // prepare last store
+ movi0 ar.lc = save_lc
+} { .mmi
+(p_yy) st2 [ptr1] = value, 4 // fill 2 (aligned) bytes
+(p_yy) st2 [ptr2] = value, 4 // fill 2 (aligned) bytes
+(p_yy) add cnt = -4, cnt // [11, 10 (o less) left]
+;; }
+{ .mmi
+(p_y) cmp.le.unc p_yy, p0 = 8, cnt
+ add ptr3 = -1, ptr3 // last store
+ tbit.nz p_scr, p0 = cnt, 1 // will there be a st2 at the end ?
+} { .mmi
+(p_y) st2 [ptr1] = value, 4 // fill 2 (aligned) bytes
+(p_y) st2 [ptr2] = value, 4 // fill 2 (aligned) bytes
+(p_y) add cnt = -4, cnt // [7, 6 (or less) left]
+;; }
+{ .mmi
+(p_yy) st2 [ptr1] = value, 4 // fill 2 (aligned) bytes
+(p_yy) st2 [ptr2] = value, 4 // fill 2 (aligned) bytes
+ // [3, 2 (or less) left]
+ tbit.nz p_y, p0 = cnt, 0 // will there be a st1 at the end ?
+} { .mmi
+(p_yy) add cnt = -4, cnt
+;; }
+{ .mmb
+(p_scr) st2 [ptr1] = value // fill 2 (aligned) bytes
+(p_y) st1 [ptr3] = value // fill last byte (using ptr3)
+ br.ret.sptk.many rp
+;; }
END(memset)
+
| Index Nav: | [Date Index] [Subject Index] [Author Index] [Thread Index] | |
|---|---|---|
| Message Nav: | [Date Prev] [Date Next] | [Thread Prev] [Thread Next] |