[PATCH 10/20] alpha: add EV6/EV7 assembly memmove
Matt Turner
mattst88@gmail.com
Wed Aug 12 01:19:47 GMT 2026
Alpha used the generic C memmove. Add an assembly one built out of the ev6
memcpy: when the destination does not overlap the source from below -- that
is, (uintptr) dst - (uintptr) src >= n, which also covers dst < src -- the
copy runs forward and is exactly memcpy, with the same-alignment quadword
fast path, the wh64 unrolled loop on the 21264 and the prefetch loop on the
21364, and the rotating ldq_u path for mismatched alignment. Only the
genuinely overlapping case (src < dst < src+n) copies backward, and that is
rare and not performance critical, so it uses a straightforward descending
aligned-quadword-or-byte loop.
Two details of the forward path are specific to memmove and are not
inherited from memcpy, because memcpy never sees an overlapping copy.
First, wh64 claims a cache line for writing without filling it from memory,
which makes the whole aligned 64-byte block UNPREDICTABLE unless every byte
of it is stored. In a forward overlapping copy the hint runs ahead of the
stores, so it can land on source bytes that have not been read yet and
destroy them before the copy consumes them. At trip k the hint covers
[dst_k+128, dst_k+192) and the unread source begins at dst_k + (src-dst),
so the copy is safe exactly when src-dst >= 192. Below that the copy takes
the prefetch loop instead, which substitutes LDS to F31 for the wh64 -- a
prefetch with modify intent that only reads, and so cannot destroy the
source -- and is correct at any overlap because it reads a whole block into
registers before storing any of it. Note that bound tracks the wh64
distance: it is the hint's far edge, so it has to move whenever that moves.
Second, the guard sends a too-small gap to the 64-byte prefetch loop rather
than to the scalar tail loop, which would cost roughly 8x on exactly the
copies it catches. A Python list.remove() moves its ob_item array with a
gap of a single pointer, so it always takes this path.
Verified on an EV68CB against the C memmove over every size class from 0 to
64KB, at 72 destination alignments for the non-overlapping case and across
source-destination gaps from 8 bytes to 1KB for the overlapping one, with
512-byte guard regions checked for stray writes.
---
sysdeps/alpha/alphaev6/memmove.S | 384 +++++++++++++++++++++++++++++++
1 file changed, 384 insertions(+)
create mode 100644 sysdeps/alpha/alphaev6/memmove.S
diff --git ./sysdeps/alpha/alphaev6/memmove.S ./sysdeps/alpha/alphaev6/memmove.S
new file mode 100644
index 0000000000..7c0cca1391
--- /dev/null
+++ ./sysdeps/alpha/alphaev6/memmove.S
@@ -0,0 +1,384 @@
+/* Copyright (C) 2026 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library. If not, see
+ <https://www.gnu.org/licenses/>. */
+
+/*
+ * EV6/EV7 memmove. Alpha previously used the generic C memmove.
+ *
+ * When the destination does not overlap the source from below -- i.e.
+ * (uintptr)dst - (uintptr)src >= n, which also covers dst < src -- the copy
+ * runs forward and is exactly the ev6 memcpy (same-alignment quadword fast
+ * path with the EV7 prefetch unrolled loop, and the rotating ldq_u path for
+ * mismatched alignment). Only the genuinely overlapping case (src < dst <
+ * src+n) copies backward; that is rare and not performance critical, so it
+ * uses a straightforward aligned-quadword-or-byte descending loop.
+ *
+ * Temp usage mirrors memcpy ($0 result, $1..$7,$22 scratch); the backward
+ * path additionally keeps the running pointers in $16/$17 and uses $1,$2.
+ */
+
+#include <sysdep.h>
+
+ .arch ev6
+ .set noreorder
+ .set noat
+
+ENTRY(memmove)
+ .prologue 0
+
+ mov $16, $0 # E : return value = dest
+ ble $18, $nomoredata # U : nothing to do
+ subq $16, $17, $1 # E : dst - src
+ cmpult $1, $18, $2 # E : (dst-src) < n -> overlap from below
+
+ bne $2, $backward # U : overlapping -> copy backward
+ xor $16, $17, $1 # E : forward: same alignment?
+ and $1, 7, $1 # E : ... mod 8
+ bne $1, $misaligned # U : no -> rotating path
+
+ and $16, 7, $1 # E : both 0mod8?
+ beq $1, $both_0mod8 # U : yes
+ nop # E :
+
+ /* Same misalignment: copy bytes until both reach a 0mod8 boundary. */
+$head_align:
+ ldbu $1, 0($17) # L : grab a byte
+ subq $18, 1, $18 # E : count--
+ addq $17, 1, $17 # E : src++
+ stb $1, 0($16) # L :
+ addq $16, 1, $16 # E : dest++
+ and $16, 7, $1 # E : 0mod8 yet?
+ ble $18, $nomoredata # U : done?
+ bne $1, $head_align # U :
+
+$both_0mod8:
+ cmple $18, 127, $1 # E : Can we unroll the loop?
+ bne $1, $no_unroll # U :
+ and $16, 63, $1 # E : get mod64 alignment
+ beq $1, $do_unroll # U : no single quads to fiddle
+
+$single_head_quad:
+ ldq $1, 0($17) # L : get 8 bytes
+ subq $18, 8, $18 # E : count -= 8
+ addq $17, 8, $17 # E : src += 8
+ nop # E :
+
+ stq $1, 0($16) # L : store
+ addq $16, 8, $16 # E : dest += 8
+ and $16, 63, $1 # E : get mod64 alignment
+ bne $1, $single_head_quad # U : still not fully aligned
+
+$do_unroll:
+ /*
+ * Same IMPLVER dispatch as memcpy: the 21264 (EV6/67/68) uses wh64 two
+ * lines ahead; the 21364 (EV7) takes a loop that software-prefetches both
+ * streams (read on the source, modify intent on the destination) six
+ * cache lines ahead. IMPLVER returns 2 for the 21264 family, 3 for EV7.
+ */
+ cmple $18, 127, $1 # E : Can we go through the unrolled loop?
+ bne $1, $tail_quads # U : Nope
+ implver $2 # E : 2 = 21264, 3 = 21364
+ subq $2, 3, $2 # E : EV7 -> 0
+
+ beq $2, $ev7_unroll # U : EV7 takes the prefetch-modify path
+
+ /*
+ * wh64 claims a cache line for writing without filling it from memory,
+ * which makes the entire 64-byte block UNPREDICTABLE unless every byte
+ * of it is stored. In a forward overlapping copy (dst < src) the hint
+ * runs ahead of the stores and can therefore land on source bytes that
+ * have not been read yet, destroying them before the copy consumes
+ * them.
+ *
+ * At trip k the hint covers [dst_k+128, dst_k+192) and the unread
+ * source begins at src_k = dst_k + (src-dst), so the copy is safe
+ * exactly when src-dst >= 192. Note this bound tracks the wh64
+ * distance: it is the hint's far edge, so it has to be updated whenever
+ * that distance changes. When dst >= src+n (no overlap at all) src-dst
+ * wraps to a huge unsigned value, cmpult is false, and the wh64 path is
+ * used as normal.
+ *
+ * Too close a gap falls through to $ev7_unroll rather than the scalar
+ * $tail_quads loop. That path copies 64 bytes a trip like this one and
+ * merely swaps wh64 for LDS to F31, a prefetch-with-modify-intent that
+ * only reads and so cannot destroy the source; ReadBlkMod is
+ * implemented on the 21264 (it is the evict-next variants that are
+ * 21364-only). It reads a whole block into registers before storing
+ * any of it, so it is correct at any overlap. Landing on the scalar
+ * loop instead would cost roughly 8x on exactly the small-gap copies
+ * this guard catches -- a Python list.remove() moves its ob_item array
+ * with a gap of one pointer, so it always takes this path.
+ */
+ subq $17, $16, $1 # E : src - dst
+ cmpult $1, 192, $2 # E : (src-dst) < 192 ?
+ bne $2, $ev7_unroll # U : too close -- use the non-wh64 loop
+
+ /*
+ * Initial wh64 target, two lines ahead of the first block. Only 128
+ * bytes are known to remain here, so unlike the steady-state hint this
+ * one can overrun; fall back to the first block when it would.
+ */
+ addq $16, 128, $7 # E : initial (+2 line) wh64 address
+ subq $18, 192, $2 # E : room for [$16+128, $16+192)?
+ cmovlt $2, $16, $7 # E : no -- hint the first block instead
+
+$unroll_body:
+ wh64 ($7) # L1 : 64 bytes at ($7) about to be overwritten
+ ldq $6, 0($17) # L0 : bytes 0..7
+ nop # E :
+ nop # E :
+
+ ldq $4, 8($17) # L : bytes 8..15
+ ldq $5, 16($17) # L : bytes 16..23
+ addq $7, 64, $7 # E : Update next wh64 address
+ nop # E :
+
+ ldq $3, 24($17) # L : bytes 24..31
+ addq $16, 64, $23 # E : fallback wh64 address (see below)
+ nop # E :
+ nop # E :
+
+ addq $17, 32, $17 # E : src += 32 bytes
+ stq $6, 0($16) # L : bytes 0..7
+ nop # E :
+ nop # E :
+
+ stq $4, 8($16) # L : bytes 8..15
+ stq $5, 16($16) # L : bytes 16..23
+ lda $2, -256($18) # E : room for the +2 line hint next trip?
+ # it covers [$16+128, $16+192) there,
+ # so that trip needs 192 and this one
+ # needs 256
+ nop # E :
+
+ stq $3, 24($16) # L : bytes 24..31
+ addq $16, 32, $16 # E : dest += 32 bytes
+ nop # E :
+ nop # E :
+
+ ldq $6, 0($17) # L : bytes 0..7
+ ldq $4, 8($17) # L : bytes 8..15
+ nop # E :
+ nop # E :
+
+ ldq $5, 16($17) # L : bytes 16..23
+ ldq $3, 24($17) # L : bytes 24..31
+ addq $16, 32, $16 # E : dest += 32
+ subq $18, 64, $18 # E : count -= 64
+
+ addq $17, 32, $17 # E : src += 32
+ stq $6, -32($16) # L : bytes 0..7
+ stq $4, -24($16) # L : bytes 8..15
+ cmple $18, 63, $1 # E : At least one more trip?
+
+ /*
+ * $7 is deliberately finalized late, in the group with the loop branch
+ * rather than alongside the loads above, and the fallback address lives
+ * in $23 so that it survives the cmple that reuses $1 for the loop
+ * condition. This is not cosmetic: $7 is what the next trip's wh64
+ * issues against, and letting that hint issue while the previous trip's
+ * stores are still in flight provokes Mbox replay traps in the store
+ * stream. Measured on an EV68CB over a cold 256KB copy, moving the
+ * cmov down cuts replays (perf r4) from about 1000 per call to about
+ * 750 and is worth 1.4-1.8% from 2KB upwards, for no extra instructions.
+ * Moving it back up, or reusing $1 so that it has to move back up,
+ * gives up that gain.
+ */
+ stq $5, -16($16) # L : bytes 16..23
+ stq $3, -8($16) # L : bytes 24..31
+ cmovlt $2, $23, $7 # E : fallback if < 2 more trips (late; see
+ # the note above -- do not hoist this)
+ beq $1, $unroll_body
+ br $31, $tail_quads # U : 1..15 trailing quads/bytes
+
+ .align 4
+$ev7_unroll:
+ lds $f31, 6*64($16) # L : dest prefetch w/ modify intent
+ ldl $31, 6*64($17) # L : src prefetch (read)
+ ldq $1, 0($17) # L : bytes 0..7
+ ldq $2, 8($17) # L : bytes 8..15
+ ldq $3, 16($17) # L : bytes 16..23
+ ldq $4, 24($17) # L : bytes 24..31
+ ldq $5, 32($17) # L : bytes 32..39
+ ldq $6, 40($17) # L : bytes 40..47
+ ldq $7, 48($17) # L : bytes 48..55
+ ldq $22, 56($17) # L : bytes 56..63
+
+ stq $1, 0($16) # L :
+ stq $2, 8($16) # L :
+ stq $3, 16($16) # L :
+ stq $4, 24($16) # L :
+ stq $5, 32($16) # L :
+ stq $6, 40($16) # L :
+ stq $7, 48($16) # L :
+ stq $22, 56($16) # L :
+
+ subq $18, 64, $18 # E : count -= 64
+ addq $17, 64, $17 # E : src += 64
+ addq $16, 64, $16 # E : dest += 64
+ cmple $18, 63, $1 # E : less than one more full trip?
+ beq $1, $ev7_unroll # U : keep going
+
+$tail_quads:
+$no_unroll:
+ .align 4
+ subq $18, 8, $18 # E : At least a quad left?
+ blt $18, $less_than_8 # U : Nope
+ nop # E :
+ nop # E :
+
+$move_a_quad:
+ ldq $1, 0($17) # L : fetch 8
+ subq $18, 8, $18 # E : count -= 8
+ addq $17, 8, $17 # E : src += 8
+ nop # E :
+
+ stq $1, 0($16) # L : store 8
+ addq $16, 8, $16 # E : dest += 8
+ bge $18, $move_a_quad # U :
+ nop # E :
+
+$less_than_8:
+ .align 4
+ addq $18, 8, $18 # E : add back for trailing bytes
+ ble $18, $nomoredata # U : All-done
+ nop # E :
+ nop # E :
+
+$tail_bytes:
+ subq $18, 1, $18 # E : count--
+ ldbu $1, 0($17) # L : fetch a byte
+ addq $17, 1, $17 # E : src++
+ nop # E :
+
+ stb $1, 0($16) # L : store a byte
+ addq $16, 1, $16 # E : dest++
+ bgt $18, $tail_bytes # U : more to be done?
+ nop # E :
+
+ ret $31, ($26), 1 # L0 :
+ nop # E :
+ nop # E :
+ nop # E :
+
+$misaligned:
+ mov $0, $4 # E : dest temp
+ and $0, 7, $1 # E : dest alignment mod8
+ beq $1, $dest_0mod8 # U : life doesn't totally suck
+ nop
+
+$aligndest:
+ ble $18, $nomoredata # U :
+ ldbu $1, 0($17) # L : fetch a byte
+ subq $18, 1, $18 # E : count--
+ addq $17, 1, $17 # E : src++
+
+ stb $1, 0($4) # L : store it
+ addq $4, 1, $4 # E : dest++
+ and $4, 7, $1 # E : dest 0mod8 yet?
+ bne $1, $aligndest # U : go until we are aligned.
+
+ /* Source has unknown alignment, but dest is known to be 0mod8 */
+$dest_0mod8:
+ subq $18, 8, $18 # E : At least a quad left?
+ blt $18, $misalign_tail # U : Nope
+ ldq_u $3, 0($17) # L : seed (rotating load) of 8 bytes
+ nop # E :
+
+$mis_quad:
+ ldq_u $16, 8($17) # L : Fetch next 8
+ extql $3, $17, $3 # U : masking
+ extqh $16, $17, $1 # U : masking
+ bis $3, $1, $1 # E : merged bytes to store
+
+ subq $18, 8, $18 # E : count -= 8
+ addq $17, 8, $17 # E : src += 8
+ stq $1, 0($4) # L : store 8 (aligned)
+ mov $16, $3 # E : "rotate" source data
+
+ addq $4, 8, $4 # E : dest += 8
+ bge $18, $mis_quad # U : More quads to move
+ nop
+ nop
+
+$misalign_tail:
+ addq $18, 8, $18 # E : account for tail stuff
+ ble $18, $nomoredata # U :
+ nop
+ nop
+
+$misalign_byte:
+ ldbu $1, 0($17) # L : fetch 1
+ subq $18, 1, $18 # E : count--
+ addq $17, 1, $17 # E : src++
+ nop # E :
+
+ stb $1, 0($4) # L : store
+ addq $4, 1, $4 # E : dest++
+ bgt $18, $misalign_byte # U : more to go?
+ nop
+
+$nomoredata:
+ ret $31, ($26), 1 # L0 :
+
+ /*
+ * Overlapping move (src < dst < src+n): copy from the high end down.
+ * Aligned quadwords when the two ends share alignment, else bytes.
+ * $0 already holds the original dest for the return value.
+ */
+$backward:
+ addq $16, $18, $16 # E : dest end (one past last byte)
+ addq $17, $18, $17 # E : src end
+ xor $16, $17, $1 # E : ends share alignment?
+ and $1, 7, $1 # E :
+ bne $1, $back_bytes # U : no -> byte copy
+
+$back_head:
+ and $16, 7, $1 # E : dest end 0mod8 yet?
+ beq $1, $back_quads # U :
+ lda $16, -1($16) # E : dest--
+ lda $17, -1($17) # E : src--
+ ldbu $1, 0($17) # L :
+ stb $1, 0($16) # L :
+ subq $18, 1, $18 # E : count--
+ beq $18, $nomoredata # U :
+ br $31, $back_head # U :
+
+$back_quads:
+ subq $18, 8, $2 # E : (count - 8)
+ blt $2, $back_bytes # U : < 8 left -> bytes
+$back_q:
+ lda $16, -8($16) # E : dest -= 8
+ lda $17, -8($17) # E : src -= 8
+ ldq $1, 0($17) # L :
+ stq $1, 0($16) # L :
+ mov $2, $18 # E : commit count -= 8
+ subq $18, 8, $2 # E : (count - 8)
+ bge $2, $back_q # U :
+
+$back_bytes:
+ beq $18, $nomoredata # U :
+$back_b:
+ lda $16, -1($16) # E :
+ lda $17, -1($17) # E :
+ ldbu $1, 0($17) # L :
+ stb $1, 0($16) # L :
+ subq $18, 1, $18 # E :
+ bne $18, $back_b # U :
+ ret $31, ($26), 1 # L0 :
+
+END(memmove)
+libc_hidden_builtin_def (memmove)
--
2.54.0
More information about the Libc-alpha
mailing list