[PATCH] aarch64: Optimize memcmp for Kunpeng 950 with SVE

[Wilco Dijkstra]

Hi Weihong,

A few comments/questions:

+ENTRY (__memcmp_kunpeng950)
+    whilelo p0.b, xzr, cnt
+    b.none  L(equal)
+    rdvl    off_vl, #1

Why use RDVL rather than CNTB? CNTB is faster for simple cases on various
SVE implementations. And if needed you can compute 2x/3x/4x using simple
shifts/adds.

+    ld1b    z0.b, p0/z, [src1]
+    ld1b    z1.b, p0/z, [src2]
+    cmpne   p1.b, p0/z, z0.b, z1.b
+    b.any   L(mismatch0)
+
+    whilelo p2.b, off_vl, cnt
+    b.none  L(equal)
+    rdvl    off_vlx2, #2
+    ld1b    z2.b, p2/z, [src1, off_vl]
+    ld1b    z3.b, p2/z, [src2, off_vl]

Is there a reason not to use [src1, 1, mul vl] here? It should be cheap.

+    cmpne   p3.b, p2/z, z2.b, z3.b
+    b.any   L(mismatch1)

Why not reuse z0.b/z1.b/p1.b and branch to mismatch0 rather than repeat the code?

+    whilelo p4.b, off_vlx2, cnt
+    b.none  L(equal)
+    rdvl    off_vlx3, #3
+    ld1b    z6.b, p4/z, [src1, off_vlx2]
+    ld1b    z7.b, p4/z, [src2, off_vlx2]
+    cmpne   p5.b, p4/z, z6.b, z7.b
+    b.any   L(mismatch2)
+
+    whilelo p6.b, off_vlx3, cnt
+    b.none  L(equal)
+    rdvl    current, #4
+    ld1b    z16.b, p6/z, [src1, off_vlx3]
+    ld1b    z17.b, p6/z, [src2, off_vlx3]
+    cmpne   p7.b, p6/z, z16.b, z17.b
+    b.any   L(mismatch3)

Is it really worth doing 4x unrolling here?

+    subs    safe_limit, cnt, current
+    csel    safe_limit, safe_limit, xzr, hs // Saturate to 0 if cnt < 4VL to prevent unsigned underflow

Rather than use CSEL, we can just do b.lo L(equal) and skip all the tail code.

+    cmp     current, safe_limit
+    b.hi    L(tail_4xvl)
+
+    ptrue   p0.b

No need for this - the first WHILELO p0.b ensures we have all true in p0.

+    .p2align 4
+L(loop_full):
+    addvl   off_vl, current, #1
+    addvl   off_vlx2, current, #2
+    addvl   off_vlx3, current, #3

That's a lot of unnecessary ADDVL - it would be better to increment src1/src2 and
use 1/2/3, mul vl as offset. Or just reuse off_vl, off_vlx2, off_vlx3.

+    ld1b    z0.b, p0/z, [src1, current]
+    ld1b    z1.b, p0/z, [src2, current]
+    cmpne   p1.b, p0/z, z0.b, z1.b
+    b.any   L(mismatch0)
+
+    ld1b    z2.b, p0/z, [src1, off_vl]
+    ld1b    z3.b, p0/z, [src2, off_vl]
+    cmpne   p3.b, p0/z, z2.b, z3.b
+    b.any   L(mismatch1_full)

It's likely better to compute 2 vectors at a time similar to current memcmp.

Also it's not clear to me why 4x unrolling is used, if we avoid the 3 ADDVL,
is it really faster than 2x unrolling?

+    ld1b    z6.b, p0/z, [src1, off_vlx2]
+    ld1b    z7.b, p0/z, [src2, off_vlx2]
+    cmpne   p5.b, p0/z, z6.b, z7.b
+    b.any   L(mismatch2_full)
+
+    ld1b    z16.b, p0/z, [src1, off_vlx3]
+    ld1b    z17.b, p0/z, [src2, off_vlx3]
+    cmpne   p7.b, p0/z, z16.b, z17.b
+    b.any   L(mismatch3_full)
+
+    addvl   current, current, #4
+    cmp     current, safe_limit
+    b.le    L(loop_full)
+
+L(tail_4xvl):
+    whilelo p0.b, current, cnt
+    b.none  L(equal)
+    addvl   off_vl, current, #1
+    ld1b    z0.b, p0/z, [src1, current]
+    ld1b    z1.b, p0/z, [src2, current]
+    cmpne   p1.b, p0/z, z0.b, z1.b
+    b.any   L(mismatch0)
+
+    whilelo p2.b, off_vl, cnt
+    b.none  L(equal)
+    addvl   off_vlx2, current, #2
+    ld1b    z2.b, p2/z, [src1, off_vl]
+    ld1b    z3.b, p2/z, [src2, off_vl]
+    cmpne   p3.b, p2/z, z2.b, z3.b
+    b.any   L(mismatch1)
+
+    whilelo p4.b, off_vlx2, cnt
+    b.none  L(equal)
+    addvl   off_vlx3, current, #3
+    ld1b    z6.b, p4/z, [src1, off_vlx2]
+    ld1b    z7.b, p4/z, [src2, off_vlx2]
+    cmpne   p5.b, p4/z, z6.b, z7.b
+    b.any   L(mismatch2)
+
+    whilelo p6.b, off_vlx3, cnt
+    b.none  L(equal)
+    ld1b    z16.b, p6/z, [src1, off_vlx3]
+    ld1b    z17.b, p6/z, [src2, off_vlx3]
+    cmpne   p7.b, p6/z, z16.b, z17.b
+    b.any   L(mismatch3)
+
+    b   L(equal)

Does 4x unrolling the tail code really help performance?

+L(mismatch1_full):
+    brkb    p2.b, p0/z, p3.b
+    lasta   w0, p2, z2.b
+    lasta   w1, p2, z3.b
+    sub     x0, x0, x1
+    ret
+
+L(mismatch2_full):
+    brkb    p2.b, p0/z, p5.b
+    lasta   w0, p2, z6.b
+    lasta   w1, p2, z7.b
+    sub     x0, x0, x1
+    ret
+
+L(mismatch3_full):
+    brkb    p2.b, p0/z, p7.b
+    lasta   w0, p2, z16.b
+    lasta   w1, p2, z17.b
+    sub     x0, x0, x1
+    ret
+
+L(mismatch0):
+    brkb    p2.b, p0/z, p1.b
+    lasta   w0, p2, z0.b
+    lasta   w1, p2, z1.b
+    sub     x0, x0, x1
+    ret
+
+L(mismatch1):
+    brkb    p0.b, p2/z, p3.b
+    lasta   w0, p0, z2.b
+    lasta   w1, p0, z3.b
+    sub     x0, x0, x1
+    ret
+
+L(mismatch2):
+    brkb    p6.b, p4/z, p5.b
+    lasta   w0, p6, z6.b
+    lasta   w1, p6, z7.b
+    sub     x0, x0, x1
+    ret
+
+L(mismatch3):
+    brkb    p4.b, p6/z, p7.b
+    lasta   w0, p4, z16.b
+    lasta   w1, p4, z17.b
+    sub     x0, x0, x1
+    ret

That's 7 copies of the same sequence - by using the same register/predicates it should
be feasible to use just 1 copy.

+L(equal):
+    mov x0, #0
+    ret
+END (__memcmp_kunpeng950)

Cheers,
Wilco