[PATCH] malloc: Cleanup clear_memory
Wilco Dijkstra
Wilco.Dijkstra@arm.com
Thu Nov 19 16:35:28 GMT 2026
Use an end pointer to simplify addressing. Use original size in all calculations.
Add an expansion for when MALLOC_USER_SIZE_ALIGNMENT is 2*SIZE_SZ. The resulting
code is far better, eg. on AArch64 the tail of __libc_calloc is now:
str xzr, [x0, #8]
cmp x2, #0x48
b.hi 4afc <__GI___libc_calloc+0x158> // b.pmore
str xzr, [x0]
add x1, x0, x2
cmp x2, #0x20
b.ls 4a50 <__GI___libc_calloc+0xac> // b.plast
stp xzr, xzr, [x0, #16]
stp xzr, xzr, [x1, #-32]
4a50: stp xzr, xzr, [x1, #-16]
ret
Passes regress, OK for commit?
---
diff --git a/sysdeps/generic/calloc-clear-memory.h b/sysdeps/generic/calloc-clear-memory.h
index 0fe980e332aa51f75702a459204f81248cb3fc2f..26dc286f0ff50cc31f88f63e0322e8477737fe6d 100644
--- a/sysdeps/generic/calloc-clear-memory.h
+++ b/sysdeps/generic/calloc-clear-memory.h
@@ -22,37 +22,45 @@ static __always_inline void *
clear_memory (void *p, unsigned long clearsize)
{
INTERNAL_SIZE_T *d = (INTERNAL_SIZE_T *) p;
- /* Unroll clear memory size up to 9 * INTERNAL_SIZE_T bytes. We know
- that contents have an odd number of INTERNAL_SIZE_T-sized words;
- minimally 3 words. */
- unsigned long nclears = clearsize / SIZE_SZ;
+ INTERNAL_SIZE_T *end = (INTERNAL_SIZE_T *) ((uintptr_t) p + clearsize);
- if (nclears > 9)
+ /* Clearsize is the usable size of a chunk. It is a multiple of SIZE_SZ
+ and incremenets by 2 * SIZE_SZ. Depending on the chunk layout, the
+ minimum is either 2 or 3 * SIZE_SZ. */
+ if (clearsize > 9 * SIZE_SZ)
return memset (d, 0, clearsize);
- /* NB: The VRP pass in GCC 14.2 will optimize it out for the default size
- calculations. */
- if (nclears < 3)
- {
#ifdef MALLOC_USER_SIZE_ALIGNMENT
- return memset (d, 0, clearsize);
-#else
- __builtin_unreachable ();
-#endif
+ /* Clear 2-8 words using overlapping stores to reduce branching. */
+ if (MALLOC_USER_SIZE_ALIGNMENT >= SIZE_SZ * 2)
+ {
+ d[0] = 0;
+ d[1] = 0;
+ if (clearsize > 4 * SIZE_SZ)
+ {
+ d[2] = 0;
+ d[3] = 0;
+ end[-4] = 0;
+ end[-3] = 0;
+ }
+ end[-2] = 0;
+ end[-1] = 0;
+ return d;
}
+#endif
- /* Use repeated stores with 1 branch, instead of up to 3. */
- *(d + 0) = 0;
- *(d + 1) = 0;
- *(d + 2) = 0;
- *(d + nclears - 2) = 0;
- *(d + nclears - 2 + 1) = 0;
- if (nclears > 5)
+ /* Clear 3-9 words using overlapping stores to reduce branching. */
+ d[0] = 0;
+ d[1] = 0;
+ d[2] = 0;
+ end[-2] = 0;
+ end[-1] = 0;
+ if (clearsize > 5 * SIZE_SZ)
{
- *(d + 3) = 0;
- *(d + 3 + 1) = 0;
- *(d + nclears - 4) = 0;
- *(d + nclears - 4 + 1) = 0;
+ d[3] = 0;
+ d[4] = 0;
+ end[-4] = 0;
+ end[-3] = 0;
}
return d;
More information about the Libc-alpha
mailing list