[PATCH] malloc: Improve performance of __libc_calloc

[Wilco Dijkstra]

Hi DJ,

> I have two comments:
>
> 1. It no longer applies due to your removal of initialization code in
>   your previous patch.  Looks like a trivial rebase.
>
> 2. "sz" changes type from INTERNAL_SIZE_T to size_t.  At the moment this
>    doesn't matter because those types are the same on all supported
>    targets, but we allow a configuration to have 64-bit size_t but
>    32-bit INTENRAL_SIZE_T to save memory.  This should at least be
>    addressed.

Malloc doesn't build at all with a different INTERNAL_SIZE_T. But basically that would be
an unchecked cast of size_t to a smaller INTERNAL_SIZE_T, thus incorrectly dealing with
huge user allocations.

> The rest of the code looks OK to me, please post a v2 that's applyable
> (for the record) and ponder if we care about #2 above.

See v2 below - I also fixed another oddity: a signed ptrdiff_t was used in the overflow check.
Using size_t makes more sense since checked_request2size deals with huge sizes.

Cheers,
Wilco


v2: rebase, remove odd ptrdiff_t in overflow check

Improve performance of __libc_calloc by splitting it into 2 parts: first handle
the tcache fastpath, then do the rest in a separate tailcalled function.
This results in significant performance gains since __libc_calloc doesn't need
to setup a frame.

On Neoverse V2, bench-calloc-simple improves by 5.0% overall. Bench-calloc-tcache 32
improves by 24%.

Regress passed, OK for commit?

---

diff --git a/malloc/malloc.c b/malloc/malloc.c
index afb74d0665e790fc0d1d57b3c4904e7639741a96..fe56a631bce6f5271d94096d18c2bef2c5fd0050 100644
--- a/malloc/malloc.c
+++ b/malloc/malloc.c
@@ -591,6 +591,8 @@ tag_at (void *ptr)
 void*  __libc_malloc(size_t);
 libc_hidden_proto (__libc_malloc)
 
+static void *__libc_calloc2 (size_t);
+
 /*
   free(void* p)
   Releases the chunk of memory pointed to by p, that had been previously
@@ -3324,6 +3326,13 @@ tcache_init(void)
 
 }
 
+static void * __attribute_noinline__
+tcache_calloc_init (size_t bytes)
+{
+  tcache_init ();
+  return __libc_calloc2 (bytes);
+}
+
 # define MAYBE_INIT_TCACHE() \
   if (__glibc_unlikely (tcache == NULL)) \
     tcache_init();
@@ -3711,38 +3720,13 @@ __libc_pvalloc (size_t bytes)
 }
 
 void *
-__libc_calloc (size_t n, size_t elem_size)
+__libc_calloc2 (size_t sz)
 {
   mstate av;
   mchunkptr oldtop, p;
-  INTERNAL_SIZE_T sz, oldtopsize, csz;
+  INTERNAL_SIZE_T oldtopsize, csz;
   void *mem;
   unsigned long clearsize;
-  ptrdiff_t bytes;
-
-  if (__glibc_unlikely (__builtin_mul_overflow (n, elem_size, &bytes)))
-    {
-       __set_errno (ENOMEM);
-       return NULL;
-    }
-
-  sz = bytes;
-
-#if USE_TCACHE
-  size_t tc_idx = usize2tidx (bytes);
-  if (tcache_available (tc_idx))
-    {
-      mem = tcache_get (tc_idx);
-      p = mem2chunk (mem);
-      if (__glibc_unlikely (mtag_enabled))
-	return tag_new_zero_region (mem, memsize (p));
-
-      csz = chunksize (p);
-      clearsize = csz - SIZE_SZ;
-      return clear_memory ((INTERNAL_SIZE_T *) mem, clearsize);
-    }
-  MAYBE_INIT_TCACHE ();
-#endif
 
   if (SINGLE_THREAD_P)
     av = &main_arena;
@@ -3828,6 +3812,38 @@ __libc_calloc (size_t n, size_t elem_size)
   clearsize = csz - SIZE_SZ;
   return clear_memory ((INTERNAL_SIZE_T *) mem, clearsize);
 }
+
+void *
+__libc_calloc (size_t n, size_t elem_size)
+{
+  size_t bytes;
+
+  if (__glibc_unlikely (__builtin_mul_overflow (n, elem_size, &bytes)))
+    {
+       __set_errno (ENOMEM);
+       return NULL;
+    }
+
+#if USE_TCACHE
+  size_t tc_idx = usize2tidx (bytes);
+  if (__glibc_likely (tc_idx < mp_.tcache_bins))
+    {
+      if (__glibc_unlikely (tcache == NULL))
+        return tcache_calloc_init (bytes);
+
+      if (__glibc_likely (tcache->entries[tc_idx] != NULL))
+	{
+	  void *mem = tcache_get (tc_idx);
+
+	  if (__glibc_unlikely (mtag_enabled))
+            return tag_new_zero_region (mem, memsize (mem2chunk (mem)));
+
+          return clear_memory ((INTERNAL_SIZE_T *) mem, tidx2usize (tc_idx));
+	}
+    }
+#endif
+  return __libc_calloc2 (bytes);
+}
 #endif /* IS_IN (libc) */
 
 /*