[PATCH] malloc: retire mechanism for older non used tcached chunks
Cupertino Miranda
cupertino.miranda@oracle.com
Mon Nov 17 14:15:09 GMT 2025
This patch implements a different retention mechanism for large tcaches.
With this mechanism large cached chunks no longer will be retained
indefinitely by tcache if they are never requested by the application.
Each thread will contain in the tcache structure a monotonically
increasing value (alloc_size_state). Each time any chunk is added to a
tcache this value will be increased by the chunk size.
Also, when a chunk is added to a large tcache bin, it will be annotated
(in cached_at) with the current value of alloc_size_state.
Each time a large tcache bin is traversed (by tcache_location_large), it
will compare alloc_size_state to cached_at values to verify if the chunk
should be retired from the cache.
The retiring should happen if the difference is bigger than
tcache_retire_size.
The tcache_retire_size is set by default to be 5 times the size of
tcache_max value. In any case this patch implements the tunable
glibc.malloc.tcache_retire_size to allow to override it.
In order to quickly get the number of elements in each of the tcache
entries (bins) the implementation still increments and decrements the
num_slots values for large tcaches.
It now uses num_slots, not to limit the number of chunks to cache but
rather to decide to force a full traversal of the bin in order to retire
larger older bins.
Retired chunks are added to a special bin (return_to_arena) which will
only actually free the chunks back to arena, when:
- it fails to find a proper chunk in tcache during allocation,
- the num_slots for a deallocation are filled up (i.e. num_slots is
equal or lower to 0) during deallocation,
- and malloc_trim is executed by the application.
This strategy gives priority for performance, instead of memory usage, by
delaying the release of the chunks back to the the arena only the next
time an actual arena lock is required.
Changes from v1:
- Fix mistake in RETIRE_OLD_CACHED_CHUNKS definition and usage in
tcache_put_large.
- Fixed comments style.
- Change __libc_free to _int_free_chunk also preemptively locking the
arena and allowing to do multiple frees with a single lock.
---
Rules | 2 +-
elf/dl-tunables.list | 4 ++
malloc/arena.c | 3 +
malloc/malloc.c | 159 +++++++++++++++++++++++++++++++++++++------
4 files changed, 146 insertions(+), 22 deletions(-)
diff --git a/Rules b/Rules
index dd319e8013..07fdcf676e 100644
--- a/Rules
+++ b/Rules
@@ -362,7 +362,7 @@ $(foreach t,$(tests-malloc-hugetlb2),$(eval $(call malloc-hugetlb2-ENVS,$(t))))
# All malloc-largetcache tests will be run with GLIBC_TUNABLE=glibc.malloc.tcache_max=1048576
define malloc-largetcache-ENVS
-$(1)-malloc-largetcache-ENV += GLIBC_TUNABLES=glibc.malloc.tcache_max=1048576
+$(1)-malloc-largetcache-ENV += GLIBC_TUNABLES=glibc.malloc.tcache_max=1048576:glibc.malloc.mmap_threshold=1048576
endef
$(foreach t,$(tests-malloc-largetcache),$(eval $(call malloc-largetcache-ENVS,$(t))))
diff --git a/elf/dl-tunables.list b/elf/dl-tunables.list
index c03c9967f0..b169aaf7c1 100644
--- a/elf/dl-tunables.list
+++ b/elf/dl-tunables.list
@@ -74,6 +74,10 @@ glibc {
tcache_unsorted_limit {
type: SIZE_T
}
+ tcache_retire_size {
+ type: SIZE_T
+ minval: 1
+ }
mxfast {
type: SIZE_T
minval: 0
diff --git a/malloc/arena.c b/malloc/arena.c
index 2551cb7749..61413fb03a 100644
--- a/malloc/arena.c
+++ b/malloc/arena.c
@@ -239,6 +239,7 @@ TUNABLE_CALLBACK_FNDECL (set_arena_test, size_t)
TUNABLE_CALLBACK_FNDECL (set_tcache_max, size_t)
TUNABLE_CALLBACK_FNDECL (set_tcache_count, size_t)
TUNABLE_CALLBACK_FNDECL (set_tcache_unsorted_limit, size_t)
+TUNABLE_CALLBACK_FNDECL (set_tcache_retire_size, size_t)
#endif
TUNABLE_CALLBACK_FNDECL (set_mxfast, size_t)
TUNABLE_CALLBACK_FNDECL (set_hugetlb, size_t)
@@ -293,6 +294,8 @@ __ptmalloc_init (void)
TUNABLE_GET (tcache_count, size_t, TUNABLE_CALLBACK (set_tcache_count));
TUNABLE_GET (tcache_unsorted_limit, size_t,
TUNABLE_CALLBACK (set_tcache_unsorted_limit));
+ TUNABLE_GET (tcache_retire_size, size_t,
+ TUNABLE_CALLBACK (set_tcache_retire_size));
# endif
TUNABLE_GET (mxfast, size_t, TUNABLE_CALLBACK (set_mxfast));
TUNABLE_GET (hugetlb, size_t, TUNABLE_CALLBACK (set_hugetlb));
diff --git a/malloc/malloc.c b/malloc/malloc.c
index 0b21bdf1bd..eaa6d10563 100644
--- a/malloc/malloc.c
+++ b/malloc/malloc.c
@@ -299,6 +299,7 @@
# define TCACHE_LARGE_BINS 12 /* Up to 4M chunks */
# define TCACHE_MAX_BINS (TCACHE_SMALL_BINS + TCACHE_LARGE_BINS)
# define MAX_TCACHE_SMALL_SIZE tidx2csize (TCACHE_SMALL_BINS-1)
+# define TCACHE_DEFAULT_RETIRE_MULTIPLIER 5
# define tidx2csize(idx) (((size_t) idx) * MALLOC_ALIGNMENT + MINSIZE)
# define tidx2usize(idx) (((size_t) idx) * MALLOC_ALIGNMENT + MINSIZE - SIZE_SZ)
@@ -1947,6 +1948,8 @@ struct malloc_par
/* Maximum number of chunks to remove from the unsorted list, which
aren't used to prefill the cache. */
size_t tcache_unsorted_limit;
+ uintptr_t tcache_retire_size;
+ size_t tcache_trim;
#endif
};
@@ -1979,7 +1982,12 @@ static struct malloc_par mp_ =
.tcache_count = TCACHE_FILL_COUNT,
.tcache_small_bins = TCACHE_SMALL_BINS,
.tcache_max_bytes = MAX_TCACHE_SMALL_SIZE + 1,
- .tcache_unsorted_limit = 0 /* No limit. */
+ .tcache_unsorted_limit = 0, /* No limit. */
+ /* The default does not enables large tcache support.
+ Capacity default is set to 0 to allow to set a default capacity of 10
+ times the tcache_max value when the tunable is set. */
+ .tcache_retire_size = 0,
+ .tcache_trim = 0
#endif
};
@@ -3088,6 +3096,7 @@ typedef struct tcache_entry
struct tcache_entry *next;
/* This field exists to detect double frees. */
uintptr_t key;
+ uintptr_t cached_at;
} tcache_entry;
/* There is one of these for each thread, which contains the
@@ -3100,6 +3109,9 @@ typedef struct tcache_perthread_struct
{
uint16_t num_slots[TCACHE_MAX_BINS];
tcache_entry *entries[TCACHE_MAX_BINS];
+ tcache_entry *return_to_arena;
+ uintptr_t alloc_size_state;
+ size_t trim_id;
} tcache_perthread_struct;
static const union
@@ -3201,7 +3213,7 @@ large_csize2tidx(size_t nb)
/* Caller must ensure that we know tc_idx is valid and there's room
for more chunks. */
static __always_inline void
-tcache_put_n (mchunkptr chunk, size_t tc_idx, tcache_entry **ep, bool mangled)
+tcache_put_n (mchunkptr chunk, tcache_entry **ep, uint16_t *num_slots, bool mangled)
{
tcache_entry *e = (tcache_entry *) chunk2mem (chunk);
@@ -3219,14 +3231,14 @@ tcache_put_n (mchunkptr chunk, size_t tc_idx, tcache_entry **ep, bool mangled)
e->next = PROTECT_PTR (&e->next, REVEAL_PTR (*ep));
*ep = PROTECT_PTR (ep, e);
}
- --(tcache->num_slots[tc_idx]);
+ --(*num_slots);
}
/* Caller must ensure that we know tc_idx is valid and there's
available chunks to remove. Removes chunk from the middle of the
list. */
static __always_inline void *
-tcache_get_n (size_t tc_idx, tcache_entry **ep, bool mangled)
+tcache_get_n (tcache_entry **ep, uint16_t *num_slots, bool mangled)
{
tcache_entry *e;
if (!mangled)
@@ -3242,7 +3254,7 @@ tcache_get_n (size_t tc_idx, tcache_entry **ep, bool mangled)
else
*ep = PROTECT_PTR (ep, REVEAL_PTR (e->next));
- ++(tcache->num_slots[tc_idx]);
+ ++(*num_slots);
e->key = 0;
return (void *) e;
}
@@ -3250,25 +3262,38 @@ tcache_get_n (size_t tc_idx, tcache_entry **ep, bool mangled)
static __always_inline void
tcache_put (mchunkptr chunk, size_t tc_idx)
{
- tcache_put_n (chunk, tc_idx, &tcache->entries[tc_idx], false);
+ tcache_put_n (chunk, &tcache->entries[tc_idx], &tcache->num_slots[tc_idx], false);
}
/* Like the above, but removes from the head of the list. */
static __always_inline void *
tcache_get (size_t tc_idx)
{
- return tcache_get_n (tc_idx, &tcache->entries[tc_idx], false);
+ return tcache_get_n (&tcache->entries[tc_idx], &tcache->num_slots[tc_idx], false);
}
static __always_inline tcache_entry **
-tcache_location_large (size_t nb, size_t tc_idx,
+tcache_location_large (tcache_entry **tep, size_t nb, uint16_t *num_slots,
bool *mangled, tcache_entry **demangled_ptr)
{
- tcache_entry **tep = &(tcache->entries[tc_idx]);
- tcache_entry *te = *tep;
+ tcache_entry *te = *mangled == false ? *tep : REVEAL_PTR (*tep);
while (te != NULL
&& __glibc_unlikely (chunksize (mem2chunk (te)) < nb))
{
+ /* Move chunk to retire bin if chunk is beyond tcache capacity.
+ We should always do (alloc_size_state - cached_at) before attempting
+ any comparissons. This is a requirement to keep the comparisson valid
+ in the cases where one of the values has wraparound. */
+ if (tcache->alloc_size_state - te->cached_at > mp_.tcache_retire_size)
+ {
+ uint16_t not_used = 1;
+ tcache_entry *e = tcache_get_n (tep, num_slots, *mangled);
+ tcache_put_n (mem2chunk (e), &tcache->return_to_arena, ¬_used,
+ false);
+ te = *mangled == false ? *tep : REVEAL_PTR (*tep);
+ continue;
+ }
+
tep = & (te->next);
te = REVEAL_PTR (te->next);
*mangled = true;
@@ -3278,15 +3303,79 @@ tcache_location_large (size_t nb, size_t tc_idx,
return tep;
}
+/* Force for tcache_location_large to traverse the full bin by requesting a too
+ big of chunk. This allows it to mark old chunks to be released from tcache. */
+#define RETIRE_OLD_CACHED_CHUNKS(ENTRY_PTR, TC_IDX, MANGLED_PTR) { \
+ tcache_entry *te; \
+ tcache_location_large (ENTRY_PTR, PTRDIFF_MAX, &tcache->num_slots[TC_IDX], \
+ MANGLED_PTR, &te); \
+}
+
+static __always_inline void
+tcache_large_cleanup (void)
+{
+ if (!tcache_enabled ())
+ return;
+
+ mstate current_locked_arena = NULL;
+ while (tcache->return_to_arena)
+ {
+ tcache_entry *e = tcache->return_to_arena;
+ if (__glibc_unlikely (misaligned_mem (e)))
+ malloc_printerr ("tcache_thread_shutdown(): "
+ "unaligned tcache chunk detected");
+ tcache->return_to_arena = REVEAL_PTR (e->next);
+ mchunkptr p = mem2chunk (e);
+ size_t size = chunksize (p);
+ mstate chunk_arena = arena_for_chunk (p);
+
+ if (current_locked_arena != chunk_arena)
+ {
+ if (current_locked_arena != NULL)
+ __libc_lock_unlock (current_locked_arena->mutex);
+ __libc_lock_lock (chunk_arena->mutex);
+ current_locked_arena = chunk_arena;
+ }
+ _int_free_chunk (arena_for_chunk (p), p, size, 1);
+ }
+ if (current_locked_arena != NULL)
+ __libc_lock_unlock (current_locked_arena->mutex);
+
+ /* Traverse all tcache bins and retire old cached chunks. */
+ if (mp_.tcache_trim - tcache->trim_id != 0)
+ {
+ tcache->trim_id += mp_.tcache_trim;
+ for (size_t tc_idx = TCACHE_SMALL_BINS; tc_idx < TCACHE_MAX_BINS; tc_idx++)
+ {
+ bool mangled = false;
+ tcache_entry **tep = &(tcache->entries[tc_idx]);
+ RETIRE_OLD_CACHED_CHUNKS (tep, tc_idx, &mangled);
+ }
+ }
+}
+
static __always_inline void
tcache_put_large (mchunkptr chunk, size_t tc_idx)
{
tcache_entry **entry;
+ tcache_entry *e = chunk2mem (chunk);
bool mangled = false;
tcache_entry *te;
- entry = tcache_location_large (chunksize (chunk), tc_idx, &mangled, &te);
+ tcache_entry **tep = &(tcache->entries[tc_idx]);
+ entry = tcache_location_large (tep, chunksize (chunk),
+ &tcache->num_slots[tc_idx], &mangled, &te);
- return tcache_put_n (chunk, tc_idx, entry, mangled);
+ e->cached_at = tcache->alloc_size_state;
+ tcache->alloc_size_state += chunksize (chunk);
+ tcache_put_n (chunk, entry, &tcache->num_slots[tc_idx], mangled);
+
+ /* If the bin is full, execute tcache_location_large again from entry forward
+ with a much bigger size entry such that it would traverse the full bin. */
+ if (tcache->num_slots[tc_idx] <= 0)
+ {
+ RETIRE_OLD_CACHED_CHUNKS (entry, tc_idx, &mangled);
+ tcache_large_cleanup ();
+ }
}
static __always_inline void *
@@ -3295,12 +3384,16 @@ tcache_get_large (size_t tc_idx, size_t nb)
tcache_entry **entry;
bool mangled = false;
tcache_entry *te;
- entry = tcache_location_large (nb, tc_idx, &mangled, &te);
+ tcache_entry **tep = &(tcache->entries[tc_idx]);
+ entry = tcache_location_large (tep, nb, &tcache->num_slots[tc_idx], &mangled, &te);
if (te == NULL || nb != chunksize (mem2chunk (te)))
- return NULL;
+ {
+ tcache_large_cleanup ();
+ return NULL;
+ }
- return tcache_get_n (tc_idx, entry, mangled);
+ return tcache_get_n (entry, &tcache->num_slots[tc_idx], mangled);
}
static void tcache_init (mstate av);
@@ -3312,7 +3405,7 @@ tcache_get_align (size_t nb, size_t alignment)
{
size_t tc_idx = csize2tidx (nb);
if (__glibc_unlikely (tc_idx >= TCACHE_SMALL_BINS))
- tc_idx = large_csize2tidx (nb);
+ tc_idx = large_csize2tidx (nb);
/* The tcache itself isn't encoded, but the chain is. */
tcache_entry **tep = & tcache->entries[tc_idx];
@@ -3338,7 +3431,7 @@ tcache_get_align (size_t nb, size_t alignment)
if (te != NULL
&& csize == nb
&& PTR_IS_ALIGNED (te, alignment))
- return tag_new_usable (tcache_get_n (tc_idx, tep, mangled));
+ return tag_new_usable (tcache_get_n (tep, &tcache->num_slots[tc_idx], mangled));
DIAG_POP_NEEDS_COMMENT;
}
return NULL;
@@ -3358,7 +3451,7 @@ tcache_double_free_verify (tcache_entry *e)
tmp;
tmp = REVEAL_PTR (tmp->next), ++cnt)
{
- if (cnt >= mp_.tcache_count)
+ if (tc_idx < TCACHE_SMALL_BINS && cnt >= mp_.tcache_count)
malloc_printerr ("free(): too many chunks detected in tcache");
if (__glibc_unlikely (misaligned_mem (tmp)))
malloc_printerr ("free(): unaligned chunk detected in tcache 2");
@@ -3398,6 +3491,15 @@ tcache_thread_shutdown (void)
tcache_tmp->entries[i] = REVEAL_PTR (e->next);
__libc_free (e);
}
+ while (tcache_tmp->return_to_arena)
+ {
+ tcache_entry *e = tcache_tmp->return_to_arena;
+ if (__glibc_unlikely (misaligned_mem (e)))
+ malloc_printerr ("tcache_thread_shutdown(): "
+ "unaligned tcache chunk detected");
+ tcache_tmp->return_to_arena = REVEAL_PTR (e->next);
+ __libc_free (e);
+ }
}
__libc_free (tcache_tmp);
@@ -3430,6 +3532,9 @@ tcache_init (mstate av)
memset (tcache, 0, bytes);
for (int i = 0; i < TCACHE_MAX_BINS; i++)
tcache->num_slots[i] = mp_.tcache_count;
+ tcache->alloc_size_state = 0;
+ tcache->return_to_arena = NULL;
+ tcache->trim_id = 0;
}
}
@@ -3559,8 +3664,7 @@ __libc_free (void *mem)
else
{
tc_idx = large_csize2tidx (size);
- if (size >= MINSIZE
- && __glibc_likely (tcache->num_slots[tc_idx] != 0))
+ if (size >= MINSIZE && tcache_enabled ())
return tcache_put_large (p, tc_idx);
}
@@ -5281,6 +5385,9 @@ int
__malloc_trim (size_t s)
{
int result = 0;
+#if USE_TCACHE
+ mp_.tcache_trim += 1;
+#endif
mstate ar_ptr = &main_arena;
do
@@ -5569,11 +5676,13 @@ do_set_tcache_max (size_t value)
LIBC_PROBE (memory_tunable_tcache_max_bytes, 2, value, mp_.tcache_max_bytes);
- if (tc_idx < TCACHE_MAX_BINS)
+ if (tc_idx < TCACHE_MAX_BINS || mp_.tcache_retire_size == 0)
{
if (tc_idx < TCACHE_SMALL_BINS)
mp_.tcache_small_bins = tc_idx + 1;
mp_.tcache_max_bytes = nb + 1;
+ if (tc_idx >= TCACHE_SMALL_BINS && mp_.tcache_retire_size == 0)
+ mp_.tcache_retire_size = nb * TCACHE_DEFAULT_RETIRE_MULTIPLIER;
return 1;
}
@@ -5599,6 +5708,14 @@ do_set_tcache_unsorted_limit (size_t value)
mp_.tcache_unsorted_limit = value;
return 1;
}
+
+static __always_inline int
+do_set_tcache_retire_size (size_t value)
+{
+ LIBC_PROBE (memory_tunable_tcache_retire_size, 2, value, mp_.tcache_retire_size);
+ mp_.tcache_retire_size = value;
+ return 1;
+}
#endif
static __always_inline int
--
2.30.2
More information about the Libc-alpha
mailing list