[PATCH] malloc: retire mechanism for older non used tcached chunks
Cupertino Miranda
cupertino.miranda@oracle.com
Mon Dec 15 15:13:12 GMT 2025
PING!
On 17-11-2025 14:17, Cupertino Miranda wrote:
> This should have been versioned v2.
>
> On 17-11-2025 14:15, Cupertino Miranda wrote:
>> This patch implements a different retention mechanism for large tcaches.
>> With this mechanism large cached chunks no longer will be retained
>> indefinitely by tcache if they are never requested by the application.
>>
>> Each thread will contain in the tcache structure a monotonically
>> increasing value (alloc_size_state). Each time any chunk is added to a
>> tcache this value will be increased by the chunk size.
>> Also, when a chunk is added to a large tcache bin, it will be annotated
>> (in cached_at) with the current value of alloc_size_state.
>>
>> Each time a large tcache bin is traversed (by tcache_location_large), it
>> will compare alloc_size_state to cached_at values to verify if the chunk
>> should be retired from the cache.
>> The retiring should happen if the difference is bigger than
>> tcache_retire_size.
>>
>> The tcache_retire_size is set by default to be 5 times the size of
>> tcache_max value. In any case this patch implements the tunable
>> glibc.malloc.tcache_retire_size to allow to override it.
>>
>> In order to quickly get the number of elements in each of the tcache
>> entries (bins) the implementation still increments and decrements the
>> num_slots values for large tcaches.
>> It now uses num_slots, not to limit the number of chunks to cache but
>> rather to decide to force a full traversal of the bin in order to retire
>> larger older bins.
>>
>> Retired chunks are added to a special bin (return_to_arena) which will
>> only actually free the chunks back to arena, when:
>> - it fails to find a proper chunk in tcache during allocation,
>> - the num_slots for a deallocation are filled up (i.e. num_slots is
>> equal or lower to 0) during deallocation,
>> - and malloc_trim is executed by the application.
>> This strategy gives priority for performance, instead of memory usage, by
>> delaying the release of the chunks back to the the arena only the next
>> time an actual arena lock is required.
>>
>> Changes from v1:
>> - Fix mistake in RETIRE_OLD_CACHED_CHUNKS definition and usage in
>> tcache_put_large.
>> - Fixed comments style.
>> - Change __libc_free to _int_free_chunk also preemptively locking the
>> arena and allowing to do multiple frees with a single lock.
>> ---
>> Rules | 2 +-
>> elf/dl-tunables.list | 4 ++
>> malloc/arena.c | 3 +
>> malloc/malloc.c | 159 +++++++++++++++++++++++++++++++++++++------
>> 4 files changed, 146 insertions(+), 22 deletions(-)
>>
>> diff --git a/Rules b/Rules
>> index dd319e8013..07fdcf676e 100644
>> --- a/Rules
>> +++ b/Rules
>> @@ -362,7 +362,7 @@ $(foreach t,$(tests-malloc-hugetlb2),$(eval $(call
>> malloc-hugetlb2-ENVS,$(t))))
>> # All malloc-largetcache tests will be run with
>> GLIBC_TUNABLE=glibc.malloc.tcache_max=1048576
>> define malloc-largetcache-ENVS
>> -$(1)-malloc-largetcache-ENV +=
>> GLIBC_TUNABLES=glibc.malloc.tcache_max=1048576
>> +$(1)-malloc-largetcache-ENV +=
>> GLIBC_TUNABLES=glibc.malloc.tcache_max=1048576:glibc.malloc.mmap_threshold=1048576
>> endef
>> $(foreach t,$(tests-malloc-largetcache),$(eval $(call malloc-
>> largetcache-ENVS,$(t))))
>> diff --git a/elf/dl-tunables.list b/elf/dl-tunables.list
>> index c03c9967f0..b169aaf7c1 100644
>> --- a/elf/dl-tunables.list
>> +++ b/elf/dl-tunables.list
>> @@ -74,6 +74,10 @@ glibc {
>> tcache_unsorted_limit {
>> type: SIZE_T
>> }
>> + tcache_retire_size {
>> + type: SIZE_T
>> + minval: 1
>> + }
>> mxfast {
>> type: SIZE_T
>> minval: 0
>> diff --git a/malloc/arena.c b/malloc/arena.c
>> index 2551cb7749..61413fb03a 100644
>> --- a/malloc/arena.c
>> +++ b/malloc/arena.c
>> @@ -239,6 +239,7 @@ TUNABLE_CALLBACK_FNDECL (set_arena_test, size_t)
>> TUNABLE_CALLBACK_FNDECL (set_tcache_max, size_t)
>> TUNABLE_CALLBACK_FNDECL (set_tcache_count, size_t)
>> TUNABLE_CALLBACK_FNDECL (set_tcache_unsorted_limit, size_t)
>> +TUNABLE_CALLBACK_FNDECL (set_tcache_retire_size, size_t)
>> #endif
>> TUNABLE_CALLBACK_FNDECL (set_mxfast, size_t)
>> TUNABLE_CALLBACK_FNDECL (set_hugetlb, size_t)
>> @@ -293,6 +294,8 @@ __ptmalloc_init (void)
>> TUNABLE_GET (tcache_count, size_t, TUNABLE_CALLBACK
>> (set_tcache_count));
>> TUNABLE_GET (tcache_unsorted_limit, size_t,
>> TUNABLE_CALLBACK (set_tcache_unsorted_limit));
>> + TUNABLE_GET (tcache_retire_size, size_t,
>> + TUNABLE_CALLBACK (set_tcache_retire_size));
>> # endif
>> TUNABLE_GET (mxfast, size_t, TUNABLE_CALLBACK (set_mxfast));
>> TUNABLE_GET (hugetlb, size_t, TUNABLE_CALLBACK (set_hugetlb));
>> diff --git a/malloc/malloc.c b/malloc/malloc.c
>> index 0b21bdf1bd..eaa6d10563 100644
>> --- a/malloc/malloc.c
>> +++ b/malloc/malloc.c
>> @@ -299,6 +299,7 @@
>> # define TCACHE_LARGE_BINS 12 /* Up to 4M chunks */
>> # define TCACHE_MAX_BINS (TCACHE_SMALL_BINS + TCACHE_LARGE_BINS)
>> # define MAX_TCACHE_SMALL_SIZE tidx2csize (TCACHE_SMALL_BINS-1)
>> +# define TCACHE_DEFAULT_RETIRE_MULTIPLIER 5
>> # define tidx2csize(idx) (((size_t) idx) * MALLOC_ALIGNMENT +
>> MINSIZE)
>> # define tidx2usize(idx) (((size_t) idx) * MALLOC_ALIGNMENT +
>> MINSIZE - SIZE_SZ)
>> @@ -1947,6 +1948,8 @@ struct malloc_par
>> /* Maximum number of chunks to remove from the unsorted list, which
>> aren't used to prefill the cache. */
>> size_t tcache_unsorted_limit;
>> + uintptr_t tcache_retire_size;
>> + size_t tcache_trim;
>> #endif
>> };
>> @@ -1979,7 +1982,12 @@ static struct malloc_par mp_ =
>> .tcache_count = TCACHE_FILL_COUNT,
>> .tcache_small_bins = TCACHE_SMALL_BINS,
>> .tcache_max_bytes = MAX_TCACHE_SMALL_SIZE + 1,
>> - .tcache_unsorted_limit = 0 /* No limit. */
>> + .tcache_unsorted_limit = 0, /* No limit. */
>> + /* The default does not enables large tcache support.
>> + Capacity default is set to 0 to allow to set a default capacity
>> of 10
>> + times the tcache_max value when the tunable is set. */
>> + .tcache_retire_size = 0,
>> + .tcache_trim = 0
>> #endif
>> };
>> @@ -3088,6 +3096,7 @@ typedef struct tcache_entry
>> struct tcache_entry *next;
>> /* This field exists to detect double frees. */
>> uintptr_t key;
>> + uintptr_t cached_at;
>> } tcache_entry;
>> /* There is one of these for each thread, which contains the
>> @@ -3100,6 +3109,9 @@ typedef struct tcache_perthread_struct
>> {
>> uint16_t num_slots[TCACHE_MAX_BINS];
>> tcache_entry *entries[TCACHE_MAX_BINS];
>> + tcache_entry *return_to_arena;
>> + uintptr_t alloc_size_state;
>> + size_t trim_id;
>> } tcache_perthread_struct;
>> static const union
>> @@ -3201,7 +3213,7 @@ large_csize2tidx(size_t nb)
>> /* Caller must ensure that we know tc_idx is valid and there's room
>> for more chunks. */
>> static __always_inline void
>> -tcache_put_n (mchunkptr chunk, size_t tc_idx, tcache_entry **ep, bool
>> mangled)
>> +tcache_put_n (mchunkptr chunk, tcache_entry **ep, uint16_t
>> *num_slots, bool mangled)
>> {
>> tcache_entry *e = (tcache_entry *) chunk2mem (chunk);
>> @@ -3219,14 +3231,14 @@ tcache_put_n (mchunkptr chunk, size_t tc_idx,
>> tcache_entry **ep, bool mangled)
>> e->next = PROTECT_PTR (&e->next, REVEAL_PTR (*ep));
>> *ep = PROTECT_PTR (ep, e);
>> }
>> - --(tcache->num_slots[tc_idx]);
>> + --(*num_slots);
>> }
>> /* Caller must ensure that we know tc_idx is valid and there's
>> available chunks to remove. Removes chunk from the middle of the
>> list. */
>> static __always_inline void *
>> -tcache_get_n (size_t tc_idx, tcache_entry **ep, bool mangled)
>> +tcache_get_n (tcache_entry **ep, uint16_t *num_slots, bool mangled)
>> {
>> tcache_entry *e;
>> if (!mangled)
>> @@ -3242,7 +3254,7 @@ tcache_get_n (size_t tc_idx, tcache_entry **ep,
>> bool mangled)
>> else
>> *ep = PROTECT_PTR (ep, REVEAL_PTR (e->next));
>> - ++(tcache->num_slots[tc_idx]);
>> + ++(*num_slots);
>> e->key = 0;
>> return (void *) e;
>> }
>> @@ -3250,25 +3262,38 @@ tcache_get_n (size_t tc_idx, tcache_entry
>> **ep, bool mangled)
>> static __always_inline void
>> tcache_put (mchunkptr chunk, size_t tc_idx)
>> {
>> - tcache_put_n (chunk, tc_idx, &tcache->entries[tc_idx], false);
>> + tcache_put_n (chunk, &tcache->entries[tc_idx], &tcache-
>> >num_slots[tc_idx], false);
>> }
>> /* Like the above, but removes from the head of the list. */
>> static __always_inline void *
>> tcache_get (size_t tc_idx)
>> {
>> - return tcache_get_n (tc_idx, &tcache->entries[tc_idx], false);
>> + return tcache_get_n (&tcache->entries[tc_idx], &tcache-
>> >num_slots[tc_idx], false);
>> }
>> static __always_inline tcache_entry **
>> -tcache_location_large (size_t nb, size_t tc_idx,
>> +tcache_location_large (tcache_entry **tep, size_t nb, uint16_t
>> *num_slots,
>> bool *mangled, tcache_entry **demangled_ptr)
>> {
>> - tcache_entry **tep = &(tcache->entries[tc_idx]);
>> - tcache_entry *te = *tep;
>> + tcache_entry *te = *mangled == false ? *tep : REVEAL_PTR (*tep);
>> while (te != NULL
>> && __glibc_unlikely (chunksize (mem2chunk (te)) < nb))
>> {
>> + /* Move chunk to retire bin if chunk is beyond tcache capacity.
>> + We should always do (alloc_size_state - cached_at) before
>> attempting
>> + any comparissons. This is a requirement to keep the comparisson
>> valid
>> + in the cases where one of the values has wraparound. */
>> + if (tcache->alloc_size_state - te->cached_at >
>> mp_.tcache_retire_size)
>> + {
>> + uint16_t not_used = 1;
>> + tcache_entry *e = tcache_get_n (tep, num_slots, *mangled);
>> + tcache_put_n (mem2chunk (e), &tcache->return_to_arena, ¬_used,
>> + false);
>> + te = *mangled == false ? *tep : REVEAL_PTR (*tep);
>> + continue;
>> + }
>> +
>> tep = & (te->next);
>> te = REVEAL_PTR (te->next);
>> *mangled = true;
>> @@ -3278,15 +3303,79 @@ tcache_location_large (size_t nb, size_t tc_idx,
>> return tep;
>> }
>> +/* Force for tcache_location_large to traverse the full bin by
>> requesting a too
>> + big of chunk. This allows it to mark old chunks to be released
>> from tcache. */
>> +#define RETIRE_OLD_CACHED_CHUNKS(ENTRY_PTR, TC_IDX, MANGLED_PTR) { \
>> + tcache_entry *te; \
>> + tcache_location_large (ENTRY_PTR, PTRDIFF_MAX, &tcache-
>> >num_slots[TC_IDX], \
>> + MANGLED_PTR, &te); \
>> +}
>> +
>> +static __always_inline void
>> +tcache_large_cleanup (void)
>> +{
>> + if (!tcache_enabled ())
>> + return;
>> +
>> + mstate current_locked_arena = NULL;
>> + while (tcache->return_to_arena)
>> + {
>> + tcache_entry *e = tcache->return_to_arena;
>> + if (__glibc_unlikely (misaligned_mem (e)))
>> + malloc_printerr ("tcache_thread_shutdown(): "
>> + "unaligned tcache chunk detected");
>> + tcache->return_to_arena = REVEAL_PTR (e->next);
>> + mchunkptr p = mem2chunk (e);
>> + size_t size = chunksize (p);
>> + mstate chunk_arena = arena_for_chunk (p);
>> +
>> + if (current_locked_arena != chunk_arena)
>> + {
>> + if (current_locked_arena != NULL)
>> + __libc_lock_unlock (current_locked_arena->mutex);
>> + __libc_lock_lock (chunk_arena->mutex);
>> + current_locked_arena = chunk_arena;
>> + }
>> + _int_free_chunk (arena_for_chunk (p), p, size, 1);
>> + }
>> + if (current_locked_arena != NULL)
>> + __libc_lock_unlock (current_locked_arena->mutex);
>> +
>> + /* Traverse all tcache bins and retire old cached chunks. */
>> + if (mp_.tcache_trim - tcache->trim_id != 0)
>> + {
>> + tcache->trim_id += mp_.tcache_trim;
>> + for (size_t tc_idx = TCACHE_SMALL_BINS; tc_idx <
>> TCACHE_MAX_BINS; tc_idx++)
>> + {
>> + bool mangled = false;
>> + tcache_entry **tep = &(tcache->entries[tc_idx]);
>> + RETIRE_OLD_CACHED_CHUNKS (tep, tc_idx, &mangled);
>> + }
>> + }
>> +}
>> +
>> static __always_inline void
>> tcache_put_large (mchunkptr chunk, size_t tc_idx)
>> {
>> tcache_entry **entry;
>> + tcache_entry *e = chunk2mem (chunk);
>> bool mangled = false;
>> tcache_entry *te;
>> - entry = tcache_location_large (chunksize (chunk), tc_idx, &mangled,
>> &te);
>> + tcache_entry **tep = &(tcache->entries[tc_idx]);
>> + entry = tcache_location_large (tep, chunksize (chunk),
>> + &tcache->num_slots[tc_idx], &mangled, &te);
>> - return tcache_put_n (chunk, tc_idx, entry, mangled);
>> + e->cached_at = tcache->alloc_size_state;
>> + tcache->alloc_size_state += chunksize (chunk);
>> + tcache_put_n (chunk, entry, &tcache->num_slots[tc_idx], mangled);
>> +
>> + /* If the bin is full, execute tcache_location_large again from
>> entry forward
>> + with a much bigger size entry such that it would traverse the
>> full bin. */
>> + if (tcache->num_slots[tc_idx] <= 0)
>> + {
>> + RETIRE_OLD_CACHED_CHUNKS (entry, tc_idx, &mangled);
>> + tcache_large_cleanup ();
>> + }
>> }
>> static __always_inline void *
>> @@ -3295,12 +3384,16 @@ tcache_get_large (size_t tc_idx, size_t nb)
>> tcache_entry **entry;
>> bool mangled = false;
>> tcache_entry *te;
>> - entry = tcache_location_large (nb, tc_idx, &mangled, &te);
>> + tcache_entry **tep = &(tcache->entries[tc_idx]);
>> + entry = tcache_location_large (tep, nb, &tcache->num_slots[tc_idx],
>> &mangled, &te);
>> if (te == NULL || nb != chunksize (mem2chunk (te)))
>> - return NULL;
>> + {
>> + tcache_large_cleanup ();
>> + return NULL;
>> + }
>> - return tcache_get_n (tc_idx, entry, mangled);
>> + return tcache_get_n (entry, &tcache->num_slots[tc_idx], mangled);
>> }
>> static void tcache_init (mstate av);
>> @@ -3312,7 +3405,7 @@ tcache_get_align (size_t nb, size_t alignment)
>> {
>> size_t tc_idx = csize2tidx (nb);
>> if (__glibc_unlikely (tc_idx >= TCACHE_SMALL_BINS))
>> - tc_idx = large_csize2tidx (nb);
>> + tc_idx = large_csize2tidx (nb);
>> /* The tcache itself isn't encoded, but the chain is. */
>> tcache_entry **tep = & tcache->entries[tc_idx];
>> @@ -3338,7 +3431,7 @@ tcache_get_align (size_t nb, size_t alignment)
>> if (te != NULL
>> && csize == nb
>> && PTR_IS_ALIGNED (te, alignment))
>> - return tag_new_usable (tcache_get_n (tc_idx, tep, mangled));
>> + return tag_new_usable (tcache_get_n (tep, &tcache-
>> >num_slots[tc_idx], mangled));
>> DIAG_POP_NEEDS_COMMENT;
>> }
>> return NULL;
>> @@ -3358,7 +3451,7 @@ tcache_double_free_verify (tcache_entry *e)
>> tmp;
>> tmp = REVEAL_PTR (tmp->next), ++cnt)
>> {
>> - if (cnt >= mp_.tcache_count)
>> + if (tc_idx < TCACHE_SMALL_BINS && cnt >= mp_.tcache_count)
>> malloc_printerr ("free(): too many chunks detected in tcache");
>> if (__glibc_unlikely (misaligned_mem (tmp)))
>> malloc_printerr ("free(): unaligned chunk detected in tcache
>> 2");
>> @@ -3398,6 +3491,15 @@ tcache_thread_shutdown (void)
>> tcache_tmp->entries[i] = REVEAL_PTR (e->next);
>> __libc_free (e);
>> }
>> + while (tcache_tmp->return_to_arena)
>> + {
>> + tcache_entry *e = tcache_tmp->return_to_arena;
>> + if (__glibc_unlikely (misaligned_mem (e)))
>> + malloc_printerr ("tcache_thread_shutdown(): "
>> + "unaligned tcache chunk detected");
>> + tcache_tmp->return_to_arena = REVEAL_PTR (e->next);
>> + __libc_free (e);
>> + }
>> }
>> __libc_free (tcache_tmp);
>> @@ -3430,6 +3532,9 @@ tcache_init (mstate av)
>> memset (tcache, 0, bytes);
>> for (int i = 0; i < TCACHE_MAX_BINS; i++)
>> tcache->num_slots[i] = mp_.tcache_count;
>> + tcache->alloc_size_state = 0;
>> + tcache->return_to_arena = NULL;
>> + tcache->trim_id = 0;
>> }
>> }
>> @@ -3559,8 +3664,7 @@ __libc_free (void *mem)
>> else
>> {
>> tc_idx = large_csize2tidx (size);
>> - if (size >= MINSIZE
>> - && __glibc_likely (tcache->num_slots[tc_idx] != 0))
>> + if (size >= MINSIZE && tcache_enabled ())
>> return tcache_put_large (p, tc_idx);
>> }
>> @@ -5281,6 +5385,9 @@ int
>> __malloc_trim (size_t s)
>> {
>> int result = 0;
>> +#if USE_TCACHE
>> + mp_.tcache_trim += 1;
>> +#endif
>> mstate ar_ptr = &main_arena;
>> do
>> @@ -5569,11 +5676,13 @@ do_set_tcache_max (size_t value)
>> LIBC_PROBE (memory_tunable_tcache_max_bytes, 2, value,
>> mp_.tcache_max_bytes);
>> - if (tc_idx < TCACHE_MAX_BINS)
>> + if (tc_idx < TCACHE_MAX_BINS || mp_.tcache_retire_size == 0)
>> {
>> if (tc_idx < TCACHE_SMALL_BINS)
>> mp_.tcache_small_bins = tc_idx + 1;
>> mp_.tcache_max_bytes = nb + 1;
>> + if (tc_idx >= TCACHE_SMALL_BINS && mp_.tcache_retire_size == 0)
>> + mp_.tcache_retire_size = nb * TCACHE_DEFAULT_RETIRE_MULTIPLIER;
>> return 1;
>> }
>> @@ -5599,6 +5708,14 @@ do_set_tcache_unsorted_limit (size_t value)
>> mp_.tcache_unsorted_limit = value;
>> return 1;
>> }
>> +
>> +static __always_inline int
>> +do_set_tcache_retire_size (size_t value)
>> +{
>> + LIBC_PROBE (memory_tunable_tcache_retire_size, 2, value,
>> mp_.tcache_retire_size);
>> + mp_.tcache_retire_size = value;
>> + return 1;
>> +}
>> #endif
>> static __always_inline int
>
More information about the Libc-alpha
mailing list