[PATCH] malloc: retire mechanism for older non used tcached chunks
Cupertino Miranda
cupertino.miranda@oracle.com
Mon Nov 17 14:17:28 GMT 2025
This should have been versioned v2.
On 17-11-2025 14:15, Cupertino Miranda wrote:
> This patch implements a different retention mechanism for large tcaches.
> With this mechanism large cached chunks no longer will be retained
> indefinitely by tcache if they are never requested by the application.
>
> Each thread will contain in the tcache structure a monotonically
> increasing value (alloc_size_state). Each time any chunk is added to a
> tcache this value will be increased by the chunk size.
> Also, when a chunk is added to a large tcache bin, it will be annotated
> (in cached_at) with the current value of alloc_size_state.
>
> Each time a large tcache bin is traversed (by tcache_location_large), it
> will compare alloc_size_state to cached_at values to verify if the chunk
> should be retired from the cache.
> The retiring should happen if the difference is bigger than
> tcache_retire_size.
>
> The tcache_retire_size is set by default to be 5 times the size of
> tcache_max value. In any case this patch implements the tunable
> glibc.malloc.tcache_retire_size to allow to override it.
>
> In order to quickly get the number of elements in each of the tcache
> entries (bins) the implementation still increments and decrements the
> num_slots values for large tcaches.
> It now uses num_slots, not to limit the number of chunks to cache but
> rather to decide to force a full traversal of the bin in order to retire
> larger older bins.
>
> Retired chunks are added to a special bin (return_to_arena) which will
> only actually free the chunks back to arena, when:
> - it fails to find a proper chunk in tcache during allocation,
> - the num_slots for a deallocation are filled up (i.e. num_slots is
> equal or lower to 0) during deallocation,
> - and malloc_trim is executed by the application.
> This strategy gives priority for performance, instead of memory usage, by
> delaying the release of the chunks back to the the arena only the next
> time an actual arena lock is required.
>
> Changes from v1:
> - Fix mistake in RETIRE_OLD_CACHED_CHUNKS definition and usage in
> tcache_put_large.
> - Fixed comments style.
> - Change __libc_free to _int_free_chunk also preemptively locking the
> arena and allowing to do multiple frees with a single lock.
> ---
> Rules | 2 +-
> elf/dl-tunables.list | 4 ++
> malloc/arena.c | 3 +
> malloc/malloc.c | 159 +++++++++++++++++++++++++++++++++++++------
> 4 files changed, 146 insertions(+), 22 deletions(-)
>
> diff --git a/Rules b/Rules
> index dd319e8013..07fdcf676e 100644
> --- a/Rules
> +++ b/Rules
> @@ -362,7 +362,7 @@ $(foreach t,$(tests-malloc-hugetlb2),$(eval $(call malloc-hugetlb2-ENVS,$(t))))
>
> # All malloc-largetcache tests will be run with GLIBC_TUNABLE=glibc.malloc.tcache_max=1048576
> define malloc-largetcache-ENVS
> -$(1)-malloc-largetcache-ENV += GLIBC_TUNABLES=glibc.malloc.tcache_max=1048576
> +$(1)-malloc-largetcache-ENV += GLIBC_TUNABLES=glibc.malloc.tcache_max=1048576:glibc.malloc.mmap_threshold=1048576
> endef
> $(foreach t,$(tests-malloc-largetcache),$(eval $(call malloc-largetcache-ENVS,$(t))))
>
> diff --git a/elf/dl-tunables.list b/elf/dl-tunables.list
> index c03c9967f0..b169aaf7c1 100644
> --- a/elf/dl-tunables.list
> +++ b/elf/dl-tunables.list
> @@ -74,6 +74,10 @@ glibc {
> tcache_unsorted_limit {
> type: SIZE_T
> }
> + tcache_retire_size {
> + type: SIZE_T
> + minval: 1
> + }
> mxfast {
> type: SIZE_T
> minval: 0
> diff --git a/malloc/arena.c b/malloc/arena.c
> index 2551cb7749..61413fb03a 100644
> --- a/malloc/arena.c
> +++ b/malloc/arena.c
> @@ -239,6 +239,7 @@ TUNABLE_CALLBACK_FNDECL (set_arena_test, size_t)
> TUNABLE_CALLBACK_FNDECL (set_tcache_max, size_t)
> TUNABLE_CALLBACK_FNDECL (set_tcache_count, size_t)
> TUNABLE_CALLBACK_FNDECL (set_tcache_unsorted_limit, size_t)
> +TUNABLE_CALLBACK_FNDECL (set_tcache_retire_size, size_t)
> #endif
> TUNABLE_CALLBACK_FNDECL (set_mxfast, size_t)
> TUNABLE_CALLBACK_FNDECL (set_hugetlb, size_t)
> @@ -293,6 +294,8 @@ __ptmalloc_init (void)
> TUNABLE_GET (tcache_count, size_t, TUNABLE_CALLBACK (set_tcache_count));
> TUNABLE_GET (tcache_unsorted_limit, size_t,
> TUNABLE_CALLBACK (set_tcache_unsorted_limit));
> + TUNABLE_GET (tcache_retire_size, size_t,
> + TUNABLE_CALLBACK (set_tcache_retire_size));
> # endif
> TUNABLE_GET (mxfast, size_t, TUNABLE_CALLBACK (set_mxfast));
> TUNABLE_GET (hugetlb, size_t, TUNABLE_CALLBACK (set_hugetlb));
> diff --git a/malloc/malloc.c b/malloc/malloc.c
> index 0b21bdf1bd..eaa6d10563 100644
> --- a/malloc/malloc.c
> +++ b/malloc/malloc.c
> @@ -299,6 +299,7 @@
> # define TCACHE_LARGE_BINS 12 /* Up to 4M chunks */
> # define TCACHE_MAX_BINS (TCACHE_SMALL_BINS + TCACHE_LARGE_BINS)
> # define MAX_TCACHE_SMALL_SIZE tidx2csize (TCACHE_SMALL_BINS-1)
> +# define TCACHE_DEFAULT_RETIRE_MULTIPLIER 5
>
> # define tidx2csize(idx) (((size_t) idx) * MALLOC_ALIGNMENT + MINSIZE)
> # define tidx2usize(idx) (((size_t) idx) * MALLOC_ALIGNMENT + MINSIZE - SIZE_SZ)
> @@ -1947,6 +1948,8 @@ struct malloc_par
> /* Maximum number of chunks to remove from the unsorted list, which
> aren't used to prefill the cache. */
> size_t tcache_unsorted_limit;
> + uintptr_t tcache_retire_size;
> + size_t tcache_trim;
> #endif
> };
>
> @@ -1979,7 +1982,12 @@ static struct malloc_par mp_ =
> .tcache_count = TCACHE_FILL_COUNT,
> .tcache_small_bins = TCACHE_SMALL_BINS,
> .tcache_max_bytes = MAX_TCACHE_SMALL_SIZE + 1,
> - .tcache_unsorted_limit = 0 /* No limit. */
> + .tcache_unsorted_limit = 0, /* No limit. */
> + /* The default does not enables large tcache support.
> + Capacity default is set to 0 to allow to set a default capacity of 10
> + times the tcache_max value when the tunable is set. */
> + .tcache_retire_size = 0,
> + .tcache_trim = 0
> #endif
> };
>
> @@ -3088,6 +3096,7 @@ typedef struct tcache_entry
> struct tcache_entry *next;
> /* This field exists to detect double frees. */
> uintptr_t key;
> + uintptr_t cached_at;
> } tcache_entry;
>
> /* There is one of these for each thread, which contains the
> @@ -3100,6 +3109,9 @@ typedef struct tcache_perthread_struct
> {
> uint16_t num_slots[TCACHE_MAX_BINS];
> tcache_entry *entries[TCACHE_MAX_BINS];
> + tcache_entry *return_to_arena;
> + uintptr_t alloc_size_state;
> + size_t trim_id;
> } tcache_perthread_struct;
>
> static const union
> @@ -3201,7 +3213,7 @@ large_csize2tidx(size_t nb)
> /* Caller must ensure that we know tc_idx is valid and there's room
> for more chunks. */
> static __always_inline void
> -tcache_put_n (mchunkptr chunk, size_t tc_idx, tcache_entry **ep, bool mangled)
> +tcache_put_n (mchunkptr chunk, tcache_entry **ep, uint16_t *num_slots, bool mangled)
> {
> tcache_entry *e = (tcache_entry *) chunk2mem (chunk);
>
> @@ -3219,14 +3231,14 @@ tcache_put_n (mchunkptr chunk, size_t tc_idx, tcache_entry **ep, bool mangled)
> e->next = PROTECT_PTR (&e->next, REVEAL_PTR (*ep));
> *ep = PROTECT_PTR (ep, e);
> }
> - --(tcache->num_slots[tc_idx]);
> + --(*num_slots);
> }
>
> /* Caller must ensure that we know tc_idx is valid and there's
> available chunks to remove. Removes chunk from the middle of the
> list. */
> static __always_inline void *
> -tcache_get_n (size_t tc_idx, tcache_entry **ep, bool mangled)
> +tcache_get_n (tcache_entry **ep, uint16_t *num_slots, bool mangled)
> {
> tcache_entry *e;
> if (!mangled)
> @@ -3242,7 +3254,7 @@ tcache_get_n (size_t tc_idx, tcache_entry **ep, bool mangled)
> else
> *ep = PROTECT_PTR (ep, REVEAL_PTR (e->next));
>
> - ++(tcache->num_slots[tc_idx]);
> + ++(*num_slots);
> e->key = 0;
> return (void *) e;
> }
> @@ -3250,25 +3262,38 @@ tcache_get_n (size_t tc_idx, tcache_entry **ep, bool mangled)
> static __always_inline void
> tcache_put (mchunkptr chunk, size_t tc_idx)
> {
> - tcache_put_n (chunk, tc_idx, &tcache->entries[tc_idx], false);
> + tcache_put_n (chunk, &tcache->entries[tc_idx], &tcache->num_slots[tc_idx], false);
> }
>
> /* Like the above, but removes from the head of the list. */
> static __always_inline void *
> tcache_get (size_t tc_idx)
> {
> - return tcache_get_n (tc_idx, &tcache->entries[tc_idx], false);
> + return tcache_get_n (&tcache->entries[tc_idx], &tcache->num_slots[tc_idx], false);
> }
>
> static __always_inline tcache_entry **
> -tcache_location_large (size_t nb, size_t tc_idx,
> +tcache_location_large (tcache_entry **tep, size_t nb, uint16_t *num_slots,
> bool *mangled, tcache_entry **demangled_ptr)
> {
> - tcache_entry **tep = &(tcache->entries[tc_idx]);
> - tcache_entry *te = *tep;
> + tcache_entry *te = *mangled == false ? *tep : REVEAL_PTR (*tep);
> while (te != NULL
> && __glibc_unlikely (chunksize (mem2chunk (te)) < nb))
> {
> + /* Move chunk to retire bin if chunk is beyond tcache capacity.
> + We should always do (alloc_size_state - cached_at) before attempting
> + any comparissons. This is a requirement to keep the comparisson valid
> + in the cases where one of the values has wraparound. */
> + if (tcache->alloc_size_state - te->cached_at > mp_.tcache_retire_size)
> + {
> + uint16_t not_used = 1;
> + tcache_entry *e = tcache_get_n (tep, num_slots, *mangled);
> + tcache_put_n (mem2chunk (e), &tcache->return_to_arena, ¬_used,
> + false);
> + te = *mangled == false ? *tep : REVEAL_PTR (*tep);
> + continue;
> + }
> +
> tep = & (te->next);
> te = REVEAL_PTR (te->next);
> *mangled = true;
> @@ -3278,15 +3303,79 @@ tcache_location_large (size_t nb, size_t tc_idx,
> return tep;
> }
>
> +/* Force for tcache_location_large to traverse the full bin by requesting a too
> + big of chunk. This allows it to mark old chunks to be released from tcache. */
> +#define RETIRE_OLD_CACHED_CHUNKS(ENTRY_PTR, TC_IDX, MANGLED_PTR) { \
> + tcache_entry *te; \
> + tcache_location_large (ENTRY_PTR, PTRDIFF_MAX, &tcache->num_slots[TC_IDX], \
> + MANGLED_PTR, &te); \
> +}
> +
> +static __always_inline void
> +tcache_large_cleanup (void)
> +{
> + if (!tcache_enabled ())
> + return;
> +
> + mstate current_locked_arena = NULL;
> + while (tcache->return_to_arena)
> + {
> + tcache_entry *e = tcache->return_to_arena;
> + if (__glibc_unlikely (misaligned_mem (e)))
> + malloc_printerr ("tcache_thread_shutdown(): "
> + "unaligned tcache chunk detected");
> + tcache->return_to_arena = REVEAL_PTR (e->next);
> + mchunkptr p = mem2chunk (e);
> + size_t size = chunksize (p);
> + mstate chunk_arena = arena_for_chunk (p);
> +
> + if (current_locked_arena != chunk_arena)
> + {
> + if (current_locked_arena != NULL)
> + __libc_lock_unlock (current_locked_arena->mutex);
> + __libc_lock_lock (chunk_arena->mutex);
> + current_locked_arena = chunk_arena;
> + }
> + _int_free_chunk (arena_for_chunk (p), p, size, 1);
> + }
> + if (current_locked_arena != NULL)
> + __libc_lock_unlock (current_locked_arena->mutex);
> +
> + /* Traverse all tcache bins and retire old cached chunks. */
> + if (mp_.tcache_trim - tcache->trim_id != 0)
> + {
> + tcache->trim_id += mp_.tcache_trim;
> + for (size_t tc_idx = TCACHE_SMALL_BINS; tc_idx < TCACHE_MAX_BINS; tc_idx++)
> + {
> + bool mangled = false;
> + tcache_entry **tep = &(tcache->entries[tc_idx]);
> + RETIRE_OLD_CACHED_CHUNKS (tep, tc_idx, &mangled);
> + }
> + }
> +}
> +
> static __always_inline void
> tcache_put_large (mchunkptr chunk, size_t tc_idx)
> {
> tcache_entry **entry;
> + tcache_entry *e = chunk2mem (chunk);
> bool mangled = false;
> tcache_entry *te;
> - entry = tcache_location_large (chunksize (chunk), tc_idx, &mangled, &te);
> + tcache_entry **tep = &(tcache->entries[tc_idx]);
> + entry = tcache_location_large (tep, chunksize (chunk),
> + &tcache->num_slots[tc_idx], &mangled, &te);
>
> - return tcache_put_n (chunk, tc_idx, entry, mangled);
> + e->cached_at = tcache->alloc_size_state;
> + tcache->alloc_size_state += chunksize (chunk);
> + tcache_put_n (chunk, entry, &tcache->num_slots[tc_idx], mangled);
> +
> + /* If the bin is full, execute tcache_location_large again from entry forward
> + with a much bigger size entry such that it would traverse the full bin. */
> + if (tcache->num_slots[tc_idx] <= 0)
> + {
> + RETIRE_OLD_CACHED_CHUNKS (entry, tc_idx, &mangled);
> + tcache_large_cleanup ();
> + }
> }
>
> static __always_inline void *
> @@ -3295,12 +3384,16 @@ tcache_get_large (size_t tc_idx, size_t nb)
> tcache_entry **entry;
> bool mangled = false;
> tcache_entry *te;
> - entry = tcache_location_large (nb, tc_idx, &mangled, &te);
> + tcache_entry **tep = &(tcache->entries[tc_idx]);
> + entry = tcache_location_large (tep, nb, &tcache->num_slots[tc_idx], &mangled, &te);
>
> if (te == NULL || nb != chunksize (mem2chunk (te)))
> - return NULL;
> + {
> + tcache_large_cleanup ();
> + return NULL;
> + }
>
> - return tcache_get_n (tc_idx, entry, mangled);
> + return tcache_get_n (entry, &tcache->num_slots[tc_idx], mangled);
> }
>
> static void tcache_init (mstate av);
> @@ -3312,7 +3405,7 @@ tcache_get_align (size_t nb, size_t alignment)
> {
> size_t tc_idx = csize2tidx (nb);
> if (__glibc_unlikely (tc_idx >= TCACHE_SMALL_BINS))
> - tc_idx = large_csize2tidx (nb);
> + tc_idx = large_csize2tidx (nb);
>
> /* The tcache itself isn't encoded, but the chain is. */
> tcache_entry **tep = & tcache->entries[tc_idx];
> @@ -3338,7 +3431,7 @@ tcache_get_align (size_t nb, size_t alignment)
> if (te != NULL
> && csize == nb
> && PTR_IS_ALIGNED (te, alignment))
> - return tag_new_usable (tcache_get_n (tc_idx, tep, mangled));
> + return tag_new_usable (tcache_get_n (tep, &tcache->num_slots[tc_idx], mangled));
> DIAG_POP_NEEDS_COMMENT;
> }
> return NULL;
> @@ -3358,7 +3451,7 @@ tcache_double_free_verify (tcache_entry *e)
> tmp;
> tmp = REVEAL_PTR (tmp->next), ++cnt)
> {
> - if (cnt >= mp_.tcache_count)
> + if (tc_idx < TCACHE_SMALL_BINS && cnt >= mp_.tcache_count)
> malloc_printerr ("free(): too many chunks detected in tcache");
> if (__glibc_unlikely (misaligned_mem (tmp)))
> malloc_printerr ("free(): unaligned chunk detected in tcache 2");
> @@ -3398,6 +3491,15 @@ tcache_thread_shutdown (void)
> tcache_tmp->entries[i] = REVEAL_PTR (e->next);
> __libc_free (e);
> }
> + while (tcache_tmp->return_to_arena)
> + {
> + tcache_entry *e = tcache_tmp->return_to_arena;
> + if (__glibc_unlikely (misaligned_mem (e)))
> + malloc_printerr ("tcache_thread_shutdown(): "
> + "unaligned tcache chunk detected");
> + tcache_tmp->return_to_arena = REVEAL_PTR (e->next);
> + __libc_free (e);
> + }
> }
>
> __libc_free (tcache_tmp);
> @@ -3430,6 +3532,9 @@ tcache_init (mstate av)
> memset (tcache, 0, bytes);
> for (int i = 0; i < TCACHE_MAX_BINS; i++)
> tcache->num_slots[i] = mp_.tcache_count;
> + tcache->alloc_size_state = 0;
> + tcache->return_to_arena = NULL;
> + tcache->trim_id = 0;
> }
> }
>
> @@ -3559,8 +3664,7 @@ __libc_free (void *mem)
> else
> {
> tc_idx = large_csize2tidx (size);
> - if (size >= MINSIZE
> - && __glibc_likely (tcache->num_slots[tc_idx] != 0))
> + if (size >= MINSIZE && tcache_enabled ())
> return tcache_put_large (p, tc_idx);
> }
>
> @@ -5281,6 +5385,9 @@ int
> __malloc_trim (size_t s)
> {
> int result = 0;
> +#if USE_TCACHE
> + mp_.tcache_trim += 1;
> +#endif
>
> mstate ar_ptr = &main_arena;
> do
> @@ -5569,11 +5676,13 @@ do_set_tcache_max (size_t value)
>
> LIBC_PROBE (memory_tunable_tcache_max_bytes, 2, value, mp_.tcache_max_bytes);
>
> - if (tc_idx < TCACHE_MAX_BINS)
> + if (tc_idx < TCACHE_MAX_BINS || mp_.tcache_retire_size == 0)
> {
> if (tc_idx < TCACHE_SMALL_BINS)
> mp_.tcache_small_bins = tc_idx + 1;
> mp_.tcache_max_bytes = nb + 1;
> + if (tc_idx >= TCACHE_SMALL_BINS && mp_.tcache_retire_size == 0)
> + mp_.tcache_retire_size = nb * TCACHE_DEFAULT_RETIRE_MULTIPLIER;
> return 1;
> }
>
> @@ -5599,6 +5708,14 @@ do_set_tcache_unsorted_limit (size_t value)
> mp_.tcache_unsorted_limit = value;
> return 1;
> }
> +
> +static __always_inline int
> +do_set_tcache_retire_size (size_t value)
> +{
> + LIBC_PROBE (memory_tunable_tcache_retire_size, 2, value, mp_.tcache_retire_size);
> + mp_.tcache_retire_size = value;
> + return 1;
> +}
> #endif
>
> static __always_inline int
More information about the Libc-alpha
mailing list