[RFC v2] Annotate malloc conditions with glibc_likely.
Ondřej Bílka
neleai@seznam.cz
Sat Dec 21 21:07:00 GMT 2013
On Tue, Dec 10, 2013 at 09:54:31AM +0000, Will Newton wrote:
> On 9 December 2013 20:44, OndÅ™ej BÃlka <neleai@seznam.cz> wrote:
> > Hi,
> >
> > Another little optimization in malloc is annotate that fastbins are
> > likely case. These should be fast path and given that other path
> > slower it should pay off.
>
> In general I am not in favour of adding these types of annotations
> unless we can show they improve performance as they make code harder
> to read. They are also something of a blunt instrument - does "likely"
> mean will happen with 75% likelihood? 90%? 99%? I guess the results
> will vary with compiler and architecture as to whether or not the
> annotation helps or not.
>
I wrote a test that I send, I added more annotations that make quite a
difference when fastbin is full.
When fastbin is empty half of time then branch mistprediction starts being
a problem which this benchmark catches and there is only small benefit.
That is problem of implementation for small requests a better way is
on empty condition allocate 8 chunks at once which also causes better
cache locality.
A results without and with attached patch are following.
old
allocations in range 1-8: 118.689
allocations in range 2-16: 116.139
allocations in range 4-32: 122.606
allocations in range 8-64: 135.817
allocations in range 16-128: 160.102
allocations in range 32-256: 229.255
allocations in range 64-512: 281.434
allocations in range 128-1024: 400
allocations in range 256-2048: 621.67
allocations in range 512-4096: 1104.91
allocations in range 1024-8192: 2421.7
allocations in range 2048-16384: 4834.78
allocations in range 4096-32768: 9865.6
allocations in range 1-8: 118.25
allocations in range 2-16: 116.122
allocations in range 4-32: 121.751
allocations in range 8-64: 136.429
allocations in range 16-128: 160.606
allocations in range 32-256: 227.734
allocations in range 64-512: 281.236
allocations in range 128-1024: 394.561
allocations in range 256-2048: 609.688
allocations in range 512-4096: 1082.73
allocations in range 1024-8192: 2296.92
allocations in range 2048-16384: 4722.11
allocations in range 4096-32768: 9748.97
allocations in range 1-8: 118.287
allocations in range 2-16: 116.089
allocations in range 4-32: 121.822
allocations in range 8-64: 136.193
allocations in range 16-128: 160.722
allocations in range 32-256: 227.183
allocations in range 64-512: 280.237
allocations in range 128-1024: 393.878
allocations in range 256-2048: 607.415
allocations in range 512-4096: 1079.1
allocations in range 1024-8192: 2269.06
allocations in range 2048-16384: 4698.77
allocations in range 4096-32768: 9709.76
allocations in range 1-8: 118.371
allocations in range 2-16: 116.09
allocations in range 4-32: 121.652
allocations in range 8-64: 136.227
allocations in range 16-128: 160.741
allocations in range 32-256: 227.857
allocations in range 64-512: 280.756
allocations in range 128-1024: 394.021
allocations in range 256-2048: 609.028
allocations in range 512-4096: 1077.23
allocations in range 1024-8192: 2270.23
allocations in range 2048-16384: 4698.65
allocations in range 4096-32768: 9708.73
allocations in range 1-8: 118.061
allocations in range 2-16: 116.27
allocations in range 4-32: 121.893
allocations in range 8-64: 136.55
allocations in range 16-128: 160.553
allocations in range 32-256: 227.641
allocations in range 64-512: 280.883
allocations in range 128-1024: 394.121
allocations in range 256-2048: 607.112
allocations in range 512-4096: 1079.12
allocations in range 1024-8192: 2267.24
allocations in range 2048-16384: 4700.08
allocations in range 4096-32768: 9709.41
new
allocations in range 1-8: 107.242
allocations in range 2-16: 104.76
allocations in range 4-32: 115.903
allocations in range 8-64: 129.071
allocations in range 16-128: 157.852
allocations in range 32-256: 228.652
allocations in range 64-512: 280.58
allocations in range 128-1024: 406.251
allocations in range 256-2048: 638.335
allocations in range 512-4096: 1128.08
allocations in range 1024-8192: 2337.02
allocations in range 2048-16384: 4810.68
allocations in range 4096-32768: 9824.05
allocations in range 1-8: 106.484
allocations in range 2-16: 105.134
allocations in range 4-32: 113.38
allocations in range 8-64: 129.916
allocations in range 16-128: 158.132
allocations in range 32-256: 226.758
allocations in range 64-512: 280.586
allocations in range 128-1024: 400.147
allocations in range 256-2048: 627.112
allocations in range 512-4096: 1096.87
allocations in range 1024-8192: 2278.98
allocations in range 2048-16384: 4695.18
allocations in range 4096-32768: 9722.37
allocations in range 1-8: 106.837
allocations in range 2-16: 104.966
allocations in range 4-32: 114.843
allocations in range 8-64: 138.233
allocations in range 16-128: 162.073
allocations in range 32-256: 227.041
allocations in range 64-512: 297.998
allocations in range 128-1024: 408.122
allocations in range 256-2048: 645.366
allocations in range 512-4096: 1130.57
allocations in range 1024-8192: 2326.98
allocations in range 2048-16384: 4824.57
allocations in range 4096-32768: 9800.23
allocations in range 1-8: 106.691
allocations in range 2-16: 105.315
allocations in range 4-32: 113.97
allocations in range 8-64: 130.445
allocations in range 16-128: 156.499
allocations in range 32-256: 226.695
allocations in range 64-512: 280.039
allocations in range 128-1024: 400.835
allocations in range 256-2048: 625.75
allocations in range 512-4096: 1093.93
allocations in range 1024-8192: 2280.61
allocations in range 2048-16384: 4695.94
allocations in range 4096-32768: 9677.53
allocations in range 1-8: 106.672
allocations in range 2-16: 105.131
allocations in range 4-32: 113.929
allocations in range 8-64: 130.109
allocations in range 16-128: 157.513
allocations in range 32-256: 226.601
allocations in range 64-512: 280.75
allocations in range 128-1024: 400.561
allocations in range 256-2048: 627.846
allocations in range 512-4096: 1095.34
allocations in range 1024-8192: 2280.27
allocations in range 2048-16384: 4693.53
allocations in range 4096-32768: 9678.55
-------------- next part --------------
diff --git a/malloc/arena.c b/malloc/arena.c
index 9d49f93..9c13daa 100644
--- a/malloc/arena.c
+++ b/malloc/arena.c
@@ -113,7 +113,7 @@ int __malloc_initialized = -1;
} while(0)
# define arena_lock(ptr, size) do { \
- if(ptr) \
+ if(__glibc_likely (ptr)) \
(void)mutex_lock(&ptr->mutex); \
else \
ptr = arena_get2(ptr, (size), NULL); \
diff --git a/malloc/malloc.c b/malloc/malloc.c
index b1668b5..374eb09 100644
--- a/malloc/malloc.c
+++ b/malloc/malloc.c
@@ -1849,14 +1849,14 @@ static int check_action = DEFAULT_CHECK_ACTION;
static int perturb_byte;
-static inline void
+static inline __attribute__ ((always_inline)) void
alloc_perturb (char *p, size_t n)
{
if (__glibc_unlikely (perturb_byte))
memset (p, perturb_byte ^ 0xff, n);
}
-static inline void
+static inline __attribute__ ((always_inline)) void
free_perturb (char *p, size_t n)
{
if (__glibc_unlikely (perturb_byte))
@@ -2835,10 +2835,10 @@ __libc_malloc(size_t bytes)
arena_lookup(ar_ptr);
arena_lock(ar_ptr, bytes);
- if(!ar_ptr)
+ if(__glibc_unlikely (!ar_ptr))
return 0;
victim = _int_malloc(ar_ptr, bytes);
- if(!victim) {
+ if(__glibc_unlikely (!victim)) {
LIBC_PROBE (memory_malloc_retry, 1, bytes);
ar_ptr = arena_get_retry(ar_ptr, bytes);
if (__builtin_expect(ar_ptr != NULL, 1)) {
@@ -2866,12 +2866,12 @@ __libc_free(void* mem)
return;
}
- if (mem == 0) /* free(0) has no effect */
+ if (__glibc_unlikely (mem == 0)) /* free(0) has no effect */
return;
p = mem2chunk(mem);
- if (chunk_is_mmapped(p)) /* release mmapped memory. */
+ if (__glibc_unlikely (chunk_is_mmapped(p))) /* release mmapped memory. */
{
/* see if the dynamic brk/mmap threshold needs adjusting */
if (!mp_.no_dyn_threshold
@@ -3251,19 +3251,21 @@ _int_malloc(mstate av, size_t bytes)
can try it without checking, which saves some time on this fast path.
*/
- if ((unsigned long)(nb) <= (unsigned long)(get_max_fast ())) {
+ if (__glibc_likely ((unsigned long)(nb) <= (unsigned long)(get_max_fast ())))
+ {
idx = fastbin_index(nb);
mfastbinptr* fb = &fastbin (av, idx);
mchunkptr pp = *fb;
do
{
victim = pp;
- if (victim == NULL)
+ if (__glibc_unlikely (victim == NULL))
break;
}
- while ((pp = catomic_compare_and_exchange_val_acq (fb, victim->fd, victim))
+ while (__glibc_unlikely (
+ (pp = catomic_compare_and_exchange_val_acq (fb, victim->fd, victim)))
!= victim);
- if (victim != 0) {
+ if (__glibc_likely (victim != 0)) {
if (__builtin_expect (fastbin_index (chunksize (victim)) != idx, 0))
{
errstr = "malloc(): memory corruption (fast)";
@@ -3741,7 +3743,7 @@ _int_free(mstate av, mchunkptr p, int have_lock)
and used quickly in malloc.
*/
- if ((unsigned long)(size) <= (unsigned long)(get_max_fast ())
+ if (__glibc_likely ((unsigned long)(size) <= (unsigned long)(get_max_fast ()))
#if TRIM_FASTBINS
/*
@@ -3795,13 +3797,14 @@ _int_free(mstate av, mchunkptr p, int have_lock)
errstr = "double free or corruption (fasttop)";
goto errout;
}
- if (old != NULL)
+ if (__glibc_likely (old != NULL))
old_idx = fastbin_index(chunksize(old));
p->fd = fd = old;
}
- while ((old = catomic_compare_and_exchange_val_rel (fb, p, fd)) != fd);
+ while (__glibc_unlikely (
+ (old = catomic_compare_and_exchange_val_rel (fb, p, fd)) != fd));
- if (fd != NULL && __builtin_expect (old_idx != idx, 0))
+ if (__glibc_likely (fd != NULL) && __builtin_expect (old_idx != idx, 0))
{
errstr = "invalid fastbin entry (free)";
goto errout;
--
1.7.10.4
More information about the Libc-alpha
mailing list