This is the mail archive of the
libc-help@sourceware.org
mailing list for the glibc project.
Adding priority inheritance to glibc malloc locking
- From: Ben Jackson <ben at ben dot com>
- To: libc-help at sourceware dot org
- Date: Thu, 16 Sep 2010 11:00:34 -0700
- Subject: Adding priority inheritance to glibc malloc locking
I work on a soft realtime system running embedded Linux. We ran into a
problem where high priority threads experienced unexpected delays. One
developer traced a problem to a std::queue operation and from there we
determined that malloc/free (primarily free, due to the way glibc's malloc
works) was showing very high latency jitter.
This was caused by priority inversions when there was malloc arena
contention. We were exacerbating the problem by having several threads
start and initialize at low priority before moving up to their normal
running priority. The thread starts all start over at the first arena
so we were experiencing many more contention events than you would for
long-running threads (though any producer/consumer type operation that
pumps memory between threads would also be subject to this if their
priorities differ).
lll_lock is implemented in terms of atomic operations and private futexes,
but it does not use the FUTEX_LOCK_PI idiom. Converting it to do so
would be problematic (I'm happy to discuss my reasoning here, but I will
omit it in this initial description). Instead my solution was to add
another extra-libs library to glibc that builds the regular malloc under
NOT_IN_libc which changing the mapping of __libc_lock to pthreads rather
than lll_lock. Then I enabled PI on the __libc_lock_init calls. This
library can be linked directly or forced with LD_PRELOAD for testing.
I'd welcome any comments on the general topic of PI locking in glibc and
malloc in particular. Also, I started from a state of complete ignorance
about the glibc build, so any review of my attached patch would be helpful.
Here's a description of the patch broken down by major area:
Add libptmalloc to the build:
* nptl/Makefile: Add libptmalloc to extra-libs and add malloc
components to libptmalloc-routines. Use vpath to ../malloc.
* Versions.def: Add libptmalloc stanza with versions referenced
in malloc/Versions, plus 2.2.5 due to nptl/shlib-versions (?)
* nptl/Versions: Add libptmalloc stanza with exports copied from
malloc/Versions.
* nptl/shlib-versions: Copy libpthread stanza to libptmalloc without
understanding it at all. Required to get symbols to bind properly
and versioned .so installation.
Make malloc build outside of libc:
* nptl/Makefile: Add _itoa and itoa-udigits to libptmalloc-routines,
required for mtrace. Could have also exported them from libc as
GLIBC_PRIVATE, but was unsure about namespace issues. Access with
vpath to ../stdio_common.
* nptl/Versions: Add __libc_message, __libc_freeres to libc GLIBC_PRIVATE.
* malloc/malloc.c: If IS_IN_libptmalloc don't remap open, mmap and friends.
* malloc/malloc.c: Change reference to __libc_argv[0] to __progname.
* malloc/mtrace.c: If IS_IN_libptmalloc don't remap setvbuf, fwrite.
* stdio-common/_itoa.c: Extend existing NOT_IN_libc INTUSE fencing
for _itoa_*_digits.
Set PTHREAD_PRIO_INHERIT for NOT_IN_libc __libc_lock_init:
* bits/libc-lock.h: Modify all calls pthread_mutex_init to include
pthread_mutexattr_setprotocol (&attr, PTHREAD_PRIO_INHERIT). This
only affects defines that are fenced for NOT_IN_libc already but it
does affect all extra-libs __libc_lock users (to include the new
libptmalloc, which is the main goal).
The symbol versioning of pthread_mutexattr_setprotocol may be wrong
because there is no __pthread_mutexattr_setprotocol and maybe I
should have made one?
Thanks,
--
Ben Jackson AD7GD
<ben@ben.com>
http://www.ben.com/
diff -r -u glibc-2.7.orig/malloc/malloc.c glibc-2.7/malloc/malloc.c
--- glibc-2.7.orig/malloc/malloc.c 2007-10-01 20:52:03.000000000 -0700
+++ glibc-2.7/malloc/malloc.c 2010-09-14 10:37:50.000000000 -0700
@@ -483,11 +483,13 @@
#define public_gET_STATe __malloc_get_state
#define public_sET_STATe __malloc_set_state
#define malloc_getpagesize __getpagesize()
+#ifndef IS_IN_libptmalloc
#define open __open
#define mmap __mmap
#define munmap __munmap
#define mremap __mremap
#define mprotect __mprotect
+#endif /* !IS_IN_libptmalloc */
#define MORECORE (*__morecore)
#define MORECORE_FAILURE 0
@@ -5872,7 +5874,7 @@
/* Helper code. */
-extern char **__libc_argv attribute_hidden;
+extern char *__progname;
static void
malloc_printerr(int action, const char *str, void *ptr)
@@ -5890,7 +5892,7 @@
__libc_message (action & 2,
"*** glibc detected *** %s: %s: 0x%s ***\n",
- __libc_argv[0] ?: "<unknown>", str, cp);
+ __progname ?: "<unknown>", str, cp);
}
else if (action & 2)
abort ();
diff -r -u glibc-2.7.orig/malloc/mtrace.c glibc-2.7/malloc/mtrace.c
--- glibc-2.7.orig/malloc/mtrace.c 2007-01-24 16:43:38.000000000 -0800
+++ glibc-2.7/malloc/mtrace.c 2010-09-10 15:32:42.000000000 -0700
@@ -38,8 +38,10 @@
# include <libc-internal.h>
# include <libio/iolibio.h>
+#ifndef IS_IN_libptmalloc
# define setvbuf(s, b, f, l) INTUSE(_IO_setvbuf) (s, b, f, l)
# define fwrite(buf, size, count, fp) _IO_fwrite (buf, size, count, fp)
+#endif /* !IS_IN_libptmalloc */
#endif
#ifndef attribute_hidden
diff -r -u glibc-2.7.orig/nptl/Makefile glibc-2.7/nptl/Makefile
--- glibc-2.7.orig/nptl/Makefile 2007-08-21 16:54:26.000000000 -0700
+++ glibc-2.7/nptl/Makefile 2010-09-14 10:40:51.000000000 -0700
@@ -23,10 +23,18 @@
headers := pthread.h semaphore.h bits/semaphore.h
-extra-libs := libpthread
+extra-libs := libpthread libptmalloc
extra-libs-others := $(extra-libs)
install-lib-ldscripts := libpthread.so
+libptmalloc-routines = malloc morecore mcheck mtrace _itoa itoa-udigits
+vpath m%.c ../malloc
+vpath _itoa.c ../stdio-common
+vpath itoa%.c ../stdio-common
+
+# Extra dependencies for ptmalloc
+$(foreach o,$(all-object-suffixes),$(objpfx)malloc$(o)): arena.c hooks.c
+
routines = alloca_cutoff forward libc-lowlevellock libc-cancellation
shared-only-routines = forward
@@ -516,6 +524,10 @@
$(common-objpfx)libc_nonshared.a \
$(if $(filter yes,$(elf)), $(elfobjdir)/ld.so)
+$(objpfx)libptmalloc.so: $(common-objpfx)libc.so \
+ $(common-objpfx)libc_nonshared.a \
+ $(if $(filter yes,$(elf)), $(elfobjdir)/ld.so)
+
# Make sure we link with the thread library.
ifeq ($(build-shared),yes)
$(addprefix $(objpfx), \
diff -r -u glibc-2.7.orig/nptl/shlib-versions glibc-2.7/nptl/shlib-versions
--- glibc-2.7.orig/nptl/shlib-versions 2007-01-10 15:24:02.000000000 -0800
+++ glibc-2.7/nptl/shlib-versions 2010-09-13 15:24:39.000000000 -0700
@@ -8,3 +8,14 @@
x86_64-.*-linux.* libpthread=0 GLIBC_2.2.5
powerpc64-.*-linux.* libpthread=0 GLIBC_2.3
.*-.*-linux.* libpthread=0
+
+mips.*-.*-linux.* libptmalloc=0 GLIBC_2.0 GLIBC_2.2
+sparc64.*-.*-linux.* libptmalloc=0 GLIBC_2.2
+sh.*-.*-linux.* libptmalloc=0 GLIBC_2.2
+ia64.*-.*-linux.* libptmalloc=0 GLIBC_2.2
+hppa.*-.*-linux.* libptmalloc=0 GLIBC_2.2
+s390x-.*-linux.* libptmalloc=0 GLIBC_2.2
+cris-.*-linux.* libptmalloc=0 GLIBC_2.2
+x86_64-.*-linux.* libptmalloc=0 GLIBC_2.2.5
+powerpc64-.*-linux.* libptmalloc=0 GLIBC_2.3
+.*-.*-linux.* libptmalloc=0
diff -r -u glibc-2.7.orig/nptl/sysdeps/pthread/bits/libc-lock.h glibc-2.7/nptl/sysdeps/pthread/bits/libc-lock.h
--- glibc-2.7.orig/nptl/sysdeps/pthread/bits/libc-lock.h 2007-10-10 08:59:42.000000000 -0700
+++ glibc-2.7/nptl/sysdeps/pthread/bits/libc-lock.h 2010-09-10 16:28:59.000000000 -0700
@@ -170,7 +170,16 @@
# define __libc_lock_init(NAME) ((NAME) = LLL_LOCK_INITIALIZER, 0)
#else
# define __libc_lock_init(NAME) \
- __libc_maybe_call (__pthread_mutex_init, (&(NAME), NULL), 0)
+ do { \
+ if (__pthread_mutex_init != NULL) \
+ { \
+ pthread_mutexattr_t __attr; \
+ __pthread_mutexattr_init (&__attr); \
+ pthread_mutexattr_setprotocol (&__attr, PTHREAD_PRIO_INHERIT); \
+ __pthread_mutex_init (&(NAME), &__attr); \
+ __pthread_mutexattr_destroy (&__attr); \
+ } \
+ } while (0)
#endif
#if defined SHARED && !defined NOT_IN_libc
/* ((NAME) = (__libc_rwlock_t) PTHREAD_RWLOCK_INITIALIZER, 0) is
@@ -194,6 +203,7 @@
pthread_mutexattr_t __attr; \
__pthread_mutexattr_init (&__attr); \
__pthread_mutexattr_settype (&__attr, PTHREAD_MUTEX_RECURSIVE_NP); \
+ pthread_mutexattr_setprotocol (&__attr, PTHREAD_PRIO_INHERIT); \
__pthread_mutex_init (&(NAME).mutex, &__attr); \
__pthread_mutexattr_destroy (&__attr); \
} \
@@ -207,6 +217,7 @@
pthread_mutexattr_t __attr; \
__pthread_mutexattr_init (&__attr); \
__pthread_mutexattr_settype (&__attr, PTHREAD_MUTEX_RECURSIVE_NP); \
+ pthread_mutexattr_setprotocol (&__attr, PTHREAD_PRIO_INHERIT); \
__pthread_mutex_init (&(NAME).mutex, &__attr); \
__pthread_mutexattr_destroy (&__attr); \
} \
@@ -539,6 +550,7 @@
weak_extern (BP_SYM (__pthread_mutexattr_init))
weak_extern (BP_SYM (__pthread_mutexattr_destroy))
weak_extern (BP_SYM (__pthread_mutexattr_settype))
+weak_extern (BP_SYM ( pthread_mutexattr_setprotocol))
weak_extern (BP_SYM (__pthread_rwlock_init))
weak_extern (BP_SYM (__pthread_rwlock_destroy))
weak_extern (BP_SYM (__pthread_rwlock_rdlock))
@@ -564,6 +576,7 @@
# pragma weak __pthread_mutexattr_init
# pragma weak __pthread_mutexattr_destroy
# pragma weak __pthread_mutexattr_settype
+# pragma weak pthread_mutexattr_setprotocol
# pragma weak __pthread_rwlock_destroy
# pragma weak __pthread_rwlock_rdlock
# pragma weak __pthread_rwlock_tryrdlock
diff -r -u glibc-2.7.orig/nptl/Versions glibc-2.7/nptl/Versions
--- glibc-2.7.orig/nptl/Versions 2006-02-28 01:36:05.000000000 -0800
+++ glibc-2.7/nptl/Versions 2010-09-13 13:08:41.000000000 -0700
@@ -29,6 +29,9 @@
GLIBC_PRIVATE {
# Internal libc interface to libpthread
__libc_dl_error_tsd;
+
+ # Expose to malloc
+ __libc_message; __libc_freeres;
}
}
@@ -246,3 +249,62 @@
__pthread_unwind;
}
}
+libptmalloc {
+ GLIBC_2.0 {
+ # global variables
+
+ # interface of malloc functions
+ __libc_calloc; __libc_free; __libc_mallinfo; __libc_malloc;
+ __libc_mallopt; __libc_memalign; __libc_pvalloc; __libc_realloc;
+ __libc_valloc;
+ __malloc_initialize_hook; __free_hook; __malloc_hook; __realloc_hook;
+ __memalign_hook; __after_morecore_hook;
+ __malloc_initialized; __default_morecore; __morecore;
+
+ # functions used in inline functions or macros
+
+ # variables in normal name space
+ mallwatch;
+
+ # c*
+ calloc; cfree;
+
+ # f*
+ free;
+
+ # m*
+ mallinfo; malloc; malloc_get_state; malloc_set_state; malloc_stats;
+ malloc_trim; malloc_usable_size; mallopt; memalign; mprobe; mtrace;
+ muntrace;
+
+ # p*
+ pvalloc;
+
+ # r*
+ realloc;
+
+ # t*
+ tr_break;
+
+ # v*
+ valloc;
+ }
+ GLIBC_2.1 {
+ # Special functions.
+ __libc_freeres;
+ }
+ GLIBC_2.2 {
+ # m*
+ mcheck_check_all; mcheck_pedantic;
+
+ # p*
+ posix_memalign;
+ }
+ GLIBC_PRIVATE {
+ # Internal startup hook for libpthread.
+ __libc_malloc_pthread_startup;
+
+ # Internal destructor hook for libpthread.
+ __libc_thread_freeres;
+ }
+}
diff -r -u glibc-2.7.orig/stdio-common/_itoa.c glibc-2.7/stdio-common/_itoa.c
--- glibc-2.7.orig/stdio-common/_itoa.c 2007-01-23 08:39:21.000000000 -0800
+++ glibc-2.7/stdio-common/_itoa.c 2010-09-10 15:46:34.000000000 -0700
@@ -169,6 +169,7 @@
extern const char _itoa_upper_digits_internal[] attribute_hidden;
+#ifndef NOT_IN_libc
char *
_itoa_word (unsigned long value, char *buflim,
unsigned int base, int upper_case)
@@ -203,6 +204,7 @@
return buflim;
}
#undef SPECIAL
+#endif /* !NOT_IN_libc */
#if LLONG_MAX != LONG_MAX
@@ -214,8 +216,14 @@
int upper_case;
{
const char *digits = (upper_case
+#if !defined NOT_IN_libc || defined IS_IN_rtld
? INTUSE(_itoa_upper_digits)
- : INTUSE(_itoa_lower_digits));
+ : INTUSE(_itoa_lower_digits)
+#else
+ ? _itoa_upper_digits
+ : _itoa_lower_digits
+#endif
+ );
const struct base_table_t *brec = &_itoa_base_table[base - 2];
switch (base)
diff -r -u glibc-2.7.orig/Versions.def glibc-2.7/Versions.def
--- glibc-2.7.orig/Versions.def 2007-09-15 15:32:08.000000000 -0700
+++ glibc-2.7/Versions.def 2010-09-13 15:37:18.000000000 -0700
@@ -87,6 +87,13 @@
GLIBC_2.6
GLIBC_PRIVATE
}
+libptmalloc {
+ GLIBC_2.0
+ GLIBC_2.1
+ GLIBC_2.2
+ GLIBC_2.2.5
+ GLIBC_PRIVATE
+}
libresolv {
GLIBC_2.0
GLIBC_2.2