From 97e38514ba24dfacb5a20036c0112b401de7fcc7 Mon Sep 17 00:00:00 2001 From: Darren Hart Date: Mon, 10 May 2010 20:35:44 -0400 Subject: [PATCH 1/4] pi-condvars: add protocol support to pthread_condattr_t When using a PTHREAD_PRIO_INHERIT mutex with a condvar, the pthread_cond* calls can still cause an unbounded priority inversion via the internal condvar lock. The POSIX specification doesn't provide a mechanism to specify the protocol of the condvar. We would like to do this at runtime, but unfortunately it is legal to call pthread_cond_signal() or pthread_cond_broadcast() without first waiting on the lock, so the mutex type may not be known the first time the condvar is used. A new API, pthread_condattr_setprotocol_np() and pthread_condattr_getprotocol_np() allow the user to create a PTHREAD_PRIO_INHERIT condvar. This uses a PTHREAD_PRIO_INHERIT mutex for the internal condvar lock, eliminating the potential for hitting an unbounded priority inversion on that lock. Signed-off-by: Dinakar Guniguntala Signed-off-by: Darren Hart --- Versions.def | 1 + nptl/Makefile | 1 + nptl/Versions | 4 + nptl/pthread_cond_broadcast.c | 59 ++++++++----- nptl/pthread_cond_init.c | 23 +++++- nptl/pthread_cond_signal.c | 37 +++++++-- nptl/pthread_cond_timedwait.c | 40 +++++++-- nptl/pthread_cond_wait.c | 89 +++++++++++++++++--- nptl/pthread_condattr_getclock.c | 7 +- nptl/pthread_condattr_getprotocol_np.c | 34 ++++++++ nptl/pthread_condattr_setclock.c | 6 +- nptl/pthread_condattr_setprotocol_np.c | 39 +++++++++ nptl/sysdeps/pthread/cond-lock.h | 58 +++++++++++++ nptl/sysdeps/pthread/pthread.h | 12 +++ nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h | 88 +++++++++++++++++++- nptl/sysdeps/unix/sysv/linux/internaltypes.h | 30 +++++-- nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h | 40 +++++++++ 17 files changed, 502 insertions(+), 66 deletions(-) create mode 100644 nptl/pthread_condattr_getprotocol_np.c create mode 100644 nptl/pthread_condattr_setprotocol_np.c create mode 100644 nptl/sysdeps/pthread/cond-lock.h diff --git a/Versions.def b/Versions.def index eab006b..26535b5 100644 --- a/Versions.def +++ b/Versions.def @@ -92,6 +92,7 @@ libpthread { GLIBC_2.6 GLIBC_2.11 GLIBC_2.12 + GLIBC_2.13 GLIBC_PRIVATE } libresolv { diff --git a/nptl/Makefile b/nptl/Makefile index 982db8e..6d1913e 100644 --- a/nptl/Makefile +++ b/nptl/Makefile @@ -75,6 +75,7 @@ libpthread-routines = nptl-init vars events version \ old_pthread_cond_signal old_pthread_cond_broadcast \ pthread_condattr_init pthread_condattr_destroy \ pthread_condattr_getpshared pthread_condattr_setpshared \ + pthread_condattr_getprotocol_np pthread_condattr_setprotocol_np \ pthread_condattr_getclock pthread_condattr_setclock \ pthread_spin_init pthread_spin_destroy \ pthread_spin_lock pthread_spin_trylock \ diff --git a/nptl/Versions b/nptl/Versions index f74941f..49a89e8 100644 --- a/nptl/Versions +++ b/nptl/Versions @@ -251,6 +251,10 @@ libpthread { pthread_setname_np; pthread_getname_np; }; + GLIBC_2.13 { + pthread_condattr_getprotocol_np; pthread_condattr_setprotocol_np; + } + GLIBC_PRIVATE { __pthread_initialize_minimal; __pthread_clock_gettime; __pthread_clock_settime; diff --git a/nptl/pthread_cond_broadcast.c b/nptl/pthread_cond_broadcast.c index 22523c2..d5d4348 100644 --- a/nptl/pthread_cond_broadcast.c +++ b/nptl/pthread_cond_broadcast.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2003, 2004, 2006, 2007 Free Software Foundation, Inc. +/* Copyright (C) 2003, 2004, 2006, 2007, 2010 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Martin Schwidefsky , 2003. @@ -27,6 +27,8 @@ #include #include +#include "cond-lock.h" + int __pthread_cond_broadcast (cond) @@ -34,8 +36,9 @@ __pthread_cond_broadcast (cond) { int pshared = (cond->__data.__mutex == (void *) ~0l) ? LLL_SHARED : LLL_PRIVATE; + /* Make sure we are alone. */ - lll_lock (cond->__data.__lock, pshared); + cond_lock(cond, pshared); /* Are there any waiters to be woken? */ if (cond->__data.__total_seq > cond->__data.__wakeup_seq) @@ -45,36 +48,48 @@ __pthread_cond_broadcast (cond) cond->__data.__woken_seq = cond->__data.__total_seq; cond->__data.__futex = (unsigned int) cond->__data.__total_seq * 2; int futex_val = cond->__data.__futex; + pthread_mutex_t *mut = (pthread_mutex_t *) cond->__data.__mutex; + int err; /* Signal that a broadcast happened. */ ++cond->__data.__broadcast_seq; /* We are done. */ - lll_unlock (cond->__data.__lock, pshared); + cond_unlock(cond, pshared); /* Do not use requeue for pshared condvars. */ if (cond->__data.__mutex == (void *) ~0l) goto wake_all; - /* Wake everybody. */ - pthread_mutex_t *mut = (pthread_mutex_t *) cond->__data.__mutex; - - /* XXX: Kernel so far doesn't support requeue to PI futex. */ - /* XXX: Kernel so far can only requeue to the same type of futex, - in this case private (we don't requeue for pshared condvars). */ + /* We don't requeue for pshared condvars */ if (__builtin_expect (mut->__data.__kind - & (PTHREAD_MUTEX_PRIO_INHERIT_NP - | PTHREAD_MUTEX_PSHARED_BIT), 0)) - goto wake_all; - - /* lll_futex_requeue returns 0 for success and non-zero - for errors. */ - if (__builtin_expect (lll_futex_requeue (&cond->__data.__futex, 1, - INT_MAX, &mut->__data.__lock, - futex_val, LLL_PRIVATE), 0)) + & PTHREAD_MUTEX_PSHARED_BIT, 0)) + goto wake_all; + + if (__builtin_expect (mut->__data.__kind + & PTHREAD_MUTEX_PRIO_INHERIT_NP, 0)) + { + /* lll_futex_requeue_pi returns the number of tasks requeue'd on + success and negative for errors. */ + err = lll_futex_requeue_pi (&cond->__data.__futex, 1, INT_MAX, + &mut->__data.__lock, futex_val, + LLL_PRIVATE); + /* The requeue_pi functionality is not available. */ + if (__builtin_expect (err == -ENOSYS, 0)) + goto wake_all; + } + else { - /* The requeue functionality is not available. */ - wake_all: - lll_futex_wake (&cond->__data.__futex, INT_MAX, pshared); + /* lll_futex_requeue_pi returns the number of tasks requeue'd on + success and negative for errors. */ + err = lll_futex_requeue (&cond->__data.__futex, 1, INT_MAX, + &mut->__data.__lock, futex_val, + LLL_PRIVATE); + if (__builtin_expect (err == -ENOSYS, 0)) + { + /* The requeue functionality is not available. */ + wake_all: + lll_futex_wake (&cond->__data.__futex, INT_MAX, pshared); + } } /* That's all. */ @@ -82,7 +97,7 @@ __pthread_cond_broadcast (cond) } /* We are done. */ - lll_unlock (cond->__data.__lock, pshared); + cond_unlock(cond, pshared); return 0; } diff --git a/nptl/pthread_cond_init.c b/nptl/pthread_cond_init.c index 65c01b1..33bb6bb 100644 --- a/nptl/pthread_cond_init.c +++ b/nptl/pthread_cond_init.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2004, 2005, 2007, 2008 +/* Copyright (C) 2002, 2003, 2004, 2005, 2007, 2008, 2010 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper , 2002. @@ -32,9 +32,26 @@ __pthread_cond_init (cond, cond_attr) cond->__data.__lock = LLL_LOCK_INITIALIZER; cond->__data.__futex = 0; cond->__data.__nwaiters = (icond_attr != NULL - ? ((icond_attr->value >> 1) - & ((1 << COND_NWAITERS_SHIFT) - 1)) + ? ((icond_attr->value >> CONDATTR_CLOCKID_SHIFT) + & ((1 << COND_PROTOCOL_SHIFT) - 1)) : CLOCK_REALTIME); + if (icond_attr != NULL) + { + switch (icond_attr->value & CONDATTR_PROTOCOL_MASK) + { + case PTHREAD_PRIO_INHERIT << CONDATTR_PROTOCOL_SHIFT: + cond->__data.__nwaiters |= COND_PRIO_INHERIT; + break; + + case PTHREAD_PRIO_PROTECT << CONDATTR_PROTOCOL_SHIFT: + cond->__data.__nwaiters |= COND_PRIO_PROTECT; + break; + + default: + break; + } + } + cond->__data.__total_seq = 0; cond->__data.__wakeup_seq = 0; cond->__data.__woken_seq = 0; diff --git a/nptl/pthread_cond_signal.c b/nptl/pthread_cond_signal.c index 023bbb5..53e883d 100644 --- a/nptl/pthread_cond_signal.c +++ b/nptl/pthread_cond_signal.c @@ -27,16 +27,19 @@ #include #include +#include "cond-lock.h" + int __pthread_cond_signal (cond) pthread_cond_t *cond; { - int pshared = (cond->__data.__mutex == (void *) ~0l) - ? LLL_SHARED : LLL_PRIVATE; + int err, futex_val; + pthread_mutex_t *mutex = (pthread_mutex_t *) cond->__data.__mutex; + int pshared = (mutex == (void *) ~0l) ? LLL_SHARED : LLL_PRIVATE; /* Make sure we are alone. */ - lll_lock (cond->__data.__lock, pshared); + cond_lock(cond, pshared); /* Are there any waiters to be woken? */ if (cond->__data.__total_seq > cond->__data.__wakeup_seq) @@ -44,18 +47,38 @@ __pthread_cond_signal (cond) /* Yes. Mark one of them as woken. */ ++cond->__data.__wakeup_seq; ++cond->__data.__futex; + futex_val = cond->__data.__futex; + pthread_mutex_t *mut = (pthread_mutex_t *) cond->__data.__mutex; + /* We dont do requeue-pi for pshared mutexes */ + if (pshared == LLL_PRIVATE && + mut->__data.__kind & PTHREAD_MUTEX_PRIO_INHERIT_NP) + { + /* lll_futex_requeue_pi returns the number of tasks requeue'd on + success and negative for errors. */ + err = lll_futex_requeue_pi (&cond->__data.__futex, 1, 0, + &mut->__data.__lock, futex_val, + pshared); + /* The requeue_pi functionality is not available. */ + if (__builtin_expect (err != -ENOSYS, 0)) + goto done; + } +#if 0 + /* This is not needed for the x86_64 or i686 arches */ /* Wake one. */ - if (! __builtin_expect (lll_futex_wake_unlock (&cond->__data.__futex, 1, - 1, &cond->__data.__lock, - pshared), 0)) + else if (! __builtin_expect (lll_futex_wake_unlock (&cond->__data.__futex, + 1, 1, + &cond->__data.__lock, + pshared), 0)) return 0; +#endif lll_futex_wake (&cond->__data.__futex, 1, pshared); } +done: /* We are done. */ - lll_unlock (cond->__data.__lock, pshared); + cond_unlock(cond, pshared); return 0; } diff --git a/nptl/pthread_cond_timedwait.c b/nptl/pthread_cond_timedwait.c index 7278ec4..648162a 100644 --- a/nptl/pthread_cond_timedwait.c +++ b/nptl/pthread_cond_timedwait.c @@ -27,6 +27,8 @@ #include +#include "cond-lock.h" + #ifndef HAVE_CLOCK_GETTIME_VSYSCALL # undef INTERNAL_VSYSCALL # define INTERNAL_VSYSCALL INTERNAL_SYSCALL @@ -57,6 +59,7 @@ __pthread_cond_timedwait (cond, mutex, abstime) struct _pthread_cleanup_buffer buffer; struct _condvar_cleanup_buffer cbuffer; int result = 0; + int pi_requeued; /* Catch invalid parameters. */ if (abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000) @@ -65,14 +68,14 @@ __pthread_cond_timedwait (cond, mutex, abstime) int pshared = (cond->__data.__mutex == (void *) ~0l) ? LLL_SHARED : LLL_PRIVATE; - /* Make sure we are along. */ - lll_lock (cond->__data.__lock, pshared); + /* Make sure we are alone. */ + cond_lock(cond, pshared); /* Now we can release the mutex. */ int err = __pthread_mutex_unlock_usercnt (mutex, 0); if (err) { - lll_unlock (cond->__data.__lock, pshared); + cond_unlock(cond, pshared); return err; } @@ -105,6 +108,7 @@ __pthread_cond_timedwait (cond, mutex, abstime) while (1) { + pi_requeued = 0; struct timespec rt; { #ifdef __NR_clock_gettime @@ -112,7 +116,7 @@ __pthread_cond_timedwait (cond, mutex, abstime) int ret; ret = INTERNAL_VSYSCALL (clock_gettime, err, 2, (cond->__data.__nwaiters - & ((1 << COND_NWAITERS_SHIFT) - 1)), + & ((1 << COND_PROTOCOL_SHIFT) - 1)), &rt); # ifndef __ASSUME_POSIX_TIMERS if (__builtin_expect (INTERNAL_SYSCALL_ERROR_P (ret, err), 0)) @@ -158,20 +162,34 @@ __pthread_cond_timedwait (cond, mutex, abstime) unsigned int futex_val = cond->__data.__futex; /* Prepare to wait. Release the condvar futex. */ - lll_unlock (cond->__data.__lock, pshared); + cond_unlock(cond, pshared); /* Enable asynchronous cancellation. Required by the standard. */ cbuffer.oldtype = __pthread_enable_asynccancel (); + if (mutex->__data.__kind & PTHREAD_MUTEX_PRIO_INHERIT_NP) + { + /* Try requeueing to the PI mutex, if no support in the kernel + try the non-requeue syscall. */ + err = lll_futex_timed_wait_requeue_pi (cond, futex_val, abstime, + &mutex->__data.__lock, + pshared); + if (__builtin_expect (!err, 1)) + pi_requeued = 1; + if (__builtin_expect (err != -ENOSYS, 0)) + goto woken; + } + /* Wait until woken by signal or broadcast. */ err = lll_futex_timed_wait (&cond->__data.__futex, futex_val, &rt, pshared); +woken: /* Disable asynchronous cancellation. */ __pthread_disable_asynccancel (cbuffer.oldtype); /* We are going to look at shared data again, so get the lock. */ - lll_lock (cond->__data.__lock, pshared); + cond_lock(cond, pshared); /* If a broadcast happened, we are done. */ if (cbuffer.bc_seq != cond->__data.__broadcast_seq) @@ -211,13 +229,17 @@ __pthread_cond_timedwait (cond, mutex, abstime) lll_futex_wake (&cond->__data.__nwaiters, 1, pshared); /* We are done with the condvar. */ - lll_unlock (cond->__data.__lock, pshared); + cond_unlock(cond, pshared); /* The cancellation handling is back to normal, remove the handler. */ __pthread_cleanup_pop (&buffer, 0); - /* Get the mutex before returning. */ - err = __pthread_mutex_cond_lock (mutex); + /* Get the mutex before returning. If the requeue_pi call above was successful, + the lock is already held in the kernel, so just return to the application. */ + if (pi_requeued) + __pthread_mutex_cond_lock_adjust (mutex); + else + err = __pthread_mutex_cond_lock (mutex); return err ?: result; } diff --git a/nptl/pthread_cond_wait.c b/nptl/pthread_cond_wait.c index 670fba5..4afa028 100644 --- a/nptl/pthread_cond_wait.c +++ b/nptl/pthread_cond_wait.c @@ -17,15 +17,19 @@ Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. */ +#include #include #include #include #include +#include #include #include #include +#include "cond-lock.h" + struct _condvar_cleanup_buffer { @@ -38,16 +42,55 @@ struct _condvar_cleanup_buffer void __attribute__ ((visibility ("hidden"))) +lll_pi_lock(int *futexp, int private) +{ + pid_t id = THREAD_GETMEM (THREAD_SELF, tid); + int newval = id; + int ret; + + newval |= FUTEX_WAITERS; + ret = atomic_compare_and_exchange_val_acq (futexp, newval, 0); + + if (ret != 0) + { + /* The mutex is locked. The kernel will now take care of + everything. */ + INTERNAL_SYSCALL_DECL (__err); + int e = INTERNAL_SYSCALL (futex, __err, 4, futexp, + __lll_private_flag (FUTEX_LOCK_PI, private), + 1, 0); + } +} + + +void +__attribute__ ((visibility ("hidden"))) +lll_pi_unlock(int *futexp, int private) +{ + + if ((*futexp & FUTEX_WAITERS) != 0 + || atomic_compare_and_exchange_bool_acq (futexp, 0, + THREAD_GETMEM (THREAD_SELF, + tid))) + { + INTERNAL_SYSCALL_DECL (__err); + INTERNAL_SYSCALL (futex, __err, 2, futexp, + __lll_private_flag (FUTEX_UNLOCK_PI, private)); + } +} + + +void +__attribute__ ((visibility ("hidden"))) __condvar_cleanup (void *arg) { struct _condvar_cleanup_buffer *cbuffer = (struct _condvar_cleanup_buffer *) arg; unsigned int destroying; - int pshared = (cbuffer->cond->__data.__mutex == (void *) ~0l) - ? LLL_SHARED : LLL_PRIVATE; + int pshared = (cbuffer->mutex == (void *) ~0l) ? LLL_SHARED : LLL_PRIVATE; /* We are going to modify shared data. */ - lll_lock (cbuffer->cond->__data.__lock, pshared); + cond_lock(cbuffer->cond, pshared); if (cbuffer->bc_seq == cbuffer->cond->__data.__broadcast_seq) { @@ -78,7 +121,7 @@ __condvar_cleanup (void *arg) } /* We are done. */ - lll_unlock (cbuffer->cond->__data.__lock, pshared); + cond_unlock(cbuffer->cond, pshared); /* Wake everybody to make sure no condvar signal gets lost. */ if (! destroying) @@ -97,18 +140,19 @@ __pthread_cond_wait (cond, mutex) { struct _pthread_cleanup_buffer buffer; struct _condvar_cleanup_buffer cbuffer; - int err; int pshared = (cond->__data.__mutex == (void *) ~0l) ? LLL_SHARED : LLL_PRIVATE; + int pi_requeued; + int err; /* Make sure we are along. */ - lll_lock (cond->__data.__lock, pshared); + cond_lock(cond, pshared); /* Now we can release the mutex. */ err = __pthread_mutex_unlock_usercnt (mutex, 0); if (__builtin_expect (err, 0)) { - lll_unlock (cond->__data.__lock, pshared); + cond_unlock(cond, pshared); return err; } @@ -142,21 +186,35 @@ __pthread_cond_wait (cond, mutex) do { unsigned int futex_val = cond->__data.__futex; + pi_requeued = 0; /* Prepare to wait. Release the condvar futex. */ - lll_unlock (cond->__data.__lock, pshared); + cond_unlock(cond, pshared); /* Enable asynchronous cancellation. Required by the standard. */ cbuffer.oldtype = __pthread_enable_asynccancel (); + if (mutex->__data.__kind & PTHREAD_MUTEX_PRIO_INHERIT_NP) + { + /* Try requeueing to the PI mutex, if no support in the kernel + try the non-requeue syscall. */ + err = lll_futex_wait_requeue_pi (cond, futex_val, + &mutex->__data.__lock, pshared); + if (__builtin_expect (!err, 1)) + pi_requeued = 1; + if (__builtin_expect (err != -ENOSYS, 0)) + goto woken; + } + /* Wait until woken by signal or broadcast. */ lll_futex_wait (&cond->__data.__futex, futex_val, pshared); +woken: /* Disable asynchronous cancellation. */ __pthread_disable_asynccancel (cbuffer.oldtype); /* We are going to look at shared data again, so get the lock. */ - lll_lock (cond->__data.__lock, pshared); + cond_lock(cond, pshared); /* If a broadcast happened, we are done. */ if (cbuffer.bc_seq != cond->__data.__broadcast_seq) @@ -182,13 +240,20 @@ __pthread_cond_wait (cond, mutex) lll_futex_wake (&cond->__data.__nwaiters, 1, pshared); /* We are done with the condvar. */ - lll_unlock (cond->__data.__lock, pshared); + cond_unlock(cond, pshared); /* The cancellation handling is back to normal, remove the handler. */ __pthread_cleanup_pop (&buffer, 0); - /* Get the mutex before returning. */ - return __pthread_mutex_cond_lock (mutex); + /* Get the mutex before returning. If the requeue_pi call above was successful, + the lock is already held in the kernel, so just return 0 to application. */ + if (pi_requeued) + { + __pthread_mutex_cond_lock_adjust (mutex); + return 0; + } + else + return __pthread_mutex_cond_lock (mutex); } versioned_symbol (libpthread, __pthread_cond_wait, pthread_cond_wait, diff --git a/nptl/pthread_condattr_getclock.c b/nptl/pthread_condattr_getclock.c index 3eedeb1..9f034b3 100644 --- a/nptl/pthread_condattr_getclock.c +++ b/nptl/pthread_condattr_getclock.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2003, 2004, 2007 Free Software Foundation, Inc. +/* Copyright (C) 2003,2004,2007,2010 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper , 2003. @@ -25,7 +25,8 @@ pthread_condattr_getclock (attr, clock_id) const pthread_condattr_t *attr; clockid_t *clock_id; { - *clock_id = (((((const struct pthread_condattr *) attr)->value) >> 1) - & ((1 << COND_NWAITERS_SHIFT) - 1)); + *clock_id = (((((const struct pthread_condattr *) attr)->value) + >> CONDATTR_CLOCKID_SHIFT) + & ((1 << COND_PROTOCOL_SHIFT) - 1)); return 0; } diff --git a/nptl/pthread_condattr_getprotocol_np.c b/nptl/pthread_condattr_getprotocol_np.c new file mode 100644 index 0000000..6281f2a --- /dev/null +++ b/nptl/pthread_condattr_getprotocol_np.c @@ -0,0 +1,34 @@ +/* Copyright (C) 2010 Free Software Foundation, Inc. + This file is part of the GNU C Library. + Contributed by Dinakar Guniguntala , 2010. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, write to the Free + Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA + 02111-1307 USA. */ + +#include "pthreadP.h" + + +int +pthread_condattr_getprotocol_np (attr, protocol) + const pthread_condattr_t *attr; + int *protocol; +{ + *protocol = ((const struct pthread_condattr *) attr)->value; + + *protocol = ((*protocol & CONDATTR_PROTOCOL_MASK) + >> CONDATTR_PROTOCOL_SHIFT); + + return 0; +} diff --git a/nptl/pthread_condattr_setclock.c b/nptl/pthread_condattr_setclock.c index 5c54f76..97e9595 100644 --- a/nptl/pthread_condattr_setclock.c +++ b/nptl/pthread_condattr_setclock.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2003, 2004, 2007, 2008 Free Software Foundation, Inc. +/* Copyright (C) 2003,2004,2007,2008,2010 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper , 2003. @@ -62,11 +62,11 @@ pthread_condattr_setclock (attr, clock_id) return EINVAL; /* Make sure the value fits in the bits we reserved. */ - assert (clock_id < (1 << COND_NWAITERS_SHIFT)); + assert (clock_id < (1 << COND_PROTOCOL_SHIFT)); int *valuep = &((struct pthread_condattr *) attr)->value; - *valuep = ((*valuep & ~(((1 << COND_NWAITERS_SHIFT) - 1) << 1)) + *valuep = ((*valuep & ~(((1 << COND_PROTOCOL_SHIFT) - 1) << 1)) | (clock_id << 1)); return 0; diff --git a/nptl/pthread_condattr_setprotocol_np.c b/nptl/pthread_condattr_setprotocol_np.c new file mode 100644 index 0000000..5859a27 --- /dev/null +++ b/nptl/pthread_condattr_setprotocol_np.c @@ -0,0 +1,39 @@ +/* Copyright (C) 2010 Free Software Foundation, Inc. + This file is part of the GNU C Library. + Contributed by Dinakar Guniguntala , 2010. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, write to the Free + Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA + 02111-1307 USA. */ + +#include +#include + +int +pthread_condattr_setprotocol_np (attr, protocol) + pthread_condattr_t *attr; + int protocol; +{ + if (protocol != PTHREAD_PRIO_NONE + && protocol != PTHREAD_PRIO_INHERIT + && __builtin_expect (protocol != PTHREAD_PRIO_PROTECT, 0)) + return EINVAL; + + int *valuep = &((struct pthread_condattr *) attr)->value; + + *valuep = ((*valuep & ~CONDATTR_PROTOCOL_MASK) + | (protocol << CONDATTR_PROTOCOL_SHIFT)); + + return 0; +} diff --git a/nptl/sysdeps/pthread/cond-lock.h b/nptl/sysdeps/pthread/cond-lock.h new file mode 100644 index 0000000..acf3f6d --- /dev/null +++ b/nptl/sysdeps/pthread/cond-lock.h @@ -0,0 +1,58 @@ +/* Copyright (C) 2010 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, write to the Free + Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA + 02111-1307 USA. */ + +#ifndef _COND_LOCK_H +#define _COND_LOCK_H 1 + + +extern void lll_pi_lock (int *futex, int pshared) + __attribute__ ((visibility ("hidden"))); +extern void lll_pi_unlock (int *futex, int pshared) + __attribute__ ((visibility ("hidden"))); + +static inline void cond_lock(pthread_cond_t *cond, + int pshared); + +static inline void cond_unlock(pthread_cond_t *cond, + int pshared); + +static inline void cond_lock(cond, pshared) + pthread_cond_t *cond; + int pshared; +{ + if (pshared == LLL_PRIVATE + && ((cond->__data.__nwaiters & COND_PROTOCOL_MASK) + == COND_PRIO_INHERIT)) + lll_pi_lock (&cond->__data.__lock, pshared); + else + lll_lock (cond->__data.__lock, pshared); +} + +static inline void cond_unlock(cond, pshared) + pthread_cond_t *cond; + int pshared; +{ + if (pshared == LLL_PRIVATE + && ((cond->__data.__nwaiters & COND_PROTOCOL_MASK) + == COND_PRIO_INHERIT)) + lll_pi_unlock (&cond->__data.__lock, pshared); + else + lll_unlock (cond->__data.__lock, pshared); +} + +#endif diff --git a/nptl/sysdeps/pthread/pthread.h b/nptl/sysdeps/pthread/pthread.h index 44cf9f0..ef5e093 100644 --- a/nptl/sysdeps/pthread/pthread.h +++ b/nptl/sysdeps/pthread/pthread.h @@ -1006,6 +1006,18 @@ extern int pthread_condattr_getpshared (__const pthread_condattr_t * extern int pthread_condattr_setpshared (pthread_condattr_t *__attr, int __pshared) __THROW __nonnull ((1)); +/* Get the protocol flag of the condition variable attribute ATTR. */ +extern int pthread_condattr_getprotocol_np (__const pthread_condattr_t * + __restrict __attr, + int *__restrict __protocol) + __THROW __nonnull ((1, 2)); + +/* Set the cond protocol attribute in ATTR to protocol (one of + PTHREAD_PRIO_NONE, PTHREAD_PRIO_INHERIT or PTHREAD_PRIO_PROTECT). */ +extern int pthread_condattr_setprotocol_np (pthread_condattr_t *__attr, + int __protocol) + __THROW __nonnull ((1)); + #ifdef __USE_XOPEN2K /* Get the clock selected for the conditon variable attribute ATTR. */ extern int pthread_condattr_getclock (__const pthread_condattr_t * diff --git a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h index 4bb585a..3144d6f 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h @@ -1,4 +1,5 @@ -/* Copyright (C) 2002-2004, 2006-2008, 2009 Free Software Foundation, Inc. +/* Copyright (C) 2002,2003,2004,2006,2007,2008,2010 + Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper , 2002. @@ -238,6 +239,91 @@ LLL_STUB_UNWIND_INFO_END } while (0) +#define lll_futex_wait_requeue_pi(cond, val, mutex, private) \ + lll_futex_timed_wait_requeue_pi (cond, val, NULL, mutex, private) + + +#define lll_futex_timed_wait_requeue_pi(cond, val, time, mutex, private) \ + ({ \ + INTERNAL_SYSCALL_DECL (__err); \ + long int __ret; \ + unsigned int *__futexp = &cond->__data.__futex; \ + int __flag = FUTEX_WAIT_REQUEUE_PI; \ + __flag = (cond->__data.__nwaiters & COND_CLOCK_MASK) ? \ + __flag : __flag | FUTEX_CLOCK_REALTIME; \ + __ret = INTERNAL_SYSCALL (futex, __err, 5, \ + (__futexp), \ + __lll_private_flag (__flag, private), \ + (val), time, \ + (mutex)); \ + INTERNAL_SYSCALL_ERROR_P (__ret, __err)? -__ret : __ret; \ + }) + + +#define lll_futex_requeue(futex, nr_wake, nr_move, mutex, val, private) \ + ({ \ + int __status; \ + int __save_ebx; \ + int __dummy; \ + int __flag = FUTEX_CMP_REQUEUE; \ + register __typeof (nr_move) _nr_move asm ("esi") = (nr_move); \ + __asm __volatile ("mov %%ebx, %0\n\t" \ + "mov %1, %%ebx\n\t" \ + LLL_EBX_LOAD \ + "pushl %%ebp\n\t" \ + "mov %3, %%ebp\n\t" \ + : "=m" (__save_ebx) \ + : "c" (mutex), LLL_EBX_REG (futex), \ + "a" (val)); \ + __asm __volatile (LLL_ENTER_KERNEL \ + "popl %%ebp\n\t" \ + : "=a" (__status) \ + : "0" (SYS_futex), \ + "c" (__lll_private_flag (__flag, private)), \ + "d" (nr_wake), "S" (_nr_move), \ + "i" (0) /* phony, to align next arg's number */, \ + "i" (offsetof (tcbhead_t, sysinfo)) \ + : "memory"); \ + __asm __volatile (LLL_EBX_LOAD \ + "mov %1, %%ebx\n\t" \ + : "=m" (__dummy) \ + : "m" (__save_ebx), LLL_EBX_REG (futex)); \ + __status; \ + }) + + +#define lll_futex_requeue_pi(futex, nr_wake, nr_move, mutex, val, private) \ + ({ \ + int __status; \ + int __save_ebx; \ + int __dummy; \ + int __flag = FUTEX_CMP_REQUEUE_PI; \ + register __typeof (nr_move) _nr_move asm ("esi") = (nr_move); \ + __asm __volatile ("mov %%ebx, %0\n\t" \ + "mov %1, %%ebx\n\t" \ + LLL_EBX_LOAD \ + "pushl %%ebp\n\t" \ + "mov %3, %%ebp\n\t" \ + : "=m" (__save_ebx) \ + : "c" (mutex), LLL_EBX_REG (futex), \ + "a" (val)); \ + __asm __volatile (LLL_ENTER_KERNEL \ + "popl %%ebp\n\t" \ + : "=a" (__status) \ + : "0" (SYS_futex), \ + "c" (__lll_private_flag (__flag, private)), \ + "d" (nr_wake), "S" (_nr_move), \ + "i" (0) /* phony, to align next arg's number */, \ + "i" (offsetof (tcbhead_t, sysinfo)) \ + : "memory"); \ + __asm __volatile (LLL_EBX_LOAD \ + "mov %1, %%ebx\n\t" \ + : "=m" (__dummy) \ + : "m" (__save_ebx), LLL_EBX_REG (futex)); \ + __status; \ + }) + + /* NB: in the lll_trylock macro we simply return the value in %eax after the cmpxchg instruction. In case the operation succeded this value is zero. In case the operation failed, the cmpxchg instruction diff --git a/nptl/sysdeps/unix/sysv/linux/internaltypes.h b/nptl/sysdeps/unix/sysv/linux/internaltypes.h index add20b6..876ca79 100644 --- a/nptl/sysdeps/unix/sysv/linux/internaltypes.h +++ b/nptl/sysdeps/unix/sysv/linux/internaltypes.h @@ -67,20 +67,38 @@ struct pthread_condattr { /* Combination of values: - Bit 0 : flag whether coditional variable will be shareable between + Bit 0 : flag whether conditional variable will be shareable between processes. - Bit 1-7: clock ID. */ + Bit 1-7: clock ID. + Bit 8-9: protocol. One of PTHREAD_PRIO_NONE, PTHREAD_PRIO_INHERIT + or PTHREAD_PRIO_PROTECT. */ int value; }; +#define CONDATTR_PSHARED_MASK 0x00000001 +#define CONDATTR_CLOCKID_MASK 0x000000FE +#define CONDATTR_CLOCKID_SHIFT 1 +#define CONDATTR_PROTOCOL_MASK 0x00000300 +#define CONDATTR_PROTOCOL_SHIFT 8 + + +enum { + COND_PRIO_INHERIT = 2, + COND_PRIO_PROTECT +}; + + /* The __NWAITERS field is used as a counter and to house the number - of bits for other purposes. COND_CLOCK_BITS is the number - of bits needed to represent the ID of the clock. COND_NWAITERS_SHIFT + of bits for other purposes. COND_CLOCK_MASK defines the bits used + to represent the ID of the clock. COND_PROTOCOL_MASK defines the + bits used to represent cond protocol attrbutes. COND_NWAITERS_SHIFT is the number of bits reserved for other purposes like the clock. */ -#define COND_CLOCK_BITS 1 -#define COND_NWAITERS_SHIFT 1 +#define COND_CLOCK_MASK 0x00000001 +#define COND_PROTOCOL_SHIFT 1 +#define COND_PROTOCOL_MASK 0x00000006 +#define COND_NWAITERS_SHIFT 3 /* Read-write lock variable attribute data structure. */ diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h index 9b15bfb..511de06 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h @@ -223,6 +223,31 @@ LLL_STUB_UNWIND_INFO_END }) +#define lll_futex_wait_requeue_pi(cond, val, mutex, private) \ + lll_futex_timed_wait_requeue_pi(cond, val, NULL, mutex, private) + + +#define lll_futex_timed_wait_requeue_pi(cond, val, timeout, mutex, private) \ + ({ \ + register const struct timespec *__to __asm ("r10") = timeout; \ + register void *__mutex __asm ("r8") = mutex; \ + unsigned int *__futexp = &cond->__data.__futex; \ + int __flag = FUTEX_WAIT_REQUEUE_PI; \ + __flag = (cond->__data.__nwaiters & COND_CLOCK_MASK) ? \ + __flag : __flag | FUTEX_CLOCK_REALTIME; \ + int __status; \ + register __typeof (val) _val __asm ("edx") = (val); \ + __asm __volatile ("syscall" \ + : "=a" (__status) \ + : "0" (SYS_futex), "D" (__futexp), \ + "S" (__lll_private_flag (__flag, private)), \ + "d" (_val), "r" (__to), \ + "r" (__mutex) \ + : "memory", "cc", "r11", "cx"); \ + __status; \ + }) + + #define lll_futex_wake(futex, nr, private) \ do { \ int __ignore; \ @@ -554,6 +579,21 @@ LLL_STUB_UNWIND_INFO_END : "cx", "r11", "cc", "memory"); \ __res < 0; }) +/* Returns non-zero if error happened, zero if success. */ +#define lll_futex_requeue_pi(ftx, nr_wake, nr_move, mutex, val, private) \ + ({ int __res; \ + register int __nr_move __asm ("r10") = nr_move; \ + register void *__mutex __asm ("r8") = mutex; \ + register int __val __asm ("r9") = val; \ + __asm __volatile ("syscall" \ + : "=a" (__res) \ + : "0" (__NR_futex), "D" ((void *) ftx), \ + "S" (__lll_private_flag (FUTEX_CMP_REQUEUE_PI, \ + private)), "d" (nr_wake), \ + "r" (__nr_move), "r" (__mutex), "r" (__val) \ + : "cx", "r11", "cc", "memory"); \ + __res < 0; }) + #define lll_islocked(futex) \ (futex != LLL_LOCK_INITIALIZER) -- 1.7.0.4