This is the mail archive of the libc-alpha@sourceware.org mailing list for the glibc project.


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]
Other format: [Raw text]

Re: [PATCH 05/14] Add lock elision to rwlocks


I need some consensus on this issue.

In POSIX Issue 7:
http://pubs.opengroup.org/onlinepubs/9699919799/functions/pthread_rwlock_trywrlock.html

The language for pthread_rlock_wrlock and pthread_rwlock_trywrlock says
that it "may" deadlock, or it "may" also return EDEADLK.

My reading of this is that Torvald's guideline is incorrect:
http://sourceware.org/glibc/wiki/LockElisionGuide#Compability_with_pthreads_rwlock

In that wrlock/trywrlock are not *required* to have a deadlock on relock
semantic by the standard.

The same soft language of "may deadlock" is used for rdlock and thus the same
flexibility is granted there.

Therefore this patch #5 falls within the semantic guarantees of the standard
and therefore can be considered correct.

Comments?

> 2013-06-24  Andi Kleen <ak@linux.intel.com>
>             Hongjiu Lu <hongjiu.lu@intel.com>
> 
> 	* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S: dito.
> 	* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S: dito
> 	* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S: dito.
> 	* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S: dito.
> 	* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S: dito.
> 	* sysdeps/unix/sysv/linux/internaltypes.h (struct pthread_condattr):
> 	(struct pthread_rwlockattr): Support elision flags in internal rwlock.
> 	* sysdeps/unix/sysv/linux/lowlevelrwlock.sym: Add RW_ELISION.
> 	* sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h: Add elision flags
> 	* sysdeps/unix/sysv/linux/x86/pthread_rwlock_tryrdlock.c: New file
> 	* sysdeps/unix/sysv/linux/x86/pthread_rwlock_trywrlock.c: dito.
> 	* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S: Handle elision.
> 	* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S: dito.
> 	* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S: dito.
> 	* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S: dito.
> 	* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S: dito.
> ---
>  nptl/pthread_rwlock_tryrdlock.c                    |  2 +
>  nptl/pthread_rwlock_trywrlock.c                    |  2 +
>  .../sysv/linux/i386/i486/pthread_rwlock_rdlock.S   | 65 ++++++++++++++++++++++
>  .../linux/i386/i486/pthread_rwlock_timedrdlock.S   | 60 ++++++++++++++++++++
>  .../linux/i386/i486/pthread_rwlock_timedwrlock.S   | 46 ++++++++++++++-
>  .../sysv/linux/i386/i486/pthread_rwlock_unlock.S   | 19 +++++++
>  .../sysv/linux/i386/i486/pthread_rwlock_wrlock.S   | 52 +++++++++++++++++
>  nptl/sysdeps/unix/sysv/linux/internaltypes.h       | 16 +++++-
>  nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym    |  5 ++
>  .../unix/sysv/linux/x86/bits/pthreadtypes.h        |  6 +-
>  .../unix/sysv/linux/x86/pthread_rwlock_tryrdlock.c | 52 +++++++++++++++++
>  .../unix/sysv/linux/x86/pthread_rwlock_trywrlock.c | 51 +++++++++++++++++
>  .../unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S | 48 ++++++++++++++++
>  .../sysv/linux/x86_64/pthread_rwlock_timedrdlock.S | 48 ++++++++++++++++
>  .../sysv/linux/x86_64/pthread_rwlock_timedwrlock.S | 32 +++++++++++
>  .../unix/sysv/linux/x86_64/pthread_rwlock_unlock.S | 17 +++++-
>  .../unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S | 31 +++++++++++
>  17 files changed, 546 insertions(+), 6 deletions(-)
>  create mode 100644 nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_tryrdlock.c
>  create mode 100644 nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_trywrlock.c
> 
> diff --git a/nptl/pthread_rwlock_tryrdlock.c b/nptl/pthread_rwlock_tryrdlock.c
> index 935ac87..896809b 100644
> --- a/nptl/pthread_rwlock_tryrdlock.c
> +++ b/nptl/pthread_rwlock_tryrdlock.c
> @@ -45,4 +45,6 @@ __pthread_rwlock_tryrdlock (pthread_rwlock_t *rwlock)
>  
>    return result;
>  }
> +#ifndef __pthread_rwlock_tryrdlock
>  strong_alias (__pthread_rwlock_tryrdlock, pthread_rwlock_tryrdlock)
> +#endif
> diff --git a/nptl/pthread_rwlock_trywrlock.c b/nptl/pthread_rwlock_trywrlock.c
> index 01754ae..6d8a8c4 100644
> --- a/nptl/pthread_rwlock_trywrlock.c
> +++ b/nptl/pthread_rwlock_trywrlock.c
> @@ -38,4 +38,6 @@ __pthread_rwlock_trywrlock (pthread_rwlock_t *rwlock)
>  
>    return result;
>  }
> +#ifndef __pthread_rwlock_trywrlock
>  strong_alias (__pthread_rwlock_trywrlock, pthread_rwlock_trywrlock)
> +#endif
> diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S
> index 6c46ba6..4151b03 100644
> --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S
> +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S
> @@ -21,9 +21,16 @@
>  #include <lowlevelrwlock.h>
>  #include <pthread-errnos.h>
>  #include <kernel-features.h>
> +#include <hle.h>
>  
>  #include <stap-probe.h>
>  
> +#ifdef PIC
> +#define MO(x) x##@GOTOFF(%edx)
> +#else
> +#define MO(x) x
> +#endif
> +
>  	.text
>  
>  	.globl	__pthread_rwlock_rdlock
> @@ -43,6 +50,62 @@ __pthread_rwlock_rdlock:
>  
>  	LIBC_PROBE (rdlock_entry, 1, %ebx)
>  
> +#ifdef PIC
> +	SETUP_PIC_REG(dx)
> +	addl $_GLOBAL_OFFSET_TABLE_,%edx
> +#endif
> +
> +	cmpb  $0,RW_ELISION(%ebx)
> +	js    not_elided_rdlock
> +	jnz   2f
> +	/* zero: use default  */
> +
> +	cmpl	$0,MO(__rwlock_rtm_enabled)
> +	jz    not_elided_rdlock
> +
> +2:
> +	mov   MO(__rwlock_rtm_read_retries),%ecx
> +
> +try_trans_rdlock:
> +	XBEGIN abort_rdlock
> +
> +	/* Lock writer/reader free?  */
> +	cmpl  $0,WRITER(%ebx)
> +	jnz   1f
> +	cmpl  $0,NR_READERS(%ebx)
> +	jnz   1f
> +
> +	/* Lock is free. Run with transaction  */
> +	xor   %eax,%eax
> +
> +	pop   %ebx
> +	cfi_adjust_cfa_offset(-4)
> +	pop   %esi
> +	cfi_adjust_cfa_offset(-4)
> +	ret
> +
> +	/* Lock is not free. Run  */
> +1:	XABORT 0xff
> +	jmp not_elided_rdlock
> +
> +	/* Abort happened.  */
> +abort_rdlock:
> +	testl $_XABORT_CONFLICT,%eax
> +	jz    not_elided_rdlock
> +
> +	/* For a reader that aborts due a conflict retry speculation
> +	   a limited number of times. This way when some reader aborts
> +	   because the reader count is written the other readers will
> +	   still elide, at the cost of retrying the speculation.  */
> +
> +	dec   %ecx
> +	jnz   try_trans_rdlock
> +
> +	/* Otherwise we just fall back directly to the lock.
> +	   Here's the place to add more adaptation.  */
> +
> +not_elided_rdlock:
> +
>  	/* Get the lock.  */
>  	movl	$1, %edx
>  	xorl	%eax, %eax
> @@ -188,5 +251,7 @@ __pthread_rwlock_rdlock:
>  	cfi_endproc
>  	.size	__pthread_rwlock_rdlock,.-__pthread_rwlock_rdlock
>  
> +#ifndef __pthread_rwlock_rdlock
>  strong_alias (__pthread_rwlock_rdlock, pthread_rwlock_rdlock)
>  hidden_def (__pthread_rwlock_rdlock)
> +#endif
> diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S
> index 1908f6f..e710c49 100644
> --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S
> +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S
> @@ -21,7 +21,13 @@
>  #include <lowlevelrwlock.h>
>  #include <pthread-errnos.h>
>  #include <kernel-features.h>
> +#include <hle.h>
>  
> +#ifdef PIC
> +#define MO(x) x##@GOTOFF(%edx)
> +#else
> +#define MO(x) x
> +#endif
>  
>  	.text
>  
> @@ -48,6 +54,59 @@ pthread_rwlock_timedrdlock:
>  	movl	28(%esp), %ebp
>  	movl	32(%esp), %edi
>  
> +#ifdef PIC
> +	SETUP_PIC_REG(dx)
> +	addl $_GLOBAL_OFFSET_TABLE_,%edx
> +#endif
> +
> +	cmpb  $0,RW_ELISION(%ebp)
> +	js    not_elided_trdlock
> +	jnz   2f
> +	/* zero: use default  */
> +
> +	cmpl  $0,MO(__rwlock_rtm_enabled)
> +	jz    not_elided_trdlock
> +
> +2:
> +	mov   MO(__rwlock_rtm_read_retries),%ecx
> +
> +try_trans_trdlock:
> +	XBEGIN abort_trdlock
> +
> +	/* Lock writer/reader free?  */
> +	cmpl  $0,WRITER(%ebp)
> +	jnz   1f
> +	cmpl  $0,NR_READERS(%ebp)
> +	jnz   1f
> +
> +	/* Lock is free. Run with transaction  */
> +	xor   %eax,%eax
> +	jmp   77f
> +
> +
> +	/* Lock is not free. Run  */
> +1:	XABORT 0xff
> +	jmp not_elided_trdlock
> +
> +	/* Abort happened.  */
> +abort_trdlock:
> +	testl $_XABORT_CONFLICT,%eax
> +	jz    not_elided_trdlock
> +
> +	/* For a reader that aborts due a conflict retry speculation
> +	   a limited number of times. This way when some reader aborts
> +	   because the reader count is written the other readers will
> +	   still elide, at the cost of retrying the speculation.  */
> +
> +	dec   %ecx
> +	jnz   try_trans_trdlock
> +
> +	/* Otherwise we just fall back directly to the lock.
> +	   Here's the place to add more adaptation.  */
> +
> +not_elided_trdlock:
> +
> +
>  	/* Get the lock.  */
>  	movl	$1, %edx
>  	xorl	%eax, %eax
> @@ -158,6 +217,7 @@ pthread_rwlock_timedrdlock:
>  
>  7:	movl	%edx, %eax
>  
> +77:
>  	addl	$8, %esp
>  	cfi_adjust_cfa_offset(-8)
>  	popl	%ebp
> diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S
> index e0fc809..c23f270 100644
> --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S
> +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S
> @@ -21,7 +21,13 @@
>  #include <lowlevelrwlock.h>
>  #include <pthread-errnos.h>
>  #include <kernel-features.h>
> +#include <hle.h>
>  
> +#ifdef PIC
> +#define MO(x) x##@GOTOFF(%edx)
> +#else
> +#define MO(x) x
> +#endif
>  
>  	.text
>  
> @@ -48,6 +54,44 @@ pthread_rwlock_timedwrlock:
>  	movl	28(%esp), %ebp
>  	movl	32(%esp), %edi
>  
> +#ifdef PIC
> +	SETUP_PIC_REG(dx)
> +	addl $_GLOBAL_OFFSET_TABLE_,%edx
> +#endif
> +	cmpb  $0,RW_ELISION(%ebp)
> +	js    not_elided_twrlock
> +	jnz   try_trans_wrlock
> +	/* zero: use default  */
> +
> +	cmpl	$0,MO(__rwlock_rtm_enabled)
> +	jz	not_elided_twrlock
> +
> +try_trans_wrlock:
> +	XBEGIN abort_twrlock
> +
> +	/* Lock writer free?  */
> +	cmpl  $0,WRITER(%ebp)
> +	jnz   1f
> +	cmpl  $0,NR_READERS(%ebp)
> +	jnz   1f
> +
> +	/* Lock is free. Run with transaction  */
> +	xor   %eax,%eax
> +
> +	jmp   77f
> +
> +	/* Lock is not free. Run  */
> +1:	XABORT 0xff
> +	jmp   not_elided_twrlock
> +
> +	/* Abort happened.  */
> +abort_twrlock:
> +
> +	/* Otherwise we just fall back directly to the lock.
> +	   Here's the place to add more adaptation.  */
> +
> +not_elided_twrlock:
> +
>  	/* Get the lock.  */
>  	movl	$1, %edx
>  	xorl	%eax, %eax
> @@ -156,7 +200,7 @@ pthread_rwlock_timedwrlock:
>  
>  7:	movl	%edx, %eax
>  
> -	addl	$8, %esp
> +77:	addl	$8, %esp
>  	cfi_adjust_cfa_offset(-8)
>  	popl	%ebp
>  	cfi_adjust_cfa_offset(-4)
> diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S
> index 708e31c..9b384bb 100644
> --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S
> +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S
> @@ -20,6 +20,7 @@
>  #include <lowlevellock.h>
>  #include <lowlevelrwlock.h>
>  #include <kernel-features.h>
> +#include <hle.h>
>  
>  
>  	.text
> @@ -38,6 +39,24 @@ __pthread_rwlock_unlock:
>  
>  	movl	12(%esp), %edi
>  
> +	/* Is lock free?  */
> +	cmpl $0,WRITER(%edi)
> +	jnz  1f
> +	cmpl $0,NR_READERS(%edi)
> +	jnz  1f
> +
> +	/* Looks free. Assume transaction.
> +	   If you crash here you unlocked a free lock.  */
> +	XEND
> +	xor  %eax,%eax
> +
> +	pop	%edi
> +	cfi_adjust_cfa_offset(-4)
> +	pop	%ebx
> +	cfi_adjust_cfa_offset(-4)
> +	ret
> +
> +1:
>  	/* Get the lock.  */
>  	movl	$1, %edx
>  	xorl	%eax, %eax
> diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S
> index 6ea17f7..e0c3428 100644
> --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S
> +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S
> @@ -21,9 +21,16 @@
>  #include <lowlevelrwlock.h>
>  #include <pthread-errnos.h>
>  #include <kernel-features.h>
> +#include <hle.h>
>  
>  #include <stap-probe.h>
>  
> +#ifdef PIC
> +#define MO(x) x@GOTOFF(%edx)
> +#else
> +#define MO(x) x
> +#endif
> +
>  	.text
>  
>  	.globl	__pthread_rwlock_wrlock
> @@ -43,6 +50,49 @@ __pthread_rwlock_wrlock:
>  
>  	LIBC_PROBE (wrlock_entry, 1, %ebx)
>  
> +#ifdef PIC
> +	SETUP_PIC_REG(dx)
> +	addl $_GLOBAL_OFFSET_TABLE_,%edx
> +#endif
> +	cmpb  $0,RW_ELISION(%ebx)
> +	js    not_elided_wrlock
> +	jnz   try_trans_wrlock
> +	/* zero: use default  */
> +
> +	cmpl    $0,MO(__rwlock_rtm_enabled)
> +	jz      not_elided_wrlock
> +
> +try_trans_wrlock:
> +	XBEGIN abort_wrlock
> +
> +	/* Lock writer free?  */
> +	/* Ignore readers because we don't need them  */
> +	cmpl  $0,WRITER(%ebx)
> +	jnz   1f
> +	cmpl  $0,NR_READERS(%ebx)
> +	jnz   1f
> +
> +	/* Lock is free. Run with transaction  */
> +	xor   %eax,%eax
> +
> +	pop   %ebx
> +	cfi_adjust_cfa_offset(-4)
> +	pop   %esi
> +	cfi_adjust_cfa_offset(-4)
> +	ret
> +
> +	/* Lock is not free. Run  */
> +1:	XABORT 0xff
> +	jmp   not_elided_wrlock
> +
> +	/* Abort happened.  */
> +abort_wrlock:
> +
> +	/* Otherwise we just fall back directly to the lock.
> +	   Here's the place to add more adaptation.  */
> +
> +not_elided_wrlock:
> +
>  	/* Get the lock.  */
>  	movl	$1, %edx
>  	xorl	%eax, %eax
> @@ -179,5 +229,7 @@ __pthread_rwlock_wrlock:
>  	cfi_endproc
>  	.size	__pthread_rwlock_wrlock,.-__pthread_rwlock_wrlock
>  
> +#ifndef __pthread_rwlock_wrlock
>  strong_alias (__pthread_rwlock_wrlock, pthread_rwlock_wrlock)
>  hidden_def (__pthread_rwlock_wrlock)
> +#endif
> diff --git a/nptl/sysdeps/unix/sysv/linux/internaltypes.h b/nptl/sysdeps/unix/sysv/linux/internaltypes.h
> index 699a618..70aaa40 100644
> --- a/nptl/sysdeps/unix/sysv/linux/internaltypes.h
> +++ b/nptl/sysdeps/unix/sysv/linux/internaltypes.h
> @@ -20,7 +20,7 @@
>  #define _INTERNALTYPES_H	1
>  
>  #include <stdint.h>
> -
> +#include <endian.h>
>  
>  struct pthread_attr
>  {
> @@ -86,7 +86,19 @@ struct pthread_condattr
>  struct pthread_rwlockattr
>  {
>    int lockkind;
> -  int pshared;
> +  /* This used to be an int pshared. We keep the pshared at the
> +     same location.
> +     PTHREAD_PROCESS_SHARED and PTHREAD_PROCESS_PRIVATE, both fit into one
> +     byte.  */
> +#if __BYTE_ORDER == __LITTLE_ENDIAN
> +  short pshared;
> +  char  rw_elision;
> +  char  pad;
> +#else
> +  char  rw_elision;
> +  char  pad;
> +  short pshared;
> +#endif
>  };
>  
>  
> diff --git a/nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym b/nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym
> index f50b25b..6d9d201 100644
> --- a/nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym
> +++ b/nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym
> @@ -3,6 +3,10 @@
>  #include <bits/pthreadtypes.h>
>  #include <bits/wordsize.h>
>  
> +#ifndef __PTHREAD_RWLOCK_ELISION
> +#define __rw_elision __shared
> +#endif
> +
>  --
>  
>  MUTEX		offsetof (pthread_rwlock_t, __data.__lock)
> @@ -14,3 +18,4 @@ WRITERS_QUEUED	offsetof (pthread_rwlock_t, __data.__nr_writers_queued)
>  FLAGS		offsetof (pthread_rwlock_t, __data.__flags)
>  WRITER		offsetof (pthread_rwlock_t, __data.__writer)
>  PSHARED		offsetof (pthread_rwlock_t, __data.__shared)
> +RW_ELISION		offsetof (pthread_rwlock_t, __data.__rw_elision)
> diff --git a/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h
> index 1852e07..c3325d9 100644
> --- a/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h
> +++ b/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h
> @@ -183,12 +183,13 @@ typedef union
>      unsigned int __nr_writers_queued;
>      int __writer;
>      int __shared;
> -    unsigned long int __pad1;
> +    long __rw_elision;
>      unsigned long int __pad2;
>      /* FLAGS must stay at this position in the structure to maintain
>         binary compatibility.  */
>      unsigned int __flags;
>  # define __PTHREAD_RWLOCK_INT_FLAGS_SHARED	1
> +# define __PTHREAD_RWLOCK_ELISION		1
>    } __data;
>  # else
>    struct
> @@ -203,8 +204,9 @@ typedef union
>         binary compatibility.  */
>      unsigned char __flags;
>      unsigned char __shared;
> -    unsigned char __pad1;
> +    char __rw_elision;
>      unsigned char __pad2;
> +# define __PTHREAD_RWLOCK_ELISION		2
>      int __writer;
>    } __data;
>  # endif
> diff --git a/nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_tryrdlock.c b/nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_tryrdlock.c
> new file mode 100644
> index 0000000..f18295c
> --- /dev/null
> +++ b/nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_tryrdlock.c
> @@ -0,0 +1,52 @@
> +/* pthread_rwlock_tryrdlock: Lock elision version of pthreads rwlock_tryrdlock.
> +   Copyright (C) 2013 Free Software Foundation, Inc.
> +   This file is part of the GNU C Library.
> +
> +   The GNU C Library is free software; you can redistribute it and/or
> +   modify it under the terms of the GNU Lesser General Public
> +   License as published by the Free Software Foundation; either
> +   version 2.1 of the License, or (at your option) any later version.
> +
> +   The GNU C Library is distributed in the hope that it will be useful,
> +   but WITHOUT ANY WARRANTY; without even the implied warranty of
> +   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> +   Lesser General Public License for more details.
> +
> +   You should have received a copy of the GNU Lesser General Public
> +   License along with the GNU C Library; if not, see
> +   <http://www.gnu.org/licenses/>.  */
> +#include <pthread.h>
> +#include <pthreadP.h>
> +#include <hle.h>
> +#include <elision-conf.h>
> +#include "init-arch.h"
> +
> +#define __pthread_rwlock_tryrdlock __full_pthread_rwlock_tryrdlock
> +#include <nptl/pthread_rwlock_tryrdlock.c>
> +#undef __pthread_rwlock_tryrdlock
> +
> +int
> +__pthread_rwlock_tryrdlock(pthread_rwlock_t *rwlock)
> +{
> +  unsigned status;
> +
> +  if (rwlock->__data.__rw_elision > 0
> +      || (rwlock->__data.__rw_elision == 0 && __rwlock_rtm_enabled))
> +    {
> +      if ((status = _xbegin()) == _XBEGIN_STARTED)
> +	{
> +	  if (rwlock->__data.__writer == 0
> +	      && rwlock->__data.__nr_readers == 0)
> +	    return 0;
> +	  /* Lock was busy. Fall back to normal locking.
> +	     Could also _xend here but xabort with 0xff code
> +	     is more visible in the profiler.  */
> +	  _xabort (_ABORT_LOCK_BUSY);
> +	}
> +      /* Aborts come here  */
> +    }
> +
> +  return __full_pthread_rwlock_tryrdlock (rwlock);
> +}
> +
> +strong_alias(__pthread_rwlock_tryrdlock, pthread_rwlock_tryrdlock);
> diff --git a/nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_trywrlock.c b/nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_trywrlock.c
> new file mode 100644
> index 0000000..0f5cd3c
> --- /dev/null
> +++ b/nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_trywrlock.c
> @@ -0,0 +1,51 @@
> +/* pthread_rwlock_trywrlock: Lock elision version of pthreads rwlock trywrlock.
> +   Copyright (C) 2013 Free Software Foundation, Inc.
> +   This file is part of the GNU C Library.
> +
> +   The GNU C Library is free software; you can redistribute it and/or
> +   modify it under the terms of the GNU Lesser General Public
> +   License as published by the Free Software Foundation; either
> +   version 2.1 of the License, or (at your option) any later version.
> +
> +   The GNU C Library is distributed in the hope that it will be useful,
> +   but WITHOUT ANY WARRANTY; without even the implied warranty of
> +   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> +   Lesser General Public License for more details.
> +
> +   You should have received a copy of the GNU Lesser General Public
> +   License along with the GNU C Library; if not, see
> +   <http://www.gnu.org/licenses/>.  */
> +#include <pthread.h>
> +#include <pthreadP.h>
> +#include <hle.h>
> +#include <elision-conf.h>
> +#include "init-arch.h"
> +
> +#define __pthread_rwlock_trywrlock __full_pthread_rwlock_trywrlock
> +#include <nptl/pthread_rwlock_trywrlock.c>
> +#undef __pthread_rwlock_trywrlock
> +
> +int
> +__pthread_rwlock_trywrlock(pthread_rwlock_t *rwlock)
> +{
> +  unsigned status;
> +
> +  if ((rwlock->__data.__rw_elision == 0 && __rwlock_rtm_enabled)
> +       || rwlock->__data.__rw_elision > 0)
> +    {
> +      _xabort (_ABORT_NESTED_TRYLOCK);
> +      if ((status = _xbegin()) == _XBEGIN_STARTED)
> +	{
> +	  if (rwlock->__data.__writer == 0
> +	      && rwlock->__data.__nr_readers == 0)
> +	    return 0;
> +	  /* Lock was busy. Fall back to normal locking.
> +	     Could also _xend here but xabort with 0xff code
> +	     is more visible in the profiler.  */
> +	  _xabort (_ABORT_LOCK_BUSY);
> +	}
> +    }
> +
> +  return __full_pthread_rwlock_trywrlock (rwlock);
> +}
> +strong_alias(__pthread_rwlock_trywrlock, pthread_rwlock_trywrlock);
> diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S
> index 7681818..3b335bc 100644
> --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S
> +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S
> @@ -22,6 +22,7 @@
>  #include <pthread-errnos.h>
>  #include <kernel-features.h>
>  #include <stap-probe.h>
> +#include <hle.h>
>  
>  	.text
>  
> @@ -33,6 +34,51 @@ __pthread_rwlock_rdlock:
>  
>  	LIBC_PROBE (rdlock_entry, 1, %rdi)
>  
> +	cmpq  $0,RW_ELISION(%rdi)
> +	js    not_elided_rdlock
> +	jnz   2f
> +	/* zero: use default  */
> +
> +	cmpl  $0,__rwlock_rtm_enabled(%rip)
> +	jz    not_elided_rdlock
> +
> +2:
> +	mov   __rwlock_rtm_read_retries(%rip),%esi
> +
> +try_trans_rdlock:
> +	XBEGIN abort_rdlock
> +
> +	/* Lock reader/writer free?  */
> +	cmpl  $0,WRITER(%rdi)
> +	jnz   1f
> +	cmpl  $0,NR_READERS(%rdi)
> +	jnz   1f
> +
> +	/* Lock is free. Run with transaction  */
> +	xor   %eax,%eax
> +	ret
> +
> +	/* Lock is not free. Run  */
> +1:	XABORT 0xff
> +	jmp not_elided_rdlock
> +
> +	/* Abort happened.  */
> +abort_rdlock:
> +	testl $_XABORT_CONFLICT,%eax
> +	jz    not_elided_rdlock
> +
> +	/* For a reader that aborts due a conflict retry speculation
> +	   a limited number of times. This way when some reader aborts
> +	   because the reader count is written the other readers will
> +	   still elide, at the cost of retrying the speculation.  */
> +
> +	dec   %esi
> +	jnz   try_trans_rdlock
> +
> +	/* Otherwise we just fall back directly to the lock.
> +	   Here's the place to add more adaptation.  */
> +
> +not_elided_rdlock:
>  	xorq	%r10, %r10
>  
>  	/* Get the lock.  */
> @@ -173,5 +219,7 @@ __pthread_rwlock_rdlock:
>  	cfi_endproc
>  	.size	__pthread_rwlock_rdlock,.-__pthread_rwlock_rdlock
>  
> +#ifndef __pthread_rwlock_rdlock
>  strong_alias (__pthread_rwlock_rdlock, pthread_rwlock_rdlock)
>  hidden_def (__pthread_rwlock_rdlock)
> +#endif
> diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S
> index 57fe1e9..2bc9037 100644
> --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S
> +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S
> @@ -21,6 +21,7 @@
>  #include <lowlevelrwlock.h>
>  #include <pthread-errnos.h>
>  #include <kernel-features.h>
> +#include <hle.h>
>  
>  	.text
>  
> @@ -29,6 +30,53 @@
>  	.align	16
>  pthread_rwlock_timedrdlock:
>  	cfi_startproc
> +
> +	cmpq  $0,RW_ELISION(%rdi)
> +	js    not_elided_timedrdlock
> +	jnz   2f
> +	/* zero: use default  */
> +
> +	cmpl  $0,__rwlock_rtm_enabled(%rip)
> +	jz    not_elided_timedrdlock
> +
> +2:	
> +	mov   __rwlock_rtm_read_retries(%rip),%ecx
> +
> +try_trans_timedrdlock:
> +	XBEGIN abort_timedrdlock
> +
> +	/* Lock reader/writer free?  */
> +	cmpl  $0,WRITER(%rdi)
> +	jnz   1f
> +	cmpl  $0,NR_READERS(%rdi)
> +	jnz   1f
> +
> +	/* Lock is free. Run with transaction  */
> +	xor   %eax,%eax
> +	ret
> +
> +	/* Lock is not free. Run  */
> +1:	XABORT 0xff
> +	jmp not_elided_timedrdlock
> +
> +	/* Abort happened.  */
> +abort_timedrdlock:
> +	testl $_XABORT_CONFLICT,%eax
> +	jz    not_elided_timedrdlock
> +
> +	/* For a reader that aborts due a conflict retry speculation
> +	   a limited number of times. This way when some reader aborts
> +	   because the reader count is written the other readers will
> +	   still elide, at the cost of retrying the speculation.  */
> +
> +	dec   %ecx
> +	jnz   try_trans_timedrdlock
> +
> +	/* Otherwise we just fall back directly to the lock.
> +	   Here's the place to add more adaptation.  */
> +
> +not_elided_timedrdlock:
> +
>  	pushq	%r12
>  	cfi_adjust_cfa_offset(8)
>  	cfi_rel_offset(%r12, 0)
> diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S
> index 391be17..701c07a 100644
> --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S
> +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S
> @@ -21,6 +21,7 @@
>  #include <lowlevelrwlock.h>
>  #include <pthread-errnos.h>
>  #include <kernel-features.h>
> +#include <hle.h>
>  
>  	.text
>  
> @@ -29,6 +30,37 @@
>  	.align	16
>  pthread_rwlock_timedwrlock:
>  	cfi_startproc
> +
> +	cmpq  $0,RW_ELISION(%rdi)
> +	js    not_elided_timedwrlock
> +	jnz   2f
> +	/* zero: use default  */
> +
> +	cmpl  $0,__rwlock_rtm_enabled(%rip)
> +	jz    not_elided_timedwrlock
> +
> +2:
> +	XBEGIN abort
> +
> +	/* Lock free?  */
> +	cmpl  $0,NR_READERS(%rdi)
> +	jnz   1f
> +	cmpl  $0,WRITER(%rdi)
> +	jnz   1f
> +
> +	/* Lock is free. Run with transaction  */
> +	xor   %rax, %rax
> +	ret
> +
> +	/* Lock is not free. Run  */
> +1:	XABORT 0xff
> +	jmp   not_elided_timedwrlock
> +
> +	/* Abort happened.  */
> +abort:
> +	/* No retries here  */
> +
> +not_elided_timedwrlock:
>  	pushq	%r12
>  	cfi_adjust_cfa_offset(8)
>  	cfi_rel_offset(%r12, 0)
> diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S
> index 86dda05..f1b9516 100644
> --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S
> +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S
> @@ -20,7 +20,7 @@
>  #include <lowlevellock.h>
>  #include <lowlevelrwlock.h>
>  #include <kernel-features.h>
> -
> +#include <hle.h>
>  
>  	.text
>  
> @@ -29,6 +29,21 @@
>  	.align	16
>  __pthread_rwlock_unlock:
>  	cfi_startproc
> +
> +	/* Is lock free?  */
> +	cmpl $0,WRITER(%rdi)
> +	jnz  1f
> +	cmpl $0,NR_READERS(%rdi)
> +	jnz  1f
> +
> +	/* Looks free. Assume transaction.
> +	   If you crash here you unlocked a free lock.  */
> +	XEND
> +	xor  %rax,%rax
> +	ret
> +
> +1:
> +
>  	/* Get the lock.  */
>  	movl	$1, %esi
>  	xorl	%eax, %eax
> diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S
> index 734bee3..28d1b4a 100644
> --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S
> +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S
> @@ -22,6 +22,7 @@
>  #include <pthread-errnos.h>
>  #include <kernel-features.h>
>  #include <stap-probe.h>
> +#include <hle.h>
>  
>  	.text
>  
> @@ -33,6 +34,34 @@ __pthread_rwlock_wrlock:
>  
>  	LIBC_PROBE (wrlock_entry, 1, %rdi)
>  
> +	cmpq  $0,RW_ELISION(%rdi)
> +	js    not_elided_wrlock
> +	jnz   2f
> +	/* zero: use default  */
> +
> +	cmpl  $0,__rwlock_rtm_enabled(%rip)
> +	jz    not_elided_wrlock
> +
> +2:
> +	XBEGIN abort_wrlock
> +
> +	/* Lock free?  */
> +	cmpl  $0,WRITER(%rdi)
> +	jnz   1f
> +	cmpl  $0,NR_READERS(%rdi)
> +	jnz   1f
> +
> +	/* Lock is free. Run with transaction  */
> +	xor   %eax,%eax
> +	ret
> +
> +	/* Lock is not free. End transaction  */
> +1:	XABORT 0xff
> +	jmp   not_elided_wrlock
> +
> +	/* Abort happened.  */
> +abort_wrlock:
> +not_elided_wrlock:
>  	xorq	%r10, %r10
>  
>  	/* Get the lock.  */
> @@ -161,5 +190,7 @@ __pthread_rwlock_wrlock:
>  	cfi_endproc
>  	.size	__pthread_rwlock_wrlock,.-__pthread_rwlock_wrlock
>  
> +#ifndef __pthread_rwlock_wrlock
>  strong_alias (__pthread_rwlock_wrlock, pthread_rwlock_wrlock)
>  hidden_def (__pthread_rwlock_wrlock)
> +#endif
> 


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]