[glibc.git] / sysdeps / unix / sysv / linux / i386 / lowlevellock.h

/* Copyright (C) 2002-2014 Free Software Foundation, Inc.
   This file is part of the GNU C Library.
   Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library; if not, see
   <http://www.gnu.org/licenses/>.  */

#ifndef _LOWLEVELLOCK_H
#define _LOWLEVELLOCK_H	1

#include <stap-probe.h>

#ifndef __ASSEMBLER__
# include <time.h>
# include <sys/param.h>
# include <bits/pthreadtypes.h>
# include <kernel-features.h>
# include <tcb-offsets.h>

# ifndef LOCK_INSTR
#  ifdef UP
#   define LOCK_INSTR	/* nothing */
#  else
#   define LOCK_INSTR "lock;"
#  endif
# endif
#else
# ifndef LOCK
#  ifdef UP
#   define LOCK
#  else
#   define LOCK lock
#  endif
# endif
#endif

#define SYS_futex		240
#define FUTEX_WAIT		0
#define FUTEX_WAKE		1
#define FUTEX_CMP_REQUEUE	4
#define FUTEX_WAKE_OP		5
#define FUTEX_LOCK_PI		6
#define FUTEX_UNLOCK_PI		7
#define FUTEX_TRYLOCK_PI	8
#define FUTEX_WAIT_BITSET	9
#define FUTEX_WAKE_BITSET	10
#define FUTEX_WAIT_REQUEUE_PI	11
#define FUTEX_CMP_REQUEUE_PI	12
#define FUTEX_PRIVATE_FLAG	128
#define FUTEX_CLOCK_REALTIME	256

#define FUTEX_BITSET_MATCH_ANY	0xffffffff

#define FUTEX_OP_CLEAR_WAKE_IF_GT_ONE	((4 << 24) | 1)

/* Values for 'private' parameter of locking macros.  Yes, the
   definition seems to be backwards.  But it is not.  The bit will be
   reversed before passing to the system call.  */
#define LLL_PRIVATE	0
#define LLL_SHARED	FUTEX_PRIVATE_FLAG


#if !defined NOT_IN_libc || defined IS_IN_rtld
/* In libc.so or ld.so all futexes are private.  */
# ifdef __ASSUME_PRIVATE_FUTEX
#  define __lll_private_flag(fl, private) \
  ((fl) | FUTEX_PRIVATE_FLAG)
# else
#  define __lll_private_flag(fl, private) \
  ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex))
# endif
#else
# ifdef __ASSUME_PRIVATE_FUTEX
#  define __lll_private_flag(fl, private) \
  (((fl) | FUTEX_PRIVATE_FLAG) ^ (private))
# else
#  define __lll_private_flag(fl, private) \
  (__builtin_constant_p (private)					      \
   ? ((private) == 0							      \
      ? ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex))	      \
      : (fl))								      \
   : ({ unsigned int __fl = ((private) ^ FUTEX_PRIVATE_FLAG);		      \
	asm ("andl %%gs:%P1, %0" : "+r" (__fl)				      \
	     : "i" (offsetof (struct pthread, header.private_futex)));	      \
	__fl | (fl); }))
# endif
#endif

#ifndef __ASSEMBLER__

/* Initializer for compatibility lock.  */
#define LLL_LOCK_INITIALIZER		(0)
#define LLL_LOCK_INITIALIZER_LOCKED	(1)
#define LLL_LOCK_INITIALIZER_WAITERS	(2)


#ifdef PIC
# define LLL_EBX_LOAD	"xchgl %2, %%ebx\n"
# define LLL_EBX_REG	"D"
#else
# define LLL_EBX_LOAD
# define LLL_EBX_REG	"b"
#endif

#ifdef I386_USE_SYSENTER
# ifdef SHARED
#  define LLL_ENTER_KERNEL	"call *%%gs:%P6\n\t"
# else
#  define LLL_ENTER_KERNEL	"call *_dl_sysinfo\n\t"
# endif
#else
# define LLL_ENTER_KERNEL	"int $0x80\n\t"
#endif

/* Delay in spinlock loop.  */
#define BUSY_WAIT_NOP	asm ("rep; nop")

#define lll_futex_wait(futex, val, private) \
  lll_futex_timed_wait (futex, val, NULL, private)


#define lll_futex_timed_wait(futex, val, timeout, private) \
  ({									      \
    int __status;							      \
    register __typeof (val) _val asm ("edx") = (val);			      \
    __asm __volatile (LLL_EBX_LOAD					      \
		      LLL_ENTER_KERNEL					      \
		      LLL_EBX_LOAD					      \
		      : "=a" (__status)					      \
		      : "0" (SYS_futex), LLL_EBX_REG (futex), "S" (timeout),  \
			"c" (__lll_private_flag (FUTEX_WAIT, private)),	      \
			"d" (_val), "i" (offsetof (tcbhead_t, sysinfo))	      \
		      : "memory");					      \
    __status;								      \
  })


#define lll_futex_wake(futex, nr, private) \
  ({									      \
    int __status;							      \
    register __typeof (nr) _nr asm ("edx") = (nr);			      \
    LIBC_PROBE (lll_futex_wake, 3, futex, nr, private);                       \
    __asm __volatile (LLL_EBX_LOAD					      \
		      LLL_ENTER_KERNEL					      \
		      LLL_EBX_LOAD					      \
		      : "=a" (__status)					      \
		      : "0" (SYS_futex), LLL_EBX_REG (futex),		      \
			"c" (__lll_private_flag (FUTEX_WAKE, private)),	      \
			"d" (_nr),					      \
			"i" (0) /* phony, to align next arg's number */,      \
			"i" (offsetof (tcbhead_t, sysinfo)));		      \
    __status;								      \
  })


/* NB: in the lll_trylock macro we simply return the value in %eax
   after the cmpxchg instruction.  In case the operation succeded this
   value is zero.  In case the operation failed, the cmpxchg instruction
   has loaded the current value of the memory work which is guaranteed
   to be nonzero.  */
#if defined NOT_IN_libc || defined UP
# define __lll_trylock_asm LOCK_INSTR "cmpxchgl %2, %1"
#else
# define __lll_trylock_asm "cmpl $0, %%gs:%P5\n\t" \
			   "je 0f\n\t"					      \
			   "lock\n"					      \
			   "0:\tcmpxchgl %2, %1"
#endif

#define lll_trylock(futex) \
  ({ int ret;								      \
     __asm __volatile (__lll_trylock_asm				      \
		       : "=a" (ret), "=m" (futex)			      \
		       : "r" (LLL_LOCK_INITIALIZER_LOCKED), "m" (futex),      \
			 "0" (LLL_LOCK_INITIALIZER),			      \
			 "i" (MULTIPLE_THREADS_OFFSET)			      \
		       : "memory");					      \
     ret; })


#define lll_cond_trylock(futex) \
  ({ int ret;								      \
     __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1"			      \
		       : "=a" (ret), "=m" (futex)			      \
		       : "r" (LLL_LOCK_INITIALIZER_WAITERS),		      \
			 "m" (futex), "0" (LLL_LOCK_INITIALIZER)	      \
		       : "memory");					      \
     ret; })

#if defined NOT_IN_libc || defined UP
# define __lll_lock_asm_start LOCK_INSTR "cmpxchgl %1, %2\n\t"
#else
# define __lll_lock_asm_start "cmpl $0, %%gs:%P6\n\t"			      \
			      "je 0f\n\t"				      \
			      "lock\n"					      \
			      "0:\tcmpxchgl %1, %2\n\t"
#endif

#define lll_lock(futex, private) \
  (void)								      \
    ({ int ignore1, ignore2;						      \
       if (__builtin_constant_p (private) && (private) == LLL_PRIVATE)	      \
	 __asm __volatile (__lll_lock_asm_start				      \
			   "jz 18f\n\t"				      \
			   "1:\tleal %2, %%ecx\n"			      \
			   "2:\tcall __lll_lock_wait_private\n" 	      \
			   "18:"					      \
			   : "=a" (ignore1), "=c" (ignore2), "=m" (futex)     \
			   : "0" (0), "1" (1), "m" (futex),		      \
			     "i" (MULTIPLE_THREADS_OFFSET)		      \
			   : "memory");					      \
       else								      \
	 {								      \
	   int ignore3;							      \
	   __asm __volatile (__lll_lock_asm_start			      \
			     "jz 18f\n\t"			 	      \
			     "1:\tleal %2, %%edx\n"			      \
			     "0:\tmovl %8, %%ecx\n"			      \
			     "2:\tcall __lll_lock_wait\n"		      \
			     "18:"					      \
			     : "=a" (ignore1), "=c" (ignore2),		      \
			       "=m" (futex), "=&d" (ignore3) 		      \
			     : "1" (1), "m" (futex),			      \
			       "i" (MULTIPLE_THREADS_OFFSET), "0" (0),	      \
			       "g" ((int) (private))			      \
			     : "memory");				      \
	 }								      \
    })

#define lll_robust_lock(futex, id, private) \
  ({ int result, ignore1, ignore2;					      \
     __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t"			      \
		       "jz 18f\n\t"					      \
		       "1:\tleal %2, %%edx\n"				      \
		       "0:\tmovl %7, %%ecx\n"				      \
		       "2:\tcall __lll_robust_lock_wait\n"		      \
		       "18:"						      \
		       : "=a" (result), "=c" (ignore1), "=m" (futex),	      \
			 "=&d" (ignore2)				      \
		       : "0" (0), "1" (id), "m" (futex), "g" ((int) (private))\
		       : "memory");					      \
     result; })


/* Special version of lll_lock which causes the unlock function to
   always wakeup waiters.  */
#define lll_cond_lock(futex, private) \
  (void)								      \
    ({ int ignore1, ignore2, ignore3;					      \
       __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t"		      \
			 "jz 18f\n\t"					      \
			 "1:\tleal %2, %%edx\n"				      \
			 "0:\tmovl %7, %%ecx\n"				      \
			 "2:\tcall __lll_lock_wait\n"			      \
			 "18:"						      \
			 : "=a" (ignore1), "=c" (ignore2), "=m" (futex),      \
			   "=&d" (ignore3)				      \
			 : "0" (0), "1" (2), "m" (futex), "g" ((int) (private))\
			 : "memory");					      \
    })


#define lll_robust_cond_lock(futex, id, private) \
  ({ int result, ignore1, ignore2;					      \
     __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t"			      \
		       "jz 18f\n\t"					      \
		       "1:\tleal %2, %%edx\n"				      \
		       "0:\tmovl %7, %%ecx\n"				      \
		       "2:\tcall __lll_robust_lock_wait\n"		      \
		       "18:"						      \
		       : "=a" (result), "=c" (ignore1), "=m" (futex),	      \
			 "=&d" (ignore2)				      \
		       : "0" (0), "1" (id | FUTEX_WAITERS), "m" (futex),      \
			 "g" ((int) (private))				      \
		       : "memory");					      \
     result; })


#define lll_timedlock(futex, timeout, private) \
  ({ int result, ignore1, ignore2, ignore3;				      \
     __asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t"			      \
		       "jz 18f\n\t"					      \
		       "1:\tleal %3, %%ecx\n"				      \
		       "0:\tmovl %8, %%edx\n"				      \
		       "2:\tcall __lll_timedlock_wait\n"		      \
		       "18:"						      \
		       : "=a" (result), "=c" (ignore1), "=&d" (ignore2),      \
			 "=m" (futex), "=S" (ignore3)			      \
		       : "0" (0), "1" (1), "m" (futex), "m" (timeout),	      \
			 "4" ((int) (private))				      \
		       : "memory");					      \
     result; })

extern int __lll_timedlock_elision (int *futex, short *adapt_count,
					 const struct timespec *timeout,
					 int private) attribute_hidden;

#define lll_timedlock_elision(futex, adapt_count, timeout, private)	\
  __lll_timedlock_elision(&(futex), &(adapt_count), timeout, private)

#define lll_robust_timedlock(futex, timeout, id, private) \
  ({ int result, ignore1, ignore2, ignore3;				      \
     __asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t"			      \
		       "jz 18f\n\t"			   		      \
		       "1:\tleal %3, %%ecx\n"				      \
		       "0:\tmovl %8, %%edx\n"				      \
		       "2:\tcall __lll_robust_timedlock_wait\n"		      \
		       "18:"						      \
		       : "=a" (result), "=c" (ignore1), "=&d" (ignore2),      \
			 "=m" (futex), "=S" (ignore3)			      \
		       : "0" (0), "1" (id), "m" (futex), "m" (timeout),	      \
			 "4" ((int) (private))				      \
		       : "memory");					      \
     result; })

#if defined NOT_IN_libc || defined UP
# define __lll_unlock_asm LOCK_INSTR "subl $1, %0\n\t"
#else
# define __lll_unlock_asm "cmpl $0, %%gs:%P3\n\t"			      \
			  "je 0f\n\t"					      \
			  "lock\n"					      \
			  "0:\tsubl $1,%0\n\t"
#endif

#define lll_unlock(futex, private) \
  (void)								      \
    ({ int ignore;							      \
       if (__builtin_constant_p (private) && (private) == LLL_PRIVATE)	      \
	 __asm __volatile (__lll_unlock_asm				      \
			   "je 18f\n\t"					      \
			   "1:\tleal %0, %%eax\n"			      \
			   "2:\tcall __lll_unlock_wake_private\n"	      \
			   "18:"					      \
			   : "=m" (futex), "=&a" (ignore)		      \
			   : "m" (futex), "i" (MULTIPLE_THREADS_OFFSET)	      \
			   : "memory");					      \
       else								      \
	 {								      \
	   int ignore2;							      \
	   __asm __volatile (__lll_unlock_asm				      \
			     "je 18f\n\t"				      \
			     "1:\tleal %0, %%eax\n"			      \
			     "0:\tmovl %5, %%ecx\n"			      \
			     "2:\tcall __lll_unlock_wake\n"		      \
			     "18:"					      \
			     : "=m" (futex), "=&a" (ignore), "=&c" (ignore2)  \
			     : "i" (MULTIPLE_THREADS_OFFSET), "m" (futex),    \
			       "g" ((int) (private))			      \
			     : "memory");				      \
	 }								      \
    })

#define lll_robust_unlock(futex, private) \
  (void)								      \
    ({ int ignore, ignore2;						      \
       __asm __volatile (LOCK_INSTR "andl %3, %0\n\t"			      \
			 "je 18f\n\t"					      \
			 "1:\tleal %0, %%eax\n"				      \
			 "0:\tmovl %5, %%ecx\n"				      \
			 "2:\tcall __lll_unlock_wake\n"			      \
			 "18:"						      \
			 : "=m" (futex), "=&a" (ignore), "=&c" (ignore2)      \
			 : "i" (FUTEX_WAITERS), "m" (futex),		      \
			   "g" ((int) (private))			      \
			 : "memory");					      \
    })


#define lll_islocked(futex) \
  (futex != LLL_LOCK_INITIALIZER)

/* The kernel notifies a process which uses CLONE_CHILD_CLEARTID via futex
   wakeup when the clone terminates.  The memory location contains the
   thread ID while the clone is running and is reset to zero
   afterwards.

   The macro parameter must not have any side effect.  */
#define lll_wait_tid(tid) \
  do {									      \
    int __ignore;							      \
    register __typeof (tid) _tid asm ("edx") = (tid);			      \
    if (_tid != 0)							      \
      __asm __volatile (LLL_EBX_LOAD					      \
			"1:\tmovl %1, %%eax\n\t"			      \
			LLL_ENTER_KERNEL				      \
			"cmpl $0, (%%ebx)\n\t"				      \
			"jne 1b\n\t"					      \
			LLL_EBX_LOAD					      \
			: "=&a" (__ignore)				      \
			: "i" (SYS_futex), LLL_EBX_REG (&tid), "S" (0),	      \
			  "c" (FUTEX_WAIT), "d" (_tid),			      \
			  "i" (offsetof (tcbhead_t, sysinfo))		      \
			: "memory");					      \
  } while (0)

extern int __lll_timedwait_tid (int *tid, const struct timespec *abstime)
     __attribute__ ((regparm (2))) attribute_hidden;
#define lll_timedwait_tid(tid, abstime) \
  ({									      \
    int __result = 0;							      \
    if (tid != 0)							      \
      {									      \
	if (abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000)	      \
	  __result = EINVAL;						      \
	else								      \
	  __result = __lll_timedwait_tid (&tid, abstime);		      \
      }									      \
    __result; })

extern int __lll_lock_elision (int *futex, short *adapt_count, int private)
  attribute_hidden;

extern int __lll_unlock_elision(int *lock, int private)
  attribute_hidden;

extern int __lll_trylock_elision(int *lock, short *adapt_count)
  attribute_hidden;

#define lll_lock_elision(futex, adapt_count, private) \
  __lll_lock_elision (&(futex), &(adapt_count), private)
#define lll_unlock_elision(futex, private) \
  __lll_unlock_elision (&(futex), private)
#define lll_trylock_elision(futex, adapt_count) \
  __lll_trylock_elision(&(futex), &(adapt_count))

#endif  /* !__ASSEMBLER__ */

#endif	/* lowlevellock.h */
Commit	Line	Data
d4697bc9	1	/* Copyright (C) 2002-2014 Free Software Foundation, Inc.
76a50749 UD	2	This file is part of the GNU C Library.
	3	Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
	4
	5	The GNU C Library is free software; you can redistribute it and/or
	6	modify it under the terms of the GNU Lesser General Public
	7	License as published by the Free Software Foundation; either
	8	version 2.1 of the License, or (at your option) any later version.
	9
	10	The GNU C Library is distributed in the hope that it will be useful,
	11	but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	Lesser General Public License for more details.
	14
	15	You should have received a copy of the GNU Lesser General Public
59ba27a6 PE	16	License along with the GNU C Library; if not, see
59ba27a6 PE	17	<http://www.gnu.org/licenses/>. */
76a50749 UD	18
	19	#ifndef _LOWLEVELLOCK_H
	20	#define _LOWLEVELLOCK_H 1
	21
17557282 RH	22	#include <stap-probe.h>
17557282 RH	23
e51deae7 UD	24	#ifndef __ASSEMBLER__
	25	# include <time.h>
	26	# include <sys/param.h>
	27	# include <bits/pthreadtypes.h>
	28	# include <kernel-features.h>
	29	# include <tcb-offsets.h>
	30
	31	# ifndef LOCK_INSTR
	32	# ifdef UP
	33	# define LOCK_INSTR /* nothing */
	34	# else
	35	# define LOCK_INSTR "lock;"
	36	# endif
	37	# endif
	38	#else
	39	# ifndef LOCK
	40	# ifdef UP
	41	# define LOCK
	42	# else
	43	# define LOCK lock
	44	# endif
76a50749 UD	45	# endif
	46	#endif
	47
	48	#define SYS_futex 240
	49	#define FUTEX_WAIT 0
	50	#define FUTEX_WAKE 1
e51deae7 UD	51	#define FUTEX_CMP_REQUEUE 4
e51deae7 UD	52	#define FUTEX_WAKE_OP 5
d5ba53f9 UD	53	#define FUTEX_LOCK_PI 6
	54	#define FUTEX_UNLOCK_PI 7
	55	#define FUTEX_TRYLOCK_PI 8
da5ac135 UD	56	#define FUTEX_WAIT_BITSET 9
da5ac135 UD	57	#define FUTEX_WAKE_BITSET 10
75956694 DG	58	#define FUTEX_WAIT_REQUEUE_PI 11
75956694 DG	59	#define FUTEX_CMP_REQUEUE_PI 12
5a8075b1	60	#define FUTEX_PRIVATE_FLAG 128
7dd650d7 UD	61	#define FUTEX_CLOCK_REALTIME 256
	62
	63	#define FUTEX_BITSET_MATCH_ANY 0xffffffff
76a50749	64
e51deae7	65	#define FUTEX_OP_CLEAR_WAKE_IF_GT_ONE ((4 << 24) \| 1)
76a50749	66
835abc5c UD	67	/* Values for 'private' parameter of locking macros. Yes, the
	68	definition seems to be backwards. But it is not. The bit will be
	69	reversed before passing to the system call. */
	70	#define LLL_PRIVATE 0
	71	#define LLL_SHARED FUTEX_PRIVATE_FLAG
	72
	73
1168be45 JJ	74	#if !defined NOT_IN_libc \|\| defined IS_IN_rtld
	75	/* In libc.so or ld.so all futexes are private. */
	76	# ifdef __ASSUME_PRIVATE_FUTEX
	77	# define __lll_private_flag(fl, private) \
	78	((fl) \| FUTEX_PRIVATE_FLAG)
	79	# else
	80	# define __lll_private_flag(fl, private) \
	81	((fl) \| THREAD_GETMEM (THREAD_SELF, header.private_futex))
	82	# endif
	83	#else
	84	# ifdef __ASSUME_PRIVATE_FUTEX
	85	# define __lll_private_flag(fl, private) \
	86	(((fl) \| FUTEX_PRIVATE_FLAG) ^ (private))
	87	# else
	88	# define __lll_private_flag(fl, private) \
	89	(__builtin_constant_p (private) \
	90	? ((private) == 0 \
	91	? ((fl) \| THREAD_GETMEM (THREAD_SELF, header.private_futex)) \
	92	: (fl)) \
	93	: ({ unsigned int __fl = ((private) ^ FUTEX_PRIVATE_FLAG); \
22502ea2	94	asm ("andl %%gs:%P1, %0" : "+r" (__fl) \
1168be45 JJ	95	: "i" (offsetof (struct pthread, header.private_futex))); \
1168be45 JJ	96	__fl \| (fl); }))
da5ac135	97	# endif
1168be45 JJ	98	#endif
1168be45 JJ	99
e51deae7	100	#ifndef __ASSEMBLER__
1168be45	101
76a50749	102	/* Initializer for compatibility lock. */
e51deae7 UD	103	#define LLL_LOCK_INITIALIZER (0)
	104	#define LLL_LOCK_INITIALIZER_LOCKED (1)
	105	#define LLL_LOCK_INITIALIZER_WAITERS (2)
76a50749 UD	106
76a50749 UD	107
949ec764 UD	108	#ifdef PIC
	109	# define LLL_EBX_LOAD "xchgl %2, %%ebx\n"
	110	# define LLL_EBX_REG "D"
	111	#else
	112	# define LLL_EBX_LOAD
	113	# define LLL_EBX_REG "b"
	114	#endif
	115
	116	#ifdef I386_USE_SYSENTER
	117	# ifdef SHARED
20945457	118	# define LLL_ENTER_KERNEL "call *%%gs:%P6\n\t"
949ec764	119	# else
20945457	120	# define LLL_ENTER_KERNEL "call *_dl_sysinfo\n\t"
949ec764 UD	121	# endif
	122	#else
	123	# define LLL_ENTER_KERNEL "int $0x80\n\t"
	124	#endif
	125
2c0b891a	126	/* Delay in spinlock loop. */
e51deae7	127	#define BUSY_WAIT_NOP asm ("rep; nop")
2c0b891a	128
835abc5c UD	129	#define lll_futex_wait(futex, val, private) \
	130	lll_futex_timed_wait (futex, val, NULL, private)
	131
	132
	133	#define lll_futex_timed_wait(futex, val, timeout, private) \
679d83ba UD	134	({ \
679d83ba UD	135	int __status; \
92d83c72	136	register __typeof (val) _val asm ("edx") = (val); \
949ec764	137	__asm __volatile (LLL_EBX_LOAD \
92d83c72	138	LLL_ENTER_KERNEL \
949ec764	139	LLL_EBX_LOAD \
679d83ba	140	: "=a" (__status) \
835abc5c	141	: "0" (SYS_futex), LLL_EBX_REG (futex), "S" (timeout), \
1168be45 JJ	142	"c" (__lll_private_flag (FUTEX_WAIT, private)), \
1168be45 JJ	143	"d" (_val), "i" (offsetof (tcbhead_t, sysinfo)) \
b576fca1	144	: "memory"); \
679d83ba UD	145	__status; \
	146	})
	147
	148
835abc5c	149	#define lll_futex_wake(futex, nr, private) \
be11d713 CD	150	({ \
be11d713 CD	151	int __status; \
835abc5c	152	register __typeof (nr) _nr asm ("edx") = (nr); \
17557282	153	LIBC_PROBE (lll_futex_wake, 3, futex, nr, private); \
835abc5c UD	154	__asm __volatile (LLL_EBX_LOAD \
	155	LLL_ENTER_KERNEL \
	156	LLL_EBX_LOAD \
be11d713	157	: "=a" (__status) \
835abc5c	158	: "0" (SYS_futex), LLL_EBX_REG (futex), \
1168be45 JJ	159	"c" (__lll_private_flag (FUTEX_WAKE, private)), \
1168be45 JJ	160	"d" (_nr), \
949ec764	161	"i" (0) /* phony, to align next arg's number */, \
92d83c72	162	"i" (offsetof (tcbhead_t, sysinfo))); \
be11d713 CD	163	__status; \
be11d713 CD	164	})
92d83c72 UD	165
92d83c72 UD	166
e51deae7	167	/* NB: in the lll_trylock macro we simply return the value in %eax
e408880b UD	168	after the cmpxchg instruction. In case the operation succeded this
	169	value is zero. In case the operation failed, the cmpxchg instruction
	170	has loaded the current value of the memory work which is guaranteed
	171	to be nonzero. */
e51deae7 UD	172	#if defined NOT_IN_libc \|\| defined UP
	173	# define __lll_trylock_asm LOCK_INSTR "cmpxchgl %2, %1"
	174	#else
	175	# define __lll_trylock_asm "cmpl $0, %%gs:%P5\n\t" \
	176	"je 0f\n\t" \
	177	"lock\n" \
	178	"0:\tcmpxchgl %2, %1"
	179	#endif
	180
	181	#define lll_trylock(futex) \
e408880b	182	({ int ret; \
e51deae7	183	__asm __volatile (__lll_trylock_asm \
76a50749	184	: "=a" (ret), "=m" (futex) \
e51deae7 UD	185	: "r" (LLL_LOCK_INITIALIZER_LOCKED), "m" (futex), \
	186	"0" (LLL_LOCK_INITIALIZER), \
	187	"i" (MULTIPLE_THREADS_OFFSET) \
7a5cdb30	188	: "memory"); \
76a50749 UD	189	ret; })
76a50749 UD	190
683040c3	191
e51deae7	192	#define lll_cond_trylock(futex) \
2c0b891a UD	193	({ int ret; \
	194	__asm __volatile (LOCK_INSTR "cmpxchgl %2, %1" \
	195	: "=a" (ret), "=m" (futex) \
e51deae7 UD	196	: "r" (LLL_LOCK_INITIALIZER_WAITERS), \
e51deae7 UD	197	"m" (futex), "0" (LLL_LOCK_INITIALIZER) \
2c0b891a UD	198	: "memory"); \
	199	ret; })
	200
e51deae7 UD	201	#if defined NOT_IN_libc \|\| defined UP
	202	# define __lll_lock_asm_start LOCK_INSTR "cmpxchgl %1, %2\n\t"
	203	#else
	204	# define __lll_lock_asm_start "cmpl $0, %%gs:%P6\n\t" \
	205	"je 0f\n\t" \
	206	"lock\n" \
	207	"0:\tcmpxchgl %1, %2\n\t"
	208	#endif
2c0b891a	209
e51deae7 UD	210	#define lll_lock(futex, private) \
	211	(void) \
	212	({ int ignore1, ignore2; \
	213	if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \
	214	__asm __volatile (__lll_lock_asm_start \
df5b85da	215	"jz 18f\n\t" \
e51deae7 UD	216	"1:\tleal %2, %%ecx\n" \
e51deae7 UD	217	"2:\tcall __lll_lock_wait_private\n" \
e51deae7 UD	218	"18:" \
	219	: "=a" (ignore1), "=c" (ignore2), "=m" (futex) \
	220	: "0" (0), "1" (1), "m" (futex), \
	221	"i" (MULTIPLE_THREADS_OFFSET) \
	222	: "memory"); \
	223	else \
	224	{ \
	225	int ignore3; \
	226	__asm __volatile (__lll_lock_asm_start \
df5b85da	227	"jz 18f\n\t" \
e51deae7 UD	228	"1:\tleal %2, %%edx\n" \
	229	"0:\tmovl %8, %%ecx\n" \
	230	"2:\tcall __lll_lock_wait\n" \
e51deae7 UD	231	"18:" \
	232	: "=a" (ignore1), "=c" (ignore2), \
	233	"=m" (futex), "=&d" (ignore3) \
	234	: "1" (1), "m" (futex), \
	235	"i" (MULTIPLE_THREADS_OFFSET), "0" (0), \
2458c748	236	"g" ((int) (private)) \
e51deae7 UD	237	: "memory"); \
	238	} \
	239	})
	240
	241	#define lll_robust_lock(futex, id, private) \
	242	({ int result, ignore1, ignore2; \
683040c3	243	__asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t" \
df5b85da	244	"jz 18f\n\t" \
e51deae7 UD	245	"1:\tleal %2, %%edx\n" \
	246	"0:\tmovl %7, %%ecx\n" \
	247	"2:\tcall __lll_robust_lock_wait\n" \
cd248c3f	248	"18:" \
e51deae7 UD	249	: "=a" (result), "=c" (ignore1), "=m" (futex), \
e51deae7 UD	250	"=&d" (ignore2) \
2458c748	251	: "0" (0), "1" (id), "m" (futex), "g" ((int) (private))\
683040c3 UD	252	: "memory"); \
	253	result; })
	254
	255
e51deae7	256	/* Special version of lll_lock which causes the unlock function to
69431c9a	257	always wakeup waiters. */
e51deae7 UD	258	#define lll_cond_lock(futex, private) \
	259	(void) \
	260	({ int ignore1, ignore2, ignore3; \
	261	__asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t" \
df5b85da	262	"jz 18f\n\t" \
e51deae7 UD	263	"1:\tleal %2, %%edx\n" \
	264	"0:\tmovl %7, %%ecx\n" \
	265	"2:\tcall __lll_lock_wait\n" \
e51deae7 UD	266	"18:" \
	267	: "=a" (ignore1), "=c" (ignore2), "=m" (futex), \
	268	"=&d" (ignore3) \
2458c748	269	: "0" (0), "1" (2), "m" (futex), "g" ((int) (private))\
e51deae7 UD	270	: "memory"); \
	271	})
	272
	273
	274	#define lll_robust_cond_lock(futex, id, private) \
	275	({ int result, ignore1, ignore2; \
683040c3	276	__asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t" \
df5b85da	277	"jz 18f\n\t" \
e51deae7 UD	278	"1:\tleal %2, %%edx\n" \
	279	"0:\tmovl %7, %%ecx\n" \
	280	"2:\tcall __lll_robust_lock_wait\n" \
cd248c3f	281	"18:" \
e51deae7 UD	282	: "=a" (result), "=c" (ignore1), "=m" (futex), \
	283	"=&d" (ignore2) \
	284	: "0" (0), "1" (id \| FUTEX_WAITERS), "m" (futex), \
2458c748	285	"g" ((int) (private)) \
683040c3 UD	286	: "memory"); \
	287	result; })
	288
	289
e51deae7 UD	290	#define lll_timedlock(futex, timeout, private) \
e51deae7 UD	291	({ int result, ignore1, ignore2, ignore3; \
71451de2	292	__asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t" \
df5b85da	293	"jz 18f\n\t" \
cd248c3f	294	"1:\tleal %3, %%ecx\n" \
e51deae7 UD	295	"0:\tmovl %8, %%edx\n" \
e51deae7 UD	296	"2:\tcall __lll_timedlock_wait\n" \
cd248c3f	297	"18:" \
71451de2	298	: "=a" (result), "=c" (ignore1), "=&d" (ignore2), \
e51deae7 UD	299	"=m" (futex), "=S" (ignore3) \
e51deae7 UD	300	: "0" (0), "1" (1), "m" (futex), "m" (timeout), \
2458c748	301	"4" ((int) (private)) \
7a5cdb30	302	: "memory"); \
76a50749 UD	303	result; })
76a50749 UD	304
1cdbe579 AK	305	extern int __lll_timedlock_elision (int futex, short adapt_count,
	306	const struct timespec *timeout,
	307	int private) attribute_hidden;
	308
	309	#define lll_timedlock_elision(futex, adapt_count, timeout, private) \
	310	__lll_timedlock_elision(&(futex), &(adapt_count), timeout, private)
76a50749	311
e51deae7 UD	312	#define lll_robust_timedlock(futex, timeout, id, private) \
e51deae7 UD	313	({ int result, ignore1, ignore2, ignore3; \
683040c3	314	__asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t" \
df5b85da	315	"jz 18f\n\t" \
cd248c3f	316	"1:\tleal %3, %%ecx\n" \
e51deae7 UD	317	"0:\tmovl %8, %%edx\n" \
e51deae7 UD	318	"2:\tcall __lll_robust_timedlock_wait\n" \
cd248c3f	319	"18:" \
683040c3	320	: "=a" (result), "=c" (ignore1), "=&d" (ignore2), \
e51deae7 UD	321	"=m" (futex), "=S" (ignore3) \
e51deae7 UD	322	: "0" (0), "1" (id), "m" (futex), "m" (timeout), \
2458c748	323	"4" ((int) (private)) \
683040c3 UD	324	: "memory"); \
	325	result; })
	326
14e7aece	327	#if defined NOT_IN_libc \|\| defined UP
e51deae7	328	# define __lll_unlock_asm LOCK_INSTR "subl $1, %0\n\t"
14e7aece	329	#else
e51deae7 UD	330	# define __lll_unlock_asm "cmpl $0, %%gs:%P3\n\t" \
	331	"je 0f\n\t" \
	332	"lock\n" \
	333	"0:\tsubl $1,%0\n\t"
14e7aece	334	#endif
76a50749	335
e51deae7 UD	336	#define lll_unlock(futex, private) \
	337	(void) \
	338	({ int ignore; \
	339	if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \
	340	__asm __volatile (__lll_unlock_asm \
df5b85da	341	"je 18f\n\t" \
e51deae7 UD	342	"1:\tleal %0, %%eax\n" \
e51deae7 UD	343	"2:\tcall __lll_unlock_wake_private\n" \
e51deae7 UD	344	"18:" \
	345	: "=m" (futex), "=&a" (ignore) \
	346	: "m" (futex), "i" (MULTIPLE_THREADS_OFFSET) \
	347	: "memory"); \
	348	else \
	349	{ \
	350	int ignore2; \
	351	__asm __volatile (__lll_unlock_asm \
df5b85da	352	"je 18f\n\t" \
e51deae7 UD	353	"1:\tleal %0, %%eax\n" \
	354	"0:\tmovl %5, %%ecx\n" \
	355	"2:\tcall __lll_unlock_wake\n" \
e51deae7 UD	356	"18:" \
	357	: "=m" (futex), "=&a" (ignore), "=&c" (ignore2) \
	358	: "i" (MULTIPLE_THREADS_OFFSET), "m" (futex), \
2458c748	359	"g" ((int) (private)) \
e51deae7 UD	360	: "memory"); \
	361	} \
	362	})
	363
	364	#define lll_robust_unlock(futex, private) \
	365	(void) \
	366	({ int ignore, ignore2; \
	367	__asm __volatile (LOCK_INSTR "andl %3, %0\n\t" \
df5b85da	368	"je 18f\n\t" \
e51deae7 UD	369	"1:\tleal %0, %%eax\n" \
	370	"0:\tmovl %5, %%ecx\n" \
	371	"2:\tcall __lll_unlock_wake\n" \
e51deae7 UD	372	"18:" \
e51deae7 UD	373	: "=m" (futex), "=&a" (ignore), "=&c" (ignore2) \
2458c748 AJ	374	: "i" (FUTEX_WAITERS), "m" (futex), \
2458c748 AJ	375	"g" ((int) (private)) \
e51deae7 UD	376	: "memory"); \
	377	})
	378
	379
76a50749	380	#define lll_islocked(futex) \
71451de2	381	(futex != LLL_LOCK_INITIALIZER)
76a50749	382
adcdc775	383	/* The kernel notifies a process which uses CLONE_CHILD_CLEARTID via futex
76a50749 UD	384	wakeup when the clone terminates. The memory location contains the
	385	thread ID while the clone is running and is reset to zero
	386	afterwards.
	387
	388	The macro parameter must not have any side effect. */
76a50749 UD	389	#define lll_wait_tid(tid) \
	390	do { \
	391	int __ignore; \
	392	register __typeof (tid) _tid asm ("edx") = (tid); \
	393	if (_tid != 0) \
949ec764	394	__asm __volatile (LLL_EBX_LOAD \
76a50749	395	"1:\tmovl %1, %%eax\n\t" \
949ec764	396	LLL_ENTER_KERNEL \
76a50749	397	"cmpl $0, (%%ebx)\n\t" \
ab9a9ff8	398	"jne 1b\n\t" \
949ec764	399	LLL_EBX_LOAD \
76a50749	400	: "=&a" (__ignore) \
949ec764	401	: "i" (SYS_futex), LLL_EBX_REG (&tid), "S" (0), \
097eca29	402	"c" (FUTEX_WAIT), "d" (_tid), \
b576fca1 UD	403	"i" (offsetof (tcbhead_t, sysinfo)) \
b576fca1 UD	404	: "memory"); \
76a50749 UD	405	} while (0)
	406
	407	extern int __lll_timedwait_tid (int tid, const struct timespec abstime)
	408	__attribute__ ((regparm (2))) attribute_hidden;
	409	#define lll_timedwait_tid(tid, abstime) \
	410	({ \
	411	int __result = 0; \
	412	if (tid != 0) \
	413	{ \
31195be2	414	if (abstime->tv_nsec < 0 \|\| abstime->tv_nsec >= 1000000000) \
76a50749 UD	415	__result = EINVAL; \
	416	else \
	417	__result = __lll_timedwait_tid (&tid, abstime); \
	418	} \
	419	__result; })
	420
1cdbe579 AK	421	extern int __lll_lock_elision (int futex, short adapt_count, int private)
	422	attribute_hidden;
	423
	424	extern int __lll_unlock_elision(int *lock, int private)
	425	attribute_hidden;
	426
	427	extern int __lll_trylock_elision(int lock, short adapt_count)
	428	attribute_hidden;
	429
	430	#define lll_lock_elision(futex, adapt_count, private) \
	431	__lll_lock_elision (&(futex), &(adapt_count), private)
	432	#define lll_unlock_elision(futex, private) \
	433	__lll_unlock_elision (&(futex), private)
	434	#define lll_trylock_elision(futex, adapt_count) \
	435	__lll_trylock_elision(&(futex), &(adapt_count))
	436
e51deae7	437	#endif /* !__ASSEMBLER__ */
76a50749 UD	438
76a50749 UD	439	#endif /* lowlevellock.h */