[glibc.git] / nptl / sysdeps / unix / sysv / linux / i386 / lowlevellock.h

/* Copyright (C) 2002-2004, 2006-2008, 2009 Free Software Foundation, Inc.
   This file is part of the GNU C Library.
   Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library; if not, write to the Free
   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
   02111-1307 USA.  */

#ifndef _LOWLEVELLOCK_H
#define _LOWLEVELLOCK_H	1

#ifndef __ASSEMBLER__
# include <time.h>
# include <sys/param.h>
# include <bits/pthreadtypes.h>
# include <kernel-features.h>
# include <tcb-offsets.h>

# ifndef LOCK_INSTR
#  ifdef UP
#   define LOCK_INSTR	/* nothing */
#  else
#   define LOCK_INSTR "lock;"
#  endif
# endif
#else
# ifndef LOCK
#  ifdef UP
#   define LOCK
#  else
#   define LOCK lock
#  endif
# endif
#endif

#define SYS_futex		240
#define FUTEX_WAIT		0
#define FUTEX_WAKE		1
#define FUTEX_CMP_REQUEUE	4
#define FUTEX_WAKE_OP		5
#define FUTEX_LOCK_PI		6
#define FUTEX_UNLOCK_PI		7
#define FUTEX_TRYLOCK_PI	8
#define FUTEX_WAIT_BITSET	9
#define FUTEX_WAKE_BITSET	10
#define FUTEX_WAIT_REQUEUE_PI	11
#define FUTEX_CMP_REQUEUE_PI	12
#define FUTEX_PRIVATE_FLAG	128
#define FUTEX_CLOCK_REALTIME	256

#define FUTEX_BITSET_MATCH_ANY	0xffffffff

#define FUTEX_OP_CLEAR_WAKE_IF_GT_ONE	((4 << 24) | 1)

/* Values for 'private' parameter of locking macros.  Yes, the
   definition seems to be backwards.  But it is not.  The bit will be
   reversed before passing to the system call.  */
#define LLL_PRIVATE	0
#define LLL_SHARED	FUTEX_PRIVATE_FLAG


#if !defined NOT_IN_libc || defined IS_IN_rtld
/* In libc.so or ld.so all futexes are private.  */
# ifdef __ASSUME_PRIVATE_FUTEX
#  define __lll_private_flag(fl, private) \
  ((fl) | FUTEX_PRIVATE_FLAG)
# else
#  define __lll_private_flag(fl, private) \
  ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex))
# endif
#else
# ifdef __ASSUME_PRIVATE_FUTEX
#  define __lll_private_flag(fl, private) \
  (((fl) | FUTEX_PRIVATE_FLAG) ^ (private))
# else
#  define __lll_private_flag(fl, private) \
  (__builtin_constant_p (private)					      \
   ? ((private) == 0							      \
      ? ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex))	      \
      : (fl))								      \
   : ({ unsigned int __fl = ((private) ^ FUTEX_PRIVATE_FLAG);		      \
	asm ("andl %%gs:%P1, %0" : "+r" (__fl)				      \
	     : "i" (offsetof (struct pthread, header.private_futex)));	      \
	__fl | (fl); }))
# endif
#endif

#ifndef __ASSEMBLER__

/* Initializer for compatibility lock.  */
#define LLL_LOCK_INITIALIZER		(0)
#define LLL_LOCK_INITIALIZER_LOCKED	(1)
#define LLL_LOCK_INITIALIZER_WAITERS	(2)


#ifdef PIC
# define LLL_EBX_LOAD	"xchgl %2, %%ebx\n"
# define LLL_EBX_REG	"D"
#else
# define LLL_EBX_LOAD
# define LLL_EBX_REG	"b"
#endif

#ifdef I386_USE_SYSENTER
# ifdef SHARED
#  define LLL_ENTER_KERNEL	"call *%%gs:%P6\n\t"
# else
#  define LLL_ENTER_KERNEL	"call *_dl_sysinfo\n\t"
# endif
#else
# define LLL_ENTER_KERNEL	"int $0x80\n\t"
#endif

/* Delay in spinlock loop.  */
#define BUSY_WAIT_NOP	asm ("rep; nop")


#define LLL_STUB_UNWIND_INFO_START \
	".section	.eh_frame,\"a\",@progbits\n"		\
"5:\t"	".long	7f-6f	# Length of Common Information Entry\n"	\
"6:\t"	".long	0x0	# CIE Identifier Tag\n\t"		\
	".byte	0x1	# CIE Version\n\t"			\
	".ascii \"zR\\0\"	# CIE Augmentation\n\t"		\
	".uleb128 0x1	# CIE Code Alignment Factor\n\t"	\
	".sleb128 -4	# CIE Data Alignment Factor\n\t"	\
	".byte	0x8	# CIE RA Column\n\t"			\
	".uleb128 0x1	# Augmentation size\n\t"		\
	".byte	0x1b	# FDE Encoding (pcrel sdata4)\n\t"	\
	".byte	0xc	# DW_CFA_def_cfa\n\t"			\
	".uleb128 0x4\n\t"					\
	".uleb128 0x0\n\t"					\
	".align 4\n"						\
"7:\t"	".long	17f-8f	# FDE Length\n"				\
"8:\t"	".long	8b-5b	# FDE CIE offset\n\t"			\
	".long	1b-.	# FDE initial location\n\t"		\
	".long	4b-1b	# FDE address range\n\t"		\
	".uleb128 0x0	# Augmentation size\n\t"		\
	".byte	0x16	# DW_CFA_val_expression\n\t"		\
	".uleb128 0x8\n\t"					\
	".uleb128 10f-9f\n"					\
"9:\t"	".byte	0x78	# DW_OP_breg8\n\t"			\
	".sleb128 3b-1b\n"
#define LLL_STUB_UNWIND_INFO_END \
	".byte	0x16	# DW_CFA_val_expression\n\t"		\
	".uleb128 0x8\n\t"					\
	".uleb128 12f-11f\n"					\
"11:\t"	".byte	0x78	# DW_OP_breg8\n\t"			\
	".sleb128 3b-2b\n"					\
"12:\t"	".byte	0x40 + (3b-2b-1) # DW_CFA_advance_loc\n\t"	\
	".byte	0x16	# DW_CFA_val_expression\n\t"		\
	".uleb128 0x8\n\t"					\
	".uleb128 16f-13f\n"					\
"13:\t"	".byte	0x78	# DW_OP_breg8\n\t"			\
	".sleb128 15f-14f\n\t"					\
	".byte	0x0d	# DW_OP_const4s\n"			\
"14:\t"	".4byte	3b-.\n\t"					\
	".byte	0x1c	# DW_OP_minus\n\t"			\
	".byte	0x0d	# DW_OP_const4s\n"			\
"15:\t"	".4byte	18f-.\n\t"					\
	".byte	0x22	# DW_OP_plus\n"				\
"16:\t"	".align 4\n"						\
"17:\t"	".previous\n"

/* Unwind info for
   1: lea ..., ...
   2: call ...
   3: jmp 18f
   4:
   snippet.  */
#define LLL_STUB_UNWIND_INFO_3 \
LLL_STUB_UNWIND_INFO_START					\
"10:\t"	".byte	0x40 + (2b-1b) # DW_CFA_advance_loc\n\t"	\
LLL_STUB_UNWIND_INFO_END

/* Unwind info for
   1: lea ..., ...
   0: movl ..., ...
   2: call ...
   3: jmp 18f
   4:
   snippet.  */
#define LLL_STUB_UNWIND_INFO_4 \
LLL_STUB_UNWIND_INFO_START					\
"10:\t"	".byte	0x40 + (0b-1b) # DW_CFA_advance_loc\n\t"	\
	".byte	0x16	# DW_CFA_val_expression\n\t"		\
	".uleb128 0x8\n\t"					\
	".uleb128 20f-19f\n"					\
"19:\t"	".byte	0x78	# DW_OP_breg8\n\t"			\
	".sleb128 3b-0b\n"					\
"20:\t"	".byte	0x40 + (2b-0b) # DW_CFA_advance_loc\n\t"	\
LLL_STUB_UNWIND_INFO_END


#define lll_futex_wait(futex, val, private) \
  lll_futex_timed_wait (futex, val, NULL, private)


#define lll_futex_timed_wait(futex, val, timeout, private) \
  ({									      \
    int __status;							      \
    register __typeof (val) _val asm ("edx") = (val);			      \
    __asm __volatile (LLL_EBX_LOAD					      \
		      LLL_ENTER_KERNEL					      \
		      LLL_EBX_LOAD					      \
		      : "=a" (__status)					      \
		      : "0" (SYS_futex), LLL_EBX_REG (futex), "S" (timeout),  \
			"c" (__lll_private_flag (FUTEX_WAIT, private)),	      \
			"d" (_val), "i" (offsetof (tcbhead_t, sysinfo))	      \
		      : "memory");					      \
    __status;								      \
  })


#define lll_futex_wake(futex, nr, private) \
  do {									      \
    int __ignore;							      \
    register __typeof (nr) _nr asm ("edx") = (nr);			      \
    __asm __volatile (LLL_EBX_LOAD					      \
		      LLL_ENTER_KERNEL					      \
		      LLL_EBX_LOAD					      \
		      : "=a" (__ignore)					      \
		      : "0" (SYS_futex), LLL_EBX_REG (futex),		      \
			"c" (__lll_private_flag (FUTEX_WAKE, private)),	      \
			"d" (_nr),					      \
			"i" (0) /* phony, to align next arg's number */,      \
			"i" (offsetof (tcbhead_t, sysinfo)));		      \
  } while (0)


/* NB: in the lll_trylock macro we simply return the value in %eax
   after the cmpxchg instruction.  In case the operation succeded this
   value is zero.  In case the operation failed, the cmpxchg instruction
   has loaded the current value of the memory work which is guaranteed
   to be nonzero.  */
#if defined NOT_IN_libc || defined UP
# define __lll_trylock_asm LOCK_INSTR "cmpxchgl %2, %1"
#else
# define __lll_trylock_asm "cmpl $0, %%gs:%P5\n\t" \
			   "je 0f\n\t"					      \
			   "lock\n"					      \
			   "0:\tcmpxchgl %2, %1"
#endif

#define lll_trylock(futex) \
  ({ int ret;								      \
     __asm __volatile (__lll_trylock_asm				      \
		       : "=a" (ret), "=m" (futex)			      \
		       : "r" (LLL_LOCK_INITIALIZER_LOCKED), "m" (futex),      \
			 "0" (LLL_LOCK_INITIALIZER),			      \
			 "i" (MULTIPLE_THREADS_OFFSET)			      \
		       : "memory");					      \
     ret; })

#define lll_robust_trylock(futex, id) \
  ({ int ret;								      \
     __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1"			      \
		       : "=a" (ret), "=m" (futex)			      \
		       : "r" (id), "m" (futex),				      \
			 "0" (LLL_LOCK_INITIALIZER)			      \
		       : "memory");					      \
     ret; })


#define lll_cond_trylock(futex) \
  ({ int ret;								      \
     __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1"			      \
		       : "=a" (ret), "=m" (futex)			      \
		       : "r" (LLL_LOCK_INITIALIZER_WAITERS),		      \
			 "m" (futex), "0" (LLL_LOCK_INITIALIZER)	      \
		       : "memory");					      \
     ret; })

#if defined NOT_IN_libc || defined UP
# define __lll_lock_asm_start LOCK_INSTR "cmpxchgl %1, %2\n\t"
#else
# define __lll_lock_asm_start "cmpl $0, %%gs:%P6\n\t"			      \
			      "je 0f\n\t"				      \
			      "lock\n"					      \
			      "0:\tcmpxchgl %1, %2\n\t"
#endif

#define lll_lock(futex, private) \
  (void)								      \
    ({ int ignore1, ignore2;						      \
       if (__builtin_constant_p (private) && (private) == LLL_PRIVATE)	      \
	 __asm __volatile (__lll_lock_asm_start				      \
			   "jnz _L_lock_%=\n\t"				      \
			   ".subsection 1\n\t"				      \
			   ".type _L_lock_%=,@function\n"		      \
			   "_L_lock_%=:\n"				      \
			   "1:\tleal %2, %%ecx\n"			      \
			   "2:\tcall __lll_lock_wait_private\n" 	      \
			   "3:\tjmp 18f\n"				      \
			   "4:\t.size _L_lock_%=, 4b-1b\n\t"		      \
			   ".previous\n"				      \
			   LLL_STUB_UNWIND_INFO_3			      \
			   "18:"					      \
			   : "=a" (ignore1), "=c" (ignore2), "=m" (futex)     \
			   : "0" (0), "1" (1), "m" (futex),		      \
			     "i" (MULTIPLE_THREADS_OFFSET)		      \
			   : "memory");					      \
       else								      \
	 {								      \
	   int ignore3;							      \
	   __asm __volatile (__lll_lock_asm_start			      \
			     "jnz _L_lock_%=\n\t"			      \
			     ".subsection 1\n\t"			      \
			     ".type _L_lock_%=,@function\n"		      \
			     "_L_lock_%=:\n"				      \
			     "1:\tleal %2, %%edx\n"			      \
			     "0:\tmovl %8, %%ecx\n"			      \
			     "2:\tcall __lll_lock_wait\n"		      \
			     "3:\tjmp 18f\n"				      \
			     "4:\t.size _L_lock_%=, 4b-1b\n\t"		      \
			     ".previous\n"				      \
			     LLL_STUB_UNWIND_INFO_4			      \
			     "18:"					      \
			     : "=a" (ignore1), "=c" (ignore2),		      \
			       "=m" (futex), "=&d" (ignore3) 		      \
			     : "1" (1), "m" (futex),			      \
			       "i" (MULTIPLE_THREADS_OFFSET), "0" (0),	      \
			       "g" ((int) (private))			      \
			     : "memory");				      \
	 }								      \
    })

#define lll_robust_lock(futex, id, private) \
  ({ int result, ignore1, ignore2;					      \
     __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t"			      \
		       "jnz _L_robust_lock_%=\n\t"			      \
		       ".subsection 1\n\t"				      \
		       ".type _L_robust_lock_%=,@function\n"		      \
		       "_L_robust_lock_%=:\n"				      \
		       "1:\tleal %2, %%edx\n"				      \
		       "0:\tmovl %7, %%ecx\n"				      \
		       "2:\tcall __lll_robust_lock_wait\n"		      \
		       "3:\tjmp 18f\n"					      \
		       "4:\t.size _L_robust_lock_%=, 4b-1b\n\t"		      \
		       ".previous\n"					      \
		       LLL_STUB_UNWIND_INFO_4				      \
		       "18:"						      \
		       : "=a" (result), "=c" (ignore1), "=m" (futex),	      \
			 "=&d" (ignore2)				      \
		       : "0" (0), "1" (id), "m" (futex), "g" ((int) (private))\
		       : "memory");					      \
     result; })


/* Special version of lll_lock which causes the unlock function to
   always wakeup waiters.  */
#define lll_cond_lock(futex, private) \
  (void)								      \
    ({ int ignore1, ignore2, ignore3;					      \
       __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t"		      \
			 "jnz _L_cond_lock_%=\n\t"			      \
			 ".subsection 1\n\t"				      \
			 ".type _L_cond_lock_%=,@function\n"		      \
			 "_L_cond_lock_%=:\n"				      \
			 "1:\tleal %2, %%edx\n"				      \
			 "0:\tmovl %7, %%ecx\n"				      \
			 "2:\tcall __lll_lock_wait\n"			      \
			 "3:\tjmp 18f\n"				      \
			 "4:\t.size _L_cond_lock_%=, 4b-1b\n\t"		      \
			 ".previous\n"					      \
			 LLL_STUB_UNWIND_INFO_4				      \
			 "18:"						      \
			 : "=a" (ignore1), "=c" (ignore2), "=m" (futex),      \
			   "=&d" (ignore3)				      \
			 : "0" (0), "1" (2), "m" (futex), "g" ((int) (private))\
			 : "memory");					      \
    })


#define lll_robust_cond_lock(futex, id, private) \
  ({ int result, ignore1, ignore2;					      \
     __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t"			      \
		       "jnz _L_robust_cond_lock_%=\n\t"			      \
		       ".subsection 1\n\t"				      \
		       ".type _L_robust_cond_lock_%=,@function\n"	      \
		       "_L_robust_cond_lock_%=:\n"			      \
		       "1:\tleal %2, %%edx\n"				      \
		       "0:\tmovl %7, %%ecx\n"				      \
		       "2:\tcall __lll_robust_lock_wait\n"		      \
		       "3:\tjmp 18f\n"					      \
		       "4:\t.size _L_robust_cond_lock_%=, 4b-1b\n\t"	      \
		       ".previous\n"					      \
		       LLL_STUB_UNWIND_INFO_4				      \
		       "18:"						      \
		       : "=a" (result), "=c" (ignore1), "=m" (futex),	      \
			 "=&d" (ignore2)				      \
		       : "0" (0), "1" (id | FUTEX_WAITERS), "m" (futex),      \
			 "g" ((int) (private))				      \
		       : "memory");					      \
     result; })


#define lll_timedlock(futex, timeout, private) \
  ({ int result, ignore1, ignore2, ignore3;				      \
     __asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t"			      \
		       "jnz _L_timedlock_%=\n\t"			      \
		       ".subsection 1\n\t"				      \
		       ".type _L_timedlock_%=,@function\n"		      \
		       "_L_timedlock_%=:\n"				      \
		       "1:\tleal %3, %%ecx\n"				      \
		       "0:\tmovl %8, %%edx\n"				      \
		       "2:\tcall __lll_timedlock_wait\n"		      \
		       "3:\tjmp 18f\n"					      \
		       "4:\t.size _L_timedlock_%=, 4b-1b\n\t"		      \
		       ".previous\n"					      \
		       LLL_STUB_UNWIND_INFO_4				      \
		       "18:"						      \
		       : "=a" (result), "=c" (ignore1), "=&d" (ignore2),      \
			 "=m" (futex), "=S" (ignore3)			      \
		       : "0" (0), "1" (1), "m" (futex), "m" (timeout),	      \
			 "4" ((int) (private))				      \
		       : "memory");					      \
     result; })


#define lll_robust_timedlock(futex, timeout, id, private) \
  ({ int result, ignore1, ignore2, ignore3;				      \
     __asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t"			      \
		       "jnz _L_robust_timedlock_%=\n\t"			      \
		       ".subsection 1\n\t"				      \
		       ".type _L_robust_timedlock_%=,@function\n"	      \
		       "_L_robust_timedlock_%=:\n"			      \
		       "1:\tleal %3, %%ecx\n"				      \
		       "0:\tmovl %8, %%edx\n"				      \
		       "2:\tcall __lll_robust_timedlock_wait\n"		      \
		       "3:\tjmp 18f\n"					      \
		       "4:\t.size _L_robust_timedlock_%=, 4b-1b\n\t"	      \
		       ".previous\n"					      \
		       LLL_STUB_UNWIND_INFO_4				      \
		       "18:"						      \
		       : "=a" (result), "=c" (ignore1), "=&d" (ignore2),      \
			 "=m" (futex), "=S" (ignore3)			      \
		       : "0" (0), "1" (id), "m" (futex), "m" (timeout),	      \
			 "4" ((int) (private))				      \
		       : "memory");					      \
     result; })

#if defined NOT_IN_libc || defined UP
# define __lll_unlock_asm LOCK_INSTR "subl $1, %0\n\t"
#else
# define __lll_unlock_asm "cmpl $0, %%gs:%P3\n\t"			      \
			  "je 0f\n\t"					      \
			  "lock\n"					      \
			  "0:\tsubl $1,%0\n\t"
#endif

#define lll_unlock(futex, private) \
  (void)								      \
    ({ int ignore;							      \
       if (__builtin_constant_p (private) && (private) == LLL_PRIVATE)	      \
	 __asm __volatile (__lll_unlock_asm				      \
			   "jne _L_unlock_%=\n\t"			      \
			   ".subsection 1\n\t"				      \
			   ".type _L_unlock_%=,@function\n"		      \
			   "_L_unlock_%=:\n"				      \
			   "1:\tleal %0, %%eax\n"			      \
			   "2:\tcall __lll_unlock_wake_private\n"	      \
			   "3:\tjmp 18f\n"				      \
			   "4:\t.size _L_unlock_%=, 4b-1b\n\t"		      \
			   ".previous\n"				      \
			   LLL_STUB_UNWIND_INFO_3			      \
			   "18:"					      \
			   : "=m" (futex), "=&a" (ignore)		      \
			   : "m" (futex), "i" (MULTIPLE_THREADS_OFFSET)	      \
			   : "memory");					      \
       else								      \
	 {								      \
	   int ignore2;							      \
	   __asm __volatile (__lll_unlock_asm				      \
			     "jne _L_unlock_%=\n\t"			      \
			     ".subsection 1\n\t"			      \
			     ".type _L_unlock_%=,@function\n"		      \
			     "_L_unlock_%=:\n"				      \
			     "1:\tleal %0, %%eax\n"			      \
			     "0:\tmovl %5, %%ecx\n"			      \
			     "2:\tcall __lll_unlock_wake\n"		      \
			     "3:\tjmp 18f\n"				      \
			     "4:\t.size _L_unlock_%=, 4b-1b\n\t"	      \
			     ".previous\n"				      \
			     LLL_STUB_UNWIND_INFO_4			      \
			     "18:"					      \
			     : "=m" (futex), "=&a" (ignore), "=&c" (ignore2)  \
			     : "i" (MULTIPLE_THREADS_OFFSET), "m" (futex),    \
			       "g" ((int) (private))			      \
			     : "memory");				      \
	 }								      \
    })

#define lll_robust_unlock(futex, private) \
  (void)								      \
    ({ int ignore, ignore2;						      \
       __asm __volatile (LOCK_INSTR "andl %3, %0\n\t"			      \
			 "jne _L_robust_unlock_%=\n\t"			      \
			 ".subsection 1\n\t"				      \
			 ".type _L_robust_unlock_%=,@function\n"	      \
			 "_L_robust_unlock_%=:\n\t"			      \
			 "1:\tleal %0, %%eax\n"				      \
			 "0:\tmovl %5, %%ecx\n"				      \
			 "2:\tcall __lll_unlock_wake\n"			      \
			 "3:\tjmp 18f\n"				      \
			 "4:\t.size _L_robust_unlock_%=, 4b-1b\n\t"	      \
			 ".previous\n"					      \
			 LLL_STUB_UNWIND_INFO_4				      \
			 "18:"						      \
			 : "=m" (futex), "=&a" (ignore), "=&c" (ignore2)      \
			 : "i" (FUTEX_WAITERS), "m" (futex),		      \
			   "g" ((int) (private))			      \
			 : "memory");					      \
    })


#define lll_robust_dead(futex, private) \
  (void)								      \
    ({ int __ignore;							      \
       register int _nr asm ("edx") = 1;				      \
       __asm __volatile (LOCK_INSTR "orl %5, (%2)\n\t"			      \
			 LLL_EBX_LOAD					      \
			 LLL_ENTER_KERNEL				      \
			 LLL_EBX_LOAD					      \
			 : "=a" (__ignore)				      \
			 : "0" (SYS_futex), LLL_EBX_REG (&(futex)),	      \
			   "c" (__lll_private_flag (FUTEX_WAKE, private)),    \
			   "d" (_nr), "i" (FUTEX_OWNER_DIED),		      \
			   "i" (offsetof (tcbhead_t, sysinfo)));	      \
    })

#define lll_islocked(futex) \
  (futex != LLL_LOCK_INITIALIZER)

/* The kernel notifies a process with uses CLONE_CLEARTID via futex
   wakeup when the clone terminates.  The memory location contains the
   thread ID while the clone is running and is reset to zero
   afterwards.

   The macro parameter must not have any side effect.  */
#define lll_wait_tid(tid) \
  do {									      \
    int __ignore;							      \
    register __typeof (tid) _tid asm ("edx") = (tid);			      \
    if (_tid != 0)							      \
      __asm __volatile (LLL_EBX_LOAD					      \
			"1:\tmovl %1, %%eax\n\t"			      \
			LLL_ENTER_KERNEL				      \
			"cmpl $0, (%%ebx)\n\t"				      \
			"jne 1b\n\t"					      \
			LLL_EBX_LOAD					      \
			: "=&a" (__ignore)				      \
			: "i" (SYS_futex), LLL_EBX_REG (&tid), "S" (0),	      \
			  "c" (FUTEX_WAIT), "d" (_tid),			      \
			  "i" (offsetof (tcbhead_t, sysinfo))		      \
			: "memory");					      \
  } while (0)

extern int __lll_timedwait_tid (int *tid, const struct timespec *abstime)
     __attribute__ ((regparm (2))) attribute_hidden;
#define lll_timedwait_tid(tid, abstime) \
  ({									      \
    int __result = 0;							      \
    if (tid != 0)							      \
      {									      \
	if (abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000)	      \
	  __result = EINVAL;						      \
	else								      \
	  __result = __lll_timedwait_tid (&tid, abstime);		      \
      }									      \
    __result; })

#endif  /* !__ASSEMBLER__ */

#endif	/* lowlevellock.h */
Commit	Line	Data
7dd650d7	1	/* Copyright (C) 2002-2004, 2006-2008, 2009 Free Software Foundation, Inc.
76a50749 UD	2	This file is part of the GNU C Library.
	3	Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
	4
	5	The GNU C Library is free software; you can redistribute it and/or
	6	modify it under the terms of the GNU Lesser General Public
	7	License as published by the Free Software Foundation; either
	8	version 2.1 of the License, or (at your option) any later version.
	9
	10	The GNU C Library is distributed in the hope that it will be useful,
	11	but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	Lesser General Public License for more details.
	14
	15	You should have received a copy of the GNU Lesser General Public
	16	License along with the GNU C Library; if not, write to the Free
	17	Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
	18	02111-1307 USA. */
	19
	20	#ifndef _LOWLEVELLOCK_H
	21	#define _LOWLEVELLOCK_H 1
	22
e51deae7 UD	23	#ifndef __ASSEMBLER__
	24	# include <time.h>
	25	# include <sys/param.h>
	26	# include <bits/pthreadtypes.h>
	27	# include <kernel-features.h>
	28	# include <tcb-offsets.h>
	29
	30	# ifndef LOCK_INSTR
	31	# ifdef UP
	32	# define LOCK_INSTR /* nothing */
	33	# else
	34	# define LOCK_INSTR "lock;"
	35	# endif
	36	# endif
	37	#else
	38	# ifndef LOCK
	39	# ifdef UP
	40	# define LOCK
	41	# else
	42	# define LOCK lock
	43	# endif
76a50749 UD	44	# endif
	45	#endif
	46
	47	#define SYS_futex 240
	48	#define FUTEX_WAIT 0
	49	#define FUTEX_WAKE 1
e51deae7 UD	50	#define FUTEX_CMP_REQUEUE 4
e51deae7 UD	51	#define FUTEX_WAKE_OP 5
d5ba53f9 UD	52	#define FUTEX_LOCK_PI 6
	53	#define FUTEX_UNLOCK_PI 7
	54	#define FUTEX_TRYLOCK_PI 8
da5ac135 UD	55	#define FUTEX_WAIT_BITSET 9
da5ac135 UD	56	#define FUTEX_WAKE_BITSET 10
75956694 DG	57	#define FUTEX_WAIT_REQUEUE_PI 11
75956694 DG	58	#define FUTEX_CMP_REQUEUE_PI 12
5a8075b1	59	#define FUTEX_PRIVATE_FLAG 128
7dd650d7 UD	60	#define FUTEX_CLOCK_REALTIME 256
	61
	62	#define FUTEX_BITSET_MATCH_ANY 0xffffffff
76a50749	63
e51deae7	64	#define FUTEX_OP_CLEAR_WAKE_IF_GT_ONE ((4 << 24) \| 1)
76a50749	65
835abc5c UD	66	/* Values for 'private' parameter of locking macros. Yes, the
	67	definition seems to be backwards. But it is not. The bit will be
	68	reversed before passing to the system call. */
	69	#define LLL_PRIVATE 0
	70	#define LLL_SHARED FUTEX_PRIVATE_FLAG
	71
	72
1168be45 JJ	73	#if !defined NOT_IN_libc \|\| defined IS_IN_rtld
	74	/* In libc.so or ld.so all futexes are private. */
	75	# ifdef __ASSUME_PRIVATE_FUTEX
	76	# define __lll_private_flag(fl, private) \
	77	((fl) \| FUTEX_PRIVATE_FLAG)
	78	# else
	79	# define __lll_private_flag(fl, private) \
	80	((fl) \| THREAD_GETMEM (THREAD_SELF, header.private_futex))
	81	# endif
	82	#else
	83	# ifdef __ASSUME_PRIVATE_FUTEX
	84	# define __lll_private_flag(fl, private) \
	85	(((fl) \| FUTEX_PRIVATE_FLAG) ^ (private))
	86	# else
	87	# define __lll_private_flag(fl, private) \
	88	(__builtin_constant_p (private) \
	89	? ((private) == 0 \
	90	? ((fl) \| THREAD_GETMEM (THREAD_SELF, header.private_futex)) \
	91	: (fl)) \
	92	: ({ unsigned int __fl = ((private) ^ FUTEX_PRIVATE_FLAG); \
22502ea2	93	asm ("andl %%gs:%P1, %0" : "+r" (__fl) \
1168be45 JJ	94	: "i" (offsetof (struct pthread, header.private_futex))); \
1168be45 JJ	95	__fl \| (fl); }))
da5ac135	96	# endif
1168be45 JJ	97	#endif
1168be45 JJ	98
e51deae7	99	#ifndef __ASSEMBLER__
1168be45	100
76a50749	101	/* Initializer for compatibility lock. */
e51deae7 UD	102	#define LLL_LOCK_INITIALIZER (0)
	103	#define LLL_LOCK_INITIALIZER_LOCKED (1)
	104	#define LLL_LOCK_INITIALIZER_WAITERS (2)
76a50749 UD	105
76a50749 UD	106
949ec764 UD	107	#ifdef PIC
	108	# define LLL_EBX_LOAD "xchgl %2, %%ebx\n"
	109	# define LLL_EBX_REG "D"
	110	#else
	111	# define LLL_EBX_LOAD
	112	# define LLL_EBX_REG "b"
	113	#endif
	114
	115	#ifdef I386_USE_SYSENTER
	116	# ifdef SHARED
20945457	117	# define LLL_ENTER_KERNEL "call *%%gs:%P6\n\t"
949ec764	118	# else
20945457	119	# define LLL_ENTER_KERNEL "call *_dl_sysinfo\n\t"
949ec764 UD	120	# endif
	121	#else
	122	# define LLL_ENTER_KERNEL "int $0x80\n\t"
	123	#endif
	124
2c0b891a	125	/* Delay in spinlock loop. */
e51deae7	126	#define BUSY_WAIT_NOP asm ("rep; nop")
2c0b891a	127
92d83c72	128
cd248c3f UD	129	#define LLL_STUB_UNWIND_INFO_START \
	130	".section .eh_frame,\"a\",@progbits\n" \
	131	"5:\t" ".long 7f-6f # Length of Common Information Entry\n" \
	132	"6:\t" ".long 0x0 # CIE Identifier Tag\n\t" \
	133	".byte 0x1 # CIE Version\n\t" \
	134	".ascii \"zR\\0\" # CIE Augmentation\n\t" \
	135	".uleb128 0x1 # CIE Code Alignment Factor\n\t" \
	136	".sleb128 -4 # CIE Data Alignment Factor\n\t" \
	137	".byte 0x8 # CIE RA Column\n\t" \
	138	".uleb128 0x1 # Augmentation size\n\t" \
	139	".byte 0x1b # FDE Encoding (pcrel sdata4)\n\t" \
	140	".byte 0xc # DW_CFA_def_cfa\n\t" \
	141	".uleb128 0x4\n\t" \
	142	".uleb128 0x0\n\t" \
	143	".align 4\n" \
	144	"7:\t" ".long 17f-8f # FDE Length\n" \
	145	"8:\t" ".long 8b-5b # FDE CIE offset\n\t" \
	146	".long 1b-. # FDE initial location\n\t" \
	147	".long 4b-1b # FDE address range\n\t" \
	148	".uleb128 0x0 # Augmentation size\n\t" \
	149	".byte 0x16 # DW_CFA_val_expression\n\t" \
	150	".uleb128 0x8\n\t" \
	151	".uleb128 10f-9f\n" \
	152	"9:\t" ".byte 0x78 # DW_OP_breg8\n\t" \
	153	".sleb128 3b-1b\n"
	154	#define LLL_STUB_UNWIND_INFO_END \
	155	".byte 0x16 # DW_CFA_val_expression\n\t" \
	156	".uleb128 0x8\n\t" \
	157	".uleb128 12f-11f\n" \
	158	"11:\t" ".byte 0x78 # DW_OP_breg8\n\t" \
	159	".sleb128 3b-2b\n" \
	160	"12:\t" ".byte 0x40 + (3b-2b-1) # DW_CFA_advance_loc\n\t" \
	161	".byte 0x16 # DW_CFA_val_expression\n\t" \
	162	".uleb128 0x8\n\t" \
	163	".uleb128 16f-13f\n" \
	164	"13:\t" ".byte 0x78 # DW_OP_breg8\n\t" \
	165	".sleb128 15f-14f\n\t" \
	166	".byte 0x0d # DW_OP_const4s\n" \
	167	"14:\t" ".4byte 3b-.\n\t" \
	168	".byte 0x1c # DW_OP_minus\n\t" \
	169	".byte 0x0d # DW_OP_const4s\n" \
	170	"15:\t" ".4byte 18f-.\n\t" \
	171	".byte 0x22 # DW_OP_plus\n" \
	172	"16:\t" ".align 4\n" \
	173	"17:\t" ".previous\n"
	174
	175	/* Unwind info for
	176	1: lea ..., ...
	177	2: call ...
	178	3: jmp 18f
	179	4:
	180	snippet. */
	181	#define LLL_STUB_UNWIND_INFO_3 \
	182	LLL_STUB_UNWIND_INFO_START \
	183	"10:\t" ".byte 0x40 + (2b-1b) # DW_CFA_advance_loc\n\t" \
	184	LLL_STUB_UNWIND_INFO_END
	185
	186	/* Unwind info for
	187	1: lea ..., ...
	188	0: movl ..., ...
	189	2: call ...
	190	3: jmp 18f
	191	4:
	192	snippet. */
193	#define LLL_STUB_UNWIND_INFO_4 \
194	LLL_STUB_UNWIND_INFO_START \
195	"10:\t" ".byte 0x40 + (0b-1b) # DW_CFA_advance_loc\n\t" \
196	".byte 0x16 # DW_CFA_val_expression\n\t" \
197	".uleb128 0x8\n\t" \
198	".uleb128 20f-19f\n" \
199	"19:\t" ".byte 0x78 # DW_OP_breg8\n\t" \
200	".sleb128 3b-0b\n" \
201	"20:\t" ".byte 0x40 + (2b-0b) # DW_CFA_advance_loc\n\t" \
202	LLL_STUB_UNWIND_INFO_END
203
204
835abc5c UD	205	#define lll_futex_wait(futex, val, private) \
	206	lll_futex_timed_wait (futex, val, NULL, private)
	207
	208
	209	#define lll_futex_timed_wait(futex, val, timeout, private) \
679d83ba UD	210	({ \
679d83ba UD	211	int __status; \
92d83c72	212	register __typeof (val) _val asm ("edx") = (val); \
949ec764	213	__asm __volatile (LLL_EBX_LOAD \
92d83c72	214	LLL_ENTER_KERNEL \
949ec764	215	LLL_EBX_LOAD \
679d83ba	216	: "=a" (__status) \
835abc5c	217	: "0" (SYS_futex), LLL_EBX_REG (futex), "S" (timeout), \
1168be45 JJ	218	"c" (__lll_private_flag (FUTEX_WAIT, private)), \
1168be45 JJ	219	"d" (_val), "i" (offsetof (tcbhead_t, sysinfo)) \
b576fca1	220	: "memory"); \
679d83ba UD	221	__status; \
	222	})
	223
	224
835abc5c UD	225	#define lll_futex_wake(futex, nr, private) \
	226	do { \
	227	int __ignore; \
	228	register __typeof (nr) _nr asm ("edx") = (nr); \
	229	__asm __volatile (LLL_EBX_LOAD \
	230	LLL_ENTER_KERNEL \
	231	LLL_EBX_LOAD \
	232	: "=a" (__ignore) \
	233	: "0" (SYS_futex), LLL_EBX_REG (futex), \
1168be45 JJ	234	"c" (__lll_private_flag (FUTEX_WAKE, private)), \
1168be45 JJ	235	"d" (_nr), \
949ec764	236	"i" (0) /* phony, to align next arg's number */, \
92d83c72 UD	237	"i" (offsetof (tcbhead_t, sysinfo))); \
	238	} while (0)
	239
	240
e51deae7	241	/* NB: in the lll_trylock macro we simply return the value in %eax
e408880b UD	242	after the cmpxchg instruction. In case the operation succeded this
	243	value is zero. In case the operation failed, the cmpxchg instruction
	244	has loaded the current value of the memory work which is guaranteed
	245	to be nonzero. */
e51deae7 UD	246	#if defined NOT_IN_libc \|\| defined UP
	247	# define __lll_trylock_asm LOCK_INSTR "cmpxchgl %2, %1"
	248	#else
	249	# define __lll_trylock_asm "cmpl $0, %%gs:%P5\n\t" \
	250	"je 0f\n\t" \
	251	"lock\n" \
	252	"0:\tcmpxchgl %2, %1"
	253	#endif
	254
	255	#define lll_trylock(futex) \
e408880b	256	({ int ret; \
e51deae7	257	__asm __volatile (__lll_trylock_asm \
76a50749	258	: "=a" (ret), "=m" (futex) \
e51deae7 UD	259	: "r" (LLL_LOCK_INITIALIZER_LOCKED), "m" (futex), \
	260	"0" (LLL_LOCK_INITIALIZER), \
	261	"i" (MULTIPLE_THREADS_OFFSET) \
7a5cdb30	262	: "memory"); \
76a50749 UD	263	ret; })
76a50749 UD	264
e51deae7	265	#define lll_robust_trylock(futex, id) \
683040c3 UD	266	({ int ret; \
	267	__asm __volatile (LOCK_INSTR "cmpxchgl %2, %1" \
	268	: "=a" (ret), "=m" (futex) \
	269	: "r" (id), "m" (futex), \
e51deae7	270	"0" (LLL_LOCK_INITIALIZER) \
683040c3 UD	271	: "memory"); \
	272	ret; })
	273
	274
e51deae7	275	#define lll_cond_trylock(futex) \
2c0b891a UD	276	({ int ret; \
	277	__asm __volatile (LOCK_INSTR "cmpxchgl %2, %1" \
	278	: "=a" (ret), "=m" (futex) \
e51deae7 UD	279	: "r" (LLL_LOCK_INITIALIZER_WAITERS), \
e51deae7 UD	280	"m" (futex), "0" (LLL_LOCK_INITIALIZER) \
2c0b891a UD	281	: "memory"); \
	282	ret; })
	283
e51deae7 UD	284	#if defined NOT_IN_libc \|\| defined UP
	285	# define __lll_lock_asm_start LOCK_INSTR "cmpxchgl %1, %2\n\t"
	286	#else
	287	# define __lll_lock_asm_start "cmpl $0, %%gs:%P6\n\t" \
	288	"je 0f\n\t" \
	289	"lock\n" \
	290	"0:\tcmpxchgl %1, %2\n\t"
	291	#endif
2c0b891a	292
e51deae7 UD	293	#define lll_lock(futex, private) \
	294	(void) \
	295	({ int ignore1, ignore2; \
	296	if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \
	297	__asm __volatile (__lll_lock_asm_start \
	298	"jnz _L_lock_%=\n\t" \
	299	".subsection 1\n\t" \
	300	".type _L_lock_%=,@function\n" \
	301	"_L_lock_%=:\n" \
	302	"1:\tleal %2, %%ecx\n" \
	303	"2:\tcall __lll_lock_wait_private\n" \
	304	"3:\tjmp 18f\n" \
	305	"4:\t.size _L_lock_%=, 4b-1b\n\t" \
	306	".previous\n" \
	307	LLL_STUB_UNWIND_INFO_3 \
	308	"18:" \
	309	: "=a" (ignore1), "=c" (ignore2), "=m" (futex) \
	310	: "0" (0), "1" (1), "m" (futex), \
	311	"i" (MULTIPLE_THREADS_OFFSET) \
	312	: "memory"); \
	313	else \
	314	{ \
	315	int ignore3; \
	316	__asm __volatile (__lll_lock_asm_start \
	317	"jnz _L_lock_%=\n\t" \
	318	".subsection 1\n\t" \
	319	".type _L_lock_%=,@function\n" \
	320	"_L_lock_%=:\n" \
	321	"1:\tleal %2, %%edx\n" \
	322	"0:\tmovl %8, %%ecx\n" \
	323	"2:\tcall __lll_lock_wait\n" \
	324	"3:\tjmp 18f\n" \
	325	"4:\t.size _L_lock_%=, 4b-1b\n\t" \
	326	".previous\n" \
	327	LLL_STUB_UNWIND_INFO_4 \
	328	"18:" \
	329	: "=a" (ignore1), "=c" (ignore2), \
	330	"=m" (futex), "=&d" (ignore3) \
	331	: "1" (1), "m" (futex), \
	332	"i" (MULTIPLE_THREADS_OFFSET), "0" (0), \
2458c748	333	"g" ((int) (private)) \
e51deae7 UD	334	: "memory"); \
	335	} \
	336	})
	337
	338	#define lll_robust_lock(futex, id, private) \
	339	({ int result, ignore1, ignore2; \
683040c3	340	__asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t" \
e51deae7	341	"jnz _L_robust_lock_%=\n\t" \
683040c3	342	".subsection 1\n\t" \
e51deae7 UD	343	".type _L_robust_lock_%=,@function\n" \
	344	"_L_robust_lock_%=:\n" \
	345	"1:\tleal %2, %%edx\n" \
	346	"0:\tmovl %7, %%ecx\n" \
	347	"2:\tcall __lll_robust_lock_wait\n" \
cd248c3f	348	"3:\tjmp 18f\n" \
e51deae7	349	"4:\t.size _L_robust_lock_%=, 4b-1b\n\t" \
683040c3	350	".previous\n" \
e51deae7	351	LLL_STUB_UNWIND_INFO_4 \
cd248c3f	352	"18:" \
e51deae7 UD	353	: "=a" (result), "=c" (ignore1), "=m" (futex), \
e51deae7 UD	354	"=&d" (ignore2) \
2458c748	355	: "0" (0), "1" (id), "m" (futex), "g" ((int) (private))\
683040c3 UD	356	: "memory"); \
	357	result; })
	358
	359
e51deae7	360	/* Special version of lll_lock which causes the unlock function to
69431c9a	361	always wakeup waiters. */
e51deae7 UD	362	#define lll_cond_lock(futex, private) \
	363	(void) \
	364	({ int ignore1, ignore2, ignore3; \
	365	__asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t" \
	366	"jnz _L_cond_lock_%=\n\t" \
	367	".subsection 1\n\t" \
	368	".type _L_cond_lock_%=,@function\n" \
	369	"_L_cond_lock_%=:\n" \
	370	"1:\tleal %2, %%edx\n" \
	371	"0:\tmovl %7, %%ecx\n" \
	372	"2:\tcall __lll_lock_wait\n" \
	373	"3:\tjmp 18f\n" \
	374	"4:\t.size _L_cond_lock_%=, 4b-1b\n\t" \
	375	".previous\n" \
	376	LLL_STUB_UNWIND_INFO_4 \
	377	"18:" \
	378	: "=a" (ignore1), "=c" (ignore2), "=m" (futex), \
	379	"=&d" (ignore3) \
2458c748	380	: "0" (0), "1" (2), "m" (futex), "g" ((int) (private))\
e51deae7 UD	381	: "memory"); \
	382	})
	383
	384
	385	#define lll_robust_cond_lock(futex, id, private) \
	386	({ int result, ignore1, ignore2; \
683040c3	387	__asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t" \
e51deae7	388	"jnz _L_robust_cond_lock_%=\n\t" \
683040c3	389	".subsection 1\n\t" \
e51deae7 UD	390	".type _L_robust_cond_lock_%=,@function\n" \
	391	"_L_robust_cond_lock_%=:\n" \
	392	"1:\tleal %2, %%edx\n" \
	393	"0:\tmovl %7, %%ecx\n" \
	394	"2:\tcall __lll_robust_lock_wait\n" \
cd248c3f	395	"3:\tjmp 18f\n" \
e51deae7	396	"4:\t.size _L_robust_cond_lock_%=, 4b-1b\n\t" \
683040c3	397	".previous\n" \
e51deae7	398	LLL_STUB_UNWIND_INFO_4 \
cd248c3f	399	"18:" \
e51deae7 UD	400	: "=a" (result), "=c" (ignore1), "=m" (futex), \
	401	"=&d" (ignore2) \
	402	: "0" (0), "1" (id \| FUTEX_WAITERS), "m" (futex), \
2458c748	403	"g" ((int) (private)) \
683040c3 UD	404	: "memory"); \
	405	result; })
	406
	407
e51deae7 UD	408	#define lll_timedlock(futex, timeout, private) \
e51deae7 UD	409	({ int result, ignore1, ignore2, ignore3; \
71451de2	410	__asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t" \
e51deae7	411	"jnz _L_timedlock_%=\n\t" \
8348dcc8	412	".subsection 1\n\t" \
e51deae7 UD	413	".type _L_timedlock_%=,@function\n" \
e51deae7 UD	414	"_L_timedlock_%=:\n" \
cd248c3f	415	"1:\tleal %3, %%ecx\n" \
e51deae7 UD	416	"0:\tmovl %8, %%edx\n" \
e51deae7 UD	417	"2:\tcall __lll_timedlock_wait\n" \
cd248c3f	418	"3:\tjmp 18f\n" \
e51deae7	419	"4:\t.size _L_timedlock_%=, 4b-1b\n\t" \
7a5cdb30	420	".previous\n" \
cd248c3f UD	421	LLL_STUB_UNWIND_INFO_4 \
cd248c3f UD	422	"18:" \
71451de2	423	: "=a" (result), "=c" (ignore1), "=&d" (ignore2), \
e51deae7 UD	424	"=m" (futex), "=S" (ignore3) \
e51deae7 UD	425	: "0" (0), "1" (1), "m" (futex), "m" (timeout), \
2458c748	426	"4" ((int) (private)) \
7a5cdb30	427	: "memory"); \
76a50749 UD	428	result; })
	429
	430
e51deae7 UD	431	#define lll_robust_timedlock(futex, timeout, id, private) \
e51deae7 UD	432	({ int result, ignore1, ignore2, ignore3; \
683040c3	433	__asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t" \
e51deae7	434	"jnz _L_robust_timedlock_%=\n\t" \
683040c3	435	".subsection 1\n\t" \
e51deae7 UD	436	".type _L_robust_timedlock_%=,@function\n" \
e51deae7 UD	437	"_L_robust_timedlock_%=:\n" \
cd248c3f	438	"1:\tleal %3, %%ecx\n" \
e51deae7 UD	439	"0:\tmovl %8, %%edx\n" \
e51deae7 UD	440	"2:\tcall __lll_robust_timedlock_wait\n" \
cd248c3f	441	"3:\tjmp 18f\n" \
e51deae7	442	"4:\t.size _L_robust_timedlock_%=, 4b-1b\n\t" \
683040c3	443	".previous\n" \
cd248c3f UD	444	LLL_STUB_UNWIND_INFO_4 \
cd248c3f UD	445	"18:" \
683040c3	446	: "=a" (result), "=c" (ignore1), "=&d" (ignore2), \
e51deae7 UD	447	"=m" (futex), "=S" (ignore3) \
e51deae7 UD	448	: "0" (0), "1" (id), "m" (futex), "m" (timeout), \
2458c748	449	"4" ((int) (private)) \
683040c3 UD	450	: "memory"); \
	451	result; })
	452
14e7aece	453	#if defined NOT_IN_libc \|\| defined UP
e51deae7	454	# define __lll_unlock_asm LOCK_INSTR "subl $1, %0\n\t"
14e7aece	455	#else
e51deae7 UD	456	# define __lll_unlock_asm "cmpl $0, %%gs:%P3\n\t" \
	457	"je 0f\n\t" \
	458	"lock\n" \
	459	"0:\tsubl $1,%0\n\t"
14e7aece	460	#endif
76a50749	461
e51deae7 UD	462	#define lll_unlock(futex, private) \
	463	(void) \
	464	({ int ignore; \
	465	if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \
	466	__asm __volatile (__lll_unlock_asm \
	467	"jne _L_unlock_%=\n\t" \
	468	".subsection 1\n\t" \
	469	".type _L_unlock_%=,@function\n" \
	470	"_L_unlock_%=:\n" \
	471	"1:\tleal %0, %%eax\n" \
	472	"2:\tcall __lll_unlock_wake_private\n" \
	473	"3:\tjmp 18f\n" \
	474	"4:\t.size _L_unlock_%=, 4b-1b\n\t" \
	475	".previous\n" \
	476	LLL_STUB_UNWIND_INFO_3 \
	477	"18:" \
	478	: "=m" (futex), "=&a" (ignore) \
	479	: "m" (futex), "i" (MULTIPLE_THREADS_OFFSET) \
	480	: "memory"); \
	481	else \
	482	{ \
	483	int ignore2; \
	484	__asm __volatile (__lll_unlock_asm \
	485	"jne _L_unlock_%=\n\t" \
	486	".subsection 1\n\t" \
	487	".type _L_unlock_%=,@function\n" \
	488	"_L_unlock_%=:\n" \
	489	"1:\tleal %0, %%eax\n" \
	490	"0:\tmovl %5, %%ecx\n" \
	491	"2:\tcall __lll_unlock_wake\n" \
	492	"3:\tjmp 18f\n" \
	493	"4:\t.size _L_unlock_%=, 4b-1b\n\t" \
	494	".previous\n" \
	495	LLL_STUB_UNWIND_INFO_4 \
	496	"18:" \
	497	: "=m" (futex), "=&a" (ignore), "=&c" (ignore2) \
	498	: "i" (MULTIPLE_THREADS_OFFSET), "m" (futex), \
2458c748	499	"g" ((int) (private)) \
e51deae7 UD	500	: "memory"); \
	501	} \
	502	})
	503
	504	#define lll_robust_unlock(futex, private) \
	505	(void) \
	506	({ int ignore, ignore2; \
	507	__asm __volatile (LOCK_INSTR "andl %3, %0\n\t" \
	508	"jne _L_robust_unlock_%=\n\t" \
	509	".subsection 1\n\t" \
	510	".type _L_robust_unlock_%=,@function\n" \
	511	"_L_robust_unlock_%=:\n\t" \
	512	"1:\tleal %0, %%eax\n" \
	513	"0:\tmovl %5, %%ecx\n" \
	514	"2:\tcall __lll_unlock_wake\n" \
	515	"3:\tjmp 18f\n" \
	516	"4:\t.size _L_robust_unlock_%=, 4b-1b\n\t" \
	517	".previous\n" \
	518	LLL_STUB_UNWIND_INFO_4 \
	519	"18:" \
	520	: "=m" (futex), "=&a" (ignore), "=&c" (ignore2) \
2458c748 AJ	521	: "i" (FUTEX_WAITERS), "m" (futex), \
2458c748 AJ	522	"g" ((int) (private)) \
e51deae7 UD	523	: "memory"); \
	524	})
	525
	526
	527	#define lll_robust_dead(futex, private) \
	528	(void) \
	529	({ int __ignore; \
	530	register int _nr asm ("edx") = 1; \
	531	__asm __volatile (LOCK_INSTR "orl %5, (%2)\n\t" \
	532	LLL_EBX_LOAD \
	533	LLL_ENTER_KERNEL \
	534	LLL_EBX_LOAD \
	535	: "=a" (__ignore) \
	536	: "0" (SYS_futex), LLL_EBX_REG (&(futex)), \
	537	"c" (__lll_private_flag (FUTEX_WAKE, private)), \
	538	"d" (_nr), "i" (FUTEX_OWNER_DIED), \
	539	"i" (offsetof (tcbhead_t, sysinfo))); \
	540	})
76a50749 UD	541
76a50749 UD	542	#define lll_islocked(futex) \
71451de2	543	(futex != LLL_LOCK_INITIALIZER)
76a50749	544
76a50749 UD	545	/* The kernel notifies a process with uses CLONE_CLEARTID via futex
	546	wakeup when the clone terminates. The memory location contains the
	547	thread ID while the clone is running and is reset to zero
	548	afterwards.
	549
	550	The macro parameter must not have any side effect. */
76a50749 UD	551	#define lll_wait_tid(tid) \
	552	do { \
	553	int __ignore; \
	554	register __typeof (tid) _tid asm ("edx") = (tid); \
	555	if (_tid != 0) \
949ec764	556	__asm __volatile (LLL_EBX_LOAD \
76a50749	557	"1:\tmovl %1, %%eax\n\t" \
949ec764	558	LLL_ENTER_KERNEL \
76a50749	559	"cmpl $0, (%%ebx)\n\t" \
ab9a9ff8	560	"jne 1b\n\t" \
949ec764	561	LLL_EBX_LOAD \
76a50749	562	: "=&a" (__ignore) \
949ec764	563	: "i" (SYS_futex), LLL_EBX_REG (&tid), "S" (0), \
097eca29	564	"c" (FUTEX_WAIT), "d" (_tid), \
b576fca1 UD	565	"i" (offsetof (tcbhead_t, sysinfo)) \
b576fca1 UD	566	: "memory"); \
76a50749 UD	567	} while (0)
	568
	569	extern int __lll_timedwait_tid (int tid, const struct timespec abstime)
	570	__attribute__ ((regparm (2))) attribute_hidden;
	571	#define lll_timedwait_tid(tid, abstime) \
	572	({ \
	573	int __result = 0; \
	574	if (tid != 0) \
	575	{ \
31195be2	576	if (abstime->tv_nsec < 0 \|\| abstime->tv_nsec >= 1000000000) \
76a50749 UD	577	__result = EINVAL; \
	578	else \
	579	__result = __lll_timedwait_tid (&tid, abstime); \
	580	} \
	581	__result; })
	582
e51deae7	583	#endif /* !__ASSEMBLER__ */
76a50749 UD	584
76a50749 UD	585	#endif /* lowlevellock.h */