This is the mail archive of the
libc-alpha@sourceware.org
mailing list for the glibc project.
[PATCH 4/4] x86: Remove arch-specific low level lock implementation
- From: Adhemerval Zanella <adhemerval dot zanella at linaro dot org>
- To: libc-alpha at sourceware dot org
- Date: Fri, 22 Feb 2019 16:27:03 -0300
- Subject: [PATCH 4/4] x86: Remove arch-specific low level lock implementation
- References: <20190222192703.18177-1-adhemerval.zanella@linaro.org>
This patch removes the arch-specific x86 assembly implementation for
low level locking and consolidate both 64 bits and 32 bits in a
single implementation.
Different than other architectures, x86 lll_trylock, lll_lock, and
lll_unlock implements a single-thread optimization to avoid atomic
operation, using cmpxchgl instead. This patch implements by using
the new single-thread.h definitions in a generic way, although using
the previous semantic.
The lll_cond_trylock, lll_cond_lock, and lll_timedlock just use
atomic operations plus calls to lll_lock_wait*.
For __lll_lock_wait_private and __lll_lock_wait the generic implemtation
there is no indication that assembly implementation is required
performance-wise.
Checked on x86_64-linux-gnu and i686-linux-gnu.
* sysdeps/nptl/lowlevellock.h (__lll_trylock): New macro.
(lll_trylock): Call __lll_trylock.
* sysdeps/unix/sysv/linux/i386/libc-lowlevellock.S: Remove file.
* sysdeps/unix/sysv/linux/i386/lll_timedlock_wait.c: Likewise.
* sysdeps/unix/sysv/linux/i386/lowlevellock.S: Likewise.
* sysdeps/unix/sysv/linux/i386/lowlevellock.h: Likewise.
* sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S: Likewise.
* sysdeps/unix/sysv/linux/x86_64/lll_timedlock_wait.c: Likewise.
* sysdeps/unix/sysv/linux/x86_64/lowlevellock.S: Likewise.
* sysdeps/unix/sysv/linux/x86_64/lowlevellock.h: Likewise.
* sysdeps/unix/sysv/linux/x86/lowlevellock.h: New file.
* sysdeps/unix/sysv/linux/x86_64/cancellation.S: Include
lowlevellock-futex.h.
---
sysdeps/nptl/lowlevellock.h | 4 +-
.../unix/sysv/linux/i386/libc-lowlevellock.S | 19 -
.../unix/sysv/linux/i386/lll_timedlock_wait.c | 1 -
sysdeps/unix/sysv/linux/i386/lowlevellock.S | 368 ------------------
sysdeps/unix/sysv/linux/i386/lowlevellock.h | 240 ------------
sysdeps/unix/sysv/linux/x86/lowlevellock.h | 110 ++++++
sysdeps/unix/sysv/linux/x86_64/cancellation.S | 2 +-
.../sysv/linux/x86_64/libc-lowlevellock.S | 19 -
.../sysv/linux/x86_64/lll_timedlock_wait.c | 1 -
sysdeps/unix/sysv/linux/x86_64/lowlevellock.S | 348 -----------------
sysdeps/unix/sysv/linux/x86_64/lowlevellock.h | 243 ------------
11 files changed, 114 insertions(+), 1241 deletions(-)
delete mode 100644 sysdeps/unix/sysv/linux/i386/libc-lowlevellock.S
delete mode 100644 sysdeps/unix/sysv/linux/i386/lll_timedlock_wait.c
delete mode 100644 sysdeps/unix/sysv/linux/i386/lowlevellock.S
delete mode 100644 sysdeps/unix/sysv/linux/i386/lowlevellock.h
create mode 100644 sysdeps/unix/sysv/linux/x86/lowlevellock.h
delete mode 100644 sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S
delete mode 100644 sysdeps/unix/sysv/linux/x86_64/lll_timedlock_wait.c
delete mode 100644 sysdeps/unix/sysv/linux/x86_64/lowlevellock.S
delete mode 100644 sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
diff --git a/sysdeps/nptl/lowlevellock.h b/sysdeps/nptl/lowlevellock.h
index 6f017afdd5..e905829ee4 100644
--- a/sysdeps/nptl/lowlevellock.h
+++ b/sysdeps/nptl/lowlevellock.h
@@ -63,8 +63,10 @@
/* If LOCK is 0 (not acquired), set to 1 (acquired with no waiters) and return
0. Otherwise leave lock unchanged and return non-zero to indicate that the
lock was not acquired. */
+#define __lll_trylock(lock) \
+ __glibc_unlikely (atomic_compare_and_exchange_bool_acq ((lock), 1, 0))
#define lll_trylock(lock) \
- __glibc_unlikely (atomic_compare_and_exchange_bool_acq (&(lock), 1, 0))
+ __lll_trylock (&(lock))
/* If LOCK is 0 (not acquired), set to 2 (acquired, possibly with waiters) and
return 0. Otherwise leave lock unchanged and return non-zero to indicate
diff --git a/sysdeps/unix/sysv/linux/i386/libc-lowlevellock.S b/sysdeps/unix/sysv/linux/i386/libc-lowlevellock.S
deleted file mode 100644
index 32025277f3..0000000000
--- a/sysdeps/unix/sysv/linux/i386/libc-lowlevellock.S
+++ /dev/null
@@ -1,19 +0,0 @@
-/* Copyright (C) 2002-2019 Free Software Foundation, Inc.
- This file is part of the GNU C Library.
- Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
-
- The GNU C Library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- The GNU C Library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with the GNU C Library; if not, see
- <http://www.gnu.org/licenses/>. */
-
-#include "lowlevellock.S"
diff --git a/sysdeps/unix/sysv/linux/i386/lll_timedlock_wait.c b/sysdeps/unix/sysv/linux/i386/lll_timedlock_wait.c
deleted file mode 100644
index f6875b8f89..0000000000
--- a/sysdeps/unix/sysv/linux/i386/lll_timedlock_wait.c
+++ /dev/null
@@ -1 +0,0 @@
-/* __lll_timedlock_wait is in lowlevellock.S. */
diff --git a/sysdeps/unix/sysv/linux/i386/lowlevellock.S b/sysdeps/unix/sysv/linux/i386/lowlevellock.S
deleted file mode 100644
index 83191a3899..0000000000
--- a/sysdeps/unix/sysv/linux/i386/lowlevellock.S
+++ /dev/null
@@ -1,368 +0,0 @@
-/* Copyright (C) 2002-2019 Free Software Foundation, Inc.
- This file is part of the GNU C Library.
- Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
-
- The GNU C Library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- The GNU C Library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with the GNU C Library; if not, see
- <http://www.gnu.org/licenses/>. */
-
-#include <sysdep.h>
-#include <pthread-errnos.h>
-#include <kernel-features.h>
-#include <lowlevellock.h>
-
-#include <stap-probe.h>
-
- .text
-
-#define LOAD_PRIVATE_FUTEX_WAIT(reg) \
- movl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg
-#define LOAD_PRIVATE_FUTEX_WAKE(reg) \
- movl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg
-#define LOAD_FUTEX_WAIT(reg) \
- xorl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg
-#define LOAD_FUTEX_WAIT_ABS(reg) \
- xorl $(FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME), reg
-#define LOAD_FUTEX_WAKE(reg) \
- xorl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg
-
- .globl __lll_lock_wait_private
- .type __lll_lock_wait_private,@function
- .hidden __lll_lock_wait_private
- .align 16
-__lll_lock_wait_private:
- cfi_startproc
- pushl %edx
- cfi_adjust_cfa_offset(4)
- pushl %ebx
- cfi_adjust_cfa_offset(4)
- pushl %esi
- cfi_adjust_cfa_offset(4)
- cfi_offset(%edx, -8)
- cfi_offset(%ebx, -12)
- cfi_offset(%esi, -16)
-
- movl $2, %edx
- movl %ecx, %ebx
- xorl %esi, %esi /* No timeout. */
- LOAD_PRIVATE_FUTEX_WAIT (%ecx)
-
- cmpl %edx, %eax /* NB: %edx == 2 */
- jne 2f
-
-1: LIBC_PROBE (lll_lock_wait_private, 1, %ebx)
- movl $SYS_futex, %eax
- ENTER_KERNEL
-
-2: movl %edx, %eax
- xchgl %eax, (%ebx) /* NB: lock is implied */
-
- testl %eax, %eax
- jnz 1b
-
- popl %esi
- cfi_adjust_cfa_offset(-4)
- cfi_restore(%esi)
- popl %ebx
- cfi_adjust_cfa_offset(-4)
- cfi_restore(%ebx)
- popl %edx
- cfi_adjust_cfa_offset(-4)
- cfi_restore(%edx)
- ret
- cfi_endproc
- .size __lll_lock_wait_private,.-__lll_lock_wait_private
-
-#if !IS_IN (libc)
- .globl __lll_lock_wait
- .type __lll_lock_wait,@function
- .hidden __lll_lock_wait
- .align 16
-__lll_lock_wait:
- cfi_startproc
- pushl %edx
- cfi_adjust_cfa_offset(4)
- pushl %ebx
- cfi_adjust_cfa_offset(4)
- pushl %esi
- cfi_adjust_cfa_offset(4)
- cfi_offset(%edx, -8)
- cfi_offset(%ebx, -12)
- cfi_offset(%esi, -16)
-
- movl %edx, %ebx
- movl $2, %edx
- xorl %esi, %esi /* No timeout. */
- LOAD_FUTEX_WAIT (%ecx)
-
- cmpl %edx, %eax /* NB: %edx == 2 */
- jne 2f
-
-1: movl $SYS_futex, %eax
- ENTER_KERNEL
-
-2: movl %edx, %eax
- xchgl %eax, (%ebx) /* NB: lock is implied */
-
- testl %eax, %eax
- jnz 1b
-
- popl %esi
- cfi_adjust_cfa_offset(-4)
- cfi_restore(%esi)
- popl %ebx
- cfi_adjust_cfa_offset(-4)
- cfi_restore(%ebx)
- popl %edx
- cfi_adjust_cfa_offset(-4)
- cfi_restore(%edx)
- ret
- cfi_endproc
- .size __lll_lock_wait,.-__lll_lock_wait
-
- /* %ecx: futex
- %esi: flags
- %edx: timeout
- %eax: futex value
- */
- .globl __lll_timedlock_wait
- .type __lll_timedlock_wait,@function
- .hidden __lll_timedlock_wait
- .align 16
-__lll_timedlock_wait:
- cfi_startproc
- pushl %ebp
- cfi_adjust_cfa_offset(4)
- cfi_rel_offset(%ebp, 0)
- pushl %ebx
- cfi_adjust_cfa_offset(4)
- cfi_rel_offset(%ebx, 0)
-
-# ifndef __ASSUME_FUTEX_CLOCK_REALTIME
-# ifdef PIC
- LOAD_PIC_REG (bx)
- cmpl $0, __have_futex_clock_realtime@GOTOFF(%ebx)
-# else
- cmpl $0, __have_futex_clock_realtime
-# endif
- je .Lreltmo
-# endif
-
- cmpl $0, (%edx)
- js 8f
-
- movl %ecx, %ebx
- movl %esi, %ecx
- movl %edx, %esi
- movl $0xffffffff, %ebp
- LOAD_FUTEX_WAIT_ABS (%ecx)
-
- movl $2, %edx
- cmpl %edx, %eax
- jne 2f
-
-1: movl $SYS_futex, %eax
- movl $2, %edx
- ENTER_KERNEL
-
-2: xchgl %edx, (%ebx) /* NB: lock is implied */
-
- testl %edx, %edx
- jz 3f
-
- cmpl $-ETIMEDOUT, %eax
- je 4f
- cmpl $-EINVAL, %eax
- jne 1b
-4: movl %eax, %edx
- negl %edx
-
-3: movl %edx, %eax
-7: popl %ebx
- cfi_adjust_cfa_offset(-4)
- cfi_restore(%ebx)
- popl %ebp
- cfi_adjust_cfa_offset(-4)
- cfi_restore(%ebp)
- ret
-
-8: movl $ETIMEDOUT, %eax
- jmp 7b
-
-# ifndef __ASSUME_FUTEX_CLOCK_REALTIME
-.Lreltmo:
- /* Check for a valid timeout value. */
- cmpl $1000000000, 4(%edx)
- jae 3f
-
- pushl %esi
- cfi_adjust_cfa_offset(4)
- cfi_rel_offset(%esi, 0)
- pushl %edi
- cfi_adjust_cfa_offset(4)
- cfi_rel_offset(%edi, 0)
-
- /* Stack frame for the timespec and timeval structs. */
- subl $8, %esp
- cfi_adjust_cfa_offset(8)
-
- movl %ecx, %ebp
- movl %edx, %edi
-
- movl $2, %edx
- xchgl %edx, (%ebp)
-
- test %edx, %edx
- je 6f
-
-1:
- /* Get current time. */
- movl %esp, %ebx
- xorl %ecx, %ecx
- movl $__NR_gettimeofday, %eax
- ENTER_KERNEL
-
- /* Compute relative timeout. */
- movl 4(%esp), %eax
- movl $1000, %edx
- mul %edx /* Milli seconds to nano seconds. */
- movl (%edi), %ecx
- movl 4(%edi), %edx
- subl (%esp), %ecx
- subl %eax, %edx
- jns 4f
- addl $1000000000, %edx
- subl $1, %ecx
-4: testl %ecx, %ecx
- js 2f /* Time is already up. */
-
- /* Store relative timeout. */
- movl %ecx, (%esp)
- movl %edx, 4(%esp)
-
- /* Futex call. */
- movl %ebp, %ebx
- movl $2, %edx
- movl %esp, %esi
- movl 16(%esp), %ecx
- LOAD_FUTEX_WAIT (%ecx)
- movl $SYS_futex, %eax
- ENTER_KERNEL
-
- /* NB: %edx == 2 */
- xchgl %edx, (%ebp)
-
- testl %edx, %edx
- je 6f
-
- cmpl $-ETIMEDOUT, %eax
- jne 1b
-2: movl $ETIMEDOUT, %edx
-
-6: addl $8, %esp
- cfi_adjust_cfa_offset(-8)
- popl %edi
- cfi_adjust_cfa_offset(-4)
- cfi_restore(%edi)
- popl %esi
- cfi_adjust_cfa_offset(-4)
- cfi_restore(%esi)
-7: popl %ebx
- cfi_adjust_cfa_offset(-4)
- cfi_restore(%ebx)
- popl %ebp
- cfi_adjust_cfa_offset(-4)
- cfi_restore(%ebp)
- movl %edx, %eax
- ret
-
-3: movl $EINVAL, %edx
- jmp 7b
-# endif
- cfi_endproc
- .size __lll_timedlock_wait,.-__lll_timedlock_wait
-#endif
-
- .globl __lll_unlock_wake_private
- .type __lll_unlock_wake_private,@function
- .hidden __lll_unlock_wake_private
- .align 16
-__lll_unlock_wake_private:
- cfi_startproc
- pushl %ebx
- cfi_adjust_cfa_offset(4)
- pushl %ecx
- cfi_adjust_cfa_offset(4)
- pushl %edx
- cfi_adjust_cfa_offset(4)
- cfi_offset(%ebx, -8)
- cfi_offset(%ecx, -12)
- cfi_offset(%edx, -16)
-
- movl %eax, %ebx
- movl $0, (%eax)
- LOAD_PRIVATE_FUTEX_WAKE (%ecx)
- movl $1, %edx /* Wake one thread. */
- movl $SYS_futex, %eax
- ENTER_KERNEL
-
- popl %edx
- cfi_adjust_cfa_offset(-4)
- cfi_restore(%edx)
- popl %ecx
- cfi_adjust_cfa_offset(-4)
- cfi_restore(%ecx)
- popl %ebx
- cfi_adjust_cfa_offset(-4)
- cfi_restore(%ebx)
- ret
- cfi_endproc
- .size __lll_unlock_wake_private,.-__lll_unlock_wake_private
-
-#if !IS_IN (libc)
- .globl __lll_unlock_wake
- .type __lll_unlock_wake,@function
- .hidden __lll_unlock_wake
- .align 16
-__lll_unlock_wake:
- cfi_startproc
- pushl %ebx
- cfi_adjust_cfa_offset(4)
- pushl %ecx
- cfi_adjust_cfa_offset(4)
- pushl %edx
- cfi_adjust_cfa_offset(4)
- cfi_offset(%ebx, -8)
- cfi_offset(%ecx, -12)
- cfi_offset(%edx, -16)
-
- movl %eax, %ebx
- movl $0, (%eax)
- LOAD_FUTEX_WAKE (%ecx)
- movl $1, %edx /* Wake one thread. */
- movl $SYS_futex, %eax
- ENTER_KERNEL
-
- popl %edx
- cfi_adjust_cfa_offset(-4)
- cfi_restore(%edx)
- popl %ecx
- cfi_adjust_cfa_offset(-4)
- cfi_restore(%ecx)
- popl %ebx
- cfi_adjust_cfa_offset(-4)
- cfi_restore(%ebx)
- ret
- cfi_endproc
- .size __lll_unlock_wake,.-__lll_unlock_wake
-#endif
diff --git a/sysdeps/unix/sysv/linux/i386/lowlevellock.h b/sysdeps/unix/sysv/linux/i386/lowlevellock.h
deleted file mode 100644
index 94dccc4ce7..0000000000
--- a/sysdeps/unix/sysv/linux/i386/lowlevellock.h
+++ /dev/null
@@ -1,240 +0,0 @@
-/* Copyright (C) 2002-2019 Free Software Foundation, Inc.
- This file is part of the GNU C Library.
- Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
-
- The GNU C Library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- The GNU C Library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with the GNU C Library; if not, see
- <http://www.gnu.org/licenses/>. */
-
-#ifndef _LOWLEVELLOCK_H
-#define _LOWLEVELLOCK_H 1
-
-#ifndef __ASSEMBLER__
-# include <time.h>
-# include <sys/param.h>
-# include <bits/pthreadtypes.h>
-# include <kernel-features.h>
-/* <tcb-offsets.h> is generated from tcb-offsets.sym to define offsets
- and sizes of types in <tls.h> as well as <pthread.h> which includes
- <lowlevellock.h> via nptl/descr.h. Don't include <tcb-offsets.h>
- when generating <tcb-offsets.h> to avoid circular dependency which
- may lead to build hang on a many-core machine. */
-# ifndef GEN_AS_CONST_HEADERS
-# include <tcb-offsets.h>
-# endif
-
-# ifndef LOCK_INSTR
-# ifdef UP
-# define LOCK_INSTR /* nothing */
-# else
-# define LOCK_INSTR "lock;"
-# endif
-# endif
-#else
-# ifndef LOCK
-# ifdef UP
-# define LOCK
-# else
-# define LOCK lock
-# endif
-# endif
-#endif
-
-#include <lowlevellock-futex.h>
-
-/* XXX Remove when no assembler code uses futexes anymore. */
-#define SYS_futex __NR_futex
-
-#ifndef __ASSEMBLER__
-
-/* Initializer for compatibility lock. */
-#define LLL_LOCK_INITIALIZER (0)
-#define LLL_LOCK_INITIALIZER_LOCKED (1)
-#define LLL_LOCK_INITIALIZER_WAITERS (2)
-
-
-/* NB: in the lll_trylock macro we simply return the value in %eax
- after the cmpxchg instruction. In case the operation succeded this
- value is zero. In case the operation failed, the cmpxchg instruction
- has loaded the current value of the memory work which is guaranteed
- to be nonzero. */
-#if !IS_IN (libc) || defined UP
-# define __lll_trylock_asm LOCK_INSTR "cmpxchgl %2, %1"
-#else
-# define __lll_trylock_asm "cmpl $0, %%gs:%P5\n\t" \
- "je 0f\n\t" \
- "lock\n" \
- "0:\tcmpxchgl %2, %1"
-#endif
-
-#define lll_trylock(futex) \
- ({ int ret; \
- __asm __volatile (__lll_trylock_asm \
- : "=a" (ret), "=m" (futex) \
- : "r" (LLL_LOCK_INITIALIZER_LOCKED), "m" (futex), \
- "0" (LLL_LOCK_INITIALIZER), \
- "i" (MULTIPLE_THREADS_OFFSET) \
- : "memory"); \
- ret; })
-
-
-#define lll_cond_trylock(futex) \
- ({ int ret; \
- __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1" \
- : "=a" (ret), "=m" (futex) \
- : "r" (LLL_LOCK_INITIALIZER_WAITERS), \
- "m" (futex), "0" (LLL_LOCK_INITIALIZER) \
- : "memory"); \
- ret; })
-
-#if !IS_IN (libc) || defined UP
-# define __lll_lock_asm_start LOCK_INSTR "cmpxchgl %1, %2\n\t"
-#else
-# define __lll_lock_asm_start "cmpl $0, %%gs:%P6\n\t" \
- "je 0f\n\t" \
- "lock\n" \
- "0:\tcmpxchgl %1, %2\n\t"
-#endif
-
-#define lll_lock(futex, private) \
- (void) \
- ({ int ignore1, ignore2; \
- if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \
- __asm __volatile (__lll_lock_asm_start \
- "jz 18f\n\t" \
- "1:\tleal %2, %%ecx\n" \
- "2:\tcall __lll_lock_wait_private\n" \
- "18:" \
- : "=a" (ignore1), "=c" (ignore2), "=m" (futex) \
- : "0" (0), "1" (1), "m" (futex), \
- "i" (MULTIPLE_THREADS_OFFSET) \
- : "memory"); \
- else \
- { \
- int ignore3; \
- __asm __volatile (__lll_lock_asm_start \
- "jz 18f\n\t" \
- "1:\tleal %2, %%edx\n" \
- "0:\tmovl %8, %%ecx\n" \
- "2:\tcall __lll_lock_wait\n" \
- "18:" \
- : "=a" (ignore1), "=c" (ignore2), \
- "=m" (futex), "=&d" (ignore3) \
- : "1" (1), "m" (futex), \
- "i" (MULTIPLE_THREADS_OFFSET), "0" (0), \
- "g" ((int) (private)) \
- : "memory"); \
- } \
- })
-
-
-/* Special version of lll_lock which causes the unlock function to
- always wakeup waiters. */
-#define lll_cond_lock(futex, private) \
- (void) \
- ({ int ignore1, ignore2, ignore3; \
- __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t" \
- "jz 18f\n\t" \
- "1:\tleal %2, %%edx\n" \
- "0:\tmovl %7, %%ecx\n" \
- "2:\tcall __lll_lock_wait\n" \
- "18:" \
- : "=a" (ignore1), "=c" (ignore2), "=m" (futex), \
- "=&d" (ignore3) \
- : "0" (0), "1" (2), "m" (futex), "g" ((int) (private))\
- : "memory"); \
- })
-
-
-#define lll_timedlock(futex, timeout, private) \
- ({ int result, ignore1, ignore2, ignore3; \
- __asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t" \
- "jz 18f\n\t" \
- "1:\tleal %3, %%ecx\n" \
- "0:\tmovl %8, %%edx\n" \
- "2:\tcall __lll_timedlock_wait\n" \
- "18:" \
- : "=a" (result), "=c" (ignore1), "=&d" (ignore2), \
- "=m" (futex), "=S" (ignore3) \
- : "0" (0), "1" (1), "m" (futex), "m" (timeout), \
- "4" ((int) (private)) \
- : "memory"); \
- result; })
-
-extern int __lll_timedlock_elision (int *futex, short *adapt_count,
- const struct timespec *timeout,
- int private) attribute_hidden;
-
-#define lll_timedlock_elision(futex, adapt_count, timeout, private) \
- __lll_timedlock_elision(&(futex), &(adapt_count), timeout, private)
-
-#if !IS_IN (libc) || defined UP
-# define __lll_unlock_asm LOCK_INSTR "subl $1, %0\n\t"
-#else
-# define __lll_unlock_asm "cmpl $0, %%gs:%P3\n\t" \
- "je 0f\n\t" \
- "lock\n" \
- "0:\tsubl $1,%0\n\t"
-#endif
-
-#define lll_unlock(futex, private) \
- (void) \
- ({ int ignore; \
- if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \
- __asm __volatile (__lll_unlock_asm \
- "je 18f\n\t" \
- "1:\tleal %0, %%eax\n" \
- "2:\tcall __lll_unlock_wake_private\n" \
- "18:" \
- : "=m" (futex), "=&a" (ignore) \
- : "m" (futex), "i" (MULTIPLE_THREADS_OFFSET) \
- : "memory"); \
- else \
- { \
- int ignore2; \
- __asm __volatile (__lll_unlock_asm \
- "je 18f\n\t" \
- "1:\tleal %0, %%eax\n" \
- "0:\tmovl %5, %%ecx\n" \
- "2:\tcall __lll_unlock_wake\n" \
- "18:" \
- : "=m" (futex), "=&a" (ignore), "=&c" (ignore2) \
- : "i" (MULTIPLE_THREADS_OFFSET), "m" (futex), \
- "g" ((int) (private)) \
- : "memory"); \
- } \
- })
-
-
-#define lll_islocked(futex) \
- (futex != LLL_LOCK_INITIALIZER)
-
-extern int __lll_lock_elision (int *futex, short *adapt_count, int private)
- attribute_hidden;
-
-extern int __lll_unlock_elision(int *lock, int private)
- attribute_hidden;
-
-extern int __lll_trylock_elision(int *lock, short *adapt_count)
- attribute_hidden;
-
-#define lll_lock_elision(futex, adapt_count, private) \
- __lll_lock_elision (&(futex), &(adapt_count), private)
-#define lll_unlock_elision(futex, adapt_count, private) \
- __lll_unlock_elision (&(futex), private)
-#define lll_trylock_elision(futex, adapt_count) \
- __lll_trylock_elision(&(futex), &(adapt_count))
-
-#endif /* !__ASSEMBLER__ */
-
-#endif /* lowlevellock.h */
diff --git a/sysdeps/unix/sysv/linux/x86/lowlevellock.h b/sysdeps/unix/sysv/linux/x86/lowlevellock.h
new file mode 100644
index 0000000000..2652a00736
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/x86/lowlevellock.h
@@ -0,0 +1,110 @@
+/* Low-level lock implementation, x86 version.
+ Copyright (C) 2019 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#ifndef _X86_64_LOWLEVELLOCK_H
+#define _X86_64_LOWLEVELLOCK_H 1
+
+#ifndef __ASSEMBLER__
+#include <sysdeps/nptl/lowlevellock.h>
+#include <single-thread.h>
+
+/* The lll_trylock, lll_lock, and lll_unlock implements a single-thread
+ optimization using the cmpxchgl instruction. It checks if the process
+ is single thread and avoid a more expensive atomic instruction. */
+
+/* The single-thread optimization only works for libc itself, we need
+ atomicity for libpthread in case of shared futexes. */
+#if !IS_IN(libc)
+# define is_single_thread 0
+#else
+# define is_single_thread SINGLE_THREAD_P
+#endif
+
+/* In the __lllc_as we simply return the value in %eax after the cmpxchg
+ instruction. In case the operation succeded this value is zero. In case
+ the operation failed, the cmpxchg instruction has loaded the current value
+ of the memory work which is guaranteed to be nonzero. */
+static inline int
+__attribute__ ((always_inline))
+__lll_cas_lock (int *futex)
+{
+ int ret;
+ asm volatile ("cmpxchgl %2, %1"
+ : "=a" (ret), "=m" (*futex)
+ : "r" (1), "m" (*futex), "0" (0)
+ : "memory");
+ return ret;
+}
+
+#undef lll_trylock
+#define lll_trylock(lock) \
+ ({ \
+ int __ret; \
+ if (is_single_thread) \
+ __ret = __lll_cas_lock (&(lock)); \
+ else \
+ __ret = __lll_trylock (&(lock)); \
+ __ret; \
+ })
+
+#undef lll_lock
+#define lll_lock(lock, private) \
+ ((void) \
+ ({ \
+ if (is_single_thread) \
+ __lll_cas_lock (&(lock)); \
+ else \
+ __lll_lock (&(lock), private); \
+ }))
+
+#undef lll_unlock
+#define lll_unlock(lock, private) \
+ ((void) \
+ ({ \
+ if (is_single_thread) \
+ (lock)--; \
+ else \
+ __lll_unlock (&(lock), private); \
+ }))
+
+extern int __lll_timedlock_elision (int *futex, short *adapt_count,
+ const struct timespec *timeout,
+ int private) attribute_hidden;
+
+#define lll_timedlock_elision(futex, adapt_count, timeout, private) \
+ __lll_timedlock_elision(&(futex), &(adapt_count), timeout, private)
+
+extern int __lll_lock_elision (int *futex, short *adapt_count, int private)
+ attribute_hidden;
+
+extern int __lll_unlock_elision (int *lock, int private)
+ attribute_hidden;
+
+extern int __lll_trylock_elision (int *lock, short *adapt_count)
+ attribute_hidden;
+
+#define lll_lock_elision(futex, adapt_count, private) \
+ __lll_lock_elision (&(futex), &(adapt_count), private)
+#define lll_unlock_elision(futex, adapt_count, private) \
+ __lll_unlock_elision (&(futex), private)
+#define lll_trylock_elision(futex, adapt_count) \
+ __lll_trylock_elision (&(futex), &(adapt_count))
+
+#endif /* !__ASSEMBLER__ */
+
+#endif /* lowlevellock.h */
diff --git a/sysdeps/unix/sysv/linux/x86_64/cancellation.S b/sysdeps/unix/sysv/linux/x86_64/cancellation.S
index 7d169d9aca..bb4910764a 100644
--- a/sysdeps/unix/sysv/linux/x86_64/cancellation.S
+++ b/sysdeps/unix/sysv/linux/x86_64/cancellation.S
@@ -19,7 +19,7 @@
#include <sysdep.h>
#include <tcb-offsets.h>
#include <kernel-features.h>
-#include "lowlevellock.h"
+#include <lowlevellock-futex.h>
#define PTHREAD_UNWIND JUMPTARGET(__pthread_unwind)
#if IS_IN (libpthread)
diff --git a/sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S b/sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S
deleted file mode 100644
index 32025277f3..0000000000
--- a/sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S
+++ /dev/null
@@ -1,19 +0,0 @@
-/* Copyright (C) 2002-2019 Free Software Foundation, Inc.
- This file is part of the GNU C Library.
- Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
-
- The GNU C Library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- The GNU C Library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with the GNU C Library; if not, see
- <http://www.gnu.org/licenses/>. */
-
-#include "lowlevellock.S"
diff --git a/sysdeps/unix/sysv/linux/x86_64/lll_timedlock_wait.c b/sysdeps/unix/sysv/linux/x86_64/lll_timedlock_wait.c
deleted file mode 100644
index f6875b8f89..0000000000
--- a/sysdeps/unix/sysv/linux/x86_64/lll_timedlock_wait.c
+++ /dev/null
@@ -1 +0,0 @@
-/* __lll_timedlock_wait is in lowlevellock.S. */
diff --git a/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S b/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S
deleted file mode 100644
index 166dbcfd3b..0000000000
--- a/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S
+++ /dev/null
@@ -1,348 +0,0 @@
-/* Copyright (C) 2002-2019 Free Software Foundation, Inc.
- This file is part of the GNU C Library.
- Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
-
- The GNU C Library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- The GNU C Library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with the GNU C Library; if not, see
- <http://www.gnu.org/licenses/>. */
-
-#include <sysdep.h>
-#include <pthread-errnos.h>
-#include <kernel-features.h>
-#include <lowlevellock.h>
-
-#include <stap-probe.h>
-
- .text
-
-#define LOAD_PRIVATE_FUTEX_WAIT(reg) \
- movl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg
-#define LOAD_PRIVATE_FUTEX_WAKE(reg) \
- movl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg
-#define LOAD_FUTEX_WAIT(reg) \
- xorl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg
-#define LOAD_FUTEX_WAIT_ABS(reg) \
- xorl $(FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME), reg
-#define LOAD_FUTEX_WAKE(reg) \
- xorl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg
-
-
- .globl __lll_lock_wait_private
- .type __lll_lock_wait_private,@function
- .hidden __lll_lock_wait_private
- .align 16
-__lll_lock_wait_private:
- cfi_startproc
- pushq %r10
- cfi_adjust_cfa_offset(8)
- pushq %rdx
- cfi_adjust_cfa_offset(8)
- cfi_offset(%r10, -16)
- cfi_offset(%rdx, -24)
- xorq %r10, %r10 /* No timeout. */
- movl $2, %edx
- LOAD_PRIVATE_FUTEX_WAIT (%esi)
-
- cmpl %edx, %eax /* NB: %edx == 2 */
- jne 2f
-
-1: LIBC_PROBE (lll_lock_wait_private, 1, %rdi)
- movl $SYS_futex, %eax
- syscall
-
-2: movl %edx, %eax
- xchgl %eax, (%rdi) /* NB: lock is implied */
-
- testl %eax, %eax
- jnz 1b
-
- popq %rdx
- cfi_adjust_cfa_offset(-8)
- cfi_restore(%rdx)
- popq %r10
- cfi_adjust_cfa_offset(-8)
- cfi_restore(%r10)
- retq
- cfi_endproc
- .size __lll_lock_wait_private,.-__lll_lock_wait_private
-
-#if !IS_IN (libc)
- .globl __lll_lock_wait
- .type __lll_lock_wait,@function
- .hidden __lll_lock_wait
- .align 16
-__lll_lock_wait:
- cfi_startproc
- pushq %r10
- cfi_adjust_cfa_offset(8)
- pushq %rdx
- cfi_adjust_cfa_offset(8)
- cfi_offset(%r10, -16)
- cfi_offset(%rdx, -24)
- xorq %r10, %r10 /* No timeout. */
- movl $2, %edx
- LOAD_FUTEX_WAIT (%esi)
-
- cmpl %edx, %eax /* NB: %edx == 2 */
- jne 2f
-
-1: LIBC_PROBE (lll_lock_wait, 2, %rdi, %rsi)
- movl $SYS_futex, %eax
- syscall
-
-2: movl %edx, %eax
- xchgl %eax, (%rdi) /* NB: lock is implied */
-
- testl %eax, %eax
- jnz 1b
-
- popq %rdx
- cfi_adjust_cfa_offset(-8)
- cfi_restore(%rdx)
- popq %r10
- cfi_adjust_cfa_offset(-8)
- cfi_restore(%r10)
- retq
- cfi_endproc
- .size __lll_lock_wait,.-__lll_lock_wait
-
- /* %rdi: futex
- %rsi: flags
- %rdx: timeout
- %eax: futex value
- */
- .globl __lll_timedlock_wait
- .type __lll_timedlock_wait,@function
- .hidden __lll_timedlock_wait
- .align 16
-__lll_timedlock_wait:
- cfi_startproc
-# ifndef __ASSUME_FUTEX_CLOCK_REALTIME
-# ifdef PIC
- cmpl $0, __have_futex_clock_realtime(%rip)
-# else
- cmpl $0, __have_futex_clock_realtime
-# endif
- je .Lreltmo
-# endif
-
- cmpq $0, (%rdx)
- js 5f
-
- pushq %r9
- cfi_adjust_cfa_offset(8)
- cfi_rel_offset(%r9, 0)
-
- movq %rdx, %r10
- movl $0xffffffff, %r9d
- LOAD_FUTEX_WAIT_ABS (%esi)
-
- movl $2, %edx
- cmpl %edx, %eax
- jne 2f
-
-1: movl $SYS_futex, %eax
- movl $2, %edx
- syscall
-
-2: xchgl %edx, (%rdi) /* NB: lock is implied */
-
- testl %edx, %edx
- jz 3f
-
- cmpl $-ETIMEDOUT, %eax
- je 4f
- cmpl $-EINVAL, %eax
- jne 1b
-4: movl %eax, %edx
- negl %edx
-
-3: movl %edx, %eax
- popq %r9
- cfi_adjust_cfa_offset(-8)
- cfi_restore(%r9)
- retq
-
-5: movl $ETIMEDOUT, %eax
- retq
-
-# ifndef __ASSUME_FUTEX_CLOCK_REALTIME
-.Lreltmo:
- /* Check for a valid timeout value. */
- cmpq $1000000000, 8(%rdx)
- jae 3f
-
- pushq %r8
- cfi_adjust_cfa_offset(8)
- pushq %r9
- cfi_adjust_cfa_offset(8)
- pushq %r12
- cfi_adjust_cfa_offset(8)
- pushq %r13
- cfi_adjust_cfa_offset(8)
- pushq %r14
- cfi_adjust_cfa_offset(8)
- cfi_offset(%r8, -16)
- cfi_offset(%r9, -24)
- cfi_offset(%r12, -32)
- cfi_offset(%r13, -40)
- cfi_offset(%r14, -48)
- pushq %rsi
- cfi_adjust_cfa_offset(8)
-
- /* Stack frame for the timespec and timeval structs. */
- subq $24, %rsp
- cfi_adjust_cfa_offset(24)
-
- movq %rdi, %r12
- movq %rdx, %r13
-
- movl $2, %edx
- xchgl %edx, (%r12)
-
- testl %edx, %edx
- je 6f
-
-1:
- /* Get current time. */
- movq %rsp, %rdi
- xorl %esi, %esi
- /* This call works because we directly jump to a system call entry
- which preserves all the registers. */
- call JUMPTARGET(__gettimeofday)
-
- /* Compute relative timeout. */
- movq 8(%rsp), %rax
- movl $1000, %edi
- mul %rdi /* Milli seconds to nano seconds. */
- movq (%r13), %rdi
- movq 8(%r13), %rsi
- subq (%rsp), %rdi
- subq %rax, %rsi
- jns 4f
- addq $1000000000, %rsi
- decq %rdi
-4: testq %rdi, %rdi
- js 2f /* Time is already up. */
-
- /* Store relative timeout. */
- movq %rdi, (%rsp)
- movq %rsi, 8(%rsp)
-
- /* Futex call. */
- movl $2, %edx
- movl $1, %eax
- movq %rsp, %r10
- movl 24(%rsp), %esi
- LOAD_FUTEX_WAIT (%esi)
- movq %r12, %rdi
- movl $SYS_futex, %eax
- syscall
-
- /* NB: %edx == 2 */
- xchgl %edx, (%r12)
-
- testl %edx, %edx
- je 6f
-
- cmpl $-ETIMEDOUT, %eax
- jne 1b
-2: movl $ETIMEDOUT, %edx
-
-6: addq $32, %rsp
- cfi_adjust_cfa_offset(-32)
- popq %r14
- cfi_adjust_cfa_offset(-8)
- cfi_restore(%r14)
- popq %r13
- cfi_adjust_cfa_offset(-8)
- cfi_restore(%r13)
- popq %r12
- cfi_adjust_cfa_offset(-8)
- cfi_restore(%r12)
- popq %r9
- cfi_adjust_cfa_offset(-8)
- cfi_restore(%r9)
- popq %r8
- cfi_adjust_cfa_offset(-8)
- cfi_restore(%r8)
- movl %edx, %eax
- retq
-
-3: movl $EINVAL, %eax
- retq
-# endif
- cfi_endproc
- .size __lll_timedlock_wait,.-__lll_timedlock_wait
-#endif
-
-
- .globl __lll_unlock_wake_private
- .type __lll_unlock_wake_private,@function
- .hidden __lll_unlock_wake_private
- .align 16
-__lll_unlock_wake_private:
- cfi_startproc
- pushq %rsi
- cfi_adjust_cfa_offset(8)
- pushq %rdx
- cfi_adjust_cfa_offset(8)
- cfi_offset(%rsi, -16)
- cfi_offset(%rdx, -24)
-
- movl $0, (%rdi)
- LOAD_PRIVATE_FUTEX_WAKE (%esi)
- movl $1, %edx /* Wake one thread. */
- movl $SYS_futex, %eax
- syscall
-
- popq %rdx
- cfi_adjust_cfa_offset(-8)
- cfi_restore(%rdx)
- popq %rsi
- cfi_adjust_cfa_offset(-8)
- cfi_restore(%rsi)
- retq
- cfi_endproc
- .size __lll_unlock_wake_private,.-__lll_unlock_wake_private
-
-#if !IS_IN (libc)
- .globl __lll_unlock_wake
- .type __lll_unlock_wake,@function
- .hidden __lll_unlock_wake
- .align 16
-__lll_unlock_wake:
- cfi_startproc
- pushq %rsi
- cfi_adjust_cfa_offset(8)
- pushq %rdx
- cfi_adjust_cfa_offset(8)
- cfi_offset(%rsi, -16)
- cfi_offset(%rdx, -24)
-
- movl $0, (%rdi)
- LOAD_FUTEX_WAKE (%esi)
- movl $1, %edx /* Wake one thread. */
- movl $SYS_futex, %eax
- syscall
-
- popq %rdx
- cfi_adjust_cfa_offset(-8)
- cfi_restore(%rdx)
- popq %rsi
- cfi_adjust_cfa_offset(-8)
- cfi_restore(%rsi)
- retq
- cfi_endproc
- .size __lll_unlock_wake,.-__lll_unlock_wake
-#endif
diff --git a/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h b/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
deleted file mode 100644
index 8cbc1caa5b..0000000000
--- a/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
+++ /dev/null
@@ -1,243 +0,0 @@
-/* Copyright (C) 2002-2019 Free Software Foundation, Inc.
- This file is part of the GNU C Library.
- Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
-
- The GNU C Library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- The GNU C Library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with the GNU C Library; if not, see
- <http://www.gnu.org/licenses/>. */
-
-#ifndef _LOWLEVELLOCK_H
-#define _LOWLEVELLOCK_H 1
-
-#ifndef __ASSEMBLER__
-# include <time.h>
-# include <sys/param.h>
-# include <bits/pthreadtypes.h>
-# include <kernel-features.h>
-
-# ifndef LOCK_INSTR
-# ifdef UP
-# define LOCK_INSTR /* nothing */
-# else
-# define LOCK_INSTR "lock;"
-# endif
-# endif
-#else
-# ifndef LOCK
-# ifdef UP
-# define LOCK
-# else
-# define LOCK lock
-# endif
-# endif
-#endif
-
-#include <lowlevellock-futex.h>
-
-/* XXX Remove when no assembler code uses futexes anymore. */
-#define SYS_futex __NR_futex
-
-#ifndef __ASSEMBLER__
-
-/* Initializer for lock. */
-#define LLL_LOCK_INITIALIZER (0)
-#define LLL_LOCK_INITIALIZER_LOCKED (1)
-#define LLL_LOCK_INITIALIZER_WAITERS (2)
-
-
-/* NB: in the lll_trylock macro we simply return the value in %eax
- after the cmpxchg instruction. In case the operation succeded this
- value is zero. In case the operation failed, the cmpxchg instruction
- has loaded the current value of the memory work which is guaranteed
- to be nonzero. */
-#if !IS_IN (libc) || defined UP
-# define __lll_trylock_asm LOCK_INSTR "cmpxchgl %2, %1"
-#else
-# define __lll_trylock_asm "cmpl $0, __libc_multiple_threads(%%rip)\n\t" \
- "je 0f\n\t" \
- "lock; cmpxchgl %2, %1\n\t" \
- "jmp 1f\n\t" \
- "0:\tcmpxchgl %2, %1\n\t" \
- "1:"
-#endif
-
-#define lll_trylock(futex) \
- ({ int ret; \
- __asm __volatile (__lll_trylock_asm \
- : "=a" (ret), "=m" (futex) \
- : "r" (LLL_LOCK_INITIALIZER_LOCKED), "m" (futex), \
- "0" (LLL_LOCK_INITIALIZER) \
- : "memory"); \
- ret; })
-
-#define lll_cond_trylock(futex) \
- ({ int ret; \
- __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1" \
- : "=a" (ret), "=m" (futex) \
- : "r" (LLL_LOCK_INITIALIZER_WAITERS), \
- "m" (futex), "0" (LLL_LOCK_INITIALIZER) \
- : "memory"); \
- ret; })
-
-#if !IS_IN (libc) || defined UP
-# define __lll_lock_asm_start LOCK_INSTR "cmpxchgl %4, %2\n\t" \
- "jz 24f\n\t"
-#else
-# define __lll_lock_asm_start "cmpl $0, __libc_multiple_threads(%%rip)\n\t" \
- "je 0f\n\t" \
- "lock; cmpxchgl %4, %2\n\t" \
- "jnz 1f\n\t" \
- "jmp 24f\n" \
- "0:\tcmpxchgl %4, %2\n\t" \
- "jz 24f\n\t"
-#endif
-
-#define lll_lock(futex, private) \
- (void) \
- ({ int ignore1, ignore2, ignore3; \
- if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \
- __asm __volatile (__lll_lock_asm_start \
- "1:\tlea %2, %%" RDI_LP "\n" \
- "2:\tsub $128, %%" RSP_LP "\n" \
- ".cfi_adjust_cfa_offset 128\n" \
- "3:\tcallq __lll_lock_wait_private\n" \
- "4:\tadd $128, %%" RSP_LP "\n" \
- ".cfi_adjust_cfa_offset -128\n" \
- "24:" \
- : "=S" (ignore1), "=&D" (ignore2), "=m" (futex), \
- "=a" (ignore3) \
- : "0" (1), "m" (futex), "3" (0) \
- : "cx", "r11", "cc", "memory"); \
- else \
- __asm __volatile (__lll_lock_asm_start \
- "1:\tlea %2, %%" RDI_LP "\n" \
- "2:\tsub $128, %%" RSP_LP "\n" \
- ".cfi_adjust_cfa_offset 128\n" \
- "3:\tcallq __lll_lock_wait\n" \
- "4:\tadd $128, %%" RSP_LP "\n" \
- ".cfi_adjust_cfa_offset -128\n" \
- "24:" \
- : "=S" (ignore1), "=D" (ignore2), "=m" (futex), \
- "=a" (ignore3) \
- : "1" (1), "m" (futex), "3" (0), "0" (private) \
- : "cx", "r11", "cc", "memory"); \
- }) \
-
-#define lll_cond_lock(futex, private) \
- (void) \
- ({ int ignore1, ignore2, ignore3; \
- __asm __volatile (LOCK_INSTR "cmpxchgl %4, %2\n\t" \
- "jz 24f\n" \
- "1:\tlea %2, %%" RDI_LP "\n" \
- "2:\tsub $128, %%" RSP_LP "\n" \
- ".cfi_adjust_cfa_offset 128\n" \
- "3:\tcallq __lll_lock_wait\n" \
- "4:\tadd $128, %%" RSP_LP "\n" \
- ".cfi_adjust_cfa_offset -128\n" \
- "24:" \
- : "=S" (ignore1), "=D" (ignore2), "=m" (futex), \
- "=a" (ignore3) \
- : "1" (2), "m" (futex), "3" (0), "0" (private) \
- : "cx", "r11", "cc", "memory"); \
- })
-
-#define lll_timedlock(futex, timeout, private) \
- ({ int result, ignore1, ignore2, ignore3; \
- __asm __volatile (LOCK_INSTR "cmpxchgl %1, %4\n\t" \
- "jz 24f\n" \
- "1:\tlea %4, %%" RDI_LP "\n" \
- "0:\tmov %8, %%" RDX_LP "\n" \
- "2:\tsub $128, %%" RSP_LP "\n" \
- ".cfi_adjust_cfa_offset 128\n" \
- "3:\tcallq __lll_timedlock_wait\n" \
- "4:\tadd $128, %%" RSP_LP "\n" \
- ".cfi_adjust_cfa_offset -128\n" \
- "24:" \
- : "=a" (result), "=D" (ignore1), "=S" (ignore2), \
- "=&d" (ignore3), "=m" (futex) \
- : "0" (0), "1" (1), "m" (futex), "m" (timeout), \
- "2" (private) \
- : "memory", "cx", "cc", "r10", "r11"); \
- result; })
-
-extern int __lll_timedlock_elision (int *futex, short *adapt_count,
- const struct timespec *timeout,
- int private) attribute_hidden;
-
-#define lll_timedlock_elision(futex, adapt_count, timeout, private) \
- __lll_timedlock_elision(&(futex), &(adapt_count), timeout, private)
-
-#if !IS_IN (libc) || defined UP
-# define __lll_unlock_asm_start LOCK_INSTR "decl %0\n\t" \
- "je 24f\n\t"
-#else
-# define __lll_unlock_asm_start "cmpl $0, __libc_multiple_threads(%%rip)\n\t" \
- "je 0f\n\t" \
- "lock; decl %0\n\t" \
- "jne 1f\n\t" \
- "jmp 24f\n\t" \
- "0:\tdecl %0\n\t" \
- "je 24f\n\t"
-#endif
-
-#define lll_unlock(futex, private) \
- (void) \
- ({ int ignore; \
- if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \
- __asm __volatile (__lll_unlock_asm_start \
- "1:\tlea %0, %%" RDI_LP "\n" \
- "2:\tsub $128, %%" RSP_LP "\n" \
- ".cfi_adjust_cfa_offset 128\n" \
- "3:\tcallq __lll_unlock_wake_private\n" \
- "4:\tadd $128, %%" RSP_LP "\n" \
- ".cfi_adjust_cfa_offset -128\n" \
- "24:" \
- : "=m" (futex), "=&D" (ignore) \
- : "m" (futex) \
- : "ax", "cx", "r11", "cc", "memory"); \
- else \
- __asm __volatile (__lll_unlock_asm_start \
- "1:\tlea %0, %%" RDI_LP "\n" \
- "2:\tsub $128, %%" RSP_LP "\n" \
- ".cfi_adjust_cfa_offset 128\n" \
- "3:\tcallq __lll_unlock_wake\n" \
- "4:\tadd $128, %%" RSP_LP "\n" \
- ".cfi_adjust_cfa_offset -128\n" \
- "24:" \
- : "=m" (futex), "=&D" (ignore) \
- : "m" (futex), "S" (private) \
- : "ax", "cx", "r11", "cc", "memory"); \
- })
-
-#define lll_islocked(futex) \
- (futex != LLL_LOCK_INITIALIZER)
-
-extern int __lll_lock_elision (int *futex, short *adapt_count, int private)
- attribute_hidden;
-
-extern int __lll_unlock_elision (int *lock, int private)
- attribute_hidden;
-
-extern int __lll_trylock_elision (int *lock, short *adapt_count)
- attribute_hidden;
-
-#define lll_lock_elision(futex, adapt_count, private) \
- __lll_lock_elision (&(futex), &(adapt_count), private)
-#define lll_unlock_elision(futex, adapt_count, private) \
- __lll_unlock_elision (&(futex), private)
-#define lll_trylock_elision(futex, adapt_count) \
- __lll_trylock_elision (&(futex), &(adapt_count))
-
-#endif /* !__ASSEMBLER__ */
-
-#endif /* lowlevellock.h */
--
2.17.1