nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h

   1 /* Copyright (C) 2002, 2003 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, write to the Free
  17    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
  18    02111-1307 USA.  */
  19
  20 #ifndef _LOWLEVELLOCK_H
  21 #define _LOWLEVELLOCK_H 1
  22
  23 #include <time.h>
  24 #include <sys/param.h>
  25 #include <bits/pthreadtypes.h>
  26
  27 #ifndef LOCK_INSTR
  28 # ifdef UP
  29 #  define LOCK_INSTR    /* nothing */
  30 # else
  31 #  define LOCK_INSTR "lock;"
  32 # endif
  33 #endif
  34
  35 #define SYS_futex               202
  36 #define FUTEX_WAIT              0
  37 #define FUTEX_WAKE              1
  38
  39
  40 /* Initializer for compatibility lock.  */
  41 #define LLL_MUTEX_LOCK_INITIALIZER              (0)
  42 #define LLL_MUTEX_LOCK_INITIALIZER_LOCKED       (1)
  43
  44
  45 #define lll_futex_wait(futex, val) \
  46   do {                                                                        \
  47     int __ignore;                                                             \
  48     register __typeof (val) _val asm ("edx") = (val);                         \
  49     __asm __volatile ("xorq %%r10, %%r10\n\t"                                 \
  50                       "syscall"                                               \
  51                       : "=a" (__ignore)                                       \
  52                       : "0" (SYS_futex), "D" (&futex), "S" (FUTEX_WAIT),      \
  53                         "d" (_val)                                            \
  54                       : "memory", "cc", "r10", "r11", "cx");                  \
  55   } while (0)
  56
  57
  58 #define lll_futex_wake(futex, nr) \
  59   do {                                                                        \
  60     int __ignore;                                                             \
  61     register __typeof (nr) _nr asm ("edx") = (nr);                            \
  62     __asm __volatile ("syscall"                                               \
  63                       : "=a" (__ignore)                                       \
  64                       : "0" (SYS_futex), "D" (&futex), "S" (FUTEX_WAKE),      \
  65                         "d" (_nr)                                             \
  66                       : "memory", "cc", "r10", "r11", "cx");                  \
  67   } while (0)
  68
  69
  70 /* Does not preserve %eax and %ecx.  */
  71 extern int __lll_mutex_lock_wait (int *__futex, int __val) attribute_hidden;
  72 /* Does not preserver %eax, %ecx, and %edx.  */
  73 extern int __lll_mutex_timedlock_wait (int *__futex, int __val,
  74                                        const struct timespec *__abstime)
  75      attribute_hidden;
  76 /* Preserves all registers but %eax.  */
  77 extern int __lll_mutex_unlock_wait (int *__futex) attribute_hidden;
  78
  79
  80 #define lll_mutex_trylock(futex) \
  81   ({ unsigned char ret;                                                       \
  82      __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1; setne %0"                 \
  83                        : "=a" (ret), "=m" (futex)                             \
  84                        : "r" (LLL_MUTEX_LOCK_INITIALIZER_LOCKED), "m" (futex),\
  85                          "0" (LLL_MUTEX_LOCK_INITIALIZER)                     \
  86                        : "memory");                                           \
  87      ret; })
  88
  89
  90 #define lll_mutex_lock(futex) \
  91   (void) ({ int ignore1, ignore2, ignore3;                                    \
  92             __asm __volatile (LOCK_INSTR "cmpxchgl %0, %2\n\t"                \
  93                               "jnz 1f\n\t"                                    \
  94                               ".subsection 1\n"                               \
  95                               "1:\tleaq %2, %%rdi\n\t"                        \
  96                               "subq $128, %%rsp\n\t"                          \
  97                               "callq __lll_mutex_lock_wait\n\t"               \
  98                               "addq $128, %%rsp\n\t"                          \
  99                               "jmp 2f\n\t"                                    \
 100                               ".previous\n"                                   \
 101                               "2:"                                            \
 102                               : "=S" (ignore1), "=&D" (ignore2), "=m" (futex),\
 103                                 "=a" (ignore3)                                \
 104                               : "0" (1), "m" (futex), "3" (0)                 \
 105                               : "cx", "r11", "cc", "memory"); })
 106
 107
 108 #define lll_mutex_cond_lock(futex) \
 109   (void) ({ int ignore1, ignore2, ignore3;                                    \
 110             __asm __volatile (LOCK_INSTR "cmpxchgl %0, %2\n\t"                \
 111                               "jnz 1f\n\t"                                    \
 112                               ".subsection 1\n"                               \
 113                               "1:\tleaq %2, %%rdi\n\t"                        \
 114                               "subq $128, %%rsp\n\t"                          \
 115                               "callq __lll_mutex_lock_wait\n\t"               \
 116                               "addq $128, %%rsp\n\t"                          \
 117                               "jmp 2f\n\t"                                    \
 118                               ".previous\n"                                   \
 119                               "2:"                                            \
 120                               : "=S" (ignore1), "=&D" (ignore2), "=m" (futex),\
 121                                 "=a" (ignore3)                                \
 122                               : "0" (2), "m" (futex), "3" (0)                 \
 123                               : "cx", "r11", "cc", "memory"); })
 124
 125
 126 #define lll_mutex_timedlock(futex, timeout) \
 127   ({ int result, ignore1, ignore2, ignore3;                                   \
 128      __asm __volatile (LOCK_INSTR "cmpxchgl %2, %4\n\t"                       \
 129                        "jnz 1f\n\t"                                           \
 130                        ".subsection 1\n"                                      \
 131                        "1:\tleaq %4, %%rdi\n\t"                               \
 132                        "movq %8, %%rdx\n\t"                                   \
 133                        "subq $128, %%rsp\n\t"                                 \
 134                        "callq __lll_mutex_timedlock_wait\n\t"                 \
 135                        "addq $128, %%rsp\n\t"                                 \
 136                        "jmp 2f\n\t"                                           \
 137                        ".previous\n"                                          \
 138                        "2:"                                                   \
 139                        : "=a" (result), "=&D" (ignore1), "=S" (ignore2),      \
 140                          "=&d" (ignore3), "=m" (futex)                        \
 141                        : "0" (0), "2" (1), "m" (futex), "m" (timeout)         \
 142                        : "memory", "cx", "cc", "r10", "r11");                 \
 143      result; })
 144
 145
 146 #define lll_mutex_unlock(futex) \
 147   (void) ({ int ignore;                                                       \
 148             __asm __volatile (LOCK_INSTR "decl %0\n\t"                        \
 149                               "jne 1f\n\t"                                    \
 150                               ".subsection 1\n"                               \
 151                               "1:\tleaq %0, %%rdi\n\t"                        \
 152                               "subq $128, %%rsp\n\t"                          \
 153                               "callq __lll_mutex_unlock_wake\n\t"             \
 154                               "addq $128, %%rsp\n\t"                          \
 155                               "jmp 2f\n\t"                                    \
 156                               ".previous\n"                                   \
 157                               "2:"                                            \
 158                               : "=m" (futex), "=&D" (ignore)                  \
 159                               : "m" (futex)                                   \
 160                               : "ax", "cx", "r11", "cc", "memory"); })
 161
 162
 163 #define lll_mutex_islocked(futex) \
 164   (futex != LLL_MUTEX_LOCK_INITIALIZER)
 165
 166
 167 /* We have a separate internal lock implementation which is not tied
 168    to binary compatibility.  */
 169
 170 /* Type for lock object.  */
 171 typedef int lll_lock_t;
 172
 173 /* Initializers for lock.  */
 174 #define LLL_LOCK_INITIALIZER            (0)
 175 #define LLL_LOCK_INITIALIZER_LOCKED     (1)
 176
 177
 178 extern int lll_unlock_wake_cb (int *__futex) attribute_hidden;
 179
 180
 181 /* The states of a lock are:
 182     0  -  untaken
 183     1  -  taken by one user
 184     2  -  taken by more users */
 185
 186
 187 #if defined NOT_IN_libc || defined UP
 188 # define lll_trylock(futex) lll_mutex_trylock (futex)
 189 # define lll_lock(futex) lll_mutex_lock (futex)
 190 # define lll_unlock(futex) lll_mutex_unlock (futex)
 191 #else
 192 /* Special versions of the macros for use in libc itself.  They avoid
 193    the lock prefix when the thread library is not used.
 194
 195    XXX In future we might even want to avoid it on UP machines.  */
 196
 197 # define lll_trylock(futex) \
 198   ({ unsigned char ret;                                                       \
 199      __asm __volatile ("cmpl $0, __libc_multiple_threads(%%rip)\n\t"          \
 200                        "je 0f\n\t"                                            \
 201                        "lock\n"                                               \
 202                        "0:\tcmpxchgl %2, %1; setne %0"                        \
 203                        : "=a" (ret), "=m" (futex)                             \
 204                        : "r" (LLL_MUTEX_LOCK_INITIALIZER_LOCKED), "m" (futex),\
 205                          "0" (LLL_MUTEX_LOCK_INITIALIZER)                     \
 206                        : "memory");                                           \
 207      ret; })
 208
 209
 210 # define lll_lock(futex) \
 211   (void) ({ int ignore1, ignore2, ignore3;                                    \
 212             __asm __volatile ("cmpl $0, __libc_multiple_threads(%%rip)\n\t"   \
 213                               "je 0f\n\t"                                     \
 214                               "lock\n"                                        \
 215                               "0:\tcmpxchgl %0, %2\n\t"                       \
 216                               "jnz 1f\n\t"                                    \
 217                               ".subsection 1\n"                               \
 218                               "1:\tleaq %2, %%rdi\n\t"                        \
 219                               "subq $128, %%rsp\n\t"                          \
 220                               "callq __lll_mutex_lock_wait\n\t"               \
 221                               "addq $128, %%rsp\n\t"                          \
 222                               "jmp 2f\n\t"                                    \
 223                               ".previous\n"                                   \
 224                               "2:"                                            \
 225                               : "=S" (ignore1), "=&D" (ignore2), "=m" (futex),\
 226                                 "=a" (ignore3)                                \
 227                               : "0" (1), "m" (futex), "3" (0)                 \
 228                               : "cx", "r11", "cc", "memory"); })
 229
 230
 231 # define lll_unlock(futex) \
 232   (void) ({ int ignore;                                                       \
 233             __asm __volatile ("cmpl $0, __libc_multiple_threads(%%rip)\n\t"   \
 234                               "je 0f\n\t"                                     \
 235                               "lock\n"                                        \
 236                               "0:\tdecl %0\n\t"                               \
 237                               "jne 1f\n\t"                                    \
 238                               ".subsection 1\n"                               \
 239                               "1:\tleaq %0, %%rdi\n\t"                        \
 240                               "subq $128, %%rsp\n\t"                          \
 241                               "callq __lll_mutex_unlock_wake\n\t"             \
 242                               "addq $128, %%rsp\n\t"                          \
 243                               "jmp 2f\n\t"                                    \
 244                               ".previous\n"                                   \
 245                               "2:"                                            \
 246                               : "=m" (futex), "=&D" (ignore)                  \
 247                               : "0" (futex)                                   \
 248                               : "ax", "cx", "r11", "cc", "memory"); })
 249 #endif
 250
 251
 252 #define lll_islocked(futex) \
 253   (futex != LLL_MUTEX_LOCK_INITIALIZER)
 254
 255
 256 /* The kernel notifies a process with uses CLONE_CLEARTID via futex
 257    wakeup when the clone terminates.  The memory location contains the
 258    thread ID while the clone is running and is reset to zero
 259    afterwards.
 260
 261    The macro parameter must not have any side effect.  */
 262 #define lll_wait_tid(tid) \
 263   do {                                                                        \
 264     int __ignore;                                                             \
 265     register __typeof (tid) _tid asm ("edx") = (tid);                         \
 266     if (_tid != 0)                                                            \
 267       __asm __volatile ("xorq %%r10, %%r10\n\t"                               \
 268                         "1:\tmovq %2, %%rax\n\t"                              \
 269                         "syscall\n\t"                                         \
 270                         "cmpl $0, (%%rdi)\n\t"                                \
 271                         "jne 1b"                                              \
 272                         : "=&a" (__ignore)                                    \
 273                         : "S" (FUTEX_WAIT), "i" (SYS_futex), "D" (&tid),      \
 274                           "d" (_tid)                                          \
 275                         : "memory", "cc", "r10", "r11", "cx");                \
 276   } while (0)
 277
 278 extern int __lll_timedwait_tid (int *tid, const struct timespec *abstime)
 279      attribute_hidden;
 280 #define lll_timedwait_tid(tid, abstime) \
 281   ({                                                                          \
 282     int __result = 0;                                                         \
 283     if (tid != 0)                                                             \
 284       {                                                                       \
 285         if (abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000)           \
 286           __result = EINVAL;                                                  \
 287         else                                                                  \
 288           __result = __lll_timedwait_tid (&tid, abstime);                     \
 289       }                                                                       \
 290     __result; })
 291
 292
 293 /* Conditional variable handling.  */
 294
 295 extern void __lll_cond_wait (pthread_cond_t *cond) attribute_hidden;
 296 extern int __lll_cond_timedwait (pthread_cond_t *cond,
 297                                  const struct timespec *abstime)
 298      attribute_hidden;
 299 extern void __lll_cond_wake (pthread_cond_t *cond) attribute_hidden;
 300 extern void __lll_cond_broadcast (pthread_cond_t *cond) attribute_hidden;
 301
 302
 303 #define lll_cond_wait(cond) \
 304   __lll_cond_wait (cond)
 305 #define lll_cond_timedwait(cond, abstime) \
 306   __lll_cond_timedwait (cond, abstime)
 307 #define lll_cond_wake(cond) \
 308   __lll_cond_wake (cond)
 309 #define lll_cond_broadcast(cond) \
 310   __lll_cond_broadcast (cond)
 311
 312
 313 #endif  /* lowlevellock.h */