nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S

   1 /* Copyright (C) 2002-2012 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <http://www.gnu.org/licenses/>.  */
  18
  19 #include <sysdep.h>
  20 #include <shlib-compat.h>
  21 #include <lowlevellock.h>
  22 #include <lowlevelcond.h>
  23 #include <pthread-pi-defines.h>
  24 #include <pthread-errnos.h>
  25 #include <stap-probe.h>
  26
  27 #include <kernel-features.h>
  28
  29
  30         .text
  31
  32
  33 /* int pthread_cond_timedwait (pthread_cond_t *cond, pthread_mutex_t *mutex,
  34                                const struct timespec *abstime)  */
  35         .globl  __pthread_cond_timedwait
  36         .type   __pthread_cond_timedwait, @function
  37         .align  16
  38 __pthread_cond_timedwait:
  39 .LSTARTCODE:
  40         cfi_startproc
  41 #ifdef SHARED
  42         cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect,
  43                         DW.ref.__gcc_personality_v0)
  44         cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART)
  45 #else
  46         cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0)
  47         cfi_lsda(DW_EH_PE_udata4, .LexceptSTART)
  48 #endif
  49
  50         pushq   %r12
  51         cfi_adjust_cfa_offset(8)
  52         cfi_rel_offset(%r12, 0)
  53         pushq   %r13
  54         cfi_adjust_cfa_offset(8)
  55         cfi_rel_offset(%r13, 0)
  56         pushq   %r14
  57         cfi_adjust_cfa_offset(8)
  58         cfi_rel_offset(%r14, 0)
  59         pushq   %r15
  60         cfi_adjust_cfa_offset(8)
  61         cfi_rel_offset(%r15, 0)
  62 #ifdef __ASSUME_FUTEX_CLOCK_REALTIME
  63 # define FRAME_SIZE (32+8)
  64 #else
  65 # define FRAME_SIZE (48+8)
  66 #endif
  67         subq    $FRAME_SIZE, %rsp
  68         cfi_adjust_cfa_offset(FRAME_SIZE)
  69         cfi_remember_state
  70
  71         LIBC_PROBE (cond_timedwait, 3, %rdi, %rsi, %rdx)
  72
  73         cmpq    $1000000000, 8(%rdx)
  74         movl    $EINVAL, %eax
  75         jae     48f
  76
  77         /* Stack frame:
  78
  79            rsp + 48
  80                     +--------------------------+
  81            rsp + 32 | timeout value            |
  82                     +--------------------------+
  83            rsp + 24 | old wake_seq value       |
  84                     +--------------------------+
  85            rsp + 16 | mutex pointer            |
  86                     +--------------------------+
  87            rsp +  8 | condvar pointer          |
  88                     +--------------------------+
  89            rsp +  4 | old broadcast_seq value  |
  90                     +--------------------------+
  91            rsp +  0 | old cancellation mode    |
  92                     +--------------------------+
  93         */
  94
  95         LP_OP(cmp) $-1, dep_mutex(%rdi)
  96
  97         /* Prepare structure passed to cancellation handler.  */
  98         movq    %rdi, 8(%rsp)
  99         movq    %rsi, 16(%rsp)
 100         movq    %rdx, %r13
 101
 102         je      22f
 103         mov     %RSI_LP, dep_mutex(%rdi)
 104
 105 22:
 106         xorl    %r15d, %r15d
 107
 108 #ifndef __ASSUME_FUTEX_CLOCK_REALTIME
 109 #  ifdef PIC
 110         cmpl    $0, __have_futex_clock_realtime(%rip)
 111 #  else
 112         cmpl    $0, __have_futex_clock_realtime
 113 #  endif
 114         je      .Lreltmo
 115 #endif
 116
 117         /* Get internal lock.  */
 118         movl    $1, %esi
 119         xorl    %eax, %eax
 120         LOCK
 121 #if cond_lock == 0
 122         cmpxchgl %esi, (%rdi)
 123 #else
 124         cmpxchgl %esi, cond_lock(%rdi)
 125 #endif
 126         jnz     31f
 127
 128         /* Unlock the mutex.  */
 129 32:     movq    16(%rsp), %rdi
 130         xorl    %esi, %esi
 131         callq   __pthread_mutex_unlock_usercnt
 132
 133         testl   %eax, %eax
 134         jne     46f
 135
 136         movq    8(%rsp), %rdi
 137         incq    total_seq(%rdi)
 138         incl    cond_futex(%rdi)
 139         addl    $(1 << nwaiters_shift), cond_nwaiters(%rdi)
 140
 141         /* Get and store current wakeup_seq value.  */
 142         movq    8(%rsp), %rdi
 143         movq    wakeup_seq(%rdi), %r9
 144         movl    broadcast_seq(%rdi), %edx
 145         movq    %r9, 24(%rsp)
 146         movl    %edx, 4(%rsp)
 147
 148         cmpq    $0, (%r13)
 149         movq    $-ETIMEDOUT, %r14
 150         js      36f
 151
 152 38:     movl    cond_futex(%rdi), %r12d
 153
 154         /* Unlock.  */
 155         LOCK
 156 #if cond_lock == 0
 157         decl    (%rdi)
 158 #else
 159         decl    cond_lock(%rdi)
 160 #endif
 161         jne     33f
 162
 163 .LcleanupSTART1:
 164 34:     callq   __pthread_enable_asynccancel
 165         movl    %eax, (%rsp)
 166
 167         movq    %r13, %r10
 168         movl    $FUTEX_WAIT_BITSET, %esi
 169         LP_OP(cmp) $-1, dep_mutex(%rdi)
 170         je      60f
 171
 172         mov     dep_mutex(%rdi), %R8_LP
 173         /* Requeue to a non-robust PI mutex if the PI bit is set and
 174         the robust bit is not set.  */
 175         movl    MUTEX_KIND(%r8), %eax
 176         andl    $(ROBUST_BIT|PI_BIT), %eax
 177         cmpl    $PI_BIT, %eax
 178         jne     61f
 179
 180         movl    $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi
 181         xorl    %eax, %eax
 182         /* The following only works like this because we only support
 183            two clocks, represented using a single bit.  */
 184         testl   $1, cond_nwaiters(%rdi)
 185         movl    $FUTEX_CLOCK_REALTIME, %edx
 186         cmove   %edx, %eax
 187         orl     %eax, %esi
 188         movq    %r12, %rdx
 189         addq    $cond_futex, %rdi
 190         movl    $SYS_futex, %eax
 191         syscall
 192
 193         movl    $1, %r15d
 194 #ifdef __ASSUME_REQUEUE_PI
 195         jmp     62f
 196 #else
 197         cmpq    $-4095, %rax
 198         jnae    62f
 199
 200         subq    $cond_futex, %rdi
 201 #endif
 202
 203 61:     movl    $(FUTEX_WAIT_BITSET|FUTEX_PRIVATE_FLAG), %esi
 204 60:     xorl    %r15d, %r15d
 205         xorl    %eax, %eax
 206         /* The following only works like this because we only support
 207            two clocks, represented using a single bit.  */
 208         testl   $1, cond_nwaiters(%rdi)
 209         movl    $FUTEX_CLOCK_REALTIME, %edx
 210         movl    $0xffffffff, %r9d
 211         cmove   %edx, %eax
 212         orl     %eax, %esi
 213         movq    %r12, %rdx
 214         addq    $cond_futex, %rdi
 215         movl    $SYS_futex, %eax
 216         syscall
 217 62:     movq    %rax, %r14
 218
 219         movl    (%rsp), %edi
 220         callq   __pthread_disable_asynccancel
 221 .LcleanupEND1:
 222
 223         /* Lock.  */
 224         movq    8(%rsp), %rdi
 225         movl    $1, %esi
 226         xorl    %eax, %eax
 227         LOCK
 228 #if cond_lock == 0
 229         cmpxchgl %esi, (%rdi)
 230 #else
 231         cmpxchgl %esi, cond_lock(%rdi)
 232 #endif
 233         jne     35f
 234
 235 36:     movl    broadcast_seq(%rdi), %edx
 236
 237         movq    woken_seq(%rdi), %rax
 238
 239         movq    wakeup_seq(%rdi), %r9
 240
 241         cmpl    4(%rsp), %edx
 242         jne     53f
 243
 244         cmpq    24(%rsp), %r9
 245         jbe     45f
 246
 247         cmpq    %rax, %r9
 248         ja      39f
 249
 250 45:     cmpq    $-ETIMEDOUT, %r14
 251         jne     38b
 252
 253 99:     incq    wakeup_seq(%rdi)
 254         incl    cond_futex(%rdi)
 255         movl    $ETIMEDOUT, %r14d
 256         jmp     44f
 257
 258 53:     xorq    %r14, %r14
 259         jmp     54f
 260
 261 39:     xorq    %r14, %r14
 262 44:     incq    woken_seq(%rdi)
 263
 264 54:     subl    $(1 << nwaiters_shift), cond_nwaiters(%rdi)
 265
 266         /* Wake up a thread which wants to destroy the condvar object.  */
 267         cmpq    $0xffffffffffffffff, total_seq(%rdi)
 268         jne     55f
 269         movl    cond_nwaiters(%rdi), %eax
 270         andl    $~((1 << nwaiters_shift) - 1), %eax
 271         jne     55f
 272
 273         addq    $cond_nwaiters, %rdi
 274         LP_OP(cmp) $-1, dep_mutex-cond_nwaiters(%rdi)
 275         movl    $1, %edx
 276 #ifdef __ASSUME_PRIVATE_FUTEX
 277         movl    $FUTEX_WAKE, %eax
 278         movl    $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
 279         cmove   %eax, %esi
 280 #else
 281         movl    $0, %eax
 282         movl    %fs:PRIVATE_FUTEX, %esi
 283         cmove   %eax, %esi
 284         orl     $FUTEX_WAKE, %esi
 285 #endif
 286         movl    $SYS_futex, %eax
 287         syscall
 288         subq    $cond_nwaiters, %rdi
 289
 290 55:     LOCK
 291 #if cond_lock == 0
 292         decl    (%rdi)
 293 #else
 294         decl    cond_lock(%rdi)
 295 #endif
 296         jne     40f
 297
 298         /* If requeue_pi is used the kernel performs the locking of the
 299            mutex. */
 300 41:     movq    16(%rsp), %rdi
 301         testl   %r15d, %r15d
 302         jnz     64f
 303
 304         callq   __pthread_mutex_cond_lock
 305
 306 63:     testq   %rax, %rax
 307         cmoveq  %r14, %rax
 308
 309 48:     addq    $FRAME_SIZE, %rsp
 310         cfi_adjust_cfa_offset(-FRAME_SIZE)
 311         popq    %r15
 312         cfi_adjust_cfa_offset(-8)
 313         cfi_restore(%r15)
 314         popq    %r14
 315         cfi_adjust_cfa_offset(-8)
 316         cfi_restore(%r14)
 317         popq    %r13
 318         cfi_adjust_cfa_offset(-8)
 319         cfi_restore(%r13)
 320         popq    %r12
 321         cfi_adjust_cfa_offset(-8)
 322         cfi_restore(%r12)
 323
 324         retq
 325
 326         cfi_restore_state
 327
 328 64:     callq   __pthread_mutex_cond_lock_adjust
 329         movq    %r14, %rax
 330         jmp     48b
 331
 332         /* Initial locking failed.  */
 333 31:
 334 #if cond_lock != 0
 335         addq    $cond_lock, %rdi
 336 #endif
 337         LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
 338         movl    $LLL_PRIVATE, %eax
 339         movl    $LLL_SHARED, %esi
 340         cmovne  %eax, %esi
 341         callq   __lll_lock_wait
 342         jmp     32b
 343
 344         /* Unlock in loop requires wakeup.  */
 345 33:
 346 #if cond_lock != 0
 347         addq    $cond_lock, %rdi
 348 #endif
 349         LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
 350         movl    $LLL_PRIVATE, %eax
 351         movl    $LLL_SHARED, %esi
 352         cmovne  %eax, %esi
 353         callq   __lll_unlock_wake
 354         jmp     34b
 355
 356         /* Locking in loop failed.  */
 357 35:
 358 #if cond_lock != 0
 359         addq    $cond_lock, %rdi
 360 #endif
 361         LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
 362         movl    $LLL_PRIVATE, %eax
 363         movl    $LLL_SHARED, %esi
 364         cmovne  %eax, %esi
 365         callq   __lll_lock_wait
 366 #if cond_lock != 0
 367         subq    $cond_lock, %rdi
 368 #endif
 369         jmp     36b
 370
 371         /* Unlock after loop requires wakeup.  */
 372 40:
 373 #if cond_lock != 0
 374         addq    $cond_lock, %rdi
 375 #endif
 376         LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
 377         movl    $LLL_PRIVATE, %eax
 378         movl    $LLL_SHARED, %esi
 379         cmovne  %eax, %esi
 380         callq   __lll_unlock_wake
 381         jmp     41b
 382
 383         /* The initial unlocking of the mutex failed.  */
 384 46:     movq    8(%rsp), %rdi
 385         movq    %rax, (%rsp)
 386         LOCK
 387 #if cond_lock == 0
 388         decl    (%rdi)
 389 #else
 390         decl    cond_lock(%rdi)
 391 #endif
 392         jne     47f
 393
 394 #if cond_lock != 0
 395         addq    $cond_lock, %rdi
 396 #endif
 397         LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
 398         movl    $LLL_PRIVATE, %eax
 399         movl    $LLL_SHARED, %esi
 400         cmovne  %eax, %esi
 401         callq   __lll_unlock_wake
 402
 403 47:     movq    (%rsp), %rax
 404         jmp     48b
 405
 406
 407 #ifndef __ASSUME_FUTEX_CLOCK_REALTIME
 408 .Lreltmo:
 409         /* Get internal lock.  */
 410         movl    $1, %esi
 411         xorl    %eax, %eax
 412         LOCK
 413 # if cond_lock == 0
 414         cmpxchgl %esi, (%rdi)
 415 # else
 416         cmpxchgl %esi, cond_lock(%rdi)
 417 # endif
 418         jnz     1f
 419
 420         /* Unlock the mutex.  */
 421 2:      movq    16(%rsp), %rdi
 422         xorl    %esi, %esi
 423         callq   __pthread_mutex_unlock_usercnt
 424
 425         testl   %eax, %eax
 426         jne     46b
 427
 428         movq    8(%rsp), %rdi
 429         incq    total_seq(%rdi)
 430         incl    cond_futex(%rdi)
 431         addl    $(1 << nwaiters_shift), cond_nwaiters(%rdi)
 432
 433         /* Get and store current wakeup_seq value.  */
 434         movq    8(%rsp), %rdi
 435         movq    wakeup_seq(%rdi), %r9
 436         movl    broadcast_seq(%rdi), %edx
 437         movq    %r9, 24(%rsp)
 438         movl    %edx, 4(%rsp)
 439
 440         /* Get the current time.  */
 441 8:
 442 # ifdef __NR_clock_gettime
 443         /* Get the clock number.  Note that the field in the condvar
 444            structure stores the number minus 1.  */
 445         movq    8(%rsp), %rdi
 446         movl    cond_nwaiters(%rdi), %edi
 447         andl    $((1 << nwaiters_shift) - 1), %edi
 448         /* Only clocks 0 and 1 are allowed so far.  Both are handled in the
 449            kernel.  */
 450         leaq    32(%rsp), %rsi
 451 #  ifdef SHARED
 452         mov     __vdso_clock_gettime@GOTPCREL(%rip), %RAX_LP
 453         mov     (%rax), %RAX_LP
 454         PTR_DEMANGLE (%RAX_LP)
 455         call    *%rax
 456 #  else
 457         movl    $__NR_clock_gettime, %eax
 458         syscall
 459 #  endif
 460 #  ifndef __ASSUME_POSIX_TIMERS
 461         cmpq    $-ENOSYS, %rax
 462         je      19f
 463 #  endif
 464
 465         /* Compute relative timeout.  */
 466         movq    (%r13), %rcx
 467         movq    8(%r13), %rdx
 468         subq    32(%rsp), %rcx
 469         subq    40(%rsp), %rdx
 470 # else
 471         leaq    24(%rsp), %rdi
 472         xorl    %esi, %esi
 473         /* This call works because we directly jump to a system call entry
 474            which preserves all the registers.  */
 475         call    JUMPTARGET(__gettimeofday)
 476
 477         /* Compute relative timeout.  */
 478         movq    40(%rsp), %rax
 479         movl    $1000, %edx
 480         mul     %rdx            /* Milli seconds to nano seconds.  */
 481         movq    (%r13), %rcx
 482         movq    8(%r13), %rdx
 483         subq    32(%rsp), %rcx
 484         subq    %rax, %rdx
 485 # endif
 486         jns     12f
 487         addq    $1000000000, %rdx
 488         decq    %rcx
 489 12:     testq   %rcx, %rcx
 490         movq    8(%rsp), %rdi
 491         movq    $-ETIMEDOUT, %r14
 492         js      6f
 493
 494         /* Store relative timeout.  */
 495 21:     movq    %rcx, 32(%rsp)
 496         movq    %rdx, 40(%rsp)
 497
 498         movl    cond_futex(%rdi), %r12d
 499
 500         /* Unlock.  */
 501         LOCK
 502 # if cond_lock == 0
 503         decl    (%rdi)
 504 # else
 505         decl    cond_lock(%rdi)
 506 # endif
 507         jne     3f
 508
 509 .LcleanupSTART2:
 510 4:      callq   __pthread_enable_asynccancel
 511         movl    %eax, (%rsp)
 512
 513         leaq    32(%rsp), %r10
 514         LP_OP(cmp) $-1, dep_mutex(%rdi)
 515         movq    %r12, %rdx
 516 # ifdef __ASSUME_PRIVATE_FUTEX
 517         movl    $FUTEX_WAIT, %eax
 518         movl    $(FUTEX_WAIT|FUTEX_PRIVATE_FLAG), %esi
 519         cmove   %eax, %esi
 520 # else
 521         movl    $0, %eax
 522         movl    %fs:PRIVATE_FUTEX, %esi
 523         cmove   %eax, %esi
 524 #  if FUTEX_WAIT != 0
 525         orl     $FUTEX_WAIT, %esi
 526 #  endif
 527 # endif
 528         addq    $cond_futex, %rdi
 529         movl    $SYS_futex, %eax
 530         syscall
 531         movq    %rax, %r14
 532
 533         movl    (%rsp), %edi
 534         callq   __pthread_disable_asynccancel
 535 .LcleanupEND2:
 536
 537         /* Lock.  */
 538         movq    8(%rsp), %rdi
 539         movl    $1, %esi
 540         xorl    %eax, %eax
 541         LOCK
 542 # if cond_lock == 0
 543         cmpxchgl %esi, (%rdi)
 544 # else
 545         cmpxchgl %esi, cond_lock(%rdi)
 546 # endif
 547         jne     5f
 548
 549 6:      movl    broadcast_seq(%rdi), %edx
 550
 551         movq    woken_seq(%rdi), %rax
 552
 553         movq    wakeup_seq(%rdi), %r9
 554
 555         cmpl    4(%rsp), %edx
 556         jne     53b
 557
 558         cmpq    24(%rsp), %r9
 559         jbe     15f
 560
 561         cmpq    %rax, %r9
 562         ja      39b
 563
 564 15:     cmpq    $-ETIMEDOUT, %r14
 565         jne     8b
 566
 567         jmp     99b
 568
 569         /* Initial locking failed.  */
 570 1:
 571 # if cond_lock != 0
 572         addq    $cond_lock, %rdi
 573 # endif
 574         LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
 575         movl    $LLL_PRIVATE, %eax
 576         movl    $LLL_SHARED, %esi
 577         cmovne  %eax, %esi
 578         callq   __lll_lock_wait
 579         jmp     2b
 580
 581         /* Unlock in loop requires wakeup.  */
 582 3:
 583 # if cond_lock != 0
 584         addq    $cond_lock, %rdi
 585 # endif
 586         LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
 587         movl    $LLL_PRIVATE, %eax
 588         movl    $LLL_SHARED, %esi
 589         cmovne  %eax, %esi
 590         callq   __lll_unlock_wake
 591         jmp     4b
 592
 593         /* Locking in loop failed.  */
 594 5:
 595 # if cond_lock != 0
 596         addq    $cond_lock, %rdi
 597 # endif
 598         LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
 599         movl    $LLL_PRIVATE, %eax
 600         movl    $LLL_SHARED, %esi
 601         cmovne  %eax, %esi
 602         callq   __lll_lock_wait
 603 # if cond_lock != 0
 604         subq    $cond_lock, %rdi
 605 # endif
 606         jmp     6b
 607
 608 # if defined __NR_clock_gettime && !defined __ASSUME_POSIX_TIMERS
 609         /* clock_gettime not available.  */
 610 19:     leaq    32(%rsp), %rdi
 611         xorl    %esi, %esi
 612         /* This call works because we directly jump to a system call entry
 613            which preserves all the registers.  */
 614         call    JUMPTARGET(__gettimeofday)
 615
 616         /* Compute relative timeout.  */
 617         movq    40(%rsp), %rax
 618         movl    $1000, %edx
 619         mul     %rdx            /* Milli seconds to nano seconds.  */
 620         movq    (%r13), %rcx
 621         movq    8(%r13), %rdx
 622         subq    32(%rsp), %rcx
 623         subq    %rax, %rdx
 624         jns     20f
 625         addq    $1000000000, %rdx
 626         decq    %rcx
 627 20:     testq   %rcx, %rcx
 628         movq    8(%rsp), %rdi
 629         movq    $-ETIMEDOUT, %r14
 630         js      6b
 631         jmp     21b
 632 # endif
 633 #endif
 634         .size   __pthread_cond_timedwait, .-__pthread_cond_timedwait
 635 versioned_symbol (libpthread, __pthread_cond_timedwait, pthread_cond_timedwait,
 636                   GLIBC_2_3_2)
 637
 638
 639         .align  16
 640         .type   __condvar_cleanup2, @function
 641 __condvar_cleanup2:
 642         /* Stack frame:
 643
 644            rsp + 72
 645                     +--------------------------+
 646            rsp + 64 | %r12                     |
 647                     +--------------------------+
 648            rsp + 56 | %r13                     |
 649                     +--------------------------+
 650            rsp + 48 | %r14                     |
 651                     +--------------------------+
 652            rsp + 24 | unused                   |
 653                     +--------------------------+
 654            rsp + 16 | mutex pointer            |
 655                     +--------------------------+
 656            rsp +  8 | condvar pointer          |
 657                     +--------------------------+
 658            rsp +  4 | old broadcast_seq value  |
 659                     +--------------------------+
 660            rsp +  0 | old cancellation mode    |
 661                     +--------------------------+
 662         */
 663
 664         movq    %rax, 24(%rsp)
 665
 666         /* Get internal lock.  */
 667         movq    8(%rsp), %rdi
 668         movl    $1, %esi
 669         xorl    %eax, %eax
 670         LOCK
 671 #if cond_lock == 0
 672         cmpxchgl %esi, (%rdi)
 673 #else
 674         cmpxchgl %esi, cond_lock(%rdi)
 675 #endif
 676         jz      1f
 677
 678 #if cond_lock != 0
 679         addq    $cond_lock, %rdi
 680 #endif
 681         LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
 682         movl    $LLL_PRIVATE, %eax
 683         movl    $LLL_SHARED, %esi
 684         cmovne  %eax, %esi
 685         callq   __lll_lock_wait
 686 #if cond_lock != 0
 687         subq    $cond_lock, %rdi
 688 #endif
 689
 690 1:      movl    broadcast_seq(%rdi), %edx
 691         cmpl    4(%rsp), %edx
 692         jne     3f
 693
 694         /* We increment the wakeup_seq counter only if it is lower than
 695            total_seq.  If this is not the case the thread was woken and
 696            then canceled.  In this case we ignore the signal.  */
 697         movq    total_seq(%rdi), %rax
 698         cmpq    wakeup_seq(%rdi), %rax
 699         jbe     6f
 700         incq    wakeup_seq(%rdi)
 701         incl    cond_futex(%rdi)
 702 6:      incq    woken_seq(%rdi)
 703
 704 3:      subl    $(1 << nwaiters_shift), cond_nwaiters(%rdi)
 705
 706         /* Wake up a thread which wants to destroy the condvar object.  */
 707         xorq    %r12, %r12
 708         cmpq    $0xffffffffffffffff, total_seq(%rdi)
 709         jne     4f
 710         movl    cond_nwaiters(%rdi), %eax
 711         andl    $~((1 << nwaiters_shift) - 1), %eax
 712         jne     4f
 713
 714         LP_OP(cmp) $-1, dep_mutex(%rdi)
 715         leaq    cond_nwaiters(%rdi), %rdi
 716         movl    $1, %edx
 717 #ifdef __ASSUME_PRIVATE_FUTEX
 718         movl    $FUTEX_WAKE, %eax
 719         movl    $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
 720         cmove   %eax, %esi
 721 #else
 722         movl    $0, %eax
 723         movl    %fs:PRIVATE_FUTEX, %esi
 724         cmove   %eax, %esi
 725         orl     $FUTEX_WAKE, %esi
 726 #endif
 727         movl    $SYS_futex, %eax
 728         syscall
 729         subq    $cond_nwaiters, %rdi
 730         movl    $1, %r12d
 731
 732 4:      LOCK
 733 #if cond_lock == 0
 734         decl    (%rdi)
 735 #else
 736         decl    cond_lock(%rdi)
 737 #endif
 738         je      2f
 739 #if cond_lock != 0
 740         addq    $cond_lock, %rdi
 741 #endif
 742         LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
 743         movl    $LLL_PRIVATE, %eax
 744         movl    $LLL_SHARED, %esi
 745         cmovne  %eax, %esi
 746         callq   __lll_unlock_wake
 747
 748         /* Wake up all waiters to make sure no signal gets lost.  */
 749 2:      testq   %r12, %r12
 750         jnz     5f
 751         addq    $cond_futex, %rdi
 752         LP_OP(cmp) $-1, dep_mutex-cond_futex(%rdi)
 753         movl    $0x7fffffff, %edx
 754 #ifdef __ASSUME_PRIVATE_FUTEX
 755         movl    $FUTEX_WAKE, %eax
 756         movl    $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
 757         cmove   %eax, %esi
 758 #else
 759         movl    $0, %eax
 760         movl    %fs:PRIVATE_FUTEX, %esi
 761         cmove   %eax, %esi
 762         orl     $FUTEX_WAKE, %esi
 763 #endif
 764         movl    $SYS_futex, %eax
 765         syscall
 766
 767 5:      movq    16(%rsp), %rdi
 768         callq   __pthread_mutex_cond_lock
 769
 770         movq    24(%rsp), %rdi
 771         movq    FRAME_SIZE(%rsp), %r15
 772         movq    FRAME_SIZE+8(%rsp), %r14
 773         movq    FRAME_SIZE+16(%rsp), %r13
 774         movq    FRAME_SIZE+24(%rsp), %r12
 775 .LcallUR:
 776         call    _Unwind_Resume@PLT
 777         hlt
 778 .LENDCODE:
 779         cfi_endproc
 780         .size   __condvar_cleanup2, .-__condvar_cleanup2
 781
 782
 783         .section .gcc_except_table,"a",@progbits
 784 .LexceptSTART:
 785         .byte   DW_EH_PE_omit                   # @LPStart format
 786         .byte   DW_EH_PE_omit                   # @TType format
 787         .byte   DW_EH_PE_uleb128                # call-site format
 788         .uleb128 .Lcstend-.Lcstbegin
 789 .Lcstbegin:
 790         .uleb128 .LcleanupSTART1-.LSTARTCODE
 791         .uleb128 .LcleanupEND1-.LcleanupSTART1
 792         .uleb128 __condvar_cleanup2-.LSTARTCODE
 793         .uleb128  0
 794 #ifndef __ASSUME_FUTEX_CLOCK_REALTIME
 795         .uleb128 .LcleanupSTART2-.LSTARTCODE
 796         .uleb128 .LcleanupEND2-.LcleanupSTART2
 797         .uleb128 __condvar_cleanup2-.LSTARTCODE
 798         .uleb128  0
 799 #endif
 800         .uleb128 .LcallUR-.LSTARTCODE
 801         .uleb128 .LENDCODE-.LcallUR
 802         .uleb128 0
 803         .uleb128  0
 804 .Lcstend:
 805
 806
 807 #ifdef SHARED
 808         .hidden DW.ref.__gcc_personality_v0
 809         .weak   DW.ref.__gcc_personality_v0
 810         .section .gnu.linkonce.d.DW.ref.__gcc_personality_v0,"aw",@progbits
 811         .align  LP_SIZE
 812         .type   DW.ref.__gcc_personality_v0, @object
 813         .size   DW.ref.__gcc_personality_v0, LP_SIZE
 814 DW.ref.__gcc_personality_v0:
 815         ASM_ADDR __gcc_personality_v0
 816 #endif