[PATCH] nptl: Disable asynchronous cancellation on __do_cancel (BZ 32782)

[Adhemerval Zanella]

Similar to __pthread_unwind, called from pthread_exit, once cancellation
starts the cancellation signal handler (sigcancel_handler) should not
restart the cancellation process (and libgcc unwind is not reentrant).
So also disables asynchronous cancellation on __do_cancel, any
cancellation signal received after it is ignored (cancel_async_enabled
will return false).

Checked on x86_64-linux-gnu and aarch64-linux-gnu.
---
 sysdeps/nptl/pthreadP.h        | 12 +++++-
 sysdeps/pthread/Makefile       |  1 +
 sysdeps/pthread/tst-cancel32.c | 73 ++++++++++++++++++++++++++++++++++
 3 files changed, 84 insertions(+), 2 deletions(-)
 create mode 100644 sysdeps/pthread/tst-cancel32.c

diff --git a/sysdeps/nptl/pthreadP.h b/sysdeps/nptl/pthreadP.h
index 2d620ed20d..63de8904f4 100644
--- a/sysdeps/nptl/pthreadP.h
+++ b/sysdeps/nptl/pthreadP.h
@@ -267,8 +267,16 @@ __do_cancel (void *result)
 
   self->result = result;
 
-  /* Make sure we get no more cancellations.  */
-  atomic_fetch_or_relaxed (&self->cancelhandling, EXITING_BITMASK);
+  /* Disable asynchronous cancellation and signal that thread is exiting.  */
+  int cancelhandling = atomic_load_relaxed (&self->cancelhandling);
+  int newval;
+   do
+     {
+       newval = (cancelhandling & ~CANCELTYPE_BITMASK) | EXITING_BITMASK;
+     }
+   while (!atomic_compare_exchange_weak_acquire (&self->cancelhandling,
+                                                 &cancelhandling,
+                                                 newval));
 
   __pthread_unwind ((__pthread_unwind_buf_t *)
 		    THREAD_GETMEM (self, cleanup_jmp_buf));
diff --git a/sysdeps/pthread/Makefile b/sysdeps/pthread/Makefile
index 70e62b2e1b..d4869c624b 100644
--- a/sysdeps/pthread/Makefile
+++ b/sysdeps/pthread/Makefile
@@ -106,6 +106,7 @@ tests += \
   tst-cancel28 \
   tst-cancel29 \
   tst-cancel30 \
+  tst-cancel32 \
   tst-cleanup0 \
   tst-cleanup1 \
   tst-cleanup2 \
diff --git a/sysdeps/pthread/tst-cancel32.c b/sysdeps/pthread/tst-cancel32.c
new file mode 100644
index 0000000000..1c6a4d8f47
--- /dev/null
+++ b/sysdeps/pthread/tst-cancel32.c
@@ -0,0 +1,73 @@
+/* Check if pthread_setcanceltype disables asynchronous cancellation
+   once cancellation happens (BZ 32782)
+
+   Copyright (C) 2025 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+/* The pthread_setcanceltype is a cancellation entrypoint, and if
+   asynchronous is enabled and the cancellation starts (on the second
+   pthread_setcanceltype call), the asynchronous should not restart
+   the process.  */
+
+#include <support/xthread.h>
+
+#define NITER     1000
+#define NTHREADS     8
+
+static void
+tf_cleanup (void *arg)
+{
+}
+
+static void *
+tf (void *closure)
+{
+  pthread_cleanup_push (tf_cleanup, NULL);
+  for (;;)
+    {
+      /* The only failure possible for pthread_setcanceltype is and
+	 invalid state type.  */
+      pthread_setcanceltype (PTHREAD_CANCEL_ASYNCHRONOUS, NULL);
+      pthread_setcanceltype (PTHREAD_CANCEL_DEFERRED, NULL);
+    }
+  pthread_cleanup_pop (1);
+
+  return NULL;
+}
+
+static void
+poll_threads (int nthreads)
+{
+  pthread_t thr[nthreads];
+  for (int i = 0; i < nthreads; i++)
+    thr[i] = xpthread_create (NULL, tf, NULL);
+  for (int i = 0; i < nthreads; i++)
+    xpthread_cancel (thr[i]);
+  for (int i = 0; i < nthreads; i++)
+    xpthread_join (thr[i]);
+}
+
+static int
+do_test (void)
+{
+  for (int k = 0; k < NITER; k++)
+    poll_threads (NTHREADS);
+
+  return 0;
+}
+
+#include <support/test-driver.c>
-- 
2.43.0