[PATCH] malloc: Fix ABBA deadlock in fork handlers

[Cuishuangjin]

From 7b2df68f726741d2e39df9933a9044290e6f5a2e Mon Sep 17 00:00:00 2001
From: cys43405 <YS.cuishuangjin@h3c.com>
Date: Sat, 9 May 2026 20:30:09 +0800
Subject: [PATCH] glibc: Fix ABBA deadlock between fork handlers and atfork
registration
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Detailed description:
- Problem analysis: glibc 2.38-50 has an ABBA deadlock issue. When thread A executes prepare_handler during fork phase while thread B concurrently registers pthread_atfork handlers, a deadlock occurs where thread A holds malloc_lock and waits for atfork_lock, while thread B holds atfork_lock and waits for malloc_lock.

- Timeline:
  Timeline    │ Thread A (__run_prefork_handlers)        │ Thread B (__register_atfork)
  ────────────┼──────────────────────────────────────────┼──────────────────────────────
     T1       │ Holds atfork_lock                        │
     T2       │   lll_unlock(atfork_lock)  ────────────► │
     T3       │ Executes prepare_handler()               │   lll_lock(atfork_lock)
     T4       │ └─► ptmalloc_lock_all()                  │ Holds atfork_lock
     T5       │ Holds malloc_lock                        │   fork_handler_list_emplace()
     T6       │   lll_lock(atfork_lock) ◄──── Blocked ── │ └─► malloc() ◄──── Blocked ─┐

- Solution: Weaken definitions of __malloc_fork_lock_parent, __malloc_fork_unlock_parent, and __malloc_fork_unlock_child in glibc 2.38-50 to prevent hard-linking inconsistencies. Strengthen malloc_lock management in __register_atfork prepare_handler functions for ptmalloc and tcmalloc libraries.

- Testing method: Use unmodified .so library with one process continuously forking while another registers pthread_atfork handlers to verify deadlock occurrence. Compare with modified .so to confirm issue resolution.

Signed-off-by: shuangjin cui <YS.cuishuangjin@h3c.com>
---
malloc/Versions          | 5 +++++
malloc/arena.c           | 6 +++---
malloc/malloc-internal.h | 6 +++---
3 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/malloc/Versions b/malloc/Versions
index a9ce4a035f..40532c89dc 100644
--- a/malloc/Versions
+++ b/malloc/Versions
@@ -43,6 +43,11 @@ libc {
     # v*
     valloc;
+
+    # weak malloc fork lock/unlock
+    __malloc_fork_lock_parent;
+    __malloc_fork_unlock_parent;
+    __malloc_fork_unlock_child;
   }
   GLIBC_2.1 {
     # Special functions.
diff --git a/malloc/arena.c b/malloc/arena.c
index ddde32c712..032e0dd69d 100644
--- a/malloc/arena.c
+++ b/malloc/arena.c
@@ -162,7 +162,7 @@ arena_for_chunk (mchunkptr ptr)
    called, so that other fork handlers can use the malloc
    subsystem.  */
-void
+void __attribute__((weak))
__malloc_fork_lock_parent (void)
{
   /* We do not acquire free_list_lock here because we completely
@@ -179,7 +179,7 @@ __malloc_fork_lock_parent (void)
     }
}
-void
+void __attribute__((weak))
__malloc_fork_unlock_parent (void)
{
   for (mstate ar_ptr = &main_arena;; )
@@ -192,7 +192,7 @@ __malloc_fork_unlock_parent (void)
   __libc_lock_unlock (list_lock);
}
-void
+void __attribute__((weak))
__malloc_fork_unlock_child (void)
{
   /* Push all arenas to the free list, except thread_arena, which is
diff --git a/malloc/malloc-internal.h b/malloc/malloc-internal.h
index a6340bfd88..978d6bb1d4 100644
--- a/malloc/malloc-internal.h
+++ b/malloc/malloc-internal.h
@@ -26,13 +26,13 @@
#include <calloc-clear-memory.h>
 /* Called in the parent process before a fork.  */
-void __malloc_fork_lock_parent (void) attribute_hidden;
+void __attribute__((weak)) __malloc_fork_lock_parent (void) ;
 /* Called in the parent process after a fork.  */
-void __malloc_fork_unlock_parent (void) attribute_hidden;
+void __attribute__((weak)) __malloc_fork_unlock_parent (void) ;
 /* Called in the child process after a fork.  */
-void __malloc_fork_unlock_child (void) attribute_hidden;
+void __attribute__((weak)) __malloc_fork_unlock_child (void) ;
 /* Called as part of the thread shutdown sequence.  */
void __malloc_arena_thread_freeres (void) attribute_hidden;
--
2.33.0

-------------------------------------------------------------------------------------------------------------------------------------
本邮件及其附件含有新华三集团的保密信息，仅限于发送给上面地址中列出的个人或群组。
禁止任何其他人以任何形式使用（包括但不限于全部或部分地泄露、复制、或散发）本邮件中的信息。
如果您错收了本邮件，请您立即电话或邮件通知发件人并删除本邮件！
This e-mail and its attachments contain confidential information from New H3C, which is intended only for the person or entity whose address is listed above.
Any use of the information contained herein in any way (including, but not limited to, total or partial disclosure, reproduction, or dissemination) by persons other than the intended recipient(s) is prohibited.
If you receive this e-mail in error, please notify the sender by phone or email immediately and delete it!
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://sourceware.org/pipermail/libc-alpha/attachments/20260511/efce9522/attachment-0001.htm>