[PATCH 5/6] malloc: aarch64: Add MTE ifunc resolvers
Muhammad Kamran
muhammad.kamran@arm.com
Thu Sep 24 14:35:08 GMT 2026
From: Muhammad Kamran <Muhammad.Kamran@arm.com>
We introduce the ARCH_INIT_MALLOC() macro that is defined in
the sysdep headers to allow doing target-specific initialisation
in __ptmalloc_init. On AArch64 we use this macro to modify
global variables in malloc.c when MTE is enabled based on the
runtime value calculated from the glibc.cpu.mtemode tunable.
On all targets this macro should initialise extra_mmap_prot.
We create AArch64-specific version of the 'malloc-size.h' header
to provide custom 'memsize()' and 'checked_request2size()'. This
is required to support MTE memory tagging without the need to
check its status every time.
We modify ifunc resolvers for AArch64 to return MTE-aware versions
of malloc functions when MTE is active. At this point these *_mte
functions don't actually do any memory tagging. This will be done
in a subsequent commit.
Finally, we add the glibc.cpu.mtemode tunable to all malloc
tests on AArch64. On systems without MTE this will have no effect
but on systems with MTE it will make sure that tests are executed
with memory tagging enabled in sync mode.
Co-authored-by: Yury Khrustalev <yury.khrustalev@arm.com>
---
malloc/arena.c | 9 ++-
malloc/malloc.c | 8 +-
sysdeps/aarch64/multiarch/Makefile | 1 +
sysdeps/aarch64/multiarch/malloc-ifuncs.c | 30 ++++----
sysdeps/aarch64/multiarch/malloc-ifuncs.h | 34 +++++++++
sysdeps/aarch64/multiarch/malloc-init.h | 33 ++++++++
sysdeps/aarch64/multiarch/malloc-mte.c | 91 +++++++++++++++++++++++
sysdeps/aarch64/multiarch/malloc-size.h | 71 ++++++++++++++++++
sysdeps/generic/malloc-init.h | 25 +++++++
sysdeps/unix/sysv/linux/aarch64/Makefile | 16 ++++
10 files changed, 297 insertions(+), 21 deletions(-)
create mode 100644 sysdeps/aarch64/multiarch/malloc-init.h
create mode 100644 sysdeps/aarch64/multiarch/malloc-mte.c
create mode 100644 sysdeps/aarch64/multiarch/malloc-size.h
create mode 100644 sysdeps/generic/malloc-init.h
diff --git a/malloc/arena.c b/malloc/arena.c
index 1ed419a305..5ec168a8ce 100644
--- a/malloc/arena.c
+++ b/malloc/arena.c
@@ -22,6 +22,8 @@
#define TUNABLE_NAMESPACE malloc
#include <elf/dl-tunables.h>
+#include <malloc-init.h>
+
/* Compile-time constants. */
#define HEAP_MIN_SIZE (32 * 1024)
@@ -248,6 +250,9 @@ static void tcache_key_initialize (void);
void
__ptmalloc_init (void)
{
+ /* Perform any target-specific initialisation. */
+ ARCH_INIT_MALLOC ();
+
#if USE_TCACHE
tcache_key_initialize ();
#endif
@@ -403,7 +408,7 @@ alloc_new_heap (size_t size, size_t top_pad, size_t pagesize,
}
}
}
- if (__mprotect (p2, size, extra_mmap_prot | PROT_READ | PROT_WRITE) != 0)
+ if (__mprotect (p2, size, extra_mmap_prot) != 0)
{
__munmap (p2, max_size);
return NULL;
@@ -457,7 +462,7 @@ grow_heap (heap_info *h, long diff)
{
if (__mprotect ((char *) h + h->mprotect_size,
(unsigned long) new_size - h->mprotect_size,
- extra_mmap_prot | PROT_READ | PROT_WRITE) != 0)
+ extra_mmap_prot) != 0)
return -2;
h->mprotect_size = new_size;
diff --git a/malloc/malloc.c b/malloc/malloc.c
index 9e1b981c77..3cae7cc642 100644
--- a/malloc/malloc.c
+++ b/malloc/malloc.c
@@ -1865,9 +1865,7 @@ sysmalloc_mmap (INTERNAL_SIZE_T nb, size_t pagesize, int extra_flags)
size_t padding = MALLOC_ALIGNMENT - CHUNK_HDR_SZ;
size_t size = ALIGN_UP (nb + padding + CHUNK_HDR_SZ, pagesize);
- char *mm = (char *) MMAP (NULL, size,
- extra_mmap_prot | PROT_READ | PROT_WRITE,
- extra_flags);
+ char *mm = (char *) MMAP (NULL, size, extra_mmap_prot, extra_flags);
if (mm == MAP_FAILED)
return mm;
if (extra_flags == 0)
@@ -1906,9 +1904,7 @@ sysmalloc_mmap_fallback (size_t *s, size_t size, size_t minsize,
if (size < minsize)
size = minsize;
- char *mbrk = (char *) (MMAP (NULL, size,
- extra_mmap_prot | PROT_READ | PROT_WRITE,
- extra_flags));
+ char *mbrk = (char *) MMAP (NULL, size, extra_mmap_prot, extra_flags);
if (mbrk == MAP_FAILED)
return MAP_FAILED;
diff --git a/sysdeps/aarch64/multiarch/Makefile b/sysdeps/aarch64/multiarch/Makefile
index a8a03a0491..333324aa0a 100644
--- a/sysdeps/aarch64/multiarch/Makefile
+++ b/sysdeps/aarch64/multiarch/Makefile
@@ -24,5 +24,6 @@ endif # ifeq ($(subdir),string)
ifeq ($(subdir),malloc)
sysdep_routines += \
malloc-ifuncs \
+ malloc-mte \
# sysdep_routines
endif # ifeq ($(subdir),malloc)
diff --git a/sysdeps/aarch64/multiarch/malloc-ifuncs.c b/sysdeps/aarch64/multiarch/malloc-ifuncs.c
index 4b5db036ff..8e21039d1e 100644
--- a/sysdeps/aarch64/multiarch/malloc-ifuncs.c
+++ b/sysdeps/aarch64/multiarch/malloc-ifuncs.c
@@ -22,55 +22,59 @@
# if IS_IN (libc)
#include <init-arch.h>
-#include <malloc-api.h>
+#include <malloc-ifuncs.h>
#include <shlib-compat.h>
+#define MTE_ACTIVE (mte && \
+ ((GL (dl_aarch64_mte_scope) & MTE_SCOPE_HEAP) != 0) \
+ && (GL (dl_aarch64_mte_mode) != MTE_MODE_DISABLED))
+
libc_ifunc_hidden (__libc_malloc, __libc_malloc_redirect,
- __libc_malloc)
+ MTE_ACTIVE ? __libc_malloc_mte : __libc_malloc)
strong_alias (__libc_malloc_redirect, malloc)
libc_ifunc_hidden (__libc_calloc, __libc_calloc_redirect,
- __libc_calloc)
+ MTE_ACTIVE ? __libc_calloc_mte : __libc_calloc)
weak_alias (__libc_calloc_redirect, calloc)
libc_ifunc_hidden (__libc_memalign, __libc_memalign_redirect,
- __libc_memalign)
+ MTE_ACTIVE ? __libc_memalign_mte : __libc_memalign)
weak_alias (__libc_memalign_redirect, memalign)
libc_ifunc_hidden (__libc_valloc, __libc_valloc_redirect,
- __libc_valloc)
+ MTE_ACTIVE ? __libc_valloc_mte : __libc_valloc)
weak_alias (__libc_valloc_redirect, valloc)
libc_ifunc_hidden (__libc_pvalloc, __libc_pvalloc_redirect,
- __libc_pvalloc)
+ MTE_ACTIVE ? __libc_pvalloc_mte : __libc_pvalloc)
weak_alias (__libc_pvalloc_redirect, pvalloc)
libc_ifunc_hidden (__libc_realloc, __libc_realloc_redirect,
- __libc_realloc)
+ MTE_ACTIVE ? __libc_realloc_mte : __libc_realloc)
strong_alias (__libc_realloc_redirect, realloc)
libc_ifunc_hidden (__libc_free, __libc_free_redirect,
- __libc_free)
+ MTE_ACTIVE ? __libc_free_mte : __libc_free)
strong_alias (__libc_free_redirect, free)
libc_ifunc_hidden (__malloc_usable_size, __malloc_usable_size_redirect,
- __malloc_usable_size)
+ MTE_ACTIVE ? __malloc_usable_size_mte : __malloc_usable_size)
weak_alias (__malloc_usable_size_redirect, malloc_usable_size)
libc_ifunc_hidden (__posix_memalign, __posix_memalign_redirect,
- __posix_memalign)
+ MTE_ACTIVE ? __posix_memalign_mte : __posix_memalign)
weak_alias (__posix_memalign_redirect, posix_memalign)
libc_ifunc_hidden (__aligned_alloc, __aligned_alloc_redirect,
- __aligned_alloc)
+ MTE_ACTIVE ? __aligned_alloc_mte : __aligned_alloc)
weak_alias (__aligned_alloc_redirect, aligned_alloc)
libc_ifunc_hidden (__free_sized, __free_sized_redirect,
- __free_sized)
+ MTE_ACTIVE ? __free_sized_mte : __free_sized)
weak_alias (__free_sized_redirect, free_sized)
libc_ifunc_hidden (__free_aligned_sized, __free_aligned_sized_redirect,
- __free_aligned_sized)
+ MTE_ACTIVE ? __free_aligned_sized_mte : __free_aligned_sized)
weak_alias (__free_aligned_sized_redirect, free_aligned_sized)
# endif /* IS_IN (libc) */
diff --git a/sysdeps/aarch64/multiarch/malloc-ifuncs.h b/sysdeps/aarch64/multiarch/malloc-ifuncs.h
index abd5d3ef19..cdaafd0cf3 100644
--- a/sysdeps/aarch64/multiarch/malloc-ifuncs.h
+++ b/sysdeps/aarch64/multiarch/malloc-ifuncs.h
@@ -22,4 +22,38 @@
/* AArch64 target provides ifuncs for malloc functions. */
#define USE_MULTIARCH_MALLOC 1
+#include <malloc-api.h>
+
+/* MTE implementations of malloc functions. */
+void *__libc_malloc_mte (size_t);
+libc_hidden_proto (__libc_malloc_mte)
+void *__libc_calloc_mte (size_t, size_t);
+libc_hidden_proto (__libc_calloc_mte)
+void *__libc_memalign_mte (size_t, size_t);
+libc_hidden_proto (__libc_memalign_mte)
+void *__libc_valloc_mte (size_t);
+libc_hidden_proto (__libc_valloc_mte)
+void *__libc_pvalloc_mte (size_t);
+libc_hidden_proto (__libc_pvalloc_mte)
+void *__libc_realloc_mte (void *, size_t);
+libc_hidden_proto (__libc_realloc_mte)
+void __libc_free_mte (void *);
+libc_hidden_proto (__libc_free_mte)
+size_t __malloc_usable_size_mte (void *);
+libc_hidden_proto (__malloc_usable_size_mte)
+
+/* For additions of POSIX: MTE version. */
+int __posix_memalign_mte (void **, size_t, size_t);
+libc_hidden_proto (__posix_memalign_mte)
+
+/* For ISO C17: MTE version. */
+void *__aligned_alloc_mte (size_t, size_t);
+libc_hidden_proto (__aligned_alloc_mte)
+
+/* For ISO C23: MTE version. */
+void __free_sized_mte (void *, size_t);
+libc_hidden_proto (__free_sized_mte)
+void __free_aligned_sized_mte (void *, size_t, size_t);
+libc_hidden_proto (__free_aligned_sized_mte)
+
#endif /* _AARCH64_MALLOC_IFUNCS_H */
diff --git a/sysdeps/aarch64/multiarch/malloc-init.h b/sysdeps/aarch64/multiarch/malloc-init.h
new file mode 100644
index 0000000000..11f86578a6
--- /dev/null
+++ b/sysdeps/aarch64/multiarch/malloc-init.h
@@ -0,0 +1,33 @@
+/* Definitions for malloc init: aarch64 version.
+ Copyright (C) 2026 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#ifndef _AARCH64_MALLOC_INIT_H
+#define _AARCH64_MALLOC_INIT_H
+
+#define MTE_ACTIVE ((GL (dl_aarch64_mte_scope) & MTE_SCOPE_HEAP) != 0 \
+ && (GL (dl_aarch64_mte_mode) != MTE_MODE_DISABLED))
+
+#define ARCH_INIT_MALLOC() \
+ extra_mmap_prot = PROT_READ | PROT_WRITE; \
+ if (MTE_ACTIVE) \
+ { \
+ extra_mmap_prot |= PROT_MTE; \
+ __always_fail_morecore = true; \
+ }
+
+#endif /* _AARCH64_MALLOC_INIT_H */
diff --git a/sysdeps/aarch64/multiarch/malloc-mte.c b/sysdeps/aarch64/multiarch/malloc-mte.c
new file mode 100644
index 0000000000..772e2319c1
--- /dev/null
+++ b/sysdeps/aarch64/multiarch/malloc-mte.c
@@ -0,0 +1,91 @@
+/* Implementation for MTE (memory tagging) wrappers in malloc.
+ Copyright (C) 2026 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <malloc-ifuncs.h>
+
+void *__libc_malloc_mte (size_t bytes)
+{
+ return __libc_malloc (bytes);
+}
+libc_hidden_def (__libc_malloc_mte)
+
+void *__libc_calloc_mte (size_t n, size_t elem_size)
+{
+ return __libc_calloc (n, elem_size);
+}
+libc_hidden_def (__libc_calloc_mte)
+
+void *__libc_memalign_mte (size_t alignment, size_t bytes)
+{
+ return __libc_memalign (alignment, bytes);
+}
+libc_hidden_def (__libc_memalign_mte)
+
+void *__libc_valloc_mte (size_t bytes)
+{
+ return __libc_valloc (bytes);
+}
+libc_hidden_def (__libc_valloc_mte)
+
+void *__libc_pvalloc_mte (size_t bytes)
+{
+ return __libc_pvalloc (bytes);
+}
+libc_hidden_def (__libc_pvalloc_mte)
+
+void *__libc_realloc_mte (void *oldmem, size_t bytes)
+{
+ return __libc_realloc (oldmem, bytes);
+}
+libc_hidden_def (__libc_realloc_mte)
+
+void __libc_free_mte (void *mem)
+{
+ __libc_free (mem);
+}
+libc_hidden_def (__libc_free_mte)
+
+size_t __malloc_usable_size_mte (void *m)
+{
+ return __malloc_usable_size (m);
+}
+libc_hidden_def (__malloc_usable_size_mte)
+
+int __posix_memalign_mte (void **memptr, size_t alignment, size_t size)
+{
+ return __posix_memalign (memptr, alignment, size);
+}
+libc_hidden_def (__posix_memalign_mte)
+
+void *__aligned_alloc_mte (size_t alignment, size_t bytes)
+{
+ return __aligned_alloc (alignment, bytes);
+}
+libc_hidden_def (__aligned_alloc_mte)
+
+void __free_sized_mte (void *ptr, size_t size)
+{
+ __free_sized (ptr, size);
+}
+libc_hidden_def (__free_sized_mte)
+
+void __free_aligned_sized_mte (void *ptr, size_t alignment, size_t size)
+{
+ __free_aligned_sized (ptr, alignment, size);
+}
+libc_hidden_def (__free_aligned_sized_mte)
diff --git a/sysdeps/aarch64/multiarch/malloc-size.h b/sysdeps/aarch64/multiarch/malloc-size.h
new file mode 100644
index 0000000000..6742e4712c
--- /dev/null
+++ b/sysdeps/aarch64/multiarch/malloc-size.h
@@ -0,0 +1,71 @@
+/* Size-related definitions for malloc: aarch64 version.
+ Copyright (C) 2026 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#ifndef _AARCH64_MALLOC_SIZE_H
+#define _AARCH64_MALLOC_SIZE_H
+
+#include <malloc-chunk.h>
+#include <stdint.h>
+#include <sys/cdefs.h>
+
+/* The smallest size we can malloc is an aligned minimal chunk. */
+#define MINSIZE \
+ (unsigned long)(((MIN_CHUNK_SIZE + MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK))
+
+/* Pad request bytes into a usable size -- internal version. Note: This must
+ be a macro that evaluates to a compile time constant if passed a literal
+ constant. */
+#define request2size(req) \
+ (((req) + CHUNK_HDR_SZ + MALLOC_ALIGN_MASK < MINSIZE) ? \
+ MINSIZE : \
+ ((req) + CHUNK_HDR_SZ + MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK)
+
+/* MTE uses 16-byte granules. */
+#define MTE_GRANULE_SIZE 16
+
+/* MTE granules must be aligned by malloc alignment. */
+verify (MTE_GRANULE_SIZE <= MALLOC_ALIGNMENT);
+verify (MALLOC_ALIGNMENT % MTE_GRANULE_SIZE == 0);
+
+/* memsize relies on the chunk header occupying a complete tag granule. */
+verify (MTE_GRANULE_SIZE == CHUNK_HDR_SZ);
+
+/* Check if REQ overflows when padded and aligned and if the resulting
+ value is less than PTRDIFF_T. Returns the requested size or
+ MINSIZE in case the value is less than MINSIZE, or SIZE_MAX if any
+ of the previous checks fail. */
+static __always_inline __attribute_maybe_unused__ size_t
+checked_request2size (size_t req)
+{
+ if (__glibc_unlikely (req > PTRDIFF_MAX))
+ return SIZE_MAX;
+ return request2size (req);
+}
+
+/* Like chunksize, but do not mask SIZE_BITS. */
+#define chunksize_nomask(p) ((p)->mchunk_size)
+
+/* Get size, ignoring use bits. */
+#define chunksize(p) (chunksize_nomask (p) & ~(SIZE_BITS))
+
+/* This is the size of the real usable data in the chunk. Not valid for
+ dumped heap chunks. */
+#define chunksize2usable(n) (n - CHUNK_HDR_SZ)
+#define memsize(p) (chunksize2usable (chunksize (p)))
+
+#endif /* _AARCH64_MALLOC_SIZE_H */
diff --git a/sysdeps/generic/malloc-init.h b/sysdeps/generic/malloc-init.h
new file mode 100644
index 0000000000..5fb329311b
--- /dev/null
+++ b/sysdeps/generic/malloc-init.h
@@ -0,0 +1,25 @@
+/* Definition for malloc init: generic version.
+ Copyright (C) 2026 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#ifndef _GENERIC_MALLOC_INIT_H
+#define _GENERIC_MALLOC_INIT_H
+
+#define ARCH_INIT_MALLOC() \
+ extra_mmap_prot = PROT_READ | PROT_WRITE;
+
+#endif /* _GENERIC_MALLOC_INIT_H */
diff --git a/sysdeps/unix/sysv/linux/aarch64/Makefile b/sysdeps/unix/sysv/linux/aarch64/Makefile
index 4debe41df6..e8cdad8ba9 100644
--- a/sysdeps/unix/sysv/linux/aarch64/Makefile
+++ b/sysdeps/unix/sysv/linux/aarch64/Makefile
@@ -386,6 +386,22 @@ ifeq ($(subdir),stdlib)
gen-as-const-headers += ucontext_i.sym
endif
+ifeq ($(subdir),malloc)
+# Add MTE tunable to all malloc tests except malloc-check and mcheck variants
+tests-malloc-mte = \
+ $(tests) \
+ $(tests-malloc-hugetlb1:%=%-malloc-hugetlb1) \
+ $(tests-malloc-hugetlb2:%=%-malloc-hugetlb2) \
+ $(tests-malloc-largetcache:%=%-malloc-largetcache) \
+ # tests-malloc-mte
+
+define tests-malloc-mte-ENVS
+$(1)-TUNABLES += glibc.cpu.mtemode=sync
+endef
+
+$(foreach t,$(tests-malloc-mte),$(eval $(call tests-malloc-mte-ENVS,$(t))))
+endif # ifeq ($(subdir),malloc)
+
abi-variants := lp64
abi-variants += lp64_be
--
2.43.0
More information about the Libc-alpha
mailing list