[PATCH v2 3/4] elf: Align large load segments to PMD huge page size for THP
WANG Rui
wangrui@loongson.cn
Thu Feb 26 06:33:04 GMT 2026
Mapping segments that are at least the size of a PMD huge page to
huge-page-aligned addresses helps make them eligible for Transparent
Huge Pages (THP).
This patch introduces a Linux-specific helper, `_dl_map_segment_align`,
to determine an appropriate maximum alignment for ELF load segments on
64-bit systems based on the system THP policy. The optimization is
enabled only when the glibc tunable `glibc.malloc.hugetlb=1` is set and
THP is configured to be used unconditionally.
When enabled, the helper queries the default THP page size and uses it
to align sufficiently large load segments that are already properly
aligned in both virtual address and file offset.
For eligible segments, the alignment is bumped to the THP page size,
which improves THP eligibility, reduces TLB pressure, and improves
performance for large objects. To avoid excessive address space padding
on systems with very large THP sizes, the alignment is capped at 32MB.
The optimization is applied only to writable segments, matching typical
THP usage.
* sysdeps/unix/sysv/linux/wordsize-64/dl-map-segment-align.c
(_dl_map_segment_align): Determine segment alignment based on THP
policy and default huge page size.
* sysdeps/unix/sysv/linux/wordsize-64/dl-map-segment-align.h: New header.
* sysdeps/unix/sysv/linux/Makefile (dl-routines): Add
dl-map-segment-align.
* sysdeps/unix/sysv/linux/malloc-hugepages.c: Exclude THP helpers from
rtld.
Signed-off-by: WANG Rui <wangrui@loongson.cn>
---
sysdeps/unix/sysv/linux/Makefile | 1 +
sysdeps/unix/sysv/linux/malloc-hugepages.c | 4 ++
.../linux/wordsize-64/dl-map-segment-align.c | 56 +++++++++++++++++++
.../linux/wordsize-64/dl-map-segment-align.h | 23 ++++++++
4 files changed, 84 insertions(+)
create mode 100644 sysdeps/unix/sysv/linux/wordsize-64/dl-map-segment-align.c
create mode 100644 sysdeps/unix/sysv/linux/wordsize-64/dl-map-segment-align.h
diff --git a/sysdeps/unix/sysv/linux/Makefile b/sysdeps/unix/sysv/linux/Makefile
index c6bd97abf1..9879fec9a9 100644
--- a/sysdeps/unix/sysv/linux/Makefile
+++ b/sysdeps/unix/sysv/linux/Makefile
@@ -673,6 +673,7 @@ endif
ifeq ($(subdir),elf)
dl-routines += \
+ dl-map-segment-align \
dl-rseq-symbols \
# dl-routines
diff --git a/sysdeps/unix/sysv/linux/malloc-hugepages.c b/sysdeps/unix/sysv/linux/malloc-hugepages.c
index 77b49374e6..28930c0e53 100644
--- a/sysdeps/unix/sysv/linux/malloc-hugepages.c
+++ b/sysdeps/unix/sysv/linux/malloc-hugepages.c
@@ -81,6 +81,8 @@ __malloc_thp_mode (void)
return malloc_thp_mode_not_supported;
}
+#if !IS_IN (rtld)
+
static size_t
malloc_default_hugepage_size (void)
{
@@ -205,3 +207,5 @@ __malloc_hugepage_config (size_t requested, size_t *pagesize, int *flags)
__close_nocancel (dirfd);
}
+
+#endif /* !IS_IN(rtld) */
diff --git a/sysdeps/unix/sysv/linux/wordsize-64/dl-map-segment-align.c b/sysdeps/unix/sysv/linux/wordsize-64/dl-map-segment-align.c
new file mode 100644
index 0000000000..3c57d593bd
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/wordsize-64/dl-map-segment-align.c
@@ -0,0 +1,56 @@
+/* _dl_map_segment_align. Linux version.
+ Copyright (C) 2026 Free Software Foundation, Inc.
+ Copyright The GNU Toolchain Authors.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <dl-map-segment-align.h>
+#include <dl-tunables.h>
+#include <malloc-hugepages.h>
+
+#define MAX_THP_PAGESIZE (32 * 1024 * 1024)
+
+ElfW (Addr)
+_dl_map_segment_align (const struct loadcmd *c, ElfW (Addr) p_align_max)
+{
+ static enum malloc_thp_mode_t thp_mode = malloc_thp_mode_not_supported;
+ static unsigned long int thp_pagesize;
+
+#ifndef DL_MAP_SEGMENT_ALIGN_THP_DEFAULT
+ if (TUNABLE_GET (glibc, malloc, hugetlb, size_t, NULL) != 1)
+ return p_align_max;
+#endif
+
+ if (__glibc_unlikely (thp_mode == malloc_thp_mode_not_supported
+ || thp_pagesize == 0))
+ {
+ thp_mode = __malloc_thp_mode ();
+ thp_pagesize = __malloc_default_thp_pagesize ();
+ }
+
+ /* Aligning load segments that are large enough to the PMD size helps
+ improve THP eligibility and reduces TLB pressure.
+ We cap the huge page size at MAX_THP_PAGESIZE to avoid over-aligning
+ on systems with very large normal pages (like 64K pages with 512M
+ huge pages). */
+ if (thp_mode == malloc_thp_mode_always && thp_pagesize <= MAX_THP_PAGESIZE
+ && ((c->mapstart | c->mapoff) & (thp_pagesize - 1)) == 0
+ && (c->mapend - c->mapstart) >= thp_pagesize
+ && p_align_max < thp_pagesize && (c->prot & PROT_WRITE) == 0)
+ return thp_pagesize;
+
+ return p_align_max;
+}
diff --git a/sysdeps/unix/sysv/linux/wordsize-64/dl-map-segment-align.h b/sysdeps/unix/sysv/linux/wordsize-64/dl-map-segment-align.h
new file mode 100644
index 0000000000..2893408133
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/wordsize-64/dl-map-segment-align.h
@@ -0,0 +1,23 @@
+/* _dl_map_segment_align. Linux version.
+ Copyright (C) 2026 Free Software Foundation, Inc.
+ Copyright The GNU Toolchain Authors.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <dl-load.h>
+
+ElfW (Addr)
+ _dl_map_segment_align (const struct loadcmd *c, ElfW (Addr) p_align_max);
--
2.53.0
More information about the Libc-alpha
mailing list