[PATCH v8 5/6] elf: Align large load segments to PMD huge page size for THP

[H.J. Lu]

On Sun, Apr 5, 2026 at 11:56 AM WANG Rui <wangrui@loongson.cn> wrote:
>
> Mapping segments that are at least the size of a PMD huge page to
> huge-page-aligned addresses helps make them eligible for Transparent
> Huge Pages (THP).
>
> This patch introduces a Linux-specific helper, `_dl_map_segment_align`,
> to determine an appropriate maximum alignment for ELF load segments based
> on the system THP policy. The optimization is enabled only when the glibc
> tunable `glibc.elf.thp=1` is set and THP is configured to be used
> unconditionally.
>
> The optimization depends on Linux kernel support for file-backed THP,
> specifically:
>
> * `CONFIG_READ_ONLY_THP_FOR_FS` (available since Linux kernel 5.4), and
> * `CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS`.
>
> When enabled, the helper queries the default THP page size and uses it
> to align sufficiently large load segments that are already properly
> aligned in both virtual address and file offset (e.g., zero).
>
> For eligible segments, the alignment is bumped to the THP page size,
> which improves THP eligibility, reduces TLB pressure, and improves
> performance for large objects. To avoid excessive address space padding
> on systems with very large THP sizes, the alignment is capped at 32MB.
> The optimization is applied only to non-writable segments, matching
> typical THP usage.
>
> Signed-off-by: WANG Rui <wangrui@loongson.cn>
> ---
>  sysdeps/unix/sysv/linux/Makefile              |  1 +
>  .../unix/sysv/linux/dl-map-segment-align.c    | 53 +++++++++++++++++++
>  .../unix/sysv/linux/dl-map-segment-align.h    | 27 ++++++++++
>  3 files changed, 81 insertions(+)
>  create mode 100644 sysdeps/unix/sysv/linux/dl-map-segment-align.c
>  create mode 100644 sysdeps/unix/sysv/linux/dl-map-segment-align.h
>
> diff --git a/sysdeps/unix/sysv/linux/Makefile b/sysdeps/unix/sysv/linux/Makefile
> index c6bd97abf1a..9879fec9a93 100644
> --- a/sysdeps/unix/sysv/linux/Makefile
> +++ b/sysdeps/unix/sysv/linux/Makefile
> @@ -673,6 +673,7 @@ endif
>
>  ifeq ($(subdir),elf)
>  dl-routines += \
> +  dl-map-segment-align \
>    dl-rseq-symbols \
>    # dl-routines
>
> diff --git a/sysdeps/unix/sysv/linux/dl-map-segment-align.c b/sysdeps/unix/sysv/linux/dl-map-segment-align.c
> new file mode 100644
> index 00000000000..75fb0496730
> --- /dev/null
> +++ b/sysdeps/unix/sysv/linux/dl-map-segment-align.c
> @@ -0,0 +1,53 @@
> +/* _dl_map_segment_align.  Linux version.
> +   Copyright (C) 2026 Free Software Foundation, Inc.
> +   Copyright The GNU Toolchain Authors.
> +   This file is part of the GNU C Library.
> +
> +   The GNU C Library is free software; you can redistribute it and/or
> +   modify it under the terms of the GNU Lesser General Public
> +   License as published by the Free Software Foundation; either
> +   version 2.1 of the License, or (at your option) any later version.
> +
> +   The GNU C Library is distributed in the hope that it will be useful,
> +   but WITHOUT ANY WARRANTY; without even the implied warranty of
> +   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> +   Lesser General Public License for more details.
> +
> +   You should have received a copy of the GNU Lesser General Public
> +   License along with the GNU C Library; if not, see
> +   <https://www.gnu.org/licenses/>.  */
> +
> +#include <dl-map-segment-align.h>
> +#include <dl-tunables.h>
> +#include <hugepages.h>
> +
> +ElfW (Addr)
> +_dl_map_segment_align (const struct loadcmd *c, ElfW (Addr) p_align_max)
> +{
> +  static enum thp_mode_t thp_mode = thp_mode_not_supported;
> +  static unsigned long int thp_pagesize;
> +
> +  if (TUNABLE_GET (glibc, elf, thp, int32_t, NULL) == 0)
> +    return p_align_max;
> +
> +  if (__glibc_unlikely (thp_mode == thp_mode_not_supported
> +                       || thp_pagesize == 0))
> +    {
> +      unsigned long int default_thp_pagesize = DL_MAP_DEFAULT_THP_PAGESIZE;
> +      thp_mode = default_thp_pagesize ? thp_mode_always : __get_thp_mode ();
> +      thp_pagesize = default_thp_pagesize ? : __get_thp_size ();
> +    }
> +
> +  /* Aligning load segments that are large enough to the PMD size helps
> +     improve THP eligibility and reduces TLB pressure.
> +     We cap the huge page size at MAX_THP_PAGESIZE to avoid over-aligning
> +     on systems with very large normal pages (like 64K pages with 512M
> +     huge pages). */
> +  if (thp_mode == thp_mode_always && thp_pagesize <= MAX_THP_PAGESIZE
> +      && ((c->mapstart | c->mapoff) & (thp_pagesize - 1)) == 0
> +      && (c->mapend - c->mapstart) >= thp_pagesize
> +      && p_align_max < thp_pagesize && (c->prot & PROT_WRITE) == 0)
> +    return thp_pagesize;
> +
> +  return p_align_max;
> +}
> diff --git a/sysdeps/unix/sysv/linux/dl-map-segment-align.h b/sysdeps/unix/sysv/linux/dl-map-segment-align.h
> new file mode 100644
> index 00000000000..83794a03986
> --- /dev/null
> +++ b/sysdeps/unix/sysv/linux/dl-map-segment-align.h
> @@ -0,0 +1,27 @@
> +/* _dl_map_segment_align.  Linux version.
> +   Copyright (C) 2026 Free Software Foundation, Inc.
> +   Copyright The GNU Toolchain Authors.
> +   This file is part of the GNU C Library.
> +
> +   The GNU C Library is free software; you can redistribute it and/or
> +   modify it under the terms of the GNU Lesser General Public
> +   License as published by the Free Software Foundation; either
> +   version 2.1 of the License, or (at your option) any later version.
> +
> +   The GNU C Library is distributed in the hope that it will be useful,
> +   but WITHOUT ANY WARRANTY; without even the implied warranty of
> +   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> +   Lesser General Public License for more details.
> +
> +   You should have received a copy of the GNU Lesser General Public
> +   License along with the GNU C Library; if not, see
> +   <https://www.gnu.org/licenses/>.  */
> +
> +#include <dl-load.h>
> +
> +#ifndef DL_MAP_DEFAULT_THP_PAGESIZE
> +# define DL_MAP_DEFAULT_THP_PAGESIZE   0
> +#endif
> +
> +ElfW (Addr)
> +    _dl_map_segment_align (const struct loadcmd *c, ElfW (Addr) p_align_max);

extern ElfW (Addr) _dl_map_segment_align (const struct loadcmd *, ElfW (Addr));

> --
> 2.53.0
>


-- 
H.J.