[PATCH v10] elf: Support THP segment load with madvise enabled THP
Adhemerval Zanella Netto
adhemerval.zanella@linaro.org
Tue May 26 23:01:26 GMT 2026
On 26/05/26 19:44, H.J. Lu wrote:
>>> # define __rtld_global_attribute__
>>> diff --git a/sysdeps/unix/sysv/linux/Makefile b/sysdeps/unix/sysv/linux/Makefile
>>> index 63e7046cb3..edf48c93de 100644
>>> --- a/sysdeps/unix/sysv/linux/Makefile
>>> +++ b/sysdeps/unix/sysv/linux/Makefile
>>> @@ -698,11 +698,15 @@ $(objpfx)pldd: $(objpfx)xmalloc.o
>>> tests += \
>>> tst-rseq-tls-range \
>>> tst-rseq-tls-range-4096 \
>>> + tst-thp-1 \
>>> + tst-thp-1-pde \
>>> + tst-thp-1-static \
>>
>> I think there is no need to duplicate this here.
>
> It is needed. Otherwise, tst-thp-1-static won't run.
Ack, sometime I forgot about this limitation (and some Makefile does include
both using different strategies).
>
>>> tst-thp-align \
>>> # tests
>>> tests-static += \
>>> tst-rseq-tls-range-4096-static \
>>> tst-rseq-tls-range-static \
>>> + tst-thp-1-static \
>>> # tests-static
>>> modules-names += \
>>> tst-rseq-tls-range-mod \
>>> @@ -712,15 +716,12 @@ CFLAGS-tst-rseq-tls-range.c += -DMAIN_TLS_ALIGN=4
>>> CFLAGS-tst-rseq-tls-range-4096.c += -DMAIN_TLS_ALIGN=4096
>>> CFLAGS-tst-rseq-tls-range-static.c += -DMAIN_TLS_ALIGN=4
>>> CFLAGS-tst-rseq-tls-range-4096-static.c += -DMAIN_TLS_ALIGN=4096
>>> -LDFLAGS-tst-thp-size-mod.so += -Wl,-z,noseparate-code
>>> $(objpfx)tst-rseq-tls-range.out: $(objpfx)tst-rseq-tls-range-mod.so
>>> $(objpfx)tst-rseq-tls-range-4096.out: $(objpfx)tst-rseq-tls-range-mod.so
>>> $(objpfx)tst-rseq-tls-range-static.out: $(objpfx)tst-rseq-tls-range-mod.so
>>> $(objpfx)tst-rseq-tls-range-4096-static.out: $(objpfx)tst-rseq-tls-range-mod.so
>>> -$(objpfx)tst-thp-align.out: $(objpfx)tst-thp-size-mod.so
>>> tst-rseq-tls-range-static-ENV = LD_LIBRARY_PATH=$(objpfx):$(common-objpfx)
>>> tst-rseq-tls-range-4096-static-ENV = LD_LIBRARY_PATH=$(objpfx):$(common-objpfx)
>>> -tst-thp-align-ENV = GLIBC_TUNABLES=glibc.elf.thp=1
>>>
>>> test-internal-extras += tst-nolink-libc
>>> ifeq ($(run-built-tests),yes)
>>> @@ -729,6 +730,151 @@ tests-special += \
>>> $(objpfx)tst-nolink-libc-2.out \
>>> # tests-special
>>> endif
>>> +
>>> +ifndef THP-PAGE-SIZE
>>> +# Align PT_LOAD segments in THP tests to THP page size so that kernel will
>>> +# map PIE to the address aligned to THP page size. Default THP page size
>>> +# to 2MB which can be overridden in Makefile in subdirectories.
>>> +THP-PAGE-SIZE = 0x200000
>>> +endif
>>> +
>>> +THP-PAGE-SIZE-LDFLAGS = -Wl,-z,max-page-size=$(THP-PAGE-SIZE)
>>> +
>>> +LDFLAGS-tst-thp-size-mod.so = -Wl,-z,noseparate-code \
>>> + $(THP-PAGE-SIZE-LDFLAGS)
>>> +tst-thp-align-ENV = GLIBC_TUNABLES=glibc.elf.thp=1
>>> +$(objpfx)tst-thp-align.out: $(objpfx)tst-thp-size-mod.so
>>> +
>>> +tests += \
>>> + tst-thp-1-no-s-code \
>>> + tst-thp-1-no-s-code-pde \
>>> + tst-thp-1-no-s-code-static \
>>> +# tests
>>> +tests-static += \
>>> + tst-thp-1-no-s-code-static \
>>> +# tests-static
>>> +
>>> +LDFLAGS-tst-thp-1 = -Wl,-z,separate-code $(THP-PAGE-SIZE-LDFLAGS)
>>> +LDFLAGS-tst-thp-1-pde = -Wl,-z,separate-code $(THP-PAGE-SIZE-LDFLAGS)
>>> +LDFLAGS-tst-thp-1-static = -Wl,-z,separate-code $(THP-PAGE-SIZE-LDFLAGS)
>>> +LDFLAGS-tst-thp-1-no-s-code = -Wl,-z,noseparate-code \
>>> + $(THP-PAGE-SIZE-LDFLAGS)
>>> +LDFLAGS-tst-thp-1-no-s-code-pde = -Wl,-z,noseparate-code \
>>> + $(THP-PAGE-SIZE-LDFLAGS)
>>> +LDFLAGS-tst-thp-1-no-s-code-static = -Wl,-z,noseparate-code \
>>> + $(THP-PAGE-SIZE-LDFLAGS)
>>> +
>>> +$(objpfx)tst-thp-1-no-s-code: $(objpfx)tst-thp-size-mod.o
>>> +$(objpfx)tst-thp-1-no-s-code-pde: $(objpfx)tst-thp-size-mod.o
>>> +$(objpfx)tst-thp-1-no-s-code-static: $(objpfx)tst-thp-size-mod.o
>>> +
>>> +tst-thp-1-no-s-code-ENV = GLIBC_TUNABLES=glibc.elf.thp=1
>>> +tst-thp-1-no-s-code-pde-ENV = GLIBC_TUNABLES=glibc.elf.thp=1
>>> +tst-thp-1-no-s-code-static-ENV = GLIBC_TUNABLES=glibc.elf.thp=1
>>> +
>>> +tst-thp-1-no-s-code-pde-no-pie = yes
>>> +
>>> +tst-thp-1-ENV = GLIBC_TUNABLES=glibc.elf.thp=1
>>> +tst-thp-1-pde-ENV = GLIBC_TUNABLES=glibc.elf.thp=1
>>> +tst-thp-1-static-ENV = GLIBC_TUNABLES=glibc.elf.thp=1
>>> +
>>> +$(objpfx)tst-thp-1: $(objpfx)tst-thp-size-mod.o
>>> +$(objpfx)tst-thp-1-pde: $(objpfx)tst-thp-size-mod.o
>>> +$(objpfx)tst-thp-1-static: $(objpfx)tst-thp-size-mod.o
>>> +
>>> +tst-thp-1-pde-no-pie = yes
>>> +
>>> +# Don't run strace tests for cross-compiling.
>>> +ifeq (no,$(cross-compiling))
>>> +thp-kernel-status = $(shell grep madvise /sys/kernel/mm/transparent_hugepage/enabled)
>>> +# Verify that madvise is called with MADV_HUGEPAGE when THP is enabled
>>> +# under madvise THP kernel.
>>> +ifneq ($(findstring [madvise],$(thp-kernel-status)),)
>>> +tests-special += \
>>> + $(objpfx)strace-tst-thp-1-disabled.out \
>>> + $(objpfx)strace-tst-thp-1-enabled.out \
>>> + $(objpfx)strace-tst-thp-1-pde-disabled.out \
>>> + $(objpfx)strace-tst-thp-1-pde-enabled.out \
>>> + $(objpfx)strace-tst-thp-1-static-disabled.out \
>>> + $(objpfx)strace-tst-thp-1-static-enabled.out \
>>> + $(objpfx)strace-tst-thp-align-default.out \
>>> + $(objpfx)strace-tst-thp-align-disabled.out \
>>> + $(objpfx)strace-tst-thp-align-enabled.out \
>>> +# tests-special
>>> +
>>> +$(objpfx)strace-tst-thp-1-enabled.out: \
>>> + $(..)sysdeps/unix/sysv/linux/strace-tst-thp.sh $(objpfx)ld.so \
>>> + $(objpfx)tst-thp-1
>>> + $(SHELL) $< $(objpfx)ld.so '$(test-wrapper-env)' \
>>> + '$(run-program-env) GLIBC_TUNABLES=glibc.elf.thp=1' \
>>> + '$(rpath-link)' $(objpfx)tst-thp-1 > $@; \
>>> + $(evaluate-test)
>>> +
>>> +$(objpfx)strace-tst-thp-1-disabled.out: \
>>> + $(..)sysdeps/unix/sysv/linux/strace-tst-thp.sh $(objpfx)ld.so \
>>> + $(objpfx)tst-thp-1
>>> + $(SHELL) $< $(objpfx)ld.so '$(test-wrapper-env)' \
>>> + '$(run-program-env) GLIBC_TUNABLES=glibc.elf.thp=0' \
>>> + '$(rpath-link)' $(objpfx)tst-thp-1 > $@; \
>>> + $(evaluate-test)
>>> +
>>> +$(objpfx)strace-tst-thp-1-pde-enabled.out: \
>>> + $(..)sysdeps/unix/sysv/linux/strace-tst-thp.sh $(objpfx)ld.so \
>>> + $(objpfx)tst-thp-1-pde
>>> + $(SHELL) $< $(objpfx)ld.so '$(test-wrapper-env)' \
>>> + '$(run-program-env) GLIBC_TUNABLES=glibc.elf.thp=1' \
>>> + '$(rpath-link)' $(objpfx)tst-thp-1-pde > $@; \
>>> + $(evaluate-test)
>>> +
>>> +$(objpfx)strace-tst-thp-1-pde-disabled.out: \
>>> + $(..)sysdeps/unix/sysv/linux/strace-tst-thp.sh $(objpfx)ld.so \
>>> + $(objpfx)tst-thp-1-pde
>>> + $(SHELL) $< $(objpfx)ld.so '$(test-wrapper-env)' \
>>> + '$(run-program-env) GLIBC_TUNABLES=glibc.elf.thp=0' \
>>> + '$(rpath-link)' $(objpfx)tst-thp-1-pde > $@; \
>>> + $(evaluate-test)
>>> +
>>> +$(objpfx)strace-tst-thp-1-static-enabled.out: \
>>> + $(..)sysdeps/unix/sysv/linux/strace-tst-thp.sh $(objpfx)ld.so \
>>> + $(objpfx)tst-thp-1-static
>>> + $(SHELL) $< $(objpfx)ld.so '$(test-wrapper-env)' \
>>> + '$(run-program-env) GLIBC_TUNABLES=glibc.elf.thp=1' \
>>> + '$(rpath-link)' $(objpfx)tst-thp-1-static > $@; \
>>> + $(evaluate-test)
>>> +
>>> +$(objpfx)strace-tst-thp-1-static-disabled.out: \
>>> + $(..)sysdeps/unix/sysv/linux/strace-tst-thp.sh $(objpfx)ld.so \
>>> + $(objpfx)tst-thp-1-static
>>> + $(SHELL) $< $(objpfx)ld.so '$(test-wrapper-env)' \
>>> + '$(run-program-env) GLIBC_TUNABLES=glibc.elf.thp=0' \
>>> + '$(rpath-link)' $(objpfx)tst-thp-1-static > $@; \
>>> + $(evaluate-test)
>>
>> This would check the loader fails patch to load a static binary, not the
>> tst-thp-1-static binary itselt. I think it passes for accidental reasons,
>> so it would be better to issue the test directly.
>
> Fixed.
>
>>> +
>>> +$(objpfx)strace-tst-thp-align-default.out: \
>>> + $(..)sysdeps/unix/sysv/linux/strace-tst-thp.sh $(objpfx)ld.so \
>>> + $(objpfx)tst-thp-align
>>> + $(SHELL) $< $(objpfx)ld.so '$(test-wrapper-env)' \
>>> + '$(run-program-env)' \
>>> + '$(rpath-link)' $(objpfx)tst-thp-align > $@; \
>>> + $(evaluate-test)
>>> +
>>> +$(objpfx)strace-tst-thp-align-enabled.out: \
>>> + $(..)sysdeps/unix/sysv/linux/strace-tst-thp.sh $(objpfx)ld.so \
>>> + $(objpfx)tst-thp-align
>>> + $(SHELL) $< $(objpfx)ld.so '$(test-wrapper-env)' \
>>> + '$(run-program-env) GLIBC_TUNABLES=glibc.elf.thp=1' \
>>> + '$(rpath-link)' $(objpfx)tst-thp-align > $@; \
>>> + $(evaluate-test)
>>> +
>>> +$(objpfx)strace-tst-thp-align-disabled.out: \
>>> + $(..)sysdeps/unix/sysv/linux/strace-tst-thp.sh $(objpfx)ld.so \
>>> + $(objpfx)tst-thp-align
>>> + $(SHELL) $< $(objpfx)ld.so '$(test-wrapper-env)' \
>>> + '$(run-program-env) GLIBC_TUNABLES=glibc.elf.thp=0' \
>>> + '$(rpath-link)' $(objpfx)tst-thp-align > $@; \
>>> + $(evaluate-test)
>>> +endif # [madvise]
>>> +endif # $(cross-compiling)
>>> endif # $(subdir) == elf
>>>
>>> ifeq ($(subdir),rt)
>>> diff --git a/sysdeps/unix/sysv/linux/arm/Makefile b/sysdeps/unix/sysv/linux/arm/Makefile
>>> index e73ce4f811..1ee8bec9b9 100644
>>> --- a/sysdeps/unix/sysv/linux/arm/Makefile
>>> +++ b/sysdeps/unix/sysv/linux/arm/Makefile
>>> @@ -3,6 +3,13 @@ sysdep-rtld-routines += aeabi_read_tp libc-do-syscall
>>> # The test uses INTERNAL_SYSCALL_CALL. In thumb mode, this uses
>>> # an undefined reference to __libc_do_syscall.
>>> CFLAGS-tst-nolink-libc.c += -marm
>>> +
>>> +# These tests fail on arm due to limitations of arm32 kABI:
>>> +# https://sourceware.org/bugzilla/show_bug.cgi?id=34096
>>> +test-xfail-tst-thp-1-no-s-code-pde = yes
>>> +test-xfail-tst-thp-1-no-s-code-static = yes
>>> +test-xfail-tst-thp-1-pde = yes
>>> +test-xfail-tst-thp-1-static = yes
>>> endif
>>>
>>> ifeq ($(subdir),misc)
>>> diff --git a/sysdeps/unix/sysv/linux/dl-exec-post.h b/sysdeps/unix/sysv/linux/dl-exec-post.h
>>> new file mode 100644
>>> index 0000000000..3ce7a53b33
>>> --- /dev/null
>>> +++ b/sysdeps/unix/sysv/linux/dl-exec-post.h
>>> @@ -0,0 +1,126 @@
>>> +/* _dl_executable_postprocess. Linux version.
>>> + Copyright (C) 2026 Free Software Foundation, Inc.
>>> + Copyright The GNU Toolchain Authors.
>>> + This file is part of the GNU C Library.
>>> +
>>> + The GNU C Library is free software; you can redistribute it and/or
>>> + modify it under the terms of the GNU Lesser General Public
>>> + License as published by the Free Software Foundation; either
>>> + version 2.1 of the License, or (at your option) any later version.
>>> +
>>> + The GNU C Library is distributed in the hope that it will be useful,
>>> + but WITHOUT ANY WARRANTY; without even the implied warranty of
>>> + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
>>> + Lesser General Public License for more details.
>>> +
>>> + You should have received a copy of the GNU Lesser General Public
>>> + License along with the GNU C Library; if not, see
>>> + <https://www.gnu.org/licenses/>. */
>>> +
>>
>> I think it is worth a comment that when the architecture defines
>> DL_MAP_DEFAULT_THP_PAGESIZE, the sysfs is not read and variable OMD
>> are not considered. Also add the rationale why ssyfs is not consulted
>> and madvise is always applied unconditionally.
>
> There are
>
> /* NB: Accessing /sys/kernel/mm files is quite expensive and the file
> may not be accessible in containers. If DL_MAP_DEFAULT_THP_PAGESIZE
> is non-zero, assume THP mode is madvise and always call madvise.
> Since madvise is a fast system call, it adds only a small overhead
> compared to the cost of accessing /sys/kernel/mm files. */
> if (DL_MAP_DEFAULT_THP_PAGESIZE != 0)
> {
> GL(dl_elf_thp_pagesize) = DL_MAP_DEFAULT_THP_PAGESIZE;
> GL(dl_thp_mode) = thp_mode_madvise;
> }
>
> How should comments be improved?
Right, this does seems to cover my point.
>
>>> +static inline void
>>> +_dl_get_thp_config (void)
>>> +{
>>> + /* Check if there is GLIBC_TUNABLES=glibc.elf.thp=[0|1]. */
>>> + GL(dl_elf_thp_control) = TUNABLE_GET_FULL (glibc, elf, thp, int32_t,
>>> + NULL);
>>> +
>>> + /* Return if the tunable is not set or THP is disabled by the
>>> + tunable. */
>>> + if (GL(dl_elf_thp_control) == 0)
>>> + return;
>>> +
>>> + _Static_assert (DL_MAP_DEFAULT_THP_PAGESIZE <= MAX_THP_PAGESIZE,
>>> + "DL_MAP_DEFAULT_THP_PAGESIZE <= MAX_THP_PAGESIZE");
>>> +
>>> + /* NB: Accessing /sys/kernel/mm files is quite expensive and the file
>>> + may not be accessible in containers. If DL_MAP_DEFAULT_THP_PAGESIZE
>>> + is non-zero, assume THP mode is madvise and always call madvise.
>>> + Since madvise is a fast systemcall, it adds only a small overhead
>>> + compared to the cost of accessing /sys/kernel/mm files. */
>>> + if (DL_MAP_DEFAULT_THP_PAGESIZE != 0)
>>> + {
>>> + GL(dl_elf_thp_pagesize) = DL_MAP_DEFAULT_THP_PAGESIZE;
>>> + GL(dl_thp_mode) = thp_mode_madvise;
>>> + }
>>> + else
>>> + {
>>> + GL(dl_thp_mode) = __get_thp_mode ();
>>> + if (GL(dl_thp_mode) == thp_mode_always
>>> + || GL(dl_thp_mode) == thp_mode_madvise)
>>> + {
>>> + GL(dl_elf_thp_pagesize) = __get_thp_size ();
>>> + /* We cap the huge page size at MAX_THP_PAGESIZE to avoid
>>> + over-aligning on systems with very large normal pages
>>> + (like 64K pages with 512M huge pages). */
>>> + if (GL(dl_elf_thp_pagesize) > MAX_THP_PAGESIZE)
>>> + GL(dl_elf_thp_pagesize) = 0;
>>> + }
>>> + else
>>> + GL(dl_elf_thp_pagesize) = 0;
>>> +
>>> + if (GL(dl_elf_thp_pagesize) == 0)
>>> + {
>>> + GL(dl_elf_thp_control) = 0;
>>> + GL(dl_thp_mode) = thp_mode_not_supported;
>>> + }
>>> + }
>>> +}
>>> +
>>> +static inline void
>>> +_dl_executable_postprocess (struct link_map *main_map,
>>> + const ElfW(Phdr) *phdr, ElfW(Word) phnum)
>>> +{
>>> + /* NB: In static executable, PT_GNU_PROPERTY is processed in target
>>> + libc-start.h if it is needed by target. When ld.so is used, if
>>> + a target doesn't need PT_GNU_PROPERTY, _dl_process_pt_gnu_property
>>> + is an empty function. */
>>> +#ifdef SHARED
>>> + /* Process program headers again, but scan them backwards since
>>> + PT_GNU_PROPERTY is close to the end of program headers. */
>>> + for (const ElfW(Phdr) *ph = &phdr[phnum]; ph != phdr; --ph)
>>> + if (ph[-1].p_type == PT_GNU_PROPERTY)
>>> + {
>>> + _dl_process_pt_gnu_property (main_map, -1, &ph[-1]);
>>> + break;
>>> + }
>>> +#endif
>>> +
>>> + /* If THP state was not yet initialized, the main executable was mapped
>>> + by the kernel; in that case this function is the only place that can
>>> + apply MADV_HUGEPAGE to the main executable's segments. Otherwise,
>>> + _dl_get_thp_config has already run earlier in dl_main and
>>> + _dl_map_segments has just mapped the main executable, so
>>> + _dl_postprocess_loadcmd_extra has already done the madvise pass; do
>>> + not repeat it here. */
>>> + if (GL(dl_elf_thp_control) != -1)
>>> + return;
>>> +
>>> + _dl_get_thp_config ();
>>> +
>>> + /* Return if THP segment load isn't enabled. */
>>> + if (GL(dl_elf_thp_control) <= 0)
>>> + return;
>>> +
>>> + /* NB: If DL_MAP_DEFAULT_THP_PAGESIZE is non-zero, dl_thp_mode is set
>>> + to thp_mode_madvise. */
>>> + if (DL_MAP_DEFAULT_THP_PAGESIZE == 0
>>> + && GL(dl_thp_mode) != thp_mode_madvise)
>>> + return;
>>> +
>>> + /* When we get here, the main executable have been mapped in. Call
>>> + madvise with MADV_HUGEPAGE for all THP eligible PT_LOAD segments. */
>>> +
>>> + const ElfW(Phdr) *ph;
>>> +
>>> + size_t thp_pagesize = GL(dl_elf_thp_pagesize);
>>> +
>>> + /* Call __madvise if offset and address of the PT_LOAD segment are
>>> + aligned to THP page size and it is read-only. */
>>> + for (ph = phdr; ph < &phdr[phnum]; ++ph)
>>> + if (ph->p_type == PT_LOAD
>>> + && ph->p_memsz >= thp_pagesize
>>> + && ((ph->p_vaddr | ph->p_offset) & (thp_pagesize - 1)) == 0
>>> + && (ph->p_flags & (PF_W | PF_R)) == PF_R)
>>> + __madvise ((void *) (main_map->l_addr + ph->p_vaddr),
>>> + ph->p_memsz, MADV_HUGEPAGE);
>>> +}
>>> diff --git a/sysdeps/unix/sysv/linux/dl-load-post.h b/sysdeps/unix/sysv/linux/dl-load-post.h
>>> new file mode 100644
>>> index 0000000000..88467764d4
>>> --- /dev/null
>>> +++ b/sysdeps/unix/sysv/linux/dl-load-post.h
>>> @@ -0,0 +1,32 @@
>>> +/* _dl_postprocess_loadcmd_extra. Linux version.
>>> + Copyright (C) 2026 Free Software Foundation, Inc.
>>> + Copyright The GNU Toolchain Authors.
>>> + This file is part of the GNU C Library.
>>> +
>>> + The GNU C Library is free software; you can redistribute it and/or
>>> + modify it under the terms of the GNU Lesser General Public
>>> + License as published by the Free Software Foundation; either
>>> + version 2.1 of the License, or (at your option) any later version.
>>> +
>>> + The GNU C Library is distributed in the hope that it will be useful,
>>> + but WITHOUT ANY WARRANTY; without even the implied warranty of
>>> + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
>>> + Lesser General Public License for more details.
>>> +
>>> + You should have received a copy of the GNU Lesser General Public
>>> + License along with the GNU C Library; if not, see
>>> + <https://www.gnu.org/licenses/>. */
>>> +
>>> +static bool _dl_segment_thp_eligible (const struct loadcmd *, size_t);
>>> +
>>> +/* After L has been mapped in, call madvise with MADV_HUGEPAGE for THP
>>> + madvise mode if L is THP eligible. */
>>> +
>>> +static inline void
>>> +_dl_postprocess_loadcmd_extra (struct link_map *l, const struct loadcmd *c)
>>> +{
>>> + if (GL(dl_thp_mode) == thp_mode_madvise
>>> + && _dl_segment_thp_eligible (c, GL(dl_elf_thp_pagesize)))
>>> + __madvise ((void *) (l->l_addr + c->mapstart),
>>> + c->mapend - c->mapstart, MADV_HUGEPAGE);
>>> +}
>>> diff --git a/sysdeps/unix/sysv/linux/dl-map-segment-align.c b/sysdeps/unix/sysv/linux/dl-map-segment-align.c
>>> index a39e74d91b..1260b26c22 100644
>>> --- a/sysdeps/unix/sysv/linux/dl-map-segment-align.c
>>> +++ b/sysdeps/unix/sysv/linux/dl-map-segment-align.c
>>> @@ -17,38 +17,23 @@
>>> License along with the GNU C Library; if not, see
>>> <https://www.gnu.org/licenses/>. */
>>>
>>> +#include <ldsodefs.h>
>>> #include <dl-map-segment-align.h>
>>> -#include <dl-tunables.h>
>>> -#include <hugepages.h>
>>> +
>>> +/* Return the alignment of the PT_LOAD segment for THP. P_ALIGN_MAX is
>>> + the maximum p_align value in the PT_LOAD segment. */
>>>
>>> ElfW (Addr)
>>> _dl_map_segment_align (const struct loadcmd *c, ElfW (Addr) p_align_max)
>>> {
>>> - static enum thp_mode_t thp_mode = thp_mode_not_supported;
>>> - static unsigned long int thp_pagesize;
>>> + size_t thp_pagesize = GL(dl_elf_thp_pagesize);
>>>
>>> - if (TUNABLE_GET (glibc, elf, thp, int32_t, NULL) == 0)
>>> + if (GL(dl_elf_thp_control) <= 0 || p_align_max >= thp_pagesize)
>>> return p_align_max;
>>>
>>> - if (__glibc_unlikely (thp_mode == thp_mode_not_supported
>>> - || thp_pagesize == 0))
>>> - {
>>> - unsigned long int default_thp_pagesize = DL_MAP_DEFAULT_THP_PAGESIZE;
>>> - thp_mode = default_thp_pagesize ? thp_mode_always : __get_thp_mode ();
>>> - thp_pagesize = default_thp_pagesize ? : __get_thp_size ();
>>> - }
>>> -
>>> - /* Aligning load segments that are large enough to the PMD size helps
>>> - improve THP eligibility and reduces TLB pressure.
>>> - We cap the huge page size at MAX_THP_PAGESIZE to avoid over-aligning
>>> - on systems with very large normal pages (like 64K pages with 512M
>>> - huge pages). */
>>> - if (thp_mode == thp_mode_always
>>> - && thp_pagesize <= MAX_THP_PAGESIZE
>>> - && ((c->mapstart | c->mapoff) & (thp_pagesize - 1)) == 0
>>> - && (c->mapend - c->mapstart) >= thp_pagesize
>>> - && p_align_max < thp_pagesize
>>> - && (c->prot & PROT_WRITE) == 0)
>>
>> Maybe now that 'thp_pagesize <= MAX_THP_PAGESIZE' is removed add a:
>>
>> _Static_assert (DL_MAP_DEFAULT_THP_PAGESIZE <= MAX_THP_PAGESIZE, ...)
>>
>
> These are the removed codes. The new one is
>
> ElfW (Addr)
> _dl_map_segment_align (const struct loadcmd *c, ElfW (Addr) p_align_max)
> {
> size_t thp_pagesize = GL(dl_elf_thp_pagesize);
>
> if (GL(dl_elf_thp_control) <= 0 || p_align_max >= thp_pagesize)
> return p_align_max;
>
> /* Return true if the segment is THP eligible. It helps improve THP
> eligibility and reduces TLB pressure. */
> if (_dl_segment_thp_eligible (c, thp_pagesize))
> return thp_pagesize;
>
> return p_align_max;
> }
Ack, I just noticed the _Static_assert on _dl_get_thp_config. I should have finished
the patch before pressed send.
More information about the Libc-alpha
mailing list