Currently testing H.J's latest patchset (v5) of the mmap work [0] on top of trunk. So far, package builds have gone well, but when building some Rust software earlier, I noticed ld took over 10 minutes (!). I've attached a tarball which reproduces it, I've not tried to minimise it as it's Rust with its gazillion static library deps (sorry). The hanging command is: ``` /usr/bin/x86_64-pc-linux-gnu-gcc -m64 /var/tmp/portage/sys-process/procs-0.14.5/temp/rustca2ri5G/symbols.o /var/tmp/portage/sys-process/procs-0.14.5/work/procs-0.14.5/target/release/deps/procs-bf23c8fa176eac21.procs.d1865a8196e0707d-cgu.0.rcgu.o -Wl,--as-needed -L /var/tmp/portage/sys-process/procs-0.14.5/work/procs-0.14.5/target/release/deps -L /usr/lib/rust/1.75.0/lib/rustlib/x86_64-unknown-linux-gnu/lib -Wl,-Bstatic /usr/lib/rust/1.75.0/lib/rustlib/x86_64-unknown-linux-gnu/lib/libcompiler_builtins-275fd5f39e82e176.rlib -Wl,-Bdynamic -lc -lgcc_s -lutil -lrt -lpthread -lm -ldl -lc -Wl,--eh-frame-hdr -Wl,-z,noexecstack -L /usr/lib/rust/1.75.0/lib/rustlib/x86_64-unknown-linux-gnu/lib -o /var/tmp/portage/sys-process/procs-0.14.5/work/procs-0.14.5/target/release/deps/procs-bf23c8fa176eac21 -Wl,--gc-sections -pie -Wl,-z,relro,-z,now -nodefaultlibs ``` If I append `-fuse-ld=lld`, it completes with <1s. After running for 3 minutes, attaching gdb gives: ``` 0x00007f26fd2ed2e3 in bfd_getl64 (p=0x7f26fa3f1530) at /usr/src/debug/sys-devel/binutils-9999/binutils/bfd/libbfd.c:913 913 } (gdb) bt #0 0x00007f26fd2ed2e3 in bfd_getl64 (p=0x7f26fa3f1530) at /usr/src/debug/sys-devel/binutils-9999/binutils/bfd/libbfd.c:913 #1 0x00007f26fd310ca7 in bfd_elf64_swap_reloca_in (abfd=abfd@entry=0x55f06a8871e0, s=s@entry=0x7f26fa3f1528 "\f\033\020", dst=dst@entry=0x55f079aab3c8) at /usr/src/debug/sys-devel/binutils-9999/binutils/bfd/elfcode.h:445 #2 0x00007f26fd33702d in elf_link_read_relocs_from_section (abfd=abfd@entry=0x55f06a8871e0, sec=sec@entry=0x55f0717bf2c8, shdr=0x55f0717bfa78, external_relocs_addr=<optimized out>, external_relocs_size=<optimized out>, internal_relocs=<optimized out>) at /usr/src/debug/sys-devel/binutils-9999/binutils/bfd/elflink.c:2708 #3 0x00007f26fd3373bc in _bfd_elf_link_info_read_relocs (abfd=0x55f06a8871e0, info=<optimized out>, o=0x55f0717bf2c8, external_relocs=<optimized out>, internal_relocs=0x55f0799a8c90, keep_memory=<optimized out>) at /usr/src/debug/sys-devel/binutils-9999/binutils/bfd/elflink.c:2832 #4 0x00007f26fd352b0f in init_reloc_cookie_rels (cookie=0x7ffe635e0ff0, info=0x55f068a4fec0 <link_info>, abfd=<optimized out>, sec=0x55f0717bf2c8) at /usr/src/debug/sys-devel/binutils-9999/binutils/bfd/elflink.c:13668 #5 init_reloc_cookie_for_section (cookie=cookie@entry=0x7ffe635e0ff0, info=info@entry=0x55f068a4fec0 <link_info>, sec=sec@entry=0x55f0717bf2c8) at /usr/src/debug/sys-devel/binutils-9999/binutils/bfd/elflink.c:13700 #6 0x00007f26fd35307c in _bfd_elf_gc_mark (info=info@entry=0x55f068a4fec0 <link_info>, sec=sec@entry=0x55f06dcb2158, gc_mark_hook=gc_mark_hook@entry=0x7f26fd3110e0 <_bfd_x86_elf_gc_mark_hook>) at /usr/src/debug/sys-devel/binutils-9999/binutils/bfd/elflink.c:13929 #7 0x00007f26fd352dc6 in _bfd_elf_gc_mark_reloc (info=info@entry=0x55f068a4fec0 <link_info>, sec=sec@entry=0x55f06dcb0c28, gc_mark_hook=gc_mark_hook@entry=0x7f26fd3110e0 <_bfd_x86_elf_gc_mark_hook>, cookie=cookie@entry=0x7ffe635e10d0) at /usr/src/debug/sys-devel/binutils-9999/binutils/bfd/elflink.c:13872 #8 0x00007f26fd35300a in _bfd_elf_gc_mark (info=info@entry=0x55f068a4fec0 <link_info>, sec=sec@entry=0x55f06dcb0c28, gc_mark_hook=gc_mark_hook@entry=0x7f26fd3110e0 <_bfd_x86_elf_gc_mark_hook>) at /usr/src/debug/sys-devel/binutils-9999/binutils/bfd/elflink.c:13916 #9 0x00007f26fd352dc6 in _bfd_elf_gc_mark_reloc (info=info@entry=0x55f068a4fec0 <link_info>, sec=sec@entry=0x55f06dcb09c8, gc_mark_hook=gc_mark_hook@entry=0x7f26fd3110e0 <_bfd_x86_elf_gc_mark_hook>, cookie=cookie@entry=0x7ffe635e11b0) at /usr/src/debug/sys-devel/binutils-9999/binutils/bfd/elflink.c:13872 #10 0x00007f26fd35300a in _bfd_elf_gc_mark (info=info@entry=0x55f068a4fec0 <link_info>, sec=sec@entry=0x55f06dcb09c8, gc_mark_hook=gc_mark_hook@entry=0x7f26fd3110e0 <_bfd_x86_elf_gc_mark_hook>) at /usr/src/debug/sys-devel/binutils-9999/binutils/bfd/elflink.c:13916 #11 0x00007f26fd352dc6 in _bfd_elf_gc_mark_reloc (info=info@entry=0x55f068a4fec0 <link_info>, sec=sec@entry=0x55f06dcfe618, gc_mark_hook=gc_mark_hook@entry=0x7f26fd3110e0 <_bfd_x86_elf_gc_mark_hook>, cookie=cookie@entry=0x7ffe635e1290) at /usr/src/debug/sys-devel/binutils-9999/binutils/bfd/elflink.c:13872 #12 0x00007f26fd35300a in _bfd_elf_gc_mark (info=info@entry=0x55f068a4fec0 <link_info>, sec=sec@entry=0x55f06dcfe618, gc_mark_hook=gc_mark_hook@entry=0x7f26fd3110e0 <_bfd_x86_elf_gc_mark_hook>) at /usr/src/debug/sys-devel/binutils-9999/binutils/bfd/elflink.c:13916 #13 0x00007f26fd352dc6 in _bfd_elf_gc_mark_reloc (info=info@entry=0x55f068a4fec0 <link_info>, sec=sec@entry=0x55f06dcfe4e8, gc_mark_hook=gc_mark_hook@entry=0x7f26fd3110e0 <_bfd_x86_elf_gc_mark_hook>, cookie=cookie@entry=0x7ffe635e1370) at /usr/src/debug/sys-devel/binutils-9999/binutils/bfd/elflink.c:13872 #14 0x00007f26fd35300a in _bfd_elf_gc_mark (info=info@entry=0x55f068a4fec0 <link_info>, sec=sec@entry=0x55f06dcfe4e8, gc_mark_hook=gc_mark_hook@entry=0x7f26fd3110e0 <_bfd_x86_elf_gc_mark_hook>) at /usr/src/debug/sys-devel/binutils-9999/binutils/bfd/elflink.c:13916 #15 0x00007f26fd352dc6 in _bfd_elf_gc_mark_reloc (info=info@entry=0x55f068a4fec0 <link_info>, sec=sec@entry=0x55f06b1b7618, gc_mark_hook=gc_mark_hook@entry=0x7f26fd3110e0 <_bfd_x86_elf_gc_mark_hook>, cookie=cookie@entry=0x7ffe635e1450) at /usr/src/debug/sys-devel/binutils-9999/binutils/bfd/elflink.c:13872 #16 0x00007f26fd35300a in _bfd_elf_gc_mark (info=info@entry=0x55f068a4fec0 <link_info>, sec=sec@entry=0x55f06b1b7618, gc_mark_hook=gc_mark_hook@entry=0x7f26fd3110e0 <_bfd_x86_elf_gc_mark_hook>) at /usr/src/debug/sys-devel/binutils-9999/binutils/bfd/elflink.c:13916 #17 0x00007f26fd352dc6 in _bfd_elf_gc_mark_reloc (info=info@entry=0x55f068a4fec0 <link_info>, sec=sec@entry=0x55f06b1c5078, gc_mark_hook=gc_mark_hook@entry=0x7f26fd3110e0 <_bfd_x86_elf_gc_mark_hook>, cookie=cookie@entry=0x7ffe635e1530) at /usr/src/debug/sys-devel/binutils-9999/binutils/bfd/elflink.c:13872 #18 0x00007f26fd35300a in _bfd_elf_gc_mark (info=info@entry=0x55f068a4fec0 <link_info>, sec=sec@entry=0x55f06b1c5078, gc_mark_hook=gc_mark_hook@entry=0x7f26fd3110e0 <_bfd_x86_elf_gc_mark_hook>) at /usr/src/debug/sys-devel/binutils-9999/binutils/bfd/elflink.c:13916 #19 0x00007f26fd352dc6 in _bfd_elf_gc_mark_reloc (info=info@entry=0x55f068a4fec0 <link_info>, sec=sec@entry=0x55f06b1cecf8, gc_mark_hook=gc_mark_hook@entry=0x7f26fd3110e0 <_bfd_x86_elf_gc_mark_hook>, cookie=cookie@entry=0x7ffe635e1610) at /usr/src/debug/sys-devel/binutils-9999/binutils/bfd/elflink.c:13872 --Type <RET> for more, q to quit, c to continue without paging-- #20 0x00007f26fd35300a in _bfd_elf_gc_mark (info=info@entry=0x55f068a4fec0 <link_info>, sec=sec@entry=0x55f06b1cecf8, gc_mark_hook=gc_mark_hook@entry=0x7f26fd3110e0 <_bfd_x86_elf_gc_mark_hook>) at /usr/src/debug/sys-devel/binutils-9999/binutils/bfd/elflink.c:13916 #21 0x00007f26fd352dc6 in _bfd_elf_gc_mark_reloc (info=info@entry=0x55f068a4fec0 <link_info>, sec=sec@entry=0x55f06b1ce378, gc_mark_hook=gc_mark_hook@entry=0x7f26fd3110e0 <_bfd_x86_elf_gc_mark_hook>, cookie=cookie@entry=0x7ffe635e16f0) at /usr/src/debug/sys-devel/binutils-9999/binutils/bfd/elflink.c:13872 #22 0x00007f26fd35300a in _bfd_elf_gc_mark (info=info@entry=0x55f068a4fec0 <link_info>, sec=sec@entry=0x55f06b1ce378, gc_mark_hook=gc_mark_hook@entry=0x7f26fd3110e0 <_bfd_x86_elf_gc_mark_hook>) at /usr/src/debug/sys-devel/binutils-9999/binutils/bfd/elflink.c:13916 #23 0x00007f26fd352dc6 in _bfd_elf_gc_mark_reloc (info=info@entry=0x55f068a4fec0 <link_info>, sec=sec@entry=0x55f06b1ce118, gc_mark_hook=gc_mark_hook@entry=0x7f26fd3110e0 <_bfd_x86_elf_gc_mark_hook>, cookie=cookie@entry=0x7ffe635e17d0) at /usr/src/debug/sys-devel/binutils-9999/binutils/bfd/elflink.c:13872 #24 0x00007f26fd35300a in _bfd_elf_gc_mark (info=info@entry=0x55f068a4fec0 <link_info>, sec=sec@entry=0x55f06b1ce118, gc_mark_hook=gc_mark_hook@entry=0x7f26fd3110e0 <_bfd_x86_elf_gc_mark_hook>) at /usr/src/debug/sys-devel/binutils-9999/binutils/bfd/elflink.c:13916 #25 0x00007f26fd352dc6 in _bfd_elf_gc_mark_reloc (info=info@entry=0x55f068a4fec0 <link_info>, sec=sec@entry=0x55f06b340fc8, gc_mark_hook=gc_mark_hook@entry=0x7f26fd3110e0 <_bfd_x86_elf_gc_mark_hook>, cookie=cookie@entry=0x7ffe635e18b0) at /usr/src/debug/sys-devel/binutils-9999/binutils/bfd/elflink.c:13872 #26 0x00007f26fd35300a in _bfd_elf_gc_mark (info=info@entry=0x55f068a4fec0 <link_info>, sec=sec@entry=0x55f06b340fc8, gc_mark_hook=gc_mark_hook@entry=0x7f26fd3110e0 <_bfd_x86_elf_gc_mark_hook>) at /usr/src/debug/sys-devel/binutils-9999/binutils/bfd/elflink.c:13916 #27 0x00007f26fd352dc6 in _bfd_elf_gc_mark_reloc (info=info@entry=0x55f068a4fec0 <link_info>, sec=sec@entry=0x55f06a860a28, gc_mark_hook=gc_mark_hook@entry=0x7f26fd3110e0 <_bfd_x86_elf_gc_mark_hook>, cookie=cookie@entry=0x7ffe635e1990) at /usr/src/debug/sys-devel/binutils-9999/binutils/bfd/elflink.c:13872 #28 0x00007f26fd35300a in _bfd_elf_gc_mark (info=info@entry=0x55f068a4fec0 <link_info>, sec=sec@entry=0x55f06a860a28, gc_mark_hook=gc_mark_hook@entry=0x7f26fd3110e0 <_bfd_x86_elf_gc_mark_hook>) at /usr/src/debug/sys-devel/binutils-9999/binutils/bfd/elflink.c:13916 #29 0x00007f26fd356b55 in bfd_elf_gc_sections (abfd=0x55f06a84b8a0, info=0x55f068a4fec0 <link_info>) at /usr/src/debug/sys-devel/binutils-9999/binutils/bfd/elflink.c:14483 #30 0x000055f0687cbb66 in lang_gc_sections () at /usr/src/debug/sys-devel/binutils-9999/binutils/ld/ldlang.c:7756 #31 lang_process () at /usr/src/debug/sys-devel/binutils-9999/binutils/ld/ldlang.c:8346 #32 0x000055f0688e6676 in main (argc=51, argv=<optimized out>) at /usr/src/debug/sys-devel/binutils-9999/binutils/ld/ldmain.c:511 ``` perf report says: ``` 51.41% ld libbfd-2.42.50.20240308.gentoo-sys-devel-binutils-mt.so [.] bfd_elf64_swap_symbol_in 11.99% ld libbfd-2.42.50.20240308.gentoo-sys-devel-binutils-mt.so [.] bfd_elf_get_elf_syms 5.85% ld libbfd-2.42.50.20240308.gentoo-sys-devel-binutils-mt.so [.] bfd_elf64_swap_reloca_in 4.44% ld [kernel.kallsyms] [k] clear_page_rep 2.94% ld libbfd-2.42.50.20240308.gentoo-sys-devel-binutils-mt.so [.] bfd_getl32 2.89% ld libbfd-2.42.50.20240308.gentoo-sys-devel-binutils-mt.so [.] bfd_getl64 1.71% ld libbfd-2.42.50.20240308.gentoo-sys-devel-binutils-mt.so [.] elf_link_read_relocs_from_section 1.22% ld [kernel.kallsyms] [k] page_remove_rmap 1.21% ld [kernel.kallsyms] [k] __rmqueue_pcplist 0.92% ld libbfd-2.42.50.20240308.gentoo-sys-devel-binutils-mt.so [.] bfd_getl16 0.89% ld [kernel.kallsyms] [k] stackleak_erase 0.74% ld [kernel.kallsyms] [k] __free_one_page 0.58% ld [kernel.kallsyms] [k] free_unref_page_prepare 0.56% ld [kernel.kallsyms] [k] native_irq_return_iret 0.47% ld [kernel.kallsyms] [k] __handle_mm_fault [...] ``` [0] https://inbox.sourceware.org/binutils/20240308152443.13534-1-hjl.tools@gmail.com/T/#t
Tarball is at PR31466.tar.xz">https://dev.gentoo.org/~sam/bugs/sourceware/31466/ld-hang-PR31466.tar.xz. Inside, run: ``` gcc -m64 symbols.o procs-bf23c8fa176eac21.procs.d1865a8196e0707d-cgu.0.rcgu.o -Wl,--as-needed -L deps -L lib -Wl,-Bstatic libcompiler_builtins-275fd5f39e82e176.rlib -Wl,-Bdynamic -lc -lgcc_s -lutil -lrt -lpthread -lm -ldl -lc -Wl,--eh-frame-hdr -Wl,-z,noexecstack -o procs-bf23c8fa176eac21 -Wl,--gc-sections -pie -Wl,-z,relro,-z,now -nodefaultlibs ``` It should hang.
With binutils-2.42, it takes about 1-2s.
As the backtrace hinted, if I drop -Wl,--gc-sections, it completes quickly.
It also happens on master with --no-keep-memory.
It doesn't hang. It just takes a LONG time: $ /bin/time ../ld.master --no-keep-memory `cat doit` 371.06user 180.83system 9:13.36elapsed 99%CPU (0avgtext+0avgdata 1185900maxresident)k 0inputs+50408outputs (0major+71941278minor)pagefaults 0swaps
Created attachment 15394 [details] Try this
That's looking good, thanks! I'll run it overnight on top of mmap (needed trivial rebasing) with the package builds. If needed, I can do a separate large build with -Wl,--no-keep-memory and just this patch tomorrow.
Created attachment 15395 [details] This patch is on top of the mmap patches
Perhaps it would be better to just drop the mmap change in _bfd_elf_link_keep_memory. I did wonder when reviewing that patch whether it was a good idea to discard things just based on whether mmap was enabled.
(In reply to Alan Modra from comment #9) > Perhaps it would be better to just drop the mmap change in > _bfd_elf_link_keep_memory. I did wonder when reviewing that patch whether > it was a good idea to discard things just based on whether mmap was enabled. I will add --keep-memory to always cache the symbol and relocation tables.
The master branch has been updated by H.J. Lu <hjl@sourceware.org>: https://sourceware.org/git/gitweb.cgi?p=binutils-gdb.git;h=3428c771aef0cebe229ac0f02a654d2ef78f2427 commit 3428c771aef0cebe229ac0f02a654d2ef78f2427 Author: H.J. Lu <hjl.tools@gmail.com> Date: Fri Mar 8 16:21:42 2024 -0800 elf: Always keep symbol table and relocation info for eh_frame When --no-keep-memory is used, the symbol table and relocation info for eh_frame are freed after they are retrieved for each text section in the input object. If an input object has many text sections, the same data is retrieved and freed many times which can take a very long time. Update _bfd_elf_gc_mark to keep the symbol table and relocation info for eh_frame to avoid it. Data to link the 3.5GB clang executable in LLVM 17 debug build on Linux/x86-64 with 32GB RAM is: before after improvement user 86.31 86.44 -0.2% system 8.77 8.63 1.6% total 95.58 96.81 -1.3% maximum set(GB) 13.1 13.1 0% page faults 3024752 3028699 -1.3% and data to link the 275M cc1plus executable in GCC 14 stage 1 build is: user 5.49 5.46 -0.5% system 0.73 0.73 0% total 6.26 6.25 0.3% maximum set(MB) 964 964 0% page faults 235173 235796 -0.3% The memory usage impact is minimum and the link time of the Rust binary in https://sourceware.org/bugzilla/show_bug.cgi?id=31466 is reduced from 500+ seconds to 1.44 seconds, a 300x speedup. PR ld/31466 * elflink.c (init_reloc_cookie): Add a bool argument, keep_memory, for keeping memory. Always keep memory if keep_memory is true. (init_reloc_cookie_rels): Likewise (init_reloc_cookie_for_section): Add a bool argument for keeping memory and pass it to init_reloc_cookie and init_reloc_cookie_rels. (_bfd_elf_gc_mark_reloc): Pass false to _bfd_elf_gc_mark. (_bfd_elf_gc_mark): Pass true to init_reloc_cookie_for_section for the eh_frame section. Pass false to init_reloc_cookie_for_section for other sections. (_bfd_elf_gc_mark_extra_sections): Add Add a bool argument for keeping memory and pass it to _bfd_elf_gc_mark. (bfd_elf_parse_eh_frame_entries): Pass false to init_reloc_cookie and init_reloc_cookie_rels. (bfd_elf_gc_sections): Pass false to init_reloc_cookie_for_section and _bfd_elf_gc_mark. (bfd_elf_discard_info): Pass false to init_reloc_cookie_for_section and init_reloc_cookie.
Fixed for 2.43.