1 /* Thread-local storage handling in the ELF dynamic linker. Generic version.
2 Copyright (C) 2002-2021 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <https://www.gnu.org/licenses/>. */
25 #include <sys/param.h>
36 #define TUNABLE_NAMESPACE rtld
37 #include <dl-tunables.h>
39 /* Surplus static TLS, GLRO(dl_tls_static_surplus), is used for
41 - IE TLS in libc.so for all dlmopen namespaces except in the initial
42 one where libc.so is not loaded dynamically but at startup time,
43 - IE TLS in other libraries which may be dynamically loaded even in the
45 - and optionally for optimizing dynamic TLS access.
47 The maximum number of namespaces is DL_NNS, but to support that many
48 namespaces correctly the static TLS allocation should be significantly
49 increased, which may cause problems with small thread stacks due to the
50 way static TLS is accounted (bug 11787).
52 So there is a rtld.nns tunable limit on the number of supported namespaces
53 that affects the size of the static TLS and by default it's small enough
54 not to cause problems with existing applications. The limit is not
55 enforced or checked: it is the user's responsibility to increase rtld.nns
56 if more dlmopen namespaces are used.
58 Audit modules use their own namespaces, they are not included in rtld.nns,
59 but come on top when computing the number of namespaces. */
61 /* Size of initial-exec TLS in libc.so. This should be the maximum of
62 observed PT_GNU_TLS sizes across all architectures. Some
63 architectures have lower values due to differences in type sizes
64 and link editor capabilities. */
65 #define LIBC_IE_TLS 144
67 /* Size of initial-exec TLS in libraries other than libc.so.
68 This should be large enough to cover runtime libraries of the
69 compiler such as libgomp and libraries in libc other than libc.so. */
70 #define OTHER_IE_TLS 144
72 /* Default number of namespaces. */
75 /* Default for dl_tls_static_optional. */
76 #define OPTIONAL_TLS 512
78 /* Compute the static TLS surplus based on the namespace count and the
79 TLS space that can be used for optimizations. */
81 tls_static_surplus (int nns
, int opt_tls
)
83 return (nns
- 1) * LIBC_IE_TLS
+ nns
* OTHER_IE_TLS
+ opt_tls
;
86 /* This value is chosen so that with default values for the tunables,
87 the computation of dl_tls_static_surplus in
88 _dl_tls_static_surplus_init yields the historic value 1664, for
89 backwards compatibility. */
90 #define LEGACY_TLS (1664 - tls_static_surplus (DEFAULT_NNS, OPTIONAL_TLS))
92 /* Calculate the size of the static TLS surplus, when the given
93 number of audit modules are loaded. Must be called after the
94 number of audit modules is known and before static TLS allocation. */
96 _dl_tls_static_surplus_init (size_t naudit
)
101 nns
= TUNABLE_GET (nns
, size_t, NULL
);
102 opt_tls
= TUNABLE_GET (optional_static_tls
, size_t, NULL
);
104 /* Default values of the tunables. */
106 opt_tls
= OPTIONAL_TLS
;
110 if (DL_NNS
- nns
< naudit
)
111 _dl_fatal_printf ("Failed loading %lu audit modules, %lu are supported.\n",
112 (unsigned long) naudit
, (unsigned long) (DL_NNS
- nns
));
115 GL(dl_tls_static_optional
) = opt_tls
;
116 assert (LEGACY_TLS
>= 0);
117 GLRO(dl_tls_static_surplus
) = tls_static_surplus (nns
, opt_tls
) + LEGACY_TLS
;
120 /* Out-of-memory handler. */
122 __attribute__ ((__noreturn__
))
125 _dl_fatal_printf ("cannot allocate memory for thread-local data: ABORT\n");
130 _dl_assign_tls_modid (struct link_map
*l
)
134 if (__builtin_expect (GL(dl_tls_dtv_gaps
), false))
137 struct dtv_slotinfo_list
*runp
= GL(dl_tls_dtv_slotinfo_list
);
139 /* Note that this branch will never be executed during program
140 start since there are no gaps at that time. Therefore it
141 does not matter that the dl_tls_dtv_slotinfo is not allocated
142 yet when the function is called for the first times.
144 NB: the offset +1 is due to the fact that DTV[0] is used
145 for something else. */
146 result
= GL(dl_tls_static_nelem
) + 1;
147 if (result
<= GL(dl_tls_max_dtv_idx
))
150 while (result
- disp
< runp
->len
)
152 if (runp
->slotinfo
[result
- disp
].map
== NULL
)
156 assert (result
<= GL(dl_tls_max_dtv_idx
) + 1);
159 if (result
- disp
< runp
->len
)
161 /* Mark the entry as used, so any dependency see it. */
162 atomic_store_relaxed (&runp
->slotinfo
[result
- disp
].map
, l
);
168 while ((runp
= runp
->next
) != NULL
);
170 if (result
> GL(dl_tls_max_dtv_idx
))
172 /* The new index must indeed be exactly one higher than the
174 assert (result
== GL(dl_tls_max_dtv_idx
) + 1);
175 /* There is no gap anymore. */
176 GL(dl_tls_dtv_gaps
) = false;
183 /* No gaps, allocate a new entry. */
186 result
= GL(dl_tls_max_dtv_idx
) + 1;
187 /* Can be read concurrently. */
188 atomic_store_relaxed (&GL(dl_tls_max_dtv_idx
), result
);
191 l
->l_tls_modid
= result
;
196 _dl_count_modids (void)
198 /* The count is the max unless dlclose or failed dlopen created gaps. */
199 if (__glibc_likely (!GL(dl_tls_dtv_gaps
)))
200 return GL(dl_tls_max_dtv_idx
);
202 /* We have gaps and are forced to count the non-NULL entries. */
204 struct dtv_slotinfo_list
*runp
= GL(dl_tls_dtv_slotinfo_list
);
207 for (size_t i
= 0; i
< runp
->len
; ++i
)
208 if (runp
->slotinfo
[i
].map
!= NULL
)
220 _dl_determine_tlsoffset (void)
222 size_t max_align
= TCB_ALIGNMENT
;
224 size_t freebottom
= 0;
226 /* The first element of the dtv slot info list is allocated. */
227 assert (GL(dl_tls_dtv_slotinfo_list
) != NULL
);
228 /* There is at this point only one element in the
229 dl_tls_dtv_slotinfo_list list. */
230 assert (GL(dl_tls_dtv_slotinfo_list
)->next
== NULL
);
232 struct dtv_slotinfo
*slotinfo
= GL(dl_tls_dtv_slotinfo_list
)->slotinfo
;
234 /* Determining the offset of the various parts of the static TLS
235 block has several dependencies. In addition we have to work
236 around bugs in some toolchains.
238 Each TLS block from the objects available at link time has a size
239 and an alignment requirement. The GNU ld computes the alignment
240 requirements for the data at the positions *in the file*, though.
241 I.e, it is not simply possible to allocate a block with the size
242 of the TLS program header entry. The data is layed out assuming
243 that the first byte of the TLS block fulfills
245 p_vaddr mod p_align == &TLS_BLOCK mod p_align
247 This means we have to add artificial padding at the beginning of
248 the TLS block. These bytes are never used for the TLS data in
249 this module but the first byte allocated must be aligned
250 according to mod p_align == 0 so that the first byte of the TLS
251 block is aligned according to p_vaddr mod p_align. This is ugly
252 and the linker can help by computing the offsets in the TLS block
253 assuming the first byte of the TLS block is aligned according to
256 The extra space which might be allocated before the first byte of
257 the TLS block need not go unused. The code below tries to use
258 that memory for the next TLS block. This can work if the total
259 memory requirement for the next TLS block is smaller than the
263 /* We simply start with zero. */
266 for (size_t cnt
= 0; slotinfo
[cnt
].map
!= NULL
; ++cnt
)
268 assert (cnt
< GL(dl_tls_dtv_slotinfo_list
)->len
);
270 size_t firstbyte
= (-slotinfo
[cnt
].map
->l_tls_firstbyte_offset
271 & (slotinfo
[cnt
].map
->l_tls_align
- 1));
273 max_align
= MAX (max_align
, slotinfo
[cnt
].map
->l_tls_align
);
275 if (freebottom
- freetop
>= slotinfo
[cnt
].map
->l_tls_blocksize
)
277 off
= roundup (freetop
+ slotinfo
[cnt
].map
->l_tls_blocksize
278 - firstbyte
, slotinfo
[cnt
].map
->l_tls_align
)
280 if (off
<= freebottom
)
284 /* XXX For some architectures we perhaps should store the
286 slotinfo
[cnt
].map
->l_tls_offset
= off
;
291 off
= roundup (offset
+ slotinfo
[cnt
].map
->l_tls_blocksize
- firstbyte
,
292 slotinfo
[cnt
].map
->l_tls_align
) + firstbyte
;
293 if (off
> offset
+ slotinfo
[cnt
].map
->l_tls_blocksize
294 + (freebottom
- freetop
))
297 freebottom
= off
- slotinfo
[cnt
].map
->l_tls_blocksize
;
301 /* XXX For some architectures we perhaps should store the
303 slotinfo
[cnt
].map
->l_tls_offset
= off
;
306 GL(dl_tls_static_used
) = offset
;
307 GLRO (dl_tls_static_size
) = (roundup (offset
+ GLRO(dl_tls_static_surplus
),
311 /* The TLS blocks start right after the TCB. */
312 size_t offset
= TLS_TCB_SIZE
;
314 for (size_t cnt
= 0; slotinfo
[cnt
].map
!= NULL
; ++cnt
)
316 assert (cnt
< GL(dl_tls_dtv_slotinfo_list
)->len
);
318 size_t firstbyte
= (-slotinfo
[cnt
].map
->l_tls_firstbyte_offset
319 & (slotinfo
[cnt
].map
->l_tls_align
- 1));
321 max_align
= MAX (max_align
, slotinfo
[cnt
].map
->l_tls_align
);
323 if (slotinfo
[cnt
].map
->l_tls_blocksize
<= freetop
- freebottom
)
325 off
= roundup (freebottom
, slotinfo
[cnt
].map
->l_tls_align
);
326 if (off
- freebottom
< firstbyte
)
327 off
+= slotinfo
[cnt
].map
->l_tls_align
;
328 if (off
+ slotinfo
[cnt
].map
->l_tls_blocksize
- firstbyte
<= freetop
)
330 slotinfo
[cnt
].map
->l_tls_offset
= off
- firstbyte
;
331 freebottom
= (off
+ slotinfo
[cnt
].map
->l_tls_blocksize
337 off
= roundup (offset
, slotinfo
[cnt
].map
->l_tls_align
);
338 if (off
- offset
< firstbyte
)
339 off
+= slotinfo
[cnt
].map
->l_tls_align
;
341 slotinfo
[cnt
].map
->l_tls_offset
= off
- firstbyte
;
342 if (off
- firstbyte
- offset
> freetop
- freebottom
)
345 freetop
= off
- firstbyte
;
348 offset
= off
+ slotinfo
[cnt
].map
->l_tls_blocksize
- firstbyte
;
351 GL(dl_tls_static_used
) = offset
;
352 GLRO (dl_tls_static_size
) = roundup (offset
+ GLRO(dl_tls_static_surplus
),
355 # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
358 /* The alignment requirement for the static TLS block. */
359 GLRO (dl_tls_static_align
) = max_align
;
364 allocate_dtv (void *result
)
369 /* Relaxed MO, because the dtv size is later rechecked, not relied on. */
370 size_t max_modid
= atomic_load_relaxed (&GL(dl_tls_max_dtv_idx
));
371 /* We allocate a few more elements in the dtv than are needed for the
372 initial set of modules. This should avoid in most cases expansions
374 dtv_length
= max_modid
+ DTV_SURPLUS
;
375 dtv
= calloc (dtv_length
+ 2, sizeof (dtv_t
));
378 /* This is the initial length of the dtv. */
379 dtv
[0].counter
= dtv_length
;
381 /* The rest of the dtv (including the generation counter) is
382 Initialize with zero to indicate nothing there. */
384 /* Add the dtv to the thread data structures. */
385 INSTALL_DTV (result
, dtv
);
393 /* Get size and alignment requirements of the static TLS block. This
394 function is no longer used by glibc itself, but the GCC sanitizers
395 use it despite the GLIBC_PRIVATE status. */
397 _dl_get_tls_static_info (size_t *sizep
, size_t *alignp
)
399 *sizep
= GLRO (dl_tls_static_size
);
400 *alignp
= GLRO (dl_tls_static_align
);
403 /* Derive the location of the pointer to the start of the original
404 allocation (before alignment) from the pointer to the TCB. */
405 static inline void **
406 tcb_to_pointer_to_free_location (void *tcb
)
409 /* The TCB follows the TLS blocks, and the pointer to the front
411 void **original_pointer_location
= tcb
+ TLS_TCB_SIZE
;
413 /* The TCB comes first, preceded by the pre-TCB, and the pointer is
415 void **original_pointer_location
= tcb
- TLS_PRE_TCB_SIZE
- sizeof (void *);
417 return original_pointer_location
;
421 _dl_allocate_tls_storage (void)
424 size_t size
= GLRO (dl_tls_static_size
);
428 [ TLS_PRE_TCB_SIZE ] [ TLS_TCB_SIZE ] [ TLS blocks ]
429 ^ This should be returned. */
430 size
+= TLS_PRE_TCB_SIZE
;
433 /* Perform the allocation. Reserve space for the required alignment
434 and the pointer to the original allocation. */
435 size_t alignment
= GLRO (dl_tls_static_align
);
436 void *allocated
= malloc (size
+ alignment
+ sizeof (void *));
437 if (__glibc_unlikely (allocated
== NULL
))
440 /* Perform alignment and allocate the DTV. */
442 /* The TCB follows the TLS blocks, which determine the alignment.
443 (TCB alignment requirements have been taken into account when
444 calculating GLRO (dl_tls_static_align).) */
445 void *aligned
= (void *) roundup ((uintptr_t) allocated
, alignment
);
446 result
= aligned
+ size
- TLS_TCB_SIZE
;
448 /* Clear the TCB data structure. We can't ask the caller (i.e.
449 libpthread) to do it, because we will initialize the DTV et al. */
450 memset (result
, '\0', TLS_TCB_SIZE
);
452 /* Pre-TCB and TCB come before the TLS blocks. The layout computed
453 in _dl_determine_tlsoffset assumes that the TCB is aligned to the
454 TLS block alignment, and not just the TLS blocks after it. This
455 can leave an unused alignment gap between the TCB and the TLS
457 result
= (void *) roundup
458 (sizeof (void *) + TLS_PRE_TCB_SIZE
+ (uintptr_t) allocated
,
461 /* Clear the TCB data structure and TLS_PRE_TCB_SIZE bytes before
462 it. We can't ask the caller (i.e. libpthread) to do it, because
463 we will initialize the DTV et al. */
464 memset (result
- TLS_PRE_TCB_SIZE
, '\0', TLS_PRE_TCB_SIZE
+ TLS_TCB_SIZE
);
467 /* Record the value of the original pointer for later
469 *tcb_to_pointer_to_free_location (result
) = allocated
;
471 result
= allocate_dtv (result
);
479 extern dtv_t _dl_static_dtv
[];
480 # define _dl_initial_dtv (&_dl_static_dtv[1])
484 _dl_resize_dtv (dtv_t
*dtv
, size_t max_modid
)
486 /* Resize the dtv. */
488 size_t newsize
= max_modid
+ DTV_SURPLUS
;
489 size_t oldsize
= dtv
[-1].counter
;
491 if (dtv
== GL(dl_initial_dtv
))
493 /* This is the initial dtv that was either statically allocated in
494 __libc_setup_tls or allocated during rtld startup using the
495 dl-minimal.c malloc instead of the real malloc. We can't free
496 it, we have to abandon the old storage. */
498 newp
= malloc ((2 + newsize
) * sizeof (dtv_t
));
501 memcpy (newp
, &dtv
[-1], (2 + oldsize
) * sizeof (dtv_t
));
505 newp
= realloc (&dtv
[-1],
506 (2 + newsize
) * sizeof (dtv_t
));
511 newp
[0].counter
= newsize
;
513 /* Clear the newly allocated part. */
514 memset (newp
+ 2 + oldsize
, '\0',
515 (newsize
- oldsize
) * sizeof (dtv_t
));
517 /* Return the generation counter. */
523 _dl_allocate_tls_init (void *result
)
526 /* The memory allocation failed. */
529 dtv_t
*dtv
= GET_DTV (result
);
530 struct dtv_slotinfo_list
*listp
;
534 /* Protects global dynamic TLS related state. */
535 __rtld_lock_lock_recursive (GL(dl_load_tls_lock
));
537 /* Check if the current dtv is big enough. */
538 if (dtv
[-1].counter
< GL(dl_tls_max_dtv_idx
))
540 /* Resize the dtv. */
541 dtv
= _dl_resize_dtv (dtv
, GL(dl_tls_max_dtv_idx
));
543 /* Install this new dtv in the thread data structures. */
544 INSTALL_DTV (result
, &dtv
[-1]);
547 /* We have to prepare the dtv for all currently loaded modules using
548 TLS. For those which are dynamically loaded we add the values
549 indicating deferred allocation. */
550 listp
= GL(dl_tls_dtv_slotinfo_list
);
555 for (cnt
= total
== 0 ? 1 : 0; cnt
< listp
->len
; ++cnt
)
557 struct link_map
*map
;
560 /* Check for the total number of used slots. */
561 if (total
+ cnt
> GL(dl_tls_max_dtv_idx
))
564 map
= listp
->slotinfo
[cnt
].map
;
569 /* Keep track of the maximum generation number. This might
570 not be the generation counter. */
571 assert (listp
->slotinfo
[cnt
].gen
<= GL(dl_tls_generation
));
572 maxgen
= MAX (maxgen
, listp
->slotinfo
[cnt
].gen
);
574 dtv
[map
->l_tls_modid
].pointer
.val
= TLS_DTV_UNALLOCATED
;
575 dtv
[map
->l_tls_modid
].pointer
.to_free
= NULL
;
577 if (map
->l_tls_offset
== NO_TLS_OFFSET
578 || map
->l_tls_offset
== FORCED_DYNAMIC_TLS_OFFSET
)
581 assert (map
->l_tls_modid
== total
+ cnt
);
582 assert (map
->l_tls_blocksize
>= map
->l_tls_initimage_size
);
584 assert ((size_t) map
->l_tls_offset
>= map
->l_tls_blocksize
);
585 dest
= (char *) result
- map
->l_tls_offset
;
587 dest
= (char *) result
+ map
->l_tls_offset
;
589 # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
592 /* Set up the DTV entry. The simplified __tls_get_addr that
593 some platforms use in static programs requires it. */
594 dtv
[map
->l_tls_modid
].pointer
.val
= dest
;
596 /* Copy the initialization image and clear the BSS part. */
597 memset (__mempcpy (dest
, map
->l_tls_initimage
,
598 map
->l_tls_initimage_size
), '\0',
599 map
->l_tls_blocksize
- map
->l_tls_initimage_size
);
603 if (total
> GL(dl_tls_max_dtv_idx
))
607 assert (listp
!= NULL
);
609 __rtld_lock_unlock_recursive (GL(dl_load_tls_lock
));
611 /* The DTV version is up-to-date now. */
612 dtv
[0].counter
= maxgen
;
616 rtld_hidden_def (_dl_allocate_tls_init
)
619 _dl_allocate_tls (void *mem
)
621 return _dl_allocate_tls_init (mem
== NULL
622 ? _dl_allocate_tls_storage ()
623 : allocate_dtv (mem
));
625 rtld_hidden_def (_dl_allocate_tls
)
629 _dl_deallocate_tls (void *tcb
, bool dealloc_tcb
)
631 dtv_t
*dtv
= GET_DTV (tcb
);
633 /* We need to free the memory allocated for non-static TLS. */
634 for (size_t cnt
= 0; cnt
< dtv
[-1].counter
; ++cnt
)
635 free (dtv
[1 + cnt
].pointer
.to_free
);
637 /* The array starts with dtv[-1]. */
638 if (dtv
!= GL(dl_initial_dtv
))
642 free (*tcb_to_pointer_to_free_location (tcb
));
644 rtld_hidden_def (_dl_deallocate_tls
)
648 /* The __tls_get_addr function has two basic forms which differ in the
649 arguments. The IA-64 form takes two parameters, the module ID and
650 offset. The form used, among others, on IA-32 takes a reference to
651 a special structure which contain the same information. The second
652 form seems to be more often used (in the moment) so we default to
653 it. Users of the IA-64 form have to provide adequate definitions
654 of the following macros. */
655 # ifndef GET_ADDR_ARGS
656 # define GET_ADDR_ARGS tls_index *ti
657 # define GET_ADDR_PARAM ti
659 # ifndef GET_ADDR_MODULE
660 # define GET_ADDR_MODULE ti->ti_module
662 # ifndef GET_ADDR_OFFSET
663 # define GET_ADDR_OFFSET ti->ti_offset
666 /* Allocate one DTV entry. */
667 static struct dtv_pointer
668 allocate_dtv_entry (size_t alignment
, size_t size
)
670 if (powerof2 (alignment
) && alignment
<= _Alignof (max_align_t
))
672 /* The alignment is supported by malloc. */
673 void *ptr
= malloc (size
);
674 return (struct dtv_pointer
) { ptr
, ptr
};
677 /* Emulate memalign to by manually aligning a pointer returned by
678 malloc. First compute the size with an overflow check. */
679 size_t alloc_size
= size
+ alignment
;
680 if (alloc_size
< size
)
681 return (struct dtv_pointer
) {};
683 /* Perform the allocation. This is the pointer we need to free
685 void *start
= malloc (alloc_size
);
687 return (struct dtv_pointer
) {};
689 /* Find the aligned position within the larger allocation. */
690 void *aligned
= (void *) roundup ((uintptr_t) start
, alignment
);
692 return (struct dtv_pointer
) { .val
= aligned
, .to_free
= start
};
695 static struct dtv_pointer
696 allocate_and_init (struct link_map
*map
)
698 struct dtv_pointer result
= allocate_dtv_entry
699 (map
->l_tls_align
, map
->l_tls_blocksize
);
700 if (result
.val
== NULL
)
703 /* Initialize the memory. */
704 memset (__mempcpy (result
.val
, map
->l_tls_initimage
,
705 map
->l_tls_initimage_size
),
706 '\0', map
->l_tls_blocksize
- map
->l_tls_initimage_size
);
713 _dl_update_slotinfo (unsigned long int req_modid
)
715 struct link_map
*the_map
= NULL
;
716 dtv_t
*dtv
= THREAD_DTV ();
718 /* The global dl_tls_dtv_slotinfo array contains for each module
719 index the generation counter current when the entry was created.
720 This array never shrinks so that all module indices which were
721 valid at some time can be used to access it. Before the first
722 use of a new module index in this function the array was extended
723 appropriately. Access also does not have to be guarded against
724 modifications of the array. It is assumed that pointer-size
725 values can be read atomically even in SMP environments. It is
726 possible that other threads at the same time dynamically load
727 code and therefore add to the slotinfo list. This is a problem
728 since we must not pick up any information about incomplete work.
729 The solution to this is to ignore all dtv slots which were
730 created after the one we are currently interested. We know that
731 dynamic loading for this module is completed and this is the last
732 load operation we know finished. */
733 unsigned long int idx
= req_modid
;
734 struct dtv_slotinfo_list
*listp
= GL(dl_tls_dtv_slotinfo_list
);
736 while (idx
>= listp
->len
)
742 if (dtv
[0].counter
< listp
->slotinfo
[idx
].gen
)
744 /* CONCURRENCY NOTES:
746 Here the dtv needs to be updated to new_gen generation count.
748 This code may be called during TLS access when GL(dl_load_tls_lock)
749 is not held. In that case the user code has to synchronize with
750 dlopen and dlclose calls of relevant modules. A module m is
751 relevant if the generation of m <= new_gen and dlclose of m is
752 synchronized: a memory access here happens after the dlopen and
753 before the dlclose of relevant modules. The dtv entries for
754 relevant modules need to be updated, other entries can be
757 This e.g. means that the first part of the slotinfo list can be
758 accessed race free, but the tail may be concurrently extended.
759 Similarly relevant slotinfo entries can be read race free, but
760 other entries are racy. However updating a non-relevant dtv
761 entry does not affect correctness. For a relevant module m,
762 max_modid >= modid of m. */
763 size_t new_gen
= listp
->slotinfo
[idx
].gen
;
765 size_t max_modid
= atomic_load_relaxed (&GL(dl_tls_max_dtv_idx
));
766 assert (max_modid
>= req_modid
);
768 /* We have to look through the entire dtv slotinfo list. */
769 listp
= GL(dl_tls_dtv_slotinfo_list
);
772 for (size_t cnt
= total
== 0 ? 1 : 0; cnt
< listp
->len
; ++cnt
)
774 size_t modid
= total
+ cnt
;
776 /* Later entries are not relevant. */
777 if (modid
> max_modid
)
780 size_t gen
= atomic_load_relaxed (&listp
->slotinfo
[cnt
].gen
);
786 /* If the entry is older than the current dtv layout we
787 know we don't have to handle it. */
788 if (gen
<= dtv
[0].counter
)
791 /* If there is no map this means the entry is empty. */
793 = atomic_load_relaxed (&listp
->slotinfo
[cnt
].map
);
794 /* Check whether the current dtv array is large enough. */
795 if (dtv
[-1].counter
< modid
)
800 /* Resize the dtv. */
801 dtv
= _dl_resize_dtv (dtv
, max_modid
);
803 assert (modid
<= dtv
[-1].counter
);
805 /* Install this new dtv in the thread data
807 INSTALL_NEW_DTV (dtv
);
810 /* If there is currently memory allocate for this
811 dtv entry free it. */
812 /* XXX Ideally we will at some point create a memory
814 free (dtv
[modid
].pointer
.to_free
);
815 dtv
[modid
].pointer
.val
= TLS_DTV_UNALLOCATED
;
816 dtv
[modid
].pointer
.to_free
= NULL
;
818 if (modid
== req_modid
)
823 if (total
> max_modid
)
826 /* Synchronize with _dl_add_to_slotinfo. Ideally this would
827 be consume MO since we only need to order the accesses to
828 the next node after the read of the address and on most
829 hardware (other than alpha) a normal load would do that
830 because of the address dependency. */
831 listp
= atomic_load_acquire (&listp
->next
);
833 while (listp
!= NULL
);
835 /* This will be the new maximum generation counter. */
836 dtv
[0].counter
= new_gen
;
844 __attribute_noinline__
845 tls_get_addr_tail (GET_ADDR_ARGS
, dtv_t
*dtv
, struct link_map
*the_map
)
847 /* The allocation was deferred. Do it now. */
850 /* Find the link map for this module. */
851 size_t idx
= GET_ADDR_MODULE
;
852 struct dtv_slotinfo_list
*listp
= GL(dl_tls_dtv_slotinfo_list
);
854 while (idx
>= listp
->len
)
860 the_map
= listp
->slotinfo
[idx
].map
;
863 /* Make sure that, if a dlopen running in parallel forces the
864 variable into static storage, we'll wait until the address in the
865 static TLS block is set up, and use that. If we're undecided
866 yet, make sure we make the decision holding the lock as well. */
867 if (__glibc_unlikely (the_map
->l_tls_offset
868 != FORCED_DYNAMIC_TLS_OFFSET
))
870 __rtld_lock_lock_recursive (GL(dl_load_tls_lock
));
871 if (__glibc_likely (the_map
->l_tls_offset
== NO_TLS_OFFSET
))
873 the_map
->l_tls_offset
= FORCED_DYNAMIC_TLS_OFFSET
;
874 __rtld_lock_unlock_recursive (GL(dl_load_tls_lock
));
876 else if (__glibc_likely (the_map
->l_tls_offset
877 != FORCED_DYNAMIC_TLS_OFFSET
))
880 void *p
= (char *) THREAD_SELF
- the_map
->l_tls_offset
;
882 void *p
= (char *) THREAD_SELF
+ the_map
->l_tls_offset
+ TLS_PRE_TCB_SIZE
;
884 # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
886 __rtld_lock_unlock_recursive (GL(dl_load_tls_lock
));
888 dtv
[GET_ADDR_MODULE
].pointer
.to_free
= NULL
;
889 dtv
[GET_ADDR_MODULE
].pointer
.val
= p
;
891 return (char *) p
+ GET_ADDR_OFFSET
;
894 __rtld_lock_unlock_recursive (GL(dl_load_tls_lock
));
896 struct dtv_pointer result
= allocate_and_init (the_map
);
897 dtv
[GET_ADDR_MODULE
].pointer
= result
;
898 assert (result
.to_free
!= NULL
);
900 return (char *) result
.val
+ GET_ADDR_OFFSET
;
904 static struct link_map
*
905 __attribute_noinline__
906 update_get_addr (GET_ADDR_ARGS
)
908 struct link_map
*the_map
= _dl_update_slotinfo (GET_ADDR_MODULE
);
909 dtv_t
*dtv
= THREAD_DTV ();
911 void *p
= dtv
[GET_ADDR_MODULE
].pointer
.val
;
913 if (__glibc_unlikely (p
== TLS_DTV_UNALLOCATED
))
914 return tls_get_addr_tail (GET_ADDR_PARAM
, dtv
, the_map
);
916 return (void *) p
+ GET_ADDR_OFFSET
;
919 /* For all machines that have a non-macro version of __tls_get_addr, we
920 want to use rtld_hidden_proto/rtld_hidden_def in order to call the
921 internal alias for __tls_get_addr from ld.so. This avoids a PLT entry
922 in ld.so for __tls_get_addr. */
924 #ifndef __tls_get_addr
925 extern void * __tls_get_addr (GET_ADDR_ARGS
);
926 rtld_hidden_proto (__tls_get_addr
)
927 rtld_hidden_def (__tls_get_addr
)
930 /* The generic dynamic and local dynamic model cannot be used in
931 statically linked applications. */
933 __tls_get_addr (GET_ADDR_ARGS
)
935 dtv_t
*dtv
= THREAD_DTV ();
937 /* Update is needed if dtv[0].counter < the generation of the accessed
938 module. The global generation counter is used here as it is easier
939 to check. Synchronization for the relaxed MO access is guaranteed
940 by user code, see CONCURRENCY NOTES in _dl_update_slotinfo. */
941 size_t gen
= atomic_load_relaxed (&GL(dl_tls_generation
));
942 if (__glibc_unlikely (dtv
[0].counter
!= gen
))
943 return update_get_addr (GET_ADDR_PARAM
);
945 void *p
= dtv
[GET_ADDR_MODULE
].pointer
.val
;
947 if (__glibc_unlikely (p
== TLS_DTV_UNALLOCATED
))
948 return tls_get_addr_tail (GET_ADDR_PARAM
, dtv
, NULL
);
950 return (char *) p
+ GET_ADDR_OFFSET
;
955 /* Look up the module's TLS block as for __tls_get_addr,
956 but never touch anything. Return null if it's not allocated yet. */
958 _dl_tls_get_addr_soft (struct link_map
*l
)
960 if (__glibc_unlikely (l
->l_tls_modid
== 0))
961 /* This module has no TLS segment. */
964 dtv_t
*dtv
= THREAD_DTV ();
965 /* This may be called without holding the GL(dl_load_tls_lock). Reading
966 arbitrary gen value is fine since this is best effort code. */
967 size_t gen
= atomic_load_relaxed (&GL(dl_tls_generation
));
968 if (__glibc_unlikely (dtv
[0].counter
!= gen
))
970 /* This thread's DTV is not completely current,
971 but it might already cover this module. */
973 if (l
->l_tls_modid
>= dtv
[-1].counter
)
977 size_t idx
= l
->l_tls_modid
;
978 struct dtv_slotinfo_list
*listp
= GL(dl_tls_dtv_slotinfo_list
);
979 while (idx
>= listp
->len
)
985 /* We've reached the slot for this module.
986 If its generation counter is higher than the DTV's,
987 this thread does not know about this module yet. */
988 if (dtv
[0].counter
< listp
->slotinfo
[idx
].gen
)
992 void *data
= dtv
[l
->l_tls_modid
].pointer
.val
;
993 if (__glibc_unlikely (data
== TLS_DTV_UNALLOCATED
))
994 /* The DTV is current, but this thread has not yet needed
995 to allocate this module's segment. */
1003 _dl_add_to_slotinfo (struct link_map
*l
, bool do_add
)
1005 /* Now that we know the object is loaded successfully add
1006 modules containing TLS data to the dtv info table. We
1007 might have to increase its size. */
1008 struct dtv_slotinfo_list
*listp
;
1009 struct dtv_slotinfo_list
*prevp
;
1010 size_t idx
= l
->l_tls_modid
;
1012 /* Find the place in the dtv slotinfo list. */
1013 listp
= GL(dl_tls_dtv_slotinfo_list
);
1014 prevp
= NULL
; /* Needed to shut up gcc. */
1017 /* Does it fit in the array of this list element? */
1018 if (idx
< listp
->len
)
1022 listp
= listp
->next
;
1024 while (listp
!= NULL
);
1028 /* When we come here it means we have to add a new element
1029 to the slotinfo list. And the new module must be in
1033 listp
= (struct dtv_slotinfo_list
*)
1034 malloc (sizeof (struct dtv_slotinfo_list
)
1035 + TLS_SLOTINFO_SURPLUS
* sizeof (struct dtv_slotinfo
));
1038 /* We ran out of memory while resizing the dtv slotinfo list. */
1039 _dl_signal_error (ENOMEM
, "dlopen", NULL
, N_("\
1040 cannot create TLS data structures"));
1043 listp
->len
= TLS_SLOTINFO_SURPLUS
;
1045 memset (listp
->slotinfo
, '\0',
1046 TLS_SLOTINFO_SURPLUS
* sizeof (struct dtv_slotinfo
));
1047 /* Synchronize with _dl_update_slotinfo. */
1048 atomic_store_release (&prevp
->next
, listp
);
1051 /* Add the information into the slotinfo data structure. */
1054 /* Can be read concurrently. See _dl_update_slotinfo. */
1055 atomic_store_relaxed (&listp
->slotinfo
[idx
].map
, l
);
1056 atomic_store_relaxed (&listp
->slotinfo
[idx
].gen
,
1057 GL(dl_tls_generation
) + 1);
1062 static inline void __attribute__((always_inline
))
1063 init_one_static_tls (struct pthread
*curp
, struct link_map
*map
)
1066 void *dest
= (char *) curp
- map
->l_tls_offset
;
1067 # elif TLS_DTV_AT_TP
1068 void *dest
= (char *) curp
+ map
->l_tls_offset
+ TLS_PRE_TCB_SIZE
;
1070 # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
1073 /* Initialize the memory. */
1074 memset (__mempcpy (dest
, map
->l_tls_initimage
, map
->l_tls_initimage_size
),
1075 '\0', map
->l_tls_blocksize
- map
->l_tls_initimage_size
);
1079 _dl_init_static_tls (struct link_map
*map
)
1081 lll_lock (GL (dl_stack_cache_lock
), LLL_PRIVATE
);
1083 /* Iterate over the list with system-allocated threads first. */
1085 list_for_each (runp
, &GL (dl_stack_used
))
1086 init_one_static_tls (list_entry (runp
, struct pthread
, list
), map
);
1088 /* Now the list with threads using user-allocated stacks. */
1089 list_for_each (runp
, &GL (dl_stack_user
))
1090 init_one_static_tls (list_entry (runp
, struct pthread
, list
), map
);
1092 lll_unlock (GL (dl_stack_cache_lock
), LLL_PRIVATE
);
1094 #endif /* PTHREAD_IN_LIBC */