[PATCH] rtld: cache cpuid results on the stack for intel
Fabian Rast
fabian.rast@tum.de
Sun Feb 1 19:35:09 GMT 2026
Previously, the same cpuid leaves were queried multiple
times. Do the query once, and cache its result on the stack.
Signed-off-by: Fabian Rast <fabian.rast@tum.de>
---
sysdeps/x86/dl-cacheinfo.h | 112 +++++++++++++++++++++----------------
1 file changed, 65 insertions(+), 47 deletions(-)
diff --git a/sysdeps/x86/dl-cacheinfo.h b/sysdeps/x86/dl-cacheinfo.h
index b6520bddaa..32f6ef8007 100644
--- a/sysdeps/x86/dl-cacheinfo.h
+++ b/sysdeps/x86/dl-cacheinfo.h
@@ -108,6 +108,11 @@ static const struct intel_02_cache_info
#define nintel_02_known (sizeof (intel_02_known) / sizeof (intel_02_known [0]))
+struct intel_cpuid_cache {
+ char leaf2_valid, leaf4_valid;
+ unsigned int leaf2[4], leaf4[0x10][4];
+};
+
static int
intel_02_known_compare (const void *p1, const void *p2)
{
@@ -128,7 +133,8 @@ static long int
__attribute__ ((noinline))
intel_check_word (int name, unsigned int value, bool *has_level_2,
bool *no_level_2_or_3,
- const struct cpu_features *cpu_features)
+ const struct cpu_features *cpu_features,
+ struct intel_cpuid_cache *cache)
{
if ((value & 0x80000000) != 0)
/* The register value is reserved. */
@@ -162,7 +168,19 @@ intel_check_word (int name, unsigned int value, bool *has_level_2,
unsigned int round = 0;
while (1)
{
- __cpuid_count (4, round, eax, ebx, ecx, edx);
+ if (round < cache->leaf4_valid)
+ eax = cache->leaf4[round][0], ebx = cache->leaf4[round][1],
+ ecx = cache->leaf4[round][2], edx = cache->leaf4[round][3];
+ else if (round == cache->leaf4_valid
+ && round < sizeof(cache->leaf4)/sizeof(*cache->leaf4))
+ {
+ __cpuid_count (4, round, eax, ebx, ecx, edx);
+ cache->leaf4[round][0] = eax, cache->leaf4[round][1] = ebx;
+ cache->leaf4[round][2] = edx, cache->leaf4[round][3] = edx;
+ cache->leaf4_valid++;
+ }
+ else
+ __cpuid_count (4, round, eax, ebx, ecx, edx);
enum { null = 0, data = 1, inst = 2, uni = 3 } type = eax & 0x1f;
if (type == null)
@@ -258,7 +276,8 @@ intel_check_word (int name, unsigned int value, bool *has_level_2,
static long int __attribute__ ((noinline))
-handle_intel (int name, const struct cpu_features *cpu_features)
+handle_intel (int name, const struct cpu_features *cpu_features,
+ struct intel_cpuid_cache *cache)
{
unsigned int maxidx = cpu_features->basic.max_cpuid;
@@ -271,41 +290,33 @@ handle_intel (int name, const struct cpu_features *cpu_features)
long int result = 0;
bool no_level_2_or_3 = false;
bool has_level_2 = false;
- unsigned int eax;
- unsigned int ebx;
- unsigned int ecx;
- unsigned int edx;
- __cpuid (2, eax, ebx, ecx, edx);
+ int i;
+
+ if (!cache->leaf2_valid)
+ {
+ __cpuid (2, cache->leaf2[0], cache->leaf2[1],
+ cache->leaf2[2], cache->leaf2[3]);
+ cache->leaf2_valid = 1;
+ }
/* The low byte of EAX of CPUID leaf 2 should always return 1 and it
should be ignored. If it isn't 1, use CPUID leaf 4 instead. */
- if ((eax & 0xff) != 1)
+ if ((cache->leaf2[0] & 0xff) != 1)
return intel_check_word (name, 0xff, &has_level_2, &no_level_2_or_3,
- cpu_features);
- else
- {
- eax &= 0xffffff00;
+ cpu_features, cache);
- /* Process the individual registers' value. */
- result = intel_check_word (name, eax, &has_level_2,
- &no_level_2_or_3, cpu_features);
- if (result != 0)
- return result;
-
- result = intel_check_word (name, ebx, &has_level_2,
- &no_level_2_or_3, cpu_features);
- if (result != 0)
- return result;
+ /* Process all descriptors in leaf 2. */
+ result = intel_check_word (name, cache->leaf2[0]>>8, &has_level_2,
+ &no_level_2_or_3, cpu_features, cache);
+ if (result != 0)
+ return result;
- result = intel_check_word (name, ecx, &has_level_2,
- &no_level_2_or_3, cpu_features);
- if (result != 0)
- return result;
-
- result = intel_check_word (name, edx, &has_level_2,
- &no_level_2_or_3, cpu_features);
+ for (i = 1; i < 4; i++)
+ {
+ result = intel_check_word (name, cache->leaf2[i], &has_level_2,
+ &no_level_2_or_3, cpu_features, cache);
if (result != 0)
- return result;
+ return result;
}
if (name >= _SC_LEVEL2_CACHE_SIZE && name <= _SC_LEVEL3_CACHE_LINESIZE
@@ -622,7 +633,7 @@ handle_hygon (int name)
static void
get_common_cache_info (long int *shared_ptr, long int * shared_per_thread_ptr, unsigned int *threads_ptr,
- long int core)
+ long int core, struct intel_cpuid_cache *cache)
{
unsigned int eax;
unsigned int ebx;
@@ -680,7 +691,11 @@ get_common_cache_info (long int *shared_ptr, long int * shared_per_thread_ptr, u
int check = 0x1 | (threads_l3 == 0) << 1;
do
{
- __cpuid_count (4, i++, eax, ebx, ecx, edx);
+ if (cache && i < cache->leaf4_valid)
+ eax = cache->leaf4[i][0], ebx = cache->leaf4[i][1],
+ ecx = cache->leaf4[i][2], edx = cache->leaf4[i++][3];
+ else
+ __cpuid_count (4, i++, eax, ebx, ecx, edx);
/* There seems to be a bug in at least some Pentium Ds
which sometimes fail to iterate all cache parameters.
@@ -860,35 +875,38 @@ dl_init_cacheinfo (struct cpu_features *cpu_features)
if (cpu_features->basic.kind == arch_kind_intel)
{
- data = handle_intel (_SC_LEVEL1_DCACHE_SIZE, cpu_features);
- shared = handle_intel (_SC_LEVEL3_CACHE_SIZE, cpu_features);
+ struct intel_cpuid_cache cache;
+ cache.leaf2_valid = cache.leaf4_valid = 0;
+
+ data = handle_intel (_SC_LEVEL1_DCACHE_SIZE, cpu_features, &cache);
+ shared = handle_intel (_SC_LEVEL3_CACHE_SIZE, cpu_features, &cache);
shared_per_thread = shared;
level1_icache_size
- = handle_intel (_SC_LEVEL1_ICACHE_SIZE, cpu_features);
+ = handle_intel (_SC_LEVEL1_ICACHE_SIZE, cpu_features, &cache);
level1_icache_linesize
- = handle_intel (_SC_LEVEL1_ICACHE_LINESIZE, cpu_features);
+ = handle_intel (_SC_LEVEL1_ICACHE_LINESIZE, cpu_features, &cache);
level1_dcache_size = data;
level1_dcache_assoc
- = handle_intel (_SC_LEVEL1_DCACHE_ASSOC, cpu_features);
+ = handle_intel (_SC_LEVEL1_DCACHE_ASSOC, cpu_features, &cache);
level1_dcache_linesize
- = handle_intel (_SC_LEVEL1_DCACHE_LINESIZE, cpu_features);
+ = handle_intel (_SC_LEVEL1_DCACHE_LINESIZE, cpu_features, &cache);
level2_cache_size
- = handle_intel (_SC_LEVEL2_CACHE_SIZE, cpu_features);
+ = handle_intel (_SC_LEVEL2_CACHE_SIZE, cpu_features, &cache);
level2_cache_assoc
- = handle_intel (_SC_LEVEL2_CACHE_ASSOC, cpu_features);
+ = handle_intel (_SC_LEVEL2_CACHE_ASSOC, cpu_features, &cache);
level2_cache_linesize
- = handle_intel (_SC_LEVEL2_CACHE_LINESIZE, cpu_features);
+ = handle_intel (_SC_LEVEL2_CACHE_LINESIZE, cpu_features, &cache);
level3_cache_size = shared;
level3_cache_assoc
- = handle_intel (_SC_LEVEL3_CACHE_ASSOC, cpu_features);
+ = handle_intel (_SC_LEVEL3_CACHE_ASSOC, cpu_features, &cache);
level3_cache_linesize
- = handle_intel (_SC_LEVEL3_CACHE_LINESIZE, cpu_features);
+ = handle_intel (_SC_LEVEL3_CACHE_LINESIZE, cpu_features, &cache);
level4_cache_size
- = handle_intel (_SC_LEVEL4_CACHE_SIZE, cpu_features);
+ = handle_intel (_SC_LEVEL4_CACHE_SIZE, cpu_features, &cache);
get_common_cache_info (&shared, &shared_per_thread, &threads,
- level2_cache_size);
+ level2_cache_size, &cache);
}
else if (cpu_features->basic.kind == arch_kind_zhaoxin)
{
@@ -909,7 +927,7 @@ dl_init_cacheinfo (struct cpu_features *cpu_features)
level3_cache_linesize = handle_zhaoxin (_SC_LEVEL3_CACHE_LINESIZE);
get_common_cache_info (&shared, &shared_per_thread, &threads,
- level2_cache_size);
+ level2_cache_size, NULL);
}
else if (cpu_features->basic.kind == arch_kind_amd)
{
--
2.52.0
Hello,
the function `intel_check_word` was suprisingly prominent on a
specific processor when profiling the dynamic loader.
Apparently, cpuid queries are very expensive on this processor.
The cacheinfo detection code wastefully queries the same leaves
mulitple times. This patch introduces a cache on the stack
to cache these redundant cpuid queries while keeping the general
logic (repeated calls to `handle_intel`) intact.
Performance is measured using the following example program
that repeatedly executes itself to trigger dynamic loader startup
many times.
```
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
int main(int argc, char *argv[argc]) {
if (argc < 2) return 1;
unsigned long i = strtoul(argv[1], NULL, 10);
if (!i) return 0;
sprintf(argv[1], "%lu", i-1);
execv(argv[0], argv);
}
```
Test results for the two processors I am able to test on:
# Intel(R) Xeon(R) Gold 6430:
Benchmark 1: ./test_master 100
Time (mean ± σ): 30.2 ms ± 0.8 ms [User: 10.1 ms, System: 20.1 ms]
Range (min … max): 29.3 ms … 34.2 ms 101 runs
Benchmark 2: ./test_patch 100
Time (mean ± σ): 27.4 ms ± 0.4 ms [User: 7.1 ms, System: 20.3 ms]
Range (min … max): 26.7 ms … 28.8 ms 107 runs
Summary
./test_patch 100 ran
1.10 ± 0.03 times faster than ./test_master 100
using perf stat with 10000 repititions, running with --list on an empty program:
master -> cache cpuid
cycles: 516676 (0.05) -> 456290 (0.05) -11.69%
instructions: 618398 (0.01) -> 617579 (0.01) -0.13%
ref-cycles: 565544 (0.35) -> 490761 (0.35) -13.22%
duration_time: 582230 (0.3) -> 547735 (0.29) -5.92%
# Intel(R) Core(TM) i5-4300U:
Benchmark 1: ./test_master 100
Time (mean ± σ): 57.4 ms ± 2.0 ms [User: 13.1 ms, System: 43.2 ms]
Range (min … max): 54.8 ms … 62.4 ms 52 runs
Benchmark 2: ./test_patch 100
Time (mean ± σ): 57.2 ms ± 2.5 ms [User: 13.2 ms, System: 42.9 ms]
Range (min … max): 54.3 ms … 65.2 ms 53 runs
Summary
./test_patch 100 ran
1.00 ± 0.06 times faster than ./test_master 100
perf stat:
master -> cache cpuid
cycles: 556277 (0.05) -> 546479 (0.05) -1.76%
instructions: 429355 (0.02) -> 429038 (0.02) -0.07%
ref-cycles: 1068601 (0.14) -> 1051253 (0.14) -1.62%
duration_time: 1132718 (0.15) -> 1115771 (0.15) -1.5%
cpuid on the xeon processor seems particularly slow, while there is pretty
much no improvement on the i5-4300U.
I am left with a couple of questions:
Does anyone have experience with slow cpuid that could explain whats going on?
Can this be reproduced on other intel processors?
The man page for getauxval defines several tags regarding the same information
that is queried in dl_init_cacheinfo, but the loader does not try to read
them and my kernel does not set them. If almost every program wants
this information, why is getting it through the aux vector not the preferred
approach?
Any feedback on the patch or insight regarding my questions is much appreciated.
Cheers,
Fabian Rast
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 293 bytes
Desc: not available
URL: <https://sourceware.org/pipermail/libc-alpha/attachments/20260201/d7bf9123/attachment-0001.sig>
More information about the Libc-alpha
mailing list