[PATCH] hugepages: cache THP mode and page size [BZ #34083]

[Shamil Abdulaev]

__get_thp_mode and __get_thp_size read /sys files and are used by
both the dynamic linker and malloc.  Cache their results in rtld_global
so the expensive THP probes are performed once per process.

Tested on x86_64-linux-gnu.  No new test failures.

Signed-off-by: Shamil Abdulaev <ashamil435@gmail.com>
---
 elf/dl-support.c                    |  6 +++
 sysdeps/generic/hugepages.h         |  7 ++++
 sysdeps/generic/ldsodefs.h          |  6 +++
 sysdeps/unix/sysv/linux/hugepages.c | 58 ++++++++++++++++++++++++-----
 4 files changed, 68 insertions(+), 9 deletions(-)

diff --git a/elf/dl-support.c b/elf/dl-support.c
index 0508d6113b..ec77cb3e7a 100644
--- a/elf/dl-support.c
+++ b/elf/dl-support.c
@@ -144,6 +144,12 @@ int _dl_inhibit_cache;
 /* All known directories in sorted order.  */
 struct r_search_path_elem *_dl_all_dirs;
 
+/* Cached Transparent Huge Page mode and page size.  */
+int _dl_thp_mode;
+unsigned long int _dl_thp_pagesize;
+int _dl_thp_mode_state;
+int _dl_thp_pagesize_state;
+
 /* All directories after startup.  */
 struct r_search_path_elem *_dl_init_all_dirs;
 
diff --git a/sysdeps/generic/hugepages.h b/sysdeps/generic/hugepages.h
index 5fc9b5c8de..68be2ee242 100644
--- a/sysdeps/generic/hugepages.h
+++ b/sysdeps/generic/hugepages.h
@@ -32,6 +32,13 @@ enum thp_mode_t
   thp_mode_not_supported
 };
 
+enum thp_cache_state_t
+{
+  thp_cache_uninitialized,
+  thp_cache_initializing,
+  thp_cache_initialized
+};
+
 enum thp_mode_t __get_thp_mode (void) attribute_hidden;
 
 /* Return the supported huge page size from the REQUESTED sizes on PAGESIZE
diff --git a/sysdeps/generic/ldsodefs.h b/sysdeps/generic/ldsodefs.h
index 15c4659853..4b2e337a81 100644
--- a/sysdeps/generic/ldsodefs.h
+++ b/sysdeps/generic/ldsodefs.h
@@ -395,6 +395,12 @@ struct rtld_global
   /* List of search directories.  */
   EXTERN struct r_search_path_elem *_dl_all_dirs;
 
+  /* Cached Transparent Huge Page mode and page size.  */
+  EXTERN int _dl_thp_mode;
+  EXTERN unsigned long int _dl_thp_pagesize;
+  EXTERN int _dl_thp_mode_state;
+  EXTERN int _dl_thp_pagesize_state;
+
   /* Get architecture specific definitions.  */
 #define PROCINFO_DECL
 #ifndef PROCINFO_CLASS
diff --git a/sysdeps/unix/sysv/linux/hugepages.c b/sysdeps/unix/sysv/linux/hugepages.c
index 46e05151a3..c32127f6a4 100644
--- a/sysdeps/unix/sysv/linux/hugepages.c
+++ b/sysdeps/unix/sysv/linux/hugepages.c
@@ -16,6 +16,8 @@
    License along with the GNU C Library; see the file COPYING.LIB.  If
    not, see <https://www.gnu.org/licenses/>.  */
 
+#include <atomic.h>
+#include <ldsodefs.h>
 #include <intprops.h>
 #include <dirent.h>
 #include <hugepages.h>
@@ -25,18 +27,32 @@
 unsigned long int
 __get_thp_size (void)
 {
+  if (atomic_load_acquire (&GL (dl_thp_pagesize_state))
+      == thp_cache_initialized)
+    return GL (dl_thp_pagesize);
+
+  int state = thp_cache_uninitialized;
+  if (!atomic_compare_exchange_acquire (&GL (dl_thp_pagesize_state),
+					&state, thp_cache_initializing))
+    {
+      while (atomic_load_acquire (&GL (dl_thp_pagesize_state))
+	     != thp_cache_initialized)
+	atomic_spin_nop ();
+      return GL (dl_thp_pagesize);
+    }
+
+  unsigned long int r = 0;
   int fd = __open64_nocancel (
     "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size", O_RDONLY);
   if (fd == -1)
-    return 0;
+    goto out;
 
   char str[INT_BUFSIZE_BOUND (unsigned long int)];
   ssize_t s = __read_nocancel (fd, str, sizeof (str));
   __close_nocancel (fd);
   if (s < 0)
-    return 0;
+    goto out;
 
-  unsigned long int r = 0;
   for (ssize_t i = 0; i < s; i++)
     {
       if (str[i] == '\n')
@@ -44,16 +60,36 @@ __get_thp_size (void)
       r *= 10;
       r += str[i] - '0';
     }
+
+out:
+  GL (dl_thp_pagesize) = r;
+  atomic_store_release (&GL (dl_thp_pagesize_state),
+			thp_cache_initialized);
   return r;
 }
 
 enum thp_mode_t
 __get_thp_mode (void)
 {
+  if (atomic_load_acquire (&GL (dl_thp_mode_state))
+      == thp_cache_initialized)
+    return (enum thp_mode_t) GL (dl_thp_mode);
+
+  int state = thp_cache_uninitialized;
+  if (!atomic_compare_exchange_acquire (&GL (dl_thp_mode_state),
+					&state, thp_cache_initializing))
+    {
+      while (atomic_load_acquire (&GL (dl_thp_mode_state))
+	     != thp_cache_initialized)
+	atomic_spin_nop ();
+      return (enum thp_mode_t) GL (dl_thp_mode);
+    }
+
+  enum thp_mode_t mode = thp_mode_not_supported;
   int fd = __open64_nocancel ("/sys/kernel/mm/transparent_hugepage/enabled",
 			      O_RDONLY);
   if (fd == -1)
-    return thp_mode_not_supported;
+    goto out;
 
   static const char mode_always[]  = "[always] madvise never\n";
   static const char mode_madvise[] = "always [madvise] never\n";
@@ -63,19 +99,23 @@ __get_thp_mode (void)
   ssize_t s = __read_nocancel (fd, str, sizeof (str));
   __close_nocancel (fd);
   if (s >= sizeof str || s < 0)
-    return thp_mode_not_supported;
+    goto out;
   str[s] = '\0';
 
   if (s == sizeof (mode_always) - 1)
     {
       if (strcmp (str, mode_always) == 0)
-	return thp_mode_always;
+	mode = thp_mode_always;
       else if (strcmp (str, mode_madvise) == 0)
-	return thp_mode_madvise;
+	mode = thp_mode_madvise;
       else if (strcmp (str, mode_never) == 0)
-	return thp_mode_never;
+	mode = thp_mode_never;
     }
-  return thp_mode_not_supported;
+
+out:
+  GL (dl_thp_mode) = mode;
+  atomic_store_release (&GL (dl_thp_mode_state), thp_cache_initialized);
+  return mode;
 }
 
 #if !IS_IN (rtld)
-- 
2.54.0