A per-user or per-application ld.so.cache?

[Florian Weimer]

On 02/15/2016 07:30 PM, Ben Woodard wrote:
> I’ve been talking to the HPC tools and system guys and to my surprise they favor Florian’s approach which is to change glibc ld.so to cache the full directories of the visited in the process of finding a library. Subsequent lookups would first look in this cache before looking in subsequent directories in library search paths.

Thanks.

Before we start working on this, I would like to double-check that their
storage copes reasonably well with parallel readdir load.

Could you ask them to run the attached benchmark program on their
cluster, in a massively parallel fashion?  All the directories on a
typical library search path have to be listed as command line arguments
(separately, i.e. not joined as one argument and separated with colons).

The results will show if the directory listing overhead is acceptable.
It is unlikely that an ld.so implementation Median and maximum job
execution time should be sufficient, but the benchmark program produces
additional diagnostic output to identify specific bottlenecks.  For
example, if the file system reports a large block size, opendir may
allocate an equally large amount of memory.

Thanks,
Florian

-------------- next part --------------
/* Test program for measuring readdir speed.  */

#define _GNU_SOURCE
#include <dirent.h>
#include <errno.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/statvfs.h>
#include <sys/time.h>
#include <time.h>

static struct timeval
current (void)
{
  struct timeval tv;
  if (gettimeofday (&tv, NULL) != 0)
    {
      perror ("gettimeofday");
      abort ();
    }
  return tv;
}

static double
diff (const struct timeval a, const struct timeval b)
{
  double a_sec = a.tv_sec;
  double b_sec = b.tv_sec;
  return (a_sec - b_sec) + (a.tv_usec - b.tv_usec) * 1e-6;
}

static void
print_time (const struct timeval tv)
{
  struct tm tm;
  if (gmtime_r (&tv.tv_sec, &tm) == NULL)
    {
      perror ("gmtime_r");
      abort ();
    }
  printf ("%04d-%02d-%02dT%02d:%02d:%02d.%06d ",
          1900 + tm.tm_year,
          1 + tm.tm_mon,
          tm.tm_mday,
          tm.tm_hour,
          tm.tm_min,
          tm.tm_sec,
          (int) tv.tv_usec);
}

static void
list_directory (const char *path)
{
  struct timeval before = current ();
  print_time (before);
  printf ("%s: listing directory\n", path);

  DIR *dir = opendir (path);
  if (dir == NULL)
    {
      fprintf (stderr, "opendir (\"%s\"): %m", path);
      return;
    }
  {
    struct statvfs st;
    if (fstatvfs (dirfd (dir), &st) != 0)
      fprintf (stderr, "fstatvfs (\"%s\"): %m\n", path);
    else
      {
        print_time (current ());
        printf ("%s: file system block size: %lu\n", path, st.f_bsize);
      }
  }
  unsigned long long count = 0;
  while (true)
    {
      errno = 0;
      struct dirent64 *e = readdir64 (dir);
      if (e == NULL)
        {
          if (errno != 0)
            {
              perror ("readdir");
              closedir (dir);
              return;
            }
          else
            break;
        }
      ++count;
    }
  closedir (dir);
  struct timeval after = current ();
  print_time (after);
  printf ("%s: read %llu directory entries in %g seconds\n",
          path, count, diff (after, before));
}

int
main (int argc, char **argv)
{
  struct timeval before = current ();
  print_time (before);
  printf (" starting\n");
  ++argv;
  while (*argv)
    {
      list_directory (*argv);
      ++argv;
    }
  struct timeval after = current ();
  print_time (after);
  printf ("total: %g seconds\n", diff (after, before));
}