How thread safe is dlopen?

[Toby Gray <toby dot gray at realvnc dot com>]

Hello,

I've come across an issue with my use of dlopen and pthreads in my code 
and glibc 2.19. Using valgrind I've tracked it down to the following issue:

==00:00:00:05.063 23870== Thread 30:
==00:00:00:05.063 23870== Invalid write of size 8
==00:00:00:05.063 23870==    at 0x04012d4c: _dl_allocate_tls_init 
(dl-tls.c:417)
==00:00:00:05.063 23870==    by 0x05043715: pthread_create@@GLIBC_2.2.5 
(allocatestack.c:252)
==00:00:00:05.063 23870==    by 0x00400e44: thread_function (within 
/home/tg/tmp/MOB-11244/a.out)
==00:00:00:05.063 23870==    by 0x05043181: start_thread 
(pthread_create.c:312)
==00:00:00:05.063 23870==    by 0x05353fbc: clone (clone.S:111)
==00:00:00:05.063 23870==    Address 0x64fbad0 is 0 bytes after a block 
of size 272 alloc'd
==00:00:00:05.063 23870==    at 0x04c2cc70: calloc (in 
/usr/lib/valgrind/vgpreload_memcheck-amd64-linux.so)
==00:00:00:05.063 23870==    by 0x04012e54: _dl_allocate_tls (dl-tls.c:296)
==00:00:00:05.063 23870==    by 0x05043da0: pthread_create@@GLIBC_2.2.5 
(allocatestack.c:589)
==00:00:00:05.063 23870==    by 0x00400e44: thread_function (within 
/home/tg/tmp/MOB-11244/a.out)
==00:00:00:05.063 23870==    by 0x05043181: start_thread 
(pthread_create.c:312)
==00:00:00:05.063 23870==    by 0x05353fbc: clone (clone.S:111)
==00:00:00:05.063 23870==

Which looks to be related to the size of data needed for TLS.

I'd originally thought that to be able to reproduce this issue in my 
code it was necessary to be creating threads with pthread_create() while 
another thread is calling dlopen(). However if that was the case then 
making sure that pthread_create() and dlopen() aren't called together 
would solve the issue, but that doesn't seem to help.

I've attached the source for my test program.

Is there something wrong with my expectations about multi-threaded use 
of dlopen and pthreads?

Regards,

Toby

/*
 * Build with:
 *
 *   gcc -pthread main.c -ldl && ./a.out /usr/lib/x86_64-linux-gnu
 */

#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <dlfcn.h>
#include <assert.h>
#include <sys/types.h>
#include <dirent.h>
#include <gnu/libc-version.h>

#define THREAD_COUNT 16
#define ITER_COUNT 1000

#define EXTENSION ".so"

pthread_mutex_t dl_mutex = PTHREAD_MUTEX_INITIALIZER;

pthread_mutex_t test_libs_mutex = PTHREAD_MUTEX_INITIALIZER;
char** test_libs = NULL;
size_t test_libs_count = 0;

void *dlopen_wrap(const char *filename, int flag)
{
  void * ret;
  pthread_mutex_lock(&dl_mutex);
  ret = dlopen(filename, flag);
  pthread_mutex_unlock(&dl_mutex);
  return ret;
}

int dlclose_wrap(void * handle)
{
  int ret;
  pthread_mutex_lock(&dl_mutex);
  ret = dlclose(handle);
  pthread_mutex_unlock(&dl_mutex);
  return ret;
}

int pthread_create_wrap(pthread_t *thread, const pthread_attr_t *attr,
                        void *(*start_routine) (void *), void *arg)
{
  int ret;
#ifdef LOCK_PTHREAD
  pthread_mutex_lock(&dl_mutex);
#endif
  ret = pthread_create(thread, attr, start_routine, arg);
#ifdef LOCK_PTHREAD
  pthread_mutex_unlock(&dl_mutex);
#endif
  return ret;
}

int pthread_join_wrap(pthread_t thread, void **retval)
{
  int ret;
  ret = pthread_join(thread, retval);
  return ret;
}

void load_test_libs(const char * dir_name)
{
  struct dirent entry, * result;
  DIR* dir = opendir(dir_name);
  assert(dir);
  do {
    assert(readdir_r(dir, &entry, &result) == 0);
    // Add the library to the list if it contains the extension
    if (result && strstr(result->d_name, EXTENSION) != 0) {
      char * name = strdup(result->d_name);
      assert(name);
      test_libs = (char**)
        realloc(test_libs, sizeof(char*) * (test_libs_count + 1));
      assert(test_libs);
      test_libs[test_libs_count] = name;
      ++test_libs_count;
    }
  } while (result);

  closedir(dir);
  printf("Found %zd libraries with " EXTENSION " ending in %s\n",
         test_libs_count, dir_name);
  assert(test_libs_count > 0);
}

const char * pick_lib_name()
{
  char * ret;
  pthread_mutex_lock(&test_libs_mutex);
  ret = test_libs[random() % test_libs_count];
  pthread_mutex_unlock(&test_libs_mutex);
  return ret;
}

void* null_thread_function(void* ctx)
{
  void * lib_handle = dlopen_wrap(pick_lib_name(), RTLD_NOW);
  if (lib_handle) dlclose_wrap(lib_handle);
  return NULL;
}

void* thread_function(void* ctx)
{
  int i;
  for (i = 0; i < ITER_COUNT; ++i)
  {
    pthread_t thread;
    void * lib_handle;
    assert(pthread_create_wrap(&thread, NULL, null_thread_function, NULL) == 0);

    lib_handle = dlopen_wrap(pick_lib_name(), RTLD_NOW);

    pthread_join_wrap(thread, NULL);

    if (lib_handle) dlclose_wrap(lib_handle);
  }
  return NULL;
}

int main(int argc, char* argv[])
{
  pthread_t threads[THREAD_COUNT];
  int i;

  if (argc < 2) {
    printf("Usage %s <library directory>\n", argv[0]);
    return 1;
  }
  printf("GNU libc version: %s\n", gnu_get_libc_version());
  load_test_libs(argv[1]);

  for (i = 0; i < THREAD_COUNT; ++i)
  {
    assert(pthread_create_wrap(&threads[i], NULL,
                          thread_function, NULL) == 0);
  }

  printf("All threads launched, waiting for termination\n");
  for (i = 0; i < THREAD_COUNT; ++i)
  {
    pthread_join_wrap(threads[i], NULL);
  }

  printf("All threads exited, quiting\n");
  return 0;
}