[patch] Reduce ridiculous stack use in _dl_map_object_deps

[ppluzhnikov at google dot com (Paul Pluzhnikov)]

Greetings,

When dlopen()ing a library with lots of NEEDED dependencies,
_dl_map_object_deps may alloca 250+ bytes for each such dependency, which
is a problem if thread stack is limited.

Attached test case, crashes on my system (Linux/x86_64) when dlopen()ing
a library with 220 dependencies using ToT glibc.

Attached patch makes the test pass with 1200+ such dependencies, by only
calling alloca if the previous alloca is not large enough.

Thanks,

--

Repro details:

gcc -pthread pthread-stack-size.c -ldl
./gen.pl 250
./a.out

enter thread
Segmentation fault


cat pthread-stack-size.c

#include <pthread.h>
#include <stdio.h>
#include <string.h>
#include <dlfcn.h>

void *thread_fn(void *p)
{
  void *h;
  printf("enter thread\n");
  h = dlopen("./foo.so", RTLD_LAZY);
  if (h == NULL) {
    fprintf(stderr, "%s\n", dlerror());
  }
  printf("exit thread\n");
  return p;
}

int main(int argc, char *argv[])
{
  pthread_t tid;
  pthread_attr_t attr;
  size_t required_stack = 1 << 16;  // 64K
  int rc;

  pthread_attr_init(&attr);
  rc = pthread_attr_setstacksize(&attr, required_stack);
  if (rc) {
    fprintf(stderr, "pthread_attr_setstacksize: %d (%s)\n", rc, strerror(rc));
    return rc;
  }
  rc = pthread_create(&tid, &attr, thread_fn, NULL);
  if (rc) {
    fprintf(stderr, "pthread_create: %d (%s)\n", rc, strerror(rc));
    return rc;
  }
  pthread_join(tid, NULL);
  return 0;
}


cat gen.pl

#!/usr/bin/perl

# Generate foo.so with N DT_NEEDED

$N = ($ARGV[0] or 1000);

$lib = "";
for $i (0 .. $N) {
  if (! -e "libfoo$i.so") {
    print "build libfoo$i\n";
    system("echo 'int foo$i() { return $i; }' | gcc -xc -shared -fPIC -o libfoo$i.so -");
  }
  $lib .= "-lfoo$i ";
}

print "Final link\n";
system("echo 'int foo() { return 0; }' | gcc -xc - -shared -fPIC -o foo.so -L. -Wl,-rpath=. $lib");



2011-10-29  Paul Pluzhnikov  <ppluzhnikov@google.com>

	* elf/dl-deps.c (_dl_map_object_deps): Reuse alloca space to reduce
	stack usage.


diff --git a/elf/dl-deps.c b/elf/dl-deps.c
index 95b1088..36ae80e 100644
--- a/elf/dl-deps.c
+++ b/elf/dl-deps.c
@@ -156,6 +156,8 @@ _dl_map_object_deps (struct link_map *map,
   int errno_reason;
   const char *errstring;
   const char *objname;
+  struct link_map *needed_space;
+  unsigned int n_needed_space;
 
   auto inline void preload (struct link_map *map);
 
@@ -175,6 +177,10 @@ _dl_map_object_deps (struct link_map *map,
   /* No loaded object so far.  */
   nlist = 0;
 
+  /* No alloca'd space yet.  */
+  needed_space = NULL;
+  n_needed_space = 0;
+
   /* First load MAP itself.  */
   preload (map);
 
@@ -216,8 +222,16 @@ _dl_map_object_deps (struct link_map *map,
 	 dependencies of this object.  */
       if (l->l_searchlist.r_list == NULL && l->l_initfini == NULL
 	  && l != map && l->l_ldnum > 0)
-	needed = (struct link_map **) alloca (l->l_ldnum
-					      * sizeof (struct link_map *));
+	{
+	  if (l->l_ldnum > n_needed_space)
+	    {
+	      n_needed_space = l->l_ldnum;
+	      needed_space
+		= (struct link_map **) alloca (n_needed_space
+					       * sizeof (struct link_map *));
+	    }
+	  needed = needed_space;
+	}
 
       if (l->l_info[DT_NEEDED] || l->l_info[AUXTAG] || l->l_info[FILTERTAG])
 	{