Re: [Patch v2] Dryrun framework.

[OndÅej BÃlka <neleai at seznam dot cz>]

On Wed, Apr 10, 2013 at 03:12:53PM -0400, Carlos O'Donell wrote:
> On 04/09/2013 03:22 PM, OndÅej BÃlka wrote:
> > OK to commit?
> 
> Not yet. Look at my comments and post v2.
>
Here is v2 
> > Ondra
> > 
 	* dryrun/common/common.h: New file.
	* dryrun/common/record_common.h: New file.
 	* dryrun/common/replay_common.h: New file.
 	* dryrun/exp/Makefile: New file.
	* dryrun/exp/analyze.i: New file.
 	* dryrun/exp/layout.h: New file.
 	* dryrun/exp/record.c: New file.
 	* dryrun/exp/replay.c: New file.
 	* dryrun/exp/show.c: New file.
 	* dryrun/sysdeps/generic.h: New file.
 	* dryrun/sysdeps/x86_64.h: New file.

 > +
> > +/* TODO Placeholder function.  */
> > +/* We can use backtrace as it depends on libgcc which causes dlopen->backtrace->dlopen infinite loop.  */
> 
> Line too long.
> 
> If we don't have backtrace support remove it from the code.
>
I have it in x86, I added dryrun/sysdeps/x86_64.h file that I planed add
later. 
> > +static uint64_t __attribute__ ((noinline)) backtrace2 ()
> > +{
> > +  return 0;
> > +}
> > +
> > +/* TODO now only linux specific.  */
> 
> We shouldn't have any todo's, Just comment that it's linux specific.
> 
Its work in progress, best to see what I will do.

I also added placeholder to python bindings. I plan to generate them with swig.

---
 dryrun/common/common.h        |   44 +++++++++++++++++++++++
 dryrun/common/record_common.h |   75 ++++++++++++++++++++++++++++++++++++++++
 dryrun/common/replay_common.h |   42 ++++++++++++++++++++++
 dryrun/exp/Makefile           |    6 +++
 dryrun/exp/analyze.i          |   18 ++++++++++
 dryrun/exp/layout.h           |   22 ++++++++++++
 dryrun/exp/record.c           |   41 ++++++++++++++++++++++
 dryrun/exp/replay.c           |   29 +++++++++++++++
 dryrun/exp/show.c             |   27 ++++++++++++++
 dryrun/sysdeps/generic.h      |   76 +++++++++++++++++++++++++++++++++++++++++
 dryrun/sysdeps/x86_64.h       |   38 ++++++++++++++++++++
 11 files changed, 418 insertions(+), 0 deletions(-)
 create mode 100644 dryrun/common/common.h
 create mode 100644 dryrun/common/record_common.h
 create mode 100644 dryrun/common/replay_common.h
 create mode 100644 dryrun/exp/Makefile
 create mode 100644 dryrun/exp/analyze.i
 create mode 100644 dryrun/exp/layout.h
 create mode 100644 dryrun/exp/record.c
 create mode 100644 dryrun/exp/replay.c
 create mode 100644 dryrun/exp/show.c
 create mode 100644 dryrun/sysdeps/generic.h
 create mode 100644 dryrun/sysdeps/x86_64.h

diff --git a/dryrun/common/common.h b/dryrun/common/common.h
new file mode 100644
index 0000000..3469e06
--- /dev/null
+++ b/dryrun/common/common.h
@@ -0,0 +1,44 @@
+/* Copyright (C) 2013 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+#define BINARY_NAME_SIZE 48
+
+#include "../sysdeps/generic.h"
+
+
+typedef struct
+{
+  uint64_t time_start;
+  uint64_t time_end;
+  uint64_t backtrace;
+} common_s;
+
+#include "layout.h"
+
+typedef struct
+{
+  char binary_name[BINARY_NAME_SIZE];
+  uint64_t size, capacity;
+  uint64_t magic_constant; /* TODO unique for each compilation.  */
+  rec items[];
+} header;
+
+static header *h;
diff --git a/dryrun/common/record_common.h b/dryrun/common/record_common.h
new file mode 100644
index 0000000..6fd1e15
--- /dev/null
+++ b/dryrun/common/record_common.h
@@ -0,0 +1,75 @@
+/* Copyright (C) 2013 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include "common.h"
+
+static void init_fn();
+static const int initial_capacity = 1000;
+static const int record_limit = 1000000;
+
+__attribute__ ((constructor)) static void 
+__start_profile ()
+{
+  h = malloc (sizeof (header) + initial_capacity * sizeof (rec));
+  h->size = 0;
+  h->capacity = initial_capacity;
+  init_fn ();
+}
+
+
+__attribute__ ((destructor)) static void 
+__end_profile ()
+{
+  binary_name (h->binary_name);
+  char *fname = FNAME;
+  if (getenv ("DRYRUN_FILE"))
+    fname = getenv ("DRYRUN_FILE");
+
+  /* TODO We want limits on file size.  For each binary we should record
+     only first record_limit calls.  Then we only count number of calls.  */
+
+	/* TODO best representation to explain collected data is stochastic context
+     free grammar. This naturaly arises from call graph and constructing
+     grammar with probabilities obtained from profiler. 
+     Try hidden markov model method.  */
+
+  FILE* fi = fopen (fname, "a");
+  if (fi)
+    {
+      lock_file (fi);
+      fwrite (h, sizeof (header) + h->size * sizeof (rec), 1, fi);
+      fclose (fi);
+    }
+}
+
+rec*
+add_record ()
+{
+  if (h->size < record_limit) /* TODO account extra calls.  */
+    h->size++;
+  if (h->size == h->capacity)
+    {
+      h->capacity *= 2;
+      h = realloc (h, sizeof (header) + h->capacity * sizeof (rec));
+    }
+
+  rec* r = h->items + (h->size - 1);
+  r->common.time_start = get_ticks ();
+  r->common.backtrace = backtrace2 ();
+
+  return r;
+}
diff --git a/dryrun/common/replay_common.h b/dryrun/common/replay_common.h
new file mode 100644
index 0000000..fddde01
--- /dev/null
+++ b/dryrun/common/replay_common.h
@@ -0,0 +1,42 @@
+/* Copyright (C) 2013 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include "common.h"
+
+void replay (rec* r, int size);
+
+int
+main (int argc, char** argv)
+{
+  rec* buf;
+  char* fname = argv[1];
+  if (argc == 1)
+    fname = FNAME;
+  FILE* fi = fopen (fname, "r");
+  h = malloc (sizeof (header));
+  while (!feof (fi))
+    {
+      fread (h, sizeof (header), 1, fi);
+      buf = malloc (h->size * sizeof (rec));
+      fread (buf, h->size * sizeof (rec), 1, fi);
+      printf ("replaying %s\n", h->binary_name);
+
+      replay (buf, h->size);
+      free (buf);
+    }
+  return 0;
+}
diff --git a/dryrun/exp/Makefile b/dryrun/exp/Makefile
new file mode 100644
index 0000000..cabbc07
--- /dev/null
+++ b/dryrun/exp/Makefile
@@ -0,0 +1,6 @@
+all:
+	gcc -I. -Wall -std=gnu99 -pedantic  -I../common -g -DFNAME='"${CWD}/record.rec"' record.c -lm -lrt -ldl -shared -fPIC -o record.so
+	gcc -I. -Wall -std=gnu99 -pedantic -I../common -g -DFNAME='"${CWD}/record.rec"' replay.c -lm -lrt -o replay
+	gcc -I. -Wall -std=gnu99 -pedantic  -I../common -g -DFNAME='"${CWD}/record.rec"' show.c -lm -lrt -o show
+swig:
+	swig -python interface.i
diff --git a/dryrun/exp/analyze.i b/dryrun/exp/analyze.i
new file mode 100644
index 0000000..8dd22a7
--- /dev/null
+++ b/dryrun/exp/analyze.i
@@ -0,0 +1,18 @@
+/* Copyright (C) 2013 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+/* TODO wrap and access structure.  */
diff --git a/dryrun/exp/layout.h b/dryrun/exp/layout.h
new file mode 100644
index 0000000..303d10e
--- /dev/null
+++ b/dryrun/exp/layout.h
@@ -0,0 +1,22 @@
+/* Copyright (C) 2013 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+typedef struct
+{
+  common_s common;
+  double x;
+} rec;
diff --git a/dryrun/exp/record.c b/dryrun/exp/record.c
new file mode 100644
index 0000000..31b1f66
--- /dev/null
+++ b/dryrun/exp/record.c
@@ -0,0 +1,41 @@
+/* Copyright (C) 2013 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <dlfcn.h>
+#include "record_common.h"
+
+static void *libm_handle;
+static double (*fn) (double);
+
+static void
+init_fn ()
+{
+  libm_handle = dlopen ("libm.so", RTLD_NOW);
+  if (!libm_handle)
+    abort ();
+  fn = (double (**) (double)) dlsym (libm_handle, "exp");
+  if (!fn)
+    abort ();
+}
+
+double exp 
+(double x)
+{
+  rec *r = add_record ();
+  r->x = x;
+  return fn (x);
+}
diff --git a/dryrun/exp/replay.c b/dryrun/exp/replay.c
new file mode 100644
index 0000000..8afde6c
--- /dev/null
+++ b/dryrun/exp/replay.c
@@ -0,0 +1,29 @@
+/* Copyright (C) 2013 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include "replay_common.h"
+#include <math.h>
+
+void 
+replay (rec *r, int size)
+{
+  int i;
+  for (i = 0; i < size; i++)
+    {
+      exp (r[i].x);
+    }
+}
diff --git a/dryrun/exp/show.c b/dryrun/exp/show.c
new file mode 100644
index 0000000..eb823d1
--- /dev/null
+++ b/dryrun/exp/show.c
@@ -0,0 +1,27 @@
+/* Copyright (C) 2013 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include "replay_common.h"
+void 
+replay (rec *r, int size)
+{
+  int i;
+  for (i = 0; i < size; i++)
+    {
+      printf ("%lf\n", r[i].x);
+    }
+}
diff --git a/dryrun/sysdeps/generic.h b/dryrun/sysdeps/generic.h
new file mode 100644
index 0000000..698a38d
--- /dev/null
+++ b/dryrun/sysdeps/generic.h
@@ -0,0 +1,76 @@
+/* Copyright (C) 2013 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <stdint.h>
+#include <time.h>
+#include <dlfcn.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <fcntl.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <string.h>
+
+/* TODO Placeholder function.  */
+/* We can use backtrace as it depends on libgcc which causes 
+   dlopen->backtrace->dlopen infinite loop.  */
+
+static uint64_t __attribute__ ((noinline)) __attribute ((unused))
+backtrace2 ()
+{
+  return 0;
+}
+
+/* TODO now only linux specific.  */
+static void __attribute ((unused))
+binary_name (char *x)
+{
+  int i;
+  sprintf (x, "/proc/%i/cmdline", getpid ());
+  FILE *f = fopen (x, "r");
+  if (f)
+    fgets (x, BINARY_NAME_SIZE, f);
+
+  /* Do not leak command attributes.  */	
+  for (i = strlen (x) ; i < BINARY_NAME_SIZE ; i++)
+    x[i] = 0;
+
+  x[BINARY_NAME_SIZE - 1] = 0;
+}
+
+/* TODO add platform specific clocks that measure cycles.  */
+static uint64_t  __attribute__ ((noinline)) __attribute ((unused))
+get_ticks ()
+{
+  struct timespec ts;
+  if (clock_gettime (CLOCK_REALTIME, &ts) == 0)
+    return 0;
+  return 1000000000 * ts.tv_sec + ts.tv_nsec;
+}
+
+static void __attribute ((unused))
+lock_file (FILE *fi)
+{
+  struct flock fl;
+  int fd = fileno (fi);
+  fl.l_type = F_WRLCK;
+  fl.l_whence = SEEK_SET;
+  fl.l_start = 0;
+  fl.l_len = 0;
+  fl.l_pid = getpid ();
+  fcntl (fd, F_SETLKW, &fl);
+}
diff --git a/dryrun/sysdeps/x86_64.h b/dryrun/sysdeps/x86_64.h
new file mode 100644
index 0000000..b4415e4
--- /dev/null
+++ b/dryrun/sysdeps/x86_64.h
@@ -0,0 +1,38 @@
+/* Copyright (C) 2013 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+__attribute__((noinline)) 
+void *
+backtrace2() 
+  {
+    register void *ebp __asm__ ("ebp");
+    void **ptr = ebp;
+    if(!ptr) 
+      return NULL;
+    ptr = *ptr;
+    if(!ptr) 
+      return NULL;
+    return ptr[1]
+  }
+
+static uint64_t  __attribute__((noinline)) 
+get_ticks()
+  {
+    uint32_t lo, hi;
+    __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi));
+    return (uint64_t) hi << 32 | lo;
+  }
-- 
1.7.4.4