From 6109115cda35fe9ced82dcd638f8e73008b757f5 Mon Sep 17 00:00:00 2001
From: Josh Stone <jistone@redhat.com>
Date: Mon, 24 Sep 2012 16:15:55 -0700
Subject: [PATCH] stapdyn: Enable end/error probes

The dyninst exit hook runs too late for us to still call anything in the
mutatee, so the systemtap_module_exit() call which should run all of the
end/error probes wasn't happening.

Now we use a destructor function in the mutatee, so our exit path always
runs after main() returns or after an exit() call.  Functions like
_exit() are still problematic though.

This now also makes a distinction between initializing process-local vs.
session resources, so we are more ready for operating with multiple
mutatees at once.  See dyninst/runtime.h for design comments.
---
 runtime/dyninst/runtime.h | 58 +++++++++++++++++++++++++++++++++++----
 stapdyn/stapdyn.cxx       | 13 ++++-----
 2 files changed, 58 insertions(+), 13 deletions(-)

diff --git a/runtime/dyninst/runtime.h b/runtime/dyninst/runtime.h
index 43b6a9927..f66b6a59d 100644
--- a/runtime/dyninst/runtime.h
+++ b/runtime/dyninst/runtime.h
@@ -119,23 +119,71 @@ static unsigned long stap_hash_seed; /* Init during module startup */
 
 static int _stp_mem_fd = -1;
 
-int stp_dummy_init(void)
+
+/*
+ * For stapdyn to work in a multiprocess environment, the module must be
+ * prepared to be loaded multiple times in different processes.  Thus, we have
+ * to distinguish between process-level resource initialization and
+ * "session"-level (like probe begin/end/error).
+ *
+ * So stp_dyninst_ctor/dtor are the process-level functions, using gcc
+ * attributes to get called at the right time.
+ *
+ * The session-level resources have to be started by the stapdyn mutator, since
+ * only it knows which process is the "master" mutatee, so it can call
+ * stp_dyninst_session_init only in the right one.  That process will run the
+ * session exit in the dtor, since dyninst doesn't have a suitable exit hook.
+ * It may still be invoked manually from stapdyn for detaching though.
+ *
+ * The stp_dyninst_master is set as a PID, so it can be checked and made not to
+ * inherit across forks.
+ *
+ * XXX Once we have a shared-memory area (which will be necessary anyway for a
+ * multiprocess session to share globals), then a process refcount may be
+ * better for begin/end than this "master" designation.
+ *
+ * XXX Functions like _exit() which bypass destructors are a problem...
+ */
+
+static pid_t stp_dyninst_master = 0;
+
+__attribute__((constructor))
+static void stp_dyninst_ctor(void)
 {
     stap_hash_seed = _stp_random_u ((unsigned long)-1);
     _stp_mem_fd = open("/proc/self/mem", O_RDWR /*| O_LARGEFILE*/);
+}
+
+int stp_dyninst_session_init(void)
+{
+    /* We don't have a chance to indicate errors in the ctor, so do it here. */
     if (_stp_mem_fd < 0) {
 	return -errno;
     }
-    return systemtap_module_init();
+
+    int rc = systemtap_module_init();
+    if (rc == 0) {
+	stp_dyninst_master = getpid();
+    }
+    return rc;
 }
 
-int stp_dummy_exit(void)
+void stp_dyninst_session_exit(void)
 {
-    systemtap_module_exit();
+    if (stp_dyninst_master == getpid()) {
+	systemtap_module_exit();
+	stp_dyninst_master = 0;
+    }
+}
+
+__attribute__((destructor))
+static void stp_dyninst_dtor(void)
+{
+    stp_dyninst_session_exit();
+
     if (_stp_mem_fd != -1) {
 	close (_stp_mem_fd);
     }
-    return 0;
 }
 
 #endif /* _STAPDYN_RUNTIME_H_ */
diff --git a/stapdyn/stapdyn.cxx b/stapdyn/stapdyn.cxx
index 6964f681d..9714222ad 100644
--- a/stapdyn/stapdyn.cxx
+++ b/stapdyn/stapdyn.cxx
@@ -59,12 +59,6 @@ call_inferior_function(BPatch_process *app, const string& name)
     }
 }
 
-static void
-exit_hook(BPatch_thread *thread, BPatch_exitType)
-{
-  call_inferior_function(thread->getProcess(), "stp_dummy_exit");
-}
-
 // XXX: copied from runtime/dyninst/uprobes-dyninst.c
 struct stapdu_target {
 	char filename[240];
@@ -248,13 +242,16 @@ main(int argc, char * const argv[])
 
   find_uprobes(app, module);
 
-  call_inferior_function(app, "stp_dummy_init");
-  patch.registerExitCallback(exit_hook);
+  call_inferior_function(app, "stp_dyninst_session_init");
 
   app->continueExecution();
   while (!app->isTerminated())
     patch.waitForStatusChange();
 
+  /* When we get process detaching (rather than just exiting), need:
+   *   call_inferior_function(app, "stp_dyninst_session_exit");
+   */
+
   return 0;
 }
 
-- 
2.43.5