This is the mail archive of the systemtap@sourceware.org mailing list for the systemtap project.


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]
Other format: [Raw text]

Re: Controlling probe overhead


Josh,

I took your patch and updated it a small bit. Sorry for not respondingn sooner. See comments at the end.

Stone, Joshua I wrote:
Here's a simple patch to put a upper-bound on how much system time is
consumed by SystemTap.  It keeps a cumulative sum of time spent in
probes on each cpu, and if that exceeds the threshold within an
interval, the session is aborted.

Advantages:
* Accounting is maintained per-cpu.
* The frequency of get_cycles() is irrelevant to calculations, even if
it changes.
* The code is very simple. :)

Disadvantages:
* Doesn't account for overhead in firing the probe (e.g., the cost of a
debug trap).
* Doesn't detect excessive overhead from multiple scripts (e.g., ten
scripts that individually run under the threshold could together bear
down the system).

TODO:
* Share some code with STP_TIMING (which also tracks probe duration).

The patch now shares code with STP_TIMING.


* Make the threshold tunable (by gurus only?).
* Account for other runtime overhead (transport, timer sync, etc.).
* Make exceptions for some probes (begin, end)

Making exceptions for begin/end probe is going to be a bit difficult, since the common_probe_entryfn_prologue() and common_probe_entryfn_epilogue() functions don't really have any context of where they are called from.


Comments and suggestions are welcome...

Besides sharing code with STP_TIMING, I also added a command-line switch to turn this new functionality off (but your idea of tunable thresholds is probably better).


I do have a question. I'm afraid I don't quite understand the logic behind (and the differences between) STP_ACCOUNTING_THRESHOLD and STP_ACCOUNTING_INTERVAL. Why isn't STP_ACCOUNTING_THRESHOLD enough? Could you explain that a bit?

Thanks.

--
David Smith
dsmith@redhat.com
Red Hat
http://www.redhat.com
256.217.0141 (direct)
256.837.0057 (fax)
? stp_throttle.patch
? systemtap-0.5.13.tar.gz
Index: main.cxx
===================================================================
RCS file: /cvs/systemtap/src/main.cxx,v
retrieving revision 1.66
diff -u -p -r1.66 main.cxx
--- main.cxx	9 Feb 2007 13:45:49 -0000	1.66
+++ main.cxx	7 Mar 2007 18:48:39 -0000
@@ -100,6 +100,7 @@ usage (systemtap_session& s, int exitcod
     << endl
     << "   -x PID     sets target() to PID" << endl
     << "   -t         benchmarking timing information generated" << endl
+    << "   -T         turn off automatic probe throttling" << endl
     ;
   // -d: dump safety-related external references
 
@@ -196,6 +197,7 @@ main (int argc, char * const argv [])
   s.architecture = string (buf.machine);
   s.verbose = 0;
   s.timing = 0;
+  s.throttle = true;
   s.guru_mode = false;
   s.bulk_mode = false;
   s.unoptimized = false;
@@ -266,7 +268,7 @@ main (int argc, char * const argv [])
   while (true)
     {
       // NB: also see find_hash(), help(), switch stmt below, stap.1 man page
-      int grc = getopt (argc, argv, "hVMvtp:I:e:o:R:r:m:kgPc:x:D:bs:u");
+      int grc = getopt (argc, argv, "hVMvtp:I:e:o:R:r:m:kgPc:x:D:bs:uT");
       if (grc < 0)
         break;
       switch (grc)
@@ -287,6 +289,10 @@ main (int argc, char * const argv [])
 	  s.timing ++;
 	  break;
 
+        case 'T':
+	  s.throttle = false;
+	  break;
+
         case 'p':
           s.last_pass = atoi (optarg);
           if (s.last_pass < 1 || s.last_pass > 5)
Index: session.h
===================================================================
RCS file: /cvs/systemtap/src/session.h,v
retrieving revision 1.16
diff -u -p -r1.16 session.h
--- session.h	9 Feb 2007 13:45:49 -0000	1.16
+++ session.h	7 Mar 2007 18:48:39 -0000
@@ -95,6 +95,7 @@ struct systemtap_session
   unsigned perfmon;
   bool symtab;
   bool prologue_searching;
+  bool throttle;
 
   // Cache data
   bool use_cache;
Index: tapsets.cxx
===================================================================
RCS file: /cvs/systemtap/src/tapsets.cxx,v
retrieving revision 1.184
diff -u -p -r1.184 tapsets.cxx
--- tapsets.cxx	7 Mar 2007 15:45:23 -0000	1.184
+++ tapsets.cxx	7 Mar 2007 18:48:39 -0000
@@ -133,7 +133,7 @@ common_probe_entryfn_prologue (translato
   o->newline() << "struct context* __restrict__ c;";
   o->newline() << "unsigned long flags;";
 
-  o->newline() << "#ifdef STP_TIMING";
+  o->newline() << "#if defined(STP_TIMING) || defined(STP_THROTTLE)";
   // NB: we truncate cycles counts to 32 bits.  Perhaps it should be
   // fewer, if the hardware counter rolls over really quickly.  See
   // also ...epilogue().
@@ -194,7 +194,7 @@ common_probe_entryfn_prologue (translato
 void
 common_probe_entryfn_epilogue (translator_output* o)
 {
-  o->newline() << "#ifdef STP_TIMING";
+  o->newline() << "#if defined(STP_TIMING) || defined(STP_THROTTLE)";
   o->newline() << "{";
   o->newline(1) << "int32_t cycles_atend = (int32_t) get_cycles ();";
   // Handle 32-bit wraparound.
@@ -202,7 +202,23 @@ common_probe_entryfn_epilogue (translato
   o->newline(1) << "? (cycles_atend - cycles_atstart)";
   o->newline() << ": (~(int32_t)0) - cycles_atstart + cycles_atend + 1;";
 
+  o->newline() << "#ifdef STP_TIMING";
   o->newline() << "if (likely (c->statp)) _stp_stat_add(*c->statp, cycles_elapsed);";
+  o->newline() << "#endif";
+
+  o->newline() << "#ifdef STP_THROTTLE";
+  o->newline() << "c->cycles_sum += cycles_elapsed;";
+  o->newline() << "if (cycles_atend - c->cycles_base > STP_ACCOUNTING_INTERVAL) {";
+  o->newline(1) << "if (c->cycles_sum > STP_ACCOUNTING_THRESHOLD) {";
+  o->newline(1) << "_stp_error (\"probe overhead exceeded threshold\");";
+  o->newline() << "atomic_set (&session_state, STAP_SESSION_ERROR);";
+  o->newline(-1) << "}";
+
+  o->newline() << "c->cycles_base = cycles_atend;";
+  o->newline() << "c->cycles_sum = 0;";
+  o->newline(-1) << "}";
+  o->newline() << "#endif";
+
   o->indent(-1);
   o->newline(-1) << "}";
   o->newline() << "#endif";
Index: translate.cxx
===================================================================
RCS file: /cvs/systemtap/src/translate.cxx,v
retrieving revision 1.159
diff -u -p -r1.159 translate.cxx
--- translate.cxx	23 Feb 2007 15:30:15 -0000	1.159
+++ translate.cxx	7 Mar 2007 18:48:39 -0000
@@ -805,6 +805,9 @@ c_unparser::emit_common_header ()
   o->newline() << "atomic_t error_count = ATOMIC_INIT (0);";
   o->newline() << "atomic_t skipped_count = ATOMIC_INIT (0);";
   o->newline();
+  o->newline() << "#define STP_ACCOUNTING_INTERVAL 1000000000LL";
+  o->newline() << "#define STP_ACCOUNTING_THRESHOLD 500000000LL";
+  o->newline();
   o->newline() << "struct context {";
   o->newline(1) << "atomic_t busy;";
   o->newline() << "const char *probe_point;";
@@ -822,6 +825,10 @@ c_unparser::emit_common_header ()
   o->newline() << "#ifdef STP_TIMING";
   o->newline() << "Stat *statp;";
   o->newline() << "#endif";
+  o->newline() << "#ifdef STP_THROTTLE";
+  o->newline() << "cycles_t cycles_base;";
+  o->newline() << "cycles_t cycles_sum;";
+  o->newline() << "#endif";
   o->newline() << "union {";
   o->indent(1);
 
@@ -4091,7 +4098,10 @@ translate_pass (systemtap_session& s)
 	}
 
       if (s.timing)
-	s.op->newline() << "#define STP_TIMING" << " " << s.timing ;
+	s.op->newline() << "#define STP_TIMING" << " " << s.timing;
+
+      if (s.throttle)
+	s.op->newline() << "#define STP_THROTTLE" << " " << s.throttle;
 
       if (s.perfmon)
 	s.op->newline() << "#define STP_PERFMON";

Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]