From 2aaf9213218689757a1e215c64e4e8fd3eefe739 Mon Sep 17 00:00:00 2001 From: Noah Sanci Date: Fri, 8 Apr 2022 11:12:31 -0400 Subject: [PATCH] add stap-profile-annotate tool New tool to profile a process or userspace generally, then produce a hit-counted annotated version of all the relevant sources. Downloading all the debuginfo & source files requires a working debuginfod-find with a set $DEBUGINFOD_URLS. Includes tests and man page. Signed-off-by: Noah Sanci Signed-off-by: Frank Ch. Eigler --- Makefile.am | 2 +- Makefile.in | 7 +- NEWS | 4 + configure | 4 + configure.ac | 1 + doc/Makefile.in | 2 +- httpd/Makefile.in | 2 +- java/Makefile.in | 2 +- man/stap-profile-annotate.1.in | 201 +++++++++++ python/Makefile.in | 2 +- stap-profile-annotate.in | 334 ++++++++++++++++++ systemtap.spec | 1 + tapset/context.stp | 34 ++ testsuite/systemtap.apps/profile-annotate.exp | 114 ++++++ 14 files changed, 703 insertions(+), 7 deletions(-) create mode 100644 man/stap-profile-annotate.1.in create mode 100755 stap-profile-annotate.in create mode 100644 testsuite/systemtap.apps/profile-annotate.exp diff --git a/Makefile.am b/Makefile.am index 85a0821a3..1f100cea5 100644 --- a/Makefile.am +++ b/Makefile.am @@ -20,7 +20,7 @@ AM_CPPFLAGS = -DBINDIR='"$(bindir)"' \ AM_CFLAGS = -D_GNU_SOURCE -fexceptions -Wall -Wextra -Werror -Wunused -Wformat=2 -W AM_CXXFLAGS = -Wall -Wextra -Werror -bin_SCRIPTS = stap-report +bin_SCRIPTS = stap-report stap-profile-annotate pkglibexec_SCRIPTS = stap-env oldinclude_HEADERS = includes/sys/sdt.h includes/sys/sdt-config.h diff --git a/Makefile.in b/Makefile.in index 68bebc0d2..c0b223b83 100644 --- a/Makefile.in +++ b/Makefile.in @@ -151,7 +151,7 @@ CONFIG_CLEAN_FILES = includes/sys/sdt-config.h \ initscript/99stap/check run-stap dtrace \ java/org/systemtap/byteman/helper/HelperSDT.java \ staprun/guest/stapshd staprun/guest/stapsh-daemon \ - staprun/guest/stapsh@.service + staprun/guest/stapsh@.service stap-profile-annotate CONFIG_CLEAN_VPATH_FILES = @BUILD_TRANSLATOR_TRUE@am__EXEEXT_1 = stap$(EXEEXT) @BUILD_TRANSLATOR_TRUE@@BUILD_VIRT_TRUE@am__EXEEXT_2 = \ @@ -676,7 +676,8 @@ AM_CPPFLAGS = -DBINDIR='"$(bindir)"' \ AM_CFLAGS = -D_GNU_SOURCE -fexceptions -Wall -Wextra -Werror -Wunused -Wformat=2 -W AM_CXXFLAGS = -Wall -Wextra -Werror -bin_SCRIPTS = stap-report $(am__append_2) $(am__append_5) +bin_SCRIPTS = stap-report stap-profile-annotate $(am__append_2) \ + $(am__append_5) pkglibexec_SCRIPTS = stap-env $(am__append_6) oldinclude_HEADERS = includes/sys/sdt.h includes/sys/sdt-config.h @BUILD_TRANSLATOR_TRUE@stap_SOURCES = main.cxx session.cxx parse.cxx \ @@ -876,6 +877,8 @@ staprun/guest/stapsh-daemon: $(top_builddir)/config.status $(top_srcdir)/staprun cd $(top_builddir) && $(SHELL) ./config.status $@ staprun/guest/stapsh@.service: $(top_builddir)/config.status $(top_srcdir)/staprun/guest/stapsh@.service.in cd $(top_builddir) && $(SHELL) ./config.status $@ +stap-profile-annotate: $(top_builddir)/config.status $(srcdir)/stap-profile-annotate.in + cd $(top_builddir) && $(SHELL) ./config.status $@ install-binPROGRAMS: $(bin_PROGRAMS) @$(NORMAL_INSTALL) @list='$(bin_PROGRAMS)'; test -n "$(bindir)" || list=; \ diff --git a/NEWS b/NEWS index 8957e83e0..6d0691524 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,9 @@ * What's new in version 4.7, PRERELEASE +- Includes new tool stap-profile-annotate, combining systemtap & + debuginfod to collect system-wide profiling statistics, and + produce annotated source files for all relevant programs/libraries. + - target processes given with the -c/-x parameters are now always added to the -d list for possible symbol/unwind data extraction, for simplifying profiling invocations. Consider --ldd. diff --git a/configure b/configure index 6ac022ec4..5cd9d3577 100755 --- a/configure +++ b/configure @@ -13478,6 +13478,8 @@ ac_config_files="$ac_config_files staprun/guest/stapsh@.service" ac_config_files="$ac_config_files stap-exporter/Makefile" +ac_config_files="$ac_config_files stap-profile-annotate" + # Setup "shadow" directory doc/beginners that has the basic directories setup for @@ -14383,6 +14385,7 @@ do "staprun/guest/stapsh-daemon") CONFIG_FILES="$CONFIG_FILES staprun/guest/stapsh-daemon" ;; "staprun/guest/stapsh@.service") CONFIG_FILES="$CONFIG_FILES staprun/guest/stapsh@.service" ;; "stap-exporter/Makefile") CONFIG_FILES="$CONFIG_FILES stap-exporter/Makefile" ;; + "stap-profile-annotate") CONFIG_FILES="$CONFIG_FILES stap-profile-annotate" ;; "doc/beginners") CONFIG_COMMANDS="$CONFIG_COMMANDS doc/beginners" ;; *) as_fn_error $? "invalid argument: \`$ac_config_target'" "$LINENO" 5;; @@ -15195,6 +15198,7 @@ See \`config.log' for more details" "$LINENO" 5; } "staprun/run-staprun":F) chmod +x staprun/run-staprun ;; "staprun/guest/stapshd":F) chmod +x staprun/guest/stapshd ;; "staprun/guest/stapsh-daemon":F) chmod +x staprun/guest/stapsh-daemon ;; + "stap-profile-annotate":F) chmod +x stap-profile-annotate ;; "doc/beginners":C) rm -f $ac_abs_top_builddir/doc/beginners/en-US $ac_abs_top_builddir/doc/beginners/build/en-US/testsuite && mkdir -p $ac_abs_top_builddir/doc/beginners/build/en-US && ln -s $ac_abs_top_srcdir/doc/SystemTap_Beginners_Guide/en-US $ac_abs_top_builddir/doc/beginners/en-US && ln -s $ac_abs_top_srcdir/testsuite $ac_abs_top_builddir/doc/beginners/build/en-US/testsuite ;; esac diff --git a/configure.ac b/configure.ac index 895f50695..6b3b399ef 100644 --- a/configure.ac +++ b/configure.ac @@ -981,6 +981,7 @@ AC_CONFIG_FILES([staprun/guest/stapshd], [chmod +x staprun/guest/stapshd]) AC_CONFIG_FILES([staprun/guest/stapsh-daemon], [chmod +x staprun/guest/stapsh-daemon]) AC_CONFIG_FILES([staprun/guest/stapsh@.service]) AC_CONFIG_FILES(stap-exporter/Makefile) +AC_CONFIG_FILES([stap-profile-annotate], [chmod +x stap-profile-annotate]) dnl AC_CONFIG_FILES([macros.systemtap]) dnl ^^^ not that one, because we want to expand $vars etc. to fqdn's, diff --git a/doc/Makefile.in b/doc/Makefile.in index dc8a9a284..c50719f9d 100644 --- a/doc/Makefile.in +++ b/doc/Makefile.in @@ -542,8 +542,8 @@ distclean-generic: maintainer-clean-generic: @echo "This command is intended for maintainers to use" @echo "it deletes files that may require special tools to rebuild." -@BUILD_DOCS_FALSE@clean-local: @BUILD_DOCS_FALSE@uninstall-local: +@BUILD_DOCS_FALSE@clean-local: @BUILD_DOCS_FALSE@install-data-hook: clean: clean-recursive diff --git a/httpd/Makefile.in b/httpd/Makefile.in index 2ba877c0b..ac4e7d43f 100644 --- a/httpd/Makefile.in +++ b/httpd/Makefile.in @@ -937,8 +937,8 @@ maintainer-clean-generic: @echo "This command is intended for maintainers to use" @echo "it deletes files that may require special tools to rebuild." -test -z "$(BUILT_SOURCES)" || rm -f $(BUILT_SOURCES) -@HAVE_HTTP_SUPPORT_FALSE@uninstall-local: @HAVE_HTTP_SUPPORT_FALSE@install-data-local: +@HAVE_HTTP_SUPPORT_FALSE@uninstall-local: clean: clean-recursive clean-am: clean-generic clean-pkglibexecPROGRAMS mostlyclean-am diff --git a/java/Makefile.in b/java/Makefile.in index d36aac3de..28ef8e29d 100644 --- a/java/Makefile.in +++ b/java/Makefile.in @@ -686,8 +686,8 @@ maintainer-clean-generic: @echo "This command is intended for maintainers to use" @echo "it deletes files that may require special tools to rebuild." -test -z "$(BUILT_SOURCES)" || rm -f $(BUILT_SOURCES) -@HAVE_JAVA_FALSE@install-exec-local: @HAVE_JAVA_FALSE@install-data-local: +@HAVE_JAVA_FALSE@install-exec-local: @HAVE_JAVA_FALSE@uninstall-local: clean: clean-am diff --git a/man/stap-profile-annotate.1.in b/man/stap-profile-annotate.1.in new file mode 100644 index 000000000..48ccbb15a --- /dev/null +++ b/man/stap-profile-annotate.1.in @@ -0,0 +1,201 @@ +.TH STAP-PROFILE-ANNOTATE 1 +.SH NAME +stap-profile-annotate \- Annotate source files of running programs. + +.\" macros +.\" do not nest SAMPLEs +.de SAMPLE +.br + +.nr oldin \\n(.i +.nf +.nh +.. +.de ESAMPLE +.hy +.fi +.in \\n[oldin]u + +.. + +.SH SYNOPSIS + +.br +.B strace-profile-annotate [ \fIOPTIONS\fR ] \-d \fIBINARY\fR ... +.br +.B strace-profile-annotate [ \fIOPTIONS\fR ] \-x \fIPID\fR +.br +.B strace-profile-annotate [ \fIOPTIONS\fR ] \-c \fICMD\fR + +.SH DESCRIPTION + +The stap-profile-annotate command profiles selected user-space +processes, based on selected profiling events over a selected period +of time, then produces an annotated source code listings from all the +hits on their executables & shared libraries. The annotation +identifes number of profiling event hits on the source files' +individual lines. + +The selection of user-space processes and shared libraries may be +system-wide (in which case use the \fB-d\fP option to enumerate all +potentially interesting binaries), or may be focused on single +pre-existing process (\fB-x\fP) or a newly run process hierarchy +(\fB-c\fP). SystemTap automatically adds dependent shared libraries +for any binary explicitly given by the user. + +The selection of profiling event may be the default kernel profiling +timer (a few hundred Hz), or any desired set of systemtap probe +points (\fB-e\fP). + +The selected time for the profiling session can be the lifetime of the +targeted process, or a specified timeout (\fB-T\fP), or may be +interrupted by the user at any time. + +The stap-profile-annotate program uses debuginfod to fetch debuginfo +and source files, and therefore requires a configured +\fBdebuginfod-find\fP program. If profiling locally built programs +that are not available in central debuginfod servers, consider +running a local private server temporarily, possibly federated to +upstream debuginfod servers. +.SAMPLE + % export DEBUGINFOD_URLS=https://UPSTREAM.SERVER/ + % debuginfod -p 8002 -d :memory: -F /BUILD/TREE1 /BUILD/TREE2 & + + # export DEBUGINFOD_URLS=http://localhost:8002/ + # stap-profile-annotate [...] +.ESAMPLE + +The resulting annotated source files may be written into +subdirectories based on the buildid of the binaries, or printed to +standard output (\fB-p\fP). + +A profiling session can end before the process it's targeting +finishes. In this case, sending SIGINT to stap-profile-annotate +will yield the script's desired output if sent after +"Stopped stap data collector" is in stdout. You must still +manually kill the targeted process after sending +stap-profile-annotate SIGINT. + +When stap is run it adds user-space libraries into the kernel +module by default. However, when profiling a module which is not a +user-space shared library the -d option must be used to add that +module's information into the SystemTap kernel module. + +stap-profile-annotate relies heavily on the use of debuginfod. Debuginfod +allows the script to fetch debuginfo and translate virtual memory +offsets into source line numbers. It is also used to fetch the +actual source files to annotate. For these reasons, ensure your +debuginfod client is properly configured for use. + +.SH OPTIONS +.PP + +.TP +.B \fB\-h\fR +Show help message. + +.TP +.B \fB\-x\fR, \fB\-\-pid\fR \fIPID\fR +Pre-existing process PID for SystemTap to target. Its symbol information +is automatically included. + +.TP +.B \fB\-c\fR, \fB\-\-cmd\fR \fICMD\fR +Command for SystemTap to run and then target. Its symbol information +is automatically included. + +.TP +.B \fB\-d\fR \fIBINARY\fR +Add symbol information for another binary (executable or shared library) +and its referenced libraries. This option may be repeated. + +.TP +.B \fB\-e\fR \fIEVENTS\fR, \fB\-\-events \fIEVENTS\fR +Use the given SystemTap probe points (comma-separated), instead of the +default \fItimer.profile\fP to catch profiling hits. Consider +\fIperf.hw.branch_misses\fP. + +.TP +.B \fB\-T\fR, \fB\-\-timeout \fITIMEOUT\fR +stap-profile-annotate will exit after TIMEOUT seconds. \fBNote:\fR if +\fB\-x\fR or \fB\-c\fR, any targeted processes will not be killed +after this timeout. Targeted processes either need their own +timeouts specified or to be killed manually. + +.TP +.B \fB\-p\fR, \fB\-\-print\fR +Print annotated source files to standard output instead of to +individual files named like \fIprofile-BUILDID/source/PATH/FOO.c\fP. + +.TP +.B \fB\-w\fR, \fB\-\-context\-width\fR \fIWIDTH\fR +This option limits the number of lines of context before and +after each hit source line. Without this option, the default is +to use unlimited context, i.e., print all lines. + +.TP +.B \fB\-s\fR, \fB\-\-stap\fR \fIPATH\fR +Override the path to the systemtap program. + +.TP +.B \fB\-v\fR, \fB\-\-verbose\fR +Increase verbosity. May be repeated for more verbosity. + +.SH EXAMPLES + +.SS Command + +.SAMPLE +export DEBUGINFOD_URLS=https://debuginfod.elfutils.org/ # if needed +stap-profile-annotate -w 1 -c '/usr/bin/stress -t 5 --cpu 2' +.ESAMPLE + +.SS Output + +.SAMPLE +Starting stap data collector. +stress: info: [88582] dispatching hogs: 4 cpu, 0 io, 0 vm, 0 hdd +stress: info: [88582] successful run completed in 5s +Counted 19798 known userspace hits. +Ignored 12342 kernel hits. +Stopped stap data collector. +Consumed 321 profile records of 19798 hits across 3 buildids. +0001228 (6.20%) hits in profile-10da92de76b289c2e9cfb145ca1edc4e850ec4da/source/usr/src/debug/ + glibc-2.32-10.fc33.x86_64/stdlib/rand.c over 3 lines. +0004241 (21.42%) hits in profile-10da92de76b289c2e9cfb145ca1edc4e850ec4da/source/usr/src/debug/ + glibc-2.32-10.fc33.x86_64/stdlib/random_r.c over 18 lines. +0000094 (0.47%) hits in profile-10da92de76b289c2e9cfb145ca1edc4e850ec4da/source/usr/src/debug/ + glibc-2.32-10.fc33.x86_64/stdlib/../sysdeps/unix/sysv/linux/x86/lowlevellock.h over 1 lines. +0004769 (24.09%) hits in profile-10da92de76b289c2e9cfb145ca1edc4e850ec4da/source/usr/src/debug/ + glibc-2.32-10.fc33.x86_64/stdlib/random.c over 6 lines. +0001401 (7.08%) hits in profile-520e2b0352d6ac72cb3b58df4f44137e29a94250/source/usr/src/debug/ + stress-1.0.4-26.fc33.x86_64/src/stress.c over 1 lines. +00001038 (5.24%) hits in buildid 520e2b0352d6ac72cb3b58df4f44137e29a94250 with unknown source +0003109 (15.70%) hits in profile-07ae52cfc7f4eda1d13383c04564e3236e059993/source/usr/src/debug/ + glibc-2.32-10.fc33.x86_64/math/../sysdeps/ieee754/dbl-64/e_sqrt.c over 3 lines. +0003918 (19.79%) hits in profile-07ae52cfc7f4eda1d13383c04564e3236e059993/source/usr/src/debug/ + glibc-2.32-10.fc33.x86_64/math/w_sqrt_compat.c over 3 lines. +.ESAMPLE + +.SS Example Annotated Source File + +Found in: \fBprofile-07ae52cfc7f4eda1d13383c04564e3236e059993/source/\fR +\fBusr/src/debug/glibc-2.32-10.fc33.x86_64/math/w_sqrt_compat.c\fR + +.SAMPLE + { +0000730 if (__builtin_expect (isless (x, 0.0), 0) && _LIB_VERSION != _IEEE_) +0000353 return __kernel_standard (x, x, 26); /* sqrt(negative) */ + +0002799 return __ieee754_sqrt (x); + } + libm_alias_double (__sqrt, sqrt) +.ESAMPLE + + +.SH SEE ALSO +.nh +.nf +.IR stap (1), +.IR stapprobes (3stap), +.IR debuginfod-find (1) diff --git a/python/Makefile.in b/python/Makefile.in index d696a0e63..d3778d8f7 100644 --- a/python/Makefile.in +++ b/python/Makefile.in @@ -531,8 +531,8 @@ distclean-generic: maintainer-clean-generic: @echo "This command is intended for maintainers to use" @echo "it deletes files that may require special tools to rebuild." -@HAVE_PYTHON_PROBES_FALSE@clean-local: @HAVE_PYTHON_PROBES_FALSE@install-exec-local: +@HAVE_PYTHON_PROBES_FALSE@clean-local: clean: clean-am clean-am: clean-generic clean-local mostlyclean-am diff --git a/stap-profile-annotate.in b/stap-profile-annotate.in new file mode 100755 index 000000000..70233a108 --- /dev/null +++ b/stap-profile-annotate.in @@ -0,0 +1,334 @@ +#!/usr/bin/python3 + +# This script uses tapset/hit-count.stp to profile a specific process +# or the kernel. It may take a context width, module path, pid, cmd, and timeout. +# It generates folders based on buildid, containing subdirectories +# leading to sourcefiles where one may read how many times the pc +# was at a certain line in that sourcefile. + + +import argparse +import sys +import os +import re +import subprocess +import tempfile +from collections import defaultdict + +parser = argparse.ArgumentParser() +pid_cmd_group = parser.add_mutually_exclusive_group() +pid_cmd_group.add_argument("-x", "--pid", help='PID for systemtap to target.', type=int) +pid_cmd_group.add_argument("-c", "--cmd", help='Command for systemtap to target.', type=str) +parser.add_argument('-d', metavar="BINARY", help='Add symbol information for given binary and its shared libraries.', type=str, action='append', default=[]) +parser.add_argument("-e", "--events", help='Override the list of profiling probe points.', type=str, default='timer.profile') +parser.add_argument("-T", "--timeout", help="Exit in 'timeout' seconds.", type=int) +parser.add_argument("-p", "--print", help="Print annotated source files to stdout instead of files.", action='store_true') +parser.add_argument("-w", "--context-width", metavar="WIDTH", help='Limit number of lines of context around each hit. Defaults to unlimited.', type=int, default=-1) +parser.add_argument("-s", "--stap", metavar="PATH", help='Override the path to the stap interpreter.', type=str) +parser.add_argument("-v", "--verbose", help="Increase verbosity.", action='count', default=0) + +args = parser.parse_args() +verbosity = args.verbose + +def vprint(level,*args): + if (verbosity >= level): + print(*args) + + +stap_script=""" +global count +global unknown +global kernel +global user +probe begin { + system(\"echo Starting stap data collector.\") # sent to stdout of stap-profile-annotate process +} +probe """ + args.events + """ { + if (! user_mode()) { + kernel <<< 1 + next + } + try { + if (target()==0 || target_set_pid(pid())) + { + buildid = umodbuildid(uaddr()); + addr= umodaddr(uaddr()); + count[buildid,addr] <<< 1; + user <<< 1 + } + } + catch /*(e)*/ { unknown <<< 1 /* printf ("%s", e) */ } +} + +probe timer.s(1),end +{ + println (\"BEGIN\"); + foreach ( [buildid, addr] in count) + { + c = @count(count[buildid,addr]); + println(buildid, " " , addr, " ", c); + } + println (\"END\"); + delete count +} +probe end,error +{ + printf (\"Counted %d known userspace hits.\\n\", @count(user)) + if (@count(kernel)) + printf (\"Ignored %d kernel hits.\\n\", @count(kernel)) + if (@count(unknown)) + printf (\"Ignored %d unknown userspace hits.\\n\", @count(unknown)) + println(\"Stopped stap data collector.\") +} +""" + +# buildid class +class BuildIDProfile: + def __init__(self,buildid): + self.counts = defaultdict(lambda: 0) + self.buildid = buildid + self.filename = self.buildid + 'addrs.txt' + self.sources = {} + + def __str__(self): + return "BuildIDProfile(buildid %s) items: %s sources: %s" % (self.buildid, self.counts.items(), self.sources.items()) + + # Build the 'counts' dict by adding the hit count to its associated address + def accumulate(self,pc,count): + self.counts[pc] += count + + # Get the Find the sources of relative addresses from self.counts.keys() + def get_sources(self): + vprint(1,"Computing addr2line for %s" % (self.buildid,)) + # Used to maintain order of writing + ordered_keys = list(self.counts.keys()) + # create addr file in /tmp/ + with open('/tmp/'+self.filename, 'w') as f: + for k in ordered_keys: + f.write(str(hex(k)) + '\n') + vprint(2,"Dumped addresses") + # Get source:linenum info + dbginfo = self.get_debuginfo() + # Split the lines into a list divided by newlines + lines = dbginfo.split('\n') + + for i in range(0,len(lines)): + if lines[i] == '': + continue + split = lines[i].split(':') + src = split[0] + line_number = split[1] + if line_number == None: + continue + if src not in self.sources.keys(): + self.sources[src] = SourceLineProfile(self.buildid,src) + + # Sometimes addr2line reponds with a string of format ("linenum" discriminator "num") + # trim this to yield "linenum" using a regular expression: + m = re.search('[0-9]+',line_number) + # If m doesn't contain the above regex, it has no number so don't accumulate it + if m == None: + continue + line_number = int(m.group(0)) + # eu-addr2line gives outputs beginning at 1, where as in SourceLineProfiler.report + # the line numbering begins at 0. This offset of 1 must be reomved from eu-addr2line + # to ensure compatibility with SourceLineProfiler.report + self.sources[src].accumulate(line_number-1, self.counts[ordered_keys[i]]) + vprint(2,"Mapped to %d source files" % (len(self.sources),)) + # Remove tempfile + os.remove('/tmp/'+self.filename) + + # Report information for this buildid's source files + def report(self,totalhits): + for so in self.sources.values(): + so.report(totalhits) + + # Get source:linenum information. Assumes self.filename has relative address information + def get_debuginfo(self): + try: + #Get the debuginfo of the bulidid retrieved from stap + p = subprocess.Popen(['debuginfod-find', 'debuginfo', self.buildid],stdout=subprocess.PIPE) + dbg_file,err = p.communicate() + dbg_file = dbg_file.decode('utf-8').rstrip() + vprint(2, "Stored debuginfod-find debuginfo file as %s" % (dbg_file)) + #Use the debuginfo attained from the above process + process = subprocess.Popen(['sh','-c', 'eu-addr2line -A -e ' + dbg_file + ' < /tmp/' + self.filename], stdout=subprocess.PIPE) + out,err = process.communicate() + except Exception as e: + print (e) + pass + return out.decode('utf-8') + + +# Contains information related to each source of a buildid +class SourceLineProfile: + def __init__(self, bid, source): + self.bid = bid + self.source = source + self.counts = defaultdict(lambda: 0) + + def __str__(self): + return "SourceLineProfile(bid %s, source %s) counts: %s" % (self.bid, self.source, self.counts.items()) + + # Accumulate hits on a line + def accumulate(self, line, count): + self.counts[line] += count + + # Get the source file associated with a buildid + def get_source_file(self): + try: + p = subprocess.Popen(['debuginfod-find', 'source', self.bid, self.source],stdout=subprocess.PIPE) + sourcefile,err = p.communicate() + sourcefile = sourcefile.decode('utf-8').rstrip() + if sourcefile == '' or sourcefile == None: + raise Exception("No source file for bid %s, source %s from debuginfod servers: %s" % (self.bid, self.source, os.getenv("DEBUGINFOD_URLS"))) + elif err != '' and err != None: + raise Exception(err.decode('utf-8').rstrip()) + vprint(2, "Stored debuginfod-find source file as %s" % (sourcefile)) + return sourcefile + except Exception as e: + print (e) + + # Reporting function for the source file + def report(self, totalhits): + filehits=sum(self.counts.values()) + if self.source == '??' or self.source == '': + vprint(0,"%08d (%.2f%%) hits in buildid %s with unknown source" % (filehits, filehits/totalhits*100, + self.bid)) + return + # Retrieve the sourcefile's name + sourcefile = self.get_source_file() + if sourcefile == None or sourcefile == '': + return + + outfile = os.path.join('profile-'+self.bid, (sourcefile.split('/')[-1]).replace('##','/')) + + # Try creating the appropriate directory + if not args.print: + try: + os.makedirs(os.path.dirname(outfile)) + # XXX: what if outfile has lots of ../../../../../'s in it? + except: + pass + + # Output source code to 'outfile' and if a line has associated hits (read out of sourcefile) + # then add the line number and hit count before that line. If a context_width is present use + # print the surrounding lines for context in accordance with context_width + vprint(0,"%07d (%.2f%%) hits in %s over %d lines." % (filehits, filehits/totalhits*100, + outfile, len(self.counts))) + with open(sourcefile,'r') as f: + of = sys.stdout if args.print else open(outfile, 'w') + + hitlines = sorted( list(self.counts.keys()) ) + width = args.context_width + vprint(2, "Writing with width %d." % (width)) + # Set the first upper bound + upper_bound = sys.maxsize if width == -1 else hitlines[0]+width + lower_bound = 0 if width == -1 else hitlines[0] - width + + for linenum, line, in list(enumerate(f)): + # The lines by default have a new line at the end, remove those + line = line.rstrip() + + # Adjust upper bound and next key if necessary. When there is only one + # key left there is no need to adjust, it also causes an error + if upper_bound <= linenum and len(hitlines) > 1: + hitlines.pop(0) + upper_bound = sys.maxsize if width == -1 else hitlines[0] + width + lower_bound = 0 if width == -1 else hitlines[0] - width + + # If we have found a line with hits, output info + # otherwise if there is no width, don't take it into account + # otherwise if the current line is within the desired width + # print it for context + if hitlines and linenum == hitlines[0]: + of.write("%07d %s\n" % (self.counts[linenum], line)) + hitlines.pop(0) + elif width == -1: + of.write("%7s %s\n" % ("", line)) + elif lower_bound <= linenum and linenum <= upper_bound: + of.write("%7s %s\n" % ("", line)) + + if not args.print: # don't close stdout + of.close() + +def __main__(): + # We require $DEBUGINFOD_URLS + if (not os.getenv("DEBUGINFOD_URLS")): + raise Exception("Required DEBUGINFOD_URLS is unset.") + + # Run SystemTap + (tmpfd,tmpfilename) = tempfile.mkstemp() + stap_cmd = "@prefix@/bin/stap" # not @ bindir @ because autoconf expands that to shell $var expressions + stap_args = ['--ldd', '-o'+tmpfilename] + + if args.cmd: + stap_args += ['-c', args.cmd] + if args.timeout: + if args.timeout < 0: + raise Exception("Timeout must be positive") + stap_args += ['-T', str(args.timeout)] + if args.pid: + if args.pid < 0: + raise Exception("pid must be positive") + stap_args += ['-x', str(args.pid)] + for d in args.d: + stap_args += ['-d', d] + if args.stap: + stap_cmd = args.stap + if args.context_width and args.context_width < -1: + raise Exception("context_width must be positive or -1 (for all file)") + stap_args += ['-e', stap_script] + + vprint(1,"Building stap data collector.") + vprint(2,"%s %s" % (stap_cmd, stap_args)) + + try: + p = subprocess.Popen([stap_cmd] + stap_args) + p.communicate() # wait until process exits + except KeyboardInterrupt: + pass + p.kill() + + buildids = {} # dict from buildid hexcode to BuildIdProfile object + + outp_begin = False + proflines = 0 + totalhits = 0 + + for line in open(tmpfilename,"r"): # read stap output, text mode + line = line.rstrip() + # All relevant output is after BEGIN and before END + if "BEGIN" in line: + outp_begin = True + elif "END" in line: + outp_begin = False + elif outp_begin == False: + if line != "": # diagnostic message + vprint(0,line) + else: + pass + else: # an actual profile record + try: + proflines += 1 + (buildid,pc,hits) = line.split() + vprint(3,"(%s,%s,%s)" % (buildid,pc,hits)) + totalhits += int(hits) + bidp = buildids.setdefault(buildid, BuildIDProfile(buildid)) + # Accumulate hits for offset pc + bidp.accumulate(int(pc),int(hits)) + except Exception as e: # parse error? + vprint(2,e) + + os.remove(tmpfilename) + + vprint(0, "Consumed %d profile records of %d hits across %d buildids." % (proflines, totalhits, len(buildids))) + + # Output source information for each buildid + totalhits = sum([sum(bid.counts.values()) for bid in buildids.values()]) + for buildid, bidp in buildids.items(): + bidp.get_sources() + bidp.report(totalhits) + +if __name__ == '__main__': + __main__() diff --git a/systemtap.spec b/systemtap.spec index 9d14468a7..c1e5630b9 100644 --- a/systemtap.spec +++ b/systemtap.spec @@ -1078,6 +1078,7 @@ exit 0 %files devel -f systemtap.lang %{_bindir}/stap %{_bindir}/stap-prep +%{_bindir}/stap-profile-annotate %{_bindir}/stap-report %dir %{_datadir}/systemtap %{_datadir}/systemtap/runtime diff --git a/tapset/context.stp b/tapset/context.stp index c379c904b..45b9a70e2 100644 --- a/tapset/context.stp +++ b/tapset/context.stp @@ -26,6 +26,40 @@ function print_regs () } %} +function umodbuildid:string (address:long) +%{ /* pragma:vma */ + void *ubid_struct = NULL; + int i,j; + struct _stp_module * p; + static const char hex[] = "0123456789abcdef"; + stap_find_vma_map_info(current, STAP_ARG_address, + NULL, NULL, NULL, NULL, &ubid_struct); + + p = (struct _stp_module*) ubid_struct; + if (p == NULL){ + STAP_ERROR("umodbuildid 0x%llx unknown\n ", STAP_ARG_address); + } + + for(i=0,j=0; j < p->build_id_len; ++j && i < MAXSTRINGLEN) + { + unsigned char temp = p->build_id_bits[j]; + STAP_RETVALUE[i++] = hex[temp >> 4]; + STAP_RETVALUE[i++] = hex[temp & 15]; + } +%} + + +# Finds the module-relative offset of the given address in the current user process +function umodaddr:long (address:long) +%{ /* pragma:vma */ + long vm_start = -1; + stap_find_vma_map_info(current, STAP_ARG_address, + &vm_start, NULL, NULL, NULL,NULL); + if(vm_start == -1) + STAP_ERROR("umodaddr 0x%llx unknown\n ", STAP_ARG_address); + STAP_RETURN(STAP_ARG_address - vm_start); +%} + /** * sfunction pp - Returns the active probe point * diff --git a/testsuite/systemtap.apps/profile-annotate.exp b/testsuite/systemtap.apps/profile-annotate.exp new file mode 100644 index 000000000..5b8e2bc30 --- /dev/null +++ b/testsuite/systemtap.apps/profile-annotate.exp @@ -0,0 +1,114 @@ +set outp [] + +# Used to get file length for testing when -w is not specified. +# The debuginfod file and annotated source file should be of equal length. +# file is the filename to check the length of +proc file_len {filename} { + set fd [open $filename r] + set i 0 + while { [gets $fd line] > -1 } { incr i } + close $fd + return $i +} + + +set test "stap-profile-annotate" +set c_arg "/usr/bin/stress --cpu 4 -t 10" + +if {![installtest_p]} { untested $test; return } + +if { ! [ file exists "/usr/bin/stress" ] } { + untested "$test: /usr/bin/stress not present. Skipping..."; + return +} + +if { ! [info exists env(DEBUGINFOD_URLS)] } { + untested "$test: DEBUGINFOD_URLS un set" + return +} + + +# Command to find a lib to profile on the system +# There may be multiple, so pick the last one, hoping it's the one +# /usr/bin/stress is linked against. OTOH since stap-profile-annotate +# invokes stap with -c stress --ldd, the right one will be found anyway. +set lib [lindex [glob /lib*/libc.so.*] end] + +set test "source-annotate -c, -w=null, debuginfod functionality" +# Flag to check if -c is found +set c 0 +# Lists for checking if debuginfod files exists and +# annotate files are annotated appropriately +set annotated_paths [] +set dbg_paths [] +set buildids [] + +spawn stap-profile-annotate -vvv -d $lib -c $c_arg -T 50 +expect { + -timeout 180 + -re "'-c', '$c_arg'" { incr c ; exp_continue } + +# Collect debuginfod file locations + -re "Stored debuginfod-find source file as ((.*)\#\#(\S+\..))" { append dbg_paths "$expect_out(1,string) "; exp_continue } + +# Collect profile-'buildid' locations + -re {hits in (profile-(.*?)/.*\..+ )over} { append annotated_paths $expect_out(1,string); + append buildids "$expect_out(2,string) "; exp_continue } + eof {append outp $expect_out(buffer);} + timeout { fail "$test: Unexpected timeout" } +} + +# If c is not found +if { !$c } { + fail "$test: -c option not found in stap_args"; +} else { pass $test } + +# Remove duplicates from both path lists +set failme 0 +foreach b $buildids { + set cur_dbg [] + set cur_ann [] + foreach d $dbg_paths { + if { [ regexp "$b" $d ] } { append cur_dbg "$d "} + } + foreach a $annotated_paths { + if { [ regexp "$b" $a ] } { append cur_ann "$a " } + } + # This for loop assumes that both $annotated_paths and dbg_paths + # are in an order such that annotated_paths[0] is the annotated + # version of dbg_paths[0] + for { set i 0 } { $i < [ llength $cur_dbg ] } { incr i } { + set dbg [lindex $cur_dbg $i] + set ann [lindex $cur_ann $i] + set dbg_len [ file_len $dbg ] + set ann_len [ file_len $ann ] + if {$dbg_len != $ann_len} { set failme 1 } + } +} + +if { $failme > 0 } { fail "$test length mismatch" } else { pass $test } + + + +# Test to see if -w 0 only prints annotated source lines into files +set test "source-annotate -w=0" +set annotated_paths [] +spawn stap-profile-annotate -w 0 -T 30 -d $lib -vvv +expect { + -timeout 180 +# Collect profile-'buildid' locations + -re {hits in (profile-(.*?)/.*\..+ )over} { append annotated_paths $expect_out(1,string); exp_continue} + timeout { fail "$test: Unexpected timeout" } +} + +set failme 0 +foreach a $annotated_paths { + set fd [open $a r] + while { [gets $fd line] > -1 } { + if { ![regexp {[0-9]{7} } $line] } { + set failme 1 + } + } +} + +if { $failme > 0 } then { fail "$test: Line present in $a without hit count" } else { pass $test } -- 2.43.5