From: Frank Ch. Eigler Date: Wed, 15 Aug 2018 19:36:54 +0000 (-0400) Subject: stap-exporter: rework configuration X-Git-Tag: release-4.0~120 X-Git-Url: https://sourceware.org/git/?a=commitdiff_plain;h=b85506271be4159251f30b1c870f6a08f3f4371f;p=systemtap.git stap-exporter: rework configuration Stop hardcoding "stap --example URLPIECE" into the python module; instead run the "URLPIECE" script from under the /etc/stap-exporter directory. This way, one can have some non-default stap options added. (The default set of scripts is stored in the default/ subdirectory in the source tree.) Command line options for stap-exporter can now be overridden from a /etc/sysconfig/stap-exporter file suitable for use by systemd EnvironmentFile=. Move scripts directory to /etc/stap-exporter; search *.stp files systematically to compute candidate URLs; simplify implementation. Expand the stap-exporter.8 man page. stap-exporter/procfs: stop special "__prometheus" name mapping There's no need to mangle the procfs parameter name. --- diff --git a/configure b/configure index d3b3a3c0f..9d9012e8a 100755 --- a/configure +++ b/configure @@ -13062,6 +13062,8 @@ STAP_PREFIX="$stap_prefix" ac_config_headers="$ac_config_headers config.h:config.in" + + ac_config_files="$ac_config_files Makefile doc/Makefile man/Makefile man/cs/Makefile doc/beginners/Makefile doc/SystemTap_Tapset_Reference/Makefile man/stap.1 man/stappaths.7 man/systemtap-service.8 man/cs/stap.1 man/cs/stappaths.7 man/cs/systemtap.8 initscript/config.systemtap initscript/config.stap-server initscript/systemtap initscript/stap-server initscript/99stap/module-setup.sh initscript/99stap/install initscript/99stap/check" diff --git a/configure.ac b/configure.ac index 68b6d619c..caa62ee28 100644 --- a/configure.ac +++ b/configure.ac @@ -996,6 +996,10 @@ AC_DEFINE_UNQUOTED(STAP_PREFIX, "$stap_prefix", [configure prefix location]) AC_SUBST(STAP_PREFIX, "$stap_prefix") AC_CONFIG_HEADERS([config.h:config.in]) + +dnl XXX: we'd like fully expanded path names for the @macros@ in there, +dnl not like exec_prefix=${prefix} + AC_CONFIG_FILES([Makefile doc/Makefile man/Makefile man/cs/Makefile \ doc/beginners/Makefile doc/SystemTap_Tapset_Reference/Makefile \ man/stap.1 man/stappaths.7 man/systemtap-service.8 \ diff --git a/java/Makefile.in b/java/Makefile.in index 07a1728a1..35a1b2c00 100644 --- a/java/Makefile.in +++ b/java/Makefile.in @@ -673,9 +673,9 @@ maintainer-clean-generic: @echo "This command is intended for maintainers to use" @echo "it deletes files that may require special tools to rebuild." -test -z "$(BUILT_SOURCES)" || rm -f $(BUILT_SOURCES) -@HAVE_JAVA_FALSE@uninstall-local: -@HAVE_JAVA_FALSE@install-exec-local: @HAVE_JAVA_FALSE@install-data-local: +@HAVE_JAVA_FALSE@install-exec-local: +@HAVE_JAVA_FALSE@uninstall-local: clean: clean-am clean-am: clean-binPROGRAMS clean-generic clean-noinstPROGRAMS \ diff --git a/man/stappaths.7.in b/man/stappaths.7.in index 73824442e..d178561af 100644 --- a/man/stappaths.7.in +++ b/man/stappaths.7.in @@ -45,6 +45,9 @@ environment variable. The auxiliary program supervising module loading, interaction, and unloading. .TP +@sysconfdir@/stap\-exporter +The default directory to search for \fB*.stp\fR files, for exporting to HTTP. +.TP @libexecdir@/systemtap/stapio The auxiliary program for module input and output handling. .TP @@ -97,44 +100,37 @@ The location of kernel module building infrastructure. @prefix@/share/doc/systemtap*/examples Examples with greater detail can be found here. Each example comes with a .txt or .meta file explaining what the example, sample or demo does and how it is -ordinarily run. See also +ordinarily run. See also online at: .nh .IR http://sourceware.org/systemtap/examples/ .hy .TP $SYSTEMTAP_DIR/ssl/server -User's server\-side SSL certificate database. If SYSTEMTAP_DIR is not +User's server-side SSL certificate database. If SYSTEMTAP_DIR is not set, the default is $HOME/.systemtap. .TP $SYSTEMTAP_DIR/ssl/client -User's private client\-side SSL certificate database. If SYSTEMTAP_DIR is not +User's private client-side SSL certificate database. If SYSTEMTAP_DIR is not set, the default is $HOME/.systemtap. .TP @sysconfdir@/systemtap/ssl/client -Global client\-side SSL certificate database. +Global client-side SSL certificate database. .TP @sysconfdir@/systemtap/staprun/ \fIstaprun\fR\[aq]s trusted signer certificate database. .TP -@sysconfdir@/sysconfig/stap\-server/ +@sysconfdir@/sysconfig/stap\-server stap\-server service global configuration file. .TP -@sysconfdir@/stap\-server/conf.d/*.conf -stap\-server service configuration files for default servers. +@sysconfdir@/sysconfig/stap\-exporter +stap\-exporter service global configuration file. .TP -/var/run/stap\-server/ +@localstatedir@/run/stap\-server/ stap\-server service default location of status files for running servers. .TP -/var/log/stap\-server/log +@localstatedir@/log/stap\-server/log stap\-server service default log file. - -.PP -.SH FILES -.nh -.IR @prefix@/share/systemtap/tapset -.hy - .SH SEE ALSO .nh .nf diff --git a/python/Makefile.in b/python/Makefile.in index ce01b25dc..52540a24a 100644 --- a/python/Makefile.in +++ b/python/Makefile.in @@ -529,8 +529,8 @@ distclean-generic: maintainer-clean-generic: @echo "This command is intended for maintainers to use" @echo "it deletes files that may require special tools to rebuild." -@HAVE_PYTHON_PROBES_FALSE@clean-local: @HAVE_PYTHON_PROBES_FALSE@install-exec-local: +@HAVE_PYTHON_PROBES_FALSE@clean-local: clean: clean-am clean-am: clean-generic clean-local mostlyclean-am diff --git a/stap-exporter/Makefile.am b/stap-exporter/Makefile.am index b8c7c14ed..1062fd5d7 100644 --- a/stap-exporter/Makefile.am +++ b/stap-exporter/Makefile.am @@ -9,13 +9,17 @@ if HAVE_PYTHON3_PROBES man8_MANS = stap-exporter.8 install-data-local: - $(MKDIR_P) "$(DESTDIR)$(sysconfdir)/systemtap/stap-exporter" - $(MKDIR_P) "$(DESTDIR)$(sysconfdir)/systemtap/stap-exporter/autostart" + $(MKDIR_P) "$(DESTDIR)$(sysconfdir)/stap-exporter" + cd $(srcdir)/default; find . | cpio -pdmv "$(DESTDIR)$(sysconfdir)/stap-exporter/" $(MKDIR_P) "$(DESTDIR)$(prefix)/lib/systemd/system" + $(MKDIR_P) "$(DESTDIR)$(sysconfdir)/sysconfig" $(INSTALL_DATA) $(srcdir)/stap-exporter.service "$(DESTDIR)$(prefix)/lib/systemd/system" + $(INSTALL_DATA) $(srcdir)/stap-exporter.options "$(DESTDIR)$(sysconfdir)/sysconfig/stap-exporter" uninstall-local: rm -f "$(DESTDIR)$(prefix)/lib/systemd/system/stap-exporter.service" + rm -f "$(DESTDIR)$(sysconfdir)/sysconfig/stap-exporter" + rm -rf "$(DESTDIR)$(sysconfdir)/stap-exporter" sbin_SCRIPTS = stap-exporter diff --git a/stap-exporter/Makefile.in b/stap-exporter/Makefile.in index 437795e00..98abfa582 100644 --- a/stap-exporter/Makefile.in +++ b/stap-exporter/Makefile.in @@ -517,9 +517,9 @@ distclean-generic: maintainer-clean-generic: @echo "This command is intended for maintainers to use" @echo "it deletes files that may require special tools to rebuild." -@HAVE_PYTHON3_PROBES_FALSE@install-data-local: -@HAVE_PYTHON3_PROBES_FALSE@clean-local: @HAVE_PYTHON3_PROBES_FALSE@uninstall-local: +@HAVE_PYTHON3_PROBES_FALSE@clean-local: +@HAVE_PYTHON3_PROBES_FALSE@install-data-local: clean: clean-am clean-am: clean-generic clean-local mostlyclean-am @@ -607,13 +607,17 @@ uninstall-man: uninstall-man8 @HAVE_PYTHON3_PROBES_TRUE@install-data-local: -@HAVE_PYTHON3_PROBES_TRUE@ $(MKDIR_P) "$(DESTDIR)$(sysconfdir)/systemtap/stap-exporter" -@HAVE_PYTHON3_PROBES_TRUE@ $(MKDIR_P) "$(DESTDIR)$(sysconfdir)/systemtap/stap-exporter/autostart" +@HAVE_PYTHON3_PROBES_TRUE@ $(MKDIR_P) "$(DESTDIR)$(sysconfdir)/stap-exporter" +@HAVE_PYTHON3_PROBES_TRUE@ cd $(srcdir)/default; find . | cpio -pdmv "$(DESTDIR)$(sysconfdir)/stap-exporter/" @HAVE_PYTHON3_PROBES_TRUE@ $(MKDIR_P) "$(DESTDIR)$(prefix)/lib/systemd/system" +@HAVE_PYTHON3_PROBES_TRUE@ $(MKDIR_P) "$(DESTDIR)$(sysconfdir)/sysconfig" @HAVE_PYTHON3_PROBES_TRUE@ $(INSTALL_DATA) $(srcdir)/stap-exporter.service "$(DESTDIR)$(prefix)/lib/systemd/system" +@HAVE_PYTHON3_PROBES_TRUE@ $(INSTALL_DATA) $(srcdir)/stap-exporter.options "$(DESTDIR)$(sysconfdir)/sysconfig/stap-exporter" @HAVE_PYTHON3_PROBES_TRUE@uninstall-local: @HAVE_PYTHON3_PROBES_TRUE@ rm -f "$(DESTDIR)$(prefix)/lib/systemd/system/stap-exporter.service" +@HAVE_PYTHON3_PROBES_TRUE@ rm -f "$(DESTDIR)$(sysconfdir)/sysconfig/stap-exporter" +@HAVE_PYTHON3_PROBES_TRUE@ rm -rf "$(DESTDIR)$(sysconfdir)/stap-exporter" @HAVE_PYTHON3_PROBES_TRUE@stap-exporter: $(srcdir)/stap-exporter.in @HAVE_PYTHON3_PROBES_TRUE@ sed -e "s#%sysconfdir%#$(sysconfdir)#g" < $< > $@ diff --git a/stap-exporter/default/EXAMPLE b/stap-exporter/default/EXAMPLE new file mode 100755 index 000000000..eb4520f24 --- /dev/null +++ b/stap-exporter/default/EXAMPLE @@ -0,0 +1,4 @@ +#! /bin/sh + +# link to this generic shell script to invoke a non-guru stap example +exec stap -v --example $@ `basename $0 | sed -e s,autostart-,,` diff --git a/stap-exporter/default/EXAMPLE_NOERROR b/stap-exporter/default/EXAMPLE_NOERROR new file mode 100755 index 000000000..335fc85bb --- /dev/null +++ b/stap-exporter/default/EXAMPLE_NOERROR @@ -0,0 +1,4 @@ +#! /bin/sh + +# link to this generic shell script to invoke a non-guru stap example +exec stap -v --suppress-handler-errors --example $@ `basename $0 | sed -e s,autostart-,,` diff --git a/stap-exporter/default/autostart-also_ran.stp b/stap-exporter/default/autostart-also_ran.stp new file mode 120000 index 000000000..0c15c9d54 --- /dev/null +++ b/stap-exporter/default/autostart-also_ran.stp @@ -0,0 +1 @@ +EXAMPLE \ No newline at end of file diff --git a/stap-exporter/default/syscallsbypid.stp b/stap-exporter/default/syscallsbypid.stp new file mode 120000 index 000000000..e12ec3db1 --- /dev/null +++ b/stap-exporter/default/syscallsbypid.stp @@ -0,0 +1 @@ +EXAMPLE_NOERROR \ No newline at end of file diff --git a/stap-exporter/stap-exporter.8 b/stap-exporter/stap-exporter.8 index e6f12d267..be2f970e0 100644 --- a/stap-exporter/stap-exporter.8 +++ b/stap-exporter/stap-exporter.8 @@ -32,49 +32,63 @@ stap-exporter \- systemtap-prometheus interoperation mechanism .SH DESCRIPTION .I stap-exporter -is a small program that manages systemtap session and relays prometheus metrics -from the sessions to remote requesters on demand. stap-exporter runs as a systemd -service and listens to a configurable TCP port for requests. stap-exporter is -capable of running multiple -.I stap -scripts at a time. Metrics for a given script are available at the URL -[HOSTNAME]:[PORT]/[SCRIPTNAME]. PORT defaults to 9900 but is configurable, -see OPTIONS below. After installation, the stap-exporter systemd service -should be manually enabled. - -.PP -If stap-exporter receives a request for metrics from a script that is already running, -the script's prometheus probes will trigger. The metrics sent back to the requester -are specified using the prometheus_dump_arrayN family of macros (see EXAMPLE below). -If stap-exporter receives a request for a script that is not currently running, stap-exporter -will search systemtap.examples for scripts with script name given in the URL. -If found, stap-exporter will begin running this script. At launch, stap-exporter will -automatically run all scripts found in /etc/systemtap/stap-exporter directory. -.\" parametrize ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +runs a set of systemtap scripts and relays their procfs outputs to +remote HTTP clients on demand. This makes systemtap scripts directly +usable as individual prometheus exporters. This is assisted by a +set of macros provided in the \fBprometheus.stpm\fR tapset file. .SH OPTIONS + The .I stap-exporter -program supports the following options. Any other option prints a list of -supported options. +program supports the following options. .TP -.B \-p \-\-port -Listen to the specified TCP port for requests. Port 9900 is used by default. +.B \-p \-\-port PORT +Listen to the specified TCP port for HTTP requests. Port 9900 is used by default. .TP -.B \-t \-\-timeout -Scripts that run longer than TIMEOUT seconds will be automatically shutdown. -Each time stap-exporter recieves a request for a particular script, it resets -that script's remaining time until shutdown. By default, scripts will run until -stap-exporter is terminated. +.B \-k \-\-keepalive KEEPALIVE +Scripts that run longer than KEEPALIVE seconds beyond the last request are shut down. +There is no timeout by default, so once started, scripts are kept running. +.TP +.B \-s \-\-scripts SCRIPTS +Search the directory SCRIPTS for \fB*.stp\fR files to be exposed. The default is +given in the \fBstappaths.7\fR man page. + .TP .B \-h \-\-help Print help message. -.SH EXAMPLES -Suppose that example.stp contains the following stap script: + +.SH OPERATION + +Upon startup, +.I stap-exporter +searches the directory specified by the \fB\-s\fR directory for files +named \fB*.stp\fR. The name of each file becomes available as a URL +component for subsequent GET HTTP requests. For example, when an HTTP +client asks for "/foo.stp", and the "foo.stp" script (executable / +shell-script) was known, then it is spawned with additional \fIstap\fR +options to set a module name. This predictable module name makes it +possible for stap-exporter to transcribe a procfs file from that +running script to HTTP clients. + +After a configurable period of disuse (\fB\-k\fR or +\-fB\-\-keepalive\fR option), a systemtap script is terminated. It +will be restarted again if a client requests. + +All files whose name includes the substring \fBautostart\fR are +started immediately (and restarted if they stop), rather than +on-demand. These are excluded from keepalive considerations. Scripts +that may be too slow to start or wish to report long-term statistics +are candidates for this treatment. + +.SH EXAMPLE + +Suppose that example.stp contains the following script. It counts +read syscalls on a per-thread & per-cpu basis. .SAMPLE -global arr +global arr% probe syscall.read { arr[tid(), cpu()]++ @@ -85,19 +99,22 @@ probe prometheus { } .ESAMPLE -The prometheus_dump_arrayN macros are used to produce metrics from an array. -Systemtap provides a prometheus_dump_arrayN macro for all N from 1 to 8. +The \fIprometheus_dump_array\fR macros are used to produce metrics from an array. +Systemtap provides a \fIprometheus_dump_arrayN\fR macro for all N from 1 to 8. The first argument of the macros represents an array with N-element keys. The second argument represents the name of the metric. The remaining N arguments represent the names of the metric's labels. +One may launch stap\-exporter as root, or equivalent \fIstapdev\fR privileges, then +after a brief delay, use any web client to fetch data: + .SAMPLE -$ stap-exporter -p 9999 -t 60 +# stap-exporter -p 9999 -k 60 -c . & -$ curl localhost:9999/example.stp -Refresh page to access metrics. +$ curl http://localhost:9999/example.stp +Refresh page to access metrics. [...] -$ curl localhost:9999/example.stp +$ curl http://localhost:9999/example.stp count{tid="12614",cpu="0"} 9 count{tid="12170",cpu="3"} 107 count{tid="1802",cpu="0"} 33687 @@ -105,18 +122,57 @@ count{tid="12617",cpu="1"} 99 [...] .ESAMPLE +The same URL may be added to a Prometheus server's scrape_config section, +or a Performance Co-Pilot pmdaprometheus config.d directory, to collect +this data into a monitoring system. + + .SH SAFETY AND SECURITY -See the -.IR stap (1) -manual page for additional information on safety and security. + +The stap\-exporter server does not enforce any particular security mechanisms. +Therefore, deployment in an untrusted environment needs to consider: + +.TP +script selection +Since systemtap scripts are run under the privileges +of the stap\-exporter process (probably \fIroot\fR), the system +administrator must select only safe & robust scripts. +Check the scripts installed by default before activating the +service. +Scripts cannot take input from the web clients. + +.TP +TCP/IP firewalling +Since stap\-exporter exposes the selected TCP/HTTP port to all interfaces +on the host, it may be necessary to add a firewall. It is unlikely to be +appropriate to expose such a service to an untrusted network. + +.TP +HTTP filtering +Since stap\-exporter exposes the configured systemtap scripts to all HTTP +clients without authentication, it may be necessary to protect it from +abuse even on mostly trusted networks. An HTTP proxy may be used to +impose URL- or client- or usage- or authentication-dependent filters. + +.TP +HTTPS +Since stap\-exporter speaks only plain HTTP, an HTTP proxy may be used to +support HTTPS secure protocols. + .SH SEE ALSO .IR stap (1), .IR stapprobes (3stap), .IR stappaths (7) +.IR tapset::prometheus (7) .SH BUGS Use the Bugzilla link of the project web page or our mailing list. .nh .BR http://sourceware.org/systemtap/ ", " . .hy +.PP +.IR error::reporting (7stap), +.nh +.BR https://sourceware.org/systemtap/wiki/HowToReportBugs +.hy diff --git a/stap-exporter/stap-exporter.in b/stap-exporter/stap-exporter.in index 2b5907319..2706d3f49 100644 --- a/stap-exporter/stap-exporter.in +++ b/stap-exporter/stap-exporter.in @@ -1,5 +1,22 @@ #!/usr/bin/python3 +""" +Copyright (C) 2018 Red Hat, Inc. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + import os import sys import argparse @@ -9,132 +26,149 @@ from http.server import BaseHTTPRequestHandler, HTTPServer from urllib.parse import urlparse from time import time -script_dir = "%sysconfdir%/systemtap/stap-exporter/" -proc_path = "/proc/systemtap/__systemtap_exporter" +# globals +script_dir = "%sysconfdir%/stap-exporter" +proc_basename = "__stap_exporter_" + str(os.getpid()) # permit concurrent exporters +proc_path = "/proc/systemtap" +sessmgr = None -class Session: - def __init__(self, name, sess_id): +class Session: + """Represent a single systemtap script found $script_dir, whether or not + associated with a running systemtap process.""" + + def __init__(self, dirname, name, sess_id, ka): + self.dirname = dirname # config directory self.name = name - self.id = sess_id - self.cmd = self.get_cmd(name) - self.process = None - self.start_time = None - - def begin(self): - self.process = subprocess.Popen(shlex.split(self.cmd)) + self.id = sess_id # permanent id# for procfs naming + self.keepalive = ka # how many seconds to keep process alive after a use or None + self.process = None # live process + self.killafter = None # time for euthenasia + self.proc_subdirname = "%s_%d" % (proc_basename, self.id) + print("session %s found" % (self.name,)) + + def get_cmd(self): + return "%s/%s -m %s" % (self.dirname, self.name, self.proc_subdirname) - def get_proc_path(self): - return proc_path + str(self.id) + "/" + self.name - - def set_start_time(self): - self.start_time = time() - - def get_cmd(self, script): - return "stap -m __systemtap_exporter%d --example %s" % (self.id, - script) - + def poll(self, killthem): + if self.process and killthem: + self.process.terminate() + print("session %s shut down" % (self.name,)) + return + + # bring out your dead + if self.process and self.process.poll() is not None: # died? + # self.process.wait(0) # clean up zombie? + self.process = None + print("session %s stopped" % (self.name,)) + + # (re)start autostarted sessions + if not self.process and "autostart" in self.name: + cmd = self.get_cmd() + self.process = subprocess.Popen(shlex.split(cmd)) + print("session %s autostart %s" % (self.name, cmd)) + + # kill any non-autostart scripts that have been unused too long + if self.process and "autostart" not in self.name: + if self.killafter is not None and self.killafter < time(): + self.process.terminate() + print("session %s autokill" % (self.name,)) + + + def collect_output(self): + # start it if not already running + if not self.process: + cmd = self.get_cmd() + self.process = subprocess.Popen(shlex.split(cmd)) + print("session %s start %s" % (self.name, cmd)) + + # reset the killafter time + if self.keepalive is not None: + self.killafter = time() + self.keepalive + + path = proc_path + "/" + self.proc_subdirname + "/__prometheus" + with open(path) as metrics: + return bytes(metrics.read(), 'utf-8') class SessionMgr: + """Represent the set of possible systemtap scripts that can be exported. Searches the $script_dir + once at startup.""" - def __init__(self): + def __init__(self, scdir, ka): self.counter = 0 - self.port = None - self.timeout = None self.sessions = {} - self.parse_cmdline() - self.run_autostart_scripts() - - def start_sess(self, script_name): - """ Begin execution of script and record start time """ - s = Session(script_name, self.get_new_id()) - self.sessions[script_name] = s - s.begin() - s.set_start_time() - print("Started %s via %s" % (s.name, s.cmd)) - - def parse_cmdline(self): - p = argparse.ArgumentParser(description='Systemtap-prometheus interoperation mechanism') - p.add_argument('-p', '--port', nargs=1, default=[9900], type=int) - p.add_argument('-t', '--timeout', nargs=1, default=[None], type=int) - - opts = p.parse_args() - self.port = opts.port[0] - self.timeout = opts.timeout[0] - - def run_autostart_scripts(self): - scripts = os.listdir(script_dir + "autostart/") - for name in scripts: - self.start_sess(name) - - def sess_started(self, name): - return name in self.sessions - - def get_new_id(self): - ret = self.counter - self.counter += 1 - return ret - - def check_timeouts(self): - term = [] + for n in os.listdir(scdir): + if n.endswith(".stp"): + self.counter += 1 + self.sessions[n] = Session(scdir, n, self.counter, ka) + + def poll(self, killthem): for (name, sess) in self.sessions.items(): - if ((sess.start_time is not None - and self.timeout is not None - and time() - sess.start_time >= self.timeout) - or sess.process.poll() is not None): - print("Terminating " + name) - sess.process.terminate() - term.append(name) - for name in term: - self.sessions[name].process.wait(1) - self.sessions.pop(name, None) + try: + sess.poll(killthem) + except Exception as e: + print("session %s poll failure %s" % (name, str(e))) + def session(self,name): + # NB: will throw if session with given name doesn't exist + return self.sessions[name] -class HTTPHandler(BaseHTTPRequestHandler): - sessmgr = SessionMgr() + # XXX: add facility for synthetic sessionmgr metrics - def set_headers(self, code, content_type): - self.send_response(code) - self.send_header('Content-type', content_type) - self.end_headers() - def send_metrics(self, sess): - metrics_path = sess.get_proc_path() - try: - with open(metrics_path) as metrics: - self.set_headers(200, 'text/plain') - self.wfile.write(bytes(metrics.read(), 'utf-8')) - # XXX: log metrics.size - except: - self.set_headers(501, 'text/plain') - self.wfile.write(bytes('Metrics currently unavailable', 'utf-8')) - sess.set_start_time() +class HTTPHandler(BaseHTTPRequestHandler): def send_msg(self, code, msg): - self.set_headers(code, 'text/plain') - self.wfile.write(bytes(msg, 'utf-8')) + self.send_response(code) + self.send_header('Content-type', 'text/plain') + self.end_headers() + self.wfile.write(msg) def do_GET(self): # remove the preceeding '/' from url name = urlparse(self.path).path[1:] - mgr = self.sessmgr - if mgr.sess_started(name): - # session is already running, send metrics - self.send_metrics(mgr.sessions[name]) - else: - # launch session - mgr.start_sess(name) - self.send_msg(301, "Refresh page to access metrics.") + try: + s = sessmgr.session(name) + except: + self.send_msg(404, + bytes("script not found", + 'utf-8')) + return + try: + promdata = s.collect_output() + self.send_msg(200, promdata) + except Exception as e: + self.send_msg(301, + bytes("cannot read script output, try again later, %s" % (str(e),), + 'utf-8')) + print("session %s collect-output failure %s" % (name, str(e))) + if __name__ == "__main__": - sessmgr = HTTPHandler.sessmgr - server_address = ('', sessmgr.port) + p = argparse.ArgumentParser(description='systemtap prometheus-exporter') + p.add_argument('-p', '--port', nargs=1, default=[9900], type=int) + p.add_argument('-s', '--scripts', nargs=1, default=[script_dir], type=str) + p.add_argument('-k', '--keepalive', nargs=1, default=[None], type=int) + + opts = p.parse_args() + scripts = opts.scripts[0] + port = opts.port[0] + keepalive = opts.keepalive[0] + + # NB: global + sessmgr = SessionMgr(scripts,keepalive) + + server_address = ('', port) httpd = HTTPServer(server_address, HTTPHandler) - httpd.timeout = 5 + httpd.timeout = 5 # parametrize? + print("Exporter initialization complete") - print("Listening on port %d" % sessmgr.port) - - while 1: - httpd.handle_request() - sessmgr.check_timeouts() + print("Listening on port %d" % port) + + try: + while True: + sessmgr.poll(False) + httpd.handle_request() + except: + sessmgr.poll(True) diff --git a/stap-exporter/stap-exporter.options b/stap-exporter/stap-exporter.options new file mode 100644 index 000000000..53dabf643 --- /dev/null +++ b/stap-exporter/stap-exporter.options @@ -0,0 +1,12 @@ +# OPTIONS + +#PORT= +PORT="-p 9900" + +# KEEPALIVE= +KEEPALIVE="-k 300" + +SCRIPTS= +#SCRIPTS="-s /path/to/your/stap/scripts" + +OPTIONS= diff --git a/stap-exporter/stap-exporter.service b/stap-exporter/stap-exporter.service index cd7d0142f..d4f4c6e9d 100644 --- a/stap-exporter/stap-exporter.service +++ b/stap-exporter/stap-exporter.service @@ -1,8 +1,10 @@ [Unit] -Description=stap-exporter +Description=Export metrics from designated systemtap scripts to Prometheus. [Service] -ExecStart=/usr/bin/stap-exporter +EnvironmentFile=-/etc/sysconfig/stap-exporter +ExecStart=/usr/sbin/stap-exporter $PORT $KEEPALIVE $SCRIPTS $OPTIONS +Restart=always [Install] WantedBy=multi-user.target diff --git a/systemtap.spec b/systemtap.spec index 78d5dc044..7b0936504 100644 --- a/systemtap.spec +++ b/systemtap.spec @@ -123,6 +123,7 @@ Source: ftp://sourceware.org/pub/systemtap/releases/systemtap-%{version}.tar.gz # Build* BuildRequires: gcc-c++ +BuildRequires: cpio BuildRequires: gettext-devel BuildRequires: pkgconfig(nss) BuildRequires: pkgconfig(avahi-client) @@ -463,14 +464,14 @@ that probe python 3 processes. %endif %if %{with_python3} -%package stap-exporter +%package exporter Summary: Systemtap-prometheus interoperation mechanism Group: Development/System License: GPLv2+ URL: http://sourceware.org/systemtap/ Requires: systemtap-runtime = %{version}-%{release} -%description stap-exporter +%description exporter This package includes files for a systemd service that manages systemtap sessions and relays prometheus metrics from the sessions to remote requesters on demand. @@ -950,11 +951,22 @@ fi exit 0 %if %{with_python3} -%preun stap-exporter -/bin/systemctl stop stap-exporter.service >/dev/null 2>&1 || : -/bin/systemctl disable stap-exporter.service >/dev/null 2>&1 || : -%endif +%if %{with_systemd} +%preun exporter +if [ $1 = 0 ] ; then + /bin/systemctl stop stap-exporter.service >/dev/null 2>&1 || : + /bin/systemctl disable stap-exporter.service >/dev/null 2>&1 || : +fi +exit 0 +%postun exporter +# Restart service if this is an upgrade rather than an uninstall +if [ "$1" -ge "1" ]; then + /bin/systemctl condrestart stap-exporter >/dev/null 2>&1 || : +fi +exit 0 +%endif +%endif %post # Remove any previously-built uprobes.ko materials @@ -1242,12 +1254,12 @@ done %endif %if %{with_python3} -%files stap-exporter -%{_sysconfdir}/systemtap/stap-exporter +%files exporter +%{_sysconfdir}/stap-exporter +%{_sysconfdir}/sysconfig/stap-exporter %{_unitdir}/stap-exporter.service %{_mandir}/man8/stap-exporter.8* %{_sbindir}/stap-exporter -%{_etcdir}/stap-exporter %endif # ------------------------------------------------------------------------ diff --git a/tapset-procfs.cxx b/tapset-procfs.cxx index 142479931..e7d294f18 100644 --- a/tapset-procfs.cxx +++ b/tapset-procfs.cxx @@ -631,20 +631,6 @@ procfs_builder::build(systemtap_session & sess, throw SEMANTIC_ERROR (_("maxsize must be greater than 0")); } - if (path == "__prometheus") - { - // Intended for use by stap-exporter. Replace path with the script name. - string p = sess.script_file; - - if (p.empty()) - throw SEMANTIC_ERROR (_("Script name must be specified")); - - if (p.find_last_of('/') == p.length() - 1) - p = p.substr(0, p.length() - 1); - - path = p.substr(p.find_last_of('/') + 1); - } - // If no procfs path, default to "command". The runtime will do // this for us, but if we don't do it here, we'll think the // following 2 probes are attached to different paths: diff --git a/testsuite/systemtap.examples/profiling/syscallsbypid.stp b/testsuite/systemtap.examples/profiling/syscallsbypid.stp index e60f2afff..9b1af01a1 100755 --- a/testsuite/systemtap.examples/profiling/syscallsbypid.stp +++ b/testsuite/systemtap.examples/profiling/syscallsbypid.stp @@ -11,12 +11,23 @@ # but are otherwise harmless. global arr%[20000] +global procname% + +probe kernel.trace("sys_enter") { + arr[pid(), $id]++ + procname[pid()] = execname() +} -probe kernel.trace("sys_enter") { arr[pid(), $id]++ } +probe scheduler.process_exit { + delete arr[pid(),*] + delete procname[pid()] +} -probe scheduler.process_exit { delete arr[pid(),*] } +function process_name(pid) { + return sprintf("%s[%d]", procname[pid], pid) +} probe prometheus { - @prometheus_dump_array2_map(arr, "count", "pid", "syscall", - sprint, sprint, syscall_name ) + @prometheus_dump_array2_map(arr, "count", "pid", "syscall", + sprint, process_name, syscall_name ) }