From: Sagar Patel Date: Thu, 29 Aug 2019 15:40:21 +0000 (-0400) Subject: PR23285 (2): enable prometheus-exporter type scripts on stapbpf X-Git-Tag: release-4.2~42 X-Git-Url: https://sourceware.org/git/?a=commitdiff_plain;h=0801a5e1814d8b73413ac47b2de4e2c00ff7a3a5;p=systemtap.git PR23285 (2): enable prometheus-exporter type scripts on stapbpf The eBPF backend now supports prometheus-exporter scripts. This implementation introduces character escaping macros enabling the array dump macros on stapbpf. However, there are some issues with foreach loops which are used in the macros (PR24953). 1) Developed character escaping macros helper functions. 2) Added prometheus probe tapset for stapbpf. 3) Added stapbpf procfs file path to stap-exporter. 4) Introduced a sample stapbpf prometheus-exporter script in EXAMPLES. 5) Updated NEWS. --- diff --git a/NEWS b/NEWS index d2115bc39..1177c8362 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,7 @@ * What's new in version 4.2, PRERELEASE +- The stapbpf backend now supports stap-exporter extensions. + - The stapbpf backend now supports procfs probes. The implementation uses FIFO special files in /var/tmp/systemtap-$EFFUSER/MODNAME instead of the proc filesystem files. diff --git a/bpf-internal.h b/bpf-internal.h index a3d0c381f..db8ede7f1 100644 --- a/bpf-internal.h +++ b/bpf-internal.h @@ -180,7 +180,9 @@ const opcode BPF_LD_MAP = BPF_LD | BPF_IMM | BPF_DW | (BPF_PSEUDO_MAP_FD << 8); FN(set_procfs_value), \ FN(append_procfs_value), \ FN(get_procfs_value), \ - FN(str_concat), + FN(str_concat), \ + FN(text_str), \ + FN(string_quoted), const bpf_func_id BPF_FUNC_map_get_next_key = (bpf_func_id) -1; const bpf_func_id BPF_FUNC_sprintf = (bpf_func_id) -2; @@ -191,6 +193,8 @@ const bpf_func_id BPF_FUNC_set_procfs_value = (bpf_func_id) -6; const bpf_func_id BPF_FUNC_append_procfs_value = (bpf_func_id) -7; const bpf_func_id BPF_FUNC_get_procfs_value = (bpf_func_id) -8; const bpf_func_id BPF_FUNC_str_concat = (bpf_func_id) -9; +const bpf_func_id BPF_FUNC_text_str = (bpf_func_id) -10; +const bpf_func_id BPF_FUNC_string_quoted = (bpf_func_id) -11; struct insn { diff --git a/stap-exporter/default/EXAMPLE-BPF b/stap-exporter/default/EXAMPLE-BPF new file mode 100755 index 000000000..c8f1798ec --- /dev/null +++ b/stap-exporter/default/EXAMPLE-BPF @@ -0,0 +1,4 @@ +#! /bin/sh + +# link to this generic shell script to invoke a non-guru stap example +exec stap --bpf -v --example $@ `basename $0 | sed -e s,autostart-,,` diff --git a/stap-exporter/default/syscallsrw.stp b/stap-exporter/default/syscallsrw.stp new file mode 120000 index 000000000..3a166173a --- /dev/null +++ b/stap-exporter/default/syscallsrw.stp @@ -0,0 +1 @@ +EXAMPLE-BPF \ No newline at end of file diff --git a/stap-exporter/stap-exporter.in b/stap-exporter/stap-exporter.in index 8d07a80bf..e35d18acb 100644 --- a/stap-exporter/stap-exporter.in +++ b/stap-exporter/stap-exporter.in @@ -31,9 +31,9 @@ from time import time script_dir = "%sysconfdir%/stap-exporter" proc_basename = "__stap_exporter_" + str(os.getpid()) # permit concurrent exporters -proc_path = "/proc/systemtap" sessmgr = None - +proc_path_lkm = "/proc/systemtap" +proc_path_bpf = "/var/tmp/systemtap-root" class Session: """Represent a single systemtap script found $script_dir, whether or not @@ -91,9 +91,21 @@ class Session: if self.keepalive is not None: self.killafter = time() + self.keepalive - path = proc_path + "/" + self.proc_subdirname + "/__prometheus" - with open(path) as metrics: - return bytes(metrics.read(), 'utf-8') + path_lkm = proc_path_lkm + "/" + self.proc_subdirname + "/__prometheus" + path_bpf = proc_path_bpf + "/" + self.proc_subdirname + "/__prometheus" + + try: + with open(path_lkm) as metrics: + return bytes(metrics.read(), 'utf-8') + except FileNotFoundError: + pass + + try: + with open(path_bpf) as metrics: + return bytes(metrics.read(), 'utf-8') + except FileNotFoundError: + raise Exception("[Error] Unable to find procfs files under search paths. " + "Try again once the script has created the procfs files.") class SessionMgr: """Represent the set of possible systemtap scripts that can be exported. Searches the $script_dir @@ -179,6 +191,8 @@ if __name__ == "__main__": httpd.timeout = 5 # parametrize? print("listening on port %d" % port) + print("procfs files will be searched under: %s and %s" % (proc_path_lkm, proc_path_bpf)) + try: while True: sessmgr.poll() diff --git a/stapbpf/bpfinterp.cxx b/stapbpf/bpfinterp.cxx index 9298b1a2f..ffdbdd108 100644 --- a/stapbpf/bpfinterp.cxx +++ b/stapbpf/bpfinterp.cxx @@ -397,6 +397,48 @@ bpf_sprintf(std::vector &strings, char *fstr, return reinterpret_cast(strings.back().c_str()); } +uint64_t +bpf_text_str(std::vector &strings, char* input, bool quoted) +{ + std::string str(input); + std::stringstream stream; + + for (std::string::iterator it = str.begin(); it != str.end(); ++it) + { + unsigned char c = *it; + unsigned int i = c; + + if (std::isprint(c) && i < 128 && c != '\\' && c != '"') + stream << c; + else + { + stream << '\\'; + switch (c) { + case '\0': stream << '0'; break; // Not handled by translate_escapes + case '\a': stream << 'a'; break; // Not handled by translate_escapes + case '\b': stream << 'b'; break; // Not handled by translate_escapes + case '\f': stream << 'f'; break; + case '\n': stream << 'n'; break; + case '\r': stream << 'r'; break; + case '\t': stream << 't'; break; + case '\v': stream << 'v'; break; + case '"': stream << '"'; break; + case '\\': stream << '\\'; break; + default: + stream << "x" << std::setfill('0') << std::setw(2) << std::hex << i; + break; + } + } + } + + if (quoted) + strings.push_back("\"" + stream.str() + "\""); + else + strings.push_back(stream.str()); + + return reinterpret_cast(strings.back().c_str()); +} + uint64_t bpf_str_concat(std::vector &strings, char* left, char* right) { @@ -937,6 +979,12 @@ bpf_interpret(size_t ninsns, const struct bpf_insn insns[], dr = bpf_sprintf(strings, as_str(regs[1]), regs[3], regs[4], regs[5]); break; + case bpf::BPF_FUNC_text_str: + dr = bpf_text_str(strings, as_str(regs[1]), false); + break; + case bpf::BPF_FUNC_string_quoted: + dr = bpf_text_str(strings, as_str(regs[1]), true); + break; case bpf::BPF_FUNC_str_concat: dr = bpf_str_concat(strings, as_str(regs[1]), as_str(regs[2])); @@ -953,7 +1001,7 @@ bpf_interpret(size_t ninsns, const struct bpf_insn insns[], case bpf::BPF_FUNC_gettimeofday_ns: dr = bpf_gettimeofday_ns(); break; - case bpf::BPF_FUNC_get_target: + case bpf::BPF_FUNC_get_target: dr = bpf_get_target(); break; case bpf::BPF_FUNC_set_procfs_value: diff --git a/stapbpf/stapbpf.cxx b/stapbpf/stapbpf.cxx index 74f9f61cf..9b8037ec2 100644 --- a/stapbpf/stapbpf.cxx +++ b/stapbpf/stapbpf.cxx @@ -1861,8 +1861,9 @@ procfs_read_event_loop (procfsprobe_data* data, bpf_transport_context* uctx) // Run the probe and collect the message. bpf_interpret(prog->d_size / sizeof(bpf_insn), static_cast(prog->d_buf), uctx); - // Make a copy of the message. + // Make a copy of the message and reset it. std::string msg = uctx->procfs_msg; + uctx->procfs_msg.clear(); procfs_lock.unlock(); diff --git a/tapset/prometheus.stp b/tapset/prometheus.stp index cb12ceec0..170ca3687 100644 --- a/tapset/prometheus.stp +++ b/tapset/prometheus.stp @@ -1 +1,7 @@ -probe prometheus = procfs("__prometheus").read.maxsize(0x10000) { } +probe prometheus = +%( runtime != "bpf" %? + procfs("__prometheus").read.maxsize(0x10000) +%: + procfs("__prometheus").read +%) +{ } diff --git a/tapset/string.stp b/tapset/string.stp index f9792ea73..df2c7a708 100644 --- a/tapset/string.stp +++ b/tapset/string.stp @@ -100,12 +100,20 @@ function strpos:long(s1:string,s2:string) * printable are replaced by the corresponding escape sequence in the * returned string. */ -function text_str:string(input:string) -%{ /* pure */ /* unprivileged */ /* unmodified-fnargs */ - if (_stp_text_str(STAP_RETVALUE, STAP_ARG_input, 0, 0, 0, 0, 0) < 0) { - STAP_RETVALUE[0] = '\0'; - } -%} +function text_str:string(input:string) +%( runtime != "bpf" %? + %{ /* pure */ /* unprivileged */ /* unmodified-fnargs */ + if (_stp_text_str(STAP_RETVALUE, STAP_ARG_input, 0, 0, 0, 0, 0) < 0) { + STAP_RETVALUE[0] = '\0'; + } + %} +%: + %{ /* bpf */ + 0xbf, 1, $input, -, -; /* mov r1, $input */ + 0x85, 0, 0, 0, -10; /* call BPF_FUNC_TEXT_STR */ + 0xbf, $$, 0, -, -; /* return r0 */ + %} +%) /** * sfunction text_strn - Escape any non-printable chars in a string @@ -203,9 +211,17 @@ function isdigit:long(str:string) * string. Note that the string will be surrounded by double quotes. */ function string_quoted:string (str:string) -%{ /* pure */ /* unmodified-fnargs */ - /* This can't fail, since the input string is already in stap context space. */ - (void) _stp_text_str(STAP_RETVALUE, - (char *)(uintptr_t)STAP_ARG_str, - MAXSTRINGLEN, MAXSTRINGLEN, 1, 0, 0); -%} +%( runtime != "bpf" %? + %{ /* pure */ /* unmodified-fnargs */ + /* This can't fail, since the input string is already in stap context space. */ + (void) _stp_text_str(STAP_RETVALUE, + (char *)(uintptr_t)STAP_ARG_str, + MAXSTRINGLEN, MAXSTRINGLEN, 1, 0, 0); + %} +%: + %{ /* bpf */ + 0xbf, 1, $str, -, -; /* mov r1, $input */ + 0x85, 0, 0, 0, -11; /* call BPF_FUNC_STRING_QUOTED */ + 0xbf, $$, 0, -, -; /* return r0 */ + %} +%) diff --git a/testsuite/systemtap.examples/index.html b/testsuite/systemtap.examples/index.html index 77223cff3..519ecad5b 100644 --- a/testsuite/systemtap.examples/index.html +++ b/testsuite/systemtap.examples/index.html @@ -38,7 +38,7 @@
  • By Keyword
    • -

    For systemtap version 4.1.

    Best Examples

    +

    For systemtap version 4.2.

    Best Examples

    -

    For systemtap version 4.1.

    Examples by Keyword

    -

    _BEST(17) APPLICATION(2) BACKTRACE(8) BPF(1) CALLGRAPH(4) CONTAINER(1) DIAGRAM(1) DISK(5) FILE(2) FILESYSTEM(4) FORMAT(2) FUNCTION(4) FUTEX(2) GURU(19) INTERACTIVE(2) INTERRUPT(3) IO(30) JSON(2) KVM(3) LIMITS(3) LOCKING(6) MEMORY(21) MONITORING(9) NANOSLEEP(1) NETWORK(21) NFS(5) NUMA(1) PACKETS(1) PROCESS(27) PROFILING(24) PROMETHEUS(5) QEMU(2) REGEX(1) RETRANSMISSION(1) SCHEDULER(9) SCSI(1) SECURITY(13) SIGNALS(5) SIMPLE(10) SOCKET(8) SPECULATION(1) STAPGAMES(5) STATISTICS(10) SYSCALL(18) TCP(4) THREAD(2) TIME(4) TRACE(11) TRACEPOINT(8) TRAFFIC(8) TTY(1) VIRTUALIZATION(3) WATCHDOG(1) WATCHPOINT(2)

    +

    For systemtap version 4.2.

    Examples by Keyword

    +

    _BEST(17) APPLICATION(2) BACKTRACE(8) BPF(9) CALLGRAPH(4) CONTAINER(1) DIAGRAM(1) DISK(5) DYNINST(1) FILE(2) FILESYSTEM(4) FORMAT(2) FUNCTION(4) FUTEX(2) GURU(19) INTERACTIVE(2) INTERRUPT(3) IO(30) JSON(2) KVM(3) LIMITS(3) LOCKING(6) MEMORY(20) MONITORING(9) NANOSLEEP(1) NETWORK(21) NFS(5) NUMA(1) PACKETS(1) PROCESS(27) PROFILING(24) PROMETHEUS(6) QEMU(2) REGEX(1) RETRANSMISSION(1) SCHEDULER(9) SCSI(1) SECURITY(13) SIGNALS(5) SIMPLE(10) SOCKET(8) SPECULATION(1) STAPGAMES(5) STATISTICS(10) SYSCALL(19) TCP(4) THREAD(2) TIME(4) TRACE(11) TRACEPOINT(8) TRAFFIC(8) TTY(1) VIRTUALIZATION(3) WATCHDOG(1) WATCHPOINT(2)

    _BEST

    • general/helloworld.stp - SystemTap "Hello World" Program
      -keywords: _BEST SIMPLE
      +keywords: _BEST SIMPLE BPF

      A basic "Hello World" program implemented in SystemTap script. It prints out "hello world" message and then immediately exits.

      # stap helloworld.stp

    • general/para-callgraph.stp - Callgraph Tracing with Arguments
      keywords: _BEST TRACE CALLGRAPH
      @@ -97,7 +97,7 @@ keywords: _BEST APPLICATION
      • apps/libguestfs_log.stp - Trace libguestfs startup
        -keywords: APPLICATION TRACE
        +keywords: APPLICATION TRACE BPF

        The libguestfs_log.stp script prints a log of when various libgueststartup steps are encountered. The first column is microseconds since the script started. The second column is the time elapsed in microseconds between the previous and current events and the third column is the event name.

        # stap libguestfs_log.stp -T 1

      • apps/stap_time.stp - Provide elapsed times for Passes of SystemTap script compilation
        keywords: APPLICATION TRACE
        @@ -133,9 +133,33 @@ keywords: _BEST BPF
          -
        • memory/cachestat.stp - Count Page Cache Hits and Misses
          +
        • apps/libguestfs_log.stp - Trace libguestfs startup
          +keywords: APPLICATION TRACE BPF
          +

          The libguestfs_log.stp script prints a log of when various libgueststartup steps are encountered. The first column is microseconds since the script started. The second column is the time elapsed in microseconds between the previous and current events and the third column is the event name.

          # stap libguestfs_log.stp -T 1

          +
        • general/ansi_colors.stp - Color Table for ansi_set_color()
          +keywords: FORMAT BPF
          +

          The script prints a table showing the available color combinations for the ansi_set_color() function in the ansi.stp tapset.

          # stap ansi_colors.stp

          +
        • general/ansi_colors2.stp - Show Attribues in Table for ansi_set_color()
          +keywords: FORMAT BPF
          +

          The script prints a table showing the available attributes (bold, underline, and inverse) with color combinations for the ans_set_color() function in the ansi.stp tapset.

          # stap ansi_colors2.stp

          +
        • general/helloworld.stp - SystemTap "Hello World" Program
          +keywords: _BEST SIMPLE BPF
          +

          A basic "Hello World" program implemented in SystemTap script. It prints out "hello world" message and then immediately exits.

          # stap helloworld.stp

          +
        • memory/cachestat.stp - Count Page Cache Hits and Misses
          +keywords: MEMORY BPF
          +

          Monitors hits and misses to the page cache and reports a count every 5 seconds. Based on a bpftrace tool by David Valin.

          # stap cachestat.stp -T 1

          +
        • memory/mmfilepage.stp - Track Virtual Memory System Actions on File Backed Pages
          keywords: MEMORY BPF
          -

          Monitors hits and misses to the page cache and reports a count every 5 seconds. Based on a bpftrace tool by David Valin.

          # stap --bpf cachestat.stp -T 1

          +

          The mmfilepage.stp script uses the virtual memory tracepoints available in some kernels to track the number of faults, copy on writes mapping, and unmapping operations for file backed pages. When the script is terminated the counts are printed for each process that allocated pages while the script was running. The mmfilepage.stp script is useful in debugging leaks in the mapped file regions of a process.

          # stap mmfilepage.stp -T 1

          +
        • memory/mmwriteback.stp - Track Virtual Memory System Writing to Disk
          +keywords: MEMORY BPF
          +

          The mmwriteback.stp script uses the virtual memory tracepoints available in some kernels to report all of the file writebacks that occur form kupdate, pdflush and kjournald while the script is running. It's useful in determining where writes are coming from on a supposedly idle system that is experiencing unexpected IO.

          # stap mmwriteback.stp -T 1

          +
        • network/sk_stream_wait_memory.stp - Track Start and Stop of Processes Due to Network Buffer Space
          +keywords: NETWORK TCP PROCESS BPF
          +

          The sk_stream-wait_memory.stp prints a time stamp, executable, and pid each time a process blocks due to the send buffer being full. A similar entry is printed each time a process continues because there is room in the buffer.

          # stap sk_stream_wait_memory.stp -T 1

          +
        • profiling/syscallsrw.stp - Provide a tally of read and write syscalls run on the system
          +keywords: PROMETHEUS SYSCALL BPF
          +

          The syscall_read_write.stp script tallies the read and write syscalls. This is a demo script for stapbpf prometheus-exporter scripts.

        CALLGRAPH

          @@ -183,6 +207,12 @@ keywords: IO NFS DISK

          The nfsd_unlink.stp script lists the ip address and file name each time time a file is being removed or unlinked by the nfsd. This script is run on the nfs server.

          # stap nfsd_unlink.stp -T 1

        +

        DYNINST

        +
          +
        • apps/php-trace.stp - Tracing of PHP code execution
          +keywords: TRACE DYNINST
          +

          Trace of executing PHP code using the enabled markers.

          # stap php-trace.stp -c 'php -f hello.php'

          +

        FILE

        • io/iotime.stp - Trace Time Spent in Read and Write for Files
          @@ -210,10 +240,10 @@ keywords: IO FORMAT
          • general/ansi_colors.stp - Color Table for ansi_set_color()
            -keywords: FORMAT
            +keywords: FORMAT BPF

            The script prints a table showing the available color combinations for the ansi_set_color() function in the ansi.stp tapset.

            # stap ansi_colors.stp

          • general/ansi_colors2.stp - Show Attribues in Table for ansi_set_color()
            -keywords: FORMAT
            +keywords: FORMAT BPF

            The script prints a table showing the available attributes (bold, underline, and inverse) with color combinations for the ans_set_color() function in the ansi.stp tapset.

            # stap ansi_colors2.stp

          FUNCTION

          @@ -477,11 +507,8 @@ keywords: STATISTICS STATISTICS MEMORY INTERACTIVE

          This script prints the size of a type, based on dwarf debuginfo for any kernel or userspace module, or trial-compilation of a given header file name. Types and corresponding locations are provided to the script at runtime via keyboard input. The format of the input is identical to that of sizeof.stp (see script source for more information). Types and locations can be repeatedly given until the process is terminated.

          sample usage in general/sizeof_interactive.txt

        • memory/cachestat.stp - Count Page Cache Hits and Misses
          -keywords: MEMORY
          -

          Monitors hits and misses to the page cache and reports a count every 5 seconds. Based on a bpftrace tool by David Valin.

          # stap cachestat.stp -T 1

          -
        • memory/cachestat.stp - Count Page Cache Hits and Misses
          keywords: MEMORY BPF
          -

          Monitors hits and misses to the page cache and reports a count every 5 seconds. Based on a bpftrace tool by David Valin.

          # stap --bpf cachestat.stp -T 1

          +

          Monitors hits and misses to the page cache and reports a count every 5 seconds. Based on a bpftrace tool by David Valin.

          # stap cachestat.stp -T 1

        • memory/glibc-malloc.stp - Overview glibc malloc internal operations
          keywords: MEMORY PROCESS

          This script reports on internal statistics of the glibc malloc implementation, as used by a process restricted by stap -x/-c

          # stap glibc-malloc.stp -c 'stap --dump-functions'

          @@ -514,13 +541,13 @@ keywords: MEMORY MEMORY

          The mmanonpage.stp script uses the virtual memory tracepoints available in some kernels to track the number of faults, user space frees, page ins, copy on writes and unmaps for anonymous pages. When the script is terminated the counts are printed for each process that allocated pages while the script was running. This script displays the anonymous page statistics for each process that ran while the script is active. It's useful in debugging leaks in the anonymous regions of a process.

          # stap mmanonpage.stp -T 1

        • memory/mmfilepage.stp - Track Virtual Memory System Actions on File Backed Pages
          -keywords: MEMORY
          +keywords: MEMORY BPF

          The mmfilepage.stp script uses the virtual memory tracepoints available in some kernels to track the number of faults, copy on writes mapping, and unmapping operations for file backed pages. When the script is terminated the counts are printed for each process that allocated pages while the script was running. The mmfilepage.stp script is useful in debugging leaks in the mapped file regions of a process.

          # stap mmfilepage.stp -T 1

        • memory/mmreclaim.stp - Track Virtual Memory System Page Reclamation
          keywords: MEMORY

          The mmreclaim.stp script uses the virtual memory tracepoints available in some kernels to track page reclaim activity that occurred while the script was running. It's useful in debugging performance problems that occur due to page reclamation.

          # stap mmreclaim.stp -T 1

        • memory/mmwriteback.stp - Track Virtual Memory System Writing to Disk
          -keywords: MEMORY
          +keywords: MEMORY BPF

          The mmwriteback.stp script uses the virtual memory tracepoints available in some kernels to report all of the file writebacks that occur form kupdate, pdflush and kjournald while the script is running. It's useful in determining where writes are coming from on a supposedly idle system that is experiencing unexpected IO.

          # stap mmwriteback.stp -T 1

        • memory/numa_faults.stp - Summarize Process Misses across NUMA Nodes
          keywords: MEMORY NUMA
          @@ -608,7 +635,7 @@ keywords: NETWORK NETWORK TRAFFIC

          The packet_contents.stp script displays the length of each network packet and its contents in both hexadecimal and ASCII. Systemtap strings are MAXSTRINGLEN in length by default which may not be enough for larger packets. In order to print larger packets, this limit can be increased by passing in the "-DMAXSTRINGLEN=65536" command line option.

          sample usage in network/packet_contents.txt

        • network/sk_stream_wait_memory.stp - Track Start and Stop of Processes Due to Network Buffer Space
          -keywords: NETWORK TCP PROCESS
          +keywords: NETWORK TCP PROCESS BPF

          The sk_stream-wait_memory.stp prints a time stamp, executable, and pid each time a process blocks due to the send buffer being full. A similar entry is printed each time a process continues because there is room in the buffer.

          # stap sk_stream_wait_memory.stp -T 1

        • network/socket-trace.stp - Trace Functions Called in Network Socket Code
          keywords: NETWORK SOCKET
          @@ -683,7 +710,7 @@ keywords: MEMORY NETWORK SOCKET PROCESS

          The connect_stat.stp script prints a task's entire ancestry (parent process name/uid/gid) whenever it attempts an outgoing socket connection to a given IP address.

          # stap connect_stat.stp 127.0.0.1 -T 1

        • network/sk_stream_wait_memory.stp - Track Start and Stop of Processes Due to Network Buffer Space
          -keywords: NETWORK TCP PROCESS
          +keywords: NETWORK TCP PROCESS BPF

          The sk_stream-wait_memory.stp prints a time stamp, executable, and pid each time a process blocks due to the send buffer being full. A similar entry is printed each time a process continues because there is room in the buffer.

          # stap sk_stream_wait_memory.stp -T 1

        • process/cycle_thief.stp - Track IRQ's and Other Processes Stealing Cycles from a Task
          keywords: _BEST PROCESS SCHEDULER TIME TRACEPOINT INTERRUPT
          @@ -846,6 +873,9 @@ keywords: PROMETHEUS profiling/syscallsbypid.stp - Provide a per-process syscall tally on the system
          keywords: PROMETHEUS PROCESS SYSCALL TRACEPOINT

          The syscallsbypid.stp script tallies each syscall for each running process. This information can be useful to determine the activity of various processes on the system. The script makes the information available via procfs in Prometheus readable format. When a process exits its data will be eliminated from the prometheus output. To avoid exceeding the storage limitations of SystemTap older entries maybe overwritten by newer entries. This can lead to some active process syscall counts disappearing and/or later reappearing with a lower value. Also note that the script does not properly name syscalls for 32-bit applications running on 64-bit machines.

          # stap syscallsbypid.stp -T 1

          +
        • profiling/syscallsrw.stp - Provide a tally of read and write syscalls run on the system
          +keywords: PROMETHEUS SYSCALL BPF
          +

          The syscall_read_write.stp script tallies the read and write syscalls. This is a demo script for stapbpf prometheus-exporter scripts.

        QEMU

          @@ -970,7 +1000,7 @@ keywords: SIGNALS
          keywords: SIMPLE TRACE CALLGRAPH

          Print a timed per-thread microsecond-timed nested callgraph. The first parameter names the function probe points to trace.

          sample usage in general/callgraph.txt

        • general/helloworld.stp - SystemTap "Hello World" Program
          -keywords: _BEST SIMPLE
          +keywords: _BEST SIMPLE BPF

          A basic "Hello World" program implemented in SystemTap script. It prints out "hello world" message and then immediately exits.

          # stap helloworld.stp

        • general/key.stp - make keyboard noises
          keywords: SIMPLE
          @@ -1135,6 +1165,9 @@ keywords: PROMETHEUS profiling/syscallsbypid.stp - Provide a per-process syscall tally on the system
          keywords: PROMETHEUS PROCESS SYSCALL TRACEPOINT

          The syscallsbypid.stp script tallies each syscall for each running process. This information can be useful to determine the activity of various processes on the system. The script makes the information available via procfs in Prometheus readable format. When a process exits its data will be eliminated from the prometheus output. To avoid exceeding the storage limitations of SystemTap older entries maybe overwritten by newer entries. This can lead to some active process syscall counts disappearing and/or later reappearing with a lower value. Also note that the script does not properly name syscalls for 32-bit applications running on 64-bit machines.

          # stap syscallsbypid.stp -T 1

          +
        • profiling/syscallsrw.stp - Provide a tally of read and write syscalls run on the system
          +keywords: PROMETHEUS SYSCALL BPF
          +

          The syscall_read_write.stp script tallies the read and write syscalls. This is a demo script for stapbpf prometheus-exporter scripts.

        • profiling/ucalls.stp - Profile method invocations in Java, Perl, Php, Python, Ruby, and Tcl scripts
          keywords: PROFILING SYSCALL

          The ucalls.stp script is modeled after the BCC ucalls script (https://github.com/iovisor/bcc/blob/master/tools/lib/ucalls.py) by Sasha Goldshtein. The ucalls.stp script monitors the process indicated by the -x or -c option. When the scripts exits it prints out information about the number of times that each method is invoked for code written in Java, Perl, Php, Python, Ruby, and Tcl. Include the word "syscalls" on the command line to count syscalls invoked by the process. If you want latency information, include "latency" on the command line. Note that the latency option does not work for recursive functions

          # stap ucalls.stp syscalls latency -c "python -c 'print(\"hello world\")' \
@@ -1143,7 +1176,7 @@ keywords: PROFILING  TCP
 
            
 
          • network/sk_stream_wait_memory.stp - Track Start and Stop of Processes Due to Network Buffer Space
            
-keywords: NETWORK TCP PROCESS 
            
+keywords: NETWORK TCP PROCESS BPF 
            
 
            The sk_stream-wait_memory.stp prints a time stamp, executable, and pid each time a process blocks due to the send buffer being full. A similar entry is printed each time a process continues because there is room in the buffer.
            # stap sk_stream_wait_memory.stp -T 1
            
 
          • network/tcp_connections.stp - Track Creation of Incoming TCP Connections
            
 keywords: NETWORK TCP SOCKET 
            
@@ -1185,10 +1218,10 @@ keywords: PROCESS TRACE 
            
 
            The gmalloc_watch.stp script from Colin Walters' blog (https://blog.verbum.org/2011/03/19/analyzing-memory-use-with-systemtap/) traces the allocation of glib2 memory using the markers in glib2.
            # stap gmalloc_watch.stp -T 1
            
 
          • apps/libguestfs_log.stp - Trace libguestfs startup
            
-keywords: APPLICATION TRACE 
            
+keywords: APPLICATION TRACE BPF 
            
 
            The libguestfs_log.stp script prints a log of when various libgueststartup steps are encountered.   The first column is microseconds since the script started.  The second column is the time elapsed in microseconds between the previous and current events and the third column is the event name.
            # stap libguestfs_log.stp -T 1
            
 
          • apps/php-trace.stp - Tracing of PHP code execution
            
-keywords: TRACE 
            
+keywords: TRACE DYNINST 
            
 
            Trace of executing PHP code using the enabled markers.
            # stap php-trace.stp -c 'php -f hello.php'
            
 
          • apps/stap_time.stp - Provide elapsed times for Passes of SystemTap script compilation
            
 keywords: APPLICATION TRACE 
            
diff --git a/testsuite/systemtap.examples/keyword-index.txt b/testsuite/systemtap.examples/keyword-index.txt
index f07fbf3cc..0ffbfe781 100644
--- a/testsuite/systemtap.examples/keyword-index.txt
+++ b/testsuite/systemtap.examples/keyword-index.txt
@@ -2,12 +2,12 @@ SYSTEMTAP EXAMPLES INDEX BY KEYWORD
 (see also index.txt)
 
 
-For systemtap version 4.1.
+For systemtap version 4.2.
 
 = _BEST =
 
 general/helloworld.stp - SystemTap "Hello World" Program
-keywords: _best simple
+keywords: _best simple bpf
 
   A basic "Hello World" program implemented in SystemTap script. It
   prints out "hello world" message and then immediately exits.
@@ -214,7 +214,7 @@ keywords: _best virtualization kvm
 = APPLICATION =
 
 apps/libguestfs_log.stp - Trace libguestfs startup
-keywords: application trace
+keywords: application trace bpf
 
   The libguestfs_log.stp script prints a log of when various
   libgueststartup steps are encountered.   The first column is
@@ -337,13 +337,98 @@ keywords: _best profiling backtrace
 
 = BPF =
 
+apps/libguestfs_log.stp - Trace libguestfs startup
+keywords: application trace bpf
+
+  The libguestfs_log.stp script prints a log of when various
+  libgueststartup steps are encountered.   The first column is
+  microseconds since the script started.  The second column is the time
+  elapsed in microseconds between the previous and current events and
+  the third column is the event name.
+
+  # stap libguestfs_log.stp -T 1
+
+
+general/ansi_colors.stp - Color Table for ansi_set_color()
+keywords: format bpf
+
+  The script prints a table showing the available color combinations
+  for the ansi_set_color() function in the ansi.stp tapset.
+
+  # stap ansi_colors.stp
+
+
+general/ansi_colors2.stp - Show Attribues in Table for ansi_set_color()
+keywords: format bpf
+
+  The script prints a table showing the available attributes (bold,
+  underline, and inverse) with color combinations for the
+  ans_set_color() function in the ansi.stp tapset.
+
+  # stap ansi_colors2.stp
+
+
+general/helloworld.stp - SystemTap "Hello World" Program
+keywords: _best simple bpf
+
+  A basic "Hello World" program implemented in SystemTap script. It
+  prints out "hello world" message and then immediately exits.
+
+  # stap helloworld.stp
+
+
 memory/cachestat.stp - Count Page Cache Hits and Misses
 keywords: memory bpf
 
   Monitors hits and misses to the page cache and reports a count every
   5 seconds. Based on a bpftrace tool by David Valin.
 
-  # stap --bpf cachestat.stp -T 1
+  # stap cachestat.stp -T 1
+
+
+memory/mmfilepage.stp - Track Virtual Memory System Actions on File Backed Pages
+keywords: memory bpf
+
+  The mmfilepage.stp script uses the virtual memory tracepoints
+  available in some kernels to track the number of faults, copy on
+  writes mapping, and unmapping operations for file backed pages. When
+  the script is terminated the counts are printed for each process that
+  allocated pages while the script was running. The mmfilepage.stp
+  script is useful in debugging leaks in the mapped file regions of a
+  process.
+
+  # stap mmfilepage.stp -T 1
+
+
+memory/mmwriteback.stp - Track Virtual Memory System Writing to Disk
+keywords: memory bpf
+
+  The mmwriteback.stp script uses the virtual memory tracepoints
+  available in some kernels to report all of the file writebacks that
+  occur form kupdate, pdflush and kjournald while the script is
+  running.  It's useful in determining where writes are coming from on
+  a supposedly idle system that is experiencing unexpected IO.
+
+  # stap mmwriteback.stp -T 1
+
+
+network/sk_stream_wait_memory.stp - Track Start and Stop of Processes Due to Network Buffer Space
+keywords: network tcp process bpf
+
+  The sk_stream-wait_memory.stp prints a time stamp, executable, and
+  pid each time a process blocks due to the send buffer being full. A
+  similar entry is printed each time a process continues because there
+  is room in the buffer.
+
+  # stap sk_stream_wait_memory.stp -T 1
+
+
+profiling/syscallsrw.stp - Provide a tally of read and write syscalls run on the system 
+keywords: prometheus syscall bpf
+
+  The syscall_read_write.stp script tallies the read and write
+  syscalls. This is a demo script for stapbpf prometheus-exporter
+  scripts.
 
 
 = CALLGRAPH =
@@ -478,6 +563,16 @@ keywords: nfs disk
   # stap nfsd_unlink.stp -T 1
 
 
+= DYNINST =
+
+apps/php-trace.stp - Tracing of PHP code execution
+keywords: trace dyninst
+
+  Trace of executing PHP code using the enabled markers.
+
+  # stap php-trace.stp -c 'php -f hello.php'
+
+
 = FILE =
 
 io/iotime.stp - Trace Time Spent in Read and Write for Files 
@@ -557,7 +652,7 @@ keywords: io filesystem
 = FORMAT =
 
 general/ansi_colors.stp - Color Table for ansi_set_color()
-keywords: format
+keywords: format bpf
 
   The script prints a table showing the available color combinations
   for the ansi_set_color() function in the ansi.stp tapset.
@@ -566,7 +661,7 @@ keywords: format
 
 
 general/ansi_colors2.stp - Show Attribues in Table for ansi_set_color()
-keywords: format
+keywords: format bpf
 
   The script prints a table showing the available attributes (bold,
   underline, and inverse) with color combinations for the
@@ -1402,22 +1497,13 @@ keywords: statistics memory interactive
   # echo "FILE ''" | stap sizeof_interactive.stp -T 1
 
 
-memory/cachestat.stp - Count Page Cache Hits and Misses
-keywords: memory
-
-  Monitors hits and misses to the page cache and reports a count every
-  5 seconds. Based on a bpftrace tool by David Valin.
-
-  # stap cachestat.stp -T 1
-
-
 memory/cachestat.stp - Count Page Cache Hits and Misses
 keywords: memory bpf
 
   Monitors hits and misses to the page cache and reports a count every
   5 seconds. Based on a bpftrace tool by David Valin.
 
-  # stap --bpf cachestat.stp -T 1
+  # stap cachestat.stp -T 1
 
 
 memory/glibc-malloc.stp - Overview glibc malloc internal operations
@@ -1550,7 +1636,7 @@ keywords: memory
 
 
 memory/mmfilepage.stp - Track Virtual Memory System Actions on File Backed Pages
-keywords: memory
+keywords: memory bpf
 
   The mmfilepage.stp script uses the virtual memory tracepoints
   available in some kernels to track the number of faults, copy on
@@ -1575,7 +1661,7 @@ keywords: memory
 
 
 memory/mmwriteback.stp - Track Virtual Memory System Writing to Disk
-keywords: memory
+keywords: memory bpf
 
   The mmwriteback.stp script uses the virtual memory tracepoints
   available in some kernels to report all of the file writebacks that
@@ -1879,7 +1965,7 @@ keywords: network traffic
 
 
 network/sk_stream_wait_memory.stp - Track Start and Stop of Processes Due to Network Buffer Space
-keywords: network tcp process
+keywords: network tcp process bpf
 
   The sk_stream-wait_memory.stp prints a time stamp, executable, and
   pid each time a process blocks due to the send buffer being full. A
@@ -2111,7 +2197,7 @@ keywords: network socket process
 
 
 network/sk_stream_wait_memory.stp - Track Start and Stop of Processes Due to Network Buffer Space
-keywords: network tcp process
+keywords: network tcp process bpf
 
   The sk_stream-wait_memory.stp prints a time stamp, executable, and
   pid each time a process blocks due to the send buffer being full. A
@@ -2772,6 +2858,14 @@ keywords: prometheus process syscall tracepoint
   # stap syscallsbypid.stp -T 1
 
 
+profiling/syscallsrw.stp - Provide a tally of read and write syscalls run on the system 
+keywords: prometheus syscall bpf
+
+  The syscall_read_write.stp script tallies the read and write
+  syscalls. This is a demo script for stapbpf prometheus-exporter
+  scripts.
+
+
 = QEMU =
 
 virtualization/qemu_count.stp - Tally the Number of User-Space QEMU Events
@@ -3099,7 +3193,7 @@ keywords: simple trace callgraph
 
 
 general/helloworld.stp - SystemTap "Hello World" Program
-keywords: _best simple
+keywords: _best simple bpf
 
   A basic "Hello World" program implemented in SystemTap script. It
   prints out "hello world" message and then immediately exits.
@@ -3669,6 +3763,14 @@ keywords: prometheus process syscall tracepoint
   # stap syscallsbypid.stp -T 1
 
 
+profiling/syscallsrw.stp - Provide a tally of read and write syscalls run on the system 
+keywords: prometheus syscall bpf
+
+  The syscall_read_write.stp script tallies the read and write
+  syscalls. This is a demo script for stapbpf prometheus-exporter
+  scripts.
+
+
 profiling/ucalls.stp - Profile method invocations in Java, Perl, Php, Python, Ruby, and Tcl scripts
 keywords: profiling syscall
 
@@ -3690,7 +3792,7 @@ keywords: profiling syscall
 = TCP =
 
 network/sk_stream_wait_memory.stp - Track Start and Stop of Processes Due to Network Buffer Space
-keywords: network tcp process
+keywords: network tcp process bpf
 
   The sk_stream-wait_memory.stp prints a time stamp, executable, and
   pid each time a process blocks due to the send buffer being full. A
@@ -3824,7 +3926,7 @@ keywords: trace
 
 
 apps/libguestfs_log.stp - Trace libguestfs startup
-keywords: application trace
+keywords: application trace bpf
 
   The libguestfs_log.stp script prints a log of when various
   libgueststartup steps are encountered.   The first column is
@@ -3836,7 +3938,7 @@ keywords: application trace
 
 
 apps/php-trace.stp - Tracing of PHP code execution
-keywords: trace
+keywords: trace dyninst
 
   Trace of executing PHP code using the enabled markers.
 
diff --git a/testsuite/systemtap.examples/metadatabase.db b/testsuite/systemtap.examples/metadatabase.db
index a02b06e93..d3cccaa4f 100644
Binary files a/testsuite/systemtap.examples/metadatabase.db and b/testsuite/systemtap.examples/metadatabase.db differ
diff --git a/testsuite/systemtap.examples/profiling/syscallsrw.meta b/testsuite/systemtap.examples/profiling/syscallsrw.meta
new file mode 100644
index 000000000..64fd74a99
--- /dev/null
+++ b/testsuite/systemtap.examples/profiling/syscallsrw.meta
@@ -0,0 +1,13 @@
+title: Provide a tally of read and write syscalls run on the system 
+name: syscallsrw.stp
+version: 1.0
+author: Sagar Patel
+keywords: prometheus syscall bpf
+subsystem: process
+status: experimental
+exit: user-controlled
+output: proc
+scope: system-wide
+description: The syscall_read_write.stp script tallies the read and write syscalls. This is a demo script for stapbpf prometheus-exporter scripts.
+test_check_bpf: stap --bpf -p4 syscallsrw.stp
+test_installcheck_bpf: stap --bpf syscallsrw.stp -T 1
diff --git a/testsuite/systemtap.examples/profiling/syscallsrw.stp b/testsuite/systemtap.examples/profiling/syscallsrw.stp
new file mode 100755
index 000000000..159c053ba
--- /dev/null
+++ b/testsuite/systemtap.examples/profiling/syscallsrw.stp
@@ -0,0 +1,18 @@
+#! /usr/bin/stap --bpf
+
+global arr
+
+probe kernel.function("ksys_read") !, kernel.function("vfs_read") {
+    arr[0]++;
+}
+
+probe kernel.function("ksys_write") !, kernel.function("vfs_write") {
+    arr[1]++;
+}
+
+probe prometheus {
+
+    $value .= ("count{syscall=\"ksys_read\"} " . sprint(arr[0]) . "\n")
+    $value .= ("count{syscall=\"ksys_write\"} " . sprint(arr[1]) . "\n")
+
+}